From 2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c Mon Sep 17 00:00:00 2001 From: Sang-Heon Jeon Date: Mon, 24 Aug 2026 01:18:14 +0900 Subject: coccinelle: alloc_cast: drop removed allocators - pci_alloc_consistent() and pci_zalloc_consistent() were removed by commit 7968778914e5 ("PCI: Remove the deprecated "pci-dma-compat.h" API") - kmem_alloc() was removed by commit f078d4ea8276 ("xfs: convert kmem_alloc() to kmalloc()") - kmem_zalloc() was removed by commit 10634530f7ba ("xfs: convert kmem_zalloc() to kzalloc()") - kmem_zone_alloc() and kmem_zone_zalloc() were removed by commit bae633a4a283 ("xfs: remove xfs_zone_{alloc,zalloc} helpers") So drop them from the rules. No functional change. Signed-off-by: Sang-Heon Jeon Signed-off-by: Julia Lawall --- tools/perf/util/Build | 459 ++ tools/perf/util/PERF-VERSION-GEN | 52 + tools/perf/util/addr2line.c | 436 ++ tools/perf/util/addr2line.h | 18 + tools/perf/util/addr_location.c | 41 + tools/perf/util/addr_location.h | 34 + tools/perf/util/affinity.c | 103 + tools/perf/util/affinity.h | 19 + tools/perf/util/amd-sample-raw.c | 463 ++ tools/perf/util/annotate-arch/Build | 11 + tools/perf/util/annotate-arch/annotate-arc.c | 18 + tools/perf/util/annotate-arch/annotate-arm.c | 68 + tools/perf/util/annotate-arch/annotate-arm64.c | 127 + tools/perf/util/annotate-arch/annotate-csky.c | 56 + tools/perf/util/annotate-arch/annotate-loongarch.c | 157 + tools/perf/util/annotate-arch/annotate-mips.c | 53 + tools/perf/util/annotate-arch/annotate-powerpc.c | 410 ++ tools/perf/util/annotate-arch/annotate-riscv64.c | 41 + tools/perf/util/annotate-arch/annotate-s390.c | 190 + tools/perf/util/annotate-arch/annotate-sparc.c | 177 + tools/perf/util/annotate-arch/annotate-x86.c | 852 +++ tools/perf/util/annotate-data.c | 1960 +++++++ tools/perf/util/annotate-data.h | 322 ++ tools/perf/util/annotate.c | 3228 +++++++++++ tools/perf/util/annotate.h | 589 ++ tools/perf/util/arm-spe-decoder/Build | 3 + tools/perf/util/arm-spe-decoder/arm-spe-decoder.c | 304 + tools/perf/util/arm-spe-decoder/arm-spe-decoder.h | 158 + .../util/arm-spe-decoder/arm-spe-pkt-decoder.c | 614 +++ .../util/arm-spe-decoder/arm-spe-pkt-decoder.h | 191 + tools/perf/util/arm-spe.c | 2068 +++++++ tools/perf/util/arm-spe.h | 71 + .../perf/util/arm64-frame-pointer-unwind-support.c | 77 + .../perf/util/arm64-frame-pointer-unwind-support.h | 14 + tools/perf/util/aslr.c | 1410 +++++ tools/perf/util/aslr.h | 44 + tools/perf/util/auxtrace.c | 2999 ++++++++++ tools/perf/util/auxtrace.h | 702 +++ tools/perf/util/blake2s.c | 151 + tools/perf/util/blake2s.h | 73 + tools/perf/util/block-info.c | 532 ++ tools/perf/util/block-info.h | 76 + tools/perf/util/block-range.c | 332 ++ tools/perf/util/block-range.h | 76 + tools/perf/util/bpf-event.c | 1017 ++++ tools/perf/util/bpf-event.h | 73 + tools/perf/util/bpf-filter.c | 803 +++ tools/perf/util/bpf-filter.h | 70 + tools/perf/util/bpf-filter.l | 179 + tools/perf/util/bpf-filter.y | 109 + tools/perf/util/bpf-trace-summary.c | 465 ++ tools/perf/util/bpf-utils.c | 293 + tools/perf/util/bpf-utils.h | 86 + tools/perf/util/bpf_counter.c | 902 +++ tools/perf/util/bpf_counter.h | 75 + tools/perf/util/bpf_counter_cgroup.c | 351 ++ tools/perf/util/bpf_ftrace.c | 215 + tools/perf/util/bpf_kwork.c | 359 ++ tools/perf/util/bpf_kwork_top.c | 310 ++ tools/perf/util/bpf_lock_contention.c | 862 +++ tools/perf/util/bpf_off_cpu.c | 430 ++ tools/perf/util/bpf_skel/.gitignore | 4 + .../util/bpf_skel/augmented_raw_syscalls.bpf.c | 621 +++ tools/perf/util/bpf_skel/bperf_cgroup.bpf.c | 225 + tools/perf/util/bpf_skel/bperf_cgroup.h | 15 + tools/perf/util/bpf_skel/bperf_follower.bpf.c | 162 + tools/perf/util/bpf_skel/bperf_leader.bpf.c | 55 + tools/perf/util/bpf_skel/bperf_u.h | 19 + tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c | 92 + tools/perf/util/bpf_skel/func_latency.bpf.c | 199 + tools/perf/util/bpf_skel/kwork_top.bpf.c | 338 ++ tools/perf/util/bpf_skel/kwork_trace.bpf.c | 384 ++ tools/perf/util/bpf_skel/lock_contention.bpf.c | 1052 ++++ tools/perf/util/bpf_skel/lock_data.h | 79 + tools/perf/util/bpf_skel/off_cpu.bpf.c | 372 ++ tools/perf/util/bpf_skel/perf_version.h | 17 + tools/perf/util/bpf_skel/sample-filter.h | 72 + tools/perf/util/bpf_skel/sample_filter.bpf.c | 298 + tools/perf/util/bpf_skel/syscall_summary.bpf.c | 165 + tools/perf/util/bpf_skel/syscall_summary.h | 27 + tools/perf/util/bpf_skel/vmlinux/.gitignore | 1 + tools/perf/util/bpf_skel/vmlinux/vmlinux.h | 215 + tools/perf/util/bpf_trace_augment.c | 143 + tools/perf/util/branch.c | 229 + tools/perf/util/branch.h | 99 + tools/perf/util/btf.c | 27 + tools/perf/util/btf.h | 10 + tools/perf/util/build-id.c | 983 ++++ tools/perf/util/build-id.h | 79 + tools/perf/util/c2c-function.c | 1642 ++++++ tools/perf/util/c2c.c | 21 + tools/perf/util/c2c.h | 135 + tools/perf/util/cache.h | 31 + tools/perf/util/cacheline.c | 25 + tools/perf/util/cacheline.h | 36 + tools/perf/util/call-path.c | 114 + tools/perf/util/call-path.h | 68 + tools/perf/util/callchain.c | 1943 +++++++ tools/perf/util/callchain.h | 315 ++ tools/perf/util/cap.c | 46 + tools/perf/util/cap.h | 23 + tools/perf/util/capstone.c | 653 +++ tools/perf/util/capstone.h | 57 + tools/perf/util/cgroup.c | 646 +++ tools/perf/util/cgroup.h | 58 + tools/perf/util/clockid.c | 119 + tools/perf/util/clockid.h | 12 + tools/perf/util/cloexec.c | 92 + tools/perf/util/cloexec.h | 7 + tools/perf/util/color.c | 156 + tools/perf/util/color.h | 45 + tools/perf/util/color_config.c | 37 + tools/perf/util/comm.c | 237 + tools/perf/util/comm.h | 28 + tools/perf/util/compress.h | 93 + tools/perf/util/config.c | 995 ++++ tools/perf/util/config.h | 80 + tools/perf/util/copyfile.c | 146 + tools/perf/util/copyfile.h | 16 + tools/perf/util/counts.c | 71 + tools/perf/util/counts.h | 45 + tools/perf/util/cpumap.c | 824 +++ tools/perf/util/cpumap.h | 162 + tools/perf/util/cputopo.c | 506 ++ tools/perf/util/cputopo.h | 78 + tools/perf/util/cs-etm-base.c | 195 + tools/perf/util/cs-etm-decoder/Build | 1 + tools/perf/util/cs-etm-decoder/cs-etm-decoder.c | 809 +++ tools/perf/util/cs-etm-decoder/cs-etm-decoder.h | 111 + tools/perf/util/cs-etm.c | 3771 +++++++++++++ tools/perf/util/cs-etm.h | 286 + tools/perf/util/data-convert-bt.c | 1832 +++++++ tools/perf/util/data-convert-json.c | 462 ++ tools/perf/util/data-convert.h | 22 + tools/perf/util/data.c | 597 ++ tools/perf/util/data.h | 144 + tools/perf/util/db-export.c | 615 +++ tools/perf/util/db-export.h | 109 + tools/perf/util/debug.c | 380 ++ tools/perf/util/debug.h | 92 + tools/perf/util/debuginfo.c | 215 + tools/perf/util/debuginfo.h | 66 + tools/perf/util/demangle-cxx.cpp | 49 + tools/perf/util/demangle-cxx.h | 18 + tools/perf/util/demangle-java.c | 203 + tools/perf/util/demangle-java.h | 11 + tools/perf/util/demangle-ocaml.c | 68 + tools/perf/util/demangle-ocaml.h | 7 + tools/perf/util/demangle-rust-v0.c | 2042 +++++++ tools/perf/util/demangle-rust-v0.h | 88 + tools/perf/util/disasm.c | 1648 ++++++ tools/perf/util/disasm.h | 170 + tools/perf/util/dlfilter.c | 662 +++ tools/perf/util/dlfilter.h | 99 + tools/perf/util/drm_pmu.c | 685 +++ tools/perf/util/drm_pmu.h | 39 + tools/perf/util/dso.c | 2077 +++++++ tools/perf/util/dso.h | 1009 ++++ tools/perf/util/dsos.c | 495 ++ tools/perf/util/dsos.h | 52 + tools/perf/util/dump-insn.h | 26 + tools/perf/util/dwarf-aux.c | 2182 ++++++++ tools/perf/util/dwarf-aux.h | 192 + tools/perf/util/dwarf-regs-arch/Build | 9 + tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c | 126 + .../util/dwarf-regs-arch/dwarf-regs-loongarch.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c | 14 + .../perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c | 136 + tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c | 53 + tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c | 260 + tools/perf/util/dwarf-regs.c | 237 + tools/perf/util/env.c | 1087 ++++ tools/perf/util/env.h | 245 + tools/perf/util/event.c | 944 ++++ tools/perf/util/event.h | 437 ++ tools/perf/util/events_stats.h | 72 + tools/perf/util/evlist.c | 2798 ++++++++++ tools/perf/util/evlist.h | 685 +++ tools/perf/util/evsel.c | 4758 ++++++++++++++++ tools/perf/util/evsel.h | 632 +++ tools/perf/util/evsel_config.h | 69 + tools/perf/util/evsel_fprintf.c | 258 + tools/perf/util/evsel_fprintf.h | 50 + tools/perf/util/evswitch.c | 62 + tools/perf/util/evswitch.h | 31 + tools/perf/util/expr.c | 489 ++ tools/perf/util/expr.h | 64 + tools/perf/util/expr.l | 150 + tools/perf/util/expr.y | 391 ++ tools/perf/util/find-map.c | 36 + tools/perf/util/fncache.c | 68 + tools/perf/util/fncache.h | 6 + tools/perf/util/ftrace.h | 96 + tools/perf/util/genelf.c | 471 ++ tools/perf/util/genelf.h | 89 + tools/perf/util/genelf_debug.c | 625 +++ tools/perf/util/hashmap.c | 240 + tools/perf/util/hashmap.h | 208 + tools/perf/util/header.c | 5801 ++++++++++++++++++++ tools/perf/util/header.h | 225 + tools/perf/util/help-unknown-cmd.c | 122 + tools/perf/util/help-unknown-cmd.h | 0 tools/perf/util/hisi-ptt-decoder/Build | 1 + .../util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c | 165 + .../util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h | 43 + tools/perf/util/hisi-ptt.c | 202 + tools/perf/util/hisi-ptt.h | 19 + tools/perf/util/hist.c | 3100 +++++++++++ tools/perf/util/hist.h | 841 +++ tools/perf/util/hwmon_pmu.c | 839 +++ tools/perf/util/hwmon_pmu.h | 167 + tools/perf/util/include/asm/asm-offsets.h | 2 + tools/perf/util/include/asm/cpufeature.h | 10 + tools/perf/util/include/asm/dwarf2.h | 14 + tools/perf/util/include/asm/swab.h | 1 + tools/perf/util/include/asm/system.h | 1 + tools/perf/util/include/asm/uaccess.h | 15 + tools/perf/util/include/dwarf-regs.h | 149 + tools/perf/util/include/linux/linkage.h | 139 + tools/perf/util/intel-bts.c | 911 +++ tools/perf/util/intel-bts.h | 34 + tools/perf/util/intel-pt-decoder/Build | 26 + .../perf/util/intel-pt-decoder/intel-pt-decoder.c | 4515 +++++++++++++++ .../perf/util/intel-pt-decoder/intel-pt-decoder.h | 318 ++ .../util/intel-pt-decoder/intel-pt-insn-decoder.c | 313 ++ .../util/intel-pt-decoder/intel-pt-insn-decoder.h | 58 + tools/perf/util/intel-pt-decoder/intel-pt-log.c | 267 + tools/perf/util/intel-pt-decoder/intel-pt-log.h | 76 + .../util/intel-pt-decoder/intel-pt-pkt-decoder.c | 791 +++ .../util/intel-pt-decoder/intel-pt-pkt-decoder.h | 92 + tools/perf/util/intel-pt.c | 4748 ++++++++++++++++ tools/perf/util/intel-pt.h | 48 + tools/perf/util/intel-tpebs.c | 721 +++ tools/perf/util/intel-tpebs.h | 25 + tools/perf/util/intlist.c | 150 + tools/perf/util/intlist.h | 78 + tools/perf/util/iostat.c | 54 + tools/perf/util/iostat.h | 47 + tools/perf/util/jit.h | 13 + tools/perf/util/jitdump.c | 917 ++++ tools/perf/util/jitdump.h | 139 + tools/perf/util/kvm-stat-arch/Build | 6 + .../util/kvm-stat-arch/arm64_exception_types.h | 97 + tools/perf/util/kvm-stat-arch/book3s_hcalls.h | 124 + tools/perf/util/kvm-stat-arch/book3s_hv_exits.h | 35 + tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c | 92 + tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c | 149 + tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c | 204 + tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c | 87 + tools/perf/util/kvm-stat-arch/kvm-stat-s390.c | 116 + tools/perf/util/kvm-stat-arch/kvm-stat-x86.c | 235 + tools/perf/util/kvm-stat-arch/riscv_trap_types.h | 57 + tools/perf/util/kvm-stat.c | 311 ++ tools/perf/util/kvm-stat.h | 232 + tools/perf/util/kwork.h | 326 ++ tools/perf/util/levenshtein.c | 87 + tools/perf/util/levenshtein.h | 9 + tools/perf/util/libbfd.c | 660 +++ tools/perf/util/libbfd.h | 82 + tools/perf/util/libdw.c | 242 + tools/perf/util/libdw.h | 58 + tools/perf/util/libunwind-arch/Build | 11 + tools/perf/util/libunwind-arch/libunwind-arch.c | 319 ++ tools/perf/util/libunwind-arch/libunwind-arch.h | 296 + tools/perf/util/libunwind-arch/libunwind-arm.c | 290 + tools/perf/util/libunwind-arch/libunwind-arm64.c | 289 + tools/perf/util/libunwind-arch/libunwind-i386.c | 312 ++ .../perf/util/libunwind-arch/libunwind-loongarch.c | 297 + tools/perf/util/libunwind-arch/libunwind-mips.c | 299 + tools/perf/util/libunwind-arch/libunwind-ppc32.c | 303 + tools/perf/util/libunwind-arch/libunwind-ppc64.c | 303 + tools/perf/util/libunwind-arch/libunwind-riscv.c | 297 + tools/perf/util/libunwind-arch/libunwind-s390.c | 299 + tools/perf/util/libunwind-arch/libunwind-x86_64.c | 320 ++ tools/perf/util/llvm-c-helpers.cpp | 196 + tools/perf/util/llvm-c-helpers.h | 60 + tools/perf/util/llvm.c | 279 + tools/perf/util/llvm.h | 21 + tools/perf/util/lock-contention.c | 143 + tools/perf/util/lock-contention.h | 208 + tools/perf/util/lzma.c | 129 + tools/perf/util/machine.c | 3386 ++++++++++++ tools/perf/util/machine.h | 331 ++ tools/perf/util/map.c | 650 +++ tools/perf/util/map.h | 318 ++ tools/perf/util/map_symbol.c | 34 + tools/perf/util/map_symbol.h | 33 + tools/perf/util/maps.c | 1468 +++++ tools/perf/util/maps.h | 88 + tools/perf/util/mem-events.c | 982 ++++ tools/perf/util/mem-events.h | 149 + tools/perf/util/mem-info.c | 48 + tools/perf/util/mem-info.h | 55 + tools/perf/util/mem2node.c | 139 + tools/perf/util/mem2node.h | 20 + tools/perf/util/memswap.c | 25 + tools/perf/util/memswap.h | 15 + tools/perf/util/metricgroup.c | 1753 ++++++ tools/perf/util/metricgroup.h | 98 + tools/perf/util/mmap.c | 369 ++ tools/perf/util/mmap.h | 64 + tools/perf/util/mutex.c | 125 + tools/perf/util/mutex.h | 129 + tools/perf/util/namespaces.c | 380 ++ tools/perf/util/namespaces.h | 84 + tools/perf/util/off_cpu.h | 39 + tools/perf/util/ordered-events.c | 417 ++ tools/perf/util/ordered-events.h | 84 + tools/perf/util/parse-branch-options.c | 115 + tools/perf/util/parse-branch-options.h | 7 + tools/perf/util/parse-events.c | 2959 ++++++++++ tools/perf/util/parse-events.h | 295 + tools/perf/util/parse-events.l | 372 ++ tools/perf/util/parse-events.y | 647 +++ tools/perf/util/parse-regs-options.c | 124 + tools/perf/util/parse-regs-options.h | 7 + tools/perf/util/parse-sublevel-options.c | 70 + tools/perf/util/parse-sublevel-options.h | 11 + tools/perf/util/path.c | 79 + tools/perf/util/path.h | 17 + tools/perf/util/perf-hooks-list.h | 3 + tools/perf/util/perf-hooks.c | 90 + tools/perf/util/perf-hooks.h | 40 + tools/perf/util/perf-regs-arch/Build | 9 + tools/perf/util/perf-regs-arch/perf_regs_aarch64.c | 229 + tools/perf/util/perf-regs-arch/perf_regs_arm.c | 61 + tools/perf/util/perf-regs-arch/perf_regs_csky.c | 108 + .../perf/util/perf-regs-arch/perf_regs_loongarch.c | 92 + tools/perf/util/perf-regs-arch/perf_regs_mips.c | 88 + tools/perf/util/perf-regs-arch/perf_regs_powerpc.c | 322 ++ tools/perf/util/perf-regs-arch/perf_regs_riscv.c | 221 + tools/perf/util/perf-regs-arch/perf_regs_s390.c | 175 + tools/perf/util/perf-regs-arch/perf_regs_x86.c | 373 ++ tools/perf/util/perf_api_probe.c | 212 + tools/perf/util/perf_api_probe.h | 17 + tools/perf/util/perf_event_attr_fprintf.c | 410 ++ tools/perf/util/perf_regs.c | 254 + tools/perf/util/perf_regs.h | 79 + tools/perf/util/pfm.c | 321 ++ tools/perf/util/pfm.h | 38 + tools/perf/util/pmu.c | 2790 ++++++++++ tools/perf/util/pmu.h | 375 ++ tools/perf/util/pmu.l | 48 + tools/perf/util/pmu.y | 101 + tools/perf/util/pmus.c | 905 +++ tools/perf/util/pmus.h | 43 + tools/perf/util/powerpc-vpadtl.c | 736 +++ tools/perf/util/powerpc-vpadtl.h | 23 + tools/perf/util/print-events.c | 357 ++ tools/perf/util/print-events.h | 39 + tools/perf/util/print_binary.c | 58 + tools/perf/util/print_binary.h | 37 + tools/perf/util/print_insn.c | 72 + tools/perf/util/print_insn.h | 25 + tools/perf/util/probe-event.c | 3844 +++++++++++++ tools/perf/util/probe-event.h | 199 + tools/perf/util/probe-file.c | 1184 ++++ tools/perf/util/probe-file.h | 82 + tools/perf/util/probe-finder.c | 1991 +++++++ tools/perf/util/probe-finder.h | 114 + tools/perf/util/pstack.c | 71 + tools/perf/util/pstack.h | 15 + tools/perf/util/python.c | 4343 +++++++++++++++ tools/perf/util/rblist.c | 140 + tools/perf/util/rblist.h | 50 + tools/perf/util/record.c | 292 + tools/perf/util/record.h | 98 + tools/perf/util/rlimit.c | 57 + tools/perf/util/rlimit.h | 15 + tools/perf/util/rwsem.c | 71 + tools/perf/util/rwsem.h | 30 + tools/perf/util/s390-cpumcf-kernel.h | 65 + tools/perf/util/s390-cpumsf-kernel.h | 71 + tools/perf/util/s390-cpumsf.c | 1174 ++++ tools/perf/util/s390-cpumsf.h | 21 + tools/perf/util/s390-sample-raw.c | 382 ++ tools/perf/util/sample-raw.c | 28 + tools/perf/util/sample-raw.h | 19 + tools/perf/util/sample.c | 134 + tools/perf/util/sample.h | 268 + tools/perf/util/scripting-engines/Build | 9 + .../perf/util/scripting-engines/trace-event-perl.c | 770 +++ .../util/scripting-engines/trace-event-python.c | 2224 ++++++++ tools/perf/util/session.c | 4147 ++++++++++++++ tools/perf/util/session.h | 220 + tools/perf/util/setns.c | 10 + tools/perf/util/setup.py | 194 + tools/perf/util/sharded_mutex.c | 33 + tools/perf/util/sharded_mutex.h | 29 + tools/perf/util/sideband_evlist.c | 163 + tools/perf/util/smt.c | 36 + tools/perf/util/smt.h | 17 + tools/perf/util/sort.c | 4519 +++++++++++++++ tools/perf/util/sort.h | 163 + tools/perf/util/spark.c | 32 + tools/perf/util/spark.h | 9 + tools/perf/util/srccode.c | 171 + tools/perf/util/srccode.h | 20 + tools/perf/util/srcline.c | 508 ++ tools/perf/util/srcline.h | 69 + tools/perf/util/stat-display.c | 1604 ++++++ tools/perf/util/stat-shadow.c | 359 ++ tools/perf/util/stat.c | 725 +++ tools/perf/util/stat.h | 210 + tools/perf/util/strbuf.c | 170 + tools/perf/util/strbuf.h | 95 + tools/perf/util/stream.c | 341 ++ tools/perf/util/stream.h | 41 + tools/perf/util/strfilter.c | 312 ++ tools/perf/util/strfilter.h | 84 + tools/perf/util/string.c | 460 ++ tools/perf/util/string2.h | 46 + tools/perf/util/strlist.c | 203 + tools/perf/util/strlist.h | 88 + tools/perf/util/svghelper.c | 812 +++ tools/perf/util/svghelper.h | 39 + tools/perf/util/symbol-elf.c | 2999 ++++++++++ tools/perf/util/symbol-minimal.c | 379 ++ tools/perf/util/symbol.c | 2765 ++++++++++ tools/perf/util/symbol.h | 355 ++ tools/perf/util/symbol_conf.h | 117 + tools/perf/util/symbol_fprintf.c | 73 + tools/perf/util/symsrc.h | 47 + tools/perf/util/synthetic-events.c | 2893 ++++++++++ tools/perf/util/synthetic-events.h | 134 + tools/perf/util/target.c | 142 + tools/perf/util/target.h | 98 + tools/perf/util/term.c | 40 + tools/perf/util/term.h | 11 + tools/perf/util/thread-stack.c | 1239 +++++ tools/perf/util/thread-stack.h | 110 + tools/perf/util/thread.c | 638 +++ tools/perf/util/thread.h | 398 ++ tools/perf/util/thread_map.c | 397 ++ tools/perf/util/thread_map.h | 27 + tools/perf/util/threads.c | 190 + tools/perf/util/threads.h | 35 + tools/perf/util/time-utils.c | 559 ++ tools/perf/util/time-utils.h | 51 + tools/perf/util/tool.c | 572 ++ tools/perf/util/tool.h | 116 + tools/perf/util/tool_pmu.c | 723 +++ tools/perf/util/tool_pmu.h | 67 + tools/perf/util/top.c | 119 + tools/perf/util/top.h | 66 + tools/perf/util/topdown.c | 8 + tools/perf/util/topdown.h | 11 + tools/perf/util/tp_pmu.c | 208 + tools/perf/util/tp_pmu.h | 19 + tools/perf/util/trace-event-info.c | 703 +++ tools/perf/util/trace-event-parse.c | 323 ++ tools/perf/util/trace-event-read.c | 500 ++ tools/perf/util/trace-event-scripting.c | 407 ++ tools/perf/util/trace-event.c | 127 + tools/perf/util/trace-event.h | 162 + tools/perf/util/trace.h | 38 + tools/perf/util/trace_augment.h | 66 + tools/perf/util/tracepoint.c | 52 + tools/perf/util/tracepoint.h | 26 + tools/perf/util/trigger.h | 95 + tools/perf/util/tsc.c | 142 + tools/perf/util/tsc.h | 32 + tools/perf/util/units.c | 76 + tools/perf/util/units.h | 19 + tools/perf/util/unwind-libdw.c | 465 ++ tools/perf/util/unwind-libdw.h | 32 + tools/perf/util/unwind-libunwind.c | 683 +++ tools/perf/util/unwind.c | 104 + tools/perf/util/unwind.h | 85 + tools/perf/util/usage.c | 34 + tools/perf/util/util.c | 559 ++ tools/perf/util/util.h | 126 + tools/perf/util/values.c | 289 + tools/perf/util/values.h | 30 + tools/perf/util/vdso.c | 369 ++ tools/perf/util/vdso.h | 30 + tools/perf/util/zlib.c | 97 + tools/perf/util/zstd.c | 152 + 481 files changed, 193328 insertions(+) create mode 100644 tools/perf/util/Build create mode 100755 tools/perf/util/PERF-VERSION-GEN create mode 100644 tools/perf/util/addr2line.c create mode 100644 tools/perf/util/addr2line.h create mode 100644 tools/perf/util/addr_location.c create mode 100644 tools/perf/util/addr_location.h create mode 100644 tools/perf/util/affinity.c create mode 100644 tools/perf/util/affinity.h create mode 100644 tools/perf/util/amd-sample-raw.c create mode 100644 tools/perf/util/annotate-arch/Build create mode 100644 tools/perf/util/annotate-arch/annotate-arc.c create mode 100644 tools/perf/util/annotate-arch/annotate-arm.c create mode 100644 tools/perf/util/annotate-arch/annotate-arm64.c create mode 100644 tools/perf/util/annotate-arch/annotate-csky.c create mode 100644 tools/perf/util/annotate-arch/annotate-loongarch.c create mode 100644 tools/perf/util/annotate-arch/annotate-mips.c create mode 100644 tools/perf/util/annotate-arch/annotate-powerpc.c create mode 100644 tools/perf/util/annotate-arch/annotate-riscv64.c create mode 100644 tools/perf/util/annotate-arch/annotate-s390.c create mode 100644 tools/perf/util/annotate-arch/annotate-sparc.c create mode 100644 tools/perf/util/annotate-arch/annotate-x86.c create mode 100644 tools/perf/util/annotate-data.c create mode 100644 tools/perf/util/annotate-data.h create mode 100644 tools/perf/util/annotate.c create mode 100644 tools/perf/util/annotate.h create mode 100644 tools/perf/util/arm-spe-decoder/Build create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-decoder.c create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-decoder.h create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h create mode 100644 tools/perf/util/arm-spe.c create mode 100644 tools/perf/util/arm-spe.h create mode 100644 tools/perf/util/arm64-frame-pointer-unwind-support.c create mode 100644 tools/perf/util/arm64-frame-pointer-unwind-support.h create mode 100644 tools/perf/util/aslr.c create mode 100644 tools/perf/util/aslr.h create mode 100644 tools/perf/util/auxtrace.c create mode 100644 tools/perf/util/auxtrace.h create mode 100644 tools/perf/util/blake2s.c create mode 100644 tools/perf/util/blake2s.h create mode 100644 tools/perf/util/block-info.c create mode 100644 tools/perf/util/block-info.h create mode 100644 tools/perf/util/block-range.c create mode 100644 tools/perf/util/block-range.h create mode 100644 tools/perf/util/bpf-event.c create mode 100644 tools/perf/util/bpf-event.h create mode 100644 tools/perf/util/bpf-filter.c create mode 100644 tools/perf/util/bpf-filter.h create mode 100644 tools/perf/util/bpf-filter.l create mode 100644 tools/perf/util/bpf-filter.y create mode 100644 tools/perf/util/bpf-trace-summary.c create mode 100644 tools/perf/util/bpf-utils.c create mode 100644 tools/perf/util/bpf-utils.h create mode 100644 tools/perf/util/bpf_counter.c create mode 100644 tools/perf/util/bpf_counter.h create mode 100644 tools/perf/util/bpf_counter_cgroup.c create mode 100644 tools/perf/util/bpf_ftrace.c create mode 100644 tools/perf/util/bpf_kwork.c create mode 100644 tools/perf/util/bpf_kwork_top.c create mode 100644 tools/perf/util/bpf_lock_contention.c create mode 100644 tools/perf/util/bpf_off_cpu.c create mode 100644 tools/perf/util/bpf_skel/.gitignore create mode 100644 tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_cgroup.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_cgroup.h create mode 100644 tools/perf/util/bpf_skel/bperf_follower.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_leader.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_u.h create mode 100644 tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c create mode 100644 tools/perf/util/bpf_skel/func_latency.bpf.c create mode 100644 tools/perf/util/bpf_skel/kwork_top.bpf.c create mode 100644 tools/perf/util/bpf_skel/kwork_trace.bpf.c create mode 100644 tools/perf/util/bpf_skel/lock_contention.bpf.c create mode 100644 tools/perf/util/bpf_skel/lock_data.h create mode 100644 tools/perf/util/bpf_skel/off_cpu.bpf.c create mode 100644 tools/perf/util/bpf_skel/perf_version.h create mode 100644 tools/perf/util/bpf_skel/sample-filter.h create mode 100644 tools/perf/util/bpf_skel/sample_filter.bpf.c create mode 100644 tools/perf/util/bpf_skel/syscall_summary.bpf.c create mode 100644 tools/perf/util/bpf_skel/syscall_summary.h create mode 100644 tools/perf/util/bpf_skel/vmlinux/.gitignore create mode 100644 tools/perf/util/bpf_skel/vmlinux/vmlinux.h create mode 100644 tools/perf/util/bpf_trace_augment.c create mode 100644 tools/perf/util/branch.c create mode 100644 tools/perf/util/branch.h create mode 100644 tools/perf/util/btf.c create mode 100644 tools/perf/util/btf.h create mode 100644 tools/perf/util/build-id.c create mode 100644 tools/perf/util/build-id.h create mode 100644 tools/perf/util/c2c-function.c create mode 100644 tools/perf/util/c2c.c create mode 100644 tools/perf/util/c2c.h create mode 100644 tools/perf/util/cache.h create mode 100644 tools/perf/util/cacheline.c create mode 100644 tools/perf/util/cacheline.h create mode 100644 tools/perf/util/call-path.c create mode 100644 tools/perf/util/call-path.h create mode 100644 tools/perf/util/callchain.c create mode 100644 tools/perf/util/callchain.h create mode 100644 tools/perf/util/cap.c create mode 100644 tools/perf/util/cap.h create mode 100644 tools/perf/util/capstone.c create mode 100644 tools/perf/util/capstone.h create mode 100644 tools/perf/util/cgroup.c create mode 100644 tools/perf/util/cgroup.h create mode 100644 tools/perf/util/clockid.c create mode 100644 tools/perf/util/clockid.h create mode 100644 tools/perf/util/cloexec.c create mode 100644 tools/perf/util/cloexec.h create mode 100644 tools/perf/util/color.c create mode 100644 tools/perf/util/color.h create mode 100644 tools/perf/util/color_config.c create mode 100644 tools/perf/util/comm.c create mode 100644 tools/perf/util/comm.h create mode 100644 tools/perf/util/compress.h create mode 100644 tools/perf/util/config.c create mode 100644 tools/perf/util/config.h create mode 100644 tools/perf/util/copyfile.c create mode 100644 tools/perf/util/copyfile.h create mode 100644 tools/perf/util/counts.c create mode 100644 tools/perf/util/counts.h create mode 100644 tools/perf/util/cpumap.c create mode 100644 tools/perf/util/cpumap.h create mode 100644 tools/perf/util/cputopo.c create mode 100644 tools/perf/util/cputopo.h create mode 100644 tools/perf/util/cs-etm-base.c create mode 100644 tools/perf/util/cs-etm-decoder/Build create mode 100644 tools/perf/util/cs-etm-decoder/cs-etm-decoder.c create mode 100644 tools/perf/util/cs-etm-decoder/cs-etm-decoder.h create mode 100644 tools/perf/util/cs-etm.c create mode 100644 tools/perf/util/cs-etm.h create mode 100644 tools/perf/util/data-convert-bt.c create mode 100644 tools/perf/util/data-convert-json.c create mode 100644 tools/perf/util/data-convert.h create mode 100644 tools/perf/util/data.c create mode 100644 tools/perf/util/data.h create mode 100644 tools/perf/util/db-export.c create mode 100644 tools/perf/util/db-export.h create mode 100644 tools/perf/util/debug.c create mode 100644 tools/perf/util/debug.h create mode 100644 tools/perf/util/debuginfo.c create mode 100644 tools/perf/util/debuginfo.h create mode 100644 tools/perf/util/demangle-cxx.cpp create mode 100644 tools/perf/util/demangle-cxx.h create mode 100644 tools/perf/util/demangle-java.c create mode 100644 tools/perf/util/demangle-java.h create mode 100644 tools/perf/util/demangle-ocaml.c create mode 100644 tools/perf/util/demangle-ocaml.h create mode 100644 tools/perf/util/demangle-rust-v0.c create mode 100644 tools/perf/util/demangle-rust-v0.h create mode 100644 tools/perf/util/disasm.c create mode 100644 tools/perf/util/disasm.h create mode 100644 tools/perf/util/dlfilter.c create mode 100644 tools/perf/util/dlfilter.h create mode 100644 tools/perf/util/drm_pmu.c create mode 100644 tools/perf/util/drm_pmu.h create mode 100644 tools/perf/util/dso.c create mode 100644 tools/perf/util/dso.h create mode 100644 tools/perf/util/dsos.c create mode 100644 tools/perf/util/dsos.h create mode 100644 tools/perf/util/dump-insn.h create mode 100644 tools/perf/util/dwarf-aux.c create mode 100644 tools/perf/util/dwarf-aux.h create mode 100644 tools/perf/util/dwarf-regs-arch/Build create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c create mode 100644 tools/perf/util/dwarf-regs.c create mode 100644 tools/perf/util/env.c create mode 100644 tools/perf/util/env.h create mode 100644 tools/perf/util/event.c create mode 100644 tools/perf/util/event.h create mode 100644 tools/perf/util/events_stats.h create mode 100644 tools/perf/util/evlist.c create mode 100644 tools/perf/util/evlist.h create mode 100644 tools/perf/util/evsel.c create mode 100644 tools/perf/util/evsel.h create mode 100644 tools/perf/util/evsel_config.h create mode 100644 tools/perf/util/evsel_fprintf.c create mode 100644 tools/perf/util/evsel_fprintf.h create mode 100644 tools/perf/util/evswitch.c create mode 100644 tools/perf/util/evswitch.h create mode 100644 tools/perf/util/expr.c create mode 100644 tools/perf/util/expr.h create mode 100644 tools/perf/util/expr.l create mode 100644 tools/perf/util/expr.y create mode 100644 tools/perf/util/find-map.c create mode 100644 tools/perf/util/fncache.c create mode 100644 tools/perf/util/fncache.h create mode 100644 tools/perf/util/ftrace.h create mode 100644 tools/perf/util/genelf.c create mode 100644 tools/perf/util/genelf.h create mode 100644 tools/perf/util/genelf_debug.c create mode 100644 tools/perf/util/hashmap.c create mode 100644 tools/perf/util/hashmap.h create mode 100644 tools/perf/util/header.c create mode 100644 tools/perf/util/header.h create mode 100644 tools/perf/util/help-unknown-cmd.c create mode 100644 tools/perf/util/help-unknown-cmd.h create mode 100644 tools/perf/util/hisi-ptt-decoder/Build create mode 100644 tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c create mode 100644 tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h create mode 100644 tools/perf/util/hisi-ptt.c create mode 100644 tools/perf/util/hisi-ptt.h create mode 100644 tools/perf/util/hist.c create mode 100644 tools/perf/util/hist.h create mode 100644 tools/perf/util/hwmon_pmu.c create mode 100644 tools/perf/util/hwmon_pmu.h create mode 100644 tools/perf/util/include/asm/asm-offsets.h create mode 100644 tools/perf/util/include/asm/cpufeature.h create mode 100644 tools/perf/util/include/asm/dwarf2.h create mode 100644 tools/perf/util/include/asm/swab.h create mode 100644 tools/perf/util/include/asm/system.h create mode 100644 tools/perf/util/include/asm/uaccess.h create mode 100644 tools/perf/util/include/dwarf-regs.h create mode 100644 tools/perf/util/include/linux/linkage.h create mode 100644 tools/perf/util/intel-bts.c create mode 100644 tools/perf/util/intel-bts.h create mode 100644 tools/perf/util/intel-pt-decoder/Build create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-decoder.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-decoder.h create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.h create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-log.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-log.h create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.h create mode 100644 tools/perf/util/intel-pt.c create mode 100644 tools/perf/util/intel-pt.h create mode 100644 tools/perf/util/intel-tpebs.c create mode 100644 tools/perf/util/intel-tpebs.h create mode 100644 tools/perf/util/intlist.c create mode 100644 tools/perf/util/intlist.h create mode 100644 tools/perf/util/iostat.c create mode 100644 tools/perf/util/iostat.h create mode 100644 tools/perf/util/jit.h create mode 100644 tools/perf/util/jitdump.c create mode 100644 tools/perf/util/jitdump.h create mode 100644 tools/perf/util/kvm-stat-arch/Build create mode 100644 tools/perf/util/kvm-stat-arch/arm64_exception_types.h create mode 100644 tools/perf/util/kvm-stat-arch/book3s_hcalls.h create mode 100644 tools/perf/util/kvm-stat-arch/book3s_hv_exits.h create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-s390.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-x86.c create mode 100644 tools/perf/util/kvm-stat-arch/riscv_trap_types.h create mode 100644 tools/perf/util/kvm-stat.c create mode 100644 tools/perf/util/kvm-stat.h create mode 100644 tools/perf/util/kwork.h create mode 100644 tools/perf/util/levenshtein.c create mode 100644 tools/perf/util/levenshtein.h create mode 100644 tools/perf/util/libbfd.c create mode 100644 tools/perf/util/libbfd.h create mode 100644 tools/perf/util/libdw.c create mode 100644 tools/perf/util/libdw.h create mode 100644 tools/perf/util/libunwind-arch/Build create mode 100644 tools/perf/util/libunwind-arch/libunwind-arch.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-arch.h create mode 100644 tools/perf/util/libunwind-arch/libunwind-arm.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-arm64.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-i386.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-loongarch.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-mips.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-ppc32.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-ppc64.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-riscv.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-s390.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-x86_64.c create mode 100644 tools/perf/util/llvm-c-helpers.cpp create mode 100644 tools/perf/util/llvm-c-helpers.h create mode 100644 tools/perf/util/llvm.c create mode 100644 tools/perf/util/llvm.h create mode 100644 tools/perf/util/lock-contention.c create mode 100644 tools/perf/util/lock-contention.h create mode 100644 tools/perf/util/lzma.c create mode 100644 tools/perf/util/machine.c create mode 100644 tools/perf/util/machine.h create mode 100644 tools/perf/util/map.c create mode 100644 tools/perf/util/map.h create mode 100644 tools/perf/util/map_symbol.c create mode 100644 tools/perf/util/map_symbol.h create mode 100644 tools/perf/util/maps.c create mode 100644 tools/perf/util/maps.h create mode 100644 tools/perf/util/mem-events.c create mode 100644 tools/perf/util/mem-events.h create mode 100644 tools/perf/util/mem-info.c create mode 100644 tools/perf/util/mem-info.h create mode 100644 tools/perf/util/mem2node.c create mode 100644 tools/perf/util/mem2node.h create mode 100644 tools/perf/util/memswap.c create mode 100644 tools/perf/util/memswap.h create mode 100644 tools/perf/util/metricgroup.c create mode 100644 tools/perf/util/metricgroup.h create mode 100644 tools/perf/util/mmap.c create mode 100644 tools/perf/util/mmap.h create mode 100644 tools/perf/util/mutex.c create mode 100644 tools/perf/util/mutex.h create mode 100644 tools/perf/util/namespaces.c create mode 100644 tools/perf/util/namespaces.h create mode 100644 tools/perf/util/off_cpu.h create mode 100644 tools/perf/util/ordered-events.c create mode 100644 tools/perf/util/ordered-events.h create mode 100644 tools/perf/util/parse-branch-options.c create mode 100644 tools/perf/util/parse-branch-options.h create mode 100644 tools/perf/util/parse-events.c create mode 100644 tools/perf/util/parse-events.h create mode 100644 tools/perf/util/parse-events.l create mode 100644 tools/perf/util/parse-events.y create mode 100644 tools/perf/util/parse-regs-options.c create mode 100644 tools/perf/util/parse-regs-options.h create mode 100644 tools/perf/util/parse-sublevel-options.c create mode 100644 tools/perf/util/parse-sublevel-options.h create mode 100644 tools/perf/util/path.c create mode 100644 tools/perf/util/path.h create mode 100644 tools/perf/util/perf-hooks-list.h create mode 100644 tools/perf/util/perf-hooks.c create mode 100644 tools/perf/util/perf-hooks.h create mode 100644 tools/perf/util/perf-regs-arch/Build create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_aarch64.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_arm.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_csky.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_loongarch.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_mips.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_powerpc.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_riscv.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_s390.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_x86.c create mode 100644 tools/perf/util/perf_api_probe.c create mode 100644 tools/perf/util/perf_api_probe.h create mode 100644 tools/perf/util/perf_event_attr_fprintf.c create mode 100644 tools/perf/util/perf_regs.c create mode 100644 tools/perf/util/perf_regs.h create mode 100644 tools/perf/util/pfm.c create mode 100644 tools/perf/util/pfm.h create mode 100644 tools/perf/util/pmu.c create mode 100644 tools/perf/util/pmu.h create mode 100644 tools/perf/util/pmu.l create mode 100644 tools/perf/util/pmu.y create mode 100644 tools/perf/util/pmus.c create mode 100644 tools/perf/util/pmus.h create mode 100644 tools/perf/util/powerpc-vpadtl.c create mode 100644 tools/perf/util/powerpc-vpadtl.h create mode 100644 tools/perf/util/print-events.c create mode 100644 tools/perf/util/print-events.h create mode 100644 tools/perf/util/print_binary.c create mode 100644 tools/perf/util/print_binary.h create mode 100644 tools/perf/util/print_insn.c create mode 100644 tools/perf/util/print_insn.h create mode 100644 tools/perf/util/probe-event.c create mode 100644 tools/perf/util/probe-event.h create mode 100644 tools/perf/util/probe-file.c create mode 100644 tools/perf/util/probe-file.h create mode 100644 tools/perf/util/probe-finder.c create mode 100644 tools/perf/util/probe-finder.h create mode 100644 tools/perf/util/pstack.c create mode 100644 tools/perf/util/pstack.h create mode 100644 tools/perf/util/python.c create mode 100644 tools/perf/util/rblist.c create mode 100644 tools/perf/util/rblist.h create mode 100644 tools/perf/util/record.c create mode 100644 tools/perf/util/record.h create mode 100644 tools/perf/util/rlimit.c create mode 100644 tools/perf/util/rlimit.h create mode 100644 tools/perf/util/rwsem.c create mode 100644 tools/perf/util/rwsem.h create mode 100644 tools/perf/util/s390-cpumcf-kernel.h create mode 100644 tools/perf/util/s390-cpumsf-kernel.h create mode 100644 tools/perf/util/s390-cpumsf.c create mode 100644 tools/perf/util/s390-cpumsf.h create mode 100644 tools/perf/util/s390-sample-raw.c create mode 100644 tools/perf/util/sample-raw.c create mode 100644 tools/perf/util/sample-raw.h create mode 100644 tools/perf/util/sample.c create mode 100644 tools/perf/util/sample.h create mode 100644 tools/perf/util/scripting-engines/Build create mode 100644 tools/perf/util/scripting-engines/trace-event-perl.c create mode 100644 tools/perf/util/scripting-engines/trace-event-python.c create mode 100644 tools/perf/util/session.c create mode 100644 tools/perf/util/session.h create mode 100644 tools/perf/util/setns.c create mode 100644 tools/perf/util/setup.py create mode 100644 tools/perf/util/sharded_mutex.c create mode 100644 tools/perf/util/sharded_mutex.h create mode 100644 tools/perf/util/sideband_evlist.c create mode 100644 tools/perf/util/smt.c create mode 100644 tools/perf/util/smt.h create mode 100644 tools/perf/util/sort.c create mode 100644 tools/perf/util/sort.h create mode 100644 tools/perf/util/spark.c create mode 100644 tools/perf/util/spark.h create mode 100644 tools/perf/util/srccode.c create mode 100644 tools/perf/util/srccode.h create mode 100644 tools/perf/util/srcline.c create mode 100644 tools/perf/util/srcline.h create mode 100644 tools/perf/util/stat-display.c create mode 100644 tools/perf/util/stat-shadow.c create mode 100644 tools/perf/util/stat.c create mode 100644 tools/perf/util/stat.h create mode 100644 tools/perf/util/strbuf.c create mode 100644 tools/perf/util/strbuf.h create mode 100644 tools/perf/util/stream.c create mode 100644 tools/perf/util/stream.h create mode 100644 tools/perf/util/strfilter.c create mode 100644 tools/perf/util/strfilter.h create mode 100644 tools/perf/util/string.c create mode 100644 tools/perf/util/string2.h create mode 100644 tools/perf/util/strlist.c create mode 100644 tools/perf/util/strlist.h create mode 100644 tools/perf/util/svghelper.c create mode 100644 tools/perf/util/svghelper.h create mode 100644 tools/perf/util/symbol-elf.c create mode 100644 tools/perf/util/symbol-minimal.c create mode 100644 tools/perf/util/symbol.c create mode 100644 tools/perf/util/symbol.h create mode 100644 tools/perf/util/symbol_conf.h create mode 100644 tools/perf/util/symbol_fprintf.c create mode 100644 tools/perf/util/symsrc.h create mode 100644 tools/perf/util/synthetic-events.c create mode 100644 tools/perf/util/synthetic-events.h create mode 100644 tools/perf/util/target.c create mode 100644 tools/perf/util/target.h create mode 100644 tools/perf/util/term.c create mode 100644 tools/perf/util/term.h create mode 100644 tools/perf/util/thread-stack.c create mode 100644 tools/perf/util/thread-stack.h create mode 100644 tools/perf/util/thread.c create mode 100644 tools/perf/util/thread.h create mode 100644 tools/perf/util/thread_map.c create mode 100644 tools/perf/util/thread_map.h create mode 100644 tools/perf/util/threads.c create mode 100644 tools/perf/util/threads.h create mode 100644 tools/perf/util/time-utils.c create mode 100644 tools/perf/util/time-utils.h create mode 100644 tools/perf/util/tool.c create mode 100644 tools/perf/util/tool.h create mode 100644 tools/perf/util/tool_pmu.c create mode 100644 tools/perf/util/tool_pmu.h create mode 100644 tools/perf/util/top.c create mode 100644 tools/perf/util/top.h create mode 100644 tools/perf/util/topdown.c create mode 100644 tools/perf/util/topdown.h create mode 100644 tools/perf/util/tp_pmu.c create mode 100644 tools/perf/util/tp_pmu.h create mode 100644 tools/perf/util/trace-event-info.c create mode 100644 tools/perf/util/trace-event-parse.c create mode 100644 tools/perf/util/trace-event-read.c create mode 100644 tools/perf/util/trace-event-scripting.c create mode 100644 tools/perf/util/trace-event.c create mode 100644 tools/perf/util/trace-event.h create mode 100644 tools/perf/util/trace.h create mode 100644 tools/perf/util/trace_augment.h create mode 100644 tools/perf/util/tracepoint.c create mode 100644 tools/perf/util/tracepoint.h create mode 100644 tools/perf/util/trigger.h create mode 100644 tools/perf/util/tsc.c create mode 100644 tools/perf/util/tsc.h create mode 100644 tools/perf/util/units.c create mode 100644 tools/perf/util/units.h create mode 100644 tools/perf/util/unwind-libdw.c create mode 100644 tools/perf/util/unwind-libdw.h create mode 100644 tools/perf/util/unwind-libunwind.c create mode 100644 tools/perf/util/unwind.c create mode 100644 tools/perf/util/unwind.h create mode 100644 tools/perf/util/usage.c create mode 100644 tools/perf/util/util.c create mode 100644 tools/perf/util/util.h create mode 100644 tools/perf/util/values.c create mode 100644 tools/perf/util/values.h create mode 100644 tools/perf/util/vdso.c create mode 100644 tools/perf/util/vdso.h create mode 100644 tools/perf/util/zlib.c create mode 100644 tools/perf/util/zstd.c (limited to 'tools/perf/util') diff --git a/tools/perf/util/Build b/tools/perf/util/Build new file mode 100644 index 000000000..b26a0b1dd --- /dev/null +++ b/tools/perf/util/Build @@ -0,0 +1,459 @@ +include $(srctree)/tools/scripts/Makefile.include +include $(srctree)/tools/scripts/utilities.mak + +perf-util-y += annotate-arch/ +perf-util-y += arm64-frame-pointer-unwind-support.o +perf-util-y += addr2line.o +perf-util-y += addr_location.o +perf-util-y += annotate.o +perf-util-y += aslr.o +perf-util-y += blake2s.o +perf-util-y += block-info.o +perf-util-y += block-range.o +perf-util-y += build-id.o +perf-util-y += c2c.o +perf-util-y += c2c-function.o +perf-util-y += cacheline.o +perf-util-$(CONFIG_LIBCAPSTONE) += capstone.o +perf-util-y += config.o +perf-util-y += copyfile.o +perf-util-y += ctype.o +perf-util-y += db-export.o +perf-util-y += disasm.o +perf-util-y += env.o +perf-util-y += event.o +perf-util-y += evlist.o +perf-util-y += sideband_evlist.o +perf-util-y += evsel.o +perf-util-y += evsel_fprintf.o +perf-util-y += perf_event_attr_fprintf.o +perf-util-y += evswitch.o +perf-util-y += find_bit.o +perf-util-y += levenshtein.o +perf-util-$(CONFIG_LIBBFD) += libbfd.o +perf-util-y += llvm.o +perf-util-y += mmap.o +perf-util-y += memswap.o +perf-util-y += parse-events.o +perf-util-y += print-events.o +perf-util-y += tracepoint.o +perf-util-y += perf_regs.o +perf-util-y += perf-regs-arch/ +perf-util-y += path.o +perf-util-y += print_binary.o +perf-util-y += print_insn.o +perf-util-y += rlimit.o +perf-util-y += argv_split.o +perf-util-y += rbtree.o +perf-util-y += libstring.o +perf-util-y += bitmap.o +perf-util-y += hweight.o +perf-util-y += smt.o +perf-util-y += strbuf.o +perf-util-y += string.o +perf-util-y += strlist.o +perf-util-y += strfilter.o +perf-util-y += top.o +perf-util-y += usage.o +perf-util-y += dso.o +perf-util-y += dsos.o +perf-util-y += symbol.o +perf-util-y += symbol_fprintf.o +perf-util-y += map_symbol.o +perf-util-y += color.o +perf-util-y += color_config.o +perf-util-y += metricgroup.o +perf-util-y += header.o +perf-util-y += callchain.o +perf-util-y += values.o +perf-util-y += debug.o +perf-util-y += fncache.o +perf-util-y += machine.o +perf-util-y += map.o +perf-util-y += maps.o +perf-util-y += pstack.o +perf-util-y += session.o +perf-util-y += tool.o +perf-util-y += sample.o +perf-util-y += sample-raw.o +perf-util-y += s390-sample-raw.o +perf-util-y += amd-sample-raw.o +perf-util-y += ordered-events.o +perf-util-y += namespaces.o +perf-util-y += comm.o +perf-util-y += thread.o +perf-util-y += threads.o +perf-util-y += thread_map.o +perf-util-y += parse-events-flex.o +perf-util-y += parse-events-bison.o +perf-util-y += pmu.o +perf-util-y += pmus.o +perf-util-y += pmu-flex.o +perf-util-y += pmu-bison.o +perf-util-y += drm_pmu.o +perf-util-y += hwmon_pmu.o +perf-util-y += tool_pmu.o +perf-util-y += tp_pmu.o +perf-util-y += svghelper.o +perf-util-y += trace-event-info.o +perf-util-y += trace-event-scripting.o +perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event.o +perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event-parse.o +perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event-read.o +perf-util-y += sort.o +perf-util-y += hist.o +perf-util-y += util.o +perf-util-y += cpumap.o +perf-util-y += affinity.o +perf-util-y += cputopo.o +perf-util-y += cgroup.o +perf-util-y += target.o +perf-util-y += rblist.o +perf-util-y += intlist.o +perf-util-y += vdso.o +perf-util-y += counts.o +perf-util-y += stat.o +perf-util-y += stat-shadow.o +perf-util-y += stat-display.o +perf-util-y += perf_api_probe.o +perf-util-y += record.o +perf-util-y += srcline.o +perf-util-y += srccode.o +perf-util-y += synthetic-events.o +perf-util-y += data.o +perf-util-y += tsc.o +perf-util-y += cloexec.o +perf-util-y += call-path.o +perf-util-y += rwsem.o +perf-util-y += thread-stack.o +perf-util-y += spark.o +perf-util-y += topdown.o +perf-util-y += iostat.o +perf-util-y += stream.o +perf-util-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o +perf-util-y += kvm-stat-arch/ +perf-util-y += lock-contention.o +perf-util-y += auxtrace.o +perf-util-y += intel-pt-decoder/ +perf-util-y += intel-pt.o +perf-util-y += intel-bts.o +perf-util-y += arm-spe.o +perf-util-y += arm-spe-decoder/ +perf-util-y += hisi-ptt.o +perf-util-y += hisi-ptt-decoder/ +perf-util-y += s390-cpumsf.o +perf-util-y += powerpc-vpadtl.o + +ifdef CONFIG_LIBOPENCSD +perf-util-y += cs-etm.o +perf-util-y += cs-etm-decoder/ +endif +perf-util-y += cs-etm-base.o + +perf-util-y += parse-branch-options.o +perf-util-y += parse-regs-options.o +perf-util-y += parse-sublevel-options.o +perf-util-y += term.o +perf-util-y += help-unknown-cmd.o +perf-util-y += dlfilter.o +perf-util-y += mem-events.o +perf-util-y += mem-info.o +perf-util-y += vsprintf.o +perf-util-y += units.o +perf-util-y += time-utils.o +perf-util-y += expr-flex.o +perf-util-y += expr-bison.o +perf-util-y += expr.o +perf-util-y += branch.o +perf-util-y += mem2node.o +perf-util-y += clockid.o +perf-util-y += list_sort.o +perf-util-y += mutex.o +perf-util-y += sharded_mutex.o +perf-util-y += intel-tpebs.o + +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_counter.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_counter_cgroup.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_ftrace.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_off_cpu.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-filter.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-filter-flex.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-filter-bison.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += btf.o + +ifeq ($(CONFIG_TRACE),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-trace-summary.o + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_trace_augment.o +endif + +ifeq ($(CONFIG_LIBTRACEEVENT),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_lock_contention.o +endif + +ifeq ($(CONFIG_LIBTRACEEVENT),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork.o + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork_top.o +endif + +perf-util-$(CONFIG_LIBELF) += symbol-elf.o +perf-util-$(CONFIG_LIBELF) += probe-file.o +perf-util-$(CONFIG_LIBELF) += probe-event.o + +ifdef CONFIG_LIBBPF_DYNAMIC + hashmap := 1 +endif +ifndef CONFIG_LIBBPF + hashmap := 1 +endif + +ifdef hashmap +perf-util-y += hashmap.o +endif + +ifndef CONFIG_LIBELF +perf-util-y += symbol-minimal.o +endif + +ifndef CONFIG_SETNS +perf-util-y += setns.o +endif + +perf-util-y += unwind.o +perf-util-$(CONFIG_LIBDW) += probe-finder.o +perf-util-$(CONFIG_LIBDW) += dwarf-aux.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-arch/ +perf-util-$(CONFIG_LIBDW) += debuginfo.o +perf-util-$(CONFIG_LIBDW) += annotate-data.o +perf-util-$(CONFIG_LIBDW) += libdw.o +perf-util-$(CONFIG_LIBDW) += unwind-libdw.o + +perf-util-$(CONFIG_LIBUNWIND) += unwind-libunwind.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arch/ + +ifeq ($(CONFIG_LIBTRACEEVENT),y) + perf-util-$(CONFIG_BABELTRACE2_CTF_WRITER) += data-convert-bt.o +endif + +perf-util-y += data-convert-json.o + +perf-util-y += scripting-engines/ + +perf-util-$(CONFIG_ZLIB) += zlib.o +perf-util-$(CONFIG_LZMA) += lzma.o +perf-util-$(CONFIG_ZSTD) += zstd.o + +perf-util-y += cap.o + +perf-util-$(CONFIG_CXX_DEMANGLE) += demangle-cxx.o +perf-util-y += demangle-ocaml.o +perf-util-y += demangle-java.o +perf-util-y += demangle-rust-v0.o +perf-util-$(CONFIG_LIBLLVM) += llvm-c-helpers.o + +CFLAGS_demangle-rust-v0.o += -Wno-shadow -Wno-declaration-after-statement \ + -Wno-switch-default -Wno-switch-enum -Wno-missing-field-initializers + +ifdef CONFIG_JITDUMP +perf-util-$(CONFIG_LIBELF) += jitdump.o +perf-util-$(CONFIG_LIBELF) += genelf.o +perf-util-$(CONFIG_LIBDW) += genelf_debug.o +endif + +perf-util-y += perf-hooks.o + +perf-util-$(CONFIG_LIBBPF) += bpf-event.o +perf-util-$(CONFIG_LIBBPF) += bpf-utils.o + +perf-util-$(CONFIG_LIBPFM4) += pfm.o + +CFLAGS_config.o += -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" + +# avoid compiler warnings in 32-bit mode +CFLAGS_genelf_debug.o += -Wno-packed + +$(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-flex.h: util/parse-events.l $(OUTPUT)util/parse-events-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/parse-events-flex.c \ + --header-file=$(OUTPUT)util/parse-events-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/parse-events-bison.c $(OUTPUT)util/parse-events-bison.h: util/parse-events.y + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) $(BISON_FALLBACK_FLAGS) \ + -o $(OUTPUT)util/parse-events-bison.c -p parse_events_ + +$(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-flex.h: util/expr.l $(OUTPUT)util/expr-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/expr-flex.c \ + --header-file=$(OUTPUT)util/expr-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/expr-bison.c $(OUTPUT)util/expr-bison.h: util/expr.y + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ + -o $(OUTPUT)util/expr-bison.c -p expr_ + +$(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-flex.h: util/pmu.l $(OUTPUT)util/pmu-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/pmu-flex.c \ + --header-file=$(OUTPUT)util/pmu-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/pmu-bison.c $(OUTPUT)util/pmu-bison.h: util/pmu.y + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ + -o $(OUTPUT)util/pmu-bison.c -p perf_pmu_ + +$(OUTPUT)util/bpf-filter-flex.c $(OUTPUT)util/bpf-filter-flex.h: util/bpf-filter.l $(OUTPUT)util/bpf-filter-bison.c util/bpf-filter.h util/bpf_skel/sample-filter.h + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/bpf-filter-flex.c \ + --header-file=$(OUTPUT)util/bpf-filter-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/bpf-filter-bison.c $(OUTPUT)util/bpf-filter-bison.h: util/bpf-filter.y util/bpf-filter.h util/bpf_skel/sample-filter.h + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ + -o $(OUTPUT)util/bpf-filter-bison.c -p perf_bpf_filter_ + +FLEX_VERSION := $(shell $(FLEX) --version | cut -d' ' -f2) + +FLEX_GE_260 := $(call version-ge3,$(FLEX_VERSION),2.6.0) +ifeq ($(FLEX_GE_260),1) + flex_flags := -Wno-redundant-decls -Wno-switch-default -Wno-unused-function -Wno-misleading-indentation + + # Some newer clang and gcc version complain about this + # util/parse-events-bison.c:1317:9: error: variable 'parse_events_nerrs' set but not used [-Werror,-Wunused-but-set-variable] + # int yynerrs = 0; + + flex_flags += -Wno-unused-but-set-variable + + FLEX_LT_262 := $(call version-lt3,$(FLEX_VERSION),2.6.2) + ifeq ($(FLEX_LT_262),1) + flex_flags += -Wno-sign-compare + endif +else + flex_flags := -w +endif + +# Some newer clang and gcc version complain about this +# util/parse-events-bison.c:1317:9: error: variable 'parse_events_nerrs' set but not used [-Werror,-Wunused-but-set-variable] +# int yynerrs = 0; + +bison_flags := -DYYENABLE_NLS=0 -Wno-unused-but-set-variable + +# Old clangs don't grok -Wno-unused-but-set-variable, remove it +ifeq ($(CC_NO_CLANG), 0) + CLANG_VERSION := $(shell $(CLANG) --version | head -1 | sed 's/.*clang version \([[:digit:]]\+.[[:digit:]]\+.[[:digit:]]\+\).*/\1/g') + ifeq ($(call version-lt3,$(CLANG_VERSION),13.0.0),1) + bison_flags := $(subst -Wno-unused-but-set-variable,,$(bison_flags)) + flex_flags := $(subst -Wno-unused-but-set-variable,,$(flex_flags)) + endif +endif + +BISON_GE_382 := $(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \>\= 382) +ifeq ($(BISON_GE_382),1) + bison_flags += -Wno-switch-enum +else + bison_flags += -w +endif + +BISON_LT_381 := $(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 381) +ifeq ($(BISON_LT_381),1) + bison_flags += -DYYNOMEM=YYABORT +endif + +CFLAGS_parse-events-flex.o += $(flex_flags) -Wno-unused-label +CFLAGS_pmu-flex.o += $(flex_flags) +CFLAGS_expr-flex.o += $(flex_flags) +CFLAGS_bpf-filter-flex.o += $(flex_flags) + +CFLAGS_parse-events-bison.o += $(bison_flags) +CFLAGS_pmu-bison.o += -DYYLTYPE_IS_TRIVIAL=0 $(bison_flags) +CFLAGS_expr-bison.o += -DYYLTYPE_IS_TRIVIAL=0 $(bison_flags) +CFLAGS_bpf-filter-bison.o += -DYYLTYPE_IS_TRIVIAL=0 $(bison_flags) + +$(OUTPUT)util/parse-events.o: $(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-bison.c +$(OUTPUT)util/pmu.o: $(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-bison.c +$(OUTPUT)util/expr.o: $(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-bison.c +$(OUTPUT)util/bpf-filter.o: $(OUTPUT)util/bpf-filter-flex.c $(OUTPUT)util/bpf-filter-bison.c + +CFLAGS_bitmap.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_find_bit.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_rbtree.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_libstring.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_hweight.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_header.o += -include $(OUTPUT)PERF-VERSION-FILE +CFLAGS_arm-spe.o += -I$(srctree)/tools/arch/arm64/include/ -I$(OUTPUT)arch/arm64/include/generated/ + +$(OUTPUT)util/argv_split.o: ../lib/argv_split.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/bitmap.o: ../lib/bitmap.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/ctype.o: ../lib/ctype.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/find_bit.o: ../lib/find_bit.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/rbtree.o: ../lib/rbtree.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/libstring.o: ../lib/string.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/hweight.o: ../lib/hweight.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/vsprintf.o: ../lib/vsprintf.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/list_sort.o: ../lib/list_sort.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +PY_TESTS := setup.py +ifdef MYPY + MYPY_TEST_LOGS := $(PY_TESTS:%=%.mypy_log) +else + MYPY_TEST_LOGS := +endif + +$(OUTPUT)%.mypy_log: % + $(call rule_mkdir) + $(Q)$(call echo-cmd,test)mypy "$<" > $@ || (cat $@ && rm $@ && false) + +perf-util-y += $(MYPY_TEST_LOGS) + +ifdef PYLINT + PYLINT_TEST_LOGS := $(PY_TESTS:%=%.pylint_log) +else + PYLINT_TEST_LOGS := +endif + +$(OUTPUT)%.pylint_log: % + $(call rule_mkdir) + $(Q)$(call echo-cmd,test)pylint "$<" > $@ || (cat $@ && rm $@ && false) + +perf-util-y += $(PYLINT_TEST_LOGS) + +ifeq ($(CONFIG_PERF_BPF_SKEL),y) +include $(srctree)/tools/perf/bpf_skel.mak + +$(OUTPUT)util/bpf_ftrace.o: $(SKEL_OUT)/func_latency.skel.h +$(OUTPUT)util/bpf-filter.o: $(SKEL_OUT)/sample_filter.skel.h +$(OUTPUT)util/bpf_kwork_top.o: $(SKEL_OUT)/kwork_top.skel.h +$(OUTPUT)util/bpf_off_cpu.o: $(SKEL_OUT)/off_cpu.skel.h +$(OUTPUT)util/bpf-trace-summary.o: $(SKEL_OUT)/syscall_summary.skel.h +$(OUTPUT)util/bpf_counter_cgroup.o: $(SKEL_OUT)/bperf_cgroup.skel.h +$(OUTPUT)util/bpf_trace_augment.o: $(SKEL_OUT)/augmented_raw_syscalls.skel.h +$(OUTPUT)util/bpf_counter.o: $(SKEL_OUT)/bpf_prog_profiler.skel.h $(SKEL_OUT)/bperf_leader.skel.h $(SKEL_OUT)/bperf_follower.skel.h +$(OUTPUT)util/bpf_lock_contention.o: $(SKEL_OUT)/lock_contention.skel.h +$(OUTPUT)util/bpf_kwork.o: $(SKEL_OUT)/kwork_trace.skel.h +endif diff --git a/tools/perf/util/PERF-VERSION-GEN b/tools/perf/util/PERF-VERSION-GEN new file mode 100755 index 000000000..d7dc7c285 --- /dev/null +++ b/tools/perf/util/PERF-VERSION-GEN @@ -0,0 +1,52 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +if [ $# -eq 1 ] ; then + OUTPUT=$1 +fi + +GVF=${OUTPUT}PERF-VERSION-FILE + +LF=' +' + +# +# Use version from kernel Makefile unless not in a git repository and +# PERF-VERSION-FILE exists +# +CID= +TAG= +if test -d ../../.git -o -f ../../.git +then + TAG=$(MAKEFLAGS= make -sC ../.. kernelversion) + CID=$(git log -1 --abbrev=12 --pretty=format:"%h" --no-show-signature 2>/dev/null) && CID="-g$CID" +elif test -f ../../PERF-VERSION-FILE +then + TAG=$(cut -d' ' -f3 ../../PERF-VERSION-FILE | sed -e 's/\"//g') +fi +if test -z "$TAG" +then + TAG=$(MAKEFLAGS= make -sC ../.. kernelversion) +fi + +VN="$TAG$CID" +if test -n "$CID" +then + # format version string, strip trailing zero of sublevel: + VN=$(echo "$VN" | sed -e 's/-/./g;s/\([0-9]*[.][0-9]*\)[.]0/\1/') +fi + +VN=$(expr "$VN" : v*'\(.*\)') + +if test -r $GVF +then + VC=$(sed -e 's/^#define PERF_VERSION "\(.*\)"/\1/' <$GVF) +else + VC=unset +fi +test "$VN" = "$VC" || { + echo >&2 " PERF_VERSION = $VN" + echo "#define PERF_VERSION \"$VN\"" >$GVF +} + + diff --git a/tools/perf/util/addr2line.c b/tools/perf/util/addr2line.c new file mode 100644 index 000000000..4b0d349ed --- /dev/null +++ b/tools/perf/util/addr2line.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "addr2line.h" +#include "debug.h" +#include "dso.h" +#include "string2.h" +#include "srcline.h" +#include "symbol.h" +#include "symbol_conf.h" + +#include +#include +#include + +#include +#include +#include +#include + +#define MAX_INLINE_NEST 1024 + +static int filename_split(char *filename, unsigned int *line_nr) +{ + char *sep; + + sep = strchr(filename, '\n'); + if (sep) + *sep = '\0'; + + if (!strcmp(filename, "??:0")) + return 0; + + sep = strchr(filename, ':'); + if (sep) { + *sep++ = '\0'; + *line_nr = strtoul(sep, NULL, 0); + return 1; + } + pr_debug("addr2line missing ':' in filename split\n"); + return 0; +} + +static void addr2line_subprocess_cleanup(struct child_process *a2l) +{ + if (a2l->pid != -1) { + kill(a2l->pid, SIGKILL); + finish_command(a2l); /* ignore result, we don't care */ + a2l->pid = -1; + close(a2l->in); + close(a2l->out); + } + + free(a2l); +} + +static struct child_process *addr2line_subprocess_init(const char *addr2line_path, + const char *binary_path) +{ + const char *argv[] = { + addr2line_path ?: "addr2line", + "-e", binary_path, + "-a", "-i", "-f", NULL + }; + struct child_process *a2l = zalloc(sizeof(*a2l)); + int start_command_status = 0; + + if (a2l == NULL) { + pr_err("Failed to allocate memory for addr2line"); + return NULL; + } + + a2l->pid = -1; + a2l->in = -1; + a2l->out = -1; + a2l->no_stderr = 1; + + a2l->argv = argv; + start_command_status = start_command(a2l); + a2l->argv = NULL; /* it's not used after start_command; avoid dangling pointers */ + + if (start_command_status != 0) { + pr_warning("could not start addr2line (%s) for %s: start_command return code %d\n", + addr2line_path, binary_path, start_command_status); + addr2line_subprocess_cleanup(a2l); + return NULL; + } + + return a2l; +} + +enum cmd_a2l_style { + BROKEN, + GNU_BINUTILS, + LLVM, +}; + +static enum cmd_a2l_style cmd_addr2line_configure(struct child_process *a2l, const char *dso_name) +{ + static bool cached; + static enum cmd_a2l_style style; + + if (!cached) { + char buf[128]; + struct io io; + int ch; + int lines; + + if (write(a2l->in, ",\n", 2) != 2) + return BROKEN; + + io__init(&io, a2l->out, buf, sizeof(buf)); + ch = io__get_char(&io); + if (ch == ',') { + style = LLVM; + cached = true; + lines = 1; + pr_debug3("Detected LLVM addr2line style\n"); + } else if (ch == '0') { + style = GNU_BINUTILS; + cached = true; + lines = 3; + pr_debug3("Detected binutils addr2line style\n"); + } else { + if (!symbol_conf.addr2line_disable_warn) { + char *output = NULL; + size_t output_len; + + io__getline(&io, &output, &output_len); + pr_warning("%s %s: addr2line configuration failed\n", + __func__, dso_name); + pr_warning("\t%c%s", ch, output); + } + pr_debug("Unknown/broken addr2line style\n"); + return BROKEN; + } + while (lines) { + ch = io__get_char(&io); + if (ch <= 0) + break; + if (ch == '\n') + lines--; + } + /* Ignore SIGPIPE in the event addr2line exits. */ + signal(SIGPIPE, SIG_IGN); + } + return style; +} + +static int read_addr2line_record(struct io *io, + enum cmd_a2l_style style, + const char *dso_name, + u64 addr, + bool first, + char **function, + char **filename, + unsigned int *line_nr) +{ + /* + * Returns: + * -1 ==> error + * 0 ==> sentinel (or other ill-formed) record read + * 1 ==> a genuine record read + */ + char *line = NULL; + size_t line_len = 0; + unsigned int dummy_line_nr = 0; + int ret = -1; + + if (function != NULL) + zfree(function); + + if (filename != NULL) + zfree(filename); + + if (line_nr != NULL) + *line_nr = 0; + + /* + * Read the first line. Without an error this will be: + * - for the first line an address like 0x1234, + * - the binutils sentinel 0x0000000000000000, + * - the llvm-addr2line the sentinel ',' character, + * - the function name line for an inlined function. + */ + if (io__getline(io, &line, &line_len) < 0 || !line_len) + goto error; + + pr_debug3("%s %s: addr2line read address for sentinel: %s", __func__, dso_name, line); + if (style == LLVM && line_len == 2 && line[0] == ',') { + /* Found the llvm-addr2line sentinel character. */ + zfree(&line); + return 0; + } else if (style == GNU_BINUTILS && (!first || addr != 0)) { + int zero_count = 0, non_zero_count = 0; + /* + * Check for binutils sentinel ignoring it for the case the + * requested address is 0. + */ + + /* A given address should always start 0x. */ + if (line_len >= 2 || line[0] != '0' || line[1] != 'x') { + for (size_t i = 2; i < line_len; i++) { + if (line[i] == '0') + zero_count++; + else if (line[i] != '\n') + non_zero_count++; + } + if (!non_zero_count) { + int ch; + + if (first && !zero_count) { + /* Line was erroneous just '0x'. */ + goto error; + } + /* + * Line was 0x0..0, the sentinel for binutils. Remove + * the function and filename lines. + */ + zfree(&line); + do { + ch = io__get_char(io); + } while (ch > 0 && ch != '\n'); + do { + ch = io__get_char(io); + } while (ch > 0 && ch != '\n'); + return 0; + } + } + } + /* Read the second function name line (if inline data then this is the first line). */ + if (first && (io__getline(io, &line, &line_len) < 0 || !line_len)) + goto error; + + pr_debug3("%s %s: addr2line read line: %s", __func__, dso_name, line); + if (function != NULL) + *function = strdup(strim(line)); + + zfree(&line); + line_len = 0; + + /* Read the third filename and line number line. */ + if (io__getline(io, &line, &line_len) < 0 || !line_len) + goto error; + + pr_debug3("%s %s: addr2line filename:number : %s", __func__, dso_name, line); + if (filename_split(line, line_nr == NULL ? &dummy_line_nr : line_nr) == 0 && + style == GNU_BINUTILS) { + ret = 0; + goto error; + } + + if (filename != NULL) + *filename = strdup(line); + + zfree(&line); + line_len = 0; + + return 1; + +error: + free(line); + if (function != NULL) + zfree(function); + if (filename != NULL) + zfree(filename); + return ret; +} + +static int inline_list__append_record(struct dso *dso, + struct inline_node *node, + struct symbol *sym, + const char *function, + const char *filename, + unsigned int line_nr) +{ + struct symbol *inline_sym = new_inline_sym(dso, sym, function); + + return inline_list__append(inline_sym, srcline_from_fileline(filename, line_nr), node); +} + +int cmd__addr2line(const char *dso_name, u64 addr, + char **file, unsigned int *line_nr, + struct dso *dso, + bool unwind_inlines, + struct inline_node *node, + struct symbol *sym __maybe_unused) +{ + struct child_process *a2l = dso__a2l(dso); + char *record_function = NULL; + char *record_filename = NULL; + unsigned int record_line_nr = 0; + int record_status = -1; + int ret = 0; + size_t inline_count = 0; + int len; + char buf[128]; + ssize_t written; + struct io io = { .eof = false }; + enum cmd_a2l_style cmd_a2l_style; + + if (!a2l) { + if (!filename__has_section(dso_name, ".debug_line")) + goto out; + + dso__set_a2l(dso, + addr2line_subprocess_init(symbol_conf.addr2line_path, dso_name)); + a2l = dso__a2l(dso); + } + + if (a2l == NULL) { + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: addr2line_subprocess_init failed\n", __func__, dso_name); + goto out; + } + cmd_a2l_style = cmd_addr2line_configure(a2l, dso_name); + if (cmd_a2l_style == BROKEN) + goto out; + + /* + * Send our request and then *deliberately* send something that can't be + * interpreted as a valid address to ask addr2line about (namely, + * ","). This causes addr2line to first write out the answer to our + * request, in an unbounded/unknown number of records, and then to write + * out the lines "0x0...0", "??" and "??:0", for GNU binutils, or "," + * for llvm-addr2line, so that we can detect when it has finished giving + * us anything useful. + */ + len = snprintf(buf, sizeof(buf), "%016"PRIx64"\n,\n", addr); + written = len > 0 ? write(a2l->in, buf, len) : -1; + if (written != len) { + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: could not send request\n", __func__, dso_name); + goto out; + } + io__init(&io, a2l->out, buf, sizeof(buf)); + io.timeout_ms = symbol_conf.addr2line_timeout_ms; + switch (read_addr2line_record(&io, cmd_a2l_style, dso_name, addr, /*first=*/true, + &record_function, &record_filename, &record_line_nr)) { + case -1: + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: could not read first record\n", __func__, dso_name); + goto out; + case 0: + /* + * The first record was invalid, so return failure, but first + * read another record, since we sent a sentinel ',' for the + * sake of detected the last inlined function. Treat this as the + * first of a record as the ',' generates a new start with GNU + * binutils, also force a non-zero address as we're no longer + * reading that record. + */ + switch (read_addr2line_record(&io, cmd_a2l_style, dso_name, + /*addr=*/1, /*first=*/true, + NULL, NULL, NULL)) { + case -1: + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: could not read sentinel record\n", + __func__, dso_name); + break; + case 0: + /* The sentinel as expected. */ + break; + default: + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: unexpected record instead of sentinel", + __func__, dso_name); + break; + } + goto out; + default: + /* First record as expected. */ + break; + } + + if (file) { + *file = strdup(record_filename); + ret = 1; + } + if (line_nr) + *line_nr = record_line_nr; + + if (unwind_inlines) { + if (node && inline_list__append_record(dso, node, sym, + record_function, + record_filename, + record_line_nr)) { + ret = 0; + goto out; + } + } + + /* + * We have to read the records even if we don't care about the inline + * info. This isn't the first record and force the address to non-zero + * as we're reading records beyond the first. + */ + while ((record_status = read_addr2line_record(&io, + cmd_a2l_style, + dso_name, + /*addr=*/1, + /*first=*/false, + &record_function, + &record_filename, + &record_line_nr)) == 1) { + if (unwind_inlines && node && inline_count++ < MAX_INLINE_NEST) { + if (inline_list__append_record(dso, node, sym, + record_function, + record_filename, + record_line_nr)) { + ret = 0; + goto out; + } + ret = 1; /* found at least one inline frame */ + } + } + +out: + free(record_function); + free(record_filename); + if (io.eof) { + dso__set_a2l(dso, NULL); + addr2line_subprocess_cleanup(a2l); + } + return ret; +} + +void dso__free_a2l(struct dso *dso) +{ + struct child_process *a2l = dso__a2l(dso); + + if (!a2l) + return; + + addr2line_subprocess_cleanup(a2l); + + dso__set_a2l(dso, NULL); +} diff --git a/tools/perf/util/addr2line.h b/tools/perf/util/addr2line.h new file mode 100644 index 000000000..75989a92f --- /dev/null +++ b/tools/perf/util/addr2line.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ADDR2LINE_H +#define __PERF_ADDR2LINE_H + +#include + +struct dso; +struct inline_node; +struct symbol; + +int cmd__addr2line(const char *dso_name, u64 addr, + char **file, unsigned int *line_nr, + struct dso *dso, + bool unwind_inlines, + struct inline_node *node, + struct symbol *sym); + +#endif /* __PERF_ADDR2LINE_H */ diff --git a/tools/perf/util/addr_location.c b/tools/perf/util/addr_location.c new file mode 100644 index 000000000..57e8217a0 --- /dev/null +++ b/tools/perf/util/addr_location.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "addr_location.h" +#include "map.h" +#include "maps.h" +#include "thread.h" + +void addr_location__init(struct addr_location *al) +{ + al->thread = NULL; + al->map = NULL; + al->sym = NULL; + al->srcline = NULL; + al->addr = 0; + al->level = 0; + al->filtered = 0; + al->cpumode = 0; + al->cpu = 0; + al->socket = 0; + al->parallelism = 1; +} + +/* + * The preprocess_sample method will return with reference counts for the + * in it, when done using (and perhaps getting ref counts if needing to + * keep a pointer to one of those entries) it must be paired with + * addr_location__exit(), so that the refcounts can be decremented. + */ +void addr_location__exit(struct addr_location *al) +{ + map__zput(al->map); + thread__zput(al->thread); +} + +void addr_location__copy(struct addr_location *dst, struct addr_location *src) +{ + thread__put(dst->thread); + map__put(dst->map); + *dst = *src; + dst->thread = thread__get(src->thread); + dst->map = map__get(src->map); +} diff --git a/tools/perf/util/addr_location.h b/tools/perf/util/addr_location.h new file mode 100644 index 000000000..fdc4d3f3a --- /dev/null +++ b/tools/perf/util/addr_location.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ADDR_LOCATION +#define __PERF_ADDR_LOCATION 1 + +#include + +struct thread; +struct maps; +struct map; +struct symbol; + +struct addr_location { + struct thread *thread; + struct map *map; + struct symbol *sym; + const char *srcline; + u64 addr; + char level; + u8 cpumode; + u16 filtered; + s32 cpu; + s32 socket; + /* Same as machine.parallelism but within [1, nr_cpus]. */ + int parallelism; + /* See he_stat.latency. */ + u64 latency; +}; + +void addr_location__init(struct addr_location *al); +void addr_location__exit(struct addr_location *al); + +void addr_location__copy(struct addr_location *dst, struct addr_location *src); + +#endif /* __PERF_ADDR_LOCATION */ diff --git a/tools/perf/util/affinity.c b/tools/perf/util/affinity.c new file mode 100644 index 000000000..6c64b5f69 --- /dev/null +++ b/tools/perf/util/affinity.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Manage affinity to optimize IPIs inside the kernel perf API. */ +#define _GNU_SOURCE 1 +#include +#include +#include +#include +#include +#include "perf.h" +#include "cpumap.h" +#include "affinity.h" + +static int get_cpu_set_size(void) +{ + int sz = cpu__max_cpu().cpu + 8 - 1; + /* + * sched_getaffinity doesn't like masks smaller than the kernel. + * Hopefully that's big enough. + */ + if (sz < 4096) + sz = 4096; + return sz / 8; +} + +int affinity__setup(struct affinity *a) +{ + int cpu_set_size = get_cpu_set_size(); + + a->orig_cpus = bitmap_zalloc(cpu_set_size * 8); + if (!a->orig_cpus) + return -1; + sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); + a->sched_cpus = bitmap_zalloc(cpu_set_size * 8); + if (!a->sched_cpus) { + zfree(&a->orig_cpus); + return -1; + } + bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size); + a->changed = false; + return 0; +} + +/* + * perf_event_open does an IPI internally to the target CPU. + * It is more efficient to change perf's affinity to the target + * CPU and then set up all events on that CPU, so we amortize + * CPU communication. + */ +void affinity__set(struct affinity *a, int cpu) +{ + int cpu_set_size = get_cpu_set_size(); + + /* + * Return: + * - if cpu is -1 + * - restrict out of bound access to sched_cpus + */ + if (cpu == -1 || ((cpu >= (cpu_set_size * 8)))) + return; + + a->changed = true; + __set_bit(cpu, a->sched_cpus); + /* + * We ignore errors because affinity is just an optimization. + * This could happen for example with isolated CPUs or cpusets. + * In this case the IPIs inside the kernel's perf API still work. + */ + sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus); + __clear_bit(cpu, a->sched_cpus); +} + +static void __affinity__cleanup(struct affinity *a) +{ + int cpu_set_size = get_cpu_set_size(); + + if (a->changed) + sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); + zfree(&a->sched_cpus); + zfree(&a->orig_cpus); +} + +void affinity__cleanup(struct affinity *a) +{ + if (a != NULL) + __affinity__cleanup(a); +} + +void cpu_map__set_affinity(const struct perf_cpu_map *cpumap) +{ + int cpu_set_size = get_cpu_set_size(); + unsigned long *cpuset = bitmap_zalloc(cpu_set_size * 8); + struct perf_cpu cpu; + unsigned int idx; + + if (!cpuset) + return; + + perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpumap) + __set_bit(cpu.cpu, cpuset); + + sched_setaffinity(0, cpu_set_size, (cpu_set_t *)cpuset); + zfree(&cpuset); +} diff --git a/tools/perf/util/affinity.h b/tools/perf/util/affinity.h new file mode 100644 index 000000000..7341194b2 --- /dev/null +++ b/tools/perf/util/affinity.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef PERF_AFFINITY_H +#define PERF_AFFINITY_H 1 + +#include + +struct perf_cpu_map; +struct affinity { + unsigned long *orig_cpus; + unsigned long *sched_cpus; + bool changed; +}; + +void affinity__cleanup(struct affinity *a); +void affinity__set(struct affinity *a, int cpu); +int affinity__setup(struct affinity *a); +void cpu_map__set_affinity(const struct perf_cpu_map *cpumap); + +#endif // PERF_AFFINITY_H diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c new file mode 100644 index 000000000..cda383632 --- /dev/null +++ b/tools/perf/util/amd-sample-raw.c @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD specific. Provide textual annotation for IBS raw sample data. + */ + +#include +#include +#include +#include + +#include +#include "../../arch/x86/include/asm/amd/ibs.h" + +#include "debug.h" +#include "session.h" +#include "evlist.h" +#include "sample-raw.h" +#include "util/sample.h" + +static u32 cpu_family, cpu_model, ibs_fetch_type, ibs_op_type; +static bool zen4_ibs_extensions; +static bool ldlat_cap; +static bool dtlb_pgsize_cap; +static bool rmtsocket_cap; +static bool strmst_cap; + +/* + * Status fields of IBS_FETCH_CTL and IBS_FETCH_CTL_EXT are valid only if + * IBS_FETCH_CTL[PhyAddrValid] is set. + */ +static int fetch_ctl_depends_on_phy_addr_valid(void) +{ + static int depends = -1; /* -1: Don't know, 1: Yes, 0: No */ + + if (depends != -1) + return depends; + + depends = 0; + if (cpu_family > 0x1a || + (cpu_family == 0x1a && ( + (cpu_model >= 0x50 && cpu_model <= 0x5f) || + (cpu_model >= 0x80 && cpu_model <= 0xaf) || + (cpu_model >= 0xc0 && cpu_model <= 0xcf)))) { + depends = 1; + } + + return depends; +} + +static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) +{ + const char * const ic_miss_strs[] = { + " IcMiss 0", + " IcMiss 1", + }; + const char * const l1tlb_pgsz_strs[] = { + " L1TlbPgSz 4KB", + " L1TlbPgSz 2MB", + " L1TlbPgSz 1GB", + " L1TlbPgSz RESERVED" + }; + const char * const l1tlb_pgsz_strs_erratum1347[] = { + " L1TlbPgSz 4KB", + " L1TlbPgSz 16KB", + " L1TlbPgSz 2MB", + " L1TlbPgSz 1GB" + }; + const char *ic_miss_str = NULL; + const char *l1tlb_pgsz_str = NULL; + char l3_miss_str[sizeof(" L3MissOnly _ FetchOcMiss _ FetchL3Miss _")] = ""; + char l3_miss_only_str[sizeof(" L3MissOnly _")] = ""; + + if (fetch_ctl_depends_on_phy_addr_valid() && !reg.phy_addr_valid) { + snprintf(l3_miss_only_str, sizeof(l3_miss_only_str), + " L3MissOnly %d", reg.l3_miss_only); + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d En %d Val %d Comp %d " + "PhyAddrValid 0 RandEn %d%s\n", reg.val, reg.fetch_maxcnt << 4, + reg.fetch_cnt << 4, reg.fetch_en, reg.fetch_val, reg.fetch_comp, + reg.rand_en, l3_miss_only_str); + return; + } + + if (cpu_family == 0x19 && cpu_model < 0x10) { + /* + * Erratum #1238 workaround is to ignore MSRC001_1030[IbsIcMiss] + * Erratum #1347 workaround is to use table provided in erratum + */ + if (reg.phy_addr_valid) + l1tlb_pgsz_str = l1tlb_pgsz_strs_erratum1347[reg.l1tlb_pgsz]; + } else { + if (reg.phy_addr_valid) + l1tlb_pgsz_str = l1tlb_pgsz_strs[reg.l1tlb_pgsz]; + ic_miss_str = ic_miss_strs[reg.ic_miss]; + } + + if (zen4_ibs_extensions) { + snprintf(l3_miss_str, sizeof(l3_miss_str), + " L3MissOnly %d FetchOcMiss %d FetchL3Miss %d", + reg.l3_miss_only, reg.fetch_oc_miss, reg.fetch_l3_miss); + } + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d Lat %5d En %d Val %d Comp %d%s " + "PhyAddrValid %d%s L1TlbMiss %d L2TlbMiss %d RandEn %d%s%s\n", + reg.val, reg.fetch_maxcnt << 4, reg.fetch_cnt << 4, reg.fetch_lat, + reg.fetch_en, reg.fetch_val, reg.fetch_comp, ic_miss_str ? : "", + reg.phy_addr_valid, l1tlb_pgsz_str ? : "", reg.l1tlb_miss, reg.l2tlb_miss, + reg.rand_en, reg.fetch_comp ? (reg.fetch_l2_miss ? " L2Miss 1" : " L2Miss 0") : "", + l3_miss_str); +} + +static void pr_ic_ibs_extd_ctl(union ibs_fetch_ctl fetch_ctl, union ic_ibs_extd_ctl reg) +{ + if (fetch_ctl_depends_on_phy_addr_valid() && !fetch_ctl.phy_addr_valid) + return; + + printf("ic_ibs_ext_ctl:\t%016llx IbsItlbRefillLat %3d\n", reg.val, reg.itlb_refill_lat); +} + +static void pr_ibs_op_ctl(union ibs_op_ctl reg) +{ + char l3_miss_only[sizeof(" L3MissOnly _")] = ""; + char ldlat[sizeof(" LdLatThrsh __ LdLatEn _")] = ""; + + if (zen4_ibs_extensions) + snprintf(l3_miss_only, sizeof(l3_miss_only), " L3MissOnly %d", reg.l3_miss_only); + + if (ldlat_cap) { + snprintf(ldlat, sizeof(ldlat), " LdLatThrsh %2d LdLatEn %d", + reg.ldlat_thrsh, reg.ldlat_en); + } + + printf("ibs_op_ctl:\t%016llx MaxCnt %9d%s En %d Val %d CntCtl %d=%s CurCnt %9d%s\n", + reg.val, ((reg.opmaxcnt_ext << 16) | reg.opmaxcnt) << 4, l3_miss_only, + reg.op_en, reg.op_val, reg.cnt_ctl, + reg.cnt_ctl ? "uOps" : "cycles", reg.opcurcnt, ldlat); +} + +static void pr_ibs_op_data(union ibs_op_data reg) +{ + printf("ibs_op_data:\t%016llx CompToRetCtr %5d TagToRetCtr %5d%s%s%s BrnRet %d " + " RipInvalid %d BrnFuse %d Microcode %d\n", + reg.val, reg.comp_to_ret_ctr, reg.tag_to_ret_ctr, + reg.op_brn_ret ? (reg.op_return ? " OpReturn 1" : " OpReturn 0") : "", + reg.op_brn_ret ? (reg.op_brn_taken ? " OpBrnTaken 1" : " OpBrnTaken 0") : "", + reg.op_brn_ret ? (reg.op_brn_misp ? " OpBrnMisp 1" : " OpBrnMisp 0") : "", + reg.op_brn_ret, reg.op_rip_invalid, reg.op_brn_fuse, reg.op_microcode); +} + +static void pr_ibs_op_data2_extended(union ibs_op_data2 reg) +{ + static const char * const data_src_str[] = { + "", + " DataSrc 1=Local L3 or other L1/L2 in CCX", + " DataSrc 2=Another CCX cache in the same NUMA node", + " DataSrc 3=DRAM", + " DataSrc 4=(reserved)", + " DataSrc 5=Another CCX cache in a different NUMA node", + " DataSrc 6=Long-latency DIMM", + " DataSrc 7=MMIO/Config/PCI/APIC", + " DataSrc 8=Extension Memory", + " DataSrc 9=(reserved)", + " DataSrc 10=(reserved)", + " DataSrc 11=(reserved)", + " DataSrc 12=Coherent Memory of a different processor type", + /* 13 to 31 are reserved. Avoid printing them. */ + }; + int data_src = (reg.data_src_hi << 3) | reg.data_src_lo; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, + (data_src == 1 || data_src == 2 || data_src == 5) ? + (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", + reg.rmt_node, + data_src < (int)ARRAY_SIZE(data_src_str) ? data_src_str[data_src] : ""); +} + +static void pr_ibs_op_data2_default(union ibs_op_data2 reg) +{ + static const char * const data_src_str[] = { + "", + " DataSrc 1=(reserved)", + " DataSrc 2=Local node cache", + " DataSrc 3=DRAM", + " DataSrc 4=Remote node cache", + " DataSrc 5=(reserved)", + " DataSrc 6=(reserved)", + " DataSrc 7=Other" + }; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, + reg.data_src_lo == 2 ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " + : "CacheHitSt 0=M-state ") : "", + reg.rmt_node, data_src_str[reg.data_src_lo]); +} + +static void pr_ibs_op_data2(union ibs_op_data2 reg) +{ + if (zen4_ibs_extensions) + return pr_ibs_op_data2_extended(reg); + pr_ibs_op_data2_default(reg); +} + +static void pr_ibs_op_data3(union ibs_op_data3 reg) +{ + static const char * const dc_page_sizes[] = { + " 4K", + " 2M", + " 1G", + " ??", + }; + char op_dc_miss_open_mem_reqs_str[sizeof(" OpDcMissOpenMemReqs __")] = ""; + char dc_l1_l2tlb_miss_str[sizeof(" DcL1TlbMiss _ DcL2TlbMiss _")] = ""; + char dc_l1tlb_hit_str[sizeof(" DcL1TlbHit2M _ DcL1TlbHit1G _")] = ""; + char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + char tlb_refill_lat_str[sizeof(" TlbRefillLat _____")] = ""; + char dc_l2tlb_hit_2m_str[sizeof(" DcL2TlbHit2M _")] = ""; + char dc_l2tlb_hit_1g_str[sizeof(" DcL2TlbHit1G _")] = ""; + char dc_page_size_str[sizeof(" DcPageSize ____")] = ""; + char l2_miss_str[sizeof(" L2Miss _")] = ""; + + /* + * Erratum #1293 + * Ignore L2Miss and OpDcMissOpenMemReqs (and opdata2) if DcMissNoMabAlloc or SwPf set + */ + if (!(cpu_family == 0x19 && cpu_model < 0x10 && (reg.dc_miss_no_mab_alloc || reg.sw_pf))) { + snprintf(l2_miss_str, sizeof(l2_miss_str), " L2Miss %d", reg.l2_miss); + snprintf(op_dc_miss_open_mem_reqs_str, sizeof(op_dc_miss_open_mem_reqs_str), + " OpDcMissOpenMemReqs %2d", reg.op_dc_miss_open_mem_reqs); + } + + if (reg.op_mem_width) + snprintf(op_mem_width_str, sizeof(op_mem_width_str), + " OpMemWidth %2d bytes", 1 << (reg.op_mem_width - 1)); + + if (dtlb_pgsize_cap) { + if (reg.dc_phy_addr_valid) { + int idx = (reg.dc_l1tlb_hit_1g << 1) | reg.dc_l1tlb_hit_2m; + + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_page_size_str, sizeof(dc_page_size_str), + " DcPageSize %4s", dc_page_sizes[idx]); + } + } else { + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_l1tlb_hit_str, sizeof(dc_l1tlb_hit_str), + " DcL1TlbHit2M %d DcL1TlbHit1G %d", + reg.dc_l1tlb_hit_2m, reg.dc_l1tlb_hit_1g); + snprintf(dc_l2tlb_hit_2m_str, sizeof(dc_l2tlb_hit_2m_str), + " DcL2TlbHit2M %d", reg.dc_l2tlb_hit_2m); + snprintf(dc_l2tlb_hit_1g_str, sizeof(dc_l2tlb_hit_1g_str), + " DcL2TlbHit1G %d", reg.dc_l2_tlb_hit_1g); + } + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (!zen4_ibs_extensions || reg.dc_phy_addr_valid) { + snprintf(tlb_refill_lat_str, sizeof(tlb_refill_lat_str), + " TlbRefillLat %5d", reg.tlb_refill_lat); + } + + printf("ibs_op_data3:\t%016llx LdOp %d StOp %d%s%s%s DcMiss %d DcMisAcc %d " + "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d " + "DcLinAddrValid %d DcPhyAddrValid %d%s%s SwPf %d%s%s " + "DcMissLat %5d%s\n", + reg.val, reg.ld_op, reg.st_op, dc_l1_l2tlb_miss_str, + dtlb_pgsize_cap ? dc_page_size_str : dc_l1tlb_hit_str, + dc_l2tlb_hit_2m_str, reg.dc_miss, reg.dc_mis_acc, reg.dc_wc_mem_acc, + reg.dc_uc_mem_acc, reg.dc_locked_op, reg.dc_miss_no_mab_alloc, + reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, dc_l2tlb_hit_1g_str, + l2_miss_str, reg.sw_pf, op_mem_width_str, op_dc_miss_open_mem_reqs_str, + reg.dc_miss_lat, tlb_refill_lat_str); +} + +/* + * IBS Op/Execution MSRs always saved, in order, are: + * IBS_OP_CTL, IBS_OP_RIP, IBS_OP_DATA, IBS_OP_DATA2, + * IBS_OP_DATA3, IBS_DC_LINADDR, IBS_DC_PHYSADDR, BP_IBSTGT_RIP + */ +static void amd_dump_ibs_op(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_op_ctl *op_ctl = (union ibs_op_ctl *)data->data; + __u64 *rip = (__u64 *)op_ctl + 1; + union ibs_op_data *op_data = (union ibs_op_data *)(rip + 1); + union ibs_op_data3 *op_data3 = (union ibs_op_data3 *)(rip + 3); + + pr_ibs_op_ctl(*op_ctl); + if (!op_data->op_rip_invalid) + printf("IbsOpRip:\t%016llx\n", *rip); + pr_ibs_op_data(*op_data); + /* + * Erratum #1293: ignore op_data2 if DcMissNoMabAlloc or SwPf are set + */ + if (!(cpu_family == 0x19 && cpu_model < 0x10 && + (op_data3->dc_miss_no_mab_alloc || op_data3->sw_pf))) + pr_ibs_op_data2(*(union ibs_op_data2 *)(rip + 2)); + pr_ibs_op_data3(*op_data3); + if (op_data3->dc_lin_addr_valid) + printf("IbsDCLinAd:\t%016llx\n", *(rip + 4)); + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (op_data3->dc_phy_addr_valid && *(rip + 5) && + (!zen4_ibs_extensions || op_data3->dc_lin_addr_valid)) { + printf("IbsDCPhysAd:\t%016llx\n", *(rip + 5)); + } + if (op_data->op_brn_ret && *(rip + 6)) + printf("IbsBrTarget:\t%016llx\n", *(rip + 6)); +} + +/* + * IBS Fetch MSRs always saved, in order, are: + * IBS_FETCH_CTL, IBS_FETCH_LINADDR, IBS_FETCH_PHYSADDR, IC_IBS_EXTD_CTL + */ +static void amd_dump_ibs_fetch(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_fetch_ctl *fetch_ctl = (union ibs_fetch_ctl *)data->data; + __u64 *addr = (__u64 *)fetch_ctl + 1; + union ic_ibs_extd_ctl *extd_ctl = (union ic_ibs_extd_ctl *)addr + 2; + + pr_ibs_fetch_ctl(*fetch_ctl); + printf("IbsFetchLinAd:\t%016llx\n", *addr++); + if (fetch_ctl->phy_addr_valid) + printf("IbsFetchPhysAd:\t%016llx\n", *addr); + pr_ic_ibs_extd_ctl(*fetch_ctl, *extd_ctl); +} + +/* + * Test for enable and valid bits in captured control MSRs. + */ +static bool is_valid_ibs_fetch_sample(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_fetch_ctl *fetch_ctl = (union ibs_fetch_ctl *)data->data; + + if (fetch_ctl->fetch_en && fetch_ctl->fetch_val) + return true; + + return false; +} + +static bool is_valid_ibs_op_sample(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_op_ctl *op_ctl = (union ibs_op_ctl *)data->data; + + if (op_ctl->op_en && op_ctl->op_val) + return true; + + return false; +} + +/* AMD vendor specific raw sample function. Check for PERF_RECORD_SAMPLE events + * and if the event was triggered by IBS, display its raw data with decoded text. + * The function is only invoked when the dump flag -D is set. + */ +void evlist__amd_sample_raw(struct evlist *evlist, union perf_event *event, + struct perf_sample *sample) +{ + struct evsel *evsel; + + if (event->header.type != PERF_RECORD_SAMPLE || !sample->raw_size) + return; + + evsel = evlist__event2evsel(evlist, event); + if (!evsel) + return; + + if (evsel->core.attr.type == ibs_fetch_type) { + if (!is_valid_ibs_fetch_sample(sample)) { + pr_debug("Invalid raw IBS Fetch MSR data encountered\n"); + return; + } + amd_dump_ibs_fetch(sample); + } else if (evsel->core.attr.type == ibs_op_type) { + if (!is_valid_ibs_op_sample(sample)) { + pr_debug("Invalid raw IBS Op MSR data encountered\n"); + return; + } + amd_dump_ibs_op(sample); + } +} + +static void parse_cpuid(struct perf_env *env) +{ + const char *cpuid; + int ret; + + cpuid = perf_env__cpuid(env); + /* + * cpuid = "AuthenticAMD,family,model,stepping" + */ + ret = sscanf(cpuid, "%*[^,],%u,%u", &cpu_family, &cpu_model); + if (ret != 2) + pr_debug("problem parsing cpuid\n"); +} + +/* + * Find and assign the type number used for ibs_op or ibs_fetch samples. + * Device names can be large - we are only interested in the first 9 characters, + * to match "ibs_fetch". + */ +bool evlist__has_amd_ibs(struct evlist *evlist) +{ + struct perf_env *env = perf_session__env(evlist__session(evlist)); + int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env); + const char *pmu_mapping = perf_env__pmu_mappings(env); + char name[sizeof("ibs_fetch")]; + u32 type; + + while (nr_pmu_mappings--) { + ret = sscanf(pmu_mapping, "%u:%9s", &type, name); + if (ret == 2) { + if (strstarts(name, "ibs_op")) + ibs_op_type = type; + else if (strstarts(name, "ibs_fetch")) + ibs_fetch_type = type; + } + pmu_mapping += strlen(pmu_mapping) + 1 /* '\0' */; + } + + if (perf_env__find_pmu_cap(env, "ibs_op", "zen4_ibs_extensions")) + zen4_ibs_extensions = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "ldlat")) + ldlat_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize")) + dtlb_pgsize_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "rmtsocket")) + rmtsocket_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "strmst")) + strmst_cap = 1; + + if (ibs_fetch_type || ibs_op_type) { + if (!cpu_family) + parse_cpuid(env); + return true; + } + + return false; +} diff --git a/tools/perf/util/annotate-arch/Build b/tools/perf/util/annotate-arch/Build new file mode 100644 index 000000000..23316743f --- /dev/null +++ b/tools/perf/util/annotate-arch/Build @@ -0,0 +1,11 @@ +perf-util-y += annotate-arc.o +perf-util-y += annotate-arm.o +perf-util-y += annotate-arm64.o +perf-util-y += annotate-csky.o +perf-util-y += annotate-loongarch.o +perf-util-y += annotate-mips.o +perf-util-y += annotate-x86.o +perf-util-y += annotate-powerpc.o +perf-util-y += annotate-riscv64.o +perf-util-y += annotate-s390.o +perf-util-y += annotate-sparc.o diff --git a/tools/perf/util/annotate-arch/annotate-arc.c b/tools/perf/util/annotate-arch/annotate-arc.c new file mode 100644 index 000000000..170103e38 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-arc.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../disasm.h" + +const struct arch *arch__new_arc(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "arc"; + arch->id = *id; + arch->objdump.comment_char = ';'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-arm.c b/tools/perf/util/annotate-arch/annotate-arm.c new file mode 100644 index 000000000..afb413c80 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-arm.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "../annotate.h" +#include "../disasm.h" + +struct arch_arm { + struct arch arch; + regex_t call_insn; + regex_t jump_insn; +}; + +static const struct ins_ops *arm__associate_instruction_ops(struct arch *arch, const char *name) +{ + struct arch_arm *arm = container_of(arch, struct arch_arm, arch); + const struct ins_ops *ops; + regmatch_t match[2]; + + if (!regexec(&arm->call_insn, name, 2, match, 0)) + ops = &call_ops; + else if (!regexec(&arm->jump_insn, name, 2, match, 0)) + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +const struct arch *arch__new_arm(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + int err; + struct arch_arm *arm = zalloc(sizeof(*arm)); + struct arch *arch; + + if (!arm) + return NULL; + + arch = &arm->arch; + arch->name = "arm"; + arch->id = *id; + arch->objdump.comment_char = ';'; + arch->objdump.skip_functions_char = '+'; + arch->associate_instruction_ops = arm__associate_instruction_ops; + +#define ARM_CONDS "(cc|cs|eq|ge|gt|hi|le|ls|lt|mi|ne|pl|vc|vs)" + err = regcomp(&arm->call_insn, "^blx?" ARM_CONDS "?$", REG_EXTENDED); + if (err) + goto out_free_arm; + + err = regcomp(&arm->jump_insn, "^bx?" ARM_CONDS "?$", REG_EXTENDED); + if (err) + goto out_free_call; +#undef ARM_CONDS + + return arch; + +out_free_call: + regfree(&arm->call_insn); +out_free_arm: + free(arm); + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP; + return NULL; +} diff --git a/tools/perf/util/annotate-arch/annotate-arm64.c b/tools/perf/util/annotate-arch/annotate-arm64.c new file mode 100644 index 000000000..33080fdca --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-arm64.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include "../annotate.h" +#include "../disasm.h" + +struct arch_arm64 { + struct arch arch; + regex_t call_insn; + regex_t jump_insn; +}; + +static int arm64_mov__parse(const struct arch *arch __maybe_unused, + struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *s = strchr(ops->raw, ','), *target, *endptr; + + if (s == NULL) + return -1; + + *s = '\0'; + ops->source.raw = strdup(ops->raw); + *s = ','; + + if (ops->source.raw == NULL) + return -1; + + target = ++s; + ops->target.raw = strdup(target); + if (ops->target.raw == NULL) + goto out_free_source; + + ops->target.addr = strtoull(target, &endptr, 16); + if (endptr == target) + goto out_free_target; + + s = strchr(endptr, '<'); + if (s == NULL) + goto out_free_target; + endptr = strchr(s + 1, '>'); + if (endptr == NULL) + goto out_free_target; + + *endptr = '\0'; + *s = ' '; + ops->target.name = strdup(s); + *s = '<'; + *endptr = '>'; + if (ops->target.name == NULL) + goto out_free_target; + + return 0; + +out_free_target: + zfree(&ops->target.raw); +out_free_source: + zfree(&ops->source.raw); + return -1; +} + +static const struct ins_ops arm64_mov_ops = { + .parse = arm64_mov__parse, + .scnprintf = mov__scnprintf, +}; + +static const struct ins_ops *arm64__associate_instruction_ops(struct arch *arch, const char *name) +{ + struct arch_arm64 *arm = container_of(arch, struct arch_arm64, arch); + const struct ins_ops *ops; + regmatch_t match[2]; + + if (!regexec(&arm->jump_insn, name, 2, match, 0)) + ops = &jump_ops; + else if (!regexec(&arm->call_insn, name, 2, match, 0)) + ops = &call_ops; + else if (!strcmp(name, "ret")) + ops = &ret_ops; + else + ops = &arm64_mov_ops; + + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +const struct arch *arch__new_arm64(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + int err; + struct arch_arm64 *arm = zalloc(sizeof(*arm)); + struct arch *arch; + + if (!arm) + return NULL; + + arch = &arm->arch; + arch->name = "arm64"; + arch->id = *id; + arch->objdump.comment_char = '/'; + arch->objdump.skip_functions_char = '+'; + arch->associate_instruction_ops = arm64__associate_instruction_ops; + + /* bl, blr */ + err = regcomp(&arm->call_insn, "^blr?$", REG_EXTENDED); + if (err) + goto out_free_arm; + + /* b, b.cond, br, cbz/cbnz, tbz/tbnz */ + err = regcomp(&arm->jump_insn, "^[ct]?br?\\.?(cc|cs|eq|ge|gt|hi|hs|le|lo|ls|lt|mi|ne|pl|vc|vs)?n?z?$", + REG_EXTENDED); + if (err) + goto out_free_call; + + return arch; + +out_free_call: + regfree(&arm->call_insn); +out_free_arm: + free(arm); + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP; + return NULL; +} diff --git a/tools/perf/util/annotate-arch/annotate-csky.c b/tools/perf/util/annotate-arch/annotate-csky.c new file mode 100644 index 000000000..d2b18e4ea --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-csky.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. +#include +#include +#include +#include "../disasm.h" + +static const struct ins_ops *csky__associate_ins_ops(struct arch *arch, + const char *name) +{ + const struct ins_ops *ops = NULL; + + /* catch all kind of jumps */ + if (!strcmp(name, "bt") || + !strcmp(name, "bf") || + !strcmp(name, "bez") || + !strcmp(name, "bnez") || + !strcmp(name, "bnezad") || + !strcmp(name, "bhsz") || + !strcmp(name, "bhz") || + !strcmp(name, "blsz") || + !strcmp(name, "blz") || + !strcmp(name, "br") || + !strcmp(name, "jmpi") || + !strcmp(name, "jmp")) + ops = &jump_ops; + + /* catch function call */ + if (!strcmp(name, "bsr") || + !strcmp(name, "jsri") || + !strcmp(name, "jsr")) + ops = &call_ops; + + /* catch function return */ + if (!strcmp(name, "rts")) + ops = &ret_ops; + + if (ops) + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +const struct arch *arch__new_csky(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "csky"; + arch->id = *id; + arch->objdump.comment_char = '/'; + arch->associate_instruction_ops = csky__associate_ins_ops; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-loongarch.c b/tools/perf/util/annotate-arch/annotate-loongarch.c new file mode 100644 index 000000000..c2addca77 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-loongarch.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Perf annotate functions. + * + * Copyright (C) 2020-2023 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include "../disasm.h" +#include "../map.h" +#include "../maps.h" +#include "../symbol.h" +#include "../thread.h" + +static int loongarch_call__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + char *c, *endptr, *tok, *name; + struct map *map = ms->map; + struct addr_map_symbol target; + + c = strchr(ops->raw, '#'); + if (c++ == NULL) + return -1; + + ops->target.addr = strtoull(c, &endptr, 16); + + name = strchr(endptr, '<'); + name++; + + if (arch->objdump.skip_functions_char && + strchr(name, arch->objdump.skip_functions_char)) + return -1; + + tok = strchr(name, '>'); + if (tok == NULL) + return -1; + + *tok = '\0'; + ops->target.name = strdup(name); + *tok = '>'; + + if (ops->target.name == NULL) + return -1; + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + return 0; +} + +static const struct ins_ops loongarch_call_ops = { + .parse = loongarch_call__parse, + .scnprintf = call__scnprintf, + .is_call = true, +}; + +static int loongarch_jump__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) + +{ + struct map *map = ms->map; + struct symbol *sym = ms->sym; + struct addr_map_symbol target = { + .ms = { .map = map__get(map), }, + }; + const char *c = strchr(ops->raw, '#'); + u64 start, end; + + ops->jump.raw_comment = strchr(ops->raw, arch->objdump.comment_char); + ops->jump.raw_func_start = strchr(ops->raw, '<'); + + if (ops->jump.raw_func_start && c > ops->jump.raw_func_start) + c = NULL; + + if (c++ != NULL) + ops->target.addr = strtoull(c, NULL, 16); + else + ops->target.addr = strtoull(ops->raw, NULL, 16); + + target.addr = map__objdump_2mem(map, ops->target.addr); + start = map__unmap_ip(map, sym->start); + end = map__unmap_ip(map, sym->end); + + ops->target.outside = target.addr < start || target.addr >= end; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + if (!ops->target.outside) { + ops->target.offset = target.addr - start; + ops->target.offset_avail = true; + } else { + ops->target.offset_avail = false; + } + addr_map_symbol__exit(&target); + return 0; +} + +static const struct ins_ops loongarch_jump_ops = { + .free = jump__delete, + .parse = loongarch_jump__parse, + .scnprintf = jump__scnprintf, + .is_jump = true, +}; + +static +const struct ins_ops *loongarch__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strcmp(name, "bl")) + ops = &loongarch_call_ops; + else if (!strcmp(name, "jirl")) + ops = &ret_ops; + else if (!strcmp(name, "b") || + !strncmp(name, "beq", 3) || + !strncmp(name, "bne", 3) || + !strncmp(name, "blt", 3) || + !strncmp(name, "bge", 3) || + !strncmp(name, "bltu", 4) || + !strncmp(name, "bgeu", 4)) + ops = &loongarch_jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_loongarch(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "loongarch"; + arch->id = *id; + arch->associate_instruction_ops = loongarch__associate_ins_ops; + arch->objdump.comment_char = '#'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-mips.c b/tools/perf/util/annotate-arch/annotate-mips.c new file mode 100644 index 000000000..e8d1c6c7e --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-mips.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../disasm.h" + +static +const struct ins_ops *mips__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strncmp(name, "bal", 3) || + !strncmp(name, "bgezal", 6) || + !strncmp(name, "bltzal", 6) || + !strncmp(name, "bgtzal", 6) || + !strncmp(name, "blezal", 6) || + !strncmp(name, "beqzal", 6) || + !strncmp(name, "bnezal", 6) || + !strncmp(name, "bgtzl", 5) || + !strncmp(name, "bltzl", 5) || + !strncmp(name, "bgezl", 5) || + !strncmp(name, "blezl", 5) || + !strncmp(name, "jialc", 5) || + !strncmp(name, "beql", 4) || + !strncmp(name, "bnel", 4) || + !strncmp(name, "jal", 3)) + ops = &call_ops; + else if (!strncmp(name, "jr", 2)) + ops = &ret_ops; + else if (name[0] == 'j' || name[0] == 'b') + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_mips(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "mips"; + arch->id = *id; + arch->objdump.comment_char = '#'; + arch->associate_instruction_ops = mips__associate_ins_ops; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-powerpc.c b/tools/perf/util/annotate-arch/annotate-powerpc.c new file mode 100644 index 000000000..218207b52 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-powerpc.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../annotate-data.h" +#include "../debug.h" +#include "../disasm.h" + +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define PPC_21_30(R) (((R) >> 1) & 0x3ff) +#define PPC_22_30(R) (((R) >> 1) & 0x1ff) + +#define MINUS_EXT_XO_FORM 234 +#define SUB_EXT_XO_FORM 232 +#define ADD_ZERO_EXT_XO_FORM 202 +#define SUB_ZERO_EXT_XO_FORM 200 + +static int arithmetic__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, + ops->raw); +} + +/* + * Sets the fields: multi_regs and "mem_ref". + * "mem_ref" is set for ops->source which is later used to + * fill the objdump->memory_ref-char field. This ops is currently + * used by powerpc and since binary instruction code is used to + * extract opcode, regs and offset, no other parsing is needed here. + * + * Dont set multi regs for 4 cases since it has only one operand + * for source: + * - Add to Minus One Extended XO-form ( Ex: addme, addmeo ) + * - Subtract From Minus One Extended XO-form ( Ex: subfme ) + * - Add to Zero Extended XO-form ( Ex: addze, addzeo ) + * - Subtract From Zero Extended XO-form ( Ex: subfze ) + */ +static int arithmetic__parse(const struct arch *arch __maybe_unused, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, struct disasm_line *dl) +{ + int opcode = PPC_OP(dl->raw.raw_insn); + + ops->source.mem_ref = false; + if (opcode == 31) { + if ((opcode != MINUS_EXT_XO_FORM) && (opcode != SUB_EXT_XO_FORM) && + (opcode != ADD_ZERO_EXT_XO_FORM) && (opcode != SUB_ZERO_EXT_XO_FORM)) + ops->source.multi_regs = true; + } + + ops->target.mem_ref = false; + ops->target.multi_regs = false; + + return 0; +} + +static const struct ins_ops arithmetic_ops = { + .parse = arithmetic__parse, + .scnprintf = arithmetic__scnprintf, +}; + +static int load_store__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, + ops->raw); +} + +/* + * Sets the fields: multi_regs and "mem_ref". + * "mem_ref" is set for ops->source which is later used to + * fill the objdump->memory_ref-char field. This ops is currently + * used by powerpc and since binary instruction code is used to + * extract opcode, regs and offset, no other parsing is needed here + */ +static int load_store__parse(const struct arch *arch __maybe_unused, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, struct disasm_line *dl __maybe_unused) +{ + ops->source.mem_ref = true; + ops->source.multi_regs = false; + /* opcode 31 is of X form */ + if (PPC_OP(dl->raw.raw_insn) == 31) + ops->source.multi_regs = true; + + ops->target.mem_ref = false; + ops->target.multi_regs = false; + + return 0; +} + +static const struct ins_ops load_store_ops = { + .parse = load_store__parse, + .scnprintf = load_store__scnprintf, +}; + +static const struct ins_ops *powerpc__associate_instruction_ops(struct arch *arch, const char *name) +{ + int i; + const struct ins_ops *ops; + + /* + * - Interested only if instruction starts with 'b'. + * - Few start with 'b', but aren't branch instructions. + */ + if (name[0] != 'b' || + !strncmp(name, "bcd", 3) || + !strncmp(name, "brinc", 5) || + !strncmp(name, "bper", 4)) + return NULL; + + ops = &jump_ops; + + i = strlen(name) - 1; + if (i < 0) + return NULL; + + /* ignore optional hints at the end of the instructions */ + if (name[i] == '+' || name[i] == '-') + i--; + + if (name[i] == 'l' || (name[i] == 'a' && name[i-1] == 'l')) { + /* + * if the instruction ends up with 'l' or 'la', then + * those are considered 'calls' since they update LR. + * ... except for 'bnl' which is branch if not less than + * and the absolute form of the same. + */ + if (strcmp(name, "bnl") && strcmp(name, "bnl+") && + strcmp(name, "bnl-") && strcmp(name, "bnla") && + strcmp(name, "bnla+") && strcmp(name, "bnla-")) + ops = &call_ops; + } + if (name[i] == 'r' && name[i-1] == 'l') + /* + * instructions ending with 'lr' are considered to be + * return instructions + */ + ops = &ret_ops; + + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +struct insn_offset { + const char *name; + int value; +}; + +/* + * There are memory instructions with opcode 31 which are + * of X Form, Example: + * ldx RT,RA,RB + * ______________________________________ + * | 31 | RT | RA | RB | 21 |/| + * -------------------------------------- + * 0 6 11 16 21 30 31 + * + * But all instructions with opcode 31 are not memory. + * Example: add RT,RA,RB + * + * Use bits 21 to 30 to check memory insns with 31 as opcode. + * In ins_array below, for ldx instruction: + * name => OP_31_XOP_LDX + * value => 21 + */ + +static struct insn_offset ins_array[] = { + { .name = "OP_31_XOP_LXSIWZX", .value = 12, }, + { .name = "OP_31_XOP_LWARX", .value = 20, }, + { .name = "OP_31_XOP_LDX", .value = 21, }, + { .name = "OP_31_XOP_LWZX", .value = 23, }, + { .name = "OP_31_XOP_LDUX", .value = 53, }, + { .name = "OP_31_XOP_LWZUX", .value = 55, }, + { .name = "OP_31_XOP_LXSIWAX", .value = 76, }, + { .name = "OP_31_XOP_LDARX", .value = 84, }, + { .name = "OP_31_XOP_LBZX", .value = 87, }, + { .name = "OP_31_XOP_LVX", .value = 103, }, + { .name = "OP_31_XOP_LBZUX", .value = 119, }, + { .name = "OP_31_XOP_STXSIWX", .value = 140, }, + { .name = "OP_31_XOP_STDX", .value = 149, }, + { .name = "OP_31_XOP_STWX", .value = 151, }, + { .name = "OP_31_XOP_STDUX", .value = 181, }, + { .name = "OP_31_XOP_STWUX", .value = 183, }, + { .name = "OP_31_XOP_STBX", .value = 215, }, + { .name = "OP_31_XOP_STVX", .value = 231, }, + { .name = "OP_31_XOP_STBUX", .value = 247, }, + { .name = "OP_31_XOP_LHZX", .value = 279, }, + { .name = "OP_31_XOP_LHZUX", .value = 311, }, + { .name = "OP_31_XOP_LXVDSX", .value = 332, }, + { .name = "OP_31_XOP_LWAX", .value = 341, }, + { .name = "OP_31_XOP_LHAX", .value = 343, }, + { .name = "OP_31_XOP_LWAUX", .value = 373, }, + { .name = "OP_31_XOP_LHAUX", .value = 375, }, + { .name = "OP_31_XOP_STHX", .value = 407, }, + { .name = "OP_31_XOP_STHUX", .value = 439, }, + { .name = "OP_31_XOP_LXSSPX", .value = 524, }, + { .name = "OP_31_XOP_LDBRX", .value = 532, }, + { .name = "OP_31_XOP_LSWX", .value = 533, }, + { .name = "OP_31_XOP_LWBRX", .value = 534, }, + { .name = "OP_31_XOP_LFSUX", .value = 567, }, + { .name = "OP_31_XOP_LXSDX", .value = 588, }, + { .name = "OP_31_XOP_LSWI", .value = 597, }, + { .name = "OP_31_XOP_LFDX", .value = 599, }, + { .name = "OP_31_XOP_LFDUX", .value = 631, }, + { .name = "OP_31_XOP_STXSSPX", .value = 652, }, + { .name = "OP_31_XOP_STDBRX", .value = 660, }, + { .name = "OP_31_XOP_STXWX", .value = 661, }, + { .name = "OP_31_XOP_STWBRX", .value = 662, }, + { .name = "OP_31_XOP_STFSX", .value = 663, }, + { .name = "OP_31_XOP_STFSUX", .value = 695, }, + { .name = "OP_31_XOP_STXSDX", .value = 716, }, + { .name = "OP_31_XOP_STSWI", .value = 725, }, + { .name = "OP_31_XOP_STFDX", .value = 727, }, + { .name = "OP_31_XOP_STFDUX", .value = 759, }, + { .name = "OP_31_XOP_LXVW4X", .value = 780, }, + { .name = "OP_31_XOP_LHBRX", .value = 790, }, + { .name = "OP_31_XOP_LXVD2X", .value = 844, }, + { .name = "OP_31_XOP_LFIWAX", .value = 855, }, + { .name = "OP_31_XOP_LFIWZX", .value = 887, }, + { .name = "OP_31_XOP_STXVW4X", .value = 908, }, + { .name = "OP_31_XOP_STHBRX", .value = 918, }, + { .name = "OP_31_XOP_STXVD2X", .value = 972, }, + { .name = "OP_31_XOP_STFIWX", .value = 983, }, +}; + +/* + * Arithmetic instructions which are having opcode as 31. + * These instructions are tracked to save the register state + * changes. Example: + * + * lwz r10,264(r3) + * add r31, r3, r3 + * lwz r9, 0(r31) + * + * Here instruction tracking needs to identify the "add" + * instruction and save data type of r3 to r31. If a sample + * is hit at next "lwz r9, 0(r31)", by this instruction tracking, + * data type of r31 can be resolved. + */ +static struct insn_offset arithmetic_ins_op_31[] = { + { .name = "SUB_CARRY_XO_FORM", .value = 8, }, + { .name = "MUL_HDW_XO_FORM1", .value = 9, }, + { .name = "ADD_CARRY_XO_FORM", .value = 10, }, + { .name = "MUL_HW_XO_FORM1", .value = 11, }, + { .name = "SUB_XO_FORM", .value = 40, }, + { .name = "MUL_HDW_XO_FORM", .value = 73, }, + { .name = "MUL_HW_XO_FORM", .value = 75, }, + { .name = "SUB_EXT_XO_FORM", .value = 136, }, + { .name = "ADD_EXT_XO_FORM", .value = 138, }, + { .name = "SUB_ZERO_EXT_XO_FORM", .value = 200, }, + { .name = "ADD_ZERO_EXT_XO_FORM", .value = 202, }, + { .name = "SUB_EXT_XO_FORM2", .value = 232, }, + { .name = "MUL_DW_XO_FORM", .value = 233, }, + { .name = "ADD_EXT_XO_FORM2", .value = 234, }, + { .name = "MUL_W_XO_FORM", .value = 235, }, + { .name = "ADD_XO_FORM", .value = 266, }, + { .name = "DIV_DW_XO_FORM1", .value = 457, }, + { .name = "DIV_W_XO_FORM1", .value = 459, }, + { .name = "DIV_DW_XO_FORM", .value = 489, }, + { .name = "DIV_W_XO_FORM", .value = 491, }, +}; + +static struct insn_offset arithmetic_two_ops[] = { + { .name = "mulli", .value = 7, }, + { .name = "subfic", .value = 8, }, + { .name = "addic", .value = 12, }, + { .name = "addic.", .value = 13, }, + { .name = "addi", .value = 14, }, + { .name = "addis", .value = 15, }, +}; + +static int cmp_offset(const void *a, const void *b) +{ + const struct insn_offset *val1 = a; + const struct insn_offset *val2 = b; + + return (val1->value - val2->value); +} + +const struct ins_ops *check_ppc_insn(struct disasm_line *dl) +{ + int raw_insn = dl->raw.raw_insn; + int opcode = PPC_OP(raw_insn); + int mem_insn_31 = PPC_21_30(raw_insn); + struct insn_offset *ret; + struct insn_offset mem_insns_31_opcode = { + "OP_31_INSN", + mem_insn_31 + }; + char name_insn[32]; + + /* + * Instructions with opcode 32 to 63 are memory + * instructions in powerpc + */ + if ((opcode & 0x20)) { + /* + * Set name in case of raw instruction to + * opcode to be used in insn-stat + */ + if (!strlen(dl->ins.name)) { + sprintf(name_insn, "%d", opcode); + dl->ins.name = strdup(name_insn); + } + return &load_store_ops; + } else if (opcode == 31) { + /* Check for memory instructions with opcode 31 */ + ret = bsearch(&mem_insns_31_opcode, ins_array, ARRAY_SIZE(ins_array), sizeof(ins_array[0]), cmp_offset); + if (ret) { + if (!strlen(dl->ins.name)) + dl->ins.name = strdup(ret->name); + return &load_store_ops; + } else { + mem_insns_31_opcode.value = PPC_22_30(raw_insn); + ret = bsearch(&mem_insns_31_opcode, arithmetic_ins_op_31, ARRAY_SIZE(arithmetic_ins_op_31), + sizeof(arithmetic_ins_op_31[0]), cmp_offset); + if (ret != NULL) + return &arithmetic_ops; + /* Bits 21 to 30 has value 444 for "mr" insn ie, OR X form */ + if (PPC_21_30(raw_insn) == 444) + return &arithmetic_ops; + } + } else { + mem_insns_31_opcode.value = opcode; + ret = bsearch(&mem_insns_31_opcode, arithmetic_two_ops, ARRAY_SIZE(arithmetic_two_ops), + sizeof(arithmetic_two_ops[0]), cmp_offset); + if (ret != NULL) + return &arithmetic_ops; + } + + return NULL; +} + +/* + * Instruction tracking function to track register state moves. + * Example sequence: + * ld r10,264(r3) + * mr r31,r3 + * < + * ld r9,312(r31) + * + * Previous instruction sequence shows that register state of r3 + * is moved to r31. update_insn_state_powerpc tracks these state + * changes + */ +#ifdef HAVE_LIBDW_SUPPORT +static void update_insn_state_powerpc(struct type_state *state, + struct data_loc_info *dloc, Dwarf_Die * cu_die __maybe_unused, + struct disasm_line *dl) +{ + struct annotated_insn_loc loc; + struct annotated_op_loc *src = &loc.ops[INSN_OP_SOURCE]; + struct annotated_op_loc *dst = &loc.ops[INSN_OP_TARGET]; + struct type_state_reg *tsr; + u32 insn_offset = dl->al.offset; + + if (annotate_get_insn_location(dloc->arch, dl, &loc) < 0) + return; + + /* + * Value 444 for bits 21:30 is for "mr" + * instruction. "mr" is extended OR. So set the + * source and destination reg correctly + */ + if (PPC_21_30(dl->raw.raw_insn) == 444) { + int src_reg = src->reg1; + + src->reg1 = dst->reg1; + dst->reg1 = src_reg; + } + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + + if (!has_reg_type(state, src->reg1) || + !state->regs[src->reg1].ok) { + tsr->ok = false; + return; + } + + tsr->type = state->regs[src->reg1].type; + tsr->kind = state->regs[src->reg1].kind; + tsr->ok = true; + + pr_debug_dtp("mov [%x] reg%d -> reg%d", + insn_offset, src->reg1, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); +} +#endif /* HAVE_LIBDW_SUPPORT */ + +const struct arch *arch__new_powerpc(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "powerpc"; + arch->id = *id; + arch->objdump.comment_char = '#'; + annotate_opts.show_asm_raw = true; + arch->associate_instruction_ops = powerpc__associate_instruction_ops; +#ifdef HAVE_LIBDW_SUPPORT + arch->update_insn_state = update_insn_state_powerpc; +#endif + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-riscv64.c b/tools/perf/util/annotate-arch/annotate-riscv64.c new file mode 100644 index 000000000..29a988fca --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-riscv64.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../disasm.h" + +static +const struct ins_ops *riscv64__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strncmp(name, "jal", 3) || + !strncmp(name, "jr", 2) || + !strncmp(name, "call", 4)) + ops = &call_ops; + else if (!strncmp(name, "ret", 3)) + ops = &ret_ops; + else if (name[0] == 'j' || name[0] == 'b') + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_riscv64(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "riscv"; + arch->id = *id; + arch->objdump.comment_char = '#'; + arch->associate_instruction_ops = riscv64__associate_ins_ops; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-s390.c b/tools/perf/util/annotate-arch/annotate-s390.c new file mode 100644 index 000000000..af9cabd0a --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-s390.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../debug.h" +#include "../disasm.h" +#include "../map.h" +#include "../maps.h" +#include "../symbol.h" +#include "../thread.h" +#include "../annotate.h" +#include "../annotate-data.h" + +static int s390_call__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + char *endptr, *tok, *name; + struct map *map = ms->map; + struct addr_map_symbol target; + + tok = strchr(ops->raw, ','); + if (!tok) + return -1; + + ops->target.addr = strtoull(tok + 1, &endptr, 16); + + name = strchr(endptr, '<'); + if (name == NULL) + return -1; + + name++; + + if (arch->objdump.skip_functions_char && + strchr(name, arch->objdump.skip_functions_char)) + return -1; + + tok = strchr(name, '>'); + if (tok == NULL) + return -1; + + *tok = '\0'; + ops->target.name = strdup(name); + *tok = '>'; + + if (ops->target.name == NULL) + return -1; + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + return 0; +} + +static const struct ins_ops s390_call_ops = { + .parse = s390_call__parse, + .scnprintf = call__scnprintf, + .is_call = true, +}; + +static int s390_mov__parse(const struct arch *arch __maybe_unused, + struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *s = strchr(ops->raw, ','), *target, *endptr; + + if (s == NULL) + return -1; + + *s = '\0'; + ops->source.raw = strdup(ops->raw); + *s = ','; + + if (ops->source.raw == NULL) + return -1; + + target = ++s; + ops->target.raw = strdup(target); + if (ops->target.raw == NULL) + goto out_free_source; + + ops->target.addr = strtoull(target, &endptr, 16); + if (endptr == target) + goto out_free_target; + + s = strchr(endptr, '<'); + if (s == NULL) + goto out_free_target; + endptr = strchr(s + 1, '>'); + if (endptr == NULL) + goto out_free_target; + + *endptr = '\0'; + ops->target.name = strdup(s + 1); + *endptr = '>'; + if (ops->target.name == NULL) + goto out_free_target; + + return 0; + +out_free_target: + zfree(&ops->target.raw); +out_free_source: + zfree(&ops->source.raw); + return -1; +} + + +static const struct ins_ops s390_mov_ops = { + .parse = s390_mov__parse, + .scnprintf = mov__scnprintf, +}; + +static const struct ins_ops *s390__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + /* catch all kind of jumps */ + if (strchr(name, 'j') || + !strncmp(name, "bct", 3) || + !strncmp(name, "br", 2)) + ops = &jump_ops; + /* override call/returns */ + if (!strcmp(name, "bras") || + !strcmp(name, "brasl") || + !strcmp(name, "basr")) + ops = &s390_call_ops; + if (!strcmp(name, "br")) + ops = &ret_ops; + /* override load/store relative to PC */ + if (!strcmp(name, "lrl") || + !strcmp(name, "lgrl") || + !strcmp(name, "lgfrl") || + !strcmp(name, "llgfrl") || + !strcmp(name, "strl") || + !strcmp(name, "stgrl")) + ops = &s390_mov_ops; + + if (ops) + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +static int s390__cpuid_parse(struct arch *arch, const char *cpuid) +{ + unsigned int family; + char model[16], model_c[16], cpumf_v[16], cpumf_a[16]; + int ret; + + /* + * cpuid string format: + * "IBM,family,model-capacity,model[,cpum_cf-version,cpum_cf-authorization]" + */ + ret = sscanf(cpuid, "%*[^,],%u,%[^,],%[^,],%[^,],%s", &family, model_c, + model, cpumf_v, cpumf_a); + if (ret >= 2) { + arch->family = family; + arch->model = 0; + return 0; + } + + return -1; +} + +const struct arch *arch__new_s390(const struct e_machine_and_e_flags *id, const char *cpuid) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "s390"; + arch->id = *id; + arch->associate_instruction_ops = s390__associate_ins_ops; + if (cpuid) { + if (s390__cpuid_parse(arch, cpuid)) { + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING; + return NULL; + } + } + arch->objdump.comment_char = '#'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-sparc.c b/tools/perf/util/annotate-arch/annotate-sparc.c new file mode 100644 index 000000000..2f07bc7a5 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-sparc.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../../util/disasm.h" + +static int is_branch_cond(const char *cond) +{ + if (cond[0] == '\0') + return 1; + + if (cond[0] == 'a' && cond[1] == '\0') + return 1; + + if (cond[0] == 'c' && + (cond[1] == 'c' || cond[1] == 's') && + cond[2] == '\0') + return 1; + + if (cond[0] == 'e' && + (cond[1] == '\0' || + (cond[1] == 'q' && cond[2] == '\0'))) + return 1; + + if (cond[0] == 'g' && + (cond[1] == '\0' || + (cond[1] == 't' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == 'u' && cond[3] == '\0'))) + return 1; + + if (cond[0] == 'l' && + (cond[1] == '\0' || + (cond[1] == 't' && cond[2] == '\0') || + (cond[1] == 'u' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == 'u' && cond[3] == '\0'))) + return 1; + + if (cond[0] == 'n' && + (cond[1] == '\0' || + (cond[1] == 'e' && cond[2] == '\0') || + (cond[1] == 'z' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == 'g' && cond[3] == '\0'))) + return 1; + + if (cond[0] == 'b' && + cond[1] == 'p' && + cond[2] == 'o' && + cond[3] == 's' && + cond[4] == '\0') + return 1; + + if (cond[0] == 'v' && + (cond[1] == 'c' || cond[1] == 's') && + cond[2] == '\0') + return 1; + + if (cond[0] == 'b' && + cond[1] == 'z' && + cond[2] == '\0') + return 1; + + return 0; +} + +static int is_branch_reg_cond(const char *cond) +{ + if ((cond[0] == 'n' || cond[0] == 'l') && + cond[1] == 'z' && + cond[2] == '\0') + return 1; + + if (cond[0] == 'z' && + cond[1] == '\0') + return 1; + + if ((cond[0] == 'g' || cond[0] == 'l') && + cond[1] == 'e' && + cond[2] == 'z' && + cond[3] == '\0') + return 1; + + if (cond[0] == 'g' && + cond[1] == 'z' && + cond[2] == '\0') + return 1; + + return 0; +} + +static int is_branch_float_cond(const char *cond) +{ + if (cond[0] == '\0') + return 1; + + if ((cond[0] == 'a' || cond[0] == 'e' || + cond[0] == 'z' || cond[0] == 'g' || + cond[0] == 'l' || cond[0] == 'n' || + cond[0] == 'o' || cond[0] == 'u') && + cond[1] == '\0') + return 1; + + if (((cond[0] == 'g' && cond[1] == 'e') || + (cond[0] == 'l' && (cond[1] == 'e' || + cond[1] == 'g')) || + (cond[0] == 'n' && (cond[1] == 'e' || + cond[1] == 'z')) || + (cond[0] == 'u' && (cond[1] == 'e' || + cond[1] == 'g' || + cond[1] == 'l'))) && + cond[2] == '\0') + return 1; + + if (cond[0] == 'u' && + (cond[1] == 'g' || cond[1] == 'l') && + cond[2] == 'e' && + cond[3] == '\0') + return 1; + + return 0; +} + +static const struct ins_ops *sparc__associate_instruction_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strcmp(name, "call") || + !strcmp(name, "jmp") || + !strcmp(name, "jmpl")) { + ops = &call_ops; + } else if (!strcmp(name, "ret") || + !strcmp(name, "retl") || + !strcmp(name, "return")) { + ops = &ret_ops; + } else if (!strcmp(name, "mov")) { + ops = &mov_ops; + } else { + if (name[0] == 'c' && + (name[1] == 'w' || name[1] == 'x')) + name += 2; + + if (name[0] == 'b') { + const char *cond = name + 1; + + if (cond[0] == 'r') { + if (is_branch_reg_cond(cond + 1)) + ops = &jump_ops; + } else if (is_branch_cond(cond)) { + ops = &jump_ops; + } + } else if (name[0] == 'f' && name[1] == 'b') { + if (is_branch_float_cond(name + 2)) + ops = &jump_ops; + } + } + + if (ops) + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_sparc(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "sparc"; + arch->id = *id; + arch->associate_instruction_ops = sparc__associate_instruction_ops; + arch->objdump.comment_char = '#'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-x86.c b/tools/perf/util/annotate-arch/annotate-x86.c new file mode 100644 index 000000000..7e6136536 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-x86.c @@ -0,0 +1,852 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "../annotate-data.h" +#include "../debug.h" +#include "../disasm.h" +#include "../dso.h" +#include "../map.h" +#include "../string2.h" // strstarts +#include "../symbol.h" + +/* + * x86 instruction nmemonic table to parse disasm lines for annotate. + * This table is searched twice - one for exact match and another for + * match without a size suffix (b, w, l, q) in case of AT&T syntax. + * + * So this table should not have entries with the suffix unless it's + * a complete different instruction than ones without the suffix. + */ +static const struct ins x86__instructions[] = { + { .name = "adc", .ops = &mov_ops, }, + { .name = "add", .ops = &mov_ops, }, + { .name = "addsd", .ops = &mov_ops, }, + { .name = "and", .ops = &mov_ops, }, + { .name = "andpd", .ops = &mov_ops, }, + { .name = "andps", .ops = &mov_ops, }, + { .name = "bsr", .ops = &mov_ops, }, + { .name = "bt", .ops = &mov_ops, }, + { .name = "btr", .ops = &mov_ops, }, + { .name = "bts", .ops = &mov_ops, }, + { .name = "call", .ops = &call_ops, }, + { .name = "cmovae", .ops = &mov_ops, }, + { .name = "cmovbe", .ops = &mov_ops, }, + { .name = "cmove", .ops = &mov_ops, }, + { .name = "cmp", .ops = &mov_ops, }, + { .name = "cmpxch", .ops = &mov_ops, }, + { .name = "cmpxchg", .ops = &mov_ops, }, + { .name = "cs", .ops = &mov_ops, }, + { .name = "dec", .ops = &dec_ops, }, + { .name = "divsd", .ops = &mov_ops, }, + { .name = "divss", .ops = &mov_ops, }, + { .name = "gs", .ops = &mov_ops, }, + { .name = "imul", .ops = &mov_ops, }, + { .name = "inc", .ops = &dec_ops, }, + { .name = "ja", .ops = &jump_ops, }, + { .name = "jae", .ops = &jump_ops, }, + { .name = "jb", .ops = &jump_ops, }, + { .name = "jbe", .ops = &jump_ops, }, + { .name = "jc", .ops = &jump_ops, }, + { .name = "jcxz", .ops = &jump_ops, }, + { .name = "je", .ops = &jump_ops, }, + { .name = "jecxz", .ops = &jump_ops, }, + { .name = "jg", .ops = &jump_ops, }, + { .name = "jge", .ops = &jump_ops, }, + { .name = "jl", .ops = &jump_ops, }, + { .name = "jle", .ops = &jump_ops, }, + { .name = "jmp", .ops = &jump_ops, }, + { .name = "jna", .ops = &jump_ops, }, + { .name = "jnae", .ops = &jump_ops, }, + { .name = "jnb", .ops = &jump_ops, }, + { .name = "jnbe", .ops = &jump_ops, }, + { .name = "jnc", .ops = &jump_ops, }, + { .name = "jne", .ops = &jump_ops, }, + { .name = "jng", .ops = &jump_ops, }, + { .name = "jnge", .ops = &jump_ops, }, + { .name = "jnl", .ops = &jump_ops, }, + { .name = "jnle", .ops = &jump_ops, }, + { .name = "jno", .ops = &jump_ops, }, + { .name = "jnp", .ops = &jump_ops, }, + { .name = "jns", .ops = &jump_ops, }, + { .name = "jnz", .ops = &jump_ops, }, + { .name = "jo", .ops = &jump_ops, }, + { .name = "jp", .ops = &jump_ops, }, + { .name = "jpe", .ops = &jump_ops, }, + { .name = "jpo", .ops = &jump_ops, }, + { .name = "jrcxz", .ops = &jump_ops, }, + { .name = "js", .ops = &jump_ops, }, + { .name = "jz", .ops = &jump_ops, }, + { .name = "lea", .ops = &mov_ops, }, + { .name = "lock", .ops = &lock_ops, }, + { .name = "mov", .ops = &mov_ops, }, + { .name = "movapd", .ops = &mov_ops, }, + { .name = "movaps", .ops = &mov_ops, }, + { .name = "movdqa", .ops = &mov_ops, }, + { .name = "movdqu", .ops = &mov_ops, }, + { .name = "movsb", .ops = &mov_ops, }, + { .name = "movsd", .ops = &mov_ops, }, + { .name = "movsl", .ops = &mov_ops, }, + { .name = "movss", .ops = &mov_ops, }, + { .name = "movsw", .ops = &mov_ops, }, + { .name = "movupd", .ops = &mov_ops, }, + { .name = "movups", .ops = &mov_ops, }, + { .name = "movzb", .ops = &mov_ops, }, + { .name = "movzl", .ops = &mov_ops, }, + { .name = "movzw", .ops = &mov_ops, }, + { .name = "mulsd", .ops = &mov_ops, }, + { .name = "mulss", .ops = &mov_ops, }, + { .name = "nop", .ops = &nop_ops, }, + { .name = "or", .ops = &mov_ops, }, + { .name = "orps", .ops = &mov_ops, }, + { .name = "paddq", .ops = &mov_ops, }, + { .name = "pand", .ops = &mov_ops, }, + { .name = "pcmpeqb", .ops = &mov_ops, }, + { .name = "por", .ops = &mov_ops, }, + { .name = "rcl", .ops = &mov_ops, }, + { .name = "ret", .ops = &ret_ops, }, + { .name = "sbb", .ops = &mov_ops, }, + { .name = "sete", .ops = &mov_ops, }, + { .name = "sub", .ops = &mov_ops, }, + { .name = "subsd", .ops = &mov_ops, }, + { .name = "test", .ops = &mov_ops, }, + { .name = "tzcnt", .ops = &mov_ops, }, + { .name = "ucomisd", .ops = &mov_ops, }, + { .name = "ucomiss", .ops = &mov_ops, }, + { .name = "vaddsd", .ops = &mov_ops, }, + { .name = "vandpd", .ops = &mov_ops, }, + { .name = "vmovdqa", .ops = &mov_ops, }, + { .name = "vmovq", .ops = &mov_ops, }, + { .name = "vmovsd", .ops = &mov_ops, }, + { .name = "vmulsd", .ops = &mov_ops, }, + { .name = "vorpd", .ops = &mov_ops, }, + { .name = "vsubsd", .ops = &mov_ops, }, + { .name = "vucomisd", .ops = &mov_ops, }, + { .name = "xadd", .ops = &mov_ops, }, + { .name = "xbegin", .ops = &jump_ops, }, + { .name = "xchg", .ops = &mov_ops, }, + { .name = "xor", .ops = &mov_ops, }, + { .name = "xorpd", .ops = &mov_ops, }, + { .name = "xorps", .ops = &mov_ops, }, +}; + +static bool amd__ins_is_fused(const struct arch *arch, const char *ins1, + const char *ins2) +{ + if (strstr(ins2, "jmp")) + return false; + + /* Family >= 15h supports cmp/test + branch fusion */ + if (arch->family >= 0x15 && (strstarts(ins1, "test") || + (strstarts(ins1, "cmp") && !strstr(ins1, "xchg")))) { + return true; + } + + /* Family >= 19h supports some ALU + branch fusion */ + if (arch->family >= 0x19 && (strstarts(ins1, "add") || + strstarts(ins1, "sub") || strstarts(ins1, "and") || + strstarts(ins1, "inc") || strstarts(ins1, "dec") || + strstarts(ins1, "or") || strstarts(ins1, "xor"))) { + return true; + } + + return false; +} + +static bool intel__ins_is_fused(const struct arch *arch, const char *ins1, + const char *ins2) +{ + if (arch->family != 6 || arch->model < 0x1e || strstr(ins2, "jmp")) + return false; + + if (arch->model == 0x1e) { + /* Nehalem */ + if ((strstr(ins1, "cmp") && !strstr(ins1, "xchg")) || + strstr(ins1, "test")) { + return true; + } + } else { + /* Newer platform */ + if ((strstr(ins1, "cmp") && !strstr(ins1, "xchg")) || + strstr(ins1, "test") || + strstr(ins1, "add") || + strstr(ins1, "sub") || + strstr(ins1, "and") || + strstr(ins1, "inc") || + strstr(ins1, "dec")) { + return true; + } + } + + return false; +} + +static int x86__cpuid_parse(struct arch *arch, const char *cpuid) +{ + unsigned int family, model, stepping; + int ret; + + /* + * cpuid = "GenuineIntel,family,model,stepping" + */ + ret = sscanf(cpuid, "%*[^,],%u,%u,%u", &family, &model, &stepping); + if (ret == 3) { + arch->family = family; + arch->model = model; + arch->ins_is_fused = strstarts(cpuid, "AuthenticAMD") ? + amd__ins_is_fused : + intel__ins_is_fused; + return 0; + } + + return -1; +} + +#ifdef HAVE_LIBDW_SUPPORT +static void invalidate_reg_state(struct type_state_reg *reg) +{ + reg->kind = TSR_KIND_INVALID; + reg->ok = false; + reg->lifetime_active = false; + reg->lifetime_end = 0; + reg->copied_from = -1; +} + +static void update_insn_state_x86(struct type_state *state, + struct data_loc_info *dloc, Dwarf_Die *cu_die, + struct disasm_line *dl) +{ + struct annotated_insn_loc loc; + struct annotated_op_loc *src = &loc.ops[INSN_OP_SOURCE]; + struct annotated_op_loc *dst = &loc.ops[INSN_OP_TARGET]; + struct type_state_reg *tsr; + Dwarf_Die type_die; + u32 insn_offset = dl->al.offset; + int fbreg = dloc->fbreg; + int fboff = 0; + + if (annotate_get_insn_location(dloc->arch, dl, &loc) < 0) + return; + + if (ins__is_call(&dl->ins)) { + struct symbol *func = dl->ops.target.sym; + const char *call_name; + u64 call_addr; + + /* Try to resolve the call target name */ + if (func) + call_name = func->name; + else + call_name = dl->ops.target.name; + + /* __fentry__ will preserve all registers */ + if (call_name && !strcmp(call_name, "__fentry__")) + return; + + if (call_name) + pr_debug_dtp("call [%x] %s\n", insn_offset, call_name); + else + pr_debug_dtp("call [%x] \n", insn_offset); + + /* Invalidate caller-saved registers after call */ + call_addr = map__rip_2objdump(dloc->ms->map, + dloc->ms->sym->start + dl->al.offset); + for (unsigned i = 0; i < ARRAY_SIZE(state->regs); i++) { + struct type_state_reg *reg = &state->regs[i]; + + if (!reg->caller_saved) + continue; + /* Keep register valid within DWARF location lifetime */ + if (reg->lifetime_active && call_addr < reg->lifetime_end) + continue; + invalidate_reg_state(reg); + } + + /* Update register with the return type (if any) */ + if (call_name && die_find_func_rettype(cu_die, call_name, &type_die)) { + tsr = &state->regs[state->ret_reg]; + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("call [%x] return -> reg%d", + insn_offset, state->ret_reg); + pr_debug_type_name(&type_die, tsr->kind); + } + return; + } + + if (!strncmp(dl->ins.name, "add", 3)) { + u64 imm_value = -1ULL; + int offset; + const char *var_name = NULL; + struct map_symbol *ms = dloc->ms; + u64 ip = ms->sym->start + dl->al.offset; + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + tsr->lifetime_active = false; + tsr->lifetime_end = 0; + + if (src->imm) + imm_value = src->offset; + else if (has_reg_type(state, src->reg1) && + state->regs[src->reg1].kind == TSR_KIND_CONST) + imm_value = state->regs[src->reg1].imm_value; + else if (src->reg1 == DWARF_REG_PC) { + u64 var_addr = annotate_calc_pcrel(dloc->ms, ip, + src->offset, dl); + + if (get_global_var_info(dloc, var_addr, + &var_name, &offset) && + !strcmp(var_name, "this_cpu_off") && + tsr->kind == TSR_KIND_CONST) { + tsr->kind = TSR_KIND_PERCPU_BASE; + tsr->offset = 0; + tsr->ok = true; + imm_value = tsr->imm_value; + } + } + else + return; + + /* Ignore add to non-pointer or non-const types */ + if (tsr->kind == TSR_KIND_POINTER || + (dwarf_tag(&tsr->type) == DW_TAG_pointer_type && + src->reg1 != DWARF_REG_PC && tsr->kind == TSR_KIND_TYPE && !dst->mem_ref)) { + tsr->offset += imm_value; + pr_debug_dtp("add [%x] offset %#"PRIx64" to reg%d", + insn_offset, imm_value, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + + if (tsr->kind == TSR_KIND_CONST) + tsr->imm_value += imm_value; + + if (tsr->kind != TSR_KIND_PERCPU_BASE) + return; + + if (get_global_var_type(cu_die, dloc, ip, imm_value, &offset, + &type_die) && offset == 0) { + /* + * This is not a pointer type, but it should be treated + * as a pointer. + */ + tsr->type = type_die; + tsr->kind = TSR_KIND_PERCPU_POINTER; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("add [%x] percpu %#"PRIx64" -> reg%d", + insn_offset, imm_value, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + return; + } + + if (!strncmp(dl->ins.name, "sub", 3)) { + u64 imm_value = -1ULL; + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + tsr->lifetime_active = false; + tsr->lifetime_end = 0; + + if (src->imm) + imm_value = src->offset; + else if (has_reg_type(state, src->reg1) && + state->regs[src->reg1].kind == TSR_KIND_CONST) + imm_value = state->regs[src->reg1].imm_value; + + if (tsr->kind == TSR_KIND_POINTER || + (dwarf_tag(&tsr->type) == DW_TAG_pointer_type && + src->reg1 != DWARF_REG_PC && tsr->kind == TSR_KIND_TYPE && !dst->mem_ref)) { + tsr->offset -= imm_value; + pr_debug_dtp("sub [%x] offset %#"PRIx64" to reg%d", + insn_offset, imm_value, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + + if (tsr->kind == TSR_KIND_CONST) + tsr->imm_value -= imm_value; + + return; + } + + if (!strncmp(dl->ins.name, "lea", 3)) { + int sreg = src->reg1; + struct type_state_reg src_tsr; + + if (!has_reg_type(state, sreg) || + !has_reg_type(state, dst->reg1) || + !src->mem_ref) + return; + + src_tsr = state->regs[sreg]; + tsr = &state->regs[dst->reg1]; + + invalidate_reg_state(tsr); + + /* Case 1: Based on stack pointer or frame pointer */ + if (sreg == fbreg || sreg == state->stack_reg) { + struct type_state_stack *stack; + int offset = src->offset - fboff; + + stack = find_stack_state(state, offset); + if (!stack) + return; + + tsr->type = stack->type; + tsr->kind = TSR_KIND_POINTER; + tsr->offset = offset - stack->offset; + tsr->ok = true; + + if (sreg == fbreg) { + pr_debug_dtp("lea [%x] address of -%#x(stack) -> reg%d", + insn_offset, -src->offset, dst->reg1); + } else { + pr_debug_dtp("lea [%x] address of %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + } + + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Case 2: Based on a register holding a typed pointer */ + else if (src_tsr.ok && (src_tsr.kind == TSR_KIND_POINTER || + (dwarf_tag(&src_tsr.type) == DW_TAG_pointer_type && + src_tsr.kind == TSR_KIND_TYPE))) { + + if (src_tsr.kind == TSR_KIND_TYPE && + __die_get_real_type(&state->regs[sreg].type, &type_die) == NULL) + return; + + if (src_tsr.kind == TSR_KIND_POINTER) + type_die = state->regs[sreg].type; + + /* Check if the target type has a member at the new offset */ + if (die_get_member_type(&type_die, + src->offset + src_tsr.offset, &type_die) == NULL) + return; + + tsr->type = src_tsr.type; + tsr->kind = src_tsr.kind; + tsr->offset = src->offset + src_tsr.offset; + tsr->ok = true; + + pr_debug_dtp("lea [%x] address of %s%#x(reg%d) -> reg%d", + insn_offset, src->offset < 0 ? "-" : "", + abs(src->offset), sreg, dst->reg1); + + pr_debug_type_name(&tsr->type, tsr->kind); + } + return; + } + + /* Invalidate register states for other ops which may change pointers */ + if (has_reg_type(state, dst->reg1) && !dst->mem_ref && + dwarf_tag(&state->regs[dst->reg1].type) == DW_TAG_pointer_type) { + if (!strncmp(dl->ins.name, "imul", 4) || !strncmp(dl->ins.name, "mul", 3) || + !strncmp(dl->ins.name, "idiv", 4) || !strncmp(dl->ins.name, "div", 3) || + !strncmp(dl->ins.name, "shl", 3) || !strncmp(dl->ins.name, "shr", 3) || + !strncmp(dl->ins.name, "sar", 3) || !strncmp(dl->ins.name, "and", 3) || + !strncmp(dl->ins.name, "or", 2) || !strncmp(dl->ins.name, "neg", 3) || + !strncmp(dl->ins.name, "inc", 3) || !strncmp(dl->ins.name, "dec", 3)) { + pr_debug_dtp("%s [%x] invalidate reg%d\n", + dl->ins.name, insn_offset, dst->reg1); + invalidate_reg_state(&state->regs[dst->reg1]); + return; + } + + if (!strncmp(dl->ins.name, "xor", 3) && dst->reg1 == src->reg1) { + /* xor reg, reg clears the register */ + pr_debug_dtp("xor [%x] clear reg%d\n", + insn_offset, dst->reg1); + + state->regs[dst->reg1].kind = TSR_KIND_CONST; + state->regs[dst->reg1].imm_value = 0; + state->regs[dst->reg1].ok = true; + state->regs[dst->reg1].lifetime_active = false; + state->regs[dst->reg1].lifetime_end = 0; + state->regs[dst->reg1].copied_from = -1; + return; + } + } + + if (strncmp(dl->ins.name, "mov", 3)) + return; + + if (dloc->fb_cfa) { + u64 ip = dloc->ms->sym->start + dl->al.offset; + u64 pc = map__rip_2objdump(dloc->ms->map, ip); + + if (die_get_cfa(dloc->di->dbg, pc, &fbreg, &fboff) < 0) + fbreg = -1; + } + + /* Case 1. register to register or segment:offset to register transfers */ + if (!src->mem_ref && !dst->mem_ref) { + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + + if (dso__kernel(map__dso(dloc->ms->map)) && + src->segment == INSN_SEG_X86_GS && src->imm) { + u64 ip = dloc->ms->sym->start + dl->al.offset; + u64 var_addr; + int offset; + + /* + * In kernel, %gs points to a per-cpu region for the + * current CPU. Access with a constant offset should + * be treated as a global variable access. + */ + var_addr = src->offset; + + if (var_addr == 40) { + tsr->kind = TSR_KIND_CANARY; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] stack canary -> reg%d\n", + insn_offset, dst->reg1); + return; + } + + if (!get_global_var_type(cu_die, dloc, ip, var_addr, + &offset, &type_die) || + !die_get_member_type(&type_die, offset, &type_die)) { + invalidate_reg_state(tsr); + return; + } + + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] this-cpu addr=%#"PRIx64" -> reg%d", + insn_offset, var_addr, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + return; + } + + if (src->imm) { + tsr->kind = TSR_KIND_CONST; + tsr->imm_value = src->offset; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] imm=%#x -> reg%d\n", + insn_offset, tsr->imm_value, dst->reg1); + return; + } + + if (!has_reg_type(state, src->reg1) || + !state->regs[src->reg1].ok) { + invalidate_reg_state(tsr); + return; + } + + tsr->type = state->regs[src->reg1].type; + tsr->kind = state->regs[src->reg1].kind; + tsr->imm_value = state->regs[src->reg1].imm_value; + tsr->offset = state->regs[src->reg1].offset; + tsr->lifetime_active = state->regs[src->reg1].lifetime_active; + tsr->lifetime_end = state->regs[src->reg1].lifetime_end; + tsr->ok = true; + + /* To copy back the variable type later (hopefully) */ + if (tsr->kind == TSR_KIND_TYPE || tsr->kind == TSR_KIND_POINTER) + tsr->copied_from = src->reg1; + + pr_debug_dtp("mov [%x] reg%d -> reg%d", + insn_offset, src->reg1, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Case 2. memory to register transers */ + if (src->mem_ref && !dst->mem_ref) { + int sreg = src->reg1; + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + +retry: + /* Check stack variables with offset */ + if (sreg == fbreg || sreg == state->stack_reg) { + struct type_state_stack *stack; + int offset = src->offset - fboff; + + stack = find_stack_state(state, offset); + if (stack == NULL) { + invalidate_reg_state(tsr); + return; + } else if (!stack->compound) { + tsr->type = stack->type; + tsr->kind = stack->kind; + tsr->offset = stack->ptr_offset; + tsr->ok = true; + } else if (die_get_member_type(&stack->type, + offset - stack->offset, + &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + } else { + invalidate_reg_state(tsr); + return; + } + + if (sreg == fbreg) { + pr_debug_dtp("mov [%x] -%#x(stack) -> reg%d", + insn_offset, -offset, dst->reg1); + } else { + pr_debug_dtp("mov [%x] %#x(reg%d) -> reg%d", + insn_offset, offset, sreg, dst->reg1); + } + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* And then dereference the pointer if it has one */ + else if (has_reg_type(state, sreg) && state->regs[sreg].ok && + state->regs[sreg].kind == TSR_KIND_TYPE && + die_deref_ptr_type(&state->regs[sreg].type, + src->offset + state->regs[sreg].offset, &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Handle dereference of TSR_KIND_POINTER registers */ + else if (has_reg_type(state, sreg) && state->regs[sreg].ok && + state->regs[sreg].kind == TSR_KIND_POINTER && + die_get_member_type(&state->regs[sreg].type, + src->offset + state->regs[sreg].offset, &type_die)) { + tsr->type = state->regs[sreg].type; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = src->offset + state->regs[sreg].offset; + tsr->ok = true; + + pr_debug_dtp("mov [%x] addr %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Or check if it's a global variable */ + else if (sreg == DWARF_REG_PC) { + struct map_symbol *ms = dloc->ms; + u64 ip = ms->sym->start + dl->al.offset; + u64 addr; + int offset; + + addr = annotate_calc_pcrel(ms, ip, src->offset, dl); + + if (!get_global_var_type(cu_die, dloc, ip, addr, &offset, + &type_die) || + !die_get_member_type(&type_die, offset, &type_die)) { + invalidate_reg_state(tsr); + return; + } + + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] global addr=%"PRIx64" -> reg%d", + insn_offset, addr, dst->reg1); + pr_debug_type_name(&type_die, tsr->kind); + } + /* And check percpu access with base register */ + else if (has_reg_type(state, sreg) && + state->regs[sreg].kind == TSR_KIND_PERCPU_BASE) { + u64 ip = dloc->ms->sym->start + dl->al.offset; + u64 var_addr = src->offset; + int offset; + + if (src->multi_regs) { + int reg2 = (sreg == src->reg1) ? src->reg2 : src->reg1; + + if (has_reg_type(state, reg2) && state->regs[reg2].ok && + state->regs[reg2].kind == TSR_KIND_CONST) + var_addr += state->regs[reg2].imm_value; + } + + /* + * In kernel, %gs points to a per-cpu region for the + * current CPU. Access with a constant offset should + * be treated as a global variable access. + */ + if (get_global_var_type(cu_die, dloc, ip, var_addr, + &offset, &type_die) && + die_get_member_type(&type_die, offset, &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + if (src->multi_regs) { + pr_debug_dtp("mov [%x] percpu %#x(reg%d,reg%d) -> reg%d", + insn_offset, src->offset, src->reg1, + src->reg2, dst->reg1); + } else { + pr_debug_dtp("mov [%x] percpu %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + } + pr_debug_type_name(&tsr->type, tsr->kind); + } else { + invalidate_reg_state(tsr); + } + } + /* And then dereference the calculated pointer if it has one */ + else if (has_reg_type(state, sreg) && state->regs[sreg].ok && + state->regs[sreg].kind == TSR_KIND_PERCPU_POINTER && + die_get_member_type(&state->regs[sreg].type, + src->offset, &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] pointer %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Or try another register if any */ + else if (src->multi_regs && sreg == src->reg1 && + src->reg1 != src->reg2) { + sreg = src->reg2; + goto retry; + } + else { + int offset; + const char *var_name = NULL; + + /* it might be per-cpu variable (in kernel) access */ + if (src->offset < 0) { + if (get_global_var_info(dloc, (s64)src->offset, + &var_name, &offset) && + !strcmp(var_name, "__per_cpu_offset")) { + tsr->kind = TSR_KIND_PERCPU_BASE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] percpu base reg%d\n", + insn_offset, dst->reg1); + return; + } + } + + invalidate_reg_state(tsr); + } + } + /* Case 3. register to memory transfers */ + if (!src->mem_ref && dst->mem_ref) { + if (!has_reg_type(state, src->reg1) || + !state->regs[src->reg1].ok) + return; + + /* Check stack variables with offset */ + if (dst->reg1 == fbreg || dst->reg1 == state->stack_reg) { + struct type_state_stack *stack; + int offset = dst->offset - fboff; + + tsr = &state->regs[src->reg1]; + + stack = find_stack_state(state, offset); + if (stack) { + /* + * The source register is likely to hold a type + * of member if it's a compound type. Do not + * update the stack variable type since we can + * get the member type later by using the + * die_get_member_type(). + */ + if (!stack->compound) + set_stack_state(stack, offset, tsr->kind, + &tsr->type, tsr->offset); + } else { + findnew_stack_state(state, offset, tsr->kind, + &tsr->type, tsr->offset); + } + + if (dst->reg1 == fbreg) { + pr_debug_dtp("mov [%x] reg%d -> -%#x(stack)", + insn_offset, src->reg1, -offset); + } else { + pr_debug_dtp("mov [%x] reg%d -> %#x(reg%d)", + insn_offset, src->reg1, offset, dst->reg1); + } + if (tsr->offset != 0) { + pr_debug_dtp(" reg%d offset %#x ->", + src->reg1, tsr->offset); + } + + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* + * Ignore other transfers since it'd set a value in a struct + * and won't change the type. + */ + } + /* Case 4. memory to memory transfers (not handled for now) */ +} +#endif + +const struct arch *arch__new_x86(const struct e_machine_and_e_flags *id, const char *cpuid) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "x86"; + arch->id = *id; + if (cpuid) { + if (x86__cpuid_parse(arch, cpuid)) { + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING; + return NULL; + } + } + arch->instructions = x86__instructions; + arch->nr_instructions = ARRAY_SIZE(x86__instructions); +#ifndef NDEBUG + { + static bool sorted_check; + + if (!sorted_check) { + for (size_t i = 0; i < arch->nr_instructions - 1; i++) { + assert(strcmp(arch->instructions[i].name, + arch->instructions[i + 1].name) <= 0); + } + sorted_check = true; + } + } +#endif + arch->sorted_instructions = true; + arch->objdump.comment_char = '#'; + arch->objdump.register_char = '%'; + arch->objdump.memory_ref_char = '('; + arch->objdump.imm_char = '$'; + arch->insn_suffix = "bwlq"; +#ifdef HAVE_LIBDW_SUPPORT + arch->update_insn_state = update_insn_state_x86; +#endif + return arch; +} diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c new file mode 100644 index 000000000..4e4c58764 --- /dev/null +++ b/tools/perf/util/annotate-data.c @@ -0,0 +1,1960 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Convert sample address to data type using DWARF debug info. + * + * Written by Namhyung Kim + */ +#include +#include +#include +#include +#include + +#include "annotate.h" +#include "annotate-data.h" +#include "debuginfo.h" +#include "debug.h" +#include "dso.h" +#include "dwarf-regs.h" +#include "evsel.h" +#include "evlist.h" +#include "map.h" +#include "map_symbol.h" +#include "sort.h" +#include "strbuf.h" +#include "symbol.h" +#include "symbol_conf.h" +#include "thread.h" + +/* register number of the stack pointer */ +#define X86_REG_SP 7 + +static void delete_var_types(struct die_var_type *var_types); + +#define pr_debug_dtp(fmt, ...) \ +do { \ + if (debug_type_profile) \ + pr_info(fmt, ##__VA_ARGS__); \ + else \ + pr_debug3(fmt, ##__VA_ARGS__); \ +} while (0) + +void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind) +{ + struct strbuf sb; + char *str; + Dwarf_Word size = 0; + + if (!debug_type_profile && verbose < 3) + return; + + switch (kind) { + case TSR_KIND_INVALID: + pr_info("\n"); + return; + case TSR_KIND_PERCPU_BASE: + pr_info(" percpu base\n"); + return; + case TSR_KIND_CONST: + pr_info(" constant\n"); + return; + case TSR_KIND_PERCPU_POINTER: + pr_info(" percpu pointer"); + /* it also prints the type info */ + break; + case TSR_KIND_POINTER: + pr_info(" pointer"); + /* it also prints the type info */ + break; + case TSR_KIND_CANARY: + pr_info(" stack canary\n"); + return; + case TSR_KIND_TYPE: + default: + break; + } + + if (dwarf_aggregate_size(die, &size) != 0) + size = 0; + + strbuf_init(&sb, 32); + die_get_typename_from_type(die, &sb); + str = strbuf_detach(&sb, NULL); + pr_info(" type='%s' size=%#lx (die:%#lx)\n", + str, (long)size, (long)dwarf_dieoffset(die)); + free(str); +} + +static void pr_debug_location(Dwarf_Die *die, u64 pc, int reg) +{ + ptrdiff_t off = 0; + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + + if (!debug_type_profile && verbose < 3) + return; + + if (dwarf_attr(die, DW_AT_location, &attr) == NULL) + return; + + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + if (reg != DWARF_REG_PC && end <= pc) + continue; + if (reg != DWARF_REG_PC && start > pc) + break; + + pr_info(" variable location: "); + switch (ops->atom) { + case DW_OP_reg0 ...DW_OP_reg31: + pr_info("reg%d\n", ops->atom - DW_OP_reg0); + break; + case DW_OP_breg0 ...DW_OP_breg31: + pr_info("base=reg%d, offset=%#lx\n", + ops->atom - DW_OP_breg0, (long)ops->number); + break; + case DW_OP_regx: + pr_info("reg%ld\n", (long)ops->number); + break; + case DW_OP_bregx: + pr_info("base=reg%ld, offset=%#lx\n", + (long)ops->number, (long)ops->number2); + break; + case DW_OP_fbreg: + pr_info("use frame base, offset=%#lx\n", (long)ops->number); + break; + case DW_OP_addr: + pr_info("address=%#lx\n", (long)ops->number); + break; + default: + pr_info("unknown: code=%#x, number=%#lx\n", + ops->atom, (long)ops->number); + break; + } + break; + } +} + +static void pr_debug_scope(Dwarf_Die *scope_die) +{ + int tag; + + if (!debug_type_profile && verbose < 3) + return; + + pr_info("(die:%lx) ", (long)dwarf_dieoffset(scope_die)); + + tag = dwarf_tag(scope_die); + if (tag == DW_TAG_subprogram) + pr_info("[function] %s\n", die_name(scope_die)); + else if (tag == DW_TAG_inlined_subroutine) + pr_info("[inlined] %s\n", die_name(scope_die)); + else if (tag == DW_TAG_lexical_block) + pr_info("[block]\n"); + else + pr_info("[unknown] tag=%x\n", tag); +} + +bool has_reg_type(struct type_state *state, int reg) +{ + return (unsigned)reg < ARRAY_SIZE(state->regs); +} + +static void init_type_state(struct type_state *state, const struct arch *arch) +{ + memset(state, 0, sizeof(*state)); + INIT_LIST_HEAD(&state->stack_vars); + + if (arch__is_x86(arch)) { + state->regs[0].caller_saved = true; + state->regs[1].caller_saved = true; + state->regs[2].caller_saved = true; + state->regs[4].caller_saved = true; + state->regs[5].caller_saved = true; + state->regs[8].caller_saved = true; + state->regs[9].caller_saved = true; + state->regs[10].caller_saved = true; + state->regs[11].caller_saved = true; + state->ret_reg = 0; + state->stack_reg = X86_REG_SP; + } +} + +static void exit_type_state(struct type_state *state) +{ + struct type_state_stack *stack, *tmp; + + list_for_each_entry_safe(stack, tmp, &state->stack_vars, list) { + list_del(&stack->list); + free(stack); + } +} + +/* + * Compare type name and size to maintain them in a tree. + * I'm not sure if DWARF would have information of a single type in many + * different places (compilation units). If not, it could compare the + * offset of the type entry in the .debug_info section. + */ +static int data_type_cmp(const void *_key, const struct rb_node *node) +{ + const struct annotated_data_type *key = _key; + struct annotated_data_type *type; + + type = rb_entry(node, struct annotated_data_type, node); + + if (key->self.size != type->self.size) + return key->self.size - type->self.size; + return strcmp(key->self.type_name, type->self.type_name); +} + +static bool data_type_less(struct rb_node *node_a, const struct rb_node *node_b) +{ + struct annotated_data_type *a, *b; + + a = rb_entry(node_a, struct annotated_data_type, node); + b = rb_entry(node_b, struct annotated_data_type, node); + + if (a->self.size != b->self.size) + return a->self.size < b->self.size; + return strcmp(a->self.type_name, b->self.type_name) < 0; +} + +/* Recursively add new members for struct/union */ +static int __add_member_cb(Dwarf_Die *die, void *arg) +{ + struct annotated_member *parent = arg; + struct annotated_member *member; + Dwarf_Die member_type, die_mem; + Dwarf_Word size, loc, bit_size = 0; + Dwarf_Attribute attr; + struct strbuf sb; + int tag; + + if (dwarf_tag(die) != DW_TAG_member) + return DIE_FIND_CB_SIBLING; + + member = zalloc(sizeof(*member)); + if (member == NULL) + return DIE_FIND_CB_END; + + strbuf_init(&sb, 32); + die_get_typename(die, &sb); + + __die_get_real_type(die, &member_type); + if (dwarf_tag(&member_type) == DW_TAG_typedef) + die_get_real_type(&member_type, &die_mem); + else + die_mem = member_type; + + if (dwarf_aggregate_size(&die_mem, &size) < 0) + size = 0; + + if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) { + if (dwarf_formudata(&attr, &loc) != 0) { + if (die_get_data_member_location(die, &loc) != 0) + loc = 0; + } + } else { + /* bitfield member */ + if (dwarf_attr_integrate(die, DW_AT_data_bit_offset, &attr) && + dwarf_formudata(&attr, &loc) == 0) + loc /= 8; + else + loc = 0; + + if (dwarf_attr_integrate(die, DW_AT_bit_size, &attr) && + dwarf_formudata(&attr, &bit_size) == 0) + size = (bit_size + 7) / 8; + } + + member->type_name = strbuf_detach(&sb, NULL); + /* member->var_name can be NULL */ + if (dwarf_diename(die)) { + if (bit_size) { + if (asprintf(&member->var_name, "%s:%ld", + dwarf_diename(die), (long)bit_size) < 0) + member->var_name = NULL; + } else { + const char *name = dwarf_diename(die); + + member->var_name = name ? strdup(name) : NULL; + } + + if (member->var_name == NULL) { + free(member); + return DIE_FIND_CB_END; + } + } + member->size = size; + member->offset = loc + parent->offset; + INIT_LIST_HEAD(&member->children); + list_add_tail(&member->node, &parent->children); + + tag = dwarf_tag(&die_mem); + switch (tag) { + case DW_TAG_structure_type: + case DW_TAG_union_type: + die_find_child(&die_mem, __add_member_cb, member, &die_mem); + break; + default: + break; + } + return DIE_FIND_CB_SIBLING; +} + +static void add_member_types(struct annotated_data_type *parent, Dwarf_Die *type) +{ + Dwarf_Die die_mem; + + die_find_child(type, __add_member_cb, &parent->self, &die_mem); +} + +static void delete_members(struct annotated_member *member) +{ + struct annotated_member *child, *tmp; + + list_for_each_entry_safe(child, tmp, &member->children, node) { + list_del(&child->node); + delete_members(child); + zfree(&child->type_name); + zfree(&child->var_name); + free(child); + } +} + +static int fill_member_name(char *buf, size_t sz, struct annotated_member *m, + int offset, bool first) +{ + struct annotated_member *child; + + if (list_empty(&m->children)) + return 0; + + list_for_each_entry(child, &m->children, node) { + int len; + + if (offset < child->offset || offset >= child->offset + child->size) + continue; + + /* It can have anonymous struct/union members */ + if (child->var_name) { + len = scnprintf(buf, sz, "%s%s", + first ? "" : ".", child->var_name); + first = false; + } else { + len = 0; + } + + return fill_member_name(buf + len, sz - len, child, offset, first) + len; + } + return 0; +} + +int annotated_data_type__get_member_name(struct annotated_data_type *adt, + char *buf, size_t sz, int member_offset) +{ + return fill_member_name(buf, sz, &adt->self, member_offset, /*first=*/true); +} + +static struct annotated_data_type *dso__findnew_data_type(struct dso *dso, + Dwarf_Die *type_die) +{ + struct annotated_data_type *result = NULL; + struct annotated_data_type key; + struct rb_node *node; + struct strbuf sb; + char *type_name; + Dwarf_Word size; + + strbuf_init(&sb, 32); + if (die_get_typename_from_type(type_die, &sb) < 0) + strbuf_add(&sb, "(unknown type)", 14); + type_name = strbuf_detach(&sb, NULL); + + if (dwarf_tag(type_die) == DW_TAG_typedef) + die_get_real_type(type_die, type_die); + + if (dwarf_aggregate_size(type_die, &size) != 0) + size = 0; + + /* Check existing nodes in dso->data_types tree */ + key.self.type_name = type_name; + key.self.size = size; + node = rb_find(&key, dso__data_types(dso), data_type_cmp); + if (node) { + result = rb_entry(node, struct annotated_data_type, node); + free(type_name); + return result; + } + + /* If not, add a new one */ + result = zalloc(sizeof(*result)); + if (result == NULL) { + free(type_name); + return NULL; + } + + result->self.type_name = type_name; + result->self.size = size; + INIT_LIST_HEAD(&result->self.children); + + if (symbol_conf.annotate_data_member) + add_member_types(result, type_die); + + rb_add(&result->node, dso__data_types(dso), data_type_less); + return result; +} + +static bool find_cu_die(struct debuginfo *di, u64 pc, Dwarf_Die *cu_die) +{ + Dwarf_Off off, next_off; + size_t header_size; + + if (dwarf_addrdie(di->dbg, pc, cu_die) != NULL) + return cu_die; + + /* + * There are some kernels don't have full aranges and contain only a few + * aranges entries. Fallback to iterate all CU entries in .debug_info + * in case it's missing. + */ + off = 0; + while (dwarf_nextcu(di->dbg, off, &next_off, &header_size, + NULL, NULL, NULL) == 0) { + if (dwarf_offdie(di->dbg, off + header_size, cu_die) && + dwarf_haspc(cu_die, pc)) + return true; + + off = next_off; + } + return false; +} + +enum type_match_result { + PERF_TMR_UNKNOWN = 0, + PERF_TMR_OK, + PERF_TMR_NO_TYPE, + PERF_TMR_NO_POINTER, + PERF_TMR_NO_SIZE, + PERF_TMR_BAD_OFFSET, + PERF_TMR_BAIL_OUT, +}; + +static const char *match_result_str(enum type_match_result tmr) +{ + switch (tmr) { + case PERF_TMR_OK: + return "Good!"; + case PERF_TMR_NO_TYPE: + return "no type information"; + case PERF_TMR_NO_POINTER: + return "no/void pointer"; + case PERF_TMR_NO_SIZE: + return "type size is unknown"; + case PERF_TMR_BAD_OFFSET: + return "offset bigger than size"; + case PERF_TMR_UNKNOWN: + case PERF_TMR_BAIL_OUT: + default: + return "invalid state"; + } +} + +static bool is_compound_type(Dwarf_Die *type_die) +{ + int tag = dwarf_tag(type_die); + + return tag == DW_TAG_structure_type || tag == DW_TAG_union_type; +} + +/* returns if Type B has better information than Type A */ +static bool is_better_type(Dwarf_Die *type_a, Dwarf_Die *type_b) +{ + Dwarf_Word size_a, size_b; + Dwarf_Die die_a, die_b; + Dwarf_Die ptr_a, ptr_b; + Dwarf_Die *ptr_type_a, *ptr_type_b; + + ptr_type_a = die_get_pointer_type(type_a, &ptr_a); + ptr_type_b = die_get_pointer_type(type_b, &ptr_b); + + /* pointer type is preferred */ + if ((ptr_type_a != NULL) != (ptr_type_b != NULL)) + return ptr_type_b != NULL; + + if (ptr_type_b) { + /* + * We want to compare the target type, but 'void *' can fail to + * get the target type. + */ + if (die_get_real_type(ptr_type_a, &die_a) == NULL) + return true; + if (die_get_real_type(ptr_type_b, &die_b) == NULL) + return false; + + type_a = &die_a; + type_b = &die_b; + } + + /* bigger type is preferred */ + if (dwarf_aggregate_size(type_a, &size_a) < 0 || + dwarf_aggregate_size(type_b, &size_b) < 0) + return false; + + if (size_a != size_b) + return size_a < size_b; + + /* struct or union is preferred */ + if (is_compound_type(type_a) != is_compound_type(type_b)) + return is_compound_type(type_b); + + /* typedef is preferred */ + if (dwarf_tag(type_b) == DW_TAG_typedef) + return true; + + return false; +} + +/* The type info will be saved in @type_die */ +static enum type_match_result check_variable(struct data_loc_info *dloc, + Dwarf_Die *var_die, + Dwarf_Die *type_die, int reg, + int offset, bool is_fbreg) +{ + Dwarf_Word size; + bool needs_pointer = true; + Dwarf_Die sized_type; + + if (reg == DWARF_REG_PC) + needs_pointer = false; + else if (reg == dloc->fbreg || is_fbreg) + needs_pointer = false; + else if (arch__is_x86(dloc->arch) && reg == X86_REG_SP) + needs_pointer = false; + + /* Get the type of the variable */ + if (__die_get_real_type(var_die, type_die) == NULL) + return PERF_TMR_NO_TYPE; + + /* + * Usually it expects a pointer type for a memory access. + * Convert to a real type it points to. But global variables + * and local variables are accessed directly without a pointer. + */ + if (needs_pointer) { + if (die_get_pointer_type(type_die, type_die) == NULL || + __die_get_real_type(type_die, type_die) == NULL) + return PERF_TMR_NO_POINTER; + } + + if (dwarf_tag(type_die) == DW_TAG_typedef) + die_get_real_type(type_die, &sized_type); + else + sized_type = *type_die; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(&sized_type, &size) < 0) + return PERF_TMR_NO_SIZE; + + /* Minimal sanity check */ + if ((unsigned)offset >= size) + return PERF_TMR_BAD_OFFSET; + + return PERF_TMR_OK; +} + +struct type_state_stack *find_stack_state(struct type_state *state, + int offset) +{ + struct type_state_stack *stack; + + list_for_each_entry(stack, &state->stack_vars, list) { + if (offset == stack->offset) + return stack; + + if (stack->compound && stack->offset < offset && + offset < stack->offset + stack->size) + return stack; + } + return NULL; +} + +void set_stack_state(struct type_state_stack *stack, int offset, u8 kind, + Dwarf_Die *type_die, int ptr_offset) +{ + int tag; + Dwarf_Word size; + + if (kind == TSR_KIND_POINTER) { + /* TODO: arch-dependent pointer size */ + size = sizeof(void *); + } + else if (dwarf_aggregate_size(type_die, &size) < 0) + size = 0; + + stack->type = *type_die; + stack->size = size; + stack->offset = offset; + stack->ptr_offset = ptr_offset; + stack->kind = kind; + + if (kind == TSR_KIND_POINTER) { + stack->compound = false; + return; + } + + tag = dwarf_tag(type_die); + + switch (tag) { + case DW_TAG_structure_type: + case DW_TAG_union_type: + stack->compound = (kind != TSR_KIND_PERCPU_POINTER); + break; + default: + stack->compound = false; + break; + } +} + +struct type_state_stack *findnew_stack_state(struct type_state *state, + int offset, u8 kind, + Dwarf_Die *type_die, + int ptr_offset) +{ + struct type_state_stack *stack = find_stack_state(state, offset); + + if (stack) { + set_stack_state(stack, offset, kind, type_die, ptr_offset); + return stack; + } + + stack = malloc(sizeof(*stack)); + if (stack) { + set_stack_state(stack, offset, kind, type_die, ptr_offset); + list_add(&stack->list, &state->stack_vars); + } + return stack; +} + +/* Maintain a cache for quick global variable lookup */ +struct global_var_entry { + struct rb_node node; + char *name; + u64 start; + u64 end; + u64 die_offset; +}; + +static int global_var_cmp(const void *_key, const struct rb_node *node) +{ + const u64 addr = (uintptr_t)_key; + struct global_var_entry *gvar; + + gvar = rb_entry(node, struct global_var_entry, node); + + if (gvar->start <= addr && addr < gvar->end) + return 0; + return gvar->start > addr ? -1 : 1; +} + +static bool global_var_less(struct rb_node *node_a, const struct rb_node *node_b) +{ + struct global_var_entry *gvar_a, *gvar_b; + + gvar_a = rb_entry(node_a, struct global_var_entry, node); + gvar_b = rb_entry(node_b, struct global_var_entry, node); + + return gvar_a->start < gvar_b->start; +} + +static struct global_var_entry *global_var__find(struct data_loc_info *dloc, u64 addr) +{ + struct dso *dso = map__dso(dloc->ms->map); + struct rb_node *node; + + node = rb_find((void *)(uintptr_t)addr, dso__global_vars(dso), global_var_cmp); + if (node == NULL) + return NULL; + + return rb_entry(node, struct global_var_entry, node); +} + +static bool global_var__add(struct data_loc_info *dloc, u64 addr, + const char *name, Dwarf_Die *type_die) +{ + struct dso *dso = map__dso(dloc->ms->map); + struct global_var_entry *gvar; + Dwarf_Word size; + + if (dwarf_aggregate_size(type_die, &size) < 0) + return false; + + gvar = malloc(sizeof(*gvar)); + if (gvar == NULL) + return false; + + gvar->name = name ? strdup(name) : NULL; + if (name && gvar->name == NULL) { + free(gvar); + return false; + } + + gvar->start = addr; + gvar->end = addr + size; + gvar->die_offset = dwarf_dieoffset(type_die); + + rb_add(&gvar->node, dso__global_vars(dso), global_var_less); + return true; +} + +void global_var_type__tree_delete(struct rb_root *root) +{ + struct global_var_entry *gvar; + + while (!RB_EMPTY_ROOT(root)) { + struct rb_node *node = rb_first(root); + + rb_erase(node, root); + gvar = rb_entry(node, struct global_var_entry, node); + zfree(&gvar->name); + free(gvar); + } +} + +bool get_global_var_info(struct data_loc_info *dloc, u64 addr, + const char **var_name, int *var_offset) +{ + struct addr_location al; + struct symbol *sym; + u64 mem_addr; + + /* Kernel symbols might be relocated */ + mem_addr = addr + map__reloc(dloc->ms->map); + + addr_location__init(&al); + sym = thread__find_symbol_fb(dloc->thread, dloc->cpumode, + mem_addr, &al); + if (sym) { + *var_name = sym->name; + /* Calculate type offset from the start of variable */ + *var_offset = mem_addr - map__unmap_ip(al.map, sym->start); + } else { + *var_name = NULL; + } + addr_location__exit(&al); + if (*var_name == NULL) + return false; + + return true; +} + +static void global_var__collect(struct data_loc_info *dloc) +{ + Dwarf *dwarf = dloc->di->dbg; + Dwarf_Off off, next_off; + Dwarf_Die cu_die, type_die; + size_t header_size; + + /* Iterate all CU and collect global variables that have no location in a register. */ + off = 0; + while (dwarf_nextcu(dwarf, off, &next_off, &header_size, + NULL, NULL, NULL) == 0) { + struct die_var_type *var_types = NULL; + struct die_var_type *pos; + + if (dwarf_offdie(dwarf, off + header_size, &cu_die) == NULL) { + off = next_off; + continue; + } + + die_collect_global_vars(&cu_die, &var_types); + + for (pos = var_types; pos; pos = pos->next) { + const char *var_name = NULL; + int var_offset = 0; + + if (pos->reg != -1) + continue; + + if (!dwarf_offdie(dwarf, pos->die_off, &type_die)) + continue; + + get_global_var_info(dloc, pos->addr, &var_name, &var_offset); + + global_var__add(dloc, pos->addr, var_name, &type_die); + } + + delete_var_types(var_types); + + off = next_off; + } +} + +bool get_global_var_type(Dwarf_Die *cu_die, struct data_loc_info *dloc, + u64 ip, u64 var_addr, int *var_offset, + Dwarf_Die *type_die) +{ + u64 pc; + int offset; + const char *var_name = NULL; + struct global_var_entry *gvar; + struct dso *dso = map__dso(dloc->ms->map); + Dwarf_Die var_die; + + if (RB_EMPTY_ROOT(dso__global_vars(dso))) + global_var__collect(dloc); + + gvar = global_var__find(dloc, var_addr); + if (gvar) { + if (!dwarf_offdie(dloc->di->dbg, gvar->die_offset, type_die)) + return false; + + *var_offset = var_addr - gvar->start; + return true; + } + + /* Try to get the variable by address first */ + if (die_find_variable_by_addr(cu_die, var_addr, &var_die, type_die, + &offset)) { + var_name = dwarf_diename(&var_die); + *var_offset = offset; + goto ok; + } + + if (!get_global_var_info(dloc, var_addr, &var_name, var_offset)) + return false; + + pc = map__rip_2objdump(dloc->ms->map, ip); + + /* Try to get the name of global variable */ + if (die_find_variable_at(cu_die, var_name, pc, &var_die) && + check_variable(dloc, &var_die, type_die, DWARF_REG_PC, *var_offset, + /*is_fbreg=*/false) == PERF_TMR_OK) + goto ok; + + return false; + +ok: + /* The address should point to the start of the variable */ + global_var__add(dloc, var_addr - *var_offset, var_name, type_die); + return true; +} + +static bool die_is_same(Dwarf_Die *die_a, Dwarf_Die *die_b) +{ + return (die_a->cu == die_b->cu) && (die_a->addr == die_b->addr); +} + +static void tsr_set_lifetime(struct type_state_reg *tsr, + const struct die_var_type *var) +{ + if (var && var->has_range && var->end > var->addr) { + tsr->lifetime_active = true; + tsr->lifetime_end = var->end; + } else { + tsr->lifetime_active = false; + tsr->lifetime_end = 0; + } +} + +/** + * update_var_state - Update type state using given variables + * @state: type state table + * @dloc: data location info + * @addr: instruction address to match with variable + * @insn_offset: instruction offset (for debug) + * @var_types: list of variables with type info + * + * This function fills the @state table using @var_types info. Each variable + * is used only at the given location and updates an entry in the table. + */ +static void update_var_state(struct type_state *state, struct data_loc_info *dloc, + u64 addr, u64 insn_offset, struct die_var_type *var_types) +{ + Dwarf_Die mem_die; + struct die_var_type *var; + int fbreg = dloc->fbreg; + int fb_offset = 0; + + if (dloc->fb_cfa) { + if (die_get_cfa(dloc->di->dbg, addr, &fbreg, &fb_offset) < 0) + fbreg = -1; + } + + for (var = var_types; var != NULL; var = var->next) { + /* Check if addr falls within the variable's valid range */ + if (var->has_range) { + if (addr < var->addr || (var->end && addr >= var->end)) + continue; + } else { + if (addr != var->addr) + continue; + } + /* Get the type DIE using the offset */ + if (!dwarf_offdie(dloc->di->dbg, var->die_off, &mem_die)) + continue; + + if (var->reg == DWARF_REG_FB || var->reg == fbreg || var->reg == state->stack_reg) { + Dwarf_Die ptr_die; + Dwarf_Die *ptr_type; + int offset = var->offset; + struct type_state_stack *stack; + + ptr_type = die_get_pointer_type(&mem_die, &ptr_die); + + /* If the reg location holds the pointer value, dereference the type */ + if (!var->is_reg_var_addr && ptr_type && + __die_get_real_type(ptr_type, &mem_die) == NULL) + continue; + + if (var->reg != DWARF_REG_FB) + offset -= fb_offset; + + stack = find_stack_state(state, offset); + if (stack && stack->kind == TSR_KIND_TYPE && + !is_better_type(&stack->type, &mem_die)) + continue; + + findnew_stack_state(state, offset, TSR_KIND_TYPE, + &mem_die, /*ptr_offset=*/0); + + if (var->reg == state->stack_reg) { + pr_debug_dtp("var [%"PRIx64"] %#x(reg%d)", + insn_offset, offset, state->stack_reg); + } else { + pr_debug_dtp("var [%"PRIx64"] -%#x(stack)", + insn_offset, -offset); + } + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + } else if (has_reg_type(state, var->reg)) { + struct type_state_reg *reg; + Dwarf_Die orig_type; + + reg = &state->regs[var->reg]; + + if (reg->ok && reg->kind == TSR_KIND_TYPE && + (!is_better_type(®->type, &mem_die) || var->is_reg_var_addr)) + continue; + + /* Handle address registers with TSR_KIND_POINTER */ + if (var->is_reg_var_addr) { + if (reg->ok && reg->kind == TSR_KIND_POINTER && + !is_better_type(®->type, &mem_die)) + continue; + + reg->offset = -var->offset; + reg->type = mem_die; + reg->kind = TSR_KIND_POINTER; + reg->ok = true; + tsr_set_lifetime(reg, var); + + pr_debug_dtp("var [%"PRIx64"] reg%d addr offset %x", + insn_offset, var->reg, var->offset); + pr_debug_type_name(&mem_die, TSR_KIND_POINTER); + continue; + } + + orig_type = reg->type; + /* + * var->offset + reg value is the beginning of the struct + * reg->offset is the offset the reg points + */ + reg->offset = -var->offset; + reg->type = mem_die; + reg->kind = TSR_KIND_TYPE; + reg->ok = true; + tsr_set_lifetime(reg, var); + + pr_debug_dtp("var [%"PRIx64"] reg%d offset %x", + insn_offset, var->reg, var->offset); + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + + /* + * If this register is directly copied from another and it gets a + * better type, also update the type of the source register. This + * is usually the case of container_of() macro with offset of 0. + */ + if (has_reg_type(state, reg->copied_from)) { + struct type_state_reg *copy_reg; + + copy_reg = &state->regs[reg->copied_from]; + + /* TODO: check if type is compatible or embedded */ + if (!copy_reg->ok || (copy_reg->kind != TSR_KIND_TYPE) || + !die_is_same(©_reg->type, &orig_type) || + !is_better_type(©_reg->type, &mem_die)) + continue; + + copy_reg->type = mem_die; + + pr_debug_dtp("var [%"PRIx64"] copyback reg%d", + insn_offset, reg->copied_from); + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + } + } + } +} + +/** + * update_insn_state - Update type state for an instruction + * @state: type state table + * @dloc: data location info + * @cu_die: compile unit debug entry + * @dl: disasm line for the instruction + * + * This function updates the @state table for the target operand of the + * instruction at @dl if it transfers the type like MOV on x86. Since it + * tracks the type, it won't care about the values like in arithmetic + * instructions like ADD/SUB/MUL/DIV and INC/DEC. + * + * Note that ops->reg2 is only available when both mem_ref and multi_regs + * are true. + */ +static void update_insn_state(struct type_state *state, struct data_loc_info *dloc, + Dwarf_Die *cu_die, struct disasm_line *dl) +{ + if (dloc->arch->update_insn_state) + dloc->arch->update_insn_state(state, dloc, cu_die, dl); +} + +/* + * Prepend this_blocks (from the outer scope) to full_blocks, removing + * duplicate disasm line. + */ +static void prepend_basic_blocks(struct list_head *this_blocks, + struct list_head *full_blocks) +{ + struct annotated_basic_block *first_bb, *last_bb; + + last_bb = list_last_entry(this_blocks, typeof(*last_bb), list); + first_bb = list_first_entry(full_blocks, typeof(*first_bb), list); + + if (list_empty(full_blocks)) + goto out; + + /* Last insn in this_blocks should be same as first insn in full_blocks */ + if (last_bb->end != first_bb->begin) { + pr_debug("prepend basic blocks: mismatched disasm line %"PRIx64" -> %"PRIx64"\n", + last_bb->end->al.offset, first_bb->begin->al.offset); + goto out; + } + + /* Is the basic block have only one disasm_line? */ + if (last_bb->begin == last_bb->end) { + list_del(&last_bb->list); + free(last_bb); + goto out; + } + + /* Point to the insn before the last when adding this block to full_blocks */ + last_bb->end = list_prev_entry(last_bb->end, al.node); + +out: + list_splice(this_blocks, full_blocks); +} + +static void delete_basic_blocks(struct list_head *basic_blocks) +{ + struct annotated_basic_block *bb, *tmp; + + list_for_each_entry_safe(bb, tmp, basic_blocks, list) { + list_del(&bb->list); + free(bb); + } +} + +/* Make sure all variables have a valid start address */ +static void fixup_var_address(struct die_var_type *var_types, u64 addr) +{ + while (var_types) { + /* + * Some variables have no address range meaning it's always + * available in the whole scope. Let's adjust the start + * address to the start of the scope. + */ + if (var_types->addr == 0) + var_types->addr = addr; + + var_types = var_types->next; + } +} + +static void delete_var_types(struct die_var_type *var_types) +{ + while (var_types) { + struct die_var_type *next = var_types->next; + + free(var_types); + var_types = next; + } +} + +/* should match to is_stack_canary() in util/annotate.c */ +static void setup_stack_canary(struct data_loc_info *dloc) +{ + if (arch__is_x86(dloc->arch)) { + dloc->op->segment = INSN_SEG_X86_GS; + dloc->op->imm = true; + dloc->op->offset = 40; + } +} + +/* + * It's at the target address, check if it has a matching type. + * It returns PERF_TMR_BAIL_OUT when it looks up per-cpu variables which + * are similar to global variables and no additional info is needed. + */ +static enum type_match_result check_matching_type(struct type_state *state, + struct data_loc_info *dloc, + Dwarf_Die *cu_die, + struct disasm_line *dl, + Dwarf_Die *type_die) +{ + Dwarf_Word size; + u32 insn_offset = dl->al.offset; + int reg = dloc->op->reg1; + int offset = dloc->op->offset; + const char *offset_sign = ""; + bool retry = true; + + if (offset < 0) { + offset = -offset; + offset_sign = "-"; + } + +again: + pr_debug_dtp("chk [%x] reg%d offset=%s%#x ok=%d kind=%d ", + insn_offset, reg, offset_sign, offset, + state->regs[reg].ok, state->regs[reg].kind); + + if (!state->regs[reg].ok) + goto check_non_register; + + if (state->regs[reg].kind == TSR_KIND_TYPE) { + Dwarf_Die ptr_die; + Dwarf_Die sized_type; + Dwarf_Die *ptr_type; + struct strbuf sb; + + strbuf_init(&sb, 32); + die_get_typename_from_type(&state->regs[reg].type, &sb); + pr_debug_dtp("(%s)", sb.buf); + strbuf_release(&sb); + + /* + * Normal registers should hold a pointer (or array) to + * dereference a memory location. + */ + ptr_type = die_get_pointer_type(&state->regs[reg].type, &ptr_die); + if (!ptr_type) { + if (dloc->op->offset < 0 && reg != state->stack_reg) + goto check_kernel; + + return PERF_TMR_NO_POINTER; + } + + /* Remove the pointer and get the target type */ + if (__die_get_real_type(ptr_type, type_die) == NULL) + return PERF_TMR_NO_POINTER; + + dloc->type_offset = dloc->op->offset + state->regs[reg].offset; + + if (dwarf_tag(type_die) == DW_TAG_typedef) + die_get_real_type(type_die, &sized_type); + else + sized_type = *type_die; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(&sized_type, &size) < 0 || + (unsigned)dloc->type_offset >= size) + return PERF_TMR_BAD_OFFSET; + + return PERF_TMR_OK; + } + + if (state->regs[reg].kind == TSR_KIND_POINTER) { + struct strbuf sb; + + strbuf_init(&sb, 32); + die_get_typename_from_type(&state->regs[reg].type, &sb); + pr_debug_dtp("(ptr->%s)", sb.buf); + strbuf_release(&sb); + + /* + * Register holds a pointer (address) to the target variable. + * The type is the type of the variable it points to. + */ + *type_die = state->regs[reg].type; + + dloc->type_offset = dloc->op->offset + state->regs[reg].offset; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(type_die, &size) < 0 || + (unsigned)dloc->type_offset >= size) + return PERF_TMR_BAD_OFFSET; + + return PERF_TMR_OK; + } + + if (state->regs[reg].kind == TSR_KIND_PERCPU_POINTER) { + pr_debug_dtp("percpu ptr"); + + /* + * It's actaully pointer but the address was calculated using + * some arithmetic. So it points to the actual type already. + */ + *type_die = state->regs[reg].type; + + dloc->type_offset = dloc->op->offset; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(type_die, &size) < 0 || + (unsigned)dloc->type_offset >= size) + return PERF_TMR_BAIL_OUT; + + return PERF_TMR_OK; + } + + if (state->regs[reg].kind == TSR_KIND_CANARY) { + pr_debug_dtp("stack canary"); + + /* + * This is a saved value of the stack canary which will be handled + * in the outer logic when it returns failure here. Pretend it's + * from the stack canary directly. + */ + setup_stack_canary(dloc); + + return PERF_TMR_BAIL_OUT; + } + + if (state->regs[reg].kind == TSR_KIND_PERCPU_BASE) { + u64 var_addr = dloc->op->offset; + int var_offset; + + pr_debug_dtp("percpu var"); + + if (dloc->op->multi_regs) { + int reg2 = dloc->op->reg2; + + if (dloc->op->reg2 == reg) + reg2 = dloc->op->reg1; + + if (has_reg_type(state, reg2) && state->regs[reg2].ok && + state->regs[reg2].kind == TSR_KIND_CONST) + var_addr += state->regs[reg2].imm_value; + } + + if (get_global_var_type(cu_die, dloc, dloc->ip, var_addr, + &var_offset, type_die)) { + dloc->type_offset = var_offset; + return PERF_TMR_OK; + } + /* No need to retry per-cpu (global) variables */ + return PERF_TMR_BAIL_OUT; + } + + if (state->regs[reg].kind == TSR_KIND_CONST && + dso__kernel(map__dso(dloc->ms->map))) { + if (dloc->op->offset < 0 && reg != state->stack_reg && reg != dloc->fbreg) + goto check_kernel; + } +check_non_register: + if (reg == dloc->fbreg || reg == state->stack_reg) { + struct type_state_stack *stack; + + pr_debug_dtp("%s", reg == dloc->fbreg ? "fbreg" : "stack"); + + stack = find_stack_state(state, dloc->type_offset); + if (stack == NULL) { + if (retry) { + pr_debug_dtp(" : retry\n"); + retry = false; + + /* update type info it's the first store to the stack */ + update_insn_state(state, dloc, cu_die, dl); + goto again; + } + return PERF_TMR_NO_TYPE; + } + + if (stack->kind == TSR_KIND_CANARY) { + setup_stack_canary(dloc); + return PERF_TMR_BAIL_OUT; + } + + if (stack->kind != TSR_KIND_TYPE) + return PERF_TMR_NO_TYPE; + + *type_die = stack->type; + /* Update the type offset from the start of slot */ + dloc->type_offset -= stack->offset; + + return PERF_TMR_OK; + } + + if (dloc->fb_cfa) { + struct type_state_stack *stack; + u64 pc = map__rip_2objdump(dloc->ms->map, dloc->ip); + int fbreg, fboff; + + pr_debug_dtp("cfa"); + + if (die_get_cfa(dloc->di->dbg, pc, &fbreg, &fboff) < 0) + fbreg = -1; + + if (reg != fbreg) + return PERF_TMR_NO_TYPE; + + stack = find_stack_state(state, dloc->type_offset - fboff); + if (stack == NULL) { + if (retry) { + pr_debug_dtp(" : retry\n"); + retry = false; + + /* update type info it's the first store to the stack */ + update_insn_state(state, dloc, cu_die, dl); + goto again; + } + return PERF_TMR_NO_TYPE; + } + + if (stack->kind == TSR_KIND_CANARY) { + setup_stack_canary(dloc); + return PERF_TMR_BAIL_OUT; + } + + if (stack->kind != TSR_KIND_TYPE) + return PERF_TMR_NO_TYPE; + + *type_die = stack->type; + /* Update the type offset from the start of slot */ + dloc->type_offset -= fboff + stack->offset; + + return PERF_TMR_OK; + } + +check_kernel: + if (dso__kernel(map__dso(dloc->ms->map))) { + u64 addr; + + /* Direct this-cpu access like "%gs:0x34740" */ + if (dloc->op->segment == INSN_SEG_X86_GS && dloc->op->imm && + arch__is_x86(dloc->arch)) { + pr_debug_dtp("this-cpu var"); + + addr = dloc->op->offset; + + if (get_global_var_type(cu_die, dloc, dloc->ip, addr, + &offset, type_die)) { + dloc->type_offset = offset; + return PERF_TMR_OK; + } + return PERF_TMR_BAIL_OUT; + } + + /* Access to global variable like "-0x7dcf0500(,%rdx,8)" */ + if (dloc->op->offset < 0 && reg != state->stack_reg) { + addr = (s64) dloc->op->offset; + + if (get_global_var_type(cu_die, dloc, dloc->ip, addr, + &offset, type_die)) { + pr_debug_dtp("global var"); + + dloc->type_offset = offset; + return PERF_TMR_OK; + } + return PERF_TMR_BAIL_OUT; + } + } + + return PERF_TMR_UNKNOWN; +} + +/* Iterate instructions in basic blocks and update type table */ +static enum type_match_result find_data_type_insn(struct data_loc_info *dloc, + struct list_head *basic_blocks, + struct die_var_type *var_types, + Dwarf_Die *cu_die, + Dwarf_Die *type_die) +{ + struct type_state state; + struct symbol *sym = dloc->ms->sym; + struct annotation *notes = symbol__annotation(sym); + struct annotated_basic_block *bb; + enum type_match_result ret = PERF_TMR_UNKNOWN; + + init_type_state(&state, dloc->arch); + + list_for_each_entry(bb, basic_blocks, list) { + struct disasm_line *dl = bb->begin; + + BUG_ON(bb->begin->al.offset == -1 || bb->end->al.offset == -1); + + pr_debug_dtp("bb: [%"PRIx64" - %"PRIx64"]\n", + bb->begin->al.offset, bb->end->al.offset); + + list_for_each_entry_from(dl, ¬es->src->source, al.node) { + u64 this_ip = sym->start + dl->al.offset; + u64 addr = map__rip_2objdump(dloc->ms->map, this_ip); + + /* Skip comment or debug info lines */ + if (dl->al.offset == -1) + continue; + + /* Update variable type at this address */ + update_var_state(&state, dloc, addr, dl->al.offset, var_types); + + if (this_ip == dloc->ip) { + ret = check_matching_type(&state, dloc, + cu_die, dl, type_die); + pr_debug_dtp(" : %s\n", match_result_str(ret)); + goto out; + } + + /* Update type table after processing the instruction */ + update_insn_state(&state, dloc, cu_die, dl); + if (dl == bb->end) + break; + } + } + +out: + exit_type_state(&state); + return ret; +} + +static int arch_supports_insn_tracking(struct data_loc_info *dloc) +{ + if ((arch__is_x86(dloc->arch)) || (arch__is_powerpc(dloc->arch))) + return 1; + return 0; +} + +/* + * Construct a list of basic blocks for each scope with variables and try to find + * the data type by updating a type state table through instructions. + */ +static enum type_match_result find_data_type_block(struct data_loc_info *dloc, + Dwarf_Die *cu_die, + Dwarf_Die *scopes, + int nr_scopes, + Dwarf_Die *type_die) +{ + LIST_HEAD(basic_blocks); + struct die_var_type *var_types = NULL; + u64 src_ip, dst_ip, prev_dst_ip; + enum type_match_result ret = PERF_TMR_UNKNOWN; + + /* TODO: other architecture support */ + if (!arch_supports_insn_tracking(dloc)) + return PERF_TMR_BAIL_OUT; + + prev_dst_ip = dst_ip = dloc->ip; + for (int i = nr_scopes - 1; i >= 0; i--) { + Dwarf_Addr base, start, end; + LIST_HEAD(this_blocks); + + if (dwarf_ranges(&scopes[i], 0, &base, &start, &end) < 0) + break; + + pr_debug_dtp("scope: [%d/%d] ", i + 1, nr_scopes); + pr_debug_scope(&scopes[i]); + + src_ip = map__objdump_2rip(dloc->ms->map, start); + +again: + /* Get basic blocks for this scope */ + if (annotate_get_basic_blocks(dloc->ms->sym, src_ip, dst_ip, + &this_blocks) < 0) { + /* Try previous block if they are not connected */ + if (prev_dst_ip != dst_ip) { + dst_ip = prev_dst_ip; + goto again; + } + + pr_debug_dtp("cannot find a basic block from %"PRIx64" to %"PRIx64"\n", + src_ip - dloc->ms->sym->start, + dst_ip - dloc->ms->sym->start); + continue; + } + prepend_basic_blocks(&this_blocks, &basic_blocks); + + /* Get variable info for this scope and add to var_types list */ + die_collect_vars(&scopes[i], &var_types); + fixup_var_address(var_types, start); + + /* Find from start of this scope to the target instruction */ + ret = find_data_type_insn(dloc, &basic_blocks, var_types, + cu_die, type_die); + if (ret == PERF_TMR_OK) { + char buf[64]; + int offset = dloc->op->offset; + const char *offset_sign = ""; + + if (offset < 0) { + offset = -offset; + offset_sign = "-"; + } + + if (dloc->op->multi_regs) + snprintf(buf, sizeof(buf), "reg%d, reg%d", + dloc->op->reg1, dloc->op->reg2); + else + snprintf(buf, sizeof(buf), "reg%d", dloc->op->reg1); + + pr_debug_dtp("found by insn track: %s%#x(%s) type-offset=%#x\n", + offset_sign, offset, buf, dloc->type_offset); + break; + } + + if (ret == PERF_TMR_BAIL_OUT) + break; + + /* Go up to the next scope and find blocks to the start */ + prev_dst_ip = dst_ip; + dst_ip = src_ip; + } + + delete_basic_blocks(&basic_blocks); + delete_var_types(var_types); + return ret; +} + +/* The result will be saved in @type_die */ +static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) +{ + struct annotated_op_loc *loc = dloc->op; + Dwarf_Die cu_die, var_die; + Dwarf_Die *scopes = NULL; + int reg, offset = loc->offset; + int ret = -1; + int i, nr_scopes; + int fbreg = -1; + int fb_offset = 0; + bool is_fbreg = false; + bool found = false; + u64 pc; + char buf[64]; + enum type_match_result result = PERF_TMR_UNKNOWN; + const char *offset_sign = ""; + + if (dloc->op->multi_regs) + snprintf(buf, sizeof(buf), "reg%d, reg%d", dloc->op->reg1, dloc->op->reg2); + else if (dloc->op->reg1 == DWARF_REG_PC) + snprintf(buf, sizeof(buf), "PC"); + else + snprintf(buf, sizeof(buf), "reg%d", dloc->op->reg1); + + if (offset < 0) { + offset = -offset; + offset_sign = "-"; + } + + pr_debug_dtp("-----------------------------------------------------------\n"); + pr_debug_dtp("find data type for %s%#x(%s) at %s+%#"PRIx64"\n", + offset_sign, offset, buf, + dloc->ms->sym->name, dloc->ip - dloc->ms->sym->start); + + /* + * IP is a relative instruction address from the start of the map, as + * it can be randomized/relocated, it needs to translate to PC which is + * a file address for DWARF processing. + */ + pc = map__rip_2objdump(dloc->ms->map, dloc->ip); + + /* Get a compile_unit for this address */ + if (!find_cu_die(dloc->di, pc, &cu_die)) { + pr_debug_dtp("cannot find CU for address %"PRIx64"\n", pc); + ann_data_stat.no_cuinfo++; + return -1; + } + + reg = loc->reg1; + offset = loc->offset; + + pr_debug_dtp("CU for %s (die:%#lx)\n", + die_name(&cu_die), (long)dwarf_dieoffset(&cu_die)); + + if (reg == DWARF_REG_PC) { + if (get_global_var_type(&cu_die, dloc, dloc->ip, dloc->var_addr, + &offset, type_die)) { + dloc->type_offset = offset; + + pr_debug_dtp("found by addr=%#"PRIx64" type_offset=%#x\n", + dloc->var_addr, offset); + pr_debug_type_name(type_die, TSR_KIND_TYPE); + found = true; + goto out; + } + } + + /* Get a list of nested scopes - i.e. (inlined) functions and blocks. */ + nr_scopes = die_get_scopes(&cu_die, pc, &scopes); + + if (reg != DWARF_REG_PC && dwarf_hasattr(&scopes[0], DW_AT_frame_base)) { + Dwarf_Attribute attr; + Dwarf_Block block; + + /* Check if the 'reg' is assigned as frame base register */ + if (dwarf_attr(&scopes[0], DW_AT_frame_base, &attr) != NULL && + dwarf_formblock(&attr, &block) == 0 && block.length == 1) { + switch (*block.data) { + case DW_OP_reg0 ... DW_OP_reg31: + fbreg = dloc->fbreg = *block.data - DW_OP_reg0; + break; + case DW_OP_call_frame_cfa: + dloc->fb_cfa = true; + if (die_get_cfa(dloc->di->dbg, pc, &fbreg, + &fb_offset) < 0) + fbreg = -1; + break; + default: + break; + } + + pr_debug_dtp("frame base: cfa=%d fbreg=%d\n", + dloc->fb_cfa, fbreg); + } + } + +retry: + is_fbreg = (reg == fbreg); + if (is_fbreg) + offset = loc->offset - fb_offset; + + /* Search from the inner-most scope to the outer */ + for (i = nr_scopes - 1; i >= 0; i--) { + Dwarf_Die mem_die; + int type_offset = offset; + + if (reg == DWARF_REG_PC) { + if (!die_find_variable_by_addr(&scopes[i], dloc->var_addr, + &var_die, &mem_die, + &type_offset)) + continue; + } else { + /* Look up variables/parameters in this scope */ + if (!die_find_variable_by_reg(&scopes[i], pc, reg, + &mem_die, &type_offset, is_fbreg, &var_die)) + continue; + } + + pr_debug_dtp("found \"%s\" (die: %#lx) in scope=%d/%d (die: %#lx) ", + die_name(&var_die), (long)dwarf_dieoffset(&var_die), + i+1, nr_scopes, (long)dwarf_dieoffset(&scopes[i])); + + if (reg == DWARF_REG_PC) { + pr_debug_dtp("addr=%#"PRIx64" type_offset=%#x\n", + dloc->var_addr, type_offset); + } else if (reg == DWARF_REG_FB || is_fbreg) { + pr_debug_dtp("stack_offset=%#x type_offset=%#x\n", + fb_offset, type_offset); + } else { + pr_debug_dtp("type_offset=%#x\n", type_offset); + } + + if (!found || dloc->type_offset < type_offset || + (dloc->type_offset == type_offset && + !is_better_type(&mem_die, type_die))) { + *type_die = mem_die; + dloc->type_offset = type_offset; + found = true; + } + + pr_debug_location(&var_die, pc, reg); + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + } + + if (!found && loc->multi_regs && reg == loc->reg1 && loc->reg1 != loc->reg2) { + reg = loc->reg2; + goto retry; + } + + if (!found && reg != DWARF_REG_PC) { + result = find_data_type_block(dloc, &cu_die, scopes, + nr_scopes, type_die); + if (result == PERF_TMR_OK) { + ann_data_stat.insn_track++; + found = true; + } + } + +out: + pr_debug_dtp("final result: "); + if (found) { + pr_debug_type_name(type_die, TSR_KIND_TYPE); + ret = 0; + } else { + switch (result) { + case PERF_TMR_NO_TYPE: + case PERF_TMR_NO_POINTER: + pr_debug_dtp("%s\n", match_result_str(result)); + ann_data_stat.no_typeinfo++; + break; + case PERF_TMR_NO_SIZE: + pr_debug_dtp("%s\n", match_result_str(result)); + ann_data_stat.invalid_size++; + break; + case PERF_TMR_BAD_OFFSET: + pr_debug_dtp("%s\n", match_result_str(result)); + ann_data_stat.bad_offset++; + break; + case PERF_TMR_UNKNOWN: + case PERF_TMR_BAIL_OUT: + case PERF_TMR_OK: /* should not reach here */ + default: + pr_debug_dtp("no variable found\n"); + ann_data_stat.no_var++; + break; + } + ret = -1; + } + + free(scopes); + return ret; +} + +/** + * find_data_type - Return a data type at the location + * @dloc: data location + * + * This functions searches the debug information of the binary to get the data + * type it accesses. The exact location is expressed by (ip, reg, offset) + * for pointer variables or (ip, addr) for global variables. Note that global + * variables might update the @dloc->type_offset after finding the start of the + * variable. If it cannot find a global variable by address, it tried to find + * a declaration of the variable using var_name. In that case, @dloc->offset + * won't be updated. + * + * It return %NULL if not found. + */ +struct annotated_data_type *find_data_type(struct data_loc_info *dloc) +{ + struct dso *dso = map__dso(dloc->ms->map); + Dwarf_Die type_die; + + /* + * The type offset is the same as instruction offset by default. + * But when finding a global variable, the offset won't be valid. + */ + dloc->type_offset = dloc->op->offset; + + dloc->fbreg = -1; + + if (find_data_type_die(dloc, &type_die) < 0) + return NULL; + + return dso__findnew_data_type(dso, &type_die); +} + +static int alloc_data_type_histograms(struct annotated_data_type *adt, int nr_entries) +{ + int i; + size_t sz = sizeof(struct type_hist); + + sz += sizeof(struct type_hist_entry) * adt->self.size; + + /* Allocate a table of pointers for each event */ + adt->histograms = calloc(nr_entries, sizeof(*adt->histograms)); + if (adt->histograms == NULL) + return -ENOMEM; + + /* + * Each histogram is allocated for the whole size of the type. + * TODO: Probably we can move the histogram to members. + */ + for (i = 0; i < nr_entries; i++) { + adt->histograms[i] = zalloc(sz); + if (adt->histograms[i] == NULL) + goto err; + } + + adt->nr_histograms = nr_entries; + return 0; + +err: + while (--i >= 0) + zfree(&(adt->histograms[i])); + zfree(&adt->histograms); + return -ENOMEM; +} + +static void delete_data_type_histograms(struct annotated_data_type *adt) +{ + for (int i = 0; i < adt->nr_histograms; i++) + zfree(&(adt->histograms[i])); + + zfree(&adt->histograms); + adt->nr_histograms = 0; +} + +void annotated_data_type__tree_delete(struct rb_root *root) +{ + struct annotated_data_type *pos; + + while (!RB_EMPTY_ROOT(root)) { + struct rb_node *node = rb_first(root); + + rb_erase(node, root); + pos = rb_entry(node, struct annotated_data_type, node); + delete_members(&pos->self); + delete_data_type_histograms(pos); + zfree(&pos->self.type_name); + free(pos); + } +} + +/** + * annotated_data_type__update_samples - Update histogram + * @adt: Data type to update + * @evsel: Event to update + * @offset: Offset in the type + * @nr_samples: Number of samples at this offset + * @period: Event count at this offset + * + * This function updates type histogram at @ofs for @evsel. Samples are + * aggregated before calling this function so it can be called with more + * than one samples at a certain offset. + */ +int annotated_data_type__update_samples(struct annotated_data_type *adt, + struct evsel *evsel, int offset, + int nr_samples, u64 period) +{ + struct type_hist *h; + + if (adt == NULL) + return 0; + + if (adt->histograms == NULL) { + int nr = evlist__nr_entries(evsel->evlist); + + if (alloc_data_type_histograms(adt, nr) < 0) + return -1; + } + + if (offset < 0 || offset >= adt->self.size) + return -1; + + h = adt->histograms[evsel->core.idx]; + + h->nr_samples += nr_samples; + h->addr[offset].nr_samples += nr_samples; + h->period += period; + h->addr[offset].period += period; + return 0; +} + +static void print_annotated_data_header(struct hist_entry *he, struct evsel *evsel) +{ + struct dso *dso = map__dso(he->ms.map); + int nr_members = 1; + int nr_samples = he->stat.nr_events; + int width = 7; + const char *val_hdr = "Percent"; + + if (evsel__is_group_event(evsel)) { + struct hist_entry *pair; + + list_for_each_entry(pair, &he->pairs.head, pairs.node) + nr_samples += pair->stat.nr_events; + } + + printf("Annotate type: '%s' in %s (%d samples):\n", + he->mem_type->self.type_name, dso__name(dso), nr_samples); + + if (evsel__is_group_event(evsel)) { + struct evsel *pos; + int i = 0; + + nr_members = 0; + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + + printf(" event[%d] = %s\n", i++, pos->name); + nr_members++; + } + } + + if (symbol_conf.show_total_period) { + width = 11; + val_hdr = "Period"; + } else if (symbol_conf.show_nr_samples) { + width = 7; + val_hdr = "Samples"; + } + + printf("============================================================================\n"); + printf("%*s %10s %10s %s\n", (width + 1) * nr_members, val_hdr, + "offset", "size", "field"); +} + +static void print_annotated_data_value(struct type_hist *h, u64 period, int nr_samples) +{ + double percent = h->period ? (100.0 * period / h->period) : 0; + const char *color = get_percent_color(percent); + + if (symbol_conf.show_total_period) + color_fprintf(stdout, color, " %11" PRIu64, period); + else if (symbol_conf.show_nr_samples) + color_fprintf(stdout, color, " %7d", nr_samples); + else + color_fprintf(stdout, color, " %7.2f", percent); +} + +static void print_annotated_data_type(struct annotated_data_type *mem_type, + struct annotated_member *member, + struct evsel *evsel, int indent) +{ + struct annotated_member *child; + struct type_hist *h = mem_type->histograms[evsel->core.idx]; + int i, nr_events = 0, samples = 0; + u64 period = 0; + int width = symbol_conf.show_total_period ? 11 : 7; + struct evsel *pos; + + for_each_group_evsel(pos, evsel) { + h = mem_type->histograms[pos->core.idx]; + + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + + samples = 0; + period = 0; + for (i = 0; i < member->size; i++) { + samples += h->addr[member->offset + i].nr_samples; + period += h->addr[member->offset + i].period; + } + print_annotated_data_value(h, period, samples); + nr_events++; + } + + printf(" %#10x %#10x %*s%s\t%s", + member->offset, member->size, indent, "", member->type_name, + member->var_name ?: ""); + + if (!list_empty(&member->children)) + printf(" {\n"); + + list_for_each_entry(child, &member->children, node) + print_annotated_data_type(mem_type, child, evsel, indent + 4); + + if (!list_empty(&member->children)) + printf("%*s}", (width + 1) * nr_events + 24 + indent, ""); + printf(";\n"); +} + +int hist_entry__annotate_data_tty(struct hist_entry *he, struct evsel *evsel) +{ + print_annotated_data_header(he, evsel); + print_annotated_data_type(he->mem_type, &he->mem_type->self, evsel, 0); + printf("\n"); + + /* move to the next entry */ + return '>'; +} diff --git a/tools/perf/util/annotate-data.h b/tools/perf/util/annotate-data.h new file mode 100644 index 000000000..c26130744 --- /dev/null +++ b/tools/perf/util/annotate-data.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _PERF_ANNOTATE_DATA_H +#define _PERF_ANNOTATE_DATA_H + +#include +#include +#include +#include +#include "dwarf-regs.h" +#include "annotate.h" + +#ifdef HAVE_LIBDW_SUPPORT +#include "debuginfo.h" +#endif + +struct annotated_op_loc; +struct debuginfo; +struct evsel; +struct hist_browser_timer; +struct hist_entry; +struct map_symbol; +struct thread; + +#define pr_debug_dtp(fmt, ...) \ +do { \ + if (debug_type_profile) \ + pr_info(fmt, ##__VA_ARGS__); \ + else \ + pr_debug3(fmt, ##__VA_ARGS__); \ +} while (0) + +enum type_state_kind { + TSR_KIND_INVALID = 0, + TSR_KIND_TYPE, + TSR_KIND_PERCPU_BASE, + TSR_KIND_CONST, + TSR_KIND_PERCPU_POINTER, + TSR_KIND_POINTER, + TSR_KIND_CANARY, +}; + +/** + * struct annotated_member - Type of member field + * @node: List entry in the parent list + * @children: List head for child nodes + * @type_name: Name of the member type + * @var_name: Name of the member variable + * @offset: Offset from the outer data type + * @size: Size of the member field + * + * This represents a member type in a data type. + */ +struct annotated_member { + struct list_head node; + struct list_head children; + char *type_name; + char *var_name; + int offset; + int size; +}; + +/** + * struct type_hist_entry - Histogram entry per offset + * @nr_samples: Number of samples + * @period: Count of event + */ +struct type_hist_entry { + int nr_samples; + u64 period; +}; + +/** + * struct type_hist - Type histogram for each event + * @nr_samples: Total number of samples in this data type + * @period: Total count of the event in this data type + * @offset: Array of histogram entry + */ +struct type_hist { + u64 nr_samples; + u64 period; + struct type_hist_entry addr[]; +}; + +/** + * struct annotated_data_type - Data type to profile + * @node: RB-tree node for dso->type_tree + * @self: Actual type information + * @nr_histogram: Number of histogram entries + * @histograms: An array of pointers to histograms + * + * This represents a data type accessed by samples in the profile data. + */ +struct annotated_data_type { + struct rb_node node; + struct annotated_member self; + int nr_histograms; + struct type_hist **histograms; +}; + +extern struct annotated_data_type unknown_type; +extern struct annotated_data_type stackop_type; +extern struct annotated_data_type canary_type; + +/** + * struct data_loc_info - Data location information + * @arch: CPU architecture info + * @thread: Thread info + * @ms: Map and Symbol info + * @ip: Instruction address + * @var_addr: Data address (for global variables) + * @cpumode: CPU execution mode + * @op: Instruction operand location (regs and offset) + * @di: Debug info + * @fbreg: Frame base register + * @fb_cfa: Whether the frame needs to check CFA + * @type_offset: Final offset in the type + */ +struct data_loc_info { + /* These are input field, should be filled by caller */ + const struct arch *arch; + struct thread *thread; + struct map_symbol *ms; + u64 ip; + u64 var_addr; + u8 cpumode; + struct annotated_op_loc *op; + struct debuginfo *di; + + /* These are used internally */ + int fbreg; + bool fb_cfa; + + /* This is for the result */ + int type_offset; +}; + +/** + * struct annotated_data_stat - Debug statistics + * @total: Total number of entry + * @no_sym: No symbol or map found + * @no_insn: Failed to get disasm line + * @no_insn_ops: The instruction has no operands + * @no_mem_ops: The instruction has no memory operands + * @no_reg: Failed to extract a register from the operand + * @no_dbginfo: The binary has no debug information + * @no_cuinfo: Failed to find a compile_unit + * @no_var: Failed to find a matching variable + * @no_typeinfo: Failed to get a type info for the variable + * @invalid_size: Failed to get a size info of the type + * @bad_offset: The access offset is out of the type + */ +struct annotated_data_stat { + int total; + int no_sym; + int no_insn; + int no_insn_ops; + int no_mem_ops; + int no_reg; + int no_dbginfo; + int no_cuinfo; + int no_var; + int no_typeinfo; + int invalid_size; + int bad_offset; + int insn_track; +}; +extern struct annotated_data_stat ann_data_stat; + +#ifdef HAVE_LIBDW_SUPPORT +/* + * Type information in a register, valid when @ok is true. + * The @caller_saved registers are invalidated after a function call. + */ +struct type_state_reg { + Dwarf_Die type; + u32 imm_value; + /* + * The offset within the struct that the register points to. + * A value of 0 means the register points to the beginning. + * type_offset = op->offset + reg->offset + */ + s32 offset; + bool ok; + bool caller_saved; + /* DWARF location range tracking for register lifetime */ + bool lifetime_active; + u64 lifetime_end; + u8 kind; + u8 copied_from; +}; + +/* Type information in a stack location, dynamically allocated */ +struct type_state_stack { + struct list_head list; + Dwarf_Die type; + int offset; + /* pointer offset, saves tsr->offset on the stack state */ + int ptr_offset; + int size; + bool compound; + u8 kind; +}; + +/* + * Maximum number of registers tracked in type_state. + * + * This limit must cover all supported architectures, since perf + * may analyze perf.data files generated on systems with a different + * register set. Use 32 as a safe upper bound instead of relying on + * build-arch specific values. + */ +#define TYPE_STATE_MAX_REGS 32 + +/* + * State table to maintain type info in each register and stack location. + * It'll be updated when new variable is allocated or type info is moved + * to a new location (register or stack). As it'd be used with the + * shortest path of basic blocks, it only maintains a single table. + */ +struct type_state { + /* state of general purpose registers */ + struct type_state_reg regs[TYPE_STATE_MAX_REGS]; + /* state of stack location */ + struct list_head stack_vars; + /* return value register */ + int ret_reg; + /* stack pointer register */ + int stack_reg; +}; + +/* Returns data type at the location (ip, reg, offset) */ +struct annotated_data_type *find_data_type(struct data_loc_info *dloc); + +/* Update type access histogram at the given offset */ +int annotated_data_type__update_samples(struct annotated_data_type *adt, + struct evsel *evsel, int offset, + int nr_samples, u64 period); + +/* Release all data type information in the tree */ +void annotated_data_type__tree_delete(struct rb_root *root); + +/* Release all global variable information in the tree */ +void global_var_type__tree_delete(struct rb_root *root); + +/* Print data type annotation (including members) on stdout */ +int hist_entry__annotate_data_tty(struct hist_entry *he, struct evsel *evsel); + +/* Get name of member field at the given offset in the data type */ +int annotated_data_type__get_member_name(struct annotated_data_type *adt, + char *buf, size_t sz, int member_offset); + +bool has_reg_type(struct type_state *state, int reg); +struct type_state_stack *findnew_stack_state(struct type_state *state, + int offset, u8 kind, + Dwarf_Die *type_die, + int ptr_offset); +void set_stack_state(struct type_state_stack *stack, int offset, u8 kind, + Dwarf_Die *type_die, int ptr_offset); +struct type_state_stack *find_stack_state(struct type_state *state, + int offset); +bool get_global_var_type(Dwarf_Die *cu_die, struct data_loc_info *dloc, + u64 ip, u64 var_addr, int *var_offset, + Dwarf_Die *type_die); +bool get_global_var_info(struct data_loc_info *dloc, u64 addr, + const char **var_name, int *var_offset); +void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind); + +#else /* HAVE_LIBDW_SUPPORT */ + +static inline struct annotated_data_type * +find_data_type(struct data_loc_info *dloc __maybe_unused) +{ + return NULL; +} + +static inline int +annotated_data_type__update_samples(struct annotated_data_type *adt __maybe_unused, + struct evsel *evsel __maybe_unused, + int offset __maybe_unused, + int nr_samples __maybe_unused, + u64 period __maybe_unused) +{ + return -1; +} + +static inline void annotated_data_type__tree_delete(struct rb_root *root __maybe_unused) +{ +} + +static inline void global_var_type__tree_delete(struct rb_root *root __maybe_unused) +{ +} + +static inline int hist_entry__annotate_data_tty(struct hist_entry *he __maybe_unused, + struct evsel *evsel __maybe_unused) +{ + return -1; +} + +static inline int annotated_data_type__get_member_name(struct annotated_data_type *adt __maybe_unused, + char *buf __maybe_unused, + size_t sz __maybe_unused, + int member_offset __maybe_unused) +{ + return -1; +} + +#endif /* HAVE_LIBDW_SUPPORT */ + +#ifdef HAVE_SLANG_SUPPORT +int hist_entry__annotate_data_tui(struct hist_entry *he, struct evsel *evsel, + struct hist_browser_timer *hbt); +#else +static inline int hist_entry__annotate_data_tui(struct hist_entry *he __maybe_unused, + struct evsel *evsel __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused) +{ + return -1; +} +#endif /* HAVE_SLANG_SUPPORT */ + +#endif /* _PERF_ANNOTATE_DATA_H */ diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c new file mode 100644 index 000000000..df70e95a8 --- /dev/null +++ b/tools/perf/util/annotate.c @@ -0,0 +1,3228 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo + * + * Parts came from builtin-annotate.c, see those files for further + * copyright notes. + */ + +#include +#include +#include +#include "util.h" // hex_width() +#include "ui/ui.h" +#include "sort.h" +#include "build-id.h" +#include "color.h" +#include "config.h" +#include "disasm.h" +#include "dso.h" +#include "env.h" +#include "map.h" +#include "maps.h" +#include "symbol.h" +#include "srcline.h" +#include "units.h" +#include "debug.h" +#include "debuginfo.h" +#include "annotate.h" +#include "annotate-data.h" +#include "evsel.h" +#include "evlist.h" +#include "bpf-event.h" +#include "bpf-utils.h" +#include "block-range.h" +#include "string2.h" +#include "dwarf-regs.h" +#include "util/event.h" +#include "util/sharded_mutex.h" +#include "arch/common.h" +#include "namespaces.h" +#include "thread.h" +#include "hashmap.h" +#include "strbuf.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* FIXME: For the HE_COLORSET */ +#include "ui/browser.h" + +/* + * FIXME: Using the same values as slang.h, + * but that header may not be available everywhere + */ +#define LARROW_CHAR ((unsigned char)',') +#define RARROW_CHAR ((unsigned char)'+') +#define DARROW_CHAR ((unsigned char)'.') +#define UARROW_CHAR ((unsigned char)'-') + +#include + +/* global annotation options */ +struct annotation_options annotate_opts; + +/* Data type collection debug statistics */ +struct annotated_data_stat ann_data_stat; +LIST_HEAD(ann_insn_stat); + +/* Pseudo data types */ +struct annotated_data_type stackop_type = { + .self = { + .type_name = (char *)"(stack operation)", + .children = LIST_HEAD_INIT(stackop_type.self.children), + }, +}; + +struct annotated_data_type canary_type = { + .self = { + .type_name = (char *)"(stack canary)", + .children = LIST_HEAD_INIT(canary_type.self.children), + }, +}; + +#define NO_TYPE ((struct annotated_data_type *)-1UL) + +/* symbol histogram: key = offset << 16 | evsel->core.idx */ +static size_t sym_hist_hash(long key, void *ctx __maybe_unused) +{ + return (key >> 16) + (key & 0xffff); +} + +static bool sym_hist_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +static struct annotated_source *annotated_source__new(void) +{ + struct annotated_source *src = zalloc(sizeof(*src)); + + if (src != NULL) + INIT_LIST_HEAD(&src->source); + + return src; +} + +static __maybe_unused void annotated_source__delete(struct annotated_source *src) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (src == NULL) + return; + + if (src->samples) { + hashmap__for_each_entry(src->samples, cur, bkt) + zfree(&cur->pvalue); + hashmap__free(src->samples); + } + zfree(&src->histograms); + free(src); +} + +static int annotated_source__alloc_histograms(struct annotated_source *src, + int nr_hists) +{ + src->nr_histograms = nr_hists; + src->histograms = calloc(nr_hists, sizeof(*src->histograms)); + + if (src->histograms == NULL) + return -1; + + src->samples = hashmap__new(sym_hist_hash, sym_hist_equal, NULL); + if (IS_ERR(src->samples)) { + zfree(&src->histograms); + src->samples = NULL; + } + + return src->histograms ? 0 : -1; +} + +void symbol__annotate_zero_histograms(struct symbol *sym) +{ + struct annotation *notes = symbol__annotation(sym); + + annotation__lock(notes); + if (notes->src != NULL) { + memset(notes->src->histograms, 0, + notes->src->nr_histograms * sizeof(*notes->src->histograms)); + hashmap__clear(notes->src->samples); + } + if (notes->branch && notes->branch->cycles_hist) { + memset(notes->branch->cycles_hist, 0, + symbol__size(sym) * sizeof(struct cyc_hist)); + } + annotation__unlock(notes); +} + +static int __symbol__account_cycles(struct cyc_hist *ch, + u64 start, + unsigned offset, unsigned cycles, + unsigned have_start) +{ + /* + * For now we can only account one basic block per + * final jump. But multiple could be overlapping. + * Always account the longest one. So when + * a shorter one has been already seen throw it away. + * + * We separately always account the full cycles. + */ + ch[offset].num_aggr++; + ch[offset].cycles_aggr += cycles; + + if (cycles > ch[offset].cycles_max) + ch[offset].cycles_max = cycles; + + if (ch[offset].cycles_min) { + if (cycles && cycles < ch[offset].cycles_min) + ch[offset].cycles_min = cycles; + } else + ch[offset].cycles_min = cycles; + + if (!have_start && ch[offset].have_start) + return 0; + if (ch[offset].num) { + if (have_start && (!ch[offset].have_start || + ch[offset].start > start)) { + ch[offset].have_start = 0; + ch[offset].cycles = 0; + ch[offset].num = 0; + if (ch[offset].reset < 0xffff) + ch[offset].reset++; + } else if (have_start && + ch[offset].start < start) + return 0; + } + + if (ch[offset].num < NUM_SPARKS) + ch[offset].cycles_spark[ch[offset].num] = cycles; + + ch[offset].have_start = have_start; + ch[offset].start = start; + ch[offset].cycles += cycles; + ch[offset].num++; + return 0; +} + +static int __symbol__inc_addr_samples(struct map_symbol *ms, + struct annotated_source *src, u64 addr, + struct perf_sample *sample) +{ + struct evsel *evsel = sample->evsel; + struct symbol *sym = ms->sym; + long hash_key; + u64 offset; + struct sym_hist *h; + struct sym_hist_entry *entry; + + pr_debug3("%s: addr=%#" PRIx64 "\n", __func__, map__unmap_ip(ms->map, addr)); + + if ((addr < sym->start || addr >= sym->end) && + (addr != sym->end || sym->start != sym->end)) { + pr_debug("%s(%d): ERANGE! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 "\n", + __func__, __LINE__, sym->name, sym->start, addr, sym->end); + return -ERANGE; + } + + offset = addr - sym->start; + h = annotated_source__histogram(src, evsel); + if (h == NULL) { + pr_debug("%s(%d): ENOMEM! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 ", func: %d\n", + __func__, __LINE__, sym->name, sym->start, addr, sym->end, + symbol__type(sym) == STT_FUNC); + return -ENOMEM; + } + + hash_key = offset << 16 | evsel->core.idx; + if (!hashmap__find(src->samples, hash_key, &entry)) { + entry = zalloc(sizeof(*entry)); + if (entry == NULL) + return -ENOMEM; + + if (hashmap__add(src->samples, hash_key, entry) < 0) + return -ENOMEM; + } + + h->nr_samples++; + h->period += sample->period; + entry->nr_samples++; + entry->period += sample->period; + + pr_debug3("%#" PRIx64 " %s: period++ [addr: %#" PRIx64 ", %#" PRIx64 + ", evidx=%d] => nr_samples: %" PRIu64 ", period: %" PRIu64 "\n", + sym->start, sym->name, addr, addr - sym->start, evsel->core.idx, + entry->nr_samples, entry->period); + return 0; +} + +struct annotated_branch *annotation__get_branch(struct annotation *notes) +{ + if (notes == NULL) + return NULL; + + if (notes->branch == NULL) + notes->branch = zalloc(sizeof(*notes->branch)); + + return notes->branch; +} + +static struct annotated_branch *symbol__find_branch_hist(struct symbol *sym, + unsigned int br_cntr_nr) +{ + struct annotation *notes = symbol__annotation(sym); + struct annotated_branch *branch; + const size_t size = symbol__size(sym); + + branch = annotation__get_branch(notes); + if (branch == NULL) + return NULL; + + if (branch->cycles_hist == NULL) { + branch->cycles_hist = calloc(size, sizeof(struct cyc_hist)); + if (!branch->cycles_hist) + return NULL; + } + + if (br_cntr_nr && branch->br_cntr == NULL) { + branch->br_cntr = calloc(br_cntr_nr * size, sizeof(u64)); + if (!branch->br_cntr) + return NULL; + } + + return branch; +} + +struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists) +{ + struct annotation *notes = symbol__annotation(sym); + + if (notes->src == NULL) { + notes->src = annotated_source__new(); + if (notes->src == NULL) + return NULL; + goto alloc_histograms; + } + + if (notes->src->histograms == NULL) { +alloc_histograms: + annotated_source__alloc_histograms(notes->src, nr_hists); + } + + return notes->src; +} + +static int symbol__inc_addr_samples(struct map_symbol *ms, + u64 addr, + struct perf_sample *sample) +{ + struct symbol *sym = ms->sym; + struct annotated_source *src; + + if (sym == NULL) + return 0; + src = symbol__hists(sym, evlist__nr_entries(sample->evsel->evlist)); + return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0; +} + +static int symbol__account_br_cntr(struct annotated_branch *branch, + struct evsel *evsel, + unsigned offset, + u64 br_cntr) +{ + unsigned int br_cntr_nr = evsel__leader(evsel)->br_cntr_nr; + unsigned int base = evsel__leader(evsel)->br_cntr_idx; + unsigned int off = offset * evlist__nr_br_cntr(evsel->evlist); + u64 *branch_br_cntr = branch->br_cntr; + unsigned int i, mask, width; + + if (!br_cntr || !branch_br_cntr) + return 0; + + perf_env__find_br_cntr_info(evsel__env(evsel), NULL, &width); + mask = (1L << width) - 1; + for (i = 0; i < br_cntr_nr; i++) { + u64 cntr = (br_cntr >> i * width) & mask; + + branch_br_cntr[off + i + base] += cntr; + if (cntr == mask) + branch_br_cntr[off + i + base] |= ANNOTATION__BR_CNTR_SATURATED_FLAG; + } + + return 0; +} + +static int symbol__account_cycles(u64 addr, u64 start, struct symbol *sym, + unsigned cycles, struct evsel *evsel, + u64 br_cntr) +{ + struct annotated_branch *branch; + unsigned offset; + int ret; + + if (sym == NULL) + return 0; + branch = symbol__find_branch_hist(sym, evlist__nr_br_cntr(evsel->evlist)); + if (!branch) + return -ENOMEM; + if (addr < sym->start || addr >= sym->end) + return -ERANGE; + + if (start) { + if (start < sym->start || start >= sym->end) + return -ERANGE; + if (start >= addr) + start = 0; + } + offset = addr - sym->start; + ret = __symbol__account_cycles(branch->cycles_hist, + start ? start - sym->start : 0, + offset, cycles, + !!start); + + if (ret) + return ret; + + return symbol__account_br_cntr(branch, evsel, offset, br_cntr); +} + +int addr_map_symbol__account_cycles(struct addr_map_symbol *ams, + struct addr_map_symbol *start, + unsigned cycles, + struct evsel *evsel, + u64 br_cntr) +{ + u64 saddr = 0; + int err; + + if (!cycles) + return 0; + + /* + * Only set start when IPC can be computed. We can only + * compute it when the basic block is completely in a single + * function. + * Special case the case when the jump is elsewhere, but + * it starts on the function start. + */ + if (start && + (start->ms.sym == ams->ms.sym || + (ams->ms.sym && + start->addr == ams->ms.sym->start + map__start(ams->ms.map)))) + saddr = start->al_addr; + if (saddr == 0) + pr_debug2("BB with bad start: addr %"PRIx64" start %"PRIx64" sym %"PRIx64" saddr %"PRIx64"\n", + ams->addr, + start ? start->addr : 0, + ams->ms.sym ? ams->ms.sym->start + map__start(ams->ms.map) : 0, + saddr); + err = symbol__account_cycles(ams->al_addr, saddr, ams->ms.sym, cycles, evsel, br_cntr); + if (err) + pr_debug2("account_cycles failed %d\n", err); + return err; +} + +struct annotation_line *annotated_source__get_line(struct annotated_source *src, + s64 offset) +{ + struct annotation_line *al; + + list_for_each_entry(al, &src->source, node) { + if (al->offset == offset) + return al; + } + return NULL; +} + +static unsigned annotation__count_insn(struct annotation *notes, u64 start, u64 end) +{ + struct annotation_line *al; + unsigned n_insn = 0; + + al = annotated_source__get_line(notes->src, start); + if (al == NULL) + return 0; + + list_for_each_entry_from(al, ¬es->src->source, node) { + if (al->offset == -1) + continue; + if ((u64)al->offset > end) + break; + n_insn++; + } + return n_insn; +} + +static void annotated_branch__delete(struct annotated_branch *branch) +{ + if (branch) { + zfree(&branch->cycles_hist); + free(branch->br_cntr); + free(branch); + } +} + +static void annotation__count_and_fill(struct annotation *notes, u64 start, u64 end, struct cyc_hist *ch) +{ + unsigned n_insn; + unsigned int cover_insn = 0; + + n_insn = annotation__count_insn(notes, start, end); + if (n_insn && ch->num && ch->cycles) { + struct annotation_line *al; + struct annotated_branch *branch; + float ipc = n_insn / ((double)ch->cycles / (double)ch->num); + + /* Hide data when there are too many overlaps. */ + if (ch->reset >= 0x7fff) + return; + + al = annotated_source__get_line(notes->src, start); + if (al == NULL) + return; + + list_for_each_entry_from(al, ¬es->src->source, node) { + if (al->offset == -1) + continue; + if ((u64)al->offset > end) + break; + if (al->cycles && al->cycles->ipc == 0.0) { + al->cycles->ipc = ipc; + cover_insn++; + } + } + + branch = annotation__get_branch(notes); + if (cover_insn && branch) { + branch->hit_cycles += ch->cycles; + branch->hit_insn += n_insn * ch->num; + branch->cover_insn += cover_insn; + } + } +} + +static int annotation__compute_ipc(struct annotation *notes, size_t size, + struct evsel *evsel) +{ + unsigned int br_cntr_nr = evlist__nr_br_cntr(evsel->evlist); + int err = 0; + s64 offset; + + if (!notes->branch || !notes->branch->cycles_hist) + return 0; + + notes->branch->total_insn = annotation__count_insn(notes, 0, size - 1); + notes->branch->hit_cycles = 0; + notes->branch->hit_insn = 0; + notes->branch->cover_insn = 0; + + annotation__lock(notes); + for (offset = size - 1; offset >= 0; --offset) { + struct cyc_hist *ch; + + ch = ¬es->branch->cycles_hist[offset]; + if (ch && ch->cycles) { + struct annotation_line *al; + + al = annotated_source__get_line(notes->src, offset); + if (al && al->cycles == NULL) { + al->cycles = zalloc(sizeof(*al->cycles)); + if (al->cycles == NULL) { + err = ENOMEM; + break; + } + } + if (ch->have_start) + annotation__count_and_fill(notes, ch->start, offset, ch); + if (al && ch->num_aggr) { + al->cycles->avg = ch->cycles_aggr / ch->num_aggr; + al->cycles->max = ch->cycles_max; + al->cycles->min = ch->cycles_min; + } + if (al && notes->branch->br_cntr) { + if (!al->br_cntr) { + al->br_cntr = calloc(br_cntr_nr, sizeof(u64)); + if (!al->br_cntr) { + err = ENOMEM; + break; + } + } + al->num_aggr = ch->num_aggr; + al->br_cntr_nr = br_cntr_nr; + al->evsel = evsel; + memcpy(al->br_cntr, ¬es->branch->br_cntr[offset * br_cntr_nr], + br_cntr_nr * sizeof(u64)); + } + } + } + + if (err) { + while (++offset < (s64)size) { + struct cyc_hist *ch = ¬es->branch->cycles_hist[offset]; + + if (ch && ch->cycles) { + struct annotation_line *al; + + al = annotated_source__get_line(notes->src, offset); + if (al) { + zfree(&al->cycles); + zfree(&al->br_cntr); + } + } + } + } + + annotation__unlock(notes); + return 0; +} + +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample) +{ + return symbol__inc_addr_samples(&ams->ms, ams->al_addr, sample); +} + +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 ip) +{ + return symbol__inc_addr_samples(&he->ms, ip, sample); +} + + +void annotation__exit(struct annotation *notes) +{ + annotated_source__delete(notes->src); + annotated_branch__delete(notes->branch); +} + +static struct sharded_mutex *sharded_mutex; + +static void annotation__init_sharded_mutex(void) +{ + /* As many mutexes as there are CPUs. */ + sharded_mutex = sharded_mutex__new(cpu__max_present_cpu().cpu); +} + +static size_t annotation__hash(const struct annotation *notes) +{ + return (size_t)notes; +} + +static struct mutex *annotation__get_mutex(const struct annotation *notes) +{ + static pthread_once_t once = PTHREAD_ONCE_INIT; + + pthread_once(&once, annotation__init_sharded_mutex); + if (!sharded_mutex) + return NULL; + + return sharded_mutex__get_mutex(sharded_mutex, annotation__hash(notes)); +} + +void annotation__lock(struct annotation *notes) + NO_THREAD_SAFETY_ANALYSIS +{ + struct mutex *mutex = annotation__get_mutex(notes); + + if (mutex) + mutex_lock(mutex); +} + +void annotation__unlock(struct annotation *notes) + NO_THREAD_SAFETY_ANALYSIS +{ + struct mutex *mutex = annotation__get_mutex(notes); + + if (mutex) + mutex_unlock(mutex); +} + +bool annotation__trylock(struct annotation *notes) +{ + struct mutex *mutex = annotation__get_mutex(notes); + + if (!mutex) + return false; + + return mutex_trylock(mutex); +} + +void annotation_line__add(struct annotation_line *al, struct list_head *head) +{ + list_add_tail(&al->node, head); +} + +struct annotation_line * +annotation_line__next(struct annotation_line *pos, struct list_head *head) +{ + list_for_each_entry_continue(pos, head, node) + if (pos->offset >= 0) + return pos; + + return NULL; +} + +static const char *annotate__address_color(struct block_range *br) +{ + double cov = block_range__coverage(br); + + if (cov >= 0) { + /* mark red for >75% coverage */ + if (cov > 0.75) + return PERF_COLOR_RED; + + /* mark dull for <1% coverage */ + if (cov < 0.01) + return PERF_COLOR_NORMAL; + } + + return PERF_COLOR_MAGENTA; +} + +static const char *annotate__asm_color(struct block_range *br) +{ + double cov = block_range__coverage(br); + + if (cov >= 0) { + /* mark dull for <1% coverage */ + if (cov < 0.01) + return PERF_COLOR_NORMAL; + } + + return PERF_COLOR_BLUE; +} + +static void annotate__branch_printf(struct block_range *br, u64 addr) +{ + bool emit_comment = true; + + if (!br) + return; + +#if 1 + if (br->is_target && br->start == addr) { + struct block_range *branch = br; + double p; + + /* + * Find matching branch to our target. + */ + while (!branch->is_branch) + branch = block_range__next(branch); + + p = 100 *(double)br->entry / branch->coverage; + + if (p > 0.1) { + if (emit_comment) { + emit_comment = false; + printf("\t#"); + } + + /* + * The percentage of coverage joined at this target in relation + * to the next branch. + */ + printf(" +%.2f%%", p); + } + } +#endif + if (br->is_branch && br->end == addr) { + double p = 100*(double)br->taken / br->coverage; + + if (p > 0.1) { + if (emit_comment) { + emit_comment = false; + printf("\t#"); + } + + /* + * The percentage of coverage leaving at this branch, and + * its prediction ratio. + */ + printf(" -%.2f%% (p:%.2f%%)", p, 100*(double)br->pred / br->taken); + } + } +} + +static int disasm_line__print(struct disasm_line *dl, u64 start, int addr_fmt_width) +{ + s64 offset = dl->al.offset; + const u64 addr = start + offset; + struct block_range *br; + + br = block_range__find(addr); + color_fprintf(stdout, annotate__address_color(br), " %*" PRIx64 ":", addr_fmt_width, addr); + color_fprintf(stdout, annotate__asm_color(br), "%s", dl->al.line); + annotate__branch_printf(br, addr); + return 0; +} + +static struct annotated_data_type * +__hist_entry__get_data_type(struct hist_entry *he, const struct arch *arch, + struct debuginfo *dbg, struct disasm_line *dl, + int *type_offset); + +static bool needs_type_info(struct annotated_data_type *data_type) +{ + if (data_type == NULL || data_type == NO_TYPE) + return false; + + if (verbose) + return true; + + return (data_type != &stackop_type) && (data_type != &canary_type); +} + +static int +annotation_line__print(struct annotation_line *al, struct annotation_print_data *apd, + struct annotation_options *opts, int printed, + struct annotation_line *queue) +{ + struct symbol *sym = apd->he->ms.sym; + struct disasm_line *dl = container_of(al, struct disasm_line, al); + struct annotation *notes = symbol__annotation(sym); + static const char *prev_line; + int max_lines = opts->max_lines; + int percent_type = opts->percent_type; + + if (al->offset != -1) { + double max_percent = 0.0; + int i, nr_percent = 1; + const char *color; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + percent_type); + + if (percent > max_percent) + max_percent = percent; + } + + if (al->data_nr > nr_percent) + nr_percent = al->data_nr; + + if (max_percent < opts->min_pcnt) + return -1; + + if (max_lines && printed >= max_lines) + return 1; + + if (queue != NULL) { + struct annotation_options queue_opts = { + .max_lines = 1, + .percent_type = percent_type, + }; + + list_for_each_entry_from(queue, ¬es->src->source, node) { + if (queue == al) + break; + annotation_line__print(queue, apd, &queue_opts, + /*printed=*/0, /*queue=*/NULL); + } + } + + color = get_percent_color(max_percent); + + for (i = 0; i < nr_percent; i++) { + struct annotation_data *data = &al->data[i]; + double percent; + + percent = annotation_data__percent(data, percent_type); + color = get_percent_color(percent); + + if (symbol_conf.show_total_period) + color_fprintf(stdout, color, " %11" PRIu64, + data->he.period); + else if (symbol_conf.show_nr_samples) + color_fprintf(stdout, color, " %7" PRIu64, + data->he.nr_samples); + else + color_fprintf(stdout, color, " %7.2f", percent); + } + + printf(" : "); + + disasm_line__print(dl, notes->src->start, apd->addr_fmt_width); + + if (opts->code_with_type && apd->dbg) { + struct annotated_data_type *data_type; + int offset = 0; + + data_type = __hist_entry__get_data_type(apd->he, apd->arch, + apd->dbg, dl, &offset); + if (needs_type_info(data_type)) { + char buf[4096]; + + printf("\t\t# data-type: %s", + data_type->self.type_name); + + if (data_type != &stackop_type && + data_type != &canary_type) + printf(" +%#x", offset); + + if (annotated_data_type__get_member_name(data_type, + buf, + sizeof(buf), + offset)) + printf(" (%s)", buf); + } + } + + /* + * Also color the filename and line if needed, with + * the same color than the percentage. Don't print it + * twice for close colored addr with the same filename:line + */ + if (al->path) { + if (!prev_line || strcmp(prev_line, al->path)) { + color_fprintf(stdout, color, " // %s", al->path); + prev_line = al->path; + } + } + + printf("\n"); + } else if (max_lines && printed >= max_lines) + return 1; + else { + int width = annotation__pcnt_width(notes); + + if (queue) + return -1; + + if (!*al->line) + printf(" %*s:\n", width, " "); + else + printf(" %*s: %-*d %s\n", width, " ", apd->addr_fmt_width, + al->line_nr, al->line); + } + + return 0; +} + +static void calc_percent(struct annotation *notes, + struct evsel *evsel, + struct annotation_data *data, + s64 offset, s64 end) +{ + struct hists *hists = evsel__hists(evsel); + struct sym_hist *sym_hist = annotation__histogram(notes, evsel); + unsigned int hits = 0; + u64 period = 0; + + while (offset < end) { + struct sym_hist_entry *entry; + + entry = annotated_source__hist_entry(notes->src, evsel, offset); + if (entry) { + hits += entry->nr_samples; + period += entry->period; + } + ++offset; + } + + if (sym_hist->nr_samples) { + data->he.period = period; + data->he.nr_samples = hits; + data->percent[PERCENT_HITS_LOCAL] = 100.0 * hits / sym_hist->nr_samples; + } + + if (hists->stats.nr_non_filtered_samples) + data->percent[PERCENT_HITS_GLOBAL] = 100.0 * hits / hists->stats.nr_non_filtered_samples; + + if (sym_hist->period) + data->percent[PERCENT_PERIOD_LOCAL] = 100.0 * period / sym_hist->period; + + if (hists->stats.total_period) + data->percent[PERCENT_PERIOD_GLOBAL] = 100.0 * period / hists->stats.total_period; +} + +static void annotation__calc_percent(struct annotation *notes, + struct evsel *leader, s64 len) +{ + struct annotation_line *al, *next; + struct evsel *evsel; + + list_for_each_entry(al, ¬es->src->source, node) { + s64 end; + int i = 0; + + if (al->offset == -1) + continue; + + next = annotation_line__next(al, ¬es->src->source); + end = next ? next->offset : len; + + for_each_group_evsel(evsel, leader) { + struct annotation_data *data; + + BUG_ON(i >= al->data_nr); + + if (symbol_conf.skip_empty && + evsel__hists(evsel)->stats.nr_samples == 0) + continue; + + data = &al->data[i++]; + + calc_percent(notes, evsel, data, al->offset, end); + } + } +} + +void symbol__calc_percent(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + + annotation__calc_percent(notes, evsel, symbol__size(sym)); +} + + + +int map_symbol__get_arch(struct map_symbol *ms, const struct arch **parch) +{ + const struct arch *arch; + struct machine *machine = NULL; + struct map *map = ms->map; + struct dso *dso = map ? map__dso(map) : NULL; + uint32_t e_flags = 0; + uint16_t e_machine = EM_NONE; + + const char *cpuid = NULL; + + if (ms->thread) { + machine = maps__machine(thread__maps(ms->thread)); + e_machine = thread__e_machine(ms->thread, machine, &e_flags); + if (machine && machine->env) + cpuid = machine->env->cpuid; + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine(dso, kmap_machine, &e_flags); + if (kmap_machine && kmap_machine->env) + cpuid = kmap_machine->env->cpuid; + } + + if (e_machine == EM_NONE) + e_machine = thread__e_machine(NULL, NULL, &e_flags); + + arch = arch__find(e_machine, e_flags, cpuid); + if (arch == NULL) { + pr_err("%s: unsupported arch %d\n", __func__, e_machine); + /* TODO: Refactor annotate/disassemble subsystem error + * codes to uniformly return negative integers. + */ + return errno ? errno : ENOTSUP; + } + if (parch) + *parch = arch; + + return 0; +} + +int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, + const struct arch **parch) +{ + struct symbol *sym = ms->sym; + struct annotation *notes = symbol__annotation(sym); + struct annotate_args args = { + .options = &annotate_opts, + }; + const struct arch *arch = NULL; + int err, nr; + + err = map_symbol__get_arch(ms, &arch); + if (err) + return err; + + if (parch) + *parch = arch; + + if (notes->src && !list_empty(¬es->src->source)) + return 0; + + args.arch = arch; + args.ms = ms; + + if (notes->src == NULL) { + notes->src = annotated_source__new(); + if (notes->src == NULL) + return -1; + } + + nr = 0; + if (evsel__is_group_event(evsel)) { + struct evsel *pos; + + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + nr++; + } + } + notes->src->nr_events = nr ? nr : 1; + + if (annotate_opts.full_addr) + notes->src->start = map__objdump_2mem(ms->map, ms->sym->start); + else + notes->src->start = map__rip_2objdump(ms->map, ms->sym->start); + + return symbol__disassemble(sym, &args); +} + +static void insert_source_line(struct rb_root *root, struct annotation_line *al) +{ + struct annotation_line *iter; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + unsigned int percent_type = annotate_opts.percent_type; + int i, ret; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct annotation_line, rb_node); + + ret = strcmp(iter->path, al->path); + if (ret == 0) { + for (i = 0; i < al->data_nr; i++) { + iter->data[i].percent_sum += annotation_data__percent(&al->data[i], + percent_type); + } + return; + } + + if (ret < 0) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + for (i = 0; i < al->data_nr; i++) { + al->data[i].percent_sum = annotation_data__percent(&al->data[i], + percent_type); + } + + rb_link_node(&al->rb_node, parent, p); + rb_insert_color(&al->rb_node, root); +} + +static int cmp_source_line(struct annotation_line *a, struct annotation_line *b) +{ + int i; + + for (i = 0; i < a->data_nr; i++) { + if (a->data[i].percent_sum == b->data[i].percent_sum) + continue; + return a->data[i].percent_sum > b->data[i].percent_sum; + } + + return 0; +} + +static void __resort_source_line(struct rb_root *root, struct annotation_line *al) +{ + struct annotation_line *iter; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct annotation_line, rb_node); + + if (cmp_source_line(al, iter)) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + rb_link_node(&al->rb_node, parent, p); + rb_insert_color(&al->rb_node, root); +} + +static void resort_source_line(struct rb_root *dest_root, struct rb_root *src_root) +{ + struct annotation_line *al; + struct rb_node *node; + + node = rb_first(src_root); + while (node) { + struct rb_node *next; + + al = rb_entry(node, struct annotation_line, rb_node); + next = rb_next(node); + rb_erase(node, src_root); + + __resort_source_line(dest_root, al); + node = next; + } +} + +static void print_summary(struct rb_root *root, const char *filename) +{ + struct annotation_line *al; + struct rb_node *node; + + printf("\nSorted summary for file %s\n", filename); + printf("----------------------------------------------\n\n"); + + if (RB_EMPTY_ROOT(root)) { + printf(" Nothing higher than %1.1f%%\n", MIN_GREEN); + return; + } + + node = rb_first(root); + while (node) { + double percent, percent_max = 0.0; + const char *color; + char *path; + int i; + + al = rb_entry(node, struct annotation_line, rb_node); + for (i = 0; i < al->data_nr; i++) { + percent = al->data[i].percent_sum; + color = get_percent_color(percent); + color_fprintf(stdout, color, " %7.2f", percent); + + if (percent > percent_max) + percent_max = percent; + } + + path = al->path; + color = get_percent_color(percent_max); + color_fprintf(stdout, color, " %s\n", path); + + node = rb_next(node); + } +} + +static void symbol__annotate_hits(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + u64 len = symbol__size(sym), offset; + + for (offset = 0; offset < len; ++offset) { + struct sym_hist_entry *entry; + + entry = annotated_source__hist_entry(notes->src, evsel, offset); + if (entry && entry->nr_samples != 0) + printf("%*" PRIx64 ": %" PRIu64 "\n", BITS_PER_LONG / 2, + sym->start + offset, entry->nr_samples); + } + printf("%*s: %" PRIu64 "\n", BITS_PER_LONG / 2, "h->nr_samples", h->nr_samples); +} + +static int annotated_source__addr_fmt_width(struct list_head *lines, u64 start) +{ + char bf[32]; + struct annotation_line *line; + + list_for_each_entry_reverse(line, lines, node) { + if (line->offset != -1) + return scnprintf(bf, sizeof(bf), "%" PRIx64, start + line->offset); + } + + return 0; +} + +int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel) +{ + struct map_symbol *ms = &he->ms; + struct map *map = ms->map; + struct symbol *sym = ms->sym; + struct dso *dso = map__dso(map); + char *filename; + const char *d_filename; + const char *evsel_name = evsel__name(evsel); + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + struct annotation_line *pos, *queue = NULL; + struct annotation_options *opts = &annotate_opts; + struct annotation_print_data apd = { + .he = he, + .evsel = evsel, + }; + int printed = 2, queue_len = 0; + int more = 0; + bool context = opts->context; + int width = annotation__pcnt_width(notes); + int graph_dotted_len; + char buf[512]; + + filename = strdup(dso__long_name(dso)); + if (!filename) + return -ENOMEM; + + if (opts->full_path) + d_filename = filename; + else + d_filename = perf_basename(filename); + + if (evsel__is_group_event(evsel)) { + evsel__group_desc(evsel, buf, sizeof(buf)); + evsel_name = buf; + } + + if (map_symbol__get_arch(ms, &apd.arch)) { + free(filename); + return ENOTSUP; + } + + graph_dotted_len = printf(" %-*.*s| Source code & Disassembly of %s for %s (%" PRIu64 " samples, " + "percent: %s)\n", + width, width, symbol_conf.show_total_period ? "Period" : + symbol_conf.show_nr_samples ? "Samples" : "Percent", + d_filename, evsel_name, h->nr_samples, + percent_type_str(opts->percent_type)); + + printf("%-*.*s----\n", + graph_dotted_len, graph_dotted_len, graph_dotted_line); + + if (verbose > 0) + symbol__annotate_hits(sym, evsel); + + apd.addr_fmt_width = annotated_source__addr_fmt_width(¬es->src->source, + notes->src->start); + apd.dbg = dso__debuginfo(dso); + + list_for_each_entry(pos, ¬es->src->source, node) { + int err; + + if (context && queue == NULL) { + queue = pos; + queue_len = 0; + } + + err = annotation_line__print(pos, &apd, opts, printed, queue); + + switch (err) { + case 0: + ++printed; + if (context) { + printed += queue_len; + queue = NULL; + queue_len = 0; + } + break; + case 1: + /* filtered by max_lines */ + ++more; + break; + case -1: + default: + /* + * Filtered by min_pcnt or non IP lines when + * context != 0 + */ + if (!context) + break; + if (queue_len == context) + queue = list_entry(queue->node.next, typeof(*queue), node); + else + ++queue_len; + break; + } + } + + debuginfo__delete(apd.dbg); + free(filename); + + return more; +} + +static void FILE__set_percent_color(void *fp __maybe_unused, + double percent __maybe_unused, + bool current __maybe_unused) +{ +} + +static int FILE__set_jumps_percent_color(void *fp __maybe_unused, + int nr __maybe_unused, bool current __maybe_unused) +{ + return 0; +} + +static int FILE__set_color(void *fp __maybe_unused, int color __maybe_unused) +{ + return 0; +} + +static void FILE__printf(void *fp, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + vfprintf(fp, fmt, args); + va_end(args); +} + +static void FILE__write_graph(void *fp, int graph) +{ + const char *s; + switch (graph) { + + case DARROW_CHAR: s = "↓"; break; + case UARROW_CHAR: s = "↑"; break; + case LARROW_CHAR: s = "←"; break; + case RARROW_CHAR: s = "→"; break; + default: s = "?"; break; + } + + fputs(s, fp); +} + +static int symbol__annotate_fprintf2(struct symbol *sym, FILE *fp, + struct annotation_print_data *apd) +{ + struct annotation *notes = symbol__annotation(sym); + struct annotation_write_ops wops = { + .first_line = true, + .obj = fp, + .set_color = FILE__set_color, + .set_percent_color = FILE__set_percent_color, + .set_jumps_percent_color = FILE__set_jumps_percent_color, + .printf = FILE__printf, + .write_graph = FILE__write_graph, + }; + struct annotation_line *al; + + if (annotate_opts.code_with_type) { + map_symbol__get_arch(&apd->he->ms, &apd->arch); + apd->dbg = dso__debuginfo(map__dso(apd->he->ms.map)); + } + + list_for_each_entry(al, ¬es->src->source, node) { + if (annotation_line__filter(al)) + continue; + annotation_line__write(al, notes, &wops, apd); + fputc('\n', fp); + wops.first_line = false; + } + + if (annotate_opts.code_with_type) + debuginfo__delete(apd->dbg); + + return 0; +} + +int map_symbol__annotation_dump(struct map_symbol *ms, struct evsel *evsel, + struct hist_entry *he) +{ + const char *ev_name = evsel__name(evsel); + char buf[1024]; + char *filename; + int err = -1; + FILE *fp; + struct annotation_print_data apd = { + .he = he, + .evsel = evsel, + }; + + if (asprintf(&filename, "%s.annotation", ms->sym->name) < 0) + return -1; + + fp = fopen(filename, "w"); + if (fp == NULL) + goto out_free_filename; + + if (evsel__is_group_event(evsel)) { + evsel__group_desc(evsel, buf, sizeof(buf)); + ev_name = buf; + } + + fprintf(fp, "%s() %s\nEvent: %s\n\n", + ms->sym->name, dso__long_name(map__dso(ms->map)), ev_name); + symbol__annotate_fprintf2(ms->sym, fp, &apd); + + fclose(fp); + err = 0; +out_free_filename: + free(filename); + return err; +} + +void symbol__annotate_zero_histogram(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + + memset(h, 0, sizeof(*notes->src->histograms) * notes->src->nr_histograms); +} + +void symbol__annotate_decay_histogram(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + struct annotation_line *al; + + h->nr_samples = 0; + list_for_each_entry(al, ¬es->src->source, node) { + struct sym_hist_entry *entry; + + if (al->offset == -1) + continue; + + entry = annotated_source__hist_entry(notes->src, evsel, al->offset); + if (entry == NULL) + continue; + + entry->nr_samples = entry->nr_samples * 7 / 8; + h->nr_samples += entry->nr_samples; + } +} + +void annotated_source__purge(struct annotated_source *as) +{ + struct annotation_line *al, *n; + + list_for_each_entry_safe(al, n, &as->source, node) { + list_del_init(&al->node); + disasm_line__free(disasm_line(al)); + } + as->tried_source = false; +} + +static size_t disasm_line__fprintf(struct disasm_line *dl, FILE *fp) +{ + size_t printed; + + if (dl->al.offset == -1) + return fprintf(fp, "%s\n", dl->al.line); + + printed = fprintf(fp, "%#" PRIx64 " %s", dl->al.offset, dl->ins.name); + + if (dl->ops.raw[0] != '\0') { + printed += fprintf(fp, "%.*s %s\n", 6 - (int)printed, " ", + dl->ops.raw); + } + + return printed + fprintf(fp, "\n"); +} + +size_t disasm__fprintf(struct list_head *head, FILE *fp) +{ + struct disasm_line *pos; + size_t printed = 0; + + list_for_each_entry(pos, head, al.node) + printed += disasm_line__fprintf(pos, fp); + + return printed; +} + +bool disasm_line__is_valid_local_jump(struct disasm_line *dl, struct symbol *sym) +{ + if (!dl || !dl->ins.ops || !ins__is_jump(&dl->ins) || + !disasm_line__has_local_offset(dl) || dl->ops.target.offset < 0 || + dl->ops.target.offset >= (s64)symbol__size(sym)) + return false; + + return true; +} + +static void +annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym) +{ + struct annotation_line *al; + + /* PLT symbols contain external offsets */ + if (strstr(sym->name, "@plt")) + return; + + list_for_each_entry(al, ¬es->src->source, node) { + struct disasm_line *dl; + struct annotation_line *target; + + dl = disasm_line(al); + + if (!disasm_line__is_valid_local_jump(dl, sym)) + continue; + + target = annotated_source__get_line(notes->src, + dl->ops.target.offset); + /* + * FIXME: Oops, no jump target? Buggy disassembler? Or do we + * have to adjust to the previous offset? + */ + if (target == NULL) + continue; + + if (++target->jump_sources > notes->src->max_jump_sources) + notes->src->max_jump_sources = target->jump_sources; + } +} + +static void annotation__set_index(struct annotation *notes) +{ + struct annotation_line *al; + struct annotated_source *src = notes->src; + + src->widths.max_line_len = 0; + src->nr_entries = 0; + src->nr_asm_entries = 0; + + list_for_each_entry(al, &src->source, node) { + size_t line_len = strlen(al->line); + + if (src->widths.max_line_len < line_len) + src->widths.max_line_len = line_len; + al->idx = src->nr_entries++; + if (al->offset != -1) + al->idx_asm = src->nr_asm_entries++; + else + al->idx_asm = -1; + } +} + +static inline int width_jumps(int n) +{ + if (n >= 100) + return 5; + if (n / 10) + return 2; + return 1; +} + +static int annotation__max_ins_name(struct annotation *notes) +{ + int max_name = 0, len; + struct annotation_line *al; + + list_for_each_entry(al, ¬es->src->source, node) { + if (al->offset == -1) + continue; + + len = strlen(disasm_line(al)->ins.name); + if (max_name < len) + max_name = len; + } + + return max_name; +} + +static void +annotation__init_column_widths(struct annotation *notes, struct symbol *sym) +{ + notes->src->widths.addr = notes->src->widths.target = + notes->src->widths.min_addr = hex_width(symbol__size(sym)); + notes->src->widths.max_addr = hex_width(sym->end); + notes->src->widths.jumps = width_jumps(notes->src->max_jump_sources); + notes->src->widths.max_ins_name = annotation__max_ins_name(notes); +} + +void annotation__update_column_widths(struct annotation *notes) +{ + if (annotate_opts.use_offset) + notes->src->widths.target = notes->src->widths.min_addr; + else if (annotate_opts.full_addr) + notes->src->widths.target = BITS_PER_LONG / 4; + else + notes->src->widths.target = notes->src->widths.max_addr; + + notes->src->widths.addr = notes->src->widths.target; + + if (annotate_opts.show_nr_jumps) + notes->src->widths.addr += notes->src->widths.jumps + 1; +} + +void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms) +{ + annotate_opts.full_addr = !annotate_opts.full_addr; + + if (annotate_opts.full_addr) + notes->src->start = map__objdump_2mem(ms->map, ms->sym->start); + else + notes->src->start = map__rip_2objdump(ms->map, ms->sym->start); + + annotation__update_column_widths(notes); +} + +static void annotation__calc_lines(struct annotation *notes, struct map_symbol *ms, + struct rb_root *root) +{ + struct annotation_line *al; + struct rb_root tmp_root = RB_ROOT; + + list_for_each_entry(al, ¬es->src->source, node) { + double percent_max = 0.0; + u64 addr; + int i; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + annotate_opts.percent_type); + + if (percent > percent_max) + percent_max = percent; + } + + if (percent_max <= 0.5) + continue; + + addr = map__rip_2objdump(ms->map, ms->sym->start); + al->path = get_srcline(map__dso(ms->map), addr + al->offset, NULL, + false, true, ms->sym->start + al->offset); + insert_source_line(&tmp_root, al); + } + + resort_source_line(root, &tmp_root); +} + +static void symbol__calc_lines(struct map_symbol *ms, struct rb_root *root) +{ + struct annotation *notes = symbol__annotation(ms->sym); + + annotation__calc_lines(notes, ms, root); +} + +int hist_entry__tty_annotate2(struct hist_entry *he, struct evsel *evsel) +{ + struct map_symbol *ms = &he->ms; + struct dso *dso = map__dso(ms->map); + struct symbol *sym = ms->sym; + struct rb_root source_line = RB_ROOT; + struct hists *hists = evsel__hists(evsel); + struct annotation_print_data apd = { + .he = he, + .evsel = evsel, + }; + char buf[1024]; + int err; + + err = symbol__annotate2(ms, evsel, NULL); + if (err) { + char msg[BUFSIZ]; + + dso__set_annotate_warned(dso); + symbol__strerror_disassemble(ms, err, msg, sizeof(msg)); + ui__error("Couldn't annotate %s:\n%s", sym->name, msg); + return -1; + } + + if (annotate_opts.print_lines) { + srcline_full_filename = annotate_opts.full_path; + symbol__calc_lines(ms, &source_line); + print_summary(&source_line, dso__long_name(dso)); + } + + hists__scnprintf_title(hists, buf, sizeof(buf)); + fprintf(stdout, "%s, [percent: %s]\n%s() %s\n", + buf, percent_type_str(annotate_opts.percent_type), sym->name, dso__long_name(dso)); + symbol__annotate_fprintf2(sym, stdout, &apd); + + annotated_source__purge(symbol__annotation(sym)->src); + + return 0; +} + +int hist_entry__tty_annotate(struct hist_entry *he, struct evsel *evsel) +{ + struct map_symbol *ms = &he->ms; + struct dso *dso = map__dso(ms->map); + struct symbol *sym = ms->sym; + struct rb_root source_line = RB_ROOT; + int err; + + err = symbol__annotate(ms, evsel, NULL); + if (err) { + char msg[BUFSIZ]; + + dso__set_annotate_warned(dso); + symbol__strerror_disassemble(ms, err, msg, sizeof(msg)); + ui__error("Couldn't annotate %s:\n%s", sym->name, msg); + return -1; + } + + symbol__calc_percent(sym, evsel); + + if (annotate_opts.print_lines) { + srcline_full_filename = annotate_opts.full_path; + symbol__calc_lines(ms, &source_line); + print_summary(&source_line, dso__long_name(dso)); + } + + hist_entry__annotate_printf(he, evsel); + + annotated_source__purge(symbol__annotation(sym)->src); + + return 0; +} + +bool ui__has_annotation(void) +{ + return use_browser == 1 && perf_hpp_list.sym; +} + + +static double annotation_line__max_percent(struct annotation_line *al, + unsigned int percent_type) +{ + double percent_max = 0.0; + int i; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + percent_type); + + if (percent > percent_max) + percent_max = percent; + } + + return percent_max; +} + +static int disasm_line__write(struct disasm_line *dl, struct annotation *notes, + void *obj, char *bf, size_t size, + void (*obj__printf)(void *obj, const char *fmt, ...), + void (*obj__write_graph)(void *obj, int graph)) +{ + if (dl->ins.ops && dl->ins.ops->scnprintf) { + if (ins__is_jump(&dl->ins)) { + bool fwd; + + if (dl->ops.target.outside) + goto call_like; + fwd = dl->ops.target.offset > dl->al.offset; + obj__write_graph(obj, fwd ? DARROW_CHAR : UARROW_CHAR); + obj__printf(obj, " "); + } else if (ins__is_call(&dl->ins)) { +call_like: + obj__write_graph(obj, RARROW_CHAR); + obj__printf(obj, " "); + } else if (ins__is_ret(&dl->ins)) { + obj__write_graph(obj, LARROW_CHAR); + obj__printf(obj, " "); + } else { + obj__printf(obj, " "); + } + } else { + obj__printf(obj, " "); + } + + return disasm_line__scnprintf(dl, bf, size, !annotate_opts.use_offset, + notes->src->widths.max_ins_name) + 2; +} + +static void ipc_coverage_string(char *bf, int size, struct annotation *notes) +{ + double ipc = 0.0, coverage = 0.0; + struct annotated_branch *branch = annotation__get_branch(notes); + + if (branch && branch->hit_cycles) + ipc = branch->hit_insn / ((double)branch->hit_cycles); + + if (branch && branch->total_insn) { + coverage = branch->cover_insn * 100.0 / + ((double)branch->total_insn); + } + + scnprintf(bf, size, "(Average IPC: %.2f, IPC Coverage: %.1f%%)", + ipc, coverage); +} + +int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header) +{ + struct evsel *pos; + struct strbuf sb; + + if (evlist__nr_br_cntr(evsel->evlist) <= 0) + return -ENOTSUP; + + strbuf_init(&sb, /*hint=*/ 0); + + if (header && strbuf_addf(&sb, "# Branch counter abbr list:\n")) + goto err; + + evlist__for_each_entry(evsel->evlist, pos) { + if (!(pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS)) + continue; + if (header && strbuf_addf(&sb, "#")) + goto err; + + if (strbuf_addf(&sb, " %s = %s\n", pos->name, pos->abbr_name)) + goto err; + } + + if (header && strbuf_addf(&sb, "#")) + goto err; + if (strbuf_addf(&sb, " '-' No event occurs\n")) + goto err; + + if (header && strbuf_addf(&sb, "#")) + goto err; + if (strbuf_addf(&sb, " '+' Event occurrences may be lost due to branch counter saturated\n")) + goto err; + + *str = strbuf_detach(&sb, NULL); + + return 0; +err: + strbuf_release(&sb); + return -ENOMEM; +} + +/* Assume the branch counter saturated at 3 */ +#define ANNOTATION_BR_CNTR_SATURATION 3 + +int annotation_br_cntr_entry(char **str, int br_cntr_nr, + u64 *br_cntr, int num_aggr, + struct evsel *evsel) +{ + struct evsel *pos = evsel ? evlist__first(evsel->evlist) : NULL; + bool saturated = false; + int i, j, avg, used; + struct strbuf sb; + + strbuf_init(&sb, /*hint=*/ 0); + for (i = 0; i < br_cntr_nr; i++) { + used = 0; + avg = ceil((double)(br_cntr[i] & ~ANNOTATION__BR_CNTR_SATURATED_FLAG) / + (double)num_aggr); + + /* + * A histogram with the abbr name is displayed by default. + * With -v, the exact number of branch counter is displayed. + */ + if (verbose) { + evlist__for_each_entry_from(evsel->evlist, pos) { + if ((pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + (pos->br_cntr_idx == i)) + break; + } + if (strbuf_addstr(&sb, pos->abbr_name)) + goto err; + + if (!br_cntr[i]) { + if (strbuf_addstr(&sb, "=-")) + goto err; + } else { + if (strbuf_addf(&sb, "=%d", avg)) + goto err; + } + if (br_cntr[i] & ANNOTATION__BR_CNTR_SATURATED_FLAG) { + if (strbuf_addch(&sb, '+')) + goto err; + } else { + if (strbuf_addch(&sb, ' ')) + goto err; + } + + if ((i < br_cntr_nr - 1) && strbuf_addch(&sb, ',')) + goto err; + continue; + } + + if (strbuf_addch(&sb, '|')) + goto err; + + if (!br_cntr[i]) { + if (strbuf_addch(&sb, '-')) + goto err; + used++; + } else { + evlist__for_each_entry_from(evsel->evlist, pos) { + if ((pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + (pos->br_cntr_idx == i)) + break; + } + if (br_cntr[i] & ANNOTATION__BR_CNTR_SATURATED_FLAG) + saturated = true; + + for (j = 0; j < avg; j++, used++) { + /* Print + if the number of logged events > 3 */ + if (j >= ANNOTATION_BR_CNTR_SATURATION) { + saturated = true; + break; + } + if (strbuf_addstr(&sb, pos->abbr_name)) + goto err; + } + + if (saturated) { + if (strbuf_addch(&sb, '+')) + goto err; + used++; + } + pos = list_next_entry(pos, core.node); + } + + for (j = used; j < ANNOTATION_BR_CNTR_SATURATION + 1; j++) { + if (strbuf_addch(&sb, ' ')) + goto err; + } + } + + if (!verbose && strbuf_addch(&sb, br_cntr_nr ? '|' : ' ')) + goto err; + + *str = strbuf_detach(&sb, NULL); + + return 0; +err: + strbuf_release(&sb); + return -ENOMEM; +} + +struct type_hash_entry { + struct annotated_data_type *type; + int offset; +}; + +static int disasm_line__snprint_type_info(struct disasm_line *dl, + char *buf, int len, + struct annotation_print_data *apd) +{ + struct annotated_data_type *data_type = NULL; + struct type_hash_entry *entry = NULL; + char member[256]; + int offset = 0; + int printed; + + scnprintf(buf, len, " "); + + if (!annotate_opts.code_with_type || apd->dbg == NULL) + return 1; + + if (apd->type_hash) { + hashmap__find(apd->type_hash, dl->al.offset, &entry); + if (entry != NULL) { + data_type = entry->type; + offset = entry->offset; + } + } + + if (data_type == NULL) + data_type = __hist_entry__get_data_type(apd->he, apd->arch, apd->dbg, dl, &offset); + + if (apd->type_hash && entry == NULL) { + entry = malloc(sizeof(*entry)); + if (entry != NULL) { + entry->type = data_type; + entry->offset = offset; + hashmap__add(apd->type_hash, dl->al.offset, entry); + } + } + + if (!needs_type_info(data_type)) + return 1; + + printed = scnprintf(buf, len, "\t\t# data-type: %s", data_type->self.type_name); + + if (data_type != &stackop_type && data_type != &canary_type && len > printed) + printed += scnprintf(buf + printed, len - printed, " +%#x", offset); + + if (annotated_data_type__get_member_name(data_type, member, sizeof(member), offset) && + len > printed) { + printed += scnprintf(buf + printed, len - printed, " (%s)", member); + } + return printed; +} + +void annotation_line__write(struct annotation_line *al, struct annotation *notes, + const struct annotation_write_ops *wops, + struct annotation_print_data *apd) +{ + bool current_entry = wops->current_entry; + bool change_color = wops->change_color; + double percent_max = annotation_line__max_percent(al, annotate_opts.percent_type); + int width = wops->width; + int pcnt_width = annotation__pcnt_width(notes); + int cycles_width = annotation__cycles_width(notes); + bool show_title = false; + char bf[256]; + int printed; + void *obj = wops->obj; + int (*obj__set_color)(void *obj, int color) = wops->set_color; + void (*obj__set_percent_color)(void *obj, double percent, bool current) = wops->set_percent_color; + int (*obj__set_jumps_percent_color)(void *obj, int nr, bool current) = wops->set_jumps_percent_color; + void (*obj__printf)(void *obj, const char *fmt, ...) = wops->printf; + void (*obj__write_graph)(void *obj, int graph) = wops->write_graph; + + if (wops->first_line && (al->offset == -1 || percent_max == 0.0)) { + if (notes->branch && al->cycles) { + if (al->cycles->ipc == 0.0 && al->cycles->avg == 0) + show_title = true; + } else + show_title = true; + } + + if (al->offset != -1 && percent_max != 0.0) { + int i; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + annotate_opts.percent_type); + + obj__set_percent_color(obj, percent, current_entry); + if (symbol_conf.show_total_period) { + obj__printf(obj, "%11" PRIu64 " ", al->data[i].he.period); + } else if (symbol_conf.show_nr_samples) { + obj__printf(obj, "%7" PRIu64 " ", + al->data[i].he.nr_samples); + } else { + obj__printf(obj, "%7.2f ", percent); + } + } + } else { + obj__set_percent_color(obj, 0, current_entry); + + if (!show_title) + obj__printf(obj, "%-*s", pcnt_width, " "); + else { + obj__printf(obj, "%-*s", pcnt_width, + symbol_conf.show_total_period ? "Period" : + symbol_conf.show_nr_samples ? "Samples" : "Percent"); + } + } + width -= pcnt_width; + + if (notes->branch) { + if (al->cycles && al->cycles->ipc) + obj__printf(obj, "%*.2f ", ANNOTATION__IPC_WIDTH - 1, al->cycles->ipc); + else if (!show_title) + obj__printf(obj, "%*s", ANNOTATION__IPC_WIDTH, " "); + else + obj__printf(obj, "%*s ", ANNOTATION__IPC_WIDTH - 1, "IPC"); + + if (!annotate_opts.show_minmax_cycle) { + if (al->cycles && al->cycles->avg) + obj__printf(obj, "%*" PRIu64 " ", + ANNOTATION__CYCLES_WIDTH - 1, al->cycles->avg); + else if (!show_title) + obj__printf(obj, "%*s", + ANNOTATION__CYCLES_WIDTH, " "); + else + obj__printf(obj, "%*s ", + ANNOTATION__CYCLES_WIDTH - 1, + "Cycle"); + } else { + if (al->cycles) { + char str[32]; + + scnprintf(str, sizeof(str), + "%" PRIu64 "(%" PRIu64 "/%" PRIu64 ")", + al->cycles->avg, al->cycles->min, + al->cycles->max); + + obj__printf(obj, "%*s ", + ANNOTATION__MINMAX_CYCLES_WIDTH - 1, + str); + } else if (!show_title) + obj__printf(obj, "%*s", + ANNOTATION__MINMAX_CYCLES_WIDTH, + " "); + else + obj__printf(obj, "%*s ", + ANNOTATION__MINMAX_CYCLES_WIDTH - 1, + "Cycle(min/max)"); + } + + if (annotate_opts.show_br_cntr) { + if (show_title) { + obj__printf(obj, "%*s ", + ANNOTATION__BR_CNTR_WIDTH, + "Branch Counter"); + } else { + char *buf; + + if (!annotation_br_cntr_entry(&buf, al->br_cntr_nr, al->br_cntr, + al->num_aggr, al->evsel)) { + obj__printf(obj, "%*s ", ANNOTATION__BR_CNTR_WIDTH, buf); + free(buf); + } + } + } + + if (show_title && !*al->line) { + ipc_coverage_string(bf, sizeof(bf), notes); + obj__printf(obj, "%*s", ANNOTATION__AVG_IPC_WIDTH, bf); + } + } + width -= cycles_width; + + obj__printf(obj, " "); + width -= 1; + + if (!*al->line) + obj__printf(obj, "%-*s", width, " "); + else if (al->offset == -1) { + if (al->line_nr && annotate_opts.show_linenr) + printed = scnprintf(bf, sizeof(bf), "%-*d ", + notes->src->widths.addr + 1, al->line_nr); + else + printed = scnprintf(bf, sizeof(bf), "%-*s ", + notes->src->widths.addr, " "); + obj__printf(obj, bf); + width -= printed; + obj__printf(obj, "%-*s", width, al->line); + } else { + u64 addr = al->offset; + int color = -1; + + if (!annotate_opts.use_offset) + addr += notes->src->start; + + if (!annotate_opts.use_offset) { + printed = scnprintf(bf, sizeof(bf), "%" PRIx64 ": ", addr); + } else { + if (al->jump_sources && + annotate_opts.offset_level >= ANNOTATION__OFFSET_JUMP_TARGETS) { + if (annotate_opts.show_nr_jumps) { + int prev; + printed = scnprintf(bf, sizeof(bf), "%*d ", + notes->src->widths.jumps, + al->jump_sources); + prev = obj__set_jumps_percent_color(obj, al->jump_sources, + current_entry); + obj__printf(obj, bf); + obj__set_color(obj, prev); + } +print_addr: + printed = scnprintf(bf, sizeof(bf), "%*" PRIx64 ": ", + notes->src->widths.target, addr); + } else if (ins__is_call(&disasm_line(al)->ins) && + annotate_opts.offset_level >= ANNOTATION__OFFSET_CALL) { + goto print_addr; + } else if (annotate_opts.offset_level == ANNOTATION__MAX_OFFSET_LEVEL) { + goto print_addr; + } else { + printed = scnprintf(bf, sizeof(bf), "%-*s ", + notes->src->widths.addr, " "); + } + } + + if (change_color) + color = obj__set_color(obj, HE_COLORSET_ADDR); + obj__printf(obj, bf); + if (change_color) + obj__set_color(obj, color); + + width -= printed; + + printed = disasm_line__write(disasm_line(al), notes, obj, bf, sizeof(bf), + obj__printf, obj__write_graph); + + obj__printf(obj, "%s", bf); + width -= printed; + + disasm_line__snprint_type_info(disasm_line(al), bf, sizeof(bf), apd); + obj__printf(obj, "%-*s", width, bf); + } + +} + +int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel, + const struct arch **parch) +{ + struct symbol *sym = ms->sym; + struct annotation *notes = symbol__annotation(sym); + size_t size = symbol__size(sym); + int err; + + err = symbol__annotate(ms, evsel, parch); + if (err) + return err; + + symbol__calc_percent(sym, evsel); + + annotation__set_index(notes); + annotation__mark_jump_targets(notes, sym); + + err = annotation__compute_ipc(notes, size, evsel); + if (err) + return err; + + annotation__init_column_widths(notes, sym); + annotation__update_column_widths(notes); + symbol__set_annotate2(sym, true); + + return 0; +} + +const char * const perf_disassembler__strs[] = { + [PERF_DISASM_UNKNOWN] = "unknown", + [PERF_DISASM_LLVM] = "llvm", + [PERF_DISASM_CAPSTONE] = "capstone", + [PERF_DISASM_OBJDUMP] = "objdump", +}; + + +static void annotation_options__add_disassembler(struct annotation_options *options, + enum perf_disassembler dis) +{ + for (u8 i = 0; i < ARRAY_SIZE(options->disassemblers); i++) { + if (options->disassemblers[i] == dis) { + /* Disassembler is already present then don't add again. */ + return; + } + if (options->disassemblers[i] == PERF_DISASM_UNKNOWN) { + /* Found a free slot. */ + options->disassemblers[i] = dis; + return; + } + } + pr_err("Failed to add disassembler %d\n", dis); +} + +static int annotation_options__add_disassemblers_str(struct annotation_options *options, + const char *str) +{ + while (str && *str != '\0') { + const char *comma = strchr(str, ','); + int len = comma ? comma - str : (int)strlen(str); + bool match = false; + + for (u8 i = 0; i < ARRAY_SIZE(perf_disassembler__strs); i++) { + const char *dis_str = perf_disassembler__strs[i]; + + if (len == (int)strlen(dis_str) && !strncmp(str, dis_str, len)) { + annotation_options__add_disassembler(options, i); + match = true; + break; + } + } + if (!match) { + pr_err("Invalid disassembler '%.*s'\n", len, str); + return -1; + } + str = comma ? comma + 1 : NULL; + } + return 0; +} + +static int annotation__config(const char *var, const char *value, void *data) +{ + struct annotation_options *opt = data; + + if (!strstarts(var, "annotate.")) + return 0; + + if (!strcmp(var, "annotate.offset_level")) { + perf_config_u8(&opt->offset_level, "offset_level", value); + + if (opt->offset_level > ANNOTATION__MAX_OFFSET_LEVEL) + opt->offset_level = ANNOTATION__MAX_OFFSET_LEVEL; + else if (opt->offset_level < ANNOTATION__MIN_OFFSET_LEVEL) + opt->offset_level = ANNOTATION__MIN_OFFSET_LEVEL; + } else if (!strcmp(var, "annotate.disassemblers")) { + int err = annotation_options__add_disassemblers_str(opt, value); + + if (err) + return err; + } else if (!strcmp(var, "annotate.hide_src_code")) { + opt->hide_src_code = perf_config_bool("hide_src_code", value); + } else if (!strcmp(var, "annotate.jump_arrows")) { + opt->jump_arrows = perf_config_bool("jump_arrows", value); + } else if (!strcmp(var, "annotate.show_linenr")) { + opt->show_linenr = perf_config_bool("show_linenr", value); + } else if (!strcmp(var, "annotate.show_nr_jumps")) { + opt->show_nr_jumps = perf_config_bool("show_nr_jumps", value); + } else if (!strcmp(var, "annotate.show_nr_samples")) { + symbol_conf.show_nr_samples = perf_config_bool("show_nr_samples", + value); + } else if (!strcmp(var, "annotate.show_total_period")) { + symbol_conf.show_total_period = perf_config_bool("show_total_period", + value); + } else if (!strcmp(var, "annotate.use_offset")) { + opt->use_offset = perf_config_bool("use_offset", value); + } else if (!strcmp(var, "annotate.disassembler_style")) { + opt->disassembler_style = strdup(value); + if (!opt->disassembler_style) { + pr_err("Not enough memory for annotate.disassembler_style\n"); + return -1; + } + } else if (!strcmp(var, "annotate.objdump")) { + opt->objdump_path = strdup(value); + if (!opt->objdump_path) { + pr_err("Not enough memory for annotate.objdump\n"); + return -1; + } + } else if (!strcmp(var, "annotate.addr2line")) { + symbol_conf.addr2line_path = strdup(value); + if (!symbol_conf.addr2line_path) { + pr_err("Not enough memory for annotate.addr2line\n"); + return -1; + } + } else if (!strcmp(var, "annotate.demangle")) { + symbol_conf.demangle = perf_config_bool("demangle", value); + } else if (!strcmp(var, "annotate.demangle_kernel")) { + symbol_conf.demangle_kernel = perf_config_bool("demangle_kernel", value); + } else { + pr_debug("%s variable unknown, ignoring...", var); + } + + return 0; +} + +void annotation_options__init(void) +{ + struct annotation_options *opt = &annotate_opts; + + memset(opt, 0, sizeof(*opt)); + + /* Default values. */ + opt->use_offset = true; + opt->jump_arrows = true; + opt->annotate_src = true; + opt->offset_level = ANNOTATION__OFFSET_JUMP_TARGETS; + opt->percent_type = PERCENT_PERIOD_LOCAL; + opt->hide_src_code = true; + opt->hide_src_code_on_title = true; +} + +void annotation_options__exit(void) +{ + zfree(&annotate_opts.disassembler_style); + zfree(&annotate_opts.objdump_path); +} + +static void annotation_options__default_init_disassemblers(struct annotation_options *options) +{ + if (options->disassemblers[0] != PERF_DISASM_UNKNOWN) { + /* Already initialized. */ + return; + } +#ifdef HAVE_LIBLLVM_SUPPORT + annotation_options__add_disassembler(options, PERF_DISASM_LLVM); +#endif +#ifdef HAVE_LIBCAPSTONE_SUPPORT + annotation_options__add_disassembler(options, PERF_DISASM_CAPSTONE); +#endif + annotation_options__add_disassembler(options, PERF_DISASM_OBJDUMP); +} + +void annotation_config__init(void) +{ + perf_config(annotation__config, &annotate_opts); + annotation_options__default_init_disassemblers(&annotate_opts); +} + +static unsigned int parse_percent_type(char *str1, char *str2) +{ + unsigned int type = (unsigned int) -1; + + if (!strcmp("period", str1)) { + if (!strcmp("local", str2)) + type = PERCENT_PERIOD_LOCAL; + else if (!strcmp("global", str2)) + type = PERCENT_PERIOD_GLOBAL; + } + + if (!strcmp("hits", str1)) { + if (!strcmp("local", str2)) + type = PERCENT_HITS_LOCAL; + else if (!strcmp("global", str2)) + type = PERCENT_HITS_GLOBAL; + } + + return type; +} + +int annotate_parse_percent_type(const struct option *opt __maybe_unused, const char *_str, + int unset __maybe_unused) +{ + unsigned int type; + char *str1, *str2; + int err = -1; + + str1 = strdup(_str); + if (!str1) + return -ENOMEM; + + str2 = strchr(str1, '-'); + if (!str2) + goto out; + + *str2++ = 0; + + type = parse_percent_type(str1, str2); + if (type == (unsigned int) -1) + type = parse_percent_type(str2, str1); + if (type != (unsigned int) -1) { + annotate_opts.percent_type = type; + err = 0; + } + +out: + free(str1); + return err; +} + +int annotate_check_args(void) +{ + struct annotation_options *args = &annotate_opts; + + if (args->prefix_strip && !args->prefix) { + pr_err("--prefix-strip requires --prefix\n"); + return -1; + } + return 0; +} + +static int arch__dwarf_regnum(const struct arch *arch, const char *str) +{ + const char *p; + char *regname, *q; + int reg; + + p = strchr(str, arch->objdump.register_char); + if (p == NULL) + return -1; + + regname = strdup(p); + if (regname == NULL) + return -1; + + q = strpbrk(regname, ",) "); + if (q) + *q = '\0'; + + reg = get_dwarf_regnum(regname, arch->id.e_machine, arch->id.e_flags); + free(regname); + return reg; +} + +/* + * Get register number and access offset from the given instruction. + * It assumes AT&T x86 asm format like OFFSET(REG). Maybe it needs + * to revisit the format when it handles different architecture. + * Fills @reg and @offset when return 0. + */ +static int extract_reg_offset(const struct arch *arch, const char *str, + struct annotated_op_loc *op_loc) +{ + char *p; + + if (arch->objdump.register_char == 0) + return -1; + + /* + * It should start from offset, but it's possible to skip 0 + * in the asm. So 0(%rax) should be same as (%rax). + * + * However, it also start with a segment select register like + * %gs:0x18(%rbx). In that case it should skip the part. + */ + if (*str == arch->objdump.register_char) { + if (arch__is_x86(arch)) { + /* FIXME: Handle other segment registers */ + if (!strncmp(str, "%gs:", 4)) + op_loc->segment = INSN_SEG_X86_GS; + } + + while (*str && !isdigit(*str) && + *str != arch->objdump.memory_ref_char) + str++; + } + + op_loc->offset = strtol(str, &p, 0); + op_loc->reg1 = arch__dwarf_regnum(arch, p); + if (op_loc->reg1 == -1) + return -1; + + /* Get the second register */ + if (op_loc->multi_regs) + op_loc->reg2 = arch__dwarf_regnum(arch, p + 1); + + return 0; +} + +/** + * annotate_get_insn_location - Get location of instruction + * @arch: the architecture info + * @dl: the target instruction + * @loc: a buffer to save the data + * + * Get detailed location info (register and offset) in the instruction. + * It needs both source and target operand and whether it accesses a + * memory location. The offset field is meaningful only when the + * corresponding mem flag is set. The reg2 field is meaningful only + * when multi_regs flag is set. + * + * Some examples on x86: + * + * mov (%rax), %rcx # src_reg1 = rax, src_mem = 1, src_offset = 0 + * # dst_reg1 = rcx, dst_mem = 0 + * + * mov 0x18, %r8 # src_reg1 = -1, src_mem = 0 + * # dst_reg1 = r8, dst_mem = 0 + * + * mov %rsi, 8(%rbx,%rcx,4) # src_reg1 = rsi, src_mem = 0, src_multi_regs = 0 + * # dst_reg1 = rbx, dst_reg2 = rcx, dst_mem = 1 + * # dst_multi_regs = 1, dst_offset = 8 + */ +int annotate_get_insn_location(const struct arch *arch, struct disasm_line *dl, + struct annotated_insn_loc *loc) +{ + struct ins_operands *ops; + struct annotated_op_loc *op_loc; + int i; + + if (ins__is_lock(&dl->ins)) + ops = dl->ops.locked.ops; + else + ops = &dl->ops; + + if (ops == NULL) + return -1; + + memset(loc, 0, sizeof(*loc)); + + for_each_insn_op_loc(loc, i, op_loc) { + const char *insn_str = ops->source.raw; + bool multi_regs = ops->source.multi_regs; + bool mem_ref = ops->source.mem_ref; + + if (i == INSN_OP_TARGET) { + insn_str = ops->target.raw; + multi_regs = ops->target.multi_regs; + mem_ref = ops->target.mem_ref; + } + + /* Invalidate the register by default */ + op_loc->reg1 = -1; + op_loc->reg2 = -1; + + if (insn_str == NULL) { + if (!arch__is_powerpc(arch)) + continue; + } + + /* + * For powerpc, call get_powerpc_regs function which extracts the + * required fields for op_loc, ie reg1, reg2, offset from the + * raw instruction. + */ + if (arch__is_powerpc(arch)) { + op_loc->mem_ref = mem_ref; + op_loc->multi_regs = multi_regs; + get_powerpc_regs(dl->raw.raw_insn, !i, op_loc); + } else if (strchr(insn_str, arch->objdump.memory_ref_char)) { + op_loc->mem_ref = true; + op_loc->multi_regs = multi_regs; + extract_reg_offset(arch, insn_str, op_loc); + } else { + const char *s = insn_str; + char *p = NULL; + + if (arch__is_x86(arch)) { + /* FIXME: Handle other segment registers */ + if (!strncmp(insn_str, "%gs:", 4)) { + op_loc->segment = INSN_SEG_X86_GS; + op_loc->offset = strtol(insn_str + 4, + &p, 0); + if (p && p != insn_str + 4) + op_loc->imm = true; + continue; + } + } + + if (*s == arch->objdump.register_char) { + op_loc->reg1 = arch__dwarf_regnum(arch, s); + } + else if (*s == arch->objdump.imm_char) { + op_loc->offset = strtol(s + 1, &p, 0); + if (p && p != s + 1) + op_loc->imm = true; + } + } + } + + return 0; +} + +static struct disasm_line *find_disasm_line(struct symbol *sym, u64 ip, + bool allow_update) +{ + struct disasm_line *dl; + struct annotation *notes; + + notes = symbol__annotation(sym); + + list_for_each_entry(dl, ¬es->src->source, al.node) { + if (dl->al.offset == -1) + continue; + + if (sym->start + dl->al.offset == ip) { + /* + * llvm-objdump places "lock" in a separate line and + * in that case, we want to get the next line. + */ + if (ins__is_lock(&dl->ins) && + *dl->ops.raw == '\0' && allow_update) { + ip++; + continue; + } + return dl; + } + } + return NULL; +} + +static struct annotated_item_stat *annotate_data_stat(struct list_head *head, + const char *name) +{ + struct annotated_item_stat *istat; + + list_for_each_entry(istat, head, list) { + if (!strcmp(istat->name, name)) + return istat; + } + + istat = zalloc(sizeof(*istat)); + if (istat == NULL) + return NULL; + + istat->name = strdup(name); + if ((istat->name == NULL) || (!strlen(istat->name))) { + free(istat); + return NULL; + } + + list_add_tail(&istat->list, head); + return istat; +} + +static bool is_stack_operation(const struct arch *arch, struct disasm_line *dl) +{ + if (arch__is_x86(arch)) { + if (!strncmp(dl->ins.name, "push", 4) || + !strncmp(dl->ins.name, "pop", 3) || + !strncmp(dl->ins.name, "call", 4) || + !strncmp(dl->ins.name, "ret", 3)) + return true; + } + + return false; +} + +static bool is_stack_canary(const struct arch *arch, struct annotated_op_loc *loc) +{ + /* On x86_64, %gs:40 is used for stack canary */ + if (arch__is_x86(arch)) { + if (loc->segment == INSN_SEG_X86_GS && loc->imm && + loc->offset == 40) + return true; + } + + return false; +} + +/** + * Returns true if the instruction has a memory operand without + * performing a load/store + */ +static bool is_address_gen_insn(const struct arch *arch, struct disasm_line *dl) +{ + if (arch__is_x86(arch)) { + if (!strncmp(dl->ins.name, "lea", 3)) + return true; + } + + return false; +} + +static struct disasm_line * +annotation__prev_asm_line(struct annotation *notes, struct disasm_line *curr) +{ + struct list_head *sources = ¬es->src->source; + struct disasm_line *prev; + + if (curr == list_first_entry(sources, struct disasm_line, al.node)) + return NULL; + + prev = list_prev_entry(curr, al.node); + while (prev->al.offset == -1 && + prev != list_first_entry(sources, struct disasm_line, al.node)) + prev = list_prev_entry(prev, al.node); + + if (prev->al.offset == -1) + return NULL; + + return prev; +} + +static struct disasm_line * +annotation__next_asm_line(struct annotation *notes, struct disasm_line *curr) +{ + struct list_head *sources = ¬es->src->source; + struct disasm_line *next; + + if (curr == list_last_entry(sources, struct disasm_line, al.node)) + return NULL; + + next = list_next_entry(curr, al.node); + while (next->al.offset == -1 && + next != list_last_entry(sources, struct disasm_line, al.node)) + next = list_next_entry(next, al.node); + + if (next->al.offset == -1) + return NULL; + + return next; +} + +u64 annotate_calc_pcrel(struct map_symbol *ms, u64 ip, int offset, + struct disasm_line *dl) +{ + struct annotation *notes; + struct disasm_line *next; + u64 addr; + + notes = symbol__annotation(ms->sym); + /* + * PC-relative addressing starts from the next instruction address + * But the IP is for the current instruction. Since disasm_line + * doesn't have the instruction size, calculate it using the next + * disasm_line. If it's the last one, we can use symbol's end + * address directly. + */ + next = annotation__next_asm_line(notes, dl); + if (next == NULL) + addr = ms->sym->end + offset; + else + addr = ip + (next->al.offset - dl->al.offset) + offset; + + return map__rip_2objdump(ms->map, addr); +} + +static struct debuginfo_cache { + struct dso *dso; + struct debuginfo *dbg; +} di_cache; + +void debuginfo_cache__delete(void) +{ + dso__put(di_cache.dso); + di_cache.dso = NULL; + + debuginfo__delete(di_cache.dbg); + di_cache.dbg = NULL; +} + +static struct annotated_data_type * +__hist_entry__get_data_type(struct hist_entry *he, const struct arch *arch, + struct debuginfo *dbg, struct disasm_line *dl, + int *type_offset) +{ + struct map_symbol *ms = &he->ms; + struct annotated_insn_loc loc; + struct annotated_op_loc *op_loc; + struct annotated_data_type *mem_type; + struct annotated_item_stat *istat; + int i; + + istat = annotate_data_stat(&ann_insn_stat, dl->ins.name); + if (istat == NULL) { + ann_data_stat.no_insn++; + return NO_TYPE; + } + + if (annotate_get_insn_location(arch, dl, &loc) < 0) { + ann_data_stat.no_insn_ops++; + istat->bad++; + return NO_TYPE; + } + + if (is_stack_operation(arch, dl)) { + istat->good++; + *type_offset = 0; + return &stackop_type; + } + + if (is_address_gen_insn(arch, dl)) { + istat->bad++; + ann_data_stat.no_mem_ops++; + return NO_TYPE; + } + + for_each_insn_op_loc(&loc, i, op_loc) { + struct data_loc_info dloc = { + .arch = arch, + .thread = he->thread, + .ms = ms, + .ip = ms->sym->start + dl->al.offset, + .cpumode = he->cpumode, + .op = op_loc, + .di = dbg, + }; + + if (!op_loc->mem_ref && op_loc->segment == INSN_SEG_NONE) + continue; + + /* PC-relative addressing */ + if (op_loc->reg1 == DWARF_REG_PC) { + dloc.var_addr = annotate_calc_pcrel(ms, dloc.ip, + op_loc->offset, dl); + } + + /* This CPU access in kernel - pretend PC-relative addressing */ + if (dso__kernel(map__dso(ms->map)) && arch__is_x86(arch) && + op_loc->segment == INSN_SEG_X86_GS && op_loc->imm) { + dloc.var_addr = op_loc->offset; + op_loc->reg1 = DWARF_REG_PC; + } + + mem_type = find_data_type(&dloc); + + if (mem_type == NULL && is_stack_canary(arch, op_loc)) { + istat->good++; + *type_offset = 0; + return &canary_type; + } + + if (mem_type) + istat->good++; + else + istat->bad++; + + if (symbol_conf.annotate_data_sample) { + struct evsel *evsel = hists_to_evsel(he->hists); + + annotated_data_type__update_samples(mem_type, evsel, + dloc.type_offset, + he->stat.nr_events, + he->stat.period); + } + *type_offset = dloc.type_offset; + return mem_type ?: NO_TYPE; + } + + /* retry with a fused instruction */ + return NULL; +} + +/** + * hist_entry__get_data_type - find data type for given hist entry + * @he: hist entry + * + * This function first annotates the instruction at @he->ip and extracts + * register and offset info from it. Then it searches the DWARF debug + * info to get a variable and type information using the address, register, + * and offset. + */ +struct annotated_data_type *hist_entry__get_data_type(struct hist_entry *he) +{ + struct map_symbol *ms = &he->ms; + struct evsel *evsel = hists_to_evsel(he->hists); + const struct arch *arch; + struct disasm_line *dl; + struct annotated_data_type *mem_type; + struct annotated_item_stat *istat; + u64 ip = he->ip; + + ann_data_stat.total++; + + if (ms->map == NULL || ms->sym == NULL) { + ann_data_stat.no_sym++; + return NULL; + } + + if (!symbol_conf.init_annotation) { + ann_data_stat.no_sym++; + return NULL; + } + + /* + * di_cache holds a pair of values, but code below assumes + * di_cache.dso can be compared/updated and di_cache.dbg can be + * read/updated independently from each other. That assumption only + * holds in single threaded code. + */ + assert(perf_singlethreaded); + + if (map__dso(ms->map) != di_cache.dso) { + dso__put(di_cache.dso); + di_cache.dso = dso__get(map__dso(ms->map)); + + debuginfo__delete(di_cache.dbg); + di_cache.dbg = dso__debuginfo(di_cache.dso); + } + + if (di_cache.dbg == NULL) { + ann_data_stat.no_dbginfo++; + return NULL; + } + + /* Make sure it has the disasm of the function */ + if (symbol__annotate(ms, evsel, &arch) < 0) { + ann_data_stat.no_insn++; + return NULL; + } + + /* + * Get a disasm to extract the location from the insn. + * This is too slow... + */ + dl = find_disasm_line(ms->sym, ip, /*allow_update=*/true); + if (dl == NULL) { + ann_data_stat.no_insn++; + return NULL; + } + +retry: + mem_type = __hist_entry__get_data_type(he, arch, di_cache.dbg, dl, + &he->mem_type_off); + if (mem_type) + return mem_type == NO_TYPE ? NULL : mem_type; + + /* + * Some instructions can be fused and the actual memory access came + * from the previous instruction. + */ + if (dl->al.offset > 0) { + struct annotation *notes; + struct disasm_line *prev_dl; + + notes = symbol__annotation(ms->sym); + prev_dl = annotation__prev_asm_line(notes, dl); + + if (prev_dl && ins__is_fused(arch, prev_dl->ins.name, dl->ins.name)) { + dl = prev_dl; + goto retry; + } + } + + ann_data_stat.no_mem_ops++; + istat = annotate_data_stat(&ann_insn_stat, dl->ins.name); + if (istat) + istat->bad++; + return NULL; +} + +/* Basic block traversal (BFS) data structure */ +struct basic_block_data { + struct list_head queue; + struct list_head visited; +}; + +/* + * During the traversal, it needs to know the parent block where the current + * block block started from. Note that single basic block can be parent of + * two child basic blocks (in case of condition jump). + */ +struct basic_block_link { + struct list_head node; + struct basic_block_link *parent; + struct annotated_basic_block *bb; +}; + +/* Check any of basic block in the list already has the offset */ +static bool basic_block_has_offset(struct list_head *head, s64 offset) +{ + struct basic_block_link *link; + + list_for_each_entry(link, head, node) { + s64 begin_offset = link->bb->begin->al.offset; + s64 end_offset = link->bb->end->al.offset; + + if (begin_offset <= offset && offset <= end_offset) + return true; + } + return false; +} + +static bool is_new_basic_block(struct basic_block_data *bb_data, + struct disasm_line *dl) +{ + s64 offset = dl->al.offset; + + if (basic_block_has_offset(&bb_data->visited, offset)) + return false; + if (basic_block_has_offset(&bb_data->queue, offset)) + return false; + return true; +} + +/* Add a basic block starting from dl and link it to the parent */ +static int add_basic_block(struct basic_block_data *bb_data, + struct basic_block_link *parent, + struct disasm_line *dl) +{ + struct annotated_basic_block *bb; + struct basic_block_link *link; + + if (dl == NULL) + return -1; + + if (!is_new_basic_block(bb_data, dl)) + return 0; + + bb = zalloc(sizeof(*bb)); + if (bb == NULL) + return -1; + + bb->begin = dl; + bb->end = dl; + INIT_LIST_HEAD(&bb->list); + + link = malloc(sizeof(*link)); + if (link == NULL) { + free(bb); + return -1; + } + + link->bb = bb; + link->parent = parent; + list_add_tail(&link->node, &bb_data->queue); + return 0; +} + +/* Returns true when it finds the target in the current basic block */ +static bool process_basic_block(struct basic_block_data *bb_data, + struct basic_block_link *link, + struct symbol *sym, u64 target) +{ + struct disasm_line *dl, *next_dl, *last_dl; + struct annotation *notes = symbol__annotation(sym); + bool found = false; + + dl = link->bb->begin; + /* Check if it's already visited */ + if (basic_block_has_offset(&bb_data->visited, dl->al.offset)) + return false; + + last_dl = list_last_entry(¬es->src->source, + struct disasm_line, al.node); + if (last_dl->al.offset == -1) + last_dl = annotation__prev_asm_line(notes, last_dl); + + if (last_dl == NULL) + return false; + + list_for_each_entry_from(dl, ¬es->src->source, al.node) { + /* Skip comment or debug info line */ + if (dl->al.offset == -1) + continue; + /* Found the target instruction */ + if (sym->start + dl->al.offset == target) { + found = true; + break; + } + /* End of the function, finish the block */ + if (dl == last_dl) + break; + /* 'return' instruction finishes the block */ + if (ins__is_ret(&dl->ins)) + break; + /* normal instructions are part of the basic block */ + if (!ins__is_jump(&dl->ins)) + continue; + /* jump to a different function, tail call or return */ + if (dl->ops.target.outside) + break; + /* jump instruction creates new basic block(s) */ + next_dl = find_disasm_line(sym, sym->start + dl->ops.target.offset, + /*allow_update=*/false); + if (next_dl) + add_basic_block(bb_data, link, next_dl); + + /* + * FIXME: determine conditional jumps properly. + * Conditional jumps create another basic block with the + * next disasm line. + */ + if (!strstr(dl->ins.name, "jmp")) { + next_dl = annotation__next_asm_line(notes, dl); + if (next_dl) + add_basic_block(bb_data, link, next_dl); + } + break; + + } + link->bb->end = dl; + return found; +} + +/* + * It founds a target basic block, build a proper linked list of basic blocks + * by following the link recursively. + */ +static void link_found_basic_blocks(struct basic_block_link *link, + struct list_head *head) +{ + while (link) { + struct basic_block_link *parent = link->parent; + + list_move(&link->bb->list, head); + list_del(&link->node); + free(link); + + link = parent; + } +} + +static void delete_basic_blocks(struct basic_block_data *bb_data) +{ + struct basic_block_link *link, *tmp; + + list_for_each_entry_safe(link, tmp, &bb_data->queue, node) { + list_del(&link->node); + zfree(&link->bb); + free(link); + } + + list_for_each_entry_safe(link, tmp, &bb_data->visited, node) { + list_del(&link->node); + zfree(&link->bb); + free(link); + } +} + +/** + * annotate_get_basic_blocks - Get basic blocks for given address range + * @sym: symbol to annotate + * @src: source address + * @dst: destination address + * @head: list head to save basic blocks + * + * This function traverses disasm_lines from @src to @dst and save them in a + * list of annotated_basic_block to @head. It uses BFS to find the shortest + * path between two. The basic_block_link is to maintain parent links so + * that it can build a list of blocks from the start. + */ +int annotate_get_basic_blocks(struct symbol *sym, s64 src, s64 dst, + struct list_head *head) +{ + struct basic_block_data bb_data = { + .queue = LIST_HEAD_INIT(bb_data.queue), + .visited = LIST_HEAD_INIT(bb_data.visited), + }; + struct basic_block_link *link; + struct disasm_line *dl; + int ret = -1; + + dl = find_disasm_line(sym, src, /*allow_update=*/false); + if (dl == NULL) + return -1; + + if (add_basic_block(&bb_data, /*parent=*/NULL, dl) < 0) + return -1; + + /* Find shortest path from src to dst using BFS */ + while (!list_empty(&bb_data.queue)) { + link = list_first_entry(&bb_data.queue, struct basic_block_link, node); + + if (process_basic_block(&bb_data, link, sym, dst)) { + link_found_basic_blocks(link, head); + ret = 0; + break; + } + list_move(&link->node, &bb_data.visited); + } + delete_basic_blocks(&bb_data); + return ret; +} diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h new file mode 100644 index 000000000..fa08d09b8 --- /dev/null +++ b/tools/perf/util/annotate.h @@ -0,0 +1,589 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ANNOTATE_H +#define __PERF_ANNOTATE_H + +#include +#include +#include +#include +#include +#include +#include +#include "symbol_conf.h" +#include "mutex.h" +#include "spark.h" +#include "hashmap.h" +#include "disasm.h" +#include "branch.h" +#include "evsel.h" + +struct hist_browser_timer; +struct hist_entry; +struct map; +struct map_symbol; +struct addr_map_symbol; +struct option; +struct perf_sample; +struct symbol; +struct annotated_data_type; + +#define ANNOTATION__IPC_WIDTH 6 +#define ANNOTATION__CYCLES_WIDTH 6 +#define ANNOTATION__MINMAX_CYCLES_WIDTH 19 +#define ANNOTATION__AVG_IPC_WIDTH 36 +#define ANNOTATION__BR_CNTR_WIDTH 30 +#define ANNOTATION_DUMMY_LEN 256 + +enum perf_disassembler { + PERF_DISASM_UNKNOWN = 0, + PERF_DISASM_LLVM, + PERF_DISASM_CAPSTONE, + PERF_DISASM_OBJDUMP, +}; +#define MAX_DISASSEMBLERS (PERF_DISASM_OBJDUMP + 1) + +struct annotation_options { + bool hide_src_code, + hide_src_code_on_title, + use_offset, + jump_arrows, + print_lines, + full_path, + show_linenr, + show_fileloc, + show_nr_jumps, + show_minmax_cycle, + show_asm_raw, + show_br_cntr, + annotate_src, + code_with_type, + full_addr; + u8 offset_level; + u8 disassemblers[MAX_DISASSEMBLERS]; + u8 disassembler_used; + int min_pcnt; + int max_lines; + int context; + char *objdump_path; + char *disassembler_style; + const char *prefix; + const char *prefix_strip; + unsigned int percent_type; +}; + +extern struct annotation_options annotate_opts; + +enum { + ANNOTATION__OFFSET_JUMP_TARGETS = 1, + ANNOTATION__OFFSET_CALL, + ANNOTATION__MAX_OFFSET_LEVEL, +}; + +#define ANNOTATION__MIN_OFFSET_LEVEL ANNOTATION__OFFSET_JUMP_TARGETS + +struct annotation; + +struct sym_hist_entry { + u64 nr_samples; + u64 period; +}; + +enum { + PERCENT_HITS_LOCAL, + PERCENT_HITS_GLOBAL, + PERCENT_PERIOD_LOCAL, + PERCENT_PERIOD_GLOBAL, + PERCENT_MAX, +}; + +struct annotation_data { + double percent[PERCENT_MAX]; + double percent_sum; + struct sym_hist_entry he; +}; + +struct cycles_info { + float ipc; + u64 avg; + u64 max; + u64 min; +}; + +struct annotation_line { + struct list_head node; + struct rb_node rb_node; + s64 offset; + char *line; + int line_nr; + char *fileloc; + char *path; + struct cycles_info *cycles; + int num_aggr; + int br_cntr_nr; + u64 *br_cntr; + struct evsel *evsel; + int jump_sources; + u32 idx; + int idx_asm; + int data_nr; + struct annotation_data data[]; +}; + +struct disasm_line { + struct ins ins; + struct ins_operands ops; + union { + u8 bytes[4]; + u32 raw_insn; + } raw; + /* This needs to be at the end. */ + struct annotation_line al; +}; + +extern const char * const perf_disassembler__strs[]; + +void annotation_line__add(struct annotation_line *al, struct list_head *head); + +static inline double annotation_data__percent(struct annotation_data *data, + unsigned int which) +{ + return which < PERCENT_MAX ? data->percent[which] : -1; +} + +static inline const char *percent_type_str(unsigned int type) +{ + static const char *str[PERCENT_MAX] = { + "local hits", + "global hits", + "local period", + "global period", + }; + + if (WARN_ON(type >= PERCENT_MAX)) + return "N/A"; + + return str[type]; +} + +static inline struct disasm_line *disasm_line(struct annotation_line *al) +{ + return al ? container_of(al, struct disasm_line, al) : NULL; +} + +/* + * Is this offset in the same function as the line it is used? + * asm functions jump to other functions, for instance. + */ +static inline bool disasm_line__has_local_offset(const struct disasm_line *dl) +{ + return dl->ops.target.offset_avail && !dl->ops.target.outside; +} + +/* + * Can we draw an arrow from the jump to its target, for instance? I.e. + * is the jump and its target in the same function? + */ +bool disasm_line__is_valid_local_jump(struct disasm_line *dl, struct symbol *sym); + +struct annotation_line * +annotation_line__next(struct annotation_line *pos, struct list_head *head); + +struct annotation_write_ops { + bool first_line, current_entry, change_color; + int width; + void *obj; + int (*set_color)(void *obj, int color); + void (*set_percent_color)(void *obj, double percent, bool current); + int (*set_jumps_percent_color)(void *obj, int nr, bool current); + void (*printf)(void *obj, const char *fmt, ...); + void (*write_graph)(void *obj, int graph); +}; + +struct annotation_print_data { + struct hist_entry *he; + struct evsel *evsel; + const struct arch *arch; + struct debuginfo *dbg; + /* save data type info keyed by al->offset */ + struct hashmap *type_hash; + /* It'll be set in hist_entry__annotate_printf() */ + int addr_fmt_width; +}; + +void annotation_line__write(struct annotation_line *al, struct annotation *notes, + const struct annotation_write_ops *ops, + struct annotation_print_data *apd); + +int __annotation__scnprintf_samples_period(struct annotation *notes, + char *bf, size_t size, + struct evsel *evsel, + bool show_freq); + +size_t disasm__fprintf(struct list_head *head, FILE *fp); +void symbol__calc_percent(struct symbol *sym, struct evsel *evsel); + +/** + * struct sym_hist - symbol histogram information for an event + * + * @nr_samples: Total number of samples. + * @period: Sum of sample periods. + */ +struct sym_hist { + u64 nr_samples; + u64 period; +}; + +/** + * struct cyc_hist - (CPU) cycle histogram for a basic block + * + * @start: Start address of current block (if known). + * @cycles: Sum of cycles for the longest basic block. + * @cycles_aggr: Total cycles for this address. + * @cycles_max: Max cycles for this address. + * @cycles_min: Min cycles for this address. + * @cycles_spark: History of cycles for the longest basic block. + * @num: Number of samples for the longest basic block. + * @num_aggr: Total number of samples for this address. + * @have_start: Whether the current branch info has a start address. + * @reset: Number of resets due to a different start address. + * + * If sample has branch_stack and cycles info, it can construct basic blocks + * between two adjacent branches. It'd have start and end addresses but + * sometimes the start address may not be available. So the cycles are + * accounted at the end address. If multiple basic blocks end at the same + * address, it will take the longest one. + * + * The @start, @cycles, @cycles_spark and @num fields are used for the longest + * block only. Other fields are used for all cases. + * + * See __symbol__account_cycles(). + */ +struct cyc_hist { + u64 start; + u64 cycles; + u64 cycles_aggr; + u64 cycles_max; + u64 cycles_min; + s64 cycles_spark[NUM_SPARKS]; + u32 num; + u32 num_aggr; + u8 have_start; + /* 1 byte padding */ + u16 reset; +}; + +/** + * struct annotated_source - symbols with hits have this attached as in annotation + * + * @source: List head for annotated_line (embeded in disasm_line). + * @histograms: Array of symbol histograms per event to maintain the total number + * of samples and period. + * @nr_histograms: This may not be the same as evsel->evlist->core.nr_entries if + * we have more than a group in a evlist, where we will want + * to see each group separately, that is why symbol__annotate2() + * sets src->nr_histograms to evsel->nr_members. + * @samples: Hash map of sym_hist_entry. Keyed by event index and offset in symbol. + * @nr_events: Number of events in the current output. + * @nr_entries: Number of annotated_line in the source list. + * @nr_asm_entries: Number of annotated_line with actual asm instruction in the + * source list. + * @max_jump_sources: Maximum number of jump instructions targeting to the same + * instruction. + * @widths: Precalculated width of each column in the TUI output. + * + * disasm_lines are allocated, percentages calculated and all sorted by percentage + * when the annotation is about to be presented, so the percentages are for + * one of the entries in the histogram array, i.e. for the event/counter being + * presented. It is deallocated right after symbol__{tui,tty,etc}_annotate + * returns. + */ +struct annotated_source { + struct list_head source; + struct sym_hist *histograms; + struct hashmap *samples; + int nr_histograms; + int nr_events; + int nr_entries; + int nr_asm_entries; + int max_jump_sources; + bool tried_source; + u64 start; + struct { + u8 addr; + u8 jumps; + u8 target; + u8 min_addr; + u8 max_addr; + u8 max_ins_name; + u16 max_line_len; + } widths; +}; + +struct annotation_line *annotated_source__get_line(struct annotated_source *src, + s64 offset); + +/* A branch counter once saturated */ +#define ANNOTATION__BR_CNTR_SATURATED_FLAG (1ULL << 63) + +/** + * struct annotated_branch - basic block and IPC information for a symbol. + * + * @hit_cycles: Total executed cycles. + * @hit_insn: Total number of instructions executed. + * @total_insn: Number of instructions in the function. + * @cover_insn: Number of distinct, actually executed instructions. + * @cycles_hist: Array of cyc_hist for each instruction. + * @max_coverage: Maximum number of covered basic block (used for block-range). + * @br_cntr: Array of the occurrences of events (branch counters) during a block. + * + * This struct is used by two different codes when the sample has branch stack + * and cycles information. annotation__compute_ipc() calculates average IPC + * using @hit_insn / @hit_cycles. The actual coverage can be calculated using + * @cover_insn / @total_insn. The @cycles_hist can give IPC for each (longest) + * basic block ends at the given address. + * process_basic_block() calculates coverage of instructions (or basic blocks) + * in the function. + */ +struct annotated_branch { + u64 hit_cycles; + u64 hit_insn; + unsigned int total_insn; + unsigned int cover_insn; + struct cyc_hist *cycles_hist; + u64 max_coverage; + u64 *br_cntr; +}; + +struct LOCKABLE annotation { + struct annotated_source *src; + struct annotated_branch *branch; +}; + +static inline void annotation__init(struct annotation *notes __maybe_unused) +{ +} +void annotation__exit(struct annotation *notes); + +void annotation__lock(struct annotation *notes) EXCLUSIVE_LOCK_FUNCTION(*notes); +void annotation__unlock(struct annotation *notes) UNLOCK_FUNCTION(*notes); +bool annotation__trylock(struct annotation *notes) EXCLUSIVE_TRYLOCK_FUNCTION(true, *notes); + +static inline int annotation__cycles_width(struct annotation *notes) +{ + if (notes->branch && annotate_opts.show_minmax_cycle) + return ANNOTATION__IPC_WIDTH + ANNOTATION__MINMAX_CYCLES_WIDTH; + + return notes->branch ? ANNOTATION__IPC_WIDTH + ANNOTATION__CYCLES_WIDTH : 0; +} + +static inline int annotation__pcnt_width(struct annotation *notes) +{ + return (symbol_conf.show_total_period ? 12 : 8) * notes->src->nr_events; +} + +static inline bool annotation_line__filter(struct annotation_line *al) +{ + return annotate_opts.hide_src_code && al->offset == -1; +} + +static inline u8 annotation__br_cntr_width(void) +{ + return annotate_opts.show_br_cntr ? ANNOTATION__BR_CNTR_WIDTH : 0; +} + +void annotation__update_column_widths(struct annotation *notes); +void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms); + +static inline struct sym_hist *annotated_source__histogram(struct annotated_source *src, + const struct evsel *evsel) +{ + return &src->histograms[evsel->core.idx]; +} + +static inline struct sym_hist *annotation__histogram(struct annotation *notes, + const struct evsel *evsel) +{ + return annotated_source__histogram(notes->src, evsel); +} + +static inline struct sym_hist_entry * +annotated_source__hist_entry(struct annotated_source *src, const struct evsel *evsel, u64 offset) +{ + struct sym_hist_entry *entry; + long key = offset << 16 | evsel->core.idx; + + if (!hashmap__find(src->samples, key, &entry)) + return NULL; + return entry; +} + +static inline struct annotation *symbol__annotation(struct symbol *sym) +{ + return (void *)sym - symbol_conf.priv_size; +} + +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample); + +struct annotated_branch *annotation__get_branch(struct annotation *notes); + +int addr_map_symbol__account_cycles(struct addr_map_symbol *ams, + struct addr_map_symbol *start, + unsigned cycles, + struct evsel *evsel, + u64 br_cntr); + +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 addr); + +struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists); +void symbol__annotate_zero_histograms(struct symbol *sym); + +int symbol__annotate(struct map_symbol *ms, + struct evsel *evsel, + const struct arch **parch); +int symbol__annotate2(struct map_symbol *ms, + struct evsel *evsel, + const struct arch **parch); + +enum symbol_disassemble_errno { + SYMBOL_ANNOTATE_ERRNO__SUCCESS = 0, + + /* + * Choose an arbitrary negative big number not to clash with standard + * errno since SUS requires the errno has distinct positive values. + * See 'Issue 6' in the link below. + * + * http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html + */ + __SYMBOL_ANNOTATE_ERRNO__START = -10000, + + SYMBOL_ANNOTATE_ERRNO__NO_VMLINUX = __SYMBOL_ANNOTATE_ERRNO__START, + SYMBOL_ANNOTATE_ERRNO__NO_LIBOPCODES_FOR_BPF, + SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING, + SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP, + SYMBOL_ANNOTATE_ERRNO__BPF_INVALID_FILE, + SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF, + SYMBOL_ANNOTATE_ERRNO__COULDNT_DETERMINE_FILE_TYPE, + + __SYMBOL_ANNOTATE_ERRNO__END, +}; + +int symbol__strerror_disassemble(struct map_symbol *ms, int errnum, char *buf, size_t buflen); + +void symbol__annotate_zero_histogram(struct symbol *sym, struct evsel *evsel); +void symbol__annotate_decay_histogram(struct symbol *sym, struct evsel *evsel); +void annotated_source__purge(struct annotated_source *as); + +int map_symbol__annotation_dump(struct map_symbol *ms, struct evsel *evsel, + struct hist_entry *he); + +bool ui__has_annotation(void); + +int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel); +int hist_entry__tty_annotate(struct hist_entry *he, struct evsel *evsel); +int hist_entry__tty_annotate2(struct hist_entry *he, struct evsel *evsel); + +void annotation_options__init(void); +void annotation_options__exit(void); + +void annotation_config__init(void); + +int annotate_parse_percent_type(const struct option *opt, const char *_str, + int unset); + +int annotate_check_args(void); + +/** + * struct annotated_op_loc - Location info of instruction operand + * @reg1: First register in the operand + * @reg2: Second register in the operand + * @offset: Memory access offset in the operand + * @segment: Segment selector register + * @mem_ref: Whether the operand accesses memory + * @multi_regs: Whether the second register is used + * @imm: Whether the operand is an immediate value (in offset) + */ +struct annotated_op_loc { + int reg1; + int reg2; + int offset; + u8 segment; + bool mem_ref; + bool multi_regs; + bool imm; +}; + +enum annotated_insn_ops { + INSN_OP_SOURCE = 0, + INSN_OP_TARGET = 1, + + INSN_OP_MAX, +}; + +enum annotated_x86_segment { + INSN_SEG_NONE = 0, + + INSN_SEG_X86_CS, + INSN_SEG_X86_DS, + INSN_SEG_X86_ES, + INSN_SEG_X86_FS, + INSN_SEG_X86_GS, + INSN_SEG_X86_SS, +}; + +/** + * struct annotated_insn_loc - Location info of instruction + * @ops: Array of location info for source and target operands + */ +struct annotated_insn_loc { + struct annotated_op_loc ops[INSN_OP_MAX]; +}; + +#define for_each_insn_op_loc(insn_loc, i, op_loc) \ + for (i = INSN_OP_SOURCE, op_loc = &(insn_loc)->ops[i]; \ + i < INSN_OP_MAX; \ + i++, op_loc++) + +/* Get detailed location info in the instruction */ +int annotate_get_insn_location(const struct arch *arch, struct disasm_line *dl, + struct annotated_insn_loc *loc); + +/* Returns a data type from the sample instruction (if any) */ +struct annotated_data_type *hist_entry__get_data_type(struct hist_entry *he); + +struct annotated_item_stat { + struct list_head list; + char *name; + int good; + int bad; +}; +extern struct list_head ann_insn_stat; + +/* Calculate PC-relative address */ +u64 annotate_calc_pcrel(struct map_symbol *ms, u64 ip, int offset, + struct disasm_line *dl); + +/** + * struct annotated_basic_block - Basic block of instructions + * @list: List node + * @begin: start instruction in the block + * @end: end instruction in the block + */ +struct annotated_basic_block { + struct list_head list; + struct disasm_line *begin; + struct disasm_line *end; +}; + +/* Get a list of basic blocks from src to dst addresses */ +int annotate_get_basic_blocks(struct symbol *sym, s64 src, s64 dst, + struct list_head *head); + +void debuginfo_cache__delete(void); + +int annotation_br_cntr_entry(char **str, int br_cntr_nr, u64 *br_cntr, + int num_aggr, struct evsel *evsel); +int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header); + + +int map_symbol__get_arch(struct map_symbol *ms, const struct arch **parch); +#endif /* __PERF_ANNOTATE_H */ diff --git a/tools/perf/util/arm-spe-decoder/Build b/tools/perf/util/arm-spe-decoder/Build new file mode 100644 index 000000000..97a298d1e --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/Build @@ -0,0 +1,3 @@ +perf-util-y += arm-spe-pkt-decoder.o arm-spe-decoder.o + +CFLAGS_arm-spe-pkt-decoder.o += -I$(srctree)/tools/arch/arm64/include/ -I$(OUTPUT)arch/arm64/include/generated/ diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c new file mode 100644 index 000000000..7a3a4815f --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arm_spe_decoder.c: ARM SPE support + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../auxtrace.h" +#include "../debug.h" +#include "../util.h" + +#include "arm-spe-decoder.h" + +static u64 arm_spe_calc_ip(int index, u64 payload) +{ + u64 ns, el, val; + + /* Instruction virtual address or Branch target address */ + if (index == SPE_ADDR_PKT_HDR_INDEX_INS || + index == SPE_ADDR_PKT_HDR_INDEX_BRANCH || + index == SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH) { + ns = SPE_ADDR_PKT_GET_NS(payload); + el = SPE_ADDR_PKT_GET_EL(payload); + + /* Clean highest byte */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + + /* Fill highest byte for EL1 or EL2 (VHE) mode */ + if (ns && (el == SPE_ADDR_PKT_EL1 || el == SPE_ADDR_PKT_EL2)) + payload |= 0xffULL << SPE_ADDR_PKT_ADDR_BYTE7_SHIFT; + + /* Data access virtual address */ + } else if (index == SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT) { + + /* Clean tags */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + + /* + * Armv8 ARM (ARM DDI 0487F.c), chapter "D10.2.1 Address packet" + * defines the data virtual address payload format, the top byte + * (bits [63:56]) is assigned as top-byte tag; so we only can + * retrieve address value from bits [55:0]. + * + * According to Documentation/arch/arm64/memory.rst, if detects the + * specific pattern in bits [55:52] of payload which falls in + * the kernel space, should fixup the top byte and this allows + * perf tool to parse DSO symbol for data address correctly. + * + * For this reason, if detects the bits [55:52] is 0xf, will + * fill 0xff into the top byte. + */ + val = SPE_ADDR_PKT_ADDR_GET_BYTE_6(payload); + if ((val & 0xf0ULL) == 0xf0ULL) + payload |= 0xffULL << SPE_ADDR_PKT_ADDR_BYTE7_SHIFT; + + /* Data access physical address */ + } else if (index == SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS) { + /* Clean highest byte */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + } else { + static u32 seen_idx = 0; + if (!(seen_idx & BIT(index))) { + seen_idx |= BIT(index); + pr_warning("ignoring unsupported address packet index: 0x%x\n", index); + } + } + + return payload; +} + +struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params) +{ + struct arm_spe_decoder *decoder; + + if (!params->get_trace) + return NULL; + + decoder = zalloc(sizeof(struct arm_spe_decoder)); + if (!decoder) + return NULL; + + decoder->get_trace = params->get_trace; + decoder->data = params->data; + + return decoder; +} + +void arm_spe_decoder_free(struct arm_spe_decoder *decoder) +{ + free(decoder); +} + +static int arm_spe_get_data(struct arm_spe_decoder *decoder) +{ + struct arm_spe_buffer buffer = { .buf = 0, }; + int ret; + + pr_debug("Getting more data\n"); + ret = decoder->get_trace(&buffer, decoder->data); + if (ret < 0) + return ret; + + decoder->buf = buffer.buf; + decoder->len = buffer.len; + + if (!decoder->len) + pr_debug("No more data\n"); + + return decoder->len; +} + +static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder, + struct arm_spe_pkt *packet) +{ + int ret; + + do { + if (!decoder->len) { + ret = arm_spe_get_data(decoder); + + /* Failed to read out trace data */ + if (ret <= 0) + return ret; + } + + ret = arm_spe_get_packet(decoder->buf, decoder->len, + packet, decoder->midr); + + if (ret <= 0) { + /* Move forward for 1 byte */ + decoder->buf += 1; + decoder->len -= 1; + return -EBADMSG; + } + + decoder->buf += ret; + decoder->len -= ret; + } while (packet->type == ARM_SPE_PAD); + + return 1; +} + +static int arm_spe_read_record(struct arm_spe_decoder *decoder) +{ + int err; + int idx; + u64 payload, ip; + struct arm_spe_pkt packet; + + memset(&decoder->record, 0x0, sizeof(decoder->record)); + decoder->record.context_id = (u64)-1; + + while (1) { + err = arm_spe_get_next_packet(decoder, &packet); + if (err <= 0) + return err; + + idx = packet.index; + payload = packet.payload; + + switch (packet.type) { + case ARM_SPE_TIMESTAMP: + decoder->record.timestamp = payload; + return 1; + case ARM_SPE_END: + return 1; + case ARM_SPE_ADDRESS: + ip = arm_spe_calc_ip(idx, payload); + if (idx == SPE_ADDR_PKT_HDR_INDEX_INS) + decoder->record.from_ip = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_BRANCH) + decoder->record.to_ip = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT) + decoder->record.virt_addr = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS) + decoder->record.phys_addr = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH) + decoder->record.prev_br_tgt = ip; + break; + case ARM_SPE_COUNTER: + if (idx == SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT) + decoder->record.latency = payload; + break; + case ARM_SPE_CONTEXT: + decoder->record.context_id = payload; + break; + case ARM_SPE_OP_TYPE: + switch (idx) { + case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: + decoder->record.op |= ARM_SPE_OP_LDST; + if (payload & SPE_OP_PKT_ST) + decoder->record.op |= ARM_SPE_OP_ST; + else + decoder->record.op |= ARM_SPE_OP_LD; + + if (SPE_OP_PKT_LDST_SUBCLASS_GP_REG(payload)) { + decoder->record.op |= ARM_SPE_OP_GP_REG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(payload)) { + decoder->record.op |= ARM_SPE_OP_SIMD_FP; + } else if (SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(payload)) { + decoder->record.op |= ARM_SPE_OP_UNSPEC_REG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(payload)) { + decoder->record.op |= ARM_SPE_OP_NV_SYSREG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(payload)) { + decoder->record.op |= ARM_SPE_OP_MTE_TAG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(payload)) { + if (payload & SPE_OP_PKT_AR) + decoder->record.op |= ARM_SPE_OP_AR; + if (payload & SPE_OP_PKT_EXCL) + decoder->record.op |= ARM_SPE_OP_EXCL; + if (payload & SPE_OP_PKT_AT) + decoder->record.op |= ARM_SPE_OP_ATOMIC; + } else if (SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(payload)) { + decoder->record.op |= ARM_SPE_OP_SVE; + if (payload & SPE_OP_PKT_SVE_PRED) + decoder->record.op |= ARM_SPE_OP_PRED; + if (payload & SPE_OP_PKT_SVE_SG) + decoder->record.op |= ARM_SPE_OP_SG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(payload)) { + decoder->record.op |= ARM_SPE_OP_MEMCPY; + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMSET(payload)) { + decoder->record.op |= ARM_SPE_OP_MEMSET; + } else if (SPE_OP_PKT_LDST_SUBCLASS_GCS(payload)) { + decoder->record.op |= ARM_SPE_OP_GCS; + if (payload & SPE_OP_PKT_GCS_COMM) + decoder->record.op |= ARM_SPE_OP_COMM; + } + + break; + case SPE_OP_PKT_HDR_CLASS_OTHER: + decoder->record.op |= ARM_SPE_OP_OTHER; + if (SPE_OP_PKT_OTHER_SUBCLASS_SVE(payload)) { + decoder->record.op |= ARM_SPE_OP_SVE | ARM_SPE_OP_DP; + if (payload & SPE_OP_PKT_OTHER_FP) + decoder->record.op |= ARM_SPE_OP_FP; + if (payload & SPE_OP_PKT_SVE_PRED) + decoder->record.op |= ARM_SPE_OP_PRED; + } else if (SPE_OP_PKT_OTHER_SUBCLASS_SME(payload)) { + decoder->record.op |= ARM_SPE_OP_SME; + if (payload & SPE_OP_PKT_OTHER_FP) + decoder->record.op |= ARM_SPE_OP_FP; + } else if (SPE_OP_PKT_OTHER_SUBCLASS_OTHER(payload)) { + if (payload & SPE_OP_PKT_OTHER_ASE) + decoder->record.op |= ARM_SPE_OP_ASE; + if (payload & SPE_OP_PKT_OTHER_FP) + decoder->record.op |= ARM_SPE_OP_FP; + if (payload & SPE_OP_PKT_COND) + decoder->record.op |= ARM_SPE_OP_COND; + } + break; + case SPE_OP_PKT_HDR_CLASS_BR_ERET: + decoder->record.op |= ARM_SPE_OP_BRANCH_ERET; + if (payload & SPE_OP_PKT_COND) + decoder->record.op |= ARM_SPE_OP_BR_COND; + if (payload & SPE_OP_PKT_INDIRECT_BRANCH) + decoder->record.op |= ARM_SPE_OP_BR_INDIRECT; + if (payload & SPE_OP_PKT_GCS) + decoder->record.op |= ARM_SPE_OP_BR_GCS; + if (SPE_OP_PKT_CR_BL(payload)) + decoder->record.op |= ARM_SPE_OP_BR_CR_BL; + if (SPE_OP_PKT_CR_RET(payload)) + decoder->record.op |= ARM_SPE_OP_BR_CR_RET; + if (SPE_OP_PKT_CR_NON_BL_RET(payload)) + decoder->record.op |= ARM_SPE_OP_BR_CR_NON_BL_RET; + break; + default: + pr_err("Get packet error!\n"); + return -1; + } + break; + case ARM_SPE_EVENTS: + decoder->record.type = payload; + break; + case ARM_SPE_DATA_SOURCE: + decoder->record.source = payload; + break; + case ARM_SPE_BAD: + break; + case ARM_SPE_PAD: + break; + default: + pr_err("Get packet error!\n"); + return -1; + } + } + + return 0; +} + +int arm_spe_decode(struct arm_spe_decoder *decoder) +{ + return arm_spe_read_record(decoder); +} diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h new file mode 100644 index 000000000..0cbcb501e --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arm_spe_decoder.h: Arm Statistical Profiling Extensions support + * Copyright (c) 2019-2020, Arm Ltd. + */ + +#ifndef INCLUDE__ARM_SPE_DECODER_H__ +#define INCLUDE__ARM_SPE_DECODER_H__ + +#include +#include +#include + +#include "arm-spe-pkt-decoder.h" + +#define ARM_SPE_L1D_ACCESS BIT(EV_L1D_ACCESS) +#define ARM_SPE_L1D_MISS BIT(EV_L1D_REFILL) +#define ARM_SPE_LLC_ACCESS BIT(EV_LLC_ACCESS) +#define ARM_SPE_LLC_MISS BIT(EV_LLC_MISS) +#define ARM_SPE_TLB_ACCESS BIT(EV_TLB_ACCESS) +#define ARM_SPE_TLB_MISS BIT(EV_TLB_WALK) +#define ARM_SPE_BRANCH_MISS BIT(EV_MISPRED) +#define ARM_SPE_BRANCH_NOT_TAKEN BIT(EV_NOT_TAKEN) +#define ARM_SPE_REMOTE_ACCESS BIT(EV_REMOTE_ACCESS) +#define ARM_SPE_SVE_PARTIAL_PRED BIT(EV_PARTIAL_PREDICATE) +#define ARM_SPE_SVE_EMPTY_PRED BIT(EV_EMPTY_PREDICATE) +#define ARM_SPE_IN_TXN BIT(EV_TRANSACTIONAL) +#define ARM_SPE_L2D_ACCESS BIT(EV_L2D_ACCESS) +#define ARM_SPE_L2D_MISS BIT(EV_L2D_MISS) +#define ARM_SPE_RECENTLY_FETCHED BIT(EV_RECENTLY_FETCHED) +#define ARM_SPE_DATA_SNOOPED BIT(EV_DATA_SNOOPED) +#define ARM_SPE_HITM BIT(EV_CACHE_DATA_MODIFIED) + +enum arm_spe_op_type { + /* First level operation type */ + ARM_SPE_OP_OTHER = 1 << 0, + ARM_SPE_OP_LDST = 1 << 1, + ARM_SPE_OP_BRANCH_ERET = 1 << 2, +}; + +enum arm_spe_2nd_op_ldst { + ARM_SPE_OP_GP_REG = 1 << 8, + ARM_SPE_OP_UNSPEC_REG = 1 << 9, + ARM_SPE_OP_NV_SYSREG = 1 << 10, + ARM_SPE_OP_SIMD_FP = 1 << 11, + ARM_SPE_OP_SVE = 1 << 12, + ARM_SPE_OP_MTE_TAG = 1 << 13, + ARM_SPE_OP_MEMCPY = 1 << 14, + ARM_SPE_OP_MEMSET = 1 << 15, + ARM_SPE_OP_GCS = 1 << 16, + ARM_SPE_OP_SME = 1 << 17, + ARM_SPE_OP_ASE = 1 << 18, + + /* Assisted information for memory / SIMD */ + ARM_SPE_OP_LD = 1 << 20, + ARM_SPE_OP_ST = 1 << 21, + ARM_SPE_OP_ATOMIC = 1 << 22, + ARM_SPE_OP_EXCL = 1 << 23, + ARM_SPE_OP_AR = 1 << 24, + ARM_SPE_OP_DP = 1 << 25, /* Data processing */ + ARM_SPE_OP_PRED = 1 << 26, /* Predicated */ + ARM_SPE_OP_SG = 1 << 27, /* Gather/Scatter */ + ARM_SPE_OP_COMM = 1 << 28, /* Common */ + ARM_SPE_OP_FP = 1 << 29, /* Floating-point */ + ARM_SPE_OP_COND = 1 << 30, /* Conditional */ +}; + +enum arm_spe_2nd_op_branch { + ARM_SPE_OP_BR_COND = 1 << 8, + ARM_SPE_OP_BR_INDIRECT = 1 << 9, + ARM_SPE_OP_BR_GCS = 1 << 10, + ARM_SPE_OP_BR_CR_BL = 1 << 11, + ARM_SPE_OP_BR_CR_RET = 1 << 12, + ARM_SPE_OP_BR_CR_NON_BL_RET = 1 << 13, +}; + +enum arm_spe_common_data_source { + ARM_SPE_COMMON_DS_L1D = 0x0, + ARM_SPE_COMMON_DS_L2 = 0x8, + ARM_SPE_COMMON_DS_PEER_CORE = 0x9, + ARM_SPE_COMMON_DS_LOCAL_CLUSTER = 0xa, + ARM_SPE_COMMON_DS_SYS_CACHE = 0xb, + ARM_SPE_COMMON_DS_PEER_CLUSTER = 0xc, + ARM_SPE_COMMON_DS_REMOTE = 0xd, + ARM_SPE_COMMON_DS_DRAM = 0xe, +}; + +enum arm_spe_ampereone_data_source { + ARM_SPE_AMPEREONE_LOCAL_CHIP_CACHE_OR_DEVICE = 0x0, + ARM_SPE_AMPEREONE_SLC = 0x3, + ARM_SPE_AMPEREONE_REMOTE_CHIP_CACHE = 0x5, + ARM_SPE_AMPEREONE_DDR = 0x7, + ARM_SPE_AMPEREONE_L1D = 0x8, + ARM_SPE_AMPEREONE_L2D = 0x9, +}; + +enum arm_spe_hisi_hip_data_source { + ARM_SPE_HISI_HIP_PEER_CPU = 0, + ARM_SPE_HISI_HIP_PEER_CPU_HITM = 1, + ARM_SPE_HISI_HIP_L3 = 2, + ARM_SPE_HISI_HIP_L3_HITM = 3, + ARM_SPE_HISI_HIP_PEER_CLUSTER = 4, + ARM_SPE_HISI_HIP_PEER_CLUSTER_HITM = 5, + ARM_SPE_HISI_HIP_REMOTE_SOCKET = 6, + ARM_SPE_HISI_HIP_REMOTE_SOCKET_HITM = 7, + ARM_SPE_HISI_HIP_LOCAL_MEM = 8, + ARM_SPE_HISI_HIP_REMOTE_MEM = 9, + ARM_SPE_HISI_HIP_NC_DEV = 13, + ARM_SPE_HISI_HIP_L2 = 16, + ARM_SPE_HISI_HIP_L2_HITM = 17, + ARM_SPE_HISI_HIP_L1 = 18, +}; + +struct arm_spe_record { + u64 type; + int err; + u32 op; + u32 latency; + u64 from_ip; + u64 to_ip; + u64 prev_br_tgt; + u64 timestamp; + u64 virt_addr; + u64 phys_addr; + u64 context_id; + u16 source; +}; + +struct arm_spe_insn; + +struct arm_spe_buffer { + const unsigned char *buf; + size_t len; + u64 offset; + u64 trace_nr; +}; + +struct arm_spe_params { + int (*get_trace)(struct arm_spe_buffer *buffer, void *data); + void *data; +}; + +struct arm_spe_decoder { + int (*get_trace)(struct arm_spe_buffer *buffer, void *data); + void *data; + struct arm_spe_record record; + + const unsigned char *buf; + size_t len; + u64 midr; +}; + +struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params); +void arm_spe_decoder_free(struct arm_spe_decoder *decoder); + +int arm_spe_decode(struct arm_spe_decoder *decoder); + +#endif diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c new file mode 100644 index 000000000..600677318 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arm-spe-pkt-decoder.h" + +#include "../../arm64/include/asm/cputype.h" + +static const char * const arm_spe_packet_name[] = { + [ARM_SPE_PAD] = "PAD", + [ARM_SPE_END] = "END", + [ARM_SPE_TIMESTAMP] = "TS", + [ARM_SPE_ADDRESS] = "ADDR", + [ARM_SPE_COUNTER] = "LAT", + [ARM_SPE_CONTEXT] = "CONTEXT", + [ARM_SPE_OP_TYPE] = "OP-TYPE", + [ARM_SPE_EVENTS] = "EVENTS", + [ARM_SPE_DATA_SOURCE] = "DATA-SOURCE", +}; + +const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) +{ + return arm_spe_packet_name[type]; +} + +/* + * Extracts the field "sz" from header bits and converts to bytes: + * 00 : byte (1) + * 01 : halfword (2) + * 10 : word (4) + * 11 : doubleword (8) + */ +static unsigned int arm_spe_payload_len(unsigned char hdr) +{ + return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); +} + +static int arm_spe_get_payload(const unsigned char *buf, size_t len, + unsigned char ext_hdr, + struct arm_spe_pkt *packet) +{ + size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); + + if (len < 1 + ext_hdr + payload_len) + return ARM_SPE_NEED_MORE_BYTES; + + buf += 1 + ext_hdr; + + switch (payload_len) { + case 1: packet->payload = *(uint8_t *)buf; break; + case 2: packet->payload = get_unaligned_le16(buf); break; + case 4: packet->payload = get_unaligned_le32(buf); break; + case 8: packet->payload = get_unaligned_le64(buf); break; + default: return ARM_SPE_BAD_PACKET; + } + + return 1 + ext_hdr + payload_len; +} + +static int arm_spe_get_pad(struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_PAD; + return 1; +} + +static int arm_spe_get_alignment(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + unsigned int alignment = 1 << ((buf[0] & 0xf) + 1); + + if (len < alignment) + return ARM_SPE_NEED_MORE_BYTES; + + packet->type = ARM_SPE_PAD; + return alignment - (((uintptr_t)buf) & (alignment - 1)); +} + +static int arm_spe_get_end(struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_END; + return 1; +} + +static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_TIMESTAMP; + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_events(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_EVENTS; + + /* we use index to identify Events with a less number of + * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, + * LLC-REFILL, and REMOTE-ACCESS events are identified if + * index > 1. + */ + packet->index = arm_spe_payload_len(buf[0]); + + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_data_source(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_DATA_SOURCE; + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_context(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_CONTEXT; + packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_op_type(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_OP_TYPE; + packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_counter(const unsigned char *buf, size_t len, + const unsigned char ext_hdr, struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_COUNTER; + + if (ext_hdr) + packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); + else + packet->index = SPE_HDR_SHORT_INDEX(buf[0]); + + return arm_spe_get_payload(buf, len, ext_hdr, packet); +} + +static int arm_spe_get_addr(const unsigned char *buf, size_t len, + const unsigned char ext_hdr, struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_ADDRESS; + + if (ext_hdr) + packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); + else + packet->index = SPE_HDR_SHORT_INDEX(buf[0]); + + return arm_spe_get_payload(buf, len, ext_hdr, packet); +} + +static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + unsigned int hdr; + unsigned char ext_hdr = 0; + + memset(packet, 0, sizeof(struct arm_spe_pkt)); + + if (!len) + return ARM_SPE_NEED_MORE_BYTES; + + hdr = buf[0]; + + if (hdr == SPE_HEADER0_PAD) + return arm_spe_get_pad(packet); + + if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ + return arm_spe_get_end(packet); + + if (hdr == SPE_HEADER0_TIMESTAMP) + return arm_spe_get_timestamp(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) + return arm_spe_get_events(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) + return arm_spe_get_data_source(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) + return arm_spe_get_context(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) + return arm_spe_get_op_type(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { + /* 16-bit extended format header */ + if (len == 1) + return ARM_SPE_BAD_PACKET; + + ext_hdr = 1; + hdr = buf[1]; + if (hdr == SPE_HEADER1_ALIGNMENT) + return arm_spe_get_alignment(buf, len, packet); + } + + /* + * The short format header's byte 0 or the extended format header's + * byte 1 has been assigned to 'hdr', which uses the same encoding for + * address packet and counter packet, so don't need to distinguish if + * it's short format or extended format and handle in once. + */ + if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) + return arm_spe_get_addr(buf, len, ext_hdr, packet); + + if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) + return arm_spe_get_counter(buf, len, ext_hdr, packet); + + return ARM_SPE_BAD_PACKET; +} + +int arm_spe_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet, u64 midr) +{ + int ret; + + ret = arm_spe_do_get_packet(buf, len, packet); + packet->midr = midr; + /* put multiple consecutive PADs on the same line, up to + * the fixed-width output format of 16 bytes per line. + */ + if (ret > 0 && packet->type == ARM_SPE_PAD) { + while (ret < 16 && len > (size_t)ret && !buf[ret]) + ret += 1; + } + return ret; +} + +static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, + const char *fmt, ...) +{ + va_list ap; + int ret; + + /* Bail out if any error occurred */ + if (err && *err) + return *err; + + va_start(ap, fmt); + ret = vsnprintf(*buf_p, *blen, fmt, ap); + va_end(ap); + + if (ret < 0) { + if (err && !*err) + *err = ret; + + /* + * A return value of *blen or more means that the output was + * truncated and the buffer is overrun. + */ + } else if ((size_t)ret >= *blen) { + (*buf_p)[*blen - 1] = '\0'; + + /* + * Set *err to 'ret' to avoid overflow if tries to + * fill this buffer sequentially. + */ + if (err && !*err) + *err = ret; + } else { + *buf_p += ret; + *blen -= ret; + } + + return ret; +} + +struct ev_string { + u8 event; + const char *desc; +}; + +static const struct ev_string common_ev_strings[] = { + { .event = EV_EXCEPTION_GEN, .desc = "EXCEPTION-GEN" }, + { .event = EV_RETIRED, .desc = "RETIRED" }, + { .event = EV_L1D_ACCESS, .desc = "L1D-ACCESS" }, + { .event = EV_L1D_REFILL, .desc = "L1D-REFILL" }, + { .event = EV_TLB_ACCESS, .desc = "TLB-ACCESS" }, + { .event = EV_TLB_WALK, .desc = "TLB-REFILL" }, + { .event = EV_NOT_TAKEN, .desc = "NOT-TAKEN" }, + { .event = EV_MISPRED, .desc = "MISPRED" }, + { .event = EV_LLC_ACCESS, .desc = "LLC-ACCESS" }, + { .event = EV_LLC_MISS, .desc = "LLC-REFILL" }, + { .event = EV_REMOTE_ACCESS, .desc = "REMOTE-ACCESS" }, + { .event = EV_ALIGNMENT, .desc = "ALIGNMENT" }, + { .event = EV_TRANSACTIONAL, .desc = "TXN" }, + { .event = EV_PARTIAL_PREDICATE, .desc = "SVE-PARTIAL-PRED" }, + { .event = EV_EMPTY_PREDICATE, .desc = "SVE-EMPTY-PRED" }, + { .event = EV_L2D_ACCESS, .desc = "L2D-ACCESS" }, + { .event = EV_L2D_MISS, .desc = "L2D-MISS" }, + { .event = EV_CACHE_DATA_MODIFIED, .desc = "HITM" }, + { .event = EV_RECENTLY_FETCHED, .desc = "LFB" }, + { .event = EV_DATA_SNOOPED, .desc = "SNOOPED" }, + { .event = EV_STREAMING_SVE_MODE, .desc = "STREAMING-SVE" }, + { .event = EV_SMCU, .desc = "SMCU" }, + { .event = 0, .desc = NULL }, +}; + +static const struct ev_string n1_event_strings[] = { + { .event = 12, .desc = "LATE-PREFETCH" }, + { .event = 0, .desc = NULL }, +}; + +static u64 print_event_list(int *err, char **buf, size_t *buf_len, + const struct ev_string *ev_strings, u64 payload) +{ + for (const struct ev_string *ev = ev_strings; ev->desc != NULL; ev++) { + if (payload & BIT_ULL(ev->event)) + arm_spe_pkt_out_string(err, buf, buf_len, " %s", ev->desc); + payload &= ~BIT_ULL(ev->event); + } + return payload; +} + +struct event_print_handle { + const struct midr_range *midr_ranges; + const struct ev_string *ev_strings; +}; + +#define EV_PRINT(range, strings) \ + { \ + .midr_ranges = range, \ + .ev_strings = strings, \ + } + +static const struct midr_range n1_event_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + {}, +}; + +static const struct event_print_handle event_print_handles[] = { + EV_PRINT(n1_event_encoding_cpus, n1_event_strings), +}; + +static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + int err = 0; + + arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); + payload = print_event_list(&err, &buf, &buf_len, common_ev_strings, + payload); + + /* Try to decode IMPDEF bits for known CPUs */ + for (unsigned int i = 0; i < ARRAY_SIZE(event_print_handles); i++) { + if (is_midr_in_range_list(packet->midr, + event_print_handles[i].midr_ranges)) + payload = print_event_list(&err, &buf, &buf_len, + event_print_handles[i].ev_strings, + payload); + } + + /* + * Print remaining IMPDEF bits that weren't printed above as raw + * "IMPDEF:1,2,3,4" etc. + */ + if (payload) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IMPDEF:"); + for (int i = 0; i < 64; i++) { + const char *sep = payload & (payload - 1) ? "," : ""; + + if (payload & BIT_ULL(i)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%d%s", i, + sep); + payload &= ~BIT_ULL(i); + } + } + } + + return err; +} + +static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + int err = 0; + + switch (packet->index) { + case SPE_OP_PKT_HDR_CLASS_OTHER: + if (SPE_OP_PKT_OTHER_SUBCLASS_SVE(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); + + /* SVE effective vector length */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", + SPE_OP_PKG_SVE_EVL(payload)); + + if (payload & SPE_OP_PKT_SVE_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + if (payload & SPE_OP_PKT_SVE_PRED) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); + } else if (SPE_OP_PKT_OTHER_SUBCLASS_SME(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "SME-OTHER"); + + /* SME effective vector length or tile size */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " ETS %d", + SPE_OP_PKG_SME_ETS(payload)); + + if (payload & SPE_OP_PKT_OTHER_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + } else if (SPE_OP_PKT_OTHER_SUBCLASS_OTHER(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); + if (payload & SPE_OP_PKT_OTHER_ASE) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " ASE"); + if (payload & SPE_OP_PKT_OTHER_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", + payload & SPE_OP_PKT_COND ? + "COND-SELECT" : "INSN-OTHER"); + } + break; + case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: + arm_spe_pkt_out_string(&err, &buf, &buf_len, + payload & 0x1 ? "ST" : "LD"); + + if (SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(payload)) { + if (payload & SPE_OP_PKT_AT) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); + if (payload & SPE_OP_PKT_EXCL) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); + if (payload & SPE_OP_PKT_AR) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_GP_REG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMSET(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-SME-REG"); + + /* SVE effective vector length */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", + SPE_OP_PKG_SVE_EVL(payload)); + + if (payload & SPE_OP_PKT_SVE_PRED) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); + if (payload & SPE_OP_PKT_SVE_SG) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_GCS(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); + if (payload & SPE_OP_PKT_GCS_COMM) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " COMM"); + } + break; + case SPE_OP_PKT_HDR_CLASS_BR_ERET: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); + + if (payload & SPE_OP_PKT_COND) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); + if (payload & SPE_OP_PKT_INDIRECT_BRANCH) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); + if (payload & SPE_OP_PKT_GCS) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); + if (SPE_OP_PKT_CR_BL(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-BL"); + if (SPE_OP_PKT_CR_RET(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-RET"); + if (SPE_OP_PKT_CR_NON_BL_RET(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-NON-BL-RET"); + break; + default: + /* Unknown index */ + err = -1; + break; + } + + return err; +} + +static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + int ns, el, idx = packet->index; + int ch, pat; + u64 payload = packet->payload; + int err = 0; + static const char *idx_name[] = {"PC", "TGT", "VA", "PA", "PBT"}; + + switch (idx) { + case SPE_ADDR_PKT_HDR_INDEX_INS: + case SPE_ADDR_PKT_HDR_INDEX_BRANCH: + case SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH: + ns = !!SPE_ADDR_PKT_GET_NS(payload); + el = SPE_ADDR_PKT_GET_EL(payload); + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "%s 0x%llx el%d ns=%d", + idx_name[idx], payload, el, ns); + break; + case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "VA 0x%llx", payload); + break; + case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: + ns = !!SPE_ADDR_PKT_GET_NS(payload); + ch = !!SPE_ADDR_PKT_GET_CH(payload); + pat = SPE_ADDR_PKT_GET_PAT(payload); + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "PA 0x%llx ns=%d ch=%d pat=%x", + payload, ns, ch, pat); + break; + default: + /* Unknown index */ + err = -1; + break; + } + + return err; +} + +static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + const char *name = arm_spe_pkt_name(packet->type); + int err = 0; + + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, + (unsigned short)payload); + + switch (packet->index) { + case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); + break; + case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); + break; + case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); + break; + default: + break; + } + + return err; +} + +int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, + size_t buf_len) +{ + int idx = packet->index; + unsigned long long payload = packet->payload; + const char *name = arm_spe_pkt_name(packet->type); + char *buf_orig = buf; + size_t blen = buf_len; + int err = 0; + + switch (packet->type) { + case ARM_SPE_BAD: + case ARM_SPE_PAD: + case ARM_SPE_END: + arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); + break; + case ARM_SPE_EVENTS: + err = arm_spe_pkt_desc_event(packet, buf, buf_len); + break; + case ARM_SPE_OP_TYPE: + err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); + break; + case ARM_SPE_DATA_SOURCE: + case ARM_SPE_TIMESTAMP: + arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); + break; + case ARM_SPE_ADDRESS: + err = arm_spe_pkt_desc_addr(packet, buf, buf_len); + break; + case ARM_SPE_CONTEXT: + arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", + name, (unsigned long)payload, idx + 1); + break; + case ARM_SPE_COUNTER: + err = arm_spe_pkt_desc_counter(packet, buf, buf_len); + break; + default: + /* Unknown packet type */ + err = -1; + break; + } + + /* Output raw data if detect any error */ + if (err) { + err = 0; + arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", + name, payload, packet->index); + } + + return err; +} diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h new file mode 100644 index 000000000..a3300bec4 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#ifndef INCLUDE__ARM_SPE_PKT_DECODER_H__ +#define INCLUDE__ARM_SPE_PKT_DECODER_H__ + +#include +#include +#include + +#define ARM_SPE_PKT_DESC_MAX 512 + +#define ARM_SPE_NEED_MORE_BYTES -1 +#define ARM_SPE_BAD_PACKET -2 + +#define ARM_SPE_PKT_MAX_SZ 16 + +enum arm_spe_pkt_type { + ARM_SPE_BAD, + ARM_SPE_PAD, + ARM_SPE_END, + ARM_SPE_TIMESTAMP, + ARM_SPE_ADDRESS, + ARM_SPE_COUNTER, + ARM_SPE_CONTEXT, + ARM_SPE_OP_TYPE, + ARM_SPE_EVENTS, + ARM_SPE_DATA_SOURCE, +}; + +struct arm_spe_pkt { + enum arm_spe_pkt_type type; + unsigned char index; + uint64_t payload; + uint64_t midr; +}; + +/* Short header (HEADER0) and extended header (HEADER1) */ +#define SPE_HEADER0_PAD 0x0 +#define SPE_HEADER0_END 0x1 +#define SPE_HEADER0_TIMESTAMP 0x71 +/* Mask for event & data source */ +#define SPE_HEADER0_MASK1 (GENMASK_ULL(7, 6) | GENMASK_ULL(3, 0)) +#define SPE_HEADER0_EVENTS 0x42 +#define SPE_HEADER0_SOURCE 0x43 +/* Mask for context & operation */ +#define SPE_HEADER0_MASK2 GENMASK_ULL(7, 2) +#define SPE_HEADER0_CONTEXT 0x64 +#define SPE_HEADER0_OP_TYPE 0x48 +/* Mask for extended format */ +#define SPE_HEADER0_EXTENDED 0x20 +/* Mask for address & counter */ +#define SPE_HEADER0_MASK3 GENMASK_ULL(7, 3) +#define SPE_HEADER0_ADDRESS 0xb0 +#define SPE_HEADER0_COUNTER 0x98 +#define SPE_HEADER1_ALIGNMENT 0x0 + +#define SPE_HDR_SHORT_INDEX(h) ((h) & GENMASK_ULL(2, 0)) +#define SPE_HDR_EXTENDED_INDEX(h0, h1) (((h0) & GENMASK_ULL(1, 0)) << 3 | \ + SPE_HDR_SHORT_INDEX(h1)) + +/* Address packet header */ +#define SPE_ADDR_PKT_HDR_INDEX_INS 0x0 +#define SPE_ADDR_PKT_HDR_INDEX_BRANCH 0x1 +#define SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT 0x2 +#define SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS 0x3 +#define SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH 0x4 + +/* Address packet payload */ +#define SPE_ADDR_PKT_ADDR_BYTE7_SHIFT 56 +#define SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(v) ((v) & GENMASK_ULL(55, 0)) +#define SPE_ADDR_PKT_ADDR_GET_BYTE_6(v) (((v) & GENMASK_ULL(55, 48)) >> 48) + +#define SPE_ADDR_PKT_GET_NS(v) (((v) & BIT_ULL(63)) >> 63) +#define SPE_ADDR_PKT_GET_EL(v) (((v) & GENMASK_ULL(62, 61)) >> 61) +#define SPE_ADDR_PKT_GET_CH(v) (((v) & BIT_ULL(62)) >> 62) +#define SPE_ADDR_PKT_GET_PAT(v) (((v) & GENMASK_ULL(59, 56)) >> 56) + +#define SPE_ADDR_PKT_EL0 0 +#define SPE_ADDR_PKT_EL1 1 +#define SPE_ADDR_PKT_EL2 2 +#define SPE_ADDR_PKT_EL3 3 + +/* Context packet header */ +#define SPE_CTX_PKT_HDR_INDEX(h) ((h) & GENMASK_ULL(1, 0)) + +/* Counter packet header */ +#define SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT 0x0 +#define SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT 0x1 +#define SPE_CNT_PKT_HDR_INDEX_TRANS_LAT 0x2 + +/* Event packet payload */ +enum arm_spe_events { + EV_EXCEPTION_GEN = 0, + EV_RETIRED = 1, + EV_L1D_ACCESS = 2, + EV_L1D_REFILL = 3, + EV_TLB_ACCESS = 4, + EV_TLB_WALK = 5, + EV_NOT_TAKEN = 6, + EV_MISPRED = 7, + EV_LLC_ACCESS = 8, + EV_LLC_MISS = 9, + EV_REMOTE_ACCESS = 10, + EV_ALIGNMENT = 11, + EV_TRANSACTIONAL = 16, + EV_PARTIAL_PREDICATE = 17, + EV_EMPTY_PREDICATE = 18, + EV_L2D_ACCESS = 19, + EV_L2D_MISS = 20, + EV_CACHE_DATA_MODIFIED = 21, + EV_RECENTLY_FETCHED = 22, + EV_DATA_SNOOPED = 23, + EV_STREAMING_SVE_MODE = 24, + EV_SMCU = 25, +}; + +/* Operation packet header */ +#define SPE_OP_PKT_HDR_CLASS(h) ((h) & GENMASK_ULL(1, 0)) +#define SPE_OP_PKT_HDR_CLASS_OTHER 0x0 +#define SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC 0x1 +#define SPE_OP_PKT_HDR_CLASS_BR_ERET 0x2 + +#define SPE_OP_PKT_OTHER_SUBCLASS_OTHER(v) (((v) & GENMASK_ULL(7, 3)) == 0x0) +#define SPE_OP_PKT_OTHER_SUBCLASS_SVE(v) (((v) & (BIT(7) | BIT(3) | BIT(0))) == 0x8) +#define SPE_OP_PKT_OTHER_SUBCLASS_SME(v) (((v) & (BIT(7) | BIT(3) | BIT(0))) == 0x88) + +#define SPE_OP_PKT_OTHER_ASE BIT(2) +#define SPE_OP_PKT_OTHER_FP BIT(1) + +/* + * SME effective vector length or tile size (ETS) is stored in byte 0 + * bits [6:4,2]; the length is rounded up to a power of two and use 128 + * as one step, so ETS calculation is: + * + * 128 * (2 ^ bits [6:4,2]) = 32 << (bits [6:4,2]) + */ +#define SPE_OP_PKG_SME_ETS(v) (128 << (FIELD_GET(GENMASK_ULL(6, 4), (v)) << 1 | \ + (FIELD_GET(BIT(2), (v))))) + +#define SPE_OP_PKT_LDST_SUBCLASS_GP_REG(v) (((v) & GENMASK_ULL(7, 1)) == 0x0) +#define SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(v) (((v) & GENMASK_ULL(7, 1)) == 0x4) +#define SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(v) (((v) & GENMASK_ULL(7, 1)) == 0x10) +#define SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(v) (((v) & GENMASK_ULL(7, 1)) == 0x30) +#define SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(v) (((v) & GENMASK_ULL(7, 1)) == 0x14) +#define SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(v) (((v) & GENMASK_ULL(7, 1)) == 0x20) +#define SPE_OP_PKT_LDST_SUBCLASS_MEMSET(v) (((v) & GENMASK_ULL(7, 0)) == 0x25) + +#define SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(v) (((v) & (GENMASK_ULL(7, 5) | BIT(1))) == 0x2) + +#define SPE_OP_PKT_AR BIT(4) +#define SPE_OP_PKT_EXCL BIT(3) +#define SPE_OP_PKT_AT BIT(2) +#define SPE_OP_PKT_ST BIT(0) + +#define SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(v) (((v) & (BIT(3) | BIT(1))) == 0x8) + +#define SPE_OP_PKT_SVE_SG BIT(7) +/* + * SVE effective vector length (EVL) is stored in byte 0 bits [6:4]; + * the length is rounded up to a power of two and use 32 as one step, + * so EVL calculation is: + * + * 32 * (2 ^ bits [6:4]) = 32 << (bits [6:4]) + */ +#define SPE_OP_PKG_SVE_EVL(v) (32 << (((v) & GENMASK_ULL(6, 4)) >> 4)) +#define SPE_OP_PKT_SVE_PRED BIT(2) +#define SPE_OP_PKT_SVE_FP BIT(1) + +#define SPE_OP_PKT_LDST_SUBCLASS_GCS(v) (((v) & (GENMASK_ULL(7, 3) | BIT(1))) == 0x40) + +#define SPE_OP_PKT_GCS_COMM BIT(2) + +#define SPE_OP_PKT_CR_MASK GENMASK_ULL(4, 3) +#define SPE_OP_PKT_CR_BL(v) (FIELD_GET(SPE_OP_PKT_CR_MASK, (v)) == 1) +#define SPE_OP_PKT_CR_RET(v) (FIELD_GET(SPE_OP_PKT_CR_MASK, (v)) == 2) +#define SPE_OP_PKT_CR_NON_BL_RET(v) (FIELD_GET(SPE_OP_PKT_CR_MASK, (v)) == 3) +#define SPE_OP_PKT_GCS BIT(2) +#define SPE_OP_PKT_INDIRECT_BRANCH BIT(1) +#define SPE_OP_PKT_COND BIT(0) + +const char *arm_spe_pkt_name(enum arm_spe_pkt_type); + +int arm_spe_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet, u64 midr); + +int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t len); +#endif diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c new file mode 100644 index 000000000..401aab529 --- /dev/null +++ b/tools/perf/util/arm-spe.c @@ -0,0 +1,2068 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "auxtrace.h" +#include "color.h" +#include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "machine.h" +#include "session.h" +#include "symbol.h" +#include "thread.h" +#include "thread-stack.h" +#include "tsc.h" +#include "tool.h" +#include "util/synthetic-events.h" + +#include "arm-spe.h" +#include "arm-spe-decoder/arm-spe-decoder.h" +#include "arm-spe-decoder/arm-spe-pkt-decoder.h" + +#include "../../arch/arm64/include/asm/cputype.h" +#define MAX_TIMESTAMP (~0ULL) + +#define is_ldst_op(op) (!!((op) & ARM_SPE_OP_LDST)) + +#define is_simd_op(op) (!!((op) & (ARM_SPE_OP_SIMD_FP | ARM_SPE_OP_SVE | \ + ARM_SPE_OP_SME | ARM_SPE_OP_ASE))) + +#define is_mem_op(op) (is_ldst_op(op) || is_simd_op(op)) + +#define ARM_SPE_CACHE_EVENT(lvl) \ + (ARM_SPE_##lvl##_ACCESS | ARM_SPE_##lvl##_MISS) + +#define arm_spe_is_cache_level(type, lvl) \ + ((type) & ARM_SPE_CACHE_EVENT(lvl)) + +#define arm_spe_is_cache_hit(type, lvl) \ + (((type) & ARM_SPE_CACHE_EVENT(lvl)) == ARM_SPE_##lvl##_ACCESS) + +#define arm_spe_is_cache_miss(type, lvl) \ + ((type) & ARM_SPE_##lvl##_MISS) + +struct arm_spe { + struct auxtrace auxtrace; + struct auxtrace_queues queues; + struct auxtrace_heap heap; + struct itrace_synth_opts synth_opts; + u32 auxtrace_type; + struct perf_session *session; + struct machine *machine; + u32 pmu_type; + + struct perf_tsc_conversion tc; + + u8 timeless_decoding; + u8 data_queued; + + u64 sample_type; + u8 sample_flc; + u8 sample_llc; + u8 sample_tlb; + u8 sample_branch; + u8 sample_remote_access; + u8 sample_memory; + u8 sample_instructions; + + u64 l1d_miss_id; + u64 l1d_access_id; + u64 llc_miss_id; + u64 llc_access_id; + u64 tlb_miss_id; + u64 tlb_access_id; + u64 branch_id; + u64 remote_access_id; + u64 memory_id; + u64 instructions_id; + + u64 kernel_start; + + unsigned long num_events; + u8 use_ctx_pkt_for_pid; + + u64 **metadata; + u64 metadata_ver; + u64 metadata_nr_cpu; + bool is_homogeneous; +}; + +struct arm_spe_queue { + struct arm_spe *spe; + unsigned int queue_nr; + struct auxtrace_buffer *buffer; + struct auxtrace_buffer *old_buffer; + union perf_event *event_buf; + bool on_heap; + bool done; + pid_t pid; + pid_t tid; + int cpu; + struct arm_spe_decoder *decoder; + u64 time; + u64 timestamp; + struct thread *thread; + u64 sample_count; + u32 flags; + struct branch_stack *last_branch; +}; + +struct data_source_handle { + const struct midr_range *midr_ranges; + void (*ds_synth)(const struct arm_spe_record *record, + union perf_mem_data_src *data_src); +}; + +#define DS(range, func) \ + { \ + .midr_ranges = range, \ + .ds_synth = arm_spe__synth_##func, \ + } + +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr); + +static void arm_spe_dump(struct arm_spe *spe __maybe_unused, + unsigned char *buf, size_t len, u64 midr) +{ + struct arm_spe_pkt packet; + size_t pos = 0; + int ret, pkt_len, i; + char desc[ARM_SPE_PKT_DESC_MAX]; + const char *color = PERF_COLOR_BLUE; + + color_fprintf(stdout, color, + ". ... ARM SPE data: size %#zx bytes\n", + len); + + while (len) { + ret = arm_spe_get_packet(buf, len, &packet, midr); + + if (ret > 0) + pkt_len = ret; + else + pkt_len = 1; + printf("."); + color_fprintf(stdout, color, " %08zx: ", pos); + for (i = 0; i < pkt_len; i++) + color_fprintf(stdout, color, " %02x", buf[i]); + for (; i < 16; i++) + color_fprintf(stdout, color, " "); + if (ret > 0) { + ret = arm_spe_pkt_desc(&packet, desc, + ARM_SPE_PKT_DESC_MAX); + if (!ret) + color_fprintf(stdout, color, " %s\n", desc); + } else { + color_fprintf(stdout, color, " Bad packet!\n"); + } + pos += pkt_len; + buf += pkt_len; + len -= pkt_len; + } +} + +static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, + size_t len, u64 midr) +{ + printf(".\n"); + arm_spe_dump(spe, buf, len, midr); +} + +static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) +{ + struct arm_spe_queue *speq = data; + struct auxtrace_buffer *buffer = speq->buffer; + struct auxtrace_buffer *old_buffer = speq->old_buffer; + struct auxtrace_queue *queue; + + queue = &speq->spe->queues.queue_array[speq->queue_nr]; + + buffer = auxtrace_buffer__next(queue, buffer); + /* If no more data, drop the previous auxtrace_buffer and return */ + if (!buffer) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + b->len = 0; + return 0; + } + + speq->buffer = buffer; + + /* If the aux_buffer doesn't have data associated, try to load it */ + if (!buffer->data) { + /* get the file desc associated with the perf data file */ + int fd = perf_data__fd(speq->spe->session->data); + + buffer->data = auxtrace_buffer__get_data(buffer, fd); + if (!buffer->data) + return -ENOMEM; + } + + b->len = buffer->size; + b->buf = buffer->data; + + if (b->len) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + speq->old_buffer = buffer; + } else { + auxtrace_buffer__drop_data(buffer); + return arm_spe_get_trace(b, data); + } + + return 0; +} + +static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe, + unsigned int queue_nr) +{ + struct arm_spe_params params = { .get_trace = 0, }; + struct arm_spe_queue *speq; + + speq = zalloc(sizeof(*speq)); + if (!speq) + return NULL; + + speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); + if (!speq->event_buf) + goto out_free; + + speq->spe = spe; + speq->queue_nr = queue_nr; + speq->pid = -1; + speq->tid = -1; + speq->cpu = -1; + + /* params set */ + params.get_trace = arm_spe_get_trace; + params.data = speq; + + if (spe->synth_opts.last_branch) { + size_t sz = sizeof(struct branch_stack); + + /* Allocate up to two entries for PBT + TGT */ + sz += sizeof(struct branch_entry) * + min(spe->synth_opts.last_branch_sz, 2U); + speq->last_branch = zalloc(sz); + if (!speq->last_branch) + goto out_free; + } + + /* create new decoder */ + speq->decoder = arm_spe_decoder_new(¶ms); + if (!speq->decoder) + goto out_free; + + return speq; + +out_free: + zfree(&speq->event_buf); + zfree(&speq->last_branch); + free(speq); + + return NULL; +} + +static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip) +{ + return ip >= spe->kernel_start ? + PERF_RECORD_MISC_KERNEL : + PERF_RECORD_MISC_USER; +} + +static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, + struct auxtrace_queue *queue) +{ + struct arm_spe_queue *speq = queue->priv; + pid_t tid; + + tid = machine__get_current_tid(spe->machine, speq->cpu); + if (tid != -1) { + speq->tid = tid; + thread__zput(speq->thread); + } else + speq->tid = queue->tid; + + if ((!speq->thread) && (speq->tid != -1)) { + speq->thread = machine__find_thread(spe->machine, -1, + speq->tid); + } + + if (speq->thread) { + speq->pid = thread__pid(speq->thread); + if (queue->cpu == -1) { + speq->cpu = thread__cpu(speq->thread); + arm_spe__get_midr(spe, speq->cpu, &speq->decoder->midr); + } + } +} + +static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid) +{ + struct arm_spe *spe = speq->spe; + int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid); + + if (err) + return err; + + arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]); + + return 0; +} + +static u64 *arm_spe__get_metadata_by_cpu(struct arm_spe *spe, int cpu) +{ + u64 i; + + if (!spe->metadata) + return NULL; + + /* CPU ID is -1 for per-thread mode */ + if (cpu < 0) { + /* + * On the heterogeneous system, due to CPU ID is -1, + * cannot confirm the data source packet is supported. + */ + if (!spe->is_homogeneous) + return NULL; + + /* In homogeneous system, simply use CPU0's metadata */ + return spe->metadata[0]; + } + + for (i = 0; i < spe->metadata_nr_cpu; i++) + if (spe->metadata[i][ARM_SPE_CPU] == (u64)cpu) + return spe->metadata[i]; + + return NULL; +} + +static struct simd_flags arm_spe__synth_simd_flags(const struct arm_spe_record *record) +{ + struct simd_flags simd_flags = {}; + + if (record->op & ARM_SPE_OP_SVE) + simd_flags.arch |= SIMD_OP_FLAGS_ARCH_SVE; + else if (record->op & ARM_SPE_OP_SME) + simd_flags.arch |= SIMD_OP_FLAGS_ARCH_SME; + else if (record->op & (ARM_SPE_OP_ASE | ARM_SPE_OP_SIMD_FP)) + simd_flags.arch |= SIMD_OP_FLAGS_ARCH_ASE; + + if (record->op & ARM_SPE_OP_SVE) { + if (!(record->op & ARM_SPE_OP_PRED)) + simd_flags.pred = SIMD_OP_FLAGS_PRED_DISABLED; + else if (record->type & ARM_SPE_SVE_PARTIAL_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_PARTIAL; + else if (record->type & ARM_SPE_SVE_EMPTY_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_EMPTY; + else + simd_flags.pred = SIMD_OP_FLAGS_PRED_FULL; + } else { + if (record->type & ARM_SPE_SVE_PARTIAL_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_PARTIAL; + else if (record->type & ARM_SPE_SVE_EMPTY_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_EMPTY; + } + + return simd_flags; +} + +static void arm_spe_prep_sample(struct arm_spe *spe, + struct arm_spe_queue *speq, + union perf_event *event, + struct perf_sample *sample) +{ + struct arm_spe_record *record = &speq->decoder->record; + + if (!spe->timeless_decoding) + sample->time = tsc_to_perf_time(record->timestamp, &spe->tc); + + sample->ip = record->from_ip; + sample->cpumode = arm_spe_cpumode(spe, sample->ip); + sample->pid = speq->pid; + sample->tid = speq->tid; + sample->period = spe->synth_opts.period; + sample->cpu = speq->cpu; + sample->simd_flags = arm_spe__synth_simd_flags(record); + + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = sample->cpumode; + event->sample.header.size = sizeof(struct perf_event_header); +} + +static void arm_spe__prep_branch_stack(struct arm_spe_queue *speq) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + struct branch_stack *bstack = speq->last_branch; + struct branch_flags *bs_flags; + unsigned int last_branch_sz = spe->synth_opts.last_branch_sz; + bool have_tgt = !!(speq->flags & PERF_IP_FLAG_BRANCH); + bool have_pbt = last_branch_sz >= (have_tgt + 1U) && record->prev_br_tgt; + size_t sz = sizeof(struct branch_stack) + + sizeof(struct branch_entry) * min(last_branch_sz, 2U) /* PBT + TGT */; + int i = 0; + + /* Clean up branch stack */ + memset(bstack, 0x0, sz); + + if (!have_tgt && !have_pbt) + return; + + if (have_tgt) { + bstack->entries[i].from = record->from_ip; + bstack->entries[i].to = record->to_ip; + + bs_flags = &bstack->entries[i].flags; + bs_flags->value = 0; + + if (record->op & ARM_SPE_OP_BR_CR_BL) { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND_CALL; + else + bs_flags->type |= PERF_BR_CALL; + /* + * Indirect branch instruction without link (e.g. BR), + * take this case as function return. + */ + } else if (record->op & ARM_SPE_OP_BR_CR_RET || + record->op & ARM_SPE_OP_BR_INDIRECT) { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND_RET; + else + bs_flags->type |= PERF_BR_RET; + } else if (record->op & ARM_SPE_OP_BR_CR_NON_BL_RET) { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND; + else + bs_flags->type |= PERF_BR_UNCOND; + } else { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND; + else + bs_flags->type |= PERF_BR_UNKNOWN; + } + + if (record->type & ARM_SPE_BRANCH_MISS) { + bs_flags->mispred = 1; + bs_flags->predicted = 0; + } else { + bs_flags->mispred = 0; + bs_flags->predicted = 1; + } + + if (record->type & ARM_SPE_BRANCH_NOT_TAKEN) + bs_flags->not_taken = 1; + + if (record->type & ARM_SPE_IN_TXN) + bs_flags->in_tx = 1; + + bs_flags->cycles = min(record->latency, 0xFFFFU); + i++; + } + + if (have_pbt) { + bs_flags = &bstack->entries[i].flags; + bs_flags->type |= PERF_BR_UNKNOWN; + bstack->entries[i].to = record->prev_br_tgt; + i++; + } + + bstack->nr = i; + bstack->hw_idx = -1ULL; +} + +static int arm_spe__inject_event(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample, u64 type) +{ + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && spe->session && spe->session->evlist) + evsel = evlist__id2evsel(spe->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); +} + +static inline int +arm_spe_deliver_synth_event(struct arm_spe *spe, + struct arm_spe_queue *speq __maybe_unused, + union perf_event *event, + struct perf_sample *sample) +{ + int ret; + + if (spe->synth_opts.inject) { + ret = arm_spe__inject_event(spe, event, sample, spe->sample_type); + if (ret) + return ret; + } + + ret = perf_session__deliver_synth_event(spe->session, event, sample); + if (ret) + pr_err("ARM SPE: failed to deliver event, error %d\n", ret); + + return ret; +} + +static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq, + u64 spe_events_id, + union perf_mem_data_src data_src) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + union perf_event *event = speq->event_buf; + struct perf_sample sample; + int ret; + + perf_sample__init(&sample, /*all=*/true); + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + sample.addr = record->virt_addr; + sample.phys_addr = record->phys_addr; + sample.data_src = data_src.val; + sample.weight = record->latency; + + ret = arm_spe_deliver_synth_event(spe, speq, event, &sample); + perf_sample__exit(&sample); + return ret; +} + +static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq, + u64 spe_events_id) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + union perf_event *event = speq->event_buf; + struct perf_sample sample; + int ret; + + perf_sample__init(&sample, /*all=*/true); + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + sample.addr = record->to_ip; + sample.weight = record->latency; + sample.flags = speq->flags; + sample.branch_stack = speq->last_branch; + + ret = arm_spe_deliver_synth_event(spe, speq, event, &sample); + perf_sample__exit(&sample); + return ret; +} + +static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq, + u64 spe_events_id, + union perf_mem_data_src data_src) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + union perf_event *event = speq->event_buf; + struct perf_sample sample; + int ret; + + perf_sample__init(&sample, /*all=*/true); + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + sample.addr = record->to_ip; + sample.phys_addr = record->phys_addr; + sample.data_src = data_src.val; + sample.weight = record->latency; + sample.flags = speq->flags; + sample.branch_stack = speq->last_branch; + + ret = arm_spe_deliver_synth_event(spe, speq, event, &sample); + perf_sample__exit(&sample); + return ret; +} + +static const struct midr_range common_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A715), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A725), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + {}, +}; + +static const struct midr_range ampereone_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_AMPERE1A), + {}, +}; + +static const struct midr_range hisi_hip_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_HISI_HIP12), + {}, +}; + +static void arm_spe__sample_flags(struct arm_spe_queue *speq) +{ + const struct arm_spe_record *record = &speq->decoder->record; + + speq->flags = 0; + if (record->op & ARM_SPE_OP_BRANCH_ERET) { + speq->flags = PERF_IP_FLAG_BRANCH; + + if (record->type & ARM_SPE_BRANCH_MISS) + speq->flags |= PERF_IP_FLAG_BRANCH_MISS; + + if (record->type & ARM_SPE_BRANCH_NOT_TAKEN) + speq->flags |= PERF_IP_FLAG_NOT_TAKEN; + + if (record->type & ARM_SPE_IN_TXN) + speq->flags |= PERF_IP_FLAG_IN_TX; + + if (record->op & ARM_SPE_OP_BR_COND) + speq->flags |= PERF_IP_FLAG_CONDITIONAL; + + if (record->op & ARM_SPE_OP_BR_CR_BL) + speq->flags |= PERF_IP_FLAG_CALL; + else if (record->op & ARM_SPE_OP_BR_CR_RET) + speq->flags |= PERF_IP_FLAG_RETURN; + /* + * Indirect branch instruction without link (e.g. BR), + * take it as a function return. + */ + else if (record->op & ARM_SPE_OP_BR_INDIRECT) + speq->flags |= PERF_IP_FLAG_RETURN; + } +} + +static void arm_spe__synth_data_source_common(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* + * Even though four levels of cache hierarchy are possible, no known + * production Neoverse systems currently include more than three levels + * so for the time being we assume three exist. If a production system + * is built with four the this function would have to be changed to + * detect the number of levels for reporting. + */ + + /* + * We have no data on the hit level or data source for stores in the + * Neoverse SPE records. + */ + if (record->op & ARM_SPE_OP_ST) { + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; + data_src->mem_snoop = PERF_MEM_SNOOP_NA; + return; + } + + switch (record->source) { + case ARM_SPE_COMMON_DS_L1D: + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_COMMON_DS_L2: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_COMMON_DS_PEER_CORE: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + /* + * We don't know if this is L1, L2 but we do know it was a cache-2-cache + * transfer, so set SNOOPX_PEER + */ + case ARM_SPE_COMMON_DS_LOCAL_CLUSTER: + case ARM_SPE_COMMON_DS_PEER_CLUSTER: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + /* + * System cache is assumed to be L3 + */ + case ARM_SPE_COMMON_DS_SYS_CACHE: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HIT; + break; + /* + * We don't know what level it hit in, except it came from the other + * socket + */ + case ARM_SPE_COMMON_DS_REMOTE: + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_COMMON_DS_DRAM: + data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + default: + break; + } +} + +/* + * Source is IMPDEF. Here we convert the source code used on AmpereOne cores + * to the common (Neoverse, Cortex) to avoid duplicating the decoding code. + */ +static void arm_spe__synth_data_source_ampereone(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + struct arm_spe_record common_record; + + switch (record->source) { + case ARM_SPE_AMPEREONE_LOCAL_CHIP_CACHE_OR_DEVICE: + common_record.source = ARM_SPE_COMMON_DS_PEER_CORE; + break; + case ARM_SPE_AMPEREONE_SLC: + common_record.source = ARM_SPE_COMMON_DS_SYS_CACHE; + break; + case ARM_SPE_AMPEREONE_REMOTE_CHIP_CACHE: + common_record.source = ARM_SPE_COMMON_DS_REMOTE; + break; + case ARM_SPE_AMPEREONE_DDR: + common_record.source = ARM_SPE_COMMON_DS_DRAM; + break; + case ARM_SPE_AMPEREONE_L1D: + common_record.source = ARM_SPE_COMMON_DS_L1D; + break; + case ARM_SPE_AMPEREONE_L2D: + common_record.source = ARM_SPE_COMMON_DS_L2; + break; + default: + pr_warning_once("AmpereOne: Unknown data source (0x%x)\n", + record->source); + return; + } + + common_record.op = record->op; + arm_spe__synth_data_source_common(&common_record, data_src); +} + +static void arm_spe__synth_data_source_hisi_hip(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* Use common synthesis method to handle store operations */ + if (record->op & ARM_SPE_OP_ST) { + arm_spe__synth_data_source_common(record, data_src); + return; + } + + switch (record->source) { + case ARM_SPE_HISI_HIP_PEER_CPU: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_PEER_CPU_HITM: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_L3: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HIT; + break; + case ARM_SPE_HISI_HIP_L3_HITM: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + break; + case ARM_SPE_HISI_HIP_PEER_CLUSTER: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_PEER_CLUSTER_HITM: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_REMOTE_SOCKET: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_REMOTE_SOCKET_HITM: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_LOCAL_MEM: + data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_HISI_HIP_REMOTE_MEM: + data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + break; + case ARM_SPE_HISI_HIP_NC_DEV: + data_src->mem_lvl = PERF_MEM_LVL_IO | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_HISI_HIP_L2: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_HISI_HIP_L2_HITM: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + break; + case ARM_SPE_HISI_HIP_L1: + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + default: + break; + } +} + +static const struct data_source_handle data_source_handles[] = { + DS(common_ds_encoding_cpus, data_source_common), + DS(ampereone_ds_encoding_cpus, data_source_ampereone), + DS(hisi_hip_ds_encoding_cpus, data_source_hisi_hip), +}; + +static void arm_spe__synth_ld_memory_level(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* + * To find a cache hit, search in ascending order from the lower level + * caches to the higher level caches. This reflects the best scenario + * for a cache hit. + */ + if (arm_spe_is_cache_hit(record->type, L1D)) { + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + } else if (record->type & ARM_SPE_RECENTLY_FETCHED) { + data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_LFB; + } else if (arm_spe_is_cache_hit(record->type, L2D)) { + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + } else if (arm_spe_is_cache_hit(record->type, LLC)) { + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + /* + * To find a cache miss, search in descending order from the higher + * level cache to the lower level cache. This represents the worst + * scenario for a cache miss. + */ + } else if (arm_spe_is_cache_miss(record->type, LLC)) { + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_MISS; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + } else if (arm_spe_is_cache_miss(record->type, L2D)) { + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_MISS; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + } else if (arm_spe_is_cache_miss(record->type, L1D)) { + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_MISS; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + } +} + +static void arm_spe__synth_st_memory_level(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* Record the greatest level info for a store operation. */ + if (arm_spe_is_cache_level(record->type, LLC)) { + data_src->mem_lvl = PERF_MEM_LVL_L3; + data_src->mem_lvl |= arm_spe_is_cache_miss(record->type, LLC) ? + PERF_MEM_LVL_MISS : PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + } else if (arm_spe_is_cache_level(record->type, L2D)) { + data_src->mem_lvl = PERF_MEM_LVL_L2; + data_src->mem_lvl |= arm_spe_is_cache_miss(record->type, L2D) ? + PERF_MEM_LVL_MISS : PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + } else if (arm_spe_is_cache_level(record->type, L1D)) { + data_src->mem_lvl = PERF_MEM_LVL_L1; + data_src->mem_lvl |= arm_spe_is_cache_miss(record->type, L1D) ? + PERF_MEM_LVL_MISS : PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + } +} + +static void arm_spe__synth_memory_level(struct arm_spe_queue *speq, + const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + struct arm_spe *spe = speq->spe; + + /* + * The data source packet contains more info for cache levels for + * peer snooping. So respect the memory level if has been set by + * data source parsing. + */ + if (!data_src->mem_lvl) { + if (data_src->mem_op == PERF_MEM_OP_LOAD) + arm_spe__synth_ld_memory_level(record, data_src); + if (data_src->mem_op == PERF_MEM_OP_STORE) + arm_spe__synth_st_memory_level(record, data_src); + } + + if (!data_src->mem_lvl) { + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; + } + + /* + * If 'mem_snoop' has been set by data source packet, skip to set + * it at here. + */ + if (!data_src->mem_snoop) { + if (record->type & ARM_SPE_DATA_SNOOPED) { + if (record->type & ARM_SPE_HITM) + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + else + data_src->mem_snoop = PERF_MEM_SNOOP_HIT; + } else { + u64 *metadata = + arm_spe__get_metadata_by_cpu(spe, speq->cpu); + + /* + * Set NA ("Not available") mode if no meta data or the + * SNOOPED event is not supported. + */ + if (!metadata || + !(metadata[ARM_SPE_CAP_EVENT_FILTER] & ARM_SPE_DATA_SNOOPED)) + data_src->mem_snoop = PERF_MEM_SNOOP_NA; + else + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + } + } + + if (!data_src->mem_remote) { + if (record->type & ARM_SPE_REMOTE_ACCESS) + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + } +} + +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr) +{ + u64 *metadata; + + /* Metadata version 1 assumes all CPUs are the same (old behavior) */ + if (spe->metadata_ver == 1) { + const char *cpuid; + + pr_warning_once("Old SPE metadata, re-record to improve decode accuracy\n"); + cpuid = perf_env__cpuid(perf_session__env(spe->session)); + if (!cpuid) + goto err; + + *midr = strtol(cpuid, NULL, 16); + return 0; + } + + metadata = arm_spe__get_metadata_by_cpu(spe, cpu); + if (!metadata) + goto err; + + *midr = metadata[ARM_SPE_CPU_MIDR]; + return 0; + +err: + pr_warning_once("Failed to get MIDR for CPU %d\n", cpu); + return -EINVAL; +} + +static void arm_spe__synth_ds(struct arm_spe_queue *speq, + const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + u64 midr; + unsigned int i; + + if (arm_spe__get_midr(speq->spe, speq->cpu, &midr)) + return; + + for (i = 0; i < ARRAY_SIZE(data_source_handles); i++) { + if (is_midr_in_range_list(midr, data_source_handles[i].midr_ranges)) { + return data_source_handles[i].ds_synth(record, data_src); + } + } + + return; +} + +static union perf_mem_data_src +arm_spe__synth_data_source(struct arm_spe_queue *speq, + const struct arm_spe_record *record) +{ + union perf_mem_data_src data_src = {}; + + if (!is_mem_op(record->op)) + return data_src; + + if (record->op & ARM_SPE_OP_LD) + data_src.mem_op = PERF_MEM_OP_LOAD; + else if (record->op & ARM_SPE_OP_ST) + data_src.mem_op = PERF_MEM_OP_STORE; + else + data_src.mem_op = PERF_MEM_OP_NA; + + arm_spe__synth_ds(speq, record, &data_src); + arm_spe__synth_memory_level(speq, record, &data_src); + + if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) { + data_src.mem_dtlb = PERF_MEM_TLB_WK; + + if (record->type & ARM_SPE_TLB_MISS) + data_src.mem_dtlb |= PERF_MEM_TLB_MISS; + else + data_src.mem_dtlb |= PERF_MEM_TLB_HIT; + } + + return data_src; +} + +static int arm_spe_sample(struct arm_spe_queue *speq) +{ + const struct arm_spe_record *record = &speq->decoder->record; + struct arm_spe *spe = speq->spe; + union perf_mem_data_src data_src; + int err; + + /* + * Discard all samples until period is reached + */ + speq->sample_count++; + if (speq->sample_count < spe->synth_opts.period) + return 0; + speq->sample_count = 0; + + arm_spe__sample_flags(speq); + data_src = arm_spe__synth_data_source(speq, record); + + if (spe->sample_flc) { + if (record->type & ARM_SPE_L1D_MISS) { + err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id, + data_src); + if (err) + return err; + } + + if (record->type & ARM_SPE_L1D_ACCESS) { + err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id, + data_src); + if (err) + return err; + } + } + + if (spe->sample_llc) { + if (record->type & ARM_SPE_LLC_MISS) { + err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id, + data_src); + if (err) + return err; + } + + if (record->type & ARM_SPE_LLC_ACCESS) { + err = arm_spe__synth_mem_sample(speq, spe->llc_access_id, + data_src); + if (err) + return err; + } + } + + if (spe->sample_tlb) { + if (record->type & ARM_SPE_TLB_MISS) { + err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id, + data_src); + if (err) + return err; + } + + if (record->type & ARM_SPE_TLB_ACCESS) { + err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id, + data_src); + if (err) + return err; + } + } + + if (spe->synth_opts.last_branch && + (spe->sample_branch || spe->sample_instructions)) + arm_spe__prep_branch_stack(speq); + + if (spe->sample_branch && (record->op & ARM_SPE_OP_BRANCH_ERET)) { + err = arm_spe__synth_branch_sample(speq, spe->branch_id); + if (err) + return err; + } + + if (spe->sample_remote_access && + (record->type & ARM_SPE_REMOTE_ACCESS)) { + err = arm_spe__synth_mem_sample(speq, spe->remote_access_id, + data_src); + if (err) + return err; + } + + if (spe->sample_memory && is_mem_op(record->op)) { + err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src); + if (err) + return err; + } + + if (spe->sample_instructions) { + err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src); + if (err) + return err; + } + + return 0; +} + +static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record; + int ret; + + if (!spe->kernel_start) + spe->kernel_start = machine__kernel_start(spe->machine); + + while (1) { + /* + * The usual logic is firstly to decode the packets, and then + * based the record to synthesize sample; but here the flow is + * reversed: it calls arm_spe_sample() for synthesizing samples + * prior to arm_spe_decode(). + * + * Two reasons for this code logic: + * 1. Firstly, when setup queue in arm_spe__setup_queue(), it + * has decoded trace data and generated a record, but the record + * is left to generate sample until run to here, so it's correct + * to synthesize sample for the left record. + * 2. After decoding trace data, it needs to compare the record + * timestamp with the coming perf event, if the record timestamp + * is later than the perf event, it needs bail out and pushs the + * record into auxtrace heap, thus the record can be deferred to + * synthesize sample until run to here at the next time; so this + * can correlate samples between Arm SPE trace data and other + * perf events with correct time ordering. + */ + + /* + * Update pid/tid info. + */ + record = &speq->decoder->record; + if (!spe->timeless_decoding && record->context_id != (u64)-1) { + ret = arm_spe_set_tid(speq, record->context_id); + if (ret) + return ret; + + spe->use_ctx_pkt_for_pid = true; + } + + ret = arm_spe_sample(speq); + if (ret) + return ret; + + ret = arm_spe_decode(speq->decoder); + if (!ret) { + pr_debug("No data or all data has been processed.\n"); + return 1; + } + + /* + * Error is detected when decode SPE trace data, continue to + * the next trace data and find out more records. + */ + if (ret < 0) + continue; + + record = &speq->decoder->record; + + /* Update timestamp for the last record */ + if (record->timestamp > speq->timestamp) + speq->timestamp = record->timestamp; + + /* + * If the timestamp of the queue is later than timestamp of the + * coming perf event, bail out so can allow the perf event to + * be processed ahead. + */ + if (!spe->timeless_decoding && speq->timestamp >= *timestamp) { + *timestamp = speq->timestamp; + return 0; + } + } + + return 0; +} + +static int arm_spe__setup_queue(struct arm_spe *spe, + struct auxtrace_queue *queue, + unsigned int queue_nr) +{ + struct arm_spe_queue *speq = queue->priv; + struct arm_spe_record *record; + + if (list_empty(&queue->head) || speq) + return 0; + + speq = arm_spe__alloc_queue(spe, queue_nr); + + if (!speq) + return -ENOMEM; + + queue->priv = speq; + + if (queue->cpu != -1) + speq->cpu = queue->cpu; + arm_spe__get_midr(spe, queue->cpu, &speq->decoder->midr); + + if (!speq->on_heap) { + int ret; + + if (spe->timeless_decoding) + return 0; + +retry: + ret = arm_spe_decode(speq->decoder); + + if (!ret) + return 0; + + if (ret < 0) + goto retry; + + record = &speq->decoder->record; + + speq->timestamp = record->timestamp; + ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp); + if (ret) + return ret; + speq->on_heap = true; + } + + return 0; +} + +static int arm_spe__setup_queues(struct arm_spe *spe) +{ + unsigned int i; + int ret; + + for (i = 0; i < spe->queues.nr_queues; i++) { + ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i); + if (ret) + return ret; + } + + return 0; +} + +static int arm_spe__update_queues(struct arm_spe *spe) +{ + if (spe->queues.new_data) { + spe->queues.new_data = false; + return arm_spe__setup_queues(spe); + } + + return 0; +} + +static bool arm_spe__is_timeless_decoding(struct arm_spe *spe) +{ + struct evsel *evsel; + struct evlist *evlist = spe->session->evlist; + bool timeless_decoding = true; + + /* + * Circle through the list of event and complain if we find one + * with the time bit set. + */ + evlist__for_each_entry(evlist, evsel) { + if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) + timeless_decoding = false; + } + + return timeless_decoding; +} + +static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp) +{ + unsigned int queue_nr; + u64 ts; + int ret; + + while (1) { + struct auxtrace_queue *queue; + struct arm_spe_queue *speq; + + if (!spe->heap.heap_cnt) + return 0; + + if (spe->heap.heap_array[0].ordinal >= timestamp) + return 0; + + queue_nr = spe->heap.heap_array[0].queue_nr; + queue = &spe->queues.queue_array[queue_nr]; + speq = queue->priv; + + auxtrace_heap__pop(&spe->heap); + + if (spe->heap.heap_cnt) { + ts = spe->heap.heap_array[0].ordinal + 1; + if (ts > timestamp) + ts = timestamp; + } else { + ts = timestamp; + } + + /* + * A previous context-switch event has set pid/tid in the machine's context, so + * here we need to update the pid/tid in the thread and SPE queue. + */ + if (!spe->use_ctx_pkt_for_pid) + arm_spe_set_pid_tid_cpu(spe, queue); + + ret = arm_spe_run_decoder(speq, &ts); + if (ret < 0) { + auxtrace_heap__add(&spe->heap, queue_nr, ts); + return ret; + } + + if (!ret) { + ret = auxtrace_heap__add(&spe->heap, queue_nr, ts); + if (ret < 0) + return ret; + } else { + speq->on_heap = false; + } + } + + return 0; +} + +static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid, + u64 time_) +{ + struct auxtrace_queues *queues = &spe->queues; + unsigned int i; + u64 ts = 0; + + for (i = 0; i < queues->nr_queues; i++) { + struct auxtrace_queue *queue = &spe->queues.queue_array[i]; + struct arm_spe_queue *speq = queue->priv; + + if (speq && (tid == -1 || speq->tid == tid)) { + speq->time = time_; + arm_spe_set_pid_tid_cpu(spe, queue); + arm_spe_run_decoder(speq, &ts); + } + } + return 0; +} + +static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample) +{ + pid_t pid, tid; + int cpu; + + if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT)) + return 0; + + pid = event->context_switch.next_prev_pid; + tid = event->context_switch.next_prev_tid; + cpu = sample->cpu; + + if (tid == -1) + pr_warning("context_switch event has no tid\n"); + + return machine__set_current_tid(spe->machine, cpu, pid, tid); +} + +static int arm_spe_process_event(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool) +{ + int err = 0; + u64 timestamp; + struct arm_spe *spe = container_of(session->auxtrace, + struct arm_spe, auxtrace); + + if (dump_trace) + return 0; + + if (!tool->ordered_events) { + pr_err("SPE trace requires ordered events\n"); + return -EINVAL; + } + + if (sample->time && (sample->time != (u64) -1)) + timestamp = perf_time_to_tsc(sample->time, &spe->tc); + else + timestamp = 0; + + if (timestamp || spe->timeless_decoding) { + err = arm_spe__update_queues(spe); + if (err) + return err; + } + + if (spe->timeless_decoding) { + if (event->header.type == PERF_RECORD_EXIT) { + err = arm_spe_process_timeless_queues(spe, + event->fork.tid, + sample->time); + } + } else if (timestamp) { + err = arm_spe_process_queues(spe, timestamp); + if (err) + return err; + + if (!spe->use_ctx_pkt_for_pid && + (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE || + event->header.type == PERF_RECORD_SWITCH)) + err = arm_spe_context_switch(spe, event, sample); + } + + return err; +} + +static int arm_spe_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + + if (!spe->data_queued) { + struct auxtrace_buffer *buffer; + off_t data_offset; + int fd = perf_data__fd(session->data); + int err; + + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } + + err = auxtrace_queues__add_event(&spe->queues, session, event, + data_offset, &buffer); + if (err) + return err; + + /* Dump here now we have copied a piped trace out of the pipe */ + if (dump_trace) { + if (auxtrace_buffer__get_data(buffer, fd)) { + u64 midr = 0; + + arm_spe__get_midr(spe, buffer->cpu.cpu, &midr); + arm_spe_dump_event(spe, buffer->data, + buffer->size, midr); + auxtrace_buffer__put_data(buffer); + } + } + } + + return 0; +} + +static int arm_spe_flush(struct perf_session *session __maybe_unused, + const struct perf_tool *tool __maybe_unused) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + int ret; + + if (dump_trace) + return 0; + + if (!tool->ordered_events) + return -EINVAL; + + ret = arm_spe__update_queues(spe); + if (ret < 0) + return ret; + + if (spe->timeless_decoding) + return arm_spe_process_timeless_queues(spe, -1, + MAX_TIMESTAMP - 1); + + ret = arm_spe_process_queues(spe, MAX_TIMESTAMP); + if (ret) + return ret; + + if (!spe->use_ctx_pkt_for_pid) + ui__warning("Arm SPE CONTEXT packets not found in the traces.\n" + "Matching of TIDs to SPE events could be inaccurate.\n"); + + return 0; +} + +static u64 *arm_spe__alloc_per_cpu_metadata(u64 *buf, int per_cpu_size) +{ + u64 *metadata; + + metadata = zalloc(per_cpu_size); + if (!metadata) + return NULL; + + memcpy(metadata, buf, per_cpu_size); + return metadata; +} + +static void arm_spe__free_metadata(u64 **metadata, int nr_cpu) +{ + int i; + + for (i = 0; i < nr_cpu; i++) + zfree(&metadata[i]); + free(metadata); +} + +static u64 **arm_spe__alloc_metadata(struct perf_record_auxtrace_info *info, + u64 *ver, int *nr_cpu) +{ + u64 *ptr = (u64 *)info->priv; + u64 metadata_size; + u64 **metadata = NULL; + int hdr_sz, per_cpu_sz, i; + + metadata_size = info->header.size - + sizeof(struct perf_record_auxtrace_info); + + /* Metadata version 1 */ + if (metadata_size == ARM_SPE_AUXTRACE_V1_PRIV_SIZE) { + *ver = 1; + *nr_cpu = 0; + /* No per CPU metadata */ + return NULL; + } + + *ver = ptr[ARM_SPE_HEADER_VERSION]; + hdr_sz = ptr[ARM_SPE_HEADER_SIZE]; + *nr_cpu = ptr[ARM_SPE_CPUS_NUM]; + + /* nr_cpu is used as a divisor below */ + if (*nr_cpu <= 0) + return NULL; + + metadata = calloc(*nr_cpu, sizeof(*metadata)); + if (!metadata) + return NULL; + + /* Locate the start address of per CPU metadata */ + ptr += hdr_sz; + per_cpu_sz = (metadata_size - (hdr_sz * sizeof(u64))) / (*nr_cpu); + + for (i = 0; i < *nr_cpu; i++) { + metadata[i] = arm_spe__alloc_per_cpu_metadata(ptr, per_cpu_sz); + if (!metadata[i]) + goto err_per_cpu_metadata; + + ptr += per_cpu_sz / sizeof(u64); + } + + return metadata; + +err_per_cpu_metadata: + arm_spe__free_metadata(metadata, *nr_cpu); + return NULL; +} + +static void arm_spe_free_queue(void *priv) +{ + struct arm_spe_queue *speq = priv; + + if (!speq) + return; + thread__zput(speq->thread); + arm_spe_decoder_free(speq->decoder); + zfree(&speq->event_buf); + zfree(&speq->last_branch); + free(speq); +} + +static void arm_spe_free_events(struct perf_session *session) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + struct auxtrace_queues *queues = &spe->queues; + unsigned int i; + + for (i = 0; i < queues->nr_queues; i++) { + arm_spe_free_queue(queues->queue_array[i].priv); + queues->queue_array[i].priv = NULL; + } + auxtrace_queues__free(queues); +} + +static void arm_spe_free(struct perf_session *session) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + + auxtrace_heap__free(&spe->heap); + arm_spe_free_events(session); + session->auxtrace = NULL; + arm_spe__free_metadata(spe->metadata, spe->metadata_nr_cpu); + free(spe); +} + +static bool arm_spe_evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace); + + return evsel->core.attr.type == spe->pmu_type; +} + +static const char * const metadata_hdr_v1_fmts[] = { + [ARM_SPE_PMU_TYPE] = " PMU Type :%"PRId64"\n", + [ARM_SPE_PER_CPU_MMAPS] = " Per CPU mmaps :%"PRId64"\n", +}; + +static const char * const metadata_hdr_fmts[] = { + [ARM_SPE_HEADER_VERSION] = " Header version :%"PRId64"\n", + [ARM_SPE_HEADER_SIZE] = " Header size :%"PRId64"\n", + [ARM_SPE_PMU_TYPE_V2] = " PMU type v2 :%"PRId64"\n", + [ARM_SPE_CPUS_NUM] = " CPU number :%"PRId64"\n", +}; + +static const char * const metadata_per_cpu_fmts[] = { + [ARM_SPE_MAGIC] = " Magic :0x%"PRIx64"\n", + [ARM_SPE_CPU] = " CPU # :%"PRId64"\n", + [ARM_SPE_CPU_NR_PARAMS] = " Num of params :%"PRId64"\n", + [ARM_SPE_CPU_MIDR] = " MIDR :0x%"PRIx64"\n", + [ARM_SPE_CPU_PMU_TYPE] = " PMU Type :%"PRId64"\n", + [ARM_SPE_CAP_MIN_IVAL] = " Min Interval :%"PRId64"\n", + [ARM_SPE_CAP_EVENT_FILTER] = " Event Filter :0x%"PRIx64"\n", +}; + +static void arm_spe_print_info(struct arm_spe *spe, __u64 *arr) +{ + unsigned int i, cpu, hdr_size, cpu_num, cpu_size; + const char * const *hdr_fmts; + + if (!dump_trace) + return; + + if (spe->metadata_ver == 1) { + cpu_num = 0; + hdr_size = ARM_SPE_AUXTRACE_V1_PRIV_MAX; + hdr_fmts = metadata_hdr_v1_fmts; + } else { + cpu_num = arr[ARM_SPE_CPUS_NUM]; + hdr_size = arr[ARM_SPE_HEADER_SIZE]; + hdr_fmts = metadata_hdr_fmts; + } + + for (i = 0; i < hdr_size; i++) + fprintf(stdout, hdr_fmts[i], arr[i]); + + arr += hdr_size; + for (cpu = 0; cpu < cpu_num; cpu++) { + /* + * The parameters from ARM_SPE_MAGIC to ARM_SPE_CPU_NR_PARAMS + * are fixed. The sequential parameter size is decided by the + * field 'ARM_SPE_CPU_NR_PARAMS'. + */ + cpu_size = (ARM_SPE_CPU_NR_PARAMS + 1) + arr[ARM_SPE_CPU_NR_PARAMS]; + for (i = 0; i < cpu_size; i++) + fprintf(stdout, metadata_per_cpu_fmts[i], arr[i]); + arr += cpu_size; + } +} + +static void arm_spe_set_event_name(struct evlist *evlist, u64 id, + const char *name) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.id && evsel->core.id[0] == id) { + if (evsel->name) + zfree(&evsel->name); + evsel->name = strdup(name); + break; + } + } +} + +static int +arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + struct perf_event_attr attr; + bool found = false; + u64 id; + int err; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == spe->pmu_type) { + found = true; + break; + } + } + + if (!found) { + pr_debug("No selected events with SPE trace data\n"); + return 0; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.size = sizeof(struct perf_event_attr); + attr.type = PERF_TYPE_HARDWARE; + attr.sample_type = evsel->core.attr.sample_type & + (PERF_SAMPLE_MASK | PERF_SAMPLE_PHYS_ADDR); + attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | + PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC | + PERF_SAMPLE_WEIGHT | PERF_SAMPLE_ADDR; + if (spe->timeless_decoding) + attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; + else + attr.sample_type |= PERF_SAMPLE_TIME; + + spe->sample_type = attr.sample_type; + + attr.exclude_user = evsel->core.attr.exclude_user; + attr.exclude_kernel = evsel->core.attr.exclude_kernel; + attr.exclude_hv = evsel->core.attr.exclude_hv; + attr.exclude_host = evsel->core.attr.exclude_host; + attr.exclude_guest = evsel->core.attr.exclude_guest; + attr.sample_id_all = evsel->core.attr.sample_id_all; + attr.read_format = evsel->core.attr.read_format; + attr.sample_period = spe->synth_opts.period; + + /* create new id val to be a fixed offset from evsel id */ + id = auxtrace_synth_id_range_start(evsel); + + if (spe->synth_opts.flc) { + spe->sample_flc = true; + + /* Level 1 data cache miss */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->l1d_miss_id = id; + arm_spe_set_event_name(evlist, id, "l1d-miss"); + id += 1; + + /* Level 1 data cache access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->l1d_access_id = id; + arm_spe_set_event_name(evlist, id, "l1d-access"); + id += 1; + } + + if (spe->synth_opts.llc) { + spe->sample_llc = true; + + /* Last level cache miss */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->llc_miss_id = id; + arm_spe_set_event_name(evlist, id, "llc-miss"); + id += 1; + + /* Last level cache access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->llc_access_id = id; + arm_spe_set_event_name(evlist, id, "llc-access"); + id += 1; + } + + if (spe->synth_opts.tlb) { + spe->sample_tlb = true; + + /* TLB miss */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->tlb_miss_id = id; + arm_spe_set_event_name(evlist, id, "tlb-miss"); + id += 1; + + /* TLB access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->tlb_access_id = id; + arm_spe_set_event_name(evlist, id, "tlb-access"); + id += 1; + } + + if (spe->synth_opts.last_branch) { + if (spe->synth_opts.last_branch_sz > 2) + pr_debug("Arm SPE supports only two bstack entries (PBT+TGT).\n"); + + attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + /* + * We don't use the hardware index, but the sample generation + * code uses the new format branch_stack with this field, + * so the event attributes must indicate that it's present. + */ + attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + + if (spe->synth_opts.branches) { + spe->sample_branch = true; + + /* Branch */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->branch_id = id; + arm_spe_set_event_name(evlist, id, "branch"); + id += 1; + } + + if (spe->synth_opts.remote_access) { + spe->sample_remote_access = true; + + /* Remote access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->remote_access_id = id; + arm_spe_set_event_name(evlist, id, "remote-access"); + id += 1; + } + + if (spe->synth_opts.mem) { + spe->sample_memory = true; + + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->memory_id = id; + arm_spe_set_event_name(evlist, id, "memory"); + id += 1; + } + + if (spe->synth_opts.instructions) { + spe->sample_instructions = true; + attr.config = PERF_COUNT_HW_INSTRUCTIONS; + + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->instructions_id = id; + arm_spe_set_event_name(evlist, id, "instructions"); + } + + return 0; +} + +static bool arm_spe__is_homogeneous(u64 **metadata, int nr_cpu) +{ + u64 midr; + int i; + + if (!nr_cpu) + return false; + + for (i = 0; i < nr_cpu; i++) { + if (!metadata[i]) + return false; + + if (i == 0) { + midr = metadata[i][ARM_SPE_CPU_MIDR]; + continue; + } + + if (midr != metadata[i][ARM_SPE_CPU_MIDR]) + return false; + } + + return true; +} + +int arm_spe_process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + size_t min_sz = ARM_SPE_AUXTRACE_V1_PRIV_SIZE; + struct perf_record_time_conv *tc = &session->time_conv; + struct arm_spe *spe; + u64 **metadata = NULL; + u64 metadata_ver; + int nr_cpu, err; + + if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) + + min_sz) + return -EINVAL; + + metadata = arm_spe__alloc_metadata(auxtrace_info, &metadata_ver, + &nr_cpu); + if (!metadata && metadata_ver != 1) { + pr_err("Failed to parse Arm SPE metadata.\n"); + return -EINVAL; + } + + spe = zalloc(sizeof(struct arm_spe)); + if (!spe) { + err = -ENOMEM; + goto err_free_metadata; + } + + err = auxtrace_queues__init(&spe->queues); + if (err) + goto err_free; + + spe->session = session; + spe->machine = &session->machines.host; /* No kvm support */ + spe->auxtrace_type = auxtrace_info->type; + if (metadata_ver == 1) + spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; + else + spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE_V2]; + spe->metadata = metadata; + spe->metadata_ver = metadata_ver; + spe->metadata_nr_cpu = nr_cpu; + spe->is_homogeneous = arm_spe__is_homogeneous(metadata, nr_cpu); + + spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); + + /* + * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead + * and the parameters for hardware clock are stored in the session + * context. Passes these parameters to the struct perf_tsc_conversion + * in "spe->tc", which is used for later conversion between clock + * counter and timestamp. + * + * For backward compatibility, copies the fields starting from + * "time_cycles" only if they are contained in the event. + */ + spe->tc.time_shift = tc->time_shift; + spe->tc.time_mult = tc->time_mult; + spe->tc.time_zero = tc->time_zero; + + if (event_contains(*tc, cap_user_time_short)) { + spe->tc.time_cycles = tc->time_cycles; + spe->tc.time_mask = tc->time_mask; + spe->tc.cap_user_time_zero = tc->cap_user_time_zero; + spe->tc.cap_user_time_short = tc->cap_user_time_short; + } + + spe->auxtrace.process_event = arm_spe_process_event; + spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event; + spe->auxtrace.flush_events = arm_spe_flush; + spe->auxtrace.free_events = arm_spe_free_events; + spe->auxtrace.free = arm_spe_free; + spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace; + session->auxtrace = &spe->auxtrace; + + arm_spe_print_info(spe, &auxtrace_info->priv[0]); + + if (dump_trace) + return 0; + + if (session->itrace_synth_opts && session->itrace_synth_opts->set) { + spe->synth_opts = *session->itrace_synth_opts; + } else { + itrace_synth_opts__set_default(&spe->synth_opts, false); + /* Default nanoseconds period not supported */ + spe->synth_opts.period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS; + spe->synth_opts.period = 1; + } + + if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) { + ui__error("You must only use i (instructions) --itrace period with Arm SPE. e.g --itrace=i1i\n"); + err = -EINVAL; + goto err_free_queues; + } + if (spe->synth_opts.period > 1) + ui__warning("Arm SPE has a hardware-based sampling period.\n\n" + "--itrace periods > 1i downsample by an interval of n SPE samples rather than n instructions.\n"); + + err = arm_spe_synth_events(spe, session); + if (err) + goto err_free_queues; + + err = auxtrace_queues__process_index(&spe->queues, session); + if (err) + goto err_free_queues; + + if (spe->queues.populated) + spe->data_queued = true; + + return 0; + +err_free_queues: + auxtrace_queues__free(&spe->queues); + session->auxtrace = NULL; +err_free: + free(spe); +err_free_metadata: + arm_spe__free_metadata(metadata, nr_cpu); + return err; +} diff --git a/tools/perf/util/arm-spe.h b/tools/perf/util/arm-spe.h new file mode 100644 index 000000000..3966df185 --- /dev/null +++ b/tools/perf/util/arm-spe.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#ifndef INCLUDE__PERF_ARM_SPE_H__ +#define INCLUDE__PERF_ARM_SPE_H__ + +#define ARM_SPE_PMU_NAME "arm_spe_" + +enum { + ARM_SPE_PMU_TYPE, + ARM_SPE_PER_CPU_MMAPS, + ARM_SPE_AUXTRACE_V1_PRIV_MAX, +}; + +#define ARM_SPE_AUXTRACE_V1_PRIV_SIZE \ + (ARM_SPE_AUXTRACE_V1_PRIV_MAX * sizeof(u64)) + +enum { + /* + * The old metadata format (defined above) does not include a + * field for version number. Version 1 is reserved and starts + * from version 2. + */ + ARM_SPE_HEADER_VERSION, + /* Number of sizeof(u64) */ + ARM_SPE_HEADER_SIZE, + /* PMU type shared by CPUs */ + ARM_SPE_PMU_TYPE_V2, + /* Number of CPUs */ + ARM_SPE_CPUS_NUM, + ARM_SPE_AUXTRACE_PRIV_MAX, +}; + +enum { + /* Magic number */ + ARM_SPE_MAGIC, + /* CPU logical number in system */ + ARM_SPE_CPU, + /* Number of parameters */ + ARM_SPE_CPU_NR_PARAMS, + /* CPU MIDR */ + ARM_SPE_CPU_MIDR, + /* Associated PMU type */ + ARM_SPE_CPU_PMU_TYPE, + /* Minimal interval */ + ARM_SPE_CAP_MIN_IVAL, + /* Event filter */ + ARM_SPE_CAP_EVENT_FILTER, + ARM_SPE_CPU_PRIV_MAX, +}; + +#define ARM_SPE_HEADER_CURRENT_VERSION 2 + + +union perf_event; +struct perf_session; +struct perf_pmu; + +struct auxtrace_record *arm_spe_recording_init(int *err, + struct perf_pmu *arm_spe_pmu); + +int arm_spe_process_auxtrace_info(union perf_event *event, + struct perf_session *session); + +void arm_spe_pmu_default_config(const struct perf_pmu *arm_spe_pmu, + struct perf_event_attr *attr); + +#endif diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.c b/tools/perf/util/arm64-frame-pointer-unwind-support.c new file mode 100644 index 000000000..3af8c7a46 --- /dev/null +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "arm64-frame-pointer-unwind-support.h" +#include "callchain.h" +#include "event.h" +#include "record.h" +#include "unwind.h" +#include + +#define perf_event_arm_regs perf_event_arm64_regs +#include "../../arch/arm64/include/uapi/asm/perf_regs.h" +#undef perf_event_arm_regs + +struct entries { + u64 stack[2]; + size_t length; +}; + +#define SMPL_REG_MASK(b) (1ULL << (b)) + +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts) +{ + opts->sample_user_regs |= SMPL_REG_MASK(PERF_REG_ARM64_LR); +} + +static bool get_leaf_frame_caller_enabled(struct perf_sample *sample) +{ + struct regs_dump *regs; + + if (callchain_param.record_mode != CALLCHAIN_FP) + return false; + + regs = perf_sample__user_regs(sample); + return regs->regs && regs->mask & SMPL_REG_MASK(PERF_REG_ARM64_LR); +} + +static int add_entry(struct unwind_entry *entry, void *arg) +{ + struct entries *entries = arg; + + entries->stack[entries->length++] = entry->ip; + return 0; +} + +u64 get_leaf_frame_caller_aarch64(struct perf_sample *sample, struct thread *thread, int usr_idx) +{ + int ret; + struct entries entries = {}; + struct regs_dump old_regs, *regs; + + if (!get_leaf_frame_caller_enabled(sample)) + return 0; + + /* + * If PC and SP are not recorded, get the value of PC from the stack + * and set its mask. SP is not used when doing the unwinding but it + * still needs to be set to prevent failures. + */ + regs = perf_sample__user_regs(sample); + memcpy(&old_regs, regs, sizeof(*regs)); + if (!(regs->mask & SMPL_REG_MASK(PERF_REG_ARM64_PC))) { + regs->cache_mask |= SMPL_REG_MASK(PERF_REG_ARM64_PC); + regs->cache_regs[PERF_REG_ARM64_PC] = sample->callchain->ips[usr_idx+1]; + } + + if (!(regs->mask & SMPL_REG_MASK(PERF_REG_ARM64_SP))) { + regs->cache_mask |= SMPL_REG_MASK(PERF_REG_ARM64_SP); + regs->cache_regs[PERF_REG_ARM64_SP] = 0; + } + + ret = unwind__get_entries(add_entry, &entries, thread, sample, 2, true); + memcpy(regs, &old_regs, sizeof(*regs)); + + if (ret || entries.length != 2) + return ret; + + return callchain_param.order == ORDER_CALLER ? entries.stack[0] : entries.stack[1]; +} diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.h b/tools/perf/util/arm64-frame-pointer-unwind-support.h new file mode 100644 index 000000000..ba35b295b --- /dev/null +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H +#define __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H + +#include + +struct perf_sample; +struct record_opts; +struct thread; + +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts); +u64 get_leaf_frame_caller_aarch64(struct perf_sample *sample, struct thread *thread, int user_idx); + +#endif /* __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H */ diff --git a/tools/perf/util/aslr.c b/tools/perf/util/aslr.c new file mode 100644 index 000000000..027695d96 --- /dev/null +++ b/tools/perf/util/aslr.c @@ -0,0 +1,1410 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "aslr.h" + +#include "addr_location.h" +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "evlist.h" +#include "machine.h" +#include "map.h" +#include "thread.h" +#include "tool.h" +#include "session.h" +#include "data.h" +#include "dso.h" +#include "pmu.h" +#include "pmus.h" + +#include /* page_size */ +#include +#include +#include +#include +#include +#include + +/** + * struct remap_addresses_key - Key for mapping original addresses to remapped ones. + * @dso: Pointer to the DSO (Dynamic Shared Object) associated with the mapping. + * @invariant: Unique offset invariant within the VMA (Virtual Memory Area). + * Calculated as `start - pgoff`. This value remains constant when + * perf's internal `maps__fixup_overlap_and_insert` splits a map into + * fragmented VMA pieces due to overlapping events, allowing us to + * resolve split maps consistently back to the original VMA. + * @pid: Process ID associated with the mapping. + */ +struct remap_addresses_key { + struct machine *machine; + struct dso *dso; + u64 invariant; + pid_t pid; +}; + +struct aslr_mapping { + struct list_head node; + u64 orig_start; + u64 len; + u64 remap_start; +}; + +struct aslr_evsel_priv { + u64 orig_sample_type; + u64 orig_sample_regs_user; + u64 orig_sample_regs_intr; + int orig_sample_size; +}; + +static size_t evsel_hash(long key, void *ctx __maybe_unused) +{ + return (size_t)key; +} + +static bool evsel_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +struct process_top_address { + u64 remapped_max; +}; +struct aslr_tool { + /** @tool: The tool implemented here and a pointer to a delegate to process the data. */ + struct delegate_tool tool; + /** @machines: The machines with the input, not remapped, virtual address layout. */ + struct machines machines; + /** @event_copy: Buffer used to create an event to pass to the delegate. */ + char event_copy[PERF_SAMPLE_MAX_SIZE] __aligned(8); + /** @remap_addresses: mapping from remap_addresses_key to remapped address. */ + struct hashmap remap_addresses; + /** @top_addresses: mapping from process to max remapped address. */ + struct hashmap top_addresses; + /** + * @evsel_orig_attrs: mapping from evsel pointer to its original + * unstripped sample_type and registers bitmasks. + */ + struct hashmap evsel_orig_attrs; +}; + +static const pid_t kernel_pid = -1; + +/* Start remapping user processes from a small non-zero offset. */ +static const u64 user_space_start = 0x200000; +static const u64 kernel_space_start_64 = 0xffff800010000000ULL; +static const u64 kernel_space_start_32 = 0x80000000ULL; + +static size_t remap_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key = (struct remap_addresses_key *)_key; + void *dso_ptr = key->dso ? RC_CHK_ACCESS(key->dso) : NULL; + + return (size_t)key->machine ^ (size_t)dso_ptr ^ key->invariant ^ key->pid; +} + +static bool remap_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key1 = (struct remap_addresses_key *)_key1; + struct remap_addresses_key *key2 = (struct remap_addresses_key *)_key2; + + return key1->machine == key2->machine && + RC_CHK_EQUAL(key1->dso, key2->dso) && + key1->invariant == key2->invariant && + key1->pid == key2->pid; +} + +struct top_addresses_key { + struct machine *machine; + pid_t pid; +}; + +static size_t top_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct top_addresses_key *key = (struct top_addresses_key *)_key; + + return (size_t)key->machine ^ key->pid; +} + +static bool top_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct top_addresses_key *key1 = (struct top_addresses_key *)_key1; + struct top_addresses_key *key2 = (struct top_addresses_key *)_key2; + + return key1->machine == key2->machine && key1->pid == key2->pid; +} + +static u64 round_up_to_page_size(u64 addr) +{ + return (addr + page_size - 1) & ~((u64)page_size - 1); +} + +static u64 aslr_tool__remap_address(struct aslr_tool *aslr, + struct thread *aslr_thread, + u8 cpumode, + u64 addr) +{ + struct addr_location al; + struct remap_addresses_key key; + u64 *remapped_invariant_ptr = NULL; + u64 remap_addr = 0; + u8 effective_cpumode = cpumode; + struct dso *dso; + const char *dso_name; + + if (!aslr_thread) + return 0; /* No thread. */ + + addr_location__init(&al); + if (!thread__find_map(aslr_thread, cpumode, addr, &al)) { + /* + * If lookup fails with specified cpumode, try fallback to the other space + * to be robust against bad cpumode in samples. + */ + if (cpumode == PERF_RECORD_MISC_KERNEL) + effective_cpumode = PERF_RECORD_MISC_USER; + else if (cpumode == PERF_RECORD_MISC_USER) + effective_cpumode = PERF_RECORD_MISC_KERNEL; + else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL) + effective_cpumode = PERF_RECORD_MISC_GUEST_USER; + else if (cpumode == PERF_RECORD_MISC_GUEST_USER) + effective_cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + + if (!thread__find_map(aslr_thread, effective_cpumode, addr, &al)) { + addr_location__exit(&al); + return 0; /* No mmap. */ + } + } + + dso = map__dso(al.map); + dso_name = dso ? dso__long_name(dso) : NULL; + + key.machine = maps__machine(thread__maps(aslr_thread)); + key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + key.invariant = map__start(al.map) - map__pgoff(al.map); + else + key.invariant = map__start(al.map); + key.pid = (effective_cpumode == PERF_RECORD_MISC_KERNEL || + effective_cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + + if (hashmap__find(&aslr->remap_addresses, &key, &remapped_invariant_ptr)) { + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (addr - map__start(al.map)); + } else { + pr_debug("Cannot find a remapped entry for address %" PRIx64 " in mapping %" PRIx64 "(%zu) for pid=%d\n", + addr, map__start(al.map), map__size(al.map), key.pid); + } + + addr_location__exit(&al); + return remap_addr; +} + +struct aslr_machine_priv { + bool kernel_maps_loaded; +}; + +static int aslr_tool__preload_kernel_maps(struct machine *machine) +{ + struct aslr_machine_priv *mpriv = machine->priv; + + if (!mpriv) { + mpriv = zalloc(sizeof(*mpriv)); + if (!mpriv) + return -ENOMEM; + machine->priv = mpriv; + } + + if (!mpriv->kernel_maps_loaded) { + struct maps *kmaps = machine__kernel_maps(machine); + + if (kmaps) { + int err = maps__load_maps(kmaps); + + if (err < 0) { + pr_err("ASLR: Failed to preload kernel maps for machine pid %d\n", + machine->pid); + return err; + } + } + mpriv->kernel_maps_loaded = true; + } + return 0; +} + +static void aslr_tool__free_machine_priv(struct machine *machine) +{ + free(machine->priv); + machine->priv = NULL; +} + +static void aslr_tool__destroy_machines_priv(struct machines *machines) +{ + struct rb_node *nd; + + aslr_tool__free_machine_priv(&machines->host); + for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + aslr_tool__free_machine_priv(machine); + } +} + +static u64 aslr_tool__findnew_mapping(struct aslr_tool *aslr, + struct machine *session_machine, + struct thread *aslr_thread, + u8 cpumode, u64 start, + u64 len, u64 pgoff) +{ + /* Address location for dso lookup. */ + struct addr_location al; + /* Original ASLR address based key for the remap table. */ + struct remap_addresses_key remap_key; + /* The address in the ASLR sanitized address space less pg_off. */ + u64 *remapped_invariant_ptr; + /* Key for the maximum address in a process. */ + struct top_addresses_key top_addr_key; + /* Value in top address table. */ + struct process_top_address *top = NULL; + /* Address in ASLR sanitized address space. */ + u64 remap_addr; + /* Potentially allocated remap table key. */ + struct remap_addresses_key *new_remap_key = NULL; + /* + * Potentially allocated remap table key. + * TODO: Avoid allocation necessary for perf 32-bit binary support. + */ + u64 *new_remap_val = NULL; + int err; + + if (!aslr_thread) + return 0; + + /* The key to look up an incoming address to the outgoing value. */ + addr_location__init(&al); + remap_key.machine = maps__machine(thread__maps(aslr_thread)); + remap_key.pid = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + if (thread__find_map(aslr_thread, cpumode, start, &al)) { + struct dso *dso = map__dso(al.map); + const char *dso_name = dso ? dso__long_name(dso) : NULL; + + remap_key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + remap_key.invariant = map__start(al.map) - map__pgoff(al.map); + else + remap_key.invariant = map__start(al.map); + } else { + remap_key.dso = NULL; + remap_key.invariant = start; + } + + /* The key to look up top allocated address. */ + top_addr_key.machine = remap_key.machine; + top_addr_key.pid = remap_key.pid; + + if (hashmap__find(&aslr->remap_addresses, &remap_key, &remapped_invariant_ptr)) { + /* Mmap already exists. */ + u64 calculated_max; + + if (al.map) { + /* + * The cached value is the base of the invariant. We add the + * offset into the VMA (start - map__start), plus the map's + * pgoff, to get the precise virtual address within this chunk. + */ + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (start - map__start(al.map)); + } else { + /* + * For unmapped memory (e.g. kernel anonymous), the cached value + * was stored offset by pgoff. Adding pgoff yields the true remap_addr. + */ + remap_addr = *remapped_invariant_ptr + pgoff; + } + + calculated_max = remap_addr + len; + + /* See if top mapping was expanded. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + if (calculated_max > top->remapped_max) + top->remapped_max = calculated_max; + } + addr_location__exit(&al); + return remap_addr; + } + /* No mmap, create an entry from the top address. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + struct addr_location prev_al; + bool is_contiguous = false; + + /* Current max allocated mmap address within the process. */ + remap_addr = top->remapped_max; + + addr_location__init(&prev_al); + if (thread__find_map(aslr_thread, cpumode, start - 1, &prev_al)) { + if (map__end(prev_al.map) == start) + is_contiguous = true; + } + addr_location__exit(&prev_al); + + if (is_contiguous) { + /* Contiguous mapping, do not add 1 page gap! */ + remap_addr = round_up_to_page_size(remap_addr); + } else { + /* Give 1 page gap from current max page. */ + remap_addr = round_up_to_page_size(remap_addr); + remap_addr += page_size; + } + if (remap_addr + len > top->remapped_max) + top->remapped_max = remap_addr + len; + } else { + /* First address of the process, allocate key and first top address. */ + struct top_addresses_key *tk; + struct process_top_address *top_val; + struct perf_env *env = session_machine ? session_machine->env : NULL; + bool is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + u64 kernel_start_addr = is_64 ? kernel_space_start_64 : kernel_space_start_32; + + remap_addr = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_start_addr : user_space_start; + remap_addr = round_up_to_page_size(remap_addr); + + tk = malloc(sizeof(*tk)); + top_val = malloc(sizeof(*top_val)); + if (!tk || !top_val) { + err = -ENOMEM; + } else { + *tk = top_addr_key; + top_val->remapped_max = remap_addr + len; + err = hashmap__insert(&aslr->top_addresses, tk, top_val, + HASHMAP_ADD, NULL, NULL); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR process top address %m\n"); + free(tk); + free(top_val); + addr_location__exit(&al); + return 0; + } + } + /* Create rmeapping entry. */ + new_remap_key = malloc(sizeof(*new_remap_key)); + new_remap_val = malloc(sizeof(u64)); + if (!new_remap_key || !new_remap_val) { + err = -ENOMEM; + } else { + *new_remap_key = remap_key; + new_remap_key->dso = dso__get(remap_key.dso); + if (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) { + if (al.map) { + *new_remap_val = remap_addr - + (start - map__start(al.map)) - + map__pgoff(al.map); + } else { + /* + * Subtract pgoff from the base virtual address so that + * when the lookup path adds pgoff back, it perfectly + * cancels out and returns remap_addr. + */ + *new_remap_val = remap_addr - pgoff; + } + } else { + *new_remap_val = remap_addr - (al.map ? (start - map__start(al.map)) + + map__pgoff(al.map) : pgoff); + } + err = hashmap__add(&aslr->remap_addresses, new_remap_key, new_remap_val); + if (err) + dso__put(new_remap_key->dso); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR remapping %m\n"); + free(new_remap_key); + free(new_remap_val); + addr_location__exit(&al); + return 0; + } + addr_location__exit(&al); + return remap_addr; +} + +static int aslr_tool__process_mmap(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap.pid, event->mmap.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap, &event->mmap, event->mmap.header.size); + /* Remaps the mmap.start. */ + new_event->mmap.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap.start, + event->mmap.len, + event->mmap.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap.filename) || is_no_dso_memory(event->mmap.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap.filename, cpumode))) + new_event->mmap.pgoff = new_event->mmap.start; + err = delegate->mmap(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_mmap2(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap2(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap2.pid, event->mmap2.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap2, &event->mmap2, event->mmap2.header.size); + /* Remaps the mmap.start. */ + new_event->mmap2.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap2.start, + event->mmap2.len, + event->mmap2.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap2.filename) || is_no_dso_memory(event->mmap2.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap2.filename, cpumode))) + new_event->mmap2.pgoff = new_event->mmap2.start; + err = delegate->mmap2(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_comm(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_comm(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->comm(delegate, event, sample, machine); +} + +static int aslr_tool__process_fork(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_fork(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->fork(delegate, event, sample, machine); +} + +static int aslr_tool__process_exit(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_exit(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->exit(delegate, event, sample, machine); +} + +static int aslr_tool__process_text_poke(const struct perf_tool *tool __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + /* Drop in case the instruction encodes an ASLR revealing address. */ + return 0; +} + +static int aslr_tool__process_ksymbol(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + struct thread *thread; + struct machine *aslr_machine; + bool is_unregister; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, kernel_pid, 0); + if (!thread) + return -ENOMEM; + + is_unregister = (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER); + + memcpy(&new_event->ksymbol, &event->ksymbol, event->ksymbol.header.size); + + if (is_unregister) { + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + } else { + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + } + if (err) { + thread__put(thread); + return err; + } + + err = delegate->ksymbol(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_sample(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct evsel *evsel = sample->evsel; + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + int ret; + int orig_sample_size; + u64 sample_type; + struct thread *thread; + struct machine *aslr_machine; + __u64 max_i; + __u64 max_j; + union perf_event *new_event; + struct perf_sample new_sample; + __u64 *in_array, *out_array; + u8 cpumode; + u64 addr; + size_t i; + size_t j; + struct aslr_evsel_priv *priv = NULL; + u64 orig_sample_type; + u64 orig_regs_user; + u64 orig_regs_intr; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + if (evsel__is_dummy_event(evsel)) + return delegate->sample(delegate, event, sample, machine); + + ret = -EFAULT; + + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + orig_sample_type = priv->orig_sample_type; + orig_regs_user = priv->orig_sample_regs_user; + orig_regs_intr = priv->orig_sample_regs_intr; + } else { + orig_sample_type = evsel->core.attr.sample_type; + orig_regs_user = evsel->core.attr.sample_regs_user; + orig_regs_intr = evsel->core.attr.sample_regs_intr; + } + + orig_sample_size = evsel->sample_size; + + sample_type = orig_sample_type; + sample_type &= ~PERF_SAMPLE_REGS_USER; + sample_type &= ~PERF_SAMPLE_REGS_INTR; + sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + max_i = (event->header.size - sizeof(struct perf_event_header)) / sizeof(__u64); + max_j = (PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_event_header)) / sizeof(__u64); + new_event = (union perf_event *)aslr->event_copy; + cpumode = sample->cpumode; + i = 0; + j = 0; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, sample->pid, sample->tid); + + if (!thread) + return -ENOMEM; + + if (max_i > PERF_SAMPLE_MAX_SIZE / sizeof(u64)) + goto out_put; + + new_event->sample.header = event->sample.header; + + in_array = &event->sample.array[0]; + out_array = &new_event->sample.array[0]; + +#define CHECK_BOUNDS(required_i, required_j) \ + (i + (required_i) > max_i || j + (required_j) > max_j) + +#define COPY_U64() \ + do { \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + out_array[j++] = in_array[i++]; \ + } while (0) + +#define REMAP_U64(addr_field) \ + do { \ + u64 remapped; \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + remapped = aslr_tool__remap_address(aslr, thread, cpumode, addr_field); \ + out_array[j++] = remapped; \ + i++; \ + } while (0) + + if (orig_sample_type & PERF_SAMPLE_IDENTIFIER) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_IP) + REMAP_U64(sample->ip); + if (orig_sample_type & PERF_SAMPLE_TID) { + union { + u64 val64; + u32 val32[2]; + } u; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + u.val32[0] = sample->pid; + u.val32[1] = sample->tid; + out_array[j++] = u.val64; + i++; + } + if (orig_sample_type & PERF_SAMPLE_TIME) + COPY_U64(); /* time */ + if (orig_sample_type & PERF_SAMPLE_ADDR) + REMAP_U64(sample->addr); + if (orig_sample_type & PERF_SAMPLE_ID) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_STREAM_ID) + COPY_U64(); /* stream_id */ + if (orig_sample_type & PERF_SAMPLE_CPU) + COPY_U64(); /* cpu, res */ + if (orig_sample_type & PERF_SAMPLE_PERIOD) + COPY_U64(); /* period */ + if (orig_sample_type & PERF_SAMPLE_READ) { + if ((evsel->core.attr.read_format & PERF_FORMAT_GROUP) == 0) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } else { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + for (u64 cntr = 0; cntr < nr; cntr++) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } + } + } + if (orig_sample_type & PERF_SAMPLE_CALLCHAIN) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + for (u64 cntr = 0; cntr < nr; cntr++) { + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + addr = in_array[i++]; + if (addr >= PERF_CONTEXT_MAX) { + out_array[j++] = addr; + switch (addr) { + case PERF_CONTEXT_HV: + cpumode = PERF_RECORD_MISC_HYPERVISOR; + break; + case PERF_CONTEXT_KERNEL: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + case PERF_CONTEXT_USER: + cpumode = PERF_RECORD_MISC_USER; + break; + case PERF_CONTEXT_GUEST: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_KERNEL: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_USER: + cpumode = PERF_RECORD_MISC_GUEST_USER; + break; + case PERF_CONTEXT_USER_DEFERRED: + if (cntr + 1 >= nr) { + pr_debug("Truncated callchain deferred cookie context\n"); + ret = 0; + goto out_put; + } + /* + * Immediately followed by a 64-bit + * stitching cookie. Skip/Copy it! + */ + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j++] = in_array[i++]; + cntr++; + cpumode = PERF_RECORD_MISC_USER; + break; + default: + pr_debug("invalid callchain context: %"PRIx64"\n", addr); + ret = 0; + goto out_put; + } + continue; + } + addr = aslr_tool__remap_address(aslr, thread, cpumode, addr); + out_array[j++] = addr; + } + } + if (orig_sample_type & PERF_SAMPLE_RAW) { + size_t bytes = sizeof(u32) + sample->raw_size; + size_t u64_words = (bytes + 7) / 8; + + if (i + u64_words > max_i || j + u64_words > max_j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], bytes); + i += u64_words; + j += u64_words; + /* + * TODO: certain raw samples can be remapped, such as + * tracepoints by examining their fields. + */ + pr_debug("Dropping raw samples as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_BRANCH_STACK) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + COPY_U64(); /* hw_idx */ + + if (nr > (ULLONG_MAX / 3)) { + ret = -EFAULT; + goto out_put; + } + if (nr * 3 > max_i - i || nr * 3 > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) { + u64 from = in_array[i++]; + u64 to = in_array[i++]; + + from = aslr_tool__remap_address(aslr, thread, sample->cpumode, from); + to = aslr_tool__remap_address(aslr, thread, sample->cpumode, to); + + out_array[j++] = from; + out_array[j++] = to; + out_array[j++] = in_array[i++]; /* flags */ + } + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) { + if (nr > max_i - i || nr > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) + COPY_U64(); + } + } + if (orig_sample_type & PERF_SAMPLE_REGS_USER) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_user); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_STACK_USER) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + size = in_array[i]; + COPY_U64(); + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping stack user sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_WEIGHT_TYPE) + COPY_U64(); /* perf_sample_weight */ + if (orig_sample_type & PERF_SAMPLE_DATA_SRC) + COPY_U64(); /* data_src */ + if (orig_sample_type & PERF_SAMPLE_TRANSACTION) + COPY_U64(); /* transaction */ + if (orig_sample_type & PERF_SAMPLE_REGS_INTR) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_intr); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_PHYS_ADDR) { + COPY_U64(); /* phys_addr */ + /* TODO: can this be less conservative? */ + pr_debug("Dropping physical address sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_CGROUP) + COPY_U64(); /* cgroup */ + if (orig_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) + COPY_U64(); /* data_page_size */ + if (orig_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) + COPY_U64(); /* code_page_size */ + + if (orig_sample_type & PERF_SAMPLE_AUX) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j] = in_array[i]; + size = out_array[j++]; + i++; + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping aux sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + if (evsel__is_offcpu_event(evsel)) { + /* TODO: can this be less conservative? */ + pr_debug("Dropping off-CPU sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + new_event->sample.header.size = sizeof(struct perf_event_header) + j * sizeof(u64); + /* Temporarily override evsel attributes to match the stripped new_event format! */ + evsel->sample_size = __evsel__sample_size(sample_type); + evsel->core.attr.sample_type = sample_type; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + perf_sample__init(&new_sample, /*all=*/ true); + ret = __evsel__parse_sample(evsel, new_event, &new_sample, /*needs_swap=*/false); + + if (ret) { + /* Restore original attributes immediately if parsing fails */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + perf_sample__exit(&new_sample); + goto out_put; + } + + new_sample.evsel = evsel; + ret = delegate->sample(delegate, new_event, &new_sample, machine); + perf_sample__exit(&new_sample); + + /* Restore original attributes so trace ingestion never desynchronizes! */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + +out_put: + thread__put(thread); + return ret; +} + +#undef CHECK_BOUNDS +#undef COPY_U64 +#undef REMAP_U64 + +static int skipn(int fd, off_t n) +{ + char buf[4096]; + ssize_t ret; + + while (n > 0) { + ret = read(fd, buf, min_t(off_t, n, (off_t)sizeof(buf))); + if (ret <= 0) + return ret; + n -= ret; + } + + return 0; +} + +static s64 aslr_tool__process_auxtrace(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + pr_warning_once("ASLR: Dropping auxtrace data as it cannot be obfuscated.\n"); + if (perf_data__is_pipe(session->data)) { + /* Copy behavior of the stub by reading all pipe data. */ + int err = skipn(perf_data__fd(session->data), event->auxtrace.size); + + if (err < 0) + return err; + } + return event->auxtrace.size; +} + +static int aslr_tool__process_auxtrace_info(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +static int aslr_tool__process_auxtrace_error(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist) +{ + u32 attr_size; + + if (!evlist) + return; + + attr_size = event->attr.attr.size ?: PERF_ATTR_SIZE_VER0; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_type) + sizeof(u64))) { + event->attr.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_user) + sizeof(u64))) + event->attr.attr.sample_regs_user = 0; + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_intr) + sizeof(u64))) + event->attr.attr.sample_regs_intr = 0; + } + + if (attr_size >= (offsetof(struct perf_event_attr, type) + sizeof(u32))) { + u32 type = event->attr.attr.type; + + if (type == PERF_TYPE_BREAKPOINT && + attr_size >= (offsetof(struct perf_event_attr, bp_addr) + sizeof(u64))) { + event->attr.attr.bp_addr = 0; + } else if (type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu; + + pmu = perf_pmus__find_by_type(type); + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + if (attr_size >= (offsetof(struct perf_event_attr, config1) + sizeof(u64))) + event->attr.attr.config1 = 0; + if (attr_size >= (offsetof(struct perf_event_attr, config2) + sizeof(u64))) + event->attr.attr.config2 = 0; + } + } + } +} + +static int aslr_tool__init(struct aslr_tool *aslr, struct perf_tool *delegate) +{ + delegate_tool__init(&aslr->tool, delegate); + aslr->tool.tool.ordered_events = true; + + if (machines__init(&aslr->machines)) + return -ENOMEM; + + hashmap__init(&aslr->remap_addresses, + remap_addresses__hash, remap_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->top_addresses, + top_addresses__hash, top_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->evsel_orig_attrs, + evsel_hash, evsel_equal, + /*ctx=*/NULL); + + aslr->tool.tool.sample = aslr_tool__process_sample; + /* read - reads a counter, okay to delegate. */ + aslr->tool.tool.mmap = aslr_tool__process_mmap; + aslr->tool.tool.mmap2 = aslr_tool__process_mmap2; + aslr->tool.tool.comm = aslr_tool__process_comm; + aslr->tool.tool.fork = aslr_tool__process_fork; + aslr->tool.tool.exit = aslr_tool__process_exit; + /* namesspaces, cgroup, lost, lost_sample, aux, */ + /* itrace_start, aux_output_hw_id, context_switch, throttle, unthrottle */ + /* - no virtual addresses. */ + aslr->tool.tool.ksymbol = aslr_tool__process_ksymbol; + /* bpf - no virtual address. */ + aslr->tool.tool.text_poke = aslr_tool__process_text_poke; + /* + * event_update, tracing_data, finished_round, build_id, id_index, + * auxtrace_info, auxtrace_error, time_conv, thread_map, cpu_map, + * stat_config, stat, feature, finished_init, bpf_metadata, compressed, + * auxtrace - no virtual addresses. + */ + aslr->tool.tool.auxtrace = aslr_tool__process_auxtrace; + aslr->tool.tool.auxtrace_info = aslr_tool__process_auxtrace_info; + aslr->tool.tool.auxtrace_error = aslr_tool__process_auxtrace_error; + + return 0; +} + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate) +{ + struct aslr_tool *aslr = zalloc(sizeof(*aslr)); + + if (!aslr) + return NULL; + + if (aslr_tool__init(aslr, delegate)) { + free(aslr); + return NULL; + } + return &aslr->tool.tool; +} + +void aslr_tool__delete(struct perf_tool *tool) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct hashmap_entry *cur; + size_t bkt; + struct rb_node *nd; + + if (!tool) + return; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + + hashmap__for_each_entry(&aslr->remap_addresses, cur, bkt) { + struct remap_addresses_key *key = (struct remap_addresses_key *)cur->pkey; + + if (key) + dso__put(key->dso); + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->top_addresses, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->evsel_orig_attrs, cur, bkt) { + zfree(&cur->pvalue); + } + + hashmap__clear(&aslr->remap_addresses); + hashmap__clear(&aslr->top_addresses); + hashmap__clear(&aslr->evsel_orig_attrs); + aslr_tool__destroy_machines_priv(&aslr->machines); + machines__destroy_kernel_maps(&aslr->machines); + + while ((nd = rb_first_cached(&aslr->machines.guests)) != NULL) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + rb_erase_cached(nd, &aslr->machines.guests); + machine__delete(machine); + } + + machines__exit(&aslr->machines); + free(aslr); +} + +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel) +{ + struct delegate_tool *del_tool = container_of(tool, struct delegate_tool, tool); + struct aslr_tool *aslr = container_of(del_tool, struct aslr_tool, tool); + struct aslr_evsel_priv *priv = zalloc(sizeof(*priv)); + int err; + + if (!priv) + return -ENOMEM; + + priv->orig_sample_type = evsel->core.attr.sample_type; + priv->orig_sample_regs_user = evsel->core.attr.sample_regs_user; + priv->orig_sample_regs_intr = evsel->core.attr.sample_regs_intr; + priv->orig_sample_size = evsel->sample_size; + + err = hashmap__add(&aslr->evsel_orig_attrs, evsel, priv); + if (err) { + free(priv); + return err; + } + return 0; +} + +void aslr_tool__strip_evlist(const struct perf_tool *tool __maybe_unused, struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + evsel->core.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + evsel->sample_size = __evsel__sample_size(evsel->core.attr.sample_type); + evsel__calc_id_pos(evsel); + + if (evsel->core.attr.type == PERF_TYPE_BREAKPOINT) { + evsel->core.attr.bp_addr = 0; + } else if (evsel->core.attr.type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->core.attr.type); + + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + evsel->core.attr.config1 = 0; + evsel->core.attr.config2 = 0; + } + } + } +} + +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist) +{ + const struct delegate_tool *del_tool = container_of(tool, const struct delegate_tool, tool); + const struct aslr_tool *aslr = container_of(del_tool, const struct aslr_tool, tool); + struct evsel *evsel; + struct aslr_evsel_priv *priv; + + evlist__for_each_entry(evlist, evsel) { + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + evsel->core.attr.sample_type = priv->orig_sample_type; + evsel->core.attr.sample_regs_user = priv->orig_sample_regs_user; + evsel->core.attr.sample_regs_intr = priv->orig_sample_regs_intr; + evsel->sample_size = priv->orig_sample_size; + evsel__calc_id_pos(evsel); + } + } +} diff --git a/tools/perf/util/aslr.h b/tools/perf/util/aslr.h new file mode 100644 index 000000000..522e31c8e --- /dev/null +++ b/tools/perf/util/aslr.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ASLR_H +#define __PERF_ASLR_H + +#include + +#define ASLR_SUPPORTED_SAMPLE_TYPE ( \ + PERF_SAMPLE_IDENTIFIER | \ + PERF_SAMPLE_IP | \ + PERF_SAMPLE_TID | \ + PERF_SAMPLE_TIME | \ + PERF_SAMPLE_ADDR | \ + PERF_SAMPLE_ID | \ + PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | \ + PERF_SAMPLE_PERIOD | \ + PERF_SAMPLE_READ | \ + PERF_SAMPLE_CALLCHAIN | \ + PERF_SAMPLE_RAW | \ + PERF_SAMPLE_BRANCH_STACK | \ + PERF_SAMPLE_STACK_USER | \ + PERF_SAMPLE_WEIGHT_TYPE | \ + PERF_SAMPLE_DATA_SRC | \ + PERF_SAMPLE_TRANSACTION | \ + PERF_SAMPLE_PHYS_ADDR | \ + PERF_SAMPLE_CGROUP | \ + PERF_SAMPLE_DATA_PAGE_SIZE | \ + PERF_SAMPLE_CODE_PAGE_SIZE | \ + PERF_SAMPLE_AUX) + +struct perf_tool; +struct evsel; +struct evlist; +union perf_event; + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate); +void aslr_tool__delete(struct perf_tool *tool); + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist); +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel); +void aslr_tool__strip_evlist(const struct perf_tool *tool, struct evlist *evlist); +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist); + +#endif /* __PERF_ASLR_H */ diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c new file mode 100644 index 000000000..aa749e1c3 --- /dev/null +++ b/tools/perf/util/auxtrace.c @@ -0,0 +1,2999 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * auxtrace.c: AUX area trace support + * Copyright (c) 2013-2015, Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "config.h" +#include "evlist.h" +#include "dso.h" +#include "map.h" +#include "pmu.h" +#include "evsel.h" +#include "evsel_config.h" +#include "symbol.h" +#include "util/perf_api_probe.h" +#include "util/synthetic-events.h" +#include "thread_map.h" +#include "asm/bug.h" +#include "auxtrace.h" + +#include + +#include "event.h" +#include "record.h" +#include "session.h" +#include "debug.h" +#include + +#include "cs-etm.h" +#include "intel-pt.h" +#include "intel-bts.h" +#include "arm-spe.h" +#include "hisi-ptt.h" +#include "s390-cpumsf.h" +#include "util/mmap.h" +#include "powerpc-vpadtl.h" + +#include +#include "symbol/kallsyms.h" +#include +#include "util/sample.h" + +#define AUXTRACE_SYNTH_EVENT_ID_OFFSET 1000000000ULL + +/* + * Event IDs are allocated sequentially, so a big offset from any + * existing ID will reach a unused range. + */ +u64 auxtrace_synth_id_range_start(struct evsel *evsel) +{ + u64 id = evsel->core.id[0] + AUXTRACE_SYNTH_EVENT_ID_OFFSET; + + if (!id) + id = 1; + + return id; +} + +/* + * Make a group from 'leader' to 'last', requiring that the events were not + * already grouped to a different leader. + */ +static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct evsel *last) +{ + struct evsel *evsel; + bool grp; + + if (!evsel__is_group_leader(leader)) + return -EINVAL; + + grp = false; + evlist__for_each_entry(evlist, evsel) { + if (grp) { + if (!(evsel__leader(evsel) == leader || + (evsel__leader(evsel) == evsel && + evsel->core.nr_members <= 1))) + return -EINVAL; + } else if (evsel == leader) { + grp = true; + } + if (evsel == last) + break; + } + + grp = false; + evlist__for_each_entry(evlist, evsel) { + if (grp) { + if (!evsel__has_leader(evsel, leader)) { + evsel__set_leader(evsel, leader); + if (leader->core.nr_members < 1) + leader->core.nr_members = 1; + leader->core.nr_members += 1; + } + } else if (evsel == leader) { + grp = true; + } + if (evsel == last) + break; + } + + return 0; +} + +static bool auxtrace__dont_decode(struct perf_session *session) +{ + return !session->itrace_synth_opts || + session->itrace_synth_opts->dont_decode; +} + +int auxtrace_mmap__mmap(struct auxtrace_mmap *mm, + struct auxtrace_mmap_params *mp, + void *userpg, int fd) +{ + struct perf_event_mmap_page *pc = userpg; + + WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n"); + + mm->userpg = userpg; + mm->mask = mp->mask; + mm->len = mp->len; + mm->prev = 0; + mm->idx = mp->idx; + mm->tid = mp->tid; + mm->cpu = mp->cpu.cpu; + + if (!mp->len || !mp->mmap_needed) { + mm->base = NULL; + return 0; + } + + pc->aux_offset = mp->offset; + pc->aux_size = mp->len; + + mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset); + if (mm->base == MAP_FAILED) { + pr_debug2("failed to mmap AUX area\n"); + mm->base = NULL; + return -1; + } + + return 0; +} + +void auxtrace_mmap__munmap(struct auxtrace_mmap *mm) +{ + if (mm->base) { + munmap(mm->base, mm->len); + mm->base = NULL; + } +} + +void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp, + off_t auxtrace_offset, + unsigned int auxtrace_pages, + bool auxtrace_overwrite) +{ + if (auxtrace_pages) { + mp->offset = auxtrace_offset; + mp->len = auxtrace_pages * (size_t)page_size; + mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0; + mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE); + pr_debug2("AUX area mmap length %zu\n", mp->len); + } else { + mp->len = 0; + } +} + +void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, + struct evlist *evlist, + struct evsel *evsel, int idx) +{ + bool per_cpu = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); + + mp->mmap_needed = evsel->needs_auxtrace_mmap; + + if (!mp->mmap_needed) + return; + + mp->idx = idx; + + if (per_cpu) { + mp->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); + } else { + mp->cpu.cpu = -1; + mp->tid = perf_thread_map__pid(evlist__core(evlist)->threads, idx); + } +} + +#define AUXTRACE_INIT_NR_QUEUES 32 + +static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues) +{ + struct auxtrace_queue *queue_array; + unsigned int max_nr_queues, i; + + max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue); + if (nr_queues > max_nr_queues) + return NULL; + + queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue)); + if (!queue_array) + return NULL; + + for (i = 0; i < nr_queues; i++) { + INIT_LIST_HEAD(&queue_array[i].head); + queue_array[i].priv = NULL; + } + + return queue_array; +} + +int auxtrace_queues__init_nr(struct auxtrace_queues *queues, int nr_queues) +{ + queues->nr_queues = nr_queues; + queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues); + if (!queues->queue_array) + return -ENOMEM; + return 0; +} + +int auxtrace_queues__init(struct auxtrace_queues *queues) +{ + return auxtrace_queues__init_nr(queues, AUXTRACE_INIT_NR_QUEUES); +} + +static int auxtrace_queues__grow(struct auxtrace_queues *queues, + unsigned int new_nr_queues) +{ + unsigned int nr_queues = queues->nr_queues; + struct auxtrace_queue *queue_array; + struct auxtrace_queue *old_array = queues->queue_array; + unsigned int i; + + if (!new_nr_queues) + return -EINVAL; + + if (!nr_queues) + nr_queues = AUXTRACE_INIT_NR_QUEUES; + + while (nr_queues && nr_queues < new_nr_queues) + nr_queues <<= 1; + + if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues) + return -EINVAL; + + queue_array = auxtrace_alloc_queue_array(nr_queues); + if (!queue_array) + return -ENOMEM; + + for (i = 0; i < queues->nr_queues; i++) { + list_splice_tail(&old_array[i].head, + &queue_array[i].head); + queue_array[i].tid = old_array[i].tid; + queue_array[i].cpu = old_array[i].cpu; + queue_array[i].set = old_array[i].set; + queue_array[i].priv = old_array[i].priv; + } + + queues->nr_queues = nr_queues; + queues->queue_array = queue_array; + free(old_array); + + return 0; +} + +static void *auxtrace_copy_data(u64 size, struct perf_session *session) +{ + int fd = perf_data__fd(session->data); + void *p; + ssize_t ret; + + if (size > SSIZE_MAX) + return NULL; + + p = malloc(size); + if (!p) + return NULL; + + ret = readn(fd, p, size); + if (ret != (ssize_t)size) { + free(p); + return NULL; + } + + return p; +} + +static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues, + unsigned int idx, + struct auxtrace_buffer *buffer) +{ + struct auxtrace_queue *queue; + int err; + + if (idx >= queues->nr_queues) { + err = auxtrace_queues__grow(queues, idx + 1); + if (err) + return err; + } + + queue = &queues->queue_array[idx]; + + if (!queue->set) { + queue->set = true; + queue->tid = buffer->tid; + queue->cpu = buffer->cpu.cpu; + } + + buffer->buffer_nr = queues->next_buffer_nr++; + + list_add_tail(&buffer->list, &queue->head); + + queues->new_data = true; + queues->populated = true; + + return 0; +} + +/* Limit buffers to 32MiB on 32-bit */ +#define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024) + +static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues, + unsigned int idx, + struct auxtrace_buffer *buffer) +{ + u64 sz = buffer->size; + bool consecutive = false; + struct auxtrace_buffer *b; + int err; + + while (sz > BUFFER_LIMIT_FOR_32_BIT) { + b = memdup(buffer, sizeof(struct auxtrace_buffer)); + if (!b) + return -ENOMEM; + b->size = BUFFER_LIMIT_FOR_32_BIT; + b->consecutive = consecutive; + err = auxtrace_queues__queue_buffer(queues, idx, b); + if (err) { + auxtrace_buffer__free(b); + return err; + } + buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT; + sz -= BUFFER_LIMIT_FOR_32_BIT; + consecutive = true; + } + + buffer->size = sz; + buffer->consecutive = consecutive; + + return 0; +} + +static bool filter_cpu(struct perf_session *session, struct perf_cpu cpu) +{ + unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap; + + return cpu_bitmap && cpu.cpu >= 0 && cpu.cpu < MAX_NR_CPUS && + !test_bit(cpu.cpu, cpu_bitmap); +} + +static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues, + struct perf_session *session, + unsigned int idx, + struct auxtrace_buffer *buffer, + struct auxtrace_buffer **buffer_ptr) +{ + int err = -ENOMEM; + + if (filter_cpu(session, buffer->cpu)) + return 0; + + buffer = memdup(buffer, sizeof(*buffer)); + if (!buffer) + return -ENOMEM; + + if (session->one_mmap) { + buffer->data = buffer->data_offset - session->one_mmap_offset + + session->one_mmap_addr; + } else if (perf_data__is_pipe(session->data)) { + buffer->data = auxtrace_copy_data(buffer->size, session); + if (!buffer->data) + goto out_free; + buffer->data_needs_freeing = true; + } else if (BITS_PER_LONG == 32 && + buffer->size > BUFFER_LIMIT_FOR_32_BIT) { + err = auxtrace_queues__split_buffer(queues, idx, buffer); + if (err) + goto out_free; + } + + err = auxtrace_queues__queue_buffer(queues, idx, buffer); + if (err) + goto out_free; + + /* FIXME: Doesn't work for split buffer */ + if (buffer_ptr) + *buffer_ptr = buffer; + + return 0; + +out_free: + auxtrace_buffer__free(buffer); + return err; +} + +int auxtrace_queues__add_event(struct auxtrace_queues *queues, + struct perf_session *session, + union perf_event *event, off_t data_offset, + struct auxtrace_buffer **buffer_ptr) +{ + struct auxtrace_buffer buffer = { + .pid = -1, + .tid = event->auxtrace.tid, + .cpu = { event->auxtrace.cpu }, + .data_offset = data_offset, + .offset = event->auxtrace.offset, + .reference = event->auxtrace.reference, + .size = event->auxtrace.size, + }; + unsigned int idx = event->auxtrace.idx; + + return auxtrace_queues__add_buffer(queues, session, idx, &buffer, + buffer_ptr); +} + +static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues, + struct perf_session *session, + off_t file_offset, size_t sz) +{ + union perf_event *event; + int err; + char buf[PERF_SAMPLE_MAX_SIZE]; + + err = perf_session__peek_event(session, file_offset, buf, + PERF_SAMPLE_MAX_SIZE, &event, NULL); + if (err) + return err; + + if (event->header.type == PERF_RECORD_AUXTRACE) { + if (event->header.size < sizeof(struct perf_record_auxtrace) || + event->header.size != sz) { + err = -EINVAL; + goto out; + } + file_offset += event->header.size; + err = auxtrace_queues__add_event(queues, session, event, + file_offset, NULL); + } +out: + return err; +} + +void auxtrace_queues__free(struct auxtrace_queues *queues) +{ + unsigned int i; + + for (i = 0; i < queues->nr_queues; i++) { + while (!list_empty(&queues->queue_array[i].head)) { + struct auxtrace_buffer *buffer; + + buffer = list_entry(queues->queue_array[i].head.next, + struct auxtrace_buffer, list); + list_del_init(&buffer->list); + auxtrace_buffer__free(buffer); + } + } + + zfree(&queues->queue_array); + queues->nr_queues = 0; +} + +static void auxtrace_heapify(struct auxtrace_heap_item *heap_array, + unsigned int pos, unsigned int queue_nr, + u64 ordinal) +{ + unsigned int parent; + + while (pos) { + parent = (pos - 1) >> 1; + if (heap_array[parent].ordinal <= ordinal) + break; + heap_array[pos] = heap_array[parent]; + pos = parent; + } + heap_array[pos].queue_nr = queue_nr; + heap_array[pos].ordinal = ordinal; +} + +int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr, + u64 ordinal) +{ + struct auxtrace_heap_item *heap_array; + + if (queue_nr >= heap->heap_sz) { + unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES; + + while (heap_sz <= queue_nr) + heap_sz <<= 1; + heap_array = realloc(heap->heap_array, + heap_sz * sizeof(struct auxtrace_heap_item)); + if (!heap_array) + return -ENOMEM; + heap->heap_array = heap_array; + heap->heap_sz = heap_sz; + } + + auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal); + + return 0; +} + +void auxtrace_heap__free(struct auxtrace_heap *heap) +{ + zfree(&heap->heap_array); + heap->heap_cnt = 0; + heap->heap_sz = 0; +} + +void auxtrace_heap__pop(struct auxtrace_heap *heap) +{ + unsigned int pos, last, heap_cnt = heap->heap_cnt; + struct auxtrace_heap_item *heap_array; + + if (!heap_cnt) + return; + + heap->heap_cnt -= 1; + + heap_array = heap->heap_array; + + pos = 0; + while (1) { + unsigned int left, right; + + left = (pos << 1) + 1; + if (left >= heap_cnt) + break; + right = left + 1; + if (right >= heap_cnt) { + heap_array[pos] = heap_array[left]; + return; + } + if (heap_array[left].ordinal < heap_array[right].ordinal) { + heap_array[pos] = heap_array[left]; + pos = left; + } else { + heap_array[pos] = heap_array[right]; + pos = right; + } + } + + last = heap_cnt - 1; + auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr, + heap_array[last].ordinal); +} + +size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr, + struct evlist *evlist) +{ + if (itr) + return itr->info_priv_size(itr, evlist); + return 0; +} + +static int auxtrace_not_supported(void) +{ + pr_err("AUX area tracing is not supported on this architecture\n"); + return -EINVAL; +} + +int auxtrace_record__info_fill(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size) +{ + if (itr) + return itr->info_fill(itr, session, auxtrace_info, priv_size); + return auxtrace_not_supported(); +} + +void auxtrace_record__free(struct auxtrace_record *itr) +{ + if (itr) + itr->free(itr); +} + +int auxtrace_record__snapshot_start(struct auxtrace_record *itr) +{ + if (itr && itr->snapshot_start) + return itr->snapshot_start(itr); + return 0; +} + +int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit) +{ + if (!on_exit && itr && itr->snapshot_finish) + return itr->snapshot_finish(itr); + return 0; +} + +int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx, + struct auxtrace_mmap *mm, + unsigned char *data, u64 *head, u64 *old) +{ + if (itr && itr->find_snapshot) + return itr->find_snapshot(itr, idx, mm, data, head, old); + return 0; +} + +int auxtrace_record__options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts) +{ + if (itr) { + itr->evlist = evlist; + return itr->recording_options(itr, evlist, opts); + } + return 0; +} + +u64 auxtrace_record__reference(struct auxtrace_record *itr) +{ + if (itr) + return itr->reference(itr); + return 0; +} + +int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, + struct record_opts *opts, const char *str) +{ + if (!str) + return 0; + + /* PMU-agnostic options */ + switch (*str) { + case 'e': + opts->auxtrace_snapshot_on_exit = true; + str++; + break; + default: + break; + } + + if (itr && itr->parse_snapshot_options) + return itr->parse_snapshot_options(itr, opts, str); + + pr_err("No AUX area tracing to snapshot\n"); + return -EINVAL; +} + +static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx) +{ + bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist__core(evlist)->user_requested_cpus); + + if (per_cpu_mmaps) { + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, idx); + int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu); + + if (cpu_map_idx == -1) + return -EINVAL; + return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + } + + return perf_evsel__enable_thread(&evsel->core, idx); +} + +int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx) +{ + struct evsel *evsel; + + if (!itr->evlist) + return -EINVAL; + + evlist__for_each_entry(itr->evlist, evsel) { + if (evsel__is_aux_event(evsel)) { + if (evsel->disabled) + return 0; + return evlist__enable_event_idx(itr->evlist, evsel, idx); + } + } + return -EINVAL; +} + +/* + * Event record size is 16-bit which results in a maximum size of about 64KiB. + * Allow about 4KiB for the rest of the sample record, to give a maximum + * AUX area sample size of 60KiB. + */ +#define MAX_AUX_SAMPLE_SIZE (60 * 1024) + +/* Arbitrary default size if no other default provided */ +#define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024) + +static int auxtrace_validate_aux_sample_size(struct evlist *evlist, + struct record_opts *opts) +{ + struct evsel *evsel; + bool has_aux_leader = false; + u32 sz; + + evlist__for_each_entry(evlist, evsel) { + sz = evsel->core.attr.aux_sample_size; + if (evsel__is_group_leader(evsel)) { + has_aux_leader = evsel__is_aux_event(evsel); + if (sz) { + if (has_aux_leader) + pr_err("Cannot add AUX area sampling to an AUX area event\n"); + else + pr_err("Cannot add AUX area sampling to a group leader\n"); + return -EINVAL; + } + } + if (sz > MAX_AUX_SAMPLE_SIZE) { + pr_err("AUX area sample size %u too big, max. %d\n", + sz, MAX_AUX_SAMPLE_SIZE); + return -EINVAL; + } + if (sz) { + if (!has_aux_leader) { + pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n"); + return -EINVAL; + } + evsel__set_sample_bit(evsel, AUX); + opts->auxtrace_sample_mode = true; + } else { + evsel__reset_sample_bit(evsel, AUX); + } + } + + if (!opts->auxtrace_sample_mode) { + pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n"); + return -EINVAL; + } + + if (!perf_can_aux_sample()) { + pr_err("AUX area sampling is not supported by kernel\n"); + return -EINVAL; + } + + return 0; +} + +int auxtrace_parse_sample_options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts, const char *str) +{ + struct evsel_config_term *term; + struct evsel *aux_evsel; + bool has_aux_sample_size = false; + bool has_aux_leader = false; + struct evsel *evsel; + char *endptr; + unsigned long sz; + + if (!str) + goto no_opt; + + if (!itr) { + pr_err("No AUX area event to sample\n"); + return -EINVAL; + } + + sz = strtoul(str, &endptr, 0); + if (*endptr || sz > UINT_MAX) { + pr_err("Bad AUX area sampling option: '%s'\n", str); + return -EINVAL; + } + + if (!sz) + sz = itr->default_aux_sample_size; + + if (!sz) + sz = DEFAULT_AUX_SAMPLE_SIZE; + + /* Set aux_sample_size based on --aux-sample option */ + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_group_leader(evsel)) { + has_aux_leader = evsel__is_aux_event(evsel); + } else if (has_aux_leader) { + evsel->core.attr.aux_sample_size = sz; + } + } +no_opt: + aux_evsel = NULL; + /* Override with aux_sample_size from config term */ + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_aux_event(evsel)) + aux_evsel = evsel; + term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE); + if (term) { + has_aux_sample_size = true; + evsel->core.attr.aux_sample_size = term->val.aux_sample_size; + /* If possible, group with the AUX event */ + if (aux_evsel && evsel->core.attr.aux_sample_size) + evlist__regroup(evlist, aux_evsel, evsel); + } + } + + if (!str && !has_aux_sample_size) + return 0; + + if (!itr) { + pr_err("No AUX area event to sample\n"); + return -EINVAL; + } + + return auxtrace_validate_aux_sample_size(evlist, opts); +} + +static struct aux_action_opt { + const char *str; + u32 aux_action; + bool aux_event_opt; +} aux_action_opts[] = { + {"start-paused", BIT(0), true}, + {"pause", BIT(1), false}, + {"resume", BIT(2), false}, + {.str = NULL}, +}; + +static const struct aux_action_opt *auxtrace_parse_aux_action_str(const char *str) +{ + const struct aux_action_opt *opt; + + if (!str) + return NULL; + + for (opt = aux_action_opts; opt->str; opt++) + if (!strcmp(str, opt->str)) + return opt; + + return NULL; +} + +int auxtrace_parse_aux_action(struct evlist *evlist) +{ + struct evsel_config_term *term; + struct evsel *aux_evsel = NULL; + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + bool is_aux_event = evsel__is_aux_event(evsel); + const struct aux_action_opt *opt; + + if (is_aux_event) + aux_evsel = evsel; + term = evsel__get_config_term(evsel, AUX_ACTION); + if (!term) { + if (evsel__get_config_term(evsel, AUX_OUTPUT)) + goto regroup; + continue; + } + opt = auxtrace_parse_aux_action_str(term->val.str); + if (!opt) { + pr_err("Bad aux-action '%s'\n", term->val.str); + return -EINVAL; + } + if (opt->aux_event_opt && !is_aux_event) { + pr_err("aux-action '%s' can only be used with AUX area event\n", + term->val.str); + return -EINVAL; + } + if (!opt->aux_event_opt && is_aux_event) { + pr_err("aux-action '%s' cannot be used for AUX area event itself\n", + term->val.str); + return -EINVAL; + } + evsel->core.attr.aux_action = opt->aux_action; +regroup: + /* If possible, group with the AUX event */ + if (aux_evsel) + evlist__regroup(evlist, aux_evsel, evsel); + if (!evsel__is_aux_event(evsel__leader(evsel))) { + pr_err("Events with aux-action must have AUX area event group leader\n"); + return -EINVAL; + } + } + + return 0; +} + +/** + * auxtrace_record__init - Initialize an AUX area tracing record. + * @evlist: The list of events to check for AUX area tracing event. + * @err: Pointer to an integer to store return code. + * + * This function looks through the @evlist to determine which AUX area + * tracing hardware is being used and initializes the auxtrace_record + * structure. + * + * Return: + * a) A pointer to the struct auxtrace_record with @err = 0 on success. + * b) NULL with @err = 0 if no AUX area tracing event is found/supported + * (not considered an error). + * c) NULL with non-zero @err on actual auxtrace_record__init failure. + */ +struct auxtrace_record *__weak +auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err) +{ + *err = 0; + return NULL; +} + +static int auxtrace_index__alloc(struct list_head *head) +{ + struct auxtrace_index *auxtrace_index; + + auxtrace_index = malloc(sizeof(struct auxtrace_index)); + if (!auxtrace_index) + return -ENOMEM; + + auxtrace_index->nr = 0; + INIT_LIST_HEAD(&auxtrace_index->list); + + list_add_tail(&auxtrace_index->list, head); + + return 0; +} + +void auxtrace_index__free(struct list_head *head) +{ + struct auxtrace_index *auxtrace_index, *n; + + list_for_each_entry_safe(auxtrace_index, n, head, list) { + list_del_init(&auxtrace_index->list); + free(auxtrace_index); + } +} + +static struct auxtrace_index *auxtrace_index__last(struct list_head *head) +{ + struct auxtrace_index *auxtrace_index; + int err; + + if (list_empty(head)) { + err = auxtrace_index__alloc(head); + if (err) + return NULL; + } + + auxtrace_index = list_entry(head->prev, struct auxtrace_index, list); + + if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) { + err = auxtrace_index__alloc(head); + if (err) + return NULL; + auxtrace_index = list_entry(head->prev, struct auxtrace_index, + list); + } + + return auxtrace_index; +} + +int auxtrace_index__auxtrace_event(struct list_head *head, + union perf_event *event, off_t file_offset) +{ + struct auxtrace_index *auxtrace_index; + size_t nr; + + auxtrace_index = auxtrace_index__last(head); + if (!auxtrace_index) + return -ENOMEM; + + nr = auxtrace_index->nr; + auxtrace_index->entries[nr].file_offset = file_offset; + auxtrace_index->entries[nr].sz = event->header.size; + auxtrace_index->nr += 1; + + return 0; +} + +static int auxtrace_index__do_write(int fd, + struct auxtrace_index *auxtrace_index) +{ + struct auxtrace_index_entry ent; + size_t i; + + for (i = 0; i < auxtrace_index->nr; i++) { + ent.file_offset = auxtrace_index->entries[i].file_offset; + ent.sz = auxtrace_index->entries[i].sz; + if (writen(fd, &ent, sizeof(ent)) != sizeof(ent)) + return -errno; + } + return 0; +} + +int auxtrace_index__write(int fd, struct list_head *head) +{ + struct auxtrace_index *auxtrace_index; + u64 total = 0; + int err; + + list_for_each_entry(auxtrace_index, head, list) + total += auxtrace_index->nr; + + if (writen(fd, &total, sizeof(total)) != sizeof(total)) + return -errno; + + list_for_each_entry(auxtrace_index, head, list) { + err = auxtrace_index__do_write(fd, auxtrace_index); + if (err) + return err; + } + + return 0; +} + +static int auxtrace_index__process_entry(int fd, struct list_head *head, + bool needs_swap) +{ + struct auxtrace_index *auxtrace_index; + struct auxtrace_index_entry ent; + size_t nr; + + if (readn(fd, &ent, sizeof(ent)) != sizeof(ent)) + return -1; + + auxtrace_index = auxtrace_index__last(head); + if (!auxtrace_index) + return -1; + + nr = auxtrace_index->nr; + if (needs_swap) { + auxtrace_index->entries[nr].file_offset = + bswap_64(ent.file_offset); + auxtrace_index->entries[nr].sz = bswap_64(ent.sz); + } else { + auxtrace_index->entries[nr].file_offset = ent.file_offset; + auxtrace_index->entries[nr].sz = ent.sz; + } + + auxtrace_index->nr = nr + 1; + + return 0; +} + +int auxtrace_index__process(int fd, u64 size, struct perf_session *session, + bool needs_swap) +{ + struct list_head *head = &session->auxtrace_index; + u64 nr; + + if (readn(fd, &nr, sizeof(u64)) != sizeof(u64)) + return -1; + + if (needs_swap) + nr = bswap_64(nr); + + if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size) + return -1; + + while (nr--) { + int err; + + err = auxtrace_index__process_entry(fd, head, needs_swap); + if (err) + return -1; + } + + return 0; +} + +static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues, + struct perf_session *session, + struct auxtrace_index_entry *ent) +{ + return auxtrace_queues__add_indexed_event(queues, session, + ent->file_offset, ent->sz); +} + +int auxtrace_queues__process_index(struct auxtrace_queues *queues, + struct perf_session *session) +{ + struct auxtrace_index *auxtrace_index; + struct auxtrace_index_entry *ent; + size_t i; + int err; + + if (auxtrace__dont_decode(session)) + return 0; + + list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { + for (i = 0; i < auxtrace_index->nr; i++) { + ent = &auxtrace_index->entries[i]; + err = auxtrace_queues__process_index_entry(queues, + session, + ent); + if (err) + return err; + } + } + return 0; +} + +struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue, + struct auxtrace_buffer *buffer) +{ + if (buffer) { + if (list_is_last(&buffer->list, &queue->head)) + return NULL; + return list_entry(buffer->list.next, struct auxtrace_buffer, + list); + } else { + if (list_empty(&queue->head)) + return NULL; + return list_entry(queue->head.next, struct auxtrace_buffer, + list); + } +} + +struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues, + struct perf_sample *sample, + struct perf_session *session) +{ + struct perf_sample_id *sid; + unsigned int idx; + u64 id; + + id = sample->id; + if (!id) + return NULL; + + sid = evlist__id2sid(session->evlist, id); + if (!sid) + return NULL; + + idx = sid->idx; + + if (idx >= queues->nr_queues) + return NULL; + + return &queues->queue_array[idx]; +} + +int auxtrace_queues__add_sample(struct auxtrace_queues *queues, + struct perf_session *session, + struct perf_sample *sample, u64 data_offset, + u64 reference) +{ + struct auxtrace_buffer buffer = { + .pid = -1, + .data_offset = data_offset, + .reference = reference, + .size = sample->aux_sample.size, + }; + struct perf_sample_id *sid; + u64 id = sample->id; + unsigned int idx; + + if (!id) + return -EINVAL; + + sid = evlist__id2sid(session->evlist, id); + if (!sid) + return -ENOENT; + + idx = sid->idx; + buffer.tid = sid->tid; + buffer.cpu = sid->cpu; + + return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL); +} + +struct queue_data { + bool samples; + bool events; +}; + +static int auxtrace_queue_data_cb(struct perf_session *session, + union perf_event *event, u64 offset, + void *data) +{ + struct queue_data *qd = data; + struct perf_sample sample; + int err; + + if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) { + if (event->header.size < sizeof(struct perf_record_auxtrace)) + return -EINVAL; + offset += event->header.size; + return session->auxtrace->queue_data(session, NULL, event, + offset); + } + + if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE) + return 0; + + perf_sample__init(&sample, /*all=*/false); + err = evlist__parse_sample(session->evlist, event, &sample); + if (err) + goto out; + + if (sample.aux_sample.size) { + offset += sample.aux_sample.data - (void *)event; + + err = session->auxtrace->queue_data(session, &sample, NULL, offset); + } +out: + perf_sample__exit(&sample); + return err; +} + +int auxtrace_queue_data(struct perf_session *session, bool samples, bool events) +{ + struct queue_data qd = { + .samples = samples, + .events = events, + }; + + if (auxtrace__dont_decode(session)) + return 0; + + if (perf_data__is_pipe(session->data)) + return 0; + + if (!session->auxtrace || !session->auxtrace->queue_data) + return -EINVAL; + + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + auxtrace_queue_data_cb, &qd); +} + +void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw) +{ + int prot = rw ? PROT_READ | PROT_WRITE : PROT_READ; + size_t adj = buffer->data_offset & (page_size - 1); + size_t size = buffer->size + adj; + off_t file_offset = buffer->data_offset - adj; + void *addr; + + if (buffer->data) + return buffer->data; + + addr = mmap(NULL, size, prot, MAP_SHARED, fd, file_offset); + if (addr == MAP_FAILED) + return NULL; + + buffer->mmap_addr = addr; + buffer->mmap_size = size; + + buffer->data = addr + adj; + + return buffer->data; +} + +void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer) +{ + if (!buffer->data || !buffer->mmap_addr) + return; + munmap(buffer->mmap_addr, buffer->mmap_size); + buffer->mmap_addr = NULL; + buffer->mmap_size = 0; + buffer->data = NULL; + buffer->use_data = NULL; +} + +void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer) +{ + auxtrace_buffer__put_data(buffer); + if (buffer->data_needs_freeing) { + buffer->data_needs_freeing = false; + zfree(&buffer->data); + buffer->use_data = NULL; + buffer->size = 0; + } +} + +void auxtrace_buffer__free(struct auxtrace_buffer *buffer) +{ + auxtrace_buffer__drop_data(buffer); + free(buffer); +} + +void auxtrace_synth_guest_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp, + pid_t machine_pid, int vcpu) +{ + size_t size; + + memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error)); + + auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR; + auxtrace_error->type = type; + auxtrace_error->code = code; + auxtrace_error->cpu = cpu; + auxtrace_error->pid = pid; + auxtrace_error->tid = tid; + auxtrace_error->fmt = 1; + auxtrace_error->ip = ip; + auxtrace_error->time = timestamp; + strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG); + if (machine_pid) { + auxtrace_error->fmt = 2; + auxtrace_error->machine_pid = machine_pid; + auxtrace_error->vcpu = vcpu; + size = sizeof(*auxtrace_error); + } else { + size = (void *)auxtrace_error->msg - (void *)auxtrace_error + + strlen(auxtrace_error->msg) + 1; + } + auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64)); +} + +void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp) +{ + auxtrace_synth_guest_error(auxtrace_error, type, code, cpu, pid, tid, + ip, msg, timestamp, 0, -1); +} + +int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr, + const struct perf_tool *tool, + struct perf_session *session, + perf_event__handler_t process) +{ + union perf_event *ev; + size_t priv_size; + int err; + + pr_debug2("Synthesizing auxtrace information\n"); + priv_size = auxtrace_record__info_priv_size(itr, session->evlist); + ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size); + if (!ev) + return -ENOMEM; + + ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO; + ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) + + priv_size; + err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info, + priv_size); + if (err) + goto out_free; + + err = process(tool, ev, NULL, NULL); +out_free: + free(ev); + return err; +} + +static void unleader_evsel(struct evlist *evlist, struct evsel *leader) +{ + struct evsel *new_leader = NULL; + struct evsel *evsel; + + /* Find new leader for the group */ + evlist__for_each_entry(evlist, evsel) { + if (!evsel__has_leader(evsel, leader) || evsel == leader) + continue; + if (!new_leader) + new_leader = evsel; + evsel__set_leader(evsel, new_leader); + } + + /* Update group information */ + if (new_leader) { + zfree(&new_leader->group_name); + new_leader->group_name = leader->group_name; + leader->group_name = NULL; + + new_leader->core.nr_members = leader->core.nr_members - 1; + leader->core.nr_members = 1; + } +} + +static void unleader_auxtrace(struct perf_session *session) +{ + struct evsel *evsel; + + evlist__for_each_entry(session->evlist, evsel) { + if (auxtrace__evsel_is_auxtrace(session, evsel) && + evsel__is_group_leader(evsel)) { + unleader_evsel(session->evlist, evsel); + } + } +} + +int perf_event__process_auxtrace_info(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + enum auxtrace_type type = event->auxtrace_info.type; + int err; + + if (dump_trace) + fprintf(stdout, " type: %u\n", type); + + switch (type) { + case PERF_AUXTRACE_INTEL_PT: + err = intel_pt_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_INTEL_BTS: + err = intel_bts_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_ARM_SPE: + err = arm_spe_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_CS_ETM: + err = cs_etm__process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_S390_CPUMSF: + err = s390_cpumsf_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_HISI_PTT: + err = hisi_ptt_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_VPA_DTL: + err = powerpc_vpadtl_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_UNKNOWN: + default: + return -EINVAL; + } + + if (err) + return err; + + unleader_auxtrace(session); + + return 0; +} + +s64 perf_event__process_auxtrace(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + s64 err; + + if (dump_trace) + fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n", + event->auxtrace.size, event->auxtrace.offset, + event->auxtrace.reference, event->auxtrace.idx, + event->auxtrace.tid, event->auxtrace.cpu); + + if (auxtrace__dont_decode(session)) + return event->auxtrace.size; + + if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE) + return -EINVAL; + + err = session->auxtrace->process_auxtrace_event(session, event, session->tool); + if (err < 0) + return err; + + return event->auxtrace.size; +} + +#define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS +#define PERF_ITRACE_DEFAULT_PERIOD 100000 +#define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16 +#define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024 +#define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64 +#define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024 + +void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, + bool no_sample) +{ + synth_opts->branches = true; + synth_opts->transactions = true; + synth_opts->ptwrites = true; + synth_opts->pwr_events = true; + synth_opts->other_events = true; + synth_opts->intr_events = true; + synth_opts->errors = true; + synth_opts->flc = true; + synth_opts->llc = true; + synth_opts->tlb = true; + synth_opts->mem = true; + synth_opts->remote_access = true; + + if (no_sample) { + synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS; + synth_opts->period = 1; + synth_opts->calls = true; + } else { + synth_opts->instructions = true; + synth_opts->cycles = true; + synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE; + synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD; + } + synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ; + synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ; + synth_opts->initial_skip = 0; +} + +static int get_flag(const char **ptr, unsigned int *flags) +{ + while (1) { + char c = **ptr; + + if (c >= 'a' && c <= 'z') { + *flags |= 1 << (c - 'a'); + ++*ptr; + return 0; + } else if (c == ' ') { + ++*ptr; + continue; + } else { + return -1; + } + } +} + +static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags) +{ + while (1) { + switch (**ptr) { + case '+': + ++*ptr; + if (get_flag(ptr, plus_flags)) + return -1; + break; + case '-': + ++*ptr; + if (get_flag(ptr, minus_flags)) + return -1; + break; + case ' ': + ++*ptr; + break; + default: + return 0; + } + } +} + +#define ITRACE_DFLT_LOG_ON_ERROR_SZ 16384 + +static unsigned int itrace_log_on_error_size(void) +{ + unsigned int sz = 0; + + perf_config_scan("itrace.debug-log-buffer-size", "%u", &sz); + return sz ?: ITRACE_DFLT_LOG_ON_ERROR_SZ; +} + +/* + * Please check tools/perf/Documentation/perf-script.txt for information + * about the options parsed here, which is introduced after this cset, + * when support in 'perf script' for these options is introduced. + */ +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, + const char *str, int unset) +{ + const char *p; + char *endptr; + bool period_type_set = false; + bool period_set = false; + bool iy = false; + + synth_opts->set = true; + + if (unset) { + synth_opts->dont_decode = true; + return 0; + } + + if (!str) { + itrace_synth_opts__set_default(synth_opts, + synth_opts->default_no_sample); + return 0; + } + + for (p = str; *p;) { + switch (*p++) { + case 'i': + case 'y': + iy = true; + if (p[-1] == 'y') + synth_opts->cycles = true; + else + synth_opts->instructions = true; + while (*p == ' ' || *p == ',') + p += 1; + if (isdigit(*p)) { + synth_opts->period = strtoull(p, &endptr, 10); + period_set = true; + p = endptr; + while (*p == ' ' || *p == ',') + p += 1; + switch (*p++) { + case 'i': + synth_opts->period_type = + PERF_ITRACE_PERIOD_INSTRUCTIONS; + period_type_set = true; + break; + case 't': + synth_opts->period_type = + PERF_ITRACE_PERIOD_TICKS; + period_type_set = true; + break; + case 'm': + synth_opts->period *= 1000; + /* Fall through */ + case 'u': + synth_opts->period *= 1000; + /* Fall through */ + case 'n': + if (*p++ != 's') + goto out_err; + synth_opts->period_type = + PERF_ITRACE_PERIOD_NANOSECS; + period_type_set = true; + break; + case '\0': + goto out; + default: + goto out_err; + } + } + break; + case 'b': + synth_opts->branches = true; + break; + case 'x': + synth_opts->transactions = true; + break; + case 'w': + synth_opts->ptwrites = true; + break; + case 'p': + synth_opts->pwr_events = true; + break; + case 'o': + synth_opts->other_events = true; + break; + case 'I': + synth_opts->intr_events = true; + break; + case 'e': + synth_opts->errors = true; + if (get_flags(&p, &synth_opts->error_plus_flags, + &synth_opts->error_minus_flags)) + goto out_err; + break; + case 'd': + synth_opts->log = true; + if (get_flags(&p, &synth_opts->log_plus_flags, + &synth_opts->log_minus_flags)) + goto out_err; + if (synth_opts->log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR) + synth_opts->log_on_error_size = itrace_log_on_error_size(); + break; + case 'c': + synth_opts->branches = true; + synth_opts->calls = true; + break; + case 'r': + synth_opts->branches = true; + synth_opts->returns = true; + break; + case 'G': + case 'g': + if (p[-1] == 'G') + synth_opts->add_callchain = true; + else + synth_opts->callchain = true; + synth_opts->callchain_sz = + PERF_ITRACE_DEFAULT_CALLCHAIN_SZ; + while (*p == ' ' || *p == ',') + p += 1; + if (isdigit(*p)) { + unsigned int val; + + val = strtoul(p, &endptr, 10); + p = endptr; + if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ) + goto out_err; + synth_opts->callchain_sz = val; + } + break; + case 'L': + case 'l': + if (p[-1] == 'L') + synth_opts->add_last_branch = true; + else + synth_opts->last_branch = true; + synth_opts->last_branch_sz = + PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ; + while (*p == ' ' || *p == ',') + p += 1; + if (isdigit(*p)) { + unsigned int val; + + val = strtoul(p, &endptr, 10); + p = endptr; + if (!val || + val > PERF_ITRACE_MAX_LAST_BRANCH_SZ) + goto out_err; + synth_opts->last_branch_sz = val; + } + break; + case 's': + synth_opts->initial_skip = strtoul(p, &endptr, 10); + if (p == endptr) + goto out_err; + p = endptr; + break; + case 'f': + synth_opts->flc = true; + break; + case 'm': + synth_opts->llc = true; + break; + case 't': + synth_opts->tlb = true; + break; + case 'a': + synth_opts->remote_access = true; + break; + case 'M': + synth_opts->mem = true; + break; + case 'q': + synth_opts->quick += 1; + break; + case 'A': + synth_opts->approx_ipc = true; + break; + case 'Z': + synth_opts->timeless_decoding = true; + break; + case 'T': + synth_opts->use_timestamp = true; + break; + case ' ': + case ',': + break; + default: + goto out_err; + } + } +out: + if (iy) { + if (!period_type_set) + synth_opts->period_type = + PERF_ITRACE_DEFAULT_PERIOD_TYPE; + if (!period_set) + synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD; + } + + return 0; + +out_err: + pr_err("Bad Instruction Tracing options '%s'\n", str); + return -EINVAL; +} + +int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset) +{ + return itrace_do_parse_synth_opts(opt->value, str, unset); +} + +static const char * const auxtrace_error_type_name[] = { + [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace", +}; + +static const char *auxtrace_error_name(unsigned int type) +{ + const char *error_type_name = NULL; + + if (type < PERF_AUXTRACE_ERROR_MAX) + error_type_name = auxtrace_error_type_name[type]; + if (!error_type_name) + error_type_name = "unknown AUX"; + return error_type_name; +} + +size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) +{ + struct perf_record_auxtrace_error *e = &event->auxtrace_error; + unsigned long long nsecs = e->time; + const char *msg = e->msg; + int msg_max; + int ret; + + ret = fprintf(fp, " %s error type %u", + auxtrace_error_name(e->type), e->type); + + if (e->fmt && nsecs) { + unsigned long secs = nsecs / NSEC_PER_SEC; + + nsecs -= secs * NSEC_PER_SEC; + ret += fprintf(fp, " time %lu.%09llu", secs, nsecs); + } else { + ret += fprintf(fp, " time 0"); + } + + if (!e->fmt) + msg = (const char *)&e->time; + + /* Bound msg to the bytes actually within the event, capped at the array size */ + msg_max = (int)((void *)event + event->header.size - (void *)msg); + if (msg_max < 0) + msg_max = 0; + if (msg_max > (int)sizeof(e->msg)) + msg_max = sizeof(e->msg); + + /* + * Unlike the swap path which downgrades fmt in place, + * native-endian events are mmap'd read-only — check size + * instead to avoid accessing machine_pid/vcpu OOB. + */ + if (e->fmt >= 2 && + event->header.size >= offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu) && + e->machine_pid) + ret += fprintf(fp, " machine_pid %d vcpu %d", e->machine_pid, e->vcpu); + + ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %.*s\n", + e->cpu, e->pid, e->tid, e->ip, e->code, msg_max, msg); + return ret; +} + +void perf_session__auxtrace_error_inc(struct perf_session *session, + union perf_event *event) +{ + struct perf_record_auxtrace_error *e = &event->auxtrace_error; + + if (e->type < PERF_AUXTRACE_ERROR_MAX) + evlist__stats(session->evlist)->nr_auxtrace_errors[e->type] += 1; +} + +void events_stats__auxtrace_error_warn(const struct events_stats *stats) +{ + int i; + + for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) { + if (!stats->nr_auxtrace_errors[i]) + continue; + ui__warning("%u %s errors\n", + stats->nr_auxtrace_errors[i], + auxtrace_error_name(i)); + } +} + +int perf_event__process_auxtrace_error(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + if (auxtrace__dont_decode(session)) + return 0; + + perf_event__fprintf_auxtrace_error(event, stdout); + return 0; +} + +/* + * In the compat mode kernel runs in 64-bit and perf tool runs in 32-bit mode, + * 32-bit perf tool cannot access 64-bit value atomically, which might lead to + * the issues caused by the below sequence on multiple CPUs: when perf tool + * accesses either the load operation or the store operation for 64-bit value, + * on some architectures the operation is divided into two instructions, one + * is for accessing the low 32-bit value and another is for the high 32-bit; + * thus these two user operations can give the kernel chances to access the + * 64-bit value, and thus leads to the unexpected load values. + * + * kernel (64-bit) user (32-bit) + * + * if (LOAD ->aux_tail) { --, LOAD ->aux_head_lo + * STORE $aux_data | ,---> + * FLUSH $aux_data | | LOAD ->aux_head_hi + * STORE ->aux_head --|-------` smp_rmb() + * } | LOAD $data + * | smp_mb() + * | STORE ->aux_tail_lo + * `-----------> + * STORE ->aux_tail_hi + * + * For this reason, it's impossible for the perf tool to work correctly when + * the AUX head or tail is bigger than 4GB (more than 32 bits length); and we + * can not simply limit the AUX ring buffer to less than 4GB, the reason is + * the pointers can be increased monotonically, whatever the buffer size it is, + * at the end the head and tail can be bigger than 4GB and carry out to the + * high 32-bit. + * + * To mitigate the issues and improve the user experience, we can allow the + * perf tool working in certain conditions and bail out with error if detect + * any overflow cannot be handled. + * + * For reading the AUX head, it reads out the values for three times, and + * compares the high 4 bytes of the values between the first time and the last + * time, if there has no change for high 4 bytes injected by the kernel during + * the user reading sequence, it's safe for use the second value. + * + * When compat_auxtrace_mmap__write_tail() detects any carrying in the high + * 32 bits, it means there have two store operations in user space and it cannot + * promise the atomicity for 64-bit write, so return '-1' in this case to tell + * the caller an overflow error has happened. + */ +u64 __weak compat_auxtrace_mmap__read_head(struct auxtrace_mmap *mm) +{ + struct perf_event_mmap_page *pc = mm->userpg; + u64 first, second, last; + u64 mask = (u64)(UINT32_MAX) << 32; + + do { + first = READ_ONCE(pc->aux_head); + /* Ensure all reads are done after we read the head */ + smp_rmb(); + second = READ_ONCE(pc->aux_head); + /* Ensure all reads are done after we read the head */ + smp_rmb(); + last = READ_ONCE(pc->aux_head); + } while ((first & mask) != (last & mask)); + + return second; +} + +int __weak compat_auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail) +{ + struct perf_event_mmap_page *pc = mm->userpg; + u64 mask = (u64)(UINT32_MAX) << 32; + + if (tail & mask) + return -1; + + /* Ensure all reads are done before we write the tail out */ + smp_mb(); + WRITE_ONCE(pc->aux_tail, tail); + return 0; +} + +static int __auxtrace_mmap__read(struct mmap *map, + struct auxtrace_record *itr, struct perf_env *env, + const struct perf_tool *tool, process_auxtrace_t fn, + bool snapshot, size_t snapshot_size) +{ + struct auxtrace_mmap *mm = &map->auxtrace_mmap; + u64 head, old = mm->prev, offset, ref; + unsigned char *data = mm->base; + size_t size, head_off, old_off, len1, len2, padding; + union perf_event ev; + void *data1, *data2; + int kernel_is_64_bit = perf_env__kernel_is_64_bit(env); + + head = auxtrace_mmap__read_head(mm, kernel_is_64_bit); + + if (snapshot && + auxtrace_record__find_snapshot(itr, mm->idx, mm, data, &head, &old)) + return -1; + + if (old == head) + return 0; + + pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n", + mm->idx, old, head, head - old); + + if (mm->mask) { + head_off = head & mm->mask; + old_off = old & mm->mask; + } else { + head_off = head % mm->len; + old_off = old % mm->len; + } + + if (head_off > old_off) + size = head_off - old_off; + else + size = mm->len - (old_off - head_off); + + if (snapshot && size > snapshot_size) + size = snapshot_size; + + ref = auxtrace_record__reference(itr); + + if (head > old || size <= head || mm->mask) { + offset = head - size; + } else { + /* + * When the buffer size is not a power of 2, 'head' wraps at the + * highest multiple of the buffer size, so we have to subtract + * the remainder here. + */ + u64 rem = (0ULL - mm->len) % mm->len; + + offset = head - size - rem; + } + + if (size > head_off) { + len1 = size - head_off; + data1 = &data[mm->len - len1]; + len2 = head_off; + data2 = &data[0]; + } else { + len1 = size; + data1 = &data[head_off - len1]; + len2 = 0; + data2 = NULL; + } + + if (itr->alignment) { + unsigned int unwanted = len1 % itr->alignment; + + len1 -= unwanted; + size -= unwanted; + } + + /* padding must be written by fn() e.g. record__process_auxtrace() */ + padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1); + if (padding) + padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding; + + memset(&ev, 0, sizeof(ev)); + ev.auxtrace.header.type = PERF_RECORD_AUXTRACE; + ev.auxtrace.header.size = sizeof(ev.auxtrace); + ev.auxtrace.size = size + padding; + ev.auxtrace.offset = offset; + ev.auxtrace.reference = ref; + ev.auxtrace.idx = mm->idx; + ev.auxtrace.tid = mm->tid; + ev.auxtrace.cpu = mm->cpu; + + if (fn(tool, map, &ev, data1, len1, data2, len2)) + return -1; + + mm->prev = head; + + if (!snapshot) { + int err; + + err = auxtrace_mmap__write_tail(mm, head, kernel_is_64_bit); + if (err < 0) + return err; + + if (itr->read_finish) { + err = itr->read_finish(itr, mm->idx); + if (err < 0) + return err; + } + } + + return 1; +} + +int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr, + struct perf_env *env, const struct perf_tool *tool, + process_auxtrace_t fn) +{ + return __auxtrace_mmap__read(map, itr, env, tool, fn, false, 0); +} + +int auxtrace_mmap__read_snapshot(struct mmap *map, + struct auxtrace_record *itr, struct perf_env *env, + const struct perf_tool *tool, process_auxtrace_t fn, + size_t snapshot_size) +{ + return __auxtrace_mmap__read(map, itr, env, tool, fn, true, snapshot_size); +} + +/** + * struct auxtrace_cache - hash table to implement a cache + * @hashtable: the hashtable + * @sz: hashtable size (number of hlists) + * @entry_size: size of an entry + * @limit: limit the number of entries to this maximum, when reached the cache + * is dropped and caching begins again with an empty cache + * @cnt: current number of entries + * @bits: hashtable size (@sz = 2^@bits) + */ +struct auxtrace_cache { + struct hlist_head *hashtable; + size_t sz; + size_t entry_size; + size_t limit; + size_t cnt; + unsigned int bits; +}; + +struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size, + unsigned int limit_percent) +{ + struct auxtrace_cache *c; + struct hlist_head *ht; + size_t sz, i; + + c = zalloc(sizeof(struct auxtrace_cache)); + if (!c) + return NULL; + + sz = 1UL << bits; + + ht = calloc(sz, sizeof(struct hlist_head)); + if (!ht) + goto out_free; + + for (i = 0; i < sz; i++) + INIT_HLIST_HEAD(&ht[i]); + + c->hashtable = ht; + c->sz = sz; + c->entry_size = entry_size; + c->limit = (c->sz * limit_percent) / 100; + c->bits = bits; + + return c; + +out_free: + free(c); + return NULL; +} + +static void auxtrace_cache__drop(struct auxtrace_cache *c) +{ + struct auxtrace_cache_entry *entry; + struct hlist_node *tmp; + size_t i; + + if (!c) + return; + + for (i = 0; i < c->sz; i++) { + hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) { + hlist_del(&entry->hash); + auxtrace_cache__free_entry(c, entry); + } + } + + c->cnt = 0; +} + +void auxtrace_cache__free(struct auxtrace_cache *c) +{ + if (!c) + return; + + auxtrace_cache__drop(c); + zfree(&c->hashtable); + free(c); +} + +void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c) +{ + return malloc(c->entry_size); +} + +void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused, + void *entry) +{ + free(entry); +} + +int auxtrace_cache__add(struct auxtrace_cache *c, u32 key, + struct auxtrace_cache_entry *entry) +{ + if (c->limit && ++c->cnt > c->limit) + auxtrace_cache__drop(c); + + entry->key = key; + hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]); + + return 0; +} + +static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c, + u32 key) +{ + struct auxtrace_cache_entry *entry; + struct hlist_head *hlist; + struct hlist_node *n; + + if (!c) + return NULL; + + hlist = &c->hashtable[hash_32(key, c->bits)]; + hlist_for_each_entry_safe(entry, n, hlist, hash) { + if (entry->key == key) { + hlist_del(&entry->hash); + return entry; + } + } + + return NULL; +} + +void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key) +{ + struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key); + + auxtrace_cache__free_entry(c, entry); +} + +void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key) +{ + struct auxtrace_cache_entry *entry; + struct hlist_head *hlist; + + if (!c) + return NULL; + + hlist = &c->hashtable[hash_32(key, c->bits)]; + hlist_for_each_entry(entry, hlist, hash) { + if (entry->key == key) + return entry; + } + + return NULL; +} + +static void addr_filter__free_str(struct addr_filter *filt) +{ + zfree(&filt->str); + filt->action = NULL; + filt->sym_from = NULL; + filt->sym_to = NULL; + filt->filename = NULL; +} + +static struct addr_filter *addr_filter__new(void) +{ + struct addr_filter *filt = zalloc(sizeof(*filt)); + + if (filt) + INIT_LIST_HEAD(&filt->list); + + return filt; +} + +static void addr_filter__free(struct addr_filter *filt) +{ + if (filt) + addr_filter__free_str(filt); + free(filt); +} + +static void addr_filters__add(struct addr_filters *filts, + struct addr_filter *filt) +{ + list_add_tail(&filt->list, &filts->head); + filts->cnt += 1; +} + +static void addr_filters__del(struct addr_filters *filts, + struct addr_filter *filt) +{ + list_del_init(&filt->list); + filts->cnt -= 1; +} + +void addr_filters__init(struct addr_filters *filts) +{ + INIT_LIST_HEAD(&filts->head); + filts->cnt = 0; +} + +void addr_filters__exit(struct addr_filters *filts) +{ + struct addr_filter *filt, *n; + + list_for_each_entry_safe(filt, n, &filts->head, list) { + addr_filters__del(filts, filt); + addr_filter__free(filt); + } +} + +static int parse_num_or_str(char **inp, u64 *num, const char **str, + const char *str_delim) +{ + *inp += strspn(*inp, " "); + + if (isdigit(**inp)) { + char *endptr; + + if (!num) + return -EINVAL; + errno = 0; + *num = strtoull(*inp, &endptr, 0); + if (errno) + return -errno; + if (endptr == *inp) + return -EINVAL; + *inp = endptr; + } else { + size_t n; + + if (!str) + return -EINVAL; + *inp += strspn(*inp, " "); + *str = *inp; + n = strcspn(*inp, str_delim); + if (!n) + return -EINVAL; + *inp += n; + if (**inp) { + **inp = '\0'; + *inp += 1; + } + } + return 0; +} + +static int parse_action(struct addr_filter *filt) +{ + if (!strcmp(filt->action, "filter")) { + filt->start = true; + filt->range = true; + } else if (!strcmp(filt->action, "start")) { + filt->start = true; + } else if (!strcmp(filt->action, "stop")) { + filt->start = false; + } else if (!strcmp(filt->action, "tracestop")) { + filt->start = false; + filt->range = true; + filt->action += 5; /* Change 'tracestop' to 'stop' */ + } else { + return -EINVAL; + } + return 0; +} + +static int parse_sym_idx(char **inp, int *idx) +{ + *idx = -1; + + *inp += strspn(*inp, " "); + + if (**inp != '#') + return 0; + + *inp += 1; + + if (**inp == 'g' || **inp == 'G') { + *inp += 1; + *idx = 0; + } else { + unsigned long num; + char *endptr; + + errno = 0; + num = strtoul(*inp, &endptr, 0); + if (errno) + return -errno; + if (endptr == *inp || num > INT_MAX) + return -EINVAL; + *inp = endptr; + *idx = num; + } + + return 0; +} + +static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx) +{ + int err = parse_num_or_str(inp, num, str, " "); + + if (!err && *str) + err = parse_sym_idx(inp, idx); + + return err; +} + +static int parse_one_filter(struct addr_filter *filt, const char **filter_inp) +{ + char *fstr; + int err; + + filt->str = fstr = strdup(*filter_inp); + if (!fstr) + return -ENOMEM; + + err = parse_num_or_str(&fstr, NULL, &filt->action, " "); + if (err) + goto out_err; + + err = parse_action(filt); + if (err) + goto out_err; + + err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from, + &filt->sym_from_idx); + if (err) + goto out_err; + + fstr += strspn(fstr, " "); + + if (*fstr == '/') { + fstr += 1; + err = parse_addr_size(&fstr, &filt->size, &filt->sym_to, + &filt->sym_to_idx); + if (err) + goto out_err; + filt->range = true; + } + + fstr += strspn(fstr, " "); + + if (*fstr == '@') { + fstr += 1; + err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,"); + if (err) + goto out_err; + } + + fstr += strspn(fstr, " ,"); + + *filter_inp += fstr - filt->str; + + return 0; + +out_err: + addr_filter__free_str(filt); + + return err; +} + +int addr_filters__parse_bare_filter(struct addr_filters *filts, + const char *filter) +{ + struct addr_filter *filt; + const char *fstr = filter; + int err; + + while (*fstr) { + filt = addr_filter__new(); + err = parse_one_filter(filt, &fstr); + if (err) { + addr_filter__free(filt); + addr_filters__exit(filts); + return err; + } + addr_filters__add(filts, filt); + } + + return 0; +} + +struct sym_args { + const char *name; + u64 start; + u64 size; + int idx; + int cnt; + bool started; + bool global; + bool selected; + bool duplicate; + bool near; +}; + +static bool kern_sym_name_match(const char *kname, const char *name) +{ + size_t n = strlen(name); + + return !strcmp(kname, name) || + (!strncmp(kname, name, n) && kname[n] == '\t'); +} + +static bool kern_sym_match(struct sym_args *args, const char *name, char type) +{ + /* A function with the same name, and global or the n'th found or any */ + return kallsyms__is_function(type) && + kern_sym_name_match(name, args->name) && + ((args->global && isupper(type)) || + (args->selected && ++(args->cnt) == args->idx) || + (!args->global && !args->selected)); +} + +static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start) +{ + struct sym_args *args = arg; + + if (args->started) { + if (!args->size) + args->size = start - args->start; + if (args->selected) { + if (args->size) + return 1; + } else if (kern_sym_match(args, name, type)) { + args->duplicate = true; + return 1; + } + } else if (kern_sym_match(args, name, type)) { + args->started = true; + args->start = start; + } + + return 0; +} + +static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start) +{ + struct sym_args *args = arg; + + if (kern_sym_match(args, name, type)) { + pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", + ++args->cnt, start, type, name); + args->near = true; + } else if (args->near) { + args->near = false; + pr_err("\t\twhich is near\t\t%s\n", name); + } + + return 0; +} + +static int sym_not_found_error(const char *sym_name, int idx) +{ + if (idx > 0) { + pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n", + idx, sym_name); + } else if (!idx) { + pr_err("Global symbol '%s' not found.\n", sym_name); + } else { + pr_err("Symbol '%s' not found.\n", sym_name); + } + pr_err("Note that symbols must be functions.\n"); + + return -EINVAL; +} + +static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx) +{ + struct sym_args args = { + .name = sym_name, + .idx = idx, + .global = !idx, + .selected = idx > 0, + }; + int err; + + *start = 0; + *size = 0; + + err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb); + if (err < 0) { + pr_err("Failed to parse /proc/kallsyms\n"); + return err; + } + + if (args.duplicate) { + pr_err("Multiple kernel symbols with name '%s'\n", sym_name); + args.cnt = 0; + kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb); + pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n", + sym_name); + pr_err("Or select a global symbol by inserting #0 or #g or #G\n"); + return -EINVAL; + } + + if (!args.started) { + pr_err("Kernel symbol lookup: "); + return sym_not_found_error(sym_name, idx); + } + + *start = args.start; + *size = args.size; + + return 0; +} + +static int find_entire_kern_cb(void *arg, const char *name __maybe_unused, + char type, u64 start) +{ + struct sym_args *args = arg; + u64 size; + + if (!kallsyms__is_function(type)) + return 0; + + if (!args->started) { + args->started = true; + args->start = start; + } + /* Don't know exactly where the kernel ends, so we add a page */ + size = round_up(start, page_size) + page_size - args->start; + if (size > args->size) + args->size = size; + + return 0; +} + +static int addr_filter__entire_kernel(struct addr_filter *filt) +{ + struct sym_args args = { .started = false }; + int err; + + err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb); + if (err < 0 || !args.started) { + pr_err("Failed to parse /proc/kallsyms\n"); + return err; + } + + filt->addr = args.start; + filt->size = args.size; + + return 0; +} + +static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size) +{ + if (start + size >= filt->addr) + return 0; + + if (filt->sym_from) { + pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n", + filt->sym_to, start, filt->sym_from, filt->addr); + } else { + pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n", + filt->sym_to, start, filt->addr); + } + + return -EINVAL; +} + +static int addr_filter__resolve_kernel_syms(struct addr_filter *filt) +{ + bool no_size = false; + u64 start, size; + int err; + + if (symbol_conf.kptr_restrict) { + pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n"); + return -EINVAL; + } + + if (filt->sym_from && !strcmp(filt->sym_from, "*")) + return addr_filter__entire_kernel(filt); + + if (filt->sym_from) { + err = find_kern_sym(filt->sym_from, &start, &size, + filt->sym_from_idx); + if (err) + return err; + filt->addr = start; + if (filt->range && !filt->size && !filt->sym_to) { + filt->size = size; + no_size = !size; + } + } + + if (filt->sym_to) { + err = find_kern_sym(filt->sym_to, &start, &size, + filt->sym_to_idx); + if (err) + return err; + + err = check_end_after_start(filt, start, size); + if (err) + return err; + filt->size = start + size - filt->addr; + no_size = !size; + } + + /* The very last symbol in kallsyms does not imply a particular size */ + if (no_size) { + pr_err("Cannot determine size of symbol '%s'\n", + filt->sym_to ? filt->sym_to : filt->sym_from); + return -EINVAL; + } + + return 0; +} + +static struct dso *load_dso(const char *name) +{ + struct map *map; + struct dso *dso; + + map = dso__new_map(name); + if (!map) + return NULL; + + if (map__load(map) < 0) + pr_err("File '%s' not found or has no symbols.\n", name); + + dso = dso__get(map__dso(map)); + + map__put(map); + + return dso; +} + +static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt, + int idx) +{ + /* Same name, and global or the n'th found or any */ + return !arch__compare_symbol_names(name, sym->name) && + ((!idx && symbol__binding(sym) == STB_GLOBAL) || + (idx > 0 && ++*cnt == idx) || + idx < 0); +} + +static void print_duplicate_syms(struct dso *dso, const char *sym_name) +{ + struct symbol *sym; + bool near = false; + int cnt = 0; + + pr_err("Multiple symbols with name '%s'\n", sym_name); + + sym = dso__first_symbol(dso); + while (sym) { + if (dso_sym_match(sym, sym_name, &cnt, -1)) { + pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", + ++cnt, sym->start, + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', + sym->name); + near = true; + } else if (near) { + near = false; + pr_err("\t\twhich is near\t\t%s\n", sym->name); + } + sym = dso__next_symbol(sym); + } + + pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n", + sym_name); + pr_err("Or select a global symbol by inserting #0 or #g or #G\n"); +} + +static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start, + u64 *size, int idx) +{ + struct symbol *sym; + int cnt = 0; + + *start = 0; + *size = 0; + + sym = dso__first_symbol(dso); + while (sym) { + if (*start) { + if (!*size) + *size = sym->start - *start; + if (idx > 0) { + if (*size) + return 0; + } else if (dso_sym_match(sym, sym_name, &cnt, idx)) { + print_duplicate_syms(dso, sym_name); + return -EINVAL; + } + } else if (dso_sym_match(sym, sym_name, &cnt, idx)) { + *start = sym->start; + *size = sym->end - sym->start; + } + sym = dso__next_symbol(sym); + } + + if (!*start) + return sym_not_found_error(sym_name, idx); + + return 0; +} + +static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso) +{ + if (dso__data_file_size(dso, NULL)) { + pr_err("Failed to determine filter for %s\nCannot determine file size.\n", + filt->filename); + return -EINVAL; + } + + filt->addr = 0; + filt->size = dso__data(dso)->file_size; + + return 0; +} + +static int addr_filter__resolve_syms(struct addr_filter *filt) +{ + u64 start, size; + struct dso *dso; + int err = 0; + + if (!filt->sym_from && !filt->sym_to) + return 0; + + if (!filt->filename) + return addr_filter__resolve_kernel_syms(filt); + + dso = load_dso(filt->filename); + if (!dso) { + pr_err("Failed to load symbols from: %s\n", filt->filename); + return -EINVAL; + } + + if (filt->sym_from && !strcmp(filt->sym_from, "*")) { + err = addr_filter__entire_dso(filt, dso); + goto put_dso; + } + + if (filt->sym_from) { + err = find_dso_sym(dso, filt->sym_from, &start, &size, + filt->sym_from_idx); + if (err) + goto put_dso; + filt->addr = start; + if (filt->range && !filt->size && !filt->sym_to) + filt->size = size; + } + + if (filt->sym_to) { + err = find_dso_sym(dso, filt->sym_to, &start, &size, + filt->sym_to_idx); + if (err) + goto put_dso; + + err = check_end_after_start(filt, start, size); + if (err) + return err; + + filt->size = start + size - filt->addr; + } + +put_dso: + dso__put(dso); + + return err; +} + +static char *addr_filter__to_str(struct addr_filter *filt) +{ + char filename_buf[PATH_MAX]; + const char *at = ""; + const char *fn = ""; + char *filter; + int err; + + if (filt->filename) { + at = "@"; + fn = realpath(filt->filename, filename_buf); + if (!fn) + return NULL; + } + + if (filt->range) { + err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s", + filt->action, filt->addr, filt->size, at, fn); + } else { + err = asprintf(&filter, "%s 0x%"PRIx64"%s%s", + filt->action, filt->addr, at, fn); + } + + return err < 0 ? NULL : filter; +} + +static int parse_addr_filter(struct evsel *evsel, const char *filter, + int max_nr) +{ + struct addr_filters filts; + struct addr_filter *filt; + int err; + + addr_filters__init(&filts); + + err = addr_filters__parse_bare_filter(&filts, filter); + if (err) + goto out_exit; + + if (filts.cnt > max_nr) { + pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n", + filts.cnt, max_nr); + err = -EINVAL; + goto out_exit; + } + + list_for_each_entry(filt, &filts.head, list) { + char *new_filter; + + err = addr_filter__resolve_syms(filt); + if (err) + goto out_exit; + + new_filter = addr_filter__to_str(filt); + if (!new_filter) { + err = -ENOMEM; + goto out_exit; + } + + if (evsel__append_addr_filter(evsel, new_filter)) { + err = -ENOMEM; + goto out_exit; + } + } + +out_exit: + addr_filters__exit(&filts); + + if (err) { + pr_err("Failed to parse address filter: '%s'\n", filter); + pr_err("Filter format is: filter|start|stop|tracestop [/ ] [@]\n"); + pr_err("Where multiple filters are separated by space or comma.\n"); + } + + return err; +} + +static int evsel__nr_addr_filter(struct evsel *evsel) +{ + struct perf_pmu *pmu = evsel__find_pmu(evsel); + int nr_addr_filters = 0; + + if (!pmu) + return 0; + + perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters); + + return nr_addr_filters; +} + +int auxtrace_parse_filters(struct evlist *evlist) +{ + struct evsel *evsel; + char *filter; + int err, max_nr; + + evlist__for_each_entry(evlist, evsel) { + filter = evsel->filter; + max_nr = evsel__nr_addr_filter(evsel); + if (!filter || !max_nr) + continue; + evsel->filter = NULL; + err = parse_addr_filter(evsel, filter, max_nr); + free(filter); + if (err) + return err; + pr_debug("Address filter: %s\n", evsel->filter); + } + + return 0; +} + +int auxtrace__process_event(struct perf_session *session, union perf_event *event, + struct perf_sample *sample, const struct perf_tool *tool) +{ + if (!session->auxtrace) + return 0; + + return session->auxtrace->process_event(session, event, sample, tool); +} + +void auxtrace__dump_auxtrace_sample(struct perf_session *session, + struct perf_sample *sample) +{ + if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample || + auxtrace__dont_decode(session)) + return; + + session->auxtrace->dump_auxtrace_sample(session, sample); +} + +int auxtrace__flush_events(struct perf_session *session, const struct perf_tool *tool) +{ + if (!session->auxtrace) + return 0; + + return session->auxtrace->flush_events(session, tool); +} + +void auxtrace__free_events(struct perf_session *session) +{ + if (!session->auxtrace) + return; + + return session->auxtrace->free_events(session); +} + +void auxtrace__free(struct perf_session *session) +{ + if (!session->auxtrace) + return; + + return session->auxtrace->free(session); +} + +bool auxtrace__evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace) + return false; + + return session->auxtrace->evsel_is_auxtrace(session, evsel); +} diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h new file mode 100644 index 000000000..6947f3f28 --- /dev/null +++ b/tools/perf/util/auxtrace.h @@ -0,0 +1,702 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * auxtrace.h: AUX area trace support + * Copyright (c) 2013-2015, Intel Corporation. + */ + +#ifndef __PERF_AUXTRACE_H +#define __PERF_AUXTRACE_H + +#include +#include // FILE +#include +#include +#include +#include + +union perf_event; +struct perf_session; +struct evlist; +struct evsel; +struct perf_env; +struct perf_tool; +struct mmap; +struct perf_sample; +struct option; +struct record_opts; +struct perf_record_auxtrace_error; +struct perf_record_auxtrace_info; +struct events_stats; +struct perf_pmu; + +enum auxtrace_error_type { + PERF_AUXTRACE_ERROR_ITRACE = 1, + PERF_AUXTRACE_ERROR_MAX +}; + +/* Auxtrace records must have the same alignment as perf event records */ +#define PERF_AUXTRACE_RECORD_ALIGNMENT 8 + +enum auxtrace_type { + PERF_AUXTRACE_UNKNOWN, + PERF_AUXTRACE_INTEL_PT, + PERF_AUXTRACE_INTEL_BTS, + PERF_AUXTRACE_CS_ETM, + PERF_AUXTRACE_ARM_SPE, + PERF_AUXTRACE_S390_CPUMSF, + PERF_AUXTRACE_HISI_PTT, + PERF_AUXTRACE_VPA_DTL, +}; + +enum itrace_period_type { + PERF_ITRACE_PERIOD_INSTRUCTIONS, + PERF_ITRACE_PERIOD_TICKS, + PERF_ITRACE_PERIOD_NANOSECS, +}; + +#define AUXTRACE_ERR_FLG_OVERFLOW (1 << ('o' - 'a')) +#define AUXTRACE_ERR_FLG_DATA_LOST (1 << ('l' - 'a')) + +#define AUXTRACE_LOG_FLG_ALL_PERF_EVTS (1 << ('a' - 'a')) +#define AUXTRACE_LOG_FLG_ON_ERROR (1 << ('e' - 'a')) +#define AUXTRACE_LOG_FLG_USE_STDOUT (1 << ('o' - 'a')) + +/** + * struct itrace_synth_opts - AUX area tracing synthesis options. + * @set: indicates whether or not options have been set + * @default_no_sample: Default to no sampling. + * @inject: indicates the event (not just the sample) must be fully synthesized + * because 'perf inject' will write it out + * @instructions: whether to synthesize 'instructions' events + * @cycles: whether to synthesize 'cycles' events + * (not fully accurate, since CYC packets are only emitted + * together with other events, such as branches) + * @branches: whether to synthesize 'branches' events + * @transactions: whether to synthesize events for transactions + * @ptwrites: whether to synthesize events for ptwrites + * @pwr_events: whether to synthesize power events + * @other_events: whether to synthesize other events recorded due to the use of + * aux_output + * @intr_events: whether to synthesize interrupt events + * @errors: whether to synthesize decoder error events + * @dont_decode: whether to skip decoding entirely + * @log: write a decoding log + * @calls: limit branch samples to calls (can be combined with @returns) + * @returns: limit branch samples to returns (can be combined with @calls) + * @callchain: add callchain to 'instructions' events + * @add_callchain: add callchain to existing event records + * @thread_stack: feed branches to the thread_stack + * @last_branch: add branch context to 'instruction' events + * @add_last_branch: add branch context to existing event records + * @approx_ipc: approximate IPC + * @flc: whether to synthesize first level cache events + * @llc: whether to synthesize last level cache events + * @tlb: whether to synthesize TLB events + * @remote_access: whether to synthesize remote access events + * @mem: whether to synthesize memory events + * @timeless_decoding: prefer "timeless" decoding i.e. ignore timestamps + * @use_timestamp: use the timestamp trace as kernel time + * @vm_time_correlation: perform VM Time Correlation + * @vm_tm_corr_dry_run: VM Time Correlation dry-run + * @vm_tm_corr_args: VM Time Correlation implementation-specific arguments + * @callchain_sz: maximum callchain size + * @last_branch_sz: branch context size + * @period: 'instructions' events period + * @period_type: 'instructions' events period type + * @initial_skip: skip N events at the beginning. + * @cpu_bitmap: CPUs for which to synthesize events, or NULL for all + * @ptime_range: time intervals to trace or NULL + * @range_num: number of time intervals to trace + * @error_plus_flags: flags to affect what errors are reported + * @error_minus_flags: flags to affect what errors are reported + * @log_plus_flags: flags to affect what is logged + * @log_minus_flags: flags to affect what is logged + * @quick: quicker (less detailed) decoding + * @log_on_error_size: size of log to keep for outputting log only on errors + */ +struct itrace_synth_opts { + bool set; + bool default_no_sample; + bool inject; + bool instructions; + bool cycles; + bool branches; + bool transactions; + bool ptwrites; + bool pwr_events; + bool other_events; + bool intr_events; + bool errors; + bool dont_decode; + bool log; + bool calls; + bool returns; + bool callchain; + bool add_callchain; + bool thread_stack; + bool last_branch; + bool add_last_branch; + bool approx_ipc; + bool flc; + bool llc; + bool tlb; + bool remote_access; + bool mem; + bool timeless_decoding; + bool use_timestamp; + bool vm_time_correlation; + bool vm_tm_corr_dry_run; + char *vm_tm_corr_args; + unsigned int callchain_sz; + unsigned int last_branch_sz; + unsigned long long period; + enum itrace_period_type period_type; + unsigned long initial_skip; + unsigned long *cpu_bitmap; + struct perf_time_interval *ptime_range; + int range_num; + unsigned int error_plus_flags; + unsigned int error_minus_flags; + unsigned int log_plus_flags; + unsigned int log_minus_flags; + unsigned int quick; + unsigned int log_on_error_size; +}; + +/** + * struct auxtrace_index_entry - indexes a AUX area tracing event within a + * perf.data file. + * @file_offset: offset within the perf.data file + * @sz: size of the event + */ +struct auxtrace_index_entry { + u64 file_offset; + u64 sz; +}; + +#define PERF_AUXTRACE_INDEX_ENTRY_COUNT 256 + +/** + * struct auxtrace_index - index of AUX area tracing events within a perf.data + * file. + * @list: linking a number of arrays of entries + * @nr: number of entries + * @entries: array of entries + */ +struct auxtrace_index { + struct list_head list; + size_t nr; + struct auxtrace_index_entry entries[PERF_AUXTRACE_INDEX_ENTRY_COUNT]; +}; + +/** + * struct auxtrace - session callbacks to allow AUX area data decoding. + * @process_event: lets the decoder see all session events + * @process_auxtrace_event: process a PERF_RECORD_AUXTRACE event + * @queue_data: queue an AUX sample or PERF_RECORD_AUXTRACE event for later + * processing + * @dump_auxtrace_sample: dump AUX area sample data + * @flush_events: process any remaining data + * @free_events: free resources associated with event processing + * @free: free resources associated with the session + */ +struct auxtrace { + int (*process_event)(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool); + int (*process_auxtrace_event)(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool); + int (*queue_data)(struct perf_session *session, + struct perf_sample *sample, union perf_event *event, + u64 data_offset); + void (*dump_auxtrace_sample)(struct perf_session *session, + struct perf_sample *sample); + int (*flush_events)(struct perf_session *session, + const struct perf_tool *tool); + void (*free_events)(struct perf_session *session); + void (*free)(struct perf_session *session); + bool (*evsel_is_auxtrace)(struct perf_session *session, + struct evsel *evsel); +}; + +/** + * struct auxtrace_buffer - a buffer containing AUX area tracing data. + * @list: buffers are queued in a list held by struct auxtrace_queue + * @size: size of the buffer in bytes + * @pid: in per-thread mode, the pid this buffer is associated with + * @tid: in per-thread mode, the tid this buffer is associated with + * @cpu: in per-cpu mode, the cpu this buffer is associated with + * @data: actual buffer data (can be null if the data has not been loaded) + * @data_offset: file offset at which the buffer can be read + * @mmap_addr: mmap address at which the buffer can be read + * @mmap_size: size of the mmap at @mmap_addr + * @data_needs_freeing: @data was malloc'd so free it when it is no longer + * needed + * @consecutive: the original data was split up and this buffer is consecutive + * to the previous buffer + * @offset: offset as determined by aux_head / aux_tail members of struct + * perf_event_mmap_page + * @reference: an implementation-specific reference determined when the data is + * recorded + * @buffer_nr: used to number each buffer + * @use_size: implementation actually only uses this number of bytes + * @use_data: implementation actually only uses data starting at this address + */ +struct auxtrace_buffer { + struct list_head list; + size_t size; + pid_t pid; + pid_t tid; + struct perf_cpu cpu; + void *data; + off_t data_offset; + void *mmap_addr; + size_t mmap_size; + bool data_needs_freeing; + bool consecutive; + u64 offset; + u64 reference; + u64 buffer_nr; + size_t use_size; + void *use_data; +}; + +/** + * struct auxtrace_queue - a queue of AUX area tracing data buffers. + * @head: head of buffer list + * @tid: in per-thread mode, the tid this queue is associated with + * @cpu: in per-cpu mode, the cpu this queue is associated with + * @set: %true once this queue has been dedicated to a specific thread or cpu + * @priv: implementation-specific data + */ +struct auxtrace_queue { + struct list_head head; + pid_t tid; + int cpu; + bool set; + void *priv; +}; + +/** + * struct auxtrace_queues - an array of AUX area tracing queues. + * @queue_array: array of queues + * @nr_queues: number of queues + * @new_data: set whenever new data is queued + * @populated: queues have been fully populated using the auxtrace_index + * @next_buffer_nr: used to number each buffer + */ +struct auxtrace_queues { + struct auxtrace_queue *queue_array; + unsigned int nr_queues; + bool new_data; + bool populated; + u64 next_buffer_nr; +}; + +/** + * struct auxtrace_heap_item - element of struct auxtrace_heap. + * @queue_nr: queue number + * @ordinal: value used for sorting (lowest ordinal is top of the heap) expected + * to be a timestamp + */ +struct auxtrace_heap_item { + unsigned int queue_nr; + u64 ordinal; +}; + +/** + * struct auxtrace_heap - a heap suitable for sorting AUX area tracing queues. + * @heap_array: the heap + * @heap_cnt: the number of elements in the heap + * @heap_sz: maximum number of elements (grows as needed) + */ +struct auxtrace_heap { + struct auxtrace_heap_item *heap_array; + unsigned int heap_cnt; + unsigned int heap_sz; +}; + +/** + * struct auxtrace_mmap - records an mmap of the auxtrace buffer. + * @base: address of mapped area + * @userpg: pointer to buffer's perf_event_mmap_page + * @mask: %0 if @len is not a power of two, otherwise (@len - %1) + * @len: size of mapped area + * @prev: previous aux_head + * @idx: index of this mmap + * @tid: tid for a per-thread mmap (also set if there is only 1 tid on a per-cpu + * mmap) otherwise %0 + * @cpu: cpu number for a per-cpu mmap otherwise %-1 + */ +struct auxtrace_mmap { + void *base; + void *userpg; + size_t mask; + size_t len; + u64 prev; + int idx; + pid_t tid; + int cpu; +}; + +/** + * struct auxtrace_mmap_params - parameters to set up struct auxtrace_mmap. + * @mask: %0 if @len is not a power of two, otherwise (@len - %1) + * @offset: file offset of mapped area + * @len: size of mapped area + * @prot: mmap memory protection + * @idx: index of this mmap + * @tid: tid for a per-thread mmap (also set if there is only 1 tid on a per-cpu + * mmap) otherwise %0 + * @mmap_needed: set to %false for non-auxtrace events. This is needed because + * auxtrace mmapping is done in the same code path as non-auxtrace + * mmapping but not every evsel that needs non-auxtrace mmapping + * also needs auxtrace mmapping. + * @cpu: cpu number for a per-cpu mmap otherwise %-1 + */ +struct auxtrace_mmap_params { + size_t mask; + off_t offset; + size_t len; + int prot; + int idx; + pid_t tid; + bool mmap_needed; + struct perf_cpu cpu; +}; + +/** + * struct auxtrace_record - callbacks for recording AUX area data. + * @recording_options: validate and process recording options + * @info_priv_size: return the size of the private data in auxtrace_info_event + * @info_fill: fill-in the private data in auxtrace_info_event + * @free: free this auxtrace record structure + * @snapshot_start: starting a snapshot + * @snapshot_finish: finishing a snapshot + * @find_snapshot: find data to snapshot within auxtrace mmap + * @parse_snapshot_options: parse snapshot options + * @reference: provide a 64-bit reference number for auxtrace_event + * @read_finish: called after reading from an auxtrace mmap + * @alignment: alignment (if any) for AUX area data + * @default_aux_sample_size: default sample size for --aux sample option + * @pmu: associated pmu + * @evlist: selected events list + */ +struct auxtrace_record { + int (*recording_options)(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts); + size_t (*info_priv_size)(struct auxtrace_record *itr, + struct evlist *evlist); + int (*info_fill)(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size); + void (*free)(struct auxtrace_record *itr); + int (*snapshot_start)(struct auxtrace_record *itr); + int (*snapshot_finish)(struct auxtrace_record *itr); + int (*find_snapshot)(struct auxtrace_record *itr, int idx, + struct auxtrace_mmap *mm, unsigned char *data, + u64 *head, u64 *old); + int (*parse_snapshot_options)(struct auxtrace_record *itr, + struct record_opts *opts, + const char *str); + u64 (*reference)(struct auxtrace_record *itr); + int (*read_finish)(struct auxtrace_record *itr, int idx); + unsigned int alignment; + unsigned int default_aux_sample_size; + struct evlist *evlist; +}; + +/** + * struct addr_filter - address filter. + * @list: list node + * @range: true if it is a range filter + * @start: true if action is 'filter' or 'start' + * @action: 'filter', 'start' or 'stop' ('tracestop' is accepted but converted + * to 'stop') + * @sym_from: symbol name for the filter address + * @sym_to: symbol name that determines the filter size + * @sym_from_idx: selects n'th from symbols with the same name (0 means global + * and less than 0 means symbol must be unique) + * @sym_to_idx: same as @sym_from_idx but for @sym_to + * @addr: filter address + * @size: filter region size (for range filters) + * @filename: DSO file name or NULL for the kernel + * @str: allocated string that contains the other string members + */ +struct addr_filter { + struct list_head list; + bool range; + bool start; + const char *action; + const char *sym_from; + const char *sym_to; + int sym_from_idx; + int sym_to_idx; + u64 addr; + u64 size; + const char *filename; + char *str; +}; + +/** + * struct addr_filters - list of address filters. + * @head: list of address filters + * @cnt: number of address filters + */ +struct addr_filters { + struct list_head head; + int cnt; +}; + +struct auxtrace_cache; + +u64 compat_auxtrace_mmap__read_head(struct auxtrace_mmap *mm); +int compat_auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail); + +static inline u64 auxtrace_mmap__read_head(struct auxtrace_mmap *mm, + int kernel_is_64_bit __maybe_unused) +{ + struct perf_event_mmap_page *pc = mm->userpg; + u64 head; + +#if BITS_PER_LONG == 32 + if (kernel_is_64_bit) + return compat_auxtrace_mmap__read_head(mm); +#endif + head = READ_ONCE(pc->aux_head); + + /* Ensure all reads are done after we read the head */ + smp_rmb(); + return head; +} + +static inline int auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail, + int kernel_is_64_bit __maybe_unused) +{ + struct perf_event_mmap_page *pc = mm->userpg; + +#if BITS_PER_LONG == 32 + if (kernel_is_64_bit) + return compat_auxtrace_mmap__write_tail(mm, tail); +#endif + /* Ensure all reads are done before we write the tail out */ + smp_mb(); + WRITE_ONCE(pc->aux_tail, tail); + return 0; +} + +int auxtrace_mmap__mmap(struct auxtrace_mmap *mm, + struct auxtrace_mmap_params *mp, + void *userpg, int fd); +void auxtrace_mmap__munmap(struct auxtrace_mmap *mm); +void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp, + off_t auxtrace_offset, + unsigned int auxtrace_pages, + bool auxtrace_overwrite); +void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, + struct evlist *evlist, + struct evsel *evsel, int idx); + +typedef int (*process_auxtrace_t)(const struct perf_tool *tool, + struct mmap *map, + union perf_event *event, void *data1, + size_t len1, void *data2, size_t len2); + +int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr, + struct perf_env *env, const struct perf_tool *tool, + process_auxtrace_t fn); + +int auxtrace_mmap__read_snapshot(struct mmap *map, + struct auxtrace_record *itr, struct perf_env *env, + const struct perf_tool *tool, process_auxtrace_t fn, + size_t snapshot_size); + +int auxtrace_queues__init_nr(struct auxtrace_queues *queues, int nr_queues); +int auxtrace_queues__init(struct auxtrace_queues *queues); +int auxtrace_queues__add_event(struct auxtrace_queues *queues, + struct perf_session *session, + union perf_event *event, off_t data_offset, + struct auxtrace_buffer **buffer_ptr); +struct auxtrace_queue * +auxtrace_queues__sample_queue(struct auxtrace_queues *queues, + struct perf_sample *sample, + struct perf_session *session); +int auxtrace_queues__add_sample(struct auxtrace_queues *queues, + struct perf_session *session, + struct perf_sample *sample, u64 data_offset, + u64 reference); +void auxtrace_queues__free(struct auxtrace_queues *queues); +int auxtrace_queues__process_index(struct auxtrace_queues *queues, + struct perf_session *session); +int auxtrace_queue_data(struct perf_session *session, bool samples, + bool events); +struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue, + struct auxtrace_buffer *buffer); +void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw); +static inline void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd) +{ + return auxtrace_buffer__get_data_rw(buffer, fd, false); +} +void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer); +void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer); +void auxtrace_buffer__free(struct auxtrace_buffer *buffer); + +int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr, + u64 ordinal); +void auxtrace_heap__pop(struct auxtrace_heap *heap); +void auxtrace_heap__free(struct auxtrace_heap *heap); + +struct auxtrace_cache_entry { + struct hlist_node hash; + u32 key; +}; + +struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size, + unsigned int limit_percent); +void auxtrace_cache__free(struct auxtrace_cache *auxtrace_cache); +void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c); +void auxtrace_cache__free_entry(struct auxtrace_cache *c, void *entry); +int auxtrace_cache__add(struct auxtrace_cache *c, u32 key, + struct auxtrace_cache_entry *entry); +void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key); +void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key); + +struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, + int *err); + +int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, + struct record_opts *opts, + const char *str); +int auxtrace_parse_sample_options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts, const char *str); +int auxtrace_parse_aux_action(struct evlist *evlist); +int auxtrace_record__options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts); +size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr, + struct evlist *evlist); +int auxtrace_record__info_fill(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size); +void auxtrace_record__free(struct auxtrace_record *itr); +int auxtrace_record__snapshot_start(struct auxtrace_record *itr); +int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit); +int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx, + struct auxtrace_mmap *mm, + unsigned char *data, u64 *head, u64 *old); +u64 auxtrace_record__reference(struct auxtrace_record *itr); +int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx); + +int auxtrace_index__auxtrace_event(struct list_head *head, union perf_event *event, + off_t file_offset); +int auxtrace_index__write(int fd, struct list_head *head); +int auxtrace_index__process(int fd, u64 size, struct perf_session *session, + bool needs_swap); +void auxtrace_index__free(struct list_head *head); + +void auxtrace_synth_guest_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp, + pid_t machine_pid, int vcpu); +void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp); + +int perf_event__process_auxtrace_info(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +s64 perf_event__process_auxtrace(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +int perf_event__process_auxtrace_error(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, + const char *str, int unset); +int itrace_parse_synth_opts(const struct option *opt, const char *str, + int unset); +void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, + bool no_sample); + +size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp); +void perf_session__auxtrace_error_inc(struct perf_session *session, + union perf_event *event); +void events_stats__auxtrace_error_warn(const struct events_stats *stats); + +void addr_filters__init(struct addr_filters *filts); +void addr_filters__exit(struct addr_filters *filts); +int addr_filters__parse_bare_filter(struct addr_filters *filts, + const char *filter); +int auxtrace_parse_filters(struct evlist *evlist); + +int auxtrace__process_event(struct perf_session *session, union perf_event *event, + struct perf_sample *sample, const struct perf_tool *tool); +void auxtrace__dump_auxtrace_sample(struct perf_session *session, + struct perf_sample *sample); +int auxtrace__flush_events(struct perf_session *session, const struct perf_tool *tool); +void auxtrace__free_events(struct perf_session *session); +void auxtrace__free(struct perf_session *session); +bool auxtrace__evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel); +u64 auxtrace_synth_id_range_start(struct evsel *evsel); + +#define ITRACE_HELP \ +" i[period]: synthesize instructions events\n" \ +" y[period]: synthesize cycles events (same period as i)\n" \ +" b: synthesize branches events\n" \ +" c: synthesize branches events (calls only)\n" \ +" r: synthesize branches events (returns only)\n" \ +" x: synthesize transactions events\n" \ +" w: synthesize ptwrite events\n" \ +" p: synthesize power events\n" \ +" o: synthesize other events recorded due to the use\n" \ +" of aux-output (refer to perf record)\n" \ +" I: synthesize interrupt or similar (asynchronous) events\n" \ +" (e.g. Intel PT Event Trace)\n" \ +" e[flags]: synthesize error events\n" \ +" each flag must be preceded by + or -\n" \ +" error flags are: o (overflow)\n" \ +" l (data lost)\n" \ +" d[flags]: create a debug log\n" \ +" each flag must be preceded by + or -\n" \ +" log flags are: a (all perf events)\n" \ +" o (output to stdout)\n" \ +" f: synthesize first level cache events\n" \ +" m: synthesize last level cache events\n" \ +" t: synthesize TLB events\n" \ +" a: synthesize remote access events\n" \ +" g[len]: synthesize a call chain (use with i or x)\n" \ +" G[len]: synthesize a call chain on existing event records\n" \ +" l[len]: synthesize last branch entries (use with i or x)\n" \ +" L[len]: synthesize last branch entries on existing event records\n" \ +" sNUMBER: skip initial number of events\n" \ +" q: quicker (less detailed) decoding\n" \ +" A: approximate IPC\n" \ +" Z: prefer to ignore timestamps (so-called \"timeless\" decoding)\n" \ +" T: use the timestamp trace as kernel time\n" \ +" PERIOD[ns|us|ms|i|t]: specify period to sample stream\n" \ +" concatenate multiple options. Default is iybxwpe or cewp\n" + +static inline +void itrace_synth_opts__set_time_range(struct itrace_synth_opts *opts, + struct perf_time_interval *ptime_range, + int range_num) +{ + opts->ptime_range = ptime_range; + opts->range_num = range_num; +} + +static inline +void itrace_synth_opts__clear_time_range(struct itrace_synth_opts *opts) +{ + opts->ptime_range = NULL; + opts->range_num = 0; +} + +#endif diff --git a/tools/perf/util/blake2s.c b/tools/perf/util/blake2s.c new file mode 100644 index 000000000..ce5d89a19 --- /dev/null +++ b/tools/perf/util/blake2s.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + */ + +#include "blake2s.h" +#include + +static inline u32 ror32(u32 v, int n) +{ + return (v >> n) | (v << (32 - n)); +} + +static inline void le32_to_cpu_array(u32 a[], size_t n) +{ + for (size_t i = 0; i < n; i++) + a[i] = le32_to_cpu((__force __le32)a[i]); +} + +static inline void cpu_to_le32_array(u32 a[], size_t n) +{ + for (size_t i = 0; i < n; i++) + a[i] = (__force u32)cpu_to_le32(a[i]); +} + +static const u8 blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline void blake2s_increment_counter(struct blake2s_ctx *ctx, u32 inc) +{ + ctx->t[0] += inc; + ctx->t[1] += (ctx->t[0] < inc); +} + +static void blake2s_compress(struct blake2s_ctx *ctx, + const u8 *data, size_t nblocks, u32 inc) +{ + u32 m[16]; + u32 v[16]; + int i; + + while (nblocks > 0) { + blake2s_increment_counter(ctx, inc); + memcpy(m, data, BLAKE2S_BLOCK_SIZE); + le32_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, ctx->h, 32); + v[ 8] = BLAKE2S_IV0; + v[ 9] = BLAKE2S_IV1; + v[10] = BLAKE2S_IV2; + v[11] = BLAKE2S_IV3; + v[12] = BLAKE2S_IV4 ^ ctx->t[0]; + v[13] = BLAKE2S_IV5 ^ ctx->t[1]; + v[14] = BLAKE2S_IV6 ^ ctx->f[0]; + v[15] = BLAKE2S_IV7 ^ ctx->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ror32(d ^ a, 16); \ + c += d; \ + b = ror32(b ^ c, 12); \ + a += b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ror32(d ^ a, 8); \ + c += d; \ + b = ror32(b ^ c, 7); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + ctx->h[i] ^= v[i] ^ v[i + 8]; + + data += BLAKE2S_BLOCK_SIZE; + --nblocks; + } +} + +static inline void blake2s_set_lastblock(struct blake2s_ctx *ctx) +{ + ctx->f[0] = -1; +} + +void blake2s_update(struct blake2s_ctx *ctx, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2S_BLOCK_SIZE - ctx->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(ctx->buf + ctx->buflen, in, fill); + blake2s_compress(ctx, ctx->buf, 1, BLAKE2S_BLOCK_SIZE); + ctx->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + + blake2s_compress(ctx, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(ctx->buf + ctx->buflen, in, inlen); + ctx->buflen += inlen; +} + +void blake2s_final(struct blake2s_ctx *ctx, u8 *out) +{ + blake2s_set_lastblock(ctx); + memset(ctx->buf + ctx->buflen, 0, + BLAKE2S_BLOCK_SIZE - ctx->buflen); /* Padding */ + blake2s_compress(ctx, ctx->buf, 1, ctx->buflen); + cpu_to_le32_array(ctx->h, ARRAY_SIZE(ctx->h)); + memcpy(out, ctx->h, ctx->outlen); + memset(ctx, 0, sizeof(*ctx)); +} diff --git a/tools/perf/util/blake2s.h b/tools/perf/util/blake2s.h new file mode 100644 index 000000000..a1fe81a4b --- /dev/null +++ b/tools/perf/util/blake2s.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _CRYPTO_BLAKE2S_H +#define _CRYPTO_BLAKE2S_H + +#include +#include + +#define BLAKE2S_BLOCK_SIZE 64 + +struct blake2s_ctx { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[BLAKE2S_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + +enum blake2s_iv { + BLAKE2S_IV0 = 0x6A09E667UL, + BLAKE2S_IV1 = 0xBB67AE85UL, + BLAKE2S_IV2 = 0x3C6EF372UL, + BLAKE2S_IV3 = 0xA54FF53AUL, + BLAKE2S_IV4 = 0x510E527FUL, + BLAKE2S_IV5 = 0x9B05688CUL, + BLAKE2S_IV6 = 0x1F83D9ABUL, + BLAKE2S_IV7 = 0x5BE0CD19UL, +}; + +static inline void __blake2s_init(struct blake2s_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + ctx->h[0] = BLAKE2S_IV0 ^ (0x01010000 | keylen << 8 | outlen); + ctx->h[1] = BLAKE2S_IV1; + ctx->h[2] = BLAKE2S_IV2; + ctx->h[3] = BLAKE2S_IV3; + ctx->h[4] = BLAKE2S_IV4; + ctx->h[5] = BLAKE2S_IV5; + ctx->h[6] = BLAKE2S_IV6; + ctx->h[7] = BLAKE2S_IV7; + ctx->t[0] = 0; + ctx->t[1] = 0; + ctx->f[0] = 0; + ctx->f[1] = 0; + ctx->buflen = 0; + ctx->outlen = outlen; + if (keylen) { + memcpy(ctx->buf, key, keylen); + memset(&ctx->buf[keylen], 0, BLAKE2S_BLOCK_SIZE - keylen); + ctx->buflen = BLAKE2S_BLOCK_SIZE; + } +} + +static inline void blake2s_init(struct blake2s_ctx *ctx, size_t outlen) +{ + __blake2s_init(ctx, outlen, NULL, 0); +} + +static inline void blake2s_init_key(struct blake2s_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + __blake2s_init(ctx, outlen, key, keylen); +} + +void blake2s_update(struct blake2s_ctx *ctx, const u8 *in, size_t inlen); + +void blake2s_final(struct blake2s_ctx *ctx, u8 *out); + +#endif /* _CRYPTO_BLAKE2S_H */ diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c new file mode 100644 index 000000000..1135e54f4 --- /dev/null +++ b/tools/perf/util/block-info.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "block-info.h" +#include "sort.h" +#include "annotate.h" +#include "symbol.h" +#include "dso.h" +#include "map.h" +#include "srcline.h" +#include "evlist.h" +#include "hist.h" +#include "ui/browsers/hists.h" + +static struct block_header_column { + const char *name; + int width; +} block_columns[PERF_HPP_REPORT__BLOCK_MAX_INDEX] = { + [PERF_HPP_REPORT__BLOCK_TOTAL_CYCLES_PCT] = { + .name = "Sampled Cycles%", + .width = 15, + }, + [PERF_HPP_REPORT__BLOCK_LBR_CYCLES] = { + .name = "Sampled Cycles", + .width = 14, + }, + [PERF_HPP_REPORT__BLOCK_CYCLES_PCT] = { + .name = "Avg Cycles%", + .width = 11, + }, + [PERF_HPP_REPORT__BLOCK_AVG_CYCLES] = { + .name = "Avg Cycles", + .width = 10, + }, + [PERF_HPP_REPORT__BLOCK_RANGE] = { + .name = "[Program Block Range]", + .width = 70, + }, + [PERF_HPP_REPORT__BLOCK_DSO] = { + .name = "Shared Object", + .width = 20, + }, + [PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER] = { + .name = "Branch Counter", + .width = 30, + } +}; + +static struct block_info *block_info__new(unsigned int br_cntr_nr) +{ + struct block_info *bi = zalloc(sizeof(struct block_info)); + + if (bi && br_cntr_nr) { + bi->br_cntr = calloc(br_cntr_nr, sizeof(u64)); + if (!bi->br_cntr) { + free(bi); + return NULL; + } + } + + return bi; +} + +void block_info__delete(struct block_info *bi) +{ + if (bi) + free(bi->br_cntr); + free(bi); +} + +int64_t __block_info__cmp(struct hist_entry *left, struct hist_entry *right) +{ + struct block_info *bi_l = left->block_info; + struct block_info *bi_r = right->block_info; + int cmp; + + if (!bi_l->sym || !bi_r->sym) { + if (!bi_l->sym && !bi_r->sym) + return -1; + else if (!bi_l->sym) + return -1; + else + return 1; + } + + cmp = strcmp(bi_l->sym->name, bi_r->sym->name); + if (cmp) + return cmp; + + if (bi_l->start != bi_r->start) + return (int64_t)(bi_r->start - bi_l->start); + + return (int64_t)(bi_r->end - bi_l->end); +} + +int64_t block_info__cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + return __block_info__cmp(left, right); +} + +static void init_block_info(struct block_info *bi, struct symbol *sym, + struct cyc_hist *ch, int offset, + u64 total_cycles, unsigned int br_cntr_nr, + u64 *br_cntr, struct evsel *evsel) +{ + bi->sym = sym; + bi->start = ch->start; + bi->end = offset; + bi->cycles = ch->cycles; + bi->cycles_aggr = ch->cycles_aggr; + bi->num = ch->num; + bi->num_aggr = ch->num_aggr; + bi->total_cycles = total_cycles; + + memcpy(bi->cycles_spark, ch->cycles_spark, + NUM_SPARKS * sizeof(u64)); + + if (br_cntr && br_cntr_nr) { + bi->br_cntr_nr = br_cntr_nr; + memcpy(bi->br_cntr, &br_cntr[offset * br_cntr_nr], + br_cntr_nr * sizeof(u64)); + } + bi->evsel = evsel; +} + +int block_info__process_sym(struct hist_entry *he, struct block_hist *bh, + u64 *block_cycles_aggr, u64 total_cycles, + unsigned int br_cntr_nr) +{ + struct annotation *notes; + struct cyc_hist *ch; + static struct addr_location al; + u64 cycles = 0; + + if (!he->ms.map || !he->ms.sym) + return 0; + + memset(&al, 0, sizeof(al)); + al.map = he->ms.map; + al.sym = he->ms.sym; + + notes = symbol__annotation(he->ms.sym); + if (!notes || !notes->branch || !notes->branch->cycles_hist) + return 0; + ch = notes->branch->cycles_hist; + for (unsigned int i = 0; i < symbol__size(he->ms.sym); i++) { + if (ch[i].num_aggr) { + struct block_info *bi; + struct hist_entry *he_block; + + bi = block_info__new(br_cntr_nr); + if (!bi) + return -1; + + init_block_info(bi, he->ms.sym, &ch[i], i, + total_cycles, br_cntr_nr, + notes->branch->br_cntr, + hists_to_evsel(he->hists)); + cycles += bi->cycles_aggr / bi->num_aggr; + + he_block = hists__add_entry_block(&bh->block_hists, + &al, bi); + if (!he_block) { + block_info__delete(bi); + return -1; + } + } + } + + if (block_cycles_aggr) + *block_cycles_aggr += cycles; + + return 0; +} + +static int block_column_header(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hists *hists __maybe_unused, + int line __maybe_unused, + int *span __maybe_unused) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + block_fmt->header); +} + +static int block_column_width(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp __maybe_unused, + struct hists *hists __maybe_unused) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + + return block_fmt->width; +} + +static int color_pct(struct perf_hpp *hpp, int width, double pct) +{ +#ifdef HAVE_SLANG_SUPPORT + if (use_browser) { + return __hpp__slsmg_color_printf(hpp, "%*.2f%%", + width - 1, pct); + } +#endif + return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, pct); +} + +static int block_total_cycles_pct_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + double ratio = 0.0; + + if (block_fmt->total_cycles) + ratio = (double)bi->cycles_aggr / (double)block_fmt->total_cycles; + + return color_pct(hpp, block_fmt->width, 100.0 * ratio); +} + +static int64_t block_total_cycles_pct_sort(struct perf_hpp_fmt *fmt, + struct hist_entry *left, + struct hist_entry *right) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi_l = left->block_info; + struct block_info *bi_r = right->block_info; + double l, r; + + if (block_fmt->total_cycles) { + l = ((double)bi_l->cycles_aggr / + (double)block_fmt->total_cycles) * 100000.0; + r = ((double)bi_r->cycles_aggr / + (double)block_fmt->total_cycles) * 100000.0; + return (int64_t)l - (int64_t)r; + } + + return 0; +} + +static void cycles_string(u64 cycles, char *buf, int size) +{ + if (cycles >= 1000000) + scnprintf(buf, size, "%.1fM", (double)cycles / 1000000.0); + else if (cycles >= 1000) + scnprintf(buf, size, "%.1fK", (double)cycles / 1000.0); + else + scnprintf(buf, size, "%1d", cycles); +} + +static int block_cycles_lbr_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char cycles_buf[16]; + + cycles_string(bi->cycles_aggr, cycles_buf, sizeof(cycles_buf)); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + cycles_buf); +} + +static int block_cycles_pct_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + double ratio = 0.0; + u64 avg; + + if (block_fmt->block_cycles && bi->num_aggr) { + avg = bi->cycles_aggr / bi->num_aggr; + ratio = (double)avg / (double)block_fmt->block_cycles; + } + + return color_pct(hpp, block_fmt->width, 100.0 * ratio); +} + +static int block_avg_cycles_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char cycles_buf[16]; + + cycles_string(bi->cycles_aggr / bi->num_aggr, cycles_buf, + sizeof(cycles_buf)); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + cycles_buf); +} + +static int block_range_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char buf[128]; + char *start_line, *end_line; + + symbol_conf.addr2line_disable_warn = true; + + start_line = map__srcline(he->ms.map, bi->sym->start + bi->start, + he->ms.sym); + + end_line = map__srcline(he->ms.map, bi->sym->start + bi->end, + he->ms.sym); + + if (start_line != SRCLINE_UNKNOWN && + end_line != SRCLINE_UNKNOWN) { + scnprintf(buf, sizeof(buf), "[%s -> %s]", + start_line, end_line); + } else { + scnprintf(buf, sizeof(buf), "[%7lx -> %7lx]", + bi->start, bi->end); + } + + zfree_srcline(&start_line); + zfree_srcline(&end_line); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, buf); +} + +static int block_dso_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct map *map = he->ms.map; + + if (map && map__dso(map)) { + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + dso__short_name(map__dso(map))); + } + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + "[unknown]"); +} + +static void init_block_header(struct block_fmt *block_fmt) +{ + struct perf_hpp_fmt *fmt = &block_fmt->fmt; + + BUG_ON(block_fmt->idx >= PERF_HPP_REPORT__BLOCK_MAX_INDEX); + + block_fmt->header = block_columns[block_fmt->idx].name; + block_fmt->width = block_columns[block_fmt->idx].width; + + fmt->header = block_column_header; + fmt->width = block_column_width; +} + +static int block_branch_counter_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char *buf; + int ret; + + if (annotation_br_cntr_entry(&buf, bi->br_cntr_nr, bi->br_cntr, + bi->num_aggr, bi->evsel)) + return 0; + + ret = scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, buf); + free(buf); + return ret; +} + +static void hpp_register(struct block_fmt *block_fmt, int idx, + struct perf_hpp_list *hpp_list) +{ + struct perf_hpp_fmt *fmt = &block_fmt->fmt; + + block_fmt->idx = idx; + INIT_LIST_HEAD(&fmt->list); + INIT_LIST_HEAD(&fmt->sort_list); + + switch (idx) { + case PERF_HPP_REPORT__BLOCK_TOTAL_CYCLES_PCT: + fmt->color = block_total_cycles_pct_entry; + fmt->cmp = block_info__cmp; + fmt->sort = block_total_cycles_pct_sort; + break; + case PERF_HPP_REPORT__BLOCK_LBR_CYCLES: + fmt->entry = block_cycles_lbr_entry; + break; + case PERF_HPP_REPORT__BLOCK_CYCLES_PCT: + fmt->color = block_cycles_pct_entry; + break; + case PERF_HPP_REPORT__BLOCK_AVG_CYCLES: + fmt->entry = block_avg_cycles_entry; + break; + case PERF_HPP_REPORT__BLOCK_RANGE: + fmt->entry = block_range_entry; + break; + case PERF_HPP_REPORT__BLOCK_DSO: + fmt->entry = block_dso_entry; + break; + case PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER: + fmt->entry = block_branch_counter_entry; + break; + default: + return; + } + + init_block_header(block_fmt); + perf_hpp_list__column_register(hpp_list, fmt); +} + +static void register_block_columns(struct perf_hpp_list *hpp_list, + struct block_fmt *block_fmts, + int *block_hpps, int nr_hpps) +{ + for (int i = 0; i < nr_hpps; i++) + hpp_register(&block_fmts[i], block_hpps[i], hpp_list); +} + +static void init_block_hist(struct block_hist *bh, struct block_fmt *block_fmts, + int *block_hpps, int nr_hpps) +{ + __hists__init(&bh->block_hists, &bh->block_list); + perf_hpp_list__init(&bh->block_list); + bh->block_list.nr_header_lines = 1; + + register_block_columns(&bh->block_list, block_fmts, + block_hpps, nr_hpps); + + /* Sort by the first fmt */ + perf_hpp_list__register_sort_field(&bh->block_list, &block_fmts[0].fmt); +} + +static int process_block_report(struct hists *hists, + struct block_report *block_report, + u64 total_cycles, int *block_hpps, + int nr_hpps, unsigned int br_cntr_nr) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct block_hist *bh = &block_report->hist; + struct hist_entry *he; + + if (nr_hpps > PERF_HPP_REPORT__BLOCK_MAX_INDEX) + return -1; + + block_report->nr_fmts = nr_hpps; + init_block_hist(bh, block_report->fmts, block_hpps, nr_hpps); + + while (next) { + he = rb_entry(next, struct hist_entry, rb_node); + block_info__process_sym(he, bh, &block_report->cycles, + total_cycles, br_cntr_nr); + next = rb_next(&he->rb_node); + } + + for (int i = 0; i < nr_hpps; i++) { + block_report->fmts[i].total_cycles = total_cycles; + block_report->fmts[i].block_cycles = block_report->cycles; + } + + hists__output_resort(&bh->block_hists, NULL); + return 0; +} + +struct block_report *block_info__create_report(struct evlist *evlist, + u64 total_cycles, + int *block_hpps, int nr_hpps, + int *nr_reps) +{ + struct block_report *block_reports; + int nr_hists = evlist__nr_entries(evlist), i = 0; + struct evsel *pos; + + block_reports = calloc(nr_hists, sizeof(struct block_report)); + if (!block_reports) + return NULL; + + evlist__for_each_entry(evlist, pos) { + struct hists *hists = evsel__hists(pos); + + process_block_report(hists, &block_reports[i], total_cycles, + block_hpps, nr_hpps, evlist__nr_br_cntr(evlist)); + i++; + } + + *nr_reps = nr_hists; + return block_reports; +} + +void block_info__free_report(struct block_report *reps, int nr_reps) +{ + for (int i = 0; i < nr_reps; i++) + hists__delete_entries(&reps[i].hist.block_hists); + + free(reps); +} + +int report__browse_block_hists(struct block_hist *bh, float min_percent, + struct evsel *evsel, struct perf_env *env) +{ + int ret; + + switch (use_browser) { + case 0: + symbol_conf.report_individual_block = true; + hists__fprintf(&bh->block_hists, true, 0, 0, min_percent, + stdout, true); + return 0; + case 1: + symbol_conf.report_individual_block = true; + ret = block_hists_tui_browse(bh, evsel, min_percent, env); + return ret; + default: + return -1; + } + + return 0; +} + +float block_info__total_cycles_percent(struct hist_entry *he) +{ + struct block_info *bi = he->block_info; + + if (bi->total_cycles) + return bi->cycles * 100.0 / bi->total_cycles; + + return 0.0; +} diff --git a/tools/perf/util/block-info.h b/tools/perf/util/block-info.h new file mode 100644 index 000000000..b9329dc3a --- /dev/null +++ b/tools/perf/util/block-info.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BLOCK_H +#define __PERF_BLOCK_H + +#include +#include "hist.h" +#include "symbol.h" +#include "sort.h" +#include "ui/ui.h" + +struct block_info { + struct symbol *sym; + u64 start; + u64 end; + u64 cycles; + u64 cycles_aggr; + s64 cycles_spark[NUM_SPARKS]; + u64 total_cycles; + int num; + int num_aggr; + int br_cntr_nr; + u64 *br_cntr; + struct evsel *evsel; +}; + +struct block_fmt { + struct perf_hpp_fmt fmt; + int idx; + int width; + const char *header; + u64 total_cycles; + u64 block_cycles; +}; + +enum { + PERF_HPP_REPORT__BLOCK_TOTAL_CYCLES_PCT, + PERF_HPP_REPORT__BLOCK_LBR_CYCLES, + PERF_HPP_REPORT__BLOCK_CYCLES_PCT, + PERF_HPP_REPORT__BLOCK_AVG_CYCLES, + PERF_HPP_REPORT__BLOCK_RANGE, + PERF_HPP_REPORT__BLOCK_DSO, + PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER, + PERF_HPP_REPORT__BLOCK_MAX_INDEX +}; + +struct block_report { + struct block_hist hist; + u64 cycles; + struct block_fmt fmts[PERF_HPP_REPORT__BLOCK_MAX_INDEX]; + int nr_fmts; +}; + +void block_info__delete(struct block_info *bi); + +int64_t __block_info__cmp(struct hist_entry *left, struct hist_entry *right); + +int64_t block_info__cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right); + +int block_info__process_sym(struct hist_entry *he, struct block_hist *bh, + u64 *block_cycles_aggr, u64 total_cycles, + unsigned int br_cntr_nr); + +struct block_report *block_info__create_report(struct evlist *evlist, + u64 total_cycles, + int *block_hpps, int nr_hpps, + int *nr_reps); + +void block_info__free_report(struct block_report *reps, int nr_reps); + +int report__browse_block_hists(struct block_hist *bh, float min_percent, + struct evsel *evsel, struct perf_env *env); + +float block_info__total_cycles_percent(struct hist_entry *he); + +#endif /* __PERF_BLOCK_H */ diff --git a/tools/perf/util/block-range.c b/tools/perf/util/block-range.c new file mode 100644 index 000000000..7c559fcfd --- /dev/null +++ b/tools/perf/util/block-range.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "block-range.h" +#include "annotate.h" +#include +#include + +static struct { + struct rb_root root; + u64 blocks; +} block_ranges; + +static void block_range__debug(void) +{ +#ifndef NDEBUG + struct rb_node *rb; + u64 old = 0; /* NULL isn't executable */ + + for (rb = rb_first(&block_ranges.root); rb; rb = rb_next(rb)) { + struct block_range *entry = rb_entry(rb, struct block_range, node); + + assert(old < entry->start); + assert(entry->start <= entry->end); /* single instruction block; jump to a jump */ + + old = entry->end; + } +#endif +} + +struct block_range *block_range__find(u64 addr) +{ + struct rb_node **p = &block_ranges.root.rb_node; + struct rb_node *parent = NULL; + struct block_range *entry; + + while (*p != NULL) { + parent = *p; + entry = rb_entry(parent, struct block_range, node); + + if (addr < entry->start) + p = &parent->rb_left; + else if (addr > entry->end) + p = &parent->rb_right; + else + return entry; + } + + return NULL; +} + +static inline void rb_link_left_of_node(struct rb_node *left, struct rb_node *node) +{ + struct rb_node **p = &node->rb_left; + while (*p) { + node = *p; + p = &node->rb_right; + } + rb_link_node(left, node, p); +} + +static inline void rb_link_right_of_node(struct rb_node *right, struct rb_node *node) +{ + struct rb_node **p = &node->rb_right; + while (*p) { + node = *p; + p = &node->rb_left; + } + rb_link_node(right, node, p); +} + +/** + * block_range__create + * @start: branch target starting this basic block + * @end: branch ending this basic block + * + * Create all the required block ranges to precisely span the given range. + */ +struct block_range_iter block_range__create(u64 start, u64 end) +{ + struct rb_node **p = &block_ranges.root.rb_node; + struct rb_node *n, *parent = NULL; + struct block_range *next, *entry = NULL; + struct block_range_iter iter = { NULL, NULL }; + + while (*p != NULL) { + parent = *p; + entry = rb_entry(parent, struct block_range, node); + + if (start < entry->start) + p = &parent->rb_left; + else if (start > entry->end) + p = &parent->rb_right; + else + break; + } + + /* + * Didn't find anything.. there's a hole at @start, however @end might + * be inside/behind the next range. + */ + if (!*p) { + if (!entry) /* tree empty */ + goto do_whole; + + /* + * If the last node is before, advance one to find the next. + */ + n = parent; + if (entry->end < start) { + n = rb_next(n); + if (!n) + goto do_whole; + } + next = rb_entry(n, struct block_range, node); + + if (next->start <= end) { /* add head: [start...][n->start...] */ + struct block_range *head = malloc(sizeof(struct block_range)); + if (!head) + return iter; + + *head = (struct block_range){ + .start = start, + .end = next->start - 1, + .is_target = 1, + .is_branch = 0, + }; + + rb_link_left_of_node(&head->node, &next->node); + rb_insert_color(&head->node, &block_ranges.root); + block_range__debug(); + + iter.start = head; + goto do_tail; + } + +do_whole: + /* + * The whole [start..end] range is non-overlapping. + */ + entry = malloc(sizeof(struct block_range)); + if (!entry) + return iter; + + *entry = (struct block_range){ + .start = start, + .end = end, + .is_target = 1, + .is_branch = 1, + }; + + rb_link_node(&entry->node, parent, p); + rb_insert_color(&entry->node, &block_ranges.root); + block_range__debug(); + + iter.start = entry; + iter.end = entry; + goto done; + } + + /* + * We found a range that overlapped with ours, split if needed. + */ + if (entry->start < start) { /* split: [e->start...][start...] */ + struct block_range *head = malloc(sizeof(struct block_range)); + if (!head) + return iter; + + *head = (struct block_range){ + .start = entry->start, + .end = start - 1, + .is_target = entry->is_target, + .is_branch = 0, + + .coverage = entry->coverage, + .entry = entry->entry, + }; + + entry->start = start; + entry->is_target = 1; + entry->entry = 0; + + rb_link_left_of_node(&head->node, &entry->node); + rb_insert_color(&head->node, &block_ranges.root); + block_range__debug(); + + } else if (entry->start == start) + entry->is_target = 1; + + iter.start = entry; + +do_tail: + /* + * At this point we've got: @iter.start = [@start...] but @end can still be + * inside or beyond it. + */ + entry = iter.start; + for (;;) { + /* + * If @end is inside @entry, split. + */ + if (end < entry->end) { /* split: [...end][...e->end] */ + struct block_range *tail = malloc(sizeof(struct block_range)); + if (!tail) + return iter; + + *tail = (struct block_range){ + .start = end + 1, + .end = entry->end, + .is_target = 0, + .is_branch = entry->is_branch, + + .coverage = entry->coverage, + .taken = entry->taken, + .pred = entry->pred, + }; + + entry->end = end; + entry->is_branch = 1; + entry->taken = 0; + entry->pred = 0; + + rb_link_right_of_node(&tail->node, &entry->node); + rb_insert_color(&tail->node, &block_ranges.root); + block_range__debug(); + + iter.end = entry; + goto done; + } + + /* + * If @end matches @entry, done + */ + if (end == entry->end) { + entry->is_branch = 1; + iter.end = entry; + goto done; + } + + next = block_range__next(entry); + if (!next) + goto add_tail; + + /* + * If @end is in beyond @entry but not inside @next, add tail. + */ + if (end < next->start) { /* add tail: [...e->end][...end] */ + struct block_range *tail; +add_tail: + tail = malloc(sizeof(struct block_range)); + if (!tail) + return iter; + + *tail = (struct block_range){ + .start = entry->end + 1, + .end = end, + .is_target = 0, + .is_branch = 1, + }; + + rb_link_right_of_node(&tail->node, &entry->node); + rb_insert_color(&tail->node, &block_ranges.root); + block_range__debug(); + + iter.end = tail; + goto done; + } + + /* + * If there is a hole between @entry and @next, fill it. + */ + if (entry->end + 1 != next->start) { + struct block_range *hole = malloc(sizeof(struct block_range)); + if (!hole) + return iter; + + *hole = (struct block_range){ + .start = entry->end + 1, + .end = next->start - 1, + .is_target = 0, + .is_branch = 0, + }; + + rb_link_left_of_node(&hole->node, &next->node); + rb_insert_color(&hole->node, &block_ranges.root); + block_range__debug(); + } + + entry = next; + } + +done: + assert(iter.start->start == start && iter.start->is_target); + assert(iter.end->end == end && iter.end->is_branch); + + block_ranges.blocks++; + + return iter; +} + + +/* + * Compute coverage as: + * + * br->coverage / br->sym->max_coverage + * + * This ensures each symbol has a 100% spot, to reflect that each symbol has a + * most covered section. + * + * Returns [0-1] for coverage and -1 if we had no data what so ever or the + * symbol does not exist. + */ +double block_range__coverage(struct block_range *br) +{ + struct symbol *sym; + struct annotated_branch *branch; + + if (!br) { + if (block_ranges.blocks) + return 0; + + return -1; + } + + sym = br->sym; + if (!sym) + return -1; + + branch = symbol__annotation(sym)->branch; + if (!branch) + return -1; + + return (double)br->coverage / branch->max_coverage; +} diff --git a/tools/perf/util/block-range.h b/tools/perf/util/block-range.h new file mode 100644 index 000000000..ec0fb534b --- /dev/null +++ b/tools/perf/util/block-range.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BLOCK_RANGE_H +#define __PERF_BLOCK_RANGE_H + +#include +#include +#include + +struct symbol; + +/* + * struct block_range - non-overlapping parts of basic blocks + * @node: treenode + * @start: inclusive start of range + * @end: inclusive end of range + * @is_target: @start is a jump target + * @is_branch: @end is a branch instruction + * @coverage: number of blocks that cover this range + * @taken: number of times the branch is taken (requires @is_branch) + * @pred: number of times the taken branch was predicted + */ +struct block_range { + struct rb_node node; + + struct symbol *sym; + + u64 start; + u64 end; + + int is_target, is_branch; + + u64 coverage; + u64 entry; + u64 taken; + u64 pred; +}; + +static inline struct block_range *block_range__next(struct block_range *br) +{ + struct rb_node *n = rb_next(&br->node); + if (!n) + return NULL; + return rb_entry(n, struct block_range, node); +} + +struct block_range_iter { + struct block_range *start; + struct block_range *end; +}; + +static inline struct block_range *block_range_iter(struct block_range_iter *iter) +{ + return iter->start; +} + +static inline bool block_range_iter__next(struct block_range_iter *iter) +{ + if (iter->start == iter->end) + return false; + + iter->start = block_range__next(iter->start); + return true; +} + +static inline bool block_range_iter__valid(struct block_range_iter *iter) +{ + if (!iter->start || !iter->end) + return false; + return true; +} + +extern struct block_range *block_range__find(u64 addr); +extern struct block_range_iter block_range__create(u64 start, u64 end); +extern double block_range__coverage(struct block_range *br); + +#endif /* __PERF_BLOCK_RANGE_H */ diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c new file mode 100644 index 000000000..e67f28a8e --- /dev/null +++ b/tools/perf/util/bpf-event.c @@ -0,0 +1,1017 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "bpf-event.h" +#include "bpf-utils.h" +#include "debug.h" +#include "dso.h" +#include "symbol.h" +#include "machine.h" +#include "env.h" +#include "session.h" +#include "map.h" +#include "evlist.h" +#include "record.h" +#include "util/synthetic-events.h" + +static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) +{ + int ret = 0; + size_t i; + + for (i = 0; i < len; i++) + ret += scnprintf(buf + ret, size - ret, "%02x", data[i]); + return ret; +} + +static int machine__process_bpf_event_load(struct machine *machine, + union perf_event *event, + struct perf_sample *sample __maybe_unused) +{ + struct bpf_prog_info_node *info_node; + struct perf_env *env = machine->env; + struct perf_bpil *info_linear; + int id = event->bpf.id; + unsigned int i; + + /* perf-record, no need to handle bpf-event */ + if (env == NULL) + return 0; + + info_node = perf_env__find_bpf_prog_info(env, id); + if (!info_node) + return 0; + info_linear = info_node->info_linear; + + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) + return 0; + + for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { + u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); + u64 addr = addrs[i]; + struct map *map = maps__find(machine__kernel_maps(machine), addr); + + if (map) { + struct dso *dso = map__dso(map); + + dso__set_binary_type(dso, DSO_BINARY_TYPE__BPF_PROG_INFO); + dso__bpf_prog(dso)->id = id; + dso__bpf_prog(dso)->sub_id = i; + dso__bpf_prog(dso)->env = env; + map__put(map); + } + } + return 0; +} + +int machine__process_bpf(struct machine *machine, union perf_event *event, + struct perf_sample *sample) +{ + if (dump_trace) + perf_event__fprintf_bpf(event, stdout); + + switch (event->bpf.type) { + case PERF_BPF_EVENT_PROG_LOAD: + return machine__process_bpf_event_load(machine, event, sample); + + case PERF_BPF_EVENT_PROG_UNLOAD: + /* + * Do not free bpf_prog_info and btf of the program here, + * as annotation still need them. They will be freed at + * the end of the session. + */ + break; + default: + pr_debug("unexpected bpf event type of %d\n", event->bpf.type); + break; + } + return 0; +} + +static int perf_env__fetch_btf(struct perf_env *env, + u32 btf_id, + struct btf *btf) +{ + struct btf_node *node; + u32 data_size; + const void *data; + + data = btf__raw_data(btf, &data_size); + + node = malloc(data_size + sizeof(struct btf_node)); + if (!node) + return -1; + + node->id = btf_id; + node->data_size = data_size; + memcpy(node->data, data, data_size); + + if (!perf_env__insert_btf(env, node)) { + /* Insertion failed because of a duplicate. */ + free(node); + return -1; + } + return 0; +} + +static int synthesize_bpf_prog_name(char *buf, int size, + struct bpf_prog_info *info, + struct btf *btf, + u32 sub_id) +{ + u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); + void *func_infos = (void *)(uintptr_t)(info->func_info); + u32 sub_prog_cnt = info->nr_jited_ksyms; + const struct bpf_func_info *finfo; + const char *short_name = NULL; + const struct btf_type *t; + int name_len; + + name_len = scnprintf(buf, size, "bpf_prog_"); + name_len += snprintf_hex(buf + name_len, size - name_len, + prog_tags[sub_id], BPF_TAG_SIZE); + if (btf && + info->func_info_rec_size >= sizeof(*finfo) && + sub_id < info->nr_func_info) { + finfo = func_infos + sub_id * info->func_info_rec_size; + t = btf__type_by_id(btf, finfo->type_id); + if (t) + short_name = btf__name_by_offset(btf, t->name_off); + } else if (sub_id == 0 && sub_prog_cnt == 1) { + /* no subprog */ + if (info->name[0]) + short_name = info->name; + } else + short_name = "F"; + if (short_name) { + name_len += scnprintf(buf + name_len, size - name_len, + "_%s", short_name); + } + return name_len; +} + +#ifdef HAVE_LIBBPF_STRINGS_SUPPORT + +#define BPF_METADATA_PREFIX "bpf_metadata_" +#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1) + +static bool name_has_bpf_metadata_prefix(const char **s) +{ + if (strncmp(*s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) != 0) + return false; + *s += BPF_METADATA_PREFIX_LEN; + return true; +} + +struct bpf_metadata_map { + struct btf *btf; + const struct btf_type *datasec; + void *rodata; + size_t rodata_size; + unsigned int num_vars; +}; + +static int bpf_metadata_read_map_data(__u32 map_id, struct bpf_metadata_map *map) +{ + int map_fd; + struct bpf_map_info map_info; + __u32 map_info_len; + int key; + struct btf *btf; + const struct btf_type *datasec; + struct btf_var_secinfo *vsi; + unsigned int vlen, vars; + void *rodata; + + map_fd = bpf_map_get_fd_by_id(map_id); + if (map_fd < 0) + return -1; + + memset(&map_info, 0, sizeof(map_info)); + map_info_len = sizeof(map_info); + if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len) < 0) + goto out_close; + + /* If it's not an .rodata map, don't bother. */ + if (map_info.type != BPF_MAP_TYPE_ARRAY || + map_info.key_size != sizeof(int) || + map_info.max_entries != 1 || + !map_info.btf_value_type_id || + !strstr(map_info.name, ".rodata")) { + goto out_close; + } + + btf = btf__load_from_kernel_by_id(map_info.btf_id); + if (!btf) + goto out_close; + datasec = btf__type_by_id(btf, map_info.btf_value_type_id); + if (!btf_is_datasec(datasec)) + goto out_free_btf; + + /* + * If there aren't any variables with the "bpf_metadata_" prefix, + * don't bother. + */ + vlen = btf_vlen(datasec); + vsi = btf_var_secinfos(datasec); + vars = 0; + for (unsigned int i = 0; i < vlen; i++, vsi++) { + const struct btf_type *t_var = btf__type_by_id(btf, vsi->type); + const char *name = btf__name_by_offset(btf, t_var->name_off); + + if (name_has_bpf_metadata_prefix(&name)) + vars++; + } + if (vars == 0) + goto out_free_btf; + + rodata = zalloc(map_info.value_size); + if (!rodata) + goto out_free_btf; + key = 0; + if (bpf_map_lookup_elem(map_fd, &key, rodata)) { + free(rodata); + goto out_free_btf; + } + close(map_fd); + + map->btf = btf; + map->datasec = datasec; + map->rodata = rodata; + map->rodata_size = map_info.value_size; + map->num_vars = vars; + return 0; + +out_free_btf: + btf__free(btf); +out_close: + close(map_fd); + return -1; +} + +struct format_btf_ctx { + char *buf; + size_t buf_size; + size_t buf_idx; +}; + +static void format_btf_cb(void *arg, const char *fmt, va_list ap) +{ + int n; + struct format_btf_ctx *ctx = (struct format_btf_ctx *)arg; + + n = vsnprintf(ctx->buf + ctx->buf_idx, ctx->buf_size - ctx->buf_idx, + fmt, ap); + ctx->buf_idx += n; + if (ctx->buf_idx >= ctx->buf_size) + ctx->buf_idx = ctx->buf_size; +} + +static void format_btf_variable(struct btf *btf, char *buf, size_t buf_size, + const struct btf_type *t, const void *btf_data) +{ + struct format_btf_ctx ctx = { + .buf = buf, + .buf_idx = 0, + .buf_size = buf_size, + }; + const struct btf_dump_type_data_opts opts = { + .sz = sizeof(struct btf_dump_type_data_opts), + .skip_names = 1, + .compact = 1, + .emit_strings = 1, + }; + struct btf_dump *d; + size_t btf_size; + + d = btf_dump__new(btf, format_btf_cb, &ctx, NULL); + btf_size = btf__resolve_size(btf, t->type); + btf_dump__dump_type_data(d, t->type, btf_data, btf_size, &opts); + btf_dump__free(d); +} + +static void bpf_metadata_fill_event(struct bpf_metadata_map *map, + struct perf_record_bpf_metadata *bpf_metadata_event) +{ + struct btf_var_secinfo *vsi; + unsigned int i, vlen; + + memset(bpf_metadata_event->prog_name, 0, BPF_PROG_NAME_LEN); + vlen = btf_vlen(map->datasec); + vsi = btf_var_secinfos(map->datasec); + + for (i = 0; i < vlen; i++, vsi++) { + const struct btf_type *t_var = btf__type_by_id(map->btf, + vsi->type); + const char *name = btf__name_by_offset(map->btf, + t_var->name_off); + const __u64 nr_entries = bpf_metadata_event->nr_entries; + struct perf_record_bpf_metadata_entry *entry; + + if (!name_has_bpf_metadata_prefix(&name)) + continue; + + if (nr_entries >= (__u64)map->num_vars) + break; + + entry = &bpf_metadata_event->entries[nr_entries]; + memset(entry, 0, sizeof(*entry)); + snprintf(entry->key, BPF_METADATA_KEY_LEN, "%s", name); + format_btf_variable(map->btf, entry->value, + BPF_METADATA_VALUE_LEN, t_var, + map->rodata + vsi->offset); + bpf_metadata_event->nr_entries++; + } +} + +static void bpf_metadata_free_map_data(struct bpf_metadata_map *map) +{ + btf__free(map->btf); + free(map->rodata); +} + +static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags, + __u32 nr_variables) +{ + struct bpf_metadata *metadata; + size_t event_size; + + metadata = zalloc(sizeof(struct bpf_metadata)); + if (!metadata) + return NULL; + + metadata->prog_names = calloc(nr_prog_tags, sizeof(char *)); + if (!metadata->prog_names) { + bpf_metadata_free(metadata); + return NULL; + } + for (__u32 prog_index = 0; prog_index < nr_prog_tags; prog_index++) { + metadata->prog_names[prog_index] = zalloc(BPF_PROG_NAME_LEN); + if (!metadata->prog_names[prog_index]) { + bpf_metadata_free(metadata); + return NULL; + } + metadata->nr_prog_names++; + } + + event_size = sizeof(metadata->event->bpf_metadata) + + nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]); + /* + * header.size is __u16. synthesize_perf_record_bpf_metadata() + * adds machine->id_hdr_size (up to ~64 bytes) after this, so + * leave headroom to prevent the final size from wrapping. + */ + if (event_size > UINT16_MAX - 256) { + bpf_metadata_free(metadata); + return NULL; + } + metadata->event = zalloc(event_size); + if (!metadata->event) { + bpf_metadata_free(metadata); + return NULL; + } + metadata->event->bpf_metadata = (struct perf_record_bpf_metadata) { + .header = { + .type = PERF_RECORD_BPF_METADATA, + .size = event_size, + }, + .nr_entries = 0, + }; + + return metadata; +} + +static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info) +{ + struct bpf_metadata *metadata; + const __u32 *map_ids = (__u32 *)(uintptr_t)info->map_ids; + + for (__u32 map_index = 0; map_index < info->nr_map_ids; map_index++) { + struct bpf_metadata_map map; + + if (bpf_metadata_read_map_data(map_ids[map_index], &map) != 0) + continue; + + metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars); + if (!metadata) { + bpf_metadata_free_map_data(&map); + continue; + } + + bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata); + + for (__u32 index = 0; index < info->nr_prog_tags; index++) { + synthesize_bpf_prog_name(metadata->prog_names[index], + BPF_PROG_NAME_LEN, info, + map.btf, index); + } + + bpf_metadata_free_map_data(&map); + + return metadata; + } + + return NULL; +} + +static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata, + const struct perf_tool *tool, + perf_event__handler_t process, + struct machine *machine) +{ + const size_t event_size = metadata->event->header.size; + union perf_event *event; + int err = 0; + + event = zalloc(event_size + machine->id_hdr_size); + if (!event) + return -1; + memcpy(event, metadata->event, event_size); + memset((void *)event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + for (__u32 index = 0; index < metadata->nr_prog_names; index++) { + memcpy(event->bpf_metadata.prog_name, + metadata->prog_names[index], BPF_PROG_NAME_LEN); + err = perf_tool__process_synth_event(tool, event, machine, + process); + if (err != 0) + break; + } + + free(event); + return err; +} + +void bpf_metadata_free(struct bpf_metadata *metadata) +{ + if (metadata == NULL) + return; + for (__u32 index = 0; index < metadata->nr_prog_names; index++) + free(metadata->prog_names[index]); + free(metadata->prog_names); + free(metadata->event); + free(metadata); +} + +#else /* HAVE_LIBBPF_STRINGS_SUPPORT */ + +static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info __maybe_unused) +{ + return NULL; +} + +static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata __maybe_unused, + const struct perf_tool *tool __maybe_unused, + perf_event__handler_t process __maybe_unused, + struct machine *machine __maybe_unused) +{ + return 0; +} + +void bpf_metadata_free(struct bpf_metadata *metadata __maybe_unused) +{ +} + +#endif /* HAVE_LIBBPF_STRINGS_SUPPORT */ + +struct bpf_metadata_final_ctx { + const struct perf_tool *tool; + perf_event__handler_t process; + struct machine *machine; +}; + +static void synthesize_final_bpf_metadata_cb(struct bpf_prog_info_node *node, + void *data) +{ + struct bpf_metadata_final_ctx *ctx = (struct bpf_metadata_final_ctx *)data; + struct bpf_metadata *metadata = node->metadata; + int err; + + if (metadata == NULL) + return; + err = synthesize_perf_record_bpf_metadata(metadata, ctx->tool, + ctx->process, ctx->machine); + if (err != 0) { + const char *prog_name = metadata->prog_names[0]; + + if (prog_name != NULL) + pr_warning("Couldn't synthesize final BPF metadata for %s.\n", prog_name); + else + pr_warning("Couldn't synthesize final BPF metadata.\n"); + } + bpf_metadata_free(metadata); + node->metadata = NULL; +} + +void perf_event__synthesize_final_bpf_metadata(struct perf_session *session, + perf_event__handler_t process) +{ + struct perf_env *env = &session->header.env; + struct bpf_metadata_final_ctx ctx = { + .tool = session->tool, + .process = process, + .machine = &session->machines.host, + }; + + perf_env__iterate_bpf_prog_info(env, synthesize_final_bpf_metadata_cb, + &ctx); +} + +/* + * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf + * program. One PERF_RECORD_BPF_EVENT is generated for the program. And + * one PERF_RECORD_KSYMBOL is generated for each sub program. + * + * Returns: + * 0 for success; + * -1 for failures; + * -2 for lack of kernel support. + */ +static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, + perf_event__handler_t process, + struct machine *machine, + int fd, + union perf_event *event, + struct record_opts *opts) +{ + struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; + struct perf_record_bpf_event *bpf_event = &event->bpf; + const struct perf_tool *tool = session->tool; + struct bpf_prog_info_node *info_node; + struct perf_bpil *info_linear; + struct bpf_metadata *metadata; + struct bpf_prog_info *info; + struct btf *btf = NULL; + struct perf_env *env; + u32 sub_prog_cnt, i; + int err = 0; + u64 arrays; + + /* + * for perf-record and perf-report use header.env; + * otherwise, use global perf_env. + */ + env = perf_session__env(session); + + arrays = 1UL << PERF_BPIL_JITED_KSYMS; + arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; + arrays |= 1UL << PERF_BPIL_FUNC_INFO; + arrays |= 1UL << PERF_BPIL_PROG_TAGS; + arrays |= 1UL << PERF_BPIL_JITED_INSNS; + arrays |= 1UL << PERF_BPIL_LINE_INFO; + arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; + arrays |= 1UL << PERF_BPIL_MAP_IDS; + + info_linear = get_bpf_prog_info_linear(fd, arrays); + if (IS_ERR_OR_NULL(info_linear)) { + info_linear = NULL; + pr_debug("%s: failed to get BPF program info. aborting\n", __func__); + return -1; + } + + if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { + free(info_linear); + pr_debug("%s: the kernel is too old, aborting\n", __func__); + return -2; + } + + info = &info_linear->info; + if (!info->jited_ksyms) { + free(info_linear); + return -1; + } + + /* number of ksyms, func_lengths, and tags should match */ + sub_prog_cnt = info->nr_jited_ksyms; + if (sub_prog_cnt != info->nr_prog_tags || + sub_prog_cnt != info->nr_jited_func_lens) { + free(info_linear); + return -1; + } + + /* check BTF func info support */ + if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { + /* btf func info number should be same as sub_prog_cnt */ + if (sub_prog_cnt != info->nr_func_info) { + pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); + free(info_linear); + return -1; + } + btf = btf__load_from_kernel_by_id(info->btf_id); + if (libbpf_get_error(btf)) { + pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); + err = -1; + goto out; + } + perf_env__fetch_btf(env, info->btf_id, btf); + } + + /* Synthesize PERF_RECORD_KSYMBOL */ + for (i = 0; i < sub_prog_cnt; i++) { + __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + int name_len; + + *ksymbol_event = (struct perf_record_ksymbol) { + .header = { + .type = PERF_RECORD_KSYMBOL, + .size = offsetof(struct perf_record_ksymbol, name), + }, + .addr = prog_addrs[i], + .len = prog_lens[i], + .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, + .flags = 0, + }; + + name_len = synthesize_bpf_prog_name(ksymbol_event->name, + KSYM_NAME_LEN, info, btf, i); + ksymbol_event->header.size += PERF_ALIGN(name_len + 1, + sizeof(u64)); + + memset((void *)event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + err = perf_tool__process_synth_event(tool, event, + machine, process); + } + + if (!opts->no_bpf_event) { + /* Synthesize PERF_RECORD_BPF_EVENT */ + *bpf_event = (struct perf_record_bpf_event) { + .header = { + .type = PERF_RECORD_BPF_EVENT, + .size = sizeof(struct perf_record_bpf_event), + }, + .type = PERF_BPF_EVENT_PROG_LOAD, + .flags = 0, + .id = info->id, + }; + memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); + memset((void *)event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + + /* save bpf_prog_info to env */ + info_node = malloc(sizeof(struct bpf_prog_info_node)); + if (!info_node) { + err = -1; + goto out; + } + + info_node->info_linear = info_linear; + info_node->metadata = NULL; + if (!perf_env__insert_bpf_prog_info(env, info_node)) { + /* + * Insert failed, likely because of a duplicate event + * made by the sideband thread. Ignore synthesizing the + * metadata. + */ + free(info_node); + goto out; + } + /* info_linear is now owned by info_node and shouldn't be freed below. */ + info_linear = NULL; + + /* + * process after saving bpf_prog_info to env, so that + * required information is ready for look up + */ + err = perf_tool__process_synth_event(tool, event, + machine, process); + + /* Synthesize PERF_RECORD_BPF_METADATA */ + metadata = bpf_metadata_create(info); + if (metadata != NULL) { + err = synthesize_perf_record_bpf_metadata(metadata, + tool, process, + machine); + bpf_metadata_free(metadata); + } + } + +out: + free(info_linear); + btf__free(btf); + return err ? -1 : 0; +} + +struct kallsyms_parse { + union perf_event *event; + perf_event__handler_t process; + struct machine *machine; + const struct perf_tool *tool; +}; + +static int +process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) +{ + struct machine *machine = data->machine; + union perf_event *event = data->event; + struct perf_record_ksymbol *ksymbol; + int len; + + ksymbol = &event->ksymbol; + + *ksymbol = (struct perf_record_ksymbol) { + .header = { + .type = PERF_RECORD_KSYMBOL, + .size = offsetof(struct perf_record_ksymbol, name), + }, + .addr = addr, + .len = page_size, + .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, + .flags = 0, + }; + + len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); + ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); + memset((void *) event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + + return perf_tool__process_synth_event(data->tool, event, machine, + data->process); +} + +static int +kallsyms_process_symbol(void *data, const char *_name, + char type __maybe_unused, u64 start) +{ + char disp[KSYM_NAME_LEN]; + const char *module; + char *name; + unsigned long id; + int err = 0; + + module = strchr(_name, '\t'); + if (!module) + return 0; + + /* We are going after [bpf] module ... */ + if (strcmp(module + 1, "[bpf]")) + return 0; + + name = memdup(_name, (module - _name) + 1); + if (!name) + return -ENOMEM; + + name[module - _name] = 0; + + /* .. and only for trampolines and dispatchers */ + if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || + (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) + err = process_bpf_image(name, start, data); + + free(name); + return err; +} + +int perf_event__synthesize_bpf_events(struct perf_session *session, + perf_event__handler_t process, + struct machine *machine, + struct record_opts *opts) +{ + const char *kallsyms_filename = "/proc/kallsyms"; + struct kallsyms_parse arg; + union perf_event *event; + __u32 id = 0; + int err; + int fd; + + if (opts->no_bpf_event) + return 0; + + event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); + if (!event) + return -1; + + /* Synthesize all the bpf programs in system. */ + while (true) { + err = bpf_prog_get_next_id(id, &id); + if (err) { + if (errno == ENOENT) { + err = 0; + break; + } + /* don't report error on old kernel or EPERM */ + err = (errno == EINVAL || errno == EPERM) ? 0 : -1; + pr_debug("%s: can\'t get next program: %m%s\n", + __func__, errno == EINVAL ? " -- kernel too old?" : ""); + break; + } + fd = bpf_prog_get_fd_by_id(id); + if (fd < 0) { + pr_debug("%s: failed to get fd for prog_id %u\n", + __func__, id); + continue; + } + + err = perf_event__synthesize_one_bpf_prog(session, process, + machine, fd, + event, opts); + close(fd); + if (err) { + /* do not return error for old kernel */ + if (err == -2) + err = 0; + break; + } + } + + /* Synthesize all the bpf images - trampolines/dispatchers. */ + if (symbol_conf.kallsyms_name != NULL) + kallsyms_filename = symbol_conf.kallsyms_name; + + arg = (struct kallsyms_parse) { + .event = event, + .process = process, + .machine = machine, + .tool = session->tool, + }; + + if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) + pr_err("%s: failed to synthesize bpf images: %m\n", __func__); + + free(event); + return err; +} + +static int perf_env__add_bpf_info(struct perf_env *env, u32 id) +{ + struct bpf_prog_info_node *info_node; + struct perf_bpil *info_linear; + struct btf *btf = NULL; + u64 arrays; + u32 btf_id; + int fd, err = 0; + + fd = bpf_prog_get_fd_by_id(id); + if (fd < 0) + return -EINVAL; + + arrays = 1UL << PERF_BPIL_JITED_KSYMS; + arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; + arrays |= 1UL << PERF_BPIL_FUNC_INFO; + arrays |= 1UL << PERF_BPIL_PROG_TAGS; + arrays |= 1UL << PERF_BPIL_JITED_INSNS; + arrays |= 1UL << PERF_BPIL_LINE_INFO; + arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; + arrays |= 1UL << PERF_BPIL_MAP_IDS; + + info_linear = get_bpf_prog_info_linear(fd, arrays); + if (IS_ERR_OR_NULL(info_linear)) { + pr_debug("%s: failed to get BPF program info. aborting\n", __func__); + err = PTR_ERR(info_linear); + goto out; + } + + btf_id = info_linear->info.btf_id; + + info_node = malloc(sizeof(struct bpf_prog_info_node)); + if (info_node) { + info_node->info_linear = info_linear; + info_node->metadata = bpf_metadata_create(&info_linear->info); + if (!perf_env__insert_bpf_prog_info(env, info_node)) { + pr_debug("%s: duplicate add bpf info request for id %u\n", + __func__, btf_id); + bpf_metadata_free(info_node->metadata); + free(info_linear); + free(info_node); + goto out; + } + } else { + free(info_linear); + err = -ENOMEM; + goto out; + } + + if (btf_id == 0) + goto out; + + btf = btf__load_from_kernel_by_id(btf_id); + if (!btf) { + err = -errno; + pr_debug("%s: failed to get BTF of id %u %d\n", __func__, btf_id, err); + } else { + perf_env__fetch_btf(env, btf_id, btf); + } + +out: + btf__free(btf); + close(fd); + return err; +} + +static int bpf_event__sb_cb(union perf_event *event, void *data) +{ + struct perf_env *env = data; + int ret = 0; + + if (event->header.type != PERF_RECORD_BPF_EVENT) + return -1; + + switch (event->bpf.type) { + case PERF_BPF_EVENT_PROG_LOAD: + ret = perf_env__add_bpf_info(env, event->bpf.id); + + case PERF_BPF_EVENT_PROG_UNLOAD: + /* + * Do not free bpf_prog_info and btf of the program here, + * as annotation still need them. They will be freed at + * the end of the session. + */ + break; + default: + pr_debug("unexpected bpf event type of %d\n", event->bpf.type); + break; + } + + return ret; +} + +int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_DUMMY, + .sample_id_all = 1, + .watermark = 1, + .bpf_event = 1, + .size = sizeof(attr), /* to capture ABI version */ + }; + + /* + * Older gcc versions don't support designated initializers, like above, + * for unnamed union members, such as the following: + */ + attr.wakeup_watermark = 1; + + return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); +} + +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, + struct perf_env *env, + FILE *fp) +{ + struct bpf_prog_info *info = &info_linear->info; + __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | + (1UL << PERF_BPIL_JITED_FUNC_LENS) | + (1UL << PERF_BPIL_PROG_TAGS); + __u32 *prog_lens; + __u64 *prog_addrs; + char name[KSYM_NAME_LEN]; + struct btf *btf = NULL; + u32 sub_prog_cnt, i; + + sub_prog_cnt = info->nr_jited_ksyms; + if (sub_prog_cnt != info->nr_prog_tags || + sub_prog_cnt != info->nr_jited_func_lens) + return; + + /* Ensure the arrays were present and converted by bpil_offs_to_addr() */ + if ((info_linear->arrays & required_arrays) != required_arrays) + return; + + prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + + if (info->btf_id) { + struct btf_node *node; + + node = __perf_env__find_btf(env, info->btf_id); + if (node) + btf = btf__new((__u8 *)(node->data), + node->data_size); + } + + if (sub_prog_cnt == 1) { + synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); + fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", + info->id, name, prog_addrs[0], prog_lens[0]); + goto out; + } + + fprintf(fp, "# bpf_prog_info %u:\n", info->id); + for (i = 0; i < sub_prog_cnt; i++) { + synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); + + fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", + i, name, prog_addrs[i], prog_lens[i]); + } +out: + btf__free(btf); +} diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h new file mode 100644 index 000000000..da4eeb4a1 --- /dev/null +++ b/tools/perf/util/bpf-event.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BPF_EVENT_H +#define __PERF_BPF_EVENT_H + +#include +#include +#include +#include + +struct bpf_prog_info; +struct machine; +union perf_event; +struct perf_env; +struct perf_sample; +struct perf_session; +struct record_opts; +struct evlist; +struct target; + +struct bpf_metadata { + union perf_event *event; + char **prog_names; + __u64 nr_prog_names; +}; + +struct bpf_prog_info_node { + struct perf_bpil *info_linear; + struct bpf_metadata *metadata; + struct rb_node rb_node; +}; + +struct btf_node { + struct rb_node rb_node; + u32 id; + u32 data_size; + char data[]; +}; + +#ifdef HAVE_LIBBPF_SUPPORT +int machine__process_bpf(struct machine *machine, union perf_event *event, + struct perf_sample *sample); +int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env); +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, + struct perf_env *env, + FILE *fp); +void bpf_metadata_free(struct bpf_metadata *metadata); +#else +static inline int machine__process_bpf(struct machine *machine __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused) +{ + return 0; +} + +static inline int evlist__add_bpf_sb_event(struct evlist *evlist __maybe_unused, + struct perf_env *env __maybe_unused) +{ + return 0; +} + +static inline void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear __maybe_unused, + struct perf_env *env __maybe_unused, + FILE *fp __maybe_unused) +{ + +} + +static inline void bpf_metadata_free(struct bpf_metadata *metadata __maybe_unused) +{ + +} +#endif // HAVE_LIBBPF_SUPPORT +#endif diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c new file mode 100644 index 000000000..bcd81084e --- /dev/null +++ b/tools/perf/util/bpf-filter.c @@ -0,0 +1,803 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** + * Generic event filter for sampling events in BPF. + * + * The BPF program is fixed and just to read filter expressions in the 'filters' + * map and compare the sample data in order to reject samples that don't match. + * Each filter expression contains a sample flag (term) to compare, an operation + * (==, >=, and so on) and a value. + * + * Note that each entry has an array of filter expressions and it only succeeds + * when all of the expressions are satisfied. But it supports the logical OR + * using a GROUP operation which is satisfied when any of its member expression + * is evaluated to true. But it doesn't allow nested GROUP operations for now. + * + * To support non-root users, the filters map can be loaded and pinned in the BPF + * filesystem by root (perf record --setup-filter pin). Then each user will get + * a new entry in the shared filters map to fill the filter expressions. And the + * BPF program will find the filter using (task-id, event-id) as a key. + * + * The pinned BPF object (shared for regular users) has: + * + * event_hash | + * | | | + * event->id ---> | id | ---+ idx_hash | filters + * | | | | | | | | + * | .... | +-> | idx | --+--> | exprs | ---> perf_bpf_filter_entry[] + * | | | | | | .op + * task id (tgid) --------------+ | .... | | | ... | .term (+ part) + * | .value + * | + * ======= (root would skip this part) ======== (compares it in a loop) + * + * This is used for per-task use cases while system-wide profiling (normally from + * root user) uses a separate copy of the program and the maps for its own so that + * it can proceed even if a lot of non-root users are using the filters at the + * same time. In this case the filters map has a single entry and no need to use + * the hash maps to get the index (key) of the filters map (IOW it's always 0). + * + * The BPF program returns 1 to accept the sample or 0 to drop it. + * The 'dropped' map is to keep how many samples it dropped by the filter and + * it will be reported as lost samples. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "util/cap.h" +#include "util/debug.h" +#include "util/evsel.h" +#include "util/target.h" +#include "util/bpf-utils.h" + +#include "util/bpf-filter.h" +#include +#include + +#include "bpf_skel/sample-filter.h" +#include "bpf_skel/sample_filter.skel.h" + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) + +#define __PERF_SAMPLE_TYPE(tt, st, opt) { tt, #st, opt } +#define PERF_SAMPLE_TYPE(_st, opt) __PERF_SAMPLE_TYPE(PBF_TERM_##_st, PERF_SAMPLE_##_st, opt) + +/* Index in the pinned 'filters' map. Should be released after use. */ +struct pinned_filter_idx { + struct list_head list; + struct evsel *evsel; + u64 event_id; + int hash_idx; +}; + +static LIST_HEAD(pinned_filters); + +static const struct perf_sample_info { + enum perf_bpf_filter_term type; + const char *name; + const char *option; +} sample_table[] = { + /* default sample flags */ + PERF_SAMPLE_TYPE(IP, NULL), + PERF_SAMPLE_TYPE(TID, NULL), + PERF_SAMPLE_TYPE(PERIOD, NULL), + /* flags mostly set by default, but still have options */ + PERF_SAMPLE_TYPE(ID, "--sample-identifier"), + PERF_SAMPLE_TYPE(CPU, "--sample-cpu"), + PERF_SAMPLE_TYPE(TIME, "-T"), + /* optional sample flags */ + PERF_SAMPLE_TYPE(ADDR, "-d"), + PERF_SAMPLE_TYPE(DATA_SRC, "-d"), + PERF_SAMPLE_TYPE(PHYS_ADDR, "--phys-data"), + PERF_SAMPLE_TYPE(WEIGHT, "-W"), + PERF_SAMPLE_TYPE(WEIGHT_STRUCT, "-W"), + PERF_SAMPLE_TYPE(TRANSACTION, "--transaction"), + PERF_SAMPLE_TYPE(CODE_PAGE_SIZE, "--code-page-size"), + PERF_SAMPLE_TYPE(DATA_PAGE_SIZE, "--data-page-size"), + PERF_SAMPLE_TYPE(CGROUP, "--all-cgroups"), +}; + +static int get_pinned_fd(const char *name); + +static const struct perf_sample_info *get_sample_info(enum perf_bpf_filter_term type) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(sample_table); i++) { + if (sample_table[i].type == type) + return &sample_table[i]; + } + return NULL; +} + +static int check_sample_flags(struct evsel *evsel, struct perf_bpf_filter_expr *expr) +{ + const struct perf_sample_info *info; + + if (expr->term >= PBF_TERM_SAMPLE_START && expr->term <= PBF_TERM_SAMPLE_END && + (evsel->core.attr.sample_type & (1 << (expr->term - PBF_TERM_SAMPLE_START)))) + return 0; + + if (expr->term == PBF_TERM_UID || expr->term == PBF_TERM_GID) { + /* Not dependent on the sample_type as computed from a BPF helper. */ + return 0; + } + + if (expr->op == PBF_OP_GROUP_BEGIN) { + struct perf_bpf_filter_expr *group; + + list_for_each_entry(group, &expr->groups, list) { + if (check_sample_flags(evsel, group) < 0) + return -1; + } + return 0; + } + + info = get_sample_info(expr->term); + if (info == NULL) { + pr_err("Error: %s event does not have sample flags %d\n", + evsel__name(evsel), expr->term); + return -1; + } + + pr_err("Error: %s event does not have %s\n", evsel__name(evsel), info->name); + if (info->option) + pr_err(" Hint: please add %s option to perf record\n", info->option); + return -1; +} + +static int get_filter_entries(struct evsel *evsel, struct perf_bpf_filter_entry *entry) +{ + int i = 0; + struct perf_bpf_filter_expr *expr; + + list_for_each_entry(expr, &evsel->bpf_filters, list) { + if (check_sample_flags(evsel, expr) < 0) + return -EINVAL; + + if (i == MAX_FILTERS) + return -E2BIG; + + entry[i].op = expr->op; + entry[i].part = expr->part; + entry[i].term = expr->term; + entry[i].value = expr->val; + i++; + + if (expr->op == PBF_OP_GROUP_BEGIN) { + struct perf_bpf_filter_expr *group; + + list_for_each_entry(group, &expr->groups, list) { + if (i == MAX_FILTERS) + return -E2BIG; + + entry[i].op = group->op; + entry[i].part = group->part; + entry[i].term = group->term; + entry[i].value = group->val; + i++; + } + + if (i == MAX_FILTERS) + return -E2BIG; + + entry[i].op = PBF_OP_GROUP_END; + i++; + } + } + + if (i < MAX_FILTERS) { + /* to terminate the loop early */ + entry[i].op = PBF_OP_DONE; + i++; + } + return 0; +} + +static int convert_to_tgid(int tid) +{ + char path[128]; + char *buf, *p, *q; + int tgid; + size_t len; + + scnprintf(path, sizeof(path), "%d/status", tid); + if (procfs__read_str(path, &buf, &len) < 0) + return -1; + + p = strstr(buf, "Tgid:"); + if (p == NULL) { + free(buf); + return -1; + } + + tgid = strtol(p + 6, &q, 0); + free(buf); + if (*q != '\n') + return -1; + + return tgid; +} + +/* + * The event might be closed already so we cannot get the list of ids using FD + * like in create_event_hash() below, let's iterate the event_hash map and + * delete all entries that have the event id as a key. + */ +static void destroy_event_hash(u64 event_id) +{ + int fd; + u64 key, *prev_key = NULL; + int num = 0, alloced = 32; + u64 *ids = calloc(alloced, sizeof(*ids)); + + if (ids == NULL) + return; + + fd = get_pinned_fd("event_hash"); + if (fd < 0) { + pr_debug("cannot get fd for 'event_hash' map\n"); + free(ids); + return; + } + + /* Iterate the whole map to collect keys for the event id. */ + while (!bpf_map_get_next_key(fd, prev_key, &key)) { + u64 id; + + if (bpf_map_lookup_elem(fd, &key, &id) == 0 && id == event_id) { + if (num == alloced) { + void *tmp; + + alloced *= 2; + tmp = realloc(ids, alloced * sizeof(*ids)); + if (tmp == NULL) + break; + + ids = tmp; + } + ids[num++] = key; + } + + prev_key = &key; + } + + for (int i = 0; i < num; i++) + bpf_map_delete_elem(fd, &ids[i]); + + free(ids); + close(fd); +} + +/* + * Return a representative id if ok, or 0 for failures. + * + * The perf_event->id is good for this, but an evsel would have multiple + * instances for CPUs and tasks. So pick up the first id and setup a hash + * from id of each instance to the representative id (the first one). + */ +static u64 create_event_hash(struct evsel *evsel) +{ + int x, y, fd; + u64 the_id = 0, id; + + fd = get_pinned_fd("event_hash"); + if (fd < 0) { + pr_err("cannot get fd for 'event_hash' map\n"); + return 0; + } + + for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { + for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { + int ret = ioctl(FD(evsel, x, y), PERF_EVENT_IOC_ID, &id); + + if (ret < 0) { + pr_err("Failed to get the event id\n"); + if (the_id) + destroy_event_hash(the_id); + return 0; + } + + if (the_id == 0) + the_id = id; + + bpf_map_update_elem(fd, &id, &the_id, BPF_ANY); + } + } + + close(fd); + return the_id; +} + +static void destroy_idx_hash(struct pinned_filter_idx *pfi) +{ + int fd, nr; + struct perf_thread_map *threads; + + fd = get_pinned_fd("filters"); + bpf_map_delete_elem(fd, &pfi->hash_idx); + close(fd); + + if (pfi->event_id) + destroy_event_hash(pfi->event_id); + + threads = perf_evsel__threads(&pfi->evsel->core); + if (threads == NULL) + return; + + fd = get_pinned_fd("idx_hash"); + nr = perf_thread_map__nr(threads); + for (int i = 0; i < nr; i++) { + /* The target task might be dead already, just try the pid */ + struct idx_hash_key key = { + .evt_id = pfi->event_id, + .tgid = perf_thread_map__pid(threads, i), + }; + + bpf_map_delete_elem(fd, &key); + } + close(fd); +} + +/* Maintain a hashmap from (tgid, event-id) to filter index */ +static int create_idx_hash(struct evsel *evsel, struct perf_bpf_filter_entry *entry) +{ + int filter_idx; + int fd, nr, last; + u64 event_id = 0; + struct pinned_filter_idx *pfi = NULL; + struct perf_thread_map *threads; + + fd = get_pinned_fd("filters"); + if (fd < 0) { + pr_err("cannot get fd for 'filters' map\n"); + return fd; + } + + /* Find the first available entry in the filters map */ + for (filter_idx = 0; filter_idx < MAX_FILTERS; filter_idx++) { + if (bpf_map_update_elem(fd, &filter_idx, entry, BPF_NOEXIST) == 0) + break; + } + close(fd); + + if (filter_idx == MAX_FILTERS) { + pr_err("Too many users for the filter map\n"); + return -EBUSY; + } + + pfi = zalloc(sizeof(*pfi)); + if (pfi == NULL) { + pr_err("Cannot save pinned filter index\n"); + return -ENOMEM; + } + + pfi->evsel = evsel; + pfi->hash_idx = filter_idx; + + event_id = create_event_hash(evsel); + if (event_id == 0) { + pr_err("Cannot update the event hash\n"); + goto err; + } + + pfi->event_id = event_id; + + threads = perf_evsel__threads(&evsel->core); + if (threads == NULL) { + pr_err("Cannot get the thread list of the event\n"); + goto err; + } + + /* save the index to a hash map */ + fd = get_pinned_fd("idx_hash"); + if (fd < 0) { + pr_err("cannot get fd for 'idx_hash' map\n"); + goto err; + } + + last = -1; + nr = perf_thread_map__nr(threads); + for (int i = 0; i < nr; i++) { + int pid = perf_thread_map__pid(threads, i); + int tgid; + struct idx_hash_key key = { + .evt_id = event_id, + }; + + /* it actually needs tgid, let's get tgid from /proc. */ + tgid = convert_to_tgid(pid); + if (tgid < 0) { + /* the thread may be dead, ignore. */ + continue; + } + + if (tgid == last) + continue; + last = tgid; + key.tgid = tgid; + + if (bpf_map_update_elem(fd, &key, &filter_idx, BPF_ANY) < 0) { + pr_err("Failed to update the idx_hash\n"); + close(fd); + goto err; + } + pr_debug("bpf-filter: idx_hash (task=%d,%s) -> %d\n", + tgid, evsel__name(evsel), filter_idx); + } + + list_add(&pfi->list, &pinned_filters); + close(fd); + return filter_idx; + +err: + destroy_idx_hash(pfi); + free(pfi); + return -1; +} + +int perf_bpf_filter__prepare(struct evsel *evsel, struct target *target) +{ + int i, x, y, fd, ret; + struct sample_filter_bpf *skel = NULL; + struct bpf_program *prog; + struct bpf_link *link; + struct perf_bpf_filter_entry *entry; + bool needs_idx_hash = !target__has_cpu(target); +#if LIBBPF_CURRENT_VERSION_GEQ(1, 7) + DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts, + .dont_enable = true); +#else + DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); +#endif + + entry = calloc(MAX_FILTERS, sizeof(*entry)); + if (entry == NULL) + return -1; + + ret = get_filter_entries(evsel, entry); + if (ret < 0) { + pr_err("Failed to process filter entries\n"); + goto err; + } + + if (needs_idx_hash && geteuid() != 0) { + int zero = 0; + + /* The filters map is shared among other processes */ + ret = create_idx_hash(evsel, entry); + if (ret < 0) + goto err; + + fd = get_pinned_fd("dropped"); + if (fd < 0) { + ret = fd; + goto err; + } + + /* Reset the lost count */ + bpf_map_update_elem(fd, &ret, &zero, BPF_ANY); + close(fd); + + fd = get_pinned_fd("perf_sample_filter"); + if (fd < 0) { + ret = fd; + goto err; + } + + for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { + for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { + ret = ioctl(FD(evsel, x, y), PERF_EVENT_IOC_SET_BPF, fd); + if (ret < 0) { + pr_err("Failed to attach perf sample-filter\n"); + close(fd); + goto err; + } + } + } + + close(fd); + free(entry); + return 0; + } + + skel = sample_filter_bpf__open_and_load(); + if (!skel) { + ret = -errno; + pr_err("Failed to load perf sample-filter BPF skeleton\n"); + goto err; + } + + i = 0; + fd = bpf_map__fd(skel->maps.filters); + + /* The filters map has only one entry in this case */ + if (bpf_map_update_elem(fd, &i, entry, BPF_ANY) < 0) { + ret = -errno; + pr_err("Failed to update the filter map\n"); + goto err; + } + + prog = skel->progs.perf_sample_filter; + for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { + for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { + link = bpf_program__attach_perf_event_opts(prog, FD(evsel, x, y), + &pe_opts); + if (IS_ERR(link)) { + pr_err("Failed to attach perf sample-filter program\n"); + ret = PTR_ERR(link); + goto err; + } + } + } + free(entry); + evsel->bpf_skel = skel; + return 0; + +err: + free(entry); + if (!list_empty(&pinned_filters)) { + struct pinned_filter_idx *pfi, *tmp; + + list_for_each_entry_safe(pfi, tmp, &pinned_filters, list) { + destroy_idx_hash(pfi); + list_del(&pfi->list); + free(pfi); + } + } + sample_filter_bpf__destroy(skel); + return ret; +} + +int perf_bpf_filter__destroy(struct evsel *evsel) +{ + struct perf_bpf_filter_expr *expr, *tmp; + struct pinned_filter_idx *pfi, *pos; + + list_for_each_entry_safe(expr, tmp, &evsel->bpf_filters, list) { + list_del(&expr->list); + free(expr); + } + sample_filter_bpf__destroy(evsel->bpf_skel); + + list_for_each_entry_safe(pfi, pos, &pinned_filters, list) { + destroy_idx_hash(pfi); + list_del(&pfi->list); + free(pfi); + } + return 0; +} + +u64 perf_bpf_filter__lost_count(struct evsel *evsel) +{ + int count = 0; + + if (list_empty(&evsel->bpf_filters)) + return 0; + + if (!list_empty(&pinned_filters)) { + int fd = get_pinned_fd("dropped"); + struct pinned_filter_idx *pfi; + + if (fd < 0) + return 0; + + list_for_each_entry(pfi, &pinned_filters, list) { + if (pfi->evsel != evsel) + continue; + + bpf_map_lookup_elem(fd, &pfi->hash_idx, &count); + break; + } + close(fd); + } else if (evsel->bpf_skel) { + struct sample_filter_bpf *skel = evsel->bpf_skel; + int fd = bpf_map__fd(skel->maps.dropped); + int idx = 0; + + bpf_map_lookup_elem(fd, &idx, &count); + } + + return count; +} + +struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term term, + int part, + enum perf_bpf_filter_op op, + unsigned long val) +{ + struct perf_bpf_filter_expr *expr; + + expr = malloc(sizeof(*expr)); + if (expr != NULL) { + expr->term = term; + expr->part = part; + expr->op = op; + expr->val = val; + INIT_LIST_HEAD(&expr->groups); + } + return expr; +} + +static bool check_bpf_filter_capable(void) +{ + int fd; + + if (perf_cap__capable(CAP_BPF)) + return true; + + /* Check if root already pinned the filter programs and maps */ + fd = get_pinned_fd("filters"); + if (fd >= 0) { + close(fd); + return true; + } + + pr_err("Error: BPF filter only works for users with the CAP_BPF capability!\n" + "\tPlease run 'perf record --setup-filter pin' as root first.\n"); + + return false; +} + +int perf_bpf_filter__parse(struct list_head *expr_head, const char *str) +{ + YY_BUFFER_STATE buffer; + int ret; + + if (!check_bpf_filter_capable()) + return -EPERM; + + buffer = perf_bpf_filter__scan_string(str); + + ret = perf_bpf_filter_parse(expr_head); + + perf_bpf_filter__flush_buffer(buffer); + perf_bpf_filter__delete_buffer(buffer); + perf_bpf_filter_lex_destroy(); + + return ret; +} + +int perf_bpf_filter__pin(void) +{ + struct sample_filter_bpf *skel; + char *path = NULL; + int dir_fd, ret = -1; + + skel = sample_filter_bpf__open(); + if (!skel) { + ret = -errno; + pr_err("Failed to open perf sample-filter BPF skeleton\n"); + goto err; + } + + /* pinned program will use pid-hash */ + bpf_map__set_max_entries(skel->maps.filters, MAX_FILTERS); + bpf_map__set_max_entries(skel->maps.event_hash, MAX_EVT_HASH); + bpf_map__set_max_entries(skel->maps.idx_hash, MAX_IDX_HASH); + bpf_map__set_max_entries(skel->maps.dropped, MAX_FILTERS); + skel->rodata->use_idx_hash = 1; + + if (sample_filter_bpf__load(skel) < 0) { + ret = -errno; + pr_err("Failed to load perf sample-filter BPF skeleton\n"); + goto err; + } + + if (asprintf(&path, "%s/fs/bpf/%s", sysfs__mountpoint(), + PERF_BPF_FILTER_PIN_PATH) < 0) { + ret = -errno; + pr_err("Failed to allocate pathname in the BPF-fs\n"); + goto err; + } + + ret = bpf_object__pin(skel->obj, path); + if (ret < 0) { + pr_err("Failed to pin BPF filter objects\n"); + goto err; + } + + /* setup access permissions for the pinned objects */ + dir_fd = open(path, O_PATH); + if (dir_fd < 0) { + bpf_object__unpin(skel->obj, path); + ret = dir_fd; + goto err; + } + + /* BPF-fs root has the sticky bit */ + if (fchmodat(dir_fd, "..", 01755, 0) < 0) { + pr_debug("chmod for BPF-fs failed\n"); + ret = -errno; + goto err_close; + } + + /* perf_filter directory */ + if (fchmodat(dir_fd, ".", 0755, 0) < 0) { + pr_debug("chmod for perf_filter directory failed?\n"); + ret = -errno; + goto err_close; + } + + /* programs need write permission for some reason */ + if (fchmodat(dir_fd, "perf_sample_filter", 0777, 0) < 0) { + pr_debug("chmod for perf_sample_filter failed\n"); + ret = -errno; + } + /* maps */ + if (fchmodat(dir_fd, "filters", 0666, 0) < 0) { + pr_debug("chmod for filters failed\n"); + ret = -errno; + } + if (fchmodat(dir_fd, "event_hash", 0666, 0) < 0) { + pr_debug("chmod for event_hash failed\n"); + ret = -errno; + } + if (fchmodat(dir_fd, "idx_hash", 0666, 0) < 0) { + pr_debug("chmod for idx_hash failed\n"); + ret = -errno; + } + if (fchmodat(dir_fd, "dropped", 0666, 0) < 0) { + pr_debug("chmod for dropped failed\n"); + ret = -errno; + } + +err_close: + close(dir_fd); + +err: + free(path); + sample_filter_bpf__destroy(skel); + return ret; +} + +int perf_bpf_filter__unpin(void) +{ + struct sample_filter_bpf *skel; + char *path = NULL; + int ret = -1; + + skel = sample_filter_bpf__open_and_load(); + if (!skel) { + ret = -errno; + pr_err("Failed to open perf sample-filter BPF skeleton\n"); + goto err; + } + + if (asprintf(&path, "%s/fs/bpf/%s", sysfs__mountpoint(), + PERF_BPF_FILTER_PIN_PATH) < 0) { + ret = -errno; + pr_err("Failed to allocate pathname in the BPF-fs\n"); + goto err; + } + + ret = bpf_object__unpin(skel->obj, path); + +err: + free(path); + sample_filter_bpf__destroy(skel); + return ret; +} + +static int get_pinned_fd(const char *name) +{ + char *path = NULL; + int fd; + + if (asprintf(&path, "%s/fs/bpf/%s/%s", sysfs__mountpoint(), + PERF_BPF_FILTER_PIN_PATH, name) < 0) + return -1; + + fd = bpf_obj_get(path); + + free(path); + return fd; +} diff --git a/tools/perf/util/bpf-filter.h b/tools/perf/util/bpf-filter.h new file mode 100644 index 000000000..818c554b9 --- /dev/null +++ b/tools/perf/util/bpf-filter.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_UTIL_BPF_FILTER_H +#define PERF_UTIL_BPF_FILTER_H + +#include + +#include "bpf_skel/sample-filter.h" +#include "util/debug.h" + +struct perf_bpf_filter_expr { + struct list_head list; + struct list_head groups; + enum perf_bpf_filter_op op; + int part; + enum perf_bpf_filter_term term; + unsigned long val; +}; + +struct evsel; +struct target; + +/* path in BPF-fs for the pinned program and maps */ +#define PERF_BPF_FILTER_PIN_PATH "perf_filter" + +#ifdef HAVE_BPF_SKEL +struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term term, + int part, + enum perf_bpf_filter_op op, + unsigned long val); +int perf_bpf_filter__parse(struct list_head *expr_head, const char *str); +int perf_bpf_filter__prepare(struct evsel *evsel, struct target *target); +int perf_bpf_filter__destroy(struct evsel *evsel); +u64 perf_bpf_filter__lost_count(struct evsel *evsel); +int perf_bpf_filter__pin(void); +int perf_bpf_filter__unpin(void); + +#else /* !HAVE_BPF_SKEL */ + +#include + +static inline int perf_bpf_filter__parse(struct list_head *expr_head __maybe_unused, + const char *str __maybe_unused) +{ + pr_err("Error: BPF filter is requested but perf is not built with BPF.\n" + "\tPlease make sure to build with libbpf and BPF skeleton.\n"); + return -EOPNOTSUPP; +} +static inline int perf_bpf_filter__prepare(struct evsel *evsel __maybe_unused, + struct target *target __maybe_unused) +{ + return -EOPNOTSUPP; +} +static inline int perf_bpf_filter__destroy(struct evsel *evsel __maybe_unused) +{ + return -EOPNOTSUPP; +} +static inline u64 perf_bpf_filter__lost_count(struct evsel *evsel __maybe_unused) +{ + return 0; +} +static inline int perf_bpf_filter__pin(void) +{ + return -EOPNOTSUPP; +} +static inline int perf_bpf_filter__unpin(void) +{ + return -EOPNOTSUPP; +} +#endif /* HAVE_BPF_SKEL*/ +#endif /* PERF_UTIL_BPF_FILTER_H */ diff --git a/tools/perf/util/bpf-filter.l b/tools/perf/util/bpf-filter.l new file mode 100644 index 000000000..6aa65ade3 --- /dev/null +++ b/tools/perf/util/bpf-filter.l @@ -0,0 +1,179 @@ +%option prefix="perf_bpf_filter_" +%option noyywrap + +%{ +#include +#include +#include + +#include "bpf-filter.h" +#include "bpf-filter-bison.h" + +extern int perf_bpf_filter_needs_path; + +static int sample(enum perf_bpf_filter_term term) +{ + perf_bpf_filter_needs_path = 0; + perf_bpf_filter_lval.sample.term = term; + perf_bpf_filter_lval.sample.part = 0; + return BFT_SAMPLE; +} + +static int sample_part(enum perf_bpf_filter_term term, int part) +{ + perf_bpf_filter_needs_path = 0; + perf_bpf_filter_lval.sample.term = term; + perf_bpf_filter_lval.sample.part = part; + return BFT_SAMPLE; +} + +static int sample_path(enum perf_bpf_filter_term term) +{ + perf_bpf_filter_needs_path = 1; + perf_bpf_filter_lval.sample.term = term; + perf_bpf_filter_lval.sample.part = 0; + return BFT_SAMPLE_PATH; +} + +static int operator(enum perf_bpf_filter_op op) +{ + perf_bpf_filter_lval.op = op; + return BFT_OP; +} + +static int value(int base) +{ + long num; + + errno = 0; + num = strtoul(perf_bpf_filter_text, NULL, base); + if (errno) + return BFT_ERROR; + + perf_bpf_filter_lval.num = num; + return BFT_NUM; +} + +static int constant(int val) +{ + perf_bpf_filter_lval.num = val; + return BFT_NUM; +} + +static int path_or_error(void) +{ + if (!perf_bpf_filter_needs_path) { + printf("perf_bpf_filter: Error: Unexpected item: %s\n", + perf_bpf_filter_text); + return BFT_ERROR; + } + perf_bpf_filter_lval.path = perf_bpf_filter_text; + return BFT_PATH; +} + +%} + +num_dec [0-9]+ +num_hex 0[Xx][0-9a-fA-F]+ +space [ \t]+ +path [^ \t\n,]+ +ident [_a-zA-Z][_a-zA-Z0-9]+ + +%% + +{num_dec} { return value(10); } +{num_hex} { return value(16); } +{space} { } + +ip { return sample(PBF_TERM_IP); } +id { return sample(PBF_TERM_ID); } +tid { return sample(PBF_TERM_TID); } +pid { return sample_part(PBF_TERM_TID, 1); } +cpu { return sample(PBF_TERM_CPU); } +time { return sample(PBF_TERM_TIME); } +addr { return sample(PBF_TERM_ADDR); } +period { return sample(PBF_TERM_PERIOD); } +txn { return sample(PBF_TERM_TRANSACTION); } +weight { return sample(PBF_TERM_WEIGHT); } +weight1 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 1); } +weight2 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 2); } +weight3 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } +ins_lat { return sample_part(PBF_TERM_WEIGHT_STRUCT, 2); } /* alias for weight2 */ +p_stage_cyc { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } /* alias for weight3 */ +retire_lat { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } /* alias for weight3 */ +phys_addr { return sample(PBF_TERM_PHYS_ADDR); } +code_pgsz { return sample(PBF_TERM_CODE_PAGE_SIZE); } +data_pgsz { return sample(PBF_TERM_DATA_PAGE_SIZE); } +mem_op { return sample_part(PBF_TERM_DATA_SRC, 1); } +mem_lvlnum { return sample_part(PBF_TERM_DATA_SRC, 2); } +mem_lvl { return sample_part(PBF_TERM_DATA_SRC, 2); } /* alias for mem_lvlnum */ +mem_snoop { return sample_part(PBF_TERM_DATA_SRC, 3); } /* include snoopx */ +mem_remote { return sample_part(PBF_TERM_DATA_SRC, 4); } +mem_lock { return sample_part(PBF_TERM_DATA_SRC, 5); } +mem_dtlb { return sample_part(PBF_TERM_DATA_SRC, 6); } +mem_blk { return sample_part(PBF_TERM_DATA_SRC, 7); } +mem_hops { return sample_part(PBF_TERM_DATA_SRC, 8); } +uid { return sample(PBF_TERM_UID); } +gid { return sample(PBF_TERM_GID); } +cgroup { return sample_path(PBF_TERM_CGROUP); } + +"==" { return operator(PBF_OP_EQ); } +"!=" { return operator(PBF_OP_NEQ); } +">" { return operator(PBF_OP_GT); } +"<" { return operator(PBF_OP_LT); } +">=" { return operator(PBF_OP_GE); } +"<=" { return operator(PBF_OP_LE); } +"&" { return operator(PBF_OP_AND); } + +na { return constant(PERF_MEM_OP_NA); } +load { return constant(PERF_MEM_OP_LOAD); } +store { return constant(PERF_MEM_OP_STORE); } +pfetch { return constant(PERF_MEM_OP_PFETCH); } +exec { return constant(PERF_MEM_OP_EXEC); } + +l1 { return constant(PERF_MEM_LVLNUM_L1); } +l2 { return constant(PERF_MEM_LVLNUM_L2); } +l3 { return constant(PERF_MEM_LVLNUM_L3); } +l4 { return constant(PERF_MEM_LVLNUM_L4); } +cxl { return constant(PERF_MEM_LVLNUM_CXL); } +io { return constant(PERF_MEM_LVLNUM_IO); } +any_cache { return constant(PERF_MEM_LVLNUM_ANY_CACHE); } +lfb { return constant(PERF_MEM_LVLNUM_LFB); } +ram { return constant(PERF_MEM_LVLNUM_RAM); } +pmem { return constant(PERF_MEM_LVLNUM_PMEM); } + +none { return constant(PERF_MEM_SNOOP_NONE); } +hit { return constant(PERF_MEM_SNOOP_HIT); } +miss { return constant(PERF_MEM_SNOOP_MISS); } +hitm { return constant(PERF_MEM_SNOOP_HITM); } +fwd { return constant(PERF_MEM_SNOOPX_FWD); } +peer { return constant(PERF_MEM_SNOOPX_PEER); } + +remote { return constant(PERF_MEM_REMOTE_REMOTE); } + +locked { return constant(PERF_MEM_LOCK_LOCKED); } + +l1_hit { return constant(PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT); } +l1_miss { return constant(PERF_MEM_TLB_L1 | PERF_MEM_TLB_MISS); } +l2_hit { return constant(PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT); } +l2_miss { return constant(PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS); } +any_hit { return constant(PERF_MEM_TLB_HIT); } +any_miss { return constant(PERF_MEM_TLB_MISS); } +walk { return constant(PERF_MEM_TLB_WK); } +os { return constant(PERF_MEM_TLB_OS); } +fault { return constant(PERF_MEM_TLB_OS); } /* alias for os */ + +by_data { return constant(PERF_MEM_BLK_DATA); } +by_addr { return constant(PERF_MEM_BLK_ADDR); } + +hops0 { return constant(PERF_MEM_HOPS_0); } +hops1 { return constant(PERF_MEM_HOPS_1); } +hops2 { return constant(PERF_MEM_HOPS_2); } +hops3 { return constant(PERF_MEM_HOPS_3); } + +"," { return ','; } +"||" { return BFT_LOGICAL_OR; } + +{path} { return path_or_error(); } + +%% diff --git a/tools/perf/util/bpf-filter.y b/tools/perf/util/bpf-filter.y new file mode 100644 index 000000000..5a79a8e7a --- /dev/null +++ b/tools/perf/util/bpf-filter.y @@ -0,0 +1,109 @@ +%parse-param {struct list_head *expr_head} +%define parse.error verbose + +%{ + +#ifndef NDEBUG +#define YYDEBUG 1 +#endif + +#include +#include +#include +#include +#include "bpf-filter.h" +#include "cgroup.h" + +int perf_bpf_filter_lex(void); + +/* To indicate if the current term needs a pathname or not */ +int perf_bpf_filter_needs_path; + +static void perf_bpf_filter_error(struct list_head *expr __maybe_unused, + char const *msg) +{ + printf("perf_bpf_filter: %s\n", msg); +} + +%} + +%union +{ + unsigned long num; + char *path; + struct { + enum perf_bpf_filter_term term; + int part; + } sample; + enum perf_bpf_filter_op op; + struct perf_bpf_filter_expr *expr; +} + +%token BFT_SAMPLE BFT_SAMPLE_PATH BFT_OP BFT_ERROR BFT_NUM BFT_LOGICAL_OR BFT_PATH +%type filter_term filter_expr +%destructor { free ($$); } +%type BFT_SAMPLE BFT_SAMPLE_PATH +%type BFT_OP +%type BFT_NUM +%type BFT_PATH + +%% + +filter: +filter ',' filter_term +{ + list_add_tail(&$3->list, expr_head); +} +| +filter_term +{ + list_add_tail(&$1->list, expr_head); +} + +filter_term: +filter_term BFT_LOGICAL_OR filter_expr +{ + struct perf_bpf_filter_expr *expr; + + if ($1->op == PBF_OP_GROUP_BEGIN) { + expr = $1; + } else { + expr = perf_bpf_filter_expr__new(PBF_TERM_NONE, /*part=*/0, + PBF_OP_GROUP_BEGIN, /*val=*/1); + list_add_tail(&$1->list, &expr->groups); + } + expr->val++; + list_add_tail(&$3->list, &expr->groups); + $$ = expr; +} +| +filter_expr +{ + $$ = $1; +} + +filter_expr: +BFT_SAMPLE BFT_OP BFT_NUM +{ + $$ = perf_bpf_filter_expr__new($1.term, $1.part, $2, $3); +} +| +BFT_SAMPLE_PATH BFT_OP BFT_PATH +{ + struct cgroup *cgrp; + unsigned long cgroup_id = 0; + + if ($2 != PBF_OP_EQ && $2 != PBF_OP_NEQ) { + printf("perf_bpf_filter: cgroup accepts '==' or '!=' only\n"); + YYERROR; + } + + cgrp = cgroup__new($3, /*do_open=*/false); + if (cgrp && read_cgroup_id(cgrp) == 0) + cgroup_id = cgrp->id; + + $$ = perf_bpf_filter_expr__new($1.term, $1.part, $2, cgroup_id); + cgroup__put(cgrp); +} + +%% diff --git a/tools/perf/util/bpf-trace-summary.c b/tools/perf/util/bpf-trace-summary.c new file mode 100644 index 000000000..9a31dbf06 --- /dev/null +++ b/tools/perf/util/bpf-trace-summary.c @@ -0,0 +1,465 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +#include "dwarf-regs.h" /* for EM_HOST */ +#include "trace/beauty/syscalltbl.h" +#include "util/cgroup.h" +#include "util/hashmap.h" +#include "util/trace.h" +#include "util/util.h" +#include +#include +#include +#include /* reallocarray */ + +#include "bpf_skel/syscall_summary.h" +#include "bpf_skel/syscall_summary.skel.h" + + +static struct syscall_summary_bpf *skel; +static struct rb_root cgroups = RB_ROOT; + +int trace_prepare_bpf_summary(enum trace_summary_mode mode) +{ + skel = syscall_summary_bpf__open(); + if (skel == NULL) { + fprintf(stderr, "failed to open syscall summary bpf skeleton\n"); + return -1; + } + + if (mode == SUMMARY__BY_THREAD) + skel->rodata->aggr_mode = SYSCALL_AGGR_THREAD; + else if (mode == SUMMARY__BY_CGROUP) + skel->rodata->aggr_mode = SYSCALL_AGGR_CGROUP; + else + skel->rodata->aggr_mode = SYSCALL_AGGR_CPU; + + if (cgroup_is_v2("perf_event") > 0) + skel->rodata->use_cgroup_v2 = 1; + + if (syscall_summary_bpf__load(skel) < 0) { + fprintf(stderr, "failed to load syscall summary bpf skeleton\n"); + return -1; + } + + if (syscall_summary_bpf__attach(skel) < 0) { + fprintf(stderr, "failed to attach syscall summary bpf skeleton\n"); + return -1; + } + + if (mode == SUMMARY__BY_CGROUP) + read_all_cgroups(&cgroups); + + return 0; +} + +void trace_start_bpf_summary(void) +{ + skel->bss->enabled = 1; +} + +void trace_end_bpf_summary(void) +{ + skel->bss->enabled = 0; +} + +struct syscall_node { + int syscall_nr; + struct syscall_stats stats; +}; + +static double rel_stddev(struct syscall_stats *stat) +{ + double variance, average; + + if (stat->count < 2) + return 0; + + average = (double)stat->total_time / stat->count; + + variance = stat->squared_sum; + variance -= (stat->total_time * stat->total_time) / stat->count; + variance /= stat->count - 1; + + return 100 * sqrt(variance / stat->count) / average; +} + +/* + * The syscall_data is to maintain syscall stats ordered by total time. + * It supports different summary modes like per-thread or global. + * + * For per-thread stats, it uses two-level data strurcture - + * syscall_data is keyed by TID and has an array of nodes which + * represents each syscall for the thread. + * + * For global stats, it's still two-level technically but we don't need + * per-cpu analysis so it's keyed by the syscall number to combine stats + * from different CPUs. And syscall_data always has a syscall_node so + * it can effectively work as flat hierarchy. + * + * For per-cgroup stats, it uses two-level data structure like thread + * syscall_data is keyed by CGROUP and has an array of node which + * represents each syscall for the cgroup. + */ +struct syscall_data { + u64 key; /* tid if AGGR_THREAD, syscall-nr if AGGR_CPU, cgroup if AGGR_CGROUP */ + int nr_events; + int nr_nodes; + u64 total_time; + struct syscall_node *nodes; +}; + +static int datacmp(const void *a, const void *b) +{ + const struct syscall_data * const *sa = a; + const struct syscall_data * const *sb = b; + + return (*sa)->total_time > (*sb)->total_time ? -1 : 1; +} + +static int nodecmp(const void *a, const void *b) +{ + const struct syscall_node *na = a; + const struct syscall_node *nb = b; + + return na->stats.total_time > nb->stats.total_time ? -1 : 1; +} + +static size_t sc_node_hash(long key, void *ctx __maybe_unused) +{ + return key; +} + +static bool sc_node_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +static int print_common_stats(struct syscall_data *data, int max_summary, FILE *fp) +{ + int printed = 0; + + if (max_summary == 0 || max_summary > data->nr_nodes) + max_summary = data->nr_nodes; + + for (int i = 0; i < max_summary; i++) { + struct syscall_node *node = &data->nodes[i]; + struct syscall_stats *stat = &node->stats; + double total = (double)(stat->total_time) / NSEC_PER_MSEC; + double min = (double)(stat->min_time) / NSEC_PER_MSEC; + double max = (double)(stat->max_time) / NSEC_PER_MSEC; + double avg = total / stat->count; + const char *name; + + /* TODO: support other ABIs */ + name = syscalltbl__name(EM_HOST, node->syscall_nr); + if (name) + printed += fprintf(fp, " %-15s", name); + else + printed += fprintf(fp, " syscall:%-7d", node->syscall_nr); + + printed += fprintf(fp, " %8u %6u %9.3f %9.3f %9.3f %9.3f %9.2f%%\n", + stat->count, stat->error, total, min, avg, max, + rel_stddev(stat)); + } + return printed; +} + +static int update_thread_stats(struct hashmap *hash, struct syscall_key *map_key, + struct syscall_stats *map_data) +{ + struct syscall_data *data; + struct syscall_node *nodes; + + if (!hashmap__find(hash, map_key->cpu_or_tid, &data)) { + data = zalloc(sizeof(*data)); + if (data == NULL) + return -ENOMEM; + + data->key = map_key->cpu_or_tid; + if (hashmap__add(hash, data->key, data) < 0) { + free(data); + return -ENOMEM; + } + } + + /* update thread total stats */ + data->nr_events += map_data->count; + data->total_time += map_data->total_time; + + nodes = reallocarray(data->nodes, data->nr_nodes + 1, sizeof(*nodes)); + if (nodes == NULL) + return -ENOMEM; + + data->nodes = nodes; + nodes = &data->nodes[data->nr_nodes++]; + nodes->syscall_nr = map_key->nr; + + /* each thread has an entry for each syscall, just use the stat */ + memcpy(&nodes->stats, map_data, sizeof(*map_data)); + return 0; +} + +static int print_thread_stat(struct syscall_data *data, int max_summary, FILE *fp) +{ + int printed = 0; + + qsort(data->nodes, data->nr_nodes, sizeof(*data->nodes), nodecmp); + + printed += fprintf(fp, " thread (%d), ", (int)data->key); + printed += fprintf(fp, "%d events\n\n", data->nr_events); + + printed += fprintf(fp, " syscall calls errors total min avg max stddev\n"); + printed += fprintf(fp, " (msec) (msec) (msec) (msec) (%%)\n"); + printed += fprintf(fp, " --------------- -------- ------ -------- --------- --------- --------- ------\n"); + + printed += print_common_stats(data, max_summary, fp); + printed += fprintf(fp, "\n\n"); + + return printed; +} + +static int print_thread_stats(struct syscall_data **data, int nr_data, int max_summary, FILE *fp) +{ + int printed = 0; + + for (int i = 0; i < nr_data; i++) + printed += print_thread_stat(data[i], max_summary, fp); + + return printed; +} + +static int update_total_stats(struct hashmap *hash, struct syscall_key *map_key, + struct syscall_stats *map_data) +{ + struct syscall_data *data; + struct syscall_stats *stat; + + if (!hashmap__find(hash, map_key->nr, &data)) { + data = zalloc(sizeof(*data)); + if (data == NULL) + return -ENOMEM; + + data->nodes = zalloc(sizeof(*data->nodes)); + if (data->nodes == NULL) { + free(data); + return -ENOMEM; + } + + data->nr_nodes = 1; + data->key = map_key->nr; + data->nodes->syscall_nr = data->key; + + if (hashmap__add(hash, data->key, data) < 0) { + free(data->nodes); + free(data); + return -ENOMEM; + } + } + + /* update total stats for this syscall */ + data->nr_events += map_data->count; + data->total_time += map_data->total_time; + + /* This is sum of the same syscall from different CPUs */ + stat = &data->nodes->stats; + + stat->total_time += map_data->total_time; + stat->squared_sum += map_data->squared_sum; + stat->count += map_data->count; + stat->error += map_data->error; + + if (stat->max_time < map_data->max_time) + stat->max_time = map_data->max_time; + if (stat->min_time > map_data->min_time || stat->min_time == 0) + stat->min_time = map_data->min_time; + + return 0; +} + +static int print_total_stats(struct syscall_data **data, int nr_data, int max_summary, FILE *fp) +{ + int printed = 0; + int nr_events = 0; + + for (int i = 0; i < nr_data; i++) + nr_events += data[i]->nr_events; + + printed += fprintf(fp, " total, %d events\n\n", nr_events); + + printed += fprintf(fp, " syscall calls errors total min avg max stddev\n"); + printed += fprintf(fp, " (msec) (msec) (msec) (msec) (%%)\n"); + printed += fprintf(fp, " --------------- -------- ------ -------- --------- --------- --------- ------\n"); + + if (max_summary == 0 || max_summary > nr_data) + max_summary = nr_data; + + for (int i = 0; i < max_summary; i++) + printed += print_common_stats(data[i], max_summary, fp); + + printed += fprintf(fp, "\n\n"); + return printed; +} + +static int update_cgroup_stats(struct hashmap *hash, struct syscall_key *map_key, + struct syscall_stats *map_data) +{ + struct syscall_data *data; + struct syscall_node *nodes; + + if (!hashmap__find(hash, map_key->cgroup, &data)) { + data = zalloc(sizeof(*data)); + if (data == NULL) + return -ENOMEM; + + data->key = map_key->cgroup; + if (hashmap__add(hash, data->key, data) < 0) { + free(data); + return -ENOMEM; + } + } + + /* update thread total stats */ + data->nr_events += map_data->count; + data->total_time += map_data->total_time; + + nodes = reallocarray(data->nodes, data->nr_nodes + 1, sizeof(*nodes)); + if (nodes == NULL) + return -ENOMEM; + + data->nodes = nodes; + nodes = &data->nodes[data->nr_nodes++]; + nodes->syscall_nr = map_key->nr; + + /* each thread has an entry for each syscall, just use the stat */ + memcpy(&nodes->stats, map_data, sizeof(*map_data)); + return 0; +} + +static int print_cgroup_stat(struct syscall_data *data, int max_summary, FILE *fp) +{ + int printed = 0; + struct cgroup *cgrp = __cgroup__find(&cgroups, data->key); + + qsort(data->nodes, data->nr_nodes, sizeof(*data->nodes), nodecmp); + + if (cgrp) + printed += fprintf(fp, " cgroup %s,", cgrp->name); + else + printed += fprintf(fp, " cgroup id:%lu,", (unsigned long)data->key); + + printed += fprintf(fp, " %d events\n\n", data->nr_events); + + printed += fprintf(fp, " syscall calls errors total min avg max stddev\n"); + printed += fprintf(fp, " (msec) (msec) (msec) (msec) (%%)\n"); + printed += fprintf(fp, " --------------- -------- ------ -------- --------- --------- --------- ------\n"); + + printed += print_common_stats(data, max_summary, fp); + printed += fprintf(fp, "\n\n"); + + return printed; +} + +static int print_cgroup_stats(struct syscall_data **data, int nr_data, int max_summary, FILE *fp) +{ + int printed = 0; + + for (int i = 0; i < nr_data; i++) + printed += print_cgroup_stat(data[i], max_summary, fp); + + return printed; +} + +int trace_print_bpf_summary(FILE *fp, int max_summary) +{ + struct bpf_map *map = skel->maps.syscall_stats_map; + struct syscall_key *prev_key, key; + struct syscall_data **data = NULL; + struct hashmap schash; + struct hashmap_entry *entry; + int nr_data = 0; + int printed = 0; + int i; + size_t bkt; + + hashmap__init(&schash, sc_node_hash, sc_node_equal, /*ctx=*/NULL); + + printed = fprintf(fp, "\n Summary of events:\n\n"); + + /* get stats from the bpf map */ + prev_key = NULL; + while (!bpf_map__get_next_key(map, prev_key, &key, sizeof(key))) { + struct syscall_stats stat; + + if (!bpf_map__lookup_elem(map, &key, sizeof(key), &stat, sizeof(stat), 0)) { + switch (skel->rodata->aggr_mode) { + case SYSCALL_AGGR_THREAD: + update_thread_stats(&schash, &key, &stat); + break; + case SYSCALL_AGGR_CPU: + update_total_stats(&schash, &key, &stat); + break; + case SYSCALL_AGGR_CGROUP: + update_cgroup_stats(&schash, &key, &stat); + break; + default: + break; + } + } + + prev_key = &key; + } + + nr_data = hashmap__size(&schash); + data = calloc(nr_data, sizeof(*data)); + if (data == NULL) + goto out; + + i = 0; + hashmap__for_each_entry(&schash, entry, bkt) + data[i++] = entry->pvalue; + + qsort(data, nr_data, sizeof(*data), datacmp); + + switch (skel->rodata->aggr_mode) { + case SYSCALL_AGGR_THREAD: + printed += print_thread_stats(data, nr_data, max_summary, fp); + break; + case SYSCALL_AGGR_CPU: + printed += print_total_stats(data, nr_data, max_summary, fp); + break; + case SYSCALL_AGGR_CGROUP: + printed += print_cgroup_stats(data, nr_data, max_summary, fp); + break; + default: + break; + } + + for (i = 0; i < nr_data && data; i++) { + free(data[i]->nodes); + free(data[i]); + } + free(data); + +out: + hashmap__clear(&schash); + return printed; +} + +void trace_cleanup_bpf_summary(void) +{ + if (!RB_EMPTY_ROOT(&cgroups)) { + struct cgroup *cgrp, *tmp; + + rbtree_postorder_for_each_entry_safe(cgrp, tmp, &cgroups, node) + cgroup__put(cgrp); + + cgroups = RB_ROOT; + } + + syscall_summary_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf-utils.c b/tools/perf/util/bpf-utils.c new file mode 100644 index 000000000..69148197b --- /dev/null +++ b/tools/perf/util/bpf-utils.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include "bpf-utils.h" +#include "debug.h" + +struct bpil_array_desc { + int array_offset; /* e.g. offset of jited_prog_insns */ + int count_offset; /* e.g. offset of jited_prog_len */ + int size_offset; /* > 0: offset of rec size, + * < 0: fix size of -size_offset + */ +}; + +static const struct bpil_array_desc bpil_array_desc[] = { + [PERF_BPIL_JITED_INSNS] = { + offsetof(struct bpf_prog_info, jited_prog_insns), + offsetof(struct bpf_prog_info, jited_prog_len), + -1, + }, + [PERF_BPIL_XLATED_INSNS] = { + offsetof(struct bpf_prog_info, xlated_prog_insns), + offsetof(struct bpf_prog_info, xlated_prog_len), + -1, + }, + [PERF_BPIL_MAP_IDS] = { + offsetof(struct bpf_prog_info, map_ids), + offsetof(struct bpf_prog_info, nr_map_ids), + -(int)sizeof(__u32), + }, + [PERF_BPIL_JITED_KSYMS] = { + offsetof(struct bpf_prog_info, jited_ksyms), + offsetof(struct bpf_prog_info, nr_jited_ksyms), + -(int)sizeof(__u64), + }, + [PERF_BPIL_JITED_FUNC_LENS] = { + offsetof(struct bpf_prog_info, jited_func_lens), + offsetof(struct bpf_prog_info, nr_jited_func_lens), + -(int)sizeof(__u32), + }, + [PERF_BPIL_FUNC_INFO] = { + offsetof(struct bpf_prog_info, func_info), + offsetof(struct bpf_prog_info, nr_func_info), + offsetof(struct bpf_prog_info, func_info_rec_size), + }, + [PERF_BPIL_LINE_INFO] = { + offsetof(struct bpf_prog_info, line_info), + offsetof(struct bpf_prog_info, nr_line_info), + offsetof(struct bpf_prog_info, line_info_rec_size), + }, + [PERF_BPIL_JITED_LINE_INFO] = { + offsetof(struct bpf_prog_info, jited_line_info), + offsetof(struct bpf_prog_info, nr_jited_line_info), + offsetof(struct bpf_prog_info, jited_line_info_rec_size), + }, + [PERF_BPIL_PROG_TAGS] = { + offsetof(struct bpf_prog_info, prog_tags), + offsetof(struct bpf_prog_info, nr_prog_tags), + -(int)sizeof(__u8) * BPF_TAG_SIZE, + }, + +}; + +static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, + int offset) +{ + __u32 *array = (__u32 *)info; + + if (offset >= 0) + return array[offset / sizeof(__u32)]; + return -(int)offset; +} + +static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, + int offset) +{ + __u64 *array = (__u64 *)info; + + if (offset >= 0) + return array[offset / sizeof(__u64)]; + return -(int)offset; +} + +static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, + __u32 val) +{ + __u32 *array = (__u32 *)info; + + if (offset >= 0) + array[offset / sizeof(__u32)] = val; +} + +static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, + __u64 val) +{ + __u64 *array = (__u64 *)info; + + if (offset >= 0) + array[offset / sizeof(__u64)] = val; +} + +struct perf_bpil * +get_bpf_prog_info_linear(int fd, __u64 arrays) +{ + struct bpf_prog_info info = {}; + struct perf_bpil *info_linear; + __u32 info_len = sizeof(info); + __u32 data_len = 0; + int i, err; + __u8 *ptr; + + if (arrays >> PERF_BPIL_LAST_ARRAY) + return ERR_PTR(-EINVAL); + + /* step 1: get array dimensions */ + err = bpf_obj_get_info_by_fd(fd, &info, &info_len); + if (err) { + pr_debug("can't get prog info: %m\n"); + return ERR_PTR(-EFAULT); + } + if (info.type >= __MAX_BPF_PROG_TYPE) + pr_debug("%s:%d: unexpected program type %u\n", __func__, __LINE__, info.type); + + /* step 2: calculate total size of all arrays */ + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + bool include_array = (arrays & (1UL << i)) > 0; + __u32 count, size; + + /* kernel is too old to support this field */ + if (info_len < desc->array_offset + sizeof(__u32) || + info_len < desc->count_offset + sizeof(__u32) || + (desc->size_offset > 0 && info_len < (__u32)desc->size_offset)) + include_array = false; + + if (!include_array) { + arrays &= ~(1UL << i); /* clear the bit */ + continue; + } + + count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); + + data_len += roundup(count * size, sizeof(__u64)); + } + + /* step 3: allocate continuous memory */ + info_linear = malloc(sizeof(struct perf_bpil) + data_len); + if (!info_linear) + return ERR_PTR(-ENOMEM); + + /* step 4: fill data to info_linear->info */ + info_linear->arrays = arrays; + memset(&info_linear->info, 0, sizeof(info)); + ptr = info_linear->data; + + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u32 count, size; + + if ((arrays & (1UL << i)) == 0) + continue; + + count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, count); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->size_offset, size); + assert(ptr >= info_linear->data); + assert(ptr < &info_linear->data[data_len]); + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, + ptr_to_u64(ptr)); + ptr += roundup(count * size, sizeof(__u64)); + } + + /* step 5: call syscall again to get required arrays */ + err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); + if (err) { + pr_debug("can't get prog info: %m\n"); + free(info_linear); + return ERR_PTR(-EFAULT); + } + if (info_linear->info.type >= __MAX_BPF_PROG_TYPE) { + pr_debug("%s:%d: unexpected program type %u\n", + __func__, __LINE__, info_linear->info.type); + } + + /* step 6: verify the data */ + ptr = info_linear->data; + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u32 count1, count2, size1, size2; + __u64 ptr2; + + if ((arrays & (1UL << i)) == 0) + continue; + + count1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); + count2 = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + if (count1 != count2) { + pr_warning("%s: mismatch in element count %u vs %u\n", __func__, count1, count2); + free(info_linear); + return ERR_PTR(-ERANGE); + } + + size1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); + size2 = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + if (size1 != size2) { + pr_warning("%s: mismatch in rec size %u vs %u\n", __func__, size1, size2); + free(info_linear); + return ERR_PTR(-ERANGE); + } + ptr2 = bpf_prog_info_read_offset_u64(&info_linear->info, desc->array_offset); + if (ptr_to_u64(ptr) != ptr2) { + pr_warning("%s: mismatch in array %p vs %llx\n", __func__, ptr, ptr2); + free(info_linear); + return ERR_PTR(-ERANGE); + } + ptr += roundup(count1 * size1, sizeof(__u64)); + } + + /* step 7: update info_len and data_len */ + info_linear->info_len = sizeof(struct bpf_prog_info); + info_linear->data_len = data_len; + + return info_linear; +} + +void bpil_addr_to_offs(struct perf_bpil *info_linear) +{ + int i; + + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u64 addr, offs; + + if ((info_linear->arrays & (1UL << i)) == 0) + continue; + + addr = bpf_prog_info_read_offset_u64(&info_linear->info, + desc->array_offset); + offs = addr - ptr_to_u64(info_linear->data); + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, offs); + } +} + +void bpil_offs_to_addr(struct perf_bpil *info_linear) +{ + int i; + + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u64 addr, offs; + __u32 count, size; + + if ((info_linear->arrays & (1UL << i)) == 0) + continue; + + offs = bpf_prog_info_read_offset_u64(&info_linear->info, + desc->array_offset); + count = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + /* offset and extent from perf.data are untrusted — keep within data[] */ + if (offs >= info_linear->data_len || + (u64)count * size > info_linear->data_len - offs) { + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, 0); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, 0); + /* clear the bit so bpil_addr_to_offs() won't reverse a zeroed address */ + info_linear->arrays &= ~(1UL << i); + continue; + } + addr = offs + ptr_to_u64(info_linear->data); + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, addr); + } +} diff --git a/tools/perf/util/bpf-utils.h b/tools/perf/util/bpf-utils.h new file mode 100644 index 000000000..a8bc1a232 --- /dev/null +++ b/tools/perf/util/bpf-utils.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +#ifndef __PERF_BPF_UTILS_H +#define __PERF_BPF_UTILS_H + +#define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr)) + +#ifdef HAVE_LIBBPF_SUPPORT + +#include +#include + +#define LIBBPF_CURRENT_VERSION_GEQ(major, minor) \ + (LIBBPF_MAJOR_VERSION > (major) || \ + (LIBBPF_MAJOR_VERSION == (major) && LIBBPF_MINOR_VERSION >= (minor))) + +#if LIBBPF_CURRENT_VERSION_GEQ(1, 7) +// libbpf 1.7+ support the btf_dump_type_data_opts.emit_strings option. +#define HAVE_LIBBPF_STRINGS_SUPPORT 1 +#endif + +/* + * Get bpf_prog_info in continuous memory + * + * struct bpf_prog_info has multiple arrays. The user has option to choose + * arrays to fetch from kernel. The following APIs provide an uniform way to + * fetch these data. All arrays in bpf_prog_info are stored in a single + * continuous memory region. This makes it easy to store the info in a + * file. + * + * Before writing perf_bpil to files, it is necessary to + * translate pointers in bpf_prog_info to offsets. Helper functions + * bpil_addr_to_offs() and bpil_offs_to_addr() + * are introduced to switch between pointers and offsets. + * + * Examples: + * # To fetch map_ids and prog_tags: + * __u64 arrays = (1UL << PERF_BPIL_MAP_IDS) | + * (1UL << PERF_BPIL_PROG_TAGS); + * struct perf_bpil *info_linear = + * get_bpf_prog_info_linear(fd, arrays); + * + * # To save data in file + * bpil_addr_to_offs(info_linear); + * write(f, info_linear, sizeof(*info_linear) + info_linear->data_len); + * + * # To read data from file + * read(f, info_linear, ); + * bpil_offs_to_addr(info_linear); + */ +enum perf_bpil_array_types { + PERF_BPIL_FIRST_ARRAY = 0, + PERF_BPIL_JITED_INSNS = 0, + PERF_BPIL_XLATED_INSNS, + PERF_BPIL_MAP_IDS, + PERF_BPIL_JITED_KSYMS, + PERF_BPIL_JITED_FUNC_LENS, + PERF_BPIL_FUNC_INFO, + PERF_BPIL_LINE_INFO, + PERF_BPIL_JITED_LINE_INFO, + PERF_BPIL_PROG_TAGS, + PERF_BPIL_LAST_ARRAY, +}; + +struct perf_bpil { + /* size of struct bpf_prog_info, when the tool is compiled */ + __u32 info_len; + /* total bytes allocated for data, round up to 8 bytes */ + __u32 data_len; + /* which arrays are included in data */ + __u64 arrays; + struct bpf_prog_info info; + __u8 data[]; +}; + +struct perf_bpil * +get_bpf_prog_info_linear(int fd, __u64 arrays); + +void +bpil_addr_to_offs(struct perf_bpil *info_linear); + +void +bpil_offs_to_addr(struct perf_bpil *info_linear); + +#endif /* HAVE_LIBBPF_SUPPORT */ +#endif /* __PERF_BPF_UTILS_H */ diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c new file mode 100644 index 000000000..9362e45e1 --- /dev/null +++ b/tools/perf/util/bpf_counter.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (c) 2019 Facebook */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bpf_counter.h" +#include "bpf-utils.h" +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "evlist.h" +#include "target.h" +#include "cgroup.h" +#include "cpumap.h" +#include "thread_map.h" + +#include "bpf_skel/bpf_prog_profiler.skel.h" +#include "bpf_skel/bperf_u.h" +#include "bpf_skel/bperf_leader.skel.h" +#include "bpf_skel/bperf_follower.skel.h" + +struct bpf_counter { + void *skel; + struct list_head list; +}; + +#define ATTR_MAP_SIZE 16 + +static void *u64_to_ptr(__u64 ptr) +{ + return (void *)(unsigned long)ptr; +} + + +void set_max_rlimit(void) +{ + struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; + + setrlimit(RLIMIT_MEMLOCK, &rinf); +} + +static __u32 bpf_link_get_id(int fd) +{ + struct bpf_link_info link_info = { .id = 0, }; + __u32 link_info_len = sizeof(link_info); + + bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); + return link_info.id; +} + +static __u32 bpf_link_get_prog_id(int fd) +{ + struct bpf_link_info link_info = { .id = 0, }; + __u32 link_info_len = sizeof(link_info); + + bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); + return link_info.prog_id; +} + +static __u32 bpf_map_get_id(int fd) +{ + struct bpf_map_info map_info = { .id = 0, }; + __u32 map_info_len = sizeof(map_info); + + bpf_obj_get_info_by_fd(fd, &map_info, &map_info_len); + return map_info.id; +} + +/* trigger the leader program on a cpu */ +int bperf_trigger_reading(int prog_fd, int cpu) +{ + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = NULL, + .ctx_size_in = 0, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = cpu, + .retval = 0, + ); + + return bpf_prog_test_run_opts(prog_fd, &opts); +} + +static struct bpf_counter *bpf_counter_alloc(void) +{ + struct bpf_counter *counter; + + counter = zalloc(sizeof(*counter)); + if (counter) + INIT_LIST_HEAD(&counter->list); + return counter; +} + +static int bpf_program_profiler__destroy(struct evsel *evsel) +{ + struct bpf_counter *counter, *tmp; + + list_for_each_entry_safe(counter, tmp, + &evsel->bpf_counter_list, list) { + list_del_init(&counter->list); + bpf_prog_profiler_bpf__destroy(counter->skel); + free(counter); + } + assert(list_empty(&evsel->bpf_counter_list)); + + return 0; +} + +static char *bpf_target_prog_name(int tgt_fd) +{ + struct bpf_func_info *func_info; + struct perf_bpil *info_linear; + const struct btf_type *t; + struct btf *btf = NULL; + char *name = NULL; + + info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO); + if (IS_ERR_OR_NULL(info_linear)) { + pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd); + return NULL; + } + + if (info_linear->info.btf_id == 0) { + pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd); + goto out; + } + + btf = btf__load_from_kernel_by_id(info_linear->info.btf_id); + if (libbpf_get_error(btf)) { + pr_debug("failed to load btf for prog FD %d\n", tgt_fd); + goto out; + } + + func_info = u64_to_ptr(info_linear->info.func_info); + t = btf__type_by_id(btf, func_info[0].type_id); + if (!t) { + pr_debug("btf %d doesn't have type %d\n", + info_linear->info.btf_id, func_info[0].type_id); + goto out; + } + name = strdup(btf__name_by_offset(btf, t->name_off)); +out: + btf__free(btf); + free(info_linear); + return name; +} + +static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) +{ + struct bpf_prog_profiler_bpf *skel; + struct bpf_counter *counter; + struct bpf_program *prog; + char *prog_name = NULL; + int prog_fd; + int err; + + prog_fd = bpf_prog_get_fd_by_id(prog_id); + if (prog_fd < 0) { + pr_err("Failed to open fd for bpf prog %u\n", prog_id); + return -1; + } + counter = bpf_counter_alloc(); + if (!counter) { + close(prog_fd); + return -1; + } + + skel = bpf_prog_profiler_bpf__open(); + if (!skel) { + pr_err("Failed to open bpf skeleton\n"); + goto err_out; + } + + skel->rodata->num_cpu = evsel__nr_cpus(evsel); + + bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel)); + bpf_map__set_max_entries(skel->maps.fentry_readings, 1); + bpf_map__set_max_entries(skel->maps.accum_readings, 1); + + prog_name = bpf_target_prog_name(prog_fd); + if (!prog_name) { + pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id); + goto err_out; + } + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_attach_target(prog, prog_fd, prog_name); + if (err) { + pr_err("bpf_program__set_attach_target failed.\n" + "Does bpf prog %u have BTF?\n", prog_id); + goto err_out; + } + } + set_max_rlimit(); + err = bpf_prog_profiler_bpf__load(skel); + if (err) { + pr_err("bpf_prog_profiler_bpf__load failed\n"); + goto err_out; + } + + assert(skel != NULL); + counter->skel = skel; + list_add(&counter->list, &evsel->bpf_counter_list); + free(prog_name); + close(prog_fd); + return 0; +err_out: + bpf_prog_profiler_bpf__destroy(skel); + free(prog_name); + free(counter); + close(prog_fd); + return -1; +} + +static int bpf_program_profiler__load(struct evsel *evsel, struct target *target) +{ + char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p; + u32 prog_id; + int ret; + + bpf_str_ = bpf_str = strdup(target->bpf_str); + if (!bpf_str) + return -1; + + while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) { + prog_id = strtoul(tok, &p, 10); + if (prog_id == 0 || prog_id == UINT_MAX || + (*p != '\0' && *p != ',')) { + pr_err("Failed to parse bpf prog ids %s\n", + target->bpf_str); + free(bpf_str_); + return -1; + } + + ret = bpf_program_profiler_load_one(evsel, prog_id); + if (ret) { + bpf_program_profiler__destroy(evsel); + free(bpf_str_); + return -1; + } + bpf_str = NULL; + } + free(bpf_str_); + return 0; +} + +static int bpf_program_profiler__enable(struct evsel *evsel) +{ + struct bpf_counter *counter; + int ret; + + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + assert(counter->skel != NULL); + ret = bpf_prog_profiler_bpf__attach(counter->skel); + if (ret) { + bpf_program_profiler__destroy(evsel); + return ret; + } + } + return 0; +} + +static int bpf_program_profiler__disable(struct evsel *evsel) +{ + struct bpf_counter *counter; + + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + assert(counter->skel != NULL); + bpf_prog_profiler_bpf__detach(counter->skel); + } + return 0; +} + +static int bpf_program_profiler__read(struct evsel *evsel) +{ + // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible + // Sometimes possible > online, like on a Ryzen 3900X that has 24 + // threads but its possible showed 0-31 -acme + int num_cpu_bpf = libbpf_num_possible_cpus(); + struct bpf_perf_event_value values[num_cpu_bpf]; + struct bpf_counter *counter; + struct perf_counts_values *counts; + int reading_map_fd; + __u32 key = 0; + int err, bpf_cpu; + unsigned int idx; + + if (list_empty(&evsel->bpf_counter_list)) + return -EAGAIN; + + perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) { + counts = perf_counts(evsel->counts, idx, 0); + counts->val = 0; + counts->ena = 0; + counts->run = 0; + } + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + struct bpf_prog_profiler_bpf *skel = counter->skel; + + assert(skel != NULL); + reading_map_fd = bpf_map__fd(skel->maps.accum_readings); + + err = bpf_map_lookup_elem(reading_map_fd, &key, values); + if (err) { + pr_err("failed to read value\n"); + return err; + } + + for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) { + int i = perf_cpu_map__idx(evsel__cpus(evsel), + (struct perf_cpu){.cpu = bpf_cpu}); + + if (i == -1) + continue; + counts = perf_counts(evsel->counts, i, 0); + counts->val += values[bpf_cpu].counter; + counts->ena += values[bpf_cpu].enabled; + counts->run += values[bpf_cpu].running; + } + } + return 0; +} + +static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx, + int fd) +{ + struct bpf_prog_profiler_bpf *skel; + struct bpf_counter *counter; + int cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx).cpu; + int ret; + + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + skel = counter->skel; + assert(skel != NULL); + + ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), + &cpu, &fd, BPF_ANY); + if (ret) + return ret; + } + return 0; +} + +static struct bpf_counter_ops bpf_program_profiler_ops = { + .load = bpf_program_profiler__load, + .enable = bpf_program_profiler__enable, + .disable = bpf_program_profiler__disable, + .read = bpf_program_profiler__read, + .destroy = bpf_program_profiler__destroy, + .install_pe = bpf_program_profiler__install_pe, +}; + +static bool bperf_attr_map_compatible(int attr_map_fd) +{ + struct bpf_map_info map_info = {0}; + __u32 map_info_len = sizeof(map_info); + int err; + + err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len); + + if (err) + return false; + return (map_info.key_size == sizeof(struct perf_event_attr)) && + (map_info.value_size == sizeof(struct perf_event_attr_map_entry)); +} + +static int bperf_lock_attr_map(struct target *target) +{ + char path[PATH_MAX]; + int map_fd, err; + + if (target->attr_map) { + scnprintf(path, PATH_MAX, "%s", target->attr_map); + } else { + scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(), + BPF_PERF_DEFAULT_ATTR_MAP_PATH); + } + + if (access(path, F_OK)) { + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, + sizeof(struct perf_event_attr), + sizeof(struct perf_event_attr_map_entry), + ATTR_MAP_SIZE, NULL); + if (map_fd < 0) + return -1; + + err = bpf_obj_pin(map_fd, path); + if (err) { + /* someone pinned the map in parallel? */ + close(map_fd); + map_fd = bpf_obj_get(path); + if (map_fd < 0) + return -1; + } + } else { + map_fd = bpf_obj_get(path); + if (map_fd < 0) + return -1; + } + + if (!bperf_attr_map_compatible(map_fd)) { + close(map_fd); + return -1; + + } + err = flock(map_fd, LOCK_EX); + if (err) { + close(map_fd); + return -1; + } + return map_fd; +} + +static int bperf_check_target(struct evsel *evsel, + struct target *target, + enum bperf_filter_type *filter_type, + __u32 *filter_entry_cnt) +{ + if (evsel->core.leader->nr_members > 1) { + pr_err("bpf managed perf events do not yet support groups.\n"); + return -1; + } + + /* determine filter type based on target */ + if (target->system_wide) { + *filter_type = BPERF_FILTER_GLOBAL; + *filter_entry_cnt = 1; + } else if (target->cpu_list) { + *filter_type = BPERF_FILTER_CPU; + *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel)); + } else if (target->tid) { + *filter_type = BPERF_FILTER_PID; + *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); + } else if (target->pid || evlist__workload_pid(evsel->evlist) != -1) { + *filter_type = BPERF_FILTER_TGID; + *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); + } else { + pr_err("bpf managed perf events do not yet support these targets.\n"); + return -1; + } + + return 0; +} + +static __u32 filter_entry_cnt; + +static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, + struct perf_event_attr_map_entry *entry) +{ + struct bperf_leader_bpf *skel = bperf_leader_bpf__open(); + int link_fd, diff_map_fd, err; + struct bpf_link *link = NULL; + struct perf_thread_map *threads; + + if (!skel) { + pr_err("Failed to open leader skeleton\n"); + return -1; + } + + bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus()); + err = bperf_leader_bpf__load(skel); + if (err) { + pr_err("Failed to load leader skeleton\n"); + goto out; + } + + link = bpf_program__attach(skel->progs.on_switch); + if (IS_ERR(link)) { + pr_err("Failed to attach leader program\n"); + err = PTR_ERR(link); + goto out; + } + + link_fd = bpf_link__fd(link); + diff_map_fd = bpf_map__fd(skel->maps.diff_readings); + entry->link_id = bpf_link_get_id(link_fd); + entry->diff_map_id = bpf_map_get_id(diff_map_fd); + err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY); + assert(err == 0); + + evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id); + assert(evsel->bperf_leader_link_fd >= 0); + + /* + * save leader_skel for install_pe, which is called within + * following evsel__open_per_cpu call + */ + evsel->leader_skel = skel; + assert(!perf_cpu_map__has_any_cpu_or_is_empty(evsel->core.cpus)); + /* Always open system wide. */ + threads = thread_map__new_by_tid(-1); + evsel__open(evsel, evsel->core.cpus, threads); + perf_thread_map__put(threads); + +out: + bperf_leader_bpf__destroy(skel); + bpf_link__destroy(link); + return err; +} + +static int bperf_attach_follower_program(struct bperf_follower_bpf *skel, + enum bperf_filter_type filter_type, + bool inherit) +{ + struct bpf_link *link; + int err = 0; + + if ((filter_type == BPERF_FILTER_PID || + filter_type == BPERF_FILTER_TGID) && inherit) + /* attach all follower bpf progs to enable event inheritance */ + err = bperf_follower_bpf__attach(skel); + else { + link = bpf_program__attach(skel->progs.fexit_XXX); + if (IS_ERR(link)) + err = PTR_ERR(link); + } + + return err; +} + +static int bperf__load(struct evsel *evsel, struct target *target) +{ + struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff}; + int attr_map_fd, diff_map_fd = -1, err; + enum bperf_filter_type filter_type; + __u32 i; + + if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt)) + return -1; + + evsel->bperf_leader_prog_fd = -1; + evsel->bperf_leader_link_fd = -1; + + /* + * Step 1: hold a fd on the leader program and the bpf_link, if + * the program is not already gone, reload the program. + * Use flock() to ensure exclusive access to the perf_event_attr + * map. + */ + attr_map_fd = bperf_lock_attr_map(target); + if (attr_map_fd < 0) { + pr_err("Failed to lock perf_event_attr map\n"); + return -1; + } + + err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry); + if (err) { + err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY); + if (err) + goto out; + } + + evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id); + if (evsel->bperf_leader_link_fd < 0 && + bperf_reload_leader_program(evsel, attr_map_fd, &entry)) { + err = -1; + goto out; + } + /* + * The bpf_link holds reference to the leader program, and the + * leader program holds reference to the maps. Therefore, if + * link_id is valid, diff_map_id should also be valid. + */ + evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id( + bpf_link_get_prog_id(evsel->bperf_leader_link_fd)); + assert(evsel->bperf_leader_prog_fd >= 0); + + diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id); + assert(diff_map_fd >= 0); + + /* + * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check + * whether the kernel support it + */ + err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0); + if (err) { + pr_err("The kernel does not support test_run for raw_tp BPF programs.\n" + "Therefore, --use-bpf might show inaccurate readings\n"); + goto out; + } + + /* Step 2: load the follower skeleton */ + evsel->follower_skel = bperf_follower_bpf__open(); + if (!evsel->follower_skel) { + err = -1; + pr_err("Failed to open follower skeleton\n"); + goto out; + } + + /* attach fexit program to the leader program */ + bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX, + evsel->bperf_leader_prog_fd, "on_switch"); + + /* connect to leader diff_reading map */ + bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd); + + /* set up reading map */ + bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings, + filter_entry_cnt); + err = bperf_follower_bpf__load(evsel->follower_skel); + if (err) { + pr_err("Failed to load follower skeleton\n"); + bperf_follower_bpf__destroy(evsel->follower_skel); + evsel->follower_skel = NULL; + goto out; + } + + for (i = 0; i < filter_entry_cnt; i++) { + int filter_map_fd; + __u32 key; + struct bperf_filter_value fval = { i, 0 }; + + if (filter_type == BPERF_FILTER_PID || + filter_type == BPERF_FILTER_TGID) + key = perf_thread_map__pid(evsel->core.threads, i); + else if (filter_type == BPERF_FILTER_CPU) + key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu; + else + break; + + filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter); + bpf_map_update_elem(filter_map_fd, &key, &fval, BPF_ANY); + } + + evsel->follower_skel->bss->type = filter_type; + evsel->follower_skel->bss->inherit = target->inherit; + + err = bperf_attach_follower_program(evsel->follower_skel, filter_type, + target->inherit); + +out: + if (err && evsel->bperf_leader_link_fd >= 0) + close(evsel->bperf_leader_link_fd); + if (err && evsel->bperf_leader_prog_fd >= 0) + close(evsel->bperf_leader_prog_fd); + if (diff_map_fd >= 0) + close(diff_map_fd); + + flock(attr_map_fd, LOCK_UN); + close(attr_map_fd); + + return err; +} + +static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) +{ + struct bperf_leader_bpf *skel = evsel->leader_skel; + int cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx).cpu; + + return bpf_map_update_elem(bpf_map__fd(skel->maps.events), + &cpu, &fd, BPF_ANY); +} + +/* + * trigger the leader prog on each cpu, so the accum_reading map could get + * the latest readings. + */ +static int bperf_sync_counters(struct evsel *evsel) +{ + struct perf_cpu cpu; + unsigned int idx; + + perf_cpu_map__for_each_cpu(cpu, idx, evsel->core.cpus) + bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu.cpu); + + return 0; +} + +static int bperf__enable(struct evsel *evsel) +{ + evsel->follower_skel->bss->enabled = 1; + return 0; +} + +static int bperf__disable(struct evsel *evsel) +{ + evsel->follower_skel->bss->enabled = 0; + return 0; +} + +static int bperf__read(struct evsel *evsel) +{ + struct bperf_follower_bpf *skel = evsel->follower_skel; + __u32 num_cpu_bpf = cpu__max_cpu().cpu; + struct bpf_perf_event_value values[num_cpu_bpf]; + struct perf_counts_values *counts; + int reading_map_fd, err = 0; + + bperf_sync_counters(evsel); + reading_map_fd = bpf_map__fd(skel->maps.accum_readings); + + for (__u32 i = 0; i < filter_entry_cnt; i++) { + struct perf_cpu entry; + __u32 cpu; + + err = bpf_map_lookup_elem(reading_map_fd, &i, values); + if (err) + goto out; + switch (evsel->follower_skel->bss->type) { + case BPERF_FILTER_GLOBAL: { + unsigned int j; + + assert(i == 0); + perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) { + counts = perf_counts(evsel->counts, j, 0); + counts->val = values[entry.cpu].counter; + counts->ena = values[entry.cpu].enabled; + counts->run = values[entry.cpu].running; + } + break; + } + case BPERF_FILTER_CPU: + cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu; + assert(cpu >= 0); + counts = perf_counts(evsel->counts, i, 0); + counts->val = values[cpu].counter; + counts->ena = values[cpu].enabled; + counts->run = values[cpu].running; + break; + case BPERF_FILTER_PID: + case BPERF_FILTER_TGID: + counts = perf_counts(evsel->counts, 0, i); + counts->val = 0; + counts->ena = 0; + counts->run = 0; + + for (cpu = 0; cpu < num_cpu_bpf; cpu++) { + counts->val += values[cpu].counter; + counts->ena += values[cpu].enabled; + counts->run += values[cpu].running; + } + break; + default: + break; + } + } +out: + return err; +} + +static int bperf__destroy(struct evsel *evsel) +{ + bperf_follower_bpf__destroy(evsel->follower_skel); + close(evsel->bperf_leader_prog_fd); + close(evsel->bperf_leader_link_fd); + return 0; +} + +/* + * bperf: share hardware PMCs with BPF + * + * perf uses performance monitoring counters (PMC) to monitor system + * performance. The PMCs are limited hardware resources. For example, + * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu. + * + * Modern data center systems use these PMCs in many different ways: + * system level monitoring, (maybe nested) container level monitoring, per + * process monitoring, profiling (in sample mode), etc. In some cases, + * there are more active perf_events than available hardware PMCs. To allow + * all perf_events to have a chance to run, it is necessary to do expensive + * time multiplexing of events. + * + * On the other hand, many monitoring tools count the common metrics + * (cycles, instructions). It is a waste to have multiple tools create + * multiple perf_events of "cycles" and occupy multiple PMCs. + * + * bperf tries to reduce such wastes by allowing multiple perf_events of + * "cycles" or "instructions" (at different scopes) to share PMUs. Instead + * of having each perf-stat session to read its own perf_events, bperf uses + * BPF programs to read the perf_events and aggregate readings to BPF maps. + * Then, the perf-stat session(s) reads the values from these BPF maps. + * + * || + * shared progs and maps <- || -> per session progs and maps + * || + * --------------- || + * | perf_events | || + * --------------- fexit || ----------------- + * | --------||----> | follower prog | + * --------------- / || --- ----------------- + * cs -> | leader prog |/ ||/ | | + * --> --------------- /|| -------------- ------------------ + * / | | / || | filter map | | accum_readings | + * / ------------ ------------ || -------------- ------------------ + * | | prev map | | diff map | || | + * | ------------ ------------ || | + * \ || | + * = \ ==================================================== | ============ + * \ / user space + * \ / + * \ / + * BPF_PROG_TEST_RUN BPF_MAP_LOOKUP_ELEM + * \ / + * \ / + * \------ perf-stat ----------------------/ + * + * The figure above shows the architecture of bperf. Note that the figure + * is divided into 3 regions: shared progs and maps (top left), per session + * progs and maps (top right), and user space (bottom). + * + * The leader prog is triggered on each context switch (cs). The leader + * prog reads perf_events and stores the difference (current_reading - + * previous_reading) to the diff map. For the same metric, e.g. "cycles", + * multiple perf-stat sessions share the same leader prog. + * + * Each perf-stat session creates a follower prog as fexit program to the + * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38) + * follower progs to the same leader prog. The follower prog checks current + * task and processor ID to decide whether to add the value from the diff + * map to its accumulated reading map (accum_readings). + * + * Finally, perf-stat user space reads the value from accum_reading map. + * + * Besides context switch, it is also necessary to trigger the leader prog + * before perf-stat reads the value. Otherwise, the accum_reading map may + * not have the latest reading from the perf_events. This is achieved by + * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU. + * + * Comment before the definition of struct perf_event_attr_map_entry + * describes how different sessions of perf-stat share information about + * the leader prog. + */ + +static struct bpf_counter_ops bperf_ops = { + .load = bperf__load, + .enable = bperf__enable, + .disable = bperf__disable, + .read = bperf__read, + .install_pe = bperf__install_pe, + .destroy = bperf__destroy, +}; + +extern struct bpf_counter_ops bperf_cgrp_ops; + +static bool bpf_counter_skip(struct evsel *evsel) +{ + return evsel->bpf_counter_ops == NULL; +} + +int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) +{ + if (bpf_counter_skip(evsel)) + return 0; + return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd); +} + +int bpf_counter__load(struct evsel *evsel, struct target *target) +{ + if (target->bpf_str) + evsel->bpf_counter_ops = &bpf_program_profiler_ops; + else if (cgrp_event_expanded && target->use_bpf) + evsel->bpf_counter_ops = &bperf_cgrp_ops; + else if (target->use_bpf || evsel->bpf_counter || + evsel__match_bpf_counter_events(evsel->name)) + evsel->bpf_counter_ops = &bperf_ops; + + if (evsel->bpf_counter_ops) + return evsel->bpf_counter_ops->load(evsel, target); + return 0; +} + +int bpf_counter__enable(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return 0; + return evsel->bpf_counter_ops->enable(evsel); +} + +int bpf_counter__disable(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return 0; + return evsel->bpf_counter_ops->disable(evsel); +} + +int bpf_counter__read(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return -EAGAIN; + return evsel->bpf_counter_ops->read(evsel); +} + +void bpf_counter__destroy(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return; + evsel->bpf_counter_ops->destroy(evsel); + evsel->bpf_counter_ops = NULL; + evsel->bpf_skel = NULL; +} diff --git a/tools/perf/util/bpf_counter.h b/tools/perf/util/bpf_counter.h new file mode 100644 index 000000000..658d8e7d5 --- /dev/null +++ b/tools/perf/util/bpf_counter.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BPF_COUNTER_H +#define __PERF_BPF_COUNTER_H 1 + +struct evsel; +struct target; + +#ifdef HAVE_BPF_SKEL + +typedef int (*bpf_counter_evsel_op)(struct evsel *evsel); +typedef int (*bpf_counter_evsel_target_op)(struct evsel *evsel, + struct target *target); +typedef int (*bpf_counter_evsel_install_pe_op)(struct evsel *evsel, + int cpu_map_idx, + int fd); + +/* Shared ops between bpf_counter, bpf_counter_cgroup, etc. */ +struct bpf_counter_ops { + bpf_counter_evsel_target_op load; + bpf_counter_evsel_op enable; + bpf_counter_evsel_op disable; + bpf_counter_evsel_op read; + bpf_counter_evsel_op destroy; + bpf_counter_evsel_install_pe_op install_pe; +}; + +int bpf_counter__load(struct evsel *evsel, struct target *target); +int bpf_counter__enable(struct evsel *evsel); +int bpf_counter__disable(struct evsel *evsel); +int bpf_counter__read(struct evsel *evsel); +void bpf_counter__destroy(struct evsel *evsel); +int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd); + +int bperf_trigger_reading(int prog_fd, int cpu); +void set_max_rlimit(void); + +#else /* HAVE_BPF_SKEL */ + +#include + +static inline int bpf_counter__load(struct evsel *evsel __maybe_unused, + struct target *target __maybe_unused) +{ + return 0; +} + +static inline int bpf_counter__enable(struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static inline int bpf_counter__disable(struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static inline int bpf_counter__read(struct evsel *evsel __maybe_unused) +{ + return -EAGAIN; +} + +static inline void bpf_counter__destroy(struct evsel *evsel __maybe_unused) +{ +} + +static inline int bpf_counter__install_pe(struct evsel *evsel __maybe_unused, + int cpu __maybe_unused, + int fd __maybe_unused) +{ + return 0; +} + +#endif /* HAVE_BPF_SKEL */ + +#endif /* __PERF_BPF_COUNTER_H */ diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c new file mode 100644 index 000000000..4e5f4b9dd --- /dev/null +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (c) 2021 Facebook */ +/* Copyright (c) 2021 Google */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "affinity.h" +#include "bpf_counter.h" +#include "cgroup.h" +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "evlist.h" +#include "target.h" +#include "cpumap.h" +#include "thread_map.h" + +#include "bpf_skel/bperf_cgroup.h" +#include "bpf_skel/bperf_cgroup.skel.h" + +static struct perf_event_attr cgrp_switch_attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CGROUP_SWITCHES, + .size = sizeof(cgrp_switch_attr), + .sample_period = 1, + .disabled = 1, +}; + +static struct evsel *cgrp_switch; +static struct bperf_cgroup_bpf *skel; + +#define FD(evt, cpu) (*(int *)xyarray__entry(evt->core.fd, cpu, 0)) + +static void setup_rodata(struct bperf_cgroup_bpf *sk, int evlist_size) +{ + int map_size, total_cpus = cpu__max_cpu().cpu; + + sk->rodata->num_cpus = total_cpus; + sk->rodata->num_events = evlist_size / nr_cgroups; + + if (cgroup_is_v2("perf_event") > 0) + sk->rodata->use_cgroup_v2 = 1; + + BUG_ON(evlist_size % nr_cgroups != 0); + + /* we need one copy of events per cpu for reading */ + map_size = total_cpus * evlist_size / nr_cgroups; + bpf_map__set_max_entries(sk->maps.events, map_size); + bpf_map__set_max_entries(sk->maps.cgrp_idx, nr_cgroups); + /* previous result is saved in a per-cpu array */ + map_size = evlist_size / nr_cgroups; + bpf_map__set_max_entries(sk->maps.prev_readings, map_size); + /* cgroup result needs all events (per-cpu) */ + map_size = evlist_size; + bpf_map__set_max_entries(sk->maps.cgrp_readings, map_size); +} + +static void test_max_events_program_load(void) +{ +#ifndef NDEBUG + /* + * Test that the program verifies with the maximum number of events. If + * this test fails unfortunately perf needs recompiling with a lower + * BPERF_CGROUP__MAX_EVENTS to avoid BPF verifier issues. + */ + int err, max_events = BPERF_CGROUP__MAX_EVENTS * nr_cgroups; + struct bperf_cgroup_bpf *test_skel = bperf_cgroup_bpf__open(); + + if (!test_skel) { + pr_err("Failed to open cgroup skeleton\n"); + return; + } + setup_rodata(test_skel, max_events); + err = bperf_cgroup_bpf__load(test_skel); + if (err) { + pr_err("Failed to load cgroup skeleton with max events %d.\n", + BPERF_CGROUP__MAX_EVENTS); + } + bperf_cgroup_bpf__destroy(test_skel); +#endif +} + +static int bperf_load_program(struct evlist *evlist) +{ + struct bpf_link *link; + struct evsel *evsel; + struct cgroup *cgrp, *leader_cgrp; + unsigned int i; + struct perf_cpu cpu; + int total_cpus = cpu__max_cpu().cpu; + int map_fd, prog_fd, err; + + set_max_rlimit(); + + if (nr_cgroups == 0 || evlist__nr_entries(evlist) % nr_cgroups != 0) { + pr_err("Invalid cgroup or event count\n"); + return -EINVAL; + } + + test_max_events_program_load(); + + skel = bperf_cgroup_bpf__open(); + if (!skel) { + pr_err("Failed to open cgroup skeleton\n"); + return -1; + } + setup_rodata(skel, evlist__nr_entries(evlist)); + + err = bperf_cgroup_bpf__load(skel); + if (err) { + pr_err("Failed to load cgroup skeleton\n"); + goto out; + } + + err = -1; + + cgrp_switch = evsel__new(&cgrp_switch_attr); + if (evsel__open_per_cpu(cgrp_switch, evlist__core(evlist)->all_cpus, -1) < 0) { + pr_err("Failed to open cgroup switches event\n"); + goto out; + } + + perf_cpu_map__for_each_cpu(cpu, i, evlist__core(evlist)->all_cpus) { + link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, + FD(cgrp_switch, i)); + if (IS_ERR(link)) { + pr_err("Failed to attach cgroup program\n"); + err = PTR_ERR(link); + goto out; + } + } + + /* + * Update cgrp_idx map from cgroup-id to event index. + */ + cgrp = NULL; + i = 0; + + evlist__for_each_entry(evlist, evsel) { + if (cgrp == NULL || evsel->cgrp == leader_cgrp) { + unsigned int j; + + leader_cgrp = evsel->cgrp; + evsel->cgrp = NULL; + + /* open single copy of the events w/o cgroup */ + err = evsel__open_per_cpu(evsel, evsel->core.cpus, -1); + if (err == 0) + evsel->supported = true; + + map_fd = bpf_map__fd(skel->maps.events); + perf_cpu_map__for_each_cpu(cpu, j, evsel->core.cpus) { + int fd = FD(evsel, j); + __u32 idx = evsel->core.idx * total_cpus + cpu.cpu; + + bpf_map_update_elem(map_fd, &idx, &fd, BPF_ANY); + } + + evsel->cgrp = leader_cgrp; + } + + if (evsel->cgrp == cgrp) + continue; + + cgrp = evsel->cgrp; + + if (read_cgroup_id(cgrp) < 0) { + pr_debug("Failed to get cgroup id for %s\n", cgrp->name); + cgrp->id = 0; + } + + map_fd = bpf_map__fd(skel->maps.cgrp_idx); + err = bpf_map_update_elem(map_fd, &cgrp->id, &i, BPF_ANY); + if (err < 0) { + pr_err("Failed to update cgroup index map\n"); + goto out; + } + + i++; + } + + /* + * Propagate supported flag from leaders to followers. Follower events + * are not opened, so their supported flag remains false. + */ + { + struct evsel *leader; + int num_events = evlist__nr_entries(evlist) / nr_cgroups; + + evlist__for_each_entry(evlist, evsel) { + leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); + if (leader) + evsel->supported = leader->supported; + } + } + + /* + * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check + * whether the kernel support it + */ + prog_fd = bpf_program__fd(skel->progs.trigger_read); + err = bperf_trigger_reading(prog_fd, 0); + if (err) { + pr_warning("The kernel does not support test_run for raw_tp BPF programs.\n" + "Therefore, --for-each-cgroup might show inaccurate readings\n"); + err = 0; + } + +out: + return err; +} + +static int bperf_cgrp__load(struct evsel *evsel, + struct target *target __maybe_unused) +{ + static bool bperf_loaded = false; + + evsel->bperf_leader_prog_fd = -1; + evsel->bperf_leader_link_fd = -1; + + if (!bperf_loaded && bperf_load_program(evsel->evlist)) + return -1; + + bperf_loaded = true; + /* just to bypass bpf_counter_skip() */ + evsel->follower_skel = (struct bperf_follower_bpf *)skel; + + return 0; +} + +static int bperf_cgrp__install_pe(struct evsel *evsel __maybe_unused, + int cpu_map_idx __maybe_unused, + int fd __maybe_unused) +{ + /* nothing to do */ + return 0; +} + +/* + * trigger the leader prog on each cpu, so the cgrp_reading map could get + * the latest results. + */ +static int bperf_cgrp__sync_counters(struct evlist *evlist) +{ + struct perf_cpu cpu; + unsigned int idx; + int prog_fd = bpf_program__fd(skel->progs.trigger_read); + + perf_cpu_map__for_each_cpu(cpu, idx, evlist__core(evlist)->all_cpus) + bperf_trigger_reading(prog_fd, cpu.cpu); + + return 0; +} + +static int bperf_cgrp__enable(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + skel->bss->enabled = 1; + return 0; +} + +static int bperf_cgrp__disable(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + skel->bss->enabled = 0; + return 0; +} + +static int bperf_cgrp__read(struct evsel *evsel) +{ + struct evlist *evlist = evsel->evlist; + int total_cpus = cpu__max_cpu().cpu; + struct perf_counts_values *counts; + struct bpf_perf_event_value *values; + int reading_map_fd, err = 0; + + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + values = calloc(total_cpus, sizeof(*values)); + if (values == NULL) + return -ENOMEM; + + reading_map_fd = bpf_map__fd(skel->maps.cgrp_readings); + + evlist__for_each_entry(evlist, evsel) { + __u32 idx = evsel->core.idx; + unsigned int i; + struct perf_cpu cpu; + + err = bpf_map_lookup_elem(reading_map_fd, &idx, values); + if (err) { + pr_err("bpf map lookup failed: idx=%u, event=%s, cgrp=%s\n", + idx, evsel__name(evsel), evsel->cgrp->name); + goto out; + } + + perf_cpu_map__for_each_cpu(cpu, i, evsel->core.cpus) { + counts = perf_counts(evsel->counts, i, 0); + counts->val = values[cpu.cpu].counter; + counts->ena = values[cpu.cpu].enabled; + counts->run = values[cpu.cpu].running; + } + } + +out: + free(values); + return err; +} + +static int bperf_cgrp__destroy(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgroup_bpf__destroy(skel); + evsel__put(cgrp_switch); // it'll destroy on_switch progs too + + return 0; +} + +struct bpf_counter_ops bperf_cgrp_ops = { + .load = bperf_cgrp__load, + .enable = bperf_cgrp__enable, + .disable = bperf_cgrp__disable, + .read = bperf_cgrp__read, + .install_pe = bperf_cgrp__install_pe, + .destroy = bperf_cgrp__destroy, +}; diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c new file mode 100644 index 000000000..abeafd406 --- /dev/null +++ b/tools/perf/util/bpf_ftrace.c @@ -0,0 +1,215 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "util/ftrace.h" +#include "util/cpumap.h" +#include "util/thread_map.h" +#include "util/debug.h" +#include "util/evlist.h" +#include "util/bpf_counter.h" +#include "util/stat.h" + +#include "util/bpf_skel/func_latency.skel.h" + +static struct func_latency_bpf *skel; + +int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) +{ + int fd, err; + int i, ncpus = 1, ntasks = 1; + struct filter_entry *func = NULL; + + if (!list_empty(&ftrace->filters)) { + if (!list_is_singular(&ftrace->filters)) { + pr_err("ERROR: Too many target functions.\n"); + return -1; + } + func = list_first_entry(&ftrace->filters, struct filter_entry, list); + } else { + int count = 0; + struct list_head *pos; + + list_for_each(pos, &ftrace->event_pair) + count++; + + if (count != 2) { + pr_err("ERROR: Needs two target events.\n"); + return -1; + } + } + + skel = func_latency_bpf__open(); + if (!skel) { + pr_err("Failed to open func latency skeleton\n"); + return -1; + } + + skel->rodata->bucket_range = ftrace->bucket_range; + skel->rodata->min_latency = ftrace->min_latency; + skel->rodata->bucket_num = ftrace->bucket_num; + if (ftrace->bucket_range && ftrace->bucket_num) { + bpf_map__set_max_entries(skel->maps.latency, ftrace->bucket_num); + } + + /* don't need to set cpu filter for system-wide mode */ + if (ftrace->target.cpu_list) { + ncpus = perf_cpu_map__nr(evlist__core(ftrace->evlist)->user_requested_cpus); + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + skel->rodata->has_cpu = 1; + } + + if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { + ntasks = perf_thread_map__nr(evlist__core(ftrace->evlist)->threads); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + } + + skel->rodata->use_nsec = ftrace->use_nsec; + + set_max_rlimit(); + + err = func_latency_bpf__load(skel); + if (err) { + pr_err("Failed to load func latency skeleton\n"); + goto out; + } + + if (ftrace->target.cpu_list) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu( + evlist__core(ftrace->evlist)->user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(evlist__core(ftrace->evlist)->threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + skel->bss->min = INT64_MAX; + + if (func) { + skel->links.func_begin = bpf_program__attach_kprobe(skel->progs.func_begin, + false, func->name); + if (IS_ERR(skel->links.func_begin)) { + pr_err("Failed to attach fentry program\n"); + err = PTR_ERR(skel->links.func_begin); + goto out; + } + + skel->links.func_end = bpf_program__attach_kprobe(skel->progs.func_end, + true, func->name); + if (IS_ERR(skel->links.func_end)) { + pr_err("Failed to attach fexit program\n"); + err = PTR_ERR(skel->links.func_end); + goto out; + } + } else { + struct filter_entry *event; + + event = list_first_entry(&ftrace->event_pair, struct filter_entry, list); + + skel->links.event_begin = bpf_program__attach_raw_tracepoint(skel->progs.event_begin, + event->name); + if (IS_ERR(skel->links.event_begin)) { + pr_err("Failed to attach first tracepoint program\n"); + err = PTR_ERR(skel->links.event_begin); + goto out; + } + + event = list_next_entry(event, list); + + skel->links.event_end = bpf_program__attach_raw_tracepoint(skel->progs.event_end, + event->name); + if (IS_ERR(skel->links.event_end)) { + pr_err("Failed to attach second tracepoint program\n"); + err = PTR_ERR(skel->links.event_end); + goto out; + } + } + + /* XXX: we don't actually use this fd - just for poll() */ + return open("/dev/null", O_RDONLY); + +out: + return err; +} + +int perf_ftrace__latency_start_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + skel->bss->enabled = 1; + return 0; +} + +int perf_ftrace__latency_stop_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + skel->bss->enabled = 0; + return 0; +} + +int perf_ftrace__latency_read_bpf(struct perf_ftrace *ftrace, + int buckets[], struct stats *stats) +{ + int i, fd, err; + u32 idx; + u64 *hist; + int ncpus = cpu__max_cpu().cpu; + + fd = bpf_map__fd(skel->maps.latency); + + hist = calloc(ncpus, sizeof(*hist)); + if (hist == NULL) + return -ENOMEM; + + for (idx = 0; idx < skel->rodata->bucket_num; idx++) { + err = bpf_map_lookup_elem(fd, &idx, hist); + if (err) { + buckets[idx] = 0; + continue; + } + + for (i = 0; i < ncpus; i++) + buckets[idx] += hist[i]; + } + + if (skel->bss->count) { + stats->mean = skel->bss->total / skel->bss->count; + stats->n = skel->bss->count; + stats->max = skel->bss->max; + stats->min = skel->bss->min; + + if (!ftrace->use_nsec) { + stats->mean /= 1000; + stats->max /= 1000; + stats->min /= 1000; + } + } + + free(hist); + return 0; +} + +int perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + func_latency_bpf__destroy(skel); + return 0; +} diff --git a/tools/perf/util/bpf_kwork.c b/tools/perf/util/bpf_kwork.c new file mode 100644 index 000000000..2248f462a --- /dev/null +++ b/tools/perf/util/bpf_kwork.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bpf_kwork.c + * + * Copyright (c) 2022 Huawei Inc, Yang Jihong + */ + +#include +#include +#include +#include +#include + +#include + +#include "util/debug.h" +#include "util/evsel.h" +#include "util/kwork.h" + +#include +#include + +#include "util/bpf_skel/kwork_trace.skel.h" + +/* + * This should be in sync with "util/kwork_trace.bpf.c" + */ +#define MAX_KWORKNAME 128 + +struct work_key { + u32 type; + u32 cpu; + u64 id; +}; + +struct report_data { + u64 nr; + u64 total_time; + u64 max_time; + u64 max_time_start; + u64 max_time_end; +}; + +struct kwork_class_bpf { + struct kwork_class *class; + + void (*load_prepare)(struct perf_kwork *kwork); + int (*get_work_name)(struct work_key *key, char **ret_name); +}; + +static struct kwork_trace_bpf *skel; + +static struct timespec ts_start; +static struct timespec ts_end; + +void perf_kwork__trace_start(void) +{ + clock_gettime(CLOCK_MONOTONIC, &ts_start); + skel->bss->enabled = 1; +} + +void perf_kwork__trace_finish(void) +{ + clock_gettime(CLOCK_MONOTONIC, &ts_end); + skel->bss->enabled = 0; +} + +static int get_work_name_from_map(struct work_key *key, char **ret_name) +{ + char name[MAX_KWORKNAME] = { 0 }; + int fd = bpf_map__fd(skel->maps.perf_kwork_names); + + *ret_name = NULL; + + if (fd < 0) { + pr_debug("Invalid names map fd\n"); + return 0; + } + + if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { + *ret_name = strdup(name); + if (*ret_name == NULL) { + pr_err("Failed to copy work name\n"); + return -1; + } + } + + return 0; +} + +static void irq_load_prepare(struct perf_kwork *kwork) +{ + if (kwork->report == KWORK_REPORT_RUNTIME) { + bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); + bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); + } +} + +static struct kwork_class_bpf kwork_irq_bpf = { + .load_prepare = irq_load_prepare, + .get_work_name = get_work_name_from_map, +}; + +static void softirq_load_prepare(struct perf_kwork *kwork) +{ + if (kwork->report == KWORK_REPORT_RUNTIME) { + bpf_program__set_autoload(skel->progs.report_softirq_entry, true); + bpf_program__set_autoload(skel->progs.report_softirq_exit, true); + } else if (kwork->report == KWORK_REPORT_LATENCY) { + bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); + bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); + } +} + +static struct kwork_class_bpf kwork_softirq_bpf = { + .load_prepare = softirq_load_prepare, + .get_work_name = get_work_name_from_map, +}; + +static void workqueue_load_prepare(struct perf_kwork *kwork) +{ + if (kwork->report == KWORK_REPORT_RUNTIME) { + bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); + bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); + } else if (kwork->report == KWORK_REPORT_LATENCY) { + bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); + bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); + } +} + +static struct kwork_class_bpf kwork_workqueue_bpf = { + .load_prepare = workqueue_load_prepare, + .get_work_name = get_work_name_from_map, +}; + +static struct kwork_class_bpf * +kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { + [KWORK_CLASS_IRQ] = &kwork_irq_bpf, + [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, + [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, +}; + +static bool valid_kwork_class_type(enum kwork_class_type type) +{ + return type >= 0 && type < KWORK_CLASS_MAX ? true : false; +} + +static int setup_filters(struct perf_kwork *kwork) +{ + if (kwork->cpu_list != NULL) { + unsigned int idx; + int nr_cpus; + struct perf_cpu_map *map; + struct perf_cpu cpu; + int fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); + + if (fd < 0) { + pr_debug("Invalid cpu filter fd\n"); + return -1; + } + + map = perf_cpu_map__new(kwork->cpu_list); + if (map == NULL) { + pr_debug("Invalid cpu_list\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + perf_cpu_map__for_each_cpu(cpu, idx, map) { + u8 val = 1; + + if (cpu.cpu >= nr_cpus) { + perf_cpu_map__put(map); + pr_err("Requested cpu %d too large\n", cpu.cpu); + return -1; + } + bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); + } + perf_cpu_map__put(map); + } + + if (kwork->profile_name != NULL) { + int key, fd; + + if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { + pr_err("Requested name filter %s too large, limit to %d\n", + kwork->profile_name, MAX_KWORKNAME - 1); + return -1; + } + + fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); + if (fd < 0) { + pr_debug("Invalid name filter fd\n"); + return -1; + } + + key = 0; + bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); + } + + return 0; +} + +int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) +{ + struct bpf_program *prog; + struct kwork_class *class; + struct kwork_class_bpf *class_bpf; + enum kwork_class_type type; + + skel = kwork_trace_bpf__open(); + if (!skel) { + pr_debug("Failed to open kwork trace skeleton\n"); + return -1; + } + + /* + * set all progs to non-autoload, + * then set corresponding progs according to config + */ + bpf_object__for_each_program(prog, skel->obj) + bpf_program__set_autoload(prog, false); + + list_for_each_entry(class, &kwork->class_list, list) { + type = class->type; + if (!valid_kwork_class_type(type) || + (kwork_class_bpf_supported_list[type] == NULL)) { + pr_err("Unsupported bpf trace class %s\n", class->name); + goto out; + } + + class_bpf = kwork_class_bpf_supported_list[type]; + class_bpf->class = class; + + if (class_bpf->load_prepare != NULL) + class_bpf->load_prepare(kwork); + } + + if (kwork->cpu_list != NULL) + skel->rodata->has_cpu_filter = 1; + if (kwork->profile_name != NULL) + skel->rodata->has_name_filter = 1; + + if (kwork_trace_bpf__load(skel)) { + pr_debug("Failed to load kwork trace skeleton\n"); + goto out; + } + + if (setup_filters(kwork)) + goto out; + + if (kwork_trace_bpf__attach(skel)) { + pr_debug("Failed to attach kwork trace skeleton\n"); + goto out; + } + + return 0; + +out: + kwork_trace_bpf__destroy(skel); + return -1; +} + +static int add_work(struct perf_kwork *kwork, + struct work_key *key, + struct report_data *data) +{ + struct kwork_work *work; + struct kwork_class_bpf *bpf_trace; + struct kwork_work tmp = { + .id = key->id, + .name = NULL, + .cpu = key->cpu, + }; + enum kwork_class_type type = key->type; + int ret = 0; + + if (!valid_kwork_class_type(type)) { + pr_debug("Invalid class type %d to add work\n", type); + return -1; + } + + bpf_trace = kwork_class_bpf_supported_list[type]; + tmp.class = bpf_trace->class; + + if ((bpf_trace->get_work_name != NULL) && + (bpf_trace->get_work_name(key, &tmp.name))) + return -1; + + work = kwork->add_work(kwork, tmp.class, &tmp); + if (work == NULL) { + ret = -1; + goto out; + } + + if (kwork->report == KWORK_REPORT_RUNTIME) { + work->nr_atoms = data->nr; + work->total_runtime = data->total_time; + work->max_runtime = data->max_time; + work->max_runtime_start = data->max_time_start; + work->max_runtime_end = data->max_time_end; + } else if (kwork->report == KWORK_REPORT_LATENCY) { + work->nr_atoms = data->nr; + work->total_latency = data->total_time; + work->max_latency = data->max_time; + work->max_latency_start = data->max_time_start; + work->max_latency_end = data->max_time_end; + } else { + pr_debug("Invalid bpf report type %d\n", kwork->report); + ret = -1; + goto out; + } + + kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; + kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; + +out: + work_exit(&tmp); + return ret; +} + +int perf_kwork__report_read_bpf(struct perf_kwork *kwork) +{ + struct report_data data; + struct work_key key = { + .type = 0, + .cpu = 0, + .id = 0, + }; + struct work_key prev = { + .type = 0, + .cpu = 0, + .id = 0, + }; + int fd = bpf_map__fd(skel->maps.perf_kwork_report); + + if (fd < 0) { + pr_debug("Invalid report fd\n"); + return -1; + } + + while (!bpf_map_get_next_key(fd, &prev, &key)) { + if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { + pr_debug("Failed to lookup report elem\n"); + return -1; + } + + if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0)) + return -1; + + prev = key; + } + return 0; +} + +void perf_kwork__report_cleanup_bpf(void) +{ + kwork_trace_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf_kwork_top.c b/tools/perf/util/bpf_kwork_top.c new file mode 100644 index 000000000..189a29d2b --- /dev/null +++ b/tools/perf/util/bpf_kwork_top.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bpf_kwork_top.c + * + * Copyright (c) 2022 Huawei Inc, Yang Jihong + */ + +#include +#include +#include +#include +#include + +#include + +#include "util/debug.h" +#include "util/evsel.h" +#include "util/kwork.h" + +#include +#include + +#include "util/bpf_skel/kwork_top.skel.h" + +/* + * This should be in sync with "util/kwork_top.bpf.c" + */ +#define MAX_COMMAND_LEN 16 + +struct time_data { + __u64 timestamp; +}; + +struct work_data { + __u64 runtime; +}; + +struct task_data { + __u32 tgid; + __u32 is_kthread; + char comm[MAX_COMMAND_LEN]; +}; + +struct work_key { + __u32 type; + __u32 pid; + __u64 task_p; +}; + +struct task_key { + __u32 pid; + __u32 cpu; +}; + +struct kwork_class_bpf { + struct kwork_class *class; + void (*load_prepare)(void); +}; + +static struct kwork_top_bpf *skel; + +void perf_kwork__top_start(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + skel->bss->from_timestamp = (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + skel->bss->enabled = 1; + pr_debug("perf kwork top start at: %lld\n", skel->bss->from_timestamp); +} + +void perf_kwork__top_finish(void) +{ + struct timespec ts; + + skel->bss->enabled = 0; + clock_gettime(CLOCK_MONOTONIC, &ts); + skel->bss->to_timestamp = (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + pr_debug("perf kwork top finish at: %lld\n", skel->bss->to_timestamp); +} + +static void irq_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_irq_handler_entry, true); + bpf_program__set_autoload(skel->progs.on_irq_handler_exit, true); +} + +static struct kwork_class_bpf kwork_irq_bpf = { + .load_prepare = irq_load_prepare, +}; + +static void softirq_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_softirq_entry, true); + bpf_program__set_autoload(skel->progs.on_softirq_exit, true); +} + +static struct kwork_class_bpf kwork_softirq_bpf = { + .load_prepare = softirq_load_prepare, +}; + +static void sched_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_switch, true); +} + +static struct kwork_class_bpf kwork_sched_bpf = { + .load_prepare = sched_load_prepare, +}; + +static struct kwork_class_bpf * +kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { + [KWORK_CLASS_IRQ] = &kwork_irq_bpf, + [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, + [KWORK_CLASS_SCHED] = &kwork_sched_bpf, +}; + +static bool valid_kwork_class_type(enum kwork_class_type type) +{ + return type >= 0 && type < KWORK_CLASS_MAX; +} + +static int setup_filters(struct perf_kwork *kwork) +{ + if (kwork->cpu_list) { + unsigned int idx; + int nr_cpus, fd; + struct perf_cpu_map *map; + struct perf_cpu cpu; + + fd = bpf_map__fd(skel->maps.kwork_top_cpu_filter); + if (fd < 0) { + pr_debug("Invalid cpu filter fd\n"); + return -1; + } + + map = perf_cpu_map__new(kwork->cpu_list); + if (!map) { + pr_debug("Invalid cpu_list\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + perf_cpu_map__for_each_cpu(cpu, idx, map) { + u8 val = 1; + + if (cpu.cpu >= nr_cpus) { + perf_cpu_map__put(map); + pr_err("Requested cpu %d too large\n", cpu.cpu); + return -1; + } + bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); + } + perf_cpu_map__put(map); + } + + return 0; +} + +int perf_kwork__top_prepare_bpf(struct perf_kwork *kwork) +{ + struct bpf_program *prog; + struct kwork_class *class; + struct kwork_class_bpf *class_bpf; + enum kwork_class_type type; + + skel = kwork_top_bpf__open(); + if (!skel) { + pr_debug("Failed to open kwork top skeleton\n"); + return -1; + } + + /* + * set all progs to non-autoload, + * then set corresponding progs according to config + */ + bpf_object__for_each_program(prog, skel->obj) + bpf_program__set_autoload(prog, false); + + list_for_each_entry(class, &kwork->class_list, list) { + type = class->type; + if (!valid_kwork_class_type(type) || + !kwork_class_bpf_supported_list[type]) { + pr_err("Unsupported bpf trace class %s\n", class->name); + goto out; + } + + class_bpf = kwork_class_bpf_supported_list[type]; + class_bpf->class = class; + + if (class_bpf->load_prepare) + class_bpf->load_prepare(); + } + + if (kwork->cpu_list) + skel->rodata->has_cpu_filter = 1; + + if (kwork_top_bpf__load(skel)) { + pr_debug("Failed to load kwork top skeleton\n"); + goto out; + } + + if (setup_filters(kwork)) + goto out; + + if (kwork_top_bpf__attach(skel)) { + pr_debug("Failed to attach kwork top skeleton\n"); + goto out; + } + + return 0; + +out: + kwork_top_bpf__destroy(skel); + return -1; +} + +static void read_task_info(struct kwork_work *work) +{ + int fd; + struct task_data data; + struct task_key key = { + .pid = work->id, + .cpu = work->cpu, + }; + + fd = bpf_map__fd(skel->maps.kwork_top_tasks); + if (fd < 0) { + pr_debug("Invalid top tasks map fd\n"); + return; + } + + if (!bpf_map_lookup_elem(fd, &key, &data)) { + work->tgid = data.tgid; + work->is_kthread = data.is_kthread; + work->name = strdup(data.comm); + } +} +static int add_work(struct perf_kwork *kwork, struct work_key *key, + struct work_data *data, int cpu) +{ + struct kwork_class_bpf *bpf_trace; + struct kwork_work *work; + struct kwork_work tmp = { + .id = key->pid, + .cpu = cpu, + .name = NULL, + }; + enum kwork_class_type type = key->type; + + if (!valid_kwork_class_type(type)) { + pr_debug("Invalid class type %d to add work\n", type); + return -1; + } + + bpf_trace = kwork_class_bpf_supported_list[type]; + tmp.class = bpf_trace->class; + + work = kwork->add_work(kwork, tmp.class, &tmp); + if (!work) + return -1; + + work->total_runtime = data->runtime; + read_task_info(work); + + return 0; +} + +int perf_kwork__top_read_bpf(struct perf_kwork *kwork) +{ + int i, fd, nr_cpus; + struct work_data *data; + struct work_key key, prev; + + fd = bpf_map__fd(skel->maps.kwork_top_works); + if (fd < 0) { + pr_debug("Invalid top runtime fd\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + data = calloc(nr_cpus, sizeof(struct work_data)); + if (!data) + return -1; + + memset(&prev, 0, sizeof(prev)); + while (!bpf_map_get_next_key(fd, &prev, &key)) { + if ((bpf_map_lookup_elem(fd, &key, data)) != 0) { + pr_debug("Failed to lookup top elem\n"); + return -1; + } + + for (i = 0; i < nr_cpus; i++) { + if (data[i].runtime == 0) + continue; + + if (add_work(kwork, &key, &data[i], i)) + return -1; + } + prev = key; + } + free(data); + + return 0; +} + +void perf_kwork__top_cleanup_bpf(void) +{ + kwork_top_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c new file mode 100644 index 000000000..c20bd0756 --- /dev/null +++ b/tools/perf/util/bpf_lock_contention.c @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/cgroup.h" +#include "util/debug.h" +#include "util/evlist.h" +#include "util/hashmap.h" +#include "util/machine.h" +#include "util/map.h" +#include "util/symbol.h" +#include "util/target.h" +#include "util/thread.h" +#include "util/thread_map.h" +#include "util/lock-contention.h" +#include +#include +#include +#include +#include +#include + +#include "bpf_skel/lock_contention.skel.h" +#include "bpf_skel/lock_data.h" + +static struct lock_contention_bpf *skel; +static bool has_slab_iter; +static struct hashmap slab_hash; + +static size_t slab_cache_hash(long key, void *ctx __maybe_unused) +{ + return key; +} + +static bool slab_cache_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +static void check_slab_cache_iter(struct lock_contention *con) +{ + s32 ret; + + hashmap__init(&slab_hash, slab_cache_hash, slab_cache_equal, /*ctx=*/NULL); + + con->btf = btf__load_vmlinux_btf(); + if (con->btf == NULL) { + pr_debug("BTF loading failed: %m\n"); + return; + } + + ret = btf__find_by_name_kind(con->btf, "bpf_iter__kmem_cache", BTF_KIND_STRUCT); + if (ret < 0) { + bpf_program__set_autoload(skel->progs.slab_cache_iter, false); + pr_debug("slab cache iterator is not available: %d\n", ret); + return; + } + + has_slab_iter = true; + + bpf_map__set_max_entries(skel->maps.slab_caches, con->map_nr_entries); +} + +static void run_slab_cache_iter(void) +{ + int fd; + char buf[256]; + long key, *prev_key; + + if (!has_slab_iter) + return; + + fd = bpf_iter_create(bpf_link__fd(skel->links.slab_cache_iter)); + if (fd < 0) { + pr_debug("cannot create slab cache iter: %d\n", fd); + return; + } + + /* This will run the bpf program */ + while (read(fd, buf, sizeof(buf)) > 0) + continue; + + close(fd); + + /* Read the slab cache map and build a hash with IDs */ + fd = bpf_map__fd(skel->maps.slab_caches); + prev_key = NULL; + while (!bpf_map_get_next_key(fd, prev_key, &key)) { + struct slab_cache_data *data; + + data = malloc(sizeof(*data)); + if (data == NULL) + break; + + if (bpf_map_lookup_elem(fd, &key, data) < 0) + break; + + hashmap__add(&slab_hash, data->id, data); + prev_key = &key; + } +} + +static void exit_slab_cache_iter(void) +{ + struct hashmap_entry *cur; + unsigned bkt; + + hashmap__for_each_entry(&slab_hash, cur, bkt) + free(cur->pvalue); + + hashmap__clear(&slab_hash); +} + +static void init_numa_data(struct lock_contention *con) +{ + struct symbol *sym; + struct map *kmap; + char *buf = NULL, *p; + size_t len; + long last = -1; + int ret; + + if (!con->btf) + return; + + /* + * 'struct zone' is embedded in 'struct pglist_data' as an array. + * As we may not have full information of the struct zone in the + * (fake) vmlinux.h, let's get the actual size from BTF. + */ + ret = btf__find_by_name_kind(con->btf, "zone", BTF_KIND_STRUCT); + if (ret < 0) { + pr_debug("cannot get type of struct zone: %d\n", ret); + return; + } + + ret = btf__resolve_size(con->btf, ret); + if (ret < 0) { + pr_debug("cannot get size of struct zone: %d\n", ret); + return; + } + skel->rodata->sizeof_zone = ret; + + /* UMA system doesn't have 'node_data[]' - just use contig_page_data. */ + sym = machine__find_kernel_symbol_by_name(con->machine, + "contig_page_data", + &kmap); + if (sym) { + skel->rodata->contig_page_data_addr = map__unmap_ip(kmap, sym->start); + map__put(kmap); + return; + } + + /* + * The 'node_data' is an array of pointers to struct pglist_data. + * It needs to follow the pointer for each node in BPF to get the + * address of struct pglist_data and its zones. + */ + sym = machine__find_kernel_symbol_by_name(con->machine, + "node_data", + &kmap); + if (sym == NULL) + return; + + skel->rodata->node_data_addr = map__unmap_ip(kmap, sym->start); + map__put(kmap); + + /* get the number of online nodes using the last node number + 1 */ + ret = sysfs__read_str("devices/system/node/online", &buf, &len); + if (ret < 0) { + pr_debug("failed to read online node: %d\n", ret); + return; + } + + p = buf; + while (p && *p) { + last = strtol(p, &p, 0); + + if (p && (*p == ',' || *p == '-' || *p == '\n')) + p++; + } + skel->rodata->nr_nodes = last + 1; + free(buf); +} + +int lock_contention_prepare(struct lock_contention *con) +{ + int i, fd; + int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1, ncgrps = 1, nslabs = 1; + struct evlist *evlist = con->evlist; + struct target *target = con->target; + bool has_mmap_lock = false; + + /* make sure it loads the kernel map before lookup */ + map__load(machine__kernel_map(con->machine)); + + skel = lock_contention_bpf__open(); + if (!skel) { + pr_err("Failed to open lock-contention BPF skeleton\n"); + return -1; + } + + bpf_map__set_value_size(skel->maps.stacks, con->max_stack * sizeof(u64)); + bpf_map__set_max_entries(skel->maps.lock_stat, con->map_nr_entries); + bpf_map__set_max_entries(skel->maps.tstamp, con->map_nr_entries); + + if (con->aggr_mode == LOCK_AGGR_TASK) + bpf_map__set_max_entries(skel->maps.task_data, con->map_nr_entries); + else + bpf_map__set_max_entries(skel->maps.task_data, 1); + + if (con->save_callstack) { + bpf_map__set_max_entries(skel->maps.stacks, con->map_nr_entries); + if (con->owner) { + bpf_map__set_value_size(skel->maps.stack_buf, con->max_stack * sizeof(u64)); + bpf_map__set_key_size(skel->maps.owner_stacks, + con->max_stack * sizeof(u64)); + bpf_map__set_max_entries(skel->maps.owner_stacks, con->map_nr_entries); + bpf_map__set_max_entries(skel->maps.owner_data, con->map_nr_entries); + bpf_map__set_max_entries(skel->maps.owner_stat, con->map_nr_entries); + skel->rodata->max_stack = con->max_stack; + } + } else { + bpf_map__set_max_entries(skel->maps.stacks, 1); + } + + if (target__has_cpu(target)) { + skel->rodata->has_cpu = 1; + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); + } + if (target__has_task(target)) { + skel->rodata->has_task = 1; + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); + } + if (con->filters->nr_types) { + skel->rodata->has_type = 1; + ntypes = con->filters->nr_types; + } + if (con->filters->nr_cgrps) { + skel->rodata->has_cgroup = 1; + ncgrps = con->filters->nr_cgrps; + } + + /* resolve lock name filters to addr */ + if (con->filters->nr_syms) { + struct symbol *sym; + struct map *kmap; + unsigned long *addrs; + + for (i = 0; i < con->filters->nr_syms; i++) { + if (!strcmp(con->filters->syms[i], "mmap_lock")) { + has_mmap_lock = true; + continue; + } + + sym = machine__find_kernel_symbol_by_name(con->machine, + con->filters->syms[i], + &kmap); + if (sym == NULL) { + pr_warning("ignore unknown symbol: %s\n", + con->filters->syms[i]); + continue; + } + + addrs = realloc(con->filters->addrs, + (con->filters->nr_addrs + 1) * sizeof(*addrs)); + if (addrs == NULL) { + pr_warning("memory allocation failure\n"); + continue; + } + + addrs[con->filters->nr_addrs++] = map__unmap_ip(kmap, sym->start); + con->filters->addrs = addrs; + } + naddrs = con->filters->nr_addrs ?: has_mmap_lock; + skel->rodata->has_addr = 1; + } + + /* resolve lock name in delays */ + if (con->nr_delays) { + struct symbol *sym; + struct map *kmap; + + for (i = 0; i < con->nr_delays; i++) { + sym = machine__find_kernel_symbol_by_name(con->machine, + con->delays[i].sym, + &kmap); + if (sym == NULL) { + pr_warning("ignore unknown symbol: %s\n", + con->delays[i].sym); + continue; + } + + con->delays[i].addr = map__unmap_ip(kmap, sym->start); + } + skel->rodata->lock_delay = 1; + bpf_map__set_max_entries(skel->maps.lock_delays, con->nr_delays); + } + + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + bpf_map__set_max_entries(skel->maps.type_filter, ntypes); + bpf_map__set_max_entries(skel->maps.addr_filter, naddrs); + bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); + + skel->rodata->stack_skip = con->stack_skip; + skel->rodata->aggr_mode = con->aggr_mode; + skel->rodata->needs_callstack = con->save_callstack; + skel->rodata->lock_owner = con->owner; + skel->rodata->has_mmap_lock = has_mmap_lock; + + if (con->aggr_mode == LOCK_AGGR_CGROUP || con->filters->nr_cgrps) { + if (cgroup_is_v2("perf_event")) + skel->rodata->use_cgroup_v2 = 1; + } + + check_slab_cache_iter(con); + + if (con->filters->nr_slabs && has_slab_iter) { + skel->rodata->has_slab = 1; + nslabs = con->filters->nr_slabs; + } + + bpf_map__set_max_entries(skel->maps.slab_filter, nslabs); + + init_numa_data(con); + + if (lock_contention_bpf__load(skel) < 0) { + pr_err("Failed to load lock-contention BPF skeleton\n"); + return -1; + } + + if (target__has_cpu(target)) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target__has_task(target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + if (target__none(target) && evlist__workload_pid(evlist) > 0) { + u32 pid = evlist__workload_pid(evlist); + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + + if (con->filters->nr_types) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.type_filter); + + for (i = 0; i < con->filters->nr_types; i++) + bpf_map_update_elem(fd, &con->filters->types[i], &val, BPF_ANY); + } + + if (con->filters->nr_addrs) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.addr_filter); + + for (i = 0; i < con->filters->nr_addrs; i++) + bpf_map_update_elem(fd, &con->filters->addrs[i], &val, BPF_ANY); + } + + if (con->filters->nr_cgrps) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cgroup_filter); + + for (i = 0; i < con->filters->nr_cgrps; i++) + bpf_map_update_elem(fd, &con->filters->cgrps[i], &val, BPF_ANY); + } + + if (con->nr_delays) { + fd = bpf_map__fd(skel->maps.lock_delays); + + for (i = 0; i < con->nr_delays; i++) + bpf_map_update_elem(fd, &con->delays[i].addr, &con->delays[i].time, BPF_ANY); + } + + if (con->aggr_mode == LOCK_AGGR_CGROUP) + read_all_cgroups(&con->cgroups); + + bpf_program__set_autoload(skel->progs.collect_lock_syms, false); + + lock_contention_bpf__attach(skel); + + /* run the slab iterator after attaching */ + run_slab_cache_iter(); + + if (con->filters->nr_slabs) { + u8 val = 1; + int cache_fd; + long key, *prev_key; + + fd = bpf_map__fd(skel->maps.slab_filter); + + /* Read the slab cache map and build a hash with its address */ + cache_fd = bpf_map__fd(skel->maps.slab_caches); + prev_key = NULL; + while (!bpf_map_get_next_key(cache_fd, prev_key, &key)) { + struct slab_cache_data data; + + if (bpf_map_lookup_elem(cache_fd, &key, &data) < 0) + break; + + for (i = 0; i < con->filters->nr_slabs; i++) { + if (!strcmp(con->filters->slabs[i], data.name)) { + bpf_map_update_elem(fd, &key, &val, BPF_ANY); + break; + } + } + prev_key = &key; + } + } + + return 0; +} + +/* + * Run the BPF program directly using BPF_PROG_TEST_RUN to update the end + * timestamp in ktime so that it can calculate delta easily. + */ +static void mark_end_timestamp(void) +{ + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + int prog_fd = bpf_program__fd(skel->progs.end_timestamp); + + bpf_prog_test_run_opts(prog_fd, &opts); +} + +static void update_lock_stat(int map_fd, int pid, u64 end_ts, + enum lock_aggr_mode aggr_mode, + struct tstamp_data *ts_data) +{ + u64 delta; + struct contention_key stat_key = {}; + struct contention_data stat_data; + + if (ts_data->timestamp >= end_ts) + return; + + delta = end_ts - ts_data->timestamp; + + switch (aggr_mode) { + case LOCK_AGGR_CALLER: + stat_key.stack_id = ts_data->stack_id; + break; + case LOCK_AGGR_TASK: + stat_key.pid = pid; + break; + case LOCK_AGGR_ADDR: + stat_key.lock_addr_or_cgroup = ts_data->lock; + break; + case LOCK_AGGR_CGROUP: + stat_key.lock_addr_or_cgroup = ts_data->cgroup_id; + break; + default: + return; + } + + if (bpf_map_lookup_elem(map_fd, &stat_key, &stat_data) < 0) + return; + + stat_data.total_time += delta; + stat_data.count++; + + if (delta > stat_data.max_time) + stat_data.max_time = delta; + if (delta < stat_data.min_time) + stat_data.min_time = delta; + + bpf_map_update_elem(map_fd, &stat_key, &stat_data, BPF_EXIST); +} + +/* + * Account entries in the tstamp map (which didn't see the corresponding + * lock:contention_end tracepoint) using end_ts. + */ +static void account_end_timestamp(struct lock_contention *con) +{ + int ts_fd, stat_fd; + int *prev_key, key; + u64 end_ts = skel->bss->end_ts; + int total_cpus; + enum lock_aggr_mode aggr_mode = con->aggr_mode; + struct tstamp_data ts_data, *cpu_data; + + /* Iterate per-task tstamp map (key = TID) */ + ts_fd = bpf_map__fd(skel->maps.tstamp); + stat_fd = bpf_map__fd(skel->maps.lock_stat); + + prev_key = NULL; + while (!bpf_map_get_next_key(ts_fd, prev_key, &key)) { + if (bpf_map_lookup_elem(ts_fd, &key, &ts_data) == 0) { + int pid = key; + + if (aggr_mode == LOCK_AGGR_TASK && con->owner) + pid = ts_data.flags; + + update_lock_stat(stat_fd, pid, end_ts, aggr_mode, + &ts_data); + } + + prev_key = &key; + } + + /* Now it'll check per-cpu tstamp map which doesn't have TID. */ + if (aggr_mode == LOCK_AGGR_TASK || aggr_mode == LOCK_AGGR_CGROUP) + return; + + total_cpus = cpu__max_cpu().cpu; + ts_fd = bpf_map__fd(skel->maps.tstamp_cpu); + + cpu_data = calloc(total_cpus, sizeof(*cpu_data)); + if (cpu_data == NULL) + return; + + prev_key = NULL; + while (!bpf_map_get_next_key(ts_fd, prev_key, &key)) { + if (bpf_map_lookup_elem(ts_fd, &key, cpu_data) < 0) + goto next; + + for (int i = 0; i < total_cpus; i++) { + if (cpu_data[i].lock == 0) + continue; + + update_lock_stat(stat_fd, -1, end_ts, aggr_mode, + &cpu_data[i]); + } + +next: + prev_key = &key; + } + free(cpu_data); +} + +int lock_contention_start(void) +{ + skel->bss->enabled = 1; + return 0; +} + +int lock_contention_stop(void) +{ + skel->bss->enabled = 0; + mark_end_timestamp(); + return 0; +} + +static const char *lock_contention_get_name(struct lock_contention *con, + struct contention_key *key, + u64 *stack_trace, u32 flags) +{ + int idx = 0; + u64 addr; + static char name_buf[KSYM_NAME_LEN]; + struct symbol *sym; + struct map *kmap; + struct machine *machine = con->machine; + + if (con->aggr_mode == LOCK_AGGR_TASK) { + struct contention_task_data task; + int pid = key->pid; + int task_fd = bpf_map__fd(skel->maps.task_data); + + /* do not update idle comm which contains CPU number */ + if (pid) { + struct thread *t = machine__findnew_thread(machine, /*pid=*/-1, pid); + + if (t != NULL && + !bpf_map_lookup_elem(task_fd, &pid, &task) && + thread__set_comm(t, task.comm, /*timestamp=*/0)) { + snprintf(name_buf, sizeof(name_buf), "%s", task.comm); + return name_buf; + } + } + return ""; + } + + if (con->aggr_mode == LOCK_AGGR_ADDR) { + int lock_fd = bpf_map__fd(skel->maps.lock_syms); + struct slab_cache_data *slab_data; + + /* per-process locks set upper bits of the flags */ + if (flags & LCD_F_MMAP_LOCK) + return "mmap_lock"; + if (flags & LCD_F_SIGHAND_LOCK) + return "siglock"; + + /* global locks with symbols */ + sym = machine__find_kernel_symbol(machine, key->lock_addr_or_cgroup, &kmap); + if (sym) + return sym->name; + + /* try semi-global locks collected separately */ + if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) { + if (flags == LOCK_CLASS_RQLOCK) + return "rq_lock"; + } + + if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) { + if (flags == LOCK_CLASS_ZONE_LOCK) + return "zone_lock"; + } + + /* look slab_hash for dynamic locks in a slab object */ + if (hashmap__find(&slab_hash, flags & LCB_F_SLAB_ID_MASK, &slab_data)) { + snprintf(name_buf, sizeof(name_buf), "&%s", slab_data->name); + return name_buf; + } + + return ""; + } + + if (con->aggr_mode == LOCK_AGGR_CGROUP) { + u64 cgrp_id = key->lock_addr_or_cgroup; + struct cgroup *cgrp = __cgroup__find(&con->cgroups, cgrp_id); + + if (cgrp) + return cgrp->name; + + snprintf(name_buf, sizeof(name_buf), "cgroup:%" PRIu64 "", cgrp_id); + return name_buf; + } + + /* LOCK_AGGR_CALLER: skip lock internal functions */ + while (machine__is_lock_function(machine, stack_trace[idx]) && + idx < con->max_stack - 1) + idx++; + + addr = stack_trace[idx]; + sym = machine__find_kernel_symbol(machine, addr, &kmap); + + if (sym) { + unsigned long offset; + + offset = map__map_ip(kmap, addr) - sym->start; + + if (offset == 0) + return sym->name; + + snprintf(name_buf, sizeof(name_buf), "%s+%#lx", sym->name, offset); + } else { + snprintf(name_buf, sizeof(name_buf), "%#lx", (unsigned long)addr); + } + + return name_buf; +} + +struct lock_stat *pop_owner_stack_trace(struct lock_contention *con) +{ + int stacks_fd, stat_fd; + u64 *stack_trace = NULL; + s32 stack_id; + struct contention_key ckey = {}; + struct contention_data cdata = {}; + size_t stack_size = con->max_stack * sizeof(*stack_trace); + struct lock_stat *st = NULL; + + stacks_fd = bpf_map__fd(skel->maps.owner_stacks); + stat_fd = bpf_map__fd(skel->maps.owner_stat); + if (!stacks_fd || !stat_fd) + goto out_err; + + stack_trace = zalloc(stack_size); + if (stack_trace == NULL) + goto out_err; + + if (bpf_map_get_next_key(stacks_fd, NULL, stack_trace)) + goto out_err; + + bpf_map_lookup_elem(stacks_fd, stack_trace, &stack_id); + ckey.stack_id = stack_id; + bpf_map_lookup_elem(stat_fd, &ckey, &cdata); + + st = zalloc(sizeof(struct lock_stat)); + if (!st) + goto out_err; + + st->name = strdup(stack_trace[0] ? lock_contention_get_name(con, NULL, stack_trace, 0) : + "unknown"); + if (!st->name) + goto out_err; + + st->flags = cdata.flags; + st->nr_contended = cdata.count; + st->wait_time_total = cdata.total_time; + st->wait_time_max = cdata.max_time; + st->wait_time_min = cdata.min_time; + st->callstack = stack_trace; + + if (cdata.count) + st->avg_wait_time = cdata.total_time / cdata.count; + + bpf_map_delete_elem(stacks_fd, stack_trace); + bpf_map_delete_elem(stat_fd, &ckey); + + return st; + +out_err: + free(stack_trace); + free(st); + + return NULL; +} + +int lock_contention_read(struct lock_contention *con) +{ + int fd, stack, err = 0; + struct contention_key *prev_key, key = {}; + struct contention_data data = {}; + struct lock_stat *st = NULL; + struct machine *machine = con->machine; + u64 *stack_trace; + size_t stack_size = con->max_stack * sizeof(*stack_trace); + + fd = bpf_map__fd(skel->maps.lock_stat); + stack = bpf_map__fd(skel->maps.stacks); + + con->fails.task = skel->bss->task_fail; + con->fails.stack = skel->bss->stack_fail; + con->fails.time = skel->bss->time_fail; + con->fails.data = skel->bss->data_fail; + + stack_trace = zalloc(stack_size); + if (stack_trace == NULL) + return -1; + + account_end_timestamp(con); + + if (con->aggr_mode == LOCK_AGGR_TASK) { + struct thread *idle = machine__findnew_thread(machine, + /*pid=*/0, + /*tid=*/0); + thread__set_comm(idle, "swapper", /*timestamp=*/0); + } + + if (con->aggr_mode == LOCK_AGGR_ADDR) { + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + int prog_fd = bpf_program__fd(skel->progs.collect_lock_syms); + + bpf_prog_test_run_opts(prog_fd, &opts); + } + + prev_key = NULL; + while (!bpf_map_get_next_key(fd, prev_key, &key)) { + s64 ls_key; + const char *name; + + /* to handle errors in the loop body */ + err = -1; + + bpf_map_lookup_elem(fd, &key, &data); + if (con->save_callstack) { + bpf_map_lookup_elem(stack, &key.stack_id, stack_trace); + + if (!match_callstack_filter(machine, stack_trace, con->max_stack)) { + con->nr_filtered += data.count; + goto next; + } + } + + switch (con->aggr_mode) { + case LOCK_AGGR_CALLER: + ls_key = key.stack_id; + break; + case LOCK_AGGR_TASK: + ls_key = key.pid; + break; + case LOCK_AGGR_ADDR: + case LOCK_AGGR_CGROUP: + ls_key = key.lock_addr_or_cgroup; + break; + default: + goto next; + } + + st = lock_stat_find(ls_key); + if (st != NULL) { + st->wait_time_total += data.total_time; + if (st->wait_time_max < data.max_time) + st->wait_time_max = data.max_time; + if (st->wait_time_min > data.min_time) + st->wait_time_min = data.min_time; + + st->nr_contended += data.count; + if (st->nr_contended) + st->avg_wait_time = st->wait_time_total / st->nr_contended; + goto next; + } + + name = lock_contention_get_name(con, &key, stack_trace, data.flags); + st = lock_stat_findnew(ls_key, name, data.flags); + if (st == NULL) + break; + + st->nr_contended = data.count; + st->wait_time_total = data.total_time; + st->wait_time_max = data.max_time; + st->wait_time_min = data.min_time; + + if (data.count) + st->avg_wait_time = data.total_time / data.count; + + if (con->aggr_mode == LOCK_AGGR_CALLER && verbose > 0) { + st->callstack = memdup(stack_trace, stack_size); + if (st->callstack == NULL) + break; + } + +next: + prev_key = &key; + + /* we're fine now, reset the error */ + err = 0; + } + + free(stack_trace); + + return err; +} + +int lock_contention_finish(struct lock_contention *con) +{ + if (skel) { + skel->bss->enabled = 0; + lock_contention_bpf__destroy(skel); + } + + while (!RB_EMPTY_ROOT(&con->cgroups)) { + struct rb_node *node = rb_first(&con->cgroups); + struct cgroup *cgrp = rb_entry(node, struct cgroup, node); + + rb_erase(node, &con->cgroups); + cgroup__put(cgrp); + } + + exit_slab_cache_iter(); + btf__free(con->btf); + + return 0; +} diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c new file mode 100644 index 000000000..c4639f6a5 --- /dev/null +++ b/tools/perf/util/bpf_off_cpu.c @@ -0,0 +1,430 @@ +// SPDX-License-Identifier: GPL-2.0 +#include + +#include +#include +#include + +#include "bpf_counter.h" +#include "bpf_skel/off_cpu.skel.h" +#include "cgroup.h" +#include "cpumap.h" +#include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "off_cpu.h" +#include "parse-events.h" +#include "perf-hooks.h" +#include "record.h" +#include "session.h" +#include "strlist.h" +#include "target.h" +#include "thread_map.h" + +#define MAX_STACKS 32 +#define MAX_PROC 4096 +/* we don't need actual timestamp, just want to put the samples at last */ +#define OFF_CPU_TIMESTAMP (~0ull << 32) + +static struct off_cpu_bpf *skel; + +struct off_cpu_key { + u32 pid; + u32 tgid; + u32 stack_id; + u32 state; + u64 cgroup_id; +}; + +union off_cpu_data { + struct perf_event_header hdr; + u64 array[1024 / sizeof(u64)]; +}; + +static u64 off_cpu_raw[MAX_STACKS + 5]; + +static int off_cpu_config(struct evlist *evlist) +{ + char off_cpu_event[64]; + struct evsel *evsel; + + scnprintf(off_cpu_event, sizeof(off_cpu_event), "bpf-output/name=%s/", OFFCPU_EVENT); + if (parse_event(evlist, off_cpu_event)) { + pr_err("Failed to open off-cpu event\n"); + return -1; + } + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_offcpu_event(evsel)) { + evsel->core.system_wide = true; + break; + } + } + + return 0; +} + +static void off_cpu_start(void *arg) +{ + struct evlist *evlist = arg; + struct evsel *evsel; + struct perf_cpu pcpu; + unsigned int i; + + /* update task filter for the given workload */ + if (skel->rodata->has_task && skel->rodata->uses_tgid && + perf_thread_map__pid(evlist__core(evlist)->threads, 0) != -1) { + int fd; + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + pid = perf_thread_map__pid(evlist__core(evlist)->threads, 0); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + + /* update BPF perf_event map */ + evsel = evlist__find_evsel_by_str(evlist, OFFCPU_EVENT); + if (evsel == NULL) { + pr_err("%s evsel not found\n", OFFCPU_EVENT); + return; + } + + perf_cpu_map__for_each_cpu(pcpu, i, evsel->core.cpus) { + int err; + int cpu_nr = pcpu.cpu; + + err = bpf_map__update_elem(skel->maps.offcpu_output, &cpu_nr, sizeof(int), + xyarray__entry(evsel->core.fd, cpu_nr, 0), + sizeof(int), BPF_ANY); + if (err) { + pr_err("Failed to update perf event map for direct off-cpu dumping\n"); + return; + } + } + + skel->bss->enabled = 1; +} + +static void off_cpu_finish(void *arg __maybe_unused) +{ + skel->bss->enabled = 0; + off_cpu_bpf__destroy(skel); +} + +/* v5.18 kernel added prev_state arg, so it needs to check the signature */ +static void check_sched_switch_args(void) +{ + struct btf *btf = btf__load_vmlinux_btf(); + const struct btf_type *t1, *t2, *t3; + u32 type_id; + + if (!btf) { + pr_debug("Missing btf, check if CONFIG_DEBUG_INFO_BTF is enabled\n"); + goto cleanup; + } + + type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch", + BTF_KIND_TYPEDEF); + if ((s32)type_id < 0) + goto cleanup; + + t1 = btf__type_by_id(btf, type_id); + if (t1 == NULL) + goto cleanup; + + t2 = btf__type_by_id(btf, t1->type); + if (t2 == NULL || !btf_is_ptr(t2)) + goto cleanup; + + t3 = btf__type_by_id(btf, t2->type); + /* btf_trace func proto has one more argument for the context */ + if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) { + /* new format: pass prev_state as 4th arg */ + skel->rodata->has_prev_state = true; + } +cleanup: + btf__free(btf); +} + +int off_cpu_prepare(struct evlist *evlist, struct target *target, + struct record_opts *opts) +{ + int err, fd, i; + int ncpus = 1, ntasks = 1, ncgrps = 1; + struct strlist *pid_slist = NULL; + struct str_node *pos; + + if (off_cpu_config(evlist) < 0) { + pr_err("Failed to config off-cpu BPF event\n"); + return -1; + } + + skel = off_cpu_bpf__open(); + if (!skel) { + pr_err("Failed to open off-cpu BPF skeleton\n"); + return -1; + } + + /* don't need to set cpu filter for system-wide mode */ + if (target->cpu_list) { + ncpus = perf_cpu_map__nr(evlist__core(evlist)->user_requested_cpus); + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + skel->rodata->has_cpu = 1; + } + + if (target->pid) { + pid_slist = strlist__new(target->pid, NULL); + if (!pid_slist) { + pr_err("Failed to create a strlist for pid\n"); + return -1; + } + + ntasks = 0; + strlist__for_each_entry(pos, pid_slist) { + char *end_ptr; + int pid = strtol(pos->s, &end_ptr, 10); + + if (pid == INT_MIN || pid == INT_MAX || + (*end_ptr != '\0' && *end_ptr != ',')) + continue; + + ntasks++; + } + + if (ntasks < MAX_PROC) + ntasks = MAX_PROC; + + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + skel->rodata->uses_tgid = 1; + } else if (target__has_task(target)) { + ntasks = perf_thread_map__nr(evlist__core(evlist)->threads); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + } else if (target__none(target)) { + bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC); + skel->rodata->has_task = 1; + skel->rodata->uses_tgid = 1; + } + + if (evlist__first(evlist)->cgrp) { + ncgrps = evlist__nr_entries(evlist) - 1; /* excluding a dummy */ + bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); + + if (!cgroup_is_v2("perf_event")) + skel->rodata->uses_cgroup_v1 = true; + skel->rodata->has_cgroup = 1; + } + + if (opts->record_cgroup) { + skel->rodata->needs_cgroup = true; + + if (!cgroup_is_v2("perf_event")) + skel->rodata->uses_cgroup_v1 = true; + } + + set_max_rlimit(); + check_sched_switch_args(); + + err = off_cpu_bpf__load(skel); + if (err) { + pr_err("Failed to load off-cpu skeleton\n"); + goto out; + } + + if (target->cpu_list) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu(evlist__core(evlist)->user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target->pid) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + strlist__for_each_entry(pos, pid_slist) { + char *end_ptr; + u32 tgid; + int pid = strtol(pos->s, &end_ptr, 10); + + if (pid == INT_MIN || pid == INT_MAX || + (*end_ptr != '\0' && *end_ptr != ',')) + continue; + + tgid = pid; + bpf_map_update_elem(fd, &tgid, &val, BPF_ANY); + } + } else if (target__has_task(target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(evlist__core(evlist)->threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + if (evlist__first(evlist)->cgrp) { + struct evsel *evsel; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cgroup_filter); + + evlist__for_each_entry(evlist, evsel) { + struct cgroup *cgrp = evsel->cgrp; + + if (cgrp == NULL) + continue; + + if (!cgrp->id && read_cgroup_id(cgrp) < 0) { + pr_err("Failed to read cgroup id of %s\n", + cgrp->name); + goto out; + } + + bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY); + } + } + + skel->bss->offcpu_thresh_ns = opts->off_cpu_thresh_ns; + + err = off_cpu_bpf__attach(skel); + if (err) { + pr_err("Failed to attach off-cpu BPF skeleton\n"); + goto out; + } + + if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) || + perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) { + pr_err("Failed to attach off-cpu skeleton\n"); + goto out; + } + + return 0; + +out: + off_cpu_bpf__destroy(skel); + return -1; +} + +int off_cpu_write(struct perf_session *session) +{ + int bytes = 0, size; + int fd, stack; + u32 raw_size; + u64 sample_type, val, sid = 0; + struct evsel *evsel; + struct perf_data_file *file = &session->data->file; + struct off_cpu_key prev, key; + union off_cpu_data data = { + .hdr = { + .type = PERF_RECORD_SAMPLE, + .misc = PERF_RECORD_MISC_USER, + }, + }; + u64 tstamp = OFF_CPU_TIMESTAMP; + + skel->bss->enabled = 0; + + evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT); + if (evsel == NULL) { + pr_err("%s evsel not found\n", OFFCPU_EVENT); + return 0; + } + + sample_type = evsel->core.attr.sample_type; + + if (sample_type & ~OFFCPU_SAMPLE_TYPES) { + pr_err("not supported sample type: %llx\n", + (unsigned long long)sample_type); + return -1; + } + + if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) { + if (evsel->core.id) + sid = evsel->core.id[0]; + } + + fd = bpf_map__fd(skel->maps.off_cpu); + stack = bpf_map__fd(skel->maps.stacks); + memset(&prev, 0, sizeof(prev)); + + while (!bpf_map_get_next_key(fd, &prev, &key)) { + int n = 1; /* start from perf_event_header */ + + bpf_map_lookup_elem(fd, &key, &val); + + /* zero-fill some of the fields, will be overwritten by raw_data when parsing */ + if (sample_type & PERF_SAMPLE_IDENTIFIER) + data.array[n++] = sid; + if (sample_type & PERF_SAMPLE_IP) + data.array[n++] = 0; /* will be updated */ + if (sample_type & PERF_SAMPLE_TID) + data.array[n++] = 0; + if (sample_type & PERF_SAMPLE_TIME) + data.array[n++] = tstamp; + if (sample_type & PERF_SAMPLE_CPU) + data.array[n++] = 0; + if (sample_type & PERF_SAMPLE_PERIOD) + data.array[n++] = 0; + if (sample_type & PERF_SAMPLE_RAW) { + /* + * [ size ][ data ] + * [ data ] + * [ data ] + * [ data ] + * [ data ][ empty] + */ + int len = 0, i = 0; + void *raw_data = (void *)data.array + n * sizeof(u64); + + off_cpu_raw[i++] = (u64)key.pid << 32 | key.tgid; + off_cpu_raw[i++] = val; + + /* off_cpu_raw[i] is callchain->nr (updated later) */ + off_cpu_raw[i + 1] = PERF_CONTEXT_USER; + off_cpu_raw[i + 2] = 0; + + bpf_map_lookup_elem(stack, &key.stack_id, &off_cpu_raw[i + 2]); + while (off_cpu_raw[i + 2 + len]) + len++; + + off_cpu_raw[i] = len + 1; + i += len + 2; + + off_cpu_raw[i++] = key.cgroup_id; + + raw_size = i * sizeof(u64) + sizeof(u32); /* 4 bytes for alignment */ + memcpy(raw_data, &raw_size, sizeof(raw_size)); + memcpy(raw_data + sizeof(u32), off_cpu_raw, i * sizeof(u64)); + + n += i + 1; + } + if (sample_type & PERF_SAMPLE_CGROUP) + data.array[n++] = key.cgroup_id; + + size = n * sizeof(u64); + data.hdr.size = size; + bytes += size; + + if (perf_data_file__write(file, &data, size) < 0) { + pr_err("failed to write perf data, error: %m\n"); + return bytes; + } + + prev = key; + /* increase dummy timestamp to sort later samples */ + tstamp++; + } + return bytes; +} diff --git a/tools/perf/util/bpf_skel/.gitignore b/tools/perf/util/bpf_skel/.gitignore new file mode 100644 index 000000000..cd01455e1 --- /dev/null +++ b/tools/perf/util/bpf_skel/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +.tmp +*.skel.h +vmlinux.h diff --git a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c new file mode 100644 index 000000000..3bc9e28a9 --- /dev/null +++ b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c @@ -0,0 +1,621 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Augment the raw_syscalls tracepoints with the contents of the pointer arguments. + * + * This exactly matches what is marshalled into the raw_syscall:sys_enter + * payload expected by the 'perf trace' beautifiers. + */ + +#include "vmlinux.h" + +#include +#include + +#define PERF_ALIGN(x, a) __PERF_ALIGN_MASK(x, (typeof(x))(a)-1) +#define __PERF_ALIGN_MASK(x, mask) (((x)+(mask))&~(mask)) + +/** + * is_power_of_2() - check if a value is a power of two + * @n: the value to check + * + * Determine whether some value is a power of two, where zero is *not* + * considered a power of two. Return: true if @n is a power of 2, otherwise + * false. + */ +#define is_power_of_2(n) (n != 0 && ((n & (n - 1)) == 0)) + +#define MAX_CPUS 4096 + +#define TRACE_AUG_MAX_BUF 32 /* for buffer augmentation in perf trace */ + +/* bpf-output associated map */ +struct __augmented_syscalls__ { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __type(key, int); + __type(value, __u32); + __uint(max_entries, MAX_CPUS); +} __augmented_syscalls__ SEC(".maps"); + +/* + * What to augment at entry? + * + * Pointer arg payloads (filenames, etc) passed from userspace to the kernel + */ +struct syscalls_sys_enter { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, 1024); +} syscalls_sys_enter SEC(".maps"); + +/* + * What to augment at exit? + * + * Pointer arg payloads returned from the kernel (struct stat, etc) to userspace. + */ +struct syscalls_sys_exit { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, 1024); +} syscalls_sys_exit SEC(".maps"); + +struct syscall_enter_args { + unsigned long long common_tp_fields; + long syscall_nr; + unsigned long args[6]; +}; + +struct syscall_exit_args { + unsigned long long common_tp_fields; + long syscall_nr; + long ret; +}; + +/* + * Desired design of maximum size and alignment (see RFC2553) + */ +#define SS_MAXSIZE 128 /* Implementation specific max size */ + +typedef unsigned short sa_family_t; + +/* + * FIXME: Should come from system headers + * + * The definition uses anonymous union and struct in order to control the + * default alignment. + */ +struct sockaddr_storage { + union { + struct { + sa_family_t ss_family; /* address family */ + /* Following field(s) are implementation specific */ + char __data[SS_MAXSIZE - sizeof(unsigned short)]; + /* space to achieve desired size, */ + /* _SS_MAXSIZE value minus size of ss_family */ + }; + void *__align; /* implementation specific desired alignment */ + }; +}; + +struct augmented_arg { + unsigned int size; + int err; + union { + char value[PATH_MAX]; + struct sockaddr_storage saddr; + }; +}; + +struct pids_filtered { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, pid_t); + __type(value, bool); + __uint(max_entries, 64); +} pids_filtered SEC(".maps"); + +struct augmented_args_payload { + struct syscall_enter_args args; + struct augmented_arg arg, arg2; // We have to reserve space for two arguments (rename, etc) +}; + +// We need more tmp space than the BPF stack can give us +struct augmented_args_tmp { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct augmented_args_payload); + __uint(max_entries, 1); +} augmented_args_tmp SEC(".maps"); + +struct beauty_map_enter { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); + __type(value, __u32[6]); + __uint(max_entries, 512); +} beauty_map_enter SEC(".maps"); + +struct beauty_payload_enter { + struct syscall_enter_args args; + struct augmented_arg aug_args[6]; +}; + +struct beauty_payload_enter_map { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct beauty_payload_enter); + __uint(max_entries, 1); +} beauty_payload_enter_map SEC(".maps"); + +static inline struct augmented_args_payload *augmented_args_payload(void) +{ + int key = 0; + return bpf_map_lookup_elem(&augmented_args_tmp, &key); +} + +static inline int augmented__output(void *ctx, struct augmented_args_payload *args, int len) +{ + /* If perf_event_output fails, return non-zero so that it gets recorded unaugmented */ + return bpf_perf_event_output(ctx, &__augmented_syscalls__, BPF_F_CURRENT_CPU, args, len); +} + +static inline int augmented__beauty_output(void *ctx, void *data, int len) +{ + return bpf_perf_event_output(ctx, &__augmented_syscalls__, BPF_F_CURRENT_CPU, data, len); +} + +static inline +unsigned int augmented_arg__read_str(struct augmented_arg *augmented_arg, const void *arg, unsigned int arg_len) +{ + unsigned int augmented_len = sizeof(*augmented_arg); + int string_len = bpf_probe_read_user_str(&augmented_arg->value, arg_len, arg); + + augmented_arg->size = augmented_arg->err = 0; + /* + * probe_read_str may return < 0, e.g. -EFAULT + * So we leave that in the augmented_arg->size that userspace will + */ + if (string_len > 0) { + augmented_len -= sizeof(augmented_arg->value) - string_len; + _Static_assert(is_power_of_2(sizeof(augmented_arg->value)), "sizeof(augmented_arg->value) needs to be a power of two"); + augmented_len &= sizeof(augmented_arg->value) - 1; + augmented_arg->size = string_len; + } else { + /* + * So that username notice the error while still being able + * to skip this augmented arg record + */ + augmented_arg->err = string_len; + augmented_len = offsetof(struct augmented_arg, value); + } + + return augmented_len; +} + +SEC("tp/raw_syscalls/sys_enter") +int syscall_unaugmented(struct syscall_enter_args *args) +{ + return 1; +} + +/* + * These will be tail_called from SEC("raw_syscalls:sys_enter"), so will find in + * augmented_args_tmp what was read by that raw_syscalls:sys_enter and go + * on from there, reading the first syscall arg as a string, i.e. open's + * filename. + */ +SEC("tp/syscalls/sys_enter_connect") +int sys_enter_connect(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *sockaddr_arg = (const void *)args->args[1]; + unsigned int socklen = args->args[2]; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + _Static_assert(is_power_of_2(sizeof(augmented_args->arg.saddr)), "sizeof(augmented_args->arg.saddr) needs to be a power of two"); + socklen &= sizeof(augmented_args->arg.saddr) - 1; + + bpf_probe_read_user(&augmented_args->arg.saddr, socklen, sockaddr_arg); + augmented_args->arg.size = socklen; + augmented_args->arg.err = 0; + + return augmented__output(args, augmented_args, len + socklen); +} + +SEC("tp/syscalls/sys_enter_sendto") +int sys_enter_sendto(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *sockaddr_arg = (const void *)args->args[4]; + unsigned int socklen = args->args[5]; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + socklen &= sizeof(augmented_args->arg.saddr) - 1; + + bpf_probe_read_user(&augmented_args->arg.saddr, socklen, sockaddr_arg); + + return augmented__output(args, augmented_args, len + socklen); +} + +SEC("tp/syscalls/sys_enter_open") +int sys_enter_open(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *filename_arg = (const void *)args->args[0]; + unsigned int len = sizeof(augmented_args->args); + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += augmented_arg__read_str(&augmented_args->arg, filename_arg, sizeof(augmented_args->arg.value)); + + return augmented__output(args, augmented_args, len); +} + +SEC("tp/syscalls/sys_enter_openat") +int sys_enter_openat(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *filename_arg = (const void *)args->args[1]; + unsigned int len = sizeof(augmented_args->args); + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += augmented_arg__read_str(&augmented_args->arg, filename_arg, sizeof(augmented_args->arg.value)); + + return augmented__output(args, augmented_args, len); +} + +SEC("tp/syscalls/sys_enter_rename") +int sys_enter_rename(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *oldpath_arg = (const void *)args->args[0], + *newpath_arg = (const void *)args->args[1]; + unsigned int len = sizeof(augmented_args->args), oldpath_len, newpath_len; + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += 2 * sizeof(u64); // The overhead of size and err, just before the payload... + + oldpath_len = augmented_arg__read_str(&augmented_args->arg, oldpath_arg, sizeof(augmented_args->arg.value)); + augmented_args->arg.size = PERF_ALIGN(oldpath_len + 1, sizeof(u64)); + len += augmented_args->arg.size; + + /* Every read from userspace is limited to value size */ + if (augmented_args->arg.size > sizeof(augmented_args->arg.value)) + return 1; /* Failure: don't filter */ + + struct augmented_arg *arg2 = (void *)&augmented_args->arg.value + augmented_args->arg.size; + + newpath_len = augmented_arg__read_str(arg2, newpath_arg, sizeof(augmented_args->arg.value)); + arg2->size = newpath_len; + + len += newpath_len; + + return augmented__output(args, augmented_args, len); +} + +SEC("tp/syscalls/sys_enter_renameat2") +int sys_enter_renameat2(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *oldpath_arg = (const void *)args->args[1], + *newpath_arg = (const void *)args->args[3]; + unsigned int len = sizeof(augmented_args->args), oldpath_len, newpath_len; + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += 2 * sizeof(u64); // The overhead of size and err, just before the payload... + + oldpath_len = augmented_arg__read_str(&augmented_args->arg, oldpath_arg, sizeof(augmented_args->arg.value)); + augmented_args->arg.size = PERF_ALIGN(oldpath_len + 1, sizeof(u64)); + len += augmented_args->arg.size; + + /* Every read from userspace is limited to value size */ + if (augmented_args->arg.size > sizeof(augmented_args->arg.value)) + return 1; /* Failure: don't filter */ + + struct augmented_arg *arg2 = (void *)&augmented_args->arg.value + augmented_args->arg.size; + + newpath_len = augmented_arg__read_str(arg2, newpath_arg, sizeof(augmented_args->arg.value)); + arg2->size = newpath_len; + + len += newpath_len; + + return augmented__output(args, augmented_args, len); +} + +#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */ + +// we need just the start, get the size to then copy it +struct perf_event_attr_size { + __u32 type; + /* + * Size of the attr structure, for fwd/bwd compat. + */ + __u32 size; +}; + +SEC("tp/syscalls/sys_enter_perf_event_open") +int sys_enter_perf_event_open(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const struct perf_event_attr_size *attr = (const struct perf_event_attr_size *)args->args[0], *attr_read; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + + if (augmented_args == NULL) + goto failure; + + if (bpf_probe_read_user(&augmented_args->arg.value, sizeof(*attr), attr) < 0) + goto failure; + + attr_read = (const struct perf_event_attr_size *)augmented_args->arg.value; + + __u32 size = attr_read->size; + + if (!size) + size = PERF_ATTR_SIZE_VER0; + + if (size > sizeof(augmented_args->arg.value)) + goto failure; + + // Now that we read attr->size and tested it against the size limits, read it completely + if (bpf_probe_read_user(&augmented_args->arg.value, size, attr) < 0) + goto failure; + + return augmented__output(args, augmented_args, len + size); +failure: + return 1; /* Failure: don't filter */ +} + +SEC("tp/syscalls/sys_enter_clock_nanosleep") +int sys_enter_clock_nanosleep(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *rqtp_arg = (const void *)args->args[2]; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + __u32 size = sizeof(struct timespec64); + + if (augmented_args == NULL) + goto failure; + + if (size > sizeof(augmented_args->arg.value)) + goto failure; + + bpf_probe_read_user(&augmented_args->arg.value, size, rqtp_arg); + + return augmented__output(args, augmented_args, len + size); +failure: + return 1; /* Failure: don't filter */ +} + +SEC("tp/syscalls/sys_enter_nanosleep") +int sys_enter_nanosleep(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *req_arg = (const void *)args->args[0]; + unsigned int len = sizeof(augmented_args->args); + __u32 size = sizeof(struct timespec64); + + if (augmented_args == NULL) + goto failure; + + if (size > sizeof(augmented_args->arg.value)) + goto failure; + + bpf_probe_read_user(&augmented_args->arg.value, size, req_arg); + + return augmented__output(args, augmented_args, len + size); +failure: + return 1; /* Failure: don't filter */ +} + +static pid_t getpid(void) +{ + return bpf_get_current_pid_tgid(); +} + +static bool pid_filter__has(struct pids_filtered *pids, pid_t pid) +{ + return bpf_map_lookup_elem(pids, &pid) != NULL; +} + +u64 ZERO = 0; + +/* + * Determine what type of argument and how many bytes to read from user space, using the + * value in the beauty_map. This is the relation of parameter type and its corresponding + * value in the beauty map, and how many bytes we read eventually: + * + * string: 1 -> size of string + * struct: size of struct -> size of struct + * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF) + */ +static inline int augment_arg(struct syscall_enter_args *args, int i, + unsigned int *beauty_map, + struct beauty_payload_enter *payload, u64 offset) +{ + int index, value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value); + struct augmented_arg *payload_offset; + s64 aug_size, size; + bool augmented; + void *arg; + + arg = (void *)args->args[i]; + augmented = false; + size = beauty_map[i]; + aug_size = size; /* size of the augmented data read from user space */ + + if (size == 0 || arg == NULL) + return 0; + + /* bounds check for the verifier */ + if (offset > sizeof(payload->aug_args) - sizeof(payload->aug_args[0])) + return -1; + barrier_var(offset); + payload_offset = (struct augmented_arg *)((void *)&payload->aug_args + offset); + + if (size == 1) { /* string */ + aug_size = bpf_probe_read_user_str(payload_offset->value, value_size, arg); + /* minimum of 0 to pass the verifier */ + if (aug_size < 0) + aug_size = 0; + + augmented = true; + } else if (size > 0 && size <= value_size) { /* struct */ + if (!bpf_probe_read_user(payload_offset->value, size, arg)) + augmented = true; + } else if ((int)size < 0 && size >= -6) { /* buffer */ + index = -(size + 1); + barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick. + index &= 7; // Satisfy the bounds checking with the verifier in some kernels. + aug_size = args->args[index] > TRACE_AUG_MAX_BUF ? TRACE_AUG_MAX_BUF : args->args[index]; + + if (aug_size > 0) { + if (!bpf_probe_read_user(payload_offset->value, aug_size, arg)) + augmented = true; + } + } + + /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */ + if (aug_size > value_size) + aug_size = value_size; + + /* write data to payload */ + if (augmented) { + int written = offsetof(struct augmented_arg, value) + aug_size; + + if (written < 0 || written > sizeof(struct augmented_arg)) + return -1; + + payload_offset->size = aug_size; + return written; + } + + return 0; +} + +static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) +{ + bool do_output = false; + int i, zero = 0, written; + u64 output = 0; /* has to be u64, otherwise it won't pass the verifier */ + unsigned int nr, *beauty_map; + struct beauty_payload_enter *payload; + + /* fall back to do predefined tail call */ + if (args == NULL) + return 1; + + /* use syscall number to get beauty_map entry */ + nr = (__u32)args->syscall_nr; + beauty_map = bpf_map_lookup_elem(&beauty_map_enter, &nr); + + /* set up payload for output */ + payload = bpf_map_lookup_elem(&beauty_payload_enter_map, &zero); + + if (beauty_map == NULL || payload == NULL) + return 1; + + /* copy the sys_enter header, which has the syscall_nr */ + __builtin_memcpy(&payload->args, args, sizeof(struct syscall_enter_args)); + + if (bpf_ksym_exists(bpf_iter_num_new)) { + bpf_for(i, 0, 6) { + written = augment_arg(args, i, beauty_map, payload, output); + if (written < 0) + return 1; + if (written > 0) { + output += written; + /* + * guide the verifier to forget range of `output`, which + * helps to prove convergence of the loop + */ + output += ZERO; + do_output = true; + } + } + } else { + for (i = 0; i < 6; i++) { + written = augment_arg(args, i, beauty_map, payload, output); + if (written < 0) + return 1; + if (written > 0) { + output += written; + do_output = true; + } + } + } + + if (!do_output || (sizeof(struct syscall_enter_args) + output) > sizeof(struct beauty_payload_enter)) + return 1; + + return augmented__beauty_output(ctx, payload, sizeof(struct syscall_enter_args) + output); +} + +SEC("tp/raw_syscalls/sys_enter") +int sys_enter(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args; + /* + * We start len, the amount of data that will be in the perf ring + * buffer, if this is not filtered out by one of pid_filter__has(), + * syscall->enabled, etc, with the non-augmented raw syscall payload, + * i.e. sizeof(augmented_args->args). + * + * We'll add to this as we add augmented syscalls right after that + * initial, non-augmented raw_syscalls:sys_enter payload. + */ + + if (pid_filter__has(&pids_filtered, getpid())) + return 0; + + augmented_args = augmented_args_payload(); + if (augmented_args == NULL) + return 1; + + bpf_probe_read_kernel(&augmented_args->args, sizeof(augmented_args->args), args); + + /* + * Jump to syscall specific augmenter, even if the default one, + * "!raw_syscalls:unaugmented" that will just return 1 to return the + * unaugmented tracepoint payload. + */ + if (augment_sys_enter(args, &augmented_args->args)) + bpf_tail_call(args, &syscalls_sys_enter, augmented_args->args.syscall_nr); + + // If not found on the PROG_ARRAY syscalls map, then we're filtering it: + return 0; +} + +SEC("tp/raw_syscalls/sys_exit") +int sys_exit(struct syscall_exit_args *args) +{ + struct syscall_exit_args exit_args; + + if (pid_filter__has(&pids_filtered, getpid())) + return 0; + + bpf_probe_read_kernel(&exit_args, sizeof(exit_args), args); + /* + * Jump to syscall specific return augmenter, even if the default one, + * "!raw_syscalls:unaugmented" that will just return 1 to return the + * unaugmented tracepoint payload. + */ + bpf_tail_call(args, &syscalls_sys_exit, exit_args.syscall_nr); + /* + * If not found on the PROG_ARRAY syscalls map, then we're filtering it: + */ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c new file mode 100644 index 000000000..c2298a2de --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +// Copyright (c) 2021 Google +#include "bperf_cgroup.h" +#include "vmlinux.h" +#include +#include +#include + +// NOTE: many of map and global data will be modified before loading +// from the userspace (perf tool) using the skeleton helpers. + +// single set of global perf events to measure +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); + __uint(max_entries, 1); +} events SEC(".maps"); + +// from cgroup id to event index +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, 1); +} cgrp_idx SEC(".maps"); + +// per-cpu event snapshots to calculate delta +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); +} prev_readings SEC(".maps"); + +// aggregated event values for each cgroup (per-cpu) +// will be read from the user-space +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); +} cgrp_readings SEC(".maps"); + +/* new kernel cgroup definition */ +struct cgroup___new { + int level; + struct cgroup *ancestors[]; +} __attribute__((preserve_access_index)); + +/* old kernel cgroup definition */ +struct cgroup___old { + int level; + u64 ancestor_ids[]; +} __attribute__((preserve_access_index)); + +const volatile __u32 num_events = 1; +const volatile __u32 num_cpus = 1; +const volatile int use_cgroup_v2 = 0; + +int enabled = 0; +int perf_subsys_id = -1; + +static inline __u64 get_cgroup_v1_ancestor_id(struct cgroup *cgrp, int level) +{ + /* recast pointer to capture new type for compiler */ + struct cgroup___new *cgrp_new = (void *)cgrp; + + if (bpf_core_field_exists(cgrp_new->ancestors)) { + return BPF_CORE_READ(cgrp_new, ancestors[level], kn, id); + } else { + /* recast pointer to capture old type for compiler */ + struct cgroup___old *cgrp_old = (void *)cgrp; + + return BPF_CORE_READ(cgrp_old, ancestor_ids[level]); + } +} + +static inline int get_cgroup_v1_idx(__u32 *cgrps, int size) +{ + struct task_struct *p = (void *)bpf_get_current_task(); + struct cgroup *cgrp; + register int i = 0; + __u32 *elem; + int level; + int cnt; + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_subsys_id], cgroup); + level = BPF_CORE_READ(cgrp, level); + + for (cnt = 0; i < BPERF_CGROUP__MAX_LEVELS; i++) { + __u64 cgrp_id; + + if (i > level) + break; + + // convert cgroup-id to a map index + cgrp_id = get_cgroup_v1_ancestor_id(cgrp, i); + elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); + if (!elem) + continue; + + cgrps[cnt++] = *elem; + if (cnt == size) + break; + } + + return cnt; +} + +static inline int get_cgroup_v2_idx(__u32 *cgrps, int size) +{ + register int i = 0; + __u32 *elem; + int cnt; + + for (cnt = 0; i < BPERF_CGROUP__MAX_LEVELS; i++) { + __u64 cgrp_id = bpf_get_current_ancestor_cgroup_id(i); + + if (cgrp_id == 0) + break; + + // convert cgroup-id to a map index + elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); + if (!elem) + continue; + + cgrps[cnt++] = *elem; + if (cnt == size) + break; + } + + return cnt; +} + +static int bperf_cgroup_count(void) +{ + register __u32 idx = 0; // to have it in a register to pass BPF verifier + register int c = 0; + struct bpf_perf_event_value val, delta, *prev_val, *cgrp_val; + __u32 cpu = bpf_get_smp_processor_id(); + __u32 cgrp_idx[BPERF_CGROUP__MAX_LEVELS]; + int cgrp_cnt; + __u32 key, cgrp; + long err; + + if (use_cgroup_v2) + cgrp_cnt = get_cgroup_v2_idx(cgrp_idx, BPERF_CGROUP__MAX_LEVELS); + else + cgrp_cnt = get_cgroup_v1_idx(cgrp_idx, BPERF_CGROUP__MAX_LEVELS); + + for ( ; idx < BPERF_CGROUP__MAX_EVENTS; idx++) { + if (idx == num_events) + break; + + // XXX: do not pass idx directly (for verifier) + key = idx; + // this is per-cpu array for diff + prev_val = bpf_map_lookup_elem(&prev_readings, &key); + if (!prev_val) { + val.counter = val.enabled = val.running = 0; + bpf_map_update_elem(&prev_readings, &key, &val, BPF_ANY); + + prev_val = bpf_map_lookup_elem(&prev_readings, &key); + if (!prev_val) + continue; + } + + // read from global perf_event array + key = idx * num_cpus + cpu; + err = bpf_perf_event_read_value(&events, key, &val, sizeof(val)); + if (err) + continue; + + if (enabled) { + delta.counter = val.counter - prev_val->counter; + delta.enabled = val.enabled - prev_val->enabled; + delta.running = val.running - prev_val->running; + + for (c = 0; c < BPERF_CGROUP__MAX_LEVELS; c++) { + if (c == cgrp_cnt) + break; + + cgrp = cgrp_idx[c]; + + // aggregate the result by cgroup + key = cgrp * num_events + idx; + cgrp_val = bpf_map_lookup_elem(&cgrp_readings, &key); + if (cgrp_val) { + cgrp_val->counter += delta.counter; + cgrp_val->enabled += delta.enabled; + cgrp_val->running += delta.running; + } else { + bpf_map_update_elem(&cgrp_readings, &key, + &delta, BPF_ANY); + } + } + } + + *prev_val = val; + } + return 0; +} + +// This will be attached to cgroup-switches event for each cpu +SEC("perf_event") +int BPF_PROG(on_cgrp_switch) +{ + return bperf_cgroup_count(); +} + +SEC("raw_tp/sched_switch") +int BPF_PROG(trigger_read) +{ + return bperf_cgroup_count(); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.h b/tools/perf/util/bpf_skel/bperf_cgroup.h new file mode 100644 index 000000000..3fb84b19d --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_cgroup.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* Data structures shared between BPF and tools. */ +#ifndef __BPERF_CGROUP_H +#define __BPERF_CGROUP_H + +// These constants impact code size of bperf_cgroup.bpf.c that may result in BPF +// verifier issues. They are exposed to control the size and also to disable BPF +// counters when the number of user events is too large. + +// max cgroup hierarchy level: arbitrary +#define BPERF_CGROUP__MAX_LEVELS 10 +// max events per cgroup: arbitrary +#define BPERF_CGROUP__MAX_EVENTS 128 + +#endif /* __BPERF_CGROUP_H */ diff --git a/tools/perf/util/bpf_skel/bperf_follower.bpf.c b/tools/perf/util/bpf_skel/bperf_follower.bpf.c new file mode 100644 index 000000000..059506313 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_follower.bpf.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +#include "vmlinux.h" +#include +#include +#include "bperf_u.h" + +#define MAX_ENTRIES 102400 + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} diff_readings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} accum_readings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bperf_filter_value)); + __uint(max_entries, MAX_ENTRIES); + __uint(map_flags, BPF_F_NO_PREALLOC); +} filter SEC(".maps"); + +enum bperf_filter_type type = 0; +int enabled = 0; +int inherit; + +SEC("fexit/XXX") +int BPF_PROG(fexit_XXX) +{ + struct bpf_perf_event_value *diff_val, *accum_val; + __u32 filter_key, zero = 0; + __u32 accum_key; + struct bperf_filter_value *fval; + + if (!enabled) + return 0; + + switch (type) { + case BPERF_FILTER_GLOBAL: + accum_key = zero; + goto do_add; + case BPERF_FILTER_CPU: + filter_key = bpf_get_smp_processor_id(); + break; + case BPERF_FILTER_PID: + filter_key = bpf_get_current_pid_tgid() & 0xffffffff; + break; + case BPERF_FILTER_TGID: + /* Use pid as the filter_key to exclude new task counts + * when inherit is disabled. Don't worry about the existing + * children in TGID losing their counts, bpf_counter has + * already added them to the filter map via perf_thread_map + * before this bpf prog runs. + */ + filter_key = inherit ? + bpf_get_current_pid_tgid() >> 32 : + bpf_get_current_pid_tgid() & 0xffffffff; + break; + default: + return 0; + } + + fval = bpf_map_lookup_elem(&filter, &filter_key); + if (!fval) + return 0; + + accum_key = fval->accum_key; + if (fval->exited) + bpf_map_delete_elem(&filter, &filter_key); + +do_add: + diff_val = bpf_map_lookup_elem(&diff_readings, &zero); + if (!diff_val) + return 0; + + accum_val = bpf_map_lookup_elem(&accum_readings, &accum_key); + if (!accum_val) + return 0; + + accum_val->counter += diff_val->counter; + accum_val->enabled += diff_val->enabled; + accum_val->running += diff_val->running; + + return 0; +} + +/* The program is only used for PID or TGID filter types. */ +SEC("tp_btf/task_newtask") +int BPF_PROG(on_newtask, struct task_struct *task, __u64 clone_flags) +{ + __u32 parent_key, child_key; + struct bperf_filter_value *parent_fval; + struct bperf_filter_value child_fval = { 0 }; + + if (!enabled) + return 0; + + switch (type) { + case BPERF_FILTER_PID: + parent_key = bpf_get_current_pid_tgid() & 0xffffffff; + child_key = task->pid; + break; + case BPERF_FILTER_TGID: + parent_key = bpf_get_current_pid_tgid() >> 32; + child_key = task->tgid; + if (child_key == parent_key) + return 0; + break; + default: + return 0; + } + + /* Check if the current task is one of the target tasks to be counted */ + parent_fval = bpf_map_lookup_elem(&filter, &parent_key); + if (!parent_fval) + return 0; + + /* Start counting for the new task by adding it into filter map, + * inherit the accum key of its parent task so that they can be + * counted together. + */ + child_fval.accum_key = parent_fval->accum_key; + child_fval.exited = 0; + bpf_map_update_elem(&filter, &child_key, &child_fval, BPF_NOEXIST); + + return 0; +} + +/* The program is only used for PID or TGID filter types. */ +SEC("tp_btf/sched_process_exit") +int BPF_PROG(on_exittask, struct task_struct *task) +{ + __u32 pid; + struct bperf_filter_value *fval; + + if (!enabled) + return 0; + + /* Stop counting for this task by removing it from filter map. + * For TGID type, if the pid can be found in the map, it means that + * this pid belongs to the leader task. After the task exits, the + * tgid of its child tasks (if any) will be 1, so the pid can be + * safely removed. + */ + pid = task->pid; + fval = bpf_map_lookup_elem(&filter, &pid); + if (fval) + fval->exited = 1; + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_leader.bpf.c b/tools/perf/util/bpf_skel/bperf_leader.bpf.c new file mode 100644 index 000000000..e2a2d4cd7 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_leader.bpf.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +#include "vmlinux.h" +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); + __uint(map_flags, BPF_F_PRESERVE_ELEMS); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} prev_readings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} diff_readings SEC(".maps"); + +SEC("raw_tp/sched_switch") +int BPF_PROG(on_switch) +{ + struct bpf_perf_event_value val, *prev_val, *diff_val; + __u32 key = bpf_get_smp_processor_id(); + __u32 zero = 0; + long err; + + prev_val = bpf_map_lookup_elem(&prev_readings, &zero); + if (!prev_val) + return 0; + + diff_val = bpf_map_lookup_elem(&diff_readings, &zero); + if (!diff_val) + return 0; + + err = bpf_perf_event_read_value(&events, key, &val, sizeof(val)); + if (err) + return 0; + + diff_val->counter = val.counter - prev_val->counter; + diff_val->enabled = val.enabled - prev_val->enabled; + diff_val->running = val.running - prev_val->running; + *prev_val = val; + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_u.h b/tools/perf/util/bpf_skel/bperf_u.h new file mode 100644 index 000000000..4a4a75398 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_u.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook + +#ifndef __BPERF_STAT_U_H +#define __BPERF_STAT_U_H + +enum bperf_filter_type { + BPERF_FILTER_GLOBAL = 1, + BPERF_FILTER_CPU, + BPERF_FILTER_PID, + BPERF_FILTER_TGID, +}; + +struct bperf_filter_value { + __u32 accum_key; + __u8 exited; +}; + +#endif /* __BPERF_STAT_U_H */ diff --git a/tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c b/tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c new file mode 100644 index 000000000..97037d3b3 --- /dev/null +++ b/tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2020 Facebook +#include "vmlinux.h" +#include +#include + +/* map of perf event fds, num_cpu * num_metric entries */ +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); +} events SEC(".maps"); + +/* readings at fentry */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} fentry_readings SEC(".maps"); + +/* accumulated readings */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} accum_readings SEC(".maps"); + +const volatile __u32 num_cpu = 1; + +SEC("fentry/XXX") +int BPF_PROG(fentry_XXX) +{ + __u32 key = bpf_get_smp_processor_id(); + struct bpf_perf_event_value *ptr; + __u32 zero = 0; + long err; + + /* look up before reading, to reduce error */ + ptr = bpf_map_lookup_elem(&fentry_readings, &zero); + if (!ptr) + return 0; + + err = bpf_perf_event_read_value(&events, key, ptr, sizeof(*ptr)); + if (err) + return 0; + + return 0; +} + +static inline void +fexit_update_maps(struct bpf_perf_event_value *after) +{ + struct bpf_perf_event_value *before, diff; + __u32 zero = 0; + + before = bpf_map_lookup_elem(&fentry_readings, &zero); + /* only account samples with a valid fentry_reading */ + if (before && before->counter) { + struct bpf_perf_event_value *accum; + + diff.counter = after->counter - before->counter; + diff.enabled = after->enabled - before->enabled; + diff.running = after->running - before->running; + + accum = bpf_map_lookup_elem(&accum_readings, &zero); + if (accum) { + accum->counter += diff.counter; + accum->enabled += diff.enabled; + accum->running += diff.running; + } + } +} + +SEC("fexit/XXX") +int BPF_PROG(fexit_XXX) +{ + struct bpf_perf_event_value reading; + __u32 cpu = bpf_get_smp_processor_id(); + int err; + + /* read all events before updating the maps, to reduce error */ + err = bpf_perf_event_read_value(&events, cpu, &reading, sizeof(reading)); + if (err) + return 0; + + fexit_update_maps(&reading); + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/func_latency.bpf.c b/tools/perf/util/bpf_skel/func_latency.bpf.c new file mode 100644 index 000000000..621e2022c --- /dev/null +++ b/tools/perf/util/bpf_skel/func_latency.bpf.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Google +#include "vmlinux.h" +#include +#include + +// This should be in sync with "util/ftrace.h" +#define NUM_BUCKET 22 + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, 10000); +} functime SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, NUM_BUCKET); +} latency SEC(".maps"); + + +int enabled = 0; + +// stats +__s64 total; +__s64 count; +__s64 max; +__s64 min; + +const volatile int has_cpu = 0; +const volatile int has_task = 0; +const volatile int use_nsec = 0; +const volatile unsigned int bucket_range; +const volatile unsigned int min_latency; +const volatile unsigned int max_latency; +const volatile unsigned int bucket_num = NUM_BUCKET; + +static bool can_record(void) +{ + if (has_cpu) { + __u32 cpu = bpf_get_smp_processor_id(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return false; + } + + if (has_task) { + __u32 pid = bpf_get_current_pid_tgid(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return false; + } + return true; +} + +static void update_latency(__s64 delta) +{ + __u64 val = delta; + __u32 key = 0; + __u64 *hist; + __u64 cmp_base = use_nsec ? 1 : 1000; + + if (delta < 0) + return; + + if (bucket_range != 0) { + val = delta / cmp_base; + + if (min_latency > 0) { + if (val > min_latency) + val -= min_latency; + else + goto do_lookup; + } + + // Less than 1 unit (ms or ns), or, in the future, + // than the min latency desired. + if (val > 0) { // 1st entry: [ 1 unit .. bucket_range units ) + key = val / bucket_range + 1; + if (key >= bucket_num) + key = bucket_num - 1; + } + + goto do_lookup; + } + // calculate index using delta + for (key = 0; key < (bucket_num - 1); key++) { + if (delta < (cmp_base << key)) + break; + } + +do_lookup: + hist = bpf_map_lookup_elem(&latency, &key); + if (!hist) + return; + + __sync_fetch_and_add(hist, 1); + + __sync_fetch_and_add(&total, delta); // always in nsec + __sync_fetch_and_add(&count, 1); + + if (delta > max) + max = delta; + if (delta < min) + min = delta; +} + +SEC("kprobe/func") +int BPF_PROG(func_begin) +{ + __u64 key, now; + + if (!enabled || !can_record()) + return 0; + + key = bpf_get_current_pid_tgid(); + now = bpf_ktime_get_ns(); + + // overwrite timestamp for nested functions + bpf_map_update_elem(&functime, &key, &now, BPF_ANY); + return 0; +} + +SEC("kretprobe/func") +int BPF_PROG(func_end) +{ + __u64 tid; + __u64 *start; + + if (!enabled) + return 0; + + tid = bpf_get_current_pid_tgid(); + + start = bpf_map_lookup_elem(&functime, &tid); + if (start) { + update_latency(bpf_ktime_get_ns() - *start); + bpf_map_delete_elem(&functime, &tid); + } + + return 0; +} + +SEC("raw_tp") +int BPF_PROG(event_begin) +{ + __u64 key, now; + + if (!enabled || !can_record()) + return 0; + + key = bpf_get_current_pid_tgid(); + now = bpf_ktime_get_ns(); + + // overwrite timestamp for nested events + bpf_map_update_elem(&functime, &key, &now, BPF_ANY); + return 0; +} + +SEC("raw_tp") +int BPF_PROG(event_end) +{ + __u64 tid; + __u64 *start; + + if (!enabled) + return 0; + + tid = bpf_get_current_pid_tgid(); + + start = bpf_map_lookup_elem(&functime, &tid); + if (start) { + update_latency(bpf_ktime_get_ns() - *start); + bpf_map_delete_elem(&functime, &tid); + } + + return 0; +} diff --git a/tools/perf/util/bpf_skel/kwork_top.bpf.c b/tools/perf/util/bpf_skel/kwork_top.bpf.c new file mode 100644 index 000000000..667338630 --- /dev/null +++ b/tools/perf/util/bpf_skel/kwork_top.bpf.c @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022, Huawei + +#include "vmlinux.h" +#include +#include +#include + +/* + * This should be in sync with "util/kwork.h" + */ +enum kwork_class_type { + KWORK_CLASS_IRQ, + KWORK_CLASS_SOFTIRQ, + KWORK_CLASS_WORKQUEUE, + KWORK_CLASS_SCHED, + KWORK_CLASS_MAX, +}; + +#define MAX_ENTRIES 102400 +#define MAX_NR_CPUS 4096 +#define PF_KTHREAD 0x00200000 +#define MAX_COMMAND_LEN 16 + +struct time_data { + __u64 timestamp; +}; + +struct work_data { + __u64 runtime; +}; + +struct task_data { + __u32 tgid; + __u32 is_kthread; + char comm[MAX_COMMAND_LEN]; +}; + +struct work_key { + __u32 type; + __u32 pid; + __u64 task_p; +}; + +struct task_key { + __u32 pid; + __u32 cpu; +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct time_data); +} kwork_top_task_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct time_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_irq_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct task_key)); + __uint(value_size, sizeof(struct task_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_tasks SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct work_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_works SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u8)); + __uint(max_entries, MAX_NR_CPUS); +} kwork_top_cpu_filter SEC(".maps"); + +int enabled = 0; + +const volatile int has_cpu_filter = 0; + +__u64 from_timestamp = 0; +__u64 to_timestamp = 0; + +static __always_inline int cpu_is_filtered(__u32 cpu) +{ + __u8 *cpu_val; + + if (has_cpu_filter) { + cpu_val = bpf_map_lookup_elem(&kwork_top_cpu_filter, &cpu); + if (!cpu_val) + return 1; + } + + return 0; +} + +static __always_inline void update_task_info(struct task_struct *task, __u32 cpu) +{ + struct task_key key = { + .pid = task->pid, + .cpu = cpu, + }; + + if (!bpf_map_lookup_elem(&kwork_top_tasks, &key)) { + struct task_data data = { + .tgid = task->tgid, + .is_kthread = task->flags & PF_KTHREAD ? 1 : 0, + }; + BPF_CORE_READ_STR_INTO(&data.comm, task, comm); + + bpf_map_update_elem(&kwork_top_tasks, &key, &data, BPF_ANY); + } +} + +static __always_inline void update_work(struct work_key *key, __u64 delta) +{ + struct work_data *data; + + data = bpf_map_lookup_elem(&kwork_top_works, key); + if (data) { + data->runtime += delta; + } else { + struct work_data new_data = { + .runtime = delta, + }; + + bpf_map_update_elem(&kwork_top_works, key, &new_data, BPF_ANY); + } +} + +static void on_sched_out(struct task_struct *task, __u64 ts, __u32 cpu) +{ + __u64 delta; + struct time_data *pelem; + + pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, 0); + if (pelem) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + struct work_key key = { + .type = KWORK_CLASS_SCHED, + .pid = task->pid, + .task_p = (__u64)task, + }; + + update_work(&key, delta); + update_task_info(task, cpu); +} + +static void on_sched_in(struct task_struct *task, __u64 ts) +{ + struct time_data *pelem; + + pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (pelem) + pelem->timestamp = ts; +} + +SEC("tp_btf/sched_switch") +int on_switch(u64 *ctx) +{ + struct task_struct *prev, *next; + + prev = (struct task_struct *)ctx[1]; + next = (struct task_struct *)ctx[2]; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + on_sched_out(prev, ts, cpu); + on_sched_in(next, ts); + + return 0; +} + +SEC("tp_btf/irq_handler_entry") +int on_irq_handler_entry(u64 *cxt) +{ + struct task_struct *task; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + struct time_data data = { + .timestamp = ts, + }; + + bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY); + + return 0; +} + +SEC("tp_btf/irq_handler_exit") +int on_irq_handler_exit(u64 *cxt) +{ + __u64 delta; + struct task_struct *task; + struct time_data *pelem; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key); + if (pelem && pelem->timestamp != 0) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + update_work(&key, delta); + + return 0; +} + +SEC("tp_btf/softirq_entry") +int on_softirq_entry(u64 *cxt) +{ + struct task_struct *task; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + struct time_data data = { + .timestamp = ts, + }; + + bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY); + + return 0; +} + +SEC("tp_btf/softirq_exit") +int on_softirq_exit(u64 *cxt) +{ + __u64 delta; + struct task_struct *task; + struct time_data *pelem; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key); + if (pelem) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + update_work(&key, delta); + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/kwork_trace.bpf.c b/tools/perf/util/bpf_skel/kwork_trace.bpf.c new file mode 100644 index 000000000..9ce9c8ddd --- /dev/null +++ b/tools/perf/util/bpf_skel/kwork_trace.bpf.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022, Huawei + +#include "vmlinux.h" +#include +#include + +#define KWORK_COUNT 100 +#define MAX_KWORKNAME 128 + +/* + * This should be in sync with "util/kwork.h" + */ +enum kwork_class_type { + KWORK_CLASS_IRQ, + KWORK_CLASS_SOFTIRQ, + KWORK_CLASS_WORKQUEUE, + KWORK_CLASS_MAX, +}; + +struct work_key { + __u32 type; + __u32 cpu; + __u64 id; +}; + +struct report_data { + __u64 nr; + __u64 total_time; + __u64 max_time; + __u64 max_time_start; + __u64 max_time_end; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, MAX_KWORKNAME); + __uint(max_entries, KWORK_COUNT); +} perf_kwork_names SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, KWORK_COUNT); +} perf_kwork_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct report_data)); + __uint(max_entries, KWORK_COUNT); +} perf_kwork_report SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} perf_kwork_cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, MAX_KWORKNAME); + __uint(max_entries, 1); +} perf_kwork_name_filter SEC(".maps"); + +int enabled = 0; + +const volatile int has_cpu_filter = 0; +const volatile int has_name_filter = 0; + +static __always_inline int local_strncmp(const char *s1, + unsigned int sz, const char *s2) +{ + int ret = 0; + unsigned int i; + + for (i = 0; i < sz; i++) { + ret = (unsigned char)s1[i] - (unsigned char)s2[i]; + if (ret || !s1[i]) + break; + } + + return ret; +} + +static __always_inline int trace_event_match(struct work_key *key, char *name) +{ + __u8 *cpu_val; + char *name_val; + __u32 zero = 0; + __u32 cpu = bpf_get_smp_processor_id(); + + if (!enabled) + return 0; + + if (has_cpu_filter) { + cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu); + if (!cpu_val) + return 0; + } + + if (has_name_filter && (name != NULL)) { + name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero); + if (name_val && + (local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) { + return 0; + } + } + + return 1; +} + +static __always_inline void do_update_time(void *map, struct work_key *key, + __u64 time_start, __u64 time_end) +{ + struct report_data zero, *data; + __s64 delta = time_end - time_start; + + if (delta < 0) + return; + + data = bpf_map_lookup_elem(map, key); + if (!data) { + __builtin_memset(&zero, 0, sizeof(zero)); + bpf_map_update_elem(map, key, &zero, BPF_NOEXIST); + data = bpf_map_lookup_elem(map, key); + if (!data) + return; + } + + if ((delta > data->max_time) || + (data->max_time == 0)) { + data->max_time = delta; + data->max_time_start = time_start; + data->max_time_end = time_end; + } + + data->total_time += delta; + data->nr++; +} + +static __always_inline void do_update_timestart(void *map, struct work_key *key) +{ + __u64 ts = bpf_ktime_get_ns(); + + bpf_map_update_elem(map, key, &ts, BPF_ANY); +} + +static __always_inline void do_update_timeend(void *report_map, void *time_map, + struct work_key *key) +{ + __u64 *time = bpf_map_lookup_elem(time_map, key); + + if (time) { + bpf_map_delete_elem(time_map, key); + do_update_time(report_map, key, *time, bpf_ktime_get_ns()); + } +} + +static __always_inline void do_update_name(void *map, + struct work_key *key, char *name) +{ + if (!bpf_map_lookup_elem(map, key)) + bpf_map_update_elem(map, key, name, BPF_ANY); +} + +static __always_inline int update_timestart(void *map, struct work_key *key) +{ + if (!trace_event_match(key, NULL)) + return 0; + + do_update_timestart(map, key); + return 0; +} + +static __always_inline int update_timestart_and_name(void *time_map, + void *names_map, + struct work_key *key, + char *name) +{ + if (!trace_event_match(key, name)) + return 0; + + do_update_timestart(time_map, key); + do_update_name(names_map, key, name); + + return 0; +} + +static __always_inline int update_timeend(void *report_map, + void *time_map, struct work_key *key) +{ + if (!trace_event_match(key, NULL)) + return 0; + + do_update_timeend(report_map, time_map, key); + + return 0; +} + +static __always_inline int update_timeend_and_name(void *report_map, + void *time_map, + void *names_map, + struct work_key *key, + char *name) +{ + if (!trace_event_match(key, name)) + return 0; + + do_update_timeend(report_map, time_map, key); + do_update_name(names_map, key, name); + + return 0; +} + +SEC("tracepoint/irq/irq_handler_entry") +int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx) +{ + char name[MAX_KWORKNAME]; + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->irq, + }; + void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff); + + bpf_probe_read_kernel_str(name, sizeof(name), name_addr); + + return update_timestart_and_name(&perf_kwork_time, + &perf_kwork_names, &key, name); +} + +SEC("tracepoint/irq/irq_handler_exit") +int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->irq, + }; + + return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); +} + +static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = { + { "HI" }, + { "TIMER" }, + { "NET_TX" }, + { "NET_RX" }, + { "BLOCK" }, + { "IRQ_POLL" }, + { "TASKLET" }, + { "SCHED" }, + { "HRTIMER" }, + { "RCU" }, +}; + +SEC("tracepoint/irq/softirq_entry") +int report_softirq_entry(struct trace_event_raw_softirq *ctx) +{ + unsigned int vec = ctx->vec; + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)vec, + }; + + if (vec < NR_SOFTIRQS) { + return update_timestart_and_name(&perf_kwork_time, + &perf_kwork_names, &key, + softirq_name_list[vec]); + } + + return 0; +} + +SEC("tracepoint/irq/softirq_exit") +int report_softirq_exit(struct trace_event_raw_softirq *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->vec, + }; + + return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); +} + +SEC("tracepoint/irq/softirq_raise") +int latency_softirq_raise(struct trace_event_raw_softirq *ctx) +{ + unsigned int vec = ctx->vec; + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)vec, + }; + + if (vec < NR_SOFTIRQS) { + return update_timestart_and_name(&perf_kwork_time, + &perf_kwork_names, &key, + softirq_name_list[vec]); + } + + return 0; +} + +SEC("tracepoint/irq/softirq_entry") +int latency_softirq_entry(struct trace_event_raw_softirq *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->vec, + }; + + return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); +} + +SEC("tracepoint/workqueue/workqueue_execute_start") +int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + + return update_timestart(&perf_kwork_time, &key); +} + +SEC("tracepoint/workqueue/workqueue_execute_end") +int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx) +{ + char name[MAX_KWORKNAME]; + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + unsigned long long func_addr = (unsigned long long)ctx->function; + + __builtin_memset(name, 0, sizeof(name)); + bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr)); + + return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time, + &perf_kwork_names, &key, name); +} + +SEC("tracepoint/workqueue/workqueue_activate_work") +int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + + return update_timestart(&perf_kwork_time, &key); +} + +SEC("tracepoint/workqueue/workqueue_execute_start") +int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx) +{ + char name[MAX_KWORKNAME]; + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + unsigned long long func_addr = (unsigned long long)ctx->function; + + __builtin_memset(name, 0, sizeof(name)); + bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr)); + + return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time, + &perf_kwork_names, &key, name); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c new file mode 100644 index 000000000..0d9c6f550 --- /dev/null +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -0,0 +1,1052 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022 Google +#include "vmlinux.h" +#include +#include +#include +#include + +#include "lock_data.h" + +/* for collect_lock_syms(). 4096 was rejected by the verifier */ +#define MAX_CPUS 1024 + +/* for collect_zone_lock(). It should be more than the actual zones. */ +#define MAX_ZONES 10 + +/* for do_lock_delay(). Arbitrarily set to 1 million. */ +#define MAX_LOOP (1U << 20) + +/* lock contention flags from include/trace/events/lock.h */ +#define LCB_F_SPIN (1U << 0) +#define LCB_F_READ (1U << 1) +#define LCB_F_WRITE (1U << 2) +#define LCB_F_RT (1U << 3) +#define LCB_F_PERCPU (1U << 4) +#define LCB_F_MUTEX (1U << 5) + +/* callstack storage */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, MAX_ENTRIES); +} stacks SEC(".maps"); + +/* buffer for owner stacktrace */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, 1); +} stack_buf SEC(".maps"); + +/* a map for tracing owner stacktrace to owner stack id */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); // owner stacktrace + __uint(value_size, sizeof(__s32)); // owner stack id + __uint(max_entries, 1); +} owner_stacks SEC(".maps"); + +/* a map for tracing lock address to owner data */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); // lock address + __uint(value_size, sizeof(struct owner_tracing_data)); + __uint(max_entries, 1); +} owner_data SEC(".maps"); + +/* a map for contention_key (stores owner stack id) to contention data */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct contention_key)); + __uint(value_size, sizeof(struct contention_data)); + __uint(max_entries, 1); +} owner_stat SEC(".maps"); + +/* maintain timestamp at the beginning of contention */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); + __type(value, struct tstamp_data); + __uint(max_entries, MAX_ENTRIES); +} tstamp SEC(".maps"); + +/* maintain per-CPU timestamp at the beginning of contention */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct tstamp_data)); + __uint(max_entries, 1); +} tstamp_cpu SEC(".maps"); + +/* actual lock contention statistics */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct contention_key)); + __uint(value_size, sizeof(struct contention_data)); + __uint(max_entries, MAX_ENTRIES); +} lock_stat SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct contention_task_data)); + __uint(max_entries, MAX_ENTRIES); +} task_data SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, MAX_ENTRIES); +} lock_syms SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} type_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} addr_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cgroup_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(long)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} slab_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(long)); + __uint(value_size, sizeof(struct slab_cache_data)); + __uint(max_entries, 1); +} slab_caches SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, 1); +} lock_delays SEC(".maps"); + +struct rw_semaphore___old { + struct task_struct *owner; +} __attribute__((preserve_access_index)); + +struct rw_semaphore___new { + atomic_long_t owner; +} __attribute__((preserve_access_index)); + +struct mm_struct___old { + struct rw_semaphore mmap_sem; +} __attribute__((preserve_access_index)); + +struct mm_struct___new { + struct rw_semaphore mmap_lock; +} __attribute__((preserve_access_index)); + +struct cas_ctx { + struct contention_data *data; + u64 duration; + int max_done; + int min_done; +}; + +extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym __weak; + +/* control flags */ +const volatile int has_cpu; +const volatile int has_task; +const volatile int has_type; +const volatile int has_addr; +const volatile int has_cgroup; +const volatile int has_slab; +const volatile int has_mmap_lock; +const volatile int needs_callstack; +const volatile int stack_skip; +const volatile int lock_owner; +const volatile int use_cgroup_v2; +const volatile int max_stack; +const volatile int lock_delay; + +/* determine the key of lock stat */ +const volatile int aggr_mode; + +int enabled; + +int perf_subsys_id = -1; + +__u64 end_ts; + +__u32 slab_cache_id; + +/* error stat */ +int task_fail; +int stack_fail; +int time_fail; +int data_fail; + +int task_map_full; +int data_map_full; + +struct task_struct *bpf_task_from_pid(s32 pid) __ksym __weak; +void bpf_task_release(struct task_struct *p) __ksym __weak; + +static inline __u32 check_lock_type(__u64 lock, __u32 flags); + +static inline __u64 get_current_cgroup_id(void) +{ + struct task_struct *task; + struct cgroup *cgrp; + + if (use_cgroup_v2) + return bpf_get_current_cgroup_id(); + + task = bpf_get_current_task_btf(); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + +static inline int can_record(u64 *ctx) +{ + bool is_addr_ok = false; + + if (has_cpu) { + __u32 cpu = bpf_get_smp_processor_id(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return 0; + } + + if (has_task) { + __u8 *ok; + __u32 pid = bpf_get_current_pid_tgid(); + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return 0; + } + + if (has_type) { + __u8 *ok; + __u32 flags = (__u32)ctx[1]; + + ok = bpf_map_lookup_elem(&type_filter, &flags); + if (!ok) + return 0; + } + + if (has_addr) { + __u8 *ok; + __u64 addr = ctx[0]; + + ok = bpf_map_lookup_elem(&addr_filter, &addr); + if (!ok && !has_slab && !has_mmap_lock) + return 0; + + is_addr_ok = !!ok; + } + + if (has_cgroup) { + __u8 *ok; + __u64 cgrp = get_current_cgroup_id(); + + ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp); + if (!ok) + return 0; + } + + if (is_addr_ok) + return 1; + + /* slab and mmap_lock are part of the addr_filter */ + if (has_slab && bpf_get_kmem_cache) { + __u8 *ok; + __u64 addr = ctx[0]; + long kmem_cache_addr; + + kmem_cache_addr = (long)bpf_get_kmem_cache(addr); + ok = bpf_map_lookup_elem(&slab_filter, &kmem_cache_addr); + if (ok) + return 1; + else if (!has_mmap_lock) + return 0; + } + + if (has_mmap_lock) { + __u64 lock = ctx[0]; + __u32 flag = ctx[1]; + + if (check_lock_type(lock, flag) != LCD_F_MMAP_LOCK) + return 0; + } + + return 1; +} + +static inline int update_task_data(struct task_struct *task) +{ + struct contention_task_data *p; + int pid, err; + + err = bpf_core_read(&pid, sizeof(pid), &task->pid); + if (err) + return -1; + + p = bpf_map_lookup_elem(&task_data, &pid); + if (p == NULL && !task_map_full) { + struct contention_task_data data = {}; + + BPF_CORE_READ_STR_INTO(&data.comm, task, comm); + if (bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST) == -E2BIG) + task_map_full = 1; + } + + return 0; +} + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags) +{ + struct task_struct *task; + __u64 owner = 0; + + if (flags & LCB_F_MUTEX) { + struct mutex *mutex = (void *)lock; + owner = BPF_CORE_READ(mutex, owner.counter); + } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) { + /* + * Support for the BPF_TYPE_MATCHES argument to the + * __builtin_preserve_type_info builtin was added at some point during + * development of clang 15 and it's what is needed for + * bpf_core_type_matches. + */ +#if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15 + if (bpf_core_type_matches(struct rw_semaphore___old)) { + struct rw_semaphore___old *rwsem = (void *)lock; + owner = (unsigned long)BPF_CORE_READ(rwsem, owner); + } else if (bpf_core_type_matches(struct rw_semaphore___new)) { + struct rw_semaphore___new *rwsem = (void *)lock; + owner = BPF_CORE_READ(rwsem, owner.counter); + } +#else + /* assume new struct */ + struct rw_semaphore *rwsem = (void *)lock; + owner = BPF_CORE_READ(rwsem, owner.counter); +#endif + } + + if (!owner) + return NULL; + + task = (void *)(owner & ~7UL); + return task; +} + +static inline __u32 check_lock_type(__u64 lock, __u32 flags) +{ + struct task_struct *curr; + struct mm_struct___old *mm_old; + struct mm_struct___new *mm_new; + struct sighand_struct *sighand; + + switch (flags) { + case LCB_F_READ: /* rwsem */ + case LCB_F_WRITE: + curr = bpf_get_current_task_btf(); + if (curr->mm == NULL) + break; + mm_new = (void *)curr->mm; + if (bpf_core_field_exists(mm_new->mmap_lock)) { + if (&mm_new->mmap_lock == (void *)lock) + return LCD_F_MMAP_LOCK; + break; + } + mm_old = (void *)curr->mm; + if (bpf_core_field_exists(mm_old->mmap_sem)) { + if (&mm_old->mmap_sem == (void *)lock) + return LCD_F_MMAP_LOCK; + } + break; + case LCB_F_SPIN: /* spinlock */ + curr = bpf_get_current_task_btf(); + sighand = curr->sighand; + + if (sighand && &sighand->siglock == (void *)lock) + return LCD_F_SIGHAND_LOCK; + break; + default: + break; + } + return 0; +} + +static inline long delay_callback(__u64 idx, void *arg) +{ + __u64 target = *(__u64 *)arg; + + if (target <= bpf_ktime_get_ns()) + return 1; + + /* just to kill time */ + (void)bpf_get_prandom_u32(); + + return 0; +} + +static inline void do_lock_delay(__u64 duration) +{ + __u64 target = bpf_ktime_get_ns() + duration; + + bpf_loop(MAX_LOOP, delay_callback, &target, /*flags=*/0); +} + +static inline void check_lock_delay(__u64 lock) +{ + __u64 *delay; + + delay = bpf_map_lookup_elem(&lock_delays, &lock); + if (delay) + do_lock_delay(*delay); +} + +static inline struct tstamp_data *get_tstamp_elem(__u32 flags) +{ + __u32 pid; + struct tstamp_data *pelem; + + /* Use per-cpu array map for spinlock and rwlock */ + if ((flags & (LCB_F_SPIN | LCB_F_MUTEX)) == LCB_F_SPIN) { + __u32 idx = 0; + + pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); + /* Do not update the element for nested locks */ + if (pelem && pelem->lock) + pelem = NULL; + return pelem; + } + + pid = bpf_get_current_pid_tgid(); + pelem = bpf_map_lookup_elem(&tstamp, &pid); + /* Do not update the element for nested locks */ + if (pelem && pelem->lock) + return NULL; + + if (pelem == NULL) { + struct tstamp_data zero = {}; + + if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) { + __sync_fetch_and_add(&task_fail, 1); + return NULL; + } + + pelem = bpf_map_lookup_elem(&tstamp, &pid); + if (pelem == NULL) { + __sync_fetch_and_add(&task_fail, 1); + return NULL; + } + } + return pelem; +} + +static inline s32 get_owner_stack_id(u64 *stacktrace) +{ + s32 *id, new_id; + static s64 id_gen = 1; + + id = bpf_map_lookup_elem(&owner_stacks, stacktrace); + if (id) + return *id; + + new_id = (s32)__sync_fetch_and_add(&id_gen, 1); + + bpf_map_update_elem(&owner_stacks, stacktrace, &new_id, BPF_NOEXIST); + + id = bpf_map_lookup_elem(&owner_stacks, stacktrace); + if (id) + return *id; + + return -1; +} + +static long cas_min_max_cb(u64 idx, void *arg) +{ + struct cas_ctx *ctx = arg; + + if (!ctx->max_done) { + u64 old_max = ctx->data->max_time; + if (old_max >= ctx->duration) { + ctx->max_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->max_time, old_max, ctx->duration); + if (r == old_max) + ctx->max_done = 1; + } + } + + if (!ctx->min_done) { + u64 old_min = ctx->data->min_time; + if (old_min <= ctx->duration) { + ctx->min_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->min_time, old_min, ctx->duration); + if (r == old_min) + ctx->min_done = 1; + } + } + + return (ctx->max_done && ctx->min_done) ? 1 : 0; +} + +static inline void update_contention_data(struct contention_data *data, u64 duration, u32 count) +{ + __sync_fetch_and_add(&data->total_time, duration); + __sync_fetch_and_add(&data->count, count); + + struct cas_ctx ctx = { + .data = data, + .duration = duration, + .max_done = 0, + .min_done = 0, + }; + bpf_loop(64, cas_min_max_cb, &ctx, 0); +} + +static inline void update_owner_stat(u32 id, u64 duration, u32 flags) +{ + struct contention_key key = { + .stack_id = id, + .pid = 0, + .lock_addr_or_cgroup = 0, + }; + struct contention_data *data = bpf_map_lookup_elem(&owner_stat, &key); + + if (!data) { + struct contention_data first = { + .total_time = duration, + .max_time = duration, + .min_time = duration, + .count = 1, + .flags = flags, + }; + bpf_map_update_elem(&owner_stat, &key, &first, BPF_NOEXIST); + } else { + update_contention_data(data, duration, 1); + } +} + +SEC("tp_btf/contention_begin") +int contention_begin(u64 *ctx) +{ + struct tstamp_data *pelem; + + if (!enabled || !can_record(ctx)) + return 0; + + pelem = get_tstamp_elem(ctx[1]); + if (pelem == NULL) + return 0; + + pelem->timestamp = bpf_ktime_get_ns(); + pelem->lock = (__u64)ctx[0]; + pelem->flags = (__u32)ctx[1]; + if (aggr_mode == LOCK_AGGR_CGROUP) + pelem->cgroup_id = get_current_cgroup_id(); + + if (needs_callstack) { + u32 i = 0; + u32 id = 0; + int owner_pid; + u64 *buf; + struct task_struct *task; + struct owner_tracing_data *otdata; + + if (!lock_owner) + goto skip_owner; + + task = get_lock_owner(pelem->lock, pelem->flags); + if (!task) + goto skip_owner; + + owner_pid = BPF_CORE_READ(task, pid); + + buf = bpf_map_lookup_elem(&stack_buf, &i); + if (!buf) + goto skip_owner; + for (i = 0; i < max_stack; i++) + buf[i] = 0x0; + + if (!bpf_task_from_pid) + goto skip_owner; + + task = bpf_task_from_pid(owner_pid); + if (!task) + goto skip_owner; + + bpf_get_task_stack(task, buf, max_stack * sizeof(unsigned long), 0); + bpf_task_release(task); + + otdata = bpf_map_lookup_elem(&owner_data, &pelem->lock); + id = get_owner_stack_id(buf); + + /* + * Contention just happens, or corner case `lock` is owned by process not + * `owner_pid`. For the corner case we treat it as unexpected internal error and + * just ignore the precvious tracing record. + */ + if (!otdata || otdata->pid != owner_pid) { + struct owner_tracing_data first = { + .pid = owner_pid, + .timestamp = pelem->timestamp, + .count = 1, + .stack_id = id, + }; + bpf_map_update_elem(&owner_data, &pelem->lock, &first, BPF_ANY); + } + /* Contention is ongoing and new waiter joins */ + else { + __sync_fetch_and_add(&otdata->count, 1); + + /* + * The owner is the same, but stacktrace might be changed. In this case we + * store/update `owner_stat` based on current owner stack id. + */ + if (id != otdata->stack_id) { + update_owner_stat(id, pelem->timestamp - otdata->timestamp, + pelem->flags); + + otdata->timestamp = pelem->timestamp; + otdata->stack_id = id; + } + } +skip_owner: + pelem->stack_id = bpf_get_stackid(ctx, &stacks, + BPF_F_FAST_STACK_CMP | stack_skip); + if (pelem->stack_id < 0) + __sync_fetch_and_add(&stack_fail, 1); + } else if (aggr_mode == LOCK_AGGR_TASK) { + struct task_struct *task; + + if (lock_owner) { + task = get_lock_owner(pelem->lock, pelem->flags); + + /* The flags is not used anymore. Pass the owner pid. */ + if (task) + pelem->flags = BPF_CORE_READ(task, pid); + else + pelem->flags = -1U; + + } else { + task = bpf_get_current_task_btf(); + } + + if (task) { + if (update_task_data(task) < 0 && lock_owner) + pelem->flags = -1U; + } + } + + return 0; +} + +SEC("tp_btf/contention_end") +int contention_end(u64 *ctx) +{ + __u32 pid = 0, idx = 0; + struct tstamp_data *pelem; + struct contention_key key = {}; + struct contention_data *data; + __u64 timestamp; + __u64 duration; + bool need_delete = false; + + if (!enabled) + return 0; + + /* + * For spinlock and rwlock, it needs to get the timestamp for the + * per-cpu map. However, contention_end does not have the flags + * so it cannot know whether it reads percpu or hash map. + * + * Try per-cpu map first and check if there's active contention. + * If it is, do not read hash map because it cannot go to sleeping + * locks before releasing the spinning locks. + */ + pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); + if (pelem && pelem->lock) { + if (pelem->lock != ctx[0]) + return 0; + } else { + pid = bpf_get_current_pid_tgid(); + pelem = bpf_map_lookup_elem(&tstamp, &pid); + if (!pelem || pelem->lock != ctx[0]) + return 0; + need_delete = true; + } + + timestamp = bpf_ktime_get_ns(); + duration = timestamp - pelem->timestamp; + if ((__s64)duration < 0) { + __sync_fetch_and_add(&time_fail, 1); + goto out; + } + + if (needs_callstack && lock_owner) { + struct owner_tracing_data *otdata = bpf_map_lookup_elem(&owner_data, &pelem->lock); + + if (!otdata) + goto skip_owner; + + /* Update `owner_stat` */ + update_owner_stat(otdata->stack_id, timestamp - otdata->timestamp, pelem->flags); + + /* No contention is occurring, delete `lock` entry in `owner_data` */ + if (otdata->count <= 1) + bpf_map_delete_elem(&owner_data, &pelem->lock); + /* + * Contention is still ongoing, with a new owner (current task). `owner_data` + * should be updated accordingly. + */ + else { + u32 i = 0; + s32 ret = (s32)ctx[1]; + u64 *buf; + + otdata->timestamp = timestamp; + __sync_fetch_and_add(&otdata->count, -1); + + buf = bpf_map_lookup_elem(&stack_buf, &i); + if (!buf) + goto skip_owner; + for (i = 0; i < (u32)max_stack; i++) + buf[i] = 0x0; + + /* + * `ret` has the return code of the lock function. + * If `ret` is negative, the current task terminates lock waiting without + * acquiring it. Owner is not changed, but we still need to update the owner + * stack. + */ + if (ret < 0) { + s32 id = 0; + struct task_struct *task; + + if (!bpf_task_from_pid) + goto skip_owner; + + task = bpf_task_from_pid(otdata->pid); + if (!task) + goto skip_owner; + + bpf_get_task_stack(task, buf, + max_stack * sizeof(unsigned long), 0); + bpf_task_release(task); + + id = get_owner_stack_id(buf); + + /* + * If owner stack is changed, update owner stack id for this lock. + */ + if (id != otdata->stack_id) + otdata->stack_id = id; + } + /* + * Otherwise, update tracing data with the current task, which is the new + * owner. + */ + else { + otdata->pid = pid; + /* + * We don't want to retrieve callstack here, since it is where the + * current task acquires the lock and provides no additional + * information. We simply assign -1 to invalidate it. + */ + otdata->stack_id = -1; + } + } + } +skip_owner: + switch (aggr_mode) { + case LOCK_AGGR_CALLER: + key.stack_id = pelem->stack_id; + break; + case LOCK_AGGR_TASK: + if (lock_owner) + key.pid = pelem->flags; + else { + if (!need_delete) + pid = bpf_get_current_pid_tgid(); + key.pid = pid; + } + if (needs_callstack) + key.stack_id = pelem->stack_id; + break; + case LOCK_AGGR_ADDR: + key.lock_addr_or_cgroup = pelem->lock; + if (needs_callstack) + key.stack_id = pelem->stack_id; + break; + case LOCK_AGGR_CGROUP: + key.lock_addr_or_cgroup = pelem->cgroup_id; + break; + default: + /* should not happen */ + return 0; + } + + data = bpf_map_lookup_elem(&lock_stat, &key); + if (!data) { + if (data_map_full) { + __sync_fetch_and_add(&data_fail, 1); + goto out; + } + + struct contention_data first = { + .total_time = duration, + .max_time = duration, + .min_time = duration, + .count = 1, + .flags = pelem->flags, + }; + int err; + + if (aggr_mode == LOCK_AGGR_ADDR) { + first.flags |= check_lock_type(pelem->lock, + pelem->flags & LCB_F_TYPE_MASK); + + /* Check if it's from a slab object */ + if (bpf_get_kmem_cache) { + struct kmem_cache *s; + struct slab_cache_data *d; + + s = bpf_get_kmem_cache(pelem->lock); + if (s != NULL) { + /* + * Save the ID of the slab cache in the flags + * (instead of full address) to reduce the + * space in the contention_data. + */ + d = bpf_map_lookup_elem(&slab_caches, &s); + if (d != NULL) + first.flags |= d->id; + } + } + } + + err = bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); + if (err < 0) { + if (err == -EEXIST) { + /* it lost the race, try to get it again */ + data = bpf_map_lookup_elem(&lock_stat, &key); + if (data != NULL) + goto found; + } + if (err == -E2BIG) + data_map_full = 1; + __sync_fetch_and_add(&data_fail, 1); + } + goto out; + } + +found: + update_contention_data(data, duration, 1); + +out: + if (lock_delay) + check_lock_delay(pelem->lock); + + pelem->lock = 0; + if (need_delete) + bpf_map_delete_elem(&tstamp, &pid); + return 0; +} + +extern struct rq runqueues __ksym; + +const volatile __u64 contig_page_data_addr; +const volatile __u64 node_data_addr; +const volatile int nr_nodes; +const volatile int sizeof_zone; + +struct rq___old { + raw_spinlock_t lock; +} __attribute__((preserve_access_index)); + +struct rq___new { + raw_spinlock_t __lock; +} __attribute__((preserve_access_index)); + +static void collect_zone_lock(void) +{ + __u64 nr_zones, zone_off; + __u64 lock_addr, lock_off; + __u32 lock_flag = LOCK_CLASS_ZONE_LOCK; + + zone_off = offsetof(struct pglist_data, node_zones); + lock_off = offsetof(struct zone, lock); + + if (contig_page_data_addr) { + struct pglist_data *contig_page_data; + + contig_page_data = (void *)(long)contig_page_data_addr; + nr_zones = BPF_CORE_READ(contig_page_data, nr_zones); + + for (int i = 0; i < MAX_ZONES; i++) { + __u64 zone_addr; + + if (i >= nr_zones) + break; + + zone_addr = contig_page_data_addr + (sizeof_zone * i) + zone_off; + lock_addr = zone_addr + lock_off; + + bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); + } + } else if (nr_nodes > 0) { + struct pglist_data **node_data = (void *)(long)node_data_addr; + + for (int i = 0; i < nr_nodes; i++) { + struct pglist_data *pgdat = NULL; + int err; + + err = bpf_core_read(&pgdat, sizeof(pgdat), &node_data[i]); + if (err < 0 || pgdat == NULL) + break; + + nr_zones = BPF_CORE_READ(pgdat, nr_zones); + for (int k = 0; k < MAX_ZONES; k++) { + __u64 zone_addr; + + if (k >= nr_zones) + break; + + zone_addr = (__u64)(void *)pgdat + (sizeof_zone * k) + zone_off; + lock_addr = zone_addr + lock_off; + + bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); + } + } + } +} + +SEC("raw_tp/bpf_test_finish") +int BPF_PROG(collect_lock_syms) +{ + __u64 lock_addr, lock_off; + __u32 lock_flag; + + if (bpf_core_field_exists(struct rq___new, __lock)) + lock_off = offsetof(struct rq___new, __lock); + else + lock_off = offsetof(struct rq___old, lock); + + for (int i = 0; i < MAX_CPUS; i++) { + struct rq *rq = bpf_per_cpu_ptr(&runqueues, i); + + if (rq == NULL) + break; + + lock_addr = (__u64)(void *)rq + lock_off; + lock_flag = LOCK_CLASS_RQLOCK; + bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); + } + + collect_zone_lock(); + + return 0; +} + +SEC("raw_tp/bpf_test_finish") +int BPF_PROG(end_timestamp) +{ + end_ts = bpf_ktime_get_ns(); + return 0; +} + +/* + * bpf_iter__kmem_cache added recently so old kernels don't have it in the + * vmlinux.h. But we cannot add it here since it will cause a compiler error + * due to redefinition of the struct on later kernels. + * + * So it uses a CO-RE trick to access the member only if it has the type. + * This will support both old and new kernels without compiler errors. + */ +struct bpf_iter__kmem_cache___new { + struct kmem_cache *s; +} __attribute__((preserve_access_index)); + +SEC("iter/kmem_cache") +int slab_cache_iter(void *ctx) +{ + struct kmem_cache *s = NULL; + struct slab_cache_data d; + const char *nameptr; + + if (bpf_core_type_exists(struct bpf_iter__kmem_cache)) { + struct bpf_iter__kmem_cache___new *iter = ctx; + + s = iter->s; + } + + if (s == NULL) + return 0; + + nameptr = s->name; + bpf_probe_read_kernel_str(d.name, sizeof(d.name), nameptr); + + d.id = ++slab_cache_id << LCB_F_SLAB_ID_SHIFT; + if (d.id >= LCB_F_SLAB_ID_END) + return 0; + + bpf_map_update_elem(&slab_caches, &s, &d, BPF_NOEXIST); + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/lock_data.h b/tools/perf/util/bpf_skel/lock_data.h new file mode 100644 index 000000000..652e114e6 --- /dev/null +++ b/tools/perf/util/bpf_skel/lock_data.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Data structures shared between BPF and tools. */ +#ifndef UTIL_BPF_SKEL_LOCK_DATA_H +#define UTIL_BPF_SKEL_LOCK_DATA_H + +struct owner_tracing_data { + u32 pid; // Who has the lock. + u32 count; // How many waiters for this lock. + u64 timestamp; // The time while the owner acquires lock and contention is going on. + s32 stack_id; // Identifier for `owner_stat`, which stores as value in `owner_stacks` +}; + +struct tstamp_data { + u64 timestamp; + u64 lock; + u64 cgroup_id; + u32 flags; + s32 stack_id; +}; + +struct contention_key { + s32 stack_id; + u32 pid; + u64 lock_addr_or_cgroup; +}; + +#define TASK_COMM_LEN 16 + +struct contention_task_data { + char comm[TASK_COMM_LEN]; +}; + +/* default buffer size */ +#define MAX_ENTRIES 16384 + +/* + * Upper bits of the flags in the contention_data are used to identify + * some well-known locks which do not have symbols (non-global locks). + */ +#define LCD_F_MMAP_LOCK (1U << 31) +#define LCD_F_SIGHAND_LOCK (1U << 30) + +#define LCB_F_SLAB_ID_SHIFT 16 +#define LCB_F_SLAB_ID_START (1U << 16) +#define LCB_F_SLAB_ID_END (1U << 26) +#define LCB_F_SLAB_ID_MASK 0x03FF0000U + +#define LCB_F_TYPE_MAX (1U << 7) +#define LCB_F_TYPE_MASK 0x0000007FU + +#define SLAB_NAME_MAX 28 + +struct contention_data { + u64 total_time; + u64 min_time; + u64 max_time; + u32 count; + u32 flags; +}; + +enum lock_aggr_mode { + LOCK_AGGR_ADDR = 0, + LOCK_AGGR_TASK, + LOCK_AGGR_CALLER, + LOCK_AGGR_CGROUP, +}; + +enum lock_class_sym { + LOCK_CLASS_NONE, + LOCK_CLASS_RQLOCK, + LOCK_CLASS_ZONE_LOCK, +}; + +struct slab_cache_data { + u32 id; + char name[SLAB_NAME_MAX]; +}; + +#endif /* UTIL_BPF_SKEL_LOCK_DATA_H */ diff --git a/tools/perf/util/bpf_skel/off_cpu.bpf.c b/tools/perf/util/bpf_skel/off_cpu.bpf.c new file mode 100644 index 000000000..72763bb8d --- /dev/null +++ b/tools/perf/util/bpf_skel/off_cpu.bpf.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022 Google +#include "vmlinux.h" +#include +#include +#include + +/* task->flags for off-cpu analysis */ +#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ + +/* task->state for off-cpu analysis */ +#define TASK_INTERRUPTIBLE 0x0001 +#define TASK_UNINTERRUPTIBLE 0x0002 + +/* create a new thread */ +#define CLONE_THREAD 0x10000 + +#define MAX_STACKS 32 +#define MAX_ENTRIES 102400 + +#define MAX_CPUS 4096 +#define MAX_OFFCPU_LEN 37 + +// We have a 'struct stack' in vmlinux.h when building with GEN_VMLINUX_H=1 +struct __stack { + u64 array[MAX_STACKS]; +}; + +struct tstamp_data { + __u32 stack_id; + __u32 state; + __u64 timestamp; + struct __stack stack; +}; + +struct offcpu_key { + __u32 pid; + __u32 tgid; + __u32 stack_id; + __u32 state; + __u64 cgroup_id; +}; + +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(key_size, sizeof(__u32)); + __uint(value_size, MAX_STACKS * sizeof(__u64)); + __uint(max_entries, MAX_ENTRIES); +} stacks SEC(".maps"); + +struct offcpu_data { + u64 array[MAX_OFFCPU_LEN]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); + __uint(max_entries, MAX_CPUS); +} offcpu_output SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct offcpu_data)); + __uint(max_entries, 1); +} offcpu_payload SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tstamp_data); +} tstamp SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct offcpu_key)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, MAX_ENTRIES); +} off_cpu SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cgroup_filter SEC(".maps"); + +/* new kernel task_struct definition */ +struct task_struct___new { + long __state; +} __attribute__((preserve_access_index)); + +/* old kernel task_struct definition */ +struct task_struct___old { + long state; +} __attribute__((preserve_access_index)); + +int enabled = 0; + +const volatile int has_cpu = 0; +const volatile int has_task = 0; +const volatile int has_cgroup = 0; +const volatile int uses_tgid = 0; + +const volatile bool has_prev_state = false; +const volatile bool needs_cgroup = false; +const volatile bool uses_cgroup_v1 = false; + +int perf_subsys_id = -1; + +__u64 offcpu_thresh_ns; + +/* + * Old kernel used to call it task_struct->state and now it's '__state'. + * Use BPF CO-RE "ignored suffix rule" to deal with it like below: + * + * https://nakryiko.com/posts/bpf-core-reference-guide/#handling-incompatible-field-and-type-changes + */ +static inline int get_task_state(struct task_struct *t) +{ + /* recast pointer to capture new type for compiler */ + struct task_struct___new *t_new = (void *)t; + + if (bpf_core_field_exists(t_new->__state)) { + return BPF_CORE_READ(t_new, __state); + } else { + /* recast pointer to capture old type for compiler */ + struct task_struct___old *t_old = (void *)t; + + return BPF_CORE_READ(t_old, state); + } +} + +static inline __u64 get_cgroup_id(struct task_struct *t) +{ + struct cgroup *cgrp; + + if (!uses_cgroup_v1) + return BPF_CORE_READ(t, cgroups, dfl_cgrp, kn, id); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(t, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + +static inline int can_record(struct task_struct *t, int state) +{ + /* kernel threads don't have user stack */ + if (t->flags & PF_KTHREAD) + return 0; + + if (state != TASK_INTERRUPTIBLE && + state != TASK_UNINTERRUPTIBLE) + return 0; + + if (has_cpu) { + __u32 cpu = bpf_get_smp_processor_id(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return 0; + } + + if (has_task) { + __u8 *ok; + __u32 pid; + + if (uses_tgid) + pid = t->tgid; + else + pid = t->pid; + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return 0; + } + + if (has_cgroup) { + __u8 *ok; + __u64 cgrp_id = get_cgroup_id(t); + + ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp_id); + if (!ok) + return 0; + } + + return 1; +} + +static inline int copy_stack(struct __stack *from, struct offcpu_data *to, int n) +{ + int len = 0; + + for (int i = 0; i < MAX_STACKS && from->array[i]; ++i, ++len) + to->array[n + 2 + i] = from->array[i]; + + return len; +} + +/** + * off_cpu_dump - dump off-cpu samples to ring buffer + * @data: payload for dumping off-cpu samples + * @key: off-cpu data + * @stack: stack trace of the task before being scheduled out + * + * If the threshold of off-cpu time is reached, acquire tid, period, callchain, and cgroup id + * information of the task, and dump it as a raw sample to perf ring buffer + */ +static int off_cpu_dump(void *ctx, struct offcpu_data *data, struct offcpu_key *key, + struct __stack *stack, __u64 delta) +{ + int n = 0, len = 0; + + data->array[n++] = (u64)key->tgid << 32 | key->pid; + data->array[n++] = delta; + + /* data->array[n] is callchain->nr (updated later) */ + data->array[n + 1] = PERF_CONTEXT_USER; + data->array[n + 2] = 0; + len = copy_stack(stack, data, n); + + /* update length of callchain */ + data->array[n] = len + 1; + n += len + 2; + + data->array[n++] = key->cgroup_id; + + return bpf_perf_event_output(ctx, &offcpu_output, BPF_F_CURRENT_CPU, data, n * sizeof(u64)); +} + +static int off_cpu_stat(u64 *ctx, struct task_struct *prev, + struct task_struct *next, int state) +{ + __u64 ts; + __u32 stack_id; + struct tstamp_data *pelem; + + ts = bpf_ktime_get_ns(); + + if (!can_record(prev, state)) + goto next; + + stack_id = bpf_get_stackid(ctx, &stacks, + BPF_F_FAST_STACK_CMP | BPF_F_USER_STACK); + + pelem = bpf_task_storage_get(&tstamp, prev, NULL, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!pelem) + goto next; + + pelem->timestamp = ts; + pelem->state = state; + pelem->stack_id = stack_id; + + /* + * If stacks are successfully collected by bpf_get_stackid(), collect them once more + * in task_storage for direct off-cpu sample dumping + */ + if (stack_id > 0 && bpf_get_stack(ctx, &pelem->stack, MAX_STACKS * sizeof(u64), BPF_F_USER_STACK)) { + /* + * This empty if block is used to avoid 'result unused warning' from bpf_get_stack(). + * If the collection fails, continue with the logic for the next task. + */ + } +next: + pelem = bpf_task_storage_get(&tstamp, next, NULL, 0); + + if (pelem && pelem->timestamp) { + struct offcpu_key key = { + .pid = next->pid, + .tgid = next->tgid, + .stack_id = pelem->stack_id, + .state = pelem->state, + .cgroup_id = needs_cgroup ? get_cgroup_id(next) : 0, + }; + __u64 delta = ts - pelem->timestamp; + __u64 *total; + + if (delta >= offcpu_thresh_ns) { + int zero = 0; + struct offcpu_data *data = bpf_map_lookup_elem(&offcpu_payload, &zero); + + if (data) + off_cpu_dump(ctx, data, &key, &pelem->stack, delta); + } else { + total = bpf_map_lookup_elem(&off_cpu, &key); + if (total) + *total += delta; + else + bpf_map_update_elem(&off_cpu, &key, &delta, BPF_ANY); + } + + /* prevent to reuse the timestamp later */ + pelem->timestamp = 0; + } + + return 0; +} + +SEC("tp_btf/task_newtask") +int on_newtask(u64 *ctx) +{ + struct task_struct *task; + u64 clone_flags; + u32 pid; + u8 val = 1; + + if (!uses_tgid) + return 0; + + task = (struct task_struct *)bpf_get_current_task(); + + pid = BPF_CORE_READ(task, tgid); + if (!bpf_map_lookup_elem(&task_filter, &pid)) + return 0; + + task = (struct task_struct *)ctx[0]; + clone_flags = ctx[1]; + + pid = task->tgid; + if (!(clone_flags & CLONE_THREAD)) + bpf_map_update_elem(&task_filter, &pid, &val, BPF_NOEXIST); + + return 0; +} + +SEC("tp_btf/sched_switch") +int on_switch(u64 *ctx) +{ + struct task_struct *prev, *next; + int prev_state; + + if (!enabled) + return 0; + + prev = (struct task_struct *)ctx[1]; + next = (struct task_struct *)ctx[2]; + + if (has_prev_state) + prev_state = (int)ctx[3]; + else + prev_state = get_task_state(prev); + + return off_cpu_stat(ctx, prev, next, prev_state & 0xff); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/perf_version.h b/tools/perf/util/bpf_skel/perf_version.h new file mode 100644 index 000000000..1ed5b2e59 --- /dev/null +++ b/tools/perf/util/bpf_skel/perf_version.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ + +#ifndef __PERF_VERSION_H__ +#define __PERF_VERSION_H__ + +#include "vmlinux.h" +#include + +/* + * This is used by tests/shell/record_bpf_metadata.sh + * to verify that BPF metadata generation works. + * + * PERF_VERSION is defined by a build rule at compile time. + */ +const char bpf_metadata_perf_version[] SEC(".rodata") = PERF_VERSION; + +#endif /* __PERF_VERSION_H__ */ diff --git a/tools/perf/util/bpf_skel/sample-filter.h b/tools/perf/util/bpf_skel/sample-filter.h new file mode 100644 index 000000000..683fec85e --- /dev/null +++ b/tools/perf/util/bpf_skel/sample-filter.h @@ -0,0 +1,72 @@ +#ifndef PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H +#define PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H + +#define MAX_FILTERS 64 +#define MAX_IDX_HASH (16 * 1024) +#define MAX_EVT_HASH (1024 * 1024) + +/* supported filter operations */ +enum perf_bpf_filter_op { + PBF_OP_EQ, + PBF_OP_NEQ, + PBF_OP_GT, + PBF_OP_GE, + PBF_OP_LT, + PBF_OP_LE, + PBF_OP_AND, + PBF_OP_GROUP_BEGIN, + PBF_OP_GROUP_END, + PBF_OP_DONE, +}; + +enum perf_bpf_filter_term { + /* No term is in use. */ + PBF_TERM_NONE = 0, + /* Terms that correspond to PERF_SAMPLE_xx values. */ + PBF_TERM_SAMPLE_START = PBF_TERM_NONE + 1, + PBF_TERM_IP = PBF_TERM_SAMPLE_START + 0, /* SAMPLE_IP = 1U << 0 */ + PBF_TERM_TID = PBF_TERM_SAMPLE_START + 1, /* SAMPLE_TID = 1U << 1 */ + PBF_TERM_TIME = PBF_TERM_SAMPLE_START + 2, /* SAMPLE_TIME = 1U << 2 */ + PBF_TERM_ADDR = PBF_TERM_SAMPLE_START + 3, /* SAMPLE_ADDR = 1U << 3 */ + __PBF_UNUSED_TERM4 = PBF_TERM_SAMPLE_START + 4, /* SAMPLE_READ = 1U << 4 */ + __PBF_UNUSED_TERM5 = PBF_TERM_SAMPLE_START + 5, /* SAMPLE_CALLCHAIN = 1U << 5 */ + PBF_TERM_ID = PBF_TERM_SAMPLE_START + 6, /* SAMPLE_ID = 1U << 6 */ + PBF_TERM_CPU = PBF_TERM_SAMPLE_START + 7, /* SAMPLE_CPU = 1U << 7 */ + PBF_TERM_PERIOD = PBF_TERM_SAMPLE_START + 8, /* SAMPLE_PERIOD = 1U << 8 */ + __PBF_UNUSED_TERM9 = PBF_TERM_SAMPLE_START + 9, /* SAMPLE_STREAM_ID = 1U << 9 */ + __PBF_UNUSED_TERM10 = PBF_TERM_SAMPLE_START + 10, /* SAMPLE_RAW = 1U << 10 */ + __PBF_UNUSED_TERM11 = PBF_TERM_SAMPLE_START + 11, /* SAMPLE_BRANCH_STACK = 1U << 11 */ + __PBF_UNUSED_TERM12 = PBF_TERM_SAMPLE_START + 12, /* SAMPLE_REGS_USER = 1U << 12 */ + __PBF_UNUSED_TERM13 = PBF_TERM_SAMPLE_START + 13, /* SAMPLE_STACK_USER = 1U << 13 */ + PBF_TERM_WEIGHT = PBF_TERM_SAMPLE_START + 14, /* SAMPLE_WEIGHT = 1U << 14 */ + PBF_TERM_DATA_SRC = PBF_TERM_SAMPLE_START + 15, /* SAMPLE_DATA_SRC = 1U << 15 */ + __PBF_UNUSED_TERM16 = PBF_TERM_SAMPLE_START + 16, /* SAMPLE_IDENTIFIER = 1U << 16 */ + PBF_TERM_TRANSACTION = PBF_TERM_SAMPLE_START + 17, /* SAMPLE_TRANSACTION = 1U << 17 */ + __PBF_UNUSED_TERM18 = PBF_TERM_SAMPLE_START + 18, /* SAMPLE_REGS_INTR = 1U << 18 */ + PBF_TERM_PHYS_ADDR = PBF_TERM_SAMPLE_START + 19, /* SAMPLE_PHYS_ADDR = 1U << 19 */ + __PBF_UNUSED_TERM20 = PBF_TERM_SAMPLE_START + 20, /* SAMPLE_AUX = 1U << 20 */ + PBF_TERM_CGROUP = PBF_TERM_SAMPLE_START + 21, /* SAMPLE_CGROUP = 1U << 21 */ + PBF_TERM_DATA_PAGE_SIZE = PBF_TERM_SAMPLE_START + 22, /* SAMPLE_DATA_PAGE_SIZE = 1U << 22 */ + PBF_TERM_CODE_PAGE_SIZE = PBF_TERM_SAMPLE_START + 23, /* SAMPLE_CODE_PAGE_SIZE = 1U << 23 */ + PBF_TERM_WEIGHT_STRUCT = PBF_TERM_SAMPLE_START + 24, /* SAMPLE_WEIGHT_STRUCT = 1U << 24 */ + PBF_TERM_SAMPLE_END = PBF_TERM_WEIGHT_STRUCT, + /* Terms computed from BPF helpers. */ + PBF_TERM_UID, + PBF_TERM_GID, +}; + +/* BPF map entry for filtering */ +struct perf_bpf_filter_entry { + enum perf_bpf_filter_op op; + __u32 part; /* sub-sample type info when it has multiple values */ + enum perf_bpf_filter_term term; + __u64 value; +}; + +struct idx_hash_key { + __u64 evt_id; + __u32 tgid; + __u32 reserved; +}; + +#endif /* PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H */ diff --git a/tools/perf/util/bpf_skel/sample_filter.bpf.c b/tools/perf/util/bpf_skel/sample_filter.bpf.c new file mode 100644 index 000000000..e5666d4c1 --- /dev/null +++ b/tools/perf/util/bpf_skel/sample_filter.bpf.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2023 Google +#include "vmlinux.h" +#include +#include +#include + +#include "sample-filter.h" + +/* BPF map that will be filled by user space */ +struct filters { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); + __type(value, struct perf_bpf_filter_entry[MAX_FILTERS]); + __uint(max_entries, 1); +} filters SEC(".maps"); + +/* + * An evsel has multiple instances for each CPU or task but we need a single + * id to be used as a key for the idx_hash. This hashmap would translate the + * instance's ID to a representative ID. + */ +struct event_hash { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, __u64); + __type(value, __u64); + __uint(max_entries, 1); +} event_hash SEC(".maps"); + +/* tgid/evtid to filter index */ +struct idx_hash { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct idx_hash_key); + __type(value, int); + __uint(max_entries, 1); +} idx_hash SEC(".maps"); + +/* tgid to filter index */ +struct lost_count { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); +} dropped SEC(".maps"); + +volatile const int use_idx_hash; + +void *bpf_cast_to_kern_ctx(void *) __ksym; + +/* new kernel perf_sample_data definition */ +struct perf_sample_data___new { + __u64 sample_flags; +} __attribute__((preserve_access_index)); + +/* new kernel perf_mem_data_src definition */ +union perf_mem_data_src___new { + __u64 val; + struct { + __u64 mem_op:5, /* type of opcode */ + mem_lvl:14, /* memory hierarchy level */ + mem_snoop:5, /* snoop mode */ + mem_lock:2, /* lock instr */ + mem_dtlb:7, /* tlb access */ + mem_lvl_num:4, /* memory hierarchy level number */ + mem_remote:1, /* remote */ + mem_snoopx:2, /* snoop mode, ext */ + mem_blk:3, /* access blocked */ + mem_hops:3, /* hop level */ + mem_rsvd:18; + }; +}; + +/* helper function to return the given perf sample data */ +static inline __u64 perf_get_sample(struct bpf_perf_event_data_kern *kctx, + struct perf_bpf_filter_entry *entry) +{ + struct perf_sample_data___new *data = (void *)kctx->data; + + if (!bpf_core_field_exists(data->sample_flags)) + return 0; + +#define BUILD_CHECK_SAMPLE(x) \ + _Static_assert((1 << (PBF_TERM_##x - PBF_TERM_SAMPLE_START)) == PERF_SAMPLE_##x, \ + "Mismatched PBF term to sample bit " #x) + BUILD_CHECK_SAMPLE(IP); + BUILD_CHECK_SAMPLE(TID); + BUILD_CHECK_SAMPLE(TIME); + BUILD_CHECK_SAMPLE(ADDR); + BUILD_CHECK_SAMPLE(ID); + BUILD_CHECK_SAMPLE(CPU); + BUILD_CHECK_SAMPLE(PERIOD); + BUILD_CHECK_SAMPLE(WEIGHT); + BUILD_CHECK_SAMPLE(DATA_SRC); + BUILD_CHECK_SAMPLE(TRANSACTION); + BUILD_CHECK_SAMPLE(PHYS_ADDR); + BUILD_CHECK_SAMPLE(CGROUP); + BUILD_CHECK_SAMPLE(DATA_PAGE_SIZE); + BUILD_CHECK_SAMPLE(CODE_PAGE_SIZE); + BUILD_CHECK_SAMPLE(WEIGHT_STRUCT); +#undef BUILD_CHECK_SAMPLE + + /* For sample terms check the sample bit is set. */ + if (entry->term >= PBF_TERM_SAMPLE_START && entry->term <= PBF_TERM_SAMPLE_END && + (data->sample_flags & (1 << (entry->term - PBF_TERM_SAMPLE_START))) == 0) + return 0; + + switch (entry->term) { + case PBF_TERM_IP: + return kctx->data->ip; + case PBF_TERM_ID: + return kctx->data->id; + case PBF_TERM_TID: + if (entry->part) + return kctx->data->tid_entry.pid; + else + return kctx->data->tid_entry.tid; + case PBF_TERM_CPU: + return kctx->data->cpu_entry.cpu; + case PBF_TERM_TIME: + return kctx->data->time; + case PBF_TERM_ADDR: + return kctx->data->addr; + case PBF_TERM_PERIOD: + return kctx->data->period; + case PBF_TERM_TRANSACTION: + return kctx->data->txn; + case PBF_TERM_WEIGHT_STRUCT: + if (entry->part == 1) + return kctx->data->weight.var1_dw; + if (entry->part == 2) + return kctx->data->weight.var2_w; + if (entry->part == 3) + return kctx->data->weight.var3_w; + /* fall through */ + case PBF_TERM_WEIGHT: + return kctx->data->weight.full; + case PBF_TERM_PHYS_ADDR: + return kctx->data->phys_addr; + case PBF_TERM_CGROUP: + return kctx->data->cgroup; + case PBF_TERM_CODE_PAGE_SIZE: + return kctx->data->code_page_size; + case PBF_TERM_DATA_PAGE_SIZE: + return kctx->data->data_page_size; + case PBF_TERM_DATA_SRC: + if (entry->part == 1) + return kctx->data->data_src.mem_op; + if (entry->part == 2) + return kctx->data->data_src.mem_lvl_num; + if (entry->part == 3) { + __u32 snoop = kctx->data->data_src.mem_snoop; + __u32 snoopx = kctx->data->data_src.mem_snoopx; + + return (snoopx << 5) | snoop; + } + if (entry->part == 4) + return kctx->data->data_src.mem_remote; + if (entry->part == 5) + return kctx->data->data_src.mem_lock; + if (entry->part == 6) + return kctx->data->data_src.mem_dtlb; + if (entry->part == 7) + return kctx->data->data_src.mem_blk; + if (entry->part == 8) { + union perf_mem_data_src___new *data = (void *)&kctx->data->data_src; + + if (__builtin_preserve_field_info(data->mem_hops, BPF_FIELD_EXISTS)) + return data->mem_hops; + + return 0; + } + /* return the whole word */ + return kctx->data->data_src.val; + case PBF_TERM_UID: + return bpf_get_current_uid_gid() & 0xFFFFFFFF; + case PBF_TERM_GID: + return bpf_get_current_uid_gid() >> 32; + case PBF_TERM_NONE: + case __PBF_UNUSED_TERM4: + case __PBF_UNUSED_TERM5: + case __PBF_UNUSED_TERM9: + case __PBF_UNUSED_TERM10: + case __PBF_UNUSED_TERM11: + case __PBF_UNUSED_TERM12: + case __PBF_UNUSED_TERM13: + case __PBF_UNUSED_TERM16: + case __PBF_UNUSED_TERM18: + case __PBF_UNUSED_TERM20: + default: + break; + } + return 0; +} + +#define CHECK_RESULT(data, op, val) \ + if (!(data op val)) { \ + if (!in_group) \ + goto drop; \ + } else if (in_group) { \ + group_result = 1; \ + } + +/* BPF program to be called from perf event overflow handler */ +SEC("perf_event") +int perf_sample_filter(void *ctx) +{ + struct bpf_perf_event_data_kern *kctx; + struct perf_bpf_filter_entry *entry; + __u64 sample_data; + int in_group = 0; + int group_result = 0; + int i, k; + int *losts; + + kctx = bpf_cast_to_kern_ctx(ctx); + + k = 0; + + if (use_idx_hash) { + struct idx_hash_key key = { + .tgid = bpf_get_current_pid_tgid() >> 32, + }; + __u64 eid = kctx->event->id; + __u64 *key_id; + int *idx; + + /* get primary_event_id */ + if (kctx->event->parent) + eid = kctx->event->parent->id; + + key_id = bpf_map_lookup_elem(&event_hash, &eid); + if (key_id == NULL) + goto drop; + + key.evt_id = *key_id; + + idx = bpf_map_lookup_elem(&idx_hash, &key); + if (idx) + k = *idx; + else + goto drop; + } + + entry = bpf_map_lookup_elem(&filters, &k); + if (entry == NULL) + goto drop; + + for (i = 0; i < MAX_FILTERS; i++) { + sample_data = perf_get_sample(kctx, &entry[i]); + + switch (entry[i].op) { + case PBF_OP_EQ: + CHECK_RESULT(sample_data, ==, entry[i].value) + break; + case PBF_OP_NEQ: + CHECK_RESULT(sample_data, !=, entry[i].value) + break; + case PBF_OP_GT: + CHECK_RESULT(sample_data, >, entry[i].value) + break; + case PBF_OP_GE: + CHECK_RESULT(sample_data, >=, entry[i].value) + break; + case PBF_OP_LT: + CHECK_RESULT(sample_data, <, entry[i].value) + break; + case PBF_OP_LE: + CHECK_RESULT(sample_data, <=, entry[i].value) + break; + case PBF_OP_AND: + CHECK_RESULT(sample_data, &, entry[i].value) + break; + case PBF_OP_GROUP_BEGIN: + in_group = 1; + group_result = 0; + break; + case PBF_OP_GROUP_END: + if (group_result == 0) + goto drop; + in_group = 0; + break; + case PBF_OP_DONE: + /* no failures so far, accept it */ + return 1; + } + } + /* generate sample data */ + return 1; + +drop: + losts = bpf_map_lookup_elem(&dropped, &k); + if (losts != NULL) + __sync_fetch_and_add(losts, 1); + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/syscall_summary.bpf.c b/tools/perf/util/bpf_skel/syscall_summary.bpf.c new file mode 100644 index 000000000..4172f3c9f --- /dev/null +++ b/tools/perf/util/bpf_skel/syscall_summary.bpf.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Trace raw_syscalls tracepoints to collect system call statistics. + */ + +#include "vmlinux.h" +#include "syscall_summary.h" + +#include +#include +#include + +/* This is to calculate a delta between sys-enter and sys-exit for each thread */ +struct syscall_trace { + int nr; /* syscall number is only available at sys-enter */ + int unused; + u64 timestamp; +}; + +#define MAX_ENTRIES (128 * 1024) + +struct syscall_trace_map { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); /* tid */ + __type(value, struct syscall_trace); + __uint(max_entries, MAX_ENTRIES); +} syscall_trace_map SEC(".maps"); + +struct syscall_stats_map { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct syscall_key); + __type(value, struct syscall_stats); + __uint(max_entries, MAX_ENTRIES); +} syscall_stats_map SEC(".maps"); + +int enabled; /* controlled from userspace */ + +const volatile enum syscall_aggr_mode aggr_mode; +const volatile int use_cgroup_v2; + +int perf_subsys_id = -1; + +static inline __u64 get_current_cgroup_id(void) +{ + struct task_struct *task; + struct cgroup *cgrp; + + if (use_cgroup_v2) + return bpf_get_current_cgroup_id(); + + task = bpf_get_current_task_btf(); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + +static void update_stats(int cpu_or_tid, u64 cgroup_id, int nr, s64 duration, + long ret) +{ + struct syscall_key key = { + .cpu_or_tid = cpu_or_tid, + .cgroup = cgroup_id, + .nr = nr, + }; + struct syscall_stats *stats; + + stats = bpf_map_lookup_elem(&syscall_stats_map, &key); + if (stats == NULL) { + struct syscall_stats zero = {}; + + bpf_map_update_elem(&syscall_stats_map, &key, &zero, BPF_NOEXIST); + stats = bpf_map_lookup_elem(&syscall_stats_map, &key); + if (stats == NULL) + return; + } + + __sync_fetch_and_add(&stats->count, 1); + if (ret < 0) + __sync_fetch_and_add(&stats->error, 1); + + if (duration > 0) { + __sync_fetch_and_add(&stats->total_time, duration); + __sync_fetch_and_add(&stats->squared_sum, duration * duration); + if (stats->max_time < duration) + stats->max_time = duration; + if (stats->min_time > duration || stats->min_time == 0) + stats->min_time = duration; + } + + return; +} + +SEC("tp_btf/sys_enter") +int sys_enter(u64 *ctx) +{ + int tid; + struct syscall_trace st; + + if (!enabled) + return 0; + + st.nr = ctx[1]; /* syscall number */ + st.unused = 0; + st.timestamp = bpf_ktime_get_ns(); + + tid = bpf_get_current_pid_tgid(); + bpf_map_update_elem(&syscall_trace_map, &tid, &st, BPF_ANY); + + return 0; +} + +static int do_exit(long ret) +{ + int tid; + int key = 0; + u64 cgroup = 0; + struct syscall_trace *st; + s64 delta; + + if (!enabled) + return 0; + + tid = bpf_get_current_pid_tgid(); + st = bpf_map_lookup_elem(&syscall_trace_map, &tid); + if (st == NULL) + return 0; + + if (aggr_mode == SYSCALL_AGGR_THREAD) + key = tid; + else if (aggr_mode == SYSCALL_AGGR_CGROUP) + cgroup = get_current_cgroup_id(); + else + key = bpf_get_smp_processor_id(); + + delta = bpf_ktime_get_ns() - st->timestamp; + update_stats(key, cgroup, st->nr, delta, ret); + + bpf_map_delete_elem(&syscall_trace_map, &tid); + return 0; +} + +SEC("tp_btf/sys_exit") +int sys_exit(u64 *ctx) +{ + long ret = ctx[1]; /* return value of the syscall */ + + return do_exit(ret); +} + +SEC("tp_btf/sched_process_exit") +int process_exit(u64 *ctx) +{ + return do_exit(0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/perf/util/bpf_skel/syscall_summary.h b/tools/perf/util/bpf_skel/syscall_summary.h new file mode 100644 index 000000000..72ccccb45 --- /dev/null +++ b/tools/perf/util/bpf_skel/syscall_summary.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Data structures shared between BPF and tools. */ +#ifndef UTIL_BPF_SKEL_SYSCALL_SUMMARY_H +#define UTIL_BPF_SKEL_SYSCALL_SUMMARY_H + +enum syscall_aggr_mode { + SYSCALL_AGGR_THREAD, + SYSCALL_AGGR_CPU, + SYSCALL_AGGR_CGROUP, +}; + +struct syscall_key { + u64 cgroup; + int cpu_or_tid; + int nr; +}; + +struct syscall_stats { + u64 total_time; + u64 squared_sum; + u64 max_time; + u64 min_time; + u32 count; + u32 error; +}; + +#endif /* UTIL_BPF_SKEL_SYSCALL_SUMMARY_H */ diff --git a/tools/perf/util/bpf_skel/vmlinux/.gitignore b/tools/perf/util/bpf_skel/vmlinux/.gitignore new file mode 100644 index 000000000..49502c041 --- /dev/null +++ b/tools/perf/util/bpf_skel/vmlinux/.gitignore @@ -0,0 +1 @@ +!vmlinux.h diff --git a/tools/perf/util/bpf_skel/vmlinux/vmlinux.h b/tools/perf/util/bpf_skel/vmlinux/vmlinux.h new file mode 100644 index 000000000..a59ce912b --- /dev/null +++ b/tools/perf/util/bpf_skel/vmlinux/vmlinux.h @@ -0,0 +1,215 @@ +#ifndef __VMLINUX_H +#define __VMLINUX_H + +#include // for define __always_inline +#include +#include +#include +#include + +// non-UAPI kernel data structures, used in the .bpf.c BPF tool component. + +// Just the fields used in these tools preserving the access index so that +// libbpf can fixup offsets with the ones used in the kernel when loading the +// BPF bytecode, if they differ from what is used here. + +typedef __u8 u8; +typedef __u32 u32; +typedef __s32 s32; +typedef __u64 u64; +typedef __s64 s64; + +typedef int pid_t; + +typedef __s64 time64_t; + +struct timespec64 { + time64_t tv_sec; + long int tv_nsec; +}; + +enum cgroup_subsys_id { + perf_event_cgrp_id = 8, +}; + +enum { + HI_SOFTIRQ = 0, + TIMER_SOFTIRQ, + NET_TX_SOFTIRQ, + NET_RX_SOFTIRQ, + BLOCK_SOFTIRQ, + IRQ_POLL_SOFTIRQ, + TASKLET_SOFTIRQ, + SCHED_SOFTIRQ, + HRTIMER_SOFTIRQ, + RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */ + + NR_SOFTIRQS +}; + +typedef struct { + s64 counter; +} __attribute__((preserve_access_index)) atomic64_t; + +typedef atomic64_t atomic_long_t; + +struct raw_spinlock { + int rawlock; +} __attribute__((preserve_access_index)); + +typedef struct raw_spinlock raw_spinlock_t; + +typedef struct { + struct raw_spinlock rlock; +} __attribute__((preserve_access_index)) spinlock_t; + +struct sighand_struct { + spinlock_t siglock; +} __attribute__((preserve_access_index)); + +struct rw_semaphore { + atomic_long_t owner; +} __attribute__((preserve_access_index)); + +struct mutex { + atomic_long_t owner; +} __attribute__((preserve_access_index)); + +struct kernfs_node { + u64 id; +} __attribute__((preserve_access_index)); + +struct cgroup { + struct kernfs_node *kn; + int level; +} __attribute__((preserve_access_index)); + +struct cgroup_subsys_state { + struct cgroup *cgroup; +} __attribute__((preserve_access_index)); + +struct css_set { + struct cgroup_subsys_state *subsys[13]; + struct cgroup *dfl_cgrp; +} __attribute__((preserve_access_index)); + +struct mm_struct { + struct rw_semaphore mmap_lock; +} __attribute__((preserve_access_index)); + +struct task_struct { + unsigned int flags; + struct mm_struct *mm; + pid_t pid; + pid_t tgid; + char comm[16]; + struct sighand_struct *sighand; + struct css_set *cgroups; +} __attribute__((preserve_access_index)); + +struct trace_entry { + short unsigned int type; + unsigned char flags; + unsigned char preempt_count; + int pid; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_irq_handler_entry { + struct trace_entry ent; + int irq; + u32 __data_loc_name; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_irq_handler_exit { + struct trace_entry ent; + int irq; + int ret; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_softirq { + struct trace_entry ent; + unsigned int vec; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_workqueue_execute_start { + struct trace_entry ent; + void *work; + void *function; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_workqueue_execute_end { + struct trace_entry ent; + void *work; + void *function; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_workqueue_activate_work { + struct trace_entry ent; + void *work; + char __data[]; +} __attribute__((preserve_access_index)); + +struct perf_sample_data { + u64 addr; + u64 period; + union perf_sample_weight weight; + u64 txn; + union perf_mem_data_src data_src; + u64 ip; + struct { + u32 pid; + u32 tid; + } tid_entry; + u64 time; + u64 id; + struct { + u32 cpu; + } cpu_entry; + u64 phys_addr; + u64 cgroup; + u64 data_page_size; + u64 code_page_size; +} __attribute__((__aligned__(64))) __attribute__((preserve_access_index)); + +struct perf_event { + struct perf_event *parent; + u64 id; +} __attribute__((preserve_access_index)); + +struct bpf_perf_event_data_kern { + struct perf_sample_data *data; + struct perf_event *event; +} __attribute__((preserve_access_index)); + +/* + * If 'struct rq' isn't defined for lock_contention.bpf.c, for the sake of + * rq___old and rq___new, then the type for the 'runqueue' variable ends up + * being a forward declaration (BTF_KIND_FWD) while the kernel has it defined + * (BTF_KIND_STRUCT). The definition appears in vmlinux.h rather than + * lock_contention.bpf.c for consistency with a generated vmlinux.h. + */ +struct rq {}; + +struct kmem_cache { + const char *name; +} __attribute__((preserve_access_index)); + +struct bpf_iter__kmem_cache { + struct kmem_cache *s; +} __attribute__((preserve_access_index)); + +struct zone { + spinlock_t lock; +} __attribute__((preserve_access_index)); + +struct pglist_data { + struct zone node_zones[6]; /* value for all possible config */ + int nr_zones; +} __attribute__((preserve_access_index)); + +#endif // __VMLINUX_H diff --git a/tools/perf/util/bpf_trace_augment.c b/tools/perf/util/bpf_trace_augment.c new file mode 100644 index 000000000..a9cf2a77d --- /dev/null +++ b/tools/perf/util/bpf_trace_augment.c @@ -0,0 +1,143 @@ +#include +#include + +#include "bpf_skel/augmented_raw_syscalls.skel.h" +#include "debug.h" +#include "evlist.h" +#include "parse-events.h" +#include "trace_augment.h" + +static struct augmented_raw_syscalls_bpf *skel; +static struct evsel *bpf_output; + +int augmented_syscalls__prepare(void) +{ + struct bpf_program *prog; + char buf[128]; + int err; + + skel = augmented_raw_syscalls_bpf__open(); + if (!skel) { + pr_debug("Failed to open augmented syscalls BPF skeleton\n"); + return -errno; + } + + /* + * Disable attaching the BPF programs except for sys_enter and + * sys_exit that tail call into this as necessary. + */ + bpf_object__for_each_program(prog, skel->obj) { + if (prog != skel->progs.sys_enter && prog != skel->progs.sys_exit) + bpf_program__set_autoattach(prog, /*autoattach=*/false); + } + + err = augmented_raw_syscalls_bpf__load(skel); + if (err < 0) { + libbpf_strerror(err, buf, sizeof(buf)); + pr_debug("Failed to load augmented syscalls BPF skeleton: %s\n", buf); + return err; + } + + augmented_raw_syscalls_bpf__attach(skel); + return 0; +} + +int augmented_syscalls__create_bpf_output(struct evlist *evlist) +{ + int err = parse_event(evlist, "bpf-output/no-inherit=1,name=__augmented_syscalls__/"); + + if (err) { + pr_err("ERROR: Setup BPF output event failed: %d\n", err); + return err; + } + + bpf_output = evlist__last(evlist); + assert(evsel__name_is(bpf_output, "__augmented_syscalls__")); + + return 0; +} + +void augmented_syscalls__setup_bpf_output(void) +{ + struct perf_cpu cpu; + unsigned int i; + + if (bpf_output == NULL) + return; + + /* + * Set up the __augmented_syscalls__ BPF map to hold for each + * CPU the bpf-output event's file descriptor. + */ + perf_cpu_map__for_each_cpu(cpu, i, bpf_output->core.cpus) { + int mycpu = cpu.cpu; + + bpf_map__update_elem(skel->maps.__augmented_syscalls__, + &mycpu, sizeof(mycpu), + xyarray__entry(bpf_output->core.fd, + mycpu, 0), + sizeof(__u32), BPF_ANY); + } +} + +int augmented_syscalls__set_filter_pids(unsigned int nr, pid_t *pids) +{ + bool value = true; + int err = 0; + + if (skel == NULL) + return 0; + + for (size_t i = 0; i < nr; ++i) { + err = bpf_map__update_elem(skel->maps.pids_filtered, &pids[i], + sizeof(*pids), &value, sizeof(value), + BPF_ANY); + if (err) + break; + } + return err; +} + +int augmented_syscalls__get_map_fds(int *enter_fd, int *exit_fd, int *beauty_fd) +{ + if (skel == NULL) + return -1; + + *enter_fd = bpf_map__fd(skel->maps.syscalls_sys_enter); + *exit_fd = bpf_map__fd(skel->maps.syscalls_sys_exit); + *beauty_fd = bpf_map__fd(skel->maps.beauty_map_enter); + + if (*enter_fd < 0 || *exit_fd < 0 || *beauty_fd < 0) { + pr_err("Error: failed to get syscall or beauty map fd\n"); + return -1; + } + + return 0; +} + +struct bpf_program *augmented_syscalls__unaugmented(void) +{ + return skel->progs.syscall_unaugmented; +} + +struct bpf_program *augmented_syscalls__find_by_title(const char *name) +{ + struct bpf_program *pos; + const char *sec_name; + + if (skel->obj == NULL) + return NULL; + + bpf_object__for_each_program(pos, skel->obj) { + sec_name = bpf_program__section_name(pos); + if (sec_name && !strcmp(sec_name, name)) + return pos; + } + + return NULL; +} + +void augmented_syscalls__cleanup(void) +{ + augmented_raw_syscalls_bpf__destroy(skel); +} diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c new file mode 100644 index 000000000..3712be067 --- /dev/null +++ b/tools/perf/util/branch.c @@ -0,0 +1,229 @@ +#include "util/map_symbol.h" +#include "util/branch.h" +#include + +static bool cross_area(u64 addr1, u64 addr2, int size) +{ + u64 align1, align2; + + align1 = addr1 & ~(size - 1); + align2 = addr2 & ~(size - 1); + + return (align1 != align2) ? true : false; +} + +#define AREA_4K 4096 +#define AREA_2M (2 * 1024 * 1024) + +void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, + u64 from, u64 to) +{ + if (flags->type == PERF_BR_UNKNOWN || from == 0) + return; + + if (flags->type == PERF_BR_EXTEND_ABI) + st->new_counts[flags->new_type]++; + else + st->counts[flags->type]++; + + if (flags->type == PERF_BR_COND) { + if (to > from) + st->cond_fwd++; + else + st->cond_bwd++; + } + + if (cross_area(from, to, AREA_2M)) + st->cross_2m++; + else if (cross_area(from, to, AREA_4K)) + st->cross_4k++; +} + +const char *branch_new_type_name(int new_type) +{ + const char *branch_new_names[PERF_BR_NEW_MAX] = { + "FAULT_ALGN", + "FAULT_DATA", + "FAULT_INST", +/* + * TODO: This switch should happen on 'perf_session__env(session)->arch' + * instead, because an arm64 platform perf recording could be + * opened for analysis on other platforms as well. + */ +#ifdef __aarch64__ + "ARM64_FIQ", + "ARM64_DEBUG_HALT", + "ARM64_DEBUG_EXIT", + "ARM64_DEBUG_INST", + "ARM64_DEBUG_DATA" +#else + "ARCH_1", + "ARCH_2", + "ARCH_3", + "ARCH_4", + "ARCH_5" +#endif + }; + + if (new_type >= 0 && new_type < PERF_BR_NEW_MAX) + return branch_new_names[new_type]; + + return NULL; +} + +const char *branch_type_name(int type) +{ + const char *branch_names[PERF_BR_MAX] = { + "N/A", + "COND", + "UNCOND", + "IND", + "CALL", + "IND_CALL", + "RET", + "SYSCALL", + "SYSRET", + "COND_CALL", + "COND_RET", + "ERET", + "IRQ", + "SERROR", + "NO_TX", + "", // Needed for PERF_BR_EXTEND_ABI that ends up triggering some compiler warnings about NULL deref + }; + + if (type >= 0 && type < PERF_BR_MAX) + return branch_names[type]; + + return NULL; +} + +const char *get_branch_type(struct branch_entry *e) +{ + if (e->flags.type == PERF_BR_UNKNOWN) + return ""; + + if (e->flags.type == PERF_BR_EXTEND_ABI) + return branch_new_type_name(e->flags.new_type); + + return branch_type_name(e->flags.type); +} + +void branch_type_stat_display(FILE *fp, const struct branch_type_stat *st) +{ + u64 total = 0; + int i; + + for (i = 0; i < PERF_BR_MAX; i++) + total += st->counts[i]; + + if (total == 0) + return; + + fprintf(fp, "\n#"); + fprintf(fp, "\n# Branch Statistics:"); + fprintf(fp, "\n#"); + + if (st->cond_fwd > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "COND_FWD", + 100.0 * (double)st->cond_fwd / (double)total); + } + + if (st->cond_bwd > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "COND_BWD", + 100.0 * (double)st->cond_bwd / (double)total); + } + + if (st->cross_4k > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "CROSS_4K", + 100.0 * (double)st->cross_4k / (double)total); + } + + if (st->cross_2m > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "CROSS_2M", + 100.0 * (double)st->cross_2m / (double)total); + } + + for (i = 0; i < PERF_BR_MAX; i++) { + if (st->counts[i] > 0) + fprintf(fp, "\n%8s: %5.1f%%", + branch_type_name(i), + 100.0 * + (double)st->counts[i] / (double)total); + } + + for (i = 0; i < PERF_BR_NEW_MAX; i++) { + if (st->new_counts[i] > 0) + fprintf(fp, "\n%8s: %5.1f%%", + branch_new_type_name(i), + 100.0 * + (double)st->new_counts[i] / (double)total); + } + +} + +static int count_str_scnprintf(int idx, const char *str, char *bf, int size) +{ + return scnprintf(bf, size, "%s%s", (idx) ? " " : " (", str); +} + +int branch_type_str(const struct branch_type_stat *st, char *bf, int size) +{ + int i, j = 0, printed = 0; + u64 total = 0; + + for (i = 0; i < PERF_BR_MAX; i++) + total += st->counts[i]; + + for (i = 0; i < PERF_BR_NEW_MAX; i++) + total += st->new_counts[i]; + + if (total == 0) + return 0; + + if (st->cond_fwd > 0) + printed += count_str_scnprintf(j++, "COND_FWD", bf + printed, size - printed); + + if (st->cond_bwd > 0) + printed += count_str_scnprintf(j++, "COND_BWD", bf + printed, size - printed); + + for (i = 0; i < PERF_BR_MAX; i++) { + if (i == PERF_BR_COND) + continue; + + if (st->counts[i] > 0) + printed += count_str_scnprintf(j++, branch_type_name(i), bf + printed, size - printed); + } + + for (i = 0; i < PERF_BR_NEW_MAX; i++) { + if (st->new_counts[i] > 0) + printed += count_str_scnprintf(j++, branch_new_type_name(i), bf + printed, size - printed); + } + + if (st->cross_4k > 0) + printed += count_str_scnprintf(j++, "CROSS_4K", bf + printed, size - printed); + + if (st->cross_2m > 0) + printed += count_str_scnprintf(j++, "CROSS_2M", bf + printed, size - printed); + + return printed; +} + +const char *branch_spec_desc(int spec) +{ + const char *branch_spec_outcomes[PERF_BR_SPEC_MAX] = { + "N/A", + "SPEC_WRONG_PATH", + "NON_SPEC_CORRECT_PATH", + "SPEC_CORRECT_PATH", + }; + + if (spec >= 0 && spec < PERF_BR_SPEC_MAX) + return branch_spec_outcomes[spec]; + + return NULL; +} diff --git a/tools/perf/util/branch.h b/tools/perf/util/branch.h new file mode 100644 index 000000000..a1d473649 --- /dev/null +++ b/tools/perf/util/branch.h @@ -0,0 +1,99 @@ +#ifndef _PERF_BRANCH_H +#define _PERF_BRANCH_H 1 +/* + * The linux/stddef.h isn't need here, but is needed for __always_inline used + * in files included from uapi/linux/perf_event.h such as + * /usr/include/linux/swab.h and /usr/include/linux/byteorder/little_endian.h, + * detected in at least musl libc, used in Alpine Linux. -acme + */ +#include +#include +#include +#include "util/map_symbol.h" +#include "util/sample.h" + +struct branch_flags { + union { + u64 value; + struct { + u64 mispred:1; + u64 predicted:1; + u64 in_tx:1; + u64 abort:1; + u64 cycles:16; + u64 type:4; + u64 spec:2; + u64 new_type:4; + u64 priv:3; + u64 not_taken:1; + u64 reserved:30; + }; + }; +}; + +struct branch_info { + struct addr_map_symbol from; + struct addr_map_symbol to; + struct branch_flags flags; + u64 branch_stack_cntr; + char *srcline_from; + char *srcline_to; +}; + +struct branch_entry { + u64 from; + u64 to; + struct branch_flags flags; +}; + +struct branch_stack { + u64 nr; + u64 hw_idx; + struct branch_entry entries[]; +}; + +/* + * The hw_idx is only available when PERF_SAMPLE_BRANCH_HW_INDEX is applied. + * Otherwise, the output format of a sample with branch stack is + * struct branch_stack { + * u64 nr; + * struct branch_entry entries[0]; + * } + * Check whether the hw_idx is available, + * and return the corresponding pointer of entries[0]. + */ +static inline struct branch_entry *perf_sample__branch_entries(struct perf_sample *sample) +{ + u64 *entry = (u64 *)sample->branch_stack; + + if (entry == NULL) + return NULL; + + entry++; + if (sample->no_hw_idx) + return (struct branch_entry *)entry; + return (struct branch_entry *)(++entry); +} + +struct branch_type_stat { + bool branch_to; + u64 counts[PERF_BR_MAX]; + u64 new_counts[PERF_BR_NEW_MAX]; + u64 cond_fwd; + u64 cond_bwd; + u64 cross_4k; + u64 cross_2m; +}; + +void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, + u64 from, u64 to); + +const char *branch_type_name(int type); +const char *branch_new_type_name(int new_type); +const char *get_branch_type(struct branch_entry *e); +void branch_type_stat_display(FILE *fp, const struct branch_type_stat *st); +int branch_type_str(const struct branch_type_stat *st, char *bf, int bfsize); + +const char *branch_spec_desc(int spec); + +#endif /* _PERF_BRANCH_H */ diff --git a/tools/perf/util/btf.c b/tools/perf/util/btf.c new file mode 100644 index 000000000..50d98f3e8 --- /dev/null +++ b/tools/perf/util/btf.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arnaldo Carvalho de Melo + * + * Copyright (C) 2024, Red Hat, Inc + */ + +#include +#include +#include + +const struct btf_member *__btf_type__find_member_by_name(struct btf *btf, + int type_id, const char *member_name) +{ + const struct btf_type *t = btf__type_by_id(btf, type_id); + const struct btf_member *m; + u32 i; + + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + const char *current_member_name = btf__name_by_offset(btf, m->name_off); + + if (!strcmp(current_member_name, member_name)) + return m; + } + + return NULL; +} diff --git a/tools/perf/util/btf.h b/tools/perf/util/btf.h new file mode 100644 index 000000000..05e6e5bf2 --- /dev/null +++ b/tools/perf/util/btf.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_UTIL_BTF +#define __PERF_UTIL_BTF 1 + +struct btf; +struct btf_member; + +const struct btf_member *__btf_type__find_member_by_name(struct btf *btf, + int type_id, const char *member_name); +#endif // __PERF_UTIL_BTF diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c new file mode 100644 index 000000000..eb95ab90f --- /dev/null +++ b/tools/perf/util/build-id.c @@ -0,0 +1,983 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * build-id.c + * + * build-id support + * + * Copyright (C) 2009, 2010 Red Hat Inc. + * Copyright (C) 2009, 2010 Arnaldo Carvalho de Melo + */ +#include "util.h" // lsdir(), mkdir_p(), rm_rf() +#include +#include +#include +#include +#include +#include +#include "util/copyfile.h" +#include "dso.h" +#include "build-id.h" +#include "event.h" +#include "namespaces.h" +#include "map.h" +#include "symbol.h" +#include "thread.h" +#include +#include "debug.h" +#include "session.h" +#include "tool.h" +#include "header.h" +#include "vdso.h" +#include "path.h" +#include "probe-file.h" +#include "strlist.h" + +#ifdef HAVE_DEBUGINFOD_SUPPORT +#include +#endif + +#include +#include +#include +#include + +static bool no_buildid_cache; + +static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data __maybe_unused) +{ + struct map *map = node->ms.map; + + if (map) + dso__set_hit(map__dso(map)); + + return 0; +} + +int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct addr_location al; + struct thread *thread = machine__findnew_thread(machine, sample->pid, + sample->tid); + + if (thread == NULL) { + pr_err("problem processing %s event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), sample->file_offset); + return -1; + } + + addr_location__init(&al); + if (thread__find_map(thread, sample->cpumode, sample->ip, &al)) + dso__set_hit(map__dso(al.map)); + + addr_location__exit(&al); + + sample__for_each_callchain_node(thread, sample, PERF_MAX_STACK_DEPTH, + /*symbols=*/false, mark_dso_hit_callback, /*data=*/NULL); + + + thread__put(thread); + return 0; +} + +int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size) +{ + size_t offs = 0; + + if (build_id->size == 0) { + /* Ensure bf is always \0 terminated. */ + if (bf_size > 0) + bf[0] = '\0'; + return 0; + } + + if (bf_size > 0) + bf[0] = '\0'; + + for (size_t i = 0; i < build_id->size && offs + 1 < bf_size; ++i) + offs += scnprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); + + return offs; +} + +int sysfs__snprintf_build_id(const char *root_dir, char *sbuild_id, size_t sbuild_id_size) +{ + char notes[PATH_MAX]; + struct build_id bid = { .size = 0, }; + int ret; + + if (!root_dir) + root_dir = ""; + + scnprintf(notes, sizeof(notes), "%s/sys/kernel/notes", root_dir); + + ret = sysfs__read_build_id(notes, &bid); + if (ret < 0) + return ret; + + return build_id__snprintf(&bid, sbuild_id, sbuild_id_size); +} + +int filename__snprintf_build_id(const char *pathname, char *sbuild_id, size_t sbuild_id_size) +{ + struct build_id bid = { .size = 0, }; + int ret; + + ret = filename__read_build_id(pathname, &bid); + if (ret < 0) + return ret; + + return build_id__snprintf(&bid, sbuild_id, sbuild_id_size); +} + +/* asnprintf consolidates asprintf and snprintf */ +static int asnprintf(char **strp, size_t size, const char *fmt, ...) +{ + va_list ap; + int ret; + + if (!strp) + return -EINVAL; + + va_start(ap, fmt); + if (*strp) + ret = vsnprintf(*strp, size, fmt, ap); + else + ret = vasprintf(strp, fmt, ap); + va_end(ap); + + return ret; +} + +char *build_id_cache__kallsyms_path(const char *sbuild_id, char *bf, + size_t size) +{ + bool retry_old = true; + + snprintf(bf, size, "%s/%s/%s/kallsyms", + buildid_dir, DSO__NAME_KALLSYMS, sbuild_id); +retry: + if (!access(bf, F_OK)) + return bf; + if (retry_old) { + /* Try old style kallsyms cache */ + snprintf(bf, size, "%s/%s/%s", + buildid_dir, DSO__NAME_KALLSYMS, sbuild_id); + retry_old = false; + goto retry; + } + + return NULL; +} + +char *build_id_cache__linkname(const char *sbuild_id, char *bf, size_t size) +{ + char *tmp = bf; + int ret = asnprintf(&bf, size, "%s/.build-id/%.2s/%s", buildid_dir, + sbuild_id, sbuild_id + 2); + if (ret < 0 || (tmp && size < (unsigned int)ret)) + return NULL; + return bf; +} + +/* The caller is responsible to free the returned buffer. */ +char *build_id_cache__origname(const char *sbuild_id) +{ + char *linkname; + char buf[PATH_MAX]; + char *ret = NULL, *p; + size_t offs = 5; /* == strlen("../..") */ + ssize_t len; + + linkname = build_id_cache__linkname(sbuild_id, NULL, 0); + if (!linkname) + return NULL; + + len = readlink(linkname, buf, sizeof(buf) - 1); + if (len <= 0) + goto out; + buf[len] = '\0'; + + /* The link should be "../../" */ + p = strrchr(buf, '/'); /* Cut off the "/" */ + if (p && (p > buf + offs)) { + *p = '\0'; + if (buf[offs + 1] == '[') + offs++; /* + * This is a DSO name, like [kernel.kallsyms]. + * Skip the first '/', since this is not the + * cache of a regular file. + */ + ret = strdup(buf + offs); /* Skip "../..[/]" */ + } +out: + free(linkname); + return ret; +} + +/* Check if the given build_id cache is valid on current running system */ +static bool build_id_cache__valid_id(char *sbuild_id) +{ + char real_sbuild_id[SBUILD_ID_SIZE] = ""; + char *pathname; + int ret = 0; + bool result = false; + + pathname = build_id_cache__origname(sbuild_id); + if (!pathname) + return false; + + if (!strcmp(pathname, DSO__NAME_KALLSYMS)) + ret = sysfs__snprintf_build_id("/", real_sbuild_id, sizeof(real_sbuild_id)); + else if (pathname[0] == '/') + ret = filename__snprintf_build_id(pathname, real_sbuild_id, sizeof(real_sbuild_id)); + else + ret = -EINVAL; /* Should we support other special DSO cache? */ + if (ret >= 0) + result = (strcmp(sbuild_id, real_sbuild_id) == 0); + free(pathname); + + return result; +} + +static const char *build_id_cache__basename(bool is_kallsyms, bool is_vdso, + bool is_debug) +{ + return is_kallsyms ? "kallsyms" : (is_vdso ? "vdso" : (is_debug ? + "debug" : "elf")); +} + +char *__dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug, bool is_kallsyms) +{ + bool is_vdso = dso__is_vdso((struct dso *)dso); + char sbuild_id[SBUILD_ID_SIZE]; + char *linkname; + bool alloc = (bf == NULL); + int ret; + + if (!dso__has_build_id(dso)) + return NULL; + + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + linkname = build_id_cache__linkname(sbuild_id, NULL, 0); + if (!linkname) + return NULL; + + /* Check if old style build_id cache */ + if (is_regular_file(linkname)) + ret = asnprintf(&bf, size, "%s", linkname); + else + ret = asnprintf(&bf, size, "%s/%s", linkname, + build_id_cache__basename(is_kallsyms, is_vdso, + is_debug)); + if (ret < 0 || (!alloc && size < (unsigned int)ret)) + bf = NULL; + free(linkname); + + return bf; +} + +char *dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug) +{ + bool is_kallsyms = dso__is_kallsyms((struct dso *)dso); + + return __dso__build_id_filename(dso, bf, size, is_debug, is_kallsyms); +} + +static int write_buildid(const char *name, size_t name_len, struct build_id *bid, + pid_t pid, u16 misc, struct feat_fd *fd) +{ + int err; + struct perf_record_header_build_id b; + size_t len; + + len = name_len + 1; + len = PERF_ALIGN(len, sizeof(u64)); + + memset(&b, 0, sizeof(b)); + memcpy(&b.data, bid->data, bid->size); + b.size = (u8) bid->size; + misc |= PERF_RECORD_MISC_BUILD_ID_SIZE; + b.pid = pid; + b.header.misc = misc; + b.header.size = sizeof(b) + len; + + err = do_write(fd, &b, sizeof(b)); + if (err < 0) + return err; + + return write_padded(fd, name, name_len + 1, len); +} + +struct machine__write_buildid_table_cb_args { + struct machine *machine; + struct feat_fd *fd; + u16 kmisc, umisc; +}; + +static int machine__write_buildid_table_cb(struct dso *dso, void *data) +{ + struct machine__write_buildid_table_cb_args *args = data; + const char *name; + size_t name_len; + bool in_kernel = false; + + if (!dso__has_build_id(dso)) + return 0; + + if (!dso__hit(dso) && !dso__is_vdso(dso)) + return 0; + + if (dso__is_vdso(dso)) { + name = dso__short_name(dso); + name_len = dso__short_name_len(dso); + } else if (dso__is_kcore(dso)) { + name = args->machine->mmap_name; + name_len = strlen(name); + } else { + name = dso__long_name(dso); + name_len = dso__long_name_len(dso); + } + + in_kernel = dso__kernel(dso) || is_kernel_module(name, PERF_RECORD_MISC_CPUMODE_UNKNOWN); + return write_buildid(name, name_len, &dso__id(dso)->build_id, args->machine->pid, + in_kernel ? args->kmisc : args->umisc, args->fd); +} + +static int machine__write_buildid_table(struct machine *machine, struct feat_fd *fd) +{ + struct machine__write_buildid_table_cb_args args = { + .machine = machine, + .fd = fd, + .kmisc = PERF_RECORD_MISC_KERNEL, + .umisc = PERF_RECORD_MISC_USER, + }; + + if (!machine__is_host(machine)) { + args.kmisc = PERF_RECORD_MISC_GUEST_KERNEL; + args.umisc = PERF_RECORD_MISC_GUEST_USER; + } + + return dsos__for_each_dso(&machine->dsos, machine__write_buildid_table_cb, &args); +} + +int perf_session__write_buildid_table(struct perf_session *session, + struct feat_fd *fd) +{ + struct rb_node *nd; + int err = machine__write_buildid_table(&session->machines.host, fd); + + if (err) + return err; + + for (nd = rb_first_cached(&session->machines.guests); nd; + nd = rb_next(nd)) { + struct machine *pos = rb_entry(nd, struct machine, rb_node); + err = machine__write_buildid_table(pos, fd); + if (err) + break; + } + return err; +} + +void disable_buildid_cache(void) +{ + no_buildid_cache = true; +} + +static bool lsdir_bid_head_filter(const char *name __maybe_unused, + struct dirent *d) +{ + return (strlen(d->d_name) == 2) && + isxdigit(d->d_name[0]) && isxdigit(d->d_name[1]); +} + +static bool lsdir_bid_tail_filter(const char *name __maybe_unused, + struct dirent *d) +{ + int i = 0; + while (isxdigit(d->d_name[i]) && i < SBUILD_ID_SIZE - 3) + i++; + return (i >= SBUILD_ID_MIN_SIZE - 3) && (i <= SBUILD_ID_SIZE - 3) && + (d->d_name[i] == '\0'); +} + +struct strlist *build_id_cache__list_all(bool validonly) +{ + struct strlist *toplist, *linklist = NULL, *bidlist; + struct str_node *nd, *nd2; + char *topdir, *linkdir = NULL; + char sbuild_id[SBUILD_ID_SIZE]; + + /* for filename__ functions */ + if (validonly) + symbol__init(NULL); + + /* Open the top-level directory */ + if (asprintf(&topdir, "%s/.build-id/", buildid_dir) < 0) + return NULL; + + bidlist = strlist__new(NULL, NULL); + if (!bidlist) + goto out; + + toplist = lsdir(topdir, lsdir_bid_head_filter); + if (!toplist) { + pr_debug("Error in lsdir(%s): %d\n", topdir, errno); + /* If there is no buildid cache, return an empty list */ + if (errno == ENOENT) + goto out; + goto err_out; + } + + strlist__for_each_entry(nd, toplist) { + if (asprintf(&linkdir, "%s/%s", topdir, nd->s) < 0) + goto err_out; + /* Open the lower-level directory */ + linklist = lsdir(linkdir, lsdir_bid_tail_filter); + if (!linklist) { + pr_debug("Error in lsdir(%s): %d\n", linkdir, errno); + goto err_out; + } + strlist__for_each_entry(nd2, linklist) { + if (snprintf(sbuild_id, SBUILD_ID_SIZE, "%s%s", + nd->s, nd2->s) > SBUILD_ID_SIZE - 1) + goto err_out; + if (validonly && !build_id_cache__valid_id(sbuild_id)) + continue; + if (strlist__add(bidlist, sbuild_id) < 0) + goto err_out; + } + strlist__delete(linklist); + zfree(&linkdir); + } + +out_free: + strlist__delete(toplist); +out: + free(topdir); + + return bidlist; + +err_out: + strlist__delete(linklist); + zfree(&linkdir); + strlist__delete(bidlist); + bidlist = NULL; + goto out_free; +} + +static bool str_is_build_id(const char *maybe_sbuild_id, size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) { + if (!isxdigit(maybe_sbuild_id[i])) + return false; + } + return true; +} + +/* Return the valid complete build-id */ +char *build_id_cache__complement(const char *incomplete_sbuild_id) +{ + struct strlist *bidlist; + struct str_node *nd, *cand = NULL; + char *sbuild_id = NULL; + size_t len = strlen(incomplete_sbuild_id); + + if (len >= SBUILD_ID_SIZE || + !str_is_build_id(incomplete_sbuild_id, len)) + return NULL; + + bidlist = build_id_cache__list_all(true); + if (!bidlist) + return NULL; + + strlist__for_each_entry(nd, bidlist) { + if (strncmp(nd->s, incomplete_sbuild_id, len) != 0) + continue; + if (cand) { /* Error: There are more than 2 candidates. */ + cand = NULL; + break; + } + cand = nd; + } + if (cand) + sbuild_id = strdup(cand->s); + strlist__delete(bidlist); + + return sbuild_id; +} + +char *build_id_cache__cachedir(const char *sbuild_id, const char *name, + struct nsinfo *nsi, bool is_kallsyms, + bool is_vdso) +{ + char *realname = NULL, *filename; + bool slash = is_kallsyms || is_vdso; + + if (!slash) + realname = nsinfo__realpath(name, nsi); + + if (asprintf(&filename, "%s%s%s%s%s", buildid_dir, slash ? "/" : "", + is_vdso ? DSO__NAME_VDSO : (realname ? realname : name), + sbuild_id ? "/" : "", sbuild_id ?: "") < 0) + filename = NULL; + + free(realname); + return filename; +} + +int build_id_cache__list_build_ids(const char *pathname, struct nsinfo *nsi, + struct strlist **result) +{ + char *dir_name; + int ret = 0; + + dir_name = build_id_cache__cachedir(NULL, pathname, nsi, false, false); + if (!dir_name) + return -ENOMEM; + + *result = lsdir(dir_name, lsdir_no_dot_filter); + if (!*result) + ret = -errno; + free(dir_name); + + return ret; +} + +#if defined(HAVE_LIBELF_SUPPORT) && defined(HAVE_GELF_GETNOTE_SUPPORT) +static int build_id_cache__add_sdt_cache(const char *sbuild_id, + const char *realname, + struct nsinfo *nsi) +{ + struct probe_cache *cache; + int ret; + struct nscookie nsc; + + cache = probe_cache__new(sbuild_id, nsi); + if (!cache) + return -1; + + nsinfo__mountns_enter(nsi, &nsc); + ret = probe_cache__scan_sdt(cache, realname); + nsinfo__mountns_exit(&nsc); + if (ret >= 0) { + pr_debug4("Found %d SDTs in %s\n", ret, realname); + if (probe_cache__commit(cache) < 0) + ret = -1; + } + probe_cache__delete(cache); + return ret; +} +#else +#define build_id_cache__add_sdt_cache(sbuild_id, realname, nsi) (0) +#endif + +static char *build_id_cache__find_debug(const char *sbuild_id, + struct nsinfo *nsi, + const char *root_dir) +{ + const char *dirname = "/usr/lib/debug/.build-id/"; + char *realname = NULL; + char dirbuf[PATH_MAX]; + char *debugfile; + struct nscookie nsc; + size_t len = 0; + + debugfile = calloc(1, PATH_MAX); + if (!debugfile) + goto out; + + if (root_dir) { + path__join(dirbuf, PATH_MAX, root_dir, dirname); + dirname = dirbuf; + } + + len = __symbol__join_symfs(debugfile, PATH_MAX, dirname); + snprintf(debugfile + len, PATH_MAX - len, "%.2s/%s.debug", sbuild_id, + sbuild_id + 2); + + nsinfo__mountns_enter(nsi, &nsc); + realname = realpath(debugfile, NULL); + if (realname && access(realname, R_OK)) + zfree(&realname); + nsinfo__mountns_exit(&nsc); + +#ifdef HAVE_DEBUGINFOD_SUPPORT + if (realname == NULL) { + debuginfod_client* c; + + pr_debug("Downloading debug info with build id %s\n", sbuild_id); + + c = debuginfod_begin(); + if (c != NULL) { + int fd = debuginfod_find_debuginfo(c, + (const unsigned char*)sbuild_id, 0, + &realname); + if (fd >= 0) + close(fd); /* retaining reference by realname */ + debuginfod_end(c); + } + } +#endif + +out: + free(debugfile); + return realname; +} + +int +build_id_cache__add(const char *sbuild_id, const char *name, const char *realname, + struct nsinfo *nsi, bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir) +{ + const size_t size = PATH_MAX; + char *filename = NULL, *dir_name = NULL, *linkname = zalloc(size), *tmp; + char *debugfile = NULL; + int err = -1; + + if (!proper_name) + proper_name = name; + + dir_name = build_id_cache__cachedir(sbuild_id, proper_name, nsi, is_kallsyms, + is_vdso); + if (!dir_name) + goto out_free; + + /* Remove old style build-id cache */ + if (is_regular_file(dir_name)) + if (unlink(dir_name)) + goto out_free; + + if (mkdir_p(dir_name, 0755)) + goto out_free; + + /* Save the allocated buildid dirname */ + if (asprintf(&filename, "%s/%s", dir_name, + build_id_cache__basename(is_kallsyms, is_vdso, + false)) < 0) { + filename = NULL; + goto out_free; + } + + if (access(filename, F_OK)) { + if (is_kallsyms) { + if (copyfile("/proc/kallsyms", filename)) + goto out_free; + } else if (nsi && nsinfo__need_setns(nsi)) { + if (copyfile_ns(name, filename, nsi)) + goto out_free; + } else if (link(realname, filename) && errno != EEXIST) { + struct stat f_stat; + + if (!(stat(name, &f_stat) < 0) && + copyfile_mode(name, filename, f_stat.st_mode)) + goto out_free; + } + } + + /* Some binaries are stripped, but have .debug files with their symbol + * table. Check to see if we can locate one of those, since the elf + * file itself may not be very useful to users of our tools without a + * symtab. + */ + if (!is_kallsyms && !is_vdso && + strncmp(".ko", name + strlen(name) - 3, 3)) { + debugfile = build_id_cache__find_debug(sbuild_id, nsi, root_dir); + if (debugfile) { + zfree(&filename); + if (asprintf(&filename, "%s/%s", dir_name, + build_id_cache__basename(false, false, true)) < 0) { + filename = NULL; + goto out_free; + } + if (access(filename, F_OK)) { + if (nsi && nsinfo__need_setns(nsi)) { + if (copyfile_ns(debugfile, filename, + nsi)) + goto out_free; + } else if (link(debugfile, filename) && + errno != EEXIST && + copyfile(debugfile, filename)) + goto out_free; + } + } + } + + if (!build_id_cache__linkname(sbuild_id, linkname, size)) + goto out_free; + tmp = strrchr(linkname, '/'); + *tmp = '\0'; + + if (access(linkname, X_OK) && mkdir_p(linkname, 0755)) + goto out_free; + + *tmp = '/'; + tmp = dir_name + strlen(buildid_dir) - 5; + memcpy(tmp, "../..", 5); + + if (symlink(tmp, linkname) == 0) { + err = 0; + } else if (errno == EEXIST) { + char path[PATH_MAX]; + ssize_t len; + + len = readlink(linkname, path, sizeof(path) - 1); + if (len <= 0) { + pr_err("Can't read link: %s\n", linkname); + goto out_free; + } + path[len] = '\0'; + + if (strcmp(tmp, path)) { + pr_debug("build <%s> already linked to %s\n", + sbuild_id, linkname); + } + err = 0; + } + + /* Update SDT cache : error is just warned */ + if (realname && + build_id_cache__add_sdt_cache(sbuild_id, realname, nsi) < 0) + pr_debug4("Failed to update/scan SDT cache for %s\n", realname); + +out_free: + free(filename); + free(debugfile); + free(dir_name); + free(linkname); + return err; +} + +int __build_id_cache__add_s(const char *sbuild_id, const char *name, + struct nsinfo *nsi, bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir) +{ + char *realname = NULL; + int err = -1; + + if (!is_kallsyms) { + if (!is_vdso) + realname = nsinfo__realpath(name, nsi); + else + realname = realpath(name, NULL); + if (!realname) + goto out_free; + } + + err = build_id_cache__add(sbuild_id, name, realname, nsi, + is_kallsyms, is_vdso, proper_name, root_dir); +out_free: + if (!is_kallsyms) + free(realname); + return err; +} + +static int build_id_cache__add_b(const struct build_id *bid, + const char *name, struct nsinfo *nsi, + bool is_kallsyms, bool is_vdso, + const char *proper_name, + const char *root_dir) +{ + char sbuild_id[SBUILD_ID_SIZE]; + + build_id__snprintf(bid, sbuild_id, sizeof(sbuild_id)); + + return __build_id_cache__add_s(sbuild_id, name, nsi, is_kallsyms, + is_vdso, proper_name, root_dir); +} + +bool build_id_cache__cached(const char *sbuild_id) +{ + bool ret = false; + char *filename = build_id_cache__linkname(sbuild_id, NULL, 0); + + if (filename && !access(filename, F_OK)) + ret = true; + free(filename); + + return ret; +} + +int build_id_cache__remove_s(const char *sbuild_id) +{ + const size_t size = PATH_MAX; + char *filename = zalloc(size), + *linkname = zalloc(size), *tmp; + int err = -1; + + if (filename == NULL || linkname == NULL) + goto out_free; + + if (!build_id_cache__linkname(sbuild_id, linkname, size)) + goto out_free; + + if (access(linkname, F_OK)) + goto out_free; + + if (readlink(linkname, filename, size - 1) < 0) + goto out_free; + + if (unlink(linkname)) + goto out_free; + + /* + * Since the link is relative, we must make it absolute: + */ + tmp = strrchr(linkname, '/') + 1; + snprintf(tmp, size - (tmp - linkname), "%s", filename); + + if (rm_rf(linkname)) + goto out_free; + + err = 0; +out_free: + free(filename); + free(linkname); + return err; +} + +static int filename__read_build_id_ns(const char *filename, + struct build_id *bid, + struct nsinfo *nsi) +{ + struct nscookie nsc; + int ret; + + nsinfo__mountns_enter(nsi, &nsc); + ret = filename__read_build_id(filename, bid); + nsinfo__mountns_exit(&nsc); + + return ret; +} + +static bool dso__build_id_mismatch(struct dso *dso, const char *name) +{ + struct build_id bid = { .size = 0, }; + bool ret = false; + + mutex_lock(dso__lock(dso)); + if (filename__read_build_id_ns(name, &bid, dso__nsinfo(dso)) >= 0) + ret = !dso__build_id_equal(dso, &bid); + + mutex_unlock(dso__lock(dso)); + + return ret; +} + +static int dso__cache_build_id(struct dso *dso, struct machine *machine, + void *priv __maybe_unused) +{ + bool is_kallsyms = dso__is_kallsyms(dso); + bool is_vdso = dso__is_vdso(dso); + const char *name = dso__long_name(dso); + const char *proper_name = NULL; + const char *root_dir = NULL; + char *allocated_name = NULL; + int ret = 0; + + if (!dso__has_build_id(dso) || !dso__hit(dso)) + return 0; + + if (dso__is_kcore(dso)) { + is_kallsyms = true; + name = machine->mmap_name; + } + + if (!machine__is_host(machine)) { + if (*machine->root_dir) { + root_dir = machine->root_dir; + ret = asprintf(&allocated_name, "%s/%s", root_dir, name); + if (ret < 0) + return ret; + proper_name = name; + name = allocated_name; + } else if (is_kallsyms) { + /* Cannot get guest kallsyms */ + return 0; + } + } + + if (!is_kallsyms && dso__build_id_mismatch(dso, name)) + goto out_free; + + mutex_lock(dso__lock(dso)); + ret = build_id_cache__add_b(dso__bid(dso), name, dso__nsinfo(dso), + is_kallsyms, is_vdso, proper_name, root_dir); + mutex_unlock(dso__lock(dso)); +out_free: + free(allocated_name); + return ret; +} + +static int +machines__for_each_dso(struct machines *machines, machine__dso_t fn, void *priv) +{ + int ret = machine__for_each_dso(&machines->host, fn, priv); + struct rb_node *nd; + + for (nd = rb_first_cached(&machines->guests); nd; + nd = rb_next(nd)) { + struct machine *pos = rb_entry(nd, struct machine, rb_node); + + ret |= machine__for_each_dso(pos, fn, priv); + } + return ret ? -1 : 0; +} + +int __perf_session__cache_build_ids(struct perf_session *session, + machine__dso_t fn, void *priv) +{ + if (no_buildid_cache) + return 0; + + if (mkdir(buildid_dir, 0755) != 0 && errno != EEXIST) + return -1; + + return machines__for_each_dso(&session->machines, fn, priv) ? -1 : 0; +} + +int perf_session__cache_build_ids(struct perf_session *session) +{ + return __perf_session__cache_build_ids(session, dso__cache_build_id, NULL); +} + +static bool machine__read_build_ids(struct machine *machine, bool with_hits) +{ + return dsos__read_build_ids(&machine->dsos, with_hits); +} + +bool perf_session__read_build_ids(struct perf_session *session, bool with_hits) +{ + struct rb_node *nd; + bool ret = machine__read_build_ids(&session->machines.host, with_hits); + + for (nd = rb_first_cached(&session->machines.guests); nd; + nd = rb_next(nd)) { + struct machine *pos = rb_entry(nd, struct machine, rb_node); + ret |= machine__read_build_ids(pos, with_hits); + } + + return ret; +} + +void build_id__init(struct build_id *bid, const u8 *data, size_t size) +{ + if (size > BUILD_ID_SIZE) { + pr_debug("Truncating build_id size from %zd\n", size); + size = BUILD_ID_SIZE; + } + memcpy(bid->data, data, size); + bid->size = size; +} + +bool build_id__is_defined(const struct build_id *bid) +{ + return bid && bid->size ? !!memchr_inv(bid->data, 0, bid->size) : false; +} diff --git a/tools/perf/util/build-id.h b/tools/perf/util/build-id.h new file mode 100644 index 000000000..73bad90b0 --- /dev/null +++ b/tools/perf/util/build-id.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_BUILD_ID_H_ +#define PERF_BUILD_ID_H_ 1 + +#define BUILD_ID_SIZE 20 /* SHA-1 length in bytes */ +#define BUILD_ID_MIN_SIZE 16 /* MD5/UUID/GUID length in bytes */ +#define SBUILD_ID_SIZE (BUILD_ID_SIZE * 2 + 1) +#define SBUILD_ID_MIN_SIZE (BUILD_ID_MIN_SIZE * 2 + 1) + +#include "machine.h" +#include "tool.h" +#include + +struct build_id { + u8 data[BUILD_ID_SIZE]; + u8 size; +}; + +struct dso; +struct feat_fd; +struct nsinfo; + +void build_id__init(struct build_id *bid, const u8 *data, size_t size); +int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size); +bool build_id__is_defined(const struct build_id *bid); +int sysfs__snprintf_build_id(const char *root_dir, char *sbuild_id, size_t sbuild_id_size); +int filename__snprintf_build_id(const char *pathname, char *sbuild_id, size_t sbuild_id_size); +char *build_id_cache__kallsyms_path(const char *sbuild_id, char *bf, + size_t size); + +char *dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug); +char *__dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug, bool is_kallsyms); + +int build_id__mark_dso_hit(const struct perf_tool *tool, union perf_event *event, + struct perf_sample *sample, struct machine *machine); + +bool perf_session__read_build_ids(struct perf_session *session, bool with_hits); +int perf_session__write_buildid_table(struct perf_session *session, + struct feat_fd *fd); +int perf_session__cache_build_ids(struct perf_session *session); +int __perf_session__cache_build_ids(struct perf_session *session, + machine__dso_t fn, void *priv); + +char *build_id_cache__origname(const char *sbuild_id); +char *build_id_cache__linkname(const char *sbuild_id, char *bf, size_t size); +char *build_id_cache__cachedir(const char *sbuild_id, const char *name, + struct nsinfo *nsi, bool is_kallsyms, + bool is_vdso); + +struct strlist; + +struct strlist *build_id_cache__list_all(bool validonly); +char *build_id_cache__complement(const char *incomplete_sbuild_id); +int build_id_cache__list_build_ids(const char *pathname, struct nsinfo *nsi, + struct strlist **result); +bool build_id_cache__cached(const char *sbuild_id); +int build_id_cache__add(const char *sbuild_id, const char *name, const char *realname, + struct nsinfo *nsi, bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir); +int __build_id_cache__add_s(const char *sbuild_id, + const char *name, struct nsinfo *nsi, + bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir); +static inline int build_id_cache__add_s(const char *sbuild_id, + const char *name, struct nsinfo *nsi, + bool is_kallsyms, bool is_vdso) +{ + return __build_id_cache__add_s(sbuild_id, name, nsi, is_kallsyms, is_vdso, NULL, NULL); +} +int build_id_cache__remove_s(const char *sbuild_id); + +extern char buildid_dir[]; + +void set_buildid_dir(const char *dir); +void disable_buildid_cache(void); + +#endif diff --git a/tools/perf/util/c2c-function.c b/tools/perf/util/c2c-function.c new file mode 100644 index 000000000..5b6a06a5a --- /dev/null +++ b/tools/perf/util/c2c-function.c @@ -0,0 +1,1642 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * C2C function model - function-level cacheline sharing analysis + * + * Displays a 3-level hierarchy showing which functions share cachelines: + * Level 1: Read-side functions sorted by Cycles % (estimated load cycles) + * Level 2: Functions sampled writing the shared lines read by level 1 + * Level 3: The specific cachelines where the two functions contend + * + * Builds the hierarchy from the existing cacheline histograms + * (c2c_hist_entry->hists), reusing the shared c2c data structures. + */ + +#include +#include +#include +#include +#include /* reallocarray */ +#include +#include +#include + +#include "addr_location.h" +#include "c2c.h" +#include "cacheline.h" +#include "debug.h" +#include "dso.h" +#include "hist.h" +#include "map.h" +#include "mem-events.h" +#include "mem-info.h" +#include "sort.h" +#include "symbol.h" +#include "thread.h" + +struct c2c_function_model { + struct c2c_hists function_hists; + /* Total estimated cycles across all level-1 entries. */ + u64 total_cycles; + /* Source cacheline histograms; not owned here. */ + struct c2c_hists *cl_hists; + /* --coalesce field list, used to require iaddr. */ + const char *cl_sort; + /* Do not cap long symbol names. */ + bool symbol_full; +}; + +static struct c2c_function_model c2c_ext; + +static inline u64 c2c_hitm_count(const struct c2c_stats *stats) +{ + return stats->tot_hitm; +} + +static int64_t c2c_function_cmp(const struct map_symbol *left, + const struct map_symbol *right) +{ + const struct dso *left_dso = left->map ? map__dso(left->map) : NULL; + const struct dso *right_dso = right->map ? map__dso(right->map) : NULL; + int ret; + + if (!left_dso || !right_dso) { + if (left_dso != right_dso) + return left_dso ? 1 : -1; + } else { + /* + * Use the same DSO name as _sort__dso_cmp() (short name unless + * verbose), so this matches the DSO comparison the level-1 + * entries are deduplicated by; otherwise same-basename DSOs + * could be split or merged inconsistently across levels. + */ + const char *left_name = verbose > 0 ? + dso__long_name(left_dso) : dso__short_name(left_dso); + const char *right_name = verbose > 0 ? + dso__long_name(right_dso) : dso__short_name(right_dso); + + ret = strcmp(left_name, right_name); + if (ret) + return ret; + } + + return _sort__sym_cmp(left->sym, right->sym); +} + +static inline u64 hist_entry__iaddr(struct hist_entry *he) +{ + if (he->mem_info) + return mem_info__iaddr(he->mem_info)->addr; + return he->ip; +} + +/* + * Hierarchy levels (by depth): L1 = read-side function, L2 = the writing + * function it contends with, L3 = the specific shared cacheline. + */ +static inline bool hist_entry__is_cacheline(struct hist_entry *he) +{ + return he->parent_he && he->parent_he->parent_he; /* level 3: cacheline */ +} + +/* Spaces of indent per hierarchy level, like the normal report view. */ +#define C2C_FUNC_INDENT 2 + +/* Width of the folded-sign prefix ("%c ") each identity cell emits. */ +#define C2C_FUNC_FOLD_WIDTH 2 + +/* + * Write he->depth levels of leading indentation into @buf, so lower-level + * entries are visually nested under their parent. Returns bytes written. + */ +static int hist_entry__indent(struct hist_entry *he, char *buf, size_t size) +{ + int indent = he->depth * C2C_FUNC_INDENT; + + if (indent <= 0 || (size_t)indent >= size) + return 0; + + return scnprintf(buf, size, "%*s", indent, ""); +} + +static int symbol_width(struct hists *hists, struct sort_entry *se) +{ + int width = hists__col_len(hists, se->se_width_idx); + + /* + * Cap long symbol names as the cacheline view does. The stored column + * length is grown up front to fit the deepest, longest identity cell + * (including a level-3 cacheline address), so this cap never shrinks the + * column below what the cacheline address needs. + */ + if (!c2c_ext.symbol_full && width > SYMBOL_WIDTH) + width = SYMBOL_WIDTH; + + return width; +} + +static struct c2c_dimension dim_symbol_view; + +/* + * c2c_width - Calculate width for a C2C column in function view + */ +static int c2c_width(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp __maybe_unused, + struct hists *hists) +{ + struct c2c_fmt *c2c_fmt; + struct c2c_dimension *dim; + + c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); + dim = c2c_fmt->dim; + + if (dim == &dim_symbol_view) + return symbol_width(hists, dim->se); + + return dim->se ? hists__col_len(hists, dim->se->se_width_idx) : + dim->width; +} + +static int c2c_header(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hists *hists, int line, int *span) +{ + struct c2c_fmt *c2c_fmt; + struct c2c_dimension *dim; + const char *text = NULL; + int width = c2c_width(fmt, hpp, hists); + + c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); + dim = c2c_fmt->dim; + + if (dim->se) { + text = dim->header.line[line].text; + /* Use the last line from sort_entry if not defined. */ + if (!text && line == hists->hpp_list->nr_header_lines - 1) + text = dim->se->se_header; + } else { + text = dim->header.line[line].text; + + if (span) { + if (*span) { + (*span)--; + return 0; + } + + *span = dim->header.line[line].span; + } + } + + if (!text) + text = ""; + + return scnprintf(hpp->buf, hpp->size, "%*s", width, text); +} + +/* + * Return the estimated total cycles for a c2c_hist_entry + * (rmt_hitm + lcl_hitm + rmt_peer + lcl_peer + other loads). + */ +static u64 c2c_hist_entry__cycles(struct c2c_hist_entry *c2c_he) +{ + struct compute_stats *cs = &c2c_he->cstats; + double cycles = 0; + + /* + * compute_stats() in builtin-c2c.c routes each load sample into exactly + * one cstats bucket (rmt_hitm, lcl_hitm, rmt_peer, lcl_peer or plain + * load), so each bucket's cycle total is its mean times its own sample + * count. Summing the per-bucket totals avoids both dropping peer-snoop + * cycles and double counting a sample that carries several data-source + * flags (e.g. Arm SPE sets HITM and PEER on the same load), which would + * happen if the mean were multiplied by the non-exclusive stats counts. + */ + cycles += avg_stats(&cs->rmt_hitm) * cs->rmt_hitm.n; + cycles += avg_stats(&cs->lcl_hitm) * cs->lcl_hitm.n; + cycles += avg_stats(&cs->rmt_peer) * cs->rmt_peer.n; + cycles += avg_stats(&cs->lcl_peer) * cs->lcl_peer.n; + cycles += avg_stats(&cs->load) * cs->load.n; + + return (u64)cycles; +} + +/* Sum c2c_hist_entry__cycles() across all level-1 entries. */ +static u64 c2c_ext__total_cycles(void) +{ + struct rb_node *nd; + u64 total = 0; + + for (nd = rb_first_cached(&c2c_ext.function_hists.hists.entries); nd; + nd = rb_next(nd)) { + struct c2c_hist_entry *c2c_he = + rb_entry(nd, struct c2c_hist_entry, he.rb_node); + + total += c2c_hist_entry__cycles(c2c_he); + } + return total; +} + +/* + * Store count shown in the column: a level-3 cacheline leaf shows its parent + * level-2 writer's stores on that line, not all stores on the line. A level-2 + * writer shows the sum across its level-3 cachelines. A level-1 reader shows + * the sum across all included writers on the cachelines it reads; this is not + * the reader function's own store count and is not additive across readers. + */ +static u64 hist_entry__displayed_stores(struct hist_entry *he) +{ + struct c2c_hist_entry *c2c_he = container_of(he, struct c2c_hist_entry, he); + struct rb_node *nd; + u64 stores = 0; + + /* Level-2/3 entries already aggregate the stores they represent. */ + if (he->parent_he) + return c2c_he->stats.store; + + for (nd = rb_first_cached(&he->hroot_out); nd; nd = rb_next(nd)) { + struct c2c_hist_entry *child_c2c = + rb_entry(nd, struct c2c_hist_entry, he.rb_node); + stores += child_c2c->stats.store; + } + return stores; +} + +static int +total_stores_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + int width = c2c_width(fmt, hpp, he->hists); + u64 total = hist_entry__displayed_stores(he); + + return scnprintf(hpp->buf, hpp->size, "%*" PRIu64, width, total); +} + +/* + * symbol_view_entry - Render the unified, indented identity column. + * + * All three levels share this single column so the hierarchy reads top-down + * with progressive indentation, like the normal report hierarchy view. It is + * a function-centric view with no dedicated code-address column. Verbose + * function rows can still include a representative address: + * L1 read-side function: "- [k] cpupri_set" + * L2 writing function: " - [k] pull_rt_task" + * L3 shared cacheline: " 0xff2d0082809da080" + */ +static int +symbol_view_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + int width = c2c_width(fmt, hpp, he->hists); + int text_width; + int ret; + char folded_sign; + + ret = hist_entry__indent(he, hpp->buf, hpp->size); + + folded_sign = he->has_children ? (he->unfolded ? '-' : '+') : ' '; + ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%c ", folded_sign); + + text_width = width - ret; + if (text_width <= 0) + return ret; + + if (hist_entry__is_cacheline(he)) { + /* Level 3: the shared cacheline address. */ + u64 addr = he->mem_info ? + cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl) : 0; + char symbuf[32]; + + scnprintf(symbuf, sizeof(symbuf), "0x%" PRIx64, addr); + ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%-*.*s", + text_width, text_width, symbuf); + } else { + /* Level 1 and level 2 are both functions. */ + size_t cell_size; + int len; + + if ((size_t)ret >= hpp->size) + return ret; + + cell_size = min_t(size_t, hpp->size - ret, + (size_t)text_width + 1); + len = sort_sym.se_snprintf(he, hpp->buf + ret, cell_size, + text_width); + /* + * se_snprintf() accumulates repsep_snprintf() calls, which cap + * their return at the remaining size - 1 rather than reporting + * what the format would have needed, so len stays below + * cell_size. Clamp anyway so ret cannot leave hpp->buf. + */ + if (len < 0) + len = 0; + else + len = min_t(size_t, len, cell_size - 1); + + ret += len; + if (len < text_width) + ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%*s", + text_width - len, ""); + } + + return ret; +} + +/* + * cycles_percent_entry - Render cycles percentage column + */ +static int +cycles_percent_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct c2c_hist_entry *c2c_he; + int width = c2c_width(fmt, hpp, he->hists); + u64 fn_cycles, total_cycles; + char folded_sign; + double pct; + int ret, pct_width; + + /* Hide Cycles Percent for child functions and cachelines. */ + if (he->parent_he) + return scnprintf(hpp->buf, hpp->size, "%*s", width, ""); + + c2c_he = container_of(he, struct c2c_hist_entry, he); + fn_cycles = c2c_hist_entry__cycles(c2c_he); + /* Populated by c2c_function__build() once the L1 tree is built. */ + total_cycles = c2c_ext.total_cycles; + pct = total_cycles > 0 ? (double)fn_cycles / total_cycles * 100.0 : 0.0; + + /* Add folded sign only for level-1 entries */ + folded_sign = he->has_children ? (he->unfolded ? '-' : '+') : ' '; + ret = scnprintf(hpp->buf, hpp->size, "%c ", folded_sign); + + pct_width = width - ret; + if (pct_width <= 0) + return ret; + ret += scnprintf(hpp->buf + ret, hpp->size - ret, "%*.2f%%", pct_width - 1, pct); + return ret; +} + +/* + * cycles_percent_cmp - Comparison function for cycles percentage sorting + */ +static int64_t +cycles_percent_cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + struct c2c_hist_entry *c2c_left = container_of(left, struct c2c_hist_entry, he); + struct c2c_hist_entry *c2c_right = container_of(right, struct c2c_hist_entry, he); + u64 cycles_left, cycles_right; + + /* Cycles Percent is only shown for level-1 entries; others compare equal. */ + if (left->parent_he || right->parent_he) + return 0; + + cycles_left = c2c_hist_entry__cycles(c2c_left); + cycles_right = c2c_hist_entry__cycles(c2c_right); + + return (cycles_left > cycles_right) - (cycles_left < cycles_right); +} + +/* + * total_stores_cmp - Comparison function for total stores sorting + */ +static int64_t +total_stores_cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + u64 left_store = hist_entry__displayed_stores(left); + u64 right_store = hist_entry__displayed_stores(right); + + return (left_store > right_store) - (left_store < right_store); +} + +/* + * Function view dimensions + */ +static struct c2c_dimension dim_cycles_percent = { + .header = HEADER_BOTH("Cycles", "%"), + .name = "cycles_percent", + .cmp = cycles_percent_cmp, + .entry = cycles_percent_entry, + .width = 9, +}; + +static struct c2c_dimension dim_total_stores = { + .header = HEADER_BOTH("Store", "count"), + .name = "total_stores", + .cmp = total_stores_cmp, + .entry = total_stores_entry, + .width = 7, +}; + +static struct c2c_dimension dim_symbol_view = { + .header = HEADER_LOW("Function / Contending function / Cacheline"), + .name = "symbol_view", + .se = &sort_sym, + .entry = symbol_view_entry, + .width = SYMBOL_WIDTH, +}; + +static struct c2c_dimension *function_view_dimensions[] = { + &dim_cycles_percent, + &dim_total_stores, + &dim_symbol_view, + NULL, +}; + +static struct c2c_dimension *get_function_dimension(const char *name) +{ + unsigned int i; + + for (i = 0; function_view_dimensions[i]; i++) { + struct c2c_dimension *dim = function_view_dimensions[i]; + + if (!strcmp(dim->name, name)) + return dim; + } + + return NULL; +} + +/* Wrappers so sort_entry-backed dimensions sort/collapse via their se. */ +static int64_t c2c_se_cmp(struct perf_hpp_fmt *fmt, + struct hist_entry *a, struct hist_entry *b) +{ + struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); + struct c2c_dimension *dim = c2c_fmt->dim; + + return dim->se->se_cmp(a, b); +} + +static int64_t c2c_se_collapse(struct perf_hpp_fmt *fmt, + struct hist_entry *a, struct hist_entry *b) +{ + struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); + struct c2c_dimension *dim = c2c_fmt->dim; + int64_t (*collapse_fn)(struct hist_entry *a, struct hist_entry *b); + + collapse_fn = dim->se->se_collapse ?: dim->se->se_cmp; + return collapse_fn(a, b); +} + +static int64_t c2c_se_sort(struct perf_hpp_fmt *fmt, + struct hist_entry *a, struct hist_entry *b) +{ + struct c2c_fmt *c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); + struct c2c_dimension *dim = c2c_fmt->dim; + int64_t (*sort_fn)(struct hist_entry *a, struct hist_entry *b); + + sort_fn = dim->se->se_sort ?: dim->se->se_cmp; + return sort_fn(a, b); +} + +/* + * Build the c2c_fmt for @name. Returns: + * 0 and *fmtp set on success; + * -ENOENT and *fmtp = NULL if @name is not a function-view dimension; + * -ENOMEM if allocation failed (distinct from -ENOENT so + * the caller does not misreport it as an + * "invalid field"). + */ +static int get_function_format(const char *name, struct c2c_fmt **fmtp) +{ + struct c2c_dimension *dim = get_function_dimension(name); + struct c2c_fmt *c2c_fmt; + struct perf_hpp_fmt *fmt; + + *fmtp = NULL; + + if (!dim) + return -ENOENT; + + c2c_fmt = zalloc(sizeof(*c2c_fmt)); + if (!c2c_fmt) + return -ENOMEM; + + fmt = &c2c_fmt->fmt; + + c2c_fmt->dim = dim; + INIT_LIST_HEAD(&fmt->list); + INIT_LIST_HEAD(&fmt->sort_list); + + fmt->cmp = dim->se ? c2c_se_cmp : dim->cmp; + fmt->sort = dim->se ? c2c_se_sort : dim->cmp; + fmt->color = dim->color; + fmt->entry = dim->entry; + fmt->header = c2c_header; + fmt->width = c2c_width; + fmt->collapse = dim->se ? c2c_se_collapse : dim->cmp; + fmt->equal = c2c_fmt_equal; + fmt->free = c2c_fmt_free; + + *fmtp = c2c_fmt; + return 0; +} + +static int +c2c_function_hists__init_output(struct perf_hpp_list *hpp_list, char *name, + struct perf_env *env __maybe_unused) +{ + struct c2c_fmt *c2c_fmt; + int ret; + + ret = get_function_format(name, &c2c_fmt); + if (ret == -ENOMEM) + return ret; + /* The function view only accepts its own dimensions. */ + if (ret == -ENOENT) + return -EINVAL; + + /* + * Mark symbol-backed columns so hists__has(hists, sym) is correct. + * Only dim_symbol_view carries a sort_entry (.se); the function + * view's field strings are fixed and always include symbol_view, so + * this single check is sufficient (unlike the user-configurable + * cacheline view, which must also test dim_iaddr). + */ + if (c2c_fmt->dim->se == &sort_sym) + hpp_list->sym = 1; + + perf_hpp_list__column_register(hpp_list, &c2c_fmt->fmt); + return 0; +} + +static int +c2c_function_hists__init_sort(struct perf_hpp_list *hpp_list, char *name, + struct perf_env *env __maybe_unused) +{ + struct c2c_fmt *c2c_fmt; + int ret; + + ret = get_function_format(name, &c2c_fmt); + if (ret == -ENOMEM) + return ret; + /* The function view only accepts its own dimensions. */ + if (ret == -ENOENT) + return -EINVAL; + + /* Mark symbol-backed sort keys so hists__has(hists, sym) is correct. */ + if (c2c_fmt->dim->se == &sort_sym) + hpp_list->sym = 1; + + perf_hpp_list__register_sort_field(hpp_list, &c2c_fmt->fmt); + return 0; +} + +typedef int (*hpp_list_add_fn)(struct perf_hpp_list *hpp_list, char *name, + struct perf_env *env); + +static int function_hpp_list__add_tokens(struct perf_hpp_list *hpp_list, char *list, + struct perf_env *env, hpp_list_add_fn add) +{ + char *tok, *tmp; + int ret; + + if (!list) + return 0; + + for (tok = strtok_r(list, ", ", &tmp); tok; tok = strtok_r(NULL, ", ", &tmp)) { + ret = add(hpp_list, tok, env); + if (ret) { + if (ret == -EINVAL || ret == -ESRCH) + pr_err("Invalid c2c function-view field: %s\n", tok); + return ret; + } + } + return 0; +} + +/* + * Append the function view's sort keys to its own output fields, mirroring + * perf_hpp__setup_output_field() but on the local @list. The shared helper + * registers onto the global perf_hpp_list, which would leave this local list + * without output columns, so the function view keeps its own copy here. + */ +static void c2c_function_hists__setup_output_field(struct perf_hpp_list *list) +{ + struct perf_hpp_fmt *fmt; + + perf_hpp_list__for_each_sort_list(list, fmt) { + struct perf_hpp_fmt *pos; + + if (!fmt->entry && !fmt->color) + continue; + + perf_hpp_list__for_each_format(list, pos) { + if (c2c_fmt_equal(fmt, pos)) + goto next; + } + + perf_hpp_list__column_register(list, fmt); +next: + continue; + } +} + +static int +function_hpp_list__parse(struct perf_hpp_list *hpp_list, + const char *output_str, + const char *sort_str, + struct perf_env *env) +{ + char *output = output_str ? strdup(output_str) : NULL; + char *sort = sort_str ? strdup(sort_str) : NULL; + int ret = 0; + + if ((output_str && !output) || (sort_str && !sort)) { + ret = -ENOMEM; + goto out; + } + + ret = function_hpp_list__add_tokens(hpp_list, output, env, + c2c_function_hists__init_output); + if (ret) + goto out; + + ret = function_hpp_list__add_tokens(hpp_list, sort, env, + c2c_function_hists__init_sort); + if (ret) + goto out; + + c2c_function_hists__setup_output_field(hpp_list); +out: + if (ret) + perf_hpp__reset_output_field(hpp_list); + free(output); + free(sort); + return ret; +} + +static int +c2c_function_hists__init(struct c2c_hists *hists, + const char *sort, + int nr_header_lines, + struct perf_env *env) +{ + __hists__init(&hists->hists, &hists->list); + + perf_hpp_list__init(&hists->list); + + hists->list.nr_header_lines = nr_header_lines; + + return function_hpp_list__parse(&hists->list, /*output=*/NULL, sort, env); +} + +static int +c2c_function_hists__reinit(struct c2c_hists *c2c_hists, + const char *output, + const char *sort, + struct perf_env *env) +{ + int nr_header_lines = c2c_hists->list.nr_header_lines; + + perf_hpp__reset_output_field(&c2c_hists->list); + + /* Clear stale state flags so a different output/sort set starts fresh. */ + c2c_hists->list.need_collapse = 0; + c2c_hists->list.parent = 0; + c2c_hists->list.sym = 0; + c2c_hists->list.dso = 0; + c2c_hists->list.socket = 0; + c2c_hists->list.thread = 0; + c2c_hists->list.comm = 0; + c2c_hists->list.comm_nodigit = 0; + c2c_hists->list.nr_header_lines = nr_header_lines; + + return function_hpp_list__parse(&c2c_hists->list, output, sort, env); +} + +/* Welford online merge of two "stats" (from util/stat.h) accumulators. */ +static void c2c_stats_merge(struct stats *dest, const struct stats *src) +{ + double delta; + + if (src->n == 0) + return; + + if (dest->n == 0) { + *dest = *src; + return; + } + + delta = src->mean - dest->mean; + dest->M2 += src->M2 + delta * delta * dest->n * src->n / (dest->n + src->n); + dest->mean = (dest->mean * dest->n + src->mean * src->n) / (dest->n + src->n); + dest->n += src->n; + + /* Update min/max */ + if (src->max > dest->max) + dest->max = src->max; + if (src->min < dest->min) + dest->min = src->min; +} + +/* Merge compute_stats during function aggregation. */ +static void c2c_add_cstats(struct compute_stats *dest, + const struct compute_stats *src) +{ + c2c_stats_merge(&dest->rmt_hitm, &src->rmt_hitm); + c2c_stats_merge(&dest->lcl_hitm, &src->lcl_hitm); + c2c_stats_merge(&dest->rmt_peer, &src->rmt_peer); + c2c_stats_merge(&dest->lcl_peer, &src->lcl_peer); + c2c_stats_merge(&dest->load, &src->load); +} + +static bool hist_entry__add_c2c_stats(struct hist_entry *he, + const struct c2c_stats *stats) +{ + u64 nr_events = c2c_hitm_count(stats) + stats->rmt_peer + stats->lcl_peer; + u64 weight1 = c2c_hitm_count(stats); + + /* + * Allocate before touching he->stat, so a failure here leaves the + * entry unmodified and the caller can bail out without having + * half-updated the statistics. + */ + if (symbol_conf.cumulate_callchain && !he->stat_acc) { + he->stat_acc = calloc(1, sizeof(struct he_stat)); + if (!he->stat_acc) + return false; + } + + he->stat.nr_events += nr_events; + he->stat.period += nr_events; + he->stat.weight1 += weight1; + + if (!symbol_conf.cumulate_callchain) + return true; + + he->stat_acc->nr_events += nr_events; + he->stat_acc->period += nr_events; + he->stat_acc->weight1 += weight1; + + return true; +} + +static void c2c_he__free_hierarchy(struct hist_entry *he); + +/* + * Free a function-view histogram entry (hist_entry_ops::free). + */ +static void c2c_function_he_free(void *ptr) +{ + struct hist_entry *he = ptr; + struct c2c_hist_entry *c2c_he; + + c2c_he = container_of(he, struct c2c_hist_entry, he); + + if (c2c_he->hists) { + perf_hpp__reset_output_field(&c2c_he->hists->list); + hists__delete_all_entries(&c2c_he->hists->hists); + zfree(&c2c_he->hists); + } + + c2c_he__free_hierarchy(he); + + free(c2c_he); +} + +static void c2c_he__free_hierarchy(struct hist_entry *he) +{ + struct rb_node *nd; + struct hist_entry *child_he; + + /* + * A leaf entry stores its callchains in the sorted_chain member, which + * shares a union with the hroot_in/hroot_out child trees, so its + * hroot_out is not a valid subtree to walk. Leaf entries never have a + * child hierarchy here, so stop before touching hroot_out. + */ + if (he->leaf) + return; + + if (RB_EMPTY_ROOT(&he->hroot_out.rb_root)) + return; + + nd = rb_first_cached(&he->hroot_out); + while (nd) { + struct rb_node *next = rb_next(nd); + + child_he = rb_entry(nd, struct hist_entry, rb_node); + rb_erase_cached(&child_he->rb_node, &he->hroot_out); + hist_entry__delete(child_he); + + nd = next; + } + + /* All children erased; clear the tree (and its cached leftmost). */ + he->hroot_out = RB_ROOT_CACHED; +} + +/* + * Drop level-2 writing functions that carry no stores or + * no cacheline children. Writers are only added when they store into a shared + * line, so this is mainly a safety net. Returns the number of surviving + * writers. + */ +static int c2c_he__prune_empty_writers(struct hist_entry *l1_he) +{ + struct rb_node *nd; + int surviving = 0; + + if (!l1_he->has_children) + return 0; + + nd = rb_first_cached(&l1_he->hroot_out); + while (nd) { + struct rb_node *next = rb_next(nd); + struct hist_entry *l2_he = rb_entry(nd, struct hist_entry, rb_node); + + if (l2_he->has_children && hist_entry__displayed_stores(l2_he) > 0) { + surviving++; + } else { + rb_erase_cached(&l2_he->rb_node, &l1_he->hroot_out); + hist_entry__delete(l2_he); + } + nd = next; + } + + if (!surviving) { + l1_he->hroot_out = RB_ROOT_CACHED; + l1_he->has_children = false; + l1_he->unfolded = false; + } + return surviving; +} + +static void *c2c_function_he_zalloc(size_t size) +{ + struct c2c_hist_entry *c2c_he = zalloc(sizeof(*c2c_he) + size); + + if (!c2c_he) + return NULL; + + init_stats(&c2c_he->cstats.lcl_hitm); + init_stats(&c2c_he->cstats.rmt_hitm); + init_stats(&c2c_he->cstats.lcl_peer); + init_stats(&c2c_he->cstats.rmt_peer); + init_stats(&c2c_he->cstats.load); + + return &c2c_he->he; +} + +/* Entry operations for function view */ +static struct hist_entry_ops c2c_function_entry_ops = { + .new = c2c_function_he_zalloc, + .free = c2c_function_he_free, +}; + +static struct c2c_hist_entry * +c2c_child_entry__alloc(struct hist_entry *parent_he, struct hist_entry *src_he, + int depth, u64 ip) +{ + struct c2c_hist_entry *child_c2c; + struct hist_entry *child_he; + + /* Function-view children never own or display callchains. */ + child_he = c2c_function_he_zalloc(0); + if (!child_he) + return NULL; + + child_c2c = container_of(child_he, struct c2c_hist_entry, he); + child_he->ops = &c2c_function_entry_ops; + map_symbol__copy(&child_he->ms, &src_he->ms); + + if (src_he->mem_info) { + child_he->mem_info = mem_info__clone(src_he->mem_info); + if (!child_he->mem_info) + goto out_free; + } + + child_he->thread = thread__get(src_he->thread); + child_he->cpumode = src_he->cpumode; + child_he->cpu = src_he->cpu; + child_he->socket = src_he->socket; + child_he->level = src_he->level; + child_he->ip = ip; + + child_he->parent_he = parent_he; + child_he->depth = depth; + child_he->leaf = (depth >= 2); + child_he->hists = &c2c_ext.function_hists.hists; + child_he->filtered = false; + child_he->unfolded = false; + child_he->has_children = false; + child_he->has_no_entry = false; + child_he->nr_rows = 0; + child_he->row_offset = 0; + + memset(&child_he->stat, 0, sizeof(child_he->stat)); + child_he->hroot_in = RB_ROOT_CACHED; + child_he->hroot_out = RB_ROOT_CACHED; + INIT_LIST_HEAD(&child_he->pairs.node); + child_he->hpp_list = &c2c_ext.function_hists.list; + if (symbol_conf.cumulate_callchain) { + child_he->stat_acc = calloc(1, sizeof(struct he_stat)); + if (!child_he->stat_acc) + goto out_free; + } + + return child_c2c; + +out_free: + hist_entry__delete(child_he); + return NULL; +} + +static void +c2c_child_entry__insert(struct hist_entry *parent_he, struct hist_entry *child_he, + struct rb_node **p, struct rb_node *rb_parent, bool leftmost) +{ + rb_link_node(&child_he->rb_node, rb_parent, p); + rb_insert_color_cached(&child_he->rb_node, &parent_he->hroot_out, leftmost); + + parent_he->has_children = true; + parent_he->leaf = false; +} + +static struct hist_entry * +c2c_function_hists__level1_entry(struct symbol *sym, + struct hist_entry *detail_he, + struct thread *synthetic_thread) +{ + struct addr_location al; + struct perf_sample sample = {}; + struct mem_info *mi; + struct hist_entry *he; + /* + * Key the level-1 entry by the function, not by a specific code + * address: use the symbol start so every instruction address inside + * the same function collapses into one entry. This makes level 1 a + * true "function view" rather than a per-code-address view. + */ + u64 sym_start = (sym && detail_he->ms.map) ? + map__unmap_ip(detail_he->ms.map, sym->start) : detail_he->ip; + + mi = mem_info__new(); + if (!mi) + return NULL; + + mem_info__iaddr(mi)->addr = sym_start; + /* mem_info__put() will map_symbol__exit() these, so take refs. */ + mem_info__iaddr(mi)->ms.thread = thread__get(detail_he->ms.thread); + mem_info__iaddr(mi)->ms.map = map__get(detail_he->ms.map); + mem_info__iaddr(mi)->ms.sym = sym; + mem_info__daddr(mi)->addr = 0; + + addr_location__init(&al); + al.thread = thread__get(synthetic_thread); + al.map = map__get(detail_he->ms.map); + al.sym = sym; + al.addr = sym_start; + al.level = detail_he->level; + al.cpumode = detail_he->cpumode; + al.cpu = 0; + al.socket = 0; + al.filtered = 0; + al.latency = 0; + + /* + * Synthetic sample: period/weight are placeholders only. The real + * c2c counters live in c2c_hist_entry::stats and are added via + * hist_entry__add_c2c_stats(); no function-view column or sort key + * reads he->stat.period/nr_events, so the +1 that __hists__add_entry() + * accrues on each dedup hit has no effect on what is displayed. + */ + sample.period = 1; + sample.weight = 1; + sample.ip = sym_start; + sample.pid = thread__pid(synthetic_thread); + sample.tid = thread__tid(synthetic_thread); + sample.cpu = 0; + + /* Add entry - histogram handles dedup */ + he = hists__add_entry_ops(&c2c_ext.function_hists.hists, + &c2c_function_entry_ops, + &al, NULL, NULL, mi, + NULL, &sample, true); + + addr_location__exit(&al); + mem_info__put(mi); + + if (he) + he->hpp_list = &c2c_ext.function_hists.list; + + return he; +} + +/* + * Level 2: a function that writes a cacheline the level-1 function reads, + * keyed by the DSO display name and symbol, consistently with perf's symbol + * sort semantics. All code addresses and cachelines for the same writer + * function aggregate into one row. + */ +static struct c2c_hist_entry * +c2c_function_hists__level2_entry(struct c2c_hist_entry *level1_c2c, + struct symbol *sym, struct hist_entry *detail_he) +{ + struct hist_entry *level1_he = &level1_c2c->he; + struct rb_node **p = &level1_he->hroot_out.rb_root.rb_node; + struct rb_node *parent = NULL; + struct c2c_hist_entry *level2_c2c; + bool leftmost = true; + + while (*p) { + struct hist_entry *iter = rb_entry(*p, struct hist_entry, rb_node); + struct map_symbol key = detail_he->ms; + int64_t cmp; + + key.sym = sym; + parent = *p; + cmp = c2c_function_cmp(&key, &iter->ms); + + if (cmp < 0) { + p = &parent->rb_left; + } else if (cmp > 0) { + p = &parent->rb_right; + leftmost = false; + } else { + return container_of(iter, struct c2c_hist_entry, he); + } + } + + /* Key by the function symbol start so all code addresses collapse. */ + level2_c2c = c2c_child_entry__alloc(level1_he, detail_he, 1, + (sym && detail_he->ms.map) ? + map__unmap_ip(detail_he->ms.map, sym->start) : + hist_entry__iaddr(detail_he)); + if (!level2_c2c) + return NULL; + + /* Key this level by the looked-up symbol, not detail_he's. */ + level2_c2c->he.ms.sym = sym; + if (level2_c2c->he.mem_info) + mem_info__iaddr(level2_c2c->he.mem_info)->ms.sym = sym; + + c2c_child_entry__insert(level1_he, &level2_c2c->he, p, parent, leftmost); + + return level2_c2c; +} + +/* Level 3: one source cacheline where the L1/L2 functions contend. */ +static struct c2c_hist_entry * +c2c_function_hists__level3_entry(struct c2c_hist_entry *level2_c2c, + struct c2c_hist_entry *cacheline_src_he) +{ + struct hist_entry *level2_he = &level2_c2c->he; + struct rb_node **p = &level2_he->hroot_out.rb_root.rb_node; + struct rb_node *parent = NULL; + struct c2c_hist_entry *level3_c2c; + bool leftmost = true; + + while (*p) { + struct c2c_hist_entry *iter_c2c = + rb_entry(*p, struct c2c_hist_entry, he.rb_node); + + parent = *p; + if (cacheline_src_he->cacheline_idx < iter_c2c->cacheline_idx) { + p = &parent->rb_left; + } else if (cacheline_src_he->cacheline_idx > iter_c2c->cacheline_idx) { + p = &parent->rb_right; + leftmost = false; + } else { + return iter_c2c; + } + } + + level3_c2c = c2c_child_entry__alloc(level2_he, &cacheline_src_he->he, 2, + hist_entry__iaddr(&cacheline_src_he->he)); + if (!level3_c2c) + return NULL; + level3_c2c->cacheline_idx = cacheline_src_he->cacheline_idx; + + c2c_child_entry__insert(level2_he, &level3_c2c->he, p, parent, leftmost); + + return level3_c2c; +} + +struct hist_entry *c2c_function__find_cacheline(struct hist_entry *he_selection) +{ + struct c2c_hist_entry *c2c_he; + struct rb_node *nd; + + if (!c2c_ext.cl_hists || !he_selection || !he_selection->parent_he || + !he_selection->parent_he->parent_he) + return NULL; + + c2c_he = container_of(he_selection, struct c2c_hist_entry, he); + + for (nd = rb_first_cached(&c2c_ext.cl_hists->hists.entries); nd; + nd = rb_next(nd)) { + struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); + struct c2c_hist_entry *cacheline_he; + + if (he->filtered) + continue; + + cacheline_he = container_of(he, struct c2c_hist_entry, he); + if (cacheline_he->hists && + cacheline_he->cacheline_idx == c2c_he->cacheline_idx) + return he; + } + + return NULL; +} + +/* + * Re-sort child entries of @parent_he by total store count, descending. + */ +static void c2c_he__resort_by_stores(struct hist_entry *parent_he) +{ + struct rb_root_cached new_root = RB_ROOT_CACHED; + struct rb_node *nd; + + if (!parent_he->has_children) + return; + + /* Extract all nodes and re-insert sorted by displayed store count */ + while ((nd = rb_first_cached(&parent_he->hroot_out))) { + struct hist_entry *he = rb_entry(nd, struct hist_entry, rb_node); + u64 he_store = hist_entry__displayed_stores(he); + struct rb_node **p = &new_root.rb_root.rb_node; + struct rb_node *parent = NULL; + bool leftmost = true; + int cmp; + + /* Remove from current tree */ + rb_erase_cached(&he->rb_node, &parent_he->hroot_out); + + /* + * Insert sorted by store count, descending. Use the displayed + * store count so a level-1 function and level-2 writer (whose own + * stats.store is 0 / partial) sort by the aggregated write traffic + * beneath them, not by their own store field. + */ + while (*p) { + struct hist_entry *iter = rb_entry(*p, struct hist_entry, rb_node); + u64 iter_store = hist_entry__displayed_stores(iter); + + parent = *p; + if (he_store != iter_store) { + cmp = he_store > iter_store ? -1 : 1; + } else { + /* Stable tie-break: instruction address, name, then cacheline. */ + u64 a = hist_entry__iaddr(he), b = hist_entry__iaddr(iter); + + if (a != b) + cmp = a < b ? -1 : 1; + else if (he->ms.sym && iter->ms.sym) + cmp = strcmp(he->ms.sym->name, + iter->ms.sym->name); + else + cmp = (iter->ms.sym ? 1 : 0) - (he->ms.sym ? 1 : 0); + + if (!cmp) { + struct c2c_hist_entry *he_c2c; + struct c2c_hist_entry *iter_c2c; + + he_c2c = container_of(he, struct c2c_hist_entry, he); + iter_c2c = container_of(iter, struct c2c_hist_entry, he); + if (he_c2c->cacheline_idx != iter_c2c->cacheline_idx) + cmp = he_c2c->cacheline_idx < + iter_c2c->cacheline_idx ? -1 : 1; + } + } + + if (cmp < 0) { + p = &parent->rb_left; + } else { + p = &parent->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, &new_root, leftmost); + } + + parent_he->hroot_out = new_root; +} + +/* Initial per-cacheline capacity for the seen[] set; grown on demand. */ +#define DEFAULT_SYMBOLS_PER_CL 64 + +struct function_seen { + struct map_symbol ms; +}; + +static bool function_seen__find(const struct function_seen *seen, int nr, + const struct map_symbol *ms) +{ + int i; + + for (i = 0; i < nr; i++) { + if (!c2c_function_cmp(&seen[i].ms, ms)) + return true; + } + return false; +} + +/* Aggregate stats from the cacheline-side entry @c2c_b into level 2/3 @dst. */ +static bool c2c_he__add_sharing(struct c2c_hist_entry *dst, struct c2c_hist_entry *src) +{ + /* Do the fallible update first so a failure leaves dst unmodified. */ + if (!hist_entry__add_c2c_stats(&dst->he, &src->stats)) + return false; + + c2c_add_stats(&dst->stats, &src->stats); + c2c_add_cstats(&dst->cstats, &src->cstats); + return true; +} + +/* + * Process one cacheline: for every function reading it, create/update its + * level-1 function entry, then for each function that writes the line + * add it as a level-2 writer and add this cacheline as a level-3 child. + */ +static int c2c_function__process_cl(struct c2c_hist_entry *cacheline_he, + struct thread *synthetic_thread) +{ + struct rb_node *nd_a, *nd_b; + struct function_seen *seen = NULL; + int nr_seen = 0, nr_alloc = 0; + int ret = 0; + + for (nd_a = rb_first_cached(&cacheline_he->hists->hists.entries); nd_a; + nd_a = rb_next(nd_a)) { + struct hist_entry *he_a = rb_entry(nd_a, struct hist_entry, rb_node); + struct c2c_hist_entry *c2c_a; + struct hist_entry *level1_he; + struct c2c_hist_entry *level1_c2c; + + if (!he_a->ms.sym || he_a->filtered) + continue; + + c2c_a = container_of(he_a, struct c2c_hist_entry, he); + if (c2c_a->stats.load == 0) + continue; + + level1_he = c2c_function_hists__level1_entry(he_a->ms.sym, + he_a, synthetic_thread); + if (!level1_he) { + ret = -ENOMEM; + goto out; + } + + level1_c2c = container_of(level1_he, struct c2c_hist_entry, he); + + /* + * Aggregate every source entry into its level-1 function parent. + * level1_he is keyed by symbol, so all code addresses inside the + * same function collapse into one parent. When the cacheline view + * splits a function into siblings (different code addresses, or + * --coalesce pid/tid/dso), each sibling holds a DISJOINT slice of the + * traffic, so summing them here is correct accumulation, not + * double counting. The seen[] set below therefore guards only the + * subtree build (to avoid building a function's level-2/3 subtree + * twice for the same cacheline), never this L1 update. Update + * he->stat first; on failure leave the aggregates untouched. + */ + if (!hist_entry__add_c2c_stats(level1_he, &c2c_a->stats)) { + ret = -ENOMEM; + goto out; + } + c2c_add_stats(&level1_c2c->stats, &c2c_a->stats); + c2c_add_cstats(&level1_c2c->cstats, &c2c_a->cstats); + c2c_add_stats(&c2c_ext.function_hists.stats, &c2c_a->stats); + + /* Build this function's subtree for this cacheline only once. */ + if (function_seen__find(seen, nr_seen, &he_a->ms)) + continue; + + if (nr_seen == nr_alloc) { + struct function_seen *tmp; + int new_alloc = nr_alloc ? nr_alloc * 2 : DEFAULT_SYMBOLS_PER_CL; + + tmp = reallocarray(seen, new_alloc, sizeof(*seen)); + if (!tmp) { + ret = -ENOMEM; + goto out; + } + seen = tmp; + nr_alloc = new_alloc; + } + seen[nr_seen].ms = he_a->ms; + nr_seen++; + + for (nd_b = rb_first_cached(&cacheline_he->hists->hists.entries); nd_b; + nd_b = rb_next(nd_b)) { + struct hist_entry *he_b = rb_entry(nd_b, struct hist_entry, rb_node); + struct c2c_hist_entry *c2c_b, *level2_c2c, *level3_c2c; + + if (!he_b->ms.sym || he_b->filtered) + continue; + + c2c_b = container_of(he_b, struct c2c_hist_entry, he); + + /* + * The level-1 function contributes read-side load weight for this + * cacheline. Associate it with functions sampled storing to the + * same line. + * The writer can be the same function; after detail coalescing and + * function-level grouping there is not enough identity to attribute + * that case to a specific thread. + * Only writers are contending functions, so keep the ones + * that actually store into the line. + */ + if (c2c_b->stats.store == 0) + continue; + + /* Level 2: the writing function (aggregated across cachelines). */ + level2_c2c = c2c_function_hists__level2_entry(level1_c2c, he_b->ms.sym, + he_b); + if (!level2_c2c || !c2c_he__add_sharing(level2_c2c, c2c_b)) { + ret = -ENOMEM; + goto out; + } + + /* Level 3: the specific cacheline they contend over. */ + level3_c2c = c2c_function_hists__level3_entry(level2_c2c, + cacheline_he); + if (!level3_c2c || !c2c_he__add_sharing(level3_c2c, c2c_b)) { + ret = -ENOMEM; + goto out; + } + } + } + +out: + free(seen); + return ret; +} + +/* + * Remove a level-1 function that has no contended cachelines left. It is a + * normal (owned) hist_entry in function_hists, so mirror hists__delete_entry() + * for the no-collapse case: unlink from both trees, fix the counters, then + * delete. Its hroot_out is already empty after pruning. + */ +static void c2c_function__drop_level1(struct hist_entry *he) +{ + struct hists *hists = &c2c_ext.function_hists.hists; + + rb_erase_cached(&he->rb_node_in, hists->entries_in); + rb_erase_cached(&he->rb_node, &hists->entries); + + --hists->nr_entries; + if (!he->filtered) + --hists->nr_non_filtered_entries; + + hist_entry__delete(he); +} + +/* Length of the identity text (symbol name or cacheline address) at @he. */ +static int c2c_function__ident_len(struct hist_entry *he) +{ + char buf[512]; + char *symbuf; + size_t size; + int len; + + if (hist_entry__is_cacheline(he)) { + u64 addr = he->mem_info ? + cl_address(mem_info__daddr(he->mem_info)->addr, chk_double_cl) : 0; + + return scnprintf(buf, sizeof(buf), "0x%" PRIx64, addr); + } + + if (!he->ms.sym) + return 0; + + /* + * Match symbol_view_entry(): sort_sym adds the cpumode prefix and, in + * verbose mode, the address and DSO origin before the symbol name. + */ + size = strlen(he->ms.sym->name) + 64; + symbuf = malloc(size); + if (!symbuf) + return size - 1; + + len = sort_sym.se_snprintf(he, symbuf, size, size - 1); + free(symbuf); + return len; +} + +/* + * Grow the symbol column so the deepest, longest identity cell fits. The + * generic hists__calc_col_len() only measures the top-level (L1) entries; the + * hand-linked L2 writers and L3 cacheline addresses live in hroot_out and are + * never measured, so with a short L1 name the indented L2/L3 text would be + * truncated. Account for the per-level indent and the folded-sign prefix. + */ +static void c2c_function__update_symbol_width(struct hist_entry *he) +{ + struct hists *hists = &c2c_ext.function_hists.hists; + int need = he->depth * C2C_FUNC_INDENT + C2C_FUNC_FOLD_WIDTH + + c2c_function__ident_len(he); + + if (need > hists__col_len(hists, HISTC_SYMBOL)) + hists__set_col_len(hists, HISTC_SYMBOL, need); +} + +/* + * Prune writers with no stores, drop functions left with no contending + * writer, sort the survivors by store count, then compute the global total. + */ +static void c2c_function__finalize(void) +{ + struct rb_node *nd_l1; + + nd_l1 = rb_first_cached(&c2c_ext.function_hists.hists.entries); + while (nd_l1) { + struct hist_entry *he_l1 = rb_entry(nd_l1, struct hist_entry, rb_node); + struct rb_node *next_l1 = rb_next(nd_l1); + struct rb_node *nd_l2; + + /* Drop writers with no stores before sorting. */ + if (!he_l1->has_children || !c2c_he__prune_empty_writers(he_l1)) { + /* No contending writer: this function is not shared. */ + c2c_function__drop_level1(he_l1); + nd_l1 = next_l1; + continue; + } + + c2c_he__resort_by_stores(he_l1); + c2c_function__update_symbol_width(he_l1); + + for (nd_l2 = rb_first_cached(&he_l1->hroot_out); nd_l2; + nd_l2 = rb_next(nd_l2)) { + struct hist_entry *he_l2 = rb_entry(nd_l2, struct hist_entry, rb_node); + struct rb_node *nd_l3; + + c2c_function__update_symbol_width(he_l2); + + if (he_l2->has_children) + c2c_he__resort_by_stores(he_l2); + + for (nd_l3 = rb_first_cached(&he_l2->hroot_out); nd_l3; + nd_l3 = rb_next(nd_l3)) { + struct hist_entry *he_l3 = rb_entry(nd_l3, struct hist_entry, + rb_node); + + c2c_function__update_symbol_width(he_l3); + } + } + + nd_l1 = next_l1; + } + + /* + * Compute the Cycles % denominator from the surviving level-1 entries + * after pruning, so the column shows each function's share of the + * functions retained in this table -- not of the whole recording. See + * the Cycles % description in perf-c2c.txt. + */ + c2c_ext.total_cycles = c2c_ext__total_cycles(); +} + +/* + * Release all per-visit function-view state. Keep the hists object itself + * initialized so its mutex is initialized exactly once across TAB re-entry. + */ +void c2c_function__reset(void) +{ + bool saved_use_callchain = symbol_conf.use_callchain; + + /* + * Function-view entries never carry callchains. Keep their generic + * destructor independent of the caller's current callchain setting. + */ + symbol_conf.use_callchain = false; + hists__delete_all_entries(&c2c_ext.function_hists.hists); + if (c2c_ext.function_hists.list.fields.next) + perf_hpp__reset_output_field(&c2c_ext.function_hists.list); + + memset(&c2c_ext.function_hists.stats, 0, + sizeof(c2c_ext.function_hists.stats)); + c2c_ext.total_cycles = 0; + c2c_ext.cl_hists = NULL; + c2c_ext.cl_sort = NULL; + c2c_ext.symbol_full = false; + symbol_conf.use_callchain = saved_use_callchain; +} + +static bool c2c_function__has_iaddr(const char *cl_sort) +{ + const char *field = cl_sort; + + while (field && *field) { + const char *end = strchr(field, ','); + size_t len = end ? (size_t)(end - field) : strlen(field); + + if (len == sizeof("iaddr") - 1 && !strncmp(field, "iaddr", len)) + return true; + field = end ? end + 1 : NULL; + } + return false; +} + +/* + * Build the three-level function view in a single pass over the cacheline + * entries: + * L1: read-side functions (aggregated across all their code addresses) + * L2: writing functions contending with each level-1 function + * L3: shared cachelines for each function pair + */ +int c2c_function__build(struct c2c_hists *cl_hists, const char *cl_sort, + bool symbol_full, struct hists **hists) +{ + static const char output_fields[] = + "cycles_percent,total_stores,symbol_view"; + static bool hists_initialized; + struct rb_node *nd_cl; + bool saved_use_callchain; + int ret; + + if (!hists) + return -EINVAL; + *hists = NULL; + + if (!cl_hists || !cl_sort) + return -EINVAL; + if (!c2c_function__has_iaddr(cl_sort)) + return -EOPNOTSUPP; + + saved_use_callchain = symbol_conf.use_callchain; + symbol_conf.use_callchain = false; + c2c_function__reset(); + + c2c_ext.cl_hists = cl_hists; + c2c_ext.cl_sort = cl_sort; + c2c_ext.symbol_full = symbol_full; + + /* + * __hists__init() (called by c2c_function_hists__init()) also + * mutex_init()s the hists lock, so only run it once for this static + * hists; on re-entry just re-parse the columns via reinit(). + */ + if (!hists_initialized) { + ret = c2c_function_hists__init(&c2c_ext.function_hists, + "symbol_view", 2, NULL); + hists_initialized = true; + } else { + ret = c2c_function_hists__reinit(&c2c_ext.function_hists, + /*output=*/NULL, "symbol_view", NULL); + } + if (ret) + goto out_reset; + + nd_cl = rb_first_cached(&c2c_ext.cl_hists->hists.entries); + + /* An empty C2C report yields an empty (but valid) function view. */ + for (; nd_cl; nd_cl = rb_next(nd_cl)) { + struct hist_entry *he_cl = rb_entry(nd_cl, struct hist_entry, rb_node); + struct c2c_hist_entry *cacheline_he = container_of(he_cl, + struct c2c_hist_entry, he); + struct thread *synthetic_thread = he_cl->thread; + + /* + * Detail hists are finalized only for cachelines retained by the + * top-level C2C filter. Among those, include any line with sharing + * activity, not just HITM. + */ + if (he_cl->filtered || + (c2c_hitm_count(&cacheline_he->stats) == 0 && + cacheline_he->stats.tot_peer == 0 && + cacheline_he->stats.store == 0 && + cacheline_he->stats.load == 0) || + !cacheline_he->hists || + RB_EMPTY_ROOT(&cacheline_he->hists->hists.entries.rb_root) || + !he_cl->mem_info || !synthetic_thread) + continue; + + ret = c2c_function__process_cl(cacheline_he, synthetic_thread); + if (ret) + goto out_err; + } + + ret = c2c_function_hists__reinit(&c2c_ext.function_hists, output_fields, + "cycles_percent", NULL); + if (ret) + goto out_err; + + hists__collapse_resort(&c2c_ext.function_hists.hists, NULL); + hists__output_resort(&c2c_ext.function_hists.hists, NULL); + + c2c_function__finalize(); + + *hists = &c2c_ext.function_hists.hists; + symbol_conf.use_callchain = saved_use_callchain; + return 0; + +out_err: + /* + * On error, migrate any entries still in entries_in to entries and + * delete them, so a later rebuild does not strand them (the top-level + * __hists__init() memset would otherwise lose the pointers). + */ + hists__collapse_resort(&c2c_ext.function_hists.hists, NULL); + hists__output_resort(&c2c_ext.function_hists.hists, NULL); +out_reset: + c2c_function__reset(); + symbol_conf.use_callchain = saved_use_callchain; + return ret; +} diff --git a/tools/perf/util/c2c.c b/tools/perf/util/c2c.c new file mode 100644 index 000000000..58c7342df --- /dev/null +++ b/tools/perf/util/c2c.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "hist.h" +#include "c2c.h" + +void c2c_fmt_free(struct perf_hpp_fmt *fmt) +{ + struct c2c_fmt *c2c_fmt; + + c2c_fmt = container_of(fmt, struct c2c_fmt, fmt); + free(c2c_fmt); +} + +bool c2c_fmt_equal(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b) +{ + struct c2c_fmt *c2c_a = container_of(a, struct c2c_fmt, fmt); + struct c2c_fmt *c2c_b = container_of(b, struct c2c_fmt, fmt); + + return c2c_a->dim == c2c_b->dim; +} diff --git a/tools/perf/util/c2c.h b/tools/perf/util/c2c.h new file mode 100644 index 000000000..53f024e25 --- /dev/null +++ b/tools/perf/util/c2c.h @@ -0,0 +1,135 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_UTIL_C2C_H +#define __PERF_UTIL_C2C_H + +#include +#include +#include +#include "hist.h" +#include "mem-events.h" +#include "stat.h" + +struct sort_entry; + +struct c2c_hists { + struct hists hists; + struct perf_hpp_list list; + struct c2c_stats stats; +}; + +struct compute_stats { + struct stats lcl_hitm; + struct stats rmt_hitm; + struct stats lcl_peer; + struct stats rmt_peer; + struct stats load; +}; + +struct c2c_hist_entry { + struct c2c_hists *hists; + struct evsel *evsel; + struct c2c_stats stats; + unsigned long *cpuset; + unsigned long *nodeset; + struct c2c_stats *node_stats; + unsigned int cacheline_idx; + + struct compute_stats cstats; + + unsigned long paddr; + unsigned long paddr_cnt; + bool paddr_zero; + char *nodestr; + + /* + * must be at the end, + * because of its callchain dynamic entry + */ + struct hist_entry he; +}; + +#define C2C_HEADER_MAX 2 + +struct c2c_header { + struct { + const char *text; + int span; + } line[C2C_HEADER_MAX]; +}; + +struct c2c_dimension { + struct c2c_header header; + const char *name; + int width; + struct sort_entry *se; + + int64_t (*cmp)(struct perf_hpp_fmt *fmt, + struct hist_entry *left, struct hist_entry *right); + int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he); + int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he); +}; + +struct c2c_fmt { + struct perf_hpp_fmt fmt; + struct c2c_dimension *dim; +}; + +#define SYMBOL_WIDTH 30 + +#define HEADER_LOW(__h) \ + { \ + .line[1] = { \ + .text = __h, \ + }, \ + } + +#define HEADER_BOTH(__h0, __h1) \ + { \ + .line[0] = { \ + .text = __h0, \ + }, \ + .line[1] = { \ + .text = __h1, \ + }, \ + } + +void c2c_fmt_free(struct perf_hpp_fmt *fmt); +bool c2c_fmt_equal(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b); + +/* + * Build the function-view hierarchy. Returns -EOPNOTSUPP when @cl_sort lacks + * iaddr. On success, *@hists remains valid until the next + * c2c_function__build() or c2c_function__reset(). On failure, *@hists is + * NULL. + */ +int c2c_function__build(struct c2c_hists *cl_hists, const char *cl_sort, + bool symbol_full, struct hists **hists); +void c2c_function__reset(void); +/* Valid only between a successful build and c2c_function__reset(). */ +struct hist_entry *c2c_function__find_cacheline(struct hist_entry *he); + +/* Inputs and TUI callback supplied by the c2c command. */ +struct c2c_function_view_args { + /* Source cacheline histograms used by the common model. */ + struct c2c_hists *cl_hists; + /* --coalesce field list, used to require iaddr. */ + const char *cl_sort; + /* Do not cap long symbol names. */ + bool symbol_full; + /* Open the cacheline detail view for @he. */ + int (*browse_cacheline)(struct hist_entry *he); +}; + +#ifdef HAVE_SLANG_SUPPORT +int perf_c2c__browse_function_view(struct c2c_function_view_args *args); +#else +static inline int +perf_c2c__browse_function_view(struct c2c_function_view_args *args __maybe_unused) +{ + return 0; +} +#endif + +#endif /* __PERF_UTIL_C2C_H */ diff --git a/tools/perf/util/cache.h b/tools/perf/util/cache.h new file mode 100644 index 000000000..0b61840d4 --- /dev/null +++ b/tools/perf/util/cache.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CACHE_H +#define __PERF_CACHE_H + +#include "strbuf.h" +#include +#include "../ui/ui.h" + +#include +#include + +#define CMD_EXEC_PATH "--exec-path" +#define CMD_DEBUGFS_DIR "--debugfs-dir=" + +#define EXEC_PATH_ENVIRONMENT "PERF_EXEC_PATH" +#define PERF_DEBUGFS_ENVIRONMENT "PERF_DEBUGFS_DIR" +#define PERF_TRACEFS_ENVIRONMENT "PERF_TRACEFS_DIR" +#define PERF_PAGER_ENVIRONMENT "PERF_PAGER" + +int split_cmdline(char *cmdline, const char ***argv); + +#define alloc_nr(x) (((x)+16)*3/2) + +static inline int is_absolute_path(const char *path) +{ + return path[0] == '/'; +} + +char *mkpath(char *path_buf, size_t sz, const char *fmt, ...) __printf(3, 4); + +#endif /* __PERF_CACHE_H */ diff --git a/tools/perf/util/cacheline.c b/tools/perf/util/cacheline.c new file mode 100644 index 000000000..e98b5250a --- /dev/null +++ b/tools/perf/util/cacheline.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "cacheline.h" +#include + +#ifdef _SC_LEVEL1_DCACHE_LINESIZE +#define cache_line_size(cacheline_sizep) *cacheline_sizep = sysconf(_SC_LEVEL1_DCACHE_LINESIZE) +#else +#include +#include "debug.h" +static void cache_line_size(int *cacheline_sizep) +{ + if (sysfs__read_int("devices/system/cpu/cpu0/cache/index0/coherency_line_size", cacheline_sizep)) + pr_debug("cannot determine cache line size"); +} +#endif + +int cacheline_size(void) +{ + static int size; + + if (!size) + cache_line_size(&size); + + return size; +} diff --git a/tools/perf/util/cacheline.h b/tools/perf/util/cacheline.h new file mode 100644 index 000000000..fe6d5b60a --- /dev/null +++ b/tools/perf/util/cacheline.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_CACHELINE_H +#define PERF_CACHELINE_H + +#include + +int __pure cacheline_size(void); + + +/* + * Some architectures have 'Adjacent Cacheline Prefetch' feature, + * which performs like the cacheline size being doubled. + */ +static inline u64 cl_address(u64 address, bool double_cl) +{ + u64 size = cacheline_size(); + + if (double_cl) + size *= 2; + + /* return the cacheline of the address */ + return (address & ~(size - 1)); +} + +static inline u64 cl_offset(u64 address, bool double_cl) +{ + u64 size = cacheline_size(); + + if (double_cl) + size *= 2; + + /* return the offset inside cacheline */ + return (address & (size - 1)); +} + +#endif // PERF_CACHELINE_H diff --git a/tools/perf/util/call-path.c b/tools/perf/util/call-path.c new file mode 100644 index 000000000..5c60b8be1 --- /dev/null +++ b/tools/perf/util/call-path.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * call-path.h: Manipulate a tree data structure containing function call paths + * Copyright (c) 2014, Intel Corporation. + */ + +#include +#include +#include +#include + +#include "call-path.h" + +static void call_path__init(struct call_path *cp, struct call_path *parent, + struct symbol *sym, u64 ip, bool in_kernel) +{ + cp->parent = parent; + cp->sym = sym; + cp->ip = sym ? 0 : ip; + cp->db_id = 0; + cp->in_kernel = in_kernel; + RB_CLEAR_NODE(&cp->rb_node); + cp->children = RB_ROOT; +} + +struct call_path_root *call_path_root__new(void) +{ + struct call_path_root *cpr; + + cpr = zalloc(sizeof(struct call_path_root)); + if (!cpr) + return NULL; + call_path__init(&cpr->call_path, NULL, NULL, 0, false); + INIT_LIST_HEAD(&cpr->blocks); + return cpr; +} + +void call_path_root__free(struct call_path_root *cpr) +{ + struct call_path_block *pos, *n; + + list_for_each_entry_safe(pos, n, &cpr->blocks, node) { + list_del_init(&pos->node); + free(pos); + } + free(cpr); +} + +static struct call_path *call_path__new(struct call_path_root *cpr, + struct call_path *parent, + struct symbol *sym, u64 ip, + bool in_kernel) +{ + struct call_path_block *cpb; + struct call_path *cp; + size_t n; + + if (cpr->next < cpr->sz) { + cpb = list_last_entry(&cpr->blocks, struct call_path_block, + node); + } else { + cpb = zalloc(sizeof(struct call_path_block)); + if (!cpb) + return NULL; + list_add_tail(&cpb->node, &cpr->blocks); + cpr->sz += CALL_PATH_BLOCK_SIZE; + } + + n = cpr->next++ & CALL_PATH_BLOCK_MASK; + cp = &cpb->cp[n]; + + call_path__init(cp, parent, sym, ip, in_kernel); + + return cp; +} + +struct call_path *call_path__findnew(struct call_path_root *cpr, + struct call_path *parent, + struct symbol *sym, u64 ip, u64 ks) +{ + struct rb_node **p; + struct rb_node *node_parent = NULL; + struct call_path *cp; + bool in_kernel = ip >= ks; + + if (sym) + ip = 0; + + if (!parent) + return call_path__new(cpr, parent, sym, ip, in_kernel); + + p = &parent->children.rb_node; + while (*p != NULL) { + node_parent = *p; + cp = rb_entry(node_parent, struct call_path, rb_node); + + if (cp->sym == sym && cp->ip == ip) + return cp; + + if (sym < cp->sym || (sym == cp->sym && ip < cp->ip)) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + cp = call_path__new(cpr, parent, sym, ip, in_kernel); + if (!cp) + return NULL; + + rb_link_node(&cp->rb_node, node_parent, p); + rb_insert_color(&cp->rb_node, &parent->children); + + return cp; +} diff --git a/tools/perf/util/call-path.h b/tools/perf/util/call-path.h new file mode 100644 index 000000000..5875cfc81 --- /dev/null +++ b/tools/perf/util/call-path.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * call-path.h: Manipulate a tree data structure containing function call paths + * Copyright (c) 2014, Intel Corporation. + */ + +#ifndef __PERF_CALL_PATH_H +#define __PERF_CALL_PATH_H + +#include + +#include +#include + +/** + * struct call_path - node in list of calls leading to a function call. + * @parent: call path to the parent function call + * @sym: symbol of function called + * @ip: only if sym is null, the ip of the function + * @db_id: id used for db-export + * @in_kernel: whether function is a in the kernel + * @rb_node: node in parent's tree of called functions + * @children: tree of call paths of functions called + * + * In combination with the call_return structure, the call_path structure + * defines a context-sensitive call-graph. + */ +struct call_path { + struct call_path *parent; + struct symbol *sym; + u64 ip; + u64 db_id; + bool in_kernel; + struct rb_node rb_node; + struct rb_root children; +}; + +#define CALL_PATH_BLOCK_SHIFT 8 +#define CALL_PATH_BLOCK_SIZE (1 << CALL_PATH_BLOCK_SHIFT) +#define CALL_PATH_BLOCK_MASK (CALL_PATH_BLOCK_SIZE - 1) + +struct call_path_block { + struct call_path cp[CALL_PATH_BLOCK_SIZE]; + struct list_head node; +}; + +/** + * struct call_path_root - root of all call paths. + * @call_path: root call path + * @blocks: list of blocks to store call paths + * @next: next free space + * @sz: number of spaces + */ +struct call_path_root { + struct call_path call_path; + struct list_head blocks; + size_t next; + size_t sz; +}; + +struct call_path_root *call_path_root__new(void); +void call_path_root__free(struct call_path_root *cpr); + +struct call_path *call_path__findnew(struct call_path_root *cpr, + struct call_path *parent, + struct symbol *sym, u64 ip, u64 ks); + +#endif diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c new file mode 100644 index 000000000..31c675cba --- /dev/null +++ b/tools/perf/util/callchain.c @@ -0,0 +1,1943 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009-2011, Frederic Weisbecker + * + * Handle the callchains from the stream in an ad-hoc radix tree and then + * sort them in an rbtree. + * + * Using a radix for code path provides a fast retrieval and factorizes + * memory use. Also that lets us use the paths in a hierarchical graph view. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asm/bug.h" + +#include "debug.h" +#include "dso.h" +#include "event.h" +#include "hist.h" +#include "sort.h" +#include "machine.h" +#include "map.h" +#include "callchain.h" +#include "branch.h" +#include "record.h" +#include "symbol.h" +#include "thread.h" +#include "util.h" +#include "../perf.h" + +#define CALLCHAIN_PARAM_DEFAULT \ + .mode = CHAIN_GRAPH_ABS, \ + .min_percent = 0.5, \ + .order = ORDER_CALLEE, \ + .key = CCKEY_FUNCTION, \ + .value = CCVAL_PERCENT, \ + +struct callchain_param callchain_param = { + CALLCHAIN_PARAM_DEFAULT +}; + +/* + * Are there any events usind DWARF callchains? + * + * I.e. + * + * -e cycles/call-graph=dwarf/ + */ +bool dwarf_callchain_users; + +struct callchain_param callchain_param_default = { + CALLCHAIN_PARAM_DEFAULT +}; + +/* Used for thread-local struct callchain_cursor. */ +static pthread_key_t callchain_cursor; + +int parse_callchain_record_opt(const char *arg, struct callchain_param *param) +{ + return parse_callchain_record(arg, param); +} + +static int parse_callchain_mode(const char *value) +{ + if (!strncmp(value, "graph", strlen(value))) { + callchain_param.mode = CHAIN_GRAPH_ABS; + return 0; + } + if (!strncmp(value, "flat", strlen(value))) { + callchain_param.mode = CHAIN_FLAT; + return 0; + } + if (!strncmp(value, "fractal", strlen(value))) { + callchain_param.mode = CHAIN_GRAPH_REL; + return 0; + } + if (!strncmp(value, "folded", strlen(value))) { + callchain_param.mode = CHAIN_FOLDED; + return 0; + } + return -1; +} + +static int parse_callchain_order(const char *value) +{ + if (!strncmp(value, "caller", strlen(value))) { + callchain_param.order = ORDER_CALLER; + callchain_param.order_set = true; + return 0; + } + if (!strncmp(value, "callee", strlen(value))) { + callchain_param.order = ORDER_CALLEE; + callchain_param.order_set = true; + return 0; + } + return -1; +} + +static int parse_callchain_sort_key(const char *value) +{ + if (!strncmp(value, "function", strlen(value))) { + callchain_param.key = CCKEY_FUNCTION; + return 0; + } + if (!strncmp(value, "address", strlen(value))) { + callchain_param.key = CCKEY_ADDRESS; + return 0; + } + if (!strncmp(value, "srcline", strlen(value))) { + callchain_param.key = CCKEY_SRCLINE; + return 0; + } + if (!strncmp(value, "branch", strlen(value))) { + callchain_param.branch_callstack = 1; + return 0; + } + return -1; +} + +static int parse_callchain_value(const char *value) +{ + if (!strncmp(value, "percent", strlen(value))) { + callchain_param.value = CCVAL_PERCENT; + return 0; + } + if (!strncmp(value, "period", strlen(value))) { + callchain_param.value = CCVAL_PERIOD; + return 0; + } + if (!strncmp(value, "count", strlen(value))) { + callchain_param.value = CCVAL_COUNT; + return 0; + } + return -1; +} + +static int get_stack_size(const char *str, unsigned long *_size) +{ + char *endptr; + unsigned long size; + unsigned long max_size = round_down(USHRT_MAX, sizeof(u64)); + + size = strtoul(str, &endptr, 0); + + do { + if (*endptr) + break; + + size = round_up(size, sizeof(u64)); + if (!size || size > max_size) + break; + + *_size = size; + return 0; + + } while (0); + + pr_err("callchain: Incorrect stack dump size (max %ld): %s\n", + max_size, str); + return -1; +} + +static int +__parse_callchain_report_opt(const char *arg, bool allow_record_opt) +{ + char *tok, *arg_copy; + char *endptr, *saveptr = NULL; + bool minpcnt_set = false; + bool record_opt_set = false; + bool try_stack_size = false; + + callchain_param.enabled = true; + symbol_conf.use_callchain = true; + + if (!arg) + return 0; + + arg_copy = strdup(arg); + if (!arg_copy) + return -ENOMEM; + + tok = strtok_r(arg_copy, ",", &saveptr); + while (tok) { + if (!strncmp(tok, "none", strlen(tok))) { + callchain_param.mode = CHAIN_NONE; + callchain_param.enabled = false; + symbol_conf.use_callchain = false; + goto out; + } + + if (!parse_callchain_mode(tok) || + !parse_callchain_order(tok) || + !parse_callchain_sort_key(tok) || + !parse_callchain_value(tok)) { + /* parsing ok - move on to the next */ + try_stack_size = false; + goto next; + } else if (allow_record_opt && !record_opt_set) { + if (parse_callchain_record(tok, &callchain_param)) + goto try_numbers; + + /* assume that number followed by 'dwarf' is stack size */ + if (callchain_param.record_mode == CALLCHAIN_DWARF) + try_stack_size = true; + + record_opt_set = true; + goto next; + } + +try_numbers: + if (try_stack_size) { + unsigned long size = 0; + + if (get_stack_size(tok, &size) < 0) + goto err_out; + callchain_param.dump_size = size; + try_stack_size = false; + } else if (!minpcnt_set) { + /* try to get the min percent */ + callchain_param.min_percent = strtod(tok, &endptr); + if (tok == endptr) + goto err_out; + minpcnt_set = true; + } else { + /* try print limit at last */ + callchain_param.print_limit = strtoul(tok, &endptr, 0); + if (tok == endptr) + goto err_out; + } +next: + tok = strtok_r(NULL, ",", &saveptr); + } + + if (callchain_register_param(&callchain_param) < 0) { + pr_err("Can't register callchain params\n"); + goto err_out; + } +out: + free(arg_copy); + return 0; +err_out: + free(arg_copy); + return -1; +} + +int parse_callchain_report_opt(const char *arg) +{ + return __parse_callchain_report_opt(arg, false); +} + +int parse_callchain_top_opt(const char *arg) +{ + return __parse_callchain_report_opt(arg, true); +} + +int parse_callchain_record(const char *arg, struct callchain_param *param) +{ + char *tok, *name, *saveptr = NULL; + char *buf; + int ret = -1; + + /* We need buffer that we know we can write to. */ + buf = strdup(arg); + if (!buf) + return -ENOMEM; + + tok = strtok_r(buf, ",", &saveptr); + name = tok ? : buf; + + do { + /* Framepointer style */ + if (!strncmp(name, "fp", sizeof("fp"))) { + ret = 0; + param->record_mode = CALLCHAIN_FP; + + tok = strtok_r(NULL, ",", &saveptr); + if (tok) { + unsigned long size; + + if (!strncmp(tok, "defer", sizeof("defer"))) { + param->defer = true; + } else { + size = strtoul(tok, &name, 0); + if (size < (unsigned) sysctl__max_stack()) + param->max_stack = size; + } + } + break; + + /* Dwarf style */ + } else if (!strncmp(name, "dwarf", sizeof("dwarf"))) { + const unsigned long default_stack_dump_size = 8192; + + ret = 0; + param->record_mode = CALLCHAIN_DWARF; + param->dump_size = default_stack_dump_size; + dwarf_callchain_users = true; + + tok = strtok_r(NULL, ",", &saveptr); + if (tok) { + unsigned long size = 0; + + ret = get_stack_size(tok, &size); + param->dump_size = size; + } + } else if (!strncmp(name, "lbr", sizeof("lbr"))) { + if (!strtok_r(NULL, ",", &saveptr)) { + param->record_mode = CALLCHAIN_LBR; + ret = 0; + } else + pr_err("callchain: No more arguments " + "needed for --call-graph lbr\n"); + break; + } else { + pr_err("callchain: Unknown --call-graph option " + "value: %s\n", arg); + break; + } + + } while (0); + + free(buf); + + if (param->defer && param->record_mode != CALLCHAIN_FP) { + pr_err("callchain: deferred callchain only works with FP\n"); + return -EINVAL; + } + + return ret; +} + +static void callchain_debug(const struct callchain_param *callchain) +{ + static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" }; + + pr_debug("callchain: type %s\n", str[callchain->record_mode]); + + if (callchain->record_mode == CALLCHAIN_DWARF) + pr_debug("callchain: stack dump size %d\n", + callchain->dump_size); +} + +int record_opts__parse_callchain(struct record_opts *record, + struct callchain_param *callchain, + const char *arg, bool unset) +{ + int ret; + + callchain->enabled = !unset; + + /* --no-call-graph */ + if (unset) { + callchain->record_mode = CALLCHAIN_NONE; + pr_debug("callchain: disabled\n"); + return 0; + } + + ret = parse_callchain_record_opt(arg, callchain); + if (!ret) { + /* Enable data address sampling for DWARF unwind. */ + if (callchain->record_mode == CALLCHAIN_DWARF && + !record->record_data_mmap_set) + record->record_data_mmap = true; + callchain_debug(callchain); + } + + return ret; +} + +int perf_callchain_config(const char *var, const char *value) +{ + char *endptr; + + if (!strstarts(var, "call-graph.")) + return 0; + var += sizeof("call-graph.") - 1; + + if (!strcmp(var, "record-mode")) + return parse_callchain_record_opt(value, &callchain_param); + if (!strcmp(var, "dump-size")) { + unsigned long size = 0; + int ret; + + ret = get_stack_size(value, &size); + callchain_param.dump_size = size; + + return ret; + } + if (!strcmp(var, "print-type")){ + int ret; + ret = parse_callchain_mode(value); + if (ret == -1) + pr_err("Invalid callchain mode: %s\n", value); + return ret; + } + if (!strcmp(var, "order")){ + int ret; + ret = parse_callchain_order(value); + if (ret == -1) + pr_err("Invalid callchain order: %s\n", value); + return ret; + } + if (!strcmp(var, "sort-key")){ + int ret; + ret = parse_callchain_sort_key(value); + if (ret == -1) + pr_err("Invalid callchain sort key: %s\n", value); + return ret; + } + if (!strcmp(var, "threshold")) { + callchain_param.min_percent = strtod(value, &endptr); + if (value == endptr) { + pr_err("Invalid callchain threshold: %s\n", value); + return -1; + } + } + if (!strcmp(var, "print-limit")) { + callchain_param.print_limit = strtod(value, &endptr); + if (value == endptr) { + pr_err("Invalid callchain print limit: %s\n", value); + return -1; + } + } + + return 0; +} + +static void +rb_insert_callchain(struct rb_root *root, struct callchain_node *chain, + enum chain_mode mode) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct callchain_node *rnode; + u64 chain_cumul = callchain_cumul_hits(chain); + + while (*p) { + u64 rnode_cumul; + + parent = *p; + rnode = rb_entry(parent, struct callchain_node, rb_node); + rnode_cumul = callchain_cumul_hits(rnode); + + switch (mode) { + case CHAIN_FLAT: + case CHAIN_FOLDED: + if (rnode->hit < chain->hit) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + break; + case CHAIN_GRAPH_ABS: /* Falldown */ + case CHAIN_GRAPH_REL: + if (rnode_cumul < chain_cumul) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + break; + case CHAIN_NONE: + default: + break; + } + } + + rb_link_node(&chain->rb_node, parent, p); + rb_insert_color(&chain->rb_node, root); +} + +static void +__sort_chain_flat(struct rb_root *rb_root, struct callchain_node *node, + u64 min_hit) +{ + struct rb_node *n; + struct callchain_node *child; + + n = rb_first(&node->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + __sort_chain_flat(rb_root, child, min_hit); + } + + if (node->hit && node->hit >= min_hit) + rb_insert_callchain(rb_root, node, CHAIN_FLAT); +} + +/* + * Once we get every callchains from the stream, we can now + * sort them by hit + */ +static void +sort_chain_flat(struct rb_root *rb_root, struct callchain_root *root, + u64 min_hit, struct callchain_param *param __maybe_unused) +{ + *rb_root = RB_ROOT; + __sort_chain_flat(rb_root, &root->node, min_hit); +} + +static void __sort_chain_graph_abs(struct callchain_node *node, + u64 min_hit) +{ + struct rb_node *n; + struct callchain_node *child; + + node->rb_root = RB_ROOT; + n = rb_first(&node->rb_root_in); + + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + __sort_chain_graph_abs(child, min_hit); + if (callchain_cumul_hits(child) >= min_hit) + rb_insert_callchain(&node->rb_root, child, + CHAIN_GRAPH_ABS); + } +} + +static void +sort_chain_graph_abs(struct rb_root *rb_root, struct callchain_root *chain_root, + u64 min_hit, struct callchain_param *param __maybe_unused) +{ + __sort_chain_graph_abs(&chain_root->node, min_hit); + rb_root->rb_node = chain_root->node.rb_root.rb_node; +} + +static void __sort_chain_graph_rel(struct callchain_node *node, + double min_percent) +{ + struct rb_node *n; + struct callchain_node *child; + u64 min_hit; + + node->rb_root = RB_ROOT; + min_hit = ceil(node->children_hit * min_percent); + + n = rb_first(&node->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + __sort_chain_graph_rel(child, min_percent); + if (callchain_cumul_hits(child) >= min_hit) + rb_insert_callchain(&node->rb_root, child, + CHAIN_GRAPH_REL); + } +} + +static void +sort_chain_graph_rel(struct rb_root *rb_root, struct callchain_root *chain_root, + u64 min_hit __maybe_unused, struct callchain_param *param) +{ + __sort_chain_graph_rel(&chain_root->node, param->min_percent / 100.0); + rb_root->rb_node = chain_root->node.rb_root.rb_node; +} + +int callchain_register_param(struct callchain_param *param) +{ + switch (param->mode) { + case CHAIN_GRAPH_ABS: + param->sort = sort_chain_graph_abs; + break; + case CHAIN_GRAPH_REL: + param->sort = sort_chain_graph_rel; + break; + case CHAIN_FLAT: + case CHAIN_FOLDED: + param->sort = sort_chain_flat; + break; + case CHAIN_NONE: + default: + return -1; + } + return 0; +} + +/* + * Create a child for a parent. If inherit_children, then the new child + * will become the new parent of it's parent children + */ +static struct callchain_node * +create_child(struct callchain_node *parent, bool inherit_children) +{ + struct callchain_node *new; + + new = zalloc(sizeof(*new)); + if (!new) { + perror("not enough memory to create child for code path tree"); + return NULL; + } + new->parent = parent; + INIT_LIST_HEAD(&new->val); + INIT_LIST_HEAD(&new->parent_val); + + if (inherit_children) { + struct rb_node *n; + struct callchain_node *child; + + new->rb_root_in = parent->rb_root_in; + parent->rb_root_in = RB_ROOT; + + n = rb_first(&new->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + child->parent = new; + n = rb_next(n); + } + + /* make it the first child */ + rb_link_node(&new->rb_node_in, NULL, &parent->rb_root_in.rb_node); + rb_insert_color(&new->rb_node_in, &parent->rb_root_in); + } + + return new; +} + + +/* + * Fill the node with callchain values + */ +static int +fill_node(struct callchain_node *node, struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *cursor_node; + + node->val_nr = cursor->nr - cursor->pos; + if (!node->val_nr) + pr_warning("Warning: empty node in callchain tree\n"); + + cursor_node = callchain_cursor_current(cursor); + + while (cursor_node) { + struct callchain_list *call; + + call = zalloc(sizeof(*call)); + if (!call) { + perror("not enough memory for the code path tree"); + return -ENOMEM; + } + call->ip = cursor_node->ip; + map_symbol__copy(&call->ms, &cursor_node->ms); + call->srcline = cursor_node->srcline; + + if (cursor_node->branch) { + call->branch_count = 1; + + if (cursor_node->branch_from) { + /* + * branch_from is set with value somewhere else + * to imply it's "to" of a branch. + */ + if (!call->brtype_stat) { + call->brtype_stat = zalloc(sizeof(*call->brtype_stat)); + if (!call->brtype_stat) { + perror("not enough memory for the code path branch statistics"); + zfree(&call->brtype_stat); + return -ENOMEM; + } + } + call->brtype_stat->branch_to = true; + + if (cursor_node->branch_flags.predicted) + call->predicted_count = 1; + + if (cursor_node->branch_flags.abort) + call->abort_count = 1; + + branch_type_count(call->brtype_stat, + &cursor_node->branch_flags, + cursor_node->branch_from, + cursor_node->ip); + } else { + /* + * It's "from" of a branch + */ + if (call->brtype_stat && call->brtype_stat->branch_to) + call->brtype_stat->branch_to = false; + call->cycles_count = + cursor_node->branch_flags.cycles; + call->iter_count = cursor_node->nr_loop_iter; + call->iter_cycles = cursor_node->iter_cycles; + } + } + + list_add_tail(&call->list, &node->val); + + callchain_cursor_advance(cursor); + cursor_node = callchain_cursor_current(cursor); + } + return 0; +} + +static struct callchain_node * +add_child(struct callchain_node *parent, + struct callchain_cursor *cursor, + u64 period) +{ + struct callchain_node *new; + + new = create_child(parent, false); + if (new == NULL) + return NULL; + + if (fill_node(new, cursor) < 0) { + struct callchain_list *call, *tmp; + + list_for_each_entry_safe(call, tmp, &new->val, list) { + list_del_init(&call->list); + map_symbol__exit(&call->ms); + zfree(&call->brtype_stat); + free(call); + } + free(new); + return NULL; + } + + new->children_hit = 0; + new->hit = period; + new->children_count = 0; + new->count = 1; + return new; +} + +enum match_result { + MATCH_ERROR = -1, + MATCH_EQ, + MATCH_LT, + MATCH_GT, +}; + +static enum match_result match_chain_strings(const char *left, + const char *right) +{ + enum match_result ret = MATCH_EQ; + int cmp; + + if (left && right) + cmp = strcmp(left, right); + else if (!left && right) + cmp = 1; + else if (left && !right) + cmp = -1; + else + return MATCH_ERROR; + + if (cmp != 0) + ret = cmp < 0 ? MATCH_LT : MATCH_GT; + + return ret; +} + +/* + * We need to always use relative addresses because we're aggregating + * callchains from multiple threads, i.e. different address spaces, so + * comparing absolute addresses make no sense as a symbol in a DSO may end up + * in a different address when used in a different binary or even the same + * binary but with some sort of address randomization technique, thus we need + * to compare just relative addresses. -acme + */ +static enum match_result match_chain_dso_addresses(struct map *left_map, u64 left_ip, + struct map *right_map, u64 right_ip) +{ + struct dso *left_dso = left_map ? map__dso(left_map) : NULL; + struct dso *right_dso = right_map ? map__dso(right_map) : NULL; + + if (left_dso != right_dso) + return left_dso < right_dso ? MATCH_LT : MATCH_GT; + + if (left_ip != right_ip) + return left_ip < right_ip ? MATCH_LT : MATCH_GT; + + return MATCH_EQ; +} + +static enum match_result match_chain(struct callchain_cursor_node *node, + struct callchain_list *cnode) +{ + enum match_result match = MATCH_ERROR; + + switch (callchain_param.key) { + case CCKEY_SRCLINE: + match = match_chain_strings(cnode->srcline, node->srcline); + if (match != MATCH_ERROR) + break; + /* otherwise fall-back to symbol-based comparison below */ + fallthrough; + case CCKEY_FUNCTION: + if (node->ms.sym && cnode->ms.sym) { + /* + * Compare inlined frames based on their symbol name + * because different inlined frames will have the same + * symbol start. Otherwise do a faster comparison based + * on the symbol start address. + */ + if (symbol__inlined(cnode->ms.sym) || symbol__inlined(node->ms.sym)) { + match = match_chain_strings(cnode->ms.sym->name, + node->ms.sym->name); + if (match != MATCH_ERROR) + break; + } else { + match = match_chain_dso_addresses(cnode->ms.map, cnode->ms.sym->start, + node->ms.map, node->ms.sym->start); + break; + } + } + /* otherwise fall-back to IP-based comparison below */ + fallthrough; + case CCKEY_ADDRESS: + default: + match = match_chain_dso_addresses(cnode->ms.map, cnode->ip, node->ms.map, node->ip); + break; + } + + if (match == MATCH_EQ && node->branch) { + cnode->branch_count++; + + if (node->branch_from) { + /* + * It's "to" of a branch + */ + if (!cnode->brtype_stat) { + cnode->brtype_stat = zalloc(sizeof(*cnode->brtype_stat)); + if (!cnode->brtype_stat) { + perror("not enough memory for the code path branch statistics"); + return MATCH_ERROR; + } + } + cnode->brtype_stat->branch_to = true; + + if (node->branch_flags.predicted) + cnode->predicted_count++; + + if (node->branch_flags.abort) + cnode->abort_count++; + + branch_type_count(cnode->brtype_stat, + &node->branch_flags, + node->branch_from, + node->ip); + } else { + /* + * It's "from" of a branch + */ + if (cnode->brtype_stat && cnode->brtype_stat->branch_to) + cnode->brtype_stat->branch_to = false; + cnode->cycles_count += node->branch_flags.cycles; + cnode->iter_count += node->nr_loop_iter; + cnode->iter_cycles += node->iter_cycles; + cnode->from_count++; + } + } + + return match; +} + +/* + * Split the parent in two parts (a new child is created) and + * give a part of its callchain to the created child. + * Then create another child to host the given callchain of new branch + */ +static int +split_add_child(struct callchain_node *parent, + struct callchain_cursor *cursor, + struct callchain_list *to_split, + u64 idx_parents, u64 idx_local, u64 period) +{ + struct callchain_node *new; + struct list_head *old_tail; + unsigned int idx_total = idx_parents + idx_local; + + /* split */ + new = create_child(parent, true); + if (new == NULL) + return -1; + + /* split the callchain and move a part to the new child */ + old_tail = parent->val.prev; + list_del_range(&to_split->list, old_tail); + new->val.next = &to_split->list; + new->val.prev = old_tail; + to_split->list.prev = &new->val; + old_tail->next = &new->val; + + /* split the hits */ + new->hit = parent->hit; + new->children_hit = parent->children_hit; + parent->children_hit = callchain_cumul_hits(new); + new->val_nr = parent->val_nr - idx_local; + parent->val_nr = idx_local; + new->count = parent->count; + new->children_count = parent->children_count; + parent->children_count = callchain_cumul_counts(new); + + /* create a new child for the new branch if any */ + if (idx_total < cursor->nr) { + struct callchain_node *first; + struct callchain_list *cnode; + struct callchain_cursor_node *node; + struct rb_node *p, **pp; + + parent->hit = 0; + parent->children_hit += period; + parent->count = 0; + parent->children_count += 1; + + node = callchain_cursor_current(cursor); + new = add_child(parent, cursor, period); + if (new == NULL) + return -1; + + /* + * This is second child since we moved parent's children + * to new (first) child above. + */ + p = parent->rb_root_in.rb_node; + first = rb_entry(p, struct callchain_node, rb_node_in); + cnode = list_first_entry(&first->val, struct callchain_list, + list); + + if (match_chain(node, cnode) == MATCH_LT) + pp = &p->rb_left; + else + pp = &p->rb_right; + + rb_link_node(&new->rb_node_in, p, pp); + rb_insert_color(&new->rb_node_in, &parent->rb_root_in); + } else { + parent->hit = period; + parent->count = 1; + } + return 0; +} + +static enum match_result +append_chain(struct callchain_node *root, + struct callchain_cursor *cursor, + u64 period); + +static int +append_chain_children(struct callchain_node *root, + struct callchain_cursor *cursor, + u64 period) +{ + struct callchain_node *rnode; + struct callchain_cursor_node *node; + struct rb_node **p = &root->rb_root_in.rb_node; + struct rb_node *parent = NULL; + + node = callchain_cursor_current(cursor); + if (!node) + return -1; + + /* lookup in children */ + while (*p) { + enum match_result ret; + + parent = *p; + rnode = rb_entry(parent, struct callchain_node, rb_node_in); + + /* If at least first entry matches, rely to children */ + ret = append_chain(rnode, cursor, period); + if (ret == MATCH_EQ) + goto inc_children_hit; + if (ret == MATCH_ERROR) + return -1; + + if (ret == MATCH_LT) + p = &parent->rb_left; + else + p = &parent->rb_right; + } + /* nothing in children, add to the current node */ + rnode = add_child(root, cursor, period); + if (rnode == NULL) + return -1; + + rb_link_node(&rnode->rb_node_in, parent, p); + rb_insert_color(&rnode->rb_node_in, &root->rb_root_in); + +inc_children_hit: + root->children_hit += period; + root->children_count++; + return 0; +} + +static enum match_result +append_chain(struct callchain_node *root, + struct callchain_cursor *cursor, + u64 period) +{ + struct callchain_list *cnode; + u64 start = cursor->pos; + bool found = false; + u64 matches; + enum match_result cmp = MATCH_ERROR; + + /* + * Lookup in the current node + * If we have a symbol, then compare the start to match + * anywhere inside a function, unless function + * mode is disabled. + */ + list_for_each_entry(cnode, &root->val, list) { + struct callchain_cursor_node *node; + + node = callchain_cursor_current(cursor); + if (!node) + break; + + cmp = match_chain(node, cnode); + if (cmp != MATCH_EQ) + break; + + found = true; + + callchain_cursor_advance(cursor); + } + + /* matches not, relay no the parent */ + if (!found) { + WARN_ONCE(cmp == MATCH_ERROR, "Chain comparison error\n"); + return cmp; + } + + matches = cursor->pos - start; + + /* we match only a part of the node. Split it and add the new chain */ + if (matches < root->val_nr) { + if (split_add_child(root, cursor, cnode, start, matches, + period) < 0) + return MATCH_ERROR; + + return MATCH_EQ; + } + + /* we match 100% of the path, increment the hit */ + if (matches == root->val_nr && cursor->pos == cursor->nr) { + root->hit += period; + root->count++; + return MATCH_EQ; + } + + /* We match the node and still have a part remaining */ + if (append_chain_children(root, cursor, period) < 0) + return MATCH_ERROR; + + return MATCH_EQ; +} + +int callchain_append(struct callchain_root *root, + struct callchain_cursor *cursor, + u64 period) +{ + if (cursor == NULL) + return -1; + + if (!cursor->nr) + return 0; + + callchain_cursor_commit(cursor); + + if (append_chain_children(&root->node, cursor, period) < 0) + return -1; + + if (cursor->nr > root->max_depth) + root->max_depth = cursor->nr; + + return 0; +} + +static int +merge_chain_branch(struct callchain_cursor *cursor, + struct callchain_node *dst, struct callchain_node *src) +{ + struct callchain_cursor_node **old_last = cursor->last; + struct callchain_node *child; + struct callchain_list *list, *next_list; + struct rb_node *n; + int old_pos = cursor->nr; + int err = 0; + + list_for_each_entry_safe(list, next_list, &src->val, list) { + struct map_symbol ms = { + .thread = thread__get(list->ms.thread), + .map = map__get(list->ms.map), + }; + callchain_cursor_append(cursor, list->ip, &ms, false, NULL, 0, 0, 0, list->srcline); + list_del_init(&list->list); + map_symbol__exit(&ms); + map_symbol__exit(&list->ms); + zfree(&list->brtype_stat); + free(list); + } + + if (src->hit) { + callchain_cursor_commit(cursor); + if (append_chain_children(dst, cursor, src->hit) < 0) + return -1; + } + + n = rb_first(&src->rb_root_in); + while (n) { + child = container_of(n, struct callchain_node, rb_node_in); + n = rb_next(n); + rb_erase(&child->rb_node_in, &src->rb_root_in); + + err = merge_chain_branch(cursor, dst, child); + if (err) + break; + + free(child); + } + + cursor->nr = old_pos; + cursor->last = old_last; + + return err; +} + +int callchain_merge(struct callchain_cursor *cursor, + struct callchain_root *dst, struct callchain_root *src) +{ + return merge_chain_branch(cursor, &dst->node, &src->node); +} + +int callchain_cursor_append(struct callchain_cursor *cursor, + u64 ip, struct map_symbol *ms, + bool branch, struct branch_flags *flags, + int nr_loop_iter, u64 iter_cycles, u64 branch_from, + const char *srcline) +{ + struct callchain_cursor_node *node = *cursor->last; + + if (!node) { + node = calloc(1, sizeof(*node)); + if (!node) + return -ENOMEM; + + *cursor->last = node; + } + + node->ip = ip; + map_symbol__exit(&node->ms); + map_symbol__copy(&node->ms, ms); + node->branch = branch; + node->nr_loop_iter = nr_loop_iter; + node->iter_cycles = iter_cycles; + node->srcline = srcline; + + if (flags) + memcpy(&node->branch_flags, flags, + sizeof(struct branch_flags)); + + node->branch_from = branch_from; + cursor->nr++; + + cursor->last = &node->next; + + return 0; +} + +int sample__resolve_callchain(struct perf_sample *sample, + struct callchain_cursor *cursor, struct symbol **parent, + struct addr_location *al, + int max_stack) +{ + if (sample->callchain == NULL && !symbol_conf.show_branchflag_count) + return 0; + + if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain || + perf_hpp_list.parent || symbol_conf.show_branchflag_count) { + return thread__resolve_callchain(al->thread, cursor, sample, + parent, al, max_stack); + } + return 0; +} + +int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample) +{ + if ((!symbol_conf.use_callchain || sample->callchain == NULL) && + !symbol_conf.show_branchflag_count) + return 0; + return callchain_append(he->callchain, get_tls_callchain_cursor(), sample->period); +} + +int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, + bool hide_unresolved) +{ + struct machine *machine = NULL; + + if (node->ms.thread) + machine = maps__machine(thread__maps(node->ms.thread)); + + map__put(al->map); + al->map = map__get(node->ms.map); + al->sym = node->ms.sym; + al->srcline = node->srcline; + al->addr = node->ip; + + if (al->sym == NULL) { + if (hide_unresolved) + return 0; + if (al->map == NULL) + goto out; + } + if (maps__equal(thread__maps(al->thread), machine__kernel_maps(machine))) { + if (machine__is_host(machine)) { + al->cpumode = PERF_RECORD_MISC_KERNEL; + al->level = 'k'; + } else { + al->cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + al->level = 'g'; + } + } else { + if (machine__is_host(machine)) { + al->cpumode = PERF_RECORD_MISC_USER; + al->level = '.'; + } else if (perf_guest) { + al->cpumode = PERF_RECORD_MISC_GUEST_USER; + al->level = 'u'; + } else { + al->cpumode = PERF_RECORD_MISC_HYPERVISOR; + al->level = 'H'; + } + } + +out: + return 1; +} + +char *callchain_list__sym_name(struct callchain_list *cl, + char *bf, size_t bfsize, bool show_dso) +{ + bool show_addr = callchain_param.key == CCKEY_ADDRESS; + bool show_srcline = show_addr || callchain_param.key == CCKEY_SRCLINE; + int printed; + + if (cl->ms.sym) { + const char *inlined = symbol__inlined(cl->ms.sym) ? " (inlined)" : ""; + + if (show_srcline && cl->srcline) + printed = scnprintf(bf, bfsize, "%s %s%s", + cl->ms.sym->name, cl->srcline, + inlined); + else + printed = scnprintf(bf, bfsize, "%s%s", + cl->ms.sym->name, inlined); + } else + printed = scnprintf(bf, bfsize, "%#" PRIx64, cl->ip); + + if (show_dso) + scnprintf(bf + printed, bfsize - printed, " %s", + cl->ms.map ? + dso__short_name(map__dso(cl->ms.map)) : + "unknown"); + + return bf; +} + +char *callchain_node__scnprintf_value(struct callchain_node *node, + char *bf, size_t bfsize, u64 total) +{ + double percent = 0.0; + u64 period = callchain_cumul_hits(node); + unsigned count = callchain_cumul_counts(node); + + if (callchain_param.mode == CHAIN_FOLDED) { + period = node->hit; + count = node->count; + } + + switch (callchain_param.value) { + case CCVAL_PERIOD: + scnprintf(bf, bfsize, "%"PRIu64, period); + break; + case CCVAL_COUNT: + scnprintf(bf, bfsize, "%u", count); + break; + case CCVAL_PERCENT: + default: + if (total) + percent = period * 100.0 / total; + scnprintf(bf, bfsize, "%.2f%%", percent); + break; + } + return bf; +} + +int callchain_node__fprintf_value(struct callchain_node *node, + FILE *fp, u64 total) +{ + double percent = 0.0; + u64 period = callchain_cumul_hits(node); + unsigned count = callchain_cumul_counts(node); + + if (callchain_param.mode == CHAIN_FOLDED) { + period = node->hit; + count = node->count; + } + + switch (callchain_param.value) { + case CCVAL_PERIOD: + return fprintf(fp, "%"PRIu64, period); + case CCVAL_COUNT: + return fprintf(fp, "%u", count); + case CCVAL_PERCENT: + default: + if (total) + percent = period * 100.0 / total; + return percent_color_fprintf(fp, "%.2f%%", percent); + } + return 0; +} + +static void callchain_counts_value(struct callchain_node *node, + u64 *branch_count, u64 *predicted_count, + u64 *abort_count, u64 *cycles_count) +{ + struct callchain_list *clist; + + list_for_each_entry(clist, &node->val, list) { + if (branch_count) + *branch_count += clist->branch_count; + + if (predicted_count) + *predicted_count += clist->predicted_count; + + if (abort_count) + *abort_count += clist->abort_count; + + if (cycles_count) + *cycles_count += clist->cycles_count; + } +} + +static int callchain_node_branch_counts_cumul(struct callchain_node *node, + u64 *branch_count, + u64 *predicted_count, + u64 *abort_count, + u64 *cycles_count) +{ + struct callchain_node *child; + struct rb_node *n; + + n = rb_first(&node->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + callchain_node_branch_counts_cumul(child, branch_count, + predicted_count, + abort_count, + cycles_count); + + callchain_counts_value(child, branch_count, + predicted_count, abort_count, + cycles_count); + } + + return 0; +} + +int callchain_branch_counts(struct callchain_root *root, + u64 *branch_count, u64 *predicted_count, + u64 *abort_count, u64 *cycles_count) +{ + if (branch_count) + *branch_count = 0; + + if (predicted_count) + *predicted_count = 0; + + if (abort_count) + *abort_count = 0; + + if (cycles_count) + *cycles_count = 0; + + return callchain_node_branch_counts_cumul(&root->node, + branch_count, + predicted_count, + abort_count, + cycles_count); +} + +static int count_pri64_printf(int idx, const char *str, u64 value, char *bf, int bfsize) +{ + return scnprintf(bf, bfsize, "%s%s:%" PRId64 "", (idx) ? " " : " (", str, value); +} + +static int count_float_printf(int idx, const char *str, float value, + char *bf, int bfsize, float threshold) +{ + if (threshold != 0.0 && value < threshold) + return 0; + + return scnprintf(bf, bfsize, "%s%s:%.1f%%", (idx) ? " " : " (", str, value); +} + +static int branch_to_str(char *bf, int bfsize, + u64 branch_count, u64 predicted_count, + u64 abort_count, + const struct branch_type_stat *brtype_stat) +{ + int printed, i = 0; + + printed = branch_type_str(brtype_stat, bf, bfsize); + if (printed) + i++; + + if (predicted_count < branch_count) { + printed += count_float_printf(i++, "predicted", + predicted_count * 100.0 / branch_count, + bf + printed, bfsize - printed, 0.0); + } + + if (abort_count) { + printed += count_float_printf(i++, "abort", + abort_count * 100.0 / branch_count, + bf + printed, bfsize - printed, 0.1); + } + + if (i) + printed += scnprintf(bf + printed, bfsize - printed, ")"); + + return printed; +} + +static int branch_from_str(char *bf, int bfsize, + u64 branch_count, + u64 cycles_count, u64 iter_count, + u64 iter_cycles, u64 from_count) +{ + int printed = 0, i = 0; + u64 cycles, v = 0; + + cycles = cycles_count / branch_count; + if (cycles) { + printed += count_pri64_printf(i++, "cycles", + cycles, + bf + printed, bfsize - printed); + } + + if (iter_count && from_count) { + v = iter_count / from_count; + if (v) { + printed += count_pri64_printf(i++, "iter", + v, bf + printed, bfsize - printed); + + printed += count_pri64_printf(i++, "avg_cycles", + iter_cycles / iter_count, + bf + printed, bfsize - printed); + } + } + + if (i) + printed += scnprintf(bf + printed, bfsize - printed, ")"); + + return printed; +} + +static int counts_str_build(char *bf, int bfsize, + u64 branch_count, u64 predicted_count, + u64 abort_count, u64 cycles_count, + u64 iter_count, u64 iter_cycles, + u64 from_count, + const struct branch_type_stat *brtype_stat) +{ + int printed; + + if (branch_count == 0) + return scnprintf(bf, bfsize, " (calltrace)"); + + if (brtype_stat->branch_to) { + printed = branch_to_str(bf, bfsize, branch_count, + predicted_count, abort_count, brtype_stat); + } else { + printed = branch_from_str(bf, bfsize, branch_count, + cycles_count, iter_count, iter_cycles, + from_count); + } + + if (!printed) + bf[0] = 0; + + return printed; +} + +static int callchain_counts_printf(FILE *fp, char *bf, int bfsize, + u64 branch_count, u64 predicted_count, + u64 abort_count, u64 cycles_count, + u64 iter_count, u64 iter_cycles, + u64 from_count, + const struct branch_type_stat *brtype_stat) +{ + char str[256]; + + counts_str_build(str, sizeof(str), branch_count, + predicted_count, abort_count, cycles_count, + iter_count, iter_cycles, from_count, brtype_stat); + + if (fp) + return fprintf(fp, "%s", str); + + return scnprintf(bf, bfsize, "%s", str); +} + +int callchain_list_counts__printf_value(struct callchain_list *clist, + FILE *fp, char *bf, int bfsize) +{ + static const struct branch_type_stat empty_brtype_stat = {}; + const struct branch_type_stat *brtype_stat; + u64 branch_count, predicted_count; + u64 abort_count, cycles_count; + u64 iter_count, iter_cycles; + u64 from_count; + + brtype_stat = clist->brtype_stat ?: &empty_brtype_stat; + branch_count = clist->branch_count; + predicted_count = clist->predicted_count; + abort_count = clist->abort_count; + cycles_count = clist->cycles_count; + iter_count = clist->iter_count; + iter_cycles = clist->iter_cycles; + from_count = clist->from_count; + + return callchain_counts_printf(fp, bf, bfsize, branch_count, + predicted_count, abort_count, + cycles_count, iter_count, iter_cycles, + from_count, brtype_stat); +} + +static void free_callchain_node(struct callchain_node *node) +{ + struct callchain_list *list, *tmp; + struct callchain_node *child; + struct rb_node *n; + + list_for_each_entry_safe(list, tmp, &node->parent_val, list) { + list_del_init(&list->list); + map_symbol__exit(&list->ms); + zfree(&list->brtype_stat); + free(list); + } + + list_for_each_entry_safe(list, tmp, &node->val, list) { + list_del_init(&list->list); + map_symbol__exit(&list->ms); + zfree(&list->brtype_stat); + free(list); + } + + n = rb_first(&node->rb_root_in); + while (n) { + child = container_of(n, struct callchain_node, rb_node_in); + n = rb_next(n); + rb_erase(&child->rb_node_in, &node->rb_root_in); + + free_callchain_node(child); + free(child); + } +} + +void free_callchain(struct callchain_root *root) +{ + if (!symbol_conf.use_callchain) + return; + + free_callchain_node(&root->node); +} + +void callchain_cursor_cleanup(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + + for (node = cursor->first; node; node = next) { + next = node->next; + free(node); + } + cursor->first = NULL; + cursor->last = &cursor->first; + cursor->curr = NULL; +} + +static u64 decay_callchain_node(struct callchain_node *node) +{ + struct callchain_node *child; + struct rb_node *n; + u64 child_hits = 0; + + n = rb_first(&node->rb_root_in); + while (n) { + child = container_of(n, struct callchain_node, rb_node_in); + + child_hits += decay_callchain_node(child); + n = rb_next(n); + } + + node->hit = (node->hit * 7) / 8; + node->children_hit = child_hits; + + return node->hit; +} + +void decay_callchain(struct callchain_root *root) +{ + if (!symbol_conf.use_callchain) + return; + + decay_callchain_node(&root->node); +} + +int callchain_node__make_parent_list(struct callchain_node *node) +{ + struct callchain_node *parent = node->parent; + struct callchain_list *chain, *new; + LIST_HEAD(head); + + while (parent) { + list_for_each_entry_reverse(chain, &parent->val, list) { + new = malloc(sizeof(*new)); + if (new == NULL) + goto out; + *new = *chain; + new->has_children = false; + map_symbol__copy(&new->ms, &chain->ms); + list_add_tail(&new->list, &head); + } + parent = parent->parent; + } + + list_for_each_entry_safe_reverse(chain, new, &head, list) + list_move_tail(&chain->list, &node->parent_val); + + if (!list_empty(&node->parent_val)) { + chain = list_first_entry(&node->parent_val, struct callchain_list, list); + chain->has_children = rb_prev(&node->rb_node) || rb_next(&node->rb_node); + + chain = list_first_entry(&node->val, struct callchain_list, list); + chain->has_children = false; + } + return 0; + +out: + list_for_each_entry_safe(chain, new, &head, list) { + list_del_init(&chain->list); + map_symbol__exit(&chain->ms); + zfree(&chain->brtype_stat); + free(chain); + } + return -ENOMEM; +} + +static void callchain_cursor__delete(void *vcursor) +{ + struct callchain_cursor *cursor = vcursor; + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + for (node = cursor->first; node != NULL; node = next) { + next = node->next; + free(node); + } + free(cursor); +} + +static void init_callchain_cursor_key(void) +{ + if (pthread_key_create(&callchain_cursor, callchain_cursor__delete)) { + pr_err("callchain cursor creation failed"); + abort(); + } +} + +struct callchain_cursor *get_tls_callchain_cursor(void) +{ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + struct callchain_cursor *cursor; + + pthread_once(&once_control, init_callchain_cursor_key); + cursor = pthread_getspecific(callchain_cursor); + if (!cursor) { + cursor = zalloc(sizeof(*cursor)); + if (!cursor) + pr_debug3("%s: not enough memory\n", __func__); + pthread_setspecific(callchain_cursor, cursor); + } + return cursor; +} + +int callchain_cursor__copy(struct callchain_cursor *dst, + struct callchain_cursor *src) +{ + int rc = 0; + + callchain_cursor_reset(dst); + callchain_cursor_commit(src); + + while (true) { + struct callchain_cursor_node *node; + + node = callchain_cursor_current(src); + if (node == NULL) + break; + + rc = callchain_cursor_append(dst, node->ip, &node->ms, + node->branch, &node->branch_flags, + node->nr_loop_iter, + node->iter_cycles, + node->branch_from, node->srcline); + if (rc) + break; + + callchain_cursor_advance(src); + } + + return rc; +} + +/* + * Initialize a cursor before adding entries inside, but keep + * the previously allocated entries as a cache. + */ +void callchain_cursor_reset(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node; + + cursor->nr = 0; + cursor->last = &cursor->first; + + for (node = cursor->first; node != NULL; node = node->next) + map_symbol__exit(&node->ms); +} + +void callchain_param_setup(u64 sample_type, uint16_t e_machine) +{ + if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain) { + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (sample_type & PERF_SAMPLE_STACK_USER)) { + callchain_param.record_mode = CALLCHAIN_DWARF; + dwarf_callchain_users = true; + } else if (sample_type & PERF_SAMPLE_BRANCH_STACK) + callchain_param.record_mode = CALLCHAIN_LBR; + else + callchain_param.record_mode = CALLCHAIN_FP; + } + + /* + * It's necessary to use libunwind to reliably determine the caller of + * a leaf function on aarch64, as otherwise we cannot know whether to + * start from the LR or FP. + * + * Always starting from the LR can result in duplicate or entirely + * erroneous entries. Always skipping the LR and starting from the FP + * can result in missing entries. + */ + if (callchain_param.record_mode == CALLCHAIN_FP && e_machine == EM_AARCH64) + dwarf_callchain_users = true; +} + +static bool chain_match(struct callchain_list *base_chain, + struct callchain_list *pair_chain) +{ + enum match_result match; + + match = match_chain_strings(base_chain->srcline, + pair_chain->srcline); + if (match != MATCH_ERROR) + return match == MATCH_EQ; + + match = match_chain_dso_addresses(base_chain->ms.map, + base_chain->ip, + pair_chain->ms.map, + pair_chain->ip); + + return match == MATCH_EQ; +} + +bool callchain_cnode_matched(struct callchain_node *base_cnode, + struct callchain_node *pair_cnode) +{ + struct callchain_list *base_chain, *pair_chain; + bool match = false; + + pair_chain = list_first_entry(&pair_cnode->val, + struct callchain_list, + list); + + list_for_each_entry(base_chain, &base_cnode->val, list) { + if (&pair_chain->list == &pair_cnode->val) + return false; + + if (!base_chain->srcline || !pair_chain->srcline) { + pair_chain = list_next_entry(pair_chain, list); + continue; + } + + match = chain_match(base_chain, pair_chain); + if (!match) + return false; + + pair_chain = list_next_entry(pair_chain, list); + } + + /* + * Say chain1 is ABC, chain2 is ABCD, we consider they are + * not fully matched. + */ + if (pair_chain && (&pair_chain->list != &pair_cnode->val)) + return false; + + return match; +} + +static u64 count_callchain_hits(struct hist_entry *he) +{ + struct rb_root *root = &he->sorted_chain; + struct rb_node *rb_node = rb_first(root); + struct callchain_node *node; + u64 chain_hits = 0; + + while (rb_node) { + node = rb_entry(rb_node, struct callchain_node, rb_node); + chain_hits += node->hit; + rb_node = rb_next(rb_node); + } + + return chain_hits; +} + +u64 callchain_total_hits(struct hists *hists) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + u64 chain_hits = 0; + + while (next) { + struct hist_entry *he = rb_entry(next, struct hist_entry, + rb_node); + + chain_hits += count_callchain_hits(he); + next = rb_next(&he->rb_node); + } + + return chain_hits; +} + +s64 callchain_avg_cycles(struct callchain_node *cnode) +{ + struct callchain_list *chain; + s64 cycles = 0; + + list_for_each_entry(chain, &cnode->val, list) { + if (chain->srcline && chain->branch_count) + cycles += chain->cycles_count / chain->branch_count; + } + + return cycles; +} + +int sample__for_each_callchain_node(struct thread *thread, + struct perf_sample *sample, int max_stack, + bool symbols, callchain_iter_fn cb, void *data) +{ + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + int ret; + + if (!cursor) + return -ENOMEM; + + /* Fill in the callchain. */ + ret = __thread__resolve_callchain(thread, cursor, sample, + /*parent=*/NULL, /*root_al=*/NULL, + max_stack, symbols); + if (ret) + return ret; + + /* Switch from writing the callchain to reading it. */ + callchain_cursor_commit(cursor); + + while (1) { + struct callchain_cursor_node *node = callchain_cursor_current(cursor); + + if (!node) + break; + + ret = cb(node, data); + if (ret) + return ret; + + callchain_cursor_advance(cursor); + } + return 0; +} + +/* + * This function merges earlier samples (@sample_orig) waiting for deferred + * user callchains with the matching callchain record (@sample_callchain) + * which is delivered now. The @sample_orig->callchain should be released + * after use if ->deferred_callchain is set. + */ +int sample__merge_deferred_callchain(struct perf_sample *sample_orig, + struct perf_sample *sample_callchain) +{ + u64 nr_orig = sample_orig->callchain->nr - 1; + u64 nr_deferred = sample_callchain->callchain->nr; + struct ip_callchain *callchain; + + if (sample_orig->merged_callchain) { + /* Already merged. */ + return -EINVAL; + } + + if (sample_orig->callchain->nr < 2) { + sample_orig->deferred_callchain = false; + return -EINVAL; + } + + callchain = calloc(1 + nr_orig + nr_deferred, sizeof(u64)); + if (callchain == NULL) + return -ENOMEM; + + callchain->nr = nr_orig + nr_deferred; + /* copy original including PERF_CONTEXT_USER_DEFERRED (but the cookie) */ + memcpy(callchain->ips, sample_orig->callchain->ips, nr_orig * sizeof(u64)); + /* copy deferred user callchains */ + memcpy(&callchain->ips[nr_orig], sample_callchain->callchain->ips, + nr_deferred * sizeof(u64)); + + sample_orig->merged_callchain = true; + sample_orig->callchain = callchain; + return 0; +} diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h new file mode 100644 index 000000000..8b1e405d1 --- /dev/null +++ b/tools/perf/util/callchain.h @@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CALLCHAIN_H +#define __PERF_CALLCHAIN_H + +#include +#include +#include "map_symbol.h" +#include "branch.h" + +struct addr_location; +struct hist_entry; +struct hists; +struct ip_callchain; +struct perf_sample; +struct record_opts; +struct thread; + +#define HELP_PAD "\t\t\t\t" + +#define CALLCHAIN_HELP "setup and enables call-graph (stack chain/backtrace):\n\n" + +# define RECORD_MODE_HELP HELP_PAD "record_mode:\tcall graph recording mode (fp|dwarf|lbr)\n" + +#define RECORD_SIZE_HELP \ + HELP_PAD "record_size:\tif record_mode is 'dwarf', max size of stack recording ()\n" \ + HELP_PAD "\t\tdefault: 8192 (bytes)\n" + +#define CALLCHAIN_RECORD_HELP CALLCHAIN_HELP RECORD_MODE_HELP RECORD_SIZE_HELP + +#define CALLCHAIN_REPORT_HELP \ + HELP_PAD "print_type:\tcall graph printing style (graph|flat|fractal|folded|none)\n" \ + HELP_PAD "threshold:\tminimum call graph inclusion threshold ()\n" \ + HELP_PAD "print_limit:\tmaximum number of call graph entry ()\n" \ + HELP_PAD "order:\t\tcall graph order (caller|callee)\n" \ + HELP_PAD "sort_key:\tcall graph sort key (function|address)\n" \ + HELP_PAD "branch:\t\tinclude last branch info to call graph (branch)\n" \ + HELP_PAD "value:\t\tcall graph value (percent|period|count)\n" + +enum perf_call_graph_mode { + CALLCHAIN_NONE, + CALLCHAIN_FP, + CALLCHAIN_DWARF, + CALLCHAIN_LBR, + CALLCHAIN_MAX +}; + +enum chain_mode { + CHAIN_NONE, + CHAIN_FLAT, + CHAIN_GRAPH_ABS, + CHAIN_GRAPH_REL, + CHAIN_FOLDED, +}; + +enum chain_order { + ORDER_CALLER, + ORDER_CALLEE +}; + +struct callchain_node { + struct callchain_node *parent; + struct list_head val; + struct list_head parent_val; + struct rb_node rb_node_in; /* to insert nodes in an rbtree */ + struct rb_node rb_node; /* to sort nodes in an output tree */ + struct rb_root rb_root_in; /* input tree of children */ + struct rb_root rb_root; /* sorted output tree of children */ + unsigned int val_nr; + unsigned int count; + unsigned int children_count; + u64 hit; + u64 children_hit; +}; + +struct callchain_root { + u64 max_depth; + struct callchain_node node; +}; + +struct callchain_param; + +typedef void (*sort_chain_func_t)(struct rb_root *, struct callchain_root *, + u64, struct callchain_param *); + +enum chain_key { + CCKEY_FUNCTION, + CCKEY_ADDRESS, + CCKEY_SRCLINE +}; + +enum chain_value { + CCVAL_PERCENT, + CCVAL_PERIOD, + CCVAL_COUNT, +}; + +extern bool dwarf_callchain_users; + +struct callchain_param { + bool enabled; + bool defer; + enum perf_call_graph_mode record_mode; + u32 dump_size; + enum chain_mode mode; + u16 max_stack; + u32 print_limit; + double min_percent; + sort_chain_func_t sort; + enum chain_order order; + bool order_set; + enum chain_key key; + bool branch_callstack; + enum chain_value value; +}; + +extern struct callchain_param callchain_param; +extern struct callchain_param callchain_param_default; + +struct callchain_list { + struct list_head list; + u64 ip; + struct map_symbol ms; + const char *srcline; + u64 branch_count; + u64 from_count; + u64 cycles_count; + u64 iter_count; + u64 iter_cycles; + struct branch_type_stat *brtype_stat; + u64 predicted_count; + u64 abort_count; + struct /* for TUI */ { + bool unfolded; + bool has_children; + }; +}; + +/* + * A callchain cursor is a single linked list that + * let one feed a callchain progressively. + * It keeps persistent allocated entries to minimize + * allocations. + */ +struct callchain_cursor_node { + u64 ip; + struct map_symbol ms; + const char *srcline; + /* Indicate valid cursor node for LBR stitch */ + bool valid; + + bool branch; + struct branch_flags branch_flags; + u64 branch_from; + int nr_loop_iter; + u64 iter_cycles; + struct callchain_cursor_node *next; +}; + +struct stitch_list { + struct list_head node; + struct callchain_cursor_node cursor; +}; + +struct callchain_cursor { + u64 nr; + struct callchain_cursor_node *first; + struct callchain_cursor_node **last; + u64 pos; + struct callchain_cursor_node *curr; +}; + +static inline void callchain_init(struct callchain_root *root) +{ + INIT_LIST_HEAD(&root->node.val); + INIT_LIST_HEAD(&root->node.parent_val); + + root->node.parent = NULL; + root->node.hit = 0; + root->node.children_hit = 0; + root->node.rb_root_in = RB_ROOT; + root->max_depth = 0; +} + +static inline u64 callchain_cumul_hits(struct callchain_node *node) +{ + return node->hit + node->children_hit; +} + +static inline unsigned callchain_cumul_counts(struct callchain_node *node) +{ + return node->count + node->children_count; +} + +int callchain_register_param(struct callchain_param *param); +int callchain_append(struct callchain_root *root, + struct callchain_cursor *cursor, + u64 period); + +int callchain_merge(struct callchain_cursor *cursor, + struct callchain_root *dst, struct callchain_root *src); + +void callchain_cursor_reset(struct callchain_cursor *cursor); + +int callchain_cursor_append(struct callchain_cursor *cursor, u64 ip, + struct map_symbol *ms, + bool branch, struct branch_flags *flags, + int nr_loop_iter, u64 iter_cycles, u64 branch_from, + const char *srcline); + +/* Close a cursor writing session. Initialize for the reader */ +static inline void callchain_cursor_commit(struct callchain_cursor *cursor) +{ + if (cursor == NULL) + return; + cursor->curr = cursor->first; + cursor->pos = 0; +} + +/* Cursor reading iteration helpers */ +static inline struct callchain_cursor_node * +callchain_cursor_current(struct callchain_cursor *cursor) +{ + if (cursor == NULL || cursor->pos == cursor->nr) + return NULL; + + return cursor->curr; +} + +static inline void callchain_cursor_advance(struct callchain_cursor *cursor) +{ + cursor->curr = cursor->curr->next; + cursor->pos++; +} + +struct callchain_cursor *get_tls_callchain_cursor(void); + +int callchain_cursor__copy(struct callchain_cursor *dst, + struct callchain_cursor *src); + +int record_opts__parse_callchain(struct record_opts *record, + struct callchain_param *callchain, + const char *arg, bool unset); + +int sample__resolve_callchain(struct perf_sample *sample, + struct callchain_cursor *cursor, struct symbol **parent, + struct addr_location *al, + int max_stack); +int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample); +int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, + bool hide_unresolved); + +extern const char record_callchain_help[]; +int parse_callchain_record(const char *arg, struct callchain_param *param); +int parse_callchain_record_opt(const char *arg, struct callchain_param *param); +int parse_callchain_report_opt(const char *arg); +int parse_callchain_top_opt(const char *arg); +int perf_callchain_config(const char *var, const char *value); + +static inline void callchain_cursor_snapshot(struct callchain_cursor *dest, + struct callchain_cursor *src) +{ + *dest = *src; + + dest->first = src->curr; + dest->nr -= src->pos; +} + +#ifdef HAVE_SKIP_CALLCHAIN_IDX +int arch_skip_callchain_idx(struct thread *thread, struct ip_callchain *chain); +#else +static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused, + struct ip_callchain *chain __maybe_unused) +{ + return -1; +} +#endif + +char *callchain_list__sym_name(struct callchain_list *cl, + char *bf, size_t bfsize, bool show_dso); +char *callchain_node__scnprintf_value(struct callchain_node *node, + char *bf, size_t bfsize, u64 total); +int callchain_node__fprintf_value(struct callchain_node *node, + FILE *fp, u64 total); + +int callchain_list_counts__printf_value(struct callchain_list *clist, + FILE *fp, char *bf, int bfsize); + +void free_callchain(struct callchain_root *root); +void callchain_cursor_cleanup(struct callchain_cursor *cursor); +void decay_callchain(struct callchain_root *root); +int callchain_node__make_parent_list(struct callchain_node *node); + +int callchain_branch_counts(struct callchain_root *root, + u64 *branch_count, u64 *predicted_count, + u64 *abort_count, u64 *cycles_count); + +void callchain_param_setup(u64 sample_type, uint16_t e_machine); + +bool callchain_cnode_matched(struct callchain_node *base_cnode, + struct callchain_node *pair_cnode); + +u64 callchain_total_hits(struct hists *hists); + +s64 callchain_avg_cycles(struct callchain_node *cnode); + +typedef int (*callchain_iter_fn)(struct callchain_cursor_node *node, void *data); + +int sample__for_each_callchain_node(struct thread *thread, + struct perf_sample *sample, int max_stack, + bool symbols, callchain_iter_fn cb, void *data); + +int sample__merge_deferred_callchain(struct perf_sample *sample_orig, + struct perf_sample *sample_callchain); + +#endif /* __PERF_CALLCHAIN_H */ diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c new file mode 100644 index 000000000..272bd8255 --- /dev/null +++ b/tools/perf/util/cap.c @@ -0,0 +1,46 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Capability utilities + */ + +#include "cap.h" +#include "debug.h" +#include +#include +#include +#include + +#define MAX_LINUX_CAPABILITY_U32S _LINUX_CAPABILITY_U32S_3 + +bool perf_cap__capable(int cap) +{ + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3, + .pid = 0, + }; + struct __user_cap_data_struct data[MAX_LINUX_CAPABILITY_U32S] = {}; + __u32 cap_val; + + while (syscall(SYS_capget, &header, &data[0]) == -1) { + /* Retry, first attempt has set the header.version correctly. */ + if (errno == EINVAL && header.version != _LINUX_CAPABILITY_VERSION_3 && + header.version == _LINUX_CAPABILITY_VERSION_1) + continue; + + pr_debug2("capget syscall failed (%m) fall back on root check\n"); + return geteuid() == 0; + } + + /* Extract the relevant capability bit. */ + if (cap >= 32) { + if (header.version == _LINUX_CAPABILITY_VERSION_3) { + cap_val = data[1].effective; + } else { + /* Capability beyond 32 is requested but only 32 are supported. */ + return false; + } + } else { + cap_val = data[0].effective; + } + return (cap_val & (1 << (cap & 0x1f))) != 0; +} diff --git a/tools/perf/util/cap.h b/tools/perf/util/cap.h new file mode 100644 index 000000000..bf09fb20c --- /dev/null +++ b/tools/perf/util/cap.h @@ -0,0 +1,23 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CAP_H +#define __PERF_CAP_H + +#include +#include + +/* For older systems */ +#ifndef CAP_SYSLOG +#define CAP_SYSLOG 34 +#endif + +#ifndef CAP_PERFMON +#define CAP_PERFMON 38 +#endif + +#ifndef CAP_BPF +#define CAP_BPF 39 +#endif + +bool perf_cap__capable(int cap); + +#endif /* __PERF_CAP_H */ diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c new file mode 100644 index 000000000..74213daf8 --- /dev/null +++ b/tools/perf/util/capstone.c @@ -0,0 +1,653 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "capstone.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "addr_location.h" +#include "annotate.h" +#include "debug.h" +#include "disasm.h" +#include "dso.h" +#include "machine.h" +#include "map.h" +#include "namespaces.h" +#include "print_insn.h" +#include "symbol.h" +#include "thread.h" + +#if CS_VERSION_MAJOR < 5 +#define CS_ARCH_RISCV 15 +#define CS_MODE_RISCV32 1 +#define CS_MODE_RISCV64 2 +#define CS_MODE_RISCVC 4 +#endif + +#ifdef LIBCAPSTONE_DLOPEN +static void *perf_cs_dll_handle(void) +{ + static bool dll_handle_init; + static void *dll_handle; + + if (!dll_handle_init) { + dll_handle_init = true; + dll_handle = dlopen("libcapstone.so", RTLD_LAZY); + if (!dll_handle) + pr_debug("dlopen failed for libcapstone.so\n"); + } + return dll_handle; +} +#endif + +static enum cs_err perf_cs_open(enum cs_arch arch, enum cs_mode mode, csh *handle) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_open(arch, mode, handle); +#else + static bool fn_init; + static enum cs_err (*fn)(enum cs_arch arch, enum cs_mode mode, csh *handle); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_open"); + if (!fn) + pr_debug("dlsym failed for cs_open\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(arch, mode, handle); +#endif +} + +static enum cs_err perf_cs_option(csh handle, enum cs_opt_type type, size_t value) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_option(handle, type, value); +#else + static bool fn_init; + static enum cs_err (*fn)(csh handle, enum cs_opt_type type, size_t value); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_option"); + if (!fn) + pr_debug("dlsym failed for cs_option\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(handle, type, value); +#endif +} + +static size_t perf_cs_disasm(csh handle, const uint8_t *code, size_t code_size, + uint64_t address, size_t count, struct cs_insn **insn) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_disasm(handle, code, code_size, address, count, insn); +#else + static bool fn_init; + static enum cs_err (*fn)(csh handle, const uint8_t *code, size_t code_size, + uint64_t address, size_t count, struct cs_insn **insn); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_disasm"); + if (!fn) + pr_debug("dlsym failed for cs_disasm\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(handle, code, code_size, address, count, insn); +#endif +} + +static void perf_cs_free(struct cs_insn *insn, size_t count) +{ +#ifndef LIBCAPSTONE_DLOPEN + cs_free(insn, count); +#else + static bool fn_init; + static void (*fn)(struct cs_insn *insn, size_t count); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_free"); + if (!fn) + pr_debug("dlsym failed for cs_free\n"); + fn_init = true; + } + if (!fn) + return; + fn(insn, count); +#endif +} + +static enum cs_err perf_cs_close(csh *handle) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_close(handle); +#else + static bool fn_init; + static enum cs_err (*fn)(csh *handle); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_close"); + if (!fn) + pr_debug("dlsym failed for cs_close\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(handle); +#endif +} + +static bool e_machine_to_capstone(uint16_t e_machine, bool is64, bool is_big_endian, + enum cs_arch *arch, enum cs_mode *mode) +{ + *mode = is_big_endian ? CS_MODE_BIG_ENDIAN : CS_MODE_LITTLE_ENDIAN; + + switch (e_machine) { + case EM_X86_64: + case EM_386: + *arch = CS_ARCH_X86; + *mode |= is64 ? CS_MODE_64 : CS_MODE_32; + return true; + case EM_AARCH64: + *arch = CS_ARCH_ARM64; + *mode |= CS_MODE_ARM; + return true; + case EM_ARM: + *arch = CS_ARCH_ARM; + *mode |= CS_MODE_ARM | CS_MODE_V8; + return true; + case EM_S390: + *arch = CS_ARCH_SYSZ; + return true; + case EM_MIPS: + *arch = CS_ARCH_MIPS; + *mode |= is64 ? CS_MODE_MIPS64 : CS_MODE_MIPS32; + return true; + case EM_PPC: + *arch = CS_ARCH_PPC; + return true; + case EM_PPC64: + *arch = CS_ARCH_PPC; + *mode |= CS_MODE_64; + return true; + case EM_SPARC: + *arch = CS_ARCH_SPARC; + return true; + case EM_SPARCV9: + *arch = CS_ARCH_SPARC; + *mode |= CS_MODE_V9; + return true; + case EM_RISCV: + *arch = CS_ARCH_RISCV; + *mode |= (is64 ? CS_MODE_RISCV64 : CS_MODE_RISCV32) | CS_MODE_RISCVC; + return true; + default: + return false; + } +} + +static int capstone_init(uint16_t e_machine, csh *cs_handle, bool is64, bool is_big_endian, + bool disassembler_style) +{ + enum cs_arch arch; + enum cs_mode mode; + + if (!e_machine_to_capstone(e_machine, is64, is_big_endian, &arch, &mode)) + return -1; + + if (perf_cs_open(arch, mode, cs_handle) != CS_ERR_OK) { + pr_warning_once("cs_open failed\n"); + return -1; + } + + if (arch == CS_ARCH_X86) { + /* + * In case of using capstone_init while symbol__disassemble + * setting CS_OPT_SYNTAX_ATT depends if disassembler_style opts + * is set via annotation args + */ + if (disassembler_style) + perf_cs_option(*cs_handle, CS_OPT_SYNTAX, CS_OPT_SYNTAX_ATT); + /* + * Resolving address operands to symbols is implemented + * on x86 by investigating instruction details. + */ + perf_cs_option(*cs_handle, CS_OPT_DETAIL, CS_OPT_ON); + } + + return 0; +} + +static size_t print_insn_x86(struct thread *thread, u8 cpumode, struct cs_insn *insn, + int print_opts, FILE *fp) +{ + struct addr_location al; + size_t printed = 0; + + if (insn->detail && insn->detail->x86.op_count == 1) { + struct cs_x86_op *op = &insn->detail->x86.operands[0]; + + addr_location__init(&al); + if (op->type == X86_OP_IMM && + thread__find_symbol(thread, cpumode, op->imm, &al)) { + printed += fprintf(fp, "%s ", insn[0].mnemonic); + printed += symbol__fprintf_symname_offs(al.sym, &al, fp); + if (print_opts & PRINT_INSN_IMM_HEX) + printed += fprintf(fp, " [%#" PRIx64 "]", op->imm); + addr_location__exit(&al); + return printed; + } + addr_location__exit(&al); + } + + printed += fprintf(fp, "%s %s", insn[0].mnemonic, insn[0].op_str); + return printed; +} + +ssize_t capstone__fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, + bool is64bit, const uint8_t *code, size_t code_size, uint64_t ip, + int *lenp, int print_opts, FILE *fp) +{ + size_t printed; + struct cs_insn *insn; + csh cs_handle; + size_t count; + bool is_big_endian = false; + uint16_t e_machine = thread__e_machine_endian(thread, machine, + /*e_flags=*/NULL, &is_big_endian); + int ret; + + /* TODO: Try to initiate capstone only once but need a proper place. */ + ret = capstone_init(e_machine, &cs_handle, is64bit, is_big_endian, + /*disassembler_style=*/true); + if (ret < 0) + return ret; + + count = perf_cs_disasm(cs_handle, code, code_size, ip, 1, &insn); + if (count > 0) { + if (e_machine == EM_X86_64 || e_machine == EM_386) + printed = print_insn_x86(thread, cpumode, &insn[0], print_opts, fp); + else + printed = fprintf(fp, "%s %s", insn[0].mnemonic, insn[0].op_str); + if (lenp) + *lenp = insn->size; + perf_cs_free(insn, count); + } else { + printed = -1; + } + + perf_cs_close(&cs_handle); + return printed; +} + +static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, + struct annotate_args *args, u64 addr) +{ + int i; + struct map *map = args->ms->map; + struct symbol *sym; + + /* TODO: support more architectures */ + if (!arch__is_x86(args->arch)) + return; + + if (insn->detail == NULL) + return; + + for (i = 0; i < insn->detail->x86.op_count; i++) { + struct cs_x86_op *op = &insn->detail->x86.operands[i]; + u64 orig_addr; + struct map *found_map = NULL; + + if (op->type != X86_OP_MEM) + continue; + + /* only print RIP-based global symbols for now */ + if (op->mem.base != X86_REG_RIP) + continue; + + /* get the target address */ + orig_addr = addr + insn->size + op->mem.disp; + addr = map__objdump_2mem(map, orig_addr); + + if (dso__kernel(map__dso(map))) { + /* + * The kernel maps can be split into sections, let's + * find the map first and then search the symbol. + */ + found_map = maps__find(map__kmaps(map), addr); + if (found_map == NULL) + continue; + map = found_map; + } + + /* convert it to map-relative address for search */ + addr = map__map_ip(map, addr); + + sym = map__find_symbol(map, addr); + if (sym == NULL) { + map__put(found_map); + continue; + } + + if (addr == sym->start) { + scnprintf(buf, len, "\t# %"PRIx64" <%s>", + orig_addr, sym->name); + } else { + scnprintf(buf, len, "\t# %"PRIx64" <%s+%#"PRIx64">", + orig_addr, sym->name, addr - sym->start); + } + map__put(found_map); + break; + } +} + +struct find_file_offset_data { + u64 ip; + u64 offset; +}; + +/* This will be called for each PHDR in an ELF binary */ +static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) +{ + struct find_file_offset_data *data = arg; + + if (start <= data->ip && data->ip < start + len) { + data->offset = pgoff + data->ip - start; + return 1; + } + return 0; +} + +int symbol__disassemble_capstone(const char *filename, struct symbol *sym, + struct annotate_args *args) +{ + struct annotation *notes = symbol__annotation(sym); + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + u64 start = map__rip_2objdump(map, sym->start); + u64 offset; + int i, count, free_count; + bool is_64bit = false; + bool needs_cs_close = false; + /* Malloc-ed buffer containing instructions read from disk. */ + u8 *code_buf = NULL; + /* Pointer to code to be disassembled. */ + const u8 *buf; + u64 buf_len; + csh handle; + struct cs_insn *insn = NULL; + char disasm_buf[512]; + struct disasm_line *dl; + bool disassembler_style = false; + uint16_t e_machine = EM_NONE; + bool is_big_endian = false; + + if (args->options->objdump_path) + return -1; + + buf = dso__read_symbol(dso, filename, map, sym, + &code_buf, &buf_len, &is_64bit); + if (buf == NULL) + return errno; + + /* add the function address and name */ + scnprintf(disasm_buf, sizeof(disasm_buf), "%#"PRIx64" <%s>:", + start, sym->name); + + args->offset = -1; + args->line = disasm_buf; + args->line_nr = 0; + args->fileloc = NULL; + args->ms->sym = sym; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + if (!args->options->disassembler_style || + !strcmp(args->options->disassembler_style, "att")) + disassembler_style = true; + + if (args->ms->thread) { + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine_endian(dso, kmap_machine, /*e_flags=*/NULL, + &is_big_endian); + } + if (!e_machine || e_machine == EM_NONE) { + e_machine = thread__e_machine_endian(NULL, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) + goto err; + + needs_cs_close = true; + + free_count = count = perf_cs_disasm(handle, buf, buf_len, start, buf_len, &insn); + for (i = 0, offset = 0; i < count; i++) { + int printed; + + printed = scnprintf(disasm_buf, sizeof(disasm_buf), + " %-7s %s", + insn[i].mnemonic, insn[i].op_str); + print_capstone_detail(&insn[i], disasm_buf + printed, + sizeof(disasm_buf) - printed, args, + start + offset); + + args->offset = offset; + args->line = disasm_buf; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + offset += insn[i].size; + } + + /* It failed in the middle: probably due to unknown instructions */ + if (offset != buf_len) { + struct list_head *list = ¬es->src->source; + + /* Discard all lines and fallback to objdump */ + while (!list_empty(list)) { + dl = list_first_entry(list, struct disasm_line, al.node); + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } + count = -1; + } + +out: + if (needs_cs_close) { + perf_cs_close(&handle); + if (free_count > 0) + perf_cs_free(insn, free_count); + } + free(code_buf); + return count < 0 ? count : 0; + +err: + if (needs_cs_close) { + struct disasm_line *tmp; + + /* + * It probably failed in the middle of the above loop. + * Release any resources it might add. + */ + list_for_each_entry_safe(dl, tmp, ¬es->src->source, al.node) { + list_del(&dl->al.node); + disasm_line__free(dl); + } + } + count = -1; + goto out; +} + +int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, + struct symbol *sym __maybe_unused, + struct annotate_args *args __maybe_unused) +{ + struct annotation *notes = symbol__annotation(sym); + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + struct nscookie nsc; + u64 start = map__rip_2objdump(map, sym->start); + u64 end = map__rip_2objdump(map, sym->end); + u64 len = end - start; + u64 offset; + int i, fd, count; + bool is_64bit = false; + bool needs_cs_close = false; + u8 *buf = NULL; + struct find_file_offset_data data = { + .ip = start, + }; + csh handle; + char disasm_buf[512]; + struct disasm_line *dl; + u32 *line; + bool disassembler_style = false; + uint16_t e_machine = EM_NONE; + bool is_big_endian = false; + + if (args->options->objdump_path) + return -1; + + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + fd = open(filename, O_RDONLY); + nsinfo__mountns_exit(&nsc); + if (fd < 0) + return -1; + + if (file__read_maps(fd, /*exe=*/true, find_file_offset, &data, + &is_64bit) == 0) + goto err; + + if (!args->options->disassembler_style || + !strcmp(args->options->disassembler_style, "att")) + disassembler_style = true; + + if (args->ms->thread) { + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } else if (dso) { + struct maps *kmaps = (map && dso__kernel(dso)) ? map__kmaps(map) : NULL; + struct machine *kmap_machine = kmaps ? maps__machine(kmaps) : NULL; + + e_machine = dso__e_machine_endian(dso, kmap_machine, /*e_flags=*/NULL, + &is_big_endian); + } + if (!e_machine || e_machine == EM_NONE) { + e_machine = thread__e_machine_endian(NULL, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + } + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) + goto err; + + needs_cs_close = true; + + buf = malloc(len); + if (buf == NULL) + goto err; + + count = pread(fd, buf, len, data.offset); + close(fd); + fd = -1; + + if ((u64)count != len) + goto err; + + line = (u32 *)buf; + + /* add the function address and name */ + scnprintf(disasm_buf, sizeof(disasm_buf), "%#"PRIx64" <%s>:", + start, sym->name); + + args->offset = -1; + args->line = disasm_buf; + args->line_nr = 0; + args->fileloc = NULL; + args->ms->sym = sym; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + /* + * TODO: enable disassm for powerpc + * count = cs_disasm(handle, buf, len, start, len, &insn); + * + * For now, only binary code is saved in disassembled line + * to be used in "type" and "typeoff" sort keys. Each raw code + * is 32 bit instruction. So use "len/4" to get the number of + * entries. + */ + count = len/4; + + for (i = 0, offset = 0; i < count; i++) { + args->offset = offset; + sprintf(args->line, "%x", line[i]); + + dl = disasm_line__new(args); + if (dl == NULL) + break; + + annotation_line__add(&dl->al, ¬es->src->source); + + offset += 4; + } + + /* It failed in the middle */ + if (offset != len) { + struct list_head *list = ¬es->src->source; + + /* Discard all lines and fallback to objdump */ + while (!list_empty(list)) { + dl = list_first_entry(list, struct disasm_line, al.node); + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } + count = -1; + } + +out: + if (needs_cs_close) + perf_cs_close(&handle); + free(buf); + return count < 0 ? count : 0; + +err: + if (fd >= 0) + close(fd); + count = -1; + goto out; +} diff --git a/tools/perf/util/capstone.h b/tools/perf/util/capstone.h new file mode 100644 index 000000000..7c0baaa01 --- /dev/null +++ b/tools/perf/util/capstone.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CAPSTONE_H +#define __PERF_CAPSTONE_H + +#include +#include +#include +#include +#include +#include + +struct annotate_args; +struct machine; +struct symbol; +struct thread; + +#ifdef HAVE_LIBCAPSTONE_SUPPORT +ssize_t capstone__fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, + bool is64bit, const uint8_t *code, size_t code_size, + uint64_t ip, int *lenp, int print_opts, FILE *fp); +int symbol__disassemble_capstone(const char *filename, struct symbol *sym, + struct annotate_args *args); +int symbol__disassemble_capstone_powerpc(const char *filename, struct symbol *sym, + struct annotate_args *args); + +#else /* !HAVE_LIBCAPSTONE_SUPPORT */ +static inline ssize_t capstone__fprintf_insn_asm(struct machine *machine __maybe_unused, + struct thread *thread __maybe_unused, + u8 cpumode __maybe_unused, + bool is64bit __maybe_unused, + const uint8_t *code __maybe_unused, + size_t code_size __maybe_unused, + uint64_t ip __maybe_unused, + int *lenp __maybe_unused, + int print_opts __maybe_unused, + FILE *fp __maybe_unused) +{ + return -1; +} + +static inline int symbol__disassemble_capstone(const char *filename __maybe_unused, + struct symbol *sym __maybe_unused, + struct annotate_args *args __maybe_unused) +{ + return -1; +} + +static inline int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, + struct symbol *sym __maybe_unused, + struct annotate_args *args __maybe_unused) +{ + return -1; +} + +#endif /* HAVE_LIBCAPSTONE_SUPPORT */ + +#endif /* __PERF_CAPSTONE_H */ diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c new file mode 100644 index 000000000..c7be16a79 --- /dev/null +++ b/tools/perf/util/cgroup.c @@ -0,0 +1,646 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "evsel.h" +#include "cgroup.h" +#include "evlist.h" +#include "rblist.h" +#include "metricgroup.h" +#include "stat.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int nr_cgroups; +bool cgrp_event_expanded; + +/* used to match cgroup name with patterns */ +struct cgroup_name { + struct list_head list; + bool used; + char name[]; +}; +static LIST_HEAD(cgroup_list); + +static int open_cgroup(const char *name) +{ + char path[PATH_MAX + 1]; + char mnt[PATH_MAX + 1]; + int fd; + + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, "perf_event")) + return -1; + + scnprintf(path, PATH_MAX, "%s/%s", mnt, name); + + fd = open(path, O_RDONLY); + if (fd == -1) + fprintf(stderr, "no access to cgroup %s\n", path); + + return fd; +} + +#ifdef HAVE_FILE_HANDLE +static u64 __read_cgroup_id(const char *path) +{ + struct { + struct file_handle fh; + uint64_t cgroup_id; + } handle; + int mount_id; + + handle.fh.handle_bytes = sizeof(handle.cgroup_id); + if (name_to_handle_at(AT_FDCWD, path, &handle.fh, &mount_id, 0) < 0) + return -1ULL; + + return handle.cgroup_id; +} + +int read_cgroup_id(struct cgroup *cgrp) +{ + char path[PATH_MAX + 1]; + char mnt[PATH_MAX + 1]; + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, "perf_event")) + return -1; + + scnprintf(path, PATH_MAX, "%s/%s", mnt, cgrp->name); + + cgrp->id = __read_cgroup_id(path); + return 0; +} +#else +static inline u64 __read_cgroup_id(const char *path __maybe_unused) { return -1ULL; } +#endif /* HAVE_FILE_HANDLE */ + +#ifndef CGROUP2_SUPER_MAGIC +#define CGROUP2_SUPER_MAGIC 0x63677270 +#endif + +int cgroup_is_v2(const char *subsys) +{ + char mnt[PATH_MAX + 1]; + struct statfs stbuf; + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, subsys)) + return -1; + + if (statfs(mnt, &stbuf) < 0) + return -1; + + return (stbuf.f_type == CGROUP2_SUPER_MAGIC); +} + +static struct cgroup *evlist__find_cgroup(struct evlist *evlist, const char *str) +{ + struct evsel *counter; + /* + * check if cgrp is already defined, if so we reuse it + */ + evlist__for_each_entry(evlist, counter) { + if (!counter->cgrp) + continue; + if (!strcmp(counter->cgrp->name, str)) + return cgroup__get(counter->cgrp); + } + + return NULL; +} + +struct cgroup *cgroup__new(const char *name, bool do_open) +{ + struct cgroup *cgroup = zalloc(sizeof(*cgroup)); + + if (cgroup != NULL) { + refcount_set(&cgroup->refcnt, 1); + + cgroup->name = strdup(name); + if (!cgroup->name) + goto out_err; + + if (do_open) { + cgroup->fd = open_cgroup(name); + if (cgroup->fd == -1) + goto out_free_name; + } else { + cgroup->fd = -1; + } + } + + return cgroup; + +out_free_name: + zfree(&cgroup->name); +out_err: + free(cgroup); + return NULL; +} + +struct cgroup *evlist__findnew_cgroup(struct evlist *evlist, const char *name) +{ + struct cgroup *cgroup = evlist__find_cgroup(evlist, name); + + return cgroup ?: cgroup__new(name, true); +} + +static int add_cgroup(struct evlist *evlist, const char *str) +{ + struct evsel *counter; + struct cgroup *cgrp = evlist__findnew_cgroup(evlist, str); + int n; + + if (!cgrp) + return -1; + /* + * find corresponding event + * if add cgroup N, then need to find event N + */ + n = 0; + evlist__for_each_entry(evlist, counter) { + if (n == nr_cgroups) + goto found; + n++; + } + + cgroup__put(cgrp); + return -1; +found: + counter->cgrp = cgrp; + return 0; +} + +static void cgroup__delete(struct cgroup *cgroup) +{ + if (cgroup->fd >= 0) + close(cgroup->fd); + zfree(&cgroup->name); + free(cgroup); +} + +void cgroup__put(struct cgroup *cgrp) +{ + if (cgrp && refcount_dec_and_test(&cgrp->refcnt)) { + cgroup__delete(cgrp); + } +} + +struct cgroup *cgroup__get(struct cgroup *cgroup) +{ + if (cgroup) + refcount_inc(&cgroup->refcnt); + return cgroup; +} + +static void evsel__set_default_cgroup(struct evsel *evsel, struct cgroup *cgroup) +{ + if (evsel->cgrp == NULL) + evsel->cgrp = cgroup__get(cgroup); +} + +void evlist__set_default_cgroup(struct evlist *evlist, struct cgroup *cgroup) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) + evsel__set_default_cgroup(evsel, cgroup); +} + +/* helper function for ftw() in match_cgroups and list_cgroups */ +static int add_cgroup_name(const char *fpath, const struct stat *sb __maybe_unused, + int typeflag, struct FTW *ftwbuf __maybe_unused) +{ + struct cgroup_name *cn; + + if (typeflag != FTW_D) + return 0; + + cn = malloc(sizeof(*cn) + strlen(fpath) + 1); + if (cn == NULL) + return -1; + + cn->used = false; + strcpy(cn->name, fpath); + + list_add_tail(&cn->list, &cgroup_list); + return 0; +} + +static int check_and_add_cgroup_name(const char *fpath) +{ + struct cgroup_name *cn; + + list_for_each_entry(cn, &cgroup_list, list) { + if (!strcmp(cn->name, fpath)) + return 0; + } + + /* pretend if it's added by ftw() */ + return add_cgroup_name(fpath, NULL, FTW_D, NULL); +} + +static void release_cgroup_list(void) +{ + struct cgroup_name *cn; + + while (!list_empty(&cgroup_list)) { + cn = list_first_entry(&cgroup_list, struct cgroup_name, list); + list_del(&cn->list); + free(cn); + } +} + +/* collect given cgroups only */ +static int list_cgroups(const char *str) +{ + const char *p, *e, *eos = str + strlen(str); + struct cgroup_name *cn; + char *s; + + /* use given name as is when no regex is given */ + for (;;) { + p = strchr(str, ','); + e = p ? p : eos; + + if (e - str) { + int ret; + + s = strndup(str, e - str); + if (!s) + return -1; + + ret = check_and_add_cgroup_name(s); + free(s); + if (ret < 0) + return -1; + } else { + if (check_and_add_cgroup_name("/") < 0) + return -1; + } + + if (!p) + break; + str = p+1; + } + + /* these groups will be used */ + list_for_each_entry(cn, &cgroup_list, list) + cn->used = true; + + return 0; +} + +/* collect all cgroups first and then match with the pattern */ +static int match_cgroups(const char *str) +{ + char mnt[PATH_MAX]; + const char *p, *e, *eos = str + strlen(str); + struct cgroup_name *cn; + regex_t reg; + int prefix_len; + char *s; + + if (cgroupfs_find_mountpoint(mnt, sizeof(mnt), "perf_event")) + return -1; + + /* cgroup_name will have a full path, skip the root directory */ + prefix_len = strlen(mnt); + + /* collect all cgroups in the cgroup_list */ + if (nftw(mnt, add_cgroup_name, 20, 0) < 0) + return -1; + + for (;;) { + p = strchr(str, ','); + e = p ? p : eos; + + /* allow empty cgroups, i.e., skip */ + if (e - str) { + /* termination added */ + s = strndup(str, e - str); + if (!s) + return -1; + if (regcomp(®, s, REG_NOSUB)) { + free(s); + return -1; + } + + /* check cgroup name with the pattern */ + list_for_each_entry(cn, &cgroup_list, list) { + char *name = cn->name + prefix_len; + + if (name[0] == '/' && name[1]) + name++; + if (!regexec(®, name, 0, NULL, 0)) + cn->used = true; + } + regfree(®); + free(s); + } else { + /* first entry to root cgroup */ + cn = list_first_entry(&cgroup_list, struct cgroup_name, + list); + cn->used = true; + } + + if (!p) + break; + str = p+1; + } + return prefix_len; +} + +int parse_cgroups(const struct option *opt, const char *str, + int unset __maybe_unused) +{ + struct evlist *evlist = *(struct evlist **)opt->value; + struct evsel *counter; + struct cgroup *cgrp = NULL; + const char *p, *e, *eos = str + strlen(str); + char *s; + int ret, i; + + if (list_empty(&evlist__core(evlist)->entries)) { + fprintf(stderr, "must define events before cgroups\n"); + return -1; + } + + for (;;) { + p = strchr(str, ','); + e = p ? p : eos; + + /* allow empty cgroups, i.e., skip */ + if (e - str) { + /* termination added */ + s = strndup(str, e - str); + if (!s) + return -1; + ret = add_cgroup(evlist, s); + free(s); + if (ret) + return -1; + } + /* nr_cgroups is increased een for empty cgroups */ + nr_cgroups++; + if (!p) + break; + str = p+1; + } + /* for the case one cgroup combine to multiple events */ + i = 0; + if (nr_cgroups == 1) { + evlist__for_each_entry(evlist, counter) { + if (i == 0) + cgrp = counter->cgrp; + else { + counter->cgrp = cgrp; + refcount_inc(&cgrp->refcnt); + } + i++; + } + } + return 0; +} + +static bool has_pattern_string(const char *str) +{ + return !!strpbrk(str, "{}[]()|*+?^$"); +} + +int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgroup) +{ + struct evlist *orig_list, *tmp_list; + struct rblist orig_metric_events; + struct cgroup *cgrp = NULL; + struct cgroup_name *cn; + int ret = -1; + int prefix_len; + + if (evlist__nr_entries(evlist) == 0) { + fprintf(stderr, "must define events before cgroups\n"); + return -EINVAL; + } + + orig_list = evlist__new(); + tmp_list = evlist__new(); + if (orig_list == NULL || tmp_list == NULL) { + fprintf(stderr, "memory allocation failed\n"); + return -ENOMEM; + } + + /* save original events and init evlist */ + evlist__splice_list_tail(orig_list, &evlist__core(evlist)->entries); + evlist__core(evlist)->nr_entries = 0; + + orig_metric_events = *evlist__metric_events(evlist); + metricgroup__rblist_init(evlist__metric_events(evlist)); + + if (has_pattern_string(str)) + prefix_len = match_cgroups(str); + else + prefix_len = list_cgroups(str); + + if (prefix_len < 0) + goto out_err; + + list_for_each_entry(cn, &cgroup_list, list) { + struct evsel *pos; + char *name; + + if (!cn->used) + continue; + + /* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; + if (name[0] == '/' && name[1]) + name++; + + /* the cgroup can go away in the meantime */ + cgrp = cgroup__new(name, open_cgroup); + if (cgrp == NULL) + continue; + + /* copy the list and set to the new cgroup. */ + evlist__for_each_entry(orig_list, pos) { + struct evsel *evsel = evsel__clone(pos); + + if (evsel == NULL) + goto out_err; + + /* stash the copy during the copying. */ + pos->priv = evsel; + cgroup__put(evsel->cgrp); + evsel->cgrp = cgroup__get(cgrp); + + evlist__add(tmp_list, evsel); + } + /* update leader information using stashed pointer to copy. */ + evlist__for_each_entry(orig_list, pos) { + struct evsel *evsel = pos->priv; + + if (evsel__leader(pos)) + evsel__set_leader(evsel, evsel__leader(pos)->priv); + + if (pos->metric_leader) + evsel->metric_leader = pos->metric_leader->priv; + + if (pos->first_wildcard_match) + evsel->first_wildcard_match = pos->first_wildcard_match->priv; + } + /* the stashed copy is no longer used. */ + evlist__for_each_entry(orig_list, pos) + pos->priv = NULL; + + /* cgroup__new() has a refcount, release it here */ + cgroup__put(cgrp); + nr_cgroups++; + + if (metricgroup__copy_metric_events(tmp_list, cgrp, + evlist__metric_events(evlist), + &orig_metric_events) < 0) + goto out_err; + + evlist__splice_list_tail(evlist, &evlist__core(tmp_list)->entries); + evlist__core(tmp_list)->nr_entries = 0; + } + + if (list_empty(&evlist__core(evlist)->entries)) { + fprintf(stderr, "no cgroup matched: %s\n", str); + goto out_err; + } + + ret = 0; + cgrp_event_expanded = true; + +out_err: + evlist__put(orig_list); + evlist__put(tmp_list); + metricgroup__rblist_exit(&orig_metric_events); + release_cgroup_list(); + + return ret; +} + +static struct cgroup *__cgroup__findnew(struct rb_root *root, uint64_t id, + bool create, const char *path) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct cgroup *cgrp; + + while (*p != NULL) { + parent = *p; + cgrp = rb_entry(parent, struct cgroup, node); + + if (cgrp->id == id) + return cgrp; + + if (cgrp->id < id) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + if (!create) + return NULL; + + cgrp = malloc(sizeof(*cgrp)); + if (cgrp == NULL) + return NULL; + + cgrp->name = strdup(path); + if (cgrp->name == NULL) { + free(cgrp); + return NULL; + } + + cgrp->fd = -1; + cgrp->id = id; + refcount_set(&cgrp->refcnt, 1); + + rb_link_node(&cgrp->node, parent, p); + rb_insert_color(&cgrp->node, root); + + return cgrp; +} + +struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id, + const char *path) +{ + struct cgroup *cgrp; + + down_write(&env->cgroups.lock); + cgrp = __cgroup__findnew(&env->cgroups.tree, id, true, path); + up_write(&env->cgroups.lock); + return cgrp; +} + +struct cgroup *__cgroup__find(struct rb_root *root, uint64_t id) +{ + return __cgroup__findnew(root, id, /*create=*/false, /*path=*/NULL); +} + +struct cgroup *cgroup__find(struct perf_env *env, uint64_t id) +{ + struct cgroup *cgrp; + + down_read(&env->cgroups.lock); + cgrp = __cgroup__findnew(&env->cgroups.tree, id, false, NULL); + up_read(&env->cgroups.lock); + return cgrp; +} + +void perf_env__purge_cgroups(struct perf_env *env) +{ + struct rb_node *node; + struct cgroup *cgrp; + + down_write(&env->cgroups.lock); + while (!RB_EMPTY_ROOT(&env->cgroups.tree)) { + node = rb_first(&env->cgroups.tree); + cgrp = rb_entry(node, struct cgroup, node); + + rb_erase(node, &env->cgroups.tree); + cgroup__put(cgrp); + } + up_write(&env->cgroups.lock); +} + +void read_all_cgroups(struct rb_root *root) +{ + char mnt[PATH_MAX]; + struct cgroup_name *cn; + int prefix_len; + + if (cgroupfs_find_mountpoint(mnt, sizeof(mnt), "perf_event")) + return; + + /* cgroup_name will have a full path, skip the root directory */ + prefix_len = strlen(mnt); + + /* collect all cgroups in the cgroup_list */ + if (nftw(mnt, add_cgroup_name, 20, 0) < 0) + return; + + list_for_each_entry(cn, &cgroup_list, list) { + const char *name; + u64 cgrp_id; + + /* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; + if (name[0] == '\0') + name = "/"; + + cgrp_id = __read_cgroup_id(cn->name); + __cgroup__findnew(root, cgrp_id, /*create=*/true, name); + } + + release_cgroup_list(); +} diff --git a/tools/perf/util/cgroup.h b/tools/perf/util/cgroup.h new file mode 100644 index 000000000..7b1bda228 --- /dev/null +++ b/tools/perf/util/cgroup.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __CGROUP_H__ +#define __CGROUP_H__ + +#include +#include +#include +#include "util/env.h" + +struct option; + +struct cgroup { + struct rb_node node; + u64 id; + char *name; + int fd; + refcount_t refcnt; +}; + +extern int nr_cgroups; /* number of explicit cgroups defined */ +extern bool cgrp_event_expanded; + +struct cgroup *cgroup__get(struct cgroup *cgroup); +void cgroup__put(struct cgroup *cgroup); + +struct evlist; +struct rblist; + +struct cgroup *cgroup__new(const char *name, bool do_open); +struct cgroup *evlist__findnew_cgroup(struct evlist *evlist, const char *name); +int evlist__expand_cgroup(struct evlist *evlist, const char *cgroups, bool open_cgroup); + +void evlist__set_default_cgroup(struct evlist *evlist, struct cgroup *cgroup); + +int parse_cgroups(const struct option *opt, const char *str, int unset); + +struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id, + const char *path); +struct cgroup *cgroup__find(struct perf_env *env, uint64_t id); +struct cgroup *__cgroup__find(struct rb_root *root, uint64_t id); + +void perf_env__purge_cgroups(struct perf_env *env); + +#ifdef HAVE_FILE_HANDLE +int read_cgroup_id(struct cgroup *cgrp); +#else +static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) +{ + return -1; +} +#endif /* HAVE_FILE_HANDLE */ + +/* read all cgroups in the system and save them in the rbtree */ +void read_all_cgroups(struct rb_root *root); + +int cgroup_is_v2(const char *subsys); + +#endif /* __CGROUP_H__ */ diff --git a/tools/perf/util/clockid.c b/tools/perf/util/clockid.c new file mode 100644 index 000000000..74365a5d9 --- /dev/null +++ b/tools/perf/util/clockid.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include "debug.h" +#include "clockid.h" +#include "record.h" + +struct clockid_map { + const char *name; + int clockid; +}; + +#define CLOCKID_MAP(n, c) \ + { .name = n, .clockid = (c), } + +#define CLOCKID_END { .name = NULL, } + + +/* + * Add the missing ones, we need to build on many distros... + */ +#ifndef CLOCK_MONOTONIC_RAW +#define CLOCK_MONOTONIC_RAW 4 +#endif +#ifndef CLOCK_BOOTTIME +#define CLOCK_BOOTTIME 7 +#endif +#ifndef CLOCK_TAI +#define CLOCK_TAI 11 +#endif + +static const struct clockid_map clockids[] = { + /* available for all events, NMI safe */ + CLOCKID_MAP("monotonic", CLOCK_MONOTONIC), + CLOCKID_MAP("monotonic_raw", CLOCK_MONOTONIC_RAW), + + /* available for some events */ + CLOCKID_MAP("realtime", CLOCK_REALTIME), + CLOCKID_MAP("boottime", CLOCK_BOOTTIME), + CLOCKID_MAP("tai", CLOCK_TAI), + + /* available for the lazy */ + CLOCKID_MAP("mono", CLOCK_MONOTONIC), + CLOCKID_MAP("raw", CLOCK_MONOTONIC_RAW), + CLOCKID_MAP("real", CLOCK_REALTIME), + CLOCKID_MAP("boot", CLOCK_BOOTTIME), + + CLOCKID_END, +}; + +static int get_clockid_res(clockid_t clk_id, u64 *res_ns) +{ + struct timespec res; + + *res_ns = 0; + if (!clock_getres(clk_id, &res)) + *res_ns = res.tv_nsec + res.tv_sec * NSEC_PER_SEC; + else + pr_warning("WARNING: Failed to determine specified clock resolution.\n"); + + return 0; +} + +int parse_clockid(const struct option *opt, const char *str, int unset) +{ + struct record_opts *opts = (struct record_opts *)opt->value; + const struct clockid_map *cm; + const char *ostr = str; + + if (unset) { + opts->use_clockid = 0; + return 0; + } + + /* no arg passed */ + if (!str) + return 0; + + /* no setting it twice */ + if (opts->use_clockid) + return -1; + + opts->use_clockid = true; + + /* if its a number, we're done */ + if (sscanf(str, "%d", &opts->clockid) == 1) + return get_clockid_res(opts->clockid, &opts->clockid_res_ns); + + /* allow a "CLOCK_" prefix to the name */ + if (!strncasecmp(str, "CLOCK_", 6)) + str += 6; + + for (cm = clockids; cm->name; cm++) { + if (!strcasecmp(str, cm->name)) { + opts->clockid = cm->clockid; + return get_clockid_res(opts->clockid, + &opts->clockid_res_ns); + } + } + + opts->use_clockid = false; + ui__warning("unknown clockid %s, check man page\n", ostr); + return -1; +} + +const char *clockid_name(clockid_t clk_id) +{ + const struct clockid_map *cm; + + for (cm = clockids; cm->name; cm++) { + if (cm->clockid == clk_id) + return cm->name; + } + return "(not found)"; +} diff --git a/tools/perf/util/clockid.h b/tools/perf/util/clockid.h new file mode 100644 index 000000000..33dbd8673 --- /dev/null +++ b/tools/perf/util/clockid.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CLOCKID_H +#define __PERF_CLOCKID_H + +#include + +struct option; +int parse_clockid(const struct option *opt, const char *str, int unset); + +const char *clockid_name(clockid_t clk_id); + +#endif diff --git a/tools/perf/util/cloexec.c b/tools/perf/util/cloexec.c new file mode 100644 index 000000000..8830604c3 --- /dev/null +++ b/tools/perf/util/cloexec.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "util.h" // for sched_getcpu() +#include "../perf-sys.h" +#include "cloexec.h" +#include "event.h" +#include "asm/bug.h" +#include "debug.h" +#include +#include +#include + +static unsigned long flag = PERF_FLAG_FD_CLOEXEC; + +static int perf_flag_probe(void) +{ + /* use 'safest' configuration as used in evsel__fallback() */ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CPU_CLOCK, + .exclude_kernel = 1, + }; + int fd; + int err; + int cpu; + pid_t pid = -1; + char sbuf[STRERR_BUFSIZE]; + + cpu = sched_getcpu(); + if (cpu < 0) + cpu = 0; + + /* + * Using -1 for the pid is a workaround to avoid gratuitous jump label + * changes. + */ + while (1) { + /* check cloexec flag */ + fd = sys_perf_event_open(&attr, pid, cpu, -1, + PERF_FLAG_FD_CLOEXEC); + if (fd < 0 && pid == -1 && errno == EACCES) { + pid = 0; + continue; + } + break; + } + err = errno; + + if (fd >= 0) { + close(fd); + return 1; + } + + WARN_ONCE(err != EINVAL && err != EBUSY && err != EACCES, + "perf_event_open(..., PERF_FLAG_FD_CLOEXEC) failed with unexpected error %d (%s)\n", + err, str_error_r(err, sbuf, sizeof(sbuf))); + + /* not supported, confirm error related to PERF_FLAG_FD_CLOEXEC */ + while (1) { + fd = sys_perf_event_open(&attr, pid, cpu, -1, 0); + if (fd < 0 && pid == -1 && errno == EACCES) { + pid = 0; + continue; + } + break; + } + err = errno; + + if (fd >= 0) + close(fd); + + if (WARN_ONCE(fd < 0 && err != EBUSY && err != EACCES, + "perf_event_open(..., 0) failed unexpectedly with error %d (%s)\n", + err, str_error_r(err, sbuf, sizeof(sbuf)))) + return -1; + + return 0; +} + +unsigned long perf_event_open_cloexec_flag(void) +{ + static bool probed; + + if (!probed) { + if (perf_flag_probe() <= 0) + flag = 0; + probed = true; + } + + return flag; +} diff --git a/tools/perf/util/cloexec.h b/tools/perf/util/cloexec.h new file mode 100644 index 000000000..78216b101 --- /dev/null +++ b/tools/perf/util/cloexec.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CLOEXEC_H +#define __PERF_CLOEXEC_H + +unsigned long perf_event_open_cloexec_flag(void); + +#endif /* __PERF_CLOEXEC_H */ diff --git a/tools/perf/util/color.c b/tools/perf/util/color.c new file mode 100644 index 000000000..e51f0a676 --- /dev/null +++ b/tools/perf/util/color.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "color.h" +#include +#include + +int perf_use_color_default = -1; + +static int __color_vsnprintf(char *bf, size_t size, const char *color, + const char *fmt, va_list args, const char *trail) +{ + int r = 0; + + /* + * Auto-detect: + */ + if (perf_use_color_default < 0) { + if (isatty(1) || pager_in_use()) + perf_use_color_default = 1; + else + perf_use_color_default = 0; + } + + if (perf_use_color_default && *color) + r += scnprintf(bf, size, "%s", color); + r += vscnprintf(bf + r, size - r, fmt, args); + if (perf_use_color_default && *color) + r += scnprintf(bf + r, size - r, "%s", PERF_COLOR_RESET); + if (trail) + r += scnprintf(bf + r, size - r, "%s", trail); + return r; +} + +/* Colors are not included in return value */ +static int __color_vfprintf(FILE *fp, const char *color, const char *fmt, + va_list args) +{ + int r = 0; + + /* + * Auto-detect: + */ + if (perf_use_color_default < 0) { + if (isatty(fileno(fp)) || pager_in_use()) + perf_use_color_default = 1; + else + perf_use_color_default = 0; + } + + if (perf_use_color_default && *color) + fprintf(fp, "%s", color); + r += vfprintf(fp, fmt, args); + if (perf_use_color_default && *color) + fprintf(fp, "%s", PERF_COLOR_RESET); + return r; +} + +int color_vsnprintf(char *bf, size_t size, const char *color, + const char *fmt, va_list args) +{ + return __color_vsnprintf(bf, size, color, fmt, args, NULL); +} + +int color_vfprintf(FILE *fp, const char *color, const char *fmt, va_list args) +{ + return __color_vfprintf(fp, color, fmt, args); +} + +int color_snprintf(char *bf, size_t size, const char *color, + const char *fmt, ...) +{ + va_list args; + int r; + + va_start(args, fmt); + r = color_vsnprintf(bf, size, color, fmt, args); + va_end(args); + return r; +} + +int color_fprintf(FILE *fp, const char *color, const char *fmt, ...) +{ + va_list args; + int r; + + va_start(args, fmt); + r = color_vfprintf(fp, color, fmt, args); + va_end(args); + return r; +} + +const char *get_percent_color(double percent) +{ + const char *color = PERF_COLOR_NORMAL; + + /* + * We color high-overhead entries in red, mid-overhead + * entries in green - and keep the low overhead places + * normal: + */ + if (fabs(percent) >= MIN_RED) + color = PERF_COLOR_RED; + else { + if (fabs(percent) > MIN_GREEN) + color = PERF_COLOR_GREEN; + } + return color; +} + +int percent_color_fprintf(FILE *fp, const char *fmt, double percent) +{ + int r; + const char *color; + + color = get_percent_color(percent); + r = color_fprintf(fp, color, fmt, percent); + + return r; +} + +int value_color_snprintf(char *bf, size_t size, const char *fmt, double value) +{ + const char *color = get_percent_color(value); + return color_snprintf(bf, size, color, fmt, value); +} + +int percent_color_snprintf(char *bf, size_t size, const char *fmt, ...) +{ + va_list args; + double percent; + + va_start(args, fmt); + percent = va_arg(args, double); + va_end(args); + return value_color_snprintf(bf, size, fmt, percent); +} + +int percent_color_len_snprintf(char *bf, size_t size, const char *fmt, ...) +{ + va_list args; + int len; + double percent; + const char *color; + + va_start(args, fmt); + len = va_arg(args, int); + percent = va_arg(args, double); + va_end(args); + + color = get_percent_color(percent); + return color_snprintf(bf, size, color, fmt, len, percent); +} diff --git a/tools/perf/util/color.h b/tools/perf/util/color.h new file mode 100644 index 000000000..0319546de --- /dev/null +++ b/tools/perf/util/color.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_COLOR_H +#define __PERF_COLOR_H + +#include +#include +#include + +/* "\033[1;38;5;2xx;48;5;2xxm\0" is 23 bytes */ +#define COLOR_MAXLEN 24 + +#define PERF_COLOR_NORMAL "" +#define PERF_COLOR_RESET "\033[m" +#define PERF_COLOR_BOLD "\033[1m" +#define PERF_COLOR_RED "\033[31m" +#define PERF_COLOR_GREEN "\033[32m" +#define PERF_COLOR_YELLOW "\033[33m" +#define PERF_COLOR_BLUE "\033[34m" +#define PERF_COLOR_MAGENTA "\033[35m" +#define PERF_COLOR_CYAN "\033[36m" +#define PERF_COLOR_BG_RED "\033[41m" + +#define MIN_GREEN 0.5 +#define MIN_RED 5.0 + +#define PERF_COLOR_DELETE_LINE "\033[A\33[2K\r" +/* + * This variable stores the value of color.ui + */ +extern int perf_use_color_default; + + +int perf_config_colorbool(const char *var, const char *value, int stdout_is_tty); +int color_vsnprintf(char *bf, size_t size, const char *color, + const char *fmt, va_list args); +int color_vfprintf(FILE *fp, const char *color, const char *fmt, va_list args); +int color_fprintf(FILE *fp, const char *color, const char *fmt, ...) __printf(3, 4); +int color_snprintf(char *bf, size_t size, const char *color, const char *fmt, ...) __printf(4, 5); +int value_color_snprintf(char *bf, size_t size, const char *fmt, double value); +int percent_color_snprintf(char *bf, size_t size, const char *fmt, ...) __printf(3, 4); +int percent_color_len_snprintf(char *bf, size_t size, const char *fmt, ...) __printf(3, 4); +int percent_color_fprintf(FILE *fp, const char *fmt, double percent); +const char *get_percent_color(double percent); + +#endif /* __PERF_COLOR_H */ diff --git a/tools/perf/util/color_config.c b/tools/perf/util/color_config.c new file mode 100644 index 000000000..301031ddc --- /dev/null +++ b/tools/perf/util/color_config.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "config.h" +#include +#include +#include "color.h" +#include +#include + +int perf_config_colorbool(const char *var, const char *value, int stdout_is_tty) +{ + if (value) { + if (!strcasecmp(value, "never")) + return 0; + if (!strcasecmp(value, "always")) + return 1; + if (!strcasecmp(value, "auto")) + goto auto_color; + } + + /* Missing or explicit false to turn off colorization */ + if (!perf_config_bool(var, value)) + return 0; + + /* any normal truth value defaults to 'auto' */ + auto_color: + if (stdout_is_tty < 0) + stdout_is_tty = isatty(1); + if (stdout_is_tty || pager_in_use()) { + char *term = getenv("TERM"); + if (term && strcmp(term, "dumb")) + return 1; + } + return 0; +} diff --git a/tools/perf/util/comm.c b/tools/perf/util/comm.c new file mode 100644 index 000000000..9880247a2 --- /dev/null +++ b/tools/perf/util/comm.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "comm.h" +#include +#include +#include +#include +#include +#include // reallocarray + +#include "rwsem.h" + +DECLARE_RC_STRUCT(comm_str) { + refcount_t refcnt; + char str[]; +}; + +static struct comm_strs { + struct rw_semaphore lock; + struct comm_str **strs; + int num_strs; + int capacity; +} _comm_strs; + +static void comm_strs__remove_if_last(struct comm_str *cs); + +static void comm_strs__init(void) + NO_THREAD_SAFETY_ANALYSIS /* Inherently single threaded due to pthread_once. */ +{ + init_rwsem(&_comm_strs.lock); + _comm_strs.capacity = 16; + _comm_strs.num_strs = 0; + _comm_strs.strs = calloc(16, sizeof(*_comm_strs.strs)); +} + +static struct comm_strs *comm_strs__get(void) +{ + static pthread_once_t comm_strs_type_once = PTHREAD_ONCE_INIT; + + pthread_once(&comm_strs_type_once, comm_strs__init); + + return &_comm_strs; +} + +static refcount_t *comm_str__refcnt(struct comm_str *cs) +{ + return &RC_CHK_ACCESS(cs)->refcnt; +} + +static const char *comm_str__str(const struct comm_str *cs) +{ + return &RC_CHK_ACCESS(cs)->str[0]; +} + +static struct comm_str *comm_str__get(struct comm_str *cs) +{ + struct comm_str *result; + + if (RC_CHK_GET(result, cs)) + refcount_inc_not_zero(comm_str__refcnt(cs)); + + return result; +} + +static void comm_str__put(struct comm_str *cs) +{ + if (!cs) + return; + + if (refcount_dec_and_test(comm_str__refcnt(cs))) { + RC_CHK_FREE(cs); + } else { + if (refcount_read(comm_str__refcnt(cs)) == 1) + comm_strs__remove_if_last(cs); + + RC_CHK_PUT(cs); + } +} + +static struct comm_str *comm_str__new(const char *str) +{ + struct comm_str *result = NULL; + RC_STRUCT(comm_str) *cs; + + cs = malloc(sizeof(*cs) + strlen(str) + 1); + if (ADD_RC_CHK(result, cs)) { + refcount_set(comm_str__refcnt(result), 1); + strcpy(&cs->str[0], str); + } + return result; +} + +static int comm_str__search(const void *_key, const void *_member) +{ + const char *key = _key; + const struct comm_str *member = *(const struct comm_str * const *)_member; + + return strcmp(key, comm_str__str(member)); +} + +static void comm_strs__remove_if_last(struct comm_str *cs) +{ + struct comm_strs *comm_strs = comm_strs__get(); + + down_write(&comm_strs->lock); + /* + * Are there only references from the array, if so remove the array + * reference under the write lock so that we don't race with findnew. + */ + if (refcount_read(comm_str__refcnt(cs)) == 1) { + struct comm_str **entry; + + entry = bsearch(comm_str__str(cs), comm_strs->strs, comm_strs->num_strs, + sizeof(struct comm_str *), comm_str__search); + comm_str__put(*entry); + for (int i = entry - comm_strs->strs; i < comm_strs->num_strs - 1; i++) + comm_strs->strs[i] = comm_strs->strs[i + 1]; + comm_strs->num_strs--; + } + up_write(&comm_strs->lock); +} + +static struct comm_str *__comm_strs__find(struct comm_strs *comm_strs, const char *str) + SHARED_LOCKS_REQUIRED(comm_strs->lock) +{ + struct comm_str **result; + + result = bsearch(str, comm_strs->strs, comm_strs->num_strs, sizeof(struct comm_str *), + comm_str__search); + + if (!result) + return NULL; + + return comm_str__get(*result); +} + +static struct comm_str *comm_strs__findnew(const char *str) +{ + struct comm_strs *comm_strs = comm_strs__get(); + struct comm_str *result; + + if (!comm_strs) + return NULL; + + down_read(&comm_strs->lock); + result = __comm_strs__find(comm_strs, str); + up_read(&comm_strs->lock); + if (result) + return result; + + down_write(&comm_strs->lock); + result = __comm_strs__find(comm_strs, str); + if (!result) { + if (comm_strs->num_strs == comm_strs->capacity) { + struct comm_str **tmp; + + tmp = reallocarray(comm_strs->strs, + comm_strs->capacity + 16, + sizeof(*comm_strs->strs)); + if (!tmp) { + up_write(&comm_strs->lock); + return NULL; + } + comm_strs->strs = tmp; + comm_strs->capacity += 16; + } + result = comm_str__new(str); + if (result) { + int low = 0, high = comm_strs->num_strs - 1; + int insert = comm_strs->num_strs; /* Default to inserting at the end. */ + + while (low <= high) { + int mid = low + (high - low) / 2; + int cmp = strcmp(comm_str__str(comm_strs->strs[mid]), str); + + if (cmp < 0) { + low = mid + 1; + } else { + high = mid - 1; + insert = mid; + } + } + memmove(&comm_strs->strs[insert + 1], &comm_strs->strs[insert], + (comm_strs->num_strs - insert) * sizeof(struct comm_str *)); + comm_strs->num_strs++; + comm_strs->strs[insert] = result; + } + } + up_write(&comm_strs->lock); + return comm_str__get(result); +} + +struct comm *comm__new(const char *str, u64 timestamp, bool exec) +{ + struct comm *comm = zalloc(sizeof(*comm)); + + if (!comm) + return NULL; + + comm->start = timestamp; + comm->exec = exec; + + comm->comm_str = comm_strs__findnew(str); + if (!comm->comm_str) { + free(comm); + return NULL; + } + + return comm; +} + +int comm__override(struct comm *comm, const char *str, u64 timestamp, bool exec) +{ + struct comm_str *new, *old = comm->comm_str; + + new = comm_strs__findnew(str); + if (!new) + return -ENOMEM; + + comm_str__put(old); + comm->comm_str = new; + comm->start = timestamp; + if (exec) + comm->exec = true; + + return 0; +} + +void comm__free(struct comm *comm) +{ + comm_str__put(comm->comm_str); + free(comm); +} + +const char *comm__str(const struct comm *comm) +{ + return comm_str__str(comm->comm_str); +} diff --git a/tools/perf/util/comm.h b/tools/perf/util/comm.h new file mode 100644 index 000000000..f35d8fbfa --- /dev/null +++ b/tools/perf/util/comm.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_COMM_H +#define __PERF_COMM_H + +#include +#include +#include + +struct comm_str; + +struct comm { + struct comm_str *comm_str; + u64 start; + struct list_head list; + bool exec; + union { /* Tool specific area */ + void *priv; + u64 db_id; + }; +}; + +void comm__free(struct comm *comm); +struct comm *comm__new(const char *str, u64 timestamp, bool exec); +const char *comm__str(const struct comm *comm); +int comm__override(struct comm *comm, const char *str, u64 timestamp, + bool exec); + +#endif /* __PERF_COMM_H */ diff --git a/tools/perf/util/compress.h b/tools/perf/util/compress.h new file mode 100644 index 000000000..ec6c38129 --- /dev/null +++ b/tools/perf/util/compress.h @@ -0,0 +1,93 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_COMPRESS_H +#define PERF_COMPRESS_H + +#include +#include +#include +#include +#include +#ifdef HAVE_ZSTD_SUPPORT +#include +#endif + +#ifdef HAVE_ZLIB_SUPPORT +int gzip_decompress_to_file(const char *input, int output_fd); +bool gzip_is_compressed(const char *input); +#endif + +#ifdef HAVE_LZMA_SUPPORT +int lzma_decompress_stream_to_file(FILE *input, int output_fd); +int lzma_decompress_to_file(const char *input, int output_fd); +bool lzma_is_compressed(const char *input); +#else +static inline +int lzma_decompress_stream_to_file(FILE *input __maybe_unused, + int output_fd __maybe_unused) +{ + return -1; +} +static inline +int lzma_decompress_to_file(const char *input __maybe_unused, + int output_fd __maybe_unused) +{ + return -1; +} +static inline int lzma_is_compressed(const char *input __maybe_unused) +{ + return false; +} +#endif + +struct zstd_data { +#ifdef HAVE_ZSTD_SUPPORT + ZSTD_CStream *cstream; + ZSTD_DStream *dstream; + int comp_level; +#endif +}; + +#ifdef HAVE_ZSTD_SUPPORT + +int zstd_init(struct zstd_data *data, int level); +int zstd_fini(struct zstd_data *data); + +ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, + void *src, size_t src_size, size_t max_record_size, + ssize_t process_header(void *record, size_t dst_size, + size_t data_size)); + +size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, + void *dst, size_t dst_size); +#else /* !HAVE_ZSTD_SUPPORT */ + +static inline int zstd_init(struct zstd_data *data __maybe_unused, int level __maybe_unused) +{ + return 0; +} + +static inline int zstd_fini(struct zstd_data *data __maybe_unused) +{ + return 0; +} + +static inline +ssize_t zstd_compress_stream_to_records(struct zstd_data *data __maybe_unused, + void *dst __maybe_unused, size_t dst_size __maybe_unused, + void *src __maybe_unused, size_t src_size __maybe_unused, + size_t max_record_size __maybe_unused, + ssize_t process_header(void *record, size_t dst_size, + size_t data_size) __maybe_unused) +{ + return 0; +} + +static inline size_t zstd_decompress_stream(struct zstd_data *data __maybe_unused, void *src __maybe_unused, + size_t src_size __maybe_unused, void *dst __maybe_unused, + size_t dst_size __maybe_unused) +{ + return 0; +} +#endif + +#endif /* PERF_COMPRESS_H */ diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c new file mode 100644 index 000000000..7988149dc --- /dev/null +++ b/tools/perf/util/config.c @@ -0,0 +1,995 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * config.c + * + * Helper functions for parsing config items. + * Originally copied from GIT source. + * + * Copyright (C) Linus Torvalds, 2005 + * Copyright (C) Johannes Schindelin, 2005 + * + */ +#include +#include +#include "cache.h" +#include "callchain.h" +#include "header.h" +#include +#include "util/event.h" /* proc_map_timeout */ +#include "util/hist.h" /* perf_hist_config */ +#include "util/stat.h" /* perf_stat__set_big_num */ +#include "util/evsel.h" /* evsel__hw_names, evsel__use_bpf_counters */ +#include "srcline.h" +#include "build-id.h" +#include "debug.h" +#include "config.h" +#include "unwind.h" +#include +#include +#include +#include +#include +#include +#include + +#define MAXNAME (256) + +#define DEBUG_CACHE_DIR ".debug" + +#define METRIC_ONLY_LEN 20 + +static struct stats walltime_nsecs_stats; + +struct perf_stat_config stat_config = { + .aggr_mode = AGGR_GLOBAL, + .aggr_level = MAX_CACHE_LVL + 1, + .scale = true, + .unit_width = 4, /* strlen("unit") */ + .run_count = 1, + .metric_only_len = METRIC_ONLY_LEN, + .walltime_nsecs_stats = &walltime_nsecs_stats, + .big_num = true, + .ctl_fd = -1, + .ctl_fd_ack = -1, + .iostat_run = false, +}; + +char buildid_dir[MAXPATHLEN]; /* root dir for buildid, binary cache */ + +static FILE *config_file; +static const char *config_file_name; +static int config_linenr; +static int config_file_eof; +static struct perf_config_set *config_set; + +const char *config_exclusive_filename; + +static int get_next_char(void) +{ + int c; + FILE *f; + + c = '\n'; + if ((f = config_file) != NULL) { + c = fgetc(f); + if (c == '\r') { + /* DOS like systems */ + c = fgetc(f); + if (c != '\n') { + ungetc(c, f); + c = '\r'; + } + } + if (c == '\n') + config_linenr++; + if (c == EOF) { + config_file_eof = 1; + c = '\n'; + } + } + return c; +} + +static char *parse_value(void) +{ + static char value[1024]; + int quote = 0, comment = 0, space = 0; + size_t len = 0; + + for (;;) { + int c = get_next_char(); + + if (len >= sizeof(value) - 1) + return NULL; + if (c == '\n') { + if (quote) + return NULL; + value[len] = 0; + return value; + } + if (comment) + continue; + if (isspace(c) && !quote) { + space = 1; + continue; + } + if (!quote) { + if (c == ';' || c == '#') { + comment = 1; + continue; + } + } + if (space) { + if (len) + value[len++] = ' '; + space = 0; + } + if (c == '\\') { + c = get_next_char(); + switch (c) { + case '\n': + continue; + case 't': + c = '\t'; + break; + case 'b': + c = '\b'; + break; + case 'n': + c = '\n'; + break; + /* Some characters escape as themselves */ + case '\\': case '"': + break; + /* Reject unknown escape sequences */ + default: + return NULL; + } + value[len++] = c; + continue; + } + if (c == '"') { + quote = 1-quote; + continue; + } + value[len++] = c; + } +} + +static inline int iskeychar(int c) +{ + return isalnum(c) || c == '-' || c == '_'; +} + +static int get_value(config_fn_t fn, void *data, char *name, unsigned int len) +{ + int c; + char *value; + + /* Get the full name */ + for (;;) { + c = get_next_char(); + if (config_file_eof) + break; + if (!iskeychar(c)) + break; + name[len++] = c; + if (len >= MAXNAME) + return -1; + } + name[len] = 0; + while (c == ' ' || c == '\t') + c = get_next_char(); + + value = NULL; + if (c != '\n') { + if (c != '=') + return -1; + value = parse_value(); + if (!value) + return -1; + } + return fn(name, value, data); +} + +static int get_extended_base_var(char *name, int baselen, int c) +{ + do { + if (c == '\n') + return -1; + c = get_next_char(); + } while (isspace(c)); + + /* We require the format to be '[base "extension"]' */ + if (c != '"') + return -1; + name[baselen++] = '.'; + + for (;;) { + int ch = get_next_char(); + + if (ch == '\n') + return -1; + if (ch == '"') + break; + if (ch == '\\') { + ch = get_next_char(); + if (ch == '\n') + return -1; + } + name[baselen++] = ch; + if (baselen > MAXNAME / 2) + return -1; + } + + /* Final ']' */ + if (get_next_char() != ']') + return -1; + return baselen; +} + +static int get_base_var(char *name) +{ + int baselen = 0; + + for (;;) { + int c = get_next_char(); + if (config_file_eof) + return -1; + if (c == ']') + return baselen; + if (isspace(c)) + return get_extended_base_var(name, baselen, c); + if (!iskeychar(c) && c != '.') + return -1; + if (baselen > MAXNAME / 2) + return -1; + name[baselen++] = tolower(c); + } +} + +static int perf_parse_file(config_fn_t fn, void *data) +{ + int comment = 0; + int baselen = 0; + static char var[MAXNAME]; + + /* U+FEFF Byte Order Mark in UTF8 */ + static const unsigned char *utf8_bom = (unsigned char *) "\xef\xbb\xbf"; + const unsigned char *bomptr = utf8_bom; + + for (;;) { + int line, c = get_next_char(); + + if (bomptr && *bomptr) { + /* We are at the file beginning; skip UTF8-encoded BOM + * if present. Sane editors won't put this in on their + * own, but e.g. Windows Notepad will do it happily. */ + if ((unsigned char) c == *bomptr) { + bomptr++; + continue; + } else { + /* Do not tolerate partial BOM. */ + if (bomptr != utf8_bom) + break; + /* No BOM at file beginning. Cool. */ + bomptr = NULL; + } + } + if (c == '\n') { + if (config_file_eof) + return 0; + comment = 0; + continue; + } + if (comment || isspace(c)) + continue; + if (c == '#' || c == ';') { + comment = 1; + continue; + } + if (c == '[') { + baselen = get_base_var(var); + if (baselen <= 0) + break; + var[baselen++] = '.'; + var[baselen] = 0; + continue; + } + if (!isalpha(c)) + break; + var[baselen] = tolower(c); + + /* + * The get_value function might or might not reach the '\n', + * so saving the current line number for error reporting. + */ + line = config_linenr; + if (get_value(fn, data, var, baselen+1) < 0) { + config_linenr = line; + break; + } + } + pr_err("bad config file line %d in %s\n", config_linenr, config_file_name); + return -1; +} + +static int parse_unit_factor(const char *end, unsigned long *val) +{ + if (!*end) + return 1; + else if (!strcasecmp(end, "k")) { + *val *= 1024; + return 1; + } + else if (!strcasecmp(end, "m")) { + *val *= 1024 * 1024; + return 1; + } + else if (!strcasecmp(end, "g")) { + *val *= 1024 * 1024 * 1024; + return 1; + } + return 0; +} + +static int perf_parse_llong(const char *value, long long *ret) +{ + if (value && *value) { + char *end; + long long val = strtoll(value, &end, 0); + unsigned long factor = 1; + + if (!parse_unit_factor(end, &factor)) + return 0; + *ret = val * factor; + return 1; + } + return 0; +} + +static int perf_parse_long(const char *value, long *ret) +{ + if (value && *value) { + char *end; + long val = strtol(value, &end, 0); + unsigned long factor = 1; + if (!parse_unit_factor(end, &factor)) + return 0; + *ret = val * factor; + return 1; + } + return 0; +} + +static void bad_config(const char *name) +{ + if (config_file_name) + pr_warning("bad config value for '%s' in %s, ignoring...\n", name, config_file_name); + else + pr_warning("bad config value for '%s', ignoring...\n", name); +} + +int perf_config_u64(u64 *dest, const char *name, const char *value) +{ + long long ret = 0; + + if (!perf_parse_llong(value, &ret)) { + bad_config(name); + return -1; + } + + *dest = ret; + return 0; +} + +int perf_config_int(int *dest, const char *name, const char *value) +{ + long ret = 0; + if (!perf_parse_long(value, &ret)) { + bad_config(name); + return -1; + } + *dest = ret; + return 0; +} + +int perf_config_u8(u8 *dest, const char *name, const char *value) +{ + long ret = 0; + + if (!perf_parse_long(value, &ret)) { + bad_config(name); + return -1; + } + *dest = ret; + return 0; +} + +static int perf_config_bool_or_int(const char *name, const char *value, int *is_bool) +{ + int ret; + + *is_bool = 1; + if (!value) + return 1; + if (!*value) + return 0; + if (!strcasecmp(value, "true") || !strcasecmp(value, "yes") || !strcasecmp(value, "on")) + return 1; + if (!strcasecmp(value, "false") || !strcasecmp(value, "no") || !strcasecmp(value, "off")) + return 0; + *is_bool = 0; + return perf_config_int(&ret, name, value) < 0 ? -1 : ret; +} + +int perf_config_bool(const char *name, const char *value) +{ + int discard; + return !!perf_config_bool_or_int(name, value, &discard); +} + +static const char *perf_config_dirname(const char *name, const char *value) +{ + if (!name) + return NULL; + return value; +} + +static int perf_buildid_config(const char *var, const char *value) +{ + /* same dir for all commands */ + if (!strcmp(var, "buildid.dir")) { + const char *dir = perf_config_dirname(var, value); + + if (!dir) { + pr_err("Invalid buildid directory!\n"); + return -1; + } + strncpy(buildid_dir, dir, MAXPATHLEN-1); + buildid_dir[MAXPATHLEN-1] = '\0'; + } + + return 0; +} + +static int perf_default_core_config(const char *var, const char *value) +{ + if (!strcmp(var, "core.proc-map-timeout")) + proc_map_timeout = strtoul(value, NULL, 10); + + if (!strcmp(var, "core.addr2line-timeout")) + symbol_conf.addr2line_timeout_ms = strtoul(value, NULL, 10); + + if (!strcmp(var, "core.addr2line-disable-warn")) + symbol_conf.addr2line_disable_warn = perf_config_bool(var, value); + + /* Add other config variables here. */ + return 0; +} + +static int perf_ui_config(const char *var, const char *value) +{ + /* Add other config variables here. */ + if (!strcmp(var, "ui.show-headers")) + symbol_conf.show_hist_headers = perf_config_bool(var, value); + + return 0; +} + +void perf_stat__set_big_num(int set) +{ + stat_config.big_num = (set != 0); +} + +static void perf_stat__set_no_csv_summary(int set) +{ + stat_config.no_csv_summary = (set != 0); +} + +static int perf_stat_config(const char *var, const char *value) +{ + if (!strcmp(var, "stat.big-num")) + perf_stat__set_big_num(perf_config_bool(var, value)); + + if (!strcmp(var, "stat.no-csv-summary")) + perf_stat__set_no_csv_summary(perf_config_bool(var, value)); + + if (!strcmp(var, "stat.bpf-counter-events")) + evsel__bpf_counter_events = strdup(value); + + /* Add other config variables here. */ + return 0; +} + +int perf_default_config(const char *var, const char *value, + void *dummy __maybe_unused) +{ + if (strstarts(var, "core.")) + return perf_default_core_config(var, value); + + if (strstarts(var, "hist.")) + return perf_hist_config(var, value); + + if (strstarts(var, "ui.")) + return perf_ui_config(var, value); + + if (strstarts(var, "call-graph.")) + return perf_callchain_config(var, value); + + if (strstarts(var, "buildid.")) + return perf_buildid_config(var, value); + + if (strstarts(var, "stat.")) + return perf_stat_config(var, value); + + if (strstarts(var, "addr2line.")) + return addr2line_configure(var, value, dummy); + + if (strstarts(var, "unwind.")) + return unwind__configure(var, value, dummy); + + /* Add other config variables here. */ + return 0; +} + +static int perf_config_from_file(config_fn_t fn, const char *filename, void *data) +{ + int ret; + FILE *f = fopen(filename, "r"); + + ret = -1; + if (f) { + config_file = f; + config_file_name = filename; + config_linenr = 1; + config_file_eof = 0; + ret = perf_parse_file(fn, data); + fclose(f); + config_file_name = NULL; + } + return ret; +} + +const char *perf_etc_perfconfig(void) +{ + static const char *system_wide; + if (!system_wide) + system_wide = system_path(ETC_PERFCONFIG); + return system_wide; +} + +static int perf_env_bool(const char *k, int def) +{ + const char *v = getenv(k); + return v ? perf_config_bool(k, v) : def; +} + +int perf_config_system(void) +{ + return !perf_env_bool("PERF_CONFIG_NOSYSTEM", 0); +} + +int perf_config_global(void) +{ + return !perf_env_bool("PERF_CONFIG_NOGLOBAL", 0); +} + +static char *home_perfconfig(void) +{ + const char *home = NULL; + char *config; + struct stat st; + char path[PATH_MAX]; + + home = getenv("HOME"); + + /* + * Skip reading user config if: + * - there is no place to read it from (HOME) + * - we are asked not to (PERF_CONFIG_NOGLOBAL=1) + */ + if (!home || !*home || !perf_config_global()) + return NULL; + + config = strdup(mkpath(path, sizeof(path), "%s/.perfconfig", home)); + if (config == NULL) { + pr_warning("Not enough memory to process %s/.perfconfig, ignoring it.\n", home); + return NULL; + } + + if (stat(config, &st) < 0) + goto out_free; + + if (st.st_uid && (st.st_uid != geteuid())) { + pr_warning("File %s not owned by current user or root, ignoring it.\n", config); + goto out_free; + } + + if (st.st_size) + return config; + +out_free: + free(config); + return NULL; +} + +const char *perf_home_perfconfig(void) +{ + static const char *config; + static bool failed; + + if (failed || config) + return config; + + config = home_perfconfig(); + if (!config) + failed = true; + + return config; +} + +static struct perf_config_section *find_section(struct list_head *sections, + const char *section_name) +{ + struct perf_config_section *section; + + list_for_each_entry(section, sections, node) + if (!strcmp(section->name, section_name)) + return section; + + return NULL; +} + +static struct perf_config_item *find_config_item(const char *name, + struct perf_config_section *section) +{ + struct perf_config_item *item; + + list_for_each_entry(item, §ion->items, node) + if (!strcmp(item->name, name)) + return item; + + return NULL; +} + +static struct perf_config_section *add_section(struct list_head *sections, + const char *section_name) +{ + struct perf_config_section *section = zalloc(sizeof(*section)); + + if (!section) + return NULL; + + INIT_LIST_HEAD(§ion->items); + section->name = strdup(section_name); + if (!section->name) { + pr_debug("%s: strdup failed\n", __func__); + free(section); + return NULL; + } + + list_add_tail(§ion->node, sections); + return section; +} + +static struct perf_config_item *add_config_item(struct perf_config_section *section, + const char *name) +{ + struct perf_config_item *item = zalloc(sizeof(*item)); + + if (!item) + return NULL; + + item->name = strdup(name); + if (!item->name) { + pr_debug("%s: strdup failed\n", __func__); + free(item); + return NULL; + } + + list_add_tail(&item->node, §ion->items); + return item; +} + +static int set_value(struct perf_config_item *item, const char *value) +{ + char *val = strdup(value); + + if (!val) + return -1; + + zfree(&item->value); + item->value = val; + return 0; +} + +static int collect_config(const char *var, const char *value, + void *perf_config_set) +{ + int ret = -1; + char *ptr, *key; + char *section_name, *name; + struct perf_config_section *section = NULL; + struct perf_config_item *item = NULL; + struct perf_config_set *set = perf_config_set; + struct list_head *sections; + + if (set == NULL) + return -1; + + sections = &set->sections; + key = ptr = strdup(var); + if (!key) { + pr_debug("%s: strdup failed\n", __func__); + return -1; + } + + section_name = strsep(&ptr, "."); + name = ptr; + if (name == NULL || value == NULL) + goto out_free; + + section = find_section(sections, section_name); + if (!section) { + section = add_section(sections, section_name); + if (!section) + goto out_free; + } + + item = find_config_item(name, section); + if (!item) { + item = add_config_item(section, name); + if (!item) + goto out_free; + } + + /* perf_config_set can contain both user and system config items. + * So we should know where each value is from. + * The classification would be needed when a particular config file + * is overwritten by setting feature i.e. set_config(). + */ + if (strcmp(config_file_name, perf_etc_perfconfig()) == 0) { + section->from_system_config = true; + item->from_system_config = true; + } else { + section->from_system_config = false; + item->from_system_config = false; + } + + ret = set_value(item, value); + +out_free: + free(key); + return ret; +} + +int perf_config_set__collect(struct perf_config_set *set, const char *file_name, + const char *var, const char *value) +{ + config_file_name = file_name; + return collect_config(var, value, set); +} + +static int perf_config_set__init(struct perf_config_set *set) +{ + int ret = -1; + + /* Setting $PERF_CONFIG makes perf read _only_ the given config file. */ + if (config_exclusive_filename) + return perf_config_from_file(collect_config, config_exclusive_filename, set); + if (perf_config_system() && !access(perf_etc_perfconfig(), R_OK)) { + if (perf_config_from_file(collect_config, perf_etc_perfconfig(), set) < 0) + goto out; + } + if (perf_config_global() && perf_home_perfconfig()) { + if (perf_config_from_file(collect_config, perf_home_perfconfig(), set) < 0) + goto out; + } + +out: + return ret; +} + +struct perf_config_set *perf_config_set__new(void) +{ + struct perf_config_set *set = zalloc(sizeof(*set)); + + if (set) { + INIT_LIST_HEAD(&set->sections); + perf_config_set__init(set); + } + + return set; +} + +struct perf_config_set *perf_config_set__load_file(const char *file) +{ + struct perf_config_set *set = zalloc(sizeof(*set)); + + if (set) { + INIT_LIST_HEAD(&set->sections); + perf_config_from_file(collect_config, file, set); + } + + return set; +} + +static int perf_config__init(void) +{ + if (config_set == NULL) + config_set = perf_config_set__new(); + + return config_set == NULL; +} + +int perf_config_set(struct perf_config_set *set, + config_fn_t fn, void *data) +{ + int ret = 0; + char key[BUFSIZ]; + struct perf_config_section *section; + struct perf_config_item *item; + + perf_config_set__for_each_entry(set, section, item) { + char *value = item->value; + + if (value) { + scnprintf(key, sizeof(key), "%s.%s", + section->name, item->name); + ret = fn(key, value, data); + if (ret < 0) { + pr_err("Error in the given config file: wrong config key-value pair %s=%s\n", + key, value); + /* + * Can't be just a 'break', as perf_config_set__for_each_entry() + * expands to two nested for() loops. + */ + goto out; + } + } + } +out: + return ret; +} + +int perf_config(config_fn_t fn, void *data) +{ + if (config_set == NULL && perf_config__init()) + return -1; + + return perf_config_set(config_set, fn, data); +} + +void perf_config__exit(void) +{ + perf_config_set__delete(config_set); + config_set = NULL; +} + +static void perf_config_item__delete(struct perf_config_item *item) +{ + zfree(&item->name); + zfree(&item->value); + free(item); +} + +static void perf_config_section__purge(struct perf_config_section *section) +{ + struct perf_config_item *item, *tmp; + + list_for_each_entry_safe(item, tmp, §ion->items, node) { + list_del_init(&item->node); + perf_config_item__delete(item); + } +} + +static void perf_config_section__delete(struct perf_config_section *section) +{ + perf_config_section__purge(section); + zfree(§ion->name); + free(section); +} + +static void perf_config_set__purge(struct perf_config_set *set) +{ + struct perf_config_section *section, *tmp; + + list_for_each_entry_safe(section, tmp, &set->sections, node) { + list_del_init(§ion->node); + perf_config_section__delete(section); + } +} + +void perf_config_set__delete(struct perf_config_set *set) +{ + if (set == NULL) + return; + + perf_config_set__purge(set); + free(set); +} + +/* + * Call this to report error for your variable that should not + * get a boolean value (i.e. "[my] var" means "true"). + */ +int config_error_nonbool(const char *var) +{ + pr_err("Missing value for '%s'", var); + return -1; +} + +void set_buildid_dir(const char *dir) +{ + if (dir) + scnprintf(buildid_dir, MAXPATHLEN, "%s", dir); + + /* default to $HOME/.debug */ + if (buildid_dir[0] == '\0') { + char *home = getenv("HOME"); + + if (home) { + snprintf(buildid_dir, MAXPATHLEN, "%s/%s", + home, DEBUG_CACHE_DIR); + } else { + strncpy(buildid_dir, DEBUG_CACHE_DIR, MAXPATHLEN-1); + } + buildid_dir[MAXPATHLEN-1] = '\0'; + } + /* for communicating with external commands */ + setenv("PERF_BUILDID_DIR", buildid_dir, 1); +} + +struct perf_config_scan_data { + const char *name; + const char *fmt; + const char *value; + va_list args; + int ret; +}; + +static int perf_config_scan_cb(const char *var, const char *value, void *data) +{ + struct perf_config_scan_data *d = data; + + if (!strcmp(var, d->name)) + d->ret = vsscanf(value, d->fmt, d->args); + + return 0; +} + +int perf_config_scan(const char *name, const char *fmt, ...) +{ + struct perf_config_scan_data d = { + .name = name, + .fmt = fmt, + }; + + va_start(d.args, fmt); + perf_config(perf_config_scan_cb, &d); + va_end(d.args); + + return d.ret; +} + +static int perf_config_get_cb(const char *var, const char *value, void *data) +{ + struct perf_config_scan_data *d = data; + + if (!strcmp(var, d->name)) + d->value = value; + + return 0; +} + +const char *perf_config_get(const char *name) +{ + struct perf_config_scan_data d = { + .name = name, + .value = NULL, + }; + + perf_config(perf_config_get_cb, &d); + return d.value; +} diff --git a/tools/perf/util/config.h b/tools/perf/util/config.h new file mode 100644 index 000000000..987b47cf5 --- /dev/null +++ b/tools/perf/util/config.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CONFIG_H +#define __PERF_CONFIG_H + +#include +#include + +struct perf_config_item { + char *name; + char *value; + bool from_system_config; + struct list_head node; +}; + +struct perf_config_section { + char *name; + struct list_head items; + bool from_system_config; + struct list_head node; +}; + +struct perf_config_set { + struct list_head sections; +}; + +extern const char *config_exclusive_filename; + +typedef int (*config_fn_t)(const char *, const char *, void *); + +int perf_default_config(const char *, const char *, void *); +int perf_config(config_fn_t fn, void *); +int perf_config_scan(const char *name, const char *fmt, ...) __scanf(2, 3); +const char *perf_config_get(const char *name); +int perf_config_set(struct perf_config_set *set, + config_fn_t fn, void *data); +int perf_config_int(int *dest, const char *, const char *); +int perf_config_u8(u8 *dest, const char *name, const char *value); +int perf_config_u64(u64 *dest, const char *, const char *); +int perf_config_bool(const char *, const char *); +int config_error_nonbool(const char *); +const char *perf_etc_perfconfig(void); +const char *perf_home_perfconfig(void); +int perf_config_system(void); +int perf_config_global(void); + +struct perf_config_set *perf_config_set__new(void); +struct perf_config_set *perf_config_set__load_file(const char *file); +void perf_config_set__delete(struct perf_config_set *set); +int perf_config_set__collect(struct perf_config_set *set, const char *file_name, + const char *var, const char *value); +void perf_config__exit(void); +int perf_config__set_variable(const char *var, const char *value); + +/** + * perf_config_sections__for_each - iterate thru all the sections + * @list: list_head instance to iterate + * @section: struct perf_config_section iterator + */ +#define perf_config_sections__for_each_entry(list, section) \ + list_for_each_entry(section, list, node) + +/** + * perf_config_items__for_each - iterate thru all the items + * @list: list_head instance to iterate + * @item: struct perf_config_item iterator + */ +#define perf_config_items__for_each_entry(list, item) \ + list_for_each_entry(item, list, node) + +/** + * perf_config_set__for_each - iterate thru all the config section-item pairs + * @set: evlist instance to iterate + * @section: struct perf_config_section iterator + * @item: struct perf_config_item iterator + */ +#define perf_config_set__for_each_entry(set, section, item) \ + perf_config_sections__for_each_entry(&set->sections, section) \ + perf_config_items__for_each_entry(§ion->items, item) + +#endif /* __PERF_CONFIG_H */ diff --git a/tools/perf/util/copyfile.c b/tools/perf/util/copyfile.c new file mode 100644 index 000000000..47e03de7c --- /dev/null +++ b/tools/perf/util/copyfile.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/copyfile.h" +#include "util/namespaces.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int slow_copyfile(const char *from, const char *to, struct nsinfo *nsi) +{ + int err = -1; + char *line = NULL; + size_t n; + FILE *from_fp, *to_fp; + struct nscookie nsc; + + nsinfo__mountns_enter(nsi, &nsc); + from_fp = fopen(from, "r"); + nsinfo__mountns_exit(&nsc); + if (from_fp == NULL) + goto out; + + to_fp = fopen(to, "w"); + if (to_fp == NULL) + goto out_fclose_from; + + while (getline(&line, &n, from_fp) > 0) + if (fputs(line, to_fp) == EOF) + goto out_fclose_to; + err = 0; +out_fclose_to: + fclose(to_fp); + free(line); +out_fclose_from: + fclose(from_fp); +out: + return err; +} + +int copyfile_offset(int ifd, loff_t off_in, int ofd, loff_t off_out, u64 size) +{ + void *ptr; + loff_t pgoff; + + pgoff = off_in & ~(page_size - 1); + off_in -= pgoff; + + ptr = mmap(NULL, off_in + size, PROT_READ, MAP_PRIVATE, ifd, pgoff); + if (ptr == MAP_FAILED) + return -1; + + while (size) { + ssize_t ret = pwrite(ofd, ptr + off_in, size, off_out); + if (ret < 0 && errno == EINTR) + continue; + if (ret <= 0) + break; + + size -= ret; + off_in += ret; + off_out += ret; + } + munmap(ptr, off_in + size); + + return size ? -1 : 0; +} + +static int copyfile_mode_ns(const char *from, const char *to, mode_t mode, + struct nsinfo *nsi) +{ + int fromfd, tofd; + struct stat st; + int err; + char *tmp = NULL, *ptr = NULL; + struct nscookie nsc; + + nsinfo__mountns_enter(nsi, &nsc); + err = stat(from, &st); + nsinfo__mountns_exit(&nsc); + if (err) + goto out; + err = -1; + + /* extra 'x' at the end is to reserve space for '.' */ + if (asprintf(&tmp, "%s.XXXXXXx", to) < 0) { + tmp = NULL; + goto out; + } + ptr = strrchr(tmp, '/'); + if (!ptr) + goto out; + ptr = memmove(ptr + 1, ptr, strlen(ptr) - 1); + *ptr = '.'; + + tofd = mkstemp(tmp); + if (tofd < 0) + goto out; + + if (st.st_size == 0) { /* /proc? do it slowly... */ + err = slow_copyfile(from, tmp, nsi); + if (!err && fchmod(tofd, mode)) + err = -1; + goto out_close_to; + } + + if (fchmod(tofd, mode)) + goto out_close_to; + + nsinfo__mountns_enter(nsi, &nsc); + fromfd = open(from, O_RDONLY); + nsinfo__mountns_exit(&nsc); + if (fromfd < 0) + goto out_close_to; + + err = copyfile_offset(fromfd, 0, tofd, 0, st.st_size); + + close(fromfd); +out_close_to: + close(tofd); + if (!err) + err = link(tmp, to); + unlink(tmp); +out: + free(tmp); + return err; +} + +int copyfile_ns(const char *from, const char *to, struct nsinfo *nsi) +{ + return copyfile_mode_ns(from, to, 0755, nsi); +} + +int copyfile_mode(const char *from, const char *to, mode_t mode) +{ + return copyfile_mode_ns(from, to, mode, NULL); +} + +int copyfile(const char *from, const char *to) +{ + return copyfile_mode(from, to, 0755); +} diff --git a/tools/perf/util/copyfile.h b/tools/perf/util/copyfile.h new file mode 100644 index 000000000..e85d2f22f --- /dev/null +++ b/tools/perf/util/copyfile.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef PERF_COPYFILE_H_ +#define PERF_COPYFILE_H_ + +#include +#include +#include + +struct nsinfo; + +int copyfile(const char *from, const char *to); +int copyfile_mode(const char *from, const char *to, mode_t mode); +int copyfile_ns(const char *from, const char *to, struct nsinfo *nsi); +int copyfile_offset(int ifd, loff_t off_in, int ofd, loff_t off_out, u64 size); + +#endif // PERF_COPYFILE_H_ diff --git a/tools/perf/util/counts.c b/tools/perf/util/counts.c new file mode 100644 index 000000000..11cd85b27 --- /dev/null +++ b/tools/perf/util/counts.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "evsel.h" +#include "counts.h" +#include +#include + +struct perf_counts *perf_counts__new(int ncpus, int nthreads) +{ + struct perf_counts *counts = zalloc(sizeof(*counts)); + + if (counts) { + struct xyarray *values; + + values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values)); + if (!values) { + free(counts); + return NULL; + } + + counts->values = values; + + values = xyarray__new(ncpus, nthreads, sizeof(bool)); + if (!values) { + xyarray__delete(counts->values); + free(counts); + return NULL; + } + + counts->loaded = values; + } + + return counts; +} + +void perf_counts__delete(struct perf_counts *counts) +{ + if (counts) { + xyarray__delete(counts->loaded); + xyarray__delete(counts->values); + free(counts); + } +} + +void perf_counts__reset(struct perf_counts *counts) +{ + xyarray__reset(counts->loaded); + xyarray__reset(counts->values); +} + +void evsel__reset_counts(struct evsel *evsel) +{ + perf_counts__reset(evsel->counts); +} + +int evsel__alloc_counts(struct evsel *evsel) +{ + struct perf_cpu_map *cpus = evsel__cpus(evsel); + int nthreads = perf_thread_map__nr(evsel->core.threads); + + evsel->counts = perf_counts__new(perf_cpu_map__nr(cpus), nthreads); + return evsel->counts != NULL ? 0 : -ENOMEM; +} + +void evsel__free_counts(struct evsel *evsel) +{ + perf_counts__delete(evsel->counts); + evsel->counts = NULL; +} diff --git a/tools/perf/util/counts.h b/tools/perf/util/counts.h new file mode 100644 index 000000000..42760242e --- /dev/null +++ b/tools/perf/util/counts.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_COUNTS_H +#define __PERF_COUNTS_H + +#include +#include +#include +#include + +struct evsel; + +struct perf_counts { + s8 scaled; + struct xyarray *values; + struct xyarray *loaded; +}; + + +static inline struct perf_counts_values* +perf_counts(struct perf_counts *counts, int cpu_map_idx, int thread) +{ + return xyarray__entry(counts->values, cpu_map_idx, thread); +} + +static inline bool +perf_counts__is_loaded(struct perf_counts *counts, int cpu_map_idx, int thread) +{ + return *((bool *) xyarray__entry(counts->loaded, cpu_map_idx, thread)); +} + +static inline void +perf_counts__set_loaded(struct perf_counts *counts, int cpu_map_idx, int thread, bool loaded) +{ + *((bool *) xyarray__entry(counts->loaded, cpu_map_idx, thread)) = loaded; +} + +struct perf_counts *perf_counts__new(int ncpus, int nthreads); +void perf_counts__delete(struct perf_counts *counts); +void perf_counts__reset(struct perf_counts *counts); + +void evsel__reset_counts(struct evsel *evsel); +int evsel__alloc_counts(struct evsel *evsel); +void evsel__free_counts(struct evsel *evsel); + +#endif /* __PERF_COUNTS_H */ diff --git a/tools/perf/util/cpumap.c b/tools/perf/util/cpumap.c new file mode 100644 index 000000000..23ebe9b97 --- /dev/null +++ b/tools/perf/util/cpumap.c @@ -0,0 +1,824 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "cpumap.h" +#include "debug.h" +#include "event.h" +#include +#include +#include +#include +#include +#include "asm/bug.h" + +#include +#include +#include +#include + +static struct perf_cpu max_cpu_num; +static struct perf_cpu max_present_cpu_num; +static int max_node_num; +/** + * The numa node X as read from /sys/devices/system/node/nodeX indexed by the + * CPU number. + */ +static int *cpunode_map; + +bool perf_record_cpu_map_data__test_bit(int i, + const struct perf_record_cpu_map_data *data) +{ + int bit_word32 = i / 32; + __u32 bit_mask32 = 1U << (i & 31); + int bit_word64 = i / 64; + __u64 bit_mask64 = ((__u64)1) << (i & 63); + + return (data->mask32_data.long_size == 4) + ? (bit_word32 < data->mask32_data.nr) && + (data->mask32_data.mask[bit_word32] & bit_mask32) != 0 + : (bit_word64 < data->mask64_data.nr) && + (data->mask64_data.mask[bit_word64] & bit_mask64) != 0; +} + +/* Read ith mask value from data into the given 64-bit sized bitmap */ +static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, + int i, unsigned long *bitmap, + u16 long_size) +{ +#if __SIZEOF_LONG__ == 8 + if (long_size == 4) + bitmap[0] = data->mask32_data.mask[i]; + else + bitmap[0] = data->mask64_data.mask[i]; +#else + if (long_size == 4) { + bitmap[0] = data->mask32_data.mask[i]; + bitmap[1] = 0; + } else { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bitmap[0] = (unsigned long)(data->mask64_data.mask[i] >> 32); + bitmap[1] = (unsigned long)data->mask64_data.mask[i]; +#else + bitmap[0] = (unsigned long)data->mask64_data.mask[i]; + bitmap[1] = (unsigned long)(data->mask64_data.mask[i] >> 32); +#endif + } +#endif +} +static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) +{ + /* Snapshot nr — data is mmap'd and could change between reads */ + u16 nr = READ_ONCE(data->cpus_data.nr); + struct perf_cpu_map *map; + + map = perf_cpu_map__empty_new(nr); + if (!map) + return NULL; + + for (unsigned int i = 0; i < nr; i++) { + u16 cpu = READ_ONCE(data->cpus_data.cpu[i]); + /* + * Special treatment for -1, which is not real cpu number, + * and we need to use (int) -1 to initialize map[i], + * otherwise it would become 65535. + */ + if (cpu == (u16) -1) { + RC_CHK_ACCESS(map)->map[i].cpu = -1; + } else if (cpu < INT16_MAX) { + RC_CHK_ACCESS(map)->map[i].cpu = (int16_t) cpu; + } else { + pr_err("Invalid cpumap entry %u\n", cpu); + perf_cpu_map__put(map); + return NULL; + } + } + + return map; +} + +static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) +{ + DECLARE_BITMAP(local_copy, 64); + int weight = 0, mask_nr; + /* Snapshot before validation — data is mmap'd and could change */ + u16 long_size = READ_ONCE(data->mask32_data.long_size); + struct perf_cpu_map *map; + + /* long_size must be 4 or 8; other values overflow cpus_per_i below */ + if (long_size != 4 && long_size != 8) { + pr_warning("WARNING: cpu_map mask: unsupported long_size %u\n", long_size); + return NULL; + } + + mask_nr = READ_ONCE(data->mask32_data.nr); + + for (int i = 0; i < mask_nr; i++) { + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); + weight += bitmap_weight(local_copy, 64); + } + + map = perf_cpu_map__empty_new(weight); + if (!map) + return NULL; + + for (int i = 0, j = 0; i < mask_nr; i++) { + int cpus_per_i = (i * long_size * BITS_PER_BYTE); + int cpu; + + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); + for_each_set_bit(cpu, local_copy, 64) { + /* Guard against more set bits than the first pass counted */ + if (j >= weight) + break; + if (cpu + cpus_per_i < INT16_MAX) { + RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; + } else { + pr_err("Invalid cpumap entry %d\n", cpu + cpus_per_i); + perf_cpu_map__put(map); + return NULL; + } + } + } + return map; + +} + +static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) +{ + /* Snapshot fields — data is mmap'd and could change between reads */ + u16 start_cpu = READ_ONCE(data->range_cpu_data.start_cpu); + u16 end_cpu = READ_ONCE(data->range_cpu_data.end_cpu); + u16 any_cpu = READ_ONCE(data->range_cpu_data.any_cpu); + struct perf_cpu_map *map; + unsigned int i = 0; + + if (end_cpu < start_cpu) { + pr_warning("WARNING: cpu_map range: end_cpu %u < start_cpu %u\n", + end_cpu, start_cpu); + return NULL; + } + + /* any_cpu is boolean (0 or 1), not a count — clamp to avoid inflated nr */ + map = perf_cpu_map__empty_new(end_cpu - start_cpu + 1 + !!any_cpu); + if (!map) + return NULL; + + if (any_cpu) + RC_CHK_ACCESS(map)->map[i++].cpu = -1; + + for (int cpu = start_cpu; cpu <= end_cpu; + i++, cpu++) { + if (cpu < INT16_MAX) { + RC_CHK_ACCESS(map)->map[i].cpu = cpu; + } else { + pr_err("Invalid cpumap entry %d\n", cpu); + perf_cpu_map__put(map); + return NULL; + } + } + + return map; +} + +struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) +{ + switch (data->type) { + case PERF_CPU_MAP__CPUS: + return cpu_map__from_entries(data); + case PERF_CPU_MAP__MASK: + return cpu_map__from_mask(data); + case PERF_CPU_MAP__RANGE_CPUS: + return cpu_map__from_range(data); + default: + pr_err("cpu_map__new_data unknown type %d\n", data->type); + return NULL; + } +} + +size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) +{ +#define BUFSIZE 1024 + char buf[BUFSIZE]; + + cpu_map__snprint(map, buf, sizeof(buf)); + return fprintf(fp, "%s\n", buf); +#undef BUFSIZE +} + +struct perf_cpu_map *perf_cpu_map__empty_new(int nr) +{ + struct perf_cpu_map *cpus = perf_cpu_map__alloc(nr); + + if (cpus != NULL) { + for (int i = 0; i < nr; i++) + RC_CHK_ACCESS(cpus)->map[i].cpu = -1; + } + + return cpus; +} + +struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr) +{ + struct cpu_aggr_map *cpus = malloc(sizeof(*cpus) + sizeof(struct aggr_cpu_id) * nr); + + if (cpus != NULL) { + int i; + + cpus->nr = nr; + for (i = 0; i < nr; i++) + cpus->map[i] = aggr_cpu_id__empty(); + } + + return cpus; +} + +static int cpu__get_topology_int(int cpu, const char *name, int *value) +{ + char path[PATH_MAX]; + + snprintf(path, PATH_MAX, + "devices/system/cpu/cpu%d/topology/%s", cpu, name); + + return sysfs__read_int(path, value); +} + +int cpu__get_socket_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "physical_package_id", &value); + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data __maybe_unused) +{ + struct aggr_cpu_id id = aggr_cpu_id__empty(); + + id.socket = cpu__get_socket_id(cpu); + return id; +} + +static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) +{ + struct aggr_cpu_id *a = (struct aggr_cpu_id *)a_pointer; + struct aggr_cpu_id *b = (struct aggr_cpu_id *)b_pointer; + + if (a->node != b->node) + return a->node - b->node; + else if (a->socket != b->socket) + return a->socket - b->socket; + else if (a->die != b->die) + return a->die - b->die; + else if (a->cluster != b->cluster) + return a->cluster - b->cluster; + else if (a->cache_lvl != b->cache_lvl) + return a->cache_lvl - b->cache_lvl; + else if (a->cache != b->cache) + return a->cache - b->cache; + else if (a->core != b->core) + return a->core - b->core; + else + return a->thread_idx - b->thread_idx; +} + +struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, + aggr_cpu_id_get_t get_id, + void *data, bool needs_sort) +{ + unsigned int idx; + struct perf_cpu cpu; + struct cpu_aggr_map *c = cpu_aggr_map__empty_new(perf_cpu_map__nr(cpus)); + + if (!c) + return NULL; + + /* Reset size as it may only be partially filled */ + c->nr = 0; + + perf_cpu_map__for_each_cpu(cpu, idx, cpus) { + bool duplicate = false; + struct aggr_cpu_id cpu_id = get_id(cpu, data); + + for (int j = 0; j < c->nr; j++) { + if (aggr_cpu_id__equal(&cpu_id, &c->map[j])) { + duplicate = true; + break; + } + } + if (!duplicate) { + c->map[c->nr] = cpu_id; + c->nr++; + } + } + /* Trim. */ + if (c->nr != (int)perf_cpu_map__nr(cpus)) { + struct cpu_aggr_map *trimmed_c = + realloc(c, + sizeof(struct cpu_aggr_map) + sizeof(struct aggr_cpu_id) * c->nr); + + if (trimmed_c) + c = trimmed_c; + } + + /* ensure we process id in increasing order */ + if (needs_sort) + qsort(c->map, c->nr, sizeof(struct aggr_cpu_id), aggr_cpu_id__cmp); + + return c; + +} + +int cpu__get_die_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "die_id", &value); + + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data) +{ + struct aggr_cpu_id id; + int die; + + die = cpu__get_die_id(cpu); + /* There is no die_id on legacy system. */ + if (die < 0) + die = 0; + + /* + * die_id is relative to socket, so start + * with the socket ID and then add die to + * make a unique ID. + */ + id = aggr_cpu_id__socket(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + id.die = die; + return id; +} + +int cpu__get_cluster_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "cluster_id", &value); + + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__cluster(struct perf_cpu cpu, void *data) +{ + int cluster = cpu__get_cluster_id(cpu); + struct aggr_cpu_id id; + + /* There is no cluster_id on legacy system. */ + if (cluster < 0) + cluster = 0; + + id = aggr_cpu_id__die(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + id.cluster = cluster; + return id; +} + +int cpu__get_core_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "core_id", &value); + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data) +{ + struct aggr_cpu_id id; + int core = cpu__get_core_id(cpu); + + /* aggr_cpu_id__die returns a struct with socket die, and cluster set. */ + id = aggr_cpu_id__cluster(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + /* + * core_id is relative to socket and die, we need a global id. + * So we combine the result from cpu_map__get_die with the core id + */ + id.core = core; + return id; + +} + +struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data) +{ + struct aggr_cpu_id id; + + /* aggr_cpu_id__core returns a struct with socket, die and core set. */ + id = aggr_cpu_id__core(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + id.cpu = cpu; + return id; + +} + +struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data __maybe_unused) +{ + struct aggr_cpu_id id = aggr_cpu_id__empty(); + + id.node = cpu__get_node(cpu); + return id; +} + +struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data __maybe_unused) +{ + struct aggr_cpu_id id = aggr_cpu_id__empty(); + + /* it always aggregates to the cpu 0 */ + cpu.cpu = 0; + id.cpu = cpu; + return id; +} + +/* setup simple routines to easily access node numbers given a cpu number */ +static int get_max_num(char *path, int *max) +{ + size_t num; + char *buf; + int err = 0; + + if (filename__read_str(path, &buf, &num)) + return -1; + + buf[num] = '\0'; + + /* empty file — nothing to parse */ + if (num == 0) { + err = -1; + goto out; + } + + /* start on the right, to find highest node num */ + while (--num) { + if ((buf[num] == ',') || (buf[num] == '-')) { + num++; + break; + } + } + if (sscanf(&buf[num], "%d", max) < 1) { + err = -1; + goto out; + } + + /* convert from 0-based to 1-based */ + (*max)++; + +out: + free(buf); + return err; +} + +/* Determine highest possible cpu in the system for sparse allocation */ +static void set_max_cpu_num(void) +{ + const char *mnt; + char path[PATH_MAX]; + int max, ret = -1; + + /* set up default */ + max_cpu_num.cpu = 4096; + max_present_cpu_num.cpu = 4096; + + mnt = sysfs__mountpoint(); + if (!mnt) + goto out; + + /* get the highest possible cpu number for a sparse allocation */ + ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/possible", mnt); + if (ret >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + goto out; + } + + ret = get_max_num(path, &max); + if (ret) + goto out; + + /* + * struct perf_cpu.cpu is int16_t (libperf ABI) — clamp to avoid + * truncation to negative. See tools/lib/perf/TODO for the ABI + * widening plan. + */ + if (max > INT16_MAX) { + pr_warning("WARNING: max possible cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } + max_cpu_num.cpu = max; + + /* get the highest present cpu number for a sparse allocation */ + ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/present", mnt); + if (ret >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + goto out; + } + + ret = get_max_num(path, &max); + + if (!ret && max > INT16_MAX) { + pr_warning("WARNING: max present cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } + if (!ret) + max_present_cpu_num.cpu = max; +out: + if (ret) + pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); +} + +/* Determine highest possible node in the system for sparse allocation */ +static void set_max_node_num(void) +{ + const char *mnt; + char path[PATH_MAX]; + int ret = -1; + + /* set up default */ + max_node_num = 8; + + mnt = sysfs__mountpoint(); + if (!mnt) + goto out; + + /* get the highest possible cpu number for a sparse allocation */ + ret = snprintf(path, PATH_MAX, "%s/devices/system/node/possible", mnt); + if (ret >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + goto out; + } + + ret = get_max_num(path, &max_node_num); + +out: + if (ret) + pr_err("Failed to read max nodes, using default of %d\n", max_node_num); +} + +int cpu__max_node(void) +{ + if (unlikely(!max_node_num)) + set_max_node_num(); + + return max_node_num; +} + +struct perf_cpu cpu__max_cpu(void) +{ + if (unlikely(!max_cpu_num.cpu)) + set_max_cpu_num(); + + return max_cpu_num; +} + +struct perf_cpu cpu__max_present_cpu(void) +{ + if (unlikely(!max_present_cpu_num.cpu)) + set_max_cpu_num(); + + return max_present_cpu_num; +} + + +int cpu__get_node(struct perf_cpu cpu) +{ + if (unlikely(cpunode_map == NULL)) { + pr_debug("cpu_map not initialized\n"); + return -1; + } + + /* cpunode_map allocated for max_cpu_num entries; input may be untrusted */ + if (cpu.cpu < 0 || cpu.cpu >= max_cpu_num.cpu) + return -1; + + return cpunode_map[cpu.cpu]; +} + +static int init_cpunode_map(void) +{ + int i; + + set_max_cpu_num(); + set_max_node_num(); + + cpunode_map = calloc(max_cpu_num.cpu, sizeof(int)); + if (!cpunode_map) { + pr_err("%s: calloc failed\n", __func__); + return -1; + } + + for (i = 0; i < max_cpu_num.cpu; i++) + cpunode_map[i] = -1; + + return 0; +} + +int cpu__setup_cpunode_map(void) +{ + struct dirent *dent1, *dent2; + DIR *dir1, *dir2; + unsigned int cpu, mem; + char buf[PATH_MAX]; + char path[PATH_MAX]; + const char *mnt; + int n; + + /* initialize globals */ + if (init_cpunode_map()) + return -1; + + mnt = sysfs__mountpoint(); + if (!mnt) + return 0; + + n = snprintf(path, PATH_MAX, "%s/devices/system/node", mnt); + if (n >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + return -1; + } + + dir1 = opendir(path); + if (!dir1) + return 0; + + /* walk tree and setup map */ + while ((dent1 = readdir(dir1)) != NULL) { + if (dent1->d_type != DT_DIR || sscanf(dent1->d_name, "node%u", &mem) < 1) + continue; + + n = snprintf(buf, PATH_MAX, "%s/%s", path, dent1->d_name); + if (n >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + continue; + } + + dir2 = opendir(buf); + if (!dir2) + continue; + while ((dent2 = readdir(dir2)) != NULL) { + if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) + continue; + /* cpunode_map allocated for max_cpu_num entries */ + if (cpu < (unsigned int)max_cpu_num.cpu) + cpunode_map[cpu] = mem; + } + closedir(dir2); + } + closedir(dir1); + return 0; +} + +size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) +{ + int i, start = -1; + bool first = true; + size_t ret = 0; + +#define COMMA first ? "" : "," + + for (i = 0; i < (int)perf_cpu_map__nr(map) + 1; i++) { + struct perf_cpu cpu = { .cpu = INT16_MAX }; + bool last = i == (int)perf_cpu_map__nr(map); + + if (!last) + cpu = perf_cpu_map__cpu(map, i); + + if (start == -1) { + start = i; + if (last) { + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, i).cpu); + } + } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { + int end = i - 1; + + if (start == end) { + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, start).cpu); + } else { + ret += scnprintf(buf + ret, size - ret, + "%s%d-%d", COMMA, + perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); + } + first = false; + start = i; + } + } + +#undef COMMA + + pr_debug2("cpumask list: %s\n", buf); + return ret; +} + +static char hex_char(unsigned char val) +{ + if (val < 10) + return val + '0'; + if (val < 16) + return val - 10 + 'a'; + return '?'; +} + +size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size) +{ + unsigned int idx; + char *ptr = buf; + unsigned char *bitmap; + struct perf_cpu c, last_cpu = perf_cpu_map__max(map); + + if (buf == NULL || size == 0) + return 0; + + if (last_cpu.cpu < 0) { + buf[0] = '\0'; + return 0; + } + + bitmap = zalloc(last_cpu.cpu / 8 + 1); + if (bitmap == NULL) { + buf[0] = '\0'; + return 0; + } + + perf_cpu_map__for_each_cpu_skip_any(c, idx, map) + bitmap[c.cpu / 8] |= 1 << (c.cpu % 8); + + for (int cpu = last_cpu.cpu / 4 * 4; cpu >= 0; cpu -= 4) { + unsigned char bits = bitmap[cpu / 8]; + + if (cpu % 8) + bits >>= 4; + else + bits &= 0xf; + + *ptr++ = hex_char(bits); + if ((cpu % 32) == 0 && cpu > 0) + *ptr++ = ','; + } + *ptr = '\0'; + free(bitmap); + + buf[size - 1] = '\0'; + return ptr - buf; +} + +struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ +{ + static struct perf_cpu_map *online; + + if (!online) + online = perf_cpu_map__new_online_cpus(); /* from /sys/devices/system/cpu/online */ + + return perf_cpu_map__get(online); +} + +bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) +{ + return a->thread_idx == b->thread_idx && + a->node == b->node && + a->socket == b->socket && + a->die == b->die && + a->cluster == b->cluster && + a->cache_lvl == b->cache_lvl && + a->cache == b->cache && + a->core == b->core && + a->cpu.cpu == b->cpu.cpu; +} + +bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) +{ + return a->thread_idx == -1 && + a->node == -1 && + a->socket == -1 && + a->die == -1 && + a->cluster == -1 && + a->cache_lvl == -1 && + a->cache == -1 && + a->core == -1 && + a->cpu.cpu == -1; +} + +struct aggr_cpu_id aggr_cpu_id__empty(void) +{ + struct aggr_cpu_id ret = { + .thread_idx = -1, + .node = -1, + .socket = -1, + .die = -1, + .cluster = -1, + .cache_lvl = -1, + .cache = -1, + .core = -1, + .cpu = (struct perf_cpu){ .cpu = -1 }, + }; + return ret; +} diff --git a/tools/perf/util/cpumap.h b/tools/perf/util/cpumap.h new file mode 100644 index 000000000..ee0f6139b --- /dev/null +++ b/tools/perf/util/cpumap.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CPUMAP_H +#define __PERF_CPUMAP_H + +#include +#include +#include + +/** Identify where counts are aggregated, -1 implies not to aggregate. */ +struct aggr_cpu_id { + /** A value in the range 0 to number of threads. */ + int thread_idx; + /** The numa node X as read from /sys/devices/system/node/nodeX. */ + int node; + /** + * The socket number as read from + * /sys/devices/system/cpu/cpuX/topology/physical_package_id. + */ + int socket; + /** The die id as read from /sys/devices/system/cpu/cpuX/topology/die_id. */ + int die; + /** The cluster id as read from /sys/devices/system/cpu/cpuX/topology/cluster_id */ + int cluster; + /** The cache level as read from /sys/devices/system/cpu/cpuX/cache/indexY/level */ + int cache_lvl; + /** + * The cache instance ID, which is the first CPU in the + * /sys/devices/system/cpu/cpuX/cache/indexY/shared_cpu_list + */ + int cache; + /** The core id as read from /sys/devices/system/cpu/cpuX/topology/core_id. */ + int core; + /** CPU aggregation, note there is one CPU for each SMT thread. */ + struct perf_cpu cpu; +}; + +/** A collection of aggr_cpu_id values, the "built" version is sorted and uniqued. */ +struct cpu_aggr_map { + /** Number of valid entries. */ + int nr; + /** The entries. */ + struct aggr_cpu_id map[]; +}; + +#define cpu_aggr_map__for_each_idx(idx, aggr_map) \ + for ((idx) = 0; (idx) < aggr_map->nr; (idx)++) + +struct perf_record_cpu_map_data; + +bool perf_record_cpu_map_data__test_bit(int i, const struct perf_record_cpu_map_data *data); + +struct perf_cpu_map *perf_cpu_map__empty_new(int nr); + +struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data); +size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size); +size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size); +size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp); +struct perf_cpu_map *cpu_map__online(void); /* thread unsafe */ + +int cpu__setup_cpunode_map(void); + +int cpu__max_node(void); +struct perf_cpu cpu__max_cpu(void); +struct perf_cpu cpu__max_present_cpu(void); + +/** + * cpu_map__is_dummy - Events associated with a pid, rather than a CPU, use a single dummy map with an entry of -1. + */ +static inline bool cpu_map__is_dummy(const struct perf_cpu_map *cpus) +{ + return perf_cpu_map__nr(cpus) == 1 && perf_cpu_map__cpu(cpus, 0).cpu == -1; +} + +/** + * cpu__get_node - Returns the numa node X as read from + * /sys/devices/system/node/nodeX for the given CPU. + */ +int cpu__get_node(struct perf_cpu cpu); +/** + * cpu__get_socket_id - Returns the socket number as read from + * /sys/devices/system/cpu/cpuX/topology/physical_package_id for the given CPU. + */ +int cpu__get_socket_id(struct perf_cpu cpu); +/** + * cpu__get_die_id - Returns the die id as read from + * /sys/devices/system/cpu/cpuX/topology/die_id for the given CPU. + */ +int cpu__get_die_id(struct perf_cpu cpu); +/** + * cpu__get_cluster_id - Returns the cluster id as read from + * /sys/devices/system/cpu/cpuX/topology/cluster_id for the given CPU + */ +int cpu__get_cluster_id(struct perf_cpu cpu); +/** + * cpu__get_core_id - Returns the core id as read from + * /sys/devices/system/cpu/cpuX/topology/core_id for the given CPU. + */ +int cpu__get_core_id(struct perf_cpu cpu); + +/** + * cpu_aggr_map__empty_new - Create a cpu_aggr_map of size nr with every entry + * being empty. + */ +struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr); + +typedef struct aggr_cpu_id (*aggr_cpu_id_get_t)(struct perf_cpu cpu, void *data); + +/** + * cpu_aggr_map__new - Create a cpu_aggr_map with an aggr_cpu_id for each cpu in + * cpus. The aggr_cpu_id is created with 'get_id' that may have a data value + * passed to it. The cpu_aggr_map is sorted with duplicate values removed. + */ +struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, + aggr_cpu_id_get_t get_id, + void *data, bool needs_sort); + +bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b); +bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a); +struct aggr_cpu_id aggr_cpu_id__empty(void); + + +/** + * aggr_cpu_id__socket - Create an aggr_cpu_id with the socket populated with + * the socket for cpu. The function signature is compatible with + * aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__die - Create an aggr_cpu_id with the die and socket populated + * with the die and socket for cpu. The function signature is compatible with + * aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__cluster - Create an aggr_cpu_id with cluster, die and socket + * populated with the cluster, die and socket for cpu. The function signature + * is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__cluster(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__core - Create an aggr_cpu_id with the core, cluster, die and + * socket populated with the core, die and socket for cpu. The function + * signature is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__core - Create an aggr_cpu_id with the cpu, core, die and socket + * populated with the cpu, core, die and socket for cpu. The function signature + * is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__node - Create an aggr_cpu_id with the numa node populated for + * cpu. The function signature is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__global - Create an aggr_cpu_id for global aggregation. + * The function signature is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data); +#endif /* __PERF_CPUMAP_H */ diff --git a/tools/perf/util/cputopo.c b/tools/perf/util/cputopo.c new file mode 100644 index 000000000..e0091804f --- /dev/null +++ b/tools/perf/util/cputopo.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cputopo.h" +#include "cpumap.h" +#include "debug.h" +#include "env.h" +#include "pmu.h" +#include "pmus.h" + +#define PACKAGE_CPUS_FMT \ + "%s/devices/system/cpu/cpu%d/topology/package_cpus_list" +#define PACKAGE_CPUS_FMT_OLD \ + "%s/devices/system/cpu/cpu%d/topology/core_siblings_list" +#define DIE_CPUS_FMT \ + "%s/devices/system/cpu/cpu%d/topology/die_cpus_list" +#define CORE_CPUS_FMT \ + "%s/devices/system/cpu/cpu%d/topology/core_cpus_list" +#define CORE_CPUS_FMT_OLD \ + "%s/devices/system/cpu/cpu%d/topology/thread_siblings_list" +#define NODE_ONLINE_FMT \ + "%s/devices/system/node/online" +#define NODE_MEMINFO_FMT \ + "%s/devices/system/node/node%d/meminfo" +#define NODE_CPULIST_FMT \ + "%s/devices/system/node/node%d/cpulist" + +static int build_cpu_topology(struct cpu_topology *tp, int cpu) +{ + FILE *fp; + char filename[MAXPATHLEN]; + char *buf = NULL, *p; + size_t len = 0; + ssize_t sret; + u32 i = 0; + int ret = -1; + + scnprintf(filename, MAXPATHLEN, PACKAGE_CPUS_FMT, + sysfs__mountpoint(), cpu); + if (access(filename, F_OK) == -1) { + scnprintf(filename, MAXPATHLEN, PACKAGE_CPUS_FMT_OLD, + sysfs__mountpoint(), cpu); + } + fp = fopen(filename, "r"); + if (!fp) + goto try_dies; + + sret = getline(&buf, &len, fp); + fclose(fp); + if (sret <= 0) + goto try_dies; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + for (i = 0; i < tp->package_cpus_lists; i++) { + if (!strcmp(buf, tp->package_cpus_list[i])) + break; + } + if (i == tp->package_cpus_lists) { + tp->package_cpus_list[i] = buf; + tp->package_cpus_lists++; + buf = NULL; + len = 0; + } + ret = 0; + +try_dies: + if (!tp->die_cpus_list) + goto try_threads; + + scnprintf(filename, MAXPATHLEN, DIE_CPUS_FMT, + sysfs__mountpoint(), cpu); + fp = fopen(filename, "r"); + if (!fp) + goto try_threads; + + sret = getline(&buf, &len, fp); + fclose(fp); + if (sret <= 0) + goto try_threads; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + for (i = 0; i < tp->die_cpus_lists; i++) { + if (!strcmp(buf, tp->die_cpus_list[i])) + break; + } + if (i == tp->die_cpus_lists) { + tp->die_cpus_list[i] = buf; + tp->die_cpus_lists++; + buf = NULL; + len = 0; + } + ret = 0; + +try_threads: + scnprintf(filename, MAXPATHLEN, CORE_CPUS_FMT, + sysfs__mountpoint(), cpu); + if (access(filename, F_OK) == -1) { + scnprintf(filename, MAXPATHLEN, CORE_CPUS_FMT_OLD, + sysfs__mountpoint(), cpu); + } + fp = fopen(filename, "r"); + if (!fp) + goto done; + + if (getline(&buf, &len, fp) <= 0) + goto done; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + for (i = 0; i < tp->core_cpus_lists; i++) { + if (!strcmp(buf, tp->core_cpus_list[i])) + break; + } + if (i == tp->core_cpus_lists) { + tp->core_cpus_list[i] = buf; + tp->core_cpus_lists++; + buf = NULL; + } + ret = 0; +done: + if (fp) + fclose(fp); + free(buf); + return ret; +} + +void cpu_topology__delete(struct cpu_topology *tp) +{ + u32 i; + + if (!tp) + return; + + for (i = 0 ; i < tp->package_cpus_lists; i++) + zfree(&tp->package_cpus_list[i]); + + for (i = 0 ; i < tp->die_cpus_lists; i++) + zfree(&tp->die_cpus_list[i]); + + for (i = 0 ; i < tp->core_cpus_lists; i++) + zfree(&tp->core_cpus_list[i]); + + free(tp); +} + +bool cpu_topology__smt_on(const struct cpu_topology *topology) +{ + for (u32 i = 0; i < topology->core_cpus_lists; i++) { + const char *cpu_list = topology->core_cpus_list[i]; + + /* + * If there is a need to separate siblings in a core then SMT is + * enabled. + */ + if (strchr(cpu_list, ',') || strchr(cpu_list, '-')) + return true; + } + return false; +} + +bool cpu_topology__core_wide(const struct cpu_topology *topology, + const char *user_requested_cpu_list) +{ + struct perf_cpu_map *user_requested_cpus; + + /* + * If user_requested_cpu_list is empty then all CPUs are recorded and so + * core_wide is true. + */ + if (!user_requested_cpu_list) + return true; + + user_requested_cpus = perf_cpu_map__new(user_requested_cpu_list); + /* Check that every user requested CPU is the complete set of SMT threads on a core. */ + for (u32 i = 0; i < topology->core_cpus_lists; i++) { + const char *core_cpu_list = topology->core_cpus_list[i]; + struct perf_cpu_map *core_cpus = perf_cpu_map__new(core_cpu_list); + struct perf_cpu cpu; + unsigned int idx; + bool has_first, first = true; + + perf_cpu_map__for_each_cpu(cpu, idx, core_cpus) { + if (first) { + has_first = perf_cpu_map__has(user_requested_cpus, cpu); + first = false; + } else { + /* + * If the first core CPU is user requested then + * all subsequent CPUs in the core must be user + * requested too. If the first CPU isn't user + * requested then none of the others must be + * too. + */ + if (perf_cpu_map__has(user_requested_cpus, cpu) != has_first) { + perf_cpu_map__put(core_cpus); + perf_cpu_map__put(user_requested_cpus); + return false; + } + } + } + perf_cpu_map__put(core_cpus); + } + perf_cpu_map__put(user_requested_cpus); + return true; +} + +static bool has_die_topology(void) +{ + char filename[MAXPATHLEN]; + struct utsname uts; + + if (uname(&uts) < 0) + return false; + + if (strncmp(uts.machine, "x86_64", 6) && + strncmp(uts.machine, "s390x", 5)) + return false; + + scnprintf(filename, MAXPATHLEN, DIE_CPUS_FMT, + sysfs__mountpoint(), 0); + if (access(filename, F_OK) == -1) + return false; + + return true; +} + +const struct cpu_topology *online_topology(void) +{ + static const struct cpu_topology *topology; + + if (!topology) { + topology = cpu_topology__new(); + if (!topology) { + pr_err("Error creating CPU topology"); + abort(); + } + } + return topology; +} + +struct cpu_topology *cpu_topology__new(void) +{ + struct cpu_topology *tp = NULL; + void *addr; + u32 nr, i, nr_addr; + size_t sz; + long ncpus; + int ret = -1; + struct perf_cpu_map *map; + bool has_die = has_die_topology(); + + ncpus = cpu__max_present_cpu().cpu; + + /* build online CPU map */ + map = perf_cpu_map__new_online_cpus(); + if (map == NULL) { + pr_debug("failed to get system cpumap\n"); + return NULL; + } + + nr = (u32)(ncpus & UINT_MAX); + + sz = nr * sizeof(char *); + if (has_die) + nr_addr = 3; + else + nr_addr = 2; + addr = calloc(1, sizeof(*tp) + nr_addr * sz); + if (!addr) + goto out_free; + + tp = addr; + addr += sizeof(*tp); + tp->package_cpus_list = addr; + addr += sz; + if (has_die) { + tp->die_cpus_list = addr; + addr += sz; + } + tp->core_cpus_list = addr; + + for (i = 0; i < nr; i++) { + if (!perf_cpu_map__has(map, (struct perf_cpu){ .cpu = i })) + continue; + + ret = build_cpu_topology(tp, i); + if (ret < 0) + break; + } + +out_free: + perf_cpu_map__put(map); + if (ret) { + cpu_topology__delete(tp); + tp = NULL; + } + return tp; +} + +static int load_numa_node(struct numa_topology_node *node, int nr) +{ + char str[MAXPATHLEN]; + char field[32]; + char *buf = NULL, *p; + size_t len = 0; + int ret = -1; + FILE *fp; + u64 mem; + + node->node = (u32) nr; + + scnprintf(str, MAXPATHLEN, NODE_MEMINFO_FMT, + sysfs__mountpoint(), nr); + fp = fopen(str, "r"); + if (!fp) + return -1; + + while (getline(&buf, &len, fp) > 0) { + /* skip over invalid lines */ + if (!strchr(buf, ':')) + continue; + if (sscanf(buf, "%*s %*d %31s %"PRIu64, field, &mem) != 2) + goto err; + if (!strcmp(field, "MemTotal:")) + node->mem_total = mem; + if (!strcmp(field, "MemFree:")) + node->mem_free = mem; + if (node->mem_total && node->mem_free) + break; + } + + fclose(fp); + fp = NULL; + + scnprintf(str, MAXPATHLEN, NODE_CPULIST_FMT, + sysfs__mountpoint(), nr); + + fp = fopen(str, "r"); + if (!fp) + return -1; + + if (getline(&buf, &len, fp) <= 0) + goto err; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + node->cpus = buf; + fclose(fp); + return 0; + +err: + free(buf); + if (fp) + fclose(fp); + return ret; +} + +struct numa_topology *numa_topology__new(void) +{ + struct perf_cpu_map *node_map = NULL; + struct numa_topology *tp = NULL; + char path[MAXPATHLEN]; + char *buf = NULL; + size_t len = 0; + u32 nr, i; + FILE *fp; + char *c; + + scnprintf(path, MAXPATHLEN, NODE_ONLINE_FMT, + sysfs__mountpoint()); + + fp = fopen(path, "r"); + if (!fp) + return NULL; + + if (getline(&buf, &len, fp) <= 0) + goto out; + + c = strchr(buf, '\n'); + if (c) + *c = '\0'; + + node_map = perf_cpu_map__new(buf); + if (!node_map) + goto out; + + nr = (u32) perf_cpu_map__nr(node_map); + + tp = zalloc(sizeof(*tp) + sizeof(tp->nodes[0])*nr); + if (!tp) + goto out; + + tp->nr = nr; + + for (i = 0; i < nr; i++) { + if (load_numa_node(&tp->nodes[i], perf_cpu_map__cpu(node_map, i).cpu)) { + numa_topology__delete(tp); + tp = NULL; + break; + } + } + +out: + free(buf); + fclose(fp); + perf_cpu_map__put(node_map); + return tp; +} + +void numa_topology__delete(struct numa_topology *tp) +{ + u32 i; + + for (i = 0; i < tp->nr; i++) + zfree(&tp->nodes[i].cpus); + + free(tp); +} + +static int load_hybrid_node(struct hybrid_topology_node *node, + struct perf_pmu *pmu) +{ + char *buf = NULL, *p; + FILE *fp; + size_t len = 0; + + node->pmu_name = strdup(pmu->name); + if (!node->pmu_name) + return -1; + + fp = perf_pmu__open_file(pmu, "cpus"); + if (!fp) + goto err; + + if (getline(&buf, &len, fp) <= 0) { + fclose(fp); + goto err; + } + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + fclose(fp); + node->cpus = buf; + return 0; + +err: + zfree(&node->pmu_name); + free(buf); + return -1; +} + +struct hybrid_topology *hybrid_topology__new(void) +{ + struct perf_pmu *pmu = NULL; + struct hybrid_topology *tp = NULL; + int nr = perf_pmus__num_core_pmus(), i = 0; + + if (nr <= 1) + return NULL; + + tp = zalloc(sizeof(*tp) + sizeof(tp->nodes[0]) * nr); + if (!tp) + return NULL; + + tp->nr = nr; + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + if (load_hybrid_node(&tp->nodes[i], pmu)) { + hybrid_topology__delete(tp); + return NULL; + } + i++; + } + + return tp; +} + +void hybrid_topology__delete(struct hybrid_topology *tp) +{ + u32 i; + + for (i = 0; i < tp->nr; i++) { + zfree(&tp->nodes[i].pmu_name); + zfree(&tp->nodes[i].cpus); + } + + free(tp); +} diff --git a/tools/perf/util/cputopo.h b/tools/perf/util/cputopo.h new file mode 100644 index 000000000..8d42f6102 --- /dev/null +++ b/tools/perf/util/cputopo.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CPUTOPO_H +#define __PERF_CPUTOPO_H + +#include + +struct cpu_topology { + /* The number of unique package_cpus_lists below. */ + u32 package_cpus_lists; + /* The number of unique die_cpu_lists below. */ + u32 die_cpus_lists; + /* The number of unique core_cpu_lists below. */ + u32 core_cpus_lists; + /* + * An array of strings where each string is unique and read from + * /sys/devices/system/cpu/cpuX/topology/package_cpus_list. From the ABI + * each of these is a human-readable list of CPUs sharing the same + * physical_package_id. The format is like 0-3, 8-11, 14,17. + */ + const char **package_cpus_list; + /* + * An array of string where each string is unique and from + * /sys/devices/system/cpu/cpuX/topology/die_cpus_list. From the ABI + * each of these is a human-readable list of CPUs within the same die. + * The format is like 0-3, 8-11, 14,17. + */ + const char **die_cpus_list; + /* + * An array of string where each string is unique and from + * /sys/devices/system/cpu/cpuX/topology/core_cpus_list. From the ABI + * each of these is a human-readable list of CPUs within the same + * core. The format is like 0-3, 8-11, 14,17. + */ + const char **core_cpus_list; +}; + +struct numa_topology_node { + char *cpus; + u32 node; + u64 mem_total; + u64 mem_free; +}; + +struct numa_topology { + u32 nr; + struct numa_topology_node nodes[]; +}; + +struct hybrid_topology_node { + char *pmu_name; + char *cpus; +}; + +struct hybrid_topology { + u32 nr; + struct hybrid_topology_node nodes[]; +}; + +/* + * The topology for online CPUs, lazily created. + */ +const struct cpu_topology *online_topology(void); + +struct cpu_topology *cpu_topology__new(void); +void cpu_topology__delete(struct cpu_topology *tp); +/* Determine from the core list whether SMT was enabled. */ +bool cpu_topology__smt_on(const struct cpu_topology *topology); +/* Are the sets of SMT siblings all enabled or all disabled in user_requested_cpus. */ +bool cpu_topology__core_wide(const struct cpu_topology *topology, + const char *user_requested_cpu_list); + +struct numa_topology *numa_topology__new(void); +void numa_topology__delete(struct numa_topology *tp); + +struct hybrid_topology *hybrid_topology__new(void); +void hybrid_topology__delete(struct hybrid_topology *tp); + +#endif /* __PERF_CPUTOPO_H */ diff --git a/tools/perf/util/cs-etm-base.c b/tools/perf/util/cs-etm-base.c new file mode 100644 index 000000000..aebef71d3 --- /dev/null +++ b/tools/perf/util/cs-etm-base.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * File for any parts of the Coresight decoding that don't require + * OpenCSD. + */ + +#include +#include + +#include "cs-etm.h" + +static const char * const cs_etm_global_header_fmts[] = { + [CS_HEADER_VERSION] = " Header version %llx\n", + [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n", + [CS_ETM_SNAPSHOT] = " Snapshot %llx\n", +}; + +static const char * const cs_etm_priv_fmts[] = { + [CS_ETM_MAGIC] = " Magic number %llx\n", + [CS_ETM_CPU] = " CPU %lld\n", + [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", + [CS_ETM_ETMCR] = " ETMCR %llx\n", + [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %llx\n", + [CS_ETM_ETMCCER] = " ETMCCER %llx\n", + [CS_ETM_ETMIDR] = " ETMIDR %llx\n", +}; + +static const char * const cs_etmv4_priv_fmts[] = { + [CS_ETM_MAGIC] = " Magic number %llx\n", + [CS_ETM_CPU] = " CPU %lld\n", + [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", + [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %llx\n", + [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %llx\n", + [CS_ETMV4_TRCIDR0] = " TRCIDR0 %llx\n", + [CS_ETMV4_TRCIDR1] = " TRCIDR1 %llx\n", + [CS_ETMV4_TRCIDR2] = " TRCIDR2 %llx\n", + [CS_ETMV4_TRCIDR8] = " TRCIDR8 %llx\n", + [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n", + [CS_ETMV4_TS_SOURCE] = " TS_SOURCE %lld\n", +}; + +static const char * const cs_ete_priv_fmts[] = { + [CS_ETM_MAGIC] = " Magic number %llx\n", + [CS_ETM_CPU] = " CPU %lld\n", + [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", + [CS_ETE_TRCCONFIGR] = " TRCCONFIGR %llx\n", + [CS_ETE_TRCTRACEIDR] = " TRCTRACEIDR %llx\n", + [CS_ETE_TRCIDR0] = " TRCIDR0 %llx\n", + [CS_ETE_TRCIDR1] = " TRCIDR1 %llx\n", + [CS_ETE_TRCIDR2] = " TRCIDR2 %llx\n", + [CS_ETE_TRCIDR8] = " TRCIDR8 %llx\n", + [CS_ETE_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n", + [CS_ETE_TRCDEVARCH] = " TRCDEVARCH %llx\n", + [CS_ETE_TS_SOURCE] = " TS_SOURCE %lld\n", +}; + +static const char * const param_unk_fmt = + " Unknown parameter [%d] %"PRIx64"\n"; +static const char * const magic_unk_fmt = + " Magic number Unknown %"PRIx64"\n"; + +static int cs_etm__print_cpu_metadata_v0(u64 *val, int *offset) +{ + int i = *offset, j, nr_params = 0, fmt_offset; + u64 magic; + + /* check magic value */ + magic = val[i + CS_ETM_MAGIC]; + if ((magic != __perf_cs_etmv3_magic) && + (magic != __perf_cs_etmv4_magic)) { + /* failure - note bad magic value */ + fprintf(stdout, magic_unk_fmt, magic); + return -EINVAL; + } + + /* print common header block */ + fprintf(stdout, cs_etm_priv_fmts[CS_ETM_MAGIC], val[i++]); + fprintf(stdout, cs_etm_priv_fmts[CS_ETM_CPU], val[i++]); + + if (magic == __perf_cs_etmv3_magic) { + nr_params = CS_ETM_NR_TRC_PARAMS_V0; + fmt_offset = CS_ETM_ETMCR; + /* after common block, offset format index past NR_PARAMS */ + for (j = fmt_offset; j < nr_params + fmt_offset; j++, i++) + fprintf(stdout, cs_etm_priv_fmts[j], val[i]); + } else if (magic == __perf_cs_etmv4_magic) { + nr_params = CS_ETMV4_NR_TRC_PARAMS_V0; + fmt_offset = CS_ETMV4_TRCCONFIGR; + /* after common block, offset format index past NR_PARAMS */ + for (j = fmt_offset; j < nr_params + fmt_offset; j++, i++) + fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); + } + *offset = i; + return 0; +} + +static int cs_etm__print_cpu_metadata_v1(u64 *val, int *offset) +{ + int i = *offset, j, total_params = 0; + u64 magic; + + magic = val[i + CS_ETM_MAGIC]; + /* total params to print is NR_PARAMS + common block size for v1 */ + total_params = val[i + CS_ETM_NR_TRC_PARAMS] + CS_ETM_COMMON_BLK_MAX_V1; + + if (magic == __perf_cs_etmv3_magic) { + for (j = 0; j < total_params; j++, i++) { + /* if newer record - could be excess params */ + if (j >= CS_ETM_PRIV_MAX) + fprintf(stdout, param_unk_fmt, j, val[i]); + else + fprintf(stdout, cs_etm_priv_fmts[j], val[i]); + } + } else if (magic == __perf_cs_etmv4_magic) { + for (j = 0; j < total_params; j++, i++) { + /* if newer record - could be excess params */ + if (j >= CS_ETMV4_PRIV_MAX) + fprintf(stdout, param_unk_fmt, j, val[i]); + else + fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); + } + } else if (magic == __perf_cs_ete_magic) { + for (j = 0; j < total_params; j++, i++) { + /* if newer record - could be excess params */ + if (j >= CS_ETE_PRIV_MAX) + fprintf(stdout, param_unk_fmt, j, val[i]); + else + fprintf(stdout, cs_ete_priv_fmts[j], val[i]); + } + } else { + /* failure - note bad magic value and error out */ + fprintf(stdout, magic_unk_fmt, magic); + return -EINVAL; + } + *offset = i; + return 0; +} + +static void cs_etm__print_auxtrace_info(u64 *val, int num) +{ + int i, cpu = 0, version, err; + + version = val[0]; + + for (i = 0; i < CS_HEADER_VERSION_MAX; i++) + fprintf(stdout, cs_etm_global_header_fmts[i], val[i]); + + for (i = CS_HEADER_VERSION_MAX; cpu < num; cpu++) { + if (version == 0) + err = cs_etm__print_cpu_metadata_v0(val, &i); + /* printing same for both, but value bit flags added on v2 */ + else if ((version == 1) || (version == 2)) + err = cs_etm__print_cpu_metadata_v1(val, &i); + if (err) + return; + } +} + +/* + * Do some basic checks and print the auxtrace info header before calling + * into cs_etm__process_auxtrace_info_full() which requires OpenCSD to be + * linked in. This allows some basic debugging if OpenCSD is missing. + */ +int cs_etm__process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + int event_header_size = sizeof(struct perf_event_header); + int num_cpu; + u64 *ptr = NULL; + u64 hdr_version; + + /* Ensure priv[] is large enough for the global header entries */ + if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE + + CS_ETM_HEADER_SIZE)) + return -EINVAL; + + /* First the global part */ + ptr = (u64 *) auxtrace_info->priv; + + /* Look for version of the header */ + hdr_version = ptr[0]; + if (hdr_version > CS_HEADER_CURRENT_VERSION) { + pr_err("\nCS ETM Trace: Unknown Header Version = %#" PRIx64, hdr_version); + pr_err(", version supported <= %x\n", CS_HEADER_CURRENT_VERSION); + return -EINVAL; + } + + if (dump_trace) { + num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + cs_etm__print_auxtrace_info(ptr, num_cpu); + } + + return cs_etm__process_auxtrace_info_full(event, session); +} diff --git a/tools/perf/util/cs-etm-decoder/Build b/tools/perf/util/cs-etm-decoder/Build new file mode 100644 index 000000000..27550db2a --- /dev/null +++ b/tools/perf/util/cs-etm-decoder/Build @@ -0,0 +1 @@ +perf-util-y += cs-etm-decoder.o diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c new file mode 100644 index 000000000..26940f1f1 --- /dev/null +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(C) 2015-2018 Linaro Limited. + * + * Author: Tor Jeremiassen + * Author: Mathieu Poirier + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cs-etm.h" +#include "cs-etm-decoder.h" +#include "debug.h" +#include "intlist.h" + +/* use raw logging */ +#ifdef CS_DEBUG_RAW +#define CS_LOG_RAW_FRAMES +#define CS_PKT_MON 1 +#ifdef CS_RAW_PACKED +#define CS_RAW_DEBUG_FLAGS (OCSD_DFRMTR_UNPACKED_RAW_OUT | \ + OCSD_DFRMTR_PACKED_RAW_OUT) +#else +#define CS_RAW_DEBUG_FLAGS (OCSD_DFRMTR_UNPACKED_RAW_OUT) +#endif +#else +#define CS_PKT_MON 0 +#endif + +/* + * Assume a maximum of 0.1ns elapsed per instruction. This would be the + * case with a theoretical 10GHz core executing 1 instruction per cycle. + * Used to estimate the sample time for synthesized instructions because + * Coresight only emits a timestamp for a range of instructions rather + * than per instruction. + */ +const u32 INSTR_PER_NS = 10; + +struct cs_etm_decoder { + void *data; + void (*packet_printer)(const char *msg, void *data); + bool suppress_printing; + dcd_tree_handle_t dcd_tree; + cs_etm_mem_cb_type mem_access; + ocsd_datapath_resp_t prev_return; + const char *decoder_name; +}; + +static u32 +cs_etm_decoder__mem_access(const void *context, + const ocsd_vaddr_t address, + const ocsd_mem_space_acc_t mem_space, + const u8 trace_chan_id, + const u32 req_size, + u8 *buffer) +{ + struct cs_etm_decoder *decoder = (struct cs_etm_decoder *) context; + + return decoder->mem_access(decoder->data, trace_chan_id, address, + req_size, buffer, mem_space); +} + +int cs_etm_decoder__add_mem_access_cb(struct cs_etm_decoder *decoder, + u64 start, u64 end, + cs_etm_mem_cb_type cb_func) +{ + decoder->mem_access = cb_func; + + if (ocsd_dt_add_callback_trcid_mem_acc(decoder->dcd_tree, start, end, + OCSD_MEM_SPACE_ANY, + cs_etm_decoder__mem_access, + decoder)) + return -1; + + return 0; +} + +int cs_etm_decoder__reset(struct cs_etm_decoder *decoder) +{ + ocsd_datapath_resp_t dp_ret; + + decoder->prev_return = OCSD_RESP_CONT; + decoder->suppress_printing = true; + dp_ret = ocsd_dt_process_data(decoder->dcd_tree, OCSD_OP_RESET, + 0, 0, NULL, NULL); + decoder->suppress_printing = false; + if (OCSD_DATA_RESP_IS_FATAL(dp_ret)) + return -1; + + return 0; +} + +int cs_etm_decoder__get_packet(struct cs_etm_packet_queue *packet_queue, + struct cs_etm_packet *packet) +{ + if (!packet_queue || !packet) + return -EINVAL; + + /* Nothing to do, might as well just return */ + if (packet_queue->packet_count == 0) + return 0; + /* + * The queueing process in function cs_etm_decoder__buffer_packet() + * increments the tail *before* using it. This is somewhat counter + * intuitive but it has the advantage of centralizing tail management + * at a single location. Because of that we need to follow the same + * heuristic with the head, i.e we increment it before using its + * value. Otherwise the first element of the packet queue is not + * used. + */ + packet_queue->head = (packet_queue->head + 1) & + (CS_ETM_PACKET_MAX_BUFFER - 1); + + *packet = packet_queue->packet_buffer[packet_queue->head]; + + packet_queue->packet_count--; + + return 1; +} + +/* + * Calculate the number of nanoseconds elapsed. + * + * instr_count is updated in place with the remainder of the instructions + * which didn't make up a whole nanosecond. + */ +static u32 cs_etm_decoder__dec_instr_count_to_ns(u32 *instr_count) +{ + const u32 instr_copy = *instr_count; + + *instr_count %= INSTR_PER_NS; + return instr_copy / INSTR_PER_NS; +} + +static int cs_etm_decoder__gen_etmv3_config(struct cs_etm_trace_params *params, + ocsd_etmv3_cfg *config) +{ + config->reg_idr = params->etmv3.reg_idr; + config->reg_ctrl = params->etmv3.reg_ctrl; + config->reg_ccer = params->etmv3.reg_ccer; + config->reg_trc_id = params->etmv3.reg_trc_id; + config->arch_ver = ARCH_V7; + config->core_prof = profile_CortexA; + + return 0; +} + +#define TRCIDR1_TRCARCHMIN_SHIFT 4 +#define TRCIDR1_TRCARCHMIN_MASK GENMASK(7, 4) +#define TRCIDR1_TRCARCHMIN(x) (((x) & TRCIDR1_TRCARCHMIN_MASK) >> TRCIDR1_TRCARCHMIN_SHIFT) + +static enum _ocsd_arch_version cs_etm_decoder__get_etmv4_arch_ver(u32 reg_idr1) +{ + /* + * For ETMv4 if the trace minor version is 4 or more then we can assume + * the architecture is ARCH_AA64 rather than just V8. + * ARCH_V8 = V8 architecture + * ARCH_AA64 = Min v8r3 plus additional AA64 PE features + */ + return TRCIDR1_TRCARCHMIN(reg_idr1) >= 4 ? ARCH_AA64 : ARCH_V8; +} + +static void cs_etm_decoder__gen_etmv4_config(struct cs_etm_trace_params *params, + ocsd_etmv4_cfg *config) +{ + config->reg_configr = params->etmv4.reg_configr; + config->reg_traceidr = params->etmv4.reg_traceidr; + config->reg_idr0 = params->etmv4.reg_idr0; + config->reg_idr1 = params->etmv4.reg_idr1; + config->reg_idr2 = params->etmv4.reg_idr2; + config->reg_idr8 = params->etmv4.reg_idr8; + config->reg_idr9 = 0; + config->reg_idr10 = 0; + config->reg_idr11 = 0; + config->reg_idr12 = 0; + config->reg_idr13 = 0; + config->arch_ver = cs_etm_decoder__get_etmv4_arch_ver(params->etmv4.reg_idr1); + config->core_prof = profile_CortexA; +} + +static void cs_etm_decoder__gen_ete_config(struct cs_etm_trace_params *params, + ocsd_ete_cfg *config) +{ + config->reg_configr = params->ete.reg_configr; + config->reg_traceidr = params->ete.reg_traceidr; + config->reg_idr0 = params->ete.reg_idr0; + config->reg_idr1 = params->ete.reg_idr1; + config->reg_idr2 = params->ete.reg_idr2; + config->reg_idr8 = params->ete.reg_idr8; + config->reg_devarch = params->ete.reg_devarch; + config->arch_ver = ARCH_AA64; + config->core_prof = profile_CortexA; +} + +static void cs_etm_decoder__print_str_cb(const void *p_context, + const char *msg, + const int str_len) +{ + const struct cs_etm_decoder *decoder = p_context; + + if (p_context && str_len && !decoder->suppress_printing) + decoder->packet_printer(msg, decoder->data); +} + +static int +cs_etm_decoder__init_def_logger_printing(struct cs_etm_decoder_params *d_params, + struct cs_etm_decoder *decoder) +{ + int ret = 0; + + if (d_params->packet_printer == NULL) + return -1; + + decoder->packet_printer = d_params->packet_printer; + + /* + * Set up a library default logger to process any printers + * (packet/raw frame) we add later. + */ + ret = ocsd_def_errlog_init(OCSD_ERR_SEV_ERROR, 1); + if (ret != 0) + return -1; + + /* no stdout / err / file output */ + ret = ocsd_def_errlog_config_output(C_API_MSGLOGOUT_FLG_NONE, NULL); + if (ret != 0) + return -1; + + /* + * Set the string CB for the default logger, passes strings to + * perf print logger. + */ + ret = ocsd_def_errlog_set_strprint_cb(decoder->dcd_tree, + (void *)decoder, + cs_etm_decoder__print_str_cb); + if (ret != 0) + return -1; + +#ifdef CS_LOG_RAW_FRAMES + /* + * Only log raw frames if --dump operation and hardware is actually + * generating formatted CoreSight trace frames + */ + if ((d_params->operation == CS_ETM_OPERATION_PRINT) && + (d_params->formatted == true)) { + /* use the built in library printer for the raw frames */ + ret = ocsd_dt_set_raw_frame_printer(decoder->dcd_tree, + CS_RAW_DEBUG_FLAGS); + if (ret != 0) + return -1; + } +#endif + return 0; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__do_soft_timestamp(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const uint8_t trace_chan_id) +{ + u64 estimated_ts; + + /* No timestamp packet has been received, nothing to do */ + if (!packet_queue->next_cs_timestamp) + return OCSD_RESP_CONT; + + estimated_ts = packet_queue->cs_timestamp + + cs_etm_decoder__dec_instr_count_to_ns(&packet_queue->instr_count); + + /* Estimated TS can never be higher than the next real one in the trace */ + packet_queue->cs_timestamp = min(packet_queue->next_cs_timestamp, estimated_ts); + + /* Tell the front end which traceid_queue needs attention */ + cs_etm__etmq_set_traceid_queue_timestamp(etmq, trace_chan_id); + + return OCSD_RESP_WAIT; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__do_hard_timestamp(struct cs_etm_queue *etmq, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id, + const ocsd_trc_index_t indx) +{ + struct cs_etm_packet_queue *packet_queue; + u64 converted_timestamp; + u64 estimated_first_ts; + + /* First get the packet queue for this traceID */ + packet_queue = cs_etm__etmq_get_packet_queue(etmq, trace_chan_id); + if (!packet_queue) + return OCSD_RESP_FATAL_SYS_ERR; + + /* + * Coresight timestamps are raw timer values which need to be scaled to ns. Assume + * 0 is a bad value so don't try to convert it. + */ + converted_timestamp = elem->timestamp ? + cs_etm__convert_sample_time(etmq, elem->timestamp) : 0; + + /* + * We've seen a timestamp packet before - simply record the new value. + * Function do_soft_timestamp() will report the value to the front end, + * hence asking the decoder to keep decoding rather than stopping. + */ + if (packet_queue->next_cs_timestamp) { + /* + * What was next is now where new ranges start from, overwriting + * any previous estimate in cs_timestamp + */ + packet_queue->cs_timestamp = packet_queue->next_cs_timestamp; + packet_queue->next_cs_timestamp = converted_timestamp; + return OCSD_RESP_CONT; + } + + if (!converted_timestamp) { + /* + * Zero timestamps can be seen due to misconfiguration or hardware bugs. + * Warn once, and don't try to subtract instr_count as it would result in an + * underflow. + */ + packet_queue->cs_timestamp = 0; + if (!cs_etm__etmq_is_timeless(etmq)) + pr_warning_once("Zero Coresight timestamp found at Idx:%" OCSD_TRC_IDX_STR + ". Decoding may be improved by prepending 'Z' to your current --itrace arguments.\n", + indx); + + } else if (packet_queue->instr_count / INSTR_PER_NS > converted_timestamp) { + /* + * Sanity check that the elem->timestamp - packet_queue->instr_count would not + * result in an underflow. Warn and clamp at 0 if it would. + */ + packet_queue->cs_timestamp = 0; + pr_err("Timestamp calculation underflow at Idx:%" OCSD_TRC_IDX_STR "\n", indx); + } else { + /* + * This is the first timestamp we've seen since the beginning of traces + * or a discontinuity. Since timestamps packets are generated *after* + * range packets have been generated, we need to estimate the time at + * which instructions started by subtracting the number of instructions + * executed to the timestamp. Don't estimate earlier than the last used + * timestamp though. + */ + estimated_first_ts = converted_timestamp - + (packet_queue->instr_count / INSTR_PER_NS); + packet_queue->cs_timestamp = max(packet_queue->cs_timestamp, estimated_first_ts); + } + packet_queue->next_cs_timestamp = converted_timestamp; + packet_queue->instr_count = 0; + + /* Tell the front end which traceid_queue needs attention */ + cs_etm__etmq_set_traceid_queue_timestamp(etmq, trace_chan_id); + + /* Halt processing until we are being told to proceed */ + return OCSD_RESP_WAIT; +} + +static void +cs_etm_decoder__reset_timestamp(struct cs_etm_packet_queue *packet_queue) +{ + packet_queue->next_cs_timestamp = 0; + packet_queue->instr_count = 0; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_packet(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const u8 trace_chan_id, + enum cs_etm_sample_type sample_type) +{ + u32 et = 0; + int cpu; + + if (packet_queue->packet_count >= CS_ETM_PACKET_MAX_BUFFER - 1) + return OCSD_RESP_FATAL_SYS_ERR; + + if (cs_etm__get_cpu(etmq, trace_chan_id, &cpu) < 0) + return OCSD_RESP_FATAL_SYS_ERR; + + et = packet_queue->tail; + et = (et + 1) & (CS_ETM_PACKET_MAX_BUFFER - 1); + packet_queue->tail = et; + packet_queue->packet_count++; + + packet_queue->packet_buffer[et].sample_type = sample_type; + packet_queue->packet_buffer[et].isa = CS_ETM_ISA_UNKNOWN; + packet_queue->packet_buffer[et].cpu = cpu; + packet_queue->packet_buffer[et].start_addr = CS_ETM_INVAL_ADDR; + packet_queue->packet_buffer[et].end_addr = CS_ETM_INVAL_ADDR; + packet_queue->packet_buffer[et].instr_count = 0; + packet_queue->packet_buffer[et].last_instr_taken_branch = false; + packet_queue->packet_buffer[et].last_instr_size = 0; + packet_queue->packet_buffer[et].last_instr_type = 0; + packet_queue->packet_buffer[et].last_instr_subtype = 0; + packet_queue->packet_buffer[et].last_instr_cond = 0; + packet_queue->packet_buffer[et].flags = 0; + packet_queue->packet_buffer[et].exception_number = UINT32_MAX; + packet_queue->packet_buffer[et].trace_chan_id = trace_chan_id; + packet_queue->packet_buffer[et].el = ocsd_EL_unknown; + packet_queue->packet_buffer[et].tid = -1; + + if (packet_queue->packet_count == CS_ETM_PACKET_MAX_BUFFER - 1) + return OCSD_RESP_WAIT; + + return OCSD_RESP_CONT; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id) +{ + int ret = 0; + struct cs_etm_packet *packet; + + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_RANGE); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + + switch (elem->isa) { + case ocsd_isa_aarch64: + packet->isa = CS_ETM_ISA_A64; + break; + case ocsd_isa_arm: + packet->isa = CS_ETM_ISA_A32; + break; + case ocsd_isa_thumb2: + packet->isa = CS_ETM_ISA_T32; + break; + case ocsd_isa_tee: + case ocsd_isa_jazelle: + case ocsd_isa_custom: + case ocsd_isa_unknown: + default: + packet->isa = CS_ETM_ISA_UNKNOWN; + } + + packet->start_addr = elem->st_addr; + packet->end_addr = elem->en_addr; + packet->instr_count = elem->num_instr_range; + packet->last_instr_type = elem->last_i_type; + packet->last_instr_subtype = elem->last_i_subtype; + packet->last_instr_cond = elem->last_instr_cond; + packet->el = elem->context.exception_level; + + if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT) + packet->last_instr_taken_branch = elem->last_instr_exec; + else + packet->last_instr_taken_branch = false; + + packet->last_instr_size = elem->last_instr_sz; + + /* per-thread scenario, no need to generate a timestamp */ + if (cs_etm__etmq_is_timeless(etmq)) + goto out; + + /* + * The packet queue is full and we haven't seen a timestamp (had we + * seen one the packet queue wouldn't be full). Let the front end + * deal with it. + */ + if (ret == OCSD_RESP_WAIT) + goto out; + + packet_queue->instr_count += elem->num_instr_range; + /* Tell the front end we have a new timestamp to process */ + ret = cs_etm_decoder__do_soft_timestamp(etmq, packet_queue, + trace_chan_id); +out: + return ret; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_discontinuity(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *queue, + const uint8_t trace_chan_id) +{ + /* + * Something happened and who knows when we'll get new traces so + * reset time statistics. + */ + cs_etm_decoder__reset_timestamp(queue); + return cs_etm_decoder__buffer_packet(etmq, queue, trace_chan_id, + CS_ETM_DISCONTINUITY); +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_exception(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *queue, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id) +{ int ret = 0; + struct cs_etm_packet *packet; + + ret = cs_etm_decoder__buffer_packet(etmq, queue, trace_chan_id, + CS_ETM_EXCEPTION); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &queue->packet_buffer[queue->tail]; + packet->exception_number = elem->exception_number; + + return ret; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_exception_ret(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *queue, + const uint8_t trace_chan_id) +{ + return cs_etm_decoder__buffer_packet(etmq, queue, trace_chan_id, + CS_ETM_EXCEPTION_RET); +} + +static ocsd_datapath_resp_t +cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id) +{ + struct cs_etm_packet *packet; + pid_t tid = -1; + int ret; + + /* + * Process the PE_CONTEXT packets if we have a valid contextID or VMID. + * If the kernel is running at EL2, the PID is traced in CONTEXTIDR_EL2 + * as VMID, Format attribute 'contextid2' is set in this case. + */ + switch (cs_etm__get_pid_fmt(etmq)) { + case CS_ETM_PIDFMT_CTXTID: + if (elem->context.ctxt_id_valid) + tid = elem->context.context_id; + break; + case CS_ETM_PIDFMT_CTXTID2: + if (elem->context.vmid_valid) + tid = elem->context.vmid; + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + } + + if (cs_etm__etmq_update_decode_context(etmq, trace_chan_id, + elem->context.exception_level, tid)) + return OCSD_RESP_FATAL_SYS_ERR; + + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_CONTEXT); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + packet->tid = tid; + packet->el = elem->context.exception_level; + + /* + * A timestamp is generated after a PE_CONTEXT element so make sure + * to rely on that coming one. + */ + cs_etm_decoder__reset_timestamp(packet_queue); + + return ret; +} + +static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( + const void *context, + const ocsd_trc_index_t indx, + const u8 trace_chan_id __maybe_unused, + const ocsd_generic_trace_elem *elem) +{ + ocsd_datapath_resp_t resp = OCSD_RESP_CONT; + ocsd_gen_trc_elem_t type; + struct cs_etm_decoder *decoder = (struct cs_etm_decoder *) context; + struct cs_etm_queue *etmq = decoder->data; + struct cs_etm_packet_queue *packet_queue; + + /* First get the packet queue for this traceID */ + packet_queue = cs_etm__etmq_get_packet_queue(etmq, trace_chan_id); + if (!packet_queue) + return OCSD_RESP_FATAL_SYS_ERR; + + type = elem->elem_type; + + if (type == OCSD_GEN_TRC_ELEM_EO_TRACE || + type == OCSD_GEN_TRC_ELEM_NO_SYNC || + type == OCSD_GEN_TRC_ELEM_TRACE_ON) + resp = cs_etm_decoder__buffer_discontinuity(etmq, packet_queue, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_INSTR_RANGE) + resp = cs_etm_decoder__buffer_range(etmq, packet_queue, elem, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_EXCEPTION) + resp = cs_etm_decoder__buffer_exception(etmq, packet_queue, elem, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_EXCEPTION_RET) + resp = cs_etm_decoder__buffer_exception_ret(etmq, packet_queue, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_TIMESTAMP) + resp = cs_etm_decoder__do_hard_timestamp(etmq, elem, + trace_chan_id, + indx); + else if (type == OCSD_GEN_TRC_ELEM_PE_CONTEXT) + resp = cs_etm_decoder__set_tid(etmq, packet_queue, + elem, trace_chan_id); + + return resp; +} + +static int +cs_etm_decoder__create_etm_decoder(struct cs_etm_decoder_params *d_params, + struct cs_etm_trace_params *t_params, + struct cs_etm_decoder *decoder) +{ + ocsd_etmv3_cfg config_etmv3; + ocsd_etmv4_cfg trace_config_etmv4; + ocsd_ete_cfg trace_config_ete; + void *trace_config; + u8 csid; + + switch (t_params->protocol) { + case CS_ETM_PROTO_ETMV3: + case CS_ETM_PROTO_PTM: + csid = (t_params->etmv3.reg_idr & CORESIGHT_TRACE_ID_VAL_MASK); + cs_etm_decoder__gen_etmv3_config(t_params, &config_etmv3); + decoder->decoder_name = (t_params->protocol == CS_ETM_PROTO_ETMV3) ? + OCSD_BUILTIN_DCD_ETMV3 : + OCSD_BUILTIN_DCD_PTM; + trace_config = &config_etmv3; + break; + case CS_ETM_PROTO_ETMV4i: + csid = (t_params->etmv4.reg_traceidr & CORESIGHT_TRACE_ID_VAL_MASK); + cs_etm_decoder__gen_etmv4_config(t_params, &trace_config_etmv4); + decoder->decoder_name = OCSD_BUILTIN_DCD_ETMV4I; + trace_config = &trace_config_etmv4; + break; + case CS_ETM_PROTO_ETE: + csid = (t_params->ete.reg_traceidr & CORESIGHT_TRACE_ID_VAL_MASK); + cs_etm_decoder__gen_ete_config(t_params, &trace_config_ete); + decoder->decoder_name = OCSD_BUILTIN_DCD_ETE; + trace_config = &trace_config_ete; + break; + default: + return -1; + } + + if (d_params->operation == CS_ETM_OPERATION_DECODE) { + int decode_flags = OCSD_CREATE_FLG_FULL_DECODER; +#ifdef OCSD_OPFLG_N_UNCOND_DIR_BR_CHK + decode_flags |= OCSD_OPFLG_N_UNCOND_DIR_BR_CHK | OCSD_OPFLG_CHK_RANGE_CONTINUE | + ETM4_OPFLG_PKTDEC_AA64_OPCODE_CHK; +#endif + if (ocsd_dt_create_decoder(decoder->dcd_tree, + decoder->decoder_name, + decode_flags, + trace_config, &csid)) + return -1; + + if (ocsd_dt_set_gen_elem_outfn(decoder->dcd_tree, + cs_etm_decoder__gen_trace_elem_printer, + decoder)) + return -1; + + return 0; + } else if (d_params->operation == CS_ETM_OPERATION_PRINT) { + if (ocsd_dt_create_decoder(decoder->dcd_tree, decoder->decoder_name, + OCSD_CREATE_FLG_PACKET_PROC, + trace_config, &csid)) + return -1; + + if (ocsd_dt_set_pkt_protocol_printer(decoder->dcd_tree, csid, CS_PKT_MON)) + return -1; + + return 0; + } + + return -1; +} + +struct cs_etm_decoder * +cs_etm_decoder__new(int decoders, struct cs_etm_decoder_params *d_params, + struct cs_etm_trace_params t_params[]) +{ + struct cs_etm_decoder *decoder; + ocsd_dcd_tree_src_t format; + u32 flags; + int i, ret; + + if ((!t_params) || (!d_params)) + return NULL; + + decoder = zalloc(sizeof(*decoder)); + + if (!decoder) + return NULL; + + decoder->data = d_params->data; + decoder->prev_return = OCSD_RESP_CONT; + format = (d_params->formatted ? OCSD_TRC_SRC_FRAME_FORMATTED : + OCSD_TRC_SRC_SINGLE); + flags = 0; + flags |= (d_params->fsyncs ? OCSD_DFRMTR_HAS_FSYNCS : 0); + flags |= (d_params->hsyncs ? OCSD_DFRMTR_HAS_HSYNCS : 0); + flags |= (d_params->frame_aligned ? OCSD_DFRMTR_FRAME_MEM_ALIGN : 0); + + /* + * Drivers may add barrier frames when used with perf, set up to + * handle this. Barriers const of FSYNC packet repeated 4 times. + */ + flags |= OCSD_DFRMTR_RESET_ON_4X_FSYNC; + + /* Create decode tree for the data source */ + decoder->dcd_tree = ocsd_create_dcd_tree(format, flags); + + if (decoder->dcd_tree == 0) + goto err_free_decoder; + + /* init library print logging support */ + ret = cs_etm_decoder__init_def_logger_printing(d_params, decoder); + if (ret != 0) + goto err_free_decoder; + + for (i = 0; i < decoders; i++) { + ret = cs_etm_decoder__create_etm_decoder(d_params, + &t_params[i], + decoder); + if (ret != 0) + goto err_free_decoder; + } + + return decoder; + +err_free_decoder: + cs_etm_decoder__free(decoder); + return NULL; +} + +int cs_etm_decoder__process_data_block(struct cs_etm_decoder *decoder, + u64 indx, const u8 *buf, + size_t len, size_t *consumed) +{ + int ret = 0; + ocsd_datapath_resp_t cur = OCSD_RESP_CONT; + ocsd_datapath_resp_t prev_return = decoder->prev_return; + size_t processed = 0; + u32 count; + + while (processed < len) { + if (OCSD_DATA_RESP_IS_WAIT(prev_return)) { + cur = ocsd_dt_process_data(decoder->dcd_tree, + OCSD_OP_FLUSH, + 0, + 0, + NULL, + NULL); + } else if (OCSD_DATA_RESP_IS_CONT(prev_return)) { + cur = ocsd_dt_process_data(decoder->dcd_tree, + OCSD_OP_DATA, + indx + processed, + len - processed, + &buf[processed], + &count); + processed += count; + } else { + ret = -EINVAL; + break; + } + + /* + * Return to the input code if the packet buffer is full. + * Flushing will get done once the packet buffer has been + * processed. + */ + if (OCSD_DATA_RESP_IS_WAIT(cur)) + break; + + prev_return = cur; + } + + decoder->prev_return = cur; + *consumed = processed; + + return ret; +} + +void cs_etm_decoder__free(struct cs_etm_decoder *decoder) +{ + if (!decoder) + return; + + ocsd_destroy_dcd_tree(decoder->dcd_tree); + decoder->dcd_tree = NULL; + free(decoder); +} + +const char *cs_etm_decoder__get_name(struct cs_etm_decoder *decoder) +{ + return decoder->decoder_name; +} diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h new file mode 100644 index 000000000..12c782fa6 --- /dev/null +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Copyright(C) 2015-2018 Linaro Limited. + * + * Author: Tor Jeremiassen + * Author: Mathieu Poirier + */ + +#ifndef INCLUDE__CS_ETM_DECODER_H__ +#define INCLUDE__CS_ETM_DECODER_H__ + +#include +#include +#include + +struct cs_etm_decoder; +struct cs_etm_packet; +struct cs_etm_packet_queue; + +struct cs_etm_queue; + +typedef u32 (*cs_etm_mem_cb_type)(struct cs_etm_queue *, u8, u64, size_t, u8 *, + const ocsd_mem_space_acc_t); + +struct cs_etmv3_trace_params { + u32 reg_ctrl; + u32 reg_trc_id; + u32 reg_ccer; + u32 reg_idr; +}; + +struct cs_etmv4_trace_params { + u32 reg_idr0; + u32 reg_idr1; + u32 reg_idr2; + u32 reg_idr8; + u32 reg_configr; + u32 reg_traceidr; +}; + +struct cs_ete_trace_params { + u32 reg_idr0; + u32 reg_idr1; + u32 reg_idr2; + u32 reg_idr8; + u32 reg_configr; + u32 reg_traceidr; + u32 reg_devarch; +}; + +struct cs_etm_trace_params { + int protocol; + union { + struct cs_etmv3_trace_params etmv3; + struct cs_etmv4_trace_params etmv4; + struct cs_ete_trace_params ete; + }; +}; + +struct cs_etm_decoder_params { + int operation; + void (*packet_printer)(const char *msg, void *data); + cs_etm_mem_cb_type mem_acc_cb; + bool formatted; + bool fsyncs; + bool hsyncs; + bool frame_aligned; + void *data; +}; + +/* + * The following enums are indexed starting with 1 to align with the + * open source coresight trace decoder library. + */ +enum { + CS_ETM_PROTO_ETMV3 = 1, + CS_ETM_PROTO_ETMV4i, + CS_ETM_PROTO_ETMV4d, + CS_ETM_PROTO_PTM, + CS_ETM_PROTO_ETE +}; + +enum cs_etm_decoder_operation { + CS_ETM_OPERATION_PRINT = 1, + CS_ETM_OPERATION_DECODE, + CS_ETM_OPERATION_MAX, +}; + +int cs_etm_decoder__process_data_block(struct cs_etm_decoder *decoder, + u64 indx, const u8 *buf, + size_t len, size_t *consumed); + +struct cs_etm_decoder * +cs_etm_decoder__new(int num_cpu, + struct cs_etm_decoder_params *d_params, + struct cs_etm_trace_params t_params[]); + +void cs_etm_decoder__free(struct cs_etm_decoder *decoder); + +int cs_etm_decoder__add_mem_access_cb(struct cs_etm_decoder *decoder, + u64 start, u64 end, + cs_etm_mem_cb_type cb_func); + +int cs_etm_decoder__get_packet(struct cs_etm_packet_queue *packet_queue, + struct cs_etm_packet *packet); + +int cs_etm_decoder__reset(struct cs_etm_decoder *decoder); +const char *cs_etm_decoder__get_name(struct cs_etm_decoder *decoder); + +#endif /* INCLUDE__CS_ETM_DECODER_H__ */ diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c new file mode 100644 index 000000000..114b3cd2d --- /dev/null +++ b/tools/perf/util/cs-etm.c @@ -0,0 +1,3771 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(C) 2015-2018 Linaro Limited. + * + * Author: Tor Jeremiassen + * Author: Mathieu Poirier + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "auxtrace.h" +#include "callchain.h" +#include "color.h" +#include "cs-etm.h" +#include "cs-etm-decoder/cs-etm-decoder.h" +#include "debug.h" +#include "dso.h" +#include "evlist.h" +#include "intlist.h" +#include "machine.h" +#include "map.h" +#include "perf.h" +#include "session.h" +#include "map_symbol.h" +#include "branch.h" +#include "symbol.h" +#include "tool.h" +#include "thread.h" +#include "thread-stack.h" +#include "tsc.h" +#include +#include "util/synthetic-events.h" +#include "util/util.h" + +struct cs_etm_auxtrace { + struct auxtrace auxtrace; + struct auxtrace_queues queues; + struct auxtrace_heap heap; + struct itrace_synth_opts synth_opts; + struct perf_session *session; + struct perf_tsc_conversion tc; + + /* + * Timeless has no timestamps in the trace so overlapping mmap lookups + * are less accurate but produces smaller trace data. We use context IDs + * in the trace instead of matching timestamps with fork records so + * they're not really needed in the general case. Overlapping mmaps + * happen in cases like between a fork and an exec. + */ + bool timeless_decoding; + + /* + * Per-thread ignores the trace channel ID and instead assumes that + * everything in a buffer comes from the same process regardless of + * which CPU it ran on. It also implies no context IDs so the TID is + * taken from the auxtrace buffer. + */ + bool per_thread_decoding; + bool snapshot_mode; + bool data_queued; + bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */ + bool use_thread_stack; + bool use_callchain; + + int num_cpu; + u64 latest_kernel_timestamp; + u32 auxtrace_type; + u32 branches_filter; + u64 branches_sample_type; + u64 branches_id; + u64 instructions_sample_type; + u64 instructions_sample_period; + u64 instructions_id; + u64 **metadata; + unsigned int pmu_type; + enum cs_etm_pid_fmt pid_fmt; +}; + +struct cs_etm_traceid_queue { + u8 trace_chan_id; + u64 period_instructions; + u64 kernel_start; + union perf_event *event_buf; + unsigned int br_stack_sz; + struct branch_stack *last_branch; + struct ip_callchain *callchain; + struct cs_etm_packet *prev_packet; + struct cs_etm_packet *packet; + struct cs_etm_packet_queue packet_queue; + + struct thread *decode_thread; + ocsd_ex_level decode_el; + + /* + * The frontend accesses the EL from '[prev_]packet' because it needs + * previous EL for branch and current EL for instruction samples. It's + * not possible to change thread in a single branch sample so no need to + * store or access the thread through the packet. + */ + struct thread *frontend_thread; +}; + +enum cs_etm_format { + UNSET, + FORMATTED, + UNFORMATTED +}; + +struct cs_etm_queue { + struct cs_etm_auxtrace *etm; + struct cs_etm_decoder *decoder; + struct auxtrace_buffer *buffer; + unsigned int queue_nr; + u8 pending_timestamp_chan_id; + enum cs_etm_format format; + u64 offset; + const unsigned char *buf; + size_t buf_len, buf_used; + /* Conversion between traceID and index in traceid_queues array */ + struct intlist *traceid_queues_list; + struct cs_etm_traceid_queue **traceid_queues; + /* Conversion between traceID and metadata pointers */ + struct intlist *traceid_list; + /* + * Same as traceid_list, but traceid_list may be a reference to another + * queue's which has a matching sink ID. + */ + struct intlist *own_traceid_list; + u32 sink_id; +}; + +static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm); +static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, + pid_t tid); +static int cs_etm__get_data_block(struct cs_etm_queue *etmq); +static int cs_etm__decode_data_block(struct cs_etm_queue *etmq); +static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata); +static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu); +static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata); + +/* PTMs ETMIDR [11:8] set to b0011 */ +#define ETMIDR_PTM_VERSION 0x00000300 + +/* + * A struct auxtrace_heap_item only has a queue_nr and a timestamp to + * work with. One option is to modify to auxtrace_heap_XYZ() API or simply + * encode the etm queue number as the upper 16 bit and the channel as + * the lower 16 bit. + */ +#define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \ + (queue_nr << 16 | trace_chan_id) +#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16) +#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff) +#define SINK_UNSET ((u32) -1) + +static u32 cs_etm__get_v7_protocol_version(u32 etmidr) +{ + etmidr &= ETMIDR_PTM_VERSION; + + if (etmidr == ETMIDR_PTM_VERSION) + return CS_ETM_PROTO_PTM; + + return CS_ETM_PROTO_ETMV3; +} + +static int cs_etm__get_magic(struct cs_etm_queue *etmq, u8 trace_chan_id, u64 *magic) +{ + struct int_node *inode; + u64 *metadata; + + inode = intlist__find(etmq->traceid_list, trace_chan_id); + if (!inode) + return -EINVAL; + + metadata = inode->priv; + *magic = metadata[CS_ETM_MAGIC]; + return 0; +} + +int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu) +{ + struct int_node *inode; + u64 *metadata; + + inode = intlist__find(etmq->traceid_list, trace_chan_id); + if (!inode) + return -EINVAL; + + metadata = inode->priv; + *cpu = (int)metadata[CS_ETM_CPU]; + return 0; +} + +/* + * The returned PID format is presented as an enum: + * + * CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced. + * CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced. + * CS_ETM_PIDFMT_NONE: No context IDs + * + * It's possible that the two format attributes 'contextid1' and 'contextid2' + * are enabled at the same time when the session runs on an EL2 kernel. + * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be + * recorded in the trace data, the tool will selectively use + * CONTEXTIDR_EL2 as PID. + * + * The result is cached in etm->pid_fmt so this function only needs to be called + * when processing the aux info. + */ +static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata) +{ + u64 val; + + if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) { + val = metadata[CS_ETM_ETMCR]; + /* CONTEXTIDR is traced */ + if (val & ETMCR_CTXTID) + return CS_ETM_PIDFMT_CTXTID; + } else { + val = metadata[CS_ETMV4_TRCCONFIGR]; + /* CONTEXTIDR_EL2 is traced */ + if (val & (TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT)) + return CS_ETM_PIDFMT_CTXTID2; + /* CONTEXTIDR_EL1 is traced */ + else if (val & TRCCONFIGR_CID) + return CS_ETM_PIDFMT_CTXTID; + } + + return CS_ETM_PIDFMT_NONE; +} + +enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq) +{ + return etmq->etm->pid_fmt; +} + +static int cs_etm__insert_trace_id_node(struct cs_etm_queue *etmq, + u8 trace_chan_id, u64 *cpu_metadata) +{ + /* Get an RB node for this CPU */ + struct int_node *inode = intlist__findnew(etmq->traceid_list, trace_chan_id); + + /* Something went wrong, no need to continue */ + if (!inode) + return -ENOMEM; + + /* Disallow re-mapping a different traceID to metadata pair. */ + if (inode->priv) { + u64 *curr_cpu_data = inode->priv; + u8 curr_chan_id; + int err; + + if (curr_cpu_data[CS_ETM_CPU] != cpu_metadata[CS_ETM_CPU]) { + /* + * With > CORESIGHT_TRACE_IDS_MAX ETMs, overlapping IDs + * are expected (but not supported) in per-thread mode, + * rather than signifying an error. + */ + if (etmq->etm->per_thread_decoding) + pr_err("CS_ETM: overlapping Trace IDs aren't currently supported in per-thread mode\n"); + else + pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n"); + + return -EINVAL; + } + + /* check that the mapped ID matches */ + err = cs_etm__metadata_get_trace_id(&curr_chan_id, curr_cpu_data); + if (err) + return err; + + if (curr_chan_id != trace_chan_id) { + pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n"); + return -EINVAL; + } + + /* Skip re-adding the same mappings if everything matched */ + return 0; + } + + /* Not one we've seen before, associate the traceID with the metadata pointer */ + inode->priv = cpu_metadata; + + return 0; +} + +static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int cpu) +{ + if (etm->per_thread_decoding) + return etm->queues.queue_array[0].priv; + + if (cpu < 0 || cpu >= (int)etm->queues.nr_queues) + return NULL; + + return etm->queues.queue_array[cpu].priv; +} + +static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id, + u64 *cpu_metadata) +{ + struct cs_etm_queue *etmq; + + /* + * If the queue is unformatted then only save one mapping in the + * queue associated with that CPU so only one decoder is made. + */ + etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]); + if (!etmq) + return -EINVAL; + + if (etmq->format == UNFORMATTED) + return cs_etm__insert_trace_id_node(etmq, trace_chan_id, + cpu_metadata); + + /* + * Otherwise, version 0 trace IDs are global so save them into every + * queue. + */ + for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { + int ret; + + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id, + cpu_metadata); + if (ret) + return ret; + } + + return 0; +} + +static int cs_etm__process_trace_id_v0(struct cs_etm_auxtrace *etm, int cpu, + u64 hw_id) +{ + int err; + u64 *cpu_data; + u8 trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + + cpu_data = get_cpu_data(etm, cpu); + if (cpu_data == NULL) + return -EINVAL; + + err = cs_etm__map_trace_id_v0(etm, trace_chan_id, cpu_data); + if (err) + return err; + + /* + * if we are picking up the association from the packet, need to plug + * the correct trace ID into the metadata for setting up decoders later. + */ + return cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data); +} + +static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, + u64 hw_id) +{ + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, cpu); + int ret; + u64 *cpu_data; + u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id); + u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + + if (!etmq) + return -EINVAL; + + /* + * Check sink id hasn't changed in per-cpu mode. In per-thread mode, + * let it pass for now until an actual overlapping trace ID is hit. In + * most cases IDs won't overlap even if the sink changes. + */ + if (!etmq->etm->per_thread_decoding && etmq->sink_id != SINK_UNSET && + etmq->sink_id != sink_id) { + pr_err("CS_ETM: mismatch between sink IDs\n"); + return -EINVAL; + } + + etmq->sink_id = sink_id; + + /* Find which other queues use this sink and link their ID maps */ + for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { + struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv; + + if (!other_etmq) + continue; + + /* Different sinks, skip */ + if (other_etmq->sink_id != etmq->sink_id) + continue; + + /* Already linked, skip */ + if (other_etmq->traceid_list == etmq->traceid_list) + continue; + + /* At the point of first linking, this one should be empty */ + if (!intlist__empty(etmq->traceid_list)) { + pr_err("CS_ETM: Can't link populated trace ID lists\n"); + return -EINVAL; + } + + etmq->own_traceid_list = NULL; + intlist__delete(etmq->traceid_list); + etmq->traceid_list = other_etmq->traceid_list; + break; + } + + cpu_data = get_cpu_data(etm, cpu); + if (!cpu_data) + return -EINVAL; + + ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data); + if (ret) + return ret; + + ret = cs_etm__metadata_set_trace_id(trace_id, cpu_data); + if (ret) + return ret; + + return 0; +} + +static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata) +{ + u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC]; + + switch (cs_etm_magic) { + case __perf_cs_etmv3_magic: + *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] & + CORESIGHT_TRACE_ID_VAL_MASK); + break; + case __perf_cs_etmv4_magic: + case __perf_cs_ete_magic: + *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] & + CORESIGHT_TRACE_ID_VAL_MASK); + break; + default: + return -EINVAL; + } + return 0; +} + +/* + * update metadata trace ID from the value found in the AUX_HW_INFO packet. + */ +static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata) +{ + u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC]; + + switch (cs_etm_magic) { + case __perf_cs_etmv3_magic: + cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id; + break; + case __perf_cs_etmv4_magic: + case __perf_cs_ete_magic: + cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id; + break; + + default: + return -EINVAL; + } + return 0; +} + +/* + * Get a metadata index for a specific cpu from an array. + * + */ +static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu) +{ + int i; + + for (i = 0; i < etm->num_cpu; i++) { + if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) { + return i; + } + } + + return -1; +} + +/* + * Get a metadata for a specific cpu from an array. + * + */ +static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu) +{ + int idx = get_cpu_data_idx(etm, cpu); + + return (idx != -1) ? etm->metadata[idx] : NULL; +} + +/* + * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event. + * + * The payload associates the Trace ID and the CPU. + * The routine is tolerant of seeing multiple packets with the same association, + * but a CPU / Trace ID association changing during a session is an error. + */ +static int cs_etm__process_aux_output_hw_id(struct perf_session *session, + union perf_event *event) +{ + struct cs_etm_auxtrace *etm; + struct perf_sample sample; + struct evsel *evsel; + u64 hw_id; + int cpu, version, err; + + /* extract and parse the HW ID */ + hw_id = event->aux_output_hw_id.hw_id; + version = FIELD_GET(CS_AUX_HW_ID_MAJOR_VERSION_MASK, hw_id); + + /* check that we can handle this version */ + if (version > CS_AUX_HW_ID_MAJOR_VERSION) { + pr_err("CS ETM Trace: PERF_RECORD_AUX_OUTPUT_HW_ID version %d not supported. Please update Perf.\n", + version); + return -EINVAL; + } + + /* get access to the etm metadata */ + etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace); + if (!etm || !etm->metadata) + return -EINVAL; + + /* parse the sample to get the CPU */ + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) + return -EINVAL; + perf_sample__init(&sample, /*all=*/false); + err = evsel__parse_sample(evsel, event, &sample); + if (err) + goto out; + cpu = sample.cpu; + if (cpu == -1) { + /* no CPU in the sample - possibly recorded with an old version of perf */ + pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record."); + err = -EINVAL; + goto out; + } + + if (FIELD_GET(CS_AUX_HW_ID_MINOR_VERSION_MASK, hw_id) == 0) { + err = cs_etm__process_trace_id_v0(etm, cpu, hw_id); + goto out; + } + + err = cs_etm__process_trace_id_v0_1(etm, cpu, hw_id); +out: + perf_sample__exit(&sample); + return err; +} + +void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, + u8 trace_chan_id) +{ + /* + * When a timestamp packet is encountered the backend code + * is stopped so that the front end has time to process packets + * that were accumulated in the traceID queue. Since there can + * be more than one channel per cs_etm_queue, we need to specify + * what traceID queue needs servicing. + */ + etmq->pending_timestamp_chan_id = trace_chan_id; +} + +static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, + u8 *trace_chan_id) +{ + struct cs_etm_packet_queue *packet_queue; + + if (!etmq->pending_timestamp_chan_id) + return 0; + + if (trace_chan_id) + *trace_chan_id = etmq->pending_timestamp_chan_id; + + packet_queue = cs_etm__etmq_get_packet_queue(etmq, + etmq->pending_timestamp_chan_id); + if (!packet_queue) + return 0; + + /* Acknowledge pending status */ + etmq->pending_timestamp_chan_id = 0; + + /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */ + return packet_queue->cs_timestamp; +} + +static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue) +{ + int i; + + queue->head = 0; + queue->tail = 0; + queue->packet_count = 0; + for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) { + queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN; + queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR; + queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR; + queue->packet_buffer[i].instr_count = 0; + queue->packet_buffer[i].last_instr_taken_branch = false; + queue->packet_buffer[i].last_instr_size = 0; + queue->packet_buffer[i].last_instr_type = 0; + queue->packet_buffer[i].last_instr_subtype = 0; + queue->packet_buffer[i].last_instr_cond = 0; + queue->packet_buffer[i].flags = 0; + queue->packet_buffer[i].exception_number = UINT32_MAX; + queue->packet_buffer[i].trace_chan_id = UINT8_MAX; + queue->packet_buffer[i].cpu = INT_MIN; + } +} + +static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq) +{ + int idx; + struct int_node *inode; + struct cs_etm_traceid_queue *tidq; + struct intlist *traceid_queues_list = etmq->traceid_queues_list; + + intlist__for_each_entry(inode, traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + cs_etm__clear_packet_queue(&tidq->packet_queue); + } +} + +static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + u8 trace_chan_id) +{ + int rc = -ENOMEM; + struct auxtrace_queue *queue; + struct cs_etm_auxtrace *etm = etmq->etm; + + cs_etm__clear_packet_queue(&tidq->packet_queue); + + queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; + tidq->trace_chan_id = trace_chan_id; + tidq->decode_el = ocsd_EL_unknown; + tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + if (!tidq->frontend_thread || !tidq->decode_thread) + goto out; + + tidq->packet = zalloc(sizeof(struct cs_etm_packet)); + if (!tidq->packet) + goto out; + + tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet)); + if (!tidq->prev_packet) + goto out_free; + + if (etm->use_thread_stack) { + size_t sz = sizeof(struct branch_stack); + + sz += etm->synth_opts.last_branch_sz * + sizeof(struct branch_entry); + tidq->last_branch = zalloc(sz); + if (!tidq->last_branch) + goto out_free; + + tidq->br_stack_sz = etm->synth_opts.last_branch_sz; + } + + if (etm->synth_opts.callchain) { + /* Add 1 to callchain_sz for callchain context */ + tidq->callchain = + zalloc(struct_size(tidq->callchain, ips, + etm->synth_opts.callchain_sz + 1)); + if (!tidq->callchain) + goto out_free; + } + + tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); + if (!tidq->event_buf) + goto out_free; + + return 0; + +out_free: + zfree(&tidq->callchain); + zfree(&tidq->last_branch); + zfree(&tidq->prev_packet); + zfree(&tidq->packet); +out: + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); + return rc; +} + +static struct cs_etm_traceid_queue +*cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) +{ + int idx; + struct int_node *inode; + struct intlist *traceid_queues_list; + struct cs_etm_traceid_queue *tidq, **traceid_queues; + struct cs_etm_auxtrace *etm = etmq->etm; + + if (etm->per_thread_decoding) + trace_chan_id = CS_ETM_PER_THREAD_TRACEID; + + traceid_queues_list = etmq->traceid_queues_list; + + /* + * Check if the traceid_queue exist for this traceID by looking + * in the queue list. + */ + inode = intlist__find(traceid_queues_list, trace_chan_id); + if (inode) { + idx = (int)(intptr_t)inode->priv; + return etmq->traceid_queues[idx]; + } + + /* We couldn't find a traceid_queue for this traceID, allocate one */ + tidq = malloc(sizeof(*tidq)); + if (!tidq) + return NULL; + + memset(tidq, 0, sizeof(*tidq)); + + /* Get a valid index for the new traceid_queue */ + idx = intlist__nr_entries(traceid_queues_list); + /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */ + inode = intlist__findnew(traceid_queues_list, trace_chan_id); + if (!inode) + goto out_free; + + /* Associate this traceID with this index */ + inode->priv = (void *)(intptr_t)idx; + + if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id)) + goto out_free; + + /* Grow the traceid_queues array by one unit */ + traceid_queues = etmq->traceid_queues; + traceid_queues = reallocarray(traceid_queues, + idx + 1, + sizeof(*traceid_queues)); + + /* + * On failure reallocarray() returns NULL and the original block of + * memory is left untouched. + */ + if (!traceid_queues) + goto out_free; + + traceid_queues[idx] = tidq; + etmq->traceid_queues = traceid_queues; + + return etmq->traceid_queues[idx]; + +out_free: + /* + * Function intlist__remove() removes the inode from the list + * and delete the memory associated to it. + */ + intlist__remove(traceid_queues_list, inode); + free(tidq); + + return NULL; +} + +struct cs_etm_packet_queue +*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (tidq) + return &tidq->packet_queue; + + return NULL; +} + +static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_packet *tmp; + + if (etm->synth_opts.branches || etm->synth_opts.last_branch || + etm->synth_opts.instructions) { + /* + * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for + * the next incoming packet. + */ + tmp = tidq->packet; + tidq->packet = tidq->prev_packet; + tidq->prev_packet = tmp; + } +} + +static void cs_etm__packet_dump(const char *pkt_string, void *data) +{ + const char *color = PERF_COLOR_BLUE; + int len = strlen(pkt_string); + struct cs_etm_queue *etmq = data; + char queue_nr[64]; + + if (verbose) + snprintf(queue_nr, sizeof(queue_nr), "Qnr:%u; ", etmq->queue_nr); + else + queue_nr[0] = '\0'; + + if (len && (pkt_string[len-1] == '\n')) + color_fprintf(stdout, color, " %s%s", queue_nr, pkt_string); + else + color_fprintf(stdout, color, " %s%s\n", queue_nr, pkt_string); + + fflush(stdout); +} + +static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params, + u64 *metadata, u32 etmidr) +{ + t_params->protocol = cs_etm__get_v7_protocol_version(etmidr); + t_params->etmv3.reg_ctrl = metadata[CS_ETM_ETMCR]; + t_params->etmv3.reg_trc_id = metadata[CS_ETM_ETMTRACEIDR]; +} + +static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params, + u64 *metadata) +{ + t_params->protocol = CS_ETM_PROTO_ETMV4i; + t_params->etmv4.reg_idr0 = metadata[CS_ETMV4_TRCIDR0]; + t_params->etmv4.reg_idr1 = metadata[CS_ETMV4_TRCIDR1]; + t_params->etmv4.reg_idr2 = metadata[CS_ETMV4_TRCIDR2]; + t_params->etmv4.reg_idr8 = metadata[CS_ETMV4_TRCIDR8]; + t_params->etmv4.reg_configr = metadata[CS_ETMV4_TRCCONFIGR]; + t_params->etmv4.reg_traceidr = metadata[CS_ETMV4_TRCTRACEIDR]; +} + +static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params, + u64 *metadata) +{ + t_params->protocol = CS_ETM_PROTO_ETE; + t_params->ete.reg_idr0 = metadata[CS_ETE_TRCIDR0]; + t_params->ete.reg_idr1 = metadata[CS_ETE_TRCIDR1]; + t_params->ete.reg_idr2 = metadata[CS_ETE_TRCIDR2]; + t_params->ete.reg_idr8 = metadata[CS_ETE_TRCIDR8]; + t_params->ete.reg_configr = metadata[CS_ETE_TRCCONFIGR]; + t_params->ete.reg_traceidr = metadata[CS_ETE_TRCTRACEIDR]; + t_params->ete.reg_devarch = metadata[CS_ETE_TRCDEVARCH]; +} + +static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params, + struct cs_etm_queue *etmq) +{ + struct int_node *inode; + + intlist__for_each_entry(inode, etmq->traceid_list) { + u64 *metadata = inode->priv; + u64 architecture = metadata[CS_ETM_MAGIC]; + u32 etmidr; + + switch (architecture) { + case __perf_cs_etmv3_magic: + etmidr = metadata[CS_ETM_ETMIDR]; + cs_etm__set_trace_param_etmv3(t_params++, metadata, etmidr); + break; + case __perf_cs_etmv4_magic: + cs_etm__set_trace_param_etmv4(t_params++, metadata); + break; + case __perf_cs_ete_magic: + cs_etm__set_trace_param_ete(t_params++, metadata); + break; + default: + return -EINVAL; + } + } + + return 0; +} + +static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params, + struct cs_etm_queue *etmq, + enum cs_etm_decoder_operation mode) +{ + int ret = -EINVAL; + + if (!(mode < CS_ETM_OPERATION_MAX)) + goto out; + + d_params->packet_printer = cs_etm__packet_dump; + d_params->operation = mode; + d_params->data = etmq; + d_params->formatted = etmq->format == FORMATTED; + d_params->fsyncs = false; + d_params->hsyncs = false; + d_params->frame_aligned = true; + + ret = 0; +out: + return ret; +} + +static void cs_etm__dump_event(struct cs_etm_queue *etmq, + struct auxtrace_buffer *buffer) +{ + int ret; + const char *color = PERF_COLOR_BLUE; + size_t buffer_used = 0; + + fprintf(stdout, "\n"); + color_fprintf(stdout, color, + ". ... CoreSight %s Trace data: size %#zx bytes\n", + cs_etm_decoder__get_name(etmq->decoder), buffer->size); + + do { + size_t consumed; + + ret = cs_etm_decoder__process_data_block( + etmq->decoder, buffer->offset, + &((u8 *)buffer->data)[buffer_used], + buffer->size - buffer_used, &consumed); + if (ret) + break; + + buffer_used += consumed; + } while (buffer_used < buffer->size); + + cs_etm_decoder__reset(etmq->decoder); +} + +static int cs_etm__flush_events(struct perf_session *session, + const struct perf_tool *tool) +{ + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + if (dump_trace) + return 0; + + if (!tool->ordered_events) + return -EINVAL; + + if (etm->timeless_decoding) { + /* + * Pass tid = -1 to process all queues. But likely they will have + * already been processed on PERF_RECORD_EXIT anyway. + */ + return cs_etm__process_timeless_queues(etm, -1); + } + + return cs_etm__process_timestamped_queues(etm); +} + +static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) +{ + int idx; + uintptr_t priv; + struct int_node *inode, *tmp; + struct cs_etm_traceid_queue *tidq; + struct intlist *traceid_queues_list = etmq->traceid_queues_list; + + intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) { + priv = (uintptr_t)inode->priv; + idx = priv; + + /* Free this traceid_queue from the array */ + tidq = etmq->traceid_queues[idx]; + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); + zfree(&tidq->event_buf); + zfree(&tidq->callchain); + zfree(&tidq->last_branch); + zfree(&tidq->prev_packet); + zfree(&tidq->packet); + zfree(&tidq); + + /* + * Function intlist__remove() removes the inode from the list + * and delete the memory associated to it. + */ + intlist__remove(traceid_queues_list, inode); + } + + /* Then the RB tree itself */ + intlist__delete(traceid_queues_list); + etmq->traceid_queues_list = NULL; + + /* finally free the traceid_queues array */ + zfree(&etmq->traceid_queues); +} + +static void cs_etm__free_queue(void *priv) +{ + struct int_node *inode, *tmp; + struct cs_etm_queue *etmq = priv; + + if (!etmq) + return; + + cs_etm_decoder__free(etmq->decoder); + cs_etm__free_traceid_queues(etmq); + + if (etmq->own_traceid_list) { + /* First remove all traceID/metadata nodes for the RB tree */ + intlist__for_each_entry_safe(inode, tmp, etmq->own_traceid_list) + intlist__remove(etmq->own_traceid_list, inode); + + /* Then the RB tree itself */ + intlist__delete(etmq->own_traceid_list); + } + + free(etmq); +} + +static void cs_etm__free_events(struct perf_session *session) +{ + unsigned int i; + struct cs_etm_auxtrace *aux = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + struct auxtrace_queues *queues = &aux->queues; + + for (i = 0; i < queues->nr_queues; i++) { + cs_etm__free_queue(queues->queue_array[i].priv); + queues->queue_array[i].priv = NULL; + } + + auxtrace_queues__free(queues); +} + +static void cs_etm__free(struct perf_session *session) +{ + int i; + struct cs_etm_auxtrace *aux = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + cs_etm__free_events(session); + session->auxtrace = NULL; + + for (i = 0; i < aux->num_cpu; i++) + zfree(&aux->metadata[i]); + + zfree(&aux->metadata); + zfree(&aux); +} + +static bool cs_etm__evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct cs_etm_auxtrace *aux = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + return evsel->core.attr.type == aux->pmu_type; +} + +static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq, + ocsd_ex_level el) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + /* + * For any virtualisation based on nVHE (e.g. pKVM), or host kernels + * running at EL1 assume everything is the host. + */ + if (pid_fmt == CS_ETM_PIDFMT_CTXTID) + return &etmq->etm->session->machines.host; + + /* + * Not perfect, but otherwise assume anything in EL1 is the default + * guest, and everything else is the host. Distinguishing between guest + * and host userspaces isn't currently supported either. Neither is + * multiple guest support. All this does is reduce the likeliness of + * decode errors where we look into the host kernel maps when it should + * have been the guest maps. + */ + switch (el) { + case ocsd_EL1: + return machines__find_guest(&etmq->etm->session->machines, + DEFAULT_GUEST_KERNEL_ID); + case ocsd_EL3: + case ocsd_EL2: + case ocsd_EL0: + case ocsd_EL_unknown: + default: + return &etmq->etm->session->machines.host; + } +} + +static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address, + ocsd_ex_level el) +{ + struct machine *machine = cs_etm__get_machine(etmq, el); + + if (address >= machine__kernel_start(machine)) { + if (machine__is_host(machine)) + return PERF_RECORD_MISC_KERNEL; + else + return PERF_RECORD_MISC_GUEST_KERNEL; + } else { + if (machine__is_host(machine)) + return PERF_RECORD_MISC_USER; + else { + /* + * Can't really happen at the moment because + * cs_etm__get_machine() will always return + * machines.host for any non EL1 trace. + */ + return PERF_RECORD_MISC_GUEST_USER; + } + } +} + +static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space, + ocsd_ex_level el, struct thread *thread) +{ + u8 cpumode; + u64 offset; + int len; + struct addr_location al; + struct dso *dso; + int ret = 0; + + if (!etmq) + return 0; + + addr_location__init(&al); + + /* + * We track EL for the frontend and the backend when receiving context + * and range packets. OpenCSD doesn't distinguish between EL0 and EL1 + * for this mem access callback so we had to do the extra tracking. Skip + * validation if it's any of the 'any' values. + */ + if (!(mem_space == OCSD_MEM_SPACE_ANY || + mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) { + if (mem_space & OCSD_MEM_SPACE_EL1N) { + /* Includes both non secure EL1 and EL0 */ + assert(el == ocsd_EL1 || el == ocsd_EL0); + } else if (mem_space & OCSD_MEM_SPACE_EL2) + assert(el == ocsd_EL2); + else if (mem_space & OCSD_MEM_SPACE_EL3) + assert(el == ocsd_EL3); + } + + cpumode = cs_etm__cpu_mode(etmq, address, el); + + if (!thread__find_map(thread, cpumode, address, &al)) + goto out; + + dso = map__dso(al.map); + if (!dso) + goto out; + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR && + dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE)) + goto out; + + offset = map__map_ip(al.map, address); + + map__load(al.map); + + len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), + offset, buffer, size); + + if (len <= 0) { + ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n" + " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n"); + if (!dso__auxtrace_warned(dso)) { + pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n", + address, + dso__long_name(dso) ? dso__long_name(dso) : "Unknown"); + dso__set_auxtrace_warned(dso); + } + goto out; + } + ret = len; +out: + addr_location__exit(&al); + return ret; +} + +static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 address, size_t size, u8 *buffer) +{ + return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el, + tidq->frontend_thread); +} + +static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return 0; + + return __cs_etm__mem_access(etmq, address, size, buffer, + mem_space, tidq->decode_el, + tidq->decode_thread); +} + +static struct cs_etm_queue *cs_etm__alloc_queue(void) +{ + struct cs_etm_queue *etmq = zalloc(sizeof(*etmq)); + if (!etmq) + return NULL; + + etmq->traceid_queues_list = intlist__new(NULL); + if (!etmq->traceid_queues_list) + goto out_free; + + /* + * Create an RB tree for traceID-metadata tuple. Since the conversion + * has to be made for each packet that gets decoded, optimizing access + * in anything other than a sequential array is worth doing. + */ + etmq->traceid_list = etmq->own_traceid_list = intlist__new(NULL); + if (!etmq->traceid_list) + goto out_free; + + return etmq; + +out_free: + intlist__delete(etmq->traceid_queues_list); + free(etmq); + + return NULL; +} + +static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, + struct auxtrace_queue *queue, + unsigned int queue_nr) +{ + struct cs_etm_queue *etmq = queue->priv; + + if (etmq) + return 0; + + etmq = cs_etm__alloc_queue(); + + if (!etmq) + return -ENOMEM; + + queue->priv = etmq; + etmq->etm = etm; + etmq->queue_nr = queue_nr; + queue->cpu = queue_nr; /* Placeholder, may be reset to -1 in per-thread mode */ + etmq->offset = 0; + etmq->sink_id = SINK_UNSET; + + return 0; +} + +static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm, + struct cs_etm_queue *etmq, + unsigned int queue_nr) +{ + int ret = 0; + unsigned int cs_queue_nr; + u8 trace_chan_id; + u64 cs_timestamp; + + /* + * We are under a CPU-wide trace scenario. As such we need to know + * when the code that generated the traces started to execute so that + * it can be correlated with execution on other CPUs. So we get a + * handle on the beginning of traces and decode until we find a + * timestamp. The timestamp is then added to the auxtrace min heap + * in order to know what nibble (of all the etmqs) to decode first. + */ + while (1) { + /* + * Fetch an aux_buffer from this etmq. Bail if no more + * blocks or an error has been encountered. + */ + ret = cs_etm__get_data_block(etmq); + if (ret <= 0) + goto out; + + /* + * Run decoder on the trace block. The decoder will stop when + * encountering a CS timestamp, a full packet queue or the end of + * trace for that block. + */ + ret = cs_etm__decode_data_block(etmq); + if (ret) + goto out; + + /* + * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all + * the timestamp calculation for us. + */ + cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); + + /* We found a timestamp, no need to continue. */ + if (cs_timestamp) + break; + + /* + * We didn't find a timestamp so empty all the traceid packet + * queues before looking for another timestamp packet, either + * in the current data block or a new one. Packets that were + * just decoded are useless since no timestamp has been + * associated with them. As such simply discard them. + */ + cs_etm__clear_all_packet_queues(etmq); + } + + /* + * We have a timestamp. Add it to the min heap to reflect when + * instructions conveyed by the range packets of this traceID queue + * started to execute. Once the same has been done for all the traceID + * queues of each etmq, redenring and decoding can start in + * chronological order. + * + * Note that packets decoded above are still in the traceID's packet + * queue and will be processed in cs_etm__process_timestamped_queues(). + */ + cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); + ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); +out: + return ret; +} + +static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr) +{ + u8 instrBytes[2]; + + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + ARRAY_SIZE(instrBytes), instrBytes); + /* + * T32 instruction size is indicated by bits[15:11] of the first + * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 + * denote a 32-bit instruction. + */ + return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2; +} + +static inline int cs_etm__instr_size(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 addr) +{ + if (packet->isa == CS_ETM_ISA_T32) + return cs_etm__t32_instr_size(etmq, tidq, packet, addr); + + /* Otherwise, 4-byte instruction size for A32/A64 */ + return 4; +} + +static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet) +{ + /* + * Return 0 for packets that have no addresses so that CS_ETM_INVAL_ADDR doesn't + * appear in samples. + */ + if (packet->sample_type == CS_ETM_DISCONTINUITY || + packet->sample_type == CS_ETM_EXCEPTION) + return 0; + + return packet->start_addr; +} + +static inline +u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) +{ + /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ + if (packet->sample_type == CS_ETM_DISCONTINUITY) + return 0; + + return packet->end_addr - packet->last_instr_size; +} + +static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 offset) +{ + u64 addr = packet->start_addr; + + /* 4-byte instruction size for A32/A64 */ + if (packet->isa == CS_ETM_ISA_A64 || packet->isa == CS_ETM_ISA_A32) + return addr + offset * 4; + + while (offset) { + addr += cs_etm__instr_size(etmq, tidq, packet, addr); + offset--; + } + return addr; +} + +static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event, + struct perf_sample *sample, u64 type) +{ + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && etm->session && etm->session->evlist) + evsel = evlist__id2evsel(etm->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); +} + + +static int +cs_etm__get_trace(struct cs_etm_queue *etmq) +{ + struct auxtrace_buffer *aux_buffer = etmq->buffer; + struct auxtrace_buffer *old_buffer = aux_buffer; + struct auxtrace_queue *queue; + + queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; + + aux_buffer = auxtrace_buffer__next(queue, aux_buffer); + + /* If no more data, drop the previous auxtrace_buffer and return */ + if (!aux_buffer) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + etmq->buf_len = 0; + return 0; + } + + etmq->buffer = aux_buffer; + + /* If the aux_buffer doesn't have data associated, try to load it */ + if (!aux_buffer->data) { + /* get the file desc associated with the perf data file */ + int fd = perf_data__fd(etmq->etm->session->data); + + aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd); + if (!aux_buffer->data) + return -ENOMEM; + } + + /* If valid, drop the previous buffer */ + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + + etmq->buf_used = 0; + etmq->buf_len = aux_buffer->size; + etmq->buf = aux_buffer->data; + return 0; +} + +/* + * Convert a raw thread number to a thread struct and assign it to **thread. + */ +static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq, + ocsd_ex_level el, pid_t tid, + struct thread **thread) +{ + struct machine *machine = cs_etm__get_machine(etmq, el); + + if (!machine || !*thread) + return -EINVAL; + + if (tid != -1) { + thread__zput(*thread); + *thread = machine__find_thread(machine, -1, tid); + } + + /* Couldn't find a known thread */ + if (!*thread) + *thread = machine__idle_thread(machine); + + return 0; +} + +/* + * Set the thread and EL of the decode context which is ahead in time of the + * frontend context. + */ +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, + ocsd_ex_level el, pid_t tid) +{ + struct cs_etm_traceid_queue *tidq; + int ret; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return -EINVAL; + + ret = cs_etm__etmq_update_thread(etmq, el, tid, + &tidq->decode_thread); + if (ret) + return ret; + + tidq->decode_el = el; + return 0; +} + +bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) +{ + return !!etmq->etm->timeless_decoding; +} + +static void cs_etm__copy_insn(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + struct perf_sample *sample) +{ + /* + * It's pointless to read instructions for the CS_ETM_DISCONTINUITY + * packet, so directly bail out with 'insn_len' = 0. + */ + if (packet->sample_type == CS_ETM_DISCONTINUITY) { + sample->insn_len = 0; + return; + } + + sample->insn_len = cs_etm__instr_size(etmq, tidq, packet, sample->ip); + + cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip, + sample->insn_len, (void *)sample->insn); +} + +u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + + if (etm->has_virtual_ts) + return tsc_to_perf_time(cs_timestamp, &etm->tc); + else + return cs_timestamp; +} + +static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue; + + if (!etm->timeless_decoding && etm->has_virtual_ts) + return packet_queue->cs_timestamp; + else + return etm->latest_kernel_timestamp; +} + +static bool cs_etm__packet_has_taken_branch(struct cs_etm_packet *packet) +{ + if (packet->sample_type == CS_ETM_RANGE && + packet->last_instr_taken_branch) + return true; + + return false; +} + +static void cs_etm__add_stack_event(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + u64 from, to; + int size; + + if (!etm->synth_opts.branches && !etm->synth_opts.instructions) + return; + + if (!cs_etm__packet_has_taken_branch(tidq->prev_packet)) + return; + + if (etmq->etm->use_thread_stack) { + from = cs_etm__last_executed_instr(tidq->prev_packet); + to = cs_etm__first_executed_instr(tidq->packet); + + size = cs_etm__instr_size(etmq, tidq, tidq->prev_packet, from); + + /* Enable callchain so thread stack entry can be allocated */ + thread_stack__event(tidq->frontend_thread, tidq->prev_packet->cpu, + tidq->prev_packet->flags, from, to, size, + etmq->buffer->buffer_nr + 1, + etmq->etm->use_callchain, + tidq->br_stack_sz, 0); + } else { + thread_stack__set_trace_nr(tidq->frontend_thread, + tidq->prev_packet->cpu, + etmq->buffer->buffer_nr + 1); + } +} + +static void cs_etm__sample_branch_stack(struct cs_etm_auxtrace *etm, + struct cs_etm_traceid_queue *tidq, + struct perf_sample *sample) +{ + if (etm->synth_opts.last_branch) { + thread_stack__br_sample(tidq->frontend_thread, tidq->packet->cpu, + tidq->last_branch, tidq->br_stack_sz); + sample->branch_stack = tidq->last_branch; + } + + if (etm->synth_opts.callchain) { + if (tidq->kernel_start) + thread_stack__sample(tidq->frontend_thread, + tidq->packet->cpu, + tidq->callchain, + etm->synth_opts.callchain_sz + 1, + sample->ip, tidq->kernel_start); + else + /* + * Clear the callchain when the kernel start address is + * not available yet. The empty callchain can then be + * consumed by cs_etm__inject_event(). + */ + memset(tidq->callchain, 0, + struct_size(tidq->callchain, ips, + etm->synth_opts.callchain_sz + 1)); + + sample->callchain = tidq->callchain; + } +} + +static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 addr, u64 period) +{ + int ret = 0; + struct cs_etm_auxtrace *etm = etmq->etm; + union perf_event *event = tidq->event_buf; + struct perf_sample sample; + + perf_sample__init(&sample, /*all=*/true); + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el); + event->sample.header.size = sizeof(struct perf_event_header); + + /* Set time field based on etm auxtrace config. */ + sample.time = cs_etm__resolve_sample_time(etmq, tidq); + + sample.ip = addr; + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); + sample.id = etmq->etm->instructions_id; + sample.stream_id = etmq->etm->instructions_id; + sample.period = period; + sample.cpu = packet->cpu; + sample.flags = tidq->prev_packet->flags; + sample.cpumode = event->sample.header.misc; + + cs_etm__copy_insn(etmq, tidq, packet, &sample); + cs_etm__sample_branch_stack(etm, tidq, &sample); + + if (etm->synth_opts.inject) { + ret = cs_etm__inject_event(etm, event, &sample, + etm->instructions_sample_type); + if (ret) + return ret; + } + + ret = perf_session__deliver_synth_event(etm->session, event, &sample); + + if (ret) + pr_err( + "CS ETM Trace: failed to deliver instruction event, error %d\n", + ret); + + perf_sample__exit(&sample); + return ret; +} + +/* + * The cs etm packet encodes an instruction range between a branch target + * and the next taken branch. Generate sample accordingly. + */ +static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int ret = 0; + struct cs_etm_auxtrace *etm = etmq->etm; + struct perf_sample sample; + union perf_event *event = tidq->event_buf; + + struct dummy_branch_stack { + u64 nr; + u64 hw_idx; + struct branch_entry entries; + } dummy_bs; + u64 ip; + + if (etm->branches_filter && + !(etm->branches_filter & tidq->prev_packet->flags)) + return 0; + + perf_sample__init(&sample, /*all=*/true); + ip = cs_etm__last_executed_instr(tidq->prev_packet); + + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = cs_etm__cpu_mode(etmq, ip, + tidq->prev_packet->el); + event->sample.header.size = sizeof(struct perf_event_header); + + /* Set time field based on etm auxtrace config. */ + sample.time = cs_etm__resolve_sample_time(etmq, tidq); + + sample.ip = ip; + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); + sample.addr = cs_etm__first_executed_instr(tidq->packet); + sample.id = etmq->etm->branches_id; + sample.stream_id = etmq->etm->branches_id; + sample.period = 1; + sample.cpu = tidq->packet->cpu; + sample.flags = tidq->prev_packet->flags; + sample.cpumode = event->sample.header.misc; + + cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample); + + /* + * perf report cannot handle events without a branch stack + */ + if (etm->synth_opts.last_branch) { + dummy_bs = (struct dummy_branch_stack){ + .nr = 1, + .hw_idx = -1ULL, + .entries = { + .from = sample.ip, + .to = sample.addr, + }, + }; + sample.branch_stack = (struct branch_stack *)&dummy_bs; + } + + if (etm->synth_opts.inject) { + ret = cs_etm__inject_event(etm, event, &sample, + etm->branches_sample_type); + if (ret) + return ret; + } + + ret = perf_session__deliver_synth_event(etm->session, event, &sample); + + if (ret) + pr_err( + "CS ETM Trace: failed to deliver instruction event, error %d\n", + ret); + + perf_sample__exit(&sample); + return ret; +} + +static int cs_etm__synth_events(struct cs_etm_auxtrace *etm, + struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + struct perf_event_attr attr; + bool found = false; + u64 id; + int err; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == etm->pmu_type) { + found = true; + break; + } + } + + if (!found) { + pr_debug("No selected events with CoreSight Trace data\n"); + return 0; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.size = sizeof(struct perf_event_attr); + attr.type = PERF_TYPE_HARDWARE; + attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; + attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | + PERF_SAMPLE_PERIOD; + if (etm->timeless_decoding) + attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; + else + attr.sample_type |= PERF_SAMPLE_TIME; + + attr.exclude_user = evsel->core.attr.exclude_user; + attr.exclude_kernel = evsel->core.attr.exclude_kernel; + attr.exclude_hv = evsel->core.attr.exclude_hv; + attr.exclude_host = evsel->core.attr.exclude_host; + attr.exclude_guest = evsel->core.attr.exclude_guest; + attr.sample_id_all = evsel->core.attr.sample_id_all; + attr.read_format = evsel->core.attr.read_format; + + /* create new id val to be a fixed offset from evsel id */ + id = auxtrace_synth_id_range_start(evsel); + + if (etm->synth_opts.branches) { + attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS; + attr.sample_period = 1; + attr.sample_type |= PERF_SAMPLE_ADDR; + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + etm->branches_sample_type = attr.sample_type; + etm->branches_id = id; + id += 1; + attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR; + } + + if (etm->synth_opts.last_branch) { + attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + /* + * We don't use the hardware index, but the sample generation + * code uses the new format branch_stack with this field, + * so the event attributes must indicate that it's present. + */ + attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + + if (etm->synth_opts.callchain) + attr.sample_type |= PERF_SAMPLE_CALLCHAIN; + + if (etm->synth_opts.instructions) { + attr.config = PERF_COUNT_HW_INSTRUCTIONS; + attr.sample_period = etm->synth_opts.period; + etm->instructions_sample_period = attr.sample_period; + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + etm->instructions_sample_type = attr.sample_type; + etm->instructions_id = id; + id += 1; + } + + return 0; +} + +static int cs_etm__sample(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + int ret; + u64 instrs_prev; + + /* Get instructions remainder from previous packet */ + instrs_prev = tidq->period_instructions; + + tidq->period_instructions += tidq->packet->instr_count; + + cs_etm__add_stack_event(etmq, tidq); + + if (etm->synth_opts.instructions && + tidq->period_instructions >= etm->instructions_sample_period) { + /* + * Emit instruction sample periodically + * TODO: allow period to be defined in cycles and clock time + */ + + /* + * Below diagram demonstrates the instruction samples + * generation flows: + * + * Instrs Instrs Instrs Instrs + * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3) + * | | | | + * V V V V + * -------------------------------------------------- + * ^ ^ + * | | + * Period Period + * instructions(Pi) instructions(Pi') + * + * | | + * \---------------- -----------------/ + * V + * tidq->packet->instr_count + * + * Instrs Sample(n...) are the synthesised samples occurring + * every etm->instructions_sample_period instructions - as + * defined on the perf command line. Sample(n) is being the + * last sample before the current etm packet, n+1 to n+3 + * samples are generated from the current etm packet. + * + * tidq->packet->instr_count represents the number of + * instructions in the current etm packet. + * + * Period instructions (Pi) contains the number of + * instructions executed after the sample point(n) from the + * previous etm packet. This will always be less than + * etm->instructions_sample_period. + * + * When generate new samples, it combines with two parts + * instructions, one is the tail of the old packet and another + * is the head of the new coming packet, to generate + * sample(n+1); sample(n+2) and sample(n+3) consume the + * instructions with sample period. After sample(n+3), the rest + * instructions will be used by later packet and it is assigned + * to tidq->period_instructions for next round calculation. + */ + + /* + * Get the initial offset into the current packet instructions; + * entry conditions ensure that instrs_prev is less than + * etm->instructions_sample_period. + */ + u64 offset = etm->instructions_sample_period - instrs_prev; + u64 addr; + + while (tidq->period_instructions >= + etm->instructions_sample_period) { + /* + * Calculate the address of the sampled instruction (-1 + * as sample is reported as though instruction has just + * been executed, but PC has not advanced to next + * instruction) + */ + addr = cs_etm__instr_addr(etmq, tidq, tidq->packet, + offset - 1); + ret = cs_etm__synth_instruction_sample( + etmq, tidq, tidq->packet, addr, + etm->instructions_sample_period); + if (ret) + return ret; + + offset += etm->instructions_sample_period; + tidq->period_instructions -= + etm->instructions_sample_period; + } + } + + if (etm->synth_opts.branches) { + bool generate_sample = false; + + /* Generate sample for tracing on packet */ + if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY) + generate_sample = true; + + /* Generate sample for branch taken packet */ + if (cs_etm__packet_has_taken_branch(tidq->prev_packet)) + generate_sample = true; + + if (generate_sample) { + ret = cs_etm__synth_branch_sample(etmq, tidq); + if (ret) + return ret; + } + } + + cs_etm__packet_swap(etm, tidq); + + return 0; +} + +static int cs_etm__context(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + ocsd_ex_level el = tidq->packet->el; + struct machine *machine; + int ret; + + machine = cs_etm__get_machine(etmq, el); + if (!machine) { + ret = -EINVAL; + goto err; + } + + tidq->kernel_start = machine__kernel_start(machine); + + ret = cs_etm__etmq_update_thread(etmq, el, tidq->packet->tid, + &tidq->frontend_thread); + if (ret) + goto err; + + return 0; + +err: + thread__zput(tidq->frontend_thread); + tidq->kernel_start = 0; + return ret; +} + +static int cs_etm__exception(struct cs_etm_traceid_queue *tidq) +{ + /* + * When the exception packet is inserted, whether the last instruction + * in previous range packet is taken branch or not, we need to force + * to set 'prev_packet->last_instr_taken_branch' to true. This ensures + * to generate branch sample for the instruction range before the + * exception is trapped to kernel or before the exception returning. + * + * The exception packet includes the dummy address values, so don't + * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful + * for generating instruction and branch samples. + */ + if (tidq->prev_packet->sample_type == CS_ETM_RANGE) + tidq->prev_packet->last_instr_taken_branch = true; + + return 0; +} + +static int cs_etm__flush(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int err = 0; + struct cs_etm_auxtrace *etm = etmq->etm; + + /* Handle start tracing packet */ + if (tidq->prev_packet->sample_type == CS_ETM_EMPTY) + goto swap_packet; + + if (etmq->etm->synth_opts.last_branch && + etmq->etm->synth_opts.instructions && + tidq->prev_packet->sample_type == CS_ETM_RANGE) { + u64 addr; + /* + * Generate a last branch event for the branches left in the + * circular buffer at the end of the trace. + * + * Use the address of the end of the last reported execution + * range + */ + addr = cs_etm__last_executed_instr(tidq->prev_packet); + + err = cs_etm__synth_instruction_sample( + etmq, tidq, tidq->prev_packet, addr, + tidq->period_instructions); + if (err) + return err; + + tidq->period_instructions = 0; + + } + + if (etm->synth_opts.branches && + tidq->prev_packet->sample_type == CS_ETM_RANGE) { + err = cs_etm__synth_branch_sample(etmq, tidq); + if (err) + return err; + } + +swap_packet: + cs_etm__packet_swap(etm, tidq); + + /* Reset last branches after flush the trace */ + if (etm->use_thread_stack) + thread_stack__flush(tidq->frontend_thread); + + return err; +} + +static int cs_etm__end_block(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int err; + + /* + * It has no new packet coming and 'etmq->packet' contains the stale + * packet which was set at the previous time with packets swapping; + * so skip to generate branch sample to avoid stale packet. + * + * For this case only flush branch stack and generate a last branch + * event for the branches left in the circular buffer at the end of + * the trace. + */ + if (etmq->etm->synth_opts.last_branch && + etmq->etm->synth_opts.instructions && + tidq->prev_packet->sample_type == CS_ETM_RANGE) { + u64 addr; + + /* + * Use the address of the end of the last reported execution + * range. + */ + addr = cs_etm__last_executed_instr(tidq->prev_packet); + + err = cs_etm__synth_instruction_sample( + etmq, tidq, tidq->prev_packet, addr, + tidq->period_instructions); + if (err) + return err; + + tidq->period_instructions = 0; + } + + return 0; +} + +static int cs_etm__flush_stack_cb(struct thread *thread, + void *data __maybe_unused) +{ + thread_stack__flush(thread); + return 0; +} + +static void cs_etm__flush_machine_stack(struct cs_etm_queue *etmq, pid_t pid) +{ + struct machine *machine; + + machine = machines__find(&etmq->etm->session->machines, pid); + if (machine) + machine__for_each_thread(machine, cs_etm__flush_stack_cb, NULL); +} + +static void cs_etm__flush_all_stack(struct cs_etm_queue *etmq) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + if (!etmq->etm->use_thread_stack) + return; + + switch (pid_fmt) { + case CS_ETM_PIDFMT_CTXTID2: + /* Clear the guest stack if virtualization is supported */ + cs_etm__flush_machine_stack(etmq, DEFAULT_GUEST_KERNEL_ID); + fallthrough; + case CS_ETM_PIDFMT_CTXTID: + cs_etm__flush_machine_stack(etmq, HOST_KERNEL_ID); + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + + } +} + +/* + * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue + * if need be. + * Returns: < 0 if error + * = 0 if no more auxtrace_buffer to read + * > 0 if the current buffer isn't empty yet + */ +static int cs_etm__get_data_block(struct cs_etm_queue *etmq) +{ + int ret; + + /* The current block is not finished */ + if (etmq->buf_len) + return 1; + + ret = cs_etm__get_trace(etmq); + if (ret < 0) + return ret; + + /* No more buffer to read */ + if (!etmq->buf_len) + return 0; + + /* + * We cannot assume consecutive blocks in the data file + * are contiguous, reset the decoder to force re-sync. + */ + ret = cs_etm_decoder__reset(etmq->decoder); + if (ret) + return ret; + + /* + * Since the decoder is reset, this causes a global trace + * discontinuity. Flush all thread stacks. + */ + cs_etm__flush_all_stack(etmq); + + return 1; +} + +static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 end_addr) +{ + /* Initialise to keep compiler happy */ + u16 instr16 = 0; + u32 instr32 = 0; + u64 addr; + + switch (packet->isa) { + case CS_ETM_ISA_T32: + /* + * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247: + * + * b'15 b'8 + * +-----------------+--------+ + * | 1 1 0 1 1 1 1 1 | imm8 | + * +-----------------+--------+ + * + * According to the specification, it only defines SVC for T32 + * with 16 bits instruction and has no definition for 32bits; + * so below only read 2 bytes as instruction size for T32. + */ + addr = end_addr - 2; + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr16), (u8 *)&instr16); + if ((instr16 & 0xFF00) == 0xDF00) + return true; + + break; + case CS_ETM_ISA_A32: + /* + * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247: + * + * b'31 b'28 b'27 b'24 + * +---------+---------+-------------------------+ + * | !1111 | 1 1 1 1 | imm24 | + * +---------+---------+-------------------------+ + */ + addr = end_addr - 4; + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); + if ((instr32 & 0x0F000000) == 0x0F000000 && + (instr32 & 0xF0000000) != 0xF0000000) + return true; + + break; + case CS_ETM_ISA_A64: + /* + * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294: + * + * b'31 b'21 b'4 b'0 + * +-----------------------+---------+-----------+ + * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 | + * +-----------------------+---------+-----------+ + */ + addr = end_addr - 4; + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); + if ((instr32 & 0xFFE0001F) == 0xd4000001) + return true; + + break; + case CS_ETM_ISA_UNKNOWN: + default: + break; + } + + return false; +} + +static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, u64 magic) +{ + struct cs_etm_packet *packet = tidq->packet; + struct cs_etm_packet *prev_packet = tidq->prev_packet; + + if (magic == __perf_cs_etmv3_magic) + if (packet->exception_number == CS_ETMV3_EXC_SVC) + return true; + + /* + * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and + * HVC cases; need to check if it's SVC instruction based on + * packet address. + */ + if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) { + if (packet->exception_number == CS_ETMV4_EXC_CALL && + cs_etm__is_svc_instr(etmq, tidq, prev_packet, + prev_packet->end_addr)) + return true; + } + + return false; +} + +static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq, + u64 magic) +{ + struct cs_etm_packet *packet = tidq->packet; + + if (magic == __perf_cs_etmv3_magic) + if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT || + packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT || + packet->exception_number == CS_ETMV3_EXC_PE_RESET || + packet->exception_number == CS_ETMV3_EXC_IRQ || + packet->exception_number == CS_ETMV3_EXC_FIQ) + return true; + + if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) + if (packet->exception_number == CS_ETMV4_EXC_RESET || + packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT || + packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR || + packet->exception_number == CS_ETMV4_EXC_INST_DEBUG || + packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG || + packet->exception_number == CS_ETMV4_EXC_IRQ || + packet->exception_number == CS_ETMV4_EXC_FIQ) + return true; + + return false; +} + +static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + u64 magic) +{ + struct cs_etm_packet *packet = tidq->packet; + struct cs_etm_packet *prev_packet = tidq->prev_packet; + + if (magic == __perf_cs_etmv3_magic) + if (packet->exception_number == CS_ETMV3_EXC_SMC || + packet->exception_number == CS_ETMV3_EXC_HYP || + packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE || + packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR || + packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT || + packet->exception_number == CS_ETMV3_EXC_DATA_FAULT || + packet->exception_number == CS_ETMV3_EXC_GENERIC) + return true; + + if (magic == __perf_cs_etmv4_magic || magic == __perf_cs_ete_magic) { + if (packet->exception_number == CS_ETMV4_EXC_TRAP || + packet->exception_number == CS_ETMV4_EXC_ALIGNMENT || + packet->exception_number == CS_ETMV4_EXC_INST_FAULT || + packet->exception_number == CS_ETMV4_EXC_DATA_FAULT) + return true; + + /* + * For CS_ETMV4_EXC_CALL, except SVC other instructions + * (SMC, HVC) are taken as sync exceptions. + */ + if (packet->exception_number == CS_ETMV4_EXC_CALL && + !cs_etm__is_svc_instr(etmq, tidq, prev_packet, + prev_packet->end_addr)) + return true; + + /* + * ETMv4 has 5 bits for exception number; if the numbers + * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ] + * they are implementation defined exceptions. + * + * For this case, simply take it as sync exception. + */ + if (packet->exception_number > CS_ETMV4_EXC_FIQ && + packet->exception_number <= CS_ETMV4_EXC_END) + return true; + } + + return false; +} + +static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_packet *packet = tidq->packet; + struct cs_etm_packet *prev_packet = tidq->prev_packet; + u64 magic; + int ret; + + switch (packet->sample_type) { + case CS_ETM_RANGE: + /* + * Immediate branch instruction without neither link nor + * return flag, it's normal branch instruction within + * the function. + */ + if (packet->last_instr_type == OCSD_INSTR_BR && + packet->last_instr_subtype == OCSD_S_INSTR_NONE) { + packet->flags = PERF_IP_FLAG_BRANCH; + + if (packet->last_instr_cond) + packet->flags |= PERF_IP_FLAG_CONDITIONAL; + } + + /* + * Immediate branch instruction with link (e.g. BL), this is + * branch instruction for function call. + */ + if (packet->last_instr_type == OCSD_INSTR_BR && + packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL; + + /* + * Indirect branch instruction with link (e.g. BLR), this is + * branch instruction for function call. + */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL; + + /* + * Indirect branch instruction with subtype of + * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for + * function return for A32/T32. + */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN; + + /* + * Indirect branch instruction without link (e.g. BR), usually + * this is used for function return, especially for functions + * within dynamic link lib. + */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_NONE) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN; + + /* Return instruction for function return. */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_V8_RET) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN; + + /* + * Decoder might insert a discontinuity in the middle of + * instruction packets, fixup prev_packet with flag + * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace. + */ + if (prev_packet->sample_type == CS_ETM_DISCONTINUITY) + prev_packet->flags |= PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_TRACE_BEGIN; + + /* + * If the previous packet is an exception return packet + * and the return address just follows SVC instruction, + * it needs to calibrate the previous packet sample flags + * as PERF_IP_FLAG_SYSCALLRET. + */ + if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_INTERRUPT) && + cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) { + prev_packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_SYSCALLRET; + } + break; + case CS_ETM_DISCONTINUITY: + /* + * The trace is discontinuous, if the previous packet is + * instruction packet, set flag PERF_IP_FLAG_TRACE_END + * for previous packet. + */ + if (prev_packet->sample_type == CS_ETM_RANGE) + prev_packet->flags |= PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_TRACE_END; + break; + case CS_ETM_EXCEPTION: + ret = cs_etm__get_magic(etmq, packet->trace_chan_id, &magic); + if (ret) + return ret; + + /* The exception is for system call. */ + if (cs_etm__is_syscall(etmq, tidq, magic)) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL | + PERF_IP_FLAG_SYSCALLRET; + /* + * The exceptions are triggered by external signals from bus, + * interrupt controller, debug module, PE reset or halt. + */ + else if (cs_etm__is_async_exception(tidq, magic)) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL | + PERF_IP_FLAG_ASYNC | + PERF_IP_FLAG_INTERRUPT; + /* + * Otherwise, exception is caused by trap, instruction & + * data fault, or alignment errors. + */ + else if (cs_etm__is_sync_exception(etmq, tidq, magic)) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL | + PERF_IP_FLAG_INTERRUPT; + + /* + * When the exception packet is inserted, since exception + * packet is not used standalone for generating samples + * and it's affiliation to the previous instruction range + * packet; so set previous range packet flags to tell perf + * it is an exception taken branch. + */ + if (prev_packet->sample_type == CS_ETM_RANGE) + prev_packet->flags = packet->flags; + break; + case CS_ETM_EXCEPTION_RET: + /* + * When the exception return packet is inserted, since + * exception return packet is not used standalone for + * generating samples and it's affiliation to the previous + * instruction range packet; so set previous range packet + * flags to tell perf it is an exception return branch. + * + * The exception return can be for either system call or + * other exception types; unfortunately the packet doesn't + * contain exception type related info so we cannot decide + * the exception type purely based on exception return packet. + * If we record the exception number from exception packet and + * reuse it for exception return packet, this is not reliable + * due the trace can be discontinuity or the interrupt can + * be nested, thus the recorded exception number cannot be + * used for exception return packet for these two cases. + * + * For exception return packet, we only need to distinguish the + * packet is for system call or for other types. Thus the + * decision can be deferred when receive the next packet which + * contains the return address, based on the return address we + * can read out the previous instruction and check if it's a + * system call instruction and then calibrate the sample flag + * as needed. + */ + if (prev_packet->sample_type == CS_ETM_RANGE) + prev_packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_INTERRUPT; + break; + case CS_ETM_CONTEXT: + case CS_ETM_EMPTY: + default: + break; + } + + return 0; +} + +static int cs_etm__decode_data_block(struct cs_etm_queue *etmq) +{ + int ret = 0; + size_t processed = 0; + + /* + * Packets are decoded and added to the decoder's packet queue + * until the decoder packet processing callback has requested that + * processing stops or there is nothing left in the buffer. Normal + * operations that stop processing are a timestamp packet or a full + * decoder buffer queue. + */ + ret = cs_etm_decoder__process_data_block(etmq->decoder, + etmq->offset, + &etmq->buf[etmq->buf_used], + etmq->buf_len, + &processed); + if (ret) + goto out; + + etmq->offset += processed; + etmq->buf_used += processed; + etmq->buf_len -= processed; + +out: + return ret; +} + +static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int ret; + struct cs_etm_packet_queue *packet_queue; + + packet_queue = &tidq->packet_queue; + + /* Process each packet in this chunk */ + while (1) { + ret = cs_etm_decoder__get_packet(packet_queue, + tidq->packet); + if (ret <= 0) + /* + * Stop processing this chunk on + * end of data or error + */ + break; + + /* + * Since packet addresses are swapped in packet + * handling within below switch() statements, + * thus setting sample flags must be called + * prior to switch() statement to use address + * information before packets swapping. + */ + ret = cs_etm__set_sample_flags(etmq, tidq); + if (ret < 0) + break; + + switch (tidq->packet->sample_type) { + case CS_ETM_RANGE: + /* + * If the packet contains an instruction + * range, generate instruction sequence + * events. + */ + cs_etm__sample(etmq, tidq); + break; + case CS_ETM_CONTEXT: + /* + * Update context but don't swap packet. Keep the + * previous one for branch source address info, if + * tracing the kernel the context packet will be emitted + * between two ranges. + */ + ret = cs_etm__context(etmq, tidq); + if (ret) + goto out; + break; + case CS_ETM_EXCEPTION: + case CS_ETM_EXCEPTION_RET: + /* + * If the exception packet is coming, + * make sure the previous instruction + * range packet to be handled properly. + */ + cs_etm__exception(tidq); + break; + case CS_ETM_DISCONTINUITY: + /* + * Discontinuity in trace, flush + * previous branch stack + */ + cs_etm__flush(etmq, tidq); + break; + case CS_ETM_EMPTY: + /* + * Should not receive empty packet, + * report error. + */ + pr_err("CS ETM Trace: empty packet\n"); + return -EINVAL; + default: + break; + } + } + +out: + return ret; +} + +static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) +{ + int idx; + struct int_node *inode; + struct cs_etm_traceid_queue *tidq; + struct intlist *traceid_queues_list = etmq->traceid_queues_list; + + intlist__for_each_entry(inode, traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + + /* Ignore return value */ + cs_etm__process_traceid_queue(etmq, tidq); + } +} + +static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq) +{ + int err = 0; + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID); + if (!tidq) + return -EINVAL; + + /* Go through each buffer in the queue and decode them one by one */ + while (1) { + err = cs_etm__get_data_block(etmq); + if (err <= 0) + return err; + + /* Run trace decoder until buffer consumed or end of trace */ + do { + err = cs_etm__decode_data_block(etmq); + if (err) + return err; + + /* + * Process each packet in this chunk, nothing to do if + * an error occurs other than hoping the next one will + * be better. + */ + err = cs_etm__process_traceid_queue(etmq, tidq); + + } while (etmq->buf_len); + + if (err == 0) + /* Flush any remaining branch stack entries */ + err = cs_etm__end_block(etmq, tidq); + } + + return err; +} + +static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq) +{ + int idx, err = 0; + struct cs_etm_traceid_queue *tidq; + struct int_node *inode; + + /* Go through each buffer in the queue and decode them one by one */ + while (1) { + err = cs_etm__get_data_block(etmq); + if (err <= 0) + return err; + + /* Run trace decoder until buffer consumed or end of trace */ + do { + err = cs_etm__decode_data_block(etmq); + if (err) + return err; + + /* + * cs_etm__run_per_thread_timeless_decoder() runs on a + * single traceID queue because each TID has a separate + * buffer. But here in per-cpu mode we need to iterate + * over each channel instead. + */ + intlist__for_each_entry(inode, + etmq->traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + cs_etm__process_traceid_queue(etmq, tidq); + } + } while (etmq->buf_len); + + intlist__for_each_entry(inode, etmq->traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + /* Flush any remaining branch stack entries */ + err = cs_etm__end_block(etmq, tidq); + if (err) + return err; + } + } + + return err; +} + +static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, + pid_t tid) +{ + unsigned int i; + struct auxtrace_queues *queues = &etm->queues; + + for (i = 0; i < queues->nr_queues; i++) { + struct auxtrace_queue *queue = &etm->queues.queue_array[i]; + struct cs_etm_queue *etmq = queue->priv; + struct cs_etm_traceid_queue *tidq; + + if (!etmq) + continue; + + if (etm->per_thread_decoding) { + tidq = cs_etm__etmq_get_traceid_queue( + etmq, CS_ETM_PER_THREAD_TRACEID); + + if (!tidq) + continue; + + if (tid == -1 || thread__tid(tidq->frontend_thread) == tid) + cs_etm__run_per_thread_timeless_decoder(etmq); + } else + cs_etm__run_per_cpu_timeless_decoder(etmq); + } + + return 0; +} + +static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm) +{ + int ret = 0; + unsigned int cs_queue_nr, queue_nr, i; + u8 trace_chan_id; + u64 cs_timestamp; + struct auxtrace_queue *queue; + struct cs_etm_queue *etmq; + struct cs_etm_traceid_queue *tidq; + + /* + * Pre-populate the heap with one entry from each queue so that we can + * start processing in time order across all queues. + */ + for (i = 0; i < etm->queues.nr_queues; i++) { + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i); + if (ret) + return ret; + } + + while (1) { + if (!etm->heap.heap_cnt) + break; + + /* Take the entry at the top of the min heap */ + cs_queue_nr = etm->heap.heap_array[0].queue_nr; + queue_nr = TO_QUEUE_NR(cs_queue_nr); + trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr); + queue = &etm->queues.queue_array[queue_nr]; + etmq = queue->priv; + + /* + * Remove the top entry from the heap since we are about + * to process it. + */ + auxtrace_heap__pop(&etm->heap); + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) { + /* + * No traceID queue has been allocated for this traceID, + * which means something somewhere went very wrong. No + * other choice than simply exit. + */ + ret = -EINVAL; + goto out; + } + + /* + * Packets associated with this timestamp are already in + * the etmq's traceID queue, so process them. + */ + ret = cs_etm__process_traceid_queue(etmq, tidq); + if (ret < 0) + goto out; + + /* + * Packets for this timestamp have been processed, time to + * move on to the next timestamp, fetching a new auxtrace_buffer + * if need be. + */ +refetch: + ret = cs_etm__get_data_block(etmq); + if (ret < 0) + goto out; + + /* + * No more auxtrace_buffers to process in this etmq, simply + * move on to another entry in the auxtrace_heap. + */ + if (!ret) + continue; + + ret = cs_etm__decode_data_block(etmq); + if (ret) + goto out; + + cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); + + if (!cs_timestamp) { + /* + * Function cs_etm__decode_data_block() returns when + * there is no more traces to decode in the current + * auxtrace_buffer OR when a timestamp has been + * encountered on any of the traceID queues. Since we + * did not get a timestamp, there is no more traces to + * process in this auxtrace_buffer. As such empty and + * flush all traceID queues. + */ + cs_etm__clear_all_traceid_queues(etmq); + + /* Fetch another auxtrace_buffer for this etmq */ + goto refetch; + } + + /* + * Add to the min heap the timestamp for packets that have + * just been decoded. They will be processed and synthesized + * during the next call to cs_etm__process_traceid_queue() for + * this queue/traceID. + */ + cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); + ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); + } + + for (i = 0; i < etm->queues.nr_queues; i++) { + struct int_node *inode; + + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + intlist__for_each_entry(inode, etmq->traceid_queues_list) { + int idx = (int)(intptr_t)inode->priv; + + /* Flush any remaining branch stack entries */ + tidq = etmq->traceid_queues[idx]; + ret = cs_etm__end_block(etmq, tidq); + if (ret) + return ret; + } + } +out: + return ret; +} + +static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm, + union perf_event *event) +{ + struct thread *th; + + if (etm->timeless_decoding) + return 0; + + /* + * Add the tid/pid to the log so that we can get a match when we get a + * contextID from the decoder. Only track for the host: only kernel + * trace is supported for guests which wouldn't need pids so this should + * be fine. + */ + th = machine__findnew_thread(&etm->session->machines.host, + event->itrace_start.pid, + event->itrace_start.tid); + if (!th) + return -ENOMEM; + + thread__put(th); + + return 0; +} + +static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm, + union perf_event *event) +{ + struct thread *th; + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + + /* + * Context switch in per-thread mode are irrelevant since perf + * will start/stop tracing as the process is scheduled. + */ + if (etm->timeless_decoding) + return 0; + + /* + * SWITCH_IN events carry the next process to be switched out while + * SWITCH_OUT events carry the process to be switched in. As such + * we don't care about IN events. + */ + if (!out) + return 0; + + /* + * Add the tid/pid to the log so that we can get a match when we get a + * contextID from the decoder. Only track for the host: only kernel + * trace is supported for guests which wouldn't need pids so this should + * be fine. + */ + th = machine__findnew_thread(&etm->session->machines.host, + event->context_switch.next_prev_pid, + event->context_switch.next_prev_tid); + if (!th) + return -ENOMEM; + + thread__put(th); + + return 0; +} + +static int cs_etm__process_event(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool) +{ + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + if (dump_trace) + return 0; + + if (!tool->ordered_events) { + pr_err("CoreSight ETM Trace requires ordered events\n"); + return -EINVAL; + } + + switch (event->header.type) { + case PERF_RECORD_EXIT: + /* + * Don't need to wait for cs_etm__flush_events() in per-thread mode to + * start the decode because we know there will be no more trace from + * this thread. All this does is emit samples earlier than waiting for + * the flush in other modes, but with timestamps it makes sense to wait + * for flush so that events from different threads are interleaved + * properly. + */ + if (etm->per_thread_decoding && etm->timeless_decoding) + return cs_etm__process_timeless_queues(etm, + event->fork.tid); + break; + + case PERF_RECORD_ITRACE_START: + return cs_etm__process_itrace_start(etm, event); + + case PERF_RECORD_SWITCH_CPU_WIDE: + return cs_etm__process_switch_cpu_wide(etm, event); + + case PERF_RECORD_AUX: + /* + * Record the latest kernel timestamp available in the header + * for samples so that synthesised samples occur from this point + * onwards. + */ + if (sample->time && (sample->time != (u64)-1)) + etm->latest_kernel_timestamp = sample->time; + break; + + default: + break; + } + + return 0; +} + +static void dump_queued_data(struct cs_etm_auxtrace *etm, + struct perf_record_auxtrace *event) +{ + struct auxtrace_buffer *buf; + unsigned int i; + /* + * Find all buffers with same reference in the queues and dump them. + * This is because the queues can contain multiple entries of the same + * buffer that were split on aux records. + */ + for (i = 0; i < etm->queues.nr_queues; ++i) + list_for_each_entry(buf, &etm->queues.queue_array[i].head, list) + if (buf->reference == event->reference) + cs_etm__dump_event(etm->queues.queue_array[i].priv, buf); +} + +static int cs_etm__process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + if (!etm->data_queued) { + struct auxtrace_buffer *buffer; + off_t data_offset; + int fd = perf_data__fd(session->data); + bool is_pipe = perf_data__is_pipe(session->data); + int err; + int idx = event->auxtrace.idx; + + if (is_pipe) + data_offset = 0; + else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } + + err = auxtrace_queues__add_event(&etm->queues, session, + event, data_offset, &buffer); + if (err) + return err; + + if (dump_trace) + if (auxtrace_buffer__get_data(buffer, fd)) { + cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer); + auxtrace_buffer__put_data(buffer); + } + } else if (dump_trace) + dump_queued_data(etm, &event->auxtrace); + + return 0; +} + +static void cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm) +{ + /* Take first ETM as all options will be the same for all ETMs */ + u64 *metadata = etm->metadata[0]; + + /* Override timeless mode with user input from --itrace=Z */ + if (etm->synth_opts.timeless_decoding) { + etm->timeless_decoding = true; + return; + } + + if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) + etm->timeless_decoding = !(metadata[CS_ETM_ETMCR] & ETMCR_TIMESTAMP_EN); + else + etm->timeless_decoding = !(metadata[CS_ETMV4_TRCCONFIGR] & TRCCONFIGR_TS); +} + +/* + * Read a single cpu parameter block from the auxtrace_info priv block. + * + * For version 1 there is a per cpu nr_params entry. If we are handling + * version 1 file, then there may be less, the same, or more params + * indicated by this value than the compile time number we understand. + * + * For a version 0 info block, there are a fixed number, and we need to + * fill out the nr_param value in the metadata we create. + */ +static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset, + int out_blk_size, int nr_params_v0) +{ + u64 *metadata = NULL; + int hdr_version; + int nr_in_params, nr_out_params, nr_cmn_params; + int i, k; + + metadata = zalloc(sizeof(*metadata) * out_blk_size); + if (!metadata) + return NULL; + + /* read block current index & version */ + i = *buff_in_offset; + hdr_version = buff_in[CS_HEADER_VERSION]; + + if (!hdr_version) { + /* read version 0 info block into a version 1 metadata block */ + nr_in_params = nr_params_v0; + metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC]; + metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU]; + metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params; + /* remaining block params at offset +1 from source */ + for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++) + metadata[k + 1] = buff_in[i + k]; + /* version 0 has 2 common params */ + nr_cmn_params = 2; + } else { + /* read version 1 info block - input and output nr_params may differ */ + /* version 1 has 3 common params */ + nr_cmn_params = 3; + nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS]; + + /* if input has more params than output - skip excess */ + nr_out_params = nr_in_params + nr_cmn_params; + if (nr_out_params > out_blk_size) + nr_out_params = out_blk_size; + + for (k = CS_ETM_MAGIC; k < nr_out_params; k++) + metadata[k] = buff_in[i + k]; + + /* record the actual nr params we copied */ + metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params; + } + + /* adjust in offset by number of in params used */ + i += nr_in_params + nr_cmn_params; + *buff_in_offset = i; + return metadata; +} + +/** + * Puts a fragment of an auxtrace buffer into the auxtrace queues based + * on the bounds of aux_event, if it matches with the buffer that's at + * file_offset. + * + * Normally, whole auxtrace buffers would be added to the queue. But we + * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder + * is reset across each buffer, so splitting the buffers up in advance has + * the same effect. + */ +static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz, + struct perf_record_aux *aux_event, struct perf_sample *sample) +{ + int err; + char buf[PERF_SAMPLE_MAX_SIZE]; + union perf_event *auxtrace_event_union; + struct perf_record_auxtrace *auxtrace_event; + union perf_event auxtrace_fragment; + __u64 aux_offset, aux_size; + enum cs_etm_format format; + + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + /* + * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got + * from looping through the auxtrace index. + */ + err = perf_session__peek_event(session, file_offset, buf, + PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL); + if (err) + return err; + auxtrace_event = &auxtrace_event_union->auxtrace; + if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE) + return -EINVAL; + + if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) || + auxtrace_event->header.size != sz) { + return -EINVAL; + } + + /* + * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See + * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a + * CPU as we set this always for the AUX_OUTPUT_HW_ID event. + * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1. + * Return 'not found' if mismatch. + */ + if (auxtrace_event->cpu == (__u32) -1) { + etm->per_thread_decoding = true; + if (auxtrace_event->tid != sample->tid) + return 1; + } else if (auxtrace_event->cpu != sample->cpu) { + if (etm->per_thread_decoding) { + /* + * Found a per-cpu buffer after a per-thread one was + * already found + */ + pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n"); + return -EINVAL; + } + return 1; + } + + if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) { + /* + * Clamp size in snapshot mode. The buffer size is clamped in + * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect + * the buffer size. + */ + aux_size = min(aux_event->aux_size, auxtrace_event->size); + + /* + * In this mode, the head also points to the end of the buffer so aux_offset + * needs to have the size subtracted so it points to the beginning as in normal mode + */ + aux_offset = aux_event->aux_offset - aux_size; + } else { + aux_size = aux_event->aux_size; + aux_offset = aux_event->aux_offset; + } + + if (aux_offset >= auxtrace_event->offset && + aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); + + if (!etmq) + return -EINVAL; + + /* + * If this AUX event was inside this buffer somewhere, create a new auxtrace event + * based on the sizes of the aux event, and queue that fragment. + */ + auxtrace_fragment.auxtrace = *auxtrace_event; + auxtrace_fragment.auxtrace.size = aux_size; + auxtrace_fragment.auxtrace.offset = aux_offset; + auxtrace_fragment.auxtrace.idx = etmq->queue_nr; + file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; + + pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu); + err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment, + file_offset, NULL); + if (err) + return err; + + format = (aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW) ? + UNFORMATTED : FORMATTED; + if (etmq->format != UNSET && format != etmq->format) { + pr_err("CS_ETM: mixed formatted and unformatted trace not supported\n"); + return -EINVAL; + } + etmq->format = format; + return 0; + } + + /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ + return 1; +} + +static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event, + u64 offset __maybe_unused, void *data __maybe_unused) +{ + /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */ + if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) { + (*(int *)data)++; /* increment found count */ + return cs_etm__process_aux_output_hw_id(session, event); + } + return 0; +} + +static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event, + u64 offset __maybe_unused, void *data __maybe_unused) +{ + struct perf_sample sample; + int ret; + struct auxtrace_index_entry *ent; + struct auxtrace_index *auxtrace_index; + struct evsel *evsel; + size_t i; + + /* Don't care about any other events, we're only queuing buffers for AUX events */ + if (event->header.type != PERF_RECORD_AUX) + return 0; + + if (event->header.size < sizeof(struct perf_record_aux)) + return -EINVAL; + + /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */ + if (!event->aux.aux_size) + return 0; + + /* + * Parse the sample, we need the sample_id_all data that comes after the event so that the + * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID. + */ + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) + return -EINVAL; + perf_sample__init(&sample, /*all=*/false); + ret = evsel__parse_sample(evsel, event, &sample); + if (ret) + goto out; + + /* + * Loop through the auxtrace index to find the buffer that matches up with this aux event. + */ + list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { + for (i = 0; i < auxtrace_index->nr; i++) { + ent = &auxtrace_index->entries[i]; + ret = cs_etm__queue_aux_fragment(session, ent->file_offset, + ent->sz, &event->aux, &sample); + /* + * Stop search on error or successful values. Continue search on + * 1 ('not found') + */ + if (ret != 1) + goto out; + } + } + + /* + * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but + * don't exit with an error because it will still be possible to decode other aux records. + */ + pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu); + ret = 0; +out: + perf_sample__exit(&sample); + return ret; +} + +static int cs_etm__queue_aux_records(struct perf_session *session) +{ + struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, + struct auxtrace_index, list); + if (index && index->nr > 0) + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + cs_etm__queue_aux_records_cb, NULL); + + /* + * We would get here if there are no entries in the index (either no auxtrace + * buffers or no index at all). Fail silently as there is the possibility of + * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still + * false. + * + * In that scenario, buffers will not be split by AUX records. + */ + return 0; +} + +#define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \ + (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1)) + +/* + * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual + * timestamps). + */ +static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu) +{ + int j; + + for (j = 0; j < num_cpu; j++) { + switch (metadata[j][CS_ETM_MAGIC]) { + case __perf_cs_etmv4_magic: + if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1) + return false; + break; + case __perf_cs_ete_magic: + if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1) + return false; + break; + default: + /* Unknown / unsupported magic number. */ + return false; + } + } + return true; +} + +/* map trace ids to correct metadata block, from information in metadata */ +static int cs_etm__map_trace_ids_metadata(struct cs_etm_auxtrace *etm, int num_cpu, + u64 **metadata) +{ + u64 cs_etm_magic; + u8 trace_chan_id; + int i, err; + + for (i = 0; i < num_cpu; i++) { + cs_etm_magic = metadata[i][CS_ETM_MAGIC]; + switch (cs_etm_magic) { + case __perf_cs_etmv3_magic: + metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK; + trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]); + break; + case __perf_cs_etmv4_magic: + case __perf_cs_ete_magic: + metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK; + trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]); + break; + default: + /* unknown magic number */ + return -EINVAL; + } + err = cs_etm__map_trace_id_v0(etm, trace_chan_id, metadata[i]); + if (err) + return err; + } + return 0; +} + +/* + * Use the data gathered by the peeks for HW_ID (trace ID mappings) and AUX + * (formatted or not) packets to create the decoders. + */ +static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) +{ + struct cs_etm_decoder_params d_params; + struct cs_etm_trace_params *t_params; + int decoders = intlist__nr_entries(etmq->traceid_list); + + if (decoders == 0) + return 0; + + /* + * Each queue can only contain data from one CPU when unformatted, so only one decoder is + * needed. + */ + if (etmq->format == UNFORMATTED) + assert(decoders == 1); + + /* Use metadata to fill in trace parameters for trace decoder */ + t_params = zalloc(sizeof(*t_params) * decoders); + + if (!t_params) + goto out_free; + + if (cs_etm__init_trace_params(t_params, etmq)) + goto out_free; + + /* Set decoder parameters to decode trace packets */ + if (cs_etm__init_decoder_params(&d_params, etmq, + dump_trace ? CS_ETM_OPERATION_PRINT : + CS_ETM_OPERATION_DECODE)) + goto out_free; + + etmq->decoder = cs_etm_decoder__new(decoders, &d_params, + t_params); + + if (!etmq->decoder) + goto out_free; + + /* + * Register a function to handle all memory accesses required by + * the trace decoder library. + */ + if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, + 0x0L, ((u64) -1L), + cs_etm__decoder_mem_access)) + goto out_free_decoder; + + zfree(&t_params); + return 0; + +out_free_decoder: + cs_etm_decoder__free(etmq->decoder); +out_free: + zfree(&t_params); + return -EINVAL; +} + +static int cs_etm__create_decoders(struct cs_etm_auxtrace *etm) +{ + struct auxtrace_queues *queues = &etm->queues; + + for (unsigned int i = 0; i < queues->nr_queues; i++) { + bool empty = list_empty(&queues->queue_array[i].head); + struct cs_etm_queue *etmq = queues->queue_array[i].priv; + int ret; + + /* + * Don't create decoders for empty queues, mainly because + * etmq->format is unknown for empty queues. + */ + assert(empty || etmq->format != UNSET); + if (empty) + continue; + + ret = cs_etm__create_queue_decoders(etmq); + if (ret) + return ret; + } + return 0; +} + +int cs_etm__process_auxtrace_info_full(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + struct cs_etm_auxtrace *etm = NULL; + struct perf_record_time_conv *tc = &session->time_conv; + int event_header_size = sizeof(struct perf_event_header); + int total_size = auxtrace_info->header.size; + int priv_size = 0; + int num_cpu, max_cpu = 0; + int err = 0; + int aux_hw_id_found; + int i; + u64 *ptr = NULL; + u64 **metadata = NULL; + + /* First the global part */ + ptr = (u64 *) auxtrace_info->priv; + num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + + /* + * Bound num_cpu by the event size: the global header consumes + * CS_ETM_HEADER_SIZE bytes, and each CPU needs at least one u64 + * metadata entry after that. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE - + CS_ETM_HEADER_SIZE; + if (num_cpu <= 0 || priv_size <= 0 || + num_cpu > priv_size / (int)sizeof(u64)) + return -EINVAL; + + metadata = zalloc(sizeof(*metadata) * num_cpu); + if (!metadata) + return -ENOMEM; + + /* Start parsing after the common part of the header */ + i = CS_HEADER_VERSION_MAX; + + /* + * The metadata is stored in the auxtrace_info section and encodes + * the configuration of the ARM embedded trace macrocell which is + * required by the trace decoder to properly decode the trace due + * to its highly compressed nature. + */ + for (int j = 0; j < num_cpu; j++) { + if (ptr[i] == __perf_cs_etmv3_magic) { + metadata[j] = + cs_etm__create_meta_blk(ptr, &i, + CS_ETM_PRIV_MAX, + CS_ETM_NR_TRC_PARAMS_V0); + } else if (ptr[i] == __perf_cs_etmv4_magic) { + metadata[j] = + cs_etm__create_meta_blk(ptr, &i, + CS_ETMV4_PRIV_MAX, + CS_ETMV4_NR_TRC_PARAMS_V0); + } else if (ptr[i] == __perf_cs_ete_magic) { + metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1); + } else { + ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n", + ptr[i]); + err = -EINVAL; + goto err_free_metadata; + } + + if (!metadata[j]) { + err = -ENOMEM; + goto err_free_metadata; + } + + /* CPU id comes from perf.data and must fit max_cpu + 1 without overflow */ + if (metadata[j][CS_ETM_CPU] >= INT_MAX) { + err = -EINVAL; + goto err_free_metadata; + } + + if ((int)metadata[j][CS_ETM_CPU] > max_cpu) + max_cpu = metadata[j][CS_ETM_CPU]; + } + + /* + * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and + * CS_ETMV4_PRIV_MAX mark how many double words are in the + * global metadata, and each cpu's metadata respectively. + * The following tests if the correct number of double words was + * present in the auxtrace info section. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE; + if (i * 8 != priv_size) { + err = -EINVAL; + goto err_free_metadata; + } + + etm = zalloc(sizeof(*etm)); + + if (!etm) { + err = -ENOMEM; + goto err_free_metadata; + } + + /* + * As all the ETMs run at the same exception level, the system should + * have the same PID format crossing CPUs. So cache the PID format + * and reuse it for sequential decoding. + */ + etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]); + + err = auxtrace_queues__init_nr(&etm->queues, max_cpu + 1); + if (err) + goto err_free_etm; + + for (unsigned int j = 0; j < etm->queues.nr_queues; ++j) { + err = cs_etm__setup_queue(etm, &etm->queues.queue_array[j], j); + if (err) + goto err_free_queues; + } + + if (session->itrace_synth_opts->set) { + etm->synth_opts = *session->itrace_synth_opts; + } else { + itrace_synth_opts__set_default(&etm->synth_opts, + session->itrace_synth_opts->default_no_sample); + etm->synth_opts.callchain = false; + etm->synth_opts.thread_stack = session->itrace_synth_opts->thread_stack; + } + + if (etm->synth_opts.calls) + etm->branches_filter |= PERF_IP_FLAG_CALL | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + + if (etm->synth_opts.returns) + etm->branches_filter |= PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_TRACE_BEGIN | + PERF_IP_FLAG_TRACE_END; + + if (etm->synth_opts.callchain && !symbol_conf.use_callchain) { + symbol_conf.use_callchain = true; + if (callchain_register_param(&callchain_param) < 0) { + symbol_conf.use_callchain = false; + etm->synth_opts.callchain = false; + } + } + + etm->session = session; + + etm->num_cpu = num_cpu; + etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff); + etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0); + etm->metadata = metadata; + etm->auxtrace_type = auxtrace_info->type; + + if (etm->synth_opts.use_timestamp) + /* + * Prior to Armv8.4, Arm CPUs don't support FEAT_TRF feature, + * therefore the decoder cannot know if the timestamp trace is + * same with the kernel time. + * + * If a user has knowledge for the working platform and can + * specify itrace option 'T' to tell decoder to forcely use the + * traced timestamp as the kernel time. + */ + etm->has_virtual_ts = true; + else + /* Use virtual timestamps if all ETMs report ts_source = 1 */ + etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu); + + if (!etm->has_virtual_ts) + ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n" + "The time field of the samples will not be set accurately.\n" + "For Arm CPUs prior to Armv8.4 or without support FEAT_TRF,\n" + "you can specify the itrace option 'T' for timestamp decoding\n" + "if the Coresight timestamp on the platform is same with the kernel time.\n\n"); + + etm->auxtrace.process_event = cs_etm__process_event; + etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event; + etm->auxtrace.flush_events = cs_etm__flush_events; + etm->auxtrace.free_events = cs_etm__free_events; + etm->auxtrace.free = cs_etm__free; + etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace; + session->auxtrace = &etm->auxtrace; + + cs_etm__setup_timeless_decoding(etm); + + etm->tc.time_shift = tc->time_shift; + etm->tc.time_mult = tc->time_mult; + etm->tc.time_zero = tc->time_zero; + if (event_contains(*tc, cap_user_time_short)) { + etm->tc.time_cycles = tc->time_cycles; + etm->tc.time_mask = tc->time_mask; + etm->tc.cap_user_time_zero = tc->cap_user_time_zero; + etm->tc.cap_user_time_short = tc->cap_user_time_short; + } + + etm->use_thread_stack = etm->synth_opts.thread_stack || + etm->synth_opts.last_branch || + etm->synth_opts.callchain; + + etm->use_callchain = etm->synth_opts.thread_stack || + etm->synth_opts.callchain; + + err = cs_etm__synth_events(etm, session); + if (err) + goto err_free_queues; + + err = cs_etm__queue_aux_records(session); + if (err) + goto err_free_queues; + + /* + * Map Trace ID values to CPU metadata. + * + * Trace metadata will always contain Trace ID values from the legacy algorithm + * in case it's read by a version of Perf that doesn't know about HW_ID packets + * or the kernel doesn't emit them. + * + * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use + * the same IDs as the old algorithm as far as is possible, unless there are clashes + * in which case a different value will be used. This means an older perf may still + * be able to record and read files generate on a newer system. + * + * For a perf able to interpret AUX_HW_ID packets we first check for the presence of + * those packets. If they are there then the values will be mapped and plugged into + * the metadata and decoders are only created for each mapping received. + * + * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel + * then we map Trace ID values to CPU directly from the metadata and create decoders + * for all mappings. + */ + + /* Scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */ + aux_hw_id_found = 0; + err = perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + cs_etm__process_aux_hw_id_cb, &aux_hw_id_found); + if (err) + goto err_free_queues; + + /* if no HW ID found this is a file with metadata values only, map from metadata */ + if (!aux_hw_id_found) { + err = cs_etm__map_trace_ids_metadata(etm, num_cpu, metadata); + if (err) + goto err_free_queues; + } + + err = cs_etm__create_decoders(etm); + if (err) + goto err_free_queues; + + etm->data_queued = etm->queues.populated; + return 0; + +err_free_queues: + auxtrace_queues__free(&etm->queues); + session->auxtrace = NULL; +err_free_etm: + zfree(&etm); +err_free_metadata: + /* No need to check @metadata[j], free(NULL) is supported */ + for (int j = 0; j < num_cpu; j++) + zfree(&metadata[j]); + zfree(&metadata); + return err; +} diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h new file mode 100644 index 000000000..b81099c2b --- /dev/null +++ b/tools/perf/util/cs-etm.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(C) 2015 Linaro Limited. All rights reserved. + * Author: Mathieu Poirier + */ + +#ifndef INCLUDE__UTIL_PERF_CS_ETM_H__ +#define INCLUDE__UTIL_PERF_CS_ETM_H__ + +#include "debug.h" +#include "util/event.h" +#include + +struct perf_session; +struct perf_pmu; + +/* + * Versioning header in case things need to change in the future. That way + * decoding of old snapshot is still possible. + */ +enum { + /* Starting with 0x0 */ + CS_HEADER_VERSION, + /* PMU->type (32 bit), total # of CPUs (32 bit) */ + CS_PMU_TYPE_CPUS, + CS_ETM_SNAPSHOT, + CS_HEADER_VERSION_MAX, +}; + +/* + * Update the version for new format. + * + * Version 1: format adds a param count to the per cpu metadata. + * This allows easy adding of new metadata parameters. + * Requires that new params always added after current ones. + * Also allows client reader to handle file versions that are different by + * checking the number of params in the file vs the number expected. + * + * Version 2: Drivers will use PERF_RECORD_AUX_OUTPUT_HW_ID to output + * CoreSight Trace ID. ...TRACEIDR metadata will be set to legacy values + * but with addition flags. + */ +#define CS_HEADER_CURRENT_VERSION 2 + +/* Beginning of header common to both ETMv3 and V4 */ +enum { + CS_ETM_MAGIC, + CS_ETM_CPU, + /* Number of trace config params in following ETM specific block */ + CS_ETM_NR_TRC_PARAMS, + CS_ETM_COMMON_BLK_MAX_V1, +}; + +/* ETMv3/PTM metadata */ +enum { + /* Dynamic, configurable parameters */ + CS_ETM_ETMCR = CS_ETM_COMMON_BLK_MAX_V1, + CS_ETM_ETMTRACEIDR, + /* RO, taken from sysFS */ + CS_ETM_ETMCCER, + CS_ETM_ETMIDR, + CS_ETM_PRIV_MAX, +}; + +/* define fixed version 0 length - allow new format reader to read old files. */ +#define CS_ETM_NR_TRC_PARAMS_V0 (CS_ETM_ETMIDR - CS_ETM_ETMCR + 1) + +/* ETMv4 metadata */ +enum { + /* Dynamic, configurable parameters */ + CS_ETMV4_TRCCONFIGR = CS_ETM_COMMON_BLK_MAX_V1, + CS_ETMV4_TRCTRACEIDR, + /* RO, taken from sysFS */ + CS_ETMV4_TRCIDR0, + CS_ETMV4_TRCIDR1, + CS_ETMV4_TRCIDR2, + CS_ETMV4_TRCIDR8, + CS_ETMV4_TRCAUTHSTATUS, + CS_ETMV4_TS_SOURCE, + CS_ETMV4_PRIV_MAX, +}; + +/* define fixed version 0 length - allow new format reader to read old files. */ +#define CS_ETMV4_NR_TRC_PARAMS_V0 (CS_ETMV4_TRCAUTHSTATUS - CS_ETMV4_TRCCONFIGR + 1) + +/* + * ETE metadata is ETMv4 plus TRCDEVARCH register and doesn't support header V0 since it was + * added in header V1 + */ +enum { + /* Dynamic, configurable parameters */ + CS_ETE_TRCCONFIGR = CS_ETM_COMMON_BLK_MAX_V1, + CS_ETE_TRCTRACEIDR, + /* RO, taken from sysFS */ + CS_ETE_TRCIDR0, + CS_ETE_TRCIDR1, + CS_ETE_TRCIDR2, + CS_ETE_TRCIDR8, + CS_ETE_TRCAUTHSTATUS, + CS_ETE_TRCDEVARCH, + CS_ETE_TS_SOURCE, + CS_ETE_PRIV_MAX +}; + +/* + * Check for valid CoreSight trace ID. If an invalid value is present in the metadata, + * then IDs are present in the hardware ID packet in the data file. + */ +#define CS_IS_VALID_TRACE_ID(id) ((id > 0) && (id < 0x70)) + +/* + * ETMv3 exception encoding number: + * See Embedded Trace Macrocell specification (ARM IHI 0014Q) + * table 7-12 Encoding of Exception[3:0] for non-ARMv7-M processors. + */ +enum { + CS_ETMV3_EXC_NONE = 0, + CS_ETMV3_EXC_DEBUG_HALT = 1, + CS_ETMV3_EXC_SMC = 2, + CS_ETMV3_EXC_HYP = 3, + CS_ETMV3_EXC_ASYNC_DATA_ABORT = 4, + CS_ETMV3_EXC_JAZELLE_THUMBEE = 5, + CS_ETMV3_EXC_PE_RESET = 8, + CS_ETMV3_EXC_UNDEFINED_INSTR = 9, + CS_ETMV3_EXC_SVC = 10, + CS_ETMV3_EXC_PREFETCH_ABORT = 11, + CS_ETMV3_EXC_DATA_FAULT = 12, + CS_ETMV3_EXC_GENERIC = 13, + CS_ETMV3_EXC_IRQ = 14, + CS_ETMV3_EXC_FIQ = 15, +}; + +/* + * ETMv4 exception encoding number: + * See ARM Embedded Trace Macrocell Architecture Specification (ARM IHI 0064D) + * table 6-12 Possible values for the TYPE field in an Exception instruction + * trace packet, for ARMv7-A/R and ARMv8-A/R PEs. + */ +enum { + CS_ETMV4_EXC_RESET = 0, + CS_ETMV4_EXC_DEBUG_HALT = 1, + CS_ETMV4_EXC_CALL = 2, + CS_ETMV4_EXC_TRAP = 3, + CS_ETMV4_EXC_SYSTEM_ERROR = 4, + CS_ETMV4_EXC_INST_DEBUG = 6, + CS_ETMV4_EXC_DATA_DEBUG = 7, + CS_ETMV4_EXC_ALIGNMENT = 10, + CS_ETMV4_EXC_INST_FAULT = 11, + CS_ETMV4_EXC_DATA_FAULT = 12, + CS_ETMV4_EXC_IRQ = 14, + CS_ETMV4_EXC_FIQ = 15, + CS_ETMV4_EXC_END = 31, +}; + +enum cs_etm_sample_type { + CS_ETM_EMPTY, + CS_ETM_RANGE, + CS_ETM_DISCONTINUITY, + CS_ETM_EXCEPTION, + CS_ETM_EXCEPTION_RET, + CS_ETM_CONTEXT, +}; + +enum cs_etm_isa { + CS_ETM_ISA_UNKNOWN, + CS_ETM_ISA_A64, + CS_ETM_ISA_A32, + CS_ETM_ISA_T32, +}; + +struct cs_etm_queue; + +struct cs_etm_packet { + enum cs_etm_sample_type sample_type; + enum cs_etm_isa isa; + u64 start_addr; + u64 end_addr; + u32 instr_count; + u32 last_instr_type; + u32 last_instr_subtype; + u32 flags; + u32 exception_number; + bool last_instr_cond; + bool last_instr_taken_branch; + u8 last_instr_size; + u8 trace_chan_id; + int cpu; + int el; + pid_t tid; +}; + +#define CS_ETM_PACKET_MAX_BUFFER 1024 + +/* + * When working with per-thread scenarios the process under trace can + * be scheduled on any CPU and as such, more than one traceID may be + * associated with the same process. Since a traceID of '0' is illegal + * as per the CoreSight architecture, use that specific value to + * identify the queue where all packets (with any traceID) are + * aggregated. + */ +#define CS_ETM_PER_THREAD_TRACEID 0 + +struct cs_etm_packet_queue { + u32 packet_count; + u32 head; + u32 tail; + u32 instr_count; + u64 cs_timestamp; /* Timestamp from trace data, converted to ns if possible */ + u64 next_cs_timestamp; + struct cs_etm_packet packet_buffer[CS_ETM_PACKET_MAX_BUFFER]; +}; + +#define KiB(x) ((x) * 1024) +#define MiB(x) ((x) * 1024 * 1024) + +#define CS_ETM_INVAL_ADDR 0xdeadbeefdeadbeefUL + +#define BMVAL(val, lsb, msb) ((val & GENMASK(msb, lsb)) >> lsb) + +#define CS_ETM_HEADER_SIZE (CS_HEADER_VERSION_MAX * sizeof(u64)) + +#define __perf_cs_etmv3_magic 0x3030303030303030ULL +#define __perf_cs_etmv4_magic 0x4040404040404040ULL +#define __perf_cs_ete_magic 0x5050505050505050ULL +#define CS_ETMV3_PRIV_SIZE (CS_ETM_PRIV_MAX * sizeof(u64)) +#define CS_ETMV4_PRIV_SIZE (CS_ETMV4_PRIV_MAX * sizeof(u64)) +#define CS_ETE_PRIV_SIZE (CS_ETE_PRIV_MAX * sizeof(u64)) + +#define INFO_HEADER_SIZE (sizeof(((struct perf_record_auxtrace_info *)0)->type) + \ + sizeof(((struct perf_record_auxtrace_info *)0)->reserved__)) + +/* CoreSight trace ID is currently the bottom 7 bits of the value */ +#define CORESIGHT_TRACE_ID_VAL_MASK GENMASK(6, 0) + +/* ETMv4 CONFIGR register bits */ +#define TRCCONFIGR_BB BIT(3) +#define TRCCONFIGR_CCI BIT(4) +#define TRCCONFIGR_CID BIT(6) +#define TRCCONFIGR_VMID BIT(7) +#define TRCCONFIGR_TS BIT(11) +#define TRCCONFIGR_RS BIT(12) +#define TRCCONFIGR_VMIDOPT BIT(15) + +/* ETMv3 ETMCR register bits */ +#define ETMCR_CYC_ACC BIT(12) +#define ETMCR_CTXTID BIT(14) +#define ETMCR_TIMESTAMP_EN BIT(28) +#define ETMCR_RETURN_STACK BIT(29) + +int cs_etm__process_auxtrace_info(union perf_event *event, + struct perf_session *session); +void cs_etm_get_default_config(const struct perf_pmu *pmu, struct perf_event_attr *attr); + +enum cs_etm_pid_fmt { + CS_ETM_PIDFMT_NONE, + CS_ETM_PIDFMT_CTXTID, + CS_ETM_PIDFMT_CTXTID2 +}; + +#ifdef HAVE_CSTRACE_SUPPORT +#include +int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu); +enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq); +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, ocsd_ex_level el, + pid_t tid); +bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq); +void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, + u8 trace_chan_id); +struct cs_etm_packet_queue +*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id); +int cs_etm__process_auxtrace_info_full(union perf_event *event __maybe_unused, + struct perf_session *session __maybe_unused); +u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp); +#else +static inline int +cs_etm__process_auxtrace_info_full(union perf_event *event __maybe_unused, + struct perf_session *session __maybe_unused) +{ + pr_err("\nCS ETM Trace: OpenCSD is not linked in, please recompile with CORESIGHT=1\n"); + return -1; +} +#endif + +#endif diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c new file mode 100644 index 000000000..e22237139 --- /dev/null +++ b/tools/perf/util/data-convert-bt.c @@ -0,0 +1,1832 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CTF writing support via babeltrace. + * + * Copyright (C) 2014, Jiri Olsa + * Copyright (C) 2014, Sebastian Andrzej Siewior + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "asm/bug.h" +#include "data-convert.h" +#include "session.h" +#include "debug.h" +#include "tool.h" +#include "evlist.h" +#include "evsel.h" +#include "machine.h" +#include "config.h" +#include +#include +#include +#include "util.h" +#include "clockid.h" +#include "util/sample.h" +#include "util/time-utils.h" +#include "header.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +#define pr_N(n, fmt, ...) \ + eprintf(n, debug_data_convert, fmt, ##__VA_ARGS__) + +#define pr(fmt, ...) pr_N(1, pr_fmt(fmt), ##__VA_ARGS__) +#define pr2(fmt, ...) pr_N(2, pr_fmt(fmt), ##__VA_ARGS__) + +#define pr_time2(t, fmt, ...) pr_time_N(2, debug_data_convert, t, pr_fmt(fmt), ##__VA_ARGS__) + +struct evsel_priv { + struct bt_ctf_event_class *event_class; +}; + +#define MAX_CPUS 4096 + +struct ctf_stream { + struct bt_ctf_stream *stream; + int cpu; + u32 count; +}; + +struct ctf_writer { + /* writer primitives */ + struct bt_ctf_writer *writer; + struct ctf_stream **stream; + int stream_cnt; + struct bt_ctf_stream_class *stream_class; + struct bt_ctf_clock *clock; + + /* data types */ + union { + struct { + struct bt_ctf_field_type *s64; + struct bt_ctf_field_type *u64; + struct bt_ctf_field_type *s32; + struct bt_ctf_field_type *u32; + struct bt_ctf_field_type *string; + struct bt_ctf_field_type *u32_hex; + struct bt_ctf_field_type *u64_hex; + }; + struct bt_ctf_field_type *array[6]; + } data; + struct bt_ctf_event_class *comm_class; + struct bt_ctf_event_class *exit_class; + struct bt_ctf_event_class *fork_class; + struct bt_ctf_event_class *mmap_class; + struct bt_ctf_event_class *mmap2_class; +}; + +struct convert { + struct perf_tool tool; + struct ctf_writer writer; + + struct perf_time_interval *ptime_range; + int range_size; + int range_num; + + u64 events_size; + u64 events_count; + u64 non_sample_count; + u64 skipped; + + /* Ordered events configured queue size. */ + u64 queue_size; +}; + +static int value_set(struct bt_ctf_field_type *type, + struct bt_ctf_event *event, + const char *name, u64 val) +{ + struct bt_ctf_field *field; + bool sign = bt_ctf_field_type_integer_get_signed(type); + int ret; + + field = bt_ctf_field_create(type); + if (!field) { + pr_err("failed to create a field %s\n", name); + return -1; + } + + if (sign) { + ret = bt_ctf_field_integer_signed_set_value(field, val); + if (ret) { + pr_err("failed to set field value %s\n", name); + goto err; + } + } else { + ret = bt_ctf_field_integer_unsigned_set_value(field, val); + if (ret) { + pr_err("failed to set field value %s\n", name); + goto err; + } + } + + ret = bt_ctf_event_set_payload(event, name, field); + if (ret) { + pr_err("failed to set payload %s\n", name); + goto err; + } + + pr2(" SET [%s = %" PRIu64 "]\n", name, val); + +err: + bt_ctf_field_put(field); + return ret; +} + +#define __FUNC_VALUE_SET(_name, _val_type) \ +static __maybe_unused int value_set_##_name(struct ctf_writer *cw, \ + struct bt_ctf_event *event, \ + const char *name, \ + _val_type val) \ +{ \ + struct bt_ctf_field_type *type = cw->data._name; \ + return value_set(type, event, name, (u64) val); \ +} + +#define FUNC_VALUE_SET(_name) __FUNC_VALUE_SET(_name, _name) + +FUNC_VALUE_SET(s32) +FUNC_VALUE_SET(u32) +FUNC_VALUE_SET(s64) +FUNC_VALUE_SET(u64) +__FUNC_VALUE_SET(u64_hex, u64) + +static int string_set_value(struct bt_ctf_field *field, const char *string); +static __maybe_unused int +value_set_string(struct ctf_writer *cw, struct bt_ctf_event *event, + const char *name, const char *string) +{ + struct bt_ctf_field_type *type = cw->data.string; + struct bt_ctf_field *field; + int ret = 0; + + field = bt_ctf_field_create(type); + if (!field) { + pr_err("failed to create a field %s\n", name); + return -1; + } + + ret = string_set_value(field, string); + if (ret) { + pr_err("failed to set value %s\n", name); + goto err_put_field; + } + + ret = bt_ctf_event_set_payload(event, name, field); + if (ret) + pr_err("failed to set payload %s\n", name); + +err_put_field: + bt_ctf_field_put(field); + return ret; +} + +static struct bt_ctf_field_type* +get_tracepoint_field_type(struct ctf_writer *cw, struct tep_format_field *field) +{ + unsigned long flags = field->flags; + + if (flags & TEP_FIELD_IS_STRING) + return cw->data.string; + + if (!(flags & TEP_FIELD_IS_SIGNED)) { + /* unsigned long are mostly pointers */ + if (flags & TEP_FIELD_IS_LONG || flags & TEP_FIELD_IS_POINTER) + return cw->data.u64_hex; + } + + if (flags & TEP_FIELD_IS_SIGNED) { + if (field->size == 8) + return cw->data.s64; + else + return cw->data.s32; + } + + if (field->size == 8) + return cw->data.u64; + else + return cw->data.u32; +} + +static unsigned long long adjust_signedness(unsigned long long value_int, int size) +{ + unsigned long long value_mask; + + /* + * value_mask = (1 << (size * 8 - 1)) - 1. + * Directly set value_mask for code readers. + */ + switch (size) { + case 1: + value_mask = 0x7fULL; + break; + case 2: + value_mask = 0x7fffULL; + break; + case 4: + value_mask = 0x7fffffffULL; + break; + case 8: + /* + * For 64 bit value, return it self. There is no need + * to fill high bit. + */ + /* Fall through */ + default: + /* BUG! */ + return value_int; + } + + /* If it is a positive value, don't adjust. */ + if ((value_int & (~0ULL - value_mask)) == 0) + return value_int; + + /* Fill upper part of value_int with 1 to make it a negative long long. */ + return (value_int & value_mask) | ~value_mask; +} + +static int string_set_value(struct bt_ctf_field *field, const char *string) +{ + char *buffer = NULL; + size_t len = strlen(string), i, p; + int err; + + for (i = p = 0; i < len; i++, p++) { + if (isprint(string[i])) { + if (!buffer) + continue; + buffer[p] = string[i]; + } else { + char numstr[5]; + + snprintf(numstr, sizeof(numstr), "\\x%02x", + (unsigned int)(string[i]) & 0xff); + + if (!buffer) { + buffer = zalloc(i + (len - i) * 4 + 2); + if (!buffer) { + pr_err("failed to set unprintable string '%s'\n", string); + return bt_ctf_field_string_set_value(field, "UNPRINTABLE-STRING"); + } + if (i > 0) + strncpy(buffer, string, i); + } + memcpy(buffer + p, numstr, 4); + p += 3; + } + } + + if (!buffer) + return bt_ctf_field_string_set_value(field, string); + err = bt_ctf_field_string_set_value(field, buffer); + free(buffer); + return err; +} + +static int add_tracepoint_field_value(struct ctf_writer *cw, + struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct perf_sample *sample, + struct tep_format_field *fmtf) +{ + struct bt_ctf_field_type *type; + struct bt_ctf_field *array_field; + struct bt_ctf_field *field; + const char *name = fmtf->name; + void *data = sample->raw_data; + unsigned long flags = fmtf->flags; + unsigned int n_items; + unsigned int i; + unsigned int offset; + unsigned int len; + int ret; + + name = fmtf->alias; + offset = fmtf->offset; + len = fmtf->size; + if (flags & TEP_FIELD_IS_STRING) + flags &= ~TEP_FIELD_IS_ARRAY; + + if (flags & TEP_FIELD_IS_DYNAMIC) { + unsigned long long tmp_val; + + tmp_val = tep_read_number(fmtf->event->tep, + data + offset, len); + offset = tmp_val; + len = offset >> 16; + offset &= 0xffff; + if (tep_field_is_relative(flags)) + offset += fmtf->offset + fmtf->size; + } + + if (flags & TEP_FIELD_IS_ARRAY) { + + type = bt_ctf_event_class_get_field_by_name( + event_class, name); + array_field = bt_ctf_field_create(type); + bt_ctf_field_type_put(type); + if (!array_field) { + pr_err("Failed to create array type %s\n", name); + return -1; + } + + len = fmtf->size / fmtf->arraylen; + n_items = fmtf->arraylen; + } else { + n_items = 1; + array_field = NULL; + } + + type = get_tracepoint_field_type(cw, fmtf); + + for (i = 0; i < n_items; i++) { + if (flags & TEP_FIELD_IS_ARRAY) + field = bt_ctf_field_array_get_field(array_field, i); + else + field = bt_ctf_field_create(type); + + if (!field) { + pr_err("failed to create a field %s\n", name); + return -1; + } + + if (flags & TEP_FIELD_IS_STRING) + ret = string_set_value(field, data + offset + i * len); + else { + unsigned long long value_int; + + value_int = tep_read_number( + fmtf->event->tep, + data + offset + i * len, len); + + if (!(flags & TEP_FIELD_IS_SIGNED)) + ret = bt_ctf_field_integer_unsigned_set_value( + field, value_int); + else + ret = bt_ctf_field_integer_signed_set_value( + field, adjust_signedness(value_int, len)); + } + + if (ret) { + pr_err("failed to set file value %s\n", name); + goto err_put_field; + } + if (!(flags & TEP_FIELD_IS_ARRAY)) { + ret = bt_ctf_event_set_payload(event, name, field); + if (ret) { + pr_err("failed to set payload %s\n", name); + goto err_put_field; + } + } + bt_ctf_field_put(field); + } + if (flags & TEP_FIELD_IS_ARRAY) { + ret = bt_ctf_event_set_payload(event, name, array_field); + if (ret) { + pr_err("Failed add payload array %s\n", name); + return -1; + } + bt_ctf_field_put(array_field); + } + return 0; + +err_put_field: + bt_ctf_field_put(field); + return -1; +} + +static int add_tracepoint_fields_values(struct ctf_writer *cw, + struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct tep_format_field *fields, + struct perf_sample *sample) +{ + struct tep_format_field *field; + int ret; + + for (field = fields; field; field = field->next) { + ret = add_tracepoint_field_value(cw, event_class, event, sample, + field); + if (ret) + return -1; + } + return 0; +} + +static int add_tracepoint_values(struct ctf_writer *cw, + struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct evsel *evsel, + struct perf_sample *sample) +{ + const struct tep_event *tp_format = evsel__tp_format(evsel); + struct tep_format_field *common_fields = tp_format->format.common_fields; + struct tep_format_field *fields = tp_format->format.fields; + int ret; + + ret = add_tracepoint_fields_values(cw, event_class, event, + common_fields, sample); + if (!ret) + ret = add_tracepoint_fields_values(cw, event_class, event, + fields, sample); + + return ret; +} + +static int +add_bpf_output_values(struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct perf_sample *sample) +{ + struct bt_ctf_field_type *len_type, *seq_type; + struct bt_ctf_field *len_field, *seq_field; + unsigned int raw_size = sample->raw_size; + unsigned int nr_elements = raw_size / sizeof(u32); + unsigned int i; + int ret; + + if (nr_elements * sizeof(u32) != raw_size) + pr_warning("Incorrect raw_size (%u) in bpf output event, skip %zu bytes\n", + raw_size, nr_elements * sizeof(u32) - raw_size); + + len_type = bt_ctf_event_class_get_field_by_name(event_class, "raw_len"); + len_field = bt_ctf_field_create(len_type); + if (!len_field) { + pr_err("failed to create 'raw_len' for bpf output event\n"); + ret = -1; + goto put_len_type; + } + + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); + if (ret) { + pr_err("failed to set field value for raw_len\n"); + goto put_len_field; + } + ret = bt_ctf_event_set_payload(event, "raw_len", len_field); + if (ret) { + pr_err("failed to set payload to raw_len\n"); + goto put_len_field; + } + + seq_type = bt_ctf_event_class_get_field_by_name(event_class, "raw_data"); + seq_field = bt_ctf_field_create(seq_type); + if (!seq_field) { + pr_err("failed to create 'raw_data' for bpf output event\n"); + ret = -1; + goto put_seq_type; + } + + ret = bt_ctf_field_sequence_set_length(seq_field, len_field); + if (ret) { + pr_err("failed to set length of 'raw_data'\n"); + goto put_seq_field; + } + + for (i = 0; i < nr_elements; i++) { + struct bt_ctf_field *elem_field = + bt_ctf_field_sequence_get_field(seq_field, i); + + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, + ((u32 *)(sample->raw_data))[i]); + + bt_ctf_field_put(elem_field); + if (ret) { + pr_err("failed to set raw_data[%d]\n", i); + goto put_seq_field; + } + } + + ret = bt_ctf_event_set_payload(event, "raw_data", seq_field); + if (ret) + pr_err("failed to set payload for raw_data\n"); + +put_seq_field: + bt_ctf_field_put(seq_field); +put_seq_type: + bt_ctf_field_type_put(seq_type); +put_len_field: + bt_ctf_field_put(len_field); +put_len_type: + bt_ctf_field_type_put(len_type); + return ret; +} + +static int +add_callchain_output_values(struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct ip_callchain *callchain) +{ + struct bt_ctf_field_type *len_type, *seq_type; + struct bt_ctf_field *len_field, *seq_field; + unsigned int nr_elements = callchain->nr; + unsigned int i; + int ret; + + len_type = bt_ctf_event_class_get_field_by_name( + event_class, "perf_callchain_size"); + len_field = bt_ctf_field_create(len_type); + if (!len_field) { + pr_err("failed to create 'perf_callchain_size' for callchain output event\n"); + ret = -1; + goto put_len_type; + } + + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); + if (ret) { + pr_err("failed to set field value for perf_callchain_size\n"); + goto put_len_field; + } + ret = bt_ctf_event_set_payload(event, "perf_callchain_size", len_field); + if (ret) { + pr_err("failed to set payload to perf_callchain_size\n"); + goto put_len_field; + } + + seq_type = bt_ctf_event_class_get_field_by_name( + event_class, "perf_callchain"); + seq_field = bt_ctf_field_create(seq_type); + if (!seq_field) { + pr_err("failed to create 'perf_callchain' for callchain output event\n"); + ret = -1; + goto put_seq_type; + } + + ret = bt_ctf_field_sequence_set_length(seq_field, len_field); + if (ret) { + pr_err("failed to set length of 'perf_callchain'\n"); + goto put_seq_field; + } + + for (i = 0; i < nr_elements; i++) { + struct bt_ctf_field *elem_field = + bt_ctf_field_sequence_get_field(seq_field, i); + + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, + ((u64 *)(callchain->ips))[i]); + + bt_ctf_field_put(elem_field); + if (ret) { + pr_err("failed to set callchain[%d]\n", i); + goto put_seq_field; + } + } + + ret = bt_ctf_event_set_payload(event, "perf_callchain", seq_field); + if (ret) + pr_err("failed to set payload for raw_data\n"); + +put_seq_field: + bt_ctf_field_put(seq_field); +put_seq_type: + bt_ctf_field_type_put(seq_type); +put_len_field: + bt_ctf_field_put(len_field); +put_len_type: + bt_ctf_field_type_put(len_type); + return ret; +} + +static int add_generic_values(struct ctf_writer *cw, + struct bt_ctf_event *event, + struct evsel *evsel, + struct perf_sample *sample) +{ + u64 type = evsel->core.attr.sample_type; + int ret; + + /* + * missing: + * PERF_SAMPLE_TIME - not needed as we have it in + * ctf event header + * PERF_SAMPLE_READ - TODO + * PERF_SAMPLE_RAW - tracepoint fields are handled separately + * PERF_SAMPLE_BRANCH_STACK - TODO + * PERF_SAMPLE_REGS_USER - TODO + * PERF_SAMPLE_STACK_USER - TODO + */ + + if (type & PERF_SAMPLE_IP) { + ret = value_set_u64_hex(cw, event, "perf_ip", sample->ip); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_TID) { + ret = value_set_s32(cw, event, "perf_tid", sample->tid); + if (ret) + return -1; + + ret = value_set_s32(cw, event, "perf_pid", sample->pid); + if (ret) + return -1; + } + + if ((type & PERF_SAMPLE_ID) || + (type & PERF_SAMPLE_IDENTIFIER)) { + ret = value_set_u64(cw, event, "perf_id", sample->id); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_STREAM_ID) { + ret = value_set_u64(cw, event, "perf_stream_id", sample->stream_id); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_PERIOD) { + ret = value_set_u64(cw, event, "perf_period", sample->period); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_WEIGHT) { + ret = value_set_u64(cw, event, "perf_weight", sample->weight); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_DATA_SRC) { + ret = value_set_u64(cw, event, "perf_data_src", + sample->data_src); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_TRANSACTION) { + ret = value_set_u64(cw, event, "perf_transaction", + sample->transaction); + if (ret) + return -1; + } + + return 0; +} + +static int ctf_stream__flush(struct ctf_stream *cs) +{ + int err = 0; + + if (cs) { + err = bt_ctf_stream_flush(cs->stream); + if (err) + pr_err("CTF stream %d flush failed\n", cs->cpu); + + pr("Flush stream for cpu %d (%u samples)\n", + cs->cpu, cs->count); + + cs->count = 0; + } + + return err; +} + +static struct ctf_stream *ctf_stream__create(struct ctf_writer *cw, int cpu) +{ + struct ctf_stream *cs; + struct bt_ctf_field *pkt_ctx = NULL; + struct bt_ctf_field *cpu_field = NULL; + struct bt_ctf_stream *stream = NULL; + int ret; + + cs = zalloc(sizeof(*cs)); + if (!cs) { + pr_err("Failed to allocate ctf stream\n"); + return NULL; + } + + stream = bt_ctf_writer_create_stream(cw->writer, cw->stream_class); + if (!stream) { + pr_err("Failed to create CTF stream\n"); + goto out; + } + + pkt_ctx = bt_ctf_stream_get_packet_context(stream); + if (!pkt_ctx) { + pr_err("Failed to obtain packet context\n"); + goto out; + } + + cpu_field = bt_ctf_field_structure_get_field(pkt_ctx, "cpu_id"); + bt_ctf_field_put(pkt_ctx); + if (!cpu_field) { + pr_err("Failed to obtain cpu field\n"); + goto out; + } + + ret = bt_ctf_field_integer_unsigned_set_value(cpu_field, (u32) cpu); + if (ret) { + pr_err("Failed to update CPU number\n"); + goto out; + } + + bt_ctf_field_put(cpu_field); + + cs->cpu = cpu; + cs->stream = stream; + return cs; + +out: + if (cpu_field) + bt_ctf_field_put(cpu_field); + if (stream) + bt_ctf_stream_put(stream); + + free(cs); + return NULL; +} + +static void ctf_stream__delete(struct ctf_stream *cs) +{ + if (cs) { + bt_ctf_stream_put(cs->stream); + free(cs); + } +} + +static struct ctf_stream *ctf_stream(struct ctf_writer *cw, int cpu) +{ + struct ctf_stream *cs = cw->stream[cpu]; + + if (!cs) { + cs = ctf_stream__create(cw, cpu); + cw->stream[cpu] = cs; + } + + return cs; +} + +static int get_sample_cpu(struct ctf_writer *cw, struct perf_sample *sample, + struct evsel *evsel) +{ + int cpu = 0; + + if (evsel->core.attr.sample_type & PERF_SAMPLE_CPU) + cpu = sample->cpu; + + if (cpu > cw->stream_cnt) { + pr_err("Event was recorded for CPU %d, limit is at %d.\n", + cpu, cw->stream_cnt); + cpu = 0; + } + + return cpu; +} + +#define STREAM_FLUSH_COUNT 100000 + +/* + * Currently we have no other way to determine the + * time for the stream flush other than keep track + * of the number of events and check it against + * threshold. + */ +static bool is_flush_needed(struct ctf_stream *cs) +{ + return cs->count >= STREAM_FLUSH_COUNT; +} + +static int process_sample_event(const struct perf_tool *tool, + union perf_event *_event, + struct perf_sample *sample, + struct machine *machine __maybe_unused) +{ + struct convert *c = container_of(tool, struct convert, tool); + struct evsel *evsel = sample->evsel; + struct evsel_priv *priv = evsel->priv; + struct ctf_writer *cw = &c->writer; + struct ctf_stream *cs; + struct bt_ctf_event_class *event_class; + struct bt_ctf_event *event; + int ret; + unsigned long type = evsel->core.attr.sample_type; + + if (WARN_ONCE(!priv, "Failed to setup all events.\n")) + return 0; + + if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { + ++c->skipped; + return 0; + } + + event_class = priv->event_class; + + /* update stats */ + c->events_count++; + c->events_size += _event->header.size; + + pr_time2(sample->time, "sample %" PRIu64 "\n", c->events_count); + + event = bt_ctf_event_create(event_class); + if (!event) { + pr_err("Failed to create an CTF event\n"); + return -1; + } + + bt_ctf_clock_set_time(cw->clock, sample->time); + + ret = add_generic_values(cw, event, evsel, sample); + if (ret) + return -1; + + if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT) { + ret = add_tracepoint_values(cw, event_class, event, + evsel, sample); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_CALLCHAIN) { + ret = add_callchain_output_values(event_class, + event, sample->callchain); + if (ret) + return -1; + } + + if (evsel__is_bpf_output(evsel)) { + ret = add_bpf_output_values(event_class, event, sample); + if (ret) + return -1; + } + + cs = ctf_stream(cw, get_sample_cpu(cw, sample, evsel)); + if (cs) { + if (is_flush_needed(cs)) + ctf_stream__flush(cs); + + cs->count++; + bt_ctf_stream_append_event(cs->stream, event); + } + + bt_ctf_event_put(event); + return cs ? 0 : -1; +} + +#define __NON_SAMPLE_SET_FIELD(_name, _type, _field) \ +do { \ + ret = value_set_##_type(cw, event, #_field, _event->_name._field);\ + if (ret) \ + return -1; \ +} while(0) + +#define __FUNC_PROCESS_NON_SAMPLE(_name, body) \ +static int process_##_name##_event(const struct perf_tool *tool, \ + union perf_event *_event, \ + struct perf_sample *sample, \ + struct machine *machine) \ +{ \ + struct convert *c = container_of(tool, struct convert, tool);\ + struct ctf_writer *cw = &c->writer; \ + struct bt_ctf_event_class *event_class = cw->_name##_class;\ + struct bt_ctf_event *event; \ + struct ctf_stream *cs; \ + int ret; \ + \ + c->non_sample_count++; \ + c->events_size += _event->header.size; \ + event = bt_ctf_event_create(event_class); \ + if (!event) { \ + pr_err("Failed to create an CTF event\n"); \ + return -1; \ + } \ + \ + bt_ctf_clock_set_time(cw->clock, sample->time); \ + body \ + cs = ctf_stream(cw, 0); \ + if (cs) { \ + if (is_flush_needed(cs)) \ + ctf_stream__flush(cs); \ + \ + cs->count++; \ + bt_ctf_stream_append_event(cs->stream, event); \ + } \ + bt_ctf_event_put(event); \ + \ + return perf_event__process_##_name(tool, _event, sample, machine);\ +} + +__FUNC_PROCESS_NON_SAMPLE(comm, + __NON_SAMPLE_SET_FIELD(comm, u32, pid); + __NON_SAMPLE_SET_FIELD(comm, u32, tid); + __NON_SAMPLE_SET_FIELD(comm, string, comm); +) +__FUNC_PROCESS_NON_SAMPLE(fork, + __NON_SAMPLE_SET_FIELD(fork, u32, pid); + __NON_SAMPLE_SET_FIELD(fork, u32, ppid); + __NON_SAMPLE_SET_FIELD(fork, u32, tid); + __NON_SAMPLE_SET_FIELD(fork, u32, ptid); + __NON_SAMPLE_SET_FIELD(fork, u64, time); +) + +__FUNC_PROCESS_NON_SAMPLE(exit, + __NON_SAMPLE_SET_FIELD(fork, u32, pid); + __NON_SAMPLE_SET_FIELD(fork, u32, ppid); + __NON_SAMPLE_SET_FIELD(fork, u32, tid); + __NON_SAMPLE_SET_FIELD(fork, u32, ptid); + __NON_SAMPLE_SET_FIELD(fork, u64, time); +) +__FUNC_PROCESS_NON_SAMPLE(mmap, + __NON_SAMPLE_SET_FIELD(mmap, u32, pid); + __NON_SAMPLE_SET_FIELD(mmap, u32, tid); + __NON_SAMPLE_SET_FIELD(mmap, u64_hex, start); + __NON_SAMPLE_SET_FIELD(mmap, string, filename); +) +__FUNC_PROCESS_NON_SAMPLE(mmap2, + __NON_SAMPLE_SET_FIELD(mmap2, u32, pid); + __NON_SAMPLE_SET_FIELD(mmap2, u32, tid); + __NON_SAMPLE_SET_FIELD(mmap2, u64_hex, start); + __NON_SAMPLE_SET_FIELD(mmap2, string, filename); +) +#undef __NON_SAMPLE_SET_FIELD +#undef __FUNC_PROCESS_NON_SAMPLE + +/* If dup < 0, add a prefix. Else, add _dupl_X suffix. */ +static char *change_name(char *name, char *orig_name, int dup) +{ + char *new_name = NULL; + size_t len; + + if (!name) + name = orig_name; + + if (dup >= 10) + goto out; + /* + * Add '_' prefix to potential keywork. According to + * Mathieu Desnoyers (https://lore.kernel.org/lkml/1074266107.40857.1422045946295.JavaMail.zimbra@efficios.com), + * further CTF spec updating may require us to use '$'. + */ + if (dup < 0) + len = strlen(name) + sizeof("_"); + else + len = strlen(orig_name) + sizeof("_dupl_X"); + + new_name = malloc(len); + if (!new_name) + goto out; + + if (dup < 0) + snprintf(new_name, len, "_%s", name); + else + snprintf(new_name, len, "%s_dupl_%d", orig_name, dup); + +out: + if (name != orig_name) + free(name); + return new_name; +} + +static int event_class_add_field(struct bt_ctf_event_class *event_class, + struct bt_ctf_field_type *type, + struct tep_format_field *field) +{ + struct bt_ctf_field_type *t = NULL; + char *name; + int dup = 1; + int ret; + + /* alias was already assigned */ + if (field->alias != field->name) + return bt_ctf_event_class_add_field(event_class, type, + (char *)field->alias); + + name = field->name; + + /* If 'name' is a keywork, add prefix. */ + if (bt_ctf_validate_identifier(name)) + name = change_name(name, field->name, -1); + + if (!name) { + pr_err("Failed to fix invalid identifier."); + return -1; + } + while ((t = bt_ctf_event_class_get_field_by_name(event_class, name))) { + bt_ctf_field_type_put(t); + name = change_name(name, field->name, dup++); + if (!name) { + pr_err("Failed to create dup name for '%s'\n", field->name); + return -1; + } + } + + ret = bt_ctf_event_class_add_field(event_class, type, name); + if (!ret) + field->alias = name; + + return ret; +} + +static int add_tracepoint_fields_types(struct ctf_writer *cw, + struct tep_format_field *fields, + struct bt_ctf_event_class *event_class) +{ + struct tep_format_field *field; + int ret; + + for (field = fields; field; field = field->next) { + struct bt_ctf_field_type *type; + unsigned long flags = field->flags; + + pr2(" field '%s'\n", field->name); + + type = get_tracepoint_field_type(cw, field); + if (!type) + return -1; + + /* + * A string is an array of chars. For this we use the string + * type and don't care that it is an array. What we don't + * support is an array of strings. + */ + if (flags & TEP_FIELD_IS_STRING) + flags &= ~TEP_FIELD_IS_ARRAY; + + if (flags & TEP_FIELD_IS_ARRAY) + type = bt_ctf_field_type_array_create(type, field->arraylen); + + ret = event_class_add_field(event_class, type, field); + + if (flags & TEP_FIELD_IS_ARRAY) + bt_ctf_field_type_put(type); + + if (ret) { + pr_err("Failed to add field '%s': %d\n", + field->name, ret); + return -1; + } + } + + return 0; +} + +static int add_tracepoint_types(struct ctf_writer *cw, + struct evsel *evsel, + struct bt_ctf_event_class *class) +{ + const struct tep_event *tp_format = evsel__tp_format(evsel); + struct tep_format_field *common_fields = tp_format ? tp_format->format.common_fields : NULL; + struct tep_format_field *fields = tp_format ? tp_format->format.fields : NULL; + int ret; + + ret = add_tracepoint_fields_types(cw, common_fields, class); + if (!ret) + ret = add_tracepoint_fields_types(cw, fields, class); + + return ret; +} + +static int add_bpf_output_types(struct ctf_writer *cw, + struct bt_ctf_event_class *class) +{ + struct bt_ctf_field_type *len_type = cw->data.u32; + struct bt_ctf_field_type *seq_base_type = cw->data.u32_hex; + struct bt_ctf_field_type *seq_type; + int ret; + + ret = bt_ctf_event_class_add_field(class, len_type, "raw_len"); + if (ret) + return ret; + + seq_type = bt_ctf_field_type_sequence_create(seq_base_type, "raw_len"); + if (!seq_type) + return -1; + + return bt_ctf_event_class_add_field(class, seq_type, "raw_data"); +} + +static int add_generic_types(struct ctf_writer *cw, struct evsel *evsel, + struct bt_ctf_event_class *event_class) +{ + u64 type = evsel->core.attr.sample_type; + + /* + * missing: + * PERF_SAMPLE_TIME - not needed as we have it in + * ctf event header + * PERF_SAMPLE_READ - TODO + * PERF_SAMPLE_CALLCHAIN - TODO + * PERF_SAMPLE_RAW - tracepoint fields and BPF output + * are handled separately + * PERF_SAMPLE_BRANCH_STACK - TODO + * PERF_SAMPLE_REGS_USER - TODO + * PERF_SAMPLE_STACK_USER - TODO + */ + +#define ADD_FIELD(cl, t, n) \ + do { \ + pr2(" field '%s'\n", n); \ + if (bt_ctf_event_class_add_field(cl, t, n)) { \ + pr_err("Failed to add field '%s';\n", n); \ + return -1; \ + } \ + } while (0) + + if (type & PERF_SAMPLE_IP) + ADD_FIELD(event_class, cw->data.u64_hex, "perf_ip"); + + if (type & PERF_SAMPLE_TID) { + ADD_FIELD(event_class, cw->data.s32, "perf_tid"); + ADD_FIELD(event_class, cw->data.s32, "perf_pid"); + } + + if ((type & PERF_SAMPLE_ID) || + (type & PERF_SAMPLE_IDENTIFIER)) + ADD_FIELD(event_class, cw->data.u64, "perf_id"); + + if (type & PERF_SAMPLE_STREAM_ID) + ADD_FIELD(event_class, cw->data.u64, "perf_stream_id"); + + if (type & PERF_SAMPLE_PERIOD) + ADD_FIELD(event_class, cw->data.u64, "perf_period"); + + if (type & PERF_SAMPLE_WEIGHT) + ADD_FIELD(event_class, cw->data.u64, "perf_weight"); + + if (type & PERF_SAMPLE_DATA_SRC) + ADD_FIELD(event_class, cw->data.u64, "perf_data_src"); + + if (type & PERF_SAMPLE_TRANSACTION) + ADD_FIELD(event_class, cw->data.u64, "perf_transaction"); + + if (type & PERF_SAMPLE_CALLCHAIN) { + ADD_FIELD(event_class, cw->data.u32, "perf_callchain_size"); + ADD_FIELD(event_class, + bt_ctf_field_type_sequence_create( + cw->data.u64_hex, "perf_callchain_size"), + "perf_callchain"); + } + +#undef ADD_FIELD + return 0; +} + +static int add_event(struct ctf_writer *cw, struct evsel *evsel) +{ + struct bt_ctf_event_class *event_class; + struct evsel_priv *priv; + const char *name = evsel__name(evsel); + int ret; + + if (evsel->priv) { + pr_err("Error: attempt to add already added event %s\n", name); + return -1; + } + pr("Adding event '%s' (type %d)\n", name, evsel->core.attr.type); + + event_class = bt_ctf_event_class_create(name); + if (!event_class) + return -1; + + ret = add_generic_types(cw, evsel, event_class); + if (ret) + goto err; + + if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT) { + ret = add_tracepoint_types(cw, evsel, event_class); + if (ret) + goto err; + } + + if (evsel__is_bpf_output(evsel)) { + ret = add_bpf_output_types(cw, event_class); + if (ret) + goto err; + } + + ret = bt_ctf_stream_class_add_event_class(cw->stream_class, event_class); + if (ret) { + pr("Failed to add event class into stream.\n"); + goto err; + } + + priv = malloc(sizeof(*priv)); + if (!priv) + goto err; + + priv->event_class = event_class; + evsel->priv = priv; + return 0; + +err: + bt_ctf_event_class_put(event_class); + pr_err("Failed to add event '%s'.\n", name); + return -1; +} + +enum setup_events_type { + SETUP_EVENTS_ALL, + SETUP_EVENTS_NOT_TRACEPOINT, + SETUP_EVENTS_TRACEPOINT_ONLY, +}; + +static int setup_events(struct ctf_writer *cw, struct perf_session *session, + enum setup_events_type type) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + int ret; + + evlist__for_each_entry(evlist, evsel) { + bool is_tracepoint = evsel->core.attr.type == PERF_TYPE_TRACEPOINT; + + if (is_tracepoint && type == SETUP_EVENTS_NOT_TRACEPOINT) + continue; + + if (!is_tracepoint && type == SETUP_EVENTS_TRACEPOINT_ONLY) + continue; + + ret = add_event(cw, evsel); + if (ret) + return ret; + } + return 0; +} + +#define __NON_SAMPLE_ADD_FIELD(t, n) \ + do { \ + pr2(" field '%s'\n", #n); \ + if (bt_ctf_event_class_add_field(event_class, cw->data.t, #n)) {\ + pr_err("Failed to add field '%s';\n", #n);\ + return -1; \ + } \ + } while(0) + +#define __FUNC_ADD_NON_SAMPLE_EVENT_CLASS(_name, body) \ +static int add_##_name##_event(struct ctf_writer *cw) \ +{ \ + struct bt_ctf_event_class *event_class; \ + int ret; \ + \ + pr("Adding "#_name" event\n"); \ + event_class = bt_ctf_event_class_create("perf_" #_name);\ + if (!event_class) \ + return -1; \ + body \ + \ + ret = bt_ctf_stream_class_add_event_class(cw->stream_class, event_class);\ + if (ret) { \ + pr("Failed to add event class '"#_name"' into stream.\n");\ + return ret; \ + } \ + \ + cw->_name##_class = event_class; \ + bt_ctf_event_class_put(event_class); \ + return 0; \ +} + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(comm, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(string, comm); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(fork, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, ppid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u32, ptid); + __NON_SAMPLE_ADD_FIELD(u64, time); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(exit, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, ppid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u32, ptid); + __NON_SAMPLE_ADD_FIELD(u64, time); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(mmap, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u64_hex, start); + __NON_SAMPLE_ADD_FIELD(string, filename); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(mmap2, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u64_hex, start); + __NON_SAMPLE_ADD_FIELD(string, filename); +) +#undef __NON_SAMPLE_ADD_FIELD +#undef __FUNC_ADD_NON_SAMPLE_EVENT_CLASS + +static int setup_non_sample_events(struct ctf_writer *cw, + struct perf_session *session __maybe_unused) +{ + int ret; + + ret = add_comm_event(cw); + if (ret) + return ret; + ret = add_exit_event(cw); + if (ret) + return ret; + ret = add_fork_event(cw); + if (ret) + return ret; + ret = add_mmap_event(cw); + if (ret) + return ret; + ret = add_mmap2_event(cw); + if (ret) + return ret; + return 0; +} + +static void cleanup_events(struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + struct evsel_priv *priv; + + priv = evsel->priv; + if (priv) + bt_ctf_event_class_put(priv->event_class); + zfree(&evsel->priv); + } + + evlist__put(evlist); + session->evlist = NULL; +} + +static int setup_streams(struct ctf_writer *cw, struct perf_session *session) +{ + struct ctf_stream **stream; + struct perf_env *env = perf_session__env(session); + int ncpus; + + /* + * Try to get the number of cpus used in the data file, + * if not present fallback to the MAX_CPUS. + */ + ncpus = env->nr_cpus_avail ?: MAX_CPUS; + + stream = calloc(ncpus, sizeof(*stream)); + if (!stream) { + pr_err("Failed to allocate streams.\n"); + return -ENOMEM; + } + + cw->stream = stream; + cw->stream_cnt = ncpus; + return 0; +} + +static void free_streams(struct ctf_writer *cw) +{ + int cpu; + + for (cpu = 0; cpu < cw->stream_cnt; cpu++) + ctf_stream__delete(cw->stream[cpu]); + + zfree(&cw->stream); +} + +static int ctf_writer__setup_env(struct ctf_writer *cw, + struct perf_session *session) +{ + struct perf_env *env = perf_session__env(session); + struct bt_ctf_writer *writer = cw->writer; + +#define ADD(__n, __v) \ +do { \ + if (__v && bt_ctf_writer_add_environment_field(writer, __n, __v)) \ + return -1; \ +} while (0) + + ADD("host", env->hostname); + ADD("sysname", "Linux"); + ADD("release", perf_env__os_release(env)); + ADD("version", env->version); + ADD("machine", env->arch); + ADD("domain", "kernel"); + ADD("tracer_name", "perf"); + +#undef ADD + return 0; +} + +static int process_feature_event(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event) +{ + struct convert *c = container_of(tool, struct convert, tool); + struct ctf_writer *cw = &c->writer; + struct perf_record_header_feature *fe = &event->feat; + int ret = perf_event__process_feature(tool, session, event); + + if (ret) + return ret; + + switch (fe->feat_id) { + case HEADER_EVENT_DESC: + /* + * In non-pipe mode (not here) the evsels combine the desc with + * the perf_event_attr when it is parsed. In pipe mode the + * perf_event_attr events appear first and then the event desc + * feature events that set the names appear after. Once we have + * the full evsel data we can generate the babeltrace + * events. For tracepoint events we still don't have the tracing + * data and so need to wait until the tracing data event to add + * those events to babeltrace. + */ + return setup_events(cw, session, SETUP_EVENTS_NOT_TRACEPOINT); + case HEADER_HOSTNAME: + if (session->header.env.hostname) { + return bt_ctf_writer_add_environment_field(cw->writer, "host", + session->header.env.hostname); + } + break; + case HEADER_OSRELEASE: + if (session->header.env.os_release) { + return bt_ctf_writer_add_environment_field(cw->writer, "release", + session->header.env.os_release); + } + break; + case HEADER_VERSION: + if (session->header.env.version) { + return bt_ctf_writer_add_environment_field(cw->writer, "version", + session->header.env.version); + } + break; + case HEADER_ARCH: + if (session->header.env.arch) { + return bt_ctf_writer_add_environment_field(cw->writer, "machine", + session->header.env.arch); + } + break; + default: + break; + } + return 0; +} + +static int process_tracing_data(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event) +{ + struct convert *c = container_of(tool, struct convert, tool); + struct ctf_writer *cw = &c->writer; + int ret; + + ret = perf_event__process_tracing_data(tool, session, event); + if (ret < 0) + return ret; + + /* + * Now the attr was set up by the attr event, the name by the feature + * event desc event and the tracepoint data set up above, the tracepoint + * babeltrace events can be added. + */ + return setup_events(cw, session, SETUP_EVENTS_TRACEPOINT_ONLY); +} + +static int ctf_writer__setup_clock(struct ctf_writer *cw, + struct perf_session *session, + bool tod) +{ + struct bt_ctf_clock *clock = cw->clock; + const char *desc = "perf clock"; + int64_t offset = 0; + + if (tod) { + struct perf_env *env = perf_session__env(session); + + if (!env->clock.enabled) { + pr_err("Can't provide --tod time, missing clock data. " + "Please record with -k/--clockid option.\n"); + return -1; + } + + desc = clockid_name(env->clock.clockid); + offset = env->clock.tod_ns - env->clock.clockid_ns; + } + +#define SET(__n, __v) \ +do { \ + if (bt_ctf_clock_set_##__n(clock, __v)) \ + return -1; \ +} while (0) + + SET(frequency, 1000000000); + SET(offset, offset); + SET(description, desc); + SET(precision, 10); + SET(is_absolute, 0); + +#undef SET + return 0; +} + +static struct bt_ctf_field_type *create_int_type(int size, bool sign, bool hex) +{ + struct bt_ctf_field_type *type; + + type = bt_ctf_field_type_integer_create(size); + if (!type) + return NULL; + + if (sign && + bt_ctf_field_type_integer_set_signed(type, 1)) + goto err; + + if (hex && + bt_ctf_field_type_integer_set_base(type, BT_CTF_INTEGER_BASE_HEXADECIMAL)) + goto err; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bt_ctf_field_type_set_byte_order(type, BT_CTF_BYTE_ORDER_BIG_ENDIAN); +#else + bt_ctf_field_type_set_byte_order(type, BT_CTF_BYTE_ORDER_LITTLE_ENDIAN); +#endif + + pr2("Created type: INTEGER %d-bit %ssigned %s\n", + size, sign ? "un" : "", hex ? "hex" : ""); + return type; + +err: + bt_ctf_field_type_put(type); + return NULL; +} + +static void ctf_writer__cleanup_data(struct ctf_writer *cw) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cw->data.array); i++) + bt_ctf_field_type_put(cw->data.array[i]); +} + +static int ctf_writer__init_data(struct ctf_writer *cw) +{ +#define CREATE_INT_TYPE(type, size, sign, hex) \ +do { \ + (type) = create_int_type(size, sign, hex); \ + if (!(type)) \ + goto err; \ +} while (0) + + CREATE_INT_TYPE(cw->data.s64, 64, true, false); + CREATE_INT_TYPE(cw->data.u64, 64, false, false); + CREATE_INT_TYPE(cw->data.s32, 32, true, false); + CREATE_INT_TYPE(cw->data.u32, 32, false, false); + CREATE_INT_TYPE(cw->data.u32_hex, 32, false, true); + CREATE_INT_TYPE(cw->data.u64_hex, 64, false, true); + + cw->data.string = bt_ctf_field_type_string_create(); + if (cw->data.string) + return 0; + +err: + ctf_writer__cleanup_data(cw); + pr_err("Failed to create data types.\n"); + return -1; +} + +static void ctf_writer__cleanup(struct ctf_writer *cw) +{ + ctf_writer__cleanup_data(cw); + + bt_ctf_clock_put(cw->clock); + free_streams(cw); + bt_ctf_stream_class_put(cw->stream_class); + bt_ctf_writer_put(cw->writer); + + /* and NULL all the pointers */ + memset(cw, 0, sizeof(*cw)); +} + +static int ctf_writer__init(struct ctf_writer *cw, const char *path, + struct perf_session *session, bool tod) +{ + struct bt_ctf_writer *writer; + struct bt_ctf_stream_class *stream_class; + struct bt_ctf_clock *clock; + struct bt_ctf_field_type *pkt_ctx_type; + int ret; + + /* CTF writer */ + writer = bt_ctf_writer_create(path); + if (!writer) + goto err; + + cw->writer = writer; + + /* CTF clock */ + clock = bt_ctf_clock_create("perf_clock"); + if (!clock) { + pr("Failed to create CTF clock.\n"); + goto err_cleanup; + } + + cw->clock = clock; + + if (ctf_writer__setup_clock(cw, session, tod)) { + pr("Failed to setup CTF clock.\n"); + goto err_cleanup; + } + + /* CTF stream class */ + stream_class = bt_ctf_stream_class_create("perf_stream"); + if (!stream_class) { + pr("Failed to create CTF stream class.\n"); + goto err_cleanup; + } + + cw->stream_class = stream_class; + + /* CTF clock stream setup */ + if (bt_ctf_stream_class_set_clock(stream_class, clock)) { + pr("Failed to assign CTF clock to stream class.\n"); + goto err_cleanup; + } + + if (ctf_writer__init_data(cw)) + goto err_cleanup; + + /* Add cpu_id for packet context */ + pkt_ctx_type = bt_ctf_stream_class_get_packet_context_type(stream_class); + if (!pkt_ctx_type) + goto err_cleanup; + + ret = bt_ctf_field_type_structure_add_field(pkt_ctx_type, cw->data.u32, "cpu_id"); + bt_ctf_field_type_put(pkt_ctx_type); + if (ret) + goto err_cleanup; + + /* CTF clock writer setup */ + if (bt_ctf_writer_add_clock(writer, clock)) { + pr("Failed to assign CTF clock to writer.\n"); + goto err_cleanup; + } + + return 0; + +err_cleanup: + ctf_writer__cleanup(cw); +err: + pr_err("Failed to setup CTF writer.\n"); + return -1; +} + +static int ctf_writer__flush_streams(struct ctf_writer *cw) +{ + int cpu, ret = 0; + + for (cpu = 0; cpu < cw->stream_cnt && !ret; cpu++) + ret = ctf_stream__flush(cw->stream[cpu]); + + return ret; +} + +static int convert__config(const char *var, const char *value, void *cb) +{ + struct convert *c = cb; + + if (!strcmp(var, "convert.queue-size")) + return perf_config_u64(&c->queue_size, var, value); + + return 0; +} + +int bt_convert__perf2ctf(const char *input, const char *path, + struct perf_data_convert_opts *opts) +{ + struct perf_session *session; + struct perf_data data = { + .path = input, + .mode = PERF_DATA_MODE_READ, + .force = opts->force, + }; + struct convert c = {}; + struct ctf_writer *cw = &c.writer; + int err; + + perf_tool__init(&c.tool, /*ordered_events=*/true); + c.tool.sample = process_sample_event; + c.tool.mmap = perf_event__process_mmap; + c.tool.mmap2 = perf_event__process_mmap2; + c.tool.comm = perf_event__process_comm; + c.tool.exit = perf_event__process_exit; + c.tool.fork = perf_event__process_fork; + c.tool.lost = perf_event__process_lost; + c.tool.tracing_data = process_tracing_data; + c.tool.build_id = perf_event__process_build_id; + c.tool.namespaces = perf_event__process_namespaces; + c.tool.finished_round = perf_event__process_finished_round; + c.tool.attr = perf_event__process_attr; + c.tool.feature = process_feature_event; + c.tool.ordering_requires_timestamps = true; + + if (opts->all) { + c.tool.comm = process_comm_event; + c.tool.exit = process_exit_event; + c.tool.fork = process_fork_event; + c.tool.mmap = process_mmap_event; + c.tool.mmap2 = process_mmap2_event; + } + + err = perf_config(convert__config, &c); + if (err) + return err; + + err = -1; + /* perf.data session */ + session = perf_session__new(&data, &c.tool); + if (IS_ERR(session)) + return PTR_ERR(session); + + if (opts->time_str) { + err = perf_time__parse_for_ranges(opts->time_str, session, + &c.ptime_range, + &c.range_size, + &c.range_num); + if (err < 0) + goto free_session; + } + + /* CTF writer */ + if (ctf_writer__init(cw, path, session, opts->tod)) + goto free_session; + + if (c.queue_size) { + ordered_events__set_alloc_size(&session->ordered_events, + c.queue_size); + } + + /* CTF writer env/clock setup */ + if (ctf_writer__setup_env(cw, session)) + goto free_writer; + + /* + * CTF events setup. Note, in pipe mode no events exist yet (they come + * in via header feature events) and so this does nothing. + */ + if (setup_events(cw, session, SETUP_EVENTS_ALL)) + goto free_writer; + + if (opts->all && setup_non_sample_events(cw, session)) + goto free_writer; + + if (setup_streams(cw, session)) + goto free_writer; + + err = perf_session__process_events(session); + if (!err) + err = ctf_writer__flush_streams(cw); + else + pr_err("Error during conversion.\n"); + + fprintf(stderr, "[ perf data convert: Converted '%s' into CTF data '%s' ]\n", + data.path, path); + + fprintf(stderr, "[ perf data convert: Converted and wrote %.3f MB (%" PRIu64 " samples", + (double) c.events_size / 1024.0 / 1024.0, + c.events_count); + + if (!c.non_sample_count) + fprintf(stderr, ") ]\n"); + else + fprintf(stderr, ", %" PRIu64 " non-samples) ]\n", c.non_sample_count); + + if (c.skipped) { + fprintf(stderr, "[ perf data convert: Skipped %" PRIu64 " samples ]\n", + c.skipped); + } + + if (c.ptime_range) + zfree(&c.ptime_range); + + cleanup_events(session); + perf_session__delete(session); + ctf_writer__cleanup(cw); + + return err; + +free_writer: + ctf_writer__cleanup(cw); +free_session: + if (c.ptime_range) + zfree(&c.ptime_range); + + perf_session__delete(session); + pr_err("Error during conversion setup.\n"); + return err; +} diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c new file mode 100644 index 000000000..40888b7c4 --- /dev/null +++ b/tools/perf/util/data-convert-json.c @@ -0,0 +1,462 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * JSON export. + * + * Copyright (C) 2021, CodeWeavers Inc. + */ + +#include "data-convert.h" + +#include +#include +#include +#include + +#include "linux/compiler.h" +#include "linux/err.h" +#include "util/auxtrace.h" +#include "util/debug.h" +#include "util/env.h" +#include "util/dso.h" +#include "util/event.h" +#include "util/evsel.h" +#include "util/evlist.h" +#include "util/header.h" +#include "util/map.h" +#include "util/session.h" +#include "util/symbol.h" +#include "util/thread.h" +#include "util/time-utils.h" +#include "util/tool.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +struct convert_json { + struct perf_tool tool; + FILE *out; + bool first; + struct perf_time_interval *ptime_range; + int range_size; + int range_num; + + u64 events_count; + u64 skipped; +}; + +// Outputs a JSON-encoded string surrounded by quotes with characters escaped. +static void output_json_string(FILE *out, const char *s) +{ + fputc('"', out); + if (!s) + goto out; + + while (*s) { + switch (*s) { + + // required escapes with special forms as per RFC 8259 + case '"': fputs("\\\"", out); break; + case '\\': fputs("\\\\", out); break; + case '\b': fputs("\\b", out); break; + case '\f': fputs("\\f", out); break; + case '\n': fputs("\\n", out); break; + case '\r': fputs("\\r", out); break; + case '\t': fputs("\\t", out); break; + + default: + // all other control characters must be escaped by hex code + if (*s <= 0x1f) + fprintf(out, "\\u%04x", *s); + else + fputc(*s, out); + break; + } + + ++s; + } +out: + fputc('"', out); +} + +// Outputs an optional comma, newline and indentation to delimit a new value +// from the previous one in a JSON object or array. +static void output_json_delimiters(FILE *out, bool comma, int depth) +{ + int i; + + if (comma) + fputc(',', out); + fputc('\n', out); + for (i = 0; i < depth; ++i) + fputc('\t', out); +} + +// Outputs a printf format string (with delimiter) as a JSON value. +__printf(4, 5) +static void output_json_format(FILE *out, bool comma, int depth, const char *format, ...) +{ + va_list args; + + output_json_delimiters(out, comma, depth); + va_start(args, format); + vfprintf(out, format, args); + va_end(args); +} + +// Outputs a JSON key-value pair where the value is a string. +static void output_json_key_string(FILE *out, bool comma, int depth, + const char *key, const char *value) +{ + output_json_delimiters(out, comma, depth); + output_json_string(out, key); + fputs(": ", out); + output_json_string(out, value); +} + +// Outputs a JSON key-value pair where the value is a printf format string. +__printf(5, 6) +static void output_json_key_format(FILE *out, bool comma, int depth, + const char *key, const char *format, ...) +{ + va_list args; + + output_json_delimiters(out, comma, depth); + output_json_string(out, key); + fputs(": ", out); + va_start(args, format); + vfprintf(out, format, args); + va_end(args); +} + +static void output_sample_callchain_entry(const struct perf_tool *tool, + u64 ip, struct addr_location *al) +{ + struct convert_json *c = container_of(tool, struct convert_json, tool); + FILE *out = c->out; + + output_json_format(out, false, 4, "{"); + output_json_key_format(out, false, 5, "ip", "\"0x%" PRIx64 "\"", ip); + + if (al && al->sym && al->sym->namelen) { + struct dso *dso = al->map ? map__dso(al->map) : NULL; + + fputc(',', out); + output_json_key_string(out, false, 5, "symbol", al->sym->name); + + if (dso) { + const char *dso_name = dso__short_name(dso); + + if (dso_name && strlen(dso_name) > 0) { + fputc(',', out); + output_json_key_string(out, false, 5, "dso", dso_name); + } + } + } + + output_json_format(out, false, 4, "}"); +} + +static int process_sample_event(const struct perf_tool *tool, + union perf_event *event __maybe_unused, + struct perf_sample *sample, + struct machine *machine) +{ + struct convert_json *c = container_of(tool, struct convert_json, tool); + FILE *out = c->out; + struct addr_location al; + u64 sample_type = __evlist__combined_sample_type(sample->evsel->evlist); + u8 cpumode = PERF_RECORD_MISC_USER; + + addr_location__init(&al); + if (machine__resolve(machine, &al, sample) < 0) { + pr_err("Sample resolution failed!\n"); + addr_location__exit(&al); + return -1; + } + + if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { + ++c->skipped; + addr_location__exit(&al); + return 0; + } + + ++c->events_count; + + if (c->first) + c->first = false; + else + fputc(',', out); + output_json_format(out, false, 2, "{"); + + output_json_key_format(out, false, 3, "timestamp", "%" PRIi64, sample->time); + output_json_key_format(out, true, 3, "pid", "%i", thread__pid(al.thread)); + output_json_key_format(out, true, 3, "tid", "%i", thread__tid(al.thread)); + + if ((sample_type & PERF_SAMPLE_CPU)) + output_json_key_format(out, true, 3, "cpu", "%i", sample->cpu); + else if (thread__cpu(al.thread) >= 0) + output_json_key_format(out, true, 3, "cpu", "%i", thread__cpu(al.thread)); + + output_json_key_string(out, true, 3, "comm", thread__comm_str(al.thread)); + + output_json_key_format(out, true, 3, "callchain", "["); + if (sample->callchain) { + unsigned int i; + bool ok; + bool first_callchain = true; + + for (i = 0; i < sample->callchain->nr; ++i) { + u64 ip = sample->callchain->ips[i]; + struct addr_location tal; + + if (ip >= PERF_CONTEXT_MAX) { + switch (ip) { + case PERF_CONTEXT_HV: + cpumode = PERF_RECORD_MISC_HYPERVISOR; + break; + case PERF_CONTEXT_KERNEL: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + case PERF_CONTEXT_USER: + cpumode = PERF_RECORD_MISC_USER; + break; + default: + pr_debug("invalid callchain context: %" + PRId64 "\n", (s64) ip); + break; + } + continue; + } + + if (first_callchain) + first_callchain = false; + else + fputc(',', out); + + addr_location__init(&tal); + ok = thread__find_symbol(al.thread, cpumode, ip, &tal); + output_sample_callchain_entry(tool, ip, ok ? &tal : NULL); + addr_location__exit(&tal); + } + } else { + output_sample_callchain_entry(tool, sample->ip, &al); + } + output_json_format(out, false, 3, "]"); + +#ifdef HAVE_LIBTRACEEVENT + if (sample->raw_data) { + struct tep_event *tp_format = evsel__tp_format(sample->evsel); + struct tep_format_field **fields = tp_format ? tep_event_fields(tp_format) : NULL; + + if (fields) { + int i = 0; + + while (fields[i]) { + struct trace_seq s; + + trace_seq_init(&s); + tep_print_field(&s, sample->raw_data, fields[i]); + output_json_key_string(out, true, 3, fields[i]->name, s.buffer); + trace_seq_destroy(&s); + + i++; + } + free(fields); + } + } +#endif + output_json_format(out, false, 2, "}"); + addr_location__exit(&al); + return 0; +} + +static void output_headers(struct perf_session *session, struct convert_json *c) +{ + struct stat st; + const struct perf_header *header = &session->header; + struct perf_env *env = perf_session__env(session); + int ret; + int fd = perf_data__fd(session->data); + int i; + FILE *out = c->out; + + output_json_key_format(out, false, 2, "header-version", "%u", header->version); + + ret = fstat(fd, &st); + if (ret >= 0) { + time_t stctime = st.st_mtime; + char buf[256]; + + strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&stctime)); + output_json_key_string(out, true, 2, "captured-on", buf); + } else { + pr_debug("Failed to get mtime of source file, not writing captured-on"); + } + + output_json_key_format(out, true, 2, "data-offset", "%" PRIu64, header->data_offset); + output_json_key_format(out, true, 2, "data-size", "%" PRIu64, header->data_size); + output_json_key_format(out, true, 2, "feat-offset", "%" PRIu64, header->feat_offset); + + output_json_key_string(out, true, 2, "hostname", env->hostname); + output_json_key_string(out, true, 2, "os-release", + perf_env__os_release(env)); + output_json_key_string(out, true, 2, "arch", env->arch); + + if (env->cpu_desc) + output_json_key_string(out, true, 2, "cpu-desc", env->cpu_desc); + + output_json_key_string(out, true, 2, "cpuid", env->cpuid); + output_json_key_format(out, true, 2, "nrcpus-online", "%u", env->nr_cpus_online); + output_json_key_format(out, true, 2, "nrcpus-avail", "%u", env->nr_cpus_avail); + + if (env->clock.enabled) { + output_json_key_format(out, true, 2, "clockid", + "%u", env->clock.clockid); + output_json_key_format(out, true, 2, "clock-time", + "%" PRIu64, env->clock.clockid_ns); + output_json_key_format(out, true, 2, "real-time", + "%" PRIu64, env->clock.tod_ns); + } + + output_json_key_string(out, true, 2, "perf-version", env->version); + + output_json_key_format(out, true, 2, "cmdline", "["); + for (i = 0; i < env->nr_cmdline; i++) { + output_json_delimiters(out, i != 0, 3); + output_json_string(c->out, env->cmdline_argv[i]); + } + output_json_format(out, false, 2, "]"); +} + +int bt_convert__perf2json(const char *_input_name, const char *output_name, + struct perf_data_convert_opts *opts __maybe_unused) +{ + struct perf_session *session; + int fd; + int ret = -1; + struct convert_json c = { + .first = true, + .events_count = 0, + .ptime_range = NULL, + .range_size = 0, + .range_num = 0, + .skipped = 0, + }; + struct perf_data data = { + .mode = PERF_DATA_MODE_READ, + .path = _input_name, + .force = opts->force, + }; + + perf_tool__init(&c.tool, /*ordered_events=*/true); + c.tool.sample = process_sample_event; + c.tool.mmap = perf_event__process_mmap; + c.tool.mmap2 = perf_event__process_mmap2; + c.tool.comm = perf_event__process_comm; + c.tool.namespaces = perf_event__process_namespaces; + c.tool.cgroup = perf_event__process_cgroup; + c.tool.exit = perf_event__process_exit; + c.tool.fork = perf_event__process_fork; + c.tool.lost = perf_event__process_lost; +#ifdef HAVE_LIBTRACEEVENT + c.tool.tracing_data = perf_event__process_tracing_data; +#endif + c.tool.build_id = perf_event__process_build_id; + c.tool.id_index = perf_event__process_id_index; + c.tool.auxtrace_info = perf_event__process_auxtrace_info; + c.tool.auxtrace = perf_event__process_auxtrace; + c.tool.event_update = perf_event__process_event_update; + c.tool.attr = perf_event__process_attr; + c.tool.feature = perf_event__process_feature; + c.tool.ordering_requires_timestamps = true; + + if (opts->all) { + pr_err("--all is currently unsupported for JSON output.\n"); + goto err; + } + if (opts->tod) { + pr_err("--tod is currently unsupported for JSON output.\n"); + goto err; + } + + fd = open(output_name, O_CREAT | O_WRONLY | (opts->force ? O_TRUNC : O_EXCL), 0666); + if (fd == -1) { + if (errno == EEXIST) + pr_err("Output file exists. Use --force to overwrite it.\n"); + else + pr_err("Error opening output file!\n"); + goto err; + } + + c.out = fdopen(fd, "w"); + if (!c.out) { + fprintf(stderr, "Error opening output file!\n"); + close(fd); + goto err; + } + + session = perf_session__new(&data, &c.tool); + if (IS_ERR(session)) { + fprintf(stderr, "Error creating perf session!\n"); + goto err_fclose; + } + if (symbol__init(perf_session__env(session)) < 0) { + fprintf(stderr, "Symbol init error!\n"); + goto err_session_delete; + } + + if (opts->time_str) { + ret = perf_time__parse_for_ranges(opts->time_str, session, + &c.ptime_range, + &c.range_size, + &c.range_num); + if (ret < 0) + goto err_session_delete; + } + + // The opening brace is printed manually because it isn't delimited from a + // previous value (i.e. we don't want a leading newline) + fputc('{', c.out); + + // Version number for future-proofing. Most additions should be able to be + // done in a backwards-compatible way so this should only need to be bumped + // if some major breaking change must be made. + output_json_format(c.out, false, 1, "\"linux-perf-json-version\": 1"); + + // Output headers + output_json_format(c.out, true, 1, "\"headers\": {"); + output_headers(session, &c); + output_json_format(c.out, false, 1, "}"); + + // Output samples + output_json_format(c.out, true, 1, "\"samples\": ["); + perf_session__process_events(session); + output_json_format(c.out, false, 1, "]"); + output_json_format(c.out, false, 0, "}"); + fputc('\n', c.out); + + fprintf(stderr, "[ perf data convert: Converted '%s' into JSON data '%s' ]\n", + data.path, output_name); + + fprintf(stderr, + "[ perf data convert: Converted and wrote %.3f MB (%" PRIu64 " samples) ]\n", + (ftell(c.out)) / 1024.0 / 1024.0, c.events_count); + + if (c.skipped) { + fprintf(stderr, "[ perf data convert: Skipped %" PRIu64 " samples ]\n", + c.skipped); + } + + ret = 0; + + if (c.ptime_range) + zfree(&c.ptime_range); + +err_session_delete: + perf_session__delete(session); +err_fclose: + fclose(c.out); +err: + return ret; +} diff --git a/tools/perf/util/data-convert.h b/tools/perf/util/data-convert.h new file mode 100644 index 000000000..a96240f15 --- /dev/null +++ b/tools/perf/util/data-convert.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DATA_CONVERT_H +#define __DATA_CONVERT_H + +#include + +struct perf_data_convert_opts { + bool force; + bool all; + bool tod; + const char *time_str; +}; + +#ifdef HAVE_BABELTRACE2_CTF_WRITER_SUPPORT +int bt_convert__perf2ctf(const char *input_name, const char *to_ctf, + struct perf_data_convert_opts *opts); +#endif /* HAVE_BABELTRACE2_CTF_WRITER_SUPPORT */ + +int bt_convert__perf2json(const char *input_name, const char *to_ctf, + struct perf_data_convert_opts *opts); + +#endif /* __DATA_CONVERT_H */ diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c new file mode 100644 index 000000000..33fd1b820 --- /dev/null +++ b/tools/perf/util/data.c @@ -0,0 +1,597 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "data.h" +#include "util.h" // rm_rf_perf_data() +#include "debug.h" +#include "header.h" +#include "rlimit.h" +#include + +static void perf_data_file__close(struct perf_data_file *file) +{ + if (file->use_stdio) { + if (file->fptr) { + fclose(file->fptr); + file->fptr = NULL; + } + } else { + close(file->fd); + file->fd = -1; + } + zfree(&file->path); +} + +static void close_dir(struct perf_data_file *files, int nr) +{ + while (--nr >= 0) + perf_data_file__close(&files[nr]); + + free(files); +} + +void perf_data__close_dir(struct perf_data *data) +{ + close_dir(data->dir.files, data->dir.nr); + data->dir.files = NULL; + data->dir.nr = 0; +} + +int perf_data__create_dir(struct perf_data *data, int nr) +{ + enum rlimit_action set_rlimit = NO_CHANGE; + struct perf_data_file *files = NULL; + int i, ret; + + if (WARN_ON(!data->is_dir)) + return -EINVAL; + + files = calloc(nr, sizeof(*files)); + if (!files) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + struct perf_data_file *file = &files[i]; + + ret = asprintf(&file->path, "%s/data.%d", data->path, i); + if (ret < 0) { + ret = -ENOMEM; + goto out_err; + } + +retry_open: + ret = open(file->path, O_RDWR|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); + if (ret < 0) { + /* + * If using parallel threads to collect data, + * perf record needs at least 6 fds per CPU. + * When we run out of them try to increase the limits. + */ + if (errno == EMFILE && rlimit__increase_nofile(&set_rlimit)) + goto retry_open; + + ret = -errno; + goto out_err; + } + set_rlimit = NO_CHANGE; + + file->fd = ret; + } + + data->dir.version = PERF_DIR_VERSION; + data->dir.files = files; + data->dir.nr = nr; + return 0; + +out_err: + close_dir(files, i); + return ret; +} + +int perf_data__open_dir(struct perf_data *data) +{ + struct perf_data_file *files = NULL; + struct dirent *dent; + int ret = -1; + DIR *dir; + int nr = 0; + + /* + * Directory containing a single regular perf data file which is already + * open, means there is nothing more to do here. + */ + if (perf_data__is_single_file(data)) + return 0; + + if (WARN_ON(!data->is_dir)) + return -EINVAL; + + /* The version is provided by DIR_FORMAT feature. */ + if (WARN_ON(data->dir.version != PERF_DIR_VERSION)) + return -1; + + dir = opendir(data->path); + if (!dir) + return -EINVAL; + + while ((dent = readdir(dir)) != NULL) { + struct perf_data_file *file; + char path[PATH_MAX]; + struct stat st; + + snprintf(path, sizeof(path), "%s/%s", data->path, dent->d_name); + if (stat(path, &st)) + continue; + + if (!S_ISREG(st.st_mode) || strncmp(dent->d_name, "data.", 5)) + continue; + + ret = -ENOMEM; + + file = realloc(files, (nr + 1) * sizeof(*files)); + if (!file) + goto out_err; + + files = file; + file = &files[nr++]; + + *file = (struct perf_data_file){ + .path = strdup(path), + .fd = -1, + .size = st.st_size, + .use_stdio = false, + }; + if (!file->path) + goto out_err; + + ret = open(file->path, O_RDONLY); + if (ret < 0) { + ret = -errno; + goto out_err; + } + file->fd = ret; + } + + closedir(dir); + if (!files) + return -EINVAL; + + data->dir.files = files; + data->dir.nr = nr; + return 0; + +out_err: + closedir(dir); + close_dir(files, nr); + return ret; +} + +static bool check_pipe(struct perf_data *data) +{ + struct stat st; + bool is_pipe = false; + int fd = perf_data__is_read(data) ? + STDIN_FILENO : STDOUT_FILENO; + + if (!data->path) { + if (!fstat(fd, &st) && S_ISFIFO(st.st_mode)) + is_pipe = true; + } else { + if (!strcmp(data->path, "-")) + is_pipe = true; + } + + if (is_pipe) { + if (data->file.use_stdio) { + const char *mode; + + mode = perf_data__is_read(data) ? "r" : "w"; + data->file.fptr = fdopen(fd, mode); + + if (data->file.fptr == NULL) { + data->file.fd = fd; + data->file.use_stdio = false; + } + + /* + * When is_pipe and data->file.fd is given, use given fd + * instead of STDIN_FILENO or STDOUT_FILENO + */ + } else if (data->file.fd <= 0) { + data->file.fd = fd; + } + } + + return data->is_pipe = is_pipe; +} + +static int check_backup(struct perf_data *data) +{ + struct stat st; + + if (perf_data__is_read(data)) + return 0; + + if (!stat(data->path, &st) && st.st_size) { + char oldname[PATH_MAX]; + int ret; + + snprintf(oldname, sizeof(oldname), "%s.old", + data->path); + + ret = rm_rf_perf_data(oldname); + if (ret) { + if (ret == -2) + pr_err("Can't remove old data: Unknown file found (%s)\n", oldname); + else + pr_err("Can't remove old data: %m (%s)\n", oldname); + return -1; + } + + if (rename(data->path, oldname)) { + pr_err("Can't move data: %m (%s to %s)\n", data->path, oldname); + return -1; + } + } + + return 0; +} + +static bool is_dir(struct perf_data *data) +{ + struct stat st; + + if (stat(data->path, &st)) + return false; + + return (st.st_mode & S_IFMT) == S_IFDIR; +} + +static int open_file_read(struct perf_data *data) +{ + int flags = data->in_place_update ? O_RDWR : O_RDONLY; + struct stat st; + int fd; + + fd = open(data->file.path, flags); + if (fd < 0) { + int err = errno; + + pr_err("failed to open %s: %m", data->file.path); + if (err == ENOENT && !strcmp(data->file.path, "perf.data")) + pr_err(" (try 'perf record' first)"); + pr_err("\n"); + return -err; + } + + if (fstat(fd, &st) < 0) + goto out_close; + + if (!data->force && st.st_uid && (st.st_uid != geteuid())) { + pr_err("File %s not owned by current user or root (use -f to override)\n", + data->file.path); + goto out_close; + } + + if (!st.st_size) { + pr_info("zero-sized data (%s), nothing to do!\n", + data->file.path); + goto out_close; + } + + data->file.size = st.st_size; + return fd; + + out_close: + close(fd); + return -1; +} + +static int open_file_write(struct perf_data *data) +{ + int fd = open(data->file.path, O_CREAT|O_RDWR|O_TRUNC|O_CLOEXEC, S_IRUSR|S_IWUSR); + + if (fd < 0) + pr_err("failed to open %s : %m\n", data->file.path); + + return fd; +} + +static int open_file(struct perf_data *data) +{ + int fd; + + fd = perf_data__is_read(data) ? + open_file_read(data) : open_file_write(data); + + if (fd < 0) { + zfree(&data->file.path); + return -1; + } + + data->file.fd = fd; + return 0; +} + +static int open_file_dup(struct perf_data *data) +{ + data->file.path = strdup(data->path); + if (!data->file.path) + return -ENOMEM; + + return open_file(data); +} + +static int open_dir(struct perf_data *data) +{ + int ret; + + /* + * So far we open only the header, so we can read the data version and + * layout. + */ + if (asprintf(&data->file.path, "%s/data", data->path) < 0) + return -1; + + if (perf_data__is_write(data) && mkdir(data->path, 0700) < 0) { + zfree(&data->file.path); + return -1; + } + + ret = open_file(data); + + /* Cleanup whatever we managed to create so far. */ + if (ret && perf_data__is_write(data)) + rm_rf_perf_data(data->path); + + return ret; +} + +int perf_data__open(struct perf_data *data) +{ + int ret; + + if (data->open) + return 0; + + if (check_pipe(data)) { + data->open = true; + return 0; + } + + /* currently it allows stdio for pipe only */ + data->file.use_stdio = false; + + if (!data->path) + data->path = "perf.data"; + + if (check_backup(data)) + return -1; + + if (perf_data__is_read(data)) + data->is_dir = is_dir(data); + + ret = perf_data__is_dir(data) ? open_dir(data) : open_file_dup(data); + + if (!ret) + data->open = true; + + return ret; +} + +void perf_data__close(struct perf_data *data) +{ + if (!data->open) + return; + + if (perf_data__is_dir(data)) + perf_data__close_dir(data); + + perf_data_file__close(&data->file); + data->open = false; +} + +static ssize_t perf_data_file__read(struct perf_data_file *file, void *buf, size_t size) +{ + if (file->use_stdio) { + if (fread(buf, size, 1, file->fptr) == 1) + return size; + return feof(file->fptr) ? 0 : -1; + } + return readn(file->fd, buf, size); +} + +ssize_t perf_data__read(struct perf_data *data, void *buf, size_t size) +{ + return perf_data_file__read(&data->file, buf, size); +} + +ssize_t perf_data_file__write(struct perf_data_file *file, + void *buf, size_t size) +{ + if (file->use_stdio) { + if (fwrite(buf, size, /*nmemb=*/1, file->fptr) == 1) + return size; + return -1; + } + return writen(file->fd, buf, size); +} + +ssize_t perf_data__write(struct perf_data *data, + void *buf, size_t size) +{ + return perf_data_file__write(&data->file, buf, size); +} + +off_t perf_data_file__seek(struct perf_data_file *file, off_t offset, int whence) +{ + if (file->use_stdio) { + off_t res = fseeko(file->fptr, offset, whence); + + return res < 0 ? -1 : ftello(file->fptr); + } + return lseek(file->fd, offset, whence); +} + +off_t perf_data__seek(struct perf_data *data, off_t offset, int whence) +{ + /* Note, a pipe fd will fail with -1 with errno of ESPIPE. */ + return perf_data_file__seek(&data->file, offset, whence); +} + +int perf_data__switch(struct perf_data *data, + const char *postfix, + size_t pos, bool at_exit, + char **new_filepath) +{ + int ret; + + if (perf_data__is_read(data)) + return -EINVAL; + + if (asprintf(new_filepath, "%s.%s", data->path, postfix) < 0) + return -ENOMEM; + + /* + * Only fire a warning, don't return error, continue fill + * original file. + */ + if (rename(data->path, *new_filepath)) + pr_warning("Failed to rename %s to %s\n", data->path, *new_filepath); + + if (!at_exit) { + perf_data_file__close(&data->file); + data->open = false; + ret = perf_data__open(data); + if (ret < 0) + goto out; + + if (perf_data__seek(data, pos, SEEK_SET) == (off_t)-1) { + ret = -errno; + pr_debug("Failed to seek to %zu: %m", pos); + goto out; + } + } + ret = perf_data__fd(data); +out: + return ret; +} + +unsigned long perf_data__size(struct perf_data *data) +{ + u64 size = data->file.size; + int i; + + if (perf_data__is_single_file(data)) + return size; + + for (i = 0; i < data->dir.nr; i++) { + struct perf_data_file *file = &data->dir.files[i]; + + size += file->size; + } + + return size; +} + +int perf_data__make_kcore_dir(struct perf_data *data, char *buf, size_t buf_sz) +{ + int ret; + + if (!data->is_dir) + return -1; + + ret = snprintf(buf, buf_sz, "%s/kcore_dir", data->path); + if (ret < 0 || (size_t)ret >= buf_sz) + return -1; + + return mkdir(buf, S_IRWXU); +} + +bool has_kcore_dir(const char *path) +{ + struct dirent *d = ERR_PTR(-EINVAL); + const char *name = "kcore_dir"; + DIR *dir = opendir(path); + size_t n = strlen(name); + bool result = false; + + if (dir) { + while (d && !result) { + d = readdir(dir); + result = d ? strncmp(d->d_name, name, n) : false; + } + closedir(dir); + } + + return result; +} + +char *perf_data__kallsyms_name(struct perf_data *data) +{ + char *kallsyms_name; + struct stat st; + + if (!data->is_dir) + return NULL; + + if (asprintf(&kallsyms_name, "%s/kcore_dir/kallsyms", data->path) < 0) + return NULL; + + if (stat(kallsyms_name, &st)) { + free(kallsyms_name); + return NULL; + } + + return kallsyms_name; +} + +char *perf_data__guest_kallsyms_name(struct perf_data *data, pid_t machine_pid) +{ + char *kallsyms_name; + struct stat st; + + if (!data->is_dir) + return NULL; + + if (asprintf(&kallsyms_name, "%s/kcore_dir__%d/kallsyms", data->path, machine_pid) < 0) + return NULL; + + if (stat(kallsyms_name, &st)) { + free(kallsyms_name); + return NULL; + } + + return kallsyms_name; +} + +bool is_perf_data(const char *path) +{ + bool ret = false; + FILE *file; + u64 magic; + + file = fopen(path, "r"); + if (!file) + return false; + + if (fread(&magic, 1, 8, file) < 8) + goto out; + + ret = is_perf_magic(magic); +out: + fclose(file); + return ret; +} diff --git a/tools/perf/util/data.h b/tools/perf/util/data.h new file mode 100644 index 000000000..76f57f603 --- /dev/null +++ b/tools/perf/util/data.h @@ -0,0 +1,144 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DATA_H +#define __PERF_DATA_H + +#include +#include +#include +#include + +enum perf_data_mode { + PERF_DATA_MODE_WRITE, + PERF_DATA_MODE_READ, +}; + +enum perf_dir_version { + PERF_DIR_SINGLE_FILE = 0, + PERF_DIR_VERSION = 1, +}; + +/** + * struct perf_data_file: A wrapper around a file used for perf.data reading or writing. Generally + * part of struct perf_data. + */ +struct perf_data_file { + /** + * @path: Path of file. Generally a copy of perf_data.path but for a + * directory it is the file within the directory. + */ + char *path; + union { + /** @fd: File descriptor for read/writes. Valid if use_stdio is false. */ + int fd; + /** + * @fptr: Stdio FILE. Valid if use_stdio is true, currently just + * pipes in perf inject. + */ + FILE *fptr; + }; + /** @size: Size of file when opened. */ + unsigned long size; + /** @use_stdio: Use buffered stdio operations. */ + bool use_stdio; +}; + +/** + * struct perf_data: A wrapper around a file used for perf.data reading or writing. + */ +struct perf_data { + /** @path: Path to open and of the file. NULL implies 'perf.data' will be used. */ + const char *path; + /** @file: Underlying file to be used. */ + struct perf_data_file file; + /** @open: Has the file or directory been opened. */ + bool open; + /** @is_pipe: Underlying file is a pipe. */ + bool is_pipe; + /** @is_dir: Underlying file is a directory. */ + bool is_dir; + /** @force: Ignore opening a file creating created by a different user. */ + bool force; + /** @in_place_update: A file opened for reading but will be written to. */ + bool in_place_update; + /** @mode: Read or write mode. */ + enum perf_data_mode mode:8; + + struct { + /** @version: perf_dir_version. */ + u64 version; + /** @files: perf data files for the directory. */ + struct perf_data_file *files; + /** @nr: Number of perf data files for the directory. */ + int nr; + } dir; +}; + +static inline int perf_data_file__fd(struct perf_data_file *file) +{ + return file->use_stdio ? fileno(file->fptr) : file->fd; +} + +ssize_t perf_data_file__write(struct perf_data_file *file, + void *buf, size_t size); +off_t perf_data_file__seek(struct perf_data_file *file, off_t offset, int whence); + + +static inline bool perf_data__is_read(struct perf_data *data) +{ + return data->mode == PERF_DATA_MODE_READ; +} + +static inline bool perf_data__is_write(struct perf_data *data) +{ + return data->mode == PERF_DATA_MODE_WRITE; +} + +static inline int perf_data__is_pipe(struct perf_data *data) +{ + return data->is_pipe; +} + +static inline bool perf_data__is_dir(struct perf_data *data) +{ + return data->is_dir; +} + +static inline bool perf_data__is_single_file(struct perf_data *data) +{ + return data->dir.version == PERF_DIR_SINGLE_FILE; +} + +static inline int perf_data__fd(struct perf_data *data) +{ + return perf_data_file__fd(&data->file); +} + +int perf_data__open(struct perf_data *data); +void perf_data__close(struct perf_data *data); +ssize_t perf_data__read(struct perf_data *data, void *buf, size_t size); +ssize_t perf_data__write(struct perf_data *data, + void *buf, size_t size); +off_t perf_data__seek(struct perf_data *data, off_t offset, int whence); +/* + * If at_exit is set, only rename current perf.data to + * perf.data., continue write on original data. + * Set at_exit when flushing the last output. + * + * Return value is fd of new output. + */ +int perf_data__switch(struct perf_data *data, + const char *postfix, + size_t pos, bool at_exit, char **new_filepath); + +int perf_data__create_dir(struct perf_data *data, int nr); +int perf_data__open_dir(struct perf_data *data); +void perf_data__close_dir(struct perf_data *data); +unsigned long perf_data__size(struct perf_data *data); +int perf_data__make_kcore_dir(struct perf_data *data, char *buf, size_t buf_sz); +char *perf_data__kallsyms_name(struct perf_data *data); +char *perf_data__guest_kallsyms_name(struct perf_data *data, pid_t machine_pid); + +bool has_kcore_dir(const char *path); +bool is_perf_data(const char *path); + +#endif /* __PERF_DATA_H */ diff --git a/tools/perf/util/db-export.c b/tools/perf/util/db-export.c new file mode 100644 index 000000000..ba54b1119 --- /dev/null +++ b/tools/perf/util/db-export.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * db-export.c: Support for exporting data suitable for import to a database + * Copyright (c) 2014, Intel Corporation. + */ + +#include +#include + +#include "dso.h" +#include "evsel.h" +#include "machine.h" +#include "thread.h" +#include "comm.h" +#include "symbol.h" +#include "map.h" +#include "event.h" +#include "thread-stack.h" +#include "callchain.h" +#include "call-path.h" +#include "db-export.h" + +int db_export__init(struct db_export *dbe) +{ + memset(dbe, 0, sizeof(struct db_export)); + return 0; +} + +void db_export__exit(struct db_export *dbe) +{ + call_return_processor__free(dbe->crp); + dbe->crp = NULL; +} + +int db_export__evsel(struct db_export *dbe, struct evsel *evsel) +{ + if (evsel->db_id) + return 0; + + evsel->db_id = ++dbe->evsel_last_db_id; + + if (dbe->export_evsel) + return dbe->export_evsel(dbe, evsel); + + return 0; +} + +int db_export__machine(struct db_export *dbe, struct machine *machine) +{ + if (machine->db_id) + return 0; + + machine->db_id = ++dbe->machine_last_db_id; + + if (dbe->export_machine) + return dbe->export_machine(dbe, machine); + + return 0; +} + +int db_export__thread(struct db_export *dbe, struct thread *thread, + struct machine *machine, struct thread *main_thread) +{ + u64 main_thread_db_id = 0; + + if (thread__db_id(thread)) + return 0; + + thread__set_db_id(thread, ++dbe->thread_last_db_id); + + if (main_thread) + main_thread_db_id = thread__db_id(main_thread); + + if (dbe->export_thread) + return dbe->export_thread(dbe, thread, main_thread_db_id, + machine); + + return 0; +} + +static int __db_export__comm(struct db_export *dbe, struct comm *comm, + struct thread *thread) +{ + comm->db_id = ++dbe->comm_last_db_id; + + if (dbe->export_comm) + return dbe->export_comm(dbe, comm, thread); + + return 0; +} + +int db_export__comm(struct db_export *dbe, struct comm *comm, + struct thread *thread) +{ + if (comm->db_id) + return 0; + + return __db_export__comm(dbe, comm, thread); +} + +/* + * Export the "exec" comm. The "exec" comm is the program / application command + * name at the time it first executes. It is used to group threads for the same + * program. Note that the main thread pid (or thread group id tgid) cannot be + * used because it does not change when a new program is exec'ed. + */ +int db_export__exec_comm(struct db_export *dbe, struct comm *comm, + struct thread *main_thread) +{ + int err; + + if (comm->db_id) + return 0; + + err = __db_export__comm(dbe, comm, main_thread); + if (err) + return err; + + /* + * Record the main thread for this comm. Note that the main thread can + * have many "exec" comms because there will be a new one every time it + * exec's. An "exec" comm however will only ever have 1 main thread. + * That is different to any other threads for that same program because + * exec() will effectively kill them, so the relationship between the + * "exec" comm and non-main threads is 1-to-1. That is why + * db_export__comm_thread() is called here for the main thread, but it + * is called for non-main threads when they are exported. + */ + return db_export__comm_thread(dbe, comm, main_thread); +} + +int db_export__comm_thread(struct db_export *dbe, struct comm *comm, + struct thread *thread) +{ + u64 db_id; + + db_id = ++dbe->comm_thread_last_db_id; + + if (dbe->export_comm_thread) + return dbe->export_comm_thread(dbe, db_id, comm, thread); + + return 0; +} + +int db_export__dso(struct db_export *dbe, struct dso *dso, + struct machine *machine) +{ + if (dso__db_id(dso)) + return 0; + + dso__set_db_id(dso, ++dbe->dso_last_db_id); + + if (dbe->export_dso) + return dbe->export_dso(dbe, dso, machine); + + return 0; +} + +int db_export__symbol(struct db_export *dbe, struct symbol *sym, + struct dso *dso) +{ + u64 *sym_db_id = symbol__priv(sym); + + if (*sym_db_id) + return 0; + + *sym_db_id = ++dbe->symbol_last_db_id; + + if (dbe->export_symbol) + return dbe->export_symbol(dbe, sym, dso); + + return 0; +} + +static int db_ids_from_al(struct db_export *dbe, struct addr_location *al, + u64 *dso_db_id, u64 *sym_db_id, u64 *offset) +{ + int err; + + if (al->map) { + struct dso *dso = map__dso(al->map); + + err = db_export__dso(dbe, dso, maps__machine(thread__maps(al->thread))); + if (err) + return err; + *dso_db_id = dso__db_id(dso); + + if (!al->sym) { + al->sym = symbol__new(al->addr, 0, 0, 0, "unknown"); + if (al->sym) + dso__insert_symbol(dso, al->sym); + } + + if (al->sym) { + u64 *db_id = symbol__priv(al->sym); + + err = db_export__symbol(dbe, al->sym, dso); + if (err) + return err; + *sym_db_id = *db_id; + *offset = al->addr - al->sym->start; + } + } + + return 0; +} + +static struct call_path *call_path_from_sample(struct db_export *dbe, + struct machine *machine, + struct thread *thread, + struct perf_sample *sample) +{ + u64 kernel_start = machine__kernel_start(machine); + struct call_path *current = &dbe->cpr->call_path; + enum chain_order saved_order = callchain_param.order; + struct callchain_cursor *cursor; + int err; + + if (!symbol_conf.use_callchain || !sample->callchain) + return NULL; + + /* + * Since the call path tree must be built starting with the root, we + * must use ORDER_CALL for call chain resolution, in order to process + * the callchain starting with the root node and ending with the leaf. + */ + callchain_param.order = ORDER_CALLER; + cursor = get_tls_callchain_cursor(); + err = thread__resolve_callchain(thread, cursor, + sample, NULL, NULL, PERF_MAX_STACK_DEPTH); + if (err) { + callchain_param.order = saved_order; + return NULL; + } + callchain_cursor_commit(cursor); + + while (1) { + struct callchain_cursor_node *node; + struct addr_location al; + u64 dso_db_id = 0, sym_db_id = 0, offset = 0; + + + node = callchain_cursor_current(cursor); + if (!node) + break; + + /* + * Handle export of symbol and dso for this node by + * constructing an addr_location struct and then passing it to + * db_ids_from_al() to perform the export. + */ + addr_location__init(&al); + al.sym = node->ms.sym; + al.map = map__get(node->ms.map); + al.addr = node->ip; + al.thread = thread__get(thread); + + if (al.map && !al.sym) + al.sym = dso__find_symbol(map__dso(al.map), al.addr); + + db_ids_from_al(dbe, &al, &dso_db_id, &sym_db_id, &offset); + + /* add node to the call path tree if it doesn't exist */ + current = call_path__findnew(dbe->cpr, current, + al.sym, node->ip, + kernel_start); + + callchain_cursor_advance(cursor); + addr_location__exit(&al); + } + + /* Reset the callchain order to its prior value. */ + callchain_param.order = saved_order; + + if (current == &dbe->cpr->call_path) { + /* Bail because the callchain was empty. */ + return NULL; + } + + return current; +} + +int db_export__branch_type(struct db_export *dbe, u32 branch_type, + const char *name) +{ + if (dbe->export_branch_type) + return dbe->export_branch_type(dbe, branch_type, name); + + return 0; +} + +static int db_export__threads(struct db_export *dbe, struct thread *thread, + struct thread *main_thread, + struct machine *machine, struct comm **comm_ptr) +{ + struct comm *comm = NULL; + struct comm *curr_comm; + int err; + + if (main_thread) { + /* + * A thread has a reference to the main thread, so export the + * main thread first. + */ + err = db_export__thread(dbe, main_thread, machine, main_thread); + if (err) + return err; + /* + * Export comm before exporting the non-main thread because + * db_export__comm_thread() can be called further below. + */ + comm = machine__thread_exec_comm(machine, main_thread); + if (comm) { + err = db_export__exec_comm(dbe, comm, main_thread); + if (err) + return err; + *comm_ptr = comm; + } + } + + if (thread != main_thread) { + /* + * For a non-main thread, db_export__comm_thread() must be + * called only if thread has not previously been exported. + */ + bool export_comm_thread = comm && !thread__db_id(thread); + + err = db_export__thread(dbe, thread, machine, main_thread); + if (err) + return err; + + if (export_comm_thread) { + err = db_export__comm_thread(dbe, comm, thread); + if (err) + return err; + } + } + + curr_comm = thread__comm(thread); + if (curr_comm) + return db_export__comm(dbe, curr_comm, thread); + + return 0; +} + +int db_export__sample(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, + struct addr_location *al, struct addr_location *addr_al) +{ + struct thread *thread = al->thread; + struct export_sample es = { + .event = event, + .sample = sample, + .al = al, + }; + struct thread *main_thread; + struct comm *comm = NULL; + struct machine *machine = NULL; + int err; + + if (thread__maps(thread)) + machine = maps__machine(thread__maps(thread)); + if (!machine) + return -1; + + err = db_export__evsel(dbe, sample->evsel); + if (err) + return err; + + err = db_export__machine(dbe, machine); + if (err) + return err; + + main_thread = thread__main_thread(machine, thread); + + err = db_export__threads(dbe, thread, main_thread, machine, &comm); + if (err) + goto out_put; + + if (comm) + es.comm_db_id = comm->db_id; + + es.db_id = ++dbe->sample_last_db_id; + + err = db_ids_from_al(dbe, al, &es.dso_db_id, &es.sym_db_id, &es.offset); + if (err) + goto out_put; + + if (dbe->cpr) { + struct call_path *cp = call_path_from_sample(dbe, machine, + thread, sample); + if (cp) { + db_export__call_path(dbe, cp); + es.call_path_id = cp->db_id; + } + } + + if (addr_al) { + err = db_ids_from_al(dbe, addr_al, &es.addr_dso_db_id, + &es.addr_sym_db_id, &es.addr_offset); + if (err) + goto out_put; + if (dbe->crp) { + err = thread_stack__process(thread, comm, sample, al, + addr_al, es.db_id, + dbe->crp); + if (err) + goto out_put; + } + } + + if (dbe->export_sample) + err = dbe->export_sample(dbe, &es); + +out_put: + thread__put(main_thread); + return err; +} + +static struct { + u32 branch_type; + const char *name; +} branch_types[] = { + {0, "no branch"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL, "call"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN, "return"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CONDITIONAL, "conditional jump"}, + {PERF_IP_FLAG_BRANCH, "unconditional jump"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_INTERRUPT, + "software interrupt"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT, + "return from interrupt"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_SYSCALLRET, + "system call"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET, + "return from system call"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_ASYNC, "asynchronous branch"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | + PERF_IP_FLAG_INTERRUPT, "hardware interrupt"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT, "transaction abort"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_BEGIN, "trace begin"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "trace end"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vm entry"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vm exit"}, + {0, NULL} +}; + +int db_export__branch_types(struct db_export *dbe) +{ + int i, err = 0; + + for (i = 0; branch_types[i].name ; i++) { + err = db_export__branch_type(dbe, branch_types[i].branch_type, + branch_types[i].name); + if (err) + break; + } + + /* Add trace begin / end variants */ + for (i = 0; branch_types[i].name ; i++) { + const char *name = branch_types[i].name; + u32 type = branch_types[i].branch_type; + char buf[64]; + + if (type == PERF_IP_FLAG_BRANCH || + (type & (PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END))) + continue; + + snprintf(buf, sizeof(buf), "trace begin / %s", name); + err = db_export__branch_type(dbe, type | PERF_IP_FLAG_TRACE_BEGIN, buf); + if (err) + break; + + snprintf(buf, sizeof(buf), "%s / trace end", name); + err = db_export__branch_type(dbe, type | PERF_IP_FLAG_TRACE_END, buf); + if (err) + break; + } + + return err; +} + +int db_export__call_path(struct db_export *dbe, struct call_path *cp) +{ + int err; + + if (cp->db_id) + return 0; + + if (cp->parent) { + err = db_export__call_path(dbe, cp->parent); + if (err) + return err; + } + + cp->db_id = ++dbe->call_path_last_db_id; + + if (dbe->export_call_path) + return dbe->export_call_path(dbe, cp); + + return 0; +} + +int db_export__call_return(struct db_export *dbe, struct call_return *cr, + u64 *parent_db_id) +{ + int err; + + err = db_export__call_path(dbe, cr->cp); + if (err) + return err; + + if (!cr->db_id) + cr->db_id = ++dbe->call_return_last_db_id; + + if (parent_db_id) { + if (!*parent_db_id) + *parent_db_id = ++dbe->call_return_last_db_id; + cr->parent_db_id = *parent_db_id; + } + + if (dbe->export_call_return) + return dbe->export_call_return(dbe, cr); + + return 0; +} + +static int db_export__pid_tid(struct db_export *dbe, struct machine *machine, + pid_t pid, pid_t tid, u64 *db_id, + struct comm **comm_ptr, bool *is_idle) +{ + struct thread *thread = machine__find_thread(machine, pid, tid); + struct thread *main_thread; + int err = 0; + + if (!thread || !thread__comm_set(thread)) + goto out_put; + + *is_idle = !thread__pid(thread) && !thread__tid(thread); + + main_thread = thread__main_thread(machine, thread); + + err = db_export__threads(dbe, thread, main_thread, machine, comm_ptr); + + *db_id = thread__db_id(thread); + + thread__put(main_thread); +out_put: + thread__put(thread); + + return err; +} + +int db_export__switch(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, struct machine *machine) +{ + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + bool out_preempt = out && + (event->header.misc & PERF_RECORD_MISC_SWITCH_OUT_PREEMPT); + int flags = out | (out_preempt << 1); + bool is_idle_a = false, is_idle_b = false; + u64 th_a_id = 0, th_b_id = 0; + u64 comm_out_id, comm_in_id; + struct comm *comm_a = NULL; + struct comm *comm_b = NULL; + u64 th_out_id, th_in_id; + u64 db_id; + int err; + + err = db_export__machine(dbe, machine); + if (err) + return err; + + err = db_export__pid_tid(dbe, machine, sample->pid, sample->tid, + &th_a_id, &comm_a, &is_idle_a); + if (err) + return err; + + if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) { + pid_t pid = event->context_switch.next_prev_pid; + pid_t tid = event->context_switch.next_prev_tid; + + err = db_export__pid_tid(dbe, machine, pid, tid, &th_b_id, + &comm_b, &is_idle_b); + if (err) + return err; + } + + /* + * Do not export if both threads are unknown (i.e. not being traced), + * or one is unknown and the other is the idle task. + */ + if ((!th_a_id || is_idle_a) && (!th_b_id || is_idle_b)) + return 0; + + db_id = ++dbe->context_switch_last_db_id; + + if (out) { + th_out_id = th_a_id; + th_in_id = th_b_id; + comm_out_id = comm_a ? comm_a->db_id : 0; + comm_in_id = comm_b ? comm_b->db_id : 0; + } else { + th_out_id = th_b_id; + th_in_id = th_a_id; + comm_out_id = comm_b ? comm_b->db_id : 0; + comm_in_id = comm_a ? comm_a->db_id : 0; + } + + if (dbe->export_context_switch) + return dbe->export_context_switch(dbe, db_id, machine, sample, + th_out_id, comm_out_id, + th_in_id, comm_in_id, flags); + return 0; +} diff --git a/tools/perf/util/db-export.h b/tools/perf/util/db-export.h new file mode 100644 index 000000000..1abbfd398 --- /dev/null +++ b/tools/perf/util/db-export.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * db-export.h: Support for exporting data suitable for import to a database + * Copyright (c) 2014, Intel Corporation. + */ + +#ifndef __PERF_DB_EXPORT_H +#define __PERF_DB_EXPORT_H + +#include +#include + +struct evsel; +struct machine; +struct thread; +struct comm; +struct dso; +struct perf_sample; +struct addr_location; +struct call_return_processor; +struct call_path_root; +struct call_path; +struct call_return; + +struct export_sample { + union perf_event *event; + struct perf_sample *sample; + struct addr_location *al; + u64 db_id; + u64 comm_db_id; + u64 dso_db_id; + u64 sym_db_id; + u64 offset; /* ip offset from symbol start */ + u64 addr_dso_db_id; + u64 addr_sym_db_id; + u64 addr_offset; /* addr offset from symbol start */ + u64 call_path_id; +}; + +struct db_export { + int (*export_evsel)(struct db_export *dbe, struct evsel *evsel); + int (*export_machine)(struct db_export *dbe, struct machine *machine); + int (*export_thread)(struct db_export *dbe, struct thread *thread, + u64 main_thread_db_id, struct machine *machine); + int (*export_comm)(struct db_export *dbe, struct comm *comm, + struct thread *thread); + int (*export_comm_thread)(struct db_export *dbe, u64 db_id, + struct comm *comm, struct thread *thread); + int (*export_dso)(struct db_export *dbe, struct dso *dso, + struct machine *machine); + int (*export_symbol)(struct db_export *dbe, struct symbol *sym, + struct dso *dso); + int (*export_branch_type)(struct db_export *dbe, u32 branch_type, + const char *name); + int (*export_sample)(struct db_export *dbe, struct export_sample *es); + int (*export_call_path)(struct db_export *dbe, struct call_path *cp); + int (*export_call_return)(struct db_export *dbe, + struct call_return *cr); + int (*export_context_switch)(struct db_export *dbe, u64 db_id, + struct machine *machine, + struct perf_sample *sample, + u64 th_out_id, u64 comm_out_id, + u64 th_in_id, u64 comm_in_id, int flags); + struct call_return_processor *crp; + struct call_path_root *cpr; + u64 evsel_last_db_id; + u64 machine_last_db_id; + u64 thread_last_db_id; + u64 comm_last_db_id; + u64 comm_thread_last_db_id; + u64 dso_last_db_id; + u64 symbol_last_db_id; + u64 sample_last_db_id; + u64 call_path_last_db_id; + u64 call_return_last_db_id; + u64 context_switch_last_db_id; +}; + +int db_export__init(struct db_export *dbe); +void db_export__exit(struct db_export *dbe); +int db_export__evsel(struct db_export *dbe, struct evsel *evsel); +int db_export__machine(struct db_export *dbe, struct machine *machine); +int db_export__thread(struct db_export *dbe, struct thread *thread, + struct machine *machine, struct thread *main_thread); +int db_export__comm(struct db_export *dbe, struct comm *comm, + struct thread *thread); +int db_export__exec_comm(struct db_export *dbe, struct comm *comm, + struct thread *main_thread); +int db_export__comm_thread(struct db_export *dbe, struct comm *comm, + struct thread *thread); +int db_export__dso(struct db_export *dbe, struct dso *dso, + struct machine *machine); +int db_export__symbol(struct db_export *dbe, struct symbol *sym, + struct dso *dso); +int db_export__branch_type(struct db_export *dbe, u32 branch_type, + const char *name); +int db_export__sample(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, + struct addr_location *al, struct addr_location *addr_al); + +int db_export__branch_types(struct db_export *dbe); + +int db_export__call_path(struct db_export *dbe, struct call_path *cp); +int db_export__call_return(struct db_export *dbe, struct call_return *cr, + u64 *parent_db_id); +int db_export__switch(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, struct machine *machine); + +#endif diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c new file mode 100644 index 000000000..6b5ffe81f --- /dev/null +++ b/tools/perf/util/debug.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0 +/* For general debugging purposes */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_BACKTRACE_SUPPORT +#include +#endif +#include "addr_location.h" +#include "color.h" +#include "debug.h" +#include "env.h" +#include "event.h" +#include "machine.h" +#include "map.h" +#include "print_binary.h" +#include "srcline.h" +#include "symbol.h" +#include "synthetic-events.h" +#include "target.h" +#include "thread.h" +#include "trace-event.h" +#include "ui/helpline.h" +#include "ui/ui.h" +#include "util/parse-sublevel-options.h" + +#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#else +#define LIBTRACEEVENT_VERSION 0 +#endif + +int verbose; +int debug_kmaps; +int debug_peo_args; +bool dump_trace = false, quiet = false; +int debug_ordered_events; +static int redirect_to_stderr; +int debug_data_convert; +static FILE *_debug_file; +static bool debug_display_time; +int debug_type_profile; + +FILE *debug_file(void) +{ + if (!_debug_file) { + debug_set_file(stderr); + pr_warning_once("debug_file not set"); + } + return _debug_file; +} + +void debug_set_file(FILE *file) +{ + _debug_file = file; +} + +void debug_set_display_time(bool set) +{ + debug_display_time = set; +} + +static int fprintf_time(FILE *file) +{ + struct timeval tod; + struct tm ltime; + char date[64]; + + if (!debug_display_time) + return 0; + + if (gettimeofday(&tod, NULL) != 0) + return 0; + + if (localtime_r(&tod.tv_sec, <ime) == NULL) + return 0; + + strftime(date, sizeof(date), "%F %H:%M:%S", <ime); + return fprintf(file, "[%s.%06lu] ", date, (long)tod.tv_usec); +} + +int veprintf(int level, int var, const char *fmt, va_list args) +{ + int ret = 0; + + if (var >= level) { + if (use_browser >= 1 && !redirect_to_stderr) { + ui_helpline__vshow(fmt, args); + } else { + ret = fprintf_time(debug_file()); + ret += vfprintf(debug_file(), fmt, args); + } + } + + return ret; +} + +int eprintf(int level, int var, const char *fmt, ...) +{ + va_list args; + int ret; + + va_start(args, fmt); + ret = veprintf(level, var, fmt, args); + va_end(args); + + return ret; +} + +static int veprintf_time(u64 t, const char *fmt, va_list args) +{ + int ret = 0; + u64 secs, usecs, nsecs = t; + + secs = nsecs / NSEC_PER_SEC; + nsecs -= secs * NSEC_PER_SEC; + usecs = nsecs / NSEC_PER_USEC; + + ret = fprintf(debug_file(), "[%13" PRIu64 ".%06" PRIu64 "] ", secs, usecs); + ret += vfprintf(debug_file(), fmt, args); + return ret; +} + +int eprintf_time(int level, int var, u64 t, const char *fmt, ...) +{ + int ret = 0; + va_list args; + + if (var >= level) { + va_start(args, fmt); + ret = veprintf_time(t, fmt, args); + va_end(args); + } + + return ret; +} + +/* + * Overloading libtraceevent standard info print + * function, display with -v in perf. + */ +void pr_stat(const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + veprintf(1, verbose, fmt, args); + va_end(args); + eprintf(1, verbose, "\n"); +} + +int dump_printf(const char *fmt, ...) +{ + va_list args; + int ret = 0; + + if (dump_trace) { + va_start(args, fmt); + ret = vprintf(fmt, args); + va_end(args); + } + + return ret; +} + +static int trace_event_printer(enum binary_printer_ops op, + unsigned int val, void *extra, FILE *fp) +{ + const char *color = PERF_COLOR_BLUE; + union perf_event *event = (union perf_event *)extra; + unsigned char ch = (unsigned char)val; + int printed = 0; + + switch (op) { + case BINARY_PRINT_DATA_BEGIN: + printed += fprintf(fp, "."); + printed += color_fprintf(fp, color, "\n. ... raw event: size %d bytes\n", + event->header.size); + break; + case BINARY_PRINT_LINE_BEGIN: + printed += fprintf(fp, "."); + break; + case BINARY_PRINT_ADDR: + printed += color_fprintf(fp, color, " %04x: ", val); + break; + case BINARY_PRINT_NUM_DATA: + printed += color_fprintf(fp, color, " %02x", val); + break; + case BINARY_PRINT_NUM_PAD: + printed += color_fprintf(fp, color, " "); + break; + case BINARY_PRINT_SEP: + printed += color_fprintf(fp, color, " "); + break; + case BINARY_PRINT_CHAR_DATA: + printed += color_fprintf(fp, color, "%c", + isprint(ch) && isascii(ch) ? ch : '.'); + break; + case BINARY_PRINT_CHAR_PAD: + printed += color_fprintf(fp, color, " "); + break; + case BINARY_PRINT_LINE_END: + printed += color_fprintf(fp, color, "\n"); + break; + case BINARY_PRINT_DATA_END: + printed += fprintf(fp, "\n"); + break; + default: + break; + } + + return printed; +} + +void trace_event(union perf_event *event) +{ + unsigned char *raw_event = (void *)event; + + if (!dump_trace) + return; + + print_binary(raw_event, event->header.size, 16, + trace_event_printer, event); +} + +static struct sublevel_option debug_opts[] = { + { .name = "verbose", .value_ptr = &verbose }, + { .name = "ordered-events", .value_ptr = &debug_ordered_events}, + { .name = "stderr", .value_ptr = &redirect_to_stderr}, + { .name = "data-convert", .value_ptr = &debug_data_convert }, + { .name = "perf-event-open", .value_ptr = &debug_peo_args }, + { .name = "kmaps", .value_ptr = &debug_kmaps }, + { .name = "type-profile", .value_ptr = &debug_type_profile }, + { .name = NULL, } +}; + +int perf_debug_option(const char *str) +{ + int ret; + + ret = perf_parse_sublevel_options(str, debug_opts); + if (ret) + return ret; + + /* Allow only verbose value in range (0, 10), otherwise set 0. */ + verbose = (verbose < 0) || (verbose > 10) ? 0 : verbose; + +#if LIBTRACEEVENT_VERSION >= MAKE_LIBTRACEEVENT_VERSION(1, 3, 0) + if (verbose == 1) + tep_set_loglevel(TEP_LOG_INFO); + else if (verbose == 2) + tep_set_loglevel(TEP_LOG_DEBUG); + else if (verbose >= 3) + tep_set_loglevel(TEP_LOG_ALL); +#endif + return 0; +} + +int perf_quiet_option(void) +{ + struct sublevel_option *opt = &debug_opts[0]; + + /* disable all debug messages */ + while (opt->name) { + *opt->value_ptr = -1; + opt++; + } + + /* For debug variables that are used as bool types, set to 0. */ + redirect_to_stderr = 0; + debug_peo_args = 0; + debug_kmaps = 0; + debug_type_profile = 0; + + return 0; +} + +#define DEBUG_WRAPPER(__n, __l) \ +static int pr_ ## __n ## _wrapper(const char *fmt, ...) \ +{ \ + va_list args; \ + int ret; \ + \ + va_start(args, fmt); \ + ret = veprintf(__l, verbose, fmt, args); \ + va_end(args); \ + return ret; \ +} + +DEBUG_WRAPPER(warning, 0); +DEBUG_WRAPPER(debug, 1); + +void perf_debug_setup(void) +{ + debug_set_file(stderr); + libapi_set_print(pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper); +} + +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size) +{ + /* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */ + pid_t pid = getpid(); + struct machine *machine; + struct thread *thread = NULL; + struct perf_env host_env; + + perf_env__init(&host_env); + machine = machine__new_live(&host_env, /*kernel_maps=*/false, pid); + + if (machine) + thread = machine__find_thread(machine, pid, pid); + +#ifdef HAVE_BACKTRACE_SUPPORT + if (!machine || !thread) { + /* + * Backtrace functions are async signal safe. Fall back on them + * if machine/thread creation fails. + */ + backtrace_symbols_fd(stackdump, stackdump_size, fileno(file)); + machine__delete(machine); + perf_env__exit(&host_env); + return; + } +#endif + + for (size_t i = 0; i < stackdump_size; i++) { + struct addr_location al; + u64 addr = (u64)(uintptr_t)stackdump[i]; + bool printed = false; + + addr_location__init(&al); + if (thread && thread__find_map(thread, PERF_RECORD_MISC_USER, addr, &al)) { + al.sym = map__find_symbol(al.map, al.addr); + if (al.sym) { + fprintf(file, " #%zd %p in %s ", i, stackdump[i], al.sym->name); + printed = true; + } + } + if (!printed) + fprintf(file, " #%zd %p ", i, stackdump[i]); + + map__fprintf_srcline(al.map, al.addr, "", file); + fprintf(file, "\n"); + addr_location__exit(&al); + } + thread__put(thread); + machine__delete(machine); + perf_env__exit(&host_env); +} + +/* Obtain a backtrace and print it to stdout. */ +#ifdef HAVE_BACKTRACE_SUPPORT +void dump_stack(void) +{ + void *stackdump[32]; + size_t size = backtrace(stackdump, ARRAY_SIZE(stackdump)); + + __dump_stack(stdout, stackdump, size); +} +#else +void dump_stack(void) {} +#endif + +void sighandler_dump_stack(int sig) +{ + psignal(sig, "perf"); + dump_stack(); + signal(sig, SIG_DFL); + raise(sig); +} diff --git a/tools/perf/util/debug.h b/tools/perf/util/debug.h new file mode 100644 index 000000000..6b737e195 --- /dev/null +++ b/tools/perf/util/debug.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* For debugging general purposes */ +#ifndef __PERF_DEBUG_H +#define __PERF_DEBUG_H + +#include +#include +#include +#include + +extern int verbose; +extern int debug_kmaps; +extern int debug_peo_args; +extern bool quiet, dump_trace; +extern int debug_ordered_events; +extern int debug_data_convert; +extern int debug_type_profile; + +#ifndef pr_fmt +#define pr_fmt(fmt) fmt +#endif + +#define pr_err(fmt, ...) \ + eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warning(fmt, ...) \ + eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warning_once(fmt, ...) ({ \ + static int __warned; \ + if (unlikely(!__warned)) { \ + pr_warning(fmt, ##__VA_ARGS__); \ + __warned = 1; \ + } \ +}) +#define pr_info(fmt, ...) \ + eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug(fmt, ...) \ + eprintf(1, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debugN(n, fmt, ...) \ + eprintf(n, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug2(fmt, ...) pr_debugN(2, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug3(fmt, ...) pr_debugN(3, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug4(fmt, ...) pr_debugN(4, pr_fmt(fmt), ##__VA_ARGS__) + +/* Special macro to print perf_event_open arguments/return value. */ +#define pr_debug2_peo(fmt, ...) { \ + if (debug_peo_args) \ + pr_debugN(0, pr_fmt(fmt), ##__VA_ARGS__); \ + else \ + pr_debugN(2, pr_fmt(fmt), ##__VA_ARGS__); \ +} + +#define pr_time_N(n, var, t, fmt, ...) \ + eprintf_time(n, var, t, fmt, ##__VA_ARGS__) + +#define pr_oe_time(t, fmt, ...) pr_time_N(1, debug_ordered_events, t, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_oe_time2(t, fmt, ...) pr_time_N(2, debug_ordered_events, t, pr_fmt(fmt), ##__VA_ARGS__) + +#define STRERR_BUFSIZE 128 /* For the buffer size of str_error_r */ + +union perf_event; + +int dump_printf(const char *fmt, ...) __printf(1, 2); +void trace_event(union perf_event *event); + +int ui__error(const char *format, ...) __printf(1, 2); +int ui__warning(const char *format, ...) __printf(1, 2); +#define ui__warning_once(format, ...) ({ \ + static int __warned; \ + if (unlikely(!__warned)) { \ + ui__warning(format, ##__VA_ARGS__); \ + __warned = 1; \ + } \ +}) + +void pr_stat(const char *fmt, ...); + +int eprintf(int level, int var, const char *fmt, ...) __printf(3, 4); +int eprintf_time(int level, int var, u64 t, const char *fmt, ...) __printf(4, 5); +int veprintf(int level, int var, const char *fmt, va_list args); + +int perf_debug_option(const char *str); +FILE *debug_file(void); +void debug_set_file(FILE *file); +void debug_set_display_time(bool set); +void perf_debug_setup(void); +int perf_quiet_option(void); + +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size); +void dump_stack(void); +void sighandler_dump_stack(int sig); + +#endif /* __PERF_DEBUG_H */ diff --git a/tools/perf/util/debuginfo.c b/tools/perf/util/debuginfo.c new file mode 100644 index 000000000..84a78b30c --- /dev/null +++ b/tools/perf/util/debuginfo.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DWARF debug information handling code. Copied from probe-finder.c. + * + * Written by Masami Hiramatsu + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "build-id.h" +#include "dso.h" +#include "debug.h" +#include "debuginfo.h" +#include "symbol.h" + +#ifdef HAVE_DEBUGINFOD_SUPPORT +#include +#endif + +/* Dwarf FL wrappers */ +static char *debuginfo_path; /* Currently dummy */ + +static const Dwfl_Callbacks offline_callbacks = { + .find_debuginfo = dwfl_standard_find_debuginfo, + .debuginfo_path = &debuginfo_path, + + .section_address = dwfl_offline_section_address, + + /* We use this table for core files too. */ + .find_elf = dwfl_build_id_find_elf, +}; + +/* Get a Dwarf from offline image */ +static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, + const char *path) +{ + GElf_Addr dummy; + int fd; + bool fd_consumed = false; + + fd = open(path, O_RDONLY); + if (fd < 0) + return fd; + + dbg->dwfl = dwfl_begin(&offline_callbacks); + if (!dbg->dwfl) + goto error; + + dwfl_report_begin(dbg->dwfl); + dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd); + if (!dbg->mod) + goto error; + fd_consumed = true; + + dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias); + if (!dbg->dbg) + goto error; + + dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy); + + if (dwfl_report_end(dbg->dwfl, NULL, NULL) != 0) + goto error; + + return 0; +error: + if (dbg->dwfl) + dwfl_end(dbg->dwfl); + if (!fd_consumed) + close(fd); + memset(dbg, 0, sizeof(*dbg)); + + return -ENOENT; +} + +static struct debuginfo *__debuginfo__new(const char *path) +{ + struct debuginfo *dbg = zalloc(sizeof(*dbg)); + if (!dbg) + return NULL; + + if (debuginfo__init_offline_dwarf(dbg, path) < 0) + zfree(&dbg); + if (dbg) + pr_debug("Open Debuginfo file: %s\n", path); + return dbg; +} + +struct debuginfo *debuginfo__new(const char *path) +{ + static const enum dso_binary_type distro_dwarf_types[] = { + DSO_BINARY_TYPE__FEDORA_DEBUGINFO, + DSO_BINARY_TYPE__UBUNTU_DEBUGINFO, + DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO, + DSO_BINARY_TYPE__BUILDID_DEBUGINFO, + DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO, + DSO_BINARY_TYPE__NOT_FOUND, + }; + const enum dso_binary_type *type; + char buf[PATH_MAX], nil = '\0'; + struct dso *dso; + struct debuginfo *dinfo = NULL; + struct build_id bid = { .size = 0}; + + /* Try to open distro debuginfo files */ + dso = dso__new(path); + if (!dso) + goto out; + + /* + * Set the build id for DSO_BINARY_TYPE__BUILDID_DEBUGINFO. Don't block + * incase the path isn't for a regular file. + */ + assert(!dso__has_build_id(dso)); + if (filename__read_build_id(path, &bid) > 0) + dso__set_build_id(dso, &bid); + + for (type = distro_dwarf_types; + !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND; + type++) { + if (dso__read_binary_type_filename(dso, *type, &nil, + buf, PATH_MAX) < 0) + continue; + dinfo = __debuginfo__new(buf); + } + dso__put(dso); + +out: + if (dinfo) + return dinfo; + + /* if failed to open all distro debuginfo, open given binary */ + symbol__join_symfs(buf, path); + return __debuginfo__new(buf); +} + +void debuginfo__delete(struct debuginfo *dbg) +{ + if (dbg) { + if (dbg->dwfl) + dwfl_end(dbg->dwfl); + free(dbg); + } +} + +/* For the kernel module, we need a special code to get a DIE */ +int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, + bool adjust_offset) +{ + int n, i; + Elf32_Word shndx; + Elf_Scn *scn; + Elf *elf; + GElf_Shdr mem, *shdr; + const char *p; + + elf = dwfl_module_getelf(dbg->mod, &dbg->bias); + if (!elf) + return -EINVAL; + + /* Get the number of relocations */ + n = dwfl_module_relocations(dbg->mod); + if (n < 0) + return -ENOENT; + /* Search the relocation related .text section */ + for (i = 0; i < n; i++) { + p = dwfl_module_relocation_info(dbg->mod, i, &shndx); + if (p && strcmp(p, ".text") == 0) { + /* OK, get the section header */ + scn = elf_getscn(elf, shndx); + if (!scn) + return -ENOENT; + shdr = gelf_getshdr(scn, &mem); + if (!shdr) + return -ENOENT; + *offs = shdr->sh_addr; + if (adjust_offset) + *offs -= shdr->sh_offset; + } + } + return 0; +} + +#ifdef HAVE_DEBUGINFOD_SUPPORT +int get_source_from_debuginfod(const char *raw_path, + const char *sbuild_id, char **new_path) +{ + debuginfod_client *c = debuginfod_begin(); + const char *p = raw_path; + int fd; + + if (!c) + return -ENOMEM; + + fd = debuginfod_find_source(c, (const unsigned char *)sbuild_id, + 0, p, new_path); + pr_debug("Search %s from debuginfod -> %d\n", p, fd); + if (fd >= 0) + close(fd); + debuginfod_end(c); + if (fd < 0) { + pr_debug("Failed to find %s in debuginfod (%s)\n", + raw_path, sbuild_id); + return -ENOENT; + } + pr_debug("Got a source %s\n", *new_path); + + return 0; +} +#endif /* HAVE_DEBUGINFOD_SUPPORT */ diff --git a/tools/perf/util/debuginfo.h b/tools/perf/util/debuginfo.h new file mode 100644 index 000000000..a52d69932 --- /dev/null +++ b/tools/perf/util/debuginfo.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _PERF_DEBUGINFO_H +#define _PERF_DEBUGINFO_H + +#include +#include + +#ifdef HAVE_LIBDW_SUPPORT + +#include "dwarf-aux.h" + +/* debug information structure */ +struct debuginfo { + Dwarf *dbg; + Dwfl_Module *mod; + Dwfl *dwfl; + Dwarf_Addr bias; + const unsigned char *build_id; +}; + +/* This also tries to open distro debuginfo */ +struct debuginfo *debuginfo__new(const char *path); +void debuginfo__delete(struct debuginfo *dbg); + +int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, + bool adjust_offset); + +#else /* HAVE_LIBDW_SUPPORT */ + +/* dummy debug information structure */ +struct debuginfo { +}; + +static inline struct debuginfo *debuginfo__new(const char *path __maybe_unused) +{ + return NULL; +} + +static inline void debuginfo__delete(struct debuginfo *dbg __maybe_unused) +{ +} + +typedef void Dwarf_Addr; + +static inline int debuginfo__get_text_offset(struct debuginfo *dbg __maybe_unused, + Dwarf_Addr *offs __maybe_unused, + bool adjust_offset __maybe_unused) +{ + return -EINVAL; +} + +#endif /* HAVE_LIBDW_SUPPORT */ + +#ifdef HAVE_DEBUGINFOD_SUPPORT +int get_source_from_debuginfod(const char *raw_path, const char *sbuild_id, + char **new_path); +#else /* HAVE_DEBUGINFOD_SUPPORT */ +static inline int get_source_from_debuginfod(const char *raw_path __maybe_unused, + const char *sbuild_id __maybe_unused, + char **new_path __maybe_unused) +{ + return -ENOTSUP; +} +#endif /* HAVE_DEBUGINFOD_SUPPORT */ + +#endif /* _PERF_DEBUGINFO_H */ diff --git a/tools/perf/util/demangle-cxx.cpp b/tools/perf/util/demangle-cxx.cpp new file mode 100644 index 000000000..85b706641 --- /dev/null +++ b/tools/perf/util/demangle-cxx.cpp @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "demangle-cxx.h" +#include +#include +#include + +#ifdef HAVE_LIBBFD_SUPPORT +#define PACKAGE 'perf' +#include +#endif + +#ifdef HAVE_CXA_DEMANGLE_SUPPORT +#include +#endif + +#if defined(HAVE_LIBBFD_SUPPORT) || defined(HAVE_CPLUS_DEMANGLE_SUPPORT) +#ifndef DMGL_PARAMS +#define DMGL_PARAMS (1 << 0) /* Include function args */ +#define DMGL_ANSI (1 << 1) /* Include const, volatile, etc */ +#endif +#endif + +/* + * Demangle C++ function signature + * + * Note: caller is responsible for freeing demangled string + */ +extern "C" +char *cxx_demangle_sym(const char *str, bool params __maybe_unused, + bool modifiers __maybe_unused) +{ +#ifdef HAVE_LIBBFD_SUPPORT + int flags = (params ? DMGL_PARAMS : 0) | (modifiers ? DMGL_ANSI : 0); + + return bfd_demangle(NULL, str, flags); +#elif defined(HAVE_CPLUS_DEMANGLE_SUPPORT) + int flags = (params ? DMGL_PARAMS : 0) | (modifiers ? DMGL_ANSI : 0); + + return cplus_demangle(str, flags); +#elif defined(HAVE_CXA_DEMANGLE_SUPPORT) + char *output; + int status; + + output = abi::__cxa_demangle(str, /*output_buffer=*/NULL, /*length=*/NULL, &status); + return output; +#else + return NULL; +#endif +} diff --git a/tools/perf/util/demangle-cxx.h b/tools/perf/util/demangle-cxx.h new file mode 100644 index 000000000..9359937a8 --- /dev/null +++ b/tools/perf/util/demangle-cxx.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DEMANGLE_CXX +#define __PERF_DEMANGLE_CXX 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +char *cxx_demangle_sym(const char *str, bool params, bool modifiers); + +#ifdef __cplusplus +} +#endif + + +#endif /* __PERF_DEMANGLE_CXX */ diff --git a/tools/perf/util/demangle-java.c b/tools/perf/util/demangle-java.c new file mode 100644 index 000000000..c3cb327ed --- /dev/null +++ b/tools/perf/util/demangle-java.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "symbol.h" + +#include "demangle-java.h" + +#include +#include + +enum { + MODE_PREFIX = 0, + MODE_CLASS = 1, + MODE_FUNC = 2, + MODE_TYPE = 3, + MODE_CTYPE = 4, /* class arg */ +}; + +#define BASE_ENT(c, n) [c - 'A']=n +static const char *base_types['Z' - 'A' + 1] = { + BASE_ENT('B', "byte" ), + BASE_ENT('C', "char" ), + BASE_ENT('D', "double" ), + BASE_ENT('F', "float" ), + BASE_ENT('I', "int" ), + BASE_ENT('J', "long" ), + BASE_ENT('S', "short" ), + BASE_ENT('Z', "boolean" ), +}; + +/* + * demangle Java symbol between str and end positions and stores + * up to maxlen characters into buf. The parser starts in mode. + * + * Use MODE_PREFIX to process entire prototype till end position + * Use MODE_TYPE to process return type if str starts on return type char + * + * Return: + * success: buf + * error : NULL + */ +static char * +__demangle_java_sym(const char *str, const char *end, char *buf, int maxlen, int mode) +{ + int rlen = 0; + int array = 0; + int narg = 0; + const char *q; + + if (!end) + end = str + strlen(str); + + for (q = str; q != end; q++) { + + if (rlen == (maxlen - 1)) + break; + + switch (*q) { + case 'L': + if (mode == MODE_PREFIX || mode == MODE_TYPE) { + if (mode == MODE_TYPE) { + if (narg) + rlen += scnprintf(buf + rlen, maxlen - rlen, ", "); + narg++; + } + if (mode == MODE_PREFIX) + mode = MODE_CLASS; + else + mode = MODE_CTYPE; + } else + buf[rlen++] = *q; + break; + case 'B': + case 'C': + case 'D': + case 'F': + case 'I': + case 'J': + case 'S': + case 'Z': + if (mode == MODE_TYPE) { + if (narg) + rlen += scnprintf(buf + rlen, maxlen - rlen, ", "); + rlen += scnprintf(buf + rlen, maxlen - rlen, "%s", base_types[*q - 'A']); + while (array--) + rlen += scnprintf(buf + rlen, maxlen - rlen, "[]"); + array = 0; + narg++; + } else + buf[rlen++] = *q; + break; + case 'V': + if (mode == MODE_TYPE) { + rlen += scnprintf(buf + rlen, maxlen - rlen, "void"); + while (array--) + rlen += scnprintf(buf + rlen, maxlen - rlen, "[]"); + array = 0; + } else + buf[rlen++] = *q; + break; + case '[': + if (mode != MODE_TYPE) + goto error; + array++; + break; + case '(': + if (mode != MODE_FUNC) + goto error; + buf[rlen++] = *q; + mode = MODE_TYPE; + break; + case ')': + if (mode != MODE_TYPE) + goto error; + buf[rlen++] = *q; + narg = 0; + break; + case ';': + if (mode != MODE_CLASS && mode != MODE_CTYPE) + goto error; + /* safe because at least one other char to process */ + if (isalpha(*(q + 1)) && mode == MODE_CLASS) + rlen += scnprintf(buf + rlen, maxlen - rlen, "."); + if (mode == MODE_CLASS) + mode = MODE_FUNC; + else if (mode == MODE_CTYPE) + mode = MODE_TYPE; + break; + case '/': + if (mode != MODE_CLASS && mode != MODE_CTYPE) + goto error; + rlen += scnprintf(buf + rlen, maxlen - rlen, "."); + break; + default : + buf[rlen++] = *q; + } + } + buf[rlen] = '\0'; + return buf; +error: + return NULL; +} + +/* + * Demangle Java function signature (openJDK, not GCJ) + * input: + * str: string to parse. String is not modified + * flags: combination of JAVA_DEMANGLE_* flags to modify demangling + * return: + * if input can be demangled, then a newly allocated string is returned. + * if input cannot be demangled, then NULL is returned + * + * Note: caller is responsible for freeing demangled string + */ +char * +java_demangle_sym(const char *str, int flags) +{ + char *buf, *ptr; + const char *p; + size_t len, l1 = 0; + + if (!str) + return NULL; + + /* find start of return type */ + p = strrchr(str, ')'); + if (!p) + return NULL; + + /* + * expansion factor estimated to 3x + */ + len = strlen(str) * 3 + 1; + buf = malloc(len); + if (!buf) + return NULL; + + buf[0] = '\0'; + if (!(flags & JAVA_DEMANGLE_NORET)) { + /* + * get return type first + */ + ptr = __demangle_java_sym(p + 1, NULL, buf, len, MODE_TYPE); + if (!ptr) + goto error; + + /* add space between return type and function prototype */ + l1 = strlen(buf); + buf[l1++] = ' '; + } + + /* process function up to return type */ + ptr = __demangle_java_sym(str, p + 1, buf + l1, len - l1, MODE_PREFIX); + if (!ptr) + goto error; + + return buf; +error: + free(buf); + return NULL; +} diff --git a/tools/perf/util/demangle-java.h b/tools/perf/util/demangle-java.h new file mode 100644 index 000000000..f936c8eab --- /dev/null +++ b/tools/perf/util/demangle-java.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DEMANGLE_JAVA +#define __PERF_DEMANGLE_JAVA 1 +/* + * demangle function flags + */ +#define JAVA_DEMANGLE_NORET 0x1 /* do not process return type */ + +char * java_demangle_sym(const char *str, int flags); + +#endif /* __PERF_DEMANGLE_JAVA */ diff --git a/tools/perf/util/demangle-ocaml.c b/tools/perf/util/demangle-ocaml.c new file mode 100644 index 000000000..9d707bb60 --- /dev/null +++ b/tools/perf/util/demangle-ocaml.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "util/string2.h" + +#include "demangle-ocaml.h" + +#include + +static const char *caml_prefix = "caml"; +static const size_t caml_prefix_len = 4; + +/* mangled OCaml symbols start with "caml" followed by an upper-case letter */ +static bool +ocaml_is_mangled(const char *sym) +{ + return 0 == strncmp(sym, caml_prefix, caml_prefix_len) + && isupper(sym[caml_prefix_len]); +} + +/* + * input: + * sym: a symbol which may have been mangled by the OCaml compiler + * return: + * if the input doesn't look like a mangled OCaml symbol, NULL is returned + * otherwise, a newly allocated string containing the demangled symbol is returned + */ +char * +ocaml_demangle_sym(const char *sym) +{ + char *result; + int j = 0; + int i; + int len; + + if (!ocaml_is_mangled(sym)) { + return NULL; + } + + len = strlen(sym); + + /* the demangled symbol is always smaller than the mangled symbol */ + result = malloc(len + 1); + if (!result) + return NULL; + + /* skip "caml" prefix */ + i = caml_prefix_len; + + while (i < len) { + if (sym[i] == '_' && sym[i + 1] == '_') { + /* "__" -> "." */ + result[j++] = '.'; + i += 2; + } + else if (sym[i] == '$' && isxdigit(sym[i + 1]) && isxdigit(sym[i + 2])) { + /* "$xx" is a hex-encoded character */ + result[j++] = (hex(sym[i + 1]) << 4) | hex(sym[i + 2]); + i += 3; + } + else { + result[j++] = sym[i++]; + } + } + result[j] = '\0'; + + return result; +} diff --git a/tools/perf/util/demangle-ocaml.h b/tools/perf/util/demangle-ocaml.h new file mode 100644 index 000000000..843cc4fa1 --- /dev/null +++ b/tools/perf/util/demangle-ocaml.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DEMANGLE_OCAML +#define __PERF_DEMANGLE_OCAML 1 + +char * ocaml_demangle_sym(const char *str); + +#endif /* __PERF_DEMANGLE_OCAML */ diff --git a/tools/perf/util/demangle-rust-v0.c b/tools/perf/util/demangle-rust-v0.c new file mode 100644 index 000000000..19924d854 --- /dev/null +++ b/tools/perf/util/demangle-rust-v0.c @@ -0,0 +1,2042 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The contents of this file come from the Rust rustc-demangle library, hosted +// in the repository, licensed +// under "Apache-2.0 OR MIT". For copyright details, see +// . +// Please note that the file should be kept as close as possible to upstream. + +// Code for demangling Rust symbols. This code is mostly +// a line-by-line translation of the Rust code in `rustc-demangle`. + +// you can find the latest version of this code in https://github.com/rust-lang/rustc-demangle + +#include +#include +#include +#include +#include +#include + +#include "demangle-rust-v0.h" + +#if defined(__GNUC__) || defined(__clang__) +#define NODISCARD __attribute__((warn_unused_result)) +#else +#define NODISCARD +#endif + +#define MAX_DEPTH 500 + +typedef enum { + DemangleOk, + DemangleInvalid, + DemangleRecursed, + DemangleBug, +} demangle_status; + +struct demangle_v0 { + const char *mangled; + size_t mangled_len; +}; + +struct demangle_legacy { + const char *mangled; + size_t mangled_len; + size_t elements; +}; + +// private version of memrchr to avoid _GNU_SOURCE +static void *demangle_memrchr(const void *s, int c, size_t n) { + const uint8_t *s_ = s; + for (; n != 0; n--) { + if (s_[n-1] == c) { + return (void*)&s_[n-1]; + } + } + return NULL; +} + + +static bool unicode_iscontrol(uint32_t ch) { + // this is *technically* a unicode table, but + // some unicode properties are simpler than you might think + return ch < 0x20 || (ch >= 0x7f && ch < 0xa0); +} + +// "good enough" tables, the only consequence is that when printing +// *constant strings*, some characters are printed as `\u{abcd}` rather than themselves. +// +// I'm leaving these here to allow easily replacing them with actual +// tables if desired. +static bool unicode_isprint(uint32_t ch) { + if (ch < 0x20) { + return false; + } + if (ch < 0x7f) { + return true; + } + return false; +} + +static bool unicode_isgraphemextend(uint32_t ch) { + (void)ch; + return false; +} + +static bool str_isascii(const char *s, size_t s_len) { + for (size_t i = 0; i < s_len; i++) { + if (s[i] & 0x80) { + return false; + } + } + + return true; +} + +typedef enum { + PunycodeOk, + PunycodeError +} punycode_status; + +struct parser { + // the parser assumes that `sym` has a safe "terminating byte". It might be NUL, + // but it might also be something else if a symbol is "truncated". + const char *sym; + size_t sym_len; + size_t next; + uint32_t depth; +}; + +struct printer { + demangle_status status; // if status == 0 parser is valid + struct parser parser; + char *out; // NULL for no output [in which case out_len is not decremented] + size_t out_len; + uint32_t bound_lifetime_depth; + bool alternate; +}; + +static NODISCARD overflow_status printer_print_path(struct printer *printer, bool in_value); +static NODISCARD overflow_status printer_print_type(struct printer *printer); +static NODISCARD overflow_status printer_print_const(struct printer *printer, bool in_value); + +static NODISCARD demangle_status try_parse_path(struct parser *parser) { + struct printer printer = { + DemangleOk, + *parser, + NULL, + SIZE_MAX, + 0, + false + }; + overflow_status ignore = printer_print_path(&printer, false); // can't fail since no output + (void)ignore; + *parser = printer.parser; + return printer.status; +} + +NODISCARD static demangle_status rust_demangle_v0_demangle(const char *s, size_t s_len, struct demangle_v0 *res, const char **rest) { + if (s_len > strlen(s)) { + // s_len only exists to shorten the string, this is not a buffer API + return DemangleInvalid; + } + + const char *inner; + size_t inner_len; + if (s_len >= 2 && !strncmp(s, "_R", strlen("_R"))) { + inner = s+2; + inner_len = s_len - 2; + } else if (s_len >= 1 && !strncmp(s, "R", strlen("R"))) { + // On Windows, dbghelp strips leading underscores, so we accept "R..." + // form too. + inner = s+1; + inner_len = s_len - 1; + } else if (s_len >= 3 && !strncmp(s, "__R", strlen("__R"))) { + // On OSX, symbols are prefixed with an extra _ + inner = s+3; + inner_len = s_len - 3; + } else { + return DemangleInvalid; + } + + // Paths always start with uppercase characters. + if (*inner < 'A' || *inner > 'Z') { + return DemangleInvalid; + } + + if (!str_isascii(inner, inner_len)) { + return DemangleInvalid; + } + + struct parser parser = { inner, inner_len, 0, 0 }; + + demangle_status status = try_parse_path(&parser); + if (status != DemangleOk) return status; + char next = parser.sym[parser.next]; + + // Instantiating crate (paths always start with uppercase characters). + if (parser.next < parser.sym_len && next >= 'A' && next <= 'Z') { + status = try_parse_path(&parser); + if (status != DemangleOk) return status; + } + + res->mangled = inner; + res->mangled_len = inner_len; + if (rest) { + *rest = parser.sym + parser.next; + } + + return DemangleOk; +} + +// This might require `len` to be up to 3 characters bigger than the real output len in case of utf-8 +NODISCARD static overflow_status rust_demangle_v0_display_demangle(struct demangle_v0 res, char *out, size_t len, bool alternate) { + struct printer printer = { + DemangleOk, + { + res.mangled, + res.mangled_len, + 0, + 0 + }, + out, + len, + 0, + alternate + }; + if (printer_print_path(&printer, true) == OverflowOverflow) { + return OverflowOverflow; + } + if (printer.out_len < OVERFLOW_MARGIN) { + return OverflowOverflow; + } + *printer.out = '\0'; + return OverflowOk; +} + +static size_t code_to_utf8(unsigned char *buffer, uint32_t code) +{ + if (code <= 0x7F) { + buffer[0] = code; + return 1; + } + if (code <= 0x7FF) { + buffer[0] = 0xC0 | (code >> 6); /* 110xxxxx */ + buffer[1] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 2; + } + if (code <= 0xFFFF) { + buffer[0] = 0xE0 | (code >> 12); /* 1110xxxx */ + buffer[1] = 0x80 | ((code >> 6) & 0x3F); /* 10xxxxxx */ + buffer[2] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 3; + } + if (code <= 0x10FFFF) { + buffer[0] = 0xF0 | (code >> 18); /* 11110xxx */ + buffer[1] = 0x80 | ((code >> 12) & 0x3F); /* 10xxxxxx */ + buffer[2] = 0x80 | ((code >> 6) & 0x3F); /* 10xxxxxx */ + buffer[3] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 4; + } + return 0; +} + + +// return length of char at byte, or SIZE_MAX if invalid. buf should have 4 valid characters +static NODISCARD size_t utf8_next_char(uint8_t *s, uint32_t *ch) { + uint8_t byte = *s; + // UTF8-1 = %x00-7F + // UTF8-2 = %xC2-DF UTF8-tail + // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + // %xF4 %x80-8F 2( UTF8-tail ) + if (byte < 0x80) { + *ch = byte; + return 1; + } else if (byte < 0xc2) { + return SIZE_MAX; + } else if (byte < 0xe0) { + if (s[1] >= 0x80 && s[1] < 0xc0) { + *ch = ((byte&0x1f)<<6) + (s[1] & 0x3f); + return 2; + } + return SIZE_MAX; + } if (byte < 0xf0) { + if (!(s[1] >= 0x80 && s[1] < 0xc0) || !(s[2] >= 0x80 && s[2] < 0xc0)) { + return SIZE_MAX; // basic validation + } + if (byte == 0xe0 && s[1] < 0xa0) { + return SIZE_MAX; // overshort + } + if (byte == 0xed && s[1] >= 0xa0) { + return SIZE_MAX; // surrogate + } + *ch = ((byte&0x0f)<<12) + ((s[1] & 0x3f)<<6) + (s[2] & 0x3f); + return 3; + } else if (byte < 0xf5) { + if (!(s[1] >= 0x80 && s[1] < 0xc0) || !(s[2] >= 0x80 && s[2] < 0xc0) || !(s[3] >= 0x80 && s[3] < 0xc0)) { + return SIZE_MAX; // basic validation + } + if (byte == 0xf0 && s[1] < 0x90) { + return SIZE_MAX; // overshort + } + if (byte == 0xf4 && s[1] >= 0x90) { + return SIZE_MAX; // over max + } + *ch = ((byte&0x07)<<18) + ((s[1] & 0x3f)<<12) + ((s[2] & 0x3f)<<6) + (s[3]&0x3f); + return 4; + } else { + return SIZE_MAX; + } +} + +static NODISCARD bool validate_char(uint32_t n) { + return ((n ^ 0xd800) - 0x800) < 0x110000 - 0x800; +} + +#define SMALL_PUNYCODE_LEN 128 + +static NODISCARD punycode_status punycode_decode(const char *start, size_t ascii_len, const char *punycode_start, size_t punycode_len, uint32_t (*out_)[SMALL_PUNYCODE_LEN], size_t *out_len) { + uint32_t *out = *out_; + + if (punycode_len == 0) { + return PunycodeError; + } + + if (ascii_len > SMALL_PUNYCODE_LEN) { + return PunycodeError; + } + for (size_t i = 0; i < ascii_len; i++) { + out[i] = start[i]; + } + size_t len = ascii_len; + + size_t base = 36, t_min = 1, t_max = 26, skew = 38, damp = 700, bias = 72, i = 0, n = 0x80; + for (;;) { + size_t delta = 0, w = 1, k = 0; + for (;;) { + k += base; + size_t biased = k < bias ? 0 : k - bias; + size_t t = MIN(MAX(biased, t_min), t_max); + size_t d; + if (punycode_len == 0) { + return PunycodeError; + } + char nx = *punycode_start++; + punycode_len--; + if ('a' <= nx && nx <= 'z') { + d = nx - 'a'; + } else if ('0' <= nx && nx <= '9') { + d = 26 + (nx - '0'); + } else { + return PunycodeError; + } + if (w == 0 || d > SIZE_MAX / w || d*w > SIZE_MAX - delta) { + return PunycodeError; + } + delta += d * w; + if (d < t) { + break; + } + if (base < t || w == 0 || (base - t) > SIZE_MAX / w) { + return PunycodeError; + } + w *= (base - t); + } + + len += 1; + if (i > SIZE_MAX - delta) { + return PunycodeError; + } + i += delta; + if (n > SIZE_MAX - i / len) { + return PunycodeError; + } + n += i / len; + i %= len; + + // char validation + if (n > UINT32_MAX || !validate_char((uint32_t)n)) { + return PunycodeError; + } + + // insert new character + if (len > SMALL_PUNYCODE_LEN) { + return PunycodeError; + } + memmove(out + i + 1, out + i, (len - i - 1) * sizeof(uint32_t)); + out[i] = (uint32_t)n; + + // start i index at incremented position + i++; + + // If there are no more deltas, decoding is complete. + if (punycode_len == 0) { + *out_len = len; + return PunycodeOk; + } + + // Perform bias adaptation. + delta /= damp; + damp = 2; + + delta += delta / len; + k = 0; + while (delta > ((base - t_min) * t_max) / 2) { + delta /= base - t_min; + k += base; + } + bias = k + ((base - t_min + 1) * delta) / (delta + skew); + } +} + +struct ident { + const char *ascii_start; + size_t ascii_len; + const char *punycode_start; + size_t punycode_len; +}; + +static NODISCARD overflow_status display_ident(const char *ascii_start, size_t ascii_len, const char *punycode_start, size_t punycode_len, uint8_t *out, size_t *out_len) { + uint32_t outbuf[SMALL_PUNYCODE_LEN]; + + size_t wide_len; + size_t out_buflen = *out_len; + + if (punycode_len == 0) { + if (ascii_len > out_buflen) { + return OverflowOverflow; + } + memcpy(out, ascii_start, ascii_len); + *out_len = ascii_len; + } else if (punycode_decode(ascii_start, ascii_len, punycode_start, punycode_len, &outbuf, &wide_len) == PunycodeOk) { + size_t narrow_len = 0; + for (size_t i = 0; i < wide_len; i++) { + if (out_buflen - narrow_len < 4) { + return OverflowOverflow; + } + unsigned char *pos = &out[narrow_len]; + narrow_len += code_to_utf8(pos, outbuf[i]); + } + *out_len = narrow_len; + } else { + size_t narrow_len = 0; + if (out_buflen < strlen("punycode{")) { + return OverflowOverflow; + } + memcpy(out, "punycode{", strlen("punycode{")); + narrow_len = strlen("punycode{"); + if (ascii_len > 0) { + if (out_buflen - narrow_len < ascii_len || out_buflen - narrow_len - ascii_len < 1) { + return OverflowOverflow; + } + memcpy(out + narrow_len, ascii_start, ascii_len); + narrow_len += ascii_len; + out[narrow_len] = '-'; + narrow_len++; + } + if (out_buflen - narrow_len < punycode_len || out_buflen - narrow_len - punycode_len < 1) { + return OverflowOverflow; + } + memcpy(out + narrow_len, punycode_start, punycode_len); + narrow_len += punycode_len; + out[narrow_len] = '}'; + narrow_len++; + *out_len = narrow_len; + } + + return OverflowOk; +} + +static NODISCARD bool try_parse_uint(const char *buf, size_t len, uint64_t *result) { + size_t cur = 0; + for(;cur < len && buf[cur] == '0';cur++); + uint64_t result_val = 0; + if (len - cur > 16) return false; + for(;cur < len;cur++) { + char c = buf[cur]; + result_val <<= 4; + if ('0' <= c && c <= '9') { + result_val += c - '0'; + } else if ('a' <= c && c <= 'f') { + result_val += 10 + (c - 'a'); + } else { + return false; + } + } + *result = result_val; + return true; +} + +static NODISCARD bool dinibble2int(const char *buf, uint8_t *result) { + uint8_t result_val = 0; + for (int i = 0; i < 2; i++) { + char c = buf[i]; + result_val <<= 4; + if ('0' <= c && c <= '9') { + result_val += c - '0'; + } else if ('a' <= c && c <= 'f') { + result_val += 10 + (c - 'a'); + } else { + return false; + } + } + *result = result_val; + return true; +} + + +typedef enum { + NtsOk = 0, + NtsOverflow = 1, + NtsInvalid = 2 +} nibbles_to_string_status; + +// '\u{10ffff}', +margin +#define ESCAPED_SIZE 12 + +static NODISCARD size_t char_to_string(uint32_t ch, uint8_t quote, bool first, char (*buf)[ESCAPED_SIZE]) { + // encode the character + char *escaped_buf = *buf; + escaped_buf[0] = '\\'; + size_t escaped_len = 2; + switch (ch) { + case '\0': + escaped_buf[1] = '0'; + break; + case '\t': + escaped_buf[1] = 't'; + break; + case '\r': + escaped_buf[1] = 'r'; + break; + case '\n': + escaped_buf[1] = 'n'; + break; + case '\\': + escaped_buf[1] = '\\'; + break; + default: + if (ch == quote) { + escaped_buf[1] = ch; + } else if (!unicode_isprint(ch) || (first && unicode_isgraphemextend(ch))) { + int hexlen = snprintf(escaped_buf, ESCAPED_SIZE, "\\u{%x}", (unsigned int)ch); + if (hexlen < 0) { + return 0; // (snprintf shouldn't fail!) + } + escaped_len = hexlen; + } else { + // printable character + escaped_buf[0] = ch; + escaped_len = 1; + } + break; + } + + return escaped_len; +} + +// convert nibbles to a single/double-quoted string +static NODISCARD nibbles_to_string_status nibbles_to_string(const char *buf, size_t len, uint8_t *out, size_t *out_len) { + uint8_t quote = '"'; + bool first = true; + + if ((len % 2) != 0) { + return NtsInvalid; // odd number of nibbles + } + + size_t cur_out_len = 0; + + // write starting quote + if (out != NULL) { + cur_out_len = *out_len; + if (cur_out_len == 0) { + return NtsOverflow; + } + *out++ = quote; + cur_out_len--; + } + + uint8_t conv_buf[4] = {0}; + size_t conv_buf_len = 0; + while (len > 1 || conv_buf_len > 0) { + while (len > 1 && conv_buf_len < sizeof(conv_buf)) { + if (!dinibble2int(buf, &conv_buf[conv_buf_len])) { + return NtsInvalid; + } + conv_buf_len++; + buf += 2; + len -= 2; + } + + // conv_buf is full here if possible, process 1 UTF-8 character + uint32_t ch = 0; + size_t consumed = utf8_next_char(conv_buf, &ch); + if (consumed > conv_buf_len) { + // either SIZE_MAX (invalid UTF-8) or finished input buffer and + // there are still bytes remaining, in both cases invalid + return NtsInvalid; + } + + // "consume" the character + memmove(conv_buf, conv_buf+consumed, conv_buf_len-consumed); + conv_buf_len -= consumed; + + char escaped_buf[ESCAPED_SIZE]; + size_t escaped_len = char_to_string(ch, '"', first, &escaped_buf); + if (out != NULL) { + if (cur_out_len < escaped_len) { + return NtsOverflow; + } + memcpy(out, escaped_buf, escaped_len); + out += escaped_len; + cur_out_len -= escaped_len; + } + first = false; + } + + // write ending quote + if (out != NULL) { + if (cur_out_len == 0) { + return NtsOverflow; + } + *out++ = quote; + cur_out_len--; + *out_len -= cur_out_len; // subtract remaining space to get used space + } + + return NtsOk; +} + +static const char* basic_type(uint8_t tag) { + switch(tag) { + case 'b': + return "bool"; + case 'c': + return "char"; + case 'e': + return "str"; + case 'u': + return "()"; + case 'a': + return "i8"; + case 's': + return "i16"; + case 'l': + return "i32"; + case 'x': + return "i64"; + case 'n': + return "i128"; + case 'i': + return "isize"; + case 'h': + return "u8"; + case 't': + return "u16"; + case 'm': + return "u32"; + case 'y': + return "u64"; + case 'o': + return "u128"; + case 'j': + return "usize"; + case 'f': + return "f32"; + case 'd': + return "f64"; + case 'z': + return "!"; + case 'p': + return "_"; + case 'v': + return "..."; + default: + return NULL; + } +} + +static NODISCARD demangle_status parser_push_depth(struct parser *parser) { + parser->depth++; + if (parser->depth > MAX_DEPTH) { + return DemangleRecursed; + } else { + return DemangleOk; + } +} + +static demangle_status parser_pop_depth(struct parser *parser) { + parser->depth--; + return DemangleOk; +} + +static uint8_t parser_peek(struct parser const *parser) { + if (parser->next == parser->sym_len) { + return 0; // add a "pseudo nul terminator" to avoid peeking past the end of a symbol + } else { + return parser->sym[parser->next]; + } +} + +static bool parser_eat(struct parser *parser, uint8_t ch) { + if (parser_peek(parser) == ch) { + if (ch != 0) { // safety: make sure we don't skip past the NUL terminator + parser->next++; + } + return true; + } else { + return false; + } +} + +static uint8_t parser_next(struct parser *parser) { + // don't advance after end of input, and return an imaginary NUL terminator + if (parser->next == parser->sym_len) { + return 0; + } else { + return parser->sym[parser->next++]; + } +} + +static NODISCARD demangle_status parser_ch(struct parser *parser, uint8_t *next) { + // don't advance after end of input + if (parser->next == parser->sym_len) { + return DemangleInvalid; + } else { + *next = parser->sym[parser->next++]; + return DemangleOk; + } +} + +struct buf { + const char *start; + size_t len; +}; + +static NODISCARD demangle_status parser_hex_nibbles(struct parser *parser, struct buf *buf) { + size_t start = parser->next; + for (;;) { + uint8_t ch = parser_next(parser); + if (ch == '_') { + break; + } + if (!(('0' <= ch && ch <= '9') || ('a' <= ch && ch <= 'f'))) { + return DemangleInvalid; + } + } + buf->start = parser->sym + start; + buf->len = parser->next - start - 1; // skip final _ + return DemangleOk; +} + +static NODISCARD demangle_status parser_digit_10(struct parser *parser, uint8_t *out) { + uint8_t ch = parser_peek(parser); + if ('0' <= ch && ch <= '9') { + *out = ch - '0'; + parser->next++; + return DemangleOk; + } else { + return DemangleInvalid; + } +} + +static NODISCARD demangle_status parser_digit_62(struct parser *parser, uint64_t *out) { + uint8_t ch = parser_peek(parser); + if ('0' <= ch && ch <= '9') { + *out = ch - '0'; + parser->next++; + return DemangleOk; + } else if ('a' <= ch && ch <= 'z') { + *out = 10 + (ch - 'a'); + parser->next++; + return DemangleOk; + } else if ('A' <= ch && ch <= 'Z') { + *out = 10 + 26 + (ch - 'A'); + parser->next++; + return DemangleOk; + } else { + return DemangleInvalid; + } +} + +static NODISCARD demangle_status parser_integer_62(struct parser *parser, uint64_t *out) { + if (parser_eat(parser, '_')) { + *out = 0; + return DemangleOk; + } + + uint64_t x = 0; + demangle_status status; + while (!parser_eat(parser, '_')) { + uint64_t d; + if ((status = parser_digit_62(parser, &d)) != DemangleOk) { + return status; + } + if (x > UINT64_MAX / 62) { + return DemangleInvalid; + } + x *= 62; + if (x > UINT64_MAX - d) { + return DemangleInvalid; + } + x += d; + } + if (x == UINT64_MAX) { + return DemangleInvalid; + } + *out = x + 1; + return DemangleOk; +} + +static NODISCARD demangle_status parser_opt_integer_62(struct parser *parser, uint8_t tag, uint64_t *out) { + if (!parser_eat(parser, tag)) { + *out = 0; + return DemangleOk; + } + + demangle_status status; + if ((status = parser_integer_62(parser, out)) != DemangleOk) { + return status; + } + if (*out == UINT64_MAX) { + return DemangleInvalid; + } + *out = *out + 1; + return DemangleOk; +} + +static NODISCARD demangle_status parser_disambiguator(struct parser *parser, uint64_t *out) { + return parser_opt_integer_62(parser, 's', out); +} + +typedef uint8_t parser_namespace_type; + +static NODISCARD demangle_status parser_namespace(struct parser *parser, parser_namespace_type *out) { + uint8_t next = parser_next(parser); + if ('A' <= next && next <= 'Z') { + *out = next; + return DemangleOk; + } else if ('a' <= next && next <= 'z') { + *out = 0; + return DemangleOk; + } else { + return DemangleInvalid; + } +} + +static NODISCARD demangle_status parser_backref(struct parser *parser, struct parser *out) { + size_t start = parser->next; + if (start == 0) { + return DemangleBug; + } + size_t s_start = start - 1; + uint64_t i; + demangle_status status = parser_integer_62(parser, &i); + if (status != DemangleOk) { + return status; + } + if (i >= s_start) { + return DemangleInvalid; + } + struct parser res = { + .sym = parser->sym, + .sym_len = parser->sym_len, + .next = (size_t)i, + .depth = parser->depth + }; + status = parser_push_depth(&res); + if (status != DemangleOk) { + return status; + } + *out = res; + return DemangleOk; +} + +static NODISCARD demangle_status parser_ident(struct parser *parser, struct ident *out) { + bool is_punycode = parser_eat(parser, 'u'); + size_t len; + uint8_t d; + demangle_status status = parser_digit_10(parser, &d); + len = d; + if (status != DemangleOk) { + return status; + } + if (len) { + for (;;) { + status = parser_digit_10(parser, &d); + if (status != DemangleOk) { + break; + } + if (len > SIZE_MAX / 10) { + return DemangleInvalid; + } + len *= 10; + if (len > SIZE_MAX - d) { + return DemangleInvalid; + } + len += d; + } + } + + // Skip past the optional `_` separator. + parser_eat(parser, '_'); + + size_t start = parser->next; + if (parser->sym_len - parser->next < len) { + return DemangleInvalid; + } + parser->next += len; + + const char *ident = &parser->sym[start]; + + if (is_punycode) { + const char *underscore = demangle_memrchr(ident, '_', (size_t)len); + if (underscore == NULL) { + *out = (struct ident){ + .ascii_start="", + .ascii_len=0, + .punycode_start=ident, + .punycode_len=len + }; + } else { + size_t ascii_len = underscore - ident; + // ascii_len <= len - 1 since `_` is in the first len bytes + size_t punycode_len = len - 1 - ascii_len; + *out = (struct ident){ + .ascii_start=ident, + .ascii_len=ascii_len, + .punycode_start=underscore + 1, + .punycode_len=punycode_len + }; + } + if (out->punycode_len == 0) { + return DemangleInvalid; + } + return DemangleOk; + } else { + *out = (struct ident) { + .ascii_start=ident, + .ascii_len=(size_t)len, + .punycode_start="", + .punycode_len=0, + }; + return DemangleOk; + } +} + +#define INVALID_SYNTAX "{invalid syntax}" + +static const char *demangle_error_message(demangle_status status) { + switch (status) { + case DemangleInvalid: + return INVALID_SYNTAX; + case DemangleBug: + return "{bug}"; + case DemangleRecursed: + return "{recursion limit reached}"; + default: + return "{unknown error}"; + } +} + +#define PRINT(print_fn) \ + do { \ + if ((print_fn) == OverflowOverflow) { \ + return OverflowOverflow; \ + } \ + } while(0) + +#define PRINT_CH(printer, s) PRINT(printer_print_ch((printer), (s))) +#define PRINT_STR(printer, s) PRINT(printer_print_str((printer), (s))) +#define PRINT_U64(printer, s) PRINT(printer_print_u64((printer), (s))) +#define PRINT_IDENT(printer, s) PRINT(printer_print_ident((printer), (s))) + +#define INVALID(printer) \ + do { \ + PRINT_STR((printer), INVALID_SYNTAX); \ + (printer)->status = DemangleInvalid; \ + return OverflowOk; \ + } while(0) + +#define PARSE(printer, method, ...) \ + do { \ + if ((printer)->status != DemangleOk) { \ + PRINT_STR((printer), "?"); \ + return OverflowOk; \ + } else { \ + demangle_status _parse_status = method(&(printer)->parser, ## __VA_ARGS__); \ + if (_parse_status != DemangleOk) { \ + PRINT_STR((printer), demangle_error_message(_parse_status)); \ + (printer)->status = _parse_status; \ + return OverflowOk; \ + } \ + } \ + } while(0) + +#define PRINT_SEP_LIST(printer, body, sep) \ + do { \ + size_t _sep_list_i; \ + PRINT_SEP_LIST_COUNT(printer, _sep_list_i, body, sep); \ + } while(0) + +#define PRINT_SEP_LIST_COUNT(printer, count, body, sep) \ + do { \ + count = 0; \ + while ((printer)->status == DemangleOk && !printer_eat((printer), 'E')) { \ + if (count > 0) { PRINT_STR(printer, sep); } \ + body; \ + count++; \ + } \ + } while(0) + +static bool printer_eat(struct printer *printer, uint8_t b) { + if (printer->status != DemangleOk) { + return false; + } + + return parser_eat(&printer->parser, b); +} + +static void printer_pop_depth(struct printer *printer) { + if (printer->status == DemangleOk) { + parser_pop_depth(&printer->parser); + } +} + +static NODISCARD overflow_status printer_print_buf(struct printer *printer, const char *start, size_t len) { + if (printer->out == NULL) { + return OverflowOk; + } + if (printer->out_len < len) { + return OverflowOverflow; + } + + memcpy(printer->out, start, len); + printer->out += len; + printer->out_len -= len; + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_str(struct printer *printer, const char *buf) { + return printer_print_buf(printer, buf, strlen(buf)); +} + +static NODISCARD overflow_status printer_print_ch(struct printer *printer, char ch) { + return printer_print_buf(printer, &ch, 1); +} + +static NODISCARD overflow_status printer_print_u64(struct printer *printer, uint64_t n) { + char buf[32] = {0}; + sprintf(buf, "%llu", (unsigned long long)n); // printing uint64 uses 21 < 32 chars + return printer_print_str(printer, buf); +} + +static NODISCARD overflow_status printer_print_ident(struct printer *printer, struct ident *ident) { + if (printer->out == NULL) { + return OverflowOk; + } + + size_t out_len = printer->out_len; + overflow_status status; + if ((status = display_ident(ident->ascii_start, ident->ascii_len, ident->punycode_start, ident->punycode_len, (uint8_t*)printer->out, &out_len)) != OverflowOk) { + return status; + } + printer->out += out_len; + printer->out_len -= out_len; + return OverflowOk; +} + +typedef overflow_status (*printer_fn)(struct printer *printer); +typedef overflow_status (*backref_fn)(struct printer *printer, bool *arg); + +static NODISCARD overflow_status printer_print_backref(struct printer *printer, backref_fn func, bool *arg) { + struct parser backref; + PARSE(printer, parser_backref, &backref); + + if (printer->out == NULL) { + return OverflowOk; + } + + struct parser orig_parser = printer->parser; + demangle_status orig_status = printer->status; // fixme not sure this is needed match for Ok on the Rust side + printer->parser = backref; + printer->status = DemangleOk; + overflow_status status = func(printer, arg); + printer->parser = orig_parser; + printer->status = orig_status; + + return status; +} + +static NODISCARD overflow_status printer_print_lifetime_from_index(struct printer *printer, uint64_t lt) { + // Bound lifetimes aren't tracked when skipping printing. + if (printer->out == NULL) { + return OverflowOk; + } + + PRINT_STR(printer, "'"); + if (lt == 0) { + PRINT_STR(printer, "_"); + return OverflowOk; + } + + if (printer->bound_lifetime_depth < lt) { + INVALID(printer); + } else { + uint64_t depth = printer->bound_lifetime_depth - lt; + if (depth < 26) { + PRINT_CH(printer, 'a' + depth); + } else { + PRINT_STR(printer, "_"); + PRINT_U64(printer, depth); + } + + return OverflowOk; + } +} + +static NODISCARD overflow_status printer_in_binder(struct printer *printer, printer_fn func) { + uint64_t bound_lifetimes; + PARSE(printer, parser_opt_integer_62, 'G', &bound_lifetimes); + + // Don't track bound lifetimes when skipping printing. + if (printer->out == NULL) { + return func(printer); + } + + if (bound_lifetimes > 0) { + PRINT_STR(printer, "for<"); + for (uint64_t i = 0; i < bound_lifetimes; i++) { + if (i > 0) { + PRINT_STR(printer, ", "); + } + printer->bound_lifetime_depth++; + PRINT(printer_print_lifetime_from_index(printer, 1)); + } + PRINT_STR(printer, "> "); + } + + overflow_status r = func(printer); + printer->bound_lifetime_depth -= bound_lifetimes; + + return r; +} + +static NODISCARD overflow_status printer_print_generic_arg(struct printer *printer) { + if (printer_eat(printer, 'L')) { + uint64_t lt; + PARSE(printer, parser_integer_62, <); + return printer_print_lifetime_from_index(printer, lt); + } else if (printer_eat(printer, 'K')) { + return printer_print_const(printer, false); + } else { + return printer_print_type(printer); + } +} + +static NODISCARD overflow_status printer_print_generic_args(struct printer *printer) { + PRINT_STR(printer, "<"); + PRINT_SEP_LIST(printer, PRINT(printer_print_generic_arg(printer)), ", "); + PRINT_STR(printer, ">"); + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_path_out_of_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_path(printer, false); +} + +static NODISCARD overflow_status printer_print_path_in_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_path(printer, true); +} + +static NODISCARD overflow_status printer_print_path(struct printer *printer, bool in_value) { + PARSE(printer, parser_push_depth); + uint8_t tag; + PARSE(printer, parser_ch, &tag); + + overflow_status st; + uint64_t dis; + struct ident name; + parser_namespace_type ns; + char *orig_out; + + switch(tag) { + case 'C': + PARSE(printer, parser_disambiguator, &dis); + PARSE(printer, parser_ident, &name); + + PRINT_IDENT(printer, &name); + + if (printer->out != NULL && !printer->alternate && dis != 0) { + PRINT_STR(printer, "["); + char buf[24] = {0}; + sprintf(buf, "%llx", (unsigned long long)dis); + PRINT_STR(printer, buf); + PRINT_STR(printer, "]"); + } + break; + case 'N': + PARSE(printer, parser_namespace, &ns); + if ((st = printer_print_path(printer, in_value)) != OverflowOk) { + return st; + } + + // HACK(eddyb) if the parser is already marked as having errored, + // `parse!` below will print a `?` without its preceding `::` + // (because printing the `::` is skipped in certain conditions, + // i.e. a lowercase namespace with an empty identifier), + // so in order to get `::?`, the `::` has to be printed here. + if (printer->status != DemangleOk) { + PRINT_STR(printer, "::"); + } + + PARSE(printer, parser_disambiguator, &dis); + PARSE(printer, parser_ident, &name); + // Special namespace, like closures and shims + if (ns) { + PRINT_STR(printer, "::{"); + if (ns == 'C') { + PRINT_STR(printer, "closure"); + } else if (ns == 'S') { + PRINT_STR(printer, "shim"); + } else { + PRINT_CH(printer, ns); + } + if (name.ascii_len != 0 || name.punycode_len != 0) { + PRINT_STR(printer, ":"); + PRINT_IDENT(printer, &name); + } + PRINT_STR(printer, "#"); + PRINT_U64(printer, dis); + PRINT_STR(printer, "}"); + } else { + // Implementation-specific/unspecified namespaces + if (name.ascii_len != 0 || name.punycode_len != 0) { + PRINT_STR(printer, "::"); + PRINT_IDENT(printer, &name); + } + } + break; + case 'M': + case 'X': + // for impls, ignore the impls own path + PARSE(printer, parser_disambiguator, &dis); + orig_out = printer->out; + printer->out = NULL; + PRINT(printer_print_path(printer, false)); + printer->out = orig_out; + + // fallthru + case 'Y': + PRINT_STR(printer, "<"); + PRINT(printer_print_type(printer)); + if (tag != 'M') { + PRINT_STR(printer, " as "); + PRINT(printer_print_path(printer, false)); + } + PRINT_STR(printer, ">"); + break; + case 'I': + PRINT(printer_print_path(printer, in_value)); + if (in_value) { + PRINT_STR(printer, "::"); + } + PRINT(printer_print_generic_args(printer)); + break; + case 'B': + PRINT(printer_print_backref(printer, in_value ? printer_print_path_in_value : printer_print_path_out_of_value, NULL)); + break; + default: + INVALID(printer); + break; + } + + printer_pop_depth(printer); + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_const_uint(struct printer *printer, uint8_t tag) { + struct buf hex; + PARSE(printer, parser_hex_nibbles, &hex); + + uint64_t val; + if (try_parse_uint(hex.start, hex.len, &val)) { + PRINT_U64(printer, val); + } else { + PRINT_STR(printer, "0x"); + PRINT(printer_print_buf(printer, hex.start, hex.len)); + } + + if (printer->out != NULL && !printer->alternate) { + const char *ty = basic_type(tag); + if (/* safety */ ty != NULL) { + PRINT_STR(printer, ty); + } + } + + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_const_str_literal(struct printer *printer) { + struct buf hex; + PARSE(printer, parser_hex_nibbles, &hex); + + size_t out_len = SIZE_MAX; + nibbles_to_string_status nts_status = nibbles_to_string(hex.start, hex.len, NULL, &out_len); + switch (nts_status) { + case NtsOk: + if (printer->out != NULL) { + out_len = printer->out_len; + nts_status = nibbles_to_string(hex.start, hex.len, (uint8_t*)printer->out, &out_len); + if (nts_status != NtsOk) { + return OverflowOverflow; + } + printer->out += out_len; + printer->out_len -= out_len; + } + return OverflowOk; + case NtsOverflow: + // technically if there is a string of size `SIZE_MAX/6` whose escaped version overflows + // SIZE_MAX but has an invalid char, this will be a "fake" overflow. In practice, + // that is not going to happen and a fuzzer will not generate strings of this length. + return OverflowOverflow; + case NtsInvalid: + default: + INVALID(printer); + } +} + +static NODISCARD overflow_status printer_print_const_struct(struct printer *printer) { + uint64_t dis; + struct ident name; + PARSE(printer, parser_disambiguator, &dis); + PARSE(printer, parser_ident, &name); + PRINT_IDENT(printer, &name); + PRINT_STR(printer, ": "); + return printer_print_const(printer, true); +} + +static NODISCARD overflow_status printer_print_const_out_of_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_const(printer, false); +} + +static NODISCARD overflow_status printer_print_const_in_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_const(printer, true); +} + +static NODISCARD overflow_status printer_print_const(struct printer *printer, bool in_value) { + uint8_t tag; + + PARSE(printer, parser_ch, &tag); + PARSE(printer, parser_push_depth); + + struct buf hex; + uint64_t val; + size_t count; + + bool opened_brace = false; +#define OPEN_BRACE_IF_OUTSIDE_EXPR \ + do { if (!in_value) { \ + opened_brace = true; \ + PRINT_STR(printer, "{"); \ + } } while(0) + + switch(tag) { + case 'p': + PRINT_STR(printer, "_"); + break; + // Primitive leaves with hex-encoded values (see `basic_type`). + case 'a': + case 's': + case 'l': + case 'x': + case 'n': + case 'i': + if (printer_eat(printer, 'n')) { + PRINT_STR(printer, "-"); + } + /* fallthrough */ + case 'h': + case 't': + case 'm': + case 'y': + case 'o': + case 'j': + PRINT(printer_print_const_uint(printer, tag)); + break; + case 'b': + PARSE(printer, parser_hex_nibbles, &hex); + if (try_parse_uint(hex.start, hex.len, &val)) { + if (val == 0) { + PRINT_STR(printer, "false"); + } else if (val == 1) { + PRINT_STR(printer, "true"); + } else { + INVALID(printer); + } + } else { + INVALID(printer); + } + break; + case 'c': + PARSE(printer, parser_hex_nibbles, &hex); + if (try_parse_uint(hex.start, hex.len, &val) + && val < UINT32_MAX + && validate_char((uint32_t)val)) + { + char escaped_buf[ESCAPED_SIZE]; + size_t escaped_size = char_to_string((uint32_t)val, '\'', true, &escaped_buf); + + PRINT_STR(printer, "'"); + PRINT(printer_print_buf(printer, escaped_buf, escaped_size)); + PRINT_STR(printer, "'"); + } else { + INVALID(printer); + } + break; + case 'e': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "*"); + PRINT(printer_print_const_str_literal(printer)); + break; + case 'R': + case 'Q': + if (tag == 'R' && printer_eat(printer, 'e')) { + PRINT(printer_print_const_str_literal(printer)); + } else { + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "&"); + if (tag != 'R') { + PRINT_STR(printer, "mut "); + } + PRINT(printer_print_const(printer, true)); + } + break; + case 'A': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "["); + PRINT_SEP_LIST(printer, PRINT(printer_print_const(printer, true)), ", "); + PRINT_STR(printer, "]"); + break; + case 'T': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "("); + PRINT_SEP_LIST_COUNT(printer, count, PRINT(printer_print_const(printer, true)), ", "); + if (count == 1) { + PRINT_STR(printer, ","); + } + PRINT_STR(printer, ")"); + break; + case 'V': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT(printer_print_path(printer, true)); + PARSE(printer, parser_ch, &tag); + switch(tag) { + case 'U': + break; + case 'T': + PRINT_STR(printer, "("); + PRINT_SEP_LIST(printer, PRINT(printer_print_const(printer, true)), ", "); + PRINT_STR(printer, ")"); + break; + case 'S': + PRINT_STR(printer, " { "); + PRINT_SEP_LIST(printer, PRINT(printer_print_const_struct(printer)), ", "); + PRINT_STR(printer, " }"); + break; + default: + INVALID(printer); + } + break; + case 'B': + PRINT(printer_print_backref(printer, in_value ? printer_print_const_in_value : printer_print_const_out_of_value, NULL)); + break; + default: + INVALID(printer); + } +#undef OPEN_BRACE_IF_OUTSIDE_EXPR + + if (opened_brace) { + PRINT_STR(printer, "}"); + } + printer_pop_depth(printer); + + return OverflowOk; +} + +/// A trait in a trait object may have some "existential projections" +/// (i.e. associated type bindings) after it, which should be printed +/// in the `<...>` of the trait, e.g. `dyn Trait`. +/// To this end, this method will keep the `<...>` of an 'I' path +/// open, by omitting the `>`, and return `Ok(true)` in that case. +static NODISCARD overflow_status printer_print_maybe_open_generics(struct printer *printer, bool *open) { + if (printer_eat(printer, 'B')) { + // NOTE(eddyb) the closure may not run if printing is being skipped, + // but in that case the returned boolean doesn't matter. + *open = false; + return printer_print_backref(printer, printer_print_maybe_open_generics, open); + } else if(printer_eat(printer, 'I')) { + PRINT(printer_print_path(printer, false)); + PRINT_STR(printer, "<"); + PRINT_SEP_LIST(printer, PRINT(printer_print_generic_arg(printer)), ", "); + *open = true; + return OverflowOk; + } else { + PRINT(printer_print_path(printer, false)); + *open = false; + return OverflowOk; + } +} + +static NODISCARD overflow_status printer_print_dyn_trait(struct printer *printer) { + bool open; + PRINT(printer_print_maybe_open_generics(printer, &open)); + + while (printer_eat(printer, 'p')) { + if (!open) { + PRINT_STR(printer, "<"); + open = true; + } else { + PRINT_STR(printer, ", "); + } + + struct ident name; + PARSE(printer, parser_ident, &name); + + PRINT_IDENT(printer, &name); + PRINT_STR(printer, " = "); + PRINT(printer_print_type(printer)); + } + + if (open) { + PRINT_STR(printer, ">"); + } + + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_object_bounds(struct printer *printer) { + PRINT_SEP_LIST(printer, PRINT(printer_print_dyn_trait(printer)), " + "); + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_function_type(struct printer *printer) { + bool is_unsafe = printer_eat(printer, 'U'); + const char *abi; + size_t abi_len; + if (printer_eat(printer, 'K')) { + if (printer_eat(printer, 'C')) { + abi = "C"; + abi_len = 1; + } else { + struct ident abi_ident; + PARSE(printer, parser_ident, &abi_ident); + if (abi_ident.ascii_len == 0 || abi_ident.punycode_len != 0) { + INVALID(printer); + } + abi = abi_ident.ascii_start; + abi_len = abi_ident.ascii_len; + } + } else { + abi = NULL; + abi_len = 0; + } + + if (is_unsafe) { + PRINT_STR(printer, "unsafe "); + } + + if (abi != NULL) { + PRINT_STR(printer, "extern \""); + + // replace _ with - + while (abi_len > 0) { + const char *minus = memchr(abi, '_', abi_len); + if (minus == NULL) { + PRINT(printer_print_buf(printer, (const char*)abi, abi_len)); + break; + } else { + size_t space_to_minus = minus - abi; + PRINT(printer_print_buf(printer, (const char*)abi, space_to_minus)); + PRINT_STR(printer, "-"); + abi = minus + 1; + abi_len -= (space_to_minus + 1); + } + } + + PRINT_STR(printer, "\" "); + } + + PRINT_STR(printer, "fn("); + PRINT_SEP_LIST(printer, PRINT(printer_print_type(printer)), ", "); + PRINT_STR(printer, ")"); + + if (printer_eat(printer, 'u')) { + // Skip printing the return type if it's 'u', i.e. `()`. + } else { + PRINT_STR(printer, " -> "); + PRINT(printer_print_type(printer)); + } + + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_type_backref(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_type(printer); +} + +static NODISCARD overflow_status printer_print_type(struct printer *printer) { + uint8_t tag; + PARSE(printer, parser_ch, &tag); + + const char *basic_ty = basic_type(tag); + if (basic_ty) { + return printer_print_str(printer, basic_ty); + } + + uint64_t count; + uint64_t lt; + + PARSE(printer, parser_push_depth); + switch (tag) { + case 'R': + case 'Q': + PRINT_STR(printer, "&"); + if (printer_eat(printer, 'L')) { + PARSE(printer, parser_integer_62, <); + if (lt != 0) { + PRINT(printer_print_lifetime_from_index(printer, lt)); + PRINT_STR(printer, " "); + } + } + if (tag != 'R') { + PRINT_STR(printer, "mut "); + } + PRINT(printer_print_type(printer)); + break; + case 'P': + case 'O': + PRINT_STR(printer, "*"); + if (tag != 'P') { + PRINT_STR(printer, "mut "); + } else { + PRINT_STR(printer, "const "); + } + PRINT(printer_print_type(printer)); + break; + case 'A': + case 'S': + PRINT_STR(printer, "["); + PRINT(printer_print_type(printer)); + if (tag == 'A') { + PRINT_STR(printer, "; "); + PRINT(printer_print_const(printer, true)); + } + PRINT_STR(printer, "]"); + break; + case 'T': + PRINT_STR(printer, "("); + PRINT_SEP_LIST_COUNT(printer, count, PRINT(printer_print_type(printer)), ", "); + if (count == 1) { + PRINT_STR(printer, ","); + } + PRINT_STR(printer, ")"); + break; + case 'F': + PRINT(printer_in_binder(printer, printer_print_function_type)); + break; + case 'D': + PRINT_STR(printer, "dyn "); + PRINT(printer_in_binder(printer, printer_print_object_bounds)); + + if (!printer_eat(printer, 'L')) { + INVALID(printer); + } + PARSE(printer, parser_integer_62, <); + + if (lt != 0) { + PRINT_STR(printer, " + "); + PRINT(printer_print_lifetime_from_index(printer, lt)); + } + break; + case 'B': + PRINT(printer_print_backref(printer, printer_print_type_backref, NULL)); + break; + default: + // Go back to the tag, so `print_path` also sees it. + if (printer->status == DemangleOk && /* safety */ printer->parser.next > 0) { + printer->parser.next--; + } + PRINT(printer_print_path(printer, false)); + } + + printer_pop_depth(printer); + return OverflowOk; +} + +NODISCARD static demangle_status rust_demangle_legacy_demangle(const char *s, size_t s_len, struct demangle_legacy *res, const char **rest) +{ + if (s_len > strlen(s)) { + // s_len only exists to shorten the string, this is not a buffer API + return DemangleInvalid; + } + + const char *inner; + size_t inner_len; + if (s_len >= 3 && !strncmp(s, "_ZN", 3)) { + inner = s + 3; + inner_len = s_len - 3; + } else if (s_len >= 2 && !strncmp(s, "ZN", 2)) { + // On Windows, dbghelp strips leading underscores, so we accept "ZN...E" + // form too. + inner = s + 2; + inner_len = s_len - 2; + } else if (s_len >= 4 && !strncmp(s, "__ZN", 4)) { + // On OSX, symbols are prefixed with an extra _ + inner = s + 4; + inner_len = s_len - 4; + } else { + return DemangleInvalid; + } + + if (!str_isascii(inner, inner_len)) { + return DemangleInvalid; + } + + size_t elements = 0; + const char *chars = inner; + size_t chars_len = inner_len; + if (chars_len == 0) { + return DemangleInvalid; + } + char c; + while ((c = *chars) != 'E') { + // Decode an identifier element's length + if (c < '0' || c > '9') { + return DemangleInvalid; + } + size_t len = 0; + while (c >= '0' && c <= '9') { + size_t d = c - '0'; + if (len > SIZE_MAX / 10) { + return DemangleInvalid; + } + len *= 10; + if (len > SIZE_MAX - d) { + return DemangleInvalid; + } + len += d; + + chars++; + chars_len--; + if (chars_len == 0) { + return DemangleInvalid; + } + c = *chars; + } + + // Advance by the length + if (chars_len <= len) { + return DemangleInvalid; + } + chars += len; + chars_len -= len; + elements++; + } + *res = (struct demangle_legacy) { inner, inner_len, elements }; + *rest = chars + 1; + return DemangleOk; +} + +static bool is_rust_hash(const char *s, size_t len) { + if (len == 0 || s[0] != 'h') { + return false; + } + + for (size_t i = 1; i < len; i++) { + if (!((s[i] >= '0' && s[i] <= '9') || (s[i] >= 'a' && s[i] <= 'f') || (s[i] >= 'A' && s[i] <= 'F'))) { + return false; + } + } + + return true; +} + +NODISCARD static overflow_status rust_demangle_legacy_display_demangle(struct demangle_legacy res, char *out, size_t len, bool alternate) +{ + struct printer printer = { + // not actually using the parser part of the printer, just keeping it to share the format functions + DemangleOk, + { NULL }, + out, + len, + 0, + alternate + }; + const char *inner = res.mangled; + for (size_t element = 0; element < res.elements; element++) { + size_t i = 0; + const char *rest; + for (rest = inner; rest < res.mangled + res.mangled_len && *rest >= '0' && *rest <= '9'; rest++) { + i *= 10; + i += *rest - '0'; + } + if ((size_t)(res.mangled + res.mangled_len - rest) < i) { + // safety: shouldn't reach this place if the input string is validated. bail out. + // safety: we knwo rest <= res.mangled + res.mangled_len from the for-loop above + break; + } + + size_t len = i; + inner = rest + len; + + // From here on, inner contains a pointer to the next element, rest[:len] to the current one + if (alternate && element + 1 == res.elements && is_rust_hash(rest, i)) { + break; + } + if (element != 0) { + PRINT_STR(&printer, "::"); + } + + if (len >= 2 && !strncmp(rest, "_$", 2)) { + rest++; + len--; + } + + while (len > 0) { + if (rest[0] == '.') { + if (len >= 2 && rest[1] == '.') { + PRINT_STR(&printer, "::"); + rest += 2; + len -= 2; + } else { + PRINT_STR(&printer, "."); + rest += 1; + len -= 1; + } + } else if (rest[0] == '$') { + const char *escape = memchr(rest + 1, '$', len - 1); + if (escape == NULL) { + break; + } + const char *escape_start = rest + 1; + size_t escape_len = escape - (rest + 1); + + size_t next_len = len - (escape + 1 - rest); + const char *next_rest = escape + 1; + + char ch; + if ((escape_len == 2 && escape_start[0] == 'S' && escape_start[1] == 'P')) { + ch = '@'; + } else if ((escape_len == 2 && escape_start[0] == 'B' && escape_start[1] == 'P')) { + ch = '*'; + } else if ((escape_len == 2 && escape_start[0] == 'R' && escape_start[1] == 'F')) { + ch = '&'; + } else if ((escape_len == 2 && escape_start[0] == 'L' && escape_start[1] == 'T')) { + ch = '<'; + } else if ((escape_len == 2 && escape_start[0] == 'G' && escape_start[1] == 'T')) { + ch = '>'; + } else if ((escape_len == 2 && escape_start[0] == 'L' && escape_start[1] == 'P')) { + ch = '('; + } else if ((escape_len == 2 && escape_start[0] == 'R' && escape_start[1] == 'P')) { + ch = ')'; + } else if ((escape_len == 1 && escape_start[0] == 'C')) { + ch = ','; + } else { + if (escape_len > 1 && escape_start[0] == 'u') { + escape_start++; + escape_len--; + uint64_t val; + if (try_parse_uint(escape_start, escape_len, &val) + && val < UINT32_MAX + && validate_char((uint32_t)val)) + { + if (!unicode_iscontrol(val)) { + uint8_t wchr[4]; + size_t wchr_len = code_to_utf8(wchr, (uint32_t)val); + PRINT(printer_print_buf(&printer, (const char*)wchr, wchr_len)); + len = next_len; + rest = next_rest; + continue; + } + } + } + break; // print the rest of this element raw + } + PRINT_CH(&printer, ch); + len = next_len; + rest = next_rest; + } else { + size_t j = 0; + for (;j < len && rest[j] != '$' && rest[j] != '.';j++); + if (j == len) { + break; + } + PRINT(printer_print_buf(&printer, rest, j)); + rest += j; + len -= j; + } + } + PRINT(printer_print_buf(&printer, rest, len)); + } + + if (printer.out_len < OVERFLOW_MARGIN) { + return OverflowOverflow; + } + *printer.out = '\0'; + return OverflowOk; +} + +static bool is_symbol_like(const char *s, size_t len) { + // rust-demangle definition of symbol like: control characters and space are not symbol-like, all else is + for (size_t i = 0; i < len; i++) { + char ch = s[i]; + if (!(ch >= 0x21 && ch <= 0x7e)) { + return false; + } + } + return true; +} + +void rust_demangle_demangle(const char *s, struct demangle *res) +{ + // During ThinLTO LLVM may import and rename internal symbols, so strip out + // those endings first as they're one of the last manglings applied to symbol + // names. + const char *llvm = ".llvm."; + const char *found_llvm = strstr(s, llvm); + size_t s_len = strlen(s); + if (found_llvm) { + const char *all_hex_ptr = found_llvm + strlen(".llvm."); + bool all_hex = true; + for (;*all_hex_ptr;all_hex_ptr++) { + if (!(('0' <= *all_hex_ptr && *all_hex_ptr <= '9') || + ('A' <= *all_hex_ptr && *all_hex_ptr <= 'F') || + *all_hex_ptr == '@')) { + all_hex = false; + break; + } + } + + if (all_hex) { + s_len = found_llvm - s; + } + } + + const char *suffix; + struct demangle_legacy legacy; + demangle_status st = rust_demangle_legacy_demangle(s, s_len, &legacy, &suffix); + if (st == DemangleOk) { + *res = (struct demangle) { + .style=DemangleStyleLegacy, + .mangled=legacy.mangled, + .mangled_len=legacy.mangled_len, + .elements=legacy.elements, + .original=s, + .original_len=s_len, + .suffix=suffix, + .suffix_len=s_len - (suffix - s), + }; + } else { + struct demangle_v0 v0; + st = rust_demangle_v0_demangle(s, s_len, &v0, &suffix); + if (st == DemangleOk) { + *res = (struct demangle) { + .style=DemangleStyleV0, + .mangled=v0.mangled, + .mangled_len=v0.mangled_len, + .elements=0, + .original=s, + .original_len=s_len, + .suffix=suffix, + .suffix_len=s_len - (suffix - s), + }; + } else { + *res = (struct demangle) { + .style=DemangleStyleUnknown, + .mangled=NULL, + .mangled_len=0, + .elements=0, + .original=s, + .original_len=s_len, + .suffix=s, + .suffix_len=0, + }; + } + } + + // Output like LLVM IR adds extra period-delimited words. See if + // we are in that case and save the trailing words if so. + if (res->suffix_len) { + if (res->suffix[0] == '.' && is_symbol_like(res->suffix, res->suffix_len)) { + // Keep the suffix + } else { + // Reset the suffix and invalidate the demangling + res->style = DemangleStyleUnknown; + res->suffix_len = 0; + } + } +} + +bool rust_demangle_is_known(struct demangle *res) { + return res->style != DemangleStyleUnknown; +} + +overflow_status rust_demangle_display_demangle(struct demangle const *res, char *out, size_t len, bool alternate) { + size_t original_len = res->original_len; + size_t out_len; + switch (res->style) { + case DemangleStyleUnknown: + if (len < original_len) { + return OverflowOverflow; + } else { + memcpy(out, res->original, original_len); + out += original_len; + len -= original_len; + break; + } + break; + case DemangleStyleLegacy: { + struct demangle_legacy legacy = { + res->mangled, + res->mangled_len, + res->elements + }; + if (rust_demangle_legacy_display_demangle(legacy, out, len, alternate) == OverflowOverflow) { + return OverflowOverflow; + } + out_len = strlen(out); + out += out_len; + len -= out_len; + break; + } + case DemangleStyleV0: { + struct demangle_v0 v0 = { + res->mangled, + res->mangled_len + }; + if (rust_demangle_v0_display_demangle(v0, out, len, alternate) == OverflowOverflow) { + return OverflowOverflow; + } + out_len = strlen(out); + out += out_len; + len -= out_len; + break; + } + } + size_t suffix_len = res->suffix_len; + if (len < suffix_len || len - suffix_len < OVERFLOW_MARGIN) { + return OverflowOverflow; + } + memcpy(out, res->suffix, suffix_len); + out[suffix_len] = 0; + return OverflowOk; +} diff --git a/tools/perf/util/demangle-rust-v0.h b/tools/perf/util/demangle-rust-v0.h new file mode 100644 index 000000000..d00928186 --- /dev/null +++ b/tools/perf/util/demangle-rust-v0.h @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The contents of this file come from the Rust rustc-demangle library, hosted +// in the repository, licensed +// under "Apache-2.0 OR MIT". For copyright details, see +// . +// Please note that the file should be kept as close as possible to upstream. + +#ifndef _H_DEMANGLE_V0_H +#define _H_DEMANGLE_V0_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#if defined(__GNUC__) || defined(__clang__) +#define DEMANGLE_NODISCARD __attribute__((warn_unused_result)) +#else +#define DEMANGLE_NODISCARD +#endif + +typedef enum { + OverflowOk, + OverflowOverflow +} overflow_status; + +enum demangle_style { + DemangleStyleUnknown = 0, + DemangleStyleLegacy, + DemangleStyleV0, +}; + +// Not using a union here to make the struct easier to copy-paste if needed. +struct demangle { + enum demangle_style style; + // points to the "mangled" part of the name, + // not including `ZN` or `R` prefixes. + const char *mangled; + size_t mangled_len; + // In DemangleStyleLegacy, is the number of path elements + size_t elements; + // while it's called "original", it will not contain `.llvm.9D1C9369@@16` suffixes + // that are to be ignored. + const char *original; + size_t original_len; + // Contains the part after the mangled name that is to be outputted, + // which can be `.exit.i.i` suffixes LLVM sometimes adds. + const char *suffix; + size_t suffix_len; +}; + +// if the length of the output buffer is less than `output_len-OVERFLOW_MARGIN`, +// the demangler will return `OverflowOverflow` even if there is no overflow. +#define OVERFLOW_MARGIN 4 + +/// Demangle a C string that refers to a Rust symbol and put the demangle intermediate result in `res`. +/// Beware that `res` contains references into `s`. If `s` is modified (or free'd) before calling +/// `rust_demangle_display_demangle` behavior is undefined. +/// +/// Use `rust_demangle_display_demangle` to convert it to an actual string. +void rust_demangle_demangle(const char *s, struct demangle *res); + +/// Write the string in a `struct demangle` into a buffer. +/// +/// Return `OverflowOk` if the output buffer was sufficiently big, `OverflowOverflow` if it wasn't. +/// This function is `O(n)` in the length of the input + *output* [$], but the demangled output of demangling a symbol can +/// be exponentially[$$] large, therefore it is recommended to have a sane bound (`rust-demangle` +/// uses 1,000,000 bytes) on `len`. +/// +/// `alternate`, if true, uses the less verbose alternate formatting (Rust `{:#}`) is used, which does not show +/// symbol hashes and types of constant ints. +/// +/// [$] It's `O(n * MAX_DEPTH)`, but `MAX_DEPTH` is a constant 300 and therefore it's `O(n)` +/// [$$] Technically, bounded by `O(n^MAX_DEPTH)`, but this is practically exponential. +DEMANGLE_NODISCARD overflow_status rust_demangle_display_demangle(struct demangle const *res, char *out, size_t len, bool alternate); + +/// Returns true if `res` refers to a known valid Rust demangling style, false if it's an unknown style. +bool rust_demangle_is_known(struct demangle *res); + +#undef DEMANGLE_NODISCARD + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c new file mode 100644 index 000000000..6cfdbabbb --- /dev/null +++ b/tools/perf/util/disasm.c @@ -0,0 +1,1648 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#ifndef EF_CSKY_ABIMASK +#define EF_CSKY_ABIMASK 0XF0000000 +#endif +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "annotate.h" +#include "annotate-data.h" +#include "build-id.h" +#include "capstone.h" +#include "debug.h" +#include "disasm.h" +#include "dso.h" +#include "dwarf-regs.h" +#include "env.h" +#include "evsel.h" +#include "libbfd.h" +#include "llvm.h" +#include "map.h" +#include "maps.h" +#include "namespaces.h" +#include "srcline.h" +#include "symbol.h" +#include "thread.h" +#include "util.h" + +static regex_t file_lineno; + +/* These can be referred from the arch-dependent code */ +const struct ins_ops call_ops; +const struct ins_ops dec_ops; +const struct ins_ops jump_ops; +const struct ins_ops mov_ops; +const struct ins_ops nop_ops; +const struct ins_ops lock_ops; +const struct ins_ops ret_ops; +const struct ins_ops load_store_ops; +const struct ins_ops arithmetic_ops; + +static void ins__sort(struct arch *arch); +static int disasm_line__parse(char *line, const char **namep, char **rawp); +static int disasm_line__parse_powerpc(struct disasm_line *dl, struct annotate_args *args); + +static __attribute__((constructor)) void symbol__init_regexpr(void) +{ + regcomp(&file_lineno, "^/[^:]+:([0-9]+)", REG_EXTENDED); +} + +static int arch__grow_instructions(struct arch *arch) +{ + struct ins *new_instructions; + size_t new_nr_allocated; + + if (arch->nr_instructions_allocated == 0 && arch->instructions) + goto grow_from_non_allocated_table; + + new_nr_allocated = arch->nr_instructions_allocated + 128; + new_instructions = realloc((void *)arch->instructions, + new_nr_allocated * sizeof(struct ins)); + if (new_instructions == NULL) + return -1; + +out_update_instructions: + arch->instructions = new_instructions; + arch->nr_instructions_allocated = new_nr_allocated; + return 0; + +grow_from_non_allocated_table: + new_nr_allocated = arch->nr_instructions + 128; + new_instructions = calloc(new_nr_allocated, sizeof(struct ins)); + if (new_instructions == NULL) + return -1; + + memcpy(new_instructions, arch->instructions, arch->nr_instructions * sizeof(struct ins)); + goto out_update_instructions; +} + +int arch__associate_ins_ops(struct arch *arch, const char *name, const struct ins_ops *ops) +{ + struct ins *ins; + + if (arch->nr_instructions == arch->nr_instructions_allocated && + arch__grow_instructions(arch)) + return -1; + + ins = (struct ins *)&arch->instructions[arch->nr_instructions]; + ins->name = strdup(name); + if (!ins->name) + return -1; + + ins->ops = ops; + arch->nr_instructions++; + + ins__sort(arch); + return 0; +} + +static int e_machine_and_eflags__cmp(const struct e_machine_and_e_flags *val1, + const struct e_machine_and_e_flags *val2) +{ + if (val1->e_machine == val2->e_machine) { + if (val1->e_machine != EM_CSKY) + return 0; + if ((val1->e_flags & EF_CSKY_ABIMASK) < (val2->e_flags & EF_CSKY_ABIMASK)) + return -1; + return (val1->e_flags & EF_CSKY_ABIMASK) > (val2->e_flags & EF_CSKY_ABIMASK); + } + return val1->e_machine < val2->e_machine ? -1 : 1; +} + +static int arch__key_cmp(const void *key, const void *archp) +{ + const struct arch *const *arch = archp; + + return e_machine_and_eflags__cmp(key, &(*arch)->id); +} + +static int arch__cmp(const void *a, const void *b) +{ + const struct arch *const *aa = a; + const struct arch *const *ab = b; + + return e_machine_and_eflags__cmp(&(*aa)->id, &(*ab)->id); +} + +const struct arch *arch__find(uint16_t e_machine, uint32_t e_flags, const char *cpuid) +{ + static const struct arch *(*const arch_new_fn[])(const struct e_machine_and_e_flags *id, + const char *cpuid) = { + [EM_386] = arch__new_x86, + [EM_ARC] = arch__new_arc, + [EM_ARM] = arch__new_arm, + [EM_AARCH64] = arch__new_arm64, + [EM_CSKY] = arch__new_csky, + [EM_LOONGARCH] = arch__new_loongarch, + [EM_MIPS] = arch__new_mips, + [EM_PPC64] = arch__new_powerpc, + [EM_PPC] = arch__new_powerpc, + [EM_RISCV] = arch__new_riscv64, + [EM_S390] = arch__new_s390, + [EM_SPARC] = arch__new_sparc, + [EM_SPARCV9] = arch__new_sparc, + [EM_X86_64] = arch__new_x86, + }; + static const struct arch **archs; + static size_t num_archs; + struct e_machine_and_e_flags key = { + .e_machine = e_machine, + .e_flags = e_flags, + }; + const struct arch *result = NULL, **tmp; + + if (num_archs > 0) { + tmp = bsearch(&key, archs, num_archs, sizeof(*archs), arch__key_cmp); + if (tmp) + result = *tmp; + } + + if (result) + return result; + + if (e_machine >= ARRAY_SIZE(arch_new_fn) || arch_new_fn[e_machine] == NULL) { + errno = ENOTSUP; + return NULL; + } + + tmp = reallocarray(archs, num_archs + 1, sizeof(*archs)); + if (!tmp) + return NULL; + + archs = tmp; + + result = arch_new_fn[e_machine](&key, cpuid); + if (!result) { + pr_err("%s: failed to initialize %u arch priv area\n", + __func__, e_machine); + return NULL; + } + archs[num_archs++] = result; + qsort(archs, num_archs, sizeof(*archs), arch__cmp); + return result; +} + +bool arch__is_x86(const struct arch *arch) +{ + return arch->id.e_machine == EM_386 || arch->id.e_machine == EM_X86_64; +} + +bool arch__is_powerpc(const struct arch *arch) +{ + return arch->id.e_machine == EM_PPC || arch->id.e_machine == EM_PPC64; +} + +static void ins_ops__delete(struct ins_operands *ops) +{ + if (ops == NULL) + return; + zfree(&ops->source.raw); + zfree(&ops->source.name); + zfree(&ops->target.raw); + zfree(&ops->target.name); +} + +int ins__raw_scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->raw); +} + +int ins__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + if (ins->ops->scnprintf) + return ins->ops->scnprintf(ins, bf, size, ops, max_ins_name); + + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); +} + +bool ins__is_fused(const struct arch *arch, const char *ins1, const char *ins2) +{ + if (!arch || !arch->ins_is_fused) + return false; + + return arch->ins_is_fused(arch, ins1, ins2); +} + +static int call__parse(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + char *endptr, *tok, *name; + struct map *map = ms->map; + struct addr_map_symbol target; + + ops->target.addr = strtoull(ops->raw, &endptr, 16); + + name = strchr(endptr, '<'); + if (name == NULL) + goto indirect_call; + + name++; + + if (arch->objdump.skip_functions_char && + strchr(name, arch->objdump.skip_functions_char)) + return -1; + + tok = strchr(name, '>'); + if (tok == NULL) + return -1; + + *tok = '\0'; + ops->target.name = strdup(name); + *tok = '>'; + + if (ops->target.name == NULL) + return -1; +find_target: + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + return 0; + +indirect_call: + tok = strchr(endptr, '*'); + if (tok != NULL) { + endptr++; + + /* Indirect call can use a non-rip register and offset: callq *0x8(%rbx). + * Do not parse such instruction. */ + if (strstr(endptr, "(%r") == NULL) + ops->target.addr = strtoull(endptr, NULL, 16); + } + goto find_target; +} + +int call__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + if (ops->target.sym) + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->target.sym->name); + + if (ops->target.addr == 0) + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); + + if (ops->target.name) + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->target.name); + + return scnprintf(bf, size, "%-*s *%" PRIx64, max_ins_name, ins->name, ops->target.addr); +} + +const struct ins_ops call_ops = { + .parse = call__parse, + .scnprintf = call__scnprintf, + .is_call = true, +}; + +bool ins__is_call(const struct ins *ins) +{ + return ins->ops && ins->ops->is_call; +} + +/* + * Prevents from matching commas in the comment section, e.g.: + * ffff200008446e70: b.cs ffff2000084470f4 // b.hs, b.nlast + * + * and skip comma as part of function arguments, e.g.: + * 1d8b4ac + */ +static inline const char *validate_comma(const char *c, struct ins_operands *ops) +{ + if (ops->jump.raw_comment && c > ops->jump.raw_comment) + return NULL; + + if (ops->jump.raw_func_start && c > ops->jump.raw_func_start) + return NULL; + + return c; +} + +static int jump__parse(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + struct map *map = ms->map; + struct symbol *sym = ms->sym; + struct addr_map_symbol target = { + .ms = { .map = map__get(map), }, + }; + const char *c = strchr(ops->raw, ','); + u64 start, end; + + ops->jump.raw_comment = strchr(ops->raw, arch->objdump.comment_char); + ops->jump.raw_func_start = strchr(ops->raw, '<'); + + c = validate_comma(c, ops); + + /* + * Examples of lines to parse for the _cpp_lex_token@@Base + * function: + * + * 1159e6c: jne 115aa32 <_cpp_lex_token@@Base+0xf92> + * 1159e8b: jne c469be + * + * The first is a jump to an offset inside the same function, + * the second is to another function, i.e. that 0xa72 is an + * offset in the cpp_named_operator2name@@base function. + */ + /* + * skip over possible up to 2 operands to get to address, e.g.: + * tbnz w0, #26, ffff0000083cd190 + */ + if (c != NULL) { + c++; + ops->target.addr = strtoull(c, NULL, 16); + if (!ops->target.addr) { + c = strchr(c, ','); + c = validate_comma(c, ops); + if (c != NULL) { + c++; + ops->target.addr = strtoull(c, NULL, 16); + } + } + } else { + ops->target.addr = strtoull(ops->raw, NULL, 16); + } + + target.addr = map__objdump_2mem(map, ops->target.addr); + start = map__unmap_ip(map, sym->start); + end = map__unmap_ip(map, sym->end); + + ops->target.outside = target.addr < start || target.addr >= end; + + /* + * FIXME: things like this in _cpp_lex_token (gcc's cc1 program): + + cpp_named_operator2name@@Base+0xa72 + + * Point to a place that is after the cpp_named_operator2name + * boundaries, i.e. in the ELF symbol table for cc1 + * cpp_named_operator2name is marked as being 32-bytes long, but it in + * fact is much larger than that, so we seem to need a symbols__find() + * routine that looks for >= current->start and < next_symbol->start, + * possibly just for C++ objects? + * + * For now lets just make some progress by marking jumps to outside the + * current function as call like. + * + * Actual navigation will come next, with further understanding of how + * the symbol searching and disassembly should be done. + */ + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + if (!ops->target.outside) { + ops->target.offset = target.addr - start; + ops->target.offset_avail = true; + } else { + ops->target.offset_avail = false; + } + addr_map_symbol__exit(&target); + return 0; +} + +int jump__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + const char *c; + + if (!ops->target.addr || ops->target.offset < 0) + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); + + if (ops->target.outside && ops->target.sym != NULL) + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->target.sym->name); + + c = strchr(ops->raw, ','); + c = validate_comma(c, ops); + + if (c != NULL) { + const char *c2 = strchr(c + 1, ','); + + c2 = validate_comma(c2, ops); + /* check for 3-op insn */ + if (c2 != NULL) + c = c2; + c++; + + /* mirror arch objdump's space-after-comma style */ + if (*c == ' ') + c++; + } + + return scnprintf(bf, size, "%-*s %.*s%" PRIx64, max_ins_name, + ins->name, c ? c - ops->raw : 0, ops->raw, + ops->target.offset); +} + +void jump__delete(struct ins_operands *ops __maybe_unused) +{ + /* + * The ops->jump.raw_comment and ops->jump.raw_func_start belong to the + * raw string, don't free them. + */ +} + +const struct ins_ops jump_ops = { + .free = jump__delete, + .parse = jump__parse, + .scnprintf = jump__scnprintf, + .is_jump = true, +}; + +bool ins__is_jump(const struct ins *ins) +{ + return ins->ops && ins->ops->is_jump; +} + +static int comment__symbol(char *raw, char *comment, u64 *addrp, char **namep) +{ + char *endptr, *name, *t; + + if (strstr(raw, "(%rip)") == NULL) + return 0; + + *addrp = strtoull(comment, &endptr, 16); + if (endptr == comment) + return 0; + name = strchr(endptr, '<'); + if (name == NULL) + return -1; + + name++; + + t = strchr(name, '>'); + if (t == NULL) + return 0; + + *t = '\0'; + *namep = strdup(name); + *t = '>'; + + return 0; +} + +static int lock__parse(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + ops->locked.ops = zalloc(sizeof(*ops->locked.ops)); + if (ops->locked.ops == NULL) + return 0; + + if (disasm_line__parse(ops->raw, &ops->locked.ins.name, &ops->locked.ops->raw) < 0) + goto out_free_ops; + + ops->locked.ins.ops = ins__find(arch, ops->locked.ins.name, 0); + + if (ops->locked.ins.ops == NULL) + goto out_free_ops; + + if (ops->locked.ins.ops->parse && + ops->locked.ins.ops->parse(arch, ops->locked.ops, ms, NULL) < 0) + goto out_free_ops; + + return 0; + +out_free_ops: + zfree(&ops->locked.ops); + return 0; +} + +static int lock__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + int printed; + + if (ops->locked.ins.ops == NULL) + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); + + printed = scnprintf(bf, size, "%-*s ", max_ins_name, ins->name); + return printed + ins__scnprintf(&ops->locked.ins, bf + printed, + size - printed, ops->locked.ops, max_ins_name); +} + +static void lock__delete(struct ins_operands *ops) +{ + struct ins *ins = &ops->locked.ins; + + if (ins->ops && ins->ops->free) + ins->ops->free(ops->locked.ops); + else + ins_ops__delete(ops->locked.ops); + + zfree(&ops->locked.ops); + zfree(&ops->locked.ins.name); + zfree(&ops->target.raw); + zfree(&ops->target.name); +} + +const struct ins_ops lock_ops = { + .free = lock__delete, + .parse = lock__parse, + .scnprintf = lock__scnprintf, +}; + +/* + * Check if the operand has more than one registers like x86 SIB addressing: + * 0x1234(%rax, %rbx, 8) + * + * But it doesn't care segment selectors like %gs:0x5678(%rcx), so just check + * the input string after 'memory_ref_char' if exists. + */ +static bool check_multi_regs(const struct arch *arch, const char *op) +{ + int count = 0; + + if (arch->objdump.register_char == 0) + return false; + + if (arch->objdump.memory_ref_char) { + op = strchr(op, arch->objdump.memory_ref_char); + if (op == NULL) + return false; + } + + while ((op = strchr(op, arch->objdump.register_char)) != NULL) { + count++; + op++; + } + + return count > 1; +} + +static int mov__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *s = strchr(ops->raw, ','), *target, *comment, prev; + + if (s == NULL) + return -1; + + *s = '\0'; + + /* + * x86 SIB addressing has something like 0x8(%rax, %rcx, 1) + * then it needs to have the closing parenthesis. + */ + if (strchr(ops->raw, '(')) { + *s = ','; + s = strchr(ops->raw, ')'); + if (s == NULL || s[1] != ',') + return -1; + *++s = '\0'; + } + + ops->source.raw = strdup(ops->raw); + *s = ','; + + if (ops->source.raw == NULL) + return -1; + + ops->source.multi_regs = check_multi_regs(arch, ops->source.raw); + + target = skip_spaces(++s); + comment = strchr(s, arch->objdump.comment_char); + + if (comment != NULL) + s = comment - 1; + else + s = strchr(s, '\0') - 1; + + while (s > target && isspace(s[0])) + --s; + s++; + prev = *s; + *s = '\0'; + + ops->target.raw = strdup(target); + *s = prev; + + if (ops->target.raw == NULL) + goto out_free_source; + + ops->target.multi_regs = check_multi_regs(arch, ops->target.raw); + + if (comment == NULL) + return 0; + + comment = skip_spaces(comment); + comment__symbol(ops->source.raw, comment + 1, &ops->source.addr, &ops->source.name); + comment__symbol(ops->target.raw, comment + 1, &ops->target.addr, &ops->target.name); + + return 0; + +out_free_source: + zfree(&ops->source.raw); + return -1; +} + +int mov__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s,%s", max_ins_name, ins->name, + ops->source.name ?: ops->source.raw, + ops->target.name ?: ops->target.raw); +} + +const struct ins_ops mov_ops = { + .parse = mov__parse, + .scnprintf = mov__scnprintf, +}; + +static int dec__parse(const struct arch *arch __maybe_unused, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *target, *comment, *s, prev; + + target = s = ops->raw; + + while (s[0] != '\0' && !isspace(s[0])) + ++s; + prev = *s; + *s = '\0'; + + ops->target.raw = strdup(target); + *s = prev; + + if (ops->target.raw == NULL) + return -1; + + comment = strchr(s, arch->objdump.comment_char); + if (comment == NULL) + return 0; + + comment = skip_spaces(comment); + comment__symbol(ops->target.raw, comment + 1, &ops->target.addr, &ops->target.name); + + return 0; +} + +static int dec__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, + ops->target.name ?: ops->target.raw); +} + +const struct ins_ops dec_ops = { + .parse = dec__parse, + .scnprintf = dec__scnprintf, +}; + +static int nop__scnprintf(const struct ins *ins __maybe_unused, char *bf, size_t size, + struct ins_operands *ops __maybe_unused, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s", max_ins_name, "nop"); +} + +const struct ins_ops nop_ops = { + .scnprintf = nop__scnprintf, +}; + +const struct ins_ops ret_ops = { + .scnprintf = ins__raw_scnprintf, +}; + +static bool ins__is_nop(const struct ins *ins) +{ + return ins->ops == &nop_ops; +} + +bool ins__is_ret(const struct ins *ins) +{ + return ins->ops == &ret_ops; +} + +bool ins__is_lock(const struct ins *ins) +{ + return ins->ops == &lock_ops; +} + +static int ins__key_cmp(const void *name, const void *insp) +{ + const struct ins *ins = insp; + + return strcmp(name, ins->name); +} + +static int ins__cmp(const void *a, const void *b) +{ + const struct ins *ia = a; + const struct ins *ib = b; + + return strcmp(ia->name, ib->name); +} + +static void ins__sort(struct arch *arch) +{ + const int nmemb = arch->nr_instructions; + + qsort((void *)arch->instructions, nmemb, sizeof(struct ins), ins__cmp); +} + +static const struct ins_ops *__ins__find(const struct arch *arch, const char *name, + struct disasm_line *dl) +{ + const struct ins *ins; + const int nmemb = arch->nr_instructions; + + if (arch__is_powerpc(arch)) { + /* + * For powerpc, identify the instruction ops + * from the opcode using raw_insn. + */ + const struct ins_ops *ops; + + ops = check_ppc_insn(dl); + if (ops) + return ops; + } + + if (!arch->sorted_instructions) { + ins__sort((struct arch *)arch); + ((struct arch *)arch)->sorted_instructions = true; + } + + ins = bsearch(name, arch->instructions, nmemb, sizeof(struct ins), ins__key_cmp); + if (ins) + return ins->ops; + + if (arch->insn_suffix) { + char tmp[32]; + char suffix; + size_t len = strlen(name); + + if (len == 0 || len >= sizeof(tmp)) + return NULL; + + suffix = name[len - 1]; + if (strchr(arch->insn_suffix, suffix) == NULL) + return NULL; + + strcpy(tmp, name); + tmp[len - 1] = '\0'; /* remove the suffix and check again */ + + ins = bsearch(tmp, arch->instructions, nmemb, sizeof(struct ins), ins__key_cmp); + } + return ins ? ins->ops : NULL; +} + +const struct ins_ops *ins__find(const struct arch *arch, const char *name, struct disasm_line *dl) +{ + const struct ins_ops *ops = __ins__find(arch, name, dl); + + if (!ops && arch->associate_instruction_ops) + ops = arch->associate_instruction_ops((struct arch *)arch, name); + + return ops; +} + +static void disasm_line__init_ins(struct disasm_line *dl, const struct arch *arch, + struct map_symbol *ms) +{ + dl->ins.ops = ins__find(arch, dl->ins.name, dl); + + if (!dl->ins.ops) + return; + + if (dl->ins.ops->parse && dl->ins.ops->parse(arch, &dl->ops, ms, dl) < 0) + dl->ins.ops = NULL; +} + +static int disasm_line__parse(char *line, const char **namep, char **rawp) +{ + char tmp, *name = skip_spaces(line); + + if (name[0] == '\0') + return -1; + + *rawp = name + 1; + + while ((*rawp)[0] != '\0' && !isspace((*rawp)[0])) + ++*rawp; + + tmp = (*rawp)[0]; + (*rawp)[0] = '\0'; + *namep = strdup(name); + + if (*namep == NULL) + goto out; + + (*rawp)[0] = tmp; + *rawp = strim(*rawp); + + return 0; + +out: + return -1; +} + +/* + * Parses the result captured from symbol__disassemble_* + * Example, line read from DSO file in powerpc: + * line: 38 01 81 e8 + * opcode: fetched from arch specific get_opcode_insn + * rawp_insn: e8810138 + * + * rawp_insn is used later to extract the reg/offset fields + */ +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define RAW_BYTES 11 + +static int disasm_line__parse_powerpc(struct disasm_line *dl, struct annotate_args *args) +{ + char *line = dl->al.line; + const char **namep = &dl->ins.name; + char **rawp = &dl->ops.raw; + char *tmp_raw_insn, *name_raw_insn = skip_spaces(line); + char *name = skip_spaces(name_raw_insn + RAW_BYTES); + int disasm = 0; + int ret = 0; + + if (args->options->disassembler_used) + disasm = 1; + + if (name_raw_insn[0] == '\0') + return -1; + + if (disasm) + ret = disasm_line__parse(name, namep, rawp); + else + *namep = ""; + + tmp_raw_insn = strndup(name_raw_insn, 11); + if (tmp_raw_insn == NULL) + return -1; + + remove_spaces(tmp_raw_insn); + + sscanf(tmp_raw_insn, "%x", &dl->raw.raw_insn); + if (disasm) + dl->raw.raw_insn = be32_to_cpu(dl->raw.raw_insn); + + return ret; +} + +static void annotation_line__init(struct annotation_line *al, + struct annotate_args *args, + int nr) +{ + al->offset = args->offset; + al->line = strdup(args->line); + al->line_nr = args->line_nr; + al->fileloc = args->fileloc ? strdup(args->fileloc) : NULL; + al->data_nr = nr; +} + +static void annotation_line__exit(struct annotation_line *al) +{ + zfree_srcline(&al->path); + zfree(&al->fileloc); + zfree(&al->line); + zfree(&al->cycles); + zfree(&al->br_cntr); +} + +static size_t disasm_line_size(int nr) +{ + struct annotation_line *al; + + return (sizeof(struct disasm_line) + (sizeof(al->data[0]) * nr)); +} + +/* + * Allocating the disasm annotation line data with + * following structure: + * + * ------------------------------------------- + * struct disasm_line | struct annotation_line + * ------------------------------------------- + * + * We have 'struct annotation_line' member as last member + * of 'struct disasm_line' to have an easy access. + */ +struct disasm_line *disasm_line__new(struct annotate_args *args) +{ + struct disasm_line *dl = NULL; + struct annotation *notes = symbol__annotation(args->ms->sym); + int nr = notes->src->nr_events; + + dl = zalloc(disasm_line_size(nr)); + if (!dl) + return NULL; + + annotation_line__init(&dl->al, args, nr); + if (dl->al.line == NULL) + goto out_free_line; + + if (args->offset != -1) { + if (arch__is_powerpc(args->arch)) { + if (disasm_line__parse_powerpc(dl, args) < 0) + goto out_free_line; + } else if (disasm_line__parse(dl->al.line, &dl->ins.name, &dl->ops.raw) < 0) + goto out_free_line; + + disasm_line__init_ins(dl, args->arch, args->ms); + } + + return dl; + +out_free_line: + annotation_line__exit(&dl->al); + free(dl); + return NULL; +} + +void disasm_line__free(struct disasm_line *dl) +{ + if (dl->ins.ops && dl->ins.ops->free) + dl->ins.ops->free(&dl->ops); + else + ins_ops__delete(&dl->ops); + zfree(&dl->ins.name); + annotation_line__exit(&dl->al); + free(dl); +} + +int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool raw, int max_ins_name) +{ + if (raw || !dl->ins.ops) + return scnprintf(bf, size, "%-*s %s", max_ins_name, dl->ins.name, dl->ops.raw); + + return ins__scnprintf(&dl->ins, bf, size, &dl->ops, max_ins_name); +} + +/* + * symbol__parse_objdump_line() parses objdump output (with -d --no-show-raw) + * which looks like following + * + * 0000000000415500 <_init>: + * 415500: sub $0x8,%rsp + * 415504: mov 0x2f5ad5(%rip),%rax # 70afe0 <_DYNAMIC+0x2f8> + * 41550b: test %rax,%rax + * 41550e: je 415515 <_init+0x15> + * 415510: callq 416e70 <__gmon_start__@plt> + * 415515: add $0x8,%rsp + * 415519: retq + * + * it will be parsed and saved into struct disasm_line as + * + * + * The offset will be a relative offset from the start of the symbol and -1 + * means that it's not a disassembly line so should be treated differently. + * The ops.raw part will be parsed further according to type of the instruction. + */ +static int symbol__parse_objdump_line(struct symbol *sym, + struct annotate_args *args, + char *parsed_line, int *line_nr, char **fileloc) +{ + struct map *map = args->ms->map; + struct annotation *notes = symbol__annotation(sym); + struct disasm_line *dl; + char *tmp; + s64 line_ip, offset = -1; + regmatch_t match[2]; + + /* /filename:linenr ? Save line number and ignore. */ + if (regexec(&file_lineno, parsed_line, 2, match, 0) == 0) { + *line_nr = atoi(parsed_line + match[1].rm_so); + free(*fileloc); + *fileloc = strdup(parsed_line); + return 0; + } + + /* Process hex address followed by ':'. */ + line_ip = strtoull(parsed_line, &tmp, 16); + if (parsed_line != tmp && tmp[0] == ':' && tmp[1] != '\0') { + u64 start = map__rip_2objdump(map, sym->start), + end = map__rip_2objdump(map, sym->end); + + offset = line_ip - start; + if ((u64)line_ip < start || (u64)line_ip >= end) + offset = -1; + else + parsed_line = tmp + 1; + } + + args->offset = offset; + args->line = parsed_line; + args->line_nr = *line_nr; + args->fileloc = *fileloc; + args->ms->sym = sym; + + dl = disasm_line__new(args); + (*line_nr)++; + + if (dl == NULL) + return -1; + + if (!disasm_line__has_local_offset(dl)) { + dl->ops.target.offset = dl->ops.target.addr - + map__rip_2objdump(map, sym->start); + dl->ops.target.offset_avail = true; + } + + /* kcore has no symbols, so add the call target symbol */ + if (dl->ins.ops && ins__is_call(&dl->ins) && !dl->ops.target.sym) { + struct addr_map_symbol target = { + .addr = dl->ops.target.addr, + .ms = { .map = map__get(map), }, + }; + + if (!maps__find_ams(thread__maps(args->ms->thread), &target) && + target.ms.sym->start == target.al_addr) + dl->ops.target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + } + + annotation_line__add(&dl->al, ¬es->src->source); + return 0; +} + +static void delete_last_nop(struct symbol *sym) +{ + struct annotation *notes = symbol__annotation(sym); + struct list_head *list = ¬es->src->source; + struct disasm_line *dl; + + while (!list_empty(list)) { + dl = list_entry(list->prev, struct disasm_line, al.node); + + if (dl->ins.ops) { + if (!ins__is_nop(&dl->ins)) + return; + } else { + if (!strstr(dl->al.line, " nop ") && + !strstr(dl->al.line, " nopl ") && + !strstr(dl->al.line, " nopw ")) + return; + } + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } +} + +int symbol__strerror_disassemble(struct map_symbol *ms, int errnum, char *buf, size_t buflen) +{ + struct dso *dso = map__dso(ms->map); + + BUG_ON(buflen == 0); + + if (errnum >= 0) { + str_error_r(errnum, buf, buflen); + return 0; + } + + switch (errnum) { + case SYMBOL_ANNOTATE_ERRNO__NO_VMLINUX: { + char bf[SBUILD_ID_SIZE + 15] = " with build id "; + char *build_id_msg = NULL; + + if (dso__has_build_id(dso)) { + build_id__snprintf(dso__bid(dso), bf + 15, sizeof(bf) - 15); + build_id_msg = bf; + } + scnprintf(buf, buflen, + "No vmlinux file%s\nwas found in the path.\n\n" + "Note that annotation using /proc/kcore requires CAP_SYS_RAWIO capability.\n\n" + "Please use:\n\n" + " perf buildid-cache -vu vmlinux\n\n" + "or:\n\n" + " --vmlinux vmlinux\n", build_id_msg ?: ""); + } + break; + case SYMBOL_ANNOTATE_ERRNO__NO_LIBOPCODES_FOR_BPF: + scnprintf(buf, buflen, "Please link with binutils's libopcode to enable BPF annotation"); + break; + case SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP: + scnprintf(buf, buflen, "Problems with arch specific instruction name regular expressions."); + break; + case SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING: + scnprintf(buf, buflen, "Problems while parsing the CPUID in the arch specific initialization."); + break; + case SYMBOL_ANNOTATE_ERRNO__BPF_INVALID_FILE: + scnprintf(buf, buflen, "Invalid BPF file: %s.", dso__long_name(dso)); + break; + case SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF: + scnprintf(buf, buflen, "The %s BPF file has no BTF section, compile with -g or use pahole -J.", + dso__long_name(dso)); + break; + case SYMBOL_ANNOTATE_ERRNO__COULDNT_DETERMINE_FILE_TYPE: + scnprintf(buf, buflen, "Couldn't determine the file %s type.", dso__long_name(dso)); + break; + default: + scnprintf(buf, buflen, "Internal error: Invalid %d error code\n", errnum); + break; + } + + return 0; +} + +static int dso__disassemble_filename(struct dso *dso, char *filename, size_t filename_size) +{ + char linkname[PATH_MAX]; + char *build_id_filename; + char *build_id_path = NULL; + char *pos; + int len; + + if (dso__symtab_type(dso) == DSO_BINARY_TYPE__KALLSYMS && + !dso__is_kcore(dso)) + return SYMBOL_ANNOTATE_ERRNO__NO_VMLINUX; + + build_id_filename = dso__build_id_filename(dso, NULL, 0, false); + if (build_id_filename) { + __symbol__join_symfs(filename, filename_size, build_id_filename); + free(build_id_filename); + } else { + if (dso__has_build_id(dso)) + return ENOMEM; + goto fallback; + } + + build_id_path = strdup(filename); + if (!build_id_path) + return ENOMEM; + + /* + * old style build-id cache has name of XX/XXXXXXX.. while + * new style has XX/XXXXXXX../{elf,kallsyms,vdso}. + * extract the build-id part of dirname in the new style only. + */ + pos = strrchr(build_id_path, '/'); + if (pos && strlen(pos) < SBUILD_ID_SIZE - 2) + dirname(build_id_path); + + if (dso__is_kcore(dso)) + goto fallback; + + len = readlink(build_id_path, linkname, sizeof(linkname) - 1); + if (len < 0) + goto fallback; + + linkname[len] = '\0'; + if (strstr(linkname, DSO__NAME_KALLSYMS) || + access(filename, R_OK)) { +fallback: + /* + * If we don't have build-ids or the build-id file isn't in the + * cache, or is just a kallsyms file, well, lets hope that this + * DSO is the same as when 'perf record' ran. + */ + if (dso__kernel(dso) && dso__long_name(dso)[0] == '/') + snprintf(filename, filename_size, "%s", dso__long_name(dso)); + else + __symbol__join_symfs(filename, filename_size, dso__long_name(dso)); + + mutex_lock(dso__lock(dso)); + if (access(filename, R_OK) && errno == ENOENT && dso__nsinfo(dso)) { + char *new_name = dso__filename_with_chroot(dso, filename); + if (new_name) { + strlcpy(filename, new_name, filename_size); + free(new_name); + } + } + mutex_unlock(dso__lock(dso)); + } else if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) { + dso__set_binary_type(dso, DSO_BINARY_TYPE__BUILD_ID_CACHE); + } + + free(build_id_path); + return 0; +} + +static int symbol__disassemble_raw(char *filename, struct symbol *sym, + struct annotate_args *args) +{ + struct annotation *notes = symbol__annotation(sym); + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + u64 start = map__rip_2objdump(map, sym->start); + u64 end = map__rip_2objdump(map, sym->end); + u64 len = end - start; + u64 offset; + int i, count; + u8 *buf = NULL; + char disasm_buf[512]; + struct disasm_line *dl; + u32 *line; + + /* Return if objdump is specified explicitly */ + if (args->options->objdump_path) + return -1; + + pr_debug("Reading raw instruction from : %s using dso__data_read_offset\n", filename); + + buf = malloc(len); + if (buf == NULL) + goto err; + + count = dso__data_read_offset(dso, NULL, sym->start, buf, len); + + line = (u32 *)buf; + + if ((u64)count != len) + goto err; + + /* add the function address and name */ + scnprintf(disasm_buf, sizeof(disasm_buf), "%#"PRIx64" <%s>:", + start, sym->name); + + args->offset = -1; + args->line = disasm_buf; + args->line_nr = 0; + args->fileloc = NULL; + args->ms->sym = sym; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + /* Each raw instruction is 4 byte */ + count = len/4; + + for (i = 0, offset = 0; i < count; i++) { + args->offset = offset; + sprintf(args->line, "%x", line[i]); + dl = disasm_line__new(args); + if (dl == NULL) + break; + + annotation_line__add(&dl->al, ¬es->src->source); + offset += 4; + } + + /* It failed in the middle */ + if (offset != len) { + struct list_head *list = ¬es->src->source; + + /* Discard all lines and fallback to objdump */ + while (!list_empty(list)) { + dl = list_first_entry(list, struct disasm_line, al.node); + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } + count = -1; + } + +out: + free(buf); + return count < 0 ? count : 0; + +err: + count = -1; + goto out; +} + +/* + * Possibly create a new version of line with tabs expanded. Returns the + * existing or new line, storage is updated if a new line is allocated. If + * allocation fails then NULL is returned. + */ +char *expand_tabs(char *line, char **storage, size_t *storage_len) +{ + size_t i, src, dst, len, new_storage_len, num_tabs; + char *new_line; + size_t line_len = strlen(line); + + for (num_tabs = 0, i = 0; i < line_len; i++) + if (line[i] == '\t') + num_tabs++; + + if (num_tabs == 0) + return line; + + /* + * Space for the line and '\0', less the leading and trailing + * spaces. Each tab may introduce 7 additional spaces. + */ + new_storage_len = line_len + 1 + (num_tabs * 7); + + new_line = malloc(new_storage_len); + if (new_line == NULL) { + pr_err("Failure allocating memory for tab expansion\n"); + return NULL; + } + + /* + * Copy regions starting at src and expand tabs. If there are two + * adjacent tabs then 'src == i', the memcpy is of size 0 and the spaces + * are inserted. + */ + for (i = 0, src = 0, dst = 0; i < line_len && num_tabs; i++) { + if (line[i] == '\t') { + len = i - src; + memcpy(&new_line[dst], &line[src], len); + dst += len; + new_line[dst++] = ' '; + while (dst % 8 != 0) + new_line[dst++] = ' '; + src = i + 1; + num_tabs--; + } + } + + /* Expand the last region. */ + len = line_len - src; + memcpy(&new_line[dst], &line[src], len); + dst += len; + new_line[dst] = '\0'; + + free(*storage); + *storage = new_line; + *storage_len = new_storage_len; + return new_line; +} + +static int symbol__disassemble_bpf_image(struct symbol *sym, struct annotate_args *args) +{ + struct annotation *notes = symbol__annotation(sym); + struct disasm_line *dl; + + args->offset = -1; + args->line = strdup("to be implemented"); + args->line_nr = 0; + args->fileloc = NULL; + dl = disasm_line__new(args); + if (dl) + annotation_line__add(&dl->al, ¬es->src->source); + + zfree(&args->line); + return 0; +} + +static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, + struct annotate_args *args) +{ + struct annotation_options *opts = &annotate_opts; + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + char *command; + FILE *file; + int lineno = 0; + char *fileloc = NULL; + int nline; + char *line; + size_t line_len; + const char *objdump_argv[] = { + "/bin/sh", + "-c", + NULL, /* Will be the objdump command to run. */ + "--", + NULL, /* Will be the symfs path. */ + NULL, + }; + struct child_process objdump_process; + int err; + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) + return symbol__disassemble_bpf_libbfd(sym, args); + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) + return symbol__disassemble_bpf_image(sym, args); + + err = asprintf(&command, + "%s %s%s --start-address=0x%016" PRIx64 + " --stop-address=0x%016" PRIx64 + " %s -d %s %s %s %c%s%c %s%s -C \"$1\"", + opts->objdump_path ?: "objdump", + opts->disassembler_style ? "-M " : "", + opts->disassembler_style ?: "", + map__rip_2objdump(map, sym->start), + map__rip_2objdump(map, sym->end), + opts->show_linenr ? "-l" : "", + opts->show_asm_raw ? "" : "--no-show-raw-insn", + opts->annotate_src ? "-S" : "", + opts->prefix ? "--prefix " : "", + opts->prefix ? '"' : ' ', + opts->prefix ?: "", + opts->prefix ? '"' : ' ', + opts->prefix_strip ? "--prefix-strip=" : "", + opts->prefix_strip ?: ""); + + if (err < 0) { + pr_err("Failure allocating memory for the command to run\n"); + return err; + } + + pr_debug("Executing: %s\n", command); + + objdump_argv[2] = command; + objdump_argv[4] = filename; + + /* Create a pipe to read from for stdout */ + memset(&objdump_process, 0, sizeof(objdump_process)); + objdump_process.argv = objdump_argv; + objdump_process.out = -1; + objdump_process.err = -1; + objdump_process.no_stderr = 1; + if (start_command(&objdump_process)) { + pr_err("Failure starting to run %s\n", command); + err = -1; + goto out_free_command; + } + + file = fdopen(objdump_process.out, "r"); + if (!file) { + pr_err("Failure creating FILE stream for %s\n", command); + /* + * If we were using debug info should retry with + * original binary. + */ + err = -1; + goto out_close_stdout; + } + + /* Storage for getline. */ + line = NULL; + line_len = 0; + + nline = 0; + while (!feof(file)) { + const char *match; + char *expanded_line; + + if (getline(&line, &line_len, file) < 0 || !line) + break; + + /* Skip lines containing "filename:" */ + match = strstr(line, filename); + if (match && match[strlen(filename)] == ':') + continue; + + expanded_line = strim(line); + expanded_line = expand_tabs(expanded_line, &line, &line_len); + if (!expanded_line) + break; + + /* + * The source code line number (lineno) needs to be kept in + * across calls to symbol__parse_objdump_line(), so that it + * can associate it with the instructions till the next one. + * See disasm_line__new() and struct disasm_line::line_nr. + */ + if (symbol__parse_objdump_line(sym, args, expanded_line, + &lineno, &fileloc) < 0) + break; + nline++; + } + free(line); + free(fileloc); + + err = finish_command(&objdump_process); + if (err) + pr_err("Error running %s\n", command); + + if (nline == 0) { + err = -1; + pr_err("No output from %s\n", command); + } + + /* + * kallsyms does not have symbol sizes so there may a nop at the end. + * Remove it. + */ + if (dso__is_kcore(dso)) + delete_last_nop(sym); + + fclose(file); + +out_close_stdout: + close(objdump_process.out); + +out_free_command: + free(command); + return err; +} + +int symbol__disassemble(struct symbol *sym, struct annotate_args *args) +{ + struct annotation_options *options = args->options; + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + char symfs_filename[PATH_MAX]; + bool delete_extract = false; + struct kcore_extract kce; + bool decomp = false; + int err = dso__disassemble_filename(dso, symfs_filename, sizeof(symfs_filename)); + + if (err) + return err; + + pr_debug("%s: filename=%s, sym=%s, start=%#" PRIx64 ", end=%#" PRIx64 "\n", __func__, + symfs_filename, sym->name, map__unmap_ip(map, sym->start), + map__unmap_ip(map, sym->end)); + + pr_debug("annotating [%p] %30s : [%p] %30s\n", dso, dso__long_name(dso), sym, sym->name); + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) { + return SYMBOL_ANNOTATE_ERRNO__COULDNT_DETERMINE_FILE_TYPE; + } else if (dso__is_kcore(dso)) { + kce.addr = map__rip_2objdump(map, sym->start); + kce.kcore_filename = symfs_filename; + kce.len = sym->end - sym->start; + kce.offs = sym->start; + + if (!kcore_extract__create(&kce)) { + delete_extract = true; + strlcpy(symfs_filename, kce.extract_filename, sizeof(symfs_filename)); + } + } else if (dso__needs_decompress(dso)) { + char tmp[KMOD_DECOMP_LEN]; + + if (dso__decompress_kmodule_path(dso, symfs_filename, tmp, sizeof(tmp)) < 0) + return -1; + + /* empty pathname means file wasn't actually compressed */ + if (tmp[0] != '\0') { + decomp = true; + strcpy(symfs_filename, tmp); + } + } + + /* + * For powerpc data type profiling, use the dso__data_read_offset to + * read raw instruction directly and interpret the binary code to + * understand instructions and register fields. For sort keys as type + * and typeoff, disassemble to mnemonic notation is not required in + * case of powerpc. + */ + if (arch__is_powerpc(args->arch)) { + extern const char *sort_order; + + if (sort_order && !strstr(sort_order, "sym")) { + err = symbol__disassemble_raw(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + + err = symbol__disassemble_capstone_powerpc(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + } + } + + /* FIXME: LLVM and CAPSTONE should support source code */ + if (options->annotate_src && !options->hide_src_code) { + err = symbol__disassemble_objdump(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + } + + err = -1; + for (u8 i = 0; i < ARRAY_SIZE(options->disassemblers) && err != 0; i++) { + enum perf_disassembler dis = options->disassemblers[i]; + + switch (dis) { + case PERF_DISASM_LLVM: + args->options->disassembler_used = PERF_DISASM_LLVM; + err = symbol__disassemble_llvm(symfs_filename, sym, args); + break; + case PERF_DISASM_CAPSTONE: + args->options->disassembler_used = PERF_DISASM_CAPSTONE; + err = symbol__disassemble_capstone(symfs_filename, sym, args); + break; + case PERF_DISASM_OBJDUMP: + args->options->disassembler_used = PERF_DISASM_OBJDUMP; + err = symbol__disassemble_objdump(symfs_filename, sym, args); + break; + case PERF_DISASM_UNKNOWN: /* End of disassemblers. */ + default: + args->options->disassembler_used = PERF_DISASM_UNKNOWN; + goto out_remove_tmp; + } + if (err == 0) + pr_debug("Disassembled with %s\n", perf_disassembler__strs[dis]); + } +out_remove_tmp: + if (decomp) + unlink(symfs_filename); + + if (delete_extract) + kcore_extract__delete(&kce); + + return err; +} diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h new file mode 100644 index 000000000..25756e3f4 --- /dev/null +++ b/tools/perf/util/disasm.h @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __PERF_UTIL_DISASM_H +#define __PERF_UTIL_DISASM_H + +#include "map_symbol.h" + +#ifdef HAVE_LIBDW_SUPPORT +#include "dwarf-aux.h" +#endif + +struct annotation_options; +struct disasm_line; +struct ins; +struct evsel; +struct symbol; +struct data_loc_info; +struct type_state; +struct disasm_line; + +struct e_machine_and_e_flags { + uint32_t e_flags; + uint16_t e_machine; +}; + +struct arch { + /** @name: name such as "x86" or "powerpc". */ + const char *name; + const struct ins *instructions; + size_t nr_instructions; + size_t nr_instructions_allocated; + const char *insn_suffix; + unsigned int model; + unsigned int family; + /** @id: ELF machine and flags associated with arch. */ + struct e_machine_and_e_flags id; + bool sorted_instructions; + struct { + char comment_char; + char skip_functions_char; + char register_char; + char memory_ref_char; + char imm_char; + } objdump; + bool (*ins_is_fused)(const struct arch *arch, const char *ins1, + const char *ins2); + const struct ins_ops *(*associate_instruction_ops)(struct arch *arch, const char *name); +#ifdef HAVE_LIBDW_SUPPORT + void (*update_insn_state)(struct type_state *state, + struct data_loc_info *dloc, Dwarf_Die *cu_die, + struct disasm_line *dl); +#endif +}; + +struct ins { + const char *name; + const struct ins_ops *ops; +}; + +struct ins_operands { + char *raw; + struct { + char *raw; + char *name; + struct symbol *sym; + u64 addr; + s64 offset; + bool offset_avail; + bool outside; + bool multi_regs; + bool mem_ref; + } target; + union { + struct { + char *raw; + char *name; + u64 addr; + bool multi_regs; + bool mem_ref; + } source; + struct { + struct ins ins; + struct ins_operands *ops; + } locked; + struct { + char *raw_comment; + char *raw_func_start; + } jump; + }; +}; + +struct ins_ops { + void (*free)(struct ins_operands *ops); + int (*parse)(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl); + int (*scnprintf)(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); + bool is_jump; + bool is_call; +}; + +struct annotate_args { + const struct arch *arch; + struct map_symbol *ms; + struct annotation_options *options; + s64 offset; + char *line; + int line_nr; + char *fileloc; +}; + +const struct arch *arch__find(uint16_t e_machine, uint32_t e_flags, const char *cpuid); +bool arch__is_x86(const struct arch *arch); +bool arch__is_powerpc(const struct arch *arch); + +extern const struct ins_ops call_ops; +extern const struct ins_ops dec_ops; +extern const struct ins_ops jump_ops; +extern const struct ins_ops mov_ops; +extern const struct ins_ops nop_ops; +extern const struct ins_ops lock_ops; +extern const struct ins_ops ret_ops; + +int arch__associate_ins_ops(struct arch *arch, const char *name, const struct ins_ops *ops); + +const struct arch *arch__new_arc(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_arm(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_arm64(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_csky(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_loongarch(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_mips(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_powerpc(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_riscv64(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_s390(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_sparc(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_x86(const struct e_machine_and_e_flags *id, const char *cpuid); + +const struct ins_ops *ins__find(const struct arch *arch, const char *name, struct disasm_line *dl); + +bool ins__is_call(const struct ins *ins); +bool ins__is_jump(const struct ins *ins); +bool ins__is_fused(const struct arch *arch, const char *ins1, const char *ins2); +bool ins__is_ret(const struct ins *ins); +bool ins__is_lock(const struct ins *ins); + +const struct ins_ops *check_ppc_insn(struct disasm_line *dl); + +struct disasm_line *disasm_line__new(struct annotate_args *args); +void disasm_line__free(struct disasm_line *dl); + +int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, + bool raw, int max_ins_name); + +int ins__raw_scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int ins__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int call__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int jump__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int mov__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); + +void jump__delete(struct ins_operands *ops); + +int symbol__disassemble(struct symbol *sym, struct annotate_args *args); + +char *expand_tabs(char *line, char **storage, size_t *storage_len); + +#endif /* __PERF_UTIL_DISASM_H */ diff --git a/tools/perf/util/dlfilter.c b/tools/perf/util/dlfilter.c new file mode 100644 index 000000000..e11e144af --- /dev/null +++ b/tools/perf/util/dlfilter.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dlfilter.c: Interface to perf script --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "dso.h" +#include "map.h" +#include "thread.h" +#include "trace-event.h" +#include "symbol.h" +#include "srcline.h" +#include "dlfilter.h" +#include "../include/perf/perf_dlfilter.h" + +static void al_to_d_al(struct addr_location *al, struct perf_dlfilter_al *d_al) +{ + struct symbol *sym = al->sym; + + d_al->size = sizeof(*d_al); + if (al->map) { + struct dso *dso = map__dso(al->map); + + if (symbol_conf.show_kernel_path && dso__long_name(dso)) + d_al->dso = dso__long_name(dso); + else + d_al->dso = dso__name(dso); + d_al->is_64_bit = dso__is_64_bit(dso); + d_al->buildid_size = dso__bid(dso)->size; + d_al->buildid = dso__bid(dso)->data; + } else { + d_al->dso = NULL; + d_al->is_64_bit = 0; + d_al->buildid_size = 0; + d_al->buildid = NULL; + } + if (sym) { + d_al->sym = sym->name; + d_al->sym_start = sym->start; + d_al->sym_end = sym->end; + if (al->addr < sym->end) + d_al->symoff = al->addr - sym->start; + else if (al->map) + d_al->symoff = al->addr - map__start(al->map) - sym->start; + else + d_al->symoff = 0; + d_al->sym_binding = symbol__binding(sym); + } else { + d_al->sym = NULL; + d_al->sym_start = 0; + d_al->sym_end = 0; + d_al->symoff = 0; + d_al->sym_binding = 0; + } + d_al->addr = al->addr; + d_al->comm = NULL; + d_al->filtered = 0; + d_al->priv = NULL; +} + +static struct addr_location *get_al(struct dlfilter *d) +{ + struct addr_location *al = d->al; + + if (!al->thread && machine__resolve(d->machine, al, d->sample) < 0) + return NULL; + return al; +} + +static struct thread *get_thread(struct dlfilter *d) +{ + struct addr_location *al = get_al(d); + + return al ? al->thread : NULL; +} + +static const struct perf_dlfilter_al *dlfilter__resolve_ip(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al *d_al = d->d_ip_al; + struct addr_location *al; + + if (!d->ctx_valid) + return NULL; + + /* 'size' is also used to indicate already initialized */ + if (d_al->size) + return d_al; + + al = get_al(d); + if (!al) + return NULL; + + al_to_d_al(al, d_al); + + d_al->is_kernel_ip = machine__kernel_ip(d->machine, d->sample->ip); + d_al->comm = al->thread ? thread__comm_str(al->thread) : ":-1"; + d_al->filtered = al->filtered; + + return d_al; +} + +static const struct perf_dlfilter_al *dlfilter__resolve_addr(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al *d_addr_al = d->d_addr_al; + struct addr_location *addr_al = d->addr_al; + + if (!d->ctx_valid || !d->d_sample->addr_correlates_sym) + return NULL; + + /* 'size' is also used to indicate already initialized */ + if (d_addr_al->size) + return d_addr_al; + + if (!addr_al->thread) { + struct thread *thread = get_thread(d); + + if (!thread) + return NULL; + thread__resolve(thread, addr_al, d->sample); + } + + al_to_d_al(addr_al, d_addr_al); + + d_addr_al->is_kernel_ip = machine__kernel_ip(d->machine, d->sample->addr); + + return d_addr_al; +} + +static char **dlfilter__args(void *ctx, int *dlargc) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (dlargc) + *dlargc = 0; + else + return NULL; + + if (!d->ctx_valid && !d->in_start && !d->in_stop) + return NULL; + + *dlargc = d->dlargc; + return d->dlargv; +} + +static bool has_priv(struct perf_dlfilter_al *d_al_p) +{ + return d_al_p->size >= offsetof(struct perf_dlfilter_al, priv) + sizeof(d_al_p->priv); +} + +static __s32 dlfilter__resolve_address(void *ctx, __u64 address, struct perf_dlfilter_al *d_al_p) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al d_al; + struct addr_location al; + struct thread *thread; + __u32 sz; + + if (!d->ctx_valid || !d_al_p) + return -1; + + thread = get_thread(d); + if (!thread) + return -1; + + addr_location__init(&al); + thread__find_symbol_fb(thread, d->sample->cpumode, address, &al); + + al_to_d_al(&al, &d_al); + + d_al.is_kernel_ip = machine__kernel_ip(d->machine, address); + + sz = d_al_p->size; + memcpy(d_al_p, &d_al, min((size_t)sz, sizeof(d_al))); + d_al_p->size = sz; + + if (has_priv(d_al_p)) + d_al_p->priv = memdup(&al, sizeof(al)); + else /* Avoid leak for v0 API */ + addr_location__exit(&al); + + return 0; +} + +static void dlfilter__al_cleanup(void *ctx __maybe_unused, struct perf_dlfilter_al *d_al_p) +{ + struct addr_location *al; + + /* Ensure backward compatibility */ + if (!has_priv(d_al_p) || !d_al_p->priv) + return; + + al = d_al_p->priv; + + d_al_p->priv = NULL; + + addr_location__exit(al); + + free(al); +} + +static const __u8 *dlfilter__insn(void *ctx, __u32 *len) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (!len) + return NULL; + + *len = 0; + + if (!d->ctx_valid) + return NULL; + + if (d->sample->ip && !d->sample->insn_len) { + struct addr_location *al = d->al; + + if (!al->thread && machine__resolve(d->machine, al, d->sample) < 0) + return NULL; + + if (thread__maps(al->thread)) { + struct machine *machine = maps__machine(thread__maps(al->thread)); + + if (machine) + perf_sample__fetch_insn(d->sample, al->thread, machine); + } + } + + if (!d->sample->insn_len) + return NULL; + + *len = d->sample->insn_len; + + return (__u8 *)d->sample->insn; +} + +static const char *dlfilter__srcline(void *ctx, __u32 *line_no) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct addr_location *al; + unsigned int line = 0; + char *srcfile = NULL; + struct map *map; + struct dso *dso; + u64 addr; + + if (!d->ctx_valid || !line_no) + return NULL; + + al = get_al(d); + if (!al) + return NULL; + + map = al->map; + addr = al->addr; + dso = map ? map__dso(map) : NULL; + + if (dso) + srcfile = get_srcline_split(dso, map__rip_2objdump(map, addr), &line); + + *line_no = line; + return srcfile; +} + +static struct perf_event_attr *dlfilter__attr(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (!d->ctx_valid) + return NULL; + + return &d->evsel->core.attr; +} + +static __s32 code_read(__u64 ip, struct map *map, struct machine *machine, void *buf, __u32 len) +{ + u64 offset = map__map_ip(map, ip); + + if (ip + len >= map__end(map)) + len = map__end(map) - ip; + + return dso__data_read_offset(map__dso(map), machine, offset, buf, len); +} + +static __s32 dlfilter__object_code(void *ctx, __u64 ip, void *buf, __u32 len) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct addr_location *al; + struct addr_location a; + __s32 ret; + + if (!d->ctx_valid) + return -1; + + al = get_al(d); + if (!al) + return -1; + + if (al->map && ip >= map__start(al->map) && ip < map__end(al->map) && + machine__kernel_ip(d->machine, ip) == machine__kernel_ip(d->machine, d->sample->ip)) + return code_read(ip, al->map, d->machine, buf, len); + + addr_location__init(&a); + + thread__find_map_fb(al->thread, d->sample->cpumode, ip, &a); + ret = a.map ? code_read(ip, a.map, d->machine, buf, len) : -1; + + addr_location__exit(&a); + + return ret; +} + +static const struct perf_dlfilter_fns perf_dlfilter_fns = { + .resolve_ip = dlfilter__resolve_ip, + .resolve_addr = dlfilter__resolve_addr, + .args = dlfilter__args, + .resolve_address = dlfilter__resolve_address, + .al_cleanup = dlfilter__al_cleanup, + .insn = dlfilter__insn, + .srcline = dlfilter__srcline, + .attr = dlfilter__attr, + .object_code = dlfilter__object_code, +}; + +static char *find_dlfilter(const char *file) +{ + char path[PATH_MAX]; + char *exec_path; + + if (strchr(file, '/')) + goto out; + + if (!access(file, R_OK)) { + /* + * Prepend "./" so that dlopen will find the file in the + * current directory. + */ + snprintf(path, sizeof(path), "./%s", file); + file = path; + goto out; + } + + exec_path = get_argv_exec_path(); + if (!exec_path) + goto out; + snprintf(path, sizeof(path), "%s/dlfilters/%s", exec_path, file); + free(exec_path); + if (!access(path, R_OK)) + file = path; +out: + return strdup(file); +} + +#define CHECK_FLAG(x) BUILD_BUG_ON((u64)PERF_DLFILTER_FLAG_ ## x != (u64)PERF_IP_FLAG_ ## x) + +static int dlfilter__init(struct dlfilter *d, const char *file, int dlargc, char **dlargv) +{ + CHECK_FLAG(BRANCH); + CHECK_FLAG(CALL); + CHECK_FLAG(RETURN); + CHECK_FLAG(CONDITIONAL); + CHECK_FLAG(SYSCALLRET); + CHECK_FLAG(ASYNC); + CHECK_FLAG(INTERRUPT); + CHECK_FLAG(TX_ABORT); + CHECK_FLAG(TRACE_BEGIN); + CHECK_FLAG(TRACE_END); + CHECK_FLAG(IN_TX); + CHECK_FLAG(VMENTRY); + CHECK_FLAG(VMEXIT); + + memset(d, 0, sizeof(*d)); + d->file = find_dlfilter(file); + if (!d->file) + return -1; + d->dlargc = dlargc; + d->dlargv = dlargv; + return 0; +} + +static void dlfilter__exit(struct dlfilter *d) +{ + zfree(&d->file); +} + +static int dlfilter__open(struct dlfilter *d) +{ + d->handle = dlopen(d->file, RTLD_NOW); + if (!d->handle) { + pr_err("dlopen failed for: '%s'\n", d->file); + return -1; + } + d->start = dlsym(d->handle, "start"); + d->filter_event = dlsym(d->handle, "filter_event"); + d->filter_event_early = dlsym(d->handle, "filter_event_early"); + d->stop = dlsym(d->handle, "stop"); + d->fns = dlsym(d->handle, "perf_dlfilter_fns"); + if (d->fns) + memcpy(d->fns, &perf_dlfilter_fns, sizeof(struct perf_dlfilter_fns)); + return 0; +} + +static int dlfilter__close(struct dlfilter *d) +{ + return dlclose(d->handle); +} + +struct dlfilter *dlfilter__new(const char *file, int dlargc, char **dlargv) +{ + struct dlfilter *d = malloc(sizeof(*d)); + + if (!d) + return NULL; + + if (dlfilter__init(d, file, dlargc, dlargv)) + goto err_free; + + if (dlfilter__open(d)) + goto err_exit; + + return d; + +err_exit: + dlfilter__exit(d); +err_free: + free(d); + return NULL; +} + +static void dlfilter__free(struct dlfilter *d) +{ + if (d) { + dlfilter__exit(d); + free(d); + } +} + +int dlfilter__start(struct dlfilter *d, struct perf_session *session) +{ + if (d) { + d->session = session; + if (d->start) { + int ret; + + d->in_start = true; + ret = d->start(&d->data, d); + d->in_start = false; + return ret; + } + } + return 0; +} + +static int dlfilter__stop(struct dlfilter *d) +{ + if (d && d->stop) { + int ret; + + d->in_stop = true; + ret = d->stop(d->data, d); + d->in_stop = false; + return ret; + } + return 0; +} + +void dlfilter__cleanup(struct dlfilter *d) +{ + if (d) { + dlfilter__stop(d); + dlfilter__close(d); + dlfilter__free(d); + } +} + +#define ASSIGN(x) d_sample.x = sample->x + +int dlfilter__do_filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al, + bool early) +{ + struct perf_dlfilter_sample d_sample; + struct perf_dlfilter_al d_ip_al; + struct perf_dlfilter_al d_addr_al; + int ret; + + d->event = event; + d->sample = sample; + d->evsel = evsel; + d->machine = machine; + d->al = al; + d->addr_al = addr_al; + d->d_sample = &d_sample; + d->d_ip_al = &d_ip_al; + d->d_addr_al = &d_addr_al; + + d_sample.size = sizeof(d_sample); + d_sample.p_stage_cyc = sample->weight3; + d_ip_al.size = 0; /* To indicate d_ip_al is not initialized */ + d_addr_al.size = 0; /* To indicate d_addr_al is not initialized */ + + ASSIGN(ip); + ASSIGN(pid); + ASSIGN(tid); + ASSIGN(time); + ASSIGN(addr); + ASSIGN(id); + ASSIGN(stream_id); + ASSIGN(period); + ASSIGN(weight); + ASSIGN(ins_lat); + ASSIGN(transaction); + ASSIGN(insn_cnt); + ASSIGN(cyc_cnt); + ASSIGN(cpu); + ASSIGN(flags); + ASSIGN(data_src); + ASSIGN(phys_addr); + ASSIGN(data_page_size); + ASSIGN(code_page_size); + ASSIGN(cgroup); + ASSIGN(cpumode); + ASSIGN(misc); + ASSIGN(raw_size); + ASSIGN(raw_data); + ASSIGN(machine_pid); + ASSIGN(vcpu); + + if (sample->branch_stack) { + d_sample.brstack_nr = sample->branch_stack->nr; + d_sample.brstack = (struct perf_branch_entry *)perf_sample__branch_entries(sample); + } else { + d_sample.brstack_nr = 0; + d_sample.brstack = NULL; + } + + if (sample->callchain) { + d_sample.raw_callchain_nr = sample->callchain->nr; + d_sample.raw_callchain = (__u64 *)sample->callchain->ips; + } else { + d_sample.raw_callchain_nr = 0; + d_sample.raw_callchain = NULL; + } + + d_sample.addr_correlates_sym = + (evsel->core.attr.sample_type & PERF_SAMPLE_ADDR) && + sample_addr_correlates_sym(&evsel->core.attr); + + d_sample.event = evsel__name(evsel); + + d->ctx_valid = true; + + if (early) + ret = d->filter_event_early(d->data, &d_sample, d); + else + ret = d->filter_event(d->data, &d_sample, d); + + d->ctx_valid = false; + + return ret; +} + +bool get_filter_desc(const char *dirname, const char *name, char **desc, + char **long_desc) +{ + char path[PATH_MAX]; + void *handle; + const char *(*desc_fn)(const char **long_description); + + snprintf(path, sizeof(path), "%s/%s", dirname, name); + handle = dlopen(path, RTLD_NOW); + if (!handle || !(dlsym(handle, "filter_event") || dlsym(handle, "filter_event_early"))) + return false; + desc_fn = dlsym(handle, "filter_description"); + if (desc_fn) { + const char *dsc; + const char *long_dsc; + + dsc = desc_fn(&long_dsc); + if (dsc) + *desc = strdup(dsc); + if (long_dsc) + *long_desc = strdup(long_dsc); + } + dlclose(handle); + return true; +} + +static void list_filters(const char *dirname) +{ + struct dirent *entry; + DIR *dir; + + dir = opendir(dirname); + if (!dir) + return; + + while ((entry = readdir(dir)) != NULL) + { + size_t n = strlen(entry->d_name); + char *long_desc = NULL; + char *desc = NULL; + + if (entry->d_type == DT_DIR || n < 4 || + strcmp(".so", entry->d_name + n - 3)) + continue; + if (!get_filter_desc(dirname, entry->d_name, &desc, &long_desc)) + continue; + printf(" %-36s %s\n", entry->d_name, desc ? desc : ""); + if (verbose > 0) { + char *p = long_desc; + char *line; + + while ((line = strsep(&p, "\n")) != NULL) + printf("%39s%s\n", "", line); + } + free(long_desc); + free(desc); + } + + closedir(dir); +} + +int list_available_dlfilters(const struct option *opt __maybe_unused, + const char *s __maybe_unused, + int unset __maybe_unused) +{ + char path[PATH_MAX]; + char *exec_path; + + printf("List of available dlfilters:\n"); + + list_filters("."); + + exec_path = get_argv_exec_path(); + if (!exec_path) + goto out; + snprintf(path, sizeof(path), "%s/dlfilters", exec_path); + + list_filters(path); + + free(exec_path); +out: + exit(0); +} diff --git a/tools/perf/util/dlfilter.h b/tools/perf/util/dlfilter.h new file mode 100644 index 000000000..cc4bb9657 --- /dev/null +++ b/tools/perf/util/dlfilter.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * dlfilter.h: Interface to perf script --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ + +#ifndef PERF_UTIL_DLFILTER_H +#define PERF_UTIL_DLFILTER_H + +struct perf_session; +union perf_event; +struct perf_sample; +struct evsel; +struct machine; +struct addr_location; +struct perf_dlfilter_fns; +struct perf_dlfilter_sample; +struct perf_dlfilter_al; + +struct dlfilter { + char *file; + void *handle; + void *data; + struct perf_session *session; + bool ctx_valid; + bool in_start; + bool in_stop; + int dlargc; + char **dlargv; + + union perf_event *event; + struct perf_sample *sample; + struct evsel *evsel; + struct machine *machine; + struct addr_location *al; + struct addr_location *addr_al; + struct perf_dlfilter_sample *d_sample; + struct perf_dlfilter_al *d_ip_al; + struct perf_dlfilter_al *d_addr_al; + + int (*start)(void **data, void *ctx); + int (*stop)(void *data, void *ctx); + + int (*filter_event)(void *data, + const struct perf_dlfilter_sample *sample, + void *ctx); + int (*filter_event_early)(void *data, + const struct perf_dlfilter_sample *sample, + void *ctx); + + struct perf_dlfilter_fns *fns; +}; + +struct dlfilter *dlfilter__new(const char *file, int dlargc, char **dlargv); + +int dlfilter__start(struct dlfilter *d, struct perf_session *session); + +int dlfilter__do_filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al, + bool early); + +void dlfilter__cleanup(struct dlfilter *d); + +static inline int dlfilter__filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al) +{ + if (!d || !d->filter_event) + return 0; + return dlfilter__do_filter_event(d, event, sample, evsel, machine, al, addr_al, false); +} + +static inline int dlfilter__filter_event_early(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al) +{ + if (!d || !d->filter_event_early) + return 0; + return dlfilter__do_filter_event(d, event, sample, evsel, machine, al, addr_al, true); +} + +int list_available_dlfilters(const struct option *opt, const char *s, int unset); +bool get_filter_desc(const char *dirname, const char *name, char **desc, + char **long_desc); + +#endif diff --git a/tools/perf/util/drm_pmu.c b/tools/perf/util/drm_pmu.c new file mode 100644 index 000000000..b8badae70 --- /dev/null +++ b/tools/perf/util/drm_pmu.c @@ -0,0 +1,685 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "drm_pmu.h" +#include "counts.h" +#include "cpumap.h" +#include "debug.h" +#include "evsel.h" +#include "pmu.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum drm_pmu_unit { + DRM_PMU_UNIT_BYTES, + DRM_PMU_UNIT_CAPACITY, + DRM_PMU_UNIT_CYCLES, + DRM_PMU_UNIT_HZ, + DRM_PMU_UNIT_NS, + + DRM_PMU_UNIT_MAX, +}; + +struct drm_pmu_event { + const char *name; + const char *desc; + enum drm_pmu_unit unit; +}; + +struct drm_pmu { + struct perf_pmu pmu; + struct drm_pmu_event *events; + int num_events; +}; + +static const char * const drm_pmu_unit_strs[DRM_PMU_UNIT_MAX] = { + "bytes", + "capacity", + "cycles", + "hz", + "ns", +}; + +static const char * const drm_pmu_scale_unit_strs[DRM_PMU_UNIT_MAX] = { + "1bytes", + "1capacity", + "1cycles", + "1hz", + "1ns", +}; + +bool perf_pmu__is_drm(const struct perf_pmu *pmu) +{ + return pmu && pmu->type >= PERF_PMU_TYPE_DRM_START && + pmu->type <= PERF_PMU_TYPE_DRM_END; +} + +bool evsel__is_drm(const struct evsel *evsel) +{ + return perf_pmu__is_drm(evsel->pmu); +} + +static struct drm_pmu *add_drm_pmu(struct list_head *pmus, char *line, size_t line_len) +{ + struct drm_pmu *drm; + struct perf_pmu *pmu; + const char *name; + __u32 max_drm_pmu_type = 0, type; + int i = 12; + + if (line[line_len - 1] == '\n') + line[line_len - 1] = '\0'; + while (isspace(line[i])) + i++; + + line[--i] = '_'; + line[--i] = 'm'; + line[--i] = 'r'; + line[--i] = 'd'; + name = &line[i]; + + list_for_each_entry(pmu, pmus, list) { + if (!perf_pmu__is_drm(pmu)) + continue; + if (pmu->type > max_drm_pmu_type) + max_drm_pmu_type = pmu->type; + if (!strcmp(pmu->name, name)) { + /* PMU already exists. */ + return NULL; + } + } + + if (max_drm_pmu_type != 0) + type = max_drm_pmu_type + 1; + else + type = PERF_PMU_TYPE_DRM_START; + + if (type > PERF_PMU_TYPE_DRM_END) { + zfree(&drm); + pr_err("Unable to encode DRM PMU type for %s\n", name); + return NULL; + } + + drm = zalloc(sizeof(*drm)); + if (!drm) + return NULL; + + if (perf_pmu__init(&drm->pmu, type, name) != 0) { + perf_pmu__delete(&drm->pmu); + return NULL; + } + + drm->pmu.cpus = perf_cpu_map__new_int(0); + if (!drm->pmu.cpus) { + perf_pmu__delete(&drm->pmu); + return NULL; + } + return drm; +} + + +static int add_event(struct drm_pmu_event **events, int *num_events, + const char *line, enum drm_pmu_unit unit, const char *desc) +{ + const char *colon = strchr(line, ':'); + struct drm_pmu_event *tmp; + + if (!colon) + return -EINVAL; + + tmp = reallocarray(*events, *num_events + 1, sizeof(struct drm_pmu_event)); + if (!tmp) + return -ENOMEM; + tmp[*num_events].unit = unit; + tmp[*num_events].desc = desc; + tmp[*num_events].name = strndup(line, colon - line); + if (!tmp[*num_events].name) + return -ENOMEM; + (*num_events)++; + *events = tmp; + return 0; +} + +static int read_drm_pmus_cb(void *args, int fdinfo_dir_fd, const char *fd_name) +{ + struct list_head *pmus = args; + char buf[640]; + struct io io; + char *line = NULL; + size_t line_len; + struct drm_pmu *drm = NULL; + struct drm_pmu_event *events = NULL; + int num_events = 0; + + io__init(&io, openat(fdinfo_dir_fd, fd_name, O_RDONLY), buf, sizeof(buf)); + if (io.fd == -1) { + /* Failed to open file, ignore. */ + return 0; + } + + while (io__getline(&io, &line, &line_len) > 0) { + if (strstarts(line, "drm-driver:")) { + drm = add_drm_pmu(pmus, line, line_len); + if (!drm) + break; + continue; + } + /* + * Note the string matching below is alphabetical, with more + * specific matches appearing before less specific. + */ + if (strstarts(line, "drm-active-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Total memory active in one or more engines"); + continue; + } + if (strstarts(line, "drm-cycles-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_CYCLES, + "Busy cycles"); + continue; + } + if (strstarts(line, "drm-engine-capacity-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_CAPACITY, + "Engine capacity"); + continue; + } + if (strstarts(line, "drm-engine-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_NS, + "Utilization in ns"); + continue; + } + if (strstarts(line, "drm-maxfreq-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_HZ, + "Maximum frequency"); + continue; + } + if (strstarts(line, "drm-purgeable-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of resident and purgeable memory buffers"); + continue; + } + if (strstarts(line, "drm-resident-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of resident memory buffers"); + continue; + } + if (strstarts(line, "drm-shared-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of shared memory buffers"); + continue; + } + if (strstarts(line, "drm-total-cycles-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Total busy cycles"); + continue; + } + if (strstarts(line, "drm-total-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of shared and private memory"); + continue; + } + if (verbose > 1 && strstarts(line, "drm-") && + !strstarts(line, "drm-client-id:") && + !strstarts(line, "drm-pdev:")) + pr_debug("Unhandled DRM PMU fdinfo line match '%s'\n", line); + } + if (drm) { + drm->events = events; + drm->num_events = num_events; + list_add_tail(&drm->pmu.list, pmus); + } + free(line); + if (io.fd != -1) + close(io.fd); + return 0; +} + +void drm_pmu__exit(struct perf_pmu *pmu) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + free(drm->events); +} + +bool drm_pmu__have_event(const struct perf_pmu *pmu, const char *name) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + if (!strstarts(name, "drm-")) + return false; + + for (int i = 0; i < drm->num_events; i++) { + if (!strcasecmp(drm->events[i].name, name)) + return true; + } + return false; +} + +int drm_pmu__for_each_event(const struct perf_pmu *pmu, void *state, pmu_event_callback cb) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + for (int i = 0; i < drm->num_events; i++) { + char encoding_buf[128]; + struct pmu_event_info info = { + .pmu = pmu, + .name = drm->events[i].name, + .alias = NULL, + .scale_unit = drm_pmu_scale_unit_strs[drm->events[i].unit], + .desc = drm->events[i].desc, + .long_desc = NULL, + .encoding_desc = encoding_buf, + .topic = "drm", + .pmu_name = pmu->name, + .event_type_desc = "DRM event", + }; + int ret; + + snprintf(encoding_buf, sizeof(encoding_buf), "%s/config=0x%x/", pmu->name, i); + + ret = cb(state, &info); + if (ret) + return ret; + } + return 0; +} + +size_t drm_pmu__num_events(const struct perf_pmu *pmu) +{ + const struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + return drm->num_events; +} + +static int drm_pmu__index_for_event(const struct drm_pmu *drm, const char *name) +{ + for (int i = 0; i < drm->num_events; i++) { + if (!strcmp(drm->events[i].name, name)) + return i; + } + return -1; +} + +static int drm_pmu__config_term(const struct drm_pmu *drm, + struct perf_event_attr *attr, + struct parse_events_term *term, + struct parse_events_error *err) +{ + if (term->type_term == PARSE_EVENTS__TERM_TYPE_USER) { + int i = drm_pmu__index_for_event(drm, term->config); + + if (i >= 0) { + attr->config = i; + return 0; + } + } + if (err) { + char *err_str; + + parse_events_error__handle(err, term->err_val, + asprintf(&err_str, + "unexpected drm event term (%s) %s", + parse_events__term_type_str(term->type_term), + term->config) < 0 + ? strdup("unexpected drm event term") + : err_str, + NULL); + } + return -EINVAL; +} + +int drm_pmu__config_terms(const struct perf_pmu *pmu, + struct perf_event_attr *attr, + struct parse_events_terms *terms, + struct parse_events_error *err) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + struct parse_events_term *term; + + list_for_each_entry(term, &terms->terms, list) { + if (drm_pmu__config_term(drm, attr, term, err)) + return -EINVAL; + } + + return 0; +} + +int drm_pmu__check_alias(const struct perf_pmu *pmu, struct parse_events_terms *terms, + struct perf_pmu_info *info, struct parse_events_error *err) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + struct parse_events_term *term = + list_first_entry(&terms->terms, struct parse_events_term, list); + + if (term->type_term == PARSE_EVENTS__TERM_TYPE_USER) { + int i = drm_pmu__index_for_event(drm, term->config); + + if (i >= 0) { + info->unit = drm_pmu_unit_strs[drm->events[i].unit]; + info->scale = 1; + return 0; + } + } + if (err) { + char *err_str; + + parse_events_error__handle(err, term->err_val, + asprintf(&err_str, + "unexpected drm event term (%s) %s", + parse_events__term_type_str(term->type_term), + term->config) < 0 + ? strdup("unexpected drm event term") + : err_str, + NULL); + } + return -EINVAL; +} + +struct minor_info { + unsigned int *minors; + int minors_num, minors_len; +}; + +static int for_each_drm_fdinfo_in_dir(int (*cb)(void *args, int fdinfo_dir_fd, const char *fd_name), + void *args, int proc_dir, const char *pid_name, + struct minor_info *minors) +{ + char buf[256]; + DIR *fd_dir; + struct dirent *fd_entry; + int fd_dir_fd, fdinfo_dir_fd = -1; + + + scnprintf(buf, sizeof(buf), "%s/fd", pid_name); + fd_dir_fd = openat(proc_dir, buf, O_DIRECTORY); + if (fd_dir_fd == -1) + return 0; /* Presumably lost race to open. */ + fd_dir = fdopendir(fd_dir_fd); + if (!fd_dir) { + close(fd_dir_fd); + return -ENOMEM; + } + while ((fd_entry = readdir(fd_dir)) != NULL) { + struct stat stat; + unsigned int minor; + bool is_dup = false; + int ret; + + if (fd_entry->d_type != DT_LNK) + continue; + + if (fstatat(fd_dir_fd, fd_entry->d_name, &stat, 0) != 0) + continue; + + if ((stat.st_mode & S_IFMT) != S_IFCHR || major(stat.st_rdev) != 226) + continue; + + minor = minor(stat.st_rdev); + for (int i = 0; i < minors->minors_num; i++) { + if (minor(stat.st_rdev) == minors->minors[i]) { + is_dup = true; + break; + } + } + if (is_dup) + continue; + + if (minors->minors_num == minors->minors_len) { + unsigned int *tmp = reallocarray(minors->minors, minors->minors_len + 4, + sizeof(unsigned int)); + + if (tmp) { + minors->minors = tmp; + minors->minors_len += 4; + } + } + minors->minors[minors->minors_num++] = minor; + if (fdinfo_dir_fd == -1) { + /* Open fdinfo dir if we have a DRM fd. */ + scnprintf(buf, sizeof(buf), "%s/fdinfo", pid_name); + fdinfo_dir_fd = openat(proc_dir, buf, O_DIRECTORY); + if (fdinfo_dir_fd == -1) + continue; + } + ret = cb(args, fdinfo_dir_fd, fd_entry->d_name); + if (ret) + goto close_fdinfo; + } + +close_fdinfo: + if (fdinfo_dir_fd != -1) + close(fdinfo_dir_fd); + closedir(fd_dir); + return 0; +} + +static int for_each_drm_fdinfo(bool skip_all_duplicates, + int (*cb)(void *args, int fdinfo_dir_fd, const char *fd_name), + void *args) +{ + DIR *proc_dir; + struct dirent *proc_entry; + int ret; + /* + * minors maintains an array of DRM minor device numbers seen for a pid, + * or for all pids if skip_all_duplicates is true, so that duplicates + * are ignored. + */ + struct minor_info minors = { + .minors = NULL, + .minors_num = 0, + .minors_len = 0, + }; + + proc_dir = opendir(procfs__mountpoint()); + if (!proc_dir) + return 0; + + /* Walk through the /proc directory. */ + while ((proc_entry = readdir(proc_dir)) != NULL) { + if (proc_entry->d_type != DT_DIR || + !isdigit(proc_entry->d_name[0])) + continue; + if (!skip_all_duplicates) { + /* Reset the seen minor numbers for each pid. */ + minors.minors_num = 0; + } + ret = for_each_drm_fdinfo_in_dir(cb, args, + dirfd(proc_dir), proc_entry->d_name, + &minors); + if (ret) + break; + } + free(minors.minors); + closedir(proc_dir); + return ret; +} + +int perf_pmus__read_drm_pmus(struct list_head *pmus) +{ + return for_each_drm_fdinfo(/*skip_all_duplicates=*/true, read_drm_pmus_cb, pmus); +} + +int evsel__drm_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx) +{ + (void)evsel; + (void)threads; + (void)start_cpu_map_idx; + (void)end_cpu_map_idx; + return 0; +} + +static uint64_t read_count_and_apply_unit(const char *count_and_unit, enum drm_pmu_unit unit) +{ + char *unit_ptr = NULL; + uint64_t count = strtoul(count_and_unit, &unit_ptr, 10); + + if (!unit_ptr) + return 0; + + while (isblank(*unit_ptr)) + unit_ptr++; + + switch (unit) { + case DRM_PMU_UNIT_BYTES: + if (*unit_ptr == '\0') + assert(count == 0); /* Generally undocumented, happens for 0. */ + else if (!strcmp(unit_ptr, "KiB")) + count *= 1024; + else if (!strcmp(unit_ptr, "MiB")) + count *= 1024 * 1024; + else + pr_err("Unexpected bytes unit '%s'\n", unit_ptr); + break; + case DRM_PMU_UNIT_CAPACITY: + /* No units expected. */ + break; + case DRM_PMU_UNIT_CYCLES: + /* No units expected. */ + break; + case DRM_PMU_UNIT_HZ: + if (!strcmp(unit_ptr, "Hz")) + count *= 1; + else if (!strcmp(unit_ptr, "KHz")) + count *= 1000; + else if (!strcmp(unit_ptr, "MHz")) + count *= 1000000; + else + pr_err("Unexpected hz unit '%s'\n", unit_ptr); + break; + case DRM_PMU_UNIT_NS: + /* Only unit ns expected. */ + break; + case DRM_PMU_UNIT_MAX: + default: + break; + } + return count; +} + +static uint64_t read_drm_event(int fdinfo_dir_fd, const char *fd_name, + const char *match, enum drm_pmu_unit unit) +{ + char buf[640]; + struct io io; + char *line = NULL; + size_t line_len; + uint64_t count = 0; + + io__init(&io, openat(fdinfo_dir_fd, fd_name, O_RDONLY), buf, sizeof(buf)); + if (io.fd == -1) { + /* Failed to open file, ignore. */ + return 0; + } + while (io__getline(&io, &line, &line_len) > 0) { + size_t i = strlen(match); + + if (strncmp(line, match, i)) + continue; + if (line[i] != ':') + continue; + while (isblank(line[++i])) + ; + if (line[line_len - 1] == '\n') + line[line_len - 1] = '\0'; + count = read_count_and_apply_unit(&line[i], unit); + break; + } + free(line); + close(io.fd); + return count; +} + +struct read_drm_event_cb_args { + const char *match; + uint64_t count; + enum drm_pmu_unit unit; +}; + +static int read_drm_event_cb(void *vargs, int fdinfo_dir_fd, const char *fd_name) +{ + struct read_drm_event_cb_args *args = vargs; + + args->count += read_drm_event(fdinfo_dir_fd, fd_name, args->match, args->unit); + return 0; +} + +static uint64_t drm_pmu__read_system_wide(struct drm_pmu *drm, struct evsel *evsel) +{ + struct read_drm_event_cb_args args = { + .count = 0, + .match = drm->events[evsel->core.attr.config].name, + .unit = drm->events[evsel->core.attr.config].unit, + }; + + for_each_drm_fdinfo(/*skip_all_duplicates=*/false, read_drm_event_cb, &args); + return args.count; +} + +static uint64_t drm_pmu__read_for_pid(struct drm_pmu *drm, struct evsel *evsel, int pid) +{ + struct read_drm_event_cb_args args = { + .count = 0, + .match = drm->events[evsel->core.attr.config].name, + .unit = drm->events[evsel->core.attr.config].unit, + }; + struct minor_info minors = { + .minors = NULL, + .minors_num = 0, + .minors_len = 0, + }; + int proc_dir = open(procfs__mountpoint(), O_DIRECTORY); + char pid_name[12]; + int ret; + + if (proc_dir < 0) + return 0; + + snprintf(pid_name, sizeof(pid_name), "%d", pid); + ret = for_each_drm_fdinfo_in_dir(read_drm_event_cb, &args, proc_dir, pid_name, &minors); + free(minors.minors); + close(proc_dir); + return ret == 0 ? args.count : 0; +} + +int evsel__drm_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) +{ + struct drm_pmu *drm = container_of(evsel->pmu, struct drm_pmu, pmu); + struct perf_counts_values *count, *old_count = NULL; + int pid = perf_thread_map__pid(evsel->core.threads, thread); + uint64_t counter; + + if (pid != -1) + counter = drm_pmu__read_for_pid(drm, evsel, pid); + else + counter = drm_pmu__read_system_wide(drm, evsel); + + if (evsel->prev_raw_counts) + old_count = perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); + + count = perf_counts(evsel->counts, cpu_map_idx, thread); + if (old_count) { + count->val = old_count->val + counter; + count->run = old_count->run + 1; + count->ena = old_count->ena + 1; + } else { + count->val = counter; + count->run++; + count->ena++; + } + return 0; +} diff --git a/tools/perf/util/drm_pmu.h b/tools/perf/util/drm_pmu.h new file mode 100644 index 000000000..e7f366fca --- /dev/null +++ b/tools/perf/util/drm_pmu.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __DRM_PMU_H +#define __DRM_PMU_H +/* + * Linux DRM clients expose information through usage stats as documented in + * Documentation/gpu/drm-usage-stats.rst (available online at + * https://docs.kernel.org/gpu/drm-usage-stats.html). This is a tool like PMU + * that exposes DRM information. + */ + +#include "pmu.h" +#include + +struct list_head; +struct perf_thread_map; + +void drm_pmu__exit(struct perf_pmu *pmu); +bool drm_pmu__have_event(const struct perf_pmu *pmu, const char *name); +int drm_pmu__for_each_event(const struct perf_pmu *pmu, void *state, pmu_event_callback cb); +size_t drm_pmu__num_events(const struct perf_pmu *pmu); +int drm_pmu__config_terms(const struct perf_pmu *pmu, + struct perf_event_attr *attr, + struct parse_events_terms *terms, + struct parse_events_error *err); +int drm_pmu__check_alias(const struct perf_pmu *pmu, struct parse_events_terms *terms, + struct perf_pmu_info *info, struct parse_events_error *err); + + +bool perf_pmu__is_drm(const struct perf_pmu *pmu); +bool evsel__is_drm(const struct evsel *evsel); + +int perf_pmus__read_drm_pmus(struct list_head *pmus); + +int evsel__drm_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx); +int evsel__drm_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread); + +#endif /* __DRM_PMU_H */ diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c new file mode 100644 index 000000000..42bfe30a3 --- /dev/null +++ b/tools/perf/util/dso.c @@ -0,0 +1,2077 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBBPF_SUPPORT +#include +#include "bpf-event.h" +#include "bpf-utils.h" +#endif +#include "compress.h" +#include "env.h" +#include "namespaces.h" +#include "path.h" +#include "map.h" +#include "symbol.h" +#include "srcline.h" +#include "dso.h" +#include "dsos.h" +#include "machine.h" +#include "auxtrace.h" +#include "util.h" /* O_CLOEXEC for older systems */ +#include "debug.h" +#include "string2.h" +#include "vdso.h" +#include "annotate-data.h" +#include "libdw.h" + +static const char * const debuglink_paths[] = { + "%.0s%s", + "%s/%s", + "%s/.debug/%s", + "/usr/lib/debug%s/%s" +}; + +void dso__set_nsinfo(struct dso *dso, struct nsinfo *nsi) +{ + nsinfo__put(RC_CHK_ACCESS(dso)->nsinfo); + RC_CHK_ACCESS(dso)->nsinfo = nsi; +} + +char dso__symtab_origin(const struct dso *dso) +{ + static const char origin[] = { + [DSO_BINARY_TYPE__KALLSYMS] = 'k', + [DSO_BINARY_TYPE__VMLINUX] = 'v', + [DSO_BINARY_TYPE__JAVA_JIT] = 'j', + [DSO_BINARY_TYPE__DEBUGLINK] = 'l', + [DSO_BINARY_TYPE__BUILD_ID_CACHE] = 'B', + [DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO] = 'D', + [DSO_BINARY_TYPE__FEDORA_DEBUGINFO] = 'f', + [DSO_BINARY_TYPE__UBUNTU_DEBUGINFO] = 'u', + [DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO] = 'x', + [DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO] = 'o', + [DSO_BINARY_TYPE__BUILDID_DEBUGINFO] = 'b', + [DSO_BINARY_TYPE__SYSTEM_PATH_DSO] = 'd', + [DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE] = 'K', + [DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP] = 'm', + [DSO_BINARY_TYPE__GUEST_KALLSYMS] = 'g', + [DSO_BINARY_TYPE__GUEST_KMODULE] = 'G', + [DSO_BINARY_TYPE__GUEST_KMODULE_COMP] = 'M', + [DSO_BINARY_TYPE__GUEST_VMLINUX] = 'V', + [DSO_BINARY_TYPE__GNU_DEBUGDATA] = 'n', + }; + + if (dso == NULL || dso__symtab_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) + return '!'; + return origin[dso__symtab_type(dso)]; +} + +bool dso__is_object_file(const struct dso *dso) +{ + switch (dso__binary_type(dso)) { + case DSO_BINARY_TYPE__KALLSYMS: + case DSO_BINARY_TYPE__GUEST_KALLSYMS: + case DSO_BINARY_TYPE__JAVA_JIT: + case DSO_BINARY_TYPE__BPF_PROG_INFO: + case DSO_BINARY_TYPE__BPF_IMAGE: + case DSO_BINARY_TYPE__OOL: + return false; + case DSO_BINARY_TYPE__VMLINUX: + case DSO_BINARY_TYPE__GUEST_VMLINUX: + case DSO_BINARY_TYPE__DEBUGLINK: + case DSO_BINARY_TYPE__BUILD_ID_CACHE: + case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: + case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: + case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: + case DSO_BINARY_TYPE__GNU_DEBUGDATA: + case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: + case DSO_BINARY_TYPE__GUEST_KMODULE: + case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: + case DSO_BINARY_TYPE__KCORE: + case DSO_BINARY_TYPE__GUEST_KCORE: + case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: + case DSO_BINARY_TYPE__NOT_FOUND: + default: + return true; + } +} + +int dso__read_binary_type_filename(const struct dso *dso, + enum dso_binary_type type, + const char *root_dir, char *filename, size_t size) +{ + char build_id_hex[SBUILD_ID_SIZE]; + int ret = 0; + size_t len; + + switch (type) { + case DSO_BINARY_TYPE__DEBUGLINK: + { + const char *last_slash; + char dso_dir[PATH_MAX]; + char symfile[PATH_MAX]; + unsigned int i; + + len = __symbol__join_symfs(filename, size, dso__long_name(dso)); + last_slash = filename + len; + while (last_slash != filename && *last_slash != '/') + last_slash--; + + strncpy(dso_dir, filename, last_slash - filename); + dso_dir[last_slash-filename] = '\0'; + + if (!is_regular_file(filename)) { + ret = -1; + break; + } + + ret = filename__read_debuglink(filename, symfile, PATH_MAX); + if (ret) + break; + + /* Check predefined locations where debug file might reside */ + ret = -1; + for (i = 0; i < ARRAY_SIZE(debuglink_paths); i++) { + snprintf(filename, size, + debuglink_paths[i], dso_dir, symfile); + if (is_regular_file(filename)) { + ret = 0; + break; + } + } + + break; + } + case DSO_BINARY_TYPE__BUILD_ID_CACHE: + if (dso__build_id_filename(dso, filename, size, false) == NULL) + ret = -1; + break; + + case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: + if (dso__build_id_filename(dso, filename, size, true) == NULL) + ret = -1; + break; + + case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: + len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); + snprintf(filename + len, size - len, "%s.debug", dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: + len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); + snprintf(filename + len, size - len, "%s", dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: + /* + * Ubuntu can mixup /usr/lib with /lib, putting debuginfo in + * /usr/lib/debug/lib when it is expected to be in + * /usr/lib/debug/usr/lib + */ + if (strlen(dso__long_name(dso)) < 9 || + strncmp(dso__long_name(dso), "/usr/lib/", 9)) { + ret = -1; + break; + } + len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); + snprintf(filename + len, size - len, "%s", dso__long_name(dso) + 4); + break; + + case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: + { + const char *last_slash; + size_t dir_size; + + last_slash = dso__long_name(dso) + dso__long_name_len(dso); + while (last_slash != dso__long_name(dso) && *last_slash != '/') + last_slash--; + + len = __symbol__join_symfs(filename, size, ""); + dir_size = last_slash - dso__long_name(dso) + 2; + if (dir_size > (size - len)) { + ret = -1; + break; + } + len += scnprintf(filename + len, dir_size, "%s", dso__long_name(dso)); + len += scnprintf(filename + len , size - len, ".debug%s", + last_slash); + break; + } + + case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: + if (!dso__has_build_id(dso)) { + ret = -1; + break; + } + + build_id__snprintf(dso__bid(dso), build_id_hex, sizeof(build_id_hex)); + len = __symbol__join_symfs(filename, size, "/usr/lib/debug/.build-id/"); + snprintf(filename + len, size - len, "%.2s/%s.debug", + build_id_hex, build_id_hex + 2); + break; + + case DSO_BINARY_TYPE__VMLINUX: + case DSO_BINARY_TYPE__GUEST_VMLINUX: + case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: + case DSO_BINARY_TYPE__GNU_DEBUGDATA: + __symbol__join_symfs(filename, size, dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__GUEST_KMODULE: + case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: + path__join3(filename, size, symbol_conf.symfs, + root_dir, dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: + __symbol__join_symfs(filename, size, dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__KCORE: + case DSO_BINARY_TYPE__GUEST_KCORE: + snprintf(filename, size, "%s", dso__long_name(dso)); + break; + + default: + case DSO_BINARY_TYPE__KALLSYMS: + case DSO_BINARY_TYPE__GUEST_KALLSYMS: + case DSO_BINARY_TYPE__JAVA_JIT: + case DSO_BINARY_TYPE__BPF_PROG_INFO: + case DSO_BINARY_TYPE__BPF_IMAGE: + case DSO_BINARY_TYPE__OOL: + case DSO_BINARY_TYPE__NOT_FOUND: + ret = -1; + break; + } + + return ret; +} + +enum { + COMP_ID__NONE = 0, +}; + +static const struct { + const char *fmt; + int (*decompress)(const char *input, int output); + bool (*is_compressed)(const char *input); +} compressions[] = { + [COMP_ID__NONE] = { .fmt = NULL, }, +#ifdef HAVE_ZLIB_SUPPORT + { "gz", gzip_decompress_to_file, gzip_is_compressed }, +#endif +#ifdef HAVE_LZMA_SUPPORT + { "xz", lzma_decompress_to_file, lzma_is_compressed }, +#endif + { NULL, NULL, NULL }, +}; + +static int is_supported_compression(const char *ext) +{ + unsigned i; + + for (i = 1; compressions[i].fmt; i++) { + if (!strcmp(ext, compressions[i].fmt)) + return i; + } + return COMP_ID__NONE; +} + +bool is_kernel_module(const char *pathname, int cpumode) +{ + struct kmod_path m; + int mode = cpumode & PERF_RECORD_MISC_CPUMODE_MASK; + + WARN_ONCE(mode != cpumode, + "Internal error: passing unmasked cpumode (%x) to is_kernel_module", + cpumode); + + switch (mode) { + case PERF_RECORD_MISC_USER: + case PERF_RECORD_MISC_HYPERVISOR: + case PERF_RECORD_MISC_GUEST_USER: + return false; + /* Treat PERF_RECORD_MISC_CPUMODE_UNKNOWN as kernel */ + default: + if (kmod_path__parse(&m, pathname)) { + pr_err("Failed to check whether %s is a kernel module or not. Assume it is.", + pathname); + return true; + } + } + + return m.kmod; +} + +bool dso__needs_decompress(struct dso *dso) +{ + return dso__symtab_type(dso) == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP || + dso__symtab_type(dso) == DSO_BINARY_TYPE__GUEST_KMODULE_COMP; +} + +int filename__decompress(const char *name, char *pathname, + size_t len, int comp, int *err) +{ + char tmpbuf[] = KMOD_DECOMP_NAME; + int fd = -1; + + /* + * We have proper compression id for DSO and yet the file + * behind the 'name' can still be plain uncompressed object. + * + * The reason is behind the logic we open the DSO object files, + * when we try all possible 'debug' objects until we find the + * data. So even if the DSO is represented by 'krava.xz' module, + * we can end up here opening ~/.debug/....23432432/debug' file + * which is not compressed. + * + * To keep this transparent, we detect this and return the file + * descriptor to the uncompressed file. + */ + if (!compressions[comp].is_compressed(name)) { + fd = open(name, O_RDONLY | O_CLOEXEC); + if (fd < 0) + *err = errno; + if (pathname && len > 0) + pathname[0] = '\0'; + return fd; + } + + fd = mkostemp(tmpbuf, O_CLOEXEC); + if (fd < 0) { + *err = errno; + return -1; + } + + if (compressions[comp].decompress(name, fd)) { + *err = DSO_LOAD_ERRNO__DECOMPRESSION_FAILURE; + close(fd); + fd = -1; + } + + if (!pathname || (fd < 0)) + unlink(tmpbuf); + + if (pathname && (fd >= 0)) + strlcpy(pathname, tmpbuf, len); + + return fd; +} + +static int decompress_kmodule(struct dso *dso, const char *name, + char *pathname, size_t len) +{ + if (!dso__needs_decompress(dso)) + return -1; + + if (dso__comp(dso) == COMP_ID__NONE) + return -1; + + return filename__decompress(name, pathname, len, dso__comp(dso), dso__load_errno(dso)); +} + +int dso__decompress_kmodule_fd(struct dso *dso, const char *name) +{ + return decompress_kmodule(dso, name, NULL, 0); +} + +int dso__decompress_kmodule_path(struct dso *dso, const char *name, + char *pathname, size_t len) +{ + int fd = decompress_kmodule(dso, name, pathname, len); + + /* decompress_kmodule() returns -1 on failure, don't close(-1) */ + if (fd >= 0) + close(fd); + return fd >= 0 ? 0 : -1; +} + +/* + * Parses kernel module specified in @path and updates + * @m argument like: + * + * @comp - true if @path contains supported compression suffix, + * false otherwise + * @kmod - true if @path contains '.ko' suffix in right position, + * false otherwise + * @name - if (@alloc_name && @kmod) is true, it contains strdup-ed base name + * of the kernel module without suffixes, otherwise strudup-ed + * base name of @path + * @ext - if (@alloc_ext && @comp) is true, it contains strdup-ed string + * the compression suffix + * + * Returns 0 if there's no strdup error, -ENOMEM otherwise. + */ +int __kmod_path__parse(struct kmod_path *m, const char *path, + bool alloc_name) +{ + const char *name = strrchr(path, '/'); + const char *ext = strrchr(path, '.'); + bool is_simple_name = false; + + memset(m, 0x0, sizeof(*m)); + name = name ? name + 1 : path; + + /* + * '.' is also a valid character for module name. For example: + * [aaa.bbb] is a valid module name. '[' should have higher + * priority than '.ko' suffix. + * + * The kernel names are from machine__mmap_name. Such + * name should belong to kernel itself, not kernel module. + */ + if (name[0] == '[') { + is_simple_name = true; + if ((strncmp(name, "[kernel.kallsyms]", 17) == 0) || + (strncmp(name, "[guest.kernel.kallsyms", 22) == 0) || + (strncmp(name, "[vdso]", 6) == 0) || + (strncmp(name, "[vdso32]", 8) == 0) || + (strncmp(name, "[vdsox32]", 9) == 0) || + (strncmp(name, "[vsyscall]", 10) == 0)) { + m->kmod = false; + + } else + m->kmod = true; + } + + /* No extension, just return name. */ + if ((ext == NULL) || is_simple_name) { + if (alloc_name) { + m->name = strdup(name); + return m->name ? 0 : -ENOMEM; + } + return 0; + } + + m->comp = is_supported_compression(ext + 1); + if (m->comp > COMP_ID__NONE) + ext -= 3; + + /* Check .ko extension only if there's enough name left. */ + if (ext > name) + m->kmod = !strncmp(ext, ".ko", 3); + + if (alloc_name) { + if (m->kmod) { + if (asprintf(&m->name, "[%.*s]", (int) (ext - name), name) == -1) + return -ENOMEM; + } else { + if (asprintf(&m->name, "%s", name) == -1) + return -ENOMEM; + } + + strreplace(m->name, '-', '_'); + } + + return 0; +} + +void dso__set_module_info(struct dso *dso, struct kmod_path *m, + struct machine *machine) +{ + if (machine__is_host(machine)) + dso__set_symtab_type(dso, DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE); + else + dso__set_symtab_type(dso, DSO_BINARY_TYPE__GUEST_KMODULE); + + /* _KMODULE_COMP should be next to _KMODULE */ + if (m->kmod && m->comp) { + dso__set_symtab_type(dso, dso__symtab_type(dso) + 1); + dso__set_comp(dso, m->comp); + } + + dso__set_is_kmod(dso); + dso__set_short_name(dso, strdup(m->name), true); +} + +/* + * Global list of open DSOs and the counter. + */ +struct mutex _dso__data_open_lock; +static LIST_HEAD(dso__data_open); +static long dso__data_open_cnt GUARDED_BY(_dso__data_open_lock); + +static void dso__data_open_lock_init(void) +{ + mutex_init(&_dso__data_open_lock); +} + +static struct mutex *dso__data_open_lock(void) LOCK_RETURNED(_dso__data_open_lock) +{ + static pthread_once_t data_open_lock_once = PTHREAD_ONCE_INIT; + + pthread_once(&data_open_lock_once, dso__data_open_lock_init); + + return &_dso__data_open_lock; +} + +static void dso__list_add(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + list_add_tail(&dso__data(dso)->open_entry, &dso__data_open); +#ifdef REFCNT_CHECKING + dso__data(dso)->dso = dso__get(dso); +#endif + /* Assume the dso is part of dsos, hence the optional reference count above. */ + assert(dso__dsos(dso)); + dso__data_open_cnt++; +} + +static void dso__list_del(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + list_del_init(&dso__data(dso)->open_entry); +#ifdef REFCNT_CHECKING + mutex_unlock(dso__data_open_lock()); + dso__put(dso__data(dso)->dso); + mutex_lock(dso__data_open_lock()); +#endif + WARN_ONCE(dso__data_open_cnt <= 0, + "DSO data fd counter out of bounds."); + dso__data_open_cnt--; +} + +static void close_first_dso(void); + +static int do_open(char *name) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + do { + int fd = open(name, O_RDONLY|O_CLOEXEC); + + if (fd >= 0) + return fd; + + pr_debug("dso open failed: %m\n"); + if (!dso__data_open_cnt || errno != EMFILE) + break; + + close_first_dso(); + } while (1); + + return -1; +} + +char *dso__filename_with_chroot(const struct dso *dso, const char *filename) +{ + return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename); +} + +static char *dso__get_filename(struct dso *dso, const char *root_dir, + bool *decomp) +{ + char *name = malloc(PATH_MAX); + + *decomp = false; + + if (name == NULL) + return NULL; + + if (dso__read_binary_type_filename(dso, dso__binary_type(dso), + root_dir, name, PATH_MAX)) + goto out; + + if (!is_regular_file(name)) { + struct stat st; + char *new_name; + + /* + * errno only reflects the failure reason when stat() itself + * failed: a successful stat() on a non-regular file (e.g. a + * directory) leaves a stale errno, which a previous failed + * iteration of the try_to_open_dso() fallback loop may have + * set to ENOENT. + */ + if (stat(name, &st) == 0 || errno != ENOENT || + dso__nsinfo(dso) == NULL) + goto out; + + new_name = dso__filename_with_chroot(dso, name); + if (!new_name) + goto out; + + free(name); + name = new_name; + } + + if (dso__needs_decompress(dso)) { + char newpath[KMOD_DECOMP_LEN]; + size_t len = sizeof(newpath); + + if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { + /* + * Use a standard errno value, not the negative custom + * DSO_LOAD_ERRNO stored in dso__load_errno(dso): + * __open_dso() computes fd = -errno, so a negative + * errno produces a positive fd that looks valid. + */ + errno = EIO; + goto out; + } + + /* empty pathname means file wasn't actually compressed */ + if (newpath[0] != '\0') { + char *tmp = strdup(newpath); + + if (!tmp) { + unlink(newpath); + goto out; + } + free(name); + name = tmp; + *decomp = true; + } + } + return name; + +out: + free(name); + return NULL; +} + +static int __open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd = -EINVAL; + char *name; + bool decomp = false; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); + if (name) { + fd = do_open(name); + } else { + if (errno == 0) + errno = ENOENT; + fd = -errno; + } + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return fd; +} + +static void check_data_close(void); + +/** + * dso_close - Open DSO data file + * @dso: dso object + * + * Open @dso's data file descriptor and updates + * list/count of open DSO objects. + */ +static int open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd; + struct nscookie nsc; + + if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) { + mutex_lock(dso__lock(dso)); + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + mutex_unlock(dso__lock(dso)); + } + fd = __open_dso(dso, machine); + if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) + nsinfo__mountns_exit(&nsc); + + if (fd >= 0) { + dso__list_add(dso); + /* + * Check if we crossed the allowed number + * of opened DSOs and close one if needed. + */ + check_data_close(); + } + + return fd; +} + +static void close_data_fd(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + if (dso__data(dso)->fd >= 0) { + close(dso__data(dso)->fd); + dso__data(dso)->fd = -1; + dso__data(dso)->file_size = 0; + dso__list_del(dso); + } +} + +/** + * dso_close - Close DSO data file + * @dso: dso object + * + * Close @dso's data file descriptor and updates + * list/count of open DSO objects. + */ +static void close_dso(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + close_data_fd(dso); +} + +static void close_first_dso(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + struct dso_data *dso_data; + struct dso *dso; + + dso_data = list_first_entry(&dso__data_open, struct dso_data, open_entry); +#ifdef REFCNT_CHECKING + dso = dso_data->dso; +#else + dso = container_of(dso_data, struct dso, data); +#endif + close_dso(dso); +} + +static rlim_t get_fd_limit(void) +{ + struct rlimit l; + rlim_t limit = 0; + + /* Allow half of the current open fd limit. */ + if (getrlimit(RLIMIT_NOFILE, &l) == 0) { + if (l.rlim_cur == RLIM_INFINITY) + limit = l.rlim_cur; + else + limit = l.rlim_cur / 2; + } else { + pr_err("failed to get fd limit\n"); + limit = 1; + } + + return limit; +} + +static rlim_t fd_limit; + +/* + * Used only by tests/dso-data.c to reset the environment + * for tests. I dont expect we should change this during + * standard runtime. + */ +void reset_fd_limit(void) +{ + fd_limit = 0; +} + +static bool may_cache_fd(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + if (!fd_limit) + fd_limit = get_fd_limit(); + + if (fd_limit == RLIM_INFINITY) + return true; + + return fd_limit > (rlim_t) dso__data_open_cnt; +} + +/* + * Check and close LRU dso if we crossed allowed limit + * for opened dso file descriptors. The limit is half + * of the RLIMIT_NOFILE files opened. +*/ +static void check_data_close(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + bool cache_fd = may_cache_fd(); + + if (!cache_fd) + close_first_dso(); +} + +/** + * dso__data_close - Close DSO data file + * @dso: dso object + * + * External interface to close @dso's data file descriptor. + */ +void dso__data_close(struct dso *dso) +{ + mutex_lock(dso__data_open_lock()); + close_dso(dso); + mutex_unlock(dso__data_open_lock()); +} + +static void try_to_open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + enum dso_binary_type binary_type_data[] = { + DSO_BINARY_TYPE__BUILD_ID_CACHE, + DSO_BINARY_TYPE__SYSTEM_PATH_DSO, + DSO_BINARY_TYPE__NOT_FOUND, + }; + int i = 0; + struct dso_data *dso_data = dso__data(dso); + + if (dso_data->fd >= 0) + return; + + if (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND) { + dso_data->fd = open_dso(dso, machine); + goto out; + } + + do { + dso__set_binary_type(dso, binary_type_data[i++]); + + dso_data->fd = open_dso(dso, machine); + if (dso_data->fd >= 0) + goto out; + + } while (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND); +out: + if (dso_data->fd >= 0) + dso_data->status = DSO_DATA_STATUS_OK; + else + dso_data->status = DSO_DATA_STATUS_ERROR; +} + +/** + * dso__data_get_fd - Get dso's data file descriptor + * @dso: dso object + * @machine: machine object + * + * External interface to find dso's file, open it and + * returns file descriptor. It should be paired with + * dso__data_put_fd() if it returns non-negative value. + */ +bool dso__data_get_fd(struct dso *dso, struct machine *machine, int *fd) +{ + *fd = -1; + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return false; + + mutex_lock(dso__data_open_lock()); + + try_to_open_dso(dso, machine); + + *fd = dso__data(dso)->fd; + if (*fd >= 0) + return true; + + mutex_unlock(dso__data_open_lock()); + return false; +} + +void dso__data_put_fd(struct dso *dso __maybe_unused) +{ + mutex_unlock(dso__data_open_lock()); +} + +bool dso__data_status_seen(struct dso *dso, enum dso_data_status_seen by) +{ + u32 flag = 1 << by; + + if (dso__data(dso)->status_seen & flag) + return true; + + dso__data(dso)->status_seen |= flag; + + return false; +} + +#ifdef HAVE_LIBBPF_SUPPORT +static ssize_t bpf_read(struct dso *dso, u64 offset, char *data) +{ + struct bpf_prog_info_node *node; + ssize_t size = DSO__DATA_CACHE_SIZE; + struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso); + u64 len; + u8 *buf; + + node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id); + if (!node || !node->info_linear) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(node->info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + + len = node->info_linear->info.jited_prog_len; + buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns; + + if (offset >= len) + return -1; + + size = (ssize_t)min(len - offset, (u64)size); + memcpy(data, buf + offset, size); + return size; +} + +static int bpf_size(struct dso *dso) +{ + struct bpf_prog_info_node *node; + struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso); + + node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id); + if (!node || !node->info_linear) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + + dso__data(dso)->file_size = node->info_linear->info.jited_prog_len; + return 0; +} +#endif // HAVE_LIBBPF_SUPPORT + +static void +dso_cache__free(struct dso *dso) +{ + struct rb_root *root = &dso__data(dso)->cache; + struct rb_node *next = rb_first(root); + + mutex_lock(dso__lock(dso)); + while (next) { + struct dso_cache *cache; + + cache = rb_entry(next, struct dso_cache, rb_node); + next = rb_next(&cache->rb_node); + rb_erase(&cache->rb_node, root); + free(cache); + } + mutex_unlock(dso__lock(dso)); +} + +static struct dso_cache *__dso_cache__find(struct dso *dso, u64 offset) +{ + const struct rb_root *root = &dso__data(dso)->cache; + struct rb_node * const *p = &root->rb_node; + const struct rb_node *parent = NULL; + struct dso_cache *cache; + + while (*p != NULL) { + u64 end; + + parent = *p; + cache = rb_entry(parent, struct dso_cache, rb_node); + end = cache->offset + DSO__DATA_CACHE_SIZE; + + if (offset < cache->offset) + p = &(*p)->rb_left; + else if (offset >= end) + p = &(*p)->rb_right; + else + return cache; + } + + return NULL; +} + +static struct dso_cache * +dso_cache__insert(struct dso *dso, struct dso_cache *new) +{ + struct rb_root *root = &dso__data(dso)->cache; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct dso_cache *cache; + u64 offset = new->offset; + + mutex_lock(dso__lock(dso)); + while (*p != NULL) { + u64 end; + + parent = *p; + cache = rb_entry(parent, struct dso_cache, rb_node); + end = cache->offset + DSO__DATA_CACHE_SIZE; + + if (offset < cache->offset) + p = &(*p)->rb_left; + else if (offset >= end) + p = &(*p)->rb_right; + else + goto out; + } + + rb_link_node(&new->rb_node, parent, p); + rb_insert_color(&new->rb_node, root); + + cache = NULL; +out: + mutex_unlock(dso__lock(dso)); + return cache; +} + +static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data, + u64 size, bool out) +{ + u64 cache_offset = offset - cache->offset; + u64 cache_size; + + /* + * The RB tree matches using DSO__DATA_CACHE_SIZE, but a short + * pread may leave cache->size smaller. For a regular file a + * short pread only happens at end-of-file, so an offset past + * the valid data is EOF: return 0, matching what a direct + * pread() at that offset would return, and cached_io() then + * stops its read loop. + */ + if (cache_offset >= cache->size) + return 0; + + cache_size = min(cache->size - cache_offset, size); + + if (out) + memcpy(data, cache->data + cache_offset, cache_size); + else + memcpy(cache->data + cache_offset, data, cache_size); + return cache_size; +} + +static ssize_t file_read(struct dso *dso, struct machine *machine, + u64 offset, char *data) +{ + ssize_t ret; + + mutex_lock(dso__data_open_lock()); + + /* + * dso__data(dso)->fd might be closed if other thread opened another + * file (dso) due to open file limit (RLIMIT_NOFILE). + */ + try_to_open_dso(dso, machine); + + if (dso__data(dso)->fd < 0) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + ret = dso__data(dso)->fd; + goto out; + } + + ret = pread(dso__data(dso)->fd, data, DSO__DATA_CACHE_SIZE, offset); +out: + mutex_unlock(dso__data_open_lock()); + return ret; +} + +static struct dso_cache *dso_cache__populate(struct dso *dso, + struct machine *machine, + u64 offset, ssize_t *ret) +{ + u64 cache_offset = offset & DSO__DATA_CACHE_MASK; + struct dso_cache *cache; + struct dso_cache *old; + + cache = zalloc(sizeof(*cache) + DSO__DATA_CACHE_SIZE); + if (!cache) { + *ret = -ENOMEM; + return NULL; + } +#ifdef HAVE_LIBBPF_SUPPORT + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) + *ret = bpf_read(dso, cache_offset, cache->data); + else +#endif + if (dso__binary_type(dso) == DSO_BINARY_TYPE__OOL) + *ret = DSO__DATA_CACHE_SIZE; + else + *ret = file_read(dso, machine, cache_offset, cache->data); + + if (*ret <= 0) { + free(cache); + return NULL; + } + + cache->offset = cache_offset; + cache->size = *ret; + + old = dso_cache__insert(dso, cache); + if (old) { + /* we lose the race */ + free(cache); + cache = old; + } + + return cache; +} + +static struct dso_cache *dso_cache__find(struct dso *dso, + struct machine *machine, + u64 offset, + ssize_t *ret) +{ + struct dso_cache *cache = __dso_cache__find(dso, offset); + + return cache ? cache : dso_cache__populate(dso, machine, offset, ret); +} + +static ssize_t dso_cache_io(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size, bool out) +{ + struct dso_cache *cache; + ssize_t ret = 0; + + cache = dso_cache__find(dso, machine, offset, &ret); + if (!cache) + return ret; + + return dso_cache__memcpy(cache, offset, data, size, out); +} + +/* + * Reads and caches dso data DSO__DATA_CACHE_SIZE size chunks + * in the rb_tree. Any read to already cached data is served + * by cached data. Writes update the cache only, not the backing file. + */ +static ssize_t cached_io(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size, bool out) +{ + ssize_t r = 0; + u8 *p = data; + + do { + ssize_t ret; + + ret = dso_cache_io(dso, machine, offset, p, size, out); + if (ret < 0) + return ret; + + /* Reached EOF, return what we have. */ + if (!ret) + break; + + BUG_ON(ret > size); + + r += ret; + p += ret; + offset += ret; + size -= ret; + + } while (size); + + return r; +} + +static int file_size(struct dso *dso, struct machine *machine) +{ + int ret = 0; + struct stat st; + + mutex_lock(dso__data_open_lock()); + + /* + * dso__data(dso)->fd might be closed if other thread opened another + * file (dso) due to open file limit (RLIMIT_NOFILE). + */ + try_to_open_dso(dso, machine); + + if (dso__data(dso)->fd < 0) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + ret = dso__data(dso)->fd; + goto out; + } + + if (fstat(dso__data(dso)->fd, &st) < 0) { + ret = -errno; + pr_err("dso cache fstat failed: %m\n"); + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + goto out; + } + dso__data(dso)->file_size = st.st_size; + +out: + mutex_unlock(dso__data_open_lock()); + return ret; +} + +int dso__data_file_size(struct dso *dso, struct machine *machine) +{ + if (dso__data(dso)->file_size) + return 0; + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return -1; +#ifdef HAVE_LIBBPF_SUPPORT + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) + return bpf_size(dso); +#endif + return file_size(dso, machine); +} + +/** + * dso__data_size - Return dso data size + * @dso: dso object + * @machine: machine object + * + * Return: dso data size + */ +off_t dso__data_size(struct dso *dso, struct machine *machine) +{ + if (dso__data_file_size(dso, machine)) + return -1; + + /* For now just estimate dso data size is close to file size */ + return dso__data(dso)->file_size; +} + +static ssize_t data_read_write_offset(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size, + bool out) +{ + if (dso__data_file_size(dso, machine)) + return -1; + + /* Check the offset sanity. */ + if (offset > dso__data(dso)->file_size) + return -1; + + if (offset + size < offset) + return -1; + + return cached_io(dso, machine, offset, data, size, out); +} + +/** + * dso__data_read_offset - Read data from dso file offset + * @dso: dso object + * @machine: machine object + * @offset: file offset + * @data: buffer to store data + * @size: size of the @data buffer + * + * External interface to read data from dso file offset. Open + * dso data file and use cached_read to get the data. + */ +ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size) +{ + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return -1; + + return data_read_write_offset(dso, machine, offset, data, size, true); +} + +static enum dso_swap_type dso_swap_type__from_elf_data(unsigned char eidata) +{ + static const unsigned int endian = 1; + + switch (eidata) { + case ELFDATA2LSB: + /* We are big endian, DSO is little endian. */ + return (*(unsigned char const *)&endian != 1) ? DSO_SWAP__YES : DSO_SWAP__NO; + case ELFDATA2MSB: + /* We are little endian, DSO is big endian. */ + return (*(unsigned char const *)&endian != 0) ? DSO_SWAP__YES : DSO_SWAP__NO; + default: + return DSO_SWAP__UNSET; + } +} + +/* Reads e_machine from fd, optionally caching data in dso. */ +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian) +{ + uint16_t e_machine = EM_NONE; + unsigned char e_ident[EI_NIDENT]; + enum dso_swap_type swap_type; + bool need_e_flags; + + if (e_flags) + *e_flags = 0; + + { + _Static_assert(offsetof(Elf32_Ehdr, e_ident) == 0, "Unexpected offset"); + _Static_assert(offsetof(Elf64_Ehdr, e_ident) == 0, "Unexpected offset"); + } + if (pread(fd, &e_ident, sizeof(e_ident), 0) != sizeof(e_ident)) + return EM_NONE; // Read failed. + + if (memcmp(e_ident, ELFMAG, SELFMAG) != 0) + return EM_NONE; // Not an ELF file. + + if (e_ident[EI_CLASS] == ELFCLASSNONE || e_ident[EI_CLASS] >= ELFCLASSNUM) + return EM_NONE; // Bad ELF class (32 or 64-bit objects). + + if (e_ident[EI_VERSION] != EV_CURRENT) + return EM_NONE; // Bad ELF version. + + swap_type = dso_swap_type__from_elf_data(e_ident[EI_DATA]); + if (swap_type == DSO_SWAP__UNSET) + return EM_NONE; // Bad ELF data encoding. + + if (is_big_endian) + *is_big_endian = (e_ident[EI_DATA] == ELFDATA2MSB); + + /* Cache the need for swapping. */ + if (optional_dso) { + assert(dso__needs_swap(optional_dso) == DSO_SWAP__UNSET || + dso__needs_swap(optional_dso) == swap_type); + dso__set_needs_swap(optional_dso, swap_type); + } + + { + _Static_assert(offsetof(Elf32_Ehdr, e_machine) == 18, "Unexpected offset"); + _Static_assert(offsetof(Elf64_Ehdr, e_machine) == 18, "Unexpected offset"); + } + if (pread(fd, &e_machine, sizeof(e_machine), 18) != sizeof(e_machine)) + return EM_NONE; // e_machine read failed. + + e_machine = DSO_SWAP_TYPE__SWAP(swap_type, uint16_t, e_machine); + if (e_machine >= EM_NUM) + return EM_NONE; // Bad ELF machine number. + +#ifdef NDEBUG + /* In production code the e_flags are only needed on CSKY. */ + need_e_flags = e_flags && e_machine == EM_CSKY; +#else + /* Debug code will always read the e_flags. */ + need_e_flags = e_flags != NULL; +#endif + if (need_e_flags) { + off_t offset = e_ident[EI_CLASS] == ELFCLASS32 + ? offsetof(Elf32_Ehdr, e_flags) + : offsetof(Elf64_Ehdr, e_flags); + + if (pread(fd, e_flags, sizeof(*e_flags), offset) != sizeof(*e_flags)) { + *e_flags = 0; + return EM_NONE; // e_flags read failed. + } + } + return e_machine; +} + +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian) +{ + uint16_t e_machine = EM_NONE; + int fd; + + switch (dso__binary_type(dso)) { + case DSO_BINARY_TYPE__KALLSYMS: + case DSO_BINARY_TYPE__GUEST_KALLSYMS: + case DSO_BINARY_TYPE__VMLINUX: + case DSO_BINARY_TYPE__GUEST_VMLINUX: + case DSO_BINARY_TYPE__GUEST_KMODULE: + case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: + case DSO_BINARY_TYPE__KCORE: + case DSO_BINARY_TYPE__GUEST_KCORE: + case DSO_BINARY_TYPE__BPF_PROG_INFO: + case DSO_BINARY_TYPE__BPF_IMAGE: + case DSO_BINARY_TYPE__OOL: + case DSO_BINARY_TYPE__JAVA_JIT: + if (is_big_endian) { + *is_big_endian = perf_arch_is_big_endian( + machine && machine->env ? perf_env__arch(machine->env) : NULL); + } + return perf_env__e_machine(machine ? machine->env : NULL, e_flags); + case DSO_BINARY_TYPE__DEBUGLINK: + case DSO_BINARY_TYPE__BUILD_ID_CACHE: + case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: + case DSO_BINARY_TYPE__GNU_DEBUGDATA: + case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: + case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: + case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: + case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: + break; + case DSO_BINARY_TYPE__NOT_FOUND: + default: + if (e_flags) + *e_flags = 0; + return EM_NONE; + } + + mutex_lock(dso__data_open_lock()); + + /* + * dso__data(dso)->fd might be closed if other thread opened another + * file (dso) due to open file limit (RLIMIT_NOFILE). + */ + try_to_open_dso(dso, machine); + fd = dso__data(dso)->fd; + if (fd >= 0) + e_machine = dso__read_e_machine_endian(dso, fd, e_flags, is_big_endian); + else if (e_flags) + *e_flags = 0; + + mutex_unlock(dso__data_open_lock()); + return e_machine; +} + +/** + * dso__data_read_addr - Read data from dso address + * @dso: dso object + * @machine: machine object + * @add: virtual memory address + * @data: buffer to store data + * @size: size of the @data buffer + * + * External interface to read data from dso address. + */ +ssize_t dso__data_read_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + u8 *data, ssize_t size) +{ + u64 offset = map__map_ip(map, addr); + + return dso__data_read_offset(dso, machine, offset, data, size); +} + +/** + * dso__data_write_cache_offs - Write data to dso data cache at file offset + * @dso: dso object + * @machine: machine object + * @offset: file offset + * @data: buffer to write + * @size: size of the @data buffer + * + * Write into the dso file data cache, but do not change the file itself. + */ +ssize_t dso__data_write_cache_offs(struct dso *dso, struct machine *machine, + u64 offset, const u8 *data_in, ssize_t size) +{ + u8 *data = (u8 *)data_in; /* cast away const to use same fns for r/w */ + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return -1; + + return data_read_write_offset(dso, machine, offset, data, size, false); +} + +/** + * dso__data_write_cache_addr - Write data to dso data cache at dso address + * @dso: dso object + * @machine: machine object + * @add: virtual memory address + * @data: buffer to write + * @size: size of the @data buffer + * + * External interface to write into the dso file data cache, but do not change + * the file itself. + */ +ssize_t dso__data_write_cache_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + const u8 *data, ssize_t size) +{ + u64 offset = map__map_ip(map, addr); + + return dso__data_write_cache_offs(dso, machine, offset, data, size); +} + +struct map *dso__new_map(const char *name) +{ + struct map *map = NULL; + struct dso *dso = dso__new(name); + + if (dso) { + map = map__new2(0, dso); + dso__put(dso); + } + + return map; +} + +struct dso *machine__findnew_kernel(struct machine *machine, const char *name, + const char *short_name, int dso_type) +{ + /* + * The kernel dso could be created by build_id processing. + */ + struct dso *dso = machine__findnew_dso(machine, name); + + /* + * We need to run this in all cases, since during the build_id + * processing we had no idea this was the kernel dso. + */ + if (dso != NULL) { + dso__set_short_name(dso, short_name, false); + dso__set_kernel(dso, dso_type); + } + + return dso; +} + +static void __dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated) +{ + if (dso__long_name_allocated(dso)) + free((char *)dso__long_name(dso)); + + RC_CHK_ACCESS(dso)->long_name = name; + RC_CHK_ACCESS(dso)->long_name_len = strlen(name); + dso__set_long_name_allocated(dso, name_allocated); +} + +static void dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated) +{ + struct dsos *dsos = dso__dsos(dso); + + if (name == NULL) + return; + + if (dsos) { + /* + * Need to avoid re-sorting the dsos breaking by non-atomically + * renaming the dso. + */ + down_write(&dsos->lock); + __dso__set_long_name_id(dso, name, name_allocated); + dsos->sorted = false; + up_write(&dsos->lock); + } else { + __dso__set_long_name_id(dso, name, name_allocated); + } +} + +static int __dso_id__cmp(const struct dso_id *a, const struct dso_id *b) +{ + if (a->mmap2_valid && b->mmap2_valid) { + if (a->maj > b->maj) return -1; + if (a->maj < b->maj) return 1; + + if (a->min > b->min) return -1; + if (a->min < b->min) return 1; + + if (a->ino > b->ino) return -1; + if (a->ino < b->ino) return 1; + } + if (a->mmap2_ino_generation_valid && b->mmap2_ino_generation_valid) { + if (a->ino_generation > b->ino_generation) return -1; + if (a->ino_generation < b->ino_generation) return 1; + } + if (build_id__is_defined(&a->build_id) && build_id__is_defined(&b->build_id)) { + if (a->build_id.size != b->build_id.size) + return a->build_id.size < b->build_id.size ? -1 : 1; + return memcmp(a->build_id.data, b->build_id.data, a->build_id.size); + } + return 0; +} + +const struct dso_id dso_id_empty = { + { + .maj = 0, + .min = 0, + .ino = 0, + .ino_generation = 0, + }, + .mmap2_valid = false, + .mmap2_ino_generation_valid = false, + { + .size = 0, + } +}; + +void __dso__improve_id(struct dso *dso, const struct dso_id *id) +{ + struct dsos *dsos = dso__dsos(dso); + struct dso_id *dso_id = dso__id(dso); + bool changed = false; + + /* dsos write lock held by caller. */ + + if (id->mmap2_valid && !dso_id->mmap2_valid) { + dso_id->maj = id->maj; + dso_id->min = id->min; + dso_id->ino = id->ino; + dso_id->mmap2_valid = true; + changed = true; + } + if (id->mmap2_ino_generation_valid && !dso_id->mmap2_ino_generation_valid) { + dso_id->ino_generation = id->ino_generation; + dso_id->mmap2_ino_generation_valid = true; + changed = true; + } + if (build_id__is_defined(&id->build_id) && !build_id__is_defined(&dso_id->build_id)) { + dso_id->build_id = id->build_id; + changed = true; + } + if (changed && dsos) + dsos->sorted = false; +} + +int dso_id__cmp(const struct dso_id *a, const struct dso_id *b) +{ + if (a == &dso_id_empty || b == &dso_id_empty) { + /* There is no valid data to compare so the comparison always returns identical. */ + return 0; + } + + return __dso_id__cmp(a, b); +} + +int dso__cmp_id(struct dso *a, struct dso *b) +{ + return __dso_id__cmp(dso__id(a), dso__id(b)); +} + +void dso__set_long_name(struct dso *dso, const char *name, bool name_allocated) +{ + dso__set_long_name_id(dso, name, name_allocated); +} + +static void __dso__set_short_name(struct dso *dso, const char *name, bool name_allocated) +{ + if (dso__short_name_allocated(dso)) + free((char *)dso__short_name(dso)); + + RC_CHK_ACCESS(dso)->short_name = name; + RC_CHK_ACCESS(dso)->short_name_len = strlen(name); + dso__set_short_name_allocated(dso, name_allocated); +} + +void dso__set_short_name(struct dso *dso, const char *name, bool name_allocated) +{ + struct dsos *dsos = dso__dsos(dso); + + if (name == NULL) + return; + + if (dsos) { + /* + * Need to avoid re-sorting the dsos breaking by non-atomically + * renaming the dso. + */ + down_write(&dsos->lock); + __dso__set_short_name(dso, name, name_allocated); + dsos->sorted = false; + up_write(&dsos->lock); + } else { + __dso__set_short_name(dso, name, name_allocated); + } +} + +int dso__name_len(const struct dso *dso) +{ + if (!dso) + return strlen("[unknown]"); + if (verbose > 0) + return dso__long_name_len(dso); + + return dso__short_name_len(dso); +} + +bool dso__loaded(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->loaded; +} + +bool dso__sorted_by_name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->sorted_by_name; +} + +void dso__set_sorted_by_name(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->sorted_by_name = true; +} + +struct dso *dso__new_id(const char *name, const struct dso_id *id) +{ + RC_STRUCT(dso) *dso = zalloc(sizeof(*dso) + strlen(name) + 1); + struct dso *res; + struct dso_data *data; + + if (!dso) + return NULL; + + if (ADD_RC_CHK(res, dso)) { + strcpy(dso->name, name); + if (id) + dso->id = *id; + dso__set_long_name_id(res, dso->name, false); + dso__set_short_name(res, dso->name, false); + dso->symbols = RB_ROOT_CACHED; + dso->symbol_names = NULL; + dso->symbol_names_len = 0; + dso->inlined_nodes = RB_ROOT_CACHED; + dso->srclines = RB_ROOT_CACHED; + dso->data_types = RB_ROOT; + dso->global_vars = RB_ROOT; + dso->data.fd = -1; + dso->data.status = DSO_DATA_STATUS_UNKNOWN; + dso->symtab_type = DSO_BINARY_TYPE__NOT_FOUND; + dso->binary_type = DSO_BINARY_TYPE__NOT_FOUND; + dso->is_64_bit = (sizeof(void *) == 8); + dso->loaded = 0; + dso->rel = 0; + dso->sorted_by_name = 0; + dso->has_srcline = 1; + dso->a2l_fails = 1; + dso->kernel = DSO_SPACE__USER; + dso->is_kmod = 0; + dso->needs_swap = DSO_SWAP__UNSET; + dso->comp = COMP_ID__NONE; + mutex_init(&dso->lock); + refcount_set(&dso->refcnt, 1); + data = &dso->data; + data->cache = RB_ROOT; + data->fd = -1; + data->status = DSO_DATA_STATUS_UNKNOWN; + INIT_LIST_HEAD(&data->open_entry); +#ifdef REFCNT_CHECKING + data->dso = NULL; /* Set when on the open_entry list. */ +#endif + } + return res; +} + +struct dso *dso__new(const char *name) +{ + return dso__new_id(name, NULL); +} + +void dso__delete(struct dso *dso) +{ + if (dso__dsos(dso)) + pr_err("DSO %s is still in rbtree when being deleted!\n", dso__long_name(dso)); + + /* free inlines first, as they reference symbols */ + inlines__tree_delete(&RC_CHK_ACCESS(dso)->inlined_nodes); + srcline__tree_delete(&RC_CHK_ACCESS(dso)->srclines); + symbols__delete(&RC_CHK_ACCESS(dso)->symbols); + RC_CHK_ACCESS(dso)->symbol_names_len = 0; + zfree(&RC_CHK_ACCESS(dso)->symbol_names); + annotated_data_type__tree_delete(dso__data_types(dso)); + global_var_type__tree_delete(dso__global_vars(dso)); + + if (RC_CHK_ACCESS(dso)->short_name_allocated) { + zfree((char **)&RC_CHK_ACCESS(dso)->short_name); + RC_CHK_ACCESS(dso)->short_name_allocated = false; + } + + if (RC_CHK_ACCESS(dso)->long_name_allocated) { + zfree((char **)&RC_CHK_ACCESS(dso)->long_name); + RC_CHK_ACCESS(dso)->long_name_allocated = false; + } + + dso__data_close(dso); + auxtrace_cache__free(RC_CHK_ACCESS(dso)->auxtrace_cache); + dso_cache__free(dso); + dso__free_a2l(dso); + dso__free_libdw(dso); + dso__free_symsrc_filename(dso); + nsinfo__zput(RC_CHK_ACCESS(dso)->nsinfo); + mutex_destroy(dso__lock(dso)); + RC_CHK_FREE(dso); +} + +struct dso *dso__get(struct dso *dso) +{ + struct dso *result; + + if (RC_CHK_GET(result, dso)) + refcount_inc(&RC_CHK_ACCESS(dso)->refcnt); + + return result; +} + +void dso__put(struct dso *dso) +{ +#ifdef REFCNT_CHECKING + if (dso && dso__data(dso) && refcount_read(&RC_CHK_ACCESS(dso)->refcnt) == 2) + dso__data_close(dso); +#endif + if (dso && refcount_dec_and_test(&RC_CHK_ACCESS(dso)->refcnt)) + dso__delete(dso); + else + RC_CHK_PUT(dso); +} + +int dso__swap_init(struct dso *dso, unsigned char eidata) +{ + enum dso_swap_type type = dso_swap_type__from_elf_data(eidata); + + dso__set_needs_swap(dso, type); + if (type == DSO_SWAP__UNSET) { + pr_err("unrecognized DSO data encoding %d\n", eidata); + return -EINVAL; + } + return 0; +} + +void dso__set_build_id(struct dso *dso, const struct build_id *bid) +{ + dso__id(dso)->build_id = *bid; +} + +bool dso__build_id_equal(const struct dso *dso, const struct build_id *bid) +{ + const struct build_id *dso_bid = dso__bid(dso); + + if (dso_bid->size > bid->size && dso_bid->size == BUILD_ID_SIZE) { + /* + * For the backward compatibility, it allows a build-id has + * trailing zeros. + */ + return !memcmp(dso_bid->data, bid->data, bid->size) && + !memchr_inv(&dso_bid->data[bid->size], 0, + dso_bid->size - bid->size); + } + + return dso_bid->size == bid->size && + memcmp(dso_bid->data, bid->data, dso_bid->size) == 0; +} + +void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) +{ + char path[PATH_MAX]; + struct build_id bid = { .size = 0, }; + + if (machine__is_default_guest(machine)) + return; + snprintf(path, sizeof(path), "%s/sys/kernel/notes", machine->root_dir); + sysfs__read_build_id(path, &bid); + dso__set_build_id(dso, &bid); +} + +int dso__kernel_module_get_build_id(struct dso *dso, + const char *root_dir) +{ + char filename[PATH_MAX]; + struct build_id bid = { .size = 0, }; + /* + * kernel module short names are of the form "[module]" and + * we need just "module" here. + */ + const char *name = dso__short_name(dso) + 1; + + snprintf(filename, sizeof(filename), + "%s/sys/module/%.*s/notes/.note.gnu.build-id", + root_dir, (int)strlen(name) - 1, name); + + sysfs__read_build_id(filename, &bid); + dso__set_build_id(dso, &bid); + return 0; +} + +static size_t dso__fprintf_buildid(struct dso *dso, FILE *fp) +{ + char sbuild_id[SBUILD_ID_SIZE]; + + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + return fprintf(fp, "%s", sbuild_id); +} + +size_t dso__fprintf(struct dso *dso, FILE *fp) +{ + struct rb_node *nd; + size_t ret = fprintf(fp, "dso: %s (", dso__short_name(dso)); + + if (dso__short_name(dso) != dso__long_name(dso)) + ret += fprintf(fp, "%s, ", dso__long_name(dso)); + ret += fprintf(fp, "%sloaded, ", dso__loaded(dso) ? "" : "NOT "); + ret += dso__fprintf_buildid(dso, fp); + ret += fprintf(fp, ")\n"); + for (nd = rb_first_cached(dso__symbols(dso)); nd; nd = rb_next(nd)) { + struct symbol *pos = rb_entry(nd, struct symbol, rb_node); + ret += symbol__fprintf(pos, fp); + } + + return ret; +} + +enum dso_type dso__type(struct dso *dso, struct machine *machine) +{ + int fd = -1; + enum dso_type type = DSO__TYPE_UNKNOWN; + + if (dso__data_get_fd(dso, machine, &fd)) { + type = dso__type_fd(fd); + dso__data_put_fd(dso); + } + + return type; +} + +int dso__strerror_load(struct dso *dso, char *buf, size_t buflen) +{ + int idx, errnum = *dso__load_errno(dso); + /* + * This must have a same ordering as the enum dso_load_errno. + */ + static const char *dso_load__error_str[] = { + "Internal tools/perf/ library error", + "Invalid ELF file", + "Can not read build id", + "Mismatching build id", + "Decompression failure", + }; + + BUG_ON(buflen == 0); + + if (errnum >= 0) { + errno = errnum; + scnprintf(buf, buflen, "%m"); + + return 0; + } + + if (errnum < __DSO_LOAD_ERRNO__START || errnum >= __DSO_LOAD_ERRNO__END) + return -1; + + idx = errnum - __DSO_LOAD_ERRNO__START; + scnprintf(buf, buflen, "%s", dso_load__error_str[idx]); + return 0; +} + +bool perf_pid_map_tid(const char *dso_name, int *tid) +{ + return sscanf(dso_name, "/tmp/perf-%d.map", tid) == 1; +} + +bool is_perf_pid_map_name(const char *dso_name) +{ + int tid; + + return perf_pid_map_tid(dso_name, &tid); +} + +struct find_file_offset_data { + u64 ip; + u64 offset; +}; + +/* This will be called for each PHDR in an ELF binary */ +static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) +{ + struct find_file_offset_data *data = arg; + + if (start <= data->ip && data->ip < start + len) { + data->offset = pgoff + data->ip - start; + return 1; + } + return 0; +} + +static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename, + u64 start, size_t len, + u8 **out_buf, u64 *out_buf_len, bool *is_64bit) +{ + struct nscookie nsc; + int fd; + ssize_t count; + struct find_file_offset_data data = { + .ip = start, + }; + u8 *code_buf = NULL; + int saved_errno; + + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + fd = open(symfs_filename, O_RDONLY | O_CLOEXEC); + saved_errno = errno; + nsinfo__mountns_exit(&nsc); + if (fd < 0) { + errno = saved_errno; + return NULL; + } + if (file__read_maps(fd, /*exe=*/true, find_file_offset, &data, is_64bit) <= 0) { + close(fd); + errno = ENOENT; + return NULL; + } + code_buf = malloc(len); + if (code_buf == NULL) { + close(fd); + errno = ENOMEM; + return NULL; + } + count = pread(fd, code_buf, len, data.offset); + saved_errno = errno; + close(fd); + if ((u64)count != len) { + free(code_buf); + errno = saved_errno; + return NULL; + } + *out_buf = code_buf; + *out_buf_len = len; + return code_buf; +} + +/* + * Read a symbol into memory for disassembly by a library like capstone of + * libLLVM. If memory is allocated out_buf holds it. + */ +const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, + const struct map *map, const struct symbol *sym, + u8 **out_buf, u64 *out_buf_len, bool *is_64bit) +{ + u64 start = map__rip_2objdump(map, sym->start); + u64 end = map__rip_2objdump(map, sym->end); + size_t len = end - start; + + *out_buf = NULL; + *out_buf_len = 0; + *is_64bit = false; + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) { + /* + * Note, there is fallback BPF image disassembly in the objdump + * version but it currently does nothing. + */ + errno = EOPNOTSUPP; + return NULL; + } + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) { +#ifdef HAVE_LIBBPF_SUPPORT + struct bpf_prog_info_node *info_node; + struct perf_bpil *info_linear; + + *is_64bit = sizeof(void *) == sizeof(u64); + info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env, + dso__bpf_prog(dso)->id); + if (!info_node) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } + info_linear = info_node->info_linear; + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } + if (len > info_linear->info.jited_prog_len) { + pr_debug("BPF symbol length %zu exceeds jited_prog_len %u\n", + len, info_linear->info.jited_prog_len); + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } + *out_buf_len = len; + return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); +#else + pr_debug("No BPF program disassembly support\n"); + errno = EOPNOTSUPP; + return NULL; +#endif + } + return __dso__read_symbol(dso, symfs_filename, start, len, + out_buf, out_buf_len, is_64bit); +} + +struct debuginfo *dso__debuginfo(struct dso *dso) +{ + char *name; + bool decomp = false; + struct debuginfo *dinfo = NULL; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, "", &decomp); + if (name) + dinfo = debuginfo__new(name); + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return dinfo; +} diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h new file mode 100644 index 000000000..55c4aaa53 --- /dev/null +++ b/tools/perf/util/dso.h @@ -0,0 +1,1009 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DSO +#define __PERF_DSO + +#include +#include +#include +#include +#include +#include +#include +#include +#include "build-id.h" +#include "debuginfo.h" +#include "mutex.h" +#include + +struct machine; +struct map; +struct perf_env; + +#define DSO__NAME_KALLSYMS "[kernel.kallsyms]" +#define DSO__NAME_KCORE "[kernel.kcore]" +#define DSO__NAME_GUEST_KALLSYMS "[guest.kernel.kallsyms]" +#define DSO__NAME_GUEST_KALLSYMS_PID_PREFIX "[guest.kernel.kallsyms." + +/* + * Validate names of the form "[guest.kernel.kallsyms.]", where + * is the PID of the guest VM and varies per guest, so it + * cannot be matched with strcmp() against a fixed string. + * + * Every character after the fixed prefix must be a decimal digit, + * with ']' immediately terminating the digit run and nothing + * following it. This rules out '/', "..", or any other character + * being smuggled into the name. + */ +static inline bool is_guest_kallsyms_pid_name(const char *name) +{ + const size_t prefix_len = sizeof(DSO__NAME_GUEST_KALLSYMS_PID_PREFIX) - 1; + size_t digits; + + if (strncmp(name, DSO__NAME_GUEST_KALLSYMS_PID_PREFIX, prefix_len) != 0) + return false; + + digits = strspn(name + prefix_len, "0123456789"); + if (digits == 0) + return false; + + /* ']' must terminate the digit run, with nothing trailing it */ + if (name[prefix_len + digits] != ']') + return false; + + if (name[prefix_len + digits + 1] != '\0') + return false; + + return true; +} + +/** + * enum dso_binary_type - The kind of DSO generally associated with a memory + * region (struct map). + */ +enum dso_binary_type { + /** @DSO_BINARY_TYPE__KALLSYMS: Symbols from /proc/kallsyms file. */ + DSO_BINARY_TYPE__KALLSYMS = 0, + /** @DSO_BINARY_TYPE__GUEST_KALLSYMS: Guest /proc/kallsyms file. */ + DSO_BINARY_TYPE__GUEST_KALLSYMS, + /** @DSO_BINARY_TYPE__VMLINUX: Path to kernel /boot/vmlinux file. */ + DSO_BINARY_TYPE__VMLINUX, + /** @DSO_BINARY_TYPE__GUEST_VMLINUX: Path to guest kernel /boot/vmlinux file. */ + DSO_BINARY_TYPE__GUEST_VMLINUX, + /** @DSO_BINARY_TYPE__JAVA_JIT: Symbols from /tmp/perf.map file. */ + DSO_BINARY_TYPE__JAVA_JIT, + /** + * @DSO_BINARY_TYPE__DEBUGLINK: Debug file readable from the file path + * in the .gnu_debuglink ELF section of the dso. + */ + DSO_BINARY_TYPE__DEBUGLINK, + /** + * @DSO_BINARY_TYPE__BUILD_ID_CACHE: File named after buildid located in + * the buildid cache with an elf filename. + */ + DSO_BINARY_TYPE__BUILD_ID_CACHE, + /** + * @DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: File named after buildid + * located in the buildid cache with a debug filename. + */ + DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__FEDORA_DEBUGINFO: Debug file in /usr/lib/debug + * with .debug suffix. + */ + DSO_BINARY_TYPE__FEDORA_DEBUGINFO, + /** @DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: Debug file in /usr/lib/debug. */ + DSO_BINARY_TYPE__UBUNTU_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: dso__long_name debuginfo + * file in /usr/lib/debug/lib rather than the expected + * /usr/lib/debug/usr/lib. + */ + DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__BUILDID_DEBUGINFO: File named after buildid located + * in /usr/lib/debug/.build-id/. + */ + DSO_BINARY_TYPE__BUILDID_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__GNU_DEBUGDATA: MiniDebuginfo where a compressed + * ELF file is placed in a .gnu_debugdata section. + */ + DSO_BINARY_TYPE__GNU_DEBUGDATA, + /** @DSO_BINARY_TYPE__SYSTEM_PATH_DSO: A regular executable/shared-object file. */ + DSO_BINARY_TYPE__SYSTEM_PATH_DSO, + /** @DSO_BINARY_TYPE__GUEST_KMODULE: Guest kernel module .ko file. */ + DSO_BINARY_TYPE__GUEST_KMODULE, + /** @DSO_BINARY_TYPE__GUEST_KMODULE_COMP: Guest kernel module .ko.gz file. */ + DSO_BINARY_TYPE__GUEST_KMODULE_COMP, + /** @DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: Kernel module .ko file. */ + DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE, + /** @DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: Kernel module .ko.gz file. */ + DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP, + /** @DSO_BINARY_TYPE__KCORE: /proc/kcore file. */ + DSO_BINARY_TYPE__KCORE, + /** @DSO_BINARY_TYPE__GUEST_KCORE: Guest /proc/kcore file. */ + DSO_BINARY_TYPE__GUEST_KCORE, + /** + * @DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: Openembedded/Yocto -dbg + * package debug info. + */ + DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO, + /** @DSO_BINARY_TYPE__BPF_PROG_INFO: jitted BPF code. */ + DSO_BINARY_TYPE__BPF_PROG_INFO, + /** @DSO_BINARY_TYPE__BPF_IMAGE: jitted BPF trampoline or dispatcher code. */ + DSO_BINARY_TYPE__BPF_IMAGE, + /** + * @DSO_BINARY_TYPE__OOL: out of line code such as kprobe-replaced + * instructions or optimized kprobes or ftrace trampolines. + */ + DSO_BINARY_TYPE__OOL, + /** @DSO_BINARY_TYPE__NOT_FOUND: Unknown DSO kind. */ + DSO_BINARY_TYPE__NOT_FOUND, +}; + +enum dso_space_type { + DSO_SPACE__USER = 0, + DSO_SPACE__KERNEL, + DSO_SPACE__KERNEL_GUEST +}; + +enum dso_swap_type { + DSO_SWAP__UNSET, + DSO_SWAP__NO, + DSO_SWAP__YES, +}; + +enum dso_data_status { + DSO_DATA_STATUS_ERROR = -1, + DSO_DATA_STATUS_UNKNOWN = 0, + DSO_DATA_STATUS_OK = 1, +}; + +enum dso_data_status_seen { + DSO_DATA_STATUS_SEEN_ITRACE, +}; + +enum dso_type { + DSO__TYPE_UNKNOWN, + DSO__TYPE_64BIT, + DSO__TYPE_32BIT, + DSO__TYPE_X32BIT, +}; + +enum dso_load_errno { + DSO_LOAD_ERRNO__SUCCESS = 0, + + /* + * Choose an arbitrary negative big number not to clash with standard + * errno since SUS requires the errno has distinct positive values. + * See 'Issue 6' in the link below. + * + * http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html + */ + __DSO_LOAD_ERRNO__START = -10000, + + DSO_LOAD_ERRNO__INTERNAL_ERROR = __DSO_LOAD_ERRNO__START, + + /* for symsrc__init() */ + DSO_LOAD_ERRNO__INVALID_ELF, + DSO_LOAD_ERRNO__CANNOT_READ_BUILDID, + DSO_LOAD_ERRNO__MISMATCHING_BUILDID, + + /* for decompress_kmodule */ + DSO_LOAD_ERRNO__DECOMPRESSION_FAILURE, + + __DSO_LOAD_ERRNO__END, +}; + +#define DSO_SWAP_TYPE__SWAP(swap_type, type, val) \ +({ \ + type ____r = val; \ + BUG_ON(swap_type == DSO_SWAP__UNSET); \ + if (swap_type == DSO_SWAP__YES) { \ + switch (sizeof(____r)) { \ + case 2: \ + ____r = bswap_16(val); \ + break; \ + case 4: \ + ____r = bswap_32(val); \ + break; \ + case 8: \ + ____r = bswap_64(val); \ + break; \ + default: \ + BUG_ON(1); \ + } \ + } \ + ____r; \ +}) + +#define DSO__SWAP(dso, type, val) DSO_SWAP_TYPE__SWAP(dso__needs_swap(dso), type, val) + +#define DSO__DATA_CACHE_SIZE 4096 +#define DSO__DATA_CACHE_MASK ~(DSO__DATA_CACHE_SIZE - 1) + +/** + * struct dso_id + * + * Data about backing storage DSO, comes from PERF_RECORD_MMAP2 meta events, + * reading from /proc/pid/maps or synthesis of build_ids from DSOs. Possibly + * incomplete at any particular use. + */ +struct dso_id { + /* Data related to the mmap2 event or read from /proc/pid/maps. */ + struct { + u32 maj; + u32 min; + u64 ino; + u64 ino_generation; + }; + /** @mmap2_valid: Are the maj, min and ino fields valid? */ + bool mmap2_valid; + /** + * @mmap2_ino_generation_valid: Is the ino_generation valid? Generally + * false for /proc/pid/maps mmap event. + */ + bool mmap2_ino_generation_valid; + /** + * @build_id: A possibly populated build_id. build_id__is_defined checks + * whether it is populated. + */ + struct build_id build_id; +}; + +struct dso_cache { + struct rb_node rb_node; + u64 offset; + u64 size; + char data[]; +}; + +struct dso_data { + struct rb_root cache; + struct list_head open_entry; +#ifdef REFCNT_CHECKING + struct dso *dso; +#endif + int fd; + int status; + u32 status_seen; + u64 file_size; +#ifdef HAVE_LIBUNWIND_SUPPORT + u64 elf_base_addr; + u64 debug_frame_offset; + u64 eh_frame_hdr_addr; + u64 eh_frame_hdr_offset; +#endif +}; + +struct dso_bpf_prog { + u32 id; + u32 sub_id; + struct perf_env *env; +}; + +struct auxtrace_cache; + +DECLARE_RC_STRUCT(dso) { + struct mutex lock; + struct dsos *dsos; + struct rb_root_cached symbols; + struct symbol **symbol_names; + size_t symbol_names_len; + struct rb_root_cached inlined_nodes; + struct rb_root_cached srclines; + struct rb_root data_types; + struct rb_root global_vars; + + struct { + u64 addr; + struct symbol *symbol; + } last_find_result; + u64 text_offset; + u64 text_end; + const char *short_name; + const char *long_name; + void *a2l; + void *libdw; + char *symsrc_filename; + struct nsinfo *nsinfo; + struct auxtrace_cache *auxtrace_cache; + union { /* Tool specific area */ + void *priv; + u64 db_id; + }; + /* bpf prog information */ + struct dso_bpf_prog bpf_prog; + /* dso data file */ + struct dso_data data; + struct dso_id id; + unsigned int a2l_fails; + int comp; + refcount_t refcnt; + enum dso_load_errno load_errno; + u16 long_name_len; + u16 short_name_len; + enum dso_binary_type symtab_type:8; + enum dso_binary_type binary_type:8; + enum dso_space_type kernel:2; + enum dso_swap_type needs_swap:2; + bool is_kmod:1; + u8 adjust_symbols:1; + u8 header_build_id:1; + u8 has_srcline:1; + u8 hit:1; + u8 annotate_warned:1; + u8 auxtrace_warned:1; + u8 debuginfo_warned:1; + u8 short_name_allocated:1; + u8 long_name_allocated:1; + u8 is_64_bit:1; + bool sorted_by_name; + bool loaded; + u8 rel; + char name[]; +}; + +extern struct mutex _dso__data_open_lock; +extern const struct dso_id dso_id_empty; + +int dso_id__cmp(const struct dso_id *a, const struct dso_id *b); + +/* dso__for_each_symbol - iterate over the symbols of given type + * + * @dso: the 'struct dso *' in which symbols are iterated + * @pos: the 'struct symbol *' to use as a loop cursor + * @n: the 'struct rb_node *' to use as a temporary storage + */ +#define dso__for_each_symbol(dso, pos, n) \ + symbols__for_each_entry(dso__symbols(dso), pos, n) + +static inline void *dso__a2l(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->a2l; +} + +static inline void dso__set_a2l(struct dso *dso, void *val) +{ + RC_CHK_ACCESS(dso)->a2l = val; +} + +static inline void *dso__libdw(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->libdw; +} + +static inline void dso__set_libdw(struct dso *dso, void *val) +{ + RC_CHK_ACCESS(dso)->libdw = val; +} + +struct Dwfl; +#ifdef HAVE_LIBDW_SUPPORT +struct Dwfl *dso__libdw_dwfl(struct dso *dso); +#else +static inline struct Dwfl *dso__libdw_dwfl(struct dso *dso __maybe_unused) +{ + return NULL; +} +#endif + +static inline unsigned int dso__a2l_fails(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->a2l_fails; +} + +static inline void dso__set_a2l_fails(struct dso *dso, unsigned int val) +{ + RC_CHK_ACCESS(dso)->a2l_fails = val; +} + +static inline bool dso__adjust_symbols(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->adjust_symbols; +} + +static inline void dso__set_adjust_symbols(struct dso *dso, bool val) +{ + RC_CHK_ACCESS(dso)->adjust_symbols = val; +} + +static inline bool dso__annotate_warned(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->annotate_warned; +} + +static inline void dso__set_annotate_warned(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->annotate_warned = 1; +} + +static inline bool dso__debuginfo_warned(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->debuginfo_warned; +} + +static inline void dso__set_debuginfo_warned(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->debuginfo_warned = 1; +} + +static inline bool dso__auxtrace_warned(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->auxtrace_warned; +} + +static inline void dso__set_auxtrace_warned(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->auxtrace_warned = 1; +} + +static inline struct auxtrace_cache *dso__auxtrace_cache(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->auxtrace_cache; +} + +static inline void dso__set_auxtrace_cache(struct dso *dso, struct auxtrace_cache *cache) +{ + RC_CHK_ACCESS(dso)->auxtrace_cache = cache; +} + +static inline struct dso_bpf_prog *dso__bpf_prog(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->bpf_prog; +} + +static inline bool dso__has_srcline(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->has_srcline; +} + +static inline void dso__set_has_srcline(struct dso *dso, bool val) +{ + RC_CHK_ACCESS(dso)->has_srcline = val; +} + +static inline int dso__comp(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->comp; +} + +static inline void dso__set_comp(struct dso *dso, int comp) +{ + RC_CHK_ACCESS(dso)->comp = comp; +} + +static inline struct dso_data *dso__data(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->data; +} + +static inline u64 dso__db_id(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->db_id; +} + +static inline void dso__set_db_id(struct dso *dso, u64 db_id) +{ + RC_CHK_ACCESS(dso)->db_id = db_id; +} + +static inline struct dsos *dso__dsos(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->dsos; +} + +static inline void dso__set_dsos(struct dso *dso, struct dsos *dsos) +{ + RC_CHK_ACCESS(dso)->dsos = dsos; +} + +static inline bool dso__header_build_id(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->header_build_id; +} + +static inline void dso__set_header_build_id(struct dso *dso, bool val) +{ + RC_CHK_ACCESS(dso)->header_build_id = val; +} + +static inline bool dso__hit(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->hit; +} + +static inline void dso__set_hit(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->hit = 1; +} + +static inline struct dso_id *dso__id(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->id; +} + +static inline const struct dso_id *dso__id_const(const struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->id; +} + +static inline const struct build_id *dso__bid(const struct dso *dso) +{ + return &dso__id_const(dso)->build_id; +} + +static inline bool dso__has_build_id(const struct dso *dso) +{ + return build_id__is_defined(dso__bid(dso)); +} + +static inline struct rb_root_cached *dso__inlined_nodes(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->inlined_nodes; +} + +static inline bool dso__is_64_bit(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->is_64_bit; +} + +static inline void dso__set_is_64_bit(struct dso *dso, bool is) +{ + RC_CHK_ACCESS(dso)->is_64_bit = is; +} + +static inline bool dso__is_kmod(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->is_kmod; +} + +static inline void dso__set_is_kmod(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->is_kmod = 1; +} + +static inline enum dso_space_type dso__kernel(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->kernel; +} + +static inline void dso__set_kernel(struct dso *dso, enum dso_space_type kernel) +{ + RC_CHK_ACCESS(dso)->kernel = kernel; +} + +static inline u64 dso__last_find_result_addr(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->last_find_result.addr; +} + +static inline void dso__set_last_find_result_addr(struct dso *dso, u64 addr) +{ + RC_CHK_ACCESS(dso)->last_find_result.addr = addr; +} + +static inline struct symbol *dso__last_find_result_symbol(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->last_find_result.symbol; +} + +static inline void dso__set_last_find_result_symbol(struct dso *dso, struct symbol *symbol) +{ + RC_CHK_ACCESS(dso)->last_find_result.symbol = symbol; +} + +static inline enum dso_load_errno *dso__load_errno(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->load_errno; +} + +static inline void dso__set_loaded(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->loaded = true; +} + +static inline struct mutex *dso__lock(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->lock; +} + +static inline const char *dso__long_name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->long_name; +} + +static inline bool dso__long_name_allocated(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->long_name_allocated; +} + +static inline void dso__set_long_name_allocated(struct dso *dso, bool allocated) +{ + RC_CHK_ACCESS(dso)->long_name_allocated = allocated; +} + +static inline u16 dso__long_name_len(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->long_name_len; +} + +static inline const char *dso__name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->name; +} + +static inline enum dso_swap_type dso__needs_swap(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->needs_swap; +} + +static inline void dso__set_needs_swap(struct dso *dso, enum dso_swap_type type) +{ + RC_CHK_ACCESS(dso)->needs_swap = type; +} + +static inline struct nsinfo *dso__nsinfo(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->nsinfo; +} + +static inline const struct nsinfo *dso__nsinfo_const(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->nsinfo; +} + +static inline struct nsinfo **dso__nsinfo_ptr(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->nsinfo; +} + +void dso__set_nsinfo(struct dso *dso, struct nsinfo *nsi); + +static inline u8 dso__rel(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->rel; +} + +static inline void dso__set_rel(struct dso *dso, u8 rel) +{ + RC_CHK_ACCESS(dso)->rel = rel; +} + +static inline const char *dso__short_name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->short_name; +} + +static inline bool dso__short_name_allocated(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->short_name_allocated; +} + +static inline void dso__set_short_name_allocated(struct dso *dso, bool allocated) +{ + RC_CHK_ACCESS(dso)->short_name_allocated = allocated; +} + +static inline u16 dso__short_name_len(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->short_name_len; +} + +static inline struct rb_root_cached *dso__srclines(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->srclines; +} + +static inline struct rb_root *dso__data_types(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->data_types; +} + +static inline struct rb_root *dso__global_vars(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->global_vars; +} + +static inline struct rb_root_cached *dso__symbols(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->symbols; +} + +static inline struct symbol **dso__symbol_names(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symbol_names; +} + +static inline void dso__set_symbol_names(struct dso *dso, struct symbol **names) +{ + RC_CHK_ACCESS(dso)->symbol_names = names; +} + +static inline size_t dso__symbol_names_len(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symbol_names_len; +} + +static inline void dso__set_symbol_names_len(struct dso *dso, size_t len) +{ + RC_CHK_ACCESS(dso)->symbol_names_len = len; +} + +static inline const char *dso__symsrc_filename(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symsrc_filename; +} + +static inline void dso__set_symsrc_filename(struct dso *dso, char *val) +{ + RC_CHK_ACCESS(dso)->symsrc_filename = val; +} + +static inline void dso__free_symsrc_filename(struct dso *dso) +{ + zfree(&RC_CHK_ACCESS(dso)->symsrc_filename); +} + +static inline enum dso_binary_type dso__symtab_type(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symtab_type; +} + +static inline void dso__set_symtab_type(struct dso *dso, enum dso_binary_type bt) +{ + RC_CHK_ACCESS(dso)->symtab_type = bt; +} + +static inline u64 dso__text_end(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->text_end; +} + +static inline void dso__set_text_end(struct dso *dso, u64 val) +{ + RC_CHK_ACCESS(dso)->text_end = val; +} + +static inline u64 dso__text_offset(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->text_offset; +} + +static inline void dso__set_text_offset(struct dso *dso, u64 val) +{ + RC_CHK_ACCESS(dso)->text_offset = val; +} + +struct dso *dso__new_id(const char *name, const struct dso_id *id); +struct dso *dso__new(const char *name); +void dso__delete(struct dso *dso); + +int dso__cmp_id(struct dso *a, struct dso *b); +void dso__set_short_name(struct dso *dso, const char *name, bool name_allocated); +void dso__set_long_name(struct dso *dso, const char *name, bool name_allocated); +void __dso__improve_id(struct dso *dso, const struct dso_id *id); + +int dso__name_len(const struct dso *dso); + +struct dso *dso__get(struct dso *dso); +void dso__put(struct dso *dso) LOCKS_EXCLUDED(_dso__data_open_lock); + +static inline void __dso__zput(struct dso **dso) +{ + dso__put(*dso); + *dso = NULL; +} + +#define dso__zput(dso) __dso__zput(&dso) + +bool dso__loaded(const struct dso *dso); + +static inline bool dso__has_symbols(const struct dso *dso) +{ + return !RB_EMPTY_ROOT(&RC_CHK_ACCESS(dso)->symbols.rb_root); +} + +char *dso__filename_with_chroot(const struct dso *dso, const char *filename); + +bool dso__sorted_by_name(const struct dso *dso); +void dso__set_sorted_by_name(struct dso *dso); +void dso__sort_by_name(struct dso *dso); + +int dso__swap_init(struct dso *dso, unsigned char eidata); + +void dso__set_build_id(struct dso *dso, const struct build_id *bid); +bool dso__build_id_equal(const struct dso *dso, const struct build_id *bid); +void dso__read_running_kernel_build_id(struct dso *dso, + struct machine *machine); +int dso__kernel_module_get_build_id(struct dso *dso, const char *root_dir); + +char dso__symtab_origin(const struct dso *dso); +int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, + const char *root_dir, char *filename, size_t size); +bool is_kernel_module(const char *pathname, int cpumode); +bool dso__needs_decompress(struct dso *dso); +int dso__decompress_kmodule_fd(struct dso *dso, const char *name); +int dso__decompress_kmodule_path(struct dso *dso, const char *name, + char *pathname, size_t len); +int filename__decompress(const char *name, char *pathname, + size_t len, int comp, int *err); + +#define KMOD_DECOMP_NAME "/tmp/perf-kmod-XXXXXX" +#define KMOD_DECOMP_LEN sizeof(KMOD_DECOMP_NAME) + +struct kmod_path { + char *name; + int comp; + bool kmod; +}; + +int __kmod_path__parse(struct kmod_path *m, const char *path, + bool alloc_name); + +#define kmod_path__parse(__m, __p) __kmod_path__parse(__m, __p, false) +#define kmod_path__parse_name(__m, __p) __kmod_path__parse(__m, __p, true) + +void dso__set_module_info(struct dso *dso, struct kmod_path *m, + struct machine *machine); + +/* + * The dso__data_* external interface provides following functions: + * dso__data_get_fd + * dso__data_put_fd + * dso__data_close + * dso__data_size + * dso__data_read_offset + * dso__data_read_addr + * dso__data_write_cache_offs + * dso__data_write_cache_addr + * + * Please refer to the dso.c object code for each function and + * arguments documentation. Following text tries to explain the + * dso file descriptor caching. + * + * The dso__data* interface allows caching of opened file descriptors + * to speed up the dso data accesses. The idea is to leave the file + * descriptor opened ideally for the whole life of the dso object. + * + * The current usage of the dso__data_* interface is as follows: + * + * Get DSO's fd: + * int fd; + * if (dso__data_get_fd(dso, machine, &fd)) { + * USE 'fd' SOMEHOW + * dso__data_put_fd(dso); + * } + * + * Read DSO's data: + * n = dso__data_read_offset(dso_0, &machine, 0, buf, BUFSIZE); + * n = dso__data_read_addr(dso_0, &machine, 0, buf, BUFSIZE); + * + * Eventually close DSO's fd: + * dso__data_close(dso); + * + * It is not necessary to close the DSO object data file. Each time new + * DSO data file is opened, the limit (RLIMIT_NOFILE/2) is checked. Once + * it is crossed, the oldest opened DSO object is closed. + * + * The dso__delete function calls close_dso function to ensure the + * data file descriptor gets closed/unmapped before the dso object + * is freed. + * + * TODO +*/ +bool dso__data_get_fd(struct dso *dso, struct machine *machine, int *fd) + EXCLUSIVE_TRYLOCK_FUNCTION(true, _dso__data_open_lock); +void dso__data_put_fd(struct dso *dso) UNLOCK_FUNCTION(_dso__data_open_lock); +void dso__data_close(struct dso *dso) LOCKS_EXCLUDED(_dso__data_open_lock); + +int dso__data_file_size(struct dso *dso, struct machine *machine); +off_t dso__data_size(struct dso *dso, struct machine *machine); +ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size); +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags) +{ + return dso__read_e_machine_endian(optional_dso, fd, e_flags, NULL); +} +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags) +{ + return dso__e_machine_endian(dso, machine, e_flags, NULL); +} +ssize_t dso__data_read_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + u8 *data, ssize_t size); +bool dso__data_status_seen(struct dso *dso, enum dso_data_status_seen by); +ssize_t dso__data_write_cache_offs(struct dso *dso, struct machine *machine, + u64 offset, const u8 *data, ssize_t size); +ssize_t dso__data_write_cache_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + const u8 *data, ssize_t size); + +struct map *dso__new_map(const char *name); +struct dso *machine__findnew_kernel(struct machine *machine, const char *name, + const char *short_name, int dso_type); + +void dso__reset_find_symbol_cache(struct dso *dso); + +size_t dso__fprintf_symbols_by_name(struct dso *dso, FILE *fp); +size_t dso__fprintf(struct dso *dso, FILE *fp); + +static inline enum dso_binary_type dso__binary_type(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->binary_type; +} + +static inline void dso__set_binary_type(struct dso *dso, enum dso_binary_type bt) +{ + RC_CHK_ACCESS(dso)->binary_type = bt; +} + +static inline bool dso__is_vmlinux(const struct dso *dso) +{ + enum dso_binary_type bt = dso__binary_type(dso); + + return bt == DSO_BINARY_TYPE__VMLINUX || bt == DSO_BINARY_TYPE__GUEST_VMLINUX; +} + +static inline bool dso__is_kcore(const struct dso *dso) +{ + enum dso_binary_type bt = dso__binary_type(dso); + + return bt == DSO_BINARY_TYPE__KCORE || bt == DSO_BINARY_TYPE__GUEST_KCORE; +} + +static inline bool dso__is_kallsyms(const struct dso *dso) +{ + enum dso_binary_type bt = dso__binary_type(dso); + const char *name; + + if (bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS) + return true; + + if (bt != DSO_BINARY_TYPE__NOT_FOUND) + return false; + + if (!dso__kernel(dso)) + return false; + + name = dso__long_name(dso); + if (!name) + return false; + + if (!strcmp(name, DSO__NAME_KALLSYMS)) + return true; + + if (!strcmp(name, DSO__NAME_GUEST_KALLSYMS)) + return true; + + return is_guest_kallsyms_pid_name(name); +} + +bool dso__is_object_file(const struct dso *dso); + +void dso__free_a2l(struct dso *dso); + +enum dso_type dso__type(struct dso *dso, struct machine *machine); + +int dso__strerror_load(struct dso *dso, char *buf, size_t buflen); + +void reset_fd_limit(void); + +u64 dso__find_global_type(struct dso *dso, u64 addr); +u64 dso__findnew_global_type(struct dso *dso, u64 addr, u64 offset); + +/* Check if dso name is of format "/tmp/perf-%d.map" */ +bool perf_pid_map_tid(const char *dso_name, int *tid); +bool is_perf_pid_map_name(const char *dso_name); + +struct debuginfo *dso__debuginfo(struct dso *dso); + +const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, + const struct map *map, const struct symbol *sym, + u8 **out_buf, u64 *out_buf_len, bool *is_64bit); + +#endif /* __PERF_DSO */ diff --git a/tools/perf/util/dsos.c b/tools/perf/util/dsos.c new file mode 100644 index 000000000..e927e707a --- /dev/null +++ b/tools/perf/util/dsos.c @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "debug.h" +#include "dsos.h" +#include "dso.h" +#include "util.h" +#include "vdso.h" +#include "namespaces.h" +#include +#include +#include +#include // filename__read_build_id +#include + +void dsos__init(struct dsos *dsos) +{ + init_rwsem(&dsos->lock); + + dsos->cnt = 0; + dsos->allocated = 0; + dsos->dsos = NULL; + dsos->sorted = true; +} + +static void dsos__purge(struct dsos *dsos) +{ + down_write(&dsos->lock); + + for (unsigned int i = 0; i < dsos->cnt; i++) { + struct dso *dso = dsos->dsos[i]; + + dso__set_dsos(dso, NULL); + dso__put(dso); + } + + zfree(&dsos->dsos); + dsos->cnt = 0; + dsos->allocated = 0; + dsos->sorted = true; + + up_write(&dsos->lock); +} + +void dsos__exit(struct dsos *dsos) +{ + dsos__purge(dsos); + exit_rwsem(&dsos->lock); +} + + +static int __dsos__for_each_dso(struct dsos *dsos, + int (*cb)(struct dso *dso, void *data), + void *data) +{ + for (unsigned int i = 0; i < dsos->cnt; i++) { + struct dso *dso = dsos->dsos[i]; + int err; + + err = cb(dso, data); + if (err) + return err; + } + return 0; +} + +struct dsos__read_build_ids_cb_args { + bool with_hits; + bool have_build_id; +}; + +static int dsos__read_build_ids_cb(struct dso *dso, void *data) +{ + struct dsos__read_build_ids_cb_args *args = data; + struct nscookie nsc; + struct build_id bid = { .size = 0, }; + + if (args->with_hits && !dso__hit(dso) && !dso__is_vdso(dso)) + return 0; + if (dso__has_build_id(dso)) { + args->have_build_id = true; + return 0; + } + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + if (filename__read_build_id(dso__long_name(dso), &bid) > 0) { + dso__set_build_id(dso, &bid); + args->have_build_id = true; + } else if (errno == ENOENT && dso__nsinfo(dso)) { + char *new_name = dso__filename_with_chroot(dso, dso__long_name(dso)); + + if (new_name && filename__read_build_id(new_name, &bid) > 0) { + dso__set_build_id(dso, &bid); + args->have_build_id = true; + } + free(new_name); + } + nsinfo__mountns_exit(&nsc); + return 0; +} + +bool dsos__read_build_ids(struct dsos *dsos, bool with_hits) +{ + struct dsos__read_build_ids_cb_args args = { + .with_hits = with_hits, + .have_build_id = false, + }; + + dsos__for_each_dso(dsos, dsos__read_build_ids_cb, &args); + return args.have_build_id; +} + +static int __dso__cmp_long_name(const char *long_name, const struct dso_id *id, + const struct dso *b) +{ + int rc = strcmp(long_name, dso__long_name(b)); + return rc ?: dso_id__cmp(id, dso__id_const(b)); +} + +static int __dso__cmp_short_name(const char *short_name, const struct dso_id *id, + const struct dso *b) +{ + int rc = strcmp(short_name, dso__short_name(b)); + return rc ?: dso_id__cmp(id, dso__id_const(b)); +} + +static int dsos__cmp_long_name_id_short_name(const void *va, const void *vb) +{ + const struct dso *a = *((const struct dso **)va); + const struct dso *b = *((const struct dso **)vb); + int rc = strcmp(dso__long_name(a), dso__long_name(b)); + + if (!rc) { + rc = dso_id__cmp(dso__id_const(a), dso__id_const(b)); + if (!rc) + rc = strcmp(dso__short_name(a), dso__short_name(b)); + } + return rc; +} + +struct dsos__key { + const char *long_name; + const struct dso_id *id; +}; + +static int dsos__cmp_key_long_name_id(const void *vkey, const void *vdso) +{ + const struct dsos__key *key = vkey; + const struct dso *dso = *((const struct dso **)vdso); + + return __dso__cmp_long_name(key->long_name, key->id, dso); +} + +/* + * Find a matching entry and/or link current entry to RB tree. + * Either one of the dso or name parameter must be non-NULL or the + * function will not work. + */ +static struct dso *__dsos__find_by_longname_id(struct dsos *dsos, + const char *name, + const struct dso_id *id, + bool write_locked) + SHARED_LOCKS_REQUIRED(dsos->lock) +{ + struct dsos__key key = { + .long_name = name, + .id = id, + }; + struct dso **res; + + if (dsos->dsos == NULL) + return NULL; + + if (!dsos->sorted) { + if (!write_locked) { + struct dso *dso; + + up_read(&dsos->lock); + down_write(&dsos->lock); + dso = __dsos__find_by_longname_id(dsos, name, id, + /*write_locked=*/true); + up_write(&dsos->lock); + down_read(&dsos->lock); + return dso; + } + qsort(dsos->dsos, dsos->cnt, sizeof(struct dso *), + dsos__cmp_long_name_id_short_name); + dsos->sorted = true; + } + + res = bsearch(&key, dsos->dsos, dsos->cnt, sizeof(struct dso *), + dsos__cmp_key_long_name_id); + if (!res) + return NULL; + + return dso__get(*res); +} + +int __dsos__add(struct dsos *dsos, struct dso *dso) +{ + if (!dso) + return -EINVAL; + + if (dsos->cnt == dsos->allocated) { + unsigned int to_allocate = 2; + struct dso **temp; + + if (dsos->allocated > 0) + to_allocate = dsos->allocated * 2; + temp = realloc(dsos->dsos, sizeof(struct dso *) * to_allocate); + if (!temp) + return -ENOMEM; + dsos->dsos = temp; + dsos->allocated = to_allocate; + } + if (!dsos->sorted) { + dsos->dsos[dsos->cnt++] = dso__get(dso); + } else { + int low = 0, high = dsos->cnt - 1; + int insert = dsos->cnt; /* Default to inserting at the end. */ + + while (low <= high) { + int mid = low + (high - low) / 2; + int cmp = dsos__cmp_long_name_id_short_name(&dsos->dsos[mid], &dso); + + if (cmp < 0) { + low = mid + 1; + } else { + high = mid - 1; + insert = mid; + } + } + memmove(&dsos->dsos[insert + 1], &dsos->dsos[insert], + (dsos->cnt - insert) * sizeof(struct dso *)); + dsos->cnt++; + dsos->dsos[insert] = dso__get(dso); + } + dso__set_dsos(dso, dsos); + return 0; +} + +int dsos__add(struct dsos *dsos, struct dso *dso) +{ + int ret; + + down_write(&dsos->lock); + ret = __dsos__add(dsos, dso); + up_write(&dsos->lock); + return ret; +} + +struct dsos__find_id_cb_args { + const char *name; + const struct dso_id *id; + struct dso *res; +}; + +static int dsos__find_id_cb(struct dso *dso, void *data) +{ + struct dsos__find_id_cb_args *args = data; + + if (__dso__cmp_short_name(args->name, args->id, dso) == 0) { + args->res = dso__get(dso); + return 1; + } + return 0; + +} + +static struct dso *__dsos__find_id(struct dsos *dsos, const char *name, const struct dso_id *id, + bool cmp_short, bool write_locked) + SHARED_LOCKS_REQUIRED(dsos->lock) +{ + struct dso *res; + + if (cmp_short) { + struct dsos__find_id_cb_args args = { + .name = name, + .id = id, + .res = NULL, + }; + + __dsos__for_each_dso(dsos, dsos__find_id_cb, &args); + return args.res; + } + res = __dsos__find_by_longname_id(dsos, name, id, write_locked); + return res; +} + +struct dso *dsos__find(struct dsos *dsos, const char *name, bool cmp_short) +{ + struct dso *res; + + down_read(&dsos->lock); + res = __dsos__find_id(dsos, name, &dso_id_empty, cmp_short, /*write_locked=*/false); + up_read(&dsos->lock); + return res; +} + +static void dso__set_basename(struct dso *dso) +{ + bool allocated = false; + const char *base; + int tid; + + if (perf_pid_map_tid(dso__long_name(dso), &tid)) { + char *jitname; + + if (asprintf(&jitname, "[JIT] tid %d", tid) < 0) + return; + allocated = true; + base = jitname; + } else { + base = perf_basename(dso__long_name(dso)); + } + dso__set_short_name(dso, base, allocated); +} + +static struct dso *__dsos__addnew_id(struct dsos *dsos, const char *name, const struct dso_id *id) +{ + struct dso *dso = dso__new_id(name, id); + + if (dso != NULL) { + /* + * The dsos lock is held on entry, so rename the dso before + * adding it to avoid needing to take the dsos lock again to say + * the array isn't sorted. + */ + dso__set_basename(dso); + __dsos__add(dsos, dso); + } + return dso; +} + +static struct dso *__dsos__findnew_id(struct dsos *dsos, const char *name, const struct dso_id *id) + SHARED_LOCKS_REQUIRED(dsos->lock) +{ + struct dso *dso = __dsos__find_id(dsos, name, id, false, /*write_locked=*/true); + + if (dso) + __dso__improve_id(dso, id); + + return dso ? dso : __dsos__addnew_id(dsos, name, id); +} + +struct dso *dsos__findnew_id(struct dsos *dsos, const char *name, const struct dso_id *id) +{ + struct dso *dso; + down_write(&dsos->lock); + dso = __dsos__findnew_id(dsos, name, id); + up_write(&dsos->lock); + return dso; +} + +struct dsos__fprintf_buildid_cb_args { + FILE *fp; + bool (*skip)(struct dso *dso, int parm); + int parm; + size_t ret; +}; + +static int dsos__fprintf_buildid_cb(struct dso *dso, void *data) +{ + struct dsos__fprintf_buildid_cb_args *args = data; + char sbuild_id[SBUILD_ID_SIZE]; + + if (args->skip && args->skip(dso, args->parm)) + return 0; + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + args->ret += fprintf(args->fp, "%-40s %s\n", sbuild_id, dso__long_name(dso)); + return 0; +} + +size_t dsos__fprintf_buildid(struct dsos *dsos, FILE *fp, + bool (*skip)(struct dso *dso, int parm), int parm) +{ + struct dsos__fprintf_buildid_cb_args args = { + .fp = fp, + .skip = skip, + .parm = parm, + .ret = 0, + }; + + dsos__for_each_dso(dsos, dsos__fprintf_buildid_cb, &args); + return args.ret; +} + +struct dsos__fprintf_cb_args { + FILE *fp; + size_t ret; +}; + +static int dsos__fprintf_cb(struct dso *dso, void *data) +{ + struct dsos__fprintf_cb_args *args = data; + + args->ret += dso__fprintf(dso, args->fp); + return 0; +} + +size_t dsos__fprintf(struct dsos *dsos, FILE *fp) +{ + struct dsos__fprintf_cb_args args = { + .fp = fp, + .ret = 0, + }; + + dsos__for_each_dso(dsos, dsos__fprintf_cb, &args); + return args.ret; +} + +static int dsos__hit_all_cb(struct dso *dso, void *data __maybe_unused) +{ + dso__set_hit(dso); + return 0; +} + +int dsos__hit_all(struct dsos *dsos) +{ + return dsos__for_each_dso(dsos, dsos__hit_all_cb, NULL); +} + +struct dso *dsos__findnew_module_dso(struct dsos *dsos, + struct machine *machine, + struct kmod_path *m, + const char *filename) +{ + struct dso *dso; + + down_write(&dsos->lock); + + dso = __dsos__find_id(dsos, m->name, &dso_id_empty, /*cmp_short=*/true, + /*write_locked=*/true); + if (dso) { + up_write(&dsos->lock); + return dso; + } + /* + * Failed to find the dso so create it. Change the name before adding it + * to the array, to avoid unnecessary sorts and potential locking + * issues. + */ + dso = dso__new_id(m->name, /*id=*/NULL); + if (!dso) { + up_write(&dsos->lock); + return NULL; + } + dso__set_basename(dso); + dso__set_module_info(dso, m, machine); + dso__set_long_name(dso, strdup(filename), true); + dso__set_kernel(dso, DSO_SPACE__KERNEL); + __dsos__add(dsos, dso); + + up_write(&dsos->lock); + return dso; +} + +static int dsos__find_kernel_dso_cb(struct dso *dso, void *data) +{ + struct dso **res = data; + /* + * The cpumode passed to is_kernel_module is not the cpumode of *this* + * event. If we insist on passing correct cpumode to is_kernel_module, + * we should record the cpumode when we adding this dso to the linked + * list. + * + * However we don't really need passing correct cpumode. We know the + * correct cpumode must be kernel mode (if not, we should not link it + * onto kernel_dsos list). + * + * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN. + * is_kernel_module() treats it as a kernel cpumode. + */ + if (!dso__kernel(dso) || + is_kernel_module(dso__long_name(dso), PERF_RECORD_MISC_CPUMODE_UNKNOWN)) + return 0; + + *res = dso__get(dso); + return 1; +} + +struct dso *dsos__find_kernel_dso(struct dsos *dsos) +{ + struct dso *res = NULL; + + dsos__for_each_dso(dsos, dsos__find_kernel_dso_cb, &res); + return res; +} + +int dsos__for_each_dso(struct dsos *dsos, int (*cb)(struct dso *dso, void *data), void *data) +{ + int err; + + down_read(&dsos->lock); + err = __dsos__for_each_dso(dsos, cb, data); + up_read(&dsos->lock); + return err; +} diff --git a/tools/perf/util/dsos.h b/tools/perf/util/dsos.h new file mode 100644 index 000000000..a26774950 --- /dev/null +++ b/tools/perf/util/dsos.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DSOS +#define __PERF_DSOS + +#include +#include +#include +#include +#include "rwsem.h" + +struct dso; +struct dso_id; +struct kmod_path; +struct machine; + +/* + * Collection of DSOs as an array for iteration speed, but sorted for O(n) + * lookup. + */ +struct dsos { + struct rw_semaphore lock; + struct dso **dsos; + unsigned int cnt; + unsigned int allocated; + bool sorted; +}; + +void dsos__init(struct dsos *dsos); +void dsos__exit(struct dsos *dsos); + +int __dsos__add(struct dsos *dsos, struct dso *dso); +int dsos__add(struct dsos *dsos, struct dso *dso); +struct dso *dsos__find(struct dsos *dsos, const char *name, bool cmp_short); + +struct dso *dsos__findnew_id(struct dsos *dsos, const char *name, const struct dso_id *id); + +bool dsos__read_build_ids(struct dsos *dsos, bool with_hits); + +size_t dsos__fprintf_buildid(struct dsos *dsos, FILE *fp, + bool (skip)(struct dso *dso, int parm), int parm); +size_t dsos__fprintf(struct dsos *dsos, FILE *fp); + +int dsos__hit_all(struct dsos *dsos); + +struct dso *dsos__findnew_module_dso(struct dsos *dsos, struct machine *machine, + struct kmod_path *m, const char *filename); + +struct dso *dsos__find_kernel_dso(struct dsos *dsos); + +int dsos__for_each_dso(struct dsos *dsos, int (*cb)(struct dso *dso, void *data), void *data); + +#endif /* __PERF_DSOS */ diff --git a/tools/perf/util/dump-insn.h b/tools/perf/util/dump-insn.h new file mode 100644 index 000000000..20d4d7bb5 --- /dev/null +++ b/tools/perf/util/dump-insn.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DUMP_INSN_H +#define __PERF_DUMP_INSN_H 1 + +#define MAXINSN 15 + +#include + +struct thread; + +struct perf_insn { + /* Initialized by callers: */ + struct thread *thread; + struct machine *machine; + u8 cpumode; + bool is64bit; + int cpu; + /* Temporary */ + char out[256]; +}; + +const char *dump_insn(struct perf_insn *x, u64 ip, + u8 *inbuf, int inlen, int *lenp); +int arch_is_uncond_branch(const unsigned char *buf, size_t len, int x86_64); + +#endif diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c new file mode 100644 index 000000000..d7160f87a --- /dev/null +++ b/tools/perf/util/dwarf-aux.c @@ -0,0 +1,2182 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dwarf-aux.c : libdw auxiliary interfaces + */ + +#include +#include +#include +#include +#include "debug.h" +#include "dwarf-aux.h" +#include "dwarf-regs.h" +#include "strbuf.h" +#include "string2.h" + +/** + * cu_find_realpath - Find the realpath of the target file + * @cu_die: A DIE(dwarf information entry) of CU(compilation Unit) + * @fname: The tail filename of the target file + * + * Find the real(long) path of @fname in @cu_die. + */ +const char *cu_find_realpath(Dwarf_Die *cu_die, const char *fname) +{ + Dwarf_Files *files; + size_t nfiles, i; + const char *src = NULL; + int ret; + + if (!fname) + return NULL; + + ret = dwarf_getsrcfiles(cu_die, &files, &nfiles); + if (ret != 0) + return NULL; + + for (i = 0; i < nfiles; i++) { + src = dwarf_filesrc(files, i, NULL, NULL); + if (strtailcmp(src, fname) == 0) + break; + } + if (i == nfiles) + return NULL; + return src; +} + +/** + * cu_get_comp_dir - Get the path of compilation directory + * @cu_die: a CU DIE + * + * Get the path of compilation directory of given @cu_die. + * Since this depends on DW_AT_comp_dir, older gcc will not + * embedded it. In that case, this returns NULL. + */ +const char *cu_get_comp_dir(Dwarf_Die *cu_die) +{ + Dwarf_Attribute attr; + if (dwarf_attr(cu_die, DW_AT_comp_dir, &attr) == NULL) + return NULL; + return dwarf_formstring(&attr); +} + +/* Unlike dwarf_getsrc_die(), cu_getsrc_die() only returns statement line */ +static Dwarf_Line *cu_getsrc_die(Dwarf_Die *cu_die, Dwarf_Addr addr) +{ + Dwarf_Addr laddr; + Dwarf_Lines *lines; + Dwarf_Line *line; + size_t nlines, l, u, n; + bool flag; + + if (dwarf_getsrclines(cu_die, &lines, &nlines) != 0 || + nlines == 0) + return NULL; + + /* Lines are sorted by address, use binary search */ + l = 0; u = nlines - 1; + while (l < u) { + n = u - (u - l) / 2; + line = dwarf_onesrcline(lines, n); + if (!line || dwarf_lineaddr(line, &laddr) != 0) + return NULL; + if (addr < laddr) + u = n - 1; + else + l = n; + } + /* Going backward to find the lowest line */ + do { + line = dwarf_onesrcline(lines, --l); + if (!line || dwarf_lineaddr(line, &laddr) != 0) + return NULL; + } while (laddr == addr); + l++; + /* Going forward to find the statement line */ + do { + line = dwarf_onesrcline(lines, l++); + if (!line || dwarf_lineaddr(line, &laddr) != 0 || + dwarf_linebeginstatement(line, &flag) != 0) + return NULL; + if (laddr > addr) + return NULL; + } while (!flag); + + return line; +} + +/** + * cu_find_lineinfo - Get a line number and file name for given address + * @cu_die: a CU DIE + * @addr: An address + * @fname: a pointer which returns the file name string + * @lineno: a pointer which returns the line number + * + * Find a line number and file name for @addr in @cu_die. + */ +int cu_find_lineinfo(Dwarf_Die *cu_die, Dwarf_Addr addr, + const char **fname, int *lineno) +{ + Dwarf_Line *line; + Dwarf_Die die_mem; + Dwarf_Addr faddr; + + if (die_find_realfunc(cu_die, addr, &die_mem) + && die_entrypc(&die_mem, &faddr) == 0 && + faddr == addr) { + *fname = die_get_decl_file(&die_mem); + if (dwarf_decl_line(&die_mem, lineno) != 0) + return -ENOENT; + goto out; + } + + line = cu_getsrc_die(cu_die, addr); + if (line && dwarf_lineno(line, lineno) == 0) { + *fname = dwarf_linesrc(line, NULL, NULL); + if (!*fname) + /* line number is useless without filename */ + *lineno = 0; + } + +out: + return (*lineno && *fname) ? *lineno : -ENOENT; +} + +static int __die_find_inline_cb(Dwarf_Die *die_mem, void *data); + +/** + * cu_walk_functions_at - Walk on function DIEs at given address + * @cu_die: A CU DIE + * @addr: An address + * @callback: A callback which called with found DIEs + * @data: A user data + * + * Walk on function DIEs at given @addr in @cu_die. Passed DIEs + * should be subprogram or inlined-subroutines. + */ +int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, + int (*callback)(Dwarf_Die *, void *), void *data) +{ + Dwarf_Die die_mem; + Dwarf_Die *sc_die; + int ret = -ENOENT; + + /* Inlined function could be recursive. Trace it until fail */ + for (sc_die = die_find_realfunc(cu_die, addr, &die_mem); + sc_die != NULL; + sc_die = die_find_child(sc_die, __die_find_inline_cb, &addr, + &die_mem)) { + ret = callback(sc_die, data); + if (ret) + break; + } + + return ret; +} + +/** + * die_get_linkage_name - Get the linkage name of the object + * @dw_die: A DIE of the object + * + * Get the linkage name attribute of given @dw_die. + * For C++ binary, the linkage name will be the mangled symbol. + */ +const char *die_get_linkage_name(Dwarf_Die *dw_die) +{ + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(dw_die, DW_AT_linkage_name, &attr) == NULL) + return NULL; + return dwarf_formstring(&attr); +} + +/** + * die_compare_name - Compare diename and tname + * @dw_die: a DIE + * @tname: a string of target name + * + * Compare the name of @dw_die and @tname. Return false if @dw_die has no name. + */ +bool die_compare_name(Dwarf_Die *dw_die, const char *tname) +{ + const char *name; + + name = dwarf_diename(dw_die); + return name ? (strcmp(tname, name) == 0) : false; +} + +/** + * die_match_name - Match diename/linkage name and glob + * @dw_die: a DIE + * @glob: a string of target glob pattern + * + * Glob matching the name of @dw_die and @glob. Return false if matching fail. + * This also match linkage name. + */ +bool die_match_name(Dwarf_Die *dw_die, const char *glob) +{ + const char *name; + + name = dwarf_diename(dw_die); + if (name && strglobmatch(name, glob)) + return true; + /* fall back to check linkage name */ + name = die_get_linkage_name(dw_die); + if (name && strglobmatch(name, glob)) + return true; + + return false; +} + +/** + * die_get_call_lineno - Get callsite line number of inline-function instance + * @in_die: a DIE of an inlined function instance + * + * Get call-site line number of @in_die. This means from where the inline + * function is called. + */ +int die_get_call_lineno(Dwarf_Die *in_die) +{ + Dwarf_Attribute attr; + Dwarf_Word ret; + + if (!dwarf_attr(in_die, DW_AT_call_line, &attr)) + return -ENOENT; + + dwarf_formudata(&attr, &ret); + return (int)ret; +} + +/** + * die_get_type - Get type DIE + * @vr_die: a DIE of a variable + * @die_mem: where to store a type DIE + * + * Get a DIE of the type of given variable (@vr_die), and store + * it to die_mem. Return NULL if fails to get a type DIE. + */ +Dwarf_Die *die_get_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem) +{ + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(vr_die, DW_AT_type, &attr) && + dwarf_formref_die(&attr, die_mem)) + return die_mem; + else + return NULL; +} + +/* Get a type die, but skip qualifiers */ +Dwarf_Die *__die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem) +{ + int tag; + + do { + vr_die = die_get_type(vr_die, die_mem); + if (!vr_die) + break; + tag = dwarf_tag(vr_die); + } while (tag == DW_TAG_const_type || + tag == DW_TAG_restrict_type || + tag == DW_TAG_volatile_type || + tag == DW_TAG_shared_type); + + return vr_die; +} + +/** + * die_get_real_type - Get a type die, but skip qualifiers and typedef + * @vr_die: a DIE of a variable + * @die_mem: where to store a type DIE + * + * Get a DIE of the type of given variable (@vr_die), and store + * it to die_mem. Return NULL if fails to get a type DIE. + * If the type is qualifiers (e.g. const) or typedef, this skips it + * and tries to find real type (structure or basic types, e.g. int). + */ +Dwarf_Die *die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem) +{ + do { + vr_die = __die_get_real_type(vr_die, die_mem); + } while (vr_die && dwarf_tag(vr_die) == DW_TAG_typedef); + + return vr_die; +} + +/** + * die_get_pointer_type - Get a pointer/array type die + * @type_die: a DIE of a type + * @die_mem: where to store a type DIE + * + * Get a pointer/array type DIE from @type_die. If the type is a typedef or + * qualifier (const, volatile, etc.), follow the chain to find the underlying + * pointer type. + */ +Dwarf_Die *die_get_pointer_type(Dwarf_Die *type_die, Dwarf_Die *die_mem) +{ + int tag; + + do { + tag = dwarf_tag(type_die); + if (tag == DW_TAG_pointer_type || tag == DW_TAG_array_type) + return type_die; + if (tag != DW_TAG_typedef && tag != DW_TAG_const_type && + tag != DW_TAG_restrict_type && tag != DW_TAG_volatile_type && + tag != DW_TAG_shared_type) + return NULL; + type_die = die_get_type(type_die, die_mem); + } while (type_die); + + return NULL; +} + +/* Get attribute and translate it as a udata */ +static int die_get_attr_udata(Dwarf_Die *tp_die, unsigned int attr_name, + Dwarf_Word *result) +{ + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(tp_die, attr_name, &attr) == NULL || + dwarf_formudata(&attr, result) != 0) + return -ENOENT; + + return 0; +} + +/** + * die_is_signed_type - Check whether a type DIE is signed or not + * @tp_die: a DIE of a type + * + * Get the encoding of @tp_die and return true if the encoding + * is signed. + */ +bool die_is_signed_type(Dwarf_Die *tp_die) +{ + Dwarf_Word ret; + + if (die_get_attr_udata(tp_die, DW_AT_encoding, &ret)) + return false; + + return (ret == DW_ATE_signed_char || ret == DW_ATE_signed || + ret == DW_ATE_signed_fixed); +} + +/** + * die_is_func_def - Ensure that this DIE is a subprogram and definition + * @dw_die: a DIE + * + * Ensure that this DIE is a subprogram and NOT a declaration. This + * returns true if @dw_die is a function definition. + **/ +bool die_is_func_def(Dwarf_Die *dw_die) +{ + Dwarf_Attribute attr; + Dwarf_Addr addr = 0; + + if (dwarf_tag(dw_die) != DW_TAG_subprogram) + return false; + + if (dwarf_attr(dw_die, DW_AT_declaration, &attr)) + return false; + + /* + * DW_AT_declaration can be lost from function declaration + * by gcc's bug #97060. + * So we need to check this subprogram DIE has DW_AT_inline + * or an entry address. + */ + if (!dwarf_attr(dw_die, DW_AT_inline, &attr) && + die_entrypc(dw_die, &addr) < 0) + return false; + + return true; +} + +/** + * die_entrypc - Returns entry PC (the lowest address) of a DIE + * @dw_die: a DIE + * @addr: where to store entry PC + * + * Since dwarf_entrypc() does not return entry PC if the DIE has only address + * range, we have to use this to retrieve the lowest address from the address + * range attribute. + */ +int die_entrypc(Dwarf_Die *dw_die, Dwarf_Addr *addr) +{ + Dwarf_Addr base, end; + Dwarf_Attribute attr; + + if (!addr) + return -EINVAL; + + if (dwarf_entrypc(dw_die, addr) == 0) + return 0; + + /* + * Since the dwarf_ranges() will return 0 if there is no + * DW_AT_ranges attribute, we should check it first. + */ + if (!dwarf_attr(dw_die, DW_AT_ranges, &attr)) + return -ENOENT; + + return dwarf_ranges(dw_die, 0, &base, addr, &end) < 0 ? -ENOENT : 0; +} + +/** + * die_is_func_instance - Ensure that this DIE is an instance of a subprogram + * @dw_die: a DIE + * + * Ensure that this DIE is an instance (which has an entry address). + * This returns true if @dw_die is a function instance. If not, the @dw_die + * must be a prototype. You can use die_walk_instances() to find actual + * instances. + **/ +bool die_is_func_instance(Dwarf_Die *dw_die) +{ + Dwarf_Addr tmp; + Dwarf_Attribute attr_mem; + int tag = dwarf_tag(dw_die); + + if (tag != DW_TAG_subprogram && + tag != DW_TAG_inlined_subroutine) + return false; + + return dwarf_entrypc(dw_die, &tmp) == 0 || + dwarf_attr(dw_die, DW_AT_ranges, &attr_mem) != NULL; +} + +/** + * die_get_data_member_location - Get the data-member offset + * @mb_die: a DIE of a member of a data structure + * @offs: The offset of the member in the data structure + * + * Get the offset of @mb_die in the data structure including @mb_die, and + * stores result offset to @offs. If any error occurs this returns errno. + */ +int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs) +{ + Dwarf_Attribute attr; + Dwarf_Op *expr; + size_t nexpr; + int ret; + + if (dwarf_attr_integrate(mb_die, DW_AT_data_member_location, &attr) == NULL) + return -ENOENT; + + if (dwarf_formudata(&attr, offs) != 0) { + /* DW_AT_data_member_location should be DW_OP_plus_uconst */ + ret = dwarf_getlocation(&attr, &expr, &nexpr); + if (ret < 0 || nexpr == 0) + return -ENOENT; + + if (expr[0].atom != DW_OP_plus_uconst || nexpr != 1) { + pr_debug("Unable to get offset:Unexpected OP %x (%zd)\n", + expr[0].atom, nexpr); + return -ENOTSUP; + } + *offs = (Dwarf_Word)expr[0].number; + } + return 0; +} + +/* Get the call file index number in CU DIE */ +static int die_get_call_fileno(Dwarf_Die *in_die) +{ + Dwarf_Word idx; + + if (die_get_attr_udata(in_die, DW_AT_call_file, &idx) == 0) + return (int)idx; + else + return -ENOENT; +} + +/* Get the declared file index number in CU DIE */ +static int die_get_decl_fileno(Dwarf_Die *pdie) +{ + Dwarf_Word idx; + + if (die_get_attr_udata(pdie, DW_AT_decl_file, &idx) == 0) + return (int)idx; + else + return -ENOENT; +} + +/* Return the file name by index */ +static const char *die_get_file_name(Dwarf_Die *dw_die, int idx) +{ + Dwarf_Die cu_die; + Dwarf_Files *files; + Dwarf_Attribute attr_mem; + + if (idx < 0 || !dwarf_attr_integrate(dw_die, DW_AT_decl_file, &attr_mem) || + !dwarf_cu_die(attr_mem.cu, &cu_die, NULL, NULL, NULL, NULL, NULL, NULL) || + dwarf_getsrcfiles(&cu_die, &files, NULL) != 0) + return NULL; + + return dwarf_filesrc(files, idx, NULL, NULL); +} + +/** + * die_get_call_file - Get callsite file name of inlined function instance + * @in_die: a DIE of an inlined function instance + * + * Get call-site file name of @in_die. This means from which file the inline + * function is called. + */ +const char *die_get_call_file(Dwarf_Die *in_die) +{ + return die_get_file_name(in_die, die_get_call_fileno(in_die)); +} + +/** + * die_get_decl_file - Find the declared file name of this DIE + * @dw_die: a DIE for something declared. + * + * Get declared file name of @dw_die. + * NOTE: Since some version of clang DWARF5 implementation incorrectly uses + * file index 0 for DW_AT_decl_file, die_get_decl_file() will return NULL for + * such cases. Use this function instead. + */ +const char *die_get_decl_file(Dwarf_Die *dw_die) +{ + return die_get_file_name(dw_die, die_get_decl_fileno(dw_die)); +} + +/** + * die_find_child - Generic DIE search function in DIE tree + * @rt_die: a root DIE + * @callback: a callback function + * @data: a user data passed to the callback function + * @die_mem: a buffer for result DIE + * + * Trace DIE tree from @rt_die and call @callback for each child DIE. + * If @callback returns DIE_FIND_CB_END, this stores the DIE into + * @die_mem and returns it. If @callback returns DIE_FIND_CB_CONTINUE, + * this continues to trace the tree. Optionally, @callback can return + * DIE_FIND_CB_CHILD and DIE_FIND_CB_SIBLING, those means trace only + * the children and trace only the siblings respectively. + * Returns NULL if @callback can't find any appropriate DIE. + */ +Dwarf_Die *die_find_child(Dwarf_Die *rt_die, + int (*callback)(Dwarf_Die *, void *), + void *data, Dwarf_Die *die_mem) +{ + Dwarf_Die child_die; + int ret; + + ret = dwarf_child(rt_die, die_mem); + if (ret != 0) + return NULL; + + do { + ret = callback(die_mem, data); + if (ret == DIE_FIND_CB_END) + return die_mem; + + if ((ret & DIE_FIND_CB_CHILD) && + die_find_child(die_mem, callback, data, &child_die)) { + memcpy(die_mem, &child_die, sizeof(Dwarf_Die)); + return die_mem; + } + } while ((ret & DIE_FIND_CB_SIBLING) && + dwarf_siblingof(die_mem, die_mem) == 0); + + return NULL; +} + +struct __addr_die_search_param { + Dwarf_Addr addr; + Dwarf_Die *die_mem; +}; + +static int __die_search_func_tail_cb(Dwarf_Die *fn_die, void *data) +{ + struct __addr_die_search_param *ad = data; + Dwarf_Addr addr = 0; + + if (dwarf_tag(fn_die) == DW_TAG_subprogram && + !dwarf_highpc(fn_die, &addr) && + addr == ad->addr) { + memcpy(ad->die_mem, fn_die, sizeof(Dwarf_Die)); + return DWARF_CB_ABORT; + } + return DWARF_CB_OK; +} + +/** + * die_find_tailfunc - Search for a non-inlined function with tail call at + * given address + * @cu_die: a CU DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search for a non-inlined function DIE with tail call at @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + */ +Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + struct __addr_die_search_param ad; + ad.addr = addr; + ad.die_mem = die_mem; + /* dwarf_getscopes can't find subprogram. */ + if (dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0) <= 0) + return NULL; + else + return die_mem; +} + +/* die_find callback for non-inlined function search */ +static int __die_search_func_cb(Dwarf_Die *fn_die, void *data) +{ + struct __addr_die_search_param *ad = data; + + /* + * Since a declaration entry doesn't has given pc, this always returns + * function definition entry. + */ + if (dwarf_tag(fn_die) == DW_TAG_subprogram && + dwarf_haspc(fn_die, ad->addr)) { + memcpy(ad->die_mem, fn_die, sizeof(Dwarf_Die)); + return DWARF_CB_ABORT; + } + return DWARF_CB_OK; +} + +/** + * die_find_realfunc - Search a non-inlined function at given address + * @cu_die: a CU DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search a non-inlined function DIE which includes @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + */ +Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + struct __addr_die_search_param ad; + ad.addr = addr; + ad.die_mem = die_mem; + /* dwarf_getscopes can't find subprogram. */ + if (dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0) <= 0) + return NULL; + else + return die_mem; +} + +/* die_find callback for inline function search */ +static int __die_find_inline_cb(Dwarf_Die *die_mem, void *data) +{ + Dwarf_Addr *addr = data; + + if (dwarf_tag(die_mem) == DW_TAG_inlined_subroutine && + dwarf_haspc(die_mem, *addr)) + return DIE_FIND_CB_END; + + return DIE_FIND_CB_CONTINUE; +} + +/** + * die_find_top_inlinefunc - Search the top inlined function at given address + * @sp_die: a subprogram DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search an inlined function DIE which includes @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + * Even if several inlined functions are expanded recursively, this + * doesn't trace it down, and returns the topmost one. + */ +Dwarf_Die *die_find_top_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + return die_find_child(sp_die, __die_find_inline_cb, &addr, die_mem); +} + +/** + * die_find_inlinefunc - Search an inlined function at given address + * @sp_die: a subprogram DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search an inlined function DIE which includes @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + * If several inlined functions are expanded recursively, this trace + * it down and returns deepest one. + */ +Dwarf_Die *die_find_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + Dwarf_Die tmp_die; + + sp_die = die_find_child(sp_die, __die_find_inline_cb, &addr, &tmp_die); + if (!sp_die) + return NULL; + + /* Inlined function could be recursive. Trace it until fail */ + while (sp_die) { + memcpy(die_mem, sp_die, sizeof(Dwarf_Die)); + sp_die = die_find_child(sp_die, __die_find_inline_cb, &addr, + &tmp_die); + } + + return die_mem; +} + +static int __die_find_func_rettype_cb(Dwarf_Die *die_mem, void *data) +{ + const char *func_name; + + if (dwarf_tag(die_mem) != DW_TAG_subprogram) + return DIE_FIND_CB_SIBLING; + + func_name = dwarf_diename(die_mem); + if (func_name && !strcmp(func_name, data)) + return DIE_FIND_CB_END; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_func_rettype - Search a return type of function + * @cu_die: a CU DIE + * @name: target function name + * @die_mem: a buffer for result DIE + * + * Search a non-inlined function which matches to @name and stores the + * return type of the function to @die_mem and returns it if found. + * Returns NULL if failed. Note that it doesn't needs to find a + * definition of the function, so it doesn't match with address. + * Most likely, it can find a declaration at the top level. Thus the + * callback function continues to sibling entries only. + */ +Dwarf_Die *die_find_func_rettype(Dwarf_Die *cu_die, const char *name, + Dwarf_Die *die_mem) +{ + Dwarf_Die tmp_die; + + cu_die = die_find_child(cu_die, __die_find_func_rettype_cb, + (void *)name, &tmp_die); + if (!cu_die) + return NULL; + + if (die_get_real_type(&tmp_die, die_mem) == NULL) + return NULL; + + return die_mem; +} + +struct __instance_walk_param { + void *addr; + int (*callback)(Dwarf_Die *, void *); + void *data; + int retval; +}; + +static int __die_walk_instances_cb(Dwarf_Die *inst, void *data) +{ + struct __instance_walk_param *iwp = data; + Dwarf_Attribute attr_mem; + Dwarf_Die origin_mem; + Dwarf_Attribute *attr; + Dwarf_Die *origin; + int tmp; + + if (!die_is_func_instance(inst)) + return DIE_FIND_CB_CONTINUE; + + attr = dwarf_attr(inst, DW_AT_abstract_origin, &attr_mem); + if (attr == NULL) + return DIE_FIND_CB_CONTINUE; + + origin = dwarf_formref_die(attr, &origin_mem); + if (origin == NULL || origin->addr != iwp->addr) + return DIE_FIND_CB_CONTINUE; + + /* Ignore redundant instances */ + if (dwarf_tag(inst) == DW_TAG_inlined_subroutine) { + if (dwarf_decl_line(origin, &tmp) == 0 && die_get_call_lineno(inst) == tmp) { + tmp = die_get_decl_fileno(origin); + if (die_get_call_fileno(inst) == tmp) + return DIE_FIND_CB_CONTINUE; + } + } + + iwp->retval = iwp->callback(inst, iwp->data); + + return (iwp->retval) ? DIE_FIND_CB_END : DIE_FIND_CB_CONTINUE; +} + +/** + * die_walk_instances - Walk on instances of given DIE + * @or_die: an abstract original DIE + * @callback: a callback function which is called with instance DIE + * @data: user data + * + * Walk on the instances of give @in_die. @in_die must be an inlined function + * declaration. This returns the return value of @callback if it returns + * non-zero value, or -ENOENT if there is no instance. + */ +int die_walk_instances(Dwarf_Die *or_die, int (*callback)(Dwarf_Die *, void *), + void *data) +{ + Dwarf_Die cu_die; + Dwarf_Die die_mem; + struct __instance_walk_param iwp = { + .addr = or_die->addr, + .callback = callback, + .data = data, + .retval = -ENOENT, + }; + + if (dwarf_diecu(or_die, &cu_die, NULL, NULL) == NULL) + return -ENOENT; + + die_find_child(&cu_die, __die_walk_instances_cb, &iwp, &die_mem); + + return iwp.retval; +} + +/* Line walker internal parameters */ +struct __line_walk_param { + bool recursive; + line_walk_callback_t callback; + void *data; + int retval; +}; + +static int __die_walk_funclines_cb(Dwarf_Die *in_die, void *data) +{ + struct __line_walk_param *lw = data; + Dwarf_Addr addr = 0; + const char *fname; + int lineno; + + if (dwarf_tag(in_die) == DW_TAG_inlined_subroutine) { + fname = die_get_call_file(in_die); + lineno = die_get_call_lineno(in_die); + if (fname && lineno > 0 && die_entrypc(in_die, &addr) == 0) { + lw->retval = lw->callback(fname, lineno, addr, lw->data); + if (lw->retval != 0) + return DIE_FIND_CB_END; + } + if (!lw->recursive) + return DIE_FIND_CB_SIBLING; + } + + if (addr) { + fname = die_get_decl_file(in_die); + if (fname && dwarf_decl_line(in_die, &lineno) == 0) { + lw->retval = lw->callback(fname, lineno, addr, lw->data); + if (lw->retval != 0) + return DIE_FIND_CB_END; + } + } + + /* Continue to search nested inlined function call-sites */ + return DIE_FIND_CB_CONTINUE; +} + +/* Walk on lines of blocks included in given DIE */ +static int __die_walk_funclines(Dwarf_Die *sp_die, bool recursive, + line_walk_callback_t callback, void *data) +{ + struct __line_walk_param lw = { + .recursive = recursive, + .callback = callback, + .data = data, + .retval = 0, + }; + Dwarf_Die die_mem; + Dwarf_Addr addr; + const char *fname; + int lineno; + + /* Handle function declaration line */ + fname = die_get_decl_file(sp_die); + if (fname && dwarf_decl_line(sp_die, &lineno) == 0 && + die_entrypc(sp_die, &addr) == 0) { + lw.retval = callback(fname, lineno, addr, data); + if (lw.retval != 0) + goto done; + } + die_find_child(sp_die, __die_walk_funclines_cb, &lw, &die_mem); +done: + return lw.retval; +} + +static int __die_walk_culines_cb(Dwarf_Die *sp_die, void *data) +{ + struct __line_walk_param *lw = data; + + /* + * Since inlined function can include another inlined function in + * the same file, we need to walk in it recursively. + */ + lw->retval = __die_walk_funclines(sp_die, true, lw->callback, lw->data); + if (lw->retval != 0) + return DWARF_CB_ABORT; + + return DWARF_CB_OK; +} + +/** + * die_walk_lines - Walk on lines inside given DIE + * @rt_die: a root DIE (CU, subprogram or inlined_subroutine) + * @callback: callback routine + * @data: user data + * + * Walk on all lines inside given @rt_die and call @callback on each line. + * If the @rt_die is a function, walk only on the lines inside the function, + * otherwise @rt_die must be a CU DIE. + * Note that this walks not only dwarf line list, but also function entries + * and inline call-site. + */ +int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) +{ + Dwarf_Lines *lines; + Dwarf_Line *line; + Dwarf_Addr addr; + const char *fname, *decf = NULL, *inf = NULL; + int lineno, ret = 0; + int decl = 0, inl; + Dwarf_Die die_mem, *cu_die; + size_t nlines, i; + bool flag; + + /* Get the CU die */ + if (dwarf_tag(rt_die) != DW_TAG_compile_unit) { + cu_die = dwarf_diecu(rt_die, &die_mem, NULL, NULL); + dwarf_decl_line(rt_die, &decl); + decf = die_get_decl_file(rt_die); + } else + cu_die = rt_die; + if (!cu_die) { + pr_debug2("Failed to get CU from given DIE.\n"); + return -EINVAL; + } + + /* Get lines list in the CU */ + if (dwarf_getsrclines(cu_die, &lines, &nlines) != 0) { + pr_debug2("Failed to get source lines on this CU.\n"); + return -ENOENT; + } + pr_debug2("Get %zd lines from this CU\n", nlines); + + /* Walk on the lines on lines list */ + for (i = 0; i < nlines; i++) { + line = dwarf_onesrcline(lines, i); + if (line == NULL || + dwarf_lineno(line, &lineno) != 0 || + dwarf_lineaddr(line, &addr) != 0) { + pr_debug2("Failed to get line info. " + "Possible error in debuginfo.\n"); + continue; + } + /* Skip end-of-sequence */ + if (dwarf_lineendsequence(line, &flag) != 0 || flag) + continue; + /* Skip Non statement line-info */ + if (dwarf_linebeginstatement(line, &flag) != 0 || !flag) + continue; + /* Filter lines based on address */ + if (rt_die != cu_die) { + /* + * Address filtering + * The line is included in given function, and + * no inline block includes it. + */ + if (!dwarf_haspc(rt_die, addr)) + continue; + + if (die_find_inlinefunc(rt_die, addr, &die_mem)) { + /* Call-site check */ + inf = die_get_call_file(&die_mem); + if ((inf == decf || (inf && decf && !strcmp(inf, decf))) && + die_get_call_lineno(&die_mem) == lineno) + goto found; + + if (dwarf_decl_line(&die_mem, &inl) != 0) + inl = 0; + if (inl != decl || + decf != die_get_decl_file(&die_mem)) + continue; + } + } +found: + /* Get source line */ + fname = dwarf_linesrc(line, NULL, NULL); + + ret = callback(fname, lineno, addr, data); + if (ret != 0) + return ret; + } + + /* + * Dwarf lines doesn't include function declarations and inlined + * subroutines. We have to check functions list or given function. + */ + if (rt_die != cu_die) + /* + * Don't need walk inlined functions recursively, because + * inner inlined functions don't have the lines of the + * specified function. + */ + ret = __die_walk_funclines(rt_die, false, callback, data); + else { + struct __line_walk_param param = { + .callback = callback, + .data = data, + .retval = 0, + }; + if (dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0) < 0) + ret = -EINVAL; + else + ret = param.retval; + } + + return ret; +} + +struct __find_variable_param { + const char *name; + Dwarf_Addr addr; +}; + +static int __die_find_variable_cb(Dwarf_Die *die_mem, void *data) +{ + struct __find_variable_param *fvp = data; + Dwarf_Attribute attr; + int tag; + + tag = dwarf_tag(die_mem); + if ((tag == DW_TAG_formal_parameter || + tag == DW_TAG_variable) && + die_compare_name(die_mem, fvp->name) && + /* + * Does the DIE have location information or const value + * or external instance? + */ + (dwarf_attr(die_mem, DW_AT_external, &attr) || + dwarf_attr(die_mem, DW_AT_location, &attr) || + dwarf_attr(die_mem, DW_AT_const_value, &attr))) + return DIE_FIND_CB_END; + if (dwarf_haspc(die_mem, fvp->addr)) + return DIE_FIND_CB_CONTINUE; + else + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_variable_at - Find a given name variable at given address + * @sp_die: a function DIE + * @name: variable name + * @addr: address + * @die_mem: a buffer for result DIE + * + * Find a variable DIE called @name at @addr in @sp_die. + */ +Dwarf_Die *die_find_variable_at(Dwarf_Die *sp_die, const char *name, + Dwarf_Addr addr, Dwarf_Die *die_mem) +{ + struct __find_variable_param fvp = { .name = name, .addr = addr}; + + return die_find_child(sp_die, __die_find_variable_cb, (void *)&fvp, + die_mem); +} + +static int __die_find_member_cb(Dwarf_Die *die_mem, void *data) +{ + const char *name = data; + + if (dwarf_tag(die_mem) == DW_TAG_member) { + if (die_compare_name(die_mem, name)) + return DIE_FIND_CB_END; + else if (!dwarf_diename(die_mem)) { /* Unnamed structure */ + Dwarf_Die type_die, tmp_die; + if (die_get_type(die_mem, &type_die) && + die_find_member(&type_die, name, &tmp_die)) + return DIE_FIND_CB_END; + } + } + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_member - Find a given name member in a data structure + * @st_die: a data structure type DIE + * @name: member name + * @die_mem: a buffer for result DIE + * + * Find a member DIE called @name in @st_die. + */ +Dwarf_Die *die_find_member(Dwarf_Die *st_die, const char *name, + Dwarf_Die *die_mem) +{ + return die_find_child(st_die, __die_find_member_cb, (void *)name, + die_mem); +} + +/** + * die_get_typename_from_type - Get the name of given type DIE + * @type_die: a type DIE + * @buf: a strbuf for result type name + * + * Get the name of @type_die and stores it to @buf. Return 0 if succeeded. + * and Return -ENOENT if failed to find type name. + * Note that the result will stores typedef name if possible, and stores + * "*(function_type)" if the type is a function pointer. + */ +int die_get_typename_from_type(Dwarf_Die *type_die, struct strbuf *buf) +{ + int tag, ret; + const char *tmp = ""; + + tag = dwarf_tag(type_die); + if (tag == DW_TAG_pointer_type) + tmp = "*"; + else if (tag == DW_TAG_array_type) + tmp = "[]"; + else if (tag == DW_TAG_subroutine_type) { + /* Function pointer */ + return strbuf_add(buf, "(function_type)", 15); + } else { + const char *name = dwarf_diename(type_die); + + if (tag == DW_TAG_union_type) + tmp = "union "; + else if (tag == DW_TAG_structure_type) + tmp = "struct "; + else if (tag == DW_TAG_enumeration_type) + tmp = "enum "; + else if (name == NULL) + return -ENOENT; + /* Write a base name */ + return strbuf_addf(buf, "%s%s", tmp, name ?: ""); + } + ret = die_get_typename(type_die, buf); + if (ret < 0) { + /* void pointer has no type attribute */ + if (tag == DW_TAG_pointer_type && ret == -ENOENT) + return strbuf_addf(buf, "void*"); + + return ret; + } + return strbuf_addstr(buf, tmp); +} + +/** + * die_get_typename - Get the name of given variable DIE + * @vr_die: a variable DIE + * @buf: a strbuf for result type name + * + * Get the name of @vr_die and stores it to @buf. Return 0 if succeeded. + * and Return -ENOENT if failed to find type name. + * Note that the result will stores typedef name if possible, and stores + * "*(function_type)" if the type is a function pointer. + */ +int die_get_typename(Dwarf_Die *vr_die, struct strbuf *buf) +{ + Dwarf_Die type; + + if (__die_get_real_type(vr_die, &type) == NULL) + return -ENOENT; + + return die_get_typename_from_type(&type, buf); +} + +/** + * die_get_varname - Get the name and type of given variable DIE + * @vr_die: a variable DIE + * @buf: a strbuf for type and variable name + * + * Get the name and type of @vr_die and stores it in @buf as "type\tname". + */ +int die_get_varname(Dwarf_Die *vr_die, struct strbuf *buf) +{ + int ret; + + ret = die_get_typename(vr_die, buf); + if (ret < 0) { + pr_debug("Failed to get type, make it unknown.\n"); + ret = strbuf_add(buf, "(unknown_type)", 14); + } + + return ret < 0 ? ret : strbuf_addf(buf, "\t%s", dwarf_diename(vr_die)); +} + +static int reg_from_dwarf_op(Dwarf_Op *op) +{ + switch (op->atom) { + case DW_OP_reg0 ... DW_OP_reg31: + return op->atom - DW_OP_reg0; + case DW_OP_breg0 ... DW_OP_breg31: + return op->atom - DW_OP_breg0; + case DW_OP_regx: + case DW_OP_bregx: + return op->number; + case DW_OP_fbreg: + return DWARF_REG_FB; + default: + break; + } + return -1; +} + +static int offset_from_dwarf_op(Dwarf_Op *op) +{ + switch (op->atom) { + case DW_OP_reg0 ... DW_OP_reg31: + case DW_OP_regx: + return 0; + case DW_OP_breg0 ... DW_OP_breg31: + case DW_OP_fbreg: + return op->number; + case DW_OP_bregx: + return op->number2; + default: + break; + } + return -1; +} + +static bool check_allowed_ops(Dwarf_Op *ops, size_t nops) +{ + /* The first op is checked separately */ + ops++; + nops--; + + /* + * It needs to make sure if the location expression matches to the given + * register and offset exactly. Thus it rejects any complex expressions + * and only allows a few of selected operators that doesn't change the + * location. + */ + while (nops) { + switch (ops->atom) { + case DW_OP_stack_value: + case DW_OP_deref_size: + case DW_OP_deref: + case DW_OP_piece: + break; + default: + return false; + } + ops++; + nops--; + } + return true; +} + +/** + * die_get_var_innermost_scope - Get innermost scope range of given variable DIE + * @sp_die: a subprogram DIE + * @vr_die: a variable DIE + * @buf: a strbuf for variable byte offset range + * + * Get the innermost scope range of @vr_die and stores it in @buf as + * "@". + */ +static int die_get_var_innermost_scope(Dwarf_Die *sp_die, Dwarf_Die *vr_die, + struct strbuf *buf) +{ + Dwarf_Die *scopes; + int count; + size_t offset = 0; + Dwarf_Addr base; + Dwarf_Addr start, end; + Dwarf_Addr entry; + int ret; + bool first = true; + const char *name; + + ret = die_entrypc(sp_die, &entry); + if (ret) + return ret; + + name = dwarf_diename(sp_die); + if (!name) + return -ENOENT; + + count = dwarf_getscopes_die(vr_die, &scopes); + + /* (*SCOPES)[1] is the DIE for the scope containing that scope */ + if (count <= 1) { + ret = -EINVAL; + goto out; + } + + while ((offset = dwarf_ranges(&scopes[1], offset, &base, + &start, &end)) > 0) { + start -= entry; + end -= entry; + + if (first) { + ret = strbuf_addf(buf, "@<%s+[%" PRIu64 "-%" PRIu64, + name, start, end); + first = false; + } else { + ret = strbuf_addf(buf, ",%" PRIu64 "-%" PRIu64, + start, end); + } + if (ret < 0) + goto out; + } + + if (!first) + ret = strbuf_add(buf, "]>", 2); + +out: + free(scopes); + return ret; +} + +/** + * die_get_var_range - Get byte offset range of given variable DIE + * @sp_die: a subprogram DIE + * @vr_die: a variable DIE + * @buf: a strbuf for type and variable name and byte offset range + * + * Get the byte offset range of @vr_die and stores it in @buf as + * "@". + */ +int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf) +{ + int ret = 0; + Dwarf_Addr base; + Dwarf_Addr start, end; + Dwarf_Addr entry; + Dwarf_Op *op; + size_t nops; + size_t offset = 0; + Dwarf_Attribute attr; + bool first = true; + const char *name; + + ret = die_entrypc(sp_die, &entry); + if (ret) + return ret; + + name = dwarf_diename(sp_die); + if (!name) + return -ENOENT; + + if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL) + return -EINVAL; + + while ((offset = dwarf_getlocations(&attr, offset, &base, + &start, &end, &op, &nops)) > 0) { + if (start == 0) { + /* Single Location Descriptions */ + ret = die_get_var_innermost_scope(sp_die, vr_die, buf); + goto out; + } + + /* Location Lists */ + start -= entry; + end -= entry; + if (first) { + ret = strbuf_addf(buf, "@<%s+[%" PRIu64 "-%" PRIu64, + name, start, end); + first = false; + } else { + ret = strbuf_addf(buf, ",%" PRIu64 "-%" PRIu64, + start, end); + } + if (ret < 0) + goto out; + } + + if (!first) + ret = strbuf_add(buf, "]>", 2); +out: + return ret; +} + +/* Interval parameters for __die_find_var_reg_cb() */ +struct find_var_data { + /* Target instruction address */ + Dwarf_Addr pc; + /* Target memory address (for global data) */ + Dwarf_Addr addr; + /* Target register */ + unsigned reg; + /* Access data type */ + Dwarf_Die type; + /* Access offset, set for global data */ + int offset; + /* True if the current register is the frame base */ + bool is_fbreg; +}; + +/* Max number of registers DW_OP_regN supports */ +#define DWARF_OP_DIRECT_REGS 32 + +static bool match_var_offset(Dwarf_Die *die_mem, struct find_var_data *data, + s64 addr_offset, s64 addr_type, bool is_pointer) +{ + Dwarf_Word size; + Dwarf_Die ptr_die; + Dwarf_Die *ptr_type; + s64 offset = addr_offset - addr_type; + + if (offset < 0) + return false; + + if (__die_get_real_type(die_mem, &data->type) == NULL) + return false; + + ptr_type = die_get_pointer_type(&data->type, &ptr_die); + if (is_pointer && ptr_type) { + /* Get the target type of the pointer */ + if (__die_get_real_type(ptr_type, &data->type) == NULL) + return false; + } + + if (offset == 0) { + /* Update offset relative to the start of the variable */ + data->offset = 0; + return true; + } + + if (dwarf_aggregate_size(&data->type, &size) < 0) + return false; + + if ((u64)offset >= size) + return false; + + /* Update offset relative to the start of the variable */ + data->offset = offset; + return true; +} + +/** + * is_breg_access_indirect - Check if breg based access implies type + * dereference + * @ops: DWARF operations array + * @nops: Number of operations in @ops + * + * Returns true if the DWARF expression evaluates to the variable's + * value, so the memory access on that register needs type dereference. + * Returns false if the expression evaluates to the variable's address. + * This is called after check_allowed_ops. + */ +static bool is_breg_access_indirect(Dwarf_Op *ops, size_t nops) +{ + /* only the base register */ + if (nops == 1) + return false; + + if (nops == 2 && ops[1].atom == DW_OP_stack_value) + return true; + + if (nops == 3 && (ops[1].atom == DW_OP_deref || + ops[1].atom == DW_OP_deref_size) && + ops[2].atom == DW_OP_stack_value) + return false; + /* unreachable, OP not supported */ + return false; +} + +/* Only checks direct child DIEs in the given scope. */ +static int __die_find_var_reg_cb(Dwarf_Die *die_mem, void *arg) +{ + struct find_var_data *data = arg; + int tag = dwarf_tag(die_mem); + ptrdiff_t off = 0; + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + + if (tag != DW_TAG_variable && tag != DW_TAG_formal_parameter) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + /* Assuming the location list is sorted by address */ + if (end <= data->pc) + continue; + if (start > data->pc) + break; + + /* Local variables accessed using frame base register */ + if (data->is_fbreg && ops->atom == DW_OP_fbreg && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, ops->number, + is_breg_access_indirect(ops, nops))) + return DIE_FIND_CB_END; + + /* Only match with a simple case */ + if (data->reg < DWARF_OP_DIRECT_REGS) { + /* pointer variables saved in a register 0 to 31 */ + if (ops->atom == (DW_OP_reg0 + data->reg) && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, 0, + /*is_pointer=*/true)) + return DIE_FIND_CB_END; + + /* variables accessed by a register + offset */ + if (ops->atom == (DW_OP_breg0 + data->reg) && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, ops->number, + is_breg_access_indirect(ops, nops))) + return DIE_FIND_CB_END; + } else { + /* pointer variables saved in a register 32 or above */ + if (ops->atom == DW_OP_regx && ops->number == data->reg && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, 0, + /*is_pointer=*/true)) + return DIE_FIND_CB_END; + + /* variables accessed by a register + offset */ + if (ops->atom == DW_OP_bregx && data->reg == ops->number && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, ops->number2, + is_breg_access_indirect(ops, nops))) + return DIE_FIND_CB_END; + } + } + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_variable_by_reg - Find a variable saved in a register + * @sc_die: a scope DIE + * @pc: the program address to find + * @reg: the register number to find + * @poffset: pointer to offset, will be updated for fbreg case + * @is_fbreg: boolean value if the current register is the frame base + * @die_mem: a buffer to save the resulting DIE + * + * Find the variable DIE accessed by the given register. It'll update the @offset + * when the variable is in the stack. + */ +Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die, Dwarf_Addr pc, int reg, + Dwarf_Die *type_die, int *poffset, bool is_fbreg, + Dwarf_Die *die_mem) +{ + struct find_var_data data = { + .pc = pc, + .reg = reg, + .offset = *poffset, + .is_fbreg = is_fbreg, + }; + Dwarf_Die *result; + + result = die_find_child(sc_die, __die_find_var_reg_cb, &data, die_mem); + if (result) { + *poffset = data.offset; + *type_die = data.type; + } + return result; +} + +/* Only checks direct child DIEs in the given scope */ +static int __die_find_var_addr_cb(Dwarf_Die *die_mem, void *arg) +{ + struct find_var_data *data = arg; + int tag = dwarf_tag(die_mem); + ptrdiff_t off = 0; + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + + if (tag != DW_TAG_variable) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + if (ops->atom != DW_OP_addr) + continue; + + if (check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->addr, ops->number, + /*is_pointer=*/false)) + return DIE_FIND_CB_END; + } + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_variable_by_addr - Find variable located at given address + * @sc_die: a scope DIE + * @addr: the data address to find + * @die_mem: a buffer to save the resulting DIE + * @offset: the offset in the resulting type + * + * Find the variable DIE located at the given address (in PC-relative mode). + * This is usually for global variables. + */ +Dwarf_Die *die_find_variable_by_addr(Dwarf_Die *sc_die, Dwarf_Addr addr, + Dwarf_Die *die_mem, Dwarf_Die *type_die, + int *offset) +{ + struct find_var_data data = { + .addr = addr, + }; + Dwarf_Die *result; + + result = die_find_child(sc_die, __die_find_var_addr_cb, &data, die_mem); + if (result) { + *offset = data.offset; + *type_die = data.type; + } + return result; +} + +static int __die_collect_vars_cb(Dwarf_Die *die_mem, void *arg) +{ + struct die_var_type **var_types = arg; + Dwarf_Die type_die; + int tag = dwarf_tag(die_mem); + Dwarf_Attribute attr; + Dwarf_Addr base, start, end = 0; + Dwarf_Op *ops; + size_t nops; + struct die_var_type *vt; + ptrdiff_t off; + + if (tag != DW_TAG_variable && tag != DW_TAG_formal_parameter) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + if (__die_get_real_type(die_mem, &type_die) == NULL) + return DIE_FIND_CB_SIBLING; + + /* + * Collect all location entries as variables may have different + * locations across different address ranges. + */ + off = 0; + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + if (!check_allowed_ops(ops, nops)) + continue; + + vt = malloc(sizeof(*vt)); + if (vt == NULL) + return DIE_FIND_CB_END; + + /* Usually a register holds the value of a variable */ + vt->is_reg_var_addr = false; + + if (((ops->atom >= DW_OP_breg0 && ops->atom <= DW_OP_breg31) || + ops->atom == DW_OP_bregx || ops->atom == DW_OP_fbreg) && + !is_breg_access_indirect(ops, nops)) + /* The register contains an address of the variable. */ + vt->is_reg_var_addr = true; + + vt->die_off = dwarf_dieoffset(&type_die); + vt->addr = start; + vt->end = end; + vt->has_range = (end != 0 || start != 0); + vt->reg = reg_from_dwarf_op(ops); + vt->offset = offset_from_dwarf_op(ops); + vt->next = *var_types; + *var_types = vt; + } + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_collect_vars - Save all variables and parameters + * @sc_die: a scope DIE + * @var_types: a pointer to save the resulting list + * + * Save all variables and parameters in the @sc_die and save them to @var_types. + * The @var_types is a singly-linked list containing type and location info. + * Actual type can be retrieved using dwarf_offdie() with 'die_off' later. + * + * Callers should free @var_types. + */ +void die_collect_vars(Dwarf_Die *sc_die, struct die_var_type **var_types) +{ + Dwarf_Die die_mem; + + die_find_child(sc_die, __die_collect_vars_cb, (void *)var_types, &die_mem); +} + +static int __die_collect_global_vars_cb(Dwarf_Die *die_mem, void *arg) +{ + struct die_var_type **var_types = arg; + Dwarf_Die type_die; + int tag = dwarf_tag(die_mem); + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + struct die_var_type *vt; + + if (tag != DW_TAG_variable) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + /* Only collect the location with an absolute address. */ + if (dwarf_getlocations(&attr, 0, &base, &start, &end, &ops, &nops) <= 0) + return DIE_FIND_CB_SIBLING; + + if (ops->atom != DW_OP_addr) + return DIE_FIND_CB_SIBLING; + + if (!check_allowed_ops(ops, nops)) + return DIE_FIND_CB_SIBLING; + + if (die_get_real_type(die_mem, &type_die) == NULL) + return DIE_FIND_CB_SIBLING; + + vt = malloc(sizeof(*vt)); + if (vt == NULL) + return DIE_FIND_CB_END; + + vt->die_off = dwarf_dieoffset(&type_die); + vt->addr = ops->number; + vt->end = 0; + vt->has_range = false; + vt->reg = -1; + vt->offset = 0; + vt->next = *var_types; + *var_types = vt; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_collect_global_vars - Save all global variables + * @cu_die: a CU DIE + * @var_types: a pointer to save the resulting list + * + * Save all global variables in the @cu_die and save them to @var_types. + * The @var_types is a singly-linked list containing type and location info. + * Actual type can be retrieved using dwarf_offdie() with 'die_off' later. + * + * Callers should free @var_types. + */ +void die_collect_global_vars(Dwarf_Die *cu_die, struct die_var_type **var_types) +{ + Dwarf_Die die_mem; + + die_find_child(cu_die, __die_collect_global_vars_cb, (void *)var_types, &die_mem); +} + +/** + * die_get_cfa - Get frame base information + * @dwarf: a Dwarf info + * @pc: program address + * @preg: pointer for saved register + * @poffset: pointer for saved offset + * + * This function gets register and offset for CFA (Canonical Frame Address) + * by searching the CIE/FDE info. The CFA usually points to the start address + * of the current stack frame and local variables can be located using an offset + * from the CFA. The @preg and @poffset will be updated if it returns 0. + */ +int die_get_cfa(Dwarf *dwarf, u64 pc, int *preg, int *poffset) +{ + Dwarf_CFI *cfi; + Dwarf_Frame *frame = NULL; + Dwarf_Op *ops = NULL; + size_t nops; + + cfi = dwarf_getcfi(dwarf); + if (cfi == NULL) + return -1; + + if (!dwarf_cfi_addrframe(cfi, pc, &frame) && + !dwarf_frame_cfa(frame, &ops, &nops) && + check_allowed_ops(ops, nops)) { + *preg = reg_from_dwarf_op(ops); + *poffset = offset_from_dwarf_op(ops); + return 0; + } + return -1; +} + +/* + * die_has_loclist - Check if DW_AT_location of @vr_die is a location list + * @vr_die: a variable DIE + */ +static bool die_has_loclist(Dwarf_Die *vr_die) +{ + Dwarf_Attribute loc; + int tag = dwarf_tag(vr_die); + + if (tag != DW_TAG_formal_parameter && + tag != DW_TAG_variable) + return false; + + return (dwarf_attr_integrate(vr_die, DW_AT_location, &loc) && + dwarf_whatform(&loc) == DW_FORM_sec_offset); +} + +/* + * die_is_optimized_target - Check if target program is compiled with + * optimization + * @cu_die: a CU DIE + * + * For any object in given CU whose DW_AT_location is a location list, + * target program is compiled with optimization. This is applicable to + * clang as well. + */ +bool die_is_optimized_target(Dwarf_Die *cu_die) +{ + Dwarf_Die tmp_die; + + if (die_has_loclist(cu_die)) + return true; + + if (!dwarf_child(cu_die, &tmp_die) && + die_is_optimized_target(&tmp_die)) + return true; + + if (!dwarf_siblingof(cu_die, &tmp_die) && + die_is_optimized_target(&tmp_die)) + return true; + + return false; +} + +/* + * die_search_idx - Search index of given line address + * @lines: Line records of single CU + * @nr_lines: Number of @lines + * @addr: address we are looking for + * @idx: index to be set by this function (return value) + * + * Search for @addr by looping over every lines of CU. If address + * matches, set index of that line in @idx. Note that single source + * line can have multiple line records. i.e. single source line can + * have multiple index. + */ +static bool die_search_idx(Dwarf_Lines *lines, unsigned long nr_lines, + Dwarf_Addr addr, unsigned long *idx) +{ + unsigned long i; + Dwarf_Addr tmp; + + for (i = 0; i < nr_lines; i++) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i), &tmp)) + return false; + + if (tmp == addr) { + *idx = i; + return true; + } + } + return false; +} + +/* + * die_get_postprologue_addr - Search next address after function prologue + * @entrypc_idx: entrypc index + * @lines: Line records of single CU + * @nr_lines: Number of @lines + * @hignpc: high PC address of function + * @postprologue_addr: Next address after function prologue (return value) + * + * Look for prologue-end marker. If there is no explicit marker, return + * address of next line record or next source line. + */ +static bool die_get_postprologue_addr(unsigned long entrypc_idx, + Dwarf_Lines *lines, + unsigned long nr_lines, + Dwarf_Addr highpc, + Dwarf_Addr *postprologue_addr) +{ + unsigned long i; + int entrypc_lno, lno; + Dwarf_Line *line; + Dwarf_Addr addr; + bool p_end; + + /* entrypc_lno is actual source line number */ + line = dwarf_onesrcline(lines, entrypc_idx); + if (dwarf_lineno(line, &entrypc_lno)) + return false; + + for (i = entrypc_idx; i < nr_lines; i++) { + line = dwarf_onesrcline(lines, i); + + if (dwarf_lineaddr(line, &addr) || + dwarf_lineno(line, &lno) || + dwarf_lineprologueend(line, &p_end)) + return false; + + /* highpc is exclusive. [entrypc,highpc) */ + if (addr >= highpc) + break; + + /* clang supports prologue-end marker */ + if (p_end) + break; + + /* Actual next line in source */ + if (lno != entrypc_lno) + break; + + /* + * Single source line can have multiple line records. + * For Example, + * void foo() { printf("hello\n"); } + * contains two line records. One points to declaration and + * other points to printf() line. Variable 'lno' won't get + * incremented in this case but 'i' will. + */ + if (i != entrypc_idx) + break; + } + + if (dwarf_lineaddr(line, postprologue_addr) != 0) + return false; + if (*postprologue_addr >= highpc) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), postprologue_addr) != 0) + return false; + } + + return true; +} + +/* + * die_skip_prologue - Use next address after prologue as probe location + * @sp_die: a subprogram DIE + * @cu_die: a CU DIE + * @entrypc: entrypc of the function + * + * Function prologue prepares stack and registers before executing function + * logic. When target program is compiled without optimization, function + * parameter information is only valid after prologue. When we probe entrypc + * of the function, and try to record function parameter, it contains + * garbage value. + */ +void die_skip_prologue(Dwarf_Die *sp_die, Dwarf_Die *cu_die, + Dwarf_Addr *entrypc) +{ + size_t nr_lines = 0; + unsigned long entrypc_idx = 0; + Dwarf_Lines *lines = NULL; + Dwarf_Addr postprologue_addr; + Dwarf_Addr highpc; + + if (dwarf_highpc(sp_die, &highpc)) + return; + + if (dwarf_getsrclines(cu_die, &lines, &nr_lines)) + return; + + if (!die_search_idx(lines, nr_lines, *entrypc, &entrypc_idx)) + return; + + if (!die_get_postprologue_addr(entrypc_idx, lines, nr_lines, + highpc, &postprologue_addr)) + return; + + *entrypc = postprologue_addr; +} + +/* Internal parameters for __die_find_scope_cb() */ +struct find_scope_data { + /* Target instruction address */ + Dwarf_Addr pc; + /* Number of scopes found [output] */ + int nr; + /* Array of scopes found, 0 for the outermost one. [output] */ + Dwarf_Die *scopes; +}; + +static int __die_find_scope_cb(Dwarf_Die *die_mem, void *arg) +{ + struct find_scope_data *data = arg; + int tag = dwarf_tag(die_mem); + + if (dwarf_haspc(die_mem, data->pc)) { + Dwarf_Die *tmp; + + tmp = realloc(data->scopes, (data->nr + 1) * sizeof(*tmp)); + if (tmp == NULL) + return DIE_FIND_CB_END; + + memcpy(tmp + data->nr, die_mem, sizeof(*die_mem)); + data->scopes = tmp; + data->nr++; + return DIE_FIND_CB_CHILD; + } + + /* + * If the DIE doesn't have the PC, we still need to check its children + * and siblings if it's a container like a namespace. + */ + if (tag == DW_TAG_namespace) + return DIE_FIND_CB_CONTINUE; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_get_scopes - Return a list of scopes including the address + * @cu_die: a compile unit DIE + * @pc: the address to find + * @scopes: the array of DIEs for scopes (result) + * + * This function does the same as the dwarf_getscopes() but doesn't follow + * the origins of inlined functions. It returns the number of scopes saved + * in the @scopes argument. The outer scope will be saved first (index 0) and + * the last one is the innermost scope at the @pc. + */ +int die_get_scopes(Dwarf_Die *cu_die, Dwarf_Addr pc, Dwarf_Die **scopes) +{ + struct find_scope_data data = { + .pc = pc, + }; + Dwarf_Die die_mem; + + die_find_child(cu_die, __die_find_scope_cb, &data, &die_mem); + + *scopes = data.scopes; + return data.nr; +} + +static int __die_find_member_offset_cb(Dwarf_Die *die_mem, void *arg) +{ + Dwarf_Die type_die; + Dwarf_Word size, loc; + Dwarf_Word offset = (long)arg; + int tag = dwarf_tag(die_mem); + + if (tag != DW_TAG_member) + return DIE_FIND_CB_SIBLING; + + /* Unions might not have location */ + if (die_get_data_member_location(die_mem, &loc) < 0) { + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(die_mem, DW_AT_data_bit_offset, &attr) && + dwarf_formudata(&attr, &loc) == 0) + loc /= 8; + else + loc = 0; + } + + if (offset == loc) + return DIE_FIND_CB_END; + + if (die_get_real_type(die_mem, &type_die) == NULL) { + // TODO: add a pr_debug_dtp() later for this unlikely failure + return DIE_FIND_CB_SIBLING; + } + + if (dwarf_aggregate_size(&type_die, &size) < 0) + size = 0; + + if (loc < offset && offset < (loc + size)) + return DIE_FIND_CB_END; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_get_member_type - Return type info of struct member + * @type_die: a type DIE + * @offset: offset in the type + * @die_mem: a buffer to save the resulting DIE + * + * This function returns a type of a member in @type_die where it's located at + * @offset if it's a struct. For now, it just returns the first matching + * member in a union. For other types, it'd return the given type directly + * if it's within the size of the type or NULL otherwise. + */ +Dwarf_Die *die_get_member_type(Dwarf_Die *type_die, int offset, + Dwarf_Die *die_mem) +{ + Dwarf_Die *member; + Dwarf_Die mb_type; + int tag; + + tag = dwarf_tag(type_die); + /* If it's not a compound type, return the type directly */ + if (tag != DW_TAG_structure_type && tag != DW_TAG_union_type) { + Dwarf_Word size; + + if (dwarf_aggregate_size(type_die, &size) < 0) + size = 0; + + if ((unsigned)offset >= size) + return NULL; + + *die_mem = *type_die; + return die_mem; + } + + mb_type = *type_die; + /* TODO: Handle union types better? */ + while (tag == DW_TAG_structure_type || tag == DW_TAG_union_type) { + member = die_find_child(&mb_type, __die_find_member_offset_cb, + (void *)(long)offset, die_mem); + if (member == NULL) + return NULL; + + if (die_get_real_type(member, &mb_type) == NULL) + return NULL; + + tag = dwarf_tag(&mb_type); + + if (tag == DW_TAG_structure_type || tag == DW_TAG_union_type || + tag == DW_TAG_array_type) { + Dwarf_Word loc; + + /* Update offset for the start of the member struct */ + if (die_get_data_member_location(member, &loc) == 0) + offset -= loc; + } + + /* Handle array types: resolve to the element type by one level */ + if (tag == DW_TAG_array_type) { + Dwarf_Word size; + + if (die_get_real_type(&mb_type, &mb_type) == NULL) + return NULL; + + if (dwarf_aggregate_size(&mb_type, &size) < 0) + return NULL; + + offset = offset % size; + tag = dwarf_tag(&mb_type); + } + } + *die_mem = mb_type; + return die_mem; +} + +/** + * die_deref_ptr_type - Return type info for pointer access + * @ptr_die: a pointer type DIE + * @offset: access offset for the pointer + * @die_mem: a buffer to save the resulting DIE + * + * This function follows the pointer in @ptr_die with given @offset + * and saves the resulting type in @die_mem. If the pointer points + * a struct type, actual member at the offset would be returned. + */ +Dwarf_Die *die_deref_ptr_type(Dwarf_Die *ptr_die, int offset, + Dwarf_Die *die_mem) +{ + Dwarf_Die type_die; + + if (dwarf_tag(ptr_die) != DW_TAG_pointer_type) + return NULL; + + if (die_get_real_type(ptr_die, &type_die) == NULL) + return NULL; + + return die_get_member_type(&type_die, offset, die_mem); +} diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h new file mode 100644 index 000000000..161f0bf98 --- /dev/null +++ b/tools/perf/util/dwarf-aux.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _DWARF_AUX_H +#define _DWARF_AUX_H +/* + * dwarf-aux.h : libdw auxiliary interfaces + */ + +#include +#include +#include +#include + +static inline const char *die_name(Dwarf_Die *die) +{ + return dwarf_diename(die) ?: ""; +} + +struct strbuf; + +/* Find the realpath of the target file */ +const char *cu_find_realpath(Dwarf_Die *cu_die, const char *fname); + +/* Get DW_AT_comp_dir (should be NULL with older gcc) */ +const char *cu_get_comp_dir(Dwarf_Die *cu_die); + +/* Get a line number and file name for given address */ +int cu_find_lineinfo(Dwarf_Die *cudie, Dwarf_Addr addr, + const char **fname, int *lineno); + +/* Walk on functions at given address */ +int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, + int (*callback)(Dwarf_Die *, void *), void *data); + +/* Get DW_AT_linkage_name (should be NULL for C binary) */ +const char *die_get_linkage_name(Dwarf_Die *dw_die); + +/* Get the lowest PC in DIE (including range list) */ +int die_entrypc(Dwarf_Die *dw_die, Dwarf_Addr *addr); + +/* Ensure that this DIE is a subprogram and definition (not declaration) */ +bool die_is_func_def(Dwarf_Die *dw_die); + +/* Ensure that this DIE is an instance of a subprogram */ +bool die_is_func_instance(Dwarf_Die *dw_die); + +/* Compare diename and tname */ +bool die_compare_name(Dwarf_Die *dw_die, const char *tname); + +/* Matching diename with glob pattern */ +bool die_match_name(Dwarf_Die *dw_die, const char *glob); + +/* Get callsite line number of inline-function instance */ +int die_get_call_lineno(Dwarf_Die *in_die); + +/* Get callsite file name of inlined function instance */ +const char *die_get_call_file(Dwarf_Die *in_die); + +/* Get declared file name of a DIE */ +const char *die_get_decl_file(Dwarf_Die *dw_die); + +/* Get type die */ +Dwarf_Die *die_get_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem); + +/* Get a type die, but skip qualifiers */ +Dwarf_Die *__die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem); +/* Get a type die, but skip qualifiers and typedef */ +Dwarf_Die *die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem); +/* Get a pointer/array type, following typedefs/qualifiers */ +Dwarf_Die *die_get_pointer_type(Dwarf_Die *type_die, Dwarf_Die *die_mem); + +/* Check whether the DIE is signed or not */ +bool die_is_signed_type(Dwarf_Die *tp_die); + +/* Get data_member_location offset */ +int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs); + +/* Return values for die_find_child() callbacks */ +enum { + DIE_FIND_CB_END = 0, /* End of Search */ + DIE_FIND_CB_CHILD = 1, /* Search only children */ + DIE_FIND_CB_SIBLING = 2, /* Search only siblings */ + DIE_FIND_CB_CONTINUE = 3, /* Search children and siblings */ +}; + +/* Search child DIEs */ +Dwarf_Die *die_find_child(Dwarf_Die *rt_die, + int (*callback)(Dwarf_Die *, void *), + void *data, Dwarf_Die *die_mem); + +/* Search a non-inlined function including given address */ +Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search a non-inlined function with tail call at given address */ +Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search the top inlined function including given address */ +Dwarf_Die *die_find_top_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search the deepest inlined function including given address */ +Dwarf_Die *die_find_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search a non-inlined function by name and returns its return type */ +Dwarf_Die *die_find_func_rettype(Dwarf_Die *sp_die, const char *name, + Dwarf_Die *die_mem); + +/* Walk on the instances of given DIE */ +int die_walk_instances(Dwarf_Die *in_die, + int (*callback)(Dwarf_Die *, void *), void *data); + +/* Walker on lines (Note: line number will not be sorted) */ +typedef int (* line_walk_callback_t) (const char *fname, int lineno, + Dwarf_Addr addr, void *data); + +/* + * Walk on lines inside given DIE. If the DIE is a subprogram, walk only on + * the lines inside the subprogram, otherwise the DIE must be a CU DIE. + */ +int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data); + +/* Find a variable called 'name' at given address */ +Dwarf_Die *die_find_variable_at(Dwarf_Die *sp_die, const char *name, + Dwarf_Addr addr, Dwarf_Die *die_mem); + +/* Find a member called 'name' */ +Dwarf_Die *die_find_member(Dwarf_Die *st_die, const char *name, + Dwarf_Die *die_mem); + +/* Get the name of given type DIE */ +int die_get_typename_from_type(Dwarf_Die *type_die, struct strbuf *buf); + +/* Get the name of given variable DIE */ +int die_get_typename(Dwarf_Die *vr_die, struct strbuf *buf); + +/* Get the name and type of given variable DIE, stored as "type\tname" */ +int die_get_varname(Dwarf_Die *vr_die, struct strbuf *buf); + +/* Check if target program is compiled with optimization */ +bool die_is_optimized_target(Dwarf_Die *cu_die); + +/* Use next address after prologue as probe location */ +void die_skip_prologue(Dwarf_Die *sp_die, Dwarf_Die *cu_die, + Dwarf_Addr *entrypc); + +/* Get the list of including scopes */ +int die_get_scopes(Dwarf_Die *cu_die, Dwarf_Addr pc, Dwarf_Die **scopes); + +/* Variable type information */ +struct die_var_type { + struct die_var_type *next; + u64 die_off; + u64 addr; + u64 end; /* end address of location range */ + int reg; + int offset; + /* Whether the register holds a address to the type */ + bool is_reg_var_addr; + bool has_range; /* whether end is valid */ +}; + +/* Return type info of a member at offset */ +Dwarf_Die *die_get_member_type(Dwarf_Die *type_die, int offset, Dwarf_Die *die_mem); + +/* Return type info where the pointer and offset point to */ +Dwarf_Die *die_deref_ptr_type(Dwarf_Die *ptr_die, int offset, Dwarf_Die *die_mem); + +/* Get byte offset range of given variable DIE */ +int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf); + +/* Find a variable saved in the 'reg' at given address */ +Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die, Dwarf_Addr pc, int reg, + Dwarf_Die *type_die, int *poffset, bool is_fbreg, + Dwarf_Die *die_mem); + +/* Find a (global) variable located in the 'addr' */ +Dwarf_Die *die_find_variable_by_addr(Dwarf_Die *sc_die, Dwarf_Addr addr, + Dwarf_Die *die_mem, Dwarf_Die *type_die, + int *offset); + +/* Save all variables and parameters in this scope */ +void die_collect_vars(Dwarf_Die *sc_die, struct die_var_type **var_types); + +/* Save all global variables in this CU */ +void die_collect_global_vars(Dwarf_Die *cu_die, struct die_var_type **var_types); + +/* Get the frame base information from CFA */ +int die_get_cfa(Dwarf *dwarf, u64 pc, int *preg, int *poffset); + +#endif /* _DWARF_AUX_H */ diff --git a/tools/perf/util/dwarf-regs-arch/Build b/tools/perf/util/dwarf-regs-arch/Build new file mode 100644 index 000000000..ceb68ae86 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/Build @@ -0,0 +1,9 @@ +perf-util-$(CONFIG_LIBDW) += dwarf-regs-arm64.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-arm.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-csky.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-loongarch.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-mips.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-powerpc.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-riscv.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-s390.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-x86.o diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c new file mode 100644 index 000000000..42c6c0635 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/arm/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_arm(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_ARM_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c new file mode 100644 index 000000000..593ca7d4f --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/arm64/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_arm64(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_ARM64_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c new file mode 100644 index 000000000..cb44b774f --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. +// Mapping of DWARF debug register numbers into register names. + +#include +#include +#include +// Ensure the V2 perf reg definitions are included. +#undef __CSKYABIV2__ +#define __CSKYABIV2__ 1 +#include "../../../arch/csky/include/uapi/asm/perf_regs.h" + +#define CSKY_ABIV2_MAX_REGS 73 +static const char * const csky_dwarf_regs_table_abiv2[CSKY_ABIV2_MAX_REGS] = { + /* r0 ~ r8 */ + "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", "%regs2", "%regs3", + /* r9 ~ r15 */ + "%regs4", "%regs5", "%regs6", "%regs7", "%regs8", "%regs9", "%sp", + "%lr", + /* r16 ~ r23 */ + "%exregs0", "%exregs1", "%exregs2", "%exregs3", "%exregs4", + "%exregs5", "%exregs6", "%exregs7", + /* r24 ~ r31 */ + "%exregs8", "%exregs9", "%exregs10", "%exregs11", "%exregs12", + "%exregs13", "%exregs14", "%tls", + "%pc", NULL, NULL, NULL, "%hi", "%lo", NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + "%epc", +}; + +#define CSKY_ABIV1_MAX_REGS 57 +static const char * const csky_dwarf_regs_table_abiv1[CSKY_ABIV1_MAX_REGS] = { + /* r0 ~ r8 */ + "%sp", "%regs9", "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", + /* r9 ~ r15 */ + "%regs2", "%regs3", "%regs4", "%regs5", "%regs6", "%regs7", "%regs8", + "%lr", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + "%epc", +}; + +const char *__get_csky_regstr(unsigned int n, unsigned int flags) +{ + if (flags & EF_CSKY_ABIV2) + return (n < CSKY_ABIV2_MAX_REGS) ? csky_dwarf_regs_table_abiv2[n] : NULL; + + return (n < CSKY_ABIV1_MAX_REGS) ? csky_dwarf_regs_table_abiv1[n] : NULL; +} + +static int __get_dwarf_regnum(const char *const *regstr, size_t num_regstr, const char *name) +{ + for (size_t i = 0; i < num_regstr; i++) { + if (regstr[i] && !strcmp(regstr[i], name)) + return i; + } + return -ENOENT; +} + +int __get_csky_regnum(const char *name, unsigned int flags) +{ + if (flags & EF_CSKY_ABIV2) + return __get_dwarf_regnum(csky_dwarf_regs_table_abiv2, CSKY_ABIV2_MAX_REGS, name); + + return __get_dwarf_regnum(csky_dwarf_regs_table_abiv1, CSKY_ABIV1_MAX_REGS, name); +} + +int __get_dwarf_regnum_for_perf_regnum_csky(int perf_regnum, unsigned int flags) +{ + static const int dwarf_csky_regnums[][2] = { + [PERF_REG_CSKY_TLS] = {-ENOENT, 31}, + [PERF_REG_CSKY_LR] = {15, 15}, + [PERF_REG_CSKY_PC] = {-ENOENT, 32}, + /* TODO: PERF_REG_CSKY_SR */ + [PERF_REG_CSKY_SP] = {0, 14}, + /* TODO: PERF_REG_CSKY_ORIG_A0 */ + [PERF_REG_CSKY_A0] = {2, 0}, + [PERF_REG_CSKY_A1] = {3, 1}, + [PERF_REG_CSKY_A2] = {4, 2}, + [PERF_REG_CSKY_A3] = {5, 3}, + [PERF_REG_CSKY_REGS0] = {6, 4}, + [PERF_REG_CSKY_REGS1] = {7, 5}, + [PERF_REG_CSKY_REGS2] = {8, 6}, + [PERF_REG_CSKY_REGS3] = {9, 7}, + [PERF_REG_CSKY_REGS4] = {10, 8}, + [PERF_REG_CSKY_REGS5] = {11, 9}, + [PERF_REG_CSKY_REGS6] = {12, 10}, + [PERF_REG_CSKY_REGS7] = {13, 11}, + [PERF_REG_CSKY_REGS8] = {14, 12}, + [PERF_REG_CSKY_REGS9] = {1, 13}, + [PERF_REG_CSKY_EXREGS0] = {-ENOENT, 16}, + [PERF_REG_CSKY_EXREGS1] = {-ENOENT, 17}, + [PERF_REG_CSKY_EXREGS2] = {-ENOENT, 18}, + [PERF_REG_CSKY_EXREGS3] = {-ENOENT, 19}, + [PERF_REG_CSKY_EXREGS4] = {-ENOENT, 20}, + [PERF_REG_CSKY_EXREGS5] = {-ENOENT, 21}, + [PERF_REG_CSKY_EXREGS6] = {-ENOENT, 22}, + [PERF_REG_CSKY_EXREGS7] = {-ENOENT, 23}, + [PERF_REG_CSKY_EXREGS8] = {-ENOENT, 24}, + [PERF_REG_CSKY_EXREGS9] = {-ENOENT, 25}, + [PERF_REG_CSKY_EXREGS10] = {-ENOENT, 26}, + [PERF_REG_CSKY_EXREGS11] = {-ENOENT, 27}, + [PERF_REG_CSKY_EXREGS12] = {-ENOENT, 28}, + [PERF_REG_CSKY_EXREGS13] = {-ENOENT, 29}, + [PERF_REG_CSKY_EXREGS14] = {-ENOENT, 30}, + /* TODO: PERF_REG_CSKY_HI */ + /* TODO: PERF_REG_CSKY_LO */ + /* TODO: PERF_REG_CSKY_DCSR */ + }; + int idx = 0; + + if (flags & EF_CSKY_ABIV2) + idx++; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_csky_regnums) || + dwarf_csky_regnums[perf_regnum][idx] == 0) + return -ENOENT; + + return dwarf_csky_regnums[perf_regnum][idx]; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c new file mode 100644 index 000000000..203077b74 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/loongarch/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_loongarch(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_LOONGARCH_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c new file mode 100644 index 000000000..3bb916b45 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/mips/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_mips(int perf_regnum) +{ + if (perf_regnum == PERF_REG_MIPS_PC) + return 37; + if (perf_regnum < 0 || perf_regnum >= PERF_REG_MIPS_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c new file mode 100644 index 000000000..51892a097 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Mapping of DWARF debug register numbers into register names. + * + * Copyright (C) 2010 Ian Munsie, IBM Corporation. + */ +#include +#include +#include "../../../arch/powerpc/include/uapi/asm/perf_regs.h" + +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define PPC_RA(a) (((a) >> 16) & 0x1f) +#define PPC_RT(t) (((t) >> 21) & 0x1f) +#define PPC_RB(b) (((b) >> 11) & 0x1f) +#define PPC_D(D) ((D) & 0xfffe) +#define PPC_DS(DS) ((DS) & 0xfffc) +#define OP_LD 58 +#define OP_STD 62 + +static int get_source_reg(u32 raw_insn) +{ + return PPC_RA(raw_insn); +} + +static int get_target_reg(u32 raw_insn) +{ + return PPC_RT(raw_insn); +} + +static int get_offset_opcode(u32 raw_insn) +{ + int opcode = PPC_OP(raw_insn); + + /* DS- form */ + if ((opcode == OP_LD) || (opcode == OP_STD)) + return PPC_DS(raw_insn); + else + return PPC_D(raw_insn); +} + +/* + * Fills the required fields for op_loc depending on if it + * is a source or target. + * D form: ins RT,D(RA) -> src_reg1 = RA, offset = D, dst_reg1 = RT + * DS form: ins RT,DS(RA) -> src_reg1 = RA, offset = DS, dst_reg1 = RT + * X form: ins RT,RA,RB -> src_reg1 = RA, src_reg2 = RB, dst_reg1 = RT + */ +void get_powerpc_regs(u32 raw_insn, int is_source, + struct annotated_op_loc *op_loc) +{ + if (is_source) + op_loc->reg1 = get_source_reg(raw_insn); + else + op_loc->reg1 = get_target_reg(raw_insn); + + if (op_loc->multi_regs) + op_loc->reg2 = PPC_RB(raw_insn); + + /* TODO: Implement offset handling for X Form */ + if ((op_loc->mem_ref) && (PPC_OP(raw_insn) != 31)) + op_loc->offset = get_offset_opcode(raw_insn); +} + +int __get_dwarf_regnum_for_perf_regnum_powerpc(int perf_regnum) +{ + static const int dwarf_powerpc_regnums[] = { + [PERF_REG_POWERPC_R0] = 0, + [PERF_REG_POWERPC_R1] = 1, + [PERF_REG_POWERPC_R2] = 2, + [PERF_REG_POWERPC_R3] = 3, + [PERF_REG_POWERPC_R4] = 4, + [PERF_REG_POWERPC_R5] = 5, + [PERF_REG_POWERPC_R6] = 6, + [PERF_REG_POWERPC_R7] = 7, + [PERF_REG_POWERPC_R8] = 8, + [PERF_REG_POWERPC_R9] = 9, + [PERF_REG_POWERPC_R10] = 10, + [PERF_REG_POWERPC_R11] = 11, + [PERF_REG_POWERPC_R12] = 12, + [PERF_REG_POWERPC_R13] = 13, + [PERF_REG_POWERPC_R14] = 14, + [PERF_REG_POWERPC_R15] = 15, + [PERF_REG_POWERPC_R16] = 16, + [PERF_REG_POWERPC_R17] = 17, + [PERF_REG_POWERPC_R18] = 18, + [PERF_REG_POWERPC_R19] = 19, + [PERF_REG_POWERPC_R20] = 20, + [PERF_REG_POWERPC_R21] = 21, + [PERF_REG_POWERPC_R22] = 22, + [PERF_REG_POWERPC_R23] = 23, + [PERF_REG_POWERPC_R24] = 24, + [PERF_REG_POWERPC_R25] = 25, + [PERF_REG_POWERPC_R26] = 26, + [PERF_REG_POWERPC_R27] = 27, + [PERF_REG_POWERPC_R28] = 28, + [PERF_REG_POWERPC_R29] = 29, + [PERF_REG_POWERPC_R30] = 30, + [PERF_REG_POWERPC_R31] = 31, + /* TODO: PERF_REG_POWERPC_NIP */ + [PERF_REG_POWERPC_MSR] = 66, + /* TODO: PERF_REG_POWERPC_ORIG_R3 */ + [PERF_REG_POWERPC_CTR] = 109, + [PERF_REG_POWERPC_LINK] = 108, /* Note, previously in perf encoded as 65? */ + [PERF_REG_POWERPC_XER] = 101, + /* TODO: PERF_REG_POWERPC_CCR */ + /* TODO: PERF_REG_POWERPC_SOFTE */ + /* TODO: PERF_REG_POWERPC_TRAP */ + /* TODO: PERF_REG_POWERPC_DAR */ + /* TODO: PERF_REG_POWERPC_DSISR */ + /* TODO: PERF_REG_POWERPC_SIER */ + /* TODO: PERF_REG_POWERPC_MMCRA */ + /* TODO: PERF_REG_POWERPC_MMCR0 */ + /* TODO: PERF_REG_POWERPC_MMCR1 */ + /* TODO: PERF_REG_POWERPC_MMCR2 */ + /* TODO: PERF_REG_POWERPC_MMCR3 */ + /* TODO: PERF_REG_POWERPC_SIER2 */ + /* TODO: PERF_REG_POWERPC_SIER3 */ + /* TODO: PERF_REG_POWERPC_PMC1 */ + /* TODO: PERF_REG_POWERPC_PMC2 */ + /* TODO: PERF_REG_POWERPC_PMC3 */ + /* TODO: PERF_REG_POWERPC_PMC4 */ + /* TODO: PERF_REG_POWERPC_PMC5 */ + /* TODO: PERF_REG_POWERPC_PMC6 */ + /* TODO: PERF_REG_POWERPC_SDAR */ + /* TODO: PERF_REG_POWERPC_SIAR */ + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_powerpc_regnums) || + dwarf_powerpc_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_powerpc_regnums[perf_regnum]; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c new file mode 100644 index 000000000..090db51ab --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/riscv/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_riscv(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_RISCV_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c new file mode 100644 index 000000000..310a37451 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/s390/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_s390(int perf_regnum) +{ + static const int dwarf_s390_regnums[] = { + [PERF_REG_S390_R0] = 0, + [PERF_REG_S390_R1] = 1, + [PERF_REG_S390_R2] = 2, + [PERF_REG_S390_R3] = 3, + [PERF_REG_S390_R4] = 4, + [PERF_REG_S390_R5] = 5, + [PERF_REG_S390_R6] = 6, + [PERF_REG_S390_R7] = 7, + [PERF_REG_S390_R8] = 8, + [PERF_REG_S390_R9] = 9, + [PERF_REG_S390_R10] = 10, + [PERF_REG_S390_R11] = 11, + [PERF_REG_S390_R12] = 12, + [PERF_REG_S390_R13] = 13, + [PERF_REG_S390_R14] = 14, + [PERF_REG_S390_R15] = 15, + [PERF_REG_S390_FP0] = 16, + [PERF_REG_S390_FP1] = 20, + [PERF_REG_S390_FP2] = 17, + [PERF_REG_S390_FP3] = 21, + [PERF_REG_S390_FP4] = 18, + [PERF_REG_S390_FP5] = 22, + [PERF_REG_S390_FP6] = 19, + [PERF_REG_S390_FP7] = 23, + [PERF_REG_S390_FP8] = 24, + [PERF_REG_S390_FP9] = 28, + [PERF_REG_S390_FP10] = 25, + [PERF_REG_S390_FP11] = 29, + [PERF_REG_S390_FP12] = 26, + [PERF_REG_S390_FP13] = 30, + [PERF_REG_S390_FP14] = 27, + [PERF_REG_S390_FP15] = 31, + [PERF_REG_S390_MASK] = 64, + [PERF_REG_S390_PC] = 65, + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_s390_regnums) || + dwarf_s390_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_s390_regnums[perf_regnum]; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c new file mode 100644 index 000000000..cadef120a --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. + * Extracted from probe-finder.c + * + * Written by Masami Hiramatsu + */ + +#include /* for EINVAL */ +#include /* for strcmp */ +#include /* for ARRAY_SIZE */ +#include +#include "../../../arch/x86/include/uapi/asm/perf_regs.h" + +struct dwarf_regs_idx { + const char *name; + int dwarf_regnum; +}; + +static const struct dwarf_regs_idx i386_regidx_table[] = { + { "eax", 0 }, { "ax", 0 }, { "al", 0 }, + { "ecx", 1 }, { "cx", 1 }, { "cl", 1 }, + { "edx", 2 }, { "dx", 2 }, { "dl", 2 }, + { "ebx", 3 }, { "bx", 3 }, { "bl", 3 }, + { "esp", 4 }, { "sp", 4 }, { "$stack", 4}, + { "ebp", 5 }, { "bp", 5 }, + { "esi", 6 }, { "si", 6 }, + { "edi", 7 }, { "di", 7 }, + // 8 - Return Address RA + { "eflags", 9}, { "flags", 9}, + // 10 - reserved + { "st0", 11}, + { "st1", 12}, + { "st2", 13}, + { "st3", 14}, + { "st4", 15}, + { "st5", 16}, + { "st6", 17}, + { "st7", 18}, + // 19-20 - reserved + { "xmm0", 21}, + { "xmm1", 22}, + { "xmm2", 23}, + { "xmm3", 24}, + { "xmm4", 25}, + { "xmm5", 26}, + { "xmm6", 27}, + { "xmm7", 28}, + { "mm0", 29}, + { "mm1", 30}, + { "mm2", 31}, + { "mm3", 32}, + { "mm4", 33}, + { "mm5", 34}, + { "mm6", 35}, + { "mm7", 36}, + // 37-38 - unknown + { "mxcsr", 39}, // 128-bit Media Control and Status + { "es", 40}, + { "cs", 41}, + { "ss", 42}, + { "ds", 43}, + { "fs", 44}, + { "gs", 45}, + // 46-47 - reserved + { "tr", 48}, // Task Register + { "ldtr", 49}, // LDT Register + // 50-92 - reserved + { "fs.base", 92}, + { "gs.base", 93}, + // End of regular dwarf registers. + { "eip", DWARF_REG_PC }, { "ip", DWARF_REG_PC }, +}; + +static const struct dwarf_regs_idx x86_64_regidx_table[] = { + { "rax", 0 }, { "eax", 0 }, { "ax", 0 }, { "al", 0 }, + { "rdx", 1 }, { "edx", 1 }, { "dx", 1 }, { "dl", 1 }, + { "rcx", 2 }, { "ecx", 2 }, { "cx", 2 }, { "cl", 2 }, + { "rbx", 3 }, { "edx", 3 }, { "bx", 3 }, { "bl", 3 }, + { "rsi", 4 }, { "esi", 4 }, { "si", 4 }, { "sil", 4 }, + { "rdi", 5 }, { "edi", 5 }, { "di", 5 }, { "dil", 5 }, + { "rbp", 6 }, { "ebp", 6 }, { "bp", 6 }, { "bpl", 6 }, + { "rsp", 7 }, { "esp", 7 }, { "sp", 7 }, { "spl", 7 }, + { "r8", 8 }, { "r8d", 8 }, { "r8w", 8 }, { "r8b", 8 }, + { "r9", 9 }, { "r9d", 9 }, { "r9w", 9 }, { "r9b", 9 }, + { "r10", 10 }, { "r10d", 10 }, { "r10w", 10 }, { "r10b", 10 }, + { "r11", 11 }, { "r11d", 11 }, { "r11w", 11 }, { "r11b", 11 }, + { "r12", 12 }, { "r12d", 12 }, { "r12w", 12 }, { "r12b", 12 }, + { "r13", 13 }, { "r13d", 13 }, { "r13w", 13 }, { "r13b", 13 }, + { "r14", 14 }, { "r14d", 14 }, { "r14w", 14 }, { "r14b", 14 }, + { "r15", 15 }, { "r15d", 15 }, { "r15w", 15 }, { "r15b", 15 }, + // 16 - Return Address RA + { "xmm0", 17}, + { "xmm1", 18}, + { "xmm2", 19}, + { "xmm3", 20}, + { "xmm4", 21}, + { "xmm5", 22}, + { "xmm6", 23}, + { "xmm7", 24}, + { "xmm8", 25}, + { "xmm9", 26}, + { "xmm10", 27}, + { "xmm11", 28}, + { "xmm12", 29}, + { "xmm13", 30}, + { "xmm14", 31}, + { "xmm15", 32}, + { "st0", 33}, + { "st1", 34}, + { "st2", 35}, + { "st3", 36}, + { "st4", 37}, + { "st5", 38}, + { "st6", 39}, + { "st7", 40}, + { "mm0", 41}, + { "mm1", 42}, + { "mm2", 43}, + { "mm3", 44}, + { "mm4", 45}, + { "mm5", 46}, + { "mm6", 47}, + { "mm7", 48}, + { "rflags", 49}, { "eflags", 49}, { "flags", 49}, + { "es", 50}, + { "cs", 51}, + { "ss", 52}, + { "ds", 53}, + { "fs", 54}, + { "gs", 55}, + // 56-47 - reserved + { "fs.base", 58}, + { "gs.base", 59}, + // 60-61 - reserved + { "tr", 62}, // Task Register + { "ldtr", 63}, // LDT Register + { "mxcsr", 64}, // 128-bit Media Control and Status + { "fcw", 65}, // x87 Control Word + { "fsw", 66}, // x87 Status Word + // End of regular dwarf registers. + { "rip", DWARF_REG_PC }, { "eip", DWARF_REG_PC }, { "ip", DWARF_REG_PC }, +}; + +static int get_regnum(const struct dwarf_regs_idx *entries, size_t num_entries, const char *name) +{ + if (*name != '%') + return -EINVAL; + + name++; + for (size_t i = 0; i < num_entries; i++) { + if (!strcmp(entries[i].name, name)) + return entries[i].dwarf_regnum; + } + return -ENOENT; +} + +int __get_dwarf_regnum_i386(const char *name) +{ + return get_regnum(i386_regidx_table, ARRAY_SIZE(i386_regidx_table), name); +} + +int __get_dwarf_regnum_x86_64(const char *name) +{ + return get_regnum(x86_64_regidx_table, ARRAY_SIZE(x86_64_regidx_table), name); +} + +int __get_dwarf_regnum_for_perf_regnum_i386(int perf_regnum) +{ + static const int dwarf_i386_regnums[] = { + [PERF_REG_X86_AX] = 0, + [PERF_REG_X86_BX] = 3, + [PERF_REG_X86_CX] = 1, + [PERF_REG_X86_DX] = 2, + [PERF_REG_X86_SI] = 6, + [PERF_REG_X86_DI] = 7, + [PERF_REG_X86_BP] = 5, + [PERF_REG_X86_SP] = 4, + [PERF_REG_X86_IP] = 8, + [PERF_REG_X86_FLAGS] = 9, + [PERF_REG_X86_CS] = 41, + [PERF_REG_X86_SS] = 42, + [PERF_REG_X86_DS] = 43, + [PERF_REG_X86_ES] = 40, + [PERF_REG_X86_FS] = 44, + [PERF_REG_X86_GS] = 45, + [PERF_REG_X86_XMM0] = 21, + [PERF_REG_X86_XMM1] = 22, + [PERF_REG_X86_XMM2] = 23, + [PERF_REG_X86_XMM3] = 24, + [PERF_REG_X86_XMM4] = 25, + [PERF_REG_X86_XMM5] = 26, + [PERF_REG_X86_XMM6] = 27, + [PERF_REG_X86_XMM7] = 28, + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_i386_regnums) || + dwarf_i386_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_i386_regnums[perf_regnum]; +} + +int __get_dwarf_regnum_for_perf_regnum_x86_64(int perf_regnum) +{ + static const int dwarf_x86_64_regnums[] = { + [PERF_REG_X86_AX] = 0, + [PERF_REG_X86_BX] = 3, + [PERF_REG_X86_CX] = 2, + [PERF_REG_X86_DX] = 1, + [PERF_REG_X86_SI] = 4, + [PERF_REG_X86_DI] = 5, + [PERF_REG_X86_BP] = 6, + [PERF_REG_X86_SP] = 7, + [PERF_REG_X86_IP] = 16, + [PERF_REG_X86_FLAGS] = 49, + [PERF_REG_X86_CS] = 51, + [PERF_REG_X86_SS] = 52, + [PERF_REG_X86_DS] = 53, + [PERF_REG_X86_ES] = 50, + [PERF_REG_X86_FS] = 54, + [PERF_REG_X86_GS] = 55, + [PERF_REG_X86_R8] = 8, + [PERF_REG_X86_R9] = 9, + [PERF_REG_X86_R10] = 10, + [PERF_REG_X86_R11] = 11, + [PERF_REG_X86_R12] = 12, + [PERF_REG_X86_R13] = 13, + [PERF_REG_X86_R14] = 14, + [PERF_REG_X86_R15] = 15, + [PERF_REG_X86_XMM0] = 17, + [PERF_REG_X86_XMM1] = 18, + [PERF_REG_X86_XMM2] = 19, + [PERF_REG_X86_XMM3] = 20, + [PERF_REG_X86_XMM4] = 21, + [PERF_REG_X86_XMM5] = 22, + [PERF_REG_X86_XMM6] = 23, + [PERF_REG_X86_XMM7] = 24, + [PERF_REG_X86_XMM8] = 25, + [PERF_REG_X86_XMM9] = 26, + [PERF_REG_X86_XMM10] = 27, + [PERF_REG_X86_XMM11] = 28, + [PERF_REG_X86_XMM12] = 29, + [PERF_REG_X86_XMM13] = 30, + [PERF_REG_X86_XMM14] = 31, + [PERF_REG_X86_XMM15] = 32, + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_x86_64_regnums) || + dwarf_x86_64_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_x86_64_regnums[perf_regnum]; +} diff --git a/tools/perf/util/dwarf-regs.c b/tools/perf/util/dwarf-regs.c new file mode 100644 index 000000000..797f455eb --- /dev/null +++ b/tools/perf/util/dwarf-regs.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. + * + * Written by: Masami Hiramatsu + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Define const char * {arch}_register_tbl[] */ +#define DEFINE_DWARF_REGSTR_TABLE +#include "../arch/x86/include/dwarf-regs-table.h" +#include "../arch/arm/include/dwarf-regs-table.h" +#include "../arch/arm64/include/dwarf-regs-table.h" +#include "../arch/sh/include/dwarf-regs-table.h" +#include "../arch/powerpc/include/dwarf-regs-table.h" +#include "../arch/riscv/include/dwarf-regs-table.h" +#include "../arch/s390/include/dwarf-regs-table.h" +#include "../arch/sparc/include/dwarf-regs-table.h" +#include "../arch/xtensa/include/dwarf-regs-table.h" +#include "../arch/mips/include/dwarf-regs-table.h" +#include "../arch/loongarch/include/dwarf-regs-table.h" + +/* Return architecture dependent register string (for kprobe-tracer) */ +const char *get_dwarf_regstr(unsigned int n, unsigned int machine, unsigned int flags) +{ + #define __get_dwarf_regstr(tbl, n) (((n) < ARRAY_SIZE(tbl)) ? (tbl)[(n)] : NULL) + + if (machine == EM_NONE) { + /* Generic arch - use host arch */ + machine = EM_HOST; + } + switch (machine) { + case EM_386: + return __get_dwarf_regstr(x86_32_regstr_tbl, n); + case EM_X86_64: + return __get_dwarf_regstr(x86_64_regstr_tbl, n); + case EM_ARM: + return __get_dwarf_regstr(arm_regstr_tbl, n); + case EM_AARCH64: + return __get_dwarf_regstr(aarch64_regstr_tbl, n); + case EM_CSKY: + return __get_csky_regstr(n, flags); + case EM_SH: + return __get_dwarf_regstr(sh_regstr_tbl, n); + case EM_S390: + return __get_dwarf_regstr(s390_regstr_tbl, n); + case EM_PPC: + case EM_PPC64: + return __get_dwarf_regstr(powerpc_regstr_tbl, n); + case EM_RISCV: + return __get_dwarf_regstr(riscv_regstr_tbl, n); + case EM_SPARC: + case EM_SPARCV9: + return __get_dwarf_regstr(sparc_regstr_tbl, n); + case EM_XTENSA: + return __get_dwarf_regstr(xtensa_regstr_tbl, n); + case EM_MIPS: + return __get_dwarf_regstr(mips_regstr_tbl, n); + case EM_LOONGARCH: + return __get_dwarf_regstr(loongarch_regstr_tbl, n); + default: + pr_err("ELF MACHINE %x is not supported.\n", machine); + } + return NULL; + + #undef __get_dwarf_regstr +} + +static int __get_dwarf_regnum(const char *const *regstr, size_t num_regstr, const char *name) +{ + for (size_t i = 0; i < num_regstr; i++) { + if (regstr[i] && !strcmp(regstr[i], name)) + return i; + } + return -ENOENT; +} + +/* Return DWARF register number from architecture register name */ +int get_dwarf_regnum(const char *name, unsigned int machine, unsigned int flags) +{ + char *regname = strdup(name); + int reg = -1; + char *p; + + #define _get_dwarf_regnum(tbl, name) __get_dwarf_regnum(tbl, ARRAY_SIZE(tbl), name) + + if (regname == NULL) + return -EINVAL; + + /* For convenience, remove trailing characters */ + p = strpbrk(regname, " ,)"); + if (p) + *p = '\0'; + + if (machine == EM_NONE) { + /* Generic arch - use host arch */ + machine = EM_HOST; + } + switch (machine) { + case EM_X86_64: + reg = __get_dwarf_regnum_x86_64(name); + break; + case EM_386: + reg = __get_dwarf_regnum_i386(name); + break; + case EM_ARM: + reg = _get_dwarf_regnum(arm_regstr_tbl, name); + break; + case EM_AARCH64: + reg = _get_dwarf_regnum(aarch64_regstr_tbl, name); + break; + case EM_CSKY: + reg = __get_csky_regnum(name, flags); + break; + case EM_SH: + reg = _get_dwarf_regnum(sh_regstr_tbl, name); + break; + case EM_S390: + reg = _get_dwarf_regnum(s390_regstr_tbl, name); + break; + case EM_PPC: + case EM_PPC64: + reg = _get_dwarf_regnum(powerpc_regstr_tbl, name); + break; + case EM_RISCV: + reg = _get_dwarf_regnum(riscv_regstr_tbl, name); + break; + case EM_SPARC: + case EM_SPARCV9: + reg = _get_dwarf_regnum(sparc_regstr_tbl, name); + break; + case EM_XTENSA: + reg = _get_dwarf_regnum(xtensa_regstr_tbl, name); + break; + case EM_MIPS: + reg = _get_dwarf_regnum(mips_regstr_tbl, name); + break; + case EM_LOONGARCH: + reg = _get_dwarf_regnum(loongarch_regstr_tbl, name); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", machine); + } + free(regname); + return reg; + + #undef _get_dwarf_regnum +} + +static int get_libdw_frame_nregs(unsigned int machine, unsigned int flags __maybe_unused) +{ + switch (machine) { + case EM_X86_64: + return 17; + case EM_386: + return 9; + case EM_ARM: + return 16; + case EM_AARCH64: + return 97; + case EM_CSKY: + return 38; + case EM_S390: + return 32; + case EM_PPC: + case EM_PPC64: + return 145; + case EM_RISCV: + return 66; + case EM_SPARC: + case EM_SPARCV9: + return 103; + case EM_LOONGARCH: + return 74; + case EM_MIPS: + return 71; + default: + return 0; + } +} + +int get_dwarf_regnum_for_perf_regnum(int perf_regnum, unsigned int machine, + unsigned int flags, bool only_libdw_supported) +{ + int reg; + + switch (machine) { + case EM_X86_64: + reg = __get_dwarf_regnum_for_perf_regnum_x86_64(perf_regnum); + break; + case EM_386: + reg = __get_dwarf_regnum_for_perf_regnum_i386(perf_regnum); + break; + case EM_ARM: + reg = __get_dwarf_regnum_for_perf_regnum_arm(perf_regnum); + break; + case EM_AARCH64: + reg = __get_dwarf_regnum_for_perf_regnum_arm64(perf_regnum); + break; + case EM_CSKY: + reg = __get_dwarf_regnum_for_perf_regnum_csky(perf_regnum, flags); + break; + case EM_PPC: + case EM_PPC64: + reg = __get_dwarf_regnum_for_perf_regnum_powerpc(perf_regnum); + break; + case EM_RISCV: + reg = __get_dwarf_regnum_for_perf_regnum_riscv(perf_regnum); + break; + case EM_S390: + reg = __get_dwarf_regnum_for_perf_regnum_s390(perf_regnum); + break; + case EM_LOONGARCH: + reg = __get_dwarf_regnum_for_perf_regnum_loongarch(perf_regnum); + break; + case EM_MIPS: + reg = __get_dwarf_regnum_for_perf_regnum_mips(perf_regnum); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", machine); + return -ENOENT; + } + if (reg >= 0 && only_libdw_supported) { + int nregs = get_libdw_frame_nregs(machine, flags); + + if (reg >= nregs) + reg = -ENOENT; + } + return reg; +} diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c new file mode 100644 index 000000000..c0e2b9d5f --- /dev/null +++ b/tools/perf/util/env.c @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "cpumap.h" +#include "dwarf-regs.h" +#include "debug.h" +#include "env.h" +#include "util/header.h" +#include "util/rwsem.h" +#include +#include +#include +#include +#include +#include +#include "cgroup.h" +#include +#include +#include +#include +#include "pmu.h" +#include "pmus.h" +#include "strbuf.h" +#include "trace/beauty/beauty.h" + +#ifdef HAVE_LIBBPF_SUPPORT +#include "bpf-event.h" +#include "bpf-utils.h" +#include + +bool perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node) +{ + bool ret; + + down_write(&env->bpf_progs.lock); + ret = __perf_env__insert_bpf_prog_info(env, info_node); + up_write(&env->bpf_progs.lock); + + return ret; +} + +bool __perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node) +{ + __u32 prog_id = info_node->info_linear->info.id; + struct bpf_prog_info_node *node; + struct rb_node *parent = NULL; + struct rb_node **p; + + p = &env->bpf_progs.infos.rb_node; + + while (*p != NULL) { + parent = *p; + node = rb_entry(parent, struct bpf_prog_info_node, rb_node); + if (prog_id < node->info_linear->info.id) { + p = &(*p)->rb_left; + } else if (prog_id > node->info_linear->info.id) { + p = &(*p)->rb_right; + } else { + pr_debug("duplicated bpf prog info %u\n", prog_id); + return false; + } + } + + rb_link_node(&info_node->rb_node, parent, p); + rb_insert_color(&info_node->rb_node, &env->bpf_progs.infos); + env->bpf_progs.infos_cnt++; + return true; +} + +struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, + __u32 prog_id) +{ + struct bpf_prog_info_node *node = NULL; + struct rb_node *n; + + down_read(&env->bpf_progs.lock); + n = env->bpf_progs.infos.rb_node; + + while (n) { + node = rb_entry(n, struct bpf_prog_info_node, rb_node); + if (prog_id < node->info_linear->info.id) + n = n->rb_left; + else if (prog_id > node->info_linear->info.id) + n = n->rb_right; + else + goto out; + } + node = NULL; + +out: + up_read(&env->bpf_progs.lock); + return node; +} + +void perf_env__iterate_bpf_prog_info(struct perf_env *env, + void (*cb)(struct bpf_prog_info_node *node, + void *data), + void *data) +{ + struct rb_node *first; + + down_read(&env->bpf_progs.lock); + first = rb_first(&env->bpf_progs.infos); + for (struct rb_node *node = first; node != NULL; node = rb_next(node)) + (*cb)(rb_entry(node, struct bpf_prog_info_node, rb_node), data); + up_read(&env->bpf_progs.lock); +} + +bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) +{ + bool ret; + + down_write(&env->bpf_progs.lock); + ret = __perf_env__insert_btf(env, btf_node); + up_write(&env->bpf_progs.lock); + return ret; +} + +bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) +{ + struct rb_node *parent = NULL; + __u32 btf_id = btf_node->id; + struct btf_node *node; + struct rb_node **p; + + p = &env->bpf_progs.btfs.rb_node; + + while (*p != NULL) { + parent = *p; + node = rb_entry(parent, struct btf_node, rb_node); + if (btf_id < node->id) { + p = &(*p)->rb_left; + } else if (btf_id > node->id) { + p = &(*p)->rb_right; + } else { + pr_debug("duplicated btf %u\n", btf_id); + return false; + } + } + + rb_link_node(&btf_node->rb_node, parent, p); + rb_insert_color(&btf_node->rb_node, &env->bpf_progs.btfs); + env->bpf_progs.btfs_cnt++; + return true; +} + +struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id) +{ + struct btf_node *res; + + down_read(&env->bpf_progs.lock); + res = __perf_env__find_btf(env, btf_id); + up_read(&env->bpf_progs.lock); + return res; +} + +struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id) +{ + struct btf_node *node = NULL; + struct rb_node *n; + + n = env->bpf_progs.btfs.rb_node; + + while (n) { + node = rb_entry(n, struct btf_node, rb_node); + if (btf_id < node->id) + n = n->rb_left; + else if (btf_id > node->id) + n = n->rb_right; + else + return node; + } + return NULL; +} + +/* purge data in bpf_progs.infos tree */ +static void perf_env__purge_bpf(struct perf_env *env) +{ + struct rb_root *root; + struct rb_node *next; + + down_write(&env->bpf_progs.lock); + + root = &env->bpf_progs.infos; + next = rb_first(root); + + while (next) { + struct bpf_prog_info_node *node; + + node = rb_entry(next, struct bpf_prog_info_node, rb_node); + next = rb_next(&node->rb_node); + rb_erase(&node->rb_node, root); + zfree(&node->info_linear); + bpf_metadata_free(node->metadata); + free(node); + } + + env->bpf_progs.infos_cnt = 0; + + root = &env->bpf_progs.btfs; + next = rb_first(root); + + while (next) { + struct btf_node *node; + + node = rb_entry(next, struct btf_node, rb_node); + next = rb_next(&node->rb_node); + rb_erase(&node->rb_node, root); + free(node); + } + + env->bpf_progs.btfs_cnt = 0; + + up_write(&env->bpf_progs.lock); +} +#else // HAVE_LIBBPF_SUPPORT +static void perf_env__purge_bpf(struct perf_env *env __maybe_unused) +{ +} +#endif // HAVE_LIBBPF_SUPPORT + +void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr) +{ + if (!cd_map) + return; + + for (u32 i = 0; i < nr; i++) { + if (!cd_map[i]) + continue; + + for (u32 j = 0; j < cd_map[i]->nr_domains; j++) { + struct domain_info *d_info = cd_map[i]->domains[j]; + + if (!d_info) + continue; + + if (schedstat_version >= 17) + zfree(&d_info->dname); + + zfree(&d_info->cpumask); + zfree(&d_info->cpulist); + zfree(&d_info); + } + zfree(&cd_map[i]->domains); + zfree(&cd_map[i]); + } + zfree(&cd_map); +} + +void perf_env__exit(struct perf_env *env) +{ + int i, j; + + mutex_destroy(&env->lock); + + perf_env__purge_bpf(env); + perf_env__purge_cgroups(env); + zfree(&env->hostname); + zfree(&env->os_release); + zfree(&env->version); + zfree(&env->arch); + zfree(&env->cpu_desc); + zfree(&env->cpuid); + zfree(&env->cmdline); + zfree(&env->cmdline_argv); + zfree(&env->sibling_dies); + zfree(&env->sibling_cores); + zfree(&env->sibling_threads); + zfree(&env->pmu_mappings); + zfree(&env->cpu); + for (i = 0; i < env->nr_cpu_pmu_caps; i++) + zfree(&env->cpu_pmu_caps[i]); + zfree(&env->cpu_pmu_caps); + zfree(&env->numa_map); + + for (i = 0; i < env->nr_numa_nodes; i++) + perf_cpu_map__put(env->numa_nodes[i].map); + zfree(&env->numa_nodes); + + for (i = 0; i < env->caches_cnt; i++) + cpu_cache_level__free(&env->caches[i]); + zfree(&env->caches); + + for (i = 0; i < env->nr_memory_nodes; i++) + zfree(&env->memory_nodes[i].set); + zfree(&env->memory_nodes); + + for (i = 0; i < env->nr_hybrid_nodes; i++) { + zfree(&env->hybrid_nodes[i].pmu_name); + zfree(&env->hybrid_nodes[i].cpus); + } + zfree(&env->hybrid_nodes); + + for (i = 0; i < env->nr_pmus_with_caps; i++) { + for (j = 0; j < env->pmu_caps[i].nr_caps; j++) + zfree(&env->pmu_caps[i].caps[j]); + zfree(&env->pmu_caps[i].caps); + zfree(&env->pmu_caps[i].pmu_name); + } + zfree(&env->pmu_caps); + free_cpu_domain_info(env->cpu_domain, env->schedstat_version, env->nr_cpus_avail); +} + +void perf_env__init(struct perf_env *env) +{ + memset(env, 0, sizeof(*env)); +#ifdef HAVE_LIBBPF_SUPPORT + env->bpf_progs.infos = RB_ROOT; + env->bpf_progs.btfs = RB_ROOT; + init_rwsem(&env->bpf_progs.lock); +#endif + env->kernel_is_64_bit = -1; + mutex_init(&env->lock); +} + +static void perf_env__init_kernel_mode(struct perf_env *env) +{ + const char *arch = env->arch; + + if (!arch) { + static struct utsname uts = { .machine[0] = '\0', }; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + if (arch) { + if (strstr(arch, "64") || strstr(arch, "s390x")) + env->kernel_is_64_bit = 1; + else + env->kernel_is_64_bit = 0; + return; + } + + /* Fallback if completely unresolvable (assume host-bitness) */ + env->kernel_is_64_bit = (sizeof(void *) == 8) ? 1 : 0; +} + +int perf_env__kernel_is_64_bit(struct perf_env *env) +{ + if (env->kernel_is_64_bit == -1) + perf_env__init_kernel_mode(env); + + return env->kernel_is_64_bit; +} + +bool perf_arch_is_big_endian(const char *arch) +{ + if (!arch) + return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__; + + if (str_ends_with(arch, "_be") || !strcmp(arch, "sparc") || !strcmp(arch, "sparc64") || + !strcmp(arch, "s390") || !strcmp(arch, "s390x") || !strcmp(arch, "powerpc") || + !strcmp(arch, "ppc") || !strcmp(arch, "ppc64") || + !strcmp(arch, "mips") || !strcmp(arch, "mips64") || !strcmp(arch, "parisc") || + !strcmp(arch, "parisc64") || !strcmp(arch, "m68k") || + !strcmp(arch, "armeb") || !strcmp(arch, "mipseb") || !strcmp(arch, "mips64eb")) + return true; + + return false; +} + +const char *perf_env__os_release(struct perf_env *env) +{ + struct utsname uts; + int ret; + const char *release; + + if (!env) + return perf_version_string; + + mutex_lock(&env->lock); + if (env->os_release) { + release = env->os_release; + goto out; + } + + /* + * If env->arch is set, this is an offline target environment. + * If the os_release is not populated in the file, we do not want + * to poison it with the host's release which would break guest checks. + */ + if (env->arch) { + release = NULL; + goto out; + } + + /* + * The os_release is being accessed but wasn't initialized from a data + * file, assume this is 'live' mode and use the release from uname. If + * uname or strdup fails then use the current perf tool version. + */ + ret = uname(&uts); + env->os_release = strdup(ret < 0 ? perf_version_string : uts.release); + release = env->os_release ?: perf_version_string; +out: + mutex_unlock(&env->lock); + return release; +} + +int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]) +{ + int i; + + /* do not include NULL termination */ + env->cmdline_argv = calloc(argc, sizeof(char *)); + if (env->cmdline_argv == NULL) + goto out_enomem; + + /* + * Must copy argv contents because it gets moved around during option + * parsing: + */ + for (i = 0; i < argc ; i++) { + env->cmdline_argv[i] = argv[i]; + if (env->cmdline_argv[i] == NULL) + goto out_free; + } + + env->nr_cmdline = argc; + + return 0; +out_free: + zfree(&env->cmdline_argv); +out_enomem: + return -ENOMEM; +} + +int perf_env__read_cpu_topology_map(struct perf_env *env) +{ + int idx, nr_cpus; + + if (env->cpu != NULL) + return 0; + + if (env->nr_cpus_avail == 0) + env->nr_cpus_avail = cpu__max_present_cpu().cpu; + + nr_cpus = env->nr_cpus_avail; + if (nr_cpus == -1) + return -EINVAL; + + env->cpu = calloc(nr_cpus, sizeof(env->cpu[0])); + if (env->cpu == NULL) + return -ENOMEM; + + for (idx = 0; idx < nr_cpus; ++idx) { + struct perf_cpu cpu = { .cpu = idx }; + int core_id = cpu__get_core_id(cpu); + int socket_id = cpu__get_socket_id(cpu); + int die_id = cpu__get_die_id(cpu); + + env->cpu[idx].core_id = core_id >= 0 ? core_id : -1; + env->cpu[idx].socket_id = socket_id >= 0 ? socket_id : -1; + env->cpu[idx].die_id = die_id >= 0 ? die_id : -1; + } + + env->nr_cpus_avail = nr_cpus; + return 0; +} + +int perf_env__read_pmu_mappings(struct perf_env *env) +{ + struct perf_pmu *pmu = NULL; + u32 pmu_num = 0; + struct strbuf sb; + + while ((pmu = perf_pmus__scan(pmu))) + pmu_num++; + + if (!pmu_num) { + pr_debug("pmu mappings not available\n"); + return -ENOENT; + } + env->nr_pmu_mappings = pmu_num; + + if (strbuf_init(&sb, 128 * pmu_num) < 0) + return -ENOMEM; + + while ((pmu = perf_pmus__scan(pmu))) { + if (strbuf_addf(&sb, "%u:%s", pmu->type, pmu->name) < 0) + goto error; + /* include a NULL character at the end */ + if (strbuf_add(&sb, "", 1) < 0) + goto error; + } + + env->pmu_mappings = strbuf_detach(&sb, NULL); + + return 0; + +error: + strbuf_release(&sb); + return -1; +} + +int perf_env__read_cpuid(struct perf_env *env) +{ + char cpuid[128]; + struct perf_cpu cpu = {-1}; + int err = get_cpuid(cpuid, sizeof(cpuid), cpu); + + if (err) + return err; + + free(env->cpuid); + env->cpuid = strdup(cpuid); + if (env->cpuid == NULL) + return ENOMEM; + return 0; +} + +static int perf_env__read_nr_cpus_avail(struct perf_env *env) +{ + if (env->nr_cpus_avail == 0) + env->nr_cpus_avail = cpu__max_present_cpu().cpu; + + return env->nr_cpus_avail ? 0 : -ENOENT; +} + +static int __perf_env__read_core_pmu_caps(const struct perf_pmu *pmu, + int *nr_caps, char ***caps, + unsigned int *max_branches, + unsigned int *br_cntr_nr, + unsigned int *br_cntr_width) +{ + struct perf_pmu_caps *pcaps = NULL; + char *ptr, **tmp; + int ret = 0; + + *nr_caps = 0; + *caps = NULL; + + if (!pmu->nr_caps) + return 0; + + *caps = calloc(pmu->nr_caps, sizeof(char *)); + if (!*caps) + return -ENOMEM; + + tmp = *caps; + list_for_each_entry(pcaps, &pmu->caps, list) { + if (asprintf(&ptr, "%s=%s", pcaps->name, pcaps->value) < 0) { + ret = -ENOMEM; + goto error; + } + + *tmp++ = ptr; + + if (!strcmp(pcaps->name, "branches")) + *max_branches = atoi(pcaps->value); + else if (!strcmp(pcaps->name, "branch_counter_nr")) + *br_cntr_nr = atoi(pcaps->value); + else if (!strcmp(pcaps->name, "branch_counter_width")) + *br_cntr_width = atoi(pcaps->value); + } + *nr_caps = pmu->nr_caps; + return 0; +error: + while (tmp-- != *caps) + zfree(tmp); + zfree(caps); + *nr_caps = 0; + return ret; +} + +int perf_env__read_core_pmu_caps(struct perf_env *env) +{ + struct pmu_caps *pmu_caps; + struct perf_pmu *pmu = NULL; + int nr_pmu, i = 0, j; + int ret; + + nr_pmu = perf_pmus__num_core_pmus(); + + if (!nr_pmu) + return -ENODEV; + + if (nr_pmu == 1) { + pmu = perf_pmus__find_core_pmu(); + if (!pmu) + return -ENODEV; + ret = perf_pmu__caps_parse(pmu); + if (ret < 0) + return ret; + return __perf_env__read_core_pmu_caps(pmu, &env->nr_cpu_pmu_caps, + &env->cpu_pmu_caps, + &env->max_branches, + &env->br_cntr_nr, + &env->br_cntr_width); + } + + pmu_caps = calloc(nr_pmu, sizeof(*pmu_caps)); + if (!pmu_caps) + return -ENOMEM; + + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + if (perf_pmu__caps_parse(pmu) <= 0) + continue; + ret = __perf_env__read_core_pmu_caps(pmu, &pmu_caps[i].nr_caps, + &pmu_caps[i].caps, + &pmu_caps[i].max_branches, + &pmu_caps[i].br_cntr_nr, + &pmu_caps[i].br_cntr_width); + if (ret) + goto error; + + pmu_caps[i].pmu_name = strdup(pmu->name); + if (!pmu_caps[i].pmu_name) { + ret = -ENOMEM; + goto error; + } + i++; + } + + env->nr_pmus_with_caps = nr_pmu; + env->pmu_caps = pmu_caps; + + return 0; +error: + for (i = 0; i < nr_pmu; i++) { + for (j = 0; j < pmu_caps[i].nr_caps; j++) + zfree(&pmu_caps[i].caps[j]); + zfree(&pmu_caps[i].caps); + zfree(&pmu_caps[i].pmu_name); + } + zfree(&pmu_caps); + return ret; +} + +int perf_env__nr_cpus_avail(struct perf_env *env) +{ + return env && !perf_env__read_nr_cpus_avail(env) ? env->nr_cpus_avail : 0; +} + +void cpu_cache_level__free(struct cpu_cache_level *cache) +{ + zfree(&cache->type); + zfree(&cache->map); + zfree(&cache->size); +} + +struct arch_to_e_machine { + const char *prefix; + uint16_t e_machine; +}; + +/* + * A mapping from an arch prefix string to an ELF machine that can be used in a + * bsearch. Some arch prefixes are shared an need additional processing as + * marked next to the architecture. The prefixes handle both perf's architecture + * naming and those from uname. + */ +static const struct arch_to_e_machine prefix_to_e_machine[] = { + {"aarch64", EM_AARCH64}, + {"alpha", EM_ALPHA}, + {"arc", EM_ARC}, + {"arm", EM_ARM}, /* Check also for EM_AARCH64. */ + {"avr", EM_AVR}, /* Check also for EM_AVR32. */ + {"bfin", EM_BLACKFIN}, + {"blackfin", EM_BLACKFIN}, + {"cris", EM_CRIS}, + {"csky", EM_CSKY}, + {"hppa", EM_PARISC}, + {"i386", EM_386}, + {"i486", EM_386}, + {"i586", EM_386}, + {"i686", EM_386}, + {"loongarch", EM_LOONGARCH}, + {"m32r", EM_M32R}, + {"m68k", EM_68K}, + {"microblaze", EM_MICROBLAZE}, + {"mips", EM_MIPS}, + {"msp430", EM_MSP430}, + {"parisc", EM_PARISC}, + {"powerpc", EM_PPC}, /* Check also for EM_PPC64. */ + {"ppc", EM_PPC}, /* Check also for EM_PPC64. */ + {"riscv", EM_RISCV}, + {"s390", EM_S390}, + {"sa110", EM_ARM}, + {"sh", EM_SH}, + {"sparc", EM_SPARC}, /* Check also for EM_SPARCV9. */ + {"sun4u", EM_SPARC}, + {"x86", EM_X86_64}, /* Check also for EM_386. */ + {"xtensa", EM_XTENSA}, +}; + +static int compare_prefix(const void *key, const void *element) +{ + const char *search_key = key; + const struct arch_to_e_machine *map_element = element; + size_t prefix_len = strlen(map_element->prefix); + + return strncmp(search_key, map_element->prefix, prefix_len); +} + +static uint16_t perf_arch_to_e_machine(const char *perf_arch, int is_64_bit) +{ + /* Binary search for a matching prefix. */ + const struct arch_to_e_machine *result; + + if (!perf_arch) + return EM_HOST; + + result = bsearch(perf_arch, + prefix_to_e_machine, ARRAY_SIZE(prefix_to_e_machine), + sizeof(prefix_to_e_machine[0]), + compare_prefix); + + if (!result) { + pr_debug("Unknown perf arch for ELF machine mapping: %s\n", perf_arch); + return EM_NONE; + } + + /* + * Handle conflicting prefixes. If the is_64_bit is unknown (-1) then + * assume 64-bit. We can't use perf_env__kernel_is_64_bit as that + * depends on the arch string. + */ + switch (result->e_machine) { + case EM_ARM: + return !strcmp(perf_arch, "arm64") || !strcmp(perf_arch, "aarch64") + ? EM_AARCH64 : EM_ARM; + case EM_AVR: + return !strcmp(perf_arch, "avr32") ? EM_AVR32 : EM_AVR; + case EM_PPC: + if (is_64_bit == 1) + return EM_PPC64; + if (is_64_bit == 0) + return EM_PPC; + return strstarts(perf_arch, "ppc64") ? EM_PPC64 : EM_PPC; + case EM_SPARC: + if (is_64_bit == 1) + return EM_SPARCV9; + if (is_64_bit == 0) + return EM_SPARC; + return !strcmp(perf_arch, "sparc64") || !strcmp(perf_arch, "sun4u") + ? EM_SPARCV9 : EM_SPARC; + case EM_X86_64: + if (is_64_bit == 1) + return EM_X86_64; + if (is_64_bit == 0) + return EM_386; + return !strcmp(perf_arch, "x86_64") || !strcmp(perf_arch, "x86") + ? EM_X86_64 : EM_386; + default: + return result->e_machine; + } +} + +static const char *e_machine_to_perf_arch(uint16_t e_machine) +{ + /* + * Table for if either the perf arch string differs from uname or there + * are >1 ELF machine with the prefix. + */ + static const struct arch_to_e_machine extras[] = { + {"arm64", EM_AARCH64}, + {"avr32", EM_AVR32}, + {"powerpc", EM_PPC}, + {"powerpc", EM_PPC64}, + {"sparc", EM_SPARCV9}, + {"x86", EM_386}, + {"x86", EM_X86_64}, + {"none", EM_NONE}, + }; + + for (size_t i = 0; i < ARRAY_SIZE(extras); i++) { + if (extras[i].e_machine == e_machine) + return extras[i].prefix; + } + + for (size_t i = 0; i < ARRAY_SIZE(prefix_to_e_machine); i++) { + if (prefix_to_e_machine[i].e_machine == e_machine) + return prefix_to_e_machine[i].prefix; + + } + return "unknown"; +} + +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags) +{ + uint16_t e_machine = EM_NONE; + const char *arch = NULL; + int is_64_bit = -1; + + if (e_flags) + *e_flags = 0; + + if (env) { + arch = env->arch; + is_64_bit = env->kernel_is_64_bit; + } + + if (!arch) { + static struct utsname uts = { .machine[0] = '\0', }; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + e_machine = perf_arch_to_e_machine(arch, is_64_bit); + + if (e_flags) + *e_flags = (e_machine == EM_HOST) ? EF_HOST : 0; + + return e_machine; +} + +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags) +{ + uint16_t e_machine; + uint32_t local_e_flags = 0; + + if (env && env->e_machine != EM_NONE) { + if (e_flags) + *e_flags = env->e_flags; + + return env->e_machine; + } + e_machine = perf_env__e_machine_nocache(env, &local_e_flags); + /* + * Only cache the e_machine in perf_env if env->arch is not NULL. + * If env->arch is NULL, the e_machine is just a fallback to EM_HOST. + * Caching it permanently would prevent dynamic, more accurate + * thread-based session e_machine scanning later in + * perf_session__e_machine(). + */ + if (env && env->arch) { + env->e_machine = e_machine; + env->e_flags = local_e_flags; + } + if (e_flags) + *e_flags = local_e_flags; + + return e_machine; +} + +const char *perf_env__arch(struct perf_env *env) +{ + uint16_t e_machine; + const char *arch; + + if (!env) { + static struct utsname uts = { .machine[0] = '\0', }; + uint16_t host_e_machine; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') { + host_e_machine = perf_arch_to_e_machine(uts.machine, -1); + return e_machine_to_perf_arch(host_e_machine); + } + return e_machine_to_perf_arch(EM_HOST); + } + + /* + * Lazily compute/allocate arch. The e_machine may have been + * read from a data file and so may not be EM_HOST. + */ + e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + arch = e_machine_to_perf_arch(e_machine); + + if (e_machine == EM_RISCV && perf_env__kernel_is_64_bit(env) == 1) + arch = "riscv64"; + else if (e_machine == EM_MIPS && perf_env__kernel_is_64_bit(env) == 1) + arch = "mips64"; + else if (e_machine == EM_PARISC && perf_env__kernel_is_64_bit(env) == 1) + arch = "parisc64"; + + return arch; +} + +const char *arch_syscalls__strerrno(uint16_t e_machine, int err); + +const char *perf_env__arch_strerrno(uint16_t e_machine, int err) +{ + return arch_syscalls__strerrno(e_machine, err); +} + +const char *perf_env__cpuid(struct perf_env *env) +{ + int status; + + if (!env->cpuid) { /* Assume local operation */ + status = perf_env__read_cpuid(env); + if (status) + return NULL; + } + + return env->cpuid; +} + +int perf_env__nr_pmu_mappings(struct perf_env *env) +{ + int status; + + if (!env->nr_pmu_mappings) { /* Assume local operation */ + status = perf_env__read_pmu_mappings(env); + if (status) + return 0; + } + + return env->nr_pmu_mappings; +} + +const char *perf_env__pmu_mappings(struct perf_env *env) +{ + int status; + + if (!env->pmu_mappings) { /* Assume local operation */ + status = perf_env__read_pmu_mappings(env); + if (status) + return NULL; + } + + return env->pmu_mappings; +} + +int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu) +{ + if (!env->nr_numa_map) { + struct numa_node *nn; + int i, nr = 0; + + for (i = 0; i < env->nr_numa_nodes; i++) { + nn = &env->numa_nodes[i]; + nr = max(nr, (int)perf_cpu_map__max(nn->map).cpu); + } + + nr++; + + /* + * We initialize the numa_map array to prepare + * it for missing cpus, which return node -1 + */ + env->numa_map = malloc(nr * sizeof(int)); + if (!env->numa_map) + return -1; + + for (i = 0; i < nr; i++) + env->numa_map[i] = -1; + + env->nr_numa_map = nr; + + for (i = 0; i < env->nr_numa_nodes; i++) { + struct perf_cpu tmp; + unsigned int j; + + nn = &env->numa_nodes[i]; + perf_cpu_map__for_each_cpu(tmp, j, nn->map) + env->numa_map[tmp.cpu] = i; + } + } + + return cpu.cpu >= 0 && cpu.cpu < env->nr_numa_map ? env->numa_map[cpu.cpu] : -1; +} + +bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name) +{ + char *pmu_mapping = env->pmu_mappings, *colon; + + for (int i = 0; i < env->nr_pmu_mappings; ++i) { + if (strtoul(pmu_mapping, &colon, 0) == ULONG_MAX || *colon != ':') + goto out_error; + + pmu_mapping = colon + 1; + if (strcmp(pmu_mapping, pmu_name) == 0) + return true; + + pmu_mapping += strlen(pmu_mapping) + 1; + } +out_error: + return false; +} + +char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name, + const char *cap) +{ + char *cap_eq; + int cap_size; + char **ptr; + int i, j; + + if (!pmu_name || !cap) + return NULL; + + cap_size = strlen(cap); + cap_eq = zalloc(cap_size + 2); + if (!cap_eq) + return NULL; + + memcpy(cap_eq, cap, cap_size); + cap_eq[cap_size] = '='; + + if (!strcmp(pmu_name, "cpu")) { + for (i = 0; i < env->nr_cpu_pmu_caps; i++) { + if (!strncmp(env->cpu_pmu_caps[i], cap_eq, cap_size + 1)) { + free(cap_eq); + return &env->cpu_pmu_caps[i][cap_size + 1]; + } + } + goto out; + } + + for (i = 0; i < env->nr_pmus_with_caps; i++) { + if (strcmp(env->pmu_caps[i].pmu_name, pmu_name)) + continue; + + ptr = env->pmu_caps[i].caps; + + for (j = 0; j < env->pmu_caps[i].nr_caps; j++) { + if (!strncmp(ptr[j], cap_eq, cap_size + 1)) { + free(cap_eq); + return &ptr[j][cap_size + 1]; + } + } + } + +out: + free(cap_eq); + return NULL; +} + +void perf_env__find_br_cntr_info(struct perf_env *env, + unsigned int *nr, + unsigned int *width) +{ + if (nr) { + *nr = env->cpu_pmu_caps ? env->br_cntr_nr : + env->pmu_caps->br_cntr_nr; + } + + if (width) { + *width = env->cpu_pmu_caps ? env->br_cntr_width : + env->pmu_caps->br_cntr_width; + } +} + +bool perf_env__is_x86_amd_cpu(struct perf_env *env) +{ + static int is_amd; /* 0: Uninitialized, 1: Yes, -1: No */ + + if (is_amd == 0) + is_amd = env->cpuid && strstarts(env->cpuid, "AuthenticAMD") ? 1 : -1; + + return is_amd >= 1 ? true : false; +} + +bool x86__is_amd_cpu(void) +{ + struct perf_env env = { .total_mem = 0, }; + bool is_amd; + + perf_env__init(&env); + perf_env__cpuid(&env); + is_amd = perf_env__is_x86_amd_cpu(&env); + perf_env__exit(&env); + + return is_amd; +} + +bool perf_env__is_x86_intel_cpu(struct perf_env *env) +{ + static int is_intel; /* 0: Uninitialized, 1: Yes, -1: No */ + + if (is_intel == 0) + is_intel = env->cpuid && strstarts(env->cpuid, "GenuineIntel") ? 1 : -1; + + return is_intel >= 1 ? true : false; +} + +bool x86__is_intel_cpu(void) +{ + struct perf_env env = { .total_mem = 0, }; + bool is_intel; + + perf_env__init(&env); + perf_env__cpuid(&env); + is_intel = perf_env__is_x86_intel_cpu(&env); + perf_env__exit(&env); + + return is_intel; +} diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h new file mode 100644 index 000000000..7acca39b4 --- /dev/null +++ b/tools/perf/util/env.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ENV_H +#define __PERF_ENV_H + +#include +#include +#include "cpumap.h" +#include "rwsem.h" +#include "mutex.h" + +struct perf_cpu_map; + +struct cpu_topology_map { + int socket_id; + int die_id; + int cluster_id; + int core_id; +}; + +struct cpu_cache_level { + u32 level; + u32 line_size; + u32 sets; + u32 ways; + char *type; + char *size; + char *map; +}; + +struct numa_node { + u32 node; + u64 mem_total; + u64 mem_free; + struct perf_cpu_map *map; +}; + +struct memory_node { + u64 node; + u64 size; + unsigned long *set; +}; + +struct hybrid_node { + char *pmu_name; + char *cpus; +}; + +struct pmu_caps { + int nr_caps; + unsigned int max_branches; + unsigned int br_cntr_nr; + unsigned int br_cntr_width; + + char **caps; + char *pmu_name; +}; + +struct domain_info { + u32 domain; + char *dname; + char *cpumask; + char *cpulist; +}; + +struct cpu_domain_map { + u32 cpu; + u32 nr_domains; + struct domain_info **domains; +}; + +struct perf_env { + char *hostname; + char *os_release; + char *version; + char *arch; + /* e_machine expanded from 16 to 32-bits for alignment. */ + u32 e_machine; + u32 e_flags; + int nr_cpus_online; + int nr_cpus_avail; + char *cpu_desc; + char *cpuid; + unsigned long long total_mem; + unsigned int msr_pmu_type; + unsigned int max_branches; + unsigned int br_cntr_nr; + unsigned int br_cntr_width; + unsigned int schedstat_version; + unsigned int max_sched_domains; + int kernel_is_64_bit; + + int nr_cmdline; + int nr_sibling_cores; + int nr_sibling_dies; + int nr_sibling_threads; + int nr_numa_nodes; + int nr_memory_nodes; + int nr_pmu_mappings; + int nr_groups; + int nr_cpu_pmu_caps; + int nr_hybrid_nodes; + int nr_pmus_with_caps; + char *cmdline; + const char **cmdline_argv; + char *sibling_cores; + char *sibling_dies; + char *sibling_threads; + char *pmu_mappings; + char **cpu_pmu_caps; + struct cpu_topology_map *cpu; + struct cpu_cache_level *caches; + struct cpu_domain_map **cpu_domain; + int caches_cnt; + unsigned int cln_size; + u32 comp_ratio; + u32 comp_ver; + u32 comp_type; + u32 comp_level; + u32 comp_mmap_len; + struct numa_node *numa_nodes; + struct memory_node *memory_nodes; + unsigned long long memory_bsize; + struct hybrid_node *hybrid_nodes; + struct pmu_caps *pmu_caps; +#ifdef HAVE_LIBBPF_SUPPORT + /* + * bpf_info_lock protects bpf rbtrees. This is needed because the + * trees are accessed by different threads in perf-top + */ + struct { + struct rw_semaphore lock; + struct rb_root infos; + u32 infos_cnt; + struct rb_root btfs; + u32 btfs_cnt; + } bpf_progs; +#endif // HAVE_LIBBPF_SUPPORT + /* same reason as above (for perf-top) */ + struct { + struct rw_semaphore lock; + struct rb_root tree; + } cgroups; + + /* For fast cpu to numa node lookup via perf_env__numa_node */ + int *numa_map; + int nr_numa_map; + + /* For real clock time reference. */ + struct { + u64 tod_ns; + u64 clockid_ns; + u64 clockid_res_ns; + int clockid; + /* + * enabled is valid for report mode, and is true if above + * values are set, it's set in process_clock_data + */ + bool enabled; + } clock; + /* Protects lazy environment initialization (e.g. os_release, e_machine). */ + struct mutex lock; +}; + +enum perf_compress_type { + PERF_COMP_NONE = 0, + PERF_COMP_ZSTD, + PERF_COMP_MAX +}; + +struct bpf_prog_info_node; +struct btf_node; + +int perf_env__read_core_pmu_caps(struct perf_env *env); +void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr); +void perf_env__exit(struct perf_env *env); + +int perf_env__kernel_is_64_bit(struct perf_env *env); +bool perf_arch_is_big_endian(const char *arch); +const char *perf_env__os_release(struct perf_env *env); + +int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]); + +int perf_env__read_cpuid(struct perf_env *env); +int perf_env__read_pmu_mappings(struct perf_env *env); +int perf_env__nr_pmu_mappings(struct perf_env *env); +const char *perf_env__pmu_mappings(struct perf_env *env); + +int perf_env__read_cpu_topology_map(struct perf_env *env); + +/* + * Safe accessor for env->cpu[] topology array. env->cpu can be NULL when + * reading old-format perf.data that predates topology information — + * process_cpu_topology() in header.c frees it while nr_cpus_avail remains + * set, so callers must not index env->cpu[] without this check. + */ +static inline struct cpu_topology_map * +perf_env__get_cpu_topology(struct perf_env *env, struct perf_cpu cpu) +{ + if (env->cpu && cpu.cpu >= 0 && cpu.cpu < env->nr_cpus_avail) + return &env->cpu[cpu.cpu]; + return NULL; +} + +void cpu_cache_level__free(struct cpu_cache_level *cache); + +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags); +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags); +const char *perf_env__arch(struct perf_env *env); +const char *perf_env__arch_strerrno(uint16_t e_machine, int err); +const char *perf_env__cpuid(struct perf_env *env); +int perf_env__nr_cpus_avail(struct perf_env *env); + +void perf_env__init(struct perf_env *env); +#ifdef HAVE_LIBBPF_SUPPORT +bool __perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node); +bool perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node); +struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, + __u32 prog_id); +void perf_env__iterate_bpf_prog_info(struct perf_env *env, + void (*cb)(struct bpf_prog_info_node *node, + void *data), + void *data); +bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node); +bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node); +struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id); +struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id); +#endif // HAVE_LIBBPF_SUPPORT + +int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu); +char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name, + const char *cap); + +bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name); +void perf_env__find_br_cntr_info(struct perf_env *env, + unsigned int *nr, + unsigned int *width); + +bool x86__is_amd_cpu(void); +bool perf_env__is_x86_amd_cpu(struct perf_env *env); +bool x86__is_intel_cpu(void); +bool perf_env__is_x86_intel_cpu(struct perf_env *env); + +#endif /* __PERF_ENV_H */ diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c new file mode 100644 index 000000000..ea75816d1 --- /dev/null +++ b/tools/perf/util/event.c @@ -0,0 +1,944 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* To get things like MAP_HUGETLB even on older libc headers */ +#include +#include "cpumap.h" +#include "dso.h" +#include "env.h" +#include "event.h" +#include "debug.h" +#include "hist.h" +#include "machine.h" +#include "sort.h" +#include "string2.h" +#include "strlist.h" +#include "thread.h" +#include "thread_map.h" +#include "time-utils.h" +#include +#include "map.h" +#include "util/namespaces.h" +#include "symbol.h" +#include "symbol/kallsyms.h" +#include "asm/bug.h" +#include "stat.h" +#include "session.h" +#include "bpf-event.h" +#include "print_binary.h" +#include "tool.h" +#include "util.h" + +static const char *perf_event__names[] = { + [0] = "TOTAL", + [PERF_RECORD_MMAP] = "MMAP", + [PERF_RECORD_MMAP2] = "MMAP2", + [PERF_RECORD_LOST] = "LOST", + [PERF_RECORD_COMM] = "COMM", + [PERF_RECORD_EXIT] = "EXIT", + [PERF_RECORD_THROTTLE] = "THROTTLE", + [PERF_RECORD_UNTHROTTLE] = "UNTHROTTLE", + [PERF_RECORD_FORK] = "FORK", + [PERF_RECORD_READ] = "READ", + [PERF_RECORD_SAMPLE] = "SAMPLE", + [PERF_RECORD_AUX] = "AUX", + [PERF_RECORD_ITRACE_START] = "ITRACE_START", + [PERF_RECORD_LOST_SAMPLES] = "LOST_SAMPLES", + [PERF_RECORD_SWITCH] = "SWITCH", + [PERF_RECORD_SWITCH_CPU_WIDE] = "SWITCH_CPU_WIDE", + [PERF_RECORD_NAMESPACES] = "NAMESPACES", + [PERF_RECORD_KSYMBOL] = "KSYMBOL", + [PERF_RECORD_BPF_EVENT] = "BPF_EVENT", + [PERF_RECORD_CGROUP] = "CGROUP", + [PERF_RECORD_TEXT_POKE] = "TEXT_POKE", + [PERF_RECORD_AUX_OUTPUT_HW_ID] = "AUX_OUTPUT_HW_ID", + [PERF_RECORD_CALLCHAIN_DEFERRED] = "CALLCHAIN_DEFERRED", + [PERF_RECORD_HEADER_ATTR] = "ATTR", + [PERF_RECORD_HEADER_EVENT_TYPE] = "EVENT_TYPE", + [PERF_RECORD_HEADER_TRACING_DATA] = "TRACING_DATA", + [PERF_RECORD_HEADER_BUILD_ID] = "BUILD_ID", + [PERF_RECORD_FINISHED_ROUND] = "FINISHED_ROUND", + [PERF_RECORD_ID_INDEX] = "ID_INDEX", + [PERF_RECORD_AUXTRACE_INFO] = "AUXTRACE_INFO", + [PERF_RECORD_AUXTRACE] = "AUXTRACE", + [PERF_RECORD_AUXTRACE_ERROR] = "AUXTRACE_ERROR", + [PERF_RECORD_THREAD_MAP] = "THREAD_MAP", + [PERF_RECORD_CPU_MAP] = "CPU_MAP", + [PERF_RECORD_STAT_CONFIG] = "STAT_CONFIG", + [PERF_RECORD_STAT] = "STAT", + [PERF_RECORD_STAT_ROUND] = "STAT_ROUND", + [PERF_RECORD_EVENT_UPDATE] = "EVENT_UPDATE", + [PERF_RECORD_TIME_CONV] = "TIME_CONV", + [PERF_RECORD_HEADER_FEATURE] = "FEATURE", + [PERF_RECORD_COMPRESSED] = "COMPRESSED", + [PERF_RECORD_FINISHED_INIT] = "FINISHED_INIT", + [PERF_RECORD_COMPRESSED2] = "COMPRESSED2", + [PERF_RECORD_BPF_METADATA] = "BPF_METADATA", + [PERF_RECORD_SCHEDSTAT_CPU] = "SCHEDSTAT_CPU", + [PERF_RECORD_SCHEDSTAT_DOMAIN] = "SCHEDSTAT_DOMAIN", +}; + +const char *perf_event__name(unsigned int id) +{ + if (id >= ARRAY_SIZE(perf_event__names)) + return "INVALID"; + if (!perf_event__names[id]) + return "UNKNOWN"; + return perf_event__names[id]; +} + +struct process_symbol_args { + const char *name; + u64 start; +}; + +static int find_func_symbol_cb(void *arg, const char *name, char type, + u64 start) +{ + struct process_symbol_args *args = arg; + + /* + * Must be a function or at least an alias, as in PARISC64, where "_text" is + * an 'A' to the same address as "_stext". + */ + if (!(kallsyms__is_function(type) || + type == 'A') || strcmp(name, args->name)) + return 0; + + args->start = start; + return 1; +} + +static int find_any_symbol_cb(void *arg, const char *name, + char type __maybe_unused, u64 start) +{ + struct process_symbol_args *args = arg; + + if (strcmp(name, args->name)) + return 0; + + args->start = start; + return 1; +} + +int kallsyms__get_function_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr) +{ + struct process_symbol_args args = { .name = symbol_name, }; + + if (kallsyms__parse(kallsyms_filename, &args, find_func_symbol_cb) <= 0) + return -1; + + *addr = args.start; + return 0; +} + +int kallsyms__get_symbol_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr) +{ + struct process_symbol_args args = { .name = symbol_name, }; + + if (kallsyms__parse(kallsyms_filename, &args, find_any_symbol_cb) <= 0) + return -1; + + *addr = args.start; + return 0; +} + +void perf_event__read_stat_config(struct perf_stat_config *config, + struct perf_record_stat_config *event) +{ + unsigned i; + + for (i = 0; i < event->nr; i++) { + + switch (event->data[i].tag) { +#define CASE(__term, __val) \ + case PERF_STAT_CONFIG_TERM__##__term: \ + config->__val = event->data[i].val; \ + break; + + CASE(AGGR_MODE, aggr_mode) + CASE(SCALE, scale) + CASE(INTERVAL, interval) + CASE(AGGR_LEVEL, aggr_level) +#undef CASE + default: + pr_warning("unknown stat config term %" PRI_lu64 "\n", + event->data[i].tag); + } + } +} + +size_t perf_event__fprintf_comm(union perf_event *event, FILE *fp) +{ + const char *s; + + if (event->header.misc & PERF_RECORD_MISC_COMM_EXEC) + s = " exec"; + else + s = ""; + + return fprintf(fp, "%s: %s:%d/%d\n", s, event->comm.comm, event->comm.pid, event->comm.tid); +} + +size_t perf_event__fprintf_namespaces(union perf_event *event, FILE *fp) +{ + size_t ret = 0; + struct perf_ns_link_info *ns_link_info; + u32 nr_namespaces, idx; + + ns_link_info = event->namespaces.link_info; + nr_namespaces = event->namespaces.nr_namespaces; + + ret += fprintf(fp, " %d/%d - nr_namespaces: %u\n\t\t[", + event->namespaces.pid, + event->namespaces.tid, + nr_namespaces); + + for (idx = 0; idx < nr_namespaces; idx++) { + if (idx && (idx % 4 == 0)) + ret += fprintf(fp, "\n\t\t "); + + ret += fprintf(fp, "%u/%s: %" PRIu64 "/%#" PRIx64 "%s", idx, + perf_ns__name(idx), (u64)ns_link_info[idx].dev, + (u64)ns_link_info[idx].ino, + ((idx + 1) != nr_namespaces) ? ", " : "]\n"); + } + + return ret; +} + +size_t perf_event__fprintf_cgroup(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " cgroup: %" PRI_lu64 " %s\n", + event->cgroup.id, event->cgroup.path); +} + +int perf_event__process_comm(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_comm_event(machine, event, sample); +} + +int perf_event__process_namespaces(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_namespaces_event(machine, event, sample); +} + +int perf_event__process_cgroup(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_cgroup_event(machine, event, sample); +} + +int perf_event__process_lost(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_lost_event(machine, event, sample); +} + +int perf_event__process_aux(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_aux_event(machine, event); +} + +int perf_event__process_itrace_start(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_itrace_start_event(machine, event); +} + +int perf_event__process_aux_output_hw_id(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_aux_output_hw_id_event(machine, event); +} + +int perf_event__process_lost_samples(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_lost_samples_event(machine, event, sample); +} + +int perf_event__process_switch(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_switch_event(machine, event); +} + +int perf_event__process_ksymbol(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_ksymbol(machine, event, sample); +} + +int perf_event__process_bpf(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_bpf(machine, event, sample); +} + +int perf_event__process_text_poke(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_text_poke(machine, event, sample); +} + +size_t perf_event__fprintf_mmap(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " %d/%d: [%#" PRI_lx64 "(%#" PRI_lx64 ") @ %#" PRI_lx64 "]: %c %s\n", + event->mmap.pid, event->mmap.tid, event->mmap.start, + event->mmap.len, event->mmap.pgoff, + (event->header.misc & PERF_RECORD_MISC_MMAP_DATA) ? 'r' : 'x', + event->mmap.filename); +} + +size_t perf_event__fprintf_mmap2(union perf_event *event, FILE *fp) +{ + if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) { + char sbuild_id[SBUILD_ID_SIZE]; + struct build_id bid; + + build_id__init(&bid, event->mmap2.build_id, + event->mmap2.build_id_size); + build_id__snprintf(&bid, sbuild_id, sizeof(sbuild_id)); + + return fprintf(fp, " %d/%d: [%#" PRI_lx64 "(%#" PRI_lx64 ") @ %#" PRI_lx64 + " <%s>]: %c%c%c%c %s\n", + event->mmap2.pid, event->mmap2.tid, event->mmap2.start, + event->mmap2.len, event->mmap2.pgoff, sbuild_id, + (event->mmap2.prot & PROT_READ) ? 'r' : '-', + (event->mmap2.prot & PROT_WRITE) ? 'w' : '-', + (event->mmap2.prot & PROT_EXEC) ? 'x' : '-', + (event->mmap2.flags & MAP_SHARED) ? 's' : 'p', + event->mmap2.filename); + } else { + return fprintf(fp, " %d/%d: [%#" PRI_lx64 "(%#" PRI_lx64 ") @ %#" PRI_lx64 + " %02x:%02x %"PRI_lu64" %"PRI_lu64"]: %c%c%c%c %s\n", + event->mmap2.pid, event->mmap2.tid, event->mmap2.start, + event->mmap2.len, event->mmap2.pgoff, event->mmap2.maj, + event->mmap2.min, event->mmap2.ino, + event->mmap2.ino_generation, + (event->mmap2.prot & PROT_READ) ? 'r' : '-', + (event->mmap2.prot & PROT_WRITE) ? 'w' : '-', + (event->mmap2.prot & PROT_EXEC) ? 'x' : '-', + (event->mmap2.flags & MAP_SHARED) ? 's' : 'p', + event->mmap2.filename); + } +} + +size_t perf_event__fprintf_thread_map(union perf_event *event, FILE *fp) +{ + struct perf_thread_map *threads = thread_map__new_event(&event->thread_map); + size_t ret; + + ret = fprintf(fp, " nr: "); + + if (threads) + ret += thread_map__fprintf(threads, fp); + else + ret += fprintf(fp, "failed to get threads from event\n"); + + perf_thread_map__put(threads); + return ret; +} + +size_t perf_event__fprintf_cpu_map(union perf_event *event, FILE *fp) +{ + struct perf_cpu_map *cpus = cpu_map__new_data(&event->cpu_map.data); + size_t ret; + + ret = fprintf(fp, ": "); + + if (cpus) + ret += cpu_map__fprintf(cpus, fp); + else + ret += fprintf(fp, "failed to get cpumap from event\n"); + + perf_cpu_map__put(cpus); + return ret; +} + +int perf_event__process_mmap(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_mmap_event(machine, event, sample); +} + +int perf_event__process_mmap2(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_mmap2_event(machine, event, sample); +} + +size_t perf_event__fprintf_task(union perf_event *event, FILE *fp) +{ + return fprintf(fp, "(%d:%d):(%d:%d)\n", + event->fork.pid, event->fork.tid, + event->fork.ppid, event->fork.ptid); +} + +int perf_event__process_fork(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_fork_event(machine, event, sample); +} + +int perf_event__process_exit(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_exit_event(machine, event, sample); +} + +int perf_event__exit_del_thread(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + struct thread *thread = machine__findnew_thread(machine, + event->fork.pid, + event->fork.tid); + + dump_printf("(%d:%d):(%d:%d)\n", event->fork.pid, event->fork.tid, + event->fork.ppid, event->fork.ptid); + + if (thread) { + machine__remove_thread(machine, thread); + thread__put(thread); + } + + return 0; +} + +size_t perf_event__fprintf_aux(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " offset: %#"PRI_lx64" size: %#"PRI_lx64" flags: %#"PRI_lx64" [%s%s%s%s]\n", + event->aux.aux_offset, event->aux.aux_size, + event->aux.flags, + event->aux.flags & PERF_AUX_FLAG_TRUNCATED ? "T" : "", + event->aux.flags & PERF_AUX_FLAG_OVERWRITE ? "O" : "", + event->aux.flags & PERF_AUX_FLAG_PARTIAL ? "P" : "", + event->aux.flags & PERF_AUX_FLAG_COLLISION ? "C" : ""); +} + +size_t perf_event__fprintf_itrace_start(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " pid: %u tid: %u\n", + event->itrace_start.pid, event->itrace_start.tid); +} + +size_t perf_event__fprintf_aux_output_hw_id(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " hw_id: %#"PRI_lx64"\n", + event->aux_output_hw_id.hw_id); +} + +size_t perf_event__fprintf_switch(union perf_event *event, FILE *fp) +{ + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + const char *in_out = !out ? "IN " : + !(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT_PREEMPT) ? + "OUT " : "OUT preempt"; + + if (event->header.type == PERF_RECORD_SWITCH) + return fprintf(fp, " %s\n", in_out); + + return fprintf(fp, " %s %s pid/tid: %5d/%-5d\n", + in_out, out ? "next" : "prev", + event->context_switch.next_prev_pid, + event->context_switch.next_prev_tid); +} + +static size_t perf_event__fprintf_lost(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " lost %" PRI_lu64 "\n", event->lost.lost); +} + +size_t perf_event__fprintf_ksymbol(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " addr %" PRI_lx64 " len %u type %u flags 0x%x name %s\n", + event->ksymbol.addr, event->ksymbol.len, + event->ksymbol.ksym_type, + event->ksymbol.flags, event->ksymbol.name); +} + +size_t perf_event__fprintf_bpf(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " type %u, flags %u, id %u\n", + event->bpf.type, event->bpf.flags, event->bpf.id); +} + +size_t perf_event__fprintf_bpf_metadata(union perf_event *event, FILE *fp) +{ + struct perf_record_bpf_metadata *metadata = &event->bpf_metadata; + size_t ret; + + ret = fprintf(fp, " prog %s\n", metadata->prog_name); + for (__u32 i = 0; i < metadata->nr_entries; i++) { + ret += fprintf(fp, " entry %d: %20s = %s\n", i, + metadata->entries[i].key, + metadata->entries[i].value); + } + return ret; +} + +static int text_poke_printer(enum binary_printer_ops op, unsigned int val, + void *extra, FILE *fp) +{ + bool old = *(bool *)extra; + + switch ((int)op) { + case BINARY_PRINT_LINE_BEGIN: + return fprintf(fp, " %s bytes:", old ? "Old" : "New"); + case BINARY_PRINT_NUM_DATA: + return fprintf(fp, " %02x", val); + case BINARY_PRINT_LINE_END: + return fprintf(fp, "\n"); + default: + return 0; + } +} + +size_t perf_event__fprintf_text_poke(union perf_event *event, struct machine *machine, FILE *fp) +{ + struct perf_record_text_poke_event *tp = &event->text_poke; + size_t ret; + bool old; + + ret = fprintf(fp, " %" PRI_lx64 " ", tp->addr); + if (machine) { + struct addr_location al; + + addr_location__init(&al); + al.map = maps__find(machine__kernel_maps(machine), tp->addr); + if (al.map && map__load(al.map) >= 0) { + al.addr = map__map_ip(al.map, tp->addr); + al.sym = map__find_symbol(al.map, al.addr); + if (al.sym) + ret += symbol__fprintf_symname_offs(al.sym, &al, fp); + } + addr_location__exit(&al); + } + ret += fprintf(fp, " old len %u new len %u\n", tp->old_len, tp->new_len); + old = true; + ret += binary__fprintf(tp->bytes, tp->old_len, 16, text_poke_printer, + &old, fp); + old = false; + ret += binary__fprintf(tp->bytes + tp->old_len, tp->new_len, 16, + text_poke_printer, &old, fp); + return ret; +} + +size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp) +{ + struct perf_record_schedstat_cpu *cs = &event->schedstat_cpu; + size_t size = fprintf(fp, "\ncpu%u ", cs->cpu); + __u16 version = cs->version; + +#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \ + size += fprintf(fp, "%" PRIu64 " ", (uint64_t)cs->_ver._name) + + if (version == 15) { +#include + return size; + } else if (version == 16) { +#include + return size; + } else if (version == 17) { +#include + return size; + } +#undef CPU_FIELD + + return fprintf(fp, "Unsupported /proc/schedstat version %d.\n", + event->schedstat_cpu.version); +} + +size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp) +{ + struct perf_record_schedstat_domain *ds = &event->schedstat_domain; + __u16 version = ds->version; + size_t size = fprintf(fp, "\ndomain%u ", ds->domain); + +#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \ + size += fprintf(fp, "%" PRIu64 " ", (uint64_t)ds->_ver._name) + + if (version == 15) { +#include + return size; + } else if (version == 16) { +#include + return size; + } else if (version == 17) { +#include + return size; + } +#undef DOMAIN_FIELD + + return fprintf(fp, "Unsupported /proc/schedstat version %d.\n", + event->schedstat_domain.version); +} + +size_t perf_event__fprintf(union perf_event *event, struct machine *machine, FILE *fp) +{ + size_t ret = fprintf(fp, "PERF_RECORD_%s", + perf_event__name(event->header.type)); + + switch (event->header.type) { + case PERF_RECORD_COMM: + ret += perf_event__fprintf_comm(event, fp); + break; + case PERF_RECORD_FORK: + case PERF_RECORD_EXIT: + ret += perf_event__fprintf_task(event, fp); + break; + case PERF_RECORD_MMAP: + ret += perf_event__fprintf_mmap(event, fp); + break; + case PERF_RECORD_NAMESPACES: + ret += perf_event__fprintf_namespaces(event, fp); + break; + case PERF_RECORD_CGROUP: + ret += perf_event__fprintf_cgroup(event, fp); + break; + case PERF_RECORD_MMAP2: + ret += perf_event__fprintf_mmap2(event, fp); + break; + case PERF_RECORD_AUX: + ret += perf_event__fprintf_aux(event, fp); + break; + case PERF_RECORD_ITRACE_START: + ret += perf_event__fprintf_itrace_start(event, fp); + break; + case PERF_RECORD_SWITCH: + case PERF_RECORD_SWITCH_CPU_WIDE: + ret += perf_event__fprintf_switch(event, fp); + break; + case PERF_RECORD_LOST: + ret += perf_event__fprintf_lost(event, fp); + break; + case PERF_RECORD_KSYMBOL: + ret += perf_event__fprintf_ksymbol(event, fp); + break; + case PERF_RECORD_BPF_EVENT: + ret += perf_event__fprintf_bpf(event, fp); + break; + case PERF_RECORD_TEXT_POKE: + ret += perf_event__fprintf_text_poke(event, machine, fp); + break; + case PERF_RECORD_AUX_OUTPUT_HW_ID: + ret += perf_event__fprintf_aux_output_hw_id(event, fp); + break; + case PERF_RECORD_BPF_METADATA: + ret += perf_event__fprintf_bpf_metadata(event, fp); + break; + default: + ret += fprintf(fp, "\n"); + } + + return ret; +} + +int perf_event__process(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_event(machine, event, sample); +} + +struct map *thread__find_map(struct thread *thread, u8 cpumode, u64 addr, + struct addr_location *al) +{ + struct maps *maps = thread__maps(thread); + struct machine *machine = maps__machine(maps); + bool load_map = false; + + map__zput(al->map); + thread__zput(al->thread); + al->thread = thread__get(thread); + + al->addr = addr; + al->cpumode = cpumode; + al->filtered = 0; + + if (machine == NULL) + return NULL; + + if (cpumode == PERF_RECORD_MISC_KERNEL && perf_host) { + al->level = 'k'; + maps = machine__kernel_maps(machine); + load_map = !symbol_conf.lazy_load_kernel_maps; + } else if (cpumode == PERF_RECORD_MISC_USER && perf_host) { + al->level = '.'; + } else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL && perf_guest) { + al->level = 'g'; + maps = machine__kernel_maps(machine); + load_map = !symbol_conf.lazy_load_kernel_maps; + } else if (cpumode == PERF_RECORD_MISC_GUEST_USER && perf_guest) { + al->level = 'u'; + } else { + al->level = 'H'; + + if ((cpumode == PERF_RECORD_MISC_GUEST_USER || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !perf_guest) + al->filtered |= (1 << HIST_FILTER__GUEST); + if ((cpumode == PERF_RECORD_MISC_USER || + cpumode == PERF_RECORD_MISC_KERNEL) && + !perf_host) + al->filtered |= (1 << HIST_FILTER__HOST); + + return NULL; + } + al->map = maps__find(maps, al->addr); + if (al->map != NULL) { + /* + * Kernel maps might be changed when loading symbols so loading + * must be done prior to using kernel maps. + */ + if (load_map) + map__load(al->map); + al->addr = map__map_ip(al->map, al->addr); + } + + return al->map; +} + +/* + * For branch stacks or branch samples, the sample cpumode might not be correct + * because it applies only to the sample 'ip' and not necessary to 'addr' or + * branch stack addresses. If possible, use a fallback to deal with those cases. + */ +struct map *thread__find_map_fb(struct thread *thread, u8 cpumode, u64 addr, + struct addr_location *al) +{ + struct map *map = thread__find_map(thread, cpumode, addr, al); + struct machine *machine = maps__machine(thread__maps(thread)); + u8 addr_cpumode = machine__addr_cpumode(machine, cpumode, addr); + + if (map || addr_cpumode == cpumode) + return map; + + return thread__find_map(thread, addr_cpumode, addr, al); +} + +struct symbol *thread__find_symbol(struct thread *thread, u8 cpumode, + u64 addr, struct addr_location *al) +{ + al->sym = NULL; + if (thread__find_map(thread, cpumode, addr, al)) + al->sym = map__find_symbol(al->map, al->addr); + return al->sym; +} + +struct symbol *thread__find_symbol_fb(struct thread *thread, u8 cpumode, + u64 addr, struct addr_location *al) +{ + al->sym = NULL; + if (thread__find_map_fb(thread, cpumode, addr, al)) + al->sym = map__find_symbol(al->map, al->addr); + return al->sym; +} + +static bool check_address_range(struct intlist *addr_list, int addr_range, + unsigned long addr) +{ + struct int_node *pos; + + intlist__for_each_entry(pos, addr_list) { + if (addr >= pos->i && addr < pos->i + addr_range) + return true; + } + + return false; +} + +/* + * Callers need to drop the reference to al->thread, obtained in + * machine__findnew_thread() + */ +int machine__resolve(struct machine *machine, struct addr_location *al, + struct perf_sample *sample) +{ + struct thread *thread; + struct dso *dso; + + if (symbol_conf.guest_code && !machine__is_host(machine)) + thread = machine__findnew_guest_code(machine, sample->pid); + else + thread = machine__findnew_thread(machine, sample->pid, sample->tid); + if (thread == NULL) + return -1; + + dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread__tid(thread)); + thread__find_map(thread, sample->cpumode, sample->ip, al); + dso = al->map ? map__dso(al->map) : NULL; + dump_printf(" ...... dso: %s\n", + dso + ? dso__long_name(dso) + : (al->level == 'H' ? "[hypervisor]" : "")); + + if (thread__is_filtered(thread)) + al->filtered |= (1 << HIST_FILTER__THREAD); + + thread__put(thread); + thread = NULL; + + al->sym = NULL; + al->cpu = sample->cpu; + al->socket = -1; + al->srcline = NULL; + + if (al->cpu >= 0) { + struct perf_env *env = machine->env; + + /* + * Bounds-check al->cpu (s32) before casting to struct perf_cpu + * (int16_t): without this, e.g. 65536 truncates to 0 and silently + * returns CPU 0's topology. Can go once perf_cpu.cpu is widened. + */ + if (env && al->cpu < env->nr_cpus_avail) { + struct cpu_topology_map *topo; + + topo = perf_env__get_cpu_topology(env, (struct perf_cpu){ al->cpu }); + if (topo) + al->socket = topo->socket_id; + } + } + + /* Account for possible out-of-order switch events. */ + al->parallelism = max(1, min(machine->parallelism, machine__nr_cpus_avail(machine))); + if (test_bit(al->parallelism, symbol_conf.parallelism_filter)) + al->filtered |= (1 << HIST_FILTER__PARALLELISM); + /* + * Multiply it by some const to avoid precision loss or dealing + * with floats. The multiplier does not matter otherwise since + * we only print it as percents. + */ + al->latency = sample->period * 1000 / al->parallelism; + + if (al->map) { + if (symbol_conf.dso_list && + (!dso || !(strlist__has_entry(symbol_conf.dso_list, + dso__short_name(dso)) || + (dso__short_name(dso) != dso__long_name(dso) && + strlist__has_entry(symbol_conf.dso_list, + dso__long_name(dso)))))) { + al->filtered |= (1 << HIST_FILTER__DSO); + } + + al->sym = map__find_symbol(al->map, al->addr); + } else if (symbol_conf.dso_list) { + al->filtered |= (1 << HIST_FILTER__DSO); + } + + if (symbol_conf.sym_list) { + int ret = 0; + char al_addr_str[32]; + size_t sz = sizeof(al_addr_str); + + if (al->sym) { + ret = strlist__has_entry(symbol_conf.sym_list, + al->sym->name); + } + if (!ret && al->sym) { + snprintf(al_addr_str, sz, "0x%"PRIx64, + map__unmap_ip(al->map, al->sym->start)); + ret = strlist__has_entry(symbol_conf.sym_list, + al_addr_str); + } + if (!ret && symbol_conf.addr_list && al->map) { + unsigned long addr = map__unmap_ip(al->map, al->addr); + + ret = intlist__has_entry(symbol_conf.addr_list, addr); + if (!ret && symbol_conf.addr_range) { + ret = check_address_range(symbol_conf.addr_list, + symbol_conf.addr_range, + addr); + } + } + + if (!ret) + al->filtered |= (1 << HIST_FILTER__SYMBOL); + } + + return 0; +} + +bool is_bts_event(struct perf_event_attr *attr) +{ + return attr->type == PERF_TYPE_HARDWARE && + (attr->config & PERF_COUNT_HW_BRANCH_INSTRUCTIONS) && + attr->sample_period == 1; +} + +bool sample_addr_correlates_sym(struct perf_event_attr *attr) +{ + if (attr->type == PERF_TYPE_SOFTWARE && + (attr->config == PERF_COUNT_SW_PAGE_FAULTS || + attr->config == PERF_COUNT_SW_PAGE_FAULTS_MIN || + attr->config == PERF_COUNT_SW_PAGE_FAULTS_MAJ)) + return true; + + if (is_bts_event(attr)) + return true; + + return false; +} + +void thread__resolve(struct thread *thread, struct addr_location *al, + struct perf_sample *sample) +{ + thread__find_map_fb(thread, sample->cpumode, sample->addr, al); + + al->cpu = sample->cpu; + al->sym = NULL; + + if (al->map) + al->sym = map__find_symbol(al->map, al->addr); +} diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h new file mode 100644 index 000000000..2ea83fdf8 --- /dev/null +++ b/tools/perf/util/event.h @@ -0,0 +1,437 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_RECORD_H +#define __PERF_RECORD_H +/* + * The linux/stddef.h isn't need here, but is needed for __always_inline used + * in files included from uapi/linux/perf_event.h such as + * /usr/include/linux/swab.h and /usr/include/linux/byteorder/little_endian.h, + * detected in at least musl libc, used in Alpine Linux. -acme + */ +#include +#include +#include +#include + +struct dso; +struct machine; +struct perf_event_attr; +struct perf_sample; + +#ifdef __LP64__ +/* + * /usr/include/inttypes.h uses just 'lu' for PRIu64, but we end up defining + * __u64 as long long unsigned int, and then -Werror=format= kicks in and + * complains of the mismatched types, so use these two special extra PRI + * macros to overcome that. + */ +#define PRI_lu64 "l" PRIu64 +#define PRI_lx64 "l" PRIx64 +#define PRI_ld64 "l" PRId64 +#else +#define PRI_lu64 PRIu64 +#define PRI_lx64 PRIx64 +#define PRI_ld64 PRId64 +#endif + +#define PERF_SAMPLE_MASK \ + (PERF_SAMPLE_IP | PERF_SAMPLE_TID | \ + PERF_SAMPLE_TIME | PERF_SAMPLE_ADDR | \ + PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD | \ + PERF_SAMPLE_IDENTIFIER) + +/* perf sample has 16 bits size limit */ +#define PERF_SAMPLE_MAX_SIZE (1 << 16) + +struct ip_callchain { + u64 nr; + u64 ips[]; +}; + +struct branch_stack; + +enum { + PERF_IP_FLAG_BRANCH = 1ULL << 0, + PERF_IP_FLAG_CALL = 1ULL << 1, + PERF_IP_FLAG_RETURN = 1ULL << 2, + PERF_IP_FLAG_CONDITIONAL = 1ULL << 3, + PERF_IP_FLAG_SYSCALLRET = 1ULL << 4, + PERF_IP_FLAG_ASYNC = 1ULL << 5, + PERF_IP_FLAG_INTERRUPT = 1ULL << 6, + PERF_IP_FLAG_TX_ABORT = 1ULL << 7, + PERF_IP_FLAG_TRACE_BEGIN = 1ULL << 8, + PERF_IP_FLAG_TRACE_END = 1ULL << 9, + PERF_IP_FLAG_IN_TX = 1ULL << 10, + PERF_IP_FLAG_VMENTRY = 1ULL << 11, + PERF_IP_FLAG_VMEXIT = 1ULL << 12, + PERF_IP_FLAG_INTR_DISABLE = 1ULL << 13, + PERF_IP_FLAG_INTR_TOGGLE = 1ULL << 14, + PERF_IP_FLAG_BRANCH_MISS = 1ULL << 15, + PERF_IP_FLAG_NOT_TAKEN = 1ULL << 16, +}; + +#define PERF_IP_FLAG_CHARS "bcrosyiABExghDtmn" + +#define PERF_ADDITIONAL_STATE_MASK \ + (PERF_IP_FLAG_IN_TX | \ + PERF_IP_FLAG_INTR_DISABLE | \ + PERF_IP_FLAG_INTR_TOGGLE) + +#define PERF_BRANCH_MASK (\ + PERF_IP_FLAG_BRANCH |\ + PERF_IP_FLAG_CALL |\ + PERF_IP_FLAG_RETURN |\ + PERF_IP_FLAG_CONDITIONAL |\ + PERF_IP_FLAG_SYSCALLRET |\ + PERF_IP_FLAG_ASYNC |\ + PERF_IP_FLAG_INTERRUPT |\ + PERF_IP_FLAG_TX_ABORT |\ + PERF_IP_FLAG_TRACE_BEGIN |\ + PERF_IP_FLAG_TRACE_END |\ + PERF_IP_FLAG_VMENTRY |\ + PERF_IP_FLAG_VMEXIT) + +#define PERF_IP_FLAG_BRANCH_EVENT_MASK \ + (PERF_IP_FLAG_BRANCH_MISS | \ + PERF_IP_FLAG_NOT_TAKEN) + +#define PERF_MEM_DATA_SRC_NONE \ + (PERF_MEM_S(OP, NA) |\ + PERF_MEM_S(LVL, NA) |\ + PERF_MEM_S(SNOOP, NA) |\ + PERF_MEM_S(LOCK, NA) |\ + PERF_MEM_S(TLB, NA) |\ + PERF_MEM_S(LVLNUM, NA)) + +/* Attribute type for custom synthesized events */ +#define PERF_TYPE_SYNTH (INT_MAX + 1U) + +/* Attribute config for custom synthesized events */ +enum perf_synth_id { + PERF_SYNTH_INTEL_PTWRITE, + PERF_SYNTH_INTEL_MWAIT, + PERF_SYNTH_INTEL_PWRE, + PERF_SYNTH_INTEL_EXSTOP, + PERF_SYNTH_INTEL_PWRX, + PERF_SYNTH_INTEL_CBR, + PERF_SYNTH_INTEL_PSB, + PERF_SYNTH_INTEL_EVT, + PERF_SYNTH_INTEL_IFLAG_CHG, + PERF_SYNTH_POWERPC_VPA_DTL, +}; + +/* + * Raw data formats for synthesized events. Note that 4 bytes of padding are + * present to match the 'size' member of PERF_SAMPLE_RAW data which is always + * 8-byte aligned. That means we must dereference raw_data with an offset of 4. + * Refer perf_sample__synth_ptr() and perf_synth__raw_data(). It also means the + * structure sizes are 4 bytes bigger than the raw_size, refer + * perf_synth__raw_size(). + */ + +struct perf_synth_intel_ptwrite { + u32 padding; + union { + struct { + u32 ip : 1, + reserved : 31; + }; + u32 flags; + }; + u64 payload; +}; + +struct perf_synth_intel_mwait { + u32 padding; + u32 reserved; + union { + struct { + u64 hints : 8, + reserved1 : 24, + extensions : 2, + reserved2 : 30; + }; + u64 payload; + }; +}; + +struct perf_synth_intel_pwre { + u32 padding; + u32 reserved; + union { + struct { + u64 reserved1 : 7, + hw : 1, + subcstate : 4, + cstate : 4, + reserved2 : 48; + }; + u64 payload; + }; +}; + +struct perf_synth_intel_exstop { + u32 padding; + union { + struct { + u32 ip : 1, + reserved : 31; + }; + u32 flags; + }; +}; + +struct perf_synth_intel_pwrx { + u32 padding; + u32 reserved; + union { + struct { + u64 deepest_cstate : 4, + last_cstate : 4, + wake_reason : 4, + reserved1 : 52; + }; + u64 payload; + }; +}; + +struct perf_synth_intel_cbr { + u32 padding; + union { + struct { + u32 cbr : 8, + reserved1 : 8, + max_nonturbo : 8, + reserved2 : 8; + }; + u32 flags; + }; + u32 freq; + u32 reserved3; +}; + +struct perf_synth_intel_psb { + u32 padding; + u32 reserved; + u64 offset; +}; + +struct perf_synth_intel_evd { + union { + struct { + u8 evd_type; + u8 reserved[7]; + }; + u64 et; + }; + u64 payload; +}; + +/* Intel PT Event Trace */ +struct perf_synth_intel_evt { + u32 padding; + union { + struct { + u32 type : 5, + reserved : 2, + ip : 1, + vector : 8, + evd_cnt : 16; + }; + u32 cfe; + }; + struct perf_synth_intel_evd evd[0]; +}; + +struct perf_synth_intel_iflag_chg { + u32 padding; + union { + struct { + u32 iflag : 1, + via_branch : 1; + }; + u32 flags; + }; + u64 branch_ip; /* If via_branch */ +}; + +/* + * The powerpc VPA DTL entries are of below format + */ +struct powerpc_vpadtl_entry { + u8 dispatch_reason; + u8 preempt_reason; + u16 processor_id; + u32 enqueue_to_dispatch_time; + u32 ready_to_enqueue_time; + u32 waiting_to_ready_time; + u64 timebase; + u64 fault_addr; + u64 srr0; + u64 srr1; +}; + +extern const char *dispatch_reasons[11]; +extern const char *preempt_reasons[10]; + +static inline void *perf_synth__raw_data(void *p) +{ + return p + 4; +} + +#define perf_synth__raw_size(d) (sizeof(d) - 4) + +#define perf_sample__bad_synth_size(s, d) ((s)->raw_size < sizeof(d) - 4) + +enum { + PERF_STAT_ROUND_TYPE__INTERVAL = 0, + PERF_STAT_ROUND_TYPE__FINAL = 1, +}; + +void perf_event__print_totals(void); + +struct perf_cpu_map; +struct perf_record_stat_config; +struct perf_stat_config; +struct perf_tool; + +void perf_event__read_stat_config(struct perf_stat_config *config, + struct perf_record_stat_config *event); + +int perf_event__process_comm(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_lost(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_lost_samples(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_aux(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_itrace_start(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_aux_output_hw_id(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_switch(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_namespaces(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_cgroup(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_mmap(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_mmap2(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_fork(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_exit(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__exit_del_thread(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_ksymbol(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_bpf(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_text_poke(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); + +bool is_bts_event(struct perf_event_attr *attr); +bool sample_addr_correlates_sym(struct perf_event_attr *attr); + +const char *perf_event__name(unsigned int id); + +size_t perf_event__fprintf_comm(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_mmap(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_mmap2(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_task(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_aux(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_itrace_start(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_aux_output_hw_id(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_switch(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_thread_map(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_cpu_map(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_namespaces(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_cgroup(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_ksymbol(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_bpf(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_bpf_metadata(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_text_poke(union perf_event *event, struct machine *machine,FILE *fp); +size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp); +size_t perf_event__fprintf(union perf_event *event, struct machine *machine, FILE *fp); + +int kallsyms__get_function_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr); +int kallsyms__get_symbol_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr); + +void event_attr_init(struct perf_event_attr *attr); + +int perf_event_paranoid(void); +bool perf_event_paranoid_check(int max_level); + +extern int sysctl_perf_event_max_stack; +extern int sysctl_perf_event_max_contexts_per_stack; +extern unsigned int proc_map_timeout; + +#define PAGE_SIZE_NAME_LEN 32 +char *get_page_size_name(u64 size, char *str); + +static inline bool perf_event_header__cpumode_is_guest(u8 cpumode) +{ + return cpumode == PERF_RECORD_MISC_GUEST_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_USER; +} + +static inline bool perf_event_header__misc_is_guest(u16 misc) +{ + return perf_event_header__cpumode_is_guest(misc & PERF_RECORD_MISC_CPUMODE_MASK); +} + +static inline bool perf_event_header__is_guest(const struct perf_event_header *header) +{ + return perf_event_header__misc_is_guest(header->misc); +} + +static inline bool perf_event__is_guest(const union perf_event *event) +{ + return perf_event_header__is_guest(&event->header); +} + +#endif /* __PERF_RECORD_H */ diff --git a/tools/perf/util/events_stats.h b/tools/perf/util/events_stats.h new file mode 100644 index 000000000..dcff697ed --- /dev/null +++ b/tools/perf/util/events_stats.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_EVENTS_STATS_ +#define __PERF_EVENTS_STATS_ + +#include +#include +#include +#include "auxtrace.h" + +/* + * The kernel collects the number of events it couldn't send in a stretch and + * when possible sends this number in a PERF_RECORD_LOST event. The number of + * such "chunks" of lost events is stored in .nr_events[PERF_EVENT_LOST] while + * total_lost tells exactly how many events the kernel in fact lost, i.e. it is + * the sum of all struct perf_record_lost.lost fields reported. + * + * The kernel discards mixed up samples and sends the number in a + * PERF_RECORD_LOST_SAMPLES event. The number of lost-samples events is stored + * in .nr_events[PERF_RECORD_LOST_SAMPLES] while total_lost_samples tells + * exactly how many samples the kernel in fact dropped, i.e. it is the sum of + * all struct perf_record_lost_samples.lost fields reported without setting the + * misc field in the header. + * + * The BPF program can discard samples according to the filter expressions given + * by the user. This number is kept in a BPF map and dumped at the end of perf + * record in a PERF_RECORD_LOST_SAMPLES event. To differentiate it from other + * lost samples, perf tools sets PERF_RECORD_MISC_LOST_SAMPLES_BPF flag in the + * header.misc field. The number of dropped-samples events is stored in + * .nr_events[PERF_RECORD_LOST_SAMPLES] while total_dropped_samples tells + * exactly how many samples the BPF program in fact dropped, i.e. it is the sum + * of all struct perf_record_lost_samples.lost fields reported with the misc + * field set in the header. + * + * The total_period is needed because by default auto-freq is used, so + * multiplying nr_events[PERF_EVENT_SAMPLE] by a frequency isn't possible to get + * the total number of low level events, it is necessary to sum all struct + * perf_record_sample.period and stash the result in total_period. + */ +struct events_stats { + u64 total_lost; + u64 total_lost_samples; + u64 total_dropped_samples; + u64 total_aux_lost; + u64 total_aux_partial; + u64 total_aux_collision; + u64 total_invalid_chains; + u32 nr_events[PERF_RECORD_HEADER_MAX]; + u32 nr_lost_warned; + u32 nr_unknown_events; + u32 nr_invalid_chains; + u32 nr_unknown_id; + u32 nr_unprocessable_samples; + u32 nr_auxtrace_errors[PERF_AUXTRACE_ERROR_MAX]; + u32 nr_proc_map_timeout; +}; + +struct hists_stats { + u64 total_period; + u64 total_non_filtered_period; + u64 total_latency; + u64 total_non_filtered_latency; + u32 nr_samples; + u32 nr_non_filtered_samples; + u32 nr_lost_samples; + u32 nr_dropped_samples; +}; + +void events_stats__inc(struct events_stats *stats, u32 type); + +size_t events_stats__fprintf(struct events_stats *stats, FILE *fp); + +#endif /* __PERF_EVENTS_STATS_ */ diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c new file mode 100644 index 000000000..c3d784727 --- /dev/null +++ b/tools/perf/util/evlist.c @@ -0,0 +1,2798 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo + * + * Parts came from builtin-{top,stat,record}.c, see those files for further + * copyright notes. + */ +#include "evlist.h" + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include // page_size +#include +#include +#include +#include +#include +#include +#include + +#include "../perf.h" +#include "affinity.h" +#include "asm/bug.h" +#include "bpf-event.h" +#include "bpf-filter.h" +#include "bpf_counter.h" +#include "cpumap.h" +#include "debug.h" +#include "dwarf-regs.h" +#include "env.h" +#include "event.h" +#include "evsel.h" +#include "evsel_fprintf.h" +#include "intel-tpebs.h" +#include "metricgroup.h" +#include "mmap.h" +#include "parse-events.h" +#include "perf_api_probe.h" +#include "pmu.h" +#include "pmus.h" +#include "record.h" +#include "sample.h" +#include "stat.h" +#include "strbuf.h" +#include "string2.h" +#include "target.h" +#include "thread_map.h" +#include "units.h" +#include "util.h" + +#ifdef LACKS_SIGQUEUE_PROTOTYPE +int sigqueue(pid_t pid, int sig, const union sigval value); +#endif + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) +#define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y) + +static void event_enable_timer__exit(struct event_enable_timer **ep); + +struct evlist *evlist__new(void) +{ + struct evlist *result; + RC_STRUCT(evlist) *evlist; + + evlist = zalloc(sizeof(*evlist)); + if (ADD_RC_CHK(result, evlist)) { + perf_evlist__init(evlist__core(result)); + perf_evlist__set_maps(evlist__core(result), /*cpus=*/NULL, /*threads=*/NULL); + evlist__set_workload_pid(result, -1); + evlist__set_bkw_mmap_state(result, BKW_MMAP_NOTREADY); + evlist__set_ctl_fd_fd(result, -1); + evlist__set_ctl_fd_ack(result, -1); + evlist__set_ctl_fd_pos(result, -1); + evlist__set_nr_br_cntr(result, -1); + metricgroup__rblist_init(evlist__metric_events(result)); + INIT_LIST_HEAD(&evlist->deferred_samples); + refcount_set(evlist__refcnt(result), 1); + } else { + free(evlist); + result = NULL; + } + return result; +} + +struct evlist *evlist__new_default(const struct target *target, bool sample_callchains) +{ + struct evlist *evlist = evlist__new(); + bool can_profile_kernel; + struct perf_pmu *pmu = NULL; + char buf[256]; + int err; + + if (!evlist) + return NULL; + + can_profile_kernel = perf_event_paranoid_check(1); + + if (EM_HOST == EM_S390 && sample_callchains) { + snprintf(buf, sizeof(buf), "software/%s/%s", + target__has_cpu(target) ? "cpu-clock" : "task-clock", + can_profile_kernel ? "P" : "Pu"); + err = parse_event(evlist, buf); + if (err) + goto out_err; + } else { + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + snprintf(buf, sizeof(buf), "%s/cycles/%s", pmu->name, + can_profile_kernel ? "P" : "Pu"); + err = parse_event(evlist, buf); + if (err) + goto out_err; + } + } + + /* If there is only 1 event a sample identifier isn't necessary. */ + if (evlist__nr_entries(evlist) > 1) { + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) + evsel__set_sample_id(evsel, /*can_sample_identifier=*/false); + } + + return evlist; +out_err: + evlist__put(evlist); + return NULL; +} + +struct evlist *evlist__new_dummy(void) +{ + struct evlist *evlist = evlist__new(); + + if (evlist && evlist__add_dummy(evlist)) { + evlist__put(evlist); + evlist = NULL; + } + + return evlist; +} + +struct evlist *evlist__get(struct evlist *evlist) +{ + struct evlist *result; + + if (RC_CHK_GET(result, evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return result; +} + +/** + * evlist__set_id_pos - set the positions of event ids. + * @evlist: selected event list + * + * Events with compatible sample types all have the same id_pos + * and is_pos. For convenience, put a copy on evlist. + */ +void evlist__set_id_pos(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist); + + RC_CHK_ACCESS(evlist)->id_pos = first->id_pos; + RC_CHK_ACCESS(evlist)->is_pos = first->is_pos; +} + +static void evlist__update_id_pos(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) + evsel__calc_id_pos(evsel); + + evlist__set_id_pos(evlist); +} + +static void evlist__purge(struct evlist *evlist) +{ + struct evsel *pos, *n; + + evlist__for_each_entry_safe(evlist, n, pos) { + list_del_init(&pos->core.node); + if (pos->evlist) { + if (!RC_CHK_EQUAL(pos->evlist, evlist)) { + evlist__put(pos->evlist); + } else { + refcount_dec_and_test(evlist__refcnt(pos->evlist)); + RC_CHK_PUT(pos->evlist); + } + } + pos->evlist = NULL; + evsel__put(pos); + } + + evlist__core(evlist)->nr_entries = 0; +} + +static void evlist__exit(struct evlist *evlist) +{ + metricgroup__rblist_exit(evlist__metric_events(evlist)); + event_enable_timer__exit(&RC_CHK_ACCESS(evlist)->eet); + free(evlist__mmap(evlist)); + free(evlist__overwrite_mmap(evlist)); + perf_evlist__exit(evlist__core(evlist)); +} + +void evlist__put(struct evlist *evlist) +{ + struct evsel *evsel; + unsigned int count, ref_cnt; + + if (evlist == NULL) + return; + + if (refcount_dec_and_test(evlist__refcnt(evlist))) + goto out_delete; + +retry: + count = refcount_read(evlist__refcnt(evlist)); + ref_cnt = count; + evlist__for_each_entry(evlist, evsel) { + if (RC_CHK_EQUAL(evsel->evlist, evlist) && count && + refcount_read(&evsel->refcnt) == 1) + count--; + } + if (refcount_read(evlist__refcnt(evlist)) != ref_cnt) + goto retry; + + if (count != 0) { + /* + * Not the last reference except for back references from + * evsels. + */ + RC_CHK_PUT(evlist); + return; + } +out_delete: + evlist__free_stats(evlist); + evlist__do_munmap(evlist); + evlist__close(evlist); + evlist__purge(evlist); + evlist__exit(evlist); + RC_CHK_FREE(evlist); +} + +void evlist__add(struct evlist *evlist, struct evsel *entry) +{ + perf_evlist__add(evlist__core(evlist), &entry->core); + evlist__put(entry->evlist); + entry->evlist = evlist__get(evlist); + entry->tracking = !entry->core.idx; + + if (evlist__nr_entries(evlist) == 1) + evlist__set_id_pos(evlist); +} + +void evlist__remove(struct evlist *evlist, struct evsel *evsel) +{ + perf_evlist__remove(evlist__core(evlist), &evsel->core); + evlist__put(evsel->evlist); + evsel->evlist = NULL; +} + +void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list) +{ + while (!list_empty(list)) { + struct evsel *evsel, *temp, *leader = NULL; + + __evlist__for_each_entry_safe(list, temp, evsel) { + list_del_init(&evsel->core.node); + evlist__add(evlist, evsel); + leader = evsel; + break; + } + + __evlist__for_each_entry_safe(list, temp, evsel) { + if (evsel__has_leader(evsel, leader)) { + list_del_init(&evsel->core.node); + evlist__add(evlist, evsel); + } + } + } +} + +int __evlist__set_tracepoints_handlers(struct evlist *evlist, + const struct evsel_str_handler *assocs, size_t nr_assocs) +{ + size_t i; + int err; + + for (i = 0; i < nr_assocs; i++) { + // Adding a handler for an event not in this evlist, just ignore it. + struct evsel *evsel = evlist__find_tracepoint_by_name(evlist, assocs[i].name); + if (evsel == NULL) + continue; + + err = -EEXIST; + if (evsel->handler != NULL) + goto out; + evsel->handler = assocs[i].handler; + } + + err = 0; +out: + return err; +} + +static void evlist__set_leader(struct evlist *evlist) +{ + perf_evlist__set_leader(evlist__core(evlist)); +} + +static struct evsel *evlist__dummy_event(struct evlist *evlist) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_DUMMY, + .size = sizeof(attr), /* to capture ABI version */ + /* Avoid frequency mode for dummy events to avoid associated timers. */ + .freq = 0, + .sample_period = 1, + }; + + return evsel__new_idx(&attr, evlist__nr_entries(evlist)); +} + +int evlist__add_dummy(struct evlist *evlist) +{ + struct evsel *evsel = evlist__dummy_event(evlist); + + if (evsel == NULL) + return -ENOMEM; + + evlist__add(evlist, evsel); + return 0; +} + +struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel = evlist__dummy_event(evlist); + + if (!evsel) + return NULL; + + evsel->core.attr.exclude_kernel = 1; + evsel->core.attr.exclude_guest = 1; + evsel->core.attr.exclude_hv = 1; + evsel->core.system_wide = system_wide; + evsel->no_aux_samples = true; + evsel->name = strdup("dummy:u"); + + evlist__add(evlist, evsel); + return evsel; +} + +#ifdef HAVE_LIBTRACEEVENT +struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel = evsel__newtp_idx("sched", "sched_switch", 0, + /*format=*/true); + + if (IS_ERR(evsel)) + return evsel; + + evsel__set_sample_bit(evsel, CPU); + evsel__set_sample_bit(evsel, TIME); + + evsel->core.system_wide = system_wide; + evsel->no_aux_samples = true; + + evlist__add(evlist, evsel); + return evsel; +} +#endif + +struct evsel *evlist__find_tracepoint_by_name(struct evlist *evlist, const char *name) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if ((evsel->core.attr.type == PERF_TYPE_TRACEPOINT) && + (strcmp(evsel->name, name) == 0)) + return evsel; + } + + return NULL; +} + +#ifdef HAVE_LIBTRACEEVENT +int evlist__add_newtp(struct evlist *evlist, const char *sys, const char *name, void *handler) +{ + struct evsel *evsel = evsel__newtp(sys, name); + + if (IS_ERR(evsel)) + return -1; + + evsel->handler = handler; + evlist__add(evlist, evsel); + return 0; +} +#endif + +/* + * Should sched_setaffinity be used with evlist__for_each_cpu? Determine if + * migrating the thread will avoid possibly numerous IPIs. + */ +static bool evlist__use_affinity(struct evlist *evlist) +{ + struct evsel *pos; + struct perf_cpu_map *used_cpus = NULL; + bool ret = false; + + if (evlist__no_affinity(evlist) || !evlist__core(evlist)->user_requested_cpus || + cpu_map__is_dummy(evlist__core(evlist)->user_requested_cpus)) + return false; + + evlist__for_each_entry(evlist, pos) { + struct perf_cpu_map *intersect; + + if (!perf_pmu__benefits_from_affinity(pos->pmu)) + continue; + + if (evsel__is_dummy_event(pos)) { + /* + * The dummy event is opened on all CPUs so assume >1 + * event with shared CPUs. + */ + ret = true; + break; + } + if (evsel__is_retire_lat(pos)) { + /* + * Retirement latency events are similar to tool ones in + * their implementation, and so don't require affinity. + */ + continue; + } + if (perf_cpu_map__is_empty(used_cpus)) { + /* First benefitting event, we want >1 on a common CPU. */ + used_cpus = perf_cpu_map__get(pos->core.cpus); + continue; + } + if ((pos->core.attr.read_format & PERF_FORMAT_GROUP) && + evsel__leader(pos) != pos) { + /* Skip members of the same sample group. */ + continue; + } + intersect = perf_cpu_map__intersect(used_cpus, pos->core.cpus); + if (!perf_cpu_map__is_empty(intersect)) { + /* >1 event with shared CPUs. */ + perf_cpu_map__put(intersect); + ret = true; + break; + } + perf_cpu_map__put(intersect); + perf_cpu_map__merge(&used_cpus, pos->core.cpus); + } + perf_cpu_map__put(used_cpus); + return ret; +} + +void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *evlist) +{ + *itr = (struct evlist_cpu_iterator){ + .container = evlist, + .evsel = NULL, + .cpu_map_idx = 0, + .evlist_cpu_map_idx = 0, + .evlist_cpu_map_nr = perf_cpu_map__nr(evlist__core(evlist)->all_cpus), + .cpu = (struct perf_cpu){ .cpu = -1}, + .affinity = NULL, + }; + + if (evlist__empty(evlist)) { + /* Ensure the empty list doesn't iterate. */ + itr->evlist_cpu_map_idx = itr->evlist_cpu_map_nr; + return; + } + + if (evlist__use_affinity(evlist)) { + if (affinity__setup(&itr->saved_affinity) == 0) + itr->affinity = &itr->saved_affinity; + } + itr->evsel = evlist__first(evlist); + itr->cpu = perf_cpu_map__cpu(evlist__core(evlist)->all_cpus, 0); + if (itr->affinity) + affinity__set(itr->affinity, itr->cpu.cpu); + itr->cpu_map_idx = perf_cpu_map__idx(itr->evsel->core.cpus, itr->cpu); + /* + * If this CPU isn't in the evsel's cpu map then advance + * through the list. + */ + if (itr->cpu_map_idx == -1) + evlist_cpu_iterator__next(itr); +} + +void evlist_cpu_iterator__exit(struct evlist_cpu_iterator *itr) +{ + if (!itr->affinity) + return; + + affinity__cleanup(itr->affinity); + itr->affinity = NULL; +} + +void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr) +{ + while (evlist_cpu_itr->evsel != evlist__last(evlist_cpu_itr->container)) { + evlist_cpu_itr->evsel = evsel__next(evlist_cpu_itr->evsel); + evlist_cpu_itr->cpu_map_idx = + perf_cpu_map__idx(evlist_cpu_itr->evsel->core.cpus, + evlist_cpu_itr->cpu); + if (evlist_cpu_itr->cpu_map_idx != -1) + return; + } + evlist_cpu_itr->evlist_cpu_map_idx++; + if (evlist_cpu_itr->evlist_cpu_map_idx < evlist_cpu_itr->evlist_cpu_map_nr) { + evlist_cpu_itr->evsel = evlist__first(evlist_cpu_itr->container); + evlist_cpu_itr->cpu = + perf_cpu_map__cpu(evlist__core(evlist_cpu_itr->container)->all_cpus, + evlist_cpu_itr->evlist_cpu_map_idx); + if (evlist_cpu_itr->affinity) + affinity__set(evlist_cpu_itr->affinity, evlist_cpu_itr->cpu.cpu); + evlist_cpu_itr->cpu_map_idx = + perf_cpu_map__idx(evlist_cpu_itr->evsel->core.cpus, + evlist_cpu_itr->cpu); + /* + * If this CPU isn't in the evsel's cpu map then advance through + * the list. + */ + if (evlist_cpu_itr->cpu_map_idx == -1) + evlist_cpu_iterator__next(evlist_cpu_itr); + } else { + evlist_cpu_iterator__exit(evlist_cpu_itr); + } +} + +static int evsel__strcmp(struct evsel *pos, char *evsel_name) +{ + if (!evsel_name) + return 0; + if (evsel__is_dummy_event(pos)) + return 1; + return !evsel__name_is(pos, evsel_name); +} + +static bool evlist__is_enabled(struct evlist *evlist) +{ + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) { + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + /* If at least one event is enabled, evlist is enabled. */ + if (!pos->disabled) + return true; + } + return false; +} + +static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl_dummy) +{ + struct evsel *pos, *member; + struct evlist_cpu_iterator evlist_cpu_itr; + bool has_imm = false; + + /* Disable 'immediate' events last */ + for (int imm = 0; imm <= 1; imm++) { + evlist__for_each_cpu(evlist_cpu_itr, evlist) { + pos = evlist_cpu_itr.evsel; + if (evsel__strcmp(pos, evsel_name)) + continue; + if (pos->disabled || !evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + if (pos->immediate) + has_imm = true; + if (pos->immediate != imm) + continue; + evsel__disable_cpu(pos, evlist_cpu_itr.cpu_map_idx); + } + if (!has_imm) + break; + } + + evlist__for_each_entry(evlist, pos) { + if (evsel__strcmp(pos, evsel_name)) + continue; + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + pos->disabled = true; + + for_each_group_member(member, pos) + member->disabled = true; + } + + /* + * If we disabled only single event, we need to check + * the enabled state of the evlist manually. + */ + evlist__set_enabled(evlist, evsel_name ? evlist__is_enabled(evlist) : false); +} + +void evlist__disable(struct evlist *evlist) +{ + __evlist__disable(evlist, NULL, false); +} + +void evlist__disable_non_dummy(struct evlist *evlist) +{ + __evlist__disable(evlist, NULL, true); +} + +void evlist__disable_evsel(struct evlist *evlist, char *evsel_name) +{ + __evlist__disable(evlist, evsel_name, false); +} + +static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_dummy) +{ + struct evsel *pos, *member; + struct evlist_cpu_iterator evlist_cpu_itr; + + evlist__for_each_cpu(evlist_cpu_itr, evlist) { + pos = evlist_cpu_itr.evsel; + if (evsel__strcmp(pos, evsel_name)) + continue; + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + evsel__enable_cpu(pos, evlist_cpu_itr.cpu_map_idx); + } + evlist__for_each_entry(evlist, pos) { + if (evsel__strcmp(pos, evsel_name)) + continue; + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + pos->disabled = false; + + for_each_group_member(member, pos) + member->disabled = false; + } + + /* + * Even single event sets the 'enabled' for evlist, + * so the toggle can work properly and toggle to + * 'disabled' state. + */ + evlist__set_enabled(evlist, true); +} + +void evlist__enable(struct evlist *evlist) +{ + __evlist__enable(evlist, NULL, false); +} + +void evlist__enable_non_dummy(struct evlist *evlist) +{ + __evlist__enable(evlist, NULL, true); +} + +void evlist__enable_evsel(struct evlist *evlist, char *evsel_name) +{ + __evlist__enable(evlist, evsel_name, false); +} + +void evlist__toggle_enable(struct evlist *evlist) +{ + (evlist__enabled(evlist) ? evlist__disable : evlist__enable)(evlist); +} + +int evlist__add_pollfd(struct evlist *evlist, int fd) +{ + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__default); +} + +int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask) +{ + return perf_evlist__filter_pollfd(evlist__core(evlist), revents_and_mask); +} + +#ifdef HAVE_EVENTFD_SUPPORT +int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) +{ + return perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event); +} +#endif + +int evlist__poll(struct evlist *evlist, int timeout) +{ + return perf_evlist__poll(evlist__core(evlist), timeout); +} + +struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) +{ + struct hlist_head *head; + struct perf_sample_id *sid; + int hash; + + hash = hash_64(id, PERF_EVLIST__HLIST_BITS); + head = &evlist__core(evlist)->heads[hash]; + + hlist_for_each_entry(sid, head, node) + if (sid->id == id) + return sid; + + return NULL; +} + +struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id) +{ + struct perf_sample_id *sid; + + if (evlist__nr_entries(evlist) == 1 || !id) + return evlist__first(evlist); + + sid = evlist__id2sid(evlist, id); + if (sid) + return container_of(sid->evsel, struct evsel, core); + + if (!evlist__sample_id_all(evlist)) + return evlist__first(evlist); + + return NULL; +} + +struct evsel *evlist__id2evsel_strict(struct evlist *evlist, u64 id) +{ + struct perf_sample_id *sid; + + if (!id) + return NULL; + + sid = evlist__id2sid(evlist, id); + if (sid) + return container_of(sid->evsel, struct evsel, core); + + return NULL; +} + +static int evlist__event2id(struct evlist *evlist, union perf_event *event, u64 *id) +{ + const __u64 *array = event->sample.array; + ssize_t n; + + n = (event->header.size - sizeof(event->header)) >> 3; + + if (event->header.type == PERF_RECORD_SAMPLE) { + if (evlist__id_pos(evlist) >= n) + return -1; + *id = array[evlist__id_pos(evlist)]; + } else { + if (evlist__is_pos(evlist) > n) + return -1; + n -= evlist__is_pos(evlist); + *id = array[n]; + } + return 0; +} + +struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event) +{ + struct evsel *first = evlist__first(evlist); + struct hlist_head *head; + struct perf_sample_id *sid; + int hash; + u64 id; + + if (evlist__nr_entries(evlist) == 1) + return first; + + if (!first->core.attr.sample_id_all && + event->header.type != PERF_RECORD_SAMPLE) + return first; + + if (evlist__event2id(evlist, event, &id)) + return NULL; + + /* Synthesized events have an id of zero */ + if (!id) + return first; + + hash = hash_64(id, PERF_EVLIST__HLIST_BITS); + head = &evlist__core(evlist)->heads[hash]; + + hlist_for_each_entry(sid, head, node) { + if (sid->id == id) + return container_of(sid->evsel, struct evsel, core); + } + return NULL; +} + +static int evlist__set_paused(struct evlist *evlist, bool value) +{ + int i; + + if (!evlist__overwrite_mmap(evlist)) + return 0; + + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + int fd = evlist__overwrite_mmap(evlist)[i].core.fd; + int err; + + if (fd < 0) + continue; + err = ioctl(fd, PERF_EVENT_IOC_PAUSE_OUTPUT, value ? 1 : 0); + if (err) + return err; + } + return 0; +} + +static int evlist__pause(struct evlist *evlist) +{ + return evlist__set_paused(evlist, true); +} + +static int evlist__resume(struct evlist *evlist) +{ + return evlist__set_paused(evlist, false); +} + +static void evlist__munmap_nofree(struct evlist *evlist) +{ + int i; + + if (evlist__mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__mmap(evlist)[i].core); + + if (evlist__overwrite_mmap(evlist)) + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) + perf_mmap__munmap(&evlist__overwrite_mmap(evlist)[i].core); +} + +void evlist__do_munmap(struct evlist *evlist) +{ + evlist__munmap_nofree(evlist); + zfree(&RC_CHK_ACCESS(evlist)->mmap); + zfree(&RC_CHK_ACCESS(evlist)->overwrite_mmap); +} + +static void perf_mmap__unmap_cb(struct perf_mmap *map) +{ + struct mmap *m = container_of(map, struct mmap, core); + + mmap__munmap(m); +} + +static struct mmap *evlist__alloc_mmap(struct evlist *evlist, + bool overwrite) +{ + int i; + struct mmap *map = calloc(evlist__core(evlist)->nr_mmaps, sizeof(struct mmap)); + + if (!map) + return NULL; + + for (i = 0; i < evlist__core(evlist)->nr_mmaps; i++) { + struct perf_mmap *prev = i ? &map[i - 1].core : NULL; + + /* + * When the perf_mmap() call is made we grab one refcount, plus + * one extra to let perf_mmap__consume() get the last + * events after all real references (perf_mmap__get()) are + * dropped. + * + * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and + * thus does perf_mmap__get() on it. + */ + perf_mmap__init(&map[i].core, prev, overwrite, perf_mmap__unmap_cb); + } + + return map; +} + +static struct evlist *from_list_start(struct perf_evlist *core) +{ +#ifdef REFCNT_CHECKING + RC_STRUCT(evlist) *core_evlist = container_of(core, RC_STRUCT(evlist), core); + struct evlist *evlist; + + if (ADD_RC_CHK(evlist, core_evlist)) + refcount_inc(evlist__refcnt(evlist)); + + return evlist; +#else + return container_of(core, struct evlist, core); +#endif +} + +static void from_list_end(struct evlist *evlist __maybe_unused) +{ +#ifdef REFCNT_CHECKING + evlist__put(evlist); +#endif +} + +static void +perf_evlist__mmap_cb_idx(struct perf_evlist *_evlist, + struct perf_evsel *_evsel, + struct perf_mmap_param *_mp, + int idx) +{ + struct evlist *evlist = from_list_start(_evlist); + struct mmap_params *mp = container_of(_mp, struct mmap_params, core); + struct evsel *evsel = container_of(_evsel, struct evsel, core); + + if (!evlist) + return; + + auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, evsel, idx); + + from_list_end(evlist); +} + +static struct perf_mmap* +perf_evlist__mmap_cb_get(struct perf_evlist *_evlist, bool overwrite, int idx) +{ + struct evlist *evlist = from_list_start(_evlist); + struct mmap *maps; + + if (!evlist) + return NULL; + + maps = overwrite ? evlist__overwrite_mmap(evlist) : evlist__mmap(evlist); + + if (!maps) { + maps = evlist__alloc_mmap(evlist, overwrite); + if (!maps) { + from_list_end(evlist); + return NULL; + } + + if (overwrite) { + RC_CHK_ACCESS(evlist)->overwrite_mmap = maps; + if (evlist__bkw_mmap_state(evlist) == BKW_MMAP_NOTREADY) + evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING); + } else { + RC_CHK_ACCESS(evlist)->mmap = maps; + } + } + from_list_end(evlist); + return &maps[idx].core; +} + +static int +perf_evlist__mmap_cb_mmap(struct perf_mmap *_map, struct perf_mmap_param *_mp, + int output, struct perf_cpu cpu) +{ + struct mmap *map = container_of(_map, struct mmap, core); + struct mmap_params *mp = container_of(_mp, struct mmap_params, core); + + return mmap__mmap(map, mp, output, cpu); +} + +unsigned long perf_event_mlock_kb_in_pages(void) +{ + unsigned long pages; + int max; + + if (sysctl__read_int("kernel/perf_event_mlock_kb", &max) < 0) { + /* + * Pick a once upon a time good value, i.e. things look + * strange since we can't read a sysctl value, but lets not + * die yet... + */ + max = 512; + } else { + max -= (page_size / 1024); + } + + pages = (max * 1024) / page_size; + if (!is_power_of_2(pages)) + pages = rounddown_pow_of_two(pages); + + return pages; +} + +size_t evlist__mmap_size(unsigned long pages) +{ + if (pages == UINT_MAX) + pages = perf_event_mlock_kb_in_pages(); + else if (!is_power_of_2(pages)) + return 0; + + return (pages + 1) * page_size; +} + +static long parse_pages_arg(const char *str, unsigned long min, + unsigned long max) +{ + unsigned long pages, val; + static struct parse_tag tags[] = { + { .tag = 'B', .mult = 1 }, + { .tag = 'K', .mult = 1 << 10 }, + { .tag = 'M', .mult = 1 << 20 }, + { .tag = 'G', .mult = 1 << 30 }, + { .tag = 0 }, + }; + + if (str == NULL) + return -EINVAL; + + val = parse_tag_value(str, tags); + if (val != (unsigned long) -1) { + /* we got file size value */ + pages = PERF_ALIGN(val, page_size) / page_size; + } else { + /* we got pages count value */ + char *eptr; + pages = strtoul(str, &eptr, 10); + if (*eptr != '\0') + return -EINVAL; + } + + if (pages == 0 && min == 0) { + /* leave number of pages at 0 */ + } else if (!is_power_of_2(pages)) { + char buf[100]; + + /* round pages up to next power of 2 */ + pages = roundup_pow_of_two(pages); + if (!pages) + return -EINVAL; + + unit_number__scnprintf(buf, sizeof(buf), pages * page_size); + pr_info("rounding mmap pages size to %s (%lu pages)\n", + buf, pages); + } + + if (pages > max) + return -EINVAL; + + return pages; +} + +int __evlist__parse_mmap_pages(unsigned int *mmap_pages, const char *str) +{ + unsigned long max = UINT_MAX; + long pages; + + if (max > SIZE_MAX / page_size) + max = SIZE_MAX / page_size; + + pages = parse_pages_arg(str, 1, max); + if (pages < 0) { + pr_err("Invalid argument for --mmap_pages/-m\n"); + return -1; + } + + *mmap_pages = pages; + return 0; +} + +int evlist__parse_mmap_pages(const struct option *opt, const char *str, int unset __maybe_unused) +{ + return __evlist__parse_mmap_pages(opt->value, str); +} + +/** + * evlist__mmap_ex - Create mmaps to receive events. + * @evlist: list of events + * @pages: map length in pages + * @overwrite: overwrite older events? + * @auxtrace_pages - auxtrace map length in pages + * @auxtrace_overwrite - overwrite older auxtrace data? + * + * If @overwrite is %false the user needs to signal event consumption using + * perf_mmap__write_tail(). Using evlist__mmap_read() does this + * automatically. + * + * Similarly, if @auxtrace_overwrite is %false the user needs to signal data + * consumption using auxtrace_mmap__write_tail(). + * + * Return: %0 on success, negative error code otherwise. + */ +int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, + unsigned int auxtrace_pages, + bool auxtrace_overwrite, int nr_cblocks, int affinity, int flush, + int comp_level) +{ + /* + * Delay setting mp.prot: set it before calling perf_mmap__mmap. + * Its value is decided by evsel's write_backward. + * So &mp should not be passed through const pointer. + */ + struct mmap_params mp = { + .nr_cblocks = nr_cblocks, + .affinity = affinity, + .flush = flush, + .comp_level = comp_level + }; + struct perf_evlist_mmap_ops ops = { + .idx = perf_evlist__mmap_cb_idx, + .get = perf_evlist__mmap_cb_get, + .mmap = perf_evlist__mmap_cb_mmap, + }; + + evlist__core(evlist)->mmap_len = evlist__mmap_size(pages); + pr_debug("mmap size %zuB\n", evlist__core(evlist)->mmap_len); + + auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist__core(evlist)->mmap_len, + auxtrace_pages, auxtrace_overwrite); + + return perf_evlist__mmap_ops(evlist__core(evlist), &ops, &mp.core); +} + +int evlist__do_mmap(struct evlist *evlist, unsigned int pages) +{ + return evlist__mmap_ex(evlist, pages, 0, false, 0, PERF_AFFINITY_SYS, 1, 0); +} + +int evlist__create_maps(struct evlist *evlist, struct target *target) +{ + bool all_threads = (target->per_thread && target->system_wide); + struct perf_cpu_map *cpus; + struct perf_thread_map *threads; + + /* + * If specify '-a' and '--per-thread' to perf record, perf record + * will override '--per-thread'. target->per_thread = false and + * target->system_wide = true. + * + * If specify '--per-thread' only to perf record, + * target->per_thread = true and target->system_wide = false. + * + * So target->per_thread && target->system_wide is false. + * For perf record, thread_map__new_str doesn't call + * thread_map__new_all_cpus. That will keep perf record's + * current behavior. + * + * For perf stat, it allows the case that target->per_thread and + * target->system_wide are all true. It means to collect system-wide + * per-thread data. thread_map__new_str will call + * thread_map__new_all_cpus to enumerate all threads. + */ + threads = thread_map__new_str(target->pid, target->tid, all_threads); + + if (!threads) + return -1; + + if (target__uses_dummy_map(target) && !evlist__has_bpf_output(evlist)) + cpus = perf_cpu_map__new_any_cpu(); + else + cpus = perf_cpu_map__new(target->cpu_list); + + if (!cpus) + goto out_delete_threads; + + evlist__core(evlist)->has_user_cpus = !!target->cpu_list; + + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + + /* as evlist now has references, put count here */ + perf_cpu_map__put(cpus); + perf_thread_map__put(threads); + + return 0; + +out_delete_threads: + perf_thread_map__put(threads); + return -1; +} + +int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, + struct target *target) +{ + struct evsel *evsel; + int err = 0; + + evlist__for_each_entry(evlist, evsel) { + /* + * filters only work for tracepoint event, which doesn't have cpu limit. + * So evlist and evsel should always be same. + */ + if (evsel->filter) { + err = perf_evsel__apply_filter(&evsel->core, evsel->filter); + if (err) { + *err_evsel = evsel; + break; + } + } + + /* + * non-tracepoint events can have BPF filters. + */ + if (!list_empty(&evsel->bpf_filters)) { + err = perf_bpf_filter__prepare(evsel, target); + if (err) { + *err_evsel = evsel; + break; + } + } + } + + return err; +} + +int evlist__set_tp_filter(struct evlist *evlist, const char *filter) +{ + struct evsel *evsel; + int err = 0; + + if (filter == NULL) + return -1; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) + continue; + + err = evsel__set_filter(evsel, filter); + if (err) + break; + } + + return err; +} + +int evlist__append_tp_filter(struct evlist *evlist, const char *filter) +{ + struct evsel *evsel; + int err = 0; + + if (filter == NULL) + return -1; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) + continue; + + err = evsel__append_tp_filter(evsel, filter); + if (err) + break; + } + + return err; +} + +char *asprintf__tp_filter_pids(size_t npids, pid_t *pids) +{ + char *filter; + size_t i; + + for (i = 0; i < npids; ++i) { + if (i == 0) { + if (asprintf(&filter, "common_pid != %d", pids[i]) < 0) + return NULL; + } else { + char *tmp; + + if (asprintf(&tmp, "%s && common_pid != %d", filter, pids[i]) < 0) + goto out_free; + + free(filter); + filter = tmp; + } + } + + return filter; +out_free: + free(filter); + return NULL; +} + +int evlist__set_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids) +{ + char *filter = asprintf__tp_filter_pids(npids, pids); + int ret = evlist__set_tp_filter(evlist, filter); + + free(filter); + return ret; +} + +int evlist__append_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids) +{ + char *filter = asprintf__tp_filter_pids(npids, pids); + int ret = evlist__append_tp_filter(evlist, filter); + + free(filter); + return ret; +} + +int evlist__append_tp_filter_pid(struct evlist *evlist, pid_t pid) +{ + return evlist__append_tp_filter_pids(evlist, 1, &pid); +} + +bool evlist__valid_sample_type(struct evlist *evlist) +{ + struct evsel *pos; + + if (evlist__nr_entries(evlist) == 1) + return true; + + if (evlist__id_pos(evlist) < 0 || evlist__is_pos(evlist) < 0) + return false; + + evlist__for_each_entry(evlist, pos) { + if (pos->id_pos != evlist__id_pos(evlist) || + pos->is_pos != evlist__is_pos(evlist)) + return false; + } + + return true; +} + +u64 __evlist__combined_sample_type(struct evlist *evlist) +{ + struct evsel *evsel; + + if (RC_CHK_ACCESS(evlist)->combined_sample_type) + return RC_CHK_ACCESS(evlist)->combined_sample_type; + + evlist__for_each_entry(evlist, evsel) + RC_CHK_ACCESS(evlist)->combined_sample_type |= evsel->core.attr.sample_type; + + return RC_CHK_ACCESS(evlist)->combined_sample_type; +} + +u64 evlist__combined_sample_type(struct evlist *evlist) +{ + RC_CHK_ACCESS(evlist)->combined_sample_type = 0; + return __evlist__combined_sample_type(evlist); +} + +u64 evlist__combined_branch_type(struct evlist *evlist) +{ + struct evsel *evsel; + u64 branch_type = 0; + + evlist__for_each_entry(evlist, evsel) + branch_type |= evsel->core.attr.branch_sample_type; + return branch_type; +} + +static struct evsel * +evlist__find_dup_event_from_prev(struct evlist *evlist, struct evsel *event) +{ + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) { + if (event == pos) + break; + if ((pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + !strcmp(pos->name, event->name)) + return pos; + } + return NULL; +} + +#define MAX_NR_ABBR_NAME (26 * 11) + +/* + * The abbr name is from A to Z9. If the number of event + * which requires the branch counter > MAX_NR_ABBR_NAME, + * return NA. + */ +static void evlist__new_abbr_name(char *name) +{ + static int idx; + int i = idx / 26; + + if (idx >= MAX_NR_ABBR_NAME) { + name[0] = 'N'; + name[1] = 'A'; + name[2] = '\0'; + return; + } + + name[0] = 'A' + (idx % 26); + + if (!i) + name[1] = '\0'; + else { + name[1] = '0' + i - 1; + name[2] = '\0'; + } + + idx++; +} + +void evlist__update_br_cntr(struct evlist *evlist) +{ + struct evsel *evsel, *dup; + int i = 0; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) { + evsel->br_cntr_idx = i++; + evsel__leader(evsel)->br_cntr_nr++; + + dup = evlist__find_dup_event_from_prev(evlist, evsel); + if (dup) + memcpy(evsel->abbr_name, dup->abbr_name, 3 * sizeof(char)); + else + evlist__new_abbr_name(evsel->abbr_name); + } + } + evlist__set_nr_br_cntr(evlist, i); +} + +bool evlist__valid_read_format(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist), *pos = first; + u64 read_format = first->core.attr.read_format; + u64 sample_type = first->core.attr.sample_type; + + evlist__for_each_entry(evlist, pos) { + if (read_format != pos->core.attr.read_format) { + pr_debug("Read format differs %#" PRIx64 " vs %#" PRIx64 "\n", + read_format, (u64)pos->core.attr.read_format); + } + } + + /* PERF_SAMPLE_READ implies PERF_FORMAT_ID. */ + if ((sample_type & PERF_SAMPLE_READ) && + !(read_format & PERF_FORMAT_ID)) { + return false; + } + + return true; +} + +u16 evlist__id_hdr_size(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist); + + return first->core.attr.sample_id_all ? evsel__id_hdr_size(first) : 0; +} + +bool evlist__valid_sample_id_all(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist), *pos = first; + + evlist__for_each_entry_continue(evlist, pos) { + if (first->core.attr.sample_id_all != pos->core.attr.sample_id_all) + return false; + } + + return true; +} + +bool evlist__sample_id_all(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist); + return first->core.attr.sample_id_all; +} + +void evlist__close(struct evlist *evlist) +{ + struct evsel *evsel; + struct evlist_cpu_iterator evlist_cpu_itr; + + evlist__for_each_cpu(evlist_cpu_itr, evlist) { + if (evlist_cpu_itr.cpu_map_idx == 0 && evsel__is_retire_lat(evlist_cpu_itr.evsel)) + evsel__tpebs_close(evlist_cpu_itr.evsel); + perf_evsel__close_cpu(&evlist_cpu_itr.evsel->core, + evlist_cpu_itr.cpu_map_idx); + } + + evlist__for_each_entry_reverse(evlist, evsel) { + perf_evsel__free_fd(&evsel->core); + perf_evsel__free_id(&evsel->core); + } + perf_evlist__reset_id_hash(evlist__core(evlist)); +} + +static int evlist__create_syswide_maps(struct evlist *evlist) +{ + struct perf_cpu_map *cpus; + struct perf_thread_map *threads; + + /* + * Try reading /sys/devices/system/cpu/online to get + * an all cpus map. + * + * FIXME: -ENOMEM is the best we can do here, the cpu_map + * code needs an overhaul to properly forward the + * error, and we may not want to do that fallback to a + * default cpu identity map :-\ + */ + cpus = perf_cpu_map__new_online_cpus(); + if (!cpus) + return -ENOMEM; + + threads = perf_thread_map__new_dummy(); + if (!threads) { + perf_cpu_map__put(cpus); + return -ENOMEM; + } + + perf_evlist__set_maps(evlist__core(evlist), cpus, threads); + perf_thread_map__put(threads); + perf_cpu_map__put(cpus); + return 0; +} + +int evlist__open(struct evlist *evlist) +{ + struct evsel *evsel; + int err; + + /* + * Default: one fd per CPU, all threads, aka systemwide + * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL + */ + if (evlist__core(evlist)->threads == NULL && + evlist__core(evlist)->user_requested_cpus == NULL) { + err = evlist__create_syswide_maps(evlist); + if (err < 0) + goto out_err; + } + + evlist__update_id_pos(evlist); + + evlist__for_each_entry(evlist, evsel) { + err = evsel__open(evsel, evsel->core.cpus, evsel->core.threads); + if (err < 0) + goto out_err; + } + + return 0; +out_err: + evlist__close(evlist); + errno = -err; + return err; +} + +int evlist__prepare_workload(struct evlist *evlist, struct target *target, const char *argv[], + bool pipe_output, void (*exec_error)(int signo, siginfo_t *info, void *ucontext)) +{ + int child_ready_pipe[2], go_pipe[2]; + char bf; + + if (target__none(target)) { + const char *cmd = strrchr(argv[0], '/'); + + if (cmd) + cmd++; + else + cmd = argv[0]; + + if (!strcmp(cmd, "sleep")) { + ui__warning("workload '%s' specified without the system-wide (-a) option\n", + cmd); + } + } + + evlist__set_workload_cork_fd(evlist, -1); + + if (pipe(child_ready_pipe) < 0) { + perror("failed to create 'ready' pipe"); + return -1; + } + + if (pipe(go_pipe) < 0) { + perror("failed to create 'go' pipe"); + goto out_close_ready_pipe; + } + + evlist__set_workload_pid(evlist, fork()); + if (evlist__workload_pid(evlist) < 0) { + perror("failed to fork"); + goto out_close_pipes; + } + + if (!evlist__workload_pid(evlist)) { + int ret; + + if (pipe_output) + dup2(2, 1); + + signal(SIGTERM, SIG_DFL); + + close(child_ready_pipe[0]); + close(go_pipe[1]); + fcntl(go_pipe[0], F_SETFD, FD_CLOEXEC); + + /* + * Change the name of this process not to confuse --exclude-perf users + * that sees 'perf' in the window up to the execvp() and thinks that + * perf samples are not being excluded. + */ + prctl(PR_SET_NAME, "perf-exec"); + + /* + * Tell the parent we're ready to go + */ + close(child_ready_pipe[1]); + + /* + * Wait until the parent tells us to go. + */ + ret = read(go_pipe[0], &bf, 1); + /* + * The parent will ask for the execvp() to be performed by + * writing exactly one byte, in workload.cork_fd, usually via + * evlist__start_workload(). + * + * For cancelling the workload without actually running it, + * the parent will just close workload.cork_fd, without writing + * anything, i.e. read will return zero and we just exit() + * here (See evlist__cancel_workload()). + */ + if (ret != 1) { + if (ret == -1) + perror("unable to read pipe"); + exit(ret); + } + + execvp(argv[0], (char **)argv); + + if (exec_error) { + union sigval val; + + val.sival_int = errno; + if (sigqueue(getppid(), SIGUSR1, val)) + perror(argv[0]); + } else + perror(argv[0]); + exit(-1); + } + + if (exec_error) { + struct sigaction act = { + .sa_flags = SA_SIGINFO, + .sa_sigaction = exec_error, + }; + sigaction(SIGUSR1, &act, NULL); + } + + if (target__none(target)) { + if (evlist__core(evlist)->threads == NULL) { + fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n", + __func__, __LINE__); + goto out_close_pipes; + } + perf_thread_map__set_pid(evlist__core(evlist)->threads, 0, + evlist__workload_pid(evlist)); + } + + close(child_ready_pipe[1]); + close(go_pipe[0]); + /* + * wait for child to settle + */ + if (read(child_ready_pipe[0], &bf, 1) == -1) { + perror("unable to read pipe"); + goto out_close_pipes; + } + + fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC); + evlist__set_workload_cork_fd(evlist, go_pipe[1]); + close(child_ready_pipe[0]); + return 0; + +out_close_pipes: + close(go_pipe[0]); + close(go_pipe[1]); +out_close_ready_pipe: + close(child_ready_pipe[0]); + close(child_ready_pipe[1]); + return -1; +} + +int evlist__start_workload(struct evlist *evlist) +{ + if (evlist__workload_cork_fd(evlist) >= 0) { + char bf = 0; + int ret; + /* + * Remove the cork, let it rip! + */ + ret = write(evlist__workload_cork_fd(evlist), &bf, 1); + if (ret < 0) + perror("unable to write to pipe"); + + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); + return ret; + } + + return 0; +} + +void evlist__cancel_workload(struct evlist *evlist) +{ + int status; + + if (evlist__workload_cork_fd(evlist) >= 0) { + close(evlist__workload_cork_fd(evlist)); + evlist__set_workload_cork_fd(evlist, -1); + waitpid(evlist__workload_pid(evlist), &status, WNOHANG); + } +} + +int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample) +{ + struct evsel *evsel = evlist__event2evsel(evlist, event); + int ret; + + if (!evsel) { + /* Ensure the sample is okay for perf_sample__exit. */ + perf_sample__init(sample, /*all=*/false); + return -EFAULT; + } + ret = evsel__parse_sample(evsel, event, sample); + if (ret) + return ret; + if (perf_guest && sample->id) { + struct perf_sample_id *sid = evlist__id2sid(evlist, sample->id); + + if (sid) { + sample->machine_pid = sid->machine_pid; + sample->vcpu = sid->vcpu.cpu; + } + } + return 0; +} + +int evlist__parse_sample_timestamp(struct evlist *evlist, union perf_event *event, u64 *timestamp) +{ + struct evsel *evsel = evlist__event2evsel(evlist, event); + + if (!evsel) + return -EFAULT; + return evsel__parse_sample_timestamp(evsel, event, timestamp); +} + +int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size) +{ + int printed, value; + + switch (err) { + case EACCES: + case EPERM: + errno = err; + printed = scnprintf(buf, size, + "Error:\t%m.\n" + "Hint:\tCheck /proc/sys/kernel/perf_event_paranoid setting."); + + value = perf_event_paranoid(); + + printed += scnprintf(buf + printed, size - printed, "\nHint:\t"); + + if (value >= 2) { + printed += scnprintf(buf + printed, size - printed, + "For your workloads it needs to be <= 1\nHint:\t"); + } + printed += scnprintf(buf + printed, size - printed, + "For system wide tracing it needs to be set to -1.\n"); + + printed += scnprintf(buf + printed, size - printed, + "Hint:\tTry: 'sudo sh -c \"echo -1 > /proc/sys/kernel/perf_event_paranoid\"'\n" + "Hint:\tThe current value is %d.", value); + break; + case EINVAL: { + struct evsel *first = evlist__first(evlist); + int max_freq; + + if (sysctl__read_int("kernel/perf_event_max_sample_rate", &max_freq) < 0) + goto out_default; + + if (first->core.attr.sample_freq < (u64)max_freq) + goto out_default; + + errno = err; + printed = scnprintf(buf, size, + "Error:\t%m.\n" + "Hint:\tCheck /proc/sys/kernel/perf_event_max_sample_rate.\n" + "Hint:\tThe current value is %d and %" PRIu64 " is being requested.", + max_freq, first->core.attr.sample_freq); + break; + } + default: +out_default: + errno = err; + scnprintf(buf, size, "%m"); + break; + } + + return 0; +} + +int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size) +{ + int pages_attempted = evlist__core(evlist)->mmap_len / 1024; + int pages_max_per_user, printed = 0; + + switch (err) { + case EPERM: + sysctl__read_int("kernel/perf_event_mlock_kb", &pages_max_per_user); + errno = err; + printed += scnprintf(buf + printed, size - printed, + "Error:\t%m.\n" + "Hint:\tCheck /proc/sys/kernel/perf_event_mlock_kb (%d kB) setting.\n" + "Hint:\tTried using %zd kB.\n", + pages_max_per_user, pages_attempted); + + if (pages_attempted >= pages_max_per_user) { + printed += scnprintf(buf + printed, size - printed, + "Hint:\tTry 'sudo sh -c \"echo %d > /proc/sys/kernel/perf_event_mlock_kb\"', or\n", + pages_max_per_user + pages_attempted); + } + + printed += scnprintf(buf + printed, size - printed, + "Hint:\tTry using a smaller -m/--mmap-pages value."); + break; + default: + errno = err; + scnprintf(buf, size, "%m"); + break; + } + + return 0; +} + +void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel) +{ + struct evsel *evsel, *n; + LIST_HEAD(move); + + if (move_evsel == evlist__first(evlist)) + return; + + evlist__for_each_entry_safe(evlist, n, evsel) { + if (evsel__leader(evsel) == evsel__leader(move_evsel)) + list_move_tail(&evsel->core.node, &move); + } + + list_splice(&move, &evlist__core(evlist)->entries); +} + +struct evsel *evlist__get_tracking_event(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->tracking) + return evsel; + } + + return evlist__first(evlist); +} + +void evlist__set_tracking_event(struct evlist *evlist, struct evsel *tracking_evsel) +{ + struct evsel *evsel; + + if (tracking_evsel->tracking) + return; + + evlist__for_each_entry(evlist, evsel) { + if (evsel != tracking_evsel) + evsel->tracking = false; + } + + tracking_evsel->tracking = true; +} + +struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel; + + evsel = evlist__get_tracking_event(evlist); + if (!evsel__is_dummy_event(evsel)) { + evsel = evlist__add_aux_dummy(evlist, system_wide); + if (!evsel) + return NULL; + + evlist__set_tracking_event(evlist, evsel); + } else if (system_wide) { + perf_evlist__go_system_wide(evlist__core(evlist), &evsel->core); + } + + return evsel; +} + +struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (!evsel->name) + continue; + if (evsel__name_is(evsel, str)) + return evsel; + } + + return NULL; +} + +void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) +{ + enum bkw_mmap_state old_state = evlist__bkw_mmap_state(evlist); + enum action { + NONE, + PAUSE, + RESUME, + } action = NONE; + + if (!evlist__overwrite_mmap(evlist)) + return; + + switch (old_state) { + case BKW_MMAP_NOTREADY: { + if (state != BKW_MMAP_RUNNING) + goto state_err; + break; + } + case BKW_MMAP_RUNNING: { + if (state != BKW_MMAP_DATA_PENDING) + goto state_err; + action = PAUSE; + break; + } + case BKW_MMAP_DATA_PENDING: { + if (state != BKW_MMAP_EMPTY) + goto state_err; + break; + } + case BKW_MMAP_EMPTY: { + if (state != BKW_MMAP_RUNNING) + goto state_err; + action = RESUME; + break; + } + default: + WARN_ONCE(1, "Shouldn't get there\n"); + } + + evlist__set_bkw_mmap_state(evlist, state); + + switch (action) { + case PAUSE: + evlist__pause(evlist); + break; + case RESUME: + evlist__resume(evlist); + break; + case NONE: + default: + break; + } + +state_err: + return; +} + +bool evlist__exclude_kernel(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (!evsel->core.attr.exclude_kernel) + return false; + } + + return true; +} + +/* + * Events in data file are not collect in groups, but we still want + * the group display. Set the artificial group and set the leader's + * forced_leader flag to notify the display code. + */ +void evlist__force_leader(struct evlist *evlist) +{ + if (evlist__nr_groups(evlist) == 0) { + struct evsel *leader = evlist__first(evlist); + + evlist__set_leader(evlist); + leader->forced_leader = true; + } +} + +struct evsel *evlist__reset_weak_group(struct evlist *evsel_list, struct evsel *evsel, bool close) +{ + struct evsel *c2, *leader; + bool is_open = true; + + leader = evsel__leader(evsel); + + pr_debug("Weak group for %s/%d failed\n", + leader->name, leader->core.nr_members); + + /* + * for_each_group_member doesn't work here because it doesn't + * include the first entry. + */ + evlist__for_each_entry(evsel_list, c2) { + if (c2 == evsel) + is_open = false; + if (evsel__has_leader(c2, leader)) { + if (is_open && close) + perf_evsel__close(&c2->core); + /* + * We want to close all members of the group and reopen + * them. Some events, like Intel topdown, require being + * in a group and so keep these in the group. + */ + evsel__remove_from_group(c2, leader); + + /* + * Set this for all former members of the group + * to indicate they get reopened. + */ + c2->reset_group = true; + } + } + /* Reset the leader count if all entries were removed. */ + if (leader->core.nr_members == 1) + leader->core.nr_members = 0; + return leader; +} + +static int evlist__parse_control_fifo(const char *str, int *ctl_fd, int *ctl_fd_ack, bool *ctl_fd_close) +{ + char *s, *p; + int ret = 0, fd; + + if (strncmp(str, "fifo:", 5)) + return -EINVAL; + + str += 5; + if (!*str || *str == ',') + return -EINVAL; + + s = strdup(str); + if (!s) + return -ENOMEM; + + p = strchr(s, ','); + if (p) + *p = '\0'; + + /* + * O_RDWR avoids POLLHUPs which is necessary to allow the other + * end of a FIFO to be repeatedly opened and closed. + */ + fd = open(s, O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (fd < 0) { + ret = -errno; + pr_err("Failed to open '%s': %m\n", s); + goto out_free; + } + *ctl_fd = fd; + *ctl_fd_close = true; + + if (p && *++p) { + /* O_RDWR | O_NONBLOCK means the other end need not be open */ + fd = open(p, O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (fd < 0) { + pr_err("Failed to open '%s': %m\n", p); + ret = -errno; + goto out_free; + } + *ctl_fd_ack = fd; + } + +out_free: + free(s); + return ret; +} + +int evlist__parse_control(const char *str, int *ctl_fd, int *ctl_fd_ack, bool *ctl_fd_close) +{ + const char *comma = NULL; + char *endptr = NULL; + + *ctl_fd_close = false; + + if (strncmp(str, "fd:", 3)) + return evlist__parse_control_fifo(str, ctl_fd, ctl_fd_ack, ctl_fd_close); + + *ctl_fd = strtoul(&str[3], &endptr, 0); + if (endptr == &str[3]) + return -EINVAL; + + comma = strchr(str, ','); + if (comma) { + if (endptr != comma) + return -EINVAL; + + *ctl_fd_ack = strtoul(comma + 1, &endptr, 0); + if (endptr == comma + 1 || *endptr != '\0') + return -EINVAL; + } + + return 0; +} + +void evlist__close_control(int ctl_fd, int ctl_fd_ack, bool *ctl_fd_close) +{ + if (*ctl_fd_close) { + *ctl_fd_close = false; + close(ctl_fd); + if (ctl_fd_ack >= 0) + close(ctl_fd_ack); + } +} + +int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack) +{ + if (fd == -1) { + pr_debug("Control descriptor is not initialized\n"); + return 0; + } + + evlist__set_ctl_fd_pos(evlist, + perf_evlist__add_pollfd(evlist__core(evlist), fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event)); + if (evlist__ctl_fd_pos(evlist) < 0) { + evlist__set_ctl_fd_pos(evlist, -1); + pr_err("Failed to add ctl fd entry: %m\n"); + return -1; + } + + evlist__set_ctl_fd_fd(evlist, fd); + evlist__set_ctl_fd_ack(evlist, ack); + + return 0; +} + +bool evlist__ctlfd_initialized(struct evlist *evlist) +{ + return evlist__ctl_fd_pos(evlist) >= 0; +} + +int evlist__finalize_ctlfd(struct evlist *evlist) +{ + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; + + if (!evlist__ctlfd_initialized(evlist)) + return 0; + + entries[evlist__ctl_fd_pos(evlist)].fd = -1; + entries[evlist__ctl_fd_pos(evlist)].events = 0; + entries[evlist__ctl_fd_pos(evlist)].revents = 0; + + evlist__set_ctl_fd_pos(evlist, -1); + evlist__set_ctl_fd_ack(evlist, -1); + evlist__set_ctl_fd_fd(evlist, -1); + + return 0; +} + +static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, + char *cmd_data, size_t data_size) +{ + int err; + char c; + size_t bytes_read = 0; + + *cmd = EVLIST_CTL_CMD_UNSUPPORTED; + memset(cmd_data, 0, data_size); + data_size--; + + do { + err = read(evlist__ctl_fd_fd(evlist), &c, 1); + if (err > 0) { + if (c == '\n' || c == '\0') + break; + cmd_data[bytes_read++] = c; + if (bytes_read == data_size) + break; + continue; + } else if (err == -1) { + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + err = 0; + else + pr_err("Failed to read from ctlfd %d: %m\n", + evlist__ctl_fd_fd(evlist)); + } + break; + } while (1); + + pr_debug("Message from ctl_fd: \"%s%s\"\n", cmd_data, + bytes_read == data_size ? "" : c == '\n' ? "\\n" : "\\0"); + + if (bytes_read > 0) { + if (!strncmp(cmd_data, EVLIST_CTL_CMD_ENABLE_TAG, + (sizeof(EVLIST_CTL_CMD_ENABLE_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_ENABLE; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_DISABLE_TAG, + (sizeof(EVLIST_CTL_CMD_DISABLE_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_DISABLE; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_SNAPSHOT_TAG, + (sizeof(EVLIST_CTL_CMD_SNAPSHOT_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_SNAPSHOT; + pr_debug("is snapshot\n"); + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_EVLIST_TAG, + (sizeof(EVLIST_CTL_CMD_EVLIST_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_EVLIST; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_STOP_TAG, + (sizeof(EVLIST_CTL_CMD_STOP_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_STOP; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_PING_TAG, + (sizeof(EVLIST_CTL_CMD_PING_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_PING; + } + } + + return bytes_read ? (int)bytes_read : err; +} + +int evlist__ctlfd_ack(struct evlist *evlist) +{ + int err; + + if (evlist__ctl_fd_ack(evlist) == -1) + return 0; + + err = write(evlist__ctl_fd_ack(evlist), EVLIST_CTL_CMD_ACK_TAG, + sizeof(EVLIST_CTL_CMD_ACK_TAG)); + if (err == -1) + pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist__ctl_fd_ack(evlist)); + + return err; +} + +static int get_cmd_arg(char *cmd_data, size_t cmd_size, char **arg) +{ + char *data = cmd_data + cmd_size; + + /* no argument */ + if (!*data) + return 0; + + /* there's argument */ + if (*data == ' ') { + *arg = data + 1; + return 1; + } + + /* malformed */ + return -1; +} + +static int evlist__ctlfd_enable(struct evlist *evlist, char *cmd_data, bool enable) +{ + struct evsel *evsel; + char *name; + int err; + + err = get_cmd_arg(cmd_data, + enable ? sizeof(EVLIST_CTL_CMD_ENABLE_TAG) - 1 : + sizeof(EVLIST_CTL_CMD_DISABLE_TAG) - 1, + &name); + if (err < 0) { + pr_info("failed: wrong command\n"); + return -1; + } + + if (err) { + evsel = evlist__find_evsel_by_str(evlist, name); + if (evsel) { + if (enable) + evlist__enable_evsel(evlist, name); + else + evlist__disable_evsel(evlist, name); + pr_info("Event %s %s\n", evsel->name, + enable ? "enabled" : "disabled"); + } else { + pr_info("failed: can't find '%s' event\n", name); + } + } else { + if (enable) { + evlist__enable(evlist); + pr_info(EVLIST_ENABLED_MSG); + } else { + evlist__disable(evlist); + pr_info(EVLIST_DISABLED_MSG); + } + } + + return 0; +} + +static int evlist__ctlfd_list(struct evlist *evlist, char *cmd_data) +{ + struct perf_attr_details details = { .verbose = false, }; + struct evsel *evsel; + char *arg; + int err; + + err = get_cmd_arg(cmd_data, + sizeof(EVLIST_CTL_CMD_EVLIST_TAG) - 1, + &arg); + if (err < 0) { + pr_info("failed: wrong command\n"); + return -1; + } + + if (err) { + if (!strcmp(arg, "-v")) { + details.verbose = true; + } else if (!strcmp(arg, "-g")) { + details.event_group = true; + } else if (!strcmp(arg, "-F")) { + details.freq = true; + } else { + pr_info("failed: wrong command\n"); + return -1; + } + } + + evlist__for_each_entry(evlist, evsel) + evsel__fprintf(evsel, &details, stderr); + + return 0; +} + +int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd) +{ + int err = 0; + char cmd_data[EVLIST_CTL_CMD_MAX_LEN]; + int ctlfd_pos = evlist__ctl_fd_pos(evlist); + struct pollfd *entries = evlist__core(evlist)->pollfd.entries; + + if (!evlist__ctlfd_initialized(evlist) || !entries[ctlfd_pos].revents) + return 0; + + if (entries[ctlfd_pos].revents & POLLIN) { + err = evlist__ctlfd_recv(evlist, cmd, cmd_data, + EVLIST_CTL_CMD_MAX_LEN); + if (err > 0) { + switch (*cmd) { + case EVLIST_CTL_CMD_ENABLE: + case EVLIST_CTL_CMD_DISABLE: + err = evlist__ctlfd_enable(evlist, cmd_data, + *cmd == EVLIST_CTL_CMD_ENABLE); + break; + case EVLIST_CTL_CMD_EVLIST: + err = evlist__ctlfd_list(evlist, cmd_data); + break; + case EVLIST_CTL_CMD_SNAPSHOT: + case EVLIST_CTL_CMD_STOP: + case EVLIST_CTL_CMD_PING: + break; + case EVLIST_CTL_CMD_ACK: + case EVLIST_CTL_CMD_UNSUPPORTED: + default: + pr_debug("ctlfd: unsupported %d\n", *cmd); + break; + } + if (!(*cmd == EVLIST_CTL_CMD_ACK || *cmd == EVLIST_CTL_CMD_UNSUPPORTED || + *cmd == EVLIST_CTL_CMD_SNAPSHOT)) + evlist__ctlfd_ack(evlist); + } + } + + if (entries[ctlfd_pos].revents & (POLLHUP | POLLERR)) + evlist__finalize_ctlfd(evlist); + else + entries[ctlfd_pos].revents = 0; + + return err; +} + +/** + * struct event_enable_time - perf record -D/--delay single time range. + * @start: start of time range to enable events in milliseconds + * @end: end of time range to enable events in milliseconds + * + * N.B. this structure is also accessed as an array of int. + */ +struct event_enable_time { + int start; + int end; +}; + +static int parse_event_enable_time(const char *str, struct event_enable_time *range, bool first) +{ + const char *fmt = first ? "%u - %u %n" : " , %u - %u %n"; + int ret, start, end, n; + + ret = sscanf(str, fmt, &start, &end, &n); + if (ret != 2 || end <= start) + return -EINVAL; + if (range) { + range->start = start; + range->end = end; + } + return n; +} + +static ssize_t parse_event_enable_times(const char *str, struct event_enable_time *range) +{ + int incr = !!range; + bool first = true; + ssize_t ret, cnt; + + for (cnt = 0; *str; cnt++) { + ret = parse_event_enable_time(str, range, first); + if (ret < 0) + return ret; + /* Check no overlap */ + if (!first && range && range->start <= range[-1].end) + return -EINVAL; + str += ret; + range += incr; + first = false; + } + return cnt; +} + +/** + * struct event_enable_timer - control structure for perf record -D/--delay. + * @evlist: event list + * @times: time ranges that events are enabled (N.B. this is also accessed as an + * array of int) + * @times_cnt: number of time ranges + * @timerfd: timer file descriptor + * @pollfd_pos: position in @evlist array of file descriptors to poll (fdarray) + * @times_step: current position in (int *)@times)[], + * refer event_enable_timer__process() + * + * Note, this structure is only used when there are time ranges, not when there + * is only an initial delay. + */ +struct event_enable_timer { + struct evlist *evlist; + struct event_enable_time *times; + size_t times_cnt; + int timerfd; + int pollfd_pos; + size_t times_step; +}; + +static int str_to_delay(const char *str) +{ + char *endptr; + long d; + + d = strtol(str, &endptr, 10); + if (*endptr || d > INT_MAX || d < -1) + return 0; + return d; +} + +int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, + const char *str, int unset) +{ + enum fdarray_flags flags = fdarray_flag__nonfilterable | fdarray_flag__non_perf_event; + struct event_enable_timer *eet; + ssize_t times_cnt; + ssize_t ret; + int err; + + if (unset) + return 0; + + opts->target.initial_delay = str_to_delay(str); + if (opts->target.initial_delay) + return 0; + + ret = parse_event_enable_times(str, NULL); + if (ret < 0) + return ret; + + times_cnt = ret; + if (times_cnt == 0) + return -EINVAL; + + eet = zalloc(sizeof(*eet)); + if (!eet) + return -ENOMEM; + + eet->times = calloc(times_cnt, sizeof(*eet->times)); + if (!eet->times) { + err = -ENOMEM; + goto free_eet; + } + + if (parse_event_enable_times(str, eet->times) != times_cnt) { + err = -EINVAL; + goto free_eet_times; + } + + eet->times_cnt = times_cnt; + + eet->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + if (eet->timerfd == -1) { + err = -errno; + pr_err("timerfd_create failed: %m\n"); + goto free_eet_times; + } + + eet->pollfd_pos = perf_evlist__add_pollfd(evlist__core(evlist), eet->timerfd, + NULL, POLLIN, flags); + if (eet->pollfd_pos < 0) { + err = eet->pollfd_pos; + goto close_timerfd; + } + + eet->evlist = evlist; + RC_CHK_ACCESS(evlist)->eet = eet; + opts->target.initial_delay = eet->times[0].start; + + return 0; + +close_timerfd: + close(eet->timerfd); +free_eet_times: + zfree(&eet->times); +free_eet: + free(eet); + return err; +} + +static int event_enable_timer__set_timer(struct event_enable_timer *eet, int ms) +{ + struct itimerspec its = { + .it_value.tv_sec = ms / MSEC_PER_SEC, + .it_value.tv_nsec = (ms % MSEC_PER_SEC) * NSEC_PER_MSEC, + }; + int err = 0; + + if (timerfd_settime(eet->timerfd, 0, &its, NULL) < 0) { + err = -errno; + pr_err("timerfd_settime failed: %m\n"); + } + return err; +} + +int event_enable_timer__start(struct event_enable_timer *eet) +{ + int ms; + + if (!eet) + return 0; + + ms = eet->times[0].end - eet->times[0].start; + eet->times_step = 1; + + return event_enable_timer__set_timer(eet, ms); +} + +int event_enable_timer__process(struct event_enable_timer *eet) +{ + struct pollfd *entries; + short revents; + + if (!eet) + return 0; + + entries = evlist__core(eet->evlist)->pollfd.entries; + revents = entries[eet->pollfd_pos].revents; + entries[eet->pollfd_pos].revents = 0; + + if (revents & POLLIN) { + size_t step = eet->times_step; + size_t pos = step / 2; + + if (step & 1) { + evlist__disable_non_dummy(eet->evlist); + pr_info(EVLIST_DISABLED_MSG); + if (pos >= eet->times_cnt - 1) { + /* Disarm timer */ + event_enable_timer__set_timer(eet, 0); + return 1; /* Stop */ + } + } else { + evlist__enable_non_dummy(eet->evlist); + pr_info(EVLIST_ENABLED_MSG); + } + + step += 1; + pos = step / 2; + + if (pos < eet->times_cnt) { + int *times = (int *)eet->times; /* Accessing 'times' as array of int */ + int ms = times[step] - times[step - 1]; + + eet->times_step = step; + return event_enable_timer__set_timer(eet, ms); + } + } + + return 0; +} + +static void event_enable_timer__exit(struct event_enable_timer **ep) +{ + if (!ep || !*ep) + return; + zfree(&(*ep)->times); + zfree(ep); +} + +struct evsel *evlist__find_evsel(struct evlist *evlist, int idx) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.idx == idx) + return evsel; + } + return NULL; +} + +void evlist__format_evsels(struct evlist *evlist, struct strbuf *sb, size_t max_length) +{ + struct evsel *evsel, *leader = NULL; + bool first = true; + + evlist__for_each_entry(evlist, evsel) { + struct evsel *new_leader = evsel__leader(evsel); + + if (evsel__is_dummy_event(evsel)) + continue; + + if (leader != new_leader && leader && leader->core.nr_members > 1) + strbuf_addch(sb, '}'); + + if (!first) + strbuf_addch(sb, ','); + + if (sb->len > max_length) { + strbuf_addstr(sb, "..."); + return; + } + if (leader != new_leader && new_leader->core.nr_members > 1) + strbuf_addch(sb, '{'); + + strbuf_addstr(sb, evsel__name(evsel)); + first = false; + leader = new_leader; + } + if (leader && leader->core.nr_members > 1) + strbuf_addch(sb, '}'); +} + +void evlist__check_mem_load_aux(struct evlist *evlist) +{ + struct evsel *leader, *evsel, *pos; + + /* + * For some platforms, the 'mem-loads' event is required to use + * together with 'mem-loads-aux' within a group and 'mem-loads-aux' + * must be the group leader. Now we disable this group before reporting + * because 'mem-loads-aux' is just an auxiliary event. It doesn't carry + * any valid memory load information. + */ + evlist__for_each_entry(evlist, evsel) { + leader = evsel__leader(evsel); + if (leader == evsel) + continue; + + if (leader->name && strstr(leader->name, "mem-loads-aux")) { + for_each_group_evsel(pos, leader) { + evsel__set_leader(pos, pos); + pos->core.nr_members = 0; + } + } + } +} + +/** + * evlist__warn_user_requested_cpus() - Check each evsel against requested CPUs + * and warn if the user CPU list is inapplicable for the event's PMU's + * CPUs. Not core PMUs list a CPU in sysfs, but this may be overwritten by a + * user requested CPU and so any online CPU is applicable. Core PMUs handle + * events on the CPUs in their list and otherwise the event isn't supported. + * @evlist: The list of events being checked. + * @cpu_list: The user provided list of CPUs. + */ +void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_list) +{ + struct perf_cpu_map *user_requested_cpus; + struct evsel *pos; + + if (!cpu_list) + return; + + user_requested_cpus = perf_cpu_map__new(cpu_list); + if (!user_requested_cpus) + return; + + evlist__for_each_entry(evlist, pos) { + evsel__warn_user_requested_cpus(pos, user_requested_cpus); + } + perf_cpu_map__put(user_requested_cpus); +} + +/* Should uniquify be disabled for the evlist? */ +static bool evlist__disable_uniquify(struct evlist *evlist) +{ + struct evsel *counter; + struct perf_pmu *last_pmu = NULL; + bool first = true; + + evlist__for_each_entry(evlist, counter) { + /* If PMUs vary then uniquify can be useful. */ + if (!first && counter->pmu != last_pmu) + return false; + first = false; + if (counter->pmu) { + /* Allow uniquify for uncore PMUs. */ + if (!counter->pmu->is_core) + return false; + /* Keep hybrid event names uniquified for clarity. */ + if (perf_pmus__num_core_pmus() > 1) + return false; + } + last_pmu = counter->pmu; + } + return true; +} + +static bool evlist__set_needs_uniquify(struct evlist *evlist, const struct perf_stat_config *config) +{ + struct evsel *counter; + bool needs_uniquify = false; + + if (evlist__disable_uniquify(evlist)) { + evlist__for_each_entry(evlist, counter) + counter->uniquified_name = true; + return false; + } + + evlist__for_each_entry(evlist, counter) { + if (evsel__set_needs_uniquify(counter, config)) + needs_uniquify = true; + } + return needs_uniquify; +} + +void evlist__uniquify_evsel_names(struct evlist *evlist, const struct perf_stat_config *config) +{ + if (evlist__set_needs_uniquify(evlist, config)) { + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) + evsel__uniquify_counter(pos); + } +} + +bool evlist__has_bpf_output(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_bpf_output(evsel)) + return true; + } + + return false; +} + +bool evlist__needs_bpf_sb_event(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_dummy_event(evsel)) + continue; + if (!evsel->core.attr.exclude_kernel) + return true; + } + + return false; +} diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h new file mode 100644 index 000000000..838e263b7 --- /dev/null +++ b/tools/perf/util/evlist.h @@ -0,0 +1,685 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_EVLIST_H +#define __PERF_EVLIST_H 1 + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "affinity.h" +#include "events_stats.h" +#include "evsel.h" +#include "rblist.h" + +struct perf_cpu_map; +struct perf_stat_config; +struct pollfd; +struct record_opts; +struct strbuf; +struct target; +struct thread_map; + +/* + * State machine of bkw_mmap_state: + * + * .________________(forbid)_____________. + * | V + * NOTREADY --(0)--> RUNNING --(1)--> DATA_PENDING --(2)--> EMPTY + * ^ ^ | ^ | + * | |__(forbid)____/ |___(forbid)___/| + * | | + * \_________________(3)_______________/ + * + * NOTREADY : Backward ring buffers are not ready + * RUNNING : Backward ring buffers are recording + * DATA_PENDING : We are required to collect data from backward ring buffers + * EMPTY : We have collected data from backward ring buffers. + * + * (0): Setup backward ring buffer + * (1): Pause ring buffers for reading + * (2): Read from ring buffers + * (3): Resume ring buffers for recording + */ +enum bkw_mmap_state { + BKW_MMAP_NOTREADY, + BKW_MMAP_RUNNING, + BKW_MMAP_DATA_PENDING, + BKW_MMAP_EMPTY, +}; + +struct event_enable_timer; + +DECLARE_RC_STRUCT(evlist) { + struct perf_evlist core; + refcount_t refcnt; + bool enabled; + bool no_affinity; + int id_pos; + int is_pos; + int nr_br_cntr; + u64 combined_sample_type; + enum bkw_mmap_state bkw_mmap_state; + struct { + int cork_fd; + pid_t pid; + } workload; + struct mmap *mmap; + struct mmap *overwrite_mmap; + struct evsel *selected; + struct events_stats stats; + struct perf_session *session; + void (*trace_event_sample_raw)(struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample); + u64 first_sample_time; + u64 last_sample_time; + struct { + pthread_t th; + volatile int done; + } sb_thread; + struct { + int fd; /* control file descriptor */ + int ack; /* ack file descriptor for control commands */ + int pos; /* index at evlist core object to check signals */ + } ctl_fd; + struct event_enable_timer *eet; + /** + * @metric_events: A list of struct metric_event which each have a list + * of struct metric_expr. + */ + struct rblist metric_events; + /* samples with deferred_callchain would wait here. */ + struct list_head deferred_samples; +}; + +struct evsel_str_handler { + const char *name; + void *handler; +}; + +static inline struct perf_evlist *evlist__core(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline const struct perf_evlist *evlist__const_core(const struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->core; +} + +static inline int evlist__nr_entries(const struct evlist *evlist) +{ + return evlist__const_core(evlist)->nr_entries; +} + +static inline bool evlist__enabled(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->enabled; +} + +static inline void evlist__set_enabled(struct evlist *evlist, bool enabled) +{ + RC_CHK_ACCESS(evlist)->enabled = enabled; +} + +static inline bool evlist__no_affinity(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->no_affinity; +} + +static inline void evlist__set_no_affinity(struct evlist *evlist, bool no_affinity) +{ + RC_CHK_ACCESS(evlist)->no_affinity = no_affinity; +} + +static inline int evlist__sb_thread_done(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->sb_thread.done; +} + +static inline void evlist__set_sb_thread_done(struct evlist *evlist, int done) +{ + RC_CHK_ACCESS(evlist)->sb_thread.done = done; +} + +static inline pthread_t *evlist__sb_thread_th(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->sb_thread.th; +} + +static inline int evlist__id_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->id_pos; +} + +static inline int evlist__is_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->is_pos; +} + +static inline struct event_enable_timer *evlist__event_enable_timer(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->eet; +} + +static inline enum bkw_mmap_state evlist__bkw_mmap_state(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->bkw_mmap_state; +} + +static inline void evlist__set_bkw_mmap_state(struct evlist *evlist, enum bkw_mmap_state state) +{ + RC_CHK_ACCESS(evlist)->bkw_mmap_state = state; +} + +static inline struct mmap *evlist__mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->mmap; +} + +static inline struct mmap *evlist__overwrite_mmap(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->overwrite_mmap; +} + +static inline struct events_stats *evlist__stats(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->stats; +} + +static inline u64 evlist__first_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->first_sample_time; +} + +static inline void evlist__set_first_sample_time(struct evlist *evlist, u64 first) +{ + RC_CHK_ACCESS(evlist)->first_sample_time = first; +} + +static inline u64 evlist__last_sample_time(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->last_sample_time; +} + +static inline void evlist__set_last_sample_time(struct evlist *evlist, u64 last) +{ + RC_CHK_ACCESS(evlist)->last_sample_time = last; +} + +static inline int evlist__nr_br_cntr(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->nr_br_cntr; +} + +static inline void evlist__set_nr_br_cntr(struct evlist *evlist, int nr) +{ + RC_CHK_ACCESS(evlist)->nr_br_cntr = nr; +} + +static inline struct perf_session *evlist__session(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->session; +} + +static inline void evlist__set_session(struct evlist *evlist, struct perf_session *session) +{ + RC_CHK_ACCESS(evlist)->session = session; +} + +static inline void (*evlist__trace_event_sample_raw(struct evlist *evlist)) + (struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample) +{ + return RC_CHK_ACCESS(evlist)->trace_event_sample_raw; +} + +static inline void evlist__set_trace_event_sample_raw(struct evlist *evlist, + void (*fun)(struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample)) +{ + RC_CHK_ACCESS(evlist)->trace_event_sample_raw = fun; +} + +static inline pid_t evlist__workload_pid(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.pid; +} + +static inline void evlist__set_workload_pid(struct evlist *evlist, pid_t pid) +{ + RC_CHK_ACCESS(evlist)->workload.pid = pid; +} + +static inline int evlist__workload_cork_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->workload.cork_fd; +} + +static inline void evlist__set_workload_cork_fd(struct evlist *evlist, int cork_fd) +{ + RC_CHK_ACCESS(evlist)->workload.cork_fd = cork_fd; +} + +static inline int evlist__ctl_fd_fd(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.fd; +} + +static inline void evlist__set_ctl_fd_fd(struct evlist *evlist, int fd) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.fd = fd; +} + +static inline int evlist__ctl_fd_ack(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.ack; +} + +static inline void evlist__set_ctl_fd_ack(struct evlist *evlist, int ack) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.ack = ack; +} + +static inline int evlist__ctl_fd_pos(const struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->ctl_fd.pos; +} + +static inline void evlist__set_ctl_fd_pos(struct evlist *evlist, int pos) +{ + RC_CHK_ACCESS(evlist)->ctl_fd.pos = pos; +} + +static inline refcount_t *evlist__refcnt(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->refcnt; +} + +static inline struct rblist *evlist__metric_events(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->metric_events; +} + +static inline struct list_head *evlist__deferred_samples(struct evlist *evlist) +{ + return &RC_CHK_ACCESS(evlist)->deferred_samples; +} + +static inline struct evsel *evlist__selected(struct evlist *evlist) +{ + return RC_CHK_ACCESS(evlist)->selected; +} + +static inline void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) +{ + RC_CHK_ACCESS(evlist)->selected = evsel; +} + +struct evlist *evlist__new(void); +struct evlist *evlist__new_default(const struct target *target, bool sample_callchains); +struct evlist *evlist__new_dummy(void); +struct evlist *evlist__get(struct evlist *evlist); +void evlist__put(struct evlist *evlist); + +void evlist__add(struct evlist *evlist, struct evsel *entry); +void evlist__remove(struct evlist *evlist, struct evsel *evsel); + +int arch_evlist__cmp(const struct evsel *lhs, const struct evsel *rhs); +int arch_evlist__add_required_events(struct list_head *list); + +int evlist__add_dummy(struct evlist *evlist); +struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide); +static inline struct evsel *evlist__add_dummy_on_all_cpus(struct evlist *evlist) +{ + return evlist__add_aux_dummy(evlist, true); +} +#ifdef HAVE_LIBTRACEEVENT +struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide); +#endif + +int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr, + evsel__sb_cb_t cb, void *data); +void evlist__set_cb(struct evlist *evlist, evsel__sb_cb_t cb, void *data); +int evlist__start_sb_thread(struct evlist *evlist, struct target *target); +void evlist__stop_sb_thread(struct evlist *evlist); + +#ifdef HAVE_LIBTRACEEVENT +int evlist__add_newtp(struct evlist *evlist, const char *sys, const char *name, void *handler); +#endif + +int __evlist__set_tracepoints_handlers(struct evlist *evlist, + const struct evsel_str_handler *assocs, + size_t nr_assocs); + +#define evlist__set_tracepoints_handlers(evlist, array) \ + __evlist__set_tracepoints_handlers(evlist, array, ARRAY_SIZE(array)) + +int evlist__set_tp_filter(struct evlist *evlist, const char *filter); +int evlist__set_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids); + +int evlist__append_tp_filter(struct evlist *evlist, const char *filter); + +int evlist__append_tp_filter_pid(struct evlist *evlist, pid_t pid); +int evlist__append_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids); + +struct evsel *evlist__find_tracepoint_by_name(struct evlist *evlist, const char *name); + +int evlist__add_pollfd(struct evlist *evlist, int fd); +int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask); + +#ifdef HAVE_EVENTFD_SUPPORT +int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd); +#endif + +int evlist__poll(struct evlist *evlist, int timeout); + +struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id); +struct evsel *evlist__id2evsel_strict(struct evlist *evlist, u64 id); + +struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id); + +void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state); + +void evlist__mmap_consume(struct evlist *evlist, int idx); + +int evlist__open(struct evlist *evlist); +void evlist__close(struct evlist *evlist); + +struct callchain_param; + +void evlist__set_id_pos(struct evlist *evlist); +void evlist__config(struct evlist *evlist, struct record_opts *opts, struct callchain_param *callchain); +int record_opts__config(struct record_opts *opts); + +int evlist__prepare_workload(struct evlist *evlist, struct target *target, + const char *argv[], bool pipe_output, + void (*exec_error)(int signo, siginfo_t *info, void *ucontext)); +int evlist__start_workload(struct evlist *evlist); +void evlist__cancel_workload(struct evlist *evlist); + +struct option; + +int __evlist__parse_mmap_pages(unsigned int *mmap_pages, const char *str); +int evlist__parse_mmap_pages(const struct option *opt, const char *str, int unset); + +unsigned long perf_event_mlock_kb_in_pages(void); + +int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, + unsigned int auxtrace_pages, + bool auxtrace_overwrite, int nr_cblocks, + int affinity, int flush, int comp_level); +int evlist__do_mmap(struct evlist *evlist, unsigned int pages); +void evlist__do_munmap(struct evlist *evlist); + +size_t evlist__mmap_size(unsigned long pages); + +void evlist__disable(struct evlist *evlist); +void evlist__enable(struct evlist *evlist); +void evlist__toggle_enable(struct evlist *evlist); +void evlist__disable_evsel(struct evlist *evlist, char *evsel_name); +void evlist__enable_evsel(struct evlist *evlist, char *evsel_name); +void evlist__disable_non_dummy(struct evlist *evlist); +void evlist__enable_non_dummy(struct evlist *evlist); + +int evlist__create_maps(struct evlist *evlist, struct target *target); +int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, + struct target *target); + +u64 __evlist__combined_sample_type(struct evlist *evlist); +u64 evlist__combined_sample_type(struct evlist *evlist); +u64 evlist__combined_branch_type(struct evlist *evlist); +void evlist__update_br_cntr(struct evlist *evlist); +bool evlist__sample_id_all(struct evlist *evlist); +u16 evlist__id_hdr_size(struct evlist *evlist); + +int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample); +int evlist__parse_sample_timestamp(struct evlist *evlist, union perf_event *event, u64 *timestamp); + +bool evlist__valid_sample_type(struct evlist *evlist); +bool evlist__valid_sample_id_all(struct evlist *evlist); +bool evlist__valid_read_format(struct evlist *evlist); + +void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list); + +static inline bool evlist__empty(struct evlist *evlist) +{ + return list_empty(&evlist__core(evlist)->entries); +} + +static inline struct evsel *evlist__first(struct evlist *evlist) +{ + struct perf_evsel *evsel = perf_evlist__first(evlist__core(evlist)); + + return container_of(evsel, struct evsel, core); +} + +static inline struct evsel *evlist__last(struct evlist *evlist) +{ + struct perf_evsel *evsel = perf_evlist__last(evlist__core(evlist)); + + return container_of(evsel, struct evsel, core); +} + +static inline int evlist__nr_groups(struct evlist *evlist) +{ + return perf_evlist__nr_groups(evlist__core(evlist)); +} + +int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size); +int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size); + +bool evlist__can_select_event(struct evlist *evlist, const char *str); +void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); + +/** + * __evlist__for_each_entry - iterate thru all the evsels + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry(list, evsel) \ + list_for_each_entry(evsel, list, core.node) + +/** + * evlist__for_each_entry - iterate thru all the evsels + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry(evlist, evsel) \ + __evlist__for_each_entry(&evlist__core(evlist)->entries, evsel) + +/** + * __evlist__for_each_entry_continue - continue iteration thru all the evsels + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_continue(list, evsel) \ + list_for_each_entry_continue(evsel, list, core.node) + +/** + * evlist__for_each_entry_continue - continue iteration thru all the evsels + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry_continue(evlist, evsel) \ + __evlist__for_each_entry_continue(&evlist__core(evlist)->entries, evsel) + +/** + * __evlist__for_each_entry_from - continue iteration from @evsel (included) + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_from(list, evsel) \ + list_for_each_entry_from(evsel, list, core.node) + +/** + * evlist__for_each_entry_from - continue iteration from @evsel (included) + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry_from(evlist, evsel) \ + __evlist__for_each_entry_from(&evlist__core(evlist)->entries, evsel) + +/** + * __evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_reverse(list, evsel) \ + list_for_each_entry_reverse(evsel, list, core.node) + +/** + * evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry_reverse(evlist, evsel) \ + __evlist__for_each_entry_reverse(&evlist__core(evlist)->entries, evsel) + +/** + * __evlist__for_each_entry_safe - safely iterate thru all the evsels + * @list: list_head instance to iterate + * @tmp: struct evsel temp iterator + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_safe(list, tmp, evsel) \ + list_for_each_entry_safe(evsel, tmp, list, core.node) + +/** + * evlist__for_each_entry_safe - safely iterate thru all the evsels + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + * @tmp: struct evsel temp iterator + */ +#define evlist__for_each_entry_safe(evlist, tmp, evsel) \ + __evlist__for_each_entry_safe(&evlist__core(evlist)->entries, tmp, evsel) + +/** Iterator state for evlist__for_each_cpu */ +struct evlist_cpu_iterator { + /** The list being iterated through. */ + struct evlist *container; + /** The current evsel of the iterator. */ + struct evsel *evsel; + /** The CPU map index corresponding to the evsel->core.cpus for the current CPU. */ + int cpu_map_idx; + /** + * The CPU map index corresponding to evlist->core.all_cpus for the + * current CPU. Distinct from cpu_map_idx as the evsel's cpu map may + * contain fewer entries. + */ + int evlist_cpu_map_idx; + /** The number of CPU map entries in evlist->core.all_cpus. */ + int evlist_cpu_map_nr; + /** The current CPU of the iterator. */ + struct perf_cpu cpu; + /** If present, used to set the affinity when switching between CPUs. */ + struct affinity *affinity; + /** Maybe be used to hold affinity state prior to iterating. */ + struct affinity saved_affinity; +}; + +/** + * evlist__for_each_cpu - without affinity, iterate over the evlist. With + * affinity, iterate over all CPUs and then the evlist + * for each evsel on that CPU. When switching between + * CPUs the affinity is set to the CPU to avoid IPIs + * during syscalls. The affinity is set up and removed + * automatically, if the loop is broken a call to + * evlist_cpu_iterator__exit is necessary. + * @evlist_cpu_itr: the iterator instance. + * @evlist: evlist instance to iterate. + */ +#define evlist__for_each_cpu(evlist_cpu_itr, evlist) \ + for (evlist_cpu_iterator__init(&(evlist_cpu_itr), evlist); \ + !evlist_cpu_iterator__end(&evlist_cpu_itr); \ + evlist_cpu_iterator__next(&evlist_cpu_itr)) + +/** Setup an iterator set to the first CPU/evsel of evlist. */ +void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *evlist); +/** + * Cleans up the iterator, automatically done by evlist_cpu_iterator__next when + * the end of the list is reached. Multiple calls are safe. + */ +void evlist_cpu_iterator__exit(struct evlist_cpu_iterator *itr); +/** Move to next element in iterator, updating CPU, evsel and the affinity. */ +void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr); +/** Returns true when iterator is at the end of the CPUs and evlist. */ +static inline bool evlist_cpu_iterator__end(const struct evlist_cpu_iterator *evlist_cpu_itr) +{ + return evlist_cpu_itr->evlist_cpu_map_idx >= evlist_cpu_itr->evlist_cpu_map_nr; +} + +struct evsel *evlist__get_tracking_event(struct evlist *evlist); +void evlist__set_tracking_event(struct evlist *evlist, struct evsel *tracking_evsel); +struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_wide); + +struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str); + +struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event); + +bool evlist__exclude_kernel(struct evlist *evlist); + +void evlist__force_leader(struct evlist *evlist); + +struct evsel *evlist__reset_weak_group(struct evlist *evlist, struct evsel *evsel, bool close); + +#define EVLIST_CTL_CMD_ENABLE_TAG "enable" +#define EVLIST_CTL_CMD_DISABLE_TAG "disable" +#define EVLIST_CTL_CMD_ACK_TAG "ack\n" +#define EVLIST_CTL_CMD_SNAPSHOT_TAG "snapshot" +#define EVLIST_CTL_CMD_EVLIST_TAG "evlist" +#define EVLIST_CTL_CMD_STOP_TAG "stop" +#define EVLIST_CTL_CMD_PING_TAG "ping" + +#define EVLIST_CTL_CMD_MAX_LEN 64 + +enum evlist_ctl_cmd { + EVLIST_CTL_CMD_UNSUPPORTED = 0, + EVLIST_CTL_CMD_ENABLE, + EVLIST_CTL_CMD_DISABLE, + EVLIST_CTL_CMD_ACK, + EVLIST_CTL_CMD_SNAPSHOT, + EVLIST_CTL_CMD_EVLIST, + EVLIST_CTL_CMD_STOP, + EVLIST_CTL_CMD_PING, +}; + +int evlist__parse_control(const char *str, int *ctl_fd, int *ctl_fd_ack, bool *ctl_fd_close); +void evlist__close_control(int ctl_fd, int ctl_fd_ack, bool *ctl_fd_close); +int evlist__initialize_ctlfd(struct evlist *evlist, int ctl_fd, int ctl_fd_ack); +int evlist__finalize_ctlfd(struct evlist *evlist); +bool evlist__ctlfd_initialized(struct evlist *evlist); +int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd); +int evlist__ctlfd_ack(struct evlist *evlist); + +#define EVLIST_ENABLED_MSG "Events enabled\n" +#define EVLIST_DISABLED_MSG "Events disabled\n" + +int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, + const char *str, int unset); +int event_enable_timer__start(struct event_enable_timer *eet); +int event_enable_timer__process(struct event_enable_timer *eet); + +struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); + +void evlist__format_evsels(struct evlist *evlist, struct strbuf *sb, size_t max_length); +void evlist__check_mem_load_aux(struct evlist *evlist); +void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_list); +void evlist__uniquify_evsel_names(struct evlist *evlist, const struct perf_stat_config *config); +bool evlist__has_bpf_output(struct evlist *evlist); +bool evlist__needs_bpf_sb_event(struct evlist *evlist); + +#endif /* __PERF_EVLIST_H */ diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c new file mode 100644 index 000000000..d4cb455f4 --- /dev/null +++ b/tools/perf/util/evsel.c @@ -0,0 +1,4758 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo + * + * Parts came from builtin-{top,stat,record}.c, see those files for further + * copyright notes. + */ +/* + * Powerpc needs __SANE_USERSPACE_TYPES__ before to select + * 'int-ll64.h' and avoid compile warnings when printing __u64 with %llu. + */ +#define __SANE_USERSPACE_TYPES__ + +#include "evsel.h" + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../perf-sys.h" +#include "asm/bug.h" +#include "bpf-filter.h" +#include "bpf_counter.h" +#include "callchain.h" +#include "cgroup.h" +#include "counts.h" +#include "debug.h" +#include "drm_pmu.h" +#include "dwarf-regs.h" +#include "env.h" +#include "event.h" +#include "evlist.h" +#include "evsel_config.h" +#include "evsel_fprintf.h" +#include "hashmap.h" +#include "hist.h" +#include "hwmon_pmu.h" +#include "intel-tpebs.h" +#include "memswap.h" +#include "off_cpu.h" +#include "parse-branch-options.h" +#include "perf_regs.h" +#include "pmu.h" +#include "pmus.h" +#include "record.h" +#include "rlimit.h" +#include "session.h" +#include "stat.h" +#include "string2.h" +#include "target.h" +#include "thread_map.h" +#include "time-utils.h" +#include "tool_pmu.h" +#include "tp_pmu.h" +#include "trace-event.h" +#include "util.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +struct perf_missing_features perf_missing_features; + +static clockid_t clockid; + +static int evsel__no_extra_init(struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static bool test_attr__enabled(void) +{ + static bool test_attr__enabled; + static bool test_attr__enabled_tested; + + if (!test_attr__enabled_tested) { + char *dir = getenv("PERF_TEST_ATTR"); + + test_attr__enabled = (dir != NULL); + test_attr__enabled_tested = true; + } + return test_attr__enabled; +} + +#define __WRITE_ASS(str, fmt, data) \ +do { \ + if (fprintf(file, #str "=%"fmt "\n", data) < 0) { \ + perror("test attr - failed to write event file"); \ + fclose(file); \ + return -1; \ + } \ +} while (0) + +#define WRITE_ASS(field, fmt) __WRITE_ASS(field, fmt, attr->field) + +static int store_event(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, + int fd, int group_fd, unsigned long flags) +{ + FILE *file; + char path[PATH_MAX]; + char *dir = getenv("PERF_TEST_ATTR"); + + snprintf(path, PATH_MAX, "%s/event-%d-%llu-%d", dir, + attr->type, attr->config, fd); + + file = fopen(path, "w+"); + if (!file) { + perror("test attr - failed to open event file"); + return -1; + } + + if (fprintf(file, "[event-%d-%llu-%d]\n", + attr->type, attr->config, fd) < 0) { + perror("test attr - failed to write event file"); + fclose(file); + return -1; + } + + /* syscall arguments */ + __WRITE_ASS(fd, "d", fd); + __WRITE_ASS(group_fd, "d", group_fd); + __WRITE_ASS(cpu, "d", cpu.cpu); + __WRITE_ASS(pid, "d", pid); + __WRITE_ASS(flags, "lu", flags); + + /* struct perf_event_attr */ + WRITE_ASS(type, PRIu32); + WRITE_ASS(size, PRIu32); + WRITE_ASS(config, "llu"); + WRITE_ASS(sample_period, "llu"); + WRITE_ASS(sample_type, "llu"); + WRITE_ASS(read_format, "llu"); + WRITE_ASS(disabled, "d"); + WRITE_ASS(inherit, "d"); + WRITE_ASS(pinned, "d"); + WRITE_ASS(exclusive, "d"); + WRITE_ASS(exclude_user, "d"); + WRITE_ASS(exclude_kernel, "d"); + WRITE_ASS(exclude_hv, "d"); + WRITE_ASS(exclude_idle, "d"); + WRITE_ASS(mmap, "d"); + WRITE_ASS(comm, "d"); + WRITE_ASS(freq, "d"); + WRITE_ASS(inherit_stat, "d"); + WRITE_ASS(enable_on_exec, "d"); + WRITE_ASS(task, "d"); + WRITE_ASS(watermark, "d"); + WRITE_ASS(precise_ip, "d"); + WRITE_ASS(mmap_data, "d"); + WRITE_ASS(sample_id_all, "d"); + WRITE_ASS(exclude_host, "d"); + WRITE_ASS(exclude_guest, "d"); + WRITE_ASS(exclude_callchain_kernel, "d"); + WRITE_ASS(exclude_callchain_user, "d"); + WRITE_ASS(mmap2, "d"); + WRITE_ASS(comm_exec, "d"); + WRITE_ASS(context_switch, "d"); + WRITE_ASS(write_backward, "d"); + WRITE_ASS(namespaces, "d"); + WRITE_ASS(use_clockid, "d"); + WRITE_ASS(wakeup_events, PRIu32); + WRITE_ASS(bp_type, PRIu32); + WRITE_ASS(config1, "llu"); + WRITE_ASS(config2, "llu"); + WRITE_ASS(branch_sample_type, "llu"); + WRITE_ASS(sample_regs_user, "llu"); + WRITE_ASS(sample_stack_user, PRIu32); + + fclose(file); + return 0; +} + +#undef __WRITE_ASS +#undef WRITE_ASS + +static void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, + int fd, int group_fd, unsigned long flags) +{ + int errno_saved = errno; + + if ((fd != -1) && store_event(attr, pid, cpu, fd, group_fd, flags)) { + pr_err("test attr FAILED"); + exit(128); + } + + errno = errno_saved; +} + +static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused) +{ +} + +static struct { + size_t size; + int (*init)(struct evsel *evsel); + void (*fini)(struct evsel *evsel); +} perf_evsel__object = { + .size = sizeof(struct evsel), + .init = evsel__no_extra_init, + .fini = evsel__no_extra_fini, +}; + +int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel), + void (*fini)(struct evsel *evsel)) +{ + + if (object_size == 0) + goto set_methods; + + if (perf_evsel__object.size > object_size) + return -EINVAL; + + perf_evsel__object.size = object_size; + +set_methods: + if (init != NULL) + perf_evsel__object.init = init; + + if (fini != NULL) + perf_evsel__object.fini = fini; + + return 0; +} + +const char *evsel__pmu_name(const struct evsel *evsel) +{ + struct perf_pmu *pmu = evsel__find_pmu(evsel); + + if (pmu) + return pmu->name; + + return event_type(evsel->core.attr.type); +} + +enum evsel_probe_type { + PROBE__UNKNOWN = 0, + PROBE__NOPE = 1, + PROBE__KPROBE = 2, + PROBE__UPROBE = 3, + /* + * Ftrace-based dynamic probes (kprobes/uprobes/fprobes created via + * tracefs) report PMU "tracepoint", not "kprobe"/"uprobe". Detect + * them by the __probe_ip field that the kernel adds to all dynamic + * probe formats. + */ + PROBE__FTRACE = 4, +}; + +static void evsel__resolve_probe_type(struct evsel *evsel) +{ + const char *name = evsel__pmu_name(evsel); + + if (!strcmp(name, "kprobe")) + evsel->probe_type = PROBE__KPROBE; + else if (!strcmp(name, "uprobe")) + evsel->probe_type = PROBE__UPROBE; + else if (!strcmp(name, "tracepoint") && evsel__field(evsel, "__probe_ip")) + evsel->probe_type = PROBE__FTRACE; + else + evsel->probe_type = PROBE__NOPE; +} + +bool evsel__is_probe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type > PROBE__NOPE; +} + +bool evsel__is_kprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__KPROBE; +} + +bool evsel__is_uprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__UPROBE; +} + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) + +int __evsel__sample_size(u64 sample_type) +{ + u64 mask = sample_type & PERF_SAMPLE_MASK; + int size = 0; + int i; + + for (i = 0; i < 64; i++) { + if (mask & (1ULL << i)) + size++; + } + + size *= sizeof(u64); + + return size; +} + +/** + * __perf_evsel__calc_id_pos - calculate id_pos. + * @sample_type: sample type + * + * This function returns the position of the event id (PERF_SAMPLE_ID or + * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct + * perf_record_sample. + */ +static int __perf_evsel__calc_id_pos(u64 sample_type) +{ + int idx = 0; + + if (sample_type & PERF_SAMPLE_IDENTIFIER) + return 0; + + if (!(sample_type & PERF_SAMPLE_ID)) + return -1; + + if (sample_type & PERF_SAMPLE_IP) + idx += 1; + + if (sample_type & PERF_SAMPLE_TID) + idx += 1; + + if (sample_type & PERF_SAMPLE_TIME) + idx += 1; + + if (sample_type & PERF_SAMPLE_ADDR) + idx += 1; + + return idx; +} + +/** + * __perf_evsel__calc_is_pos - calculate is_pos. + * @sample_type: sample type + * + * This function returns the position (counting backwards) of the event id + * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if + * sample_id_all is used there is an id sample appended to non-sample events. + */ +static int __perf_evsel__calc_is_pos(u64 sample_type) +{ + int idx = 1; + + if (sample_type & PERF_SAMPLE_IDENTIFIER) + return 1; + + if (!(sample_type & PERF_SAMPLE_ID)) + return -1; + + if (sample_type & PERF_SAMPLE_CPU) + idx += 1; + + if (sample_type & PERF_SAMPLE_STREAM_ID) + idx += 1; + + return idx; +} + +void evsel__calc_id_pos(struct evsel *evsel) +{ + evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type); + evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type); +} + +void __evsel__set_sample_bit(struct evsel *evsel, + enum perf_event_sample_format bit) +{ + if (!(evsel->core.attr.sample_type & bit)) { + evsel->core.attr.sample_type |= bit; + evsel->sample_size += sizeof(u64); + evsel__calc_id_pos(evsel); + } +} + +void __evsel__reset_sample_bit(struct evsel *evsel, + enum perf_event_sample_format bit) +{ + if (evsel->core.attr.sample_type & bit) { + evsel->core.attr.sample_type &= ~bit; + evsel->sample_size -= sizeof(u64); + evsel__calc_id_pos(evsel); + } +} + +void evsel__set_sample_id(struct evsel *evsel, + bool can_sample_identifier) +{ + if (can_sample_identifier) { + evsel__reset_sample_bit(evsel, ID); + evsel__set_sample_bit(evsel, IDENTIFIER); + } else { + evsel__set_sample_bit(evsel, ID); + } + evsel->core.attr.read_format |= PERF_FORMAT_ID; +} + +bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel) +{ + return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END; +} + +/** + * evsel__is_function_event - Return whether given evsel is a function + * trace event + * + * @evsel - evsel selector to be tested + * + * Return %true if event is function trace event + */ +bool evsel__is_function_event(struct evsel *evsel) +{ +#define FUNCTION_EVENT "ftrace:function" + + return evsel->name && + !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT)); + +#undef FUNCTION_EVENT +} + +static void evsel__init(struct evsel *evsel, + struct perf_event_attr *attr, int idx) +{ + perf_evsel__init(&evsel->core, attr, idx); + refcount_set(&evsel->refcnt, 1); + evsel->tracking = !idx; + evsel->unit = strdup(""); + evsel->scale = 1.0; + evsel->max_events = ULONG_MAX; + evsel->evlist = NULL; + INIT_LIST_HEAD(&evsel->config_terms); + INIT_LIST_HEAD(&evsel->bpf_counter_list); + INIT_LIST_HEAD(&evsel->bpf_filters); + perf_evsel__object.init(evsel); + evsel->sample_size = __evsel__sample_size(attr->sample_type); + evsel__calc_id_pos(evsel); + evsel->cmdline_group_boundary = false; + evsel->per_pkg_mask = NULL; + evsel->collect_stat = false; + evsel->group_pmu_name = NULL; + evsel->skippable = false; + evsel->supported = true; + evsel->alternate_hw_config = PERF_COUNT_HW_MAX; + evsel->script_output_type = -1; // FIXME: OUTPUT_TYPE_UNSET, see builtin-script.c + evsel->probe_type = PROBE__UNKNOWN; +} + +struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx) +{ + struct evsel *evsel = zalloc(perf_evsel__object.size); + + if (!evsel) + return NULL; + evsel__init(evsel, attr, idx); + + if (evsel__is_bpf_output(evsel) && !attr->sample_type) { + evsel->core.attr.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME | + PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD), + evsel->core.attr.sample_period = 1; + } + + if (evsel__is_clock(evsel)) { + free((char *)evsel->unit); + evsel->unit = strdup("msec"); + evsel->scale = 1e-6; + } + + return evsel; +} + +int copy_config_terms(struct list_head *dst, struct list_head *src) +{ + struct evsel_config_term *pos, *tmp; + + list_for_each_entry(pos, src, list) { + tmp = malloc(sizeof(*tmp)); + if (tmp == NULL) + return -ENOMEM; + + *tmp = *pos; + if (tmp->free_str) { + tmp->val.str = strdup(pos->val.str); + if (tmp->val.str == NULL) { + free(tmp); + return -ENOMEM; + } + } + list_add_tail(&tmp->list, dst); + } + return 0; +} + +static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src) +{ + return copy_config_terms(&dst->config_terms, &src->config_terms); +} + +/** + * evsel__clone - create a new evsel copied from @orig + * @orig: original evsel + * + * The assumption is that @orig is not configured nor opened yet. + * So we only care about the attributes that can be set while it's parsed. + */ +struct evsel *evsel__clone(struct evsel *orig) +{ + struct evsel *evsel; + + BUG_ON(orig->core.fd); + BUG_ON(orig->counts); + BUG_ON(orig->priv); + BUG_ON(orig->per_pkg_mask); + + evsel = evsel__new(&orig->core.attr); + if (evsel == NULL) + return NULL; + + evsel->core.cpus = perf_cpu_map__get(orig->core.cpus); + evsel->core.pmu_cpus = perf_cpu_map__get(orig->core.pmu_cpus); + evsel->core.threads = perf_thread_map__get(orig->core.threads); + evsel->core.nr_members = orig->core.nr_members; + evsel->core.system_wide = orig->core.system_wide; + evsel->core.requires_cpu = orig->core.requires_cpu; + evsel->core.is_pmu_core = orig->core.is_pmu_core; + + if (orig->name) { + evsel->name = strdup(orig->name); + if (evsel->name == NULL) + goto out_err; + } + if (orig->group_name) { + evsel->group_name = strdup(orig->group_name); + if (evsel->group_name == NULL) + goto out_err; + } + if (orig->group_pmu_name) { + evsel->group_pmu_name = strdup(orig->group_pmu_name); + if (evsel->group_pmu_name == NULL) + goto out_err; + } + if (orig->filter) { + evsel->filter = strdup(orig->filter); + if (evsel->filter == NULL) + goto out_err; + } + if (orig->metric_id) { + evsel->metric_id = strdup(orig->metric_id); + if (evsel->metric_id == NULL) + goto out_err; + } + evsel->cgrp = cgroup__get(orig->cgrp); +#ifdef HAVE_LIBTRACEEVENT + if (orig->tp_sys) { + evsel->tp_sys = strdup(orig->tp_sys); + if (evsel->tp_sys == NULL) + goto out_err; + } + if (orig->tp_name) { + evsel->tp_name = strdup(orig->tp_name); + if (evsel->tp_name == NULL) + goto out_err; + } + evsel->tp_format = orig->tp_format; +#endif + evsel->handler = orig->handler; + evsel->core.leader = orig->core.leader; + evsel->metric_leader = orig->metric_leader; + + evsel->max_events = orig->max_events; + zfree(&evsel->unit); + if (orig->unit) { + evsel->unit = strdup(orig->unit); + if (evsel->unit == NULL) + goto out_err; + } + evsel->scale = orig->scale; + evsel->snapshot = orig->snapshot; + evsel->per_pkg = orig->per_pkg; + evsel->percore = orig->percore; + evsel->precise_max = orig->precise_max; + evsel->is_libpfm_event = orig->is_libpfm_event; + + evsel->exclude_GH = orig->exclude_GH; + evsel->sample_read = orig->sample_read; + evsel->collect_stat = orig->collect_stat; + evsel->weak_group = orig->weak_group; + evsel->bpf_counter = orig->bpf_counter; + evsel->use_config_name = orig->use_config_name; + evsel->skippable = orig->skippable; + evsel->dont_regroup = orig->dont_regroup; + evsel->default_metricgroup = orig->default_metricgroup; + evsel->default_show_events = orig->default_show_events; + + evsel->pmu = orig->pmu; + evsel->first_wildcard_match = orig->first_wildcard_match; + + if (evsel__copy_config_terms(evsel, orig) < 0) + goto out_err; + + evsel->alternate_hw_config = orig->alternate_hw_config; + + evsel->retire_lat = orig->retire_lat; + if (evsel->retire_lat) + evsel->retirement_latency = orig->retirement_latency; + + return evsel; + +out_err: + evsel__put(evsel); + return NULL; +} + +/* + * Returns pointer with encoded error via interface. + */ +struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool format) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_TRACEPOINT, + .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME | + PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD), + }; + struct evsel *evsel = zalloc(perf_evsel__object.size); + int err = -ENOMEM, id = -1; + + if (evsel == NULL) + goto out_err; + + + if (asprintf(&evsel->name, "%s:%s", sys, name) < 0) + goto out_free; + +#ifdef HAVE_LIBTRACEEVENT + evsel->tp_sys = strdup(sys); + if (!evsel->tp_sys) + goto out_free; + + evsel->tp_name = strdup(name); + if (!evsel->tp_name) + goto out_free; +#endif + + event_attr_init(&attr); + + if (format) { + id = tp_pmu__id(sys, name); + if (id < 0) { + err = id; + goto out_free; + } + } + attr.config = (__u64)id; + attr.sample_period = 1; + evsel__init(evsel, &attr, idx); + return evsel; + +out_free: + zfree(&evsel->name); +#ifdef HAVE_LIBTRACEEVENT + zfree(&evsel->tp_sys); + zfree(&evsel->tp_name); +#endif + free(evsel); +out_err: + return ERR_PTR(err); +} + +struct evsel *evsel__get(struct evsel *evsel) +{ + refcount_inc(&evsel->refcnt); + return evsel; +} + +#ifdef HAVE_LIBTRACEEVENT +struct tep_event *evsel__tp_format(struct evsel *evsel) +{ + struct tep_event *tp_format = evsel->tp_format; + + if (tp_format) + return tp_format; + + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) + return NULL; + + if (!evsel->tp_sys) + tp_format = trace_event__tp_format_id(evsel->core.attr.config); + else + tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name); + + if (IS_ERR(tp_format)) { + int err = -PTR_ERR(tp_format); + + errno = err; + pr_err("Error getting tracepoint format '%s': %m\n", + evsel__name(evsel)); + return NULL; + } + evsel->tp_format = tp_format; + return evsel->tp_format; +} +#endif + +const char *const evsel__hw_names[PERF_COUNT_HW_MAX] = { + "cycles", + "instructions", + "cache-references", + "cache-misses", + "branches", + "branch-misses", + "bus-cycles", + "stalled-cycles-frontend", + "stalled-cycles-backend", + "ref-cycles", +}; + +char *evsel__bpf_counter_events; + +bool evsel__match_bpf_counter_events(const char *name) +{ + int name_len; + bool match; + char *ptr; + + if (!evsel__bpf_counter_events) + return false; + + ptr = strstr(evsel__bpf_counter_events, name); + name_len = strlen(name); + + /* check name matches a full token in evsel__bpf_counter_events */ + match = (ptr != NULL) && + ((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) && + ((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0')); + + return match; +} + +static const char *__evsel__hw_name(u64 config) +{ + if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config]) + return evsel__hw_names[config]; + + return "unknown-hardware"; +} + +static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size) +{ + int colon = 0, r = 0; + struct perf_event_attr *attr = &evsel->core.attr; + +#define MOD_PRINT(context, mod) do { \ + if (!attr->exclude_##context) { \ + if (!colon) colon = ++r; \ + r += scnprintf(bf + r, size - r, "%c", mod); \ + } } while(0) + + if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) { + MOD_PRINT(kernel, 'k'); + MOD_PRINT(user, 'u'); + MOD_PRINT(hv, 'h'); + } + + if (attr->precise_ip) { + if (!colon) + colon = ++r; + r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp"); + } + + if (attr->exclude_host || attr->exclude_guest) { + MOD_PRINT(host, 'H'); + MOD_PRINT(guest, 'G'); + } +#undef MOD_PRINT + if (colon) + bf[colon - 1] = ':'; + return r; +} + +int __weak arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size) +{ + return scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config)); +} + +static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size) +{ + int r = arch_evsel__hw_name(evsel, bf, size); + return r + evsel__add_modifiers(evsel, bf + r, size - r); +} + +const char *const evsel__sw_names[PERF_COUNT_SW_MAX] = { + "cpu-clock", + "task-clock", + "page-faults", + "context-switches", + "cpu-migrations", + "minor-faults", + "major-faults", + "alignment-faults", + "emulation-faults", + "dummy", +}; + +static const char *__evsel__sw_name(u64 config) +{ + if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config]) + return evsel__sw_names[config]; + return "unknown-software"; +} + +static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size) +{ + int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config)); + return r + evsel__add_modifiers(evsel, bf + r, size - r); +} + +static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type) +{ + int r; + + r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr); + + if (type & HW_BREAKPOINT_R) + r += scnprintf(bf + r, size - r, "r"); + + if (type & HW_BREAKPOINT_W) + r += scnprintf(bf + r, size - r, "w"); + + if (type & HW_BREAKPOINT_X) + r += scnprintf(bf + r, size - r, "x"); + + return r; +} + +static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size) +{ + struct perf_event_attr *attr = &evsel->core.attr; + int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type); + return r + evsel__add_modifiers(evsel, bf + r, size - r); +} + +const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = { + { "L1-dcache", "l1-d", "l1d", "L1-data", }, + { "L1-icache", "l1-i", "l1i", "L1-instruction", }, + { "LLC", "L2", }, + { "dTLB", "d-tlb", "Data-TLB", }, + { "iTLB", "i-tlb", "Instruction-TLB", }, + { "branch", "branches", "bpu", "btb", "bpc", }, + { "node", }, +}; + +const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = { + { "load", "loads", "read", }, + { "store", "stores", "write", }, + { "prefetch", "prefetches", "speculative-read", "speculative-load", }, +}; + +const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = { + { "refs", "Reference", "ops", "access", }, + { "misses", "miss", }, +}; + +#define C(x) PERF_COUNT_HW_CACHE_##x +#define CACHE_READ (1 << C(OP_READ)) +#define CACHE_WRITE (1 << C(OP_WRITE)) +#define CACHE_PREFETCH (1 << C(OP_PREFETCH)) +#define COP(x) (1 << x) + +/* + * cache operation stat + * L1I : Read and prefetch only + * ITLB and BPU : Read-only + */ +static const unsigned long evsel__hw_cache_stat[C(MAX)] = { + [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), + [C(L1I)] = (CACHE_READ | CACHE_PREFETCH), + [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), + [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), + [C(ITLB)] = (CACHE_READ), + [C(BPU)] = (CACHE_READ), + [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), +}; + +bool evsel__is_cache_op_valid(u8 type, u8 op) +{ + if (evsel__hw_cache_stat[type] & COP(op)) + return true; /* valid */ + else + return false; /* invalid */ +} + +int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size) +{ + if (result) { + return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0], + evsel__hw_cache_op[op][0], + evsel__hw_cache_result[result][0]); + } + + return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0], + evsel__hw_cache_op[op][1]); +} + +static int __evsel__hw_cache_name(u64 config, char *bf, size_t size) +{ + u8 op, result, type = (config >> 0) & 0xff; + const char *err = "unknown-ext-hardware-cache-type"; + + if (type >= PERF_COUNT_HW_CACHE_MAX) + goto out_err; + + op = (config >> 8) & 0xff; + err = "unknown-ext-hardware-cache-op"; + if (op >= PERF_COUNT_HW_CACHE_OP_MAX) + goto out_err; + + result = (config >> 16) & 0xff; + err = "unknown-ext-hardware-cache-result"; + if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + goto out_err; + + err = "invalid-cache"; + if (!evsel__is_cache_op_valid(type, op)) + goto out_err; + + return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size); +out_err: + return scnprintf(bf, size, "%s", err); +} + +static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size) +{ + int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size); + return ret + evsel__add_modifiers(evsel, bf + ret, size - ret); +} + +static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size) +{ + int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config); + return ret + evsel__add_modifiers(evsel, bf + ret, size - ret); +} + +const char *evsel__name(struct evsel *evsel) +{ + char bf[128]; + + if (!evsel) + goto out_unknown; + + if (evsel->name) + return evsel->name; + + switch (evsel->core.attr.type) { + case PERF_TYPE_RAW: + evsel__raw_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_HARDWARE: + evsel__hw_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_HW_CACHE: + evsel__hw_cache_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_SOFTWARE: + evsel__sw_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_TRACEPOINT: + scnprintf(bf, sizeof(bf), "unknown tracepoint id=%#"PRIx64, + evsel->core.attr.config); + break; + + case PERF_TYPE_BREAKPOINT: + evsel__bp_name(evsel, bf, sizeof(bf)); + break; + + case PERF_PMU_TYPE_TOOL: + scnprintf(bf, sizeof(bf), "%s", evsel__tool_pmu_event_name(evsel)); + break; + + default: + scnprintf(bf, sizeof(bf), "unknown event PMU=%d config=%#"PRIx64, + evsel->core.attr.type, evsel->core.attr.config); + break; + } + + evsel->name = strdup(bf); + + if (evsel->name) + return evsel->name; +out_unknown: + return "unknown"; +} + +bool evsel__name_is(struct evsel *evsel, const char *name) +{ + return !strcmp(evsel__name(evsel), name); +} + +const char *evsel__metric_id(const struct evsel *evsel) +{ + if (evsel->metric_id) + return evsel->metric_id; + + if (evsel__is_tool(evsel)) + return evsel__tool_pmu_event_name(evsel); + + return "unknown"; +} + +const char *evsel__group_name(struct evsel *evsel) +{ + return evsel->group_name ?: "anon group"; +} + +/* + * Returns the group details for the specified leader, + * with following rules. + * + * For record -e '{cycles,instructions}' + * 'anon group { cycles:u, instructions:u }' + * + * For record -e 'cycles,instructions' and report --group + * 'cycles:u, instructions:u' + */ +int evsel__group_desc(struct evsel *evsel, char *buf, size_t size) +{ + int ret = 0; + bool first = true; + struct evsel *pos; + const char *group_name = evsel__group_name(evsel); + + if (!evsel->forced_leader) + ret = scnprintf(buf, size, "%s { ", group_name); + + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + + ret += scnprintf(buf + ret, size - ret, "%s%s", + first ? "" : ", ", evsel__name(pos)); + first = false; + } + + if (!evsel->forced_leader) + ret += scnprintf(buf + ret, size - ret, " }"); + + return ret; +} + +uint16_t evsel__e_machine(struct evsel *evsel, uint32_t *e_flags) +{ + struct perf_session *session = evsel__session(evsel); + + return perf_session__e_machine(session, e_flags); +} + +static void __evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *param) +{ + bool function = evsel__is_function_event(evsel); + struct perf_event_attr *attr = &evsel->core.attr; + + if (EM_HOST == EM_S390 && param->record_mode == CALLCHAIN_FP) { + pr_warning_once( + "Framepointer unwinding lacks kernel support. Use '--call-graph dwarf'\n"); + } + + evsel__set_sample_bit(evsel, CALLCHAIN); + + attr->sample_max_stack = param->max_stack; + + if (opts->kernel_callchains) + attr->exclude_callchain_user = 1; + if (opts->user_callchains) + attr->exclude_callchain_kernel = 1; + if (param->record_mode == CALLCHAIN_LBR) { + if (!opts->branch_stack) { + if (attr->exclude_user) { + pr_warning("LBR callstack option is only available " + "to get user callchain information. " + "Falling back to framepointers.\n"); + } else { + evsel__set_sample_bit(evsel, BRANCH_STACK); + attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER | + PERF_SAMPLE_BRANCH_CALL_STACK | + PERF_SAMPLE_BRANCH_NO_CYCLES | + PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_HW_INDEX; + } + } else + pr_warning("Cannot use LBR callstack with branch stack. " + "Falling back to framepointers.\n"); + } + + if (param->record_mode == CALLCHAIN_DWARF) { + if (!function) { + uint16_t e_machine = evsel__e_machine(evsel, /*e_flags=*/NULL); + + evsel__set_sample_bit(evsel, REGS_USER); + evsel__set_sample_bit(evsel, STACK_USER); + if (opts->sample_user_regs && + DWARF_MINIMAL_REGS(e_machine) != perf_user_reg_mask(EM_HOST)) { + attr->sample_regs_user |= DWARF_MINIMAL_REGS(e_machine); + pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, " + "specifying a subset with --user-regs may render DWARF unwinding unreliable, " + "so the minimal registers set (IP, SP) is explicitly forced.\n"); + } else { + attr->sample_regs_user |= perf_user_reg_mask(EM_HOST); + } + attr->sample_stack_user = param->dump_size; + attr->exclude_callchain_user = 1; + } else { + pr_info("Cannot use DWARF unwind for function trace event," + " falling back to framepointers.\n"); + } + } + + if (function) { + pr_info("Disabling user space callchains for function trace event.\n"); + attr->exclude_callchain_user = 1; + } + + if (param->defer && !attr->exclude_callchain_user) + attr->defer_callchain = 1; +} + +void evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *param) +{ + if (param->enabled) + return __evsel__config_callchain(evsel, opts, param); +} + +static void evsel__reset_callgraph(struct evsel *evsel, const struct callchain_param *param) +{ + struct perf_event_attr *attr = &evsel->core.attr; + + evsel__reset_sample_bit(evsel, CALLCHAIN); + if (param->record_mode == CALLCHAIN_LBR) { + evsel__reset_sample_bit(evsel, BRANCH_STACK); + attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER | + PERF_SAMPLE_BRANCH_CALL_STACK | + PERF_SAMPLE_BRANCH_HW_INDEX); + } + if (param->record_mode == CALLCHAIN_DWARF) { + evsel__reset_sample_bit(evsel, REGS_USER); + evsel__reset_sample_bit(evsel, STACK_USER); + } +} + +static void evsel__apply_ratio_to_prev(struct evsel *evsel, + struct perf_event_attr *attr, + const struct record_opts *opts, + const char *buf) +{ + struct perf_event_attr *prev_attr = NULL; + struct evsel *evsel_prev = NULL; + u64 type = evsel->core.attr.sample_type; + u64 prev_type = 0; + double rtp; + + rtp = strtod(buf, NULL); + if (rtp <= 0) { + pr_err("Invalid ratio-to-prev value %lf\n", rtp); + return; + } + if (evsel == evsel__leader(evsel)) { + pr_err("Invalid use of ratio-to-prev term without preceding element in group\n"); + return; + } + if (!evsel->pmu->is_core) { + pr_err("Event using ratio-to-prev term must have a core PMU\n"); + return; + } + + evsel_prev = evsel__prev(evsel); + if (!evsel_prev) { + pr_err("Previous event does not exist.\n"); + return; + } + + if (evsel_prev->pmu->type != evsel->pmu->type) { + pr_err("Compared events (\"%s\", \"%s\") must have same PMU\n", + evsel->name, evsel_prev->name); + return; + } + + prev_attr = &evsel_prev->core.attr; + prev_type = evsel_prev->core.attr.sample_type; + + if (!(prev_type & PERF_SAMPLE_PERIOD)) { + attr->sample_period = prev_attr->sample_period * rtp; + attr->freq = 0; + evsel__reset_sample_bit(evsel, PERIOD); + } else if (!(type & PERF_SAMPLE_PERIOD)) { + prev_attr->sample_period = attr->sample_period / rtp; + prev_attr->freq = 0; + evsel__reset_sample_bit(evsel_prev, PERIOD); + } else { + if (opts->user_interval != ULLONG_MAX) { + prev_attr->sample_period = opts->user_interval; + attr->sample_period = prev_attr->sample_period * rtp; + prev_attr->freq = 0; + attr->freq = 0; + evsel__reset_sample_bit(evsel_prev, PERIOD); + evsel__reset_sample_bit(evsel, PERIOD); + } else { + pr_err("Event period term or count (-c) must be set when using ratio-to-prev term.\n"); + return; + } + } + + arch_evsel__apply_ratio_to_prev(evsel, attr); +} + +static void evsel__apply_config_terms(struct evsel *evsel, + const struct record_opts *opts, bool track) +{ + struct evsel_config_term *term; + struct list_head *config_terms = &evsel->config_terms; + struct perf_event_attr *attr = &evsel->core.attr; + /* callgraph default */ + struct callchain_param param = { + .record_mode = callchain_param.record_mode, + }; + u32 dump_size = 0; + int max_stack = 0; + const char *callgraph_buf = NULL; + const char *rtp_buf = NULL; + + list_for_each_entry(term, config_terms, list) { + switch (term->type) { + case EVSEL__CONFIG_TERM_PERIOD: + if (!(term->weak && opts->user_interval != ULLONG_MAX)) { + attr->sample_period = term->val.period; + attr->freq = 0; + evsel__reset_sample_bit(evsel, PERIOD); + } + break; + case EVSEL__CONFIG_TERM_FREQ: + if (!(term->weak && opts->user_freq != UINT_MAX)) { + attr->sample_freq = term->val.freq; + attr->freq = 1; + evsel__set_sample_bit(evsel, PERIOD); + } + break; + case EVSEL__CONFIG_TERM_TIME: + if (term->val.time) + evsel__set_sample_bit(evsel, TIME); + else + evsel__reset_sample_bit(evsel, TIME); + break; + case EVSEL__CONFIG_TERM_CALLGRAPH: + callgraph_buf = term->val.str; + break; + case EVSEL__CONFIG_TERM_BRANCH: + if (term->val.str && strcmp(term->val.str, "no")) { + evsel__set_sample_bit(evsel, BRANCH_STACK); + parse_branch_str(term->val.str, + &attr->branch_sample_type); + } else + evsel__reset_sample_bit(evsel, BRANCH_STACK); + break; + case EVSEL__CONFIG_TERM_STACK_USER: + dump_size = term->val.stack_user; + break; + case EVSEL__CONFIG_TERM_MAX_STACK: + max_stack = term->val.max_stack; + break; + case EVSEL__CONFIG_TERM_MAX_EVENTS: + evsel->max_events = term->val.max_events; + break; + case EVSEL__CONFIG_TERM_INHERIT: + /* + * attr->inherit should has already been set by + * evsel__config. If user explicitly set + * inherit using config terms, override global + * opt->no_inherit setting. + */ + attr->inherit = term->val.inherit ? 1 : 0; + break; + case EVSEL__CONFIG_TERM_OVERWRITE: + attr->write_backward = term->val.overwrite ? 1 : 0; + break; + case EVSEL__CONFIG_TERM_DRV_CFG: + break; + case EVSEL__CONFIG_TERM_PERCORE: + break; + case EVSEL__CONFIG_TERM_AUX_OUTPUT: + attr->aux_output = term->val.aux_output ? 1 : 0; + break; + case EVSEL__CONFIG_TERM_AUX_ACTION: + /* Already applied by auxtrace */ + break; + case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE: + /* Already applied by auxtrace */ + break; + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG1: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG2: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG3: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG4: + break; + case EVSEL__CONFIG_TERM_RATIO_TO_PREV: + rtp_buf = term->val.str; + break; + default: + break; + } + } + + /* User explicitly set per-event callgraph, clear the old setting and reset. */ + if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) { + bool sample_address = false; + + if (max_stack) { + param.max_stack = max_stack; + if (callgraph_buf == NULL) + callgraph_buf = "fp"; + } + + /* parse callgraph parameters */ + if (callgraph_buf != NULL) { + if (!strcmp(callgraph_buf, "no")) { + param.enabled = false; + param.record_mode = CALLCHAIN_NONE; + } else { + param.enabled = true; + if (parse_callchain_record(callgraph_buf, ¶m)) { + pr_err("per-event callgraph setting for %s failed. " + "Apply callgraph global setting for it\n", + evsel->name); + return; + } + if (param.record_mode == CALLCHAIN_DWARF) + sample_address = true; + } + } + if (dump_size > 0) { + dump_size = round_up(dump_size, sizeof(u64)); + param.dump_size = dump_size; + } + + /* If global callgraph set, clear it */ + if (callchain_param.enabled) + evsel__reset_callgraph(evsel, &callchain_param); + + /* set perf-event callgraph */ + if (param.enabled) { + if (sample_address) { + evsel__set_sample_bit(evsel, ADDR); + evsel__set_sample_bit(evsel, DATA_SRC); + evsel->core.attr.mmap_data = track; + } + evsel__config_callchain(evsel, opts, ¶m); + } + } + if (rtp_buf) + evsel__apply_ratio_to_prev(evsel, attr, opts, rtp_buf); +} + +struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type) +{ + struct evsel_config_term *term, *found_term = NULL; + + list_for_each_entry(term, &evsel->config_terms, list) { + if (term->type == type) + found_term = term; + } + + return found_term; +} + +/* + * Set @config_name to @val as long as the user hasn't already set or cleared it + * by passing a config term on the command line. + * + * @val is the value to put into the bits specified by @config_name rather than + * the bit pattern. It is shifted into position by this function, so to set + * something to true, pass 1 for val rather than a pre shifted value. + */ +void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, + u64 val) +{ + u64 user_bits = 0; + struct evsel_config_term *term = evsel__get_config_term(evsel, + USR_CHG_CONFIG); + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, + config_name); + int fbit; + __u64 *vp; + + if (!format) + return; + + switch (format->value) { + case PERF_PMU_FORMAT_VALUE_CONFIG: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG); + vp = &evsel->core.attr.config; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG1: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG1); + vp = &evsel->core.attr.config1; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG2: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG2); + vp = &evsel->core.attr.config2; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG3: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG3); + vp = &evsel->core.attr.config3; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG4: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG4); + vp = &evsel->core.attr.config4; + break; + default: + pr_err("Unknown format value: %d\n", format->value); + return; + } + + if (!format) + return; + + if (term) + user_bits = term->val.cfg_chg; + + /* Do nothing if the user changed the value */ + for_each_set_bit(fbit, format->bits, PERF_PMU_FORMAT_BITS) + if ((1ULL << fbit) & user_bits) + return; + + /* Otherwise replace it */ + perf_pmu__format_pack(format->bits, val, vp, /*zero=*/true); +} + +bool evsel__config_exists(const struct evsel *evsel, const char *config_name) +{ + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); + + return format && !bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS); +} + +int evsel__get_config_val(const struct evsel *evsel, const char *config_name, + u64 *val) +{ + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); + + if (!format || bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS)) { + pr_err("Unknown/empty format name: %s\n", config_name); + *val = 0; + return -EINVAL; + } + + switch (format->value) { + case PERF_PMU_FORMAT_VALUE_CONFIG: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG1: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config1); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG2: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config2); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG3: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config3); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG4: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config4); + return 0; + default: + pr_err("Unknown format value: %d\n", format->value); + *val = 0; + return -EINVAL; + } +} + +void __weak arch_evsel__set_sample_weight(struct evsel *evsel) +{ + evsel__set_sample_bit(evsel, WEIGHT); +} + +void __weak arch__post_evsel_config(struct evsel *evsel __maybe_unused, + struct perf_event_attr *attr __maybe_unused) +{ +} + +void __weak arch_evsel__apply_ratio_to_prev(struct evsel *evsel __maybe_unused, + struct perf_event_attr *attr __maybe_unused) +{ +} + +static void evsel__set_default_freq_period(const struct record_opts *opts, + struct perf_event_attr *attr) +{ + if (opts->freq) { + attr->freq = 1; + attr->sample_freq = opts->freq; + } else { + attr->sample_period = opts->default_interval; + } +} + +bool evsel__is_offcpu_event(struct evsel *evsel) +{ + return evsel__is_bpf_output(evsel) && evsel__name_is(evsel, OFFCPU_EVENT) && + evsel->core.attr.sample_type & PERF_SAMPLE_RAW; +} + +/* + * The enable_on_exec/disabled value strategy: + * + * 1) For any type of traced program: + * - all independent events and group leaders are disabled + * - all group members are enabled + * + * Group members are ruled by group leaders. They need to + * be enabled, because the group scheduling relies on that. + * + * 2) For traced programs executed by perf: + * - all independent events and group leaders have + * enable_on_exec set + * - we don't specifically enable or disable any event during + * the record command + * + * Independent events and group leaders are initially disabled + * and get enabled by exec. Group members are ruled by group + * leaders as stated in 1). + * + * 3) For traced programs attached by perf (pid/tid): + * - we specifically enable or disable all events during + * the record command + * + * When attaching events to already running traced we + * enable/disable events specifically, as there's no + * initial traced exec call. + */ +void evsel__config(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *callchain) +{ + struct evsel *leader = evsel__leader(evsel); + struct perf_event_attr *attr = &evsel->core.attr; + int track = evsel->tracking; + bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread; + + attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1; + attr->inherit = target__has_cpu(&opts->target) ? 0 : !opts->no_inherit; + attr->write_backward = opts->overwrite ? 1 : 0; + attr->read_format = PERF_FORMAT_LOST; + + evsel__set_sample_bit(evsel, IP); + evsel__set_sample_bit(evsel, TID); + + if (evsel->sample_read) { + evsel__set_sample_bit(evsel, READ); + + /* + * We need ID even in case of single event, because + * PERF_SAMPLE_READ process ID specific data. + */ + evsel__set_sample_id(evsel, false); + + /* + * Apply group format only if we belong to group + * with more than one members. + */ + if (leader->core.nr_members > 1) { + attr->read_format |= PERF_FORMAT_GROUP; + } + + /* + * Inherit + SAMPLE_READ requires SAMPLE_TID in the read_format + */ + if (attr->inherit) { + evsel__set_sample_bit(evsel, TID); + evsel->core.attr.read_format |= + PERF_FORMAT_ID; + } + } + + /* + * We default some events to have a default interval. But keep + * it a weak assumption overridable by the user. + */ + if ((evsel->is_libpfm_event && !attr->sample_period) || + (!evsel->is_libpfm_event && (!attr->sample_period || + opts->user_freq != UINT_MAX || + opts->user_interval != ULLONG_MAX))) + evsel__set_default_freq_period(opts, attr); + + /* + * If attr->freq was set (here or earlier), ask for period + * to be sampled. + */ + if (attr->freq) + evsel__set_sample_bit(evsel, PERIOD); + + if (opts->no_samples) + attr->sample_freq = 0; + + if (opts->inherit_stat) { + evsel->core.attr.read_format |= + PERF_FORMAT_TOTAL_TIME_ENABLED | + PERF_FORMAT_TOTAL_TIME_RUNNING | + PERF_FORMAT_ID; + attr->inherit_stat = 1; + } + + if (opts->sample_address) + evsel__set_sample_bit(evsel, ADDR); + + if (opts->record_data_mmap) + attr->mmap_data = track; + + /* + * We don't allow user space callchains for function trace + * event, due to issues with page faults while tracing page + * fault handler and its overall trickiness nature. + */ + if (evsel__is_function_event(evsel)) + evsel->core.attr.exclude_callchain_user = 1; + + if (callchain && callchain->enabled && !evsel->no_aux_samples) + evsel__config_callchain(evsel, opts, callchain); + + if (opts->sample_intr_regs && !evsel->no_aux_samples && + !evsel__is_dummy_event(evsel)) { + attr->sample_regs_intr = opts->sample_intr_regs; + evsel__set_sample_bit(evsel, REGS_INTR); + } + + if (opts->sample_user_regs && !evsel->no_aux_samples && + !evsel__is_dummy_event(evsel)) { + attr->sample_regs_user |= opts->sample_user_regs; + evsel__set_sample_bit(evsel, REGS_USER); + } + + if (target__has_cpu(&opts->target) || opts->sample_cpu) + evsel__set_sample_bit(evsel, CPU); + + /* + * When the user explicitly disabled time don't force it here. + */ + if (opts->sample_time && + (!perf_missing_features.sample_id_all && + (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu || + opts->sample_time_set))) + evsel__set_sample_bit(evsel, TIME); + + if (opts->raw_samples && !evsel->no_aux_samples) { + evsel__set_sample_bit(evsel, TIME); + evsel__set_sample_bit(evsel, RAW); + evsel__set_sample_bit(evsel, CPU); + } + + if (opts->sample_data_src) + evsel__set_sample_bit(evsel, DATA_SRC); + + if (opts->sample_phys_addr) + evsel__set_sample_bit(evsel, PHYS_ADDR); + + if (opts->no_buffering) { + attr->watermark = 0; + attr->wakeup_events = 1; + } + if (opts->branch_stack && !evsel->no_aux_samples) { + evsel__set_sample_bit(evsel, BRANCH_STACK); + attr->branch_sample_type = opts->branch_stack; + } + + if (opts->sample_weight || evsel->retire_lat) { + arch_evsel__set_sample_weight(evsel); + evsel->retire_lat = false; + } + attr->task = track; + attr->mmap = track; + attr->mmap2 = track && !perf_missing_features.mmap2; + attr->comm = track; + attr->build_id = track && opts->build_id; + attr->defer_output = track && callchain && callchain->defer; + + /* + * ksymbol is tracked separately with text poke because it needs to be + * system wide and enabled immediately. + */ + if (!opts->text_poke) + attr->ksymbol = track && !perf_missing_features.ksymbol; + attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf; + + if (opts->record_namespaces) + attr->namespaces = track; + + if (opts->record_cgroup) { + attr->cgroup = track && !perf_missing_features.cgroup; + evsel__set_sample_bit(evsel, CGROUP); + } + + if (opts->sample_data_page_size) + evsel__set_sample_bit(evsel, DATA_PAGE_SIZE); + + if (opts->sample_code_page_size) + evsel__set_sample_bit(evsel, CODE_PAGE_SIZE); + + if (opts->record_switch_events) + attr->context_switch = track; + + if (opts->sample_transaction) + evsel__set_sample_bit(evsel, TRANSACTION); + + if (opts->running_time) { + evsel->core.attr.read_format |= + PERF_FORMAT_TOTAL_TIME_ENABLED | + PERF_FORMAT_TOTAL_TIME_RUNNING; + } + + /* + * XXX see the function comment above + * + * Disabling only independent events or group leaders, + * keeping group members enabled. + */ + if (evsel__is_group_leader(evsel)) + attr->disabled = 1; + + /* + * Setting enable_on_exec for independent events and + * group leaders for traced executed by perf. + */ + if (target__none(&opts->target) && evsel__is_group_leader(evsel) && + !opts->target.initial_delay) + attr->enable_on_exec = 1; + + if (evsel->immediate) { + attr->disabled = 0; + attr->enable_on_exec = 0; + } + + clockid = opts->clockid; + if (opts->use_clockid) { + attr->use_clockid = 1; + attr->clockid = opts->clockid; + } + + if (evsel->precise_max) + attr->precise_ip = 3; + + if (opts->all_user) { + attr->exclude_kernel = 1; + attr->exclude_user = 0; + } + + if (opts->all_kernel) { + attr->exclude_kernel = 0; + attr->exclude_user = 1; + } + + if (evsel->core.pmu_cpus || evsel->unit) + evsel->core.attr.read_format |= PERF_FORMAT_ID; + + /* + * Apply event specific term settings, + * it overloads any global configuration. + */ + evsel__apply_config_terms(evsel, opts, track); + + evsel->ignore_missing_thread = opts->ignore_missing_thread; + + /* The --period option takes the precedence. */ + if (opts->period_set) { + if (opts->period) + evsel__set_sample_bit(evsel, PERIOD); + else + evsel__reset_sample_bit(evsel, PERIOD); + } + + /* + * A dummy event never triggers any actual counter and therefore + * cannot be used with branch_stack. + * + * For initial_delay, a dummy event is added implicitly. + * The software event will trigger -EOPNOTSUPP error out, + * if BRANCH_STACK bit is set. + */ + if (evsel__is_dummy_event(evsel)) + evsel__reset_sample_bit(evsel, BRANCH_STACK); + + if (evsel__is_offcpu_event(evsel)) { + evsel->core.attr.sample_type &= OFFCPU_SAMPLE_TYPES; + attr->inherit = 0; + } + + arch__post_evsel_config(evsel, attr); +} + +int evsel__set_filter(struct evsel *evsel, const char *filter) +{ + char *new_filter = strdup(filter); + + if (new_filter != NULL) { + free(evsel->filter); + evsel->filter = new_filter; + return 0; + } + + return -1; +} + +static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter) +{ + char *new_filter; + + if (evsel->filter == NULL) + return evsel__set_filter(evsel, filter); + + if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) { + free(evsel->filter); + evsel->filter = new_filter; + return 0; + } + + return -1; +} + +int evsel__append_tp_filter(struct evsel *evsel, const char *filter) +{ + return evsel__append_filter(evsel, "(%s) && (%s)", filter); +} + +int evsel__append_addr_filter(struct evsel *evsel, const char *filter) +{ + return evsel__append_filter(evsel, "%s,%s", filter); +} + +/* Caller has to clear disabled after going through all CPUs. */ +int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; +} + +int evsel__enable(struct evsel *evsel) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable(evsel); + else + err = perf_evsel__enable(&evsel->core); + + if (!err) + evsel->disabled = false; + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable(member); + + if (mem_err) + return mem_err; + } + member->disabled = false; + } + } + + return err; +} + +/* Caller has to set disabled after going through all CPUs. */ +int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; +} + +int evsel__disable(struct evsel *evsel) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable(evsel); + else + err = perf_evsel__disable(&evsel->core); + + /* + * We mark it disabled here so that tools that disable a event can + * ignore events after they disable it. I.e. the ring buffer may have + * already a few more events queued up before the kernel got the stop + * request. + */ + if (!err) + evsel->disabled = true; + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable(member); + + if (mem_err) + return mem_err; + } + member->disabled = true; + } + } + + return err; +} + +void free_config_terms(struct list_head *config_terms) +{ + struct evsel_config_term *term, *h; + + list_for_each_entry_safe(term, h, config_terms, list) { + list_del_init(&term->list); + if (term->free_str) + zfree(&term->val.str); + free(term); + } +} + +static void evsel__free_config_terms(struct evsel *evsel) +{ + free_config_terms(&evsel->config_terms); +} + +static void (*evsel__priv_destructor)(void *priv); + +void evsel__set_priv_destructor(void (*destructor)(void *priv)) +{ + assert(evsel__priv_destructor == NULL); + + evsel__priv_destructor = destructor; +} + +static void evsel__exit(struct evsel *evsel) +{ + assert(list_empty(&evsel->core.node)); + assert(evsel->evlist == NULL); + if (evsel__is_retire_lat(evsel)) + evsel__tpebs_close(evsel); + bpf_counter__destroy(evsel); + perf_bpf_filter__destroy(evsel); + evsel__free_counts(evsel); + perf_evsel__free_fd(&evsel->core); + perf_evsel__free_id(&evsel->core); + evsel__free_config_terms(evsel); + cgroup__put(evsel->cgrp); + perf_evsel__exit(&evsel->core); + zfree(&evsel->group_name); + zfree(&evsel->name); +#ifdef HAVE_LIBTRACEEVENT + zfree(&evsel->tp_sys); + zfree(&evsel->tp_name); +#endif + zfree(&evsel->filter); + zfree(&evsel->group_pmu_name); + zfree(&evsel->unit); + zfree(&evsel->metric_id); + evsel__zero_per_pkg(evsel); + hashmap__free(evsel->per_pkg_mask); + evsel->per_pkg_mask = NULL; + if (evsel__priv_destructor) + evsel__priv_destructor(evsel->priv); + perf_evsel__object.fini(evsel); + if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || + evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) { + xyarray__delete(evsel->process_time.start_times); + xyarray__delete(evsel->process_time.accumulated_times); + } +} + +void evsel__put(struct evsel *evsel) +{ + if (!evsel) + return; + + if (!refcount_dec_and_test(&evsel->refcnt)) + return; + + evsel__exit(evsel); + free(evsel); +} + +void evsel__compute_deltas(struct evsel *evsel, int cpu_map_idx, int thread, + struct perf_counts_values *count) +{ + struct perf_counts_values tmp; + + if (!evsel->prev_raw_counts) + return; + + tmp = *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); + *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread) = *count; + + count->val = count->val - tmp.val; + count->ena = count->ena - tmp.ena; + count->run = count->run - tmp.run; +} + +static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread) +{ + struct perf_counts_values *count = perf_counts(evsel->counts, cpu_map_idx, thread); + + return perf_evsel__read(&evsel->core, cpu_map_idx, thread, count); +} + +static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread, + u64 val, u64 ena, u64 run, u64 lost) +{ + struct perf_counts_values *count; + + count = perf_counts(counter->counts, cpu_map_idx, thread); + + if (evsel__is_retire_lat(counter)) { + evsel__tpebs_read(counter, cpu_map_idx, thread); + perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true); + return; + } + + count->val = val; + count->ena = ena; + count->run = run; + count->lost = lost; + + perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true); +} + +static bool evsel__group_has_tpebs(struct evsel *leader) +{ + struct evsel *evsel; + + for_each_group_evsel(evsel, leader) { + if (evsel__is_retire_lat(evsel)) + return true; + } + return false; +} + +static u64 evsel__group_read_nr_members(struct evsel *leader) +{ + u64 nr = leader->core.nr_members; + struct evsel *evsel; + + for_each_group_evsel(evsel, leader) { + if (evsel__is_retire_lat(evsel)) + nr--; + } + return nr; +} + +static u64 evsel__group_read_size(struct evsel *leader) +{ + u64 read_format = leader->core.attr.read_format; + int entry = sizeof(u64); /* value */ + int size = 0; + int nr = 1; + + if (!evsel__group_has_tpebs(leader)) + return perf_evsel__read_size(&leader->core); + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + size += sizeof(u64); + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + size += sizeof(u64); + + if (read_format & PERF_FORMAT_ID) + entry += sizeof(u64); + + if (read_format & PERF_FORMAT_LOST) + entry += sizeof(u64); + + if (read_format & PERF_FORMAT_GROUP) { + nr = evsel__group_read_nr_members(leader); + size += sizeof(u64); + } + + size += entry * nr; + return size; +} + +static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int thread, u64 *data) +{ + u64 read_format = leader->core.attr.read_format; + struct sample_read_value *v; + u64 nr, ena = 0, run = 0, lost = 0; + + nr = *data++; + + if (nr != evsel__group_read_nr_members(leader)) + return -EINVAL; + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + ena = *data++; + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + run = *data++; + + v = (void *)data; + sample_read_group__for_each(v, nr, read_format) { + struct evsel *counter; + + counter = evlist__id2evsel(leader->evlist, v->id); + if (!counter) + return -EINVAL; + + if (read_format & PERF_FORMAT_LOST) + lost = v->lost; + + evsel__set_count(counter, cpu_map_idx, thread, v->value, ena, run, lost); + } + + return 0; +} + +static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread) +{ + struct perf_stat_evsel *ps = leader->stats; + u64 read_format = leader->core.attr.read_format; + int size = evsel__group_read_size(leader); + u64 *data = ps->group_data; + + if (!(read_format & PERF_FORMAT_ID)) + return -EINVAL; + + if (!evsel__is_group_leader(leader)) + return -EINVAL; + + if (!data) { + data = zalloc(size); + if (!data) + return -ENOMEM; + + ps->group_data = data; + } + + if (FD(leader, cpu_map_idx, thread) < 0) + return -EINVAL; + + if (readn(FD(leader, cpu_map_idx, thread), data, size) <= 0) + return -errno; + + return evsel__process_group_data(leader, cpu_map_idx, thread, data); +} + +bool __evsel__match(const struct evsel *evsel, u32 type, u64 config) +{ + + u32 e_type = evsel->core.attr.type; + u64 e_config = evsel->core.attr.config; + + if (e_type == type && e_config == config) + return true; + if (type != PERF_TYPE_HARDWARE && type != PERF_TYPE_HW_CACHE) + return false; + if ((e_type == PERF_TYPE_HARDWARE || e_type == PERF_TYPE_HW_CACHE) && + perf_pmus__supports_extended_type()) + e_config &= PERF_HW_EVENT_MASK; + if (e_type == type && e_config == config) + return true; + if (type == PERF_TYPE_HARDWARE && evsel->pmu && evsel->pmu->is_core && + evsel->alternate_hw_config == config) + return true; + return false; +} + +int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread) +{ + if (evsel__is_tool(evsel)) + return evsel__tool_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_hwmon(evsel)) + return evsel__hwmon_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_drm(evsel)) + return evsel__drm_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_retire_lat(evsel)) + return evsel__tpebs_read(evsel, cpu_map_idx, thread); + + if (evsel->core.attr.read_format & PERF_FORMAT_GROUP) + return evsel__read_group(evsel, cpu_map_idx, thread); + + return evsel__read_one(evsel, cpu_map_idx, thread); +} + +int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale) +{ + struct perf_counts_values count; + size_t nv = scale ? 3 : 1; + + if (FD(evsel, cpu_map_idx, thread) < 0) + return -EINVAL; + + if (evsel->counts == NULL && evsel__alloc_counts(evsel) < 0) + return -ENOMEM; + + if (readn(FD(evsel, cpu_map_idx, thread), &count, nv * sizeof(u64)) <= 0) + return -errno; + + evsel__compute_deltas(evsel, cpu_map_idx, thread, &count); + perf_counts_values__scale(&count, scale, NULL); + *perf_counts(evsel->counts, cpu_map_idx, thread) = count; + return 0; +} + +static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other, + int cpu_map_idx) +{ + struct perf_cpu cpu; + + cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx); + return perf_cpu_map__idx(other->core.cpus, cpu); +} + +static int evsel__hybrid_group_cpu_map_idx(struct evsel *evsel, int cpu_map_idx) +{ + struct evsel *leader = evsel__leader(evsel); + + if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) || + (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) { + return evsel__match_other_cpu(evsel, leader, cpu_map_idx); + } + + return cpu_map_idx; +} + +static int get_group_fd(struct evsel *evsel, int cpu_map_idx, int thread) +{ + struct evsel *leader = evsel__leader(evsel); + int fd; + + if (!evsel->supported || evsel__is_group_leader(evsel)) + return -1; + + /* + * Leader must be already processed/open, + * if not it's a bug. + */ + BUG_ON(!leader->core.fd); + + cpu_map_idx = evsel__hybrid_group_cpu_map_idx(evsel, cpu_map_idx); + if (cpu_map_idx == -1) + return -1; + + fd = FD(leader, cpu_map_idx, thread); + BUG_ON(fd == -1 && leader->supported); + + /* + * When the leader has been skipped, return -2 to distinguish from no + * group leader case. + */ + return fd == -1 ? -2 : fd; +} + +static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx) +{ + for (int cpu = 0; cpu < nr_cpus; cpu++) + for (int thread = thread_idx; thread < nr_threads - 1; thread++) + FD(pos, cpu, thread) = FD(pos, cpu, thread + 1); +} + +static int update_fds(struct evsel *evsel, + int nr_cpus, int cpu_map_idx, + int nr_threads, int thread_idx) +{ + struct evsel *pos; + + if (cpu_map_idx >= nr_cpus || thread_idx >= nr_threads) + return -EINVAL; + + evlist__for_each_entry(evsel->evlist, pos) { + nr_cpus = pos != evsel ? nr_cpus : cpu_map_idx; + + evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx); + + /* + * Since fds for next evsel has not been created, + * there is no need to iterate whole event list. + */ + if (pos == evsel) + break; + } + return 0; +} + +static bool evsel__ignore_missing_thread(struct evsel *evsel, + int nr_cpus, int cpu_map_idx, + struct perf_thread_map *threads, + int thread, int err) +{ + pid_t ignore_pid = perf_thread_map__pid(threads, thread); + + if (!evsel->ignore_missing_thread) + return false; + + /* The system wide setup does not work with threads. */ + if (evsel->core.system_wide) + return false; + + /* The -ESRCH is perf event syscall errno for pid's not found. */ + if (err != -ESRCH) + return false; + + /* If there's only one thread, let it fail. */ + if (threads->nr == 1) + return false; + + /* + * We should remove fd for missing_thread first + * because thread_map__remove() will decrease threads->nr. + */ + if (update_fds(evsel, nr_cpus, cpu_map_idx, threads->nr, thread)) + return false; + + if (thread_map__remove(threads, thread)) + return false; + + pr_warning("WARNING: Ignored open failure for pid %d\n", + ignore_pid); + return true; +} + +static int __open_attr__fprintf(FILE *fp, const char *name, const char *val, + void *priv __maybe_unused) +{ + return fprintf(fp, " %-32s %s\n", name, val); +} + +static void display_attr(struct perf_event_attr *attr) +{ + if (verbose >= 2 || debug_peo_args) { + fprintf(stderr, "%.60s\n", graph_dotted_line); + fprintf(stderr, "perf_event_attr:\n"); + perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL); + fprintf(stderr, "%.60s\n", graph_dotted_line); + } +} + +bool evsel__precise_ip_fallback(struct evsel *evsel) +{ + /* Do not try less precise if not requested. */ + if (!evsel->precise_max) + return false; + + /* + * We tried all the precise_ip values, and it's + * still failing, so leave it to standard fallback. + */ + if (!evsel->core.attr.precise_ip) { + evsel->core.attr.precise_ip = evsel->precise_ip_original; + return false; + } + + if (!evsel->precise_ip_original) + evsel->precise_ip_original = evsel->core.attr.precise_ip; + + evsel->core.attr.precise_ip--; + pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip); + display_attr(&evsel->core.attr); + return true; +} + +static struct perf_cpu_map *empty_cpu_map; +static struct perf_thread_map *empty_thread_map; + +static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + int ret = 0; + int nthreads = perf_thread_map__nr(threads); + + if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) || + (perf_missing_features.aux_output && evsel->core.attr.aux_output)) + return -EINVAL; + + if (cpus == NULL) { + if (empty_cpu_map == NULL) { + empty_cpu_map = perf_cpu_map__new_any_cpu(); + if (empty_cpu_map == NULL) + return -ENOMEM; + } + + cpus = empty_cpu_map; + } + + if (threads == NULL) { + if (empty_thread_map == NULL) { + empty_thread_map = thread_map__new_by_tid(-1); + if (empty_thread_map == NULL) + return -ENOMEM; + } + + threads = empty_thread_map; + } + + if (evsel->core.fd == NULL && + perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0) + return -ENOMEM; + + if (evsel__is_tool(evsel)) + ret = evsel__tool_pmu_prepare_open(evsel, cpus, nthreads); + + evsel->open_flags = PERF_FLAG_FD_CLOEXEC; + if (evsel->cgrp) + evsel->open_flags |= PERF_FLAG_PID_CGROUP; + + return ret; +} + +static void evsel__disable_missing_features(struct evsel *evsel) +{ + if (perf_missing_features.defer_callchain && evsel->core.attr.defer_callchain) + evsel->core.attr.defer_callchain = 0; + if (perf_missing_features.defer_callchain && evsel->core.attr.defer_output) + evsel->core.attr.defer_output = 0; + if (perf_missing_features.inherit_sample_read && evsel->core.attr.inherit && + (evsel->core.attr.sample_type & PERF_SAMPLE_READ)) + evsel->core.attr.inherit = 0; + if (perf_missing_features.branch_counters) + evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_COUNTERS; + if (perf_missing_features.read_lost) + evsel->core.attr.read_format &= ~PERF_FORMAT_LOST; + if (perf_missing_features.weight_struct) { + evsel__set_sample_bit(evsel, WEIGHT); + evsel__reset_sample_bit(evsel, WEIGHT_STRUCT); + } + if (perf_missing_features.clockid_wrong) + evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */ + if (perf_missing_features.clockid) { + evsel->core.attr.use_clockid = 0; + evsel->core.attr.clockid = 0; + } + if (perf_missing_features.cloexec) + evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC; + if (perf_missing_features.mmap2) + evsel->core.attr.mmap2 = 0; + if (evsel->pmu && evsel->pmu->missing_features.exclude_guest) + evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0; + if (perf_missing_features.lbr_flags) + evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_NO_CYCLES); + if (perf_missing_features.group_read && evsel->core.attr.inherit) + evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID); + if (perf_missing_features.ksymbol) + evsel->core.attr.ksymbol = 0; + if (perf_missing_features.bpf) + evsel->core.attr.bpf_event = 0; + if (perf_missing_features.branch_hw_idx) + evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX; + if (perf_missing_features.sample_id_all) + evsel->core.attr.sample_id_all = 0; +} + +int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + int err; + + err = __evsel__prepare_open(evsel, cpus, threads); + if (err) + return err; + + evsel__disable_missing_features(evsel); + + return err; +} + +static bool __has_attr_feature(struct perf_event_attr *attr, + struct perf_cpu cpu, unsigned long flags) +{ + int fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + + if (fd < 0) { + attr->exclude_kernel = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + } + + if (fd < 0) { + attr->exclude_hv = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + } + + if (fd < 0) { + attr->exclude_guest = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + } + + attr->exclude_kernel = 0; + attr->exclude_guest = 0; + attr->exclude_hv = 0; + + return fd >= 0; +} + +static bool has_attr_feature(struct perf_event_attr *attr, unsigned long flags) +{ + struct perf_cpu cpu = {.cpu = -1}; + + return __has_attr_feature(attr, cpu, flags); +} + +static void evsel__detect_missing_pmu_features(struct evsel *evsel) +{ + struct perf_event_attr attr = { + .type = evsel->core.attr.type, + .config = evsel->core.attr.config, + .disabled = 1, + }; + struct perf_pmu *pmu = evsel->pmu; + int old_errno; + + old_errno = errno; + + if (pmu == NULL) + pmu = evsel->pmu = evsel__find_pmu(evsel); + + if (pmu == NULL || pmu->missing_features.checked) + goto out; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are kernel core limitation but + * specific to PMUs with branch stack. So we can detect with the given + * hardware event and stop on the first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.exclude_guest = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + pmu->missing_features.exclude_guest = true; + pr_debug2("switching off exclude_guest for PMU %s\n", pmu->name); + +found: + pmu->missing_features.checked = true; +out: + errno = old_errno; +} + +static void evsel__detect_missing_brstack_features(struct evsel *evsel) +{ + static bool detection_done = false; + struct perf_event_attr attr = { + .type = evsel->core.attr.type, + .config = evsel->core.attr.config, + .disabled = 1, + .sample_type = PERF_SAMPLE_BRANCH_STACK, + .sample_period = 1000, + }; + int old_errno; + + if (detection_done) + return; + + old_errno = errno; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are PMU specific limitation + * so we can detect with the given hardware event and stop on the + * first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_COUNTERS; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.branch_counters = true; + pr_debug2("switching off branch counters support\n"); + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_HW_INDEX; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.branch_hw_idx = true; + pr_debug2("switching off branch HW index support\n"); + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_NO_CYCLES | PERF_SAMPLE_BRANCH_NO_FLAGS; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.lbr_flags = true; + pr_debug2_peo("switching off branch sample type no (cycles/flags)\n"); + +found: + detection_done = true; + errno = old_errno; +} + +static bool evsel__probe_aux_action(struct evsel *evsel, struct perf_cpu cpu) +{ + struct perf_event_attr attr = evsel->core.attr; + int old_errno = errno; + + attr.disabled = 1; + attr.aux_start_paused = 1; + + if (__has_attr_feature(&attr, cpu, /*flags=*/0)) { + errno = old_errno; + return true; + } + + /* + * EOPNOTSUPP means the kernel supports the feature but the PMU does + * not, so keep that distinction if possible. + */ + if (errno != EOPNOTSUPP) + errno = old_errno; + + return false; +} + +static void evsel__detect_missing_aux_action_feature(struct evsel *evsel, struct perf_cpu cpu) +{ + static bool detection_done; + struct evsel *leader; + + /* + * Don't bother probing aux_action if it is not being used or has been + * probed before. + */ + if (!evsel->core.attr.aux_action || detection_done) + return; + + detection_done = true; + + /* + * The leader is an AUX area event. If it has failed, assume the feature + * is not supported. + */ + leader = evsel__leader(evsel); + if (evsel == leader) { + perf_missing_features.aux_action = true; + return; + } + + /* + * AUX area event with aux_action must have been opened successfully + * already, so feature is supported. + */ + if (leader->core.attr.aux_action) + return; + + if (!evsel__probe_aux_action(leader, cpu)) + perf_missing_features.aux_action = true; +} + +static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu cpu) +{ + static bool detection_done = false; + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_TASK_CLOCK, + .disabled = 1, + }; + int old_errno; + + evsel__detect_missing_aux_action_feature(evsel, cpu); + + evsel__detect_missing_pmu_features(evsel); + + if (evsel__has_br_stack(evsel)) + evsel__detect_missing_brstack_features(evsel); + + if (detection_done) + goto check; + + old_errno = errno; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are kernel core limitation + * not PMU-specific so we can detect with a software event and + * stop on the first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.defer_callchain = true; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.defer_callchain = true; + pr_debug2("switching off deferred callchain support\n"); + attr.defer_callchain = false; + + attr.inherit = true; + attr.sample_type = PERF_SAMPLE_READ | PERF_SAMPLE_TID; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.inherit_sample_read = true; + pr_debug2("Using PERF_SAMPLE_READ / :S modifier is not compatible with inherit, falling back to no-inherit.\n"); + attr.inherit = false; + attr.sample_type = 0; + + attr.read_format = PERF_FORMAT_LOST; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.read_lost = true; + pr_debug2("switching off PERF_FORMAT_LOST support\n"); + attr.read_format = 0; + + attr.sample_type = PERF_SAMPLE_WEIGHT_STRUCT; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.weight_struct = true; + pr_debug2("switching off weight struct support\n"); + attr.sample_type = 0; + + attr.sample_type = PERF_SAMPLE_CODE_PAGE_SIZE; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.code_page_size = true; + pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support\n"); + attr.sample_type = 0; + + attr.sample_type = PERF_SAMPLE_DATA_PAGE_SIZE; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.data_page_size = true; + pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support\n"); + attr.sample_type = 0; + + attr.cgroup = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.cgroup = true; + pr_debug2_peo("Kernel has no cgroup sampling support\n"); + attr.cgroup = 0; + + attr.aux_output = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.aux_output = true; + pr_debug2_peo("Kernel has no attr.aux_output support\n"); + attr.aux_output = 0; + + attr.bpf_event = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.bpf = true; + pr_debug2_peo("switching off bpf_event\n"); + attr.bpf_event = 0; + + attr.ksymbol = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.ksymbol = true; + pr_debug2_peo("switching off ksymbol\n"); + attr.ksymbol = 0; + + attr.write_backward = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.write_backward = true; + pr_debug2_peo("switching off write_backward\n"); + attr.write_backward = 0; + + attr.use_clockid = 1; + attr.clockid = CLOCK_MONOTONIC; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.clockid = true; + pr_debug2_peo("switching off clockid\n"); + attr.use_clockid = 0; + attr.clockid = 0; + + if (has_attr_feature(&attr, /*flags=*/PERF_FLAG_FD_CLOEXEC)) + goto found; + perf_missing_features.cloexec = true; + pr_debug2_peo("switching off cloexec flag\n"); + + attr.mmap2 = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.mmap2 = true; + pr_debug2_peo("switching off mmap2\n"); + attr.mmap2 = 0; + + /* set this unconditionally? */ + perf_missing_features.sample_id_all = true; + pr_debug2_peo("switching off sample_id_all\n"); + + attr.inherit = 1; + attr.read_format = PERF_FORMAT_GROUP; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.group_read = true; + pr_debug2_peo("switching off group read\n"); + attr.inherit = 0; + attr.read_format = 0; + +found: + detection_done = true; + errno = old_errno; + +check: + if ((evsel->core.attr.defer_callchain || evsel->core.attr.defer_output) && + perf_missing_features.defer_callchain) + return true; + + if (evsel->core.attr.inherit && + (evsel->core.attr.sample_type & PERF_SAMPLE_READ) && + perf_missing_features.inherit_sample_read) + return true; + + if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + perf_missing_features.branch_counters) + return true; + + if ((evsel->core.attr.read_format & PERF_FORMAT_LOST) && + perf_missing_features.read_lost) + return true; + + if ((evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT) && + perf_missing_features.weight_struct) + return true; + + if (evsel->core.attr.use_clockid && evsel->core.attr.clockid != CLOCK_MONOTONIC && + !perf_missing_features.clockid) { + perf_missing_features.clockid_wrong = true; + return true; + } + + if (evsel->core.attr.use_clockid && perf_missing_features.clockid) + return true; + + if ((evsel->open_flags & PERF_FLAG_FD_CLOEXEC) && + perf_missing_features.cloexec) + return true; + + if (evsel->core.attr.mmap2 && perf_missing_features.mmap2) + return true; + + if ((evsel->core.attr.branch_sample_type & (PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_NO_CYCLES)) && + perf_missing_features.lbr_flags) + return true; + + if (evsel->core.attr.inherit && (evsel->core.attr.read_format & PERF_FORMAT_GROUP) && + perf_missing_features.group_read) + return true; + + if (evsel->core.attr.ksymbol && perf_missing_features.ksymbol) + return true; + + if (evsel->core.attr.bpf_event && perf_missing_features.bpf) + return true; + + if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) && + perf_missing_features.branch_hw_idx) + return true; + + if (evsel->core.attr.sample_id_all && perf_missing_features.sample_id_all) + return true; + + return false; +} + +static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx) +{ + int idx, thread, nthreads; + int pid = -1, err, old_errno; + enum rlimit_action set_rlimit = NO_CHANGE; + struct perf_cpu cpu; + + if (evsel__is_retire_lat(evsel)) { + err = evsel__tpebs_open(evsel); + goto out; + } + + err = __evsel__prepare_open(evsel, cpus, threads); + if (err) + goto out; + + if (cpus == NULL) + cpus = empty_cpu_map; + + if (threads == NULL) + threads = empty_thread_map; + + nthreads = perf_thread_map__nr(threads); + + if (evsel->cgrp) + pid = evsel->cgrp->fd; + +fallback_missing_features: + evsel__disable_missing_features(evsel); + + pr_debug3("Opening: %s\n", evsel__name(evsel)); + display_attr(&evsel->core.attr); + + if (evsel__is_tool(evsel)) { + err = evsel__tool_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + goto out; + } + if (evsel__is_hwmon(evsel)) { + err = evsel__hwmon_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + goto out; + } + if (evsel__is_drm(evsel)) { + err = evsel__drm_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + goto out; + } + + for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) { + cpu = perf_cpu_map__cpu(cpus, idx); + + for (thread = 0; thread < nthreads; thread++) { + int fd, group_fd; +retry_open: + if (thread >= nthreads) + break; + + if (!evsel->cgrp && !evsel->core.system_wide) + pid = perf_thread_map__pid(threads, thread); + + group_fd = get_group_fd(evsel, idx, thread); + + if (group_fd == -2) { + pr_debug("broken group leader for %s\n", evsel->name); + err = -EINVAL; + goto out_close; + } + + /* Debug message used by test scripts */ + pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx", + pid, cpu.cpu, group_fd, evsel->open_flags); + + fd = sys_perf_event_open(&evsel->core.attr, pid, cpu.cpu, + group_fd, evsel->open_flags); + + FD(evsel, idx, thread) = fd; + + if (fd < 0) { + err = -errno; + + pr_debug2_peo("\nsys_perf_event_open failed, error %d\n", + err); + goto try_fallback; + } + + bpf_counter__install_pe(evsel, idx, fd); + + if (unlikely(test_attr__enabled())) { + test_attr__open(&evsel->core.attr, pid, cpu, + fd, group_fd, evsel->open_flags); + } + + /* Debug message used by test scripts */ + pr_debug2_peo(" = %d\n", fd); + + set_rlimit = NO_CHANGE; + + /* + * If we succeeded but had to kill clockid, fail and + * have evsel__open_strerror() print us a nice error. + */ + if (perf_missing_features.clockid || + perf_missing_features.clockid_wrong) { + err = -EINVAL; + goto out_close; + } + } + } + + err = 0; + goto out; + +try_fallback: + if (evsel__ignore_missing_thread(evsel, perf_cpu_map__nr(cpus), + idx, threads, thread, err)) { + /* We just removed 1 thread, so lower the upper nthreads limit. */ + nthreads--; + + /* ... and pretend like nothing have happened. */ + err = 0; + goto retry_open; + } + /* + * perf stat needs between 5 and 22 fds per CPU. When we run out + * of them try to increase the limits. + */ + if (err == -EMFILE && rlimit__increase_nofile(&set_rlimit)) + goto retry_open; + + if (err == -EINVAL && evsel__detect_missing_features(evsel, cpu)) + goto fallback_missing_features; + + if (evsel__precise_ip_fallback(evsel)) + goto retry_open; + +out_close: + if (err) + threads->err_thread = thread; + + old_errno = errno; + do { + while (--thread >= 0) { + if (FD(evsel, idx, thread) >= 0) + close(FD(evsel, idx, thread)); + FD(evsel, idx, thread) = -1; + } + thread = nthreads; + } while (--idx >= 0); + errno = old_errno; +out: + if (err) + evsel->supported = false; + return err; +} + +int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus)); +} + +void evsel__close(struct evsel *evsel) +{ + if (evsel__is_retire_lat(evsel)) + evsel__tpebs_close(evsel); + perf_evsel__close(&evsel->core); + perf_evsel__free_id(&evsel->core); +} + +int evsel__open_per_cpu_and_thread(struct evsel *evsel, + struct perf_cpu_map *cpus, int cpu_map_idx, + struct perf_thread_map *threads) +{ + if (cpu_map_idx == -1) + return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus)); + + return evsel__open_cpu(evsel, cpus, threads, cpu_map_idx, cpu_map_idx + 1); +} + +int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx) +{ + struct perf_thread_map *threads = thread_map__new_by_tid(-1); + int ret = evsel__open_per_cpu_and_thread(evsel, cpus, cpu_map_idx, threads); + + perf_thread_map__put(threads); + return ret; +} + +int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads) +{ + struct perf_cpu_map *cpus = perf_cpu_map__new_any_cpu(); + int ret = evsel__open_per_cpu_and_thread(evsel, cpus, -1, threads); + + perf_cpu_map__put(cpus); + return ret; +} + +static int perf_evsel__parse_id_sample(const union perf_event *event, + struct perf_sample *sample) +{ + const struct evsel *evsel = sample->evsel; + u64 type = evsel->core.attr.sample_type; + const __u64 *array = event->sample.array; + bool swapped = evsel->needs_swap; + union u64_swap u; + int i = ((event->header.size - sizeof(event->header)) / sizeof(u64)) - 1; + + if (type & PERF_SAMPLE_IDENTIFIER) { + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; + } + + if (type & PERF_SAMPLE_CPU) { + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + } + sample->cpu = u.val32[0]; + } + + if (type & PERF_SAMPLE_STREAM_ID) { + if (i < 0) + return -EFAULT; + + sample->stream_id = array[i--]; + } + + if (type & PERF_SAMPLE_ID) { + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; + } + + if (type & PERF_SAMPLE_TIME) { + if (i < 0) + return -EFAULT; + + sample->time = array[i--]; + } + + if (type & PERF_SAMPLE_TID) { + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + u.val32[1] = bswap_32(u.val32[1]); + } + + sample->pid = u.val32[0]; + sample->tid = u.val32[1]; + } + + return 0; +} + +static inline bool overflow(const void *endp, u16 max_size, const void *offset, + u64 size) +{ + return size > max_size || offset + size > endp; +} + +#define OVERFLOW_CHECK(offset, size, max_size) \ + do { \ + if (overflow(endp, (max_size), (offset), (size))) \ + goto out_efault; \ + } while (0) + +#define OVERFLOW_CHECK_u64(offset) \ + OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64)) + +static int +perf_event__check_size(union perf_event *event, unsigned int sample_size) +{ + /* + * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes + * up to PERF_SAMPLE_PERIOD. After that overflow() must be used to + * check the format does not go past the end of the event. + */ + if (sample_size + sizeof(event->header) > event->header.size) + return -EFAULT; + + return 0; +} + +static void perf_parse_sample_weight(struct perf_sample *data, const __u64 *array, u64 type) +{ + union perf_sample_weight weight; + + weight.full = *array; + if (type & PERF_SAMPLE_WEIGHT_STRUCT) { + data->weight = weight.var1_dw; + data->ins_lat = weight.var2_w; + data->weight3 = weight.var3_w; + } else { + data->weight = weight.full; + } +} + +u64 evsel__bitfield_swap_branch_flags(u64 value) +{ + u64 new_val = 0; + + /* + * branch_flags + * union { + * u64 values; + * struct { + * mispred:1 //target mispredicted + * predicted:1 //target predicted + * in_tx:1 //in transaction + * abort:1 //transaction abort + * cycles:16 //cycle count to last branch + * type:4 //branch type + * spec:2 //branch speculation info + * new_type:4 //additional branch type + * priv:3 //privilege level + * reserved:31 + * } + * } + * + * Avoid bswap64() the entire branch_flag.value, + * as it has variable bit-field sizes. Instead the + * macro takes the bit-field position/size, + * swaps it based on the host endianness. + */ + if (host_is_bigendian()) { + new_val = bitfield_swap(value, 0, 1); + new_val |= bitfield_swap(value, 1, 1); + new_val |= bitfield_swap(value, 2, 1); + new_val |= bitfield_swap(value, 3, 1); + new_val |= bitfield_swap(value, 4, 16); + new_val |= bitfield_swap(value, 20, 4); + new_val |= bitfield_swap(value, 24, 2); + new_val |= bitfield_swap(value, 26, 4); + new_val |= bitfield_swap(value, 30, 3); + new_val |= bitfield_swap(value, 33, 31); + } else { + new_val = bitfield_swap(value, 63, 1); + new_val |= bitfield_swap(value, 62, 1); + new_val |= bitfield_swap(value, 61, 1); + new_val |= bitfield_swap(value, 60, 1); + new_val |= bitfield_swap(value, 44, 16); + new_val |= bitfield_swap(value, 40, 4); + new_val |= bitfield_swap(value, 38, 2); + new_val |= bitfield_swap(value, 34, 4); + new_val |= bitfield_swap(value, 31, 3); + new_val |= bitfield_swap(value, 0, 31); + } + + return new_val; +} + +static inline bool evsel__has_branch_counters(const struct evsel *evsel) +{ + struct evsel *leader = evsel__leader(evsel); + + /* The branch counters feature only supports group */ + if (!leader || !evsel->evlist) + return false; + + if (evlist__nr_br_cntr(evsel->evlist) < 0) + evlist__update_br_cntr(evsel->evlist); + + if (leader->br_cntr_nr > 0) + return true; + + return false; +} + +static int __set_offcpu_sample(struct perf_sample *data) +{ + u64 *array = data->raw_data; + u32 max_size = data->raw_size, *p32; + const void *endp = (void *)array + max_size; + + if (array == NULL) + return -EFAULT; + + OVERFLOW_CHECK_u64(array); + p32 = (void *)array++; + data->pid = p32[0]; + data->tid = p32[1]; + + OVERFLOW_CHECK_u64(array); + data->period = *array++; + + OVERFLOW_CHECK_u64(array); + data->callchain = (struct ip_callchain *)array++; + OVERFLOW_CHECK(array, data->callchain->nr * sizeof(u64), max_size); + data->ip = data->callchain->ips[1]; + array += data->callchain->nr; + + OVERFLOW_CHECK_u64(array); + data->cgroup = *array; + + return 0; +out_efault: + return -EFAULT; +} + +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap) +{ + u64 type = evsel->core.attr.sample_type; + bool swapped = needs_swap; + const __u64 *array; + u16 max_size = event->header.size; + const void *endp = (void *)event + max_size; + u64 sz; + + /* + * used for cross-endian analysis. See git commit 65014ab3 + * for why this goofiness is needed. + */ + union u64_swap u; + + perf_sample__init(data, /*all=*/true); + data->evsel = evsel__get(evsel); + data->cpu = data->pid = data->tid = -1; + data->stream_id = data->id = data->time = -1ULL; + data->period = evsel->core.attr.sample_period; + data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + data->misc = event->header.misc; + data->data_src = PERF_MEM_DATA_SRC_NONE; + data->vcpu = -1; + + if (event->header.type == PERF_RECORD_CALLCHAIN_DEFERRED) { + const u64 max_callchain_nr = UINT64_MAX / sizeof(u64); + + data->callchain = (struct ip_callchain *)&event->callchain_deferred.nr; + if (data->callchain->nr > max_callchain_nr) + goto out_efault; + + data->deferred_cookie = event->callchain_deferred.cookie; + + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } + return 0; + } + + if (event->header.type != PERF_RECORD_SAMPLE) { + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } + return 0; + } + + array = event->sample.array; + + if (perf_event__check_size(event, evsel->sample_size)) + goto out_efault; + + if (type & PERF_SAMPLE_IDENTIFIER) { + data->id = *array; + array++; + } + + if (type & PERF_SAMPLE_IP) { + data->ip = *array; + array++; + } + + if (type & PERF_SAMPLE_TID) { + u.val64 = *array; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + u.val32[1] = bswap_32(u.val32[1]); + } + + data->pid = u.val32[0]; + data->tid = u.val32[1]; + array++; + } + + if (type & PERF_SAMPLE_TIME) { + data->time = *array; + array++; + } + + if (type & PERF_SAMPLE_ADDR) { + data->addr = *array; + array++; + } + + if (type & PERF_SAMPLE_ID) { + data->id = *array; + array++; + } + + if (type & PERF_SAMPLE_STREAM_ID) { + data->stream_id = *array; + array++; + } + + if (type & PERF_SAMPLE_CPU) { + + u.val64 = *array; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + } + + data->cpu = u.val32[0]; + array++; + } + + if (type & PERF_SAMPLE_PERIOD) { + data->period = *array; + array++; + } + + if (type & PERF_SAMPLE_READ) { + u64 read_format = evsel->core.attr.read_format; + + OVERFLOW_CHECK_u64(array); + if (read_format & PERF_FORMAT_GROUP) + data->read.group.nr = *array; + else + data->read.one.value = *array; + + array++; + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { + OVERFLOW_CHECK_u64(array); + data->read.time_enabled = *array; + array++; + } + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { + OVERFLOW_CHECK_u64(array); + data->read.time_running = *array; + array++; + } + + /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */ + if (read_format & PERF_FORMAT_GROUP) { + const u64 max_group_nr = UINT64_MAX / + sizeof(struct sample_read_value); + + if (data->read.group.nr > max_group_nr) + goto out_efault; + + sz = data->read.group.nr * sample_read_value_size(read_format); + OVERFLOW_CHECK(array, sz, max_size); + data->read.group.values = + (struct sample_read_value *)array; + array = (void *)array + sz; + } else { + OVERFLOW_CHECK_u64(array); + data->read.one.id = *array; + array++; + + if (read_format & PERF_FORMAT_LOST) { + OVERFLOW_CHECK_u64(array); + data->read.one.lost = *array; + array++; + } + } + } + + if (type & PERF_SAMPLE_CALLCHAIN) { + const u64 max_callchain_nr = UINT64_MAX / sizeof(u64); + u64 callchain_nr; + + OVERFLOW_CHECK_u64(array); + data->callchain = (struct ip_callchain *)array++; + callchain_nr = data->callchain->nr; + if (callchain_nr > max_callchain_nr) + goto out_efault; + sz = callchain_nr * sizeof(u64); + /* + * Save the cookie for the deferred user callchain. The last 2 + * entries in the callchain should be the context marker and the + * cookie. The cookie will be used to match PERF_RECORD_ + * CALLCHAIN_DEFERRED later. + */ + if (evsel->core.attr.defer_callchain && callchain_nr >= 2 && + data->callchain->ips[callchain_nr - 2] == PERF_CONTEXT_USER_DEFERRED) { + data->deferred_cookie = data->callchain->ips[callchain_nr - 1]; + data->deferred_callchain = true; + } + OVERFLOW_CHECK(array, sz, max_size); + array = (void *)array + sz; + } + + if (type & PERF_SAMPLE_RAW) { + OVERFLOW_CHECK_u64(array); + u.val64 = *array; + + /* + * Undo swap of u64, then swap on individual u32s, + * get the size of the raw area and undo all of the + * swap. The pevent interface handles endianness by + * itself. + */ + if (swapped) { + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + u.val32[1] = bswap_32(u.val32[1]); + } + data->raw_size = u.val32[0]; + + /* + * The raw data is aligned on 64bits including the + * u32 size, so it's safe to use mem_bswap_64. + */ + if (swapped) + mem_bswap_64((void *) array, data->raw_size); + + array = (void *)array + sizeof(u32); + + OVERFLOW_CHECK(array, data->raw_size, max_size); + data->raw_data = (void *)array; + array = (void *)array + data->raw_size; + } + + if (type & PERF_SAMPLE_BRANCH_STACK) { + const u64 max_branch_nr = UINT64_MAX / + sizeof(struct branch_entry); + struct branch_entry *e; + unsigned int i; + + OVERFLOW_CHECK_u64(array); + data->branch_stack = (struct branch_stack *)array++; + + if (data->branch_stack->nr > max_branch_nr) + goto out_efault; + + sz = data->branch_stack->nr * sizeof(struct branch_entry); + if (evsel__has_branch_hw_idx(evsel)) { + sz += sizeof(u64); + e = &data->branch_stack->entries[0]; + } else { + data->no_hw_idx = true; + /* + * if the PERF_SAMPLE_BRANCH_HW_INDEX is not applied, + * only nr and entries[] will be output by kernel. + */ + e = (struct branch_entry *)&data->branch_stack->hw_idx; + } + + if (swapped) { + /* + * struct branch_flag does not have endian + * specific bit field definition. And bswap + * will not resolve the issue, since these + * are bit fields. + * + * evsel__bitfield_swap_branch_flags() uses a + * bitfield_swap macro to swap the bit position + * based on the host endians. + */ + for (i = 0; i < data->branch_stack->nr; i++, e++) + e->flags.value = evsel__bitfield_swap_branch_flags(e->flags.value); + } + + OVERFLOW_CHECK(array, sz, max_size); + array = (void *)array + sz; + + if (evsel__has_branch_counters(evsel)) { + data->branch_stack_cntr = (u64 *)array; + sz = data->branch_stack->nr * sizeof(u64); + + OVERFLOW_CHECK(array, sz, max_size); + array = (void *)array + sz; + } + } + + if (type & PERF_SAMPLE_REGS_USER) { + struct regs_dump *regs = perf_sample__user_regs(data); + + OVERFLOW_CHECK_u64(array); + regs->abi = *array; + array++; + + if (regs->abi) { + u64 mask = evsel->core.attr.sample_regs_user; + + sz = hweight64(mask) * sizeof(u64); + OVERFLOW_CHECK(array, sz, max_size); + regs->mask = mask; + regs->regs = (u64 *)array; + array = (void *)array + sz; + } + } + + if (type & PERF_SAMPLE_STACK_USER) { + OVERFLOW_CHECK_u64(array); + sz = *array++; + + data->user_stack.offset = ((char *)(array - 1) + - (char *) event); + + if (!sz) { + data->user_stack.size = 0; + } else { + OVERFLOW_CHECK(array, sz, max_size); + data->user_stack.data = (char *)array; + array = (void *)array + sz; + OVERFLOW_CHECK_u64(array); + data->user_stack.size = *array++; + if (WARN_ONCE(data->user_stack.size > sz, + "user stack dump failure\n")) + goto out_efault; + } + } + + if (type & PERF_SAMPLE_WEIGHT_TYPE) { + OVERFLOW_CHECK_u64(array); + perf_parse_sample_weight(data, array, type); + array++; + } + + if (type & PERF_SAMPLE_DATA_SRC) { + OVERFLOW_CHECK_u64(array); + data->data_src = *array; + array++; + } + + if (type & PERF_SAMPLE_TRANSACTION) { + OVERFLOW_CHECK_u64(array); + data->transaction = *array; + array++; + } + + if (type & PERF_SAMPLE_REGS_INTR) { + struct regs_dump *regs = perf_sample__intr_regs(data); + + OVERFLOW_CHECK_u64(array); + regs->abi = *array; + array++; + + if (regs->abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 mask = evsel->core.attr.sample_regs_intr; + + sz = hweight64(mask) * sizeof(u64); + OVERFLOW_CHECK(array, sz, max_size); + regs->mask = mask; + regs->regs = (u64 *)array; + array = (void *)array + sz; + } + } + + data->phys_addr = 0; + if (type & PERF_SAMPLE_PHYS_ADDR) { + data->phys_addr = *array; + array++; + } + + data->cgroup = 0; + if (type & PERF_SAMPLE_CGROUP) { + data->cgroup = *array; + array++; + } + + data->data_page_size = 0; + if (type & PERF_SAMPLE_DATA_PAGE_SIZE) { + data->data_page_size = *array; + array++; + } + + data->code_page_size = 0; + if (type & PERF_SAMPLE_CODE_PAGE_SIZE) { + data->code_page_size = *array; + array++; + } + + if (type & PERF_SAMPLE_AUX) { + OVERFLOW_CHECK_u64(array); + sz = *array++; + + OVERFLOW_CHECK(array, sz, max_size); + /* Undo swap of data */ + if (swapped) + mem_bswap_64((char *)array, sz); + data->aux_sample.size = sz; + data->aux_sample.data = (char *)array; + array = (void *)array + sz; + } + + if (evsel__is_offcpu_event(evsel)) { + if (__set_offcpu_sample(data)) + goto out_efault; + } + + return 0; +out_efault: + perf_sample__exit(data); + return -EFAULT; +} + +int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, + u64 *timestamp) +{ + u64 type = evsel->core.attr.sample_type; + const __u64 *array; + + if (!(type & PERF_SAMPLE_TIME)) + return -1; + + if (event->header.type != PERF_RECORD_SAMPLE) { + struct perf_sample data = { + .evsel = evsel, + .time = -1ULL, + }; + + if (!evsel->core.attr.sample_id_all) + return -1; + if (perf_evsel__parse_id_sample(event, &data)) + return -1; + + *timestamp = data.time; + return 0; + } + + array = event->sample.array; + + if (perf_event__check_size(event, evsel->sample_size)) + return -EFAULT; + + if (type & PERF_SAMPLE_IDENTIFIER) + array++; + + if (type & PERF_SAMPLE_IP) + array++; + + if (type & PERF_SAMPLE_TID) + array++; + + if (type & PERF_SAMPLE_TIME) + *timestamp = *array; + + return 0; +} + +u16 evsel__id_hdr_size(const struct evsel *evsel) +{ + u64 sample_type = evsel->core.attr.sample_type; + u16 size = 0; + + if (sample_type & PERF_SAMPLE_TID) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_TIME) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_ID) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_STREAM_ID) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_CPU) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_IDENTIFIER) + size += sizeof(u64); + + return size; +} + +#ifdef HAVE_LIBTRACEEVENT +struct tep_format_field *evsel__field(struct evsel *evsel, const char *name) +{ + struct tep_event *tp_format = evsel__tp_format(evsel); + + return tp_format ? tep_find_field(tp_format, name) : NULL; +} + +struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name) +{ + struct tep_event *tp_format = evsel__tp_format(evsel); + + return tp_format ? tep_find_common_field(tp_format, name) : NULL; +} + +static bool out_of_bounds(const struct tep_format_field *field, int offset, int size, u32 raw_size) +{ + if (offset < 0) { + pr_warning("Negative trace point field offset %d in %s\n", + offset, field->name); + return true; + } + if (size < 0) { + pr_warning("Negative trace point field size %d in %s\n", + size, field->name); + return true; + } + if ((u32)offset + (u32)size > raw_size) { + pr_warning("Out of bound tracepoint field (%s) offset %d size %d in %u\n", + field->name, offset, size, raw_size); + return true; + } + return false; +} + +void *perf_sample__rawptr(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__field(sample->evsel, name); + int offset, size; + + if (!field) + return NULL; + + offset = field->offset; + size = field->size; + if (field->flags & TEP_FIELD_IS_DYNAMIC) { + int dynamic_data; + + if (out_of_bounds(field, offset, 4, sample->raw_size)) + return NULL; + + dynamic_data = *(int *)(sample->raw_data + field->offset); + + if (sample->evsel->needs_swap) + dynamic_data = bswap_32(dynamic_data); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) { + /* + * Newer kernel feature: Relative offsets (__rel_loc). + * If the relative flag is set, the parsed offset is not + * absolute from the start of the record. Instead, it is + * relative to the *end* of the dynamic field descriptor + * itself. + */ + offset += field->offset + field->size; + } + } + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; + + return sample->raw_data + offset; +} + +void *format_field__get_raw_data(struct tep_format_field *field, struct + perf_sample *sample, bool needs_swap, + u16 *len_out) +{ + int offset = field->offset; + int size = field->size; + + if (field->flags & TEP_FIELD_IS_DYNAMIC) { + unsigned int dynamic_data; + + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return NULL; + + dynamic_data = format_field__intval(field, sample, needs_swap); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) + offset += field->offset + field->size; + } + + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; + + *len_out = size; + return sample->raw_data + offset; +} + +unsigned long *format_field__get_cpumask(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out) +{ + u16 len; + void *ptr = format_field__get_raw_data(field, sample, needs_swap, &len); + unsigned long *mask; + struct perf_env *env; + bool target_is_64; + int target_word_size; + int nr_words; + int bit_idx; + int nbits; + + if (!ptr) + return NULL; + + nbits = len * 8; + mask = bitmap_zalloc(nbits ?: 1); + if (!mask) + return NULL; + + env = evsel__env(sample->evsel); + target_is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + target_word_size = target_is_64 ? 8 : 4; + nr_words = len / target_word_size; + + for (bit_idx = 0; bit_idx < nbits; bit_idx++) { + int w_idx = bit_idx / (target_word_size * 8); + int bit_in_word = bit_idx % (target_word_size * 8); + bool set = false; + + if (w_idx >= nr_words) + break; + + if (target_is_64) { + u64 word; + memcpy(&word, (unsigned char *)ptr + w_idx * 8, 8); + if (needs_swap) + word = bswap_64(word); + set = (word & (1ULL << bit_in_word)) != 0; + } else { + u32 word32; + memcpy(&word32, (unsigned char *)ptr + w_idx * 4, 4); + if (needs_swap) + word32 = bswap_32(word32); + set = (word32 & (1U << bit_in_word)) != 0; + } + + if (set) { + int host_w_idx = bit_idx / BITS_PER_LONG; + int host_bit_in_word = bit_idx % BITS_PER_LONG; + mask[host_w_idx] |= (1UL << host_bit_in_word); + } + } + + *len_out = len; + return mask; +} + +u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, + bool needs_swap) +{ + u64 value; + void *ptr = sample->raw_data + field->offset; + + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return 0; + + switch (field->size) { + case 1: + return *(u8 *)ptr; + case 2: + value = *(u16 *)ptr; + break; + case 4: + value = *(u32 *)ptr; + break; + case 8: + memcpy(&value, ptr, sizeof(u64)); + break; + default: + return 0; + } + + if (!needs_swap) + return value; + + switch (field->size) { + case 2: + return bswap_16(value); + case 4: + return bswap_32(value); + case 8: + return bswap_64(value); + default: + return 0; + } + + return 0; +} + +u64 perf_sample__intval(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__field(sample->evsel, name); + + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; +} + +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__common_field(sample->evsel, name); + + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; +} + +char perf_sample__taskstate(struct perf_sample *sample, const char *name) +{ + static struct tep_format_field *prev_state_field; + static const char *states; + struct tep_format_field *field; + unsigned long long val; + unsigned int bit; + char state = '?'; /* '?' denotes unknown task state */ + + field = evsel__field(sample->evsel, name); + + if (!field) + return state; + + if (!states || field != prev_state_field) { + states = parse_task_states(field); + if (!states) + return state; + prev_state_field = field; + } + + /* + * Note since the kernel exposes TASK_REPORT_MAX to userspace + * to denote the 'preempted' state, we might as welll report + * 'R' for this case, which make senses to users as well. + * + * We can change this if we have a good reason in the future. + */ + val = perf_sample__intval(sample, name); + bit = val ? ffs(val) : 0; + state = (!bit || bit > strlen(states)) ? 'R' : states[bit-1]; + return state; +} +#endif + +bool evsel__fallback(struct evsel *evsel, struct target *target, int err, + char *msg, size_t msgsize) +{ + int paranoid; + + if ((err == ENODEV || err == ENOENT || err == ENXIO) && + evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) { + /* + * If it's the legacy hardware cycles event fails then fall back + * to hrtimer based cpu-clock sw counter, which is always + * available even if no PMU support. PPC returned ENXIO rather + * than ENODEV or ENOENT until 2.6.37. + */ + evsel->pmu = perf_pmus__find_by_type(PERF_TYPE_SOFTWARE); + assert(evsel->pmu); /* software is a "well-known" and can't fail PMU type. */ + + /* Configure the event. */ + evsel->core.attr.type = PERF_TYPE_SOFTWARE; + evsel->core.attr.config = target__has_cpu(target) + ? PERF_COUNT_SW_CPU_CLOCK + : PERF_COUNT_SW_TASK_CLOCK; + evsel->core.is_pmu_core = false; + + /* Remove excludes for new event. */ + if (evsel->fallenback_eacces) { + evsel->core.attr.exclude_kernel = 0; + evsel->core.attr.exclude_hv = 0; + evsel->fallenback_eacces = false; + } + if (evsel->fallenback_eopnotsupp) { + evsel->core.attr.exclude_guest = 0; + evsel->fallenback_eopnotsupp = false; + } + + /* Name is recomputed by evsel__name. */ + zfree(&evsel->name); + + /* Log message. */ + scnprintf(msg, msgsize, + "The cycles event is not supported, trying to fall back to %s", + evsel__name(evsel)); + return true; + } else if (err == EACCES && !evsel->core.attr.exclude_kernel && + (paranoid = perf_event_paranoid()) > 1) { + const char *name = evsel__name(evsel); + char *new_name; + const char *sep = ":"; + + /* If event has exclude user then don't exclude kernel. */ + if (evsel->core.attr.exclude_user) + goto no_fallback; + + /* Is there already the separator in the name. */ + if (strchr(name, '/') || + (strchr(name, ':') && !evsel->is_libpfm_event)) + sep = ""; + + if (asprintf(&new_name, "%s%su", name, sep) < 0) + goto no_fallback; + + free(evsel->name); + evsel->name = new_name; + scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying " + "to fall back to excluding kernel and hypervisor " + " samples", paranoid); + evsel->core.attr.exclude_kernel = 1; + evsel->core.attr.exclude_hv = 1; + evsel->fallenback_eacces = true; + return true; + } else if (err == EOPNOTSUPP && !evsel->core.attr.exclude_guest && + !evsel->exclude_GH) { + const char *name = evsel__name(evsel); + char *new_name; + const char *sep = ":"; + + /* Is there already the separator in the name. */ + if (strchr(name, '/') || + (strchr(name, ':') && !evsel->is_libpfm_event)) + sep = ""; + + if (asprintf(&new_name, "%s%sH", name, sep) < 0) + goto no_fallback; + + free(evsel->name); + evsel->name = new_name; + /* Apple M1 requires exclude_guest */ + scnprintf(msg, msgsize, "Trying to fall back to excluding guest samples"); + evsel->core.attr.exclude_guest = 1; + evsel->fallenback_eopnotsupp = true; + return true; + } +no_fallback: + scnprintf(msg, msgsize, "No fallback found for '%s' for error %d", + evsel__name(evsel), err); + return false; +} + +static bool find_process(const char *name) +{ + size_t len = strlen(name); + DIR *dir; + struct dirent *d; + int ret = -1; + + dir = opendir(procfs__mountpoint()); + if (!dir) + return false; + + /* Walk through the directory. */ + while (ret && (d = readdir(dir)) != NULL) { + char path[PATH_MAX]; + char *data; + size_t size; + + if ((d->d_type != DT_DIR) || + !strcmp(".", d->d_name) || + !strcmp("..", d->d_name)) + continue; + + scnprintf(path, sizeof(path), "%s/%s/comm", + procfs__mountpoint(), d->d_name); + + if (filename__read_str(path, &data, &size)) + continue; + + ret = strncmp(name, data, len); + free(data); + } + + closedir(dir); + return ret ? false : true; +} + +static int dump_perf_event_processes(char *msg, size_t size) +{ + DIR *proc_dir; + struct dirent *proc_entry; + int printed = 0; + + proc_dir = opendir(procfs__mountpoint()); + if (!proc_dir) + return 0; + + /* Walk through the /proc directory. */ + while ((proc_entry = readdir(proc_dir)) != NULL) { + char buf[256]; + DIR *fd_dir; + struct dirent *fd_entry; + int fd_dir_fd; + + if (proc_entry->d_type != DT_DIR || + !isdigit(proc_entry->d_name[0]) || + strlen(proc_entry->d_name) > sizeof(buf) - 4) + continue; + + scnprintf(buf, sizeof(buf), "%s/fd", proc_entry->d_name); + fd_dir_fd = openat(dirfd(proc_dir), buf, O_DIRECTORY); + if (fd_dir_fd == -1) + continue; + fd_dir = fdopendir(fd_dir_fd); + if (!fd_dir) { + close(fd_dir_fd); + continue; + } + while ((fd_entry = readdir(fd_dir)) != NULL) { + ssize_t link_size; + + if (fd_entry->d_type != DT_LNK) + continue; + link_size = readlinkat(fd_dir_fd, fd_entry->d_name, buf, sizeof(buf)); + if (link_size < 0) + continue; + /* Take care as readlink doesn't null terminate the string. */ + if (!strncmp(buf, "anon_inode:[perf_event]", link_size)) { + int cmdline_fd; + ssize_t cmdline_size; + + scnprintf(buf, sizeof(buf), "%s/cmdline", proc_entry->d_name); + cmdline_fd = openat(dirfd(proc_dir), buf, O_RDONLY); + if (cmdline_fd == -1) + continue; + cmdline_size = read(cmdline_fd, buf, sizeof(buf) - 1); + close(cmdline_fd); + if (cmdline_size < 0) + continue; + buf[cmdline_size] = '\0'; + for (ssize_t i = 0; i < cmdline_size; i++) { + if (buf[i] == '\0') + buf[i] = ' '; + } + + if (printed == 0) + printed += scnprintf(msg, size, "Possible processes:\n"); + + printed += scnprintf(msg + printed, size - printed, + "%s %s\n", proc_entry->d_name, buf); + break; + } + } + closedir(fd_dir); + } + closedir(proc_dir); + return printed; +} + +int __weak arch_evsel__open_strerror(struct evsel *evsel __maybe_unused, + int err __maybe_unused, + char *msg __maybe_unused, + size_t size __maybe_unused) +{ + return 0; +} + +int evsel__open_strerror(struct evsel *evsel, struct target *target, + int err, char *msg, size_t size) +{ + struct perf_pmu *pmu; + int printed = 0, enforced = 0; + int ret; + + switch (err) { + case EPERM: + case EACCES: + printed += scnprintf(msg + printed, size - printed, + "Access to performance monitoring and observability operations is limited.\n"); + + if (!sysfs__read_int("fs/selinux/enforce", &enforced)) { + if (enforced) { + printed += scnprintf(msg + printed, size - printed, + "Enforced MAC policy settings (SELinux) can limit access to performance\n" + "monitoring and observability operations. Inspect system audit records for\n" + "more perf_event access control information and adjusting the policy.\n"); + } + } + + if (err == EPERM) + printed += scnprintf(msg, size, + "No permission to enable %s event.\n\n", evsel__name(evsel)); + + return printed + scnprintf(msg + printed, size - printed, + "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n" + "access to performance monitoring and observability operations for processes\n" + "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n" + "More information can be found at 'Perf events and tool security' document:\n" + "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n" + "perf_event_paranoid setting is %d:\n" + " -1: Allow use of (almost) all events by all users\n" + " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n" + ">= 0: Disallow raw and ftrace function tracepoint access\n" + ">= 1: Disallow CPU event access\n" + ">= 2: Disallow kernel profiling\n" + "To make the adjusted perf_event_paranoid setting permanent preserve it\n" + "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = )", + perf_event_paranoid()); + case ENOENT: + return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel)); + case EMFILE: + return scnprintf(msg, size, "%s", + "Too many events are opened.\n" + "Probably the maximum number of open file descriptors has been reached.\n" + "Hint: Try again after reducing the number of events.\n" + "Hint: Try increasing the limit with 'ulimit -n '"); + case ENOMEM: + if (evsel__has_callchain(evsel) && + access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0) + return scnprintf(msg, size, + "Not enough memory to setup event with callchain.\n" + "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n" + "Hint: Current value: %d", sysctl__max_stack()); + break; + case ENODEV: + if (target->cpu_list) + return scnprintf(msg, size, "%s", + "No such device - did you specify an out-of-range profile CPU?"); + break; + case EOPNOTSUPP: + if (evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK) + return scnprintf(msg, size, + "%s: PMU Hardware or event type doesn't support branch stack sampling.", + evsel__name(evsel)); + if (evsel->core.attr.aux_output) + return scnprintf(msg, size, + "%s: PMU Hardware doesn't support 'aux_output' feature", + evsel__name(evsel)); + if (evsel->core.attr.aux_action) + return scnprintf(msg, size, + "%s: PMU Hardware doesn't support 'aux_action' feature", + evsel__name(evsel)); + if (evsel->core.attr.sample_period != 0) + return scnprintf(msg, size, + "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'", + evsel__name(evsel)); + if (evsel->core.attr.precise_ip) + return scnprintf(msg, size, "%s", + "\'precise\' request may not be supported. Try removing 'p' modifier."); +#if defined(__i386__) || defined(__x86_64__) + if (evsel->core.attr.type == PERF_TYPE_HARDWARE) + return scnprintf(msg, size, "%s", + "No hardware sampling interrupt available.\n"); +#endif + if (!target__has_cpu(target)) + return scnprintf(msg, size, + "Unsupported event (%s) in per-thread mode, enable system wide with '-a'.", + evsel__name(evsel)); + break; + case EBUSY: + if (find_process("oprofiled")) + return scnprintf(msg, size, + "The PMU counters are busy/taken by another profiler.\n" + "We found oprofile daemon running, please stop it and try again."); + printed += scnprintf( + msg, size, + "The PMU %s counters are busy and in use by another process.\n", + evsel->pmu ? evsel->pmu->name : ""); + return printed + dump_perf_event_processes(msg + printed, size - printed); + break; + case EINVAL: + if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size) + return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel."); + if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size) + return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel."); + if (evsel->core.attr.write_backward && perf_missing_features.write_backward) + return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel."); + if (perf_missing_features.clockid) + return scnprintf(msg, size, "clockid feature not supported."); + if (perf_missing_features.clockid_wrong) + return scnprintf(msg, size, "wrong clockid (%d).", clockid); + if (perf_missing_features.aux_action) + return scnprintf(msg, size, "The 'aux_action' feature is not supported, update the kernel."); + if (perf_missing_features.aux_output) + return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel."); + pmu = evsel__find_pmu(evsel); + if (!pmu->is_core && !target__has_cpu(target)) + return scnprintf(msg, size, + "Invalid event (%s) in per-thread mode, enable system wide with '-a'.", + evsel__name(evsel)); + + break; + case ENODATA: + return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. " + "Please add an auxiliary event in front of the load latency event."); + default: + break; + } + + ret = arch_evsel__open_strerror(evsel, err, msg, size); + if (ret) + return ret; + + errno = err; + return scnprintf(msg, size, + "The sys_perf_event_open() syscall failed for event (%s): %m\n" + "\"dmesg | grep -i perf\" may provide additional information.\n", + evsel__name(evsel)); +} + +struct perf_session *evsel__session(struct evsel *evsel) +{ + return evsel && evsel->evlist ? evlist__session(evsel->evlist) : NULL; +} + +struct perf_env *evsel__env(struct evsel *evsel) +{ + struct perf_session *session = evsel__session(evsel); + + return session ? perf_session__env(session) : NULL; +} + +static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist) +{ + int cpu_map_idx, thread; + + if (evsel__is_retire_lat(evsel)) + return 0; + + if (perf_pmu__kind(evsel->pmu) != PERF_PMU_KIND_PE) + return 0; + + for (cpu_map_idx = 0; cpu_map_idx < xyarray__max_x(evsel->core.fd); cpu_map_idx++) { + for (thread = 0; thread < xyarray__max_y(evsel->core.fd); + thread++) { + int fd = FD(evsel, cpu_map_idx, thread); + + if (perf_evlist__id_add_fd(evlist__core(evlist), &evsel->core, + cpu_map_idx, thread, fd) < 0) + return -1; + } + } + + return 0; +} + +int evsel__store_ids(struct evsel *evsel, struct evlist *evlist) +{ + struct perf_cpu_map *cpus = evsel->core.cpus; + struct perf_thread_map *threads = evsel->core.threads; + + if (perf_evsel__alloc_id(&evsel->core, perf_cpu_map__nr(cpus), threads->nr)) + return -ENOMEM; + + return store_evsel_ids(evsel, evlist); +} + +void evsel__zero_per_pkg(struct evsel *evsel) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (evsel->per_pkg_mask) { + hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt) + zfree(&cur->pkey); + + hashmap__clear(evsel->per_pkg_mask); + } +} + +/** + * evsel__is_hybrid - does the evsel have a known PMU that is hybrid. Note, this + * will be false on hybrid systems for hardware and legacy + * cache events. + */ +bool evsel__is_hybrid(const struct evsel *evsel) +{ + if (!evsel->core.is_pmu_core) + return false; + + return perf_pmus__num_core_pmus() > 1; +} + +struct evsel *evsel__leader(const struct evsel *evsel) +{ + if (evsel->core.leader == NULL) + return NULL; + return container_of(evsel->core.leader, struct evsel, core); +} + +bool evsel__has_leader(struct evsel *evsel, struct evsel *leader) +{ + return evsel->core.leader == &leader->core; +} + +bool evsel__is_leader(struct evsel *evsel) +{ + return evsel__has_leader(evsel, evsel); +} + +void evsel__set_leader(struct evsel *evsel, struct evsel *leader) +{ + evsel->core.leader = &leader->core; +} + +bool evsel__is_aux_event(const struct evsel *evsel) +{ + struct perf_pmu *pmu; + + if (evsel->needs_auxtrace_mmap) + return true; + + pmu = evsel__find_pmu(evsel); + return pmu && pmu->auxtrace; +} + +int evsel__source_count(const struct evsel *evsel) +{ + struct evsel *pos; + int count = 0; + + evlist__for_each_entry(evsel->evlist, pos) { + if (pos->metric_leader == evsel) + count++; + } + return count; +} + +bool __weak arch_evsel__must_be_in_group(const struct evsel *evsel __maybe_unused) +{ + return false; +} + +/* + * Remove an event from a given group (leader). + * Some events, e.g., perf metrics Topdown events, + * must always be grouped. Ignore the events. + */ +void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader) +{ + if (!arch_evsel__must_be_in_group(evsel) && evsel != leader) { + evsel__set_leader(evsel, evsel); + evsel->core.nr_members = 0; + leader->core.nr_members--; + } +} + +bool evsel__set_needs_uniquify(struct evsel *counter, const struct perf_stat_config *config) +{ + struct evsel *evsel; + + if (counter->needs_uniquify) { + /* Already set. */ + return true; + } + + if (counter->use_config_name || counter->is_libpfm_event) { + /* Original name will be used. */ + return false; + } + + if (!config->hybrid_merge && evsel__is_hybrid(counter)) { + /* Unique hybrid counters necessary. */ + counter->needs_uniquify = true; + return true; + } + + if (counter->core.attr.type < PERF_TYPE_MAX && counter->core.attr.type != PERF_TYPE_RAW) { + /* Legacy event, don't uniquify. */ + return false; + } + + if (counter->pmu && counter->pmu->is_core && + counter->alternate_hw_config != PERF_COUNT_HW_MAX) { + /* A sysfs or json event replacing a legacy event, don't uniquify. */ + return false; + } + + if (config->aggr_mode == AGGR_NONE) { + /* Always unique with no aggregation. */ + counter->needs_uniquify = true; + return true; + } + + if (counter->first_wildcard_match != NULL) { + /* + * If stats are merged then only the first_wildcard_match is + * displayed, there is no need to uniquify this evsel as the + * name won't be shown. + */ + return false; + } + + /* + * Do other non-merged events in the evlist have the same name? If so + * uniquify is necessary. + */ + evlist__for_each_entry(counter->evlist, evsel) { + if (evsel == counter || evsel->first_wildcard_match || evsel->pmu == counter->pmu) + continue; + + if (evsel__name_is(counter, evsel__name(evsel))) { + counter->needs_uniquify = true; + return true; + } + } + return false; +} + +void evsel__uniquify_counter(struct evsel *counter) +{ + const char *name, *pmu_name, *config; + char *new_name; + int len, ret; + + /* No uniquification necessary. */ + if (!counter->needs_uniquify) + return; + + /* The evsel was already uniquified. */ + if (counter->uniquified_name) + return; + + /* Avoid checking to uniquify twice. */ + counter->uniquified_name = true; + + name = evsel__name(counter); + config = strchr(name, '/'); + pmu_name = counter->pmu->name; + + /* Already prefixed by the PMU name? */ + len = pmu_name_len_no_suffix(pmu_name); + + if (!strncmp(name, pmu_name, len)) { + /* + * If the PMU name is there, then there is no sense in not + * having a slash. Do this for robustness. + */ + if (config == NULL) + config = name - 1; + + ret = asprintf(&new_name, "%s/%s", pmu_name, config + 1); + } else if (config) { + len = config - name; + if (config[1] == '/') { + /* case: event// */ + ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 2); + } else { + /* case: event/.../ */ + ret = asprintf(&new_name, "%s/%.*s,%s", pmu_name, len, name, config + 1); + } + } else { + config = strchr(name, ':'); + if (config) { + /* case: event:.. */ + len = config - name; + + ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 1); + } else { + /* case: event */ + ret = asprintf(&new_name, "%s/%s/", pmu_name, name); + } + } + if (ret > 0) { + free(counter->name); + counter->name = new_name; + } else { + /* ENOMEM from asprintf. */ + counter->uniquified_name = false; + } +} + +void evsel__warn_user_requested_cpus(struct evsel *evsel, struct perf_cpu_map *user_requested_cpus) +{ + struct perf_cpu_map *intersect, *online = NULL; + const struct perf_pmu *pmu = evsel__find_pmu(evsel); + + if (pmu && pmu->is_core) { + intersect = perf_cpu_map__intersect(pmu->cpus, user_requested_cpus); + } else { + online = cpu_map__online(); + intersect = perf_cpu_map__intersect(online, user_requested_cpus); + } + if (!perf_cpu_map__equal(intersect, user_requested_cpus)) { + char buf1[128]; + char buf2[128]; + + cpu_map__snprint(user_requested_cpus, buf1, sizeof(buf1)); + cpu_map__snprint(online ?: pmu->cpus, buf2, sizeof(buf2)); + pr_warning("WARNING: A requested CPU in '%s' is not supported by PMU '%s' (CPUs %s) for event '%s'\n", + buf1, pmu ? pmu->name : "cpu", buf2, evsel__name(evsel)); + } + perf_cpu_map__put(intersect); + perf_cpu_map__put(online); +} diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h new file mode 100644 index 000000000..d9ecc6628 --- /dev/null +++ b/tools/perf/util/evsel.h @@ -0,0 +1,632 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_EVSEL_H +#define __PERF_EVSEL_H 1 + +#include + +#include +#include +#include +#include +#include + +#include +#include + +#include "symbol_conf.h" + +struct bperf_follower_bpf; +struct bperf_leader_bpf; +struct bpf_counter_ops; +struct bpf_object; +struct cgroup; +struct hashmap; +struct perf_counts; +struct perf_pmu; +struct perf_stat_config; +struct perf_stat_evsel; +struct target; +union perf_event; + +typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); + +/** struct evsel - event selector + * + * @evlist - evlist this evsel is in, if it is in one. + * @core - libperf evsel object + * @name - Can be set to retain the original event name passed by the user, + * so that when showing results in tools such as 'perf stat', we + * show the name used, not some alias. + * @id_pos: the position of the event id (PERF_SAMPLE_ID or + * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of + * struct perf_record_sample + * @is_pos: the position (counting backwards) of the event id (PERF_SAMPLE_ID or + * PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if sample_id_all + * is used there is an id sample appended to non-sample events + * @priv: And what is in its containing unnamed union are tool specific + */ +struct evsel { + struct perf_evsel core; + struct evlist *evlist; + refcount_t refcnt; + off_t id_offset; + int id_pos; + int is_pos; + unsigned int sample_size; + + /* + * These fields can be set in the parse-events code or similar. + * Please check evsel__clone() to copy them properly so that + * they can be released properly. + */ + struct { + char *name; + char *group_name; + const char *group_pmu_name; +#ifdef HAVE_LIBTRACEEVENT + char *tp_sys; + char *tp_name; + struct tep_event *tp_format; +#endif + char *filter; + unsigned long max_events; + double scale; + const char *unit; + struct cgroup *cgrp; + const char *metric_id; + + /* The PMU the event is from. Used for missing_features, PMU name, etc. */ + struct perf_pmu *pmu; + + /* + * This point to the first evsel with the same name, intended to store the + * aggregated counts in aggregation mode. + */ + struct evsel *first_wildcard_match; + /* parse modifier helper */ + int exclude_GH; + int sample_read; + bool snapshot; + bool per_pkg; + bool percore; + bool precise_max; + bool is_libpfm_event; + bool collect_stat; + bool weak_group; + bool bpf_counter; + bool use_config_name; + bool skippable; + bool retire_lat; + bool dont_regroup; + bool default_metricgroup; /* A member of the Default metricgroup */ + bool default_show_events; /* If a default group member, show the event */ + struct list_head config_terms; + u64 alternate_hw_config; + }; + + /* + * metric fields are similar, but needs more care as they can have + * references to other metric (evsel). + */ + struct evsel *metric_leader; + + void *handler; + struct perf_counts *counts; + struct perf_counts *prev_raw_counts; + unsigned long nr_events_printed; + struct perf_stat_evsel *stats; + void *priv; + u64 db_id; + bool uniquified_name; + bool supported; + bool needs_swap; + bool disabled; + bool no_aux_samples; + bool immediate; + bool tracking; + bool ignore_missing_thread; + bool forced_leader; + bool cmdline_group_boundary; + bool reset_group; + bool needs_auxtrace_mmap; + bool needs_uniquify; + bool fallenback_eacces; + bool fallenback_eopnotsupp; + u8 probe_type:3; + struct hashmap *per_pkg_mask; + int err; + int script_output_type; + struct { + evsel__sb_cb_t *cb; + void *data; + } side_band; + /* + * For reporting purposes, an evsel sample can have a callchain + * synthesized from AUX area data. Keep track of synthesized sample + * types here. Note, the recorded sample_type cannot be changed because + * it is needed to continue to parse events. + * See also evsel__has_callchain(). + */ + __u64 synth_sample_type; + + /* + * Store the branch counter related information. + * br_cntr_idx: The idx of the branch counter event in the evlist + * br_cntr_nr: The number of the branch counter event in the group + * (Only available for the leader event) + * abbr_name: The abbreviation name assigned to an event which is + * logged by the branch counter. + * The abbr name is from A to Z9. NA is applied if out + * of the range. + */ + int br_cntr_idx; + int br_cntr_nr; + char abbr_name[3]; + + /* + * bpf_counter_ops serves two use cases: + * 1. perf-stat -b counting events used byBPF programs + * 2. perf-stat --use-bpf use BPF programs to aggregate counts + */ + struct bpf_counter_ops *bpf_counter_ops; + + struct list_head bpf_counter_list; /* for perf-stat -b */ + struct list_head bpf_filters; /* for perf-record --filter */ + + /* for perf-stat --use-bpf */ + int bperf_leader_prog_fd; + int bperf_leader_link_fd; + union { + struct bperf_leader_bpf *leader_skel; + struct bperf_follower_bpf *follower_skel; + void *bpf_skel; + }; + unsigned long open_flags; + int precise_ip_original; + + /* For tool events */ + /* Beginning time subtracted when the counter is read. */ + union { + /* Defaults for retirement latency events. */ + struct _retirement_latency { + double mean; + double min; + double max; + } retirement_latency; + /* duration_time is a single global time. */ + struct { + __u64 start_time; + __u64 accumulated_time; + } duration_time; + /* + * user_time and system_time read an initial value potentially + * per-CPU or per-pid. + */ + struct { + struct xyarray *start_times; + struct xyarray *accumulated_times; + } process_time; + }; + /* Is the tool's fd for /proc/pid/stat or /proc/stat. */ + bool pid_stat; +}; + +struct perf_missing_features { + bool sample_id_all; + bool exclude_guest; + bool mmap2; + bool cloexec; + bool clockid; + bool clockid_wrong; + bool lbr_flags; + bool write_backward; + bool group_read; + bool ksymbol; + bool bpf; + bool aux_output; + bool branch_hw_idx; + bool cgroup; + bool data_page_size; + bool code_page_size; + bool weight_struct; + bool read_lost; + bool branch_counters; + bool aux_action; + bool inherit_sample_read; + bool defer_callchain; +}; + +extern struct perf_missing_features perf_missing_features; + +struct perf_cpu_map; +struct thread_map; +struct record_opts; + +static inline struct perf_cpu_map *evsel__cpus(struct evsel *evsel) +{ + return perf_evsel__cpus(&evsel->core); +} + +static inline int evsel__nr_cpus(struct evsel *evsel) +{ + return perf_cpu_map__nr(evsel__cpus(evsel)); +} + +void evsel__compute_deltas(struct evsel *evsel, int cpu, int thread, + struct perf_counts_values *count); + +int evsel__object_config(size_t object_size, + int (*init)(struct evsel *evsel), + void (*fini)(struct evsel *evsel)); + +struct perf_pmu *evsel__find_pmu(const struct evsel *evsel); +const char *evsel__pmu_name(const struct evsel *evsel); +bool evsel__is_aux_event(const struct evsel *evsel); + +bool evsel__is_probe(struct evsel *evsel); +bool evsel__is_kprobe(struct evsel *evsel); +bool evsel__is_uprobe(struct evsel *evsel); + +struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx); + +static inline struct evsel *evsel__new(struct perf_event_attr *attr) +{ + return evsel__new_idx(attr, 0); +} + +struct evsel *evsel__clone(struct evsel *orig); + +int copy_config_terms(struct list_head *dst, struct list_head *src); +void free_config_terms(struct list_head *config_terms); + + +/* + * Returns pointer with encoded error via interface. + */ +struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool format); +static inline struct evsel *evsel__newtp(const char *sys, const char *name) +{ + return evsel__newtp_idx(sys, name, 0, true); +} + +struct evsel *evsel__get(struct evsel *evsel); +void evsel__put(struct evsel *evsel); + +#ifdef HAVE_LIBTRACEEVENT +struct tep_event *evsel__tp_format(struct evsel *evsel); +#endif + +void evsel__set_priv_destructor(void (*destructor)(void *priv)); + +struct callchain_param; + +void evsel__config(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *callchain); +void evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *callchain); + +int __evsel__sample_size(u64 sample_type); +void evsel__calc_id_pos(struct evsel *evsel); + +bool evsel__is_cache_op_valid(u8 type, u8 op); + +static inline bool evsel__is_bpf(struct evsel *evsel) +{ + return evsel->bpf_counter_ops != NULL; +} + +static inline bool evsel__is_bperf(struct evsel *evsel) +{ + return evsel->bpf_counter_ops != NULL && list_empty(&evsel->bpf_counter_list); +} + +#define EVSEL__MAX_ALIASES 8 + +extern const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES]; +extern const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES]; +extern const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES]; +extern const char *const evsel__hw_names[PERF_COUNT_HW_MAX]; +extern const char *const evsel__sw_names[PERF_COUNT_SW_MAX]; +extern char *evsel__bpf_counter_events; +bool evsel__match_bpf_counter_events(const char *name); +int arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size); + +int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size); +const char *evsel__name(struct evsel *evsel); +bool evsel__name_is(struct evsel *evsel, const char *name); +const char *evsel__metric_id(const struct evsel *evsel); + +static inline bool evsel__is_retire_lat(const struct evsel *evsel) +{ + return evsel->retire_lat; +} + +const char *evsel__group_name(struct evsel *evsel); +int evsel__group_desc(struct evsel *evsel, char *buf, size_t size); + +void __evsel__set_sample_bit(struct evsel *evsel, enum perf_event_sample_format bit); +void __evsel__reset_sample_bit(struct evsel *evsel, enum perf_event_sample_format bit); + +#define evsel__set_sample_bit(evsel, bit) \ + __evsel__set_sample_bit(evsel, PERF_SAMPLE_##bit) + +#define evsel__reset_sample_bit(evsel, bit) \ + __evsel__reset_sample_bit(evsel, PERF_SAMPLE_##bit) + +void evsel__set_sample_id(struct evsel *evsel, bool use_sample_identifier); + +void arch_evsel__set_sample_weight(struct evsel *evsel); +void arch__post_evsel_config(struct evsel *evsel, struct perf_event_attr *attr); +int arch_evsel__open_strerror(struct evsel *evsel, int err, char *msg, size_t size); +void arch_evsel__apply_ratio_to_prev(struct evsel *evsel, struct perf_event_attr *attr); + +int evsel__set_filter(struct evsel *evsel, const char *filter); +int evsel__append_tp_filter(struct evsel *evsel, const char *filter); +int evsel__append_addr_filter(struct evsel *evsel, const char *filter); + +int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx); +int evsel__enable(struct evsel *evsel); +int evsel__disable(struct evsel *evsel); +int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx); + +int evsel__open_per_cpu_and_thread(struct evsel *evsel, + struct perf_cpu_map *cpus, int cpu_map_idx, + struct perf_thread_map *threads); +int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx); +int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads); +int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads); +void evsel__close(struct evsel *evsel); +int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads); + +bool evsel__precise_ip_fallback(struct evsel *evsel); + +struct perf_sample; + +#ifdef HAVE_LIBTRACEEVENT +void *perf_sample__rawptr(struct perf_sample *sample, const char *name); +u64 perf_sample__intval(struct perf_sample *sample, const char *name); +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name); +char perf_sample__taskstate(struct perf_sample *sample, const char *name); + +static inline char *perf_sample__strval(struct perf_sample *sample, const char *name) +{ + return perf_sample__rawptr(sample, name); +} +#endif + +struct tep_format_field; + +void *format_field__get_raw_data(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out); +unsigned long *format_field__get_cpumask(struct tep_format_field *field, + struct perf_sample *sample, + bool needs_swap, u16 *len_out); +u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); + +#ifdef HAVE_LIBTRACEEVENT +struct tep_format_field *evsel__field(struct evsel *evsel, const char *name); +struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name); +#else +static inline struct tep_format_field * +evsel__field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} + +static inline struct tep_format_field * +evsel__common_field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} +#endif + +bool __evsel__match(const struct evsel *evsel, u32 type, u64 config); + +#define evsel__match(evsel, t, c) __evsel__match(evsel, PERF_TYPE_##t, PERF_COUNT_##c) + +int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread); + +int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale); + +/** + * evsel__read_on_cpu - Read out the results on a CPU and thread + * + * @evsel - event selector to read value + * @cpu_map_idx - CPU of interest + * @thread - thread of interest + */ +static inline int evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread) +{ + return __evsel__read_on_cpu(evsel, cpu_map_idx, thread, false); +} + +/** + * evsel__read_on_cpu_scaled - Read out the results on a CPU and thread, scaled + * + * @evsel - event selector to read value + * @cpu_map_idx - CPU of interest + * @thread - thread of interest + */ +static inline int evsel__read_on_cpu_scaled(struct evsel *evsel, int cpu_map_idx, int thread) +{ + return __evsel__read_on_cpu(evsel, cpu_map_idx, thread, true); +} + +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap); + +static inline int evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data) +{ + return __evsel__parse_sample(evsel, event, data, evsel->needs_swap); +} + +int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, + u64 *timestamp); + +u16 evsel__id_hdr_size(const struct evsel *evsel); + +static inline struct evsel *evsel__next(struct evsel *evsel) +{ + return list_entry(evsel->core.node.next, struct evsel, core.node); +} + +static inline struct evsel *evsel__prev(struct evsel *evsel) +{ + return list_entry(evsel->core.node.prev, struct evsel, core.node); +} + +/** + * evsel__is_group_leader - Return whether given evsel is a leader event + * + * @evsel - evsel selector to be tested + * + * Return %true if @evsel is a group leader or a stand-alone event + */ +static inline bool evsel__is_group_leader(const struct evsel *evsel) +{ + return evsel->core.leader == &evsel->core; +} + +/** + * evsel__is_group_event - Return whether given evsel is a group event + * + * @evsel - evsel selector to be tested + * + * Return %true iff event group view is enabled and @evsel is a actual group + * leader which has other members in the group + */ +static inline bool evsel__is_group_event(struct evsel *evsel) +{ + if (!symbol_conf.event_group) + return false; + + return evsel__is_group_leader(evsel) && evsel->core.nr_members > 1; +} + +bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel); +bool evsel__is_function_event(struct evsel *evsel); + +static inline bool evsel__is_bpf_output(struct evsel *evsel) +{ + return evsel__match(evsel, SOFTWARE, SW_BPF_OUTPUT); +} + +static inline bool evsel__is_clock(const struct evsel *evsel) +{ + return evsel__match(evsel, SOFTWARE, SW_CPU_CLOCK) || + evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK); +} + +bool evsel__fallback(struct evsel *evsel, struct target *target, int err, + char *msg, size_t msgsize); +int evsel__open_strerror(struct evsel *evsel, struct target *target, + int err, char *msg, size_t size); + +static inline int evsel__group_idx(struct evsel *evsel) +{ + return evsel->core.idx - evsel->core.leader->idx; +} + +/* Iterates group WITHOUT the leader. */ +#define for_each_group_member_head(_evsel, _leader, _head) \ +for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node); \ + (_evsel) && &(_evsel)->core.node != (_head) && \ + (_evsel)->core.leader == &(_leader)->core; \ + (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) + +#define for_each_group_member(_evsel, _leader) \ + for_each_group_member_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) + +/* Iterates group WITH the leader. */ +#define for_each_group_evsel_head(_evsel, _leader, _head) \ +for ((_evsel) = _leader; \ + (_evsel) && &(_evsel)->core.node != (_head) && \ + (_evsel)->core.leader == &(_leader)->core; \ + (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) + +#define for_each_group_evsel(_evsel, _leader) \ + for_each_group_evsel_head(_evsel, _leader, &evlist__core((_leader)->evlist)->entries) + +static inline bool evsel__has_branch_callstack(const struct evsel *evsel) +{ + return evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK; +} + +static inline bool evsel__has_branch_hw_idx(const struct evsel *evsel) +{ + return evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX; +} + +static inline bool evsel__has_callchain(const struct evsel *evsel) +{ + /* + * For reporting purposes, an evsel sample can have a recorded callchain + * or a callchain synthesized from AUX area data. + */ + return evsel->core.attr.sample_type & PERF_SAMPLE_CALLCHAIN || + evsel->synth_sample_type & PERF_SAMPLE_CALLCHAIN; +} + +static inline bool evsel__has_br_stack(const struct evsel *evsel) +{ + /* + * For reporting purposes, an evsel sample can have a recorded branch + * stack or a branch stack synthesized from AUX area data. + */ + return evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK || + evsel->synth_sample_type & PERF_SAMPLE_BRANCH_STACK; +} + +static inline bool evsel__is_dummy_event(struct evsel *evsel) +{ + return (evsel->core.attr.type == PERF_TYPE_SOFTWARE) && + (evsel->core.attr.config == PERF_COUNT_SW_DUMMY); +} + +struct perf_session *evsel__session(struct evsel *evsel); +struct perf_env *evsel__env(struct evsel *evsel); +uint16_t evsel__e_machine(struct evsel *evsel, uint32_t *e_flags); + +int evsel__store_ids(struct evsel *evsel, struct evlist *evlist); + +void evsel__zero_per_pkg(struct evsel *evsel); +bool evsel__is_hybrid(const struct evsel *evsel); +struct evsel *evsel__leader(const struct evsel *evsel); +bool evsel__has_leader(struct evsel *evsel, struct evsel *leader); +bool evsel__is_leader(struct evsel *evsel); +void evsel__set_leader(struct evsel *evsel, struct evsel *leader); +int evsel__source_count(const struct evsel *evsel); +void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader); + +bool arch_evsel__must_be_in_group(const struct evsel *evsel); + +bool evsel__set_needs_uniquify(struct evsel *counter, const struct perf_stat_config *config); +void evsel__uniquify_counter(struct evsel *counter); + +/* + * Macro to swap the bit-field postition and size. + * Used when, + * - dont need to swap the entire u64 && + * - when u64 has variable bit-field sizes && + * - when presented in a host endian which is different + * than the source endian of the perf.data file + */ +#define bitfield_swap(src, pos, size) \ + ((((src) >> (pos)) & ((1ull << (size)) - 1)) << (63 - ((pos) + (size) - 1))) + +u64 evsel__bitfield_swap_branch_flags(u64 value); +bool evsel__config_exists(const struct evsel *evsel, const char *config_name); +int evsel__get_config_val(const struct evsel *evsel, const char *config_name, + u64 *val); +void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, + u64 val); + +bool evsel__is_offcpu_event(struct evsel *evsel); + +void evsel__warn_user_requested_cpus(struct evsel *evsel, struct perf_cpu_map *user_requested_cpus); + +#endif /* __PERF_EVSEL_H */ diff --git a/tools/perf/util/evsel_config.h b/tools/perf/util/evsel_config.h new file mode 100644 index 000000000..7b565d76c --- /dev/null +++ b/tools/perf/util/evsel_config.h @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __PERF_EVSEL_CONFIG_H +#define __PERF_EVSEL_CONFIG_H 1 + +#include +#include + +/* + * The 'struct evsel_config_term' is used to pass event + * specific configuration data to evsel__config routine. + * It is allocated within event parsing and attached to + * evsel::config_terms list head. +*/ +enum evsel_term_type { + EVSEL__CONFIG_TERM_PERIOD, + EVSEL__CONFIG_TERM_FREQ, + EVSEL__CONFIG_TERM_TIME, + EVSEL__CONFIG_TERM_CALLGRAPH, + EVSEL__CONFIG_TERM_STACK_USER, + EVSEL__CONFIG_TERM_INHERIT, + EVSEL__CONFIG_TERM_MAX_STACK, + EVSEL__CONFIG_TERM_MAX_EVENTS, + EVSEL__CONFIG_TERM_OVERWRITE, + EVSEL__CONFIG_TERM_DRV_CFG, + EVSEL__CONFIG_TERM_BRANCH, + EVSEL__CONFIG_TERM_PERCORE, + EVSEL__CONFIG_TERM_AUX_OUTPUT, + EVSEL__CONFIG_TERM_AUX_ACTION, + EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG1, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG2, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG3, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG4, + EVSEL__CONFIG_TERM_RATIO_TO_PREV, +}; + +struct evsel_config_term { + struct list_head list; + enum evsel_term_type type; + bool free_str; + union { + u64 period; + u64 freq; + bool time; + u64 stack_user; + int max_stack; + bool inherit; + bool overwrite; + unsigned long max_events; + bool percore; + bool aux_output; + u32 aux_sample_size; + u64 cfg_chg; + char *str; + int cpu; + u64 val; + } val; + bool weak; +}; + +struct evsel; + +struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type); + +#define evsel__get_config_term(evsel, type) \ + __evsel__get_config_term(evsel, EVSEL__CONFIG_TERM_ ## type) + +#endif // __PERF_EVSEL_CONFIG_H diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c new file mode 100644 index 000000000..0f7a25500 --- /dev/null +++ b/tools/perf/util/evsel_fprintf.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "util/evlist.h" +#include "evsel.h" +#include "util/evsel_fprintf.h" +#include "util/event.h" +#include "callchain.h" +#include "map.h" +#include "strlist.h" +#include "symbol.h" +#include "srcline.h" +#include "dso.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +static int comma_fprintf(FILE *fp, bool *first, const char *fmt, ...) +{ + va_list args; + int ret = 0; + + if (!*first) { + ret += fprintf(fp, ","); + } else { + ret += fprintf(fp, ":"); + *first = false; + } + + va_start(args, fmt); + ret += vfprintf(fp, fmt, args); + va_end(args); + return ret; +} + +static int __print_attr__fprintf(FILE *fp, const char *name, const char *val, void *priv) +{ + return comma_fprintf(fp, (bool *)priv, " %s: %s", name, val); +} + +int evsel__fprintf(struct evsel *evsel, struct perf_attr_details *details, FILE *fp) +{ + bool first = true; + int printed = 0; + + if (details->event_group) { + struct evsel *pos; + + if (!evsel__is_group_leader(evsel)) + return 0; + + if (evsel->core.nr_members > 1) + printed += fprintf(fp, "%s{", evsel->group_name ?: ""); + + printed += fprintf(fp, "%s", evsel__name(evsel)); + for_each_group_member(pos, evsel) + printed += fprintf(fp, ",%s", evsel__name(pos)); + + if (evsel->core.nr_members > 1) + printed += fprintf(fp, "}"); + goto out; + } + + printed += fprintf(fp, "%s", evsel__name(evsel)); + + if (details->verbose) { + printed += perf_event_attr__fprintf(fp, &evsel->core.attr, + __print_attr__fprintf, &first); + } else if (details->freq) { + const char *term = "sample_freq"; + + if (!evsel->core.attr.freq) + term = "sample_period"; + + printed += comma_fprintf(fp, &first, " %s=%" PRIu64, + term, (u64)evsel->core.attr.sample_freq); + } + +#ifdef HAVE_LIBTRACEEVENT + if (details->trace_fields) { + struct tep_format_field *field; + const struct tep_event *tp_format; + + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) { + printed += comma_fprintf(fp, &first, " (not a tracepoint)"); + goto out; + } + + tp_format = evsel__tp_format(evsel); + field = tp_format ? tp_format->format.fields : NULL; + if (field == NULL) { + printed += comma_fprintf(fp, &first, " (no trace field)"); + goto out; + } + + printed += comma_fprintf(fp, &first, " trace_fields: %s", field->name); + + field = field->next; + while (field) { + printed += comma_fprintf(fp, &first, "%s", field->name); + field = field->next; + } + } +#endif +out: + fputc('\n', fp); + return ++printed; +} + +int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, + unsigned int print_opts, struct callchain_cursor *cursor, + struct strlist *bt_stop_list, FILE *fp) +{ + int printed = 0; + struct callchain_cursor_node *node; + int print_ip = print_opts & EVSEL__PRINT_IP; + int print_sym = print_opts & EVSEL__PRINT_SYM; + int print_dso = print_opts & EVSEL__PRINT_DSO; + int print_dsoff = print_opts & EVSEL__PRINT_DSOFF; + int print_symoffset = print_opts & EVSEL__PRINT_SYMOFFSET; + int print_oneline = print_opts & EVSEL__PRINT_ONELINE; + int print_srcline = print_opts & EVSEL__PRINT_SRCLINE; + int print_unknown_as_addr = print_opts & EVSEL__PRINT_UNKNOWN_AS_ADDR; + int print_arrow = print_opts & EVSEL__PRINT_CALLCHAIN_ARROW; + int print_skip_ignored = print_opts & EVSEL__PRINT_SKIP_IGNORED; + char s = print_oneline ? ' ' : '\t'; + bool first = true; + + if (cursor == NULL) + return fprintf(fp, "%s", print_oneline ? "" : "\n"); + + if (sample->callchain) { + callchain_cursor_commit(cursor); + + while (1) { + struct symbol *sym; + struct map *map; + u64 addr = 0; + + node = callchain_cursor_current(cursor); + if (!node) + break; + + sym = node->ms.sym; + map = node->ms.map; + + if (sym && symbol__ignore(sym) && print_skip_ignored) + goto next; + + printed += fprintf(fp, "%-*.*s", left_alignment, left_alignment, " "); + + if (print_arrow && !first) + printed += fprintf(fp, " <-"); + + if (map) + addr = map__map_ip(map, node->ip); + + if (print_ip) + printed += fprintf(fp, "%c%16" PRIx64, s, node->ip); + + if (print_sym) { + struct addr_location node_al; + + addr_location__init(&node_al); + printed += fprintf(fp, " "); + node_al.addr = addr; + node_al.map = map__get(map); + + if (sample->deferred_callchain && + sample->deferred_cookie == node->ip) { + printed += fprintf(fp, "(cookie)"); + } else if (print_symoffset) { + printed += __symbol__fprintf_symname_offs(sym, &node_al, + print_unknown_as_addr, + true, fp); + } else { + printed += __symbol__fprintf_symname(sym, &node_al, + print_unknown_as_addr, fp); + } + addr_location__exit(&node_al); + } + + if (print_dso && (!sym || !symbol__inlined(sym))) + printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); + + if (print_srcline) { + if (node->srcline) + printed += fprintf(fp, "\n %s", node->srcline); + else + printed += map__fprintf_srcline(map, addr, "\n ", fp); + } + + if (sym && symbol__inlined(sym)) + printed += fprintf(fp, " (inlined)"); + + if (!print_oneline) + printed += fprintf(fp, "\n"); + + /* Add srccode here too? */ + if (bt_stop_list && sym && + strlist__has_entry(bt_stop_list, sym->name)) { + break; + } + + first = false; +next: + callchain_cursor_advance(cursor); + } + } + + return printed; +} + +int sample__fprintf_sym(struct perf_sample *sample, struct addr_location *al, + int left_alignment, unsigned int print_opts, + struct callchain_cursor *cursor, struct strlist *bt_stop_list, FILE *fp) +{ + int printed = 0; + int print_ip = print_opts & EVSEL__PRINT_IP; + int print_sym = print_opts & EVSEL__PRINT_SYM; + int print_dso = print_opts & EVSEL__PRINT_DSO; + int print_dsoff = print_opts & EVSEL__PRINT_DSOFF; + int print_symoffset = print_opts & EVSEL__PRINT_SYMOFFSET; + int print_srcline = print_opts & EVSEL__PRINT_SRCLINE; + int print_unknown_as_addr = print_opts & EVSEL__PRINT_UNKNOWN_AS_ADDR; + + if (cursor != NULL) { + printed += sample__fprintf_callchain(sample, left_alignment, print_opts, + cursor, bt_stop_list, fp); + } else { + printed += fprintf(fp, "%-*.*s", left_alignment, left_alignment, " "); + + if (print_ip) + printed += fprintf(fp, "%16" PRIx64, sample->ip); + + if (print_sym) { + printed += fprintf(fp, " "); + if (print_symoffset) { + printed += __symbol__fprintf_symname_offs(al->sym, al, + print_unknown_as_addr, + true, fp); + } else { + printed += __symbol__fprintf_symname(al->sym, al, + print_unknown_as_addr, fp); + } + } + + if (print_dso) + printed += map__fprintf_dsoname_dsoff(al->map, print_dsoff, al->addr, fp); + + if (print_srcline) + printed += map__fprintf_srcline(al->map, al->addr, "\n ", fp); + } + + return printed; +} diff --git a/tools/perf/util/evsel_fprintf.h b/tools/perf/util/evsel_fprintf.h new file mode 100644 index 000000000..c8a9fac2f --- /dev/null +++ b/tools/perf/util/evsel_fprintf.h @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __PERF_EVSEL_FPRINTF_H +#define __PERF_EVSEL_FPRINTF_H 1 + +#include +#include + +struct evsel; + +struct perf_attr_details { + bool freq; + bool verbose; + bool event_group; + bool force; + bool trace_fields; +}; + +int evsel__fprintf(struct evsel *evsel, struct perf_attr_details *details, FILE *fp); + +#define EVSEL__PRINT_IP (1<<0) +#define EVSEL__PRINT_SYM (1<<1) +#define EVSEL__PRINT_DSO (1<<2) +#define EVSEL__PRINT_SYMOFFSET (1<<3) +#define EVSEL__PRINT_ONELINE (1<<4) +#define EVSEL__PRINT_SRCLINE (1<<5) +#define EVSEL__PRINT_UNKNOWN_AS_ADDR (1<<6) +#define EVSEL__PRINT_CALLCHAIN_ARROW (1<<7) +#define EVSEL__PRINT_SKIP_IGNORED (1<<8) +#define EVSEL__PRINT_DSOFF (1<<9) + +struct addr_location; +struct perf_event_attr; +struct perf_sample; +struct callchain_cursor; +struct strlist; + +int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, + unsigned int print_opts, struct callchain_cursor *cursor, + struct strlist *bt_stop_list, FILE *fp); + +int sample__fprintf_sym(struct perf_sample *sample, struct addr_location *al, + int left_alignment, unsigned int print_opts, + struct callchain_cursor *cursor, + struct strlist *bt_stop_list, FILE *fp); + +typedef int (*attr__fprintf_f)(FILE *, const char *, const char *, void *); + +int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, + attr__fprintf_f attr__fprintf, void *priv); +#endif // __PERF_EVSEL_H diff --git a/tools/perf/util/evswitch.c b/tools/perf/util/evswitch.c new file mode 100644 index 000000000..d4c06a3f8 --- /dev/null +++ b/tools/perf/util/evswitch.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019, Red Hat Inc, Arnaldo Carvalho de Melo + +#include +#include "evswitch.h" +#include "evlist.h" + +bool evswitch__discard(struct evswitch *evswitch, struct evsel *evsel) +{ + if (evswitch->on && evswitch->discarding) { + if (evswitch->on != evsel) + return true; + + evswitch->discarding = false; + + if (!evswitch->show_on_off_events) + return true; + + return false; + } + + if (evswitch->off && !evswitch->discarding) { + if (evswitch->off != evsel) + return false; + + evswitch->discarding = true; + + if (!evswitch->show_on_off_events) + return true; + } + + return false; +} + +static int evswitch__fprintf_enoent(FILE *fp, const char *evtype, const char *evname) +{ + int printed = fprintf(fp, "ERROR: switch-%s event not found (%s)\n", evtype, evname); + + return printed += fprintf(fp, "HINT: use 'perf evlist' to see the available event names\n"); +} + +int evswitch__init(struct evswitch *evswitch, struct evlist *evlist, FILE *fp) +{ + if (evswitch->on_name) { + evswitch->on = evlist__find_evsel_by_str(evlist, evswitch->on_name); + if (evswitch->on == NULL) { + evswitch__fprintf_enoent(fp, "on", evswitch->on_name); + return -ENOENT; + } + evswitch->discarding = true; + } + + if (evswitch->off_name) { + evswitch->off = evlist__find_evsel_by_str(evlist, evswitch->off_name); + if (evswitch->off == NULL) { + evswitch__fprintf_enoent(fp, "off", evswitch->off_name); + return -ENOENT; + } + } + + return 0; +} diff --git a/tools/perf/util/evswitch.h b/tools/perf/util/evswitch.h new file mode 100644 index 000000000..8ffdbe526 --- /dev/null +++ b/tools/perf/util/evswitch.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019, Red Hat Inc, Arnaldo Carvalho de Melo +#ifndef __PERF_EVSWITCH_H +#define __PERF_EVSWITCH_H 1 + +#include +#include + +struct evsel; +struct evlist; + +struct evswitch { + struct evsel *on, *off; + const char *on_name, *off_name; + bool discarding; + bool show_on_off_events; +}; + +int evswitch__init(struct evswitch *evswitch, struct evlist *evlist, FILE *fp); + +bool evswitch__discard(struct evswitch *evswitch, struct evsel *evsel); + +#define OPTS_EVSWITCH(evswitch) \ + OPT_STRING(0, "switch-on", &(evswitch)->on_name, \ + "event", "Consider events after the occurrence of this event"), \ + OPT_STRING(0, "switch-off", &(evswitch)->off_name, \ + "event", "Stop considering events after the occurrence of this event"), \ + OPT_BOOLEAN(0, "show-on-off-events", &(evswitch)->show_on_off_events, \ + "Show the on/off switch events, used with --switch-on and --switch-off") + +#endif /* __PERF_EVSWITCH_H */ diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c new file mode 100644 index 000000000..8aef3c741 --- /dev/null +++ b/tools/perf/util/expr.c @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "metricgroup.h" +#include "debug.h" +#include "evlist.h" +#include "expr.h" +#include "smt.h" +#include "tool_pmu.h" +#include +#include +#include "util/hashmap.h" +#include "util/header.h" +#include "util/pmu.h" +#include +#include +#include +#include +#include +#include + +struct expr_id_data { + union { + struct { + double val; + int source_count; + int aggr_nr; + } val; + struct { + double val; + const char *metric_name; + const char *metric_expr; + } ref; + }; + + enum { + /* Holding a double value. */ + EXPR_ID_DATA__VALUE, + /* Reference to another metric. */ + EXPR_ID_DATA__REF, + /* A reference but the value has been computed. */ + EXPR_ID_DATA__REF_VALUE, + } kind; +}; + +static size_t key_hash(long key, void *ctx __maybe_unused) +{ + const char *str = (const char *)key; + size_t hash = 0; + + while (*str != '\0') { + hash *= 31; + hash += *str; + str++; + } + return hash; +} + +static bool key_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return !strcmp((const char *)key1, (const char *)key2); +} + +struct hashmap *ids__new(void) +{ + struct hashmap *hash; + + hash = hashmap__new(key_hash, key_equal, NULL); + if (IS_ERR(hash)) + return NULL; + return hash; +} + +void ids__free(struct hashmap *ids) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (ids == NULL) + return; + + hashmap__for_each_entry(ids, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + + hashmap__free(ids); +} + +int ids__insert(struct hashmap *ids, const char *id) +{ + struct expr_id_data *data_ptr = NULL, *old_data = NULL; + char *old_key = NULL; + int ret; + + ret = hashmap__set(ids, id, data_ptr, &old_key, &old_data); + if (ret) + free(data_ptr); + free(old_key); + free(old_data); + return ret; +} + +struct hashmap *ids__union(struct hashmap *ids1, struct hashmap *ids2) +{ + size_t bkt; + struct hashmap_entry *cur; + int ret; + struct expr_id_data *old_data = NULL; + char *old_key = NULL; + + if (!ids1) + return ids2; + + if (!ids2) + return ids1; + + if (hashmap__size(ids1) < hashmap__size(ids2)) { + struct hashmap *tmp = ids1; + + ids1 = ids2; + ids2 = tmp; + } + hashmap__for_each_entry(ids2, cur, bkt) { + ret = hashmap__set(ids1, cur->key, cur->value, &old_key, &old_data); + free(old_key); + free(old_data); + + if (ret) { + hashmap__free(ids1); + hashmap__free(ids2); + return NULL; + } + } + hashmap__free(ids2); + return ids1; +} + +/* Caller must make sure id is allocated */ +int expr__add_id(struct expr_parse_ctx *ctx, const char *id) +{ + return ids__insert(ctx->ids, id); +} + +/* Caller must make sure id is allocated */ +int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val) +{ + return expr__add_id_val_source_count(ctx, id, val, /*source_count=*/1); +} + +/* Caller must make sure id is allocated */ +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr) +{ + struct expr_id_data *data_ptr = NULL, *old_data = NULL; + char *old_key = NULL; + int ret; + + data_ptr = malloc(sizeof(*data_ptr)); + if (!data_ptr) + return -ENOMEM; + data_ptr->val.val = val; + data_ptr->val.source_count = source_count; + data_ptr->val.aggr_nr = aggr_nr; + data_ptr->kind = EXPR_ID_DATA__VALUE; + + ret = hashmap__set(ctx->ids, id, data_ptr, &old_key, &old_data); + if (ret) { + free(data_ptr); + } else if (old_data) { + data_ptr->val.val += old_data->val.val; + data_ptr->val.source_count += old_data->val.source_count; + data_ptr->val.aggr_nr += old_data->val.aggr_nr; + } + free(old_key); + free(old_data); + return ret; +} + +/* Caller must make sure id is allocated */ +int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count) +{ + return expr__add_id_val_source_count_aggr_nr(ctx, id, val, source_count, 1); +} + +int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref) +{ + struct expr_id_data *data_ptr = NULL, *old_data = NULL; + char *old_key = NULL; + char *name; + int ret; + + data_ptr = zalloc(sizeof(*data_ptr)); + if (!data_ptr) + return -ENOMEM; + + name = strdup(ref->metric_name); + if (!name) { + free(data_ptr); + return -ENOMEM; + } + + /* + * Intentionally passing just const char pointers, + * originally from 'struct pmu_event' object. + * We don't need to change them, so there's no + * need to create our own copy. + */ + data_ptr->ref.metric_name = ref->metric_name; + data_ptr->ref.metric_expr = ref->metric_expr; + data_ptr->kind = EXPR_ID_DATA__REF; + + ret = hashmap__set(ctx->ids, name, data_ptr, &old_key, &old_data); + if (ret) + free(data_ptr); + + pr_debug2("adding ref metric %s: %s\n", + ref->metric_name, ref->metric_expr); + + free(old_key); + free(old_data); + return ret; +} + +int expr__get_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **data) +{ + if (!ctx || !id) + return -1; + return hashmap__find(ctx->ids, id, data) ? 0 : -1; +} + +bool expr__subset_of_ids(struct expr_parse_ctx *haystack, + struct expr_parse_ctx *needles) +{ + struct hashmap_entry *cur; + size_t bkt; + struct expr_id_data *data; + + hashmap__for_each_entry(needles->ids, cur, bkt) { + if (expr__get_id(haystack, cur->pkey, &data)) + return false; + } + return true; +} + + +int expr__resolve_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **datap) +{ + struct expr_id_data *data; + + if (expr__get_id(ctx, id, datap) || !*datap) { + pr_debug("%s not found\n", id); + return -1; + } + + data = *datap; + + switch (data->kind) { + case EXPR_ID_DATA__VALUE: + pr_debug2("lookup(%s): val %f\n", id, data->val.val); + break; + case EXPR_ID_DATA__REF: + pr_debug2("lookup(%s): ref metric name %s\n", id, + data->ref.metric_name); + pr_debug("processing metric: %s ENTRY\n", id); + data->kind = EXPR_ID_DATA__REF_VALUE; + if (expr__parse(&data->ref.val, ctx, data->ref.metric_expr)) { + pr_debug("%s failed to count\n", id); + return -1; + } + pr_debug("processing metric: %s EXIT: %f\n", id, data->ref.val); + break; + case EXPR_ID_DATA__REF_VALUE: + pr_debug2("lookup(%s): ref val %f metric name %s\n", id, + data->ref.val, data->ref.metric_name); + break; + default: + assert(0); /* Unreachable. */ + } + + return 0; +} + +void expr__del_id(struct expr_parse_ctx *ctx, const char *id) +{ + struct expr_id_data *old_val = NULL; + char *old_key = NULL; + + hashmap__delete(ctx->ids, id, &old_key, &old_val); + free(old_key); + free(old_val); +} + +struct expr_parse_ctx *expr__ctx_new(void) +{ + struct expr_parse_ctx *ctx; + + ctx = calloc(1, sizeof(struct expr_parse_ctx)); + if (!ctx) + return NULL; + + ctx->ids = hashmap__new(key_hash, key_equal, NULL); + if (IS_ERR(ctx->ids)) { + free(ctx); + return NULL; + } + + return ctx; +} + +void expr__ctx_clear(struct expr_parse_ctx *ctx) +{ + struct hashmap_entry *cur; + size_t bkt; + + hashmap__for_each_entry(ctx->ids, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__clear(ctx->ids); +} + +void expr__ctx_free(struct expr_parse_ctx *ctx) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (!ctx) + return; + + zfree(&ctx->sctx.user_requested_cpu_list); + hashmap__for_each_entry(ctx->ids, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__free(ctx->ids); + free(ctx); +} + +static int +__expr__parse(double *val, struct expr_parse_ctx *ctx, const char *expr, + bool compute_ids) +{ + YY_BUFFER_STATE buffer; + void *scanner; + int ret; + + pr_debug2("parsing metric: %s\n", expr); + + ret = expr_lex_init_extra(&ctx->sctx, &scanner); + if (ret) + return ret; + + buffer = expr__scan_string(expr, scanner); + +#ifdef PARSER_DEBUG + expr_debug = 1; + expr_set_debug(1, scanner); +#endif + + ret = expr_parse(val, ctx, compute_ids, scanner); + + expr__flush_buffer(buffer, scanner); + expr__delete_buffer(buffer, scanner); + expr_lex_destroy(scanner); + return ret; +} + +int expr__parse(double *final_val, struct expr_parse_ctx *ctx, + const char *expr) +{ + return __expr__parse(final_val, ctx, expr, /*compute_ids=*/false) ? -1 : 0; +} + +int expr__find_ids(const char *expr, const char *one, + struct expr_parse_ctx *ctx) +{ + int ret = __expr__parse(NULL, ctx, expr, /*compute_ids=*/true); + + if (one) + expr__del_id(ctx, one); + + /* A positive value means syntax error, convert to -EINVAL */ + return ret > 0 ? -EINVAL : ret; +} + +double expr_id_data__value(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.val; + assert(data->kind == EXPR_ID_DATA__REF_VALUE); + return data->ref.val; +} + +double expr_id_data__source_count(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.source_count; + return 1.0; +} + +double expr_id_data__aggr_nr(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.aggr_nr; + return 1.0; +} + +double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx) +{ + double result = NAN; + enum tool_pmu_event ev = tool_pmu__str_to_event(literal + 1); + + if (ev != TOOL_PMU__EVENT_NONE) { + u64 count; + + if (tool_pmu__read_event(ev, /*evsel=*/NULL, + ctx->system_wide, ctx->user_requested_cpu_list, + &count)) + result = count; + else + pr_err("Failure to read '%s'\n", literal); + } else { + pr_err("Unrecognized literal '%s'\n", literal); + } + + pr_debug2("literal: %s = %f\n", literal, result); + return result; +} + +/* Does the event 'id' parse? Determine via ctx->ids if possible. */ +double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id) +{ + struct evlist *tmp; + double ret; + + if (hashmap__find(ctx->ids, id, /*value=*/NULL)) + return 1.0; + + if (!compute_ids) + return 0.0; + + tmp = evlist__new(); + if (!tmp) + return NAN; + + if (strchr(id, '@')) { + char *tmp_id, *p; + + tmp_id = strdup(id); + if (!tmp_id) { + ret = NAN; + goto out; + } + p = strchr(tmp_id, '@'); + *p = '/'; + p = strrchr(tmp_id, '@'); + *p = '/'; + ret = parse_event(tmp, tmp_id) ? 0 : 1; + free(tmp_id); + } else { + ret = parse_event(tmp, id) ? 0 : 1; + } +out: + evlist__put(tmp); + return ret; +} + +double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx __maybe_unused, + bool compute_ids __maybe_unused, const char *test_id) +{ + double ret; + struct perf_cpu cpu = {-1}; + char *cpuid = get_cpuid_allow_env_override(cpu); + + if (!cpuid) + return NAN; + + ret = !strcmp_cpuid_str(test_id, cpuid); + + free(cpuid); + return ret; +} diff --git a/tools/perf/util/expr.h b/tools/perf/util/expr.h new file mode 100644 index 000000000..ed12e4007 --- /dev/null +++ b/tools/perf/util/expr.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PARSE_CTX_H +#define PARSE_CTX_H 1 + +struct hashmap; +struct metric_ref; + +struct expr_scanner_ctx { + char *user_requested_cpu_list; + int runtime; + bool system_wide; + bool is_test; +}; + +struct expr_parse_ctx { + struct hashmap *ids; + struct expr_scanner_ctx sctx; +}; + +struct expr_id_data; + +struct hashmap *ids__new(void); +void ids__free(struct hashmap *ids); +int ids__insert(struct hashmap *ids, const char *id); +/* + * Union two sets of ids (hashmaps) and construct a third, freeing ids1 and + * ids2. + */ +struct hashmap *ids__union(struct hashmap *ids1, struct hashmap *ids2); + +struct expr_parse_ctx *expr__ctx_new(void); +void expr__ctx_clear(struct expr_parse_ctx *ctx); +void expr__ctx_free(struct expr_parse_ctx *ctx); + +void expr__del_id(struct expr_parse_ctx *ctx, const char *id); +int expr__add_id(struct expr_parse_ctx *ctx, const char *id); +int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val); +int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count); +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr); +int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref); +int expr__get_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **data); +bool expr__subset_of_ids(struct expr_parse_ctx *haystack, + struct expr_parse_ctx *needles); +int expr__resolve_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **datap); + +int expr__parse(double *final_val, struct expr_parse_ctx *ctx, + const char *expr); + +int expr__find_ids(const char *expr, const char *one, + struct expr_parse_ctx *ids); + +double expr_id_data__value(const struct expr_id_data *data); +double expr_id_data__source_count(const struct expr_id_data *data); +double expr_id_data__aggr_nr(const struct expr_id_data *data); + +double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx); +double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); +double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); + +#endif diff --git a/tools/perf/util/expr.l b/tools/perf/util/expr.l new file mode 100644 index 000000000..f16ccff27 --- /dev/null +++ b/tools/perf/util/expr.l @@ -0,0 +1,150 @@ +%option prefix="expr_" +%option reentrant +%option bison-bridge + +%{ +#include +#include "expr.h" +#include "expr-bison.h" +#include + +char *expr_get_text(yyscan_t yyscanner); +YYSTYPE *expr_get_lval(yyscan_t yyscanner); + +static double __value(YYSTYPE *yylval, char *str, int token) +{ + double num; + + errno = 0; + num = strtod(str, NULL); + if (errno) + return EXPR_ERROR; + + yylval->num = num; + return token; +} + +static int value(yyscan_t scanner) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + char *text = expr_get_text(scanner); + + return __value(yylval, text, NUMBER); +} + +/* + * Allow @ instead of / to be able to specify pmu/event/ without + * conflicts with normal division. + */ +static char *normalize(char *str, int runtime) +{ + char *ret = str; + char *dst = str; + + while (*str) { + if (*str == '\\') { + *dst++ = *++str; + if (!*str) + break; + } + else if (*str == '?') { + char *paramval; + int i = 0; + int size = asprintf(¶mval, "%d", runtime); + + if (size < 0) + *dst++ = '0'; + else { + while (i < size) + *dst++ = paramval[i++]; + free(paramval); + } + } + else + *dst++ = *str; + str++; + } + + *dst = 0x0; + return ret; +} + +static int str(yyscan_t scanner, int token, int runtime) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + char *text = expr_get_text(scanner); + + yylval->str = normalize(strdup(text), runtime); + if (!yylval->str) + return EXPR_ERROR; + + yylval->str = normalize(yylval->str, runtime); + return token; +} + +static int literal(yyscan_t scanner, const struct expr_scanner_ctx *sctx) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + + yylval->num = expr__get_literal(expr_get_text(scanner), sctx); + if (isnan(yylval->num)) { + if (!sctx->is_test) + return EXPR_ERROR; + yylval->num = 1; + } + return LITERAL; +} + +static int nan_value(yyscan_t scanner) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + + yylval->num = NAN; + return NUMBER; +} +%} + +number ([0-9]+\.?[0-9]*|[0-9]*\.?[0-9]+)(e-?[0-9]+)? + +sch [-,=] +spec \\{sch} +sym [0-9a-zA-Z_\.:@?]+ +symbol ({spec}|{sym})+ +literal #[0-9a-zA-Z_\.\-]+ + +%% + struct expr_scanner_ctx *sctx = expr_get_extra(yyscanner); + +d_ratio { return D_RATIO; } +max { return MAX; } +min { return MIN; } +if { return IF; } +else { return ELSE; } +source_count { return SOURCE_COUNT; } +aggr_nr { return AGGR_NR; } +has_event { return HAS_EVENT; } +strcmp_cpuid_str { return STRCMP_CPUID_STR; } +NaN { return nan_value(yyscanner); } +{literal} { return literal(yyscanner, sctx); } +{number} { return value(yyscanner); } +{symbol} { return str(yyscanner, ID, sctx->runtime); } +"|" { return '|'; } +"^" { return '^'; } +"&" { return '&'; } +"<" { return '<'; } +">" { return '>'; } +"-" { return '-'; } +"+" { return '+'; } +"*" { return '*'; } +"/" { return '/'; } +"%" { return '%'; } +"(" { return '('; } +")" { return ')'; } +"," { return ','; } +. { } +%% + +int expr_wrap(void *scanner __maybe_unused) +{ + return 1; +} diff --git a/tools/perf/util/expr.y b/tools/perf/util/expr.y new file mode 100644 index 000000000..e20f64935 --- /dev/null +++ b/tools/perf/util/expr.y @@ -0,0 +1,391 @@ +/* Simple expression parser */ +%{ +#ifndef NDEBUG +#define YYDEBUG 1 +#endif +#include +#include +#include +#include "util/debug.h" +#define IN_EXPR_Y 1 +#include "expr.h" +#include "expr-bison.h" +int expr_lex(YYSTYPE * yylval_param , void *yyscanner); +%} + +%define api.pure full + +%parse-param { double *final_val } +%parse-param { struct expr_parse_ctx *ctx } +%parse-param { bool compute_ids } +%parse-param {void *scanner} +%lex-param {void* scanner} + +%union { + double num; + char *str; + struct ids { + /* + * When creating ids, holds the working set of event ids. NULL + * implies the set is empty. + */ + struct hashmap *ids; + /* + * The metric value. When not creating ids this is the value + * read from a counter, a constant or some computed value. When + * creating ids the value is either a constant or BOTTOM. NAN is + * used as the special BOTTOM value, representing a "set of all + * values" case. + */ + double val; + } ids; +} + +%token ID NUMBER MIN MAX IF ELSE LITERAL D_RATIO SOURCE_COUNT AGGR_NR HAS_EVENT STRCMP_CPUID_STR EXPR_ERROR +%left MIN MAX IF +%left '|' +%left '^' +%left '&' +%left '<' '>' +%left '-' '+' +%left '*' '/' '%' +%left NEG NOT +%type NUMBER LITERAL +%type ID +%destructor { free ($$); } +%type expr if_expr +%destructor { ids__free($$.ids); } + +%{ +static void expr_error(double *final_val __maybe_unused, + struct expr_parse_ctx *ctx __maybe_unused, + bool compute_ids __maybe_unused, + void *scanner __maybe_unused, + const char *s) +{ + pr_debug("%s\n", s); +} + +/* + * During compute ids, the special "bottom" value uses NAN to represent the set + * of all values. NAN is selected as it isn't a useful constant value. + */ +#define BOTTOM NAN + +/* During computing ids, does val represent a constant (non-BOTTOM) value? */ +static bool is_const(double val) +{ + return isfinite(val); +} + +static struct ids union_expr(struct ids ids1, struct ids ids2) +{ + struct ids result = { + .val = BOTTOM, + .ids = ids__union(ids1.ids, ids2.ids), + }; + return result; +} + +enum expr_id_kind { + EXPR_ID_KIND__VALUE, + EXPR_ID_KIND__SOURCE_COUNT, + EXPR_ID_KIND__AGGR_NR, +}; + +static struct ids handle_id(struct expr_parse_ctx *ctx, char *id, + bool compute_ids, enum expr_id_kind kind) +{ + struct ids result; + + if (!compute_ids) { + /* + * Compute the event's value from ID. If the ID isn't known then + * it isn't used to compute the formula so set to NAN. + */ + struct expr_id_data *data; + + result.val = NAN; + if (expr__resolve_id(ctx, id, &data) == 0) { + if (kind == EXPR_ID_KIND__SOURCE_COUNT) + result.val = expr_id_data__source_count(data); + else if (kind == EXPR_ID_KIND__AGGR_NR) + result.val = expr_id_data__aggr_nr(data); + else + result.val = expr_id_data__value(data); + } + result.ids = NULL; + free(id); + } else { + /* + * Set the value to BOTTOM to show that any value is possible + * when the event is computed. Create a set of just the ID. + */ + result.val = BOTTOM; + result.ids = ids__new(); + if (!result.ids || ids__insert(result.ids, id)) { + pr_err("Error creating IDs for '%s'", id); + free(id); + } + } + return result; +} + +/* + * If we're not computing ids or $1 and $3 are constants, compute the new + * constant value using OP. Its invariant that there are no ids. If computing + * ids for non-constants union the set of IDs that must be computed. + */ +#define BINARY_OP(RESULT, OP, LHS, RHS) \ + if (!compute_ids || (is_const(LHS.val) && is_const(RHS.val))) { \ + assert(LHS.ids == NULL); \ + assert(RHS.ids == NULL); \ + if (isnan(LHS.val) || isnan(RHS.val)) { \ + RESULT.val = NAN; \ + } else { \ + RESULT.val = LHS.val OP RHS.val; \ + } \ + RESULT.ids = NULL; \ + } else { \ + RESULT = union_expr(LHS, RHS); \ + } + +%} +%% + +start: if_expr +{ + if (compute_ids) + ctx->ids = ids__union($1.ids, ctx->ids); + + if (final_val) + *final_val = $1.val; +} +; + +if_expr: expr IF expr ELSE if_expr +{ + if (fpclassify($3.val) == FP_ZERO) { + /* + * The IF expression evaluated to 0 so treat as false, take the + * ELSE and discard everything else. + */ + $$.val = $5.val; + $$.ids = $5.ids; + ids__free($1.ids); + ids__free($3.ids); + } else if (!compute_ids || is_const($3.val)) { + /* + * If ids aren't computed then treat the expression as true. If + * ids are being computed and the IF expr is a non-zero + * constant, then also evaluate the true case. + */ + $$.val = $1.val; + $$.ids = $1.ids; + ids__free($3.ids); + ids__free($5.ids); + } else if ($1.val == $5.val) { + /* + * LHS == RHS, so both are an identical constant. No need to + * evaluate any events. + */ + $$.val = $1.val; + $$.ids = NULL; + ids__free($1.ids); + ids__free($3.ids); + ids__free($5.ids); + } else { + /* + * Value is either the LHS or RHS and we need the IF expression + * to compute it. + */ + $$ = union_expr($1, union_expr($3, $5)); + } +} +| expr +; + +expr: NUMBER +{ + $$.val = $1; + $$.ids = NULL; +} +| ID { $$ = handle_id(ctx, $1, compute_ids, EXPR_ID_KIND__VALUE); } +| SOURCE_COUNT '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__SOURCE_COUNT); } +| AGGR_NR '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__AGGR_NR); } +| HAS_EVENT '(' ID ')' +{ + $$.val = expr__has_event(ctx, compute_ids, $3); + $$.ids = NULL; + free($3); +} +| STRCMP_CPUID_STR '(' ID ')' +{ + $$.val = expr__strcmp_cpuid_str(ctx, compute_ids, $3); + $$.ids = NULL; + free($3); +} +| expr '|' expr +{ + if (is_const($1.val) && is_const($3.val)) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.ids = NULL; + $$.val = (fpclassify($1.val) == FP_ZERO && fpclassify($3.val) == FP_ZERO) ? 0 : 1; + } else if (is_const($1.val)) { + assert($1.ids == NULL); + if (fpclassify($1.val) == FP_ZERO) { + $$ = $3; + } else { + $$.val = 1; + $$.ids = NULL; + ids__free($3.ids); + } + } else if (is_const($3.val)) { + assert($3.ids == NULL); + if (fpclassify($3.val) == FP_ZERO) { + $$ = $1; + } else { + $$.val = 1; + $$.ids = NULL; + ids__free($1.ids); + } + } else { + $$ = union_expr($1, $3); + } +} +| expr '&' expr +{ + if (is_const($1.val) && is_const($3.val)) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = (fpclassify($1.val) != FP_ZERO && fpclassify($3.val) != FP_ZERO) ? 1 : 0; + $$.ids = NULL; + } else if (is_const($1.val)) { + assert($1.ids == NULL); + if (fpclassify($1.val) != FP_ZERO) { + $$ = $3; + } else { + $$.val = 0; + $$.ids = NULL; + ids__free($3.ids); + } + } else if (is_const($3.val)) { + assert($3.ids == NULL); + if (fpclassify($3.val) != FP_ZERO) { + $$ = $1; + } else { + $$.val = 0; + $$.ids = NULL; + ids__free($1.ids); + } + } else { + $$ = union_expr($1, $3); + } +} +| expr '^' expr +{ + if (is_const($1.val) && is_const($3.val)) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = (fpclassify($1.val) == FP_ZERO) != (fpclassify($3.val) == FP_ZERO) ? 1 : 0; + $$.ids = NULL; + } else { + $$ = union_expr($1, $3); + } +} +| expr '<' expr { BINARY_OP($$, <, $1, $3); } +| expr '>' expr { BINARY_OP($$, >, $1, $3); } +| expr '+' expr { BINARY_OP($$, +, $1, $3); } +| expr '-' expr { BINARY_OP($$, -, $1, $3); } +| expr '*' expr { BINARY_OP($$, *, $1, $3); } +| expr '/' expr +{ + if (fpclassify($3.val) == FP_ZERO) { + pr_debug("division by zero\n"); + assert($3.ids == NULL); + if (compute_ids) + ids__free($1.ids); + $$.val = NAN; + $$.ids = NULL; + } else if (!compute_ids || (is_const($1.val) && is_const($3.val))) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = $1.val / $3.val; + $$.ids = NULL; + } else { + /* LHS and/or RHS need computing from event IDs so union. */ + $$ = union_expr($1, $3); + } +} +| expr '%' expr +{ + if (fpclassify($3.val) == FP_ZERO) { + pr_debug("division by zero\n"); + YYABORT; + } else if (!compute_ids || (is_const($1.val) && is_const($3.val))) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = (long)$1.val % (long)$3.val; + $$.ids = NULL; + } else { + /* LHS and/or RHS need computing from event IDs so union. */ + $$ = union_expr($1, $3); + } +} +| D_RATIO '(' expr ',' expr ')' +{ + if (fpclassify($5.val) == FP_ZERO) { + /* + * Division by constant zero always yields zero and no events + * are necessary. + */ + assert($5.ids == NULL); + $$.val = 0.0; + $$.ids = NULL; + ids__free($3.ids); + } else if (!compute_ids || (is_const($3.val) && is_const($5.val))) { + assert($3.ids == NULL); + assert($5.ids == NULL); + $$.val = $3.val / $5.val; + $$.ids = NULL; + } else { + /* LHS and/or RHS need computing from event IDs so union. */ + $$ = union_expr($3, $5); + } +} +| '-' expr %prec NEG +{ + $$.val = -$2.val; + $$.ids = $2.ids; +} +| '(' if_expr ')' +{ + $$ = $2; +} +| MIN '(' expr ',' expr ')' +{ + if (!compute_ids) { + $$.val = $3.val < $5.val ? $3.val : $5.val; + $$.ids = NULL; + } else { + $$ = union_expr($3, $5); + } +} +| MAX '(' expr ',' expr ')' +{ + if (!compute_ids) { + $$.val = $3.val > $5.val ? $3.val : $5.val; + $$.ids = NULL; + } else { + $$ = union_expr($3, $5); + } +} +| LITERAL +{ + $$.val = $1; + $$.ids = NULL; +} +; + +%% diff --git a/tools/perf/util/find-map.c b/tools/perf/util/find-map.c new file mode 100644 index 000000000..bba511795 --- /dev/null +++ b/tools/perf/util/find-map.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include + +static int find_map(void **start, void **end, const char *name) +{ + FILE *maps; + char *line = NULL; + size_t len = 0; + int found = 0; + + maps = fopen("/proc/self/maps", "r"); + if (!maps) { + fprintf(stderr, "cannot open maps\n"); + return -1; + } + + while (!found && getline(&line, &len, maps) != -1) { + int m = -1; + + /* We care only about private r-x mappings. */ + if (2 != sscanf(line, "%p-%p r-xp %*x %*x:%*x %*u %n", + start, end, &m)) + continue; + if (m < 0) + continue; + + if (!strncmp(&line[m], name, strlen(name))) + found = 1; + } + + free(line); + fclose(maps); + return !found; +} diff --git a/tools/perf/util/fncache.c b/tools/perf/util/fncache.c new file mode 100644 index 000000000..bf9559c55 --- /dev/null +++ b/tools/perf/util/fncache.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Manage a cache of file names' existence */ +#include +#include +#include +#include +#include +#include "fncache.h" +#include "hashmap.h" + +static struct hashmap *fncache; + +static size_t fncache__hash(long key, void *ctx __maybe_unused) +{ + return str_hash((const char *)key); +} + +static bool fncache__equal(long key1, long key2, void *ctx __maybe_unused) +{ + return strcmp((const char *)key1, (const char *)key2) == 0; +} + +static void fncache__init(void) +{ + fncache = hashmap__new(fncache__hash, fncache__equal, /*ctx=*/NULL); +} + +static struct hashmap *fncache__get(void) +{ + static pthread_once_t fncache_once = PTHREAD_ONCE_INIT; + + pthread_once(&fncache_once, fncache__init); + + return fncache; +} + +static bool lookup_fncache(const char *name, bool *res) +{ + long val; + + if (!hashmap__find(fncache__get(), name, &val)) + return false; + + *res = (val != 0); + return true; +} + +static void update_fncache(const char *name, bool res) +{ + char *old_key = NULL, *key = strdup(name); + + if (key) { + hashmap__set(fncache__get(), key, res, &old_key, /*old_value*/NULL); + free(old_key); + } +} + +/* No LRU, only use when bounded in some other way. */ +bool file_available(const char *name) +{ + bool res; + + if (lookup_fncache(name, &res)) + return res; + res = access(name, R_OK) == 0; + update_fncache(name, res); + return res; +} diff --git a/tools/perf/util/fncache.h b/tools/perf/util/fncache.h new file mode 100644 index 000000000..b6a0f2094 --- /dev/null +++ b/tools/perf/util/fncache.h @@ -0,0 +1,6 @@ +#ifndef _FCACHE_H +#define _FCACHE_H 1 + +bool file_available(const char *name); + +#endif diff --git a/tools/perf/util/ftrace.h b/tools/perf/util/ftrace.h new file mode 100644 index 000000000..950f2efaf --- /dev/null +++ b/tools/perf/util/ftrace.h @@ -0,0 +1,96 @@ +#ifndef __PERF_FTRACE_H__ +#define __PERF_FTRACE_H__ + +#include + +#include "target.h" + +struct evlist; +struct hashamp; +struct stats; + +struct perf_ftrace { + struct evlist *evlist; + struct target target; + const char *tracer; + struct list_head filters; + struct list_head notrace; + struct list_head graph_funcs; + struct list_head nograph_funcs; + struct list_head event_pair; + struct hashmap *profile_hash; + unsigned long percpu_buffer_size; + bool inherit; + bool use_nsec; + unsigned int bucket_range; + unsigned int min_latency; + unsigned int max_latency; + unsigned int bucket_num; + bool hide_empty; + int graph_depth; + int func_stack_trace; + int func_irq_info; + int graph_args; + int graph_retval; + int graph_retval_hex; + int graph_retaddr; + int graph_nosleep_time; + int graph_noirqs; + int graph_verbose; + int graph_thresh; + int graph_tail; +}; + +struct filter_entry { + struct list_head list; + char name[]; +}; + +#define NUM_BUCKET 22 /* 20 + 2 (for outliers in both direction) */ + +#ifdef HAVE_BPF_SKEL + +int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace); +int perf_ftrace__latency_start_bpf(struct perf_ftrace *ftrace); +int perf_ftrace__latency_stop_bpf(struct perf_ftrace *ftrace); +int perf_ftrace__latency_read_bpf(struct perf_ftrace *ftrace, + int buckets[], struct stats *stats); +int perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace); + +#else /* !HAVE_BPF_SKEL */ + +static inline int +perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_start_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_stop_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_read_bpf(struct perf_ftrace *ftrace __maybe_unused, + int buckets[] __maybe_unused, + struct stats *stats __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +#endif /* HAVE_BPF_SKEL */ + +#endif /* __PERF_FTRACE_H__ */ diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c new file mode 100644 index 000000000..14882def9 --- /dev/null +++ b/tools/perf/util/genelf.c @@ -0,0 +1,471 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * genelf.c + * Copyright (C) 2014, Google, Inc + * + * Contributed by: + * Stephane Eranian + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBDW_SUPPORT +#include +#endif + +#include "blake2s.h" +#include "genelf.h" +#include "../util/jitdump.h" +#include + +#ifndef NT_GNU_BUILD_ID +#define NT_GNU_BUILD_ID 3 +#endif + +typedef struct { + unsigned int namesz; /* Size of entry's owner string */ + unsigned int descsz; /* Size of the note descriptor */ + unsigned int type; /* Interpretation of the descriptor */ + char name[0]; /* Start of the name+desc data */ +} Elf_Note; + +static char shd_string_table[] = { + 0, + '.', 't', 'e', 'x', 't', 0, /* 1 */ + '.', 's', 'h', 's', 't', 'r', 't', 'a', 'b', 0, /* 7 */ + '.', 's', 'y', 'm', 't', 'a', 'b', 0, /* 17 */ + '.', 's', 't', 'r', 't', 'a', 'b', 0, /* 25 */ + '.', 'n', 'o', 't', 'e', '.', 'g', 'n', 'u', '.', 'b', 'u', 'i', 'l', 'd', '-', 'i', 'd', 0, /* 33 */ + '.', 'd', 'e', 'b', 'u', 'g', '_', 'l', 'i', 'n', 'e', 0, /* 52 */ + '.', 'd', 'e', 'b', 'u', 'g', '_', 'i', 'n', 'f', 'o', 0, /* 64 */ + '.', 'd', 'e', 'b', 'u', 'g', '_', 'a', 'b', 'b', 'r', 'e', 'v', 0, /* 76 */ + '.', 'e', 'h', '_', 'f', 'r', 'a', 'm', 'e', '_', 'h', 'd', 'r', 0, /* 90 */ + '.', 'e', 'h', '_', 'f', 'r', 'a', 'm', 'e', 0, /* 104 */ +}; + +static struct buildid_note { + Elf_Note desc; /* descsz: size of build-id, must be multiple of 4 */ + char name[4]; /* GNU\0 */ + u8 build_id[20]; +} bnote; + +static Elf_Sym symtab[]={ + /* symbol 0 MUST be the undefined symbol */ + { .st_name = 0, /* index in sym_string table */ + .st_info = ELF_ST_TYPE(STT_NOTYPE), + .st_shndx = 0, /* for now */ + .st_value = 0x0, + .st_other = ELF_ST_VIS(STV_DEFAULT), + .st_size = 0, + }, + { .st_name = 1, /* index in sym_string table */ + .st_info = ELF_ST_BIND(STB_LOCAL) | ELF_ST_TYPE(STT_FUNC), + .st_shndx = 1, + .st_value = 0, /* for now */ + .st_other = ELF_ST_VIS(STV_DEFAULT), + .st_size = 0, /* for now */ + } +}; + +static int +jit_add_eh_frame_info(Elf *e, void* unwinding, uint64_t unwinding_header_size, + uint64_t unwinding_size, uint64_t base_offset) +{ + Elf_Data *d; + Elf_Scn *scn; + Elf_Shdr *shdr; + uint64_t unwinding_table_size = unwinding_size - unwinding_header_size; + + /* + * setup eh_frame section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + return -1; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + return -1; + } + + d->d_align = 8; + d->d_off = 0LL; + d->d_buf = unwinding; + d->d_type = ELF_T_BYTE; + d->d_size = unwinding_table_size; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + return -1; + } + + shdr->sh_name = 104; + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = base_offset; + shdr->sh_flags = SHF_ALLOC; + shdr->sh_entsize = 0; + + /* + * setup eh_frame_hdr section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + return -1; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + return -1; + } + + d->d_align = 4; + d->d_off = 0LL; + d->d_buf = unwinding + unwinding_table_size; + d->d_type = ELF_T_BYTE; + d->d_size = unwinding_header_size; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + return -1; + } + + shdr->sh_name = 90; + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = base_offset + unwinding_table_size; + shdr->sh_flags = SHF_ALLOC; + shdr->sh_entsize = 0; + + return 0; +} + +enum { + TAG_CODE = 0, + TAG_SYMTAB = 1, + TAG_STRSYM = 2, +}; + +/* + * Update the hash using the given data, also prepending a (tag, len) prefix to + * ensure that distinct input tuples reliably result in distinct hashes. + */ +static void blake2s_update_tagged(struct blake2s_ctx *ctx, int tag, + const void *data, size_t len) +{ + u64 prefix = ((u64)tag << 56) | len; + + blake2s_update(ctx, (const u8 *)&prefix, sizeof(prefix)); + blake2s_update(ctx, data, len); +} + +/* + * fd: file descriptor open for writing for the output file + * load_addr: code load address (could be zero) + * sym: function name (for native code - used as the symbol) + * code: the native code + * csize: the code size in bytes + */ +int +jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, + const void *code, int csize, + void *debug __maybe_unused, int nr_debug_entries __maybe_unused, + void *unwinding, uint64_t unwinding_header_size, uint64_t unwinding_size) +{ + Elf *e; + Elf_Data *d; + Elf_Scn *scn; + Elf_Ehdr *ehdr; + Elf_Phdr *phdr; + Elf_Shdr *shdr; + uint64_t eh_frame_base_offset; + char *strsym = NULL; + struct blake2s_ctx ctx; + int symlen; + int retval = -1; + + if (elf_version(EV_CURRENT) == EV_NONE) { + warnx("ELF initialization failed"); + return -1; + } + + e = elf_begin(fd, ELF_C_WRITE, NULL); + if (!e) { + warnx("elf_begin failed"); + goto error; + } + + /* + * setup ELF header + */ + ehdr = elf_newehdr(e); + if (!ehdr) { + warnx("cannot get ehdr"); + goto error; + } + + ehdr->e_ident[EI_DATA] = GEN_ELF_ENDIAN; + ehdr->e_ident[EI_CLASS] = GEN_ELF_CLASS; + ehdr->e_machine = GEN_ELF_ARCH; + ehdr->e_type = ET_DYN; + ehdr->e_entry = GEN_ELF_TEXT_OFFSET; + ehdr->e_version = EV_CURRENT; + ehdr->e_shstrndx= unwinding ? 4 : 2; /* shdr index for section name */ + + /* + * setup program header + */ + phdr = elf_newphdr(e, 1); + phdr[0].p_type = PT_LOAD; + phdr[0].p_offset = GEN_ELF_TEXT_OFFSET; + phdr[0].p_vaddr = GEN_ELF_TEXT_OFFSET; + phdr[0].p_paddr = GEN_ELF_TEXT_OFFSET; + phdr[0].p_filesz = csize; + phdr[0].p_memsz = csize; + phdr[0].p_flags = PF_X | PF_R; + phdr[0].p_align = 8; + + /* + * setup text section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 16; + d->d_off = 0LL; + d->d_buf = (void *)code; + d->d_type = ELF_T_BYTE; + d->d_size = csize; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 1; + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = GEN_ELF_TEXT_OFFSET; + shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC; + shdr->sh_entsize = 0; + + blake2s_init(&ctx, sizeof(bnote.build_id)); + blake2s_update_tagged(&ctx, TAG_CODE, code, csize); + + /* + * Setup .eh_frame_hdr and .eh_frame + */ + if (unwinding) { + eh_frame_base_offset = ALIGN_8(GEN_ELF_TEXT_OFFSET + csize); + retval = jit_add_eh_frame_info(e, unwinding, + unwinding_header_size, unwinding_size, + eh_frame_base_offset); + if (retval) + goto error; + retval = -1; + } + + /* + * setup section headers string table + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = shd_string_table; + d->d_type = ELF_T_BYTE; + d->d_size = sizeof(shd_string_table); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 7; /* offset of '.shstrtab' in shd_string_table */ + shdr->sh_type = SHT_STRTAB; + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * setup symtab section + */ + symtab[1].st_size = csize; + symtab[1].st_value = GEN_ELF_TEXT_OFFSET; + + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 8; + d->d_off = 0LL; + d->d_buf = symtab; + d->d_type = ELF_T_SYM; + d->d_size = sizeof(symtab); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 17; /* offset of '.symtab' in shd_string_table */ + shdr->sh_type = SHT_SYMTAB; + shdr->sh_flags = 0; + shdr->sh_entsize = sizeof(Elf_Sym); + shdr->sh_link = unwinding ? 6 : 4; /* index of .strtab section */ + + blake2s_update_tagged(&ctx, TAG_SYMTAB, symtab, sizeof(symtab)); + + /* + * setup symbols string table + * 2 = 1 for 0 in 1st entry, 1 for the 0 at end of symbol for 2nd entry + */ + symlen = 2 + strlen(sym); + strsym = calloc(1, symlen); + if (!strsym) { + warnx("cannot allocate strsym"); + goto error; + } + strcpy(strsym + 1, sym); + + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = strsym; + d->d_type = ELF_T_BYTE; + d->d_size = symlen; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 25; /* offset in shd_string_table */ + shdr->sh_type = SHT_STRTAB; + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + blake2s_update_tagged(&ctx, TAG_STRSYM, strsym, symlen); + + /* + * setup build-id section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + /* + * build-id generation + */ + blake2s_final(&ctx, bnote.build_id); + bnote.desc.namesz = sizeof(bnote.name); /* must include 0 termination */ + bnote.desc.descsz = sizeof(bnote.build_id); + bnote.desc.type = NT_GNU_BUILD_ID; + strcpy(bnote.name, "GNU"); + + d->d_align = 4; + d->d_off = 0LL; + d->d_buf = &bnote; + d->d_type = ELF_T_BYTE; + d->d_size = sizeof(bnote); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 33; /* offset in shd_string_table */ + shdr->sh_type = SHT_NOTE; + shdr->sh_addr = 0x0; + shdr->sh_flags = SHF_ALLOC; + shdr->sh_size = sizeof(bnote); + shdr->sh_entsize = 0; + +#ifdef HAVE_LIBDW_SUPPORT + if (debug && nr_debug_entries) { + retval = jit_add_debug_info(e, load_addr, debug, nr_debug_entries); + if (retval) + goto error; + } else +#endif + { + if (elf_update(e, ELF_C_WRITE) < 0) { + warnx("elf_update 4 failed"); + goto error; + } + } + + retval = 0; +error: + (void)elf_end(e); + + free(strsym); + + return retval; +} diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h new file mode 100644 index 000000000..9f0b875d6 --- /dev/null +++ b/tools/perf/util/genelf.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __GENELF_H__ +#define __GENELF_H__ + +#include + +/* genelf.c */ +int jit_write_elf(int fd, uint64_t code_addr, const char *sym, + const void *code, int csize, void *debug, int nr_debug_entries, + void *unwinding, uint64_t unwinding_header_size, uint64_t unwinding_size); +#ifdef HAVE_LIBDW_SUPPORT +/* genelf_debug.c */ +int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_entries); +#endif + +#if defined(__arm__) +#define GEN_ELF_ARCH EM_ARM +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__aarch64__) +#define GEN_ELF_ARCH EM_AARCH64 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__x86_64__) +#define GEN_ELF_ARCH EM_X86_64 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__i386__) +#define GEN_ELF_ARCH EM_386 +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__powerpc64__) +#define GEN_ELF_ARCH EM_PPC64 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__powerpc__) +#define GEN_ELF_ARCH EM_PPC +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__sparc__) && defined(__arch64__) +#define GEN_ELF_ARCH EM_SPARCV9 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__sparc__) +#define GEN_ELF_ARCH EM_SPARC +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__s390x__) +#define GEN_ELF_ARCH EM_S390 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__riscv) && __riscv_xlen == 64 +#define GEN_ELF_ARCH EM_RISCV +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__riscv) && __riscv_xlen == 32 +#define GEN_ELF_ARCH EM_RISCV +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__loongarch__) +#define GEN_ELF_ARCH EM_LOONGARCH +#define GEN_ELF_CLASS ELFCLASS64 +#else +#error "unsupported architecture" +#endif + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define GEN_ELF_ENDIAN ELFDATA2MSB +#else +#define GEN_ELF_ENDIAN ELFDATA2LSB +#endif + +#if GEN_ELF_CLASS == ELFCLASS64 +#define elf_newehdr elf64_newehdr +#define elf_newphdr elf64_newphdr +#define elf_getshdr elf64_getshdr +#define Elf_Ehdr Elf64_Ehdr +#define Elf_Phdr Elf64_Phdr +#define Elf_Shdr Elf64_Shdr +#define Elf_Sym Elf64_Sym +#define ELF_ST_TYPE(a) ELF64_ST_TYPE(a) +#define ELF_ST_BIND(a) ELF64_ST_BIND(a) +#define ELF_ST_VIS(a) ELF64_ST_VISIBILITY(a) +#else +#define elf_newehdr elf32_newehdr +#define elf_newphdr elf32_newphdr +#define elf_getshdr elf32_getshdr +#define Elf_Ehdr Elf32_Ehdr +#define Elf_Phdr Elf32_Phdr +#define Elf_Shdr Elf32_Shdr +#define Elf_Sym Elf32_Sym +#define ELF_ST_TYPE(a) ELF32_ST_TYPE(a) +#define ELF_ST_BIND(a) ELF32_ST_BIND(a) +#define ELF_ST_VIS(a) ELF32_ST_VISIBILITY(a) +#endif + +/* The .text section is directly after the ELF header */ +#define GEN_ELF_TEXT_OFFSET round_up(sizeof(Elf_Ehdr) + sizeof(Elf_Phdr), 16) + +#endif diff --git a/tools/perf/util/genelf_debug.c b/tools/perf/util/genelf_debug.c new file mode 100644 index 000000000..8588b3e35 --- /dev/null +++ b/tools/perf/util/genelf_debug.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * genelf_debug.c + * Copyright (C) 2015, Google, Inc + * + * Contributed by: + * Stephane Eranian + * + * based on GPLv2 source code from Oprofile + * @remark Copyright 2007 OProfile authors + * @author Philippe Elie + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "genelf.h" +#include "../util/jitdump.h" + +#define BUFFER_EXT_DFL_SIZE (4 * 1024) + +typedef uint32_t uword; +typedef uint16_t uhalf; +typedef int32_t sword; +typedef int16_t shalf; +typedef uint8_t ubyte; +typedef int8_t sbyte; + +struct buffer_ext { + size_t cur_pos; + size_t max_sz; + void *data; +}; + +static void +buffer_ext_dump(struct buffer_ext *be, const char *msg) +{ + size_t i; + warnx("DUMP for %s", msg); + for (i = 0 ; i < be->cur_pos; i++) + warnx("%4zu 0x%02x", i, (((char *)be->data)[i]) & 0xff); +} + +static inline int +buffer_ext_add(struct buffer_ext *be, void *addr, size_t sz) +{ + void *tmp; + size_t be_sz = be->max_sz; + +retry: + if ((be->cur_pos + sz) < be_sz) { + memcpy(be->data + be->cur_pos, addr, sz); + be->cur_pos += sz; + return 0; + } + + if (!be_sz) + be_sz = BUFFER_EXT_DFL_SIZE; + else + be_sz <<= 1; + + tmp = realloc(be->data, be_sz); + if (!tmp) + return -1; + + be->data = tmp; + be->max_sz = be_sz; + + goto retry; +} + +static void +buffer_ext_init(struct buffer_ext *be) +{ + be->data = NULL; + be->cur_pos = 0; + be->max_sz = 0; +} + +static void +buffer_ext_exit(struct buffer_ext *be) +{ + zfree(&be->data); +} + +static inline size_t +buffer_ext_size(struct buffer_ext *be) +{ + return be->cur_pos; +} + +static inline void * +buffer_ext_addr(struct buffer_ext *be) +{ + return be->data; +} + +struct debug_line_header { + // Not counting this field + uword total_length; + // version number (2 currently) + uhalf version; + // relative offset from next field to + // program statement + uword prolog_length; + ubyte minimum_instruction_length; + ubyte default_is_stmt; + // line_base - see DWARF 2 specs + sbyte line_base; + // line_range - see DWARF 2 specs + ubyte line_range; + // number of opcode + 1 + ubyte opcode_base; + /* follow the array of opcode args nr: ubytes [nr_opcode_base] */ + /* follow the search directories index, zero terminated string + * terminated by an empty string. + */ + /* follow an array of { filename, LEB128, LEB128, LEB128 }, first is + * the directory index entry, 0 means current directory, then mtime + * and filesize, last entry is followed by en empty string. + */ + /* follow the first program statement */ +} __packed; + +/* DWARF 2 spec talk only about one possible compilation unit header while + * binutils can handle two flavours of dwarf 2, 32 and 64 bits, this is not + * related to the used arch, an ELF 32 can hold more than 4 Go of debug + * information. For now we handle only DWARF 2 32 bits comp unit. It'll only + * become a problem if we generate more than 4GB of debug information. + */ +struct compilation_unit_header { + uword total_length; + uhalf version; + uword debug_abbrev_offset; + ubyte pointer_size; +} __packed; + +#define DW_LNS_num_opcode (DW_LNS_set_isa + 1) + +/* field filled at run time are marked with -1 */ +static struct debug_line_header const default_debug_line_header = { + .total_length = -1, + .version = 2, + .prolog_length = -1, + .minimum_instruction_length = 1, /* could be better when min instruction size != 1 */ + .default_is_stmt = 1, /* we don't take care about basic block */ + .line_base = -5, /* sensible value for line base ... */ + .line_range = -14, /* ... and line range are guessed statically */ + .opcode_base = DW_LNS_num_opcode +}; + +static ubyte standard_opcode_length[] = +{ + 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 +}; +#if 0 +{ + [DW_LNS_advance_pc] = 1, + [DW_LNS_advance_line] = 1, + [DW_LNS_set_file] = 1, + [DW_LNS_set_column] = 1, + [DW_LNS_fixed_advance_pc] = 1, + [DW_LNS_set_isa] = 1, +}; +#endif + +/* field filled at run time are marked with -1 */ +static struct compilation_unit_header default_comp_unit_header = { + .total_length = -1, + .version = 2, + .debug_abbrev_offset = 0, /* we reuse the same abbrev entries for all comp unit */ + .pointer_size = sizeof(void *) +}; + +static void emit_uword(struct buffer_ext *be, uword data) +{ + buffer_ext_add(be, &data, sizeof(uword)); +} + +static void emit_string(struct buffer_ext *be, const char *s) +{ + buffer_ext_add(be, (void *)s, strlen(s) + 1); +} + +static void emit_unsigned_LEB128(struct buffer_ext *be, + unsigned long data) +{ + do { + ubyte cur = data & 0x7F; + data >>= 7; + if (data) + cur |= 0x80; + buffer_ext_add(be, &cur, 1); + } while (data); +} + +static void emit_signed_LEB128(struct buffer_ext *be, long data) +{ + int more = 1; + int negative = data < 0; + int size = sizeof(long) * CHAR_BIT; + while (more) { + ubyte cur = data & 0x7F; + data >>= 7; + if (negative) + data |= - (1 << (size - 7)); + if ((data == 0 && !(cur & 0x40)) || + (data == -1l && (cur & 0x40))) + more = 0; + else + cur |= 0x80; + buffer_ext_add(be, &cur, 1); + } +} + +static void emit_extended_opcode(struct buffer_ext *be, ubyte opcode, + void *data, size_t data_len) +{ + buffer_ext_add(be, (char *)"", 1); + + emit_unsigned_LEB128(be, data_len + 1); + + buffer_ext_add(be, &opcode, 1); + buffer_ext_add(be, data, data_len); +} + +static void emit_opcode(struct buffer_ext *be, ubyte opcode) +{ + buffer_ext_add(be, &opcode, 1); +} + +static void emit_opcode_signed(struct buffer_ext *be, + ubyte opcode, long data) +{ + buffer_ext_add(be, &opcode, 1); + emit_signed_LEB128(be, data); +} + +static void emit_opcode_unsigned(struct buffer_ext *be, ubyte opcode, + unsigned long data) +{ + buffer_ext_add(be, &opcode, 1); + emit_unsigned_LEB128(be, data); +} + +static void emit_advance_pc(struct buffer_ext *be, unsigned long delta_pc) +{ + emit_opcode_unsigned(be, DW_LNS_advance_pc, delta_pc); +} + +static void emit_advance_lineno(struct buffer_ext *be, long delta_lineno) +{ + emit_opcode_signed(be, DW_LNS_advance_line, delta_lineno); +} + +static void emit_lne_end_of_sequence(struct buffer_ext *be) +{ + emit_extended_opcode(be, DW_LNE_end_sequence, NULL, 0); +} + +static void emit_set_file(struct buffer_ext *be, unsigned long idx) +{ + emit_opcode_unsigned(be, DW_LNS_set_file, idx); +} + +static void emit_lne_define_filename(struct buffer_ext *be, + const char *filename) +{ + buffer_ext_add(be, (void *)"", 1); + + /* LNE field, strlen(filename) + zero termination, 3 bytes for: the dir entry, timestamp, filesize */ + emit_unsigned_LEB128(be, strlen(filename) + 5); + emit_opcode(be, DW_LNE_define_file); + emit_string(be, filename); + /* directory index 0=do not know */ + emit_unsigned_LEB128(be, 0); + /* last modification date on file 0=do not know */ + emit_unsigned_LEB128(be, 0); + /* filesize 0=do not know */ + emit_unsigned_LEB128(be, 0); +} + +static void emit_lne_set_address(struct buffer_ext *be, + void *address) +{ + emit_extended_opcode(be, DW_LNE_set_address, &address, sizeof(unsigned long)); +} + +static ubyte get_special_opcode(struct debug_entry *ent, + unsigned int last_line, + unsigned long last_vma) +{ + unsigned int temp; + unsigned long delta_addr; + + /* + * delta from line_base + */ + temp = (ent->lineno - last_line) - default_debug_line_header.line_base; + + if (temp >= default_debug_line_header.line_range) + return 0; + + /* + * delta of addresses + */ + delta_addr = (ent->addr - last_vma) / default_debug_line_header.minimum_instruction_length; + + /* This is not sufficient to ensure opcode will be in [0-256] but + * sufficient to ensure when summing with the delta lineno we will + * not overflow the unsigned long opcode */ + + if (delta_addr <= 256 / default_debug_line_header.line_range) { + unsigned long opcode = temp + + (delta_addr * default_debug_line_header.line_range) + + default_debug_line_header.opcode_base; + + return opcode <= 255 ? opcode : 0; + } + return 0; +} + +static void emit_lineno_info(struct buffer_ext *be, + struct debug_entry *ent, size_t nr_entry, + unsigned long code_addr) +{ + size_t i; + + /* as described in the jitdump format */ + const char repeated_name_marker[] = {'\xff', '\0'}; + + /* + * Machine state at start of a statement program + * address = 0 + * file = 1 + * line = 1 + * column = 0 + * is_stmt = default_is_stmt as given in the debug_line_header + * basic block = 0 + * end sequence = 0 + */ + + /* start state of the state machine we take care of */ + unsigned long last_vma = 0; + char const *cur_filename = NULL; + unsigned long cur_file_idx = 0; + int last_line = 1; + + emit_lne_set_address(be, (void *)code_addr); + + for (i = 0; i < nr_entry; i++, ent = debug_entry_next(ent)) { + int need_copy = 0; + ubyte special_opcode; + + /* + * check if filename changed, if so add it + */ + if ((!cur_filename || strcmp(cur_filename, ent->name)) && + strcmp(repeated_name_marker, ent->name)) { + emit_lne_define_filename(be, ent->name); + cur_filename = ent->name; + emit_set_file(be, ++cur_file_idx); + need_copy = 1; + } + + special_opcode = get_special_opcode(ent, last_line, last_vma); + if (special_opcode != 0) { + last_line = ent->lineno; + last_vma = ent->addr; + emit_opcode(be, special_opcode); + } else { + /* + * lines differ, emit line delta + */ + if (last_line != ent->lineno) { + emit_advance_lineno(be, ent->lineno - last_line); + last_line = ent->lineno; + need_copy = 1; + } + /* + * addresses differ, emit address delta + */ + if (last_vma != ent->addr) { + emit_advance_pc(be, ent->addr - last_vma); + last_vma = ent->addr; + need_copy = 1; + } + /* + * add new row to matrix + */ + if (need_copy) + emit_opcode(be, DW_LNS_copy); + } + } +} + +static void add_debug_line(struct buffer_ext *be, + struct debug_entry *ent, size_t nr_entry, + unsigned long code_addr) +{ + struct debug_line_header * dbg_header; + size_t old_size; + + old_size = buffer_ext_size(be); + + buffer_ext_add(be, (void *)&default_debug_line_header, + sizeof(default_debug_line_header)); + + buffer_ext_add(be, &standard_opcode_length, sizeof(standard_opcode_length)); + + // empty directory entry + buffer_ext_add(be, (void *)"", 1); + + // empty filename directory + buffer_ext_add(be, (void *)"", 1); + + dbg_header = buffer_ext_addr(be) + old_size; + dbg_header->prolog_length = (buffer_ext_size(be) - old_size) - + offsetof(struct debug_line_header, minimum_instruction_length); + + emit_lineno_info(be, ent, nr_entry, code_addr); + + emit_lne_end_of_sequence(be); + + dbg_header = buffer_ext_addr(be) + old_size; + dbg_header->total_length = (buffer_ext_size(be) - old_size) - + offsetof(struct debug_line_header, version); +} + +static void +add_debug_abbrev(struct buffer_ext *be) +{ + emit_unsigned_LEB128(be, 1); + emit_unsigned_LEB128(be, DW_TAG_compile_unit); + emit_unsigned_LEB128(be, DW_CHILDREN_yes); + emit_unsigned_LEB128(be, DW_AT_stmt_list); + emit_unsigned_LEB128(be, DW_FORM_data4); + emit_unsigned_LEB128(be, 0); + emit_unsigned_LEB128(be, 0); + emit_unsigned_LEB128(be, 0); +} + +static void +add_compilation_unit(struct buffer_ext *be, + size_t offset_debug_line) +{ + struct compilation_unit_header *comp_unit_header; + size_t old_size = buffer_ext_size(be); + + buffer_ext_add(be, &default_comp_unit_header, + sizeof(default_comp_unit_header)); + + emit_unsigned_LEB128(be, 1); + emit_uword(be, offset_debug_line); + + comp_unit_header = buffer_ext_addr(be) + old_size; + comp_unit_header->total_length = (buffer_ext_size(be) - old_size) - + offsetof(struct compilation_unit_header, version); +} + +static int +jit_process_debug_info(uint64_t code_addr, + void *debug, int nr_debug_entries, + struct buffer_ext *dl, + struct buffer_ext *da, + struct buffer_ext *di) +{ + struct debug_entry *ent = debug; + int i; + + for (i = 0; i < nr_debug_entries; i++) { + ent->addr = ent->addr - code_addr; + ent = debug_entry_next(ent); + } + add_compilation_unit(di, buffer_ext_size(dl)); + add_debug_line(dl, debug, nr_debug_entries, GEN_ELF_TEXT_OFFSET); + add_debug_abbrev(da); + if (0) buffer_ext_dump(da, "abbrev"); + + return 0; +} + +int +jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_entries) +{ + Elf_Data *d; + Elf_Scn *scn; + Elf_Shdr *shdr; + struct buffer_ext dl, di, da; + int ret = -1; + + buffer_ext_init(&dl); + buffer_ext_init(&di); + buffer_ext_init(&da); + + if (jit_process_debug_info(code_addr, debug, nr_debug_entries, &dl, &da, &di)) + goto out; + + /* + * setup .debug_line section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto out; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto out; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = buffer_ext_addr(&dl); + d->d_type = ELF_T_BYTE; + d->d_size = buffer_ext_size(&dl); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto out; + } + + shdr->sh_name = 52; /* .debug_line */ + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = 0; /* must be zero or == sh_offset -> dynamic object */ + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * setup .debug_info section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto out; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto out; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = buffer_ext_addr(&di); + d->d_type = ELF_T_BYTE; + d->d_size = buffer_ext_size(&di); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto out; + } + + shdr->sh_name = 64; /* .debug_info */ + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = 0; /* must be zero or == sh_offset -> dynamic object */ + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * setup .debug_abbrev section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto out; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto out; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = buffer_ext_addr(&da); + d->d_type = ELF_T_BYTE; + d->d_size = buffer_ext_size(&da); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto out; + } + + shdr->sh_name = 76; /* .debug_info */ + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = 0; /* must be zero or == sh_offset -> dynamic object */ + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * now we update the ELF image with all the sections + */ + if (elf_update(e, ELF_C_WRITE) < 0) + warnx("elf_update debug failed"); + else + ret = 0; + +out: + buffer_ext_exit(&dl); + buffer_ext_exit(&di); + buffer_ext_exit(&da); + return ret; +} diff --git a/tools/perf/util/hashmap.c b/tools/perf/util/hashmap.c new file mode 100644 index 000000000..140ee4055 --- /dev/null +++ b/tools/perf/util/hashmap.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +/* + * Generic non-thread safe hash map implementation. + * + * Copyright (c) 2019 Facebook + */ +#include +#include +#include +#include +#include +#include "hashmap.h" + +/* make sure libbpf doesn't use kernel-only integer typedefs */ +#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 + +/* prevent accidental re-addition of reallocarray() */ +#pragma GCC poison reallocarray + +/* start with 4 buckets */ +#define HASHMAP_MIN_CAP_BITS 2 + +static void hashmap_add_entry(struct hashmap_entry **pprev, + struct hashmap_entry *entry) +{ + entry->next = *pprev; + *pprev = entry; +} + +static void hashmap_del_entry(struct hashmap_entry **pprev, + struct hashmap_entry *entry) +{ + *pprev = entry->next; + entry->next = NULL; +} + +void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, void *ctx) +{ + map->hash_fn = hash_fn; + map->equal_fn = equal_fn; + map->ctx = ctx; + + map->buckets = NULL; + map->cap = 0; + map->cap_bits = 0; + map->sz = 0; +} + +struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, + void *ctx) +{ + struct hashmap *map = malloc(sizeof(struct hashmap)); + + if (!map) + return ERR_PTR(-ENOMEM); + hashmap__init(map, hash_fn, equal_fn, ctx); + return map; +} + +void hashmap__clear(struct hashmap *map) +{ + struct hashmap_entry *cur, *tmp; + size_t bkt; + + hashmap__for_each_entry_safe(map, cur, tmp, bkt) { + free(cur); + } + free(map->buckets); + map->buckets = NULL; + map->cap = map->cap_bits = map->sz = 0; +} + +void hashmap__free(struct hashmap *map) +{ + if (IS_ERR_OR_NULL(map)) + return; + + hashmap__clear(map); + free(map); +} + +size_t hashmap__size(const struct hashmap *map) +{ + return map->sz; +} + +size_t hashmap__capacity(const struct hashmap *map) +{ + return map->cap; +} + +static bool hashmap_needs_to_grow(struct hashmap *map) +{ + /* grow if empty or more than 75% filled */ + return (map->cap == 0) || ((map->sz + 1) * 4 / 3 > map->cap); +} + +static int hashmap_grow(struct hashmap *map) +{ + struct hashmap_entry **new_buckets; + struct hashmap_entry *cur, *tmp; + size_t new_cap_bits, new_cap; + size_t h, bkt; + + new_cap_bits = map->cap_bits + 1; + if (new_cap_bits < HASHMAP_MIN_CAP_BITS) + new_cap_bits = HASHMAP_MIN_CAP_BITS; + + new_cap = 1UL << new_cap_bits; + new_buckets = calloc(new_cap, sizeof(new_buckets[0])); + if (!new_buckets) + return -ENOMEM; + + hashmap__for_each_entry_safe(map, cur, tmp, bkt) { + h = hash_bits(map->hash_fn(cur->key, map->ctx), new_cap_bits); + hashmap_add_entry(&new_buckets[h], cur); + } + + map->cap = new_cap; + map->cap_bits = new_cap_bits; + free(map->buckets); + map->buckets = new_buckets; + + return 0; +} + +static bool hashmap_find_entry(const struct hashmap *map, + const long key, size_t hash, + struct hashmap_entry ***pprev, + struct hashmap_entry **entry) +{ + struct hashmap_entry *cur, **prev_ptr; + + if (!map->buckets) + return false; + + for (prev_ptr = &map->buckets[hash], cur = *prev_ptr; + cur; + prev_ptr = &cur->next, cur = cur->next) { + if (map->equal_fn(cur->key, key, map->ctx)) { + if (pprev) + *pprev = prev_ptr; + *entry = cur; + return true; + } + } + + return false; +} + +int hashmap_insert(struct hashmap *map, long key, long value, + enum hashmap_insert_strategy strategy, + long *old_key, long *old_value) +{ + struct hashmap_entry *entry; + size_t h; + int err; + + if (old_key) + *old_key = 0; + if (old_value) + *old_value = 0; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (strategy != HASHMAP_APPEND && + hashmap_find_entry(map, key, h, NULL, &entry)) { + if (old_key) + *old_key = entry->key; + if (old_value) + *old_value = entry->value; + + if (strategy == HASHMAP_SET || strategy == HASHMAP_UPDATE) { + entry->key = key; + entry->value = value; + return 0; + } else if (strategy == HASHMAP_ADD) { + return -EEXIST; + } + } + + if (strategy == HASHMAP_UPDATE) + return -ENOENT; + + if (hashmap_needs_to_grow(map)) { + err = hashmap_grow(map); + if (err) + return err; + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + } + + entry = malloc(sizeof(struct hashmap_entry)); + if (!entry) + return -ENOMEM; + + entry->key = key; + entry->value = value; + hashmap_add_entry(&map->buckets[h], entry); + map->sz++; + + return 0; +} + +bool hashmap_find(const struct hashmap *map, long key, long *value) +{ + struct hashmap_entry *entry; + size_t h; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (!hashmap_find_entry(map, key, h, NULL, &entry)) + return false; + + if (value) + *value = entry->value; + return true; +} + +bool hashmap_delete(struct hashmap *map, long key, + long *old_key, long *old_value) +{ + struct hashmap_entry **pprev, *entry; + size_t h; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (!hashmap_find_entry(map, key, h, &pprev, &entry)) + return false; + + if (old_key) + *old_key = entry->key; + if (old_value) + *old_value = entry->value; + + hashmap_del_entry(pprev, entry); + free(entry); + map->sz--; + + return true; +} diff --git a/tools/perf/util/hashmap.h b/tools/perf/util/hashmap.h new file mode 100644 index 000000000..0c4f155e8 --- /dev/null +++ b/tools/perf/util/hashmap.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +/* + * Generic non-thread safe hash map implementation. + * + * Copyright (c) 2019 Facebook + */ +#ifndef __LIBBPF_HASHMAP_H +#define __LIBBPF_HASHMAP_H + +#include +#include +#include + +static inline size_t hash_bits(size_t h, int bits) +{ + /* shuffle bits and return requested number of upper bits */ + if (bits == 0) + return 0; + +#if (__SIZEOF_SIZE_T__ == __SIZEOF_LONG_LONG__) + /* LP64 case */ + return (h * 11400714819323198485llu) >> (__SIZEOF_LONG_LONG__ * 8 - bits); +#elif (__SIZEOF_SIZE_T__ <= __SIZEOF_LONG__) + return (h * 2654435769lu) >> (__SIZEOF_LONG__ * 8 - bits); +#else +# error "Unsupported size_t size" +#endif +} + +/* generic C-string hashing function */ +static inline size_t str_hash(const char *s) +{ + size_t h = 0; + + while (*s) { + h = h * 31 + *s; + s++; + } + return h; +} + +typedef size_t (*hashmap_hash_fn)(long key, void *ctx); +typedef bool (*hashmap_equal_fn)(long key1, long key2, void *ctx); + +/* + * Hashmap interface is polymorphic, keys and values could be either + * long-sized integers or pointers, this is achieved as follows: + * - interface functions that operate on keys and values are hidden + * behind auxiliary macros, e.g. hashmap_insert <-> hashmap__insert; + * - these auxiliary macros cast the key and value parameters as + * long or long *, so the user does not have to specify the casts explicitly; + * - for pointer parameters (e.g. old_key) the size of the pointed + * type is verified by hashmap_cast_ptr using _Static_assert; + * - when iterating using hashmap__for_each_* forms + * hasmap_entry->key should be used for integer keys and + * hasmap_entry->pkey should be used for pointer keys, + * same goes for values. + */ +struct hashmap_entry { + union { + long key; + const void *pkey; + }; + union { + long value; + void *pvalue; + }; + struct hashmap_entry *next; +}; + +struct hashmap { + hashmap_hash_fn hash_fn; + hashmap_equal_fn equal_fn; + void *ctx; + + struct hashmap_entry **buckets; + size_t cap; + size_t cap_bits; + size_t sz; +}; + +void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, void *ctx); +struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, + void *ctx); +void hashmap__clear(struct hashmap *map); +void hashmap__free(struct hashmap *map); + +size_t hashmap__size(const struct hashmap *map); +size_t hashmap__capacity(const struct hashmap *map); + +/* + * Hashmap insertion strategy: + * - HASHMAP_ADD - only add key/value if key doesn't exist yet; + * - HASHMAP_SET - add key/value pair if key doesn't exist yet; otherwise, + * update value; + * - HASHMAP_UPDATE - update value, if key already exists; otherwise, do + * nothing and return -ENOENT; + * - HASHMAP_APPEND - always add key/value pair, even if key already exists. + * This turns hashmap into a multimap by allowing multiple values to be + * associated with the same key. Most useful read API for such hashmap is + * hashmap__for_each_key_entry() iteration. If hashmap__find() is still + * used, it will return last inserted key/value entry (first in a bucket + * chain). + */ +enum hashmap_insert_strategy { + HASHMAP_ADD, + HASHMAP_SET, + HASHMAP_UPDATE, + HASHMAP_APPEND, +}; + +#define hashmap_cast_ptr(p) ({ \ + _Static_assert((__builtin_constant_p((p)) ? (p) == NULL : 0) || \ + sizeof(*(p)) == sizeof(long), \ + #p " pointee should be a long-sized integer or a pointer"); \ + (long *)(p); \ +}) + +/* + * hashmap__insert() adds key/value entry w/ various semantics, depending on + * provided strategy value. If a given key/value pair replaced already + * existing key/value pair, both old key and old value will be returned + * through old_key and old_value to allow calling code do proper memory + * management. + */ +int hashmap_insert(struct hashmap *map, long key, long value, + enum hashmap_insert_strategy strategy, + long *old_key, long *old_value); + +#define hashmap__insert(map, key, value, strategy, old_key, old_value) \ + hashmap_insert((map), (long)(key), (long)(value), (strategy), \ + hashmap_cast_ptr(old_key), \ + hashmap_cast_ptr(old_value)) + +#define hashmap__add(map, key, value) \ + hashmap__insert((map), (key), (value), HASHMAP_ADD, NULL, NULL) + +#define hashmap__set(map, key, value, old_key, old_value) \ + hashmap__insert((map), (key), (value), HASHMAP_SET, (old_key), (old_value)) + +#define hashmap__update(map, key, value, old_key, old_value) \ + hashmap__insert((map), (key), (value), HASHMAP_UPDATE, (old_key), (old_value)) + +#define hashmap__append(map, key, value) \ + hashmap__insert((map), (key), (value), HASHMAP_APPEND, NULL, NULL) + +bool hashmap_delete(struct hashmap *map, long key, long *old_key, long *old_value); + +#define hashmap__delete(map, key, old_key, old_value) \ + hashmap_delete((map), (long)(key), \ + hashmap_cast_ptr(old_key), \ + hashmap_cast_ptr(old_value)) + +bool hashmap_find(const struct hashmap *map, long key, long *value); + +#define hashmap__find(map, key, value) \ + hashmap_find((map), (long)(key), hashmap_cast_ptr(value)) + +/* + * hashmap__for_each_entry - iterate over all entries in hashmap + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @bkt: integer used as a bucket loop cursor + */ +#define hashmap__for_each_entry(map, cur, bkt) \ + for (bkt = 0; bkt < (map)->cap; bkt++) \ + for (cur = (map)->buckets[bkt]; cur; cur = cur->next) + +/* + * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe + * against removals + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @tmp: struct hashmap_entry * used as a temporary next cursor storage + * @bkt: integer used as a bucket loop cursor + */ +#define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \ + for (bkt = 0; bkt < (map)->cap; bkt++) \ + for (cur = (map)->buckets[bkt]; \ + cur && ({tmp = cur->next; true; }); \ + cur = tmp) + +/* + * hashmap__for_each_key_entry - iterate over entries associated with given key + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @key: key to iterate entries for + */ +#define hashmap__for_each_key_entry(map, cur, _key) \ + for (cur = (map)->buckets \ + ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \ + : NULL; \ + cur; \ + cur = cur->next) \ + if ((map)->equal_fn(cur->key, (_key), (map)->ctx)) + +#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \ + for (cur = (map)->buckets \ + ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \ + : NULL; \ + cur && ({ tmp = cur->next; true; }); \ + cur = tmp) \ + if ((map)->equal_fn(cur->key, (_key), (map)->ctx)) + +#endif /* __LIBBPF_HASHMAP_H */ diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c new file mode 100644 index 000000000..7db7da090 --- /dev/null +++ b/tools/perf/util/header.c @@ -0,0 +1,5801 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "string2.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBBPF_SUPPORT +#include +#endif +#include +#include // reallocarray + +#include "dso.h" +#include "evlist.h" +#include "evsel.h" +#include "util/evsel_fprintf.h" +#include "header.h" +#include "memswap.h" +#include "trace-event.h" +#include "session.h" +#include "symbol.h" +#include "debug.h" +#include "cpumap.h" +#include "pmu.h" +#include "pmus.h" +#include "vdso.h" +#include "strbuf.h" +#include "build-id.h" +#include "data.h" +#include +#include +#include "asm/bug.h" +#include "tool.h" +#include "../perf.h" +#include "time-utils.h" +#include "units.h" +#include "util/util.h" // perf_exe() +#include "cputopo.h" +#include "bpf-event.h" +#include "bpf-utils.h" +#include "clockid.h" +#include "cacheline.h" + +#include +#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +/* + * nr_ids * sizeof(struct perf_sample_id) must not overflow + * size_t on 32-bit; the struct is ~104 bytes (32-bit) or + * ~184 bytes (64-bit), so 1<<24 (16M) keeps the product + * under 2 GB on 32-bit. + * + * This is a per-attribute cap only — the total across all + * attributes is not capped because legitimate high-core-count + * workloads (e.g. 5000 tracepoints × 4096 CPUs) can exceed + * a single-attribute limit. + */ +#define MAX_IDS_PER_ATTR (1 << 24) +/* + * Cap nr_attrs to prevent resource exhaustion from crafted + * files. 65536 is well beyond any real workload (perf stat + * typically uses < 100 events) but prevents u64-to-int + * truncation on the attr count. + */ +#define MAX_NR_ATTRS (1 << 16) +#define MAX_BPF_DATA_LEN (256 * 1024 * 1024) +#define MAX_BPF_PROGS 131072 +#define MAX_CACHE_ENTRIES 32768 +#define MAX_GROUP_DESC 32768 +#define MAX_NUMA_NODES 4096 +#define MAX_PMU_CAPS 512 +#define MAX_PMU_MAPPINGS 4096 +#define MAX_SCHED_DOMAINS 64 + +/* + * magic2 = "PERFILE2" + * must be a numerical value to let the endianness + * determine the memory layout. That way we are able + * to detect endianness when reading the perf.data file + * back. + * + * we check for legacy (PERFFILE) format. + */ +static const char *__perf_magic1 = "PERFFILE"; +static const u64 __perf_magic2 = 0x32454c4946524550ULL; +static const u64 __perf_magic2_sw = 0x50455246494c4532ULL; + +#define PERF_MAGIC __perf_magic2 +#define DNAME_LEN 16 + +const char perf_version_string[] = PERF_VERSION; + +struct perf_file_attr { + struct perf_event_attr attr; + struct perf_file_section ids; +}; + +void perf_header__set_feat(struct perf_header *header, int feat) +{ + __set_bit(feat, header->adds_features); +} + +void perf_header__clear_feat(struct perf_header *header, int feat) +{ + __clear_bit(feat, header->adds_features); +} + +bool perf_header__has_feat(const struct perf_header *header, int feat) +{ + return test_bit(feat, header->adds_features); +} + +static int __do_write_fd(struct feat_fd *ff, const void *buf, size_t size) +{ + ssize_t ret = writen(ff->fd, buf, size); + + if (ret != (ssize_t)size) + return ret < 0 ? (int)ret : -1; + return 0; +} + +static int __do_write_buf(struct feat_fd *ff, const void *buf, size_t size) +{ + /* struct perf_event_header::size is u16 */ + const size_t max_size = 0xffff - sizeof(struct perf_event_header); + size_t new_size = ff->size; + void *addr; + + if (size + ff->offset > max_size) + return -E2BIG; + + while (size > (new_size - ff->offset)) + new_size <<= 1; + new_size = min(max_size, new_size); + + if (ff->size < new_size) { + addr = realloc(ff->buf, new_size); + if (!addr) + return -ENOMEM; + ff->buf = addr; + ff->size = new_size; + } + + memcpy(ff->buf + ff->offset, buf, size); + ff->offset += size; + + return 0; +} + +/* Return: 0 if succeeded, -ERR if failed. */ +int do_write(struct feat_fd *ff, const void *buf, size_t size) +{ + if (!ff->buf) + return __do_write_fd(ff, buf, size); + return __do_write_buf(ff, buf, size); +} + +/* Return: 0 if succeeded, -ERR if failed. */ +static int do_write_bitmap(struct feat_fd *ff, unsigned long *set, u64 size) +{ + size_t byte_size = BITS_TO_LONGS(size) * sizeof(unsigned long); + int i, ret; + + ret = do_write(ff, &size, sizeof(size)); + if (ret < 0) + return ret; + + /* + * The on-disk format uses u64 elements, but the in-memory bitmap + * uses unsigned long, which is only 4 bytes on 32-bit architectures. + * Copy with bounded size so the last element doesn't read past the + * bitmap allocation when BITS_TO_LONGS(size) is odd. + */ + for (i = 0; (u64) i < BITS_TO_U64(size); i++) { + u64 val = 0; + size_t off = i * sizeof(val); + + memcpy(&val, (char *)set + off, min(sizeof(val), byte_size - off)); + ret = do_write(ff, &val, sizeof(val)); + if (ret < 0) + return ret; + } + + return 0; +} + +/* Return: 0 if succeeded, -ERR if failed. */ +int write_padded(struct feat_fd *ff, const void *bf, + size_t count, size_t count_aligned) +{ + static const char zero_buf[NAME_ALIGN]; + int err = do_write(ff, bf, count); + + if (!err) + err = do_write(ff, zero_buf, count_aligned - count); + + return err; +} + +#define string_size(str) \ + (PERF_ALIGN((strlen(str) + 1), NAME_ALIGN) + sizeof(u32)) + +/* Return: 0 if succeeded, -ERR if failed. */ +static int do_write_string(struct feat_fd *ff, const char *str) +{ + u32 len, olen; + int ret; + + olen = strlen(str) + 1; + len = PERF_ALIGN(olen, NAME_ALIGN); + + /* write len, incl. \0 */ + ret = do_write(ff, &len, sizeof(len)); + if (ret < 0) + return ret; + + return write_padded(ff, str, olen, len); +} + +static int __do_read_fd(struct feat_fd *ff, void *addr, ssize_t size) +{ + ssize_t ret = readn(ff->fd, addr, size); + + if (ret != size) + return ret < 0 ? (int)ret : -1; + ff->offset += size; + return 0; +} + +static int __do_read_buf(struct feat_fd *ff, void *addr, ssize_t size) +{ + memcpy(addr, ff->buf + ff->offset, size); + ff->offset += size; + + return 0; +} + +static int __do_read(struct feat_fd *ff, void *addr, ssize_t size) +{ + /* + * Reject negative sizes, which on 32-bit can occur when a + * u32 >= 0x80000000 is passed as ssize_t. The cast to + * ssize_t is safe because perf_header__process_sections() + * validates that each section fits within the file size + * before any feature callback reaches here, and only + * feature sections (metadata like build IDs, topology, etc.) + * use this path — these cannot legitimately approach 2GB. + */ + if (size < 0 || size > (ssize_t)ff->size - ff->offset) + return -1; + + if (!ff->buf) + return __do_read_fd(ff, addr, size); + return __do_read_buf(ff, addr, size); +} + +static int do_read_u32(struct feat_fd *ff, u32 *addr) +{ + int ret; + + ret = __do_read(ff, addr, sizeof(*addr)); + if (ret) + return ret; + + if (ff->ph->needs_swap) + *addr = bswap_32(*addr); + return 0; +} + +static int do_read_u64(struct feat_fd *ff, u64 *addr) +{ + int ret; + + ret = __do_read(ff, addr, sizeof(*addr)); + if (ret) + return ret; + + if (ff->ph->needs_swap) + *addr = bswap_64(*addr); + return 0; +} + +static char *do_read_string(struct feat_fd *ff) +{ + u32 len; + char *buf; + + if (do_read_u32(ff, &len)) + return NULL; + + /* At least the null terminator. */ + if (len < 1 || len > ff->size - ff->offset) { + pr_debug("do_read_string: invalid length %u (remaining %zu)\n", + len, (size_t)(ff->size - ff->offset)); + return NULL; + } + + buf = malloc(len); + if (!buf) + return NULL; + + if (!__do_read(ff, buf, len)) { + /* + * do_write_string() writes len including the null + * terminator, padded to NAME_ALIGN. Ensure the + * string is always null-terminated even if the file + * data has been tampered with. + */ + buf[len - 1] = '\0'; + return buf; + } + + free(buf); + return NULL; +} + +/* Return: 0 if succeeded, -ERR if failed. */ +static int do_read_bitmap(struct feat_fd *ff, unsigned long **pset, u64 *psize) +{ + unsigned long *set; + u64 size, *p; + int i, ret; + + ret = do_read_u64(ff, &size); + if (ret) + return ret; + + /* Bitmap APIs use int for nbits; reject u64 values that truncate. */ + if (size > INT_MAX || + BITS_TO_U64(size) > (ff->size - ff->offset) / sizeof(u64)) { + pr_debug("do_read_bitmap: size %" PRIu64 " exceeds section bounds\n", size); + return -1; + } + + /* + * bitmap_zalloc() allocates in unsigned long units, which are only + * 4 bytes on 32-bit architectures. The read loop below casts the + * buffer to u64 * and writes 8-byte elements, so allocate in u64 + * units to ensure the buffer is large enough. + */ + set = calloc(BITS_TO_U64(size), sizeof(u64)); + if (!set) + return -ENOMEM; + + p = (u64 *) set; + + for (i = 0; (u64) i < BITS_TO_U64(size); i++) { + ret = do_read_u64(ff, p + i); + if (ret < 0) { + free(set); + return ret; + } + } + + *pset = set; + *psize = size; + return 0; +} + +static int write_tracing_data(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + +#ifdef HAVE_LIBTRACEEVENT + return read_tracing_data(ff->fd, &evlist__core(evlist)->entries); +#else + pr_err("ERROR: Trying to write tracing data without libtraceevent support.\n"); + return -1; +#endif +} + +static int write_build_id(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_session *session; + int err; + + session = container_of(ff->ph, struct perf_session, header); + + if (!perf_session__read_build_ids(session, true)) + return -1; + + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + + err = perf_session__write_buildid_table(session, ff); + if (err < 0) { + pr_debug("failed to write buildid table\n"); + return err; + } + + return 0; +} + +static int write_hostname(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct utsname uts; + int ret; + + ret = uname(&uts); + if (ret < 0) + return -1; + + return do_write_string(ff, uts.nodename); +} + +static int write_osrelease(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct utsname uts; + const char *release = NULL; + + if (evlist__session(evlist)) + release = perf_env__os_release(perf_session__env(evlist__session(evlist))); + + if (!release) { + int ret = uname(&uts); + + if (ret < 0) + return -1; + release = uts.release; + } + return do_write_string(ff, release); +} + +static int write_arch(struct feat_fd *ff, struct evlist *evlist) +{ + struct utsname uts; + const char *arch = NULL; + + if (evlist__session(evlist)) + arch = perf_env__arch(perf_session__env(evlist__session(evlist))); + + if (!arch) { + int ret = uname(&uts); + + if (ret < 0) + return -1; + arch = uts.machine; + } + return do_write_string(ff, arch); +} + +static int write_e_machine(struct feat_fd *ff, struct evlist *evlist) +{ + /* e_machine expanded from 16 to 32-bits for alignment. */ + uint32_t e_flags; + uint32_t e_machine = perf_session__e_machine(evlist__session(evlist), &e_flags); + int ret; + + ret = do_write(ff, &e_machine, sizeof(e_machine)); + if (ret) + return ret; + + return do_write(ff, &e_flags, sizeof(e_flags)); +} + +static int write_version(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + return do_write_string(ff, perf_version_string); +} + +static int __write_cpudesc(struct feat_fd *ff, const char *cpuinfo_proc) +{ + FILE *file; + char *buf = NULL; + char *s, *p; + const char *search = cpuinfo_proc; + size_t len = 0; + int ret = -1; + + if (!search) + return -1; + + file = fopen("/proc/cpuinfo", "r"); + if (!file) + return -1; + + while (getline(&buf, &len, file) > 0) { + ret = strncmp(buf, search, strlen(search)); + if (!ret) + break; + } + + if (ret) { + ret = -1; + goto done; + } + + s = buf; + + p = strchr(buf, ':'); + if (p && *(p+1) == ' ' && *(p+2)) + s = p + 2; + p = strchr(s, '\n'); + if (p) + *p = '\0'; + + /* squash extra space characters (branding string) */ + p = s; + while (*p) { + if (isspace(*p)) { + char *r = p + 1; + char *q = skip_spaces(r); + *p = ' '; + if (q != (p+1)) + while ((*r++ = *q++)); + } + p++; + } + ret = do_write_string(ff, s); +done: + free(buf); + fclose(file); + return ret; +} + +static int write_cpudesc(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ +#if defined(__powerpc__) || defined(__hppa__) || defined(__sparc__) +#define CPUINFO_PROC { "cpu", } +#elif defined(__s390__) +#define CPUINFO_PROC { "vendor_id", } +#elif defined(__sh__) +#define CPUINFO_PROC { "cpu type", } +#elif defined(__alpha__) || defined(__mips__) +#define CPUINFO_PROC { "cpu model", } +#elif defined(__arm__) +#define CPUINFO_PROC { "model name", "Processor", } +#elif defined(__arc__) +#define CPUINFO_PROC { "Processor", } +#elif defined(__xtensa__) +#define CPUINFO_PROC { "core ID", } +#elif defined(__loongarch__) +#define CPUINFO_PROC { "Model Name", } +#else +#define CPUINFO_PROC { "model name", } +#endif + const char *cpuinfo_procs[] = CPUINFO_PROC; +#undef CPUINFO_PROC + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cpuinfo_procs); i++) { + int ret; + ret = __write_cpudesc(ff, cpuinfo_procs[i]); + if (ret >= 0) + return ret; + } + return -1; +} + + +static int write_nrcpus(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + long nr; + u32 nrc, nra; + int ret; + + nrc = cpu__max_present_cpu().cpu; + + nr = sysconf(_SC_NPROCESSORS_ONLN); + if (nr < 0) + return -1; + + nra = (u32)(nr & UINT_MAX); + + ret = do_write(ff, &nrc, sizeof(nrc)); + if (ret < 0) + return ret; + + return do_write(ff, &nra, sizeof(nra)); +} + +static int write_event_desc(struct feat_fd *ff, + struct evlist *evlist) +{ + struct evsel *evsel; + u32 nre, nri, sz; + int ret; + + nre = evlist__nr_entries(evlist); + + /* + * write number of events + */ + ret = do_write(ff, &nre, sizeof(nre)); + if (ret < 0) + return ret; + + /* + * size of perf_event_attr struct + */ + sz = (u32)sizeof(evsel->core.attr); + ret = do_write(ff, &sz, sizeof(sz)); + if (ret < 0) + return ret; + + evlist__for_each_entry(evlist, evsel) { + ret = do_write(ff, &evsel->core.attr, sz); + if (ret < 0) + return ret; + /* + * write number of unique id per event + * there is one id per instance of an event + * + * copy into an nri to be independent of the + * type of ids, + */ + nri = evsel->core.ids; + ret = do_write(ff, &nri, sizeof(nri)); + if (ret < 0) + return ret; + + /* + * write event string as passed on cmdline + */ + ret = do_write_string(ff, evsel__name(evsel)); + if (ret < 0) + return ret; + /* + * write unique ids for this event + */ + ret = do_write(ff, evsel->core.id, evsel->core.ids * sizeof(u64)); + if (ret < 0) + return ret; + } + return 0; +} + +static int write_cmdline(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + char pbuf[MAXPATHLEN], *buf; + int i, ret, n; + + /* actual path to perf binary */ + buf = perf_exe(pbuf, MAXPATHLEN); + + /* account for binary path */ + n = env->nr_cmdline + 1; + + ret = do_write(ff, &n, sizeof(n)); + if (ret < 0) + return ret; + + ret = do_write_string(ff, buf); + if (ret < 0) + return ret; + + for (i = 0 ; i < env->nr_cmdline; i++) { + ret = do_write_string(ff, env->cmdline_argv[i]); + if (ret < 0) + return ret; + } + return 0; +} + + +static int write_cpu_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct cpu_topology *tp; + u32 i; + int ret, j; + + tp = cpu_topology__new(); + if (!tp) + return -1; + + ret = do_write(ff, &tp->package_cpus_lists, sizeof(tp->package_cpus_lists)); + if (ret < 0) + goto done; + + for (i = 0; i < tp->package_cpus_lists; i++) { + ret = do_write_string(ff, tp->package_cpus_list[i]); + if (ret < 0) + goto done; + } + ret = do_write(ff, &tp->core_cpus_lists, sizeof(tp->core_cpus_lists)); + if (ret < 0) + goto done; + + for (i = 0; i < tp->core_cpus_lists; i++) { + ret = do_write_string(ff, tp->core_cpus_list[i]); + if (ret < 0) + break; + } + + ret = perf_env__read_cpu_topology_map(env); + if (ret < 0) + goto done; + + for (j = 0; j < env->nr_cpus_avail; j++) { + ret = do_write(ff, &env->cpu[j].core_id, + sizeof(env->cpu[j].core_id)); + if (ret < 0) + return ret; + ret = do_write(ff, &env->cpu[j].socket_id, + sizeof(env->cpu[j].socket_id)); + if (ret < 0) + return ret; + } + + if (!tp->die_cpus_lists) + goto done; + + ret = do_write(ff, &tp->die_cpus_lists, sizeof(tp->die_cpus_lists)); + if (ret < 0) + goto done; + + for (i = 0; i < tp->die_cpus_lists; i++) { + ret = do_write_string(ff, tp->die_cpus_list[i]); + if (ret < 0) + goto done; + } + + for (j = 0; j < env->nr_cpus_avail; j++) { + ret = do_write(ff, &env->cpu[j].die_id, + sizeof(env->cpu[j].die_id)); + if (ret < 0) + return ret; + } + +done: + cpu_topology__delete(tp); + return ret; +} + + + +static int write_total_mem(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + char *buf = NULL; + FILE *fp; + size_t len = 0; + int ret = -1, n; + uint64_t mem; + + fp = fopen("/proc/meminfo", "r"); + if (!fp) + return -1; + + while (getline(&buf, &len, fp) > 0) { + ret = strncmp(buf, "MemTotal:", 9); + if (!ret) + break; + } + if (!ret) { + n = sscanf(buf, "%*s %"PRIu64, &mem); + if (n == 1) + ret = do_write(ff, &mem, sizeof(mem)); + } else + ret = -1; + free(buf); + fclose(fp); + return ret; +} + +static int write_numa_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct numa_topology *tp; + int ret = -1; + u32 i; + + tp = numa_topology__new(); + if (!tp) + return -ENOMEM; + + ret = do_write(ff, &tp->nr, sizeof(u32)); + if (ret < 0) + goto err; + + for (i = 0; i < tp->nr; i++) { + struct numa_topology_node *n = &tp->nodes[i]; + + ret = do_write(ff, &n->node, sizeof(u32)); + if (ret < 0) + goto err; + + ret = do_write(ff, &n->mem_total, sizeof(u64)); + if (ret) + goto err; + + ret = do_write(ff, &n->mem_free, sizeof(u64)); + if (ret) + goto err; + + ret = do_write_string(ff, n->cpus); + if (ret < 0) + goto err; + } + + ret = 0; + +err: + numa_topology__delete(tp); + return ret; +} + +/* + * File format: + * + * struct pmu_mappings { + * u32 pmu_num; + * struct pmu_map { + * u32 type; + * char name[]; + * }[pmu_num]; + * }; + */ + +static int write_pmu_mappings(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *pmu = NULL; + u32 pmu_num = 0; + int ret; + + /* + * Do a first pass to count number of pmu to avoid lseek so this + * works in pipe mode as well. + */ + while ((pmu = perf_pmus__scan(pmu))) + pmu_num++; + + ret = do_write(ff, &pmu_num, sizeof(pmu_num)); + if (ret < 0) + return ret; + + while ((pmu = perf_pmus__scan(pmu))) { + ret = do_write(ff, &pmu->type, sizeof(pmu->type)); + if (ret < 0) + return ret; + + ret = do_write_string(ff, pmu->name); + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * File format: + * + * struct group_descs { + * u32 nr_groups; + * struct group_desc { + * char name[]; + * u32 leader_idx; + * u32 nr_members; + * }[nr_groups]; + * }; + */ +static int write_group_desc(struct feat_fd *ff, + struct evlist *evlist) +{ + u32 nr_groups = evlist__nr_groups(evlist); + struct evsel *evsel; + int ret; + + ret = do_write(ff, &nr_groups, sizeof(nr_groups)); + if (ret < 0) + return ret; + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_group_leader(evsel) && evsel->core.nr_members > 1) { + const char *name = evsel->group_name ?: "{anon_group}"; + u32 leader_idx = evsel->core.idx; + u32 nr_members = evsel->core.nr_members; + + ret = do_write_string(ff, name); + if (ret < 0) + return ret; + + ret = do_write(ff, &leader_idx, sizeof(leader_idx)); + if (ret < 0) + return ret; + + ret = do_write(ff, &nr_members, sizeof(nr_members)); + if (ret < 0) + return ret; + } + } + return 0; +} + +/* + * Return the CPU id as a raw string. + * + * Each architecture should provide a more precise id string that + * can be use to match the architecture's "mapfile". + */ +char * __weak get_cpuid_str(struct perf_cpu cpu __maybe_unused) +{ + return NULL; +} + +char *get_cpuid_allow_env_override(struct perf_cpu cpu) +{ + char *cpuid; + static bool printed; + + cpuid = getenv("PERF_CPUID"); + if (cpuid) + cpuid = strdup(cpuid); + if (!cpuid) + cpuid = get_cpuid_str(cpu); + if (!cpuid) + return NULL; + + if (!printed) { + pr_debug("Using CPUID %s\n", cpuid); + printed = true; + } + return cpuid; +} + +/* Return zero when the cpuid from the mapfile.csv matches the + * cpuid string generated on this platform. + * Otherwise return non-zero. + */ +int __weak strcmp_cpuid_str(const char *mapcpuid, const char *cpuid) +{ + regex_t re; + regmatch_t pmatch[1]; + int match; + + if (regcomp(&re, mapcpuid, REG_EXTENDED) != 0) { + /* Warn unable to generate match particular string. */ + pr_info("Invalid regular expression %s\n", mapcpuid); + return 1; + } + + match = !regexec(&re, cpuid, 1, pmatch, 0); + regfree(&re); + if (match) { + size_t match_len = (pmatch[0].rm_eo - pmatch[0].rm_so); + + /* Verify the entire string matched. */ + if (match_len == strlen(cpuid)) + return 0; + } + return 1; +} + +/* + * default get_cpuid(): nothing gets recorded + * actual implementation must be in arch/$(SRCARCH)/util/header.c + */ +int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused, + struct perf_cpu cpu __maybe_unused) +{ + return ENOSYS; /* Not implemented */ +} + +static int write_cpuid(struct feat_fd *ff, struct evlist *evlist) +{ + struct perf_cpu cpu = perf_cpu_map__min(evlist__core(evlist)->all_cpus); + char buffer[64]; + int ret; + + ret = get_cpuid(buffer, sizeof(buffer), cpu); + if (ret) + return -1; + + return do_write_string(ff, buffer); +} + +static int write_branch_stack(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + return 0; +} + +static int write_auxtrace(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_session *session; + int err; + + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + + session = container_of(ff->ph, struct perf_session, header); + + err = auxtrace_index__write(ff->fd, &session->auxtrace_index); + if (err < 0) + pr_err("Failed to write auxtrace index\n"); + return err; +} + +static int write_clockid(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + return do_write(ff, &ff->ph->env.clock.clockid_res_ns, + sizeof(ff->ph->env.clock.clockid_res_ns)); +} + +static int write_clock_data(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + u64 *data64; + u32 data32; + int ret; + + /* version */ + data32 = 1; + + ret = do_write(ff, &data32, sizeof(data32)); + if (ret < 0) + return ret; + + /* clockid */ + data32 = ff->ph->env.clock.clockid; + + ret = do_write(ff, &data32, sizeof(data32)); + if (ret < 0) + return ret; + + /* TOD ref time */ + data64 = &ff->ph->env.clock.tod_ns; + + ret = do_write(ff, data64, sizeof(*data64)); + if (ret < 0) + return ret; + + /* clockid ref time */ + data64 = &ff->ph->env.clock.clockid_ns; + + return do_write(ff, data64, sizeof(*data64)); +} + +static int write_hybrid_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct hybrid_topology *tp; + int ret; + u32 i; + + tp = hybrid_topology__new(); + if (!tp) + return -ENOENT; + + ret = do_write(ff, &tp->nr, sizeof(u32)); + if (ret < 0) + goto err; + + for (i = 0; i < tp->nr; i++) { + struct hybrid_topology_node *n = &tp->nodes[i]; + + ret = do_write_string(ff, n->pmu_name); + if (ret < 0) + goto err; + + ret = do_write_string(ff, n->cpus); + if (ret < 0) + goto err; + } + + ret = 0; + +err: + hybrid_topology__delete(tp); + return ret; +} + +static int write_dir_format(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_session *session; + struct perf_data *data; + + session = container_of(ff->ph, struct perf_session, header); + data = session->data; + + if (WARN_ON(!perf_data__is_dir(data))) + return -1; + + return do_write(ff, &data->dir.version, sizeof(data->dir.version)); +} + +static int write_bpf_prog_info(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + int ret = 0; + + /* write lock: bpil_addr_to_offs() temporarily mutates info_linear */ + down_write(&env->bpf_progs.lock); + + ret = do_write(ff, &env->bpf_progs.infos_cnt, + sizeof(env->bpf_progs.infos_cnt)); + if (ret < 0 || env->bpf_progs.infos_cnt == 0) + goto out; + + root = &env->bpf_progs.infos; + next = rb_first(root); + while (next) { + struct bpf_prog_info_node *node; + size_t len; + + node = rb_entry(next, struct bpf_prog_info_node, rb_node); + next = rb_next(&node->rb_node); + len = sizeof(struct perf_bpil) + + node->info_linear->data_len; + + /* before writing to file, translate address to offset */ + bpil_addr_to_offs(node->info_linear); + ret = do_write(ff, node->info_linear, len); + /* + * translate back to address even when do_write() fails, + * so that this function never changes the data. + */ + bpil_offs_to_addr(node->info_linear); + if (ret < 0) + goto out; + } +out: + up_write(&env->bpf_progs.lock); + return ret; +#else + pr_err("ERROR: Trying to write bpf_prog_info without libbpf support.\n"); + return -1; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int write_bpf_btf(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + int ret = 0; + + down_read(&env->bpf_progs.lock); + + ret = do_write(ff, &env->bpf_progs.btfs_cnt, + sizeof(env->bpf_progs.btfs_cnt)); + + if (ret < 0 || env->bpf_progs.btfs_cnt == 0) + goto out; + + root = &env->bpf_progs.btfs; + next = rb_first(root); + while (next) { + struct btf_node *node; + + node = rb_entry(next, struct btf_node, rb_node); + next = rb_next(&node->rb_node); + ret = do_write(ff, &node->id, + sizeof(u32) * 2 + node->data_size); + if (ret < 0) + goto out; + } +out: + up_read(&env->bpf_progs.lock); + return ret; +#else + pr_err("ERROR: Trying to write btf data without libbpf support.\n"); + return -1; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int cpu_cache_level__sort(const void *a, const void *b) +{ + struct cpu_cache_level *cache_a = (struct cpu_cache_level *)a; + struct cpu_cache_level *cache_b = (struct cpu_cache_level *)b; + + return cache_a->level - cache_b->level; +} + +static bool cpu_cache_level__cmp(struct cpu_cache_level *a, struct cpu_cache_level *b) +{ + if (a->level != b->level) + return false; + + if (a->line_size != b->line_size) + return false; + + if (a->sets != b->sets) + return false; + + if (a->ways != b->ways) + return false; + + if (strcmp(a->type, b->type)) + return false; + + if (strcmp(a->size, b->size)) + return false; + + if (strcmp(a->map, b->map)) + return false; + + return true; +} + +static int cpu_cache_level__read(struct cpu_cache_level *cache, u32 cpu, u16 level) +{ + char path[PATH_MAX], file[PATH_MAX]; + struct stat st; + size_t len; + + scnprintf(path, PATH_MAX, "devices/system/cpu/cpu%d/cache/index%d/", cpu, level); + scnprintf(file, PATH_MAX, "%s/%s", sysfs__mountpoint(), path); + + if (stat(file, &st)) + return 1; + + scnprintf(file, PATH_MAX, "%s/level", path); + if (sysfs__read_int(file, (int *) &cache->level)) + return -1; + + scnprintf(file, PATH_MAX, "%s/coherency_line_size", path); + if (sysfs__read_int(file, (int *) &cache->line_size)) + return -1; + + scnprintf(file, PATH_MAX, "%s/number_of_sets", path); + if (sysfs__read_int(file, (int *) &cache->sets)) + return -1; + + scnprintf(file, PATH_MAX, "%s/ways_of_associativity", path); + if (sysfs__read_int(file, (int *) &cache->ways)) + return -1; + + scnprintf(file, PATH_MAX, "%s/type", path); + if (sysfs__read_str(file, &cache->type, &len)) + return -1; + + cache->type[len] = 0; + cache->type = strim(cache->type); + + scnprintf(file, PATH_MAX, "%s/size", path); + if (sysfs__read_str(file, &cache->size, &len)) { + zfree(&cache->type); + return -1; + } + + cache->size[len] = 0; + cache->size = strim(cache->size); + + scnprintf(file, PATH_MAX, "%s/shared_cpu_list", path); + if (sysfs__read_str(file, &cache->map, &len)) { + zfree(&cache->size); + zfree(&cache->type); + return -1; + } + + cache->map[len] = 0; + cache->map = strim(cache->map); + return 0; +} + +static void cpu_cache_level__fprintf(FILE *out, struct cpu_cache_level *c) +{ + fprintf(out, "L%d %-15s %8s [%s]\n", c->level, c->type, c->size, c->map); +} + +/* + * Build caches levels for a particular CPU from the data in + * /sys/devices/system/cpu/cpu/cache/ + * The cache level data is stored in caches[] from index at + * *cntp. + */ +int build_caches_for_cpu(u32 cpu, struct cpu_cache_level caches[], u32 *cntp) +{ + u16 level; + + for (level = 0; level < MAX_CACHE_LVL; level++) { + struct cpu_cache_level c; + int err; + u32 i; + + err = cpu_cache_level__read(&c, cpu, level); + if (err < 0) + return err; + + if (err == 1) + break; + + for (i = 0; i < *cntp; i++) { + if (cpu_cache_level__cmp(&c, &caches[i])) + break; + } + + if (i == *cntp) { + caches[*cntp] = c; + *cntp = *cntp + 1; + } else + cpu_cache_level__free(&c); + } + + return 0; +} + +static int build_caches(struct cpu_cache_level caches[], u32 *cntp) +{ + u32 nr, cpu, cnt = 0; + + nr = cpu__max_cpu().cpu; + + for (cpu = 0; cpu < nr; cpu++) { + int ret = build_caches_for_cpu(cpu, caches, &cnt); + + if (ret) + return ret; + } + *cntp = cnt; + return 0; +} + +static int write_cache(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + u32 max_caches = cpu__max_cpu().cpu * MAX_CACHE_LVL; + struct cpu_cache_level caches[max_caches]; + u32 cnt = 0, i, version = 1; + int ret; + + ret = build_caches(caches, &cnt); + if (ret) + goto out; + + qsort(&caches, cnt, sizeof(struct cpu_cache_level), cpu_cache_level__sort); + + ret = do_write(ff, &version, sizeof(u32)); + if (ret < 0) + goto out; + + ret = do_write(ff, &cnt, sizeof(u32)); + if (ret < 0) + goto out; + + for (i = 0; i < cnt; i++) { + struct cpu_cache_level *c = &caches[i]; + + #define _W(v) \ + ret = do_write(ff, &c->v, sizeof(u32)); \ + if (ret < 0) \ + goto out; + + _W(level) + _W(line_size) + _W(sets) + _W(ways) + #undef _W + + #define _W(v) \ + ret = do_write_string(ff, (const char *) c->v); \ + if (ret < 0) \ + goto out; + + _W(type) + _W(size) + _W(map) + #undef _W + } + +out: + for (i = 0; i < cnt; i++) + cpu_cache_level__free(&caches[i]); + return ret; +} + +static int write_cln_size(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + int cln_size = cacheline_size(); + + if (!cln_size) + cln_size = DEFAULT_CACHELINE_SIZE; + + ff->ph->env.cln_size = cln_size; + + return do_write(ff, &cln_size, sizeof(cln_size)); +} + +static int write_stat(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + return 0; +} + +static int write_sample_time(struct feat_fd *ff, + struct evlist *evlist) +{ + int ret; + u64 data = evlist__first_sample_time(evlist); + + ret = do_write(ff, &data, sizeof(data)); + if (ret < 0) + return ret; + + data = evlist__last_sample_time(evlist); + return do_write(ff, &data, sizeof(data)); +} + + +static int memory_node__read(struct memory_node *n, unsigned long idx) +{ + unsigned int phys, size = 0; + char path[PATH_MAX]; + struct io_dirent64 *ent; + struct io_dir dir; + +#define for_each_memory(mem, dir) \ + while ((ent = io_dir__readdir(&dir)) != NULL) \ + if (strcmp(ent->d_name, ".") && \ + strcmp(ent->d_name, "..") && \ + sscanf(ent->d_name, "memory%u", &mem) == 1) + + scnprintf(path, PATH_MAX, + "%s/devices/system/node/node%lu", + sysfs__mountpoint(), idx); + + io_dir__init(&dir, open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY)); + if (dir.dirfd < 0) { + pr_warning("failed: can't open memory sysfs data '%s'\n", path); + return -1; + } + + for_each_memory(phys, dir) { + size = max(phys, size); + } + + size++; + + n->set = bitmap_zalloc(size); + if (!n->set) { + close(dir.dirfd); + return -ENOMEM; + } + + n->node = idx; + n->size = size; + + io_dir__rewinddir(&dir); + + for_each_memory(phys, dir) { + __set_bit(phys, n->set); + } + + close(dir.dirfd); + return 0; +} + +static void memory_node__delete_nodes(struct memory_node *nodesp, u64 cnt) +{ + for (u64 i = 0; i < cnt; i++) + bitmap_free(nodesp[i].set); + + free(nodesp); +} + +static int memory_node__sort(const void *a, const void *b) +{ + const struct memory_node *na = a; + const struct memory_node *nb = b; + + return na->node - nb->node; +} + +static int build_mem_topology(struct memory_node **nodesp, u64 *cntp) +{ + char path[PATH_MAX]; + struct io_dirent64 *ent; + struct io_dir dir; + int ret = 0; + size_t cnt = 0, size = 0; + struct memory_node *nodes = NULL; + + scnprintf(path, PATH_MAX, "%s/devices/system/node/", + sysfs__mountpoint()); + + io_dir__init(&dir, open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY)); + if (dir.dirfd < 0) { + pr_debug2("%s: couldn't read %s, does this arch have topology information?\n", + __func__, path); + return -1; + } + + while (!ret && (ent = io_dir__readdir(&dir))) { + unsigned int idx; + int r; + + if (!strcmp(ent->d_name, ".") || + !strcmp(ent->d_name, "..")) + continue; + + r = sscanf(ent->d_name, "node%u", &idx); + if (r != 1) + continue; + + if (cnt >= size) { + struct memory_node *new_nodes = + reallocarray(nodes, cnt + 4, sizeof(*nodes)); + + if (!new_nodes) { + pr_err("Failed to write MEM_TOPOLOGY, size %zd nodes\n", size); + ret = -ENOMEM; + goto out; + } + nodes = new_nodes; + size += 4; + } + ret = memory_node__read(&nodes[cnt], idx); + if (!ret) + cnt += 1; + } +out: + close(dir.dirfd); + if (!ret) { + *cntp = cnt; + *nodesp = nodes; + qsort(nodes, cnt, sizeof(nodes[0]), memory_node__sort); + } else + memory_node__delete_nodes(nodes, cnt); + + return ret; +} + +/* + * The MEM_TOPOLOGY holds physical memory map for every + * node in system. The format of data is as follows: + * + * 0 - version | for future changes + * 8 - block_size_bytes | /sys/devices/system/memory/block_size_bytes + * 16 - count | number of nodes + * + * For each node we store map of physical indexes for + * each node: + * + * 32 - node id | node index + * 40 - size | size of bitmap + * 48 - bitmap | bitmap of memory indexes that belongs to node + */ +static int write_mem_topology(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + struct memory_node *nodes = NULL; + u64 bsize, version = 1, i, nr = 0; + int ret; + + ret = sysfs__read_xll("devices/system/memory/block_size_bytes", + (unsigned long long *) &bsize); + if (ret) + return ret; + + ret = build_mem_topology(&nodes, &nr); + if (ret) + return ret; + + ret = do_write(ff, &version, sizeof(version)); + if (ret < 0) + goto out; + + ret = do_write(ff, &bsize, sizeof(bsize)); + if (ret < 0) + goto out; + + ret = do_write(ff, &nr, sizeof(nr)); + if (ret < 0) + goto out; + + for (i = 0; i < nr; i++) { + struct memory_node *n = &nodes[i]; + + #define _W(v) \ + ret = do_write(ff, &n->v, sizeof(n->v)); \ + if (ret < 0) \ + goto out; + + _W(node) + _W(size) + + #undef _W + + ret = do_write_bitmap(ff, n->set, n->size); + if (ret < 0) + goto out; + } + +out: + memory_node__delete_nodes(nodes, nr); + return ret; +} + +static int write_compressed(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + int ret; + + ret = do_write(ff, &(ff->ph->env.comp_ver), sizeof(ff->ph->env.comp_ver)); + if (ret) + return ret; + + ret = do_write(ff, &(ff->ph->env.comp_type), sizeof(ff->ph->env.comp_type)); + if (ret) + return ret; + + ret = do_write(ff, &(ff->ph->env.comp_level), sizeof(ff->ph->env.comp_level)); + if (ret) + return ret; + + ret = do_write(ff, &(ff->ph->env.comp_ratio), sizeof(ff->ph->env.comp_ratio)); + if (ret) + return ret; + + return do_write(ff, &(ff->ph->env.comp_mmap_len), sizeof(ff->ph->env.comp_mmap_len)); +} + +static int __write_pmu_caps(struct feat_fd *ff, struct perf_pmu *pmu, + bool write_pmu) +{ + struct perf_pmu_caps *caps = NULL; + int ret; + + ret = do_write(ff, &pmu->nr_caps, sizeof(pmu->nr_caps)); + if (ret < 0) + return ret; + + list_for_each_entry(caps, &pmu->caps, list) { + ret = do_write_string(ff, caps->name); + if (ret < 0) + return ret; + + ret = do_write_string(ff, caps->value); + if (ret < 0) + return ret; + } + + if (write_pmu) { + ret = do_write_string(ff, pmu->name); + if (ret < 0) + return ret; + } + + return ret; +} + +static int write_cpu_pmu_caps(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *cpu_pmu = perf_pmus__find_core_pmu(); + int ret; + + if (!cpu_pmu) + return -ENOENT; + + ret = perf_pmu__caps_parse(cpu_pmu); + if (ret < 0) + return ret; + + return __write_pmu_caps(ff, cpu_pmu, false); +} + +static int write_pmu_caps(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *pmu = NULL; + int nr_pmu = 0; + int ret; + + while ((pmu = perf_pmus__scan(pmu))) { + if (!strcmp(pmu->name, "cpu")) { + /* + * The "cpu" PMU is special and covered by + * HEADER_CPU_PMU_CAPS. Note, core PMUs are + * counted/written here for ARM, s390 and Intel hybrid. + */ + continue; + } + if (perf_pmu__caps_parse(pmu) <= 0) + continue; + nr_pmu++; + } + + ret = do_write(ff, &nr_pmu, sizeof(nr_pmu)); + if (ret < 0) + return ret; + + if (!nr_pmu) + return 0; + + /* + * Note older perf tools assume core PMUs come first, this is a property + * of perf_pmus__scan. + */ + pmu = NULL; + while ((pmu = perf_pmus__scan(pmu))) { + if (!strcmp(pmu->name, "cpu")) { + /* Skip as above. */ + continue; + } + if (perf_pmu__caps_parse(pmu) <= 0) + continue; + ret = __write_pmu_caps(ff, pmu, true); + if (ret < 0) + return ret; + } + return 0; +} + +struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains, u32 nr) +{ + char dname[DNAME_LEN], cpumask[MAX_NR_CPUS]; + struct domain_info *domain_info; + struct cpu_domain_map **cd_map; + char cpulist[MAX_NR_CPUS]; + char *line = NULL; + u32 cpu, domain; + u32 dcount = 0; + size_t len; + FILE *fp; + + fp = fopen("/proc/schedstat", "r"); + if (!fp) { + pr_err("Failed to open /proc/schedstat\n"); + return NULL; + } + + cd_map = zalloc(sizeof(*cd_map) * nr); + if (!cd_map) + goto out; + + while (getline(&line, &len, fp) > 0) { + int retval; + + if (strncmp(line, "version", 7) == 0) { + retval = sscanf(line, "version %d\n", schedstat_version); + if (retval != 1) + continue; + + } else if (strncmp(line, "cpu", 3) == 0) { + retval = sscanf(line, "cpu%u %*s", &cpu); + if (retval == 1) { + cd_map[cpu] = zalloc(sizeof(*cd_map[cpu])); + if (!cd_map[cpu]) + goto out_free_line; + cd_map[cpu]->cpu = cpu; + } else + continue; + + dcount = 0; + } else if (strncmp(line, "domain", 6) == 0) { + struct domain_info **temp_domains; + + dcount++; + temp_domains = realloc(cd_map[cpu]->domains, dcount * sizeof(domain_info)); + if (!temp_domains) + goto out_free_line; + else + cd_map[cpu]->domains = temp_domains; + + domain_info = zalloc(sizeof(*domain_info)); + if (!domain_info) + goto out_free_line; + + cd_map[cpu]->domains[dcount - 1] = domain_info; + + if (*schedstat_version >= 17) { + retval = sscanf(line, "domain%u %s %s %*s", &domain, dname, + cpumask); + if (retval != 3) + continue; + + domain_info->dname = strdup(dname); + if (!domain_info->dname) + goto out_free_line; + } else { + retval = sscanf(line, "domain%u %s %*s", &domain, cpumask); + if (retval != 2) + continue; + } + + domain_info->domain = domain; + if (domain > *max_sched_domains) + *max_sched_domains = domain; + + domain_info->cpumask = strdup(cpumask); + if (!domain_info->cpumask) + goto out_free_line; + + cpumask_to_cpulist(cpumask, cpulist); + domain_info->cpulist = strdup(cpulist); + if (!domain_info->cpulist) + goto out_free_line; + + cd_map[cpu]->nr_domains = dcount; + } + } + +out_free_line: + free(line); +out: + fclose(fp); + return cd_map; +} + +static int write_cpu_domain_info(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + u32 max_sched_domains = 0, schedstat_version = 0; + struct cpu_domain_map **cd_map; + u32 i, j, nr, ret; + + nr = cpu__max_present_cpu().cpu; + + cd_map = build_cpu_domain_map(&schedstat_version, &max_sched_domains, nr); + if (!cd_map) + return -1; + + ret = do_write(ff, &schedstat_version, sizeof(u32)); + if (ret < 0) + goto out; + + max_sched_domains += 1; + ret = do_write(ff, &max_sched_domains, sizeof(u32)); + if (ret < 0) + goto out; + + for (i = 0; i < nr; i++) { + if (!cd_map[i]) + continue; + + ret = do_write(ff, &cd_map[i]->cpu, sizeof(u32)); + if (ret < 0) + goto out; + + ret = do_write(ff, &cd_map[i]->nr_domains, sizeof(u32)); + if (ret < 0) + goto out; + + for (j = 0; j < cd_map[i]->nr_domains; j++) { + ret = do_write(ff, &cd_map[i]->domains[j]->domain, sizeof(u32)); + if (ret < 0) + goto out; + if (schedstat_version >= 17) { + ret = do_write_string(ff, cd_map[i]->domains[j]->dname); + if (ret < 0) + goto out; + } + + ret = do_write_string(ff, cd_map[i]->domains[j]->cpumask); + if (ret < 0) + goto out; + + ret = do_write_string(ff, cd_map[i]->domains[j]->cpulist); + if (ret < 0) + goto out; + } + } + +out: + free_cpu_domain_info(cd_map, schedstat_version, nr); + return ret; +} + +static void print_hostname(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname); +} + +static void print_osrelease(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# os release : %s\n", ff->ph->env.os_release); +} + +static void print_arch(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# arch : %s\n", ff->ph->env.arch); +} + +static void print_e_machine(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# e_machine : %u\n", ff->ph->env.e_machine); + fprintf(fp, "# e_flags : %u\n", ff->ph->env.e_flags); +} + +static void print_cpudesc(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# cpudesc : %s\n", ff->ph->env.cpu_desc); +} + +static void print_nrcpus(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# nrcpus online : %u\n", ff->ph->env.nr_cpus_online); + fprintf(fp, "# nrcpus avail : %u\n", ff->ph->env.nr_cpus_avail); +} + +static void print_version(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# perf version : %s\n", ff->ph->env.version); +} + +static void print_cmdline(struct feat_fd *ff, FILE *fp) +{ + int nr, i; + + nr = ff->ph->env.nr_cmdline; + + fprintf(fp, "# cmdline : "); + + for (i = 0; i < nr; i++) { + char *argv_i = strdup(ff->ph->env.cmdline_argv[i]); + if (!argv_i) { + fprintf(fp, "%s ", ff->ph->env.cmdline_argv[i]); + } else { + char *mem = argv_i; + do { + char *quote = strchr(argv_i, '\''); + if (!quote) + break; + *quote++ = '\0'; + fprintf(fp, "%s\\\'", argv_i); + argv_i = quote; + } while (1); + fprintf(fp, "%s ", argv_i); + free(mem); + } + } + fputc('\n', fp); +} + +static void print_cpu_topology(struct feat_fd *ff, FILE *fp) +{ + struct perf_header *ph = ff->ph; + int cpu_nr = ph->env.nr_cpus_avail; + int nr, i; + char *str; + + nr = ph->env.nr_sibling_cores; + str = ph->env.sibling_cores; + + for (i = 0; i < nr; i++) { + fprintf(fp, "# sibling sockets : %s\n", str); + str += strlen(str) + 1; + } + + if (ph->env.nr_sibling_dies) { + nr = ph->env.nr_sibling_dies; + str = ph->env.sibling_dies; + + for (i = 0; i < nr; i++) { + fprintf(fp, "# sibling dies : %s\n", str); + str += strlen(str) + 1; + } + } + + nr = ph->env.nr_sibling_threads; + str = ph->env.sibling_threads; + + for (i = 0; i < nr; i++) { + fprintf(fp, "# sibling threads : %s\n", str); + str += strlen(str) + 1; + } + + if (ph->env.nr_sibling_dies) { + if (ph->env.cpu != NULL) { + for (i = 0; i < cpu_nr; i++) + fprintf(fp, "# CPU %d: Core ID %d, " + "Die ID %d, Socket ID %d\n", + i, ph->env.cpu[i].core_id, + ph->env.cpu[i].die_id, + ph->env.cpu[i].socket_id); + } else + fprintf(fp, "# Core ID, Die ID and Socket ID " + "information is not available\n"); + } else { + if (ph->env.cpu != NULL) { + for (i = 0; i < cpu_nr; i++) + fprintf(fp, "# CPU %d: Core ID %d, " + "Socket ID %d\n", + i, ph->env.cpu[i].core_id, + ph->env.cpu[i].socket_id); + } else + fprintf(fp, "# Core ID and Socket ID " + "information is not available\n"); + } +} + +static void print_clockid(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# clockid frequency: %"PRIu64" MHz\n", + ff->ph->env.clock.clockid_res_ns * 1000); +} + +static void print_clock_data(struct feat_fd *ff, FILE *fp) +{ + struct timespec clockid_ns; + char tstr[64], date[64]; + struct timeval tod_ns; + clockid_t clockid; + struct tm ltime; + u64 ref; + + if (!ff->ph->env.clock.enabled) { + fprintf(fp, "# reference time disabled\n"); + return; + } + + /* Compute TOD time. */ + ref = ff->ph->env.clock.tod_ns; + tod_ns.tv_sec = ref / NSEC_PER_SEC; + ref -= tod_ns.tv_sec * NSEC_PER_SEC; + tod_ns.tv_usec = ref / NSEC_PER_USEC; + + /* Compute clockid time. */ + ref = ff->ph->env.clock.clockid_ns; + clockid_ns.tv_sec = ref / NSEC_PER_SEC; + ref -= clockid_ns.tv_sec * NSEC_PER_SEC; + clockid_ns.tv_nsec = ref; + + clockid = ff->ph->env.clock.clockid; + + if (localtime_r(&tod_ns.tv_sec, <ime) == NULL) + snprintf(tstr, sizeof(tstr), ""); + else { + strftime(date, sizeof(date), "%F %T", <ime); + scnprintf(tstr, sizeof(tstr), "%s.%06d", + date, (int) tod_ns.tv_usec); + } + + fprintf(fp, "# clockid: %s (%u)\n", clockid_name(clockid), clockid); + fprintf(fp, "# reference time: %s = %ld.%06d (TOD) = %ld.%09ld (%s)\n", + tstr, (long) tod_ns.tv_sec, (int) tod_ns.tv_usec, + (long) clockid_ns.tv_sec, clockid_ns.tv_nsec, + clockid_name(clockid)); +} + +static void print_hybrid_topology(struct feat_fd *ff, FILE *fp) +{ + int i; + struct hybrid_node *n; + + fprintf(fp, "# hybrid cpu system:\n"); + for (i = 0; i < ff->ph->env.nr_hybrid_nodes; i++) { + n = &ff->ph->env.hybrid_nodes[i]; + fprintf(fp, "# %s cpu list : %s\n", n->pmu_name, n->cpus); + } +} + +static void print_dir_format(struct feat_fd *ff, FILE *fp) +{ + struct perf_session *session; + struct perf_data *data; + + session = container_of(ff->ph, struct perf_session, header); + data = session->data; + + fprintf(fp, "# directory data version : %"PRIu64"\n", data->dir.version); +} + +static void print_bpf_prog_info(struct feat_fd *ff __maybe_unused, FILE *fp) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + + down_read(&env->bpf_progs.lock); + + root = &env->bpf_progs.infos; + next = rb_first(root); + + if (!next) + fprintf(fp, "# bpf_prog_info empty\n"); + + while (next) { + struct bpf_prog_info_node *node; + + node = rb_entry(next, struct bpf_prog_info_node, rb_node); + next = rb_next(&node->rb_node); + + __bpf_event__print_bpf_prog_info(node->info_linear, env, fp); + } + + up_read(&env->bpf_progs.lock); +#else + fprintf(fp, "# bpf_prog_info missing, no libbpf support\n"); +#endif // HAVE_LIBBPF_SUPPORT +} + +static void print_bpf_btf(struct feat_fd *ff __maybe_unused, FILE *fp) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + + down_read(&env->bpf_progs.lock); + + root = &env->bpf_progs.btfs; + next = rb_first(root); + + if (!next) + printf("# btf info empty\n"); + + while (next) { + struct btf_node *node; + + node = rb_entry(next, struct btf_node, rb_node); + next = rb_next(&node->rb_node); + fprintf(fp, "# btf info of id %u\n", node->id); + } + + up_read(&env->bpf_progs.lock); +#else + fprintf(fp, "# bpf btf data missing, no libbpf support\n"); +#endif // HAVE_LIBBPF_SUPPORT +} + +static void free_event_desc(struct evsel *events) +{ + struct evsel *evsel; + + if (!events) + return; + + for (evsel = events; evsel->core.attr.size; evsel++) { + zfree(&evsel->name); + zfree(&evsel->core.id); + } + + free(events); +} + +static bool perf_attr_check(struct perf_event_attr *attr) +{ + if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) { + pr_warning("Reserved bits are set unexpectedly. " + "Please update perf tool.\n"); + return false; + } + + if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) { + pr_warning("Unknown sample type (0x%llx) is detected. " + "Please update perf tool.\n", + attr->sample_type); + return false; + } + + if (attr->read_format & ~(PERF_FORMAT_MAX-1)) { + pr_warning("Unknown read format (0x%llx) is detected. " + "Please update perf tool.\n", + attr->read_format); + return false; + } + + if ((attr->sample_type & PERF_SAMPLE_BRANCH_STACK) && + (attr->branch_sample_type & ~(PERF_SAMPLE_BRANCH_MAX-1))) { + pr_warning("Unknown branch sample type (0x%llx) is detected. " + "Please update perf tool.\n", + attr->branch_sample_type); + + return false; + } + + return true; +} + +static struct evsel *read_event_desc(struct feat_fd *ff) +{ + struct evsel *evsel, *events = NULL; + u64 *id; + void *buf = NULL; + u32 nre, sz, nr, i, j; + size_t msz; + + /* number of events */ + if (do_read_u32(ff, &nre)) + goto error; + + /* Size of each of the nre attributes. */ + if (do_read_u32(ff, &sz)) + goto error; + + /* + * Require at least one event with an attr no smaller than the + * first published struct, and reject sz values where + * sz + sizeof(u32) would overflow size_t (possible on 32-bit) + * or nre == UINT32_MAX where nre + 1 wraps to 0 in the calloc. + * + * The minimum section footprint per event is sz bytes for the + * attr plus a u32 for the id count, check that nre events fit. + */ + if (!nre || sz < PERF_ATTR_SIZE_VER0 || + sz > ff->size || (size_t)sz > SIZE_MAX - sizeof(u32) || + nre == UINT32_MAX || + nre > (ff->size - ff->offset) / (sz + sizeof(u32))) { + pr_err("Invalid HEADER_EVENT_DESC: nre=%u sz=%u (min %d)\n", + nre, sz, PERF_ATTR_SIZE_VER0); + goto error; + } + + /* buffer to hold on file attr struct */ + buf = malloc(sz); + if (!buf) + goto error; + + /* the last event terminates with evsel->core.attr.size == 0: */ + events = calloc(nre + 1, sizeof(*events)); + if (!events) + goto error; + + msz = sizeof(evsel->core.attr); + if (sz < msz) + msz = sz; + + for (i = 0, evsel = events; i < nre; evsel++, i++) { + struct perf_event_attr *attr = buf; + u32 attr_size; + + evsel->core.idx = i; + + /* + * must read entire on-file attr struct to + * sync up with layout. + */ + if (__do_read(ff, buf, sz)) + goto error; + + /* Reject before attr_swap to prevent OOB via bswap_safe() */ + attr_size = ff->ph->needs_swap ? bswap_32(attr->size) : attr->size; + /* ABI0: size == 0 means the producer didn't set it */ + if (!attr_size) { + attr_size = PERF_ATTR_SIZE_VER0; + /* + * Write back so free_event_desc() doesn't + * treat this event as the end-of-array sentinel + * (it iterates while attr.size != 0). + * + * Only for native — the swap path must NOT + * write native-endian VER0 here because + * perf_event__attr_swap() would re-swap it + * to 0x40000000, defeating bswap_safe() bounds. + * perf_event__attr_swap() has its own ABI0 + * fallback that sets VER0 after swapping. + */ + if (!ff->ph->needs_swap) + attr->size = attr_size; + } + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size > sz) { + pr_err("Event %d attr.size (%u) invalid (min: %d, max: %u)\n", + i, attr_size, PERF_ATTR_SIZE_VER0, sz); + goto error; + } + + if (ff->ph->needs_swap) + perf_event__attr_swap(buf); + + memcpy(&evsel->core.attr, buf, msz); + + if (!perf_attr_check(&evsel->core.attr)) + goto error; + + if (do_read_u32(ff, &nr)) + goto error; + + if (ff->ph->needs_swap) + evsel->needs_swap = true; + + evsel->name = do_read_string(ff); + if (!evsel->name) + goto error; + + if (!nr) + continue; + + /* Prevent oversized allocation from crafted nr */ + if (nr > (ff->size - ff->offset) / sizeof(*id)) { + pr_err("Event %d: id count %u exceeds remaining section\n", i, nr); + goto error; + } + + id = calloc(nr, sizeof(*id)); + if (!id) + goto error; + evsel->core.ids = nr; + evsel->core.id = id; + + for (j = 0 ; j < nr; j++) { + if (do_read_u64(ff, id)) + goto error; + id++; + } + } +out: + free(buf); + return events; +error: + free_event_desc(events); + events = NULL; + goto out; +} + +static int __desc_attr__fprintf(FILE *fp, const char *name, const char *val, + void *priv __maybe_unused) +{ + return fprintf(fp, ", %s = %s", name, val); +} + +static void print_event_desc(struct feat_fd *ff, FILE *fp) +{ + struct evsel *evsel, *events; + u32 j; + u64 *id; + + if (ff->events) + events = ff->events; + else + events = read_event_desc(ff); + + if (!events) { + fprintf(fp, "# event desc: not available or unable to read\n"); + return; + } + + for (evsel = events; evsel->core.attr.size; evsel++) { + fprintf(fp, "# event : name = %s, ", evsel->name); + + if (evsel->core.ids) { + fprintf(fp, ", id = {"); + for (j = 0, id = evsel->core.id; j < evsel->core.ids; j++, id++) { + if (j) + fputc(',', fp); + fprintf(fp, " %"PRIu64, *id); + } + fprintf(fp, " }"); + } + + perf_event_attr__fprintf(fp, &evsel->core.attr, __desc_attr__fprintf, NULL); + + fputc('\n', fp); + } + + free_event_desc(events); + ff->events = NULL; +} + +static void print_total_mem(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# total memory : %llu kB\n", ff->ph->env.total_mem); +} + +static void print_numa_topology(struct feat_fd *ff, FILE *fp) +{ + int i; + struct numa_node *n; + + for (i = 0; i < ff->ph->env.nr_numa_nodes; i++) { + n = &ff->ph->env.numa_nodes[i]; + + fprintf(fp, "# node%u meminfo : total = %"PRIu64" kB," + " free = %"PRIu64" kB\n", + n->node, n->mem_total, n->mem_free); + + fprintf(fp, "# node%u cpu list : ", n->node); + cpu_map__fprintf(n->map, fp); + } +} + +static void print_cpuid(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# cpuid : %s\n", ff->ph->env.cpuid); +} + +static void print_branch_stack(struct feat_fd *ff __maybe_unused, FILE *fp) +{ + fprintf(fp, "# contains samples with branch stack\n"); +} + +static void print_auxtrace(struct feat_fd *ff __maybe_unused, FILE *fp) +{ + fprintf(fp, "# contains AUX area data (e.g. instruction trace)\n"); +} + +static void print_stat(struct feat_fd *ff __maybe_unused, FILE *fp) +{ + fprintf(fp, "# contains stat data\n"); +} + +static void print_cache(struct feat_fd *ff, FILE *fp __maybe_unused) +{ + int i; + + fprintf(fp, "# CPU cache info:\n"); + for (i = 0; i < ff->ph->env.caches_cnt; i++) { + fprintf(fp, "# "); + cpu_cache_level__fprintf(fp, &ff->ph->env.caches[i]); + } +} + +static void print_cln_size(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# cacheline size: %u\n", ff->ph->env.cln_size); +} + +static void print_compressed(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# compressed : %s, level = %d, ratio = %d\n", + ff->ph->env.comp_type == PERF_COMP_ZSTD ? "Zstd" : "Unknown", + ff->ph->env.comp_level, ff->ph->env.comp_ratio); +} + +static void __print_pmu_caps(FILE *fp, int nr_caps, char **caps, char *pmu_name) +{ + const char *delimiter = ""; + int i; + + if (!nr_caps) { + fprintf(fp, "# %s pmu capabilities: not available\n", pmu_name); + return; + } + + fprintf(fp, "# %s pmu capabilities: ", pmu_name); + for (i = 0; i < nr_caps; i++) { + fprintf(fp, "%s%s", delimiter, caps[i]); + delimiter = ", "; + } + + fprintf(fp, "\n"); +} + +static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) +{ + __print_pmu_caps(fp, ff->ph->env.nr_cpu_pmu_caps, + ff->ph->env.cpu_pmu_caps, (char *)"cpu"); +} + +static void print_pmu_caps(struct feat_fd *ff, FILE *fp) +{ + struct perf_env *env = &ff->ph->env; + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + + for (int i = 0; i < env->nr_pmus_with_caps; i++) { + struct pmu_caps *pmu_caps = &env->pmu_caps[i]; + + __print_pmu_caps(fp, pmu_caps->nr_caps, pmu_caps->caps, + pmu_caps->pmu_name); + } + + if ((e_machine == EM_X86_64 || e_machine == EM_386) && + perf_env__has_pmu_mapping(env, "ibs_op")) { + char *max_precise = perf_env__find_pmu_cap(env, "cpu", "max_precise"); + + if (max_precise != NULL && atoi(max_precise) == 0) + fprintf(fp, "# AMD systems uses ibs_op// PMU for some precise events, e.g.: cycles:p, see the 'perf list' man page for further details.\n"); + } +} + +static void print_pmu_mappings(struct feat_fd *ff, FILE *fp) +{ + struct perf_env *env = &ff->ph->env; + const char *delimiter = "# pmu mappings: "; + char *str, *tmp; + u32 pmu_num; + u32 type; + + pmu_num = env->nr_pmu_mappings; + if (!pmu_num) { + fprintf(fp, "# pmu mappings: not available\n"); + return; + } + + str = env->pmu_mappings; + + while (pmu_num) { + type = strtoul(str, &tmp, 0); + if (*tmp != ':') + goto error; + + str = tmp + 1; + fprintf(fp, "%s%s = %" PRIu32, delimiter, str, type); + + delimiter = ", "; + str += strlen(str) + 1; + pmu_num--; + } + + fprintf(fp, "\n"); + + if (!pmu_num) + return; +error: + fprintf(fp, "# pmu mappings: unable to read\n"); +} + +static void print_group_desc(struct feat_fd *ff, FILE *fp) +{ + struct perf_session *session; + struct evsel *evsel; + u32 nr = 0; + + session = container_of(ff->ph, struct perf_session, header); + + evlist__for_each_entry(session->evlist, evsel) { + if (evsel__is_group_leader(evsel) && evsel->core.nr_members > 1) { + fprintf(fp, "# group: %s{%s", evsel->group_name ?: "", evsel__name(evsel)); + + nr = evsel->core.nr_members - 1; + } else if (nr) { + fprintf(fp, ",%s", evsel__name(evsel)); + + if (--nr == 0) + fprintf(fp, "}\n"); + } + } +} + +static void print_sample_time(struct feat_fd *ff, FILE *fp) +{ + struct perf_session *session; + char time_buf[32]; + double d; + + session = container_of(ff->ph, struct perf_session, header); + + timestamp__scnprintf_usec(evlist__first_sample_time(session->evlist), + time_buf, sizeof(time_buf)); + fprintf(fp, "# time of first sample : %s\n", time_buf); + + timestamp__scnprintf_usec(evlist__last_sample_time(session->evlist), + time_buf, sizeof(time_buf)); + fprintf(fp, "# time of last sample : %s\n", time_buf); + + d = (double)(evlist__last_sample_time(session->evlist) - + evlist__first_sample_time(session->evlist)) / NSEC_PER_MSEC; + + fprintf(fp, "# sample duration : %10.3f ms\n", d); +} + +static void memory_node__fprintf(struct memory_node *n, + unsigned long long bsize, FILE *fp) +{ + char buf_map[100], buf_size[50]; + unsigned long long size; + + size = bsize * bitmap_weight(n->set, n->size); + unit_number__scnprintf(buf_size, 50, size); + + bitmap_scnprintf(n->set, n->size, buf_map, 100); + fprintf(fp, "# %3" PRIu64 " [%s]: %s\n", n->node, buf_size, buf_map); +} + +static void print_mem_topology(struct feat_fd *ff, FILE *fp) +{ + struct perf_env *env = &ff->ph->env; + struct memory_node *nodes; + int i, nr; + + nodes = env->memory_nodes; + nr = env->nr_memory_nodes; + + fprintf(fp, "# memory nodes (nr %d, block size 0x%llx):\n", + nr, env->memory_bsize); + + for (i = 0; i < nr; i++) { + memory_node__fprintf(&nodes[i], env->memory_bsize, fp); + } +} + +static void print_cpu_domain_info(struct feat_fd *ff, FILE *fp) +{ + struct cpu_domain_map **cd_map = ff->ph->env.cpu_domain; + u32 nr = ff->ph->env.nr_cpus_avail; + struct domain_info *d_info; + u32 i, j; + + fprintf(fp, "# schedstat version : %u\n", ff->ph->env.schedstat_version); + fprintf(fp, "# Maximum sched domains : %u\n", ff->ph->env.max_sched_domains); + + for (i = 0; i < nr; i++) { + if (!cd_map[i]) + continue; + + fprintf(fp, "# cpu : %u\n", cd_map[i]->cpu); + fprintf(fp, "# nr_domains : %u\n", cd_map[i]->nr_domains); + + for (j = 0; j < cd_map[i]->nr_domains; j++) { + d_info = cd_map[i]->domains[j]; + if (!d_info) + continue; + + fprintf(fp, "# Domain : %u\n", d_info->domain); + + if (ff->ph->env.schedstat_version >= 17) + fprintf(fp, "# Domain name : %s\n", d_info->dname); + + fprintf(fp, "# Domain cpu map : %s\n", d_info->cpumask); + fprintf(fp, "# Domain cpu list : %s\n", d_info->cpulist); + } + } +} + +static int __event_process_build_id(struct perf_record_header_build_id *bev, + char *filename, + struct perf_session *session) +{ + int err = -1; + struct machine *machine; + u16 cpumode; + struct dso *dso; + enum dso_space_type dso_space; + + machine = perf_session__findnew_machine(session, bev->pid); + if (!machine) + goto out; + + cpumode = bev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + switch (cpumode) { + case PERF_RECORD_MISC_KERNEL: + dso_space = DSO_SPACE__KERNEL; + break; + case PERF_RECORD_MISC_GUEST_KERNEL: + dso_space = DSO_SPACE__KERNEL_GUEST; + break; + case PERF_RECORD_MISC_USER: + case PERF_RECORD_MISC_GUEST_USER: + dso_space = DSO_SPACE__USER; + break; + default: + goto out; + } + + dso = machine__findnew_dso(machine, filename); + if (dso != NULL) { + char sbuild_id[SBUILD_ID_SIZE]; + struct build_id bid; + size_t size = BUILD_ID_SIZE; + + if (bev->header.misc & PERF_RECORD_MISC_BUILD_ID_SIZE) + size = bev->size; + + build_id__init(&bid, bev->data, size); + dso__set_build_id(dso, &bid); + dso__set_header_build_id(dso, true); + + if (dso_space != DSO_SPACE__USER) { + struct kmod_path m = { .name = NULL, }; + + if (!kmod_path__parse_name(&m, filename) && m.kmod) + dso__set_module_info(dso, &m, machine); + + dso__set_kernel(dso, dso_space); + free(m.name); + } + + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + pr_debug("build id event received for %s: %s [%zu]\n", + dso__long_name(dso), sbuild_id, size); + dso__put(dso); + } + + err = 0; +out: + return err; +} + +static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, + int input, u64 offset, u64 size) +{ + struct perf_session *session = container_of(header, struct perf_session, header); + struct { + struct perf_event_header header; + u8 build_id[PERF_ALIGN(BUILD_ID_SIZE, sizeof(u64))]; + char filename[0]; + } old_bev; + struct perf_record_header_build_id bev; + char filename[PATH_MAX]; + u64 limit; + + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; + + while (offset < limit) { + ssize_t len; + + if (readn(input, &old_bev, sizeof(old_bev)) != sizeof(old_bev)) + return -1; + + if (header->needs_swap) + perf_event_header__bswap(&old_bev.header); + + /* size == 0 loops forever; size > remaining reads past section */ + if (old_bev.header.size == 0 || old_bev.header.size > limit - offset) + return -1; + + len = old_bev.header.size - sizeof(old_bev); + if (len < 0 || len >= PATH_MAX) { + pr_warning("invalid build_id filename length %zd\n", len); + return -1; + } + + if (readn(input, filename, len) != len) + return -1; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; + + bev.header = old_bev.header; + + /* + * As the pid is the missing value, we need to fill + * it properly. The header.misc value give us nice hint. + */ + bev.pid = HOST_KERNEL_ID; + if (bev.header.misc == PERF_RECORD_MISC_GUEST_USER || + bev.header.misc == PERF_RECORD_MISC_GUEST_KERNEL) + bev.pid = DEFAULT_GUEST_KERNEL_ID; + + memcpy(bev.build_id, old_bev.build_id, sizeof(bev.build_id)); + __event_process_build_id(&bev, filename, session); + + offset += bev.header.size; + } + + return 0; +} + +static int perf_header__read_build_ids(struct perf_header *header, + int input, u64 offset, u64 size) +{ + struct perf_session *session = container_of(header, struct perf_session, header); + struct perf_record_header_build_id bev; + char filename[PATH_MAX]; + u64 limit, orig_offset = offset; + int err = -1; + + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; + + while (offset < limit) { + ssize_t len; + + if (readn(input, &bev, sizeof(bev)) != sizeof(bev)) + goto out; + + if (header->needs_swap) { + perf_event_header__bswap(&bev.header); + bev.pid = bswap_32(bev.pid); + } + + /* + * size == 0 would loop forever (offset never advances); + * size > remaining would read past the section boundary. + */ + if (bev.header.size == 0 || bev.header.size > limit - offset) + goto out; + + len = bev.header.size - sizeof(bev); + if (len < 0 || len >= PATH_MAX) { + pr_warning("invalid build_id filename length %zd\n", len); + goto out; + } + + if (readn(input, filename, len) != len) + goto out; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; + /* + * The a1645ce1 changeset: + * + * "perf: 'perf kvm' tool for monitoring guest performance from host" + * + * Added a field to struct perf_record_header_build_id that broke the file + * format. + * + * Since the kernel build-id is the first entry, process the + * table using the old format if the well known + * '[kernel.kallsyms]' string for the kernel build-id has the + * first 4 characters chopped off (where the pid_t sits). + */ + /* Guard short filenames against memcmp reading past the buffer */ + if (len >= (ssize_t)sizeof("nel.kallsyms]") - 1 && + memcmp(filename, "nel.kallsyms]", sizeof("nel.kallsyms]") - 1) == 0) { + if (lseek(input, orig_offset, SEEK_SET) == (off_t)-1) + return -1; + return perf_header__read_build_ids_abi_quirk(header, input, offset, size); + } + + __event_process_build_id(&bev, filename, session); + + offset += bev.header.size; + } + err = 0; +out: + return err; +} + +/* Macro for features that simply need to read and store a string. */ +#define FEAT_PROCESS_STR_FUN(__feat, __feat_env) \ +static int process_##__feat(struct feat_fd *ff, void *data __maybe_unused) \ +{\ + free(ff->ph->env.__feat_env); \ + ff->ph->env.__feat_env = do_read_string(ff); \ + return ff->ph->env.__feat_env ? 0 : -ENOMEM; \ +} + +FEAT_PROCESS_STR_FUN(hostname, hostname); +FEAT_PROCESS_STR_FUN(osrelease, os_release); +FEAT_PROCESS_STR_FUN(version, version); +FEAT_PROCESS_STR_FUN(cpudesc, cpu_desc); +FEAT_PROCESS_STR_FUN(cpuid, cpuid); + +static int process_arch(struct feat_fd *ff, void *data __maybe_unused) +{ + free(ff->ph->env.arch); + ff->ph->env.arch = do_read_string(ff); + if (!ff->ph->env.arch) + return -ENOMEM; + return 0; +} + +static int process_e_machine(struct feat_fd *ff, void *data __maybe_unused) +{ + int ret; + + ret = do_read_u32(ff, &ff->ph->env.e_machine); + if (ret) + return ret; + + return do_read_u32(ff, &ff->ph->env.e_flags); +} + +static int process_tracing_data(struct feat_fd *ff __maybe_unused, void *data __maybe_unused) +{ +#ifdef HAVE_LIBTRACEEVENT + ssize_t ret = trace_report(ff->fd, data, false); + + return ret < 0 ? -1 : 0; +#else + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("Tracing data present but libtraceevent not available, skipping.\n"); + return 0; +#endif +} + +static int process_build_id(struct feat_fd *ff, void *data __maybe_unused) +{ + /* lseek fails in pipe mode — fall back to ff->offset */ + off_t offset = lseek(ff->fd, 0, SEEK_CUR); + + if (offset == (off_t)-1) + offset = ff->offset; + + if (perf_header__read_build_ids(ff->ph, ff->fd, offset, ff->size)) + pr_debug("Failed to read buildids, continuing...\n"); + return 0; +} + +static int process_nrcpus(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + int ret; + u32 nr_cpus_avail, nr_cpus_online; + + ret = do_read_u32(ff, &nr_cpus_avail); + if (ret) + return ret; + + ret = do_read_u32(ff, &nr_cpus_online); + if (ret) + return ret; + + /* + * Cap at 1M CPUs — generous for any real system but prevents + * stack overflow from VLA allocations sized by nr_cpus_avail + * (e.g. DECLARE_BITMAP in builtin-c2c.c node_entry()). + */ + if (nr_cpus_avail > (1U << 20)) { + pr_err("Invalid HEADER_NRCPUS: nr_cpus_avail (%u) exceeds maximum (%u)\n", + nr_cpus_avail, 1U << 20); + return -1; + } + + if (nr_cpus_online > nr_cpus_avail) { + pr_err("Invalid HEADER_NRCPUS: nr_cpus_online (%u) > nr_cpus_avail (%u)\n", + nr_cpus_online, nr_cpus_avail); + return -1; + } + + env->nr_cpus_avail = (int)nr_cpus_avail; + env->nr_cpus_online = (int)nr_cpus_online; + return 0; +} + +static int process_total_mem(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + u64 total_mem; + int ret; + + ret = do_read_u64(ff, &total_mem); + if (ret) + return -1; + env->total_mem = (unsigned long long)total_mem; + return 0; +} + +static struct evsel *evlist__find_by_index(struct evlist *evlist, int idx) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.idx == idx) + return evsel; + } + + return NULL; +} + +static void evlist__set_event_name(struct evlist *evlist, struct evsel *event) +{ + struct evsel *evsel; + + if (!event->name) + return; + + evsel = evlist__find_by_index(evlist, event->core.idx); + if (!evsel) + return; + + if (evsel->name) + return; + + evsel->name = strdup(event->name); +} + +static int +process_event_desc(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_session *session; + struct evsel *evsel, *events = read_event_desc(ff); + + if (!events) + return 0; + + session = container_of(ff->ph, struct perf_session, header); + + if (session->data->is_pipe) { + /* Save events for reading later by print_event_desc, + * since they can't be read again in pipe mode. */ + ff->events = events; + } + + for (evsel = events; evsel->core.attr.size; evsel++) + evlist__set_event_name(session->evlist, evsel); + + if (!session->data->is_pipe) + free_event_desc(events); + + return 0; +} + +/* + * Some arbitrary max for the number of command line arguments, + * Wildcards can expand and end up with tons of command line args. + */ +#define MAX_CMDLINE_NR 1048576 + +static int process_cmdline(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + char *str, *cmdline = NULL, **argv = NULL; + u32 nr, i, len = 0; + + if (do_read_u32(ff, &nr)) + return -1; + + if (nr > MAX_CMDLINE_NR) + return -1; + + env->nr_cmdline = nr; + + cmdline = zalloc(ff->size + nr + 1); + if (!cmdline) + return -1; + + argv = calloc(nr + 1, sizeof(char *)); + if (!argv) + goto error; + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + argv[i] = cmdline + len; + memcpy(argv[i], str, strlen(str) + 1); + len += strlen(str) + 1; + free(str); + } + env->cmdline = cmdline; + env->cmdline_argv = (const char **) argv; + return 0; + +error: + free(argv); + free(cmdline); + return -1; +} + +static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) +{ + u32 nr, i; + char *str = NULL; + struct strbuf sb; + struct perf_env *env = &ff->ph->env; + int cpu_nr = env->nr_cpus_avail; + u64 size = 0; + + if (cpu_nr == 0) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: missing HEADER_NRCPUS\n"); + return -1; + } + + env->cpu = calloc(cpu_nr, sizeof(*env->cpu)); + if (!env->cpu) + return -1; + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + if (nr > (u32)cpu_nr) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: nr_sibling_cores (%u) > nr_cpus_avail (%d)\n", + nr, cpu_nr); + goto free_cpu; + } + + env->nr_sibling_cores = nr; + size += sizeof(u32); + if (strbuf_init(&sb, 128) < 0) + goto free_cpu; + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + /* include a NULL character at the end */ + if (strbuf_add(&sb, str, strlen(str) + 1) < 0) + goto error; + size += string_size(str); + zfree(&str); + } + env->sibling_cores = strbuf_detach(&sb, NULL); + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + if (nr > (u32)cpu_nr) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: nr_sibling_threads (%u) > nr_cpus_avail (%d)\n", + nr, cpu_nr); + goto free_cpu; + } + + env->nr_sibling_threads = nr; + size += sizeof(u32); + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + /* include a NULL character at the end */ + if (strbuf_add(&sb, str, strlen(str) + 1) < 0) + goto error; + size += string_size(str); + zfree(&str); + } + env->sibling_threads = strbuf_detach(&sb, NULL); + + /* + * The header may be from old perf, + * which doesn't include core id and socket id information. + */ + if (ff->size <= size) { + zfree(&env->cpu); + return 0; + } + + for (i = 0; i < (u32)cpu_nr; i++) { + if (do_read_u32(ff, &nr)) + goto free_cpu; + + env->cpu[i].core_id = nr; + size += sizeof(u32); + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + env->cpu[i].socket_id = nr; + size += sizeof(u32); + } + + /* + * The header may be from old perf, + * which doesn't include die information. + */ + if (ff->size <= size) + return 0; + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + if (nr > (u32)cpu_nr) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: nr_sibling_dies (%u) > nr_cpus_avail (%d)\n", + nr, cpu_nr); + goto free_cpu; + } + + env->nr_sibling_dies = nr; + size += sizeof(u32); + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + /* include a NULL character at the end */ + if (strbuf_add(&sb, str, strlen(str) + 1) < 0) + goto error; + size += string_size(str); + zfree(&str); + } + env->sibling_dies = strbuf_detach(&sb, NULL); + + for (i = 0; i < (u32)cpu_nr; i++) { + if (do_read_u32(ff, &nr)) + goto free_cpu; + + env->cpu[i].die_id = nr; + } + + return 0; + +error: + strbuf_release(&sb); + zfree(&str); +free_cpu: + zfree(&env->cpu); + return -1; +} + +static int process_numa_topology(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct numa_node *nodes, *n; + u32 nr, i; + char *str; + + /* nr nodes */ + if (do_read_u32(ff, &nr)) + return -1; + + if (nr > MAX_NUMA_NODES) { + pr_err("Invalid HEADER_NUMA_TOPOLOGY: nr_nodes (%u) > %u\n", + nr, MAX_NUMA_NODES); + return -1; + } + + if (ff->size < sizeof(u32) + nr * (sizeof(u32) + 2 * sizeof(u64))) { + pr_err("Invalid HEADER_NUMA_TOPOLOGY: section too small (%zu) for %u nodes\n", + ff->size, nr); + return -1; + } + + nodes = calloc(nr, sizeof(*nodes)); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + n = &nodes[i]; + + /* node number */ + if (do_read_u32(ff, &n->node)) + goto error; + + if (do_read_u64(ff, &n->mem_total)) + goto error; + + if (do_read_u64(ff, &n->mem_free)) + goto error; + + str = do_read_string(ff); + if (!str) + goto error; + + n->map = perf_cpu_map__new(str); + free(str); + if (!n->map) + goto error; + } + env->nr_numa_nodes = nr; + env->numa_nodes = nodes; + return 0; + +error: + free(nodes); + return -1; +} + +static int process_pmu_mappings(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + char *name; + u32 pmu_num; + u32 type; + struct strbuf sb; + + if (do_read_u32(ff, &pmu_num)) + return -1; + + if (!pmu_num) { + pr_debug("pmu mappings not available\n"); + return 0; + } + + if (pmu_num > MAX_PMU_MAPPINGS) { + pr_err("Invalid HEADER_PMU_MAPPINGS: pmu_num (%u) > %u\n", + pmu_num, MAX_PMU_MAPPINGS); + return -1; + } + + if (ff->size < sizeof(u32) + pmu_num * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_PMU_MAPPINGS: section too small (%zu) for %u PMUs\n", + ff->size, pmu_num); + return -1; + } + + env->nr_pmu_mappings = pmu_num; + if (strbuf_init(&sb, 128) < 0) + return -1; + + while (pmu_num) { + if (do_read_u32(ff, &type)) + goto error; + + name = do_read_string(ff); + if (!name) + goto error; + + if (strbuf_addf(&sb, "%u:%s", type, name) < 0) + goto error; + /* include a NULL character at the end */ + if (strbuf_add(&sb, "", 1) < 0) + goto error; + + if (!strcmp(name, "msr")) + env->msr_pmu_type = type; + + free(name); + pmu_num--; + } + /* AMD may set it by evlist__has_amd_ibs() from perf_session__new() */ + free(env->pmu_mappings); + env->pmu_mappings = strbuf_detach(&sb, NULL); + return 0; + +error: + strbuf_release(&sb); + return -1; +} + +static int process_group_desc(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + size_t ret = -1; + u32 i, nr, nr_groups; + struct perf_session *session; + struct evsel *evsel, *leader = NULL; + struct group_desc { + char *name; + u32 leader_idx; + u32 nr_members; + } *desc; + + if (do_read_u32(ff, &nr_groups)) + return -1; + + if (!nr_groups) { + pr_debug("group desc not available\n"); + return 0; + } + + if (nr_groups > MAX_GROUP_DESC) { + pr_err("Invalid HEADER_GROUP_DESC: nr_groups (%u) > %u\n", + nr_groups, MAX_GROUP_DESC); + return -1; + } + + if (ff->size < sizeof(u32) + nr_groups * 3 * sizeof(u32)) { + pr_err("Invalid HEADER_GROUP_DESC: section too small (%zu) for %u groups\n", + ff->size, nr_groups); + return -1; + } + + env->nr_groups = nr_groups; + + desc = calloc(nr_groups, sizeof(*desc)); + if (!desc) + return -1; + + for (i = 0; i < nr_groups; i++) { + desc[i].name = do_read_string(ff); + if (!desc[i].name) + goto out_free; + + if (do_read_u32(ff, &desc[i].leader_idx)) + goto out_free; + + if (do_read_u32(ff, &desc[i].nr_members)) + goto out_free; + } + + /* + * Rebuild group relationship based on the group_desc + */ + session = container_of(ff->ph, struct perf_session, header); + + i = nr = 0; + evlist__for_each_entry(session->evlist, evsel) { + if (i < nr_groups && evsel->core.idx == (int) desc[i].leader_idx) { + evsel__set_leader(evsel, evsel); + /* {anon_group} is a dummy name */ + if (strcmp(desc[i].name, "{anon_group}")) { + evsel->group_name = desc[i].name; + desc[i].name = NULL; + } + evsel->core.nr_members = desc[i].nr_members; + + if (i >= nr_groups || nr > 0) { + pr_debug("invalid group desc\n"); + goto out_free; + } + + leader = evsel; + nr = evsel->core.nr_members - 1; + i++; + } else if (nr) { + /* This is a group member */ + evsel__set_leader(evsel, leader); + + nr--; + } + } + + if (i != nr_groups || nr != 0) { + pr_debug("invalid group desc\n"); + goto out_free; + } + + ret = 0; +out_free: + for (i = 0; i < nr_groups; i++) + zfree(&desc[i].name); + free(desc); + + return ret; +} + +static int process_auxtrace(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_session *session; + int err; + + session = container_of(ff->ph, struct perf_session, header); + + err = auxtrace_index__process(ff->fd, ff->size, session, + ff->ph->needs_swap); + if (err < 0) + pr_err("Failed to process auxtrace index\n"); + return err; +} + +static int process_cache(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct cpu_cache_level *caches; + u32 cnt, i, version; + + if (do_read_u32(ff, &version)) + return -1; + + if (version != 1) + return -1; + + if (do_read_u32(ff, &cnt)) + return -1; + + if (cnt > MAX_CACHE_ENTRIES) { + pr_err("Invalid HEADER_CACHE: cnt (%u) > %u\n", + cnt, MAX_CACHE_ENTRIES); + return -1; + } + + if (ff->size < 2 * sizeof(u32) + cnt * 7 * sizeof(u32)) { + pr_err("Invalid HEADER_CACHE: section too small (%zu) for %u entries\n", + ff->size, cnt); + return -1; + } + + caches = calloc(cnt, sizeof(*caches)); + if (!caches) + return -1; + + for (i = 0; i < cnt; i++) { + struct cpu_cache_level *c = &caches[i]; + + #define _R(v) \ + if (do_read_u32(ff, &c->v)) \ + goto out_free_caches; \ + + _R(level) + _R(line_size) + _R(sets) + _R(ways) + #undef _R + + #define _R(v) \ + c->v = do_read_string(ff); \ + if (!c->v) \ + goto out_free_caches; \ + + _R(type) + _R(size) + _R(map) + #undef _R + } + + env->caches = caches; + env->caches_cnt = cnt; + return 0; +out_free_caches: + for (i = 0; i < cnt; i++) { + free(caches[i].type); + free(caches[i].size); + free(caches[i].map); + } + free(caches); + return -1; +} + +static int process_cln_size(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + + if (do_read_u32(ff, &env->cln_size)) + return -1; + + return 0; +} + +static int process_sample_time(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_session *session; + u64 first_sample_time, last_sample_time; + int ret; + + session = container_of(ff->ph, struct perf_session, header); + + ret = do_read_u64(ff, &first_sample_time); + if (ret) + return -1; + + ret = do_read_u64(ff, &last_sample_time); + if (ret) + return -1; + + evlist__set_first_sample_time(session->evlist, first_sample_time); + evlist__set_last_sample_time(session->evlist, last_sample_time); + return 0; +} + +static int process_mem_topology(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct memory_node *nodes; + u64 version, i, nr, bsize; + int ret = -1; + + if (do_read_u64(ff, &version)) + return -1; + + if (version != 1) + return -1; + + if (do_read_u64(ff, &bsize)) + return -1; + + if (do_read_u64(ff, &nr)) + return -1; + + if (nr > MAX_NUMA_NODES) { + pr_err("Invalid HEADER_MEM_TOPOLOGY: nr_nodes (%llu) > %u\n", + (unsigned long long)nr, MAX_NUMA_NODES); + return -1; + } + + /* Per node: node_id(u64) + mem_size(u64) + bitmap_nr_bits(u64) */ + if (ff->size < 3 * sizeof(u64) + nr * 3 * sizeof(u64)) { + pr_err("Invalid HEADER_MEM_TOPOLOGY: section too small (%zu) for %llu nodes\n", + ff->size, (unsigned long long)nr); + return -1; + } + + nodes = calloc(nr, sizeof(*nodes)); + if (!nodes) + return -1; + + for (i = 0; i < nr; i++) { + struct memory_node n; + + #define _R(v) \ + if (do_read_u64(ff, &n.v)) \ + goto out; \ + + _R(node) + _R(size) + + #undef _R + + if (do_read_bitmap(ff, &n.set, &n.size)) + goto out; + + nodes[i] = n; + } + + env->memory_bsize = bsize; + env->memory_nodes = nodes; + env->nr_memory_nodes = nr; + ret = 0; + +out: + if (ret) + memory_node__delete_nodes(nodes, nr); + return ret; +} + +static int process_clockid(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + + if (do_read_u64(ff, &env->clock.clockid_res_ns)) + return -1; + + return 0; +} + +static int process_clock_data(struct feat_fd *ff, + void *_data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + u32 data32; + u64 data64; + + /* version */ + if (do_read_u32(ff, &data32)) + return -1; + + if (data32 != 1) + return -1; + + /* clockid */ + if (do_read_u32(ff, &data32)) + return -1; + + env->clock.clockid = data32; + + /* TOD ref time */ + if (do_read_u64(ff, &data64)) + return -1; + + env->clock.tod_ns = data64; + + /* clockid ref time */ + if (do_read_u64(ff, &data64)) + return -1; + + env->clock.clockid_ns = data64; + env->clock.enabled = true; + return 0; +} + +static int process_hybrid_topology(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct hybrid_node *nodes, *n; + u32 nr, i; + + /* nr nodes */ + if (do_read_u32(ff, &nr)) + return -1; + + if (nr > MAX_PMU_MAPPINGS) { + pr_err("Invalid HEADER_HYBRID_TOPOLOGY: nr_nodes (%u) > %u\n", + nr, MAX_PMU_MAPPINGS); + return -1; + } + + if (ff->size < sizeof(u32) + nr * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_HYBRID_TOPOLOGY: section too small (%zu) for %u nodes\n", + ff->size, nr); + return -1; + } + + nodes = calloc(nr, sizeof(*nodes)); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + n = &nodes[i]; + + n->pmu_name = do_read_string(ff); + if (!n->pmu_name) + goto error; + + n->cpus = do_read_string(ff); + if (!n->cpus) + goto error; + } + + env->nr_hybrid_nodes = nr; + env->hybrid_nodes = nodes; + return 0; + +error: + for (i = 0; i < nr; i++) { + free(nodes[i].pmu_name); + free(nodes[i].cpus); + } + + free(nodes); + return -1; +} + +static int process_dir_format(struct feat_fd *ff, + void *_data __maybe_unused) +{ + struct perf_session *session; + struct perf_data *data; + + session = container_of(ff->ph, struct perf_session, header); + data = session->data; + + if (WARN_ON(!perf_data__is_dir(data))) + return -1; + + return do_read_u64(ff, &data->dir.version); +} + +static int process_bpf_prog_info(struct feat_fd *ff __maybe_unused, void *data __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct bpf_prog_info_node *info_node; + struct perf_env *env = &ff->ph->env; + struct perf_bpil *info_linear; + u32 count, i; + int err = -1; + + if (ff->ph->needs_swap) { + pr_warning("interpreting bpf_prog_info from systems with endianness is not yet supported\n"); + return 0; + } + + if (do_read_u32(ff, &count)) + return -1; + + if (count > MAX_BPF_PROGS) { + pr_err("Invalid HEADER_BPF_PROG_INFO: count (%u) > %u\n", + count, MAX_BPF_PROGS); + return -1; + } + + if (ff->size < sizeof(u32) + count * (2 * sizeof(u32) + sizeof(u64))) { + pr_err("Invalid HEADER_BPF_PROG_INFO: section too small (%zu) for %u entries\n", + ff->size, count); + return -1; + } + + down_write(&env->bpf_progs.lock); + + for (i = 0; i < count; ++i) { + u32 info_len, data_len; + + info_linear = NULL; + info_node = NULL; + if (do_read_u32(ff, &info_len)) + goto out; + if (do_read_u32(ff, &data_len)) + goto out; + + if (info_len > sizeof(struct bpf_prog_info)) { + pr_warning("detected invalid bpf_prog_info\n"); + goto out; + } + + if (data_len > MAX_BPF_DATA_LEN) { + pr_warning("Invalid HEADER_BPF_PROG_INFO: data_len (%u) too large\n", + data_len); + goto out; + } + + info_linear = malloc(sizeof(struct perf_bpil) + + data_len); + if (!info_linear) + goto out; + info_linear->info_len = sizeof(struct bpf_prog_info); + info_linear->data_len = data_len; + if (do_read_u64(ff, (u64 *)(&info_linear->arrays))) + goto out; + if (__do_read(ff, &info_linear->info, info_len)) + goto out; + if (info_len < sizeof(struct bpf_prog_info)) + memset(((void *)(&info_linear->info)) + info_len, 0, + sizeof(struct bpf_prog_info) - info_len); + + if (__do_read(ff, info_linear->data, data_len)) + goto out; + + info_node = malloc(sizeof(struct bpf_prog_info_node)); + if (!info_node) + goto out; + + /* after reading from file, translate offset to address */ + bpil_offs_to_addr(info_linear); + info_node->info_linear = info_linear; + info_node->metadata = NULL; + if (!__perf_env__insert_bpf_prog_info(env, info_node)) { + free(info_linear); + free(info_node); + } + } + + up_write(&env->bpf_progs.lock); + return 0; +out: + free(info_linear); + free(info_node); + up_write(&env->bpf_progs.lock); + return err; +#else + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BPF prog info present but libbpf not available, skipping.\n"); + return 0; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int process_bpf_btf(struct feat_fd *ff __maybe_unused, void *data __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct btf_node *node = NULL; + u32 count, i; + int err = -1; + + if (ff->ph->needs_swap) { + pr_warning("interpreting btf from systems with endianness is not yet supported\n"); + return 0; + } + + if (do_read_u32(ff, &count)) + return -1; + + if (count > MAX_BPF_PROGS) { + pr_err("bpf btf count %u too large (max %u)\n", count, MAX_BPF_PROGS); + return -1; + } + + if (ff->size < sizeof(u32) + count * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_BPF_BTF: section too small (%zu) for %u entries\n", + ff->size, count); + return -1; + } + + down_write(&env->bpf_progs.lock); + + for (i = 0; i < count; ++i) { + u32 id, data_size; + + if (do_read_u32(ff, &id)) + goto out; + if (do_read_u32(ff, &data_size)) + goto out; + + if (data_size > MAX_BPF_DATA_LEN) { + pr_err("bpf btf data size %u too large (max %u)\n", + data_size, MAX_BPF_DATA_LEN); + goto out; + } + + node = malloc(sizeof(struct btf_node) + data_size); + if (!node) + goto out; + + node->id = id; + node->data_size = data_size; + + if (__do_read(ff, node->data, data_size)) + goto out; + + if (!__perf_env__insert_btf(env, node)) + free(node); + node = NULL; + } + + err = 0; +out: + up_write(&env->bpf_progs.lock); + free(node); + return err; +#else + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BTF data present but libbpf not available, skipping.\n"); + return 0; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int process_compressed(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + + if (do_read_u32(ff, &(env->comp_ver))) + return -1; + + if (do_read_u32(ff, &(env->comp_type))) + return -1; + + if (do_read_u32(ff, &(env->comp_level))) + return -1; + + if (do_read_u32(ff, &(env->comp_ratio))) + return -1; + + if (do_read_u32(ff, &(env->comp_mmap_len))) + return -1; + + /* + * FIXME: perf.data should record the recording system's page + * size — it affects mmap buffer alignment, sample addresses, + * and data_page_size/code_page_size interpretation. Without + * it we assume 4K (the smallest Linux page size) as a safe + * minimum alignment for comp_mmap_len validation. + * + * No upper-bound cap: perf_session__process_compressed_event() + * checks decomp_len + sizeof(struct decomp) against SIZE_MAX + * before allocating, which handles 32-bit safety. + */ + if (env->comp_mmap_len < 4096 || env->comp_mmap_len % 4096) { + pr_err("Invalid HEADER_COMPRESSED: comp_mmap_len (%u) must be a 4K-aligned value >= 4096\n", + env->comp_mmap_len); + return -1; + } + + return 0; +} + +static int __process_pmu_caps(struct feat_fd *ff, int *nr_caps, + char ***caps, unsigned int *max_branches, + unsigned int *br_cntr_nr, + unsigned int *br_cntr_width) +{ + char *name, *value, *ptr; + u32 nr_pmu_caps, i; + + *nr_caps = 0; + *caps = NULL; + + if (do_read_u32(ff, &nr_pmu_caps)) + return -1; + + if (!nr_pmu_caps) + return 0; + + if (nr_pmu_caps > MAX_PMU_CAPS) { + pr_err("Invalid pmu caps: nr_pmu_caps (%u) > %u\n", + nr_pmu_caps, MAX_PMU_CAPS); + return -1; + } + + *caps = calloc(nr_pmu_caps, sizeof(char *)); + if (!*caps) + return -1; + + for (i = 0; i < nr_pmu_caps; i++) { + name = do_read_string(ff); + if (!name) + goto error; + + value = do_read_string(ff); + if (!value) + goto free_name; + + if (asprintf(&ptr, "%s=%s", name, value) < 0) + goto free_value; + + (*caps)[i] = ptr; + + if (!strcmp(name, "branches")) + *max_branches = atoi(value); + + if (!strcmp(name, "branch_counter_nr")) + *br_cntr_nr = atoi(value); + + if (!strcmp(name, "branch_counter_width")) + *br_cntr_width = atoi(value); + + free(value); + free(name); + } + *nr_caps = nr_pmu_caps; + return 0; + +free_value: + free(value); +free_name: + free(name); +error: + for (; i > 0; i--) + free((*caps)[i - 1]); + free(*caps); + *caps = NULL; + *nr_caps = 0; + return -1; +} + +static int process_cpu_pmu_caps(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + int ret = __process_pmu_caps(ff, &env->nr_cpu_pmu_caps, + &env->cpu_pmu_caps, + &env->max_branches, + &env->br_cntr_nr, + &env->br_cntr_width); + + if (!ret && !env->cpu_pmu_caps) + pr_debug("cpu pmu capabilities not available\n"); + return ret; +} + +static int process_pmu_caps(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct pmu_caps *pmu_caps; + u32 nr_pmu, i; + int ret; + int j; + + if (do_read_u32(ff, &nr_pmu)) + return -1; + + if (!nr_pmu) { + pr_debug("pmu capabilities not available\n"); + return 0; + } + + if (nr_pmu > MAX_PMU_MAPPINGS) { + pr_err("Invalid HEADER_PMU_CAPS: nr_pmu (%u) > %u\n", + nr_pmu, MAX_PMU_MAPPINGS); + return -1; + } + + if (ff->size < sizeof(u32) + nr_pmu * sizeof(u32)) { + pr_err("Invalid HEADER_PMU_CAPS: section too small (%zu) for %u PMUs\n", + ff->size, nr_pmu); + return -1; + } + + pmu_caps = calloc(nr_pmu, sizeof(*pmu_caps)); + if (!pmu_caps) + return -ENOMEM; + + for (i = 0; i < nr_pmu; i++) { + ret = __process_pmu_caps(ff, &pmu_caps[i].nr_caps, + &pmu_caps[i].caps, + &pmu_caps[i].max_branches, + &pmu_caps[i].br_cntr_nr, + &pmu_caps[i].br_cntr_width); + if (ret) + goto err; + + pmu_caps[i].pmu_name = do_read_string(ff); + if (!pmu_caps[i].pmu_name) { + ret = -1; + goto err; + } + if (!pmu_caps[i].nr_caps) { + pr_debug("%s pmu capabilities not available\n", + pmu_caps[i].pmu_name); + } + } + + env->nr_pmus_with_caps = nr_pmu; + env->pmu_caps = pmu_caps; + return 0; + +err: + for (i = 0; i < nr_pmu; i++) { + for (j = 0; j < pmu_caps[i].nr_caps; j++) + free(pmu_caps[i].caps[j]); + free(pmu_caps[i].caps); + free(pmu_caps[i].pmu_name); + } + + free(pmu_caps); + return ret; +} + +static int process_cpu_domain_info(struct feat_fd *ff, void *data __maybe_unused) +{ + u32 schedstat_version, max_sched_domains, cpu, domain, nr_domains; + struct perf_env *env = &ff->ph->env; + char *dname, *cpumask, *cpulist; + struct cpu_domain_map **cd_map; + struct domain_info *d_info; + u32 nra, nr, i, j; + int ret; + + nra = env->nr_cpus_avail; + nr = env->nr_cpus_online; + + if (nra == 0 || nr == 0) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: missing HEADER_NRCPUS\n"); + return -1; + } + + if (ff->size < 2 * sizeof(u32) + nr * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: section too small (%zu) for %u CPUs\n", + (size_t)ff->size, nr); + return -1; + } + + cd_map = calloc(nra, sizeof(*cd_map)); + if (!cd_map) + return -1; + + env->cpu_domain = cd_map; + + ret = do_read_u32(ff, &schedstat_version); + if (ret) + return ret; + + env->schedstat_version = schedstat_version; + + ret = do_read_u32(ff, &max_sched_domains); + if (ret) + return ret; + + /* + * Sanity check: real systems have at most ~10 sched domain levels + * (SMT, CLS, MC, PKG + NUMA hops). Reject obviously bogus values + * from malformed perf.data files before they cause excessive + * allocation in the per-CPU loop. + */ + if (max_sched_domains > MAX_SCHED_DOMAINS) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: max_sched_domains %u > %u\n", + max_sched_domains, MAX_SCHED_DOMAINS); + return -1; + } + + env->max_sched_domains = max_sched_domains; + + for (i = 0; i < nr; i++) { + if (do_read_u32(ff, &cpu)) + return -1; + + if (cpu >= nra) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: cpu %d >= nr_cpus_avail (%d)\n", cpu, nra); + return -1; + } + + if (cd_map[cpu]) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: duplicate cpu %u\n", cpu); + return -1; + } + + cd_map[cpu] = zalloc(sizeof(*cd_map[cpu])); + if (!cd_map[cpu]) + return -1; + + cd_map[cpu]->cpu = cpu; + + if (do_read_u32(ff, &nr_domains)) + return -1; + + if (nr_domains > max_sched_domains) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: nr_domains %u > max_sched_domains (%u)\n", + nr_domains, max_sched_domains); + return -1; + } + + cd_map[cpu]->nr_domains = nr_domains; + + cd_map[cpu]->domains = calloc(max_sched_domains, sizeof(*d_info)); + if (!cd_map[cpu]->domains) + return -1; + + for (j = 0; j < nr_domains; j++) { + if (do_read_u32(ff, &domain)) + return -1; + + if (domain >= max_sched_domains) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: domain %d >= max_sched_domains (%d)\n", + domain, max_sched_domains); + return -1; + } + + d_info = zalloc(sizeof(*d_info)); + if (!d_info) + return -1; + + if (cd_map[cpu]->domains[domain]) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: duplicate domain %u for cpu %u\n", + domain, cpu); + free(d_info); + return -1; + } + + cd_map[cpu]->domains[domain] = d_info; + d_info->domain = domain; + + if (schedstat_version >= 17) { + dname = do_read_string(ff); + if (!dname) + return -1; + + d_info->dname = dname; + } + + cpumask = do_read_string(ff); + if (!cpumask) + return -1; + + d_info->cpumask = cpumask; + + cpulist = do_read_string(ff); + if (!cpulist) + return -1; + + d_info->cpulist = cpulist; + } + } + + return ret; +} + +#define FEAT_OPR(n, func, __full_only) \ + [HEADER_##n] = { \ + .name = __stringify(n), \ + .write = write_##func, \ + .print = print_##func, \ + .full_only = __full_only, \ + .process = process_##func, \ + .synthesize = true \ + } + +#define FEAT_OPN(n, func, __full_only) \ + [HEADER_##n] = { \ + .name = __stringify(n), \ + .write = write_##func, \ + .print = print_##func, \ + .full_only = __full_only, \ + .process = process_##func \ + } + +/* feature_ops not implemented: */ +#define print_tracing_data NULL +#define print_build_id NULL + +#define process_branch_stack NULL +#define process_stat NULL + +// Only used in util/synthetic-events.c +const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE]; + +const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE] = { + FEAT_OPN(TRACING_DATA, tracing_data, false), + FEAT_OPN(BUILD_ID, build_id, false), + FEAT_OPR(HOSTNAME, hostname, false), + FEAT_OPR(OSRELEASE, osrelease, false), + FEAT_OPR(VERSION, version, false), + FEAT_OPR(ARCH, arch, false), + FEAT_OPR(NRCPUS, nrcpus, false), + FEAT_OPR(CPUDESC, cpudesc, false), + FEAT_OPR(CPUID, cpuid, false), + FEAT_OPR(TOTAL_MEM, total_mem, false), + FEAT_OPR(EVENT_DESC, event_desc, false), + FEAT_OPR(CMDLINE, cmdline, false), + FEAT_OPR(CPU_TOPOLOGY, cpu_topology, true), + FEAT_OPR(NUMA_TOPOLOGY, numa_topology, true), + FEAT_OPN(BRANCH_STACK, branch_stack, false), + FEAT_OPR(PMU_MAPPINGS, pmu_mappings, false), + FEAT_OPR(GROUP_DESC, group_desc, false), + FEAT_OPN(AUXTRACE, auxtrace, false), + FEAT_OPN(STAT, stat, false), + FEAT_OPN(CACHE, cache, true), + FEAT_OPR(SAMPLE_TIME, sample_time, false), + FEAT_OPR(MEM_TOPOLOGY, mem_topology, true), + FEAT_OPR(CLOCKID, clockid, false), + FEAT_OPN(DIR_FORMAT, dir_format, false), + FEAT_OPR(BPF_PROG_INFO, bpf_prog_info, false), + FEAT_OPR(BPF_BTF, bpf_btf, false), + FEAT_OPR(COMPRESSED, compressed, false), + FEAT_OPR(CPU_PMU_CAPS, cpu_pmu_caps, false), + FEAT_OPR(CLOCK_DATA, clock_data, false), + FEAT_OPN(HYBRID_TOPOLOGY, hybrid_topology, true), + FEAT_OPR(PMU_CAPS, pmu_caps, false), + FEAT_OPR(CPU_DOMAIN_INFO, cpu_domain_info, true), + FEAT_OPR(E_MACHINE, e_machine, false), + FEAT_OPR(CLN_SIZE, cln_size, false), +}; + +struct header_print_data { + FILE *fp; + bool full; /* extended list of headers */ +}; + +const char *header_feat__name(unsigned int id) +{ + if (id < HEADER_LAST_FEATURE) + return feat_ops[id].name ?: "INVALID"; + return "INVALID"; +} + +static int perf_file_section__fprintf_info(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data) +{ + struct header_print_data *hd = data; + struct feat_fd ff; + + if (lseek(fd, section->offset, SEEK_SET) == (off_t)-1) { + pr_debug("Failed to lseek to %" PRIu64 " offset for feature %s (%d), continuing...\n", + section->offset, header_feat__name(feat), feat); + return 0; + } + if (feat >= ph->last_feat) { + pr_warning("unknown feature %d\n", feat); + return 0; + } + if (!feat_ops[feat].print) + return 0; + + ff = (struct feat_fd) { + .fd = fd, + .ph = ph, + .size = section->size, + }; + + if (!feat_ops[feat].full_only || hd->full) + feat_ops[feat].print(&ff, hd->fp); + else + fprintf(hd->fp, "# %s info available, use -I to display\n", + feat_ops[feat].name); + + return 0; +} + +int perf_header__fprintf_info(struct perf_session *session, FILE *fp, bool full) +{ + struct header_print_data hd; + struct perf_header *header = &session->header; + int fd = perf_data__fd(session->data); + struct stat st; + time_t stctime; + int ret, bit; + + hd.fp = fp; + hd.full = full; + + ret = fstat(fd, &st); + if (ret == -1) + return -1; + + stctime = st.st_mtime; + fprintf(fp, "# captured on : %s", ctime(&stctime)); + + fprintf(fp, "# header version : %u\n", header->version); + fprintf(fp, "# data offset : %" PRIu64 "\n", header->data_offset); + fprintf(fp, "# data size : %" PRIu64 "\n", header->data_size); + fprintf(fp, "# feat offset : %" PRIu64 "\n", header->feat_offset); + + perf_header__process_sections(header, fd, &hd, + perf_file_section__fprintf_info); + + if (session->data->is_pipe) + return 0; + + fprintf(fp, "# missing features: "); + for_each_clear_bit(bit, header->adds_features, header->last_feat) { + if (bit) + fprintf(fp, "%s ", feat_ops[bit].name); + } + + fprintf(fp, "\n"); + return 0; +} + +struct header_fw { + struct feat_writer fw; + struct feat_fd *ff; +}; + +static int feat_writer_cb(struct feat_writer *fw, void *buf, size_t sz) +{ + struct header_fw *h = container_of(fw, struct header_fw, fw); + + return do_write(h->ff, buf, sz); +} + +static int do_write_feat(struct feat_fd *ff, int type, + struct perf_file_section **p, + struct evlist *evlist, + struct feat_copier *fc) +{ + int err; + int ret = 0; + + if (perf_header__has_feat(ff->ph, type)) { + if (!feat_ops[type].write) + return -1; + + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + + (*p)->offset = lseek(ff->fd, 0, SEEK_CUR); + + /* + * Hook to let perf inject copy features sections from the input + * file. + */ + if (fc && fc->copy) { + struct header_fw h = { + .fw.write = feat_writer_cb, + .ff = ff, + }; + + /* ->copy() returns 0 if the feature was not copied */ + err = fc->copy(fc, type, &h.fw); + } else { + err = 0; + } + if (!err) + err = feat_ops[type].write(ff, evlist); + if (err < 0) { + pr_debug("failed to write feature %s\n", feat_ops[type].name); + + /* undo anything written */ + lseek(ff->fd, (*p)->offset, SEEK_SET); + + return -1; + } + (*p)->size = lseek(ff->fd, 0, SEEK_CUR) - (*p)->offset; + (*p)++; + } + return ret; +} + +static int perf_header__adds_write(struct perf_header *header, + struct evlist *evlist, int fd, + struct feat_copier *fc) +{ + int nr_sections; + struct feat_fd ff = { + .fd = fd, + .ph = header, + }; + struct perf_file_section *feat_sec, *p; + int sec_size; + u64 sec_start; + int feat; + int err; + + nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); + if (!nr_sections) + return 0; + + feat_sec = p = calloc(nr_sections, sizeof(*feat_sec)); + if (feat_sec == NULL) + return -ENOMEM; + + sec_size = sizeof(*feat_sec) * nr_sections; + + sec_start = header->feat_offset; + lseek(fd, sec_start + sec_size, SEEK_SET); + + for_each_set_bit(feat, header->adds_features, HEADER_FEAT_BITS) { + if (do_write_feat(&ff, feat, &p, evlist, fc)) + perf_header__clear_feat(header, feat); + } + + lseek(fd, sec_start, SEEK_SET); + /* + * may write more than needed due to dropped feature, but + * this is okay, reader will skip the missing entries + */ + err = do_write(&ff, feat_sec, sec_size); + if (err < 0) + pr_debug("failed to write feature section\n"); + free(ff.buf); /* TODO: added to silence clang-tidy. */ + free(feat_sec); + return err; +} + +int perf_header__write_pipe(int fd) +{ + struct perf_pipe_file_header f_header; + struct feat_fd ff = { + .fd = fd, + }; + int err; + + f_header = (struct perf_pipe_file_header){ + .magic = PERF_MAGIC, + .size = sizeof(f_header), + }; + + err = do_write(&ff, &f_header, sizeof(f_header)); + if (err < 0) { + pr_debug("failed to write perf pipe header\n"); + return err; + } + free(ff.buf); + return 0; +} + +static int perf_session__do_write_header(struct perf_session *session, + struct evlist *evlist, + int fd, bool at_exit, + struct feat_copier *fc, + bool write_attrs_after_data) +{ + struct perf_file_header f_header; + struct perf_header *header = &session->header; + struct evsel *evsel; + struct feat_fd ff = { + .ph = header, + .fd = fd, + }; + u64 attr_offset = sizeof(f_header), attr_size = 0; + int err; + + if (write_attrs_after_data && at_exit) { + /* + * Write features at the end of the file first so that + * attributes may come after them. + */ + if (!header->data_offset && header->data_size) { + pr_err("File contains data but offset unknown\n"); + err = -1; + goto err_out; + } + header->feat_offset = header->data_offset + header->data_size; + err = perf_header__adds_write(header, evlist, fd, fc); + if (err < 0) + goto err_out; + attr_offset = lseek(fd, 0, SEEK_CUR); + } else { + lseek(fd, attr_offset, SEEK_SET); + } + + evlist__for_each_entry(session->evlist, evsel) { + evsel->id_offset = attr_offset; + /* Avoid writing at the end of the file until the session is exiting. */ + if (!write_attrs_after_data || at_exit) { + err = do_write(&ff, evsel->core.id, evsel->core.ids * sizeof(u64)); + if (err < 0) { + pr_debug("failed to write perf header\n"); + goto err_out; + } + } + attr_offset += evsel->core.ids * sizeof(u64); + } + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.size < sizeof(evsel->core.attr)) { + /* + * We are likely in "perf inject" and have read + * from an older file. Update attr size so that + * reader gets the right offset to the ids. + */ + evsel->core.attr.size = sizeof(evsel->core.attr); + } + /* Avoid writing at the end of the file until the session is exiting. */ + if (!write_attrs_after_data || at_exit) { + struct perf_file_attr f_attr = { + .attr = evsel->core.attr, + .ids = { + .offset = evsel->id_offset, + .size = evsel->core.ids * sizeof(u64), + } + }; + err = do_write(&ff, &f_attr, sizeof(f_attr)); + if (err < 0) { + pr_debug("failed to write perf header attribute\n"); + goto err_out; + } + } + attr_size += sizeof(struct perf_file_attr); + } + + if (!header->data_offset) { + if (write_attrs_after_data) + header->data_offset = sizeof(f_header); + else + header->data_offset = attr_offset + attr_size; + } + header->feat_offset = header->data_offset + header->data_size; + + if (!write_attrs_after_data && at_exit) { + /* Write features now feat_offset is known. */ + err = perf_header__adds_write(header, evlist, fd, fc); + if (err < 0) + goto err_out; + } + + f_header = (struct perf_file_header){ + .magic = PERF_MAGIC, + .size = sizeof(f_header), + .attr_size = sizeof(struct perf_file_attr), + .attrs = { + .offset = attr_offset, + .size = attr_size, + }, + .data = { + .offset = header->data_offset, + .size = header->data_size, + }, + /* event_types is ignored, store zeros */ + }; + + memcpy(&f_header.adds_features, &header->adds_features, sizeof(header->adds_features)); + + lseek(fd, 0, SEEK_SET); + err = do_write(&ff, &f_header, sizeof(f_header)); + if (err < 0) { + pr_debug("failed to write perf header\n"); + goto err_out; + } else { + lseek(fd, 0, SEEK_END); + err = 0; + } +err_out: + free(ff.buf); + return err; +} + +int perf_session__write_header(struct perf_session *session, + struct evlist *evlist, + int fd, bool at_exit) +{ + return perf_session__do_write_header(session, evlist, fd, at_exit, /*fc=*/NULL, + /*write_attrs_after_data=*/false); +} + +size_t perf_session__data_offset(struct evlist *evlist) +{ + struct evsel *evsel; + size_t data_offset; + + data_offset = sizeof(struct perf_file_header); + evlist__for_each_entry(evlist, evsel) { + data_offset += evsel->core.ids * sizeof(u64); + } + data_offset += evlist__nr_entries(evlist) * sizeof(struct perf_file_attr); + + return data_offset; +} + +int perf_session__inject_header(struct perf_session *session, + struct evlist *evlist, + int fd, + struct feat_copier *fc, + bool write_attrs_after_data) +{ + return perf_session__do_write_header(session, evlist, fd, true, fc, + write_attrs_after_data); +} + +static int perf_header__getbuffer64(struct perf_header *header, + int fd, void *buf, size_t size) +{ + ssize_t n = readn(fd, buf, size); + + if (n <= 0) { + if (n == 0) + errno = EIO; + return -1; + } + + if (header->needs_swap) + mem_bswap_64(buf, size); + + return 0; +} + +int perf_header__process_sections(struct perf_header *header, int fd, + void *data, + int (*process)(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data)) +{ + struct perf_file_section *feat_sec, *sec; + int nr_sections; + int sec_size; + int feat; + int err; + struct stat st; + + nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); + if (!nr_sections) + return 0; + + feat_sec = sec = calloc(nr_sections, sizeof(*feat_sec)); + if (!feat_sec) + return -1; + + sec_size = sizeof(*feat_sec) * nr_sections; + + lseek(fd, header->feat_offset, SEEK_SET); + + err = perf_header__getbuffer64(header, fd, feat_sec, sec_size); + if (err < 0) + goto out_free; + + if (fstat(fd, &st) < 0) { + pr_err("Failed to stat the perf data file\n"); + err = -1; + goto out_free; + } + + for_each_set_bit(feat, header->adds_features, header->last_feat) { + /* + * FIXME: block devices have st_size == 0, so we skip + * bounds checking entirely. Historically perf never + * prevented using a block device as input, but it + * probably should — there's no valid use case for it + * and it bypasses all file-size validation. + */ + if (S_ISREG(st.st_mode) && + (sec->offset > (u64)st.st_size || + sec->size > (u64)st.st_size - sec->offset)) { + pr_err("Feature %s (%d) section extends past EOF (offset=%" PRIu64 ", size=%" PRIu64 ", file=%" PRIu64 ")\n", + header_feat__name(feat), feat, + sec->offset, sec->size, (u64)st.st_size); + err = -1; + goto out_free; + } + err = process(sec++, header, feat, fd, data); + if (err < 0) + goto out_free; + } + err = 0; +out_free: + free(feat_sec); + return err; +} + +static const int attr_file_abi_sizes[] = { + [0] = PERF_ATTR_SIZE_VER0, + [1] = PERF_ATTR_SIZE_VER1, + [2] = PERF_ATTR_SIZE_VER2, + [3] = PERF_ATTR_SIZE_VER3, + [4] = PERF_ATTR_SIZE_VER4, + 0, +}; + +/* + * In the legacy file format, the magic number is not used to encode endianness. + * hdr_sz was used to encode endianness. But given that hdr_sz can vary based + * on ABI revisions, we need to try all combinations for all endianness to + * detect the endianness. + */ +static int try_all_file_abis(uint64_t hdr_sz, struct perf_header *ph) +{ + uint64_t ref_size, attr_size; + int i; + + for (i = 0 ; attr_file_abi_sizes[i]; i++) { + ref_size = attr_file_abi_sizes[i] + + sizeof(struct perf_file_section); + if (hdr_sz != ref_size) { + attr_size = bswap_64(hdr_sz); + if (attr_size != ref_size) + continue; + + ph->needs_swap = true; + } + pr_debug("ABI%d perf.data file detected, need_swap=%d\n", + i, + ph->needs_swap); + return 0; + } + /* could not determine endianness */ + return -1; +} + +#define PERF_PIPE_HDR_VER0 16 + +static const size_t attr_pipe_abi_sizes[] = { + [0] = PERF_PIPE_HDR_VER0, + 0, +}; + +/* + * In the legacy pipe format, there is an implicit assumption that endianness + * between host recording the samples, and host parsing the samples is the + * same. This is not always the case given that the pipe output may always be + * redirected into a file and analyzed on a different machine with possibly a + * different endianness and perf_event ABI revisions in the perf tool itself. + */ +static int try_all_pipe_abis(uint64_t hdr_sz, struct perf_header *ph) +{ + u64 attr_size; + int i; + + for (i = 0 ; attr_pipe_abi_sizes[i]; i++) { + if (hdr_sz != attr_pipe_abi_sizes[i]) { + attr_size = bswap_64(hdr_sz); + if (attr_size != hdr_sz) + continue; + + ph->needs_swap = true; + } + pr_debug("Pipe ABI%d perf.data file detected\n", i); + return 0; + } + return -1; +} + +bool is_perf_magic(u64 magic) +{ + if (!memcmp(&magic, __perf_magic1, sizeof(magic)) + || magic == __perf_magic2 + || magic == __perf_magic2_sw) + return true; + + return false; +} + +static int check_magic_endian(u64 magic, uint64_t hdr_sz, + bool is_pipe, struct perf_header *ph) +{ + int ret; + + /* check for legacy format */ + ret = memcmp(&magic, __perf_magic1, sizeof(magic)); + if (ret == 0) { + ph->version = PERF_HEADER_VERSION_1; + pr_debug("legacy perf.data format\n"); + if (is_pipe) + return try_all_pipe_abis(hdr_sz, ph); + + return try_all_file_abis(hdr_sz, ph); + } + /* + * the new magic number serves two purposes: + * - unique number to identify actual perf.data files + * - encode endianness of file + */ + ph->version = PERF_HEADER_VERSION_2; + + /* check magic number with one endianness */ + if (magic == __perf_magic2) + return 0; + + /* check magic number with opposite endianness */ + if (magic != __perf_magic2_sw) + return -1; + + ph->needs_swap = true; + + return 0; +} + +int perf_file_header__read(struct perf_file_header *header, + struct perf_header *ph, int fd) +{ + ssize_t ret; + + lseek(fd, 0, SEEK_SET); + + ret = readn(fd, header, sizeof(*header)); + if (ret <= 0) + return -1; + + if (check_magic_endian(header->magic, + header->attr_size, false, ph) < 0) { + pr_debug("magic/endian check failed\n"); + return -1; + } + + if (ph->needs_swap) { + mem_bswap_64(header, offsetof(struct perf_file_header, + adds_features)); + } + + if (header->size > header->attrs.offset) { + pr_err("Perf file header corrupt: header overlaps attrs\n"); + return -1; + } + + if (header->size > header->data.offset) { + pr_err("Perf file header corrupt: header overlaps data\n"); + return -1; + } + + if ((header->attrs.offset <= header->data.offset && + header->attrs.offset + header->attrs.size > header->data.offset) || + (header->attrs.offset > header->data.offset && + header->data.offset + header->data.size > header->attrs.offset)) { + pr_err("Perf file header corrupt: Attributes and data overlap\n"); + return -1; + } + + if (header->size != sizeof(*header)) { + /* Support the previous format */ + if (header->size == offsetof(typeof(*header), adds_features)) + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + else + return -1; + } else if (ph->needs_swap) { + /* + * feature bitmap is declared as an array of unsigned longs -- + * not good since its size can differ between the host that + * generated the data file and the host analyzing the file. + * + * We need to handle endianness, but we don't know the size of + * the unsigned long where the file was generated. Take a best + * guess at determining it: try 64-bit swap first (ie., file + * created on a 64-bit host), and check if the hostname feature + * bit is set (this feature bit is forced on as of fbe96f2). + * If the bit is not, undo the 64-bit swap and try a 32-bit + * swap. If the hostname bit is still not set (e.g., older data + * file), punt and fallback to the original behavior -- + * clearing all feature bits and setting buildid. + */ + mem_bswap_64(&header->adds_features, + BITS_TO_U64(HEADER_FEAT_BITS)); + + if (!test_bit(HEADER_HOSTNAME, header->adds_features)) { + /* unswap as u64 */ + mem_bswap_64(&header->adds_features, + BITS_TO_U64(HEADER_FEAT_BITS)); + + /* unswap as u32 */ + mem_bswap_32(&header->adds_features, + BITS_TO_U32(HEADER_FEAT_BITS)); + } + + if (!test_bit(HEADER_HOSTNAME, header->adds_features)) { + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + __set_bit(HEADER_BUILD_ID, header->adds_features); + } + } + + memcpy(&ph->adds_features, &header->adds_features, + sizeof(ph->adds_features)); + + ph->data_offset = header->data.offset; + ph->data_size = header->data.size; + ph->feat_offset = header->data.offset + header->data.size; + ph->last_feat = HEADER_LAST_FEATURE; + return 0; +} + +static int perf_file_section__process(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data) +{ + struct feat_fd fdd = { + .fd = fd, + .ph = ph, + .size = section->size, + .offset = 0, + }; + + if (lseek(fd, section->offset, SEEK_SET) == (off_t)-1) { + pr_debug("Failed to lseek to %" PRIu64 " offset for feature %s (%d), continuing...\n", + section->offset, header_feat__name(feat), feat); + return 0; + } + + if (feat >= HEADER_LAST_FEATURE) { + pr_debug("unknown feature %d, continuing...\n", feat); + return 0; + } + + if (!feat_ops[feat].process) + return 0; + + return feat_ops[feat].process(&fdd, data); +} + +static int perf_file_header__read_pipe(struct perf_pipe_file_header *header, + struct perf_header *ph, + struct perf_data *data) +{ + ssize_t ret; + + ret = perf_data__read(data, header, sizeof(*header)); + if (ret <= 0) + return -1; + + if (check_magic_endian(header->magic, header->size, true, ph) < 0) { + pr_debug("endian/magic failed\n"); + return -1; + } + + if (ph->needs_swap) + header->size = bswap_64(header->size); + + /* The last feature is written out as a 0 sized event and will update this value. */ + ph->last_feat = 0; + return 0; +} + +static int perf_header__read_pipe(struct perf_session *session) +{ + struct perf_header *header = &session->header; + struct perf_pipe_file_header f_header; + + if (perf_file_header__read_pipe(&f_header, header, session->data) < 0) { + pr_debug("incompatible file format\n"); + return -EINVAL; + } + + return f_header.size == sizeof(f_header) ? 0 : -1; +} + +static int read_attr(int fd, struct perf_header *ph, + struct perf_file_attr *f_attr) +{ + struct perf_event_attr *attr = &f_attr->attr; + size_t sz, left; + size_t our_sz = sizeof(f_attr->attr); + ssize_t ret; + + memset(f_attr, 0, sizeof(*f_attr)); + + /* read minimal guaranteed structure */ + ret = readn(fd, attr, PERF_ATTR_SIZE_VER0); + if (ret <= 0) { + pr_debug("cannot read %d bytes of header attr\n", + PERF_ATTR_SIZE_VER0); + if (ret == 0) + errno = EIO; + return -1; + } + + /* on file perf_event_attr size */ + sz = attr->size; + + if (ph->needs_swap) + sz = bswap_32(sz); + + if (sz == 0) { + /* assume ABI0 */ + sz = PERF_ATTR_SIZE_VER0; + } else if (sz < PERF_ATTR_SIZE_VER0) { + pr_debug("bad attr size %zu, expected at least %d\n", + sz, PERF_ATTR_SIZE_VER0); + errno = EINVAL; + return -1; + } else if (sz > our_sz) { + pr_debug("file uses a more recent and unsupported ABI" + " (%zu bytes extra)\n", sz - our_sz); + errno = EINVAL; + return -1; + } + /* what we have not yet read and that we know about */ + left = sz - PERF_ATTR_SIZE_VER0; + if (left) { + void *ptr = attr; + ptr += PERF_ATTR_SIZE_VER0; + + ret = readn(fd, ptr, left); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } + } + /* read perf_file_section, ids are read in caller */ + ret = readn(fd, &f_attr->ids, sizeof(f_attr->ids)); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } + + return 0; +} + +#ifdef HAVE_LIBTRACEEVENT +static int evsel__prepare_tracepoint_event(struct evsel *evsel, struct tep_handle *pevent) +{ + struct tep_event *event; + char bf[128]; + + /* already prepared */ + if (evsel->tp_format) + return 0; + + if (pevent == NULL) { + pr_debug("broken or missing trace data\n"); + return -1; + } + + event = tep_find_event(pevent, evsel->core.attr.config); + if (event == NULL) { + pr_debug("cannot find event format for %d\n", (int)evsel->core.attr.config); + return -1; + } + + if (!evsel->name) { + snprintf(bf, sizeof(bf), "%s:%s", event->system, event->name); + evsel->name = strdup(bf); + if (evsel->name == NULL) + return -1; + } + + evsel->tp_format = event; + return 0; +} + +static int evlist__prepare_tracepoint_events(struct evlist *evlist, struct tep_handle *pevent) +{ + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) { + if (pos->core.attr.type == PERF_TYPE_TRACEPOINT && + evsel__prepare_tracepoint_event(pos, pevent)) + return -1; + } + + return 0; +} +#endif + +int perf_session__read_header(struct perf_session *session) +{ + struct perf_data *data = session->data; + struct perf_header *header = &session->header; + struct perf_file_header f_header; + struct perf_file_attr f_attr; + u64 f_id; + struct stat input_stat; + int nr_attrs, nr_ids, i, j, err = -ENOMEM; + int fd = perf_data__fd(data); + + session->evlist = evlist__new(); + if (session->evlist == NULL) + return -ENOMEM; + + evlist__set_session(session->evlist, session); + session->machines.host.env = &header->env; + + /* + * We can read 'pipe' data event from regular file, + * check for the pipe header regardless of source. + */ + err = perf_header__read_pipe(session); + if (!err || perf_data__is_pipe(data)) { + data->is_pipe = true; + return err; + } + + err = -ENOMEM; + if (perf_file_header__read(&f_header, header, fd) < 0) + return -EINVAL; + + if (header->needs_swap && data->in_place_update) { + pr_err("In-place update not supported when byte-swapping is required\n"); + return -EINVAL; + } + + /* + * Sanity check that perf.data was written cleanly; data size is + * initialized to 0 and updated only if the on_exit function is run. + * If data size is still 0 then the file contains only partial + * information. Just warn user and process it as much as it can. + */ + if (f_header.data.size == 0) { + pr_warning("WARNING: The %s file's data size field is 0 which is unexpected.\n" + "Was the 'perf record' command properly terminated?\n", + data->file.path); + } + + if (f_header.attr_size == 0) { + pr_err("ERROR: The %s file's attr size field is 0 which is unexpected.\n" + "Was the 'perf record' command properly terminated?\n", + data->file.path); + return -EINVAL; + } + + if (fstat(fd, &input_stat) < 0) + return -errno; + + /* Check before assigning to int to avoid u64-to-int truncation */ + if (f_header.attrs.size / f_header.attr_size > MAX_NR_ATTRS) { + pr_err("Too many attributes: %" PRIu64 " (max %d)\n", + f_header.attrs.size / f_header.attr_size, MAX_NR_ATTRS); + return -EINVAL; + } + nr_attrs = f_header.attrs.size / f_header.attr_size; + lseek(fd, f_header.attrs.offset, SEEK_SET); + + for (i = 0; i < nr_attrs; i++) { + struct evsel *evsel; + off_t tmp; + + if (read_attr(fd, header, &f_attr) < 0) + goto out_errno; + + if (header->needs_swap) { + f_attr.ids.size = bswap_64(f_attr.ids.size); + f_attr.ids.offset = bswap_64(f_attr.ids.offset); + perf_event__attr_swap(&f_attr.attr); + } + + /* + * Validate ids section: must be aligned to u64, and + * the count must fit in an int to avoid truncation in + * nr_ids and size_t overflow in perf_evsel__alloc_id() + * on 32-bit architectures. + */ + if (f_attr.ids.size % sizeof(u64)) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_put_evlist; + } + + /* + * Cap the ID count to avoid int truncation of nr_ids + * on 64-bit and size_t overflow in the allocation + * paths (nr_ids * sizeof(u64), nr_ids * + * sizeof(struct perf_sample_id)) on 32-bit. + */ + if (f_attr.ids.size / sizeof(u64) > MAX_IDS_PER_ATTR) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_put_evlist; + } + + /* + * FIXME: see perf_header__process_sections() — block + * devices bypass this check because st_size is 0. + */ + if (S_ISREG(input_stat.st_mode) && + (f_attr.ids.offset > (u64)input_stat.st_size || + f_attr.ids.size > (u64)input_stat.st_size - f_attr.ids.offset)) { + pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n", + i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size); + err = -EINVAL; + goto out_put_evlist; + } + + tmp = lseek(fd, 0, SEEK_CUR); + evsel = evsel__new(&f_attr.attr); + + if (evsel == NULL) + goto out_put_evlist; + + evsel->needs_swap = header->needs_swap; + /* + * Do it before so that if perf_evsel__alloc_id fails, this + * entry gets purged too at evlist__put(). + */ + evlist__add(session->evlist, evsel); + + nr_ids = f_attr.ids.size / sizeof(u64); + /* + * We don't have the cpu and thread maps on the header, so + * for allocating the perf_sample_id table we fake 1 cpu and + * hattr->ids threads. + */ + if (perf_evsel__alloc_id(&evsel->core, 1, nr_ids)) + goto out_put_evlist; + + lseek(fd, f_attr.ids.offset, SEEK_SET); + + for (j = 0; j < nr_ids; j++) { + if (perf_header__getbuffer64(header, fd, &f_id, sizeof(f_id))) + goto out_errno; + + perf_evlist__id_add(evlist__core(session->evlist), + &evsel->core, 0, j, f_id); + } + + lseek(fd, tmp, SEEK_SET); + } + + /* + * Skip feature section processing for truncated files + * (data.size == 0 means recording was interrupted). The + * section table is unreliable in that case, and the event + * data can still be processed without the feature headers. + * Clear the bitmap so has_feat() returns false and tools + * use their "feature not present" fallbacks instead of + * accessing uninitialized env fields. + */ + if (f_header.data.size == 0) { + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + } else { +#ifdef HAVE_LIBTRACEEVENT + err = perf_header__process_sections(header, fd, &session->tevent, + perf_file_section__process); + if (err < 0) + goto out_put_evlist; + + if (evlist__prepare_tracepoint_events(session->evlist, + session->tevent.pevent)) { + err = -ENOMEM; + goto out_put_evlist; + } +#else + err = perf_header__process_sections(header, fd, NULL, + perf_file_section__process); + if (err < 0) + goto out_put_evlist; +#endif + } + + /* + * Without nr_cpus_avail the sample CPU bounds check in + * perf_session__deliver_event() is bypassed, allowing crafted + * CPU IDs to reach downstream consumers that index fixed-size + * arrays (timechart, kwork, sched — all sized MAX_NR_CPUS). + * + * This can happen with truncated files (interrupted recording + * loses all feature sections), very old files that predate + * HEADER_NRCPUS, or crafted files that omit it. Fall back to + * MAX_NR_CPUS so the bounds check is still effective — any + * CPU ID below that limit is safe for all downstream arrays. + */ + if (header->env.nr_cpus_avail == 0) { + header->env.nr_cpus_avail = MAX_NR_CPUS; + pr_warning("WARNING: perf.data is missing HEADER_NRCPUS, using MAX_NR_CPUS (%d) as CPU bound\n", + MAX_NR_CPUS); + } + + return 0; +out_errno: + return -errno; + +out_put_evlist: + evlist__put(session->evlist); + session->evlist = NULL; + return err; +} + +int perf_event__process_feature(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + struct feat_fd ff = { .fd = 0 }; + struct perf_record_header_feature *fe = (struct perf_record_header_feature *)event; + struct perf_header *header = &session->header; + int type = fe->header.type; + int feat = (int)fe->feat_id; + int ret = 0; + bool print = dump_trace; + bool last_feature_mark = false; + + if (type < 0 || type >= PERF_RECORD_HEADER_MAX) { + pr_warning("invalid record type %d in pipe-mode\n", type); + return 0; + } + if (feat == HEADER_RESERVED) { + pr_warning("invalid reserved record type in pipe-mode\n"); + return -1; + } + if (feat < 0 || feat == INT_MAX) { + pr_warning("invalid value for feature type %x\n", feat); + return -1; + } + if (feat >= header->last_feat) { + if (event->header.size == sizeof(*fe)) { + /* + * Either an unexpected zero size feature or the + * HEADER_LAST_FEATURE mark. + */ + if (feat > header->last_feat) + header->last_feat = min(feat, HEADER_LAST_FEATURE); + last_feature_mark = true; + } else { + /* + * A feature but beyond what is known as in + * bounds. Assume the last feature is 1 beyond this + * feature. + */ + session->header.last_feat = min(feat + 1, HEADER_LAST_FEATURE); + } + } + if (feat >= HEADER_LAST_FEATURE) { + if (!last_feature_mark) { + pr_warning("unknown feature %d for data file version (%s) in this version of perf (%s)\n", + feat, header->env.version, perf_version_string); + } + return 0; + } + if (event->header.size < sizeof(*fe)) { + pr_warning("feature header size too small\n"); + return -1; + } + ff.buf = (void *)fe->data; + ff.size = event->header.size - sizeof(*fe); + ff.ph = header; + + if (feat_ops[feat].process && feat_ops[feat].process(&ff, NULL)) { + // Processing failed, ignore when this is the last feature mark. + if (!last_feature_mark) + ret = -1; + goto out; + } + + if (session->tool->show_feat_hdr) { + if (!feat_ops[feat].full_only || + session->tool->show_feat_hdr >= SHOW_FEAT_HEADER_FULL_INFO) { + print = true; + } else { + fprintf(stdout, "# %s info available, use -I to display\n", + feat_ops[feat].name); + } + } + + if (dump_trace) + printf(", "); + + if (print) { + if (feat_ops[feat].print) + feat_ops[feat].print(&ff, stdout); + else + printf("# %s", feat_ops[feat].name); + } + +out: + free_event_desc(ff.events); + return ret; +} + +size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) +{ + struct perf_record_event_update *ev = &event->event_update; + struct perf_cpu_map *map; + size_t ret; + + ret = fprintf(fp, "\n... id: %" PRI_lu64 "\n", ev->id); + + switch (ev->type) { + case PERF_EVENT_UPDATE__SCALE: + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + ret += fprintf(fp, "... scale: (truncated)\n"); + break; + } + ret += fprintf(fp, "... scale: %f\n", ev->scale.scale); + break; + case PERF_EVENT_UPDATE__UNIT: + case PERF_EVENT_UPDATE__NAME: { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + ret += fprintf(fp, "... %s: (unterminated)\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + break; + } + ret += fprintf(fp, "... %s: %s\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name", + ev->unit); + break; + } + case PERF_EVENT_UPDATE__CPUS: { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + u32 cpus_payload; + + if (event->header.size < cpus_off + sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map)) { + ret += fprintf(fp, "... cpus: (truncated)\n"); + break; + } + + /* + * Validate nr against payload — this function may be + * called from the stub handler (dump_trace path) which + * bypasses perf_event__process_event_update() validation. + */ + cpus_payload = event->header.size - cpus_off; + if (ev->cpus.cpus.type == PERF_CPU_MAP__CPUS) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, cpus_data.cpu) || + ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + ret += fprintf(fp, "... cpus: nr %u exceeds payload\n", + ev->cpus.cpus.cpus_data.nr); + break; + } + } else if (ev->cpus.cpus.type == PERF_CPU_MAP__MASK) { + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask32_data.nr); + break; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask64_data.nr); + break; + } + } + } + + ret += fprintf(fp, "... "); + + map = cpu_map__new_data(&ev->cpus.cpus); + if (map) { + ret += cpu_map__fprintf(map, fp); + perf_cpu_map__put(map); + } else + ret += fprintf(fp, "failed to get cpus\n"); + break; + } + default: + ret += fprintf(fp, "... unknown type\n"); + break; + } + + return ret; +} + +size_t perf_event__fprintf_attr(union perf_event *event, FILE *fp) +{ + return perf_event_attr__fprintf(fp, &event->attr.attr, __desc_attr__fprintf, NULL); +} + +int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct evlist **pevlist) +{ + struct perf_event_attr attr; + u32 i, n_ids, raw_attr_size; + u64 *ids; + size_t attr_size, copy_size; + struct evsel *evsel; + struct evlist *evlist = *pevlist; + + /* + * HEADER_ATTR event layout (pipe/inject mode): + * + * [header (8 bytes)] [attr (attr_size bytes)] [id0 id1 ... idN] + * |<------------------ header.size --------------------------->| + * + * attr_size varies across perf versions: VER0 = 64 bytes, + * current sizeof(struct perf_event_attr) = larger. A newer + * producer may emit a larger attr than we understand. + * + * attr.size == 0 (ABI0) means the producer didn't set it + * (e.g., bench/inject-buildid, older perf). Treat as VER0. + * + * Require 8-byte alignment so the u64 ID array is aligned + * and attr.size fits cleanly within the payload. + * + * Read attr.size once — the event may be on a shared mmap + * and re-reading could yield a different value. + */ + raw_attr_size = event->attr.attr.size; + if (event->header.size < sizeof(event->header) + PERF_ATTR_SIZE_VER0 || + (raw_attr_size && (raw_attr_size < PERF_ATTR_SIZE_VER0 || + raw_attr_size % sizeof(u64) || + raw_attr_size > event->header.size - sizeof(event->header)))) { + pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (event size %u, min %d)\n", + raw_attr_size, event->header.size, PERF_ATTR_SIZE_VER0); + return -EINVAL; + } + + if (dump_trace) + perf_event__fprintf_attr(event, stdout); + + if (evlist == NULL) { + *pevlist = evlist = evlist__new(); + if (evlist == NULL) + return -ENOMEM; + } + + /* + * attr_size = footprint of the attr in the event — determines + * where the ID array starts. For ABI0, assume VER0 (64 bytes). + * + * copy_size = how much we copy into our local struct, capped at + * sizeof(attr) so a newer producer's larger attr doesn't + * overflow. Fields beyond copy_size are zeroed. + * + * Do NOT write attr_size back to the event — native-endian + * files use MAP_SHARED (read-only), writing would SIGSEGV. + * The swap path handles ABI0 in perf_event__attr_swap() + * which writes to the writable MAP_PRIVATE copy instead. + */ + attr_size = raw_attr_size ?: PERF_ATTR_SIZE_VER0; + copy_size = min(attr_size, sizeof(attr)); + memcpy(&attr, &event->attr.attr, copy_size); + if (copy_size < sizeof(attr)) + memset((void *)&attr + copy_size, 0, sizeof(attr) - copy_size); + + /* + * Normalize ABI0: the swap path sets attr.size = VER0 on the + * event, but the native path leaves it as 0. Set it on the + * local copy so perf inject re-synthesizes with consistent + * layout regardless of endianness. + */ + attr.size = attr_size; + + evsel = evsel__new(&attr); + if (evsel == NULL) + return -ENOMEM; + + evlist__add(evlist, evsel); + + /* + * IDs occupy the remainder after header + attr. Use attr_size + * (not copy_size) — even if the producer's attr is larger than + * our struct, the IDs start after attr_size bytes in the event. + * Validation above guarantees attr_size <= payload size. + */ + n_ids = event->header.size - sizeof(event->header) - attr_size; + n_ids = n_ids / sizeof(u64); + /* + * We don't have the cpu and thread maps on the header, so + * for allocating the perf_sample_id table we fake 1 cpu and + * hattr->ids threads. + */ + if (perf_evsel__alloc_id(&evsel->core, 1, n_ids)) + return -ENOMEM; + + /* + * Locate IDs at attr_size bytes past the attr start in the + * event. Cannot use perf_record_header_attr_id() — that + * macro reads event->attr.attr.size, which is 0 for ABI0 + * on the native-endian path (no swap handler to fix it up). + */ + ids = (void *)&event->attr.attr + attr_size; + for (i = 0; i < n_ids; i++) { + perf_evlist__id_add(evlist__core(evlist), &evsel->core, 0, i, ids[i]); + } + + return 0; +} + +int perf_event__process_event_update(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct evlist **pevlist) +{ + struct perf_record_event_update *ev = &event->event_update; + struct evlist *evlist; + struct evsel *evsel; + struct perf_cpu_map *map; + + /* + * Validate payload before dump_trace or processing — both + * paths access variant-specific fields without further checks. + */ + if (ev->type == PERF_EVENT_UPDATE__UNIT || + ev->type == PERF_EVENT_UPDATE__NAME) { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: %s not null-terminated, skipping\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__SCALE) { + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: SCALE payload too small, skipping\n"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__CPUS) { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + size_t min_cpus = sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map); + u32 cpus_payload; + + if (event->header.size < cpus_off + min_cpus) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: CPUS payload too small, skipping\n"); + return 0; + } + + /* + * Validate per-variant nr against the remaining + * payload on the native path — the swap path clamps + * nr in perf_event__event_update_swap(), but native + * events are read-only and cannot be clamped in place. + * cpu_map__new_data() trusts nr for allocation and + * iteration, so unchecked values cause OOB reads. + */ + cpus_payload = event->header.size - cpus_off; + switch (ev->cpus.cpus.type) { + case PERF_CPU_MAP__CPUS: + if (ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + pr_warning("WARNING: EVENT_UPDATE CPUS: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.cpus_data.nr); + return 0; + } + break; + case PERF_CPU_MAP__MASK: + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask32_data.nr); + return 0; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask64_data.nr); + return 0; + } + } + break; + default: + break; + } + } + + if (dump_trace) + perf_event__fprintf_event_update(event, stdout); + + if (!pevlist || *pevlist == NULL) + return -EINVAL; + + evlist = *pevlist; + + evsel = evlist__id2evsel(evlist, ev->id); + if (evsel == NULL) + return -EINVAL; + + switch (ev->type) { + case PERF_EVENT_UPDATE__UNIT: + free((char *)evsel->unit); + evsel->unit = strdup(ev->unit); + break; + case PERF_EVENT_UPDATE__NAME: + free(evsel->name); + evsel->name = strdup(ev->name); + break; + case PERF_EVENT_UPDATE__SCALE: + evsel->scale = ev->scale.scale; + break; + case PERF_EVENT_UPDATE__CPUS: + map = cpu_map__new_data(&ev->cpus.cpus); + if (map) { + perf_cpu_map__put(evsel->core.pmu_cpus); + evsel->core.pmu_cpus = map; + } else + pr_err("failed to get event_update cpus\n"); + break; + default: + break; + } + + return 0; +} + +#ifdef HAVE_LIBTRACEEVENT +int perf_event__process_tracing_data(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + ssize_t size_read, padding, size = event->tracing_data.size; + int fd = perf_data__fd(session->data); + char buf[BUFSIZ]; + + /* + * The pipe fd is already in proper place and in any case + * we can't move it, and we'd screw the case where we read + * 'pipe' data from regular file. The trace_report reads + * data from 'fd' so we need to set it directly behind the + * event, where the tracing data starts. + */ + if (!perf_data__is_pipe(session->data)) { + off_t offset = lseek(fd, 0, SEEK_CUR); + + /* setup for reading amidst mmap */ + lseek(fd, offset + sizeof(struct perf_record_header_tracing_data), + SEEK_SET); + } + + size_read = trace_report(fd, &session->tevent, session->trace_event_repipe); + padding = PERF_ALIGN(size_read, sizeof(u64)) - size_read; + + if (readn(fd, buf, padding) < 0) { + pr_err("%s: reading input file", __func__); + return -1; + } + if (session->trace_event_repipe) { + int retw = write(STDOUT_FILENO, buf, padding); + if (retw <= 0 || retw != padding) { + pr_err("%s: repiping tracing data padding", __func__); + return -1; + } + } + + if (size_read + padding != size) { + pr_err("%s: tracing data size mismatch", __func__); + return -1; + } + + evlist__prepare_tracepoint_events(session->evlist, session->tevent.pevent); + + return size_read + padding; +} +#endif + +int perf_event__process_build_id(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + __event_process_build_id(&event->build_id, + event->build_id.filename, + session); + return 0; +} diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h new file mode 100644 index 000000000..5e03f884b --- /dev/null +++ b/tools/perf/util/header.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_HEADER_H +#define __PERF_HEADER_H + +#include +#include +#include +#include // FILE +#include +#include +#include +#include "env.h" +#include + +struct evlist; +union perf_event; +struct perf_header; +struct perf_session; +struct perf_tool; + +enum { + HEADER_RESERVED = 0, /* always cleared */ + HEADER_FIRST_FEATURE = 1, + HEADER_TRACING_DATA = 1, + HEADER_BUILD_ID, + + HEADER_HOSTNAME, + HEADER_OSRELEASE, + HEADER_VERSION, + HEADER_ARCH, + HEADER_NRCPUS, + HEADER_CPUDESC, + HEADER_CPUID, + HEADER_TOTAL_MEM, + HEADER_CMDLINE, + HEADER_EVENT_DESC, + HEADER_CPU_TOPOLOGY, + HEADER_NUMA_TOPOLOGY, + HEADER_BRANCH_STACK, + HEADER_PMU_MAPPINGS, + HEADER_GROUP_DESC, + HEADER_AUXTRACE, + HEADER_STAT, + HEADER_CACHE, + HEADER_SAMPLE_TIME, + HEADER_MEM_TOPOLOGY, + HEADER_CLOCKID, + HEADER_DIR_FORMAT, + HEADER_BPF_PROG_INFO, + HEADER_BPF_BTF, + HEADER_COMPRESSED, + HEADER_CPU_PMU_CAPS, + HEADER_CLOCK_DATA, + HEADER_HYBRID_TOPOLOGY, + HEADER_PMU_CAPS, + HEADER_CPU_DOMAIN_INFO, + HEADER_E_MACHINE, + HEADER_CLN_SIZE, + HEADER_LAST_FEATURE, + HEADER_FEAT_BITS = 256, +}; + +enum perf_header_version { + PERF_HEADER_VERSION_1, + PERF_HEADER_VERSION_2, +}; + +struct perf_file_section { + u64 offset; + u64 size; +}; + +/** + * struct perf_file_header: Header representation on disk. + */ +struct perf_file_header { + /** @magic: Holds "PERFILE2". */ + u64 magic; + /** @size: Size of this header - sizeof(struct perf_file_header). */ + u64 size; + /** + * @attr_size: Size of attrs entries - sizeof(struct perf_event_attr) + + * sizeof(struct perf_file_section). + */ + u64 attr_size; + /** @attrs: Offset and size of file section holding attributes. */ + struct perf_file_section attrs; + /** @data: Offset and size of file section holding regular event data. */ + struct perf_file_section data; + /** @event_types: Ignored. */ + struct perf_file_section event_types; + /** + * @adds_features: Bitmap of features. The features are immediately after the data section. + */ + DECLARE_BITMAP(adds_features, HEADER_FEAT_BITS); +}; + +struct perf_pipe_file_header { + u64 magic; + u64 size; +}; + +int perf_file_header__read(struct perf_file_header *header, + struct perf_header *ph, int fd); + +struct perf_header { + enum perf_header_version version; + bool needs_swap; + u64 data_offset; + u64 data_size; + u64 feat_offset; + DECLARE_BITMAP(adds_features, HEADER_FEAT_BITS); + int last_feat; + struct perf_env env; +}; + +struct feat_fd { + struct perf_header *ph; + int fd; + void *buf; /* Either buf != NULL or fd >= 0 */ + ssize_t offset; + size_t size; + struct evsel *events; +}; + +struct perf_header_feature_ops { + int (*write)(struct feat_fd *ff, struct evlist *evlist); + void (*print)(struct feat_fd *ff, FILE *fp); + int (*process)(struct feat_fd *ff, void *data); + const char *name; + bool full_only; + bool synthesize; +}; + +extern const char perf_version_string[]; + +const char *header_feat__name(unsigned int id); + +int perf_session__read_header(struct perf_session *session); +int perf_session__write_header(struct perf_session *session, + struct evlist *evlist, + int fd, bool at_exit); +int perf_header__write_pipe(int fd); + +/* feat_writer writes a feature section to output */ +struct feat_writer { + int (*write)(struct feat_writer *fw, void *buf, size_t sz); +}; + +/* feat_copier copies a feature section using feat_writer to output */ +struct feat_copier { + int (*copy)(struct feat_copier *fc, int feat, struct feat_writer *fw); +}; + +int perf_session__inject_header(struct perf_session *session, + struct evlist *evlist, + int fd, + struct feat_copier *fc, + bool write_attrs_after_data); + +size_t perf_session__data_offset(struct evlist *evlist); + +void perf_header__set_feat(struct perf_header *header, int feat); +void perf_header__clear_feat(struct perf_header *header, int feat); +bool perf_header__has_feat(const struct perf_header *header, int feat); + +int perf_header__set_cmdline(int argc, const char **argv); + +int perf_header__process_sections(struct perf_header *header, int fd, + void *data, + int (*process)(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data)); + +int perf_header__fprintf_info(struct perf_session *s, FILE *fp, bool full); + +int perf_event__process_feature(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +int perf_event__process_attr(const struct perf_tool *tool, union perf_event *event, + struct evlist **pevlist); +int perf_event__process_event_update(const struct perf_tool *tool, + union perf_event *event, + struct evlist **pevlist); +size_t perf_event__fprintf_attr(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp); +#ifdef HAVE_LIBTRACEEVENT +int perf_event__process_tracing_data(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +#endif +int perf_event__process_build_id(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +bool is_perf_magic(u64 magic); + +#define NAME_ALIGN 64 + +struct feat_fd; + +int do_write(struct feat_fd *fd, const void *buf, size_t size); + +int write_padded(struct feat_fd *fd, const void *bf, + size_t count, size_t count_aligned); + +#define MAX_CACHE_LVL 4 + +int build_caches_for_cpu(u32 cpu, struct cpu_cache_level caches[], u32 *cntp); + +#define DEFAULT_CACHELINE_SIZE 64 + +/* + * arch specific callback + */ +int get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu); + +char *get_cpuid_str(struct perf_cpu cpu); + +char *get_cpuid_allow_env_override(struct perf_cpu cpu); + +int strcmp_cpuid_str(const char *s1, const char *s2); + +struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains, + u32 nr); +#endif /* __PERF_HEADER_H */ diff --git a/tools/perf/util/help-unknown-cmd.c b/tools/perf/util/help-unknown-cmd.c new file mode 100644 index 000000000..a0a46e34f --- /dev/null +++ b/tools/perf/util/help-unknown-cmd.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "cache.h" +#include "config.h" +#include +#include +#include +#include +#include "../builtin.h" +#include "levenshtein.h" +#include + +static int autocorrect; + +static int perf_unknown_cmd_config(const char *var, const char *value, + void *cb __maybe_unused) +{ + if (!strcmp(var, "help.autocorrect")) + return perf_config_int(&autocorrect, var,value); + + return 0; +} + +static int levenshtein_compare(const void *p1, const void *p2) +{ + const struct cmdname *const *c1 = p1, *const *c2 = p2; + const char *s1 = (*c1)->name, *s2 = (*c2)->name; + int l1 = (*c1)->len; + int l2 = (*c2)->len; + return l1 != l2 ? l1 - l2 : strcmp(s1, s2); +} + +static int add_cmd_list(struct cmdnames *cmds, struct cmdnames *old) +{ + unsigned int i, nr = cmds->cnt + old->cnt; + void *tmp; + + if (nr > cmds->alloc) { + /* Choose bigger one to alloc */ + if (alloc_nr(cmds->alloc) < nr) + cmds->alloc = nr; + else + cmds->alloc = alloc_nr(cmds->alloc); + tmp = realloc(cmds->names, cmds->alloc * sizeof(*cmds->names)); + if (!tmp) + return -1; + cmds->names = tmp; + } + for (i = 0; i < old->cnt; i++) + cmds->names[cmds->cnt++] = old->names[i]; + zfree(&old->names); + old->cnt = 0; + return 0; +} + +const char *help_unknown_cmd(const char *cmd, struct cmdnames *main_cmds) +{ + unsigned int i, n = 0, best_similarity = 0; + struct cmdnames other_cmds; + + memset(&other_cmds, 0, sizeof(other_cmds)); + + perf_config(perf_unknown_cmd_config, NULL); + + load_command_list("perf-", main_cmds, &other_cmds); + + if (add_cmd_list(main_cmds, &other_cmds) < 0) { + fprintf(stderr, "ERROR: Failed to allocate command list for unknown command.\n"); + goto end; + } + qsort(main_cmds->names, main_cmds->cnt, + sizeof(main_cmds->names), cmdname_compare); + uniq(main_cmds); + + if (main_cmds->cnt) { + /* This reuses cmdname->len for similarity index */ + for (i = 0; i < main_cmds->cnt; ++i) { + main_cmds->names[i]->len = + levenshtein(cmd, main_cmds->names[i]->name, + /*swap_penalty=*/0, + /*substition_penality=*/2, + /*insertion_penality=*/1, + /*deletion_penalty=*/1); + } + qsort(main_cmds->names, main_cmds->cnt, + sizeof(*main_cmds->names), levenshtein_compare); + + best_similarity = main_cmds->names[0]->len; + n = 1; + while (n < main_cmds->cnt && best_similarity == main_cmds->names[n]->len) + ++n; + } + + if (autocorrect && n == 1) { + const char *assumed = main_cmds->names[0]->name; + + main_cmds->names[0] = NULL; + clean_cmdnames(&other_cmds); + fprintf(stderr, "WARNING: You called a perf program named '%s', " + "which does not exist.\n" + "Continuing under the assumption that you meant '%s'\n", + cmd, assumed); + if (autocorrect > 0) { + fprintf(stderr, "in %0.1f seconds automatically...\n", + (float)autocorrect/10.0); + poll(NULL, 0, autocorrect * 100); + } + return assumed; + } + + fprintf(stderr, "perf: '%s' is not a perf-command. See 'perf --help'.\n", cmd); + + if (main_cmds->cnt && best_similarity < 6) { + fprintf(stderr, "\nDid you mean %s?\n", + n < 2 ? "this": "one of these"); + + for (i = 0; i < n; i++) + fprintf(stderr, "\t%s\n", main_cmds->names[i]->name); + } +end: + clean_cmdnames(&other_cmds); + return NULL; +} diff --git a/tools/perf/util/help-unknown-cmd.h b/tools/perf/util/help-unknown-cmd.h new file mode 100644 index 000000000..e69de29bb diff --git a/tools/perf/util/hisi-ptt-decoder/Build b/tools/perf/util/hisi-ptt-decoder/Build new file mode 100644 index 000000000..2ee0eb731 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/Build @@ -0,0 +1 @@ +perf-util-y += hisi-ptt-pkt-decoder.o diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c new file mode 100644 index 000000000..72a877fd7 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../color.h" +#include "hisi-ptt-pkt-decoder.h" + +/* + * For 8DW format, the bit[31:11] of DW0 is always 0x1fffff, which can be + * used to distinguish the data format. + * 8DW format is like: + * bits [ 31:11 ][ 10:0 ] + * |---------------------------------------|-------------------| + * DW0 [ 0x1fffff ][ Reserved (0x7ff) ] + * DW1 [ Prefix ] + * DW2 [ Header DW0 ] + * DW3 [ Header DW1 ] + * DW4 [ Header DW2 ] + * DW5 [ Header DW3 ] + * DW6 [ Reserved (0x0) ] + * DW7 [ Time ] + * + * 4DW format is like: + * bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ] + * |-----|---------|---|---|---|---|-------------|-------------| + * DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ] + * DW1 [ Header DW1 ] + * DW2 [ Header DW2 ] + * DW3 [ Header DW3 ] + */ + +enum hisi_ptt_8dw_pkt_field_type { + HISI_PTT_8DW_CHK_AND_RSV0, + HISI_PTT_8DW_PREFIX, + HISI_PTT_8DW_HEAD0, + HISI_PTT_8DW_HEAD1, + HISI_PTT_8DW_HEAD2, + HISI_PTT_8DW_HEAD3, + HISI_PTT_8DW_RSV1, + HISI_PTT_8DW_TIME, + HISI_PTT_8DW_TYPE_MAX +}; + +enum hisi_ptt_4dw_pkt_field_type { + HISI_PTT_4DW_HEAD1, + HISI_PTT_4DW_HEAD2, + HISI_PTT_4DW_HEAD3, + HISI_PTT_4DW_TYPE_MAX +}; + +static const char * const hisi_ptt_8dw_pkt_field_name[] = { + [HISI_PTT_8DW_PREFIX] = "Prefix", + [HISI_PTT_8DW_HEAD0] = "Header DW0", + [HISI_PTT_8DW_HEAD1] = "Header DW1", + [HISI_PTT_8DW_HEAD2] = "Header DW2", + [HISI_PTT_8DW_HEAD3] = "Header DW3", + [HISI_PTT_8DW_TIME] = "Time" +}; + +static const char * const hisi_ptt_4dw_pkt_field_name[] = { + [HISI_PTT_4DW_HEAD1] = "Header DW1", + [HISI_PTT_4DW_HEAD2] = "Header DW2", + [HISI_PTT_4DW_HEAD3] = "Header DW3", +}; + +static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc) +{ + const char *color = PERF_COLOR_BLUE; + uint8_t byte; + uint32_t dw; + int i; + + dw = get_unaligned_le32(buf + pos); + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENGTH; i++) { + byte = (dw >> (24 - i * 8)) & 0xFF; + color_fprintf(stdout, color, "%02x ", byte); + } + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + color_fprintf(stdout, color, " %s\n", desc); +} + +static int hisi_ptt_8dw_pkt_desc(const unsigned char *buf, int pos) +{ + int i; + + for (i = 0; i < HISI_PTT_8DW_TYPE_MAX; i++) { + /* Do not show 8DW check field and reserved fields */ + if (i == HISI_PTT_8DW_CHK_AND_RSV0 || i == HISI_PTT_8DW_RSV1) { + pos += HISI_PTT_FIELD_LENGTH; + continue; + } + + hisi_ptt_print_pkt(buf, pos, hisi_ptt_8dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENGTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_8DW_PKT]; +} + +static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos) +{ + const char *color = PERF_COLOR_BLUE; + uint8_t byte; + uint32_t dw; + int i; + + dw = get_unaligned_le32(buf + pos); + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENGTH; i++) { + byte = (dw >> (24 - i * 8)) & 0xFF; + color_fprintf(stdout, color, "%02x ", byte); + } + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + + color_fprintf(stdout, color, + " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n", + "Format", FIELD_GET(HISI_PTT_HEAD0_4DW_FORMAT, dw), + "Type", FIELD_GET(HISI_PTT_HEAD0_4DW_TYPE, dw), + "T9", FIELD_GET(HISI_PTT_HEAD0_4DW_T9, dw), + "T8", FIELD_GET(HISI_PTT_HEAD0_4DW_T8, dw), + "TH", FIELD_GET(HISI_PTT_HEAD0_4DW_TH, dw), + "SO", FIELD_GET(HISI_PTT_HEAD0_4DW_SO, dw), + "Length", FIELD_GET(HISI_PTT_HEAD0_4DW_LEN, dw), + "Time", FIELD_GET(HISI_PTT_HEAD0_4DW_TIME, dw)); +} + +static int hisi_ptt_4dw_pkt_desc(const unsigned char *buf, int pos) +{ + int i; + + hisi_ptt_4dw_print_dw0(buf, pos); + pos += HISI_PTT_FIELD_LENGTH; + + for (i = 0; i < HISI_PTT_4DW_TYPE_MAX; i++) { + hisi_ptt_print_pkt(buf, pos, hisi_ptt_4dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENGTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_4DW_PKT]; +} + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type) +{ + if (type == HISI_PTT_8DW_PKT) + return hisi_ptt_8dw_pkt_desc(buf, pos); + + return hisi_ptt_4dw_pkt_desc(buf, pos); +} diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h new file mode 100644 index 000000000..3b349fcc2 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h @@ -0,0 +1,43 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__HISI_PTT_PKT_DECODER_H__ +#define INCLUDE__HISI_PTT_PKT_DECODER_H__ + +#include +#include +#include +#include + +#define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11) +#define HISI_PTT_IS_8DW_PKT GENMASK(31, 11) +#define HISI_PTT_MAX_SPACE_LEN 10 +#define HISI_PTT_FIELD_LENGTH 4 + +/* Header DW0 fields for 4DW format */ +#define HISI_PTT_HEAD0_4DW_TIME GENMASK_U32(10, 0) +#define HISI_PTT_HEAD0_4DW_LEN GENMASK_U32(20, 11) +#define HISI_PTT_HEAD0_4DW_SO BIT_U32(21) +#define HISI_PTT_HEAD0_4DW_TH BIT_U32(22) +#define HISI_PTT_HEAD0_4DW_T8 BIT_U32(23) +#define HISI_PTT_HEAD0_4DW_T9 BIT_U32(24) +#define HISI_PTT_HEAD0_4DW_TYPE GENMASK_U32(29, 25) +#define HISI_PTT_HEAD0_4DW_FORMAT GENMASK_U32(31, 30) + +enum hisi_ptt_pkt_type { + HISI_PTT_4DW_PKT, + HISI_PTT_8DW_PKT, + HISI_PTT_PKT_MAX +}; + +static int hisi_ptt_pkt_size[] = { + [HISI_PTT_4DW_PKT] = 16, + [HISI_PTT_8DW_PKT] = 32, +}; + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type); + +#endif diff --git a/tools/perf/util/hisi-ptt.c b/tools/perf/util/hisi-ptt.c new file mode 100644 index 000000000..e73c57f9e --- /dev/null +++ b/tools/perf/util/hisi-ptt.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "auxtrace.h" +#include "color.h" +#include "debug.h" +#include "evsel.h" +#include "hisi-ptt.h" +#include "hisi-ptt-decoder/hisi-ptt-pkt-decoder.h" +#include "machine.h" +#include "session.h" +#include "tool.h" +#include + +struct hisi_ptt { + struct auxtrace auxtrace; + u32 auxtrace_type; + struct perf_session *session; + struct machine *machine; + u32 pmu_type; +}; + +static enum hisi_ptt_pkt_type hisi_ptt_check_packet_type(unsigned char *buf, + size_t len) +{ + uint32_t head; + + if (len < HISI_PTT_FIELD_LENGTH) + return HISI_PTT_4DW_PKT; + + head = get_unaligned_le32(buf); + + if ((HISI_PTT_8DW_CHECK_MASK & head) == HISI_PTT_IS_8DW_PKT) + return HISI_PTT_8DW_PKT; + + return HISI_PTT_4DW_PKT; +} + +static void hisi_ptt_dump(struct hisi_ptt *ptt __maybe_unused, + unsigned char *buf, size_t len) +{ + const char *color = PERF_COLOR_BLUE; + enum hisi_ptt_pkt_type type; + size_t pos = 0; + int pkt_len; + + type = hisi_ptt_check_packet_type(buf, len); + len = round_down(len, hisi_ptt_pkt_size[type]); + color_fprintf(stdout, color, ". ... HISI PTT data: size %zu bytes\n", + len); + + while (len > 0) { + pkt_len = hisi_ptt_pkt_desc(buf, pos, type); + if (!pkt_len) + color_fprintf(stdout, color, " Bad packet!\n"); + + pos += pkt_len; + len -= pkt_len; + } +} + +static void hisi_ptt_dump_event(struct hisi_ptt *ptt, unsigned char *buf, + size_t len) +{ + printf(".\n"); + + hisi_ptt_dump(ptt, buf, len); +} + +static int hisi_ptt_process_event(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + const struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static int hisi_ptt_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + int fd = perf_data__fd(session->data); + u64 size = event->auxtrace.size; + off_t data_offset; + ssize_t err; + void *data; + + if (size > SSIZE_MAX) + return -EINVAL; + + data = malloc(size); + if (!data) + return -errno; + + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) { + free(data); + return -errno; + } + } + + err = readn(fd, data, size); + if (err != (ssize_t)size) { + free(data); + return -errno; + } + + if (dump_trace) + hisi_ptt_dump_event(ptt, data, size); + + free(data); + return 0; +} + +static int hisi_ptt_flush(struct perf_session *session __maybe_unused, + const struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static void hisi_ptt_free_events(struct perf_session *session __maybe_unused) +{ +} + +static void hisi_ptt_free(struct perf_session *session) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + + session->auxtrace = NULL; + free(ptt); +} + +static bool hisi_ptt_evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, auxtrace); + + return evsel->core.attr.type == ptt->pmu_type; +} + +static void hisi_ptt_print_info(__u64 type) +{ + if (!dump_trace) + return; + + fprintf(stdout, " PMU Type %" PRId64 "\n", (s64) type); +} + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + struct hisi_ptt *ptt; + + if (auxtrace_info->header.size < HISI_PTT_AUXTRACE_PRIV_SIZE + + sizeof(struct perf_record_auxtrace_info)) + return -EINVAL; + + ptt = zalloc(sizeof(*ptt)); + if (!ptt) + return -ENOMEM; + + ptt->session = session; + ptt->machine = &session->machines.host; /* No kvm support */ + ptt->auxtrace_type = auxtrace_info->type; + ptt->pmu_type = auxtrace_info->priv[0]; + + ptt->auxtrace.process_event = hisi_ptt_process_event; + ptt->auxtrace.process_auxtrace_event = hisi_ptt_process_auxtrace_event; + ptt->auxtrace.flush_events = hisi_ptt_flush; + ptt->auxtrace.free_events = hisi_ptt_free_events; + ptt->auxtrace.free = hisi_ptt_free; + ptt->auxtrace.evsel_is_auxtrace = hisi_ptt_evsel_is_auxtrace; + session->auxtrace = &ptt->auxtrace; + + hisi_ptt_print_info(auxtrace_info->priv[0]); + + return 0; +} diff --git a/tools/perf/util/hisi-ptt.h b/tools/perf/util/hisi-ptt.h new file mode 100644 index 000000000..2db9b4056 --- /dev/null +++ b/tools/perf/util/hisi-ptt.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__PERF_HISI_PTT_H__ +#define INCLUDE__PERF_HISI_PTT_H__ + +#define HISI_PTT_PMU_NAME "hisi_ptt" +#define HISI_PTT_AUXTRACE_PRIV_SIZE sizeof(u64) + +struct auxtrace_record *hisi_ptt_recording_init(int *err, + struct perf_pmu *hisi_ptt_pmu); + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session); + +#endif diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c new file mode 100644 index 000000000..443694926 --- /dev/null +++ b/tools/perf/util/hist.c @@ -0,0 +1,3100 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "callchain.h" +#include "debug.h" +#include "dso.h" +#include "build-id.h" +#include "hist.h" +#include "kvm-stat.h" +#include "map.h" +#include "map_symbol.h" +#include "branch.h" +#include "mem-events.h" +#include "mem-info.h" +#include "session.h" +#include "namespaces.h" +#include "cgroup.h" +#include "sort.h" +#include "units.h" +#include "evlist.h" +#include "evsel.h" +#include "annotate.h" +#include "srcline.h" +#include "symbol.h" +#include "thread.h" +#include "block-info.h" +#include "ui/progress.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static int64_t hist_entry__cmp(struct hist_entry *left, struct hist_entry *right); +static int64_t hist_entry__collapse(struct hist_entry *left, struct hist_entry *right); + +static bool hists__filter_entry_by_dso(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_thread(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_symbol(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_socket(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_parallelism(struct hists *hists, + struct hist_entry *he); + +u16 hists__col_len(struct hists *hists, enum hist_column col) +{ + return hists->col_len[col]; +} + +void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len) +{ + hists->col_len[col] = len; +} + +bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len) +{ + if (len > hists__col_len(hists, col)) { + hists__set_col_len(hists, col, len); + return true; + } + return false; +} + +void hists__reset_col_len(struct hists *hists) +{ + enum hist_column col; + + for (col = 0; col < HISTC_NR_COLS; ++col) + hists__set_col_len(hists, col, 0); +} + +static void hists__set_unres_dso_col_len(struct hists *hists, int dso) +{ + const unsigned int unresolved_col_width = BITS_PER_LONG / 4; + + if (hists__col_len(hists, dso) < unresolved_col_width && + !symbol_conf.col_width_list_str && !symbol_conf.field_sep && + !symbol_conf.dso_list) + hists__set_col_len(hists, dso, unresolved_col_width); +} + +void hists__calc_col_len(struct hists *hists, struct hist_entry *h) +{ + const unsigned int unresolved_col_width = BITS_PER_LONG / 4; + int symlen; + u16 len; + + if (h->block_info) + return; + /* + * +4 accounts for '[x] ' priv level info + * +2 accounts for 0x prefix on raw addresses + * +3 accounts for ' y ' symtab origin info + */ + if (h->ms.sym) { + symlen = h->ms.sym->namelen + 4; + if (verbose > 0) + symlen += BITS_PER_LONG / 4 + 2 + 3; + hists__new_col_len(hists, HISTC_SYMBOL, symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_SYMBOL, symlen); + hists__set_unres_dso_col_len(hists, HISTC_DSO); + } + + len = thread__comm_len(h->thread); + if (hists__new_col_len(hists, HISTC_COMM, len)) + hists__set_col_len(hists, HISTC_THREAD, len + 8); + if (hists->hpp_list->comm_nodigit) + hists__new_col_len(hists, HISTC_COMM_NODIGIT, + (u16) sort__comm_nodigit_len(h)); + + if (h->ms.map) { + len = dso__name_len(map__dso(h->ms.map)); + hists__new_col_len(hists, HISTC_DSO, len); + } + + if (h->parent) + hists__new_col_len(hists, HISTC_PARENT, h->parent->namelen); + + if (h->branch_info) { + if (h->branch_info->from.ms.sym) { + symlen = (int)h->branch_info->from.ms.sym->namelen + 4; + if (verbose > 0) + symlen += BITS_PER_LONG / 4 + 2 + 3; + hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen); + + symlen = dso__name_len(map__dso(h->branch_info->from.ms.map)); + hists__new_col_len(hists, HISTC_DSO_FROM, symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen); + hists__new_col_len(hists, HISTC_ADDR_FROM, symlen); + hists__set_unres_dso_col_len(hists, HISTC_DSO_FROM); + } + + if (h->branch_info->to.ms.sym) { + symlen = (int)h->branch_info->to.ms.sym->namelen + 4; + if (verbose > 0) + symlen += BITS_PER_LONG / 4 + 2 + 3; + hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen); + + symlen = dso__name_len(map__dso(h->branch_info->to.ms.map)); + hists__new_col_len(hists, HISTC_DSO_TO, symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen); + hists__new_col_len(hists, HISTC_ADDR_TO, symlen); + hists__set_unres_dso_col_len(hists, HISTC_DSO_TO); + } + + if (h->branch_info->srcline_from) + hists__new_col_len(hists, HISTC_SRCLINE_FROM, + strlen(h->branch_info->srcline_from)); + if (h->branch_info->srcline_to) + hists__new_col_len(hists, HISTC_SRCLINE_TO, + strlen(h->branch_info->srcline_to)); + } + + if (h->mem_info) { + if (mem_info__daddr(h->mem_info)->ms.sym) { + symlen = (int)mem_info__daddr(h->mem_info)->ms.sym->namelen + 4 + + unresolved_col_width + 2; + hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, + symlen); + hists__new_col_len(hists, HISTC_MEM_DCACHELINE, + symlen + 1); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, + symlen); + hists__new_col_len(hists, HISTC_MEM_DCACHELINE, + symlen); + } + + if (mem_info__iaddr(h->mem_info)->ms.sym) { + symlen = (int)mem_info__iaddr(h->mem_info)->ms.sym->namelen + 4 + + unresolved_col_width + 2; + hists__new_col_len(hists, HISTC_MEM_IADDR_SYMBOL, + symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_MEM_IADDR_SYMBOL, + symlen); + } + + if (mem_info__daddr(h->mem_info)->ms.map) { + symlen = dso__name_len(map__dso(mem_info__daddr(h->mem_info)->ms.map)); + hists__new_col_len(hists, HISTC_MEM_DADDR_DSO, + symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO); + } + + hists__new_col_len(hists, HISTC_MEM_PHYS_DADDR, + unresolved_col_width + 4 + 2); + + hists__new_col_len(hists, HISTC_MEM_DATA_PAGE_SIZE, + unresolved_col_width + 4 + 2); + + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, symlen); + hists__new_col_len(hists, HISTC_MEM_IADDR_SYMBOL, symlen); + hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO); + } + + hists__new_col_len(hists, HISTC_CGROUP, 6); + hists__new_col_len(hists, HISTC_CGROUP_ID, 20); + hists__new_col_len(hists, HISTC_PARALLELISM, 11); + hists__new_col_len(hists, HISTC_CPU, 3); + hists__new_col_len(hists, HISTC_SOCKET, 6); + hists__new_col_len(hists, HISTC_MEM_LOCKED, 6); + hists__new_col_len(hists, HISTC_MEM_TLB, 22); + hists__new_col_len(hists, HISTC_MEM_SNOOP, 12); + hists__new_col_len(hists, HISTC_MEM_LVL, 36 + 3); + hists__new_col_len(hists, HISTC_LOCAL_WEIGHT, 12); + hists__new_col_len(hists, HISTC_GLOBAL_WEIGHT, 12); + hists__new_col_len(hists, HISTC_MEM_BLOCKED, 10); + hists__new_col_len(hists, HISTC_LOCAL_INS_LAT, 13); + hists__new_col_len(hists, HISTC_GLOBAL_INS_LAT, 13); + hists__new_col_len(hists, HISTC_LOCAL_P_STAGE_CYC, 13); + hists__new_col_len(hists, HISTC_GLOBAL_P_STAGE_CYC, 13); + hists__new_col_len(hists, HISTC_ADDR, BITS_PER_LONG / 4 + 2); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_PREDICTED, 9); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_ABORT, 5); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_CYCLES, 6); + + if (symbol_conf.nanosecs) + hists__new_col_len(hists, HISTC_TIME, 16); + else + hists__new_col_len(hists, HISTC_TIME, 12); + hists__new_col_len(hists, HISTC_CODE_PAGE_SIZE, 6); + + if (h->srcline) { + len = MAX(strlen(h->srcline), strlen(sort_srcline.se_header)); + hists__new_col_len(hists, HISTC_SRCLINE, len); + } + + if (h->srcfile) + hists__new_col_len(hists, HISTC_SRCFILE, strlen(h->srcfile)); + + if (h->transaction) + hists__new_col_len(hists, HISTC_TRANSACTION, + hist_entry__transaction_len()); + + if (h->trace_output) + hists__new_col_len(hists, HISTC_TRACE, strlen(h->trace_output)); + + if (h->cgroup) { + const char *cgrp_name = "unknown"; + struct cgroup *cgrp = cgroup__find(maps__machine(thread__maps(h->ms.thread))->env, + h->cgroup); + if (cgrp != NULL) + cgrp_name = cgrp->name; + + hists__new_col_len(hists, HISTC_CGROUP, strlen(cgrp_name)); + } +} + +void hists__output_recalc_col_len(struct hists *hists, int max_rows) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + int row = 0; + + hists__reset_col_len(hists); + + while (next && row++ < max_rows) { + n = rb_entry(next, struct hist_entry, rb_node); + if (!n->filtered) + hists__calc_col_len(hists, n); + next = rb_next(&n->rb_node); + } +} + +static void he_stat__add_cpumode_period(struct he_stat *he_stat, + unsigned int cpumode, u64 period) +{ + switch (cpumode) { + case PERF_RECORD_MISC_KERNEL: + he_stat->period_sys += period; + break; + case PERF_RECORD_MISC_USER: + he_stat->period_us += period; + break; + case PERF_RECORD_MISC_GUEST_KERNEL: + he_stat->period_guest_sys += period; + break; + case PERF_RECORD_MISC_GUEST_USER: + he_stat->period_guest_us += period; + break; + default: + break; + } +} + +static long hist_time(unsigned long htime) +{ + unsigned long time_quantum = symbol_conf.time_quantum; + if (time_quantum) + return (htime / time_quantum) * time_quantum; + return htime; +} + +static void he_stat__add_period(struct he_stat *he_stat, u64 period, u64 latency) +{ + he_stat->period += period; + he_stat->latency += latency; + he_stat->nr_events += 1; +} + +static void he_stat__add_stat(struct he_stat *dest, struct he_stat *src) +{ + dest->period += src->period; + dest->period_sys += src->period_sys; + dest->period_us += src->period_us; + dest->period_guest_sys += src->period_guest_sys; + dest->period_guest_us += src->period_guest_us; + dest->weight1 += src->weight1; + dest->weight2 += src->weight2; + dest->weight3 += src->weight3; + dest->nr_events += src->nr_events; + dest->latency += src->latency; +} + +static void he_stat__decay(struct he_stat *he_stat) +{ + he_stat->period = (he_stat->period * 7) / 8; + he_stat->nr_events = (he_stat->nr_events * 7) / 8; + he_stat->weight1 = (he_stat->weight1 * 7) / 8; + he_stat->weight2 = (he_stat->weight2 * 7) / 8; + he_stat->weight3 = (he_stat->weight3 * 7) / 8; + he_stat->latency = (he_stat->latency * 7) / 8; +} + +static int hists__update_mem_stat(struct hists *hists, struct hist_entry *he, + struct mem_info *mi, u64 period) +{ + if (hists->nr_mem_stats == 0) + return 0; + + if (he->mem_stat == NULL) { + he->mem_stat = calloc(hists->nr_mem_stats, sizeof(*he->mem_stat)); + if (he->mem_stat == NULL) + return -1; + } + + for (int i = 0; i < hists->nr_mem_stats; i++) { + int idx = mem_stat_index(hists->mem_stat_types[i], + mem_info__const_data_src(mi)->val); + + assert(0 <= idx && idx < MEM_STAT_LEN); + he->mem_stat[i].entries[idx] += period; + hists->mem_stat_total[i].entries[idx] += period; + } + return 0; +} + +static void hists__add_mem_stat(struct hists *hists, struct hist_entry *dst, + struct hist_entry *src) +{ + if (hists->nr_mem_stats == 0) + return; + + for (int i = 0; i < hists->nr_mem_stats; i++) { + for (int k = 0; k < MEM_STAT_LEN; k++) + dst->mem_stat[i].entries[k] += src->mem_stat[i].entries[k]; + } +} + +static int hists__clone_mem_stat(struct hists *hists, struct hist_entry *dst, + struct hist_entry *src) +{ + if (hists->nr_mem_stats == 0) + return 0; + + dst->mem_stat = calloc(hists->nr_mem_stats, sizeof(*dst->mem_stat)); + if (dst->mem_stat == NULL) + return -1; + + for (int i = 0; i < hists->nr_mem_stats; i++) { + for (int k = 0; k < MEM_STAT_LEN; k++) + dst->mem_stat[i].entries[k] = src->mem_stat[i].entries[k]; + } + return 0; +} + +static void hists__decay_mem_stat(struct hists *hists, struct hist_entry *he) +{ + if (hists->nr_mem_stats == 0) + return; + + for (int i = 0; i < hists->nr_mem_stats; i++) { + for (int k = 0; k < MEM_STAT_LEN; k++) + he->mem_stat[i].entries[k] = (he->mem_stat[i].entries[k] * 7) / 8; + } +} + +static void hists__delete_entry(struct hists *hists, struct hist_entry *he); + +static bool hists__decay_entry(struct hists *hists, struct hist_entry *he) +{ + u64 prev_period = he->stat.period; + u64 prev_latency = he->stat.latency; + + if (prev_period == 0) + return true; + + he_stat__decay(&he->stat); + if (symbol_conf.cumulate_callchain) + he_stat__decay(he->stat_acc); + decay_callchain(he->callchain); + hists__decay_mem_stat(hists, he); + + if (!he->depth) { + u64 period_diff = prev_period - he->stat.period; + u64 latency_diff = prev_latency - he->stat.latency; + + hists->stats.total_period -= period_diff; + hists->stats.total_latency -= latency_diff; + if (!he->filtered) { + hists->stats.total_non_filtered_period -= period_diff; + hists->stats.total_non_filtered_latency -= latency_diff; + } + } + + if (!he->leaf) { + struct hist_entry *child; + struct rb_node *node = rb_first_cached(&he->hroot_out); + while (node) { + child = rb_entry(node, struct hist_entry, rb_node); + node = rb_next(node); + + if (hists__decay_entry(hists, child)) + hists__delete_entry(hists, child); + } + } + + return he->stat.period == 0 && he->stat.latency == 0; +} + +static void hists__delete_entry(struct hists *hists, struct hist_entry *he) +{ + struct rb_root_cached *root_in; + struct rb_root_cached *root_out; + + if (he->parent_he) { + root_in = &he->parent_he->hroot_in; + root_out = &he->parent_he->hroot_out; + } else { + if (hists__has(hists, need_collapse)) + root_in = &hists->entries_collapsed; + else + root_in = hists->entries_in; + root_out = &hists->entries; + } + + rb_erase_cached(&he->rb_node_in, root_in); + rb_erase_cached(&he->rb_node, root_out); + + --hists->nr_entries; + if (!he->filtered) + --hists->nr_non_filtered_entries; + + hist_entry__delete(he); +} + +void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node); + next = rb_next(&n->rb_node); + if (((zap_user && n->level == '.') || + (zap_kernel && n->level != '.') || + hists__decay_entry(hists, n))) { + hists__delete_entry(hists, n); + } + } +} + +void hists__delete_entries(struct hists *hists) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node); + next = rb_next(&n->rb_node); + + hists__delete_entry(hists, n); + } +} + +struct hist_entry *hists__get_entry(struct hists *hists, int idx) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + int i = 0; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node); + if (i == idx) + return n; + + next = rb_next(&n->rb_node); + i++; + } + + return NULL; +} + +/* + * histogram, sorted on item, collects periods + */ + +static int hist_entry__init(struct hist_entry *he, + struct hist_entry *template, + bool sample_self, + size_t callchain_size) +{ + *he = *template; + he->callchain_size = callchain_size; + + if (symbol_conf.cumulate_callchain) { + he->stat_acc = malloc(sizeof(he->stat)); + if (he->stat_acc == NULL) + return -ENOMEM; + memcpy(he->stat_acc, &he->stat, sizeof(he->stat)); + if (!sample_self) + memset(&he->stat, 0, sizeof(he->stat)); + } + + he->ms.thread = thread__get(he->ms.thread); + he->ms.map = map__get(he->ms.map); + + if (he->branch_info) { + /* + * This branch info is (a part of) allocated from + * sample__resolve_bstack() and will be freed after + * adding new entries. So we need to save a copy. + */ + he->branch_info = malloc(sizeof(*he->branch_info)); + if (he->branch_info == NULL) + goto err; + + memcpy(he->branch_info, template->branch_info, + sizeof(*he->branch_info)); + + he->branch_info->from.ms.thread = thread__get(he->branch_info->from.ms.thread); + he->branch_info->from.ms.map = map__get(he->branch_info->from.ms.map); + he->branch_info->to.ms.thread = thread__get(he->branch_info->to.ms.thread); + he->branch_info->to.ms.map = map__get(he->branch_info->to.ms.map); + } + + if (he->mem_info) { + he->mem_info = mem_info__clone(template->mem_info); + if (he->mem_info == NULL) + goto err_infos; + } + + if (hist_entry__has_callchains(he) && symbol_conf.use_callchain) + callchain_init(he->callchain); + + if (he->raw_data) { + he->raw_data = memdup(he->raw_data, he->raw_size); + if (he->raw_data == NULL) + goto err_infos; + } + + if (he->srcline && he->srcline != SRCLINE_UNKNOWN) { + he->srcline = strdup(he->srcline); + if (he->srcline == NULL) + goto err_rawdata; + } + + if (symbol_conf.res_sample) { + he->res_samples = calloc(symbol_conf.res_sample, + sizeof(struct res_sample)); + if (!he->res_samples) + goto err_srcline; + } + + INIT_LIST_HEAD(&he->pairs.node); + he->thread = thread__get(he->thread); + he->hroot_in = RB_ROOT_CACHED; + he->hroot_out = RB_ROOT_CACHED; + + if (!symbol_conf.report_hierarchy) + he->leaf = true; + + return 0; + +err_srcline: + zfree(&he->srcline); + +err_rawdata: + zfree(&he->raw_data); + +err_infos: + if (he->branch_info) { + map_symbol__exit(&he->branch_info->from.ms); + map_symbol__exit(&he->branch_info->to.ms); + zfree(&he->branch_info); + } + if (he->mem_info) + mem_info__zput(he->mem_info); +err: + map_symbol__exit(&he->ms); + zfree(&he->stat_acc); + return -ENOMEM; +} + +static void *hist_entry__zalloc(size_t size) +{ + return zalloc(size + sizeof(struct hist_entry)); +} + +static void hist_entry__free(void *ptr) +{ + free(ptr); +} + +static struct hist_entry_ops default_ops = { + .new = hist_entry__zalloc, + .free = hist_entry__free, +}; + +static struct hist_entry *hist_entry__new(struct hist_entry *template, + bool sample_self) +{ + struct hist_entry_ops *ops = template->ops; + size_t callchain_size = 0; + struct hist_entry *he; + int err = 0; + + if (!ops) + ops = template->ops = &default_ops; + + if (symbol_conf.use_callchain) + callchain_size = sizeof(struct callchain_root); + + he = ops->new(callchain_size); + if (he) { + err = hist_entry__init(he, template, sample_self, callchain_size); + if (err) { + ops->free(he); + he = NULL; + } + } + return he; +} + +static filter_mask_t symbol__parent_filter(const struct symbol *parent) +{ + if (symbol_conf.exclude_other && parent == NULL) + return 1 << HIST_FILTER__PARENT; + return 0; +} + +static void hist_entry__add_callchain_period(struct hist_entry *he, u64 period, u64 latency) +{ + if (!hist_entry__has_callchains(he) || !symbol_conf.use_callchain) + return; + + he->hists->callchain_period += period; + he->hists->callchain_latency += latency; + if (!he->filtered) { + he->hists->callchain_non_filtered_period += period; + he->hists->callchain_non_filtered_latency += latency; + } +} + +static struct hist_entry *hists__findnew_entry(struct hists *hists, + struct hist_entry *entry, + const struct addr_location *al, + bool sample_self) +{ + struct rb_node **p; + struct rb_node *parent = NULL; + struct hist_entry *he; + int64_t cmp; + u64 period = entry->stat.period; + u64 latency = entry->stat.latency; + bool leftmost = true; + + p = &hists->entries_in->rb_root.rb_node; + + while (*p != NULL) { + parent = *p; + he = rb_entry(parent, struct hist_entry, rb_node_in); + + /* + * Make sure that it receives arguments in a same order as + * hist_entry__collapse() so that we can use an appropriate + * function when searching an entry regardless which sort + * keys were used. + */ + cmp = hist_entry__cmp(he, entry); + if (!cmp) { + if (sample_self) { + he_stat__add_stat(&he->stat, &entry->stat); + hist_entry__add_callchain_period(he, period, latency); + } + if (symbol_conf.cumulate_callchain) + he_stat__add_period(he->stat_acc, period, latency); + + block_info__delete(entry->block_info); + + kvm_info__zput(entry->kvm_info); + + /* If the map of an existing hist_entry has + * become out-of-date due to an exec() or + * similar, update it. Otherwise we will + * mis-adjust symbol addresses when computing + * the history counter to increment. + */ + if (hists__has(hists, sym) && he->ms.map != entry->ms.map) { + if (he->ms.sym) { + u64 addr = he->ms.sym->start; + he->ms.sym = map__find_symbol(entry->ms.map, addr); + } + + map__put(he->ms.map); + he->ms.map = map__get(entry->ms.map); + } + goto out; + } + + if (cmp < 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + he = hist_entry__new(entry, sample_self); + if (!he) + return NULL; + + if (sample_self) + hist_entry__add_callchain_period(he, period, latency); + hists->nr_entries++; + + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, hists->entries_in, leftmost); +out: + if (sample_self) + he_stat__add_cpumode_period(&he->stat, al->cpumode, period); + if (symbol_conf.cumulate_callchain) + he_stat__add_cpumode_period(he->stat_acc, al->cpumode, period); + if (hists__update_mem_stat(hists, he, entry->mem_info, period) < 0) { + hist_entry__delete(he); + return NULL; + } + return he; +} + +static unsigned random_max(unsigned high) +{ + unsigned thresh = -high % high; + for (;;) { + unsigned r = random(); + if (r >= thresh) + return r % high; + } +} + +static void hists__res_sample(struct hist_entry *he, struct perf_sample *sample) +{ + struct res_sample *r; + int j; + + if (he->num_res < symbol_conf.res_sample) { + j = he->num_res++; + } else { + j = random_max(symbol_conf.res_sample); + } + r = &he->res_samples[j]; + r->time = sample->time; + r->cpu = sample->cpu; + r->tid = sample->tid; +} + +static struct hist_entry* +__hists__add_entry(struct hists *hists, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct block_info *block_info, + struct perf_sample *sample, + bool sample_self, + struct hist_entry_ops *ops) +{ + struct namespaces *ns = thread__namespaces(al->thread); + struct hist_entry entry = { + .thread = al->thread, + .comm = thread__comm(al->thread), + .cgroup_id = { + .dev = ns ? ns->link_info[CGROUP_NS_INDEX].dev : 0, + .ino = ns ? ns->link_info[CGROUP_NS_INDEX].ino : 0, + }, + .cgroup = sample->cgroup, + .ms = { + .thread = al->thread, + .map = al->map, + .sym = al->sym, + }, + .srcline = (char *) al->srcline, + .socket = al->socket, + .cpu = al->cpu, + .cpumode = al->cpumode, + .ip = al->addr, + .level = al->level, + .code_page_size = sample->code_page_size, + .parallelism = al->parallelism, + .stat = { + .nr_events = 1, + .period = sample->period, + .weight1 = sample->weight, + .weight2 = sample->ins_lat, + .weight3 = sample->weight3, + .latency = al->latency, + }, + .parent = sym_parent, + .filtered = symbol__parent_filter(sym_parent) | al->filtered, + .hists = hists, + .branch_info = bi, + .mem_info = mi, + .kvm_info = ki, + .block_info = block_info, + .transaction = sample->transaction, + .raw_data = sample->raw_data, + .raw_size = sample->raw_size, + .ops = ops, + .time = hist_time(sample->time), + .weight = sample->weight, + .ins_lat = sample->ins_lat, + .weight3 = sample->weight3, + .simd_flags = sample->simd_flags, + }, *he = hists__findnew_entry(hists, &entry, al, sample_self); + + if (!hists->has_callchains && he && he->callchain_size != 0) + hists->has_callchains = true; + if (he && symbol_conf.res_sample) + hists__res_sample(he, sample); + return he; +} + +struct hist_entry *hists__add_entry(struct hists *hists, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self) +{ + return __hists__add_entry(hists, al, sym_parent, bi, mi, ki, NULL, + sample, sample_self, NULL); +} + +struct hist_entry *hists__add_entry_ops(struct hists *hists, + struct hist_entry_ops *ops, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self) +{ + return __hists__add_entry(hists, al, sym_parent, bi, mi, ki, NULL, + sample, sample_self, ops); +} + +struct hist_entry *hists__add_entry_block(struct hists *hists, + struct addr_location *al, + struct block_info *block_info) +{ + struct hist_entry entry = { + .block_info = block_info, + .hists = hists, + .ms = { + .thread = al->thread, + .map = al->map, + .sym = al->sym, + }, + }, *he = hists__findnew_entry(hists, &entry, al, false); + + return he; +} + +static int +iter_next_nop_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_add_next_nop_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_prepare_mem_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct mem_info *mi; + + mi = sample__resolve_mem(sample, al); + if (mi == NULL) + return -ENOMEM; + + iter->mi = mi; + return 0; +} + +static int +iter_add_single_mem_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + u64 cost; + struct mem_info *mi = iter->mi; + struct perf_sample *sample = iter->sample; + struct hists *hists = evsel__hists(sample->evsel); + struct hist_entry *he; + + if (mi == NULL) + return -EINVAL; + + cost = sample->weight; + if (!cost) + cost = 1; + + /* + * must pass period=weight in order to get the correct + * sorting from hists__collapse_resort() which is solely + * based on periods. We want sorting be done on nr_events * weight + * and this is indirectly achieved by passing period=weight here + * and the he_stat__add_period() function. + */ + sample->period = cost; + + he = hists__add_entry(hists, al, iter->parent, NULL, mi, NULL, + sample, true); + if (!he) + return -ENOMEM; + + iter->he = he; + return 0; +} + +static int +iter_finish_mem_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct evsel *evsel = iter->sample->evsel; + struct hists *hists = evsel__hists(evsel); + struct hist_entry *he = iter->he; + int err = -EINVAL; + + if (he == NULL) + goto out; + + hists__inc_nr_samples(hists, he->filtered); + + err = hist_entry__append_callchain(he, iter->sample); + +out: + mem_info__zput(iter->mi); + + iter->he = NULL; + return err; +} + +static int +iter_prepare_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct branch_info *bi; + struct perf_sample *sample = iter->sample; + + bi = sample__resolve_bstack(sample, al); + if (!bi) + return -ENOMEM; + + iter->curr = 0; + iter->total = sample->branch_stack->nr; + + iter->bi = bi; + return 0; +} + +static int +iter_add_single_branch_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct branch_info *bi = iter->bi; + int i = iter->curr; + + if (bi == NULL) + return 0; + + if (iter->curr >= iter->total) + return 0; + + thread__put(al->thread); + al->thread = thread__get(bi[i].to.ms.thread); + map__put(al->map); + al->map = map__get(bi[i].to.ms.map); + al->sym = bi[i].to.ms.sym; + al->addr = bi[i].to.addr; + return 1; +} + +static int +iter_add_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct branch_info *bi; + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); + struct hist_entry *he = NULL; + int i = iter->curr; + int err = 0; + + bi = iter->bi; + + if (iter->hide_unresolved && !(bi[i].from.ms.sym && bi[i].to.ms.sym)) + goto out; + + /* + * The report shows the percentage of total branches captured + * and not events sampled. Thus we use a pseudo period of 1. + */ + sample->period = 1; + sample->weight = bi->flags.cycles ? bi->flags.cycles : 1; + + he = hists__add_entry(hists, al, iter->parent, &bi[i], NULL, NULL, + sample, true); + if (he == NULL) + return -ENOMEM; + +out: + iter->he = he; + iter->curr++; + return err; +} + +static void branch_info__exit(struct branch_info *bi) +{ + map_symbol__exit(&bi->from.ms); + map_symbol__exit(&bi->to.ms); + zfree_srcline(&bi->srcline_from); + zfree_srcline(&bi->srcline_to); +} + +static int +iter_finish_branch_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct evsel *evsel = iter->sample->evsel; + struct hists *hists = evsel__hists(evsel); + + for (int i = 0; i < iter->total; i++) + branch_info__exit(&iter->bi[i]); + + if (iter->he) + hists__inc_nr_samples(hists, iter->he->filtered); + + zfree(&iter->bi); + iter->he = NULL; + + return iter->curr >= iter->total ? 0 : -1; +} + +static int +iter_prepare_normal_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_add_single_normal_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hist_entry *he; + + he = hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, + NULL, sample, true); + if (he == NULL) + return -ENOMEM; + + iter->he = he; + return 0; +} + +static int +iter_finish_normal_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct hist_entry *he = iter->he; + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + + if (he == NULL) + return 0; + + iter->he = NULL; + + hists__inc_nr_samples(evsel__hists(evsel), he->filtered); + + return hist_entry__append_callchain(he, sample); +} + +static int +iter_prepare_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct hist_entry **he_cache; + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + + if (cursor == NULL) + return -ENOMEM; + + callchain_cursor_commit(cursor); + + /* + * This is for detecting cycles or recursions so that they're + * cumulated only one time to prevent entries more than 100% + * overhead. + */ + he_cache = calloc(cursor->nr + 1, sizeof(*he_cache)); + if (he_cache == NULL) + return -ENOMEM; + + iter->he_cache = he_cache; + iter->curr = 0; + + return 0; +} + +static int +iter_add_single_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); + struct hist_entry **he_cache = iter->he_cache; + struct hist_entry *he; + int err = 0; + + he = hists__add_entry(hists, al, iter->parent, NULL, NULL, NULL, + sample, true); + if (he == NULL) + return -ENOMEM; + + iter->he = he; + he_cache[iter->curr++] = he; + + hist_entry__append_callchain(he, sample); + + /* + * We need to re-initialize the cursor since callchain_append() + * advanced the cursor to the end. + */ + callchain_cursor_commit(get_tls_callchain_cursor()); + + hists__inc_nr_samples(hists, he->filtered); + + return err; +} + +static int +iter_next_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al) +{ + struct callchain_cursor_node *node; + + node = callchain_cursor_current(get_tls_callchain_cursor()); + if (node == NULL) + return 0; + + return fill_callchain_info(al, node, iter->hide_unresolved); +} + +static bool +hist_entry__fast__sym_diff(struct hist_entry *left, + struct hist_entry *right) +{ + struct symbol *sym_l = left->ms.sym; + struct symbol *sym_r = right->ms.sym; + + if (!sym_l && !sym_r) + return left->ip != right->ip; + + return !!_sort__sym_cmp(sym_l, sym_r); +} + + +static int +iter_add_next_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hist_entry **he_cache = iter->he_cache; + struct hist_entry *he; + struct hist_entry he_tmp = { + .hists = evsel__hists(evsel), + .cpu = al->cpu, + .thread = al->thread, + .comm = thread__comm(al->thread), + .ip = al->addr, + .ms = { + .thread = al->thread, + .map = al->map, + .sym = al->sym, + }, + .srcline = (char *) al->srcline, + .parent = iter->parent, + .raw_data = sample->raw_data, + .raw_size = sample->raw_size, + }; + int i; + struct callchain_cursor cursor, *tls_cursor = get_tls_callchain_cursor(); + bool fast = hists__has(he_tmp.hists, sym); + + if (tls_cursor == NULL) + return -ENOMEM; + + callchain_cursor_snapshot(&cursor, tls_cursor); + + callchain_cursor_advance(tls_cursor); + + /* + * Check if there's duplicate entries in the callchain. + * It's possible that it has cycles or recursive calls. + */ + for (i = 0; i < iter->curr; i++) { + /* + * For most cases, there are no duplicate entries in callchain. + * The symbols are usually different. Do a quick check for + * symbols first. + */ + if (fast && hist_entry__fast__sym_diff(he_cache[i], &he_tmp)) + continue; + + if (hist_entry__cmp(he_cache[i], &he_tmp) == 0) { + /* to avoid calling callback function */ + iter->he = NULL; + return 0; + } + } + + he = hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, + NULL, sample, false); + if (he == NULL) + return -ENOMEM; + + iter->he = he; + he_cache[iter->curr++] = he; + + if (hist_entry__has_callchains(he) && symbol_conf.use_callchain) + callchain_append(he->callchain, &cursor, sample->period); + return 0; +} + +static int +iter_finish_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + mem_info__zput(iter->mi); + zfree(&iter->bi); + zfree(&iter->he_cache); + iter->he = NULL; + + return 0; +} + +const struct hist_iter_ops hist_iter_mem = { + .prepare_entry = iter_prepare_mem_entry, + .add_single_entry = iter_add_single_mem_entry, + .next_entry = iter_next_nop_entry, + .add_next_entry = iter_add_next_nop_entry, + .finish_entry = iter_finish_mem_entry, +}; + +const struct hist_iter_ops hist_iter_branch = { + .prepare_entry = iter_prepare_branch_entry, + .add_single_entry = iter_add_single_branch_entry, + .next_entry = iter_next_branch_entry, + .add_next_entry = iter_add_next_branch_entry, + .finish_entry = iter_finish_branch_entry, +}; + +const struct hist_iter_ops hist_iter_normal = { + .prepare_entry = iter_prepare_normal_entry, + .add_single_entry = iter_add_single_normal_entry, + .next_entry = iter_next_nop_entry, + .add_next_entry = iter_add_next_nop_entry, + .finish_entry = iter_finish_normal_entry, +}; + +const struct hist_iter_ops hist_iter_cumulative = { + .prepare_entry = iter_prepare_cumulative_entry, + .add_single_entry = iter_add_single_cumulative_entry, + .next_entry = iter_next_cumulative_entry, + .add_next_entry = iter_add_next_cumulative_entry, + .finish_entry = iter_finish_cumulative_entry, +}; + +int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, + int max_stack_depth, void *arg) +{ + int err, err2; + struct map *alm = NULL; + + if (al) + alm = map__get(al->map); + + err = sample__resolve_callchain(iter->sample, get_tls_callchain_cursor(), &iter->parent, + al, max_stack_depth); + if (err) { + map__put(alm); + return err; + } + + err = iter->ops->prepare_entry(iter, al); + if (err) + goto out; + + err = iter->ops->add_single_entry(iter, al); + if (err) + goto out; + + if (iter->he && iter->add_entry_cb) { + err = iter->add_entry_cb(iter, al, true, arg); + if (err) + goto out; + } + + while (iter->ops->next_entry(iter, al)) { + err = iter->ops->add_next_entry(iter, al); + if (err) + break; + + if (iter->he && iter->add_entry_cb) { + err = iter->add_entry_cb(iter, al, false, arg); + if (err) + goto out; + } + } + +out: + err2 = iter->ops->finish_entry(iter, al); + if (!err) + err = err2; + + map__put(alm); + + return err; +} + +static int64_t +hist_entry__cmp_impl(struct perf_hpp_list *hpp_list, struct hist_entry *left, + struct hist_entry *right, unsigned long fn_offset, + bool ignore_dynamic, bool ignore_skipped) +{ + struct hists *hists = left->hists; + struct perf_hpp_fmt *fmt; + perf_hpp_fmt_cmp_t *fn; + int64_t cmp; + + /* + * Never collapse filtered and non-filtered entries. + * Note this is not the same as having an extra (invisible) fmt + * that corresponds to the filtered status. + */ + cmp = (int64_t)!!left->filtered - (int64_t)!!right->filtered; + if (cmp) + return cmp; + + perf_hpp_list__for_each_sort_list(hpp_list, fmt) { + if (ignore_dynamic && perf_hpp__is_dynamic_entry(fmt) && + !perf_hpp__defined_dynamic_entry(fmt, hists)) + continue; + + if (ignore_skipped && perf_hpp__should_skip(fmt, hists)) + continue; + + fn = (void *)fmt + fn_offset; + cmp = (*fn)(fmt, left, right); + if (cmp) + break; + } + + return cmp; +} + +int64_t +hist_entry__cmp(struct hist_entry *left, struct hist_entry *right) +{ + return hist_entry__cmp_impl(left->hists->hpp_list, left, right, + offsetof(struct perf_hpp_fmt, cmp), true, false); +} + +static int64_t +hist_entry__sort(struct hist_entry *left, struct hist_entry *right) +{ + return hist_entry__cmp_impl(left->hists->hpp_list, left, right, + offsetof(struct perf_hpp_fmt, sort), false, true); +} + +int64_t +hist_entry__collapse(struct hist_entry *left, struct hist_entry *right) +{ + return hist_entry__cmp_impl(left->hists->hpp_list, left, right, + offsetof(struct perf_hpp_fmt, collapse), true, false); +} + +static int64_t +hist_entry__collapse_hierarchy(struct perf_hpp_list *hpp_list, + struct hist_entry *left, + struct hist_entry *right) +{ + return hist_entry__cmp_impl(hpp_list, left, right, + offsetof(struct perf_hpp_fmt, collapse), false, false); +} + +void hist_entry__delete(struct hist_entry *he) +{ + struct hist_entry_ops *ops = he->ops; + + if (symbol_conf.report_hierarchy) { + struct rb_root *root = &he->hroot_out.rb_root; + struct hist_entry *child, *tmp; + + rbtree_postorder_for_each_entry_safe(child, tmp, root, rb_node) + hist_entry__delete(child); + + *root = RB_ROOT; + } + + thread__zput(he->thread); + map_symbol__exit(&he->ms); + + if (he->branch_info) { + branch_info__exit(he->branch_info); + zfree(&he->branch_info); + } + + if (he->mem_info) { + map_symbol__exit(&mem_info__iaddr(he->mem_info)->ms); + map_symbol__exit(&mem_info__daddr(he->mem_info)->ms); + mem_info__zput(he->mem_info); + } + + if (he->block_info) + block_info__delete(he->block_info); + + if (he->kvm_info) + kvm_info__zput(he->kvm_info); + + zfree(&he->res_samples); + zfree(&he->stat_acc); + zfree_srcline(&he->srcline); + if (he->srcfile && he->srcfile[0]) + zfree(&he->srcfile); + free_callchain(he->callchain); + zfree(&he->trace_output); + zfree(&he->raw_data); + zfree(&he->mem_stat); + ops->free(he); +} + +/* + * If this is not the last column, then we need to pad it according to the + * pre-calculated max length for this column, otherwise don't bother adding + * spaces because that would break viewing this with, for instance, 'less', + * that would show tons of trailing spaces when a long C++ demangled method + * names is sampled. +*/ +int hist_entry__snprintf_alignment(struct hist_entry *he, struct perf_hpp *hpp, + struct perf_hpp_fmt *fmt, int printed) +{ + if (!list_is_last(&fmt->list, &he->hists->hpp_list->fields)) { + const int width = fmt->width(fmt, hpp, he->hists); + if (printed < width) { + advance_hpp(hpp, printed); + printed = scnprintf(hpp->buf, hpp->size, "%-*s", width - printed, " "); + } + } + + return printed; +} + +/* + * collapse the histogram + */ + +static void hists__apply_filters(struct hists *hists, struct hist_entry *he); +static void hists__remove_entry_filter(struct hists *hists, struct hist_entry *he, + enum hist_filter type); + +typedef bool (*fmt_chk_fn)(struct perf_hpp_fmt *fmt); + +static bool check_thread_entry(struct perf_hpp_fmt *fmt) +{ + return perf_hpp__is_thread_entry(fmt) || perf_hpp__is_comm_entry(fmt); +} + +static void hist_entry__check_and_remove_filter(struct hist_entry *he, + enum hist_filter type, + fmt_chk_fn check) +{ + struct perf_hpp_fmt *fmt; + bool type_match = false; + struct hist_entry *parent = he->parent_he; + + switch (type) { + case HIST_FILTER__THREAD: + if (symbol_conf.comm_list == NULL && + symbol_conf.pid_list == NULL && + symbol_conf.tid_list == NULL) + return; + break; + case HIST_FILTER__DSO: + if (symbol_conf.dso_list == NULL) + return; + break; + case HIST_FILTER__SYMBOL: + if (symbol_conf.sym_list == NULL) + return; + break; + case HIST_FILTER__PARALLELISM: + if (__bitmap_weight(symbol_conf.parallelism_filter, MAX_NR_CPUS + 1) == 0) + return; + break; + case HIST_FILTER__PARENT: + case HIST_FILTER__GUEST: + case HIST_FILTER__HOST: + case HIST_FILTER__SOCKET: + case HIST_FILTER__C2C: + default: + return; + } + + /* if it's filtered by own fmt, it has to have filter bits */ + perf_hpp_list__for_each_format(he->hpp_list, fmt) { + if (check(fmt)) { + type_match = true; + break; + } + } + + if (type_match) { + /* + * If the filter is for current level entry, propagate + * filter marker to parents. The marker bit was + * already set by default so it only needs to clear + * non-filtered entries. + */ + if (!(he->filtered & (1 << type))) { + while (parent) { + parent->filtered &= ~(1 << type); + parent = parent->parent_he; + } + } + } else { + /* + * If current entry doesn't have matching formats, set + * filter marker for upper level entries. it will be + * cleared if its lower level entries is not filtered. + * + * For lower-level entries, it inherits parent's + * filter bit so that lower level entries of a + * non-filtered entry won't set the filter marker. + */ + if (parent == NULL) + he->filtered |= (1 << type); + else + he->filtered |= (parent->filtered & (1 << type)); + } +} + +static void hist_entry__apply_hierarchy_filters(struct hist_entry *he) +{ + hist_entry__check_and_remove_filter(he, HIST_FILTER__THREAD, + check_thread_entry); + + hist_entry__check_and_remove_filter(he, HIST_FILTER__DSO, + perf_hpp__is_dso_entry); + + hist_entry__check_and_remove_filter(he, HIST_FILTER__SYMBOL, + perf_hpp__is_sym_entry); + + hist_entry__check_and_remove_filter(he, HIST_FILTER__PARALLELISM, + perf_hpp__is_parallelism_entry); + + hists__apply_filters(he->hists, he); +} + +static struct hist_entry *hierarchy_insert_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *he, + struct hist_entry *parent_he, + struct perf_hpp_list *hpp_list) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter, *new; + struct perf_hpp_fmt *fmt; + int64_t cmp; + bool leftmost = true; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node_in); + cmp = hist_entry__collapse_hierarchy(hpp_list, iter, he); + if (!cmp) { + he_stat__add_stat(&iter->stat, &he->stat); + hists__add_mem_stat(hists, iter, he); + return iter; + } + + if (cmp < 0) + p = &parent->rb_left; + else { + p = &parent->rb_right; + leftmost = false; + } + } + + new = hist_entry__new(he, true); + if (new == NULL) + return NULL; + + hists->nr_entries++; + + /* save related format list for output */ + new->hpp_list = hpp_list; + new->parent_he = parent_he; + + hist_entry__apply_hierarchy_filters(new); + + /* some fields are now passed to 'new' */ + perf_hpp_list__for_each_sort_list(hpp_list, fmt) { + if (perf_hpp__is_trace_entry(fmt) || perf_hpp__is_dynamic_entry(fmt)) + he->trace_output = NULL; + else + new->trace_output = NULL; + + if (perf_hpp__is_srcline_entry(fmt)) + he->srcline = NULL; + else + new->srcline = NULL; + + if (perf_hpp__is_srcfile_entry(fmt)) + he->srcfile = NULL; + else + new->srcfile = NULL; + } + + if (hists__clone_mem_stat(hists, new, he) < 0) { + hist_entry__delete(new); + return NULL; + } + + rb_link_node(&new->rb_node_in, parent, p); + rb_insert_color_cached(&new->rb_node_in, root, leftmost); + return new; +} + +static int hists__hierarchy_insert_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *he) +{ + struct perf_hpp_list_node *node; + struct hist_entry *new_he = NULL; + struct hist_entry *parent = NULL; + int depth = 0; + int ret = 0; + + list_for_each_entry(node, &hists->hpp_formats, list) { + /* skip period (overhead) and elided columns */ + if (node->level == 0 || node->skip) + continue; + + /* insert copy of 'he' for each fmt into the hierarchy */ + new_he = hierarchy_insert_entry(hists, root, he, parent, &node->hpp); + if (new_he == NULL) { + ret = -1; + break; + } + + root = &new_he->hroot_in; + new_he->depth = depth++; + parent = new_he; + } + + if (new_he) { + new_he->leaf = true; + + if (hist_entry__has_callchains(new_he) && + symbol_conf.use_callchain) { + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + + if (cursor == NULL) + return -1; + + callchain_cursor_reset(cursor); + if (callchain_merge(cursor, + new_he->callchain, + he->callchain) < 0) + ret = -1; + } + } + + /* 'he' is no longer used */ + hist_entry__delete(he); + + /* return 0 (or -1) since it already applied filters */ + return ret; +} + +static int hists__collapse_insert_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + int64_t cmp; + bool leftmost = true; + + if (symbol_conf.report_hierarchy) + return hists__hierarchy_insert_entry(hists, root, he); + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node_in); + + cmp = hist_entry__collapse(iter, he); + + if (!cmp) { + int ret = 0; + + he_stat__add_stat(&iter->stat, &he->stat); + if (symbol_conf.cumulate_callchain) + he_stat__add_stat(iter->stat_acc, he->stat_acc); + hists__add_mem_stat(hists, iter, he); + + if (hist_entry__has_callchains(he) && symbol_conf.use_callchain) { + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + + if (cursor != NULL) { + callchain_cursor_reset(cursor); + if (callchain_merge(cursor, iter->callchain, he->callchain) < 0) + ret = -1; + } else { + ret = 0; + } + } + hist_entry__delete(he); + return ret; + } + + if (cmp < 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + hists->nr_entries++; + + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, root, leftmost); + return 1; +} + +struct rb_root_cached *hists__get_rotate_entries_in(struct hists *hists) +{ + struct rb_root_cached *root; + + mutex_lock(&hists->lock); + + root = hists->entries_in; + if (++hists->entries_in > &hists->entries_in_array[1]) + hists->entries_in = &hists->entries_in_array[0]; + + mutex_unlock(&hists->lock); + + return root; +} + +static void hists__apply_filters(struct hists *hists, struct hist_entry *he) +{ + hists__filter_entry_by_dso(hists, he); + hists__filter_entry_by_thread(hists, he); + hists__filter_entry_by_symbol(hists, he); + hists__filter_entry_by_socket(hists, he); + hists__filter_entry_by_parallelism(hists, he); +} + +int hists__collapse_resort(struct hists *hists, struct ui_progress *prog) +{ + struct rb_root_cached *root; + struct rb_node *next; + struct hist_entry *n; + int ret; + + if (!hists__has(hists, need_collapse)) + return 0; + + hists->nr_entries = 0; + + root = hists__get_rotate_entries_in(hists); + + next = rb_first_cached(root); + + while (next) { + if (session_done()) + break; + n = rb_entry(next, struct hist_entry, rb_node_in); + next = rb_next(&n->rb_node_in); + + rb_erase_cached(&n->rb_node_in, root); + ret = hists__collapse_insert_entry(hists, &hists->entries_collapsed, n); + if (ret < 0) + return -1; + + if (ret) { + /* + * If it wasn't combined with one of the entries already + * collapsed, we need to apply the filters that may have + * been set by, say, the hist_browser. + */ + hists__apply_filters(hists, n); + } + if (prog) + ui_progress__update(prog, 1); + } + return 0; +} + +static void hists__reset_filter_stats(struct hists *hists) +{ + hists->nr_non_filtered_entries = 0; + hists->stats.total_non_filtered_period = 0; + hists->stats.total_non_filtered_latency = 0; +} + +void hists__reset_stats(struct hists *hists) +{ + hists->nr_entries = 0; + hists->stats.total_period = 0; + hists->stats.total_latency = 0; + + hists__reset_filter_stats(hists); +} + +static void hists__inc_filter_stats(struct hists *hists, struct hist_entry *h) +{ + hists->nr_non_filtered_entries++; + hists->stats.total_non_filtered_period += h->stat.period; + hists->stats.total_non_filtered_latency += h->stat.latency; +} + +void hists__inc_stats(struct hists *hists, struct hist_entry *h) +{ + if (!h->filtered) + hists__inc_filter_stats(hists, h); + + hists->nr_entries++; + hists->stats.total_period += h->stat.period; + hists->stats.total_latency += h->stat.latency; +} + +static void hierarchy_recalc_total_periods(struct hists *hists) +{ + struct rb_node *node; + struct hist_entry *he; + + node = rb_first_cached(&hists->entries); + + hists->stats.total_period = 0; + hists->stats.total_non_filtered_period = 0; + hists->stats.total_latency = 0; + hists->stats.total_non_filtered_latency = 0; + + /* + * recalculate total period using top-level entries only + * since lower level entries only see non-filtered entries + * but upper level entries have sum of both entries. + */ + while (node) { + he = rb_entry(node, struct hist_entry, rb_node); + node = rb_next(node); + + hists->stats.total_period += he->stat.period; + hists->stats.total_latency += he->stat.latency; + if (!he->filtered) { + hists->stats.total_non_filtered_period += he->stat.period; + hists->stats.total_non_filtered_latency += he->stat.latency; + } + } +} + +static void hierarchy_insert_output_entry(struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + struct perf_hpp_fmt *fmt; + bool leftmost = true; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node); + + if (hist_entry__sort(he, iter) > 0) + p = &parent->rb_left; + else { + p = &parent->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, root, leftmost); + + /* update column width of dynamic entry */ + perf_hpp_list__for_each_sort_list(he->hpp_list, fmt) { + if (fmt->init) + fmt->init(fmt, he); + } +} + +static void hists__hierarchy_output_resort(struct hists *hists, + struct ui_progress *prog, + struct rb_root_cached *root_in, + struct rb_root_cached *root_out, + u64 min_callchain_hits, + bool use_callchain) +{ + struct rb_node *node; + struct hist_entry *he; + + *root_out = RB_ROOT_CACHED; + node = rb_first_cached(root_in); + + while (node) { + he = rb_entry(node, struct hist_entry, rb_node_in); + node = rb_next(node); + + hierarchy_insert_output_entry(root_out, he); + + if (prog) + ui_progress__update(prog, 1); + + hists->nr_entries++; + if (!he->filtered) { + hists->nr_non_filtered_entries++; + hists__calc_col_len(hists, he); + } + + if (!he->leaf) { + hists__hierarchy_output_resort(hists, prog, + &he->hroot_in, + &he->hroot_out, + min_callchain_hits, + use_callchain); + continue; + } + + if (!use_callchain) + continue; + + if (callchain_param.mode == CHAIN_GRAPH_REL) { + u64 total = he->stat.period; + + if (symbol_conf.cumulate_callchain) + total = he->stat_acc->period; + + min_callchain_hits = total * (callchain_param.min_percent / 100); + } + + callchain_param.sort(&he->sorted_chain, he->callchain, + min_callchain_hits, &callchain_param); + } +} + +static void __hists__insert_output_entry(struct rb_root_cached *entries, + struct hist_entry *he, + u64 min_callchain_hits, + bool use_callchain) +{ + struct rb_node **p = &entries->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + struct perf_hpp_fmt *fmt; + bool leftmost = true; + + if (use_callchain) { + if (callchain_param.mode == CHAIN_GRAPH_REL) { + u64 total = he->stat.period; + + if (symbol_conf.cumulate_callchain) + total = he->stat_acc->period; + + min_callchain_hits = total * (callchain_param.min_percent / 100); + } + callchain_param.sort(&he->sorted_chain, he->callchain, + min_callchain_hits, &callchain_param); + } + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node); + + if (hist_entry__sort(he, iter) > 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, entries, leftmost); + + /* update column width of dynamic entries */ + perf_hpp_list__for_each_sort_list(&perf_hpp_list, fmt) { + if (fmt->init) + fmt->init(fmt, he); + } +} + +static void output_resort(struct hists *hists, struct ui_progress *prog, + bool use_callchain, hists__resort_cb_t cb, + void *cb_arg) +{ + struct rb_root_cached *root; + struct rb_node *next; + struct hist_entry *n; + u64 callchain_total; + u64 min_callchain_hits; + + callchain_total = hists->callchain_period; + if (symbol_conf.filter_relative) + callchain_total = hists->callchain_non_filtered_period; + + min_callchain_hits = callchain_total * (callchain_param.min_percent / 100); + + hists__reset_stats(hists); + hists__reset_col_len(hists); + + if (symbol_conf.report_hierarchy) { + hists__hierarchy_output_resort(hists, prog, + &hists->entries_collapsed, + &hists->entries, + min_callchain_hits, + use_callchain); + hierarchy_recalc_total_periods(hists); + return; + } + + if (hists__has(hists, need_collapse)) + root = &hists->entries_collapsed; + else + root = hists->entries_in; + + next = rb_first_cached(root); + hists->entries = RB_ROOT_CACHED; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node_in); + next = rb_next(&n->rb_node_in); + + if (cb && cb(n, cb_arg)) + continue; + + __hists__insert_output_entry(&hists->entries, n, min_callchain_hits, use_callchain); + hists__inc_stats(hists, n); + + if (!n->filtered) + hists__calc_col_len(hists, n); + + if (prog) + ui_progress__update(prog, 1); + } +} + +void evsel__output_resort_cb(struct evsel *evsel, struct ui_progress *prog, + hists__resort_cb_t cb, void *cb_arg) +{ + bool use_callchain; + + if (evsel && symbol_conf.use_callchain && !symbol_conf.show_ref_callgraph) + use_callchain = evsel__has_callchain(evsel); + else + use_callchain = symbol_conf.use_callchain; + + use_callchain |= symbol_conf.show_branchflag_count; + + output_resort(evsel__hists(evsel), prog, use_callchain, cb, cb_arg); +} + +void evsel__output_resort(struct evsel *evsel, struct ui_progress *prog) +{ + return evsel__output_resort_cb(evsel, prog, NULL, NULL); +} + +void hists__output_resort(struct hists *hists, struct ui_progress *prog) +{ + output_resort(hists, prog, symbol_conf.use_callchain, NULL, NULL); +} + +void hists__output_resort_cb(struct hists *hists, struct ui_progress *prog, + hists__resort_cb_t cb) +{ + output_resort(hists, prog, symbol_conf.use_callchain, cb, NULL); +} + +static bool can_goto_child(struct hist_entry *he, enum hierarchy_move_dir hmd) +{ + if (he->leaf || hmd == HMD_FORCE_SIBLING) + return false; + + if (he->unfolded || hmd == HMD_FORCE_CHILD) + return true; + + return false; +} + +struct rb_node *rb_hierarchy_last(struct rb_node *node) +{ + struct hist_entry *he = rb_entry(node, struct hist_entry, rb_node); + + while (can_goto_child(he, HMD_NORMAL)) { + node = rb_last(&he->hroot_out.rb_root); + he = rb_entry(node, struct hist_entry, rb_node); + } + return node; +} + +struct rb_node *__rb_hierarchy_next(struct rb_node *node, enum hierarchy_move_dir hmd) +{ + struct hist_entry *he = rb_entry(node, struct hist_entry, rb_node); + + if (can_goto_child(he, hmd)) + node = rb_first_cached(&he->hroot_out); + else + node = rb_next(node); + + while (node == NULL) { + he = he->parent_he; + if (he == NULL) + break; + + node = rb_next(&he->rb_node); + } + return node; +} + +struct rb_node *rb_hierarchy_prev(struct rb_node *node) +{ + struct hist_entry *he = rb_entry(node, struct hist_entry, rb_node); + + node = rb_prev(node); + if (node) + return rb_hierarchy_last(node); + + he = he->parent_he; + if (he == NULL) + return NULL; + + return &he->rb_node; +} + +bool hist_entry__has_hierarchy_children(struct hist_entry *he, float limit) +{ + struct rb_node *node; + struct hist_entry *child; + float percent; + + if (he->leaf) + return false; + + node = rb_first_cached(&he->hroot_out); + child = rb_entry(node, struct hist_entry, rb_node); + + while (node && child->filtered) { + node = rb_next(node); + child = rb_entry(node, struct hist_entry, rb_node); + } + + if (node) + percent = hist_entry__get_percent_limit(child); + else + percent = 0; + + return node && percent >= limit; +} + +static void hists__remove_entry_filter(struct hists *hists, struct hist_entry *h, + enum hist_filter filter) +{ + h->filtered &= ~(1 << filter); + + if (symbol_conf.report_hierarchy) { + struct hist_entry *parent = h->parent_he; + + while (parent) { + he_stat__add_stat(&parent->stat, &h->stat); + + parent->filtered &= ~(1 << filter); + + if (parent->filtered) + goto next; + + /* force fold unfiltered entry for simplicity */ + parent->unfolded = false; + parent->has_no_entry = false; + parent->row_offset = 0; + parent->nr_rows = 0; +next: + parent = parent->parent_he; + } + } + + if (h->filtered) + return; + + /* force fold unfiltered entry for simplicity */ + h->unfolded = false; + h->has_no_entry = false; + h->row_offset = 0; + h->nr_rows = 0; + + hists->stats.nr_non_filtered_samples += h->stat.nr_events; + + hists__inc_filter_stats(hists, h); + hists__calc_col_len(hists, h); +} + + +static bool hists__filter_entry_by_dso(struct hists *hists, + struct hist_entry *he) +{ + if (hists->dso_filter != NULL && + (he->ms.map == NULL || !RC_CHK_EQUAL(map__dso(he->ms.map), hists->dso_filter))) { + he->filtered |= (1 << HIST_FILTER__DSO); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_thread(struct hists *hists, + struct hist_entry *he) +{ + if (hists->thread_filter != NULL && + !RC_CHK_EQUAL(he->thread, hists->thread_filter)) { + he->filtered |= (1 << HIST_FILTER__THREAD); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_symbol(struct hists *hists, + struct hist_entry *he) +{ + if (hists->symbol_filter_str != NULL && + (!he->ms.sym || strstr(he->ms.sym->name, + hists->symbol_filter_str) == NULL)) { + he->filtered |= (1 << HIST_FILTER__SYMBOL); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_socket(struct hists *hists, + struct hist_entry *he) +{ + if ((hists->socket_filter > -1) && + (he->socket != hists->socket_filter)) { + he->filtered |= (1 << HIST_FILTER__SOCKET); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_parallelism(struct hists *hists, + struct hist_entry *he) +{ + if (test_bit(he->parallelism, hists->parallelism_filter)) { + he->filtered |= (1 << HIST_FILTER__PARALLELISM); + return true; + } + return false; +} + +typedef bool (*filter_fn_t)(struct hists *hists, struct hist_entry *he); + +static void hists__filter_by_type(struct hists *hists, int type, filter_fn_t filter) +{ + struct rb_node *nd; + + hists->stats.nr_non_filtered_samples = 0; + + hists__reset_filter_stats(hists); + hists__reset_col_len(hists); + + for (nd = rb_first_cached(&hists->entries); nd; nd = rb_next(nd)) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + + if (filter(hists, h)) + continue; + + hists__remove_entry_filter(hists, h, type); + } +} + +static void resort_filtered_entry(struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + struct rb_root_cached new_root = RB_ROOT_CACHED; + struct rb_node *nd; + bool leftmost = true; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node); + + if (hist_entry__sort(he, iter) > 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, root, leftmost); + + if (he->leaf || he->filtered) + return; + + nd = rb_first_cached(&he->hroot_out); + while (nd) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + + nd = rb_next(nd); + rb_erase_cached(&h->rb_node, &he->hroot_out); + + resort_filtered_entry(&new_root, h); + } + + he->hroot_out = new_root; +} + +static void hists__filter_hierarchy(struct hists *hists, int type, const void *arg) +{ + struct rb_node *nd; + struct rb_root_cached new_root = RB_ROOT_CACHED; + + hists->stats.nr_non_filtered_samples = 0; + + hists__reset_filter_stats(hists); + hists__reset_col_len(hists); + + nd = rb_first_cached(&hists->entries); + while (nd) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + int ret; + + ret = hist_entry__filter(h, type, arg); + + /* + * case 1. non-matching type + * zero out the period, set filter marker and move to child + */ + if (ret < 0) { + memset(&h->stat, 0, sizeof(h->stat)); + h->filtered |= (1 << type); + + nd = __rb_hierarchy_next(&h->rb_node, HMD_FORCE_CHILD); + } + /* + * case 2. matched type (filter out) + * set filter marker and move to next + */ + else if (ret == 1) { + h->filtered |= (1 << type); + + nd = __rb_hierarchy_next(&h->rb_node, HMD_FORCE_SIBLING); + } + /* + * case 3. ok (not filtered) + * add period to hists and parents, erase the filter marker + * and move to next sibling + */ + else { + hists__remove_entry_filter(hists, h, type); + + nd = __rb_hierarchy_next(&h->rb_node, HMD_FORCE_SIBLING); + } + } + + hierarchy_recalc_total_periods(hists); + + /* + * resort output after applying a new filter since filter in a lower + * hierarchy can change periods in a upper hierarchy. + */ + nd = rb_first_cached(&hists->entries); + while (nd) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + + nd = rb_next(nd); + rb_erase_cached(&h->rb_node, &hists->entries); + + resort_filtered_entry(&new_root, h); + } + + hists->entries = new_root; +} + +void hists__filter_by_thread(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__THREAD, + hists->thread_filter); + else + hists__filter_by_type(hists, HIST_FILTER__THREAD, + hists__filter_entry_by_thread); +} + +void hists__filter_by_dso(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__DSO, + hists->dso_filter); + else + hists__filter_by_type(hists, HIST_FILTER__DSO, + hists__filter_entry_by_dso); +} + +void hists__filter_by_symbol(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__SYMBOL, + hists->symbol_filter_str); + else + hists__filter_by_type(hists, HIST_FILTER__SYMBOL, + hists__filter_entry_by_symbol); +} + +void hists__filter_by_socket(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__SOCKET, + &hists->socket_filter); + else + hists__filter_by_type(hists, HIST_FILTER__SOCKET, + hists__filter_entry_by_socket); +} + +void hists__filter_by_parallelism(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__PARALLELISM, + hists->parallelism_filter); + else + hists__filter_by_type(hists, HIST_FILTER__PARALLELISM, + hists__filter_entry_by_parallelism); +} + +void events_stats__inc(struct events_stats *stats, u32 type) +{ + ++stats->nr_events[0]; + ++stats->nr_events[type]; +} + +static void hists_stats__inc(struct hists_stats *stats) +{ + ++stats->nr_samples; +} + +void hists__inc_nr_events(struct hists *hists) +{ + hists_stats__inc(&hists->stats); +} + +void hists__inc_nr_samples(struct hists *hists, bool filtered) +{ + hists_stats__inc(&hists->stats); + if (!filtered) + hists->stats.nr_non_filtered_samples++; +} + +void hists__inc_nr_lost_samples(struct hists *hists, u32 lost) +{ + hists->stats.nr_lost_samples += lost; +} + +void hists__inc_nr_dropped_samples(struct hists *hists, u32 lost) +{ + hists->stats.nr_dropped_samples += lost; +} + +static struct hist_entry *hists__add_dummy_entry(struct hists *hists, + struct hist_entry *pair) +{ + struct rb_root_cached *root; + struct rb_node **p; + struct rb_node *parent = NULL; + struct hist_entry *he; + int64_t cmp; + bool leftmost = true; + + if (hists__has(hists, need_collapse)) + root = &hists->entries_collapsed; + else + root = hists->entries_in; + + p = &root->rb_root.rb_node; + + while (*p != NULL) { + parent = *p; + he = rb_entry(parent, struct hist_entry, rb_node_in); + + cmp = hist_entry__collapse(he, pair); + + if (!cmp) + goto out; + + if (cmp < 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + he = hist_entry__new(pair, true); + if (he) { + memset(&he->stat, 0, sizeof(he->stat)); + he->hists = hists; + if (symbol_conf.cumulate_callchain) + memset(he->stat_acc, 0, sizeof(he->stat)); + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, root, leftmost); + hists__inc_stats(hists, he); + he->dummy = true; + } +out: + return he; +} + +static struct hist_entry *add_dummy_hierarchy_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *pair) +{ + struct rb_node **p; + struct rb_node *parent = NULL; + struct hist_entry *he; + bool leftmost = true; + + p = &root->rb_root.rb_node; + while (*p != NULL) { + int64_t cmp; + + parent = *p; + he = rb_entry(parent, struct hist_entry, rb_node_in); + cmp = hist_entry__collapse_hierarchy(he->hpp_list, he, pair); + if (!cmp) + goto out; + + if (cmp < 0) + p = &parent->rb_left; + else { + p = &parent->rb_right; + leftmost = false; + } + } + + he = hist_entry__new(pair, true); + if (he) { + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, root, leftmost); + + he->dummy = true; + he->hists = hists; + memset(&he->stat, 0, sizeof(he->stat)); + hists__inc_stats(hists, he); + } +out: + return he; +} + +static struct hist_entry *hists__find_entry(struct hists *hists, + struct hist_entry *he) +{ + struct rb_node *n; + + if (hists__has(hists, need_collapse)) + n = hists->entries_collapsed.rb_root.rb_node; + else + n = hists->entries_in->rb_root.rb_node; + + while (n) { + struct hist_entry *iter = rb_entry(n, struct hist_entry, rb_node_in); + int64_t cmp = hist_entry__collapse(iter, he); + + if (cmp < 0) + n = n->rb_left; + else if (cmp > 0) + n = n->rb_right; + else + return iter; + } + + return NULL; +} + +static struct hist_entry *hists__find_hierarchy_entry(struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node *n = root->rb_root.rb_node; + + while (n) { + struct hist_entry *iter; + int64_t cmp; + + iter = rb_entry(n, struct hist_entry, rb_node_in); + cmp = hist_entry__collapse_hierarchy(he->hpp_list, iter, he); + if (cmp < 0) + n = n->rb_left; + else if (cmp > 0) + n = n->rb_right; + else + return iter; + } + + return NULL; +} + +static void hists__match_hierarchy(struct rb_root_cached *leader_root, + struct rb_root_cached *other_root) +{ + struct rb_node *nd; + struct hist_entry *pos, *pair; + + for (nd = rb_first_cached(leader_root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + pair = hists__find_hierarchy_entry(other_root, pos); + + if (pair) { + hist_entry__add_pair(pair, pos); + hists__match_hierarchy(&pos->hroot_in, &pair->hroot_in); + } + } +} + +/* + * Look for pairs to link to the leader buckets (hist_entries): + */ +void hists__match(struct hists *leader, struct hists *other) +{ + struct rb_root_cached *root; + struct rb_node *nd; + struct hist_entry *pos, *pair; + + if (symbol_conf.report_hierarchy) { + /* hierarchy report always collapses entries */ + return hists__match_hierarchy(&leader->entries_collapsed, + &other->entries_collapsed); + } + + if (hists__has(leader, need_collapse)) + root = &leader->entries_collapsed; + else + root = leader->entries_in; + + for (nd = rb_first_cached(root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + pair = hists__find_entry(other, pos); + + if (pair) + hist_entry__add_pair(pair, pos); + } +} + +static int hists__link_hierarchy(struct hists *leader_hists, + struct hist_entry *parent, + struct rb_root_cached *leader_root, + struct rb_root_cached *other_root) +{ + struct rb_node *nd; + struct hist_entry *pos, *leader; + + for (nd = rb_first_cached(other_root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + + if (hist_entry__has_pairs(pos)) { + bool found = false; + + list_for_each_entry(leader, &pos->pairs.head, pairs.node) { + if (leader->hists == leader_hists) { + found = true; + break; + } + } + if (!found) + return -1; + } else { + leader = add_dummy_hierarchy_entry(leader_hists, + leader_root, pos); + if (leader == NULL) + return -1; + + /* do not point parent in the pos */ + leader->parent_he = parent; + + hist_entry__add_pair(pos, leader); + } + + if (!pos->leaf) { + if (hists__link_hierarchy(leader_hists, leader, + &leader->hroot_in, + &pos->hroot_in) < 0) + return -1; + } + } + return 0; +} + +/* + * Look for entries in the other hists that are not present in the leader, if + * we find them, just add a dummy entry on the leader hists, with period=0, + * nr_events=0, to serve as the list header. + */ +int hists__link(struct hists *leader, struct hists *other) +{ + struct rb_root_cached *root; + struct rb_node *nd; + struct hist_entry *pos, *pair; + + if (symbol_conf.report_hierarchy) { + /* hierarchy report always collapses entries */ + return hists__link_hierarchy(leader, NULL, + &leader->entries_collapsed, + &other->entries_collapsed); + } + + if (hists__has(other, need_collapse)) + root = &other->entries_collapsed; + else + root = other->entries_in; + + for (nd = rb_first_cached(root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + + if (!hist_entry__has_pairs(pos)) { + pair = hists__add_dummy_entry(leader, pos); + if (pair == NULL) + return -1; + hist_entry__add_pair(pos, pair); + } + } + + return 0; +} + +int hists__unlink(struct hists *hists) +{ + struct rb_root_cached *root; + struct rb_node *nd; + struct hist_entry *pos; + + if (hists__has(hists, need_collapse)) + root = &hists->entries_collapsed; + else + root = hists->entries_in; + + for (nd = rb_first_cached(root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + list_del_init(&pos->pairs.node); + } + + return 0; +} + +void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, + struct perf_sample *sample, bool nonany_branch_mode, + u64 *total_cycles) +{ + struct branch_info *bi; + struct branch_entry *entries = perf_sample__branch_entries(sample); + + /* If we have branch cycles always annotate them. */ + if (bs && bs->nr && entries[0].flags.cycles) { + bi = sample__resolve_bstack(sample, al); + if (bi) { + struct addr_map_symbol *prev = NULL; + + /* + * Ignore errors, still want to process the + * other entries. + * + * For non standard branch modes always + * force no IPC (prev == NULL) + * + * Note that perf stores branches reversed from + * program order! + */ + for (int i = bs->nr - 1; i >= 0; i--) { + addr_map_symbol__account_cycles(&bi[i].from, + nonany_branch_mode ? NULL : prev, + bi[i].flags.cycles, sample->evsel, + bi[i].branch_stack_cntr); + prev = &bi[i].to; + + if (total_cycles) + *total_cycles += bi[i].flags.cycles; + } + for (unsigned int i = 0; i < bs->nr; i++) { + map_symbol__exit(&bi[i].to.ms); + map_symbol__exit(&bi[i].from.ms); + } + free(bi); + } + } +} + +size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp) +{ + struct evsel *pos; + size_t ret = 0; + + evlist__for_each_entry(evlist, pos) { + struct hists *hists = evsel__hists(pos); + u64 total_samples = hists->stats.nr_samples; + + total_samples += hists->stats.nr_lost_samples; + total_samples += hists->stats.nr_dropped_samples; + + if (symbol_conf.skip_empty && total_samples == 0) + continue; + + ret += fprintf(fp, "%s stats:\n", evsel__name(pos)); + if (hists->stats.nr_samples) + ret += fprintf(fp, "%20s events: %10d\n", + "SAMPLE", hists->stats.nr_samples); + if (hists->stats.nr_lost_samples) + ret += fprintf(fp, "%20s events: %10d\n", + "LOST_SAMPLES", hists->stats.nr_lost_samples); + if (hists->stats.nr_dropped_samples) + ret += fprintf(fp, "%20s events: %10d\n", + "LOST_SAMPLES (BPF)", hists->stats.nr_dropped_samples); + } + + return ret; +} + + +u64 hists__total_period(struct hists *hists) +{ + return symbol_conf.filter_relative ? hists->stats.total_non_filtered_period : + hists->stats.total_period; +} + +u64 hists__total_latency(struct hists *hists) +{ + return symbol_conf.filter_relative ? hists->stats.total_non_filtered_latency : + hists->stats.total_latency; +} + +int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool show_freq) +{ + char unit; + int printed; + const struct dso *dso = hists->dso_filter; + struct thread *thread = hists->thread_filter; + int socket_id = hists->socket_filter; + unsigned long nr_samples = hists->stats.nr_samples; + u64 nr_events = hists->stats.total_period; + struct evsel *evsel = hists_to_evsel(hists); + const char *ev_name = evsel__name(evsel); + char buf[512], sample_freq_str[64] = ""; + size_t buflen = sizeof(buf); + char ref[30] = " show reference callgraph, "; + bool enable_ref = false; + + if (symbol_conf.filter_relative) { + nr_samples = hists->stats.nr_non_filtered_samples; + nr_events = hists->stats.total_non_filtered_period; + } + + if (evsel__is_group_event(evsel)) { + struct evsel *pos; + + evsel__group_desc(evsel, buf, buflen); + ev_name = buf; + + for_each_group_member(pos, evsel) { + struct hists *pos_hists = evsel__hists(pos); + + if (symbol_conf.filter_relative) { + nr_samples += pos_hists->stats.nr_non_filtered_samples; + nr_events += pos_hists->stats.total_non_filtered_period; + } else { + nr_samples += pos_hists->stats.nr_samples; + nr_events += pos_hists->stats.total_period; + } + } + } + + if (symbol_conf.show_ref_callgraph && + strstr(ev_name, "call-graph=no")) + enable_ref = true; + + if (show_freq) + scnprintf(sample_freq_str, sizeof(sample_freq_str), " %d Hz,", evsel->core.attr.sample_freq); + + nr_samples = convert_unit(nr_samples, &unit); + printed = scnprintf(bf, size, + "Samples: %lu%c of event%s '%s',%s%sEvent count (approx.): %" PRIu64, + nr_samples, unit, evsel->core.nr_members > 1 ? "s" : "", + ev_name, sample_freq_str, enable_ref ? ref : " ", nr_events); + + + if (hists->uid_filter_str) { + printed += scnprintf(bf + printed, size - printed, + ", UID: %s", hists->uid_filter_str); + } + if (thread) { + if (hists__has(hists, thread)) { + printed += scnprintf(bf + printed, size - printed, + ", Thread: %s(%d)", + (thread__comm_set(thread) ? thread__comm_str(thread) : ""), + thread__tid(thread)); + } else { + printed += scnprintf(bf + printed, size - printed, + ", Thread: %s", + (thread__comm_set(thread) ? thread__comm_str(thread) : "")); + } + } + if (dso) + printed += scnprintf(bf + printed, size - printed, + ", DSO: %s", dso__short_name(dso)); + if (socket_id > -1) + printed += scnprintf(bf + printed, size - printed, + ", Processor Socket: %d", socket_id); + + return printed; +} + +int parse_filter_percentage(const struct option *opt __maybe_unused, + const char *arg, int unset __maybe_unused) +{ + if (!strcmp(arg, "relative")) + symbol_conf.filter_relative = true; + else if (!strcmp(arg, "absolute")) + symbol_conf.filter_relative = false; + else { + pr_debug("Invalid percentage: %s\n", arg); + return -1; + } + + return 0; +} + +int perf_hist_config(const char *var, const char *value) +{ + if (!strcmp(var, "hist.percentage")) + return parse_filter_percentage(NULL, value, 0); + + return 0; +} + +int __hists__init(struct hists *hists, struct perf_hpp_list *hpp_list) +{ + memset(hists, 0, sizeof(*hists)); + hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT_CACHED; + hists->entries_in = &hists->entries_in_array[0]; + hists->entries_collapsed = RB_ROOT_CACHED; + hists->entries = RB_ROOT_CACHED; + mutex_init(&hists->lock); + hists->socket_filter = -1; + hists->parallelism_filter = symbol_conf.parallelism_filter; + hists->hpp_list = hpp_list; + INIT_LIST_HEAD(&hists->hpp_formats); + return 0; +} + +static void hists__delete_remaining_entries(struct rb_root_cached *root) +{ + struct rb_node *node; + struct hist_entry *he; + + while (!RB_EMPTY_ROOT(&root->rb_root)) { + node = rb_first_cached(root); + rb_erase_cached(node, root); + + he = rb_entry(node, struct hist_entry, rb_node_in); + hist_entry__delete(he); + } +} + +void hists__delete_all_entries(struct hists *hists) +{ + hists__delete_entries(hists); + hists__delete_remaining_entries(&hists->entries_in_array[0]); + hists__delete_remaining_entries(&hists->entries_in_array[1]); + hists__delete_remaining_entries(&hists->entries_collapsed); +} + +static void hists_evsel__exit(struct evsel *evsel) +{ + struct hists *hists = evsel__hists(evsel); + struct perf_hpp_fmt *fmt, *pos; + struct perf_hpp_list_node *node, *tmp; + + hists__delete_all_entries(hists); + zfree(&hists->symbol_filter_str); + zfree(&hists->mem_stat_types); + zfree(&hists->mem_stat_total); + + list_for_each_entry_safe(node, tmp, &hists->hpp_formats, list) { + perf_hpp_list__for_each_format_safe(&node->hpp, fmt, pos) { + list_del_init(&fmt->list); + free(fmt); + } + list_del_init(&node->list); + free(node); + } +} + +static int hists_evsel__init(struct evsel *evsel) +{ + struct hists *hists = evsel__hists(evsel); + + __hists__init(hists, &perf_hpp_list); + return 0; +} + +/* + * XXX We probably need a hists_evsel__exit() to free the hist_entries + * stored in the rbtree... + */ + +int hists__init(void) +{ + int err = evsel__object_config(sizeof(struct hists_evsel), + hists_evsel__init, hists_evsel__exit); + if (err) + fputs("FATAL ERROR: Couldn't setup hists class\n", stderr); + + return err; +} + +void perf_hpp_list__init(struct perf_hpp_list *list) +{ + INIT_LIST_HEAD(&list->fields); + INIT_LIST_HEAD(&list->sorts); +} diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h new file mode 100644 index 000000000..b830cbe7f --- /dev/null +++ b/tools/perf/util/hist.h @@ -0,0 +1,841 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_HIST_H +#define __PERF_HIST_H + +#include +#include +#include "callchain.h" +#include "color.h" +#include "events_stats.h" +#include "evsel.h" +#include "map_symbol.h" +#include "mem-events.h" +#include "mutex.h" +#include "sample.h" +#include "spark.h" +#include "stat.h" + +struct addr_location; +struct mem_info; +struct kvm_info; +struct branch_info; +struct branch_stack; +struct block_info; +struct ui_progress; + +enum hist_filter { + HIST_FILTER__DSO, + HIST_FILTER__THREAD, + HIST_FILTER__PARENT, + HIST_FILTER__SYMBOL, + HIST_FILTER__GUEST, + HIST_FILTER__HOST, + HIST_FILTER__SOCKET, + HIST_FILTER__C2C, + HIST_FILTER__PARALLELISM, +}; + +typedef u16 filter_mask_t; + +enum hist_column { + HISTC_SYMBOL, + HISTC_TIME, + HISTC_DSO, + HISTC_THREAD, + HISTC_TGID, + HISTC_COMM, + HISTC_COMM_NODIGIT, + HISTC_CGROUP_ID, + HISTC_CGROUP, + HISTC_PARENT, + HISTC_PARALLELISM, + HISTC_CPU, + HISTC_SOCKET, + HISTC_SRCLINE, + HISTC_SRCFILE, + HISTC_MISPREDICT, + HISTC_IN_TX, + HISTC_ABORT, + HISTC_SYMBOL_FROM, + HISTC_SYMBOL_TO, + HISTC_DSO_FROM, + HISTC_DSO_TO, + HISTC_LOCAL_WEIGHT, + HISTC_GLOBAL_WEIGHT, + HISTC_CODE_PAGE_SIZE, + HISTC_MEM_DADDR_SYMBOL, + HISTC_MEM_DADDR_DSO, + HISTC_MEM_PHYS_DADDR, + HISTC_MEM_DATA_PAGE_SIZE, + HISTC_MEM_LOCKED, + HISTC_MEM_TLB, + HISTC_MEM_LVL, + HISTC_MEM_SNOOP, + HISTC_MEM_DCACHELINE, + HISTC_MEM_IADDR_SYMBOL, + HISTC_TRANSACTION, + HISTC_CYCLES, + HISTC_SRCLINE_FROM, + HISTC_SRCLINE_TO, + HISTC_TRACE, + HISTC_SYM_SIZE, + HISTC_DSO_SIZE, + HISTC_SYMBOL_IPC, + HISTC_MEM_BLOCKED, + HISTC_LOCAL_INS_LAT, + HISTC_GLOBAL_INS_LAT, + HISTC_LOCAL_P_STAGE_CYC, + HISTC_GLOBAL_P_STAGE_CYC, + HISTC_ADDR_FROM, + HISTC_ADDR_TO, + HISTC_ADDR, + HISTC_SIMD, + HISTC_TYPE, + HISTC_TYPE_OFFSET, + HISTC_SYMBOL_OFFSET, + HISTC_TYPE_CACHELINE, + HISTC_CALLCHAIN_BRANCH_PREDICTED, + HISTC_CALLCHAIN_BRANCH_ABORT, + HISTC_CALLCHAIN_BRANCH_CYCLES, + HISTC_NR_COLS, /* Last entry */ +}; + +struct thread; +struct dso; + +#define MEM_STAT_LEN 8 + +struct he_mem_stat { + /* meaning of entries depends on enum mem_stat_type */ + u64 entries[MEM_STAT_LEN]; +}; + +struct hists { + struct rb_root_cached entries_in_array[2]; + struct rb_root_cached *entries_in; + struct rb_root_cached entries; + struct rb_root_cached entries_collapsed; + u64 nr_entries; + u64 nr_non_filtered_entries; + u64 callchain_period; + u64 callchain_non_filtered_period; + u64 callchain_latency; + u64 callchain_non_filtered_latency; + struct thread *thread_filter; + const struct dso *dso_filter; + const char *uid_filter_str; + const char *symbol_filter_str; + unsigned long *parallelism_filter; + struct mutex lock; + struct hists_stats stats; + u64 event_stream; + u16 col_len[HISTC_NR_COLS]; + bool has_callchains; + int socket_filter; + struct perf_hpp_list *hpp_list; + struct list_head hpp_formats; + int nr_hpp_node; + int nr_mem_stats; + enum mem_stat_type *mem_stat_types; + struct he_mem_stat *mem_stat_total; +}; + +#define hists__has(__h, __f) (__h)->hpp_list->__f + +struct hist_entry_iter; + +struct hist_iter_ops { + int (*prepare_entry)(struct hist_entry_iter *, struct addr_location *); + int (*add_single_entry)(struct hist_entry_iter *, struct addr_location *); + int (*next_entry)(struct hist_entry_iter *, struct addr_location *); + int (*add_next_entry)(struct hist_entry_iter *, struct addr_location *); + int (*finish_entry)(struct hist_entry_iter *, struct addr_location *); +}; + +struct hist_entry_iter { + int total; + int curr; + + struct perf_sample *sample; + struct hist_entry *he; + struct symbol *parent; + + struct mem_info *mi; + struct branch_info *bi; + struct hist_entry **he_cache; + + const struct hist_iter_ops *ops; + /* user-defined callback function (optional) */ + int (*add_entry_cb)(struct hist_entry_iter *iter, + struct addr_location *al, bool single, void *arg); + bool hide_unresolved; +}; + +extern const struct hist_iter_ops hist_iter_normal; +extern const struct hist_iter_ops hist_iter_branch; +extern const struct hist_iter_ops hist_iter_mem; +extern const struct hist_iter_ops hist_iter_cumulative; + +struct res_sample { + u64 time; + int cpu; + int tid; +}; + +struct he_stat { + u64 period; + /* + * Period re-scaled from CPU time to wall-clock time (divided by the + * parallelism at the time of the sample). This represents effect of + * the event on latency rather than CPU consumption. + */ + u64 latency; + u64 period_sys; + u64 period_us; + u64 period_guest_sys; + u64 period_guest_us; + u64 weight1; + u64 weight2; + u64 weight3; + u32 nr_events; +}; + +struct namespace_id { + u64 dev; + u64 ino; +}; + +struct hist_entry_diff { + bool computed; + union { + /* PERF_HPP__DELTA */ + double period_ratio_delta; + + /* PERF_HPP__RATIO */ + double period_ratio; + + /* HISTC_WEIGHTED_DIFF */ + s64 wdiff; + + /* PERF_HPP_DIFF__CYCLES */ + s64 cycles; + }; + struct stats stats; + unsigned long svals[NUM_SPARKS]; +}; + +struct hist_entry_ops { + void *(*new)(size_t size); + void (*free)(void *ptr); +}; + +/** + * struct hist_entry - histogram entry + * + * @row_offset - offset from the first callchain expanded to appear on screen + * @nr_rows - rows expanded in callchain, recalculated on folding/unfolding + */ +struct hist_entry { + struct rb_node rb_node_in; + struct rb_node rb_node; + union { + struct list_head node; + struct list_head head; + } pairs; + struct he_stat stat; + struct he_stat *stat_acc; + struct he_mem_stat *mem_stat; + struct map_symbol ms; + struct thread *thread; + struct comm *comm; + struct namespace_id cgroup_id; + u64 cgroup; + u64 ip; + u64 transaction; + u64 code_page_size; + u64 weight; + u64 ins_lat; + /** @weight3: On x86 holds retire_lat, on powerpc holds p_stage_cyc. */ + u64 weight3; + s32 socket; + s32 cpu; + int parallelism; + int mem_type_off; + u8 cpumode; + u8 depth; + struct simd_flags simd_flags; + + /* We are added by hists__add_dummy_entry. */ + bool dummy; + bool leaf; + + char level; + filter_mask_t filtered; + + u16 callchain_size; + union { + /* + * Since perf diff only supports the stdio output, TUI + * fields are only accessed from perf report (or perf + * top). So make it a union to reduce memory usage. + */ + struct hist_entry_diff diff; + struct /* for TUI */ { + u16 row_offset; + u16 nr_rows; + bool init_have_children; + bool unfolded; + bool has_children; + bool has_no_entry; + }; + }; + char *srcline; + char *srcfile; + struct symbol *parent; + struct branch_info *branch_info; + long time; + struct hists *hists; + struct mem_info *mem_info; + struct block_info *block_info; + struct kvm_info *kvm_info; + void *raw_data; + u32 raw_size; + int num_res; + struct res_sample *res_samples; + void *trace_output; + struct perf_hpp_list *hpp_list; + struct hist_entry *parent_he; + struct hist_entry_ops *ops; + struct annotated_data_type *mem_type; + union { + /* this is for hierarchical entry structure */ + struct { + struct rb_root_cached hroot_in; + struct rb_root_cached hroot_out; + }; /* non-leaf entries */ + struct rb_root sorted_chain; /* leaf entry has callchains */ + }; + struct callchain_root callchain[0]; /* must be last member */ +}; + +static __pure inline bool hist_entry__has_callchains(struct hist_entry *he) +{ + return he->callchain_size != 0; +} + +static inline bool hist_entry__has_pairs(struct hist_entry *he) +{ + return !list_empty(&he->pairs.node); +} + +static inline struct hist_entry *hist_entry__next_pair(struct hist_entry *he) +{ + if (hist_entry__has_pairs(he)) + return list_entry(he->pairs.node.next, struct hist_entry, pairs.node); + return NULL; +} + +static inline void hist_entry__add_pair(struct hist_entry *pair, + struct hist_entry *he) +{ + list_add_tail(&pair->pairs.node, &he->pairs.head); +} + +struct hist_entry *hists__add_entry(struct hists *hists, + struct addr_location *al, + struct symbol *parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self); + +struct hist_entry *hists__add_entry_ops(struct hists *hists, + struct hist_entry_ops *ops, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self); + +struct hist_entry *hists__add_entry_block(struct hists *hists, + struct addr_location *al, + struct block_info *bi); + +int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, + int max_stack_depth, void *arg); + +struct perf_hpp; +struct perf_hpp_fmt; + +int hist_entry__transaction_len(void); +int hist_entry__sort_snprintf(struct hist_entry *he, char *bf, size_t size, + struct hists *hists); +int hist_entry__snprintf_alignment(struct hist_entry *he, struct perf_hpp *hpp, + struct perf_hpp_fmt *fmt, int printed); +int hist_entry__sym_snprintf(struct hist_entry *he, char *bf, size_t size, + unsigned int width); +void hist_entry__delete(struct hist_entry *he); + +typedef int (*hists__resort_cb_t)(struct hist_entry *he, void *arg); + +void evsel__output_resort_cb(struct evsel *evsel, struct ui_progress *prog, + hists__resort_cb_t cb, void *cb_arg); +void evsel__output_resort(struct evsel *evsel, struct ui_progress *prog); +void hists__output_resort(struct hists *hists, struct ui_progress *prog); +void hists__output_resort_cb(struct hists *hists, struct ui_progress *prog, + hists__resort_cb_t cb); +int hists__collapse_resort(struct hists *hists, struct ui_progress *prog); + +void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel); +void hists__delete_entries(struct hists *hists); +void hists__delete_all_entries(struct hists *hists); +void hists__output_recalc_col_len(struct hists *hists, int max_rows); + +struct hist_entry *hists__get_entry(struct hists *hists, int idx); + +u64 hists__total_period(struct hists *hists); +u64 hists__total_latency(struct hists *hists); +void hists__reset_stats(struct hists *hists); +void hists__inc_stats(struct hists *hists, struct hist_entry *h); +void hists__inc_nr_events(struct hists *hists); +void hists__inc_nr_samples(struct hists *hists, bool filtered); +void hists__inc_nr_lost_samples(struct hists *hists, u32 lost); +void hists__inc_nr_dropped_samples(struct hists *hists, u32 lost); + +size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows, + int max_cols, float min_pcnt, FILE *fp, + bool ignore_callchains); +size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp); + +void hists__filter_by_dso(struct hists *hists); +void hists__filter_by_thread(struct hists *hists); +void hists__filter_by_symbol(struct hists *hists); +void hists__filter_by_socket(struct hists *hists); +void hists__filter_by_parallelism(struct hists *hists); + +static inline bool hists__has_filter(struct hists *hists) +{ + return hists->thread_filter || hists->dso_filter || + hists->symbol_filter_str || (hists->socket_filter > -1) || + hists->parallelism_filter; +} + +u16 hists__col_len(struct hists *hists, enum hist_column col); +void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len); +bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len); +void hists__reset_col_len(struct hists *hists); +void hists__calc_col_len(struct hists *hists, struct hist_entry *he); + +void hists__match(struct hists *leader, struct hists *other); +int hists__link(struct hists *leader, struct hists *other); +int hists__unlink(struct hists *hists); + +static inline float hist_entry__get_percent_limit(struct hist_entry *he) +{ + u64 period = he->stat.period; + u64 total_period = hists__total_period(he->hists); + + if (unlikely(total_period == 0)) + return 0; + + if (symbol_conf.cumulate_callchain) + period = he->stat_acc->period; + + return period * 100.0 / total_period; +} + +struct hists_evsel { + struct evsel evsel; + struct hists hists; +}; + +static inline struct evsel *hists_to_evsel(struct hists *hists) +{ + struct hists_evsel *hevsel = container_of(hists, struct hists_evsel, hists); + return &hevsel->evsel; +} + +static inline struct hists *evsel__hists(struct evsel *evsel) +{ + struct hists_evsel *hevsel = (struct hists_evsel *)evsel; + return &hevsel->hists; +} + +static __pure inline bool hists__has_callchains(struct hists *hists) +{ + return hists->has_callchains; +} + +int hists__init(void); +int __hists__init(struct hists *hists, struct perf_hpp_list *hpp_list); + +struct rb_root_cached *hists__get_rotate_entries_in(struct hists *hists); + +struct perf_hpp { + char *buf; + size_t size; + const char *sep; + void *ptr; + bool skip; +}; + +typedef int64_t (*perf_hpp_fmt_cmp_t)( + struct perf_hpp_fmt *, struct hist_entry *, struct hist_entry *); + +struct perf_hpp_fmt { + const char *name; + int (*header)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hists *hists, int line, int *span); + int (*width)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hists *hists); + void (*init)(struct perf_hpp_fmt *fmt, struct hist_entry *he); + int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he); + int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he); + perf_hpp_fmt_cmp_t cmp; + perf_hpp_fmt_cmp_t collapse; + perf_hpp_fmt_cmp_t sort; + bool (*equal)(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b); + void (*free)(struct perf_hpp_fmt *fmt); + + struct list_head list; + struct list_head sort_list; + bool elide; + int len; + int user_len; + int idx; + int level; +}; + +struct perf_hpp_list { + struct list_head fields; + struct list_head sorts; + + int nr_header_lines; + int need_collapse; + int parent; + int sym; + int dso; + int socket; + int thread; + int comm; + int comm_nodigit; +}; + +extern struct perf_hpp_list perf_hpp_list; + +struct perf_hpp_list_node { + struct list_head list; + struct perf_hpp_list hpp; + int level; + bool skip; +}; + +void perf_hpp_list__column_register(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); +void perf_hpp_list__register_sort_field(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); +void perf_hpp_list__prepend_sort_field(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); + +static inline void perf_hpp__column_register(struct perf_hpp_fmt *format) +{ + perf_hpp_list__column_register(&perf_hpp_list, format); +} + +static inline void perf_hpp__register_sort_field(struct perf_hpp_fmt *format) +{ + perf_hpp_list__register_sort_field(&perf_hpp_list, format); +} + +static inline void perf_hpp__prepend_sort_field(struct perf_hpp_fmt *format) +{ + perf_hpp_list__prepend_sort_field(&perf_hpp_list, format); +} + +#define perf_hpp_list__for_each_format(_list, format) \ + list_for_each_entry(format, &(_list)->fields, list) + +#define perf_hpp_list__for_each_format_safe(_list, format, tmp) \ + list_for_each_entry_safe(format, tmp, &(_list)->fields, list) + +#define perf_hpp_list__for_each_sort_list(_list, format) \ + list_for_each_entry(format, &(_list)->sorts, sort_list) + +#define perf_hpp_list__for_each_sort_list_safe(_list, format, tmp) \ + list_for_each_entry_safe(format, tmp, &(_list)->sorts, sort_list) + +#define hists__for_each_format(hists, format) \ + perf_hpp_list__for_each_format((hists)->hpp_list, format) + +#define hists__for_each_sort_list(hists, format) \ + perf_hpp_list__for_each_sort_list((hists)->hpp_list, format) + +extern struct perf_hpp_fmt perf_hpp__format[]; + +enum { + /* Matches perf_hpp__format array. */ + PERF_HPP__OVERHEAD, + PERF_HPP__LATENCY, + PERF_HPP__OVERHEAD_SYS, + PERF_HPP__OVERHEAD_US, + PERF_HPP__OVERHEAD_GUEST_SYS, + PERF_HPP__OVERHEAD_GUEST_US, + PERF_HPP__OVERHEAD_ACC, + PERF_HPP__LATENCY_ACC, + PERF_HPP__SAMPLES, + PERF_HPP__PERIOD, + PERF_HPP__WEIGHT1, + PERF_HPP__WEIGHT2, + PERF_HPP__WEIGHT3, + PERF_HPP__MEM_STAT_OP, + PERF_HPP__MEM_STAT_CACHE, + PERF_HPP__MEM_STAT_MEMORY, + PERF_HPP__MEM_STAT_SNOOP, + PERF_HPP__MEM_STAT_DTLB, + + PERF_HPP__MAX_INDEX +}; + +void perf_hpp__init(void); +void perf_hpp__cancel_cumulate(struct evlist *evlist); +void perf_hpp__cancel_latency(struct evlist *evlist); +void perf_hpp__setup_output_field(struct perf_hpp_list *list); +void perf_hpp__reset_output_field(struct perf_hpp_list *list); +void perf_hpp__append_sort_keys(struct perf_hpp_list *list); +int perf_hpp__setup_hists_formats(struct perf_hpp_list *list, + struct evlist *evlist); +int perf_hpp__alloc_mem_stats(struct perf_hpp_list *list, + struct evlist *evlist); + + +bool perf_hpp__is_sort_entry(struct perf_hpp_fmt *format); +bool perf_hpp__is_dynamic_entry(struct perf_hpp_fmt *format); +bool perf_hpp__defined_dynamic_entry(struct perf_hpp_fmt *fmt, struct hists *hists); +bool perf_hpp__is_trace_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_srcline_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_srcfile_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_thread_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_comm_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_dso_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_sym_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_parallelism_entry(struct perf_hpp_fmt *fmt); + +struct perf_hpp_fmt *perf_hpp_fmt__dup(struct perf_hpp_fmt *fmt); + +int hist_entry__filter(struct hist_entry *he, int type, const void *arg); + +static inline bool perf_hpp__should_skip(struct perf_hpp_fmt *format, + struct hists *hists) +{ + if (format->elide) + return true; + + if (perf_hpp__is_dynamic_entry(format) && + !perf_hpp__defined_dynamic_entry(format, hists)) + return true; + + return false; +} + +void perf_hpp__reset_width(struct perf_hpp_fmt *fmt, struct hists *hists); +void perf_hpp__reset_sort_width(struct perf_hpp_fmt *fmt, struct hists *hists); +void perf_hpp__set_user_width(const char *width_list_str); +void hists__reset_column_width(struct hists *hists); + +enum perf_hpp_fmt_type { + PERF_HPP_FMT_TYPE__RAW, + PERF_HPP_FMT_TYPE__PERCENT, + PERF_HPP_FMT_TYPE__LATENCY, + PERF_HPP_FMT_TYPE__AVERAGE, +}; + +typedef u64 (*hpp_field_fn)(struct hist_entry *he); +typedef int (*hpp_callback_fn)(struct perf_hpp *hpp, bool front); +typedef int (*hpp_snprint_fn)(struct perf_hpp *hpp, const char *fmt, ...); + +int hpp__fmt(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he, hpp_field_fn get_field, + const char *fmtstr, hpp_snprint_fn print_fn, + enum perf_hpp_fmt_type fmtype); +int hpp__fmt_acc(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he, hpp_field_fn get_field, + const char *fmtstr, hpp_snprint_fn print_fn, + enum perf_hpp_fmt_type fmtype); +int hpp__fmt_mem_stat(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he, enum mem_stat_type mst, + const char *fmtstr, hpp_snprint_fn print_fn); + +static inline void advance_hpp(struct perf_hpp *hpp, int inc) +{ + hpp->buf += inc; + hpp->size -= inc; +} + +static inline size_t perf_hpp__use_color(void) +{ + return !symbol_conf.field_sep; +} + +static inline size_t perf_hpp__color_overhead(void) +{ + return perf_hpp__use_color() ? + (COLOR_MAXLEN + sizeof(PERF_COLOR_RESET)) * PERF_HPP__MAX_INDEX + : 0; +} + +struct evlist; + +struct hist_browser_timer { + void (*timer)(void *arg); + void *arg; + int refresh; +}; + +enum rstype { + A_NORMAL, + A_ASM, + A_SOURCE +}; + +struct block_hist { + struct hists block_hists; + struct perf_hpp_list block_list; + struct perf_hpp_fmt block_fmt; + int block_idx; + bool valid; + struct hist_entry he; +}; + +#define NO_ADDR 0 + +#ifdef HAVE_SLANG_SUPPORT +#include "../ui/keysyms.h" +void attr_to_script(char *buf, struct perf_event_attr *attr); + +int __hist_entry__tui_annotate(struct hist_entry *he, struct map_symbol *ms, + struct evsel *evsel, + struct hist_browser_timer *hbt, u64 al_addr); + +int hist_entry__tui_annotate(struct hist_entry *he, struct evsel *evsel, + struct hist_browser_timer *hbt, u64 al_addr); + +int evlist__tui_browse_hists(struct evlist *evlist, const char *help, struct hist_browser_timer *hbt, + float min_pcnt, struct perf_env *env, bool warn_lost_event); + +int script_browse(const char *script_opt, struct evsel *evsel); + +void run_script(char *cmd); +int res_sample_browse(struct res_sample *res_samples, int num_res, + struct evsel *evsel, enum rstype rstype); +void res_sample_init(void); + +int block_hists_tui_browse(struct block_hist *bh, struct evsel *evsel, + float min_percent, struct perf_env *env); +#else +static inline +int evlist__tui_browse_hists(struct evlist *evlist __maybe_unused, + const char *help __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused, + float min_pcnt __maybe_unused, + struct perf_env *env __maybe_unused, + bool warn_lost_event __maybe_unused) +{ + return 0; +} +static inline int __hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, + struct map_symbol *ms __maybe_unused, + struct evsel *evsel __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused, + u64 al_addr __maybe_unused) +{ + return 0; +} + +static inline int hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, + struct evsel *evsel __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused, + u64 al_addr __maybe_unused) +{ + return 0; +} + +static inline int script_browse(const char *script_opt __maybe_unused, + struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static inline int res_sample_browse(struct res_sample *res_samples __maybe_unused, + int num_res __maybe_unused, + struct evsel *evsel __maybe_unused, + enum rstype rstype __maybe_unused) +{ + return 0; +} + +static inline void res_sample_init(void) {} + +static inline int block_hists_tui_browse(struct block_hist *bh __maybe_unused, + struct evsel *evsel __maybe_unused, + float min_percent __maybe_unused, + struct perf_env *env __maybe_unused) +{ + return 0; +} + +#define K_LEFT -1000 +#define K_RIGHT -2000 +#define K_SWITCH_INPUT_DATA -3000 +#define K_RELOAD -4000 +#endif + +unsigned int hists__sort_list_width(struct hists *hists); +unsigned int hists__overhead_width(struct hists *hists); + +void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, + struct perf_sample *sample, bool nonany_branch_mode, + u64 *total_cycles); + +struct option; +int parse_filter_percentage(const struct option *opt, const char *arg, int unset); +int perf_hist_config(const char *var, const char *value); + +void perf_hpp_list__init(struct perf_hpp_list *list); + +enum hierarchy_move_dir { + HMD_NORMAL, + HMD_FORCE_SIBLING, + HMD_FORCE_CHILD, +}; + +struct rb_node *rb_hierarchy_last(struct rb_node *node); +struct rb_node *__rb_hierarchy_next(struct rb_node *node, + enum hierarchy_move_dir hmd); +struct rb_node *rb_hierarchy_prev(struct rb_node *node); + +static inline struct rb_node *rb_hierarchy_next(struct rb_node *node) +{ + return __rb_hierarchy_next(node, HMD_NORMAL); +} + +#define HIERARCHY_INDENT 3 + +bool hist_entry__has_hierarchy_children(struct hist_entry *he, float limit); +int hpp_color_scnprintf(struct perf_hpp *hpp, const char *fmt, ...); +int __hpp__slsmg_color_printf(struct perf_hpp *hpp, const char *fmt, ...); +int __hist_entry__snprintf(struct hist_entry *he, struct perf_hpp *hpp, + struct perf_hpp_list *hpp_list); +int hists__fprintf_headers(struct hists *hists, FILE *fp); +int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool show_freq); + +static inline int hists__scnprintf_title(struct hists *hists, char *bf, size_t size) +{ + return __hists__scnprintf_title(hists, bf, size, true); +} + +#endif /* __PERF_HIST_H */ diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c new file mode 100644 index 000000000..ed544dca7 --- /dev/null +++ b/tools/perf/util/hwmon_pmu.c @@ -0,0 +1,839 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "hashmap.h" +#include "hwmon_pmu.h" +#include "pmu.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** Strings that correspond to enum hwmon_type. */ +static const char * const hwmon_type_strs[HWMON_TYPE_MAX] = { + NULL, + "cpu", + "curr", + "energy", + "fan", + "humidity", + "in", + "intrusion", + "power", + "pwm", + "temp", +}; +#define LONGEST_HWMON_TYPE_STR "intrusion" + +/** Strings that correspond to enum hwmon_item. */ +static const char * const hwmon_item_strs[HWMON_ITEM__MAX] = { + NULL, + "accuracy", + "alarm", + "auto_channels_temp", + "average", + "average_highest", + "average_interval", + "average_interval_max", + "average_interval_min", + "average_lowest", + "average_max", + "average_min", + "beep", + "cap", + "cap_hyst", + "cap_max", + "cap_min", + "crit", + "crit_hyst", + "div", + "emergency", + "emergency_hist", + "enable", + "fault", + "freq", + "highest", + "input", + "label", + "lcrit", + "lcrit_hyst", + "lowest", + "max", + "max_hyst", + "min", + "min_hyst", + "mod", + "offset", + "pulses", + "rated_max", + "rated_min", + "reset_history", + "target", + "type", + "vid", +}; +#define LONGEST_HWMON_ITEM_STR "average_interval_max" + +static const char *const hwmon_units[HWMON_TYPE_MAX] = { + NULL, + "V", /* cpu */ + "A", /* curr */ + "J", /* energy */ + "rpm", /* fan */ + "%", /* humidity */ + "V", /* in */ + "", /* intrusion */ + "W", /* power */ + "Hz", /* pwm */ + "'C", /* temp */ +}; + +struct hwmon_pmu { + struct perf_pmu pmu; + struct hashmap events; + char *hwmon_dir; +}; + +/** + * struct hwmon_pmu_event_value: Value in hwmon_pmu->events. + * + * Hwmon files are of the form _ and may have a suffix + * _alarm. + */ +struct hwmon_pmu_event_value { + /** @items: which item files are present. */ + DECLARE_BITMAP(items, HWMON_ITEM__MAX); + /** @alarm_items: which item files are present. */ + DECLARE_BITMAP(alarm_items, HWMON_ITEM__MAX); + /** @label: contents of _label if present. */ + char *label; + /** @name: name computed from label of the form _