From 5febf432df1cfa5b25d99b54c32103fafbdd0eb9 Mon Sep 17 00:00:00 2001 From: Miguel Ojeda Date: Sun, 16 Aug 2026 15:32:33 +0200 Subject: kbuild: rust: keep Rust objects out of Clang LTO with inline helpers Under `CONFIG_LTO_CLANG` + `CONFIG_RUST_INLINE_HELPERS`, one may hit `objtool` errors such as: vmlinux.o: error: objtool: _R..._3Gsp4boot+0xd6a: can't find jump dest instruction at .text._R..._3Gsp4boot+0x1dfd The reason is that in such builds, the Clang invocation that compiles the combined Rust plus helpers bitcode emits LLVM bitcode (again) -- the final code generation happens in the linker's LTO step, which the `-mllvm` trap options passed to Clang do not reach. This, in turn, means that unreachable traps are missing, and the impossible paths do not merely fallthrough to the next symbol, but past the end of their own section, since LTO builds place each function in its own section. Thus filter `CC_FLAGS_LTO` out of the Clang invocation, so that it always emits machine code directly, with the traps in place. Assisted-by: LLM Cc: Gary Guo Cc: Boqun Feng Cc: Alice Ryhl Cc: Matthew Maurer Cc: Josh Poimboeuf Cc: Peter Zijlstra Cc: stable@vger.kernel.org Fixes: 3a2486cc1da5 ("kbuild: rust: provide an option to inline C helpers into Rust") Acked-by: Gary Guo Link: https://patch.msgid.link/20260816133233.197500-2-ojeda@kernel.org Signed-off-by: Miguel Ojeda --- tools/perf/util/Build | 458 ++ tools/perf/util/PERF-VERSION-GEN | 52 + tools/perf/util/addr2line.c | 436 ++ tools/perf/util/addr2line.h | 18 + tools/perf/util/addr_location.c | 41 + tools/perf/util/addr_location.h | 34 + tools/perf/util/affinity.c | 103 + tools/perf/util/affinity.h | 19 + tools/perf/util/amd-sample-raw.c | 463 ++ tools/perf/util/annotate-arch/Build | 11 + tools/perf/util/annotate-arch/annotate-arc.c | 18 + tools/perf/util/annotate-arch/annotate-arm.c | 68 + tools/perf/util/annotate-arch/annotate-arm64.c | 127 + tools/perf/util/annotate-arch/annotate-csky.c | 56 + tools/perf/util/annotate-arch/annotate-loongarch.c | 157 + tools/perf/util/annotate-arch/annotate-mips.c | 53 + tools/perf/util/annotate-arch/annotate-powerpc.c | 410 ++ tools/perf/util/annotate-arch/annotate-riscv64.c | 41 + tools/perf/util/annotate-arch/annotate-s390.c | 190 + tools/perf/util/annotate-arch/annotate-sparc.c | 177 + tools/perf/util/annotate-arch/annotate-x86.c | 852 +++ tools/perf/util/annotate-data.c | 1960 +++++++ tools/perf/util/annotate-data.h | 322 ++ tools/perf/util/annotate.c | 3205 +++++++++++ tools/perf/util/annotate.h | 588 ++ tools/perf/util/arm-spe-decoder/Build | 3 + tools/perf/util/arm-spe-decoder/arm-spe-decoder.c | 304 + tools/perf/util/arm-spe-decoder/arm-spe-decoder.h | 158 + .../util/arm-spe-decoder/arm-spe-pkt-decoder.c | 614 +++ .../util/arm-spe-decoder/arm-spe-pkt-decoder.h | 191 + tools/perf/util/arm-spe.c | 2064 +++++++ tools/perf/util/arm-spe.h | 71 + .../perf/util/arm64-frame-pointer-unwind-support.c | 77 + .../perf/util/arm64-frame-pointer-unwind-support.h | 14 + tools/perf/util/aslr.c | 1409 +++++ tools/perf/util/aslr.h | 44 + tools/perf/util/auxtrace.c | 2994 ++++++++++ tools/perf/util/auxtrace.h | 702 +++ tools/perf/util/blake2s.c | 151 + tools/perf/util/blake2s.h | 73 + tools/perf/util/block-info.c | 532 ++ tools/perf/util/block-info.h | 76 + tools/perf/util/block-range.c | 332 ++ tools/perf/util/block-range.h | 76 + tools/perf/util/bpf-event.c | 1016 ++++ tools/perf/util/bpf-event.h | 73 + tools/perf/util/bpf-filter.c | 807 +++ tools/perf/util/bpf-filter.h | 70 + tools/perf/util/bpf-filter.l | 179 + tools/perf/util/bpf-filter.y | 109 + tools/perf/util/bpf-trace-summary.c | 465 ++ tools/perf/util/bpf-utils.c | 293 + tools/perf/util/bpf-utils.h | 86 + tools/perf/util/bpf_counter.c | 902 +++ tools/perf/util/bpf_counter.h | 75 + tools/perf/util/bpf_counter_cgroup.c | 351 ++ tools/perf/util/bpf_ftrace.c | 214 + tools/perf/util/bpf_kwork.c | 359 ++ tools/perf/util/bpf_kwork_top.c | 310 ++ tools/perf/util/bpf_lock_contention.c | 862 +++ tools/perf/util/bpf_off_cpu.c | 428 ++ tools/perf/util/bpf_skel/.gitignore | 4 + .../util/bpf_skel/augmented_raw_syscalls.bpf.c | 580 ++ tools/perf/util/bpf_skel/bperf_cgroup.bpf.c | 225 + tools/perf/util/bpf_skel/bperf_cgroup.h | 15 + tools/perf/util/bpf_skel/bperf_follower.bpf.c | 162 + tools/perf/util/bpf_skel/bperf_leader.bpf.c | 55 + tools/perf/util/bpf_skel/bperf_u.h | 19 + tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c | 92 + tools/perf/util/bpf_skel/func_latency.bpf.c | 199 + tools/perf/util/bpf_skel/kwork_top.bpf.c | 338 ++ tools/perf/util/bpf_skel/kwork_trace.bpf.c | 384 ++ tools/perf/util/bpf_skel/lock_contention.bpf.c | 1052 ++++ tools/perf/util/bpf_skel/lock_data.h | 79 + tools/perf/util/bpf_skel/off_cpu.bpf.c | 372 ++ tools/perf/util/bpf_skel/perf_version.h | 17 + tools/perf/util/bpf_skel/sample-filter.h | 72 + tools/perf/util/bpf_skel/sample_filter.bpf.c | 298 + tools/perf/util/bpf_skel/syscall_summary.bpf.c | 165 + tools/perf/util/bpf_skel/syscall_summary.h | 27 + tools/perf/util/bpf_skel/vmlinux/.gitignore | 1 + tools/perf/util/bpf_skel/vmlinux/vmlinux.h | 215 + tools/perf/util/bpf_trace_augment.c | 143 + tools/perf/util/branch.c | 229 + tools/perf/util/branch.h | 99 + tools/perf/util/btf.c | 27 + tools/perf/util/btf.h | 10 + tools/perf/util/build-id.c | 983 ++++ tools/perf/util/build-id.h | 79 + tools/perf/util/cache.h | 31 + tools/perf/util/cacheline.c | 25 + tools/perf/util/cacheline.h | 36 + tools/perf/util/call-path.c | 114 + tools/perf/util/call-path.h | 68 + tools/perf/util/callchain.c | 1943 +++++++ tools/perf/util/callchain.h | 315 ++ tools/perf/util/cap.c | 48 + tools/perf/util/cap.h | 24 + tools/perf/util/capstone.c | 615 +++ tools/perf/util/capstone.h | 57 + tools/perf/util/cgroup.c | 646 +++ tools/perf/util/cgroup.h | 58 + tools/perf/util/clockid.c | 119 + tools/perf/util/clockid.h | 12 + tools/perf/util/cloexec.c | 92 + tools/perf/util/cloexec.h | 7 + tools/perf/util/color.c | 156 + tools/perf/util/color.h | 45 + tools/perf/util/color_config.c | 37 + tools/perf/util/comm.c | 237 + tools/perf/util/comm.h | 28 + tools/perf/util/compress.h | 91 + tools/perf/util/config.c | 995 ++++ tools/perf/util/config.h | 80 + tools/perf/util/copyfile.c | 146 + tools/perf/util/copyfile.h | 16 + tools/perf/util/counts.c | 71 + tools/perf/util/counts.h | 45 + tools/perf/util/cpumap.c | 824 +++ tools/perf/util/cpumap.h | 162 + tools/perf/util/cputopo.c | 506 ++ tools/perf/util/cputopo.h | 78 + tools/perf/util/cs-etm-base.c | 195 + tools/perf/util/cs-etm-decoder/Build | 1 + tools/perf/util/cs-etm-decoder/cs-etm-decoder.c | 809 +++ tools/perf/util/cs-etm-decoder/cs-etm-decoder.h | 111 + tools/perf/util/cs-etm.c | 3671 +++++++++++++ tools/perf/util/cs-etm.h | 286 + tools/perf/util/data-convert-bt.c | 1832 +++++++ tools/perf/util/data-convert-json.c | 461 ++ tools/perf/util/data-convert.h | 22 + tools/perf/util/data.c | 580 ++ tools/perf/util/data.h | 142 + tools/perf/util/db-export.c | 615 +++ tools/perf/util/db-export.h | 109 + tools/perf/util/debug.c | 380 ++ tools/perf/util/debug.h | 92 + tools/perf/util/debuginfo.c | 215 + tools/perf/util/debuginfo.h | 66 + tools/perf/util/demangle-cxx.cpp | 49 + tools/perf/util/demangle-cxx.h | 18 + tools/perf/util/demangle-java.c | 203 + tools/perf/util/demangle-java.h | 11 + tools/perf/util/demangle-ocaml.c | 68 + tools/perf/util/demangle-ocaml.h | 7 + tools/perf/util/demangle-rust-v0.c | 2042 +++++++ tools/perf/util/demangle-rust-v0.h | 88 + tools/perf/util/disasm.c | 1648 ++++++ tools/perf/util/disasm.h | 170 + tools/perf/util/dlfilter.c | 662 +++ tools/perf/util/dlfilter.h | 99 + tools/perf/util/drm_pmu.c | 685 +++ tools/perf/util/drm_pmu.h | 39 + tools/perf/util/dso.c | 2045 +++++++ tools/perf/util/dso.h | 954 ++++ tools/perf/util/dsos.c | 495 ++ tools/perf/util/dsos.h | 52 + tools/perf/util/dump-insn.h | 26 + tools/perf/util/dwarf-aux.c | 2182 ++++++++ tools/perf/util/dwarf-aux.h | 192 + tools/perf/util/dwarf-regs-arch/Build | 9 + tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c | 126 + .../util/dwarf-regs-arch/dwarf-regs-loongarch.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c | 14 + .../perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c | 136 + tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c | 12 + tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c | 53 + tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c | 260 + tools/perf/util/dwarf-regs.c | 237 + tools/perf/util/env.c | 1087 ++++ tools/perf/util/env.h | 245 + tools/perf/util/event.c | 944 ++++ tools/perf/util/event.h | 437 ++ tools/perf/util/events_stats.h | 72 + tools/perf/util/evlist.c | 2701 +++++++++ tools/perf/util/evlist.h | 464 ++ tools/perf/util/evsel.c | 4667 ++++++++++++++++ tools/perf/util/evsel.h | 627 +++ tools/perf/util/evsel_config.h | 69 + tools/perf/util/evsel_fprintf.c | 258 + tools/perf/util/evsel_fprintf.h | 50 + tools/perf/util/evswitch.c | 62 + tools/perf/util/evswitch.h | 31 + tools/perf/util/expr.c | 489 ++ tools/perf/util/expr.h | 64 + tools/perf/util/expr.l | 150 + tools/perf/util/expr.y | 391 ++ tools/perf/util/find-map.c | 30 + tools/perf/util/fncache.c | 68 + tools/perf/util/fncache.h | 6 + tools/perf/util/ftrace.h | 96 + tools/perf/util/genelf.c | 471 ++ tools/perf/util/genelf.h | 89 + tools/perf/util/genelf_debug.c | 625 +++ tools/perf/util/hashmap.c | 240 + tools/perf/util/hashmap.h | 208 + tools/perf/util/header.c | 5799 ++++++++++++++++++++ tools/perf/util/header.h | 225 + tools/perf/util/help-unknown-cmd.c | 122 + tools/perf/util/help-unknown-cmd.h | 0 tools/perf/util/hisi-ptt-decoder/Build | 1 + .../util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c | 164 + .../util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h | 31 + tools/perf/util/hisi-ptt.c | 190 + tools/perf/util/hisi-ptt.h | 19 + tools/perf/util/hist.c | 3099 +++++++++++ tools/perf/util/hist.h | 841 +++ tools/perf/util/hwmon_pmu.c | 839 +++ tools/perf/util/hwmon_pmu.h | 167 + tools/perf/util/include/asm/asm-offsets.h | 2 + tools/perf/util/include/asm/cpufeature.h | 10 + tools/perf/util/include/asm/dwarf2.h | 14 + tools/perf/util/include/asm/swab.h | 1 + tools/perf/util/include/asm/system.h | 1 + tools/perf/util/include/asm/uaccess.h | 15 + tools/perf/util/include/dwarf-regs.h | 149 + tools/perf/util/include/linux/linkage.h | 139 + tools/perf/util/intel-bts.c | 911 +++ tools/perf/util/intel-bts.h | 34 + tools/perf/util/intel-pt-decoder/Build | 26 + .../perf/util/intel-pt-decoder/intel-pt-decoder.c | 4515 +++++++++++++++ .../perf/util/intel-pt-decoder/intel-pt-decoder.h | 318 ++ .../util/intel-pt-decoder/intel-pt-insn-decoder.c | 313 ++ .../util/intel-pt-decoder/intel-pt-insn-decoder.h | 58 + tools/perf/util/intel-pt-decoder/intel-pt-log.c | 267 + tools/perf/util/intel-pt-decoder/intel-pt-log.h | 76 + .../util/intel-pt-decoder/intel-pt-pkt-decoder.c | 791 +++ .../util/intel-pt-decoder/intel-pt-pkt-decoder.h | 92 + tools/perf/util/intel-pt.c | 4746 ++++++++++++++++ tools/perf/util/intel-pt.h | 48 + tools/perf/util/intel-tpebs.c | 720 +++ tools/perf/util/intel-tpebs.h | 25 + tools/perf/util/intlist.c | 150 + tools/perf/util/intlist.h | 78 + tools/perf/util/iostat.c | 54 + tools/perf/util/iostat.h | 47 + tools/perf/util/jit.h | 13 + tools/perf/util/jitdump.c | 917 ++++ tools/perf/util/jitdump.h | 139 + tools/perf/util/kvm-stat-arch/Build | 6 + .../util/kvm-stat-arch/arm64_exception_types.h | 97 + tools/perf/util/kvm-stat-arch/book3s_hcalls.h | 124 + tools/perf/util/kvm-stat-arch/book3s_hv_exits.h | 33 + tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c | 92 + tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c | 149 + tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c | 225 + tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c | 87 + tools/perf/util/kvm-stat-arch/kvm-stat-s390.c | 116 + tools/perf/util/kvm-stat-arch/kvm-stat-x86.c | 262 + tools/perf/util/kvm-stat-arch/riscv_trap_types.h | 57 + tools/perf/util/kvm-stat.c | 270 + tools/perf/util/kvm-stat.h | 234 + tools/perf/util/kwork.h | 326 ++ tools/perf/util/levenshtein.c | 87 + tools/perf/util/levenshtein.h | 9 + tools/perf/util/libbfd.c | 645 +++ tools/perf/util/libbfd.h | 82 + tools/perf/util/libdw.c | 216 + tools/perf/util/libdw.h | 58 + tools/perf/util/libunwind-arch/Build | 11 + tools/perf/util/libunwind-arch/libunwind-arch.c | 319 ++ tools/perf/util/libunwind-arch/libunwind-arch.h | 296 + tools/perf/util/libunwind-arch/libunwind-arm.c | 290 + tools/perf/util/libunwind-arch/libunwind-arm64.c | 289 + tools/perf/util/libunwind-arch/libunwind-i386.c | 312 ++ .../perf/util/libunwind-arch/libunwind-loongarch.c | 297 + tools/perf/util/libunwind-arch/libunwind-mips.c | 299 + tools/perf/util/libunwind-arch/libunwind-ppc32.c | 303 + tools/perf/util/libunwind-arch/libunwind-ppc64.c | 303 + tools/perf/util/libunwind-arch/libunwind-riscv.c | 297 + tools/perf/util/libunwind-arch/libunwind-s390.c | 299 + tools/perf/util/libunwind-arch/libunwind-x86_64.c | 320 ++ tools/perf/util/llvm-c-helpers.cpp | 196 + tools/perf/util/llvm-c-helpers.h | 60 + tools/perf/util/llvm.c | 279 + tools/perf/util/llvm.h | 21 + tools/perf/util/lock-contention.c | 143 + tools/perf/util/lock-contention.h | 208 + tools/perf/util/lzma.c | 129 + tools/perf/util/machine.c | 3369 ++++++++++++ tools/perf/util/machine.h | 331 ++ tools/perf/util/map.c | 650 +++ tools/perf/util/map.h | 319 ++ tools/perf/util/map_symbol.c | 34 + tools/perf/util/map_symbol.h | 33 + tools/perf/util/maps.c | 1468 +++++ tools/perf/util/maps.h | 88 + tools/perf/util/mem-events.c | 982 ++++ tools/perf/util/mem-events.h | 149 + tools/perf/util/mem-info.c | 48 + tools/perf/util/mem-info.h | 55 + tools/perf/util/mem2node.c | 139 + tools/perf/util/mem2node.h | 20 + tools/perf/util/memswap.c | 25 + tools/perf/util/memswap.h | 15 + tools/perf/util/metricgroup.c | 1738 ++++++ tools/perf/util/metricgroup.h | 96 + tools/perf/util/mmap.c | 369 ++ tools/perf/util/mmap.h | 64 + tools/perf/util/mutex.c | 125 + tools/perf/util/mutex.h | 129 + tools/perf/util/namespaces.c | 380 ++ tools/perf/util/namespaces.h | 84 + tools/perf/util/off_cpu.h | 39 + tools/perf/util/ordered-events.c | 417 ++ tools/perf/util/ordered-events.h | 84 + tools/perf/util/parse-branch-options.c | 115 + tools/perf/util/parse-branch-options.h | 7 + tools/perf/util/parse-events.c | 2953 ++++++++++ tools/perf/util/parse-events.h | 290 + tools/perf/util/parse-events.l | 372 ++ tools/perf/util/parse-events.y | 647 +++ tools/perf/util/parse-regs-options.c | 124 + tools/perf/util/parse-regs-options.h | 7 + tools/perf/util/parse-sublevel-options.c | 70 + tools/perf/util/parse-sublevel-options.h | 11 + tools/perf/util/path.c | 79 + tools/perf/util/path.h | 17 + tools/perf/util/perf-hooks-list.h | 3 + tools/perf/util/perf-hooks.c | 90 + tools/perf/util/perf-hooks.h | 40 + tools/perf/util/perf-regs-arch/Build | 9 + tools/perf/util/perf-regs-arch/perf_regs_aarch64.c | 229 + tools/perf/util/perf-regs-arch/perf_regs_arm.c | 61 + tools/perf/util/perf-regs-arch/perf_regs_csky.c | 108 + .../perf/util/perf-regs-arch/perf_regs_loongarch.c | 92 + tools/perf/util/perf-regs-arch/perf_regs_mips.c | 88 + tools/perf/util/perf-regs-arch/perf_regs_powerpc.c | 322 ++ tools/perf/util/perf-regs-arch/perf_regs_riscv.c | 221 + tools/perf/util/perf-regs-arch/perf_regs_s390.c | 175 + tools/perf/util/perf-regs-arch/perf_regs_x86.c | 373 ++ tools/perf/util/perf_api_probe.c | 208 + tools/perf/util/perf_api_probe.h | 17 + tools/perf/util/perf_event_attr_fprintf.c | 410 ++ tools/perf/util/perf_regs.c | 254 + tools/perf/util/perf_regs.h | 79 + tools/perf/util/pfm.c | 321 ++ tools/perf/util/pfm.h | 38 + tools/perf/util/pmu.c | 2786 ++++++++++ tools/perf/util/pmu.h | 375 ++ tools/perf/util/pmu.l | 48 + tools/perf/util/pmu.y | 101 + tools/perf/util/pmus.c | 903 +++ tools/perf/util/pmus.h | 43 + tools/perf/util/powerpc-vpadtl.c | 736 +++ tools/perf/util/powerpc-vpadtl.h | 23 + tools/perf/util/print-events.c | 357 ++ tools/perf/util/print-events.h | 39 + tools/perf/util/print_binary.c | 58 + tools/perf/util/print_binary.h | 37 + tools/perf/util/print_insn.c | 72 + tools/perf/util/print_insn.h | 25 + tools/perf/util/probe-event.c | 3844 +++++++++++++ tools/perf/util/probe-event.h | 199 + tools/perf/util/probe-file.c | 1182 ++++ tools/perf/util/probe-file.h | 82 + tools/perf/util/probe-finder.c | 1991 +++++++ tools/perf/util/probe-finder.h | 114 + tools/perf/util/pstack.c | 71 + tools/perf/util/pstack.h | 15 + tools/perf/util/python.c | 2329 ++++++++ tools/perf/util/rblist.c | 140 + tools/perf/util/rblist.h | 50 + tools/perf/util/record.c | 291 + tools/perf/util/record.h | 98 + tools/perf/util/rlimit.c | 57 + tools/perf/util/rlimit.h | 15 + tools/perf/util/rwsem.c | 71 + tools/perf/util/rwsem.h | 30 + tools/perf/util/s390-cpumcf-kernel.h | 65 + tools/perf/util/s390-cpumsf-kernel.h | 71 + tools/perf/util/s390-cpumsf.c | 1174 ++++ tools/perf/util/s390-cpumsf.h | 21 + tools/perf/util/s390-sample-raw.c | 380 ++ tools/perf/util/sample-raw.c | 28 + tools/perf/util/sample-raw.h | 19 + tools/perf/util/sample.c | 127 + tools/perf/util/sample.h | 268 + tools/perf/util/scripting-engines/Build | 9 + .../perf/util/scripting-engines/trace-event-perl.c | 770 +++ .../util/scripting-engines/trace-event-python.c | 2224 ++++++++ tools/perf/util/session.c | 4132 ++++++++++++++ tools/perf/util/session.h | 218 + tools/perf/util/setns.c | 10 + tools/perf/util/setup.py | 107 + tools/perf/util/sharded_mutex.c | 33 + tools/perf/util/sharded_mutex.h | 29 + tools/perf/util/sideband_evlist.c | 149 + tools/perf/util/smt.c | 36 + tools/perf/util/smt.h | 17 + tools/perf/util/sort.c | 4491 +++++++++++++++ tools/perf/util/sort.h | 163 + tools/perf/util/spark.c | 32 + tools/perf/util/spark.h | 9 + tools/perf/util/srccode.c | 171 + tools/perf/util/srccode.h | 20 + tools/perf/util/srcline.c | 508 ++ tools/perf/util/srcline.h | 69 + tools/perf/util/stat-display.c | 1601 ++++++ tools/perf/util/stat-shadow.c | 355 ++ tools/perf/util/stat.c | 721 +++ tools/perf/util/stat.h | 209 + tools/perf/util/strbuf.c | 170 + tools/perf/util/strbuf.h | 95 + tools/perf/util/stream.c | 341 ++ tools/perf/util/stream.h | 41 + tools/perf/util/strfilter.c | 312 ++ tools/perf/util/strfilter.h | 84 + tools/perf/util/string.c | 460 ++ tools/perf/util/string2.h | 46 + tools/perf/util/strlist.c | 203 + tools/perf/util/strlist.h | 88 + tools/perf/util/svghelper.c | 812 +++ tools/perf/util/svghelper.h | 39 + tools/perf/util/symbol-elf.c | 2990 ++++++++++ tools/perf/util/symbol-minimal.c | 379 ++ tools/perf/util/symbol.c | 2766 ++++++++++ tools/perf/util/symbol.h | 343 ++ tools/perf/util/symbol_conf.h | 117 + tools/perf/util/symbol_fprintf.c | 73 + tools/perf/util/symsrc.h | 47 + tools/perf/util/synthetic-events.c | 2769 ++++++++++ tools/perf/util/synthetic-events.h | 134 + tools/perf/util/target.c | 142 + tools/perf/util/target.h | 98 + tools/perf/util/term.c | 40 + tools/perf/util/term.h | 11 + tools/perf/util/thread-stack.c | 1239 +++++ tools/perf/util/thread-stack.h | 110 + tools/perf/util/thread.c | 638 +++ tools/perf/util/thread.h | 398 ++ tools/perf/util/thread_map.c | 397 ++ tools/perf/util/thread_map.h | 27 + tools/perf/util/threads.c | 190 + tools/perf/util/threads.h | 35 + tools/perf/util/time-utils.c | 559 ++ tools/perf/util/time-utils.h | 51 + tools/perf/util/tool.c | 572 ++ tools/perf/util/tool.h | 116 + tools/perf/util/tool_pmu.c | 723 +++ tools/perf/util/tool_pmu.h | 67 + tools/perf/util/top.c | 119 + tools/perf/util/top.h | 66 + tools/perf/util/topdown.c | 8 + tools/perf/util/topdown.h | 11 + tools/perf/util/tp_pmu.c | 208 + tools/perf/util/tp_pmu.h | 19 + tools/perf/util/trace-event-info.c | 703 +++ tools/perf/util/trace-event-parse.c | 323 ++ tools/perf/util/trace-event-read.c | 480 ++ tools/perf/util/trace-event-scripting.c | 407 ++ tools/perf/util/trace-event.c | 118 + tools/perf/util/trace-event.h | 162 + tools/perf/util/trace.h | 38 + tools/perf/util/trace_augment.h | 66 + tools/perf/util/tracepoint.c | 52 + tools/perf/util/tracepoint.h | 26 + tools/perf/util/trigger.h | 95 + tools/perf/util/tsc.c | 142 + tools/perf/util/tsc.h | 32 + tools/perf/util/units.c | 76 + tools/perf/util/units.h | 19 + tools/perf/util/unwind-libdw.c | 447 ++ tools/perf/util/unwind-libdw.h | 32 + tools/perf/util/unwind-libunwind.c | 683 +++ tools/perf/util/unwind.c | 104 + tools/perf/util/unwind.h | 85 + tools/perf/util/usage.c | 34 + tools/perf/util/util.c | 565 ++ tools/perf/util/util.h | 126 + tools/perf/util/values.c | 289 + tools/perf/util/values.h | 30 + tools/perf/util/vdso.c | 369 ++ tools/perf/util/vdso.h | 30 + tools/perf/util/zlib.c | 97 + tools/perf/util/zstd.c | 149 + 478 files changed, 188299 insertions(+) create mode 100644 tools/perf/util/Build create mode 100755 tools/perf/util/PERF-VERSION-GEN create mode 100644 tools/perf/util/addr2line.c create mode 100644 tools/perf/util/addr2line.h create mode 100644 tools/perf/util/addr_location.c create mode 100644 tools/perf/util/addr_location.h create mode 100644 tools/perf/util/affinity.c create mode 100644 tools/perf/util/affinity.h create mode 100644 tools/perf/util/amd-sample-raw.c create mode 100644 tools/perf/util/annotate-arch/Build create mode 100644 tools/perf/util/annotate-arch/annotate-arc.c create mode 100644 tools/perf/util/annotate-arch/annotate-arm.c create mode 100644 tools/perf/util/annotate-arch/annotate-arm64.c create mode 100644 tools/perf/util/annotate-arch/annotate-csky.c create mode 100644 tools/perf/util/annotate-arch/annotate-loongarch.c create mode 100644 tools/perf/util/annotate-arch/annotate-mips.c create mode 100644 tools/perf/util/annotate-arch/annotate-powerpc.c create mode 100644 tools/perf/util/annotate-arch/annotate-riscv64.c create mode 100644 tools/perf/util/annotate-arch/annotate-s390.c create mode 100644 tools/perf/util/annotate-arch/annotate-sparc.c create mode 100644 tools/perf/util/annotate-arch/annotate-x86.c create mode 100644 tools/perf/util/annotate-data.c create mode 100644 tools/perf/util/annotate-data.h create mode 100644 tools/perf/util/annotate.c create mode 100644 tools/perf/util/annotate.h create mode 100644 tools/perf/util/arm-spe-decoder/Build create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-decoder.c create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-decoder.h create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c create mode 100644 tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h create mode 100644 tools/perf/util/arm-spe.c create mode 100644 tools/perf/util/arm-spe.h create mode 100644 tools/perf/util/arm64-frame-pointer-unwind-support.c create mode 100644 tools/perf/util/arm64-frame-pointer-unwind-support.h create mode 100644 tools/perf/util/aslr.c create mode 100644 tools/perf/util/aslr.h create mode 100644 tools/perf/util/auxtrace.c create mode 100644 tools/perf/util/auxtrace.h create mode 100644 tools/perf/util/blake2s.c create mode 100644 tools/perf/util/blake2s.h create mode 100644 tools/perf/util/block-info.c create mode 100644 tools/perf/util/block-info.h create mode 100644 tools/perf/util/block-range.c create mode 100644 tools/perf/util/block-range.h create mode 100644 tools/perf/util/bpf-event.c create mode 100644 tools/perf/util/bpf-event.h create mode 100644 tools/perf/util/bpf-filter.c create mode 100644 tools/perf/util/bpf-filter.h create mode 100644 tools/perf/util/bpf-filter.l create mode 100644 tools/perf/util/bpf-filter.y create mode 100644 tools/perf/util/bpf-trace-summary.c create mode 100644 tools/perf/util/bpf-utils.c create mode 100644 tools/perf/util/bpf-utils.h create mode 100644 tools/perf/util/bpf_counter.c create mode 100644 tools/perf/util/bpf_counter.h create mode 100644 tools/perf/util/bpf_counter_cgroup.c create mode 100644 tools/perf/util/bpf_ftrace.c create mode 100644 tools/perf/util/bpf_kwork.c create mode 100644 tools/perf/util/bpf_kwork_top.c create mode 100644 tools/perf/util/bpf_lock_contention.c create mode 100644 tools/perf/util/bpf_off_cpu.c create mode 100644 tools/perf/util/bpf_skel/.gitignore create mode 100644 tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_cgroup.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_cgroup.h create mode 100644 tools/perf/util/bpf_skel/bperf_follower.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_leader.bpf.c create mode 100644 tools/perf/util/bpf_skel/bperf_u.h create mode 100644 tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c create mode 100644 tools/perf/util/bpf_skel/func_latency.bpf.c create mode 100644 tools/perf/util/bpf_skel/kwork_top.bpf.c create mode 100644 tools/perf/util/bpf_skel/kwork_trace.bpf.c create mode 100644 tools/perf/util/bpf_skel/lock_contention.bpf.c create mode 100644 tools/perf/util/bpf_skel/lock_data.h create mode 100644 tools/perf/util/bpf_skel/off_cpu.bpf.c create mode 100644 tools/perf/util/bpf_skel/perf_version.h create mode 100644 tools/perf/util/bpf_skel/sample-filter.h create mode 100644 tools/perf/util/bpf_skel/sample_filter.bpf.c create mode 100644 tools/perf/util/bpf_skel/syscall_summary.bpf.c create mode 100644 tools/perf/util/bpf_skel/syscall_summary.h create mode 100644 tools/perf/util/bpf_skel/vmlinux/.gitignore create mode 100644 tools/perf/util/bpf_skel/vmlinux/vmlinux.h create mode 100644 tools/perf/util/bpf_trace_augment.c create mode 100644 tools/perf/util/branch.c create mode 100644 tools/perf/util/branch.h create mode 100644 tools/perf/util/btf.c create mode 100644 tools/perf/util/btf.h create mode 100644 tools/perf/util/build-id.c create mode 100644 tools/perf/util/build-id.h create mode 100644 tools/perf/util/cache.h create mode 100644 tools/perf/util/cacheline.c create mode 100644 tools/perf/util/cacheline.h create mode 100644 tools/perf/util/call-path.c create mode 100644 tools/perf/util/call-path.h create mode 100644 tools/perf/util/callchain.c create mode 100644 tools/perf/util/callchain.h create mode 100644 tools/perf/util/cap.c create mode 100644 tools/perf/util/cap.h create mode 100644 tools/perf/util/capstone.c create mode 100644 tools/perf/util/capstone.h create mode 100644 tools/perf/util/cgroup.c create mode 100644 tools/perf/util/cgroup.h create mode 100644 tools/perf/util/clockid.c create mode 100644 tools/perf/util/clockid.h create mode 100644 tools/perf/util/cloexec.c create mode 100644 tools/perf/util/cloexec.h create mode 100644 tools/perf/util/color.c create mode 100644 tools/perf/util/color.h create mode 100644 tools/perf/util/color_config.c create mode 100644 tools/perf/util/comm.c create mode 100644 tools/perf/util/comm.h create mode 100644 tools/perf/util/compress.h create mode 100644 tools/perf/util/config.c create mode 100644 tools/perf/util/config.h create mode 100644 tools/perf/util/copyfile.c create mode 100644 tools/perf/util/copyfile.h create mode 100644 tools/perf/util/counts.c create mode 100644 tools/perf/util/counts.h create mode 100644 tools/perf/util/cpumap.c create mode 100644 tools/perf/util/cpumap.h create mode 100644 tools/perf/util/cputopo.c create mode 100644 tools/perf/util/cputopo.h create mode 100644 tools/perf/util/cs-etm-base.c create mode 100644 tools/perf/util/cs-etm-decoder/Build create mode 100644 tools/perf/util/cs-etm-decoder/cs-etm-decoder.c create mode 100644 tools/perf/util/cs-etm-decoder/cs-etm-decoder.h create mode 100644 tools/perf/util/cs-etm.c create mode 100644 tools/perf/util/cs-etm.h create mode 100644 tools/perf/util/data-convert-bt.c create mode 100644 tools/perf/util/data-convert-json.c create mode 100644 tools/perf/util/data-convert.h create mode 100644 tools/perf/util/data.c create mode 100644 tools/perf/util/data.h create mode 100644 tools/perf/util/db-export.c create mode 100644 tools/perf/util/db-export.h create mode 100644 tools/perf/util/debug.c create mode 100644 tools/perf/util/debug.h create mode 100644 tools/perf/util/debuginfo.c create mode 100644 tools/perf/util/debuginfo.h create mode 100644 tools/perf/util/demangle-cxx.cpp create mode 100644 tools/perf/util/demangle-cxx.h create mode 100644 tools/perf/util/demangle-java.c create mode 100644 tools/perf/util/demangle-java.h create mode 100644 tools/perf/util/demangle-ocaml.c create mode 100644 tools/perf/util/demangle-ocaml.h create mode 100644 tools/perf/util/demangle-rust-v0.c create mode 100644 tools/perf/util/demangle-rust-v0.h create mode 100644 tools/perf/util/disasm.c create mode 100644 tools/perf/util/disasm.h create mode 100644 tools/perf/util/dlfilter.c create mode 100644 tools/perf/util/dlfilter.h create mode 100644 tools/perf/util/drm_pmu.c create mode 100644 tools/perf/util/drm_pmu.h create mode 100644 tools/perf/util/dso.c create mode 100644 tools/perf/util/dso.h create mode 100644 tools/perf/util/dsos.c create mode 100644 tools/perf/util/dsos.h create mode 100644 tools/perf/util/dump-insn.h create mode 100644 tools/perf/util/dwarf-aux.c create mode 100644 tools/perf/util/dwarf-aux.h create mode 100644 tools/perf/util/dwarf-regs-arch/Build create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c create mode 100644 tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c create mode 100644 tools/perf/util/dwarf-regs.c create mode 100644 tools/perf/util/env.c create mode 100644 tools/perf/util/env.h create mode 100644 tools/perf/util/event.c create mode 100644 tools/perf/util/event.h create mode 100644 tools/perf/util/events_stats.h create mode 100644 tools/perf/util/evlist.c create mode 100644 tools/perf/util/evlist.h create mode 100644 tools/perf/util/evsel.c create mode 100644 tools/perf/util/evsel.h create mode 100644 tools/perf/util/evsel_config.h create mode 100644 tools/perf/util/evsel_fprintf.c create mode 100644 tools/perf/util/evsel_fprintf.h create mode 100644 tools/perf/util/evswitch.c create mode 100644 tools/perf/util/evswitch.h create mode 100644 tools/perf/util/expr.c create mode 100644 tools/perf/util/expr.h create mode 100644 tools/perf/util/expr.l create mode 100644 tools/perf/util/expr.y create mode 100644 tools/perf/util/find-map.c create mode 100644 tools/perf/util/fncache.c create mode 100644 tools/perf/util/fncache.h create mode 100644 tools/perf/util/ftrace.h create mode 100644 tools/perf/util/genelf.c create mode 100644 tools/perf/util/genelf.h create mode 100644 tools/perf/util/genelf_debug.c create mode 100644 tools/perf/util/hashmap.c create mode 100644 tools/perf/util/hashmap.h create mode 100644 tools/perf/util/header.c create mode 100644 tools/perf/util/header.h create mode 100644 tools/perf/util/help-unknown-cmd.c create mode 100644 tools/perf/util/help-unknown-cmd.h create mode 100644 tools/perf/util/hisi-ptt-decoder/Build create mode 100644 tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c create mode 100644 tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h create mode 100644 tools/perf/util/hisi-ptt.c create mode 100644 tools/perf/util/hisi-ptt.h create mode 100644 tools/perf/util/hist.c create mode 100644 tools/perf/util/hist.h create mode 100644 tools/perf/util/hwmon_pmu.c create mode 100644 tools/perf/util/hwmon_pmu.h create mode 100644 tools/perf/util/include/asm/asm-offsets.h create mode 100644 tools/perf/util/include/asm/cpufeature.h create mode 100644 tools/perf/util/include/asm/dwarf2.h create mode 100644 tools/perf/util/include/asm/swab.h create mode 100644 tools/perf/util/include/asm/system.h create mode 100644 tools/perf/util/include/asm/uaccess.h create mode 100644 tools/perf/util/include/dwarf-regs.h create mode 100644 tools/perf/util/include/linux/linkage.h create mode 100644 tools/perf/util/intel-bts.c create mode 100644 tools/perf/util/intel-bts.h create mode 100644 tools/perf/util/intel-pt-decoder/Build create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-decoder.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-decoder.h create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-insn-decoder.h create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-log.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-log.h create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.c create mode 100644 tools/perf/util/intel-pt-decoder/intel-pt-pkt-decoder.h create mode 100644 tools/perf/util/intel-pt.c create mode 100644 tools/perf/util/intel-pt.h create mode 100644 tools/perf/util/intel-tpebs.c create mode 100644 tools/perf/util/intel-tpebs.h create mode 100644 tools/perf/util/intlist.c create mode 100644 tools/perf/util/intlist.h create mode 100644 tools/perf/util/iostat.c create mode 100644 tools/perf/util/iostat.h create mode 100644 tools/perf/util/jit.h create mode 100644 tools/perf/util/jitdump.c create mode 100644 tools/perf/util/jitdump.h create mode 100644 tools/perf/util/kvm-stat-arch/Build create mode 100644 tools/perf/util/kvm-stat-arch/arm64_exception_types.h create mode 100644 tools/perf/util/kvm-stat-arch/book3s_hcalls.h create mode 100644 tools/perf/util/kvm-stat-arch/book3s_hv_exits.h create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-arm64.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-loongarch.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-powerpc.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-riscv.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-s390.c create mode 100644 tools/perf/util/kvm-stat-arch/kvm-stat-x86.c create mode 100644 tools/perf/util/kvm-stat-arch/riscv_trap_types.h create mode 100644 tools/perf/util/kvm-stat.c create mode 100644 tools/perf/util/kvm-stat.h create mode 100644 tools/perf/util/kwork.h create mode 100644 tools/perf/util/levenshtein.c create mode 100644 tools/perf/util/levenshtein.h create mode 100644 tools/perf/util/libbfd.c create mode 100644 tools/perf/util/libbfd.h create mode 100644 tools/perf/util/libdw.c create mode 100644 tools/perf/util/libdw.h create mode 100644 tools/perf/util/libunwind-arch/Build create mode 100644 tools/perf/util/libunwind-arch/libunwind-arch.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-arch.h create mode 100644 tools/perf/util/libunwind-arch/libunwind-arm.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-arm64.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-i386.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-loongarch.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-mips.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-ppc32.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-ppc64.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-riscv.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-s390.c create mode 100644 tools/perf/util/libunwind-arch/libunwind-x86_64.c create mode 100644 tools/perf/util/llvm-c-helpers.cpp create mode 100644 tools/perf/util/llvm-c-helpers.h create mode 100644 tools/perf/util/llvm.c create mode 100644 tools/perf/util/llvm.h create mode 100644 tools/perf/util/lock-contention.c create mode 100644 tools/perf/util/lock-contention.h create mode 100644 tools/perf/util/lzma.c create mode 100644 tools/perf/util/machine.c create mode 100644 tools/perf/util/machine.h create mode 100644 tools/perf/util/map.c create mode 100644 tools/perf/util/map.h create mode 100644 tools/perf/util/map_symbol.c create mode 100644 tools/perf/util/map_symbol.h create mode 100644 tools/perf/util/maps.c create mode 100644 tools/perf/util/maps.h create mode 100644 tools/perf/util/mem-events.c create mode 100644 tools/perf/util/mem-events.h create mode 100644 tools/perf/util/mem-info.c create mode 100644 tools/perf/util/mem-info.h create mode 100644 tools/perf/util/mem2node.c create mode 100644 tools/perf/util/mem2node.h create mode 100644 tools/perf/util/memswap.c create mode 100644 tools/perf/util/memswap.h create mode 100644 tools/perf/util/metricgroup.c create mode 100644 tools/perf/util/metricgroup.h create mode 100644 tools/perf/util/mmap.c create mode 100644 tools/perf/util/mmap.h create mode 100644 tools/perf/util/mutex.c create mode 100644 tools/perf/util/mutex.h create mode 100644 tools/perf/util/namespaces.c create mode 100644 tools/perf/util/namespaces.h create mode 100644 tools/perf/util/off_cpu.h create mode 100644 tools/perf/util/ordered-events.c create mode 100644 tools/perf/util/ordered-events.h create mode 100644 tools/perf/util/parse-branch-options.c create mode 100644 tools/perf/util/parse-branch-options.h create mode 100644 tools/perf/util/parse-events.c create mode 100644 tools/perf/util/parse-events.h create mode 100644 tools/perf/util/parse-events.l create mode 100644 tools/perf/util/parse-events.y create mode 100644 tools/perf/util/parse-regs-options.c create mode 100644 tools/perf/util/parse-regs-options.h create mode 100644 tools/perf/util/parse-sublevel-options.c create mode 100644 tools/perf/util/parse-sublevel-options.h create mode 100644 tools/perf/util/path.c create mode 100644 tools/perf/util/path.h create mode 100644 tools/perf/util/perf-hooks-list.h create mode 100644 tools/perf/util/perf-hooks.c create mode 100644 tools/perf/util/perf-hooks.h create mode 100644 tools/perf/util/perf-regs-arch/Build create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_aarch64.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_arm.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_csky.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_loongarch.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_mips.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_powerpc.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_riscv.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_s390.c create mode 100644 tools/perf/util/perf-regs-arch/perf_regs_x86.c create mode 100644 tools/perf/util/perf_api_probe.c create mode 100644 tools/perf/util/perf_api_probe.h create mode 100644 tools/perf/util/perf_event_attr_fprintf.c create mode 100644 tools/perf/util/perf_regs.c create mode 100644 tools/perf/util/perf_regs.h create mode 100644 tools/perf/util/pfm.c create mode 100644 tools/perf/util/pfm.h create mode 100644 tools/perf/util/pmu.c create mode 100644 tools/perf/util/pmu.h create mode 100644 tools/perf/util/pmu.l create mode 100644 tools/perf/util/pmu.y create mode 100644 tools/perf/util/pmus.c create mode 100644 tools/perf/util/pmus.h create mode 100644 tools/perf/util/powerpc-vpadtl.c create mode 100644 tools/perf/util/powerpc-vpadtl.h create mode 100644 tools/perf/util/print-events.c create mode 100644 tools/perf/util/print-events.h create mode 100644 tools/perf/util/print_binary.c create mode 100644 tools/perf/util/print_binary.h create mode 100644 tools/perf/util/print_insn.c create mode 100644 tools/perf/util/print_insn.h create mode 100644 tools/perf/util/probe-event.c create mode 100644 tools/perf/util/probe-event.h create mode 100644 tools/perf/util/probe-file.c create mode 100644 tools/perf/util/probe-file.h create mode 100644 tools/perf/util/probe-finder.c create mode 100644 tools/perf/util/probe-finder.h create mode 100644 tools/perf/util/pstack.c create mode 100644 tools/perf/util/pstack.h create mode 100644 tools/perf/util/python.c create mode 100644 tools/perf/util/rblist.c create mode 100644 tools/perf/util/rblist.h create mode 100644 tools/perf/util/record.c create mode 100644 tools/perf/util/record.h create mode 100644 tools/perf/util/rlimit.c create mode 100644 tools/perf/util/rlimit.h create mode 100644 tools/perf/util/rwsem.c create mode 100644 tools/perf/util/rwsem.h create mode 100644 tools/perf/util/s390-cpumcf-kernel.h create mode 100644 tools/perf/util/s390-cpumsf-kernel.h create mode 100644 tools/perf/util/s390-cpumsf.c create mode 100644 tools/perf/util/s390-cpumsf.h create mode 100644 tools/perf/util/s390-sample-raw.c create mode 100644 tools/perf/util/sample-raw.c create mode 100644 tools/perf/util/sample-raw.h create mode 100644 tools/perf/util/sample.c create mode 100644 tools/perf/util/sample.h create mode 100644 tools/perf/util/scripting-engines/Build create mode 100644 tools/perf/util/scripting-engines/trace-event-perl.c create mode 100644 tools/perf/util/scripting-engines/trace-event-python.c create mode 100644 tools/perf/util/session.c create mode 100644 tools/perf/util/session.h create mode 100644 tools/perf/util/setns.c create mode 100644 tools/perf/util/setup.py create mode 100644 tools/perf/util/sharded_mutex.c create mode 100644 tools/perf/util/sharded_mutex.h create mode 100644 tools/perf/util/sideband_evlist.c create mode 100644 tools/perf/util/smt.c create mode 100644 tools/perf/util/smt.h create mode 100644 tools/perf/util/sort.c create mode 100644 tools/perf/util/sort.h create mode 100644 tools/perf/util/spark.c create mode 100644 tools/perf/util/spark.h create mode 100644 tools/perf/util/srccode.c create mode 100644 tools/perf/util/srccode.h create mode 100644 tools/perf/util/srcline.c create mode 100644 tools/perf/util/srcline.h create mode 100644 tools/perf/util/stat-display.c create mode 100644 tools/perf/util/stat-shadow.c create mode 100644 tools/perf/util/stat.c create mode 100644 tools/perf/util/stat.h create mode 100644 tools/perf/util/strbuf.c create mode 100644 tools/perf/util/strbuf.h create mode 100644 tools/perf/util/stream.c create mode 100644 tools/perf/util/stream.h create mode 100644 tools/perf/util/strfilter.c create mode 100644 tools/perf/util/strfilter.h create mode 100644 tools/perf/util/string.c create mode 100644 tools/perf/util/string2.h create mode 100644 tools/perf/util/strlist.c create mode 100644 tools/perf/util/strlist.h create mode 100644 tools/perf/util/svghelper.c create mode 100644 tools/perf/util/svghelper.h create mode 100644 tools/perf/util/symbol-elf.c create mode 100644 tools/perf/util/symbol-minimal.c create mode 100644 tools/perf/util/symbol.c create mode 100644 tools/perf/util/symbol.h create mode 100644 tools/perf/util/symbol_conf.h create mode 100644 tools/perf/util/symbol_fprintf.c create mode 100644 tools/perf/util/symsrc.h create mode 100644 tools/perf/util/synthetic-events.c create mode 100644 tools/perf/util/synthetic-events.h create mode 100644 tools/perf/util/target.c create mode 100644 tools/perf/util/target.h create mode 100644 tools/perf/util/term.c create mode 100644 tools/perf/util/term.h create mode 100644 tools/perf/util/thread-stack.c create mode 100644 tools/perf/util/thread-stack.h create mode 100644 tools/perf/util/thread.c create mode 100644 tools/perf/util/thread.h create mode 100644 tools/perf/util/thread_map.c create mode 100644 tools/perf/util/thread_map.h create mode 100644 tools/perf/util/threads.c create mode 100644 tools/perf/util/threads.h create mode 100644 tools/perf/util/time-utils.c create mode 100644 tools/perf/util/time-utils.h create mode 100644 tools/perf/util/tool.c create mode 100644 tools/perf/util/tool.h create mode 100644 tools/perf/util/tool_pmu.c create mode 100644 tools/perf/util/tool_pmu.h create mode 100644 tools/perf/util/top.c create mode 100644 tools/perf/util/top.h create mode 100644 tools/perf/util/topdown.c create mode 100644 tools/perf/util/topdown.h create mode 100644 tools/perf/util/tp_pmu.c create mode 100644 tools/perf/util/tp_pmu.h create mode 100644 tools/perf/util/trace-event-info.c create mode 100644 tools/perf/util/trace-event-parse.c create mode 100644 tools/perf/util/trace-event-read.c create mode 100644 tools/perf/util/trace-event-scripting.c create mode 100644 tools/perf/util/trace-event.c create mode 100644 tools/perf/util/trace-event.h create mode 100644 tools/perf/util/trace.h create mode 100644 tools/perf/util/trace_augment.h create mode 100644 tools/perf/util/tracepoint.c create mode 100644 tools/perf/util/tracepoint.h create mode 100644 tools/perf/util/trigger.h create mode 100644 tools/perf/util/tsc.c create mode 100644 tools/perf/util/tsc.h create mode 100644 tools/perf/util/units.c create mode 100644 tools/perf/util/units.h create mode 100644 tools/perf/util/unwind-libdw.c create mode 100644 tools/perf/util/unwind-libdw.h create mode 100644 tools/perf/util/unwind-libunwind.c create mode 100644 tools/perf/util/unwind.c create mode 100644 tools/perf/util/unwind.h create mode 100644 tools/perf/util/usage.c create mode 100644 tools/perf/util/util.c create mode 100644 tools/perf/util/util.h create mode 100644 tools/perf/util/values.c create mode 100644 tools/perf/util/values.h create mode 100644 tools/perf/util/vdso.c create mode 100644 tools/perf/util/vdso.h create mode 100644 tools/perf/util/zlib.c create mode 100644 tools/perf/util/zstd.c (limited to 'tools/perf/util') diff --git a/tools/perf/util/Build b/tools/perf/util/Build new file mode 100644 index 000000000..5e2265018 --- /dev/null +++ b/tools/perf/util/Build @@ -0,0 +1,458 @@ +include $(srctree)/tools/scripts/Makefile.include +include $(srctree)/tools/scripts/utilities.mak + +perf-util-y += annotate-arch/ +perf-util-y += arm64-frame-pointer-unwind-support.o +perf-util-y += addr2line.o +perf-util-y += addr_location.o +perf-util-y += annotate.o +perf-util-y += aslr.o +perf-util-y += blake2s.o +perf-util-y += block-info.o +perf-util-y += block-range.o +perf-util-y += build-id.o +perf-util-y += cacheline.o +perf-util-$(CONFIG_LIBCAPSTONE) += capstone.o +perf-util-y += config.o +perf-util-y += copyfile.o +perf-util-y += ctype.o +perf-util-y += db-export.o +perf-util-y += disasm.o +perf-util-y += env.o +perf-util-y += event.o +perf-util-y += evlist.o +perf-util-y += sideband_evlist.o +perf-util-y += evsel.o +perf-util-y += evsel_fprintf.o +perf-util-y += perf_event_attr_fprintf.o +perf-util-y += evswitch.o +perf-util-y += find_bit.o +perf-util-y += levenshtein.o +perf-util-$(CONFIG_LIBBFD) += libbfd.o +perf-util-y += llvm.o +perf-util-y += mmap.o +perf-util-y += memswap.o +perf-util-y += parse-events.o +perf-util-y += print-events.o +perf-util-y += tracepoint.o +perf-util-y += perf_regs.o +perf-util-y += perf-regs-arch/ +perf-util-y += path.o +perf-util-y += print_binary.o +perf-util-y += print_insn.o +perf-util-y += rlimit.o +perf-util-y += argv_split.o +perf-util-y += rbtree.o +perf-util-y += libstring.o +perf-util-y += bitmap.o +perf-util-y += hweight.o +perf-util-y += smt.o +perf-util-y += strbuf.o +perf-util-y += string.o +perf-util-y += strlist.o +perf-util-y += strfilter.o +perf-util-y += top.o +perf-util-y += usage.o +perf-util-y += dso.o +perf-util-y += dsos.o +perf-util-y += symbol.o +perf-util-y += symbol_fprintf.o +perf-util-y += map_symbol.o +perf-util-y += color.o +perf-util-y += color_config.o +perf-util-y += metricgroup.o +perf-util-y += header.o +perf-util-y += callchain.o +perf-util-y += values.o +perf-util-y += debug.o +perf-util-y += fncache.o +perf-util-y += machine.o +perf-util-y += map.o +perf-util-y += maps.o +perf-util-y += pstack.o +perf-util-y += session.o +perf-util-y += tool.o +perf-util-y += sample.o +perf-util-y += sample-raw.o +perf-util-y += s390-sample-raw.o +perf-util-y += amd-sample-raw.o + +perf-util-y += ordered-events.o +perf-util-y += namespaces.o +perf-util-y += comm.o +perf-util-y += thread.o +perf-util-y += threads.o +perf-util-y += thread_map.o +perf-util-y += parse-events-flex.o +perf-util-y += parse-events-bison.o +perf-util-y += pmu.o +perf-util-y += pmus.o +perf-util-y += pmu-flex.o +perf-util-y += pmu-bison.o +perf-util-y += drm_pmu.o +perf-util-y += hwmon_pmu.o +perf-util-y += tool_pmu.o +perf-util-y += tp_pmu.o +perf-util-y += svghelper.o +perf-util-y += trace-event-info.o +perf-util-y += trace-event-scripting.o +perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event.o +perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event-parse.o +perf-util-$(CONFIG_LIBTRACEEVENT) += trace-event-read.o +perf-util-y += sort.o +perf-util-y += hist.o +perf-util-y += util.o +perf-util-y += cpumap.o +perf-util-y += affinity.o +perf-util-y += cputopo.o +perf-util-y += cgroup.o +perf-util-y += target.o +perf-util-y += rblist.o +perf-util-y += intlist.o +perf-util-y += vdso.o +perf-util-y += counts.o +perf-util-y += stat.o +perf-util-y += stat-shadow.o +perf-util-y += stat-display.o +perf-util-y += perf_api_probe.o +perf-util-y += record.o +perf-util-y += srcline.o +perf-util-y += srccode.o +perf-util-y += synthetic-events.o +perf-util-y += data.o +perf-util-y += tsc.o +perf-util-y += cloexec.o +perf-util-y += call-path.o +perf-util-y += rwsem.o +perf-util-y += thread-stack.o +perf-util-y += spark.o +perf-util-y += topdown.o +perf-util-y += iostat.o +perf-util-y += stream.o +perf-util-$(CONFIG_LIBTRACEEVENT) += kvm-stat.o +perf-util-y += kvm-stat-arch/ +perf-util-y += lock-contention.o +perf-util-y += auxtrace.o +perf-util-y += intel-pt-decoder/ +perf-util-y += intel-pt.o +perf-util-y += intel-bts.o +perf-util-y += arm-spe.o +perf-util-y += arm-spe-decoder/ +perf-util-y += hisi-ptt.o +perf-util-y += hisi-ptt-decoder/ +perf-util-y += s390-cpumsf.o +perf-util-y += powerpc-vpadtl.o + +ifdef CONFIG_LIBOPENCSD +perf-util-y += cs-etm.o +perf-util-y += cs-etm-decoder/ +endif +perf-util-y += cs-etm-base.o + +perf-util-y += parse-branch-options.o +perf-util-y += parse-regs-options.o +perf-util-y += parse-sublevel-options.o +perf-util-y += term.o +perf-util-y += help-unknown-cmd.o +perf-util-y += dlfilter.o +perf-util-y += mem-events.o +perf-util-y += mem-info.o +perf-util-y += vsprintf.o +perf-util-y += units.o +perf-util-y += time-utils.o +perf-util-y += expr-flex.o +perf-util-y += expr-bison.o +perf-util-y += expr.o +perf-util-y += branch.o +perf-util-y += mem2node.o +perf-util-y += clockid.o +perf-util-y += list_sort.o +perf-util-y += mutex.o +perf-util-y += sharded_mutex.o +perf-util-y += intel-tpebs.o + +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_counter.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_counter_cgroup.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_ftrace.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_off_cpu.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-filter.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-filter-flex.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-filter-bison.o +perf-util-$(CONFIG_PERF_BPF_SKEL) += btf.o + +ifeq ($(CONFIG_TRACE),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf-trace-summary.o + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_trace_augment.o +endif + +ifeq ($(CONFIG_LIBTRACEEVENT),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_lock_contention.o +endif + +ifeq ($(CONFIG_LIBTRACEEVENT),y) + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork.o + perf-util-$(CONFIG_PERF_BPF_SKEL) += bpf_kwork_top.o +endif + +perf-util-$(CONFIG_LIBELF) += symbol-elf.o +perf-util-$(CONFIG_LIBELF) += probe-file.o +perf-util-$(CONFIG_LIBELF) += probe-event.o + +ifdef CONFIG_LIBBPF_DYNAMIC + hashmap := 1 +endif +ifndef CONFIG_LIBBPF + hashmap := 1 +endif + +ifdef hashmap +perf-util-y += hashmap.o +endif + +ifndef CONFIG_LIBELF +perf-util-y += symbol-minimal.o +endif + +ifndef CONFIG_SETNS +perf-util-y += setns.o +endif + +perf-util-y += unwind.o +perf-util-$(CONFIG_LIBDW) += probe-finder.o +perf-util-$(CONFIG_LIBDW) += dwarf-aux.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-arch/ +perf-util-$(CONFIG_LIBDW) += debuginfo.o +perf-util-$(CONFIG_LIBDW) += annotate-data.o +perf-util-$(CONFIG_LIBDW) += libdw.o +perf-util-$(CONFIG_LIBDW) += unwind-libdw.o + +perf-util-$(CONFIG_LIBUNWIND) += unwind-libunwind.o +perf-util-$(CONFIG_LIBUNWIND) += libunwind-arch/ + +ifeq ($(CONFIG_LIBTRACEEVENT),y) + perf-util-$(CONFIG_BABELTRACE2_CTF_WRITER) += data-convert-bt.o +endif + +perf-util-y += data-convert-json.o + +perf-util-y += scripting-engines/ + +perf-util-$(CONFIG_ZLIB) += zlib.o +perf-util-$(CONFIG_LZMA) += lzma.o +perf-util-$(CONFIG_ZSTD) += zstd.o + +perf-util-y += cap.o + +perf-util-$(CONFIG_CXX_DEMANGLE) += demangle-cxx.o +perf-util-y += demangle-ocaml.o +perf-util-y += demangle-java.o +perf-util-y += demangle-rust-v0.o +perf-util-$(CONFIG_LIBLLVM) += llvm-c-helpers.o + +CFLAGS_demangle-rust-v0.o += -Wno-shadow -Wno-declaration-after-statement \ + -Wno-switch-default -Wno-switch-enum -Wno-missing-field-initializers + +ifdef CONFIG_JITDUMP +perf-util-$(CONFIG_LIBELF) += jitdump.o +perf-util-$(CONFIG_LIBELF) += genelf.o +perf-util-$(CONFIG_LIBDW) += genelf_debug.o +endif + +perf-util-y += perf-hooks.o + +perf-util-$(CONFIG_LIBBPF) += bpf-event.o +perf-util-$(CONFIG_LIBBPF) += bpf-utils.o + +perf-util-$(CONFIG_LIBPFM4) += pfm.o + +CFLAGS_config.o += -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" + +# avoid compiler warnings in 32-bit mode +CFLAGS_genelf_debug.o += -Wno-packed + +$(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-flex.h: util/parse-events.l $(OUTPUT)util/parse-events-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/parse-events-flex.c \ + --header-file=$(OUTPUT)util/parse-events-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/parse-events-bison.c $(OUTPUT)util/parse-events-bison.h: util/parse-events.y + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) $(BISON_FALLBACK_FLAGS) \ + -o $(OUTPUT)util/parse-events-bison.c -p parse_events_ + +$(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-flex.h: util/expr.l $(OUTPUT)util/expr-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/expr-flex.c \ + --header-file=$(OUTPUT)util/expr-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/expr-bison.c $(OUTPUT)util/expr-bison.h: util/expr.y + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ + -o $(OUTPUT)util/expr-bison.c -p expr_ + +$(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-flex.h: util/pmu.l $(OUTPUT)util/pmu-bison.c + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/pmu-flex.c \ + --header-file=$(OUTPUT)util/pmu-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/pmu-bison.c $(OUTPUT)util/pmu-bison.h: util/pmu.y + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ + -o $(OUTPUT)util/pmu-bison.c -p perf_pmu_ + +$(OUTPUT)util/bpf-filter-flex.c $(OUTPUT)util/bpf-filter-flex.h: util/bpf-filter.l $(OUTPUT)util/bpf-filter-bison.c util/bpf-filter.h util/bpf_skel/sample-filter.h + $(call rule_mkdir) + $(Q)$(call echo-cmd,flex)$(FLEX) -o $(OUTPUT)util/bpf-filter-flex.c \ + --header-file=$(OUTPUT)util/bpf-filter-flex.h $(PARSER_DEBUG_FLEX) $< + +$(OUTPUT)util/bpf-filter-bison.c $(OUTPUT)util/bpf-filter-bison.h: util/bpf-filter.y util/bpf-filter.h util/bpf_skel/sample-filter.h + $(call rule_mkdir) + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ + -o $(OUTPUT)util/bpf-filter-bison.c -p perf_bpf_filter_ + +FLEX_VERSION := $(shell $(FLEX) --version | cut -d' ' -f2) + +FLEX_GE_260 := $(call version-ge3,$(FLEX_VERSION),2.6.0) +ifeq ($(FLEX_GE_260),1) + flex_flags := -Wno-redundant-decls -Wno-switch-default -Wno-unused-function -Wno-misleading-indentation + + # Some newer clang and gcc version complain about this + # util/parse-events-bison.c:1317:9: error: variable 'parse_events_nerrs' set but not used [-Werror,-Wunused-but-set-variable] + # int yynerrs = 0; + + flex_flags += -Wno-unused-but-set-variable + + FLEX_LT_262 := $(call version-lt3,$(FLEX_VERSION),2.6.2) + ifeq ($(FLEX_LT_262),1) + flex_flags += -Wno-sign-compare + endif +else + flex_flags := -w +endif + +# Some newer clang and gcc version complain about this +# util/parse-events-bison.c:1317:9: error: variable 'parse_events_nerrs' set but not used [-Werror,-Wunused-but-set-variable] +# int yynerrs = 0; + +bison_flags := -DYYENABLE_NLS=0 -Wno-unused-but-set-variable + +# Old clangs don't grok -Wno-unused-but-set-variable, remove it +ifeq ($(CC_NO_CLANG), 0) + CLANG_VERSION := $(shell $(CLANG) --version | head -1 | sed 's/.*clang version \([[:digit:]]\+.[[:digit:]]\+.[[:digit:]]\+\).*/\1/g') + ifeq ($(call version-lt3,$(CLANG_VERSION),13.0.0),1) + bison_flags := $(subst -Wno-unused-but-set-variable,,$(bison_flags)) + flex_flags := $(subst -Wno-unused-but-set-variable,,$(flex_flags)) + endif +endif + +BISON_GE_382 := $(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \>\= 382) +ifeq ($(BISON_GE_382),1) + bison_flags += -Wno-switch-enum +else + bison_flags += -w +endif + +BISON_LT_381 := $(shell expr $(shell $(BISON) --version | grep bison | sed -e 's/.\+ \([0-9]\+\).\([0-9]\+\).\([0-9]\+\)/\1\2\3/g') \< 381) +ifeq ($(BISON_LT_381),1) + bison_flags += -DYYNOMEM=YYABORT +endif + +CFLAGS_parse-events-flex.o += $(flex_flags) -Wno-unused-label +CFLAGS_pmu-flex.o += $(flex_flags) +CFLAGS_expr-flex.o += $(flex_flags) +CFLAGS_bpf-filter-flex.o += $(flex_flags) + +CFLAGS_parse-events-bison.o += $(bison_flags) +CFLAGS_pmu-bison.o += -DYYLTYPE_IS_TRIVIAL=0 $(bison_flags) +CFLAGS_expr-bison.o += -DYYLTYPE_IS_TRIVIAL=0 $(bison_flags) +CFLAGS_bpf-filter-bison.o += -DYYLTYPE_IS_TRIVIAL=0 $(bison_flags) + +$(OUTPUT)util/parse-events.o: $(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-bison.c +$(OUTPUT)util/pmu.o: $(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-bison.c +$(OUTPUT)util/expr.o: $(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-bison.c +$(OUTPUT)util/bpf-filter.o: $(OUTPUT)util/bpf-filter-flex.c $(OUTPUT)util/bpf-filter-bison.c + +CFLAGS_bitmap.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_find_bit.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_rbtree.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_libstring.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_hweight.o += -Wno-unused-parameter -DETC_PERFCONFIG="BUILD_STR($(ETC_PERFCONFIG_SQ))" +CFLAGS_header.o += -include $(OUTPUT)PERF-VERSION-FILE +CFLAGS_arm-spe.o += -I$(srctree)/tools/arch/arm64/include/ -I$(OUTPUT)arch/arm64/include/generated/ + +$(OUTPUT)util/argv_split.o: ../lib/argv_split.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/bitmap.o: ../lib/bitmap.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/ctype.o: ../lib/ctype.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/find_bit.o: ../lib/find_bit.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/rbtree.o: ../lib/rbtree.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/libstring.o: ../lib/string.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/hweight.o: ../lib/hweight.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/vsprintf.o: ../lib/vsprintf.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +$(OUTPUT)util/list_sort.o: ../lib/list_sort.c FORCE + $(call rule_mkdir) + $(call if_changed_dep,cc_o_c) + +PY_TESTS := setup.py +ifdef MYPY + MYPY_TEST_LOGS := $(PY_TESTS:%=%.mypy_log) +else + MYPY_TEST_LOGS := +endif + +$(OUTPUT)%.mypy_log: % + $(call rule_mkdir) + $(Q)$(call echo-cmd,test)mypy "$<" > $@ || (cat $@ && rm $@ && false) + +perf-util-y += $(MYPY_TEST_LOGS) + +ifdef PYLINT + PYLINT_TEST_LOGS := $(PY_TESTS:%=%.pylint_log) +else + PYLINT_TEST_LOGS := +endif + +$(OUTPUT)%.pylint_log: % + $(call rule_mkdir) + $(Q)$(call echo-cmd,test)pylint "$<" > $@ || (cat $@ && rm $@ && false) + +perf-util-y += $(PYLINT_TEST_LOGS) + +ifeq ($(CONFIG_PERF_BPF_SKEL),y) +include $(srctree)/tools/perf/bpf_skel.mak + +$(OUTPUT)util/bpf_ftrace.o: $(SKEL_OUT)/func_latency.skel.h +$(OUTPUT)util/bpf-filter.o: $(SKEL_OUT)/sample_filter.skel.h +$(OUTPUT)util/bpf_kwork_top.o: $(SKEL_OUT)/kwork_top.skel.h +$(OUTPUT)util/bpf_off_cpu.o: $(SKEL_OUT)/off_cpu.skel.h +$(OUTPUT)util/bpf-trace-summary.o: $(SKEL_OUT)/syscall_summary.skel.h +$(OUTPUT)util/bpf_counter_cgroup.o: $(SKEL_OUT)/bperf_cgroup.skel.h +$(OUTPUT)util/bpf_trace_augment.o: $(SKEL_OUT)/augmented_raw_syscalls.skel.h +$(OUTPUT)util/bpf_counter.o: $(SKEL_OUT)/bpf_prog_profiler.skel.h $(SKEL_OUT)/bperf_leader.skel.h $(SKEL_OUT)/bperf_follower.skel.h +$(OUTPUT)util/bpf_lock_contention.o: $(SKEL_OUT)/lock_contention.skel.h +$(OUTPUT)util/bpf_kwork.o: $(SKEL_OUT)/kwork_trace.skel.h +endif diff --git a/tools/perf/util/PERF-VERSION-GEN b/tools/perf/util/PERF-VERSION-GEN new file mode 100755 index 000000000..d7dc7c285 --- /dev/null +++ b/tools/perf/util/PERF-VERSION-GEN @@ -0,0 +1,52 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-2.0 + +if [ $# -eq 1 ] ; then + OUTPUT=$1 +fi + +GVF=${OUTPUT}PERF-VERSION-FILE + +LF=' +' + +# +# Use version from kernel Makefile unless not in a git repository and +# PERF-VERSION-FILE exists +# +CID= +TAG= +if test -d ../../.git -o -f ../../.git +then + TAG=$(MAKEFLAGS= make -sC ../.. kernelversion) + CID=$(git log -1 --abbrev=12 --pretty=format:"%h" --no-show-signature 2>/dev/null) && CID="-g$CID" +elif test -f ../../PERF-VERSION-FILE +then + TAG=$(cut -d' ' -f3 ../../PERF-VERSION-FILE | sed -e 's/\"//g') +fi +if test -z "$TAG" +then + TAG=$(MAKEFLAGS= make -sC ../.. kernelversion) +fi + +VN="$TAG$CID" +if test -n "$CID" +then + # format version string, strip trailing zero of sublevel: + VN=$(echo "$VN" | sed -e 's/-/./g;s/\([0-9]*[.][0-9]*\)[.]0/\1/') +fi + +VN=$(expr "$VN" : v*'\(.*\)') + +if test -r $GVF +then + VC=$(sed -e 's/^#define PERF_VERSION "\(.*\)"/\1/' <$GVF) +else + VC=unset +fi +test "$VN" = "$VC" || { + echo >&2 " PERF_VERSION = $VN" + echo "#define PERF_VERSION \"$VN\"" >$GVF +} + + diff --git a/tools/perf/util/addr2line.c b/tools/perf/util/addr2line.c new file mode 100644 index 000000000..4b0d349ed --- /dev/null +++ b/tools/perf/util/addr2line.c @@ -0,0 +1,436 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "addr2line.h" +#include "debug.h" +#include "dso.h" +#include "string2.h" +#include "srcline.h" +#include "symbol.h" +#include "symbol_conf.h" + +#include +#include +#include + +#include +#include +#include +#include + +#define MAX_INLINE_NEST 1024 + +static int filename_split(char *filename, unsigned int *line_nr) +{ + char *sep; + + sep = strchr(filename, '\n'); + if (sep) + *sep = '\0'; + + if (!strcmp(filename, "??:0")) + return 0; + + sep = strchr(filename, ':'); + if (sep) { + *sep++ = '\0'; + *line_nr = strtoul(sep, NULL, 0); + return 1; + } + pr_debug("addr2line missing ':' in filename split\n"); + return 0; +} + +static void addr2line_subprocess_cleanup(struct child_process *a2l) +{ + if (a2l->pid != -1) { + kill(a2l->pid, SIGKILL); + finish_command(a2l); /* ignore result, we don't care */ + a2l->pid = -1; + close(a2l->in); + close(a2l->out); + } + + free(a2l); +} + +static struct child_process *addr2line_subprocess_init(const char *addr2line_path, + const char *binary_path) +{ + const char *argv[] = { + addr2line_path ?: "addr2line", + "-e", binary_path, + "-a", "-i", "-f", NULL + }; + struct child_process *a2l = zalloc(sizeof(*a2l)); + int start_command_status = 0; + + if (a2l == NULL) { + pr_err("Failed to allocate memory for addr2line"); + return NULL; + } + + a2l->pid = -1; + a2l->in = -1; + a2l->out = -1; + a2l->no_stderr = 1; + + a2l->argv = argv; + start_command_status = start_command(a2l); + a2l->argv = NULL; /* it's not used after start_command; avoid dangling pointers */ + + if (start_command_status != 0) { + pr_warning("could not start addr2line (%s) for %s: start_command return code %d\n", + addr2line_path, binary_path, start_command_status); + addr2line_subprocess_cleanup(a2l); + return NULL; + } + + return a2l; +} + +enum cmd_a2l_style { + BROKEN, + GNU_BINUTILS, + LLVM, +}; + +static enum cmd_a2l_style cmd_addr2line_configure(struct child_process *a2l, const char *dso_name) +{ + static bool cached; + static enum cmd_a2l_style style; + + if (!cached) { + char buf[128]; + struct io io; + int ch; + int lines; + + if (write(a2l->in, ",\n", 2) != 2) + return BROKEN; + + io__init(&io, a2l->out, buf, sizeof(buf)); + ch = io__get_char(&io); + if (ch == ',') { + style = LLVM; + cached = true; + lines = 1; + pr_debug3("Detected LLVM addr2line style\n"); + } else if (ch == '0') { + style = GNU_BINUTILS; + cached = true; + lines = 3; + pr_debug3("Detected binutils addr2line style\n"); + } else { + if (!symbol_conf.addr2line_disable_warn) { + char *output = NULL; + size_t output_len; + + io__getline(&io, &output, &output_len); + pr_warning("%s %s: addr2line configuration failed\n", + __func__, dso_name); + pr_warning("\t%c%s", ch, output); + } + pr_debug("Unknown/broken addr2line style\n"); + return BROKEN; + } + while (lines) { + ch = io__get_char(&io); + if (ch <= 0) + break; + if (ch == '\n') + lines--; + } + /* Ignore SIGPIPE in the event addr2line exits. */ + signal(SIGPIPE, SIG_IGN); + } + return style; +} + +static int read_addr2line_record(struct io *io, + enum cmd_a2l_style style, + const char *dso_name, + u64 addr, + bool first, + char **function, + char **filename, + unsigned int *line_nr) +{ + /* + * Returns: + * -1 ==> error + * 0 ==> sentinel (or other ill-formed) record read + * 1 ==> a genuine record read + */ + char *line = NULL; + size_t line_len = 0; + unsigned int dummy_line_nr = 0; + int ret = -1; + + if (function != NULL) + zfree(function); + + if (filename != NULL) + zfree(filename); + + if (line_nr != NULL) + *line_nr = 0; + + /* + * Read the first line. Without an error this will be: + * - for the first line an address like 0x1234, + * - the binutils sentinel 0x0000000000000000, + * - the llvm-addr2line the sentinel ',' character, + * - the function name line for an inlined function. + */ + if (io__getline(io, &line, &line_len) < 0 || !line_len) + goto error; + + pr_debug3("%s %s: addr2line read address for sentinel: %s", __func__, dso_name, line); + if (style == LLVM && line_len == 2 && line[0] == ',') { + /* Found the llvm-addr2line sentinel character. */ + zfree(&line); + return 0; + } else if (style == GNU_BINUTILS && (!first || addr != 0)) { + int zero_count = 0, non_zero_count = 0; + /* + * Check for binutils sentinel ignoring it for the case the + * requested address is 0. + */ + + /* A given address should always start 0x. */ + if (line_len >= 2 || line[0] != '0' || line[1] != 'x') { + for (size_t i = 2; i < line_len; i++) { + if (line[i] == '0') + zero_count++; + else if (line[i] != '\n') + non_zero_count++; + } + if (!non_zero_count) { + int ch; + + if (first && !zero_count) { + /* Line was erroneous just '0x'. */ + goto error; + } + /* + * Line was 0x0..0, the sentinel for binutils. Remove + * the function and filename lines. + */ + zfree(&line); + do { + ch = io__get_char(io); + } while (ch > 0 && ch != '\n'); + do { + ch = io__get_char(io); + } while (ch > 0 && ch != '\n'); + return 0; + } + } + } + /* Read the second function name line (if inline data then this is the first line). */ + if (first && (io__getline(io, &line, &line_len) < 0 || !line_len)) + goto error; + + pr_debug3("%s %s: addr2line read line: %s", __func__, dso_name, line); + if (function != NULL) + *function = strdup(strim(line)); + + zfree(&line); + line_len = 0; + + /* Read the third filename and line number line. */ + if (io__getline(io, &line, &line_len) < 0 || !line_len) + goto error; + + pr_debug3("%s %s: addr2line filename:number : %s", __func__, dso_name, line); + if (filename_split(line, line_nr == NULL ? &dummy_line_nr : line_nr) == 0 && + style == GNU_BINUTILS) { + ret = 0; + goto error; + } + + if (filename != NULL) + *filename = strdup(line); + + zfree(&line); + line_len = 0; + + return 1; + +error: + free(line); + if (function != NULL) + zfree(function); + if (filename != NULL) + zfree(filename); + return ret; +} + +static int inline_list__append_record(struct dso *dso, + struct inline_node *node, + struct symbol *sym, + const char *function, + const char *filename, + unsigned int line_nr) +{ + struct symbol *inline_sym = new_inline_sym(dso, sym, function); + + return inline_list__append(inline_sym, srcline_from_fileline(filename, line_nr), node); +} + +int cmd__addr2line(const char *dso_name, u64 addr, + char **file, unsigned int *line_nr, + struct dso *dso, + bool unwind_inlines, + struct inline_node *node, + struct symbol *sym __maybe_unused) +{ + struct child_process *a2l = dso__a2l(dso); + char *record_function = NULL; + char *record_filename = NULL; + unsigned int record_line_nr = 0; + int record_status = -1; + int ret = 0; + size_t inline_count = 0; + int len; + char buf[128]; + ssize_t written; + struct io io = { .eof = false }; + enum cmd_a2l_style cmd_a2l_style; + + if (!a2l) { + if (!filename__has_section(dso_name, ".debug_line")) + goto out; + + dso__set_a2l(dso, + addr2line_subprocess_init(symbol_conf.addr2line_path, dso_name)); + a2l = dso__a2l(dso); + } + + if (a2l == NULL) { + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: addr2line_subprocess_init failed\n", __func__, dso_name); + goto out; + } + cmd_a2l_style = cmd_addr2line_configure(a2l, dso_name); + if (cmd_a2l_style == BROKEN) + goto out; + + /* + * Send our request and then *deliberately* send something that can't be + * interpreted as a valid address to ask addr2line about (namely, + * ","). This causes addr2line to first write out the answer to our + * request, in an unbounded/unknown number of records, and then to write + * out the lines "0x0...0", "??" and "??:0", for GNU binutils, or "," + * for llvm-addr2line, so that we can detect when it has finished giving + * us anything useful. + */ + len = snprintf(buf, sizeof(buf), "%016"PRIx64"\n,\n", addr); + written = len > 0 ? write(a2l->in, buf, len) : -1; + if (written != len) { + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: could not send request\n", __func__, dso_name); + goto out; + } + io__init(&io, a2l->out, buf, sizeof(buf)); + io.timeout_ms = symbol_conf.addr2line_timeout_ms; + switch (read_addr2line_record(&io, cmd_a2l_style, dso_name, addr, /*first=*/true, + &record_function, &record_filename, &record_line_nr)) { + case -1: + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: could not read first record\n", __func__, dso_name); + goto out; + case 0: + /* + * The first record was invalid, so return failure, but first + * read another record, since we sent a sentinel ',' for the + * sake of detected the last inlined function. Treat this as the + * first of a record as the ',' generates a new start with GNU + * binutils, also force a non-zero address as we're no longer + * reading that record. + */ + switch (read_addr2line_record(&io, cmd_a2l_style, dso_name, + /*addr=*/1, /*first=*/true, + NULL, NULL, NULL)) { + case -1: + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: could not read sentinel record\n", + __func__, dso_name); + break; + case 0: + /* The sentinel as expected. */ + break; + default: + if (!symbol_conf.addr2line_disable_warn) + pr_warning("%s %s: unexpected record instead of sentinel", + __func__, dso_name); + break; + } + goto out; + default: + /* First record as expected. */ + break; + } + + if (file) { + *file = strdup(record_filename); + ret = 1; + } + if (line_nr) + *line_nr = record_line_nr; + + if (unwind_inlines) { + if (node && inline_list__append_record(dso, node, sym, + record_function, + record_filename, + record_line_nr)) { + ret = 0; + goto out; + } + } + + /* + * We have to read the records even if we don't care about the inline + * info. This isn't the first record and force the address to non-zero + * as we're reading records beyond the first. + */ + while ((record_status = read_addr2line_record(&io, + cmd_a2l_style, + dso_name, + /*addr=*/1, + /*first=*/false, + &record_function, + &record_filename, + &record_line_nr)) == 1) { + if (unwind_inlines && node && inline_count++ < MAX_INLINE_NEST) { + if (inline_list__append_record(dso, node, sym, + record_function, + record_filename, + record_line_nr)) { + ret = 0; + goto out; + } + ret = 1; /* found at least one inline frame */ + } + } + +out: + free(record_function); + free(record_filename); + if (io.eof) { + dso__set_a2l(dso, NULL); + addr2line_subprocess_cleanup(a2l); + } + return ret; +} + +void dso__free_a2l(struct dso *dso) +{ + struct child_process *a2l = dso__a2l(dso); + + if (!a2l) + return; + + addr2line_subprocess_cleanup(a2l); + + dso__set_a2l(dso, NULL); +} diff --git a/tools/perf/util/addr2line.h b/tools/perf/util/addr2line.h new file mode 100644 index 000000000..75989a92f --- /dev/null +++ b/tools/perf/util/addr2line.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ADDR2LINE_H +#define __PERF_ADDR2LINE_H + +#include + +struct dso; +struct inline_node; +struct symbol; + +int cmd__addr2line(const char *dso_name, u64 addr, + char **file, unsigned int *line_nr, + struct dso *dso, + bool unwind_inlines, + struct inline_node *node, + struct symbol *sym); + +#endif /* __PERF_ADDR2LINE_H */ diff --git a/tools/perf/util/addr_location.c b/tools/perf/util/addr_location.c new file mode 100644 index 000000000..57e8217a0 --- /dev/null +++ b/tools/perf/util/addr_location.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "addr_location.h" +#include "map.h" +#include "maps.h" +#include "thread.h" + +void addr_location__init(struct addr_location *al) +{ + al->thread = NULL; + al->map = NULL; + al->sym = NULL; + al->srcline = NULL; + al->addr = 0; + al->level = 0; + al->filtered = 0; + al->cpumode = 0; + al->cpu = 0; + al->socket = 0; + al->parallelism = 1; +} + +/* + * The preprocess_sample method will return with reference counts for the + * in it, when done using (and perhaps getting ref counts if needing to + * keep a pointer to one of those entries) it must be paired with + * addr_location__exit(), so that the refcounts can be decremented. + */ +void addr_location__exit(struct addr_location *al) +{ + map__zput(al->map); + thread__zput(al->thread); +} + +void addr_location__copy(struct addr_location *dst, struct addr_location *src) +{ + thread__put(dst->thread); + map__put(dst->map); + *dst = *src; + dst->thread = thread__get(src->thread); + dst->map = map__get(src->map); +} diff --git a/tools/perf/util/addr_location.h b/tools/perf/util/addr_location.h new file mode 100644 index 000000000..fdc4d3f3a --- /dev/null +++ b/tools/perf/util/addr_location.h @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ADDR_LOCATION +#define __PERF_ADDR_LOCATION 1 + +#include + +struct thread; +struct maps; +struct map; +struct symbol; + +struct addr_location { + struct thread *thread; + struct map *map; + struct symbol *sym; + const char *srcline; + u64 addr; + char level; + u8 cpumode; + u16 filtered; + s32 cpu; + s32 socket; + /* Same as machine.parallelism but within [1, nr_cpus]. */ + int parallelism; + /* See he_stat.latency. */ + u64 latency; +}; + +void addr_location__init(struct addr_location *al); +void addr_location__exit(struct addr_location *al); + +void addr_location__copy(struct addr_location *dst, struct addr_location *src); + +#endif /* __PERF_ADDR_LOCATION */ diff --git a/tools/perf/util/affinity.c b/tools/perf/util/affinity.c new file mode 100644 index 000000000..6c64b5f69 --- /dev/null +++ b/tools/perf/util/affinity.c @@ -0,0 +1,103 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Manage affinity to optimize IPIs inside the kernel perf API. */ +#define _GNU_SOURCE 1 +#include +#include +#include +#include +#include +#include "perf.h" +#include "cpumap.h" +#include "affinity.h" + +static int get_cpu_set_size(void) +{ + int sz = cpu__max_cpu().cpu + 8 - 1; + /* + * sched_getaffinity doesn't like masks smaller than the kernel. + * Hopefully that's big enough. + */ + if (sz < 4096) + sz = 4096; + return sz / 8; +} + +int affinity__setup(struct affinity *a) +{ + int cpu_set_size = get_cpu_set_size(); + + a->orig_cpus = bitmap_zalloc(cpu_set_size * 8); + if (!a->orig_cpus) + return -1; + sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); + a->sched_cpus = bitmap_zalloc(cpu_set_size * 8); + if (!a->sched_cpus) { + zfree(&a->orig_cpus); + return -1; + } + bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size); + a->changed = false; + return 0; +} + +/* + * perf_event_open does an IPI internally to the target CPU. + * It is more efficient to change perf's affinity to the target + * CPU and then set up all events on that CPU, so we amortize + * CPU communication. + */ +void affinity__set(struct affinity *a, int cpu) +{ + int cpu_set_size = get_cpu_set_size(); + + /* + * Return: + * - if cpu is -1 + * - restrict out of bound access to sched_cpus + */ + if (cpu == -1 || ((cpu >= (cpu_set_size * 8)))) + return; + + a->changed = true; + __set_bit(cpu, a->sched_cpus); + /* + * We ignore errors because affinity is just an optimization. + * This could happen for example with isolated CPUs or cpusets. + * In this case the IPIs inside the kernel's perf API still work. + */ + sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus); + __clear_bit(cpu, a->sched_cpus); +} + +static void __affinity__cleanup(struct affinity *a) +{ + int cpu_set_size = get_cpu_set_size(); + + if (a->changed) + sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus); + zfree(&a->sched_cpus); + zfree(&a->orig_cpus); +} + +void affinity__cleanup(struct affinity *a) +{ + if (a != NULL) + __affinity__cleanup(a); +} + +void cpu_map__set_affinity(const struct perf_cpu_map *cpumap) +{ + int cpu_set_size = get_cpu_set_size(); + unsigned long *cpuset = bitmap_zalloc(cpu_set_size * 8); + struct perf_cpu cpu; + unsigned int idx; + + if (!cpuset) + return; + + perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpumap) + __set_bit(cpu.cpu, cpuset); + + sched_setaffinity(0, cpu_set_size, (cpu_set_t *)cpuset); + zfree(&cpuset); +} diff --git a/tools/perf/util/affinity.h b/tools/perf/util/affinity.h new file mode 100644 index 000000000..7341194b2 --- /dev/null +++ b/tools/perf/util/affinity.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef PERF_AFFINITY_H +#define PERF_AFFINITY_H 1 + +#include + +struct perf_cpu_map; +struct affinity { + unsigned long *orig_cpus; + unsigned long *sched_cpus; + bool changed; +}; + +void affinity__cleanup(struct affinity *a); +void affinity__set(struct affinity *a, int cpu); +int affinity__setup(struct affinity *a); +void cpu_map__set_affinity(const struct perf_cpu_map *cpumap); + +#endif // PERF_AFFINITY_H diff --git a/tools/perf/util/amd-sample-raw.c b/tools/perf/util/amd-sample-raw.c new file mode 100644 index 000000000..394c061fb --- /dev/null +++ b/tools/perf/util/amd-sample-raw.c @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * AMD specific. Provide textual annotation for IBS raw sample data. + */ + +#include +#include +#include +#include + +#include +#include "../../arch/x86/include/asm/amd/ibs.h" + +#include "debug.h" +#include "session.h" +#include "evlist.h" +#include "sample-raw.h" +#include "util/sample.h" + +static u32 cpu_family, cpu_model, ibs_fetch_type, ibs_op_type; +static bool zen4_ibs_extensions; +static bool ldlat_cap; +static bool dtlb_pgsize_cap; +static bool rmtsocket_cap; +static bool strmst_cap; + +/* + * Status fields of IBS_FETCH_CTL and IBS_FETCH_CTL_EXT are valid only if + * IBS_FETCH_CTL[PhyAddrValid] is set. + */ +static int fetch_ctl_depends_on_phy_addr_valid(void) +{ + static int depends = -1; /* -1: Don't know, 1: Yes, 0: No */ + + if (depends != -1) + return depends; + + depends = 0; + if (cpu_family > 0x1a || + (cpu_family == 0x1a && ( + (cpu_model >= 0x50 && cpu_model <= 0x5f) || + (cpu_model >= 0x80 && cpu_model <= 0xaf) || + (cpu_model >= 0xc0 && cpu_model <= 0xcf)))) { + depends = 1; + } + + return depends; +} + +static void pr_ibs_fetch_ctl(union ibs_fetch_ctl reg) +{ + const char * const ic_miss_strs[] = { + " IcMiss 0", + " IcMiss 1", + }; + const char * const l1tlb_pgsz_strs[] = { + " L1TlbPgSz 4KB", + " L1TlbPgSz 2MB", + " L1TlbPgSz 1GB", + " L1TlbPgSz RESERVED" + }; + const char * const l1tlb_pgsz_strs_erratum1347[] = { + " L1TlbPgSz 4KB", + " L1TlbPgSz 16KB", + " L1TlbPgSz 2MB", + " L1TlbPgSz 1GB" + }; + const char *ic_miss_str = NULL; + const char *l1tlb_pgsz_str = NULL; + char l3_miss_str[sizeof(" L3MissOnly _ FetchOcMiss _ FetchL3Miss _")] = ""; + char l3_miss_only_str[sizeof(" L3MissOnly _")] = ""; + + if (fetch_ctl_depends_on_phy_addr_valid() && !reg.phy_addr_valid) { + snprintf(l3_miss_only_str, sizeof(l3_miss_only_str), + " L3MissOnly %d", reg.l3_miss_only); + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d En %d Val %d Comp %d " + "PhyAddrValid 0 RandEn %d%s\n", reg.val, reg.fetch_maxcnt << 4, + reg.fetch_cnt << 4, reg.fetch_en, reg.fetch_val, reg.fetch_comp, + reg.rand_en, l3_miss_only_str); + return; + } + + if (cpu_family == 0x19 && cpu_model < 0x10) { + /* + * Erratum #1238 workaround is to ignore MSRC001_1030[IbsIcMiss] + * Erratum #1347 workaround is to use table provided in erratum + */ + if (reg.phy_addr_valid) + l1tlb_pgsz_str = l1tlb_pgsz_strs_erratum1347[reg.l1tlb_pgsz]; + } else { + if (reg.phy_addr_valid) + l1tlb_pgsz_str = l1tlb_pgsz_strs[reg.l1tlb_pgsz]; + ic_miss_str = ic_miss_strs[reg.ic_miss]; + } + + if (zen4_ibs_extensions) { + snprintf(l3_miss_str, sizeof(l3_miss_str), + " L3MissOnly %d FetchOcMiss %d FetchL3Miss %d", + reg.l3_miss_only, reg.fetch_oc_miss, reg.fetch_l3_miss); + } + + printf("ibs_fetch_ctl:\t%016llx MaxCnt %7d Cnt %7d Lat %5d En %d Val %d Comp %d%s " + "PhyAddrValid %d%s L1TlbMiss %d L2TlbMiss %d RandEn %d%s%s\n", + reg.val, reg.fetch_maxcnt << 4, reg.fetch_cnt << 4, reg.fetch_lat, + reg.fetch_en, reg.fetch_val, reg.fetch_comp, ic_miss_str ? : "", + reg.phy_addr_valid, l1tlb_pgsz_str ? : "", reg.l1tlb_miss, reg.l2tlb_miss, + reg.rand_en, reg.fetch_comp ? (reg.fetch_l2_miss ? " L2Miss 1" : " L2Miss 0") : "", + l3_miss_str); +} + +static void pr_ic_ibs_extd_ctl(union ibs_fetch_ctl fetch_ctl, union ic_ibs_extd_ctl reg) +{ + if (fetch_ctl_depends_on_phy_addr_valid() && !fetch_ctl.phy_addr_valid) + return; + + printf("ic_ibs_ext_ctl:\t%016llx IbsItlbRefillLat %3d\n", reg.val, reg.itlb_refill_lat); +} + +static void pr_ibs_op_ctl(union ibs_op_ctl reg) +{ + char l3_miss_only[sizeof(" L3MissOnly _")] = ""; + char ldlat[sizeof(" LdLatThrsh __ LdLatEn _")] = ""; + + if (zen4_ibs_extensions) + snprintf(l3_miss_only, sizeof(l3_miss_only), " L3MissOnly %d", reg.l3_miss_only); + + if (ldlat_cap) { + snprintf(ldlat, sizeof(ldlat), " LdLatThrsh %2d LdLatEn %d", + reg.ldlat_thrsh, reg.ldlat_en); + } + + printf("ibs_op_ctl:\t%016llx MaxCnt %9d%s En %d Val %d CntCtl %d=%s CurCnt %9d%s\n", + reg.val, ((reg.opmaxcnt_ext << 16) | reg.opmaxcnt) << 4, l3_miss_only, + reg.op_en, reg.op_val, reg.cnt_ctl, + reg.cnt_ctl ? "uOps" : "cycles", reg.opcurcnt, ldlat); +} + +static void pr_ibs_op_data(union ibs_op_data reg) +{ + printf("ibs_op_data:\t%016llx CompToRetCtr %5d TagToRetCtr %5d%s%s%s BrnRet %d " + " RipInvalid %d BrnFuse %d Microcode %d\n", + reg.val, reg.comp_to_ret_ctr, reg.tag_to_ret_ctr, + reg.op_brn_ret ? (reg.op_return ? " OpReturn 1" : " OpReturn 0") : "", + reg.op_brn_ret ? (reg.op_brn_taken ? " OpBrnTaken 1" : " OpBrnTaken 0") : "", + reg.op_brn_ret ? (reg.op_brn_misp ? " OpBrnMisp 1" : " OpBrnMisp 0") : "", + reg.op_brn_ret, reg.op_rip_invalid, reg.op_brn_fuse, reg.op_microcode); +} + +static void pr_ibs_op_data2_extended(union ibs_op_data2 reg) +{ + static const char * const data_src_str[] = { + "", + " DataSrc 1=Local L3 or other L1/L2 in CCX", + " DataSrc 2=Another CCX cache in the same NUMA node", + " DataSrc 3=DRAM", + " DataSrc 4=(reserved)", + " DataSrc 5=Another CCX cache in a different NUMA node", + " DataSrc 6=Long-latency DIMM", + " DataSrc 7=MMIO/Config/PCI/APIC", + " DataSrc 8=Extension Memory", + " DataSrc 9=(reserved)", + " DataSrc 10=(reserved)", + " DataSrc 11=(reserved)", + " DataSrc 12=Coherent Memory of a different processor type", + /* 13 to 31 are reserved. Avoid printing them. */ + }; + int data_src = (reg.data_src_hi << 3) | reg.data_src_lo; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, + (data_src == 1 || data_src == 2 || data_src == 5) ? + (reg.cache_hit_st ? "CacheHitSt 1=O-State " : "CacheHitSt 0=M-state ") : "", + reg.rmt_node, + data_src < (int)ARRAY_SIZE(data_src_str) ? data_src_str[data_src] : ""); +} + +static void pr_ibs_op_data2_default(union ibs_op_data2 reg) +{ + static const char * const data_src_str[] = { + "", + " DataSrc 1=(reserved)", + " DataSrc 2=Local node cache", + " DataSrc 3=DRAM", + " DataSrc 4=Remote node cache", + " DataSrc 5=(reserved)", + " DataSrc 6=(reserved)", + " DataSrc 7=Other" + }; + char rmtsocket[sizeof("RmtSocket _ ")] = ""; + char strmst[sizeof("StrmSt _ ")] = ""; + + if (rmtsocket_cap) + snprintf(rmtsocket, sizeof(rmtsocket), "RmtSocket %d ", reg.rmt_socket); + if (strmst_cap) + snprintf(strmst, sizeof(strmst), "StrmSt %d ", reg.strm_st); + + printf("ibs_op_data2:\t%016llx %s%s%sRmtNode %d%s\n", reg.val, + rmtsocket, strmst, + reg.data_src_lo == 2 ? (reg.cache_hit_st ? "CacheHitSt 1=O-State " + : "CacheHitSt 0=M-state ") : "", + reg.rmt_node, data_src_str[reg.data_src_lo]); +} + +static void pr_ibs_op_data2(union ibs_op_data2 reg) +{ + if (zen4_ibs_extensions) + return pr_ibs_op_data2_extended(reg); + pr_ibs_op_data2_default(reg); +} + +static void pr_ibs_op_data3(union ibs_op_data3 reg) +{ + static const char * const dc_page_sizes[] = { + " 4K", + " 2M", + " 1G", + " ??", + }; + char op_dc_miss_open_mem_reqs_str[sizeof(" OpDcMissOpenMemReqs __")] = ""; + char dc_l1_l2tlb_miss_str[sizeof(" DcL1TlbMiss _ DcL2TlbMiss _")] = ""; + char dc_l1tlb_hit_str[sizeof(" DcL1TlbHit2M _ DcL1TlbHit1G _")] = ""; + char op_mem_width_str[sizeof(" OpMemWidth _____ bytes")] = ""; + char tlb_refill_lat_str[sizeof(" TlbRefillLat _____")] = ""; + char dc_l2tlb_hit_2m_str[sizeof(" DcL2TlbHit2M _")] = ""; + char dc_l2tlb_hit_1g_str[sizeof(" DcL2TlbHit1G _")] = ""; + char dc_page_size_str[sizeof(" DcPageSize ____")] = ""; + char l2_miss_str[sizeof(" L2Miss _")] = ""; + + /* + * Erratum #1293 + * Ignore L2Miss and OpDcMissOpenMemReqs (and opdata2) if DcMissNoMabAlloc or SwPf set + */ + if (!(cpu_family == 0x19 && cpu_model < 0x10 && (reg.dc_miss_no_mab_alloc || reg.sw_pf))) { + snprintf(l2_miss_str, sizeof(l2_miss_str), " L2Miss %d", reg.l2_miss); + snprintf(op_dc_miss_open_mem_reqs_str, sizeof(op_dc_miss_open_mem_reqs_str), + " OpDcMissOpenMemReqs %2d", reg.op_dc_miss_open_mem_reqs); + } + + if (reg.op_mem_width) + snprintf(op_mem_width_str, sizeof(op_mem_width_str), + " OpMemWidth %2d bytes", 1 << (reg.op_mem_width - 1)); + + if (dtlb_pgsize_cap) { + if (reg.dc_phy_addr_valid) { + int idx = (reg.dc_l1tlb_hit_1g << 1) | reg.dc_l1tlb_hit_2m; + + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_page_size_str, sizeof(dc_page_size_str), + " DcPageSize %4s", dc_page_sizes[idx]); + } + } else { + snprintf(dc_l1_l2tlb_miss_str, sizeof(dc_l1_l2tlb_miss_str), + " DcL1TlbMiss %d DcL2TlbMiss %d", + reg.dc_l1tlb_miss, reg.dc_l2tlb_miss); + snprintf(dc_l1tlb_hit_str, sizeof(dc_l1tlb_hit_str), + " DcL1TlbHit2M %d DcL1TlbHit1G %d", + reg.dc_l1tlb_hit_2m, reg.dc_l1tlb_hit_1g); + snprintf(dc_l2tlb_hit_2m_str, sizeof(dc_l2tlb_hit_2m_str), + " DcL2TlbHit2M %d", reg.dc_l2tlb_hit_2m); + snprintf(dc_l2tlb_hit_1g_str, sizeof(dc_l2tlb_hit_1g_str), + " DcL2TlbHit1G %d", reg.dc_l2_tlb_hit_1g); + } + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (!zen4_ibs_extensions || reg.dc_phy_addr_valid) { + snprintf(tlb_refill_lat_str, sizeof(tlb_refill_lat_str), + " TlbRefillLat %5d", reg.tlb_refill_lat); + } + + printf("ibs_op_data3:\t%016llx LdOp %d StOp %d%s%s%s DcMiss %d DcMisAcc %d " + "DcWcMemAcc %d DcUcMemAcc %d DcLockedOp %d DcMissNoMabAlloc %d " + "DcLinAddrValid %d DcPhyAddrValid %d%s%s SwPf %d%s%s " + "DcMissLat %5d%s\n", + reg.val, reg.ld_op, reg.st_op, dc_l1_l2tlb_miss_str, + dtlb_pgsize_cap ? dc_page_size_str : dc_l1tlb_hit_str, + dc_l2tlb_hit_2m_str, reg.dc_miss, reg.dc_mis_acc, reg.dc_wc_mem_acc, + reg.dc_uc_mem_acc, reg.dc_locked_op, reg.dc_miss_no_mab_alloc, + reg.dc_lin_addr_valid, reg.dc_phy_addr_valid, dc_l2tlb_hit_1g_str, + l2_miss_str, reg.sw_pf, op_mem_width_str, op_dc_miss_open_mem_reqs_str, + reg.dc_miss_lat, tlb_refill_lat_str); +} + +/* + * IBS Op/Execution MSRs always saved, in order, are: + * IBS_OP_CTL, IBS_OP_RIP, IBS_OP_DATA, IBS_OP_DATA2, + * IBS_OP_DATA3, IBS_DC_LINADDR, IBS_DC_PHYSADDR, BP_IBSTGT_RIP + */ +static void amd_dump_ibs_op(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_op_ctl *op_ctl = (union ibs_op_ctl *)data->data; + __u64 *rip = (__u64 *)op_ctl + 1; + union ibs_op_data *op_data = (union ibs_op_data *)(rip + 1); + union ibs_op_data3 *op_data3 = (union ibs_op_data3 *)(rip + 3); + + pr_ibs_op_ctl(*op_ctl); + if (!op_data->op_rip_invalid) + printf("IbsOpRip:\t%016llx\n", *rip); + pr_ibs_op_data(*op_data); + /* + * Erratum #1293: ignore op_data2 if DcMissNoMabAlloc or SwPf are set + */ + if (!(cpu_family == 0x19 && cpu_model < 0x10 && + (op_data3->dc_miss_no_mab_alloc || op_data3->sw_pf))) + pr_ibs_op_data2(*(union ibs_op_data2 *)(rip + 2)); + pr_ibs_op_data3(*op_data3); + if (op_data3->dc_lin_addr_valid) + printf("IbsDCLinAd:\t%016llx\n", *(rip + 4)); + + /* Use !zen4_ibs_extensions as a proxy for Zen3 and earlier */ + if (op_data3->dc_phy_addr_valid && *(rip + 5) && + (!zen4_ibs_extensions || op_data3->dc_lin_addr_valid)) { + printf("IbsDCPhysAd:\t%016llx\n", *(rip + 5)); + } + if (op_data->op_brn_ret && *(rip + 6)) + printf("IbsBrTarget:\t%016llx\n", *(rip + 6)); +} + +/* + * IBS Fetch MSRs always saved, in order, are: + * IBS_FETCH_CTL, IBS_FETCH_LINADDR, IBS_FETCH_PHYSADDR, IC_IBS_EXTD_CTL + */ +static void amd_dump_ibs_fetch(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_fetch_ctl *fetch_ctl = (union ibs_fetch_ctl *)data->data; + __u64 *addr = (__u64 *)fetch_ctl + 1; + union ic_ibs_extd_ctl *extd_ctl = (union ic_ibs_extd_ctl *)addr + 2; + + pr_ibs_fetch_ctl(*fetch_ctl); + printf("IbsFetchLinAd:\t%016llx\n", *addr++); + if (fetch_ctl->phy_addr_valid) + printf("IbsFetchPhysAd:\t%016llx\n", *addr); + pr_ic_ibs_extd_ctl(*fetch_ctl, *extd_ctl); +} + +/* + * Test for enable and valid bits in captured control MSRs. + */ +static bool is_valid_ibs_fetch_sample(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_fetch_ctl *fetch_ctl = (union ibs_fetch_ctl *)data->data; + + if (fetch_ctl->fetch_en && fetch_ctl->fetch_val) + return true; + + return false; +} + +static bool is_valid_ibs_op_sample(struct perf_sample *sample) +{ + struct perf_ibs_data *data = sample->raw_data; + union ibs_op_ctl *op_ctl = (union ibs_op_ctl *)data->data; + + if (op_ctl->op_en && op_ctl->op_val) + return true; + + return false; +} + +/* AMD vendor specific raw sample function. Check for PERF_RECORD_SAMPLE events + * and if the event was triggered by IBS, display its raw data with decoded text. + * The function is only invoked when the dump flag -D is set. + */ +void evlist__amd_sample_raw(struct evlist *evlist, union perf_event *event, + struct perf_sample *sample) +{ + struct evsel *evsel; + + if (event->header.type != PERF_RECORD_SAMPLE || !sample->raw_size) + return; + + evsel = evlist__event2evsel(evlist, event); + if (!evsel) + return; + + if (evsel->core.attr.type == ibs_fetch_type) { + if (!is_valid_ibs_fetch_sample(sample)) { + pr_debug("Invalid raw IBS Fetch MSR data encountered\n"); + return; + } + amd_dump_ibs_fetch(sample); + } else if (evsel->core.attr.type == ibs_op_type) { + if (!is_valid_ibs_op_sample(sample)) { + pr_debug("Invalid raw IBS Op MSR data encountered\n"); + return; + } + amd_dump_ibs_op(sample); + } +} + +static void parse_cpuid(struct perf_env *env) +{ + const char *cpuid; + int ret; + + cpuid = perf_env__cpuid(env); + /* + * cpuid = "AuthenticAMD,family,model,stepping" + */ + ret = sscanf(cpuid, "%*[^,],%u,%u", &cpu_family, &cpu_model); + if (ret != 2) + pr_debug("problem parsing cpuid\n"); +} + +/* + * Find and assign the type number used for ibs_op or ibs_fetch samples. + * Device names can be large - we are only interested in the first 9 characters, + * to match "ibs_fetch". + */ +bool evlist__has_amd_ibs(struct evlist *evlist) +{ + struct perf_env *env = perf_session__env(evlist->session); + int ret, nr_pmu_mappings = perf_env__nr_pmu_mappings(env); + const char *pmu_mapping = perf_env__pmu_mappings(env); + char name[sizeof("ibs_fetch")]; + u32 type; + + while (nr_pmu_mappings--) { + ret = sscanf(pmu_mapping, "%u:%9s", &type, name); + if (ret == 2) { + if (strstarts(name, "ibs_op")) + ibs_op_type = type; + else if (strstarts(name, "ibs_fetch")) + ibs_fetch_type = type; + } + pmu_mapping += strlen(pmu_mapping) + 1 /* '\0' */; + } + + if (perf_env__find_pmu_cap(env, "ibs_op", "zen4_ibs_extensions")) + zen4_ibs_extensions = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "ldlat")) + ldlat_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "dtlb_pgsize")) + dtlb_pgsize_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "rmtsocket")) + rmtsocket_cap = 1; + + if (perf_env__find_pmu_cap(env, "ibs_op", "strmst")) + strmst_cap = 1; + + if (ibs_fetch_type || ibs_op_type) { + if (!cpu_family) + parse_cpuid(env); + return true; + } + + return false; +} diff --git a/tools/perf/util/annotate-arch/Build b/tools/perf/util/annotate-arch/Build new file mode 100644 index 000000000..23316743f --- /dev/null +++ b/tools/perf/util/annotate-arch/Build @@ -0,0 +1,11 @@ +perf-util-y += annotate-arc.o +perf-util-y += annotate-arm.o +perf-util-y += annotate-arm64.o +perf-util-y += annotate-csky.o +perf-util-y += annotate-loongarch.o +perf-util-y += annotate-mips.o +perf-util-y += annotate-x86.o +perf-util-y += annotate-powerpc.o +perf-util-y += annotate-riscv64.o +perf-util-y += annotate-s390.o +perf-util-y += annotate-sparc.o diff --git a/tools/perf/util/annotate-arch/annotate-arc.c b/tools/perf/util/annotate-arch/annotate-arc.c new file mode 100644 index 000000000..170103e38 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-arc.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../disasm.h" + +const struct arch *arch__new_arc(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "arc"; + arch->id = *id; + arch->objdump.comment_char = ';'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-arm.c b/tools/perf/util/annotate-arch/annotate-arm.c new file mode 100644 index 000000000..afb413c80 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-arm.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "../annotate.h" +#include "../disasm.h" + +struct arch_arm { + struct arch arch; + regex_t call_insn; + regex_t jump_insn; +}; + +static const struct ins_ops *arm__associate_instruction_ops(struct arch *arch, const char *name) +{ + struct arch_arm *arm = container_of(arch, struct arch_arm, arch); + const struct ins_ops *ops; + regmatch_t match[2]; + + if (!regexec(&arm->call_insn, name, 2, match, 0)) + ops = &call_ops; + else if (!regexec(&arm->jump_insn, name, 2, match, 0)) + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +const struct arch *arch__new_arm(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + int err; + struct arch_arm *arm = zalloc(sizeof(*arm)); + struct arch *arch; + + if (!arm) + return NULL; + + arch = &arm->arch; + arch->name = "arm"; + arch->id = *id; + arch->objdump.comment_char = ';'; + arch->objdump.skip_functions_char = '+'; + arch->associate_instruction_ops = arm__associate_instruction_ops; + +#define ARM_CONDS "(cc|cs|eq|ge|gt|hi|le|ls|lt|mi|ne|pl|vc|vs)" + err = regcomp(&arm->call_insn, "^blx?" ARM_CONDS "?$", REG_EXTENDED); + if (err) + goto out_free_arm; + + err = regcomp(&arm->jump_insn, "^bx?" ARM_CONDS "?$", REG_EXTENDED); + if (err) + goto out_free_call; +#undef ARM_CONDS + + return arch; + +out_free_call: + regfree(&arm->call_insn); +out_free_arm: + free(arm); + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP; + return NULL; +} diff --git a/tools/perf/util/annotate-arch/annotate-arm64.c b/tools/perf/util/annotate-arch/annotate-arm64.c new file mode 100644 index 000000000..33080fdca --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-arm64.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include "../annotate.h" +#include "../disasm.h" + +struct arch_arm64 { + struct arch arch; + regex_t call_insn; + regex_t jump_insn; +}; + +static int arm64_mov__parse(const struct arch *arch __maybe_unused, + struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *s = strchr(ops->raw, ','), *target, *endptr; + + if (s == NULL) + return -1; + + *s = '\0'; + ops->source.raw = strdup(ops->raw); + *s = ','; + + if (ops->source.raw == NULL) + return -1; + + target = ++s; + ops->target.raw = strdup(target); + if (ops->target.raw == NULL) + goto out_free_source; + + ops->target.addr = strtoull(target, &endptr, 16); + if (endptr == target) + goto out_free_target; + + s = strchr(endptr, '<'); + if (s == NULL) + goto out_free_target; + endptr = strchr(s + 1, '>'); + if (endptr == NULL) + goto out_free_target; + + *endptr = '\0'; + *s = ' '; + ops->target.name = strdup(s); + *s = '<'; + *endptr = '>'; + if (ops->target.name == NULL) + goto out_free_target; + + return 0; + +out_free_target: + zfree(&ops->target.raw); +out_free_source: + zfree(&ops->source.raw); + return -1; +} + +static const struct ins_ops arm64_mov_ops = { + .parse = arm64_mov__parse, + .scnprintf = mov__scnprintf, +}; + +static const struct ins_ops *arm64__associate_instruction_ops(struct arch *arch, const char *name) +{ + struct arch_arm64 *arm = container_of(arch, struct arch_arm64, arch); + const struct ins_ops *ops; + regmatch_t match[2]; + + if (!regexec(&arm->jump_insn, name, 2, match, 0)) + ops = &jump_ops; + else if (!regexec(&arm->call_insn, name, 2, match, 0)) + ops = &call_ops; + else if (!strcmp(name, "ret")) + ops = &ret_ops; + else + ops = &arm64_mov_ops; + + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +const struct arch *arch__new_arm64(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + int err; + struct arch_arm64 *arm = zalloc(sizeof(*arm)); + struct arch *arch; + + if (!arm) + return NULL; + + arch = &arm->arch; + arch->name = "arm64"; + arch->id = *id; + arch->objdump.comment_char = '/'; + arch->objdump.skip_functions_char = '+'; + arch->associate_instruction_ops = arm64__associate_instruction_ops; + + /* bl, blr */ + err = regcomp(&arm->call_insn, "^blr?$", REG_EXTENDED); + if (err) + goto out_free_arm; + + /* b, b.cond, br, cbz/cbnz, tbz/tbnz */ + err = regcomp(&arm->jump_insn, "^[ct]?br?\\.?(cc|cs|eq|ge|gt|hi|hs|le|lo|ls|lt|mi|ne|pl|vc|vs)?n?z?$", + REG_EXTENDED); + if (err) + goto out_free_call; + + return arch; + +out_free_call: + regfree(&arm->call_insn); +out_free_arm: + free(arm); + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP; + return NULL; +} diff --git a/tools/perf/util/annotate-arch/annotate-csky.c b/tools/perf/util/annotate-arch/annotate-csky.c new file mode 100644 index 000000000..d2b18e4ea --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-csky.c @@ -0,0 +1,56 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. +#include +#include +#include +#include "../disasm.h" + +static const struct ins_ops *csky__associate_ins_ops(struct arch *arch, + const char *name) +{ + const struct ins_ops *ops = NULL; + + /* catch all kind of jumps */ + if (!strcmp(name, "bt") || + !strcmp(name, "bf") || + !strcmp(name, "bez") || + !strcmp(name, "bnez") || + !strcmp(name, "bnezad") || + !strcmp(name, "bhsz") || + !strcmp(name, "bhz") || + !strcmp(name, "blsz") || + !strcmp(name, "blz") || + !strcmp(name, "br") || + !strcmp(name, "jmpi") || + !strcmp(name, "jmp")) + ops = &jump_ops; + + /* catch function call */ + if (!strcmp(name, "bsr") || + !strcmp(name, "jsri") || + !strcmp(name, "jsr")) + ops = &call_ops; + + /* catch function return */ + if (!strcmp(name, "rts")) + ops = &ret_ops; + + if (ops) + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +const struct arch *arch__new_csky(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "csky"; + arch->id = *id; + arch->objdump.comment_char = '/'; + arch->associate_instruction_ops = csky__associate_ins_ops; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-loongarch.c b/tools/perf/util/annotate-arch/annotate-loongarch.c new file mode 100644 index 000000000..c2addca77 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-loongarch.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Perf annotate functions. + * + * Copyright (C) 2020-2023 Loongson Technology Corporation Limited + */ +#include +#include +#include +#include +#include "../disasm.h" +#include "../map.h" +#include "../maps.h" +#include "../symbol.h" +#include "../thread.h" + +static int loongarch_call__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + char *c, *endptr, *tok, *name; + struct map *map = ms->map; + struct addr_map_symbol target; + + c = strchr(ops->raw, '#'); + if (c++ == NULL) + return -1; + + ops->target.addr = strtoull(c, &endptr, 16); + + name = strchr(endptr, '<'); + name++; + + if (arch->objdump.skip_functions_char && + strchr(name, arch->objdump.skip_functions_char)) + return -1; + + tok = strchr(name, '>'); + if (tok == NULL) + return -1; + + *tok = '\0'; + ops->target.name = strdup(name); + *tok = '>'; + + if (ops->target.name == NULL) + return -1; + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + return 0; +} + +static const struct ins_ops loongarch_call_ops = { + .parse = loongarch_call__parse, + .scnprintf = call__scnprintf, + .is_call = true, +}; + +static int loongarch_jump__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) + +{ + struct map *map = ms->map; + struct symbol *sym = ms->sym; + struct addr_map_symbol target = { + .ms = { .map = map__get(map), }, + }; + const char *c = strchr(ops->raw, '#'); + u64 start, end; + + ops->jump.raw_comment = strchr(ops->raw, arch->objdump.comment_char); + ops->jump.raw_func_start = strchr(ops->raw, '<'); + + if (ops->jump.raw_func_start && c > ops->jump.raw_func_start) + c = NULL; + + if (c++ != NULL) + ops->target.addr = strtoull(c, NULL, 16); + else + ops->target.addr = strtoull(ops->raw, NULL, 16); + + target.addr = map__objdump_2mem(map, ops->target.addr); + start = map__unmap_ip(map, sym->start); + end = map__unmap_ip(map, sym->end); + + ops->target.outside = target.addr < start || target.addr >= end; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + if (!ops->target.outside) { + ops->target.offset = target.addr - start; + ops->target.offset_avail = true; + } else { + ops->target.offset_avail = false; + } + addr_map_symbol__exit(&target); + return 0; +} + +static const struct ins_ops loongarch_jump_ops = { + .free = jump__delete, + .parse = loongarch_jump__parse, + .scnprintf = jump__scnprintf, + .is_jump = true, +}; + +static +const struct ins_ops *loongarch__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strcmp(name, "bl")) + ops = &loongarch_call_ops; + else if (!strcmp(name, "jirl")) + ops = &ret_ops; + else if (!strcmp(name, "b") || + !strncmp(name, "beq", 3) || + !strncmp(name, "bne", 3) || + !strncmp(name, "blt", 3) || + !strncmp(name, "bge", 3) || + !strncmp(name, "bltu", 4) || + !strncmp(name, "bgeu", 4)) + ops = &loongarch_jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_loongarch(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "loongarch"; + arch->id = *id; + arch->associate_instruction_ops = loongarch__associate_ins_ops; + arch->objdump.comment_char = '#'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-mips.c b/tools/perf/util/annotate-arch/annotate-mips.c new file mode 100644 index 000000000..e8d1c6c7e --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-mips.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../disasm.h" + +static +const struct ins_ops *mips__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strncmp(name, "bal", 3) || + !strncmp(name, "bgezal", 6) || + !strncmp(name, "bltzal", 6) || + !strncmp(name, "bgtzal", 6) || + !strncmp(name, "blezal", 6) || + !strncmp(name, "beqzal", 6) || + !strncmp(name, "bnezal", 6) || + !strncmp(name, "bgtzl", 5) || + !strncmp(name, "bltzl", 5) || + !strncmp(name, "bgezl", 5) || + !strncmp(name, "blezl", 5) || + !strncmp(name, "jialc", 5) || + !strncmp(name, "beql", 4) || + !strncmp(name, "bnel", 4) || + !strncmp(name, "jal", 3)) + ops = &call_ops; + else if (!strncmp(name, "jr", 2)) + ops = &ret_ops; + else if (name[0] == 'j' || name[0] == 'b') + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_mips(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "mips"; + arch->id = *id; + arch->objdump.comment_char = '#'; + arch->associate_instruction_ops = mips__associate_ins_ops; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-powerpc.c b/tools/perf/util/annotate-arch/annotate-powerpc.c new file mode 100644 index 000000000..218207b52 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-powerpc.c @@ -0,0 +1,410 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../annotate-data.h" +#include "../debug.h" +#include "../disasm.h" + +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define PPC_21_30(R) (((R) >> 1) & 0x3ff) +#define PPC_22_30(R) (((R) >> 1) & 0x1ff) + +#define MINUS_EXT_XO_FORM 234 +#define SUB_EXT_XO_FORM 232 +#define ADD_ZERO_EXT_XO_FORM 202 +#define SUB_ZERO_EXT_XO_FORM 200 + +static int arithmetic__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, + ops->raw); +} + +/* + * Sets the fields: multi_regs and "mem_ref". + * "mem_ref" is set for ops->source which is later used to + * fill the objdump->memory_ref-char field. This ops is currently + * used by powerpc and since binary instruction code is used to + * extract opcode, regs and offset, no other parsing is needed here. + * + * Dont set multi regs for 4 cases since it has only one operand + * for source: + * - Add to Minus One Extended XO-form ( Ex: addme, addmeo ) + * - Subtract From Minus One Extended XO-form ( Ex: subfme ) + * - Add to Zero Extended XO-form ( Ex: addze, addzeo ) + * - Subtract From Zero Extended XO-form ( Ex: subfze ) + */ +static int arithmetic__parse(const struct arch *arch __maybe_unused, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, struct disasm_line *dl) +{ + int opcode = PPC_OP(dl->raw.raw_insn); + + ops->source.mem_ref = false; + if (opcode == 31) { + if ((opcode != MINUS_EXT_XO_FORM) && (opcode != SUB_EXT_XO_FORM) && + (opcode != ADD_ZERO_EXT_XO_FORM) && (opcode != SUB_ZERO_EXT_XO_FORM)) + ops->source.multi_regs = true; + } + + ops->target.mem_ref = false; + ops->target.multi_regs = false; + + return 0; +} + +static const struct ins_ops arithmetic_ops = { + .parse = arithmetic__parse, + .scnprintf = arithmetic__scnprintf, +}; + +static int load_store__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, + ops->raw); +} + +/* + * Sets the fields: multi_regs and "mem_ref". + * "mem_ref" is set for ops->source which is later used to + * fill the objdump->memory_ref-char field. This ops is currently + * used by powerpc and since binary instruction code is used to + * extract opcode, regs and offset, no other parsing is needed here + */ +static int load_store__parse(const struct arch *arch __maybe_unused, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, struct disasm_line *dl __maybe_unused) +{ + ops->source.mem_ref = true; + ops->source.multi_regs = false; + /* opcode 31 is of X form */ + if (PPC_OP(dl->raw.raw_insn) == 31) + ops->source.multi_regs = true; + + ops->target.mem_ref = false; + ops->target.multi_regs = false; + + return 0; +} + +static const struct ins_ops load_store_ops = { + .parse = load_store__parse, + .scnprintf = load_store__scnprintf, +}; + +static const struct ins_ops *powerpc__associate_instruction_ops(struct arch *arch, const char *name) +{ + int i; + const struct ins_ops *ops; + + /* + * - Interested only if instruction starts with 'b'. + * - Few start with 'b', but aren't branch instructions. + */ + if (name[0] != 'b' || + !strncmp(name, "bcd", 3) || + !strncmp(name, "brinc", 5) || + !strncmp(name, "bper", 4)) + return NULL; + + ops = &jump_ops; + + i = strlen(name) - 1; + if (i < 0) + return NULL; + + /* ignore optional hints at the end of the instructions */ + if (name[i] == '+' || name[i] == '-') + i--; + + if (name[i] == 'l' || (name[i] == 'a' && name[i-1] == 'l')) { + /* + * if the instruction ends up with 'l' or 'la', then + * those are considered 'calls' since they update LR. + * ... except for 'bnl' which is branch if not less than + * and the absolute form of the same. + */ + if (strcmp(name, "bnl") && strcmp(name, "bnl+") && + strcmp(name, "bnl-") && strcmp(name, "bnla") && + strcmp(name, "bnla+") && strcmp(name, "bnla-")) + ops = &call_ops; + } + if (name[i] == 'r' && name[i-1] == 'l') + /* + * instructions ending with 'lr' are considered to be + * return instructions + */ + ops = &ret_ops; + + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +struct insn_offset { + const char *name; + int value; +}; + +/* + * There are memory instructions with opcode 31 which are + * of X Form, Example: + * ldx RT,RA,RB + * ______________________________________ + * | 31 | RT | RA | RB | 21 |/| + * -------------------------------------- + * 0 6 11 16 21 30 31 + * + * But all instructions with opcode 31 are not memory. + * Example: add RT,RA,RB + * + * Use bits 21 to 30 to check memory insns with 31 as opcode. + * In ins_array below, for ldx instruction: + * name => OP_31_XOP_LDX + * value => 21 + */ + +static struct insn_offset ins_array[] = { + { .name = "OP_31_XOP_LXSIWZX", .value = 12, }, + { .name = "OP_31_XOP_LWARX", .value = 20, }, + { .name = "OP_31_XOP_LDX", .value = 21, }, + { .name = "OP_31_XOP_LWZX", .value = 23, }, + { .name = "OP_31_XOP_LDUX", .value = 53, }, + { .name = "OP_31_XOP_LWZUX", .value = 55, }, + { .name = "OP_31_XOP_LXSIWAX", .value = 76, }, + { .name = "OP_31_XOP_LDARX", .value = 84, }, + { .name = "OP_31_XOP_LBZX", .value = 87, }, + { .name = "OP_31_XOP_LVX", .value = 103, }, + { .name = "OP_31_XOP_LBZUX", .value = 119, }, + { .name = "OP_31_XOP_STXSIWX", .value = 140, }, + { .name = "OP_31_XOP_STDX", .value = 149, }, + { .name = "OP_31_XOP_STWX", .value = 151, }, + { .name = "OP_31_XOP_STDUX", .value = 181, }, + { .name = "OP_31_XOP_STWUX", .value = 183, }, + { .name = "OP_31_XOP_STBX", .value = 215, }, + { .name = "OP_31_XOP_STVX", .value = 231, }, + { .name = "OP_31_XOP_STBUX", .value = 247, }, + { .name = "OP_31_XOP_LHZX", .value = 279, }, + { .name = "OP_31_XOP_LHZUX", .value = 311, }, + { .name = "OP_31_XOP_LXVDSX", .value = 332, }, + { .name = "OP_31_XOP_LWAX", .value = 341, }, + { .name = "OP_31_XOP_LHAX", .value = 343, }, + { .name = "OP_31_XOP_LWAUX", .value = 373, }, + { .name = "OP_31_XOP_LHAUX", .value = 375, }, + { .name = "OP_31_XOP_STHX", .value = 407, }, + { .name = "OP_31_XOP_STHUX", .value = 439, }, + { .name = "OP_31_XOP_LXSSPX", .value = 524, }, + { .name = "OP_31_XOP_LDBRX", .value = 532, }, + { .name = "OP_31_XOP_LSWX", .value = 533, }, + { .name = "OP_31_XOP_LWBRX", .value = 534, }, + { .name = "OP_31_XOP_LFSUX", .value = 567, }, + { .name = "OP_31_XOP_LXSDX", .value = 588, }, + { .name = "OP_31_XOP_LSWI", .value = 597, }, + { .name = "OP_31_XOP_LFDX", .value = 599, }, + { .name = "OP_31_XOP_LFDUX", .value = 631, }, + { .name = "OP_31_XOP_STXSSPX", .value = 652, }, + { .name = "OP_31_XOP_STDBRX", .value = 660, }, + { .name = "OP_31_XOP_STXWX", .value = 661, }, + { .name = "OP_31_XOP_STWBRX", .value = 662, }, + { .name = "OP_31_XOP_STFSX", .value = 663, }, + { .name = "OP_31_XOP_STFSUX", .value = 695, }, + { .name = "OP_31_XOP_STXSDX", .value = 716, }, + { .name = "OP_31_XOP_STSWI", .value = 725, }, + { .name = "OP_31_XOP_STFDX", .value = 727, }, + { .name = "OP_31_XOP_STFDUX", .value = 759, }, + { .name = "OP_31_XOP_LXVW4X", .value = 780, }, + { .name = "OP_31_XOP_LHBRX", .value = 790, }, + { .name = "OP_31_XOP_LXVD2X", .value = 844, }, + { .name = "OP_31_XOP_LFIWAX", .value = 855, }, + { .name = "OP_31_XOP_LFIWZX", .value = 887, }, + { .name = "OP_31_XOP_STXVW4X", .value = 908, }, + { .name = "OP_31_XOP_STHBRX", .value = 918, }, + { .name = "OP_31_XOP_STXVD2X", .value = 972, }, + { .name = "OP_31_XOP_STFIWX", .value = 983, }, +}; + +/* + * Arithmetic instructions which are having opcode as 31. + * These instructions are tracked to save the register state + * changes. Example: + * + * lwz r10,264(r3) + * add r31, r3, r3 + * lwz r9, 0(r31) + * + * Here instruction tracking needs to identify the "add" + * instruction and save data type of r3 to r31. If a sample + * is hit at next "lwz r9, 0(r31)", by this instruction tracking, + * data type of r31 can be resolved. + */ +static struct insn_offset arithmetic_ins_op_31[] = { + { .name = "SUB_CARRY_XO_FORM", .value = 8, }, + { .name = "MUL_HDW_XO_FORM1", .value = 9, }, + { .name = "ADD_CARRY_XO_FORM", .value = 10, }, + { .name = "MUL_HW_XO_FORM1", .value = 11, }, + { .name = "SUB_XO_FORM", .value = 40, }, + { .name = "MUL_HDW_XO_FORM", .value = 73, }, + { .name = "MUL_HW_XO_FORM", .value = 75, }, + { .name = "SUB_EXT_XO_FORM", .value = 136, }, + { .name = "ADD_EXT_XO_FORM", .value = 138, }, + { .name = "SUB_ZERO_EXT_XO_FORM", .value = 200, }, + { .name = "ADD_ZERO_EXT_XO_FORM", .value = 202, }, + { .name = "SUB_EXT_XO_FORM2", .value = 232, }, + { .name = "MUL_DW_XO_FORM", .value = 233, }, + { .name = "ADD_EXT_XO_FORM2", .value = 234, }, + { .name = "MUL_W_XO_FORM", .value = 235, }, + { .name = "ADD_XO_FORM", .value = 266, }, + { .name = "DIV_DW_XO_FORM1", .value = 457, }, + { .name = "DIV_W_XO_FORM1", .value = 459, }, + { .name = "DIV_DW_XO_FORM", .value = 489, }, + { .name = "DIV_W_XO_FORM", .value = 491, }, +}; + +static struct insn_offset arithmetic_two_ops[] = { + { .name = "mulli", .value = 7, }, + { .name = "subfic", .value = 8, }, + { .name = "addic", .value = 12, }, + { .name = "addic.", .value = 13, }, + { .name = "addi", .value = 14, }, + { .name = "addis", .value = 15, }, +}; + +static int cmp_offset(const void *a, const void *b) +{ + const struct insn_offset *val1 = a; + const struct insn_offset *val2 = b; + + return (val1->value - val2->value); +} + +const struct ins_ops *check_ppc_insn(struct disasm_line *dl) +{ + int raw_insn = dl->raw.raw_insn; + int opcode = PPC_OP(raw_insn); + int mem_insn_31 = PPC_21_30(raw_insn); + struct insn_offset *ret; + struct insn_offset mem_insns_31_opcode = { + "OP_31_INSN", + mem_insn_31 + }; + char name_insn[32]; + + /* + * Instructions with opcode 32 to 63 are memory + * instructions in powerpc + */ + if ((opcode & 0x20)) { + /* + * Set name in case of raw instruction to + * opcode to be used in insn-stat + */ + if (!strlen(dl->ins.name)) { + sprintf(name_insn, "%d", opcode); + dl->ins.name = strdup(name_insn); + } + return &load_store_ops; + } else if (opcode == 31) { + /* Check for memory instructions with opcode 31 */ + ret = bsearch(&mem_insns_31_opcode, ins_array, ARRAY_SIZE(ins_array), sizeof(ins_array[0]), cmp_offset); + if (ret) { + if (!strlen(dl->ins.name)) + dl->ins.name = strdup(ret->name); + return &load_store_ops; + } else { + mem_insns_31_opcode.value = PPC_22_30(raw_insn); + ret = bsearch(&mem_insns_31_opcode, arithmetic_ins_op_31, ARRAY_SIZE(arithmetic_ins_op_31), + sizeof(arithmetic_ins_op_31[0]), cmp_offset); + if (ret != NULL) + return &arithmetic_ops; + /* Bits 21 to 30 has value 444 for "mr" insn ie, OR X form */ + if (PPC_21_30(raw_insn) == 444) + return &arithmetic_ops; + } + } else { + mem_insns_31_opcode.value = opcode; + ret = bsearch(&mem_insns_31_opcode, arithmetic_two_ops, ARRAY_SIZE(arithmetic_two_ops), + sizeof(arithmetic_two_ops[0]), cmp_offset); + if (ret != NULL) + return &arithmetic_ops; + } + + return NULL; +} + +/* + * Instruction tracking function to track register state moves. + * Example sequence: + * ld r10,264(r3) + * mr r31,r3 + * < + * ld r9,312(r31) + * + * Previous instruction sequence shows that register state of r3 + * is moved to r31. update_insn_state_powerpc tracks these state + * changes + */ +#ifdef HAVE_LIBDW_SUPPORT +static void update_insn_state_powerpc(struct type_state *state, + struct data_loc_info *dloc, Dwarf_Die * cu_die __maybe_unused, + struct disasm_line *dl) +{ + struct annotated_insn_loc loc; + struct annotated_op_loc *src = &loc.ops[INSN_OP_SOURCE]; + struct annotated_op_loc *dst = &loc.ops[INSN_OP_TARGET]; + struct type_state_reg *tsr; + u32 insn_offset = dl->al.offset; + + if (annotate_get_insn_location(dloc->arch, dl, &loc) < 0) + return; + + /* + * Value 444 for bits 21:30 is for "mr" + * instruction. "mr" is extended OR. So set the + * source and destination reg correctly + */ + if (PPC_21_30(dl->raw.raw_insn) == 444) { + int src_reg = src->reg1; + + src->reg1 = dst->reg1; + dst->reg1 = src_reg; + } + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + + if (!has_reg_type(state, src->reg1) || + !state->regs[src->reg1].ok) { + tsr->ok = false; + return; + } + + tsr->type = state->regs[src->reg1].type; + tsr->kind = state->regs[src->reg1].kind; + tsr->ok = true; + + pr_debug_dtp("mov [%x] reg%d -> reg%d", + insn_offset, src->reg1, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); +} +#endif /* HAVE_LIBDW_SUPPORT */ + +const struct arch *arch__new_powerpc(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "powerpc"; + arch->id = *id; + arch->objdump.comment_char = '#'; + annotate_opts.show_asm_raw = true; + arch->associate_instruction_ops = powerpc__associate_instruction_ops; +#ifdef HAVE_LIBDW_SUPPORT + arch->update_insn_state = update_insn_state_powerpc; +#endif + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-riscv64.c b/tools/perf/util/annotate-arch/annotate-riscv64.c new file mode 100644 index 000000000..29a988fca --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-riscv64.c @@ -0,0 +1,41 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../disasm.h" + +static +const struct ins_ops *riscv64__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strncmp(name, "jal", 3) || + !strncmp(name, "jr", 2) || + !strncmp(name, "call", 4)) + ops = &call_ops; + else if (!strncmp(name, "ret", 3)) + ops = &ret_ops; + else if (name[0] == 'j' || name[0] == 'b') + ops = &jump_ops; + else + return NULL; + + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_riscv64(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "riscv"; + arch->id = *id; + arch->objdump.comment_char = '#'; + arch->associate_instruction_ops = riscv64__associate_ins_ops; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-s390.c b/tools/perf/util/annotate-arch/annotate-s390.c new file mode 100644 index 000000000..af9cabd0a --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-s390.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../debug.h" +#include "../disasm.h" +#include "../map.h" +#include "../maps.h" +#include "../symbol.h" +#include "../thread.h" +#include "../annotate.h" +#include "../annotate-data.h" + +static int s390_call__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + char *endptr, *tok, *name; + struct map *map = ms->map; + struct addr_map_symbol target; + + tok = strchr(ops->raw, ','); + if (!tok) + return -1; + + ops->target.addr = strtoull(tok + 1, &endptr, 16); + + name = strchr(endptr, '<'); + if (name == NULL) + return -1; + + name++; + + if (arch->objdump.skip_functions_char && + strchr(name, arch->objdump.skip_functions_char)) + return -1; + + tok = strchr(name, '>'); + if (tok == NULL) + return -1; + + *tok = '\0'; + ops->target.name = strdup(name); + *tok = '>'; + + if (ops->target.name == NULL) + return -1; + + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + return 0; +} + +static const struct ins_ops s390_call_ops = { + .parse = s390_call__parse, + .scnprintf = call__scnprintf, + .is_call = true, +}; + +static int s390_mov__parse(const struct arch *arch __maybe_unused, + struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *s = strchr(ops->raw, ','), *target, *endptr; + + if (s == NULL) + return -1; + + *s = '\0'; + ops->source.raw = strdup(ops->raw); + *s = ','; + + if (ops->source.raw == NULL) + return -1; + + target = ++s; + ops->target.raw = strdup(target); + if (ops->target.raw == NULL) + goto out_free_source; + + ops->target.addr = strtoull(target, &endptr, 16); + if (endptr == target) + goto out_free_target; + + s = strchr(endptr, '<'); + if (s == NULL) + goto out_free_target; + endptr = strchr(s + 1, '>'); + if (endptr == NULL) + goto out_free_target; + + *endptr = '\0'; + ops->target.name = strdup(s + 1); + *endptr = '>'; + if (ops->target.name == NULL) + goto out_free_target; + + return 0; + +out_free_target: + zfree(&ops->target.raw); +out_free_source: + zfree(&ops->source.raw); + return -1; +} + + +static const struct ins_ops s390_mov_ops = { + .parse = s390_mov__parse, + .scnprintf = mov__scnprintf, +}; + +static const struct ins_ops *s390__associate_ins_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + /* catch all kind of jumps */ + if (strchr(name, 'j') || + !strncmp(name, "bct", 3) || + !strncmp(name, "br", 2)) + ops = &jump_ops; + /* override call/returns */ + if (!strcmp(name, "bras") || + !strcmp(name, "brasl") || + !strcmp(name, "basr")) + ops = &s390_call_ops; + if (!strcmp(name, "br")) + ops = &ret_ops; + /* override load/store relative to PC */ + if (!strcmp(name, "lrl") || + !strcmp(name, "lgrl") || + !strcmp(name, "lgfrl") || + !strcmp(name, "llgfrl") || + !strcmp(name, "strl") || + !strcmp(name, "stgrl")) + ops = &s390_mov_ops; + + if (ops) + arch__associate_ins_ops(arch, name, ops); + return ops; +} + +static int s390__cpuid_parse(struct arch *arch, const char *cpuid) +{ + unsigned int family; + char model[16], model_c[16], cpumf_v[16], cpumf_a[16]; + int ret; + + /* + * cpuid string format: + * "IBM,family,model-capacity,model[,cpum_cf-version,cpum_cf-authorization]" + */ + ret = sscanf(cpuid, "%*[^,],%u,%[^,],%[^,],%[^,],%s", &family, model_c, + model, cpumf_v, cpumf_a); + if (ret >= 2) { + arch->family = family; + arch->model = 0; + return 0; + } + + return -1; +} + +const struct arch *arch__new_s390(const struct e_machine_and_e_flags *id, const char *cpuid) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "s390"; + arch->id = *id; + arch->associate_instruction_ops = s390__associate_ins_ops; + if (cpuid) { + if (s390__cpuid_parse(arch, cpuid)) { + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING; + return NULL; + } + } + arch->objdump.comment_char = '#'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-sparc.c b/tools/perf/util/annotate-arch/annotate-sparc.c new file mode 100644 index 000000000..2f07bc7a5 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-sparc.c @@ -0,0 +1,177 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "../../util/disasm.h" + +static int is_branch_cond(const char *cond) +{ + if (cond[0] == '\0') + return 1; + + if (cond[0] == 'a' && cond[1] == '\0') + return 1; + + if (cond[0] == 'c' && + (cond[1] == 'c' || cond[1] == 's') && + cond[2] == '\0') + return 1; + + if (cond[0] == 'e' && + (cond[1] == '\0' || + (cond[1] == 'q' && cond[2] == '\0'))) + return 1; + + if (cond[0] == 'g' && + (cond[1] == '\0' || + (cond[1] == 't' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == 'u' && cond[3] == '\0'))) + return 1; + + if (cond[0] == 'l' && + (cond[1] == '\0' || + (cond[1] == 't' && cond[2] == '\0') || + (cond[1] == 'u' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == 'u' && cond[3] == '\0'))) + return 1; + + if (cond[0] == 'n' && + (cond[1] == '\0' || + (cond[1] == 'e' && cond[2] == '\0') || + (cond[1] == 'z' && cond[2] == '\0') || + (cond[1] == 'e' && cond[2] == 'g' && cond[3] == '\0'))) + return 1; + + if (cond[0] == 'b' && + cond[1] == 'p' && + cond[2] == 'o' && + cond[3] == 's' && + cond[4] == '\0') + return 1; + + if (cond[0] == 'v' && + (cond[1] == 'c' || cond[1] == 's') && + cond[2] == '\0') + return 1; + + if (cond[0] == 'b' && + cond[1] == 'z' && + cond[2] == '\0') + return 1; + + return 0; +} + +static int is_branch_reg_cond(const char *cond) +{ + if ((cond[0] == 'n' || cond[0] == 'l') && + cond[1] == 'z' && + cond[2] == '\0') + return 1; + + if (cond[0] == 'z' && + cond[1] == '\0') + return 1; + + if ((cond[0] == 'g' || cond[0] == 'l') && + cond[1] == 'e' && + cond[2] == 'z' && + cond[3] == '\0') + return 1; + + if (cond[0] == 'g' && + cond[1] == 'z' && + cond[2] == '\0') + return 1; + + return 0; +} + +static int is_branch_float_cond(const char *cond) +{ + if (cond[0] == '\0') + return 1; + + if ((cond[0] == 'a' || cond[0] == 'e' || + cond[0] == 'z' || cond[0] == 'g' || + cond[0] == 'l' || cond[0] == 'n' || + cond[0] == 'o' || cond[0] == 'u') && + cond[1] == '\0') + return 1; + + if (((cond[0] == 'g' && cond[1] == 'e') || + (cond[0] == 'l' && (cond[1] == 'e' || + cond[1] == 'g')) || + (cond[0] == 'n' && (cond[1] == 'e' || + cond[1] == 'z')) || + (cond[0] == 'u' && (cond[1] == 'e' || + cond[1] == 'g' || + cond[1] == 'l'))) && + cond[2] == '\0') + return 1; + + if (cond[0] == 'u' && + (cond[1] == 'g' || cond[1] == 'l') && + cond[2] == 'e' && + cond[3] == '\0') + return 1; + + return 0; +} + +static const struct ins_ops *sparc__associate_instruction_ops(struct arch *arch, const char *name) +{ + const struct ins_ops *ops = NULL; + + if (!strcmp(name, "call") || + !strcmp(name, "jmp") || + !strcmp(name, "jmpl")) { + ops = &call_ops; + } else if (!strcmp(name, "ret") || + !strcmp(name, "retl") || + !strcmp(name, "return")) { + ops = &ret_ops; + } else if (!strcmp(name, "mov")) { + ops = &mov_ops; + } else { + if (name[0] == 'c' && + (name[1] == 'w' || name[1] == 'x')) + name += 2; + + if (name[0] == 'b') { + const char *cond = name + 1; + + if (cond[0] == 'r') { + if (is_branch_reg_cond(cond + 1)) + ops = &jump_ops; + } else if (is_branch_cond(cond)) { + ops = &jump_ops; + } + } else if (name[0] == 'f' && name[1] == 'b') { + if (is_branch_float_cond(name + 2)) + ops = &jump_ops; + } + } + + if (ops) + arch__associate_ins_ops(arch, name, ops); + + return ops; +} + +const struct arch *arch__new_sparc(const struct e_machine_and_e_flags *id, + const char *cpuid __maybe_unused) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "sparc"; + arch->id = *id; + arch->associate_instruction_ops = sparc__associate_instruction_ops; + arch->objdump.comment_char = '#'; + return arch; +} diff --git a/tools/perf/util/annotate-arch/annotate-x86.c b/tools/perf/util/annotate-arch/annotate-x86.c new file mode 100644 index 000000000..7e6136536 --- /dev/null +++ b/tools/perf/util/annotate-arch/annotate-x86.c @@ -0,0 +1,852 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "../annotate-data.h" +#include "../debug.h" +#include "../disasm.h" +#include "../dso.h" +#include "../map.h" +#include "../string2.h" // strstarts +#include "../symbol.h" + +/* + * x86 instruction nmemonic table to parse disasm lines for annotate. + * This table is searched twice - one for exact match and another for + * match without a size suffix (b, w, l, q) in case of AT&T syntax. + * + * So this table should not have entries with the suffix unless it's + * a complete different instruction than ones without the suffix. + */ +static const struct ins x86__instructions[] = { + { .name = "adc", .ops = &mov_ops, }, + { .name = "add", .ops = &mov_ops, }, + { .name = "addsd", .ops = &mov_ops, }, + { .name = "and", .ops = &mov_ops, }, + { .name = "andpd", .ops = &mov_ops, }, + { .name = "andps", .ops = &mov_ops, }, + { .name = "bsr", .ops = &mov_ops, }, + { .name = "bt", .ops = &mov_ops, }, + { .name = "btr", .ops = &mov_ops, }, + { .name = "bts", .ops = &mov_ops, }, + { .name = "call", .ops = &call_ops, }, + { .name = "cmovae", .ops = &mov_ops, }, + { .name = "cmovbe", .ops = &mov_ops, }, + { .name = "cmove", .ops = &mov_ops, }, + { .name = "cmp", .ops = &mov_ops, }, + { .name = "cmpxch", .ops = &mov_ops, }, + { .name = "cmpxchg", .ops = &mov_ops, }, + { .name = "cs", .ops = &mov_ops, }, + { .name = "dec", .ops = &dec_ops, }, + { .name = "divsd", .ops = &mov_ops, }, + { .name = "divss", .ops = &mov_ops, }, + { .name = "gs", .ops = &mov_ops, }, + { .name = "imul", .ops = &mov_ops, }, + { .name = "inc", .ops = &dec_ops, }, + { .name = "ja", .ops = &jump_ops, }, + { .name = "jae", .ops = &jump_ops, }, + { .name = "jb", .ops = &jump_ops, }, + { .name = "jbe", .ops = &jump_ops, }, + { .name = "jc", .ops = &jump_ops, }, + { .name = "jcxz", .ops = &jump_ops, }, + { .name = "je", .ops = &jump_ops, }, + { .name = "jecxz", .ops = &jump_ops, }, + { .name = "jg", .ops = &jump_ops, }, + { .name = "jge", .ops = &jump_ops, }, + { .name = "jl", .ops = &jump_ops, }, + { .name = "jle", .ops = &jump_ops, }, + { .name = "jmp", .ops = &jump_ops, }, + { .name = "jna", .ops = &jump_ops, }, + { .name = "jnae", .ops = &jump_ops, }, + { .name = "jnb", .ops = &jump_ops, }, + { .name = "jnbe", .ops = &jump_ops, }, + { .name = "jnc", .ops = &jump_ops, }, + { .name = "jne", .ops = &jump_ops, }, + { .name = "jng", .ops = &jump_ops, }, + { .name = "jnge", .ops = &jump_ops, }, + { .name = "jnl", .ops = &jump_ops, }, + { .name = "jnle", .ops = &jump_ops, }, + { .name = "jno", .ops = &jump_ops, }, + { .name = "jnp", .ops = &jump_ops, }, + { .name = "jns", .ops = &jump_ops, }, + { .name = "jnz", .ops = &jump_ops, }, + { .name = "jo", .ops = &jump_ops, }, + { .name = "jp", .ops = &jump_ops, }, + { .name = "jpe", .ops = &jump_ops, }, + { .name = "jpo", .ops = &jump_ops, }, + { .name = "jrcxz", .ops = &jump_ops, }, + { .name = "js", .ops = &jump_ops, }, + { .name = "jz", .ops = &jump_ops, }, + { .name = "lea", .ops = &mov_ops, }, + { .name = "lock", .ops = &lock_ops, }, + { .name = "mov", .ops = &mov_ops, }, + { .name = "movapd", .ops = &mov_ops, }, + { .name = "movaps", .ops = &mov_ops, }, + { .name = "movdqa", .ops = &mov_ops, }, + { .name = "movdqu", .ops = &mov_ops, }, + { .name = "movsb", .ops = &mov_ops, }, + { .name = "movsd", .ops = &mov_ops, }, + { .name = "movsl", .ops = &mov_ops, }, + { .name = "movss", .ops = &mov_ops, }, + { .name = "movsw", .ops = &mov_ops, }, + { .name = "movupd", .ops = &mov_ops, }, + { .name = "movups", .ops = &mov_ops, }, + { .name = "movzb", .ops = &mov_ops, }, + { .name = "movzl", .ops = &mov_ops, }, + { .name = "movzw", .ops = &mov_ops, }, + { .name = "mulsd", .ops = &mov_ops, }, + { .name = "mulss", .ops = &mov_ops, }, + { .name = "nop", .ops = &nop_ops, }, + { .name = "or", .ops = &mov_ops, }, + { .name = "orps", .ops = &mov_ops, }, + { .name = "paddq", .ops = &mov_ops, }, + { .name = "pand", .ops = &mov_ops, }, + { .name = "pcmpeqb", .ops = &mov_ops, }, + { .name = "por", .ops = &mov_ops, }, + { .name = "rcl", .ops = &mov_ops, }, + { .name = "ret", .ops = &ret_ops, }, + { .name = "sbb", .ops = &mov_ops, }, + { .name = "sete", .ops = &mov_ops, }, + { .name = "sub", .ops = &mov_ops, }, + { .name = "subsd", .ops = &mov_ops, }, + { .name = "test", .ops = &mov_ops, }, + { .name = "tzcnt", .ops = &mov_ops, }, + { .name = "ucomisd", .ops = &mov_ops, }, + { .name = "ucomiss", .ops = &mov_ops, }, + { .name = "vaddsd", .ops = &mov_ops, }, + { .name = "vandpd", .ops = &mov_ops, }, + { .name = "vmovdqa", .ops = &mov_ops, }, + { .name = "vmovq", .ops = &mov_ops, }, + { .name = "vmovsd", .ops = &mov_ops, }, + { .name = "vmulsd", .ops = &mov_ops, }, + { .name = "vorpd", .ops = &mov_ops, }, + { .name = "vsubsd", .ops = &mov_ops, }, + { .name = "vucomisd", .ops = &mov_ops, }, + { .name = "xadd", .ops = &mov_ops, }, + { .name = "xbegin", .ops = &jump_ops, }, + { .name = "xchg", .ops = &mov_ops, }, + { .name = "xor", .ops = &mov_ops, }, + { .name = "xorpd", .ops = &mov_ops, }, + { .name = "xorps", .ops = &mov_ops, }, +}; + +static bool amd__ins_is_fused(const struct arch *arch, const char *ins1, + const char *ins2) +{ + if (strstr(ins2, "jmp")) + return false; + + /* Family >= 15h supports cmp/test + branch fusion */ + if (arch->family >= 0x15 && (strstarts(ins1, "test") || + (strstarts(ins1, "cmp") && !strstr(ins1, "xchg")))) { + return true; + } + + /* Family >= 19h supports some ALU + branch fusion */ + if (arch->family >= 0x19 && (strstarts(ins1, "add") || + strstarts(ins1, "sub") || strstarts(ins1, "and") || + strstarts(ins1, "inc") || strstarts(ins1, "dec") || + strstarts(ins1, "or") || strstarts(ins1, "xor"))) { + return true; + } + + return false; +} + +static bool intel__ins_is_fused(const struct arch *arch, const char *ins1, + const char *ins2) +{ + if (arch->family != 6 || arch->model < 0x1e || strstr(ins2, "jmp")) + return false; + + if (arch->model == 0x1e) { + /* Nehalem */ + if ((strstr(ins1, "cmp") && !strstr(ins1, "xchg")) || + strstr(ins1, "test")) { + return true; + } + } else { + /* Newer platform */ + if ((strstr(ins1, "cmp") && !strstr(ins1, "xchg")) || + strstr(ins1, "test") || + strstr(ins1, "add") || + strstr(ins1, "sub") || + strstr(ins1, "and") || + strstr(ins1, "inc") || + strstr(ins1, "dec")) { + return true; + } + } + + return false; +} + +static int x86__cpuid_parse(struct arch *arch, const char *cpuid) +{ + unsigned int family, model, stepping; + int ret; + + /* + * cpuid = "GenuineIntel,family,model,stepping" + */ + ret = sscanf(cpuid, "%*[^,],%u,%u,%u", &family, &model, &stepping); + if (ret == 3) { + arch->family = family; + arch->model = model; + arch->ins_is_fused = strstarts(cpuid, "AuthenticAMD") ? + amd__ins_is_fused : + intel__ins_is_fused; + return 0; + } + + return -1; +} + +#ifdef HAVE_LIBDW_SUPPORT +static void invalidate_reg_state(struct type_state_reg *reg) +{ + reg->kind = TSR_KIND_INVALID; + reg->ok = false; + reg->lifetime_active = false; + reg->lifetime_end = 0; + reg->copied_from = -1; +} + +static void update_insn_state_x86(struct type_state *state, + struct data_loc_info *dloc, Dwarf_Die *cu_die, + struct disasm_line *dl) +{ + struct annotated_insn_loc loc; + struct annotated_op_loc *src = &loc.ops[INSN_OP_SOURCE]; + struct annotated_op_loc *dst = &loc.ops[INSN_OP_TARGET]; + struct type_state_reg *tsr; + Dwarf_Die type_die; + u32 insn_offset = dl->al.offset; + int fbreg = dloc->fbreg; + int fboff = 0; + + if (annotate_get_insn_location(dloc->arch, dl, &loc) < 0) + return; + + if (ins__is_call(&dl->ins)) { + struct symbol *func = dl->ops.target.sym; + const char *call_name; + u64 call_addr; + + /* Try to resolve the call target name */ + if (func) + call_name = func->name; + else + call_name = dl->ops.target.name; + + /* __fentry__ will preserve all registers */ + if (call_name && !strcmp(call_name, "__fentry__")) + return; + + if (call_name) + pr_debug_dtp("call [%x] %s\n", insn_offset, call_name); + else + pr_debug_dtp("call [%x] \n", insn_offset); + + /* Invalidate caller-saved registers after call */ + call_addr = map__rip_2objdump(dloc->ms->map, + dloc->ms->sym->start + dl->al.offset); + for (unsigned i = 0; i < ARRAY_SIZE(state->regs); i++) { + struct type_state_reg *reg = &state->regs[i]; + + if (!reg->caller_saved) + continue; + /* Keep register valid within DWARF location lifetime */ + if (reg->lifetime_active && call_addr < reg->lifetime_end) + continue; + invalidate_reg_state(reg); + } + + /* Update register with the return type (if any) */ + if (call_name && die_find_func_rettype(cu_die, call_name, &type_die)) { + tsr = &state->regs[state->ret_reg]; + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("call [%x] return -> reg%d", + insn_offset, state->ret_reg); + pr_debug_type_name(&type_die, tsr->kind); + } + return; + } + + if (!strncmp(dl->ins.name, "add", 3)) { + u64 imm_value = -1ULL; + int offset; + const char *var_name = NULL; + struct map_symbol *ms = dloc->ms; + u64 ip = ms->sym->start + dl->al.offset; + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + tsr->lifetime_active = false; + tsr->lifetime_end = 0; + + if (src->imm) + imm_value = src->offset; + else if (has_reg_type(state, src->reg1) && + state->regs[src->reg1].kind == TSR_KIND_CONST) + imm_value = state->regs[src->reg1].imm_value; + else if (src->reg1 == DWARF_REG_PC) { + u64 var_addr = annotate_calc_pcrel(dloc->ms, ip, + src->offset, dl); + + if (get_global_var_info(dloc, var_addr, + &var_name, &offset) && + !strcmp(var_name, "this_cpu_off") && + tsr->kind == TSR_KIND_CONST) { + tsr->kind = TSR_KIND_PERCPU_BASE; + tsr->offset = 0; + tsr->ok = true; + imm_value = tsr->imm_value; + } + } + else + return; + + /* Ignore add to non-pointer or non-const types */ + if (tsr->kind == TSR_KIND_POINTER || + (dwarf_tag(&tsr->type) == DW_TAG_pointer_type && + src->reg1 != DWARF_REG_PC && tsr->kind == TSR_KIND_TYPE && !dst->mem_ref)) { + tsr->offset += imm_value; + pr_debug_dtp("add [%x] offset %#"PRIx64" to reg%d", + insn_offset, imm_value, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + + if (tsr->kind == TSR_KIND_CONST) + tsr->imm_value += imm_value; + + if (tsr->kind != TSR_KIND_PERCPU_BASE) + return; + + if (get_global_var_type(cu_die, dloc, ip, imm_value, &offset, + &type_die) && offset == 0) { + /* + * This is not a pointer type, but it should be treated + * as a pointer. + */ + tsr->type = type_die; + tsr->kind = TSR_KIND_PERCPU_POINTER; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("add [%x] percpu %#"PRIx64" -> reg%d", + insn_offset, imm_value, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + return; + } + + if (!strncmp(dl->ins.name, "sub", 3)) { + u64 imm_value = -1ULL; + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + tsr->lifetime_active = false; + tsr->lifetime_end = 0; + + if (src->imm) + imm_value = src->offset; + else if (has_reg_type(state, src->reg1) && + state->regs[src->reg1].kind == TSR_KIND_CONST) + imm_value = state->regs[src->reg1].imm_value; + + if (tsr->kind == TSR_KIND_POINTER || + (dwarf_tag(&tsr->type) == DW_TAG_pointer_type && + src->reg1 != DWARF_REG_PC && tsr->kind == TSR_KIND_TYPE && !dst->mem_ref)) { + tsr->offset -= imm_value; + pr_debug_dtp("sub [%x] offset %#"PRIx64" to reg%d", + insn_offset, imm_value, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + + if (tsr->kind == TSR_KIND_CONST) + tsr->imm_value -= imm_value; + + return; + } + + if (!strncmp(dl->ins.name, "lea", 3)) { + int sreg = src->reg1; + struct type_state_reg src_tsr; + + if (!has_reg_type(state, sreg) || + !has_reg_type(state, dst->reg1) || + !src->mem_ref) + return; + + src_tsr = state->regs[sreg]; + tsr = &state->regs[dst->reg1]; + + invalidate_reg_state(tsr); + + /* Case 1: Based on stack pointer or frame pointer */ + if (sreg == fbreg || sreg == state->stack_reg) { + struct type_state_stack *stack; + int offset = src->offset - fboff; + + stack = find_stack_state(state, offset); + if (!stack) + return; + + tsr->type = stack->type; + tsr->kind = TSR_KIND_POINTER; + tsr->offset = offset - stack->offset; + tsr->ok = true; + + if (sreg == fbreg) { + pr_debug_dtp("lea [%x] address of -%#x(stack) -> reg%d", + insn_offset, -src->offset, dst->reg1); + } else { + pr_debug_dtp("lea [%x] address of %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + } + + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Case 2: Based on a register holding a typed pointer */ + else if (src_tsr.ok && (src_tsr.kind == TSR_KIND_POINTER || + (dwarf_tag(&src_tsr.type) == DW_TAG_pointer_type && + src_tsr.kind == TSR_KIND_TYPE))) { + + if (src_tsr.kind == TSR_KIND_TYPE && + __die_get_real_type(&state->regs[sreg].type, &type_die) == NULL) + return; + + if (src_tsr.kind == TSR_KIND_POINTER) + type_die = state->regs[sreg].type; + + /* Check if the target type has a member at the new offset */ + if (die_get_member_type(&type_die, + src->offset + src_tsr.offset, &type_die) == NULL) + return; + + tsr->type = src_tsr.type; + tsr->kind = src_tsr.kind; + tsr->offset = src->offset + src_tsr.offset; + tsr->ok = true; + + pr_debug_dtp("lea [%x] address of %s%#x(reg%d) -> reg%d", + insn_offset, src->offset < 0 ? "-" : "", + abs(src->offset), sreg, dst->reg1); + + pr_debug_type_name(&tsr->type, tsr->kind); + } + return; + } + + /* Invalidate register states for other ops which may change pointers */ + if (has_reg_type(state, dst->reg1) && !dst->mem_ref && + dwarf_tag(&state->regs[dst->reg1].type) == DW_TAG_pointer_type) { + if (!strncmp(dl->ins.name, "imul", 4) || !strncmp(dl->ins.name, "mul", 3) || + !strncmp(dl->ins.name, "idiv", 4) || !strncmp(dl->ins.name, "div", 3) || + !strncmp(dl->ins.name, "shl", 3) || !strncmp(dl->ins.name, "shr", 3) || + !strncmp(dl->ins.name, "sar", 3) || !strncmp(dl->ins.name, "and", 3) || + !strncmp(dl->ins.name, "or", 2) || !strncmp(dl->ins.name, "neg", 3) || + !strncmp(dl->ins.name, "inc", 3) || !strncmp(dl->ins.name, "dec", 3)) { + pr_debug_dtp("%s [%x] invalidate reg%d\n", + dl->ins.name, insn_offset, dst->reg1); + invalidate_reg_state(&state->regs[dst->reg1]); + return; + } + + if (!strncmp(dl->ins.name, "xor", 3) && dst->reg1 == src->reg1) { + /* xor reg, reg clears the register */ + pr_debug_dtp("xor [%x] clear reg%d\n", + insn_offset, dst->reg1); + + state->regs[dst->reg1].kind = TSR_KIND_CONST; + state->regs[dst->reg1].imm_value = 0; + state->regs[dst->reg1].ok = true; + state->regs[dst->reg1].lifetime_active = false; + state->regs[dst->reg1].lifetime_end = 0; + state->regs[dst->reg1].copied_from = -1; + return; + } + } + + if (strncmp(dl->ins.name, "mov", 3)) + return; + + if (dloc->fb_cfa) { + u64 ip = dloc->ms->sym->start + dl->al.offset; + u64 pc = map__rip_2objdump(dloc->ms->map, ip); + + if (die_get_cfa(dloc->di->dbg, pc, &fbreg, &fboff) < 0) + fbreg = -1; + } + + /* Case 1. register to register or segment:offset to register transfers */ + if (!src->mem_ref && !dst->mem_ref) { + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + + if (dso__kernel(map__dso(dloc->ms->map)) && + src->segment == INSN_SEG_X86_GS && src->imm) { + u64 ip = dloc->ms->sym->start + dl->al.offset; + u64 var_addr; + int offset; + + /* + * In kernel, %gs points to a per-cpu region for the + * current CPU. Access with a constant offset should + * be treated as a global variable access. + */ + var_addr = src->offset; + + if (var_addr == 40) { + tsr->kind = TSR_KIND_CANARY; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] stack canary -> reg%d\n", + insn_offset, dst->reg1); + return; + } + + if (!get_global_var_type(cu_die, dloc, ip, var_addr, + &offset, &type_die) || + !die_get_member_type(&type_die, offset, &type_die)) { + invalidate_reg_state(tsr); + return; + } + + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] this-cpu addr=%#"PRIx64" -> reg%d", + insn_offset, var_addr, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + return; + } + + if (src->imm) { + tsr->kind = TSR_KIND_CONST; + tsr->imm_value = src->offset; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] imm=%#x -> reg%d\n", + insn_offset, tsr->imm_value, dst->reg1); + return; + } + + if (!has_reg_type(state, src->reg1) || + !state->regs[src->reg1].ok) { + invalidate_reg_state(tsr); + return; + } + + tsr->type = state->regs[src->reg1].type; + tsr->kind = state->regs[src->reg1].kind; + tsr->imm_value = state->regs[src->reg1].imm_value; + tsr->offset = state->regs[src->reg1].offset; + tsr->lifetime_active = state->regs[src->reg1].lifetime_active; + tsr->lifetime_end = state->regs[src->reg1].lifetime_end; + tsr->ok = true; + + /* To copy back the variable type later (hopefully) */ + if (tsr->kind == TSR_KIND_TYPE || tsr->kind == TSR_KIND_POINTER) + tsr->copied_from = src->reg1; + + pr_debug_dtp("mov [%x] reg%d -> reg%d", + insn_offset, src->reg1, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Case 2. memory to register transers */ + if (src->mem_ref && !dst->mem_ref) { + int sreg = src->reg1; + + if (!has_reg_type(state, dst->reg1)) + return; + + tsr = &state->regs[dst->reg1]; + tsr->copied_from = -1; + +retry: + /* Check stack variables with offset */ + if (sreg == fbreg || sreg == state->stack_reg) { + struct type_state_stack *stack; + int offset = src->offset - fboff; + + stack = find_stack_state(state, offset); + if (stack == NULL) { + invalidate_reg_state(tsr); + return; + } else if (!stack->compound) { + tsr->type = stack->type; + tsr->kind = stack->kind; + tsr->offset = stack->ptr_offset; + tsr->ok = true; + } else if (die_get_member_type(&stack->type, + offset - stack->offset, + &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + } else { + invalidate_reg_state(tsr); + return; + } + + if (sreg == fbreg) { + pr_debug_dtp("mov [%x] -%#x(stack) -> reg%d", + insn_offset, -offset, dst->reg1); + } else { + pr_debug_dtp("mov [%x] %#x(reg%d) -> reg%d", + insn_offset, offset, sreg, dst->reg1); + } + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* And then dereference the pointer if it has one */ + else if (has_reg_type(state, sreg) && state->regs[sreg].ok && + state->regs[sreg].kind == TSR_KIND_TYPE && + die_deref_ptr_type(&state->regs[sreg].type, + src->offset + state->regs[sreg].offset, &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Handle dereference of TSR_KIND_POINTER registers */ + else if (has_reg_type(state, sreg) && state->regs[sreg].ok && + state->regs[sreg].kind == TSR_KIND_POINTER && + die_get_member_type(&state->regs[sreg].type, + src->offset + state->regs[sreg].offset, &type_die)) { + tsr->type = state->regs[sreg].type; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = src->offset + state->regs[sreg].offset; + tsr->ok = true; + + pr_debug_dtp("mov [%x] addr %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Or check if it's a global variable */ + else if (sreg == DWARF_REG_PC) { + struct map_symbol *ms = dloc->ms; + u64 ip = ms->sym->start + dl->al.offset; + u64 addr; + int offset; + + addr = annotate_calc_pcrel(ms, ip, src->offset, dl); + + if (!get_global_var_type(cu_die, dloc, ip, addr, &offset, + &type_die) || + !die_get_member_type(&type_die, offset, &type_die)) { + invalidate_reg_state(tsr); + return; + } + + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] global addr=%"PRIx64" -> reg%d", + insn_offset, addr, dst->reg1); + pr_debug_type_name(&type_die, tsr->kind); + } + /* And check percpu access with base register */ + else if (has_reg_type(state, sreg) && + state->regs[sreg].kind == TSR_KIND_PERCPU_BASE) { + u64 ip = dloc->ms->sym->start + dl->al.offset; + u64 var_addr = src->offset; + int offset; + + if (src->multi_regs) { + int reg2 = (sreg == src->reg1) ? src->reg2 : src->reg1; + + if (has_reg_type(state, reg2) && state->regs[reg2].ok && + state->regs[reg2].kind == TSR_KIND_CONST) + var_addr += state->regs[reg2].imm_value; + } + + /* + * In kernel, %gs points to a per-cpu region for the + * current CPU. Access with a constant offset should + * be treated as a global variable access. + */ + if (get_global_var_type(cu_die, dloc, ip, var_addr, + &offset, &type_die) && + die_get_member_type(&type_die, offset, &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + if (src->multi_regs) { + pr_debug_dtp("mov [%x] percpu %#x(reg%d,reg%d) -> reg%d", + insn_offset, src->offset, src->reg1, + src->reg2, dst->reg1); + } else { + pr_debug_dtp("mov [%x] percpu %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + } + pr_debug_type_name(&tsr->type, tsr->kind); + } else { + invalidate_reg_state(tsr); + } + } + /* And then dereference the calculated pointer if it has one */ + else if (has_reg_type(state, sreg) && state->regs[sreg].ok && + state->regs[sreg].kind == TSR_KIND_PERCPU_POINTER && + die_get_member_type(&state->regs[sreg].type, + src->offset, &type_die)) { + tsr->type = type_die; + tsr->kind = TSR_KIND_TYPE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] pointer %#x(reg%d) -> reg%d", + insn_offset, src->offset, sreg, dst->reg1); + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* Or try another register if any */ + else if (src->multi_regs && sreg == src->reg1 && + src->reg1 != src->reg2) { + sreg = src->reg2; + goto retry; + } + else { + int offset; + const char *var_name = NULL; + + /* it might be per-cpu variable (in kernel) access */ + if (src->offset < 0) { + if (get_global_var_info(dloc, (s64)src->offset, + &var_name, &offset) && + !strcmp(var_name, "__per_cpu_offset")) { + tsr->kind = TSR_KIND_PERCPU_BASE; + tsr->offset = 0; + tsr->ok = true; + + pr_debug_dtp("mov [%x] percpu base reg%d\n", + insn_offset, dst->reg1); + return; + } + } + + invalidate_reg_state(tsr); + } + } + /* Case 3. register to memory transfers */ + if (!src->mem_ref && dst->mem_ref) { + if (!has_reg_type(state, src->reg1) || + !state->regs[src->reg1].ok) + return; + + /* Check stack variables with offset */ + if (dst->reg1 == fbreg || dst->reg1 == state->stack_reg) { + struct type_state_stack *stack; + int offset = dst->offset - fboff; + + tsr = &state->regs[src->reg1]; + + stack = find_stack_state(state, offset); + if (stack) { + /* + * The source register is likely to hold a type + * of member if it's a compound type. Do not + * update the stack variable type since we can + * get the member type later by using the + * die_get_member_type(). + */ + if (!stack->compound) + set_stack_state(stack, offset, tsr->kind, + &tsr->type, tsr->offset); + } else { + findnew_stack_state(state, offset, tsr->kind, + &tsr->type, tsr->offset); + } + + if (dst->reg1 == fbreg) { + pr_debug_dtp("mov [%x] reg%d -> -%#x(stack)", + insn_offset, src->reg1, -offset); + } else { + pr_debug_dtp("mov [%x] reg%d -> %#x(reg%d)", + insn_offset, src->reg1, offset, dst->reg1); + } + if (tsr->offset != 0) { + pr_debug_dtp(" reg%d offset %#x ->", + src->reg1, tsr->offset); + } + + pr_debug_type_name(&tsr->type, tsr->kind); + } + /* + * Ignore other transfers since it'd set a value in a struct + * and won't change the type. + */ + } + /* Case 4. memory to memory transfers (not handled for now) */ +} +#endif + +const struct arch *arch__new_x86(const struct e_machine_and_e_flags *id, const char *cpuid) +{ + struct arch *arch = zalloc(sizeof(*arch)); + + if (!arch) + return NULL; + + arch->name = "x86"; + arch->id = *id; + if (cpuid) { + if (x86__cpuid_parse(arch, cpuid)) { + errno = SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING; + return NULL; + } + } + arch->instructions = x86__instructions; + arch->nr_instructions = ARRAY_SIZE(x86__instructions); +#ifndef NDEBUG + { + static bool sorted_check; + + if (!sorted_check) { + for (size_t i = 0; i < arch->nr_instructions - 1; i++) { + assert(strcmp(arch->instructions[i].name, + arch->instructions[i + 1].name) <= 0); + } + sorted_check = true; + } + } +#endif + arch->sorted_instructions = true; + arch->objdump.comment_char = '#'; + arch->objdump.register_char = '%'; + arch->objdump.memory_ref_char = '('; + arch->objdump.imm_char = '$'; + arch->insn_suffix = "bwlq"; +#ifdef HAVE_LIBDW_SUPPORT + arch->update_insn_state = update_insn_state_x86; +#endif + return arch; +} diff --git a/tools/perf/util/annotate-data.c b/tools/perf/util/annotate-data.c new file mode 100644 index 000000000..63e3c54fa --- /dev/null +++ b/tools/perf/util/annotate-data.c @@ -0,0 +1,1960 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Convert sample address to data type using DWARF debug info. + * + * Written by Namhyung Kim + */ +#include +#include +#include +#include +#include + +#include "annotate.h" +#include "annotate-data.h" +#include "debuginfo.h" +#include "debug.h" +#include "dso.h" +#include "dwarf-regs.h" +#include "evsel.h" +#include "evlist.h" +#include "map.h" +#include "map_symbol.h" +#include "sort.h" +#include "strbuf.h" +#include "symbol.h" +#include "symbol_conf.h" +#include "thread.h" + +/* register number of the stack pointer */ +#define X86_REG_SP 7 + +static void delete_var_types(struct die_var_type *var_types); + +#define pr_debug_dtp(fmt, ...) \ +do { \ + if (debug_type_profile) \ + pr_info(fmt, ##__VA_ARGS__); \ + else \ + pr_debug3(fmt, ##__VA_ARGS__); \ +} while (0) + +void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind) +{ + struct strbuf sb; + char *str; + Dwarf_Word size = 0; + + if (!debug_type_profile && verbose < 3) + return; + + switch (kind) { + case TSR_KIND_INVALID: + pr_info("\n"); + return; + case TSR_KIND_PERCPU_BASE: + pr_info(" percpu base\n"); + return; + case TSR_KIND_CONST: + pr_info(" constant\n"); + return; + case TSR_KIND_PERCPU_POINTER: + pr_info(" percpu pointer"); + /* it also prints the type info */ + break; + case TSR_KIND_POINTER: + pr_info(" pointer"); + /* it also prints the type info */ + break; + case TSR_KIND_CANARY: + pr_info(" stack canary\n"); + return; + case TSR_KIND_TYPE: + default: + break; + } + + if (dwarf_aggregate_size(die, &size) != 0) + size = 0; + + strbuf_init(&sb, 32); + die_get_typename_from_type(die, &sb); + str = strbuf_detach(&sb, NULL); + pr_info(" type='%s' size=%#lx (die:%#lx)\n", + str, (long)size, (long)dwarf_dieoffset(die)); + free(str); +} + +static void pr_debug_location(Dwarf_Die *die, u64 pc, int reg) +{ + ptrdiff_t off = 0; + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + + if (!debug_type_profile && verbose < 3) + return; + + if (dwarf_attr(die, DW_AT_location, &attr) == NULL) + return; + + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + if (reg != DWARF_REG_PC && end <= pc) + continue; + if (reg != DWARF_REG_PC && start > pc) + break; + + pr_info(" variable location: "); + switch (ops->atom) { + case DW_OP_reg0 ...DW_OP_reg31: + pr_info("reg%d\n", ops->atom - DW_OP_reg0); + break; + case DW_OP_breg0 ...DW_OP_breg31: + pr_info("base=reg%d, offset=%#lx\n", + ops->atom - DW_OP_breg0, (long)ops->number); + break; + case DW_OP_regx: + pr_info("reg%ld\n", (long)ops->number); + break; + case DW_OP_bregx: + pr_info("base=reg%ld, offset=%#lx\n", + (long)ops->number, (long)ops->number2); + break; + case DW_OP_fbreg: + pr_info("use frame base, offset=%#lx\n", (long)ops->number); + break; + case DW_OP_addr: + pr_info("address=%#lx\n", (long)ops->number); + break; + default: + pr_info("unknown: code=%#x, number=%#lx\n", + ops->atom, (long)ops->number); + break; + } + break; + } +} + +static void pr_debug_scope(Dwarf_Die *scope_die) +{ + int tag; + + if (!debug_type_profile && verbose < 3) + return; + + pr_info("(die:%lx) ", (long)dwarf_dieoffset(scope_die)); + + tag = dwarf_tag(scope_die); + if (tag == DW_TAG_subprogram) + pr_info("[function] %s\n", die_name(scope_die)); + else if (tag == DW_TAG_inlined_subroutine) + pr_info("[inlined] %s\n", die_name(scope_die)); + else if (tag == DW_TAG_lexical_block) + pr_info("[block]\n"); + else + pr_info("[unknown] tag=%x\n", tag); +} + +bool has_reg_type(struct type_state *state, int reg) +{ + return (unsigned)reg < ARRAY_SIZE(state->regs); +} + +static void init_type_state(struct type_state *state, const struct arch *arch) +{ + memset(state, 0, sizeof(*state)); + INIT_LIST_HEAD(&state->stack_vars); + + if (arch__is_x86(arch)) { + state->regs[0].caller_saved = true; + state->regs[1].caller_saved = true; + state->regs[2].caller_saved = true; + state->regs[4].caller_saved = true; + state->regs[5].caller_saved = true; + state->regs[8].caller_saved = true; + state->regs[9].caller_saved = true; + state->regs[10].caller_saved = true; + state->regs[11].caller_saved = true; + state->ret_reg = 0; + state->stack_reg = X86_REG_SP; + } +} + +static void exit_type_state(struct type_state *state) +{ + struct type_state_stack *stack, *tmp; + + list_for_each_entry_safe(stack, tmp, &state->stack_vars, list) { + list_del(&stack->list); + free(stack); + } +} + +/* + * Compare type name and size to maintain them in a tree. + * I'm not sure if DWARF would have information of a single type in many + * different places (compilation units). If not, it could compare the + * offset of the type entry in the .debug_info section. + */ +static int data_type_cmp(const void *_key, const struct rb_node *node) +{ + const struct annotated_data_type *key = _key; + struct annotated_data_type *type; + + type = rb_entry(node, struct annotated_data_type, node); + + if (key->self.size != type->self.size) + return key->self.size - type->self.size; + return strcmp(key->self.type_name, type->self.type_name); +} + +static bool data_type_less(struct rb_node *node_a, const struct rb_node *node_b) +{ + struct annotated_data_type *a, *b; + + a = rb_entry(node_a, struct annotated_data_type, node); + b = rb_entry(node_b, struct annotated_data_type, node); + + if (a->self.size != b->self.size) + return a->self.size < b->self.size; + return strcmp(a->self.type_name, b->self.type_name) < 0; +} + +/* Recursively add new members for struct/union */ +static int __add_member_cb(Dwarf_Die *die, void *arg) +{ + struct annotated_member *parent = arg; + struct annotated_member *member; + Dwarf_Die member_type, die_mem; + Dwarf_Word size, loc, bit_size = 0; + Dwarf_Attribute attr; + struct strbuf sb; + int tag; + + if (dwarf_tag(die) != DW_TAG_member) + return DIE_FIND_CB_SIBLING; + + member = zalloc(sizeof(*member)); + if (member == NULL) + return DIE_FIND_CB_END; + + strbuf_init(&sb, 32); + die_get_typename(die, &sb); + + __die_get_real_type(die, &member_type); + if (dwarf_tag(&member_type) == DW_TAG_typedef) + die_get_real_type(&member_type, &die_mem); + else + die_mem = member_type; + + if (dwarf_aggregate_size(&die_mem, &size) < 0) + size = 0; + + if (dwarf_attr_integrate(die, DW_AT_data_member_location, &attr)) { + if (dwarf_formudata(&attr, &loc) != 0) { + if (die_get_data_member_location(die, &loc) != 0) + loc = 0; + } + } else { + /* bitfield member */ + if (dwarf_attr_integrate(die, DW_AT_data_bit_offset, &attr) && + dwarf_formudata(&attr, &loc) == 0) + loc /= 8; + else + loc = 0; + + if (dwarf_attr_integrate(die, DW_AT_bit_size, &attr) && + dwarf_formudata(&attr, &bit_size) == 0) + size = (bit_size + 7) / 8; + } + + member->type_name = strbuf_detach(&sb, NULL); + /* member->var_name can be NULL */ + if (dwarf_diename(die)) { + if (bit_size) { + if (asprintf(&member->var_name, "%s:%ld", + dwarf_diename(die), (long)bit_size) < 0) + member->var_name = NULL; + } else { + const char *name = dwarf_diename(die); + + member->var_name = name ? strdup(name) : NULL; + } + + if (member->var_name == NULL) { + free(member); + return DIE_FIND_CB_END; + } + } + member->size = size; + member->offset = loc + parent->offset; + INIT_LIST_HEAD(&member->children); + list_add_tail(&member->node, &parent->children); + + tag = dwarf_tag(&die_mem); + switch (tag) { + case DW_TAG_structure_type: + case DW_TAG_union_type: + die_find_child(&die_mem, __add_member_cb, member, &die_mem); + break; + default: + break; + } + return DIE_FIND_CB_SIBLING; +} + +static void add_member_types(struct annotated_data_type *parent, Dwarf_Die *type) +{ + Dwarf_Die die_mem; + + die_find_child(type, __add_member_cb, &parent->self, &die_mem); +} + +static void delete_members(struct annotated_member *member) +{ + struct annotated_member *child, *tmp; + + list_for_each_entry_safe(child, tmp, &member->children, node) { + list_del(&child->node); + delete_members(child); + zfree(&child->type_name); + zfree(&child->var_name); + free(child); + } +} + +static int fill_member_name(char *buf, size_t sz, struct annotated_member *m, + int offset, bool first) +{ + struct annotated_member *child; + + if (list_empty(&m->children)) + return 0; + + list_for_each_entry(child, &m->children, node) { + int len; + + if (offset < child->offset || offset >= child->offset + child->size) + continue; + + /* It can have anonymous struct/union members */ + if (child->var_name) { + len = scnprintf(buf, sz, "%s%s", + first ? "" : ".", child->var_name); + first = false; + } else { + len = 0; + } + + return fill_member_name(buf + len, sz - len, child, offset, first) + len; + } + return 0; +} + +int annotated_data_type__get_member_name(struct annotated_data_type *adt, + char *buf, size_t sz, int member_offset) +{ + return fill_member_name(buf, sz, &adt->self, member_offset, /*first=*/true); +} + +static struct annotated_data_type *dso__findnew_data_type(struct dso *dso, + Dwarf_Die *type_die) +{ + struct annotated_data_type *result = NULL; + struct annotated_data_type key; + struct rb_node *node; + struct strbuf sb; + char *type_name; + Dwarf_Word size; + + strbuf_init(&sb, 32); + if (die_get_typename_from_type(type_die, &sb) < 0) + strbuf_add(&sb, "(unknown type)", 14); + type_name = strbuf_detach(&sb, NULL); + + if (dwarf_tag(type_die) == DW_TAG_typedef) + die_get_real_type(type_die, type_die); + + if (dwarf_aggregate_size(type_die, &size) != 0) + size = 0; + + /* Check existing nodes in dso->data_types tree */ + key.self.type_name = type_name; + key.self.size = size; + node = rb_find(&key, dso__data_types(dso), data_type_cmp); + if (node) { + result = rb_entry(node, struct annotated_data_type, node); + free(type_name); + return result; + } + + /* If not, add a new one */ + result = zalloc(sizeof(*result)); + if (result == NULL) { + free(type_name); + return NULL; + } + + result->self.type_name = type_name; + result->self.size = size; + INIT_LIST_HEAD(&result->self.children); + + if (symbol_conf.annotate_data_member) + add_member_types(result, type_die); + + rb_add(&result->node, dso__data_types(dso), data_type_less); + return result; +} + +static bool find_cu_die(struct debuginfo *di, u64 pc, Dwarf_Die *cu_die) +{ + Dwarf_Off off, next_off; + size_t header_size; + + if (dwarf_addrdie(di->dbg, pc, cu_die) != NULL) + return cu_die; + + /* + * There are some kernels don't have full aranges and contain only a few + * aranges entries. Fallback to iterate all CU entries in .debug_info + * in case it's missing. + */ + off = 0; + while (dwarf_nextcu(di->dbg, off, &next_off, &header_size, + NULL, NULL, NULL) == 0) { + if (dwarf_offdie(di->dbg, off + header_size, cu_die) && + dwarf_haspc(cu_die, pc)) + return true; + + off = next_off; + } + return false; +} + +enum type_match_result { + PERF_TMR_UNKNOWN = 0, + PERF_TMR_OK, + PERF_TMR_NO_TYPE, + PERF_TMR_NO_POINTER, + PERF_TMR_NO_SIZE, + PERF_TMR_BAD_OFFSET, + PERF_TMR_BAIL_OUT, +}; + +static const char *match_result_str(enum type_match_result tmr) +{ + switch (tmr) { + case PERF_TMR_OK: + return "Good!"; + case PERF_TMR_NO_TYPE: + return "no type information"; + case PERF_TMR_NO_POINTER: + return "no/void pointer"; + case PERF_TMR_NO_SIZE: + return "type size is unknown"; + case PERF_TMR_BAD_OFFSET: + return "offset bigger than size"; + case PERF_TMR_UNKNOWN: + case PERF_TMR_BAIL_OUT: + default: + return "invalid state"; + } +} + +static bool is_compound_type(Dwarf_Die *type_die) +{ + int tag = dwarf_tag(type_die); + + return tag == DW_TAG_structure_type || tag == DW_TAG_union_type; +} + +/* returns if Type B has better information than Type A */ +static bool is_better_type(Dwarf_Die *type_a, Dwarf_Die *type_b) +{ + Dwarf_Word size_a, size_b; + Dwarf_Die die_a, die_b; + Dwarf_Die ptr_a, ptr_b; + Dwarf_Die *ptr_type_a, *ptr_type_b; + + ptr_type_a = die_get_pointer_type(type_a, &ptr_a); + ptr_type_b = die_get_pointer_type(type_b, &ptr_b); + + /* pointer type is preferred */ + if ((ptr_type_a != NULL) != (ptr_type_b != NULL)) + return ptr_type_b != NULL; + + if (ptr_type_b) { + /* + * We want to compare the target type, but 'void *' can fail to + * get the target type. + */ + if (die_get_real_type(ptr_type_a, &die_a) == NULL) + return true; + if (die_get_real_type(ptr_type_b, &die_b) == NULL) + return false; + + type_a = &die_a; + type_b = &die_b; + } + + /* bigger type is preferred */ + if (dwarf_aggregate_size(type_a, &size_a) < 0 || + dwarf_aggregate_size(type_b, &size_b) < 0) + return false; + + if (size_a != size_b) + return size_a < size_b; + + /* struct or union is preferred */ + if (is_compound_type(type_a) != is_compound_type(type_b)) + return is_compound_type(type_b); + + /* typedef is preferred */ + if (dwarf_tag(type_b) == DW_TAG_typedef) + return true; + + return false; +} + +/* The type info will be saved in @type_die */ +static enum type_match_result check_variable(struct data_loc_info *dloc, + Dwarf_Die *var_die, + Dwarf_Die *type_die, int reg, + int offset, bool is_fbreg) +{ + Dwarf_Word size; + bool needs_pointer = true; + Dwarf_Die sized_type; + + if (reg == DWARF_REG_PC) + needs_pointer = false; + else if (reg == dloc->fbreg || is_fbreg) + needs_pointer = false; + else if (arch__is_x86(dloc->arch) && reg == X86_REG_SP) + needs_pointer = false; + + /* Get the type of the variable */ + if (__die_get_real_type(var_die, type_die) == NULL) + return PERF_TMR_NO_TYPE; + + /* + * Usually it expects a pointer type for a memory access. + * Convert to a real type it points to. But global variables + * and local variables are accessed directly without a pointer. + */ + if (needs_pointer) { + if (die_get_pointer_type(type_die, type_die) == NULL || + __die_get_real_type(type_die, type_die) == NULL) + return PERF_TMR_NO_POINTER; + } + + if (dwarf_tag(type_die) == DW_TAG_typedef) + die_get_real_type(type_die, &sized_type); + else + sized_type = *type_die; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(&sized_type, &size) < 0) + return PERF_TMR_NO_SIZE; + + /* Minimal sanity check */ + if ((unsigned)offset >= size) + return PERF_TMR_BAD_OFFSET; + + return PERF_TMR_OK; +} + +struct type_state_stack *find_stack_state(struct type_state *state, + int offset) +{ + struct type_state_stack *stack; + + list_for_each_entry(stack, &state->stack_vars, list) { + if (offset == stack->offset) + return stack; + + if (stack->compound && stack->offset < offset && + offset < stack->offset + stack->size) + return stack; + } + return NULL; +} + +void set_stack_state(struct type_state_stack *stack, int offset, u8 kind, + Dwarf_Die *type_die, int ptr_offset) +{ + int tag; + Dwarf_Word size; + + if (kind == TSR_KIND_POINTER) { + /* TODO: arch-dependent pointer size */ + size = sizeof(void *); + } + else if (dwarf_aggregate_size(type_die, &size) < 0) + size = 0; + + stack->type = *type_die; + stack->size = size; + stack->offset = offset; + stack->ptr_offset = ptr_offset; + stack->kind = kind; + + if (kind == TSR_KIND_POINTER) { + stack->compound = false; + return; + } + + tag = dwarf_tag(type_die); + + switch (tag) { + case DW_TAG_structure_type: + case DW_TAG_union_type: + stack->compound = (kind != TSR_KIND_PERCPU_POINTER); + break; + default: + stack->compound = false; + break; + } +} + +struct type_state_stack *findnew_stack_state(struct type_state *state, + int offset, u8 kind, + Dwarf_Die *type_die, + int ptr_offset) +{ + struct type_state_stack *stack = find_stack_state(state, offset); + + if (stack) { + set_stack_state(stack, offset, kind, type_die, ptr_offset); + return stack; + } + + stack = malloc(sizeof(*stack)); + if (stack) { + set_stack_state(stack, offset, kind, type_die, ptr_offset); + list_add(&stack->list, &state->stack_vars); + } + return stack; +} + +/* Maintain a cache for quick global variable lookup */ +struct global_var_entry { + struct rb_node node; + char *name; + u64 start; + u64 end; + u64 die_offset; +}; + +static int global_var_cmp(const void *_key, const struct rb_node *node) +{ + const u64 addr = (uintptr_t)_key; + struct global_var_entry *gvar; + + gvar = rb_entry(node, struct global_var_entry, node); + + if (gvar->start <= addr && addr < gvar->end) + return 0; + return gvar->start > addr ? -1 : 1; +} + +static bool global_var_less(struct rb_node *node_a, const struct rb_node *node_b) +{ + struct global_var_entry *gvar_a, *gvar_b; + + gvar_a = rb_entry(node_a, struct global_var_entry, node); + gvar_b = rb_entry(node_b, struct global_var_entry, node); + + return gvar_a->start < gvar_b->start; +} + +static struct global_var_entry *global_var__find(struct data_loc_info *dloc, u64 addr) +{ + struct dso *dso = map__dso(dloc->ms->map); + struct rb_node *node; + + node = rb_find((void *)(uintptr_t)addr, dso__global_vars(dso), global_var_cmp); + if (node == NULL) + return NULL; + + return rb_entry(node, struct global_var_entry, node); +} + +static bool global_var__add(struct data_loc_info *dloc, u64 addr, + const char *name, Dwarf_Die *type_die) +{ + struct dso *dso = map__dso(dloc->ms->map); + struct global_var_entry *gvar; + Dwarf_Word size; + + if (dwarf_aggregate_size(type_die, &size) < 0) + return false; + + gvar = malloc(sizeof(*gvar)); + if (gvar == NULL) + return false; + + gvar->name = name ? strdup(name) : NULL; + if (name && gvar->name == NULL) { + free(gvar); + return false; + } + + gvar->start = addr; + gvar->end = addr + size; + gvar->die_offset = dwarf_dieoffset(type_die); + + rb_add(&gvar->node, dso__global_vars(dso), global_var_less); + return true; +} + +void global_var_type__tree_delete(struct rb_root *root) +{ + struct global_var_entry *gvar; + + while (!RB_EMPTY_ROOT(root)) { + struct rb_node *node = rb_first(root); + + rb_erase(node, root); + gvar = rb_entry(node, struct global_var_entry, node); + zfree(&gvar->name); + free(gvar); + } +} + +bool get_global_var_info(struct data_loc_info *dloc, u64 addr, + const char **var_name, int *var_offset) +{ + struct addr_location al; + struct symbol *sym; + u64 mem_addr; + + /* Kernel symbols might be relocated */ + mem_addr = addr + map__reloc(dloc->ms->map); + + addr_location__init(&al); + sym = thread__find_symbol_fb(dloc->thread, dloc->cpumode, + mem_addr, &al); + if (sym) { + *var_name = sym->name; + /* Calculate type offset from the start of variable */ + *var_offset = mem_addr - map__unmap_ip(al.map, sym->start); + } else { + *var_name = NULL; + } + addr_location__exit(&al); + if (*var_name == NULL) + return false; + + return true; +} + +static void global_var__collect(struct data_loc_info *dloc) +{ + Dwarf *dwarf = dloc->di->dbg; + Dwarf_Off off, next_off; + Dwarf_Die cu_die, type_die; + size_t header_size; + + /* Iterate all CU and collect global variables that have no location in a register. */ + off = 0; + while (dwarf_nextcu(dwarf, off, &next_off, &header_size, + NULL, NULL, NULL) == 0) { + struct die_var_type *var_types = NULL; + struct die_var_type *pos; + + if (dwarf_offdie(dwarf, off + header_size, &cu_die) == NULL) { + off = next_off; + continue; + } + + die_collect_global_vars(&cu_die, &var_types); + + for (pos = var_types; pos; pos = pos->next) { + const char *var_name = NULL; + int var_offset = 0; + + if (pos->reg != -1) + continue; + + if (!dwarf_offdie(dwarf, pos->die_off, &type_die)) + continue; + + get_global_var_info(dloc, pos->addr, &var_name, &var_offset); + + global_var__add(dloc, pos->addr, var_name, &type_die); + } + + delete_var_types(var_types); + + off = next_off; + } +} + +bool get_global_var_type(Dwarf_Die *cu_die, struct data_loc_info *dloc, + u64 ip, u64 var_addr, int *var_offset, + Dwarf_Die *type_die) +{ + u64 pc; + int offset; + const char *var_name = NULL; + struct global_var_entry *gvar; + struct dso *dso = map__dso(dloc->ms->map); + Dwarf_Die var_die; + + if (RB_EMPTY_ROOT(dso__global_vars(dso))) + global_var__collect(dloc); + + gvar = global_var__find(dloc, var_addr); + if (gvar) { + if (!dwarf_offdie(dloc->di->dbg, gvar->die_offset, type_die)) + return false; + + *var_offset = var_addr - gvar->start; + return true; + } + + /* Try to get the variable by address first */ + if (die_find_variable_by_addr(cu_die, var_addr, &var_die, type_die, + &offset)) { + var_name = dwarf_diename(&var_die); + *var_offset = offset; + goto ok; + } + + if (!get_global_var_info(dloc, var_addr, &var_name, var_offset)) + return false; + + pc = map__rip_2objdump(dloc->ms->map, ip); + + /* Try to get the name of global variable */ + if (die_find_variable_at(cu_die, var_name, pc, &var_die) && + check_variable(dloc, &var_die, type_die, DWARF_REG_PC, *var_offset, + /*is_fbreg=*/false) == PERF_TMR_OK) + goto ok; + + return false; + +ok: + /* The address should point to the start of the variable */ + global_var__add(dloc, var_addr - *var_offset, var_name, type_die); + return true; +} + +static bool die_is_same(Dwarf_Die *die_a, Dwarf_Die *die_b) +{ + return (die_a->cu == die_b->cu) && (die_a->addr == die_b->addr); +} + +static void tsr_set_lifetime(struct type_state_reg *tsr, + const struct die_var_type *var) +{ + if (var && var->has_range && var->end > var->addr) { + tsr->lifetime_active = true; + tsr->lifetime_end = var->end; + } else { + tsr->lifetime_active = false; + tsr->lifetime_end = 0; + } +} + +/** + * update_var_state - Update type state using given variables + * @state: type state table + * @dloc: data location info + * @addr: instruction address to match with variable + * @insn_offset: instruction offset (for debug) + * @var_types: list of variables with type info + * + * This function fills the @state table using @var_types info. Each variable + * is used only at the given location and updates an entry in the table. + */ +static void update_var_state(struct type_state *state, struct data_loc_info *dloc, + u64 addr, u64 insn_offset, struct die_var_type *var_types) +{ + Dwarf_Die mem_die; + struct die_var_type *var; + int fbreg = dloc->fbreg; + int fb_offset = 0; + + if (dloc->fb_cfa) { + if (die_get_cfa(dloc->di->dbg, addr, &fbreg, &fb_offset) < 0) + fbreg = -1; + } + + for (var = var_types; var != NULL; var = var->next) { + /* Check if addr falls within the variable's valid range */ + if (var->has_range) { + if (addr < var->addr || (var->end && addr >= var->end)) + continue; + } else { + if (addr != var->addr) + continue; + } + /* Get the type DIE using the offset */ + if (!dwarf_offdie(dloc->di->dbg, var->die_off, &mem_die)) + continue; + + if (var->reg == DWARF_REG_FB || var->reg == fbreg || var->reg == state->stack_reg) { + Dwarf_Die ptr_die; + Dwarf_Die *ptr_type; + int offset = var->offset; + struct type_state_stack *stack; + + ptr_type = die_get_pointer_type(&mem_die, &ptr_die); + + /* If the reg location holds the pointer value, dereference the type */ + if (!var->is_reg_var_addr && ptr_type && + __die_get_real_type(ptr_type, &mem_die) == NULL) + continue; + + if (var->reg != DWARF_REG_FB) + offset -= fb_offset; + + stack = find_stack_state(state, offset); + if (stack && stack->kind == TSR_KIND_TYPE && + !is_better_type(&stack->type, &mem_die)) + continue; + + findnew_stack_state(state, offset, TSR_KIND_TYPE, + &mem_die, /*ptr_offset=*/0); + + if (var->reg == state->stack_reg) { + pr_debug_dtp("var [%"PRIx64"] %#x(reg%d)", + insn_offset, offset, state->stack_reg); + } else { + pr_debug_dtp("var [%"PRIx64"] -%#x(stack)", + insn_offset, -offset); + } + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + } else if (has_reg_type(state, var->reg)) { + struct type_state_reg *reg; + Dwarf_Die orig_type; + + reg = &state->regs[var->reg]; + + if (reg->ok && reg->kind == TSR_KIND_TYPE && + (!is_better_type(®->type, &mem_die) || var->is_reg_var_addr)) + continue; + + /* Handle address registers with TSR_KIND_POINTER */ + if (var->is_reg_var_addr) { + if (reg->ok && reg->kind == TSR_KIND_POINTER && + !is_better_type(®->type, &mem_die)) + continue; + + reg->offset = -var->offset; + reg->type = mem_die; + reg->kind = TSR_KIND_POINTER; + reg->ok = true; + tsr_set_lifetime(reg, var); + + pr_debug_dtp("var [%"PRIx64"] reg%d addr offset %x", + insn_offset, var->reg, var->offset); + pr_debug_type_name(&mem_die, TSR_KIND_POINTER); + continue; + } + + orig_type = reg->type; + /* + * var->offset + reg value is the beginning of the struct + * reg->offset is the offset the reg points + */ + reg->offset = -var->offset; + reg->type = mem_die; + reg->kind = TSR_KIND_TYPE; + reg->ok = true; + tsr_set_lifetime(reg, var); + + pr_debug_dtp("var [%"PRIx64"] reg%d offset %x", + insn_offset, var->reg, var->offset); + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + + /* + * If this register is directly copied from another and it gets a + * better type, also update the type of the source register. This + * is usually the case of container_of() macro with offset of 0. + */ + if (has_reg_type(state, reg->copied_from)) { + struct type_state_reg *copy_reg; + + copy_reg = &state->regs[reg->copied_from]; + + /* TODO: check if type is compatible or embedded */ + if (!copy_reg->ok || (copy_reg->kind != TSR_KIND_TYPE) || + !die_is_same(©_reg->type, &orig_type) || + !is_better_type(©_reg->type, &mem_die)) + continue; + + copy_reg->type = mem_die; + + pr_debug_dtp("var [%"PRIx64"] copyback reg%d", + insn_offset, reg->copied_from); + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + } + } + } +} + +/** + * update_insn_state - Update type state for an instruction + * @state: type state table + * @dloc: data location info + * @cu_die: compile unit debug entry + * @dl: disasm line for the instruction + * + * This function updates the @state table for the target operand of the + * instruction at @dl if it transfers the type like MOV on x86. Since it + * tracks the type, it won't care about the values like in arithmetic + * instructions like ADD/SUB/MUL/DIV and INC/DEC. + * + * Note that ops->reg2 is only available when both mem_ref and multi_regs + * are true. + */ +static void update_insn_state(struct type_state *state, struct data_loc_info *dloc, + Dwarf_Die *cu_die, struct disasm_line *dl) +{ + if (dloc->arch->update_insn_state) + dloc->arch->update_insn_state(state, dloc, cu_die, dl); +} + +/* + * Prepend this_blocks (from the outer scope) to full_blocks, removing + * duplicate disasm line. + */ +static void prepend_basic_blocks(struct list_head *this_blocks, + struct list_head *full_blocks) +{ + struct annotated_basic_block *first_bb, *last_bb; + + last_bb = list_last_entry(this_blocks, typeof(*last_bb), list); + first_bb = list_first_entry(full_blocks, typeof(*first_bb), list); + + if (list_empty(full_blocks)) + goto out; + + /* Last insn in this_blocks should be same as first insn in full_blocks */ + if (last_bb->end != first_bb->begin) { + pr_debug("prepend basic blocks: mismatched disasm line %"PRIx64" -> %"PRIx64"\n", + last_bb->end->al.offset, first_bb->begin->al.offset); + goto out; + } + + /* Is the basic block have only one disasm_line? */ + if (last_bb->begin == last_bb->end) { + list_del(&last_bb->list); + free(last_bb); + goto out; + } + + /* Point to the insn before the last when adding this block to full_blocks */ + last_bb->end = list_prev_entry(last_bb->end, al.node); + +out: + list_splice(this_blocks, full_blocks); +} + +static void delete_basic_blocks(struct list_head *basic_blocks) +{ + struct annotated_basic_block *bb, *tmp; + + list_for_each_entry_safe(bb, tmp, basic_blocks, list) { + list_del(&bb->list); + free(bb); + } +} + +/* Make sure all variables have a valid start address */ +static void fixup_var_address(struct die_var_type *var_types, u64 addr) +{ + while (var_types) { + /* + * Some variables have no address range meaning it's always + * available in the whole scope. Let's adjust the start + * address to the start of the scope. + */ + if (var_types->addr == 0) + var_types->addr = addr; + + var_types = var_types->next; + } +} + +static void delete_var_types(struct die_var_type *var_types) +{ + while (var_types) { + struct die_var_type *next = var_types->next; + + free(var_types); + var_types = next; + } +} + +/* should match to is_stack_canary() in util/annotate.c */ +static void setup_stack_canary(struct data_loc_info *dloc) +{ + if (arch__is_x86(dloc->arch)) { + dloc->op->segment = INSN_SEG_X86_GS; + dloc->op->imm = true; + dloc->op->offset = 40; + } +} + +/* + * It's at the target address, check if it has a matching type. + * It returns PERF_TMR_BAIL_OUT when it looks up per-cpu variables which + * are similar to global variables and no additional info is needed. + */ +static enum type_match_result check_matching_type(struct type_state *state, + struct data_loc_info *dloc, + Dwarf_Die *cu_die, + struct disasm_line *dl, + Dwarf_Die *type_die) +{ + Dwarf_Word size; + u32 insn_offset = dl->al.offset; + int reg = dloc->op->reg1; + int offset = dloc->op->offset; + const char *offset_sign = ""; + bool retry = true; + + if (offset < 0) { + offset = -offset; + offset_sign = "-"; + } + +again: + pr_debug_dtp("chk [%x] reg%d offset=%s%#x ok=%d kind=%d ", + insn_offset, reg, offset_sign, offset, + state->regs[reg].ok, state->regs[reg].kind); + + if (!state->regs[reg].ok) + goto check_non_register; + + if (state->regs[reg].kind == TSR_KIND_TYPE) { + Dwarf_Die ptr_die; + Dwarf_Die sized_type; + Dwarf_Die *ptr_type; + struct strbuf sb; + + strbuf_init(&sb, 32); + die_get_typename_from_type(&state->regs[reg].type, &sb); + pr_debug_dtp("(%s)", sb.buf); + strbuf_release(&sb); + + /* + * Normal registers should hold a pointer (or array) to + * dereference a memory location. + */ + ptr_type = die_get_pointer_type(&state->regs[reg].type, &ptr_die); + if (!ptr_type) { + if (dloc->op->offset < 0 && reg != state->stack_reg) + goto check_kernel; + + return PERF_TMR_NO_POINTER; + } + + /* Remove the pointer and get the target type */ + if (__die_get_real_type(ptr_type, type_die) == NULL) + return PERF_TMR_NO_POINTER; + + dloc->type_offset = dloc->op->offset + state->regs[reg].offset; + + if (dwarf_tag(type_die) == DW_TAG_typedef) + die_get_real_type(type_die, &sized_type); + else + sized_type = *type_die; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(&sized_type, &size) < 0 || + (unsigned)dloc->type_offset >= size) + return PERF_TMR_BAD_OFFSET; + + return PERF_TMR_OK; + } + + if (state->regs[reg].kind == TSR_KIND_POINTER) { + struct strbuf sb; + + strbuf_init(&sb, 32); + die_get_typename_from_type(&state->regs[reg].type, &sb); + pr_debug_dtp("(ptr->%s)", sb.buf); + strbuf_release(&sb); + + /* + * Register holds a pointer (address) to the target variable. + * The type is the type of the variable it points to. + */ + *type_die = state->regs[reg].type; + + dloc->type_offset = dloc->op->offset + state->regs[reg].offset; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(type_die, &size) < 0 || + (unsigned)dloc->type_offset >= size) + return PERF_TMR_BAD_OFFSET; + + return PERF_TMR_OK; + } + + if (state->regs[reg].kind == TSR_KIND_PERCPU_POINTER) { + pr_debug_dtp("percpu ptr"); + + /* + * It's actaully pointer but the address was calculated using + * some arithmetic. So it points to the actual type already. + */ + *type_die = state->regs[reg].type; + + dloc->type_offset = dloc->op->offset; + + /* Get the size of the actual type */ + if (dwarf_aggregate_size(type_die, &size) < 0 || + (unsigned)dloc->type_offset >= size) + return PERF_TMR_BAIL_OUT; + + return PERF_TMR_OK; + } + + if (state->regs[reg].kind == TSR_KIND_CANARY) { + pr_debug_dtp("stack canary"); + + /* + * This is a saved value of the stack canary which will be handled + * in the outer logic when it returns failure here. Pretend it's + * from the stack canary directly. + */ + setup_stack_canary(dloc); + + return PERF_TMR_BAIL_OUT; + } + + if (state->regs[reg].kind == TSR_KIND_PERCPU_BASE) { + u64 var_addr = dloc->op->offset; + int var_offset; + + pr_debug_dtp("percpu var"); + + if (dloc->op->multi_regs) { + int reg2 = dloc->op->reg2; + + if (dloc->op->reg2 == reg) + reg2 = dloc->op->reg1; + + if (has_reg_type(state, reg2) && state->regs[reg2].ok && + state->regs[reg2].kind == TSR_KIND_CONST) + var_addr += state->regs[reg2].imm_value; + } + + if (get_global_var_type(cu_die, dloc, dloc->ip, var_addr, + &var_offset, type_die)) { + dloc->type_offset = var_offset; + return PERF_TMR_OK; + } + /* No need to retry per-cpu (global) variables */ + return PERF_TMR_BAIL_OUT; + } + + if (state->regs[reg].kind == TSR_KIND_CONST && + dso__kernel(map__dso(dloc->ms->map))) { + if (dloc->op->offset < 0 && reg != state->stack_reg && reg != dloc->fbreg) + goto check_kernel; + } +check_non_register: + if (reg == dloc->fbreg || reg == state->stack_reg) { + struct type_state_stack *stack; + + pr_debug_dtp("%s", reg == dloc->fbreg ? "fbreg" : "stack"); + + stack = find_stack_state(state, dloc->type_offset); + if (stack == NULL) { + if (retry) { + pr_debug_dtp(" : retry\n"); + retry = false; + + /* update type info it's the first store to the stack */ + update_insn_state(state, dloc, cu_die, dl); + goto again; + } + return PERF_TMR_NO_TYPE; + } + + if (stack->kind == TSR_KIND_CANARY) { + setup_stack_canary(dloc); + return PERF_TMR_BAIL_OUT; + } + + if (stack->kind != TSR_KIND_TYPE) + return PERF_TMR_NO_TYPE; + + *type_die = stack->type; + /* Update the type offset from the start of slot */ + dloc->type_offset -= stack->offset; + + return PERF_TMR_OK; + } + + if (dloc->fb_cfa) { + struct type_state_stack *stack; + u64 pc = map__rip_2objdump(dloc->ms->map, dloc->ip); + int fbreg, fboff; + + pr_debug_dtp("cfa"); + + if (die_get_cfa(dloc->di->dbg, pc, &fbreg, &fboff) < 0) + fbreg = -1; + + if (reg != fbreg) + return PERF_TMR_NO_TYPE; + + stack = find_stack_state(state, dloc->type_offset - fboff); + if (stack == NULL) { + if (retry) { + pr_debug_dtp(" : retry\n"); + retry = false; + + /* update type info it's the first store to the stack */ + update_insn_state(state, dloc, cu_die, dl); + goto again; + } + return PERF_TMR_NO_TYPE; + } + + if (stack->kind == TSR_KIND_CANARY) { + setup_stack_canary(dloc); + return PERF_TMR_BAIL_OUT; + } + + if (stack->kind != TSR_KIND_TYPE) + return PERF_TMR_NO_TYPE; + + *type_die = stack->type; + /* Update the type offset from the start of slot */ + dloc->type_offset -= fboff + stack->offset; + + return PERF_TMR_OK; + } + +check_kernel: + if (dso__kernel(map__dso(dloc->ms->map))) { + u64 addr; + + /* Direct this-cpu access like "%gs:0x34740" */ + if (dloc->op->segment == INSN_SEG_X86_GS && dloc->op->imm && + arch__is_x86(dloc->arch)) { + pr_debug_dtp("this-cpu var"); + + addr = dloc->op->offset; + + if (get_global_var_type(cu_die, dloc, dloc->ip, addr, + &offset, type_die)) { + dloc->type_offset = offset; + return PERF_TMR_OK; + } + return PERF_TMR_BAIL_OUT; + } + + /* Access to global variable like "-0x7dcf0500(,%rdx,8)" */ + if (dloc->op->offset < 0 && reg != state->stack_reg) { + addr = (s64) dloc->op->offset; + + if (get_global_var_type(cu_die, dloc, dloc->ip, addr, + &offset, type_die)) { + pr_debug_dtp("global var"); + + dloc->type_offset = offset; + return PERF_TMR_OK; + } + return PERF_TMR_BAIL_OUT; + } + } + + return PERF_TMR_UNKNOWN; +} + +/* Iterate instructions in basic blocks and update type table */ +static enum type_match_result find_data_type_insn(struct data_loc_info *dloc, + struct list_head *basic_blocks, + struct die_var_type *var_types, + Dwarf_Die *cu_die, + Dwarf_Die *type_die) +{ + struct type_state state; + struct symbol *sym = dloc->ms->sym; + struct annotation *notes = symbol__annotation(sym); + struct annotated_basic_block *bb; + enum type_match_result ret = PERF_TMR_UNKNOWN; + + init_type_state(&state, dloc->arch); + + list_for_each_entry(bb, basic_blocks, list) { + struct disasm_line *dl = bb->begin; + + BUG_ON(bb->begin->al.offset == -1 || bb->end->al.offset == -1); + + pr_debug_dtp("bb: [%"PRIx64" - %"PRIx64"]\n", + bb->begin->al.offset, bb->end->al.offset); + + list_for_each_entry_from(dl, ¬es->src->source, al.node) { + u64 this_ip = sym->start + dl->al.offset; + u64 addr = map__rip_2objdump(dloc->ms->map, this_ip); + + /* Skip comment or debug info lines */ + if (dl->al.offset == -1) + continue; + + /* Update variable type at this address */ + update_var_state(&state, dloc, addr, dl->al.offset, var_types); + + if (this_ip == dloc->ip) { + ret = check_matching_type(&state, dloc, + cu_die, dl, type_die); + pr_debug_dtp(" : %s\n", match_result_str(ret)); + goto out; + } + + /* Update type table after processing the instruction */ + update_insn_state(&state, dloc, cu_die, dl); + if (dl == bb->end) + break; + } + } + +out: + exit_type_state(&state); + return ret; +} + +static int arch_supports_insn_tracking(struct data_loc_info *dloc) +{ + if ((arch__is_x86(dloc->arch)) || (arch__is_powerpc(dloc->arch))) + return 1; + return 0; +} + +/* + * Construct a list of basic blocks for each scope with variables and try to find + * the data type by updating a type state table through instructions. + */ +static enum type_match_result find_data_type_block(struct data_loc_info *dloc, + Dwarf_Die *cu_die, + Dwarf_Die *scopes, + int nr_scopes, + Dwarf_Die *type_die) +{ + LIST_HEAD(basic_blocks); + struct die_var_type *var_types = NULL; + u64 src_ip, dst_ip, prev_dst_ip; + enum type_match_result ret = PERF_TMR_UNKNOWN; + + /* TODO: other architecture support */ + if (!arch_supports_insn_tracking(dloc)) + return PERF_TMR_BAIL_OUT; + + prev_dst_ip = dst_ip = dloc->ip; + for (int i = nr_scopes - 1; i >= 0; i--) { + Dwarf_Addr base, start, end; + LIST_HEAD(this_blocks); + + if (dwarf_ranges(&scopes[i], 0, &base, &start, &end) < 0) + break; + + pr_debug_dtp("scope: [%d/%d] ", i + 1, nr_scopes); + pr_debug_scope(&scopes[i]); + + src_ip = map__objdump_2rip(dloc->ms->map, start); + +again: + /* Get basic blocks for this scope */ + if (annotate_get_basic_blocks(dloc->ms->sym, src_ip, dst_ip, + &this_blocks) < 0) { + /* Try previous block if they are not connected */ + if (prev_dst_ip != dst_ip) { + dst_ip = prev_dst_ip; + goto again; + } + + pr_debug_dtp("cannot find a basic block from %"PRIx64" to %"PRIx64"\n", + src_ip - dloc->ms->sym->start, + dst_ip - dloc->ms->sym->start); + continue; + } + prepend_basic_blocks(&this_blocks, &basic_blocks); + + /* Get variable info for this scope and add to var_types list */ + die_collect_vars(&scopes[i], &var_types); + fixup_var_address(var_types, start); + + /* Find from start of this scope to the target instruction */ + ret = find_data_type_insn(dloc, &basic_blocks, var_types, + cu_die, type_die); + if (ret == PERF_TMR_OK) { + char buf[64]; + int offset = dloc->op->offset; + const char *offset_sign = ""; + + if (offset < 0) { + offset = -offset; + offset_sign = "-"; + } + + if (dloc->op->multi_regs) + snprintf(buf, sizeof(buf), "reg%d, reg%d", + dloc->op->reg1, dloc->op->reg2); + else + snprintf(buf, sizeof(buf), "reg%d", dloc->op->reg1); + + pr_debug_dtp("found by insn track: %s%#x(%s) type-offset=%#x\n", + offset_sign, offset, buf, dloc->type_offset); + break; + } + + if (ret == PERF_TMR_BAIL_OUT) + break; + + /* Go up to the next scope and find blocks to the start */ + prev_dst_ip = dst_ip; + dst_ip = src_ip; + } + + delete_basic_blocks(&basic_blocks); + delete_var_types(var_types); + return ret; +} + +/* The result will be saved in @type_die */ +static int find_data_type_die(struct data_loc_info *dloc, Dwarf_Die *type_die) +{ + struct annotated_op_loc *loc = dloc->op; + Dwarf_Die cu_die, var_die; + Dwarf_Die *scopes = NULL; + int reg, offset = loc->offset; + int ret = -1; + int i, nr_scopes; + int fbreg = -1; + int fb_offset = 0; + bool is_fbreg = false; + bool found = false; + u64 pc; + char buf[64]; + enum type_match_result result = PERF_TMR_UNKNOWN; + const char *offset_sign = ""; + + if (dloc->op->multi_regs) + snprintf(buf, sizeof(buf), "reg%d, reg%d", dloc->op->reg1, dloc->op->reg2); + else if (dloc->op->reg1 == DWARF_REG_PC) + snprintf(buf, sizeof(buf), "PC"); + else + snprintf(buf, sizeof(buf), "reg%d", dloc->op->reg1); + + if (offset < 0) { + offset = -offset; + offset_sign = "-"; + } + + pr_debug_dtp("-----------------------------------------------------------\n"); + pr_debug_dtp("find data type for %s%#x(%s) at %s+%#"PRIx64"\n", + offset_sign, offset, buf, + dloc->ms->sym->name, dloc->ip - dloc->ms->sym->start); + + /* + * IP is a relative instruction address from the start of the map, as + * it can be randomized/relocated, it needs to translate to PC which is + * a file address for DWARF processing. + */ + pc = map__rip_2objdump(dloc->ms->map, dloc->ip); + + /* Get a compile_unit for this address */ + if (!find_cu_die(dloc->di, pc, &cu_die)) { + pr_debug_dtp("cannot find CU for address %"PRIx64"\n", pc); + ann_data_stat.no_cuinfo++; + return -1; + } + + reg = loc->reg1; + offset = loc->offset; + + pr_debug_dtp("CU for %s (die:%#lx)\n", + die_name(&cu_die), (long)dwarf_dieoffset(&cu_die)); + + if (reg == DWARF_REG_PC) { + if (get_global_var_type(&cu_die, dloc, dloc->ip, dloc->var_addr, + &offset, type_die)) { + dloc->type_offset = offset; + + pr_debug_dtp("found by addr=%#"PRIx64" type_offset=%#x\n", + dloc->var_addr, offset); + pr_debug_type_name(type_die, TSR_KIND_TYPE); + found = true; + goto out; + } + } + + /* Get a list of nested scopes - i.e. (inlined) functions and blocks. */ + nr_scopes = die_get_scopes(&cu_die, pc, &scopes); + + if (reg != DWARF_REG_PC && dwarf_hasattr(&scopes[0], DW_AT_frame_base)) { + Dwarf_Attribute attr; + Dwarf_Block block; + + /* Check if the 'reg' is assigned as frame base register */ + if (dwarf_attr(&scopes[0], DW_AT_frame_base, &attr) != NULL && + dwarf_formblock(&attr, &block) == 0 && block.length == 1) { + switch (*block.data) { + case DW_OP_reg0 ... DW_OP_reg31: + fbreg = dloc->fbreg = *block.data - DW_OP_reg0; + break; + case DW_OP_call_frame_cfa: + dloc->fb_cfa = true; + if (die_get_cfa(dloc->di->dbg, pc, &fbreg, + &fb_offset) < 0) + fbreg = -1; + break; + default: + break; + } + + pr_debug_dtp("frame base: cfa=%d fbreg=%d\n", + dloc->fb_cfa, fbreg); + } + } + +retry: + is_fbreg = (reg == fbreg); + if (is_fbreg) + offset = loc->offset - fb_offset; + + /* Search from the inner-most scope to the outer */ + for (i = nr_scopes - 1; i >= 0; i--) { + Dwarf_Die mem_die; + int type_offset = offset; + + if (reg == DWARF_REG_PC) { + if (!die_find_variable_by_addr(&scopes[i], dloc->var_addr, + &var_die, &mem_die, + &type_offset)) + continue; + } else { + /* Look up variables/parameters in this scope */ + if (!die_find_variable_by_reg(&scopes[i], pc, reg, + &mem_die, &type_offset, is_fbreg, &var_die)) + continue; + } + + pr_debug_dtp("found \"%s\" (die: %#lx) in scope=%d/%d (die: %#lx) ", + die_name(&var_die), (long)dwarf_dieoffset(&var_die), + i+1, nr_scopes, (long)dwarf_dieoffset(&scopes[i])); + + if (reg == DWARF_REG_PC) { + pr_debug_dtp("addr=%#"PRIx64" type_offset=%#x\n", + dloc->var_addr, type_offset); + } else if (reg == DWARF_REG_FB || is_fbreg) { + pr_debug_dtp("stack_offset=%#x type_offset=%#x\n", + fb_offset, type_offset); + } else { + pr_debug_dtp("type_offset=%#x\n", type_offset); + } + + if (!found || dloc->type_offset < type_offset || + (dloc->type_offset == type_offset && + !is_better_type(&mem_die, type_die))) { + *type_die = mem_die; + dloc->type_offset = type_offset; + found = true; + } + + pr_debug_location(&var_die, pc, reg); + pr_debug_type_name(&mem_die, TSR_KIND_TYPE); + } + + if (!found && loc->multi_regs && reg == loc->reg1 && loc->reg1 != loc->reg2) { + reg = loc->reg2; + goto retry; + } + + if (!found && reg != DWARF_REG_PC) { + result = find_data_type_block(dloc, &cu_die, scopes, + nr_scopes, type_die); + if (result == PERF_TMR_OK) { + ann_data_stat.insn_track++; + found = true; + } + } + +out: + pr_debug_dtp("final result: "); + if (found) { + pr_debug_type_name(type_die, TSR_KIND_TYPE); + ret = 0; + } else { + switch (result) { + case PERF_TMR_NO_TYPE: + case PERF_TMR_NO_POINTER: + pr_debug_dtp("%s\n", match_result_str(result)); + ann_data_stat.no_typeinfo++; + break; + case PERF_TMR_NO_SIZE: + pr_debug_dtp("%s\n", match_result_str(result)); + ann_data_stat.invalid_size++; + break; + case PERF_TMR_BAD_OFFSET: + pr_debug_dtp("%s\n", match_result_str(result)); + ann_data_stat.bad_offset++; + break; + case PERF_TMR_UNKNOWN: + case PERF_TMR_BAIL_OUT: + case PERF_TMR_OK: /* should not reach here */ + default: + pr_debug_dtp("no variable found\n"); + ann_data_stat.no_var++; + break; + } + ret = -1; + } + + free(scopes); + return ret; +} + +/** + * find_data_type - Return a data type at the location + * @dloc: data location + * + * This functions searches the debug information of the binary to get the data + * type it accesses. The exact location is expressed by (ip, reg, offset) + * for pointer variables or (ip, addr) for global variables. Note that global + * variables might update the @dloc->type_offset after finding the start of the + * variable. If it cannot find a global variable by address, it tried to find + * a declaration of the variable using var_name. In that case, @dloc->offset + * won't be updated. + * + * It return %NULL if not found. + */ +struct annotated_data_type *find_data_type(struct data_loc_info *dloc) +{ + struct dso *dso = map__dso(dloc->ms->map); + Dwarf_Die type_die; + + /* + * The type offset is the same as instruction offset by default. + * But when finding a global variable, the offset won't be valid. + */ + dloc->type_offset = dloc->op->offset; + + dloc->fbreg = -1; + + if (find_data_type_die(dloc, &type_die) < 0) + return NULL; + + return dso__findnew_data_type(dso, &type_die); +} + +static int alloc_data_type_histograms(struct annotated_data_type *adt, int nr_entries) +{ + int i; + size_t sz = sizeof(struct type_hist); + + sz += sizeof(struct type_hist_entry) * adt->self.size; + + /* Allocate a table of pointers for each event */ + adt->histograms = calloc(nr_entries, sizeof(*adt->histograms)); + if (adt->histograms == NULL) + return -ENOMEM; + + /* + * Each histogram is allocated for the whole size of the type. + * TODO: Probably we can move the histogram to members. + */ + for (i = 0; i < nr_entries; i++) { + adt->histograms[i] = zalloc(sz); + if (adt->histograms[i] == NULL) + goto err; + } + + adt->nr_histograms = nr_entries; + return 0; + +err: + while (--i >= 0) + zfree(&(adt->histograms[i])); + zfree(&adt->histograms); + return -ENOMEM; +} + +static void delete_data_type_histograms(struct annotated_data_type *adt) +{ + for (int i = 0; i < adt->nr_histograms; i++) + zfree(&(adt->histograms[i])); + + zfree(&adt->histograms); + adt->nr_histograms = 0; +} + +void annotated_data_type__tree_delete(struct rb_root *root) +{ + struct annotated_data_type *pos; + + while (!RB_EMPTY_ROOT(root)) { + struct rb_node *node = rb_first(root); + + rb_erase(node, root); + pos = rb_entry(node, struct annotated_data_type, node); + delete_members(&pos->self); + delete_data_type_histograms(pos); + zfree(&pos->self.type_name); + free(pos); + } +} + +/** + * annotated_data_type__update_samples - Update histogram + * @adt: Data type to update + * @evsel: Event to update + * @offset: Offset in the type + * @nr_samples: Number of samples at this offset + * @period: Event count at this offset + * + * This function updates type histogram at @ofs for @evsel. Samples are + * aggregated before calling this function so it can be called with more + * than one samples at a certain offset. + */ +int annotated_data_type__update_samples(struct annotated_data_type *adt, + struct evsel *evsel, int offset, + int nr_samples, u64 period) +{ + struct type_hist *h; + + if (adt == NULL) + return 0; + + if (adt->histograms == NULL) { + int nr = evsel->evlist->core.nr_entries; + + if (alloc_data_type_histograms(adt, nr) < 0) + return -1; + } + + if (offset < 0 || offset >= adt->self.size) + return -1; + + h = adt->histograms[evsel->core.idx]; + + h->nr_samples += nr_samples; + h->addr[offset].nr_samples += nr_samples; + h->period += period; + h->addr[offset].period += period; + return 0; +} + +static void print_annotated_data_header(struct hist_entry *he, struct evsel *evsel) +{ + struct dso *dso = map__dso(he->ms.map); + int nr_members = 1; + int nr_samples = he->stat.nr_events; + int width = 7; + const char *val_hdr = "Percent"; + + if (evsel__is_group_event(evsel)) { + struct hist_entry *pair; + + list_for_each_entry(pair, &he->pairs.head, pairs.node) + nr_samples += pair->stat.nr_events; + } + + printf("Annotate type: '%s' in %s (%d samples):\n", + he->mem_type->self.type_name, dso__name(dso), nr_samples); + + if (evsel__is_group_event(evsel)) { + struct evsel *pos; + int i = 0; + + nr_members = 0; + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + + printf(" event[%d] = %s\n", i++, pos->name); + nr_members++; + } + } + + if (symbol_conf.show_total_period) { + width = 11; + val_hdr = "Period"; + } else if (symbol_conf.show_nr_samples) { + width = 7; + val_hdr = "Samples"; + } + + printf("============================================================================\n"); + printf("%*s %10s %10s %s\n", (width + 1) * nr_members, val_hdr, + "offset", "size", "field"); +} + +static void print_annotated_data_value(struct type_hist *h, u64 period, int nr_samples) +{ + double percent = h->period ? (100.0 * period / h->period) : 0; + const char *color = get_percent_color(percent); + + if (symbol_conf.show_total_period) + color_fprintf(stdout, color, " %11" PRIu64, period); + else if (symbol_conf.show_nr_samples) + color_fprintf(stdout, color, " %7d", nr_samples); + else + color_fprintf(stdout, color, " %7.2f", percent); +} + +static void print_annotated_data_type(struct annotated_data_type *mem_type, + struct annotated_member *member, + struct evsel *evsel, int indent) +{ + struct annotated_member *child; + struct type_hist *h = mem_type->histograms[evsel->core.idx]; + int i, nr_events = 0, samples = 0; + u64 period = 0; + int width = symbol_conf.show_total_period ? 11 : 7; + struct evsel *pos; + + for_each_group_evsel(pos, evsel) { + h = mem_type->histograms[pos->core.idx]; + + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + + samples = 0; + period = 0; + for (i = 0; i < member->size; i++) { + samples += h->addr[member->offset + i].nr_samples; + period += h->addr[member->offset + i].period; + } + print_annotated_data_value(h, period, samples); + nr_events++; + } + + printf(" %#10x %#10x %*s%s\t%s", + member->offset, member->size, indent, "", member->type_name, + member->var_name ?: ""); + + if (!list_empty(&member->children)) + printf(" {\n"); + + list_for_each_entry(child, &member->children, node) + print_annotated_data_type(mem_type, child, evsel, indent + 4); + + if (!list_empty(&member->children)) + printf("%*s}", (width + 1) * nr_events + 24 + indent, ""); + printf(";\n"); +} + +int hist_entry__annotate_data_tty(struct hist_entry *he, struct evsel *evsel) +{ + print_annotated_data_header(he, evsel); + print_annotated_data_type(he->mem_type, &he->mem_type->self, evsel, 0); + printf("\n"); + + /* move to the next entry */ + return '>'; +} diff --git a/tools/perf/util/annotate-data.h b/tools/perf/util/annotate-data.h new file mode 100644 index 000000000..c26130744 --- /dev/null +++ b/tools/perf/util/annotate-data.h @@ -0,0 +1,322 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _PERF_ANNOTATE_DATA_H +#define _PERF_ANNOTATE_DATA_H + +#include +#include +#include +#include +#include "dwarf-regs.h" +#include "annotate.h" + +#ifdef HAVE_LIBDW_SUPPORT +#include "debuginfo.h" +#endif + +struct annotated_op_loc; +struct debuginfo; +struct evsel; +struct hist_browser_timer; +struct hist_entry; +struct map_symbol; +struct thread; + +#define pr_debug_dtp(fmt, ...) \ +do { \ + if (debug_type_profile) \ + pr_info(fmt, ##__VA_ARGS__); \ + else \ + pr_debug3(fmt, ##__VA_ARGS__); \ +} while (0) + +enum type_state_kind { + TSR_KIND_INVALID = 0, + TSR_KIND_TYPE, + TSR_KIND_PERCPU_BASE, + TSR_KIND_CONST, + TSR_KIND_PERCPU_POINTER, + TSR_KIND_POINTER, + TSR_KIND_CANARY, +}; + +/** + * struct annotated_member - Type of member field + * @node: List entry in the parent list + * @children: List head for child nodes + * @type_name: Name of the member type + * @var_name: Name of the member variable + * @offset: Offset from the outer data type + * @size: Size of the member field + * + * This represents a member type in a data type. + */ +struct annotated_member { + struct list_head node; + struct list_head children; + char *type_name; + char *var_name; + int offset; + int size; +}; + +/** + * struct type_hist_entry - Histogram entry per offset + * @nr_samples: Number of samples + * @period: Count of event + */ +struct type_hist_entry { + int nr_samples; + u64 period; +}; + +/** + * struct type_hist - Type histogram for each event + * @nr_samples: Total number of samples in this data type + * @period: Total count of the event in this data type + * @offset: Array of histogram entry + */ +struct type_hist { + u64 nr_samples; + u64 period; + struct type_hist_entry addr[]; +}; + +/** + * struct annotated_data_type - Data type to profile + * @node: RB-tree node for dso->type_tree + * @self: Actual type information + * @nr_histogram: Number of histogram entries + * @histograms: An array of pointers to histograms + * + * This represents a data type accessed by samples in the profile data. + */ +struct annotated_data_type { + struct rb_node node; + struct annotated_member self; + int nr_histograms; + struct type_hist **histograms; +}; + +extern struct annotated_data_type unknown_type; +extern struct annotated_data_type stackop_type; +extern struct annotated_data_type canary_type; + +/** + * struct data_loc_info - Data location information + * @arch: CPU architecture info + * @thread: Thread info + * @ms: Map and Symbol info + * @ip: Instruction address + * @var_addr: Data address (for global variables) + * @cpumode: CPU execution mode + * @op: Instruction operand location (regs and offset) + * @di: Debug info + * @fbreg: Frame base register + * @fb_cfa: Whether the frame needs to check CFA + * @type_offset: Final offset in the type + */ +struct data_loc_info { + /* These are input field, should be filled by caller */ + const struct arch *arch; + struct thread *thread; + struct map_symbol *ms; + u64 ip; + u64 var_addr; + u8 cpumode; + struct annotated_op_loc *op; + struct debuginfo *di; + + /* These are used internally */ + int fbreg; + bool fb_cfa; + + /* This is for the result */ + int type_offset; +}; + +/** + * struct annotated_data_stat - Debug statistics + * @total: Total number of entry + * @no_sym: No symbol or map found + * @no_insn: Failed to get disasm line + * @no_insn_ops: The instruction has no operands + * @no_mem_ops: The instruction has no memory operands + * @no_reg: Failed to extract a register from the operand + * @no_dbginfo: The binary has no debug information + * @no_cuinfo: Failed to find a compile_unit + * @no_var: Failed to find a matching variable + * @no_typeinfo: Failed to get a type info for the variable + * @invalid_size: Failed to get a size info of the type + * @bad_offset: The access offset is out of the type + */ +struct annotated_data_stat { + int total; + int no_sym; + int no_insn; + int no_insn_ops; + int no_mem_ops; + int no_reg; + int no_dbginfo; + int no_cuinfo; + int no_var; + int no_typeinfo; + int invalid_size; + int bad_offset; + int insn_track; +}; +extern struct annotated_data_stat ann_data_stat; + +#ifdef HAVE_LIBDW_SUPPORT +/* + * Type information in a register, valid when @ok is true. + * The @caller_saved registers are invalidated after a function call. + */ +struct type_state_reg { + Dwarf_Die type; + u32 imm_value; + /* + * The offset within the struct that the register points to. + * A value of 0 means the register points to the beginning. + * type_offset = op->offset + reg->offset + */ + s32 offset; + bool ok; + bool caller_saved; + /* DWARF location range tracking for register lifetime */ + bool lifetime_active; + u64 lifetime_end; + u8 kind; + u8 copied_from; +}; + +/* Type information in a stack location, dynamically allocated */ +struct type_state_stack { + struct list_head list; + Dwarf_Die type; + int offset; + /* pointer offset, saves tsr->offset on the stack state */ + int ptr_offset; + int size; + bool compound; + u8 kind; +}; + +/* + * Maximum number of registers tracked in type_state. + * + * This limit must cover all supported architectures, since perf + * may analyze perf.data files generated on systems with a different + * register set. Use 32 as a safe upper bound instead of relying on + * build-arch specific values. + */ +#define TYPE_STATE_MAX_REGS 32 + +/* + * State table to maintain type info in each register and stack location. + * It'll be updated when new variable is allocated or type info is moved + * to a new location (register or stack). As it'd be used with the + * shortest path of basic blocks, it only maintains a single table. + */ +struct type_state { + /* state of general purpose registers */ + struct type_state_reg regs[TYPE_STATE_MAX_REGS]; + /* state of stack location */ + struct list_head stack_vars; + /* return value register */ + int ret_reg; + /* stack pointer register */ + int stack_reg; +}; + +/* Returns data type at the location (ip, reg, offset) */ +struct annotated_data_type *find_data_type(struct data_loc_info *dloc); + +/* Update type access histogram at the given offset */ +int annotated_data_type__update_samples(struct annotated_data_type *adt, + struct evsel *evsel, int offset, + int nr_samples, u64 period); + +/* Release all data type information in the tree */ +void annotated_data_type__tree_delete(struct rb_root *root); + +/* Release all global variable information in the tree */ +void global_var_type__tree_delete(struct rb_root *root); + +/* Print data type annotation (including members) on stdout */ +int hist_entry__annotate_data_tty(struct hist_entry *he, struct evsel *evsel); + +/* Get name of member field at the given offset in the data type */ +int annotated_data_type__get_member_name(struct annotated_data_type *adt, + char *buf, size_t sz, int member_offset); + +bool has_reg_type(struct type_state *state, int reg); +struct type_state_stack *findnew_stack_state(struct type_state *state, + int offset, u8 kind, + Dwarf_Die *type_die, + int ptr_offset); +void set_stack_state(struct type_state_stack *stack, int offset, u8 kind, + Dwarf_Die *type_die, int ptr_offset); +struct type_state_stack *find_stack_state(struct type_state *state, + int offset); +bool get_global_var_type(Dwarf_Die *cu_die, struct data_loc_info *dloc, + u64 ip, u64 var_addr, int *var_offset, + Dwarf_Die *type_die); +bool get_global_var_info(struct data_loc_info *dloc, u64 addr, + const char **var_name, int *var_offset); +void pr_debug_type_name(Dwarf_Die *die, enum type_state_kind kind); + +#else /* HAVE_LIBDW_SUPPORT */ + +static inline struct annotated_data_type * +find_data_type(struct data_loc_info *dloc __maybe_unused) +{ + return NULL; +} + +static inline int +annotated_data_type__update_samples(struct annotated_data_type *adt __maybe_unused, + struct evsel *evsel __maybe_unused, + int offset __maybe_unused, + int nr_samples __maybe_unused, + u64 period __maybe_unused) +{ + return -1; +} + +static inline void annotated_data_type__tree_delete(struct rb_root *root __maybe_unused) +{ +} + +static inline void global_var_type__tree_delete(struct rb_root *root __maybe_unused) +{ +} + +static inline int hist_entry__annotate_data_tty(struct hist_entry *he __maybe_unused, + struct evsel *evsel __maybe_unused) +{ + return -1; +} + +static inline int annotated_data_type__get_member_name(struct annotated_data_type *adt __maybe_unused, + char *buf __maybe_unused, + size_t sz __maybe_unused, + int member_offset __maybe_unused) +{ + return -1; +} + +#endif /* HAVE_LIBDW_SUPPORT */ + +#ifdef HAVE_SLANG_SUPPORT +int hist_entry__annotate_data_tui(struct hist_entry *he, struct evsel *evsel, + struct hist_browser_timer *hbt); +#else +static inline int hist_entry__annotate_data_tui(struct hist_entry *he __maybe_unused, + struct evsel *evsel __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused) +{ + return -1; +} +#endif /* HAVE_SLANG_SUPPORT */ + +#endif /* _PERF_ANNOTATE_DATA_H */ diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c new file mode 100644 index 000000000..02505222d --- /dev/null +++ b/tools/perf/util/annotate.c @@ -0,0 +1,3205 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo + * + * Parts came from builtin-annotate.c, see those files for further + * copyright notes. + */ + +#include +#include +#include +#include "util.h" // hex_width() +#include "ui/ui.h" +#include "sort.h" +#include "build-id.h" +#include "color.h" +#include "config.h" +#include "disasm.h" +#include "dso.h" +#include "env.h" +#include "map.h" +#include "maps.h" +#include "symbol.h" +#include "srcline.h" +#include "units.h" +#include "debug.h" +#include "debuginfo.h" +#include "annotate.h" +#include "annotate-data.h" +#include "evsel.h" +#include "evlist.h" +#include "bpf-event.h" +#include "bpf-utils.h" +#include "block-range.h" +#include "string2.h" +#include "dwarf-regs.h" +#include "util/event.h" +#include "util/sharded_mutex.h" +#include "arch/common.h" +#include "namespaces.h" +#include "thread.h" +#include "hashmap.h" +#include "strbuf.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* FIXME: For the HE_COLORSET */ +#include "ui/browser.h" + +/* + * FIXME: Using the same values as slang.h, + * but that header may not be available everywhere + */ +#define LARROW_CHAR ((unsigned char)',') +#define RARROW_CHAR ((unsigned char)'+') +#define DARROW_CHAR ((unsigned char)'.') +#define UARROW_CHAR ((unsigned char)'-') + +#include + +/* global annotation options */ +struct annotation_options annotate_opts; + +/* Data type collection debug statistics */ +struct annotated_data_stat ann_data_stat; +LIST_HEAD(ann_insn_stat); + +/* Pseudo data types */ +struct annotated_data_type stackop_type = { + .self = { + .type_name = (char *)"(stack operation)", + .children = LIST_HEAD_INIT(stackop_type.self.children), + }, +}; + +struct annotated_data_type canary_type = { + .self = { + .type_name = (char *)"(stack canary)", + .children = LIST_HEAD_INIT(canary_type.self.children), + }, +}; + +#define NO_TYPE ((struct annotated_data_type *)-1UL) + +/* symbol histogram: key = offset << 16 | evsel->core.idx */ +static size_t sym_hist_hash(long key, void *ctx __maybe_unused) +{ + return (key >> 16) + (key & 0xffff); +} + +static bool sym_hist_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +static struct annotated_source *annotated_source__new(void) +{ + struct annotated_source *src = zalloc(sizeof(*src)); + + if (src != NULL) + INIT_LIST_HEAD(&src->source); + + return src; +} + +static __maybe_unused void annotated_source__delete(struct annotated_source *src) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (src == NULL) + return; + + if (src->samples) { + hashmap__for_each_entry(src->samples, cur, bkt) + zfree(&cur->pvalue); + hashmap__free(src->samples); + } + zfree(&src->histograms); + free(src); +} + +static int annotated_source__alloc_histograms(struct annotated_source *src, + int nr_hists) +{ + src->nr_histograms = nr_hists; + src->histograms = calloc(nr_hists, sizeof(*src->histograms)); + + if (src->histograms == NULL) + return -1; + + src->samples = hashmap__new(sym_hist_hash, sym_hist_equal, NULL); + if (IS_ERR(src->samples)) { + zfree(&src->histograms); + src->samples = NULL; + } + + return src->histograms ? 0 : -1; +} + +void symbol__annotate_zero_histograms(struct symbol *sym) +{ + struct annotation *notes = symbol__annotation(sym); + + annotation__lock(notes); + if (notes->src != NULL) { + memset(notes->src->histograms, 0, + notes->src->nr_histograms * sizeof(*notes->src->histograms)); + hashmap__clear(notes->src->samples); + } + if (notes->branch && notes->branch->cycles_hist) { + memset(notes->branch->cycles_hist, 0, + symbol__size(sym) * sizeof(struct cyc_hist)); + } + annotation__unlock(notes); +} + +static int __symbol__account_cycles(struct cyc_hist *ch, + u64 start, + unsigned offset, unsigned cycles, + unsigned have_start) +{ + /* + * For now we can only account one basic block per + * final jump. But multiple could be overlapping. + * Always account the longest one. So when + * a shorter one has been already seen throw it away. + * + * We separately always account the full cycles. + */ + ch[offset].num_aggr++; + ch[offset].cycles_aggr += cycles; + + if (cycles > ch[offset].cycles_max) + ch[offset].cycles_max = cycles; + + if (ch[offset].cycles_min) { + if (cycles && cycles < ch[offset].cycles_min) + ch[offset].cycles_min = cycles; + } else + ch[offset].cycles_min = cycles; + + if (!have_start && ch[offset].have_start) + return 0; + if (ch[offset].num) { + if (have_start && (!ch[offset].have_start || + ch[offset].start > start)) { + ch[offset].have_start = 0; + ch[offset].cycles = 0; + ch[offset].num = 0; + if (ch[offset].reset < 0xffff) + ch[offset].reset++; + } else if (have_start && + ch[offset].start < start) + return 0; + } + + if (ch[offset].num < NUM_SPARKS) + ch[offset].cycles_spark[ch[offset].num] = cycles; + + ch[offset].have_start = have_start; + ch[offset].start = start; + ch[offset].cycles += cycles; + ch[offset].num++; + return 0; +} + +static int __symbol__inc_addr_samples(struct map_symbol *ms, + struct annotated_source *src, u64 addr, + struct perf_sample *sample) +{ + struct evsel *evsel = sample->evsel; + struct symbol *sym = ms->sym; + long hash_key; + u64 offset; + struct sym_hist *h; + struct sym_hist_entry *entry; + + pr_debug3("%s: addr=%#" PRIx64 "\n", __func__, map__unmap_ip(ms->map, addr)); + + if ((addr < sym->start || addr >= sym->end) && + (addr != sym->end || sym->start != sym->end)) { + pr_debug("%s(%d): ERANGE! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 "\n", + __func__, __LINE__, sym->name, sym->start, addr, sym->end); + return -ERANGE; + } + + offset = addr - sym->start; + h = annotated_source__histogram(src, evsel); + if (h == NULL) { + pr_debug("%s(%d): ENOMEM! sym->name=%s, start=%#" PRIx64 ", addr=%#" PRIx64 ", end=%#" PRIx64 ", func: %d\n", + __func__, __LINE__, sym->name, sym->start, addr, sym->end, + symbol__type(sym) == STT_FUNC); + return -ENOMEM; + } + + hash_key = offset << 16 | evsel->core.idx; + if (!hashmap__find(src->samples, hash_key, &entry)) { + entry = zalloc(sizeof(*entry)); + if (entry == NULL) + return -ENOMEM; + + if (hashmap__add(src->samples, hash_key, entry) < 0) + return -ENOMEM; + } + + h->nr_samples++; + h->period += sample->period; + entry->nr_samples++; + entry->period += sample->period; + + pr_debug3("%#" PRIx64 " %s: period++ [addr: %#" PRIx64 ", %#" PRIx64 + ", evidx=%d] => nr_samples: %" PRIu64 ", period: %" PRIu64 "\n", + sym->start, sym->name, addr, addr - sym->start, evsel->core.idx, + entry->nr_samples, entry->period); + return 0; +} + +struct annotated_branch *annotation__get_branch(struct annotation *notes) +{ + if (notes == NULL) + return NULL; + + if (notes->branch == NULL) + notes->branch = zalloc(sizeof(*notes->branch)); + + return notes->branch; +} + +static struct annotated_branch *symbol__find_branch_hist(struct symbol *sym, + unsigned int br_cntr_nr) +{ + struct annotation *notes = symbol__annotation(sym); + struct annotated_branch *branch; + const size_t size = symbol__size(sym); + + branch = annotation__get_branch(notes); + if (branch == NULL) + return NULL; + + if (branch->cycles_hist == NULL) { + branch->cycles_hist = calloc(size, sizeof(struct cyc_hist)); + if (!branch->cycles_hist) + return NULL; + } + + if (br_cntr_nr && branch->br_cntr == NULL) { + branch->br_cntr = calloc(br_cntr_nr * size, sizeof(u64)); + if (!branch->br_cntr) + return NULL; + } + + return branch; +} + +struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists) +{ + struct annotation *notes = symbol__annotation(sym); + + if (notes->src == NULL) { + notes->src = annotated_source__new(); + if (notes->src == NULL) + return NULL; + goto alloc_histograms; + } + + if (notes->src->histograms == NULL) { +alloc_histograms: + annotated_source__alloc_histograms(notes->src, nr_hists); + } + + return notes->src; +} + +static int symbol__inc_addr_samples(struct map_symbol *ms, + u64 addr, + struct perf_sample *sample) +{ + struct symbol *sym = ms->sym; + struct annotated_source *src; + + if (sym == NULL) + return 0; + src = symbol__hists(sym, sample->evsel->evlist->core.nr_entries); + return src ? __symbol__inc_addr_samples(ms, src, addr, sample) : 0; +} + +static int symbol__account_br_cntr(struct annotated_branch *branch, + struct evsel *evsel, + unsigned offset, + u64 br_cntr) +{ + unsigned int br_cntr_nr = evsel__leader(evsel)->br_cntr_nr; + unsigned int base = evsel__leader(evsel)->br_cntr_idx; + unsigned int off = offset * evsel->evlist->nr_br_cntr; + u64 *branch_br_cntr = branch->br_cntr; + unsigned int i, mask, width; + + if (!br_cntr || !branch_br_cntr) + return 0; + + perf_env__find_br_cntr_info(evsel__env(evsel), NULL, &width); + mask = (1L << width) - 1; + for (i = 0; i < br_cntr_nr; i++) { + u64 cntr = (br_cntr >> i * width) & mask; + + branch_br_cntr[off + i + base] += cntr; + if (cntr == mask) + branch_br_cntr[off + i + base] |= ANNOTATION__BR_CNTR_SATURATED_FLAG; + } + + return 0; +} + +static int symbol__account_cycles(u64 addr, u64 start, struct symbol *sym, + unsigned cycles, struct evsel *evsel, + u64 br_cntr) +{ + struct annotated_branch *branch; + unsigned offset; + int ret; + + if (sym == NULL) + return 0; + branch = symbol__find_branch_hist(sym, evsel->evlist->nr_br_cntr); + if (!branch) + return -ENOMEM; + if (addr < sym->start || addr >= sym->end) + return -ERANGE; + + if (start) { + if (start < sym->start || start >= sym->end) + return -ERANGE; + if (start >= addr) + start = 0; + } + offset = addr - sym->start; + ret = __symbol__account_cycles(branch->cycles_hist, + start ? start - sym->start : 0, + offset, cycles, + !!start); + + if (ret) + return ret; + + return symbol__account_br_cntr(branch, evsel, offset, br_cntr); +} + +int addr_map_symbol__account_cycles(struct addr_map_symbol *ams, + struct addr_map_symbol *start, + unsigned cycles, + struct evsel *evsel, + u64 br_cntr) +{ + u64 saddr = 0; + int err; + + if (!cycles) + return 0; + + /* + * Only set start when IPC can be computed. We can only + * compute it when the basic block is completely in a single + * function. + * Special case the case when the jump is elsewhere, but + * it starts on the function start. + */ + if (start && + (start->ms.sym == ams->ms.sym || + (ams->ms.sym && + start->addr == ams->ms.sym->start + map__start(ams->ms.map)))) + saddr = start->al_addr; + if (saddr == 0) + pr_debug2("BB with bad start: addr %"PRIx64" start %"PRIx64" sym %"PRIx64" saddr %"PRIx64"\n", + ams->addr, + start ? start->addr : 0, + ams->ms.sym ? ams->ms.sym->start + map__start(ams->ms.map) : 0, + saddr); + err = symbol__account_cycles(ams->al_addr, saddr, ams->ms.sym, cycles, evsel, br_cntr); + if (err) + pr_debug2("account_cycles failed %d\n", err); + return err; +} + +struct annotation_line *annotated_source__get_line(struct annotated_source *src, + s64 offset) +{ + struct annotation_line *al; + + list_for_each_entry(al, &src->source, node) { + if (al->offset == offset) + return al; + } + return NULL; +} + +static unsigned annotation__count_insn(struct annotation *notes, u64 start, u64 end) +{ + struct annotation_line *al; + unsigned n_insn = 0; + + al = annotated_source__get_line(notes->src, start); + if (al == NULL) + return 0; + + list_for_each_entry_from(al, ¬es->src->source, node) { + if (al->offset == -1) + continue; + if ((u64)al->offset > end) + break; + n_insn++; + } + return n_insn; +} + +static void annotated_branch__delete(struct annotated_branch *branch) +{ + if (branch) { + zfree(&branch->cycles_hist); + free(branch->br_cntr); + free(branch); + } +} + +static void annotation__count_and_fill(struct annotation *notes, u64 start, u64 end, struct cyc_hist *ch) +{ + unsigned n_insn; + unsigned int cover_insn = 0; + + n_insn = annotation__count_insn(notes, start, end); + if (n_insn && ch->num && ch->cycles) { + struct annotation_line *al; + struct annotated_branch *branch; + float ipc = n_insn / ((double)ch->cycles / (double)ch->num); + + /* Hide data when there are too many overlaps. */ + if (ch->reset >= 0x7fff) + return; + + al = annotated_source__get_line(notes->src, start); + if (al == NULL) + return; + + list_for_each_entry_from(al, ¬es->src->source, node) { + if (al->offset == -1) + continue; + if ((u64)al->offset > end) + break; + if (al->cycles && al->cycles->ipc == 0.0) { + al->cycles->ipc = ipc; + cover_insn++; + } + } + + branch = annotation__get_branch(notes); + if (cover_insn && branch) { + branch->hit_cycles += ch->cycles; + branch->hit_insn += n_insn * ch->num; + branch->cover_insn += cover_insn; + } + } +} + +static int annotation__compute_ipc(struct annotation *notes, size_t size, + struct evsel *evsel) +{ + unsigned int br_cntr_nr = evsel->evlist->nr_br_cntr; + int err = 0; + s64 offset; + + if (!notes->branch || !notes->branch->cycles_hist) + return 0; + + notes->branch->total_insn = annotation__count_insn(notes, 0, size - 1); + notes->branch->hit_cycles = 0; + notes->branch->hit_insn = 0; + notes->branch->cover_insn = 0; + + annotation__lock(notes); + for (offset = size - 1; offset >= 0; --offset) { + struct cyc_hist *ch; + + ch = ¬es->branch->cycles_hist[offset]; + if (ch && ch->cycles) { + struct annotation_line *al; + + al = annotated_source__get_line(notes->src, offset); + if (al && al->cycles == NULL) { + al->cycles = zalloc(sizeof(*al->cycles)); + if (al->cycles == NULL) { + err = ENOMEM; + break; + } + } + if (ch->have_start) + annotation__count_and_fill(notes, ch->start, offset, ch); + if (al && ch->num_aggr) { + al->cycles->avg = ch->cycles_aggr / ch->num_aggr; + al->cycles->max = ch->cycles_max; + al->cycles->min = ch->cycles_min; + } + if (al && notes->branch->br_cntr) { + if (!al->br_cntr) { + al->br_cntr = calloc(br_cntr_nr, sizeof(u64)); + if (!al->br_cntr) { + err = ENOMEM; + break; + } + } + al->num_aggr = ch->num_aggr; + al->br_cntr_nr = br_cntr_nr; + al->evsel = evsel; + memcpy(al->br_cntr, ¬es->branch->br_cntr[offset * br_cntr_nr], + br_cntr_nr * sizeof(u64)); + } + } + } + + if (err) { + while (++offset < (s64)size) { + struct cyc_hist *ch = ¬es->branch->cycles_hist[offset]; + + if (ch && ch->cycles) { + struct annotation_line *al; + + al = annotated_source__get_line(notes->src, offset); + if (al) { + zfree(&al->cycles); + zfree(&al->br_cntr); + } + } + } + } + + annotation__unlock(notes); + return 0; +} + +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample) +{ + return symbol__inc_addr_samples(&ams->ms, ams->al_addr, sample); +} + +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 ip) +{ + return symbol__inc_addr_samples(&he->ms, ip, sample); +} + + +void annotation__exit(struct annotation *notes) +{ + annotated_source__delete(notes->src); + annotated_branch__delete(notes->branch); +} + +static struct sharded_mutex *sharded_mutex; + +static void annotation__init_sharded_mutex(void) +{ + /* As many mutexes as there are CPUs. */ + sharded_mutex = sharded_mutex__new(cpu__max_present_cpu().cpu); +} + +static size_t annotation__hash(const struct annotation *notes) +{ + return (size_t)notes; +} + +static struct mutex *annotation__get_mutex(const struct annotation *notes) +{ + static pthread_once_t once = PTHREAD_ONCE_INIT; + + pthread_once(&once, annotation__init_sharded_mutex); + if (!sharded_mutex) + return NULL; + + return sharded_mutex__get_mutex(sharded_mutex, annotation__hash(notes)); +} + +void annotation__lock(struct annotation *notes) + NO_THREAD_SAFETY_ANALYSIS +{ + struct mutex *mutex = annotation__get_mutex(notes); + + if (mutex) + mutex_lock(mutex); +} + +void annotation__unlock(struct annotation *notes) + NO_THREAD_SAFETY_ANALYSIS +{ + struct mutex *mutex = annotation__get_mutex(notes); + + if (mutex) + mutex_unlock(mutex); +} + +bool annotation__trylock(struct annotation *notes) +{ + struct mutex *mutex = annotation__get_mutex(notes); + + if (!mutex) + return false; + + return mutex_trylock(mutex); +} + +void annotation_line__add(struct annotation_line *al, struct list_head *head) +{ + list_add_tail(&al->node, head); +} + +struct annotation_line * +annotation_line__next(struct annotation_line *pos, struct list_head *head) +{ + list_for_each_entry_continue(pos, head, node) + if (pos->offset >= 0) + return pos; + + return NULL; +} + +static const char *annotate__address_color(struct block_range *br) +{ + double cov = block_range__coverage(br); + + if (cov >= 0) { + /* mark red for >75% coverage */ + if (cov > 0.75) + return PERF_COLOR_RED; + + /* mark dull for <1% coverage */ + if (cov < 0.01) + return PERF_COLOR_NORMAL; + } + + return PERF_COLOR_MAGENTA; +} + +static const char *annotate__asm_color(struct block_range *br) +{ + double cov = block_range__coverage(br); + + if (cov >= 0) { + /* mark dull for <1% coverage */ + if (cov < 0.01) + return PERF_COLOR_NORMAL; + } + + return PERF_COLOR_BLUE; +} + +static void annotate__branch_printf(struct block_range *br, u64 addr) +{ + bool emit_comment = true; + + if (!br) + return; + +#if 1 + if (br->is_target && br->start == addr) { + struct block_range *branch = br; + double p; + + /* + * Find matching branch to our target. + */ + while (!branch->is_branch) + branch = block_range__next(branch); + + p = 100 *(double)br->entry / branch->coverage; + + if (p > 0.1) { + if (emit_comment) { + emit_comment = false; + printf("\t#"); + } + + /* + * The percentage of coverage joined at this target in relation + * to the next branch. + */ + printf(" +%.2f%%", p); + } + } +#endif + if (br->is_branch && br->end == addr) { + double p = 100*(double)br->taken / br->coverage; + + if (p > 0.1) { + if (emit_comment) { + emit_comment = false; + printf("\t#"); + } + + /* + * The percentage of coverage leaving at this branch, and + * its prediction ratio. + */ + printf(" -%.2f%% (p:%.2f%%)", p, 100*(double)br->pred / br->taken); + } + } +} + +static int disasm_line__print(struct disasm_line *dl, u64 start, int addr_fmt_width) +{ + s64 offset = dl->al.offset; + const u64 addr = start + offset; + struct block_range *br; + + br = block_range__find(addr); + color_fprintf(stdout, annotate__address_color(br), " %*" PRIx64 ":", addr_fmt_width, addr); + color_fprintf(stdout, annotate__asm_color(br), "%s", dl->al.line); + annotate__branch_printf(br, addr); + return 0; +} + +static struct annotated_data_type * +__hist_entry__get_data_type(struct hist_entry *he, const struct arch *arch, + struct debuginfo *dbg, struct disasm_line *dl, + int *type_offset); + +static bool needs_type_info(struct annotated_data_type *data_type) +{ + if (data_type == NULL || data_type == NO_TYPE) + return false; + + if (verbose) + return true; + + return (data_type != &stackop_type) && (data_type != &canary_type); +} + +static int +annotation_line__print(struct annotation_line *al, struct annotation_print_data *apd, + struct annotation_options *opts, int printed, + struct annotation_line *queue) +{ + struct symbol *sym = apd->he->ms.sym; + struct disasm_line *dl = container_of(al, struct disasm_line, al); + struct annotation *notes = symbol__annotation(sym); + static const char *prev_line; + int max_lines = opts->max_lines; + int percent_type = opts->percent_type; + + if (al->offset != -1) { + double max_percent = 0.0; + int i, nr_percent = 1; + const char *color; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + percent_type); + + if (percent > max_percent) + max_percent = percent; + } + + if (al->data_nr > nr_percent) + nr_percent = al->data_nr; + + if (max_percent < opts->min_pcnt) + return -1; + + if (max_lines && printed >= max_lines) + return 1; + + if (queue != NULL) { + struct annotation_options queue_opts = { + .max_lines = 1, + .percent_type = percent_type, + }; + + list_for_each_entry_from(queue, ¬es->src->source, node) { + if (queue == al) + break; + annotation_line__print(queue, apd, &queue_opts, + /*printed=*/0, /*queue=*/NULL); + } + } + + color = get_percent_color(max_percent); + + for (i = 0; i < nr_percent; i++) { + struct annotation_data *data = &al->data[i]; + double percent; + + percent = annotation_data__percent(data, percent_type); + color = get_percent_color(percent); + + if (symbol_conf.show_total_period) + color_fprintf(stdout, color, " %11" PRIu64, + data->he.period); + else if (symbol_conf.show_nr_samples) + color_fprintf(stdout, color, " %7" PRIu64, + data->he.nr_samples); + else + color_fprintf(stdout, color, " %7.2f", percent); + } + + printf(" : "); + + disasm_line__print(dl, notes->src->start, apd->addr_fmt_width); + + if (opts->code_with_type && apd->dbg) { + struct annotated_data_type *data_type; + int offset = 0; + + data_type = __hist_entry__get_data_type(apd->he, apd->arch, + apd->dbg, dl, &offset); + if (needs_type_info(data_type)) { + char buf[4096]; + + printf("\t\t# data-type: %s", + data_type->self.type_name); + + if (data_type != &stackop_type && + data_type != &canary_type) + printf(" +%#x", offset); + + if (annotated_data_type__get_member_name(data_type, + buf, + sizeof(buf), + offset)) + printf(" (%s)", buf); + } + } + + /* + * Also color the filename and line if needed, with + * the same color than the percentage. Don't print it + * twice for close colored addr with the same filename:line + */ + if (al->path) { + if (!prev_line || strcmp(prev_line, al->path)) { + color_fprintf(stdout, color, " // %s", al->path); + prev_line = al->path; + } + } + + printf("\n"); + } else if (max_lines && printed >= max_lines) + return 1; + else { + int width = annotation__pcnt_width(notes); + + if (queue) + return -1; + + if (!*al->line) + printf(" %*s:\n", width, " "); + else + printf(" %*s: %-*d %s\n", width, " ", apd->addr_fmt_width, + al->line_nr, al->line); + } + + return 0; +} + +static void calc_percent(struct annotation *notes, + struct evsel *evsel, + struct annotation_data *data, + s64 offset, s64 end) +{ + struct hists *hists = evsel__hists(evsel); + struct sym_hist *sym_hist = annotation__histogram(notes, evsel); + unsigned int hits = 0; + u64 period = 0; + + while (offset < end) { + struct sym_hist_entry *entry; + + entry = annotated_source__hist_entry(notes->src, evsel, offset); + if (entry) { + hits += entry->nr_samples; + period += entry->period; + } + ++offset; + } + + if (sym_hist->nr_samples) { + data->he.period = period; + data->he.nr_samples = hits; + data->percent[PERCENT_HITS_LOCAL] = 100.0 * hits / sym_hist->nr_samples; + } + + if (hists->stats.nr_non_filtered_samples) + data->percent[PERCENT_HITS_GLOBAL] = 100.0 * hits / hists->stats.nr_non_filtered_samples; + + if (sym_hist->period) + data->percent[PERCENT_PERIOD_LOCAL] = 100.0 * period / sym_hist->period; + + if (hists->stats.total_period) + data->percent[PERCENT_PERIOD_GLOBAL] = 100.0 * period / hists->stats.total_period; +} + +static void annotation__calc_percent(struct annotation *notes, + struct evsel *leader, s64 len) +{ + struct annotation_line *al, *next; + struct evsel *evsel; + + list_for_each_entry(al, ¬es->src->source, node) { + s64 end; + int i = 0; + + if (al->offset == -1) + continue; + + next = annotation_line__next(al, ¬es->src->source); + end = next ? next->offset : len; + + for_each_group_evsel(evsel, leader) { + struct annotation_data *data; + + BUG_ON(i >= al->data_nr); + + if (symbol_conf.skip_empty && + evsel__hists(evsel)->stats.nr_samples == 0) + continue; + + data = &al->data[i++]; + + calc_percent(notes, evsel, data, al->offset, end); + } + } +} + +void symbol__calc_percent(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + + annotation__calc_percent(notes, evsel, symbol__size(sym)); +} + +int thread__get_arch(struct thread *thread, const struct arch **parch) +{ + const struct arch *arch; + struct machine *machine; + uint32_t e_flags; + uint16_t e_machine; + + if (!thread) { + *parch = NULL; + return -1; + } + + machine = maps__machine(thread__maps(thread)); + e_machine = thread__e_machine(thread, machine, &e_flags); + arch = arch__find(e_machine, e_flags, machine->env ? machine->env->cpuid : NULL); + if (arch == NULL) { + pr_err("%s: unsupported arch %d\n", __func__, e_machine); + return errno; + } + if (parch) + *parch = arch; + + return 0; +} + +int symbol__annotate(struct map_symbol *ms, struct evsel *evsel, + const struct arch **parch) +{ + struct symbol *sym = ms->sym; + struct annotation *notes = symbol__annotation(sym); + struct annotate_args args = { + .options = &annotate_opts, + }; + const struct arch *arch = NULL; + int err, nr; + + err = thread__get_arch(ms->thread, &arch); + if (err) + return err; + + if (parch) + *parch = arch; + + if (notes->src && !list_empty(¬es->src->source)) + return 0; + + args.arch = arch; + args.ms = ms; + + if (notes->src == NULL) { + notes->src = annotated_source__new(); + if (notes->src == NULL) + return -1; + } + + nr = 0; + if (evsel__is_group_event(evsel)) { + struct evsel *pos; + + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + nr++; + } + } + notes->src->nr_events = nr ? nr : 1; + + if (annotate_opts.full_addr) + notes->src->start = map__objdump_2mem(ms->map, ms->sym->start); + else + notes->src->start = map__rip_2objdump(ms->map, ms->sym->start); + + return symbol__disassemble(sym, &args); +} + +static void insert_source_line(struct rb_root *root, struct annotation_line *al) +{ + struct annotation_line *iter; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + unsigned int percent_type = annotate_opts.percent_type; + int i, ret; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct annotation_line, rb_node); + + ret = strcmp(iter->path, al->path); + if (ret == 0) { + for (i = 0; i < al->data_nr; i++) { + iter->data[i].percent_sum += annotation_data__percent(&al->data[i], + percent_type); + } + return; + } + + if (ret < 0) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + for (i = 0; i < al->data_nr; i++) { + al->data[i].percent_sum = annotation_data__percent(&al->data[i], + percent_type); + } + + rb_link_node(&al->rb_node, parent, p); + rb_insert_color(&al->rb_node, root); +} + +static int cmp_source_line(struct annotation_line *a, struct annotation_line *b) +{ + int i; + + for (i = 0; i < a->data_nr; i++) { + if (a->data[i].percent_sum == b->data[i].percent_sum) + continue; + return a->data[i].percent_sum > b->data[i].percent_sum; + } + + return 0; +} + +static void __resort_source_line(struct rb_root *root, struct annotation_line *al) +{ + struct annotation_line *iter; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct annotation_line, rb_node); + + if (cmp_source_line(al, iter)) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + rb_link_node(&al->rb_node, parent, p); + rb_insert_color(&al->rb_node, root); +} + +static void resort_source_line(struct rb_root *dest_root, struct rb_root *src_root) +{ + struct annotation_line *al; + struct rb_node *node; + + node = rb_first(src_root); + while (node) { + struct rb_node *next; + + al = rb_entry(node, struct annotation_line, rb_node); + next = rb_next(node); + rb_erase(node, src_root); + + __resort_source_line(dest_root, al); + node = next; + } +} + +static void print_summary(struct rb_root *root, const char *filename) +{ + struct annotation_line *al; + struct rb_node *node; + + printf("\nSorted summary for file %s\n", filename); + printf("----------------------------------------------\n\n"); + + if (RB_EMPTY_ROOT(root)) { + printf(" Nothing higher than %1.1f%%\n", MIN_GREEN); + return; + } + + node = rb_first(root); + while (node) { + double percent, percent_max = 0.0; + const char *color; + char *path; + int i; + + al = rb_entry(node, struct annotation_line, rb_node); + for (i = 0; i < al->data_nr; i++) { + percent = al->data[i].percent_sum; + color = get_percent_color(percent); + color_fprintf(stdout, color, " %7.2f", percent); + + if (percent > percent_max) + percent_max = percent; + } + + path = al->path; + color = get_percent_color(percent_max); + color_fprintf(stdout, color, " %s\n", path); + + node = rb_next(node); + } +} + +static void symbol__annotate_hits(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + u64 len = symbol__size(sym), offset; + + for (offset = 0; offset < len; ++offset) { + struct sym_hist_entry *entry; + + entry = annotated_source__hist_entry(notes->src, evsel, offset); + if (entry && entry->nr_samples != 0) + printf("%*" PRIx64 ": %" PRIu64 "\n", BITS_PER_LONG / 2, + sym->start + offset, entry->nr_samples); + } + printf("%*s: %" PRIu64 "\n", BITS_PER_LONG / 2, "h->nr_samples", h->nr_samples); +} + +static int annotated_source__addr_fmt_width(struct list_head *lines, u64 start) +{ + char bf[32]; + struct annotation_line *line; + + list_for_each_entry_reverse(line, lines, node) { + if (line->offset != -1) + return scnprintf(bf, sizeof(bf), "%" PRIx64, start + line->offset); + } + + return 0; +} + +int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel) +{ + struct map_symbol *ms = &he->ms; + struct map *map = ms->map; + struct symbol *sym = ms->sym; + struct dso *dso = map__dso(map); + char *filename; + const char *d_filename; + const char *evsel_name = evsel__name(evsel); + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + struct annotation_line *pos, *queue = NULL; + struct annotation_options *opts = &annotate_opts; + struct annotation_print_data apd = { + .he = he, + .evsel = evsel, + }; + int printed = 2, queue_len = 0; + int more = 0; + bool context = opts->context; + int width = annotation__pcnt_width(notes); + int graph_dotted_len; + char buf[512]; + + filename = strdup(dso__long_name(dso)); + if (!filename) + return -ENOMEM; + + if (opts->full_path) + d_filename = filename; + else + d_filename = perf_basename(filename); + + if (evsel__is_group_event(evsel)) { + evsel__group_desc(evsel, buf, sizeof(buf)); + evsel_name = buf; + } + + graph_dotted_len = printf(" %-*.*s| Source code & Disassembly of %s for %s (%" PRIu64 " samples, " + "percent: %s)\n", + width, width, symbol_conf.show_total_period ? "Period" : + symbol_conf.show_nr_samples ? "Samples" : "Percent", + d_filename, evsel_name, h->nr_samples, + percent_type_str(opts->percent_type)); + + printf("%-*.*s----\n", + graph_dotted_len, graph_dotted_len, graph_dotted_line); + + if (verbose > 0) + symbol__annotate_hits(sym, evsel); + + apd.addr_fmt_width = annotated_source__addr_fmt_width(¬es->src->source, + notes->src->start); + thread__get_arch(ms->thread, &apd.arch); + apd.dbg = dso__debuginfo(dso); + + list_for_each_entry(pos, ¬es->src->source, node) { + int err; + + if (context && queue == NULL) { + queue = pos; + queue_len = 0; + } + + err = annotation_line__print(pos, &apd, opts, printed, queue); + + switch (err) { + case 0: + ++printed; + if (context) { + printed += queue_len; + queue = NULL; + queue_len = 0; + } + break; + case 1: + /* filtered by max_lines */ + ++more; + break; + case -1: + default: + /* + * Filtered by min_pcnt or non IP lines when + * context != 0 + */ + if (!context) + break; + if (queue_len == context) + queue = list_entry(queue->node.next, typeof(*queue), node); + else + ++queue_len; + break; + } + } + + debuginfo__delete(apd.dbg); + free(filename); + + return more; +} + +static void FILE__set_percent_color(void *fp __maybe_unused, + double percent __maybe_unused, + bool current __maybe_unused) +{ +} + +static int FILE__set_jumps_percent_color(void *fp __maybe_unused, + int nr __maybe_unused, bool current __maybe_unused) +{ + return 0; +} + +static int FILE__set_color(void *fp __maybe_unused, int color __maybe_unused) +{ + return 0; +} + +static void FILE__printf(void *fp, const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + vfprintf(fp, fmt, args); + va_end(args); +} + +static void FILE__write_graph(void *fp, int graph) +{ + const char *s; + switch (graph) { + + case DARROW_CHAR: s = "↓"; break; + case UARROW_CHAR: s = "↑"; break; + case LARROW_CHAR: s = "←"; break; + case RARROW_CHAR: s = "→"; break; + default: s = "?"; break; + } + + fputs(s, fp); +} + +static int symbol__annotate_fprintf2(struct symbol *sym, FILE *fp, + struct annotation_print_data *apd) +{ + struct annotation *notes = symbol__annotation(sym); + struct annotation_write_ops wops = { + .first_line = true, + .obj = fp, + .set_color = FILE__set_color, + .set_percent_color = FILE__set_percent_color, + .set_jumps_percent_color = FILE__set_jumps_percent_color, + .printf = FILE__printf, + .write_graph = FILE__write_graph, + }; + struct annotation_line *al; + + if (annotate_opts.code_with_type) { + thread__get_arch(apd->he->ms.thread, &apd->arch); + apd->dbg = dso__debuginfo(map__dso(apd->he->ms.map)); + } + + list_for_each_entry(al, ¬es->src->source, node) { + if (annotation_line__filter(al)) + continue; + annotation_line__write(al, notes, &wops, apd); + fputc('\n', fp); + wops.first_line = false; + } + + if (annotate_opts.code_with_type) + debuginfo__delete(apd->dbg); + + return 0; +} + +int map_symbol__annotation_dump(struct map_symbol *ms, struct evsel *evsel, + struct hist_entry *he) +{ + const char *ev_name = evsel__name(evsel); + char buf[1024]; + char *filename; + int err = -1; + FILE *fp; + struct annotation_print_data apd = { + .he = he, + .evsel = evsel, + }; + + if (asprintf(&filename, "%s.annotation", ms->sym->name) < 0) + return -1; + + fp = fopen(filename, "w"); + if (fp == NULL) + goto out_free_filename; + + if (evsel__is_group_event(evsel)) { + evsel__group_desc(evsel, buf, sizeof(buf)); + ev_name = buf; + } + + fprintf(fp, "%s() %s\nEvent: %s\n\n", + ms->sym->name, dso__long_name(map__dso(ms->map)), ev_name); + symbol__annotate_fprintf2(ms->sym, fp, &apd); + + fclose(fp); + err = 0; +out_free_filename: + free(filename); + return err; +} + +void symbol__annotate_zero_histogram(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + + memset(h, 0, sizeof(*notes->src->histograms) * notes->src->nr_histograms); +} + +void symbol__annotate_decay_histogram(struct symbol *sym, struct evsel *evsel) +{ + struct annotation *notes = symbol__annotation(sym); + struct sym_hist *h = annotation__histogram(notes, evsel); + struct annotation_line *al; + + h->nr_samples = 0; + list_for_each_entry(al, ¬es->src->source, node) { + struct sym_hist_entry *entry; + + if (al->offset == -1) + continue; + + entry = annotated_source__hist_entry(notes->src, evsel, al->offset); + if (entry == NULL) + continue; + + entry->nr_samples = entry->nr_samples * 7 / 8; + h->nr_samples += entry->nr_samples; + } +} + +void annotated_source__purge(struct annotated_source *as) +{ + struct annotation_line *al, *n; + + list_for_each_entry_safe(al, n, &as->source, node) { + list_del_init(&al->node); + disasm_line__free(disasm_line(al)); + } + as->tried_source = false; +} + +static size_t disasm_line__fprintf(struct disasm_line *dl, FILE *fp) +{ + size_t printed; + + if (dl->al.offset == -1) + return fprintf(fp, "%s\n", dl->al.line); + + printed = fprintf(fp, "%#" PRIx64 " %s", dl->al.offset, dl->ins.name); + + if (dl->ops.raw[0] != '\0') { + printed += fprintf(fp, "%.*s %s\n", 6 - (int)printed, " ", + dl->ops.raw); + } + + return printed + fprintf(fp, "\n"); +} + +size_t disasm__fprintf(struct list_head *head, FILE *fp) +{ + struct disasm_line *pos; + size_t printed = 0; + + list_for_each_entry(pos, head, al.node) + printed += disasm_line__fprintf(pos, fp); + + return printed; +} + +bool disasm_line__is_valid_local_jump(struct disasm_line *dl, struct symbol *sym) +{ + if (!dl || !dl->ins.ops || !ins__is_jump(&dl->ins) || + !disasm_line__has_local_offset(dl) || dl->ops.target.offset < 0 || + dl->ops.target.offset >= (s64)symbol__size(sym)) + return false; + + return true; +} + +static void +annotation__mark_jump_targets(struct annotation *notes, struct symbol *sym) +{ + struct annotation_line *al; + + /* PLT symbols contain external offsets */ + if (strstr(sym->name, "@plt")) + return; + + list_for_each_entry(al, ¬es->src->source, node) { + struct disasm_line *dl; + struct annotation_line *target; + + dl = disasm_line(al); + + if (!disasm_line__is_valid_local_jump(dl, sym)) + continue; + + target = annotated_source__get_line(notes->src, + dl->ops.target.offset); + /* + * FIXME: Oops, no jump target? Buggy disassembler? Or do we + * have to adjust to the previous offset? + */ + if (target == NULL) + continue; + + if (++target->jump_sources > notes->src->max_jump_sources) + notes->src->max_jump_sources = target->jump_sources; + } +} + +static void annotation__set_index(struct annotation *notes) +{ + struct annotation_line *al; + struct annotated_source *src = notes->src; + + src->widths.max_line_len = 0; + src->nr_entries = 0; + src->nr_asm_entries = 0; + + list_for_each_entry(al, &src->source, node) { + size_t line_len = strlen(al->line); + + if (src->widths.max_line_len < line_len) + src->widths.max_line_len = line_len; + al->idx = src->nr_entries++; + if (al->offset != -1) + al->idx_asm = src->nr_asm_entries++; + else + al->idx_asm = -1; + } +} + +static inline int width_jumps(int n) +{ + if (n >= 100) + return 5; + if (n / 10) + return 2; + return 1; +} + +static int annotation__max_ins_name(struct annotation *notes) +{ + int max_name = 0, len; + struct annotation_line *al; + + list_for_each_entry(al, ¬es->src->source, node) { + if (al->offset == -1) + continue; + + len = strlen(disasm_line(al)->ins.name); + if (max_name < len) + max_name = len; + } + + return max_name; +} + +static void +annotation__init_column_widths(struct annotation *notes, struct symbol *sym) +{ + notes->src->widths.addr = notes->src->widths.target = + notes->src->widths.min_addr = hex_width(symbol__size(sym)); + notes->src->widths.max_addr = hex_width(sym->end); + notes->src->widths.jumps = width_jumps(notes->src->max_jump_sources); + notes->src->widths.max_ins_name = annotation__max_ins_name(notes); +} + +void annotation__update_column_widths(struct annotation *notes) +{ + if (annotate_opts.use_offset) + notes->src->widths.target = notes->src->widths.min_addr; + else if (annotate_opts.full_addr) + notes->src->widths.target = BITS_PER_LONG / 4; + else + notes->src->widths.target = notes->src->widths.max_addr; + + notes->src->widths.addr = notes->src->widths.target; + + if (annotate_opts.show_nr_jumps) + notes->src->widths.addr += notes->src->widths.jumps + 1; +} + +void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms) +{ + annotate_opts.full_addr = !annotate_opts.full_addr; + + if (annotate_opts.full_addr) + notes->src->start = map__objdump_2mem(ms->map, ms->sym->start); + else + notes->src->start = map__rip_2objdump(ms->map, ms->sym->start); + + annotation__update_column_widths(notes); +} + +static void annotation__calc_lines(struct annotation *notes, struct map_symbol *ms, + struct rb_root *root) +{ + struct annotation_line *al; + struct rb_root tmp_root = RB_ROOT; + + list_for_each_entry(al, ¬es->src->source, node) { + double percent_max = 0.0; + u64 addr; + int i; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + annotate_opts.percent_type); + + if (percent > percent_max) + percent_max = percent; + } + + if (percent_max <= 0.5) + continue; + + addr = map__rip_2objdump(ms->map, ms->sym->start); + al->path = get_srcline(map__dso(ms->map), addr + al->offset, NULL, + false, true, ms->sym->start + al->offset); + insert_source_line(&tmp_root, al); + } + + resort_source_line(root, &tmp_root); +} + +static void symbol__calc_lines(struct map_symbol *ms, struct rb_root *root) +{ + struct annotation *notes = symbol__annotation(ms->sym); + + annotation__calc_lines(notes, ms, root); +} + +int hist_entry__tty_annotate2(struct hist_entry *he, struct evsel *evsel) +{ + struct map_symbol *ms = &he->ms; + struct dso *dso = map__dso(ms->map); + struct symbol *sym = ms->sym; + struct rb_root source_line = RB_ROOT; + struct hists *hists = evsel__hists(evsel); + struct annotation_print_data apd = { + .he = he, + .evsel = evsel, + }; + char buf[1024]; + int err; + + err = symbol__annotate2(ms, evsel, NULL); + if (err) { + char msg[BUFSIZ]; + + dso__set_annotate_warned(dso); + symbol__strerror_disassemble(ms, err, msg, sizeof(msg)); + ui__error("Couldn't annotate %s:\n%s", sym->name, msg); + return -1; + } + + if (annotate_opts.print_lines) { + srcline_full_filename = annotate_opts.full_path; + symbol__calc_lines(ms, &source_line); + print_summary(&source_line, dso__long_name(dso)); + } + + hists__scnprintf_title(hists, buf, sizeof(buf)); + fprintf(stdout, "%s, [percent: %s]\n%s() %s\n", + buf, percent_type_str(annotate_opts.percent_type), sym->name, dso__long_name(dso)); + symbol__annotate_fprintf2(sym, stdout, &apd); + + annotated_source__purge(symbol__annotation(sym)->src); + + return 0; +} + +int hist_entry__tty_annotate(struct hist_entry *he, struct evsel *evsel) +{ + struct map_symbol *ms = &he->ms; + struct dso *dso = map__dso(ms->map); + struct symbol *sym = ms->sym; + struct rb_root source_line = RB_ROOT; + int err; + + err = symbol__annotate(ms, evsel, NULL); + if (err) { + char msg[BUFSIZ]; + + dso__set_annotate_warned(dso); + symbol__strerror_disassemble(ms, err, msg, sizeof(msg)); + ui__error("Couldn't annotate %s:\n%s", sym->name, msg); + return -1; + } + + symbol__calc_percent(sym, evsel); + + if (annotate_opts.print_lines) { + srcline_full_filename = annotate_opts.full_path; + symbol__calc_lines(ms, &source_line); + print_summary(&source_line, dso__long_name(dso)); + } + + hist_entry__annotate_printf(he, evsel); + + annotated_source__purge(symbol__annotation(sym)->src); + + return 0; +} + +bool ui__has_annotation(void) +{ + return use_browser == 1 && perf_hpp_list.sym; +} + + +static double annotation_line__max_percent(struct annotation_line *al, + unsigned int percent_type) +{ + double percent_max = 0.0; + int i; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + percent_type); + + if (percent > percent_max) + percent_max = percent; + } + + return percent_max; +} + +static int disasm_line__write(struct disasm_line *dl, struct annotation *notes, + void *obj, char *bf, size_t size, + void (*obj__printf)(void *obj, const char *fmt, ...), + void (*obj__write_graph)(void *obj, int graph)) +{ + if (dl->ins.ops && dl->ins.ops->scnprintf) { + if (ins__is_jump(&dl->ins)) { + bool fwd; + + if (dl->ops.target.outside) + goto call_like; + fwd = dl->ops.target.offset > dl->al.offset; + obj__write_graph(obj, fwd ? DARROW_CHAR : UARROW_CHAR); + obj__printf(obj, " "); + } else if (ins__is_call(&dl->ins)) { +call_like: + obj__write_graph(obj, RARROW_CHAR); + obj__printf(obj, " "); + } else if (ins__is_ret(&dl->ins)) { + obj__write_graph(obj, LARROW_CHAR); + obj__printf(obj, " "); + } else { + obj__printf(obj, " "); + } + } else { + obj__printf(obj, " "); + } + + return disasm_line__scnprintf(dl, bf, size, !annotate_opts.use_offset, + notes->src->widths.max_ins_name) + 2; +} + +static void ipc_coverage_string(char *bf, int size, struct annotation *notes) +{ + double ipc = 0.0, coverage = 0.0; + struct annotated_branch *branch = annotation__get_branch(notes); + + if (branch && branch->hit_cycles) + ipc = branch->hit_insn / ((double)branch->hit_cycles); + + if (branch && branch->total_insn) { + coverage = branch->cover_insn * 100.0 / + ((double)branch->total_insn); + } + + scnprintf(bf, size, "(Average IPC: %.2f, IPC Coverage: %.1f%%)", + ipc, coverage); +} + +int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header) +{ + struct evsel *pos; + struct strbuf sb; + + if (evsel->evlist->nr_br_cntr <= 0) + return -ENOTSUP; + + strbuf_init(&sb, /*hint=*/ 0); + + if (header && strbuf_addf(&sb, "# Branch counter abbr list:\n")) + goto err; + + evlist__for_each_entry(evsel->evlist, pos) { + if (!(pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS)) + continue; + if (header && strbuf_addf(&sb, "#")) + goto err; + + if (strbuf_addf(&sb, " %s = %s\n", pos->name, pos->abbr_name)) + goto err; + } + + if (header && strbuf_addf(&sb, "#")) + goto err; + if (strbuf_addf(&sb, " '-' No event occurs\n")) + goto err; + + if (header && strbuf_addf(&sb, "#")) + goto err; + if (strbuf_addf(&sb, " '+' Event occurrences may be lost due to branch counter saturated\n")) + goto err; + + *str = strbuf_detach(&sb, NULL); + + return 0; +err: + strbuf_release(&sb); + return -ENOMEM; +} + +/* Assume the branch counter saturated at 3 */ +#define ANNOTATION_BR_CNTR_SATURATION 3 + +int annotation_br_cntr_entry(char **str, int br_cntr_nr, + u64 *br_cntr, int num_aggr, + struct evsel *evsel) +{ + struct evsel *pos = evsel ? evlist__first(evsel->evlist) : NULL; + bool saturated = false; + int i, j, avg, used; + struct strbuf sb; + + strbuf_init(&sb, /*hint=*/ 0); + for (i = 0; i < br_cntr_nr; i++) { + used = 0; + avg = ceil((double)(br_cntr[i] & ~ANNOTATION__BR_CNTR_SATURATED_FLAG) / + (double)num_aggr); + + /* + * A histogram with the abbr name is displayed by default. + * With -v, the exact number of branch counter is displayed. + */ + if (verbose) { + evlist__for_each_entry_from(evsel->evlist, pos) { + if ((pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + (pos->br_cntr_idx == i)) + break; + } + if (strbuf_addstr(&sb, pos->abbr_name)) + goto err; + + if (!br_cntr[i]) { + if (strbuf_addstr(&sb, "=-")) + goto err; + } else { + if (strbuf_addf(&sb, "=%d", avg)) + goto err; + } + if (br_cntr[i] & ANNOTATION__BR_CNTR_SATURATED_FLAG) { + if (strbuf_addch(&sb, '+')) + goto err; + } else { + if (strbuf_addch(&sb, ' ')) + goto err; + } + + if ((i < br_cntr_nr - 1) && strbuf_addch(&sb, ',')) + goto err; + continue; + } + + if (strbuf_addch(&sb, '|')) + goto err; + + if (!br_cntr[i]) { + if (strbuf_addch(&sb, '-')) + goto err; + used++; + } else { + evlist__for_each_entry_from(evsel->evlist, pos) { + if ((pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + (pos->br_cntr_idx == i)) + break; + } + if (br_cntr[i] & ANNOTATION__BR_CNTR_SATURATED_FLAG) + saturated = true; + + for (j = 0; j < avg; j++, used++) { + /* Print + if the number of logged events > 3 */ + if (j >= ANNOTATION_BR_CNTR_SATURATION) { + saturated = true; + break; + } + if (strbuf_addstr(&sb, pos->abbr_name)) + goto err; + } + + if (saturated) { + if (strbuf_addch(&sb, '+')) + goto err; + used++; + } + pos = list_next_entry(pos, core.node); + } + + for (j = used; j < ANNOTATION_BR_CNTR_SATURATION + 1; j++) { + if (strbuf_addch(&sb, ' ')) + goto err; + } + } + + if (!verbose && strbuf_addch(&sb, br_cntr_nr ? '|' : ' ')) + goto err; + + *str = strbuf_detach(&sb, NULL); + + return 0; +err: + strbuf_release(&sb); + return -ENOMEM; +} + +struct type_hash_entry { + struct annotated_data_type *type; + int offset; +}; + +static int disasm_line__snprint_type_info(struct disasm_line *dl, + char *buf, int len, + struct annotation_print_data *apd) +{ + struct annotated_data_type *data_type = NULL; + struct type_hash_entry *entry = NULL; + char member[256]; + int offset = 0; + int printed; + + scnprintf(buf, len, " "); + + if (!annotate_opts.code_with_type || apd->dbg == NULL) + return 1; + + if (apd->type_hash) { + hashmap__find(apd->type_hash, dl->al.offset, &entry); + if (entry != NULL) { + data_type = entry->type; + offset = entry->offset; + } + } + + if (data_type == NULL) + data_type = __hist_entry__get_data_type(apd->he, apd->arch, apd->dbg, dl, &offset); + + if (apd->type_hash && entry == NULL) { + entry = malloc(sizeof(*entry)); + if (entry != NULL) { + entry->type = data_type; + entry->offset = offset; + hashmap__add(apd->type_hash, dl->al.offset, entry); + } + } + + if (!needs_type_info(data_type)) + return 1; + + printed = scnprintf(buf, len, "\t\t# data-type: %s", data_type->self.type_name); + + if (data_type != &stackop_type && data_type != &canary_type && len > printed) + printed += scnprintf(buf + printed, len - printed, " +%#x", offset); + + if (annotated_data_type__get_member_name(data_type, member, sizeof(member), offset) && + len > printed) { + printed += scnprintf(buf + printed, len - printed, " (%s)", member); + } + return printed; +} + +void annotation_line__write(struct annotation_line *al, struct annotation *notes, + const struct annotation_write_ops *wops, + struct annotation_print_data *apd) +{ + bool current_entry = wops->current_entry; + bool change_color = wops->change_color; + double percent_max = annotation_line__max_percent(al, annotate_opts.percent_type); + int width = wops->width; + int pcnt_width = annotation__pcnt_width(notes); + int cycles_width = annotation__cycles_width(notes); + bool show_title = false; + char bf[256]; + int printed; + void *obj = wops->obj; + int (*obj__set_color)(void *obj, int color) = wops->set_color; + void (*obj__set_percent_color)(void *obj, double percent, bool current) = wops->set_percent_color; + int (*obj__set_jumps_percent_color)(void *obj, int nr, bool current) = wops->set_jumps_percent_color; + void (*obj__printf)(void *obj, const char *fmt, ...) = wops->printf; + void (*obj__write_graph)(void *obj, int graph) = wops->write_graph; + + if (wops->first_line && (al->offset == -1 || percent_max == 0.0)) { + if (notes->branch && al->cycles) { + if (al->cycles->ipc == 0.0 && al->cycles->avg == 0) + show_title = true; + } else + show_title = true; + } + + if (al->offset != -1 && percent_max != 0.0) { + int i; + + for (i = 0; i < al->data_nr; i++) { + double percent; + + percent = annotation_data__percent(&al->data[i], + annotate_opts.percent_type); + + obj__set_percent_color(obj, percent, current_entry); + if (symbol_conf.show_total_period) { + obj__printf(obj, "%11" PRIu64 " ", al->data[i].he.period); + } else if (symbol_conf.show_nr_samples) { + obj__printf(obj, "%7" PRIu64 " ", + al->data[i].he.nr_samples); + } else { + obj__printf(obj, "%7.2f ", percent); + } + } + } else { + obj__set_percent_color(obj, 0, current_entry); + + if (!show_title) + obj__printf(obj, "%-*s", pcnt_width, " "); + else { + obj__printf(obj, "%-*s", pcnt_width, + symbol_conf.show_total_period ? "Period" : + symbol_conf.show_nr_samples ? "Samples" : "Percent"); + } + } + width -= pcnt_width; + + if (notes->branch) { + if (al->cycles && al->cycles->ipc) + obj__printf(obj, "%*.2f ", ANNOTATION__IPC_WIDTH - 1, al->cycles->ipc); + else if (!show_title) + obj__printf(obj, "%*s", ANNOTATION__IPC_WIDTH, " "); + else + obj__printf(obj, "%*s ", ANNOTATION__IPC_WIDTH - 1, "IPC"); + + if (!annotate_opts.show_minmax_cycle) { + if (al->cycles && al->cycles->avg) + obj__printf(obj, "%*" PRIu64 " ", + ANNOTATION__CYCLES_WIDTH - 1, al->cycles->avg); + else if (!show_title) + obj__printf(obj, "%*s", + ANNOTATION__CYCLES_WIDTH, " "); + else + obj__printf(obj, "%*s ", + ANNOTATION__CYCLES_WIDTH - 1, + "Cycle"); + } else { + if (al->cycles) { + char str[32]; + + scnprintf(str, sizeof(str), + "%" PRIu64 "(%" PRIu64 "/%" PRIu64 ")", + al->cycles->avg, al->cycles->min, + al->cycles->max); + + obj__printf(obj, "%*s ", + ANNOTATION__MINMAX_CYCLES_WIDTH - 1, + str); + } else if (!show_title) + obj__printf(obj, "%*s", + ANNOTATION__MINMAX_CYCLES_WIDTH, + " "); + else + obj__printf(obj, "%*s ", + ANNOTATION__MINMAX_CYCLES_WIDTH - 1, + "Cycle(min/max)"); + } + + if (annotate_opts.show_br_cntr) { + if (show_title) { + obj__printf(obj, "%*s ", + ANNOTATION__BR_CNTR_WIDTH, + "Branch Counter"); + } else { + char *buf; + + if (!annotation_br_cntr_entry(&buf, al->br_cntr_nr, al->br_cntr, + al->num_aggr, al->evsel)) { + obj__printf(obj, "%*s ", ANNOTATION__BR_CNTR_WIDTH, buf); + free(buf); + } + } + } + + if (show_title && !*al->line) { + ipc_coverage_string(bf, sizeof(bf), notes); + obj__printf(obj, "%*s", ANNOTATION__AVG_IPC_WIDTH, bf); + } + } + width -= cycles_width; + + obj__printf(obj, " "); + width -= 1; + + if (!*al->line) + obj__printf(obj, "%-*s", width, " "); + else if (al->offset == -1) { + if (al->line_nr && annotate_opts.show_linenr) + printed = scnprintf(bf, sizeof(bf), "%-*d ", + notes->src->widths.addr + 1, al->line_nr); + else + printed = scnprintf(bf, sizeof(bf), "%-*s ", + notes->src->widths.addr, " "); + obj__printf(obj, bf); + width -= printed; + obj__printf(obj, "%-*s", width, al->line); + } else { + u64 addr = al->offset; + int color = -1; + + if (!annotate_opts.use_offset) + addr += notes->src->start; + + if (!annotate_opts.use_offset) { + printed = scnprintf(bf, sizeof(bf), "%" PRIx64 ": ", addr); + } else { + if (al->jump_sources && + annotate_opts.offset_level >= ANNOTATION__OFFSET_JUMP_TARGETS) { + if (annotate_opts.show_nr_jumps) { + int prev; + printed = scnprintf(bf, sizeof(bf), "%*d ", + notes->src->widths.jumps, + al->jump_sources); + prev = obj__set_jumps_percent_color(obj, al->jump_sources, + current_entry); + obj__printf(obj, bf); + obj__set_color(obj, prev); + } +print_addr: + printed = scnprintf(bf, sizeof(bf), "%*" PRIx64 ": ", + notes->src->widths.target, addr); + } else if (ins__is_call(&disasm_line(al)->ins) && + annotate_opts.offset_level >= ANNOTATION__OFFSET_CALL) { + goto print_addr; + } else if (annotate_opts.offset_level == ANNOTATION__MAX_OFFSET_LEVEL) { + goto print_addr; + } else { + printed = scnprintf(bf, sizeof(bf), "%-*s ", + notes->src->widths.addr, " "); + } + } + + if (change_color) + color = obj__set_color(obj, HE_COLORSET_ADDR); + obj__printf(obj, bf); + if (change_color) + obj__set_color(obj, color); + + width -= printed; + + printed = disasm_line__write(disasm_line(al), notes, obj, bf, sizeof(bf), + obj__printf, obj__write_graph); + + obj__printf(obj, "%s", bf); + width -= printed; + + disasm_line__snprint_type_info(disasm_line(al), bf, sizeof(bf), apd); + obj__printf(obj, "%-*s", width, bf); + } + +} + +int symbol__annotate2(struct map_symbol *ms, struct evsel *evsel, + const struct arch **parch) +{ + struct symbol *sym = ms->sym; + struct annotation *notes = symbol__annotation(sym); + size_t size = symbol__size(sym); + int err; + + err = symbol__annotate(ms, evsel, parch); + if (err) + return err; + + symbol__calc_percent(sym, evsel); + + annotation__set_index(notes); + annotation__mark_jump_targets(notes, sym); + + err = annotation__compute_ipc(notes, size, evsel); + if (err) + return err; + + annotation__init_column_widths(notes, sym); + annotation__update_column_widths(notes); + symbol__set_annotate2(sym, true); + + return 0; +} + +const char * const perf_disassembler__strs[] = { + [PERF_DISASM_UNKNOWN] = "unknown", + [PERF_DISASM_LLVM] = "llvm", + [PERF_DISASM_CAPSTONE] = "capstone", + [PERF_DISASM_OBJDUMP] = "objdump", +}; + + +static void annotation_options__add_disassembler(struct annotation_options *options, + enum perf_disassembler dis) +{ + for (u8 i = 0; i < ARRAY_SIZE(options->disassemblers); i++) { + if (options->disassemblers[i] == dis) { + /* Disassembler is already present then don't add again. */ + return; + } + if (options->disassemblers[i] == PERF_DISASM_UNKNOWN) { + /* Found a free slot. */ + options->disassemblers[i] = dis; + return; + } + } + pr_err("Failed to add disassembler %d\n", dis); +} + +static int annotation_options__add_disassemblers_str(struct annotation_options *options, + const char *str) +{ + while (str && *str != '\0') { + const char *comma = strchr(str, ','); + int len = comma ? comma - str : (int)strlen(str); + bool match = false; + + for (u8 i = 0; i < ARRAY_SIZE(perf_disassembler__strs); i++) { + const char *dis_str = perf_disassembler__strs[i]; + + if (len == (int)strlen(dis_str) && !strncmp(str, dis_str, len)) { + annotation_options__add_disassembler(options, i); + match = true; + break; + } + } + if (!match) { + pr_err("Invalid disassembler '%.*s'\n", len, str); + return -1; + } + str = comma ? comma + 1 : NULL; + } + return 0; +} + +static int annotation__config(const char *var, const char *value, void *data) +{ + struct annotation_options *opt = data; + + if (!strstarts(var, "annotate.")) + return 0; + + if (!strcmp(var, "annotate.offset_level")) { + perf_config_u8(&opt->offset_level, "offset_level", value); + + if (opt->offset_level > ANNOTATION__MAX_OFFSET_LEVEL) + opt->offset_level = ANNOTATION__MAX_OFFSET_LEVEL; + else if (opt->offset_level < ANNOTATION__MIN_OFFSET_LEVEL) + opt->offset_level = ANNOTATION__MIN_OFFSET_LEVEL; + } else if (!strcmp(var, "annotate.disassemblers")) { + int err = annotation_options__add_disassemblers_str(opt, value); + + if (err) + return err; + } else if (!strcmp(var, "annotate.hide_src_code")) { + opt->hide_src_code = perf_config_bool("hide_src_code", value); + } else if (!strcmp(var, "annotate.jump_arrows")) { + opt->jump_arrows = perf_config_bool("jump_arrows", value); + } else if (!strcmp(var, "annotate.show_linenr")) { + opt->show_linenr = perf_config_bool("show_linenr", value); + } else if (!strcmp(var, "annotate.show_nr_jumps")) { + opt->show_nr_jumps = perf_config_bool("show_nr_jumps", value); + } else if (!strcmp(var, "annotate.show_nr_samples")) { + symbol_conf.show_nr_samples = perf_config_bool("show_nr_samples", + value); + } else if (!strcmp(var, "annotate.show_total_period")) { + symbol_conf.show_total_period = perf_config_bool("show_total_period", + value); + } else if (!strcmp(var, "annotate.use_offset")) { + opt->use_offset = perf_config_bool("use_offset", value); + } else if (!strcmp(var, "annotate.disassembler_style")) { + opt->disassembler_style = strdup(value); + if (!opt->disassembler_style) { + pr_err("Not enough memory for annotate.disassembler_style\n"); + return -1; + } + } else if (!strcmp(var, "annotate.objdump")) { + opt->objdump_path = strdup(value); + if (!opt->objdump_path) { + pr_err("Not enough memory for annotate.objdump\n"); + return -1; + } + } else if (!strcmp(var, "annotate.addr2line")) { + symbol_conf.addr2line_path = strdup(value); + if (!symbol_conf.addr2line_path) { + pr_err("Not enough memory for annotate.addr2line\n"); + return -1; + } + } else if (!strcmp(var, "annotate.demangle")) { + symbol_conf.demangle = perf_config_bool("demangle", value); + } else if (!strcmp(var, "annotate.demangle_kernel")) { + symbol_conf.demangle_kernel = perf_config_bool("demangle_kernel", value); + } else { + pr_debug("%s variable unknown, ignoring...", var); + } + + return 0; +} + +void annotation_options__init(void) +{ + struct annotation_options *opt = &annotate_opts; + + memset(opt, 0, sizeof(*opt)); + + /* Default values. */ + opt->use_offset = true; + opt->jump_arrows = true; + opt->annotate_src = true; + opt->offset_level = ANNOTATION__OFFSET_JUMP_TARGETS; + opt->percent_type = PERCENT_PERIOD_LOCAL; + opt->hide_src_code = true; + opt->hide_src_code_on_title = true; +} + +void annotation_options__exit(void) +{ + zfree(&annotate_opts.disassembler_style); + zfree(&annotate_opts.objdump_path); +} + +static void annotation_options__default_init_disassemblers(struct annotation_options *options) +{ + if (options->disassemblers[0] != PERF_DISASM_UNKNOWN) { + /* Already initialized. */ + return; + } +#ifdef HAVE_LIBLLVM_SUPPORT + annotation_options__add_disassembler(options, PERF_DISASM_LLVM); +#endif +#ifdef HAVE_LIBCAPSTONE_SUPPORT + annotation_options__add_disassembler(options, PERF_DISASM_CAPSTONE); +#endif + annotation_options__add_disassembler(options, PERF_DISASM_OBJDUMP); +} + +void annotation_config__init(void) +{ + perf_config(annotation__config, &annotate_opts); + annotation_options__default_init_disassemblers(&annotate_opts); +} + +static unsigned int parse_percent_type(char *str1, char *str2) +{ + unsigned int type = (unsigned int) -1; + + if (!strcmp("period", str1)) { + if (!strcmp("local", str2)) + type = PERCENT_PERIOD_LOCAL; + else if (!strcmp("global", str2)) + type = PERCENT_PERIOD_GLOBAL; + } + + if (!strcmp("hits", str1)) { + if (!strcmp("local", str2)) + type = PERCENT_HITS_LOCAL; + else if (!strcmp("global", str2)) + type = PERCENT_HITS_GLOBAL; + } + + return type; +} + +int annotate_parse_percent_type(const struct option *opt __maybe_unused, const char *_str, + int unset __maybe_unused) +{ + unsigned int type; + char *str1, *str2; + int err = -1; + + str1 = strdup(_str); + if (!str1) + return -ENOMEM; + + str2 = strchr(str1, '-'); + if (!str2) + goto out; + + *str2++ = 0; + + type = parse_percent_type(str1, str2); + if (type == (unsigned int) -1) + type = parse_percent_type(str2, str1); + if (type != (unsigned int) -1) { + annotate_opts.percent_type = type; + err = 0; + } + +out: + free(str1); + return err; +} + +int annotate_check_args(void) +{ + struct annotation_options *args = &annotate_opts; + + if (args->prefix_strip && !args->prefix) { + pr_err("--prefix-strip requires --prefix\n"); + return -1; + } + return 0; +} + +static int arch__dwarf_regnum(const struct arch *arch, const char *str) +{ + const char *p; + char *regname, *q; + int reg; + + p = strchr(str, arch->objdump.register_char); + if (p == NULL) + return -1; + + regname = strdup(p); + if (regname == NULL) + return -1; + + q = strpbrk(regname, ",) "); + if (q) + *q = '\0'; + + reg = get_dwarf_regnum(regname, arch->id.e_machine, arch->id.e_flags); + free(regname); + return reg; +} + +/* + * Get register number and access offset from the given instruction. + * It assumes AT&T x86 asm format like OFFSET(REG). Maybe it needs + * to revisit the format when it handles different architecture. + * Fills @reg and @offset when return 0. + */ +static int extract_reg_offset(const struct arch *arch, const char *str, + struct annotated_op_loc *op_loc) +{ + char *p; + + if (arch->objdump.register_char == 0) + return -1; + + /* + * It should start from offset, but it's possible to skip 0 + * in the asm. So 0(%rax) should be same as (%rax). + * + * However, it also start with a segment select register like + * %gs:0x18(%rbx). In that case it should skip the part. + */ + if (*str == arch->objdump.register_char) { + if (arch__is_x86(arch)) { + /* FIXME: Handle other segment registers */ + if (!strncmp(str, "%gs:", 4)) + op_loc->segment = INSN_SEG_X86_GS; + } + + while (*str && !isdigit(*str) && + *str != arch->objdump.memory_ref_char) + str++; + } + + op_loc->offset = strtol(str, &p, 0); + op_loc->reg1 = arch__dwarf_regnum(arch, p); + if (op_loc->reg1 == -1) + return -1; + + /* Get the second register */ + if (op_loc->multi_regs) + op_loc->reg2 = arch__dwarf_regnum(arch, p + 1); + + return 0; +} + +/** + * annotate_get_insn_location - Get location of instruction + * @arch: the architecture info + * @dl: the target instruction + * @loc: a buffer to save the data + * + * Get detailed location info (register and offset) in the instruction. + * It needs both source and target operand and whether it accesses a + * memory location. The offset field is meaningful only when the + * corresponding mem flag is set. The reg2 field is meaningful only + * when multi_regs flag is set. + * + * Some examples on x86: + * + * mov (%rax), %rcx # src_reg1 = rax, src_mem = 1, src_offset = 0 + * # dst_reg1 = rcx, dst_mem = 0 + * + * mov 0x18, %r8 # src_reg1 = -1, src_mem = 0 + * # dst_reg1 = r8, dst_mem = 0 + * + * mov %rsi, 8(%rbx,%rcx,4) # src_reg1 = rsi, src_mem = 0, src_multi_regs = 0 + * # dst_reg1 = rbx, dst_reg2 = rcx, dst_mem = 1 + * # dst_multi_regs = 1, dst_offset = 8 + */ +int annotate_get_insn_location(const struct arch *arch, struct disasm_line *dl, + struct annotated_insn_loc *loc) +{ + struct ins_operands *ops; + struct annotated_op_loc *op_loc; + int i; + + if (ins__is_lock(&dl->ins)) + ops = dl->ops.locked.ops; + else + ops = &dl->ops; + + if (ops == NULL) + return -1; + + memset(loc, 0, sizeof(*loc)); + + for_each_insn_op_loc(loc, i, op_loc) { + const char *insn_str = ops->source.raw; + bool multi_regs = ops->source.multi_regs; + bool mem_ref = ops->source.mem_ref; + + if (i == INSN_OP_TARGET) { + insn_str = ops->target.raw; + multi_regs = ops->target.multi_regs; + mem_ref = ops->target.mem_ref; + } + + /* Invalidate the register by default */ + op_loc->reg1 = -1; + op_loc->reg2 = -1; + + if (insn_str == NULL) { + if (!arch__is_powerpc(arch)) + continue; + } + + /* + * For powerpc, call get_powerpc_regs function which extracts the + * required fields for op_loc, ie reg1, reg2, offset from the + * raw instruction. + */ + if (arch__is_powerpc(arch)) { + op_loc->mem_ref = mem_ref; + op_loc->multi_regs = multi_regs; + get_powerpc_regs(dl->raw.raw_insn, !i, op_loc); + } else if (strchr(insn_str, arch->objdump.memory_ref_char)) { + op_loc->mem_ref = true; + op_loc->multi_regs = multi_regs; + extract_reg_offset(arch, insn_str, op_loc); + } else { + const char *s = insn_str; + char *p = NULL; + + if (arch__is_x86(arch)) { + /* FIXME: Handle other segment registers */ + if (!strncmp(insn_str, "%gs:", 4)) { + op_loc->segment = INSN_SEG_X86_GS; + op_loc->offset = strtol(insn_str + 4, + &p, 0); + if (p && p != insn_str + 4) + op_loc->imm = true; + continue; + } + } + + if (*s == arch->objdump.register_char) { + op_loc->reg1 = arch__dwarf_regnum(arch, s); + } + else if (*s == arch->objdump.imm_char) { + op_loc->offset = strtol(s + 1, &p, 0); + if (p && p != s + 1) + op_loc->imm = true; + } + } + } + + return 0; +} + +static struct disasm_line *find_disasm_line(struct symbol *sym, u64 ip, + bool allow_update) +{ + struct disasm_line *dl; + struct annotation *notes; + + notes = symbol__annotation(sym); + + list_for_each_entry(dl, ¬es->src->source, al.node) { + if (dl->al.offset == -1) + continue; + + if (sym->start + dl->al.offset == ip) { + /* + * llvm-objdump places "lock" in a separate line and + * in that case, we want to get the next line. + */ + if (ins__is_lock(&dl->ins) && + *dl->ops.raw == '\0' && allow_update) { + ip++; + continue; + } + return dl; + } + } + return NULL; +} + +static struct annotated_item_stat *annotate_data_stat(struct list_head *head, + const char *name) +{ + struct annotated_item_stat *istat; + + list_for_each_entry(istat, head, list) { + if (!strcmp(istat->name, name)) + return istat; + } + + istat = zalloc(sizeof(*istat)); + if (istat == NULL) + return NULL; + + istat->name = strdup(name); + if ((istat->name == NULL) || (!strlen(istat->name))) { + free(istat); + return NULL; + } + + list_add_tail(&istat->list, head); + return istat; +} + +static bool is_stack_operation(const struct arch *arch, struct disasm_line *dl) +{ + if (arch__is_x86(arch)) { + if (!strncmp(dl->ins.name, "push", 4) || + !strncmp(dl->ins.name, "pop", 3) || + !strncmp(dl->ins.name, "call", 4) || + !strncmp(dl->ins.name, "ret", 3)) + return true; + } + + return false; +} + +static bool is_stack_canary(const struct arch *arch, struct annotated_op_loc *loc) +{ + /* On x86_64, %gs:40 is used for stack canary */ + if (arch__is_x86(arch)) { + if (loc->segment == INSN_SEG_X86_GS && loc->imm && + loc->offset == 40) + return true; + } + + return false; +} + +/** + * Returns true if the instruction has a memory operand without + * performing a load/store + */ +static bool is_address_gen_insn(const struct arch *arch, struct disasm_line *dl) +{ + if (arch__is_x86(arch)) { + if (!strncmp(dl->ins.name, "lea", 3)) + return true; + } + + return false; +} + +static struct disasm_line * +annotation__prev_asm_line(struct annotation *notes, struct disasm_line *curr) +{ + struct list_head *sources = ¬es->src->source; + struct disasm_line *prev; + + if (curr == list_first_entry(sources, struct disasm_line, al.node)) + return NULL; + + prev = list_prev_entry(curr, al.node); + while (prev->al.offset == -1 && + prev != list_first_entry(sources, struct disasm_line, al.node)) + prev = list_prev_entry(prev, al.node); + + if (prev->al.offset == -1) + return NULL; + + return prev; +} + +static struct disasm_line * +annotation__next_asm_line(struct annotation *notes, struct disasm_line *curr) +{ + struct list_head *sources = ¬es->src->source; + struct disasm_line *next; + + if (curr == list_last_entry(sources, struct disasm_line, al.node)) + return NULL; + + next = list_next_entry(curr, al.node); + while (next->al.offset == -1 && + next != list_last_entry(sources, struct disasm_line, al.node)) + next = list_next_entry(next, al.node); + + if (next->al.offset == -1) + return NULL; + + return next; +} + +u64 annotate_calc_pcrel(struct map_symbol *ms, u64 ip, int offset, + struct disasm_line *dl) +{ + struct annotation *notes; + struct disasm_line *next; + u64 addr; + + notes = symbol__annotation(ms->sym); + /* + * PC-relative addressing starts from the next instruction address + * But the IP is for the current instruction. Since disasm_line + * doesn't have the instruction size, calculate it using the next + * disasm_line. If it's the last one, we can use symbol's end + * address directly. + */ + next = annotation__next_asm_line(notes, dl); + if (next == NULL) + addr = ms->sym->end + offset; + else + addr = ip + (next->al.offset - dl->al.offset) + offset; + + return map__rip_2objdump(ms->map, addr); +} + +static struct debuginfo_cache { + struct dso *dso; + struct debuginfo *dbg; +} di_cache; + +void debuginfo_cache__delete(void) +{ + dso__put(di_cache.dso); + di_cache.dso = NULL; + + debuginfo__delete(di_cache.dbg); + di_cache.dbg = NULL; +} + +static struct annotated_data_type * +__hist_entry__get_data_type(struct hist_entry *he, const struct arch *arch, + struct debuginfo *dbg, struct disasm_line *dl, + int *type_offset) +{ + struct map_symbol *ms = &he->ms; + struct annotated_insn_loc loc; + struct annotated_op_loc *op_loc; + struct annotated_data_type *mem_type; + struct annotated_item_stat *istat; + int i; + + istat = annotate_data_stat(&ann_insn_stat, dl->ins.name); + if (istat == NULL) { + ann_data_stat.no_insn++; + return NO_TYPE; + } + + if (annotate_get_insn_location(arch, dl, &loc) < 0) { + ann_data_stat.no_insn_ops++; + istat->bad++; + return NO_TYPE; + } + + if (is_stack_operation(arch, dl)) { + istat->good++; + *type_offset = 0; + return &stackop_type; + } + + if (is_address_gen_insn(arch, dl)) { + istat->bad++; + ann_data_stat.no_mem_ops++; + return NO_TYPE; + } + + for_each_insn_op_loc(&loc, i, op_loc) { + struct data_loc_info dloc = { + .arch = arch, + .thread = he->thread, + .ms = ms, + .ip = ms->sym->start + dl->al.offset, + .cpumode = he->cpumode, + .op = op_loc, + .di = dbg, + }; + + if (!op_loc->mem_ref && op_loc->segment == INSN_SEG_NONE) + continue; + + /* PC-relative addressing */ + if (op_loc->reg1 == DWARF_REG_PC) { + dloc.var_addr = annotate_calc_pcrel(ms, dloc.ip, + op_loc->offset, dl); + } + + /* This CPU access in kernel - pretend PC-relative addressing */ + if (dso__kernel(map__dso(ms->map)) && arch__is_x86(arch) && + op_loc->segment == INSN_SEG_X86_GS && op_loc->imm) { + dloc.var_addr = op_loc->offset; + op_loc->reg1 = DWARF_REG_PC; + } + + mem_type = find_data_type(&dloc); + + if (mem_type == NULL && is_stack_canary(arch, op_loc)) { + istat->good++; + *type_offset = 0; + return &canary_type; + } + + if (mem_type) + istat->good++; + else + istat->bad++; + + if (symbol_conf.annotate_data_sample) { + struct evsel *evsel = hists_to_evsel(he->hists); + + annotated_data_type__update_samples(mem_type, evsel, + dloc.type_offset, + he->stat.nr_events, + he->stat.period); + } + *type_offset = dloc.type_offset; + return mem_type ?: NO_TYPE; + } + + /* retry with a fused instruction */ + return NULL; +} + +/** + * hist_entry__get_data_type - find data type for given hist entry + * @he: hist entry + * + * This function first annotates the instruction at @he->ip and extracts + * register and offset info from it. Then it searches the DWARF debug + * info to get a variable and type information using the address, register, + * and offset. + */ +struct annotated_data_type *hist_entry__get_data_type(struct hist_entry *he) +{ + struct map_symbol *ms = &he->ms; + struct evsel *evsel = hists_to_evsel(he->hists); + const struct arch *arch; + struct disasm_line *dl; + struct annotated_data_type *mem_type; + struct annotated_item_stat *istat; + u64 ip = he->ip; + + ann_data_stat.total++; + + if (ms->map == NULL || ms->sym == NULL) { + ann_data_stat.no_sym++; + return NULL; + } + + if (!symbol_conf.init_annotation) { + ann_data_stat.no_sym++; + return NULL; + } + + /* + * di_cache holds a pair of values, but code below assumes + * di_cache.dso can be compared/updated and di_cache.dbg can be + * read/updated independently from each other. That assumption only + * holds in single threaded code. + */ + assert(perf_singlethreaded); + + if (map__dso(ms->map) != di_cache.dso) { + dso__put(di_cache.dso); + di_cache.dso = dso__get(map__dso(ms->map)); + + debuginfo__delete(di_cache.dbg); + di_cache.dbg = dso__debuginfo(di_cache.dso); + } + + if (di_cache.dbg == NULL) { + ann_data_stat.no_dbginfo++; + return NULL; + } + + /* Make sure it has the disasm of the function */ + if (symbol__annotate(ms, evsel, &arch) < 0) { + ann_data_stat.no_insn++; + return NULL; + } + + /* + * Get a disasm to extract the location from the insn. + * This is too slow... + */ + dl = find_disasm_line(ms->sym, ip, /*allow_update=*/true); + if (dl == NULL) { + ann_data_stat.no_insn++; + return NULL; + } + +retry: + mem_type = __hist_entry__get_data_type(he, arch, di_cache.dbg, dl, + &he->mem_type_off); + if (mem_type) + return mem_type == NO_TYPE ? NULL : mem_type; + + /* + * Some instructions can be fused and the actual memory access came + * from the previous instruction. + */ + if (dl->al.offset > 0) { + struct annotation *notes; + struct disasm_line *prev_dl; + + notes = symbol__annotation(ms->sym); + prev_dl = annotation__prev_asm_line(notes, dl); + + if (prev_dl && ins__is_fused(arch, prev_dl->ins.name, dl->ins.name)) { + dl = prev_dl; + goto retry; + } + } + + ann_data_stat.no_mem_ops++; + istat = annotate_data_stat(&ann_insn_stat, dl->ins.name); + if (istat) + istat->bad++; + return NULL; +} + +/* Basic block traversal (BFS) data structure */ +struct basic_block_data { + struct list_head queue; + struct list_head visited; +}; + +/* + * During the traversal, it needs to know the parent block where the current + * block block started from. Note that single basic block can be parent of + * two child basic blocks (in case of condition jump). + */ +struct basic_block_link { + struct list_head node; + struct basic_block_link *parent; + struct annotated_basic_block *bb; +}; + +/* Check any of basic block in the list already has the offset */ +static bool basic_block_has_offset(struct list_head *head, s64 offset) +{ + struct basic_block_link *link; + + list_for_each_entry(link, head, node) { + s64 begin_offset = link->bb->begin->al.offset; + s64 end_offset = link->bb->end->al.offset; + + if (begin_offset <= offset && offset <= end_offset) + return true; + } + return false; +} + +static bool is_new_basic_block(struct basic_block_data *bb_data, + struct disasm_line *dl) +{ + s64 offset = dl->al.offset; + + if (basic_block_has_offset(&bb_data->visited, offset)) + return false; + if (basic_block_has_offset(&bb_data->queue, offset)) + return false; + return true; +} + +/* Add a basic block starting from dl and link it to the parent */ +static int add_basic_block(struct basic_block_data *bb_data, + struct basic_block_link *parent, + struct disasm_line *dl) +{ + struct annotated_basic_block *bb; + struct basic_block_link *link; + + if (dl == NULL) + return -1; + + if (!is_new_basic_block(bb_data, dl)) + return 0; + + bb = zalloc(sizeof(*bb)); + if (bb == NULL) + return -1; + + bb->begin = dl; + bb->end = dl; + INIT_LIST_HEAD(&bb->list); + + link = malloc(sizeof(*link)); + if (link == NULL) { + free(bb); + return -1; + } + + link->bb = bb; + link->parent = parent; + list_add_tail(&link->node, &bb_data->queue); + return 0; +} + +/* Returns true when it finds the target in the current basic block */ +static bool process_basic_block(struct basic_block_data *bb_data, + struct basic_block_link *link, + struct symbol *sym, u64 target) +{ + struct disasm_line *dl, *next_dl, *last_dl; + struct annotation *notes = symbol__annotation(sym); + bool found = false; + + dl = link->bb->begin; + /* Check if it's already visited */ + if (basic_block_has_offset(&bb_data->visited, dl->al.offset)) + return false; + + last_dl = list_last_entry(¬es->src->source, + struct disasm_line, al.node); + if (last_dl->al.offset == -1) + last_dl = annotation__prev_asm_line(notes, last_dl); + + if (last_dl == NULL) + return false; + + list_for_each_entry_from(dl, ¬es->src->source, al.node) { + /* Skip comment or debug info line */ + if (dl->al.offset == -1) + continue; + /* Found the target instruction */ + if (sym->start + dl->al.offset == target) { + found = true; + break; + } + /* End of the function, finish the block */ + if (dl == last_dl) + break; + /* 'return' instruction finishes the block */ + if (ins__is_ret(&dl->ins)) + break; + /* normal instructions are part of the basic block */ + if (!ins__is_jump(&dl->ins)) + continue; + /* jump to a different function, tail call or return */ + if (dl->ops.target.outside) + break; + /* jump instruction creates new basic block(s) */ + next_dl = find_disasm_line(sym, sym->start + dl->ops.target.offset, + /*allow_update=*/false); + if (next_dl) + add_basic_block(bb_data, link, next_dl); + + /* + * FIXME: determine conditional jumps properly. + * Conditional jumps create another basic block with the + * next disasm line. + */ + if (!strstr(dl->ins.name, "jmp")) { + next_dl = annotation__next_asm_line(notes, dl); + if (next_dl) + add_basic_block(bb_data, link, next_dl); + } + break; + + } + link->bb->end = dl; + return found; +} + +/* + * It founds a target basic block, build a proper linked list of basic blocks + * by following the link recursively. + */ +static void link_found_basic_blocks(struct basic_block_link *link, + struct list_head *head) +{ + while (link) { + struct basic_block_link *parent = link->parent; + + list_move(&link->bb->list, head); + list_del(&link->node); + free(link); + + link = parent; + } +} + +static void delete_basic_blocks(struct basic_block_data *bb_data) +{ + struct basic_block_link *link, *tmp; + + list_for_each_entry_safe(link, tmp, &bb_data->queue, node) { + list_del(&link->node); + zfree(&link->bb); + free(link); + } + + list_for_each_entry_safe(link, tmp, &bb_data->visited, node) { + list_del(&link->node); + zfree(&link->bb); + free(link); + } +} + +/** + * annotate_get_basic_blocks - Get basic blocks for given address range + * @sym: symbol to annotate + * @src: source address + * @dst: destination address + * @head: list head to save basic blocks + * + * This function traverses disasm_lines from @src to @dst and save them in a + * list of annotated_basic_block to @head. It uses BFS to find the shortest + * path between two. The basic_block_link is to maintain parent links so + * that it can build a list of blocks from the start. + */ +int annotate_get_basic_blocks(struct symbol *sym, s64 src, s64 dst, + struct list_head *head) +{ + struct basic_block_data bb_data = { + .queue = LIST_HEAD_INIT(bb_data.queue), + .visited = LIST_HEAD_INIT(bb_data.visited), + }; + struct basic_block_link *link; + struct disasm_line *dl; + int ret = -1; + + dl = find_disasm_line(sym, src, /*allow_update=*/false); + if (dl == NULL) + return -1; + + if (add_basic_block(&bb_data, /*parent=*/NULL, dl) < 0) + return -1; + + /* Find shortest path from src to dst using BFS */ + while (!list_empty(&bb_data.queue)) { + link = list_first_entry(&bb_data.queue, struct basic_block_link, node); + + if (process_basic_block(&bb_data, link, sym, dst)) { + link_found_basic_blocks(link, head); + ret = 0; + break; + } + list_move(&link->node, &bb_data.visited); + } + delete_basic_blocks(&bb_data); + return ret; +} diff --git a/tools/perf/util/annotate.h b/tools/perf/util/annotate.h new file mode 100644 index 000000000..1aa6df7d1 --- /dev/null +++ b/tools/perf/util/annotate.h @@ -0,0 +1,588 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ANNOTATE_H +#define __PERF_ANNOTATE_H + +#include +#include +#include +#include +#include +#include +#include +#include "symbol_conf.h" +#include "mutex.h" +#include "spark.h" +#include "hashmap.h" +#include "disasm.h" +#include "branch.h" +#include "evsel.h" + +struct hist_browser_timer; +struct hist_entry; +struct map; +struct map_symbol; +struct addr_map_symbol; +struct option; +struct perf_sample; +struct symbol; +struct annotated_data_type; + +#define ANNOTATION__IPC_WIDTH 6 +#define ANNOTATION__CYCLES_WIDTH 6 +#define ANNOTATION__MINMAX_CYCLES_WIDTH 19 +#define ANNOTATION__AVG_IPC_WIDTH 36 +#define ANNOTATION__BR_CNTR_WIDTH 30 +#define ANNOTATION_DUMMY_LEN 256 + +enum perf_disassembler { + PERF_DISASM_UNKNOWN = 0, + PERF_DISASM_LLVM, + PERF_DISASM_CAPSTONE, + PERF_DISASM_OBJDUMP, +}; +#define MAX_DISASSEMBLERS (PERF_DISASM_OBJDUMP + 1) + +struct annotation_options { + bool hide_src_code, + hide_src_code_on_title, + use_offset, + jump_arrows, + print_lines, + full_path, + show_linenr, + show_fileloc, + show_nr_jumps, + show_minmax_cycle, + show_asm_raw, + show_br_cntr, + annotate_src, + code_with_type, + full_addr; + u8 offset_level; + u8 disassemblers[MAX_DISASSEMBLERS]; + u8 disassembler_used; + int min_pcnt; + int max_lines; + int context; + char *objdump_path; + char *disassembler_style; + const char *prefix; + const char *prefix_strip; + unsigned int percent_type; +}; + +extern struct annotation_options annotate_opts; + +enum { + ANNOTATION__OFFSET_JUMP_TARGETS = 1, + ANNOTATION__OFFSET_CALL, + ANNOTATION__MAX_OFFSET_LEVEL, +}; + +#define ANNOTATION__MIN_OFFSET_LEVEL ANNOTATION__OFFSET_JUMP_TARGETS + +struct annotation; + +struct sym_hist_entry { + u64 nr_samples; + u64 period; +}; + +enum { + PERCENT_HITS_LOCAL, + PERCENT_HITS_GLOBAL, + PERCENT_PERIOD_LOCAL, + PERCENT_PERIOD_GLOBAL, + PERCENT_MAX, +}; + +struct annotation_data { + double percent[PERCENT_MAX]; + double percent_sum; + struct sym_hist_entry he; +}; + +struct cycles_info { + float ipc; + u64 avg; + u64 max; + u64 min; +}; + +struct annotation_line { + struct list_head node; + struct rb_node rb_node; + s64 offset; + char *line; + int line_nr; + char *fileloc; + char *path; + struct cycles_info *cycles; + int num_aggr; + int br_cntr_nr; + u64 *br_cntr; + struct evsel *evsel; + int jump_sources; + u32 idx; + int idx_asm; + int data_nr; + struct annotation_data data[]; +}; + +struct disasm_line { + struct ins ins; + struct ins_operands ops; + union { + u8 bytes[4]; + u32 raw_insn; + } raw; + /* This needs to be at the end. */ + struct annotation_line al; +}; + +extern const char * const perf_disassembler__strs[]; + +void annotation_line__add(struct annotation_line *al, struct list_head *head); + +static inline double annotation_data__percent(struct annotation_data *data, + unsigned int which) +{ + return which < PERCENT_MAX ? data->percent[which] : -1; +} + +static inline const char *percent_type_str(unsigned int type) +{ + static const char *str[PERCENT_MAX] = { + "local hits", + "global hits", + "local period", + "global period", + }; + + if (WARN_ON(type >= PERCENT_MAX)) + return "N/A"; + + return str[type]; +} + +static inline struct disasm_line *disasm_line(struct annotation_line *al) +{ + return al ? container_of(al, struct disasm_line, al) : NULL; +} + +/* + * Is this offset in the same function as the line it is used? + * asm functions jump to other functions, for instance. + */ +static inline bool disasm_line__has_local_offset(const struct disasm_line *dl) +{ + return dl->ops.target.offset_avail && !dl->ops.target.outside; +} + +/* + * Can we draw an arrow from the jump to its target, for instance? I.e. + * is the jump and its target in the same function? + */ +bool disasm_line__is_valid_local_jump(struct disasm_line *dl, struct symbol *sym); + +struct annotation_line * +annotation_line__next(struct annotation_line *pos, struct list_head *head); + +struct annotation_write_ops { + bool first_line, current_entry, change_color; + int width; + void *obj; + int (*set_color)(void *obj, int color); + void (*set_percent_color)(void *obj, double percent, bool current); + int (*set_jumps_percent_color)(void *obj, int nr, bool current); + void (*printf)(void *obj, const char *fmt, ...); + void (*write_graph)(void *obj, int graph); +}; + +struct annotation_print_data { + struct hist_entry *he; + struct evsel *evsel; + const struct arch *arch; + struct debuginfo *dbg; + /* save data type info keyed by al->offset */ + struct hashmap *type_hash; + /* It'll be set in hist_entry__annotate_printf() */ + int addr_fmt_width; +}; + +void annotation_line__write(struct annotation_line *al, struct annotation *notes, + const struct annotation_write_ops *ops, + struct annotation_print_data *apd); + +int __annotation__scnprintf_samples_period(struct annotation *notes, + char *bf, size_t size, + struct evsel *evsel, + bool show_freq); + +size_t disasm__fprintf(struct list_head *head, FILE *fp); +void symbol__calc_percent(struct symbol *sym, struct evsel *evsel); + +/** + * struct sym_hist - symbol histogram information for an event + * + * @nr_samples: Total number of samples. + * @period: Sum of sample periods. + */ +struct sym_hist { + u64 nr_samples; + u64 period; +}; + +/** + * struct cyc_hist - (CPU) cycle histogram for a basic block + * + * @start: Start address of current block (if known). + * @cycles: Sum of cycles for the longest basic block. + * @cycles_aggr: Total cycles for this address. + * @cycles_max: Max cycles for this address. + * @cycles_min: Min cycles for this address. + * @cycles_spark: History of cycles for the longest basic block. + * @num: Number of samples for the longest basic block. + * @num_aggr: Total number of samples for this address. + * @have_start: Whether the current branch info has a start address. + * @reset: Number of resets due to a different start address. + * + * If sample has branch_stack and cycles info, it can construct basic blocks + * between two adjacent branches. It'd have start and end addresses but + * sometimes the start address may not be available. So the cycles are + * accounted at the end address. If multiple basic blocks end at the same + * address, it will take the longest one. + * + * The @start, @cycles, @cycles_spark and @num fields are used for the longest + * block only. Other fields are used for all cases. + * + * See __symbol__account_cycles(). + */ +struct cyc_hist { + u64 start; + u64 cycles; + u64 cycles_aggr; + u64 cycles_max; + u64 cycles_min; + s64 cycles_spark[NUM_SPARKS]; + u32 num; + u32 num_aggr; + u8 have_start; + /* 1 byte padding */ + u16 reset; +}; + +/** + * struct annotated_source - symbols with hits have this attached as in annotation + * + * @source: List head for annotated_line (embeded in disasm_line). + * @histograms: Array of symbol histograms per event to maintain the total number + * of samples and period. + * @nr_histograms: This may not be the same as evsel->evlist->core.nr_entries if + * we have more than a group in a evlist, where we will want + * to see each group separately, that is why symbol__annotate2() + * sets src->nr_histograms to evsel->nr_members. + * @samples: Hash map of sym_hist_entry. Keyed by event index and offset in symbol. + * @nr_events: Number of events in the current output. + * @nr_entries: Number of annotated_line in the source list. + * @nr_asm_entries: Number of annotated_line with actual asm instruction in the + * source list. + * @max_jump_sources: Maximum number of jump instructions targeting to the same + * instruction. + * @widths: Precalculated width of each column in the TUI output. + * + * disasm_lines are allocated, percentages calculated and all sorted by percentage + * when the annotation is about to be presented, so the percentages are for + * one of the entries in the histogram array, i.e. for the event/counter being + * presented. It is deallocated right after symbol__{tui,tty,etc}_annotate + * returns. + */ +struct annotated_source { + struct list_head source; + struct sym_hist *histograms; + struct hashmap *samples; + int nr_histograms; + int nr_events; + int nr_entries; + int nr_asm_entries; + int max_jump_sources; + bool tried_source; + u64 start; + struct { + u8 addr; + u8 jumps; + u8 target; + u8 min_addr; + u8 max_addr; + u8 max_ins_name; + u16 max_line_len; + } widths; +}; + +struct annotation_line *annotated_source__get_line(struct annotated_source *src, + s64 offset); + +/* A branch counter once saturated */ +#define ANNOTATION__BR_CNTR_SATURATED_FLAG (1ULL << 63) + +/** + * struct annotated_branch - basic block and IPC information for a symbol. + * + * @hit_cycles: Total executed cycles. + * @hit_insn: Total number of instructions executed. + * @total_insn: Number of instructions in the function. + * @cover_insn: Number of distinct, actually executed instructions. + * @cycles_hist: Array of cyc_hist for each instruction. + * @max_coverage: Maximum number of covered basic block (used for block-range). + * @br_cntr: Array of the occurrences of events (branch counters) during a block. + * + * This struct is used by two different codes when the sample has branch stack + * and cycles information. annotation__compute_ipc() calculates average IPC + * using @hit_insn / @hit_cycles. The actual coverage can be calculated using + * @cover_insn / @total_insn. The @cycles_hist can give IPC for each (longest) + * basic block ends at the given address. + * process_basic_block() calculates coverage of instructions (or basic blocks) + * in the function. + */ +struct annotated_branch { + u64 hit_cycles; + u64 hit_insn; + unsigned int total_insn; + unsigned int cover_insn; + struct cyc_hist *cycles_hist; + u64 max_coverage; + u64 *br_cntr; +}; + +struct LOCKABLE annotation { + struct annotated_source *src; + struct annotated_branch *branch; +}; + +static inline void annotation__init(struct annotation *notes __maybe_unused) +{ +} +void annotation__exit(struct annotation *notes); + +void annotation__lock(struct annotation *notes) EXCLUSIVE_LOCK_FUNCTION(*notes); +void annotation__unlock(struct annotation *notes) UNLOCK_FUNCTION(*notes); +bool annotation__trylock(struct annotation *notes) EXCLUSIVE_TRYLOCK_FUNCTION(true, *notes); + +static inline int annotation__cycles_width(struct annotation *notes) +{ + if (notes->branch && annotate_opts.show_minmax_cycle) + return ANNOTATION__IPC_WIDTH + ANNOTATION__MINMAX_CYCLES_WIDTH; + + return notes->branch ? ANNOTATION__IPC_WIDTH + ANNOTATION__CYCLES_WIDTH : 0; +} + +static inline int annotation__pcnt_width(struct annotation *notes) +{ + return (symbol_conf.show_total_period ? 12 : 8) * notes->src->nr_events; +} + +static inline bool annotation_line__filter(struct annotation_line *al) +{ + return annotate_opts.hide_src_code && al->offset == -1; +} + +static inline u8 annotation__br_cntr_width(void) +{ + return annotate_opts.show_br_cntr ? ANNOTATION__BR_CNTR_WIDTH : 0; +} + +void annotation__update_column_widths(struct annotation *notes); +void annotation__toggle_full_addr(struct annotation *notes, struct map_symbol *ms); + +static inline struct sym_hist *annotated_source__histogram(struct annotated_source *src, + const struct evsel *evsel) +{ + return &src->histograms[evsel->core.idx]; +} + +static inline struct sym_hist *annotation__histogram(struct annotation *notes, + const struct evsel *evsel) +{ + return annotated_source__histogram(notes->src, evsel); +} + +static inline struct sym_hist_entry * +annotated_source__hist_entry(struct annotated_source *src, const struct evsel *evsel, u64 offset) +{ + struct sym_hist_entry *entry; + long key = offset << 16 | evsel->core.idx; + + if (!hashmap__find(src->samples, key, &entry)) + return NULL; + return entry; +} + +static inline struct annotation *symbol__annotation(struct symbol *sym) +{ + return (void *)sym - symbol_conf.priv_size; +} + +int addr_map_symbol__inc_samples(struct addr_map_symbol *ams, struct perf_sample *sample); + +struct annotated_branch *annotation__get_branch(struct annotation *notes); + +int addr_map_symbol__account_cycles(struct addr_map_symbol *ams, + struct addr_map_symbol *start, + unsigned cycles, + struct evsel *evsel, + u64 br_cntr); + +int hist_entry__inc_addr_samples(struct hist_entry *he, struct perf_sample *sample, u64 addr); + +struct annotated_source *symbol__hists(struct symbol *sym, int nr_hists); +void symbol__annotate_zero_histograms(struct symbol *sym); + +int symbol__annotate(struct map_symbol *ms, + struct evsel *evsel, + const struct arch **parch); +int symbol__annotate2(struct map_symbol *ms, + struct evsel *evsel, + const struct arch **parch); + +enum symbol_disassemble_errno { + SYMBOL_ANNOTATE_ERRNO__SUCCESS = 0, + + /* + * Choose an arbitrary negative big number not to clash with standard + * errno since SUS requires the errno has distinct positive values. + * See 'Issue 6' in the link below. + * + * http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html + */ + __SYMBOL_ANNOTATE_ERRNO__START = -10000, + + SYMBOL_ANNOTATE_ERRNO__NO_VMLINUX = __SYMBOL_ANNOTATE_ERRNO__START, + SYMBOL_ANNOTATE_ERRNO__NO_LIBOPCODES_FOR_BPF, + SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING, + SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP, + SYMBOL_ANNOTATE_ERRNO__BPF_INVALID_FILE, + SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF, + SYMBOL_ANNOTATE_ERRNO__COULDNT_DETERMINE_FILE_TYPE, + + __SYMBOL_ANNOTATE_ERRNO__END, +}; + +int symbol__strerror_disassemble(struct map_symbol *ms, int errnum, char *buf, size_t buflen); + +void symbol__annotate_zero_histogram(struct symbol *sym, struct evsel *evsel); +void symbol__annotate_decay_histogram(struct symbol *sym, struct evsel *evsel); +void annotated_source__purge(struct annotated_source *as); + +int map_symbol__annotation_dump(struct map_symbol *ms, struct evsel *evsel, + struct hist_entry *he); + +bool ui__has_annotation(void); + +int hist_entry__annotate_printf(struct hist_entry *he, struct evsel *evsel); +int hist_entry__tty_annotate(struct hist_entry *he, struct evsel *evsel); +int hist_entry__tty_annotate2(struct hist_entry *he, struct evsel *evsel); + +void annotation_options__init(void); +void annotation_options__exit(void); + +void annotation_config__init(void); + +int annotate_parse_percent_type(const struct option *opt, const char *_str, + int unset); + +int annotate_check_args(void); + +/** + * struct annotated_op_loc - Location info of instruction operand + * @reg1: First register in the operand + * @reg2: Second register in the operand + * @offset: Memory access offset in the operand + * @segment: Segment selector register + * @mem_ref: Whether the operand accesses memory + * @multi_regs: Whether the second register is used + * @imm: Whether the operand is an immediate value (in offset) + */ +struct annotated_op_loc { + int reg1; + int reg2; + int offset; + u8 segment; + bool mem_ref; + bool multi_regs; + bool imm; +}; + +enum annotated_insn_ops { + INSN_OP_SOURCE = 0, + INSN_OP_TARGET = 1, + + INSN_OP_MAX, +}; + +enum annotated_x86_segment { + INSN_SEG_NONE = 0, + + INSN_SEG_X86_CS, + INSN_SEG_X86_DS, + INSN_SEG_X86_ES, + INSN_SEG_X86_FS, + INSN_SEG_X86_GS, + INSN_SEG_X86_SS, +}; + +/** + * struct annotated_insn_loc - Location info of instruction + * @ops: Array of location info for source and target operands + */ +struct annotated_insn_loc { + struct annotated_op_loc ops[INSN_OP_MAX]; +}; + +#define for_each_insn_op_loc(insn_loc, i, op_loc) \ + for (i = INSN_OP_SOURCE, op_loc = &(insn_loc)->ops[i]; \ + i < INSN_OP_MAX; \ + i++, op_loc++) + +/* Get detailed location info in the instruction */ +int annotate_get_insn_location(const struct arch *arch, struct disasm_line *dl, + struct annotated_insn_loc *loc); + +/* Returns a data type from the sample instruction (if any) */ +struct annotated_data_type *hist_entry__get_data_type(struct hist_entry *he); + +struct annotated_item_stat { + struct list_head list; + char *name; + int good; + int bad; +}; +extern struct list_head ann_insn_stat; + +/* Calculate PC-relative address */ +u64 annotate_calc_pcrel(struct map_symbol *ms, u64 ip, int offset, + struct disasm_line *dl); + +/** + * struct annotated_basic_block - Basic block of instructions + * @list: List node + * @begin: start instruction in the block + * @end: end instruction in the block + */ +struct annotated_basic_block { + struct list_head list; + struct disasm_line *begin; + struct disasm_line *end; +}; + +/* Get a list of basic blocks from src to dst addresses */ +int annotate_get_basic_blocks(struct symbol *sym, s64 src, s64 dst, + struct list_head *head); + +void debuginfo_cache__delete(void); + +int annotation_br_cntr_entry(char **str, int br_cntr_nr, u64 *br_cntr, + int num_aggr, struct evsel *evsel); +int annotation_br_cntr_abbr_list(char **str, struct evsel *evsel, bool header); + +int thread__get_arch(struct thread *thread, const struct arch **parch); +#endif /* __PERF_ANNOTATE_H */ diff --git a/tools/perf/util/arm-spe-decoder/Build b/tools/perf/util/arm-spe-decoder/Build new file mode 100644 index 000000000..97a298d1e --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/Build @@ -0,0 +1,3 @@ +perf-util-y += arm-spe-pkt-decoder.o arm-spe-decoder.o + +CFLAGS_arm-spe-pkt-decoder.o += -I$(srctree)/tools/arch/arm64/include/ -I$(OUTPUT)arch/arm64/include/generated/ diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c new file mode 100644 index 000000000..7a3a4815f --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * arm_spe_decoder.c: ARM SPE support + */ + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../auxtrace.h" +#include "../debug.h" +#include "../util.h" + +#include "arm-spe-decoder.h" + +static u64 arm_spe_calc_ip(int index, u64 payload) +{ + u64 ns, el, val; + + /* Instruction virtual address or Branch target address */ + if (index == SPE_ADDR_PKT_HDR_INDEX_INS || + index == SPE_ADDR_PKT_HDR_INDEX_BRANCH || + index == SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH) { + ns = SPE_ADDR_PKT_GET_NS(payload); + el = SPE_ADDR_PKT_GET_EL(payload); + + /* Clean highest byte */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + + /* Fill highest byte for EL1 or EL2 (VHE) mode */ + if (ns && (el == SPE_ADDR_PKT_EL1 || el == SPE_ADDR_PKT_EL2)) + payload |= 0xffULL << SPE_ADDR_PKT_ADDR_BYTE7_SHIFT; + + /* Data access virtual address */ + } else if (index == SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT) { + + /* Clean tags */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + + /* + * Armv8 ARM (ARM DDI 0487F.c), chapter "D10.2.1 Address packet" + * defines the data virtual address payload format, the top byte + * (bits [63:56]) is assigned as top-byte tag; so we only can + * retrieve address value from bits [55:0]. + * + * According to Documentation/arch/arm64/memory.rst, if detects the + * specific pattern in bits [55:52] of payload which falls in + * the kernel space, should fixup the top byte and this allows + * perf tool to parse DSO symbol for data address correctly. + * + * For this reason, if detects the bits [55:52] is 0xf, will + * fill 0xff into the top byte. + */ + val = SPE_ADDR_PKT_ADDR_GET_BYTE_6(payload); + if ((val & 0xf0ULL) == 0xf0ULL) + payload |= 0xffULL << SPE_ADDR_PKT_ADDR_BYTE7_SHIFT; + + /* Data access physical address */ + } else if (index == SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS) { + /* Clean highest byte */ + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + } else { + static u32 seen_idx = 0; + if (!(seen_idx & BIT(index))) { + seen_idx |= BIT(index); + pr_warning("ignoring unsupported address packet index: 0x%x\n", index); + } + } + + return payload; +} + +struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params) +{ + struct arm_spe_decoder *decoder; + + if (!params->get_trace) + return NULL; + + decoder = zalloc(sizeof(struct arm_spe_decoder)); + if (!decoder) + return NULL; + + decoder->get_trace = params->get_trace; + decoder->data = params->data; + + return decoder; +} + +void arm_spe_decoder_free(struct arm_spe_decoder *decoder) +{ + free(decoder); +} + +static int arm_spe_get_data(struct arm_spe_decoder *decoder) +{ + struct arm_spe_buffer buffer = { .buf = 0, }; + int ret; + + pr_debug("Getting more data\n"); + ret = decoder->get_trace(&buffer, decoder->data); + if (ret < 0) + return ret; + + decoder->buf = buffer.buf; + decoder->len = buffer.len; + + if (!decoder->len) + pr_debug("No more data\n"); + + return decoder->len; +} + +static int arm_spe_get_next_packet(struct arm_spe_decoder *decoder, + struct arm_spe_pkt *packet) +{ + int ret; + + do { + if (!decoder->len) { + ret = arm_spe_get_data(decoder); + + /* Failed to read out trace data */ + if (ret <= 0) + return ret; + } + + ret = arm_spe_get_packet(decoder->buf, decoder->len, + packet, decoder->midr); + + if (ret <= 0) { + /* Move forward for 1 byte */ + decoder->buf += 1; + decoder->len -= 1; + return -EBADMSG; + } + + decoder->buf += ret; + decoder->len -= ret; + } while (packet->type == ARM_SPE_PAD); + + return 1; +} + +static int arm_spe_read_record(struct arm_spe_decoder *decoder) +{ + int err; + int idx; + u64 payload, ip; + struct arm_spe_pkt packet; + + memset(&decoder->record, 0x0, sizeof(decoder->record)); + decoder->record.context_id = (u64)-1; + + while (1) { + err = arm_spe_get_next_packet(decoder, &packet); + if (err <= 0) + return err; + + idx = packet.index; + payload = packet.payload; + + switch (packet.type) { + case ARM_SPE_TIMESTAMP: + decoder->record.timestamp = payload; + return 1; + case ARM_SPE_END: + return 1; + case ARM_SPE_ADDRESS: + ip = arm_spe_calc_ip(idx, payload); + if (idx == SPE_ADDR_PKT_HDR_INDEX_INS) + decoder->record.from_ip = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_BRANCH) + decoder->record.to_ip = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT) + decoder->record.virt_addr = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS) + decoder->record.phys_addr = ip; + else if (idx == SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH) + decoder->record.prev_br_tgt = ip; + break; + case ARM_SPE_COUNTER: + if (idx == SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT) + decoder->record.latency = payload; + break; + case ARM_SPE_CONTEXT: + decoder->record.context_id = payload; + break; + case ARM_SPE_OP_TYPE: + switch (idx) { + case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: + decoder->record.op |= ARM_SPE_OP_LDST; + if (payload & SPE_OP_PKT_ST) + decoder->record.op |= ARM_SPE_OP_ST; + else + decoder->record.op |= ARM_SPE_OP_LD; + + if (SPE_OP_PKT_LDST_SUBCLASS_GP_REG(payload)) { + decoder->record.op |= ARM_SPE_OP_GP_REG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(payload)) { + decoder->record.op |= ARM_SPE_OP_SIMD_FP; + } else if (SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(payload)) { + decoder->record.op |= ARM_SPE_OP_UNSPEC_REG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(payload)) { + decoder->record.op |= ARM_SPE_OP_NV_SYSREG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(payload)) { + decoder->record.op |= ARM_SPE_OP_MTE_TAG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(payload)) { + if (payload & SPE_OP_PKT_AR) + decoder->record.op |= ARM_SPE_OP_AR; + if (payload & SPE_OP_PKT_EXCL) + decoder->record.op |= ARM_SPE_OP_EXCL; + if (payload & SPE_OP_PKT_AT) + decoder->record.op |= ARM_SPE_OP_ATOMIC; + } else if (SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(payload)) { + decoder->record.op |= ARM_SPE_OP_SVE; + if (payload & SPE_OP_PKT_SVE_PRED) + decoder->record.op |= ARM_SPE_OP_PRED; + if (payload & SPE_OP_PKT_SVE_SG) + decoder->record.op |= ARM_SPE_OP_SG; + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(payload)) { + decoder->record.op |= ARM_SPE_OP_MEMCPY; + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMSET(payload)) { + decoder->record.op |= ARM_SPE_OP_MEMSET; + } else if (SPE_OP_PKT_LDST_SUBCLASS_GCS(payload)) { + decoder->record.op |= ARM_SPE_OP_GCS; + if (payload & SPE_OP_PKT_GCS_COMM) + decoder->record.op |= ARM_SPE_OP_COMM; + } + + break; + case SPE_OP_PKT_HDR_CLASS_OTHER: + decoder->record.op |= ARM_SPE_OP_OTHER; + if (SPE_OP_PKT_OTHER_SUBCLASS_SVE(payload)) { + decoder->record.op |= ARM_SPE_OP_SVE | ARM_SPE_OP_DP; + if (payload & SPE_OP_PKT_OTHER_FP) + decoder->record.op |= ARM_SPE_OP_FP; + if (payload & SPE_OP_PKT_SVE_PRED) + decoder->record.op |= ARM_SPE_OP_PRED; + } else if (SPE_OP_PKT_OTHER_SUBCLASS_SME(payload)) { + decoder->record.op |= ARM_SPE_OP_SME; + if (payload & SPE_OP_PKT_OTHER_FP) + decoder->record.op |= ARM_SPE_OP_FP; + } else if (SPE_OP_PKT_OTHER_SUBCLASS_OTHER(payload)) { + if (payload & SPE_OP_PKT_OTHER_ASE) + decoder->record.op |= ARM_SPE_OP_ASE; + if (payload & SPE_OP_PKT_OTHER_FP) + decoder->record.op |= ARM_SPE_OP_FP; + if (payload & SPE_OP_PKT_COND) + decoder->record.op |= ARM_SPE_OP_COND; + } + break; + case SPE_OP_PKT_HDR_CLASS_BR_ERET: + decoder->record.op |= ARM_SPE_OP_BRANCH_ERET; + if (payload & SPE_OP_PKT_COND) + decoder->record.op |= ARM_SPE_OP_BR_COND; + if (payload & SPE_OP_PKT_INDIRECT_BRANCH) + decoder->record.op |= ARM_SPE_OP_BR_INDIRECT; + if (payload & SPE_OP_PKT_GCS) + decoder->record.op |= ARM_SPE_OP_BR_GCS; + if (SPE_OP_PKT_CR_BL(payload)) + decoder->record.op |= ARM_SPE_OP_BR_CR_BL; + if (SPE_OP_PKT_CR_RET(payload)) + decoder->record.op |= ARM_SPE_OP_BR_CR_RET; + if (SPE_OP_PKT_CR_NON_BL_RET(payload)) + decoder->record.op |= ARM_SPE_OP_BR_CR_NON_BL_RET; + break; + default: + pr_err("Get packet error!\n"); + return -1; + } + break; + case ARM_SPE_EVENTS: + decoder->record.type = payload; + break; + case ARM_SPE_DATA_SOURCE: + decoder->record.source = payload; + break; + case ARM_SPE_BAD: + break; + case ARM_SPE_PAD: + break; + default: + pr_err("Get packet error!\n"); + return -1; + } + } + + return 0; +} + +int arm_spe_decode(struct arm_spe_decoder *decoder) +{ + return arm_spe_read_record(decoder); +} diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h new file mode 100644 index 000000000..0cbcb501e --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-decoder.h @@ -0,0 +1,158 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * arm_spe_decoder.h: Arm Statistical Profiling Extensions support + * Copyright (c) 2019-2020, Arm Ltd. + */ + +#ifndef INCLUDE__ARM_SPE_DECODER_H__ +#define INCLUDE__ARM_SPE_DECODER_H__ + +#include +#include +#include + +#include "arm-spe-pkt-decoder.h" + +#define ARM_SPE_L1D_ACCESS BIT(EV_L1D_ACCESS) +#define ARM_SPE_L1D_MISS BIT(EV_L1D_REFILL) +#define ARM_SPE_LLC_ACCESS BIT(EV_LLC_ACCESS) +#define ARM_SPE_LLC_MISS BIT(EV_LLC_MISS) +#define ARM_SPE_TLB_ACCESS BIT(EV_TLB_ACCESS) +#define ARM_SPE_TLB_MISS BIT(EV_TLB_WALK) +#define ARM_SPE_BRANCH_MISS BIT(EV_MISPRED) +#define ARM_SPE_BRANCH_NOT_TAKEN BIT(EV_NOT_TAKEN) +#define ARM_SPE_REMOTE_ACCESS BIT(EV_REMOTE_ACCESS) +#define ARM_SPE_SVE_PARTIAL_PRED BIT(EV_PARTIAL_PREDICATE) +#define ARM_SPE_SVE_EMPTY_PRED BIT(EV_EMPTY_PREDICATE) +#define ARM_SPE_IN_TXN BIT(EV_TRANSACTIONAL) +#define ARM_SPE_L2D_ACCESS BIT(EV_L2D_ACCESS) +#define ARM_SPE_L2D_MISS BIT(EV_L2D_MISS) +#define ARM_SPE_RECENTLY_FETCHED BIT(EV_RECENTLY_FETCHED) +#define ARM_SPE_DATA_SNOOPED BIT(EV_DATA_SNOOPED) +#define ARM_SPE_HITM BIT(EV_CACHE_DATA_MODIFIED) + +enum arm_spe_op_type { + /* First level operation type */ + ARM_SPE_OP_OTHER = 1 << 0, + ARM_SPE_OP_LDST = 1 << 1, + ARM_SPE_OP_BRANCH_ERET = 1 << 2, +}; + +enum arm_spe_2nd_op_ldst { + ARM_SPE_OP_GP_REG = 1 << 8, + ARM_SPE_OP_UNSPEC_REG = 1 << 9, + ARM_SPE_OP_NV_SYSREG = 1 << 10, + ARM_SPE_OP_SIMD_FP = 1 << 11, + ARM_SPE_OP_SVE = 1 << 12, + ARM_SPE_OP_MTE_TAG = 1 << 13, + ARM_SPE_OP_MEMCPY = 1 << 14, + ARM_SPE_OP_MEMSET = 1 << 15, + ARM_SPE_OP_GCS = 1 << 16, + ARM_SPE_OP_SME = 1 << 17, + ARM_SPE_OP_ASE = 1 << 18, + + /* Assisted information for memory / SIMD */ + ARM_SPE_OP_LD = 1 << 20, + ARM_SPE_OP_ST = 1 << 21, + ARM_SPE_OP_ATOMIC = 1 << 22, + ARM_SPE_OP_EXCL = 1 << 23, + ARM_SPE_OP_AR = 1 << 24, + ARM_SPE_OP_DP = 1 << 25, /* Data processing */ + ARM_SPE_OP_PRED = 1 << 26, /* Predicated */ + ARM_SPE_OP_SG = 1 << 27, /* Gather/Scatter */ + ARM_SPE_OP_COMM = 1 << 28, /* Common */ + ARM_SPE_OP_FP = 1 << 29, /* Floating-point */ + ARM_SPE_OP_COND = 1 << 30, /* Conditional */ +}; + +enum arm_spe_2nd_op_branch { + ARM_SPE_OP_BR_COND = 1 << 8, + ARM_SPE_OP_BR_INDIRECT = 1 << 9, + ARM_SPE_OP_BR_GCS = 1 << 10, + ARM_SPE_OP_BR_CR_BL = 1 << 11, + ARM_SPE_OP_BR_CR_RET = 1 << 12, + ARM_SPE_OP_BR_CR_NON_BL_RET = 1 << 13, +}; + +enum arm_spe_common_data_source { + ARM_SPE_COMMON_DS_L1D = 0x0, + ARM_SPE_COMMON_DS_L2 = 0x8, + ARM_SPE_COMMON_DS_PEER_CORE = 0x9, + ARM_SPE_COMMON_DS_LOCAL_CLUSTER = 0xa, + ARM_SPE_COMMON_DS_SYS_CACHE = 0xb, + ARM_SPE_COMMON_DS_PEER_CLUSTER = 0xc, + ARM_SPE_COMMON_DS_REMOTE = 0xd, + ARM_SPE_COMMON_DS_DRAM = 0xe, +}; + +enum arm_spe_ampereone_data_source { + ARM_SPE_AMPEREONE_LOCAL_CHIP_CACHE_OR_DEVICE = 0x0, + ARM_SPE_AMPEREONE_SLC = 0x3, + ARM_SPE_AMPEREONE_REMOTE_CHIP_CACHE = 0x5, + ARM_SPE_AMPEREONE_DDR = 0x7, + ARM_SPE_AMPEREONE_L1D = 0x8, + ARM_SPE_AMPEREONE_L2D = 0x9, +}; + +enum arm_spe_hisi_hip_data_source { + ARM_SPE_HISI_HIP_PEER_CPU = 0, + ARM_SPE_HISI_HIP_PEER_CPU_HITM = 1, + ARM_SPE_HISI_HIP_L3 = 2, + ARM_SPE_HISI_HIP_L3_HITM = 3, + ARM_SPE_HISI_HIP_PEER_CLUSTER = 4, + ARM_SPE_HISI_HIP_PEER_CLUSTER_HITM = 5, + ARM_SPE_HISI_HIP_REMOTE_SOCKET = 6, + ARM_SPE_HISI_HIP_REMOTE_SOCKET_HITM = 7, + ARM_SPE_HISI_HIP_LOCAL_MEM = 8, + ARM_SPE_HISI_HIP_REMOTE_MEM = 9, + ARM_SPE_HISI_HIP_NC_DEV = 13, + ARM_SPE_HISI_HIP_L2 = 16, + ARM_SPE_HISI_HIP_L2_HITM = 17, + ARM_SPE_HISI_HIP_L1 = 18, +}; + +struct arm_spe_record { + u64 type; + int err; + u32 op; + u32 latency; + u64 from_ip; + u64 to_ip; + u64 prev_br_tgt; + u64 timestamp; + u64 virt_addr; + u64 phys_addr; + u64 context_id; + u16 source; +}; + +struct arm_spe_insn; + +struct arm_spe_buffer { + const unsigned char *buf; + size_t len; + u64 offset; + u64 trace_nr; +}; + +struct arm_spe_params { + int (*get_trace)(struct arm_spe_buffer *buffer, void *data); + void *data; +}; + +struct arm_spe_decoder { + int (*get_trace)(struct arm_spe_buffer *buffer, void *data); + void *data; + struct arm_spe_record record; + + const unsigned char *buf; + size_t len; + u64 midr; +}; + +struct arm_spe_decoder *arm_spe_decoder_new(struct arm_spe_params *params); +void arm_spe_decoder_free(struct arm_spe_decoder *decoder); + +int arm_spe_decode(struct arm_spe_decoder *decoder); + +#endif diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c new file mode 100644 index 000000000..600677318 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.c @@ -0,0 +1,614 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arm-spe-pkt-decoder.h" + +#include "../../arm64/include/asm/cputype.h" + +static const char * const arm_spe_packet_name[] = { + [ARM_SPE_PAD] = "PAD", + [ARM_SPE_END] = "END", + [ARM_SPE_TIMESTAMP] = "TS", + [ARM_SPE_ADDRESS] = "ADDR", + [ARM_SPE_COUNTER] = "LAT", + [ARM_SPE_CONTEXT] = "CONTEXT", + [ARM_SPE_OP_TYPE] = "OP-TYPE", + [ARM_SPE_EVENTS] = "EVENTS", + [ARM_SPE_DATA_SOURCE] = "DATA-SOURCE", +}; + +const char *arm_spe_pkt_name(enum arm_spe_pkt_type type) +{ + return arm_spe_packet_name[type]; +} + +/* + * Extracts the field "sz" from header bits and converts to bytes: + * 00 : byte (1) + * 01 : halfword (2) + * 10 : word (4) + * 11 : doubleword (8) + */ +static unsigned int arm_spe_payload_len(unsigned char hdr) +{ + return 1U << ((hdr & GENMASK_ULL(5, 4)) >> 4); +} + +static int arm_spe_get_payload(const unsigned char *buf, size_t len, + unsigned char ext_hdr, + struct arm_spe_pkt *packet) +{ + size_t payload_len = arm_spe_payload_len(buf[ext_hdr]); + + if (len < 1 + ext_hdr + payload_len) + return ARM_SPE_NEED_MORE_BYTES; + + buf += 1 + ext_hdr; + + switch (payload_len) { + case 1: packet->payload = *(uint8_t *)buf; break; + case 2: packet->payload = get_unaligned_le16(buf); break; + case 4: packet->payload = get_unaligned_le32(buf); break; + case 8: packet->payload = get_unaligned_le64(buf); break; + default: return ARM_SPE_BAD_PACKET; + } + + return 1 + ext_hdr + payload_len; +} + +static int arm_spe_get_pad(struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_PAD; + return 1; +} + +static int arm_spe_get_alignment(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + unsigned int alignment = 1 << ((buf[0] & 0xf) + 1); + + if (len < alignment) + return ARM_SPE_NEED_MORE_BYTES; + + packet->type = ARM_SPE_PAD; + return alignment - (((uintptr_t)buf) & (alignment - 1)); +} + +static int arm_spe_get_end(struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_END; + return 1; +} + +static int arm_spe_get_timestamp(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_TIMESTAMP; + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_events(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_EVENTS; + + /* we use index to identify Events with a less number of + * comparisons in arm_spe_pkt_desc(): E.g., the LLC-ACCESS, + * LLC-REFILL, and REMOTE-ACCESS events are identified if + * index > 1. + */ + packet->index = arm_spe_payload_len(buf[0]); + + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_data_source(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_DATA_SOURCE; + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_context(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_CONTEXT; + packet->index = SPE_CTX_PKT_HDR_INDEX(buf[0]); + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_op_type(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_OP_TYPE; + packet->index = SPE_OP_PKT_HDR_CLASS(buf[0]); + return arm_spe_get_payload(buf, len, 0, packet); +} + +static int arm_spe_get_counter(const unsigned char *buf, size_t len, + const unsigned char ext_hdr, struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_COUNTER; + + if (ext_hdr) + packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); + else + packet->index = SPE_HDR_SHORT_INDEX(buf[0]); + + return arm_spe_get_payload(buf, len, ext_hdr, packet); +} + +static int arm_spe_get_addr(const unsigned char *buf, size_t len, + const unsigned char ext_hdr, struct arm_spe_pkt *packet) +{ + packet->type = ARM_SPE_ADDRESS; + + if (ext_hdr) + packet->index = SPE_HDR_EXTENDED_INDEX(buf[0], buf[1]); + else + packet->index = SPE_HDR_SHORT_INDEX(buf[0]); + + return arm_spe_get_payload(buf, len, ext_hdr, packet); +} + +static int arm_spe_do_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet) +{ + unsigned int hdr; + unsigned char ext_hdr = 0; + + memset(packet, 0, sizeof(struct arm_spe_pkt)); + + if (!len) + return ARM_SPE_NEED_MORE_BYTES; + + hdr = buf[0]; + + if (hdr == SPE_HEADER0_PAD) + return arm_spe_get_pad(packet); + + if (hdr == SPE_HEADER0_END) /* no timestamp at end of record */ + return arm_spe_get_end(packet); + + if (hdr == SPE_HEADER0_TIMESTAMP) + return arm_spe_get_timestamp(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_EVENTS) + return arm_spe_get_events(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK1) == SPE_HEADER0_SOURCE) + return arm_spe_get_data_source(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_CONTEXT) + return arm_spe_get_context(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_OP_TYPE) + return arm_spe_get_op_type(buf, len, packet); + + if ((hdr & SPE_HEADER0_MASK2) == SPE_HEADER0_EXTENDED) { + /* 16-bit extended format header */ + if (len == 1) + return ARM_SPE_BAD_PACKET; + + ext_hdr = 1; + hdr = buf[1]; + if (hdr == SPE_HEADER1_ALIGNMENT) + return arm_spe_get_alignment(buf, len, packet); + } + + /* + * The short format header's byte 0 or the extended format header's + * byte 1 has been assigned to 'hdr', which uses the same encoding for + * address packet and counter packet, so don't need to distinguish if + * it's short format or extended format and handle in once. + */ + if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_ADDRESS) + return arm_spe_get_addr(buf, len, ext_hdr, packet); + + if ((hdr & SPE_HEADER0_MASK3) == SPE_HEADER0_COUNTER) + return arm_spe_get_counter(buf, len, ext_hdr, packet); + + return ARM_SPE_BAD_PACKET; +} + +int arm_spe_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet, u64 midr) +{ + int ret; + + ret = arm_spe_do_get_packet(buf, len, packet); + packet->midr = midr; + /* put multiple consecutive PADs on the same line, up to + * the fixed-width output format of 16 bytes per line. + */ + if (ret > 0 && packet->type == ARM_SPE_PAD) { + while (ret < 16 && len > (size_t)ret && !buf[ret]) + ret += 1; + } + return ret; +} + +static int arm_spe_pkt_out_string(int *err, char **buf_p, size_t *blen, + const char *fmt, ...) +{ + va_list ap; + int ret; + + /* Bail out if any error occurred */ + if (err && *err) + return *err; + + va_start(ap, fmt); + ret = vsnprintf(*buf_p, *blen, fmt, ap); + va_end(ap); + + if (ret < 0) { + if (err && !*err) + *err = ret; + + /* + * A return value of *blen or more means that the output was + * truncated and the buffer is overrun. + */ + } else if ((size_t)ret >= *blen) { + (*buf_p)[*blen - 1] = '\0'; + + /* + * Set *err to 'ret' to avoid overflow if tries to + * fill this buffer sequentially. + */ + if (err && !*err) + *err = ret; + } else { + *buf_p += ret; + *blen -= ret; + } + + return ret; +} + +struct ev_string { + u8 event; + const char *desc; +}; + +static const struct ev_string common_ev_strings[] = { + { .event = EV_EXCEPTION_GEN, .desc = "EXCEPTION-GEN" }, + { .event = EV_RETIRED, .desc = "RETIRED" }, + { .event = EV_L1D_ACCESS, .desc = "L1D-ACCESS" }, + { .event = EV_L1D_REFILL, .desc = "L1D-REFILL" }, + { .event = EV_TLB_ACCESS, .desc = "TLB-ACCESS" }, + { .event = EV_TLB_WALK, .desc = "TLB-REFILL" }, + { .event = EV_NOT_TAKEN, .desc = "NOT-TAKEN" }, + { .event = EV_MISPRED, .desc = "MISPRED" }, + { .event = EV_LLC_ACCESS, .desc = "LLC-ACCESS" }, + { .event = EV_LLC_MISS, .desc = "LLC-REFILL" }, + { .event = EV_REMOTE_ACCESS, .desc = "REMOTE-ACCESS" }, + { .event = EV_ALIGNMENT, .desc = "ALIGNMENT" }, + { .event = EV_TRANSACTIONAL, .desc = "TXN" }, + { .event = EV_PARTIAL_PREDICATE, .desc = "SVE-PARTIAL-PRED" }, + { .event = EV_EMPTY_PREDICATE, .desc = "SVE-EMPTY-PRED" }, + { .event = EV_L2D_ACCESS, .desc = "L2D-ACCESS" }, + { .event = EV_L2D_MISS, .desc = "L2D-MISS" }, + { .event = EV_CACHE_DATA_MODIFIED, .desc = "HITM" }, + { .event = EV_RECENTLY_FETCHED, .desc = "LFB" }, + { .event = EV_DATA_SNOOPED, .desc = "SNOOPED" }, + { .event = EV_STREAMING_SVE_MODE, .desc = "STREAMING-SVE" }, + { .event = EV_SMCU, .desc = "SMCU" }, + { .event = 0, .desc = NULL }, +}; + +static const struct ev_string n1_event_strings[] = { + { .event = 12, .desc = "LATE-PREFETCH" }, + { .event = 0, .desc = NULL }, +}; + +static u64 print_event_list(int *err, char **buf, size_t *buf_len, + const struct ev_string *ev_strings, u64 payload) +{ + for (const struct ev_string *ev = ev_strings; ev->desc != NULL; ev++) { + if (payload & BIT_ULL(ev->event)) + arm_spe_pkt_out_string(err, buf, buf_len, " %s", ev->desc); + payload &= ~BIT_ULL(ev->event); + } + return payload; +} + +struct event_print_handle { + const struct midr_range *midr_ranges; + const struct ev_string *ev_strings; +}; + +#define EV_PRINT(range, strings) \ + { \ + .midr_ranges = range, \ + .ev_strings = strings, \ + } + +static const struct midr_range n1_event_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + {}, +}; + +static const struct event_print_handle event_print_handles[] = { + EV_PRINT(n1_event_encoding_cpus, n1_event_strings), +}; + +static int arm_spe_pkt_desc_event(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + int err = 0; + + arm_spe_pkt_out_string(&err, &buf, &buf_len, "EV"); + payload = print_event_list(&err, &buf, &buf_len, common_ev_strings, + payload); + + /* Try to decode IMPDEF bits for known CPUs */ + for (unsigned int i = 0; i < ARRAY_SIZE(event_print_handles); i++) { + if (is_midr_in_range_list(packet->midr, + event_print_handles[i].midr_ranges)) + payload = print_event_list(&err, &buf, &buf_len, + event_print_handles[i].ev_strings, + payload); + } + + /* + * Print remaining IMPDEF bits that weren't printed above as raw + * "IMPDEF:1,2,3,4" etc. + */ + if (payload) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IMPDEF:"); + for (int i = 0; i < 64; i++) { + const char *sep = payload & (payload - 1) ? "," : ""; + + if (payload & BIT_ULL(i)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%d%s", i, + sep); + payload &= ~BIT_ULL(i); + } + } + } + + return err; +} + +static int arm_spe_pkt_desc_op_type(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + int err = 0; + + switch (packet->index) { + case SPE_OP_PKT_HDR_CLASS_OTHER: + if (SPE_OP_PKT_OTHER_SUBCLASS_SVE(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "SVE-OTHER"); + + /* SVE effective vector length */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", + SPE_OP_PKG_SVE_EVL(payload)); + + if (payload & SPE_OP_PKT_SVE_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + if (payload & SPE_OP_PKT_SVE_PRED) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); + } else if (SPE_OP_PKT_OTHER_SUBCLASS_SME(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "SME-OTHER"); + + /* SME effective vector length or tile size */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " ETS %d", + SPE_OP_PKG_SME_ETS(payload)); + + if (payload & SPE_OP_PKT_OTHER_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + } else if (SPE_OP_PKT_OTHER_SUBCLASS_OTHER(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, "OTHER"); + if (payload & SPE_OP_PKT_OTHER_ASE) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " ASE"); + if (payload & SPE_OP_PKT_OTHER_FP) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " FP"); + arm_spe_pkt_out_string(&err, &buf, &buf_len, " %s", + payload & SPE_OP_PKT_COND ? + "COND-SELECT" : "INSN-OTHER"); + } + break; + case SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC: + arm_spe_pkt_out_string(&err, &buf, &buf_len, + payload & 0x1 ? "ST" : "LD"); + + if (SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(payload)) { + if (payload & SPE_OP_PKT_AT) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " AT"); + if (payload & SPE_OP_PKT_EXCL) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EXCL"); + if (payload & SPE_OP_PKT_AR) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " AR"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SIMD-FP"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_GP_REG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GP-REG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " UNSPEC-REG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " NV-SYSREG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MTE-TAG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMCPY"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_MEMSET(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " MEMSET"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SVE-SME-REG"); + + /* SVE effective vector length */ + arm_spe_pkt_out_string(&err, &buf, &buf_len, " EVLEN %d", + SPE_OP_PKG_SVE_EVL(payload)); + + if (payload & SPE_OP_PKT_SVE_PRED) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " PRED"); + if (payload & SPE_OP_PKT_SVE_SG) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " SG"); + } else if (SPE_OP_PKT_LDST_SUBCLASS_GCS(payload)) { + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); + if (payload & SPE_OP_PKT_GCS_COMM) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " COMM"); + } + break; + case SPE_OP_PKT_HDR_CLASS_BR_ERET: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "B"); + + if (payload & SPE_OP_PKT_COND) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " COND"); + if (payload & SPE_OP_PKT_INDIRECT_BRANCH) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " IND"); + if (payload & SPE_OP_PKT_GCS) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " GCS"); + if (SPE_OP_PKT_CR_BL(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-BL"); + if (SPE_OP_PKT_CR_RET(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-RET"); + if (SPE_OP_PKT_CR_NON_BL_RET(payload)) + arm_spe_pkt_out_string(&err, &buf, &buf_len, " CR-NON-BL-RET"); + break; + default: + /* Unknown index */ + err = -1; + break; + } + + return err; +} + +static int arm_spe_pkt_desc_addr(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + int ns, el, idx = packet->index; + int ch, pat; + u64 payload = packet->payload; + int err = 0; + static const char *idx_name[] = {"PC", "TGT", "VA", "PA", "PBT"}; + + switch (idx) { + case SPE_ADDR_PKT_HDR_INDEX_INS: + case SPE_ADDR_PKT_HDR_INDEX_BRANCH: + case SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH: + ns = !!SPE_ADDR_PKT_GET_NS(payload); + el = SPE_ADDR_PKT_GET_EL(payload); + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "%s 0x%llx el%d ns=%d", + idx_name[idx], payload, el, ns); + break; + case SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "VA 0x%llx", payload); + break; + case SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS: + ns = !!SPE_ADDR_PKT_GET_NS(payload); + ch = !!SPE_ADDR_PKT_GET_CH(payload); + pat = SPE_ADDR_PKT_GET_PAT(payload); + payload = SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(payload); + arm_spe_pkt_out_string(&err, &buf, &buf_len, + "PA 0x%llx ns=%d ch=%d pat=%x", + payload, ns, ch, pat); + break; + default: + /* Unknown index */ + err = -1; + break; + } + + return err; +} + +static int arm_spe_pkt_desc_counter(const struct arm_spe_pkt *packet, + char *buf, size_t buf_len) +{ + u64 payload = packet->payload; + const char *name = arm_spe_pkt_name(packet->type); + int err = 0; + + arm_spe_pkt_out_string(&err, &buf, &buf_len, "%s %d ", name, + (unsigned short)payload); + + switch (packet->index) { + case SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "TOT"); + break; + case SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "ISSUE"); + break; + case SPE_CNT_PKT_HDR_INDEX_TRANS_LAT: + arm_spe_pkt_out_string(&err, &buf, &buf_len, "XLAT"); + break; + default: + break; + } + + return err; +} + +int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, + size_t buf_len) +{ + int idx = packet->index; + unsigned long long payload = packet->payload; + const char *name = arm_spe_pkt_name(packet->type); + char *buf_orig = buf; + size_t blen = buf_len; + int err = 0; + + switch (packet->type) { + case ARM_SPE_BAD: + case ARM_SPE_PAD: + case ARM_SPE_END: + arm_spe_pkt_out_string(&err, &buf, &blen, "%s", name); + break; + case ARM_SPE_EVENTS: + err = arm_spe_pkt_desc_event(packet, buf, buf_len); + break; + case ARM_SPE_OP_TYPE: + err = arm_spe_pkt_desc_op_type(packet, buf, buf_len); + break; + case ARM_SPE_DATA_SOURCE: + case ARM_SPE_TIMESTAMP: + arm_spe_pkt_out_string(&err, &buf, &blen, "%s %lld", name, payload); + break; + case ARM_SPE_ADDRESS: + err = arm_spe_pkt_desc_addr(packet, buf, buf_len); + break; + case ARM_SPE_CONTEXT: + arm_spe_pkt_out_string(&err, &buf, &blen, "%s 0x%lx el%d", + name, (unsigned long)payload, idx + 1); + break; + case ARM_SPE_COUNTER: + err = arm_spe_pkt_desc_counter(packet, buf, buf_len); + break; + default: + /* Unknown packet type */ + err = -1; + break; + } + + /* Output raw data if detect any error */ + if (err) { + err = 0; + arm_spe_pkt_out_string(&err, &buf_orig, &buf_len, "%s 0x%llx (%d)", + name, payload, packet->index); + } + + return err; +} diff --git a/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h new file mode 100644 index 000000000..a3300bec4 --- /dev/null +++ b/tools/perf/util/arm-spe-decoder/arm-spe-pkt-decoder.h @@ -0,0 +1,191 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#ifndef INCLUDE__ARM_SPE_PKT_DECODER_H__ +#define INCLUDE__ARM_SPE_PKT_DECODER_H__ + +#include +#include +#include + +#define ARM_SPE_PKT_DESC_MAX 512 + +#define ARM_SPE_NEED_MORE_BYTES -1 +#define ARM_SPE_BAD_PACKET -2 + +#define ARM_SPE_PKT_MAX_SZ 16 + +enum arm_spe_pkt_type { + ARM_SPE_BAD, + ARM_SPE_PAD, + ARM_SPE_END, + ARM_SPE_TIMESTAMP, + ARM_SPE_ADDRESS, + ARM_SPE_COUNTER, + ARM_SPE_CONTEXT, + ARM_SPE_OP_TYPE, + ARM_SPE_EVENTS, + ARM_SPE_DATA_SOURCE, +}; + +struct arm_spe_pkt { + enum arm_spe_pkt_type type; + unsigned char index; + uint64_t payload; + uint64_t midr; +}; + +/* Short header (HEADER0) and extended header (HEADER1) */ +#define SPE_HEADER0_PAD 0x0 +#define SPE_HEADER0_END 0x1 +#define SPE_HEADER0_TIMESTAMP 0x71 +/* Mask for event & data source */ +#define SPE_HEADER0_MASK1 (GENMASK_ULL(7, 6) | GENMASK_ULL(3, 0)) +#define SPE_HEADER0_EVENTS 0x42 +#define SPE_HEADER0_SOURCE 0x43 +/* Mask for context & operation */ +#define SPE_HEADER0_MASK2 GENMASK_ULL(7, 2) +#define SPE_HEADER0_CONTEXT 0x64 +#define SPE_HEADER0_OP_TYPE 0x48 +/* Mask for extended format */ +#define SPE_HEADER0_EXTENDED 0x20 +/* Mask for address & counter */ +#define SPE_HEADER0_MASK3 GENMASK_ULL(7, 3) +#define SPE_HEADER0_ADDRESS 0xb0 +#define SPE_HEADER0_COUNTER 0x98 +#define SPE_HEADER1_ALIGNMENT 0x0 + +#define SPE_HDR_SHORT_INDEX(h) ((h) & GENMASK_ULL(2, 0)) +#define SPE_HDR_EXTENDED_INDEX(h0, h1) (((h0) & GENMASK_ULL(1, 0)) << 3 | \ + SPE_HDR_SHORT_INDEX(h1)) + +/* Address packet header */ +#define SPE_ADDR_PKT_HDR_INDEX_INS 0x0 +#define SPE_ADDR_PKT_HDR_INDEX_BRANCH 0x1 +#define SPE_ADDR_PKT_HDR_INDEX_DATA_VIRT 0x2 +#define SPE_ADDR_PKT_HDR_INDEX_DATA_PHYS 0x3 +#define SPE_ADDR_PKT_HDR_INDEX_PREV_BRANCH 0x4 + +/* Address packet payload */ +#define SPE_ADDR_PKT_ADDR_BYTE7_SHIFT 56 +#define SPE_ADDR_PKT_ADDR_GET_BYTES_0_6(v) ((v) & GENMASK_ULL(55, 0)) +#define SPE_ADDR_PKT_ADDR_GET_BYTE_6(v) (((v) & GENMASK_ULL(55, 48)) >> 48) + +#define SPE_ADDR_PKT_GET_NS(v) (((v) & BIT_ULL(63)) >> 63) +#define SPE_ADDR_PKT_GET_EL(v) (((v) & GENMASK_ULL(62, 61)) >> 61) +#define SPE_ADDR_PKT_GET_CH(v) (((v) & BIT_ULL(62)) >> 62) +#define SPE_ADDR_PKT_GET_PAT(v) (((v) & GENMASK_ULL(59, 56)) >> 56) + +#define SPE_ADDR_PKT_EL0 0 +#define SPE_ADDR_PKT_EL1 1 +#define SPE_ADDR_PKT_EL2 2 +#define SPE_ADDR_PKT_EL3 3 + +/* Context packet header */ +#define SPE_CTX_PKT_HDR_INDEX(h) ((h) & GENMASK_ULL(1, 0)) + +/* Counter packet header */ +#define SPE_CNT_PKT_HDR_INDEX_TOTAL_LAT 0x0 +#define SPE_CNT_PKT_HDR_INDEX_ISSUE_LAT 0x1 +#define SPE_CNT_PKT_HDR_INDEX_TRANS_LAT 0x2 + +/* Event packet payload */ +enum arm_spe_events { + EV_EXCEPTION_GEN = 0, + EV_RETIRED = 1, + EV_L1D_ACCESS = 2, + EV_L1D_REFILL = 3, + EV_TLB_ACCESS = 4, + EV_TLB_WALK = 5, + EV_NOT_TAKEN = 6, + EV_MISPRED = 7, + EV_LLC_ACCESS = 8, + EV_LLC_MISS = 9, + EV_REMOTE_ACCESS = 10, + EV_ALIGNMENT = 11, + EV_TRANSACTIONAL = 16, + EV_PARTIAL_PREDICATE = 17, + EV_EMPTY_PREDICATE = 18, + EV_L2D_ACCESS = 19, + EV_L2D_MISS = 20, + EV_CACHE_DATA_MODIFIED = 21, + EV_RECENTLY_FETCHED = 22, + EV_DATA_SNOOPED = 23, + EV_STREAMING_SVE_MODE = 24, + EV_SMCU = 25, +}; + +/* Operation packet header */ +#define SPE_OP_PKT_HDR_CLASS(h) ((h) & GENMASK_ULL(1, 0)) +#define SPE_OP_PKT_HDR_CLASS_OTHER 0x0 +#define SPE_OP_PKT_HDR_CLASS_LD_ST_ATOMIC 0x1 +#define SPE_OP_PKT_HDR_CLASS_BR_ERET 0x2 + +#define SPE_OP_PKT_OTHER_SUBCLASS_OTHER(v) (((v) & GENMASK_ULL(7, 3)) == 0x0) +#define SPE_OP_PKT_OTHER_SUBCLASS_SVE(v) (((v) & (BIT(7) | BIT(3) | BIT(0))) == 0x8) +#define SPE_OP_PKT_OTHER_SUBCLASS_SME(v) (((v) & (BIT(7) | BIT(3) | BIT(0))) == 0x88) + +#define SPE_OP_PKT_OTHER_ASE BIT(2) +#define SPE_OP_PKT_OTHER_FP BIT(1) + +/* + * SME effective vector length or tile size (ETS) is stored in byte 0 + * bits [6:4,2]; the length is rounded up to a power of two and use 128 + * as one step, so ETS calculation is: + * + * 128 * (2 ^ bits [6:4,2]) = 32 << (bits [6:4,2]) + */ +#define SPE_OP_PKG_SME_ETS(v) (128 << (FIELD_GET(GENMASK_ULL(6, 4), (v)) << 1 | \ + (FIELD_GET(BIT(2), (v))))) + +#define SPE_OP_PKT_LDST_SUBCLASS_GP_REG(v) (((v) & GENMASK_ULL(7, 1)) == 0x0) +#define SPE_OP_PKT_LDST_SUBCLASS_SIMD_FP(v) (((v) & GENMASK_ULL(7, 1)) == 0x4) +#define SPE_OP_PKT_LDST_SUBCLASS_UNSPEC_REG(v) (((v) & GENMASK_ULL(7, 1)) == 0x10) +#define SPE_OP_PKT_LDST_SUBCLASS_NV_SYSREG(v) (((v) & GENMASK_ULL(7, 1)) == 0x30) +#define SPE_OP_PKT_LDST_SUBCLASS_MTE_TAG(v) (((v) & GENMASK_ULL(7, 1)) == 0x14) +#define SPE_OP_PKT_LDST_SUBCLASS_MEMCPY(v) (((v) & GENMASK_ULL(7, 1)) == 0x20) +#define SPE_OP_PKT_LDST_SUBCLASS_MEMSET(v) (((v) & GENMASK_ULL(7, 0)) == 0x25) + +#define SPE_OP_PKT_LDST_SUBCLASS_EXTENDED(v) (((v) & (GENMASK_ULL(7, 5) | BIT(1))) == 0x2) + +#define SPE_OP_PKT_AR BIT(4) +#define SPE_OP_PKT_EXCL BIT(3) +#define SPE_OP_PKT_AT BIT(2) +#define SPE_OP_PKT_ST BIT(0) + +#define SPE_OP_PKT_LDST_SUBCLASS_SVE_SME_REG(v) (((v) & (BIT(3) | BIT(1))) == 0x8) + +#define SPE_OP_PKT_SVE_SG BIT(7) +/* + * SVE effective vector length (EVL) is stored in byte 0 bits [6:4]; + * the length is rounded up to a power of two and use 32 as one step, + * so EVL calculation is: + * + * 32 * (2 ^ bits [6:4]) = 32 << (bits [6:4]) + */ +#define SPE_OP_PKG_SVE_EVL(v) (32 << (((v) & GENMASK_ULL(6, 4)) >> 4)) +#define SPE_OP_PKT_SVE_PRED BIT(2) +#define SPE_OP_PKT_SVE_FP BIT(1) + +#define SPE_OP_PKT_LDST_SUBCLASS_GCS(v) (((v) & (GENMASK_ULL(7, 3) | BIT(1))) == 0x40) + +#define SPE_OP_PKT_GCS_COMM BIT(2) + +#define SPE_OP_PKT_CR_MASK GENMASK_ULL(4, 3) +#define SPE_OP_PKT_CR_BL(v) (FIELD_GET(SPE_OP_PKT_CR_MASK, (v)) == 1) +#define SPE_OP_PKT_CR_RET(v) (FIELD_GET(SPE_OP_PKT_CR_MASK, (v)) == 2) +#define SPE_OP_PKT_CR_NON_BL_RET(v) (FIELD_GET(SPE_OP_PKT_CR_MASK, (v)) == 3) +#define SPE_OP_PKT_GCS BIT(2) +#define SPE_OP_PKT_INDIRECT_BRANCH BIT(1) +#define SPE_OP_PKT_COND BIT(0) + +const char *arm_spe_pkt_name(enum arm_spe_pkt_type); + +int arm_spe_get_packet(const unsigned char *buf, size_t len, + struct arm_spe_pkt *packet, u64 midr); + +int arm_spe_pkt_desc(const struct arm_spe_pkt *packet, char *buf, size_t len); +#endif diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c new file mode 100644 index 000000000..552f063f1 --- /dev/null +++ b/tools/perf/util/arm-spe.c @@ -0,0 +1,2064 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "auxtrace.h" +#include "color.h" +#include "debug.h" +#include "evlist.h" +#include "evsel.h" +#include "machine.h" +#include "session.h" +#include "symbol.h" +#include "thread.h" +#include "thread-stack.h" +#include "tsc.h" +#include "tool.h" +#include "util/synthetic-events.h" + +#include "arm-spe.h" +#include "arm-spe-decoder/arm-spe-decoder.h" +#include "arm-spe-decoder/arm-spe-pkt-decoder.h" + +#include "../../arch/arm64/include/asm/cputype.h" +#define MAX_TIMESTAMP (~0ULL) + +#define is_ldst_op(op) (!!((op) & ARM_SPE_OP_LDST)) + +#define is_simd_op(op) (!!((op) & (ARM_SPE_OP_SIMD_FP | ARM_SPE_OP_SVE | \ + ARM_SPE_OP_SME | ARM_SPE_OP_ASE))) + +#define is_mem_op(op) (is_ldst_op(op) || is_simd_op(op)) + +#define ARM_SPE_CACHE_EVENT(lvl) \ + (ARM_SPE_##lvl##_ACCESS | ARM_SPE_##lvl##_MISS) + +#define arm_spe_is_cache_level(type, lvl) \ + ((type) & ARM_SPE_CACHE_EVENT(lvl)) + +#define arm_spe_is_cache_hit(type, lvl) \ + (((type) & ARM_SPE_CACHE_EVENT(lvl)) == ARM_SPE_##lvl##_ACCESS) + +#define arm_spe_is_cache_miss(type, lvl) \ + ((type) & ARM_SPE_##lvl##_MISS) + +struct arm_spe { + struct auxtrace auxtrace; + struct auxtrace_queues queues; + struct auxtrace_heap heap; + struct itrace_synth_opts synth_opts; + u32 auxtrace_type; + struct perf_session *session; + struct machine *machine; + u32 pmu_type; + + struct perf_tsc_conversion tc; + + u8 timeless_decoding; + u8 data_queued; + + u64 sample_type; + u8 sample_flc; + u8 sample_llc; + u8 sample_tlb; + u8 sample_branch; + u8 sample_remote_access; + u8 sample_memory; + u8 sample_instructions; + + u64 l1d_miss_id; + u64 l1d_access_id; + u64 llc_miss_id; + u64 llc_access_id; + u64 tlb_miss_id; + u64 tlb_access_id; + u64 branch_id; + u64 remote_access_id; + u64 memory_id; + u64 instructions_id; + + u64 kernel_start; + + unsigned long num_events; + u8 use_ctx_pkt_for_pid; + + u64 **metadata; + u64 metadata_ver; + u64 metadata_nr_cpu; + bool is_homogeneous; +}; + +struct arm_spe_queue { + struct arm_spe *spe; + unsigned int queue_nr; + struct auxtrace_buffer *buffer; + struct auxtrace_buffer *old_buffer; + union perf_event *event_buf; + bool on_heap; + bool done; + pid_t pid; + pid_t tid; + int cpu; + struct arm_spe_decoder *decoder; + u64 time; + u64 timestamp; + struct thread *thread; + u64 sample_count; + u32 flags; + struct branch_stack *last_branch; +}; + +struct data_source_handle { + const struct midr_range *midr_ranges; + void (*ds_synth)(const struct arm_spe_record *record, + union perf_mem_data_src *data_src); +}; + +#define DS(range, func) \ + { \ + .midr_ranges = range, \ + .ds_synth = arm_spe__synth_##func, \ + } + +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr); + +static void arm_spe_dump(struct arm_spe *spe __maybe_unused, + unsigned char *buf, size_t len, u64 midr) +{ + struct arm_spe_pkt packet; + size_t pos = 0; + int ret, pkt_len, i; + char desc[ARM_SPE_PKT_DESC_MAX]; + const char *color = PERF_COLOR_BLUE; + + color_fprintf(stdout, color, + ". ... ARM SPE data: size %#zx bytes\n", + len); + + while (len) { + ret = arm_spe_get_packet(buf, len, &packet, midr); + + if (ret > 0) + pkt_len = ret; + else + pkt_len = 1; + printf("."); + color_fprintf(stdout, color, " %08zx: ", pos); + for (i = 0; i < pkt_len; i++) + color_fprintf(stdout, color, " %02x", buf[i]); + for (; i < 16; i++) + color_fprintf(stdout, color, " "); + if (ret > 0) { + ret = arm_spe_pkt_desc(&packet, desc, + ARM_SPE_PKT_DESC_MAX); + if (!ret) + color_fprintf(stdout, color, " %s\n", desc); + } else { + color_fprintf(stdout, color, " Bad packet!\n"); + } + pos += pkt_len; + buf += pkt_len; + len -= pkt_len; + } +} + +static void arm_spe_dump_event(struct arm_spe *spe, unsigned char *buf, + size_t len, u64 midr) +{ + printf(".\n"); + arm_spe_dump(spe, buf, len, midr); +} + +static int arm_spe_get_trace(struct arm_spe_buffer *b, void *data) +{ + struct arm_spe_queue *speq = data; + struct auxtrace_buffer *buffer = speq->buffer; + struct auxtrace_buffer *old_buffer = speq->old_buffer; + struct auxtrace_queue *queue; + + queue = &speq->spe->queues.queue_array[speq->queue_nr]; + + buffer = auxtrace_buffer__next(queue, buffer); + /* If no more data, drop the previous auxtrace_buffer and return */ + if (!buffer) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + b->len = 0; + return 0; + } + + speq->buffer = buffer; + + /* If the aux_buffer doesn't have data associated, try to load it */ + if (!buffer->data) { + /* get the file desc associated with the perf data file */ + int fd = perf_data__fd(speq->spe->session->data); + + buffer->data = auxtrace_buffer__get_data(buffer, fd); + if (!buffer->data) + return -ENOMEM; + } + + b->len = buffer->size; + b->buf = buffer->data; + + if (b->len) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + speq->old_buffer = buffer; + } else { + auxtrace_buffer__drop_data(buffer); + return arm_spe_get_trace(b, data); + } + + return 0; +} + +static struct arm_spe_queue *arm_spe__alloc_queue(struct arm_spe *spe, + unsigned int queue_nr) +{ + struct arm_spe_params params = { .get_trace = 0, }; + struct arm_spe_queue *speq; + + speq = zalloc(sizeof(*speq)); + if (!speq) + return NULL; + + speq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); + if (!speq->event_buf) + goto out_free; + + speq->spe = spe; + speq->queue_nr = queue_nr; + speq->pid = -1; + speq->tid = -1; + speq->cpu = -1; + + /* params set */ + params.get_trace = arm_spe_get_trace; + params.data = speq; + + if (spe->synth_opts.last_branch) { + size_t sz = sizeof(struct branch_stack); + + /* Allocate up to two entries for PBT + TGT */ + sz += sizeof(struct branch_entry) * + min(spe->synth_opts.last_branch_sz, 2U); + speq->last_branch = zalloc(sz); + if (!speq->last_branch) + goto out_free; + } + + /* create new decoder */ + speq->decoder = arm_spe_decoder_new(¶ms); + if (!speq->decoder) + goto out_free; + + return speq; + +out_free: + zfree(&speq->event_buf); + zfree(&speq->last_branch); + free(speq); + + return NULL; +} + +static inline u8 arm_spe_cpumode(struct arm_spe *spe, u64 ip) +{ + return ip >= spe->kernel_start ? + PERF_RECORD_MISC_KERNEL : + PERF_RECORD_MISC_USER; +} + +static void arm_spe_set_pid_tid_cpu(struct arm_spe *spe, + struct auxtrace_queue *queue) +{ + struct arm_spe_queue *speq = queue->priv; + pid_t tid; + + tid = machine__get_current_tid(spe->machine, speq->cpu); + if (tid != -1) { + speq->tid = tid; + thread__zput(speq->thread); + } else + speq->tid = queue->tid; + + if ((!speq->thread) && (speq->tid != -1)) { + speq->thread = machine__find_thread(spe->machine, -1, + speq->tid); + } + + if (speq->thread) { + speq->pid = thread__pid(speq->thread); + if (queue->cpu == -1) { + speq->cpu = thread__cpu(speq->thread); + arm_spe__get_midr(spe, speq->cpu, &speq->decoder->midr); + } + } +} + +static int arm_spe_set_tid(struct arm_spe_queue *speq, pid_t tid) +{ + struct arm_spe *spe = speq->spe; + int err = machine__set_current_tid(spe->machine, speq->cpu, -1, tid); + + if (err) + return err; + + arm_spe_set_pid_tid_cpu(spe, &spe->queues.queue_array[speq->queue_nr]); + + return 0; +} + +static u64 *arm_spe__get_metadata_by_cpu(struct arm_spe *spe, int cpu) +{ + u64 i; + + if (!spe->metadata) + return NULL; + + /* CPU ID is -1 for per-thread mode */ + if (cpu < 0) { + /* + * On the heterogeneous system, due to CPU ID is -1, + * cannot confirm the data source packet is supported. + */ + if (!spe->is_homogeneous) + return NULL; + + /* In homogeneous system, simply use CPU0's metadata */ + return spe->metadata[0]; + } + + for (i = 0; i < spe->metadata_nr_cpu; i++) + if (spe->metadata[i][ARM_SPE_CPU] == (u64)cpu) + return spe->metadata[i]; + + return NULL; +} + +static struct simd_flags arm_spe__synth_simd_flags(const struct arm_spe_record *record) +{ + struct simd_flags simd_flags = {}; + + if (record->op & ARM_SPE_OP_SVE) + simd_flags.arch |= SIMD_OP_FLAGS_ARCH_SVE; + else if (record->op & ARM_SPE_OP_SME) + simd_flags.arch |= SIMD_OP_FLAGS_ARCH_SME; + else if (record->op & (ARM_SPE_OP_ASE | ARM_SPE_OP_SIMD_FP)) + simd_flags.arch |= SIMD_OP_FLAGS_ARCH_ASE; + + if (record->op & ARM_SPE_OP_SVE) { + if (!(record->op & ARM_SPE_OP_PRED)) + simd_flags.pred = SIMD_OP_FLAGS_PRED_DISABLED; + else if (record->type & ARM_SPE_SVE_PARTIAL_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_PARTIAL; + else if (record->type & ARM_SPE_SVE_EMPTY_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_EMPTY; + else + simd_flags.pred = SIMD_OP_FLAGS_PRED_FULL; + } else { + if (record->type & ARM_SPE_SVE_PARTIAL_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_PARTIAL; + else if (record->type & ARM_SPE_SVE_EMPTY_PRED) + simd_flags.pred = SIMD_OP_FLAGS_PRED_EMPTY; + } + + return simd_flags; +} + +static void arm_spe_prep_sample(struct arm_spe *spe, + struct arm_spe_queue *speq, + union perf_event *event, + struct perf_sample *sample) +{ + struct arm_spe_record *record = &speq->decoder->record; + + if (!spe->timeless_decoding) + sample->time = tsc_to_perf_time(record->timestamp, &spe->tc); + + sample->ip = record->from_ip; + sample->cpumode = arm_spe_cpumode(spe, sample->ip); + sample->pid = speq->pid; + sample->tid = speq->tid; + sample->period = spe->synth_opts.period; + sample->cpu = speq->cpu; + sample->simd_flags = arm_spe__synth_simd_flags(record); + + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = sample->cpumode; + event->sample.header.size = sizeof(struct perf_event_header); +} + +static void arm_spe__prep_branch_stack(struct arm_spe_queue *speq) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + struct branch_stack *bstack = speq->last_branch; + struct branch_flags *bs_flags; + unsigned int last_branch_sz = spe->synth_opts.last_branch_sz; + bool have_tgt = !!(speq->flags & PERF_IP_FLAG_BRANCH); + bool have_pbt = last_branch_sz >= (have_tgt + 1U) && record->prev_br_tgt; + size_t sz = sizeof(struct branch_stack) + + sizeof(struct branch_entry) * min(last_branch_sz, 2U) /* PBT + TGT */; + int i = 0; + + /* Clean up branch stack */ + memset(bstack, 0x0, sz); + + if (!have_tgt && !have_pbt) + return; + + if (have_tgt) { + bstack->entries[i].from = record->from_ip; + bstack->entries[i].to = record->to_ip; + + bs_flags = &bstack->entries[i].flags; + bs_flags->value = 0; + + if (record->op & ARM_SPE_OP_BR_CR_BL) { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND_CALL; + else + bs_flags->type |= PERF_BR_CALL; + /* + * Indirect branch instruction without link (e.g. BR), + * take this case as function return. + */ + } else if (record->op & ARM_SPE_OP_BR_CR_RET || + record->op & ARM_SPE_OP_BR_INDIRECT) { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND_RET; + else + bs_flags->type |= PERF_BR_RET; + } else if (record->op & ARM_SPE_OP_BR_CR_NON_BL_RET) { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND; + else + bs_flags->type |= PERF_BR_UNCOND; + } else { + if (record->op & ARM_SPE_OP_BR_COND) + bs_flags->type |= PERF_BR_COND; + else + bs_flags->type |= PERF_BR_UNKNOWN; + } + + if (record->type & ARM_SPE_BRANCH_MISS) { + bs_flags->mispred = 1; + bs_flags->predicted = 0; + } else { + bs_flags->mispred = 0; + bs_flags->predicted = 1; + } + + if (record->type & ARM_SPE_BRANCH_NOT_TAKEN) + bs_flags->not_taken = 1; + + if (record->type & ARM_SPE_IN_TXN) + bs_flags->in_tx = 1; + + bs_flags->cycles = min(record->latency, 0xFFFFU); + i++; + } + + if (have_pbt) { + bs_flags = &bstack->entries[i].flags; + bs_flags->type |= PERF_BR_UNKNOWN; + bstack->entries[i].to = record->prev_br_tgt; + i++; + } + + bstack->nr = i; + bstack->hw_idx = -1ULL; +} + +static int arm_spe__inject_event(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample, u64 type) +{ + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && spe->session && spe->session->evlist) + evsel = evlist__id2evsel(spe->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + event->header.type = PERF_RECORD_SAMPLE; + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); +} + +static inline int +arm_spe_deliver_synth_event(struct arm_spe *spe, + struct arm_spe_queue *speq __maybe_unused, + union perf_event *event, + struct perf_sample *sample) +{ + int ret; + + if (spe->synth_opts.inject) { + ret = arm_spe__inject_event(spe, event, sample, spe->sample_type); + if (ret) + return ret; + } + + ret = perf_session__deliver_synth_event(spe->session, event, sample); + if (ret) + pr_err("ARM SPE: failed to deliver event, error %d\n", ret); + + return ret; +} + +static int arm_spe__synth_mem_sample(struct arm_spe_queue *speq, + u64 spe_events_id, + union perf_mem_data_src data_src) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + union perf_event *event = speq->event_buf; + struct perf_sample sample; + int ret; + + perf_sample__init(&sample, /*all=*/true); + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + sample.addr = record->virt_addr; + sample.phys_addr = record->phys_addr; + sample.data_src = data_src.val; + sample.weight = record->latency; + + ret = arm_spe_deliver_synth_event(spe, speq, event, &sample); + perf_sample__exit(&sample); + return ret; +} + +static int arm_spe__synth_branch_sample(struct arm_spe_queue *speq, + u64 spe_events_id) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + union perf_event *event = speq->event_buf; + struct perf_sample sample; + int ret; + + perf_sample__init(&sample, /*all=*/true); + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + sample.addr = record->to_ip; + sample.weight = record->latency; + sample.flags = speq->flags; + sample.branch_stack = speq->last_branch; + + ret = arm_spe_deliver_synth_event(spe, speq, event, &sample); + perf_sample__exit(&sample); + return ret; +} + +static int arm_spe__synth_instruction_sample(struct arm_spe_queue *speq, + u64 spe_events_id, + union perf_mem_data_src data_src) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record = &speq->decoder->record; + union perf_event *event = speq->event_buf; + struct perf_sample sample; + int ret; + + perf_sample__init(&sample, /*all=*/true); + arm_spe_prep_sample(spe, speq, event, &sample); + + sample.id = spe_events_id; + sample.stream_id = spe_events_id; + sample.addr = record->to_ip; + sample.phys_addr = record->phys_addr; + sample.data_src = data_src.val; + sample.weight = record->latency; + sample.flags = speq->flags; + sample.branch_stack = speq->last_branch; + + ret = arm_spe_deliver_synth_event(spe, speq, event, &sample); + perf_sample__exit(&sample); + return ret; +} + +static const struct midr_range common_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_CORTEX_A715), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A720AE), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A725), + MIDR_ALL_VERSIONS(MIDR_CORTEX_A78C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X1C), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X3), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X4), + MIDR_ALL_VERSIONS(MIDR_CORTEX_X925), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_N2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V1), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V2), + MIDR_ALL_VERSIONS(MIDR_NEOVERSE_V3), + MIDR_ALL_VERSIONS(MIDR_NVIDIA_OLYMPUS), + {}, +}; + +static const struct midr_range ampereone_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_AMPERE1A), + {}, +}; + +static const struct midr_range hisi_hip_ds_encoding_cpus[] = { + MIDR_ALL_VERSIONS(MIDR_HISI_HIP12), + {}, +}; + +static void arm_spe__sample_flags(struct arm_spe_queue *speq) +{ + const struct arm_spe_record *record = &speq->decoder->record; + + speq->flags = 0; + if (record->op & ARM_SPE_OP_BRANCH_ERET) { + speq->flags = PERF_IP_FLAG_BRANCH; + + if (record->type & ARM_SPE_BRANCH_MISS) + speq->flags |= PERF_IP_FLAG_BRANCH_MISS; + + if (record->type & ARM_SPE_BRANCH_NOT_TAKEN) + speq->flags |= PERF_IP_FLAG_NOT_TAKEN; + + if (record->type & ARM_SPE_IN_TXN) + speq->flags |= PERF_IP_FLAG_IN_TX; + + if (record->op & ARM_SPE_OP_BR_COND) + speq->flags |= PERF_IP_FLAG_CONDITIONAL; + + if (record->op & ARM_SPE_OP_BR_CR_BL) + speq->flags |= PERF_IP_FLAG_CALL; + else if (record->op & ARM_SPE_OP_BR_CR_RET) + speq->flags |= PERF_IP_FLAG_RETURN; + /* + * Indirect branch instruction without link (e.g. BR), + * take it as a function return. + */ + else if (record->op & ARM_SPE_OP_BR_INDIRECT) + speq->flags |= PERF_IP_FLAG_RETURN; + } +} + +static void arm_spe__synth_data_source_common(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* + * Even though four levels of cache hierarchy are possible, no known + * production Neoverse systems currently include more than three levels + * so for the time being we assume three exist. If a production system + * is built with four the this function would have to be changed to + * detect the number of levels for reporting. + */ + + /* + * We have no data on the hit level or data source for stores in the + * Neoverse SPE records. + */ + if (record->op & ARM_SPE_OP_ST) { + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; + data_src->mem_snoop = PERF_MEM_SNOOP_NA; + return; + } + + switch (record->source) { + case ARM_SPE_COMMON_DS_L1D: + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_COMMON_DS_L2: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_COMMON_DS_PEER_CORE: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + /* + * We don't know if this is L1, L2 but we do know it was a cache-2-cache + * transfer, so set SNOOPX_PEER + */ + case ARM_SPE_COMMON_DS_LOCAL_CLUSTER: + case ARM_SPE_COMMON_DS_PEER_CLUSTER: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + /* + * System cache is assumed to be L3 + */ + case ARM_SPE_COMMON_DS_SYS_CACHE: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HIT; + break; + /* + * We don't know what level it hit in, except it came from the other + * socket + */ + case ARM_SPE_COMMON_DS_REMOTE: + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_COMMON_DS_DRAM: + data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + default: + break; + } +} + +/* + * Source is IMPDEF. Here we convert the source code used on AmpereOne cores + * to the common (Neoverse, Cortex) to avoid duplicating the decoding code. + */ +static void arm_spe__synth_data_source_ampereone(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + struct arm_spe_record common_record; + + switch (record->source) { + case ARM_SPE_AMPEREONE_LOCAL_CHIP_CACHE_OR_DEVICE: + common_record.source = ARM_SPE_COMMON_DS_PEER_CORE; + break; + case ARM_SPE_AMPEREONE_SLC: + common_record.source = ARM_SPE_COMMON_DS_SYS_CACHE; + break; + case ARM_SPE_AMPEREONE_REMOTE_CHIP_CACHE: + common_record.source = ARM_SPE_COMMON_DS_REMOTE; + break; + case ARM_SPE_AMPEREONE_DDR: + common_record.source = ARM_SPE_COMMON_DS_DRAM; + break; + case ARM_SPE_AMPEREONE_L1D: + common_record.source = ARM_SPE_COMMON_DS_L1D; + break; + case ARM_SPE_AMPEREONE_L2D: + common_record.source = ARM_SPE_COMMON_DS_L2; + break; + default: + pr_warning_once("AmpereOne: Unknown data source (0x%x)\n", + record->source); + return; + } + + common_record.op = record->op; + arm_spe__synth_data_source_common(&common_record, data_src); +} + +static void arm_spe__synth_data_source_hisi_hip(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* Use common synthesis method to handle store operations */ + if (record->op & ARM_SPE_OP_ST) { + arm_spe__synth_data_source_common(record, data_src); + return; + } + + switch (record->source) { + case ARM_SPE_HISI_HIP_PEER_CPU: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_PEER_CPU_HITM: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_L3: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HIT; + break; + case ARM_SPE_HISI_HIP_L3_HITM: + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + break; + case ARM_SPE_HISI_HIP_PEER_CLUSTER: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_PEER_CLUSTER_HITM: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_REMOTE_SOCKET: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_REMOTE_SOCKET_HITM: + data_src->mem_lvl = PERF_MEM_LVL_REM_CCE2; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_ANY_CACHE; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + data_src->mem_snoopx = PERF_MEM_SNOOPX_PEER; + break; + case ARM_SPE_HISI_HIP_LOCAL_MEM: + data_src->mem_lvl = PERF_MEM_LVL_LOC_RAM | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_HISI_HIP_REMOTE_MEM: + data_src->mem_lvl = PERF_MEM_LVL_REM_RAM1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_RAM; + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + break; + case ARM_SPE_HISI_HIP_NC_DEV: + data_src->mem_lvl = PERF_MEM_LVL_IO | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_IO; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_HISI_HIP_L2: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + case ARM_SPE_HISI_HIP_L2_HITM: + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + break; + case ARM_SPE_HISI_HIP_L1: + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + break; + default: + break; + } +} + +static const struct data_source_handle data_source_handles[] = { + DS(common_ds_encoding_cpus, data_source_common), + DS(ampereone_ds_encoding_cpus, data_source_ampereone), + DS(hisi_hip_ds_encoding_cpus, data_source_hisi_hip), +}; + +static void arm_spe__synth_ld_memory_level(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* + * To find a cache hit, search in ascending order from the lower level + * caches to the higher level caches. This reflects the best scenario + * for a cache hit. + */ + if (arm_spe_is_cache_hit(record->type, L1D)) { + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + } else if (record->type & ARM_SPE_RECENTLY_FETCHED) { + data_src->mem_lvl = PERF_MEM_LVL_LFB | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_LFB; + } else if (arm_spe_is_cache_hit(record->type, L2D)) { + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + } else if (arm_spe_is_cache_hit(record->type, LLC)) { + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + /* + * To find a cache miss, search in descending order from the higher + * level cache to the lower level cache. This represents the worst + * scenario for a cache miss. + */ + } else if (arm_spe_is_cache_miss(record->type, LLC)) { + data_src->mem_lvl = PERF_MEM_LVL_L3 | PERF_MEM_LVL_MISS; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + } else if (arm_spe_is_cache_miss(record->type, L2D)) { + data_src->mem_lvl = PERF_MEM_LVL_L2 | PERF_MEM_LVL_MISS; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + } else if (arm_spe_is_cache_miss(record->type, L1D)) { + data_src->mem_lvl = PERF_MEM_LVL_L1 | PERF_MEM_LVL_MISS; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + } +} + +static void arm_spe__synth_st_memory_level(const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + /* Record the greatest level info for a store operation. */ + if (arm_spe_is_cache_level(record->type, LLC)) { + data_src->mem_lvl = PERF_MEM_LVL_L3; + data_src->mem_lvl |= arm_spe_is_cache_miss(record->type, LLC) ? + PERF_MEM_LVL_MISS : PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L3; + } else if (arm_spe_is_cache_level(record->type, L2D)) { + data_src->mem_lvl = PERF_MEM_LVL_L2; + data_src->mem_lvl |= arm_spe_is_cache_miss(record->type, L2D) ? + PERF_MEM_LVL_MISS : PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L2; + } else if (arm_spe_is_cache_level(record->type, L1D)) { + data_src->mem_lvl = PERF_MEM_LVL_L1; + data_src->mem_lvl |= arm_spe_is_cache_miss(record->type, L1D) ? + PERF_MEM_LVL_MISS : PERF_MEM_LVL_HIT; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_L1; + } +} + +static void arm_spe__synth_memory_level(struct arm_spe_queue *speq, + const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + struct arm_spe *spe = speq->spe; + + /* + * The data source packet contains more info for cache levels for + * peer snooping. So respect the memory level if has been set by + * data source parsing. + */ + if (!data_src->mem_lvl) { + if (data_src->mem_op == PERF_MEM_OP_LOAD) + arm_spe__synth_ld_memory_level(record, data_src); + if (data_src->mem_op == PERF_MEM_OP_STORE) + arm_spe__synth_st_memory_level(record, data_src); + } + + if (!data_src->mem_lvl) { + data_src->mem_lvl = PERF_MEM_LVL_NA; + data_src->mem_lvl_num = PERF_MEM_LVLNUM_NA; + } + + /* + * If 'mem_snoop' has been set by data source packet, skip to set + * it at here. + */ + if (!data_src->mem_snoop) { + if (record->type & ARM_SPE_DATA_SNOOPED) { + if (record->type & ARM_SPE_HITM) + data_src->mem_snoop = PERF_MEM_SNOOP_HITM; + else + data_src->mem_snoop = PERF_MEM_SNOOP_HIT; + } else { + u64 *metadata = + arm_spe__get_metadata_by_cpu(spe, speq->cpu); + + /* + * Set NA ("Not available") mode if no meta data or the + * SNOOPED event is not supported. + */ + if (!metadata || + !(metadata[ARM_SPE_CAP_EVENT_FILTER] & ARM_SPE_DATA_SNOOPED)) + data_src->mem_snoop = PERF_MEM_SNOOP_NA; + else + data_src->mem_snoop = PERF_MEM_SNOOP_NONE; + } + } + + if (!data_src->mem_remote) { + if (record->type & ARM_SPE_REMOTE_ACCESS) + data_src->mem_remote = PERF_MEM_REMOTE_REMOTE; + } +} + +static int arm_spe__get_midr(struct arm_spe *spe, int cpu, u64 *midr) +{ + u64 *metadata; + + /* Metadata version 1 assumes all CPUs are the same (old behavior) */ + if (spe->metadata_ver == 1) { + const char *cpuid; + + pr_warning_once("Old SPE metadata, re-record to improve decode accuracy\n"); + cpuid = perf_env__cpuid(perf_session__env(spe->session)); + if (!cpuid) + goto err; + + *midr = strtol(cpuid, NULL, 16); + return 0; + } + + metadata = arm_spe__get_metadata_by_cpu(spe, cpu); + if (!metadata) + goto err; + + *midr = metadata[ARM_SPE_CPU_MIDR]; + return 0; + +err: + pr_warning_once("Failed to get MIDR for CPU %d\n", cpu); + return -EINVAL; +} + +static void arm_spe__synth_ds(struct arm_spe_queue *speq, + const struct arm_spe_record *record, + union perf_mem_data_src *data_src) +{ + u64 midr; + unsigned int i; + + if (arm_spe__get_midr(speq->spe, speq->cpu, &midr)) + return; + + for (i = 0; i < ARRAY_SIZE(data_source_handles); i++) { + if (is_midr_in_range_list(midr, data_source_handles[i].midr_ranges)) { + return data_source_handles[i].ds_synth(record, data_src); + } + } + + return; +} + +static union perf_mem_data_src +arm_spe__synth_data_source(struct arm_spe_queue *speq, + const struct arm_spe_record *record) +{ + union perf_mem_data_src data_src = {}; + + if (!is_mem_op(record->op)) + return data_src; + + if (record->op & ARM_SPE_OP_LD) + data_src.mem_op = PERF_MEM_OP_LOAD; + else if (record->op & ARM_SPE_OP_ST) + data_src.mem_op = PERF_MEM_OP_STORE; + else + data_src.mem_op = PERF_MEM_OP_NA; + + arm_spe__synth_ds(speq, record, &data_src); + arm_spe__synth_memory_level(speq, record, &data_src); + + if (record->type & (ARM_SPE_TLB_ACCESS | ARM_SPE_TLB_MISS)) { + data_src.mem_dtlb = PERF_MEM_TLB_WK; + + if (record->type & ARM_SPE_TLB_MISS) + data_src.mem_dtlb |= PERF_MEM_TLB_MISS; + else + data_src.mem_dtlb |= PERF_MEM_TLB_HIT; + } + + return data_src; +} + +static int arm_spe_sample(struct arm_spe_queue *speq) +{ + const struct arm_spe_record *record = &speq->decoder->record; + struct arm_spe *spe = speq->spe; + union perf_mem_data_src data_src; + int err; + + /* + * Discard all samples until period is reached + */ + speq->sample_count++; + if (speq->sample_count < spe->synth_opts.period) + return 0; + speq->sample_count = 0; + + arm_spe__sample_flags(speq); + data_src = arm_spe__synth_data_source(speq, record); + + if (spe->sample_flc) { + if (record->type & ARM_SPE_L1D_MISS) { + err = arm_spe__synth_mem_sample(speq, spe->l1d_miss_id, + data_src); + if (err) + return err; + } + + if (record->type & ARM_SPE_L1D_ACCESS) { + err = arm_spe__synth_mem_sample(speq, spe->l1d_access_id, + data_src); + if (err) + return err; + } + } + + if (spe->sample_llc) { + if (record->type & ARM_SPE_LLC_MISS) { + err = arm_spe__synth_mem_sample(speq, spe->llc_miss_id, + data_src); + if (err) + return err; + } + + if (record->type & ARM_SPE_LLC_ACCESS) { + err = arm_spe__synth_mem_sample(speq, spe->llc_access_id, + data_src); + if (err) + return err; + } + } + + if (spe->sample_tlb) { + if (record->type & ARM_SPE_TLB_MISS) { + err = arm_spe__synth_mem_sample(speq, spe->tlb_miss_id, + data_src); + if (err) + return err; + } + + if (record->type & ARM_SPE_TLB_ACCESS) { + err = arm_spe__synth_mem_sample(speq, spe->tlb_access_id, + data_src); + if (err) + return err; + } + } + + if (spe->synth_opts.last_branch && + (spe->sample_branch || spe->sample_instructions)) + arm_spe__prep_branch_stack(speq); + + if (spe->sample_branch && (record->op & ARM_SPE_OP_BRANCH_ERET)) { + err = arm_spe__synth_branch_sample(speq, spe->branch_id); + if (err) + return err; + } + + if (spe->sample_remote_access && + (record->type & ARM_SPE_REMOTE_ACCESS)) { + err = arm_spe__synth_mem_sample(speq, spe->remote_access_id, + data_src); + if (err) + return err; + } + + if (spe->sample_memory && is_mem_op(record->op)) { + err = arm_spe__synth_mem_sample(speq, spe->memory_id, data_src); + if (err) + return err; + } + + if (spe->sample_instructions) { + err = arm_spe__synth_instruction_sample(speq, spe->instructions_id, data_src); + if (err) + return err; + } + + return 0; +} + +static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) +{ + struct arm_spe *spe = speq->spe; + struct arm_spe_record *record; + int ret; + + if (!spe->kernel_start) + spe->kernel_start = machine__kernel_start(spe->machine); + + while (1) { + /* + * The usual logic is firstly to decode the packets, and then + * based the record to synthesize sample; but here the flow is + * reversed: it calls arm_spe_sample() for synthesizing samples + * prior to arm_spe_decode(). + * + * Two reasons for this code logic: + * 1. Firstly, when setup queue in arm_spe__setup_queue(), it + * has decoded trace data and generated a record, but the record + * is left to generate sample until run to here, so it's correct + * to synthesize sample for the left record. + * 2. After decoding trace data, it needs to compare the record + * timestamp with the coming perf event, if the record timestamp + * is later than the perf event, it needs bail out and pushs the + * record into auxtrace heap, thus the record can be deferred to + * synthesize sample until run to here at the next time; so this + * can correlate samples between Arm SPE trace data and other + * perf events with correct time ordering. + */ + + /* + * Update pid/tid info. + */ + record = &speq->decoder->record; + if (!spe->timeless_decoding && record->context_id != (u64)-1) { + ret = arm_spe_set_tid(speq, record->context_id); + if (ret) + return ret; + + spe->use_ctx_pkt_for_pid = true; + } + + ret = arm_spe_sample(speq); + if (ret) + return ret; + + ret = arm_spe_decode(speq->decoder); + if (!ret) { + pr_debug("No data or all data has been processed.\n"); + return 1; + } + + /* + * Error is detected when decode SPE trace data, continue to + * the next trace data and find out more records. + */ + if (ret < 0) + continue; + + record = &speq->decoder->record; + + /* Update timestamp for the last record */ + if (record->timestamp > speq->timestamp) + speq->timestamp = record->timestamp; + + /* + * If the timestamp of the queue is later than timestamp of the + * coming perf event, bail out so can allow the perf event to + * be processed ahead. + */ + if (!spe->timeless_decoding && speq->timestamp >= *timestamp) { + *timestamp = speq->timestamp; + return 0; + } + } + + return 0; +} + +static int arm_spe__setup_queue(struct arm_spe *spe, + struct auxtrace_queue *queue, + unsigned int queue_nr) +{ + struct arm_spe_queue *speq = queue->priv; + struct arm_spe_record *record; + + if (list_empty(&queue->head) || speq) + return 0; + + speq = arm_spe__alloc_queue(spe, queue_nr); + + if (!speq) + return -ENOMEM; + + queue->priv = speq; + + if (queue->cpu != -1) + speq->cpu = queue->cpu; + arm_spe__get_midr(spe, queue->cpu, &speq->decoder->midr); + + if (!speq->on_heap) { + int ret; + + if (spe->timeless_decoding) + return 0; + +retry: + ret = arm_spe_decode(speq->decoder); + + if (!ret) + return 0; + + if (ret < 0) + goto retry; + + record = &speq->decoder->record; + + speq->timestamp = record->timestamp; + ret = auxtrace_heap__add(&spe->heap, queue_nr, speq->timestamp); + if (ret) + return ret; + speq->on_heap = true; + } + + return 0; +} + +static int arm_spe__setup_queues(struct arm_spe *spe) +{ + unsigned int i; + int ret; + + for (i = 0; i < spe->queues.nr_queues; i++) { + ret = arm_spe__setup_queue(spe, &spe->queues.queue_array[i], i); + if (ret) + return ret; + } + + return 0; +} + +static int arm_spe__update_queues(struct arm_spe *spe) +{ + if (spe->queues.new_data) { + spe->queues.new_data = false; + return arm_spe__setup_queues(spe); + } + + return 0; +} + +static bool arm_spe__is_timeless_decoding(struct arm_spe *spe) +{ + struct evsel *evsel; + struct evlist *evlist = spe->session->evlist; + bool timeless_decoding = true; + + /* + * Circle through the list of event and complain if we find one + * with the time bit set. + */ + evlist__for_each_entry(evlist, evsel) { + if ((evsel->core.attr.sample_type & PERF_SAMPLE_TIME)) + timeless_decoding = false; + } + + return timeless_decoding; +} + +static int arm_spe_process_queues(struct arm_spe *spe, u64 timestamp) +{ + unsigned int queue_nr; + u64 ts; + int ret; + + while (1) { + struct auxtrace_queue *queue; + struct arm_spe_queue *speq; + + if (!spe->heap.heap_cnt) + return 0; + + if (spe->heap.heap_array[0].ordinal >= timestamp) + return 0; + + queue_nr = spe->heap.heap_array[0].queue_nr; + queue = &spe->queues.queue_array[queue_nr]; + speq = queue->priv; + + auxtrace_heap__pop(&spe->heap); + + if (spe->heap.heap_cnt) { + ts = spe->heap.heap_array[0].ordinal + 1; + if (ts > timestamp) + ts = timestamp; + } else { + ts = timestamp; + } + + /* + * A previous context-switch event has set pid/tid in the machine's context, so + * here we need to update the pid/tid in the thread and SPE queue. + */ + if (!spe->use_ctx_pkt_for_pid) + arm_spe_set_pid_tid_cpu(spe, queue); + + ret = arm_spe_run_decoder(speq, &ts); + if (ret < 0) { + auxtrace_heap__add(&spe->heap, queue_nr, ts); + return ret; + } + + if (!ret) { + ret = auxtrace_heap__add(&spe->heap, queue_nr, ts); + if (ret < 0) + return ret; + } else { + speq->on_heap = false; + } + } + + return 0; +} + +static int arm_spe_process_timeless_queues(struct arm_spe *spe, pid_t tid, + u64 time_) +{ + struct auxtrace_queues *queues = &spe->queues; + unsigned int i; + u64 ts = 0; + + for (i = 0; i < queues->nr_queues; i++) { + struct auxtrace_queue *queue = &spe->queues.queue_array[i]; + struct arm_spe_queue *speq = queue->priv; + + if (speq && (tid == -1 || speq->tid == tid)) { + speq->time = time_; + arm_spe_set_pid_tid_cpu(spe, queue); + arm_spe_run_decoder(speq, &ts); + } + } + return 0; +} + +static int arm_spe_context_switch(struct arm_spe *spe, union perf_event *event, + struct perf_sample *sample) +{ + pid_t pid, tid; + int cpu; + + if (!(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT)) + return 0; + + pid = event->context_switch.next_prev_pid; + tid = event->context_switch.next_prev_tid; + cpu = sample->cpu; + + if (tid == -1) + pr_warning("context_switch event has no tid\n"); + + return machine__set_current_tid(spe->machine, cpu, pid, tid); +} + +static int arm_spe_process_event(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool) +{ + int err = 0; + u64 timestamp; + struct arm_spe *spe = container_of(session->auxtrace, + struct arm_spe, auxtrace); + + if (dump_trace) + return 0; + + if (!tool->ordered_events) { + pr_err("SPE trace requires ordered events\n"); + return -EINVAL; + } + + if (sample->time && (sample->time != (u64) -1)) + timestamp = perf_time_to_tsc(sample->time, &spe->tc); + else + timestamp = 0; + + if (timestamp || spe->timeless_decoding) { + err = arm_spe__update_queues(spe); + if (err) + return err; + } + + if (spe->timeless_decoding) { + if (event->header.type == PERF_RECORD_EXIT) { + err = arm_spe_process_timeless_queues(spe, + event->fork.tid, + sample->time); + } + } else if (timestamp) { + err = arm_spe_process_queues(spe, timestamp); + if (err) + return err; + + if (!spe->use_ctx_pkt_for_pid && + (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE || + event->header.type == PERF_RECORD_SWITCH)) + err = arm_spe_context_switch(spe, event, sample); + } + + return err; +} + +static int arm_spe_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + + if (!spe->data_queued) { + struct auxtrace_buffer *buffer; + off_t data_offset; + int fd = perf_data__fd(session->data); + int err; + + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } + + err = auxtrace_queues__add_event(&spe->queues, session, event, + data_offset, &buffer); + if (err) + return err; + + /* Dump here now we have copied a piped trace out of the pipe */ + if (dump_trace) { + if (auxtrace_buffer__get_data(buffer, fd)) { + u64 midr = 0; + + arm_spe__get_midr(spe, buffer->cpu.cpu, &midr); + arm_spe_dump_event(spe, buffer->data, + buffer->size, midr); + auxtrace_buffer__put_data(buffer); + } + } + } + + return 0; +} + +static int arm_spe_flush(struct perf_session *session __maybe_unused, + const struct perf_tool *tool __maybe_unused) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + int ret; + + if (dump_trace) + return 0; + + if (!tool->ordered_events) + return -EINVAL; + + ret = arm_spe__update_queues(spe); + if (ret < 0) + return ret; + + if (spe->timeless_decoding) + return arm_spe_process_timeless_queues(spe, -1, + MAX_TIMESTAMP - 1); + + ret = arm_spe_process_queues(spe, MAX_TIMESTAMP); + if (ret) + return ret; + + if (!spe->use_ctx_pkt_for_pid) + ui__warning("Arm SPE CONTEXT packets not found in the traces.\n" + "Matching of TIDs to SPE events could be inaccurate.\n"); + + return 0; +} + +static u64 *arm_spe__alloc_per_cpu_metadata(u64 *buf, int per_cpu_size) +{ + u64 *metadata; + + metadata = zalloc(per_cpu_size); + if (!metadata) + return NULL; + + memcpy(metadata, buf, per_cpu_size); + return metadata; +} + +static void arm_spe__free_metadata(u64 **metadata, int nr_cpu) +{ + int i; + + for (i = 0; i < nr_cpu; i++) + zfree(&metadata[i]); + free(metadata); +} + +static u64 **arm_spe__alloc_metadata(struct perf_record_auxtrace_info *info, + u64 *ver, int *nr_cpu) +{ + u64 *ptr = (u64 *)info->priv; + u64 metadata_size; + u64 **metadata = NULL; + int hdr_sz, per_cpu_sz, i; + + metadata_size = info->header.size - + sizeof(struct perf_record_auxtrace_info); + + /* Metadata version 1 */ + if (metadata_size == ARM_SPE_AUXTRACE_V1_PRIV_SIZE) { + *ver = 1; + *nr_cpu = 0; + /* No per CPU metadata */ + return NULL; + } + + *ver = ptr[ARM_SPE_HEADER_VERSION]; + hdr_sz = ptr[ARM_SPE_HEADER_SIZE]; + *nr_cpu = ptr[ARM_SPE_CPUS_NUM]; + + metadata = calloc(*nr_cpu, sizeof(*metadata)); + if (!metadata) + return NULL; + + /* Locate the start address of per CPU metadata */ + ptr += hdr_sz; + per_cpu_sz = (metadata_size - (hdr_sz * sizeof(u64))) / (*nr_cpu); + + for (i = 0; i < *nr_cpu; i++) { + metadata[i] = arm_spe__alloc_per_cpu_metadata(ptr, per_cpu_sz); + if (!metadata[i]) + goto err_per_cpu_metadata; + + ptr += per_cpu_sz / sizeof(u64); + } + + return metadata; + +err_per_cpu_metadata: + arm_spe__free_metadata(metadata, *nr_cpu); + return NULL; +} + +static void arm_spe_free_queue(void *priv) +{ + struct arm_spe_queue *speq = priv; + + if (!speq) + return; + thread__zput(speq->thread); + arm_spe_decoder_free(speq->decoder); + zfree(&speq->event_buf); + zfree(&speq->last_branch); + free(speq); +} + +static void arm_spe_free_events(struct perf_session *session) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + struct auxtrace_queues *queues = &spe->queues; + unsigned int i; + + for (i = 0; i < queues->nr_queues; i++) { + arm_spe_free_queue(queues->queue_array[i].priv); + queues->queue_array[i].priv = NULL; + } + auxtrace_queues__free(queues); +} + +static void arm_spe_free(struct perf_session *session) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, + auxtrace); + + auxtrace_heap__free(&spe->heap); + arm_spe_free_events(session); + session->auxtrace = NULL; + arm_spe__free_metadata(spe->metadata, spe->metadata_nr_cpu); + free(spe); +} + +static bool arm_spe_evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct arm_spe *spe = container_of(session->auxtrace, struct arm_spe, auxtrace); + + return evsel->core.attr.type == spe->pmu_type; +} + +static const char * const metadata_hdr_v1_fmts[] = { + [ARM_SPE_PMU_TYPE] = " PMU Type :%"PRId64"\n", + [ARM_SPE_PER_CPU_MMAPS] = " Per CPU mmaps :%"PRId64"\n", +}; + +static const char * const metadata_hdr_fmts[] = { + [ARM_SPE_HEADER_VERSION] = " Header version :%"PRId64"\n", + [ARM_SPE_HEADER_SIZE] = " Header size :%"PRId64"\n", + [ARM_SPE_PMU_TYPE_V2] = " PMU type v2 :%"PRId64"\n", + [ARM_SPE_CPUS_NUM] = " CPU number :%"PRId64"\n", +}; + +static const char * const metadata_per_cpu_fmts[] = { + [ARM_SPE_MAGIC] = " Magic :0x%"PRIx64"\n", + [ARM_SPE_CPU] = " CPU # :%"PRId64"\n", + [ARM_SPE_CPU_NR_PARAMS] = " Num of params :%"PRId64"\n", + [ARM_SPE_CPU_MIDR] = " MIDR :0x%"PRIx64"\n", + [ARM_SPE_CPU_PMU_TYPE] = " PMU Type :%"PRId64"\n", + [ARM_SPE_CAP_MIN_IVAL] = " Min Interval :%"PRId64"\n", + [ARM_SPE_CAP_EVENT_FILTER] = " Event Filter :0x%"PRIx64"\n", +}; + +static void arm_spe_print_info(struct arm_spe *spe, __u64 *arr) +{ + unsigned int i, cpu, hdr_size, cpu_num, cpu_size; + const char * const *hdr_fmts; + + if (!dump_trace) + return; + + if (spe->metadata_ver == 1) { + cpu_num = 0; + hdr_size = ARM_SPE_AUXTRACE_V1_PRIV_MAX; + hdr_fmts = metadata_hdr_v1_fmts; + } else { + cpu_num = arr[ARM_SPE_CPUS_NUM]; + hdr_size = arr[ARM_SPE_HEADER_SIZE]; + hdr_fmts = metadata_hdr_fmts; + } + + for (i = 0; i < hdr_size; i++) + fprintf(stdout, hdr_fmts[i], arr[i]); + + arr += hdr_size; + for (cpu = 0; cpu < cpu_num; cpu++) { + /* + * The parameters from ARM_SPE_MAGIC to ARM_SPE_CPU_NR_PARAMS + * are fixed. The sequential parameter size is decided by the + * field 'ARM_SPE_CPU_NR_PARAMS'. + */ + cpu_size = (ARM_SPE_CPU_NR_PARAMS + 1) + arr[ARM_SPE_CPU_NR_PARAMS]; + for (i = 0; i < cpu_size; i++) + fprintf(stdout, metadata_per_cpu_fmts[i], arr[i]); + arr += cpu_size; + } +} + +static void arm_spe_set_event_name(struct evlist *evlist, u64 id, + const char *name) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.id && evsel->core.id[0] == id) { + if (evsel->name) + zfree(&evsel->name); + evsel->name = strdup(name); + break; + } + } +} + +static int +arm_spe_synth_events(struct arm_spe *spe, struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + struct perf_event_attr attr; + bool found = false; + u64 id; + int err; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == spe->pmu_type) { + found = true; + break; + } + } + + if (!found) { + pr_debug("No selected events with SPE trace data\n"); + return 0; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.size = sizeof(struct perf_event_attr); + attr.type = PERF_TYPE_HARDWARE; + attr.sample_type = evsel->core.attr.sample_type & + (PERF_SAMPLE_MASK | PERF_SAMPLE_PHYS_ADDR); + attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | + PERF_SAMPLE_PERIOD | PERF_SAMPLE_DATA_SRC | + PERF_SAMPLE_WEIGHT | PERF_SAMPLE_ADDR; + if (spe->timeless_decoding) + attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; + else + attr.sample_type |= PERF_SAMPLE_TIME; + + spe->sample_type = attr.sample_type; + + attr.exclude_user = evsel->core.attr.exclude_user; + attr.exclude_kernel = evsel->core.attr.exclude_kernel; + attr.exclude_hv = evsel->core.attr.exclude_hv; + attr.exclude_host = evsel->core.attr.exclude_host; + attr.exclude_guest = evsel->core.attr.exclude_guest; + attr.sample_id_all = evsel->core.attr.sample_id_all; + attr.read_format = evsel->core.attr.read_format; + attr.sample_period = spe->synth_opts.period; + + /* create new id val to be a fixed offset from evsel id */ + id = auxtrace_synth_id_range_start(evsel); + + if (spe->synth_opts.flc) { + spe->sample_flc = true; + + /* Level 1 data cache miss */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->l1d_miss_id = id; + arm_spe_set_event_name(evlist, id, "l1d-miss"); + id += 1; + + /* Level 1 data cache access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->l1d_access_id = id; + arm_spe_set_event_name(evlist, id, "l1d-access"); + id += 1; + } + + if (spe->synth_opts.llc) { + spe->sample_llc = true; + + /* Last level cache miss */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->llc_miss_id = id; + arm_spe_set_event_name(evlist, id, "llc-miss"); + id += 1; + + /* Last level cache access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->llc_access_id = id; + arm_spe_set_event_name(evlist, id, "llc-access"); + id += 1; + } + + if (spe->synth_opts.tlb) { + spe->sample_tlb = true; + + /* TLB miss */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->tlb_miss_id = id; + arm_spe_set_event_name(evlist, id, "tlb-miss"); + id += 1; + + /* TLB access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->tlb_access_id = id; + arm_spe_set_event_name(evlist, id, "tlb-access"); + id += 1; + } + + if (spe->synth_opts.last_branch) { + if (spe->synth_opts.last_branch_sz > 2) + pr_debug("Arm SPE supports only two bstack entries (PBT+TGT).\n"); + + attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + /* + * We don't use the hardware index, but the sample generation + * code uses the new format branch_stack with this field, + * so the event attributes must indicate that it's present. + */ + attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + + if (spe->synth_opts.branches) { + spe->sample_branch = true; + + /* Branch */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->branch_id = id; + arm_spe_set_event_name(evlist, id, "branch"); + id += 1; + } + + if (spe->synth_opts.remote_access) { + spe->sample_remote_access = true; + + /* Remote access */ + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->remote_access_id = id; + arm_spe_set_event_name(evlist, id, "remote-access"); + id += 1; + } + + if (spe->synth_opts.mem) { + spe->sample_memory = true; + + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->memory_id = id; + arm_spe_set_event_name(evlist, id, "memory"); + id += 1; + } + + if (spe->synth_opts.instructions) { + spe->sample_instructions = true; + attr.config = PERF_COUNT_HW_INSTRUCTIONS; + + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + spe->instructions_id = id; + arm_spe_set_event_name(evlist, id, "instructions"); + } + + return 0; +} + +static bool arm_spe__is_homogeneous(u64 **metadata, int nr_cpu) +{ + u64 midr; + int i; + + if (!nr_cpu) + return false; + + for (i = 0; i < nr_cpu; i++) { + if (!metadata[i]) + return false; + + if (i == 0) { + midr = metadata[i][ARM_SPE_CPU_MIDR]; + continue; + } + + if (midr != metadata[i][ARM_SPE_CPU_MIDR]) + return false; + } + + return true; +} + +int arm_spe_process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + size_t min_sz = ARM_SPE_AUXTRACE_V1_PRIV_SIZE; + struct perf_record_time_conv *tc = &session->time_conv; + struct arm_spe *spe; + u64 **metadata = NULL; + u64 metadata_ver; + int nr_cpu, err; + + if (auxtrace_info->header.size < sizeof(struct perf_record_auxtrace_info) + + min_sz) + return -EINVAL; + + metadata = arm_spe__alloc_metadata(auxtrace_info, &metadata_ver, + &nr_cpu); + if (!metadata && metadata_ver != 1) { + pr_err("Failed to parse Arm SPE metadata.\n"); + return -EINVAL; + } + + spe = zalloc(sizeof(struct arm_spe)); + if (!spe) { + err = -ENOMEM; + goto err_free_metadata; + } + + err = auxtrace_queues__init(&spe->queues); + if (err) + goto err_free; + + spe->session = session; + spe->machine = &session->machines.host; /* No kvm support */ + spe->auxtrace_type = auxtrace_info->type; + if (metadata_ver == 1) + spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; + else + spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE_V2]; + spe->metadata = metadata; + spe->metadata_ver = metadata_ver; + spe->metadata_nr_cpu = nr_cpu; + spe->is_homogeneous = arm_spe__is_homogeneous(metadata, nr_cpu); + + spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); + + /* + * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead + * and the parameters for hardware clock are stored in the session + * context. Passes these parameters to the struct perf_tsc_conversion + * in "spe->tc", which is used for later conversion between clock + * counter and timestamp. + * + * For backward compatibility, copies the fields starting from + * "time_cycles" only if they are contained in the event. + */ + spe->tc.time_shift = tc->time_shift; + spe->tc.time_mult = tc->time_mult; + spe->tc.time_zero = tc->time_zero; + + if (event_contains(*tc, cap_user_time_short)) { + spe->tc.time_cycles = tc->time_cycles; + spe->tc.time_mask = tc->time_mask; + spe->tc.cap_user_time_zero = tc->cap_user_time_zero; + spe->tc.cap_user_time_short = tc->cap_user_time_short; + } + + spe->auxtrace.process_event = arm_spe_process_event; + spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event; + spe->auxtrace.flush_events = arm_spe_flush; + spe->auxtrace.free_events = arm_spe_free_events; + spe->auxtrace.free = arm_spe_free; + spe->auxtrace.evsel_is_auxtrace = arm_spe_evsel_is_auxtrace; + session->auxtrace = &spe->auxtrace; + + arm_spe_print_info(spe, &auxtrace_info->priv[0]); + + if (dump_trace) + return 0; + + if (session->itrace_synth_opts && session->itrace_synth_opts->set) { + spe->synth_opts = *session->itrace_synth_opts; + } else { + itrace_synth_opts__set_default(&spe->synth_opts, false); + /* Default nanoseconds period not supported */ + spe->synth_opts.period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS; + spe->synth_opts.period = 1; + } + + if (spe->synth_opts.period_type != PERF_ITRACE_PERIOD_INSTRUCTIONS) { + ui__error("You must only use i (instructions) --itrace period with Arm SPE. e.g --itrace=i1i\n"); + err = -EINVAL; + goto err_free_queues; + } + if (spe->synth_opts.period > 1) + ui__warning("Arm SPE has a hardware-based sampling period.\n\n" + "--itrace periods > 1i downsample by an interval of n SPE samples rather than n instructions.\n"); + + err = arm_spe_synth_events(spe, session); + if (err) + goto err_free_queues; + + err = auxtrace_queues__process_index(&spe->queues, session); + if (err) + goto err_free_queues; + + if (spe->queues.populated) + spe->data_queued = true; + + return 0; + +err_free_queues: + auxtrace_queues__free(&spe->queues); + session->auxtrace = NULL; +err_free: + free(spe); +err_free_metadata: + arm_spe__free_metadata(metadata, nr_cpu); + return err; +} diff --git a/tools/perf/util/arm-spe.h b/tools/perf/util/arm-spe.h new file mode 100644 index 000000000..3966df185 --- /dev/null +++ b/tools/perf/util/arm-spe.h @@ -0,0 +1,71 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Arm Statistical Profiling Extensions (SPE) support + * Copyright (c) 2017-2018, Arm Ltd. + */ + +#ifndef INCLUDE__PERF_ARM_SPE_H__ +#define INCLUDE__PERF_ARM_SPE_H__ + +#define ARM_SPE_PMU_NAME "arm_spe_" + +enum { + ARM_SPE_PMU_TYPE, + ARM_SPE_PER_CPU_MMAPS, + ARM_SPE_AUXTRACE_V1_PRIV_MAX, +}; + +#define ARM_SPE_AUXTRACE_V1_PRIV_SIZE \ + (ARM_SPE_AUXTRACE_V1_PRIV_MAX * sizeof(u64)) + +enum { + /* + * The old metadata format (defined above) does not include a + * field for version number. Version 1 is reserved and starts + * from version 2. + */ + ARM_SPE_HEADER_VERSION, + /* Number of sizeof(u64) */ + ARM_SPE_HEADER_SIZE, + /* PMU type shared by CPUs */ + ARM_SPE_PMU_TYPE_V2, + /* Number of CPUs */ + ARM_SPE_CPUS_NUM, + ARM_SPE_AUXTRACE_PRIV_MAX, +}; + +enum { + /* Magic number */ + ARM_SPE_MAGIC, + /* CPU logical number in system */ + ARM_SPE_CPU, + /* Number of parameters */ + ARM_SPE_CPU_NR_PARAMS, + /* CPU MIDR */ + ARM_SPE_CPU_MIDR, + /* Associated PMU type */ + ARM_SPE_CPU_PMU_TYPE, + /* Minimal interval */ + ARM_SPE_CAP_MIN_IVAL, + /* Event filter */ + ARM_SPE_CAP_EVENT_FILTER, + ARM_SPE_CPU_PRIV_MAX, +}; + +#define ARM_SPE_HEADER_CURRENT_VERSION 2 + + +union perf_event; +struct perf_session; +struct perf_pmu; + +struct auxtrace_record *arm_spe_recording_init(int *err, + struct perf_pmu *arm_spe_pmu); + +int arm_spe_process_auxtrace_info(union perf_event *event, + struct perf_session *session); + +void arm_spe_pmu_default_config(const struct perf_pmu *arm_spe_pmu, + struct perf_event_attr *attr); + +#endif diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.c b/tools/perf/util/arm64-frame-pointer-unwind-support.c new file mode 100644 index 000000000..3af8c7a46 --- /dev/null +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.c @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "arm64-frame-pointer-unwind-support.h" +#include "callchain.h" +#include "event.h" +#include "record.h" +#include "unwind.h" +#include + +#define perf_event_arm_regs perf_event_arm64_regs +#include "../../arch/arm64/include/uapi/asm/perf_regs.h" +#undef perf_event_arm_regs + +struct entries { + u64 stack[2]; + size_t length; +}; + +#define SMPL_REG_MASK(b) (1ULL << (b)) + +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts) +{ + opts->sample_user_regs |= SMPL_REG_MASK(PERF_REG_ARM64_LR); +} + +static bool get_leaf_frame_caller_enabled(struct perf_sample *sample) +{ + struct regs_dump *regs; + + if (callchain_param.record_mode != CALLCHAIN_FP) + return false; + + regs = perf_sample__user_regs(sample); + return regs->regs && regs->mask & SMPL_REG_MASK(PERF_REG_ARM64_LR); +} + +static int add_entry(struct unwind_entry *entry, void *arg) +{ + struct entries *entries = arg; + + entries->stack[entries->length++] = entry->ip; + return 0; +} + +u64 get_leaf_frame_caller_aarch64(struct perf_sample *sample, struct thread *thread, int usr_idx) +{ + int ret; + struct entries entries = {}; + struct regs_dump old_regs, *regs; + + if (!get_leaf_frame_caller_enabled(sample)) + return 0; + + /* + * If PC and SP are not recorded, get the value of PC from the stack + * and set its mask. SP is not used when doing the unwinding but it + * still needs to be set to prevent failures. + */ + regs = perf_sample__user_regs(sample); + memcpy(&old_regs, regs, sizeof(*regs)); + if (!(regs->mask & SMPL_REG_MASK(PERF_REG_ARM64_PC))) { + regs->cache_mask |= SMPL_REG_MASK(PERF_REG_ARM64_PC); + regs->cache_regs[PERF_REG_ARM64_PC] = sample->callchain->ips[usr_idx+1]; + } + + if (!(regs->mask & SMPL_REG_MASK(PERF_REG_ARM64_SP))) { + regs->cache_mask |= SMPL_REG_MASK(PERF_REG_ARM64_SP); + regs->cache_regs[PERF_REG_ARM64_SP] = 0; + } + + ret = unwind__get_entries(add_entry, &entries, thread, sample, 2, true); + memcpy(regs, &old_regs, sizeof(*regs)); + + if (ret || entries.length != 2) + return ret; + + return callchain_param.order == ORDER_CALLER ? entries.stack[0] : entries.stack[1]; +} diff --git a/tools/perf/util/arm64-frame-pointer-unwind-support.h b/tools/perf/util/arm64-frame-pointer-unwind-support.h new file mode 100644 index 000000000..ba35b295b --- /dev/null +++ b/tools/perf/util/arm64-frame-pointer-unwind-support.h @@ -0,0 +1,14 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H +#define __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H + +#include + +struct perf_sample; +struct record_opts; +struct thread; + +void add_leaf_frame_caller_opts_aarch64(struct record_opts *opts); +u64 get_leaf_frame_caller_aarch64(struct perf_sample *sample, struct thread *thread, int user_idx); + +#endif /* __PERF_ARM_FRAME_POINTER_UNWIND_SUPPORT_H */ diff --git a/tools/perf/util/aslr.c b/tools/perf/util/aslr.c new file mode 100644 index 000000000..6a7542e7d --- /dev/null +++ b/tools/perf/util/aslr.c @@ -0,0 +1,1409 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "aslr.h" + +#include "addr_location.h" +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "evlist.h" +#include "machine.h" +#include "map.h" +#include "thread.h" +#include "tool.h" +#include "session.h" +#include "data.h" +#include "dso.h" +#include "pmus.h" + +#include /* page_size */ +#include +#include +#include +#include +#include +#include + +/** + * struct remap_addresses_key - Key for mapping original addresses to remapped ones. + * @dso: Pointer to the DSO (Dynamic Shared Object) associated with the mapping. + * @invariant: Unique offset invariant within the VMA (Virtual Memory Area). + * Calculated as `start - pgoff`. This value remains constant when + * perf's internal `maps__fixup_overlap_and_insert` splits a map into + * fragmented VMA pieces due to overlapping events, allowing us to + * resolve split maps consistently back to the original VMA. + * @pid: Process ID associated with the mapping. + */ +struct remap_addresses_key { + struct machine *machine; + struct dso *dso; + u64 invariant; + pid_t pid; +}; + +struct aslr_mapping { + struct list_head node; + u64 orig_start; + u64 len; + u64 remap_start; +}; + +struct aslr_evsel_priv { + u64 orig_sample_type; + u64 orig_sample_regs_user; + u64 orig_sample_regs_intr; + int orig_sample_size; +}; + +static size_t evsel_hash(long key, void *ctx __maybe_unused) +{ + return (size_t)key; +} + +static bool evsel_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +struct process_top_address { + u64 remapped_max; +}; +struct aslr_tool { + /** @tool: The tool implemented here and a pointer to a delegate to process the data. */ + struct delegate_tool tool; + /** @machines: The machines with the input, not remapped, virtual address layout. */ + struct machines machines; + /** @event_copy: Buffer used to create an event to pass to the delegate. */ + char event_copy[PERF_SAMPLE_MAX_SIZE] __aligned(8); + /** @remap_addresses: mapping from remap_addresses_key to remapped address. */ + struct hashmap remap_addresses; + /** @top_addresses: mapping from process to max remapped address. */ + struct hashmap top_addresses; + /** + * @evsel_orig_attrs: mapping from evsel pointer to its original + * unstripped sample_type and registers bitmasks. + */ + struct hashmap evsel_orig_attrs; +}; + +static const pid_t kernel_pid = -1; + +/* Start remapping user processes from a small non-zero offset. */ +static const u64 user_space_start = 0x200000; +static const u64 kernel_space_start_64 = 0xffff800010000000ULL; +static const u64 kernel_space_start_32 = 0x80000000ULL; + +static size_t remap_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key = (struct remap_addresses_key *)_key; + void *dso_ptr = key->dso ? RC_CHK_ACCESS(key->dso) : NULL; + + return (size_t)key->machine ^ (size_t)dso_ptr ^ key->invariant ^ key->pid; +} + +static bool remap_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct remap_addresses_key *key1 = (struct remap_addresses_key *)_key1; + struct remap_addresses_key *key2 = (struct remap_addresses_key *)_key2; + + return key1->machine == key2->machine && + RC_CHK_EQUAL(key1->dso, key2->dso) && + key1->invariant == key2->invariant && + key1->pid == key2->pid; +} + +struct top_addresses_key { + struct machine *machine; + pid_t pid; +}; + +static size_t top_addresses__hash(long _key, void *ctx __maybe_unused) +{ + struct top_addresses_key *key = (struct top_addresses_key *)_key; + + return (size_t)key->machine ^ key->pid; +} + +static bool top_addresses__equal(long _key1, long _key2, void *ctx __maybe_unused) +{ + struct top_addresses_key *key1 = (struct top_addresses_key *)_key1; + struct top_addresses_key *key2 = (struct top_addresses_key *)_key2; + + return key1->machine == key2->machine && key1->pid == key2->pid; +} + +static u64 round_up_to_page_size(u64 addr) +{ + return (addr + page_size - 1) & ~((u64)page_size - 1); +} + +static u64 aslr_tool__remap_address(struct aslr_tool *aslr, + struct thread *aslr_thread, + u8 cpumode, + u64 addr) +{ + struct addr_location al; + struct remap_addresses_key key; + u64 *remapped_invariant_ptr = NULL; + u64 remap_addr = 0; + u8 effective_cpumode = cpumode; + struct dso *dso; + const char *dso_name; + + if (!aslr_thread) + return 0; /* No thread. */ + + addr_location__init(&al); + if (!thread__find_map(aslr_thread, cpumode, addr, &al)) { + /* + * If lookup fails with specified cpumode, try fallback to the other space + * to be robust against bad cpumode in samples. + */ + if (cpumode == PERF_RECORD_MISC_KERNEL) + effective_cpumode = PERF_RECORD_MISC_USER; + else if (cpumode == PERF_RECORD_MISC_USER) + effective_cpumode = PERF_RECORD_MISC_KERNEL; + else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL) + effective_cpumode = PERF_RECORD_MISC_GUEST_USER; + else if (cpumode == PERF_RECORD_MISC_GUEST_USER) + effective_cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + + if (!thread__find_map(aslr_thread, effective_cpumode, addr, &al)) { + addr_location__exit(&al); + return 0; /* No mmap. */ + } + } + + dso = map__dso(al.map); + dso_name = dso ? dso__long_name(dso) : NULL; + + key.machine = maps__machine(thread__maps(aslr_thread)); + key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + key.invariant = map__start(al.map) - map__pgoff(al.map); + else + key.invariant = map__start(al.map); + key.pid = (effective_cpumode == PERF_RECORD_MISC_KERNEL || + effective_cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + + if (hashmap__find(&aslr->remap_addresses, &key, &remapped_invariant_ptr)) { + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (addr - map__start(al.map)); + } else { + pr_debug("Cannot find a remapped entry for address %" PRIx64 " in mapping %" PRIx64 "(%zu) for pid=%d\n", + addr, map__start(al.map), map__size(al.map), key.pid); + } + + addr_location__exit(&al); + return remap_addr; +} + +struct aslr_machine_priv { + bool kernel_maps_loaded; +}; + +static int aslr_tool__preload_kernel_maps(struct machine *machine) +{ + struct aslr_machine_priv *mpriv = machine->priv; + + if (!mpriv) { + mpriv = zalloc(sizeof(*mpriv)); + if (!mpriv) + return -ENOMEM; + machine->priv = mpriv; + } + + if (!mpriv->kernel_maps_loaded) { + struct maps *kmaps = machine__kernel_maps(machine); + + if (kmaps) { + int err = maps__load_maps(kmaps); + + if (err < 0) { + pr_err("ASLR: Failed to preload kernel maps for machine pid %d\n", + machine->pid); + return err; + } + } + mpriv->kernel_maps_loaded = true; + } + return 0; +} + +static void aslr_tool__free_machine_priv(struct machine *machine) +{ + free(machine->priv); + machine->priv = NULL; +} + +static void aslr_tool__destroy_machines_priv(struct machines *machines) +{ + struct rb_node *nd; + + aslr_tool__free_machine_priv(&machines->host); + for (nd = rb_first_cached(&machines->guests); nd; nd = rb_next(nd)) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + aslr_tool__free_machine_priv(machine); + } +} + +static u64 aslr_tool__findnew_mapping(struct aslr_tool *aslr, + struct machine *session_machine, + struct thread *aslr_thread, + u8 cpumode, u64 start, + u64 len, u64 pgoff) +{ + /* Address location for dso lookup. */ + struct addr_location al; + /* Original ASLR address based key for the remap table. */ + struct remap_addresses_key remap_key; + /* The address in the ASLR sanitized address space less pg_off. */ + u64 *remapped_invariant_ptr; + /* Key for the maximum address in a process. */ + struct top_addresses_key top_addr_key; + /* Value in top address table. */ + struct process_top_address *top = NULL; + /* Address in ASLR sanitized address space. */ + u64 remap_addr; + /* Potentially allocated remap table key. */ + struct remap_addresses_key *new_remap_key = NULL; + /* + * Potentially allocated remap table key. + * TODO: Avoid allocation necessary for perf 32-bit binary support. + */ + u64 *new_remap_val = NULL; + int err; + + if (!aslr_thread) + return 0; + + /* The key to look up an incoming address to the outgoing value. */ + addr_location__init(&al); + remap_key.machine = maps__machine(thread__maps(aslr_thread)); + remap_key.pid = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_pid : thread__pid(aslr_thread); + if (thread__find_map(aslr_thread, cpumode, start, &al)) { + struct dso *dso = map__dso(al.map); + const char *dso_name = dso ? dso__long_name(dso) : NULL; + + remap_key.dso = dso; + if (dso && !is_anon_memory(dso_name) && !is_no_dso_memory(dso_name)) + remap_key.invariant = map__start(al.map) - map__pgoff(al.map); + else + remap_key.invariant = map__start(al.map); + } else { + remap_key.dso = NULL; + remap_key.invariant = start; + } + + /* The key to look up top allocated address. */ + top_addr_key.machine = remap_key.machine; + top_addr_key.pid = remap_key.pid; + + if (hashmap__find(&aslr->remap_addresses, &remap_key, &remapped_invariant_ptr)) { + /* Mmap already exists. */ + u64 calculated_max; + + if (al.map) { + /* + * The cached value is the base of the invariant. We add the + * offset into the VMA (start - map__start), plus the map's + * pgoff, to get the precise virtual address within this chunk. + */ + remap_addr = *remapped_invariant_ptr + map__pgoff(al.map) + + (start - map__start(al.map)); + } else { + /* + * For unmapped memory (e.g. kernel anonymous), the cached value + * was stored offset by pgoff. Adding pgoff yields the true remap_addr. + */ + remap_addr = *remapped_invariant_ptr + pgoff; + } + + calculated_max = remap_addr + len; + + /* See if top mapping was expanded. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + if (calculated_max > top->remapped_max) + top->remapped_max = calculated_max; + } + addr_location__exit(&al); + return remap_addr; + } + /* No mmap, create an entry from the top address. */ + if (hashmap__find(&aslr->top_addresses, &top_addr_key, &top)) { + struct addr_location prev_al; + bool is_contiguous = false; + + /* Current max allocated mmap address within the process. */ + remap_addr = top->remapped_max; + + addr_location__init(&prev_al); + if (thread__find_map(aslr_thread, cpumode, start - 1, &prev_al)) { + if (map__end(prev_al.map) == start) + is_contiguous = true; + } + addr_location__exit(&prev_al); + + if (is_contiguous) { + /* Contiguous mapping, do not add 1 page gap! */ + remap_addr = round_up_to_page_size(remap_addr); + } else { + /* Give 1 page gap from current max page. */ + remap_addr = round_up_to_page_size(remap_addr); + remap_addr += page_size; + } + if (remap_addr + len > top->remapped_max) + top->remapped_max = remap_addr + len; + } else { + /* First address of the process, allocate key and first top address. */ + struct top_addresses_key *tk; + struct process_top_address *top_val; + struct perf_env *env = session_machine ? session_machine->env : NULL; + bool is_64 = env ? perf_env__kernel_is_64_bit(env) : (sizeof(void *) == 8); + u64 kernel_start_addr = is_64 ? kernel_space_start_64 : kernel_space_start_32; + + remap_addr = (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) ? + kernel_start_addr : user_space_start; + remap_addr = round_up_to_page_size(remap_addr); + + tk = malloc(sizeof(*tk)); + top_val = malloc(sizeof(*top_val)); + if (!tk || !top_val) { + err = -ENOMEM; + } else { + *tk = top_addr_key; + top_val->remapped_max = remap_addr + len; + err = hashmap__insert(&aslr->top_addresses, tk, top_val, + HASHMAP_ADD, NULL, NULL); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR process top address %m\n"); + free(tk); + free(top_val); + addr_location__exit(&al); + return 0; + } + } + /* Create rmeapping entry. */ + new_remap_key = malloc(sizeof(*new_remap_key)); + new_remap_val = malloc(sizeof(u64)); + if (!new_remap_key || !new_remap_val) { + err = -ENOMEM; + } else { + *new_remap_key = remap_key; + new_remap_key->dso = dso__get(remap_key.dso); + if (cpumode == PERF_RECORD_MISC_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) { + if (al.map) { + *new_remap_val = remap_addr - + (start - map__start(al.map)) - + map__pgoff(al.map); + } else { + /* + * Subtract pgoff from the base virtual address so that + * when the lookup path adds pgoff back, it perfectly + * cancels out and returns remap_addr. + */ + *new_remap_val = remap_addr - pgoff; + } + } else { + *new_remap_val = remap_addr - (al.map ? (start - map__start(al.map)) + + map__pgoff(al.map) : pgoff); + } + err = hashmap__add(&aslr->remap_addresses, new_remap_key, new_remap_val); + if (err) + dso__put(new_remap_key->dso); + } + if (err) { + errno = -err; + pr_err("Failure to add ASLR remapping %m\n"); + free(new_remap_key); + free(new_remap_val); + addr_location__exit(&al); + return 0; + } + addr_location__exit(&al); + return remap_addr; +} + +static int aslr_tool__process_mmap(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap.pid, event->mmap.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap, &event->mmap, event->mmap.header.size); + /* Remaps the mmap.start. */ + new_event->mmap.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap.start, + event->mmap.len, + event->mmap.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap.filename) || is_no_dso_memory(event->mmap.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap.filename, cpumode))) + new_event->mmap.pgoff = new_event->mmap.start; + err = delegate->mmap(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_mmap2(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + u8 cpumode; + struct thread *thread; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_mmap2(tool, event, sample, aslr_machine); + if (err) + return err; + + thread = machine__findnew_thread(aslr_machine, event->mmap2.pid, event->mmap2.tid); + if (!thread) + return -ENOMEM; + memcpy(&new_event->mmap2, &event->mmap2, event->mmap2.header.size); + /* Remaps the mmap.start. */ + new_event->mmap2.start = aslr_tool__findnew_mapping(aslr, machine, thread, cpumode, + event->mmap2.start, + event->mmap2.len, + event->mmap2.pgoff); + /* + * For anonymous memory (and kernel maps), the kernel populates the + * event's pgoff field with the original un-obfuscated virtual address + * in bytes (i.e. (addr >> PAGE_SHIFT) << PAGE_SHIFT). + * We must overwrite pgoff with the new remapped byte address to prevent + * leaking the original ASLR layout. + */ + if (is_anon_memory(event->mmap2.filename) || is_no_dso_memory(event->mmap2.filename) || + ((cpumode == PERF_RECORD_MISC_KERNEL || cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !is_kernel_module(event->mmap2.filename, cpumode))) + new_event->mmap2.pgoff = new_event->mmap2.start; + err = delegate->mmap2(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_comm(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_comm(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->comm(delegate, event, sample, machine); +} + +static int aslr_tool__process_fork(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_fork(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->fork(delegate, event, sample, machine); +} + +static int aslr_tool__process_exit(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + struct machine *aslr_machine; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + /* Create the thread, map, etc. in the ASLR before virtual address space. */ + err = perf_event__process_exit(tool, event, sample, aslr_machine); + if (err) + return err; + + return delegate->exit(delegate, event, sample, machine); +} + +static int aslr_tool__process_text_poke(const struct perf_tool *tool __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + struct machine *machine __maybe_unused) +{ + /* Drop in case the instruction encodes an ASLR revealing address. */ + return 0; +} + +static int aslr_tool__process_ksymbol(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + union perf_event *new_event; + struct thread *thread; + struct machine *aslr_machine; + bool is_unregister; + int err; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + new_event = (union perf_event *)aslr->event_copy; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, kernel_pid, 0); + if (!thread) + return -ENOMEM; + + is_unregister = (event->ksymbol.flags & PERF_RECORD_KSYMBOL_FLAGS_UNREGISTER); + + memcpy(&new_event->ksymbol, &event->ksymbol, event->ksymbol.header.size); + + if (is_unregister) { + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + } else { + err = perf_event__process_ksymbol(tool, event, sample, aslr_machine); + new_event->ksymbol.addr = aslr_tool__findnew_mapping(aslr, machine, thread, + PERF_RECORD_MISC_KERNEL, + event->ksymbol.addr, + event->ksymbol.len, + /*pgoff=*/0); + } + if (err) { + thread__put(thread); + return err; + } + + err = delegate->ksymbol(delegate, new_event, sample, machine); + thread__put(thread); + return err; +} + +static int aslr_tool__process_sample(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct evsel *evsel = sample->evsel; + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct perf_tool *delegate; + int ret; + int orig_sample_size; + u64 sample_type; + struct thread *thread; + struct machine *aslr_machine; + __u64 max_i; + __u64 max_j; + union perf_event *new_event; + struct perf_sample new_sample; + __u64 *in_array, *out_array; + u8 cpumode; + u64 addr; + size_t i; + size_t j; + struct aslr_evsel_priv *priv = NULL; + u64 orig_sample_type; + u64 orig_regs_user; + u64 orig_regs_intr; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + delegate = aslr->tool.delegate; + + if (evsel__is_dummy_event(evsel)) + return delegate->sample(delegate, event, sample, machine); + + ret = -EFAULT; + + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + orig_sample_type = priv->orig_sample_type; + orig_regs_user = priv->orig_sample_regs_user; + orig_regs_intr = priv->orig_sample_regs_intr; + } else { + orig_sample_type = evsel->core.attr.sample_type; + orig_regs_user = evsel->core.attr.sample_regs_user; + orig_regs_intr = evsel->core.attr.sample_regs_intr; + } + + orig_sample_size = evsel->sample_size; + + sample_type = orig_sample_type; + sample_type &= ~PERF_SAMPLE_REGS_USER; + sample_type &= ~PERF_SAMPLE_REGS_INTR; + sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + max_i = (event->header.size - sizeof(struct perf_event_header)) / sizeof(__u64); + max_j = (PERF_SAMPLE_MAX_SIZE - sizeof(struct perf_event_header)) / sizeof(__u64); + new_event = (union perf_event *)aslr->event_copy; + cpumode = sample->cpumode; + i = 0; + j = 0; + + aslr_machine = machines__findnew(&aslr->machines, machine->pid); + if (!aslr_machine) + return -ENOMEM; + if (aslr_tool__preload_kernel_maps(aslr_machine) < 0) + return -ENOMEM; + + thread = machine__findnew_thread(aslr_machine, sample->pid, sample->tid); + + if (!thread) + return -ENOMEM; + + if (max_i > PERF_SAMPLE_MAX_SIZE / sizeof(u64)) + goto out_put; + + new_event->sample.header = event->sample.header; + + in_array = &event->sample.array[0]; + out_array = &new_event->sample.array[0]; + +#define CHECK_BOUNDS(required_i, required_j) \ + (i + (required_i) > max_i || j + (required_j) > max_j) + +#define COPY_U64() \ + do { \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + out_array[j++] = in_array[i++]; \ + } while (0) + +#define REMAP_U64(addr_field) \ + do { \ + u64 remapped; \ + if (CHECK_BOUNDS(1, 1)) { \ + ret = -EFAULT; \ + goto out_put; \ + } \ + remapped = aslr_tool__remap_address(aslr, thread, cpumode, addr_field); \ + out_array[j++] = remapped; \ + i++; \ + } while (0) + + if (orig_sample_type & PERF_SAMPLE_IDENTIFIER) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_IP) + REMAP_U64(sample->ip); + if (orig_sample_type & PERF_SAMPLE_TID) { + union { + u64 val64; + u32 val32[2]; + } u; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + u.val32[0] = sample->pid; + u.val32[1] = sample->tid; + out_array[j++] = u.val64; + i++; + } + if (orig_sample_type & PERF_SAMPLE_TIME) + COPY_U64(); /* time */ + if (orig_sample_type & PERF_SAMPLE_ADDR) + REMAP_U64(sample->addr); + if (orig_sample_type & PERF_SAMPLE_ID) + COPY_U64(); /* id */ + if (orig_sample_type & PERF_SAMPLE_STREAM_ID) + COPY_U64(); /* stream_id */ + if (orig_sample_type & PERF_SAMPLE_CPU) + COPY_U64(); /* cpu, res */ + if (orig_sample_type & PERF_SAMPLE_PERIOD) + COPY_U64(); /* period */ + if (orig_sample_type & PERF_SAMPLE_READ) { + if ((evsel->core.attr.read_format & PERF_FORMAT_GROUP) == 0) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } else { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + COPY_U64(); /* time_enabled */ + if (evsel->core.attr.read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + COPY_U64(); /* time_running */ + for (u64 cntr = 0; cntr < nr; cntr++) { + COPY_U64(); /* value */ + if (evsel->core.attr.read_format & PERF_FORMAT_ID) + COPY_U64(); /* id */ + if (evsel->core.attr.read_format & PERF_FORMAT_LOST) + COPY_U64(); /* lost */ + } + } + } + if (orig_sample_type & PERF_SAMPLE_CALLCHAIN) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + for (u64 cntr = 0; cntr < nr; cntr++) { + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + addr = in_array[i++]; + if (addr >= PERF_CONTEXT_MAX) { + out_array[j++] = addr; + switch (addr) { + case PERF_CONTEXT_HV: + cpumode = PERF_RECORD_MISC_HYPERVISOR; + break; + case PERF_CONTEXT_KERNEL: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + case PERF_CONTEXT_USER: + cpumode = PERF_RECORD_MISC_USER; + break; + case PERF_CONTEXT_GUEST: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_KERNEL: + cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + break; + case PERF_CONTEXT_GUEST_USER: + cpumode = PERF_RECORD_MISC_GUEST_USER; + break; + case PERF_CONTEXT_USER_DEFERRED: + if (cntr + 1 >= nr) { + pr_debug("Truncated callchain deferred cookie context\n"); + ret = 0; + goto out_put; + } + /* + * Immediately followed by a 64-bit + * stitching cookie. Skip/Copy it! + */ + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j++] = in_array[i++]; + cntr++; + cpumode = PERF_RECORD_MISC_USER; + break; + default: + pr_debug("invalid callchain context: %"PRIx64"\n", addr); + ret = 0; + goto out_put; + } + continue; + } + addr = aslr_tool__remap_address(aslr, thread, cpumode, addr); + out_array[j++] = addr; + } + } + if (orig_sample_type & PERF_SAMPLE_RAW) { + size_t bytes = sizeof(u32) + sample->raw_size; + size_t u64_words = (bytes + 7) / 8; + + if (i + u64_words > max_i || j + u64_words > max_j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], bytes); + i += u64_words; + j += u64_words; + /* + * TODO: certain raw samples can be remapped, such as + * tracepoints by examining their fields. + */ + pr_debug("Dropping raw samples as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_BRANCH_STACK) { + u64 nr; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + nr = in_array[i]; + COPY_U64(); + + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) + COPY_U64(); /* hw_idx */ + + if (nr > (ULLONG_MAX / 3)) { + ret = -EFAULT; + goto out_put; + } + if (nr * 3 > max_i - i || nr * 3 > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) { + u64 from = in_array[i++]; + u64 to = in_array[i++]; + + from = aslr_tool__remap_address(aslr, thread, sample->cpumode, from); + to = aslr_tool__remap_address(aslr, thread, sample->cpumode, to); + + out_array[j++] = from; + out_array[j++] = to; + out_array[j++] = in_array[i++]; /* flags */ + } + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) { + if (nr > max_i - i || nr > max_j - j) { + ret = -EFAULT; + goto out_put; + } + for (u64 cntr = 0; cntr < nr; cntr++) + COPY_U64(); + } + } + if (orig_sample_type & PERF_SAMPLE_REGS_USER) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_user); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_STACK_USER) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + size = in_array[i]; + COPY_U64(); + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping stack user sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_WEIGHT_TYPE) + COPY_U64(); /* perf_sample_weight */ + if (orig_sample_type & PERF_SAMPLE_DATA_SRC) + COPY_U64(); /* data_src */ + if (orig_sample_type & PERF_SAMPLE_TRANSACTION) + COPY_U64(); /* transaction */ + if (orig_sample_type & PERF_SAMPLE_REGS_INTR) { + u64 abi; + + if (CHECK_BOUNDS(1, 0)) { + ret = -EFAULT; + goto out_put; + } + abi = in_array[i++]; + if (abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 nr = hweight64(orig_regs_intr); + + if (nr > max_i - i) { + ret = -EFAULT; + goto out_put; + } + i += nr; + } + } + if (orig_sample_type & PERF_SAMPLE_PHYS_ADDR) { + COPY_U64(); /* phys_addr */ + /* TODO: can this be less conservative? */ + pr_debug("Dropping physical address sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + if (orig_sample_type & PERF_SAMPLE_CGROUP) + COPY_U64(); /* cgroup */ + if (orig_sample_type & PERF_SAMPLE_DATA_PAGE_SIZE) + COPY_U64(); /* data_page_size */ + if (orig_sample_type & PERF_SAMPLE_CODE_PAGE_SIZE) + COPY_U64(); /* code_page_size */ + + if (orig_sample_type & PERF_SAMPLE_AUX) { + u64 size; + + if (CHECK_BOUNDS(1, 1)) { + ret = -EFAULT; + goto out_put; + } + out_array[j] = in_array[i]; + size = out_array[j++]; + i++; + if (size > 0) { + size_t u64_words = size / 8 + (size % 8 ? 1 : 0); + + if (u64_words > max_i - i || u64_words > max_j - j) { + ret = -EFAULT; + goto out_put; + } + memcpy(&out_array[j], &in_array[i], size); + if (size % 8) { + size_t pad = 8 - (size % 8); + + memset(((char *)&out_array[j]) + size, 0, pad); + } + i += u64_words; + j += u64_words; + } + /* TODO: can this be less conservative? */ + pr_debug("Dropping aux sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + if (evsel__is_offcpu_event(evsel)) { + /* TODO: can this be less conservative? */ + pr_debug("Dropping off-CPU sample as possible ASLR leak\n"); + ret = 0; + goto out_put; + } + + new_event->sample.header.size = sizeof(struct perf_event_header) + j * sizeof(u64); + /* Temporarily override evsel attributes to match the stripped new_event format! */ + evsel->sample_size = __evsel__sample_size(sample_type); + evsel->core.attr.sample_type = sample_type; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + perf_sample__init(&new_sample, /*all=*/ true); + ret = __evsel__parse_sample(evsel, new_event, &new_sample, /*needs_swap=*/false); + + if (ret) { + /* Restore original attributes immediately if parsing fails */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + perf_sample__exit(&new_sample); + goto out_put; + } + + new_sample.evsel = evsel; + ret = delegate->sample(delegate, new_event, &new_sample, machine); + perf_sample__exit(&new_sample); + + /* Restore original attributes so trace ingestion never desynchronizes! */ + evsel->sample_size = orig_sample_size; + evsel->core.attr.sample_type = orig_sample_type; + evsel->core.attr.sample_regs_user = orig_regs_user; + evsel->core.attr.sample_regs_intr = orig_regs_intr; + +out_put: + thread__put(thread); + return ret; +} + +#undef CHECK_BOUNDS +#undef COPY_U64 +#undef REMAP_U64 + +static int skipn(int fd, off_t n) +{ + char buf[4096]; + ssize_t ret; + + while (n > 0) { + ret = read(fd, buf, min_t(off_t, n, (off_t)sizeof(buf))); + if (ret <= 0) + return ret; + n -= ret; + } + + return 0; +} + +static s64 aslr_tool__process_auxtrace(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + pr_warning_once("ASLR: Dropping auxtrace data as it cannot be obfuscated.\n"); + if (perf_data__is_pipe(session->data)) { + /* Copy behavior of the stub by reading all pipe data. */ + int err = skipn(perf_data__fd(session->data), event->auxtrace.size); + + if (err < 0) + return err; + } + return event->auxtrace.size; +} + +static int aslr_tool__process_auxtrace_info(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +static int aslr_tool__process_auxtrace_error(const struct perf_tool *tool __maybe_unused, + struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist) +{ + u32 attr_size; + + if (!evlist) + return; + + attr_size = event->attr.attr.size ?: PERF_ATTR_SIZE_VER0; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_type) + sizeof(u64))) { + event->attr.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_user) + sizeof(u64))) + event->attr.attr.sample_regs_user = 0; + if (attr_size >= (offsetof(struct perf_event_attr, sample_regs_intr) + sizeof(u64))) + event->attr.attr.sample_regs_intr = 0; + } + + if (attr_size >= (offsetof(struct perf_event_attr, type) + sizeof(u32))) { + u32 type = event->attr.attr.type; + + if (type == PERF_TYPE_BREAKPOINT && + attr_size >= (offsetof(struct perf_event_attr, bp_addr) + sizeof(u64))) { + event->attr.attr.bp_addr = 0; + } else if (type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu; + + pmu = perf_pmus__find_by_type(type); + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + if (attr_size >= (offsetof(struct perf_event_attr, config1) + sizeof(u64))) + event->attr.attr.config1 = 0; + if (attr_size >= (offsetof(struct perf_event_attr, config2) + sizeof(u64))) + event->attr.attr.config2 = 0; + } + } + } +} + +static int aslr_tool__init(struct aslr_tool *aslr, struct perf_tool *delegate) +{ + delegate_tool__init(&aslr->tool, delegate); + aslr->tool.tool.ordered_events = true; + + if (machines__init(&aslr->machines)) + return -ENOMEM; + + hashmap__init(&aslr->remap_addresses, + remap_addresses__hash, remap_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->top_addresses, + top_addresses__hash, top_addresses__equal, + /*ctx=*/NULL); + hashmap__init(&aslr->evsel_orig_attrs, + evsel_hash, evsel_equal, + /*ctx=*/NULL); + + aslr->tool.tool.sample = aslr_tool__process_sample; + /* read - reads a counter, okay to delegate. */ + aslr->tool.tool.mmap = aslr_tool__process_mmap; + aslr->tool.tool.mmap2 = aslr_tool__process_mmap2; + aslr->tool.tool.comm = aslr_tool__process_comm; + aslr->tool.tool.fork = aslr_tool__process_fork; + aslr->tool.tool.exit = aslr_tool__process_exit; + /* namesspaces, cgroup, lost, lost_sample, aux, */ + /* itrace_start, aux_output_hw_id, context_switch, throttle, unthrottle */ + /* - no virtual addresses. */ + aslr->tool.tool.ksymbol = aslr_tool__process_ksymbol; + /* bpf - no virtual address. */ + aslr->tool.tool.text_poke = aslr_tool__process_text_poke; + /* + * event_update, tracing_data, finished_round, build_id, id_index, + * auxtrace_info, auxtrace_error, time_conv, thread_map, cpu_map, + * stat_config, stat, feature, finished_init, bpf_metadata, compressed, + * auxtrace - no virtual addresses. + */ + aslr->tool.tool.auxtrace = aslr_tool__process_auxtrace; + aslr->tool.tool.auxtrace_info = aslr_tool__process_auxtrace_info; + aslr->tool.tool.auxtrace_error = aslr_tool__process_auxtrace_error; + + return 0; +} + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate) +{ + struct aslr_tool *aslr = zalloc(sizeof(*aslr)); + + if (!aslr) + return NULL; + + if (aslr_tool__init(aslr, delegate)) { + free(aslr); + return NULL; + } + return &aslr->tool.tool; +} + +void aslr_tool__delete(struct perf_tool *tool) +{ + struct delegate_tool *del_tool; + struct aslr_tool *aslr; + struct hashmap_entry *cur; + size_t bkt; + struct rb_node *nd; + + if (!tool) + return; + + del_tool = container_of(tool, struct delegate_tool, tool); + aslr = container_of(del_tool, struct aslr_tool, tool); + + hashmap__for_each_entry(&aslr->remap_addresses, cur, bkt) { + struct remap_addresses_key *key = (struct remap_addresses_key *)cur->pkey; + + if (key) + dso__put(key->dso); + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->top_addresses, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__for_each_entry(&aslr->evsel_orig_attrs, cur, bkt) { + zfree(&cur->pvalue); + } + + hashmap__clear(&aslr->remap_addresses); + hashmap__clear(&aslr->top_addresses); + hashmap__clear(&aslr->evsel_orig_attrs); + aslr_tool__destroy_machines_priv(&aslr->machines); + machines__destroy_kernel_maps(&aslr->machines); + + while ((nd = rb_first_cached(&aslr->machines.guests)) != NULL) { + struct machine *machine = rb_entry(nd, struct machine, rb_node); + + rb_erase_cached(nd, &aslr->machines.guests); + machine__delete(machine); + } + + machines__exit(&aslr->machines); + free(aslr); +} + +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel) +{ + struct delegate_tool *del_tool = container_of(tool, struct delegate_tool, tool); + struct aslr_tool *aslr = container_of(del_tool, struct aslr_tool, tool); + struct aslr_evsel_priv *priv = zalloc(sizeof(*priv)); + int err; + + if (!priv) + return -ENOMEM; + + priv->orig_sample_type = evsel->core.attr.sample_type; + priv->orig_sample_regs_user = evsel->core.attr.sample_regs_user; + priv->orig_sample_regs_intr = evsel->core.attr.sample_regs_intr; + priv->orig_sample_size = evsel->sample_size; + + err = hashmap__add(&aslr->evsel_orig_attrs, evsel, priv); + if (err) { + free(priv); + return err; + } + return 0; +} + +void aslr_tool__strip_evlist(const struct perf_tool *tool __maybe_unused, struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + evsel->core.attr.sample_type &= ASLR_SUPPORTED_SAMPLE_TYPE; + evsel->core.attr.sample_regs_user = 0; + evsel->core.attr.sample_regs_intr = 0; + evsel->sample_size = __evsel__sample_size(evsel->core.attr.sample_type); + evsel__calc_id_pos(evsel); + + if (evsel->core.attr.type == PERF_TYPE_BREAKPOINT) { + evsel->core.attr.bp_addr = 0; + } else if (evsel->core.attr.type >= PERF_TYPE_MAX) { + struct perf_pmu *pmu = perf_pmus__find_by_type(evsel->core.attr.type); + + if (pmu && (!strcmp(pmu->name, "kprobe") || + !strcmp(pmu->name, "uprobe"))) { + evsel->core.attr.config1 = 0; + evsel->core.attr.config2 = 0; + } + } + } +} + +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist) +{ + const struct delegate_tool *del_tool = container_of(tool, const struct delegate_tool, tool); + const struct aslr_tool *aslr = container_of(del_tool, const struct aslr_tool, tool); + struct evsel *evsel; + struct aslr_evsel_priv *priv; + + evlist__for_each_entry(evlist, evsel) { + if (hashmap__find(&aslr->evsel_orig_attrs, evsel, &priv)) { + evsel->core.attr.sample_type = priv->orig_sample_type; + evsel->core.attr.sample_regs_user = priv->orig_sample_regs_user; + evsel->core.attr.sample_regs_intr = priv->orig_sample_regs_intr; + evsel->sample_size = priv->orig_sample_size; + evsel__calc_id_pos(evsel); + } + } +} diff --git a/tools/perf/util/aslr.h b/tools/perf/util/aslr.h new file mode 100644 index 000000000..522e31c8e --- /dev/null +++ b/tools/perf/util/aslr.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ASLR_H +#define __PERF_ASLR_H + +#include + +#define ASLR_SUPPORTED_SAMPLE_TYPE ( \ + PERF_SAMPLE_IDENTIFIER | \ + PERF_SAMPLE_IP | \ + PERF_SAMPLE_TID | \ + PERF_SAMPLE_TIME | \ + PERF_SAMPLE_ADDR | \ + PERF_SAMPLE_ID | \ + PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | \ + PERF_SAMPLE_PERIOD | \ + PERF_SAMPLE_READ | \ + PERF_SAMPLE_CALLCHAIN | \ + PERF_SAMPLE_RAW | \ + PERF_SAMPLE_BRANCH_STACK | \ + PERF_SAMPLE_STACK_USER | \ + PERF_SAMPLE_WEIGHT_TYPE | \ + PERF_SAMPLE_DATA_SRC | \ + PERF_SAMPLE_TRANSACTION | \ + PERF_SAMPLE_PHYS_ADDR | \ + PERF_SAMPLE_CGROUP | \ + PERF_SAMPLE_DATA_PAGE_SIZE | \ + PERF_SAMPLE_CODE_PAGE_SIZE | \ + PERF_SAMPLE_AUX) + +struct perf_tool; +struct evsel; +struct evlist; +union perf_event; + +struct perf_tool *aslr_tool__new(struct perf_tool *delegate); +void aslr_tool__delete(struct perf_tool *tool); + +void aslr_tool__strip_attr_event(union perf_event *event, struct evlist *evlist); +int aslr_tool__cache_orig_attrs(struct perf_tool *tool, struct evsel *evsel); +void aslr_tool__strip_evlist(const struct perf_tool *tool, struct evlist *evlist); +void aslr_tool__restore_evlist(const struct perf_tool *tool, struct evlist *evlist); + +#endif /* __PERF_ASLR_H */ diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c new file mode 100644 index 000000000..4cd2caf54 --- /dev/null +++ b/tools/perf/util/auxtrace.c @@ -0,0 +1,2994 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * auxtrace.c: AUX area trace support + * Copyright (c) 2013-2015, Intel Corporation. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "config.h" +#include "evlist.h" +#include "dso.h" +#include "map.h" +#include "pmu.h" +#include "evsel.h" +#include "evsel_config.h" +#include "symbol.h" +#include "util/perf_api_probe.h" +#include "util/synthetic-events.h" +#include "thread_map.h" +#include "asm/bug.h" +#include "auxtrace.h" + +#include + +#include "event.h" +#include "record.h" +#include "session.h" +#include "debug.h" +#include + +#include "cs-etm.h" +#include "intel-pt.h" +#include "intel-bts.h" +#include "arm-spe.h" +#include "hisi-ptt.h" +#include "s390-cpumsf.h" +#include "util/mmap.h" +#include "powerpc-vpadtl.h" + +#include +#include "symbol/kallsyms.h" +#include +#include "util/sample.h" + +#define AUXTRACE_SYNTH_EVENT_ID_OFFSET 1000000000ULL + +/* + * Event IDs are allocated sequentially, so a big offset from any + * existing ID will reach a unused range. + */ +u64 auxtrace_synth_id_range_start(struct evsel *evsel) +{ + u64 id = evsel->core.id[0] + AUXTRACE_SYNTH_EVENT_ID_OFFSET; + + if (!id) + id = 1; + + return id; +} + +/* + * Make a group from 'leader' to 'last', requiring that the events were not + * already grouped to a different leader. + */ +static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct evsel *last) +{ + struct evsel *evsel; + bool grp; + + if (!evsel__is_group_leader(leader)) + return -EINVAL; + + grp = false; + evlist__for_each_entry(evlist, evsel) { + if (grp) { + if (!(evsel__leader(evsel) == leader || + (evsel__leader(evsel) == evsel && + evsel->core.nr_members <= 1))) + return -EINVAL; + } else if (evsel == leader) { + grp = true; + } + if (evsel == last) + break; + } + + grp = false; + evlist__for_each_entry(evlist, evsel) { + if (grp) { + if (!evsel__has_leader(evsel, leader)) { + evsel__set_leader(evsel, leader); + if (leader->core.nr_members < 1) + leader->core.nr_members = 1; + leader->core.nr_members += 1; + } + } else if (evsel == leader) { + grp = true; + } + if (evsel == last) + break; + } + + return 0; +} + +static bool auxtrace__dont_decode(struct perf_session *session) +{ + return !session->itrace_synth_opts || + session->itrace_synth_opts->dont_decode; +} + +int auxtrace_mmap__mmap(struct auxtrace_mmap *mm, + struct auxtrace_mmap_params *mp, + void *userpg, int fd) +{ + struct perf_event_mmap_page *pc = userpg; + + WARN_ONCE(mm->base, "Uninitialized auxtrace_mmap\n"); + + mm->userpg = userpg; + mm->mask = mp->mask; + mm->len = mp->len; + mm->prev = 0; + mm->idx = mp->idx; + mm->tid = mp->tid; + mm->cpu = mp->cpu.cpu; + + if (!mp->len || !mp->mmap_needed) { + mm->base = NULL; + return 0; + } + + pc->aux_offset = mp->offset; + pc->aux_size = mp->len; + + mm->base = mmap(NULL, mp->len, mp->prot, MAP_SHARED, fd, mp->offset); + if (mm->base == MAP_FAILED) { + pr_debug2("failed to mmap AUX area\n"); + mm->base = NULL; + return -1; + } + + return 0; +} + +void auxtrace_mmap__munmap(struct auxtrace_mmap *mm) +{ + if (mm->base) { + munmap(mm->base, mm->len); + mm->base = NULL; + } +} + +void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp, + off_t auxtrace_offset, + unsigned int auxtrace_pages, + bool auxtrace_overwrite) +{ + if (auxtrace_pages) { + mp->offset = auxtrace_offset; + mp->len = auxtrace_pages * (size_t)page_size; + mp->mask = is_power_of_2(mp->len) ? mp->len - 1 : 0; + mp->prot = PROT_READ | (auxtrace_overwrite ? 0 : PROT_WRITE); + pr_debug2("AUX area mmap length %zu\n", mp->len); + } else { + mp->len = 0; + } +} + +void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, + struct evlist *evlist, + struct evsel *evsel, int idx) +{ + bool per_cpu = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + + mp->mmap_needed = evsel->needs_auxtrace_mmap; + + if (!mp->mmap_needed) + return; + + mp->idx = idx; + + if (per_cpu) { + mp->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); + mp->tid = perf_thread_map__pid(evlist->core.threads, 0); + } else { + mp->cpu.cpu = -1; + mp->tid = perf_thread_map__pid(evlist->core.threads, idx); + } +} + +#define AUXTRACE_INIT_NR_QUEUES 32 + +static struct auxtrace_queue *auxtrace_alloc_queue_array(unsigned int nr_queues) +{ + struct auxtrace_queue *queue_array; + unsigned int max_nr_queues, i; + + max_nr_queues = UINT_MAX / sizeof(struct auxtrace_queue); + if (nr_queues > max_nr_queues) + return NULL; + + queue_array = calloc(nr_queues, sizeof(struct auxtrace_queue)); + if (!queue_array) + return NULL; + + for (i = 0; i < nr_queues; i++) { + INIT_LIST_HEAD(&queue_array[i].head); + queue_array[i].priv = NULL; + } + + return queue_array; +} + +int auxtrace_queues__init_nr(struct auxtrace_queues *queues, int nr_queues) +{ + queues->nr_queues = nr_queues; + queues->queue_array = auxtrace_alloc_queue_array(queues->nr_queues); + if (!queues->queue_array) + return -ENOMEM; + return 0; +} + +int auxtrace_queues__init(struct auxtrace_queues *queues) +{ + return auxtrace_queues__init_nr(queues, AUXTRACE_INIT_NR_QUEUES); +} + +static int auxtrace_queues__grow(struct auxtrace_queues *queues, + unsigned int new_nr_queues) +{ + unsigned int nr_queues = queues->nr_queues; + struct auxtrace_queue *queue_array; + unsigned int i; + + if (!nr_queues) + nr_queues = AUXTRACE_INIT_NR_QUEUES; + + while (nr_queues && nr_queues < new_nr_queues) + nr_queues <<= 1; + + if (nr_queues < queues->nr_queues || nr_queues < new_nr_queues) + return -EINVAL; + + queue_array = auxtrace_alloc_queue_array(nr_queues); + if (!queue_array) + return -ENOMEM; + + for (i = 0; i < queues->nr_queues; i++) { + list_splice_tail(&queues->queue_array[i].head, + &queue_array[i].head); + queue_array[i].tid = queues->queue_array[i].tid; + queue_array[i].cpu = queues->queue_array[i].cpu; + queue_array[i].set = queues->queue_array[i].set; + queue_array[i].priv = queues->queue_array[i].priv; + } + + queues->nr_queues = nr_queues; + queues->queue_array = queue_array; + + return 0; +} + +static void *auxtrace_copy_data(u64 size, struct perf_session *session) +{ + int fd = perf_data__fd(session->data); + void *p; + ssize_t ret; + + if (size > SSIZE_MAX) + return NULL; + + p = malloc(size); + if (!p) + return NULL; + + ret = readn(fd, p, size); + if (ret != (ssize_t)size) { + free(p); + return NULL; + } + + return p; +} + +static int auxtrace_queues__queue_buffer(struct auxtrace_queues *queues, + unsigned int idx, + struct auxtrace_buffer *buffer) +{ + struct auxtrace_queue *queue; + int err; + + if (idx >= queues->nr_queues) { + err = auxtrace_queues__grow(queues, idx + 1); + if (err) + return err; + } + + queue = &queues->queue_array[idx]; + + if (!queue->set) { + queue->set = true; + queue->tid = buffer->tid; + queue->cpu = buffer->cpu.cpu; + } + + buffer->buffer_nr = queues->next_buffer_nr++; + + list_add_tail(&buffer->list, &queue->head); + + queues->new_data = true; + queues->populated = true; + + return 0; +} + +/* Limit buffers to 32MiB on 32-bit */ +#define BUFFER_LIMIT_FOR_32_BIT (32 * 1024 * 1024) + +static int auxtrace_queues__split_buffer(struct auxtrace_queues *queues, + unsigned int idx, + struct auxtrace_buffer *buffer) +{ + u64 sz = buffer->size; + bool consecutive = false; + struct auxtrace_buffer *b; + int err; + + while (sz > BUFFER_LIMIT_FOR_32_BIT) { + b = memdup(buffer, sizeof(struct auxtrace_buffer)); + if (!b) + return -ENOMEM; + b->size = BUFFER_LIMIT_FOR_32_BIT; + b->consecutive = consecutive; + err = auxtrace_queues__queue_buffer(queues, idx, b); + if (err) { + auxtrace_buffer__free(b); + return err; + } + buffer->data_offset += BUFFER_LIMIT_FOR_32_BIT; + sz -= BUFFER_LIMIT_FOR_32_BIT; + consecutive = true; + } + + buffer->size = sz; + buffer->consecutive = consecutive; + + return 0; +} + +static bool filter_cpu(struct perf_session *session, struct perf_cpu cpu) +{ + unsigned long *cpu_bitmap = session->itrace_synth_opts->cpu_bitmap; + + return cpu_bitmap && cpu.cpu >= 0 && cpu.cpu < MAX_NR_CPUS && + !test_bit(cpu.cpu, cpu_bitmap); +} + +static int auxtrace_queues__add_buffer(struct auxtrace_queues *queues, + struct perf_session *session, + unsigned int idx, + struct auxtrace_buffer *buffer, + struct auxtrace_buffer **buffer_ptr) +{ + int err = -ENOMEM; + + if (filter_cpu(session, buffer->cpu)) + return 0; + + buffer = memdup(buffer, sizeof(*buffer)); + if (!buffer) + return -ENOMEM; + + if (session->one_mmap) { + buffer->data = buffer->data_offset - session->one_mmap_offset + + session->one_mmap_addr; + } else if (perf_data__is_pipe(session->data)) { + buffer->data = auxtrace_copy_data(buffer->size, session); + if (!buffer->data) + goto out_free; + buffer->data_needs_freeing = true; + } else if (BITS_PER_LONG == 32 && + buffer->size > BUFFER_LIMIT_FOR_32_BIT) { + err = auxtrace_queues__split_buffer(queues, idx, buffer); + if (err) + goto out_free; + } + + err = auxtrace_queues__queue_buffer(queues, idx, buffer); + if (err) + goto out_free; + + /* FIXME: Doesn't work for split buffer */ + if (buffer_ptr) + *buffer_ptr = buffer; + + return 0; + +out_free: + auxtrace_buffer__free(buffer); + return err; +} + +int auxtrace_queues__add_event(struct auxtrace_queues *queues, + struct perf_session *session, + union perf_event *event, off_t data_offset, + struct auxtrace_buffer **buffer_ptr) +{ + struct auxtrace_buffer buffer = { + .pid = -1, + .tid = event->auxtrace.tid, + .cpu = { event->auxtrace.cpu }, + .data_offset = data_offset, + .offset = event->auxtrace.offset, + .reference = event->auxtrace.reference, + .size = event->auxtrace.size, + }; + unsigned int idx = event->auxtrace.idx; + + return auxtrace_queues__add_buffer(queues, session, idx, &buffer, + buffer_ptr); +} + +static int auxtrace_queues__add_indexed_event(struct auxtrace_queues *queues, + struct perf_session *session, + off_t file_offset, size_t sz) +{ + union perf_event *event; + int err; + char buf[PERF_SAMPLE_MAX_SIZE]; + + err = perf_session__peek_event(session, file_offset, buf, + PERF_SAMPLE_MAX_SIZE, &event, NULL); + if (err) + return err; + + if (event->header.type == PERF_RECORD_AUXTRACE) { + if (event->header.size < sizeof(struct perf_record_auxtrace) || + event->header.size != sz) { + err = -EINVAL; + goto out; + } + file_offset += event->header.size; + err = auxtrace_queues__add_event(queues, session, event, + file_offset, NULL); + } +out: + return err; +} + +void auxtrace_queues__free(struct auxtrace_queues *queues) +{ + unsigned int i; + + for (i = 0; i < queues->nr_queues; i++) { + while (!list_empty(&queues->queue_array[i].head)) { + struct auxtrace_buffer *buffer; + + buffer = list_entry(queues->queue_array[i].head.next, + struct auxtrace_buffer, list); + list_del_init(&buffer->list); + auxtrace_buffer__free(buffer); + } + } + + zfree(&queues->queue_array); + queues->nr_queues = 0; +} + +static void auxtrace_heapify(struct auxtrace_heap_item *heap_array, + unsigned int pos, unsigned int queue_nr, + u64 ordinal) +{ + unsigned int parent; + + while (pos) { + parent = (pos - 1) >> 1; + if (heap_array[parent].ordinal <= ordinal) + break; + heap_array[pos] = heap_array[parent]; + pos = parent; + } + heap_array[pos].queue_nr = queue_nr; + heap_array[pos].ordinal = ordinal; +} + +int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr, + u64 ordinal) +{ + struct auxtrace_heap_item *heap_array; + + if (queue_nr >= heap->heap_sz) { + unsigned int heap_sz = AUXTRACE_INIT_NR_QUEUES; + + while (heap_sz <= queue_nr) + heap_sz <<= 1; + heap_array = realloc(heap->heap_array, + heap_sz * sizeof(struct auxtrace_heap_item)); + if (!heap_array) + return -ENOMEM; + heap->heap_array = heap_array; + heap->heap_sz = heap_sz; + } + + auxtrace_heapify(heap->heap_array, heap->heap_cnt++, queue_nr, ordinal); + + return 0; +} + +void auxtrace_heap__free(struct auxtrace_heap *heap) +{ + zfree(&heap->heap_array); + heap->heap_cnt = 0; + heap->heap_sz = 0; +} + +void auxtrace_heap__pop(struct auxtrace_heap *heap) +{ + unsigned int pos, last, heap_cnt = heap->heap_cnt; + struct auxtrace_heap_item *heap_array; + + if (!heap_cnt) + return; + + heap->heap_cnt -= 1; + + heap_array = heap->heap_array; + + pos = 0; + while (1) { + unsigned int left, right; + + left = (pos << 1) + 1; + if (left >= heap_cnt) + break; + right = left + 1; + if (right >= heap_cnt) { + heap_array[pos] = heap_array[left]; + return; + } + if (heap_array[left].ordinal < heap_array[right].ordinal) { + heap_array[pos] = heap_array[left]; + pos = left; + } else { + heap_array[pos] = heap_array[right]; + pos = right; + } + } + + last = heap_cnt - 1; + auxtrace_heapify(heap_array, pos, heap_array[last].queue_nr, + heap_array[last].ordinal); +} + +size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr, + struct evlist *evlist) +{ + if (itr) + return itr->info_priv_size(itr, evlist); + return 0; +} + +static int auxtrace_not_supported(void) +{ + pr_err("AUX area tracing is not supported on this architecture\n"); + return -EINVAL; +} + +int auxtrace_record__info_fill(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size) +{ + if (itr) + return itr->info_fill(itr, session, auxtrace_info, priv_size); + return auxtrace_not_supported(); +} + +void auxtrace_record__free(struct auxtrace_record *itr) +{ + if (itr) + itr->free(itr); +} + +int auxtrace_record__snapshot_start(struct auxtrace_record *itr) +{ + if (itr && itr->snapshot_start) + return itr->snapshot_start(itr); + return 0; +} + +int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit) +{ + if (!on_exit && itr && itr->snapshot_finish) + return itr->snapshot_finish(itr); + return 0; +} + +int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx, + struct auxtrace_mmap *mm, + unsigned char *data, u64 *head, u64 *old) +{ + if (itr && itr->find_snapshot) + return itr->find_snapshot(itr, idx, mm, data, head, old); + return 0; +} + +int auxtrace_record__options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts) +{ + if (itr) { + itr->evlist = evlist; + return itr->recording_options(itr, evlist, opts); + } + return 0; +} + +u64 auxtrace_record__reference(struct auxtrace_record *itr) +{ + if (itr) + return itr->reference(itr); + return 0; +} + +int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, + struct record_opts *opts, const char *str) +{ + if (!str) + return 0; + + /* PMU-agnostic options */ + switch (*str) { + case 'e': + opts->auxtrace_snapshot_on_exit = true; + str++; + break; + default: + break; + } + + if (itr && itr->parse_snapshot_options) + return itr->parse_snapshot_options(itr, opts, str); + + pr_err("No AUX area tracing to snapshot\n"); + return -EINVAL; +} + +static int evlist__enable_event_idx(struct evlist *evlist, struct evsel *evsel, int idx) +{ + bool per_cpu_mmaps = !perf_cpu_map__has_any_cpu(evlist->core.user_requested_cpus); + + if (per_cpu_mmaps) { + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->core.all_cpus, idx); + int cpu_map_idx = perf_cpu_map__idx(evsel->core.cpus, evlist_cpu); + + if (cpu_map_idx == -1) + return -EINVAL; + return perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + } + + return perf_evsel__enable_thread(&evsel->core, idx); +} + +int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx) +{ + struct evsel *evsel; + + if (!itr->evlist) + return -EINVAL; + + evlist__for_each_entry(itr->evlist, evsel) { + if (evsel__is_aux_event(evsel)) { + if (evsel->disabled) + return 0; + return evlist__enable_event_idx(itr->evlist, evsel, idx); + } + } + return -EINVAL; +} + +/* + * Event record size is 16-bit which results in a maximum size of about 64KiB. + * Allow about 4KiB for the rest of the sample record, to give a maximum + * AUX area sample size of 60KiB. + */ +#define MAX_AUX_SAMPLE_SIZE (60 * 1024) + +/* Arbitrary default size if no other default provided */ +#define DEFAULT_AUX_SAMPLE_SIZE (4 * 1024) + +static int auxtrace_validate_aux_sample_size(struct evlist *evlist, + struct record_opts *opts) +{ + struct evsel *evsel; + bool has_aux_leader = false; + u32 sz; + + evlist__for_each_entry(evlist, evsel) { + sz = evsel->core.attr.aux_sample_size; + if (evsel__is_group_leader(evsel)) { + has_aux_leader = evsel__is_aux_event(evsel); + if (sz) { + if (has_aux_leader) + pr_err("Cannot add AUX area sampling to an AUX area event\n"); + else + pr_err("Cannot add AUX area sampling to a group leader\n"); + return -EINVAL; + } + } + if (sz > MAX_AUX_SAMPLE_SIZE) { + pr_err("AUX area sample size %u too big, max. %d\n", + sz, MAX_AUX_SAMPLE_SIZE); + return -EINVAL; + } + if (sz) { + if (!has_aux_leader) { + pr_err("Cannot add AUX area sampling because group leader is not an AUX area event\n"); + return -EINVAL; + } + evsel__set_sample_bit(evsel, AUX); + opts->auxtrace_sample_mode = true; + } else { + evsel__reset_sample_bit(evsel, AUX); + } + } + + if (!opts->auxtrace_sample_mode) { + pr_err("AUX area sampling requires an AUX area event group leader plus other events to which to add samples\n"); + return -EINVAL; + } + + if (!perf_can_aux_sample()) { + pr_err("AUX area sampling is not supported by kernel\n"); + return -EINVAL; + } + + return 0; +} + +int auxtrace_parse_sample_options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts, const char *str) +{ + struct evsel_config_term *term; + struct evsel *aux_evsel; + bool has_aux_sample_size = false; + bool has_aux_leader = false; + struct evsel *evsel; + char *endptr; + unsigned long sz; + + if (!str) + goto no_opt; + + if (!itr) { + pr_err("No AUX area event to sample\n"); + return -EINVAL; + } + + sz = strtoul(str, &endptr, 0); + if (*endptr || sz > UINT_MAX) { + pr_err("Bad AUX area sampling option: '%s'\n", str); + return -EINVAL; + } + + if (!sz) + sz = itr->default_aux_sample_size; + + if (!sz) + sz = DEFAULT_AUX_SAMPLE_SIZE; + + /* Set aux_sample_size based on --aux-sample option */ + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_group_leader(evsel)) { + has_aux_leader = evsel__is_aux_event(evsel); + } else if (has_aux_leader) { + evsel->core.attr.aux_sample_size = sz; + } + } +no_opt: + aux_evsel = NULL; + /* Override with aux_sample_size from config term */ + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_aux_event(evsel)) + aux_evsel = evsel; + term = evsel__get_config_term(evsel, AUX_SAMPLE_SIZE); + if (term) { + has_aux_sample_size = true; + evsel->core.attr.aux_sample_size = term->val.aux_sample_size; + /* If possible, group with the AUX event */ + if (aux_evsel && evsel->core.attr.aux_sample_size) + evlist__regroup(evlist, aux_evsel, evsel); + } + } + + if (!str && !has_aux_sample_size) + return 0; + + if (!itr) { + pr_err("No AUX area event to sample\n"); + return -EINVAL; + } + + return auxtrace_validate_aux_sample_size(evlist, opts); +} + +static struct aux_action_opt { + const char *str; + u32 aux_action; + bool aux_event_opt; +} aux_action_opts[] = { + {"start-paused", BIT(0), true}, + {"pause", BIT(1), false}, + {"resume", BIT(2), false}, + {.str = NULL}, +}; + +static const struct aux_action_opt *auxtrace_parse_aux_action_str(const char *str) +{ + const struct aux_action_opt *opt; + + if (!str) + return NULL; + + for (opt = aux_action_opts; opt->str; opt++) + if (!strcmp(str, opt->str)) + return opt; + + return NULL; +} + +int auxtrace_parse_aux_action(struct evlist *evlist) +{ + struct evsel_config_term *term; + struct evsel *aux_evsel = NULL; + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + bool is_aux_event = evsel__is_aux_event(evsel); + const struct aux_action_opt *opt; + + if (is_aux_event) + aux_evsel = evsel; + term = evsel__get_config_term(evsel, AUX_ACTION); + if (!term) { + if (evsel__get_config_term(evsel, AUX_OUTPUT)) + goto regroup; + continue; + } + opt = auxtrace_parse_aux_action_str(term->val.str); + if (!opt) { + pr_err("Bad aux-action '%s'\n", term->val.str); + return -EINVAL; + } + if (opt->aux_event_opt && !is_aux_event) { + pr_err("aux-action '%s' can only be used with AUX area event\n", + term->val.str); + return -EINVAL; + } + if (!opt->aux_event_opt && is_aux_event) { + pr_err("aux-action '%s' cannot be used for AUX area event itself\n", + term->val.str); + return -EINVAL; + } + evsel->core.attr.aux_action = opt->aux_action; +regroup: + /* If possible, group with the AUX event */ + if (aux_evsel) + evlist__regroup(evlist, aux_evsel, evsel); + if (!evsel__is_aux_event(evsel__leader(evsel))) { + pr_err("Events with aux-action must have AUX area event group leader\n"); + return -EINVAL; + } + } + + return 0; +} + +/** + * auxtrace_record__init - Initialize an AUX area tracing record. + * @evlist: The list of events to check for AUX area tracing event. + * @err: Pointer to an integer to store return code. + * + * This function looks through the @evlist to determine which AUX area + * tracing hardware is being used and initializes the auxtrace_record + * structure. + * + * Return: + * a) A pointer to the struct auxtrace_record with @err = 0 on success. + * b) NULL with @err = 0 if no AUX area tracing event is found/supported + * (not considered an error). + * c) NULL with non-zero @err on actual auxtrace_record__init failure. + */ +struct auxtrace_record *__weak +auxtrace_record__init(struct evlist *evlist __maybe_unused, int *err) +{ + *err = 0; + return NULL; +} + +static int auxtrace_index__alloc(struct list_head *head) +{ + struct auxtrace_index *auxtrace_index; + + auxtrace_index = malloc(sizeof(struct auxtrace_index)); + if (!auxtrace_index) + return -ENOMEM; + + auxtrace_index->nr = 0; + INIT_LIST_HEAD(&auxtrace_index->list); + + list_add_tail(&auxtrace_index->list, head); + + return 0; +} + +void auxtrace_index__free(struct list_head *head) +{ + struct auxtrace_index *auxtrace_index, *n; + + list_for_each_entry_safe(auxtrace_index, n, head, list) { + list_del_init(&auxtrace_index->list); + free(auxtrace_index); + } +} + +static struct auxtrace_index *auxtrace_index__last(struct list_head *head) +{ + struct auxtrace_index *auxtrace_index; + int err; + + if (list_empty(head)) { + err = auxtrace_index__alloc(head); + if (err) + return NULL; + } + + auxtrace_index = list_entry(head->prev, struct auxtrace_index, list); + + if (auxtrace_index->nr >= PERF_AUXTRACE_INDEX_ENTRY_COUNT) { + err = auxtrace_index__alloc(head); + if (err) + return NULL; + auxtrace_index = list_entry(head->prev, struct auxtrace_index, + list); + } + + return auxtrace_index; +} + +int auxtrace_index__auxtrace_event(struct list_head *head, + union perf_event *event, off_t file_offset) +{ + struct auxtrace_index *auxtrace_index; + size_t nr; + + auxtrace_index = auxtrace_index__last(head); + if (!auxtrace_index) + return -ENOMEM; + + nr = auxtrace_index->nr; + auxtrace_index->entries[nr].file_offset = file_offset; + auxtrace_index->entries[nr].sz = event->header.size; + auxtrace_index->nr += 1; + + return 0; +} + +static int auxtrace_index__do_write(int fd, + struct auxtrace_index *auxtrace_index) +{ + struct auxtrace_index_entry ent; + size_t i; + + for (i = 0; i < auxtrace_index->nr; i++) { + ent.file_offset = auxtrace_index->entries[i].file_offset; + ent.sz = auxtrace_index->entries[i].sz; + if (writen(fd, &ent, sizeof(ent)) != sizeof(ent)) + return -errno; + } + return 0; +} + +int auxtrace_index__write(int fd, struct list_head *head) +{ + struct auxtrace_index *auxtrace_index; + u64 total = 0; + int err; + + list_for_each_entry(auxtrace_index, head, list) + total += auxtrace_index->nr; + + if (writen(fd, &total, sizeof(total)) != sizeof(total)) + return -errno; + + list_for_each_entry(auxtrace_index, head, list) { + err = auxtrace_index__do_write(fd, auxtrace_index); + if (err) + return err; + } + + return 0; +} + +static int auxtrace_index__process_entry(int fd, struct list_head *head, + bool needs_swap) +{ + struct auxtrace_index *auxtrace_index; + struct auxtrace_index_entry ent; + size_t nr; + + if (readn(fd, &ent, sizeof(ent)) != sizeof(ent)) + return -1; + + auxtrace_index = auxtrace_index__last(head); + if (!auxtrace_index) + return -1; + + nr = auxtrace_index->nr; + if (needs_swap) { + auxtrace_index->entries[nr].file_offset = + bswap_64(ent.file_offset); + auxtrace_index->entries[nr].sz = bswap_64(ent.sz); + } else { + auxtrace_index->entries[nr].file_offset = ent.file_offset; + auxtrace_index->entries[nr].sz = ent.sz; + } + + auxtrace_index->nr = nr + 1; + + return 0; +} + +int auxtrace_index__process(int fd, u64 size, struct perf_session *session, + bool needs_swap) +{ + struct list_head *head = &session->auxtrace_index; + u64 nr; + + if (readn(fd, &nr, sizeof(u64)) != sizeof(u64)) + return -1; + + if (needs_swap) + nr = bswap_64(nr); + + if (sizeof(u64) + nr * sizeof(struct auxtrace_index_entry) > size) + return -1; + + while (nr--) { + int err; + + err = auxtrace_index__process_entry(fd, head, needs_swap); + if (err) + return -1; + } + + return 0; +} + +static int auxtrace_queues__process_index_entry(struct auxtrace_queues *queues, + struct perf_session *session, + struct auxtrace_index_entry *ent) +{ + return auxtrace_queues__add_indexed_event(queues, session, + ent->file_offset, ent->sz); +} + +int auxtrace_queues__process_index(struct auxtrace_queues *queues, + struct perf_session *session) +{ + struct auxtrace_index *auxtrace_index; + struct auxtrace_index_entry *ent; + size_t i; + int err; + + if (auxtrace__dont_decode(session)) + return 0; + + list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { + for (i = 0; i < auxtrace_index->nr; i++) { + ent = &auxtrace_index->entries[i]; + err = auxtrace_queues__process_index_entry(queues, + session, + ent); + if (err) + return err; + } + } + return 0; +} + +struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue, + struct auxtrace_buffer *buffer) +{ + if (buffer) { + if (list_is_last(&buffer->list, &queue->head)) + return NULL; + return list_entry(buffer->list.next, struct auxtrace_buffer, + list); + } else { + if (list_empty(&queue->head)) + return NULL; + return list_entry(queue->head.next, struct auxtrace_buffer, + list); + } +} + +struct auxtrace_queue *auxtrace_queues__sample_queue(struct auxtrace_queues *queues, + struct perf_sample *sample, + struct perf_session *session) +{ + struct perf_sample_id *sid; + unsigned int idx; + u64 id; + + id = sample->id; + if (!id) + return NULL; + + sid = evlist__id2sid(session->evlist, id); + if (!sid) + return NULL; + + idx = sid->idx; + + if (idx >= queues->nr_queues) + return NULL; + + return &queues->queue_array[idx]; +} + +int auxtrace_queues__add_sample(struct auxtrace_queues *queues, + struct perf_session *session, + struct perf_sample *sample, u64 data_offset, + u64 reference) +{ + struct auxtrace_buffer buffer = { + .pid = -1, + .data_offset = data_offset, + .reference = reference, + .size = sample->aux_sample.size, + }; + struct perf_sample_id *sid; + u64 id = sample->id; + unsigned int idx; + + if (!id) + return -EINVAL; + + sid = evlist__id2sid(session->evlist, id); + if (!sid) + return -ENOENT; + + idx = sid->idx; + buffer.tid = sid->tid; + buffer.cpu = sid->cpu; + + return auxtrace_queues__add_buffer(queues, session, idx, &buffer, NULL); +} + +struct queue_data { + bool samples; + bool events; +}; + +static int auxtrace_queue_data_cb(struct perf_session *session, + union perf_event *event, u64 offset, + void *data) +{ + struct queue_data *qd = data; + struct perf_sample sample; + int err; + + if (qd->events && event->header.type == PERF_RECORD_AUXTRACE) { + if (event->header.size < sizeof(struct perf_record_auxtrace)) + return -EINVAL; + offset += event->header.size; + return session->auxtrace->queue_data(session, NULL, event, + offset); + } + + if (!qd->samples || event->header.type != PERF_RECORD_SAMPLE) + return 0; + + perf_sample__init(&sample, /*all=*/false); + err = evlist__parse_sample(session->evlist, event, &sample); + if (err) + goto out; + + if (sample.aux_sample.size) { + offset += sample.aux_sample.data - (void *)event; + + err = session->auxtrace->queue_data(session, &sample, NULL, offset); + } +out: + perf_sample__exit(&sample); + return err; +} + +int auxtrace_queue_data(struct perf_session *session, bool samples, bool events) +{ + struct queue_data qd = { + .samples = samples, + .events = events, + }; + + if (auxtrace__dont_decode(session)) + return 0; + + if (perf_data__is_pipe(session->data)) + return 0; + + if (!session->auxtrace || !session->auxtrace->queue_data) + return -EINVAL; + + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + auxtrace_queue_data_cb, &qd); +} + +void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw) +{ + int prot = rw ? PROT_READ | PROT_WRITE : PROT_READ; + size_t adj = buffer->data_offset & (page_size - 1); + size_t size = buffer->size + adj; + off_t file_offset = buffer->data_offset - adj; + void *addr; + + if (buffer->data) + return buffer->data; + + addr = mmap(NULL, size, prot, MAP_SHARED, fd, file_offset); + if (addr == MAP_FAILED) + return NULL; + + buffer->mmap_addr = addr; + buffer->mmap_size = size; + + buffer->data = addr + adj; + + return buffer->data; +} + +void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer) +{ + if (!buffer->data || !buffer->mmap_addr) + return; + munmap(buffer->mmap_addr, buffer->mmap_size); + buffer->mmap_addr = NULL; + buffer->mmap_size = 0; + buffer->data = NULL; + buffer->use_data = NULL; +} + +void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer) +{ + auxtrace_buffer__put_data(buffer); + if (buffer->data_needs_freeing) { + buffer->data_needs_freeing = false; + zfree(&buffer->data); + buffer->use_data = NULL; + buffer->size = 0; + } +} + +void auxtrace_buffer__free(struct auxtrace_buffer *buffer) +{ + auxtrace_buffer__drop_data(buffer); + free(buffer); +} + +void auxtrace_synth_guest_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp, + pid_t machine_pid, int vcpu) +{ + size_t size; + + memset(auxtrace_error, 0, sizeof(struct perf_record_auxtrace_error)); + + auxtrace_error->header.type = PERF_RECORD_AUXTRACE_ERROR; + auxtrace_error->type = type; + auxtrace_error->code = code; + auxtrace_error->cpu = cpu; + auxtrace_error->pid = pid; + auxtrace_error->tid = tid; + auxtrace_error->fmt = 1; + auxtrace_error->ip = ip; + auxtrace_error->time = timestamp; + strlcpy(auxtrace_error->msg, msg, MAX_AUXTRACE_ERROR_MSG); + if (machine_pid) { + auxtrace_error->fmt = 2; + auxtrace_error->machine_pid = machine_pid; + auxtrace_error->vcpu = vcpu; + size = sizeof(*auxtrace_error); + } else { + size = (void *)auxtrace_error->msg - (void *)auxtrace_error + + strlen(auxtrace_error->msg) + 1; + } + auxtrace_error->header.size = PERF_ALIGN(size, sizeof(u64)); +} + +void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp) +{ + auxtrace_synth_guest_error(auxtrace_error, type, code, cpu, pid, tid, + ip, msg, timestamp, 0, -1); +} + +int perf_event__synthesize_auxtrace_info(struct auxtrace_record *itr, + const struct perf_tool *tool, + struct perf_session *session, + perf_event__handler_t process) +{ + union perf_event *ev; + size_t priv_size; + int err; + + pr_debug2("Synthesizing auxtrace information\n"); + priv_size = auxtrace_record__info_priv_size(itr, session->evlist); + ev = zalloc(sizeof(struct perf_record_auxtrace_info) + priv_size); + if (!ev) + return -ENOMEM; + + ev->auxtrace_info.header.type = PERF_RECORD_AUXTRACE_INFO; + ev->auxtrace_info.header.size = sizeof(struct perf_record_auxtrace_info) + + priv_size; + err = auxtrace_record__info_fill(itr, session, &ev->auxtrace_info, + priv_size); + if (err) + goto out_free; + + err = process(tool, ev, NULL, NULL); +out_free: + free(ev); + return err; +} + +static void unleader_evsel(struct evlist *evlist, struct evsel *leader) +{ + struct evsel *new_leader = NULL; + struct evsel *evsel; + + /* Find new leader for the group */ + evlist__for_each_entry(evlist, evsel) { + if (!evsel__has_leader(evsel, leader) || evsel == leader) + continue; + if (!new_leader) + new_leader = evsel; + evsel__set_leader(evsel, new_leader); + } + + /* Update group information */ + if (new_leader) { + zfree(&new_leader->group_name); + new_leader->group_name = leader->group_name; + leader->group_name = NULL; + + new_leader->core.nr_members = leader->core.nr_members - 1; + leader->core.nr_members = 1; + } +} + +static void unleader_auxtrace(struct perf_session *session) +{ + struct evsel *evsel; + + evlist__for_each_entry(session->evlist, evsel) { + if (auxtrace__evsel_is_auxtrace(session, evsel) && + evsel__is_group_leader(evsel)) { + unleader_evsel(session->evlist, evsel); + } + } +} + +int perf_event__process_auxtrace_info(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + enum auxtrace_type type = event->auxtrace_info.type; + int err; + + if (dump_trace) + fprintf(stdout, " type: %u\n", type); + + switch (type) { + case PERF_AUXTRACE_INTEL_PT: + err = intel_pt_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_INTEL_BTS: + err = intel_bts_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_ARM_SPE: + err = arm_spe_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_CS_ETM: + err = cs_etm__process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_S390_CPUMSF: + err = s390_cpumsf_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_HISI_PTT: + err = hisi_ptt_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_VPA_DTL: + err = powerpc_vpadtl_process_auxtrace_info(event, session); + break; + case PERF_AUXTRACE_UNKNOWN: + default: + return -EINVAL; + } + + if (err) + return err; + + unleader_auxtrace(session); + + return 0; +} + +s64 perf_event__process_auxtrace(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + s64 err; + + if (dump_trace) + fprintf(stdout, " size: %#"PRI_lx64" offset: %#"PRI_lx64" ref: %#"PRI_lx64" idx: %u tid: %d cpu: %d\n", + event->auxtrace.size, event->auxtrace.offset, + event->auxtrace.reference, event->auxtrace.idx, + event->auxtrace.tid, event->auxtrace.cpu); + + if (auxtrace__dont_decode(session)) + return event->auxtrace.size; + + if (!session->auxtrace || event->header.type != PERF_RECORD_AUXTRACE) + return -EINVAL; + + err = session->auxtrace->process_auxtrace_event(session, event, session->tool); + if (err < 0) + return err; + + return event->auxtrace.size; +} + +#define PERF_ITRACE_DEFAULT_PERIOD_TYPE PERF_ITRACE_PERIOD_NANOSECS +#define PERF_ITRACE_DEFAULT_PERIOD 100000 +#define PERF_ITRACE_DEFAULT_CALLCHAIN_SZ 16 +#define PERF_ITRACE_MAX_CALLCHAIN_SZ 1024 +#define PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ 64 +#define PERF_ITRACE_MAX_LAST_BRANCH_SZ 1024 + +void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, + bool no_sample) +{ + synth_opts->branches = true; + synth_opts->transactions = true; + synth_opts->ptwrites = true; + synth_opts->pwr_events = true; + synth_opts->other_events = true; + synth_opts->intr_events = true; + synth_opts->errors = true; + synth_opts->flc = true; + synth_opts->llc = true; + synth_opts->tlb = true; + synth_opts->mem = true; + synth_opts->remote_access = true; + + if (no_sample) { + synth_opts->period_type = PERF_ITRACE_PERIOD_INSTRUCTIONS; + synth_opts->period = 1; + synth_opts->calls = true; + } else { + synth_opts->instructions = true; + synth_opts->cycles = true; + synth_opts->period_type = PERF_ITRACE_DEFAULT_PERIOD_TYPE; + synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD; + } + synth_opts->callchain_sz = PERF_ITRACE_DEFAULT_CALLCHAIN_SZ; + synth_opts->last_branch_sz = PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ; + synth_opts->initial_skip = 0; +} + +static int get_flag(const char **ptr, unsigned int *flags) +{ + while (1) { + char c = **ptr; + + if (c >= 'a' && c <= 'z') { + *flags |= 1 << (c - 'a'); + ++*ptr; + return 0; + } else if (c == ' ') { + ++*ptr; + continue; + } else { + return -1; + } + } +} + +static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *minus_flags) +{ + while (1) { + switch (**ptr) { + case '+': + ++*ptr; + if (get_flag(ptr, plus_flags)) + return -1; + break; + case '-': + ++*ptr; + if (get_flag(ptr, minus_flags)) + return -1; + break; + case ' ': + ++*ptr; + break; + default: + return 0; + } + } +} + +#define ITRACE_DFLT_LOG_ON_ERROR_SZ 16384 + +static unsigned int itrace_log_on_error_size(void) +{ + unsigned int sz = 0; + + perf_config_scan("itrace.debug-log-buffer-size", "%u", &sz); + return sz ?: ITRACE_DFLT_LOG_ON_ERROR_SZ; +} + +/* + * Please check tools/perf/Documentation/perf-script.txt for information + * about the options parsed here, which is introduced after this cset, + * when support in 'perf script' for these options is introduced. + */ +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, + const char *str, int unset) +{ + const char *p; + char *endptr; + bool period_type_set = false; + bool period_set = false; + bool iy = false; + + synth_opts->set = true; + + if (unset) { + synth_opts->dont_decode = true; + return 0; + } + + if (!str) { + itrace_synth_opts__set_default(synth_opts, + synth_opts->default_no_sample); + return 0; + } + + for (p = str; *p;) { + switch (*p++) { + case 'i': + case 'y': + iy = true; + if (p[-1] == 'y') + synth_opts->cycles = true; + else + synth_opts->instructions = true; + while (*p == ' ' || *p == ',') + p += 1; + if (isdigit(*p)) { + synth_opts->period = strtoull(p, &endptr, 10); + period_set = true; + p = endptr; + while (*p == ' ' || *p == ',') + p += 1; + switch (*p++) { + case 'i': + synth_opts->period_type = + PERF_ITRACE_PERIOD_INSTRUCTIONS; + period_type_set = true; + break; + case 't': + synth_opts->period_type = + PERF_ITRACE_PERIOD_TICKS; + period_type_set = true; + break; + case 'm': + synth_opts->period *= 1000; + /* Fall through */ + case 'u': + synth_opts->period *= 1000; + /* Fall through */ + case 'n': + if (*p++ != 's') + goto out_err; + synth_opts->period_type = + PERF_ITRACE_PERIOD_NANOSECS; + period_type_set = true; + break; + case '\0': + goto out; + default: + goto out_err; + } + } + break; + case 'b': + synth_opts->branches = true; + break; + case 'x': + synth_opts->transactions = true; + break; + case 'w': + synth_opts->ptwrites = true; + break; + case 'p': + synth_opts->pwr_events = true; + break; + case 'o': + synth_opts->other_events = true; + break; + case 'I': + synth_opts->intr_events = true; + break; + case 'e': + synth_opts->errors = true; + if (get_flags(&p, &synth_opts->error_plus_flags, + &synth_opts->error_minus_flags)) + goto out_err; + break; + case 'd': + synth_opts->log = true; + if (get_flags(&p, &synth_opts->log_plus_flags, + &synth_opts->log_minus_flags)) + goto out_err; + if (synth_opts->log_plus_flags & AUXTRACE_LOG_FLG_ON_ERROR) + synth_opts->log_on_error_size = itrace_log_on_error_size(); + break; + case 'c': + synth_opts->branches = true; + synth_opts->calls = true; + break; + case 'r': + synth_opts->branches = true; + synth_opts->returns = true; + break; + case 'G': + case 'g': + if (p[-1] == 'G') + synth_opts->add_callchain = true; + else + synth_opts->callchain = true; + synth_opts->callchain_sz = + PERF_ITRACE_DEFAULT_CALLCHAIN_SZ; + while (*p == ' ' || *p == ',') + p += 1; + if (isdigit(*p)) { + unsigned int val; + + val = strtoul(p, &endptr, 10); + p = endptr; + if (!val || val > PERF_ITRACE_MAX_CALLCHAIN_SZ) + goto out_err; + synth_opts->callchain_sz = val; + } + break; + case 'L': + case 'l': + if (p[-1] == 'L') + synth_opts->add_last_branch = true; + else + synth_opts->last_branch = true; + synth_opts->last_branch_sz = + PERF_ITRACE_DEFAULT_LAST_BRANCH_SZ; + while (*p == ' ' || *p == ',') + p += 1; + if (isdigit(*p)) { + unsigned int val; + + val = strtoul(p, &endptr, 10); + p = endptr; + if (!val || + val > PERF_ITRACE_MAX_LAST_BRANCH_SZ) + goto out_err; + synth_opts->last_branch_sz = val; + } + break; + case 's': + synth_opts->initial_skip = strtoul(p, &endptr, 10); + if (p == endptr) + goto out_err; + p = endptr; + break; + case 'f': + synth_opts->flc = true; + break; + case 'm': + synth_opts->llc = true; + break; + case 't': + synth_opts->tlb = true; + break; + case 'a': + synth_opts->remote_access = true; + break; + case 'M': + synth_opts->mem = true; + break; + case 'q': + synth_opts->quick += 1; + break; + case 'A': + synth_opts->approx_ipc = true; + break; + case 'Z': + synth_opts->timeless_decoding = true; + break; + case 'T': + synth_opts->use_timestamp = true; + break; + case ' ': + case ',': + break; + default: + goto out_err; + } + } +out: + if (iy) { + if (!period_type_set) + synth_opts->period_type = + PERF_ITRACE_DEFAULT_PERIOD_TYPE; + if (!period_set) + synth_opts->period = PERF_ITRACE_DEFAULT_PERIOD; + } + + return 0; + +out_err: + pr_err("Bad Instruction Tracing options '%s'\n", str); + return -EINVAL; +} + +int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset) +{ + return itrace_do_parse_synth_opts(opt->value, str, unset); +} + +static const char * const auxtrace_error_type_name[] = { + [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace", +}; + +static const char *auxtrace_error_name(unsigned int type) +{ + const char *error_type_name = NULL; + + if (type < PERF_AUXTRACE_ERROR_MAX) + error_type_name = auxtrace_error_type_name[type]; + if (!error_type_name) + error_type_name = "unknown AUX"; + return error_type_name; +} + +size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp) +{ + struct perf_record_auxtrace_error *e = &event->auxtrace_error; + unsigned long long nsecs = e->time; + const char *msg = e->msg; + int msg_max; + int ret; + + ret = fprintf(fp, " %s error type %u", + auxtrace_error_name(e->type), e->type); + + if (e->fmt && nsecs) { + unsigned long secs = nsecs / NSEC_PER_SEC; + + nsecs -= secs * NSEC_PER_SEC; + ret += fprintf(fp, " time %lu.%09llu", secs, nsecs); + } else { + ret += fprintf(fp, " time 0"); + } + + if (!e->fmt) + msg = (const char *)&e->time; + + /* Bound msg to the bytes actually within the event, capped at the array size */ + msg_max = (int)((void *)event + event->header.size - (void *)msg); + if (msg_max < 0) + msg_max = 0; + if (msg_max > (int)sizeof(e->msg)) + msg_max = sizeof(e->msg); + + /* + * Unlike the swap path which downgrades fmt in place, + * native-endian events are mmap'd read-only — check size + * instead to avoid accessing machine_pid/vcpu OOB. + */ + if (e->fmt >= 2 && + event->header.size >= offsetof(typeof(event->auxtrace_error), vcpu) + + sizeof(event->auxtrace_error.vcpu) && + e->machine_pid) + ret += fprintf(fp, " machine_pid %d vcpu %d", e->machine_pid, e->vcpu); + + ret += fprintf(fp, " cpu %d pid %d tid %d ip %#"PRI_lx64" code %u: %.*s\n", + e->cpu, e->pid, e->tid, e->ip, e->code, msg_max, msg); + return ret; +} + +void perf_session__auxtrace_error_inc(struct perf_session *session, + union perf_event *event) +{ + struct perf_record_auxtrace_error *e = &event->auxtrace_error; + + if (e->type < PERF_AUXTRACE_ERROR_MAX) + session->evlist->stats.nr_auxtrace_errors[e->type] += 1; +} + +void events_stats__auxtrace_error_warn(const struct events_stats *stats) +{ + int i; + + for (i = 0; i < PERF_AUXTRACE_ERROR_MAX; i++) { + if (!stats->nr_auxtrace_errors[i]) + continue; + ui__warning("%u %s errors\n", + stats->nr_auxtrace_errors[i], + auxtrace_error_name(i)); + } +} + +int perf_event__process_auxtrace_error(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + if (auxtrace__dont_decode(session)) + return 0; + + perf_event__fprintf_auxtrace_error(event, stdout); + return 0; +} + +/* + * In the compat mode kernel runs in 64-bit and perf tool runs in 32-bit mode, + * 32-bit perf tool cannot access 64-bit value atomically, which might lead to + * the issues caused by the below sequence on multiple CPUs: when perf tool + * accesses either the load operation or the store operation for 64-bit value, + * on some architectures the operation is divided into two instructions, one + * is for accessing the low 32-bit value and another is for the high 32-bit; + * thus these two user operations can give the kernel chances to access the + * 64-bit value, and thus leads to the unexpected load values. + * + * kernel (64-bit) user (32-bit) + * + * if (LOAD ->aux_tail) { --, LOAD ->aux_head_lo + * STORE $aux_data | ,---> + * FLUSH $aux_data | | LOAD ->aux_head_hi + * STORE ->aux_head --|-------` smp_rmb() + * } | LOAD $data + * | smp_mb() + * | STORE ->aux_tail_lo + * `-----------> + * STORE ->aux_tail_hi + * + * For this reason, it's impossible for the perf tool to work correctly when + * the AUX head or tail is bigger than 4GB (more than 32 bits length); and we + * can not simply limit the AUX ring buffer to less than 4GB, the reason is + * the pointers can be increased monotonically, whatever the buffer size it is, + * at the end the head and tail can be bigger than 4GB and carry out to the + * high 32-bit. + * + * To mitigate the issues and improve the user experience, we can allow the + * perf tool working in certain conditions and bail out with error if detect + * any overflow cannot be handled. + * + * For reading the AUX head, it reads out the values for three times, and + * compares the high 4 bytes of the values between the first time and the last + * time, if there has no change for high 4 bytes injected by the kernel during + * the user reading sequence, it's safe for use the second value. + * + * When compat_auxtrace_mmap__write_tail() detects any carrying in the high + * 32 bits, it means there have two store operations in user space and it cannot + * promise the atomicity for 64-bit write, so return '-1' in this case to tell + * the caller an overflow error has happened. + */ +u64 __weak compat_auxtrace_mmap__read_head(struct auxtrace_mmap *mm) +{ + struct perf_event_mmap_page *pc = mm->userpg; + u64 first, second, last; + u64 mask = (u64)(UINT32_MAX) << 32; + + do { + first = READ_ONCE(pc->aux_head); + /* Ensure all reads are done after we read the head */ + smp_rmb(); + second = READ_ONCE(pc->aux_head); + /* Ensure all reads are done after we read the head */ + smp_rmb(); + last = READ_ONCE(pc->aux_head); + } while ((first & mask) != (last & mask)); + + return second; +} + +int __weak compat_auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail) +{ + struct perf_event_mmap_page *pc = mm->userpg; + u64 mask = (u64)(UINT32_MAX) << 32; + + if (tail & mask) + return -1; + + /* Ensure all reads are done before we write the tail out */ + smp_mb(); + WRITE_ONCE(pc->aux_tail, tail); + return 0; +} + +static int __auxtrace_mmap__read(struct mmap *map, + struct auxtrace_record *itr, struct perf_env *env, + const struct perf_tool *tool, process_auxtrace_t fn, + bool snapshot, size_t snapshot_size) +{ + struct auxtrace_mmap *mm = &map->auxtrace_mmap; + u64 head, old = mm->prev, offset, ref; + unsigned char *data = mm->base; + size_t size, head_off, old_off, len1, len2, padding; + union perf_event ev; + void *data1, *data2; + int kernel_is_64_bit = perf_env__kernel_is_64_bit(env); + + head = auxtrace_mmap__read_head(mm, kernel_is_64_bit); + + if (snapshot && + auxtrace_record__find_snapshot(itr, mm->idx, mm, data, &head, &old)) + return -1; + + if (old == head) + return 0; + + pr_debug3("auxtrace idx %d old %#"PRIx64" head %#"PRIx64" diff %#"PRIx64"\n", + mm->idx, old, head, head - old); + + if (mm->mask) { + head_off = head & mm->mask; + old_off = old & mm->mask; + } else { + head_off = head % mm->len; + old_off = old % mm->len; + } + + if (head_off > old_off) + size = head_off - old_off; + else + size = mm->len - (old_off - head_off); + + if (snapshot && size > snapshot_size) + size = snapshot_size; + + ref = auxtrace_record__reference(itr); + + if (head > old || size <= head || mm->mask) { + offset = head - size; + } else { + /* + * When the buffer size is not a power of 2, 'head' wraps at the + * highest multiple of the buffer size, so we have to subtract + * the remainder here. + */ + u64 rem = (0ULL - mm->len) % mm->len; + + offset = head - size - rem; + } + + if (size > head_off) { + len1 = size - head_off; + data1 = &data[mm->len - len1]; + len2 = head_off; + data2 = &data[0]; + } else { + len1 = size; + data1 = &data[head_off - len1]; + len2 = 0; + data2 = NULL; + } + + if (itr->alignment) { + unsigned int unwanted = len1 % itr->alignment; + + len1 -= unwanted; + size -= unwanted; + } + + /* padding must be written by fn() e.g. record__process_auxtrace() */ + padding = size & (PERF_AUXTRACE_RECORD_ALIGNMENT - 1); + if (padding) + padding = PERF_AUXTRACE_RECORD_ALIGNMENT - padding; + + memset(&ev, 0, sizeof(ev)); + ev.auxtrace.header.type = PERF_RECORD_AUXTRACE; + ev.auxtrace.header.size = sizeof(ev.auxtrace); + ev.auxtrace.size = size + padding; + ev.auxtrace.offset = offset; + ev.auxtrace.reference = ref; + ev.auxtrace.idx = mm->idx; + ev.auxtrace.tid = mm->tid; + ev.auxtrace.cpu = mm->cpu; + + if (fn(tool, map, &ev, data1, len1, data2, len2)) + return -1; + + mm->prev = head; + + if (!snapshot) { + int err; + + err = auxtrace_mmap__write_tail(mm, head, kernel_is_64_bit); + if (err < 0) + return err; + + if (itr->read_finish) { + err = itr->read_finish(itr, mm->idx); + if (err < 0) + return err; + } + } + + return 1; +} + +int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr, + struct perf_env *env, const struct perf_tool *tool, + process_auxtrace_t fn) +{ + return __auxtrace_mmap__read(map, itr, env, tool, fn, false, 0); +} + +int auxtrace_mmap__read_snapshot(struct mmap *map, + struct auxtrace_record *itr, struct perf_env *env, + const struct perf_tool *tool, process_auxtrace_t fn, + size_t snapshot_size) +{ + return __auxtrace_mmap__read(map, itr, env, tool, fn, true, snapshot_size); +} + +/** + * struct auxtrace_cache - hash table to implement a cache + * @hashtable: the hashtable + * @sz: hashtable size (number of hlists) + * @entry_size: size of an entry + * @limit: limit the number of entries to this maximum, when reached the cache + * is dropped and caching begins again with an empty cache + * @cnt: current number of entries + * @bits: hashtable size (@sz = 2^@bits) + */ +struct auxtrace_cache { + struct hlist_head *hashtable; + size_t sz; + size_t entry_size; + size_t limit; + size_t cnt; + unsigned int bits; +}; + +struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size, + unsigned int limit_percent) +{ + struct auxtrace_cache *c; + struct hlist_head *ht; + size_t sz, i; + + c = zalloc(sizeof(struct auxtrace_cache)); + if (!c) + return NULL; + + sz = 1UL << bits; + + ht = calloc(sz, sizeof(struct hlist_head)); + if (!ht) + goto out_free; + + for (i = 0; i < sz; i++) + INIT_HLIST_HEAD(&ht[i]); + + c->hashtable = ht; + c->sz = sz; + c->entry_size = entry_size; + c->limit = (c->sz * limit_percent) / 100; + c->bits = bits; + + return c; + +out_free: + free(c); + return NULL; +} + +static void auxtrace_cache__drop(struct auxtrace_cache *c) +{ + struct auxtrace_cache_entry *entry; + struct hlist_node *tmp; + size_t i; + + if (!c) + return; + + for (i = 0; i < c->sz; i++) { + hlist_for_each_entry_safe(entry, tmp, &c->hashtable[i], hash) { + hlist_del(&entry->hash); + auxtrace_cache__free_entry(c, entry); + } + } + + c->cnt = 0; +} + +void auxtrace_cache__free(struct auxtrace_cache *c) +{ + if (!c) + return; + + auxtrace_cache__drop(c); + zfree(&c->hashtable); + free(c); +} + +void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c) +{ + return malloc(c->entry_size); +} + +void auxtrace_cache__free_entry(struct auxtrace_cache *c __maybe_unused, + void *entry) +{ + free(entry); +} + +int auxtrace_cache__add(struct auxtrace_cache *c, u32 key, + struct auxtrace_cache_entry *entry) +{ + if (c->limit && ++c->cnt > c->limit) + auxtrace_cache__drop(c); + + entry->key = key; + hlist_add_head(&entry->hash, &c->hashtable[hash_32(key, c->bits)]); + + return 0; +} + +static struct auxtrace_cache_entry *auxtrace_cache__rm(struct auxtrace_cache *c, + u32 key) +{ + struct auxtrace_cache_entry *entry; + struct hlist_head *hlist; + struct hlist_node *n; + + if (!c) + return NULL; + + hlist = &c->hashtable[hash_32(key, c->bits)]; + hlist_for_each_entry_safe(entry, n, hlist, hash) { + if (entry->key == key) { + hlist_del(&entry->hash); + return entry; + } + } + + return NULL; +} + +void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key) +{ + struct auxtrace_cache_entry *entry = auxtrace_cache__rm(c, key); + + auxtrace_cache__free_entry(c, entry); +} + +void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key) +{ + struct auxtrace_cache_entry *entry; + struct hlist_head *hlist; + + if (!c) + return NULL; + + hlist = &c->hashtable[hash_32(key, c->bits)]; + hlist_for_each_entry(entry, hlist, hash) { + if (entry->key == key) + return entry; + } + + return NULL; +} + +static void addr_filter__free_str(struct addr_filter *filt) +{ + zfree(&filt->str); + filt->action = NULL; + filt->sym_from = NULL; + filt->sym_to = NULL; + filt->filename = NULL; +} + +static struct addr_filter *addr_filter__new(void) +{ + struct addr_filter *filt = zalloc(sizeof(*filt)); + + if (filt) + INIT_LIST_HEAD(&filt->list); + + return filt; +} + +static void addr_filter__free(struct addr_filter *filt) +{ + if (filt) + addr_filter__free_str(filt); + free(filt); +} + +static void addr_filters__add(struct addr_filters *filts, + struct addr_filter *filt) +{ + list_add_tail(&filt->list, &filts->head); + filts->cnt += 1; +} + +static void addr_filters__del(struct addr_filters *filts, + struct addr_filter *filt) +{ + list_del_init(&filt->list); + filts->cnt -= 1; +} + +void addr_filters__init(struct addr_filters *filts) +{ + INIT_LIST_HEAD(&filts->head); + filts->cnt = 0; +} + +void addr_filters__exit(struct addr_filters *filts) +{ + struct addr_filter *filt, *n; + + list_for_each_entry_safe(filt, n, &filts->head, list) { + addr_filters__del(filts, filt); + addr_filter__free(filt); + } +} + +static int parse_num_or_str(char **inp, u64 *num, const char **str, + const char *str_delim) +{ + *inp += strspn(*inp, " "); + + if (isdigit(**inp)) { + char *endptr; + + if (!num) + return -EINVAL; + errno = 0; + *num = strtoull(*inp, &endptr, 0); + if (errno) + return -errno; + if (endptr == *inp) + return -EINVAL; + *inp = endptr; + } else { + size_t n; + + if (!str) + return -EINVAL; + *inp += strspn(*inp, " "); + *str = *inp; + n = strcspn(*inp, str_delim); + if (!n) + return -EINVAL; + *inp += n; + if (**inp) { + **inp = '\0'; + *inp += 1; + } + } + return 0; +} + +static int parse_action(struct addr_filter *filt) +{ + if (!strcmp(filt->action, "filter")) { + filt->start = true; + filt->range = true; + } else if (!strcmp(filt->action, "start")) { + filt->start = true; + } else if (!strcmp(filt->action, "stop")) { + filt->start = false; + } else if (!strcmp(filt->action, "tracestop")) { + filt->start = false; + filt->range = true; + filt->action += 5; /* Change 'tracestop' to 'stop' */ + } else { + return -EINVAL; + } + return 0; +} + +static int parse_sym_idx(char **inp, int *idx) +{ + *idx = -1; + + *inp += strspn(*inp, " "); + + if (**inp != '#') + return 0; + + *inp += 1; + + if (**inp == 'g' || **inp == 'G') { + *inp += 1; + *idx = 0; + } else { + unsigned long num; + char *endptr; + + errno = 0; + num = strtoul(*inp, &endptr, 0); + if (errno) + return -errno; + if (endptr == *inp || num > INT_MAX) + return -EINVAL; + *inp = endptr; + *idx = num; + } + + return 0; +} + +static int parse_addr_size(char **inp, u64 *num, const char **str, int *idx) +{ + int err = parse_num_or_str(inp, num, str, " "); + + if (!err && *str) + err = parse_sym_idx(inp, idx); + + return err; +} + +static int parse_one_filter(struct addr_filter *filt, const char **filter_inp) +{ + char *fstr; + int err; + + filt->str = fstr = strdup(*filter_inp); + if (!fstr) + return -ENOMEM; + + err = parse_num_or_str(&fstr, NULL, &filt->action, " "); + if (err) + goto out_err; + + err = parse_action(filt); + if (err) + goto out_err; + + err = parse_addr_size(&fstr, &filt->addr, &filt->sym_from, + &filt->sym_from_idx); + if (err) + goto out_err; + + fstr += strspn(fstr, " "); + + if (*fstr == '/') { + fstr += 1; + err = parse_addr_size(&fstr, &filt->size, &filt->sym_to, + &filt->sym_to_idx); + if (err) + goto out_err; + filt->range = true; + } + + fstr += strspn(fstr, " "); + + if (*fstr == '@') { + fstr += 1; + err = parse_num_or_str(&fstr, NULL, &filt->filename, " ,"); + if (err) + goto out_err; + } + + fstr += strspn(fstr, " ,"); + + *filter_inp += fstr - filt->str; + + return 0; + +out_err: + addr_filter__free_str(filt); + + return err; +} + +int addr_filters__parse_bare_filter(struct addr_filters *filts, + const char *filter) +{ + struct addr_filter *filt; + const char *fstr = filter; + int err; + + while (*fstr) { + filt = addr_filter__new(); + err = parse_one_filter(filt, &fstr); + if (err) { + addr_filter__free(filt); + addr_filters__exit(filts); + return err; + } + addr_filters__add(filts, filt); + } + + return 0; +} + +struct sym_args { + const char *name; + u64 start; + u64 size; + int idx; + int cnt; + bool started; + bool global; + bool selected; + bool duplicate; + bool near; +}; + +static bool kern_sym_name_match(const char *kname, const char *name) +{ + size_t n = strlen(name); + + return !strcmp(kname, name) || + (!strncmp(kname, name, n) && kname[n] == '\t'); +} + +static bool kern_sym_match(struct sym_args *args, const char *name, char type) +{ + /* A function with the same name, and global or the n'th found or any */ + return kallsyms__is_function(type) && + kern_sym_name_match(name, args->name) && + ((args->global && isupper(type)) || + (args->selected && ++(args->cnt) == args->idx) || + (!args->global && !args->selected)); +} + +static int find_kern_sym_cb(void *arg, const char *name, char type, u64 start) +{ + struct sym_args *args = arg; + + if (args->started) { + if (!args->size) + args->size = start - args->start; + if (args->selected) { + if (args->size) + return 1; + } else if (kern_sym_match(args, name, type)) { + args->duplicate = true; + return 1; + } + } else if (kern_sym_match(args, name, type)) { + args->started = true; + args->start = start; + } + + return 0; +} + +static int print_kern_sym_cb(void *arg, const char *name, char type, u64 start) +{ + struct sym_args *args = arg; + + if (kern_sym_match(args, name, type)) { + pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", + ++args->cnt, start, type, name); + args->near = true; + } else if (args->near) { + args->near = false; + pr_err("\t\twhich is near\t\t%s\n", name); + } + + return 0; +} + +static int sym_not_found_error(const char *sym_name, int idx) +{ + if (idx > 0) { + pr_err("N'th occurrence (N=%d) of symbol '%s' not found.\n", + idx, sym_name); + } else if (!idx) { + pr_err("Global symbol '%s' not found.\n", sym_name); + } else { + pr_err("Symbol '%s' not found.\n", sym_name); + } + pr_err("Note that symbols must be functions.\n"); + + return -EINVAL; +} + +static int find_kern_sym(const char *sym_name, u64 *start, u64 *size, int idx) +{ + struct sym_args args = { + .name = sym_name, + .idx = idx, + .global = !idx, + .selected = idx > 0, + }; + int err; + + *start = 0; + *size = 0; + + err = kallsyms__parse("/proc/kallsyms", &args, find_kern_sym_cb); + if (err < 0) { + pr_err("Failed to parse /proc/kallsyms\n"); + return err; + } + + if (args.duplicate) { + pr_err("Multiple kernel symbols with name '%s'\n", sym_name); + args.cnt = 0; + kallsyms__parse("/proc/kallsyms", &args, print_kern_sym_cb); + pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n", + sym_name); + pr_err("Or select a global symbol by inserting #0 or #g or #G\n"); + return -EINVAL; + } + + if (!args.started) { + pr_err("Kernel symbol lookup: "); + return sym_not_found_error(sym_name, idx); + } + + *start = args.start; + *size = args.size; + + return 0; +} + +static int find_entire_kern_cb(void *arg, const char *name __maybe_unused, + char type, u64 start) +{ + struct sym_args *args = arg; + u64 size; + + if (!kallsyms__is_function(type)) + return 0; + + if (!args->started) { + args->started = true; + args->start = start; + } + /* Don't know exactly where the kernel ends, so we add a page */ + size = round_up(start, page_size) + page_size - args->start; + if (size > args->size) + args->size = size; + + return 0; +} + +static int addr_filter__entire_kernel(struct addr_filter *filt) +{ + struct sym_args args = { .started = false }; + int err; + + err = kallsyms__parse("/proc/kallsyms", &args, find_entire_kern_cb); + if (err < 0 || !args.started) { + pr_err("Failed to parse /proc/kallsyms\n"); + return err; + } + + filt->addr = args.start; + filt->size = args.size; + + return 0; +} + +static int check_end_after_start(struct addr_filter *filt, u64 start, u64 size) +{ + if (start + size >= filt->addr) + return 0; + + if (filt->sym_from) { + pr_err("Symbol '%s' (0x%"PRIx64") comes before '%s' (0x%"PRIx64")\n", + filt->sym_to, start, filt->sym_from, filt->addr); + } else { + pr_err("Symbol '%s' (0x%"PRIx64") comes before address 0x%"PRIx64")\n", + filt->sym_to, start, filt->addr); + } + + return -EINVAL; +} + +static int addr_filter__resolve_kernel_syms(struct addr_filter *filt) +{ + bool no_size = false; + u64 start, size; + int err; + + if (symbol_conf.kptr_restrict) { + pr_err("Kernel addresses are restricted. Unable to resolve kernel symbols.\n"); + return -EINVAL; + } + + if (filt->sym_from && !strcmp(filt->sym_from, "*")) + return addr_filter__entire_kernel(filt); + + if (filt->sym_from) { + err = find_kern_sym(filt->sym_from, &start, &size, + filt->sym_from_idx); + if (err) + return err; + filt->addr = start; + if (filt->range && !filt->size && !filt->sym_to) { + filt->size = size; + no_size = !size; + } + } + + if (filt->sym_to) { + err = find_kern_sym(filt->sym_to, &start, &size, + filt->sym_to_idx); + if (err) + return err; + + err = check_end_after_start(filt, start, size); + if (err) + return err; + filt->size = start + size - filt->addr; + no_size = !size; + } + + /* The very last symbol in kallsyms does not imply a particular size */ + if (no_size) { + pr_err("Cannot determine size of symbol '%s'\n", + filt->sym_to ? filt->sym_to : filt->sym_from); + return -EINVAL; + } + + return 0; +} + +static struct dso *load_dso(const char *name) +{ + struct map *map; + struct dso *dso; + + map = dso__new_map(name); + if (!map) + return NULL; + + if (map__load(map) < 0) + pr_err("File '%s' not found or has no symbols.\n", name); + + dso = dso__get(map__dso(map)); + + map__put(map); + + return dso; +} + +static bool dso_sym_match(struct symbol *sym, const char *name, int *cnt, + int idx) +{ + /* Same name, and global or the n'th found or any */ + return !arch__compare_symbol_names(name, sym->name) && + ((!idx && symbol__binding(sym) == STB_GLOBAL) || + (idx > 0 && ++*cnt == idx) || + idx < 0); +} + +static void print_duplicate_syms(struct dso *dso, const char *sym_name) +{ + struct symbol *sym; + bool near = false; + int cnt = 0; + + pr_err("Multiple symbols with name '%s'\n", sym_name); + + sym = dso__first_symbol(dso); + while (sym) { + if (dso_sym_match(sym, sym_name, &cnt, -1)) { + pr_err("#%d\t0x%"PRIx64"\t%c\t%s\n", + ++cnt, sym->start, + symbol__binding(sym) == STB_GLOBAL ? 'g' : + symbol__binding(sym) == STB_LOCAL ? 'l' : 'w', + sym->name); + near = true; + } else if (near) { + near = false; + pr_err("\t\twhich is near\t\t%s\n", sym->name); + } + sym = dso__next_symbol(sym); + } + + pr_err("Disambiguate symbol name by inserting #n after the name e.g. %s #2\n", + sym_name); + pr_err("Or select a global symbol by inserting #0 or #g or #G\n"); +} + +static int find_dso_sym(struct dso *dso, const char *sym_name, u64 *start, + u64 *size, int idx) +{ + struct symbol *sym; + int cnt = 0; + + *start = 0; + *size = 0; + + sym = dso__first_symbol(dso); + while (sym) { + if (*start) { + if (!*size) + *size = sym->start - *start; + if (idx > 0) { + if (*size) + return 0; + } else if (dso_sym_match(sym, sym_name, &cnt, idx)) { + print_duplicate_syms(dso, sym_name); + return -EINVAL; + } + } else if (dso_sym_match(sym, sym_name, &cnt, idx)) { + *start = sym->start; + *size = sym->end - sym->start; + } + sym = dso__next_symbol(sym); + } + + if (!*start) + return sym_not_found_error(sym_name, idx); + + return 0; +} + +static int addr_filter__entire_dso(struct addr_filter *filt, struct dso *dso) +{ + if (dso__data_file_size(dso, NULL)) { + pr_err("Failed to determine filter for %s\nCannot determine file size.\n", + filt->filename); + return -EINVAL; + } + + filt->addr = 0; + filt->size = dso__data(dso)->file_size; + + return 0; +} + +static int addr_filter__resolve_syms(struct addr_filter *filt) +{ + u64 start, size; + struct dso *dso; + int err = 0; + + if (!filt->sym_from && !filt->sym_to) + return 0; + + if (!filt->filename) + return addr_filter__resolve_kernel_syms(filt); + + dso = load_dso(filt->filename); + if (!dso) { + pr_err("Failed to load symbols from: %s\n", filt->filename); + return -EINVAL; + } + + if (filt->sym_from && !strcmp(filt->sym_from, "*")) { + err = addr_filter__entire_dso(filt, dso); + goto put_dso; + } + + if (filt->sym_from) { + err = find_dso_sym(dso, filt->sym_from, &start, &size, + filt->sym_from_idx); + if (err) + goto put_dso; + filt->addr = start; + if (filt->range && !filt->size && !filt->sym_to) + filt->size = size; + } + + if (filt->sym_to) { + err = find_dso_sym(dso, filt->sym_to, &start, &size, + filt->sym_to_idx); + if (err) + goto put_dso; + + err = check_end_after_start(filt, start, size); + if (err) + return err; + + filt->size = start + size - filt->addr; + } + +put_dso: + dso__put(dso); + + return err; +} + +static char *addr_filter__to_str(struct addr_filter *filt) +{ + char filename_buf[PATH_MAX]; + const char *at = ""; + const char *fn = ""; + char *filter; + int err; + + if (filt->filename) { + at = "@"; + fn = realpath(filt->filename, filename_buf); + if (!fn) + return NULL; + } + + if (filt->range) { + err = asprintf(&filter, "%s 0x%"PRIx64"/0x%"PRIx64"%s%s", + filt->action, filt->addr, filt->size, at, fn); + } else { + err = asprintf(&filter, "%s 0x%"PRIx64"%s%s", + filt->action, filt->addr, at, fn); + } + + return err < 0 ? NULL : filter; +} + +static int parse_addr_filter(struct evsel *evsel, const char *filter, + int max_nr) +{ + struct addr_filters filts; + struct addr_filter *filt; + int err; + + addr_filters__init(&filts); + + err = addr_filters__parse_bare_filter(&filts, filter); + if (err) + goto out_exit; + + if (filts.cnt > max_nr) { + pr_err("Error: number of address filters (%d) exceeds maximum (%d)\n", + filts.cnt, max_nr); + err = -EINVAL; + goto out_exit; + } + + list_for_each_entry(filt, &filts.head, list) { + char *new_filter; + + err = addr_filter__resolve_syms(filt); + if (err) + goto out_exit; + + new_filter = addr_filter__to_str(filt); + if (!new_filter) { + err = -ENOMEM; + goto out_exit; + } + + if (evsel__append_addr_filter(evsel, new_filter)) { + err = -ENOMEM; + goto out_exit; + } + } + +out_exit: + addr_filters__exit(&filts); + + if (err) { + pr_err("Failed to parse address filter: '%s'\n", filter); + pr_err("Filter format is: filter|start|stop|tracestop [/ ] [@]\n"); + pr_err("Where multiple filters are separated by space or comma.\n"); + } + + return err; +} + +static int evsel__nr_addr_filter(struct evsel *evsel) +{ + struct perf_pmu *pmu = evsel__find_pmu(evsel); + int nr_addr_filters = 0; + + if (!pmu) + return 0; + + perf_pmu__scan_file(pmu, "nr_addr_filters", "%d", &nr_addr_filters); + + return nr_addr_filters; +} + +int auxtrace_parse_filters(struct evlist *evlist) +{ + struct evsel *evsel; + char *filter; + int err, max_nr; + + evlist__for_each_entry(evlist, evsel) { + filter = evsel->filter; + max_nr = evsel__nr_addr_filter(evsel); + if (!filter || !max_nr) + continue; + evsel->filter = NULL; + err = parse_addr_filter(evsel, filter, max_nr); + free(filter); + if (err) + return err; + pr_debug("Address filter: %s\n", evsel->filter); + } + + return 0; +} + +int auxtrace__process_event(struct perf_session *session, union perf_event *event, + struct perf_sample *sample, const struct perf_tool *tool) +{ + if (!session->auxtrace) + return 0; + + return session->auxtrace->process_event(session, event, sample, tool); +} + +void auxtrace__dump_auxtrace_sample(struct perf_session *session, + struct perf_sample *sample) +{ + if (!session->auxtrace || !session->auxtrace->dump_auxtrace_sample || + auxtrace__dont_decode(session)) + return; + + session->auxtrace->dump_auxtrace_sample(session, sample); +} + +int auxtrace__flush_events(struct perf_session *session, const struct perf_tool *tool) +{ + if (!session->auxtrace) + return 0; + + return session->auxtrace->flush_events(session, tool); +} + +void auxtrace__free_events(struct perf_session *session) +{ + if (!session->auxtrace) + return; + + return session->auxtrace->free_events(session); +} + +void auxtrace__free(struct perf_session *session) +{ + if (!session->auxtrace) + return; + + return session->auxtrace->free(session); +} + +bool auxtrace__evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + if (!session->auxtrace || !session->auxtrace->evsel_is_auxtrace) + return false; + + return session->auxtrace->evsel_is_auxtrace(session, evsel); +} diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h new file mode 100644 index 000000000..6947f3f28 --- /dev/null +++ b/tools/perf/util/auxtrace.h @@ -0,0 +1,702 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * auxtrace.h: AUX area trace support + * Copyright (c) 2013-2015, Intel Corporation. + */ + +#ifndef __PERF_AUXTRACE_H +#define __PERF_AUXTRACE_H + +#include +#include // FILE +#include +#include +#include +#include + +union perf_event; +struct perf_session; +struct evlist; +struct evsel; +struct perf_env; +struct perf_tool; +struct mmap; +struct perf_sample; +struct option; +struct record_opts; +struct perf_record_auxtrace_error; +struct perf_record_auxtrace_info; +struct events_stats; +struct perf_pmu; + +enum auxtrace_error_type { + PERF_AUXTRACE_ERROR_ITRACE = 1, + PERF_AUXTRACE_ERROR_MAX +}; + +/* Auxtrace records must have the same alignment as perf event records */ +#define PERF_AUXTRACE_RECORD_ALIGNMENT 8 + +enum auxtrace_type { + PERF_AUXTRACE_UNKNOWN, + PERF_AUXTRACE_INTEL_PT, + PERF_AUXTRACE_INTEL_BTS, + PERF_AUXTRACE_CS_ETM, + PERF_AUXTRACE_ARM_SPE, + PERF_AUXTRACE_S390_CPUMSF, + PERF_AUXTRACE_HISI_PTT, + PERF_AUXTRACE_VPA_DTL, +}; + +enum itrace_period_type { + PERF_ITRACE_PERIOD_INSTRUCTIONS, + PERF_ITRACE_PERIOD_TICKS, + PERF_ITRACE_PERIOD_NANOSECS, +}; + +#define AUXTRACE_ERR_FLG_OVERFLOW (1 << ('o' - 'a')) +#define AUXTRACE_ERR_FLG_DATA_LOST (1 << ('l' - 'a')) + +#define AUXTRACE_LOG_FLG_ALL_PERF_EVTS (1 << ('a' - 'a')) +#define AUXTRACE_LOG_FLG_ON_ERROR (1 << ('e' - 'a')) +#define AUXTRACE_LOG_FLG_USE_STDOUT (1 << ('o' - 'a')) + +/** + * struct itrace_synth_opts - AUX area tracing synthesis options. + * @set: indicates whether or not options have been set + * @default_no_sample: Default to no sampling. + * @inject: indicates the event (not just the sample) must be fully synthesized + * because 'perf inject' will write it out + * @instructions: whether to synthesize 'instructions' events + * @cycles: whether to synthesize 'cycles' events + * (not fully accurate, since CYC packets are only emitted + * together with other events, such as branches) + * @branches: whether to synthesize 'branches' events + * @transactions: whether to synthesize events for transactions + * @ptwrites: whether to synthesize events for ptwrites + * @pwr_events: whether to synthesize power events + * @other_events: whether to synthesize other events recorded due to the use of + * aux_output + * @intr_events: whether to synthesize interrupt events + * @errors: whether to synthesize decoder error events + * @dont_decode: whether to skip decoding entirely + * @log: write a decoding log + * @calls: limit branch samples to calls (can be combined with @returns) + * @returns: limit branch samples to returns (can be combined with @calls) + * @callchain: add callchain to 'instructions' events + * @add_callchain: add callchain to existing event records + * @thread_stack: feed branches to the thread_stack + * @last_branch: add branch context to 'instruction' events + * @add_last_branch: add branch context to existing event records + * @approx_ipc: approximate IPC + * @flc: whether to synthesize first level cache events + * @llc: whether to synthesize last level cache events + * @tlb: whether to synthesize TLB events + * @remote_access: whether to synthesize remote access events + * @mem: whether to synthesize memory events + * @timeless_decoding: prefer "timeless" decoding i.e. ignore timestamps + * @use_timestamp: use the timestamp trace as kernel time + * @vm_time_correlation: perform VM Time Correlation + * @vm_tm_corr_dry_run: VM Time Correlation dry-run + * @vm_tm_corr_args: VM Time Correlation implementation-specific arguments + * @callchain_sz: maximum callchain size + * @last_branch_sz: branch context size + * @period: 'instructions' events period + * @period_type: 'instructions' events period type + * @initial_skip: skip N events at the beginning. + * @cpu_bitmap: CPUs for which to synthesize events, or NULL for all + * @ptime_range: time intervals to trace or NULL + * @range_num: number of time intervals to trace + * @error_plus_flags: flags to affect what errors are reported + * @error_minus_flags: flags to affect what errors are reported + * @log_plus_flags: flags to affect what is logged + * @log_minus_flags: flags to affect what is logged + * @quick: quicker (less detailed) decoding + * @log_on_error_size: size of log to keep for outputting log only on errors + */ +struct itrace_synth_opts { + bool set; + bool default_no_sample; + bool inject; + bool instructions; + bool cycles; + bool branches; + bool transactions; + bool ptwrites; + bool pwr_events; + bool other_events; + bool intr_events; + bool errors; + bool dont_decode; + bool log; + bool calls; + bool returns; + bool callchain; + bool add_callchain; + bool thread_stack; + bool last_branch; + bool add_last_branch; + bool approx_ipc; + bool flc; + bool llc; + bool tlb; + bool remote_access; + bool mem; + bool timeless_decoding; + bool use_timestamp; + bool vm_time_correlation; + bool vm_tm_corr_dry_run; + char *vm_tm_corr_args; + unsigned int callchain_sz; + unsigned int last_branch_sz; + unsigned long long period; + enum itrace_period_type period_type; + unsigned long initial_skip; + unsigned long *cpu_bitmap; + struct perf_time_interval *ptime_range; + int range_num; + unsigned int error_plus_flags; + unsigned int error_minus_flags; + unsigned int log_plus_flags; + unsigned int log_minus_flags; + unsigned int quick; + unsigned int log_on_error_size; +}; + +/** + * struct auxtrace_index_entry - indexes a AUX area tracing event within a + * perf.data file. + * @file_offset: offset within the perf.data file + * @sz: size of the event + */ +struct auxtrace_index_entry { + u64 file_offset; + u64 sz; +}; + +#define PERF_AUXTRACE_INDEX_ENTRY_COUNT 256 + +/** + * struct auxtrace_index - index of AUX area tracing events within a perf.data + * file. + * @list: linking a number of arrays of entries + * @nr: number of entries + * @entries: array of entries + */ +struct auxtrace_index { + struct list_head list; + size_t nr; + struct auxtrace_index_entry entries[PERF_AUXTRACE_INDEX_ENTRY_COUNT]; +}; + +/** + * struct auxtrace - session callbacks to allow AUX area data decoding. + * @process_event: lets the decoder see all session events + * @process_auxtrace_event: process a PERF_RECORD_AUXTRACE event + * @queue_data: queue an AUX sample or PERF_RECORD_AUXTRACE event for later + * processing + * @dump_auxtrace_sample: dump AUX area sample data + * @flush_events: process any remaining data + * @free_events: free resources associated with event processing + * @free: free resources associated with the session + */ +struct auxtrace { + int (*process_event)(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool); + int (*process_auxtrace_event)(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool); + int (*queue_data)(struct perf_session *session, + struct perf_sample *sample, union perf_event *event, + u64 data_offset); + void (*dump_auxtrace_sample)(struct perf_session *session, + struct perf_sample *sample); + int (*flush_events)(struct perf_session *session, + const struct perf_tool *tool); + void (*free_events)(struct perf_session *session); + void (*free)(struct perf_session *session); + bool (*evsel_is_auxtrace)(struct perf_session *session, + struct evsel *evsel); +}; + +/** + * struct auxtrace_buffer - a buffer containing AUX area tracing data. + * @list: buffers are queued in a list held by struct auxtrace_queue + * @size: size of the buffer in bytes + * @pid: in per-thread mode, the pid this buffer is associated with + * @tid: in per-thread mode, the tid this buffer is associated with + * @cpu: in per-cpu mode, the cpu this buffer is associated with + * @data: actual buffer data (can be null if the data has not been loaded) + * @data_offset: file offset at which the buffer can be read + * @mmap_addr: mmap address at which the buffer can be read + * @mmap_size: size of the mmap at @mmap_addr + * @data_needs_freeing: @data was malloc'd so free it when it is no longer + * needed + * @consecutive: the original data was split up and this buffer is consecutive + * to the previous buffer + * @offset: offset as determined by aux_head / aux_tail members of struct + * perf_event_mmap_page + * @reference: an implementation-specific reference determined when the data is + * recorded + * @buffer_nr: used to number each buffer + * @use_size: implementation actually only uses this number of bytes + * @use_data: implementation actually only uses data starting at this address + */ +struct auxtrace_buffer { + struct list_head list; + size_t size; + pid_t pid; + pid_t tid; + struct perf_cpu cpu; + void *data; + off_t data_offset; + void *mmap_addr; + size_t mmap_size; + bool data_needs_freeing; + bool consecutive; + u64 offset; + u64 reference; + u64 buffer_nr; + size_t use_size; + void *use_data; +}; + +/** + * struct auxtrace_queue - a queue of AUX area tracing data buffers. + * @head: head of buffer list + * @tid: in per-thread mode, the tid this queue is associated with + * @cpu: in per-cpu mode, the cpu this queue is associated with + * @set: %true once this queue has been dedicated to a specific thread or cpu + * @priv: implementation-specific data + */ +struct auxtrace_queue { + struct list_head head; + pid_t tid; + int cpu; + bool set; + void *priv; +}; + +/** + * struct auxtrace_queues - an array of AUX area tracing queues. + * @queue_array: array of queues + * @nr_queues: number of queues + * @new_data: set whenever new data is queued + * @populated: queues have been fully populated using the auxtrace_index + * @next_buffer_nr: used to number each buffer + */ +struct auxtrace_queues { + struct auxtrace_queue *queue_array; + unsigned int nr_queues; + bool new_data; + bool populated; + u64 next_buffer_nr; +}; + +/** + * struct auxtrace_heap_item - element of struct auxtrace_heap. + * @queue_nr: queue number + * @ordinal: value used for sorting (lowest ordinal is top of the heap) expected + * to be a timestamp + */ +struct auxtrace_heap_item { + unsigned int queue_nr; + u64 ordinal; +}; + +/** + * struct auxtrace_heap - a heap suitable for sorting AUX area tracing queues. + * @heap_array: the heap + * @heap_cnt: the number of elements in the heap + * @heap_sz: maximum number of elements (grows as needed) + */ +struct auxtrace_heap { + struct auxtrace_heap_item *heap_array; + unsigned int heap_cnt; + unsigned int heap_sz; +}; + +/** + * struct auxtrace_mmap - records an mmap of the auxtrace buffer. + * @base: address of mapped area + * @userpg: pointer to buffer's perf_event_mmap_page + * @mask: %0 if @len is not a power of two, otherwise (@len - %1) + * @len: size of mapped area + * @prev: previous aux_head + * @idx: index of this mmap + * @tid: tid for a per-thread mmap (also set if there is only 1 tid on a per-cpu + * mmap) otherwise %0 + * @cpu: cpu number for a per-cpu mmap otherwise %-1 + */ +struct auxtrace_mmap { + void *base; + void *userpg; + size_t mask; + size_t len; + u64 prev; + int idx; + pid_t tid; + int cpu; +}; + +/** + * struct auxtrace_mmap_params - parameters to set up struct auxtrace_mmap. + * @mask: %0 if @len is not a power of two, otherwise (@len - %1) + * @offset: file offset of mapped area + * @len: size of mapped area + * @prot: mmap memory protection + * @idx: index of this mmap + * @tid: tid for a per-thread mmap (also set if there is only 1 tid on a per-cpu + * mmap) otherwise %0 + * @mmap_needed: set to %false for non-auxtrace events. This is needed because + * auxtrace mmapping is done in the same code path as non-auxtrace + * mmapping but not every evsel that needs non-auxtrace mmapping + * also needs auxtrace mmapping. + * @cpu: cpu number for a per-cpu mmap otherwise %-1 + */ +struct auxtrace_mmap_params { + size_t mask; + off_t offset; + size_t len; + int prot; + int idx; + pid_t tid; + bool mmap_needed; + struct perf_cpu cpu; +}; + +/** + * struct auxtrace_record - callbacks for recording AUX area data. + * @recording_options: validate and process recording options + * @info_priv_size: return the size of the private data in auxtrace_info_event + * @info_fill: fill-in the private data in auxtrace_info_event + * @free: free this auxtrace record structure + * @snapshot_start: starting a snapshot + * @snapshot_finish: finishing a snapshot + * @find_snapshot: find data to snapshot within auxtrace mmap + * @parse_snapshot_options: parse snapshot options + * @reference: provide a 64-bit reference number for auxtrace_event + * @read_finish: called after reading from an auxtrace mmap + * @alignment: alignment (if any) for AUX area data + * @default_aux_sample_size: default sample size for --aux sample option + * @pmu: associated pmu + * @evlist: selected events list + */ +struct auxtrace_record { + int (*recording_options)(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts); + size_t (*info_priv_size)(struct auxtrace_record *itr, + struct evlist *evlist); + int (*info_fill)(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size); + void (*free)(struct auxtrace_record *itr); + int (*snapshot_start)(struct auxtrace_record *itr); + int (*snapshot_finish)(struct auxtrace_record *itr); + int (*find_snapshot)(struct auxtrace_record *itr, int idx, + struct auxtrace_mmap *mm, unsigned char *data, + u64 *head, u64 *old); + int (*parse_snapshot_options)(struct auxtrace_record *itr, + struct record_opts *opts, + const char *str); + u64 (*reference)(struct auxtrace_record *itr); + int (*read_finish)(struct auxtrace_record *itr, int idx); + unsigned int alignment; + unsigned int default_aux_sample_size; + struct evlist *evlist; +}; + +/** + * struct addr_filter - address filter. + * @list: list node + * @range: true if it is a range filter + * @start: true if action is 'filter' or 'start' + * @action: 'filter', 'start' or 'stop' ('tracestop' is accepted but converted + * to 'stop') + * @sym_from: symbol name for the filter address + * @sym_to: symbol name that determines the filter size + * @sym_from_idx: selects n'th from symbols with the same name (0 means global + * and less than 0 means symbol must be unique) + * @sym_to_idx: same as @sym_from_idx but for @sym_to + * @addr: filter address + * @size: filter region size (for range filters) + * @filename: DSO file name or NULL for the kernel + * @str: allocated string that contains the other string members + */ +struct addr_filter { + struct list_head list; + bool range; + bool start; + const char *action; + const char *sym_from; + const char *sym_to; + int sym_from_idx; + int sym_to_idx; + u64 addr; + u64 size; + const char *filename; + char *str; +}; + +/** + * struct addr_filters - list of address filters. + * @head: list of address filters + * @cnt: number of address filters + */ +struct addr_filters { + struct list_head head; + int cnt; +}; + +struct auxtrace_cache; + +u64 compat_auxtrace_mmap__read_head(struct auxtrace_mmap *mm); +int compat_auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail); + +static inline u64 auxtrace_mmap__read_head(struct auxtrace_mmap *mm, + int kernel_is_64_bit __maybe_unused) +{ + struct perf_event_mmap_page *pc = mm->userpg; + u64 head; + +#if BITS_PER_LONG == 32 + if (kernel_is_64_bit) + return compat_auxtrace_mmap__read_head(mm); +#endif + head = READ_ONCE(pc->aux_head); + + /* Ensure all reads are done after we read the head */ + smp_rmb(); + return head; +} + +static inline int auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail, + int kernel_is_64_bit __maybe_unused) +{ + struct perf_event_mmap_page *pc = mm->userpg; + +#if BITS_PER_LONG == 32 + if (kernel_is_64_bit) + return compat_auxtrace_mmap__write_tail(mm, tail); +#endif + /* Ensure all reads are done before we write the tail out */ + smp_mb(); + WRITE_ONCE(pc->aux_tail, tail); + return 0; +} + +int auxtrace_mmap__mmap(struct auxtrace_mmap *mm, + struct auxtrace_mmap_params *mp, + void *userpg, int fd); +void auxtrace_mmap__munmap(struct auxtrace_mmap *mm); +void auxtrace_mmap_params__init(struct auxtrace_mmap_params *mp, + off_t auxtrace_offset, + unsigned int auxtrace_pages, + bool auxtrace_overwrite); +void auxtrace_mmap_params__set_idx(struct auxtrace_mmap_params *mp, + struct evlist *evlist, + struct evsel *evsel, int idx); + +typedef int (*process_auxtrace_t)(const struct perf_tool *tool, + struct mmap *map, + union perf_event *event, void *data1, + size_t len1, void *data2, size_t len2); + +int auxtrace_mmap__read(struct mmap *map, struct auxtrace_record *itr, + struct perf_env *env, const struct perf_tool *tool, + process_auxtrace_t fn); + +int auxtrace_mmap__read_snapshot(struct mmap *map, + struct auxtrace_record *itr, struct perf_env *env, + const struct perf_tool *tool, process_auxtrace_t fn, + size_t snapshot_size); + +int auxtrace_queues__init_nr(struct auxtrace_queues *queues, int nr_queues); +int auxtrace_queues__init(struct auxtrace_queues *queues); +int auxtrace_queues__add_event(struct auxtrace_queues *queues, + struct perf_session *session, + union perf_event *event, off_t data_offset, + struct auxtrace_buffer **buffer_ptr); +struct auxtrace_queue * +auxtrace_queues__sample_queue(struct auxtrace_queues *queues, + struct perf_sample *sample, + struct perf_session *session); +int auxtrace_queues__add_sample(struct auxtrace_queues *queues, + struct perf_session *session, + struct perf_sample *sample, u64 data_offset, + u64 reference); +void auxtrace_queues__free(struct auxtrace_queues *queues); +int auxtrace_queues__process_index(struct auxtrace_queues *queues, + struct perf_session *session); +int auxtrace_queue_data(struct perf_session *session, bool samples, + bool events); +struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue, + struct auxtrace_buffer *buffer); +void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw); +static inline void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd) +{ + return auxtrace_buffer__get_data_rw(buffer, fd, false); +} +void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer); +void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer); +void auxtrace_buffer__free(struct auxtrace_buffer *buffer); + +int auxtrace_heap__add(struct auxtrace_heap *heap, unsigned int queue_nr, + u64 ordinal); +void auxtrace_heap__pop(struct auxtrace_heap *heap); +void auxtrace_heap__free(struct auxtrace_heap *heap); + +struct auxtrace_cache_entry { + struct hlist_node hash; + u32 key; +}; + +struct auxtrace_cache *auxtrace_cache__new(unsigned int bits, size_t entry_size, + unsigned int limit_percent); +void auxtrace_cache__free(struct auxtrace_cache *auxtrace_cache); +void *auxtrace_cache__alloc_entry(struct auxtrace_cache *c); +void auxtrace_cache__free_entry(struct auxtrace_cache *c, void *entry); +int auxtrace_cache__add(struct auxtrace_cache *c, u32 key, + struct auxtrace_cache_entry *entry); +void auxtrace_cache__remove(struct auxtrace_cache *c, u32 key); +void *auxtrace_cache__lookup(struct auxtrace_cache *c, u32 key); + +struct auxtrace_record *auxtrace_record__init(struct evlist *evlist, + int *err); + +int auxtrace_parse_snapshot_options(struct auxtrace_record *itr, + struct record_opts *opts, + const char *str); +int auxtrace_parse_sample_options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts, const char *str); +int auxtrace_parse_aux_action(struct evlist *evlist); +int auxtrace_record__options(struct auxtrace_record *itr, + struct evlist *evlist, + struct record_opts *opts); +size_t auxtrace_record__info_priv_size(struct auxtrace_record *itr, + struct evlist *evlist); +int auxtrace_record__info_fill(struct auxtrace_record *itr, + struct perf_session *session, + struct perf_record_auxtrace_info *auxtrace_info, + size_t priv_size); +void auxtrace_record__free(struct auxtrace_record *itr); +int auxtrace_record__snapshot_start(struct auxtrace_record *itr); +int auxtrace_record__snapshot_finish(struct auxtrace_record *itr, bool on_exit); +int auxtrace_record__find_snapshot(struct auxtrace_record *itr, int idx, + struct auxtrace_mmap *mm, + unsigned char *data, u64 *head, u64 *old); +u64 auxtrace_record__reference(struct auxtrace_record *itr); +int auxtrace_record__read_finish(struct auxtrace_record *itr, int idx); + +int auxtrace_index__auxtrace_event(struct list_head *head, union perf_event *event, + off_t file_offset); +int auxtrace_index__write(int fd, struct list_head *head); +int auxtrace_index__process(int fd, u64 size, struct perf_session *session, + bool needs_swap); +void auxtrace_index__free(struct list_head *head); + +void auxtrace_synth_guest_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp, + pid_t machine_pid, int vcpu); +void auxtrace_synth_error(struct perf_record_auxtrace_error *auxtrace_error, int type, + int code, int cpu, pid_t pid, pid_t tid, u64 ip, + const char *msg, u64 timestamp); + +int perf_event__process_auxtrace_info(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +s64 perf_event__process_auxtrace(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +int perf_event__process_auxtrace_error(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, + const char *str, int unset); +int itrace_parse_synth_opts(const struct option *opt, const char *str, + int unset); +void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, + bool no_sample); + +size_t perf_event__fprintf_auxtrace_error(union perf_event *event, FILE *fp); +void perf_session__auxtrace_error_inc(struct perf_session *session, + union perf_event *event); +void events_stats__auxtrace_error_warn(const struct events_stats *stats); + +void addr_filters__init(struct addr_filters *filts); +void addr_filters__exit(struct addr_filters *filts); +int addr_filters__parse_bare_filter(struct addr_filters *filts, + const char *filter); +int auxtrace_parse_filters(struct evlist *evlist); + +int auxtrace__process_event(struct perf_session *session, union perf_event *event, + struct perf_sample *sample, const struct perf_tool *tool); +void auxtrace__dump_auxtrace_sample(struct perf_session *session, + struct perf_sample *sample); +int auxtrace__flush_events(struct perf_session *session, const struct perf_tool *tool); +void auxtrace__free_events(struct perf_session *session); +void auxtrace__free(struct perf_session *session); +bool auxtrace__evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel); +u64 auxtrace_synth_id_range_start(struct evsel *evsel); + +#define ITRACE_HELP \ +" i[period]: synthesize instructions events\n" \ +" y[period]: synthesize cycles events (same period as i)\n" \ +" b: synthesize branches events\n" \ +" c: synthesize branches events (calls only)\n" \ +" r: synthesize branches events (returns only)\n" \ +" x: synthesize transactions events\n" \ +" w: synthesize ptwrite events\n" \ +" p: synthesize power events\n" \ +" o: synthesize other events recorded due to the use\n" \ +" of aux-output (refer to perf record)\n" \ +" I: synthesize interrupt or similar (asynchronous) events\n" \ +" (e.g. Intel PT Event Trace)\n" \ +" e[flags]: synthesize error events\n" \ +" each flag must be preceded by + or -\n" \ +" error flags are: o (overflow)\n" \ +" l (data lost)\n" \ +" d[flags]: create a debug log\n" \ +" each flag must be preceded by + or -\n" \ +" log flags are: a (all perf events)\n" \ +" o (output to stdout)\n" \ +" f: synthesize first level cache events\n" \ +" m: synthesize last level cache events\n" \ +" t: synthesize TLB events\n" \ +" a: synthesize remote access events\n" \ +" g[len]: synthesize a call chain (use with i or x)\n" \ +" G[len]: synthesize a call chain on existing event records\n" \ +" l[len]: synthesize last branch entries (use with i or x)\n" \ +" L[len]: synthesize last branch entries on existing event records\n" \ +" sNUMBER: skip initial number of events\n" \ +" q: quicker (less detailed) decoding\n" \ +" A: approximate IPC\n" \ +" Z: prefer to ignore timestamps (so-called \"timeless\" decoding)\n" \ +" T: use the timestamp trace as kernel time\n" \ +" PERIOD[ns|us|ms|i|t]: specify period to sample stream\n" \ +" concatenate multiple options. Default is iybxwpe or cewp\n" + +static inline +void itrace_synth_opts__set_time_range(struct itrace_synth_opts *opts, + struct perf_time_interval *ptime_range, + int range_num) +{ + opts->ptime_range = ptime_range; + opts->range_num = range_num; +} + +static inline +void itrace_synth_opts__clear_time_range(struct itrace_synth_opts *opts) +{ + opts->ptime_range = NULL; + opts->range_num = 0; +} + +#endif diff --git a/tools/perf/util/blake2s.c b/tools/perf/util/blake2s.c new file mode 100644 index 000000000..ce5d89a19 --- /dev/null +++ b/tools/perf/util/blake2s.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 OR MIT +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + * + * This is an implementation of the BLAKE2s hash and PRF functions. + * + * Information: https://blake2.net/ + */ + +#include "blake2s.h" +#include + +static inline u32 ror32(u32 v, int n) +{ + return (v >> n) | (v << (32 - n)); +} + +static inline void le32_to_cpu_array(u32 a[], size_t n) +{ + for (size_t i = 0; i < n; i++) + a[i] = le32_to_cpu((__force __le32)a[i]); +} + +static inline void cpu_to_le32_array(u32 a[], size_t n) +{ + for (size_t i = 0; i < n; i++) + a[i] = (__force u32)cpu_to_le32(a[i]); +} + +static const u8 blake2s_sigma[10][16] = { + { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 }, + { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 }, + { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 }, + { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 }, + { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 }, + { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 }, + { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 }, + { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 }, + { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 }, + { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13, 0 }, +}; + +static inline void blake2s_increment_counter(struct blake2s_ctx *ctx, u32 inc) +{ + ctx->t[0] += inc; + ctx->t[1] += (ctx->t[0] < inc); +} + +static void blake2s_compress(struct blake2s_ctx *ctx, + const u8 *data, size_t nblocks, u32 inc) +{ + u32 m[16]; + u32 v[16]; + int i; + + while (nblocks > 0) { + blake2s_increment_counter(ctx, inc); + memcpy(m, data, BLAKE2S_BLOCK_SIZE); + le32_to_cpu_array(m, ARRAY_SIZE(m)); + memcpy(v, ctx->h, 32); + v[ 8] = BLAKE2S_IV0; + v[ 9] = BLAKE2S_IV1; + v[10] = BLAKE2S_IV2; + v[11] = BLAKE2S_IV3; + v[12] = BLAKE2S_IV4 ^ ctx->t[0]; + v[13] = BLAKE2S_IV5 ^ ctx->t[1]; + v[14] = BLAKE2S_IV6 ^ ctx->f[0]; + v[15] = BLAKE2S_IV7 ^ ctx->f[1]; + +#define G(r, i, a, b, c, d) do { \ + a += b + m[blake2s_sigma[r][2 * i + 0]]; \ + d = ror32(d ^ a, 16); \ + c += d; \ + b = ror32(b ^ c, 12); \ + a += b + m[blake2s_sigma[r][2 * i + 1]]; \ + d = ror32(d ^ a, 8); \ + c += d; \ + b = ror32(b ^ c, 7); \ +} while (0) + +#define ROUND(r) do { \ + G(r, 0, v[0], v[ 4], v[ 8], v[12]); \ + G(r, 1, v[1], v[ 5], v[ 9], v[13]); \ + G(r, 2, v[2], v[ 6], v[10], v[14]); \ + G(r, 3, v[3], v[ 7], v[11], v[15]); \ + G(r, 4, v[0], v[ 5], v[10], v[15]); \ + G(r, 5, v[1], v[ 6], v[11], v[12]); \ + G(r, 6, v[2], v[ 7], v[ 8], v[13]); \ + G(r, 7, v[3], v[ 4], v[ 9], v[14]); \ +} while (0) + ROUND(0); + ROUND(1); + ROUND(2); + ROUND(3); + ROUND(4); + ROUND(5); + ROUND(6); + ROUND(7); + ROUND(8); + ROUND(9); + +#undef G +#undef ROUND + + for (i = 0; i < 8; ++i) + ctx->h[i] ^= v[i] ^ v[i + 8]; + + data += BLAKE2S_BLOCK_SIZE; + --nblocks; + } +} + +static inline void blake2s_set_lastblock(struct blake2s_ctx *ctx) +{ + ctx->f[0] = -1; +} + +void blake2s_update(struct blake2s_ctx *ctx, const u8 *in, size_t inlen) +{ + const size_t fill = BLAKE2S_BLOCK_SIZE - ctx->buflen; + + if (unlikely(!inlen)) + return; + if (inlen > fill) { + memcpy(ctx->buf + ctx->buflen, in, fill); + blake2s_compress(ctx, ctx->buf, 1, BLAKE2S_BLOCK_SIZE); + ctx->buflen = 0; + in += fill; + inlen -= fill; + } + if (inlen > BLAKE2S_BLOCK_SIZE) { + const size_t nblocks = DIV_ROUND_UP(inlen, BLAKE2S_BLOCK_SIZE); + + blake2s_compress(ctx, in, nblocks - 1, BLAKE2S_BLOCK_SIZE); + in += BLAKE2S_BLOCK_SIZE * (nblocks - 1); + inlen -= BLAKE2S_BLOCK_SIZE * (nblocks - 1); + } + memcpy(ctx->buf + ctx->buflen, in, inlen); + ctx->buflen += inlen; +} + +void blake2s_final(struct blake2s_ctx *ctx, u8 *out) +{ + blake2s_set_lastblock(ctx); + memset(ctx->buf + ctx->buflen, 0, + BLAKE2S_BLOCK_SIZE - ctx->buflen); /* Padding */ + blake2s_compress(ctx, ctx->buf, 1, ctx->buflen); + cpu_to_le32_array(ctx->h, ARRAY_SIZE(ctx->h)); + memcpy(out, ctx->h, ctx->outlen); + memset(ctx, 0, sizeof(*ctx)); +} diff --git a/tools/perf/util/blake2s.h b/tools/perf/util/blake2s.h new file mode 100644 index 000000000..a1fe81a4b --- /dev/null +++ b/tools/perf/util/blake2s.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 OR MIT */ +/* + * Copyright (C) 2015-2019 Jason A. Donenfeld . All Rights Reserved. + */ + +#ifndef _CRYPTO_BLAKE2S_H +#define _CRYPTO_BLAKE2S_H + +#include +#include + +#define BLAKE2S_BLOCK_SIZE 64 + +struct blake2s_ctx { + u32 h[8]; + u32 t[2]; + u32 f[2]; + u8 buf[BLAKE2S_BLOCK_SIZE]; + unsigned int buflen; + unsigned int outlen; +}; + +enum blake2s_iv { + BLAKE2S_IV0 = 0x6A09E667UL, + BLAKE2S_IV1 = 0xBB67AE85UL, + BLAKE2S_IV2 = 0x3C6EF372UL, + BLAKE2S_IV3 = 0xA54FF53AUL, + BLAKE2S_IV4 = 0x510E527FUL, + BLAKE2S_IV5 = 0x9B05688CUL, + BLAKE2S_IV6 = 0x1F83D9ABUL, + BLAKE2S_IV7 = 0x5BE0CD19UL, +}; + +static inline void __blake2s_init(struct blake2s_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + ctx->h[0] = BLAKE2S_IV0 ^ (0x01010000 | keylen << 8 | outlen); + ctx->h[1] = BLAKE2S_IV1; + ctx->h[2] = BLAKE2S_IV2; + ctx->h[3] = BLAKE2S_IV3; + ctx->h[4] = BLAKE2S_IV4; + ctx->h[5] = BLAKE2S_IV5; + ctx->h[6] = BLAKE2S_IV6; + ctx->h[7] = BLAKE2S_IV7; + ctx->t[0] = 0; + ctx->t[1] = 0; + ctx->f[0] = 0; + ctx->f[1] = 0; + ctx->buflen = 0; + ctx->outlen = outlen; + if (keylen) { + memcpy(ctx->buf, key, keylen); + memset(&ctx->buf[keylen], 0, BLAKE2S_BLOCK_SIZE - keylen); + ctx->buflen = BLAKE2S_BLOCK_SIZE; + } +} + +static inline void blake2s_init(struct blake2s_ctx *ctx, size_t outlen) +{ + __blake2s_init(ctx, outlen, NULL, 0); +} + +static inline void blake2s_init_key(struct blake2s_ctx *ctx, size_t outlen, + const void *key, size_t keylen) +{ + __blake2s_init(ctx, outlen, key, keylen); +} + +void blake2s_update(struct blake2s_ctx *ctx, const u8 *in, size_t inlen); + +void blake2s_final(struct blake2s_ctx *ctx, u8 *out); + +#endif /* _CRYPTO_BLAKE2S_H */ diff --git a/tools/perf/util/block-info.c b/tools/perf/util/block-info.c new file mode 100644 index 000000000..8d3a9a661 --- /dev/null +++ b/tools/perf/util/block-info.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "block-info.h" +#include "sort.h" +#include "annotate.h" +#include "symbol.h" +#include "dso.h" +#include "map.h" +#include "srcline.h" +#include "evlist.h" +#include "hist.h" +#include "ui/browsers/hists.h" + +static struct block_header_column { + const char *name; + int width; +} block_columns[PERF_HPP_REPORT__BLOCK_MAX_INDEX] = { + [PERF_HPP_REPORT__BLOCK_TOTAL_CYCLES_PCT] = { + .name = "Sampled Cycles%", + .width = 15, + }, + [PERF_HPP_REPORT__BLOCK_LBR_CYCLES] = { + .name = "Sampled Cycles", + .width = 14, + }, + [PERF_HPP_REPORT__BLOCK_CYCLES_PCT] = { + .name = "Avg Cycles%", + .width = 11, + }, + [PERF_HPP_REPORT__BLOCK_AVG_CYCLES] = { + .name = "Avg Cycles", + .width = 10, + }, + [PERF_HPP_REPORT__BLOCK_RANGE] = { + .name = "[Program Block Range]", + .width = 70, + }, + [PERF_HPP_REPORT__BLOCK_DSO] = { + .name = "Shared Object", + .width = 20, + }, + [PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER] = { + .name = "Branch Counter", + .width = 30, + } +}; + +static struct block_info *block_info__new(unsigned int br_cntr_nr) +{ + struct block_info *bi = zalloc(sizeof(struct block_info)); + + if (bi && br_cntr_nr) { + bi->br_cntr = calloc(br_cntr_nr, sizeof(u64)); + if (!bi->br_cntr) { + free(bi); + return NULL; + } + } + + return bi; +} + +void block_info__delete(struct block_info *bi) +{ + if (bi) + free(bi->br_cntr); + free(bi); +} + +int64_t __block_info__cmp(struct hist_entry *left, struct hist_entry *right) +{ + struct block_info *bi_l = left->block_info; + struct block_info *bi_r = right->block_info; + int cmp; + + if (!bi_l->sym || !bi_r->sym) { + if (!bi_l->sym && !bi_r->sym) + return -1; + else if (!bi_l->sym) + return -1; + else + return 1; + } + + cmp = strcmp(bi_l->sym->name, bi_r->sym->name); + if (cmp) + return cmp; + + if (bi_l->start != bi_r->start) + return (int64_t)(bi_r->start - bi_l->start); + + return (int64_t)(bi_r->end - bi_l->end); +} + +int64_t block_info__cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right) +{ + return __block_info__cmp(left, right); +} + +static void init_block_info(struct block_info *bi, struct symbol *sym, + struct cyc_hist *ch, int offset, + u64 total_cycles, unsigned int br_cntr_nr, + u64 *br_cntr, struct evsel *evsel) +{ + bi->sym = sym; + bi->start = ch->start; + bi->end = offset; + bi->cycles = ch->cycles; + bi->cycles_aggr = ch->cycles_aggr; + bi->num = ch->num; + bi->num_aggr = ch->num_aggr; + bi->total_cycles = total_cycles; + + memcpy(bi->cycles_spark, ch->cycles_spark, + NUM_SPARKS * sizeof(u64)); + + if (br_cntr && br_cntr_nr) { + bi->br_cntr_nr = br_cntr_nr; + memcpy(bi->br_cntr, &br_cntr[offset * br_cntr_nr], + br_cntr_nr * sizeof(u64)); + } + bi->evsel = evsel; +} + +int block_info__process_sym(struct hist_entry *he, struct block_hist *bh, + u64 *block_cycles_aggr, u64 total_cycles, + unsigned int br_cntr_nr) +{ + struct annotation *notes; + struct cyc_hist *ch; + static struct addr_location al; + u64 cycles = 0; + + if (!he->ms.map || !he->ms.sym) + return 0; + + memset(&al, 0, sizeof(al)); + al.map = he->ms.map; + al.sym = he->ms.sym; + + notes = symbol__annotation(he->ms.sym); + if (!notes || !notes->branch || !notes->branch->cycles_hist) + return 0; + ch = notes->branch->cycles_hist; + for (unsigned int i = 0; i < symbol__size(he->ms.sym); i++) { + if (ch[i].num_aggr) { + struct block_info *bi; + struct hist_entry *he_block; + + bi = block_info__new(br_cntr_nr); + if (!bi) + return -1; + + init_block_info(bi, he->ms.sym, &ch[i], i, + total_cycles, br_cntr_nr, + notes->branch->br_cntr, + hists_to_evsel(he->hists)); + cycles += bi->cycles_aggr / bi->num_aggr; + + he_block = hists__add_entry_block(&bh->block_hists, + &al, bi); + if (!he_block) { + block_info__delete(bi); + return -1; + } + } + } + + if (block_cycles_aggr) + *block_cycles_aggr += cycles; + + return 0; +} + +static int block_column_header(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hists *hists __maybe_unused, + int line __maybe_unused, + int *span __maybe_unused) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + block_fmt->header); +} + +static int block_column_width(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp __maybe_unused, + struct hists *hists __maybe_unused) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + + return block_fmt->width; +} + +static int color_pct(struct perf_hpp *hpp, int width, double pct) +{ +#ifdef HAVE_SLANG_SUPPORT + if (use_browser) { + return __hpp__slsmg_color_printf(hpp, "%*.2f%%", + width - 1, pct); + } +#endif + return hpp_color_scnprintf(hpp, "%*.2f%%", width - 1, pct); +} + +static int block_total_cycles_pct_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + double ratio = 0.0; + + if (block_fmt->total_cycles) + ratio = (double)bi->cycles_aggr / (double)block_fmt->total_cycles; + + return color_pct(hpp, block_fmt->width, 100.0 * ratio); +} + +static int64_t block_total_cycles_pct_sort(struct perf_hpp_fmt *fmt, + struct hist_entry *left, + struct hist_entry *right) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi_l = left->block_info; + struct block_info *bi_r = right->block_info; + double l, r; + + if (block_fmt->total_cycles) { + l = ((double)bi_l->cycles_aggr / + (double)block_fmt->total_cycles) * 100000.0; + r = ((double)bi_r->cycles_aggr / + (double)block_fmt->total_cycles) * 100000.0; + return (int64_t)l - (int64_t)r; + } + + return 0; +} + +static void cycles_string(u64 cycles, char *buf, int size) +{ + if (cycles >= 1000000) + scnprintf(buf, size, "%.1fM", (double)cycles / 1000000.0); + else if (cycles >= 1000) + scnprintf(buf, size, "%.1fK", (double)cycles / 1000.0); + else + scnprintf(buf, size, "%1d", cycles); +} + +static int block_cycles_lbr_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char cycles_buf[16]; + + cycles_string(bi->cycles_aggr, cycles_buf, sizeof(cycles_buf)); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + cycles_buf); +} + +static int block_cycles_pct_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + double ratio = 0.0; + u64 avg; + + if (block_fmt->block_cycles && bi->num_aggr) { + avg = bi->cycles_aggr / bi->num_aggr; + ratio = (double)avg / (double)block_fmt->block_cycles; + } + + return color_pct(hpp, block_fmt->width, 100.0 * ratio); +} + +static int block_avg_cycles_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char cycles_buf[16]; + + cycles_string(bi->cycles_aggr / bi->num_aggr, cycles_buf, + sizeof(cycles_buf)); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + cycles_buf); +} + +static int block_range_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char buf[128]; + char *start_line, *end_line; + + symbol_conf.addr2line_disable_warn = true; + + start_line = map__srcline(he->ms.map, bi->sym->start + bi->start, + he->ms.sym); + + end_line = map__srcline(he->ms.map, bi->sym->start + bi->end, + he->ms.sym); + + if (start_line != SRCLINE_UNKNOWN && + end_line != SRCLINE_UNKNOWN) { + scnprintf(buf, sizeof(buf), "[%s -> %s]", + start_line, end_line); + } else { + scnprintf(buf, sizeof(buf), "[%7lx -> %7lx]", + bi->start, bi->end); + } + + zfree_srcline(&start_line); + zfree_srcline(&end_line); + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, buf); +} + +static int block_dso_entry(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct map *map = he->ms.map; + + if (map && map__dso(map)) { + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + dso__short_name(map__dso(map))); + } + + return scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, + "[unknown]"); +} + +static void init_block_header(struct block_fmt *block_fmt) +{ + struct perf_hpp_fmt *fmt = &block_fmt->fmt; + + BUG_ON(block_fmt->idx >= PERF_HPP_REPORT__BLOCK_MAX_INDEX); + + block_fmt->header = block_columns[block_fmt->idx].name; + block_fmt->width = block_columns[block_fmt->idx].width; + + fmt->header = block_column_header; + fmt->width = block_column_width; +} + +static int block_branch_counter_entry(struct perf_hpp_fmt *fmt, + struct perf_hpp *hpp, + struct hist_entry *he) +{ + struct block_fmt *block_fmt = container_of(fmt, struct block_fmt, fmt); + struct block_info *bi = he->block_info; + char *buf; + int ret; + + if (annotation_br_cntr_entry(&buf, bi->br_cntr_nr, bi->br_cntr, + bi->num_aggr, bi->evsel)) + return 0; + + ret = scnprintf(hpp->buf, hpp->size, "%*s", block_fmt->width, buf); + free(buf); + return ret; +} + +static void hpp_register(struct block_fmt *block_fmt, int idx, + struct perf_hpp_list *hpp_list) +{ + struct perf_hpp_fmt *fmt = &block_fmt->fmt; + + block_fmt->idx = idx; + INIT_LIST_HEAD(&fmt->list); + INIT_LIST_HEAD(&fmt->sort_list); + + switch (idx) { + case PERF_HPP_REPORT__BLOCK_TOTAL_CYCLES_PCT: + fmt->color = block_total_cycles_pct_entry; + fmt->cmp = block_info__cmp; + fmt->sort = block_total_cycles_pct_sort; + break; + case PERF_HPP_REPORT__BLOCK_LBR_CYCLES: + fmt->entry = block_cycles_lbr_entry; + break; + case PERF_HPP_REPORT__BLOCK_CYCLES_PCT: + fmt->color = block_cycles_pct_entry; + break; + case PERF_HPP_REPORT__BLOCK_AVG_CYCLES: + fmt->entry = block_avg_cycles_entry; + break; + case PERF_HPP_REPORT__BLOCK_RANGE: + fmt->entry = block_range_entry; + break; + case PERF_HPP_REPORT__BLOCK_DSO: + fmt->entry = block_dso_entry; + break; + case PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER: + fmt->entry = block_branch_counter_entry; + break; + default: + return; + } + + init_block_header(block_fmt); + perf_hpp_list__column_register(hpp_list, fmt); +} + +static void register_block_columns(struct perf_hpp_list *hpp_list, + struct block_fmt *block_fmts, + int *block_hpps, int nr_hpps) +{ + for (int i = 0; i < nr_hpps; i++) + hpp_register(&block_fmts[i], block_hpps[i], hpp_list); +} + +static void init_block_hist(struct block_hist *bh, struct block_fmt *block_fmts, + int *block_hpps, int nr_hpps) +{ + __hists__init(&bh->block_hists, &bh->block_list); + perf_hpp_list__init(&bh->block_list); + bh->block_list.nr_header_lines = 1; + + register_block_columns(&bh->block_list, block_fmts, + block_hpps, nr_hpps); + + /* Sort by the first fmt */ + perf_hpp_list__register_sort_field(&bh->block_list, &block_fmts[0].fmt); +} + +static int process_block_report(struct hists *hists, + struct block_report *block_report, + u64 total_cycles, int *block_hpps, + int nr_hpps, unsigned int br_cntr_nr) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct block_hist *bh = &block_report->hist; + struct hist_entry *he; + + if (nr_hpps > PERF_HPP_REPORT__BLOCK_MAX_INDEX) + return -1; + + block_report->nr_fmts = nr_hpps; + init_block_hist(bh, block_report->fmts, block_hpps, nr_hpps); + + while (next) { + he = rb_entry(next, struct hist_entry, rb_node); + block_info__process_sym(he, bh, &block_report->cycles, + total_cycles, br_cntr_nr); + next = rb_next(&he->rb_node); + } + + for (int i = 0; i < nr_hpps; i++) { + block_report->fmts[i].total_cycles = total_cycles; + block_report->fmts[i].block_cycles = block_report->cycles; + } + + hists__output_resort(&bh->block_hists, NULL); + return 0; +} + +struct block_report *block_info__create_report(struct evlist *evlist, + u64 total_cycles, + int *block_hpps, int nr_hpps, + int *nr_reps) +{ + struct block_report *block_reports; + int nr_hists = evlist->core.nr_entries, i = 0; + struct evsel *pos; + + block_reports = calloc(nr_hists, sizeof(struct block_report)); + if (!block_reports) + return NULL; + + evlist__for_each_entry(evlist, pos) { + struct hists *hists = evsel__hists(pos); + + process_block_report(hists, &block_reports[i], total_cycles, + block_hpps, nr_hpps, evlist->nr_br_cntr); + i++; + } + + *nr_reps = nr_hists; + return block_reports; +} + +void block_info__free_report(struct block_report *reps, int nr_reps) +{ + for (int i = 0; i < nr_reps; i++) + hists__delete_entries(&reps[i].hist.block_hists); + + free(reps); +} + +int report__browse_block_hists(struct block_hist *bh, float min_percent, + struct evsel *evsel, struct perf_env *env) +{ + int ret; + + switch (use_browser) { + case 0: + symbol_conf.report_individual_block = true; + hists__fprintf(&bh->block_hists, true, 0, 0, min_percent, + stdout, true); + return 0; + case 1: + symbol_conf.report_individual_block = true; + ret = block_hists_tui_browse(bh, evsel, min_percent, env); + return ret; + default: + return -1; + } + + return 0; +} + +float block_info__total_cycles_percent(struct hist_entry *he) +{ + struct block_info *bi = he->block_info; + + if (bi->total_cycles) + return bi->cycles * 100.0 / bi->total_cycles; + + return 0.0; +} diff --git a/tools/perf/util/block-info.h b/tools/perf/util/block-info.h new file mode 100644 index 000000000..b9329dc3a --- /dev/null +++ b/tools/perf/util/block-info.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BLOCK_H +#define __PERF_BLOCK_H + +#include +#include "hist.h" +#include "symbol.h" +#include "sort.h" +#include "ui/ui.h" + +struct block_info { + struct symbol *sym; + u64 start; + u64 end; + u64 cycles; + u64 cycles_aggr; + s64 cycles_spark[NUM_SPARKS]; + u64 total_cycles; + int num; + int num_aggr; + int br_cntr_nr; + u64 *br_cntr; + struct evsel *evsel; +}; + +struct block_fmt { + struct perf_hpp_fmt fmt; + int idx; + int width; + const char *header; + u64 total_cycles; + u64 block_cycles; +}; + +enum { + PERF_HPP_REPORT__BLOCK_TOTAL_CYCLES_PCT, + PERF_HPP_REPORT__BLOCK_LBR_CYCLES, + PERF_HPP_REPORT__BLOCK_CYCLES_PCT, + PERF_HPP_REPORT__BLOCK_AVG_CYCLES, + PERF_HPP_REPORT__BLOCK_RANGE, + PERF_HPP_REPORT__BLOCK_DSO, + PERF_HPP_REPORT__BLOCK_BRANCH_COUNTER, + PERF_HPP_REPORT__BLOCK_MAX_INDEX +}; + +struct block_report { + struct block_hist hist; + u64 cycles; + struct block_fmt fmts[PERF_HPP_REPORT__BLOCK_MAX_INDEX]; + int nr_fmts; +}; + +void block_info__delete(struct block_info *bi); + +int64_t __block_info__cmp(struct hist_entry *left, struct hist_entry *right); + +int64_t block_info__cmp(struct perf_hpp_fmt *fmt __maybe_unused, + struct hist_entry *left, struct hist_entry *right); + +int block_info__process_sym(struct hist_entry *he, struct block_hist *bh, + u64 *block_cycles_aggr, u64 total_cycles, + unsigned int br_cntr_nr); + +struct block_report *block_info__create_report(struct evlist *evlist, + u64 total_cycles, + int *block_hpps, int nr_hpps, + int *nr_reps); + +void block_info__free_report(struct block_report *reps, int nr_reps); + +int report__browse_block_hists(struct block_hist *bh, float min_percent, + struct evsel *evsel, struct perf_env *env); + +float block_info__total_cycles_percent(struct hist_entry *he); + +#endif /* __PERF_BLOCK_H */ diff --git a/tools/perf/util/block-range.c b/tools/perf/util/block-range.c new file mode 100644 index 000000000..7c559fcfd --- /dev/null +++ b/tools/perf/util/block-range.c @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "block-range.h" +#include "annotate.h" +#include +#include + +static struct { + struct rb_root root; + u64 blocks; +} block_ranges; + +static void block_range__debug(void) +{ +#ifndef NDEBUG + struct rb_node *rb; + u64 old = 0; /* NULL isn't executable */ + + for (rb = rb_first(&block_ranges.root); rb; rb = rb_next(rb)) { + struct block_range *entry = rb_entry(rb, struct block_range, node); + + assert(old < entry->start); + assert(entry->start <= entry->end); /* single instruction block; jump to a jump */ + + old = entry->end; + } +#endif +} + +struct block_range *block_range__find(u64 addr) +{ + struct rb_node **p = &block_ranges.root.rb_node; + struct rb_node *parent = NULL; + struct block_range *entry; + + while (*p != NULL) { + parent = *p; + entry = rb_entry(parent, struct block_range, node); + + if (addr < entry->start) + p = &parent->rb_left; + else if (addr > entry->end) + p = &parent->rb_right; + else + return entry; + } + + return NULL; +} + +static inline void rb_link_left_of_node(struct rb_node *left, struct rb_node *node) +{ + struct rb_node **p = &node->rb_left; + while (*p) { + node = *p; + p = &node->rb_right; + } + rb_link_node(left, node, p); +} + +static inline void rb_link_right_of_node(struct rb_node *right, struct rb_node *node) +{ + struct rb_node **p = &node->rb_right; + while (*p) { + node = *p; + p = &node->rb_left; + } + rb_link_node(right, node, p); +} + +/** + * block_range__create + * @start: branch target starting this basic block + * @end: branch ending this basic block + * + * Create all the required block ranges to precisely span the given range. + */ +struct block_range_iter block_range__create(u64 start, u64 end) +{ + struct rb_node **p = &block_ranges.root.rb_node; + struct rb_node *n, *parent = NULL; + struct block_range *next, *entry = NULL; + struct block_range_iter iter = { NULL, NULL }; + + while (*p != NULL) { + parent = *p; + entry = rb_entry(parent, struct block_range, node); + + if (start < entry->start) + p = &parent->rb_left; + else if (start > entry->end) + p = &parent->rb_right; + else + break; + } + + /* + * Didn't find anything.. there's a hole at @start, however @end might + * be inside/behind the next range. + */ + if (!*p) { + if (!entry) /* tree empty */ + goto do_whole; + + /* + * If the last node is before, advance one to find the next. + */ + n = parent; + if (entry->end < start) { + n = rb_next(n); + if (!n) + goto do_whole; + } + next = rb_entry(n, struct block_range, node); + + if (next->start <= end) { /* add head: [start...][n->start...] */ + struct block_range *head = malloc(sizeof(struct block_range)); + if (!head) + return iter; + + *head = (struct block_range){ + .start = start, + .end = next->start - 1, + .is_target = 1, + .is_branch = 0, + }; + + rb_link_left_of_node(&head->node, &next->node); + rb_insert_color(&head->node, &block_ranges.root); + block_range__debug(); + + iter.start = head; + goto do_tail; + } + +do_whole: + /* + * The whole [start..end] range is non-overlapping. + */ + entry = malloc(sizeof(struct block_range)); + if (!entry) + return iter; + + *entry = (struct block_range){ + .start = start, + .end = end, + .is_target = 1, + .is_branch = 1, + }; + + rb_link_node(&entry->node, parent, p); + rb_insert_color(&entry->node, &block_ranges.root); + block_range__debug(); + + iter.start = entry; + iter.end = entry; + goto done; + } + + /* + * We found a range that overlapped with ours, split if needed. + */ + if (entry->start < start) { /* split: [e->start...][start...] */ + struct block_range *head = malloc(sizeof(struct block_range)); + if (!head) + return iter; + + *head = (struct block_range){ + .start = entry->start, + .end = start - 1, + .is_target = entry->is_target, + .is_branch = 0, + + .coverage = entry->coverage, + .entry = entry->entry, + }; + + entry->start = start; + entry->is_target = 1; + entry->entry = 0; + + rb_link_left_of_node(&head->node, &entry->node); + rb_insert_color(&head->node, &block_ranges.root); + block_range__debug(); + + } else if (entry->start == start) + entry->is_target = 1; + + iter.start = entry; + +do_tail: + /* + * At this point we've got: @iter.start = [@start...] but @end can still be + * inside or beyond it. + */ + entry = iter.start; + for (;;) { + /* + * If @end is inside @entry, split. + */ + if (end < entry->end) { /* split: [...end][...e->end] */ + struct block_range *tail = malloc(sizeof(struct block_range)); + if (!tail) + return iter; + + *tail = (struct block_range){ + .start = end + 1, + .end = entry->end, + .is_target = 0, + .is_branch = entry->is_branch, + + .coverage = entry->coverage, + .taken = entry->taken, + .pred = entry->pred, + }; + + entry->end = end; + entry->is_branch = 1; + entry->taken = 0; + entry->pred = 0; + + rb_link_right_of_node(&tail->node, &entry->node); + rb_insert_color(&tail->node, &block_ranges.root); + block_range__debug(); + + iter.end = entry; + goto done; + } + + /* + * If @end matches @entry, done + */ + if (end == entry->end) { + entry->is_branch = 1; + iter.end = entry; + goto done; + } + + next = block_range__next(entry); + if (!next) + goto add_tail; + + /* + * If @end is in beyond @entry but not inside @next, add tail. + */ + if (end < next->start) { /* add tail: [...e->end][...end] */ + struct block_range *tail; +add_tail: + tail = malloc(sizeof(struct block_range)); + if (!tail) + return iter; + + *tail = (struct block_range){ + .start = entry->end + 1, + .end = end, + .is_target = 0, + .is_branch = 1, + }; + + rb_link_right_of_node(&tail->node, &entry->node); + rb_insert_color(&tail->node, &block_ranges.root); + block_range__debug(); + + iter.end = tail; + goto done; + } + + /* + * If there is a hole between @entry and @next, fill it. + */ + if (entry->end + 1 != next->start) { + struct block_range *hole = malloc(sizeof(struct block_range)); + if (!hole) + return iter; + + *hole = (struct block_range){ + .start = entry->end + 1, + .end = next->start - 1, + .is_target = 0, + .is_branch = 0, + }; + + rb_link_left_of_node(&hole->node, &next->node); + rb_insert_color(&hole->node, &block_ranges.root); + block_range__debug(); + } + + entry = next; + } + +done: + assert(iter.start->start == start && iter.start->is_target); + assert(iter.end->end == end && iter.end->is_branch); + + block_ranges.blocks++; + + return iter; +} + + +/* + * Compute coverage as: + * + * br->coverage / br->sym->max_coverage + * + * This ensures each symbol has a 100% spot, to reflect that each symbol has a + * most covered section. + * + * Returns [0-1] for coverage and -1 if we had no data what so ever or the + * symbol does not exist. + */ +double block_range__coverage(struct block_range *br) +{ + struct symbol *sym; + struct annotated_branch *branch; + + if (!br) { + if (block_ranges.blocks) + return 0; + + return -1; + } + + sym = br->sym; + if (!sym) + return -1; + + branch = symbol__annotation(sym)->branch; + if (!branch) + return -1; + + return (double)br->coverage / branch->max_coverage; +} diff --git a/tools/perf/util/block-range.h b/tools/perf/util/block-range.h new file mode 100644 index 000000000..ec0fb534b --- /dev/null +++ b/tools/perf/util/block-range.h @@ -0,0 +1,76 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BLOCK_RANGE_H +#define __PERF_BLOCK_RANGE_H + +#include +#include +#include + +struct symbol; + +/* + * struct block_range - non-overlapping parts of basic blocks + * @node: treenode + * @start: inclusive start of range + * @end: inclusive end of range + * @is_target: @start is a jump target + * @is_branch: @end is a branch instruction + * @coverage: number of blocks that cover this range + * @taken: number of times the branch is taken (requires @is_branch) + * @pred: number of times the taken branch was predicted + */ +struct block_range { + struct rb_node node; + + struct symbol *sym; + + u64 start; + u64 end; + + int is_target, is_branch; + + u64 coverage; + u64 entry; + u64 taken; + u64 pred; +}; + +static inline struct block_range *block_range__next(struct block_range *br) +{ + struct rb_node *n = rb_next(&br->node); + if (!n) + return NULL; + return rb_entry(n, struct block_range, node); +} + +struct block_range_iter { + struct block_range *start; + struct block_range *end; +}; + +static inline struct block_range *block_range_iter(struct block_range_iter *iter) +{ + return iter->start; +} + +static inline bool block_range_iter__next(struct block_range_iter *iter) +{ + if (iter->start == iter->end) + return false; + + iter->start = block_range__next(iter->start); + return true; +} + +static inline bool block_range_iter__valid(struct block_range_iter *iter) +{ + if (!iter->start || !iter->end) + return false; + return true; +} + +extern struct block_range *block_range__find(u64 addr); +extern struct block_range_iter block_range__create(u64 start, u64 end); +extern double block_range__coverage(struct block_range *br); + +#endif /* __PERF_BLOCK_RANGE_H */ diff --git a/tools/perf/util/bpf-event.c b/tools/perf/util/bpf-event.c new file mode 100644 index 000000000..fa3ebc8ea --- /dev/null +++ b/tools/perf/util/bpf-event.c @@ -0,0 +1,1016 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "bpf-event.h" +#include "bpf-utils.h" +#include "debug.h" +#include "dso.h" +#include "symbol.h" +#include "machine.h" +#include "env.h" +#include "session.h" +#include "map.h" +#include "evlist.h" +#include "record.h" +#include "util/synthetic-events.h" + +static int snprintf_hex(char *buf, size_t size, unsigned char *data, size_t len) +{ + int ret = 0; + size_t i; + + for (i = 0; i < len; i++) + ret += scnprintf(buf + ret, size - ret, "%02x", data[i]); + return ret; +} + +static int machine__process_bpf_event_load(struct machine *machine, + union perf_event *event, + struct perf_sample *sample __maybe_unused) +{ + struct bpf_prog_info_node *info_node; + struct perf_env *env = machine->env; + struct perf_bpil *info_linear; + int id = event->bpf.id; + unsigned int i; + + /* perf-record, no need to handle bpf-event */ + if (env == NULL) + return 0; + + info_node = perf_env__find_bpf_prog_info(env, id); + if (!info_node) + return 0; + info_linear = info_node->info_linear; + + /* jited_ksyms is only valid if bpil_offs_to_addr() converted it */ + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_KSYMS))) + return 0; + + for (i = 0; i < info_linear->info.nr_jited_ksyms; i++) { + u64 *addrs = (u64 *)(uintptr_t)(info_linear->info.jited_ksyms); + u64 addr = addrs[i]; + struct map *map = maps__find(machine__kernel_maps(machine), addr); + + if (map) { + struct dso *dso = map__dso(map); + + dso__set_binary_type(dso, DSO_BINARY_TYPE__BPF_PROG_INFO); + dso__bpf_prog(dso)->id = id; + dso__bpf_prog(dso)->sub_id = i; + dso__bpf_prog(dso)->env = env; + map__put(map); + } + } + return 0; +} + +int machine__process_bpf(struct machine *machine, union perf_event *event, + struct perf_sample *sample) +{ + if (dump_trace) + perf_event__fprintf_bpf(event, stdout); + + switch (event->bpf.type) { + case PERF_BPF_EVENT_PROG_LOAD: + return machine__process_bpf_event_load(machine, event, sample); + + case PERF_BPF_EVENT_PROG_UNLOAD: + /* + * Do not free bpf_prog_info and btf of the program here, + * as annotation still need them. They will be freed at + * the end of the session. + */ + break; + default: + pr_debug("unexpected bpf event type of %d\n", event->bpf.type); + break; + } + return 0; +} + +static int perf_env__fetch_btf(struct perf_env *env, + u32 btf_id, + struct btf *btf) +{ + struct btf_node *node; + u32 data_size; + const void *data; + + data = btf__raw_data(btf, &data_size); + + node = malloc(data_size + sizeof(struct btf_node)); + if (!node) + return -1; + + node->id = btf_id; + node->data_size = data_size; + memcpy(node->data, data, data_size); + + if (!perf_env__insert_btf(env, node)) { + /* Insertion failed because of a duplicate. */ + free(node); + return -1; + } + return 0; +} + +static int synthesize_bpf_prog_name(char *buf, int size, + struct bpf_prog_info *info, + struct btf *btf, + u32 sub_id) +{ + u8 (*prog_tags)[BPF_TAG_SIZE] = (void *)(uintptr_t)(info->prog_tags); + void *func_infos = (void *)(uintptr_t)(info->func_info); + u32 sub_prog_cnt = info->nr_jited_ksyms; + const struct bpf_func_info *finfo; + const char *short_name = NULL; + const struct btf_type *t; + int name_len; + + name_len = scnprintf(buf, size, "bpf_prog_"); + name_len += snprintf_hex(buf + name_len, size - name_len, + prog_tags[sub_id], BPF_TAG_SIZE); + if (btf && + info->func_info_rec_size >= sizeof(*finfo) && + sub_id < info->nr_func_info) { + finfo = func_infos + sub_id * info->func_info_rec_size; + t = btf__type_by_id(btf, finfo->type_id); + if (t) + short_name = btf__name_by_offset(btf, t->name_off); + } else if (sub_id == 0 && sub_prog_cnt == 1) { + /* no subprog */ + if (info->name[0]) + short_name = info->name; + } else + short_name = "F"; + if (short_name) { + name_len += scnprintf(buf + name_len, size - name_len, + "_%s", short_name); + } + return name_len; +} + +#ifdef HAVE_LIBBPF_STRINGS_SUPPORT + +#define BPF_METADATA_PREFIX "bpf_metadata_" +#define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1) + +static bool name_has_bpf_metadata_prefix(const char **s) +{ + if (strncmp(*s, BPF_METADATA_PREFIX, BPF_METADATA_PREFIX_LEN) != 0) + return false; + *s += BPF_METADATA_PREFIX_LEN; + return true; +} + +struct bpf_metadata_map { + struct btf *btf; + const struct btf_type *datasec; + void *rodata; + size_t rodata_size; + unsigned int num_vars; +}; + +static int bpf_metadata_read_map_data(__u32 map_id, struct bpf_metadata_map *map) +{ + int map_fd; + struct bpf_map_info map_info; + __u32 map_info_len; + int key; + struct btf *btf; + const struct btf_type *datasec; + struct btf_var_secinfo *vsi; + unsigned int vlen, vars; + void *rodata; + + map_fd = bpf_map_get_fd_by_id(map_id); + if (map_fd < 0) + return -1; + + memset(&map_info, 0, sizeof(map_info)); + map_info_len = sizeof(map_info); + if (bpf_obj_get_info_by_fd(map_fd, &map_info, &map_info_len) < 0) + goto out_close; + + /* If it's not an .rodata map, don't bother. */ + if (map_info.type != BPF_MAP_TYPE_ARRAY || + map_info.key_size != sizeof(int) || + map_info.max_entries != 1 || + !map_info.btf_value_type_id || + !strstr(map_info.name, ".rodata")) { + goto out_close; + } + + btf = btf__load_from_kernel_by_id(map_info.btf_id); + if (!btf) + goto out_close; + datasec = btf__type_by_id(btf, map_info.btf_value_type_id); + if (!btf_is_datasec(datasec)) + goto out_free_btf; + + /* + * If there aren't any variables with the "bpf_metadata_" prefix, + * don't bother. + */ + vlen = btf_vlen(datasec); + vsi = btf_var_secinfos(datasec); + vars = 0; + for (unsigned int i = 0; i < vlen; i++, vsi++) { + const struct btf_type *t_var = btf__type_by_id(btf, vsi->type); + const char *name = btf__name_by_offset(btf, t_var->name_off); + + if (name_has_bpf_metadata_prefix(&name)) + vars++; + } + if (vars == 0) + goto out_free_btf; + + rodata = zalloc(map_info.value_size); + if (!rodata) + goto out_free_btf; + key = 0; + if (bpf_map_lookup_elem(map_fd, &key, rodata)) { + free(rodata); + goto out_free_btf; + } + close(map_fd); + + map->btf = btf; + map->datasec = datasec; + map->rodata = rodata; + map->rodata_size = map_info.value_size; + map->num_vars = vars; + return 0; + +out_free_btf: + btf__free(btf); +out_close: + close(map_fd); + return -1; +} + +struct format_btf_ctx { + char *buf; + size_t buf_size; + size_t buf_idx; +}; + +static void format_btf_cb(void *arg, const char *fmt, va_list ap) +{ + int n; + struct format_btf_ctx *ctx = (struct format_btf_ctx *)arg; + + n = vsnprintf(ctx->buf + ctx->buf_idx, ctx->buf_size - ctx->buf_idx, + fmt, ap); + ctx->buf_idx += n; + if (ctx->buf_idx >= ctx->buf_size) + ctx->buf_idx = ctx->buf_size; +} + +static void format_btf_variable(struct btf *btf, char *buf, size_t buf_size, + const struct btf_type *t, const void *btf_data) +{ + struct format_btf_ctx ctx = { + .buf = buf, + .buf_idx = 0, + .buf_size = buf_size, + }; + const struct btf_dump_type_data_opts opts = { + .sz = sizeof(struct btf_dump_type_data_opts), + .skip_names = 1, + .compact = 1, + .emit_strings = 1, + }; + struct btf_dump *d; + size_t btf_size; + + d = btf_dump__new(btf, format_btf_cb, &ctx, NULL); + btf_size = btf__resolve_size(btf, t->type); + btf_dump__dump_type_data(d, t->type, btf_data, btf_size, &opts); + btf_dump__free(d); +} + +static void bpf_metadata_fill_event(struct bpf_metadata_map *map, + struct perf_record_bpf_metadata *bpf_metadata_event) +{ + struct btf_var_secinfo *vsi; + unsigned int i, vlen; + + memset(bpf_metadata_event->prog_name, 0, BPF_PROG_NAME_LEN); + vlen = btf_vlen(map->datasec); + vsi = btf_var_secinfos(map->datasec); + + for (i = 0; i < vlen; i++, vsi++) { + const struct btf_type *t_var = btf__type_by_id(map->btf, + vsi->type); + const char *name = btf__name_by_offset(map->btf, + t_var->name_off); + const __u64 nr_entries = bpf_metadata_event->nr_entries; + struct perf_record_bpf_metadata_entry *entry; + + if (!name_has_bpf_metadata_prefix(&name)) + continue; + + if (nr_entries >= (__u64)map->num_vars) + break; + + entry = &bpf_metadata_event->entries[nr_entries]; + memset(entry, 0, sizeof(*entry)); + snprintf(entry->key, BPF_METADATA_KEY_LEN, "%s", name); + format_btf_variable(map->btf, entry->value, + BPF_METADATA_VALUE_LEN, t_var, + map->rodata + vsi->offset); + bpf_metadata_event->nr_entries++; + } +} + +static void bpf_metadata_free_map_data(struct bpf_metadata_map *map) +{ + btf__free(map->btf); + free(map->rodata); +} + +static struct bpf_metadata *bpf_metadata_alloc(__u32 nr_prog_tags, + __u32 nr_variables) +{ + struct bpf_metadata *metadata; + size_t event_size; + + metadata = zalloc(sizeof(struct bpf_metadata)); + if (!metadata) + return NULL; + + metadata->prog_names = calloc(nr_prog_tags, sizeof(char *)); + if (!metadata->prog_names) { + bpf_metadata_free(metadata); + return NULL; + } + for (__u32 prog_index = 0; prog_index < nr_prog_tags; prog_index++) { + metadata->prog_names[prog_index] = zalloc(BPF_PROG_NAME_LEN); + if (!metadata->prog_names[prog_index]) { + bpf_metadata_free(metadata); + return NULL; + } + metadata->nr_prog_names++; + } + + event_size = sizeof(metadata->event->bpf_metadata) + + nr_variables * sizeof(metadata->event->bpf_metadata.entries[0]); + /* + * header.size is __u16. synthesize_perf_record_bpf_metadata() + * adds machine->id_hdr_size (up to ~64 bytes) after this, so + * leave headroom to prevent the final size from wrapping. + */ + if (event_size > UINT16_MAX - 256) { + bpf_metadata_free(metadata); + return NULL; + } + metadata->event = zalloc(event_size); + if (!metadata->event) { + bpf_metadata_free(metadata); + return NULL; + } + metadata->event->bpf_metadata = (struct perf_record_bpf_metadata) { + .header = { + .type = PERF_RECORD_BPF_METADATA, + .size = event_size, + }, + .nr_entries = 0, + }; + + return metadata; +} + +static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info) +{ + struct bpf_metadata *metadata; + const __u32 *map_ids = (__u32 *)(uintptr_t)info->map_ids; + + for (__u32 map_index = 0; map_index < info->nr_map_ids; map_index++) { + struct bpf_metadata_map map; + + if (bpf_metadata_read_map_data(map_ids[map_index], &map) != 0) + continue; + + metadata = bpf_metadata_alloc(info->nr_prog_tags, map.num_vars); + if (!metadata) { + bpf_metadata_free_map_data(&map); + continue; + } + + bpf_metadata_fill_event(&map, &metadata->event->bpf_metadata); + + for (__u32 index = 0; index < info->nr_prog_tags; index++) { + synthesize_bpf_prog_name(metadata->prog_names[index], + BPF_PROG_NAME_LEN, info, + map.btf, index); + } + + bpf_metadata_free_map_data(&map); + + return metadata; + } + + return NULL; +} + +static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata, + const struct perf_tool *tool, + perf_event__handler_t process, + struct machine *machine) +{ + const size_t event_size = metadata->event->header.size; + union perf_event *event; + int err = 0; + + event = zalloc(event_size + machine->id_hdr_size); + if (!event) + return -1; + memcpy(event, metadata->event, event_size); + memset((void *)event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + for (__u32 index = 0; index < metadata->nr_prog_names; index++) { + memcpy(event->bpf_metadata.prog_name, + metadata->prog_names[index], BPF_PROG_NAME_LEN); + err = perf_tool__process_synth_event(tool, event, machine, + process); + if (err != 0) + break; + } + + free(event); + return err; +} + +void bpf_metadata_free(struct bpf_metadata *metadata) +{ + if (metadata == NULL) + return; + for (__u32 index = 0; index < metadata->nr_prog_names; index++) + free(metadata->prog_names[index]); + free(metadata->prog_names); + free(metadata->event); + free(metadata); +} + +#else /* HAVE_LIBBPF_STRINGS_SUPPORT */ + +static struct bpf_metadata *bpf_metadata_create(struct bpf_prog_info *info __maybe_unused) +{ + return NULL; +} + +static int synthesize_perf_record_bpf_metadata(const struct bpf_metadata *metadata __maybe_unused, + const struct perf_tool *tool __maybe_unused, + perf_event__handler_t process __maybe_unused, + struct machine *machine __maybe_unused) +{ + return 0; +} + +void bpf_metadata_free(struct bpf_metadata *metadata __maybe_unused) +{ +} + +#endif /* HAVE_LIBBPF_STRINGS_SUPPORT */ + +struct bpf_metadata_final_ctx { + const struct perf_tool *tool; + perf_event__handler_t process; + struct machine *machine; +}; + +static void synthesize_final_bpf_metadata_cb(struct bpf_prog_info_node *node, + void *data) +{ + struct bpf_metadata_final_ctx *ctx = (struct bpf_metadata_final_ctx *)data; + struct bpf_metadata *metadata = node->metadata; + int err; + + if (metadata == NULL) + return; + err = synthesize_perf_record_bpf_metadata(metadata, ctx->tool, + ctx->process, ctx->machine); + if (err != 0) { + const char *prog_name = metadata->prog_names[0]; + + if (prog_name != NULL) + pr_warning("Couldn't synthesize final BPF metadata for %s.\n", prog_name); + else + pr_warning("Couldn't synthesize final BPF metadata.\n"); + } + bpf_metadata_free(metadata); + node->metadata = NULL; +} + +void perf_event__synthesize_final_bpf_metadata(struct perf_session *session, + perf_event__handler_t process) +{ + struct perf_env *env = &session->header.env; + struct bpf_metadata_final_ctx ctx = { + .tool = session->tool, + .process = process, + .machine = &session->machines.host, + }; + + perf_env__iterate_bpf_prog_info(env, synthesize_final_bpf_metadata_cb, + &ctx); +} + +/* + * Synthesize PERF_RECORD_KSYMBOL and PERF_RECORD_BPF_EVENT for one bpf + * program. One PERF_RECORD_BPF_EVENT is generated for the program. And + * one PERF_RECORD_KSYMBOL is generated for each sub program. + * + * Returns: + * 0 for success; + * -1 for failures; + * -2 for lack of kernel support. + */ +static int perf_event__synthesize_one_bpf_prog(struct perf_session *session, + perf_event__handler_t process, + struct machine *machine, + int fd, + union perf_event *event, + struct record_opts *opts) +{ + struct perf_record_ksymbol *ksymbol_event = &event->ksymbol; + struct perf_record_bpf_event *bpf_event = &event->bpf; + const struct perf_tool *tool = session->tool; + struct bpf_prog_info_node *info_node; + struct perf_bpil *info_linear; + struct bpf_metadata *metadata; + struct bpf_prog_info *info; + struct btf *btf = NULL; + struct perf_env *env; + u32 sub_prog_cnt, i; + int err = 0; + u64 arrays; + + /* + * for perf-record and perf-report use header.env; + * otherwise, use global perf_env. + */ + env = perf_session__env(session); + + arrays = 1UL << PERF_BPIL_JITED_KSYMS; + arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; + arrays |= 1UL << PERF_BPIL_FUNC_INFO; + arrays |= 1UL << PERF_BPIL_PROG_TAGS; + arrays |= 1UL << PERF_BPIL_JITED_INSNS; + arrays |= 1UL << PERF_BPIL_LINE_INFO; + arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; + arrays |= 1UL << PERF_BPIL_MAP_IDS; + + info_linear = get_bpf_prog_info_linear(fd, arrays); + if (IS_ERR_OR_NULL(info_linear)) { + info_linear = NULL; + pr_debug("%s: failed to get BPF program info. aborting\n", __func__); + return -1; + } + + if (info_linear->info_len < offsetof(struct bpf_prog_info, prog_tags)) { + free(info_linear); + pr_debug("%s: the kernel is too old, aborting\n", __func__); + return -2; + } + + info = &info_linear->info; + if (!info->jited_ksyms) { + free(info_linear); + return -1; + } + + /* number of ksyms, func_lengths, and tags should match */ + sub_prog_cnt = info->nr_jited_ksyms; + if (sub_prog_cnt != info->nr_prog_tags || + sub_prog_cnt != info->nr_jited_func_lens) { + free(info_linear); + return -1; + } + + /* check BTF func info support */ + if (info->btf_id && info->nr_func_info && info->func_info_rec_size) { + /* btf func info number should be same as sub_prog_cnt */ + if (sub_prog_cnt != info->nr_func_info) { + pr_debug("%s: mismatch in BPF sub program count and BTF function info count, aborting\n", __func__); + free(info_linear); + return -1; + } + btf = btf__load_from_kernel_by_id(info->btf_id); + if (libbpf_get_error(btf)) { + pr_debug("%s: failed to get BTF of id %u, aborting\n", __func__, info->btf_id); + err = -1; + goto out; + } + perf_env__fetch_btf(env, info->btf_id, btf); + } + + /* Synthesize PERF_RECORD_KSYMBOL */ + for (i = 0; i < sub_prog_cnt; i++) { + __u32 *prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + __u64 *prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + int name_len; + + *ksymbol_event = (struct perf_record_ksymbol) { + .header = { + .type = PERF_RECORD_KSYMBOL, + .size = offsetof(struct perf_record_ksymbol, name), + }, + .addr = prog_addrs[i], + .len = prog_lens[i], + .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, + .flags = 0, + }; + + name_len = synthesize_bpf_prog_name(ksymbol_event->name, + KSYM_NAME_LEN, info, btf, i); + ksymbol_event->header.size += PERF_ALIGN(name_len + 1, + sizeof(u64)); + + memset((void *)event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + err = perf_tool__process_synth_event(tool, event, + machine, process); + } + + if (!opts->no_bpf_event) { + /* Synthesize PERF_RECORD_BPF_EVENT */ + *bpf_event = (struct perf_record_bpf_event) { + .header = { + .type = PERF_RECORD_BPF_EVENT, + .size = sizeof(struct perf_record_bpf_event), + }, + .type = PERF_BPF_EVENT_PROG_LOAD, + .flags = 0, + .id = info->id, + }; + memcpy(bpf_event->tag, info->tag, BPF_TAG_SIZE); + memset((void *)event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + + /* save bpf_prog_info to env */ + info_node = malloc(sizeof(struct bpf_prog_info_node)); + if (!info_node) { + err = -1; + goto out; + } + + info_node->info_linear = info_linear; + info_node->metadata = NULL; + if (!perf_env__insert_bpf_prog_info(env, info_node)) { + /* + * Insert failed, likely because of a duplicate event + * made by the sideband thread. Ignore synthesizing the + * metadata. + */ + free(info_node); + goto out; + } + /* info_linear is now owned by info_node and shouldn't be freed below. */ + info_linear = NULL; + + /* + * process after saving bpf_prog_info to env, so that + * required information is ready for look up + */ + err = perf_tool__process_synth_event(tool, event, + machine, process); + + /* Synthesize PERF_RECORD_BPF_METADATA */ + metadata = bpf_metadata_create(info); + if (metadata != NULL) { + err = synthesize_perf_record_bpf_metadata(metadata, + tool, process, + machine); + bpf_metadata_free(metadata); + } + } + +out: + free(info_linear); + btf__free(btf); + return err ? -1 : 0; +} + +struct kallsyms_parse { + union perf_event *event; + perf_event__handler_t process; + struct machine *machine; + const struct perf_tool *tool; +}; + +static int +process_bpf_image(char *name, u64 addr, struct kallsyms_parse *data) +{ + struct machine *machine = data->machine; + union perf_event *event = data->event; + struct perf_record_ksymbol *ksymbol; + int len; + + ksymbol = &event->ksymbol; + + *ksymbol = (struct perf_record_ksymbol) { + .header = { + .type = PERF_RECORD_KSYMBOL, + .size = offsetof(struct perf_record_ksymbol, name), + }, + .addr = addr, + .len = page_size, + .ksym_type = PERF_RECORD_KSYMBOL_TYPE_BPF, + .flags = 0, + }; + + len = scnprintf(ksymbol->name, KSYM_NAME_LEN, "%s", name); + ksymbol->header.size += PERF_ALIGN(len + 1, sizeof(u64)); + memset((void *) event + event->header.size, 0, machine->id_hdr_size); + event->header.size += machine->id_hdr_size; + + return perf_tool__process_synth_event(data->tool, event, machine, + data->process); +} + +static int +kallsyms_process_symbol(void *data, const char *_name, + char type __maybe_unused, u64 start) +{ + char disp[KSYM_NAME_LEN]; + const char *module; + char *name; + unsigned long id; + int err = 0; + + module = strchr(_name, '\t'); + if (!module) + return 0; + + /* We are going after [bpf] module ... */ + if (strcmp(module + 1, "[bpf]")) + return 0; + + name = memdup(_name, (module - _name) + 1); + if (!name) + return -ENOMEM; + + name[module - _name] = 0; + + /* .. and only for trampolines and dispatchers */ + if ((sscanf(name, "bpf_trampoline_%lu", &id) == 1) || + (sscanf(name, "bpf_dispatcher_%s", disp) == 1)) + err = process_bpf_image(name, start, data); + + free(name); + return err; +} + +int perf_event__synthesize_bpf_events(struct perf_session *session, + perf_event__handler_t process, + struct machine *machine, + struct record_opts *opts) +{ + const char *kallsyms_filename = "/proc/kallsyms"; + struct kallsyms_parse arg; + union perf_event *event; + __u32 id = 0; + int err; + int fd; + + if (opts->no_bpf_event) + return 0; + + event = malloc(sizeof(event->bpf) + KSYM_NAME_LEN + machine->id_hdr_size); + if (!event) + return -1; + + /* Synthesize all the bpf programs in system. */ + while (true) { + err = bpf_prog_get_next_id(id, &id); + if (err) { + if (errno == ENOENT) { + err = 0; + break; + } + /* don't report error on old kernel or EPERM */ + err = (errno == EINVAL || errno == EPERM) ? 0 : -1; + pr_debug("%s: can\'t get next program: %m%s\n", + __func__, errno == EINVAL ? " -- kernel too old?" : ""); + break; + } + fd = bpf_prog_get_fd_by_id(id); + if (fd < 0) { + pr_debug("%s: failed to get fd for prog_id %u\n", + __func__, id); + continue; + } + + err = perf_event__synthesize_one_bpf_prog(session, process, + machine, fd, + event, opts); + close(fd); + if (err) { + /* do not return error for old kernel */ + if (err == -2) + err = 0; + break; + } + } + + /* Synthesize all the bpf images - trampolines/dispatchers. */ + if (symbol_conf.kallsyms_name != NULL) + kallsyms_filename = symbol_conf.kallsyms_name; + + arg = (struct kallsyms_parse) { + .event = event, + .process = process, + .machine = machine, + .tool = session->tool, + }; + + if (kallsyms__parse(kallsyms_filename, &arg, kallsyms_process_symbol)) + pr_err("%s: failed to synthesize bpf images: %m\n", __func__); + + free(event); + return err; +} + +static int perf_env__add_bpf_info(struct perf_env *env, u32 id) +{ + struct bpf_prog_info_node *info_node; + struct perf_bpil *info_linear; + struct btf *btf = NULL; + u64 arrays; + u32 btf_id; + int fd, err = 0; + + fd = bpf_prog_get_fd_by_id(id); + if (fd < 0) + return -EINVAL; + + arrays = 1UL << PERF_BPIL_JITED_KSYMS; + arrays |= 1UL << PERF_BPIL_JITED_FUNC_LENS; + arrays |= 1UL << PERF_BPIL_FUNC_INFO; + arrays |= 1UL << PERF_BPIL_PROG_TAGS; + arrays |= 1UL << PERF_BPIL_JITED_INSNS; + arrays |= 1UL << PERF_BPIL_LINE_INFO; + arrays |= 1UL << PERF_BPIL_JITED_LINE_INFO; + arrays |= 1UL << PERF_BPIL_MAP_IDS; + + info_linear = get_bpf_prog_info_linear(fd, arrays); + if (IS_ERR_OR_NULL(info_linear)) { + pr_debug("%s: failed to get BPF program info. aborting\n", __func__); + err = PTR_ERR(info_linear); + goto out; + } + + btf_id = info_linear->info.btf_id; + + info_node = malloc(sizeof(struct bpf_prog_info_node)); + if (info_node) { + info_node->info_linear = info_linear; + info_node->metadata = bpf_metadata_create(&info_linear->info); + if (!perf_env__insert_bpf_prog_info(env, info_node)) { + pr_debug("%s: duplicate add bpf info request for id %u\n", + __func__, btf_id); + bpf_metadata_free(info_node->metadata); + free(info_linear); + free(info_node); + goto out; + } + } else { + free(info_linear); + err = -ENOMEM; + goto out; + } + + if (btf_id == 0) + goto out; + + btf = btf__load_from_kernel_by_id(btf_id); + if (!btf) { + err = -errno; + pr_debug("%s: failed to get BTF of id %u %d\n", __func__, btf_id, err); + } else { + perf_env__fetch_btf(env, btf_id, btf); + } + +out: + btf__free(btf); + close(fd); + return err; +} + +static int bpf_event__sb_cb(union perf_event *event, void *data) +{ + struct perf_env *env = data; + int ret = 0; + + if (event->header.type != PERF_RECORD_BPF_EVENT) + return -1; + + switch (event->bpf.type) { + case PERF_BPF_EVENT_PROG_LOAD: + ret = perf_env__add_bpf_info(env, event->bpf.id); + + case PERF_BPF_EVENT_PROG_UNLOAD: + /* + * Do not free bpf_prog_info and btf of the program here, + * as annotation still need them. They will be freed at + * the end of the session. + */ + break; + default: + pr_debug("unexpected bpf event type of %d\n", event->bpf.type); + break; + } + + return ret; +} + +int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_DUMMY, + .sample_id_all = 1, + .watermark = 1, + .bpf_event = 1, + .size = sizeof(attr), /* to capture ABI version */ + }; + + /* + * Older gcc versions don't support designated initializers, like above, + * for unnamed union members, such as the following: + */ + attr.wakeup_watermark = 1; + + return evlist__add_sb_event(evlist, &attr, bpf_event__sb_cb, env); +} + +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, + struct perf_env *env, + FILE *fp) +{ + struct bpf_prog_info *info = &info_linear->info; + __u64 required_arrays = (1UL << PERF_BPIL_JITED_KSYMS) | + (1UL << PERF_BPIL_JITED_FUNC_LENS); + __u32 *prog_lens; + __u64 *prog_addrs; + char name[KSYM_NAME_LEN]; + struct btf *btf = NULL; + u32 sub_prog_cnt, i; + + sub_prog_cnt = info->nr_jited_ksyms; + if (sub_prog_cnt != info->nr_prog_tags || + sub_prog_cnt != info->nr_jited_func_lens) + return; + + /* Ensure the arrays were present and converted by bpil_offs_to_addr() */ + if ((info_linear->arrays & required_arrays) != required_arrays) + return; + + prog_lens = (__u32 *)(uintptr_t)(info->jited_func_lens); + prog_addrs = (__u64 *)(uintptr_t)(info->jited_ksyms); + + if (info->btf_id) { + struct btf_node *node; + + node = __perf_env__find_btf(env, info->btf_id); + if (node) + btf = btf__new((__u8 *)(node->data), + node->data_size); + } + + if (sub_prog_cnt == 1) { + synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, 0); + fprintf(fp, "# bpf_prog_info %u: %s addr 0x%llx size %u\n", + info->id, name, prog_addrs[0], prog_lens[0]); + goto out; + } + + fprintf(fp, "# bpf_prog_info %u:\n", info->id); + for (i = 0; i < sub_prog_cnt; i++) { + synthesize_bpf_prog_name(name, KSYM_NAME_LEN, info, btf, i); + + fprintf(fp, "# \tsub_prog %u: %s addr 0x%llx size %u\n", + i, name, prog_addrs[i], prog_lens[i]); + } +out: + btf__free(btf); +} diff --git a/tools/perf/util/bpf-event.h b/tools/perf/util/bpf-event.h new file mode 100644 index 000000000..da4eeb4a1 --- /dev/null +++ b/tools/perf/util/bpf-event.h @@ -0,0 +1,73 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BPF_EVENT_H +#define __PERF_BPF_EVENT_H + +#include +#include +#include +#include + +struct bpf_prog_info; +struct machine; +union perf_event; +struct perf_env; +struct perf_sample; +struct perf_session; +struct record_opts; +struct evlist; +struct target; + +struct bpf_metadata { + union perf_event *event; + char **prog_names; + __u64 nr_prog_names; +}; + +struct bpf_prog_info_node { + struct perf_bpil *info_linear; + struct bpf_metadata *metadata; + struct rb_node rb_node; +}; + +struct btf_node { + struct rb_node rb_node; + u32 id; + u32 data_size; + char data[]; +}; + +#ifdef HAVE_LIBBPF_SUPPORT +int machine__process_bpf(struct machine *machine, union perf_event *event, + struct perf_sample *sample); +int evlist__add_bpf_sb_event(struct evlist *evlist, struct perf_env *env); +void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear, + struct perf_env *env, + FILE *fp); +void bpf_metadata_free(struct bpf_metadata *metadata); +#else +static inline int machine__process_bpf(struct machine *machine __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused) +{ + return 0; +} + +static inline int evlist__add_bpf_sb_event(struct evlist *evlist __maybe_unused, + struct perf_env *env __maybe_unused) +{ + return 0; +} + +static inline void __bpf_event__print_bpf_prog_info(struct perf_bpil *info_linear __maybe_unused, + struct perf_env *env __maybe_unused, + FILE *fp __maybe_unused) +{ + +} + +static inline void bpf_metadata_free(struct bpf_metadata *metadata __maybe_unused) +{ + +} +#endif // HAVE_LIBBPF_SUPPORT +#endif diff --git a/tools/perf/util/bpf-filter.c b/tools/perf/util/bpf-filter.c new file mode 100644 index 000000000..1a2e7b388 --- /dev/null +++ b/tools/perf/util/bpf-filter.c @@ -0,0 +1,807 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/** + * Generic event filter for sampling events in BPF. + * + * The BPF program is fixed and just to read filter expressions in the 'filters' + * map and compare the sample data in order to reject samples that don't match. + * Each filter expression contains a sample flag (term) to compare, an operation + * (==, >=, and so on) and a value. + * + * Note that each entry has an array of filter expressions and it only succeeds + * when all of the expressions are satisfied. But it supports the logical OR + * using a GROUP operation which is satisfied when any of its member expression + * is evaluated to true. But it doesn't allow nested GROUP operations for now. + * + * To support non-root users, the filters map can be loaded and pinned in the BPF + * filesystem by root (perf record --setup-filter pin). Then each user will get + * a new entry in the shared filters map to fill the filter expressions. And the + * BPF program will find the filter using (task-id, event-id) as a key. + * + * The pinned BPF object (shared for regular users) has: + * + * event_hash | + * | | | + * event->id ---> | id | ---+ idx_hash | filters + * | | | | | | | | + * | .... | +-> | idx | --+--> | exprs | ---> perf_bpf_filter_entry[] + * | | | | | | .op + * task id (tgid) --------------+ | .... | | | ... | .term (+ part) + * | .value + * | + * ======= (root would skip this part) ======== (compares it in a loop) + * + * This is used for per-task use cases while system-wide profiling (normally from + * root user) uses a separate copy of the program and the maps for its own so that + * it can proceed even if a lot of non-root users are using the filters at the + * same time. In this case the filters map has a single entry and no need to use + * the hash maps to get the index (key) of the filters map (IOW it's always 0). + * + * The BPF program returns 1 to accept the sample or 0 to drop it. + * The 'dropped' map is to keep how many samples it dropped by the filter and + * it will be reported as lost samples. + */ +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +#include "util/cap.h" +#include "util/debug.h" +#include "util/evsel.h" +#include "util/target.h" +#include "util/bpf-utils.h" + +#include "util/bpf-filter.h" +#include +#include + +#include "bpf_skel/sample-filter.h" +#include "bpf_skel/sample_filter.skel.h" + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) + +#define __PERF_SAMPLE_TYPE(tt, st, opt) { tt, #st, opt } +#define PERF_SAMPLE_TYPE(_st, opt) __PERF_SAMPLE_TYPE(PBF_TERM_##_st, PERF_SAMPLE_##_st, opt) + +/* Index in the pinned 'filters' map. Should be released after use. */ +struct pinned_filter_idx { + struct list_head list; + struct evsel *evsel; + u64 event_id; + int hash_idx; +}; + +static LIST_HEAD(pinned_filters); + +static const struct perf_sample_info { + enum perf_bpf_filter_term type; + const char *name; + const char *option; +} sample_table[] = { + /* default sample flags */ + PERF_SAMPLE_TYPE(IP, NULL), + PERF_SAMPLE_TYPE(TID, NULL), + PERF_SAMPLE_TYPE(PERIOD, NULL), + /* flags mostly set by default, but still have options */ + PERF_SAMPLE_TYPE(ID, "--sample-identifier"), + PERF_SAMPLE_TYPE(CPU, "--sample-cpu"), + PERF_SAMPLE_TYPE(TIME, "-T"), + /* optional sample flags */ + PERF_SAMPLE_TYPE(ADDR, "-d"), + PERF_SAMPLE_TYPE(DATA_SRC, "-d"), + PERF_SAMPLE_TYPE(PHYS_ADDR, "--phys-data"), + PERF_SAMPLE_TYPE(WEIGHT, "-W"), + PERF_SAMPLE_TYPE(WEIGHT_STRUCT, "-W"), + PERF_SAMPLE_TYPE(TRANSACTION, "--transaction"), + PERF_SAMPLE_TYPE(CODE_PAGE_SIZE, "--code-page-size"), + PERF_SAMPLE_TYPE(DATA_PAGE_SIZE, "--data-page-size"), + PERF_SAMPLE_TYPE(CGROUP, "--all-cgroups"), +}; + +static int get_pinned_fd(const char *name); + +static const struct perf_sample_info *get_sample_info(enum perf_bpf_filter_term type) +{ + size_t i; + + for (i = 0; i < ARRAY_SIZE(sample_table); i++) { + if (sample_table[i].type == type) + return &sample_table[i]; + } + return NULL; +} + +static int check_sample_flags(struct evsel *evsel, struct perf_bpf_filter_expr *expr) +{ + const struct perf_sample_info *info; + + if (expr->term >= PBF_TERM_SAMPLE_START && expr->term <= PBF_TERM_SAMPLE_END && + (evsel->core.attr.sample_type & (1 << (expr->term - PBF_TERM_SAMPLE_START)))) + return 0; + + if (expr->term == PBF_TERM_UID || expr->term == PBF_TERM_GID) { + /* Not dependent on the sample_type as computed from a BPF helper. */ + return 0; + } + + if (expr->op == PBF_OP_GROUP_BEGIN) { + struct perf_bpf_filter_expr *group; + + list_for_each_entry(group, &expr->groups, list) { + if (check_sample_flags(evsel, group) < 0) + return -1; + } + return 0; + } + + info = get_sample_info(expr->term); + if (info == NULL) { + pr_err("Error: %s event does not have sample flags %d\n", + evsel__name(evsel), expr->term); + return -1; + } + + pr_err("Error: %s event does not have %s\n", evsel__name(evsel), info->name); + if (info->option) + pr_err(" Hint: please add %s option to perf record\n", info->option); + return -1; +} + +static int get_filter_entries(struct evsel *evsel, struct perf_bpf_filter_entry *entry) +{ + int i = 0; + struct perf_bpf_filter_expr *expr; + + list_for_each_entry(expr, &evsel->bpf_filters, list) { + if (check_sample_flags(evsel, expr) < 0) + return -EINVAL; + + if (i == MAX_FILTERS) + return -E2BIG; + + entry[i].op = expr->op; + entry[i].part = expr->part; + entry[i].term = expr->term; + entry[i].value = expr->val; + i++; + + if (expr->op == PBF_OP_GROUP_BEGIN) { + struct perf_bpf_filter_expr *group; + + list_for_each_entry(group, &expr->groups, list) { + if (i == MAX_FILTERS) + return -E2BIG; + + entry[i].op = group->op; + entry[i].part = group->part; + entry[i].term = group->term; + entry[i].value = group->val; + i++; + } + + if (i == MAX_FILTERS) + return -E2BIG; + + entry[i].op = PBF_OP_GROUP_END; + i++; + } + } + + if (i < MAX_FILTERS) { + /* to terminate the loop early */ + entry[i].op = PBF_OP_DONE; + i++; + } + return 0; +} + +static int convert_to_tgid(int tid) +{ + char path[128]; + char *buf, *p, *q; + int tgid; + size_t len; + + scnprintf(path, sizeof(path), "%d/status", tid); + if (procfs__read_str(path, &buf, &len) < 0) + return -1; + + p = strstr(buf, "Tgid:"); + if (p == NULL) { + free(buf); + return -1; + } + + tgid = strtol(p + 6, &q, 0); + free(buf); + if (*q != '\n') + return -1; + + return tgid; +} + +/* + * The event might be closed already so we cannot get the list of ids using FD + * like in create_event_hash() below, let's iterate the event_hash map and + * delete all entries that have the event id as a key. + */ +static void destroy_event_hash(u64 event_id) +{ + int fd; + u64 key, *prev_key = NULL; + int num = 0, alloced = 32; + u64 *ids = calloc(alloced, sizeof(*ids)); + + if (ids == NULL) + return; + + fd = get_pinned_fd("event_hash"); + if (fd < 0) { + pr_debug("cannot get fd for 'event_hash' map\n"); + free(ids); + return; + } + + /* Iterate the whole map to collect keys for the event id. */ + while (!bpf_map_get_next_key(fd, prev_key, &key)) { + u64 id; + + if (bpf_map_lookup_elem(fd, &key, &id) == 0 && id == event_id) { + if (num == alloced) { + void *tmp; + + alloced *= 2; + tmp = realloc(ids, alloced * sizeof(*ids)); + if (tmp == NULL) + break; + + ids = tmp; + } + ids[num++] = key; + } + + prev_key = &key; + } + + for (int i = 0; i < num; i++) + bpf_map_delete_elem(fd, &ids[i]); + + free(ids); + close(fd); +} + +/* + * Return a representative id if ok, or 0 for failures. + * + * The perf_event->id is good for this, but an evsel would have multiple + * instances for CPUs and tasks. So pick up the first id and setup a hash + * from id of each instance to the representative id (the first one). + */ +static u64 create_event_hash(struct evsel *evsel) +{ + int x, y, fd; + u64 the_id = 0, id; + + fd = get_pinned_fd("event_hash"); + if (fd < 0) { + pr_err("cannot get fd for 'event_hash' map\n"); + return 0; + } + + for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { + for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { + int ret = ioctl(FD(evsel, x, y), PERF_EVENT_IOC_ID, &id); + + if (ret < 0) { + pr_err("Failed to get the event id\n"); + if (the_id) + destroy_event_hash(the_id); + return 0; + } + + if (the_id == 0) + the_id = id; + + bpf_map_update_elem(fd, &id, &the_id, BPF_ANY); + } + } + + close(fd); + return the_id; +} + +static void destroy_idx_hash(struct pinned_filter_idx *pfi) +{ + int fd, nr; + struct perf_thread_map *threads; + + fd = get_pinned_fd("filters"); + bpf_map_delete_elem(fd, &pfi->hash_idx); + close(fd); + + if (pfi->event_id) + destroy_event_hash(pfi->event_id); + + threads = perf_evsel__threads(&pfi->evsel->core); + if (threads == NULL) + return; + + fd = get_pinned_fd("idx_hash"); + nr = perf_thread_map__nr(threads); + for (int i = 0; i < nr; i++) { + /* The target task might be dead already, just try the pid */ + struct idx_hash_key key = { + .evt_id = pfi->event_id, + .tgid = perf_thread_map__pid(threads, i), + }; + + bpf_map_delete_elem(fd, &key); + } + close(fd); +} + +/* Maintain a hashmap from (tgid, event-id) to filter index */ +static int create_idx_hash(struct evsel *evsel, struct perf_bpf_filter_entry *entry) +{ + int filter_idx; + int fd, nr, last; + u64 event_id = 0; + struct pinned_filter_idx *pfi = NULL; + struct perf_thread_map *threads; + + fd = get_pinned_fd("filters"); + if (fd < 0) { + pr_err("cannot get fd for 'filters' map\n"); + return fd; + } + + /* Find the first available entry in the filters map */ + for (filter_idx = 0; filter_idx < MAX_FILTERS; filter_idx++) { + if (bpf_map_update_elem(fd, &filter_idx, entry, BPF_NOEXIST) == 0) + break; + } + close(fd); + + if (filter_idx == MAX_FILTERS) { + pr_err("Too many users for the filter map\n"); + return -EBUSY; + } + + pfi = zalloc(sizeof(*pfi)); + if (pfi == NULL) { + pr_err("Cannot save pinned filter index\n"); + return -ENOMEM; + } + + pfi->evsel = evsel; + pfi->hash_idx = filter_idx; + + event_id = create_event_hash(evsel); + if (event_id == 0) { + pr_err("Cannot update the event hash\n"); + goto err; + } + + pfi->event_id = event_id; + + threads = perf_evsel__threads(&evsel->core); + if (threads == NULL) { + pr_err("Cannot get the thread list of the event\n"); + goto err; + } + + /* save the index to a hash map */ + fd = get_pinned_fd("idx_hash"); + if (fd < 0) { + pr_err("cannot get fd for 'idx_hash' map\n"); + goto err; + } + + last = -1; + nr = perf_thread_map__nr(threads); + for (int i = 0; i < nr; i++) { + int pid = perf_thread_map__pid(threads, i); + int tgid; + struct idx_hash_key key = { + .evt_id = event_id, + }; + + /* it actually needs tgid, let's get tgid from /proc. */ + tgid = convert_to_tgid(pid); + if (tgid < 0) { + /* the thread may be dead, ignore. */ + continue; + } + + if (tgid == last) + continue; + last = tgid; + key.tgid = tgid; + + if (bpf_map_update_elem(fd, &key, &filter_idx, BPF_ANY) < 0) { + pr_err("Failed to update the idx_hash\n"); + close(fd); + goto err; + } + pr_debug("bpf-filter: idx_hash (task=%d,%s) -> %d\n", + tgid, evsel__name(evsel), filter_idx); + } + + list_add(&pfi->list, &pinned_filters); + close(fd); + return filter_idx; + +err: + destroy_idx_hash(pfi); + free(pfi); + return -1; +} + +int perf_bpf_filter__prepare(struct evsel *evsel, struct target *target) +{ + int i, x, y, fd, ret; + struct sample_filter_bpf *skel = NULL; + struct bpf_program *prog; + struct bpf_link *link; + struct perf_bpf_filter_entry *entry; + bool needs_idx_hash = !target__has_cpu(target); +#if LIBBPF_CURRENT_VERSION_GEQ(1, 7) + DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts, + .dont_enable = true); +#else + DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); +#endif + + entry = calloc(MAX_FILTERS, sizeof(*entry)); + if (entry == NULL) + return -1; + + ret = get_filter_entries(evsel, entry); + if (ret < 0) { + pr_err("Failed to process filter entries\n"); + goto err; + } + + if (needs_idx_hash && geteuid() != 0) { + int zero = 0; + + /* The filters map is shared among other processes */ + ret = create_idx_hash(evsel, entry); + if (ret < 0) + goto err; + + fd = get_pinned_fd("dropped"); + if (fd < 0) { + ret = fd; + goto err; + } + + /* Reset the lost count */ + bpf_map_update_elem(fd, &ret, &zero, BPF_ANY); + close(fd); + + fd = get_pinned_fd("perf_sample_filter"); + if (fd < 0) { + ret = fd; + goto err; + } + + for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { + for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { + ret = ioctl(FD(evsel, x, y), PERF_EVENT_IOC_SET_BPF, fd); + if (ret < 0) { + pr_err("Failed to attach perf sample-filter\n"); + close(fd); + goto err; + } + } + } + + close(fd); + free(entry); + return 0; + } + + skel = sample_filter_bpf__open_and_load(); + if (!skel) { + ret = -errno; + pr_err("Failed to load perf sample-filter BPF skeleton\n"); + goto err; + } + + i = 0; + fd = bpf_map__fd(skel->maps.filters); + + /* The filters map has only one entry in this case */ + if (bpf_map_update_elem(fd, &i, entry, BPF_ANY) < 0) { + ret = -errno; + pr_err("Failed to update the filter map\n"); + goto err; + } + + prog = skel->progs.perf_sample_filter; + for (x = 0; x < xyarray__max_x(evsel->core.fd); x++) { + for (y = 0; y < xyarray__max_y(evsel->core.fd); y++) { + link = bpf_program__attach_perf_event_opts(prog, FD(evsel, x, y), + &pe_opts); + if (IS_ERR(link)) { + pr_err("Failed to attach perf sample-filter program\n"); + ret = PTR_ERR(link); + goto err; + } + } + } + free(entry); + evsel->bpf_skel = skel; + return 0; + +err: + free(entry); + if (!list_empty(&pinned_filters)) { + struct pinned_filter_idx *pfi, *tmp; + + list_for_each_entry_safe(pfi, tmp, &pinned_filters, list) { + destroy_idx_hash(pfi); + list_del(&pfi->list); + free(pfi); + } + } + sample_filter_bpf__destroy(skel); + return ret; +} + +int perf_bpf_filter__destroy(struct evsel *evsel) +{ + struct perf_bpf_filter_expr *expr, *tmp; + struct pinned_filter_idx *pfi, *pos; + + list_for_each_entry_safe(expr, tmp, &evsel->bpf_filters, list) { + list_del(&expr->list); + free(expr); + } + sample_filter_bpf__destroy(evsel->bpf_skel); + + list_for_each_entry_safe(pfi, pos, &pinned_filters, list) { + destroy_idx_hash(pfi); + list_del(&pfi->list); + free(pfi); + } + return 0; +} + +u64 perf_bpf_filter__lost_count(struct evsel *evsel) +{ + int count = 0; + + if (list_empty(&evsel->bpf_filters)) + return 0; + + if (!list_empty(&pinned_filters)) { + int fd = get_pinned_fd("dropped"); + struct pinned_filter_idx *pfi; + + if (fd < 0) + return 0; + + list_for_each_entry(pfi, &pinned_filters, list) { + if (pfi->evsel != evsel) + continue; + + bpf_map_lookup_elem(fd, &pfi->hash_idx, &count); + break; + } + close(fd); + } else if (evsel->bpf_skel) { + struct sample_filter_bpf *skel = evsel->bpf_skel; + int fd = bpf_map__fd(skel->maps.dropped); + int idx = 0; + + bpf_map_lookup_elem(fd, &idx, &count); + } + + return count; +} + +struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term term, + int part, + enum perf_bpf_filter_op op, + unsigned long val) +{ + struct perf_bpf_filter_expr *expr; + + expr = malloc(sizeof(*expr)); + if (expr != NULL) { + expr->term = term; + expr->part = part; + expr->op = op; + expr->val = val; + INIT_LIST_HEAD(&expr->groups); + } + return expr; +} + +static bool check_bpf_filter_capable(void) +{ + bool used_root; + + if (perf_cap__capable(CAP_BPF, &used_root)) + return true; + + if (!used_root) { + /* Check if root already pinned the filter programs and maps */ + int fd = get_pinned_fd("filters"); + + if (fd >= 0) { + close(fd); + return true; + } + } + + pr_err("Error: BPF filter only works for %s!\n" + "\tPlease run 'perf record --setup-filter pin' as root first.\n", + used_root ? "root" : "users with the CAP_BPF capability"); + + return false; +} + +int perf_bpf_filter__parse(struct list_head *expr_head, const char *str) +{ + YY_BUFFER_STATE buffer; + int ret; + + if (!check_bpf_filter_capable()) + return -EPERM; + + buffer = perf_bpf_filter__scan_string(str); + + ret = perf_bpf_filter_parse(expr_head); + + perf_bpf_filter__flush_buffer(buffer); + perf_bpf_filter__delete_buffer(buffer); + perf_bpf_filter_lex_destroy(); + + return ret; +} + +int perf_bpf_filter__pin(void) +{ + struct sample_filter_bpf *skel; + char *path = NULL; + int dir_fd, ret = -1; + + skel = sample_filter_bpf__open(); + if (!skel) { + ret = -errno; + pr_err("Failed to open perf sample-filter BPF skeleton\n"); + goto err; + } + + /* pinned program will use pid-hash */ + bpf_map__set_max_entries(skel->maps.filters, MAX_FILTERS); + bpf_map__set_max_entries(skel->maps.event_hash, MAX_EVT_HASH); + bpf_map__set_max_entries(skel->maps.idx_hash, MAX_IDX_HASH); + bpf_map__set_max_entries(skel->maps.dropped, MAX_FILTERS); + skel->rodata->use_idx_hash = 1; + + if (sample_filter_bpf__load(skel) < 0) { + ret = -errno; + pr_err("Failed to load perf sample-filter BPF skeleton\n"); + goto err; + } + + if (asprintf(&path, "%s/fs/bpf/%s", sysfs__mountpoint(), + PERF_BPF_FILTER_PIN_PATH) < 0) { + ret = -errno; + pr_err("Failed to allocate pathname in the BPF-fs\n"); + goto err; + } + + ret = bpf_object__pin(skel->obj, path); + if (ret < 0) { + pr_err("Failed to pin BPF filter objects\n"); + goto err; + } + + /* setup access permissions for the pinned objects */ + dir_fd = open(path, O_PATH); + if (dir_fd < 0) { + bpf_object__unpin(skel->obj, path); + ret = dir_fd; + goto err; + } + + /* BPF-fs root has the sticky bit */ + if (fchmodat(dir_fd, "..", 01755, 0) < 0) { + pr_debug("chmod for BPF-fs failed\n"); + ret = -errno; + goto err_close; + } + + /* perf_filter directory */ + if (fchmodat(dir_fd, ".", 0755, 0) < 0) { + pr_debug("chmod for perf_filter directory failed?\n"); + ret = -errno; + goto err_close; + } + + /* programs need write permission for some reason */ + if (fchmodat(dir_fd, "perf_sample_filter", 0777, 0) < 0) { + pr_debug("chmod for perf_sample_filter failed\n"); + ret = -errno; + } + /* maps */ + if (fchmodat(dir_fd, "filters", 0666, 0) < 0) { + pr_debug("chmod for filters failed\n"); + ret = -errno; + } + if (fchmodat(dir_fd, "event_hash", 0666, 0) < 0) { + pr_debug("chmod for event_hash failed\n"); + ret = -errno; + } + if (fchmodat(dir_fd, "idx_hash", 0666, 0) < 0) { + pr_debug("chmod for idx_hash failed\n"); + ret = -errno; + } + if (fchmodat(dir_fd, "dropped", 0666, 0) < 0) { + pr_debug("chmod for dropped failed\n"); + ret = -errno; + } + +err_close: + close(dir_fd); + +err: + free(path); + sample_filter_bpf__destroy(skel); + return ret; +} + +int perf_bpf_filter__unpin(void) +{ + struct sample_filter_bpf *skel; + char *path = NULL; + int ret = -1; + + skel = sample_filter_bpf__open_and_load(); + if (!skel) { + ret = -errno; + pr_err("Failed to open perf sample-filter BPF skeleton\n"); + goto err; + } + + if (asprintf(&path, "%s/fs/bpf/%s", sysfs__mountpoint(), + PERF_BPF_FILTER_PIN_PATH) < 0) { + ret = -errno; + pr_err("Failed to allocate pathname in the BPF-fs\n"); + goto err; + } + + ret = bpf_object__unpin(skel->obj, path); + +err: + free(path); + sample_filter_bpf__destroy(skel); + return ret; +} + +static int get_pinned_fd(const char *name) +{ + char *path = NULL; + int fd; + + if (asprintf(&path, "%s/fs/bpf/%s/%s", sysfs__mountpoint(), + PERF_BPF_FILTER_PIN_PATH, name) < 0) + return -1; + + fd = bpf_obj_get(path); + + free(path); + return fd; +} diff --git a/tools/perf/util/bpf-filter.h b/tools/perf/util/bpf-filter.h new file mode 100644 index 000000000..818c554b9 --- /dev/null +++ b/tools/perf/util/bpf-filter.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_UTIL_BPF_FILTER_H +#define PERF_UTIL_BPF_FILTER_H + +#include + +#include "bpf_skel/sample-filter.h" +#include "util/debug.h" + +struct perf_bpf_filter_expr { + struct list_head list; + struct list_head groups; + enum perf_bpf_filter_op op; + int part; + enum perf_bpf_filter_term term; + unsigned long val; +}; + +struct evsel; +struct target; + +/* path in BPF-fs for the pinned program and maps */ +#define PERF_BPF_FILTER_PIN_PATH "perf_filter" + +#ifdef HAVE_BPF_SKEL +struct perf_bpf_filter_expr *perf_bpf_filter_expr__new(enum perf_bpf_filter_term term, + int part, + enum perf_bpf_filter_op op, + unsigned long val); +int perf_bpf_filter__parse(struct list_head *expr_head, const char *str); +int perf_bpf_filter__prepare(struct evsel *evsel, struct target *target); +int perf_bpf_filter__destroy(struct evsel *evsel); +u64 perf_bpf_filter__lost_count(struct evsel *evsel); +int perf_bpf_filter__pin(void); +int perf_bpf_filter__unpin(void); + +#else /* !HAVE_BPF_SKEL */ + +#include + +static inline int perf_bpf_filter__parse(struct list_head *expr_head __maybe_unused, + const char *str __maybe_unused) +{ + pr_err("Error: BPF filter is requested but perf is not built with BPF.\n" + "\tPlease make sure to build with libbpf and BPF skeleton.\n"); + return -EOPNOTSUPP; +} +static inline int perf_bpf_filter__prepare(struct evsel *evsel __maybe_unused, + struct target *target __maybe_unused) +{ + return -EOPNOTSUPP; +} +static inline int perf_bpf_filter__destroy(struct evsel *evsel __maybe_unused) +{ + return -EOPNOTSUPP; +} +static inline u64 perf_bpf_filter__lost_count(struct evsel *evsel __maybe_unused) +{ + return 0; +} +static inline int perf_bpf_filter__pin(void) +{ + return -EOPNOTSUPP; +} +static inline int perf_bpf_filter__unpin(void) +{ + return -EOPNOTSUPP; +} +#endif /* HAVE_BPF_SKEL*/ +#endif /* PERF_UTIL_BPF_FILTER_H */ diff --git a/tools/perf/util/bpf-filter.l b/tools/perf/util/bpf-filter.l new file mode 100644 index 000000000..6aa65ade3 --- /dev/null +++ b/tools/perf/util/bpf-filter.l @@ -0,0 +1,179 @@ +%option prefix="perf_bpf_filter_" +%option noyywrap + +%{ +#include +#include +#include + +#include "bpf-filter.h" +#include "bpf-filter-bison.h" + +extern int perf_bpf_filter_needs_path; + +static int sample(enum perf_bpf_filter_term term) +{ + perf_bpf_filter_needs_path = 0; + perf_bpf_filter_lval.sample.term = term; + perf_bpf_filter_lval.sample.part = 0; + return BFT_SAMPLE; +} + +static int sample_part(enum perf_bpf_filter_term term, int part) +{ + perf_bpf_filter_needs_path = 0; + perf_bpf_filter_lval.sample.term = term; + perf_bpf_filter_lval.sample.part = part; + return BFT_SAMPLE; +} + +static int sample_path(enum perf_bpf_filter_term term) +{ + perf_bpf_filter_needs_path = 1; + perf_bpf_filter_lval.sample.term = term; + perf_bpf_filter_lval.sample.part = 0; + return BFT_SAMPLE_PATH; +} + +static int operator(enum perf_bpf_filter_op op) +{ + perf_bpf_filter_lval.op = op; + return BFT_OP; +} + +static int value(int base) +{ + long num; + + errno = 0; + num = strtoul(perf_bpf_filter_text, NULL, base); + if (errno) + return BFT_ERROR; + + perf_bpf_filter_lval.num = num; + return BFT_NUM; +} + +static int constant(int val) +{ + perf_bpf_filter_lval.num = val; + return BFT_NUM; +} + +static int path_or_error(void) +{ + if (!perf_bpf_filter_needs_path) { + printf("perf_bpf_filter: Error: Unexpected item: %s\n", + perf_bpf_filter_text); + return BFT_ERROR; + } + perf_bpf_filter_lval.path = perf_bpf_filter_text; + return BFT_PATH; +} + +%} + +num_dec [0-9]+ +num_hex 0[Xx][0-9a-fA-F]+ +space [ \t]+ +path [^ \t\n,]+ +ident [_a-zA-Z][_a-zA-Z0-9]+ + +%% + +{num_dec} { return value(10); } +{num_hex} { return value(16); } +{space} { } + +ip { return sample(PBF_TERM_IP); } +id { return sample(PBF_TERM_ID); } +tid { return sample(PBF_TERM_TID); } +pid { return sample_part(PBF_TERM_TID, 1); } +cpu { return sample(PBF_TERM_CPU); } +time { return sample(PBF_TERM_TIME); } +addr { return sample(PBF_TERM_ADDR); } +period { return sample(PBF_TERM_PERIOD); } +txn { return sample(PBF_TERM_TRANSACTION); } +weight { return sample(PBF_TERM_WEIGHT); } +weight1 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 1); } +weight2 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 2); } +weight3 { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } +ins_lat { return sample_part(PBF_TERM_WEIGHT_STRUCT, 2); } /* alias for weight2 */ +p_stage_cyc { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } /* alias for weight3 */ +retire_lat { return sample_part(PBF_TERM_WEIGHT_STRUCT, 3); } /* alias for weight3 */ +phys_addr { return sample(PBF_TERM_PHYS_ADDR); } +code_pgsz { return sample(PBF_TERM_CODE_PAGE_SIZE); } +data_pgsz { return sample(PBF_TERM_DATA_PAGE_SIZE); } +mem_op { return sample_part(PBF_TERM_DATA_SRC, 1); } +mem_lvlnum { return sample_part(PBF_TERM_DATA_SRC, 2); } +mem_lvl { return sample_part(PBF_TERM_DATA_SRC, 2); } /* alias for mem_lvlnum */ +mem_snoop { return sample_part(PBF_TERM_DATA_SRC, 3); } /* include snoopx */ +mem_remote { return sample_part(PBF_TERM_DATA_SRC, 4); } +mem_lock { return sample_part(PBF_TERM_DATA_SRC, 5); } +mem_dtlb { return sample_part(PBF_TERM_DATA_SRC, 6); } +mem_blk { return sample_part(PBF_TERM_DATA_SRC, 7); } +mem_hops { return sample_part(PBF_TERM_DATA_SRC, 8); } +uid { return sample(PBF_TERM_UID); } +gid { return sample(PBF_TERM_GID); } +cgroup { return sample_path(PBF_TERM_CGROUP); } + +"==" { return operator(PBF_OP_EQ); } +"!=" { return operator(PBF_OP_NEQ); } +">" { return operator(PBF_OP_GT); } +"<" { return operator(PBF_OP_LT); } +">=" { return operator(PBF_OP_GE); } +"<=" { return operator(PBF_OP_LE); } +"&" { return operator(PBF_OP_AND); } + +na { return constant(PERF_MEM_OP_NA); } +load { return constant(PERF_MEM_OP_LOAD); } +store { return constant(PERF_MEM_OP_STORE); } +pfetch { return constant(PERF_MEM_OP_PFETCH); } +exec { return constant(PERF_MEM_OP_EXEC); } + +l1 { return constant(PERF_MEM_LVLNUM_L1); } +l2 { return constant(PERF_MEM_LVLNUM_L2); } +l3 { return constant(PERF_MEM_LVLNUM_L3); } +l4 { return constant(PERF_MEM_LVLNUM_L4); } +cxl { return constant(PERF_MEM_LVLNUM_CXL); } +io { return constant(PERF_MEM_LVLNUM_IO); } +any_cache { return constant(PERF_MEM_LVLNUM_ANY_CACHE); } +lfb { return constant(PERF_MEM_LVLNUM_LFB); } +ram { return constant(PERF_MEM_LVLNUM_RAM); } +pmem { return constant(PERF_MEM_LVLNUM_PMEM); } + +none { return constant(PERF_MEM_SNOOP_NONE); } +hit { return constant(PERF_MEM_SNOOP_HIT); } +miss { return constant(PERF_MEM_SNOOP_MISS); } +hitm { return constant(PERF_MEM_SNOOP_HITM); } +fwd { return constant(PERF_MEM_SNOOPX_FWD); } +peer { return constant(PERF_MEM_SNOOPX_PEER); } + +remote { return constant(PERF_MEM_REMOTE_REMOTE); } + +locked { return constant(PERF_MEM_LOCK_LOCKED); } + +l1_hit { return constant(PERF_MEM_TLB_L1 | PERF_MEM_TLB_HIT); } +l1_miss { return constant(PERF_MEM_TLB_L1 | PERF_MEM_TLB_MISS); } +l2_hit { return constant(PERF_MEM_TLB_L2 | PERF_MEM_TLB_HIT); } +l2_miss { return constant(PERF_MEM_TLB_L2 | PERF_MEM_TLB_MISS); } +any_hit { return constant(PERF_MEM_TLB_HIT); } +any_miss { return constant(PERF_MEM_TLB_MISS); } +walk { return constant(PERF_MEM_TLB_WK); } +os { return constant(PERF_MEM_TLB_OS); } +fault { return constant(PERF_MEM_TLB_OS); } /* alias for os */ + +by_data { return constant(PERF_MEM_BLK_DATA); } +by_addr { return constant(PERF_MEM_BLK_ADDR); } + +hops0 { return constant(PERF_MEM_HOPS_0); } +hops1 { return constant(PERF_MEM_HOPS_1); } +hops2 { return constant(PERF_MEM_HOPS_2); } +hops3 { return constant(PERF_MEM_HOPS_3); } + +"," { return ','; } +"||" { return BFT_LOGICAL_OR; } + +{path} { return path_or_error(); } + +%% diff --git a/tools/perf/util/bpf-filter.y b/tools/perf/util/bpf-filter.y new file mode 100644 index 000000000..5a79a8e7a --- /dev/null +++ b/tools/perf/util/bpf-filter.y @@ -0,0 +1,109 @@ +%parse-param {struct list_head *expr_head} +%define parse.error verbose + +%{ + +#ifndef NDEBUG +#define YYDEBUG 1 +#endif + +#include +#include +#include +#include +#include "bpf-filter.h" +#include "cgroup.h" + +int perf_bpf_filter_lex(void); + +/* To indicate if the current term needs a pathname or not */ +int perf_bpf_filter_needs_path; + +static void perf_bpf_filter_error(struct list_head *expr __maybe_unused, + char const *msg) +{ + printf("perf_bpf_filter: %s\n", msg); +} + +%} + +%union +{ + unsigned long num; + char *path; + struct { + enum perf_bpf_filter_term term; + int part; + } sample; + enum perf_bpf_filter_op op; + struct perf_bpf_filter_expr *expr; +} + +%token BFT_SAMPLE BFT_SAMPLE_PATH BFT_OP BFT_ERROR BFT_NUM BFT_LOGICAL_OR BFT_PATH +%type filter_term filter_expr +%destructor { free ($$); } +%type BFT_SAMPLE BFT_SAMPLE_PATH +%type BFT_OP +%type BFT_NUM +%type BFT_PATH + +%% + +filter: +filter ',' filter_term +{ + list_add_tail(&$3->list, expr_head); +} +| +filter_term +{ + list_add_tail(&$1->list, expr_head); +} + +filter_term: +filter_term BFT_LOGICAL_OR filter_expr +{ + struct perf_bpf_filter_expr *expr; + + if ($1->op == PBF_OP_GROUP_BEGIN) { + expr = $1; + } else { + expr = perf_bpf_filter_expr__new(PBF_TERM_NONE, /*part=*/0, + PBF_OP_GROUP_BEGIN, /*val=*/1); + list_add_tail(&$1->list, &expr->groups); + } + expr->val++; + list_add_tail(&$3->list, &expr->groups); + $$ = expr; +} +| +filter_expr +{ + $$ = $1; +} + +filter_expr: +BFT_SAMPLE BFT_OP BFT_NUM +{ + $$ = perf_bpf_filter_expr__new($1.term, $1.part, $2, $3); +} +| +BFT_SAMPLE_PATH BFT_OP BFT_PATH +{ + struct cgroup *cgrp; + unsigned long cgroup_id = 0; + + if ($2 != PBF_OP_EQ && $2 != PBF_OP_NEQ) { + printf("perf_bpf_filter: cgroup accepts '==' or '!=' only\n"); + YYERROR; + } + + cgrp = cgroup__new($3, /*do_open=*/false); + if (cgrp && read_cgroup_id(cgrp) == 0) + cgroup_id = cgrp->id; + + $$ = perf_bpf_filter_expr__new($1.term, $1.part, $2, cgroup_id); + cgroup__put(cgrp); +} + +%% diff --git a/tools/perf/util/bpf-trace-summary.c b/tools/perf/util/bpf-trace-summary.c new file mode 100644 index 000000000..9a31dbf06 --- /dev/null +++ b/tools/perf/util/bpf-trace-summary.c @@ -0,0 +1,465 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include +#include +#include +#include +#include + +#include "dwarf-regs.h" /* for EM_HOST */ +#include "trace/beauty/syscalltbl.h" +#include "util/cgroup.h" +#include "util/hashmap.h" +#include "util/trace.h" +#include "util/util.h" +#include +#include +#include +#include /* reallocarray */ + +#include "bpf_skel/syscall_summary.h" +#include "bpf_skel/syscall_summary.skel.h" + + +static struct syscall_summary_bpf *skel; +static struct rb_root cgroups = RB_ROOT; + +int trace_prepare_bpf_summary(enum trace_summary_mode mode) +{ + skel = syscall_summary_bpf__open(); + if (skel == NULL) { + fprintf(stderr, "failed to open syscall summary bpf skeleton\n"); + return -1; + } + + if (mode == SUMMARY__BY_THREAD) + skel->rodata->aggr_mode = SYSCALL_AGGR_THREAD; + else if (mode == SUMMARY__BY_CGROUP) + skel->rodata->aggr_mode = SYSCALL_AGGR_CGROUP; + else + skel->rodata->aggr_mode = SYSCALL_AGGR_CPU; + + if (cgroup_is_v2("perf_event") > 0) + skel->rodata->use_cgroup_v2 = 1; + + if (syscall_summary_bpf__load(skel) < 0) { + fprintf(stderr, "failed to load syscall summary bpf skeleton\n"); + return -1; + } + + if (syscall_summary_bpf__attach(skel) < 0) { + fprintf(stderr, "failed to attach syscall summary bpf skeleton\n"); + return -1; + } + + if (mode == SUMMARY__BY_CGROUP) + read_all_cgroups(&cgroups); + + return 0; +} + +void trace_start_bpf_summary(void) +{ + skel->bss->enabled = 1; +} + +void trace_end_bpf_summary(void) +{ + skel->bss->enabled = 0; +} + +struct syscall_node { + int syscall_nr; + struct syscall_stats stats; +}; + +static double rel_stddev(struct syscall_stats *stat) +{ + double variance, average; + + if (stat->count < 2) + return 0; + + average = (double)stat->total_time / stat->count; + + variance = stat->squared_sum; + variance -= (stat->total_time * stat->total_time) / stat->count; + variance /= stat->count - 1; + + return 100 * sqrt(variance / stat->count) / average; +} + +/* + * The syscall_data is to maintain syscall stats ordered by total time. + * It supports different summary modes like per-thread or global. + * + * For per-thread stats, it uses two-level data strurcture - + * syscall_data is keyed by TID and has an array of nodes which + * represents each syscall for the thread. + * + * For global stats, it's still two-level technically but we don't need + * per-cpu analysis so it's keyed by the syscall number to combine stats + * from different CPUs. And syscall_data always has a syscall_node so + * it can effectively work as flat hierarchy. + * + * For per-cgroup stats, it uses two-level data structure like thread + * syscall_data is keyed by CGROUP and has an array of node which + * represents each syscall for the cgroup. + */ +struct syscall_data { + u64 key; /* tid if AGGR_THREAD, syscall-nr if AGGR_CPU, cgroup if AGGR_CGROUP */ + int nr_events; + int nr_nodes; + u64 total_time; + struct syscall_node *nodes; +}; + +static int datacmp(const void *a, const void *b) +{ + const struct syscall_data * const *sa = a; + const struct syscall_data * const *sb = b; + + return (*sa)->total_time > (*sb)->total_time ? -1 : 1; +} + +static int nodecmp(const void *a, const void *b) +{ + const struct syscall_node *na = a; + const struct syscall_node *nb = b; + + return na->stats.total_time > nb->stats.total_time ? -1 : 1; +} + +static size_t sc_node_hash(long key, void *ctx __maybe_unused) +{ + return key; +} + +static bool sc_node_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +static int print_common_stats(struct syscall_data *data, int max_summary, FILE *fp) +{ + int printed = 0; + + if (max_summary == 0 || max_summary > data->nr_nodes) + max_summary = data->nr_nodes; + + for (int i = 0; i < max_summary; i++) { + struct syscall_node *node = &data->nodes[i]; + struct syscall_stats *stat = &node->stats; + double total = (double)(stat->total_time) / NSEC_PER_MSEC; + double min = (double)(stat->min_time) / NSEC_PER_MSEC; + double max = (double)(stat->max_time) / NSEC_PER_MSEC; + double avg = total / stat->count; + const char *name; + + /* TODO: support other ABIs */ + name = syscalltbl__name(EM_HOST, node->syscall_nr); + if (name) + printed += fprintf(fp, " %-15s", name); + else + printed += fprintf(fp, " syscall:%-7d", node->syscall_nr); + + printed += fprintf(fp, " %8u %6u %9.3f %9.3f %9.3f %9.3f %9.2f%%\n", + stat->count, stat->error, total, min, avg, max, + rel_stddev(stat)); + } + return printed; +} + +static int update_thread_stats(struct hashmap *hash, struct syscall_key *map_key, + struct syscall_stats *map_data) +{ + struct syscall_data *data; + struct syscall_node *nodes; + + if (!hashmap__find(hash, map_key->cpu_or_tid, &data)) { + data = zalloc(sizeof(*data)); + if (data == NULL) + return -ENOMEM; + + data->key = map_key->cpu_or_tid; + if (hashmap__add(hash, data->key, data) < 0) { + free(data); + return -ENOMEM; + } + } + + /* update thread total stats */ + data->nr_events += map_data->count; + data->total_time += map_data->total_time; + + nodes = reallocarray(data->nodes, data->nr_nodes + 1, sizeof(*nodes)); + if (nodes == NULL) + return -ENOMEM; + + data->nodes = nodes; + nodes = &data->nodes[data->nr_nodes++]; + nodes->syscall_nr = map_key->nr; + + /* each thread has an entry for each syscall, just use the stat */ + memcpy(&nodes->stats, map_data, sizeof(*map_data)); + return 0; +} + +static int print_thread_stat(struct syscall_data *data, int max_summary, FILE *fp) +{ + int printed = 0; + + qsort(data->nodes, data->nr_nodes, sizeof(*data->nodes), nodecmp); + + printed += fprintf(fp, " thread (%d), ", (int)data->key); + printed += fprintf(fp, "%d events\n\n", data->nr_events); + + printed += fprintf(fp, " syscall calls errors total min avg max stddev\n"); + printed += fprintf(fp, " (msec) (msec) (msec) (msec) (%%)\n"); + printed += fprintf(fp, " --------------- -------- ------ -------- --------- --------- --------- ------\n"); + + printed += print_common_stats(data, max_summary, fp); + printed += fprintf(fp, "\n\n"); + + return printed; +} + +static int print_thread_stats(struct syscall_data **data, int nr_data, int max_summary, FILE *fp) +{ + int printed = 0; + + for (int i = 0; i < nr_data; i++) + printed += print_thread_stat(data[i], max_summary, fp); + + return printed; +} + +static int update_total_stats(struct hashmap *hash, struct syscall_key *map_key, + struct syscall_stats *map_data) +{ + struct syscall_data *data; + struct syscall_stats *stat; + + if (!hashmap__find(hash, map_key->nr, &data)) { + data = zalloc(sizeof(*data)); + if (data == NULL) + return -ENOMEM; + + data->nodes = zalloc(sizeof(*data->nodes)); + if (data->nodes == NULL) { + free(data); + return -ENOMEM; + } + + data->nr_nodes = 1; + data->key = map_key->nr; + data->nodes->syscall_nr = data->key; + + if (hashmap__add(hash, data->key, data) < 0) { + free(data->nodes); + free(data); + return -ENOMEM; + } + } + + /* update total stats for this syscall */ + data->nr_events += map_data->count; + data->total_time += map_data->total_time; + + /* This is sum of the same syscall from different CPUs */ + stat = &data->nodes->stats; + + stat->total_time += map_data->total_time; + stat->squared_sum += map_data->squared_sum; + stat->count += map_data->count; + stat->error += map_data->error; + + if (stat->max_time < map_data->max_time) + stat->max_time = map_data->max_time; + if (stat->min_time > map_data->min_time || stat->min_time == 0) + stat->min_time = map_data->min_time; + + return 0; +} + +static int print_total_stats(struct syscall_data **data, int nr_data, int max_summary, FILE *fp) +{ + int printed = 0; + int nr_events = 0; + + for (int i = 0; i < nr_data; i++) + nr_events += data[i]->nr_events; + + printed += fprintf(fp, " total, %d events\n\n", nr_events); + + printed += fprintf(fp, " syscall calls errors total min avg max stddev\n"); + printed += fprintf(fp, " (msec) (msec) (msec) (msec) (%%)\n"); + printed += fprintf(fp, " --------------- -------- ------ -------- --------- --------- --------- ------\n"); + + if (max_summary == 0 || max_summary > nr_data) + max_summary = nr_data; + + for (int i = 0; i < max_summary; i++) + printed += print_common_stats(data[i], max_summary, fp); + + printed += fprintf(fp, "\n\n"); + return printed; +} + +static int update_cgroup_stats(struct hashmap *hash, struct syscall_key *map_key, + struct syscall_stats *map_data) +{ + struct syscall_data *data; + struct syscall_node *nodes; + + if (!hashmap__find(hash, map_key->cgroup, &data)) { + data = zalloc(sizeof(*data)); + if (data == NULL) + return -ENOMEM; + + data->key = map_key->cgroup; + if (hashmap__add(hash, data->key, data) < 0) { + free(data); + return -ENOMEM; + } + } + + /* update thread total stats */ + data->nr_events += map_data->count; + data->total_time += map_data->total_time; + + nodes = reallocarray(data->nodes, data->nr_nodes + 1, sizeof(*nodes)); + if (nodes == NULL) + return -ENOMEM; + + data->nodes = nodes; + nodes = &data->nodes[data->nr_nodes++]; + nodes->syscall_nr = map_key->nr; + + /* each thread has an entry for each syscall, just use the stat */ + memcpy(&nodes->stats, map_data, sizeof(*map_data)); + return 0; +} + +static int print_cgroup_stat(struct syscall_data *data, int max_summary, FILE *fp) +{ + int printed = 0; + struct cgroup *cgrp = __cgroup__find(&cgroups, data->key); + + qsort(data->nodes, data->nr_nodes, sizeof(*data->nodes), nodecmp); + + if (cgrp) + printed += fprintf(fp, " cgroup %s,", cgrp->name); + else + printed += fprintf(fp, " cgroup id:%lu,", (unsigned long)data->key); + + printed += fprintf(fp, " %d events\n\n", data->nr_events); + + printed += fprintf(fp, " syscall calls errors total min avg max stddev\n"); + printed += fprintf(fp, " (msec) (msec) (msec) (msec) (%%)\n"); + printed += fprintf(fp, " --------------- -------- ------ -------- --------- --------- --------- ------\n"); + + printed += print_common_stats(data, max_summary, fp); + printed += fprintf(fp, "\n\n"); + + return printed; +} + +static int print_cgroup_stats(struct syscall_data **data, int nr_data, int max_summary, FILE *fp) +{ + int printed = 0; + + for (int i = 0; i < nr_data; i++) + printed += print_cgroup_stat(data[i], max_summary, fp); + + return printed; +} + +int trace_print_bpf_summary(FILE *fp, int max_summary) +{ + struct bpf_map *map = skel->maps.syscall_stats_map; + struct syscall_key *prev_key, key; + struct syscall_data **data = NULL; + struct hashmap schash; + struct hashmap_entry *entry; + int nr_data = 0; + int printed = 0; + int i; + size_t bkt; + + hashmap__init(&schash, sc_node_hash, sc_node_equal, /*ctx=*/NULL); + + printed = fprintf(fp, "\n Summary of events:\n\n"); + + /* get stats from the bpf map */ + prev_key = NULL; + while (!bpf_map__get_next_key(map, prev_key, &key, sizeof(key))) { + struct syscall_stats stat; + + if (!bpf_map__lookup_elem(map, &key, sizeof(key), &stat, sizeof(stat), 0)) { + switch (skel->rodata->aggr_mode) { + case SYSCALL_AGGR_THREAD: + update_thread_stats(&schash, &key, &stat); + break; + case SYSCALL_AGGR_CPU: + update_total_stats(&schash, &key, &stat); + break; + case SYSCALL_AGGR_CGROUP: + update_cgroup_stats(&schash, &key, &stat); + break; + default: + break; + } + } + + prev_key = &key; + } + + nr_data = hashmap__size(&schash); + data = calloc(nr_data, sizeof(*data)); + if (data == NULL) + goto out; + + i = 0; + hashmap__for_each_entry(&schash, entry, bkt) + data[i++] = entry->pvalue; + + qsort(data, nr_data, sizeof(*data), datacmp); + + switch (skel->rodata->aggr_mode) { + case SYSCALL_AGGR_THREAD: + printed += print_thread_stats(data, nr_data, max_summary, fp); + break; + case SYSCALL_AGGR_CPU: + printed += print_total_stats(data, nr_data, max_summary, fp); + break; + case SYSCALL_AGGR_CGROUP: + printed += print_cgroup_stats(data, nr_data, max_summary, fp); + break; + default: + break; + } + + for (i = 0; i < nr_data && data; i++) { + free(data[i]->nodes); + free(data[i]); + } + free(data); + +out: + hashmap__clear(&schash); + return printed; +} + +void trace_cleanup_bpf_summary(void) +{ + if (!RB_EMPTY_ROOT(&cgroups)) { + struct cgroup *cgrp, *tmp; + + rbtree_postorder_for_each_entry_safe(cgrp, tmp, &cgroups, node) + cgroup__put(cgrp); + + cgroups = RB_ROOT; + } + + syscall_summary_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf-utils.c b/tools/perf/util/bpf-utils.c new file mode 100644 index 000000000..69148197b --- /dev/null +++ b/tools/perf/util/bpf-utils.c @@ -0,0 +1,293 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +#ifndef _GNU_SOURCE +#define _GNU_SOURCE +#endif + +#include +#include +#include +#include +#include +#include "bpf-utils.h" +#include "debug.h" + +struct bpil_array_desc { + int array_offset; /* e.g. offset of jited_prog_insns */ + int count_offset; /* e.g. offset of jited_prog_len */ + int size_offset; /* > 0: offset of rec size, + * < 0: fix size of -size_offset + */ +}; + +static const struct bpil_array_desc bpil_array_desc[] = { + [PERF_BPIL_JITED_INSNS] = { + offsetof(struct bpf_prog_info, jited_prog_insns), + offsetof(struct bpf_prog_info, jited_prog_len), + -1, + }, + [PERF_BPIL_XLATED_INSNS] = { + offsetof(struct bpf_prog_info, xlated_prog_insns), + offsetof(struct bpf_prog_info, xlated_prog_len), + -1, + }, + [PERF_BPIL_MAP_IDS] = { + offsetof(struct bpf_prog_info, map_ids), + offsetof(struct bpf_prog_info, nr_map_ids), + -(int)sizeof(__u32), + }, + [PERF_BPIL_JITED_KSYMS] = { + offsetof(struct bpf_prog_info, jited_ksyms), + offsetof(struct bpf_prog_info, nr_jited_ksyms), + -(int)sizeof(__u64), + }, + [PERF_BPIL_JITED_FUNC_LENS] = { + offsetof(struct bpf_prog_info, jited_func_lens), + offsetof(struct bpf_prog_info, nr_jited_func_lens), + -(int)sizeof(__u32), + }, + [PERF_BPIL_FUNC_INFO] = { + offsetof(struct bpf_prog_info, func_info), + offsetof(struct bpf_prog_info, nr_func_info), + offsetof(struct bpf_prog_info, func_info_rec_size), + }, + [PERF_BPIL_LINE_INFO] = { + offsetof(struct bpf_prog_info, line_info), + offsetof(struct bpf_prog_info, nr_line_info), + offsetof(struct bpf_prog_info, line_info_rec_size), + }, + [PERF_BPIL_JITED_LINE_INFO] = { + offsetof(struct bpf_prog_info, jited_line_info), + offsetof(struct bpf_prog_info, nr_jited_line_info), + offsetof(struct bpf_prog_info, jited_line_info_rec_size), + }, + [PERF_BPIL_PROG_TAGS] = { + offsetof(struct bpf_prog_info, prog_tags), + offsetof(struct bpf_prog_info, nr_prog_tags), + -(int)sizeof(__u8) * BPF_TAG_SIZE, + }, + +}; + +static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, + int offset) +{ + __u32 *array = (__u32 *)info; + + if (offset >= 0) + return array[offset / sizeof(__u32)]; + return -(int)offset; +} + +static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, + int offset) +{ + __u64 *array = (__u64 *)info; + + if (offset >= 0) + return array[offset / sizeof(__u64)]; + return -(int)offset; +} + +static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, + __u32 val) +{ + __u32 *array = (__u32 *)info; + + if (offset >= 0) + array[offset / sizeof(__u32)] = val; +} + +static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, + __u64 val) +{ + __u64 *array = (__u64 *)info; + + if (offset >= 0) + array[offset / sizeof(__u64)] = val; +} + +struct perf_bpil * +get_bpf_prog_info_linear(int fd, __u64 arrays) +{ + struct bpf_prog_info info = {}; + struct perf_bpil *info_linear; + __u32 info_len = sizeof(info); + __u32 data_len = 0; + int i, err; + __u8 *ptr; + + if (arrays >> PERF_BPIL_LAST_ARRAY) + return ERR_PTR(-EINVAL); + + /* step 1: get array dimensions */ + err = bpf_obj_get_info_by_fd(fd, &info, &info_len); + if (err) { + pr_debug("can't get prog info: %m\n"); + return ERR_PTR(-EFAULT); + } + if (info.type >= __MAX_BPF_PROG_TYPE) + pr_debug("%s:%d: unexpected program type %u\n", __func__, __LINE__, info.type); + + /* step 2: calculate total size of all arrays */ + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + bool include_array = (arrays & (1UL << i)) > 0; + __u32 count, size; + + /* kernel is too old to support this field */ + if (info_len < desc->array_offset + sizeof(__u32) || + info_len < desc->count_offset + sizeof(__u32) || + (desc->size_offset > 0 && info_len < (__u32)desc->size_offset)) + include_array = false; + + if (!include_array) { + arrays &= ~(1UL << i); /* clear the bit */ + continue; + } + + count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); + + data_len += roundup(count * size, sizeof(__u64)); + } + + /* step 3: allocate continuous memory */ + info_linear = malloc(sizeof(struct perf_bpil) + data_len); + if (!info_linear) + return ERR_PTR(-ENOMEM); + + /* step 4: fill data to info_linear->info */ + info_linear->arrays = arrays; + memset(&info_linear->info, 0, sizeof(info)); + ptr = info_linear->data; + + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u32 count, size; + + if ((arrays & (1UL << i)) == 0) + continue; + + count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, count); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->size_offset, size); + assert(ptr >= info_linear->data); + assert(ptr < &info_linear->data[data_len]); + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, + ptr_to_u64(ptr)); + ptr += roundup(count * size, sizeof(__u64)); + } + + /* step 5: call syscall again to get required arrays */ + err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); + if (err) { + pr_debug("can't get prog info: %m\n"); + free(info_linear); + return ERR_PTR(-EFAULT); + } + if (info_linear->info.type >= __MAX_BPF_PROG_TYPE) { + pr_debug("%s:%d: unexpected program type %u\n", + __func__, __LINE__, info_linear->info.type); + } + + /* step 6: verify the data */ + ptr = info_linear->data; + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u32 count1, count2, size1, size2; + __u64 ptr2; + + if ((arrays & (1UL << i)) == 0) + continue; + + count1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); + count2 = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + if (count1 != count2) { + pr_warning("%s: mismatch in element count %u vs %u\n", __func__, count1, count2); + free(info_linear); + return ERR_PTR(-ERANGE); + } + + size1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); + size2 = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + if (size1 != size2) { + pr_warning("%s: mismatch in rec size %u vs %u\n", __func__, size1, size2); + free(info_linear); + return ERR_PTR(-ERANGE); + } + ptr2 = bpf_prog_info_read_offset_u64(&info_linear->info, desc->array_offset); + if (ptr_to_u64(ptr) != ptr2) { + pr_warning("%s: mismatch in array %p vs %llx\n", __func__, ptr, ptr2); + free(info_linear); + return ERR_PTR(-ERANGE); + } + ptr += roundup(count1 * size1, sizeof(__u64)); + } + + /* step 7: update info_len and data_len */ + info_linear->info_len = sizeof(struct bpf_prog_info); + info_linear->data_len = data_len; + + return info_linear; +} + +void bpil_addr_to_offs(struct perf_bpil *info_linear) +{ + int i; + + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u64 addr, offs; + + if ((info_linear->arrays & (1UL << i)) == 0) + continue; + + addr = bpf_prog_info_read_offset_u64(&info_linear->info, + desc->array_offset); + offs = addr - ptr_to_u64(info_linear->data); + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, offs); + } +} + +void bpil_offs_to_addr(struct perf_bpil *info_linear) +{ + int i; + + for (i = PERF_BPIL_FIRST_ARRAY; i < PERF_BPIL_LAST_ARRAY; ++i) { + const struct bpil_array_desc *desc = &bpil_array_desc[i]; + __u64 addr, offs; + __u32 count, size; + + if ((info_linear->arrays & (1UL << i)) == 0) + continue; + + offs = bpf_prog_info_read_offset_u64(&info_linear->info, + desc->array_offset); + count = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->count_offset); + size = bpf_prog_info_read_offset_u32(&info_linear->info, + desc->size_offset); + /* offset and extent from perf.data are untrusted — keep within data[] */ + if (offs >= info_linear->data_len || + (u64)count * size > info_linear->data_len - offs) { + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, 0); + bpf_prog_info_set_offset_u32(&info_linear->info, + desc->count_offset, 0); + /* clear the bit so bpil_addr_to_offs() won't reverse a zeroed address */ + info_linear->arrays &= ~(1UL << i); + continue; + } + addr = offs + ptr_to_u64(info_linear->data); + bpf_prog_info_set_offset_u64(&info_linear->info, + desc->array_offset, addr); + } +} diff --git a/tools/perf/util/bpf-utils.h b/tools/perf/util/bpf-utils.h new file mode 100644 index 000000000..a8bc1a232 --- /dev/null +++ b/tools/perf/util/bpf-utils.h @@ -0,0 +1,86 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +#ifndef __PERF_BPF_UTILS_H +#define __PERF_BPF_UTILS_H + +#define ptr_to_u64(ptr) ((__u64)(unsigned long)(ptr)) + +#ifdef HAVE_LIBBPF_SUPPORT + +#include +#include + +#define LIBBPF_CURRENT_VERSION_GEQ(major, minor) \ + (LIBBPF_MAJOR_VERSION > (major) || \ + (LIBBPF_MAJOR_VERSION == (major) && LIBBPF_MINOR_VERSION >= (minor))) + +#if LIBBPF_CURRENT_VERSION_GEQ(1, 7) +// libbpf 1.7+ support the btf_dump_type_data_opts.emit_strings option. +#define HAVE_LIBBPF_STRINGS_SUPPORT 1 +#endif + +/* + * Get bpf_prog_info in continuous memory + * + * struct bpf_prog_info has multiple arrays. The user has option to choose + * arrays to fetch from kernel. The following APIs provide an uniform way to + * fetch these data. All arrays in bpf_prog_info are stored in a single + * continuous memory region. This makes it easy to store the info in a + * file. + * + * Before writing perf_bpil to files, it is necessary to + * translate pointers in bpf_prog_info to offsets. Helper functions + * bpil_addr_to_offs() and bpil_offs_to_addr() + * are introduced to switch between pointers and offsets. + * + * Examples: + * # To fetch map_ids and prog_tags: + * __u64 arrays = (1UL << PERF_BPIL_MAP_IDS) | + * (1UL << PERF_BPIL_PROG_TAGS); + * struct perf_bpil *info_linear = + * get_bpf_prog_info_linear(fd, arrays); + * + * # To save data in file + * bpil_addr_to_offs(info_linear); + * write(f, info_linear, sizeof(*info_linear) + info_linear->data_len); + * + * # To read data from file + * read(f, info_linear, ); + * bpil_offs_to_addr(info_linear); + */ +enum perf_bpil_array_types { + PERF_BPIL_FIRST_ARRAY = 0, + PERF_BPIL_JITED_INSNS = 0, + PERF_BPIL_XLATED_INSNS, + PERF_BPIL_MAP_IDS, + PERF_BPIL_JITED_KSYMS, + PERF_BPIL_JITED_FUNC_LENS, + PERF_BPIL_FUNC_INFO, + PERF_BPIL_LINE_INFO, + PERF_BPIL_JITED_LINE_INFO, + PERF_BPIL_PROG_TAGS, + PERF_BPIL_LAST_ARRAY, +}; + +struct perf_bpil { + /* size of struct bpf_prog_info, when the tool is compiled */ + __u32 info_len; + /* total bytes allocated for data, round up to 8 bytes */ + __u32 data_len; + /* which arrays are included in data */ + __u64 arrays; + struct bpf_prog_info info; + __u8 data[]; +}; + +struct perf_bpil * +get_bpf_prog_info_linear(int fd, __u64 arrays); + +void +bpil_addr_to_offs(struct perf_bpil *info_linear); + +void +bpil_offs_to_addr(struct perf_bpil *info_linear); + +#endif /* HAVE_LIBBPF_SUPPORT */ +#endif /* __PERF_BPF_UTILS_H */ diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c new file mode 100644 index 000000000..34b6b0da1 --- /dev/null +++ b/tools/perf/util/bpf_counter.c @@ -0,0 +1,902 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (c) 2019 Facebook */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "bpf_counter.h" +#include "bpf-utils.h" +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "evlist.h" +#include "target.h" +#include "cgroup.h" +#include "cpumap.h" +#include "thread_map.h" + +#include "bpf_skel/bpf_prog_profiler.skel.h" +#include "bpf_skel/bperf_u.h" +#include "bpf_skel/bperf_leader.skel.h" +#include "bpf_skel/bperf_follower.skel.h" + +struct bpf_counter { + void *skel; + struct list_head list; +}; + +#define ATTR_MAP_SIZE 16 + +static void *u64_to_ptr(__u64 ptr) +{ + return (void *)(unsigned long)ptr; +} + + +void set_max_rlimit(void) +{ + struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; + + setrlimit(RLIMIT_MEMLOCK, &rinf); +} + +static __u32 bpf_link_get_id(int fd) +{ + struct bpf_link_info link_info = { .id = 0, }; + __u32 link_info_len = sizeof(link_info); + + bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); + return link_info.id; +} + +static __u32 bpf_link_get_prog_id(int fd) +{ + struct bpf_link_info link_info = { .id = 0, }; + __u32 link_info_len = sizeof(link_info); + + bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); + return link_info.prog_id; +} + +static __u32 bpf_map_get_id(int fd) +{ + struct bpf_map_info map_info = { .id = 0, }; + __u32 map_info_len = sizeof(map_info); + + bpf_obj_get_info_by_fd(fd, &map_info, &map_info_len); + return map_info.id; +} + +/* trigger the leader program on a cpu */ +int bperf_trigger_reading(int prog_fd, int cpu) +{ + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = NULL, + .ctx_size_in = 0, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = cpu, + .retval = 0, + ); + + return bpf_prog_test_run_opts(prog_fd, &opts); +} + +static struct bpf_counter *bpf_counter_alloc(void) +{ + struct bpf_counter *counter; + + counter = zalloc(sizeof(*counter)); + if (counter) + INIT_LIST_HEAD(&counter->list); + return counter; +} + +static int bpf_program_profiler__destroy(struct evsel *evsel) +{ + struct bpf_counter *counter, *tmp; + + list_for_each_entry_safe(counter, tmp, + &evsel->bpf_counter_list, list) { + list_del_init(&counter->list); + bpf_prog_profiler_bpf__destroy(counter->skel); + free(counter); + } + assert(list_empty(&evsel->bpf_counter_list)); + + return 0; +} + +static char *bpf_target_prog_name(int tgt_fd) +{ + struct bpf_func_info *func_info; + struct perf_bpil *info_linear; + const struct btf_type *t; + struct btf *btf = NULL; + char *name = NULL; + + info_linear = get_bpf_prog_info_linear(tgt_fd, 1UL << PERF_BPIL_FUNC_INFO); + if (IS_ERR_OR_NULL(info_linear)) { + pr_debug("failed to get info_linear for prog FD %d\n", tgt_fd); + return NULL; + } + + if (info_linear->info.btf_id == 0) { + pr_debug("prog FD %d doesn't have valid btf\n", tgt_fd); + goto out; + } + + btf = btf__load_from_kernel_by_id(info_linear->info.btf_id); + if (libbpf_get_error(btf)) { + pr_debug("failed to load btf for prog FD %d\n", tgt_fd); + goto out; + } + + func_info = u64_to_ptr(info_linear->info.func_info); + t = btf__type_by_id(btf, func_info[0].type_id); + if (!t) { + pr_debug("btf %d doesn't have type %d\n", + info_linear->info.btf_id, func_info[0].type_id); + goto out; + } + name = strdup(btf__name_by_offset(btf, t->name_off)); +out: + btf__free(btf); + free(info_linear); + return name; +} + +static int bpf_program_profiler_load_one(struct evsel *evsel, u32 prog_id) +{ + struct bpf_prog_profiler_bpf *skel; + struct bpf_counter *counter; + struct bpf_program *prog; + char *prog_name = NULL; + int prog_fd; + int err; + + prog_fd = bpf_prog_get_fd_by_id(prog_id); + if (prog_fd < 0) { + pr_err("Failed to open fd for bpf prog %u\n", prog_id); + return -1; + } + counter = bpf_counter_alloc(); + if (!counter) { + close(prog_fd); + return -1; + } + + skel = bpf_prog_profiler_bpf__open(); + if (!skel) { + pr_err("Failed to open bpf skeleton\n"); + goto err_out; + } + + skel->rodata->num_cpu = evsel__nr_cpus(evsel); + + bpf_map__set_max_entries(skel->maps.events, evsel__nr_cpus(evsel)); + bpf_map__set_max_entries(skel->maps.fentry_readings, 1); + bpf_map__set_max_entries(skel->maps.accum_readings, 1); + + prog_name = bpf_target_prog_name(prog_fd); + if (!prog_name) { + pr_err("Failed to get program name for bpf prog %u. Does it have BTF?\n", prog_id); + goto err_out; + } + + bpf_object__for_each_program(prog, skel->obj) { + err = bpf_program__set_attach_target(prog, prog_fd, prog_name); + if (err) { + pr_err("bpf_program__set_attach_target failed.\n" + "Does bpf prog %u have BTF?\n", prog_id); + goto err_out; + } + } + set_max_rlimit(); + err = bpf_prog_profiler_bpf__load(skel); + if (err) { + pr_err("bpf_prog_profiler_bpf__load failed\n"); + goto err_out; + } + + assert(skel != NULL); + counter->skel = skel; + list_add(&counter->list, &evsel->bpf_counter_list); + free(prog_name); + close(prog_fd); + return 0; +err_out: + bpf_prog_profiler_bpf__destroy(skel); + free(prog_name); + free(counter); + close(prog_fd); + return -1; +} + +static int bpf_program_profiler__load(struct evsel *evsel, struct target *target) +{ + char *bpf_str, *bpf_str_, *tok, *saveptr = NULL, *p; + u32 prog_id; + int ret; + + bpf_str_ = bpf_str = strdup(target->bpf_str); + if (!bpf_str) + return -1; + + while ((tok = strtok_r(bpf_str, ",", &saveptr)) != NULL) { + prog_id = strtoul(tok, &p, 10); + if (prog_id == 0 || prog_id == UINT_MAX || + (*p != '\0' && *p != ',')) { + pr_err("Failed to parse bpf prog ids %s\n", + target->bpf_str); + free(bpf_str_); + return -1; + } + + ret = bpf_program_profiler_load_one(evsel, prog_id); + if (ret) { + bpf_program_profiler__destroy(evsel); + free(bpf_str_); + return -1; + } + bpf_str = NULL; + } + free(bpf_str_); + return 0; +} + +static int bpf_program_profiler__enable(struct evsel *evsel) +{ + struct bpf_counter *counter; + int ret; + + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + assert(counter->skel != NULL); + ret = bpf_prog_profiler_bpf__attach(counter->skel); + if (ret) { + bpf_program_profiler__destroy(evsel); + return ret; + } + } + return 0; +} + +static int bpf_program_profiler__disable(struct evsel *evsel) +{ + struct bpf_counter *counter; + + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + assert(counter->skel != NULL); + bpf_prog_profiler_bpf__detach(counter->skel); + } + return 0; +} + +static int bpf_program_profiler__read(struct evsel *evsel) +{ + // BPF_MAP_TYPE_PERCPU_ARRAY uses /sys/devices/system/cpu/possible + // Sometimes possible > online, like on a Ryzen 3900X that has 24 + // threads but its possible showed 0-31 -acme + int num_cpu_bpf = libbpf_num_possible_cpus(); + struct bpf_perf_event_value values[num_cpu_bpf]; + struct bpf_counter *counter; + struct perf_counts_values *counts; + int reading_map_fd; + __u32 key = 0; + int err, bpf_cpu; + unsigned int idx; + + if (list_empty(&evsel->bpf_counter_list)) + return -EAGAIN; + + perf_cpu_map__for_each_idx(idx, evsel__cpus(evsel)) { + counts = perf_counts(evsel->counts, idx, 0); + counts->val = 0; + counts->ena = 0; + counts->run = 0; + } + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + struct bpf_prog_profiler_bpf *skel = counter->skel; + + assert(skel != NULL); + reading_map_fd = bpf_map__fd(skel->maps.accum_readings); + + err = bpf_map_lookup_elem(reading_map_fd, &key, values); + if (err) { + pr_err("failed to read value\n"); + return err; + } + + for (bpf_cpu = 0; bpf_cpu < num_cpu_bpf; bpf_cpu++) { + int i = perf_cpu_map__idx(evsel__cpus(evsel), + (struct perf_cpu){.cpu = bpf_cpu}); + + if (i == -1) + continue; + counts = perf_counts(evsel->counts, i, 0); + counts->val += values[bpf_cpu].counter; + counts->ena += values[bpf_cpu].enabled; + counts->run += values[bpf_cpu].running; + } + } + return 0; +} + +static int bpf_program_profiler__install_pe(struct evsel *evsel, int cpu_map_idx, + int fd) +{ + struct bpf_prog_profiler_bpf *skel; + struct bpf_counter *counter; + int cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx).cpu; + int ret; + + list_for_each_entry(counter, &evsel->bpf_counter_list, list) { + skel = counter->skel; + assert(skel != NULL); + + ret = bpf_map_update_elem(bpf_map__fd(skel->maps.events), + &cpu, &fd, BPF_ANY); + if (ret) + return ret; + } + return 0; +} + +static struct bpf_counter_ops bpf_program_profiler_ops = { + .load = bpf_program_profiler__load, + .enable = bpf_program_profiler__enable, + .disable = bpf_program_profiler__disable, + .read = bpf_program_profiler__read, + .destroy = bpf_program_profiler__destroy, + .install_pe = bpf_program_profiler__install_pe, +}; + +static bool bperf_attr_map_compatible(int attr_map_fd) +{ + struct bpf_map_info map_info = {0}; + __u32 map_info_len = sizeof(map_info); + int err; + + err = bpf_obj_get_info_by_fd(attr_map_fd, &map_info, &map_info_len); + + if (err) + return false; + return (map_info.key_size == sizeof(struct perf_event_attr)) && + (map_info.value_size == sizeof(struct perf_event_attr_map_entry)); +} + +static int bperf_lock_attr_map(struct target *target) +{ + char path[PATH_MAX]; + int map_fd, err; + + if (target->attr_map) { + scnprintf(path, PATH_MAX, "%s", target->attr_map); + } else { + scnprintf(path, PATH_MAX, "%s/fs/bpf/%s", sysfs__mountpoint(), + BPF_PERF_DEFAULT_ATTR_MAP_PATH); + } + + if (access(path, F_OK)) { + map_fd = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, + sizeof(struct perf_event_attr), + sizeof(struct perf_event_attr_map_entry), + ATTR_MAP_SIZE, NULL); + if (map_fd < 0) + return -1; + + err = bpf_obj_pin(map_fd, path); + if (err) { + /* someone pinned the map in parallel? */ + close(map_fd); + map_fd = bpf_obj_get(path); + if (map_fd < 0) + return -1; + } + } else { + map_fd = bpf_obj_get(path); + if (map_fd < 0) + return -1; + } + + if (!bperf_attr_map_compatible(map_fd)) { + close(map_fd); + return -1; + + } + err = flock(map_fd, LOCK_EX); + if (err) { + close(map_fd); + return -1; + } + return map_fd; +} + +static int bperf_check_target(struct evsel *evsel, + struct target *target, + enum bperf_filter_type *filter_type, + __u32 *filter_entry_cnt) +{ + if (evsel->core.leader->nr_members > 1) { + pr_err("bpf managed perf events do not yet support groups.\n"); + return -1; + } + + /* determine filter type based on target */ + if (target->system_wide) { + *filter_type = BPERF_FILTER_GLOBAL; + *filter_entry_cnt = 1; + } else if (target->cpu_list) { + *filter_type = BPERF_FILTER_CPU; + *filter_entry_cnt = perf_cpu_map__nr(evsel__cpus(evsel)); + } else if (target->tid) { + *filter_type = BPERF_FILTER_PID; + *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); + } else if (target->pid || evsel->evlist->workload.pid != -1) { + *filter_type = BPERF_FILTER_TGID; + *filter_entry_cnt = perf_thread_map__nr(evsel->core.threads); + } else { + pr_err("bpf managed perf events do not yet support these targets.\n"); + return -1; + } + + return 0; +} + +static __u32 filter_entry_cnt; + +static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, + struct perf_event_attr_map_entry *entry) +{ + struct bperf_leader_bpf *skel = bperf_leader_bpf__open(); + int link_fd, diff_map_fd, err; + struct bpf_link *link = NULL; + struct perf_thread_map *threads; + + if (!skel) { + pr_err("Failed to open leader skeleton\n"); + return -1; + } + + bpf_map__set_max_entries(skel->maps.events, libbpf_num_possible_cpus()); + err = bperf_leader_bpf__load(skel); + if (err) { + pr_err("Failed to load leader skeleton\n"); + goto out; + } + + link = bpf_program__attach(skel->progs.on_switch); + if (IS_ERR(link)) { + pr_err("Failed to attach leader program\n"); + err = PTR_ERR(link); + goto out; + } + + link_fd = bpf_link__fd(link); + diff_map_fd = bpf_map__fd(skel->maps.diff_readings); + entry->link_id = bpf_link_get_id(link_fd); + entry->diff_map_id = bpf_map_get_id(diff_map_fd); + err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, entry, BPF_ANY); + assert(err == 0); + + evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry->link_id); + assert(evsel->bperf_leader_link_fd >= 0); + + /* + * save leader_skel for install_pe, which is called within + * following evsel__open_per_cpu call + */ + evsel->leader_skel = skel; + assert(!perf_cpu_map__has_any_cpu_or_is_empty(evsel->core.cpus)); + /* Always open system wide. */ + threads = thread_map__new_by_tid(-1); + evsel__open(evsel, evsel->core.cpus, threads); + perf_thread_map__put(threads); + +out: + bperf_leader_bpf__destroy(skel); + bpf_link__destroy(link); + return err; +} + +static int bperf_attach_follower_program(struct bperf_follower_bpf *skel, + enum bperf_filter_type filter_type, + bool inherit) +{ + struct bpf_link *link; + int err = 0; + + if ((filter_type == BPERF_FILTER_PID || + filter_type == BPERF_FILTER_TGID) && inherit) + /* attach all follower bpf progs to enable event inheritance */ + err = bperf_follower_bpf__attach(skel); + else { + link = bpf_program__attach(skel->progs.fexit_XXX); + if (IS_ERR(link)) + err = PTR_ERR(link); + } + + return err; +} + +static int bperf__load(struct evsel *evsel, struct target *target) +{ + struct perf_event_attr_map_entry entry = {0xffffffff, 0xffffffff}; + int attr_map_fd, diff_map_fd = -1, err; + enum bperf_filter_type filter_type; + __u32 i; + + if (bperf_check_target(evsel, target, &filter_type, &filter_entry_cnt)) + return -1; + + evsel->bperf_leader_prog_fd = -1; + evsel->bperf_leader_link_fd = -1; + + /* + * Step 1: hold a fd on the leader program and the bpf_link, if + * the program is not already gone, reload the program. + * Use flock() to ensure exclusive access to the perf_event_attr + * map. + */ + attr_map_fd = bperf_lock_attr_map(target); + if (attr_map_fd < 0) { + pr_err("Failed to lock perf_event_attr map\n"); + return -1; + } + + err = bpf_map_lookup_elem(attr_map_fd, &evsel->core.attr, &entry); + if (err) { + err = bpf_map_update_elem(attr_map_fd, &evsel->core.attr, &entry, BPF_ANY); + if (err) + goto out; + } + + evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id); + if (evsel->bperf_leader_link_fd < 0 && + bperf_reload_leader_program(evsel, attr_map_fd, &entry)) { + err = -1; + goto out; + } + /* + * The bpf_link holds reference to the leader program, and the + * leader program holds reference to the maps. Therefore, if + * link_id is valid, diff_map_id should also be valid. + */ + evsel->bperf_leader_prog_fd = bpf_prog_get_fd_by_id( + bpf_link_get_prog_id(evsel->bperf_leader_link_fd)); + assert(evsel->bperf_leader_prog_fd >= 0); + + diff_map_fd = bpf_map_get_fd_by_id(entry.diff_map_id); + assert(diff_map_fd >= 0); + + /* + * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check + * whether the kernel support it + */ + err = bperf_trigger_reading(evsel->bperf_leader_prog_fd, 0); + if (err) { + pr_err("The kernel does not support test_run for raw_tp BPF programs.\n" + "Therefore, --use-bpf might show inaccurate readings\n"); + goto out; + } + + /* Step 2: load the follower skeleton */ + evsel->follower_skel = bperf_follower_bpf__open(); + if (!evsel->follower_skel) { + err = -1; + pr_err("Failed to open follower skeleton\n"); + goto out; + } + + /* attach fexit program to the leader program */ + bpf_program__set_attach_target(evsel->follower_skel->progs.fexit_XXX, + evsel->bperf_leader_prog_fd, "on_switch"); + + /* connect to leader diff_reading map */ + bpf_map__reuse_fd(evsel->follower_skel->maps.diff_readings, diff_map_fd); + + /* set up reading map */ + bpf_map__set_max_entries(evsel->follower_skel->maps.accum_readings, + filter_entry_cnt); + err = bperf_follower_bpf__load(evsel->follower_skel); + if (err) { + pr_err("Failed to load follower skeleton\n"); + bperf_follower_bpf__destroy(evsel->follower_skel); + evsel->follower_skel = NULL; + goto out; + } + + for (i = 0; i < filter_entry_cnt; i++) { + int filter_map_fd; + __u32 key; + struct bperf_filter_value fval = { i, 0 }; + + if (filter_type == BPERF_FILTER_PID || + filter_type == BPERF_FILTER_TGID) + key = perf_thread_map__pid(evsel->core.threads, i); + else if (filter_type == BPERF_FILTER_CPU) + key = perf_cpu_map__cpu(evsel->core.cpus, i).cpu; + else + break; + + filter_map_fd = bpf_map__fd(evsel->follower_skel->maps.filter); + bpf_map_update_elem(filter_map_fd, &key, &fval, BPF_ANY); + } + + evsel->follower_skel->bss->type = filter_type; + evsel->follower_skel->bss->inherit = target->inherit; + + err = bperf_attach_follower_program(evsel->follower_skel, filter_type, + target->inherit); + +out: + if (err && evsel->bperf_leader_link_fd >= 0) + close(evsel->bperf_leader_link_fd); + if (err && evsel->bperf_leader_prog_fd >= 0) + close(evsel->bperf_leader_prog_fd); + if (diff_map_fd >= 0) + close(diff_map_fd); + + flock(attr_map_fd, LOCK_UN); + close(attr_map_fd); + + return err; +} + +static int bperf__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) +{ + struct bperf_leader_bpf *skel = evsel->leader_skel; + int cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx).cpu; + + return bpf_map_update_elem(bpf_map__fd(skel->maps.events), + &cpu, &fd, BPF_ANY); +} + +/* + * trigger the leader prog on each cpu, so the accum_reading map could get + * the latest readings. + */ +static int bperf_sync_counters(struct evsel *evsel) +{ + struct perf_cpu cpu; + unsigned int idx; + + perf_cpu_map__for_each_cpu(cpu, idx, evsel->core.cpus) + bperf_trigger_reading(evsel->bperf_leader_prog_fd, cpu.cpu); + + return 0; +} + +static int bperf__enable(struct evsel *evsel) +{ + evsel->follower_skel->bss->enabled = 1; + return 0; +} + +static int bperf__disable(struct evsel *evsel) +{ + evsel->follower_skel->bss->enabled = 0; + return 0; +} + +static int bperf__read(struct evsel *evsel) +{ + struct bperf_follower_bpf *skel = evsel->follower_skel; + __u32 num_cpu_bpf = cpu__max_cpu().cpu; + struct bpf_perf_event_value values[num_cpu_bpf]; + struct perf_counts_values *counts; + int reading_map_fd, err = 0; + + bperf_sync_counters(evsel); + reading_map_fd = bpf_map__fd(skel->maps.accum_readings); + + for (__u32 i = 0; i < filter_entry_cnt; i++) { + struct perf_cpu entry; + __u32 cpu; + + err = bpf_map_lookup_elem(reading_map_fd, &i, values); + if (err) + goto out; + switch (evsel->follower_skel->bss->type) { + case BPERF_FILTER_GLOBAL: { + unsigned int j; + + assert(i == 0); + perf_cpu_map__for_each_cpu(entry, j, evsel__cpus(evsel)) { + counts = perf_counts(evsel->counts, j, 0); + counts->val = values[entry.cpu].counter; + counts->ena = values[entry.cpu].enabled; + counts->run = values[entry.cpu].running; + } + break; + } + case BPERF_FILTER_CPU: + cpu = perf_cpu_map__cpu(evsel__cpus(evsel), i).cpu; + assert(cpu >= 0); + counts = perf_counts(evsel->counts, i, 0); + counts->val = values[cpu].counter; + counts->ena = values[cpu].enabled; + counts->run = values[cpu].running; + break; + case BPERF_FILTER_PID: + case BPERF_FILTER_TGID: + counts = perf_counts(evsel->counts, 0, i); + counts->val = 0; + counts->ena = 0; + counts->run = 0; + + for (cpu = 0; cpu < num_cpu_bpf; cpu++) { + counts->val += values[cpu].counter; + counts->ena += values[cpu].enabled; + counts->run += values[cpu].running; + } + break; + default: + break; + } + } +out: + return err; +} + +static int bperf__destroy(struct evsel *evsel) +{ + bperf_follower_bpf__destroy(evsel->follower_skel); + close(evsel->bperf_leader_prog_fd); + close(evsel->bperf_leader_link_fd); + return 0; +} + +/* + * bperf: share hardware PMCs with BPF + * + * perf uses performance monitoring counters (PMC) to monitor system + * performance. The PMCs are limited hardware resources. For example, + * Intel CPUs have 3x fixed PMCs and 4x programmable PMCs per cpu. + * + * Modern data center systems use these PMCs in many different ways: + * system level monitoring, (maybe nested) container level monitoring, per + * process monitoring, profiling (in sample mode), etc. In some cases, + * there are more active perf_events than available hardware PMCs. To allow + * all perf_events to have a chance to run, it is necessary to do expensive + * time multiplexing of events. + * + * On the other hand, many monitoring tools count the common metrics + * (cycles, instructions). It is a waste to have multiple tools create + * multiple perf_events of "cycles" and occupy multiple PMCs. + * + * bperf tries to reduce such wastes by allowing multiple perf_events of + * "cycles" or "instructions" (at different scopes) to share PMUs. Instead + * of having each perf-stat session to read its own perf_events, bperf uses + * BPF programs to read the perf_events and aggregate readings to BPF maps. + * Then, the perf-stat session(s) reads the values from these BPF maps. + * + * || + * shared progs and maps <- || -> per session progs and maps + * || + * --------------- || + * | perf_events | || + * --------------- fexit || ----------------- + * | --------||----> | follower prog | + * --------------- / || --- ----------------- + * cs -> | leader prog |/ ||/ | | + * --> --------------- /|| -------------- ------------------ + * / | | / || | filter map | | accum_readings | + * / ------------ ------------ || -------------- ------------------ + * | | prev map | | diff map | || | + * | ------------ ------------ || | + * \ || | + * = \ ==================================================== | ============ + * \ / user space + * \ / + * \ / + * BPF_PROG_TEST_RUN BPF_MAP_LOOKUP_ELEM + * \ / + * \ / + * \------ perf-stat ----------------------/ + * + * The figure above shows the architecture of bperf. Note that the figure + * is divided into 3 regions: shared progs and maps (top left), per session + * progs and maps (top right), and user space (bottom). + * + * The leader prog is triggered on each context switch (cs). The leader + * prog reads perf_events and stores the difference (current_reading - + * previous_reading) to the diff map. For the same metric, e.g. "cycles", + * multiple perf-stat sessions share the same leader prog. + * + * Each perf-stat session creates a follower prog as fexit program to the + * leader prog. It is possible to attach up to BPF_MAX_TRAMP_PROGS (38) + * follower progs to the same leader prog. The follower prog checks current + * task and processor ID to decide whether to add the value from the diff + * map to its accumulated reading map (accum_readings). + * + * Finally, perf-stat user space reads the value from accum_reading map. + * + * Besides context switch, it is also necessary to trigger the leader prog + * before perf-stat reads the value. Otherwise, the accum_reading map may + * not have the latest reading from the perf_events. This is achieved by + * triggering the event via sys_bpf(BPF_PROG_TEST_RUN) to each CPU. + * + * Comment before the definition of struct perf_event_attr_map_entry + * describes how different sessions of perf-stat share information about + * the leader prog. + */ + +static struct bpf_counter_ops bperf_ops = { + .load = bperf__load, + .enable = bperf__enable, + .disable = bperf__disable, + .read = bperf__read, + .install_pe = bperf__install_pe, + .destroy = bperf__destroy, +}; + +extern struct bpf_counter_ops bperf_cgrp_ops; + +static bool bpf_counter_skip(struct evsel *evsel) +{ + return evsel->bpf_counter_ops == NULL; +} + +int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd) +{ + if (bpf_counter_skip(evsel)) + return 0; + return evsel->bpf_counter_ops->install_pe(evsel, cpu_map_idx, fd); +} + +int bpf_counter__load(struct evsel *evsel, struct target *target) +{ + if (target->bpf_str) + evsel->bpf_counter_ops = &bpf_program_profiler_ops; + else if (cgrp_event_expanded && target->use_bpf) + evsel->bpf_counter_ops = &bperf_cgrp_ops; + else if (target->use_bpf || evsel->bpf_counter || + evsel__match_bpf_counter_events(evsel->name)) + evsel->bpf_counter_ops = &bperf_ops; + + if (evsel->bpf_counter_ops) + return evsel->bpf_counter_ops->load(evsel, target); + return 0; +} + +int bpf_counter__enable(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return 0; + return evsel->bpf_counter_ops->enable(evsel); +} + +int bpf_counter__disable(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return 0; + return evsel->bpf_counter_ops->disable(evsel); +} + +int bpf_counter__read(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return -EAGAIN; + return evsel->bpf_counter_ops->read(evsel); +} + +void bpf_counter__destroy(struct evsel *evsel) +{ + if (bpf_counter_skip(evsel)) + return; + evsel->bpf_counter_ops->destroy(evsel); + evsel->bpf_counter_ops = NULL; + evsel->bpf_skel = NULL; +} diff --git a/tools/perf/util/bpf_counter.h b/tools/perf/util/bpf_counter.h new file mode 100644 index 000000000..658d8e7d5 --- /dev/null +++ b/tools/perf/util/bpf_counter.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_BPF_COUNTER_H +#define __PERF_BPF_COUNTER_H 1 + +struct evsel; +struct target; + +#ifdef HAVE_BPF_SKEL + +typedef int (*bpf_counter_evsel_op)(struct evsel *evsel); +typedef int (*bpf_counter_evsel_target_op)(struct evsel *evsel, + struct target *target); +typedef int (*bpf_counter_evsel_install_pe_op)(struct evsel *evsel, + int cpu_map_idx, + int fd); + +/* Shared ops between bpf_counter, bpf_counter_cgroup, etc. */ +struct bpf_counter_ops { + bpf_counter_evsel_target_op load; + bpf_counter_evsel_op enable; + bpf_counter_evsel_op disable; + bpf_counter_evsel_op read; + bpf_counter_evsel_op destroy; + bpf_counter_evsel_install_pe_op install_pe; +}; + +int bpf_counter__load(struct evsel *evsel, struct target *target); +int bpf_counter__enable(struct evsel *evsel); +int bpf_counter__disable(struct evsel *evsel); +int bpf_counter__read(struct evsel *evsel); +void bpf_counter__destroy(struct evsel *evsel); +int bpf_counter__install_pe(struct evsel *evsel, int cpu_map_idx, int fd); + +int bperf_trigger_reading(int prog_fd, int cpu); +void set_max_rlimit(void); + +#else /* HAVE_BPF_SKEL */ + +#include + +static inline int bpf_counter__load(struct evsel *evsel __maybe_unused, + struct target *target __maybe_unused) +{ + return 0; +} + +static inline int bpf_counter__enable(struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static inline int bpf_counter__disable(struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static inline int bpf_counter__read(struct evsel *evsel __maybe_unused) +{ + return -EAGAIN; +} + +static inline void bpf_counter__destroy(struct evsel *evsel __maybe_unused) +{ +} + +static inline int bpf_counter__install_pe(struct evsel *evsel __maybe_unused, + int cpu __maybe_unused, + int fd __maybe_unused) +{ + return 0; +} + +#endif /* HAVE_BPF_SKEL */ + +#endif /* __PERF_BPF_COUNTER_H */ diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c new file mode 100644 index 000000000..e1ce5aa3b --- /dev/null +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -0,0 +1,351 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (c) 2021 Facebook */ +/* Copyright (c) 2021 Google */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "affinity.h" +#include "bpf_counter.h" +#include "cgroup.h" +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "evlist.h" +#include "target.h" +#include "cpumap.h" +#include "thread_map.h" + +#include "bpf_skel/bperf_cgroup.h" +#include "bpf_skel/bperf_cgroup.skel.h" + +static struct perf_event_attr cgrp_switch_attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CGROUP_SWITCHES, + .size = sizeof(cgrp_switch_attr), + .sample_period = 1, + .disabled = 1, +}; + +static struct evsel *cgrp_switch; +static struct bperf_cgroup_bpf *skel; + +#define FD(evt, cpu) (*(int *)xyarray__entry(evt->core.fd, cpu, 0)) + +static void setup_rodata(struct bperf_cgroup_bpf *sk, int evlist_size) +{ + int map_size, total_cpus = cpu__max_cpu().cpu; + + sk->rodata->num_cpus = total_cpus; + sk->rodata->num_events = evlist_size / nr_cgroups; + + if (cgroup_is_v2("perf_event") > 0) + sk->rodata->use_cgroup_v2 = 1; + + BUG_ON(evlist_size % nr_cgroups != 0); + + /* we need one copy of events per cpu for reading */ + map_size = total_cpus * evlist_size / nr_cgroups; + bpf_map__set_max_entries(sk->maps.events, map_size); + bpf_map__set_max_entries(sk->maps.cgrp_idx, nr_cgroups); + /* previous result is saved in a per-cpu array */ + map_size = evlist_size / nr_cgroups; + bpf_map__set_max_entries(sk->maps.prev_readings, map_size); + /* cgroup result needs all events (per-cpu) */ + map_size = evlist_size; + bpf_map__set_max_entries(sk->maps.cgrp_readings, map_size); +} + +static void test_max_events_program_load(void) +{ +#ifndef NDEBUG + /* + * Test that the program verifies with the maximum number of events. If + * this test fails unfortunately perf needs recompiling with a lower + * BPERF_CGROUP__MAX_EVENTS to avoid BPF verifier issues. + */ + int err, max_events = BPERF_CGROUP__MAX_EVENTS * nr_cgroups; + struct bperf_cgroup_bpf *test_skel = bperf_cgroup_bpf__open(); + + if (!test_skel) { + pr_err("Failed to open cgroup skeleton\n"); + return; + } + setup_rodata(test_skel, max_events); + err = bperf_cgroup_bpf__load(test_skel); + if (err) { + pr_err("Failed to load cgroup skeleton with max events %d.\n", + BPERF_CGROUP__MAX_EVENTS); + } + bperf_cgroup_bpf__destroy(test_skel); +#endif +} + +static int bperf_load_program(struct evlist *evlist) +{ + struct bpf_link *link; + struct evsel *evsel; + struct cgroup *cgrp, *leader_cgrp; + unsigned int i; + struct perf_cpu cpu; + int total_cpus = cpu__max_cpu().cpu; + int map_fd, prog_fd, err; + + set_max_rlimit(); + + if (nr_cgroups == 0 || evlist->core.nr_entries % nr_cgroups != 0) { + pr_err("Invalid cgroup or event count\n"); + return -EINVAL; + } + + test_max_events_program_load(); + + skel = bperf_cgroup_bpf__open(); + if (!skel) { + pr_err("Failed to open cgroup skeleton\n"); + return -1; + } + setup_rodata(skel, evlist->core.nr_entries); + + err = bperf_cgroup_bpf__load(skel); + if (err) { + pr_err("Failed to load cgroup skeleton\n"); + goto out; + } + + err = -1; + + cgrp_switch = evsel__new(&cgrp_switch_attr); + if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) { + pr_err("Failed to open cgroup switches event\n"); + goto out; + } + + perf_cpu_map__for_each_cpu(cpu, i, evlist->core.all_cpus) { + link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, + FD(cgrp_switch, i)); + if (IS_ERR(link)) { + pr_err("Failed to attach cgroup program\n"); + err = PTR_ERR(link); + goto out; + } + } + + /* + * Update cgrp_idx map from cgroup-id to event index. + */ + cgrp = NULL; + i = 0; + + evlist__for_each_entry(evlist, evsel) { + if (cgrp == NULL || evsel->cgrp == leader_cgrp) { + unsigned int j; + + leader_cgrp = evsel->cgrp; + evsel->cgrp = NULL; + + /* open single copy of the events w/o cgroup */ + err = evsel__open_per_cpu(evsel, evsel->core.cpus, -1); + if (err == 0) + evsel->supported = true; + + map_fd = bpf_map__fd(skel->maps.events); + perf_cpu_map__for_each_cpu(cpu, j, evsel->core.cpus) { + int fd = FD(evsel, j); + __u32 idx = evsel->core.idx * total_cpus + cpu.cpu; + + bpf_map_update_elem(map_fd, &idx, &fd, BPF_ANY); + } + + evsel->cgrp = leader_cgrp; + } + + if (evsel->cgrp == cgrp) + continue; + + cgrp = evsel->cgrp; + + if (read_cgroup_id(cgrp) < 0) { + pr_debug("Failed to get cgroup id for %s\n", cgrp->name); + cgrp->id = 0; + } + + map_fd = bpf_map__fd(skel->maps.cgrp_idx); + err = bpf_map_update_elem(map_fd, &cgrp->id, &i, BPF_ANY); + if (err < 0) { + pr_err("Failed to update cgroup index map\n"); + goto out; + } + + i++; + } + + /* + * Propagate supported flag from leaders to followers. Follower events + * are not opened, so their supported flag remains false. + */ + { + struct evsel *leader; + int num_events = evlist->core.nr_entries / nr_cgroups; + + evlist__for_each_entry(evlist, evsel) { + leader = evlist__find_evsel(evlist, evsel->core.idx % num_events); + if (leader) + evsel->supported = leader->supported; + } + } + + /* + * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check + * whether the kernel support it + */ + prog_fd = bpf_program__fd(skel->progs.trigger_read); + err = bperf_trigger_reading(prog_fd, 0); + if (err) { + pr_warning("The kernel does not support test_run for raw_tp BPF programs.\n" + "Therefore, --for-each-cgroup might show inaccurate readings\n"); + err = 0; + } + +out: + return err; +} + +static int bperf_cgrp__load(struct evsel *evsel, + struct target *target __maybe_unused) +{ + static bool bperf_loaded = false; + + evsel->bperf_leader_prog_fd = -1; + evsel->bperf_leader_link_fd = -1; + + if (!bperf_loaded && bperf_load_program(evsel->evlist)) + return -1; + + bperf_loaded = true; + /* just to bypass bpf_counter_skip() */ + evsel->follower_skel = (struct bperf_follower_bpf *)skel; + + return 0; +} + +static int bperf_cgrp__install_pe(struct evsel *evsel __maybe_unused, + int cpu_map_idx __maybe_unused, + int fd __maybe_unused) +{ + /* nothing to do */ + return 0; +} + +/* + * trigger the leader prog on each cpu, so the cgrp_reading map could get + * the latest results. + */ +static int bperf_cgrp__sync_counters(struct evlist *evlist) +{ + struct perf_cpu cpu; + unsigned int idx; + int prog_fd = bpf_program__fd(skel->progs.trigger_read); + + perf_cpu_map__for_each_cpu(cpu, idx, evlist->core.all_cpus) + bperf_trigger_reading(prog_fd, cpu.cpu); + + return 0; +} + +static int bperf_cgrp__enable(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + skel->bss->enabled = 1; + return 0; +} + +static int bperf_cgrp__disable(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + skel->bss->enabled = 0; + return 0; +} + +static int bperf_cgrp__read(struct evsel *evsel) +{ + struct evlist *evlist = evsel->evlist; + int total_cpus = cpu__max_cpu().cpu; + struct perf_counts_values *counts; + struct bpf_perf_event_value *values; + int reading_map_fd, err = 0; + + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + values = calloc(total_cpus, sizeof(*values)); + if (values == NULL) + return -ENOMEM; + + reading_map_fd = bpf_map__fd(skel->maps.cgrp_readings); + + evlist__for_each_entry(evlist, evsel) { + __u32 idx = evsel->core.idx; + unsigned int i; + struct perf_cpu cpu; + + err = bpf_map_lookup_elem(reading_map_fd, &idx, values); + if (err) { + pr_err("bpf map lookup failed: idx=%u, event=%s, cgrp=%s\n", + idx, evsel__name(evsel), evsel->cgrp->name); + goto out; + } + + perf_cpu_map__for_each_cpu(cpu, i, evsel->core.cpus) { + counts = perf_counts(evsel->counts, i, 0); + counts->val = values[cpu.cpu].counter; + counts->ena = values[cpu.cpu].enabled; + counts->run = values[cpu.cpu].running; + } + } + +out: + free(values); + return err; +} + +static int bperf_cgrp__destroy(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgroup_bpf__destroy(skel); + evsel__delete(cgrp_switch); // it'll destroy on_switch progs too + + return 0; +} + +struct bpf_counter_ops bperf_cgrp_ops = { + .load = bperf_cgrp__load, + .enable = bperf_cgrp__enable, + .disable = bperf_cgrp__disable, + .read = bperf_cgrp__read, + .install_pe = bperf_cgrp__install_pe, + .destroy = bperf_cgrp__destroy, +}; diff --git a/tools/perf/util/bpf_ftrace.c b/tools/perf/util/bpf_ftrace.c new file mode 100644 index 000000000..c456d24ef --- /dev/null +++ b/tools/perf/util/bpf_ftrace.c @@ -0,0 +1,214 @@ +#include +#include +#include +#include +#include + +#include +#include + +#include "util/ftrace.h" +#include "util/cpumap.h" +#include "util/thread_map.h" +#include "util/debug.h" +#include "util/evlist.h" +#include "util/bpf_counter.h" +#include "util/stat.h" + +#include "util/bpf_skel/func_latency.skel.h" + +static struct func_latency_bpf *skel; + +int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace) +{ + int fd, err; + int i, ncpus = 1, ntasks = 1; + struct filter_entry *func = NULL; + + if (!list_empty(&ftrace->filters)) { + if (!list_is_singular(&ftrace->filters)) { + pr_err("ERROR: Too many target functions.\n"); + return -1; + } + func = list_first_entry(&ftrace->filters, struct filter_entry, list); + } else { + int count = 0; + struct list_head *pos; + + list_for_each(pos, &ftrace->event_pair) + count++; + + if (count != 2) { + pr_err("ERROR: Needs two target events.\n"); + return -1; + } + } + + skel = func_latency_bpf__open(); + if (!skel) { + pr_err("Failed to open func latency skeleton\n"); + return -1; + } + + skel->rodata->bucket_range = ftrace->bucket_range; + skel->rodata->min_latency = ftrace->min_latency; + skel->rodata->bucket_num = ftrace->bucket_num; + if (ftrace->bucket_range && ftrace->bucket_num) { + bpf_map__set_max_entries(skel->maps.latency, ftrace->bucket_num); + } + + /* don't need to set cpu filter for system-wide mode */ + if (ftrace->target.cpu_list) { + ncpus = perf_cpu_map__nr(ftrace->evlist->core.user_requested_cpus); + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + skel->rodata->has_cpu = 1; + } + + if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { + ntasks = perf_thread_map__nr(ftrace->evlist->core.threads); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + } + + skel->rodata->use_nsec = ftrace->use_nsec; + + set_max_rlimit(); + + err = func_latency_bpf__load(skel); + if (err) { + pr_err("Failed to load func latency skeleton\n"); + goto out; + } + + if (ftrace->target.cpu_list) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu(ftrace->evlist->core.user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target__has_task(&ftrace->target) || target__none(&ftrace->target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(ftrace->evlist->core.threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + skel->bss->min = INT64_MAX; + + if (func) { + skel->links.func_begin = bpf_program__attach_kprobe(skel->progs.func_begin, + false, func->name); + if (IS_ERR(skel->links.func_begin)) { + pr_err("Failed to attach fentry program\n"); + err = PTR_ERR(skel->links.func_begin); + goto out; + } + + skel->links.func_end = bpf_program__attach_kprobe(skel->progs.func_end, + true, func->name); + if (IS_ERR(skel->links.func_end)) { + pr_err("Failed to attach fexit program\n"); + err = PTR_ERR(skel->links.func_end); + goto out; + } + } else { + struct filter_entry *event; + + event = list_first_entry(&ftrace->event_pair, struct filter_entry, list); + + skel->links.event_begin = bpf_program__attach_raw_tracepoint(skel->progs.event_begin, + event->name); + if (IS_ERR(skel->links.event_begin)) { + pr_err("Failed to attach first tracepoint program\n"); + err = PTR_ERR(skel->links.event_begin); + goto out; + } + + event = list_next_entry(event, list); + + skel->links.event_end = bpf_program__attach_raw_tracepoint(skel->progs.event_end, + event->name); + if (IS_ERR(skel->links.event_end)) { + pr_err("Failed to attach second tracepoint program\n"); + err = PTR_ERR(skel->links.event_end); + goto out; + } + } + + /* XXX: we don't actually use this fd - just for poll() */ + return open("/dev/null", O_RDONLY); + +out: + return err; +} + +int perf_ftrace__latency_start_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + skel->bss->enabled = 1; + return 0; +} + +int perf_ftrace__latency_stop_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + skel->bss->enabled = 0; + return 0; +} + +int perf_ftrace__latency_read_bpf(struct perf_ftrace *ftrace, + int buckets[], struct stats *stats) +{ + int i, fd, err; + u32 idx; + u64 *hist; + int ncpus = cpu__max_cpu().cpu; + + fd = bpf_map__fd(skel->maps.latency); + + hist = calloc(ncpus, sizeof(*hist)); + if (hist == NULL) + return -ENOMEM; + + for (idx = 0; idx < skel->rodata->bucket_num; idx++) { + err = bpf_map_lookup_elem(fd, &idx, hist); + if (err) { + buckets[idx] = 0; + continue; + } + + for (i = 0; i < ncpus; i++) + buckets[idx] += hist[i]; + } + + if (skel->bss->count) { + stats->mean = skel->bss->total / skel->bss->count; + stats->n = skel->bss->count; + stats->max = skel->bss->max; + stats->min = skel->bss->min; + + if (!ftrace->use_nsec) { + stats->mean /= 1000; + stats->max /= 1000; + stats->min /= 1000; + } + } + + free(hist); + return 0; +} + +int perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + func_latency_bpf__destroy(skel); + return 0; +} diff --git a/tools/perf/util/bpf_kwork.c b/tools/perf/util/bpf_kwork.c new file mode 100644 index 000000000..2248f462a --- /dev/null +++ b/tools/perf/util/bpf_kwork.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bpf_kwork.c + * + * Copyright (c) 2022 Huawei Inc, Yang Jihong + */ + +#include +#include +#include +#include +#include + +#include + +#include "util/debug.h" +#include "util/evsel.h" +#include "util/kwork.h" + +#include +#include + +#include "util/bpf_skel/kwork_trace.skel.h" + +/* + * This should be in sync with "util/kwork_trace.bpf.c" + */ +#define MAX_KWORKNAME 128 + +struct work_key { + u32 type; + u32 cpu; + u64 id; +}; + +struct report_data { + u64 nr; + u64 total_time; + u64 max_time; + u64 max_time_start; + u64 max_time_end; +}; + +struct kwork_class_bpf { + struct kwork_class *class; + + void (*load_prepare)(struct perf_kwork *kwork); + int (*get_work_name)(struct work_key *key, char **ret_name); +}; + +static struct kwork_trace_bpf *skel; + +static struct timespec ts_start; +static struct timespec ts_end; + +void perf_kwork__trace_start(void) +{ + clock_gettime(CLOCK_MONOTONIC, &ts_start); + skel->bss->enabled = 1; +} + +void perf_kwork__trace_finish(void) +{ + clock_gettime(CLOCK_MONOTONIC, &ts_end); + skel->bss->enabled = 0; +} + +static int get_work_name_from_map(struct work_key *key, char **ret_name) +{ + char name[MAX_KWORKNAME] = { 0 }; + int fd = bpf_map__fd(skel->maps.perf_kwork_names); + + *ret_name = NULL; + + if (fd < 0) { + pr_debug("Invalid names map fd\n"); + return 0; + } + + if ((bpf_map_lookup_elem(fd, key, name) == 0) && (strlen(name) != 0)) { + *ret_name = strdup(name); + if (*ret_name == NULL) { + pr_err("Failed to copy work name\n"); + return -1; + } + } + + return 0; +} + +static void irq_load_prepare(struct perf_kwork *kwork) +{ + if (kwork->report == KWORK_REPORT_RUNTIME) { + bpf_program__set_autoload(skel->progs.report_irq_handler_entry, true); + bpf_program__set_autoload(skel->progs.report_irq_handler_exit, true); + } +} + +static struct kwork_class_bpf kwork_irq_bpf = { + .load_prepare = irq_load_prepare, + .get_work_name = get_work_name_from_map, +}; + +static void softirq_load_prepare(struct perf_kwork *kwork) +{ + if (kwork->report == KWORK_REPORT_RUNTIME) { + bpf_program__set_autoload(skel->progs.report_softirq_entry, true); + bpf_program__set_autoload(skel->progs.report_softirq_exit, true); + } else if (kwork->report == KWORK_REPORT_LATENCY) { + bpf_program__set_autoload(skel->progs.latency_softirq_raise, true); + bpf_program__set_autoload(skel->progs.latency_softirq_entry, true); + } +} + +static struct kwork_class_bpf kwork_softirq_bpf = { + .load_prepare = softirq_load_prepare, + .get_work_name = get_work_name_from_map, +}; + +static void workqueue_load_prepare(struct perf_kwork *kwork) +{ + if (kwork->report == KWORK_REPORT_RUNTIME) { + bpf_program__set_autoload(skel->progs.report_workqueue_execute_start, true); + bpf_program__set_autoload(skel->progs.report_workqueue_execute_end, true); + } else if (kwork->report == KWORK_REPORT_LATENCY) { + bpf_program__set_autoload(skel->progs.latency_workqueue_activate_work, true); + bpf_program__set_autoload(skel->progs.latency_workqueue_execute_start, true); + } +} + +static struct kwork_class_bpf kwork_workqueue_bpf = { + .load_prepare = workqueue_load_prepare, + .get_work_name = get_work_name_from_map, +}; + +static struct kwork_class_bpf * +kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { + [KWORK_CLASS_IRQ] = &kwork_irq_bpf, + [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, + [KWORK_CLASS_WORKQUEUE] = &kwork_workqueue_bpf, +}; + +static bool valid_kwork_class_type(enum kwork_class_type type) +{ + return type >= 0 && type < KWORK_CLASS_MAX ? true : false; +} + +static int setup_filters(struct perf_kwork *kwork) +{ + if (kwork->cpu_list != NULL) { + unsigned int idx; + int nr_cpus; + struct perf_cpu_map *map; + struct perf_cpu cpu; + int fd = bpf_map__fd(skel->maps.perf_kwork_cpu_filter); + + if (fd < 0) { + pr_debug("Invalid cpu filter fd\n"); + return -1; + } + + map = perf_cpu_map__new(kwork->cpu_list); + if (map == NULL) { + pr_debug("Invalid cpu_list\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + perf_cpu_map__for_each_cpu(cpu, idx, map) { + u8 val = 1; + + if (cpu.cpu >= nr_cpus) { + perf_cpu_map__put(map); + pr_err("Requested cpu %d too large\n", cpu.cpu); + return -1; + } + bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); + } + perf_cpu_map__put(map); + } + + if (kwork->profile_name != NULL) { + int key, fd; + + if (strlen(kwork->profile_name) >= MAX_KWORKNAME) { + pr_err("Requested name filter %s too large, limit to %d\n", + kwork->profile_name, MAX_KWORKNAME - 1); + return -1; + } + + fd = bpf_map__fd(skel->maps.perf_kwork_name_filter); + if (fd < 0) { + pr_debug("Invalid name filter fd\n"); + return -1; + } + + key = 0; + bpf_map_update_elem(fd, &key, kwork->profile_name, BPF_ANY); + } + + return 0; +} + +int perf_kwork__trace_prepare_bpf(struct perf_kwork *kwork) +{ + struct bpf_program *prog; + struct kwork_class *class; + struct kwork_class_bpf *class_bpf; + enum kwork_class_type type; + + skel = kwork_trace_bpf__open(); + if (!skel) { + pr_debug("Failed to open kwork trace skeleton\n"); + return -1; + } + + /* + * set all progs to non-autoload, + * then set corresponding progs according to config + */ + bpf_object__for_each_program(prog, skel->obj) + bpf_program__set_autoload(prog, false); + + list_for_each_entry(class, &kwork->class_list, list) { + type = class->type; + if (!valid_kwork_class_type(type) || + (kwork_class_bpf_supported_list[type] == NULL)) { + pr_err("Unsupported bpf trace class %s\n", class->name); + goto out; + } + + class_bpf = kwork_class_bpf_supported_list[type]; + class_bpf->class = class; + + if (class_bpf->load_prepare != NULL) + class_bpf->load_prepare(kwork); + } + + if (kwork->cpu_list != NULL) + skel->rodata->has_cpu_filter = 1; + if (kwork->profile_name != NULL) + skel->rodata->has_name_filter = 1; + + if (kwork_trace_bpf__load(skel)) { + pr_debug("Failed to load kwork trace skeleton\n"); + goto out; + } + + if (setup_filters(kwork)) + goto out; + + if (kwork_trace_bpf__attach(skel)) { + pr_debug("Failed to attach kwork trace skeleton\n"); + goto out; + } + + return 0; + +out: + kwork_trace_bpf__destroy(skel); + return -1; +} + +static int add_work(struct perf_kwork *kwork, + struct work_key *key, + struct report_data *data) +{ + struct kwork_work *work; + struct kwork_class_bpf *bpf_trace; + struct kwork_work tmp = { + .id = key->id, + .name = NULL, + .cpu = key->cpu, + }; + enum kwork_class_type type = key->type; + int ret = 0; + + if (!valid_kwork_class_type(type)) { + pr_debug("Invalid class type %d to add work\n", type); + return -1; + } + + bpf_trace = kwork_class_bpf_supported_list[type]; + tmp.class = bpf_trace->class; + + if ((bpf_trace->get_work_name != NULL) && + (bpf_trace->get_work_name(key, &tmp.name))) + return -1; + + work = kwork->add_work(kwork, tmp.class, &tmp); + if (work == NULL) { + ret = -1; + goto out; + } + + if (kwork->report == KWORK_REPORT_RUNTIME) { + work->nr_atoms = data->nr; + work->total_runtime = data->total_time; + work->max_runtime = data->max_time; + work->max_runtime_start = data->max_time_start; + work->max_runtime_end = data->max_time_end; + } else if (kwork->report == KWORK_REPORT_LATENCY) { + work->nr_atoms = data->nr; + work->total_latency = data->total_time; + work->max_latency = data->max_time; + work->max_latency_start = data->max_time_start; + work->max_latency_end = data->max_time_end; + } else { + pr_debug("Invalid bpf report type %d\n", kwork->report); + ret = -1; + goto out; + } + + kwork->timestart = (u64)ts_start.tv_sec * NSEC_PER_SEC + ts_start.tv_nsec; + kwork->timeend = (u64)ts_end.tv_sec * NSEC_PER_SEC + ts_end.tv_nsec; + +out: + work_exit(&tmp); + return ret; +} + +int perf_kwork__report_read_bpf(struct perf_kwork *kwork) +{ + struct report_data data; + struct work_key key = { + .type = 0, + .cpu = 0, + .id = 0, + }; + struct work_key prev = { + .type = 0, + .cpu = 0, + .id = 0, + }; + int fd = bpf_map__fd(skel->maps.perf_kwork_report); + + if (fd < 0) { + pr_debug("Invalid report fd\n"); + return -1; + } + + while (!bpf_map_get_next_key(fd, &prev, &key)) { + if ((bpf_map_lookup_elem(fd, &key, &data)) != 0) { + pr_debug("Failed to lookup report elem\n"); + return -1; + } + + if ((data.nr != 0) && (add_work(kwork, &key, &data) != 0)) + return -1; + + prev = key; + } + return 0; +} + +void perf_kwork__report_cleanup_bpf(void) +{ + kwork_trace_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf_kwork_top.c b/tools/perf/util/bpf_kwork_top.c new file mode 100644 index 000000000..189a29d2b --- /dev/null +++ b/tools/perf/util/bpf_kwork_top.c @@ -0,0 +1,310 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * bpf_kwork_top.c + * + * Copyright (c) 2022 Huawei Inc, Yang Jihong + */ + +#include +#include +#include +#include +#include + +#include + +#include "util/debug.h" +#include "util/evsel.h" +#include "util/kwork.h" + +#include +#include + +#include "util/bpf_skel/kwork_top.skel.h" + +/* + * This should be in sync with "util/kwork_top.bpf.c" + */ +#define MAX_COMMAND_LEN 16 + +struct time_data { + __u64 timestamp; +}; + +struct work_data { + __u64 runtime; +}; + +struct task_data { + __u32 tgid; + __u32 is_kthread; + char comm[MAX_COMMAND_LEN]; +}; + +struct work_key { + __u32 type; + __u32 pid; + __u64 task_p; +}; + +struct task_key { + __u32 pid; + __u32 cpu; +}; + +struct kwork_class_bpf { + struct kwork_class *class; + void (*load_prepare)(void); +}; + +static struct kwork_top_bpf *skel; + +void perf_kwork__top_start(void) +{ + struct timespec ts; + + clock_gettime(CLOCK_MONOTONIC, &ts); + skel->bss->from_timestamp = (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + skel->bss->enabled = 1; + pr_debug("perf kwork top start at: %lld\n", skel->bss->from_timestamp); +} + +void perf_kwork__top_finish(void) +{ + struct timespec ts; + + skel->bss->enabled = 0; + clock_gettime(CLOCK_MONOTONIC, &ts); + skel->bss->to_timestamp = (u64)ts.tv_sec * NSEC_PER_SEC + ts.tv_nsec; + pr_debug("perf kwork top finish at: %lld\n", skel->bss->to_timestamp); +} + +static void irq_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_irq_handler_entry, true); + bpf_program__set_autoload(skel->progs.on_irq_handler_exit, true); +} + +static struct kwork_class_bpf kwork_irq_bpf = { + .load_prepare = irq_load_prepare, +}; + +static void softirq_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_softirq_entry, true); + bpf_program__set_autoload(skel->progs.on_softirq_exit, true); +} + +static struct kwork_class_bpf kwork_softirq_bpf = { + .load_prepare = softirq_load_prepare, +}; + +static void sched_load_prepare(void) +{ + bpf_program__set_autoload(skel->progs.on_switch, true); +} + +static struct kwork_class_bpf kwork_sched_bpf = { + .load_prepare = sched_load_prepare, +}; + +static struct kwork_class_bpf * +kwork_class_bpf_supported_list[KWORK_CLASS_MAX] = { + [KWORK_CLASS_IRQ] = &kwork_irq_bpf, + [KWORK_CLASS_SOFTIRQ] = &kwork_softirq_bpf, + [KWORK_CLASS_SCHED] = &kwork_sched_bpf, +}; + +static bool valid_kwork_class_type(enum kwork_class_type type) +{ + return type >= 0 && type < KWORK_CLASS_MAX; +} + +static int setup_filters(struct perf_kwork *kwork) +{ + if (kwork->cpu_list) { + unsigned int idx; + int nr_cpus, fd; + struct perf_cpu_map *map; + struct perf_cpu cpu; + + fd = bpf_map__fd(skel->maps.kwork_top_cpu_filter); + if (fd < 0) { + pr_debug("Invalid cpu filter fd\n"); + return -1; + } + + map = perf_cpu_map__new(kwork->cpu_list); + if (!map) { + pr_debug("Invalid cpu_list\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + perf_cpu_map__for_each_cpu(cpu, idx, map) { + u8 val = 1; + + if (cpu.cpu >= nr_cpus) { + perf_cpu_map__put(map); + pr_err("Requested cpu %d too large\n", cpu.cpu); + return -1; + } + bpf_map_update_elem(fd, &cpu.cpu, &val, BPF_ANY); + } + perf_cpu_map__put(map); + } + + return 0; +} + +int perf_kwork__top_prepare_bpf(struct perf_kwork *kwork) +{ + struct bpf_program *prog; + struct kwork_class *class; + struct kwork_class_bpf *class_bpf; + enum kwork_class_type type; + + skel = kwork_top_bpf__open(); + if (!skel) { + pr_debug("Failed to open kwork top skeleton\n"); + return -1; + } + + /* + * set all progs to non-autoload, + * then set corresponding progs according to config + */ + bpf_object__for_each_program(prog, skel->obj) + bpf_program__set_autoload(prog, false); + + list_for_each_entry(class, &kwork->class_list, list) { + type = class->type; + if (!valid_kwork_class_type(type) || + !kwork_class_bpf_supported_list[type]) { + pr_err("Unsupported bpf trace class %s\n", class->name); + goto out; + } + + class_bpf = kwork_class_bpf_supported_list[type]; + class_bpf->class = class; + + if (class_bpf->load_prepare) + class_bpf->load_prepare(); + } + + if (kwork->cpu_list) + skel->rodata->has_cpu_filter = 1; + + if (kwork_top_bpf__load(skel)) { + pr_debug("Failed to load kwork top skeleton\n"); + goto out; + } + + if (setup_filters(kwork)) + goto out; + + if (kwork_top_bpf__attach(skel)) { + pr_debug("Failed to attach kwork top skeleton\n"); + goto out; + } + + return 0; + +out: + kwork_top_bpf__destroy(skel); + return -1; +} + +static void read_task_info(struct kwork_work *work) +{ + int fd; + struct task_data data; + struct task_key key = { + .pid = work->id, + .cpu = work->cpu, + }; + + fd = bpf_map__fd(skel->maps.kwork_top_tasks); + if (fd < 0) { + pr_debug("Invalid top tasks map fd\n"); + return; + } + + if (!bpf_map_lookup_elem(fd, &key, &data)) { + work->tgid = data.tgid; + work->is_kthread = data.is_kthread; + work->name = strdup(data.comm); + } +} +static int add_work(struct perf_kwork *kwork, struct work_key *key, + struct work_data *data, int cpu) +{ + struct kwork_class_bpf *bpf_trace; + struct kwork_work *work; + struct kwork_work tmp = { + .id = key->pid, + .cpu = cpu, + .name = NULL, + }; + enum kwork_class_type type = key->type; + + if (!valid_kwork_class_type(type)) { + pr_debug("Invalid class type %d to add work\n", type); + return -1; + } + + bpf_trace = kwork_class_bpf_supported_list[type]; + tmp.class = bpf_trace->class; + + work = kwork->add_work(kwork, tmp.class, &tmp); + if (!work) + return -1; + + work->total_runtime = data->runtime; + read_task_info(work); + + return 0; +} + +int perf_kwork__top_read_bpf(struct perf_kwork *kwork) +{ + int i, fd, nr_cpus; + struct work_data *data; + struct work_key key, prev; + + fd = bpf_map__fd(skel->maps.kwork_top_works); + if (fd < 0) { + pr_debug("Invalid top runtime fd\n"); + return -1; + } + + nr_cpus = libbpf_num_possible_cpus(); + data = calloc(nr_cpus, sizeof(struct work_data)); + if (!data) + return -1; + + memset(&prev, 0, sizeof(prev)); + while (!bpf_map_get_next_key(fd, &prev, &key)) { + if ((bpf_map_lookup_elem(fd, &key, data)) != 0) { + pr_debug("Failed to lookup top elem\n"); + return -1; + } + + for (i = 0; i < nr_cpus; i++) { + if (data[i].runtime == 0) + continue; + + if (add_work(kwork, &key, &data[i], i)) + return -1; + } + prev = key; + } + free(data); + + return 0; +} + +void perf_kwork__top_cleanup_bpf(void) +{ + kwork_top_bpf__destroy(skel); +} diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c new file mode 100644 index 000000000..b1cfa63a4 --- /dev/null +++ b/tools/perf/util/bpf_lock_contention.c @@ -0,0 +1,862 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/cgroup.h" +#include "util/debug.h" +#include "util/evlist.h" +#include "util/hashmap.h" +#include "util/machine.h" +#include "util/map.h" +#include "util/symbol.h" +#include "util/target.h" +#include "util/thread.h" +#include "util/thread_map.h" +#include "util/lock-contention.h" +#include +#include +#include +#include +#include +#include + +#include "bpf_skel/lock_contention.skel.h" +#include "bpf_skel/lock_data.h" + +static struct lock_contention_bpf *skel; +static bool has_slab_iter; +static struct hashmap slab_hash; + +static size_t slab_cache_hash(long key, void *ctx __maybe_unused) +{ + return key; +} + +static bool slab_cache_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return key1 == key2; +} + +static void check_slab_cache_iter(struct lock_contention *con) +{ + s32 ret; + + hashmap__init(&slab_hash, slab_cache_hash, slab_cache_equal, /*ctx=*/NULL); + + con->btf = btf__load_vmlinux_btf(); + if (con->btf == NULL) { + pr_debug("BTF loading failed: %m\n"); + return; + } + + ret = btf__find_by_name_kind(con->btf, "bpf_iter__kmem_cache", BTF_KIND_STRUCT); + if (ret < 0) { + bpf_program__set_autoload(skel->progs.slab_cache_iter, false); + pr_debug("slab cache iterator is not available: %d\n", ret); + return; + } + + has_slab_iter = true; + + bpf_map__set_max_entries(skel->maps.slab_caches, con->map_nr_entries); +} + +static void run_slab_cache_iter(void) +{ + int fd; + char buf[256]; + long key, *prev_key; + + if (!has_slab_iter) + return; + + fd = bpf_iter_create(bpf_link__fd(skel->links.slab_cache_iter)); + if (fd < 0) { + pr_debug("cannot create slab cache iter: %d\n", fd); + return; + } + + /* This will run the bpf program */ + while (read(fd, buf, sizeof(buf)) > 0) + continue; + + close(fd); + + /* Read the slab cache map and build a hash with IDs */ + fd = bpf_map__fd(skel->maps.slab_caches); + prev_key = NULL; + while (!bpf_map_get_next_key(fd, prev_key, &key)) { + struct slab_cache_data *data; + + data = malloc(sizeof(*data)); + if (data == NULL) + break; + + if (bpf_map_lookup_elem(fd, &key, data) < 0) + break; + + hashmap__add(&slab_hash, data->id, data); + prev_key = &key; + } +} + +static void exit_slab_cache_iter(void) +{ + struct hashmap_entry *cur; + unsigned bkt; + + hashmap__for_each_entry(&slab_hash, cur, bkt) + free(cur->pvalue); + + hashmap__clear(&slab_hash); +} + +static void init_numa_data(struct lock_contention *con) +{ + struct symbol *sym; + struct map *kmap; + char *buf = NULL, *p; + size_t len; + long last = -1; + int ret; + + if (!con->btf) + return; + + /* + * 'struct zone' is embedded in 'struct pglist_data' as an array. + * As we may not have full information of the struct zone in the + * (fake) vmlinux.h, let's get the actual size from BTF. + */ + ret = btf__find_by_name_kind(con->btf, "zone", BTF_KIND_STRUCT); + if (ret < 0) { + pr_debug("cannot get type of struct zone: %d\n", ret); + return; + } + + ret = btf__resolve_size(con->btf, ret); + if (ret < 0) { + pr_debug("cannot get size of struct zone: %d\n", ret); + return; + } + skel->rodata->sizeof_zone = ret; + + /* UMA system doesn't have 'node_data[]' - just use contig_page_data. */ + sym = machine__find_kernel_symbol_by_name(con->machine, + "contig_page_data", + &kmap); + if (sym) { + skel->rodata->contig_page_data_addr = map__unmap_ip(kmap, sym->start); + map__put(kmap); + return; + } + + /* + * The 'node_data' is an array of pointers to struct pglist_data. + * It needs to follow the pointer for each node in BPF to get the + * address of struct pglist_data and its zones. + */ + sym = machine__find_kernel_symbol_by_name(con->machine, + "node_data", + &kmap); + if (sym == NULL) + return; + + skel->rodata->node_data_addr = map__unmap_ip(kmap, sym->start); + map__put(kmap); + + /* get the number of online nodes using the last node number + 1 */ + ret = sysfs__read_str("devices/system/node/online", &buf, &len); + if (ret < 0) { + pr_debug("failed to read online node: %d\n", ret); + return; + } + + p = buf; + while (p && *p) { + last = strtol(p, &p, 0); + + if (p && (*p == ',' || *p == '-' || *p == '\n')) + p++; + } + skel->rodata->nr_nodes = last + 1; + free(buf); +} + +int lock_contention_prepare(struct lock_contention *con) +{ + int i, fd; + int ncpus = 1, ntasks = 1, ntypes = 1, naddrs = 1, ncgrps = 1, nslabs = 1; + struct evlist *evlist = con->evlist; + struct target *target = con->target; + bool has_mmap_lock = false; + + /* make sure it loads the kernel map before lookup */ + map__load(machine__kernel_map(con->machine)); + + skel = lock_contention_bpf__open(); + if (!skel) { + pr_err("Failed to open lock-contention BPF skeleton\n"); + return -1; + } + + bpf_map__set_value_size(skel->maps.stacks, con->max_stack * sizeof(u64)); + bpf_map__set_max_entries(skel->maps.lock_stat, con->map_nr_entries); + bpf_map__set_max_entries(skel->maps.tstamp, con->map_nr_entries); + + if (con->aggr_mode == LOCK_AGGR_TASK) + bpf_map__set_max_entries(skel->maps.task_data, con->map_nr_entries); + else + bpf_map__set_max_entries(skel->maps.task_data, 1); + + if (con->save_callstack) { + bpf_map__set_max_entries(skel->maps.stacks, con->map_nr_entries); + if (con->owner) { + bpf_map__set_value_size(skel->maps.stack_buf, con->max_stack * sizeof(u64)); + bpf_map__set_key_size(skel->maps.owner_stacks, + con->max_stack * sizeof(u64)); + bpf_map__set_max_entries(skel->maps.owner_stacks, con->map_nr_entries); + bpf_map__set_max_entries(skel->maps.owner_data, con->map_nr_entries); + bpf_map__set_max_entries(skel->maps.owner_stat, con->map_nr_entries); + skel->rodata->max_stack = con->max_stack; + } + } else { + bpf_map__set_max_entries(skel->maps.stacks, 1); + } + + if (target__has_cpu(target)) { + skel->rodata->has_cpu = 1; + ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + } + if (target__has_task(target)) { + skel->rodata->has_task = 1; + ntasks = perf_thread_map__nr(evlist->core.threads); + } + if (con->filters->nr_types) { + skel->rodata->has_type = 1; + ntypes = con->filters->nr_types; + } + if (con->filters->nr_cgrps) { + skel->rodata->has_cgroup = 1; + ncgrps = con->filters->nr_cgrps; + } + + /* resolve lock name filters to addr */ + if (con->filters->nr_syms) { + struct symbol *sym; + struct map *kmap; + unsigned long *addrs; + + for (i = 0; i < con->filters->nr_syms; i++) { + if (!strcmp(con->filters->syms[i], "mmap_lock")) { + has_mmap_lock = true; + continue; + } + + sym = machine__find_kernel_symbol_by_name(con->machine, + con->filters->syms[i], + &kmap); + if (sym == NULL) { + pr_warning("ignore unknown symbol: %s\n", + con->filters->syms[i]); + continue; + } + + addrs = realloc(con->filters->addrs, + (con->filters->nr_addrs + 1) * sizeof(*addrs)); + if (addrs == NULL) { + pr_warning("memory allocation failure\n"); + continue; + } + + addrs[con->filters->nr_addrs++] = map__unmap_ip(kmap, sym->start); + con->filters->addrs = addrs; + } + naddrs = con->filters->nr_addrs ?: has_mmap_lock; + skel->rodata->has_addr = 1; + } + + /* resolve lock name in delays */ + if (con->nr_delays) { + struct symbol *sym; + struct map *kmap; + + for (i = 0; i < con->nr_delays; i++) { + sym = machine__find_kernel_symbol_by_name(con->machine, + con->delays[i].sym, + &kmap); + if (sym == NULL) { + pr_warning("ignore unknown symbol: %s\n", + con->delays[i].sym); + continue; + } + + con->delays[i].addr = map__unmap_ip(kmap, sym->start); + } + skel->rodata->lock_delay = 1; + bpf_map__set_max_entries(skel->maps.lock_delays, con->nr_delays); + } + + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + bpf_map__set_max_entries(skel->maps.type_filter, ntypes); + bpf_map__set_max_entries(skel->maps.addr_filter, naddrs); + bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); + + skel->rodata->stack_skip = con->stack_skip; + skel->rodata->aggr_mode = con->aggr_mode; + skel->rodata->needs_callstack = con->save_callstack; + skel->rodata->lock_owner = con->owner; + skel->rodata->has_mmap_lock = has_mmap_lock; + + if (con->aggr_mode == LOCK_AGGR_CGROUP || con->filters->nr_cgrps) { + if (cgroup_is_v2("perf_event")) + skel->rodata->use_cgroup_v2 = 1; + } + + check_slab_cache_iter(con); + + if (con->filters->nr_slabs && has_slab_iter) { + skel->rodata->has_slab = 1; + nslabs = con->filters->nr_slabs; + } + + bpf_map__set_max_entries(skel->maps.slab_filter, nslabs); + + init_numa_data(con); + + if (lock_contention_bpf__load(skel) < 0) { + pr_err("Failed to load lock-contention BPF skeleton\n"); + return -1; + } + + if (target__has_cpu(target)) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target__has_task(target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(evlist->core.threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + if (target__none(target) && evlist->workload.pid > 0) { + u32 pid = evlist->workload.pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + + if (con->filters->nr_types) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.type_filter); + + for (i = 0; i < con->filters->nr_types; i++) + bpf_map_update_elem(fd, &con->filters->types[i], &val, BPF_ANY); + } + + if (con->filters->nr_addrs) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.addr_filter); + + for (i = 0; i < con->filters->nr_addrs; i++) + bpf_map_update_elem(fd, &con->filters->addrs[i], &val, BPF_ANY); + } + + if (con->filters->nr_cgrps) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cgroup_filter); + + for (i = 0; i < con->filters->nr_cgrps; i++) + bpf_map_update_elem(fd, &con->filters->cgrps[i], &val, BPF_ANY); + } + + if (con->nr_delays) { + fd = bpf_map__fd(skel->maps.lock_delays); + + for (i = 0; i < con->nr_delays; i++) + bpf_map_update_elem(fd, &con->delays[i].addr, &con->delays[i].time, BPF_ANY); + } + + if (con->aggr_mode == LOCK_AGGR_CGROUP) + read_all_cgroups(&con->cgroups); + + bpf_program__set_autoload(skel->progs.collect_lock_syms, false); + + lock_contention_bpf__attach(skel); + + /* run the slab iterator after attaching */ + run_slab_cache_iter(); + + if (con->filters->nr_slabs) { + u8 val = 1; + int cache_fd; + long key, *prev_key; + + fd = bpf_map__fd(skel->maps.slab_filter); + + /* Read the slab cache map and build a hash with its address */ + cache_fd = bpf_map__fd(skel->maps.slab_caches); + prev_key = NULL; + while (!bpf_map_get_next_key(cache_fd, prev_key, &key)) { + struct slab_cache_data data; + + if (bpf_map_lookup_elem(cache_fd, &key, &data) < 0) + break; + + for (i = 0; i < con->filters->nr_slabs; i++) { + if (!strcmp(con->filters->slabs[i], data.name)) { + bpf_map_update_elem(fd, &key, &val, BPF_ANY); + break; + } + } + prev_key = &key; + } + } + + return 0; +} + +/* + * Run the BPF program directly using BPF_PROG_TEST_RUN to update the end + * timestamp in ktime so that it can calculate delta easily. + */ +static void mark_end_timestamp(void) +{ + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + int prog_fd = bpf_program__fd(skel->progs.end_timestamp); + + bpf_prog_test_run_opts(prog_fd, &opts); +} + +static void update_lock_stat(int map_fd, int pid, u64 end_ts, + enum lock_aggr_mode aggr_mode, + struct tstamp_data *ts_data) +{ + u64 delta; + struct contention_key stat_key = {}; + struct contention_data stat_data; + + if (ts_data->timestamp >= end_ts) + return; + + delta = end_ts - ts_data->timestamp; + + switch (aggr_mode) { + case LOCK_AGGR_CALLER: + stat_key.stack_id = ts_data->stack_id; + break; + case LOCK_AGGR_TASK: + stat_key.pid = pid; + break; + case LOCK_AGGR_ADDR: + stat_key.lock_addr_or_cgroup = ts_data->lock; + break; + case LOCK_AGGR_CGROUP: + stat_key.lock_addr_or_cgroup = ts_data->cgroup_id; + break; + default: + return; + } + + if (bpf_map_lookup_elem(map_fd, &stat_key, &stat_data) < 0) + return; + + stat_data.total_time += delta; + stat_data.count++; + + if (delta > stat_data.max_time) + stat_data.max_time = delta; + if (delta < stat_data.min_time) + stat_data.min_time = delta; + + bpf_map_update_elem(map_fd, &stat_key, &stat_data, BPF_EXIST); +} + +/* + * Account entries in the tstamp map (which didn't see the corresponding + * lock:contention_end tracepoint) using end_ts. + */ +static void account_end_timestamp(struct lock_contention *con) +{ + int ts_fd, stat_fd; + int *prev_key, key; + u64 end_ts = skel->bss->end_ts; + int total_cpus; + enum lock_aggr_mode aggr_mode = con->aggr_mode; + struct tstamp_data ts_data, *cpu_data; + + /* Iterate per-task tstamp map (key = TID) */ + ts_fd = bpf_map__fd(skel->maps.tstamp); + stat_fd = bpf_map__fd(skel->maps.lock_stat); + + prev_key = NULL; + while (!bpf_map_get_next_key(ts_fd, prev_key, &key)) { + if (bpf_map_lookup_elem(ts_fd, &key, &ts_data) == 0) { + int pid = key; + + if (aggr_mode == LOCK_AGGR_TASK && con->owner) + pid = ts_data.flags; + + update_lock_stat(stat_fd, pid, end_ts, aggr_mode, + &ts_data); + } + + prev_key = &key; + } + + /* Now it'll check per-cpu tstamp map which doesn't have TID. */ + if (aggr_mode == LOCK_AGGR_TASK || aggr_mode == LOCK_AGGR_CGROUP) + return; + + total_cpus = cpu__max_cpu().cpu; + ts_fd = bpf_map__fd(skel->maps.tstamp_cpu); + + cpu_data = calloc(total_cpus, sizeof(*cpu_data)); + if (cpu_data == NULL) + return; + + prev_key = NULL; + while (!bpf_map_get_next_key(ts_fd, prev_key, &key)) { + if (bpf_map_lookup_elem(ts_fd, &key, cpu_data) < 0) + goto next; + + for (int i = 0; i < total_cpus; i++) { + if (cpu_data[i].lock == 0) + continue; + + update_lock_stat(stat_fd, -1, end_ts, aggr_mode, + &cpu_data[i]); + } + +next: + prev_key = &key; + } + free(cpu_data); +} + +int lock_contention_start(void) +{ + skel->bss->enabled = 1; + return 0; +} + +int lock_contention_stop(void) +{ + skel->bss->enabled = 0; + mark_end_timestamp(); + return 0; +} + +static const char *lock_contention_get_name(struct lock_contention *con, + struct contention_key *key, + u64 *stack_trace, u32 flags) +{ + int idx = 0; + u64 addr; + static char name_buf[KSYM_NAME_LEN]; + struct symbol *sym; + struct map *kmap; + struct machine *machine = con->machine; + + if (con->aggr_mode == LOCK_AGGR_TASK) { + struct contention_task_data task; + int pid = key->pid; + int task_fd = bpf_map__fd(skel->maps.task_data); + + /* do not update idle comm which contains CPU number */ + if (pid) { + struct thread *t = machine__findnew_thread(machine, /*pid=*/-1, pid); + + if (t != NULL && + !bpf_map_lookup_elem(task_fd, &pid, &task) && + thread__set_comm(t, task.comm, /*timestamp=*/0)) { + snprintf(name_buf, sizeof(name_buf), "%s", task.comm); + return name_buf; + } + } + return ""; + } + + if (con->aggr_mode == LOCK_AGGR_ADDR) { + int lock_fd = bpf_map__fd(skel->maps.lock_syms); + struct slab_cache_data *slab_data; + + /* per-process locks set upper bits of the flags */ + if (flags & LCD_F_MMAP_LOCK) + return "mmap_lock"; + if (flags & LCD_F_SIGHAND_LOCK) + return "siglock"; + + /* global locks with symbols */ + sym = machine__find_kernel_symbol(machine, key->lock_addr_or_cgroup, &kmap); + if (sym) + return sym->name; + + /* try semi-global locks collected separately */ + if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) { + if (flags == LOCK_CLASS_RQLOCK) + return "rq_lock"; + } + + if (!bpf_map_lookup_elem(lock_fd, &key->lock_addr_or_cgroup, &flags)) { + if (flags == LOCK_CLASS_ZONE_LOCK) + return "zone_lock"; + } + + /* look slab_hash for dynamic locks in a slab object */ + if (hashmap__find(&slab_hash, flags & LCB_F_SLAB_ID_MASK, &slab_data)) { + snprintf(name_buf, sizeof(name_buf), "&%s", slab_data->name); + return name_buf; + } + + return ""; + } + + if (con->aggr_mode == LOCK_AGGR_CGROUP) { + u64 cgrp_id = key->lock_addr_or_cgroup; + struct cgroup *cgrp = __cgroup__find(&con->cgroups, cgrp_id); + + if (cgrp) + return cgrp->name; + + snprintf(name_buf, sizeof(name_buf), "cgroup:%" PRIu64 "", cgrp_id); + return name_buf; + } + + /* LOCK_AGGR_CALLER: skip lock internal functions */ + while (machine__is_lock_function(machine, stack_trace[idx]) && + idx < con->max_stack - 1) + idx++; + + addr = stack_trace[idx]; + sym = machine__find_kernel_symbol(machine, addr, &kmap); + + if (sym) { + unsigned long offset; + + offset = map__map_ip(kmap, addr) - sym->start; + + if (offset == 0) + return sym->name; + + snprintf(name_buf, sizeof(name_buf), "%s+%#lx", sym->name, offset); + } else { + snprintf(name_buf, sizeof(name_buf), "%#lx", (unsigned long)addr); + } + + return name_buf; +} + +struct lock_stat *pop_owner_stack_trace(struct lock_contention *con) +{ + int stacks_fd, stat_fd; + u64 *stack_trace = NULL; + s32 stack_id; + struct contention_key ckey = {}; + struct contention_data cdata = {}; + size_t stack_size = con->max_stack * sizeof(*stack_trace); + struct lock_stat *st = NULL; + + stacks_fd = bpf_map__fd(skel->maps.owner_stacks); + stat_fd = bpf_map__fd(skel->maps.owner_stat); + if (!stacks_fd || !stat_fd) + goto out_err; + + stack_trace = zalloc(stack_size); + if (stack_trace == NULL) + goto out_err; + + if (bpf_map_get_next_key(stacks_fd, NULL, stack_trace)) + goto out_err; + + bpf_map_lookup_elem(stacks_fd, stack_trace, &stack_id); + ckey.stack_id = stack_id; + bpf_map_lookup_elem(stat_fd, &ckey, &cdata); + + st = zalloc(sizeof(struct lock_stat)); + if (!st) + goto out_err; + + st->name = strdup(stack_trace[0] ? lock_contention_get_name(con, NULL, stack_trace, 0) : + "unknown"); + if (!st->name) + goto out_err; + + st->flags = cdata.flags; + st->nr_contended = cdata.count; + st->wait_time_total = cdata.total_time; + st->wait_time_max = cdata.max_time; + st->wait_time_min = cdata.min_time; + st->callstack = stack_trace; + + if (cdata.count) + st->avg_wait_time = cdata.total_time / cdata.count; + + bpf_map_delete_elem(stacks_fd, stack_trace); + bpf_map_delete_elem(stat_fd, &ckey); + + return st; + +out_err: + free(stack_trace); + free(st); + + return NULL; +} + +int lock_contention_read(struct lock_contention *con) +{ + int fd, stack, err = 0; + struct contention_key *prev_key, key = {}; + struct contention_data data = {}; + struct lock_stat *st = NULL; + struct machine *machine = con->machine; + u64 *stack_trace; + size_t stack_size = con->max_stack * sizeof(*stack_trace); + + fd = bpf_map__fd(skel->maps.lock_stat); + stack = bpf_map__fd(skel->maps.stacks); + + con->fails.task = skel->bss->task_fail; + con->fails.stack = skel->bss->stack_fail; + con->fails.time = skel->bss->time_fail; + con->fails.data = skel->bss->data_fail; + + stack_trace = zalloc(stack_size); + if (stack_trace == NULL) + return -1; + + account_end_timestamp(con); + + if (con->aggr_mode == LOCK_AGGR_TASK) { + struct thread *idle = machine__findnew_thread(machine, + /*pid=*/0, + /*tid=*/0); + thread__set_comm(idle, "swapper", /*timestamp=*/0); + } + + if (con->aggr_mode == LOCK_AGGR_ADDR) { + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .flags = BPF_F_TEST_RUN_ON_CPU, + ); + int prog_fd = bpf_program__fd(skel->progs.collect_lock_syms); + + bpf_prog_test_run_opts(prog_fd, &opts); + } + + prev_key = NULL; + while (!bpf_map_get_next_key(fd, prev_key, &key)) { + s64 ls_key; + const char *name; + + /* to handle errors in the loop body */ + err = -1; + + bpf_map_lookup_elem(fd, &key, &data); + if (con->save_callstack) { + bpf_map_lookup_elem(stack, &key.stack_id, stack_trace); + + if (!match_callstack_filter(machine, stack_trace, con->max_stack)) { + con->nr_filtered += data.count; + goto next; + } + } + + switch (con->aggr_mode) { + case LOCK_AGGR_CALLER: + ls_key = key.stack_id; + break; + case LOCK_AGGR_TASK: + ls_key = key.pid; + break; + case LOCK_AGGR_ADDR: + case LOCK_AGGR_CGROUP: + ls_key = key.lock_addr_or_cgroup; + break; + default: + goto next; + } + + st = lock_stat_find(ls_key); + if (st != NULL) { + st->wait_time_total += data.total_time; + if (st->wait_time_max < data.max_time) + st->wait_time_max = data.max_time; + if (st->wait_time_min > data.min_time) + st->wait_time_min = data.min_time; + + st->nr_contended += data.count; + if (st->nr_contended) + st->avg_wait_time = st->wait_time_total / st->nr_contended; + goto next; + } + + name = lock_contention_get_name(con, &key, stack_trace, data.flags); + st = lock_stat_findnew(ls_key, name, data.flags); + if (st == NULL) + break; + + st->nr_contended = data.count; + st->wait_time_total = data.total_time; + st->wait_time_max = data.max_time; + st->wait_time_min = data.min_time; + + if (data.count) + st->avg_wait_time = data.total_time / data.count; + + if (con->aggr_mode == LOCK_AGGR_CALLER && verbose > 0) { + st->callstack = memdup(stack_trace, stack_size); + if (st->callstack == NULL) + break; + } + +next: + prev_key = &key; + + /* we're fine now, reset the error */ + err = 0; + } + + free(stack_trace); + + return err; +} + +int lock_contention_finish(struct lock_contention *con) +{ + if (skel) { + skel->bss->enabled = 0; + lock_contention_bpf__destroy(skel); + } + + while (!RB_EMPTY_ROOT(&con->cgroups)) { + struct rb_node *node = rb_first(&con->cgroups); + struct cgroup *cgrp = rb_entry(node, struct cgroup, node); + + rb_erase(node, &con->cgroups); + cgroup__put(cgrp); + } + + exit_slab_cache_iter(); + btf__free(con->btf); + + return 0; +} diff --git a/tools/perf/util/bpf_off_cpu.c b/tools/perf/util/bpf_off_cpu.c new file mode 100644 index 000000000..a3b699a53 --- /dev/null +++ b/tools/perf/util/bpf_off_cpu.c @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/bpf_counter.h" +#include "util/debug.h" +#include "util/evsel.h" +#include "util/evlist.h" +#include "util/off_cpu.h" +#include "util/perf-hooks.h" +#include "util/record.h" +#include "util/session.h" +#include "util/target.h" +#include "util/cpumap.h" +#include "util/thread_map.h" +#include "util/cgroup.h" +#include "util/strlist.h" +#include +#include +#include +#include + +#include "bpf_skel/off_cpu.skel.h" + +#define MAX_STACKS 32 +#define MAX_PROC 4096 +/* we don't need actual timestamp, just want to put the samples at last */ +#define OFF_CPU_TIMESTAMP (~0ull << 32) + +static struct off_cpu_bpf *skel; + +struct off_cpu_key { + u32 pid; + u32 tgid; + u32 stack_id; + u32 state; + u64 cgroup_id; +}; + +union off_cpu_data { + struct perf_event_header hdr; + u64 array[1024 / sizeof(u64)]; +}; + +static u64 off_cpu_raw[MAX_STACKS + 5]; + +static int off_cpu_config(struct evlist *evlist) +{ + char off_cpu_event[64]; + struct evsel *evsel; + + scnprintf(off_cpu_event, sizeof(off_cpu_event), "bpf-output/name=%s/", OFFCPU_EVENT); + if (parse_event(evlist, off_cpu_event)) { + pr_err("Failed to open off-cpu event\n"); + return -1; + } + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_offcpu_event(evsel)) { + evsel->core.system_wide = true; + break; + } + } + + return 0; +} + +static void off_cpu_start(void *arg) +{ + struct evlist *evlist = arg; + struct evsel *evsel; + struct perf_cpu pcpu; + unsigned int i; + + /* update task filter for the given workload */ + if (skel->rodata->has_task && skel->rodata->uses_tgid && + perf_thread_map__pid(evlist->core.threads, 0) != -1) { + int fd; + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + pid = perf_thread_map__pid(evlist->core.threads, 0); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + + /* update BPF perf_event map */ + evsel = evlist__find_evsel_by_str(evlist, OFFCPU_EVENT); + if (evsel == NULL) { + pr_err("%s evsel not found\n", OFFCPU_EVENT); + return; + } + + perf_cpu_map__for_each_cpu(pcpu, i, evsel->core.cpus) { + int err; + int cpu_nr = pcpu.cpu; + + err = bpf_map__update_elem(skel->maps.offcpu_output, &cpu_nr, sizeof(int), + xyarray__entry(evsel->core.fd, cpu_nr, 0), + sizeof(int), BPF_ANY); + if (err) { + pr_err("Failed to update perf event map for direct off-cpu dumping\n"); + return; + } + } + + skel->bss->enabled = 1; +} + +static void off_cpu_finish(void *arg __maybe_unused) +{ + skel->bss->enabled = 0; + off_cpu_bpf__destroy(skel); +} + +/* v5.18 kernel added prev_state arg, so it needs to check the signature */ +static void check_sched_switch_args(void) +{ + struct btf *btf = btf__load_vmlinux_btf(); + const struct btf_type *t1, *t2, *t3; + u32 type_id; + + if (!btf) { + pr_debug("Missing btf, check if CONFIG_DEBUG_INFO_BTF is enabled\n"); + goto cleanup; + } + + type_id = btf__find_by_name_kind(btf, "btf_trace_sched_switch", + BTF_KIND_TYPEDEF); + if ((s32)type_id < 0) + goto cleanup; + + t1 = btf__type_by_id(btf, type_id); + if (t1 == NULL) + goto cleanup; + + t2 = btf__type_by_id(btf, t1->type); + if (t2 == NULL || !btf_is_ptr(t2)) + goto cleanup; + + t3 = btf__type_by_id(btf, t2->type); + /* btf_trace func proto has one more argument for the context */ + if (t3 && btf_is_func_proto(t3) && btf_vlen(t3) == 5) { + /* new format: pass prev_state as 4th arg */ + skel->rodata->has_prev_state = true; + } +cleanup: + btf__free(btf); +} + +int off_cpu_prepare(struct evlist *evlist, struct target *target, + struct record_opts *opts) +{ + int err, fd, i; + int ncpus = 1, ntasks = 1, ncgrps = 1; + struct strlist *pid_slist = NULL; + struct str_node *pos; + + if (off_cpu_config(evlist) < 0) { + pr_err("Failed to config off-cpu BPF event\n"); + return -1; + } + + skel = off_cpu_bpf__open(); + if (!skel) { + pr_err("Failed to open off-cpu BPF skeleton\n"); + return -1; + } + + /* don't need to set cpu filter for system-wide mode */ + if (target->cpu_list) { + ncpus = perf_cpu_map__nr(evlist->core.user_requested_cpus); + bpf_map__set_max_entries(skel->maps.cpu_filter, ncpus); + skel->rodata->has_cpu = 1; + } + + if (target->pid) { + pid_slist = strlist__new(target->pid, NULL); + if (!pid_slist) { + pr_err("Failed to create a strlist for pid\n"); + return -1; + } + + ntasks = 0; + strlist__for_each_entry(pos, pid_slist) { + char *end_ptr; + int pid = strtol(pos->s, &end_ptr, 10); + + if (pid == INT_MIN || pid == INT_MAX || + (*end_ptr != '\0' && *end_ptr != ',')) + continue; + + ntasks++; + } + + if (ntasks < MAX_PROC) + ntasks = MAX_PROC; + + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + skel->rodata->uses_tgid = 1; + } else if (target__has_task(target)) { + ntasks = perf_thread_map__nr(evlist->core.threads); + bpf_map__set_max_entries(skel->maps.task_filter, ntasks); + skel->rodata->has_task = 1; + } else if (target__none(target)) { + bpf_map__set_max_entries(skel->maps.task_filter, MAX_PROC); + skel->rodata->has_task = 1; + skel->rodata->uses_tgid = 1; + } + + if (evlist__first(evlist)->cgrp) { + ncgrps = evlist->core.nr_entries - 1; /* excluding a dummy */ + bpf_map__set_max_entries(skel->maps.cgroup_filter, ncgrps); + + if (!cgroup_is_v2("perf_event")) + skel->rodata->uses_cgroup_v1 = true; + skel->rodata->has_cgroup = 1; + } + + if (opts->record_cgroup) { + skel->rodata->needs_cgroup = true; + + if (!cgroup_is_v2("perf_event")) + skel->rodata->uses_cgroup_v1 = true; + } + + set_max_rlimit(); + check_sched_switch_args(); + + err = off_cpu_bpf__load(skel); + if (err) { + pr_err("Failed to load off-cpu skeleton\n"); + goto out; + } + + if (target->cpu_list) { + u32 cpu; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cpu_filter); + + for (i = 0; i < ncpus; i++) { + cpu = perf_cpu_map__cpu(evlist->core.user_requested_cpus, i).cpu; + bpf_map_update_elem(fd, &cpu, &val, BPF_ANY); + } + } + + if (target->pid) { + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + strlist__for_each_entry(pos, pid_slist) { + char *end_ptr; + u32 tgid; + int pid = strtol(pos->s, &end_ptr, 10); + + if (pid == INT_MIN || pid == INT_MAX || + (*end_ptr != '\0' && *end_ptr != ',')) + continue; + + tgid = pid; + bpf_map_update_elem(fd, &tgid, &val, BPF_ANY); + } + } else if (target__has_task(target)) { + u32 pid; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.task_filter); + + for (i = 0; i < ntasks; i++) { + pid = perf_thread_map__pid(evlist->core.threads, i); + bpf_map_update_elem(fd, &pid, &val, BPF_ANY); + } + } + + if (evlist__first(evlist)->cgrp) { + struct evsel *evsel; + u8 val = 1; + + fd = bpf_map__fd(skel->maps.cgroup_filter); + + evlist__for_each_entry(evlist, evsel) { + struct cgroup *cgrp = evsel->cgrp; + + if (cgrp == NULL) + continue; + + if (!cgrp->id && read_cgroup_id(cgrp) < 0) { + pr_err("Failed to read cgroup id of %s\n", + cgrp->name); + goto out; + } + + bpf_map_update_elem(fd, &cgrp->id, &val, BPF_ANY); + } + } + + skel->bss->offcpu_thresh_ns = opts->off_cpu_thresh_ns; + + err = off_cpu_bpf__attach(skel); + if (err) { + pr_err("Failed to attach off-cpu BPF skeleton\n"); + goto out; + } + + if (perf_hooks__set_hook("record_start", off_cpu_start, evlist) || + perf_hooks__set_hook("record_end", off_cpu_finish, evlist)) { + pr_err("Failed to attach off-cpu skeleton\n"); + goto out; + } + + return 0; + +out: + off_cpu_bpf__destroy(skel); + return -1; +} + +int off_cpu_write(struct perf_session *session) +{ + int bytes = 0, size; + int fd, stack; + u32 raw_size; + u64 sample_type, val, sid = 0; + struct evsel *evsel; + struct perf_data_file *file = &session->data->file; + struct off_cpu_key prev, key; + union off_cpu_data data = { + .hdr = { + .type = PERF_RECORD_SAMPLE, + .misc = PERF_RECORD_MISC_USER, + }, + }; + u64 tstamp = OFF_CPU_TIMESTAMP; + + skel->bss->enabled = 0; + + evsel = evlist__find_evsel_by_str(session->evlist, OFFCPU_EVENT); + if (evsel == NULL) { + pr_err("%s evsel not found\n", OFFCPU_EVENT); + return 0; + } + + sample_type = evsel->core.attr.sample_type; + + if (sample_type & ~OFFCPU_SAMPLE_TYPES) { + pr_err("not supported sample type: %llx\n", + (unsigned long long)sample_type); + return -1; + } + + if (sample_type & (PERF_SAMPLE_ID | PERF_SAMPLE_IDENTIFIER)) { + if (evsel->core.id) + sid = evsel->core.id[0]; + } + + fd = bpf_map__fd(skel->maps.off_cpu); + stack = bpf_map__fd(skel->maps.stacks); + memset(&prev, 0, sizeof(prev)); + + while (!bpf_map_get_next_key(fd, &prev, &key)) { + int n = 1; /* start from perf_event_header */ + + bpf_map_lookup_elem(fd, &key, &val); + + /* zero-fill some of the fields, will be overwritten by raw_data when parsing */ + if (sample_type & PERF_SAMPLE_IDENTIFIER) + data.array[n++] = sid; + if (sample_type & PERF_SAMPLE_IP) + data.array[n++] = 0; /* will be updated */ + if (sample_type & PERF_SAMPLE_TID) + data.array[n++] = 0; + if (sample_type & PERF_SAMPLE_TIME) + data.array[n++] = tstamp; + if (sample_type & PERF_SAMPLE_CPU) + data.array[n++] = 0; + if (sample_type & PERF_SAMPLE_PERIOD) + data.array[n++] = 0; + if (sample_type & PERF_SAMPLE_RAW) { + /* + * [ size ][ data ] + * [ data ] + * [ data ] + * [ data ] + * [ data ][ empty] + */ + int len = 0, i = 0; + void *raw_data = (void *)data.array + n * sizeof(u64); + + off_cpu_raw[i++] = (u64)key.pid << 32 | key.tgid; + off_cpu_raw[i++] = val; + + /* off_cpu_raw[i] is callchain->nr (updated later) */ + off_cpu_raw[i + 1] = PERF_CONTEXT_USER; + off_cpu_raw[i + 2] = 0; + + bpf_map_lookup_elem(stack, &key.stack_id, &off_cpu_raw[i + 2]); + while (off_cpu_raw[i + 2 + len]) + len++; + + off_cpu_raw[i] = len + 1; + i += len + 2; + + off_cpu_raw[i++] = key.cgroup_id; + + raw_size = i * sizeof(u64) + sizeof(u32); /* 4 bytes for alignment */ + memcpy(raw_data, &raw_size, sizeof(raw_size)); + memcpy(raw_data + sizeof(u32), off_cpu_raw, i * sizeof(u64)); + + n += i + 1; + } + if (sample_type & PERF_SAMPLE_CGROUP) + data.array[n++] = key.cgroup_id; + + size = n * sizeof(u64); + data.hdr.size = size; + bytes += size; + + if (perf_data_file__write(file, &data, size) < 0) { + pr_err("failed to write perf data, error: %m\n"); + return bytes; + } + + prev = key; + /* increase dummy timestamp to sort later samples */ + tstamp++; + } + return bytes; +} diff --git a/tools/perf/util/bpf_skel/.gitignore b/tools/perf/util/bpf_skel/.gitignore new file mode 100644 index 000000000..cd01455e1 --- /dev/null +++ b/tools/perf/util/bpf_skel/.gitignore @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +.tmp +*.skel.h +vmlinux.h diff --git a/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c new file mode 100644 index 000000000..2a6e61864 --- /dev/null +++ b/tools/perf/util/bpf_skel/augmented_raw_syscalls.bpf.c @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Augment the raw_syscalls tracepoints with the contents of the pointer arguments. + * + * This exactly matches what is marshalled into the raw_syscall:sys_enter + * payload expected by the 'perf trace' beautifiers. + */ + +#include "vmlinux.h" + +#include +#include + +#define PERF_ALIGN(x, a) __PERF_ALIGN_MASK(x, (typeof(x))(a)-1) +#define __PERF_ALIGN_MASK(x, mask) (((x)+(mask))&~(mask)) + +/** + * is_power_of_2() - check if a value is a power of two + * @n: the value to check + * + * Determine whether some value is a power of two, where zero is *not* + * considered a power of two. Return: true if @n is a power of 2, otherwise + * false. + */ +#define is_power_of_2(n) (n != 0 && ((n & (n - 1)) == 0)) + +#define MAX_CPUS 4096 + +#define TRACE_AUG_MAX_BUF 32 /* for buffer augmentation in perf trace */ + +/* bpf-output associated map */ +struct __augmented_syscalls__ { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __type(key, int); + __type(value, __u32); + __uint(max_entries, MAX_CPUS); +} __augmented_syscalls__ SEC(".maps"); + +/* + * What to augment at entry? + * + * Pointer arg payloads (filenames, etc) passed from userspace to the kernel + */ +struct syscalls_sys_enter { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, 1024); +} syscalls_sys_enter SEC(".maps"); + +/* + * What to augment at exit? + * + * Pointer arg payloads returned from the kernel (struct stat, etc) to userspace. + */ +struct syscalls_sys_exit { + __uint(type, BPF_MAP_TYPE_PROG_ARRAY); + __type(key, __u32); + __type(value, __u32); + __uint(max_entries, 1024); +} syscalls_sys_exit SEC(".maps"); + +struct syscall_enter_args { + unsigned long long common_tp_fields; + long syscall_nr; + unsigned long args[6]; +}; + +struct syscall_exit_args { + unsigned long long common_tp_fields; + long syscall_nr; + long ret; +}; + +/* + * Desired design of maximum size and alignment (see RFC2553) + */ +#define SS_MAXSIZE 128 /* Implementation specific max size */ + +typedef unsigned short sa_family_t; + +/* + * FIXME: Should come from system headers + * + * The definition uses anonymous union and struct in order to control the + * default alignment. + */ +struct sockaddr_storage { + union { + struct { + sa_family_t ss_family; /* address family */ + /* Following field(s) are implementation specific */ + char __data[SS_MAXSIZE - sizeof(unsigned short)]; + /* space to achieve desired size, */ + /* _SS_MAXSIZE value minus size of ss_family */ + }; + void *__align; /* implementation specific desired alignment */ + }; +}; + +struct augmented_arg { + unsigned int size; + int err; + union { + char value[PATH_MAX]; + struct sockaddr_storage saddr; + }; +}; + +struct pids_filtered { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, pid_t); + __type(value, bool); + __uint(max_entries, 64); +} pids_filtered SEC(".maps"); + +struct augmented_args_payload { + struct syscall_enter_args args; + struct augmented_arg arg, arg2; // We have to reserve space for two arguments (rename, etc) +}; + +// We need more tmp space than the BPF stack can give us +struct augmented_args_tmp { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct augmented_args_payload); + __uint(max_entries, 1); +} augmented_args_tmp SEC(".maps"); + +struct beauty_map_enter { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); + __type(value, __u32[6]); + __uint(max_entries, 512); +} beauty_map_enter SEC(".maps"); + +struct beauty_payload_enter { + struct syscall_enter_args args; + struct augmented_arg aug_args[6]; +}; + +struct beauty_payload_enter_map { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __type(key, int); + __type(value, struct beauty_payload_enter); + __uint(max_entries, 1); +} beauty_payload_enter_map SEC(".maps"); + +static inline struct augmented_args_payload *augmented_args_payload(void) +{ + int key = 0; + return bpf_map_lookup_elem(&augmented_args_tmp, &key); +} + +static inline int augmented__output(void *ctx, struct augmented_args_payload *args, int len) +{ + /* If perf_event_output fails, return non-zero so that it gets recorded unaugmented */ + return bpf_perf_event_output(ctx, &__augmented_syscalls__, BPF_F_CURRENT_CPU, args, len); +} + +static inline int augmented__beauty_output(void *ctx, void *data, int len) +{ + return bpf_perf_event_output(ctx, &__augmented_syscalls__, BPF_F_CURRENT_CPU, data, len); +} + +static inline +unsigned int augmented_arg__read_str(struct augmented_arg *augmented_arg, const void *arg, unsigned int arg_len) +{ + unsigned int augmented_len = sizeof(*augmented_arg); + int string_len = bpf_probe_read_user_str(&augmented_arg->value, arg_len, arg); + + augmented_arg->size = augmented_arg->err = 0; + /* + * probe_read_str may return < 0, e.g. -EFAULT + * So we leave that in the augmented_arg->size that userspace will + */ + if (string_len > 0) { + augmented_len -= sizeof(augmented_arg->value) - string_len; + _Static_assert(is_power_of_2(sizeof(augmented_arg->value)), "sizeof(augmented_arg->value) needs to be a power of two"); + augmented_len &= sizeof(augmented_arg->value) - 1; + augmented_arg->size = string_len; + } else { + /* + * So that username notice the error while still being able + * to skip this augmented arg record + */ + augmented_arg->err = string_len; + augmented_len = offsetof(struct augmented_arg, value); + } + + return augmented_len; +} + +SEC("tp/raw_syscalls/sys_enter") +int syscall_unaugmented(struct syscall_enter_args *args) +{ + return 1; +} + +/* + * These will be tail_called from SEC("raw_syscalls:sys_enter"), so will find in + * augmented_args_tmp what was read by that raw_syscalls:sys_enter and go + * on from there, reading the first syscall arg as a string, i.e. open's + * filename. + */ +SEC("tp/syscalls/sys_enter_connect") +int sys_enter_connect(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *sockaddr_arg = (const void *)args->args[1]; + unsigned int socklen = args->args[2]; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + _Static_assert(is_power_of_2(sizeof(augmented_args->arg.saddr)), "sizeof(augmented_args->arg.saddr) needs to be a power of two"); + socklen &= sizeof(augmented_args->arg.saddr) - 1; + + bpf_probe_read_user(&augmented_args->arg.saddr, socklen, sockaddr_arg); + augmented_args->arg.size = socklen; + augmented_args->arg.err = 0; + + return augmented__output(args, augmented_args, len + socklen); +} + +SEC("tp/syscalls/sys_enter_sendto") +int sys_enter_sendto(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *sockaddr_arg = (const void *)args->args[4]; + unsigned int socklen = args->args[5]; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + socklen &= sizeof(augmented_args->arg.saddr) - 1; + + bpf_probe_read_user(&augmented_args->arg.saddr, socklen, sockaddr_arg); + + return augmented__output(args, augmented_args, len + socklen); +} + +SEC("tp/syscalls/sys_enter_open") +int sys_enter_open(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *filename_arg = (const void *)args->args[0]; + unsigned int len = sizeof(augmented_args->args); + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += augmented_arg__read_str(&augmented_args->arg, filename_arg, sizeof(augmented_args->arg.value)); + + return augmented__output(args, augmented_args, len); +} + +SEC("tp/syscalls/sys_enter_openat") +int sys_enter_openat(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *filename_arg = (const void *)args->args[1]; + unsigned int len = sizeof(augmented_args->args); + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += augmented_arg__read_str(&augmented_args->arg, filename_arg, sizeof(augmented_args->arg.value)); + + return augmented__output(args, augmented_args, len); +} + +SEC("tp/syscalls/sys_enter_rename") +int sys_enter_rename(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *oldpath_arg = (const void *)args->args[0], + *newpath_arg = (const void *)args->args[1]; + unsigned int len = sizeof(augmented_args->args), oldpath_len, newpath_len; + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += 2 * sizeof(u64); // The overhead of size and err, just before the payload... + + oldpath_len = augmented_arg__read_str(&augmented_args->arg, oldpath_arg, sizeof(augmented_args->arg.value)); + augmented_args->arg.size = PERF_ALIGN(oldpath_len + 1, sizeof(u64)); + len += augmented_args->arg.size; + + /* Every read from userspace is limited to value size */ + if (augmented_args->arg.size > sizeof(augmented_args->arg.value)) + return 1; /* Failure: don't filter */ + + struct augmented_arg *arg2 = (void *)&augmented_args->arg.value + augmented_args->arg.size; + + newpath_len = augmented_arg__read_str(arg2, newpath_arg, sizeof(augmented_args->arg.value)); + arg2->size = newpath_len; + + len += newpath_len; + + return augmented__output(args, augmented_args, len); +} + +SEC("tp/syscalls/sys_enter_renameat2") +int sys_enter_renameat2(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *oldpath_arg = (const void *)args->args[1], + *newpath_arg = (const void *)args->args[3]; + unsigned int len = sizeof(augmented_args->args), oldpath_len, newpath_len; + + if (augmented_args == NULL) + return 1; /* Failure: don't filter */ + + len += 2 * sizeof(u64); // The overhead of size and err, just before the payload... + + oldpath_len = augmented_arg__read_str(&augmented_args->arg, oldpath_arg, sizeof(augmented_args->arg.value)); + augmented_args->arg.size = PERF_ALIGN(oldpath_len + 1, sizeof(u64)); + len += augmented_args->arg.size; + + /* Every read from userspace is limited to value size */ + if (augmented_args->arg.size > sizeof(augmented_args->arg.value)) + return 1; /* Failure: don't filter */ + + struct augmented_arg *arg2 = (void *)&augmented_args->arg.value + augmented_args->arg.size; + + newpath_len = augmented_arg__read_str(arg2, newpath_arg, sizeof(augmented_args->arg.value)); + arg2->size = newpath_len; + + len += newpath_len; + + return augmented__output(args, augmented_args, len); +} + +#define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */ + +// we need just the start, get the size to then copy it +struct perf_event_attr_size { + __u32 type; + /* + * Size of the attr structure, for fwd/bwd compat. + */ + __u32 size; +}; + +SEC("tp/syscalls/sys_enter_perf_event_open") +int sys_enter_perf_event_open(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const struct perf_event_attr_size *attr = (const struct perf_event_attr_size *)args->args[0], *attr_read; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + + if (augmented_args == NULL) + goto failure; + + if (bpf_probe_read_user(&augmented_args->arg.value, sizeof(*attr), attr) < 0) + goto failure; + + attr_read = (const struct perf_event_attr_size *)augmented_args->arg.value; + + __u32 size = attr_read->size; + + if (!size) + size = PERF_ATTR_SIZE_VER0; + + if (size > sizeof(augmented_args->arg.value)) + goto failure; + + // Now that we read attr->size and tested it against the size limits, read it completely + if (bpf_probe_read_user(&augmented_args->arg.value, size, attr) < 0) + goto failure; + + return augmented__output(args, augmented_args, len + size); +failure: + return 1; /* Failure: don't filter */ +} + +SEC("tp/syscalls/sys_enter_clock_nanosleep") +int sys_enter_clock_nanosleep(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *rqtp_arg = (const void *)args->args[2]; + unsigned int len = sizeof(u64) + sizeof(augmented_args->args); // the size + err in all 'augmented_arg' structs + __u32 size = sizeof(struct timespec64); + + if (augmented_args == NULL) + goto failure; + + if (size > sizeof(augmented_args->arg.value)) + goto failure; + + bpf_probe_read_user(&augmented_args->arg.value, size, rqtp_arg); + + return augmented__output(args, augmented_args, len + size); +failure: + return 1; /* Failure: don't filter */ +} + +SEC("tp/syscalls/sys_enter_nanosleep") +int sys_enter_nanosleep(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args = augmented_args_payload(); + const void *req_arg = (const void *)args->args[0]; + unsigned int len = sizeof(augmented_args->args); + __u32 size = sizeof(struct timespec64); + + if (augmented_args == NULL) + goto failure; + + if (size > sizeof(augmented_args->arg.value)) + goto failure; + + bpf_probe_read_user(&augmented_args->arg.value, size, req_arg); + + return augmented__output(args, augmented_args, len + size); +failure: + return 1; /* Failure: don't filter */ +} + +static pid_t getpid(void) +{ + return bpf_get_current_pid_tgid(); +} + +static bool pid_filter__has(struct pids_filtered *pids, pid_t pid) +{ + return bpf_map_lookup_elem(pids, &pid) != NULL; +} + +static int augment_sys_enter(void *ctx, struct syscall_enter_args *args) +{ + bool augmented, do_output = false; + int zero = 0, index, value_size = sizeof(struct augmented_arg) - offsetof(struct augmented_arg, value); + u64 output = 0; /* has to be u64, otherwise it won't pass the verifier */ + s64 aug_size, size; + unsigned int nr, *beauty_map; + struct beauty_payload_enter *payload; + void *arg, *payload_offset; + + /* fall back to do predefined tail call */ + if (args == NULL) + return 1; + + /* use syscall number to get beauty_map entry */ + nr = (__u32)args->syscall_nr; + beauty_map = bpf_map_lookup_elem(&beauty_map_enter, &nr); + + /* set up payload for output */ + payload = bpf_map_lookup_elem(&beauty_payload_enter_map, &zero); + payload_offset = (void *)&payload->aug_args; + + if (beauty_map == NULL || payload == NULL) + return 1; + + /* copy the sys_enter header, which has the syscall_nr */ + __builtin_memcpy(&payload->args, args, sizeof(struct syscall_enter_args)); + + /* + * Determine what type of argument and how many bytes to read from user space, using the + * value in the beauty_map. This is the relation of parameter type and its corresponding + * value in the beauty map, and how many bytes we read eventually: + * + * string: 1 -> size of string + * struct: size of struct -> size of struct + * buffer: -1 * (index of paired len) -> value of paired len (maximum: TRACE_AUG_MAX_BUF) + */ + for (int i = 0; i < 6; i++) { + arg = (void *)args->args[i]; + augmented = false; + size = beauty_map[i]; + aug_size = size; /* size of the augmented data read from user space */ + + if (size == 0 || arg == NULL) + continue; + + if (size == 1) { /* string */ + aug_size = bpf_probe_read_user_str(((struct augmented_arg *)payload_offset)->value, value_size, arg); + /* minimum of 0 to pass the verifier */ + if (aug_size < 0) + aug_size = 0; + + augmented = true; + } else if (size > 0 && size <= value_size) { /* struct */ + if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, size, arg)) + augmented = true; + } else if ((int)size < 0 && size >= -6) { /* buffer */ + index = -(size + 1); + barrier_var(index); // Prevent clang (noticed with v18) from removing the &= 7 trick. + index &= 7; // Satisfy the bounds checking with the verifier in some kernels. + aug_size = args->args[index] > TRACE_AUG_MAX_BUF ? TRACE_AUG_MAX_BUF : args->args[index]; + + if (aug_size > 0) { + if (!bpf_probe_read_user(((struct augmented_arg *)payload_offset)->value, aug_size, arg)) + augmented = true; + } + } + + /* Augmented data size is limited to sizeof(augmented_arg->unnamed union with value field) */ + if (aug_size > value_size) + aug_size = value_size; + + /* write data to payload */ + if (augmented) { + int written = offsetof(struct augmented_arg, value) + aug_size; + + if (written < 0 || written > sizeof(struct augmented_arg)) + return 1; + + ((struct augmented_arg *)payload_offset)->size = aug_size; + output += written; + payload_offset += written; + do_output = true; + } + } + + if (!do_output || (sizeof(struct syscall_enter_args) + output) > sizeof(struct beauty_payload_enter)) + return 1; + + return augmented__beauty_output(ctx, payload, sizeof(struct syscall_enter_args) + output); +} + +SEC("tp/raw_syscalls/sys_enter") +int sys_enter(struct syscall_enter_args *args) +{ + struct augmented_args_payload *augmented_args; + /* + * We start len, the amount of data that will be in the perf ring + * buffer, if this is not filtered out by one of pid_filter__has(), + * syscall->enabled, etc, with the non-augmented raw syscall payload, + * i.e. sizeof(augmented_args->args). + * + * We'll add to this as we add augmented syscalls right after that + * initial, non-augmented raw_syscalls:sys_enter payload. + */ + + if (pid_filter__has(&pids_filtered, getpid())) + return 0; + + augmented_args = augmented_args_payload(); + if (augmented_args == NULL) + return 1; + + bpf_probe_read_kernel(&augmented_args->args, sizeof(augmented_args->args), args); + + /* + * Jump to syscall specific augmenter, even if the default one, + * "!raw_syscalls:unaugmented" that will just return 1 to return the + * unaugmented tracepoint payload. + */ + if (augment_sys_enter(args, &augmented_args->args)) + bpf_tail_call(args, &syscalls_sys_enter, augmented_args->args.syscall_nr); + + // If not found on the PROG_ARRAY syscalls map, then we're filtering it: + return 0; +} + +SEC("tp/raw_syscalls/sys_exit") +int sys_exit(struct syscall_exit_args *args) +{ + struct syscall_exit_args exit_args; + + if (pid_filter__has(&pids_filtered, getpid())) + return 0; + + bpf_probe_read_kernel(&exit_args, sizeof(exit_args), args); + /* + * Jump to syscall specific return augmenter, even if the default one, + * "!raw_syscalls:unaugmented" that will just return 1 to return the + * unaugmented tracepoint payload. + */ + bpf_tail_call(args, &syscalls_sys_exit, exit_args.syscall_nr); + /* + * If not found on the PROG_ARRAY syscalls map, then we're filtering it: + */ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c new file mode 100644 index 000000000..c2298a2de --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c @@ -0,0 +1,225 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +// Copyright (c) 2021 Google +#include "bperf_cgroup.h" +#include "vmlinux.h" +#include +#include +#include + +// NOTE: many of map and global data will be modified before loading +// from the userspace (perf tool) using the skeleton helpers. + +// single set of global perf events to measure +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); + __uint(max_entries, 1); +} events SEC(".maps"); + +// from cgroup id to event index +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, 1); +} cgrp_idx SEC(".maps"); + +// per-cpu event snapshots to calculate delta +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); +} prev_readings SEC(".maps"); + +// aggregated event values for each cgroup (per-cpu) +// will be read from the user-space +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); +} cgrp_readings SEC(".maps"); + +/* new kernel cgroup definition */ +struct cgroup___new { + int level; + struct cgroup *ancestors[]; +} __attribute__((preserve_access_index)); + +/* old kernel cgroup definition */ +struct cgroup___old { + int level; + u64 ancestor_ids[]; +} __attribute__((preserve_access_index)); + +const volatile __u32 num_events = 1; +const volatile __u32 num_cpus = 1; +const volatile int use_cgroup_v2 = 0; + +int enabled = 0; +int perf_subsys_id = -1; + +static inline __u64 get_cgroup_v1_ancestor_id(struct cgroup *cgrp, int level) +{ + /* recast pointer to capture new type for compiler */ + struct cgroup___new *cgrp_new = (void *)cgrp; + + if (bpf_core_field_exists(cgrp_new->ancestors)) { + return BPF_CORE_READ(cgrp_new, ancestors[level], kn, id); + } else { + /* recast pointer to capture old type for compiler */ + struct cgroup___old *cgrp_old = (void *)cgrp; + + return BPF_CORE_READ(cgrp_old, ancestor_ids[level]); + } +} + +static inline int get_cgroup_v1_idx(__u32 *cgrps, int size) +{ + struct task_struct *p = (void *)bpf_get_current_task(); + struct cgroup *cgrp; + register int i = 0; + __u32 *elem; + int level; + int cnt; + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_subsys_id], cgroup); + level = BPF_CORE_READ(cgrp, level); + + for (cnt = 0; i < BPERF_CGROUP__MAX_LEVELS; i++) { + __u64 cgrp_id; + + if (i > level) + break; + + // convert cgroup-id to a map index + cgrp_id = get_cgroup_v1_ancestor_id(cgrp, i); + elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); + if (!elem) + continue; + + cgrps[cnt++] = *elem; + if (cnt == size) + break; + } + + return cnt; +} + +static inline int get_cgroup_v2_idx(__u32 *cgrps, int size) +{ + register int i = 0; + __u32 *elem; + int cnt; + + for (cnt = 0; i < BPERF_CGROUP__MAX_LEVELS; i++) { + __u64 cgrp_id = bpf_get_current_ancestor_cgroup_id(i); + + if (cgrp_id == 0) + break; + + // convert cgroup-id to a map index + elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); + if (!elem) + continue; + + cgrps[cnt++] = *elem; + if (cnt == size) + break; + } + + return cnt; +} + +static int bperf_cgroup_count(void) +{ + register __u32 idx = 0; // to have it in a register to pass BPF verifier + register int c = 0; + struct bpf_perf_event_value val, delta, *prev_val, *cgrp_val; + __u32 cpu = bpf_get_smp_processor_id(); + __u32 cgrp_idx[BPERF_CGROUP__MAX_LEVELS]; + int cgrp_cnt; + __u32 key, cgrp; + long err; + + if (use_cgroup_v2) + cgrp_cnt = get_cgroup_v2_idx(cgrp_idx, BPERF_CGROUP__MAX_LEVELS); + else + cgrp_cnt = get_cgroup_v1_idx(cgrp_idx, BPERF_CGROUP__MAX_LEVELS); + + for ( ; idx < BPERF_CGROUP__MAX_EVENTS; idx++) { + if (idx == num_events) + break; + + // XXX: do not pass idx directly (for verifier) + key = idx; + // this is per-cpu array for diff + prev_val = bpf_map_lookup_elem(&prev_readings, &key); + if (!prev_val) { + val.counter = val.enabled = val.running = 0; + bpf_map_update_elem(&prev_readings, &key, &val, BPF_ANY); + + prev_val = bpf_map_lookup_elem(&prev_readings, &key); + if (!prev_val) + continue; + } + + // read from global perf_event array + key = idx * num_cpus + cpu; + err = bpf_perf_event_read_value(&events, key, &val, sizeof(val)); + if (err) + continue; + + if (enabled) { + delta.counter = val.counter - prev_val->counter; + delta.enabled = val.enabled - prev_val->enabled; + delta.running = val.running - prev_val->running; + + for (c = 0; c < BPERF_CGROUP__MAX_LEVELS; c++) { + if (c == cgrp_cnt) + break; + + cgrp = cgrp_idx[c]; + + // aggregate the result by cgroup + key = cgrp * num_events + idx; + cgrp_val = bpf_map_lookup_elem(&cgrp_readings, &key); + if (cgrp_val) { + cgrp_val->counter += delta.counter; + cgrp_val->enabled += delta.enabled; + cgrp_val->running += delta.running; + } else { + bpf_map_update_elem(&cgrp_readings, &key, + &delta, BPF_ANY); + } + } + } + + *prev_val = val; + } + return 0; +} + +// This will be attached to cgroup-switches event for each cpu +SEC("perf_event") +int BPF_PROG(on_cgrp_switch) +{ + return bperf_cgroup_count(); +} + +SEC("raw_tp/sched_switch") +int BPF_PROG(trigger_read) +{ + return bperf_cgroup_count(); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.h b/tools/perf/util/bpf_skel/bperf_cgroup.h new file mode 100644 index 000000000..3fb84b19d --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_cgroup.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ +/* Data structures shared between BPF and tools. */ +#ifndef __BPERF_CGROUP_H +#define __BPERF_CGROUP_H + +// These constants impact code size of bperf_cgroup.bpf.c that may result in BPF +// verifier issues. They are exposed to control the size and also to disable BPF +// counters when the number of user events is too large. + +// max cgroup hierarchy level: arbitrary +#define BPERF_CGROUP__MAX_LEVELS 10 +// max events per cgroup: arbitrary +#define BPERF_CGROUP__MAX_EVENTS 128 + +#endif /* __BPERF_CGROUP_H */ diff --git a/tools/perf/util/bpf_skel/bperf_follower.bpf.c b/tools/perf/util/bpf_skel/bperf_follower.bpf.c new file mode 100644 index 000000000..059506313 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_follower.bpf.c @@ -0,0 +1,162 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +#include "vmlinux.h" +#include +#include +#include "bperf_u.h" + +#define MAX_ENTRIES 102400 + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} diff_readings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} accum_readings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bperf_filter_value)); + __uint(max_entries, MAX_ENTRIES); + __uint(map_flags, BPF_F_NO_PREALLOC); +} filter SEC(".maps"); + +enum bperf_filter_type type = 0; +int enabled = 0; +int inherit; + +SEC("fexit/XXX") +int BPF_PROG(fexit_XXX) +{ + struct bpf_perf_event_value *diff_val, *accum_val; + __u32 filter_key, zero = 0; + __u32 accum_key; + struct bperf_filter_value *fval; + + if (!enabled) + return 0; + + switch (type) { + case BPERF_FILTER_GLOBAL: + accum_key = zero; + goto do_add; + case BPERF_FILTER_CPU: + filter_key = bpf_get_smp_processor_id(); + break; + case BPERF_FILTER_PID: + filter_key = bpf_get_current_pid_tgid() & 0xffffffff; + break; + case BPERF_FILTER_TGID: + /* Use pid as the filter_key to exclude new task counts + * when inherit is disabled. Don't worry about the existing + * children in TGID losing their counts, bpf_counter has + * already added them to the filter map via perf_thread_map + * before this bpf prog runs. + */ + filter_key = inherit ? + bpf_get_current_pid_tgid() >> 32 : + bpf_get_current_pid_tgid() & 0xffffffff; + break; + default: + return 0; + } + + fval = bpf_map_lookup_elem(&filter, &filter_key); + if (!fval) + return 0; + + accum_key = fval->accum_key; + if (fval->exited) + bpf_map_delete_elem(&filter, &filter_key); + +do_add: + diff_val = bpf_map_lookup_elem(&diff_readings, &zero); + if (!diff_val) + return 0; + + accum_val = bpf_map_lookup_elem(&accum_readings, &accum_key); + if (!accum_val) + return 0; + + accum_val->counter += diff_val->counter; + accum_val->enabled += diff_val->enabled; + accum_val->running += diff_val->running; + + return 0; +} + +/* The program is only used for PID or TGID filter types. */ +SEC("tp_btf/task_newtask") +int BPF_PROG(on_newtask, struct task_struct *task, __u64 clone_flags) +{ + __u32 parent_key, child_key; + struct bperf_filter_value *parent_fval; + struct bperf_filter_value child_fval = { 0 }; + + if (!enabled) + return 0; + + switch (type) { + case BPERF_FILTER_PID: + parent_key = bpf_get_current_pid_tgid() & 0xffffffff; + child_key = task->pid; + break; + case BPERF_FILTER_TGID: + parent_key = bpf_get_current_pid_tgid() >> 32; + child_key = task->tgid; + if (child_key == parent_key) + return 0; + break; + default: + return 0; + } + + /* Check if the current task is one of the target tasks to be counted */ + parent_fval = bpf_map_lookup_elem(&filter, &parent_key); + if (!parent_fval) + return 0; + + /* Start counting for the new task by adding it into filter map, + * inherit the accum key of its parent task so that they can be + * counted together. + */ + child_fval.accum_key = parent_fval->accum_key; + child_fval.exited = 0; + bpf_map_update_elem(&filter, &child_key, &child_fval, BPF_NOEXIST); + + return 0; +} + +/* The program is only used for PID or TGID filter types. */ +SEC("tp_btf/sched_process_exit") +int BPF_PROG(on_exittask, struct task_struct *task) +{ + __u32 pid; + struct bperf_filter_value *fval; + + if (!enabled) + return 0; + + /* Stop counting for this task by removing it from filter map. + * For TGID type, if the pid can be found in the map, it means that + * this pid belongs to the leader task. After the task exits, the + * tgid of its child tasks (if any) will be 1, so the pid can be + * safely removed. + */ + pid = task->pid; + fval = bpf_map_lookup_elem(&filter, &pid); + if (fval) + fval->exited = 1; + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_leader.bpf.c b/tools/perf/util/bpf_skel/bperf_leader.bpf.c new file mode 100644 index 000000000..e2a2d4cd7 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_leader.bpf.c @@ -0,0 +1,55 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +#include "vmlinux.h" +#include +#include + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); + __uint(map_flags, BPF_F_PRESERVE_ELEMS); +} events SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} prev_readings SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} diff_readings SEC(".maps"); + +SEC("raw_tp/sched_switch") +int BPF_PROG(on_switch) +{ + struct bpf_perf_event_value val, *prev_val, *diff_val; + __u32 key = bpf_get_smp_processor_id(); + __u32 zero = 0; + long err; + + prev_val = bpf_map_lookup_elem(&prev_readings, &zero); + if (!prev_val) + return 0; + + diff_val = bpf_map_lookup_elem(&diff_readings, &zero); + if (!diff_val) + return 0; + + err = bpf_perf_event_read_value(&events, key, &val, sizeof(val)); + if (err) + return 0; + + diff_val->counter = val.counter - prev_val->counter; + diff_val->enabled = val.enabled - prev_val->enabled; + diff_val->running = val.running - prev_val->running; + *prev_val = val; + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/bperf_u.h b/tools/perf/util/bpf_skel/bperf_u.h new file mode 100644 index 000000000..4a4a75398 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_u.h @@ -0,0 +1,19 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook + +#ifndef __BPERF_STAT_U_H +#define __BPERF_STAT_U_H + +enum bperf_filter_type { + BPERF_FILTER_GLOBAL = 1, + BPERF_FILTER_CPU, + BPERF_FILTER_PID, + BPERF_FILTER_TGID, +}; + +struct bperf_filter_value { + __u32 accum_key; + __u8 exited; +}; + +#endif /* __BPERF_STAT_U_H */ diff --git a/tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c b/tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c new file mode 100644 index 000000000..97037d3b3 --- /dev/null +++ b/tools/perf/util/bpf_skel/bpf_prog_profiler.bpf.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2020 Facebook +#include "vmlinux.h" +#include +#include + +/* map of perf event fds, num_cpu * num_metric entries */ +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); +} events SEC(".maps"); + +/* readings at fentry */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} fentry_readings SEC(".maps"); + +/* accumulated readings */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); + __uint(max_entries, 1); +} accum_readings SEC(".maps"); + +const volatile __u32 num_cpu = 1; + +SEC("fentry/XXX") +int BPF_PROG(fentry_XXX) +{ + __u32 key = bpf_get_smp_processor_id(); + struct bpf_perf_event_value *ptr; + __u32 zero = 0; + long err; + + /* look up before reading, to reduce error */ + ptr = bpf_map_lookup_elem(&fentry_readings, &zero); + if (!ptr) + return 0; + + err = bpf_perf_event_read_value(&events, key, ptr, sizeof(*ptr)); + if (err) + return 0; + + return 0; +} + +static inline void +fexit_update_maps(struct bpf_perf_event_value *after) +{ + struct bpf_perf_event_value *before, diff; + __u32 zero = 0; + + before = bpf_map_lookup_elem(&fentry_readings, &zero); + /* only account samples with a valid fentry_reading */ + if (before && before->counter) { + struct bpf_perf_event_value *accum; + + diff.counter = after->counter - before->counter; + diff.enabled = after->enabled - before->enabled; + diff.running = after->running - before->running; + + accum = bpf_map_lookup_elem(&accum_readings, &zero); + if (accum) { + accum->counter += diff.counter; + accum->enabled += diff.enabled; + accum->running += diff.running; + } + } +} + +SEC("fexit/XXX") +int BPF_PROG(fexit_XXX) +{ + struct bpf_perf_event_value reading; + __u32 cpu = bpf_get_smp_processor_id(); + int err; + + /* read all events before updating the maps, to reduce error */ + err = bpf_perf_event_read_value(&events, cpu, &reading, sizeof(reading)); + if (err) + return 0; + + fexit_update_maps(&reading); + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/func_latency.bpf.c b/tools/perf/util/bpf_skel/func_latency.bpf.c new file mode 100644 index 000000000..621e2022c --- /dev/null +++ b/tools/perf/util/bpf_skel/func_latency.bpf.c @@ -0,0 +1,199 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Google +#include "vmlinux.h" +#include +#include + +// This should be in sync with "util/ftrace.h" +#define NUM_BUCKET 22 + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, 10000); +} functime SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, NUM_BUCKET); +} latency SEC(".maps"); + + +int enabled = 0; + +// stats +__s64 total; +__s64 count; +__s64 max; +__s64 min; + +const volatile int has_cpu = 0; +const volatile int has_task = 0; +const volatile int use_nsec = 0; +const volatile unsigned int bucket_range; +const volatile unsigned int min_latency; +const volatile unsigned int max_latency; +const volatile unsigned int bucket_num = NUM_BUCKET; + +static bool can_record(void) +{ + if (has_cpu) { + __u32 cpu = bpf_get_smp_processor_id(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return false; + } + + if (has_task) { + __u32 pid = bpf_get_current_pid_tgid(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return false; + } + return true; +} + +static void update_latency(__s64 delta) +{ + __u64 val = delta; + __u32 key = 0; + __u64 *hist; + __u64 cmp_base = use_nsec ? 1 : 1000; + + if (delta < 0) + return; + + if (bucket_range != 0) { + val = delta / cmp_base; + + if (min_latency > 0) { + if (val > min_latency) + val -= min_latency; + else + goto do_lookup; + } + + // Less than 1 unit (ms or ns), or, in the future, + // than the min latency desired. + if (val > 0) { // 1st entry: [ 1 unit .. bucket_range units ) + key = val / bucket_range + 1; + if (key >= bucket_num) + key = bucket_num - 1; + } + + goto do_lookup; + } + // calculate index using delta + for (key = 0; key < (bucket_num - 1); key++) { + if (delta < (cmp_base << key)) + break; + } + +do_lookup: + hist = bpf_map_lookup_elem(&latency, &key); + if (!hist) + return; + + __sync_fetch_and_add(hist, 1); + + __sync_fetch_and_add(&total, delta); // always in nsec + __sync_fetch_and_add(&count, 1); + + if (delta > max) + max = delta; + if (delta < min) + min = delta; +} + +SEC("kprobe/func") +int BPF_PROG(func_begin) +{ + __u64 key, now; + + if (!enabled || !can_record()) + return 0; + + key = bpf_get_current_pid_tgid(); + now = bpf_ktime_get_ns(); + + // overwrite timestamp for nested functions + bpf_map_update_elem(&functime, &key, &now, BPF_ANY); + return 0; +} + +SEC("kretprobe/func") +int BPF_PROG(func_end) +{ + __u64 tid; + __u64 *start; + + if (!enabled) + return 0; + + tid = bpf_get_current_pid_tgid(); + + start = bpf_map_lookup_elem(&functime, &tid); + if (start) { + update_latency(bpf_ktime_get_ns() - *start); + bpf_map_delete_elem(&functime, &tid); + } + + return 0; +} + +SEC("raw_tp") +int BPF_PROG(event_begin) +{ + __u64 key, now; + + if (!enabled || !can_record()) + return 0; + + key = bpf_get_current_pid_tgid(); + now = bpf_ktime_get_ns(); + + // overwrite timestamp for nested events + bpf_map_update_elem(&functime, &key, &now, BPF_ANY); + return 0; +} + +SEC("raw_tp") +int BPF_PROG(event_end) +{ + __u64 tid; + __u64 *start; + + if (!enabled) + return 0; + + tid = bpf_get_current_pid_tgid(); + + start = bpf_map_lookup_elem(&functime, &tid); + if (start) { + update_latency(bpf_ktime_get_ns() - *start); + bpf_map_delete_elem(&functime, &tid); + } + + return 0; +} diff --git a/tools/perf/util/bpf_skel/kwork_top.bpf.c b/tools/perf/util/bpf_skel/kwork_top.bpf.c new file mode 100644 index 000000000..667338630 --- /dev/null +++ b/tools/perf/util/bpf_skel/kwork_top.bpf.c @@ -0,0 +1,338 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022, Huawei + +#include "vmlinux.h" +#include +#include +#include + +/* + * This should be in sync with "util/kwork.h" + */ +enum kwork_class_type { + KWORK_CLASS_IRQ, + KWORK_CLASS_SOFTIRQ, + KWORK_CLASS_WORKQUEUE, + KWORK_CLASS_SCHED, + KWORK_CLASS_MAX, +}; + +#define MAX_ENTRIES 102400 +#define MAX_NR_CPUS 4096 +#define PF_KTHREAD 0x00200000 +#define MAX_COMMAND_LEN 16 + +struct time_data { + __u64 timestamp; +}; + +struct work_data { + __u64 runtime; +}; + +struct task_data { + __u32 tgid; + __u32 is_kthread; + char comm[MAX_COMMAND_LEN]; +}; + +struct work_key { + __u32 type; + __u32 pid; + __u64 task_p; +}; + +struct task_key { + __u32 pid; + __u32 cpu; +}; + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct time_data); +} kwork_top_task_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct time_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_irq_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct task_key)); + __uint(value_size, sizeof(struct task_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_tasks SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct work_data)); + __uint(max_entries, MAX_ENTRIES); +} kwork_top_works SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(u32)); + __uint(value_size, sizeof(u8)); + __uint(max_entries, MAX_NR_CPUS); +} kwork_top_cpu_filter SEC(".maps"); + +int enabled = 0; + +const volatile int has_cpu_filter = 0; + +__u64 from_timestamp = 0; +__u64 to_timestamp = 0; + +static __always_inline int cpu_is_filtered(__u32 cpu) +{ + __u8 *cpu_val; + + if (has_cpu_filter) { + cpu_val = bpf_map_lookup_elem(&kwork_top_cpu_filter, &cpu); + if (!cpu_val) + return 1; + } + + return 0; +} + +static __always_inline void update_task_info(struct task_struct *task, __u32 cpu) +{ + struct task_key key = { + .pid = task->pid, + .cpu = cpu, + }; + + if (!bpf_map_lookup_elem(&kwork_top_tasks, &key)) { + struct task_data data = { + .tgid = task->tgid, + .is_kthread = task->flags & PF_KTHREAD ? 1 : 0, + }; + BPF_CORE_READ_STR_INTO(&data.comm, task, comm); + + bpf_map_update_elem(&kwork_top_tasks, &key, &data, BPF_ANY); + } +} + +static __always_inline void update_work(struct work_key *key, __u64 delta) +{ + struct work_data *data; + + data = bpf_map_lookup_elem(&kwork_top_works, key); + if (data) { + data->runtime += delta; + } else { + struct work_data new_data = { + .runtime = delta, + }; + + bpf_map_update_elem(&kwork_top_works, key, &new_data, BPF_ANY); + } +} + +static void on_sched_out(struct task_struct *task, __u64 ts, __u32 cpu) +{ + __u64 delta; + struct time_data *pelem; + + pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, 0); + if (pelem) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + struct work_key key = { + .type = KWORK_CLASS_SCHED, + .pid = task->pid, + .task_p = (__u64)task, + }; + + update_work(&key, delta); + update_task_info(task, cpu); +} + +static void on_sched_in(struct task_struct *task, __u64 ts) +{ + struct time_data *pelem; + + pelem = bpf_task_storage_get(&kwork_top_task_time, task, NULL, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (pelem) + pelem->timestamp = ts; +} + +SEC("tp_btf/sched_switch") +int on_switch(u64 *ctx) +{ + struct task_struct *prev, *next; + + prev = (struct task_struct *)ctx[1]; + next = (struct task_struct *)ctx[2]; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + on_sched_out(prev, ts, cpu); + on_sched_in(next, ts); + + return 0; +} + +SEC("tp_btf/irq_handler_entry") +int on_irq_handler_entry(u64 *cxt) +{ + struct task_struct *task; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + struct time_data data = { + .timestamp = ts, + }; + + bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY); + + return 0; +} + +SEC("tp_btf/irq_handler_exit") +int on_irq_handler_exit(u64 *cxt) +{ + __u64 delta; + struct task_struct *task; + struct time_data *pelem; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key); + if (pelem && pelem->timestamp != 0) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + update_work(&key, delta); + + return 0; +} + +SEC("tp_btf/softirq_entry") +int on_softirq_entry(u64 *cxt) +{ + struct task_struct *task; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + struct time_data data = { + .timestamp = ts, + }; + + bpf_map_update_elem(&kwork_top_irq_time, &key, &data, BPF_ANY); + + return 0; +} + +SEC("tp_btf/softirq_exit") +int on_softirq_exit(u64 *cxt) +{ + __u64 delta; + struct task_struct *task; + struct time_data *pelem; + + if (!enabled) + return 0; + + __u32 cpu = bpf_get_smp_processor_id(); + + if (cpu_is_filtered(cpu)) + return 0; + + __u64 ts = bpf_ktime_get_ns(); + + task = (struct task_struct *)bpf_get_current_task(); + if (!task) + return 0; + + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .pid = BPF_CORE_READ(task, pid), + .task_p = (__u64)task, + }; + + pelem = bpf_map_lookup_elem(&kwork_top_irq_time, &key); + if (pelem) + delta = ts - pelem->timestamp; + else + delta = ts - from_timestamp; + + update_work(&key, delta); + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/kwork_trace.bpf.c b/tools/perf/util/bpf_skel/kwork_trace.bpf.c new file mode 100644 index 000000000..9ce9c8ddd --- /dev/null +++ b/tools/perf/util/bpf_skel/kwork_trace.bpf.c @@ -0,0 +1,384 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022, Huawei + +#include "vmlinux.h" +#include +#include + +#define KWORK_COUNT 100 +#define MAX_KWORKNAME 128 + +/* + * This should be in sync with "util/kwork.h" + */ +enum kwork_class_type { + KWORK_CLASS_IRQ, + KWORK_CLASS_SOFTIRQ, + KWORK_CLASS_WORKQUEUE, + KWORK_CLASS_MAX, +}; + +struct work_key { + __u32 type; + __u32 cpu; + __u64 id; +}; + +struct report_data { + __u64 nr; + __u64 total_time; + __u64 max_time; + __u64 max_time_start; + __u64 max_time_end; +}; + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, MAX_KWORKNAME); + __uint(max_entries, KWORK_COUNT); +} perf_kwork_names SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, KWORK_COUNT); +} perf_kwork_time SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct work_key)); + __uint(value_size, sizeof(struct report_data)); + __uint(max_entries, KWORK_COUNT); +} perf_kwork_report SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} perf_kwork_cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, MAX_KWORKNAME); + __uint(max_entries, 1); +} perf_kwork_name_filter SEC(".maps"); + +int enabled = 0; + +const volatile int has_cpu_filter = 0; +const volatile int has_name_filter = 0; + +static __always_inline int local_strncmp(const char *s1, + unsigned int sz, const char *s2) +{ + int ret = 0; + unsigned int i; + + for (i = 0; i < sz; i++) { + ret = (unsigned char)s1[i] - (unsigned char)s2[i]; + if (ret || !s1[i]) + break; + } + + return ret; +} + +static __always_inline int trace_event_match(struct work_key *key, char *name) +{ + __u8 *cpu_val; + char *name_val; + __u32 zero = 0; + __u32 cpu = bpf_get_smp_processor_id(); + + if (!enabled) + return 0; + + if (has_cpu_filter) { + cpu_val = bpf_map_lookup_elem(&perf_kwork_cpu_filter, &cpu); + if (!cpu_val) + return 0; + } + + if (has_name_filter && (name != NULL)) { + name_val = bpf_map_lookup_elem(&perf_kwork_name_filter, &zero); + if (name_val && + (local_strncmp(name_val, MAX_KWORKNAME, name) != 0)) { + return 0; + } + } + + return 1; +} + +static __always_inline void do_update_time(void *map, struct work_key *key, + __u64 time_start, __u64 time_end) +{ + struct report_data zero, *data; + __s64 delta = time_end - time_start; + + if (delta < 0) + return; + + data = bpf_map_lookup_elem(map, key); + if (!data) { + __builtin_memset(&zero, 0, sizeof(zero)); + bpf_map_update_elem(map, key, &zero, BPF_NOEXIST); + data = bpf_map_lookup_elem(map, key); + if (!data) + return; + } + + if ((delta > data->max_time) || + (data->max_time == 0)) { + data->max_time = delta; + data->max_time_start = time_start; + data->max_time_end = time_end; + } + + data->total_time += delta; + data->nr++; +} + +static __always_inline void do_update_timestart(void *map, struct work_key *key) +{ + __u64 ts = bpf_ktime_get_ns(); + + bpf_map_update_elem(map, key, &ts, BPF_ANY); +} + +static __always_inline void do_update_timeend(void *report_map, void *time_map, + struct work_key *key) +{ + __u64 *time = bpf_map_lookup_elem(time_map, key); + + if (time) { + bpf_map_delete_elem(time_map, key); + do_update_time(report_map, key, *time, bpf_ktime_get_ns()); + } +} + +static __always_inline void do_update_name(void *map, + struct work_key *key, char *name) +{ + if (!bpf_map_lookup_elem(map, key)) + bpf_map_update_elem(map, key, name, BPF_ANY); +} + +static __always_inline int update_timestart(void *map, struct work_key *key) +{ + if (!trace_event_match(key, NULL)) + return 0; + + do_update_timestart(map, key); + return 0; +} + +static __always_inline int update_timestart_and_name(void *time_map, + void *names_map, + struct work_key *key, + char *name) +{ + if (!trace_event_match(key, name)) + return 0; + + do_update_timestart(time_map, key); + do_update_name(names_map, key, name); + + return 0; +} + +static __always_inline int update_timeend(void *report_map, + void *time_map, struct work_key *key) +{ + if (!trace_event_match(key, NULL)) + return 0; + + do_update_timeend(report_map, time_map, key); + + return 0; +} + +static __always_inline int update_timeend_and_name(void *report_map, + void *time_map, + void *names_map, + struct work_key *key, + char *name) +{ + if (!trace_event_match(key, name)) + return 0; + + do_update_timeend(report_map, time_map, key); + do_update_name(names_map, key, name); + + return 0; +} + +SEC("tracepoint/irq/irq_handler_entry") +int report_irq_handler_entry(struct trace_event_raw_irq_handler_entry *ctx) +{ + char name[MAX_KWORKNAME]; + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->irq, + }; + void *name_addr = (void *)ctx + (ctx->__data_loc_name & 0xffff); + + bpf_probe_read_kernel_str(name, sizeof(name), name_addr); + + return update_timestart_and_name(&perf_kwork_time, + &perf_kwork_names, &key, name); +} + +SEC("tracepoint/irq/irq_handler_exit") +int report_irq_handler_exit(struct trace_event_raw_irq_handler_exit *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_IRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->irq, + }; + + return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); +} + +static char softirq_name_list[NR_SOFTIRQS][MAX_KWORKNAME] = { + { "HI" }, + { "TIMER" }, + { "NET_TX" }, + { "NET_RX" }, + { "BLOCK" }, + { "IRQ_POLL" }, + { "TASKLET" }, + { "SCHED" }, + { "HRTIMER" }, + { "RCU" }, +}; + +SEC("tracepoint/irq/softirq_entry") +int report_softirq_entry(struct trace_event_raw_softirq *ctx) +{ + unsigned int vec = ctx->vec; + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)vec, + }; + + if (vec < NR_SOFTIRQS) { + return update_timestart_and_name(&perf_kwork_time, + &perf_kwork_names, &key, + softirq_name_list[vec]); + } + + return 0; +} + +SEC("tracepoint/irq/softirq_exit") +int report_softirq_exit(struct trace_event_raw_softirq *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->vec, + }; + + return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); +} + +SEC("tracepoint/irq/softirq_raise") +int latency_softirq_raise(struct trace_event_raw_softirq *ctx) +{ + unsigned int vec = ctx->vec; + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)vec, + }; + + if (vec < NR_SOFTIRQS) { + return update_timestart_and_name(&perf_kwork_time, + &perf_kwork_names, &key, + softirq_name_list[vec]); + } + + return 0; +} + +SEC("tracepoint/irq/softirq_entry") +int latency_softirq_entry(struct trace_event_raw_softirq *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_SOFTIRQ, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->vec, + }; + + return update_timeend(&perf_kwork_report, &perf_kwork_time, &key); +} + +SEC("tracepoint/workqueue/workqueue_execute_start") +int report_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + + return update_timestart(&perf_kwork_time, &key); +} + +SEC("tracepoint/workqueue/workqueue_execute_end") +int report_workqueue_execute_end(struct trace_event_raw_workqueue_execute_end *ctx) +{ + char name[MAX_KWORKNAME]; + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + unsigned long long func_addr = (unsigned long long)ctx->function; + + __builtin_memset(name, 0, sizeof(name)); + bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr)); + + return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time, + &perf_kwork_names, &key, name); +} + +SEC("tracepoint/workqueue/workqueue_activate_work") +int latency_workqueue_activate_work(struct trace_event_raw_workqueue_activate_work *ctx) +{ + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + + return update_timestart(&perf_kwork_time, &key); +} + +SEC("tracepoint/workqueue/workqueue_execute_start") +int latency_workqueue_execute_start(struct trace_event_raw_workqueue_execute_start *ctx) +{ + char name[MAX_KWORKNAME]; + struct work_key key = { + .type = KWORK_CLASS_WORKQUEUE, + .cpu = bpf_get_smp_processor_id(), + .id = (__u64)ctx->work, + }; + unsigned long long func_addr = (unsigned long long)ctx->function; + + __builtin_memset(name, 0, sizeof(name)); + bpf_snprintf(name, sizeof(name), "%ps", &func_addr, sizeof(func_addr)); + + return update_timeend_and_name(&perf_kwork_report, &perf_kwork_time, + &perf_kwork_names, &key, name); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/lock_contention.bpf.c b/tools/perf/util/bpf_skel/lock_contention.bpf.c new file mode 100644 index 000000000..0d9c6f550 --- /dev/null +++ b/tools/perf/util/bpf_skel/lock_contention.bpf.c @@ -0,0 +1,1052 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022 Google +#include "vmlinux.h" +#include +#include +#include +#include + +#include "lock_data.h" + +/* for collect_lock_syms(). 4096 was rejected by the verifier */ +#define MAX_CPUS 1024 + +/* for collect_zone_lock(). It should be more than the actual zones. */ +#define MAX_ZONES 10 + +/* for do_lock_delay(). Arbitrarily set to 1 million. */ +#define MAX_LOOP (1U << 20) + +/* lock contention flags from include/trace/events/lock.h */ +#define LCB_F_SPIN (1U << 0) +#define LCB_F_READ (1U << 1) +#define LCB_F_WRITE (1U << 2) +#define LCB_F_RT (1U << 3) +#define LCB_F_PERCPU (1U << 4) +#define LCB_F_MUTEX (1U << 5) + +/* callstack storage */ +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, MAX_ENTRIES); +} stacks SEC(".maps"); + +/* buffer for owner stacktrace */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, 1); +} stack_buf SEC(".maps"); + +/* a map for tracing owner stacktrace to owner stack id */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); // owner stacktrace + __uint(value_size, sizeof(__s32)); // owner stack id + __uint(max_entries, 1); +} owner_stacks SEC(".maps"); + +/* a map for tracing lock address to owner data */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); // lock address + __uint(value_size, sizeof(struct owner_tracing_data)); + __uint(max_entries, 1); +} owner_data SEC(".maps"); + +/* a map for contention_key (stores owner stack id) to contention data */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct contention_key)); + __uint(value_size, sizeof(struct contention_data)); + __uint(max_entries, 1); +} owner_stat SEC(".maps"); + +/* maintain timestamp at the beginning of contention */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); + __type(value, struct tstamp_data); + __uint(max_entries, MAX_ENTRIES); +} tstamp SEC(".maps"); + +/* maintain per-CPU timestamp at the beginning of contention */ +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct tstamp_data)); + __uint(max_entries, 1); +} tstamp_cpu SEC(".maps"); + +/* actual lock contention statistics */ +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct contention_key)); + __uint(value_size, sizeof(struct contention_data)); + __uint(max_entries, MAX_ENTRIES); +} lock_stat SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct contention_task_data)); + __uint(max_entries, MAX_ENTRIES); +} task_data SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, MAX_ENTRIES); +} lock_syms SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} type_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} addr_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cgroup_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(long)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} slab_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(long)); + __uint(value_size, sizeof(struct slab_cache_data)); + __uint(max_entries, 1); +} slab_caches SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, 1); +} lock_delays SEC(".maps"); + +struct rw_semaphore___old { + struct task_struct *owner; +} __attribute__((preserve_access_index)); + +struct rw_semaphore___new { + atomic_long_t owner; +} __attribute__((preserve_access_index)); + +struct mm_struct___old { + struct rw_semaphore mmap_sem; +} __attribute__((preserve_access_index)); + +struct mm_struct___new { + struct rw_semaphore mmap_lock; +} __attribute__((preserve_access_index)); + +struct cas_ctx { + struct contention_data *data; + u64 duration; + int max_done; + int min_done; +}; + +extern struct kmem_cache *bpf_get_kmem_cache(u64 addr) __ksym __weak; + +/* control flags */ +const volatile int has_cpu; +const volatile int has_task; +const volatile int has_type; +const volatile int has_addr; +const volatile int has_cgroup; +const volatile int has_slab; +const volatile int has_mmap_lock; +const volatile int needs_callstack; +const volatile int stack_skip; +const volatile int lock_owner; +const volatile int use_cgroup_v2; +const volatile int max_stack; +const volatile int lock_delay; + +/* determine the key of lock stat */ +const volatile int aggr_mode; + +int enabled; + +int perf_subsys_id = -1; + +__u64 end_ts; + +__u32 slab_cache_id; + +/* error stat */ +int task_fail; +int stack_fail; +int time_fail; +int data_fail; + +int task_map_full; +int data_map_full; + +struct task_struct *bpf_task_from_pid(s32 pid) __ksym __weak; +void bpf_task_release(struct task_struct *p) __ksym __weak; + +static inline __u32 check_lock_type(__u64 lock, __u32 flags); + +static inline __u64 get_current_cgroup_id(void) +{ + struct task_struct *task; + struct cgroup *cgrp; + + if (use_cgroup_v2) + return bpf_get_current_cgroup_id(); + + task = bpf_get_current_task_btf(); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + +static inline int can_record(u64 *ctx) +{ + bool is_addr_ok = false; + + if (has_cpu) { + __u32 cpu = bpf_get_smp_processor_id(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return 0; + } + + if (has_task) { + __u8 *ok; + __u32 pid = bpf_get_current_pid_tgid(); + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return 0; + } + + if (has_type) { + __u8 *ok; + __u32 flags = (__u32)ctx[1]; + + ok = bpf_map_lookup_elem(&type_filter, &flags); + if (!ok) + return 0; + } + + if (has_addr) { + __u8 *ok; + __u64 addr = ctx[0]; + + ok = bpf_map_lookup_elem(&addr_filter, &addr); + if (!ok && !has_slab && !has_mmap_lock) + return 0; + + is_addr_ok = !!ok; + } + + if (has_cgroup) { + __u8 *ok; + __u64 cgrp = get_current_cgroup_id(); + + ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp); + if (!ok) + return 0; + } + + if (is_addr_ok) + return 1; + + /* slab and mmap_lock are part of the addr_filter */ + if (has_slab && bpf_get_kmem_cache) { + __u8 *ok; + __u64 addr = ctx[0]; + long kmem_cache_addr; + + kmem_cache_addr = (long)bpf_get_kmem_cache(addr); + ok = bpf_map_lookup_elem(&slab_filter, &kmem_cache_addr); + if (ok) + return 1; + else if (!has_mmap_lock) + return 0; + } + + if (has_mmap_lock) { + __u64 lock = ctx[0]; + __u32 flag = ctx[1]; + + if (check_lock_type(lock, flag) != LCD_F_MMAP_LOCK) + return 0; + } + + return 1; +} + +static inline int update_task_data(struct task_struct *task) +{ + struct contention_task_data *p; + int pid, err; + + err = bpf_core_read(&pid, sizeof(pid), &task->pid); + if (err) + return -1; + + p = bpf_map_lookup_elem(&task_data, &pid); + if (p == NULL && !task_map_full) { + struct contention_task_data data = {}; + + BPF_CORE_READ_STR_INTO(&data.comm, task, comm); + if (bpf_map_update_elem(&task_data, &pid, &data, BPF_NOEXIST) == -E2BIG) + task_map_full = 1; + } + + return 0; +} + +#ifndef __has_builtin +# define __has_builtin(x) 0 +#endif + +static inline struct task_struct *get_lock_owner(__u64 lock, __u32 flags) +{ + struct task_struct *task; + __u64 owner = 0; + + if (flags & LCB_F_MUTEX) { + struct mutex *mutex = (void *)lock; + owner = BPF_CORE_READ(mutex, owner.counter); + } else if (flags == LCB_F_READ || flags == LCB_F_WRITE) { + /* + * Support for the BPF_TYPE_MATCHES argument to the + * __builtin_preserve_type_info builtin was added at some point during + * development of clang 15 and it's what is needed for + * bpf_core_type_matches. + */ +#if __has_builtin(__builtin_preserve_type_info) && __clang_major__ >= 15 + if (bpf_core_type_matches(struct rw_semaphore___old)) { + struct rw_semaphore___old *rwsem = (void *)lock; + owner = (unsigned long)BPF_CORE_READ(rwsem, owner); + } else if (bpf_core_type_matches(struct rw_semaphore___new)) { + struct rw_semaphore___new *rwsem = (void *)lock; + owner = BPF_CORE_READ(rwsem, owner.counter); + } +#else + /* assume new struct */ + struct rw_semaphore *rwsem = (void *)lock; + owner = BPF_CORE_READ(rwsem, owner.counter); +#endif + } + + if (!owner) + return NULL; + + task = (void *)(owner & ~7UL); + return task; +} + +static inline __u32 check_lock_type(__u64 lock, __u32 flags) +{ + struct task_struct *curr; + struct mm_struct___old *mm_old; + struct mm_struct___new *mm_new; + struct sighand_struct *sighand; + + switch (flags) { + case LCB_F_READ: /* rwsem */ + case LCB_F_WRITE: + curr = bpf_get_current_task_btf(); + if (curr->mm == NULL) + break; + mm_new = (void *)curr->mm; + if (bpf_core_field_exists(mm_new->mmap_lock)) { + if (&mm_new->mmap_lock == (void *)lock) + return LCD_F_MMAP_LOCK; + break; + } + mm_old = (void *)curr->mm; + if (bpf_core_field_exists(mm_old->mmap_sem)) { + if (&mm_old->mmap_sem == (void *)lock) + return LCD_F_MMAP_LOCK; + } + break; + case LCB_F_SPIN: /* spinlock */ + curr = bpf_get_current_task_btf(); + sighand = curr->sighand; + + if (sighand && &sighand->siglock == (void *)lock) + return LCD_F_SIGHAND_LOCK; + break; + default: + break; + } + return 0; +} + +static inline long delay_callback(__u64 idx, void *arg) +{ + __u64 target = *(__u64 *)arg; + + if (target <= bpf_ktime_get_ns()) + return 1; + + /* just to kill time */ + (void)bpf_get_prandom_u32(); + + return 0; +} + +static inline void do_lock_delay(__u64 duration) +{ + __u64 target = bpf_ktime_get_ns() + duration; + + bpf_loop(MAX_LOOP, delay_callback, &target, /*flags=*/0); +} + +static inline void check_lock_delay(__u64 lock) +{ + __u64 *delay; + + delay = bpf_map_lookup_elem(&lock_delays, &lock); + if (delay) + do_lock_delay(*delay); +} + +static inline struct tstamp_data *get_tstamp_elem(__u32 flags) +{ + __u32 pid; + struct tstamp_data *pelem; + + /* Use per-cpu array map for spinlock and rwlock */ + if ((flags & (LCB_F_SPIN | LCB_F_MUTEX)) == LCB_F_SPIN) { + __u32 idx = 0; + + pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); + /* Do not update the element for nested locks */ + if (pelem && pelem->lock) + pelem = NULL; + return pelem; + } + + pid = bpf_get_current_pid_tgid(); + pelem = bpf_map_lookup_elem(&tstamp, &pid); + /* Do not update the element for nested locks */ + if (pelem && pelem->lock) + return NULL; + + if (pelem == NULL) { + struct tstamp_data zero = {}; + + if (bpf_map_update_elem(&tstamp, &pid, &zero, BPF_NOEXIST) < 0) { + __sync_fetch_and_add(&task_fail, 1); + return NULL; + } + + pelem = bpf_map_lookup_elem(&tstamp, &pid); + if (pelem == NULL) { + __sync_fetch_and_add(&task_fail, 1); + return NULL; + } + } + return pelem; +} + +static inline s32 get_owner_stack_id(u64 *stacktrace) +{ + s32 *id, new_id; + static s64 id_gen = 1; + + id = bpf_map_lookup_elem(&owner_stacks, stacktrace); + if (id) + return *id; + + new_id = (s32)__sync_fetch_and_add(&id_gen, 1); + + bpf_map_update_elem(&owner_stacks, stacktrace, &new_id, BPF_NOEXIST); + + id = bpf_map_lookup_elem(&owner_stacks, stacktrace); + if (id) + return *id; + + return -1; +} + +static long cas_min_max_cb(u64 idx, void *arg) +{ + struct cas_ctx *ctx = arg; + + if (!ctx->max_done) { + u64 old_max = ctx->data->max_time; + if (old_max >= ctx->duration) { + ctx->max_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->max_time, old_max, ctx->duration); + if (r == old_max) + ctx->max_done = 1; + } + } + + if (!ctx->min_done) { + u64 old_min = ctx->data->min_time; + if (old_min <= ctx->duration) { + ctx->min_done = 1; + } else { + u64 r = __sync_val_compare_and_swap( + &ctx->data->min_time, old_min, ctx->duration); + if (r == old_min) + ctx->min_done = 1; + } + } + + return (ctx->max_done && ctx->min_done) ? 1 : 0; +} + +static inline void update_contention_data(struct contention_data *data, u64 duration, u32 count) +{ + __sync_fetch_and_add(&data->total_time, duration); + __sync_fetch_and_add(&data->count, count); + + struct cas_ctx ctx = { + .data = data, + .duration = duration, + .max_done = 0, + .min_done = 0, + }; + bpf_loop(64, cas_min_max_cb, &ctx, 0); +} + +static inline void update_owner_stat(u32 id, u64 duration, u32 flags) +{ + struct contention_key key = { + .stack_id = id, + .pid = 0, + .lock_addr_or_cgroup = 0, + }; + struct contention_data *data = bpf_map_lookup_elem(&owner_stat, &key); + + if (!data) { + struct contention_data first = { + .total_time = duration, + .max_time = duration, + .min_time = duration, + .count = 1, + .flags = flags, + }; + bpf_map_update_elem(&owner_stat, &key, &first, BPF_NOEXIST); + } else { + update_contention_data(data, duration, 1); + } +} + +SEC("tp_btf/contention_begin") +int contention_begin(u64 *ctx) +{ + struct tstamp_data *pelem; + + if (!enabled || !can_record(ctx)) + return 0; + + pelem = get_tstamp_elem(ctx[1]); + if (pelem == NULL) + return 0; + + pelem->timestamp = bpf_ktime_get_ns(); + pelem->lock = (__u64)ctx[0]; + pelem->flags = (__u32)ctx[1]; + if (aggr_mode == LOCK_AGGR_CGROUP) + pelem->cgroup_id = get_current_cgroup_id(); + + if (needs_callstack) { + u32 i = 0; + u32 id = 0; + int owner_pid; + u64 *buf; + struct task_struct *task; + struct owner_tracing_data *otdata; + + if (!lock_owner) + goto skip_owner; + + task = get_lock_owner(pelem->lock, pelem->flags); + if (!task) + goto skip_owner; + + owner_pid = BPF_CORE_READ(task, pid); + + buf = bpf_map_lookup_elem(&stack_buf, &i); + if (!buf) + goto skip_owner; + for (i = 0; i < max_stack; i++) + buf[i] = 0x0; + + if (!bpf_task_from_pid) + goto skip_owner; + + task = bpf_task_from_pid(owner_pid); + if (!task) + goto skip_owner; + + bpf_get_task_stack(task, buf, max_stack * sizeof(unsigned long), 0); + bpf_task_release(task); + + otdata = bpf_map_lookup_elem(&owner_data, &pelem->lock); + id = get_owner_stack_id(buf); + + /* + * Contention just happens, or corner case `lock` is owned by process not + * `owner_pid`. For the corner case we treat it as unexpected internal error and + * just ignore the precvious tracing record. + */ + if (!otdata || otdata->pid != owner_pid) { + struct owner_tracing_data first = { + .pid = owner_pid, + .timestamp = pelem->timestamp, + .count = 1, + .stack_id = id, + }; + bpf_map_update_elem(&owner_data, &pelem->lock, &first, BPF_ANY); + } + /* Contention is ongoing and new waiter joins */ + else { + __sync_fetch_and_add(&otdata->count, 1); + + /* + * The owner is the same, but stacktrace might be changed. In this case we + * store/update `owner_stat` based on current owner stack id. + */ + if (id != otdata->stack_id) { + update_owner_stat(id, pelem->timestamp - otdata->timestamp, + pelem->flags); + + otdata->timestamp = pelem->timestamp; + otdata->stack_id = id; + } + } +skip_owner: + pelem->stack_id = bpf_get_stackid(ctx, &stacks, + BPF_F_FAST_STACK_CMP | stack_skip); + if (pelem->stack_id < 0) + __sync_fetch_and_add(&stack_fail, 1); + } else if (aggr_mode == LOCK_AGGR_TASK) { + struct task_struct *task; + + if (lock_owner) { + task = get_lock_owner(pelem->lock, pelem->flags); + + /* The flags is not used anymore. Pass the owner pid. */ + if (task) + pelem->flags = BPF_CORE_READ(task, pid); + else + pelem->flags = -1U; + + } else { + task = bpf_get_current_task_btf(); + } + + if (task) { + if (update_task_data(task) < 0 && lock_owner) + pelem->flags = -1U; + } + } + + return 0; +} + +SEC("tp_btf/contention_end") +int contention_end(u64 *ctx) +{ + __u32 pid = 0, idx = 0; + struct tstamp_data *pelem; + struct contention_key key = {}; + struct contention_data *data; + __u64 timestamp; + __u64 duration; + bool need_delete = false; + + if (!enabled) + return 0; + + /* + * For spinlock and rwlock, it needs to get the timestamp for the + * per-cpu map. However, contention_end does not have the flags + * so it cannot know whether it reads percpu or hash map. + * + * Try per-cpu map first and check if there's active contention. + * If it is, do not read hash map because it cannot go to sleeping + * locks before releasing the spinning locks. + */ + pelem = bpf_map_lookup_elem(&tstamp_cpu, &idx); + if (pelem && pelem->lock) { + if (pelem->lock != ctx[0]) + return 0; + } else { + pid = bpf_get_current_pid_tgid(); + pelem = bpf_map_lookup_elem(&tstamp, &pid); + if (!pelem || pelem->lock != ctx[0]) + return 0; + need_delete = true; + } + + timestamp = bpf_ktime_get_ns(); + duration = timestamp - pelem->timestamp; + if ((__s64)duration < 0) { + __sync_fetch_and_add(&time_fail, 1); + goto out; + } + + if (needs_callstack && lock_owner) { + struct owner_tracing_data *otdata = bpf_map_lookup_elem(&owner_data, &pelem->lock); + + if (!otdata) + goto skip_owner; + + /* Update `owner_stat` */ + update_owner_stat(otdata->stack_id, timestamp - otdata->timestamp, pelem->flags); + + /* No contention is occurring, delete `lock` entry in `owner_data` */ + if (otdata->count <= 1) + bpf_map_delete_elem(&owner_data, &pelem->lock); + /* + * Contention is still ongoing, with a new owner (current task). `owner_data` + * should be updated accordingly. + */ + else { + u32 i = 0; + s32 ret = (s32)ctx[1]; + u64 *buf; + + otdata->timestamp = timestamp; + __sync_fetch_and_add(&otdata->count, -1); + + buf = bpf_map_lookup_elem(&stack_buf, &i); + if (!buf) + goto skip_owner; + for (i = 0; i < (u32)max_stack; i++) + buf[i] = 0x0; + + /* + * `ret` has the return code of the lock function. + * If `ret` is negative, the current task terminates lock waiting without + * acquiring it. Owner is not changed, but we still need to update the owner + * stack. + */ + if (ret < 0) { + s32 id = 0; + struct task_struct *task; + + if (!bpf_task_from_pid) + goto skip_owner; + + task = bpf_task_from_pid(otdata->pid); + if (!task) + goto skip_owner; + + bpf_get_task_stack(task, buf, + max_stack * sizeof(unsigned long), 0); + bpf_task_release(task); + + id = get_owner_stack_id(buf); + + /* + * If owner stack is changed, update owner stack id for this lock. + */ + if (id != otdata->stack_id) + otdata->stack_id = id; + } + /* + * Otherwise, update tracing data with the current task, which is the new + * owner. + */ + else { + otdata->pid = pid; + /* + * We don't want to retrieve callstack here, since it is where the + * current task acquires the lock and provides no additional + * information. We simply assign -1 to invalidate it. + */ + otdata->stack_id = -1; + } + } + } +skip_owner: + switch (aggr_mode) { + case LOCK_AGGR_CALLER: + key.stack_id = pelem->stack_id; + break; + case LOCK_AGGR_TASK: + if (lock_owner) + key.pid = pelem->flags; + else { + if (!need_delete) + pid = bpf_get_current_pid_tgid(); + key.pid = pid; + } + if (needs_callstack) + key.stack_id = pelem->stack_id; + break; + case LOCK_AGGR_ADDR: + key.lock_addr_or_cgroup = pelem->lock; + if (needs_callstack) + key.stack_id = pelem->stack_id; + break; + case LOCK_AGGR_CGROUP: + key.lock_addr_or_cgroup = pelem->cgroup_id; + break; + default: + /* should not happen */ + return 0; + } + + data = bpf_map_lookup_elem(&lock_stat, &key); + if (!data) { + if (data_map_full) { + __sync_fetch_and_add(&data_fail, 1); + goto out; + } + + struct contention_data first = { + .total_time = duration, + .max_time = duration, + .min_time = duration, + .count = 1, + .flags = pelem->flags, + }; + int err; + + if (aggr_mode == LOCK_AGGR_ADDR) { + first.flags |= check_lock_type(pelem->lock, + pelem->flags & LCB_F_TYPE_MASK); + + /* Check if it's from a slab object */ + if (bpf_get_kmem_cache) { + struct kmem_cache *s; + struct slab_cache_data *d; + + s = bpf_get_kmem_cache(pelem->lock); + if (s != NULL) { + /* + * Save the ID of the slab cache in the flags + * (instead of full address) to reduce the + * space in the contention_data. + */ + d = bpf_map_lookup_elem(&slab_caches, &s); + if (d != NULL) + first.flags |= d->id; + } + } + } + + err = bpf_map_update_elem(&lock_stat, &key, &first, BPF_NOEXIST); + if (err < 0) { + if (err == -EEXIST) { + /* it lost the race, try to get it again */ + data = bpf_map_lookup_elem(&lock_stat, &key); + if (data != NULL) + goto found; + } + if (err == -E2BIG) + data_map_full = 1; + __sync_fetch_and_add(&data_fail, 1); + } + goto out; + } + +found: + update_contention_data(data, duration, 1); + +out: + if (lock_delay) + check_lock_delay(pelem->lock); + + pelem->lock = 0; + if (need_delete) + bpf_map_delete_elem(&tstamp, &pid); + return 0; +} + +extern struct rq runqueues __ksym; + +const volatile __u64 contig_page_data_addr; +const volatile __u64 node_data_addr; +const volatile int nr_nodes; +const volatile int sizeof_zone; + +struct rq___old { + raw_spinlock_t lock; +} __attribute__((preserve_access_index)); + +struct rq___new { + raw_spinlock_t __lock; +} __attribute__((preserve_access_index)); + +static void collect_zone_lock(void) +{ + __u64 nr_zones, zone_off; + __u64 lock_addr, lock_off; + __u32 lock_flag = LOCK_CLASS_ZONE_LOCK; + + zone_off = offsetof(struct pglist_data, node_zones); + lock_off = offsetof(struct zone, lock); + + if (contig_page_data_addr) { + struct pglist_data *contig_page_data; + + contig_page_data = (void *)(long)contig_page_data_addr; + nr_zones = BPF_CORE_READ(contig_page_data, nr_zones); + + for (int i = 0; i < MAX_ZONES; i++) { + __u64 zone_addr; + + if (i >= nr_zones) + break; + + zone_addr = contig_page_data_addr + (sizeof_zone * i) + zone_off; + lock_addr = zone_addr + lock_off; + + bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); + } + } else if (nr_nodes > 0) { + struct pglist_data **node_data = (void *)(long)node_data_addr; + + for (int i = 0; i < nr_nodes; i++) { + struct pglist_data *pgdat = NULL; + int err; + + err = bpf_core_read(&pgdat, sizeof(pgdat), &node_data[i]); + if (err < 0 || pgdat == NULL) + break; + + nr_zones = BPF_CORE_READ(pgdat, nr_zones); + for (int k = 0; k < MAX_ZONES; k++) { + __u64 zone_addr; + + if (k >= nr_zones) + break; + + zone_addr = (__u64)(void *)pgdat + (sizeof_zone * k) + zone_off; + lock_addr = zone_addr + lock_off; + + bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); + } + } + } +} + +SEC("raw_tp/bpf_test_finish") +int BPF_PROG(collect_lock_syms) +{ + __u64 lock_addr, lock_off; + __u32 lock_flag; + + if (bpf_core_field_exists(struct rq___new, __lock)) + lock_off = offsetof(struct rq___new, __lock); + else + lock_off = offsetof(struct rq___old, lock); + + for (int i = 0; i < MAX_CPUS; i++) { + struct rq *rq = bpf_per_cpu_ptr(&runqueues, i); + + if (rq == NULL) + break; + + lock_addr = (__u64)(void *)rq + lock_off; + lock_flag = LOCK_CLASS_RQLOCK; + bpf_map_update_elem(&lock_syms, &lock_addr, &lock_flag, BPF_ANY); + } + + collect_zone_lock(); + + return 0; +} + +SEC("raw_tp/bpf_test_finish") +int BPF_PROG(end_timestamp) +{ + end_ts = bpf_ktime_get_ns(); + return 0; +} + +/* + * bpf_iter__kmem_cache added recently so old kernels don't have it in the + * vmlinux.h. But we cannot add it here since it will cause a compiler error + * due to redefinition of the struct on later kernels. + * + * So it uses a CO-RE trick to access the member only if it has the type. + * This will support both old and new kernels without compiler errors. + */ +struct bpf_iter__kmem_cache___new { + struct kmem_cache *s; +} __attribute__((preserve_access_index)); + +SEC("iter/kmem_cache") +int slab_cache_iter(void *ctx) +{ + struct kmem_cache *s = NULL; + struct slab_cache_data d; + const char *nameptr; + + if (bpf_core_type_exists(struct bpf_iter__kmem_cache)) { + struct bpf_iter__kmem_cache___new *iter = ctx; + + s = iter->s; + } + + if (s == NULL) + return 0; + + nameptr = s->name; + bpf_probe_read_kernel_str(d.name, sizeof(d.name), nameptr); + + d.id = ++slab_cache_id << LCB_F_SLAB_ID_SHIFT; + if (d.id >= LCB_F_SLAB_ID_END) + return 0; + + bpf_map_update_elem(&slab_caches, &s, &d, BPF_NOEXIST); + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/lock_data.h b/tools/perf/util/bpf_skel/lock_data.h new file mode 100644 index 000000000..652e114e6 --- /dev/null +++ b/tools/perf/util/bpf_skel/lock_data.h @@ -0,0 +1,79 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Data structures shared between BPF and tools. */ +#ifndef UTIL_BPF_SKEL_LOCK_DATA_H +#define UTIL_BPF_SKEL_LOCK_DATA_H + +struct owner_tracing_data { + u32 pid; // Who has the lock. + u32 count; // How many waiters for this lock. + u64 timestamp; // The time while the owner acquires lock and contention is going on. + s32 stack_id; // Identifier for `owner_stat`, which stores as value in `owner_stacks` +}; + +struct tstamp_data { + u64 timestamp; + u64 lock; + u64 cgroup_id; + u32 flags; + s32 stack_id; +}; + +struct contention_key { + s32 stack_id; + u32 pid; + u64 lock_addr_or_cgroup; +}; + +#define TASK_COMM_LEN 16 + +struct contention_task_data { + char comm[TASK_COMM_LEN]; +}; + +/* default buffer size */ +#define MAX_ENTRIES 16384 + +/* + * Upper bits of the flags in the contention_data are used to identify + * some well-known locks which do not have symbols (non-global locks). + */ +#define LCD_F_MMAP_LOCK (1U << 31) +#define LCD_F_SIGHAND_LOCK (1U << 30) + +#define LCB_F_SLAB_ID_SHIFT 16 +#define LCB_F_SLAB_ID_START (1U << 16) +#define LCB_F_SLAB_ID_END (1U << 26) +#define LCB_F_SLAB_ID_MASK 0x03FF0000U + +#define LCB_F_TYPE_MAX (1U << 7) +#define LCB_F_TYPE_MASK 0x0000007FU + +#define SLAB_NAME_MAX 28 + +struct contention_data { + u64 total_time; + u64 min_time; + u64 max_time; + u32 count; + u32 flags; +}; + +enum lock_aggr_mode { + LOCK_AGGR_ADDR = 0, + LOCK_AGGR_TASK, + LOCK_AGGR_CALLER, + LOCK_AGGR_CGROUP, +}; + +enum lock_class_sym { + LOCK_CLASS_NONE, + LOCK_CLASS_RQLOCK, + LOCK_CLASS_ZONE_LOCK, +}; + +struct slab_cache_data { + u32 id; + char name[SLAB_NAME_MAX]; +}; + +#endif /* UTIL_BPF_SKEL_LOCK_DATA_H */ diff --git a/tools/perf/util/bpf_skel/off_cpu.bpf.c b/tools/perf/util/bpf_skel/off_cpu.bpf.c new file mode 100644 index 000000000..72763bb8d --- /dev/null +++ b/tools/perf/util/bpf_skel/off_cpu.bpf.c @@ -0,0 +1,372 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2022 Google +#include "vmlinux.h" +#include +#include +#include + +/* task->flags for off-cpu analysis */ +#define PF_KTHREAD 0x00200000 /* I am a kernel thread */ + +/* task->state for off-cpu analysis */ +#define TASK_INTERRUPTIBLE 0x0001 +#define TASK_UNINTERRUPTIBLE 0x0002 + +/* create a new thread */ +#define CLONE_THREAD 0x10000 + +#define MAX_STACKS 32 +#define MAX_ENTRIES 102400 + +#define MAX_CPUS 4096 +#define MAX_OFFCPU_LEN 37 + +// We have a 'struct stack' in vmlinux.h when building with GEN_VMLINUX_H=1 +struct __stack { + u64 array[MAX_STACKS]; +}; + +struct tstamp_data { + __u32 stack_id; + __u32 state; + __u64 timestamp; + struct __stack stack; +}; + +struct offcpu_key { + __u32 pid; + __u32 tgid; + __u32 stack_id; + __u32 state; + __u64 cgroup_id; +}; + +struct { + __uint(type, BPF_MAP_TYPE_STACK_TRACE); + __uint(key_size, sizeof(__u32)); + __uint(value_size, MAX_STACKS * sizeof(__u64)); + __uint(max_entries, MAX_ENTRIES); +} stacks SEC(".maps"); + +struct offcpu_data { + u64 array[MAX_OFFCPU_LEN]; +}; + +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(int)); + __uint(value_size, sizeof(int)); + __uint(max_entries, MAX_CPUS); +} offcpu_output SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct offcpu_data)); + __uint(max_entries, 1); +} offcpu_payload SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_TASK_STORAGE); + __uint(map_flags, BPF_F_NO_PREALLOC); + __type(key, int); + __type(value, struct tstamp_data); +} tstamp SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(struct offcpu_key)); + __uint(value_size, sizeof(__u64)); + __uint(max_entries, MAX_ENTRIES); +} off_cpu SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cpu_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} task_filter SEC(".maps"); + +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u8)); + __uint(max_entries, 1); +} cgroup_filter SEC(".maps"); + +/* new kernel task_struct definition */ +struct task_struct___new { + long __state; +} __attribute__((preserve_access_index)); + +/* old kernel task_struct definition */ +struct task_struct___old { + long state; +} __attribute__((preserve_access_index)); + +int enabled = 0; + +const volatile int has_cpu = 0; +const volatile int has_task = 0; +const volatile int has_cgroup = 0; +const volatile int uses_tgid = 0; + +const volatile bool has_prev_state = false; +const volatile bool needs_cgroup = false; +const volatile bool uses_cgroup_v1 = false; + +int perf_subsys_id = -1; + +__u64 offcpu_thresh_ns; + +/* + * Old kernel used to call it task_struct->state and now it's '__state'. + * Use BPF CO-RE "ignored suffix rule" to deal with it like below: + * + * https://nakryiko.com/posts/bpf-core-reference-guide/#handling-incompatible-field-and-type-changes + */ +static inline int get_task_state(struct task_struct *t) +{ + /* recast pointer to capture new type for compiler */ + struct task_struct___new *t_new = (void *)t; + + if (bpf_core_field_exists(t_new->__state)) { + return BPF_CORE_READ(t_new, __state); + } else { + /* recast pointer to capture old type for compiler */ + struct task_struct___old *t_old = (void *)t; + + return BPF_CORE_READ(t_old, state); + } +} + +static inline __u64 get_cgroup_id(struct task_struct *t) +{ + struct cgroup *cgrp; + + if (!uses_cgroup_v1) + return BPF_CORE_READ(t, cgroups, dfl_cgrp, kn, id); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(t, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + +static inline int can_record(struct task_struct *t, int state) +{ + /* kernel threads don't have user stack */ + if (t->flags & PF_KTHREAD) + return 0; + + if (state != TASK_INTERRUPTIBLE && + state != TASK_UNINTERRUPTIBLE) + return 0; + + if (has_cpu) { + __u32 cpu = bpf_get_smp_processor_id(); + __u8 *ok; + + ok = bpf_map_lookup_elem(&cpu_filter, &cpu); + if (!ok) + return 0; + } + + if (has_task) { + __u8 *ok; + __u32 pid; + + if (uses_tgid) + pid = t->tgid; + else + pid = t->pid; + + ok = bpf_map_lookup_elem(&task_filter, &pid); + if (!ok) + return 0; + } + + if (has_cgroup) { + __u8 *ok; + __u64 cgrp_id = get_cgroup_id(t); + + ok = bpf_map_lookup_elem(&cgroup_filter, &cgrp_id); + if (!ok) + return 0; + } + + return 1; +} + +static inline int copy_stack(struct __stack *from, struct offcpu_data *to, int n) +{ + int len = 0; + + for (int i = 0; i < MAX_STACKS && from->array[i]; ++i, ++len) + to->array[n + 2 + i] = from->array[i]; + + return len; +} + +/** + * off_cpu_dump - dump off-cpu samples to ring buffer + * @data: payload for dumping off-cpu samples + * @key: off-cpu data + * @stack: stack trace of the task before being scheduled out + * + * If the threshold of off-cpu time is reached, acquire tid, period, callchain, and cgroup id + * information of the task, and dump it as a raw sample to perf ring buffer + */ +static int off_cpu_dump(void *ctx, struct offcpu_data *data, struct offcpu_key *key, + struct __stack *stack, __u64 delta) +{ + int n = 0, len = 0; + + data->array[n++] = (u64)key->tgid << 32 | key->pid; + data->array[n++] = delta; + + /* data->array[n] is callchain->nr (updated later) */ + data->array[n + 1] = PERF_CONTEXT_USER; + data->array[n + 2] = 0; + len = copy_stack(stack, data, n); + + /* update length of callchain */ + data->array[n] = len + 1; + n += len + 2; + + data->array[n++] = key->cgroup_id; + + return bpf_perf_event_output(ctx, &offcpu_output, BPF_F_CURRENT_CPU, data, n * sizeof(u64)); +} + +static int off_cpu_stat(u64 *ctx, struct task_struct *prev, + struct task_struct *next, int state) +{ + __u64 ts; + __u32 stack_id; + struct tstamp_data *pelem; + + ts = bpf_ktime_get_ns(); + + if (!can_record(prev, state)) + goto next; + + stack_id = bpf_get_stackid(ctx, &stacks, + BPF_F_FAST_STACK_CMP | BPF_F_USER_STACK); + + pelem = bpf_task_storage_get(&tstamp, prev, NULL, + BPF_LOCAL_STORAGE_GET_F_CREATE); + if (!pelem) + goto next; + + pelem->timestamp = ts; + pelem->state = state; + pelem->stack_id = stack_id; + + /* + * If stacks are successfully collected by bpf_get_stackid(), collect them once more + * in task_storage for direct off-cpu sample dumping + */ + if (stack_id > 0 && bpf_get_stack(ctx, &pelem->stack, MAX_STACKS * sizeof(u64), BPF_F_USER_STACK)) { + /* + * This empty if block is used to avoid 'result unused warning' from bpf_get_stack(). + * If the collection fails, continue with the logic for the next task. + */ + } +next: + pelem = bpf_task_storage_get(&tstamp, next, NULL, 0); + + if (pelem && pelem->timestamp) { + struct offcpu_key key = { + .pid = next->pid, + .tgid = next->tgid, + .stack_id = pelem->stack_id, + .state = pelem->state, + .cgroup_id = needs_cgroup ? get_cgroup_id(next) : 0, + }; + __u64 delta = ts - pelem->timestamp; + __u64 *total; + + if (delta >= offcpu_thresh_ns) { + int zero = 0; + struct offcpu_data *data = bpf_map_lookup_elem(&offcpu_payload, &zero); + + if (data) + off_cpu_dump(ctx, data, &key, &pelem->stack, delta); + } else { + total = bpf_map_lookup_elem(&off_cpu, &key); + if (total) + *total += delta; + else + bpf_map_update_elem(&off_cpu, &key, &delta, BPF_ANY); + } + + /* prevent to reuse the timestamp later */ + pelem->timestamp = 0; + } + + return 0; +} + +SEC("tp_btf/task_newtask") +int on_newtask(u64 *ctx) +{ + struct task_struct *task; + u64 clone_flags; + u32 pid; + u8 val = 1; + + if (!uses_tgid) + return 0; + + task = (struct task_struct *)bpf_get_current_task(); + + pid = BPF_CORE_READ(task, tgid); + if (!bpf_map_lookup_elem(&task_filter, &pid)) + return 0; + + task = (struct task_struct *)ctx[0]; + clone_flags = ctx[1]; + + pid = task->tgid; + if (!(clone_flags & CLONE_THREAD)) + bpf_map_update_elem(&task_filter, &pid, &val, BPF_NOEXIST); + + return 0; +} + +SEC("tp_btf/sched_switch") +int on_switch(u64 *ctx) +{ + struct task_struct *prev, *next; + int prev_state; + + if (!enabled) + return 0; + + prev = (struct task_struct *)ctx[1]; + next = (struct task_struct *)ctx[2]; + + if (has_prev_state) + prev_state = (int)ctx[3]; + else + prev_state = get_task_state(prev); + + return off_cpu_stat(ctx, prev, next, prev_state & 0xff); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/perf_version.h b/tools/perf/util/bpf_skel/perf_version.h new file mode 100644 index 000000000..1ed5b2e59 --- /dev/null +++ b/tools/perf/util/bpf_skel/perf_version.h @@ -0,0 +1,17 @@ +/* SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) */ + +#ifndef __PERF_VERSION_H__ +#define __PERF_VERSION_H__ + +#include "vmlinux.h" +#include + +/* + * This is used by tests/shell/record_bpf_metadata.sh + * to verify that BPF metadata generation works. + * + * PERF_VERSION is defined by a build rule at compile time. + */ +const char bpf_metadata_perf_version[] SEC(".rodata") = PERF_VERSION; + +#endif /* __PERF_VERSION_H__ */ diff --git a/tools/perf/util/bpf_skel/sample-filter.h b/tools/perf/util/bpf_skel/sample-filter.h new file mode 100644 index 000000000..683fec85e --- /dev/null +++ b/tools/perf/util/bpf_skel/sample-filter.h @@ -0,0 +1,72 @@ +#ifndef PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H +#define PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H + +#define MAX_FILTERS 64 +#define MAX_IDX_HASH (16 * 1024) +#define MAX_EVT_HASH (1024 * 1024) + +/* supported filter operations */ +enum perf_bpf_filter_op { + PBF_OP_EQ, + PBF_OP_NEQ, + PBF_OP_GT, + PBF_OP_GE, + PBF_OP_LT, + PBF_OP_LE, + PBF_OP_AND, + PBF_OP_GROUP_BEGIN, + PBF_OP_GROUP_END, + PBF_OP_DONE, +}; + +enum perf_bpf_filter_term { + /* No term is in use. */ + PBF_TERM_NONE = 0, + /* Terms that correspond to PERF_SAMPLE_xx values. */ + PBF_TERM_SAMPLE_START = PBF_TERM_NONE + 1, + PBF_TERM_IP = PBF_TERM_SAMPLE_START + 0, /* SAMPLE_IP = 1U << 0 */ + PBF_TERM_TID = PBF_TERM_SAMPLE_START + 1, /* SAMPLE_TID = 1U << 1 */ + PBF_TERM_TIME = PBF_TERM_SAMPLE_START + 2, /* SAMPLE_TIME = 1U << 2 */ + PBF_TERM_ADDR = PBF_TERM_SAMPLE_START + 3, /* SAMPLE_ADDR = 1U << 3 */ + __PBF_UNUSED_TERM4 = PBF_TERM_SAMPLE_START + 4, /* SAMPLE_READ = 1U << 4 */ + __PBF_UNUSED_TERM5 = PBF_TERM_SAMPLE_START + 5, /* SAMPLE_CALLCHAIN = 1U << 5 */ + PBF_TERM_ID = PBF_TERM_SAMPLE_START + 6, /* SAMPLE_ID = 1U << 6 */ + PBF_TERM_CPU = PBF_TERM_SAMPLE_START + 7, /* SAMPLE_CPU = 1U << 7 */ + PBF_TERM_PERIOD = PBF_TERM_SAMPLE_START + 8, /* SAMPLE_PERIOD = 1U << 8 */ + __PBF_UNUSED_TERM9 = PBF_TERM_SAMPLE_START + 9, /* SAMPLE_STREAM_ID = 1U << 9 */ + __PBF_UNUSED_TERM10 = PBF_TERM_SAMPLE_START + 10, /* SAMPLE_RAW = 1U << 10 */ + __PBF_UNUSED_TERM11 = PBF_TERM_SAMPLE_START + 11, /* SAMPLE_BRANCH_STACK = 1U << 11 */ + __PBF_UNUSED_TERM12 = PBF_TERM_SAMPLE_START + 12, /* SAMPLE_REGS_USER = 1U << 12 */ + __PBF_UNUSED_TERM13 = PBF_TERM_SAMPLE_START + 13, /* SAMPLE_STACK_USER = 1U << 13 */ + PBF_TERM_WEIGHT = PBF_TERM_SAMPLE_START + 14, /* SAMPLE_WEIGHT = 1U << 14 */ + PBF_TERM_DATA_SRC = PBF_TERM_SAMPLE_START + 15, /* SAMPLE_DATA_SRC = 1U << 15 */ + __PBF_UNUSED_TERM16 = PBF_TERM_SAMPLE_START + 16, /* SAMPLE_IDENTIFIER = 1U << 16 */ + PBF_TERM_TRANSACTION = PBF_TERM_SAMPLE_START + 17, /* SAMPLE_TRANSACTION = 1U << 17 */ + __PBF_UNUSED_TERM18 = PBF_TERM_SAMPLE_START + 18, /* SAMPLE_REGS_INTR = 1U << 18 */ + PBF_TERM_PHYS_ADDR = PBF_TERM_SAMPLE_START + 19, /* SAMPLE_PHYS_ADDR = 1U << 19 */ + __PBF_UNUSED_TERM20 = PBF_TERM_SAMPLE_START + 20, /* SAMPLE_AUX = 1U << 20 */ + PBF_TERM_CGROUP = PBF_TERM_SAMPLE_START + 21, /* SAMPLE_CGROUP = 1U << 21 */ + PBF_TERM_DATA_PAGE_SIZE = PBF_TERM_SAMPLE_START + 22, /* SAMPLE_DATA_PAGE_SIZE = 1U << 22 */ + PBF_TERM_CODE_PAGE_SIZE = PBF_TERM_SAMPLE_START + 23, /* SAMPLE_CODE_PAGE_SIZE = 1U << 23 */ + PBF_TERM_WEIGHT_STRUCT = PBF_TERM_SAMPLE_START + 24, /* SAMPLE_WEIGHT_STRUCT = 1U << 24 */ + PBF_TERM_SAMPLE_END = PBF_TERM_WEIGHT_STRUCT, + /* Terms computed from BPF helpers. */ + PBF_TERM_UID, + PBF_TERM_GID, +}; + +/* BPF map entry for filtering */ +struct perf_bpf_filter_entry { + enum perf_bpf_filter_op op; + __u32 part; /* sub-sample type info when it has multiple values */ + enum perf_bpf_filter_term term; + __u64 value; +}; + +struct idx_hash_key { + __u64 evt_id; + __u32 tgid; + __u32 reserved; +}; + +#endif /* PERF_UTIL_BPF_SKEL_SAMPLE_FILTER_H */ diff --git a/tools/perf/util/bpf_skel/sample_filter.bpf.c b/tools/perf/util/bpf_skel/sample_filter.bpf.c new file mode 100644 index 000000000..e5666d4c1 --- /dev/null +++ b/tools/perf/util/bpf_skel/sample_filter.bpf.c @@ -0,0 +1,298 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2023 Google +#include "vmlinux.h" +#include +#include +#include + +#include "sample-filter.h" + +/* BPF map that will be filled by user space */ +struct filters { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); + __type(value, struct perf_bpf_filter_entry[MAX_FILTERS]); + __uint(max_entries, 1); +} filters SEC(".maps"); + +/* + * An evsel has multiple instances for each CPU or task but we need a single + * id to be used as a key for the idx_hash. This hashmap would translate the + * instance's ID to a representative ID. + */ +struct event_hash { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, __u64); + __type(value, __u64); + __uint(max_entries, 1); +} event_hash SEC(".maps"); + +/* tgid/evtid to filter index */ +struct idx_hash { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct idx_hash_key); + __type(value, int); + __uint(max_entries, 1); +} idx_hash SEC(".maps"); + +/* tgid to filter index */ +struct lost_count { + __uint(type, BPF_MAP_TYPE_ARRAY); + __type(key, int); + __type(value, int); + __uint(max_entries, 1); +} dropped SEC(".maps"); + +volatile const int use_idx_hash; + +void *bpf_cast_to_kern_ctx(void *) __ksym; + +/* new kernel perf_sample_data definition */ +struct perf_sample_data___new { + __u64 sample_flags; +} __attribute__((preserve_access_index)); + +/* new kernel perf_mem_data_src definition */ +union perf_mem_data_src___new { + __u64 val; + struct { + __u64 mem_op:5, /* type of opcode */ + mem_lvl:14, /* memory hierarchy level */ + mem_snoop:5, /* snoop mode */ + mem_lock:2, /* lock instr */ + mem_dtlb:7, /* tlb access */ + mem_lvl_num:4, /* memory hierarchy level number */ + mem_remote:1, /* remote */ + mem_snoopx:2, /* snoop mode, ext */ + mem_blk:3, /* access blocked */ + mem_hops:3, /* hop level */ + mem_rsvd:18; + }; +}; + +/* helper function to return the given perf sample data */ +static inline __u64 perf_get_sample(struct bpf_perf_event_data_kern *kctx, + struct perf_bpf_filter_entry *entry) +{ + struct perf_sample_data___new *data = (void *)kctx->data; + + if (!bpf_core_field_exists(data->sample_flags)) + return 0; + +#define BUILD_CHECK_SAMPLE(x) \ + _Static_assert((1 << (PBF_TERM_##x - PBF_TERM_SAMPLE_START)) == PERF_SAMPLE_##x, \ + "Mismatched PBF term to sample bit " #x) + BUILD_CHECK_SAMPLE(IP); + BUILD_CHECK_SAMPLE(TID); + BUILD_CHECK_SAMPLE(TIME); + BUILD_CHECK_SAMPLE(ADDR); + BUILD_CHECK_SAMPLE(ID); + BUILD_CHECK_SAMPLE(CPU); + BUILD_CHECK_SAMPLE(PERIOD); + BUILD_CHECK_SAMPLE(WEIGHT); + BUILD_CHECK_SAMPLE(DATA_SRC); + BUILD_CHECK_SAMPLE(TRANSACTION); + BUILD_CHECK_SAMPLE(PHYS_ADDR); + BUILD_CHECK_SAMPLE(CGROUP); + BUILD_CHECK_SAMPLE(DATA_PAGE_SIZE); + BUILD_CHECK_SAMPLE(CODE_PAGE_SIZE); + BUILD_CHECK_SAMPLE(WEIGHT_STRUCT); +#undef BUILD_CHECK_SAMPLE + + /* For sample terms check the sample bit is set. */ + if (entry->term >= PBF_TERM_SAMPLE_START && entry->term <= PBF_TERM_SAMPLE_END && + (data->sample_flags & (1 << (entry->term - PBF_TERM_SAMPLE_START))) == 0) + return 0; + + switch (entry->term) { + case PBF_TERM_IP: + return kctx->data->ip; + case PBF_TERM_ID: + return kctx->data->id; + case PBF_TERM_TID: + if (entry->part) + return kctx->data->tid_entry.pid; + else + return kctx->data->tid_entry.tid; + case PBF_TERM_CPU: + return kctx->data->cpu_entry.cpu; + case PBF_TERM_TIME: + return kctx->data->time; + case PBF_TERM_ADDR: + return kctx->data->addr; + case PBF_TERM_PERIOD: + return kctx->data->period; + case PBF_TERM_TRANSACTION: + return kctx->data->txn; + case PBF_TERM_WEIGHT_STRUCT: + if (entry->part == 1) + return kctx->data->weight.var1_dw; + if (entry->part == 2) + return kctx->data->weight.var2_w; + if (entry->part == 3) + return kctx->data->weight.var3_w; + /* fall through */ + case PBF_TERM_WEIGHT: + return kctx->data->weight.full; + case PBF_TERM_PHYS_ADDR: + return kctx->data->phys_addr; + case PBF_TERM_CGROUP: + return kctx->data->cgroup; + case PBF_TERM_CODE_PAGE_SIZE: + return kctx->data->code_page_size; + case PBF_TERM_DATA_PAGE_SIZE: + return kctx->data->data_page_size; + case PBF_TERM_DATA_SRC: + if (entry->part == 1) + return kctx->data->data_src.mem_op; + if (entry->part == 2) + return kctx->data->data_src.mem_lvl_num; + if (entry->part == 3) { + __u32 snoop = kctx->data->data_src.mem_snoop; + __u32 snoopx = kctx->data->data_src.mem_snoopx; + + return (snoopx << 5) | snoop; + } + if (entry->part == 4) + return kctx->data->data_src.mem_remote; + if (entry->part == 5) + return kctx->data->data_src.mem_lock; + if (entry->part == 6) + return kctx->data->data_src.mem_dtlb; + if (entry->part == 7) + return kctx->data->data_src.mem_blk; + if (entry->part == 8) { + union perf_mem_data_src___new *data = (void *)&kctx->data->data_src; + + if (__builtin_preserve_field_info(data->mem_hops, BPF_FIELD_EXISTS)) + return data->mem_hops; + + return 0; + } + /* return the whole word */ + return kctx->data->data_src.val; + case PBF_TERM_UID: + return bpf_get_current_uid_gid() & 0xFFFFFFFF; + case PBF_TERM_GID: + return bpf_get_current_uid_gid() >> 32; + case PBF_TERM_NONE: + case __PBF_UNUSED_TERM4: + case __PBF_UNUSED_TERM5: + case __PBF_UNUSED_TERM9: + case __PBF_UNUSED_TERM10: + case __PBF_UNUSED_TERM11: + case __PBF_UNUSED_TERM12: + case __PBF_UNUSED_TERM13: + case __PBF_UNUSED_TERM16: + case __PBF_UNUSED_TERM18: + case __PBF_UNUSED_TERM20: + default: + break; + } + return 0; +} + +#define CHECK_RESULT(data, op, val) \ + if (!(data op val)) { \ + if (!in_group) \ + goto drop; \ + } else if (in_group) { \ + group_result = 1; \ + } + +/* BPF program to be called from perf event overflow handler */ +SEC("perf_event") +int perf_sample_filter(void *ctx) +{ + struct bpf_perf_event_data_kern *kctx; + struct perf_bpf_filter_entry *entry; + __u64 sample_data; + int in_group = 0; + int group_result = 0; + int i, k; + int *losts; + + kctx = bpf_cast_to_kern_ctx(ctx); + + k = 0; + + if (use_idx_hash) { + struct idx_hash_key key = { + .tgid = bpf_get_current_pid_tgid() >> 32, + }; + __u64 eid = kctx->event->id; + __u64 *key_id; + int *idx; + + /* get primary_event_id */ + if (kctx->event->parent) + eid = kctx->event->parent->id; + + key_id = bpf_map_lookup_elem(&event_hash, &eid); + if (key_id == NULL) + goto drop; + + key.evt_id = *key_id; + + idx = bpf_map_lookup_elem(&idx_hash, &key); + if (idx) + k = *idx; + else + goto drop; + } + + entry = bpf_map_lookup_elem(&filters, &k); + if (entry == NULL) + goto drop; + + for (i = 0; i < MAX_FILTERS; i++) { + sample_data = perf_get_sample(kctx, &entry[i]); + + switch (entry[i].op) { + case PBF_OP_EQ: + CHECK_RESULT(sample_data, ==, entry[i].value) + break; + case PBF_OP_NEQ: + CHECK_RESULT(sample_data, !=, entry[i].value) + break; + case PBF_OP_GT: + CHECK_RESULT(sample_data, >, entry[i].value) + break; + case PBF_OP_GE: + CHECK_RESULT(sample_data, >=, entry[i].value) + break; + case PBF_OP_LT: + CHECK_RESULT(sample_data, <, entry[i].value) + break; + case PBF_OP_LE: + CHECK_RESULT(sample_data, <=, entry[i].value) + break; + case PBF_OP_AND: + CHECK_RESULT(sample_data, &, entry[i].value) + break; + case PBF_OP_GROUP_BEGIN: + in_group = 1; + group_result = 0; + break; + case PBF_OP_GROUP_END: + if (group_result == 0) + goto drop; + in_group = 0; + break; + case PBF_OP_DONE: + /* no failures so far, accept it */ + return 1; + } + } + /* generate sample data */ + return 1; + +drop: + losts = bpf_map_lookup_elem(&dropped, &k); + if (losts != NULL) + __sync_fetch_and_add(losts, 1); + + return 0; +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/bpf_skel/syscall_summary.bpf.c b/tools/perf/util/bpf_skel/syscall_summary.bpf.c new file mode 100644 index 000000000..4172f3c9f --- /dev/null +++ b/tools/perf/util/bpf_skel/syscall_summary.bpf.c @@ -0,0 +1,165 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Trace raw_syscalls tracepoints to collect system call statistics. + */ + +#include "vmlinux.h" +#include "syscall_summary.h" + +#include +#include +#include + +/* This is to calculate a delta between sys-enter and sys-exit for each thread */ +struct syscall_trace { + int nr; /* syscall number is only available at sys-enter */ + int unused; + u64 timestamp; +}; + +#define MAX_ENTRIES (128 * 1024) + +struct syscall_trace_map { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, int); /* tid */ + __type(value, struct syscall_trace); + __uint(max_entries, MAX_ENTRIES); +} syscall_trace_map SEC(".maps"); + +struct syscall_stats_map { + __uint(type, BPF_MAP_TYPE_HASH); + __type(key, struct syscall_key); + __type(value, struct syscall_stats); + __uint(max_entries, MAX_ENTRIES); +} syscall_stats_map SEC(".maps"); + +int enabled; /* controlled from userspace */ + +const volatile enum syscall_aggr_mode aggr_mode; +const volatile int use_cgroup_v2; + +int perf_subsys_id = -1; + +static inline __u64 get_current_cgroup_id(void) +{ + struct task_struct *task; + struct cgroup *cgrp; + + if (use_cgroup_v2) + return bpf_get_current_cgroup_id(); + + task = bpf_get_current_task_btf(); + + if (perf_subsys_id == -1) { +#if __has_builtin(__builtin_preserve_enum_value) + perf_subsys_id = bpf_core_enum_value(enum cgroup_subsys_id, + perf_event_cgrp_id); +#else + perf_subsys_id = perf_event_cgrp_id; +#endif + } + + cgrp = BPF_CORE_READ(task, cgroups, subsys[perf_subsys_id], cgroup); + return BPF_CORE_READ(cgrp, kn, id); +} + +static void update_stats(int cpu_or_tid, u64 cgroup_id, int nr, s64 duration, + long ret) +{ + struct syscall_key key = { + .cpu_or_tid = cpu_or_tid, + .cgroup = cgroup_id, + .nr = nr, + }; + struct syscall_stats *stats; + + stats = bpf_map_lookup_elem(&syscall_stats_map, &key); + if (stats == NULL) { + struct syscall_stats zero = {}; + + bpf_map_update_elem(&syscall_stats_map, &key, &zero, BPF_NOEXIST); + stats = bpf_map_lookup_elem(&syscall_stats_map, &key); + if (stats == NULL) + return; + } + + __sync_fetch_and_add(&stats->count, 1); + if (ret < 0) + __sync_fetch_and_add(&stats->error, 1); + + if (duration > 0) { + __sync_fetch_and_add(&stats->total_time, duration); + __sync_fetch_and_add(&stats->squared_sum, duration * duration); + if (stats->max_time < duration) + stats->max_time = duration; + if (stats->min_time > duration || stats->min_time == 0) + stats->min_time = duration; + } + + return; +} + +SEC("tp_btf/sys_enter") +int sys_enter(u64 *ctx) +{ + int tid; + struct syscall_trace st; + + if (!enabled) + return 0; + + st.nr = ctx[1]; /* syscall number */ + st.unused = 0; + st.timestamp = bpf_ktime_get_ns(); + + tid = bpf_get_current_pid_tgid(); + bpf_map_update_elem(&syscall_trace_map, &tid, &st, BPF_ANY); + + return 0; +} + +static int do_exit(long ret) +{ + int tid; + int key = 0; + u64 cgroup = 0; + struct syscall_trace *st; + s64 delta; + + if (!enabled) + return 0; + + tid = bpf_get_current_pid_tgid(); + st = bpf_map_lookup_elem(&syscall_trace_map, &tid); + if (st == NULL) + return 0; + + if (aggr_mode == SYSCALL_AGGR_THREAD) + key = tid; + else if (aggr_mode == SYSCALL_AGGR_CGROUP) + cgroup = get_current_cgroup_id(); + else + key = bpf_get_smp_processor_id(); + + delta = bpf_ktime_get_ns() - st->timestamp; + update_stats(key, cgroup, st->nr, delta, ret); + + bpf_map_delete_elem(&syscall_trace_map, &tid); + return 0; +} + +SEC("tp_btf/sys_exit") +int sys_exit(u64 *ctx) +{ + long ret = ctx[1]; /* return value of the syscall */ + + return do_exit(ret); +} + +SEC("tp_btf/sched_process_exit") +int process_exit(u64 *ctx) +{ + return do_exit(0); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/perf/util/bpf_skel/syscall_summary.h b/tools/perf/util/bpf_skel/syscall_summary.h new file mode 100644 index 000000000..72ccccb45 --- /dev/null +++ b/tools/perf/util/bpf_skel/syscall_summary.h @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +/* Data structures shared between BPF and tools. */ +#ifndef UTIL_BPF_SKEL_SYSCALL_SUMMARY_H +#define UTIL_BPF_SKEL_SYSCALL_SUMMARY_H + +enum syscall_aggr_mode { + SYSCALL_AGGR_THREAD, + SYSCALL_AGGR_CPU, + SYSCALL_AGGR_CGROUP, +}; + +struct syscall_key { + u64 cgroup; + int cpu_or_tid; + int nr; +}; + +struct syscall_stats { + u64 total_time; + u64 squared_sum; + u64 max_time; + u64 min_time; + u32 count; + u32 error; +}; + +#endif /* UTIL_BPF_SKEL_SYSCALL_SUMMARY_H */ diff --git a/tools/perf/util/bpf_skel/vmlinux/.gitignore b/tools/perf/util/bpf_skel/vmlinux/.gitignore new file mode 100644 index 000000000..49502c041 --- /dev/null +++ b/tools/perf/util/bpf_skel/vmlinux/.gitignore @@ -0,0 +1 @@ +!vmlinux.h diff --git a/tools/perf/util/bpf_skel/vmlinux/vmlinux.h b/tools/perf/util/bpf_skel/vmlinux/vmlinux.h new file mode 100644 index 000000000..a59ce912b --- /dev/null +++ b/tools/perf/util/bpf_skel/vmlinux/vmlinux.h @@ -0,0 +1,215 @@ +#ifndef __VMLINUX_H +#define __VMLINUX_H + +#include // for define __always_inline +#include +#include +#include +#include + +// non-UAPI kernel data structures, used in the .bpf.c BPF tool component. + +// Just the fields used in these tools preserving the access index so that +// libbpf can fixup offsets with the ones used in the kernel when loading the +// BPF bytecode, if they differ from what is used here. + +typedef __u8 u8; +typedef __u32 u32; +typedef __s32 s32; +typedef __u64 u64; +typedef __s64 s64; + +typedef int pid_t; + +typedef __s64 time64_t; + +struct timespec64 { + time64_t tv_sec; + long int tv_nsec; +}; + +enum cgroup_subsys_id { + perf_event_cgrp_id = 8, +}; + +enum { + HI_SOFTIRQ = 0, + TIMER_SOFTIRQ, + NET_TX_SOFTIRQ, + NET_RX_SOFTIRQ, + BLOCK_SOFTIRQ, + IRQ_POLL_SOFTIRQ, + TASKLET_SOFTIRQ, + SCHED_SOFTIRQ, + HRTIMER_SOFTIRQ, + RCU_SOFTIRQ, /* Preferable RCU should always be the last softirq */ + + NR_SOFTIRQS +}; + +typedef struct { + s64 counter; +} __attribute__((preserve_access_index)) atomic64_t; + +typedef atomic64_t atomic_long_t; + +struct raw_spinlock { + int rawlock; +} __attribute__((preserve_access_index)); + +typedef struct raw_spinlock raw_spinlock_t; + +typedef struct { + struct raw_spinlock rlock; +} __attribute__((preserve_access_index)) spinlock_t; + +struct sighand_struct { + spinlock_t siglock; +} __attribute__((preserve_access_index)); + +struct rw_semaphore { + atomic_long_t owner; +} __attribute__((preserve_access_index)); + +struct mutex { + atomic_long_t owner; +} __attribute__((preserve_access_index)); + +struct kernfs_node { + u64 id; +} __attribute__((preserve_access_index)); + +struct cgroup { + struct kernfs_node *kn; + int level; +} __attribute__((preserve_access_index)); + +struct cgroup_subsys_state { + struct cgroup *cgroup; +} __attribute__((preserve_access_index)); + +struct css_set { + struct cgroup_subsys_state *subsys[13]; + struct cgroup *dfl_cgrp; +} __attribute__((preserve_access_index)); + +struct mm_struct { + struct rw_semaphore mmap_lock; +} __attribute__((preserve_access_index)); + +struct task_struct { + unsigned int flags; + struct mm_struct *mm; + pid_t pid; + pid_t tgid; + char comm[16]; + struct sighand_struct *sighand; + struct css_set *cgroups; +} __attribute__((preserve_access_index)); + +struct trace_entry { + short unsigned int type; + unsigned char flags; + unsigned char preempt_count; + int pid; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_irq_handler_entry { + struct trace_entry ent; + int irq; + u32 __data_loc_name; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_irq_handler_exit { + struct trace_entry ent; + int irq; + int ret; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_softirq { + struct trace_entry ent; + unsigned int vec; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_workqueue_execute_start { + struct trace_entry ent; + void *work; + void *function; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_workqueue_execute_end { + struct trace_entry ent; + void *work; + void *function; + char __data[]; +} __attribute__((preserve_access_index)); + +struct trace_event_raw_workqueue_activate_work { + struct trace_entry ent; + void *work; + char __data[]; +} __attribute__((preserve_access_index)); + +struct perf_sample_data { + u64 addr; + u64 period; + union perf_sample_weight weight; + u64 txn; + union perf_mem_data_src data_src; + u64 ip; + struct { + u32 pid; + u32 tid; + } tid_entry; + u64 time; + u64 id; + struct { + u32 cpu; + } cpu_entry; + u64 phys_addr; + u64 cgroup; + u64 data_page_size; + u64 code_page_size; +} __attribute__((__aligned__(64))) __attribute__((preserve_access_index)); + +struct perf_event { + struct perf_event *parent; + u64 id; +} __attribute__((preserve_access_index)); + +struct bpf_perf_event_data_kern { + struct perf_sample_data *data; + struct perf_event *event; +} __attribute__((preserve_access_index)); + +/* + * If 'struct rq' isn't defined for lock_contention.bpf.c, for the sake of + * rq___old and rq___new, then the type for the 'runqueue' variable ends up + * being a forward declaration (BTF_KIND_FWD) while the kernel has it defined + * (BTF_KIND_STRUCT). The definition appears in vmlinux.h rather than + * lock_contention.bpf.c for consistency with a generated vmlinux.h. + */ +struct rq {}; + +struct kmem_cache { + const char *name; +} __attribute__((preserve_access_index)); + +struct bpf_iter__kmem_cache { + struct kmem_cache *s; +} __attribute__((preserve_access_index)); + +struct zone { + spinlock_t lock; +} __attribute__((preserve_access_index)); + +struct pglist_data { + struct zone node_zones[6]; /* value for all possible config */ + int nr_zones; +} __attribute__((preserve_access_index)); + +#endif // __VMLINUX_H diff --git a/tools/perf/util/bpf_trace_augment.c b/tools/perf/util/bpf_trace_augment.c new file mode 100644 index 000000000..9e706f0fa --- /dev/null +++ b/tools/perf/util/bpf_trace_augment.c @@ -0,0 +1,143 @@ +#include +#include + +#include "util/debug.h" +#include "util/evlist.h" +#include "util/trace_augment.h" + +#include "bpf_skel/augmented_raw_syscalls.skel.h" + +static struct augmented_raw_syscalls_bpf *skel; +static struct evsel *bpf_output; + +int augmented_syscalls__prepare(void) +{ + struct bpf_program *prog; + char buf[128]; + int err; + + skel = augmented_raw_syscalls_bpf__open(); + if (!skel) { + pr_debug("Failed to open augmented syscalls BPF skeleton\n"); + return -errno; + } + + /* + * Disable attaching the BPF programs except for sys_enter and + * sys_exit that tail call into this as necessary. + */ + bpf_object__for_each_program(prog, skel->obj) { + if (prog != skel->progs.sys_enter && prog != skel->progs.sys_exit) + bpf_program__set_autoattach(prog, /*autoattach=*/false); + } + + err = augmented_raw_syscalls_bpf__load(skel); + if (err < 0) { + libbpf_strerror(err, buf, sizeof(buf)); + pr_debug("Failed to load augmented syscalls BPF skeleton: %s\n", buf); + return err; + } + + augmented_raw_syscalls_bpf__attach(skel); + return 0; +} + +int augmented_syscalls__create_bpf_output(struct evlist *evlist) +{ + int err = parse_event(evlist, "bpf-output/no-inherit=1,name=__augmented_syscalls__/"); + + if (err) { + pr_err("ERROR: Setup BPF output event failed: %d\n", err); + return err; + } + + bpf_output = evlist__last(evlist); + assert(evsel__name_is(bpf_output, "__augmented_syscalls__")); + + return 0; +} + +void augmented_syscalls__setup_bpf_output(void) +{ + struct perf_cpu cpu; + unsigned int i; + + if (bpf_output == NULL) + return; + + /* + * Set up the __augmented_syscalls__ BPF map to hold for each + * CPU the bpf-output event's file descriptor. + */ + perf_cpu_map__for_each_cpu(cpu, i, bpf_output->core.cpus) { + int mycpu = cpu.cpu; + + bpf_map__update_elem(skel->maps.__augmented_syscalls__, + &mycpu, sizeof(mycpu), + xyarray__entry(bpf_output->core.fd, + mycpu, 0), + sizeof(__u32), BPF_ANY); + } +} + +int augmented_syscalls__set_filter_pids(unsigned int nr, pid_t *pids) +{ + bool value = true; + int err = 0; + + if (skel == NULL) + return 0; + + for (size_t i = 0; i < nr; ++i) { + err = bpf_map__update_elem(skel->maps.pids_filtered, &pids[i], + sizeof(*pids), &value, sizeof(value), + BPF_ANY); + if (err) + break; + } + return err; +} + +int augmented_syscalls__get_map_fds(int *enter_fd, int *exit_fd, int *beauty_fd) +{ + if (skel == NULL) + return -1; + + *enter_fd = bpf_map__fd(skel->maps.syscalls_sys_enter); + *exit_fd = bpf_map__fd(skel->maps.syscalls_sys_exit); + *beauty_fd = bpf_map__fd(skel->maps.beauty_map_enter); + + if (*enter_fd < 0 || *exit_fd < 0 || *beauty_fd < 0) { + pr_err("Error: failed to get syscall or beauty map fd\n"); + return -1; + } + + return 0; +} + +struct bpf_program *augmented_syscalls__unaugmented(void) +{ + return skel->progs.syscall_unaugmented; +} + +struct bpf_program *augmented_syscalls__find_by_title(const char *name) +{ + struct bpf_program *pos; + const char *sec_name; + + if (skel->obj == NULL) + return NULL; + + bpf_object__for_each_program(pos, skel->obj) { + sec_name = bpf_program__section_name(pos); + if (sec_name && !strcmp(sec_name, name)) + return pos; + } + + return NULL; +} + +void augmented_syscalls__cleanup(void) +{ + augmented_raw_syscalls_bpf__destroy(skel); +} diff --git a/tools/perf/util/branch.c b/tools/perf/util/branch.c new file mode 100644 index 000000000..3712be067 --- /dev/null +++ b/tools/perf/util/branch.c @@ -0,0 +1,229 @@ +#include "util/map_symbol.h" +#include "util/branch.h" +#include + +static bool cross_area(u64 addr1, u64 addr2, int size) +{ + u64 align1, align2; + + align1 = addr1 & ~(size - 1); + align2 = addr2 & ~(size - 1); + + return (align1 != align2) ? true : false; +} + +#define AREA_4K 4096 +#define AREA_2M (2 * 1024 * 1024) + +void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, + u64 from, u64 to) +{ + if (flags->type == PERF_BR_UNKNOWN || from == 0) + return; + + if (flags->type == PERF_BR_EXTEND_ABI) + st->new_counts[flags->new_type]++; + else + st->counts[flags->type]++; + + if (flags->type == PERF_BR_COND) { + if (to > from) + st->cond_fwd++; + else + st->cond_bwd++; + } + + if (cross_area(from, to, AREA_2M)) + st->cross_2m++; + else if (cross_area(from, to, AREA_4K)) + st->cross_4k++; +} + +const char *branch_new_type_name(int new_type) +{ + const char *branch_new_names[PERF_BR_NEW_MAX] = { + "FAULT_ALGN", + "FAULT_DATA", + "FAULT_INST", +/* + * TODO: This switch should happen on 'perf_session__env(session)->arch' + * instead, because an arm64 platform perf recording could be + * opened for analysis on other platforms as well. + */ +#ifdef __aarch64__ + "ARM64_FIQ", + "ARM64_DEBUG_HALT", + "ARM64_DEBUG_EXIT", + "ARM64_DEBUG_INST", + "ARM64_DEBUG_DATA" +#else + "ARCH_1", + "ARCH_2", + "ARCH_3", + "ARCH_4", + "ARCH_5" +#endif + }; + + if (new_type >= 0 && new_type < PERF_BR_NEW_MAX) + return branch_new_names[new_type]; + + return NULL; +} + +const char *branch_type_name(int type) +{ + const char *branch_names[PERF_BR_MAX] = { + "N/A", + "COND", + "UNCOND", + "IND", + "CALL", + "IND_CALL", + "RET", + "SYSCALL", + "SYSRET", + "COND_CALL", + "COND_RET", + "ERET", + "IRQ", + "SERROR", + "NO_TX", + "", // Needed for PERF_BR_EXTEND_ABI that ends up triggering some compiler warnings about NULL deref + }; + + if (type >= 0 && type < PERF_BR_MAX) + return branch_names[type]; + + return NULL; +} + +const char *get_branch_type(struct branch_entry *e) +{ + if (e->flags.type == PERF_BR_UNKNOWN) + return ""; + + if (e->flags.type == PERF_BR_EXTEND_ABI) + return branch_new_type_name(e->flags.new_type); + + return branch_type_name(e->flags.type); +} + +void branch_type_stat_display(FILE *fp, const struct branch_type_stat *st) +{ + u64 total = 0; + int i; + + for (i = 0; i < PERF_BR_MAX; i++) + total += st->counts[i]; + + if (total == 0) + return; + + fprintf(fp, "\n#"); + fprintf(fp, "\n# Branch Statistics:"); + fprintf(fp, "\n#"); + + if (st->cond_fwd > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "COND_FWD", + 100.0 * (double)st->cond_fwd / (double)total); + } + + if (st->cond_bwd > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "COND_BWD", + 100.0 * (double)st->cond_bwd / (double)total); + } + + if (st->cross_4k > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "CROSS_4K", + 100.0 * (double)st->cross_4k / (double)total); + } + + if (st->cross_2m > 0) { + fprintf(fp, "\n%8s: %5.1f%%", + "CROSS_2M", + 100.0 * (double)st->cross_2m / (double)total); + } + + for (i = 0; i < PERF_BR_MAX; i++) { + if (st->counts[i] > 0) + fprintf(fp, "\n%8s: %5.1f%%", + branch_type_name(i), + 100.0 * + (double)st->counts[i] / (double)total); + } + + for (i = 0; i < PERF_BR_NEW_MAX; i++) { + if (st->new_counts[i] > 0) + fprintf(fp, "\n%8s: %5.1f%%", + branch_new_type_name(i), + 100.0 * + (double)st->new_counts[i] / (double)total); + } + +} + +static int count_str_scnprintf(int idx, const char *str, char *bf, int size) +{ + return scnprintf(bf, size, "%s%s", (idx) ? " " : " (", str); +} + +int branch_type_str(const struct branch_type_stat *st, char *bf, int size) +{ + int i, j = 0, printed = 0; + u64 total = 0; + + for (i = 0; i < PERF_BR_MAX; i++) + total += st->counts[i]; + + for (i = 0; i < PERF_BR_NEW_MAX; i++) + total += st->new_counts[i]; + + if (total == 0) + return 0; + + if (st->cond_fwd > 0) + printed += count_str_scnprintf(j++, "COND_FWD", bf + printed, size - printed); + + if (st->cond_bwd > 0) + printed += count_str_scnprintf(j++, "COND_BWD", bf + printed, size - printed); + + for (i = 0; i < PERF_BR_MAX; i++) { + if (i == PERF_BR_COND) + continue; + + if (st->counts[i] > 0) + printed += count_str_scnprintf(j++, branch_type_name(i), bf + printed, size - printed); + } + + for (i = 0; i < PERF_BR_NEW_MAX; i++) { + if (st->new_counts[i] > 0) + printed += count_str_scnprintf(j++, branch_new_type_name(i), bf + printed, size - printed); + } + + if (st->cross_4k > 0) + printed += count_str_scnprintf(j++, "CROSS_4K", bf + printed, size - printed); + + if (st->cross_2m > 0) + printed += count_str_scnprintf(j++, "CROSS_2M", bf + printed, size - printed); + + return printed; +} + +const char *branch_spec_desc(int spec) +{ + const char *branch_spec_outcomes[PERF_BR_SPEC_MAX] = { + "N/A", + "SPEC_WRONG_PATH", + "NON_SPEC_CORRECT_PATH", + "SPEC_CORRECT_PATH", + }; + + if (spec >= 0 && spec < PERF_BR_SPEC_MAX) + return branch_spec_outcomes[spec]; + + return NULL; +} diff --git a/tools/perf/util/branch.h b/tools/perf/util/branch.h new file mode 100644 index 000000000..a1d473649 --- /dev/null +++ b/tools/perf/util/branch.h @@ -0,0 +1,99 @@ +#ifndef _PERF_BRANCH_H +#define _PERF_BRANCH_H 1 +/* + * The linux/stddef.h isn't need here, but is needed for __always_inline used + * in files included from uapi/linux/perf_event.h such as + * /usr/include/linux/swab.h and /usr/include/linux/byteorder/little_endian.h, + * detected in at least musl libc, used in Alpine Linux. -acme + */ +#include +#include +#include +#include "util/map_symbol.h" +#include "util/sample.h" + +struct branch_flags { + union { + u64 value; + struct { + u64 mispred:1; + u64 predicted:1; + u64 in_tx:1; + u64 abort:1; + u64 cycles:16; + u64 type:4; + u64 spec:2; + u64 new_type:4; + u64 priv:3; + u64 not_taken:1; + u64 reserved:30; + }; + }; +}; + +struct branch_info { + struct addr_map_symbol from; + struct addr_map_symbol to; + struct branch_flags flags; + u64 branch_stack_cntr; + char *srcline_from; + char *srcline_to; +}; + +struct branch_entry { + u64 from; + u64 to; + struct branch_flags flags; +}; + +struct branch_stack { + u64 nr; + u64 hw_idx; + struct branch_entry entries[]; +}; + +/* + * The hw_idx is only available when PERF_SAMPLE_BRANCH_HW_INDEX is applied. + * Otherwise, the output format of a sample with branch stack is + * struct branch_stack { + * u64 nr; + * struct branch_entry entries[0]; + * } + * Check whether the hw_idx is available, + * and return the corresponding pointer of entries[0]. + */ +static inline struct branch_entry *perf_sample__branch_entries(struct perf_sample *sample) +{ + u64 *entry = (u64 *)sample->branch_stack; + + if (entry == NULL) + return NULL; + + entry++; + if (sample->no_hw_idx) + return (struct branch_entry *)entry; + return (struct branch_entry *)(++entry); +} + +struct branch_type_stat { + bool branch_to; + u64 counts[PERF_BR_MAX]; + u64 new_counts[PERF_BR_NEW_MAX]; + u64 cond_fwd; + u64 cond_bwd; + u64 cross_4k; + u64 cross_2m; +}; + +void branch_type_count(struct branch_type_stat *st, struct branch_flags *flags, + u64 from, u64 to); + +const char *branch_type_name(int type); +const char *branch_new_type_name(int new_type); +const char *get_branch_type(struct branch_entry *e); +void branch_type_stat_display(FILE *fp, const struct branch_type_stat *st); +int branch_type_str(const struct branch_type_stat *st, char *bf, int bfsize); + +const char *branch_spec_desc(int spec); + +#endif /* _PERF_BRANCH_H */ diff --git a/tools/perf/util/btf.c b/tools/perf/util/btf.c new file mode 100644 index 000000000..50d98f3e8 --- /dev/null +++ b/tools/perf/util/btf.c @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Arnaldo Carvalho de Melo + * + * Copyright (C) 2024, Red Hat, Inc + */ + +#include +#include +#include + +const struct btf_member *__btf_type__find_member_by_name(struct btf *btf, + int type_id, const char *member_name) +{ + const struct btf_type *t = btf__type_by_id(btf, type_id); + const struct btf_member *m; + u32 i; + + for (i = 0, m = btf_members(t); i < btf_vlen(t); i++, m++) { + const char *current_member_name = btf__name_by_offset(btf, m->name_off); + + if (!strcmp(current_member_name, member_name)) + return m; + } + + return NULL; +} diff --git a/tools/perf/util/btf.h b/tools/perf/util/btf.h new file mode 100644 index 000000000..05e6e5bf2 --- /dev/null +++ b/tools/perf/util/btf.h @@ -0,0 +1,10 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_UTIL_BTF +#define __PERF_UTIL_BTF 1 + +struct btf; +struct btf_member; + +const struct btf_member *__btf_type__find_member_by_name(struct btf *btf, + int type_id, const char *member_name); +#endif // __PERF_UTIL_BTF diff --git a/tools/perf/util/build-id.c b/tools/perf/util/build-id.c new file mode 100644 index 000000000..eb95ab90f --- /dev/null +++ b/tools/perf/util/build-id.c @@ -0,0 +1,983 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * build-id.c + * + * build-id support + * + * Copyright (C) 2009, 2010 Red Hat Inc. + * Copyright (C) 2009, 2010 Arnaldo Carvalho de Melo + */ +#include "util.h" // lsdir(), mkdir_p(), rm_rf() +#include +#include +#include +#include +#include +#include +#include "util/copyfile.h" +#include "dso.h" +#include "build-id.h" +#include "event.h" +#include "namespaces.h" +#include "map.h" +#include "symbol.h" +#include "thread.h" +#include +#include "debug.h" +#include "session.h" +#include "tool.h" +#include "header.h" +#include "vdso.h" +#include "path.h" +#include "probe-file.h" +#include "strlist.h" + +#ifdef HAVE_DEBUGINFOD_SUPPORT +#include +#endif + +#include +#include +#include +#include + +static bool no_buildid_cache; + +static int mark_dso_hit_callback(struct callchain_cursor_node *node, void *data __maybe_unused) +{ + struct map *map = node->ms.map; + + if (map) + dso__set_hit(map__dso(map)); + + return 0; +} + +int build_id__mark_dso_hit(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + struct addr_location al; + struct thread *thread = machine__findnew_thread(machine, sample->pid, + sample->tid); + + if (thread == NULL) { + pr_err("problem processing %s event at offset %#" PRIx64 ", skipping it.\n", + perf_event__name(event->header.type), sample->file_offset); + return -1; + } + + addr_location__init(&al); + if (thread__find_map(thread, sample->cpumode, sample->ip, &al)) + dso__set_hit(map__dso(al.map)); + + addr_location__exit(&al); + + sample__for_each_callchain_node(thread, sample, PERF_MAX_STACK_DEPTH, + /*symbols=*/false, mark_dso_hit_callback, /*data=*/NULL); + + + thread__put(thread); + return 0; +} + +int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size) +{ + size_t offs = 0; + + if (build_id->size == 0) { + /* Ensure bf is always \0 terminated. */ + if (bf_size > 0) + bf[0] = '\0'; + return 0; + } + + if (bf_size > 0) + bf[0] = '\0'; + + for (size_t i = 0; i < build_id->size && offs + 1 < bf_size; ++i) + offs += scnprintf(bf + offs, bf_size - offs, "%02x", build_id->data[i]); + + return offs; +} + +int sysfs__snprintf_build_id(const char *root_dir, char *sbuild_id, size_t sbuild_id_size) +{ + char notes[PATH_MAX]; + struct build_id bid = { .size = 0, }; + int ret; + + if (!root_dir) + root_dir = ""; + + scnprintf(notes, sizeof(notes), "%s/sys/kernel/notes", root_dir); + + ret = sysfs__read_build_id(notes, &bid); + if (ret < 0) + return ret; + + return build_id__snprintf(&bid, sbuild_id, sbuild_id_size); +} + +int filename__snprintf_build_id(const char *pathname, char *sbuild_id, size_t sbuild_id_size) +{ + struct build_id bid = { .size = 0, }; + int ret; + + ret = filename__read_build_id(pathname, &bid); + if (ret < 0) + return ret; + + return build_id__snprintf(&bid, sbuild_id, sbuild_id_size); +} + +/* asnprintf consolidates asprintf and snprintf */ +static int asnprintf(char **strp, size_t size, const char *fmt, ...) +{ + va_list ap; + int ret; + + if (!strp) + return -EINVAL; + + va_start(ap, fmt); + if (*strp) + ret = vsnprintf(*strp, size, fmt, ap); + else + ret = vasprintf(strp, fmt, ap); + va_end(ap); + + return ret; +} + +char *build_id_cache__kallsyms_path(const char *sbuild_id, char *bf, + size_t size) +{ + bool retry_old = true; + + snprintf(bf, size, "%s/%s/%s/kallsyms", + buildid_dir, DSO__NAME_KALLSYMS, sbuild_id); +retry: + if (!access(bf, F_OK)) + return bf; + if (retry_old) { + /* Try old style kallsyms cache */ + snprintf(bf, size, "%s/%s/%s", + buildid_dir, DSO__NAME_KALLSYMS, sbuild_id); + retry_old = false; + goto retry; + } + + return NULL; +} + +char *build_id_cache__linkname(const char *sbuild_id, char *bf, size_t size) +{ + char *tmp = bf; + int ret = asnprintf(&bf, size, "%s/.build-id/%.2s/%s", buildid_dir, + sbuild_id, sbuild_id + 2); + if (ret < 0 || (tmp && size < (unsigned int)ret)) + return NULL; + return bf; +} + +/* The caller is responsible to free the returned buffer. */ +char *build_id_cache__origname(const char *sbuild_id) +{ + char *linkname; + char buf[PATH_MAX]; + char *ret = NULL, *p; + size_t offs = 5; /* == strlen("../..") */ + ssize_t len; + + linkname = build_id_cache__linkname(sbuild_id, NULL, 0); + if (!linkname) + return NULL; + + len = readlink(linkname, buf, sizeof(buf) - 1); + if (len <= 0) + goto out; + buf[len] = '\0'; + + /* The link should be "../../" */ + p = strrchr(buf, '/'); /* Cut off the "/" */ + if (p && (p > buf + offs)) { + *p = '\0'; + if (buf[offs + 1] == '[') + offs++; /* + * This is a DSO name, like [kernel.kallsyms]. + * Skip the first '/', since this is not the + * cache of a regular file. + */ + ret = strdup(buf + offs); /* Skip "../..[/]" */ + } +out: + free(linkname); + return ret; +} + +/* Check if the given build_id cache is valid on current running system */ +static bool build_id_cache__valid_id(char *sbuild_id) +{ + char real_sbuild_id[SBUILD_ID_SIZE] = ""; + char *pathname; + int ret = 0; + bool result = false; + + pathname = build_id_cache__origname(sbuild_id); + if (!pathname) + return false; + + if (!strcmp(pathname, DSO__NAME_KALLSYMS)) + ret = sysfs__snprintf_build_id("/", real_sbuild_id, sizeof(real_sbuild_id)); + else if (pathname[0] == '/') + ret = filename__snprintf_build_id(pathname, real_sbuild_id, sizeof(real_sbuild_id)); + else + ret = -EINVAL; /* Should we support other special DSO cache? */ + if (ret >= 0) + result = (strcmp(sbuild_id, real_sbuild_id) == 0); + free(pathname); + + return result; +} + +static const char *build_id_cache__basename(bool is_kallsyms, bool is_vdso, + bool is_debug) +{ + return is_kallsyms ? "kallsyms" : (is_vdso ? "vdso" : (is_debug ? + "debug" : "elf")); +} + +char *__dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug, bool is_kallsyms) +{ + bool is_vdso = dso__is_vdso((struct dso *)dso); + char sbuild_id[SBUILD_ID_SIZE]; + char *linkname; + bool alloc = (bf == NULL); + int ret; + + if (!dso__has_build_id(dso)) + return NULL; + + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + linkname = build_id_cache__linkname(sbuild_id, NULL, 0); + if (!linkname) + return NULL; + + /* Check if old style build_id cache */ + if (is_regular_file(linkname)) + ret = asnprintf(&bf, size, "%s", linkname); + else + ret = asnprintf(&bf, size, "%s/%s", linkname, + build_id_cache__basename(is_kallsyms, is_vdso, + is_debug)); + if (ret < 0 || (!alloc && size < (unsigned int)ret)) + bf = NULL; + free(linkname); + + return bf; +} + +char *dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug) +{ + bool is_kallsyms = dso__is_kallsyms((struct dso *)dso); + + return __dso__build_id_filename(dso, bf, size, is_debug, is_kallsyms); +} + +static int write_buildid(const char *name, size_t name_len, struct build_id *bid, + pid_t pid, u16 misc, struct feat_fd *fd) +{ + int err; + struct perf_record_header_build_id b; + size_t len; + + len = name_len + 1; + len = PERF_ALIGN(len, sizeof(u64)); + + memset(&b, 0, sizeof(b)); + memcpy(&b.data, bid->data, bid->size); + b.size = (u8) bid->size; + misc |= PERF_RECORD_MISC_BUILD_ID_SIZE; + b.pid = pid; + b.header.misc = misc; + b.header.size = sizeof(b) + len; + + err = do_write(fd, &b, sizeof(b)); + if (err < 0) + return err; + + return write_padded(fd, name, name_len + 1, len); +} + +struct machine__write_buildid_table_cb_args { + struct machine *machine; + struct feat_fd *fd; + u16 kmisc, umisc; +}; + +static int machine__write_buildid_table_cb(struct dso *dso, void *data) +{ + struct machine__write_buildid_table_cb_args *args = data; + const char *name; + size_t name_len; + bool in_kernel = false; + + if (!dso__has_build_id(dso)) + return 0; + + if (!dso__hit(dso) && !dso__is_vdso(dso)) + return 0; + + if (dso__is_vdso(dso)) { + name = dso__short_name(dso); + name_len = dso__short_name_len(dso); + } else if (dso__is_kcore(dso)) { + name = args->machine->mmap_name; + name_len = strlen(name); + } else { + name = dso__long_name(dso); + name_len = dso__long_name_len(dso); + } + + in_kernel = dso__kernel(dso) || is_kernel_module(name, PERF_RECORD_MISC_CPUMODE_UNKNOWN); + return write_buildid(name, name_len, &dso__id(dso)->build_id, args->machine->pid, + in_kernel ? args->kmisc : args->umisc, args->fd); +} + +static int machine__write_buildid_table(struct machine *machine, struct feat_fd *fd) +{ + struct machine__write_buildid_table_cb_args args = { + .machine = machine, + .fd = fd, + .kmisc = PERF_RECORD_MISC_KERNEL, + .umisc = PERF_RECORD_MISC_USER, + }; + + if (!machine__is_host(machine)) { + args.kmisc = PERF_RECORD_MISC_GUEST_KERNEL; + args.umisc = PERF_RECORD_MISC_GUEST_USER; + } + + return dsos__for_each_dso(&machine->dsos, machine__write_buildid_table_cb, &args); +} + +int perf_session__write_buildid_table(struct perf_session *session, + struct feat_fd *fd) +{ + struct rb_node *nd; + int err = machine__write_buildid_table(&session->machines.host, fd); + + if (err) + return err; + + for (nd = rb_first_cached(&session->machines.guests); nd; + nd = rb_next(nd)) { + struct machine *pos = rb_entry(nd, struct machine, rb_node); + err = machine__write_buildid_table(pos, fd); + if (err) + break; + } + return err; +} + +void disable_buildid_cache(void) +{ + no_buildid_cache = true; +} + +static bool lsdir_bid_head_filter(const char *name __maybe_unused, + struct dirent *d) +{ + return (strlen(d->d_name) == 2) && + isxdigit(d->d_name[0]) && isxdigit(d->d_name[1]); +} + +static bool lsdir_bid_tail_filter(const char *name __maybe_unused, + struct dirent *d) +{ + int i = 0; + while (isxdigit(d->d_name[i]) && i < SBUILD_ID_SIZE - 3) + i++; + return (i >= SBUILD_ID_MIN_SIZE - 3) && (i <= SBUILD_ID_SIZE - 3) && + (d->d_name[i] == '\0'); +} + +struct strlist *build_id_cache__list_all(bool validonly) +{ + struct strlist *toplist, *linklist = NULL, *bidlist; + struct str_node *nd, *nd2; + char *topdir, *linkdir = NULL; + char sbuild_id[SBUILD_ID_SIZE]; + + /* for filename__ functions */ + if (validonly) + symbol__init(NULL); + + /* Open the top-level directory */ + if (asprintf(&topdir, "%s/.build-id/", buildid_dir) < 0) + return NULL; + + bidlist = strlist__new(NULL, NULL); + if (!bidlist) + goto out; + + toplist = lsdir(topdir, lsdir_bid_head_filter); + if (!toplist) { + pr_debug("Error in lsdir(%s): %d\n", topdir, errno); + /* If there is no buildid cache, return an empty list */ + if (errno == ENOENT) + goto out; + goto err_out; + } + + strlist__for_each_entry(nd, toplist) { + if (asprintf(&linkdir, "%s/%s", topdir, nd->s) < 0) + goto err_out; + /* Open the lower-level directory */ + linklist = lsdir(linkdir, lsdir_bid_tail_filter); + if (!linklist) { + pr_debug("Error in lsdir(%s): %d\n", linkdir, errno); + goto err_out; + } + strlist__for_each_entry(nd2, linklist) { + if (snprintf(sbuild_id, SBUILD_ID_SIZE, "%s%s", + nd->s, nd2->s) > SBUILD_ID_SIZE - 1) + goto err_out; + if (validonly && !build_id_cache__valid_id(sbuild_id)) + continue; + if (strlist__add(bidlist, sbuild_id) < 0) + goto err_out; + } + strlist__delete(linklist); + zfree(&linkdir); + } + +out_free: + strlist__delete(toplist); +out: + free(topdir); + + return bidlist; + +err_out: + strlist__delete(linklist); + zfree(&linkdir); + strlist__delete(bidlist); + bidlist = NULL; + goto out_free; +} + +static bool str_is_build_id(const char *maybe_sbuild_id, size_t len) +{ + size_t i; + + for (i = 0; i < len; i++) { + if (!isxdigit(maybe_sbuild_id[i])) + return false; + } + return true; +} + +/* Return the valid complete build-id */ +char *build_id_cache__complement(const char *incomplete_sbuild_id) +{ + struct strlist *bidlist; + struct str_node *nd, *cand = NULL; + char *sbuild_id = NULL; + size_t len = strlen(incomplete_sbuild_id); + + if (len >= SBUILD_ID_SIZE || + !str_is_build_id(incomplete_sbuild_id, len)) + return NULL; + + bidlist = build_id_cache__list_all(true); + if (!bidlist) + return NULL; + + strlist__for_each_entry(nd, bidlist) { + if (strncmp(nd->s, incomplete_sbuild_id, len) != 0) + continue; + if (cand) { /* Error: There are more than 2 candidates. */ + cand = NULL; + break; + } + cand = nd; + } + if (cand) + sbuild_id = strdup(cand->s); + strlist__delete(bidlist); + + return sbuild_id; +} + +char *build_id_cache__cachedir(const char *sbuild_id, const char *name, + struct nsinfo *nsi, bool is_kallsyms, + bool is_vdso) +{ + char *realname = NULL, *filename; + bool slash = is_kallsyms || is_vdso; + + if (!slash) + realname = nsinfo__realpath(name, nsi); + + if (asprintf(&filename, "%s%s%s%s%s", buildid_dir, slash ? "/" : "", + is_vdso ? DSO__NAME_VDSO : (realname ? realname : name), + sbuild_id ? "/" : "", sbuild_id ?: "") < 0) + filename = NULL; + + free(realname); + return filename; +} + +int build_id_cache__list_build_ids(const char *pathname, struct nsinfo *nsi, + struct strlist **result) +{ + char *dir_name; + int ret = 0; + + dir_name = build_id_cache__cachedir(NULL, pathname, nsi, false, false); + if (!dir_name) + return -ENOMEM; + + *result = lsdir(dir_name, lsdir_no_dot_filter); + if (!*result) + ret = -errno; + free(dir_name); + + return ret; +} + +#if defined(HAVE_LIBELF_SUPPORT) && defined(HAVE_GELF_GETNOTE_SUPPORT) +static int build_id_cache__add_sdt_cache(const char *sbuild_id, + const char *realname, + struct nsinfo *nsi) +{ + struct probe_cache *cache; + int ret; + struct nscookie nsc; + + cache = probe_cache__new(sbuild_id, nsi); + if (!cache) + return -1; + + nsinfo__mountns_enter(nsi, &nsc); + ret = probe_cache__scan_sdt(cache, realname); + nsinfo__mountns_exit(&nsc); + if (ret >= 0) { + pr_debug4("Found %d SDTs in %s\n", ret, realname); + if (probe_cache__commit(cache) < 0) + ret = -1; + } + probe_cache__delete(cache); + return ret; +} +#else +#define build_id_cache__add_sdt_cache(sbuild_id, realname, nsi) (0) +#endif + +static char *build_id_cache__find_debug(const char *sbuild_id, + struct nsinfo *nsi, + const char *root_dir) +{ + const char *dirname = "/usr/lib/debug/.build-id/"; + char *realname = NULL; + char dirbuf[PATH_MAX]; + char *debugfile; + struct nscookie nsc; + size_t len = 0; + + debugfile = calloc(1, PATH_MAX); + if (!debugfile) + goto out; + + if (root_dir) { + path__join(dirbuf, PATH_MAX, root_dir, dirname); + dirname = dirbuf; + } + + len = __symbol__join_symfs(debugfile, PATH_MAX, dirname); + snprintf(debugfile + len, PATH_MAX - len, "%.2s/%s.debug", sbuild_id, + sbuild_id + 2); + + nsinfo__mountns_enter(nsi, &nsc); + realname = realpath(debugfile, NULL); + if (realname && access(realname, R_OK)) + zfree(&realname); + nsinfo__mountns_exit(&nsc); + +#ifdef HAVE_DEBUGINFOD_SUPPORT + if (realname == NULL) { + debuginfod_client* c; + + pr_debug("Downloading debug info with build id %s\n", sbuild_id); + + c = debuginfod_begin(); + if (c != NULL) { + int fd = debuginfod_find_debuginfo(c, + (const unsigned char*)sbuild_id, 0, + &realname); + if (fd >= 0) + close(fd); /* retaining reference by realname */ + debuginfod_end(c); + } + } +#endif + +out: + free(debugfile); + return realname; +} + +int +build_id_cache__add(const char *sbuild_id, const char *name, const char *realname, + struct nsinfo *nsi, bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir) +{ + const size_t size = PATH_MAX; + char *filename = NULL, *dir_name = NULL, *linkname = zalloc(size), *tmp; + char *debugfile = NULL; + int err = -1; + + if (!proper_name) + proper_name = name; + + dir_name = build_id_cache__cachedir(sbuild_id, proper_name, nsi, is_kallsyms, + is_vdso); + if (!dir_name) + goto out_free; + + /* Remove old style build-id cache */ + if (is_regular_file(dir_name)) + if (unlink(dir_name)) + goto out_free; + + if (mkdir_p(dir_name, 0755)) + goto out_free; + + /* Save the allocated buildid dirname */ + if (asprintf(&filename, "%s/%s", dir_name, + build_id_cache__basename(is_kallsyms, is_vdso, + false)) < 0) { + filename = NULL; + goto out_free; + } + + if (access(filename, F_OK)) { + if (is_kallsyms) { + if (copyfile("/proc/kallsyms", filename)) + goto out_free; + } else if (nsi && nsinfo__need_setns(nsi)) { + if (copyfile_ns(name, filename, nsi)) + goto out_free; + } else if (link(realname, filename) && errno != EEXIST) { + struct stat f_stat; + + if (!(stat(name, &f_stat) < 0) && + copyfile_mode(name, filename, f_stat.st_mode)) + goto out_free; + } + } + + /* Some binaries are stripped, but have .debug files with their symbol + * table. Check to see if we can locate one of those, since the elf + * file itself may not be very useful to users of our tools without a + * symtab. + */ + if (!is_kallsyms && !is_vdso && + strncmp(".ko", name + strlen(name) - 3, 3)) { + debugfile = build_id_cache__find_debug(sbuild_id, nsi, root_dir); + if (debugfile) { + zfree(&filename); + if (asprintf(&filename, "%s/%s", dir_name, + build_id_cache__basename(false, false, true)) < 0) { + filename = NULL; + goto out_free; + } + if (access(filename, F_OK)) { + if (nsi && nsinfo__need_setns(nsi)) { + if (copyfile_ns(debugfile, filename, + nsi)) + goto out_free; + } else if (link(debugfile, filename) && + errno != EEXIST && + copyfile(debugfile, filename)) + goto out_free; + } + } + } + + if (!build_id_cache__linkname(sbuild_id, linkname, size)) + goto out_free; + tmp = strrchr(linkname, '/'); + *tmp = '\0'; + + if (access(linkname, X_OK) && mkdir_p(linkname, 0755)) + goto out_free; + + *tmp = '/'; + tmp = dir_name + strlen(buildid_dir) - 5; + memcpy(tmp, "../..", 5); + + if (symlink(tmp, linkname) == 0) { + err = 0; + } else if (errno == EEXIST) { + char path[PATH_MAX]; + ssize_t len; + + len = readlink(linkname, path, sizeof(path) - 1); + if (len <= 0) { + pr_err("Can't read link: %s\n", linkname); + goto out_free; + } + path[len] = '\0'; + + if (strcmp(tmp, path)) { + pr_debug("build <%s> already linked to %s\n", + sbuild_id, linkname); + } + err = 0; + } + + /* Update SDT cache : error is just warned */ + if (realname && + build_id_cache__add_sdt_cache(sbuild_id, realname, nsi) < 0) + pr_debug4("Failed to update/scan SDT cache for %s\n", realname); + +out_free: + free(filename); + free(debugfile); + free(dir_name); + free(linkname); + return err; +} + +int __build_id_cache__add_s(const char *sbuild_id, const char *name, + struct nsinfo *nsi, bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir) +{ + char *realname = NULL; + int err = -1; + + if (!is_kallsyms) { + if (!is_vdso) + realname = nsinfo__realpath(name, nsi); + else + realname = realpath(name, NULL); + if (!realname) + goto out_free; + } + + err = build_id_cache__add(sbuild_id, name, realname, nsi, + is_kallsyms, is_vdso, proper_name, root_dir); +out_free: + if (!is_kallsyms) + free(realname); + return err; +} + +static int build_id_cache__add_b(const struct build_id *bid, + const char *name, struct nsinfo *nsi, + bool is_kallsyms, bool is_vdso, + const char *proper_name, + const char *root_dir) +{ + char sbuild_id[SBUILD_ID_SIZE]; + + build_id__snprintf(bid, sbuild_id, sizeof(sbuild_id)); + + return __build_id_cache__add_s(sbuild_id, name, nsi, is_kallsyms, + is_vdso, proper_name, root_dir); +} + +bool build_id_cache__cached(const char *sbuild_id) +{ + bool ret = false; + char *filename = build_id_cache__linkname(sbuild_id, NULL, 0); + + if (filename && !access(filename, F_OK)) + ret = true; + free(filename); + + return ret; +} + +int build_id_cache__remove_s(const char *sbuild_id) +{ + const size_t size = PATH_MAX; + char *filename = zalloc(size), + *linkname = zalloc(size), *tmp; + int err = -1; + + if (filename == NULL || linkname == NULL) + goto out_free; + + if (!build_id_cache__linkname(sbuild_id, linkname, size)) + goto out_free; + + if (access(linkname, F_OK)) + goto out_free; + + if (readlink(linkname, filename, size - 1) < 0) + goto out_free; + + if (unlink(linkname)) + goto out_free; + + /* + * Since the link is relative, we must make it absolute: + */ + tmp = strrchr(linkname, '/') + 1; + snprintf(tmp, size - (tmp - linkname), "%s", filename); + + if (rm_rf(linkname)) + goto out_free; + + err = 0; +out_free: + free(filename); + free(linkname); + return err; +} + +static int filename__read_build_id_ns(const char *filename, + struct build_id *bid, + struct nsinfo *nsi) +{ + struct nscookie nsc; + int ret; + + nsinfo__mountns_enter(nsi, &nsc); + ret = filename__read_build_id(filename, bid); + nsinfo__mountns_exit(&nsc); + + return ret; +} + +static bool dso__build_id_mismatch(struct dso *dso, const char *name) +{ + struct build_id bid = { .size = 0, }; + bool ret = false; + + mutex_lock(dso__lock(dso)); + if (filename__read_build_id_ns(name, &bid, dso__nsinfo(dso)) >= 0) + ret = !dso__build_id_equal(dso, &bid); + + mutex_unlock(dso__lock(dso)); + + return ret; +} + +static int dso__cache_build_id(struct dso *dso, struct machine *machine, + void *priv __maybe_unused) +{ + bool is_kallsyms = dso__is_kallsyms(dso); + bool is_vdso = dso__is_vdso(dso); + const char *name = dso__long_name(dso); + const char *proper_name = NULL; + const char *root_dir = NULL; + char *allocated_name = NULL; + int ret = 0; + + if (!dso__has_build_id(dso) || !dso__hit(dso)) + return 0; + + if (dso__is_kcore(dso)) { + is_kallsyms = true; + name = machine->mmap_name; + } + + if (!machine__is_host(machine)) { + if (*machine->root_dir) { + root_dir = machine->root_dir; + ret = asprintf(&allocated_name, "%s/%s", root_dir, name); + if (ret < 0) + return ret; + proper_name = name; + name = allocated_name; + } else if (is_kallsyms) { + /* Cannot get guest kallsyms */ + return 0; + } + } + + if (!is_kallsyms && dso__build_id_mismatch(dso, name)) + goto out_free; + + mutex_lock(dso__lock(dso)); + ret = build_id_cache__add_b(dso__bid(dso), name, dso__nsinfo(dso), + is_kallsyms, is_vdso, proper_name, root_dir); + mutex_unlock(dso__lock(dso)); +out_free: + free(allocated_name); + return ret; +} + +static int +machines__for_each_dso(struct machines *machines, machine__dso_t fn, void *priv) +{ + int ret = machine__for_each_dso(&machines->host, fn, priv); + struct rb_node *nd; + + for (nd = rb_first_cached(&machines->guests); nd; + nd = rb_next(nd)) { + struct machine *pos = rb_entry(nd, struct machine, rb_node); + + ret |= machine__for_each_dso(pos, fn, priv); + } + return ret ? -1 : 0; +} + +int __perf_session__cache_build_ids(struct perf_session *session, + machine__dso_t fn, void *priv) +{ + if (no_buildid_cache) + return 0; + + if (mkdir(buildid_dir, 0755) != 0 && errno != EEXIST) + return -1; + + return machines__for_each_dso(&session->machines, fn, priv) ? -1 : 0; +} + +int perf_session__cache_build_ids(struct perf_session *session) +{ + return __perf_session__cache_build_ids(session, dso__cache_build_id, NULL); +} + +static bool machine__read_build_ids(struct machine *machine, bool with_hits) +{ + return dsos__read_build_ids(&machine->dsos, with_hits); +} + +bool perf_session__read_build_ids(struct perf_session *session, bool with_hits) +{ + struct rb_node *nd; + bool ret = machine__read_build_ids(&session->machines.host, with_hits); + + for (nd = rb_first_cached(&session->machines.guests); nd; + nd = rb_next(nd)) { + struct machine *pos = rb_entry(nd, struct machine, rb_node); + ret |= machine__read_build_ids(pos, with_hits); + } + + return ret; +} + +void build_id__init(struct build_id *bid, const u8 *data, size_t size) +{ + if (size > BUILD_ID_SIZE) { + pr_debug("Truncating build_id size from %zd\n", size); + size = BUILD_ID_SIZE; + } + memcpy(bid->data, data, size); + bid->size = size; +} + +bool build_id__is_defined(const struct build_id *bid) +{ + return bid && bid->size ? !!memchr_inv(bid->data, 0, bid->size) : false; +} diff --git a/tools/perf/util/build-id.h b/tools/perf/util/build-id.h new file mode 100644 index 000000000..73bad90b0 --- /dev/null +++ b/tools/perf/util/build-id.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_BUILD_ID_H_ +#define PERF_BUILD_ID_H_ 1 + +#define BUILD_ID_SIZE 20 /* SHA-1 length in bytes */ +#define BUILD_ID_MIN_SIZE 16 /* MD5/UUID/GUID length in bytes */ +#define SBUILD_ID_SIZE (BUILD_ID_SIZE * 2 + 1) +#define SBUILD_ID_MIN_SIZE (BUILD_ID_MIN_SIZE * 2 + 1) + +#include "machine.h" +#include "tool.h" +#include + +struct build_id { + u8 data[BUILD_ID_SIZE]; + u8 size; +}; + +struct dso; +struct feat_fd; +struct nsinfo; + +void build_id__init(struct build_id *bid, const u8 *data, size_t size); +int build_id__snprintf(const struct build_id *build_id, char *bf, size_t bf_size); +bool build_id__is_defined(const struct build_id *bid); +int sysfs__snprintf_build_id(const char *root_dir, char *sbuild_id, size_t sbuild_id_size); +int filename__snprintf_build_id(const char *pathname, char *sbuild_id, size_t sbuild_id_size); +char *build_id_cache__kallsyms_path(const char *sbuild_id, char *bf, + size_t size); + +char *dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug); +char *__dso__build_id_filename(const struct dso *dso, char *bf, size_t size, + bool is_debug, bool is_kallsyms); + +int build_id__mark_dso_hit(const struct perf_tool *tool, union perf_event *event, + struct perf_sample *sample, struct machine *machine); + +bool perf_session__read_build_ids(struct perf_session *session, bool with_hits); +int perf_session__write_buildid_table(struct perf_session *session, + struct feat_fd *fd); +int perf_session__cache_build_ids(struct perf_session *session); +int __perf_session__cache_build_ids(struct perf_session *session, + machine__dso_t fn, void *priv); + +char *build_id_cache__origname(const char *sbuild_id); +char *build_id_cache__linkname(const char *sbuild_id, char *bf, size_t size); +char *build_id_cache__cachedir(const char *sbuild_id, const char *name, + struct nsinfo *nsi, bool is_kallsyms, + bool is_vdso); + +struct strlist; + +struct strlist *build_id_cache__list_all(bool validonly); +char *build_id_cache__complement(const char *incomplete_sbuild_id); +int build_id_cache__list_build_ids(const char *pathname, struct nsinfo *nsi, + struct strlist **result); +bool build_id_cache__cached(const char *sbuild_id); +int build_id_cache__add(const char *sbuild_id, const char *name, const char *realname, + struct nsinfo *nsi, bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir); +int __build_id_cache__add_s(const char *sbuild_id, + const char *name, struct nsinfo *nsi, + bool is_kallsyms, bool is_vdso, + const char *proper_name, const char *root_dir); +static inline int build_id_cache__add_s(const char *sbuild_id, + const char *name, struct nsinfo *nsi, + bool is_kallsyms, bool is_vdso) +{ + return __build_id_cache__add_s(sbuild_id, name, nsi, is_kallsyms, is_vdso, NULL, NULL); +} +int build_id_cache__remove_s(const char *sbuild_id); + +extern char buildid_dir[]; + +void set_buildid_dir(const char *dir); +void disable_buildid_cache(void); + +#endif diff --git a/tools/perf/util/cache.h b/tools/perf/util/cache.h new file mode 100644 index 000000000..0b61840d4 --- /dev/null +++ b/tools/perf/util/cache.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CACHE_H +#define __PERF_CACHE_H + +#include "strbuf.h" +#include +#include "../ui/ui.h" + +#include +#include + +#define CMD_EXEC_PATH "--exec-path" +#define CMD_DEBUGFS_DIR "--debugfs-dir=" + +#define EXEC_PATH_ENVIRONMENT "PERF_EXEC_PATH" +#define PERF_DEBUGFS_ENVIRONMENT "PERF_DEBUGFS_DIR" +#define PERF_TRACEFS_ENVIRONMENT "PERF_TRACEFS_DIR" +#define PERF_PAGER_ENVIRONMENT "PERF_PAGER" + +int split_cmdline(char *cmdline, const char ***argv); + +#define alloc_nr(x) (((x)+16)*3/2) + +static inline int is_absolute_path(const char *path) +{ + return path[0] == '/'; +} + +char *mkpath(char *path_buf, size_t sz, const char *fmt, ...) __printf(3, 4); + +#endif /* __PERF_CACHE_H */ diff --git a/tools/perf/util/cacheline.c b/tools/perf/util/cacheline.c new file mode 100644 index 000000000..e98b5250a --- /dev/null +++ b/tools/perf/util/cacheline.c @@ -0,0 +1,25 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "cacheline.h" +#include + +#ifdef _SC_LEVEL1_DCACHE_LINESIZE +#define cache_line_size(cacheline_sizep) *cacheline_sizep = sysconf(_SC_LEVEL1_DCACHE_LINESIZE) +#else +#include +#include "debug.h" +static void cache_line_size(int *cacheline_sizep) +{ + if (sysfs__read_int("devices/system/cpu/cpu0/cache/index0/coherency_line_size", cacheline_sizep)) + pr_debug("cannot determine cache line size"); +} +#endif + +int cacheline_size(void) +{ + static int size; + + if (!size) + cache_line_size(&size); + + return size; +} diff --git a/tools/perf/util/cacheline.h b/tools/perf/util/cacheline.h new file mode 100644 index 000000000..fe6d5b60a --- /dev/null +++ b/tools/perf/util/cacheline.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_CACHELINE_H +#define PERF_CACHELINE_H + +#include + +int __pure cacheline_size(void); + + +/* + * Some architectures have 'Adjacent Cacheline Prefetch' feature, + * which performs like the cacheline size being doubled. + */ +static inline u64 cl_address(u64 address, bool double_cl) +{ + u64 size = cacheline_size(); + + if (double_cl) + size *= 2; + + /* return the cacheline of the address */ + return (address & ~(size - 1)); +} + +static inline u64 cl_offset(u64 address, bool double_cl) +{ + u64 size = cacheline_size(); + + if (double_cl) + size *= 2; + + /* return the offset inside cacheline */ + return (address & (size - 1)); +} + +#endif // PERF_CACHELINE_H diff --git a/tools/perf/util/call-path.c b/tools/perf/util/call-path.c new file mode 100644 index 000000000..5c60b8be1 --- /dev/null +++ b/tools/perf/util/call-path.c @@ -0,0 +1,114 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * call-path.h: Manipulate a tree data structure containing function call paths + * Copyright (c) 2014, Intel Corporation. + */ + +#include +#include +#include +#include + +#include "call-path.h" + +static void call_path__init(struct call_path *cp, struct call_path *parent, + struct symbol *sym, u64 ip, bool in_kernel) +{ + cp->parent = parent; + cp->sym = sym; + cp->ip = sym ? 0 : ip; + cp->db_id = 0; + cp->in_kernel = in_kernel; + RB_CLEAR_NODE(&cp->rb_node); + cp->children = RB_ROOT; +} + +struct call_path_root *call_path_root__new(void) +{ + struct call_path_root *cpr; + + cpr = zalloc(sizeof(struct call_path_root)); + if (!cpr) + return NULL; + call_path__init(&cpr->call_path, NULL, NULL, 0, false); + INIT_LIST_HEAD(&cpr->blocks); + return cpr; +} + +void call_path_root__free(struct call_path_root *cpr) +{ + struct call_path_block *pos, *n; + + list_for_each_entry_safe(pos, n, &cpr->blocks, node) { + list_del_init(&pos->node); + free(pos); + } + free(cpr); +} + +static struct call_path *call_path__new(struct call_path_root *cpr, + struct call_path *parent, + struct symbol *sym, u64 ip, + bool in_kernel) +{ + struct call_path_block *cpb; + struct call_path *cp; + size_t n; + + if (cpr->next < cpr->sz) { + cpb = list_last_entry(&cpr->blocks, struct call_path_block, + node); + } else { + cpb = zalloc(sizeof(struct call_path_block)); + if (!cpb) + return NULL; + list_add_tail(&cpb->node, &cpr->blocks); + cpr->sz += CALL_PATH_BLOCK_SIZE; + } + + n = cpr->next++ & CALL_PATH_BLOCK_MASK; + cp = &cpb->cp[n]; + + call_path__init(cp, parent, sym, ip, in_kernel); + + return cp; +} + +struct call_path *call_path__findnew(struct call_path_root *cpr, + struct call_path *parent, + struct symbol *sym, u64 ip, u64 ks) +{ + struct rb_node **p; + struct rb_node *node_parent = NULL; + struct call_path *cp; + bool in_kernel = ip >= ks; + + if (sym) + ip = 0; + + if (!parent) + return call_path__new(cpr, parent, sym, ip, in_kernel); + + p = &parent->children.rb_node; + while (*p != NULL) { + node_parent = *p; + cp = rb_entry(node_parent, struct call_path, rb_node); + + if (cp->sym == sym && cp->ip == ip) + return cp; + + if (sym < cp->sym || (sym == cp->sym && ip < cp->ip)) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + cp = call_path__new(cpr, parent, sym, ip, in_kernel); + if (!cp) + return NULL; + + rb_link_node(&cp->rb_node, node_parent, p); + rb_insert_color(&cp->rb_node, &parent->children); + + return cp; +} diff --git a/tools/perf/util/call-path.h b/tools/perf/util/call-path.h new file mode 100644 index 000000000..5875cfc81 --- /dev/null +++ b/tools/perf/util/call-path.h @@ -0,0 +1,68 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * call-path.h: Manipulate a tree data structure containing function call paths + * Copyright (c) 2014, Intel Corporation. + */ + +#ifndef __PERF_CALL_PATH_H +#define __PERF_CALL_PATH_H + +#include + +#include +#include + +/** + * struct call_path - node in list of calls leading to a function call. + * @parent: call path to the parent function call + * @sym: symbol of function called + * @ip: only if sym is null, the ip of the function + * @db_id: id used for db-export + * @in_kernel: whether function is a in the kernel + * @rb_node: node in parent's tree of called functions + * @children: tree of call paths of functions called + * + * In combination with the call_return structure, the call_path structure + * defines a context-sensitive call-graph. + */ +struct call_path { + struct call_path *parent; + struct symbol *sym; + u64 ip; + u64 db_id; + bool in_kernel; + struct rb_node rb_node; + struct rb_root children; +}; + +#define CALL_PATH_BLOCK_SHIFT 8 +#define CALL_PATH_BLOCK_SIZE (1 << CALL_PATH_BLOCK_SHIFT) +#define CALL_PATH_BLOCK_MASK (CALL_PATH_BLOCK_SIZE - 1) + +struct call_path_block { + struct call_path cp[CALL_PATH_BLOCK_SIZE]; + struct list_head node; +}; + +/** + * struct call_path_root - root of all call paths. + * @call_path: root call path + * @blocks: list of blocks to store call paths + * @next: next free space + * @sz: number of spaces + */ +struct call_path_root { + struct call_path call_path; + struct list_head blocks; + size_t next; + size_t sz; +}; + +struct call_path_root *call_path_root__new(void); +void call_path_root__free(struct call_path_root *cpr); + +struct call_path *call_path__findnew(struct call_path_root *cpr, + struct call_path *parent, + struct symbol *sym, u64 ip, u64 ks); + +#endif diff --git a/tools/perf/util/callchain.c b/tools/perf/util/callchain.c new file mode 100644 index 000000000..31c675cba --- /dev/null +++ b/tools/perf/util/callchain.c @@ -0,0 +1,1943 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2009-2011, Frederic Weisbecker + * + * Handle the callchains from the stream in an ad-hoc radix tree and then + * sort them in an rbtree. + * + * Using a radix for code path provides a fast retrieval and factorizes + * memory use. Also that lets us use the paths in a hierarchical graph view. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "asm/bug.h" + +#include "debug.h" +#include "dso.h" +#include "event.h" +#include "hist.h" +#include "sort.h" +#include "machine.h" +#include "map.h" +#include "callchain.h" +#include "branch.h" +#include "record.h" +#include "symbol.h" +#include "thread.h" +#include "util.h" +#include "../perf.h" + +#define CALLCHAIN_PARAM_DEFAULT \ + .mode = CHAIN_GRAPH_ABS, \ + .min_percent = 0.5, \ + .order = ORDER_CALLEE, \ + .key = CCKEY_FUNCTION, \ + .value = CCVAL_PERCENT, \ + +struct callchain_param callchain_param = { + CALLCHAIN_PARAM_DEFAULT +}; + +/* + * Are there any events usind DWARF callchains? + * + * I.e. + * + * -e cycles/call-graph=dwarf/ + */ +bool dwarf_callchain_users; + +struct callchain_param callchain_param_default = { + CALLCHAIN_PARAM_DEFAULT +}; + +/* Used for thread-local struct callchain_cursor. */ +static pthread_key_t callchain_cursor; + +int parse_callchain_record_opt(const char *arg, struct callchain_param *param) +{ + return parse_callchain_record(arg, param); +} + +static int parse_callchain_mode(const char *value) +{ + if (!strncmp(value, "graph", strlen(value))) { + callchain_param.mode = CHAIN_GRAPH_ABS; + return 0; + } + if (!strncmp(value, "flat", strlen(value))) { + callchain_param.mode = CHAIN_FLAT; + return 0; + } + if (!strncmp(value, "fractal", strlen(value))) { + callchain_param.mode = CHAIN_GRAPH_REL; + return 0; + } + if (!strncmp(value, "folded", strlen(value))) { + callchain_param.mode = CHAIN_FOLDED; + return 0; + } + return -1; +} + +static int parse_callchain_order(const char *value) +{ + if (!strncmp(value, "caller", strlen(value))) { + callchain_param.order = ORDER_CALLER; + callchain_param.order_set = true; + return 0; + } + if (!strncmp(value, "callee", strlen(value))) { + callchain_param.order = ORDER_CALLEE; + callchain_param.order_set = true; + return 0; + } + return -1; +} + +static int parse_callchain_sort_key(const char *value) +{ + if (!strncmp(value, "function", strlen(value))) { + callchain_param.key = CCKEY_FUNCTION; + return 0; + } + if (!strncmp(value, "address", strlen(value))) { + callchain_param.key = CCKEY_ADDRESS; + return 0; + } + if (!strncmp(value, "srcline", strlen(value))) { + callchain_param.key = CCKEY_SRCLINE; + return 0; + } + if (!strncmp(value, "branch", strlen(value))) { + callchain_param.branch_callstack = 1; + return 0; + } + return -1; +} + +static int parse_callchain_value(const char *value) +{ + if (!strncmp(value, "percent", strlen(value))) { + callchain_param.value = CCVAL_PERCENT; + return 0; + } + if (!strncmp(value, "period", strlen(value))) { + callchain_param.value = CCVAL_PERIOD; + return 0; + } + if (!strncmp(value, "count", strlen(value))) { + callchain_param.value = CCVAL_COUNT; + return 0; + } + return -1; +} + +static int get_stack_size(const char *str, unsigned long *_size) +{ + char *endptr; + unsigned long size; + unsigned long max_size = round_down(USHRT_MAX, sizeof(u64)); + + size = strtoul(str, &endptr, 0); + + do { + if (*endptr) + break; + + size = round_up(size, sizeof(u64)); + if (!size || size > max_size) + break; + + *_size = size; + return 0; + + } while (0); + + pr_err("callchain: Incorrect stack dump size (max %ld): %s\n", + max_size, str); + return -1; +} + +static int +__parse_callchain_report_opt(const char *arg, bool allow_record_opt) +{ + char *tok, *arg_copy; + char *endptr, *saveptr = NULL; + bool minpcnt_set = false; + bool record_opt_set = false; + bool try_stack_size = false; + + callchain_param.enabled = true; + symbol_conf.use_callchain = true; + + if (!arg) + return 0; + + arg_copy = strdup(arg); + if (!arg_copy) + return -ENOMEM; + + tok = strtok_r(arg_copy, ",", &saveptr); + while (tok) { + if (!strncmp(tok, "none", strlen(tok))) { + callchain_param.mode = CHAIN_NONE; + callchain_param.enabled = false; + symbol_conf.use_callchain = false; + goto out; + } + + if (!parse_callchain_mode(tok) || + !parse_callchain_order(tok) || + !parse_callchain_sort_key(tok) || + !parse_callchain_value(tok)) { + /* parsing ok - move on to the next */ + try_stack_size = false; + goto next; + } else if (allow_record_opt && !record_opt_set) { + if (parse_callchain_record(tok, &callchain_param)) + goto try_numbers; + + /* assume that number followed by 'dwarf' is stack size */ + if (callchain_param.record_mode == CALLCHAIN_DWARF) + try_stack_size = true; + + record_opt_set = true; + goto next; + } + +try_numbers: + if (try_stack_size) { + unsigned long size = 0; + + if (get_stack_size(tok, &size) < 0) + goto err_out; + callchain_param.dump_size = size; + try_stack_size = false; + } else if (!minpcnt_set) { + /* try to get the min percent */ + callchain_param.min_percent = strtod(tok, &endptr); + if (tok == endptr) + goto err_out; + minpcnt_set = true; + } else { + /* try print limit at last */ + callchain_param.print_limit = strtoul(tok, &endptr, 0); + if (tok == endptr) + goto err_out; + } +next: + tok = strtok_r(NULL, ",", &saveptr); + } + + if (callchain_register_param(&callchain_param) < 0) { + pr_err("Can't register callchain params\n"); + goto err_out; + } +out: + free(arg_copy); + return 0; +err_out: + free(arg_copy); + return -1; +} + +int parse_callchain_report_opt(const char *arg) +{ + return __parse_callchain_report_opt(arg, false); +} + +int parse_callchain_top_opt(const char *arg) +{ + return __parse_callchain_report_opt(arg, true); +} + +int parse_callchain_record(const char *arg, struct callchain_param *param) +{ + char *tok, *name, *saveptr = NULL; + char *buf; + int ret = -1; + + /* We need buffer that we know we can write to. */ + buf = strdup(arg); + if (!buf) + return -ENOMEM; + + tok = strtok_r(buf, ",", &saveptr); + name = tok ? : buf; + + do { + /* Framepointer style */ + if (!strncmp(name, "fp", sizeof("fp"))) { + ret = 0; + param->record_mode = CALLCHAIN_FP; + + tok = strtok_r(NULL, ",", &saveptr); + if (tok) { + unsigned long size; + + if (!strncmp(tok, "defer", sizeof("defer"))) { + param->defer = true; + } else { + size = strtoul(tok, &name, 0); + if (size < (unsigned) sysctl__max_stack()) + param->max_stack = size; + } + } + break; + + /* Dwarf style */ + } else if (!strncmp(name, "dwarf", sizeof("dwarf"))) { + const unsigned long default_stack_dump_size = 8192; + + ret = 0; + param->record_mode = CALLCHAIN_DWARF; + param->dump_size = default_stack_dump_size; + dwarf_callchain_users = true; + + tok = strtok_r(NULL, ",", &saveptr); + if (tok) { + unsigned long size = 0; + + ret = get_stack_size(tok, &size); + param->dump_size = size; + } + } else if (!strncmp(name, "lbr", sizeof("lbr"))) { + if (!strtok_r(NULL, ",", &saveptr)) { + param->record_mode = CALLCHAIN_LBR; + ret = 0; + } else + pr_err("callchain: No more arguments " + "needed for --call-graph lbr\n"); + break; + } else { + pr_err("callchain: Unknown --call-graph option " + "value: %s\n", arg); + break; + } + + } while (0); + + free(buf); + + if (param->defer && param->record_mode != CALLCHAIN_FP) { + pr_err("callchain: deferred callchain only works with FP\n"); + return -EINVAL; + } + + return ret; +} + +static void callchain_debug(const struct callchain_param *callchain) +{ + static const char *str[CALLCHAIN_MAX] = { "NONE", "FP", "DWARF", "LBR" }; + + pr_debug("callchain: type %s\n", str[callchain->record_mode]); + + if (callchain->record_mode == CALLCHAIN_DWARF) + pr_debug("callchain: stack dump size %d\n", + callchain->dump_size); +} + +int record_opts__parse_callchain(struct record_opts *record, + struct callchain_param *callchain, + const char *arg, bool unset) +{ + int ret; + + callchain->enabled = !unset; + + /* --no-call-graph */ + if (unset) { + callchain->record_mode = CALLCHAIN_NONE; + pr_debug("callchain: disabled\n"); + return 0; + } + + ret = parse_callchain_record_opt(arg, callchain); + if (!ret) { + /* Enable data address sampling for DWARF unwind. */ + if (callchain->record_mode == CALLCHAIN_DWARF && + !record->record_data_mmap_set) + record->record_data_mmap = true; + callchain_debug(callchain); + } + + return ret; +} + +int perf_callchain_config(const char *var, const char *value) +{ + char *endptr; + + if (!strstarts(var, "call-graph.")) + return 0; + var += sizeof("call-graph.") - 1; + + if (!strcmp(var, "record-mode")) + return parse_callchain_record_opt(value, &callchain_param); + if (!strcmp(var, "dump-size")) { + unsigned long size = 0; + int ret; + + ret = get_stack_size(value, &size); + callchain_param.dump_size = size; + + return ret; + } + if (!strcmp(var, "print-type")){ + int ret; + ret = parse_callchain_mode(value); + if (ret == -1) + pr_err("Invalid callchain mode: %s\n", value); + return ret; + } + if (!strcmp(var, "order")){ + int ret; + ret = parse_callchain_order(value); + if (ret == -1) + pr_err("Invalid callchain order: %s\n", value); + return ret; + } + if (!strcmp(var, "sort-key")){ + int ret; + ret = parse_callchain_sort_key(value); + if (ret == -1) + pr_err("Invalid callchain sort key: %s\n", value); + return ret; + } + if (!strcmp(var, "threshold")) { + callchain_param.min_percent = strtod(value, &endptr); + if (value == endptr) { + pr_err("Invalid callchain threshold: %s\n", value); + return -1; + } + } + if (!strcmp(var, "print-limit")) { + callchain_param.print_limit = strtod(value, &endptr); + if (value == endptr) { + pr_err("Invalid callchain print limit: %s\n", value); + return -1; + } + } + + return 0; +} + +static void +rb_insert_callchain(struct rb_root *root, struct callchain_node *chain, + enum chain_mode mode) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct callchain_node *rnode; + u64 chain_cumul = callchain_cumul_hits(chain); + + while (*p) { + u64 rnode_cumul; + + parent = *p; + rnode = rb_entry(parent, struct callchain_node, rb_node); + rnode_cumul = callchain_cumul_hits(rnode); + + switch (mode) { + case CHAIN_FLAT: + case CHAIN_FOLDED: + if (rnode->hit < chain->hit) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + break; + case CHAIN_GRAPH_ABS: /* Falldown */ + case CHAIN_GRAPH_REL: + if (rnode_cumul < chain_cumul) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + break; + case CHAIN_NONE: + default: + break; + } + } + + rb_link_node(&chain->rb_node, parent, p); + rb_insert_color(&chain->rb_node, root); +} + +static void +__sort_chain_flat(struct rb_root *rb_root, struct callchain_node *node, + u64 min_hit) +{ + struct rb_node *n; + struct callchain_node *child; + + n = rb_first(&node->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + __sort_chain_flat(rb_root, child, min_hit); + } + + if (node->hit && node->hit >= min_hit) + rb_insert_callchain(rb_root, node, CHAIN_FLAT); +} + +/* + * Once we get every callchains from the stream, we can now + * sort them by hit + */ +static void +sort_chain_flat(struct rb_root *rb_root, struct callchain_root *root, + u64 min_hit, struct callchain_param *param __maybe_unused) +{ + *rb_root = RB_ROOT; + __sort_chain_flat(rb_root, &root->node, min_hit); +} + +static void __sort_chain_graph_abs(struct callchain_node *node, + u64 min_hit) +{ + struct rb_node *n; + struct callchain_node *child; + + node->rb_root = RB_ROOT; + n = rb_first(&node->rb_root_in); + + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + __sort_chain_graph_abs(child, min_hit); + if (callchain_cumul_hits(child) >= min_hit) + rb_insert_callchain(&node->rb_root, child, + CHAIN_GRAPH_ABS); + } +} + +static void +sort_chain_graph_abs(struct rb_root *rb_root, struct callchain_root *chain_root, + u64 min_hit, struct callchain_param *param __maybe_unused) +{ + __sort_chain_graph_abs(&chain_root->node, min_hit); + rb_root->rb_node = chain_root->node.rb_root.rb_node; +} + +static void __sort_chain_graph_rel(struct callchain_node *node, + double min_percent) +{ + struct rb_node *n; + struct callchain_node *child; + u64 min_hit; + + node->rb_root = RB_ROOT; + min_hit = ceil(node->children_hit * min_percent); + + n = rb_first(&node->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + __sort_chain_graph_rel(child, min_percent); + if (callchain_cumul_hits(child) >= min_hit) + rb_insert_callchain(&node->rb_root, child, + CHAIN_GRAPH_REL); + } +} + +static void +sort_chain_graph_rel(struct rb_root *rb_root, struct callchain_root *chain_root, + u64 min_hit __maybe_unused, struct callchain_param *param) +{ + __sort_chain_graph_rel(&chain_root->node, param->min_percent / 100.0); + rb_root->rb_node = chain_root->node.rb_root.rb_node; +} + +int callchain_register_param(struct callchain_param *param) +{ + switch (param->mode) { + case CHAIN_GRAPH_ABS: + param->sort = sort_chain_graph_abs; + break; + case CHAIN_GRAPH_REL: + param->sort = sort_chain_graph_rel; + break; + case CHAIN_FLAT: + case CHAIN_FOLDED: + param->sort = sort_chain_flat; + break; + case CHAIN_NONE: + default: + return -1; + } + return 0; +} + +/* + * Create a child for a parent. If inherit_children, then the new child + * will become the new parent of it's parent children + */ +static struct callchain_node * +create_child(struct callchain_node *parent, bool inherit_children) +{ + struct callchain_node *new; + + new = zalloc(sizeof(*new)); + if (!new) { + perror("not enough memory to create child for code path tree"); + return NULL; + } + new->parent = parent; + INIT_LIST_HEAD(&new->val); + INIT_LIST_HEAD(&new->parent_val); + + if (inherit_children) { + struct rb_node *n; + struct callchain_node *child; + + new->rb_root_in = parent->rb_root_in; + parent->rb_root_in = RB_ROOT; + + n = rb_first(&new->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + child->parent = new; + n = rb_next(n); + } + + /* make it the first child */ + rb_link_node(&new->rb_node_in, NULL, &parent->rb_root_in.rb_node); + rb_insert_color(&new->rb_node_in, &parent->rb_root_in); + } + + return new; +} + + +/* + * Fill the node with callchain values + */ +static int +fill_node(struct callchain_node *node, struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *cursor_node; + + node->val_nr = cursor->nr - cursor->pos; + if (!node->val_nr) + pr_warning("Warning: empty node in callchain tree\n"); + + cursor_node = callchain_cursor_current(cursor); + + while (cursor_node) { + struct callchain_list *call; + + call = zalloc(sizeof(*call)); + if (!call) { + perror("not enough memory for the code path tree"); + return -ENOMEM; + } + call->ip = cursor_node->ip; + map_symbol__copy(&call->ms, &cursor_node->ms); + call->srcline = cursor_node->srcline; + + if (cursor_node->branch) { + call->branch_count = 1; + + if (cursor_node->branch_from) { + /* + * branch_from is set with value somewhere else + * to imply it's "to" of a branch. + */ + if (!call->brtype_stat) { + call->brtype_stat = zalloc(sizeof(*call->brtype_stat)); + if (!call->brtype_stat) { + perror("not enough memory for the code path branch statistics"); + zfree(&call->brtype_stat); + return -ENOMEM; + } + } + call->brtype_stat->branch_to = true; + + if (cursor_node->branch_flags.predicted) + call->predicted_count = 1; + + if (cursor_node->branch_flags.abort) + call->abort_count = 1; + + branch_type_count(call->brtype_stat, + &cursor_node->branch_flags, + cursor_node->branch_from, + cursor_node->ip); + } else { + /* + * It's "from" of a branch + */ + if (call->brtype_stat && call->brtype_stat->branch_to) + call->brtype_stat->branch_to = false; + call->cycles_count = + cursor_node->branch_flags.cycles; + call->iter_count = cursor_node->nr_loop_iter; + call->iter_cycles = cursor_node->iter_cycles; + } + } + + list_add_tail(&call->list, &node->val); + + callchain_cursor_advance(cursor); + cursor_node = callchain_cursor_current(cursor); + } + return 0; +} + +static struct callchain_node * +add_child(struct callchain_node *parent, + struct callchain_cursor *cursor, + u64 period) +{ + struct callchain_node *new; + + new = create_child(parent, false); + if (new == NULL) + return NULL; + + if (fill_node(new, cursor) < 0) { + struct callchain_list *call, *tmp; + + list_for_each_entry_safe(call, tmp, &new->val, list) { + list_del_init(&call->list); + map_symbol__exit(&call->ms); + zfree(&call->brtype_stat); + free(call); + } + free(new); + return NULL; + } + + new->children_hit = 0; + new->hit = period; + new->children_count = 0; + new->count = 1; + return new; +} + +enum match_result { + MATCH_ERROR = -1, + MATCH_EQ, + MATCH_LT, + MATCH_GT, +}; + +static enum match_result match_chain_strings(const char *left, + const char *right) +{ + enum match_result ret = MATCH_EQ; + int cmp; + + if (left && right) + cmp = strcmp(left, right); + else if (!left && right) + cmp = 1; + else if (left && !right) + cmp = -1; + else + return MATCH_ERROR; + + if (cmp != 0) + ret = cmp < 0 ? MATCH_LT : MATCH_GT; + + return ret; +} + +/* + * We need to always use relative addresses because we're aggregating + * callchains from multiple threads, i.e. different address spaces, so + * comparing absolute addresses make no sense as a symbol in a DSO may end up + * in a different address when used in a different binary or even the same + * binary but with some sort of address randomization technique, thus we need + * to compare just relative addresses. -acme + */ +static enum match_result match_chain_dso_addresses(struct map *left_map, u64 left_ip, + struct map *right_map, u64 right_ip) +{ + struct dso *left_dso = left_map ? map__dso(left_map) : NULL; + struct dso *right_dso = right_map ? map__dso(right_map) : NULL; + + if (left_dso != right_dso) + return left_dso < right_dso ? MATCH_LT : MATCH_GT; + + if (left_ip != right_ip) + return left_ip < right_ip ? MATCH_LT : MATCH_GT; + + return MATCH_EQ; +} + +static enum match_result match_chain(struct callchain_cursor_node *node, + struct callchain_list *cnode) +{ + enum match_result match = MATCH_ERROR; + + switch (callchain_param.key) { + case CCKEY_SRCLINE: + match = match_chain_strings(cnode->srcline, node->srcline); + if (match != MATCH_ERROR) + break; + /* otherwise fall-back to symbol-based comparison below */ + fallthrough; + case CCKEY_FUNCTION: + if (node->ms.sym && cnode->ms.sym) { + /* + * Compare inlined frames based on their symbol name + * because different inlined frames will have the same + * symbol start. Otherwise do a faster comparison based + * on the symbol start address. + */ + if (symbol__inlined(cnode->ms.sym) || symbol__inlined(node->ms.sym)) { + match = match_chain_strings(cnode->ms.sym->name, + node->ms.sym->name); + if (match != MATCH_ERROR) + break; + } else { + match = match_chain_dso_addresses(cnode->ms.map, cnode->ms.sym->start, + node->ms.map, node->ms.sym->start); + break; + } + } + /* otherwise fall-back to IP-based comparison below */ + fallthrough; + case CCKEY_ADDRESS: + default: + match = match_chain_dso_addresses(cnode->ms.map, cnode->ip, node->ms.map, node->ip); + break; + } + + if (match == MATCH_EQ && node->branch) { + cnode->branch_count++; + + if (node->branch_from) { + /* + * It's "to" of a branch + */ + if (!cnode->brtype_stat) { + cnode->brtype_stat = zalloc(sizeof(*cnode->brtype_stat)); + if (!cnode->brtype_stat) { + perror("not enough memory for the code path branch statistics"); + return MATCH_ERROR; + } + } + cnode->brtype_stat->branch_to = true; + + if (node->branch_flags.predicted) + cnode->predicted_count++; + + if (node->branch_flags.abort) + cnode->abort_count++; + + branch_type_count(cnode->brtype_stat, + &node->branch_flags, + node->branch_from, + node->ip); + } else { + /* + * It's "from" of a branch + */ + if (cnode->brtype_stat && cnode->brtype_stat->branch_to) + cnode->brtype_stat->branch_to = false; + cnode->cycles_count += node->branch_flags.cycles; + cnode->iter_count += node->nr_loop_iter; + cnode->iter_cycles += node->iter_cycles; + cnode->from_count++; + } + } + + return match; +} + +/* + * Split the parent in two parts (a new child is created) and + * give a part of its callchain to the created child. + * Then create another child to host the given callchain of new branch + */ +static int +split_add_child(struct callchain_node *parent, + struct callchain_cursor *cursor, + struct callchain_list *to_split, + u64 idx_parents, u64 idx_local, u64 period) +{ + struct callchain_node *new; + struct list_head *old_tail; + unsigned int idx_total = idx_parents + idx_local; + + /* split */ + new = create_child(parent, true); + if (new == NULL) + return -1; + + /* split the callchain and move a part to the new child */ + old_tail = parent->val.prev; + list_del_range(&to_split->list, old_tail); + new->val.next = &to_split->list; + new->val.prev = old_tail; + to_split->list.prev = &new->val; + old_tail->next = &new->val; + + /* split the hits */ + new->hit = parent->hit; + new->children_hit = parent->children_hit; + parent->children_hit = callchain_cumul_hits(new); + new->val_nr = parent->val_nr - idx_local; + parent->val_nr = idx_local; + new->count = parent->count; + new->children_count = parent->children_count; + parent->children_count = callchain_cumul_counts(new); + + /* create a new child for the new branch if any */ + if (idx_total < cursor->nr) { + struct callchain_node *first; + struct callchain_list *cnode; + struct callchain_cursor_node *node; + struct rb_node *p, **pp; + + parent->hit = 0; + parent->children_hit += period; + parent->count = 0; + parent->children_count += 1; + + node = callchain_cursor_current(cursor); + new = add_child(parent, cursor, period); + if (new == NULL) + return -1; + + /* + * This is second child since we moved parent's children + * to new (first) child above. + */ + p = parent->rb_root_in.rb_node; + first = rb_entry(p, struct callchain_node, rb_node_in); + cnode = list_first_entry(&first->val, struct callchain_list, + list); + + if (match_chain(node, cnode) == MATCH_LT) + pp = &p->rb_left; + else + pp = &p->rb_right; + + rb_link_node(&new->rb_node_in, p, pp); + rb_insert_color(&new->rb_node_in, &parent->rb_root_in); + } else { + parent->hit = period; + parent->count = 1; + } + return 0; +} + +static enum match_result +append_chain(struct callchain_node *root, + struct callchain_cursor *cursor, + u64 period); + +static int +append_chain_children(struct callchain_node *root, + struct callchain_cursor *cursor, + u64 period) +{ + struct callchain_node *rnode; + struct callchain_cursor_node *node; + struct rb_node **p = &root->rb_root_in.rb_node; + struct rb_node *parent = NULL; + + node = callchain_cursor_current(cursor); + if (!node) + return -1; + + /* lookup in children */ + while (*p) { + enum match_result ret; + + parent = *p; + rnode = rb_entry(parent, struct callchain_node, rb_node_in); + + /* If at least first entry matches, rely to children */ + ret = append_chain(rnode, cursor, period); + if (ret == MATCH_EQ) + goto inc_children_hit; + if (ret == MATCH_ERROR) + return -1; + + if (ret == MATCH_LT) + p = &parent->rb_left; + else + p = &parent->rb_right; + } + /* nothing in children, add to the current node */ + rnode = add_child(root, cursor, period); + if (rnode == NULL) + return -1; + + rb_link_node(&rnode->rb_node_in, parent, p); + rb_insert_color(&rnode->rb_node_in, &root->rb_root_in); + +inc_children_hit: + root->children_hit += period; + root->children_count++; + return 0; +} + +static enum match_result +append_chain(struct callchain_node *root, + struct callchain_cursor *cursor, + u64 period) +{ + struct callchain_list *cnode; + u64 start = cursor->pos; + bool found = false; + u64 matches; + enum match_result cmp = MATCH_ERROR; + + /* + * Lookup in the current node + * If we have a symbol, then compare the start to match + * anywhere inside a function, unless function + * mode is disabled. + */ + list_for_each_entry(cnode, &root->val, list) { + struct callchain_cursor_node *node; + + node = callchain_cursor_current(cursor); + if (!node) + break; + + cmp = match_chain(node, cnode); + if (cmp != MATCH_EQ) + break; + + found = true; + + callchain_cursor_advance(cursor); + } + + /* matches not, relay no the parent */ + if (!found) { + WARN_ONCE(cmp == MATCH_ERROR, "Chain comparison error\n"); + return cmp; + } + + matches = cursor->pos - start; + + /* we match only a part of the node. Split it and add the new chain */ + if (matches < root->val_nr) { + if (split_add_child(root, cursor, cnode, start, matches, + period) < 0) + return MATCH_ERROR; + + return MATCH_EQ; + } + + /* we match 100% of the path, increment the hit */ + if (matches == root->val_nr && cursor->pos == cursor->nr) { + root->hit += period; + root->count++; + return MATCH_EQ; + } + + /* We match the node and still have a part remaining */ + if (append_chain_children(root, cursor, period) < 0) + return MATCH_ERROR; + + return MATCH_EQ; +} + +int callchain_append(struct callchain_root *root, + struct callchain_cursor *cursor, + u64 period) +{ + if (cursor == NULL) + return -1; + + if (!cursor->nr) + return 0; + + callchain_cursor_commit(cursor); + + if (append_chain_children(&root->node, cursor, period) < 0) + return -1; + + if (cursor->nr > root->max_depth) + root->max_depth = cursor->nr; + + return 0; +} + +static int +merge_chain_branch(struct callchain_cursor *cursor, + struct callchain_node *dst, struct callchain_node *src) +{ + struct callchain_cursor_node **old_last = cursor->last; + struct callchain_node *child; + struct callchain_list *list, *next_list; + struct rb_node *n; + int old_pos = cursor->nr; + int err = 0; + + list_for_each_entry_safe(list, next_list, &src->val, list) { + struct map_symbol ms = { + .thread = thread__get(list->ms.thread), + .map = map__get(list->ms.map), + }; + callchain_cursor_append(cursor, list->ip, &ms, false, NULL, 0, 0, 0, list->srcline); + list_del_init(&list->list); + map_symbol__exit(&ms); + map_symbol__exit(&list->ms); + zfree(&list->brtype_stat); + free(list); + } + + if (src->hit) { + callchain_cursor_commit(cursor); + if (append_chain_children(dst, cursor, src->hit) < 0) + return -1; + } + + n = rb_first(&src->rb_root_in); + while (n) { + child = container_of(n, struct callchain_node, rb_node_in); + n = rb_next(n); + rb_erase(&child->rb_node_in, &src->rb_root_in); + + err = merge_chain_branch(cursor, dst, child); + if (err) + break; + + free(child); + } + + cursor->nr = old_pos; + cursor->last = old_last; + + return err; +} + +int callchain_merge(struct callchain_cursor *cursor, + struct callchain_root *dst, struct callchain_root *src) +{ + return merge_chain_branch(cursor, &dst->node, &src->node); +} + +int callchain_cursor_append(struct callchain_cursor *cursor, + u64 ip, struct map_symbol *ms, + bool branch, struct branch_flags *flags, + int nr_loop_iter, u64 iter_cycles, u64 branch_from, + const char *srcline) +{ + struct callchain_cursor_node *node = *cursor->last; + + if (!node) { + node = calloc(1, sizeof(*node)); + if (!node) + return -ENOMEM; + + *cursor->last = node; + } + + node->ip = ip; + map_symbol__exit(&node->ms); + map_symbol__copy(&node->ms, ms); + node->branch = branch; + node->nr_loop_iter = nr_loop_iter; + node->iter_cycles = iter_cycles; + node->srcline = srcline; + + if (flags) + memcpy(&node->branch_flags, flags, + sizeof(struct branch_flags)); + + node->branch_from = branch_from; + cursor->nr++; + + cursor->last = &node->next; + + return 0; +} + +int sample__resolve_callchain(struct perf_sample *sample, + struct callchain_cursor *cursor, struct symbol **parent, + struct addr_location *al, + int max_stack) +{ + if (sample->callchain == NULL && !symbol_conf.show_branchflag_count) + return 0; + + if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain || + perf_hpp_list.parent || symbol_conf.show_branchflag_count) { + return thread__resolve_callchain(al->thread, cursor, sample, + parent, al, max_stack); + } + return 0; +} + +int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample) +{ + if ((!symbol_conf.use_callchain || sample->callchain == NULL) && + !symbol_conf.show_branchflag_count) + return 0; + return callchain_append(he->callchain, get_tls_callchain_cursor(), sample->period); +} + +int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, + bool hide_unresolved) +{ + struct machine *machine = NULL; + + if (node->ms.thread) + machine = maps__machine(thread__maps(node->ms.thread)); + + map__put(al->map); + al->map = map__get(node->ms.map); + al->sym = node->ms.sym; + al->srcline = node->srcline; + al->addr = node->ip; + + if (al->sym == NULL) { + if (hide_unresolved) + return 0; + if (al->map == NULL) + goto out; + } + if (maps__equal(thread__maps(al->thread), machine__kernel_maps(machine))) { + if (machine__is_host(machine)) { + al->cpumode = PERF_RECORD_MISC_KERNEL; + al->level = 'k'; + } else { + al->cpumode = PERF_RECORD_MISC_GUEST_KERNEL; + al->level = 'g'; + } + } else { + if (machine__is_host(machine)) { + al->cpumode = PERF_RECORD_MISC_USER; + al->level = '.'; + } else if (perf_guest) { + al->cpumode = PERF_RECORD_MISC_GUEST_USER; + al->level = 'u'; + } else { + al->cpumode = PERF_RECORD_MISC_HYPERVISOR; + al->level = 'H'; + } + } + +out: + return 1; +} + +char *callchain_list__sym_name(struct callchain_list *cl, + char *bf, size_t bfsize, bool show_dso) +{ + bool show_addr = callchain_param.key == CCKEY_ADDRESS; + bool show_srcline = show_addr || callchain_param.key == CCKEY_SRCLINE; + int printed; + + if (cl->ms.sym) { + const char *inlined = symbol__inlined(cl->ms.sym) ? " (inlined)" : ""; + + if (show_srcline && cl->srcline) + printed = scnprintf(bf, bfsize, "%s %s%s", + cl->ms.sym->name, cl->srcline, + inlined); + else + printed = scnprintf(bf, bfsize, "%s%s", + cl->ms.sym->name, inlined); + } else + printed = scnprintf(bf, bfsize, "%#" PRIx64, cl->ip); + + if (show_dso) + scnprintf(bf + printed, bfsize - printed, " %s", + cl->ms.map ? + dso__short_name(map__dso(cl->ms.map)) : + "unknown"); + + return bf; +} + +char *callchain_node__scnprintf_value(struct callchain_node *node, + char *bf, size_t bfsize, u64 total) +{ + double percent = 0.0; + u64 period = callchain_cumul_hits(node); + unsigned count = callchain_cumul_counts(node); + + if (callchain_param.mode == CHAIN_FOLDED) { + period = node->hit; + count = node->count; + } + + switch (callchain_param.value) { + case CCVAL_PERIOD: + scnprintf(bf, bfsize, "%"PRIu64, period); + break; + case CCVAL_COUNT: + scnprintf(bf, bfsize, "%u", count); + break; + case CCVAL_PERCENT: + default: + if (total) + percent = period * 100.0 / total; + scnprintf(bf, bfsize, "%.2f%%", percent); + break; + } + return bf; +} + +int callchain_node__fprintf_value(struct callchain_node *node, + FILE *fp, u64 total) +{ + double percent = 0.0; + u64 period = callchain_cumul_hits(node); + unsigned count = callchain_cumul_counts(node); + + if (callchain_param.mode == CHAIN_FOLDED) { + period = node->hit; + count = node->count; + } + + switch (callchain_param.value) { + case CCVAL_PERIOD: + return fprintf(fp, "%"PRIu64, period); + case CCVAL_COUNT: + return fprintf(fp, "%u", count); + case CCVAL_PERCENT: + default: + if (total) + percent = period * 100.0 / total; + return percent_color_fprintf(fp, "%.2f%%", percent); + } + return 0; +} + +static void callchain_counts_value(struct callchain_node *node, + u64 *branch_count, u64 *predicted_count, + u64 *abort_count, u64 *cycles_count) +{ + struct callchain_list *clist; + + list_for_each_entry(clist, &node->val, list) { + if (branch_count) + *branch_count += clist->branch_count; + + if (predicted_count) + *predicted_count += clist->predicted_count; + + if (abort_count) + *abort_count += clist->abort_count; + + if (cycles_count) + *cycles_count += clist->cycles_count; + } +} + +static int callchain_node_branch_counts_cumul(struct callchain_node *node, + u64 *branch_count, + u64 *predicted_count, + u64 *abort_count, + u64 *cycles_count) +{ + struct callchain_node *child; + struct rb_node *n; + + n = rb_first(&node->rb_root_in); + while (n) { + child = rb_entry(n, struct callchain_node, rb_node_in); + n = rb_next(n); + + callchain_node_branch_counts_cumul(child, branch_count, + predicted_count, + abort_count, + cycles_count); + + callchain_counts_value(child, branch_count, + predicted_count, abort_count, + cycles_count); + } + + return 0; +} + +int callchain_branch_counts(struct callchain_root *root, + u64 *branch_count, u64 *predicted_count, + u64 *abort_count, u64 *cycles_count) +{ + if (branch_count) + *branch_count = 0; + + if (predicted_count) + *predicted_count = 0; + + if (abort_count) + *abort_count = 0; + + if (cycles_count) + *cycles_count = 0; + + return callchain_node_branch_counts_cumul(&root->node, + branch_count, + predicted_count, + abort_count, + cycles_count); +} + +static int count_pri64_printf(int idx, const char *str, u64 value, char *bf, int bfsize) +{ + return scnprintf(bf, bfsize, "%s%s:%" PRId64 "", (idx) ? " " : " (", str, value); +} + +static int count_float_printf(int idx, const char *str, float value, + char *bf, int bfsize, float threshold) +{ + if (threshold != 0.0 && value < threshold) + return 0; + + return scnprintf(bf, bfsize, "%s%s:%.1f%%", (idx) ? " " : " (", str, value); +} + +static int branch_to_str(char *bf, int bfsize, + u64 branch_count, u64 predicted_count, + u64 abort_count, + const struct branch_type_stat *brtype_stat) +{ + int printed, i = 0; + + printed = branch_type_str(brtype_stat, bf, bfsize); + if (printed) + i++; + + if (predicted_count < branch_count) { + printed += count_float_printf(i++, "predicted", + predicted_count * 100.0 / branch_count, + bf + printed, bfsize - printed, 0.0); + } + + if (abort_count) { + printed += count_float_printf(i++, "abort", + abort_count * 100.0 / branch_count, + bf + printed, bfsize - printed, 0.1); + } + + if (i) + printed += scnprintf(bf + printed, bfsize - printed, ")"); + + return printed; +} + +static int branch_from_str(char *bf, int bfsize, + u64 branch_count, + u64 cycles_count, u64 iter_count, + u64 iter_cycles, u64 from_count) +{ + int printed = 0, i = 0; + u64 cycles, v = 0; + + cycles = cycles_count / branch_count; + if (cycles) { + printed += count_pri64_printf(i++, "cycles", + cycles, + bf + printed, bfsize - printed); + } + + if (iter_count && from_count) { + v = iter_count / from_count; + if (v) { + printed += count_pri64_printf(i++, "iter", + v, bf + printed, bfsize - printed); + + printed += count_pri64_printf(i++, "avg_cycles", + iter_cycles / iter_count, + bf + printed, bfsize - printed); + } + } + + if (i) + printed += scnprintf(bf + printed, bfsize - printed, ")"); + + return printed; +} + +static int counts_str_build(char *bf, int bfsize, + u64 branch_count, u64 predicted_count, + u64 abort_count, u64 cycles_count, + u64 iter_count, u64 iter_cycles, + u64 from_count, + const struct branch_type_stat *brtype_stat) +{ + int printed; + + if (branch_count == 0) + return scnprintf(bf, bfsize, " (calltrace)"); + + if (brtype_stat->branch_to) { + printed = branch_to_str(bf, bfsize, branch_count, + predicted_count, abort_count, brtype_stat); + } else { + printed = branch_from_str(bf, bfsize, branch_count, + cycles_count, iter_count, iter_cycles, + from_count); + } + + if (!printed) + bf[0] = 0; + + return printed; +} + +static int callchain_counts_printf(FILE *fp, char *bf, int bfsize, + u64 branch_count, u64 predicted_count, + u64 abort_count, u64 cycles_count, + u64 iter_count, u64 iter_cycles, + u64 from_count, + const struct branch_type_stat *brtype_stat) +{ + char str[256]; + + counts_str_build(str, sizeof(str), branch_count, + predicted_count, abort_count, cycles_count, + iter_count, iter_cycles, from_count, brtype_stat); + + if (fp) + return fprintf(fp, "%s", str); + + return scnprintf(bf, bfsize, "%s", str); +} + +int callchain_list_counts__printf_value(struct callchain_list *clist, + FILE *fp, char *bf, int bfsize) +{ + static const struct branch_type_stat empty_brtype_stat = {}; + const struct branch_type_stat *brtype_stat; + u64 branch_count, predicted_count; + u64 abort_count, cycles_count; + u64 iter_count, iter_cycles; + u64 from_count; + + brtype_stat = clist->brtype_stat ?: &empty_brtype_stat; + branch_count = clist->branch_count; + predicted_count = clist->predicted_count; + abort_count = clist->abort_count; + cycles_count = clist->cycles_count; + iter_count = clist->iter_count; + iter_cycles = clist->iter_cycles; + from_count = clist->from_count; + + return callchain_counts_printf(fp, bf, bfsize, branch_count, + predicted_count, abort_count, + cycles_count, iter_count, iter_cycles, + from_count, brtype_stat); +} + +static void free_callchain_node(struct callchain_node *node) +{ + struct callchain_list *list, *tmp; + struct callchain_node *child; + struct rb_node *n; + + list_for_each_entry_safe(list, tmp, &node->parent_val, list) { + list_del_init(&list->list); + map_symbol__exit(&list->ms); + zfree(&list->brtype_stat); + free(list); + } + + list_for_each_entry_safe(list, tmp, &node->val, list) { + list_del_init(&list->list); + map_symbol__exit(&list->ms); + zfree(&list->brtype_stat); + free(list); + } + + n = rb_first(&node->rb_root_in); + while (n) { + child = container_of(n, struct callchain_node, rb_node_in); + n = rb_next(n); + rb_erase(&child->rb_node_in, &node->rb_root_in); + + free_callchain_node(child); + free(child); + } +} + +void free_callchain(struct callchain_root *root) +{ + if (!symbol_conf.use_callchain) + return; + + free_callchain_node(&root->node); +} + +void callchain_cursor_cleanup(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + + for (node = cursor->first; node; node = next) { + next = node->next; + free(node); + } + cursor->first = NULL; + cursor->last = &cursor->first; + cursor->curr = NULL; +} + +static u64 decay_callchain_node(struct callchain_node *node) +{ + struct callchain_node *child; + struct rb_node *n; + u64 child_hits = 0; + + n = rb_first(&node->rb_root_in); + while (n) { + child = container_of(n, struct callchain_node, rb_node_in); + + child_hits += decay_callchain_node(child); + n = rb_next(n); + } + + node->hit = (node->hit * 7) / 8; + node->children_hit = child_hits; + + return node->hit; +} + +void decay_callchain(struct callchain_root *root) +{ + if (!symbol_conf.use_callchain) + return; + + decay_callchain_node(&root->node); +} + +int callchain_node__make_parent_list(struct callchain_node *node) +{ + struct callchain_node *parent = node->parent; + struct callchain_list *chain, *new; + LIST_HEAD(head); + + while (parent) { + list_for_each_entry_reverse(chain, &parent->val, list) { + new = malloc(sizeof(*new)); + if (new == NULL) + goto out; + *new = *chain; + new->has_children = false; + map_symbol__copy(&new->ms, &chain->ms); + list_add_tail(&new->list, &head); + } + parent = parent->parent; + } + + list_for_each_entry_safe_reverse(chain, new, &head, list) + list_move_tail(&chain->list, &node->parent_val); + + if (!list_empty(&node->parent_val)) { + chain = list_first_entry(&node->parent_val, struct callchain_list, list); + chain->has_children = rb_prev(&node->rb_node) || rb_next(&node->rb_node); + + chain = list_first_entry(&node->val, struct callchain_list, list); + chain->has_children = false; + } + return 0; + +out: + list_for_each_entry_safe(chain, new, &head, list) { + list_del_init(&chain->list); + map_symbol__exit(&chain->ms); + zfree(&chain->brtype_stat); + free(chain); + } + return -ENOMEM; +} + +static void callchain_cursor__delete(void *vcursor) +{ + struct callchain_cursor *cursor = vcursor; + struct callchain_cursor_node *node, *next; + + callchain_cursor_reset(cursor); + for (node = cursor->first; node != NULL; node = next) { + next = node->next; + free(node); + } + free(cursor); +} + +static void init_callchain_cursor_key(void) +{ + if (pthread_key_create(&callchain_cursor, callchain_cursor__delete)) { + pr_err("callchain cursor creation failed"); + abort(); + } +} + +struct callchain_cursor *get_tls_callchain_cursor(void) +{ + static pthread_once_t once_control = PTHREAD_ONCE_INIT; + struct callchain_cursor *cursor; + + pthread_once(&once_control, init_callchain_cursor_key); + cursor = pthread_getspecific(callchain_cursor); + if (!cursor) { + cursor = zalloc(sizeof(*cursor)); + if (!cursor) + pr_debug3("%s: not enough memory\n", __func__); + pthread_setspecific(callchain_cursor, cursor); + } + return cursor; +} + +int callchain_cursor__copy(struct callchain_cursor *dst, + struct callchain_cursor *src) +{ + int rc = 0; + + callchain_cursor_reset(dst); + callchain_cursor_commit(src); + + while (true) { + struct callchain_cursor_node *node; + + node = callchain_cursor_current(src); + if (node == NULL) + break; + + rc = callchain_cursor_append(dst, node->ip, &node->ms, + node->branch, &node->branch_flags, + node->nr_loop_iter, + node->iter_cycles, + node->branch_from, node->srcline); + if (rc) + break; + + callchain_cursor_advance(src); + } + + return rc; +} + +/* + * Initialize a cursor before adding entries inside, but keep + * the previously allocated entries as a cache. + */ +void callchain_cursor_reset(struct callchain_cursor *cursor) +{ + struct callchain_cursor_node *node; + + cursor->nr = 0; + cursor->last = &cursor->first; + + for (node = cursor->first; node != NULL; node = node->next) + map_symbol__exit(&node->ms); +} + +void callchain_param_setup(u64 sample_type, uint16_t e_machine) +{ + if (symbol_conf.use_callchain || symbol_conf.cumulate_callchain) { + if ((sample_type & PERF_SAMPLE_REGS_USER) && + (sample_type & PERF_SAMPLE_STACK_USER)) { + callchain_param.record_mode = CALLCHAIN_DWARF; + dwarf_callchain_users = true; + } else if (sample_type & PERF_SAMPLE_BRANCH_STACK) + callchain_param.record_mode = CALLCHAIN_LBR; + else + callchain_param.record_mode = CALLCHAIN_FP; + } + + /* + * It's necessary to use libunwind to reliably determine the caller of + * a leaf function on aarch64, as otherwise we cannot know whether to + * start from the LR or FP. + * + * Always starting from the LR can result in duplicate or entirely + * erroneous entries. Always skipping the LR and starting from the FP + * can result in missing entries. + */ + if (callchain_param.record_mode == CALLCHAIN_FP && e_machine == EM_AARCH64) + dwarf_callchain_users = true; +} + +static bool chain_match(struct callchain_list *base_chain, + struct callchain_list *pair_chain) +{ + enum match_result match; + + match = match_chain_strings(base_chain->srcline, + pair_chain->srcline); + if (match != MATCH_ERROR) + return match == MATCH_EQ; + + match = match_chain_dso_addresses(base_chain->ms.map, + base_chain->ip, + pair_chain->ms.map, + pair_chain->ip); + + return match == MATCH_EQ; +} + +bool callchain_cnode_matched(struct callchain_node *base_cnode, + struct callchain_node *pair_cnode) +{ + struct callchain_list *base_chain, *pair_chain; + bool match = false; + + pair_chain = list_first_entry(&pair_cnode->val, + struct callchain_list, + list); + + list_for_each_entry(base_chain, &base_cnode->val, list) { + if (&pair_chain->list == &pair_cnode->val) + return false; + + if (!base_chain->srcline || !pair_chain->srcline) { + pair_chain = list_next_entry(pair_chain, list); + continue; + } + + match = chain_match(base_chain, pair_chain); + if (!match) + return false; + + pair_chain = list_next_entry(pair_chain, list); + } + + /* + * Say chain1 is ABC, chain2 is ABCD, we consider they are + * not fully matched. + */ + if (pair_chain && (&pair_chain->list != &pair_cnode->val)) + return false; + + return match; +} + +static u64 count_callchain_hits(struct hist_entry *he) +{ + struct rb_root *root = &he->sorted_chain; + struct rb_node *rb_node = rb_first(root); + struct callchain_node *node; + u64 chain_hits = 0; + + while (rb_node) { + node = rb_entry(rb_node, struct callchain_node, rb_node); + chain_hits += node->hit; + rb_node = rb_next(rb_node); + } + + return chain_hits; +} + +u64 callchain_total_hits(struct hists *hists) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + u64 chain_hits = 0; + + while (next) { + struct hist_entry *he = rb_entry(next, struct hist_entry, + rb_node); + + chain_hits += count_callchain_hits(he); + next = rb_next(&he->rb_node); + } + + return chain_hits; +} + +s64 callchain_avg_cycles(struct callchain_node *cnode) +{ + struct callchain_list *chain; + s64 cycles = 0; + + list_for_each_entry(chain, &cnode->val, list) { + if (chain->srcline && chain->branch_count) + cycles += chain->cycles_count / chain->branch_count; + } + + return cycles; +} + +int sample__for_each_callchain_node(struct thread *thread, + struct perf_sample *sample, int max_stack, + bool symbols, callchain_iter_fn cb, void *data) +{ + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + int ret; + + if (!cursor) + return -ENOMEM; + + /* Fill in the callchain. */ + ret = __thread__resolve_callchain(thread, cursor, sample, + /*parent=*/NULL, /*root_al=*/NULL, + max_stack, symbols); + if (ret) + return ret; + + /* Switch from writing the callchain to reading it. */ + callchain_cursor_commit(cursor); + + while (1) { + struct callchain_cursor_node *node = callchain_cursor_current(cursor); + + if (!node) + break; + + ret = cb(node, data); + if (ret) + return ret; + + callchain_cursor_advance(cursor); + } + return 0; +} + +/* + * This function merges earlier samples (@sample_orig) waiting for deferred + * user callchains with the matching callchain record (@sample_callchain) + * which is delivered now. The @sample_orig->callchain should be released + * after use if ->deferred_callchain is set. + */ +int sample__merge_deferred_callchain(struct perf_sample *sample_orig, + struct perf_sample *sample_callchain) +{ + u64 nr_orig = sample_orig->callchain->nr - 1; + u64 nr_deferred = sample_callchain->callchain->nr; + struct ip_callchain *callchain; + + if (sample_orig->merged_callchain) { + /* Already merged. */ + return -EINVAL; + } + + if (sample_orig->callchain->nr < 2) { + sample_orig->deferred_callchain = false; + return -EINVAL; + } + + callchain = calloc(1 + nr_orig + nr_deferred, sizeof(u64)); + if (callchain == NULL) + return -ENOMEM; + + callchain->nr = nr_orig + nr_deferred; + /* copy original including PERF_CONTEXT_USER_DEFERRED (but the cookie) */ + memcpy(callchain->ips, sample_orig->callchain->ips, nr_orig * sizeof(u64)); + /* copy deferred user callchains */ + memcpy(&callchain->ips[nr_orig], sample_callchain->callchain->ips, + nr_deferred * sizeof(u64)); + + sample_orig->merged_callchain = true; + sample_orig->callchain = callchain; + return 0; +} diff --git a/tools/perf/util/callchain.h b/tools/perf/util/callchain.h new file mode 100644 index 000000000..8b1e405d1 --- /dev/null +++ b/tools/perf/util/callchain.h @@ -0,0 +1,315 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CALLCHAIN_H +#define __PERF_CALLCHAIN_H + +#include +#include +#include "map_symbol.h" +#include "branch.h" + +struct addr_location; +struct hist_entry; +struct hists; +struct ip_callchain; +struct perf_sample; +struct record_opts; +struct thread; + +#define HELP_PAD "\t\t\t\t" + +#define CALLCHAIN_HELP "setup and enables call-graph (stack chain/backtrace):\n\n" + +# define RECORD_MODE_HELP HELP_PAD "record_mode:\tcall graph recording mode (fp|dwarf|lbr)\n" + +#define RECORD_SIZE_HELP \ + HELP_PAD "record_size:\tif record_mode is 'dwarf', max size of stack recording ()\n" \ + HELP_PAD "\t\tdefault: 8192 (bytes)\n" + +#define CALLCHAIN_RECORD_HELP CALLCHAIN_HELP RECORD_MODE_HELP RECORD_SIZE_HELP + +#define CALLCHAIN_REPORT_HELP \ + HELP_PAD "print_type:\tcall graph printing style (graph|flat|fractal|folded|none)\n" \ + HELP_PAD "threshold:\tminimum call graph inclusion threshold ()\n" \ + HELP_PAD "print_limit:\tmaximum number of call graph entry ()\n" \ + HELP_PAD "order:\t\tcall graph order (caller|callee)\n" \ + HELP_PAD "sort_key:\tcall graph sort key (function|address)\n" \ + HELP_PAD "branch:\t\tinclude last branch info to call graph (branch)\n" \ + HELP_PAD "value:\t\tcall graph value (percent|period|count)\n" + +enum perf_call_graph_mode { + CALLCHAIN_NONE, + CALLCHAIN_FP, + CALLCHAIN_DWARF, + CALLCHAIN_LBR, + CALLCHAIN_MAX +}; + +enum chain_mode { + CHAIN_NONE, + CHAIN_FLAT, + CHAIN_GRAPH_ABS, + CHAIN_GRAPH_REL, + CHAIN_FOLDED, +}; + +enum chain_order { + ORDER_CALLER, + ORDER_CALLEE +}; + +struct callchain_node { + struct callchain_node *parent; + struct list_head val; + struct list_head parent_val; + struct rb_node rb_node_in; /* to insert nodes in an rbtree */ + struct rb_node rb_node; /* to sort nodes in an output tree */ + struct rb_root rb_root_in; /* input tree of children */ + struct rb_root rb_root; /* sorted output tree of children */ + unsigned int val_nr; + unsigned int count; + unsigned int children_count; + u64 hit; + u64 children_hit; +}; + +struct callchain_root { + u64 max_depth; + struct callchain_node node; +}; + +struct callchain_param; + +typedef void (*sort_chain_func_t)(struct rb_root *, struct callchain_root *, + u64, struct callchain_param *); + +enum chain_key { + CCKEY_FUNCTION, + CCKEY_ADDRESS, + CCKEY_SRCLINE +}; + +enum chain_value { + CCVAL_PERCENT, + CCVAL_PERIOD, + CCVAL_COUNT, +}; + +extern bool dwarf_callchain_users; + +struct callchain_param { + bool enabled; + bool defer; + enum perf_call_graph_mode record_mode; + u32 dump_size; + enum chain_mode mode; + u16 max_stack; + u32 print_limit; + double min_percent; + sort_chain_func_t sort; + enum chain_order order; + bool order_set; + enum chain_key key; + bool branch_callstack; + enum chain_value value; +}; + +extern struct callchain_param callchain_param; +extern struct callchain_param callchain_param_default; + +struct callchain_list { + struct list_head list; + u64 ip; + struct map_symbol ms; + const char *srcline; + u64 branch_count; + u64 from_count; + u64 cycles_count; + u64 iter_count; + u64 iter_cycles; + struct branch_type_stat *brtype_stat; + u64 predicted_count; + u64 abort_count; + struct /* for TUI */ { + bool unfolded; + bool has_children; + }; +}; + +/* + * A callchain cursor is a single linked list that + * let one feed a callchain progressively. + * It keeps persistent allocated entries to minimize + * allocations. + */ +struct callchain_cursor_node { + u64 ip; + struct map_symbol ms; + const char *srcline; + /* Indicate valid cursor node for LBR stitch */ + bool valid; + + bool branch; + struct branch_flags branch_flags; + u64 branch_from; + int nr_loop_iter; + u64 iter_cycles; + struct callchain_cursor_node *next; +}; + +struct stitch_list { + struct list_head node; + struct callchain_cursor_node cursor; +}; + +struct callchain_cursor { + u64 nr; + struct callchain_cursor_node *first; + struct callchain_cursor_node **last; + u64 pos; + struct callchain_cursor_node *curr; +}; + +static inline void callchain_init(struct callchain_root *root) +{ + INIT_LIST_HEAD(&root->node.val); + INIT_LIST_HEAD(&root->node.parent_val); + + root->node.parent = NULL; + root->node.hit = 0; + root->node.children_hit = 0; + root->node.rb_root_in = RB_ROOT; + root->max_depth = 0; +} + +static inline u64 callchain_cumul_hits(struct callchain_node *node) +{ + return node->hit + node->children_hit; +} + +static inline unsigned callchain_cumul_counts(struct callchain_node *node) +{ + return node->count + node->children_count; +} + +int callchain_register_param(struct callchain_param *param); +int callchain_append(struct callchain_root *root, + struct callchain_cursor *cursor, + u64 period); + +int callchain_merge(struct callchain_cursor *cursor, + struct callchain_root *dst, struct callchain_root *src); + +void callchain_cursor_reset(struct callchain_cursor *cursor); + +int callchain_cursor_append(struct callchain_cursor *cursor, u64 ip, + struct map_symbol *ms, + bool branch, struct branch_flags *flags, + int nr_loop_iter, u64 iter_cycles, u64 branch_from, + const char *srcline); + +/* Close a cursor writing session. Initialize for the reader */ +static inline void callchain_cursor_commit(struct callchain_cursor *cursor) +{ + if (cursor == NULL) + return; + cursor->curr = cursor->first; + cursor->pos = 0; +} + +/* Cursor reading iteration helpers */ +static inline struct callchain_cursor_node * +callchain_cursor_current(struct callchain_cursor *cursor) +{ + if (cursor == NULL || cursor->pos == cursor->nr) + return NULL; + + return cursor->curr; +} + +static inline void callchain_cursor_advance(struct callchain_cursor *cursor) +{ + cursor->curr = cursor->curr->next; + cursor->pos++; +} + +struct callchain_cursor *get_tls_callchain_cursor(void); + +int callchain_cursor__copy(struct callchain_cursor *dst, + struct callchain_cursor *src); + +int record_opts__parse_callchain(struct record_opts *record, + struct callchain_param *callchain, + const char *arg, bool unset); + +int sample__resolve_callchain(struct perf_sample *sample, + struct callchain_cursor *cursor, struct symbol **parent, + struct addr_location *al, + int max_stack); +int hist_entry__append_callchain(struct hist_entry *he, struct perf_sample *sample); +int fill_callchain_info(struct addr_location *al, struct callchain_cursor_node *node, + bool hide_unresolved); + +extern const char record_callchain_help[]; +int parse_callchain_record(const char *arg, struct callchain_param *param); +int parse_callchain_record_opt(const char *arg, struct callchain_param *param); +int parse_callchain_report_opt(const char *arg); +int parse_callchain_top_opt(const char *arg); +int perf_callchain_config(const char *var, const char *value); + +static inline void callchain_cursor_snapshot(struct callchain_cursor *dest, + struct callchain_cursor *src) +{ + *dest = *src; + + dest->first = src->curr; + dest->nr -= src->pos; +} + +#ifdef HAVE_SKIP_CALLCHAIN_IDX +int arch_skip_callchain_idx(struct thread *thread, struct ip_callchain *chain); +#else +static inline int arch_skip_callchain_idx(struct thread *thread __maybe_unused, + struct ip_callchain *chain __maybe_unused) +{ + return -1; +} +#endif + +char *callchain_list__sym_name(struct callchain_list *cl, + char *bf, size_t bfsize, bool show_dso); +char *callchain_node__scnprintf_value(struct callchain_node *node, + char *bf, size_t bfsize, u64 total); +int callchain_node__fprintf_value(struct callchain_node *node, + FILE *fp, u64 total); + +int callchain_list_counts__printf_value(struct callchain_list *clist, + FILE *fp, char *bf, int bfsize); + +void free_callchain(struct callchain_root *root); +void callchain_cursor_cleanup(struct callchain_cursor *cursor); +void decay_callchain(struct callchain_root *root); +int callchain_node__make_parent_list(struct callchain_node *node); + +int callchain_branch_counts(struct callchain_root *root, + u64 *branch_count, u64 *predicted_count, + u64 *abort_count, u64 *cycles_count); + +void callchain_param_setup(u64 sample_type, uint16_t e_machine); + +bool callchain_cnode_matched(struct callchain_node *base_cnode, + struct callchain_node *pair_cnode); + +u64 callchain_total_hits(struct hists *hists); + +s64 callchain_avg_cycles(struct callchain_node *cnode); + +typedef int (*callchain_iter_fn)(struct callchain_cursor_node *node, void *data); + +int sample__for_each_callchain_node(struct thread *thread, + struct perf_sample *sample, int max_stack, + bool symbols, callchain_iter_fn cb, void *data); + +int sample__merge_deferred_callchain(struct perf_sample *sample_orig, + struct perf_sample *sample_callchain); + +#endif /* __PERF_CALLCHAIN_H */ diff --git a/tools/perf/util/cap.c b/tools/perf/util/cap.c new file mode 100644 index 000000000..ac6d1d9a5 --- /dev/null +++ b/tools/perf/util/cap.c @@ -0,0 +1,48 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Capability utilities + */ + +#include "cap.h" +#include "debug.h" +#include +#include +#include +#include + +#define MAX_LINUX_CAPABILITY_U32S _LINUX_CAPABILITY_U32S_3 + +bool perf_cap__capable(int cap, bool *used_root) +{ + struct __user_cap_header_struct header = { + .version = _LINUX_CAPABILITY_VERSION_3, + .pid = 0, + }; + struct __user_cap_data_struct data[MAX_LINUX_CAPABILITY_U32S] = {}; + __u32 cap_val; + + *used_root = false; + while (syscall(SYS_capget, &header, &data[0]) == -1) { + /* Retry, first attempt has set the header.version correctly. */ + if (errno == EINVAL && header.version != _LINUX_CAPABILITY_VERSION_3 && + header.version == _LINUX_CAPABILITY_VERSION_1) + continue; + + pr_debug2("capget syscall failed (%m) fall back on root check\n"); + *used_root = true; + return geteuid() == 0; + } + + /* Extract the relevant capability bit. */ + if (cap >= 32) { + if (header.version == _LINUX_CAPABILITY_VERSION_3) { + cap_val = data[1].effective; + } else { + /* Capability beyond 32 is requested but only 32 are supported. */ + return false; + } + } else { + cap_val = data[0].effective; + } + return (cap_val & (1 << (cap & 0x1f))) != 0; +} diff --git a/tools/perf/util/cap.h b/tools/perf/util/cap.h new file mode 100644 index 000000000..c1b8ac033 --- /dev/null +++ b/tools/perf/util/cap.h @@ -0,0 +1,24 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CAP_H +#define __PERF_CAP_H + +#include +#include + +/* For older systems */ +#ifndef CAP_SYSLOG +#define CAP_SYSLOG 34 +#endif + +#ifndef CAP_PERFMON +#define CAP_PERFMON 38 +#endif + +#ifndef CAP_BPF +#define CAP_BPF 39 +#endif + +/* Query if a capability is supported, used_root is set if the fallback root check was used. */ +bool perf_cap__capable(int cap, bool *used_root); + +#endif /* __PERF_CAP_H */ diff --git a/tools/perf/util/capstone.c b/tools/perf/util/capstone.c new file mode 100644 index 000000000..5ad537fea --- /dev/null +++ b/tools/perf/util/capstone.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "capstone.h" + +#include +#include +#include + +#include +#include +#include +#include + +#include + +#include "addr_location.h" +#include "annotate.h" +#include "debug.h" +#include "disasm.h" +#include "dso.h" +#include "machine.h" +#include "map.h" +#include "namespaces.h" +#include "print_insn.h" +#include "symbol.h" +#include "thread.h" + +#ifdef LIBCAPSTONE_DLOPEN +static void *perf_cs_dll_handle(void) +{ + static bool dll_handle_init; + static void *dll_handle; + + if (!dll_handle_init) { + dll_handle_init = true; + dll_handle = dlopen("libcapstone.so", RTLD_LAZY); + if (!dll_handle) + pr_debug("dlopen failed for libcapstone.so\n"); + } + return dll_handle; +} +#endif + +static enum cs_err perf_cs_open(enum cs_arch arch, enum cs_mode mode, csh *handle) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_open(arch, mode, handle); +#else + static bool fn_init; + static enum cs_err (*fn)(enum cs_arch arch, enum cs_mode mode, csh *handle); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_open"); + if (!fn) + pr_debug("dlsym failed for cs_open\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(arch, mode, handle); +#endif +} + +static enum cs_err perf_cs_option(csh handle, enum cs_opt_type type, size_t value) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_option(handle, type, value); +#else + static bool fn_init; + static enum cs_err (*fn)(csh handle, enum cs_opt_type type, size_t value); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_option"); + if (!fn) + pr_debug("dlsym failed for cs_option\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(handle, type, value); +#endif +} + +static size_t perf_cs_disasm(csh handle, const uint8_t *code, size_t code_size, + uint64_t address, size_t count, struct cs_insn **insn) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_disasm(handle, code, code_size, address, count, insn); +#else + static bool fn_init; + static enum cs_err (*fn)(csh handle, const uint8_t *code, size_t code_size, + uint64_t address, size_t count, struct cs_insn **insn); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_disasm"); + if (!fn) + pr_debug("dlsym failed for cs_disasm\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(handle, code, code_size, address, count, insn); +#endif +} + +static void perf_cs_free(struct cs_insn *insn, size_t count) +{ +#ifndef LIBCAPSTONE_DLOPEN + cs_free(insn, count); +#else + static bool fn_init; + static void (*fn)(struct cs_insn *insn, size_t count); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_free"); + if (!fn) + pr_debug("dlsym failed for cs_free\n"); + fn_init = true; + } + if (!fn) + return; + fn(insn, count); +#endif +} + +static enum cs_err perf_cs_close(csh *handle) +{ +#ifndef LIBCAPSTONE_DLOPEN + return cs_close(handle); +#else + static bool fn_init; + static enum cs_err (*fn)(csh *handle); + + if (!fn_init) { + fn = dlsym(perf_cs_dll_handle(), "cs_close"); + if (!fn) + pr_debug("dlsym failed for cs_close\n"); + fn_init = true; + } + if (!fn) + return CS_ERR_HANDLE; + return fn(handle); +#endif +} + +static bool e_machine_to_capstone(uint16_t e_machine, bool is64, bool is_big_endian, + enum cs_arch *arch, enum cs_mode *mode) +{ + *mode = is_big_endian ? CS_MODE_BIG_ENDIAN : CS_MODE_LITTLE_ENDIAN; + + switch (e_machine) { + case EM_X86_64: + case EM_386: + *arch = CS_ARCH_X86; + *mode |= is64 ? CS_MODE_64 : CS_MODE_32; + return true; + case EM_AARCH64: + *arch = CS_ARCH_ARM64; + *mode |= CS_MODE_ARM; + return true; + case EM_ARM: + *arch = CS_ARCH_ARM; + *mode |= CS_MODE_ARM | CS_MODE_V8; + return true; + case EM_S390: + *arch = CS_ARCH_SYSZ; + return true; + case EM_MIPS: + *arch = CS_ARCH_MIPS; + *mode |= is64 ? CS_MODE_MIPS64 : CS_MODE_MIPS32; + return true; + case EM_PPC: + *arch = CS_ARCH_PPC; + return true; + case EM_PPC64: + *arch = CS_ARCH_PPC; + *mode |= CS_MODE_64; + return true; + case EM_SPARC: + *arch = CS_ARCH_SPARC; + return true; + case EM_SPARCV9: + *arch = CS_ARCH_SPARC; + *mode |= CS_MODE_V9; + return true; + case EM_RISCV: + *arch = CS_ARCH_RISCV; + *mode |= (is64 ? CS_MODE_RISCV64 : CS_MODE_RISCV32) | CS_MODE_RISCVC; + return true; + default: + return false; + } +} + +static int capstone_init(uint16_t e_machine, csh *cs_handle, bool is64, bool is_big_endian, + bool disassembler_style) +{ + enum cs_arch arch; + enum cs_mode mode; + + if (!e_machine_to_capstone(e_machine, is64, is_big_endian, &arch, &mode)) + return -1; + + if (perf_cs_open(arch, mode, cs_handle) != CS_ERR_OK) { + pr_warning_once("cs_open failed\n"); + return -1; + } + + if (arch == CS_ARCH_X86) { + /* + * In case of using capstone_init while symbol__disassemble + * setting CS_OPT_SYNTAX_ATT depends if disassembler_style opts + * is set via annotation args + */ + if (disassembler_style) + perf_cs_option(*cs_handle, CS_OPT_SYNTAX, CS_OPT_SYNTAX_ATT); + /* + * Resolving address operands to symbols is implemented + * on x86 by investigating instruction details. + */ + perf_cs_option(*cs_handle, CS_OPT_DETAIL, CS_OPT_ON); + } + + return 0; +} + +static size_t print_insn_x86(struct thread *thread, u8 cpumode, struct cs_insn *insn, + int print_opts, FILE *fp) +{ + struct addr_location al; + size_t printed = 0; + + if (insn->detail && insn->detail->x86.op_count == 1) { + struct cs_x86_op *op = &insn->detail->x86.operands[0]; + + addr_location__init(&al); + if (op->type == X86_OP_IMM && + thread__find_symbol(thread, cpumode, op->imm, &al)) { + printed += fprintf(fp, "%s ", insn[0].mnemonic); + printed += symbol__fprintf_symname_offs(al.sym, &al, fp); + if (print_opts & PRINT_INSN_IMM_HEX) + printed += fprintf(fp, " [%#" PRIx64 "]", op->imm); + addr_location__exit(&al); + return printed; + } + addr_location__exit(&al); + } + + printed += fprintf(fp, "%s %s", insn[0].mnemonic, insn[0].op_str); + return printed; +} + +ssize_t capstone__fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, + bool is64bit, const uint8_t *code, size_t code_size, uint64_t ip, + int *lenp, int print_opts, FILE *fp) +{ + size_t printed; + struct cs_insn *insn; + csh cs_handle; + size_t count; + bool is_big_endian = false; + uint16_t e_machine = thread__e_machine_endian(thread, machine, + /*e_flags=*/NULL, &is_big_endian); + int ret; + + /* TODO: Try to initiate capstone only once but need a proper place. */ + ret = capstone_init(e_machine, &cs_handle, is64bit, is_big_endian, + /*disassembler_style=*/true); + if (ret < 0) + return ret; + + count = perf_cs_disasm(cs_handle, code, code_size, ip, 1, &insn); + if (count > 0) { + if (e_machine == EM_X86_64 || e_machine == EM_386) + printed = print_insn_x86(thread, cpumode, &insn[0], print_opts, fp); + else + printed = fprintf(fp, "%s %s", insn[0].mnemonic, insn[0].op_str); + if (lenp) + *lenp = insn->size; + perf_cs_free(insn, count); + } else { + printed = -1; + } + + perf_cs_close(&cs_handle); + return printed; +} + +static void print_capstone_detail(struct cs_insn *insn, char *buf, size_t len, + struct annotate_args *args, u64 addr) +{ + int i; + struct map *map = args->ms->map; + struct symbol *sym; + + /* TODO: support more architectures */ + if (!arch__is_x86(args->arch)) + return; + + if (insn->detail == NULL) + return; + + for (i = 0; i < insn->detail->x86.op_count; i++) { + struct cs_x86_op *op = &insn->detail->x86.operands[i]; + u64 orig_addr; + + if (op->type != X86_OP_MEM) + continue; + + /* only print RIP-based global symbols for now */ + if (op->mem.base != X86_REG_RIP) + continue; + + /* get the target address */ + orig_addr = addr + insn->size + op->mem.disp; + addr = map__objdump_2mem(map, orig_addr); + + if (dso__kernel(map__dso(map))) { + /* + * The kernel maps can be split into sections, let's + * find the map first and the search the symbol. + */ + map = maps__find(map__kmaps(map), addr); + if (map == NULL) + continue; + } + + /* convert it to map-relative address for search */ + addr = map__map_ip(map, addr); + + sym = map__find_symbol(map, addr); + if (sym == NULL) + continue; + + if (addr == sym->start) { + scnprintf(buf, len, "\t# %"PRIx64" <%s>", + orig_addr, sym->name); + } else { + scnprintf(buf, len, "\t# %"PRIx64" <%s+%#"PRIx64">", + orig_addr, sym->name, addr - sym->start); + } + break; + } +} + +struct find_file_offset_data { + u64 ip; + u64 offset; +}; + +/* This will be called for each PHDR in an ELF binary */ +static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) +{ + struct find_file_offset_data *data = arg; + + if (start <= data->ip && data->ip < start + len) { + data->offset = pgoff + data->ip - start; + return 1; + } + return 0; +} + +int symbol__disassemble_capstone(const char *filename, struct symbol *sym, + struct annotate_args *args) +{ + struct annotation *notes = symbol__annotation(sym); + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + u64 start = map__rip_2objdump(map, sym->start); + u64 offset; + int i, count, free_count; + bool is_64bit = false; + bool needs_cs_close = false; + /* Malloc-ed buffer containing instructions read from disk. */ + u8 *code_buf = NULL; + /* Pointer to code to be disassembled. */ + const u8 *buf; + u64 buf_len; + csh handle; + struct cs_insn *insn = NULL; + char disasm_buf[512]; + struct disasm_line *dl; + bool disassembler_style = false; + uint16_t e_machine; + bool is_big_endian = false; + + if (args->options->objdump_path) + return -1; + + buf = dso__read_symbol(dso, filename, map, sym, + &code_buf, &buf_len, &is_64bit); + if (buf == NULL) + return errno; + + /* add the function address and name */ + scnprintf(disasm_buf, sizeof(disasm_buf), "%#"PRIx64" <%s>:", + start, sym->name); + + args->offset = -1; + args->line = disasm_buf; + args->line_nr = 0; + args->fileloc = NULL; + args->ms->sym = sym; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + if (!args->options->disassembler_style || + !strcmp(args->options->disassembler_style, "att")) + disassembler_style = true; + + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) + goto err; + + needs_cs_close = true; + + free_count = count = perf_cs_disasm(handle, buf, buf_len, start, buf_len, &insn); + for (i = 0, offset = 0; i < count; i++) { + int printed; + + printed = scnprintf(disasm_buf, sizeof(disasm_buf), + " %-7s %s", + insn[i].mnemonic, insn[i].op_str); + print_capstone_detail(&insn[i], disasm_buf + printed, + sizeof(disasm_buf) - printed, args, + start + offset); + + args->offset = offset; + args->line = disasm_buf; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + offset += insn[i].size; + } + + /* It failed in the middle: probably due to unknown instructions */ + if (offset != buf_len) { + struct list_head *list = ¬es->src->source; + + /* Discard all lines and fallback to objdump */ + while (!list_empty(list)) { + dl = list_first_entry(list, struct disasm_line, al.node); + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } + count = -1; + } + +out: + if (needs_cs_close) { + perf_cs_close(&handle); + if (free_count > 0) + perf_cs_free(insn, free_count); + } + free(code_buf); + return count < 0 ? count : 0; + +err: + if (needs_cs_close) { + struct disasm_line *tmp; + + /* + * It probably failed in the middle of the above loop. + * Release any resources it might add. + */ + list_for_each_entry_safe(dl, tmp, ¬es->src->source, al.node) { + list_del(&dl->al.node); + disasm_line__free(dl); + } + } + count = -1; + goto out; +} + +int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, + struct symbol *sym __maybe_unused, + struct annotate_args *args __maybe_unused) +{ + struct annotation *notes = symbol__annotation(sym); + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + struct nscookie nsc; + u64 start = map__rip_2objdump(map, sym->start); + u64 end = map__rip_2objdump(map, sym->end); + u64 len = end - start; + u64 offset; + int i, fd, count; + bool is_64bit = false; + bool needs_cs_close = false; + u8 *buf = NULL; + struct find_file_offset_data data = { + .ip = start, + }; + csh handle; + char disasm_buf[512]; + struct disasm_line *dl; + u32 *line; + bool disassembler_style = false; + uint16_t e_machine; + bool is_big_endian = false; + + if (args->options->objdump_path) + return -1; + + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + fd = open(filename, O_RDONLY); + nsinfo__mountns_exit(&nsc); + if (fd < 0) + return -1; + + if (file__read_maps(fd, /*exe=*/true, find_file_offset, &data, + &is_64bit) == 0) + goto err; + + if (!args->options->disassembler_style || + !strcmp(args->options->disassembler_style, "att")) + disassembler_style = true; + + e_machine = thread__e_machine_endian(args->ms->thread, + /*machine=*/NULL, + /*e_flags=*/NULL, &is_big_endian); + if (capstone_init(e_machine, &handle, is_64bit, is_big_endian, disassembler_style) < 0) + goto err; + + needs_cs_close = true; + + buf = malloc(len); + if (buf == NULL) + goto err; + + count = pread(fd, buf, len, data.offset); + close(fd); + fd = -1; + + if ((u64)count != len) + goto err; + + line = (u32 *)buf; + + /* add the function address and name */ + scnprintf(disasm_buf, sizeof(disasm_buf), "%#"PRIx64" <%s>:", + start, sym->name); + + args->offset = -1; + args->line = disasm_buf; + args->line_nr = 0; + args->fileloc = NULL; + args->ms->sym = sym; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + /* + * TODO: enable disassm for powerpc + * count = cs_disasm(handle, buf, len, start, len, &insn); + * + * For now, only binary code is saved in disassembled line + * to be used in "type" and "typeoff" sort keys. Each raw code + * is 32 bit instruction. So use "len/4" to get the number of + * entries. + */ + count = len/4; + + for (i = 0, offset = 0; i < count; i++) { + args->offset = offset; + sprintf(args->line, "%x", line[i]); + + dl = disasm_line__new(args); + if (dl == NULL) + break; + + annotation_line__add(&dl->al, ¬es->src->source); + + offset += 4; + } + + /* It failed in the middle */ + if (offset != len) { + struct list_head *list = ¬es->src->source; + + /* Discard all lines and fallback to objdump */ + while (!list_empty(list)) { + dl = list_first_entry(list, struct disasm_line, al.node); + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } + count = -1; + } + +out: + if (needs_cs_close) + perf_cs_close(&handle); + free(buf); + return count < 0 ? count : 0; + +err: + if (fd >= 0) + close(fd); + count = -1; + goto out; +} diff --git a/tools/perf/util/capstone.h b/tools/perf/util/capstone.h new file mode 100644 index 000000000..7c0baaa01 --- /dev/null +++ b/tools/perf/util/capstone.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CAPSTONE_H +#define __PERF_CAPSTONE_H + +#include +#include +#include +#include +#include +#include + +struct annotate_args; +struct machine; +struct symbol; +struct thread; + +#ifdef HAVE_LIBCAPSTONE_SUPPORT +ssize_t capstone__fprintf_insn_asm(struct machine *machine, struct thread *thread, u8 cpumode, + bool is64bit, const uint8_t *code, size_t code_size, + uint64_t ip, int *lenp, int print_opts, FILE *fp); +int symbol__disassemble_capstone(const char *filename, struct symbol *sym, + struct annotate_args *args); +int symbol__disassemble_capstone_powerpc(const char *filename, struct symbol *sym, + struct annotate_args *args); + +#else /* !HAVE_LIBCAPSTONE_SUPPORT */ +static inline ssize_t capstone__fprintf_insn_asm(struct machine *machine __maybe_unused, + struct thread *thread __maybe_unused, + u8 cpumode __maybe_unused, + bool is64bit __maybe_unused, + const uint8_t *code __maybe_unused, + size_t code_size __maybe_unused, + uint64_t ip __maybe_unused, + int *lenp __maybe_unused, + int print_opts __maybe_unused, + FILE *fp __maybe_unused) +{ + return -1; +} + +static inline int symbol__disassemble_capstone(const char *filename __maybe_unused, + struct symbol *sym __maybe_unused, + struct annotate_args *args __maybe_unused) +{ + return -1; +} + +static inline int symbol__disassemble_capstone_powerpc(const char *filename __maybe_unused, + struct symbol *sym __maybe_unused, + struct annotate_args *args __maybe_unused) +{ + return -1; +} + +#endif /* HAVE_LIBCAPSTONE_SUPPORT */ + +#endif /* __PERF_CAPSTONE_H */ diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c new file mode 100644 index 000000000..1b5664d14 --- /dev/null +++ b/tools/perf/util/cgroup.c @@ -0,0 +1,646 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "evsel.h" +#include "cgroup.h" +#include "evlist.h" +#include "rblist.h" +#include "metricgroup.h" +#include "stat.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +int nr_cgroups; +bool cgrp_event_expanded; + +/* used to match cgroup name with patterns */ +struct cgroup_name { + struct list_head list; + bool used; + char name[]; +}; +static LIST_HEAD(cgroup_list); + +static int open_cgroup(const char *name) +{ + char path[PATH_MAX + 1]; + char mnt[PATH_MAX + 1]; + int fd; + + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, "perf_event")) + return -1; + + scnprintf(path, PATH_MAX, "%s/%s", mnt, name); + + fd = open(path, O_RDONLY); + if (fd == -1) + fprintf(stderr, "no access to cgroup %s\n", path); + + return fd; +} + +#ifdef HAVE_FILE_HANDLE +static u64 __read_cgroup_id(const char *path) +{ + struct { + struct file_handle fh; + uint64_t cgroup_id; + } handle; + int mount_id; + + handle.fh.handle_bytes = sizeof(handle.cgroup_id); + if (name_to_handle_at(AT_FDCWD, path, &handle.fh, &mount_id, 0) < 0) + return -1ULL; + + return handle.cgroup_id; +} + +int read_cgroup_id(struct cgroup *cgrp) +{ + char path[PATH_MAX + 1]; + char mnt[PATH_MAX + 1]; + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, "perf_event")) + return -1; + + scnprintf(path, PATH_MAX, "%s/%s", mnt, cgrp->name); + + cgrp->id = __read_cgroup_id(path); + return 0; +} +#else +static inline u64 __read_cgroup_id(const char *path __maybe_unused) { return -1ULL; } +#endif /* HAVE_FILE_HANDLE */ + +#ifndef CGROUP2_SUPER_MAGIC +#define CGROUP2_SUPER_MAGIC 0x63677270 +#endif + +int cgroup_is_v2(const char *subsys) +{ + char mnt[PATH_MAX + 1]; + struct statfs stbuf; + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, subsys)) + return -1; + + if (statfs(mnt, &stbuf) < 0) + return -1; + + return (stbuf.f_type == CGROUP2_SUPER_MAGIC); +} + +static struct cgroup *evlist__find_cgroup(struct evlist *evlist, const char *str) +{ + struct evsel *counter; + /* + * check if cgrp is already defined, if so we reuse it + */ + evlist__for_each_entry(evlist, counter) { + if (!counter->cgrp) + continue; + if (!strcmp(counter->cgrp->name, str)) + return cgroup__get(counter->cgrp); + } + + return NULL; +} + +struct cgroup *cgroup__new(const char *name, bool do_open) +{ + struct cgroup *cgroup = zalloc(sizeof(*cgroup)); + + if (cgroup != NULL) { + refcount_set(&cgroup->refcnt, 1); + + cgroup->name = strdup(name); + if (!cgroup->name) + goto out_err; + + if (do_open) { + cgroup->fd = open_cgroup(name); + if (cgroup->fd == -1) + goto out_free_name; + } else { + cgroup->fd = -1; + } + } + + return cgroup; + +out_free_name: + zfree(&cgroup->name); +out_err: + free(cgroup); + return NULL; +} + +struct cgroup *evlist__findnew_cgroup(struct evlist *evlist, const char *name) +{ + struct cgroup *cgroup = evlist__find_cgroup(evlist, name); + + return cgroup ?: cgroup__new(name, true); +} + +static int add_cgroup(struct evlist *evlist, const char *str) +{ + struct evsel *counter; + struct cgroup *cgrp = evlist__findnew_cgroup(evlist, str); + int n; + + if (!cgrp) + return -1; + /* + * find corresponding event + * if add cgroup N, then need to find event N + */ + n = 0; + evlist__for_each_entry(evlist, counter) { + if (n == nr_cgroups) + goto found; + n++; + } + + cgroup__put(cgrp); + return -1; +found: + counter->cgrp = cgrp; + return 0; +} + +static void cgroup__delete(struct cgroup *cgroup) +{ + if (cgroup->fd >= 0) + close(cgroup->fd); + zfree(&cgroup->name); + free(cgroup); +} + +void cgroup__put(struct cgroup *cgrp) +{ + if (cgrp && refcount_dec_and_test(&cgrp->refcnt)) { + cgroup__delete(cgrp); + } +} + +struct cgroup *cgroup__get(struct cgroup *cgroup) +{ + if (cgroup) + refcount_inc(&cgroup->refcnt); + return cgroup; +} + +static void evsel__set_default_cgroup(struct evsel *evsel, struct cgroup *cgroup) +{ + if (evsel->cgrp == NULL) + evsel->cgrp = cgroup__get(cgroup); +} + +void evlist__set_default_cgroup(struct evlist *evlist, struct cgroup *cgroup) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) + evsel__set_default_cgroup(evsel, cgroup); +} + +/* helper function for ftw() in match_cgroups and list_cgroups */ +static int add_cgroup_name(const char *fpath, const struct stat *sb __maybe_unused, + int typeflag, struct FTW *ftwbuf __maybe_unused) +{ + struct cgroup_name *cn; + + if (typeflag != FTW_D) + return 0; + + cn = malloc(sizeof(*cn) + strlen(fpath) + 1); + if (cn == NULL) + return -1; + + cn->used = false; + strcpy(cn->name, fpath); + + list_add_tail(&cn->list, &cgroup_list); + return 0; +} + +static int check_and_add_cgroup_name(const char *fpath) +{ + struct cgroup_name *cn; + + list_for_each_entry(cn, &cgroup_list, list) { + if (!strcmp(cn->name, fpath)) + return 0; + } + + /* pretend if it's added by ftw() */ + return add_cgroup_name(fpath, NULL, FTW_D, NULL); +} + +static void release_cgroup_list(void) +{ + struct cgroup_name *cn; + + while (!list_empty(&cgroup_list)) { + cn = list_first_entry(&cgroup_list, struct cgroup_name, list); + list_del(&cn->list); + free(cn); + } +} + +/* collect given cgroups only */ +static int list_cgroups(const char *str) +{ + const char *p, *e, *eos = str + strlen(str); + struct cgroup_name *cn; + char *s; + + /* use given name as is when no regex is given */ + for (;;) { + p = strchr(str, ','); + e = p ? p : eos; + + if (e - str) { + int ret; + + s = strndup(str, e - str); + if (!s) + return -1; + + ret = check_and_add_cgroup_name(s); + free(s); + if (ret < 0) + return -1; + } else { + if (check_and_add_cgroup_name("/") < 0) + return -1; + } + + if (!p) + break; + str = p+1; + } + + /* these groups will be used */ + list_for_each_entry(cn, &cgroup_list, list) + cn->used = true; + + return 0; +} + +/* collect all cgroups first and then match with the pattern */ +static int match_cgroups(const char *str) +{ + char mnt[PATH_MAX]; + const char *p, *e, *eos = str + strlen(str); + struct cgroup_name *cn; + regex_t reg; + int prefix_len; + char *s; + + if (cgroupfs_find_mountpoint(mnt, sizeof(mnt), "perf_event")) + return -1; + + /* cgroup_name will have a full path, skip the root directory */ + prefix_len = strlen(mnt); + + /* collect all cgroups in the cgroup_list */ + if (nftw(mnt, add_cgroup_name, 20, 0) < 0) + return -1; + + for (;;) { + p = strchr(str, ','); + e = p ? p : eos; + + /* allow empty cgroups, i.e., skip */ + if (e - str) { + /* termination added */ + s = strndup(str, e - str); + if (!s) + return -1; + if (regcomp(®, s, REG_NOSUB)) { + free(s); + return -1; + } + + /* check cgroup name with the pattern */ + list_for_each_entry(cn, &cgroup_list, list) { + char *name = cn->name + prefix_len; + + if (name[0] == '/' && name[1]) + name++; + if (!regexec(®, name, 0, NULL, 0)) + cn->used = true; + } + regfree(®); + free(s); + } else { + /* first entry to root cgroup */ + cn = list_first_entry(&cgroup_list, struct cgroup_name, + list); + cn->used = true; + } + + if (!p) + break; + str = p+1; + } + return prefix_len; +} + +int parse_cgroups(const struct option *opt, const char *str, + int unset __maybe_unused) +{ + struct evlist *evlist = *(struct evlist **)opt->value; + struct evsel *counter; + struct cgroup *cgrp = NULL; + const char *p, *e, *eos = str + strlen(str); + char *s; + int ret, i; + + if (list_empty(&evlist->core.entries)) { + fprintf(stderr, "must define events before cgroups\n"); + return -1; + } + + for (;;) { + p = strchr(str, ','); + e = p ? p : eos; + + /* allow empty cgroups, i.e., skip */ + if (e - str) { + /* termination added */ + s = strndup(str, e - str); + if (!s) + return -1; + ret = add_cgroup(evlist, s); + free(s); + if (ret) + return -1; + } + /* nr_cgroups is increased een for empty cgroups */ + nr_cgroups++; + if (!p) + break; + str = p+1; + } + /* for the case one cgroup combine to multiple events */ + i = 0; + if (nr_cgroups == 1) { + evlist__for_each_entry(evlist, counter) { + if (i == 0) + cgrp = counter->cgrp; + else { + counter->cgrp = cgrp; + refcount_inc(&cgrp->refcnt); + } + i++; + } + } + return 0; +} + +static bool has_pattern_string(const char *str) +{ + return !!strpbrk(str, "{}[]()|*+?^$"); +} + +int evlist__expand_cgroup(struct evlist *evlist, const char *str, bool open_cgroup) +{ + struct evlist *orig_list, *tmp_list; + struct rblist orig_metric_events; + struct cgroup *cgrp = NULL; + struct cgroup_name *cn; + int ret = -1; + int prefix_len; + + if (evlist->core.nr_entries == 0) { + fprintf(stderr, "must define events before cgroups\n"); + return -EINVAL; + } + + orig_list = evlist__new(); + tmp_list = evlist__new(); + if (orig_list == NULL || tmp_list == NULL) { + fprintf(stderr, "memory allocation failed\n"); + return -ENOMEM; + } + + /* save original events and init evlist */ + evlist__splice_list_tail(orig_list, &evlist->core.entries); + evlist->core.nr_entries = 0; + + orig_metric_events = evlist->metric_events; + metricgroup__rblist_init(&evlist->metric_events); + + if (has_pattern_string(str)) + prefix_len = match_cgroups(str); + else + prefix_len = list_cgroups(str); + + if (prefix_len < 0) + goto out_err; + + list_for_each_entry(cn, &cgroup_list, list) { + struct evsel *pos; + char *name; + + if (!cn->used) + continue; + + /* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; + if (name[0] == '/' && name[1]) + name++; + + /* the cgroup can go away in the meantime */ + cgrp = cgroup__new(name, open_cgroup); + if (cgrp == NULL) + continue; + + /* copy the list and set to the new cgroup. */ + evlist__for_each_entry(orig_list, pos) { + struct evsel *evsel = evsel__clone(/*dest=*/NULL, pos); + + if (evsel == NULL) + goto out_err; + + /* stash the copy during the copying. */ + pos->priv = evsel; + cgroup__put(evsel->cgrp); + evsel->cgrp = cgroup__get(cgrp); + + evlist__add(tmp_list, evsel); + } + /* update leader information using stashed pointer to copy. */ + evlist__for_each_entry(orig_list, pos) { + struct evsel *evsel = pos->priv; + + if (evsel__leader(pos)) + evsel__set_leader(evsel, evsel__leader(pos)->priv); + + if (pos->metric_leader) + evsel->metric_leader = pos->metric_leader->priv; + + if (pos->first_wildcard_match) + evsel->first_wildcard_match = pos->first_wildcard_match->priv; + } + /* the stashed copy is no longer used. */ + evlist__for_each_entry(orig_list, pos) + pos->priv = NULL; + + /* cgroup__new() has a refcount, release it here */ + cgroup__put(cgrp); + nr_cgroups++; + + if (metricgroup__copy_metric_events(tmp_list, cgrp, + &evlist->metric_events, + &orig_metric_events) < 0) + goto out_err; + + evlist__splice_list_tail(evlist, &tmp_list->core.entries); + tmp_list->core.nr_entries = 0; + } + + if (list_empty(&evlist->core.entries)) { + fprintf(stderr, "no cgroup matched: %s\n", str); + goto out_err; + } + + ret = 0; + cgrp_event_expanded = true; + +out_err: + evlist__delete(orig_list); + evlist__delete(tmp_list); + metricgroup__rblist_exit(&orig_metric_events); + release_cgroup_list(); + + return ret; +} + +static struct cgroup *__cgroup__findnew(struct rb_root *root, uint64_t id, + bool create, const char *path) +{ + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct cgroup *cgrp; + + while (*p != NULL) { + parent = *p; + cgrp = rb_entry(parent, struct cgroup, node); + + if (cgrp->id == id) + return cgrp; + + if (cgrp->id < id) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + if (!create) + return NULL; + + cgrp = malloc(sizeof(*cgrp)); + if (cgrp == NULL) + return NULL; + + cgrp->name = strdup(path); + if (cgrp->name == NULL) { + free(cgrp); + return NULL; + } + + cgrp->fd = -1; + cgrp->id = id; + refcount_set(&cgrp->refcnt, 1); + + rb_link_node(&cgrp->node, parent, p); + rb_insert_color(&cgrp->node, root); + + return cgrp; +} + +struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id, + const char *path) +{ + struct cgroup *cgrp; + + down_write(&env->cgroups.lock); + cgrp = __cgroup__findnew(&env->cgroups.tree, id, true, path); + up_write(&env->cgroups.lock); + return cgrp; +} + +struct cgroup *__cgroup__find(struct rb_root *root, uint64_t id) +{ + return __cgroup__findnew(root, id, /*create=*/false, /*path=*/NULL); +} + +struct cgroup *cgroup__find(struct perf_env *env, uint64_t id) +{ + struct cgroup *cgrp; + + down_read(&env->cgroups.lock); + cgrp = __cgroup__findnew(&env->cgroups.tree, id, false, NULL); + up_read(&env->cgroups.lock); + return cgrp; +} + +void perf_env__purge_cgroups(struct perf_env *env) +{ + struct rb_node *node; + struct cgroup *cgrp; + + down_write(&env->cgroups.lock); + while (!RB_EMPTY_ROOT(&env->cgroups.tree)) { + node = rb_first(&env->cgroups.tree); + cgrp = rb_entry(node, struct cgroup, node); + + rb_erase(node, &env->cgroups.tree); + cgroup__put(cgrp); + } + up_write(&env->cgroups.lock); +} + +void read_all_cgroups(struct rb_root *root) +{ + char mnt[PATH_MAX]; + struct cgroup_name *cn; + int prefix_len; + + if (cgroupfs_find_mountpoint(mnt, sizeof(mnt), "perf_event")) + return; + + /* cgroup_name will have a full path, skip the root directory */ + prefix_len = strlen(mnt); + + /* collect all cgroups in the cgroup_list */ + if (nftw(mnt, add_cgroup_name, 20, 0) < 0) + return; + + list_for_each_entry(cn, &cgroup_list, list) { + const char *name; + u64 cgrp_id; + + /* cgroup_name might have a full path, skip the prefix */ + name = cn->name + prefix_len; + if (name[0] == '\0') + name = "/"; + + cgrp_id = __read_cgroup_id(cn->name); + __cgroup__findnew(root, cgrp_id, /*create=*/true, name); + } + + release_cgroup_list(); +} diff --git a/tools/perf/util/cgroup.h b/tools/perf/util/cgroup.h new file mode 100644 index 000000000..7b1bda228 --- /dev/null +++ b/tools/perf/util/cgroup.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __CGROUP_H__ +#define __CGROUP_H__ + +#include +#include +#include +#include "util/env.h" + +struct option; + +struct cgroup { + struct rb_node node; + u64 id; + char *name; + int fd; + refcount_t refcnt; +}; + +extern int nr_cgroups; /* number of explicit cgroups defined */ +extern bool cgrp_event_expanded; + +struct cgroup *cgroup__get(struct cgroup *cgroup); +void cgroup__put(struct cgroup *cgroup); + +struct evlist; +struct rblist; + +struct cgroup *cgroup__new(const char *name, bool do_open); +struct cgroup *evlist__findnew_cgroup(struct evlist *evlist, const char *name); +int evlist__expand_cgroup(struct evlist *evlist, const char *cgroups, bool open_cgroup); + +void evlist__set_default_cgroup(struct evlist *evlist, struct cgroup *cgroup); + +int parse_cgroups(const struct option *opt, const char *str, int unset); + +struct cgroup *cgroup__findnew(struct perf_env *env, uint64_t id, + const char *path); +struct cgroup *cgroup__find(struct perf_env *env, uint64_t id); +struct cgroup *__cgroup__find(struct rb_root *root, uint64_t id); + +void perf_env__purge_cgroups(struct perf_env *env); + +#ifdef HAVE_FILE_HANDLE +int read_cgroup_id(struct cgroup *cgrp); +#else +static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) +{ + return -1; +} +#endif /* HAVE_FILE_HANDLE */ + +/* read all cgroups in the system and save them in the rbtree */ +void read_all_cgroups(struct rb_root *root); + +int cgroup_is_v2(const char *subsys); + +#endif /* __CGROUP_H__ */ diff --git a/tools/perf/util/clockid.c b/tools/perf/util/clockid.c new file mode 100644 index 000000000..74365a5d9 --- /dev/null +++ b/tools/perf/util/clockid.c @@ -0,0 +1,119 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include +#include +#include +#include +#include +#include "debug.h" +#include "clockid.h" +#include "record.h" + +struct clockid_map { + const char *name; + int clockid; +}; + +#define CLOCKID_MAP(n, c) \ + { .name = n, .clockid = (c), } + +#define CLOCKID_END { .name = NULL, } + + +/* + * Add the missing ones, we need to build on many distros... + */ +#ifndef CLOCK_MONOTONIC_RAW +#define CLOCK_MONOTONIC_RAW 4 +#endif +#ifndef CLOCK_BOOTTIME +#define CLOCK_BOOTTIME 7 +#endif +#ifndef CLOCK_TAI +#define CLOCK_TAI 11 +#endif + +static const struct clockid_map clockids[] = { + /* available for all events, NMI safe */ + CLOCKID_MAP("monotonic", CLOCK_MONOTONIC), + CLOCKID_MAP("monotonic_raw", CLOCK_MONOTONIC_RAW), + + /* available for some events */ + CLOCKID_MAP("realtime", CLOCK_REALTIME), + CLOCKID_MAP("boottime", CLOCK_BOOTTIME), + CLOCKID_MAP("tai", CLOCK_TAI), + + /* available for the lazy */ + CLOCKID_MAP("mono", CLOCK_MONOTONIC), + CLOCKID_MAP("raw", CLOCK_MONOTONIC_RAW), + CLOCKID_MAP("real", CLOCK_REALTIME), + CLOCKID_MAP("boot", CLOCK_BOOTTIME), + + CLOCKID_END, +}; + +static int get_clockid_res(clockid_t clk_id, u64 *res_ns) +{ + struct timespec res; + + *res_ns = 0; + if (!clock_getres(clk_id, &res)) + *res_ns = res.tv_nsec + res.tv_sec * NSEC_PER_SEC; + else + pr_warning("WARNING: Failed to determine specified clock resolution.\n"); + + return 0; +} + +int parse_clockid(const struct option *opt, const char *str, int unset) +{ + struct record_opts *opts = (struct record_opts *)opt->value; + const struct clockid_map *cm; + const char *ostr = str; + + if (unset) { + opts->use_clockid = 0; + return 0; + } + + /* no arg passed */ + if (!str) + return 0; + + /* no setting it twice */ + if (opts->use_clockid) + return -1; + + opts->use_clockid = true; + + /* if its a number, we're done */ + if (sscanf(str, "%d", &opts->clockid) == 1) + return get_clockid_res(opts->clockid, &opts->clockid_res_ns); + + /* allow a "CLOCK_" prefix to the name */ + if (!strncasecmp(str, "CLOCK_", 6)) + str += 6; + + for (cm = clockids; cm->name; cm++) { + if (!strcasecmp(str, cm->name)) { + opts->clockid = cm->clockid; + return get_clockid_res(opts->clockid, + &opts->clockid_res_ns); + } + } + + opts->use_clockid = false; + ui__warning("unknown clockid %s, check man page\n", ostr); + return -1; +} + +const char *clockid_name(clockid_t clk_id) +{ + const struct clockid_map *cm; + + for (cm = clockids; cm->name; cm++) { + if (cm->clockid == clk_id) + return cm->name; + } + return "(not found)"; +} diff --git a/tools/perf/util/clockid.h b/tools/perf/util/clockid.h new file mode 100644 index 000000000..33dbd8673 --- /dev/null +++ b/tools/perf/util/clockid.h @@ -0,0 +1,12 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CLOCKID_H +#define __PERF_CLOCKID_H + +#include + +struct option; +int parse_clockid(const struct option *opt, const char *str, int unset); + +const char *clockid_name(clockid_t clk_id); + +#endif diff --git a/tools/perf/util/cloexec.c b/tools/perf/util/cloexec.c new file mode 100644 index 000000000..8830604c3 --- /dev/null +++ b/tools/perf/util/cloexec.c @@ -0,0 +1,92 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "util.h" // for sched_getcpu() +#include "../perf-sys.h" +#include "cloexec.h" +#include "event.h" +#include "asm/bug.h" +#include "debug.h" +#include +#include +#include + +static unsigned long flag = PERF_FLAG_FD_CLOEXEC; + +static int perf_flag_probe(void) +{ + /* use 'safest' configuration as used in evsel__fallback() */ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CPU_CLOCK, + .exclude_kernel = 1, + }; + int fd; + int err; + int cpu; + pid_t pid = -1; + char sbuf[STRERR_BUFSIZE]; + + cpu = sched_getcpu(); + if (cpu < 0) + cpu = 0; + + /* + * Using -1 for the pid is a workaround to avoid gratuitous jump label + * changes. + */ + while (1) { + /* check cloexec flag */ + fd = sys_perf_event_open(&attr, pid, cpu, -1, + PERF_FLAG_FD_CLOEXEC); + if (fd < 0 && pid == -1 && errno == EACCES) { + pid = 0; + continue; + } + break; + } + err = errno; + + if (fd >= 0) { + close(fd); + return 1; + } + + WARN_ONCE(err != EINVAL && err != EBUSY && err != EACCES, + "perf_event_open(..., PERF_FLAG_FD_CLOEXEC) failed with unexpected error %d (%s)\n", + err, str_error_r(err, sbuf, sizeof(sbuf))); + + /* not supported, confirm error related to PERF_FLAG_FD_CLOEXEC */ + while (1) { + fd = sys_perf_event_open(&attr, pid, cpu, -1, 0); + if (fd < 0 && pid == -1 && errno == EACCES) { + pid = 0; + continue; + } + break; + } + err = errno; + + if (fd >= 0) + close(fd); + + if (WARN_ONCE(fd < 0 && err != EBUSY && err != EACCES, + "perf_event_open(..., 0) failed unexpectedly with error %d (%s)\n", + err, str_error_r(err, sbuf, sizeof(sbuf)))) + return -1; + + return 0; +} + +unsigned long perf_event_open_cloexec_flag(void) +{ + static bool probed; + + if (!probed) { + if (perf_flag_probe() <= 0) + flag = 0; + probed = true; + } + + return flag; +} diff --git a/tools/perf/util/cloexec.h b/tools/perf/util/cloexec.h new file mode 100644 index 000000000..78216b101 --- /dev/null +++ b/tools/perf/util/cloexec.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CLOEXEC_H +#define __PERF_CLOEXEC_H + +unsigned long perf_event_open_cloexec_flag(void); + +#endif /* __PERF_CLOEXEC_H */ diff --git a/tools/perf/util/color.c b/tools/perf/util/color.c new file mode 100644 index 000000000..e51f0a676 --- /dev/null +++ b/tools/perf/util/color.c @@ -0,0 +1,156 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "color.h" +#include +#include + +int perf_use_color_default = -1; + +static int __color_vsnprintf(char *bf, size_t size, const char *color, + const char *fmt, va_list args, const char *trail) +{ + int r = 0; + + /* + * Auto-detect: + */ + if (perf_use_color_default < 0) { + if (isatty(1) || pager_in_use()) + perf_use_color_default = 1; + else + perf_use_color_default = 0; + } + + if (perf_use_color_default && *color) + r += scnprintf(bf, size, "%s", color); + r += vscnprintf(bf + r, size - r, fmt, args); + if (perf_use_color_default && *color) + r += scnprintf(bf + r, size - r, "%s", PERF_COLOR_RESET); + if (trail) + r += scnprintf(bf + r, size - r, "%s", trail); + return r; +} + +/* Colors are not included in return value */ +static int __color_vfprintf(FILE *fp, const char *color, const char *fmt, + va_list args) +{ + int r = 0; + + /* + * Auto-detect: + */ + if (perf_use_color_default < 0) { + if (isatty(fileno(fp)) || pager_in_use()) + perf_use_color_default = 1; + else + perf_use_color_default = 0; + } + + if (perf_use_color_default && *color) + fprintf(fp, "%s", color); + r += vfprintf(fp, fmt, args); + if (perf_use_color_default && *color) + fprintf(fp, "%s", PERF_COLOR_RESET); + return r; +} + +int color_vsnprintf(char *bf, size_t size, const char *color, + const char *fmt, va_list args) +{ + return __color_vsnprintf(bf, size, color, fmt, args, NULL); +} + +int color_vfprintf(FILE *fp, const char *color, const char *fmt, va_list args) +{ + return __color_vfprintf(fp, color, fmt, args); +} + +int color_snprintf(char *bf, size_t size, const char *color, + const char *fmt, ...) +{ + va_list args; + int r; + + va_start(args, fmt); + r = color_vsnprintf(bf, size, color, fmt, args); + va_end(args); + return r; +} + +int color_fprintf(FILE *fp, const char *color, const char *fmt, ...) +{ + va_list args; + int r; + + va_start(args, fmt); + r = color_vfprintf(fp, color, fmt, args); + va_end(args); + return r; +} + +const char *get_percent_color(double percent) +{ + const char *color = PERF_COLOR_NORMAL; + + /* + * We color high-overhead entries in red, mid-overhead + * entries in green - and keep the low overhead places + * normal: + */ + if (fabs(percent) >= MIN_RED) + color = PERF_COLOR_RED; + else { + if (fabs(percent) > MIN_GREEN) + color = PERF_COLOR_GREEN; + } + return color; +} + +int percent_color_fprintf(FILE *fp, const char *fmt, double percent) +{ + int r; + const char *color; + + color = get_percent_color(percent); + r = color_fprintf(fp, color, fmt, percent); + + return r; +} + +int value_color_snprintf(char *bf, size_t size, const char *fmt, double value) +{ + const char *color = get_percent_color(value); + return color_snprintf(bf, size, color, fmt, value); +} + +int percent_color_snprintf(char *bf, size_t size, const char *fmt, ...) +{ + va_list args; + double percent; + + va_start(args, fmt); + percent = va_arg(args, double); + va_end(args); + return value_color_snprintf(bf, size, fmt, percent); +} + +int percent_color_len_snprintf(char *bf, size_t size, const char *fmt, ...) +{ + va_list args; + int len; + double percent; + const char *color; + + va_start(args, fmt); + len = va_arg(args, int); + percent = va_arg(args, double); + va_end(args); + + color = get_percent_color(percent); + return color_snprintf(bf, size, color, fmt, len, percent); +} diff --git a/tools/perf/util/color.h b/tools/perf/util/color.h new file mode 100644 index 000000000..0319546de --- /dev/null +++ b/tools/perf/util/color.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_COLOR_H +#define __PERF_COLOR_H + +#include +#include +#include + +/* "\033[1;38;5;2xx;48;5;2xxm\0" is 23 bytes */ +#define COLOR_MAXLEN 24 + +#define PERF_COLOR_NORMAL "" +#define PERF_COLOR_RESET "\033[m" +#define PERF_COLOR_BOLD "\033[1m" +#define PERF_COLOR_RED "\033[31m" +#define PERF_COLOR_GREEN "\033[32m" +#define PERF_COLOR_YELLOW "\033[33m" +#define PERF_COLOR_BLUE "\033[34m" +#define PERF_COLOR_MAGENTA "\033[35m" +#define PERF_COLOR_CYAN "\033[36m" +#define PERF_COLOR_BG_RED "\033[41m" + +#define MIN_GREEN 0.5 +#define MIN_RED 5.0 + +#define PERF_COLOR_DELETE_LINE "\033[A\33[2K\r" +/* + * This variable stores the value of color.ui + */ +extern int perf_use_color_default; + + +int perf_config_colorbool(const char *var, const char *value, int stdout_is_tty); +int color_vsnprintf(char *bf, size_t size, const char *color, + const char *fmt, va_list args); +int color_vfprintf(FILE *fp, const char *color, const char *fmt, va_list args); +int color_fprintf(FILE *fp, const char *color, const char *fmt, ...) __printf(3, 4); +int color_snprintf(char *bf, size_t size, const char *color, const char *fmt, ...) __printf(4, 5); +int value_color_snprintf(char *bf, size_t size, const char *fmt, double value); +int percent_color_snprintf(char *bf, size_t size, const char *fmt, ...) __printf(3, 4); +int percent_color_len_snprintf(char *bf, size_t size, const char *fmt, ...) __printf(3, 4); +int percent_color_fprintf(FILE *fp, const char *fmt, double percent); +const char *get_percent_color(double percent); + +#endif /* __PERF_COLOR_H */ diff --git a/tools/perf/util/color_config.c b/tools/perf/util/color_config.c new file mode 100644 index 000000000..301031ddc --- /dev/null +++ b/tools/perf/util/color_config.c @@ -0,0 +1,37 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "config.h" +#include +#include +#include "color.h" +#include +#include + +int perf_config_colorbool(const char *var, const char *value, int stdout_is_tty) +{ + if (value) { + if (!strcasecmp(value, "never")) + return 0; + if (!strcasecmp(value, "always")) + return 1; + if (!strcasecmp(value, "auto")) + goto auto_color; + } + + /* Missing or explicit false to turn off colorization */ + if (!perf_config_bool(var, value)) + return 0; + + /* any normal truth value defaults to 'auto' */ + auto_color: + if (stdout_is_tty < 0) + stdout_is_tty = isatty(1); + if (stdout_is_tty || pager_in_use()) { + char *term = getenv("TERM"); + if (term && strcmp(term, "dumb")) + return 1; + } + return 0; +} diff --git a/tools/perf/util/comm.c b/tools/perf/util/comm.c new file mode 100644 index 000000000..9880247a2 --- /dev/null +++ b/tools/perf/util/comm.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "comm.h" +#include +#include +#include +#include +#include +#include // reallocarray + +#include "rwsem.h" + +DECLARE_RC_STRUCT(comm_str) { + refcount_t refcnt; + char str[]; +}; + +static struct comm_strs { + struct rw_semaphore lock; + struct comm_str **strs; + int num_strs; + int capacity; +} _comm_strs; + +static void comm_strs__remove_if_last(struct comm_str *cs); + +static void comm_strs__init(void) + NO_THREAD_SAFETY_ANALYSIS /* Inherently single threaded due to pthread_once. */ +{ + init_rwsem(&_comm_strs.lock); + _comm_strs.capacity = 16; + _comm_strs.num_strs = 0; + _comm_strs.strs = calloc(16, sizeof(*_comm_strs.strs)); +} + +static struct comm_strs *comm_strs__get(void) +{ + static pthread_once_t comm_strs_type_once = PTHREAD_ONCE_INIT; + + pthread_once(&comm_strs_type_once, comm_strs__init); + + return &_comm_strs; +} + +static refcount_t *comm_str__refcnt(struct comm_str *cs) +{ + return &RC_CHK_ACCESS(cs)->refcnt; +} + +static const char *comm_str__str(const struct comm_str *cs) +{ + return &RC_CHK_ACCESS(cs)->str[0]; +} + +static struct comm_str *comm_str__get(struct comm_str *cs) +{ + struct comm_str *result; + + if (RC_CHK_GET(result, cs)) + refcount_inc_not_zero(comm_str__refcnt(cs)); + + return result; +} + +static void comm_str__put(struct comm_str *cs) +{ + if (!cs) + return; + + if (refcount_dec_and_test(comm_str__refcnt(cs))) { + RC_CHK_FREE(cs); + } else { + if (refcount_read(comm_str__refcnt(cs)) == 1) + comm_strs__remove_if_last(cs); + + RC_CHK_PUT(cs); + } +} + +static struct comm_str *comm_str__new(const char *str) +{ + struct comm_str *result = NULL; + RC_STRUCT(comm_str) *cs; + + cs = malloc(sizeof(*cs) + strlen(str) + 1); + if (ADD_RC_CHK(result, cs)) { + refcount_set(comm_str__refcnt(result), 1); + strcpy(&cs->str[0], str); + } + return result; +} + +static int comm_str__search(const void *_key, const void *_member) +{ + const char *key = _key; + const struct comm_str *member = *(const struct comm_str * const *)_member; + + return strcmp(key, comm_str__str(member)); +} + +static void comm_strs__remove_if_last(struct comm_str *cs) +{ + struct comm_strs *comm_strs = comm_strs__get(); + + down_write(&comm_strs->lock); + /* + * Are there only references from the array, if so remove the array + * reference under the write lock so that we don't race with findnew. + */ + if (refcount_read(comm_str__refcnt(cs)) == 1) { + struct comm_str **entry; + + entry = bsearch(comm_str__str(cs), comm_strs->strs, comm_strs->num_strs, + sizeof(struct comm_str *), comm_str__search); + comm_str__put(*entry); + for (int i = entry - comm_strs->strs; i < comm_strs->num_strs - 1; i++) + comm_strs->strs[i] = comm_strs->strs[i + 1]; + comm_strs->num_strs--; + } + up_write(&comm_strs->lock); +} + +static struct comm_str *__comm_strs__find(struct comm_strs *comm_strs, const char *str) + SHARED_LOCKS_REQUIRED(comm_strs->lock) +{ + struct comm_str **result; + + result = bsearch(str, comm_strs->strs, comm_strs->num_strs, sizeof(struct comm_str *), + comm_str__search); + + if (!result) + return NULL; + + return comm_str__get(*result); +} + +static struct comm_str *comm_strs__findnew(const char *str) +{ + struct comm_strs *comm_strs = comm_strs__get(); + struct comm_str *result; + + if (!comm_strs) + return NULL; + + down_read(&comm_strs->lock); + result = __comm_strs__find(comm_strs, str); + up_read(&comm_strs->lock); + if (result) + return result; + + down_write(&comm_strs->lock); + result = __comm_strs__find(comm_strs, str); + if (!result) { + if (comm_strs->num_strs == comm_strs->capacity) { + struct comm_str **tmp; + + tmp = reallocarray(comm_strs->strs, + comm_strs->capacity + 16, + sizeof(*comm_strs->strs)); + if (!tmp) { + up_write(&comm_strs->lock); + return NULL; + } + comm_strs->strs = tmp; + comm_strs->capacity += 16; + } + result = comm_str__new(str); + if (result) { + int low = 0, high = comm_strs->num_strs - 1; + int insert = comm_strs->num_strs; /* Default to inserting at the end. */ + + while (low <= high) { + int mid = low + (high - low) / 2; + int cmp = strcmp(comm_str__str(comm_strs->strs[mid]), str); + + if (cmp < 0) { + low = mid + 1; + } else { + high = mid - 1; + insert = mid; + } + } + memmove(&comm_strs->strs[insert + 1], &comm_strs->strs[insert], + (comm_strs->num_strs - insert) * sizeof(struct comm_str *)); + comm_strs->num_strs++; + comm_strs->strs[insert] = result; + } + } + up_write(&comm_strs->lock); + return comm_str__get(result); +} + +struct comm *comm__new(const char *str, u64 timestamp, bool exec) +{ + struct comm *comm = zalloc(sizeof(*comm)); + + if (!comm) + return NULL; + + comm->start = timestamp; + comm->exec = exec; + + comm->comm_str = comm_strs__findnew(str); + if (!comm->comm_str) { + free(comm); + return NULL; + } + + return comm; +} + +int comm__override(struct comm *comm, const char *str, u64 timestamp, bool exec) +{ + struct comm_str *new, *old = comm->comm_str; + + new = comm_strs__findnew(str); + if (!new) + return -ENOMEM; + + comm_str__put(old); + comm->comm_str = new; + comm->start = timestamp; + if (exec) + comm->exec = true; + + return 0; +} + +void comm__free(struct comm *comm) +{ + comm_str__put(comm->comm_str); + free(comm); +} + +const char *comm__str(const struct comm *comm) +{ + return comm_str__str(comm->comm_str); +} diff --git a/tools/perf/util/comm.h b/tools/perf/util/comm.h new file mode 100644 index 000000000..f35d8fbfa --- /dev/null +++ b/tools/perf/util/comm.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_COMM_H +#define __PERF_COMM_H + +#include +#include +#include + +struct comm_str; + +struct comm { + struct comm_str *comm_str; + u64 start; + struct list_head list; + bool exec; + union { /* Tool specific area */ + void *priv; + u64 db_id; + }; +}; + +void comm__free(struct comm *comm); +struct comm *comm__new(const char *str, u64 timestamp, bool exec); +const char *comm__str(const struct comm *comm); +int comm__override(struct comm *comm, const char *str, u64 timestamp, + bool exec); + +#endif /* __PERF_COMM_H */ diff --git a/tools/perf/util/compress.h b/tools/perf/util/compress.h new file mode 100644 index 000000000..6cfecfca1 --- /dev/null +++ b/tools/perf/util/compress.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PERF_COMPRESS_H +#define PERF_COMPRESS_H + +#include +#include +#include +#include +#include +#ifdef HAVE_ZSTD_SUPPORT +#include +#endif + +#ifdef HAVE_ZLIB_SUPPORT +int gzip_decompress_to_file(const char *input, int output_fd); +bool gzip_is_compressed(const char *input); +#endif + +#ifdef HAVE_LZMA_SUPPORT +int lzma_decompress_stream_to_file(FILE *input, int output_fd); +int lzma_decompress_to_file(const char *input, int output_fd); +bool lzma_is_compressed(const char *input); +#else +static inline +int lzma_decompress_stream_to_file(FILE *input __maybe_unused, + int output_fd __maybe_unused) +{ + return -1; +} +static inline +int lzma_decompress_to_file(const char *input __maybe_unused, + int output_fd __maybe_unused) +{ + return -1; +} +static inline int lzma_is_compressed(const char *input __maybe_unused) +{ + return false; +} +#endif + +struct zstd_data { +#ifdef HAVE_ZSTD_SUPPORT + ZSTD_CStream *cstream; + ZSTD_DStream *dstream; + int comp_level; +#endif +}; + +#ifdef HAVE_ZSTD_SUPPORT + +int zstd_init(struct zstd_data *data, int level); +int zstd_fini(struct zstd_data *data); + +ssize_t zstd_compress_stream_to_records(struct zstd_data *data, void *dst, size_t dst_size, + void *src, size_t src_size, size_t max_record_size, + size_t process_header(void *record, size_t increment)); + +size_t zstd_decompress_stream(struct zstd_data *data, void *src, size_t src_size, + void *dst, size_t dst_size); +#else /* !HAVE_ZSTD_SUPPORT */ + +static inline int zstd_init(struct zstd_data *data __maybe_unused, int level __maybe_unused) +{ + return 0; +} + +static inline int zstd_fini(struct zstd_data *data __maybe_unused) +{ + return 0; +} + +static inline +ssize_t zstd_compress_stream_to_records(struct zstd_data *data __maybe_unused, + void *dst __maybe_unused, size_t dst_size __maybe_unused, + void *src __maybe_unused, size_t src_size __maybe_unused, + size_t max_record_size __maybe_unused, + size_t process_header(void *record, size_t increment) __maybe_unused) +{ + return 0; +} + +static inline size_t zstd_decompress_stream(struct zstd_data *data __maybe_unused, void *src __maybe_unused, + size_t src_size __maybe_unused, void *dst __maybe_unused, + size_t dst_size __maybe_unused) +{ + return 0; +} +#endif + +#endif /* PERF_COMPRESS_H */ diff --git a/tools/perf/util/config.c b/tools/perf/util/config.c new file mode 100644 index 000000000..7988149dc --- /dev/null +++ b/tools/perf/util/config.c @@ -0,0 +1,995 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * config.c + * + * Helper functions for parsing config items. + * Originally copied from GIT source. + * + * Copyright (C) Linus Torvalds, 2005 + * Copyright (C) Johannes Schindelin, 2005 + * + */ +#include +#include +#include "cache.h" +#include "callchain.h" +#include "header.h" +#include +#include "util/event.h" /* proc_map_timeout */ +#include "util/hist.h" /* perf_hist_config */ +#include "util/stat.h" /* perf_stat__set_big_num */ +#include "util/evsel.h" /* evsel__hw_names, evsel__use_bpf_counters */ +#include "srcline.h" +#include "build-id.h" +#include "debug.h" +#include "config.h" +#include "unwind.h" +#include +#include +#include +#include +#include +#include +#include + +#define MAXNAME (256) + +#define DEBUG_CACHE_DIR ".debug" + +#define METRIC_ONLY_LEN 20 + +static struct stats walltime_nsecs_stats; + +struct perf_stat_config stat_config = { + .aggr_mode = AGGR_GLOBAL, + .aggr_level = MAX_CACHE_LVL + 1, + .scale = true, + .unit_width = 4, /* strlen("unit") */ + .run_count = 1, + .metric_only_len = METRIC_ONLY_LEN, + .walltime_nsecs_stats = &walltime_nsecs_stats, + .big_num = true, + .ctl_fd = -1, + .ctl_fd_ack = -1, + .iostat_run = false, +}; + +char buildid_dir[MAXPATHLEN]; /* root dir for buildid, binary cache */ + +static FILE *config_file; +static const char *config_file_name; +static int config_linenr; +static int config_file_eof; +static struct perf_config_set *config_set; + +const char *config_exclusive_filename; + +static int get_next_char(void) +{ + int c; + FILE *f; + + c = '\n'; + if ((f = config_file) != NULL) { + c = fgetc(f); + if (c == '\r') { + /* DOS like systems */ + c = fgetc(f); + if (c != '\n') { + ungetc(c, f); + c = '\r'; + } + } + if (c == '\n') + config_linenr++; + if (c == EOF) { + config_file_eof = 1; + c = '\n'; + } + } + return c; +} + +static char *parse_value(void) +{ + static char value[1024]; + int quote = 0, comment = 0, space = 0; + size_t len = 0; + + for (;;) { + int c = get_next_char(); + + if (len >= sizeof(value) - 1) + return NULL; + if (c == '\n') { + if (quote) + return NULL; + value[len] = 0; + return value; + } + if (comment) + continue; + if (isspace(c) && !quote) { + space = 1; + continue; + } + if (!quote) { + if (c == ';' || c == '#') { + comment = 1; + continue; + } + } + if (space) { + if (len) + value[len++] = ' '; + space = 0; + } + if (c == '\\') { + c = get_next_char(); + switch (c) { + case '\n': + continue; + case 't': + c = '\t'; + break; + case 'b': + c = '\b'; + break; + case 'n': + c = '\n'; + break; + /* Some characters escape as themselves */ + case '\\': case '"': + break; + /* Reject unknown escape sequences */ + default: + return NULL; + } + value[len++] = c; + continue; + } + if (c == '"') { + quote = 1-quote; + continue; + } + value[len++] = c; + } +} + +static inline int iskeychar(int c) +{ + return isalnum(c) || c == '-' || c == '_'; +} + +static int get_value(config_fn_t fn, void *data, char *name, unsigned int len) +{ + int c; + char *value; + + /* Get the full name */ + for (;;) { + c = get_next_char(); + if (config_file_eof) + break; + if (!iskeychar(c)) + break; + name[len++] = c; + if (len >= MAXNAME) + return -1; + } + name[len] = 0; + while (c == ' ' || c == '\t') + c = get_next_char(); + + value = NULL; + if (c != '\n') { + if (c != '=') + return -1; + value = parse_value(); + if (!value) + return -1; + } + return fn(name, value, data); +} + +static int get_extended_base_var(char *name, int baselen, int c) +{ + do { + if (c == '\n') + return -1; + c = get_next_char(); + } while (isspace(c)); + + /* We require the format to be '[base "extension"]' */ + if (c != '"') + return -1; + name[baselen++] = '.'; + + for (;;) { + int ch = get_next_char(); + + if (ch == '\n') + return -1; + if (ch == '"') + break; + if (ch == '\\') { + ch = get_next_char(); + if (ch == '\n') + return -1; + } + name[baselen++] = ch; + if (baselen > MAXNAME / 2) + return -1; + } + + /* Final ']' */ + if (get_next_char() != ']') + return -1; + return baselen; +} + +static int get_base_var(char *name) +{ + int baselen = 0; + + for (;;) { + int c = get_next_char(); + if (config_file_eof) + return -1; + if (c == ']') + return baselen; + if (isspace(c)) + return get_extended_base_var(name, baselen, c); + if (!iskeychar(c) && c != '.') + return -1; + if (baselen > MAXNAME / 2) + return -1; + name[baselen++] = tolower(c); + } +} + +static int perf_parse_file(config_fn_t fn, void *data) +{ + int comment = 0; + int baselen = 0; + static char var[MAXNAME]; + + /* U+FEFF Byte Order Mark in UTF8 */ + static const unsigned char *utf8_bom = (unsigned char *) "\xef\xbb\xbf"; + const unsigned char *bomptr = utf8_bom; + + for (;;) { + int line, c = get_next_char(); + + if (bomptr && *bomptr) { + /* We are at the file beginning; skip UTF8-encoded BOM + * if present. Sane editors won't put this in on their + * own, but e.g. Windows Notepad will do it happily. */ + if ((unsigned char) c == *bomptr) { + bomptr++; + continue; + } else { + /* Do not tolerate partial BOM. */ + if (bomptr != utf8_bom) + break; + /* No BOM at file beginning. Cool. */ + bomptr = NULL; + } + } + if (c == '\n') { + if (config_file_eof) + return 0; + comment = 0; + continue; + } + if (comment || isspace(c)) + continue; + if (c == '#' || c == ';') { + comment = 1; + continue; + } + if (c == '[') { + baselen = get_base_var(var); + if (baselen <= 0) + break; + var[baselen++] = '.'; + var[baselen] = 0; + continue; + } + if (!isalpha(c)) + break; + var[baselen] = tolower(c); + + /* + * The get_value function might or might not reach the '\n', + * so saving the current line number for error reporting. + */ + line = config_linenr; + if (get_value(fn, data, var, baselen+1) < 0) { + config_linenr = line; + break; + } + } + pr_err("bad config file line %d in %s\n", config_linenr, config_file_name); + return -1; +} + +static int parse_unit_factor(const char *end, unsigned long *val) +{ + if (!*end) + return 1; + else if (!strcasecmp(end, "k")) { + *val *= 1024; + return 1; + } + else if (!strcasecmp(end, "m")) { + *val *= 1024 * 1024; + return 1; + } + else if (!strcasecmp(end, "g")) { + *val *= 1024 * 1024 * 1024; + return 1; + } + return 0; +} + +static int perf_parse_llong(const char *value, long long *ret) +{ + if (value && *value) { + char *end; + long long val = strtoll(value, &end, 0); + unsigned long factor = 1; + + if (!parse_unit_factor(end, &factor)) + return 0; + *ret = val * factor; + return 1; + } + return 0; +} + +static int perf_parse_long(const char *value, long *ret) +{ + if (value && *value) { + char *end; + long val = strtol(value, &end, 0); + unsigned long factor = 1; + if (!parse_unit_factor(end, &factor)) + return 0; + *ret = val * factor; + return 1; + } + return 0; +} + +static void bad_config(const char *name) +{ + if (config_file_name) + pr_warning("bad config value for '%s' in %s, ignoring...\n", name, config_file_name); + else + pr_warning("bad config value for '%s', ignoring...\n", name); +} + +int perf_config_u64(u64 *dest, const char *name, const char *value) +{ + long long ret = 0; + + if (!perf_parse_llong(value, &ret)) { + bad_config(name); + return -1; + } + + *dest = ret; + return 0; +} + +int perf_config_int(int *dest, const char *name, const char *value) +{ + long ret = 0; + if (!perf_parse_long(value, &ret)) { + bad_config(name); + return -1; + } + *dest = ret; + return 0; +} + +int perf_config_u8(u8 *dest, const char *name, const char *value) +{ + long ret = 0; + + if (!perf_parse_long(value, &ret)) { + bad_config(name); + return -1; + } + *dest = ret; + return 0; +} + +static int perf_config_bool_or_int(const char *name, const char *value, int *is_bool) +{ + int ret; + + *is_bool = 1; + if (!value) + return 1; + if (!*value) + return 0; + if (!strcasecmp(value, "true") || !strcasecmp(value, "yes") || !strcasecmp(value, "on")) + return 1; + if (!strcasecmp(value, "false") || !strcasecmp(value, "no") || !strcasecmp(value, "off")) + return 0; + *is_bool = 0; + return perf_config_int(&ret, name, value) < 0 ? -1 : ret; +} + +int perf_config_bool(const char *name, const char *value) +{ + int discard; + return !!perf_config_bool_or_int(name, value, &discard); +} + +static const char *perf_config_dirname(const char *name, const char *value) +{ + if (!name) + return NULL; + return value; +} + +static int perf_buildid_config(const char *var, const char *value) +{ + /* same dir for all commands */ + if (!strcmp(var, "buildid.dir")) { + const char *dir = perf_config_dirname(var, value); + + if (!dir) { + pr_err("Invalid buildid directory!\n"); + return -1; + } + strncpy(buildid_dir, dir, MAXPATHLEN-1); + buildid_dir[MAXPATHLEN-1] = '\0'; + } + + return 0; +} + +static int perf_default_core_config(const char *var, const char *value) +{ + if (!strcmp(var, "core.proc-map-timeout")) + proc_map_timeout = strtoul(value, NULL, 10); + + if (!strcmp(var, "core.addr2line-timeout")) + symbol_conf.addr2line_timeout_ms = strtoul(value, NULL, 10); + + if (!strcmp(var, "core.addr2line-disable-warn")) + symbol_conf.addr2line_disable_warn = perf_config_bool(var, value); + + /* Add other config variables here. */ + return 0; +} + +static int perf_ui_config(const char *var, const char *value) +{ + /* Add other config variables here. */ + if (!strcmp(var, "ui.show-headers")) + symbol_conf.show_hist_headers = perf_config_bool(var, value); + + return 0; +} + +void perf_stat__set_big_num(int set) +{ + stat_config.big_num = (set != 0); +} + +static void perf_stat__set_no_csv_summary(int set) +{ + stat_config.no_csv_summary = (set != 0); +} + +static int perf_stat_config(const char *var, const char *value) +{ + if (!strcmp(var, "stat.big-num")) + perf_stat__set_big_num(perf_config_bool(var, value)); + + if (!strcmp(var, "stat.no-csv-summary")) + perf_stat__set_no_csv_summary(perf_config_bool(var, value)); + + if (!strcmp(var, "stat.bpf-counter-events")) + evsel__bpf_counter_events = strdup(value); + + /* Add other config variables here. */ + return 0; +} + +int perf_default_config(const char *var, const char *value, + void *dummy __maybe_unused) +{ + if (strstarts(var, "core.")) + return perf_default_core_config(var, value); + + if (strstarts(var, "hist.")) + return perf_hist_config(var, value); + + if (strstarts(var, "ui.")) + return perf_ui_config(var, value); + + if (strstarts(var, "call-graph.")) + return perf_callchain_config(var, value); + + if (strstarts(var, "buildid.")) + return perf_buildid_config(var, value); + + if (strstarts(var, "stat.")) + return perf_stat_config(var, value); + + if (strstarts(var, "addr2line.")) + return addr2line_configure(var, value, dummy); + + if (strstarts(var, "unwind.")) + return unwind__configure(var, value, dummy); + + /* Add other config variables here. */ + return 0; +} + +static int perf_config_from_file(config_fn_t fn, const char *filename, void *data) +{ + int ret; + FILE *f = fopen(filename, "r"); + + ret = -1; + if (f) { + config_file = f; + config_file_name = filename; + config_linenr = 1; + config_file_eof = 0; + ret = perf_parse_file(fn, data); + fclose(f); + config_file_name = NULL; + } + return ret; +} + +const char *perf_etc_perfconfig(void) +{ + static const char *system_wide; + if (!system_wide) + system_wide = system_path(ETC_PERFCONFIG); + return system_wide; +} + +static int perf_env_bool(const char *k, int def) +{ + const char *v = getenv(k); + return v ? perf_config_bool(k, v) : def; +} + +int perf_config_system(void) +{ + return !perf_env_bool("PERF_CONFIG_NOSYSTEM", 0); +} + +int perf_config_global(void) +{ + return !perf_env_bool("PERF_CONFIG_NOGLOBAL", 0); +} + +static char *home_perfconfig(void) +{ + const char *home = NULL; + char *config; + struct stat st; + char path[PATH_MAX]; + + home = getenv("HOME"); + + /* + * Skip reading user config if: + * - there is no place to read it from (HOME) + * - we are asked not to (PERF_CONFIG_NOGLOBAL=1) + */ + if (!home || !*home || !perf_config_global()) + return NULL; + + config = strdup(mkpath(path, sizeof(path), "%s/.perfconfig", home)); + if (config == NULL) { + pr_warning("Not enough memory to process %s/.perfconfig, ignoring it.\n", home); + return NULL; + } + + if (stat(config, &st) < 0) + goto out_free; + + if (st.st_uid && (st.st_uid != geteuid())) { + pr_warning("File %s not owned by current user or root, ignoring it.\n", config); + goto out_free; + } + + if (st.st_size) + return config; + +out_free: + free(config); + return NULL; +} + +const char *perf_home_perfconfig(void) +{ + static const char *config; + static bool failed; + + if (failed || config) + return config; + + config = home_perfconfig(); + if (!config) + failed = true; + + return config; +} + +static struct perf_config_section *find_section(struct list_head *sections, + const char *section_name) +{ + struct perf_config_section *section; + + list_for_each_entry(section, sections, node) + if (!strcmp(section->name, section_name)) + return section; + + return NULL; +} + +static struct perf_config_item *find_config_item(const char *name, + struct perf_config_section *section) +{ + struct perf_config_item *item; + + list_for_each_entry(item, §ion->items, node) + if (!strcmp(item->name, name)) + return item; + + return NULL; +} + +static struct perf_config_section *add_section(struct list_head *sections, + const char *section_name) +{ + struct perf_config_section *section = zalloc(sizeof(*section)); + + if (!section) + return NULL; + + INIT_LIST_HEAD(§ion->items); + section->name = strdup(section_name); + if (!section->name) { + pr_debug("%s: strdup failed\n", __func__); + free(section); + return NULL; + } + + list_add_tail(§ion->node, sections); + return section; +} + +static struct perf_config_item *add_config_item(struct perf_config_section *section, + const char *name) +{ + struct perf_config_item *item = zalloc(sizeof(*item)); + + if (!item) + return NULL; + + item->name = strdup(name); + if (!item->name) { + pr_debug("%s: strdup failed\n", __func__); + free(item); + return NULL; + } + + list_add_tail(&item->node, §ion->items); + return item; +} + +static int set_value(struct perf_config_item *item, const char *value) +{ + char *val = strdup(value); + + if (!val) + return -1; + + zfree(&item->value); + item->value = val; + return 0; +} + +static int collect_config(const char *var, const char *value, + void *perf_config_set) +{ + int ret = -1; + char *ptr, *key; + char *section_name, *name; + struct perf_config_section *section = NULL; + struct perf_config_item *item = NULL; + struct perf_config_set *set = perf_config_set; + struct list_head *sections; + + if (set == NULL) + return -1; + + sections = &set->sections; + key = ptr = strdup(var); + if (!key) { + pr_debug("%s: strdup failed\n", __func__); + return -1; + } + + section_name = strsep(&ptr, "."); + name = ptr; + if (name == NULL || value == NULL) + goto out_free; + + section = find_section(sections, section_name); + if (!section) { + section = add_section(sections, section_name); + if (!section) + goto out_free; + } + + item = find_config_item(name, section); + if (!item) { + item = add_config_item(section, name); + if (!item) + goto out_free; + } + + /* perf_config_set can contain both user and system config items. + * So we should know where each value is from. + * The classification would be needed when a particular config file + * is overwritten by setting feature i.e. set_config(). + */ + if (strcmp(config_file_name, perf_etc_perfconfig()) == 0) { + section->from_system_config = true; + item->from_system_config = true; + } else { + section->from_system_config = false; + item->from_system_config = false; + } + + ret = set_value(item, value); + +out_free: + free(key); + return ret; +} + +int perf_config_set__collect(struct perf_config_set *set, const char *file_name, + const char *var, const char *value) +{ + config_file_name = file_name; + return collect_config(var, value, set); +} + +static int perf_config_set__init(struct perf_config_set *set) +{ + int ret = -1; + + /* Setting $PERF_CONFIG makes perf read _only_ the given config file. */ + if (config_exclusive_filename) + return perf_config_from_file(collect_config, config_exclusive_filename, set); + if (perf_config_system() && !access(perf_etc_perfconfig(), R_OK)) { + if (perf_config_from_file(collect_config, perf_etc_perfconfig(), set) < 0) + goto out; + } + if (perf_config_global() && perf_home_perfconfig()) { + if (perf_config_from_file(collect_config, perf_home_perfconfig(), set) < 0) + goto out; + } + +out: + return ret; +} + +struct perf_config_set *perf_config_set__new(void) +{ + struct perf_config_set *set = zalloc(sizeof(*set)); + + if (set) { + INIT_LIST_HEAD(&set->sections); + perf_config_set__init(set); + } + + return set; +} + +struct perf_config_set *perf_config_set__load_file(const char *file) +{ + struct perf_config_set *set = zalloc(sizeof(*set)); + + if (set) { + INIT_LIST_HEAD(&set->sections); + perf_config_from_file(collect_config, file, set); + } + + return set; +} + +static int perf_config__init(void) +{ + if (config_set == NULL) + config_set = perf_config_set__new(); + + return config_set == NULL; +} + +int perf_config_set(struct perf_config_set *set, + config_fn_t fn, void *data) +{ + int ret = 0; + char key[BUFSIZ]; + struct perf_config_section *section; + struct perf_config_item *item; + + perf_config_set__for_each_entry(set, section, item) { + char *value = item->value; + + if (value) { + scnprintf(key, sizeof(key), "%s.%s", + section->name, item->name); + ret = fn(key, value, data); + if (ret < 0) { + pr_err("Error in the given config file: wrong config key-value pair %s=%s\n", + key, value); + /* + * Can't be just a 'break', as perf_config_set__for_each_entry() + * expands to two nested for() loops. + */ + goto out; + } + } + } +out: + return ret; +} + +int perf_config(config_fn_t fn, void *data) +{ + if (config_set == NULL && perf_config__init()) + return -1; + + return perf_config_set(config_set, fn, data); +} + +void perf_config__exit(void) +{ + perf_config_set__delete(config_set); + config_set = NULL; +} + +static void perf_config_item__delete(struct perf_config_item *item) +{ + zfree(&item->name); + zfree(&item->value); + free(item); +} + +static void perf_config_section__purge(struct perf_config_section *section) +{ + struct perf_config_item *item, *tmp; + + list_for_each_entry_safe(item, tmp, §ion->items, node) { + list_del_init(&item->node); + perf_config_item__delete(item); + } +} + +static void perf_config_section__delete(struct perf_config_section *section) +{ + perf_config_section__purge(section); + zfree(§ion->name); + free(section); +} + +static void perf_config_set__purge(struct perf_config_set *set) +{ + struct perf_config_section *section, *tmp; + + list_for_each_entry_safe(section, tmp, &set->sections, node) { + list_del_init(§ion->node); + perf_config_section__delete(section); + } +} + +void perf_config_set__delete(struct perf_config_set *set) +{ + if (set == NULL) + return; + + perf_config_set__purge(set); + free(set); +} + +/* + * Call this to report error for your variable that should not + * get a boolean value (i.e. "[my] var" means "true"). + */ +int config_error_nonbool(const char *var) +{ + pr_err("Missing value for '%s'", var); + return -1; +} + +void set_buildid_dir(const char *dir) +{ + if (dir) + scnprintf(buildid_dir, MAXPATHLEN, "%s", dir); + + /* default to $HOME/.debug */ + if (buildid_dir[0] == '\0') { + char *home = getenv("HOME"); + + if (home) { + snprintf(buildid_dir, MAXPATHLEN, "%s/%s", + home, DEBUG_CACHE_DIR); + } else { + strncpy(buildid_dir, DEBUG_CACHE_DIR, MAXPATHLEN-1); + } + buildid_dir[MAXPATHLEN-1] = '\0'; + } + /* for communicating with external commands */ + setenv("PERF_BUILDID_DIR", buildid_dir, 1); +} + +struct perf_config_scan_data { + const char *name; + const char *fmt; + const char *value; + va_list args; + int ret; +}; + +static int perf_config_scan_cb(const char *var, const char *value, void *data) +{ + struct perf_config_scan_data *d = data; + + if (!strcmp(var, d->name)) + d->ret = vsscanf(value, d->fmt, d->args); + + return 0; +} + +int perf_config_scan(const char *name, const char *fmt, ...) +{ + struct perf_config_scan_data d = { + .name = name, + .fmt = fmt, + }; + + va_start(d.args, fmt); + perf_config(perf_config_scan_cb, &d); + va_end(d.args); + + return d.ret; +} + +static int perf_config_get_cb(const char *var, const char *value, void *data) +{ + struct perf_config_scan_data *d = data; + + if (!strcmp(var, d->name)) + d->value = value; + + return 0; +} + +const char *perf_config_get(const char *name) +{ + struct perf_config_scan_data d = { + .name = name, + .value = NULL, + }; + + perf_config(perf_config_get_cb, &d); + return d.value; +} diff --git a/tools/perf/util/config.h b/tools/perf/util/config.h new file mode 100644 index 000000000..987b47cf5 --- /dev/null +++ b/tools/perf/util/config.h @@ -0,0 +1,80 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CONFIG_H +#define __PERF_CONFIG_H + +#include +#include + +struct perf_config_item { + char *name; + char *value; + bool from_system_config; + struct list_head node; +}; + +struct perf_config_section { + char *name; + struct list_head items; + bool from_system_config; + struct list_head node; +}; + +struct perf_config_set { + struct list_head sections; +}; + +extern const char *config_exclusive_filename; + +typedef int (*config_fn_t)(const char *, const char *, void *); + +int perf_default_config(const char *, const char *, void *); +int perf_config(config_fn_t fn, void *); +int perf_config_scan(const char *name, const char *fmt, ...) __scanf(2, 3); +const char *perf_config_get(const char *name); +int perf_config_set(struct perf_config_set *set, + config_fn_t fn, void *data); +int perf_config_int(int *dest, const char *, const char *); +int perf_config_u8(u8 *dest, const char *name, const char *value); +int perf_config_u64(u64 *dest, const char *, const char *); +int perf_config_bool(const char *, const char *); +int config_error_nonbool(const char *); +const char *perf_etc_perfconfig(void); +const char *perf_home_perfconfig(void); +int perf_config_system(void); +int perf_config_global(void); + +struct perf_config_set *perf_config_set__new(void); +struct perf_config_set *perf_config_set__load_file(const char *file); +void perf_config_set__delete(struct perf_config_set *set); +int perf_config_set__collect(struct perf_config_set *set, const char *file_name, + const char *var, const char *value); +void perf_config__exit(void); +int perf_config__set_variable(const char *var, const char *value); + +/** + * perf_config_sections__for_each - iterate thru all the sections + * @list: list_head instance to iterate + * @section: struct perf_config_section iterator + */ +#define perf_config_sections__for_each_entry(list, section) \ + list_for_each_entry(section, list, node) + +/** + * perf_config_items__for_each - iterate thru all the items + * @list: list_head instance to iterate + * @item: struct perf_config_item iterator + */ +#define perf_config_items__for_each_entry(list, item) \ + list_for_each_entry(item, list, node) + +/** + * perf_config_set__for_each - iterate thru all the config section-item pairs + * @set: evlist instance to iterate + * @section: struct perf_config_section iterator + * @item: struct perf_config_item iterator + */ +#define perf_config_set__for_each_entry(set, section, item) \ + perf_config_sections__for_each_entry(&set->sections, section) \ + perf_config_items__for_each_entry(§ion->items, item) + +#endif /* __PERF_CONFIG_H */ diff --git a/tools/perf/util/copyfile.c b/tools/perf/util/copyfile.c new file mode 100644 index 000000000..47e03de7c --- /dev/null +++ b/tools/perf/util/copyfile.c @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "util/copyfile.h" +#include "util/namespaces.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include + +static int slow_copyfile(const char *from, const char *to, struct nsinfo *nsi) +{ + int err = -1; + char *line = NULL; + size_t n; + FILE *from_fp, *to_fp; + struct nscookie nsc; + + nsinfo__mountns_enter(nsi, &nsc); + from_fp = fopen(from, "r"); + nsinfo__mountns_exit(&nsc); + if (from_fp == NULL) + goto out; + + to_fp = fopen(to, "w"); + if (to_fp == NULL) + goto out_fclose_from; + + while (getline(&line, &n, from_fp) > 0) + if (fputs(line, to_fp) == EOF) + goto out_fclose_to; + err = 0; +out_fclose_to: + fclose(to_fp); + free(line); +out_fclose_from: + fclose(from_fp); +out: + return err; +} + +int copyfile_offset(int ifd, loff_t off_in, int ofd, loff_t off_out, u64 size) +{ + void *ptr; + loff_t pgoff; + + pgoff = off_in & ~(page_size - 1); + off_in -= pgoff; + + ptr = mmap(NULL, off_in + size, PROT_READ, MAP_PRIVATE, ifd, pgoff); + if (ptr == MAP_FAILED) + return -1; + + while (size) { + ssize_t ret = pwrite(ofd, ptr + off_in, size, off_out); + if (ret < 0 && errno == EINTR) + continue; + if (ret <= 0) + break; + + size -= ret; + off_in += ret; + off_out += ret; + } + munmap(ptr, off_in + size); + + return size ? -1 : 0; +} + +static int copyfile_mode_ns(const char *from, const char *to, mode_t mode, + struct nsinfo *nsi) +{ + int fromfd, tofd; + struct stat st; + int err; + char *tmp = NULL, *ptr = NULL; + struct nscookie nsc; + + nsinfo__mountns_enter(nsi, &nsc); + err = stat(from, &st); + nsinfo__mountns_exit(&nsc); + if (err) + goto out; + err = -1; + + /* extra 'x' at the end is to reserve space for '.' */ + if (asprintf(&tmp, "%s.XXXXXXx", to) < 0) { + tmp = NULL; + goto out; + } + ptr = strrchr(tmp, '/'); + if (!ptr) + goto out; + ptr = memmove(ptr + 1, ptr, strlen(ptr) - 1); + *ptr = '.'; + + tofd = mkstemp(tmp); + if (tofd < 0) + goto out; + + if (st.st_size == 0) { /* /proc? do it slowly... */ + err = slow_copyfile(from, tmp, nsi); + if (!err && fchmod(tofd, mode)) + err = -1; + goto out_close_to; + } + + if (fchmod(tofd, mode)) + goto out_close_to; + + nsinfo__mountns_enter(nsi, &nsc); + fromfd = open(from, O_RDONLY); + nsinfo__mountns_exit(&nsc); + if (fromfd < 0) + goto out_close_to; + + err = copyfile_offset(fromfd, 0, tofd, 0, st.st_size); + + close(fromfd); +out_close_to: + close(tofd); + if (!err) + err = link(tmp, to); + unlink(tmp); +out: + free(tmp); + return err; +} + +int copyfile_ns(const char *from, const char *to, struct nsinfo *nsi) +{ + return copyfile_mode_ns(from, to, 0755, nsi); +} + +int copyfile_mode(const char *from, const char *to, mode_t mode) +{ + return copyfile_mode_ns(from, to, mode, NULL); +} + +int copyfile(const char *from, const char *to) +{ + return copyfile_mode(from, to, 0755); +} diff --git a/tools/perf/util/copyfile.h b/tools/perf/util/copyfile.h new file mode 100644 index 000000000..e85d2f22f --- /dev/null +++ b/tools/perf/util/copyfile.h @@ -0,0 +1,16 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef PERF_COPYFILE_H_ +#define PERF_COPYFILE_H_ + +#include +#include +#include + +struct nsinfo; + +int copyfile(const char *from, const char *to); +int copyfile_mode(const char *from, const char *to, mode_t mode); +int copyfile_ns(const char *from, const char *to, struct nsinfo *nsi); +int copyfile_offset(int ifd, loff_t off_in, int ofd, loff_t off_out, u64 size); + +#endif // PERF_COPYFILE_H_ diff --git a/tools/perf/util/counts.c b/tools/perf/util/counts.c new file mode 100644 index 000000000..11cd85b27 --- /dev/null +++ b/tools/perf/util/counts.c @@ -0,0 +1,71 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "evsel.h" +#include "counts.h" +#include +#include + +struct perf_counts *perf_counts__new(int ncpus, int nthreads) +{ + struct perf_counts *counts = zalloc(sizeof(*counts)); + + if (counts) { + struct xyarray *values; + + values = xyarray__new(ncpus, nthreads, sizeof(struct perf_counts_values)); + if (!values) { + free(counts); + return NULL; + } + + counts->values = values; + + values = xyarray__new(ncpus, nthreads, sizeof(bool)); + if (!values) { + xyarray__delete(counts->values); + free(counts); + return NULL; + } + + counts->loaded = values; + } + + return counts; +} + +void perf_counts__delete(struct perf_counts *counts) +{ + if (counts) { + xyarray__delete(counts->loaded); + xyarray__delete(counts->values); + free(counts); + } +} + +void perf_counts__reset(struct perf_counts *counts) +{ + xyarray__reset(counts->loaded); + xyarray__reset(counts->values); +} + +void evsel__reset_counts(struct evsel *evsel) +{ + perf_counts__reset(evsel->counts); +} + +int evsel__alloc_counts(struct evsel *evsel) +{ + struct perf_cpu_map *cpus = evsel__cpus(evsel); + int nthreads = perf_thread_map__nr(evsel->core.threads); + + evsel->counts = perf_counts__new(perf_cpu_map__nr(cpus), nthreads); + return evsel->counts != NULL ? 0 : -ENOMEM; +} + +void evsel__free_counts(struct evsel *evsel) +{ + perf_counts__delete(evsel->counts); + evsel->counts = NULL; +} diff --git a/tools/perf/util/counts.h b/tools/perf/util/counts.h new file mode 100644 index 000000000..42760242e --- /dev/null +++ b/tools/perf/util/counts.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_COUNTS_H +#define __PERF_COUNTS_H + +#include +#include +#include +#include + +struct evsel; + +struct perf_counts { + s8 scaled; + struct xyarray *values; + struct xyarray *loaded; +}; + + +static inline struct perf_counts_values* +perf_counts(struct perf_counts *counts, int cpu_map_idx, int thread) +{ + return xyarray__entry(counts->values, cpu_map_idx, thread); +} + +static inline bool +perf_counts__is_loaded(struct perf_counts *counts, int cpu_map_idx, int thread) +{ + return *((bool *) xyarray__entry(counts->loaded, cpu_map_idx, thread)); +} + +static inline void +perf_counts__set_loaded(struct perf_counts *counts, int cpu_map_idx, int thread, bool loaded) +{ + *((bool *) xyarray__entry(counts->loaded, cpu_map_idx, thread)) = loaded; +} + +struct perf_counts *perf_counts__new(int ncpus, int nthreads); +void perf_counts__delete(struct perf_counts *counts); +void perf_counts__reset(struct perf_counts *counts); + +void evsel__reset_counts(struct evsel *evsel); +int evsel__alloc_counts(struct evsel *evsel); +void evsel__free_counts(struct evsel *evsel); + +#endif /* __PERF_COUNTS_H */ diff --git a/tools/perf/util/cpumap.c b/tools/perf/util/cpumap.c new file mode 100644 index 000000000..23ebe9b97 --- /dev/null +++ b/tools/perf/util/cpumap.c @@ -0,0 +1,824 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "cpumap.h" +#include "debug.h" +#include "event.h" +#include +#include +#include +#include +#include +#include "asm/bug.h" + +#include +#include +#include +#include + +static struct perf_cpu max_cpu_num; +static struct perf_cpu max_present_cpu_num; +static int max_node_num; +/** + * The numa node X as read from /sys/devices/system/node/nodeX indexed by the + * CPU number. + */ +static int *cpunode_map; + +bool perf_record_cpu_map_data__test_bit(int i, + const struct perf_record_cpu_map_data *data) +{ + int bit_word32 = i / 32; + __u32 bit_mask32 = 1U << (i & 31); + int bit_word64 = i / 64; + __u64 bit_mask64 = ((__u64)1) << (i & 63); + + return (data->mask32_data.long_size == 4) + ? (bit_word32 < data->mask32_data.nr) && + (data->mask32_data.mask[bit_word32] & bit_mask32) != 0 + : (bit_word64 < data->mask64_data.nr) && + (data->mask64_data.mask[bit_word64] & bit_mask64) != 0; +} + +/* Read ith mask value from data into the given 64-bit sized bitmap */ +static void perf_record_cpu_map_data__read_one_mask(const struct perf_record_cpu_map_data *data, + int i, unsigned long *bitmap, + u16 long_size) +{ +#if __SIZEOF_LONG__ == 8 + if (long_size == 4) + bitmap[0] = data->mask32_data.mask[i]; + else + bitmap[0] = data->mask64_data.mask[i]; +#else + if (long_size == 4) { + bitmap[0] = data->mask32_data.mask[i]; + bitmap[1] = 0; + } else { +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bitmap[0] = (unsigned long)(data->mask64_data.mask[i] >> 32); + bitmap[1] = (unsigned long)data->mask64_data.mask[i]; +#else + bitmap[0] = (unsigned long)data->mask64_data.mask[i]; + bitmap[1] = (unsigned long)(data->mask64_data.mask[i] >> 32); +#endif + } +#endif +} +static struct perf_cpu_map *cpu_map__from_entries(const struct perf_record_cpu_map_data *data) +{ + /* Snapshot nr — data is mmap'd and could change between reads */ + u16 nr = READ_ONCE(data->cpus_data.nr); + struct perf_cpu_map *map; + + map = perf_cpu_map__empty_new(nr); + if (!map) + return NULL; + + for (unsigned int i = 0; i < nr; i++) { + u16 cpu = READ_ONCE(data->cpus_data.cpu[i]); + /* + * Special treatment for -1, which is not real cpu number, + * and we need to use (int) -1 to initialize map[i], + * otherwise it would become 65535. + */ + if (cpu == (u16) -1) { + RC_CHK_ACCESS(map)->map[i].cpu = -1; + } else if (cpu < INT16_MAX) { + RC_CHK_ACCESS(map)->map[i].cpu = (int16_t) cpu; + } else { + pr_err("Invalid cpumap entry %u\n", cpu); + perf_cpu_map__put(map); + return NULL; + } + } + + return map; +} + +static struct perf_cpu_map *cpu_map__from_mask(const struct perf_record_cpu_map_data *data) +{ + DECLARE_BITMAP(local_copy, 64); + int weight = 0, mask_nr; + /* Snapshot before validation — data is mmap'd and could change */ + u16 long_size = READ_ONCE(data->mask32_data.long_size); + struct perf_cpu_map *map; + + /* long_size must be 4 or 8; other values overflow cpus_per_i below */ + if (long_size != 4 && long_size != 8) { + pr_warning("WARNING: cpu_map mask: unsupported long_size %u\n", long_size); + return NULL; + } + + mask_nr = READ_ONCE(data->mask32_data.nr); + + for (int i = 0; i < mask_nr; i++) { + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); + weight += bitmap_weight(local_copy, 64); + } + + map = perf_cpu_map__empty_new(weight); + if (!map) + return NULL; + + for (int i = 0, j = 0; i < mask_nr; i++) { + int cpus_per_i = (i * long_size * BITS_PER_BYTE); + int cpu; + + perf_record_cpu_map_data__read_one_mask(data, i, local_copy, long_size); + for_each_set_bit(cpu, local_copy, 64) { + /* Guard against more set bits than the first pass counted */ + if (j >= weight) + break; + if (cpu + cpus_per_i < INT16_MAX) { + RC_CHK_ACCESS(map)->map[j++].cpu = cpu + cpus_per_i; + } else { + pr_err("Invalid cpumap entry %d\n", cpu + cpus_per_i); + perf_cpu_map__put(map); + return NULL; + } + } + } + return map; + +} + +static struct perf_cpu_map *cpu_map__from_range(const struct perf_record_cpu_map_data *data) +{ + /* Snapshot fields — data is mmap'd and could change between reads */ + u16 start_cpu = READ_ONCE(data->range_cpu_data.start_cpu); + u16 end_cpu = READ_ONCE(data->range_cpu_data.end_cpu); + u16 any_cpu = READ_ONCE(data->range_cpu_data.any_cpu); + struct perf_cpu_map *map; + unsigned int i = 0; + + if (end_cpu < start_cpu) { + pr_warning("WARNING: cpu_map range: end_cpu %u < start_cpu %u\n", + end_cpu, start_cpu); + return NULL; + } + + /* any_cpu is boolean (0 or 1), not a count — clamp to avoid inflated nr */ + map = perf_cpu_map__empty_new(end_cpu - start_cpu + 1 + !!any_cpu); + if (!map) + return NULL; + + if (any_cpu) + RC_CHK_ACCESS(map)->map[i++].cpu = -1; + + for (int cpu = start_cpu; cpu <= end_cpu; + i++, cpu++) { + if (cpu < INT16_MAX) { + RC_CHK_ACCESS(map)->map[i].cpu = cpu; + } else { + pr_err("Invalid cpumap entry %d\n", cpu); + perf_cpu_map__put(map); + return NULL; + } + } + + return map; +} + +struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data) +{ + switch (data->type) { + case PERF_CPU_MAP__CPUS: + return cpu_map__from_entries(data); + case PERF_CPU_MAP__MASK: + return cpu_map__from_mask(data); + case PERF_CPU_MAP__RANGE_CPUS: + return cpu_map__from_range(data); + default: + pr_err("cpu_map__new_data unknown type %d\n", data->type); + return NULL; + } +} + +size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp) +{ +#define BUFSIZE 1024 + char buf[BUFSIZE]; + + cpu_map__snprint(map, buf, sizeof(buf)); + return fprintf(fp, "%s\n", buf); +#undef BUFSIZE +} + +struct perf_cpu_map *perf_cpu_map__empty_new(int nr) +{ + struct perf_cpu_map *cpus = perf_cpu_map__alloc(nr); + + if (cpus != NULL) { + for (int i = 0; i < nr; i++) + RC_CHK_ACCESS(cpus)->map[i].cpu = -1; + } + + return cpus; +} + +struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr) +{ + struct cpu_aggr_map *cpus = malloc(sizeof(*cpus) + sizeof(struct aggr_cpu_id) * nr); + + if (cpus != NULL) { + int i; + + cpus->nr = nr; + for (i = 0; i < nr; i++) + cpus->map[i] = aggr_cpu_id__empty(); + } + + return cpus; +} + +static int cpu__get_topology_int(int cpu, const char *name, int *value) +{ + char path[PATH_MAX]; + + snprintf(path, PATH_MAX, + "devices/system/cpu/cpu%d/topology/%s", cpu, name); + + return sysfs__read_int(path, value); +} + +int cpu__get_socket_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "physical_package_id", &value); + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data __maybe_unused) +{ + struct aggr_cpu_id id = aggr_cpu_id__empty(); + + id.socket = cpu__get_socket_id(cpu); + return id; +} + +static int aggr_cpu_id__cmp(const void *a_pointer, const void *b_pointer) +{ + struct aggr_cpu_id *a = (struct aggr_cpu_id *)a_pointer; + struct aggr_cpu_id *b = (struct aggr_cpu_id *)b_pointer; + + if (a->node != b->node) + return a->node - b->node; + else if (a->socket != b->socket) + return a->socket - b->socket; + else if (a->die != b->die) + return a->die - b->die; + else if (a->cluster != b->cluster) + return a->cluster - b->cluster; + else if (a->cache_lvl != b->cache_lvl) + return a->cache_lvl - b->cache_lvl; + else if (a->cache != b->cache) + return a->cache - b->cache; + else if (a->core != b->core) + return a->core - b->core; + else + return a->thread_idx - b->thread_idx; +} + +struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, + aggr_cpu_id_get_t get_id, + void *data, bool needs_sort) +{ + unsigned int idx; + struct perf_cpu cpu; + struct cpu_aggr_map *c = cpu_aggr_map__empty_new(perf_cpu_map__nr(cpus)); + + if (!c) + return NULL; + + /* Reset size as it may only be partially filled */ + c->nr = 0; + + perf_cpu_map__for_each_cpu(cpu, idx, cpus) { + bool duplicate = false; + struct aggr_cpu_id cpu_id = get_id(cpu, data); + + for (int j = 0; j < c->nr; j++) { + if (aggr_cpu_id__equal(&cpu_id, &c->map[j])) { + duplicate = true; + break; + } + } + if (!duplicate) { + c->map[c->nr] = cpu_id; + c->nr++; + } + } + /* Trim. */ + if (c->nr != (int)perf_cpu_map__nr(cpus)) { + struct cpu_aggr_map *trimmed_c = + realloc(c, + sizeof(struct cpu_aggr_map) + sizeof(struct aggr_cpu_id) * c->nr); + + if (trimmed_c) + c = trimmed_c; + } + + /* ensure we process id in increasing order */ + if (needs_sort) + qsort(c->map, c->nr, sizeof(struct aggr_cpu_id), aggr_cpu_id__cmp); + + return c; + +} + +int cpu__get_die_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "die_id", &value); + + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data) +{ + struct aggr_cpu_id id; + int die; + + die = cpu__get_die_id(cpu); + /* There is no die_id on legacy system. */ + if (die < 0) + die = 0; + + /* + * die_id is relative to socket, so start + * with the socket ID and then add die to + * make a unique ID. + */ + id = aggr_cpu_id__socket(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + id.die = die; + return id; +} + +int cpu__get_cluster_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "cluster_id", &value); + + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__cluster(struct perf_cpu cpu, void *data) +{ + int cluster = cpu__get_cluster_id(cpu); + struct aggr_cpu_id id; + + /* There is no cluster_id on legacy system. */ + if (cluster < 0) + cluster = 0; + + id = aggr_cpu_id__die(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + id.cluster = cluster; + return id; +} + +int cpu__get_core_id(struct perf_cpu cpu) +{ + int value, ret = cpu__get_topology_int(cpu.cpu, "core_id", &value); + return ret ?: value; +} + +struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data) +{ + struct aggr_cpu_id id; + int core = cpu__get_core_id(cpu); + + /* aggr_cpu_id__die returns a struct with socket die, and cluster set. */ + id = aggr_cpu_id__cluster(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + /* + * core_id is relative to socket and die, we need a global id. + * So we combine the result from cpu_map__get_die with the core id + */ + id.core = core; + return id; + +} + +struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data) +{ + struct aggr_cpu_id id; + + /* aggr_cpu_id__core returns a struct with socket, die and core set. */ + id = aggr_cpu_id__core(cpu, data); + if (aggr_cpu_id__is_empty(&id)) + return id; + + id.cpu = cpu; + return id; + +} + +struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data __maybe_unused) +{ + struct aggr_cpu_id id = aggr_cpu_id__empty(); + + id.node = cpu__get_node(cpu); + return id; +} + +struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data __maybe_unused) +{ + struct aggr_cpu_id id = aggr_cpu_id__empty(); + + /* it always aggregates to the cpu 0 */ + cpu.cpu = 0; + id.cpu = cpu; + return id; +} + +/* setup simple routines to easily access node numbers given a cpu number */ +static int get_max_num(char *path, int *max) +{ + size_t num; + char *buf; + int err = 0; + + if (filename__read_str(path, &buf, &num)) + return -1; + + buf[num] = '\0'; + + /* empty file — nothing to parse */ + if (num == 0) { + err = -1; + goto out; + } + + /* start on the right, to find highest node num */ + while (--num) { + if ((buf[num] == ',') || (buf[num] == '-')) { + num++; + break; + } + } + if (sscanf(&buf[num], "%d", max) < 1) { + err = -1; + goto out; + } + + /* convert from 0-based to 1-based */ + (*max)++; + +out: + free(buf); + return err; +} + +/* Determine highest possible cpu in the system for sparse allocation */ +static void set_max_cpu_num(void) +{ + const char *mnt; + char path[PATH_MAX]; + int max, ret = -1; + + /* set up default */ + max_cpu_num.cpu = 4096; + max_present_cpu_num.cpu = 4096; + + mnt = sysfs__mountpoint(); + if (!mnt) + goto out; + + /* get the highest possible cpu number for a sparse allocation */ + ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/possible", mnt); + if (ret >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + goto out; + } + + ret = get_max_num(path, &max); + if (ret) + goto out; + + /* + * struct perf_cpu.cpu is int16_t (libperf ABI) — clamp to avoid + * truncation to negative. See tools/lib/perf/TODO for the ABI + * widening plan. + */ + if (max > INT16_MAX) { + pr_warning("WARNING: max possible cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } + max_cpu_num.cpu = max; + + /* get the highest present cpu number for a sparse allocation */ + ret = snprintf(path, PATH_MAX, "%s/devices/system/cpu/present", mnt); + if (ret >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + goto out; + } + + ret = get_max_num(path, &max); + + if (!ret && max > INT16_MAX) { + pr_warning("WARNING: max present cpus %d exceeds int16_t, clamping to %d\n", + max, INT16_MAX); + max = INT16_MAX; + } + if (!ret) + max_present_cpu_num.cpu = max; +out: + if (ret) + pr_err("Failed to read max cpus, using default of %d\n", max_cpu_num.cpu); +} + +/* Determine highest possible node in the system for sparse allocation */ +static void set_max_node_num(void) +{ + const char *mnt; + char path[PATH_MAX]; + int ret = -1; + + /* set up default */ + max_node_num = 8; + + mnt = sysfs__mountpoint(); + if (!mnt) + goto out; + + /* get the highest possible cpu number for a sparse allocation */ + ret = snprintf(path, PATH_MAX, "%s/devices/system/node/possible", mnt); + if (ret >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + goto out; + } + + ret = get_max_num(path, &max_node_num); + +out: + if (ret) + pr_err("Failed to read max nodes, using default of %d\n", max_node_num); +} + +int cpu__max_node(void) +{ + if (unlikely(!max_node_num)) + set_max_node_num(); + + return max_node_num; +} + +struct perf_cpu cpu__max_cpu(void) +{ + if (unlikely(!max_cpu_num.cpu)) + set_max_cpu_num(); + + return max_cpu_num; +} + +struct perf_cpu cpu__max_present_cpu(void) +{ + if (unlikely(!max_present_cpu_num.cpu)) + set_max_cpu_num(); + + return max_present_cpu_num; +} + + +int cpu__get_node(struct perf_cpu cpu) +{ + if (unlikely(cpunode_map == NULL)) { + pr_debug("cpu_map not initialized\n"); + return -1; + } + + /* cpunode_map allocated for max_cpu_num entries; input may be untrusted */ + if (cpu.cpu < 0 || cpu.cpu >= max_cpu_num.cpu) + return -1; + + return cpunode_map[cpu.cpu]; +} + +static int init_cpunode_map(void) +{ + int i; + + set_max_cpu_num(); + set_max_node_num(); + + cpunode_map = calloc(max_cpu_num.cpu, sizeof(int)); + if (!cpunode_map) { + pr_err("%s: calloc failed\n", __func__); + return -1; + } + + for (i = 0; i < max_cpu_num.cpu; i++) + cpunode_map[i] = -1; + + return 0; +} + +int cpu__setup_cpunode_map(void) +{ + struct dirent *dent1, *dent2; + DIR *dir1, *dir2; + unsigned int cpu, mem; + char buf[PATH_MAX]; + char path[PATH_MAX]; + const char *mnt; + int n; + + /* initialize globals */ + if (init_cpunode_map()) + return -1; + + mnt = sysfs__mountpoint(); + if (!mnt) + return 0; + + n = snprintf(path, PATH_MAX, "%s/devices/system/node", mnt); + if (n >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + return -1; + } + + dir1 = opendir(path); + if (!dir1) + return 0; + + /* walk tree and setup map */ + while ((dent1 = readdir(dir1)) != NULL) { + if (dent1->d_type != DT_DIR || sscanf(dent1->d_name, "node%u", &mem) < 1) + continue; + + n = snprintf(buf, PATH_MAX, "%s/%s", path, dent1->d_name); + if (n >= PATH_MAX) { + pr_err("sysfs path crossed PATH_MAX(%d) size\n", PATH_MAX); + continue; + } + + dir2 = opendir(buf); + if (!dir2) + continue; + while ((dent2 = readdir(dir2)) != NULL) { + if (dent2->d_type != DT_LNK || sscanf(dent2->d_name, "cpu%u", &cpu) < 1) + continue; + /* cpunode_map allocated for max_cpu_num entries */ + if (cpu < (unsigned int)max_cpu_num.cpu) + cpunode_map[cpu] = mem; + } + closedir(dir2); + } + closedir(dir1); + return 0; +} + +size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size) +{ + int i, start = -1; + bool first = true; + size_t ret = 0; + +#define COMMA first ? "" : "," + + for (i = 0; i < (int)perf_cpu_map__nr(map) + 1; i++) { + struct perf_cpu cpu = { .cpu = INT16_MAX }; + bool last = i == (int)perf_cpu_map__nr(map); + + if (!last) + cpu = perf_cpu_map__cpu(map, i); + + if (start == -1) { + start = i; + if (last) { + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, i).cpu); + } + } else if (((i - start) != (cpu.cpu - perf_cpu_map__cpu(map, start).cpu)) || last) { + int end = i - 1; + + if (start == end) { + ret += scnprintf(buf + ret, size - ret, + "%s%d", COMMA, + perf_cpu_map__cpu(map, start).cpu); + } else { + ret += scnprintf(buf + ret, size - ret, + "%s%d-%d", COMMA, + perf_cpu_map__cpu(map, start).cpu, perf_cpu_map__cpu(map, end).cpu); + } + first = false; + start = i; + } + } + +#undef COMMA + + pr_debug2("cpumask list: %s\n", buf); + return ret; +} + +static char hex_char(unsigned char val) +{ + if (val < 10) + return val + '0'; + if (val < 16) + return val - 10 + 'a'; + return '?'; +} + +size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size) +{ + unsigned int idx; + char *ptr = buf; + unsigned char *bitmap; + struct perf_cpu c, last_cpu = perf_cpu_map__max(map); + + if (buf == NULL || size == 0) + return 0; + + if (last_cpu.cpu < 0) { + buf[0] = '\0'; + return 0; + } + + bitmap = zalloc(last_cpu.cpu / 8 + 1); + if (bitmap == NULL) { + buf[0] = '\0'; + return 0; + } + + perf_cpu_map__for_each_cpu_skip_any(c, idx, map) + bitmap[c.cpu / 8] |= 1 << (c.cpu % 8); + + for (int cpu = last_cpu.cpu / 4 * 4; cpu >= 0; cpu -= 4) { + unsigned char bits = bitmap[cpu / 8]; + + if (cpu % 8) + bits >>= 4; + else + bits &= 0xf; + + *ptr++ = hex_char(bits); + if ((cpu % 32) == 0 && cpu > 0) + *ptr++ = ','; + } + *ptr = '\0'; + free(bitmap); + + buf[size - 1] = '\0'; + return ptr - buf; +} + +struct perf_cpu_map *cpu_map__online(void) /* thread unsafe */ +{ + static struct perf_cpu_map *online; + + if (!online) + online = perf_cpu_map__new_online_cpus(); /* from /sys/devices/system/cpu/online */ + + return perf_cpu_map__get(online); +} + +bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b) +{ + return a->thread_idx == b->thread_idx && + a->node == b->node && + a->socket == b->socket && + a->die == b->die && + a->cluster == b->cluster && + a->cache_lvl == b->cache_lvl && + a->cache == b->cache && + a->core == b->core && + a->cpu.cpu == b->cpu.cpu; +} + +bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a) +{ + return a->thread_idx == -1 && + a->node == -1 && + a->socket == -1 && + a->die == -1 && + a->cluster == -1 && + a->cache_lvl == -1 && + a->cache == -1 && + a->core == -1 && + a->cpu.cpu == -1; +} + +struct aggr_cpu_id aggr_cpu_id__empty(void) +{ + struct aggr_cpu_id ret = { + .thread_idx = -1, + .node = -1, + .socket = -1, + .die = -1, + .cluster = -1, + .cache_lvl = -1, + .cache = -1, + .core = -1, + .cpu = (struct perf_cpu){ .cpu = -1 }, + }; + return ret; +} diff --git a/tools/perf/util/cpumap.h b/tools/perf/util/cpumap.h new file mode 100644 index 000000000..ee0f6139b --- /dev/null +++ b/tools/perf/util/cpumap.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CPUMAP_H +#define __PERF_CPUMAP_H + +#include +#include +#include + +/** Identify where counts are aggregated, -1 implies not to aggregate. */ +struct aggr_cpu_id { + /** A value in the range 0 to number of threads. */ + int thread_idx; + /** The numa node X as read from /sys/devices/system/node/nodeX. */ + int node; + /** + * The socket number as read from + * /sys/devices/system/cpu/cpuX/topology/physical_package_id. + */ + int socket; + /** The die id as read from /sys/devices/system/cpu/cpuX/topology/die_id. */ + int die; + /** The cluster id as read from /sys/devices/system/cpu/cpuX/topology/cluster_id */ + int cluster; + /** The cache level as read from /sys/devices/system/cpu/cpuX/cache/indexY/level */ + int cache_lvl; + /** + * The cache instance ID, which is the first CPU in the + * /sys/devices/system/cpu/cpuX/cache/indexY/shared_cpu_list + */ + int cache; + /** The core id as read from /sys/devices/system/cpu/cpuX/topology/core_id. */ + int core; + /** CPU aggregation, note there is one CPU for each SMT thread. */ + struct perf_cpu cpu; +}; + +/** A collection of aggr_cpu_id values, the "built" version is sorted and uniqued. */ +struct cpu_aggr_map { + /** Number of valid entries. */ + int nr; + /** The entries. */ + struct aggr_cpu_id map[]; +}; + +#define cpu_aggr_map__for_each_idx(idx, aggr_map) \ + for ((idx) = 0; (idx) < aggr_map->nr; (idx)++) + +struct perf_record_cpu_map_data; + +bool perf_record_cpu_map_data__test_bit(int i, const struct perf_record_cpu_map_data *data); + +struct perf_cpu_map *perf_cpu_map__empty_new(int nr); + +struct perf_cpu_map *cpu_map__new_data(const struct perf_record_cpu_map_data *data); +size_t cpu_map__snprint(struct perf_cpu_map *map, char *buf, size_t size); +size_t cpu_map__snprint_mask(struct perf_cpu_map *map, char *buf, size_t size); +size_t cpu_map__fprintf(struct perf_cpu_map *map, FILE *fp); +struct perf_cpu_map *cpu_map__online(void); /* thread unsafe */ + +int cpu__setup_cpunode_map(void); + +int cpu__max_node(void); +struct perf_cpu cpu__max_cpu(void); +struct perf_cpu cpu__max_present_cpu(void); + +/** + * cpu_map__is_dummy - Events associated with a pid, rather than a CPU, use a single dummy map with an entry of -1. + */ +static inline bool cpu_map__is_dummy(const struct perf_cpu_map *cpus) +{ + return perf_cpu_map__nr(cpus) == 1 && perf_cpu_map__cpu(cpus, 0).cpu == -1; +} + +/** + * cpu__get_node - Returns the numa node X as read from + * /sys/devices/system/node/nodeX for the given CPU. + */ +int cpu__get_node(struct perf_cpu cpu); +/** + * cpu__get_socket_id - Returns the socket number as read from + * /sys/devices/system/cpu/cpuX/topology/physical_package_id for the given CPU. + */ +int cpu__get_socket_id(struct perf_cpu cpu); +/** + * cpu__get_die_id - Returns the die id as read from + * /sys/devices/system/cpu/cpuX/topology/die_id for the given CPU. + */ +int cpu__get_die_id(struct perf_cpu cpu); +/** + * cpu__get_cluster_id - Returns the cluster id as read from + * /sys/devices/system/cpu/cpuX/topology/cluster_id for the given CPU + */ +int cpu__get_cluster_id(struct perf_cpu cpu); +/** + * cpu__get_core_id - Returns the core id as read from + * /sys/devices/system/cpu/cpuX/topology/core_id for the given CPU. + */ +int cpu__get_core_id(struct perf_cpu cpu); + +/** + * cpu_aggr_map__empty_new - Create a cpu_aggr_map of size nr with every entry + * being empty. + */ +struct cpu_aggr_map *cpu_aggr_map__empty_new(int nr); + +typedef struct aggr_cpu_id (*aggr_cpu_id_get_t)(struct perf_cpu cpu, void *data); + +/** + * cpu_aggr_map__new - Create a cpu_aggr_map with an aggr_cpu_id for each cpu in + * cpus. The aggr_cpu_id is created with 'get_id' that may have a data value + * passed to it. The cpu_aggr_map is sorted with duplicate values removed. + */ +struct cpu_aggr_map *cpu_aggr_map__new(const struct perf_cpu_map *cpus, + aggr_cpu_id_get_t get_id, + void *data, bool needs_sort); + +bool aggr_cpu_id__equal(const struct aggr_cpu_id *a, const struct aggr_cpu_id *b); +bool aggr_cpu_id__is_empty(const struct aggr_cpu_id *a); +struct aggr_cpu_id aggr_cpu_id__empty(void); + + +/** + * aggr_cpu_id__socket - Create an aggr_cpu_id with the socket populated with + * the socket for cpu. The function signature is compatible with + * aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__socket(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__die - Create an aggr_cpu_id with the die and socket populated + * with the die and socket for cpu. The function signature is compatible with + * aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__die(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__cluster - Create an aggr_cpu_id with cluster, die and socket + * populated with the cluster, die and socket for cpu. The function signature + * is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__cluster(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__core - Create an aggr_cpu_id with the core, cluster, die and + * socket populated with the core, die and socket for cpu. The function + * signature is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__core(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__core - Create an aggr_cpu_id with the cpu, core, die and socket + * populated with the cpu, core, die and socket for cpu. The function signature + * is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__cpu(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__node - Create an aggr_cpu_id with the numa node populated for + * cpu. The function signature is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__node(struct perf_cpu cpu, void *data); +/** + * aggr_cpu_id__global - Create an aggr_cpu_id for global aggregation. + * The function signature is compatible with aggr_cpu_id_get_t. + */ +struct aggr_cpu_id aggr_cpu_id__global(struct perf_cpu cpu, void *data); +#endif /* __PERF_CPUMAP_H */ diff --git a/tools/perf/util/cputopo.c b/tools/perf/util/cputopo.c new file mode 100644 index 000000000..e0091804f --- /dev/null +++ b/tools/perf/util/cputopo.c @@ -0,0 +1,506 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cputopo.h" +#include "cpumap.h" +#include "debug.h" +#include "env.h" +#include "pmu.h" +#include "pmus.h" + +#define PACKAGE_CPUS_FMT \ + "%s/devices/system/cpu/cpu%d/topology/package_cpus_list" +#define PACKAGE_CPUS_FMT_OLD \ + "%s/devices/system/cpu/cpu%d/topology/core_siblings_list" +#define DIE_CPUS_FMT \ + "%s/devices/system/cpu/cpu%d/topology/die_cpus_list" +#define CORE_CPUS_FMT \ + "%s/devices/system/cpu/cpu%d/topology/core_cpus_list" +#define CORE_CPUS_FMT_OLD \ + "%s/devices/system/cpu/cpu%d/topology/thread_siblings_list" +#define NODE_ONLINE_FMT \ + "%s/devices/system/node/online" +#define NODE_MEMINFO_FMT \ + "%s/devices/system/node/node%d/meminfo" +#define NODE_CPULIST_FMT \ + "%s/devices/system/node/node%d/cpulist" + +static int build_cpu_topology(struct cpu_topology *tp, int cpu) +{ + FILE *fp; + char filename[MAXPATHLEN]; + char *buf = NULL, *p; + size_t len = 0; + ssize_t sret; + u32 i = 0; + int ret = -1; + + scnprintf(filename, MAXPATHLEN, PACKAGE_CPUS_FMT, + sysfs__mountpoint(), cpu); + if (access(filename, F_OK) == -1) { + scnprintf(filename, MAXPATHLEN, PACKAGE_CPUS_FMT_OLD, + sysfs__mountpoint(), cpu); + } + fp = fopen(filename, "r"); + if (!fp) + goto try_dies; + + sret = getline(&buf, &len, fp); + fclose(fp); + if (sret <= 0) + goto try_dies; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + for (i = 0; i < tp->package_cpus_lists; i++) { + if (!strcmp(buf, tp->package_cpus_list[i])) + break; + } + if (i == tp->package_cpus_lists) { + tp->package_cpus_list[i] = buf; + tp->package_cpus_lists++; + buf = NULL; + len = 0; + } + ret = 0; + +try_dies: + if (!tp->die_cpus_list) + goto try_threads; + + scnprintf(filename, MAXPATHLEN, DIE_CPUS_FMT, + sysfs__mountpoint(), cpu); + fp = fopen(filename, "r"); + if (!fp) + goto try_threads; + + sret = getline(&buf, &len, fp); + fclose(fp); + if (sret <= 0) + goto try_threads; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + for (i = 0; i < tp->die_cpus_lists; i++) { + if (!strcmp(buf, tp->die_cpus_list[i])) + break; + } + if (i == tp->die_cpus_lists) { + tp->die_cpus_list[i] = buf; + tp->die_cpus_lists++; + buf = NULL; + len = 0; + } + ret = 0; + +try_threads: + scnprintf(filename, MAXPATHLEN, CORE_CPUS_FMT, + sysfs__mountpoint(), cpu); + if (access(filename, F_OK) == -1) { + scnprintf(filename, MAXPATHLEN, CORE_CPUS_FMT_OLD, + sysfs__mountpoint(), cpu); + } + fp = fopen(filename, "r"); + if (!fp) + goto done; + + if (getline(&buf, &len, fp) <= 0) + goto done; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + for (i = 0; i < tp->core_cpus_lists; i++) { + if (!strcmp(buf, tp->core_cpus_list[i])) + break; + } + if (i == tp->core_cpus_lists) { + tp->core_cpus_list[i] = buf; + tp->core_cpus_lists++; + buf = NULL; + } + ret = 0; +done: + if (fp) + fclose(fp); + free(buf); + return ret; +} + +void cpu_topology__delete(struct cpu_topology *tp) +{ + u32 i; + + if (!tp) + return; + + for (i = 0 ; i < tp->package_cpus_lists; i++) + zfree(&tp->package_cpus_list[i]); + + for (i = 0 ; i < tp->die_cpus_lists; i++) + zfree(&tp->die_cpus_list[i]); + + for (i = 0 ; i < tp->core_cpus_lists; i++) + zfree(&tp->core_cpus_list[i]); + + free(tp); +} + +bool cpu_topology__smt_on(const struct cpu_topology *topology) +{ + for (u32 i = 0; i < topology->core_cpus_lists; i++) { + const char *cpu_list = topology->core_cpus_list[i]; + + /* + * If there is a need to separate siblings in a core then SMT is + * enabled. + */ + if (strchr(cpu_list, ',') || strchr(cpu_list, '-')) + return true; + } + return false; +} + +bool cpu_topology__core_wide(const struct cpu_topology *topology, + const char *user_requested_cpu_list) +{ + struct perf_cpu_map *user_requested_cpus; + + /* + * If user_requested_cpu_list is empty then all CPUs are recorded and so + * core_wide is true. + */ + if (!user_requested_cpu_list) + return true; + + user_requested_cpus = perf_cpu_map__new(user_requested_cpu_list); + /* Check that every user requested CPU is the complete set of SMT threads on a core. */ + for (u32 i = 0; i < topology->core_cpus_lists; i++) { + const char *core_cpu_list = topology->core_cpus_list[i]; + struct perf_cpu_map *core_cpus = perf_cpu_map__new(core_cpu_list); + struct perf_cpu cpu; + unsigned int idx; + bool has_first, first = true; + + perf_cpu_map__for_each_cpu(cpu, idx, core_cpus) { + if (first) { + has_first = perf_cpu_map__has(user_requested_cpus, cpu); + first = false; + } else { + /* + * If the first core CPU is user requested then + * all subsequent CPUs in the core must be user + * requested too. If the first CPU isn't user + * requested then none of the others must be + * too. + */ + if (perf_cpu_map__has(user_requested_cpus, cpu) != has_first) { + perf_cpu_map__put(core_cpus); + perf_cpu_map__put(user_requested_cpus); + return false; + } + } + } + perf_cpu_map__put(core_cpus); + } + perf_cpu_map__put(user_requested_cpus); + return true; +} + +static bool has_die_topology(void) +{ + char filename[MAXPATHLEN]; + struct utsname uts; + + if (uname(&uts) < 0) + return false; + + if (strncmp(uts.machine, "x86_64", 6) && + strncmp(uts.machine, "s390x", 5)) + return false; + + scnprintf(filename, MAXPATHLEN, DIE_CPUS_FMT, + sysfs__mountpoint(), 0); + if (access(filename, F_OK) == -1) + return false; + + return true; +} + +const struct cpu_topology *online_topology(void) +{ + static const struct cpu_topology *topology; + + if (!topology) { + topology = cpu_topology__new(); + if (!topology) { + pr_err("Error creating CPU topology"); + abort(); + } + } + return topology; +} + +struct cpu_topology *cpu_topology__new(void) +{ + struct cpu_topology *tp = NULL; + void *addr; + u32 nr, i, nr_addr; + size_t sz; + long ncpus; + int ret = -1; + struct perf_cpu_map *map; + bool has_die = has_die_topology(); + + ncpus = cpu__max_present_cpu().cpu; + + /* build online CPU map */ + map = perf_cpu_map__new_online_cpus(); + if (map == NULL) { + pr_debug("failed to get system cpumap\n"); + return NULL; + } + + nr = (u32)(ncpus & UINT_MAX); + + sz = nr * sizeof(char *); + if (has_die) + nr_addr = 3; + else + nr_addr = 2; + addr = calloc(1, sizeof(*tp) + nr_addr * sz); + if (!addr) + goto out_free; + + tp = addr; + addr += sizeof(*tp); + tp->package_cpus_list = addr; + addr += sz; + if (has_die) { + tp->die_cpus_list = addr; + addr += sz; + } + tp->core_cpus_list = addr; + + for (i = 0; i < nr; i++) { + if (!perf_cpu_map__has(map, (struct perf_cpu){ .cpu = i })) + continue; + + ret = build_cpu_topology(tp, i); + if (ret < 0) + break; + } + +out_free: + perf_cpu_map__put(map); + if (ret) { + cpu_topology__delete(tp); + tp = NULL; + } + return tp; +} + +static int load_numa_node(struct numa_topology_node *node, int nr) +{ + char str[MAXPATHLEN]; + char field[32]; + char *buf = NULL, *p; + size_t len = 0; + int ret = -1; + FILE *fp; + u64 mem; + + node->node = (u32) nr; + + scnprintf(str, MAXPATHLEN, NODE_MEMINFO_FMT, + sysfs__mountpoint(), nr); + fp = fopen(str, "r"); + if (!fp) + return -1; + + while (getline(&buf, &len, fp) > 0) { + /* skip over invalid lines */ + if (!strchr(buf, ':')) + continue; + if (sscanf(buf, "%*s %*d %31s %"PRIu64, field, &mem) != 2) + goto err; + if (!strcmp(field, "MemTotal:")) + node->mem_total = mem; + if (!strcmp(field, "MemFree:")) + node->mem_free = mem; + if (node->mem_total && node->mem_free) + break; + } + + fclose(fp); + fp = NULL; + + scnprintf(str, MAXPATHLEN, NODE_CPULIST_FMT, + sysfs__mountpoint(), nr); + + fp = fopen(str, "r"); + if (!fp) + return -1; + + if (getline(&buf, &len, fp) <= 0) + goto err; + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + node->cpus = buf; + fclose(fp); + return 0; + +err: + free(buf); + if (fp) + fclose(fp); + return ret; +} + +struct numa_topology *numa_topology__new(void) +{ + struct perf_cpu_map *node_map = NULL; + struct numa_topology *tp = NULL; + char path[MAXPATHLEN]; + char *buf = NULL; + size_t len = 0; + u32 nr, i; + FILE *fp; + char *c; + + scnprintf(path, MAXPATHLEN, NODE_ONLINE_FMT, + sysfs__mountpoint()); + + fp = fopen(path, "r"); + if (!fp) + return NULL; + + if (getline(&buf, &len, fp) <= 0) + goto out; + + c = strchr(buf, '\n'); + if (c) + *c = '\0'; + + node_map = perf_cpu_map__new(buf); + if (!node_map) + goto out; + + nr = (u32) perf_cpu_map__nr(node_map); + + tp = zalloc(sizeof(*tp) + sizeof(tp->nodes[0])*nr); + if (!tp) + goto out; + + tp->nr = nr; + + for (i = 0; i < nr; i++) { + if (load_numa_node(&tp->nodes[i], perf_cpu_map__cpu(node_map, i).cpu)) { + numa_topology__delete(tp); + tp = NULL; + break; + } + } + +out: + free(buf); + fclose(fp); + perf_cpu_map__put(node_map); + return tp; +} + +void numa_topology__delete(struct numa_topology *tp) +{ + u32 i; + + for (i = 0; i < tp->nr; i++) + zfree(&tp->nodes[i].cpus); + + free(tp); +} + +static int load_hybrid_node(struct hybrid_topology_node *node, + struct perf_pmu *pmu) +{ + char *buf = NULL, *p; + FILE *fp; + size_t len = 0; + + node->pmu_name = strdup(pmu->name); + if (!node->pmu_name) + return -1; + + fp = perf_pmu__open_file(pmu, "cpus"); + if (!fp) + goto err; + + if (getline(&buf, &len, fp) <= 0) { + fclose(fp); + goto err; + } + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + fclose(fp); + node->cpus = buf; + return 0; + +err: + zfree(&node->pmu_name); + free(buf); + return -1; +} + +struct hybrid_topology *hybrid_topology__new(void) +{ + struct perf_pmu *pmu = NULL; + struct hybrid_topology *tp = NULL; + int nr = perf_pmus__num_core_pmus(), i = 0; + + if (nr <= 1) + return NULL; + + tp = zalloc(sizeof(*tp) + sizeof(tp->nodes[0]) * nr); + if (!tp) + return NULL; + + tp->nr = nr; + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + if (load_hybrid_node(&tp->nodes[i], pmu)) { + hybrid_topology__delete(tp); + return NULL; + } + i++; + } + + return tp; +} + +void hybrid_topology__delete(struct hybrid_topology *tp) +{ + u32 i; + + for (i = 0; i < tp->nr; i++) { + zfree(&tp->nodes[i].pmu_name); + zfree(&tp->nodes[i].cpus); + } + + free(tp); +} diff --git a/tools/perf/util/cputopo.h b/tools/perf/util/cputopo.h new file mode 100644 index 000000000..8d42f6102 --- /dev/null +++ b/tools/perf/util/cputopo.h @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_CPUTOPO_H +#define __PERF_CPUTOPO_H + +#include + +struct cpu_topology { + /* The number of unique package_cpus_lists below. */ + u32 package_cpus_lists; + /* The number of unique die_cpu_lists below. */ + u32 die_cpus_lists; + /* The number of unique core_cpu_lists below. */ + u32 core_cpus_lists; + /* + * An array of strings where each string is unique and read from + * /sys/devices/system/cpu/cpuX/topology/package_cpus_list. From the ABI + * each of these is a human-readable list of CPUs sharing the same + * physical_package_id. The format is like 0-3, 8-11, 14,17. + */ + const char **package_cpus_list; + /* + * An array of string where each string is unique and from + * /sys/devices/system/cpu/cpuX/topology/die_cpus_list. From the ABI + * each of these is a human-readable list of CPUs within the same die. + * The format is like 0-3, 8-11, 14,17. + */ + const char **die_cpus_list; + /* + * An array of string where each string is unique and from + * /sys/devices/system/cpu/cpuX/topology/core_cpus_list. From the ABI + * each of these is a human-readable list of CPUs within the same + * core. The format is like 0-3, 8-11, 14,17. + */ + const char **core_cpus_list; +}; + +struct numa_topology_node { + char *cpus; + u32 node; + u64 mem_total; + u64 mem_free; +}; + +struct numa_topology { + u32 nr; + struct numa_topology_node nodes[]; +}; + +struct hybrid_topology_node { + char *pmu_name; + char *cpus; +}; + +struct hybrid_topology { + u32 nr; + struct hybrid_topology_node nodes[]; +}; + +/* + * The topology for online CPUs, lazily created. + */ +const struct cpu_topology *online_topology(void); + +struct cpu_topology *cpu_topology__new(void); +void cpu_topology__delete(struct cpu_topology *tp); +/* Determine from the core list whether SMT was enabled. */ +bool cpu_topology__smt_on(const struct cpu_topology *topology); +/* Are the sets of SMT siblings all enabled or all disabled in user_requested_cpus. */ +bool cpu_topology__core_wide(const struct cpu_topology *topology, + const char *user_requested_cpu_list); + +struct numa_topology *numa_topology__new(void); +void numa_topology__delete(struct numa_topology *tp); + +struct hybrid_topology *hybrid_topology__new(void); +void hybrid_topology__delete(struct hybrid_topology *tp); + +#endif /* __PERF_CPUTOPO_H */ diff --git a/tools/perf/util/cs-etm-base.c b/tools/perf/util/cs-etm-base.c new file mode 100644 index 000000000..aebef71d3 --- /dev/null +++ b/tools/perf/util/cs-etm-base.c @@ -0,0 +1,195 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * File for any parts of the Coresight decoding that don't require + * OpenCSD. + */ + +#include +#include + +#include "cs-etm.h" + +static const char * const cs_etm_global_header_fmts[] = { + [CS_HEADER_VERSION] = " Header version %llx\n", + [CS_PMU_TYPE_CPUS] = " PMU type/num cpus %llx\n", + [CS_ETM_SNAPSHOT] = " Snapshot %llx\n", +}; + +static const char * const cs_etm_priv_fmts[] = { + [CS_ETM_MAGIC] = " Magic number %llx\n", + [CS_ETM_CPU] = " CPU %lld\n", + [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", + [CS_ETM_ETMCR] = " ETMCR %llx\n", + [CS_ETM_ETMTRACEIDR] = " ETMTRACEIDR %llx\n", + [CS_ETM_ETMCCER] = " ETMCCER %llx\n", + [CS_ETM_ETMIDR] = " ETMIDR %llx\n", +}; + +static const char * const cs_etmv4_priv_fmts[] = { + [CS_ETM_MAGIC] = " Magic number %llx\n", + [CS_ETM_CPU] = " CPU %lld\n", + [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", + [CS_ETMV4_TRCCONFIGR] = " TRCCONFIGR %llx\n", + [CS_ETMV4_TRCTRACEIDR] = " TRCTRACEIDR %llx\n", + [CS_ETMV4_TRCIDR0] = " TRCIDR0 %llx\n", + [CS_ETMV4_TRCIDR1] = " TRCIDR1 %llx\n", + [CS_ETMV4_TRCIDR2] = " TRCIDR2 %llx\n", + [CS_ETMV4_TRCIDR8] = " TRCIDR8 %llx\n", + [CS_ETMV4_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n", + [CS_ETMV4_TS_SOURCE] = " TS_SOURCE %lld\n", +}; + +static const char * const cs_ete_priv_fmts[] = { + [CS_ETM_MAGIC] = " Magic number %llx\n", + [CS_ETM_CPU] = " CPU %lld\n", + [CS_ETM_NR_TRC_PARAMS] = " NR_TRC_PARAMS %llx\n", + [CS_ETE_TRCCONFIGR] = " TRCCONFIGR %llx\n", + [CS_ETE_TRCTRACEIDR] = " TRCTRACEIDR %llx\n", + [CS_ETE_TRCIDR0] = " TRCIDR0 %llx\n", + [CS_ETE_TRCIDR1] = " TRCIDR1 %llx\n", + [CS_ETE_TRCIDR2] = " TRCIDR2 %llx\n", + [CS_ETE_TRCIDR8] = " TRCIDR8 %llx\n", + [CS_ETE_TRCAUTHSTATUS] = " TRCAUTHSTATUS %llx\n", + [CS_ETE_TRCDEVARCH] = " TRCDEVARCH %llx\n", + [CS_ETE_TS_SOURCE] = " TS_SOURCE %lld\n", +}; + +static const char * const param_unk_fmt = + " Unknown parameter [%d] %"PRIx64"\n"; +static const char * const magic_unk_fmt = + " Magic number Unknown %"PRIx64"\n"; + +static int cs_etm__print_cpu_metadata_v0(u64 *val, int *offset) +{ + int i = *offset, j, nr_params = 0, fmt_offset; + u64 magic; + + /* check magic value */ + magic = val[i + CS_ETM_MAGIC]; + if ((magic != __perf_cs_etmv3_magic) && + (magic != __perf_cs_etmv4_magic)) { + /* failure - note bad magic value */ + fprintf(stdout, magic_unk_fmt, magic); + return -EINVAL; + } + + /* print common header block */ + fprintf(stdout, cs_etm_priv_fmts[CS_ETM_MAGIC], val[i++]); + fprintf(stdout, cs_etm_priv_fmts[CS_ETM_CPU], val[i++]); + + if (magic == __perf_cs_etmv3_magic) { + nr_params = CS_ETM_NR_TRC_PARAMS_V0; + fmt_offset = CS_ETM_ETMCR; + /* after common block, offset format index past NR_PARAMS */ + for (j = fmt_offset; j < nr_params + fmt_offset; j++, i++) + fprintf(stdout, cs_etm_priv_fmts[j], val[i]); + } else if (magic == __perf_cs_etmv4_magic) { + nr_params = CS_ETMV4_NR_TRC_PARAMS_V0; + fmt_offset = CS_ETMV4_TRCCONFIGR; + /* after common block, offset format index past NR_PARAMS */ + for (j = fmt_offset; j < nr_params + fmt_offset; j++, i++) + fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); + } + *offset = i; + return 0; +} + +static int cs_etm__print_cpu_metadata_v1(u64 *val, int *offset) +{ + int i = *offset, j, total_params = 0; + u64 magic; + + magic = val[i + CS_ETM_MAGIC]; + /* total params to print is NR_PARAMS + common block size for v1 */ + total_params = val[i + CS_ETM_NR_TRC_PARAMS] + CS_ETM_COMMON_BLK_MAX_V1; + + if (magic == __perf_cs_etmv3_magic) { + for (j = 0; j < total_params; j++, i++) { + /* if newer record - could be excess params */ + if (j >= CS_ETM_PRIV_MAX) + fprintf(stdout, param_unk_fmt, j, val[i]); + else + fprintf(stdout, cs_etm_priv_fmts[j], val[i]); + } + } else if (magic == __perf_cs_etmv4_magic) { + for (j = 0; j < total_params; j++, i++) { + /* if newer record - could be excess params */ + if (j >= CS_ETMV4_PRIV_MAX) + fprintf(stdout, param_unk_fmt, j, val[i]); + else + fprintf(stdout, cs_etmv4_priv_fmts[j], val[i]); + } + } else if (magic == __perf_cs_ete_magic) { + for (j = 0; j < total_params; j++, i++) { + /* if newer record - could be excess params */ + if (j >= CS_ETE_PRIV_MAX) + fprintf(stdout, param_unk_fmt, j, val[i]); + else + fprintf(stdout, cs_ete_priv_fmts[j], val[i]); + } + } else { + /* failure - note bad magic value and error out */ + fprintf(stdout, magic_unk_fmt, magic); + return -EINVAL; + } + *offset = i; + return 0; +} + +static void cs_etm__print_auxtrace_info(u64 *val, int num) +{ + int i, cpu = 0, version, err; + + version = val[0]; + + for (i = 0; i < CS_HEADER_VERSION_MAX; i++) + fprintf(stdout, cs_etm_global_header_fmts[i], val[i]); + + for (i = CS_HEADER_VERSION_MAX; cpu < num; cpu++) { + if (version == 0) + err = cs_etm__print_cpu_metadata_v0(val, &i); + /* printing same for both, but value bit flags added on v2 */ + else if ((version == 1) || (version == 2)) + err = cs_etm__print_cpu_metadata_v1(val, &i); + if (err) + return; + } +} + +/* + * Do some basic checks and print the auxtrace info header before calling + * into cs_etm__process_auxtrace_info_full() which requires OpenCSD to be + * linked in. This allows some basic debugging if OpenCSD is missing. + */ +int cs_etm__process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + int event_header_size = sizeof(struct perf_event_header); + int num_cpu; + u64 *ptr = NULL; + u64 hdr_version; + + /* Ensure priv[] is large enough for the global header entries */ + if (auxtrace_info->header.size < (event_header_size + INFO_HEADER_SIZE + + CS_ETM_HEADER_SIZE)) + return -EINVAL; + + /* First the global part */ + ptr = (u64 *) auxtrace_info->priv; + + /* Look for version of the header */ + hdr_version = ptr[0]; + if (hdr_version > CS_HEADER_CURRENT_VERSION) { + pr_err("\nCS ETM Trace: Unknown Header Version = %#" PRIx64, hdr_version); + pr_err(", version supported <= %x\n", CS_HEADER_CURRENT_VERSION); + return -EINVAL; + } + + if (dump_trace) { + num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + cs_etm__print_auxtrace_info(ptr, num_cpu); + } + + return cs_etm__process_auxtrace_info_full(event, session); +} diff --git a/tools/perf/util/cs-etm-decoder/Build b/tools/perf/util/cs-etm-decoder/Build new file mode 100644 index 000000000..27550db2a --- /dev/null +++ b/tools/perf/util/cs-etm-decoder/Build @@ -0,0 +1 @@ +perf-util-y += cs-etm-decoder.o diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c new file mode 100644 index 000000000..26940f1f1 --- /dev/null +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -0,0 +1,809 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(C) 2015-2018 Linaro Limited. + * + * Author: Tor Jeremiassen + * Author: Mathieu Poirier + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "cs-etm.h" +#include "cs-etm-decoder.h" +#include "debug.h" +#include "intlist.h" + +/* use raw logging */ +#ifdef CS_DEBUG_RAW +#define CS_LOG_RAW_FRAMES +#define CS_PKT_MON 1 +#ifdef CS_RAW_PACKED +#define CS_RAW_DEBUG_FLAGS (OCSD_DFRMTR_UNPACKED_RAW_OUT | \ + OCSD_DFRMTR_PACKED_RAW_OUT) +#else +#define CS_RAW_DEBUG_FLAGS (OCSD_DFRMTR_UNPACKED_RAW_OUT) +#endif +#else +#define CS_PKT_MON 0 +#endif + +/* + * Assume a maximum of 0.1ns elapsed per instruction. This would be the + * case with a theoretical 10GHz core executing 1 instruction per cycle. + * Used to estimate the sample time for synthesized instructions because + * Coresight only emits a timestamp for a range of instructions rather + * than per instruction. + */ +const u32 INSTR_PER_NS = 10; + +struct cs_etm_decoder { + void *data; + void (*packet_printer)(const char *msg, void *data); + bool suppress_printing; + dcd_tree_handle_t dcd_tree; + cs_etm_mem_cb_type mem_access; + ocsd_datapath_resp_t prev_return; + const char *decoder_name; +}; + +static u32 +cs_etm_decoder__mem_access(const void *context, + const ocsd_vaddr_t address, + const ocsd_mem_space_acc_t mem_space, + const u8 trace_chan_id, + const u32 req_size, + u8 *buffer) +{ + struct cs_etm_decoder *decoder = (struct cs_etm_decoder *) context; + + return decoder->mem_access(decoder->data, trace_chan_id, address, + req_size, buffer, mem_space); +} + +int cs_etm_decoder__add_mem_access_cb(struct cs_etm_decoder *decoder, + u64 start, u64 end, + cs_etm_mem_cb_type cb_func) +{ + decoder->mem_access = cb_func; + + if (ocsd_dt_add_callback_trcid_mem_acc(decoder->dcd_tree, start, end, + OCSD_MEM_SPACE_ANY, + cs_etm_decoder__mem_access, + decoder)) + return -1; + + return 0; +} + +int cs_etm_decoder__reset(struct cs_etm_decoder *decoder) +{ + ocsd_datapath_resp_t dp_ret; + + decoder->prev_return = OCSD_RESP_CONT; + decoder->suppress_printing = true; + dp_ret = ocsd_dt_process_data(decoder->dcd_tree, OCSD_OP_RESET, + 0, 0, NULL, NULL); + decoder->suppress_printing = false; + if (OCSD_DATA_RESP_IS_FATAL(dp_ret)) + return -1; + + return 0; +} + +int cs_etm_decoder__get_packet(struct cs_etm_packet_queue *packet_queue, + struct cs_etm_packet *packet) +{ + if (!packet_queue || !packet) + return -EINVAL; + + /* Nothing to do, might as well just return */ + if (packet_queue->packet_count == 0) + return 0; + /* + * The queueing process in function cs_etm_decoder__buffer_packet() + * increments the tail *before* using it. This is somewhat counter + * intuitive but it has the advantage of centralizing tail management + * at a single location. Because of that we need to follow the same + * heuristic with the head, i.e we increment it before using its + * value. Otherwise the first element of the packet queue is not + * used. + */ + packet_queue->head = (packet_queue->head + 1) & + (CS_ETM_PACKET_MAX_BUFFER - 1); + + *packet = packet_queue->packet_buffer[packet_queue->head]; + + packet_queue->packet_count--; + + return 1; +} + +/* + * Calculate the number of nanoseconds elapsed. + * + * instr_count is updated in place with the remainder of the instructions + * which didn't make up a whole nanosecond. + */ +static u32 cs_etm_decoder__dec_instr_count_to_ns(u32 *instr_count) +{ + const u32 instr_copy = *instr_count; + + *instr_count %= INSTR_PER_NS; + return instr_copy / INSTR_PER_NS; +} + +static int cs_etm_decoder__gen_etmv3_config(struct cs_etm_trace_params *params, + ocsd_etmv3_cfg *config) +{ + config->reg_idr = params->etmv3.reg_idr; + config->reg_ctrl = params->etmv3.reg_ctrl; + config->reg_ccer = params->etmv3.reg_ccer; + config->reg_trc_id = params->etmv3.reg_trc_id; + config->arch_ver = ARCH_V7; + config->core_prof = profile_CortexA; + + return 0; +} + +#define TRCIDR1_TRCARCHMIN_SHIFT 4 +#define TRCIDR1_TRCARCHMIN_MASK GENMASK(7, 4) +#define TRCIDR1_TRCARCHMIN(x) (((x) & TRCIDR1_TRCARCHMIN_MASK) >> TRCIDR1_TRCARCHMIN_SHIFT) + +static enum _ocsd_arch_version cs_etm_decoder__get_etmv4_arch_ver(u32 reg_idr1) +{ + /* + * For ETMv4 if the trace minor version is 4 or more then we can assume + * the architecture is ARCH_AA64 rather than just V8. + * ARCH_V8 = V8 architecture + * ARCH_AA64 = Min v8r3 plus additional AA64 PE features + */ + return TRCIDR1_TRCARCHMIN(reg_idr1) >= 4 ? ARCH_AA64 : ARCH_V8; +} + +static void cs_etm_decoder__gen_etmv4_config(struct cs_etm_trace_params *params, + ocsd_etmv4_cfg *config) +{ + config->reg_configr = params->etmv4.reg_configr; + config->reg_traceidr = params->etmv4.reg_traceidr; + config->reg_idr0 = params->etmv4.reg_idr0; + config->reg_idr1 = params->etmv4.reg_idr1; + config->reg_idr2 = params->etmv4.reg_idr2; + config->reg_idr8 = params->etmv4.reg_idr8; + config->reg_idr9 = 0; + config->reg_idr10 = 0; + config->reg_idr11 = 0; + config->reg_idr12 = 0; + config->reg_idr13 = 0; + config->arch_ver = cs_etm_decoder__get_etmv4_arch_ver(params->etmv4.reg_idr1); + config->core_prof = profile_CortexA; +} + +static void cs_etm_decoder__gen_ete_config(struct cs_etm_trace_params *params, + ocsd_ete_cfg *config) +{ + config->reg_configr = params->ete.reg_configr; + config->reg_traceidr = params->ete.reg_traceidr; + config->reg_idr0 = params->ete.reg_idr0; + config->reg_idr1 = params->ete.reg_idr1; + config->reg_idr2 = params->ete.reg_idr2; + config->reg_idr8 = params->ete.reg_idr8; + config->reg_devarch = params->ete.reg_devarch; + config->arch_ver = ARCH_AA64; + config->core_prof = profile_CortexA; +} + +static void cs_etm_decoder__print_str_cb(const void *p_context, + const char *msg, + const int str_len) +{ + const struct cs_etm_decoder *decoder = p_context; + + if (p_context && str_len && !decoder->suppress_printing) + decoder->packet_printer(msg, decoder->data); +} + +static int +cs_etm_decoder__init_def_logger_printing(struct cs_etm_decoder_params *d_params, + struct cs_etm_decoder *decoder) +{ + int ret = 0; + + if (d_params->packet_printer == NULL) + return -1; + + decoder->packet_printer = d_params->packet_printer; + + /* + * Set up a library default logger to process any printers + * (packet/raw frame) we add later. + */ + ret = ocsd_def_errlog_init(OCSD_ERR_SEV_ERROR, 1); + if (ret != 0) + return -1; + + /* no stdout / err / file output */ + ret = ocsd_def_errlog_config_output(C_API_MSGLOGOUT_FLG_NONE, NULL); + if (ret != 0) + return -1; + + /* + * Set the string CB for the default logger, passes strings to + * perf print logger. + */ + ret = ocsd_def_errlog_set_strprint_cb(decoder->dcd_tree, + (void *)decoder, + cs_etm_decoder__print_str_cb); + if (ret != 0) + return -1; + +#ifdef CS_LOG_RAW_FRAMES + /* + * Only log raw frames if --dump operation and hardware is actually + * generating formatted CoreSight trace frames + */ + if ((d_params->operation == CS_ETM_OPERATION_PRINT) && + (d_params->formatted == true)) { + /* use the built in library printer for the raw frames */ + ret = ocsd_dt_set_raw_frame_printer(decoder->dcd_tree, + CS_RAW_DEBUG_FLAGS); + if (ret != 0) + return -1; + } +#endif + return 0; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__do_soft_timestamp(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const uint8_t trace_chan_id) +{ + u64 estimated_ts; + + /* No timestamp packet has been received, nothing to do */ + if (!packet_queue->next_cs_timestamp) + return OCSD_RESP_CONT; + + estimated_ts = packet_queue->cs_timestamp + + cs_etm_decoder__dec_instr_count_to_ns(&packet_queue->instr_count); + + /* Estimated TS can never be higher than the next real one in the trace */ + packet_queue->cs_timestamp = min(packet_queue->next_cs_timestamp, estimated_ts); + + /* Tell the front end which traceid_queue needs attention */ + cs_etm__etmq_set_traceid_queue_timestamp(etmq, trace_chan_id); + + return OCSD_RESP_WAIT; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__do_hard_timestamp(struct cs_etm_queue *etmq, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id, + const ocsd_trc_index_t indx) +{ + struct cs_etm_packet_queue *packet_queue; + u64 converted_timestamp; + u64 estimated_first_ts; + + /* First get the packet queue for this traceID */ + packet_queue = cs_etm__etmq_get_packet_queue(etmq, trace_chan_id); + if (!packet_queue) + return OCSD_RESP_FATAL_SYS_ERR; + + /* + * Coresight timestamps are raw timer values which need to be scaled to ns. Assume + * 0 is a bad value so don't try to convert it. + */ + converted_timestamp = elem->timestamp ? + cs_etm__convert_sample_time(etmq, elem->timestamp) : 0; + + /* + * We've seen a timestamp packet before - simply record the new value. + * Function do_soft_timestamp() will report the value to the front end, + * hence asking the decoder to keep decoding rather than stopping. + */ + if (packet_queue->next_cs_timestamp) { + /* + * What was next is now where new ranges start from, overwriting + * any previous estimate in cs_timestamp + */ + packet_queue->cs_timestamp = packet_queue->next_cs_timestamp; + packet_queue->next_cs_timestamp = converted_timestamp; + return OCSD_RESP_CONT; + } + + if (!converted_timestamp) { + /* + * Zero timestamps can be seen due to misconfiguration or hardware bugs. + * Warn once, and don't try to subtract instr_count as it would result in an + * underflow. + */ + packet_queue->cs_timestamp = 0; + if (!cs_etm__etmq_is_timeless(etmq)) + pr_warning_once("Zero Coresight timestamp found at Idx:%" OCSD_TRC_IDX_STR + ". Decoding may be improved by prepending 'Z' to your current --itrace arguments.\n", + indx); + + } else if (packet_queue->instr_count / INSTR_PER_NS > converted_timestamp) { + /* + * Sanity check that the elem->timestamp - packet_queue->instr_count would not + * result in an underflow. Warn and clamp at 0 if it would. + */ + packet_queue->cs_timestamp = 0; + pr_err("Timestamp calculation underflow at Idx:%" OCSD_TRC_IDX_STR "\n", indx); + } else { + /* + * This is the first timestamp we've seen since the beginning of traces + * or a discontinuity. Since timestamps packets are generated *after* + * range packets have been generated, we need to estimate the time at + * which instructions started by subtracting the number of instructions + * executed to the timestamp. Don't estimate earlier than the last used + * timestamp though. + */ + estimated_first_ts = converted_timestamp - + (packet_queue->instr_count / INSTR_PER_NS); + packet_queue->cs_timestamp = max(packet_queue->cs_timestamp, estimated_first_ts); + } + packet_queue->next_cs_timestamp = converted_timestamp; + packet_queue->instr_count = 0; + + /* Tell the front end which traceid_queue needs attention */ + cs_etm__etmq_set_traceid_queue_timestamp(etmq, trace_chan_id); + + /* Halt processing until we are being told to proceed */ + return OCSD_RESP_WAIT; +} + +static void +cs_etm_decoder__reset_timestamp(struct cs_etm_packet_queue *packet_queue) +{ + packet_queue->next_cs_timestamp = 0; + packet_queue->instr_count = 0; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_packet(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const u8 trace_chan_id, + enum cs_etm_sample_type sample_type) +{ + u32 et = 0; + int cpu; + + if (packet_queue->packet_count >= CS_ETM_PACKET_MAX_BUFFER - 1) + return OCSD_RESP_FATAL_SYS_ERR; + + if (cs_etm__get_cpu(etmq, trace_chan_id, &cpu) < 0) + return OCSD_RESP_FATAL_SYS_ERR; + + et = packet_queue->tail; + et = (et + 1) & (CS_ETM_PACKET_MAX_BUFFER - 1); + packet_queue->tail = et; + packet_queue->packet_count++; + + packet_queue->packet_buffer[et].sample_type = sample_type; + packet_queue->packet_buffer[et].isa = CS_ETM_ISA_UNKNOWN; + packet_queue->packet_buffer[et].cpu = cpu; + packet_queue->packet_buffer[et].start_addr = CS_ETM_INVAL_ADDR; + packet_queue->packet_buffer[et].end_addr = CS_ETM_INVAL_ADDR; + packet_queue->packet_buffer[et].instr_count = 0; + packet_queue->packet_buffer[et].last_instr_taken_branch = false; + packet_queue->packet_buffer[et].last_instr_size = 0; + packet_queue->packet_buffer[et].last_instr_type = 0; + packet_queue->packet_buffer[et].last_instr_subtype = 0; + packet_queue->packet_buffer[et].last_instr_cond = 0; + packet_queue->packet_buffer[et].flags = 0; + packet_queue->packet_buffer[et].exception_number = UINT32_MAX; + packet_queue->packet_buffer[et].trace_chan_id = trace_chan_id; + packet_queue->packet_buffer[et].el = ocsd_EL_unknown; + packet_queue->packet_buffer[et].tid = -1; + + if (packet_queue->packet_count == CS_ETM_PACKET_MAX_BUFFER - 1) + return OCSD_RESP_WAIT; + + return OCSD_RESP_CONT; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_range(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id) +{ + int ret = 0; + struct cs_etm_packet *packet; + + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_RANGE); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + + switch (elem->isa) { + case ocsd_isa_aarch64: + packet->isa = CS_ETM_ISA_A64; + break; + case ocsd_isa_arm: + packet->isa = CS_ETM_ISA_A32; + break; + case ocsd_isa_thumb2: + packet->isa = CS_ETM_ISA_T32; + break; + case ocsd_isa_tee: + case ocsd_isa_jazelle: + case ocsd_isa_custom: + case ocsd_isa_unknown: + default: + packet->isa = CS_ETM_ISA_UNKNOWN; + } + + packet->start_addr = elem->st_addr; + packet->end_addr = elem->en_addr; + packet->instr_count = elem->num_instr_range; + packet->last_instr_type = elem->last_i_type; + packet->last_instr_subtype = elem->last_i_subtype; + packet->last_instr_cond = elem->last_instr_cond; + packet->el = elem->context.exception_level; + + if (elem->last_i_type == OCSD_INSTR_BR || elem->last_i_type == OCSD_INSTR_BR_INDIRECT) + packet->last_instr_taken_branch = elem->last_instr_exec; + else + packet->last_instr_taken_branch = false; + + packet->last_instr_size = elem->last_instr_sz; + + /* per-thread scenario, no need to generate a timestamp */ + if (cs_etm__etmq_is_timeless(etmq)) + goto out; + + /* + * The packet queue is full and we haven't seen a timestamp (had we + * seen one the packet queue wouldn't be full). Let the front end + * deal with it. + */ + if (ret == OCSD_RESP_WAIT) + goto out; + + packet_queue->instr_count += elem->num_instr_range; + /* Tell the front end we have a new timestamp to process */ + ret = cs_etm_decoder__do_soft_timestamp(etmq, packet_queue, + trace_chan_id); +out: + return ret; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_discontinuity(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *queue, + const uint8_t trace_chan_id) +{ + /* + * Something happened and who knows when we'll get new traces so + * reset time statistics. + */ + cs_etm_decoder__reset_timestamp(queue); + return cs_etm_decoder__buffer_packet(etmq, queue, trace_chan_id, + CS_ETM_DISCONTINUITY); +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_exception(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *queue, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id) +{ int ret = 0; + struct cs_etm_packet *packet; + + ret = cs_etm_decoder__buffer_packet(etmq, queue, trace_chan_id, + CS_ETM_EXCEPTION); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &queue->packet_buffer[queue->tail]; + packet->exception_number = elem->exception_number; + + return ret; +} + +static ocsd_datapath_resp_t +cs_etm_decoder__buffer_exception_ret(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *queue, + const uint8_t trace_chan_id) +{ + return cs_etm_decoder__buffer_packet(etmq, queue, trace_chan_id, + CS_ETM_EXCEPTION_RET); +} + +static ocsd_datapath_resp_t +cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, + struct cs_etm_packet_queue *packet_queue, + const ocsd_generic_trace_elem *elem, + const uint8_t trace_chan_id) +{ + struct cs_etm_packet *packet; + pid_t tid = -1; + int ret; + + /* + * Process the PE_CONTEXT packets if we have a valid contextID or VMID. + * If the kernel is running at EL2, the PID is traced in CONTEXTIDR_EL2 + * as VMID, Format attribute 'contextid2' is set in this case. + */ + switch (cs_etm__get_pid_fmt(etmq)) { + case CS_ETM_PIDFMT_CTXTID: + if (elem->context.ctxt_id_valid) + tid = elem->context.context_id; + break; + case CS_ETM_PIDFMT_CTXTID2: + if (elem->context.vmid_valid) + tid = elem->context.vmid; + break; + case CS_ETM_PIDFMT_NONE: + default: + break; + } + + if (cs_etm__etmq_update_decode_context(etmq, trace_chan_id, + elem->context.exception_level, tid)) + return OCSD_RESP_FATAL_SYS_ERR; + + ret = cs_etm_decoder__buffer_packet(etmq, packet_queue, trace_chan_id, + CS_ETM_CONTEXT); + if (ret != OCSD_RESP_CONT && ret != OCSD_RESP_WAIT) + return ret; + + packet = &packet_queue->packet_buffer[packet_queue->tail]; + packet->tid = tid; + packet->el = elem->context.exception_level; + + /* + * A timestamp is generated after a PE_CONTEXT element so make sure + * to rely on that coming one. + */ + cs_etm_decoder__reset_timestamp(packet_queue); + + return ret; +} + +static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( + const void *context, + const ocsd_trc_index_t indx, + const u8 trace_chan_id __maybe_unused, + const ocsd_generic_trace_elem *elem) +{ + ocsd_datapath_resp_t resp = OCSD_RESP_CONT; + ocsd_gen_trc_elem_t type; + struct cs_etm_decoder *decoder = (struct cs_etm_decoder *) context; + struct cs_etm_queue *etmq = decoder->data; + struct cs_etm_packet_queue *packet_queue; + + /* First get the packet queue for this traceID */ + packet_queue = cs_etm__etmq_get_packet_queue(etmq, trace_chan_id); + if (!packet_queue) + return OCSD_RESP_FATAL_SYS_ERR; + + type = elem->elem_type; + + if (type == OCSD_GEN_TRC_ELEM_EO_TRACE || + type == OCSD_GEN_TRC_ELEM_NO_SYNC || + type == OCSD_GEN_TRC_ELEM_TRACE_ON) + resp = cs_etm_decoder__buffer_discontinuity(etmq, packet_queue, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_INSTR_RANGE) + resp = cs_etm_decoder__buffer_range(etmq, packet_queue, elem, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_EXCEPTION) + resp = cs_etm_decoder__buffer_exception(etmq, packet_queue, elem, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_EXCEPTION_RET) + resp = cs_etm_decoder__buffer_exception_ret(etmq, packet_queue, + trace_chan_id); + else if (type == OCSD_GEN_TRC_ELEM_TIMESTAMP) + resp = cs_etm_decoder__do_hard_timestamp(etmq, elem, + trace_chan_id, + indx); + else if (type == OCSD_GEN_TRC_ELEM_PE_CONTEXT) + resp = cs_etm_decoder__set_tid(etmq, packet_queue, + elem, trace_chan_id); + + return resp; +} + +static int +cs_etm_decoder__create_etm_decoder(struct cs_etm_decoder_params *d_params, + struct cs_etm_trace_params *t_params, + struct cs_etm_decoder *decoder) +{ + ocsd_etmv3_cfg config_etmv3; + ocsd_etmv4_cfg trace_config_etmv4; + ocsd_ete_cfg trace_config_ete; + void *trace_config; + u8 csid; + + switch (t_params->protocol) { + case CS_ETM_PROTO_ETMV3: + case CS_ETM_PROTO_PTM: + csid = (t_params->etmv3.reg_idr & CORESIGHT_TRACE_ID_VAL_MASK); + cs_etm_decoder__gen_etmv3_config(t_params, &config_etmv3); + decoder->decoder_name = (t_params->protocol == CS_ETM_PROTO_ETMV3) ? + OCSD_BUILTIN_DCD_ETMV3 : + OCSD_BUILTIN_DCD_PTM; + trace_config = &config_etmv3; + break; + case CS_ETM_PROTO_ETMV4i: + csid = (t_params->etmv4.reg_traceidr & CORESIGHT_TRACE_ID_VAL_MASK); + cs_etm_decoder__gen_etmv4_config(t_params, &trace_config_etmv4); + decoder->decoder_name = OCSD_BUILTIN_DCD_ETMV4I; + trace_config = &trace_config_etmv4; + break; + case CS_ETM_PROTO_ETE: + csid = (t_params->ete.reg_traceidr & CORESIGHT_TRACE_ID_VAL_MASK); + cs_etm_decoder__gen_ete_config(t_params, &trace_config_ete); + decoder->decoder_name = OCSD_BUILTIN_DCD_ETE; + trace_config = &trace_config_ete; + break; + default: + return -1; + } + + if (d_params->operation == CS_ETM_OPERATION_DECODE) { + int decode_flags = OCSD_CREATE_FLG_FULL_DECODER; +#ifdef OCSD_OPFLG_N_UNCOND_DIR_BR_CHK + decode_flags |= OCSD_OPFLG_N_UNCOND_DIR_BR_CHK | OCSD_OPFLG_CHK_RANGE_CONTINUE | + ETM4_OPFLG_PKTDEC_AA64_OPCODE_CHK; +#endif + if (ocsd_dt_create_decoder(decoder->dcd_tree, + decoder->decoder_name, + decode_flags, + trace_config, &csid)) + return -1; + + if (ocsd_dt_set_gen_elem_outfn(decoder->dcd_tree, + cs_etm_decoder__gen_trace_elem_printer, + decoder)) + return -1; + + return 0; + } else if (d_params->operation == CS_ETM_OPERATION_PRINT) { + if (ocsd_dt_create_decoder(decoder->dcd_tree, decoder->decoder_name, + OCSD_CREATE_FLG_PACKET_PROC, + trace_config, &csid)) + return -1; + + if (ocsd_dt_set_pkt_protocol_printer(decoder->dcd_tree, csid, CS_PKT_MON)) + return -1; + + return 0; + } + + return -1; +} + +struct cs_etm_decoder * +cs_etm_decoder__new(int decoders, struct cs_etm_decoder_params *d_params, + struct cs_etm_trace_params t_params[]) +{ + struct cs_etm_decoder *decoder; + ocsd_dcd_tree_src_t format; + u32 flags; + int i, ret; + + if ((!t_params) || (!d_params)) + return NULL; + + decoder = zalloc(sizeof(*decoder)); + + if (!decoder) + return NULL; + + decoder->data = d_params->data; + decoder->prev_return = OCSD_RESP_CONT; + format = (d_params->formatted ? OCSD_TRC_SRC_FRAME_FORMATTED : + OCSD_TRC_SRC_SINGLE); + flags = 0; + flags |= (d_params->fsyncs ? OCSD_DFRMTR_HAS_FSYNCS : 0); + flags |= (d_params->hsyncs ? OCSD_DFRMTR_HAS_HSYNCS : 0); + flags |= (d_params->frame_aligned ? OCSD_DFRMTR_FRAME_MEM_ALIGN : 0); + + /* + * Drivers may add barrier frames when used with perf, set up to + * handle this. Barriers const of FSYNC packet repeated 4 times. + */ + flags |= OCSD_DFRMTR_RESET_ON_4X_FSYNC; + + /* Create decode tree for the data source */ + decoder->dcd_tree = ocsd_create_dcd_tree(format, flags); + + if (decoder->dcd_tree == 0) + goto err_free_decoder; + + /* init library print logging support */ + ret = cs_etm_decoder__init_def_logger_printing(d_params, decoder); + if (ret != 0) + goto err_free_decoder; + + for (i = 0; i < decoders; i++) { + ret = cs_etm_decoder__create_etm_decoder(d_params, + &t_params[i], + decoder); + if (ret != 0) + goto err_free_decoder; + } + + return decoder; + +err_free_decoder: + cs_etm_decoder__free(decoder); + return NULL; +} + +int cs_etm_decoder__process_data_block(struct cs_etm_decoder *decoder, + u64 indx, const u8 *buf, + size_t len, size_t *consumed) +{ + int ret = 0; + ocsd_datapath_resp_t cur = OCSD_RESP_CONT; + ocsd_datapath_resp_t prev_return = decoder->prev_return; + size_t processed = 0; + u32 count; + + while (processed < len) { + if (OCSD_DATA_RESP_IS_WAIT(prev_return)) { + cur = ocsd_dt_process_data(decoder->dcd_tree, + OCSD_OP_FLUSH, + 0, + 0, + NULL, + NULL); + } else if (OCSD_DATA_RESP_IS_CONT(prev_return)) { + cur = ocsd_dt_process_data(decoder->dcd_tree, + OCSD_OP_DATA, + indx + processed, + len - processed, + &buf[processed], + &count); + processed += count; + } else { + ret = -EINVAL; + break; + } + + /* + * Return to the input code if the packet buffer is full. + * Flushing will get done once the packet buffer has been + * processed. + */ + if (OCSD_DATA_RESP_IS_WAIT(cur)) + break; + + prev_return = cur; + } + + decoder->prev_return = cur; + *consumed = processed; + + return ret; +} + +void cs_etm_decoder__free(struct cs_etm_decoder *decoder) +{ + if (!decoder) + return; + + ocsd_destroy_dcd_tree(decoder->dcd_tree); + decoder->dcd_tree = NULL; + free(decoder); +} + +const char *cs_etm_decoder__get_name(struct cs_etm_decoder *decoder) +{ + return decoder->decoder_name; +} diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h new file mode 100644 index 000000000..12c782fa6 --- /dev/null +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.h @@ -0,0 +1,111 @@ +/* + * SPDX-License-Identifier: GPL-2.0 + * + * Copyright(C) 2015-2018 Linaro Limited. + * + * Author: Tor Jeremiassen + * Author: Mathieu Poirier + */ + +#ifndef INCLUDE__CS_ETM_DECODER_H__ +#define INCLUDE__CS_ETM_DECODER_H__ + +#include +#include +#include + +struct cs_etm_decoder; +struct cs_etm_packet; +struct cs_etm_packet_queue; + +struct cs_etm_queue; + +typedef u32 (*cs_etm_mem_cb_type)(struct cs_etm_queue *, u8, u64, size_t, u8 *, + const ocsd_mem_space_acc_t); + +struct cs_etmv3_trace_params { + u32 reg_ctrl; + u32 reg_trc_id; + u32 reg_ccer; + u32 reg_idr; +}; + +struct cs_etmv4_trace_params { + u32 reg_idr0; + u32 reg_idr1; + u32 reg_idr2; + u32 reg_idr8; + u32 reg_configr; + u32 reg_traceidr; +}; + +struct cs_ete_trace_params { + u32 reg_idr0; + u32 reg_idr1; + u32 reg_idr2; + u32 reg_idr8; + u32 reg_configr; + u32 reg_traceidr; + u32 reg_devarch; +}; + +struct cs_etm_trace_params { + int protocol; + union { + struct cs_etmv3_trace_params etmv3; + struct cs_etmv4_trace_params etmv4; + struct cs_ete_trace_params ete; + }; +}; + +struct cs_etm_decoder_params { + int operation; + void (*packet_printer)(const char *msg, void *data); + cs_etm_mem_cb_type mem_acc_cb; + bool formatted; + bool fsyncs; + bool hsyncs; + bool frame_aligned; + void *data; +}; + +/* + * The following enums are indexed starting with 1 to align with the + * open source coresight trace decoder library. + */ +enum { + CS_ETM_PROTO_ETMV3 = 1, + CS_ETM_PROTO_ETMV4i, + CS_ETM_PROTO_ETMV4d, + CS_ETM_PROTO_PTM, + CS_ETM_PROTO_ETE +}; + +enum cs_etm_decoder_operation { + CS_ETM_OPERATION_PRINT = 1, + CS_ETM_OPERATION_DECODE, + CS_ETM_OPERATION_MAX, +}; + +int cs_etm_decoder__process_data_block(struct cs_etm_decoder *decoder, + u64 indx, const u8 *buf, + size_t len, size_t *consumed); + +struct cs_etm_decoder * +cs_etm_decoder__new(int num_cpu, + struct cs_etm_decoder_params *d_params, + struct cs_etm_trace_params t_params[]); + +void cs_etm_decoder__free(struct cs_etm_decoder *decoder); + +int cs_etm_decoder__add_mem_access_cb(struct cs_etm_decoder *decoder, + u64 start, u64 end, + cs_etm_mem_cb_type cb_func); + +int cs_etm_decoder__get_packet(struct cs_etm_packet_queue *packet_queue, + struct cs_etm_packet *packet); + +int cs_etm_decoder__reset(struct cs_etm_decoder *decoder); +const char *cs_etm_decoder__get_name(struct cs_etm_decoder *decoder); + +#endif /* INCLUDE__CS_ETM_DECODER_H__ */ diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c new file mode 100644 index 000000000..5d0664ff7 --- /dev/null +++ b/tools/perf/util/cs-etm.c @@ -0,0 +1,3671 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(C) 2015-2018 Linaro Limited. + * + * Author: Tor Jeremiassen + * Author: Mathieu Poirier + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "auxtrace.h" +#include "color.h" +#include "cs-etm.h" +#include "cs-etm-decoder/cs-etm-decoder.h" +#include "debug.h" +#include "dso.h" +#include "evlist.h" +#include "intlist.h" +#include "machine.h" +#include "map.h" +#include "perf.h" +#include "session.h" +#include "map_symbol.h" +#include "branch.h" +#include "symbol.h" +#include "tool.h" +#include "thread.h" +#include "thread-stack.h" +#include "tsc.h" +#include +#include "util/synthetic-events.h" +#include "util/util.h" + +struct cs_etm_auxtrace { + struct auxtrace auxtrace; + struct auxtrace_queues queues; + struct auxtrace_heap heap; + struct itrace_synth_opts synth_opts; + struct perf_session *session; + struct perf_tsc_conversion tc; + + /* + * Timeless has no timestamps in the trace so overlapping mmap lookups + * are less accurate but produces smaller trace data. We use context IDs + * in the trace instead of matching timestamps with fork records so + * they're not really needed in the general case. Overlapping mmaps + * happen in cases like between a fork and an exec. + */ + bool timeless_decoding; + + /* + * Per-thread ignores the trace channel ID and instead assumes that + * everything in a buffer comes from the same process regardless of + * which CPU it ran on. It also implies no context IDs so the TID is + * taken from the auxtrace buffer. + */ + bool per_thread_decoding; + bool snapshot_mode; + bool data_queued; + bool has_virtual_ts; /* Virtual/Kernel timestamps in the trace. */ + + int num_cpu; + u64 latest_kernel_timestamp; + u32 auxtrace_type; + u64 branches_sample_type; + u64 branches_id; + u64 instructions_sample_type; + u64 instructions_sample_period; + u64 instructions_id; + u64 **metadata; + unsigned int pmu_type; + enum cs_etm_pid_fmt pid_fmt; +}; + +struct cs_etm_traceid_queue { + u8 trace_chan_id; + u64 period_instructions; + size_t last_branch_pos; + union perf_event *event_buf; + struct branch_stack *last_branch; + struct branch_stack *last_branch_rb; + struct cs_etm_packet *prev_packet; + struct cs_etm_packet *packet; + struct cs_etm_packet_queue packet_queue; + + struct thread *decode_thread; + ocsd_ex_level decode_el; + + /* + * The frontend accesses the EL from '[prev_]packet' because it needs + * previous EL for branch and current EL for instruction samples. It's + * not possible to change thread in a single branch sample so no need to + * store or access the thread through the packet. + */ + struct thread *frontend_thread; +}; + +enum cs_etm_format { + UNSET, + FORMATTED, + UNFORMATTED +}; + +struct cs_etm_queue { + struct cs_etm_auxtrace *etm; + struct cs_etm_decoder *decoder; + struct auxtrace_buffer *buffer; + unsigned int queue_nr; + u8 pending_timestamp_chan_id; + enum cs_etm_format format; + u64 offset; + const unsigned char *buf; + size_t buf_len, buf_used; + /* Conversion between traceID and index in traceid_queues array */ + struct intlist *traceid_queues_list; + struct cs_etm_traceid_queue **traceid_queues; + /* Conversion between traceID and metadata pointers */ + struct intlist *traceid_list; + /* + * Same as traceid_list, but traceid_list may be a reference to another + * queue's which has a matching sink ID. + */ + struct intlist *own_traceid_list; + u32 sink_id; +}; + +static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm); +static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, + pid_t tid); +static int cs_etm__get_data_block(struct cs_etm_queue *etmq); +static int cs_etm__decode_data_block(struct cs_etm_queue *etmq); +static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata); +static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu); +static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata); + +/* PTMs ETMIDR [11:8] set to b0011 */ +#define ETMIDR_PTM_VERSION 0x00000300 + +/* + * A struct auxtrace_heap_item only has a queue_nr and a timestamp to + * work with. One option is to modify to auxtrace_heap_XYZ() API or simply + * encode the etm queue number as the upper 16 bit and the channel as + * the lower 16 bit. + */ +#define TO_CS_QUEUE_NR(queue_nr, trace_chan_id) \ + (queue_nr << 16 | trace_chan_id) +#define TO_QUEUE_NR(cs_queue_nr) (cs_queue_nr >> 16) +#define TO_TRACE_CHAN_ID(cs_queue_nr) (cs_queue_nr & 0x0000ffff) +#define SINK_UNSET ((u32) -1) + +static u32 cs_etm__get_v7_protocol_version(u32 etmidr) +{ + etmidr &= ETMIDR_PTM_VERSION; + + if (etmidr == ETMIDR_PTM_VERSION) + return CS_ETM_PROTO_PTM; + + return CS_ETM_PROTO_ETMV3; +} + +static int cs_etm__get_magic(struct cs_etm_queue *etmq, u8 trace_chan_id, u64 *magic) +{ + struct int_node *inode; + u64 *metadata; + + inode = intlist__find(etmq->traceid_list, trace_chan_id); + if (!inode) + return -EINVAL; + + metadata = inode->priv; + *magic = metadata[CS_ETM_MAGIC]; + return 0; +} + +int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu) +{ + struct int_node *inode; + u64 *metadata; + + inode = intlist__find(etmq->traceid_list, trace_chan_id); + if (!inode) + return -EINVAL; + + metadata = inode->priv; + *cpu = (int)metadata[CS_ETM_CPU]; + return 0; +} + +/* + * The returned PID format is presented as an enum: + * + * CS_ETM_PIDFMT_CTXTID: CONTEXTIDR or CONTEXTIDR_EL1 is traced. + * CS_ETM_PIDFMT_CTXTID2: CONTEXTIDR_EL2 is traced. + * CS_ETM_PIDFMT_NONE: No context IDs + * + * It's possible that the two format attributes 'contextid1' and 'contextid2' + * are enabled at the same time when the session runs on an EL2 kernel. + * This means the CONTEXTIDR_EL1 and CONTEXTIDR_EL2 both will be + * recorded in the trace data, the tool will selectively use + * CONTEXTIDR_EL2 as PID. + * + * The result is cached in etm->pid_fmt so this function only needs to be called + * when processing the aux info. + */ +static enum cs_etm_pid_fmt cs_etm__init_pid_fmt(u64 *metadata) +{ + u64 val; + + if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) { + val = metadata[CS_ETM_ETMCR]; + /* CONTEXTIDR is traced */ + if (val & ETMCR_CTXTID) + return CS_ETM_PIDFMT_CTXTID; + } else { + val = metadata[CS_ETMV4_TRCCONFIGR]; + /* CONTEXTIDR_EL2 is traced */ + if (val & (TRCCONFIGR_VMID | TRCCONFIGR_VMIDOPT)) + return CS_ETM_PIDFMT_CTXTID2; + /* CONTEXTIDR_EL1 is traced */ + else if (val & TRCCONFIGR_CID) + return CS_ETM_PIDFMT_CTXTID; + } + + return CS_ETM_PIDFMT_NONE; +} + +enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq) +{ + return etmq->etm->pid_fmt; +} + +static int cs_etm__insert_trace_id_node(struct cs_etm_queue *etmq, + u8 trace_chan_id, u64 *cpu_metadata) +{ + /* Get an RB node for this CPU */ + struct int_node *inode = intlist__findnew(etmq->traceid_list, trace_chan_id); + + /* Something went wrong, no need to continue */ + if (!inode) + return -ENOMEM; + + /* Disallow re-mapping a different traceID to metadata pair. */ + if (inode->priv) { + u64 *curr_cpu_data = inode->priv; + u8 curr_chan_id; + int err; + + if (curr_cpu_data[CS_ETM_CPU] != cpu_metadata[CS_ETM_CPU]) { + /* + * With > CORESIGHT_TRACE_IDS_MAX ETMs, overlapping IDs + * are expected (but not supported) in per-thread mode, + * rather than signifying an error. + */ + if (etmq->etm->per_thread_decoding) + pr_err("CS_ETM: overlapping Trace IDs aren't currently supported in per-thread mode\n"); + else + pr_err("CS_ETM: map mismatch between HW_ID packet CPU and Trace ID\n"); + + return -EINVAL; + } + + /* check that the mapped ID matches */ + err = cs_etm__metadata_get_trace_id(&curr_chan_id, curr_cpu_data); + if (err) + return err; + + if (curr_chan_id != trace_chan_id) { + pr_err("CS_ETM: mismatch between CPU trace ID and HW_ID packet ID\n"); + return -EINVAL; + } + + /* Skip re-adding the same mappings if everything matched */ + return 0; + } + + /* Not one we've seen before, associate the traceID with the metadata pointer */ + inode->priv = cpu_metadata; + + return 0; +} + +static struct cs_etm_queue *cs_etm__get_queue(struct cs_etm_auxtrace *etm, int cpu) +{ + if (etm->per_thread_decoding) + return etm->queues.queue_array[0].priv; + + if (cpu < 0 || cpu >= (int)etm->queues.nr_queues) + return NULL; + + return etm->queues.queue_array[cpu].priv; +} + +static int cs_etm__map_trace_id_v0(struct cs_etm_auxtrace *etm, u8 trace_chan_id, + u64 *cpu_metadata) +{ + struct cs_etm_queue *etmq; + + /* + * If the queue is unformatted then only save one mapping in the + * queue associated with that CPU so only one decoder is made. + */ + etmq = cs_etm__get_queue(etm, cpu_metadata[CS_ETM_CPU]); + if (!etmq) + return -EINVAL; + + if (etmq->format == UNFORMATTED) + return cs_etm__insert_trace_id_node(etmq, trace_chan_id, + cpu_metadata); + + /* + * Otherwise, version 0 trace IDs are global so save them into every + * queue. + */ + for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { + int ret; + + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + ret = cs_etm__insert_trace_id_node(etmq, trace_chan_id, + cpu_metadata); + if (ret) + return ret; + } + + return 0; +} + +static int cs_etm__process_trace_id_v0(struct cs_etm_auxtrace *etm, int cpu, + u64 hw_id) +{ + int err; + u64 *cpu_data; + u8 trace_chan_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + + cpu_data = get_cpu_data(etm, cpu); + if (cpu_data == NULL) + return -EINVAL; + + err = cs_etm__map_trace_id_v0(etm, trace_chan_id, cpu_data); + if (err) + return err; + + /* + * if we are picking up the association from the packet, need to plug + * the correct trace ID into the metadata for setting up decoders later. + */ + return cs_etm__metadata_set_trace_id(trace_chan_id, cpu_data); +} + +static int cs_etm__process_trace_id_v0_1(struct cs_etm_auxtrace *etm, int cpu, + u64 hw_id) +{ + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, cpu); + int ret; + u64 *cpu_data; + u32 sink_id = FIELD_GET(CS_AUX_HW_ID_SINK_ID_MASK, hw_id); + u8 trace_id = FIELD_GET(CS_AUX_HW_ID_TRACE_ID_MASK, hw_id); + + if (!etmq) + return -EINVAL; + + /* + * Check sink id hasn't changed in per-cpu mode. In per-thread mode, + * let it pass for now until an actual overlapping trace ID is hit. In + * most cases IDs won't overlap even if the sink changes. + */ + if (!etmq->etm->per_thread_decoding && etmq->sink_id != SINK_UNSET && + etmq->sink_id != sink_id) { + pr_err("CS_ETM: mismatch between sink IDs\n"); + return -EINVAL; + } + + etmq->sink_id = sink_id; + + /* Find which other queues use this sink and link their ID maps */ + for (unsigned int i = 0; i < etm->queues.nr_queues; ++i) { + struct cs_etm_queue *other_etmq = etm->queues.queue_array[i].priv; + + if (!other_etmq) + continue; + + /* Different sinks, skip */ + if (other_etmq->sink_id != etmq->sink_id) + continue; + + /* Already linked, skip */ + if (other_etmq->traceid_list == etmq->traceid_list) + continue; + + /* At the point of first linking, this one should be empty */ + if (!intlist__empty(etmq->traceid_list)) { + pr_err("CS_ETM: Can't link populated trace ID lists\n"); + return -EINVAL; + } + + etmq->own_traceid_list = NULL; + intlist__delete(etmq->traceid_list); + etmq->traceid_list = other_etmq->traceid_list; + break; + } + + cpu_data = get_cpu_data(etm, cpu); + if (!cpu_data) + return -EINVAL; + + ret = cs_etm__insert_trace_id_node(etmq, trace_id, cpu_data); + if (ret) + return ret; + + ret = cs_etm__metadata_set_trace_id(trace_id, cpu_data); + if (ret) + return ret; + + return 0; +} + +static int cs_etm__metadata_get_trace_id(u8 *trace_chan_id, u64 *cpu_metadata) +{ + u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC]; + + switch (cs_etm_magic) { + case __perf_cs_etmv3_magic: + *trace_chan_id = (u8)(cpu_metadata[CS_ETM_ETMTRACEIDR] & + CORESIGHT_TRACE_ID_VAL_MASK); + break; + case __perf_cs_etmv4_magic: + case __perf_cs_ete_magic: + *trace_chan_id = (u8)(cpu_metadata[CS_ETMV4_TRCTRACEIDR] & + CORESIGHT_TRACE_ID_VAL_MASK); + break; + default: + return -EINVAL; + } + return 0; +} + +/* + * update metadata trace ID from the value found in the AUX_HW_INFO packet. + */ +static int cs_etm__metadata_set_trace_id(u8 trace_chan_id, u64 *cpu_metadata) +{ + u64 cs_etm_magic = cpu_metadata[CS_ETM_MAGIC]; + + switch (cs_etm_magic) { + case __perf_cs_etmv3_magic: + cpu_metadata[CS_ETM_ETMTRACEIDR] = trace_chan_id; + break; + case __perf_cs_etmv4_magic: + case __perf_cs_ete_magic: + cpu_metadata[CS_ETMV4_TRCTRACEIDR] = trace_chan_id; + break; + + default: + return -EINVAL; + } + return 0; +} + +/* + * Get a metadata index for a specific cpu from an array. + * + */ +static int get_cpu_data_idx(struct cs_etm_auxtrace *etm, int cpu) +{ + int i; + + for (i = 0; i < etm->num_cpu; i++) { + if (etm->metadata[i][CS_ETM_CPU] == (u64)cpu) { + return i; + } + } + + return -1; +} + +/* + * Get a metadata for a specific cpu from an array. + * + */ +static u64 *get_cpu_data(struct cs_etm_auxtrace *etm, int cpu) +{ + int idx = get_cpu_data_idx(etm, cpu); + + return (idx != -1) ? etm->metadata[idx] : NULL; +} + +/* + * Handle the PERF_RECORD_AUX_OUTPUT_HW_ID event. + * + * The payload associates the Trace ID and the CPU. + * The routine is tolerant of seeing multiple packets with the same association, + * but a CPU / Trace ID association changing during a session is an error. + */ +static int cs_etm__process_aux_output_hw_id(struct perf_session *session, + union perf_event *event) +{ + struct cs_etm_auxtrace *etm; + struct perf_sample sample; + struct evsel *evsel; + u64 hw_id; + int cpu, version, err; + + /* extract and parse the HW ID */ + hw_id = event->aux_output_hw_id.hw_id; + version = FIELD_GET(CS_AUX_HW_ID_MAJOR_VERSION_MASK, hw_id); + + /* check that we can handle this version */ + if (version > CS_AUX_HW_ID_MAJOR_VERSION) { + pr_err("CS ETM Trace: PERF_RECORD_AUX_OUTPUT_HW_ID version %d not supported. Please update Perf.\n", + version); + return -EINVAL; + } + + /* get access to the etm metadata */ + etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace); + if (!etm || !etm->metadata) + return -EINVAL; + + /* parse the sample to get the CPU */ + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) + return -EINVAL; + perf_sample__init(&sample, /*all=*/false); + err = evsel__parse_sample(evsel, event, &sample); + if (err) + goto out; + cpu = sample.cpu; + if (cpu == -1) { + /* no CPU in the sample - possibly recorded with an old version of perf */ + pr_err("CS_ETM: no CPU AUX_OUTPUT_HW_ID sample. Use compatible perf to record."); + err = -EINVAL; + goto out; + } + + if (FIELD_GET(CS_AUX_HW_ID_MINOR_VERSION_MASK, hw_id) == 0) { + err = cs_etm__process_trace_id_v0(etm, cpu, hw_id); + goto out; + } + + err = cs_etm__process_trace_id_v0_1(etm, cpu, hw_id); +out: + perf_sample__exit(&sample); + return err; +} + +void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, + u8 trace_chan_id) +{ + /* + * When a timestamp packet is encountered the backend code + * is stopped so that the front end has time to process packets + * that were accumulated in the traceID queue. Since there can + * be more than one channel per cs_etm_queue, we need to specify + * what traceID queue needs servicing. + */ + etmq->pending_timestamp_chan_id = trace_chan_id; +} + +static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, + u8 *trace_chan_id) +{ + struct cs_etm_packet_queue *packet_queue; + + if (!etmq->pending_timestamp_chan_id) + return 0; + + if (trace_chan_id) + *trace_chan_id = etmq->pending_timestamp_chan_id; + + packet_queue = cs_etm__etmq_get_packet_queue(etmq, + etmq->pending_timestamp_chan_id); + if (!packet_queue) + return 0; + + /* Acknowledge pending status */ + etmq->pending_timestamp_chan_id = 0; + + /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */ + return packet_queue->cs_timestamp; +} + +static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue) +{ + int i; + + queue->head = 0; + queue->tail = 0; + queue->packet_count = 0; + for (i = 0; i < CS_ETM_PACKET_MAX_BUFFER; i++) { + queue->packet_buffer[i].isa = CS_ETM_ISA_UNKNOWN; + queue->packet_buffer[i].start_addr = CS_ETM_INVAL_ADDR; + queue->packet_buffer[i].end_addr = CS_ETM_INVAL_ADDR; + queue->packet_buffer[i].instr_count = 0; + queue->packet_buffer[i].last_instr_taken_branch = false; + queue->packet_buffer[i].last_instr_size = 0; + queue->packet_buffer[i].last_instr_type = 0; + queue->packet_buffer[i].last_instr_subtype = 0; + queue->packet_buffer[i].last_instr_cond = 0; + queue->packet_buffer[i].flags = 0; + queue->packet_buffer[i].exception_number = UINT32_MAX; + queue->packet_buffer[i].trace_chan_id = UINT8_MAX; + queue->packet_buffer[i].cpu = INT_MIN; + } +} + +static void cs_etm__clear_all_packet_queues(struct cs_etm_queue *etmq) +{ + int idx; + struct int_node *inode; + struct cs_etm_traceid_queue *tidq; + struct intlist *traceid_queues_list = etmq->traceid_queues_list; + + intlist__for_each_entry(inode, traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + cs_etm__clear_packet_queue(&tidq->packet_queue); + } +} + +static int cs_etm__init_traceid_queue(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + u8 trace_chan_id) +{ + int rc = -ENOMEM; + struct auxtrace_queue *queue; + struct cs_etm_auxtrace *etm = etmq->etm; + + cs_etm__clear_packet_queue(&tidq->packet_queue); + + queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; + tidq->trace_chan_id = trace_chan_id; + tidq->decode_el = ocsd_EL_unknown; + tidq->frontend_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + tidq->decode_thread = machine__findnew_thread(&etm->session->machines.host, -1, + queue->tid); + + tidq->packet = zalloc(sizeof(struct cs_etm_packet)); + if (!tidq->packet) + goto out; + + tidq->prev_packet = zalloc(sizeof(struct cs_etm_packet)); + if (!tidq->prev_packet) + goto out_free; + + if (etm->synth_opts.last_branch) { + size_t sz = sizeof(struct branch_stack); + + sz += etm->synth_opts.last_branch_sz * + sizeof(struct branch_entry); + tidq->last_branch = zalloc(sz); + if (!tidq->last_branch) + goto out_free; + tidq->last_branch_rb = zalloc(sz); + if (!tidq->last_branch_rb) + goto out_free; + } + + tidq->event_buf = malloc(PERF_SAMPLE_MAX_SIZE); + if (!tidq->event_buf) + goto out_free; + + return 0; + +out_free: + zfree(&tidq->last_branch_rb); + zfree(&tidq->last_branch); + zfree(&tidq->prev_packet); + zfree(&tidq->packet); +out: + return rc; +} + +static struct cs_etm_traceid_queue +*cs_etm__etmq_get_traceid_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) +{ + int idx; + struct int_node *inode; + struct intlist *traceid_queues_list; + struct cs_etm_traceid_queue *tidq, **traceid_queues; + struct cs_etm_auxtrace *etm = etmq->etm; + + if (etm->per_thread_decoding) + trace_chan_id = CS_ETM_PER_THREAD_TRACEID; + + traceid_queues_list = etmq->traceid_queues_list; + + /* + * Check if the traceid_queue exist for this traceID by looking + * in the queue list. + */ + inode = intlist__find(traceid_queues_list, trace_chan_id); + if (inode) { + idx = (int)(intptr_t)inode->priv; + return etmq->traceid_queues[idx]; + } + + /* We couldn't find a traceid_queue for this traceID, allocate one */ + tidq = malloc(sizeof(*tidq)); + if (!tidq) + return NULL; + + memset(tidq, 0, sizeof(*tidq)); + + /* Get a valid index for the new traceid_queue */ + idx = intlist__nr_entries(traceid_queues_list); + /* Memory for the inode is free'ed in cs_etm_free_traceid_queues () */ + inode = intlist__findnew(traceid_queues_list, trace_chan_id); + if (!inode) + goto out_free; + + /* Associate this traceID with this index */ + inode->priv = (void *)(intptr_t)idx; + + if (cs_etm__init_traceid_queue(etmq, tidq, trace_chan_id)) + goto out_free; + + /* Grow the traceid_queues array by one unit */ + traceid_queues = etmq->traceid_queues; + traceid_queues = reallocarray(traceid_queues, + idx + 1, + sizeof(*traceid_queues)); + + /* + * On failure reallocarray() returns NULL and the original block of + * memory is left untouched. + */ + if (!traceid_queues) + goto out_free; + + traceid_queues[idx] = tidq; + etmq->traceid_queues = traceid_queues; + + return etmq->traceid_queues[idx]; + +out_free: + /* + * Function intlist__remove() removes the inode from the list + * and delete the memory associated to it. + */ + intlist__remove(traceid_queues_list, inode); + free(tidq); + + return NULL; +} + +struct cs_etm_packet_queue +*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (tidq) + return &tidq->packet_queue; + + return NULL; +} + +static void cs_etm__packet_swap(struct cs_etm_auxtrace *etm, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_packet *tmp; + + if (etm->synth_opts.branches || etm->synth_opts.last_branch || + etm->synth_opts.instructions) { + /* + * Swap PACKET with PREV_PACKET: PACKET becomes PREV_PACKET for + * the next incoming packet. + */ + tmp = tidq->packet; + tidq->packet = tidq->prev_packet; + tidq->prev_packet = tmp; + } +} + +static void cs_etm__packet_dump(const char *pkt_string, void *data) +{ + const char *color = PERF_COLOR_BLUE; + int len = strlen(pkt_string); + struct cs_etm_queue *etmq = data; + char queue_nr[64]; + + if (verbose) + snprintf(queue_nr, sizeof(queue_nr), "Qnr:%u; ", etmq->queue_nr); + else + queue_nr[0] = '\0'; + + if (len && (pkt_string[len-1] == '\n')) + color_fprintf(stdout, color, " %s%s", queue_nr, pkt_string); + else + color_fprintf(stdout, color, " %s%s\n", queue_nr, pkt_string); + + fflush(stdout); +} + +static void cs_etm__set_trace_param_etmv3(struct cs_etm_trace_params *t_params, + u64 *metadata, u32 etmidr) +{ + t_params->protocol = cs_etm__get_v7_protocol_version(etmidr); + t_params->etmv3.reg_ctrl = metadata[CS_ETM_ETMCR]; + t_params->etmv3.reg_trc_id = metadata[CS_ETM_ETMTRACEIDR]; +} + +static void cs_etm__set_trace_param_etmv4(struct cs_etm_trace_params *t_params, + u64 *metadata) +{ + t_params->protocol = CS_ETM_PROTO_ETMV4i; + t_params->etmv4.reg_idr0 = metadata[CS_ETMV4_TRCIDR0]; + t_params->etmv4.reg_idr1 = metadata[CS_ETMV4_TRCIDR1]; + t_params->etmv4.reg_idr2 = metadata[CS_ETMV4_TRCIDR2]; + t_params->etmv4.reg_idr8 = metadata[CS_ETMV4_TRCIDR8]; + t_params->etmv4.reg_configr = metadata[CS_ETMV4_TRCCONFIGR]; + t_params->etmv4.reg_traceidr = metadata[CS_ETMV4_TRCTRACEIDR]; +} + +static void cs_etm__set_trace_param_ete(struct cs_etm_trace_params *t_params, + u64 *metadata) +{ + t_params->protocol = CS_ETM_PROTO_ETE; + t_params->ete.reg_idr0 = metadata[CS_ETE_TRCIDR0]; + t_params->ete.reg_idr1 = metadata[CS_ETE_TRCIDR1]; + t_params->ete.reg_idr2 = metadata[CS_ETE_TRCIDR2]; + t_params->ete.reg_idr8 = metadata[CS_ETE_TRCIDR8]; + t_params->ete.reg_configr = metadata[CS_ETE_TRCCONFIGR]; + t_params->ete.reg_traceidr = metadata[CS_ETE_TRCTRACEIDR]; + t_params->ete.reg_devarch = metadata[CS_ETE_TRCDEVARCH]; +} + +static int cs_etm__init_trace_params(struct cs_etm_trace_params *t_params, + struct cs_etm_queue *etmq) +{ + struct int_node *inode; + + intlist__for_each_entry(inode, etmq->traceid_list) { + u64 *metadata = inode->priv; + u64 architecture = metadata[CS_ETM_MAGIC]; + u32 etmidr; + + switch (architecture) { + case __perf_cs_etmv3_magic: + etmidr = metadata[CS_ETM_ETMIDR]; + cs_etm__set_trace_param_etmv3(t_params++, metadata, etmidr); + break; + case __perf_cs_etmv4_magic: + cs_etm__set_trace_param_etmv4(t_params++, metadata); + break; + case __perf_cs_ete_magic: + cs_etm__set_trace_param_ete(t_params++, metadata); + break; + default: + return -EINVAL; + } + } + + return 0; +} + +static int cs_etm__init_decoder_params(struct cs_etm_decoder_params *d_params, + struct cs_etm_queue *etmq, + enum cs_etm_decoder_operation mode) +{ + int ret = -EINVAL; + + if (!(mode < CS_ETM_OPERATION_MAX)) + goto out; + + d_params->packet_printer = cs_etm__packet_dump; + d_params->operation = mode; + d_params->data = etmq; + d_params->formatted = etmq->format == FORMATTED; + d_params->fsyncs = false; + d_params->hsyncs = false; + d_params->frame_aligned = true; + + ret = 0; +out: + return ret; +} + +static void cs_etm__dump_event(struct cs_etm_queue *etmq, + struct auxtrace_buffer *buffer) +{ + int ret; + const char *color = PERF_COLOR_BLUE; + size_t buffer_used = 0; + + fprintf(stdout, "\n"); + color_fprintf(stdout, color, + ". ... CoreSight %s Trace data: size %#zx bytes\n", + cs_etm_decoder__get_name(etmq->decoder), buffer->size); + + do { + size_t consumed; + + ret = cs_etm_decoder__process_data_block( + etmq->decoder, buffer->offset, + &((u8 *)buffer->data)[buffer_used], + buffer->size - buffer_used, &consumed); + if (ret) + break; + + buffer_used += consumed; + } while (buffer_used < buffer->size); + + cs_etm_decoder__reset(etmq->decoder); +} + +static int cs_etm__flush_events(struct perf_session *session, + const struct perf_tool *tool) +{ + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + if (dump_trace) + return 0; + + if (!tool->ordered_events) + return -EINVAL; + + if (etm->timeless_decoding) { + /* + * Pass tid = -1 to process all queues. But likely they will have + * already been processed on PERF_RECORD_EXIT anyway. + */ + return cs_etm__process_timeless_queues(etm, -1); + } + + return cs_etm__process_timestamped_queues(etm); +} + +static void cs_etm__free_traceid_queues(struct cs_etm_queue *etmq) +{ + int idx; + uintptr_t priv; + struct int_node *inode, *tmp; + struct cs_etm_traceid_queue *tidq; + struct intlist *traceid_queues_list = etmq->traceid_queues_list; + + intlist__for_each_entry_safe(inode, tmp, traceid_queues_list) { + priv = (uintptr_t)inode->priv; + idx = priv; + + /* Free this traceid_queue from the array */ + tidq = etmq->traceid_queues[idx]; + thread__zput(tidq->frontend_thread); + thread__zput(tidq->decode_thread); + zfree(&tidq->event_buf); + zfree(&tidq->last_branch); + zfree(&tidq->last_branch_rb); + zfree(&tidq->prev_packet); + zfree(&tidq->packet); + zfree(&tidq); + + /* + * Function intlist__remove() removes the inode from the list + * and delete the memory associated to it. + */ + intlist__remove(traceid_queues_list, inode); + } + + /* Then the RB tree itself */ + intlist__delete(traceid_queues_list); + etmq->traceid_queues_list = NULL; + + /* finally free the traceid_queues array */ + zfree(&etmq->traceid_queues); +} + +static void cs_etm__free_queue(void *priv) +{ + struct int_node *inode, *tmp; + struct cs_etm_queue *etmq = priv; + + if (!etmq) + return; + + cs_etm_decoder__free(etmq->decoder); + cs_etm__free_traceid_queues(etmq); + + if (etmq->own_traceid_list) { + /* First remove all traceID/metadata nodes for the RB tree */ + intlist__for_each_entry_safe(inode, tmp, etmq->own_traceid_list) + intlist__remove(etmq->own_traceid_list, inode); + + /* Then the RB tree itself */ + intlist__delete(etmq->own_traceid_list); + } + + free(etmq); +} + +static void cs_etm__free_events(struct perf_session *session) +{ + unsigned int i; + struct cs_etm_auxtrace *aux = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + struct auxtrace_queues *queues = &aux->queues; + + for (i = 0; i < queues->nr_queues; i++) { + cs_etm__free_queue(queues->queue_array[i].priv); + queues->queue_array[i].priv = NULL; + } + + auxtrace_queues__free(queues); +} + +static void cs_etm__free(struct perf_session *session) +{ + int i; + struct cs_etm_auxtrace *aux = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + cs_etm__free_events(session); + session->auxtrace = NULL; + + for (i = 0; i < aux->num_cpu; i++) + zfree(&aux->metadata[i]); + + zfree(&aux->metadata); + zfree(&aux); +} + +static bool cs_etm__evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct cs_etm_auxtrace *aux = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + return evsel->core.attr.type == aux->pmu_type; +} + +static struct machine *cs_etm__get_machine(struct cs_etm_queue *etmq, + ocsd_ex_level el) +{ + enum cs_etm_pid_fmt pid_fmt = cs_etm__get_pid_fmt(etmq); + + /* + * For any virtualisation based on nVHE (e.g. pKVM), or host kernels + * running at EL1 assume everything is the host. + */ + if (pid_fmt == CS_ETM_PIDFMT_CTXTID) + return &etmq->etm->session->machines.host; + + /* + * Not perfect, but otherwise assume anything in EL1 is the default + * guest, and everything else is the host. Distinguishing between guest + * and host userspaces isn't currently supported either. Neither is + * multiple guest support. All this does is reduce the likeliness of + * decode errors where we look into the host kernel maps when it should + * have been the guest maps. + */ + switch (el) { + case ocsd_EL1: + return machines__find_guest(&etmq->etm->session->machines, + DEFAULT_GUEST_KERNEL_ID); + case ocsd_EL3: + case ocsd_EL2: + case ocsd_EL0: + case ocsd_EL_unknown: + default: + return &etmq->etm->session->machines.host; + } +} + +static u8 cs_etm__cpu_mode(struct cs_etm_queue *etmq, u64 address, + ocsd_ex_level el) +{ + struct machine *machine = cs_etm__get_machine(etmq, el); + + if (address >= machine__kernel_start(machine)) { + if (machine__is_host(machine)) + return PERF_RECORD_MISC_KERNEL; + else + return PERF_RECORD_MISC_GUEST_KERNEL; + } else { + if (machine__is_host(machine)) + return PERF_RECORD_MISC_USER; + else { + /* + * Can't really happen at the moment because + * cs_etm__get_machine() will always return + * machines.host for any non EL1 trace. + */ + return PERF_RECORD_MISC_GUEST_USER; + } + } +} + +static u32 __cs_etm__mem_access(struct cs_etm_queue *etmq, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space, + ocsd_ex_level el, struct thread *thread) +{ + u8 cpumode; + u64 offset; + int len; + struct addr_location al; + struct dso *dso; + int ret = 0; + + if (!etmq) + return 0; + + addr_location__init(&al); + + /* + * We track EL for the frontend and the backend when receiving context + * and range packets. OpenCSD doesn't distinguish between EL0 and EL1 + * for this mem access callback so we had to do the extra tracking. Skip + * validation if it's any of the 'any' values. + */ + if (!(mem_space == OCSD_MEM_SPACE_ANY || + mem_space == OCSD_MEM_SPACE_N || mem_space == OCSD_MEM_SPACE_S)) { + if (mem_space & OCSD_MEM_SPACE_EL1N) { + /* Includes both non secure EL1 and EL0 */ + assert(el == ocsd_EL1 || el == ocsd_EL0); + } else if (mem_space & OCSD_MEM_SPACE_EL2) + assert(el == ocsd_EL2); + else if (mem_space & OCSD_MEM_SPACE_EL3) + assert(el == ocsd_EL3); + } + + cpumode = cs_etm__cpu_mode(etmq, address, el); + + if (!thread__find_map(thread, cpumode, address, &al)) + goto out; + + dso = map__dso(al.map); + if (!dso) + goto out; + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR && + dso__data_status_seen(dso, DSO_DATA_STATUS_SEEN_ITRACE)) + goto out; + + offset = map__map_ip(al.map, address); + + map__load(al.map); + + len = dso__data_read_offset(dso, maps__machine(thread__maps(thread)), + offset, buffer, size); + + if (len <= 0) { + ui__warning_once("CS ETM Trace: Missing DSO. Use 'perf archive' or debuginfod to export data from the traced system.\n" + " Enable CONFIG_PROC_KCORE or use option '-k /path/to/vmlinux' for kernel symbols.\n"); + if (!dso__auxtrace_warned(dso)) { + pr_err("CS ETM Trace: Debug data not found for address %#"PRIx64" in %s\n", + address, + dso__long_name(dso) ? dso__long_name(dso) : "Unknown"); + dso__set_auxtrace_warned(dso); + } + goto out; + } + ret = len; +out: + addr_location__exit(&al); + return ret; +} + +static u32 cs_etm__frontend_mem_access(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 address, size_t size, u8 *buffer) +{ + return __cs_etm__mem_access(etmq, address, size, buffer, 0, packet->el, + tidq->frontend_thread); +} + +static u32 cs_etm__decoder_mem_access(struct cs_etm_queue *etmq, u8 trace_chan_id, + u64 address, size_t size, u8 *buffer, + const ocsd_mem_space_acc_t mem_space) +{ + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return 0; + + return __cs_etm__mem_access(etmq, address, size, buffer, + mem_space, tidq->decode_el, + tidq->decode_thread); +} + +static struct cs_etm_queue *cs_etm__alloc_queue(void) +{ + struct cs_etm_queue *etmq = zalloc(sizeof(*etmq)); + if (!etmq) + return NULL; + + etmq->traceid_queues_list = intlist__new(NULL); + if (!etmq->traceid_queues_list) + goto out_free; + + /* + * Create an RB tree for traceID-metadata tuple. Since the conversion + * has to be made for each packet that gets decoded, optimizing access + * in anything other than a sequential array is worth doing. + */ + etmq->traceid_list = etmq->own_traceid_list = intlist__new(NULL); + if (!etmq->traceid_list) + goto out_free; + + return etmq; + +out_free: + intlist__delete(etmq->traceid_queues_list); + free(etmq); + + return NULL; +} + +static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, + struct auxtrace_queue *queue, + unsigned int queue_nr) +{ + struct cs_etm_queue *etmq = queue->priv; + + if (etmq) + return 0; + + etmq = cs_etm__alloc_queue(); + + if (!etmq) + return -ENOMEM; + + queue->priv = etmq; + etmq->etm = etm; + etmq->queue_nr = queue_nr; + queue->cpu = queue_nr; /* Placeholder, may be reset to -1 in per-thread mode */ + etmq->offset = 0; + etmq->sink_id = SINK_UNSET; + + return 0; +} + +static int cs_etm__queue_first_cs_timestamp(struct cs_etm_auxtrace *etm, + struct cs_etm_queue *etmq, + unsigned int queue_nr) +{ + int ret = 0; + unsigned int cs_queue_nr; + u8 trace_chan_id; + u64 cs_timestamp; + + /* + * We are under a CPU-wide trace scenario. As such we need to know + * when the code that generated the traces started to execute so that + * it can be correlated with execution on other CPUs. So we get a + * handle on the beginning of traces and decode until we find a + * timestamp. The timestamp is then added to the auxtrace min heap + * in order to know what nibble (of all the etmqs) to decode first. + */ + while (1) { + /* + * Fetch an aux_buffer from this etmq. Bail if no more + * blocks or an error has been encountered. + */ + ret = cs_etm__get_data_block(etmq); + if (ret <= 0) + goto out; + + /* + * Run decoder on the trace block. The decoder will stop when + * encountering a CS timestamp, a full packet queue or the end of + * trace for that block. + */ + ret = cs_etm__decode_data_block(etmq); + if (ret) + goto out; + + /* + * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all + * the timestamp calculation for us. + */ + cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); + + /* We found a timestamp, no need to continue. */ + if (cs_timestamp) + break; + + /* + * We didn't find a timestamp so empty all the traceid packet + * queues before looking for another timestamp packet, either + * in the current data block or a new one. Packets that were + * just decoded are useless since no timestamp has been + * associated with them. As such simply discard them. + */ + cs_etm__clear_all_packet_queues(etmq); + } + + /* + * We have a timestamp. Add it to the min heap to reflect when + * instructions conveyed by the range packets of this traceID queue + * started to execute. Once the same has been done for all the traceID + * queues of each etmq, redenring and decoding can start in + * chronological order. + * + * Note that packets decoded above are still in the traceID's packet + * queue and will be processed in cs_etm__process_timestamped_queues(). + */ + cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); + ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); +out: + return ret; +} + +static inline +void cs_etm__copy_last_branch_rb(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct branch_stack *bs_src = tidq->last_branch_rb; + struct branch_stack *bs_dst = tidq->last_branch; + size_t nr = 0; + + /* + * Set the number of records before early exit: ->nr is used to + * determine how many branches to copy from ->entries. + */ + bs_dst->nr = bs_src->nr; + + /* + * Early exit when there is nothing to copy. + */ + if (!bs_src->nr) + return; + + /* + * As bs_src->entries is a circular buffer, we need to copy from it in + * two steps. First, copy the branches from the most recently inserted + * branch ->last_branch_pos until the end of bs_src->entries buffer. + */ + nr = etmq->etm->synth_opts.last_branch_sz - tidq->last_branch_pos; + memcpy(&bs_dst->entries[0], + &bs_src->entries[tidq->last_branch_pos], + sizeof(struct branch_entry) * nr); + + /* + * If we wrapped around at least once, the branches from the beginning + * of the bs_src->entries buffer and until the ->last_branch_pos element + * are older valid branches: copy them over. The total number of + * branches copied over will be equal to the number of branches asked by + * the user in last_branch_sz. + */ + if (bs_src->nr >= etmq->etm->synth_opts.last_branch_sz) { + memcpy(&bs_dst->entries[nr], + &bs_src->entries[0], + sizeof(struct branch_entry) * tidq->last_branch_pos); + } +} + +static inline +void cs_etm__reset_last_branch_rb(struct cs_etm_traceid_queue *tidq) +{ + tidq->last_branch_pos = 0; + tidq->last_branch_rb->nr = 0; +} + +static inline int cs_etm__t32_instr_size(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 addr) +{ + u8 instrBytes[2]; + + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + ARRAY_SIZE(instrBytes), instrBytes); + /* + * T32 instruction size is indicated by bits[15:11] of the first + * 16-bit word of the instruction: 0b11101, 0b11110 and 0b11111 + * denote a 32-bit instruction. + */ + return ((instrBytes[1] & 0xF8) >= 0xE8) ? 4 : 2; +} + +static inline u64 cs_etm__first_executed_instr(struct cs_etm_packet *packet) +{ + /* + * Return 0 for packets that have no addresses so that CS_ETM_INVAL_ADDR doesn't + * appear in samples. + */ + if (packet->sample_type == CS_ETM_DISCONTINUITY || + packet->sample_type == CS_ETM_EXCEPTION) + return 0; + + return packet->start_addr; +} + +static inline +u64 cs_etm__last_executed_instr(const struct cs_etm_packet *packet) +{ + /* Returns 0 for the CS_ETM_DISCONTINUITY packet */ + if (packet->sample_type == CS_ETM_DISCONTINUITY) + return 0; + + return packet->end_addr - packet->last_instr_size; +} + +static inline u64 cs_etm__instr_addr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 offset) +{ + if (packet->isa == CS_ETM_ISA_T32) { + u64 addr = packet->start_addr; + + while (offset) { + addr += cs_etm__t32_instr_size(etmq, tidq, packet, + addr); + offset--; + } + return addr; + } + + /* Assume a 4 byte instruction size (A32/A64) */ + return packet->start_addr + offset * 4; +} + +static void cs_etm__update_last_branch_rb(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct branch_stack *bs = tidq->last_branch_rb; + struct branch_entry *be; + + /* + * The branches are recorded in a circular buffer in reverse + * chronological order: we start recording from the last element of the + * buffer down. After writing the first element of the stack, move the + * insert position back to the end of the buffer. + */ + if (!tidq->last_branch_pos) + tidq->last_branch_pos = etmq->etm->synth_opts.last_branch_sz; + + tidq->last_branch_pos -= 1; + + be = &bs->entries[tidq->last_branch_pos]; + be->from = cs_etm__last_executed_instr(tidq->prev_packet); + be->to = cs_etm__first_executed_instr(tidq->packet); + /* No support for mispredict */ + be->flags.mispred = 0; + be->flags.predicted = 1; + + /* + * Increment bs->nr until reaching the number of last branches asked by + * the user on the command line. + */ + if (bs->nr < etmq->etm->synth_opts.last_branch_sz) + bs->nr += 1; +} + +static int cs_etm__inject_event(struct cs_etm_auxtrace *etm, union perf_event *event, + struct perf_sample *sample, u64 type) +{ + struct evsel *evsel = sample->evsel; + u64 branch_sample_type = 0; + size_t sz; + + if (!evsel && etm->session && etm->session->evlist) + evsel = evlist__id2evsel(etm->session->evlist, sample->id); + + if (evsel) + branch_sample_type = evsel->core.attr.branch_sample_type; + + sz = perf_event__sample_event_size(sample, type, /*read_format=*/0, + branch_sample_type); + if (sz >= PERF_SAMPLE_MAX_SIZE) { + pr_err("Sample size %zu exceeds max size %d\n", sz, PERF_SAMPLE_MAX_SIZE); + return -EFAULT; + } + event->header.size = sz; + + return perf_event__synthesize_sample(event, type, /*read_format=*/0, + branch_sample_type, sample); +} + + +static int +cs_etm__get_trace(struct cs_etm_queue *etmq) +{ + struct auxtrace_buffer *aux_buffer = etmq->buffer; + struct auxtrace_buffer *old_buffer = aux_buffer; + struct auxtrace_queue *queue; + + queue = &etmq->etm->queues.queue_array[etmq->queue_nr]; + + aux_buffer = auxtrace_buffer__next(queue, aux_buffer); + + /* If no more data, drop the previous auxtrace_buffer and return */ + if (!aux_buffer) { + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + etmq->buf_len = 0; + return 0; + } + + etmq->buffer = aux_buffer; + + /* If the aux_buffer doesn't have data associated, try to load it */ + if (!aux_buffer->data) { + /* get the file desc associated with the perf data file */ + int fd = perf_data__fd(etmq->etm->session->data); + + aux_buffer->data = auxtrace_buffer__get_data(aux_buffer, fd); + if (!aux_buffer->data) + return -ENOMEM; + } + + /* If valid, drop the previous buffer */ + if (old_buffer) + auxtrace_buffer__drop_data(old_buffer); + + etmq->buf_used = 0; + etmq->buf_len = aux_buffer->size; + etmq->buf = aux_buffer->data; + + return etmq->buf_len; +} + +/* + * Convert a raw thread number to a thread struct and assign it to **thread. + */ +static int cs_etm__etmq_update_thread(struct cs_etm_queue *etmq, + ocsd_ex_level el, pid_t tid, + struct thread **thread) +{ + struct machine *machine = cs_etm__get_machine(etmq, el); + + if (!machine || !*thread) + return -EINVAL; + + if (tid != -1) { + thread__zput(*thread); + *thread = machine__find_thread(machine, -1, tid); + } + + /* Couldn't find a known thread */ + if (!*thread) + *thread = machine__idle_thread(machine); + + return 0; +} + +/* + * Set the thread and EL of the decode context which is ahead in time of the + * frontend context. + */ +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, + ocsd_ex_level el, pid_t tid) +{ + struct cs_etm_traceid_queue *tidq; + int ret; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) + return -EINVAL; + + ret = cs_etm__etmq_update_thread(etmq, el, tid, + &tidq->decode_thread); + if (ret) + return ret; + + tidq->decode_el = el; + return 0; +} + +bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq) +{ + return !!etmq->etm->timeless_decoding; +} + +static void cs_etm__copy_insn(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + struct perf_sample *sample) +{ + /* + * It's pointless to read instructions for the CS_ETM_DISCONTINUITY + * packet, so directly bail out with 'insn_len' = 0. + */ + if (packet->sample_type == CS_ETM_DISCONTINUITY) { + sample->insn_len = 0; + return; + } + + /* + * T32 instruction size might be 32-bit or 16-bit, decide by calling + * cs_etm__t32_instr_size(). + */ + if (packet->isa == CS_ETM_ISA_T32) + sample->insn_len = cs_etm__t32_instr_size(etmq, tidq, packet, + sample->ip); + /* Otherwise, A64 and A32 instruction size are always 32-bit. */ + else + sample->insn_len = 4; + + cs_etm__frontend_mem_access(etmq, tidq, packet, sample->ip, + sample->insn_len, (void *)sample->insn); +} + +u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + + if (etm->has_virtual_ts) + return tsc_to_perf_time(cs_timestamp, &etm->tc); + else + return cs_timestamp; +} + +static inline u64 cs_etm__resolve_sample_time(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + struct cs_etm_packet_queue *packet_queue = &tidq->packet_queue; + + if (!etm->timeless_decoding && etm->has_virtual_ts) + return packet_queue->cs_timestamp; + else + return etm->latest_kernel_timestamp; +} + +static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, + u64 addr, u64 period) +{ + int ret = 0; + struct cs_etm_auxtrace *etm = etmq->etm; + union perf_event *event = tidq->event_buf; + struct perf_sample sample; + + perf_sample__init(&sample, /*all=*/true); + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = cs_etm__cpu_mode(etmq, addr, packet->el); + event->sample.header.size = sizeof(struct perf_event_header); + + /* Set time field based on etm auxtrace config. */ + sample.time = cs_etm__resolve_sample_time(etmq, tidq); + + sample.ip = addr; + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); + sample.id = etmq->etm->instructions_id; + sample.stream_id = etmq->etm->instructions_id; + sample.period = period; + sample.cpu = packet->cpu; + sample.flags = tidq->prev_packet->flags; + sample.cpumode = event->sample.header.misc; + + cs_etm__copy_insn(etmq, tidq, packet, &sample); + + if (etm->synth_opts.last_branch) + sample.branch_stack = tidq->last_branch; + + if (etm->synth_opts.inject) { + ret = cs_etm__inject_event(etm, event, &sample, + etm->instructions_sample_type); + if (ret) + return ret; + } + + ret = perf_session__deliver_synth_event(etm->session, event, &sample); + + if (ret) + pr_err( + "CS ETM Trace: failed to deliver instruction event, error %d\n", + ret); + + perf_sample__exit(&sample); + return ret; +} + +/* + * The cs etm packet encodes an instruction range between a branch target + * and the next taken branch. Generate sample accordingly. + */ +static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int ret = 0; + struct cs_etm_auxtrace *etm = etmq->etm; + struct perf_sample sample = {.ip = 0,}; + union perf_event *event = tidq->event_buf; + struct dummy_branch_stack { + u64 nr; + u64 hw_idx; + struct branch_entry entries; + } dummy_bs; + u64 ip; + + ip = cs_etm__last_executed_instr(tidq->prev_packet); + + event->sample.header.type = PERF_RECORD_SAMPLE; + event->sample.header.misc = cs_etm__cpu_mode(etmq, ip, + tidq->prev_packet->el); + event->sample.header.size = sizeof(struct perf_event_header); + + /* Set time field based on etm auxtrace config. */ + sample.time = cs_etm__resolve_sample_time(etmq, tidq); + + sample.ip = ip; + sample.pid = thread__pid(tidq->frontend_thread); + sample.tid = thread__tid(tidq->frontend_thread); + sample.addr = cs_etm__first_executed_instr(tidq->packet); + sample.id = etmq->etm->branches_id; + sample.stream_id = etmq->etm->branches_id; + sample.period = 1; + sample.cpu = tidq->packet->cpu; + sample.flags = tidq->prev_packet->flags; + sample.cpumode = event->sample.header.misc; + + cs_etm__copy_insn(etmq, tidq, tidq->prev_packet, &sample); + + /* + * perf report cannot handle events without a branch stack + */ + if (etm->synth_opts.last_branch) { + dummy_bs = (struct dummy_branch_stack){ + .nr = 1, + .hw_idx = -1ULL, + .entries = { + .from = sample.ip, + .to = sample.addr, + }, + }; + sample.branch_stack = (struct branch_stack *)&dummy_bs; + } + + if (etm->synth_opts.inject) { + ret = cs_etm__inject_event(etm, event, &sample, + etm->branches_sample_type); + if (ret) + return ret; + } + + ret = perf_session__deliver_synth_event(etm->session, event, &sample); + + if (ret) + pr_err( + "CS ETM Trace: failed to deliver instruction event, error %d\n", + ret); + + return ret; +} + +static int cs_etm__synth_events(struct cs_etm_auxtrace *etm, + struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + struct perf_event_attr attr; + bool found = false; + u64 id; + int err; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type == etm->pmu_type) { + found = true; + break; + } + } + + if (!found) { + pr_debug("No selected events with CoreSight Trace data\n"); + return 0; + } + + memset(&attr, 0, sizeof(struct perf_event_attr)); + attr.size = sizeof(struct perf_event_attr); + attr.type = PERF_TYPE_HARDWARE; + attr.sample_type = evsel->core.attr.sample_type & PERF_SAMPLE_MASK; + attr.sample_type |= PERF_SAMPLE_IP | PERF_SAMPLE_TID | + PERF_SAMPLE_PERIOD; + if (etm->timeless_decoding) + attr.sample_type &= ~(u64)PERF_SAMPLE_TIME; + else + attr.sample_type |= PERF_SAMPLE_TIME; + + attr.exclude_user = evsel->core.attr.exclude_user; + attr.exclude_kernel = evsel->core.attr.exclude_kernel; + attr.exclude_hv = evsel->core.attr.exclude_hv; + attr.exclude_host = evsel->core.attr.exclude_host; + attr.exclude_guest = evsel->core.attr.exclude_guest; + attr.sample_id_all = evsel->core.attr.sample_id_all; + attr.read_format = evsel->core.attr.read_format; + + /* create new id val to be a fixed offset from evsel id */ + id = auxtrace_synth_id_range_start(evsel); + + if (etm->synth_opts.branches) { + attr.config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS; + attr.sample_period = 1; + attr.sample_type |= PERF_SAMPLE_ADDR; + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + etm->branches_sample_type = attr.sample_type; + etm->branches_id = id; + id += 1; + attr.sample_type &= ~(u64)PERF_SAMPLE_ADDR; + } + + if (etm->synth_opts.last_branch) { + attr.sample_type |= PERF_SAMPLE_BRANCH_STACK; + /* + * We don't use the hardware index, but the sample generation + * code uses the new format branch_stack with this field, + * so the event attributes must indicate that it's present. + */ + attr.branch_sample_type |= PERF_SAMPLE_BRANCH_HW_INDEX; + } + + if (etm->synth_opts.instructions) { + attr.config = PERF_COUNT_HW_INSTRUCTIONS; + attr.sample_period = etm->synth_opts.period; + etm->instructions_sample_period = attr.sample_period; + err = perf_session__deliver_synth_attr_event(session, &attr, id); + if (err) + return err; + etm->instructions_sample_type = attr.sample_type; + etm->instructions_id = id; + id += 1; + } + + return 0; +} + +static int cs_etm__sample(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_auxtrace *etm = etmq->etm; + int ret; + u64 instrs_prev; + + /* Get instructions remainder from previous packet */ + instrs_prev = tidq->period_instructions; + + tidq->period_instructions += tidq->packet->instr_count; + + /* + * Record a branch when the last instruction in + * PREV_PACKET is a branch. + */ + if (etm->synth_opts.last_branch && + tidq->prev_packet->sample_type == CS_ETM_RANGE && + tidq->prev_packet->last_instr_taken_branch) + cs_etm__update_last_branch_rb(etmq, tidq); + + if (etm->synth_opts.instructions && + tidq->period_instructions >= etm->instructions_sample_period) { + /* + * Emit instruction sample periodically + * TODO: allow period to be defined in cycles and clock time + */ + + /* + * Below diagram demonstrates the instruction samples + * generation flows: + * + * Instrs Instrs Instrs Instrs + * Sample(n) Sample(n+1) Sample(n+2) Sample(n+3) + * | | | | + * V V V V + * -------------------------------------------------- + * ^ ^ + * | | + * Period Period + * instructions(Pi) instructions(Pi') + * + * | | + * \---------------- -----------------/ + * V + * tidq->packet->instr_count + * + * Instrs Sample(n...) are the synthesised samples occurring + * every etm->instructions_sample_period instructions - as + * defined on the perf command line. Sample(n) is being the + * last sample before the current etm packet, n+1 to n+3 + * samples are generated from the current etm packet. + * + * tidq->packet->instr_count represents the number of + * instructions in the current etm packet. + * + * Period instructions (Pi) contains the number of + * instructions executed after the sample point(n) from the + * previous etm packet. This will always be less than + * etm->instructions_sample_period. + * + * When generate new samples, it combines with two parts + * instructions, one is the tail of the old packet and another + * is the head of the new coming packet, to generate + * sample(n+1); sample(n+2) and sample(n+3) consume the + * instructions with sample period. After sample(n+3), the rest + * instructions will be used by later packet and it is assigned + * to tidq->period_instructions for next round calculation. + */ + + /* + * Get the initial offset into the current packet instructions; + * entry conditions ensure that instrs_prev is less than + * etm->instructions_sample_period. + */ + u64 offset = etm->instructions_sample_period - instrs_prev; + u64 addr; + + /* Prepare last branches for instruction sample */ + if (etm->synth_opts.last_branch) + cs_etm__copy_last_branch_rb(etmq, tidq); + + while (tidq->period_instructions >= + etm->instructions_sample_period) { + /* + * Calculate the address of the sampled instruction (-1 + * as sample is reported as though instruction has just + * been executed, but PC has not advanced to next + * instruction) + */ + addr = cs_etm__instr_addr(etmq, tidq, tidq->packet, + offset - 1); + ret = cs_etm__synth_instruction_sample( + etmq, tidq, tidq->packet, addr, + etm->instructions_sample_period); + if (ret) + return ret; + + offset += etm->instructions_sample_period; + tidq->period_instructions -= + etm->instructions_sample_period; + } + } + + if (etm->synth_opts.branches) { + bool generate_sample = false; + + /* Generate sample for tracing on packet */ + if (tidq->prev_packet->sample_type == CS_ETM_DISCONTINUITY) + generate_sample = true; + + /* Generate sample for branch taken packet */ + if (tidq->prev_packet->sample_type == CS_ETM_RANGE && + tidq->prev_packet->last_instr_taken_branch) + generate_sample = true; + + if (generate_sample) { + ret = cs_etm__synth_branch_sample(etmq, tidq); + if (ret) + return ret; + } + } + + cs_etm__packet_swap(etm, tidq); + + return 0; +} + +static int cs_etm__exception(struct cs_etm_traceid_queue *tidq) +{ + /* + * When the exception packet is inserted, whether the last instruction + * in previous range packet is taken branch or not, we need to force + * to set 'prev_packet->last_instr_taken_branch' to true. This ensures + * to generate branch sample for the instruction range before the + * exception is trapped to kernel or before the exception returning. + * + * The exception packet includes the dummy address values, so don't + * swap PACKET with PREV_PACKET. This keeps PREV_PACKET to be useful + * for generating instruction and branch samples. + */ + if (tidq->prev_packet->sample_type == CS_ETM_RANGE) + tidq->prev_packet->last_instr_taken_branch = true; + + return 0; +} + +static int cs_etm__flush(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int err = 0; + struct cs_etm_auxtrace *etm = etmq->etm; + + /* Handle start tracing packet */ + if (tidq->prev_packet->sample_type == CS_ETM_EMPTY) + goto swap_packet; + + if (etmq->etm->synth_opts.last_branch && + etmq->etm->synth_opts.instructions && + tidq->prev_packet->sample_type == CS_ETM_RANGE) { + u64 addr; + + /* Prepare last branches for instruction sample */ + cs_etm__copy_last_branch_rb(etmq, tidq); + + /* + * Generate a last branch event for the branches left in the + * circular buffer at the end of the trace. + * + * Use the address of the end of the last reported execution + * range + */ + addr = cs_etm__last_executed_instr(tidq->prev_packet); + + err = cs_etm__synth_instruction_sample( + etmq, tidq, tidq->prev_packet, addr, + tidq->period_instructions); + if (err) + return err; + + tidq->period_instructions = 0; + + } + + if (etm->synth_opts.branches && + tidq->prev_packet->sample_type == CS_ETM_RANGE) { + err = cs_etm__synth_branch_sample(etmq, tidq); + if (err) + return err; + } + +swap_packet: + cs_etm__packet_swap(etm, tidq); + + /* Reset last branches after flush the trace */ + if (etm->synth_opts.last_branch) + cs_etm__reset_last_branch_rb(tidq); + + return err; +} + +static int cs_etm__end_block(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int err; + + /* + * It has no new packet coming and 'etmq->packet' contains the stale + * packet which was set at the previous time with packets swapping; + * so skip to generate branch sample to avoid stale packet. + * + * For this case only flush branch stack and generate a last branch + * event for the branches left in the circular buffer at the end of + * the trace. + */ + if (etmq->etm->synth_opts.last_branch && + etmq->etm->synth_opts.instructions && + tidq->prev_packet->sample_type == CS_ETM_RANGE) { + u64 addr; + + /* Prepare last branches for instruction sample */ + cs_etm__copy_last_branch_rb(etmq, tidq); + + /* + * Use the address of the end of the last reported execution + * range. + */ + addr = cs_etm__last_executed_instr(tidq->prev_packet); + + err = cs_etm__synth_instruction_sample( + etmq, tidq, tidq->prev_packet, addr, + tidq->period_instructions); + if (err) + return err; + + tidq->period_instructions = 0; + } + + return 0; +} +/* + * cs_etm__get_data_block: Fetch a block from the auxtrace_buffer queue + * if need be. + * Returns: < 0 if error + * = 0 if no more auxtrace_buffer to read + * > 0 if the current buffer isn't empty yet + */ +static int cs_etm__get_data_block(struct cs_etm_queue *etmq) +{ + int ret; + + if (!etmq->buf_len) { + ret = cs_etm__get_trace(etmq); + if (ret <= 0) + return ret; + /* + * We cannot assume consecutive blocks in the data file + * are contiguous, reset the decoder to force re-sync. + */ + ret = cs_etm_decoder__reset(etmq->decoder); + if (ret) + return ret; + } + + return etmq->buf_len; +} + +static bool cs_etm__is_svc_instr(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + struct cs_etm_packet *packet, u64 end_addr) +{ + /* Initialise to keep compiler happy */ + u16 instr16 = 0; + u32 instr32 = 0; + u64 addr; + + switch (packet->isa) { + case CS_ETM_ISA_T32: + /* + * The SVC of T32 is defined in ARM DDI 0487D.a, F5.1.247: + * + * b'15 b'8 + * +-----------------+--------+ + * | 1 1 0 1 1 1 1 1 | imm8 | + * +-----------------+--------+ + * + * According to the specification, it only defines SVC for T32 + * with 16 bits instruction and has no definition for 32bits; + * so below only read 2 bytes as instruction size for T32. + */ + addr = end_addr - 2; + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr16), (u8 *)&instr16); + if ((instr16 & 0xFF00) == 0xDF00) + return true; + + break; + case CS_ETM_ISA_A32: + /* + * The SVC of A32 is defined in ARM DDI 0487D.a, F5.1.247: + * + * b'31 b'28 b'27 b'24 + * +---------+---------+-------------------------+ + * | !1111 | 1 1 1 1 | imm24 | + * +---------+---------+-------------------------+ + */ + addr = end_addr - 4; + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); + if ((instr32 & 0x0F000000) == 0x0F000000 && + (instr32 & 0xF0000000) != 0xF0000000) + return true; + + break; + case CS_ETM_ISA_A64: + /* + * The SVC of A64 is defined in ARM DDI 0487D.a, C6.2.294: + * + * b'31 b'21 b'4 b'0 + * +-----------------------+---------+-----------+ + * | 1 1 0 1 0 1 0 0 0 0 0 | imm16 | 0 0 0 0 1 | + * +-----------------------+---------+-----------+ + */ + addr = end_addr - 4; + cs_etm__frontend_mem_access(etmq, tidq, packet, addr, + sizeof(instr32), (u8 *)&instr32); + if ((instr32 & 0xFFE0001F) == 0xd4000001) + return true; + + break; + case CS_ETM_ISA_UNKNOWN: + default: + break; + } + + return false; +} + +static bool cs_etm__is_syscall(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, u64 magic) +{ + struct cs_etm_packet *packet = tidq->packet; + struct cs_etm_packet *prev_packet = tidq->prev_packet; + + if (magic == __perf_cs_etmv3_magic) + if (packet->exception_number == CS_ETMV3_EXC_SVC) + return true; + + /* + * ETMv4 exception type CS_ETMV4_EXC_CALL covers SVC, SMC and + * HVC cases; need to check if it's SVC instruction based on + * packet address. + */ + if (magic == __perf_cs_etmv4_magic) { + if (packet->exception_number == CS_ETMV4_EXC_CALL && + cs_etm__is_svc_instr(etmq, tidq, prev_packet, + prev_packet->end_addr)) + return true; + } + + return false; +} + +static bool cs_etm__is_async_exception(struct cs_etm_traceid_queue *tidq, + u64 magic) +{ + struct cs_etm_packet *packet = tidq->packet; + + if (magic == __perf_cs_etmv3_magic) + if (packet->exception_number == CS_ETMV3_EXC_DEBUG_HALT || + packet->exception_number == CS_ETMV3_EXC_ASYNC_DATA_ABORT || + packet->exception_number == CS_ETMV3_EXC_PE_RESET || + packet->exception_number == CS_ETMV3_EXC_IRQ || + packet->exception_number == CS_ETMV3_EXC_FIQ) + return true; + + if (magic == __perf_cs_etmv4_magic) + if (packet->exception_number == CS_ETMV4_EXC_RESET || + packet->exception_number == CS_ETMV4_EXC_DEBUG_HALT || + packet->exception_number == CS_ETMV4_EXC_SYSTEM_ERROR || + packet->exception_number == CS_ETMV4_EXC_INST_DEBUG || + packet->exception_number == CS_ETMV4_EXC_DATA_DEBUG || + packet->exception_number == CS_ETMV4_EXC_IRQ || + packet->exception_number == CS_ETMV4_EXC_FIQ) + return true; + + return false; +} + +static bool cs_etm__is_sync_exception(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq, + u64 magic) +{ + struct cs_etm_packet *packet = tidq->packet; + struct cs_etm_packet *prev_packet = tidq->prev_packet; + + if (magic == __perf_cs_etmv3_magic) + if (packet->exception_number == CS_ETMV3_EXC_SMC || + packet->exception_number == CS_ETMV3_EXC_HYP || + packet->exception_number == CS_ETMV3_EXC_JAZELLE_THUMBEE || + packet->exception_number == CS_ETMV3_EXC_UNDEFINED_INSTR || + packet->exception_number == CS_ETMV3_EXC_PREFETCH_ABORT || + packet->exception_number == CS_ETMV3_EXC_DATA_FAULT || + packet->exception_number == CS_ETMV3_EXC_GENERIC) + return true; + + if (magic == __perf_cs_etmv4_magic) { + if (packet->exception_number == CS_ETMV4_EXC_TRAP || + packet->exception_number == CS_ETMV4_EXC_ALIGNMENT || + packet->exception_number == CS_ETMV4_EXC_INST_FAULT || + packet->exception_number == CS_ETMV4_EXC_DATA_FAULT) + return true; + + /* + * For CS_ETMV4_EXC_CALL, except SVC other instructions + * (SMC, HVC) are taken as sync exceptions. + */ + if (packet->exception_number == CS_ETMV4_EXC_CALL && + !cs_etm__is_svc_instr(etmq, tidq, prev_packet, + prev_packet->end_addr)) + return true; + + /* + * ETMv4 has 5 bits for exception number; if the numbers + * are in the range ( CS_ETMV4_EXC_FIQ, CS_ETMV4_EXC_END ] + * they are implementation defined exceptions. + * + * For this case, simply take it as sync exception. + */ + if (packet->exception_number > CS_ETMV4_EXC_FIQ && + packet->exception_number <= CS_ETMV4_EXC_END) + return true; + } + + return false; +} + +static int cs_etm__set_sample_flags(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + struct cs_etm_packet *packet = tidq->packet; + struct cs_etm_packet *prev_packet = tidq->prev_packet; + u64 magic; + int ret; + + switch (packet->sample_type) { + case CS_ETM_RANGE: + /* + * Immediate branch instruction without neither link nor + * return flag, it's normal branch instruction within + * the function. + */ + if (packet->last_instr_type == OCSD_INSTR_BR && + packet->last_instr_subtype == OCSD_S_INSTR_NONE) { + packet->flags = PERF_IP_FLAG_BRANCH; + + if (packet->last_instr_cond) + packet->flags |= PERF_IP_FLAG_CONDITIONAL; + } + + /* + * Immediate branch instruction with link (e.g. BL), this is + * branch instruction for function call. + */ + if (packet->last_instr_type == OCSD_INSTR_BR && + packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL; + + /* + * Indirect branch instruction with link (e.g. BLR), this is + * branch instruction for function call. + */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_BR_LINK) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL; + + /* + * Indirect branch instruction with subtype of + * OCSD_S_INSTR_V7_IMPLIED_RET, this is explicit hint for + * function return for A32/T32. + */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_V7_IMPLIED_RET) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN; + + /* + * Indirect branch instruction without link (e.g. BR), usually + * this is used for function return, especially for functions + * within dynamic link lib. + */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_NONE) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN; + + /* Return instruction for function return. */ + if (packet->last_instr_type == OCSD_INSTR_BR_INDIRECT && + packet->last_instr_subtype == OCSD_S_INSTR_V8_RET) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN; + + /* + * Decoder might insert a discontinuity in the middle of + * instruction packets, fixup prev_packet with flag + * PERF_IP_FLAG_TRACE_BEGIN to indicate restarting trace. + */ + if (prev_packet->sample_type == CS_ETM_DISCONTINUITY) + prev_packet->flags |= PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_TRACE_BEGIN; + + /* + * If the previous packet is an exception return packet + * and the return address just follows SVC instruction, + * it needs to calibrate the previous packet sample flags + * as PERF_IP_FLAG_SYSCALLRET. + */ + if (prev_packet->flags == (PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_INTERRUPT) && + cs_etm__is_svc_instr(etmq, tidq, packet, packet->start_addr)) { + prev_packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_SYSCALLRET; + } + break; + case CS_ETM_DISCONTINUITY: + /* + * The trace is discontinuous, if the previous packet is + * instruction packet, set flag PERF_IP_FLAG_TRACE_END + * for previous packet. + */ + if (prev_packet->sample_type == CS_ETM_RANGE) + prev_packet->flags |= PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_TRACE_END; + break; + case CS_ETM_EXCEPTION: + ret = cs_etm__get_magic(etmq, packet->trace_chan_id, &magic); + if (ret) + return ret; + + /* The exception is for system call. */ + if (cs_etm__is_syscall(etmq, tidq, magic)) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL | + PERF_IP_FLAG_SYSCALLRET; + /* + * The exceptions are triggered by external signals from bus, + * interrupt controller, debug module, PE reset or halt. + */ + else if (cs_etm__is_async_exception(tidq, magic)) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL | + PERF_IP_FLAG_ASYNC | + PERF_IP_FLAG_INTERRUPT; + /* + * Otherwise, exception is caused by trap, instruction & + * data fault, or alignment errors. + */ + else if (cs_etm__is_sync_exception(etmq, tidq, magic)) + packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_CALL | + PERF_IP_FLAG_INTERRUPT; + + /* + * When the exception packet is inserted, since exception + * packet is not used standalone for generating samples + * and it's affiliation to the previous instruction range + * packet; so set previous range packet flags to tell perf + * it is an exception taken branch. + */ + if (prev_packet->sample_type == CS_ETM_RANGE) + prev_packet->flags = packet->flags; + break; + case CS_ETM_EXCEPTION_RET: + /* + * When the exception return packet is inserted, since + * exception return packet is not used standalone for + * generating samples and it's affiliation to the previous + * instruction range packet; so set previous range packet + * flags to tell perf it is an exception return branch. + * + * The exception return can be for either system call or + * other exception types; unfortunately the packet doesn't + * contain exception type related info so we cannot decide + * the exception type purely based on exception return packet. + * If we record the exception number from exception packet and + * reuse it for exception return packet, this is not reliable + * due the trace can be discontinuity or the interrupt can + * be nested, thus the recorded exception number cannot be + * used for exception return packet for these two cases. + * + * For exception return packet, we only need to distinguish the + * packet is for system call or for other types. Thus the + * decision can be deferred when receive the next packet which + * contains the return address, based on the return address we + * can read out the previous instruction and check if it's a + * system call instruction and then calibrate the sample flag + * as needed. + */ + if (prev_packet->sample_type == CS_ETM_RANGE) + prev_packet->flags = PERF_IP_FLAG_BRANCH | + PERF_IP_FLAG_RETURN | + PERF_IP_FLAG_INTERRUPT; + break; + case CS_ETM_CONTEXT: + case CS_ETM_EMPTY: + default: + break; + } + + return 0; +} + +static int cs_etm__decode_data_block(struct cs_etm_queue *etmq) +{ + int ret = 0; + size_t processed = 0; + + /* + * Packets are decoded and added to the decoder's packet queue + * until the decoder packet processing callback has requested that + * processing stops or there is nothing left in the buffer. Normal + * operations that stop processing are a timestamp packet or a full + * decoder buffer queue. + */ + ret = cs_etm_decoder__process_data_block(etmq->decoder, + etmq->offset, + &etmq->buf[etmq->buf_used], + etmq->buf_len, + &processed); + if (ret) + goto out; + + etmq->offset += processed; + etmq->buf_used += processed; + etmq->buf_len -= processed; + +out: + return ret; +} + +static int cs_etm__process_traceid_queue(struct cs_etm_queue *etmq, + struct cs_etm_traceid_queue *tidq) +{ + int ret; + struct cs_etm_packet_queue *packet_queue; + + packet_queue = &tidq->packet_queue; + + /* Process each packet in this chunk */ + while (1) { + ret = cs_etm_decoder__get_packet(packet_queue, + tidq->packet); + if (ret <= 0) + /* + * Stop processing this chunk on + * end of data or error + */ + break; + + /* + * Since packet addresses are swapped in packet + * handling within below switch() statements, + * thus setting sample flags must be called + * prior to switch() statement to use address + * information before packets swapping. + */ + ret = cs_etm__set_sample_flags(etmq, tidq); + if (ret < 0) + break; + + switch (tidq->packet->sample_type) { + case CS_ETM_RANGE: + /* + * If the packet contains an instruction + * range, generate instruction sequence + * events. + */ + cs_etm__sample(etmq, tidq); + break; + case CS_ETM_CONTEXT: + /* + * Update context but don't swap packet. Keep the + * previous one for branch source address info, if + * tracing the kernel the context packet will be emitted + * between two ranges. + */ + ret = cs_etm__etmq_update_thread(etmq, tidq->packet->el, + tidq->packet->tid, + &tidq->frontend_thread); + if (ret) + goto out; + break; + case CS_ETM_EXCEPTION: + case CS_ETM_EXCEPTION_RET: + /* + * If the exception packet is coming, + * make sure the previous instruction + * range packet to be handled properly. + */ + cs_etm__exception(tidq); + break; + case CS_ETM_DISCONTINUITY: + /* + * Discontinuity in trace, flush + * previous branch stack + */ + cs_etm__flush(etmq, tidq); + break; + case CS_ETM_EMPTY: + /* + * Should not receive empty packet, + * report error. + */ + pr_err("CS ETM Trace: empty packet\n"); + return -EINVAL; + default: + break; + } + } + +out: + return ret; +} + +static void cs_etm__clear_all_traceid_queues(struct cs_etm_queue *etmq) +{ + int idx; + struct int_node *inode; + struct cs_etm_traceid_queue *tidq; + struct intlist *traceid_queues_list = etmq->traceid_queues_list; + + intlist__for_each_entry(inode, traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + + /* Ignore return value */ + cs_etm__process_traceid_queue(etmq, tidq); + } +} + +static int cs_etm__run_per_thread_timeless_decoder(struct cs_etm_queue *etmq) +{ + int err = 0; + struct cs_etm_traceid_queue *tidq; + + tidq = cs_etm__etmq_get_traceid_queue(etmq, CS_ETM_PER_THREAD_TRACEID); + if (!tidq) + return -EINVAL; + + /* Go through each buffer in the queue and decode them one by one */ + while (1) { + err = cs_etm__get_data_block(etmq); + if (err <= 0) + return err; + + /* Run trace decoder until buffer consumed or end of trace */ + do { + err = cs_etm__decode_data_block(etmq); + if (err) + return err; + + /* + * Process each packet in this chunk, nothing to do if + * an error occurs other than hoping the next one will + * be better. + */ + err = cs_etm__process_traceid_queue(etmq, tidq); + + } while (etmq->buf_len); + + if (err == 0) + /* Flush any remaining branch stack entries */ + err = cs_etm__end_block(etmq, tidq); + } + + return err; +} + +static int cs_etm__run_per_cpu_timeless_decoder(struct cs_etm_queue *etmq) +{ + int idx, err = 0; + struct cs_etm_traceid_queue *tidq; + struct int_node *inode; + + /* Go through each buffer in the queue and decode them one by one */ + while (1) { + err = cs_etm__get_data_block(etmq); + if (err <= 0) + return err; + + /* Run trace decoder until buffer consumed or end of trace */ + do { + err = cs_etm__decode_data_block(etmq); + if (err) + return err; + + /* + * cs_etm__run_per_thread_timeless_decoder() runs on a + * single traceID queue because each TID has a separate + * buffer. But here in per-cpu mode we need to iterate + * over each channel instead. + */ + intlist__for_each_entry(inode, + etmq->traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + cs_etm__process_traceid_queue(etmq, tidq); + } + } while (etmq->buf_len); + + intlist__for_each_entry(inode, etmq->traceid_queues_list) { + idx = (int)(intptr_t)inode->priv; + tidq = etmq->traceid_queues[idx]; + /* Flush any remaining branch stack entries */ + err = cs_etm__end_block(etmq, tidq); + if (err) + return err; + } + } + + return err; +} + +static int cs_etm__process_timeless_queues(struct cs_etm_auxtrace *etm, + pid_t tid) +{ + unsigned int i; + struct auxtrace_queues *queues = &etm->queues; + + for (i = 0; i < queues->nr_queues; i++) { + struct auxtrace_queue *queue = &etm->queues.queue_array[i]; + struct cs_etm_queue *etmq = queue->priv; + struct cs_etm_traceid_queue *tidq; + + if (!etmq) + continue; + + if (etm->per_thread_decoding) { + tidq = cs_etm__etmq_get_traceid_queue( + etmq, CS_ETM_PER_THREAD_TRACEID); + + if (!tidq) + continue; + + if (tid == -1 || thread__tid(tidq->frontend_thread) == tid) + cs_etm__run_per_thread_timeless_decoder(etmq); + } else + cs_etm__run_per_cpu_timeless_decoder(etmq); + } + + return 0; +} + +static int cs_etm__process_timestamped_queues(struct cs_etm_auxtrace *etm) +{ + int ret = 0; + unsigned int cs_queue_nr, queue_nr, i; + u8 trace_chan_id; + u64 cs_timestamp; + struct auxtrace_queue *queue; + struct cs_etm_queue *etmq; + struct cs_etm_traceid_queue *tidq; + + /* + * Pre-populate the heap with one entry from each queue so that we can + * start processing in time order across all queues. + */ + for (i = 0; i < etm->queues.nr_queues; i++) { + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + ret = cs_etm__queue_first_cs_timestamp(etm, etmq, i); + if (ret) + return ret; + } + + while (1) { + if (!etm->heap.heap_cnt) + break; + + /* Take the entry at the top of the min heap */ + cs_queue_nr = etm->heap.heap_array[0].queue_nr; + queue_nr = TO_QUEUE_NR(cs_queue_nr); + trace_chan_id = TO_TRACE_CHAN_ID(cs_queue_nr); + queue = &etm->queues.queue_array[queue_nr]; + etmq = queue->priv; + + /* + * Remove the top entry from the heap since we are about + * to process it. + */ + auxtrace_heap__pop(&etm->heap); + + tidq = cs_etm__etmq_get_traceid_queue(etmq, trace_chan_id); + if (!tidq) { + /* + * No traceID queue has been allocated for this traceID, + * which means something somewhere went very wrong. No + * other choice than simply exit. + */ + ret = -EINVAL; + goto out; + } + + /* + * Packets associated with this timestamp are already in + * the etmq's traceID queue, so process them. + */ + ret = cs_etm__process_traceid_queue(etmq, tidq); + if (ret < 0) + goto out; + + /* + * Packets for this timestamp have been processed, time to + * move on to the next timestamp, fetching a new auxtrace_buffer + * if need be. + */ +refetch: + ret = cs_etm__get_data_block(etmq); + if (ret < 0) + goto out; + + /* + * No more auxtrace_buffers to process in this etmq, simply + * move on to another entry in the auxtrace_heap. + */ + if (!ret) + continue; + + ret = cs_etm__decode_data_block(etmq); + if (ret) + goto out; + + cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); + + if (!cs_timestamp) { + /* + * Function cs_etm__decode_data_block() returns when + * there is no more traces to decode in the current + * auxtrace_buffer OR when a timestamp has been + * encountered on any of the traceID queues. Since we + * did not get a timestamp, there is no more traces to + * process in this auxtrace_buffer. As such empty and + * flush all traceID queues. + */ + cs_etm__clear_all_traceid_queues(etmq); + + /* Fetch another auxtrace_buffer for this etmq */ + goto refetch; + } + + /* + * Add to the min heap the timestamp for packets that have + * just been decoded. They will be processed and synthesized + * during the next call to cs_etm__process_traceid_queue() for + * this queue/traceID. + */ + cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); + ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); + } + + for (i = 0; i < etm->queues.nr_queues; i++) { + struct int_node *inode; + + etmq = etm->queues.queue_array[i].priv; + if (!etmq) + continue; + + intlist__for_each_entry(inode, etmq->traceid_queues_list) { + int idx = (int)(intptr_t)inode->priv; + + /* Flush any remaining branch stack entries */ + tidq = etmq->traceid_queues[idx]; + ret = cs_etm__end_block(etmq, tidq); + if (ret) + return ret; + } + } +out: + return ret; +} + +static int cs_etm__process_itrace_start(struct cs_etm_auxtrace *etm, + union perf_event *event) +{ + struct thread *th; + + if (etm->timeless_decoding) + return 0; + + /* + * Add the tid/pid to the log so that we can get a match when we get a + * contextID from the decoder. Only track for the host: only kernel + * trace is supported for guests which wouldn't need pids so this should + * be fine. + */ + th = machine__findnew_thread(&etm->session->machines.host, + event->itrace_start.pid, + event->itrace_start.tid); + if (!th) + return -ENOMEM; + + thread__put(th); + + return 0; +} + +static int cs_etm__process_switch_cpu_wide(struct cs_etm_auxtrace *etm, + union perf_event *event) +{ + struct thread *th; + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + + /* + * Context switch in per-thread mode are irrelevant since perf + * will start/stop tracing as the process is scheduled. + */ + if (etm->timeless_decoding) + return 0; + + /* + * SWITCH_IN events carry the next process to be switched out while + * SWITCH_OUT events carry the process to be switched in. As such + * we don't care about IN events. + */ + if (!out) + return 0; + + /* + * Add the tid/pid to the log so that we can get a match when we get a + * contextID from the decoder. Only track for the host: only kernel + * trace is supported for guests which wouldn't need pids so this should + * be fine. + */ + th = machine__findnew_thread(&etm->session->machines.host, + event->context_switch.next_prev_pid, + event->context_switch.next_prev_tid); + if (!th) + return -ENOMEM; + + thread__put(th); + + return 0; +} + +static int cs_etm__process_event(struct perf_session *session, + union perf_event *event, + struct perf_sample *sample, + const struct perf_tool *tool) +{ + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + if (dump_trace) + return 0; + + if (!tool->ordered_events) { + pr_err("CoreSight ETM Trace requires ordered events\n"); + return -EINVAL; + } + + switch (event->header.type) { + case PERF_RECORD_EXIT: + /* + * Don't need to wait for cs_etm__flush_events() in per-thread mode to + * start the decode because we know there will be no more trace from + * this thread. All this does is emit samples earlier than waiting for + * the flush in other modes, but with timestamps it makes sense to wait + * for flush so that events from different threads are interleaved + * properly. + */ + if (etm->per_thread_decoding && etm->timeless_decoding) + return cs_etm__process_timeless_queues(etm, + event->fork.tid); + break; + + case PERF_RECORD_ITRACE_START: + return cs_etm__process_itrace_start(etm, event); + + case PERF_RECORD_SWITCH_CPU_WIDE: + return cs_etm__process_switch_cpu_wide(etm, event); + + case PERF_RECORD_AUX: + /* + * Record the latest kernel timestamp available in the header + * for samples so that synthesised samples occur from this point + * onwards. + */ + if (sample->time && (sample->time != (u64)-1)) + etm->latest_kernel_timestamp = sample->time; + break; + + default: + break; + } + + return 0; +} + +static void dump_queued_data(struct cs_etm_auxtrace *etm, + struct perf_record_auxtrace *event) +{ + struct auxtrace_buffer *buf; + unsigned int i; + /* + * Find all buffers with same reference in the queues and dump them. + * This is because the queues can contain multiple entries of the same + * buffer that were split on aux records. + */ + for (i = 0; i < etm->queues.nr_queues; ++i) + list_for_each_entry(buf, &etm->queues.queue_array[i].head, list) + if (buf->reference == event->reference) + cs_etm__dump_event(etm->queues.queue_array[i].priv, buf); +} + +static int cs_etm__process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + if (!etm->data_queued) { + struct auxtrace_buffer *buffer; + off_t data_offset; + int fd = perf_data__fd(session->data); + bool is_pipe = perf_data__is_pipe(session->data); + int err; + int idx = event->auxtrace.idx; + + if (is_pipe) + data_offset = 0; + else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) + return -errno; + } + + err = auxtrace_queues__add_event(&etm->queues, session, + event, data_offset, &buffer); + if (err) + return err; + + if (dump_trace) + if (auxtrace_buffer__get_data(buffer, fd)) { + cs_etm__dump_event(etm->queues.queue_array[idx].priv, buffer); + auxtrace_buffer__put_data(buffer); + } + } else if (dump_trace) + dump_queued_data(etm, &event->auxtrace); + + return 0; +} + +static void cs_etm__setup_timeless_decoding(struct cs_etm_auxtrace *etm) +{ + /* Take first ETM as all options will be the same for all ETMs */ + u64 *metadata = etm->metadata[0]; + + /* Override timeless mode with user input from --itrace=Z */ + if (etm->synth_opts.timeless_decoding) { + etm->timeless_decoding = true; + return; + } + + if (metadata[CS_ETM_MAGIC] == __perf_cs_etmv3_magic) + etm->timeless_decoding = !(metadata[CS_ETM_ETMCR] & ETMCR_TIMESTAMP_EN); + else + etm->timeless_decoding = !(metadata[CS_ETMV4_TRCCONFIGR] & TRCCONFIGR_TS); +} + +/* + * Read a single cpu parameter block from the auxtrace_info priv block. + * + * For version 1 there is a per cpu nr_params entry. If we are handling + * version 1 file, then there may be less, the same, or more params + * indicated by this value than the compile time number we understand. + * + * For a version 0 info block, there are a fixed number, and we need to + * fill out the nr_param value in the metadata we create. + */ +static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset, + int out_blk_size, int nr_params_v0) +{ + u64 *metadata = NULL; + int hdr_version; + int nr_in_params, nr_out_params, nr_cmn_params; + int i, k; + + metadata = zalloc(sizeof(*metadata) * out_blk_size); + if (!metadata) + return NULL; + + /* read block current index & version */ + i = *buff_in_offset; + hdr_version = buff_in[CS_HEADER_VERSION]; + + if (!hdr_version) { + /* read version 0 info block into a version 1 metadata block */ + nr_in_params = nr_params_v0; + metadata[CS_ETM_MAGIC] = buff_in[i + CS_ETM_MAGIC]; + metadata[CS_ETM_CPU] = buff_in[i + CS_ETM_CPU]; + metadata[CS_ETM_NR_TRC_PARAMS] = nr_in_params; + /* remaining block params at offset +1 from source */ + for (k = CS_ETM_COMMON_BLK_MAX_V1 - 1; k < nr_in_params; k++) + metadata[k + 1] = buff_in[i + k]; + /* version 0 has 2 common params */ + nr_cmn_params = 2; + } else { + /* read version 1 info block - input and output nr_params may differ */ + /* version 1 has 3 common params */ + nr_cmn_params = 3; + nr_in_params = buff_in[i + CS_ETM_NR_TRC_PARAMS]; + + /* if input has more params than output - skip excess */ + nr_out_params = nr_in_params + nr_cmn_params; + if (nr_out_params > out_blk_size) + nr_out_params = out_blk_size; + + for (k = CS_ETM_MAGIC; k < nr_out_params; k++) + metadata[k] = buff_in[i + k]; + + /* record the actual nr params we copied */ + metadata[CS_ETM_NR_TRC_PARAMS] = nr_out_params - nr_cmn_params; + } + + /* adjust in offset by number of in params used */ + i += nr_in_params + nr_cmn_params; + *buff_in_offset = i; + return metadata; +} + +/** + * Puts a fragment of an auxtrace buffer into the auxtrace queues based + * on the bounds of aux_event, if it matches with the buffer that's at + * file_offset. + * + * Normally, whole auxtrace buffers would be added to the queue. But we + * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder + * is reset across each buffer, so splitting the buffers up in advance has + * the same effect. + */ +static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz, + struct perf_record_aux *aux_event, struct perf_sample *sample) +{ + int err; + char buf[PERF_SAMPLE_MAX_SIZE]; + union perf_event *auxtrace_event_union; + struct perf_record_auxtrace *auxtrace_event; + union perf_event auxtrace_fragment; + __u64 aux_offset, aux_size; + enum cs_etm_format format; + + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + /* + * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got + * from looping through the auxtrace index. + */ + err = perf_session__peek_event(session, file_offset, buf, + PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL); + if (err) + return err; + auxtrace_event = &auxtrace_event_union->auxtrace; + if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE) + return -EINVAL; + + if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) || + auxtrace_event->header.size != sz) { + return -EINVAL; + } + + /* + * In per-thread mode, auxtrace CPU is set to -1, but TID will be set instead. See + * auxtrace_mmap_params__set_idx(). However, the sample AUX event will contain a + * CPU as we set this always for the AUX_OUTPUT_HW_ID event. + * So now compare only TIDs if auxtrace CPU is -1, and CPUs if auxtrace CPU is not -1. + * Return 'not found' if mismatch. + */ + if (auxtrace_event->cpu == (__u32) -1) { + etm->per_thread_decoding = true; + if (auxtrace_event->tid != sample->tid) + return 1; + } else if (auxtrace_event->cpu != sample->cpu) { + if (etm->per_thread_decoding) { + /* + * Found a per-cpu buffer after a per-thread one was + * already found + */ + pr_err("CS ETM: Inconsistent per-thread/per-cpu mode.\n"); + return -EINVAL; + } + return 1; + } + + if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) { + /* + * Clamp size in snapshot mode. The buffer size is clamped in + * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect + * the buffer size. + */ + aux_size = min(aux_event->aux_size, auxtrace_event->size); + + /* + * In this mode, the head also points to the end of the buffer so aux_offset + * needs to have the size subtracted so it points to the beginning as in normal mode + */ + aux_offset = aux_event->aux_offset - aux_size; + } else { + aux_size = aux_event->aux_size; + aux_offset = aux_event->aux_offset; + } + + if (aux_offset >= auxtrace_event->offset && + aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { + struct cs_etm_queue *etmq = cs_etm__get_queue(etm, auxtrace_event->cpu); + + if (!etmq) + return -EINVAL; + + /* + * If this AUX event was inside this buffer somewhere, create a new auxtrace event + * based on the sizes of the aux event, and queue that fragment. + */ + auxtrace_fragment.auxtrace = *auxtrace_event; + auxtrace_fragment.auxtrace.size = aux_size; + auxtrace_fragment.auxtrace.offset = aux_offset; + auxtrace_fragment.auxtrace.idx = etmq->queue_nr; + file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; + + pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu); + err = auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment, + file_offset, NULL); + if (err) + return err; + + format = (aux_event->flags & PERF_AUX_FLAG_CORESIGHT_FORMAT_RAW) ? + UNFORMATTED : FORMATTED; + if (etmq->format != UNSET && format != etmq->format) { + pr_err("CS_ETM: mixed formatted and unformatted trace not supported\n"); + return -EINVAL; + } + etmq->format = format; + return 0; + } + + /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ + return 1; +} + +static int cs_etm__process_aux_hw_id_cb(struct perf_session *session, union perf_event *event, + u64 offset __maybe_unused, void *data __maybe_unused) +{ + /* look to handle PERF_RECORD_AUX_OUTPUT_HW_ID early to ensure decoders can be set up */ + if (event->header.type == PERF_RECORD_AUX_OUTPUT_HW_ID) { + (*(int *)data)++; /* increment found count */ + return cs_etm__process_aux_output_hw_id(session, event); + } + return 0; +} + +static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event, + u64 offset __maybe_unused, void *data __maybe_unused) +{ + struct perf_sample sample; + int ret; + struct auxtrace_index_entry *ent; + struct auxtrace_index *auxtrace_index; + struct evsel *evsel; + size_t i; + + /* Don't care about any other events, we're only queuing buffers for AUX events */ + if (event->header.type != PERF_RECORD_AUX) + return 0; + + if (event->header.size < sizeof(struct perf_record_aux)) + return -EINVAL; + + /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */ + if (!event->aux.aux_size) + return 0; + + /* + * Parse the sample, we need the sample_id_all data that comes after the event so that the + * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID. + */ + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) + return -EINVAL; + perf_sample__init(&sample, /*all=*/false); + ret = evsel__parse_sample(evsel, event, &sample); + if (ret) + goto out; + + /* + * Loop through the auxtrace index to find the buffer that matches up with this aux event. + */ + list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { + for (i = 0; i < auxtrace_index->nr; i++) { + ent = &auxtrace_index->entries[i]; + ret = cs_etm__queue_aux_fragment(session, ent->file_offset, + ent->sz, &event->aux, &sample); + /* + * Stop search on error or successful values. Continue search on + * 1 ('not found') + */ + if (ret != 1) + goto out; + } + } + + /* + * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but + * don't exit with an error because it will still be possible to decode other aux records. + */ + pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu); + ret = 0; +out: + perf_sample__exit(&sample); + return ret; +} + +static int cs_etm__queue_aux_records(struct perf_session *session) +{ + struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, + struct auxtrace_index, list); + if (index && index->nr > 0) + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + cs_etm__queue_aux_records_cb, NULL); + + /* + * We would get here if there are no entries in the index (either no auxtrace + * buffers or no index at all). Fail silently as there is the possibility of + * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still + * false. + * + * In that scenario, buffers will not be split by AUX records. + */ + return 0; +} + +#define HAS_PARAM(j, type, param) (metadata[(j)][CS_ETM_NR_TRC_PARAMS] <= \ + (CS_##type##_##param - CS_ETM_COMMON_BLK_MAX_V1)) + +/* + * Loop through the ETMs and complain if we find at least one where ts_source != 1 (virtual + * timestamps). + */ +static bool cs_etm__has_virtual_ts(u64 **metadata, int num_cpu) +{ + int j; + + for (j = 0; j < num_cpu; j++) { + switch (metadata[j][CS_ETM_MAGIC]) { + case __perf_cs_etmv4_magic: + if (HAS_PARAM(j, ETMV4, TS_SOURCE) || metadata[j][CS_ETMV4_TS_SOURCE] != 1) + return false; + break; + case __perf_cs_ete_magic: + if (HAS_PARAM(j, ETE, TS_SOURCE) || metadata[j][CS_ETE_TS_SOURCE] != 1) + return false; + break; + default: + /* Unknown / unsupported magic number. */ + return false; + } + } + return true; +} + +/* map trace ids to correct metadata block, from information in metadata */ +static int cs_etm__map_trace_ids_metadata(struct cs_etm_auxtrace *etm, int num_cpu, + u64 **metadata) +{ + u64 cs_etm_magic; + u8 trace_chan_id; + int i, err; + + for (i = 0; i < num_cpu; i++) { + cs_etm_magic = metadata[i][CS_ETM_MAGIC]; + switch (cs_etm_magic) { + case __perf_cs_etmv3_magic: + metadata[i][CS_ETM_ETMTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK; + trace_chan_id = (u8)(metadata[i][CS_ETM_ETMTRACEIDR]); + break; + case __perf_cs_etmv4_magic: + case __perf_cs_ete_magic: + metadata[i][CS_ETMV4_TRCTRACEIDR] &= CORESIGHT_TRACE_ID_VAL_MASK; + trace_chan_id = (u8)(metadata[i][CS_ETMV4_TRCTRACEIDR]); + break; + default: + /* unknown magic number */ + return -EINVAL; + } + err = cs_etm__map_trace_id_v0(etm, trace_chan_id, metadata[i]); + if (err) + return err; + } + return 0; +} + +/* + * Use the data gathered by the peeks for HW_ID (trace ID mappings) and AUX + * (formatted or not) packets to create the decoders. + */ +static int cs_etm__create_queue_decoders(struct cs_etm_queue *etmq) +{ + struct cs_etm_decoder_params d_params; + struct cs_etm_trace_params *t_params; + int decoders = intlist__nr_entries(etmq->traceid_list); + + if (decoders == 0) + return 0; + + /* + * Each queue can only contain data from one CPU when unformatted, so only one decoder is + * needed. + */ + if (etmq->format == UNFORMATTED) + assert(decoders == 1); + + /* Use metadata to fill in trace parameters for trace decoder */ + t_params = zalloc(sizeof(*t_params) * decoders); + + if (!t_params) + goto out_free; + + if (cs_etm__init_trace_params(t_params, etmq)) + goto out_free; + + /* Set decoder parameters to decode trace packets */ + if (cs_etm__init_decoder_params(&d_params, etmq, + dump_trace ? CS_ETM_OPERATION_PRINT : + CS_ETM_OPERATION_DECODE)) + goto out_free; + + etmq->decoder = cs_etm_decoder__new(decoders, &d_params, + t_params); + + if (!etmq->decoder) + goto out_free; + + /* + * Register a function to handle all memory accesses required by + * the trace decoder library. + */ + if (cs_etm_decoder__add_mem_access_cb(etmq->decoder, + 0x0L, ((u64) -1L), + cs_etm__decoder_mem_access)) + goto out_free_decoder; + + zfree(&t_params); + return 0; + +out_free_decoder: + cs_etm_decoder__free(etmq->decoder); +out_free: + zfree(&t_params); + return -EINVAL; +} + +static int cs_etm__create_decoders(struct cs_etm_auxtrace *etm) +{ + struct auxtrace_queues *queues = &etm->queues; + + for (unsigned int i = 0; i < queues->nr_queues; i++) { + bool empty = list_empty(&queues->queue_array[i].head); + struct cs_etm_queue *etmq = queues->queue_array[i].priv; + int ret; + + /* + * Don't create decoders for empty queues, mainly because + * etmq->format is unknown for empty queues. + */ + assert(empty || etmq->format != UNSET); + if (empty) + continue; + + ret = cs_etm__create_queue_decoders(etmq); + if (ret) + return ret; + } + return 0; +} + +int cs_etm__process_auxtrace_info_full(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + struct cs_etm_auxtrace *etm = NULL; + struct perf_record_time_conv *tc = &session->time_conv; + int event_header_size = sizeof(struct perf_event_header); + int total_size = auxtrace_info->header.size; + int priv_size = 0; + int num_cpu, max_cpu = 0; + int err = 0; + int aux_hw_id_found; + int i; + u64 *ptr = NULL; + u64 **metadata = NULL; + + /* First the global part */ + ptr = (u64 *) auxtrace_info->priv; + num_cpu = ptr[CS_PMU_TYPE_CPUS] & 0xffffffff; + + /* + * Bound num_cpu by the event size: the global header consumes + * CS_ETM_HEADER_SIZE bytes, and each CPU needs at least one u64 + * metadata entry after that. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE - + CS_ETM_HEADER_SIZE; + if (num_cpu <= 0 || priv_size <= 0 || + num_cpu > priv_size / (int)sizeof(u64)) + return -EINVAL; + + metadata = zalloc(sizeof(*metadata) * num_cpu); + if (!metadata) + return -ENOMEM; + + /* Start parsing after the common part of the header */ + i = CS_HEADER_VERSION_MAX; + + /* + * The metadata is stored in the auxtrace_info section and encodes + * the configuration of the ARM embedded trace macrocell which is + * required by the trace decoder to properly decode the trace due + * to its highly compressed nature. + */ + for (int j = 0; j < num_cpu; j++) { + if (ptr[i] == __perf_cs_etmv3_magic) { + metadata[j] = + cs_etm__create_meta_blk(ptr, &i, + CS_ETM_PRIV_MAX, + CS_ETM_NR_TRC_PARAMS_V0); + } else if (ptr[i] == __perf_cs_etmv4_magic) { + metadata[j] = + cs_etm__create_meta_blk(ptr, &i, + CS_ETMV4_PRIV_MAX, + CS_ETMV4_NR_TRC_PARAMS_V0); + } else if (ptr[i] == __perf_cs_ete_magic) { + metadata[j] = cs_etm__create_meta_blk(ptr, &i, CS_ETE_PRIV_MAX, -1); + } else { + ui__error("CS ETM Trace: Unrecognised magic number %#"PRIx64". File could be from a newer version of perf.\n", + ptr[i]); + err = -EINVAL; + goto err_free_metadata; + } + + if (!metadata[j]) { + err = -ENOMEM; + goto err_free_metadata; + } + + /* CPU id comes from perf.data and must fit max_cpu + 1 without overflow */ + if (metadata[j][CS_ETM_CPU] >= INT_MAX) { + err = -EINVAL; + goto err_free_metadata; + } + + if ((int)metadata[j][CS_ETM_CPU] > max_cpu) + max_cpu = metadata[j][CS_ETM_CPU]; + } + + /* + * Each of CS_HEADER_VERSION_MAX, CS_ETM_PRIV_MAX and + * CS_ETMV4_PRIV_MAX mark how many double words are in the + * global metadata, and each cpu's metadata respectively. + * The following tests if the correct number of double words was + * present in the auxtrace info section. + */ + priv_size = total_size - event_header_size - INFO_HEADER_SIZE; + if (i * 8 != priv_size) { + err = -EINVAL; + goto err_free_metadata; + } + + etm = zalloc(sizeof(*etm)); + + if (!etm) { + err = -ENOMEM; + goto err_free_metadata; + } + + /* + * As all the ETMs run at the same exception level, the system should + * have the same PID format crossing CPUs. So cache the PID format + * and reuse it for sequential decoding. + */ + etm->pid_fmt = cs_etm__init_pid_fmt(metadata[0]); + + err = auxtrace_queues__init_nr(&etm->queues, max_cpu + 1); + if (err) + goto err_free_etm; + + for (unsigned int j = 0; j < etm->queues.nr_queues; ++j) { + err = cs_etm__setup_queue(etm, &etm->queues.queue_array[j], j); + if (err) + goto err_free_queues; + } + + if (session->itrace_synth_opts->set) { + etm->synth_opts = *session->itrace_synth_opts; + } else { + itrace_synth_opts__set_default(&etm->synth_opts, + session->itrace_synth_opts->default_no_sample); + etm->synth_opts.callchain = false; + } + + etm->session = session; + + etm->num_cpu = num_cpu; + etm->pmu_type = (unsigned int) ((ptr[CS_PMU_TYPE_CPUS] >> 32) & 0xffffffff); + etm->snapshot_mode = (ptr[CS_ETM_SNAPSHOT] != 0); + etm->metadata = metadata; + etm->auxtrace_type = auxtrace_info->type; + + if (etm->synth_opts.use_timestamp) + /* + * Prior to Armv8.4, Arm CPUs don't support FEAT_TRF feature, + * therefore the decoder cannot know if the timestamp trace is + * same with the kernel time. + * + * If a user has knowledge for the working platform and can + * specify itrace option 'T' to tell decoder to forcely use the + * traced timestamp as the kernel time. + */ + etm->has_virtual_ts = true; + else + /* Use virtual timestamps if all ETMs report ts_source = 1 */ + etm->has_virtual_ts = cs_etm__has_virtual_ts(metadata, num_cpu); + + if (!etm->has_virtual_ts) + ui__warning("Virtual timestamps are not enabled, or not supported by the traced system.\n" + "The time field of the samples will not be set accurately.\n" + "For Arm CPUs prior to Armv8.4 or without support FEAT_TRF,\n" + "you can specify the itrace option 'T' for timestamp decoding\n" + "if the Coresight timestamp on the platform is same with the kernel time.\n\n"); + + etm->auxtrace.process_event = cs_etm__process_event; + etm->auxtrace.process_auxtrace_event = cs_etm__process_auxtrace_event; + etm->auxtrace.flush_events = cs_etm__flush_events; + etm->auxtrace.free_events = cs_etm__free_events; + etm->auxtrace.free = cs_etm__free; + etm->auxtrace.evsel_is_auxtrace = cs_etm__evsel_is_auxtrace; + session->auxtrace = &etm->auxtrace; + + cs_etm__setup_timeless_decoding(etm); + + etm->tc.time_shift = tc->time_shift; + etm->tc.time_mult = tc->time_mult; + etm->tc.time_zero = tc->time_zero; + if (event_contains(*tc, cap_user_time_short)) { + etm->tc.time_cycles = tc->time_cycles; + etm->tc.time_mask = tc->time_mask; + etm->tc.cap_user_time_zero = tc->cap_user_time_zero; + etm->tc.cap_user_time_short = tc->cap_user_time_short; + } + err = cs_etm__synth_events(etm, session); + if (err) + goto err_free_queues; + + err = cs_etm__queue_aux_records(session); + if (err) + goto err_free_queues; + + /* + * Map Trace ID values to CPU metadata. + * + * Trace metadata will always contain Trace ID values from the legacy algorithm + * in case it's read by a version of Perf that doesn't know about HW_ID packets + * or the kernel doesn't emit them. + * + * The updated kernel drivers that use AUX_HW_ID to sent Trace IDs will attempt to use + * the same IDs as the old algorithm as far as is possible, unless there are clashes + * in which case a different value will be used. This means an older perf may still + * be able to record and read files generate on a newer system. + * + * For a perf able to interpret AUX_HW_ID packets we first check for the presence of + * those packets. If they are there then the values will be mapped and plugged into + * the metadata and decoders are only created for each mapping received. + * + * If no AUX_HW_ID packets are present - which means a file recorded on an old kernel + * then we map Trace ID values to CPU directly from the metadata and create decoders + * for all mappings. + */ + + /* Scan for AUX_OUTPUT_HW_ID records to map trace ID values to CPU metadata */ + aux_hw_id_found = 0; + err = perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + cs_etm__process_aux_hw_id_cb, &aux_hw_id_found); + if (err) + goto err_free_queues; + + /* if no HW ID found this is a file with metadata values only, map from metadata */ + if (!aux_hw_id_found) { + err = cs_etm__map_trace_ids_metadata(etm, num_cpu, metadata); + if (err) + goto err_free_queues; + } + + err = cs_etm__create_decoders(etm); + if (err) + goto err_free_queues; + + etm->data_queued = etm->queues.populated; + return 0; + +err_free_queues: + auxtrace_queues__free(&etm->queues); + session->auxtrace = NULL; +err_free_etm: + zfree(&etm); +err_free_metadata: + /* No need to check @metadata[j], free(NULL) is supported */ + for (int j = 0; j < num_cpu; j++) + zfree(&metadata[j]); + zfree(&metadata); + return err; +} diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h new file mode 100644 index 000000000..b81099c2b --- /dev/null +++ b/tools/perf/util/cs-etm.h @@ -0,0 +1,286 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright(C) 2015 Linaro Limited. All rights reserved. + * Author: Mathieu Poirier + */ + +#ifndef INCLUDE__UTIL_PERF_CS_ETM_H__ +#define INCLUDE__UTIL_PERF_CS_ETM_H__ + +#include "debug.h" +#include "util/event.h" +#include + +struct perf_session; +struct perf_pmu; + +/* + * Versioning header in case things need to change in the future. That way + * decoding of old snapshot is still possible. + */ +enum { + /* Starting with 0x0 */ + CS_HEADER_VERSION, + /* PMU->type (32 bit), total # of CPUs (32 bit) */ + CS_PMU_TYPE_CPUS, + CS_ETM_SNAPSHOT, + CS_HEADER_VERSION_MAX, +}; + +/* + * Update the version for new format. + * + * Version 1: format adds a param count to the per cpu metadata. + * This allows easy adding of new metadata parameters. + * Requires that new params always added after current ones. + * Also allows client reader to handle file versions that are different by + * checking the number of params in the file vs the number expected. + * + * Version 2: Drivers will use PERF_RECORD_AUX_OUTPUT_HW_ID to output + * CoreSight Trace ID. ...TRACEIDR metadata will be set to legacy values + * but with addition flags. + */ +#define CS_HEADER_CURRENT_VERSION 2 + +/* Beginning of header common to both ETMv3 and V4 */ +enum { + CS_ETM_MAGIC, + CS_ETM_CPU, + /* Number of trace config params in following ETM specific block */ + CS_ETM_NR_TRC_PARAMS, + CS_ETM_COMMON_BLK_MAX_V1, +}; + +/* ETMv3/PTM metadata */ +enum { + /* Dynamic, configurable parameters */ + CS_ETM_ETMCR = CS_ETM_COMMON_BLK_MAX_V1, + CS_ETM_ETMTRACEIDR, + /* RO, taken from sysFS */ + CS_ETM_ETMCCER, + CS_ETM_ETMIDR, + CS_ETM_PRIV_MAX, +}; + +/* define fixed version 0 length - allow new format reader to read old files. */ +#define CS_ETM_NR_TRC_PARAMS_V0 (CS_ETM_ETMIDR - CS_ETM_ETMCR + 1) + +/* ETMv4 metadata */ +enum { + /* Dynamic, configurable parameters */ + CS_ETMV4_TRCCONFIGR = CS_ETM_COMMON_BLK_MAX_V1, + CS_ETMV4_TRCTRACEIDR, + /* RO, taken from sysFS */ + CS_ETMV4_TRCIDR0, + CS_ETMV4_TRCIDR1, + CS_ETMV4_TRCIDR2, + CS_ETMV4_TRCIDR8, + CS_ETMV4_TRCAUTHSTATUS, + CS_ETMV4_TS_SOURCE, + CS_ETMV4_PRIV_MAX, +}; + +/* define fixed version 0 length - allow new format reader to read old files. */ +#define CS_ETMV4_NR_TRC_PARAMS_V0 (CS_ETMV4_TRCAUTHSTATUS - CS_ETMV4_TRCCONFIGR + 1) + +/* + * ETE metadata is ETMv4 plus TRCDEVARCH register and doesn't support header V0 since it was + * added in header V1 + */ +enum { + /* Dynamic, configurable parameters */ + CS_ETE_TRCCONFIGR = CS_ETM_COMMON_BLK_MAX_V1, + CS_ETE_TRCTRACEIDR, + /* RO, taken from sysFS */ + CS_ETE_TRCIDR0, + CS_ETE_TRCIDR1, + CS_ETE_TRCIDR2, + CS_ETE_TRCIDR8, + CS_ETE_TRCAUTHSTATUS, + CS_ETE_TRCDEVARCH, + CS_ETE_TS_SOURCE, + CS_ETE_PRIV_MAX +}; + +/* + * Check for valid CoreSight trace ID. If an invalid value is present in the metadata, + * then IDs are present in the hardware ID packet in the data file. + */ +#define CS_IS_VALID_TRACE_ID(id) ((id > 0) && (id < 0x70)) + +/* + * ETMv3 exception encoding number: + * See Embedded Trace Macrocell specification (ARM IHI 0014Q) + * table 7-12 Encoding of Exception[3:0] for non-ARMv7-M processors. + */ +enum { + CS_ETMV3_EXC_NONE = 0, + CS_ETMV3_EXC_DEBUG_HALT = 1, + CS_ETMV3_EXC_SMC = 2, + CS_ETMV3_EXC_HYP = 3, + CS_ETMV3_EXC_ASYNC_DATA_ABORT = 4, + CS_ETMV3_EXC_JAZELLE_THUMBEE = 5, + CS_ETMV3_EXC_PE_RESET = 8, + CS_ETMV3_EXC_UNDEFINED_INSTR = 9, + CS_ETMV3_EXC_SVC = 10, + CS_ETMV3_EXC_PREFETCH_ABORT = 11, + CS_ETMV3_EXC_DATA_FAULT = 12, + CS_ETMV3_EXC_GENERIC = 13, + CS_ETMV3_EXC_IRQ = 14, + CS_ETMV3_EXC_FIQ = 15, +}; + +/* + * ETMv4 exception encoding number: + * See ARM Embedded Trace Macrocell Architecture Specification (ARM IHI 0064D) + * table 6-12 Possible values for the TYPE field in an Exception instruction + * trace packet, for ARMv7-A/R and ARMv8-A/R PEs. + */ +enum { + CS_ETMV4_EXC_RESET = 0, + CS_ETMV4_EXC_DEBUG_HALT = 1, + CS_ETMV4_EXC_CALL = 2, + CS_ETMV4_EXC_TRAP = 3, + CS_ETMV4_EXC_SYSTEM_ERROR = 4, + CS_ETMV4_EXC_INST_DEBUG = 6, + CS_ETMV4_EXC_DATA_DEBUG = 7, + CS_ETMV4_EXC_ALIGNMENT = 10, + CS_ETMV4_EXC_INST_FAULT = 11, + CS_ETMV4_EXC_DATA_FAULT = 12, + CS_ETMV4_EXC_IRQ = 14, + CS_ETMV4_EXC_FIQ = 15, + CS_ETMV4_EXC_END = 31, +}; + +enum cs_etm_sample_type { + CS_ETM_EMPTY, + CS_ETM_RANGE, + CS_ETM_DISCONTINUITY, + CS_ETM_EXCEPTION, + CS_ETM_EXCEPTION_RET, + CS_ETM_CONTEXT, +}; + +enum cs_etm_isa { + CS_ETM_ISA_UNKNOWN, + CS_ETM_ISA_A64, + CS_ETM_ISA_A32, + CS_ETM_ISA_T32, +}; + +struct cs_etm_queue; + +struct cs_etm_packet { + enum cs_etm_sample_type sample_type; + enum cs_etm_isa isa; + u64 start_addr; + u64 end_addr; + u32 instr_count; + u32 last_instr_type; + u32 last_instr_subtype; + u32 flags; + u32 exception_number; + bool last_instr_cond; + bool last_instr_taken_branch; + u8 last_instr_size; + u8 trace_chan_id; + int cpu; + int el; + pid_t tid; +}; + +#define CS_ETM_PACKET_MAX_BUFFER 1024 + +/* + * When working with per-thread scenarios the process under trace can + * be scheduled on any CPU and as such, more than one traceID may be + * associated with the same process. Since a traceID of '0' is illegal + * as per the CoreSight architecture, use that specific value to + * identify the queue where all packets (with any traceID) are + * aggregated. + */ +#define CS_ETM_PER_THREAD_TRACEID 0 + +struct cs_etm_packet_queue { + u32 packet_count; + u32 head; + u32 tail; + u32 instr_count; + u64 cs_timestamp; /* Timestamp from trace data, converted to ns if possible */ + u64 next_cs_timestamp; + struct cs_etm_packet packet_buffer[CS_ETM_PACKET_MAX_BUFFER]; +}; + +#define KiB(x) ((x) * 1024) +#define MiB(x) ((x) * 1024 * 1024) + +#define CS_ETM_INVAL_ADDR 0xdeadbeefdeadbeefUL + +#define BMVAL(val, lsb, msb) ((val & GENMASK(msb, lsb)) >> lsb) + +#define CS_ETM_HEADER_SIZE (CS_HEADER_VERSION_MAX * sizeof(u64)) + +#define __perf_cs_etmv3_magic 0x3030303030303030ULL +#define __perf_cs_etmv4_magic 0x4040404040404040ULL +#define __perf_cs_ete_magic 0x5050505050505050ULL +#define CS_ETMV3_PRIV_SIZE (CS_ETM_PRIV_MAX * sizeof(u64)) +#define CS_ETMV4_PRIV_SIZE (CS_ETMV4_PRIV_MAX * sizeof(u64)) +#define CS_ETE_PRIV_SIZE (CS_ETE_PRIV_MAX * sizeof(u64)) + +#define INFO_HEADER_SIZE (sizeof(((struct perf_record_auxtrace_info *)0)->type) + \ + sizeof(((struct perf_record_auxtrace_info *)0)->reserved__)) + +/* CoreSight trace ID is currently the bottom 7 bits of the value */ +#define CORESIGHT_TRACE_ID_VAL_MASK GENMASK(6, 0) + +/* ETMv4 CONFIGR register bits */ +#define TRCCONFIGR_BB BIT(3) +#define TRCCONFIGR_CCI BIT(4) +#define TRCCONFIGR_CID BIT(6) +#define TRCCONFIGR_VMID BIT(7) +#define TRCCONFIGR_TS BIT(11) +#define TRCCONFIGR_RS BIT(12) +#define TRCCONFIGR_VMIDOPT BIT(15) + +/* ETMv3 ETMCR register bits */ +#define ETMCR_CYC_ACC BIT(12) +#define ETMCR_CTXTID BIT(14) +#define ETMCR_TIMESTAMP_EN BIT(28) +#define ETMCR_RETURN_STACK BIT(29) + +int cs_etm__process_auxtrace_info(union perf_event *event, + struct perf_session *session); +void cs_etm_get_default_config(const struct perf_pmu *pmu, struct perf_event_attr *attr); + +enum cs_etm_pid_fmt { + CS_ETM_PIDFMT_NONE, + CS_ETM_PIDFMT_CTXTID, + CS_ETM_PIDFMT_CTXTID2 +}; + +#ifdef HAVE_CSTRACE_SUPPORT +#include +int cs_etm__get_cpu(struct cs_etm_queue *etmq, u8 trace_chan_id, int *cpu); +enum cs_etm_pid_fmt cs_etm__get_pid_fmt(struct cs_etm_queue *etmq); +int cs_etm__etmq_update_decode_context(struct cs_etm_queue *etmq, + u8 trace_chan_id, ocsd_ex_level el, + pid_t tid); +bool cs_etm__etmq_is_timeless(struct cs_etm_queue *etmq); +void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, + u8 trace_chan_id); +struct cs_etm_packet_queue +*cs_etm__etmq_get_packet_queue(struct cs_etm_queue *etmq, u8 trace_chan_id); +int cs_etm__process_auxtrace_info_full(union perf_event *event __maybe_unused, + struct perf_session *session __maybe_unused); +u64 cs_etm__convert_sample_time(struct cs_etm_queue *etmq, u64 cs_timestamp); +#else +static inline int +cs_etm__process_auxtrace_info_full(union perf_event *event __maybe_unused, + struct perf_session *session __maybe_unused) +{ + pr_err("\nCS ETM Trace: OpenCSD is not linked in, please recompile with CORESIGHT=1\n"); + return -1; +} +#endif + +#endif diff --git a/tools/perf/util/data-convert-bt.c b/tools/perf/util/data-convert-bt.c new file mode 100644 index 000000000..5ff46bfcd --- /dev/null +++ b/tools/perf/util/data-convert-bt.c @@ -0,0 +1,1832 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * CTF writing support via babeltrace. + * + * Copyright (C) 2014, Jiri Olsa + * Copyright (C) 2014, Sebastian Andrzej Siewior + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "asm/bug.h" +#include "data-convert.h" +#include "session.h" +#include "debug.h" +#include "tool.h" +#include "evlist.h" +#include "evsel.h" +#include "machine.h" +#include "config.h" +#include +#include +#include +#include "util.h" +#include "clockid.h" +#include "util/sample.h" +#include "util/time-utils.h" +#include "header.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +#define pr_N(n, fmt, ...) \ + eprintf(n, debug_data_convert, fmt, ##__VA_ARGS__) + +#define pr(fmt, ...) pr_N(1, pr_fmt(fmt), ##__VA_ARGS__) +#define pr2(fmt, ...) pr_N(2, pr_fmt(fmt), ##__VA_ARGS__) + +#define pr_time2(t, fmt, ...) pr_time_N(2, debug_data_convert, t, pr_fmt(fmt), ##__VA_ARGS__) + +struct evsel_priv { + struct bt_ctf_event_class *event_class; +}; + +#define MAX_CPUS 4096 + +struct ctf_stream { + struct bt_ctf_stream *stream; + int cpu; + u32 count; +}; + +struct ctf_writer { + /* writer primitives */ + struct bt_ctf_writer *writer; + struct ctf_stream **stream; + int stream_cnt; + struct bt_ctf_stream_class *stream_class; + struct bt_ctf_clock *clock; + + /* data types */ + union { + struct { + struct bt_ctf_field_type *s64; + struct bt_ctf_field_type *u64; + struct bt_ctf_field_type *s32; + struct bt_ctf_field_type *u32; + struct bt_ctf_field_type *string; + struct bt_ctf_field_type *u32_hex; + struct bt_ctf_field_type *u64_hex; + }; + struct bt_ctf_field_type *array[6]; + } data; + struct bt_ctf_event_class *comm_class; + struct bt_ctf_event_class *exit_class; + struct bt_ctf_event_class *fork_class; + struct bt_ctf_event_class *mmap_class; + struct bt_ctf_event_class *mmap2_class; +}; + +struct convert { + struct perf_tool tool; + struct ctf_writer writer; + + struct perf_time_interval *ptime_range; + int range_size; + int range_num; + + u64 events_size; + u64 events_count; + u64 non_sample_count; + u64 skipped; + + /* Ordered events configured queue size. */ + u64 queue_size; +}; + +static int value_set(struct bt_ctf_field_type *type, + struct bt_ctf_event *event, + const char *name, u64 val) +{ + struct bt_ctf_field *field; + bool sign = bt_ctf_field_type_integer_get_signed(type); + int ret; + + field = bt_ctf_field_create(type); + if (!field) { + pr_err("failed to create a field %s\n", name); + return -1; + } + + if (sign) { + ret = bt_ctf_field_integer_signed_set_value(field, val); + if (ret) { + pr_err("failed to set field value %s\n", name); + goto err; + } + } else { + ret = bt_ctf_field_integer_unsigned_set_value(field, val); + if (ret) { + pr_err("failed to set field value %s\n", name); + goto err; + } + } + + ret = bt_ctf_event_set_payload(event, name, field); + if (ret) { + pr_err("failed to set payload %s\n", name); + goto err; + } + + pr2(" SET [%s = %" PRIu64 "]\n", name, val); + +err: + bt_ctf_field_put(field); + return ret; +} + +#define __FUNC_VALUE_SET(_name, _val_type) \ +static __maybe_unused int value_set_##_name(struct ctf_writer *cw, \ + struct bt_ctf_event *event, \ + const char *name, \ + _val_type val) \ +{ \ + struct bt_ctf_field_type *type = cw->data._name; \ + return value_set(type, event, name, (u64) val); \ +} + +#define FUNC_VALUE_SET(_name) __FUNC_VALUE_SET(_name, _name) + +FUNC_VALUE_SET(s32) +FUNC_VALUE_SET(u32) +FUNC_VALUE_SET(s64) +FUNC_VALUE_SET(u64) +__FUNC_VALUE_SET(u64_hex, u64) + +static int string_set_value(struct bt_ctf_field *field, const char *string); +static __maybe_unused int +value_set_string(struct ctf_writer *cw, struct bt_ctf_event *event, + const char *name, const char *string) +{ + struct bt_ctf_field_type *type = cw->data.string; + struct bt_ctf_field *field; + int ret = 0; + + field = bt_ctf_field_create(type); + if (!field) { + pr_err("failed to create a field %s\n", name); + return -1; + } + + ret = string_set_value(field, string); + if (ret) { + pr_err("failed to set value %s\n", name); + goto err_put_field; + } + + ret = bt_ctf_event_set_payload(event, name, field); + if (ret) + pr_err("failed to set payload %s\n", name); + +err_put_field: + bt_ctf_field_put(field); + return ret; +} + +static struct bt_ctf_field_type* +get_tracepoint_field_type(struct ctf_writer *cw, struct tep_format_field *field) +{ + unsigned long flags = field->flags; + + if (flags & TEP_FIELD_IS_STRING) + return cw->data.string; + + if (!(flags & TEP_FIELD_IS_SIGNED)) { + /* unsigned long are mostly pointers */ + if (flags & TEP_FIELD_IS_LONG || flags & TEP_FIELD_IS_POINTER) + return cw->data.u64_hex; + } + + if (flags & TEP_FIELD_IS_SIGNED) { + if (field->size == 8) + return cw->data.s64; + else + return cw->data.s32; + } + + if (field->size == 8) + return cw->data.u64; + else + return cw->data.u32; +} + +static unsigned long long adjust_signedness(unsigned long long value_int, int size) +{ + unsigned long long value_mask; + + /* + * value_mask = (1 << (size * 8 - 1)) - 1. + * Directly set value_mask for code readers. + */ + switch (size) { + case 1: + value_mask = 0x7fULL; + break; + case 2: + value_mask = 0x7fffULL; + break; + case 4: + value_mask = 0x7fffffffULL; + break; + case 8: + /* + * For 64 bit value, return it self. There is no need + * to fill high bit. + */ + /* Fall through */ + default: + /* BUG! */ + return value_int; + } + + /* If it is a positive value, don't adjust. */ + if ((value_int & (~0ULL - value_mask)) == 0) + return value_int; + + /* Fill upper part of value_int with 1 to make it a negative long long. */ + return (value_int & value_mask) | ~value_mask; +} + +static int string_set_value(struct bt_ctf_field *field, const char *string) +{ + char *buffer = NULL; + size_t len = strlen(string), i, p; + int err; + + for (i = p = 0; i < len; i++, p++) { + if (isprint(string[i])) { + if (!buffer) + continue; + buffer[p] = string[i]; + } else { + char numstr[5]; + + snprintf(numstr, sizeof(numstr), "\\x%02x", + (unsigned int)(string[i]) & 0xff); + + if (!buffer) { + buffer = zalloc(i + (len - i) * 4 + 2); + if (!buffer) { + pr_err("failed to set unprintable string '%s'\n", string); + return bt_ctf_field_string_set_value(field, "UNPRINTABLE-STRING"); + } + if (i > 0) + strncpy(buffer, string, i); + } + memcpy(buffer + p, numstr, 4); + p += 3; + } + } + + if (!buffer) + return bt_ctf_field_string_set_value(field, string); + err = bt_ctf_field_string_set_value(field, buffer); + free(buffer); + return err; +} + +static int add_tracepoint_field_value(struct ctf_writer *cw, + struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct perf_sample *sample, + struct tep_format_field *fmtf) +{ + struct bt_ctf_field_type *type; + struct bt_ctf_field *array_field; + struct bt_ctf_field *field; + const char *name = fmtf->name; + void *data = sample->raw_data; + unsigned long flags = fmtf->flags; + unsigned int n_items; + unsigned int i; + unsigned int offset; + unsigned int len; + int ret; + + name = fmtf->alias; + offset = fmtf->offset; + len = fmtf->size; + if (flags & TEP_FIELD_IS_STRING) + flags &= ~TEP_FIELD_IS_ARRAY; + + if (flags & TEP_FIELD_IS_DYNAMIC) { + unsigned long long tmp_val; + + tmp_val = tep_read_number(fmtf->event->tep, + data + offset, len); + offset = tmp_val; + len = offset >> 16; + offset &= 0xffff; + if (tep_field_is_relative(flags)) + offset += fmtf->offset + fmtf->size; + } + + if (flags & TEP_FIELD_IS_ARRAY) { + + type = bt_ctf_event_class_get_field_by_name( + event_class, name); + array_field = bt_ctf_field_create(type); + bt_ctf_field_type_put(type); + if (!array_field) { + pr_err("Failed to create array type %s\n", name); + return -1; + } + + len = fmtf->size / fmtf->arraylen; + n_items = fmtf->arraylen; + } else { + n_items = 1; + array_field = NULL; + } + + type = get_tracepoint_field_type(cw, fmtf); + + for (i = 0; i < n_items; i++) { + if (flags & TEP_FIELD_IS_ARRAY) + field = bt_ctf_field_array_get_field(array_field, i); + else + field = bt_ctf_field_create(type); + + if (!field) { + pr_err("failed to create a field %s\n", name); + return -1; + } + + if (flags & TEP_FIELD_IS_STRING) + ret = string_set_value(field, data + offset + i * len); + else { + unsigned long long value_int; + + value_int = tep_read_number( + fmtf->event->tep, + data + offset + i * len, len); + + if (!(flags & TEP_FIELD_IS_SIGNED)) + ret = bt_ctf_field_integer_unsigned_set_value( + field, value_int); + else + ret = bt_ctf_field_integer_signed_set_value( + field, adjust_signedness(value_int, len)); + } + + if (ret) { + pr_err("failed to set file value %s\n", name); + goto err_put_field; + } + if (!(flags & TEP_FIELD_IS_ARRAY)) { + ret = bt_ctf_event_set_payload(event, name, field); + if (ret) { + pr_err("failed to set payload %s\n", name); + goto err_put_field; + } + } + bt_ctf_field_put(field); + } + if (flags & TEP_FIELD_IS_ARRAY) { + ret = bt_ctf_event_set_payload(event, name, array_field); + if (ret) { + pr_err("Failed add payload array %s\n", name); + return -1; + } + bt_ctf_field_put(array_field); + } + return 0; + +err_put_field: + bt_ctf_field_put(field); + return -1; +} + +static int add_tracepoint_fields_values(struct ctf_writer *cw, + struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct tep_format_field *fields, + struct perf_sample *sample) +{ + struct tep_format_field *field; + int ret; + + for (field = fields; field; field = field->next) { + ret = add_tracepoint_field_value(cw, event_class, event, sample, + field); + if (ret) + return -1; + } + return 0; +} + +static int add_tracepoint_values(struct ctf_writer *cw, + struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct evsel *evsel, + struct perf_sample *sample) +{ + const struct tep_event *tp_format = evsel__tp_format(evsel); + struct tep_format_field *common_fields = tp_format->format.common_fields; + struct tep_format_field *fields = tp_format->format.fields; + int ret; + + ret = add_tracepoint_fields_values(cw, event_class, event, + common_fields, sample); + if (!ret) + ret = add_tracepoint_fields_values(cw, event_class, event, + fields, sample); + + return ret; +} + +static int +add_bpf_output_values(struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct perf_sample *sample) +{ + struct bt_ctf_field_type *len_type, *seq_type; + struct bt_ctf_field *len_field, *seq_field; + unsigned int raw_size = sample->raw_size; + unsigned int nr_elements = raw_size / sizeof(u32); + unsigned int i; + int ret; + + if (nr_elements * sizeof(u32) != raw_size) + pr_warning("Incorrect raw_size (%u) in bpf output event, skip %zu bytes\n", + raw_size, nr_elements * sizeof(u32) - raw_size); + + len_type = bt_ctf_event_class_get_field_by_name(event_class, "raw_len"); + len_field = bt_ctf_field_create(len_type); + if (!len_field) { + pr_err("failed to create 'raw_len' for bpf output event\n"); + ret = -1; + goto put_len_type; + } + + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); + if (ret) { + pr_err("failed to set field value for raw_len\n"); + goto put_len_field; + } + ret = bt_ctf_event_set_payload(event, "raw_len", len_field); + if (ret) { + pr_err("failed to set payload to raw_len\n"); + goto put_len_field; + } + + seq_type = bt_ctf_event_class_get_field_by_name(event_class, "raw_data"); + seq_field = bt_ctf_field_create(seq_type); + if (!seq_field) { + pr_err("failed to create 'raw_data' for bpf output event\n"); + ret = -1; + goto put_seq_type; + } + + ret = bt_ctf_field_sequence_set_length(seq_field, len_field); + if (ret) { + pr_err("failed to set length of 'raw_data'\n"); + goto put_seq_field; + } + + for (i = 0; i < nr_elements; i++) { + struct bt_ctf_field *elem_field = + bt_ctf_field_sequence_get_field(seq_field, i); + + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, + ((u32 *)(sample->raw_data))[i]); + + bt_ctf_field_put(elem_field); + if (ret) { + pr_err("failed to set raw_data[%d]\n", i); + goto put_seq_field; + } + } + + ret = bt_ctf_event_set_payload(event, "raw_data", seq_field); + if (ret) + pr_err("failed to set payload for raw_data\n"); + +put_seq_field: + bt_ctf_field_put(seq_field); +put_seq_type: + bt_ctf_field_type_put(seq_type); +put_len_field: + bt_ctf_field_put(len_field); +put_len_type: + bt_ctf_field_type_put(len_type); + return ret; +} + +static int +add_callchain_output_values(struct bt_ctf_event_class *event_class, + struct bt_ctf_event *event, + struct ip_callchain *callchain) +{ + struct bt_ctf_field_type *len_type, *seq_type; + struct bt_ctf_field *len_field, *seq_field; + unsigned int nr_elements = callchain->nr; + unsigned int i; + int ret; + + len_type = bt_ctf_event_class_get_field_by_name( + event_class, "perf_callchain_size"); + len_field = bt_ctf_field_create(len_type); + if (!len_field) { + pr_err("failed to create 'perf_callchain_size' for callchain output event\n"); + ret = -1; + goto put_len_type; + } + + ret = bt_ctf_field_integer_unsigned_set_value(len_field, nr_elements); + if (ret) { + pr_err("failed to set field value for perf_callchain_size\n"); + goto put_len_field; + } + ret = bt_ctf_event_set_payload(event, "perf_callchain_size", len_field); + if (ret) { + pr_err("failed to set payload to perf_callchain_size\n"); + goto put_len_field; + } + + seq_type = bt_ctf_event_class_get_field_by_name( + event_class, "perf_callchain"); + seq_field = bt_ctf_field_create(seq_type); + if (!seq_field) { + pr_err("failed to create 'perf_callchain' for callchain output event\n"); + ret = -1; + goto put_seq_type; + } + + ret = bt_ctf_field_sequence_set_length(seq_field, len_field); + if (ret) { + pr_err("failed to set length of 'perf_callchain'\n"); + goto put_seq_field; + } + + for (i = 0; i < nr_elements; i++) { + struct bt_ctf_field *elem_field = + bt_ctf_field_sequence_get_field(seq_field, i); + + ret = bt_ctf_field_integer_unsigned_set_value(elem_field, + ((u64 *)(callchain->ips))[i]); + + bt_ctf_field_put(elem_field); + if (ret) { + pr_err("failed to set callchain[%d]\n", i); + goto put_seq_field; + } + } + + ret = bt_ctf_event_set_payload(event, "perf_callchain", seq_field); + if (ret) + pr_err("failed to set payload for raw_data\n"); + +put_seq_field: + bt_ctf_field_put(seq_field); +put_seq_type: + bt_ctf_field_type_put(seq_type); +put_len_field: + bt_ctf_field_put(len_field); +put_len_type: + bt_ctf_field_type_put(len_type); + return ret; +} + +static int add_generic_values(struct ctf_writer *cw, + struct bt_ctf_event *event, + struct evsel *evsel, + struct perf_sample *sample) +{ + u64 type = evsel->core.attr.sample_type; + int ret; + + /* + * missing: + * PERF_SAMPLE_TIME - not needed as we have it in + * ctf event header + * PERF_SAMPLE_READ - TODO + * PERF_SAMPLE_RAW - tracepoint fields are handled separately + * PERF_SAMPLE_BRANCH_STACK - TODO + * PERF_SAMPLE_REGS_USER - TODO + * PERF_SAMPLE_STACK_USER - TODO + */ + + if (type & PERF_SAMPLE_IP) { + ret = value_set_u64_hex(cw, event, "perf_ip", sample->ip); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_TID) { + ret = value_set_s32(cw, event, "perf_tid", sample->tid); + if (ret) + return -1; + + ret = value_set_s32(cw, event, "perf_pid", sample->pid); + if (ret) + return -1; + } + + if ((type & PERF_SAMPLE_ID) || + (type & PERF_SAMPLE_IDENTIFIER)) { + ret = value_set_u64(cw, event, "perf_id", sample->id); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_STREAM_ID) { + ret = value_set_u64(cw, event, "perf_stream_id", sample->stream_id); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_PERIOD) { + ret = value_set_u64(cw, event, "perf_period", sample->period); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_WEIGHT) { + ret = value_set_u64(cw, event, "perf_weight", sample->weight); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_DATA_SRC) { + ret = value_set_u64(cw, event, "perf_data_src", + sample->data_src); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_TRANSACTION) { + ret = value_set_u64(cw, event, "perf_transaction", + sample->transaction); + if (ret) + return -1; + } + + return 0; +} + +static int ctf_stream__flush(struct ctf_stream *cs) +{ + int err = 0; + + if (cs) { + err = bt_ctf_stream_flush(cs->stream); + if (err) + pr_err("CTF stream %d flush failed\n", cs->cpu); + + pr("Flush stream for cpu %d (%u samples)\n", + cs->cpu, cs->count); + + cs->count = 0; + } + + return err; +} + +static struct ctf_stream *ctf_stream__create(struct ctf_writer *cw, int cpu) +{ + struct ctf_stream *cs; + struct bt_ctf_field *pkt_ctx = NULL; + struct bt_ctf_field *cpu_field = NULL; + struct bt_ctf_stream *stream = NULL; + int ret; + + cs = zalloc(sizeof(*cs)); + if (!cs) { + pr_err("Failed to allocate ctf stream\n"); + return NULL; + } + + stream = bt_ctf_writer_create_stream(cw->writer, cw->stream_class); + if (!stream) { + pr_err("Failed to create CTF stream\n"); + goto out; + } + + pkt_ctx = bt_ctf_stream_get_packet_context(stream); + if (!pkt_ctx) { + pr_err("Failed to obtain packet context\n"); + goto out; + } + + cpu_field = bt_ctf_field_structure_get_field(pkt_ctx, "cpu_id"); + bt_ctf_field_put(pkt_ctx); + if (!cpu_field) { + pr_err("Failed to obtain cpu field\n"); + goto out; + } + + ret = bt_ctf_field_integer_unsigned_set_value(cpu_field, (u32) cpu); + if (ret) { + pr_err("Failed to update CPU number\n"); + goto out; + } + + bt_ctf_field_put(cpu_field); + + cs->cpu = cpu; + cs->stream = stream; + return cs; + +out: + if (cpu_field) + bt_ctf_field_put(cpu_field); + if (stream) + bt_ctf_stream_put(stream); + + free(cs); + return NULL; +} + +static void ctf_stream__delete(struct ctf_stream *cs) +{ + if (cs) { + bt_ctf_stream_put(cs->stream); + free(cs); + } +} + +static struct ctf_stream *ctf_stream(struct ctf_writer *cw, int cpu) +{ + struct ctf_stream *cs = cw->stream[cpu]; + + if (!cs) { + cs = ctf_stream__create(cw, cpu); + cw->stream[cpu] = cs; + } + + return cs; +} + +static int get_sample_cpu(struct ctf_writer *cw, struct perf_sample *sample, + struct evsel *evsel) +{ + int cpu = 0; + + if (evsel->core.attr.sample_type & PERF_SAMPLE_CPU) + cpu = sample->cpu; + + if (cpu > cw->stream_cnt) { + pr_err("Event was recorded for CPU %d, limit is at %d.\n", + cpu, cw->stream_cnt); + cpu = 0; + } + + return cpu; +} + +#define STREAM_FLUSH_COUNT 100000 + +/* + * Currently we have no other way to determine the + * time for the stream flush other than keep track + * of the number of events and check it against + * threshold. + */ +static bool is_flush_needed(struct ctf_stream *cs) +{ + return cs->count >= STREAM_FLUSH_COUNT; +} + +static int process_sample_event(const struct perf_tool *tool, + union perf_event *_event, + struct perf_sample *sample, + struct machine *machine __maybe_unused) +{ + struct convert *c = container_of(tool, struct convert, tool); + struct evsel *evsel = sample->evsel; + struct evsel_priv *priv = evsel->priv; + struct ctf_writer *cw = &c->writer; + struct ctf_stream *cs; + struct bt_ctf_event_class *event_class; + struct bt_ctf_event *event; + int ret; + unsigned long type = evsel->core.attr.sample_type; + + if (WARN_ONCE(!priv, "Failed to setup all events.\n")) + return 0; + + if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { + ++c->skipped; + return 0; + } + + event_class = priv->event_class; + + /* update stats */ + c->events_count++; + c->events_size += _event->header.size; + + pr_time2(sample->time, "sample %" PRIu64 "\n", c->events_count); + + event = bt_ctf_event_create(event_class); + if (!event) { + pr_err("Failed to create an CTF event\n"); + return -1; + } + + bt_ctf_clock_set_time(cw->clock, sample->time); + + ret = add_generic_values(cw, event, evsel, sample); + if (ret) + return -1; + + if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT) { + ret = add_tracepoint_values(cw, event_class, event, + evsel, sample); + if (ret) + return -1; + } + + if (type & PERF_SAMPLE_CALLCHAIN) { + ret = add_callchain_output_values(event_class, + event, sample->callchain); + if (ret) + return -1; + } + + if (evsel__is_bpf_output(evsel)) { + ret = add_bpf_output_values(event_class, event, sample); + if (ret) + return -1; + } + + cs = ctf_stream(cw, get_sample_cpu(cw, sample, evsel)); + if (cs) { + if (is_flush_needed(cs)) + ctf_stream__flush(cs); + + cs->count++; + bt_ctf_stream_append_event(cs->stream, event); + } + + bt_ctf_event_put(event); + return cs ? 0 : -1; +} + +#define __NON_SAMPLE_SET_FIELD(_name, _type, _field) \ +do { \ + ret = value_set_##_type(cw, event, #_field, _event->_name._field);\ + if (ret) \ + return -1; \ +} while(0) + +#define __FUNC_PROCESS_NON_SAMPLE(_name, body) \ +static int process_##_name##_event(const struct perf_tool *tool, \ + union perf_event *_event, \ + struct perf_sample *sample, \ + struct machine *machine) \ +{ \ + struct convert *c = container_of(tool, struct convert, tool);\ + struct ctf_writer *cw = &c->writer; \ + struct bt_ctf_event_class *event_class = cw->_name##_class;\ + struct bt_ctf_event *event; \ + struct ctf_stream *cs; \ + int ret; \ + \ + c->non_sample_count++; \ + c->events_size += _event->header.size; \ + event = bt_ctf_event_create(event_class); \ + if (!event) { \ + pr_err("Failed to create an CTF event\n"); \ + return -1; \ + } \ + \ + bt_ctf_clock_set_time(cw->clock, sample->time); \ + body \ + cs = ctf_stream(cw, 0); \ + if (cs) { \ + if (is_flush_needed(cs)) \ + ctf_stream__flush(cs); \ + \ + cs->count++; \ + bt_ctf_stream_append_event(cs->stream, event); \ + } \ + bt_ctf_event_put(event); \ + \ + return perf_event__process_##_name(tool, _event, sample, machine);\ +} + +__FUNC_PROCESS_NON_SAMPLE(comm, + __NON_SAMPLE_SET_FIELD(comm, u32, pid); + __NON_SAMPLE_SET_FIELD(comm, u32, tid); + __NON_SAMPLE_SET_FIELD(comm, string, comm); +) +__FUNC_PROCESS_NON_SAMPLE(fork, + __NON_SAMPLE_SET_FIELD(fork, u32, pid); + __NON_SAMPLE_SET_FIELD(fork, u32, ppid); + __NON_SAMPLE_SET_FIELD(fork, u32, tid); + __NON_SAMPLE_SET_FIELD(fork, u32, ptid); + __NON_SAMPLE_SET_FIELD(fork, u64, time); +) + +__FUNC_PROCESS_NON_SAMPLE(exit, + __NON_SAMPLE_SET_FIELD(fork, u32, pid); + __NON_SAMPLE_SET_FIELD(fork, u32, ppid); + __NON_SAMPLE_SET_FIELD(fork, u32, tid); + __NON_SAMPLE_SET_FIELD(fork, u32, ptid); + __NON_SAMPLE_SET_FIELD(fork, u64, time); +) +__FUNC_PROCESS_NON_SAMPLE(mmap, + __NON_SAMPLE_SET_FIELD(mmap, u32, pid); + __NON_SAMPLE_SET_FIELD(mmap, u32, tid); + __NON_SAMPLE_SET_FIELD(mmap, u64_hex, start); + __NON_SAMPLE_SET_FIELD(mmap, string, filename); +) +__FUNC_PROCESS_NON_SAMPLE(mmap2, + __NON_SAMPLE_SET_FIELD(mmap2, u32, pid); + __NON_SAMPLE_SET_FIELD(mmap2, u32, tid); + __NON_SAMPLE_SET_FIELD(mmap2, u64_hex, start); + __NON_SAMPLE_SET_FIELD(mmap2, string, filename); +) +#undef __NON_SAMPLE_SET_FIELD +#undef __FUNC_PROCESS_NON_SAMPLE + +/* If dup < 0, add a prefix. Else, add _dupl_X suffix. */ +static char *change_name(char *name, char *orig_name, int dup) +{ + char *new_name = NULL; + size_t len; + + if (!name) + name = orig_name; + + if (dup >= 10) + goto out; + /* + * Add '_' prefix to potential keywork. According to + * Mathieu Desnoyers (https://lore.kernel.org/lkml/1074266107.40857.1422045946295.JavaMail.zimbra@efficios.com), + * further CTF spec updating may require us to use '$'. + */ + if (dup < 0) + len = strlen(name) + sizeof("_"); + else + len = strlen(orig_name) + sizeof("_dupl_X"); + + new_name = malloc(len); + if (!new_name) + goto out; + + if (dup < 0) + snprintf(new_name, len, "_%s", name); + else + snprintf(new_name, len, "%s_dupl_%d", orig_name, dup); + +out: + if (name != orig_name) + free(name); + return new_name; +} + +static int event_class_add_field(struct bt_ctf_event_class *event_class, + struct bt_ctf_field_type *type, + struct tep_format_field *field) +{ + struct bt_ctf_field_type *t = NULL; + char *name; + int dup = 1; + int ret; + + /* alias was already assigned */ + if (field->alias != field->name) + return bt_ctf_event_class_add_field(event_class, type, + (char *)field->alias); + + name = field->name; + + /* If 'name' is a keywork, add prefix. */ + if (bt_ctf_validate_identifier(name)) + name = change_name(name, field->name, -1); + + if (!name) { + pr_err("Failed to fix invalid identifier."); + return -1; + } + while ((t = bt_ctf_event_class_get_field_by_name(event_class, name))) { + bt_ctf_field_type_put(t); + name = change_name(name, field->name, dup++); + if (!name) { + pr_err("Failed to create dup name for '%s'\n", field->name); + return -1; + } + } + + ret = bt_ctf_event_class_add_field(event_class, type, name); + if (!ret) + field->alias = name; + + return ret; +} + +static int add_tracepoint_fields_types(struct ctf_writer *cw, + struct tep_format_field *fields, + struct bt_ctf_event_class *event_class) +{ + struct tep_format_field *field; + int ret; + + for (field = fields; field; field = field->next) { + struct bt_ctf_field_type *type; + unsigned long flags = field->flags; + + pr2(" field '%s'\n", field->name); + + type = get_tracepoint_field_type(cw, field); + if (!type) + return -1; + + /* + * A string is an array of chars. For this we use the string + * type and don't care that it is an array. What we don't + * support is an array of strings. + */ + if (flags & TEP_FIELD_IS_STRING) + flags &= ~TEP_FIELD_IS_ARRAY; + + if (flags & TEP_FIELD_IS_ARRAY) + type = bt_ctf_field_type_array_create(type, field->arraylen); + + ret = event_class_add_field(event_class, type, field); + + if (flags & TEP_FIELD_IS_ARRAY) + bt_ctf_field_type_put(type); + + if (ret) { + pr_err("Failed to add field '%s': %d\n", + field->name, ret); + return -1; + } + } + + return 0; +} + +static int add_tracepoint_types(struct ctf_writer *cw, + struct evsel *evsel, + struct bt_ctf_event_class *class) +{ + const struct tep_event *tp_format = evsel__tp_format(evsel); + struct tep_format_field *common_fields = tp_format ? tp_format->format.common_fields : NULL; + struct tep_format_field *fields = tp_format ? tp_format->format.fields : NULL; + int ret; + + ret = add_tracepoint_fields_types(cw, common_fields, class); + if (!ret) + ret = add_tracepoint_fields_types(cw, fields, class); + + return ret; +} + +static int add_bpf_output_types(struct ctf_writer *cw, + struct bt_ctf_event_class *class) +{ + struct bt_ctf_field_type *len_type = cw->data.u32; + struct bt_ctf_field_type *seq_base_type = cw->data.u32_hex; + struct bt_ctf_field_type *seq_type; + int ret; + + ret = bt_ctf_event_class_add_field(class, len_type, "raw_len"); + if (ret) + return ret; + + seq_type = bt_ctf_field_type_sequence_create(seq_base_type, "raw_len"); + if (!seq_type) + return -1; + + return bt_ctf_event_class_add_field(class, seq_type, "raw_data"); +} + +static int add_generic_types(struct ctf_writer *cw, struct evsel *evsel, + struct bt_ctf_event_class *event_class) +{ + u64 type = evsel->core.attr.sample_type; + + /* + * missing: + * PERF_SAMPLE_TIME - not needed as we have it in + * ctf event header + * PERF_SAMPLE_READ - TODO + * PERF_SAMPLE_CALLCHAIN - TODO + * PERF_SAMPLE_RAW - tracepoint fields and BPF output + * are handled separately + * PERF_SAMPLE_BRANCH_STACK - TODO + * PERF_SAMPLE_REGS_USER - TODO + * PERF_SAMPLE_STACK_USER - TODO + */ + +#define ADD_FIELD(cl, t, n) \ + do { \ + pr2(" field '%s'\n", n); \ + if (bt_ctf_event_class_add_field(cl, t, n)) { \ + pr_err("Failed to add field '%s';\n", n); \ + return -1; \ + } \ + } while (0) + + if (type & PERF_SAMPLE_IP) + ADD_FIELD(event_class, cw->data.u64_hex, "perf_ip"); + + if (type & PERF_SAMPLE_TID) { + ADD_FIELD(event_class, cw->data.s32, "perf_tid"); + ADD_FIELD(event_class, cw->data.s32, "perf_pid"); + } + + if ((type & PERF_SAMPLE_ID) || + (type & PERF_SAMPLE_IDENTIFIER)) + ADD_FIELD(event_class, cw->data.u64, "perf_id"); + + if (type & PERF_SAMPLE_STREAM_ID) + ADD_FIELD(event_class, cw->data.u64, "perf_stream_id"); + + if (type & PERF_SAMPLE_PERIOD) + ADD_FIELD(event_class, cw->data.u64, "perf_period"); + + if (type & PERF_SAMPLE_WEIGHT) + ADD_FIELD(event_class, cw->data.u64, "perf_weight"); + + if (type & PERF_SAMPLE_DATA_SRC) + ADD_FIELD(event_class, cw->data.u64, "perf_data_src"); + + if (type & PERF_SAMPLE_TRANSACTION) + ADD_FIELD(event_class, cw->data.u64, "perf_transaction"); + + if (type & PERF_SAMPLE_CALLCHAIN) { + ADD_FIELD(event_class, cw->data.u32, "perf_callchain_size"); + ADD_FIELD(event_class, + bt_ctf_field_type_sequence_create( + cw->data.u64_hex, "perf_callchain_size"), + "perf_callchain"); + } + +#undef ADD_FIELD + return 0; +} + +static int add_event(struct ctf_writer *cw, struct evsel *evsel) +{ + struct bt_ctf_event_class *event_class; + struct evsel_priv *priv; + const char *name = evsel__name(evsel); + int ret; + + if (evsel->priv) { + pr_err("Error: attempt to add already added event %s\n", name); + return -1; + } + pr("Adding event '%s' (type %d)\n", name, evsel->core.attr.type); + + event_class = bt_ctf_event_class_create(name); + if (!event_class) + return -1; + + ret = add_generic_types(cw, evsel, event_class); + if (ret) + goto err; + + if (evsel->core.attr.type == PERF_TYPE_TRACEPOINT) { + ret = add_tracepoint_types(cw, evsel, event_class); + if (ret) + goto err; + } + + if (evsel__is_bpf_output(evsel)) { + ret = add_bpf_output_types(cw, event_class); + if (ret) + goto err; + } + + ret = bt_ctf_stream_class_add_event_class(cw->stream_class, event_class); + if (ret) { + pr("Failed to add event class into stream.\n"); + goto err; + } + + priv = malloc(sizeof(*priv)); + if (!priv) + goto err; + + priv->event_class = event_class; + evsel->priv = priv; + return 0; + +err: + bt_ctf_event_class_put(event_class); + pr_err("Failed to add event '%s'.\n", name); + return -1; +} + +enum setup_events_type { + SETUP_EVENTS_ALL, + SETUP_EVENTS_NOT_TRACEPOINT, + SETUP_EVENTS_TRACEPOINT_ONLY, +}; + +static int setup_events(struct ctf_writer *cw, struct perf_session *session, + enum setup_events_type type) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + int ret; + + evlist__for_each_entry(evlist, evsel) { + bool is_tracepoint = evsel->core.attr.type == PERF_TYPE_TRACEPOINT; + + if (is_tracepoint && type == SETUP_EVENTS_NOT_TRACEPOINT) + continue; + + if (!is_tracepoint && type == SETUP_EVENTS_TRACEPOINT_ONLY) + continue; + + ret = add_event(cw, evsel); + if (ret) + return ret; + } + return 0; +} + +#define __NON_SAMPLE_ADD_FIELD(t, n) \ + do { \ + pr2(" field '%s'\n", #n); \ + if (bt_ctf_event_class_add_field(event_class, cw->data.t, #n)) {\ + pr_err("Failed to add field '%s';\n", #n);\ + return -1; \ + } \ + } while(0) + +#define __FUNC_ADD_NON_SAMPLE_EVENT_CLASS(_name, body) \ +static int add_##_name##_event(struct ctf_writer *cw) \ +{ \ + struct bt_ctf_event_class *event_class; \ + int ret; \ + \ + pr("Adding "#_name" event\n"); \ + event_class = bt_ctf_event_class_create("perf_" #_name);\ + if (!event_class) \ + return -1; \ + body \ + \ + ret = bt_ctf_stream_class_add_event_class(cw->stream_class, event_class);\ + if (ret) { \ + pr("Failed to add event class '"#_name"' into stream.\n");\ + return ret; \ + } \ + \ + cw->_name##_class = event_class; \ + bt_ctf_event_class_put(event_class); \ + return 0; \ +} + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(comm, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(string, comm); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(fork, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, ppid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u32, ptid); + __NON_SAMPLE_ADD_FIELD(u64, time); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(exit, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, ppid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u32, ptid); + __NON_SAMPLE_ADD_FIELD(u64, time); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(mmap, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u64_hex, start); + __NON_SAMPLE_ADD_FIELD(string, filename); +) + +__FUNC_ADD_NON_SAMPLE_EVENT_CLASS(mmap2, + __NON_SAMPLE_ADD_FIELD(u32, pid); + __NON_SAMPLE_ADD_FIELD(u32, tid); + __NON_SAMPLE_ADD_FIELD(u64_hex, start); + __NON_SAMPLE_ADD_FIELD(string, filename); +) +#undef __NON_SAMPLE_ADD_FIELD +#undef __FUNC_ADD_NON_SAMPLE_EVENT_CLASS + +static int setup_non_sample_events(struct ctf_writer *cw, + struct perf_session *session __maybe_unused) +{ + int ret; + + ret = add_comm_event(cw); + if (ret) + return ret; + ret = add_exit_event(cw); + if (ret) + return ret; + ret = add_fork_event(cw); + if (ret) + return ret; + ret = add_mmap_event(cw); + if (ret) + return ret; + ret = add_mmap2_event(cw); + if (ret) + return ret; + return 0; +} + +static void cleanup_events(struct perf_session *session) +{ + struct evlist *evlist = session->evlist; + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + struct evsel_priv *priv; + + priv = evsel->priv; + if (priv) + bt_ctf_event_class_put(priv->event_class); + zfree(&evsel->priv); + } + + evlist__delete(evlist); + session->evlist = NULL; +} + +static int setup_streams(struct ctf_writer *cw, struct perf_session *session) +{ + struct ctf_stream **stream; + struct perf_env *env = perf_session__env(session); + int ncpus; + + /* + * Try to get the number of cpus used in the data file, + * if not present fallback to the MAX_CPUS. + */ + ncpus = env->nr_cpus_avail ?: MAX_CPUS; + + stream = calloc(ncpus, sizeof(*stream)); + if (!stream) { + pr_err("Failed to allocate streams.\n"); + return -ENOMEM; + } + + cw->stream = stream; + cw->stream_cnt = ncpus; + return 0; +} + +static void free_streams(struct ctf_writer *cw) +{ + int cpu; + + for (cpu = 0; cpu < cw->stream_cnt; cpu++) + ctf_stream__delete(cw->stream[cpu]); + + zfree(&cw->stream); +} + +static int ctf_writer__setup_env(struct ctf_writer *cw, + struct perf_session *session) +{ + struct perf_env *env = perf_session__env(session); + struct bt_ctf_writer *writer = cw->writer; + +#define ADD(__n, __v) \ +do { \ + if (__v && bt_ctf_writer_add_environment_field(writer, __n, __v)) \ + return -1; \ +} while (0) + + ADD("host", env->hostname); + ADD("sysname", "Linux"); + ADD("release", perf_env__os_release(env)); + ADD("version", env->version); + ADD("machine", env->arch); + ADD("domain", "kernel"); + ADD("tracer_name", "perf"); + +#undef ADD + return 0; +} + +static int process_feature_event(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event) +{ + struct convert *c = container_of(tool, struct convert, tool); + struct ctf_writer *cw = &c->writer; + struct perf_record_header_feature *fe = &event->feat; + int ret = perf_event__process_feature(tool, session, event); + + if (ret) + return ret; + + switch (fe->feat_id) { + case HEADER_EVENT_DESC: + /* + * In non-pipe mode (not here) the evsels combine the desc with + * the perf_event_attr when it is parsed. In pipe mode the + * perf_event_attr events appear first and then the event desc + * feature events that set the names appear after. Once we have + * the full evsel data we can generate the babeltrace + * events. For tracepoint events we still don't have the tracing + * data and so need to wait until the tracing data event to add + * those events to babeltrace. + */ + return setup_events(cw, session, SETUP_EVENTS_NOT_TRACEPOINT); + case HEADER_HOSTNAME: + if (session->header.env.hostname) { + return bt_ctf_writer_add_environment_field(cw->writer, "host", + session->header.env.hostname); + } + break; + case HEADER_OSRELEASE: + if (session->header.env.os_release) { + return bt_ctf_writer_add_environment_field(cw->writer, "release", + session->header.env.os_release); + } + break; + case HEADER_VERSION: + if (session->header.env.version) { + return bt_ctf_writer_add_environment_field(cw->writer, "version", + session->header.env.version); + } + break; + case HEADER_ARCH: + if (session->header.env.arch) { + return bt_ctf_writer_add_environment_field(cw->writer, "machine", + session->header.env.arch); + } + break; + default: + break; + } + return 0; +} + +static int process_tracing_data(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event) +{ + struct convert *c = container_of(tool, struct convert, tool); + struct ctf_writer *cw = &c->writer; + int ret; + + ret = perf_event__process_tracing_data(tool, session, event); + if (ret < 0) + return ret; + + /* + * Now the attr was set up by the attr event, the name by the feature + * event desc event and the tracepoint data set up above, the tracepoint + * babeltrace events can be added. + */ + return setup_events(cw, session, SETUP_EVENTS_TRACEPOINT_ONLY); +} + +static int ctf_writer__setup_clock(struct ctf_writer *cw, + struct perf_session *session, + bool tod) +{ + struct bt_ctf_clock *clock = cw->clock; + const char *desc = "perf clock"; + int64_t offset = 0; + + if (tod) { + struct perf_env *env = perf_session__env(session); + + if (!env->clock.enabled) { + pr_err("Can't provide --tod time, missing clock data. " + "Please record with -k/--clockid option.\n"); + return -1; + } + + desc = clockid_name(env->clock.clockid); + offset = env->clock.tod_ns - env->clock.clockid_ns; + } + +#define SET(__n, __v) \ +do { \ + if (bt_ctf_clock_set_##__n(clock, __v)) \ + return -1; \ +} while (0) + + SET(frequency, 1000000000); + SET(offset, offset); + SET(description, desc); + SET(precision, 10); + SET(is_absolute, 0); + +#undef SET + return 0; +} + +static struct bt_ctf_field_type *create_int_type(int size, bool sign, bool hex) +{ + struct bt_ctf_field_type *type; + + type = bt_ctf_field_type_integer_create(size); + if (!type) + return NULL; + + if (sign && + bt_ctf_field_type_integer_set_signed(type, 1)) + goto err; + + if (hex && + bt_ctf_field_type_integer_set_base(type, BT_CTF_INTEGER_BASE_HEXADECIMAL)) + goto err; + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + bt_ctf_field_type_set_byte_order(type, BT_CTF_BYTE_ORDER_BIG_ENDIAN); +#else + bt_ctf_field_type_set_byte_order(type, BT_CTF_BYTE_ORDER_LITTLE_ENDIAN); +#endif + + pr2("Created type: INTEGER %d-bit %ssigned %s\n", + size, sign ? "un" : "", hex ? "hex" : ""); + return type; + +err: + bt_ctf_field_type_put(type); + return NULL; +} + +static void ctf_writer__cleanup_data(struct ctf_writer *cw) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cw->data.array); i++) + bt_ctf_field_type_put(cw->data.array[i]); +} + +static int ctf_writer__init_data(struct ctf_writer *cw) +{ +#define CREATE_INT_TYPE(type, size, sign, hex) \ +do { \ + (type) = create_int_type(size, sign, hex); \ + if (!(type)) \ + goto err; \ +} while (0) + + CREATE_INT_TYPE(cw->data.s64, 64, true, false); + CREATE_INT_TYPE(cw->data.u64, 64, false, false); + CREATE_INT_TYPE(cw->data.s32, 32, true, false); + CREATE_INT_TYPE(cw->data.u32, 32, false, false); + CREATE_INT_TYPE(cw->data.u32_hex, 32, false, true); + CREATE_INT_TYPE(cw->data.u64_hex, 64, false, true); + + cw->data.string = bt_ctf_field_type_string_create(); + if (cw->data.string) + return 0; + +err: + ctf_writer__cleanup_data(cw); + pr_err("Failed to create data types.\n"); + return -1; +} + +static void ctf_writer__cleanup(struct ctf_writer *cw) +{ + ctf_writer__cleanup_data(cw); + + bt_ctf_clock_put(cw->clock); + free_streams(cw); + bt_ctf_stream_class_put(cw->stream_class); + bt_ctf_writer_put(cw->writer); + + /* and NULL all the pointers */ + memset(cw, 0, sizeof(*cw)); +} + +static int ctf_writer__init(struct ctf_writer *cw, const char *path, + struct perf_session *session, bool tod) +{ + struct bt_ctf_writer *writer; + struct bt_ctf_stream_class *stream_class; + struct bt_ctf_clock *clock; + struct bt_ctf_field_type *pkt_ctx_type; + int ret; + + /* CTF writer */ + writer = bt_ctf_writer_create(path); + if (!writer) + goto err; + + cw->writer = writer; + + /* CTF clock */ + clock = bt_ctf_clock_create("perf_clock"); + if (!clock) { + pr("Failed to create CTF clock.\n"); + goto err_cleanup; + } + + cw->clock = clock; + + if (ctf_writer__setup_clock(cw, session, tod)) { + pr("Failed to setup CTF clock.\n"); + goto err_cleanup; + } + + /* CTF stream class */ + stream_class = bt_ctf_stream_class_create("perf_stream"); + if (!stream_class) { + pr("Failed to create CTF stream class.\n"); + goto err_cleanup; + } + + cw->stream_class = stream_class; + + /* CTF clock stream setup */ + if (bt_ctf_stream_class_set_clock(stream_class, clock)) { + pr("Failed to assign CTF clock to stream class.\n"); + goto err_cleanup; + } + + if (ctf_writer__init_data(cw)) + goto err_cleanup; + + /* Add cpu_id for packet context */ + pkt_ctx_type = bt_ctf_stream_class_get_packet_context_type(stream_class); + if (!pkt_ctx_type) + goto err_cleanup; + + ret = bt_ctf_field_type_structure_add_field(pkt_ctx_type, cw->data.u32, "cpu_id"); + bt_ctf_field_type_put(pkt_ctx_type); + if (ret) + goto err_cleanup; + + /* CTF clock writer setup */ + if (bt_ctf_writer_add_clock(writer, clock)) { + pr("Failed to assign CTF clock to writer.\n"); + goto err_cleanup; + } + + return 0; + +err_cleanup: + ctf_writer__cleanup(cw); +err: + pr_err("Failed to setup CTF writer.\n"); + return -1; +} + +static int ctf_writer__flush_streams(struct ctf_writer *cw) +{ + int cpu, ret = 0; + + for (cpu = 0; cpu < cw->stream_cnt && !ret; cpu++) + ret = ctf_stream__flush(cw->stream[cpu]); + + return ret; +} + +static int convert__config(const char *var, const char *value, void *cb) +{ + struct convert *c = cb; + + if (!strcmp(var, "convert.queue-size")) + return perf_config_u64(&c->queue_size, var, value); + + return 0; +} + +int bt_convert__perf2ctf(const char *input, const char *path, + struct perf_data_convert_opts *opts) +{ + struct perf_session *session; + struct perf_data data = { + .path = input, + .mode = PERF_DATA_MODE_READ, + .force = opts->force, + }; + struct convert c = {}; + struct ctf_writer *cw = &c.writer; + int err; + + perf_tool__init(&c.tool, /*ordered_events=*/true); + c.tool.sample = process_sample_event; + c.tool.mmap = perf_event__process_mmap; + c.tool.mmap2 = perf_event__process_mmap2; + c.tool.comm = perf_event__process_comm; + c.tool.exit = perf_event__process_exit; + c.tool.fork = perf_event__process_fork; + c.tool.lost = perf_event__process_lost; + c.tool.tracing_data = process_tracing_data; + c.tool.build_id = perf_event__process_build_id; + c.tool.namespaces = perf_event__process_namespaces; + c.tool.finished_round = perf_event__process_finished_round; + c.tool.attr = perf_event__process_attr; + c.tool.feature = process_feature_event; + c.tool.ordering_requires_timestamps = true; + + if (opts->all) { + c.tool.comm = process_comm_event; + c.tool.exit = process_exit_event; + c.tool.fork = process_fork_event; + c.tool.mmap = process_mmap_event; + c.tool.mmap2 = process_mmap2_event; + } + + err = perf_config(convert__config, &c); + if (err) + return err; + + err = -1; + /* perf.data session */ + session = perf_session__new(&data, &c.tool); + if (IS_ERR(session)) + return PTR_ERR(session); + + if (opts->time_str) { + err = perf_time__parse_for_ranges(opts->time_str, session, + &c.ptime_range, + &c.range_size, + &c.range_num); + if (err < 0) + goto free_session; + } + + /* CTF writer */ + if (ctf_writer__init(cw, path, session, opts->tod)) + goto free_session; + + if (c.queue_size) { + ordered_events__set_alloc_size(&session->ordered_events, + c.queue_size); + } + + /* CTF writer env/clock setup */ + if (ctf_writer__setup_env(cw, session)) + goto free_writer; + + /* + * CTF events setup. Note, in pipe mode no events exist yet (they come + * in via header feature events) and so this does nothing. + */ + if (setup_events(cw, session, SETUP_EVENTS_ALL)) + goto free_writer; + + if (opts->all && setup_non_sample_events(cw, session)) + goto free_writer; + + if (setup_streams(cw, session)) + goto free_writer; + + err = perf_session__process_events(session); + if (!err) + err = ctf_writer__flush_streams(cw); + else + pr_err("Error during conversion.\n"); + + fprintf(stderr, "[ perf data convert: Converted '%s' into CTF data '%s' ]\n", + data.path, path); + + fprintf(stderr, "[ perf data convert: Converted and wrote %.3f MB (%" PRIu64 " samples", + (double) c.events_size / 1024.0 / 1024.0, + c.events_count); + + if (!c.non_sample_count) + fprintf(stderr, ") ]\n"); + else + fprintf(stderr, ", %" PRIu64 " non-samples) ]\n", c.non_sample_count); + + if (c.skipped) { + fprintf(stderr, "[ perf data convert: Skipped %" PRIu64 " samples ]\n", + c.skipped); + } + + if (c.ptime_range) + zfree(&c.ptime_range); + + cleanup_events(session); + perf_session__delete(session); + ctf_writer__cleanup(cw); + + return err; + +free_writer: + ctf_writer__cleanup(cw); +free_session: + if (c.ptime_range) + zfree(&c.ptime_range); + + perf_session__delete(session); + pr_err("Error during conversion setup.\n"); + return err; +} diff --git a/tools/perf/util/data-convert-json.c b/tools/perf/util/data-convert-json.c new file mode 100644 index 000000000..40412c3db --- /dev/null +++ b/tools/perf/util/data-convert-json.c @@ -0,0 +1,461 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * JSON export. + * + * Copyright (C) 2021, CodeWeavers Inc. + */ + +#include "data-convert.h" + +#include +#include +#include +#include + +#include "linux/compiler.h" +#include "linux/err.h" +#include "util/auxtrace.h" +#include "util/debug.h" +#include "util/env.h" +#include "util/dso.h" +#include "util/event.h" +#include "util/evsel.h" +#include "util/evlist.h" +#include "util/header.h" +#include "util/map.h" +#include "util/session.h" +#include "util/symbol.h" +#include "util/thread.h" +#include "util/time-utils.h" +#include "util/tool.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +struct convert_json { + struct perf_tool tool; + FILE *out; + bool first; + struct perf_time_interval *ptime_range; + int range_size; + int range_num; + + u64 events_count; + u64 skipped; +}; + +// Outputs a JSON-encoded string surrounded by quotes with characters escaped. +static void output_json_string(FILE *out, const char *s) +{ + fputc('"', out); + if (!s) + goto out; + + while (*s) { + switch (*s) { + + // required escapes with special forms as per RFC 8259 + case '"': fputs("\\\"", out); break; + case '\\': fputs("\\\\", out); break; + case '\b': fputs("\\b", out); break; + case '\f': fputs("\\f", out); break; + case '\n': fputs("\\n", out); break; + case '\r': fputs("\\r", out); break; + case '\t': fputs("\\t", out); break; + + default: + // all other control characters must be escaped by hex code + if (*s <= 0x1f) + fprintf(out, "\\u%04x", *s); + else + fputc(*s, out); + break; + } + + ++s; + } +out: + fputc('"', out); +} + +// Outputs an optional comma, newline and indentation to delimit a new value +// from the previous one in a JSON object or array. +static void output_json_delimiters(FILE *out, bool comma, int depth) +{ + int i; + + if (comma) + fputc(',', out); + fputc('\n', out); + for (i = 0; i < depth; ++i) + fputc('\t', out); +} + +// Outputs a printf format string (with delimiter) as a JSON value. +__printf(4, 5) +static void output_json_format(FILE *out, bool comma, int depth, const char *format, ...) +{ + va_list args; + + output_json_delimiters(out, comma, depth); + va_start(args, format); + vfprintf(out, format, args); + va_end(args); +} + +// Outputs a JSON key-value pair where the value is a string. +static void output_json_key_string(FILE *out, bool comma, int depth, + const char *key, const char *value) +{ + output_json_delimiters(out, comma, depth); + output_json_string(out, key); + fputs(": ", out); + output_json_string(out, value); +} + +// Outputs a JSON key-value pair where the value is a printf format string. +__printf(5, 6) +static void output_json_key_format(FILE *out, bool comma, int depth, + const char *key, const char *format, ...) +{ + va_list args; + + output_json_delimiters(out, comma, depth); + output_json_string(out, key); + fputs(": ", out); + va_start(args, format); + vfprintf(out, format, args); + va_end(args); +} + +static void output_sample_callchain_entry(const struct perf_tool *tool, + u64 ip, struct addr_location *al) +{ + struct convert_json *c = container_of(tool, struct convert_json, tool); + FILE *out = c->out; + + output_json_format(out, false, 4, "{"); + output_json_key_format(out, false, 5, "ip", "\"0x%" PRIx64 "\"", ip); + + if (al && al->sym && al->sym->namelen) { + struct dso *dso = al->map ? map__dso(al->map) : NULL; + + fputc(',', out); + output_json_key_string(out, false, 5, "symbol", al->sym->name); + + if (dso) { + const char *dso_name = dso__short_name(dso); + + if (dso_name && strlen(dso_name) > 0) { + fputc(',', out); + output_json_key_string(out, false, 5, "dso", dso_name); + } + } + } + + output_json_format(out, false, 4, "}"); +} + +static int process_sample_event(const struct perf_tool *tool, + union perf_event *event __maybe_unused, + struct perf_sample *sample, + struct machine *machine) +{ + struct convert_json *c = container_of(tool, struct convert_json, tool); + FILE *out = c->out; + struct addr_location al; + u64 sample_type = __evlist__combined_sample_type(sample->evsel->evlist); + u8 cpumode = PERF_RECORD_MISC_USER; + + addr_location__init(&al); + if (machine__resolve(machine, &al, sample) < 0) { + pr_err("Sample resolution failed!\n"); + addr_location__exit(&al); + return -1; + } + + if (perf_time__ranges_skip_sample(c->ptime_range, c->range_num, sample->time)) { + ++c->skipped; + addr_location__exit(&al); + return 0; + } + + ++c->events_count; + + if (c->first) + c->first = false; + else + fputc(',', out); + output_json_format(out, false, 2, "{"); + + output_json_key_format(out, false, 3, "timestamp", "%" PRIi64, sample->time); + output_json_key_format(out, true, 3, "pid", "%i", thread__pid(al.thread)); + output_json_key_format(out, true, 3, "tid", "%i", thread__tid(al.thread)); + + if ((sample_type & PERF_SAMPLE_CPU)) + output_json_key_format(out, true, 3, "cpu", "%i", sample->cpu); + else if (thread__cpu(al.thread) >= 0) + output_json_key_format(out, true, 3, "cpu", "%i", thread__cpu(al.thread)); + + output_json_key_string(out, true, 3, "comm", thread__comm_str(al.thread)); + + output_json_key_format(out, true, 3, "callchain", "["); + if (sample->callchain) { + unsigned int i; + bool ok; + bool first_callchain = true; + + for (i = 0; i < sample->callchain->nr; ++i) { + u64 ip = sample->callchain->ips[i]; + struct addr_location tal; + + if (ip >= PERF_CONTEXT_MAX) { + switch (ip) { + case PERF_CONTEXT_HV: + cpumode = PERF_RECORD_MISC_HYPERVISOR; + break; + case PERF_CONTEXT_KERNEL: + cpumode = PERF_RECORD_MISC_KERNEL; + break; + case PERF_CONTEXT_USER: + cpumode = PERF_RECORD_MISC_USER; + break; + default: + pr_debug("invalid callchain context: %" + PRId64 "\n", (s64) ip); + break; + } + continue; + } + + if (first_callchain) + first_callchain = false; + else + fputc(',', out); + + addr_location__init(&tal); + ok = thread__find_symbol(al.thread, cpumode, ip, &tal); + output_sample_callchain_entry(tool, ip, ok ? &tal : NULL); + addr_location__exit(&tal); + } + } else { + output_sample_callchain_entry(tool, sample->ip, &al); + } + output_json_format(out, false, 3, "]"); + +#ifdef HAVE_LIBTRACEEVENT + if (sample->raw_data) { + struct tep_event *tp_format = evsel__tp_format(sample->evsel); + struct tep_format_field **fields = tp_format ? tep_event_fields(tp_format) : NULL; + + if (fields) { + int i = 0; + + while (fields[i]) { + struct trace_seq s; + + trace_seq_init(&s); + tep_print_field(&s, sample->raw_data, fields[i]); + output_json_key_string(out, true, 3, fields[i]->name, s.buffer); + + i++; + } + free(fields); + } + } +#endif + output_json_format(out, false, 2, "}"); + addr_location__exit(&al); + return 0; +} + +static void output_headers(struct perf_session *session, struct convert_json *c) +{ + struct stat st; + const struct perf_header *header = &session->header; + struct perf_env *env = perf_session__env(session); + int ret; + int fd = perf_data__fd(session->data); + int i; + FILE *out = c->out; + + output_json_key_format(out, false, 2, "header-version", "%u", header->version); + + ret = fstat(fd, &st); + if (ret >= 0) { + time_t stctime = st.st_mtime; + char buf[256]; + + strftime(buf, sizeof(buf), "%FT%TZ", gmtime(&stctime)); + output_json_key_string(out, true, 2, "captured-on", buf); + } else { + pr_debug("Failed to get mtime of source file, not writing captured-on"); + } + + output_json_key_format(out, true, 2, "data-offset", "%" PRIu64, header->data_offset); + output_json_key_format(out, true, 2, "data-size", "%" PRIu64, header->data_size); + output_json_key_format(out, true, 2, "feat-offset", "%" PRIu64, header->feat_offset); + + output_json_key_string(out, true, 2, "hostname", env->hostname); + output_json_key_string(out, true, 2, "os-release", + perf_env__os_release(env)); + output_json_key_string(out, true, 2, "arch", env->arch); + + if (env->cpu_desc) + output_json_key_string(out, true, 2, "cpu-desc", env->cpu_desc); + + output_json_key_string(out, true, 2, "cpuid", env->cpuid); + output_json_key_format(out, true, 2, "nrcpus-online", "%u", env->nr_cpus_online); + output_json_key_format(out, true, 2, "nrcpus-avail", "%u", env->nr_cpus_avail); + + if (env->clock.enabled) { + output_json_key_format(out, true, 2, "clockid", + "%u", env->clock.clockid); + output_json_key_format(out, true, 2, "clock-time", + "%" PRIu64, env->clock.clockid_ns); + output_json_key_format(out, true, 2, "real-time", + "%" PRIu64, env->clock.tod_ns); + } + + output_json_key_string(out, true, 2, "perf-version", env->version); + + output_json_key_format(out, true, 2, "cmdline", "["); + for (i = 0; i < env->nr_cmdline; i++) { + output_json_delimiters(out, i != 0, 3); + output_json_string(c->out, env->cmdline_argv[i]); + } + output_json_format(out, false, 2, "]"); +} + +int bt_convert__perf2json(const char *_input_name, const char *output_name, + struct perf_data_convert_opts *opts __maybe_unused) +{ + struct perf_session *session; + int fd; + int ret = -1; + struct convert_json c = { + .first = true, + .events_count = 0, + .ptime_range = NULL, + .range_size = 0, + .range_num = 0, + .skipped = 0, + }; + struct perf_data data = { + .mode = PERF_DATA_MODE_READ, + .path = _input_name, + .force = opts->force, + }; + + perf_tool__init(&c.tool, /*ordered_events=*/true); + c.tool.sample = process_sample_event; + c.tool.mmap = perf_event__process_mmap; + c.tool.mmap2 = perf_event__process_mmap2; + c.tool.comm = perf_event__process_comm; + c.tool.namespaces = perf_event__process_namespaces; + c.tool.cgroup = perf_event__process_cgroup; + c.tool.exit = perf_event__process_exit; + c.tool.fork = perf_event__process_fork; + c.tool.lost = perf_event__process_lost; +#ifdef HAVE_LIBTRACEEVENT + c.tool.tracing_data = perf_event__process_tracing_data; +#endif + c.tool.build_id = perf_event__process_build_id; + c.tool.id_index = perf_event__process_id_index; + c.tool.auxtrace_info = perf_event__process_auxtrace_info; + c.tool.auxtrace = perf_event__process_auxtrace; + c.tool.event_update = perf_event__process_event_update; + c.tool.attr = perf_event__process_attr; + c.tool.feature = perf_event__process_feature; + c.tool.ordering_requires_timestamps = true; + + if (opts->all) { + pr_err("--all is currently unsupported for JSON output.\n"); + goto err; + } + if (opts->tod) { + pr_err("--tod is currently unsupported for JSON output.\n"); + goto err; + } + + fd = open(output_name, O_CREAT | O_WRONLY | (opts->force ? O_TRUNC : O_EXCL), 0666); + if (fd == -1) { + if (errno == EEXIST) + pr_err("Output file exists. Use --force to overwrite it.\n"); + else + pr_err("Error opening output file!\n"); + goto err; + } + + c.out = fdopen(fd, "w"); + if (!c.out) { + fprintf(stderr, "Error opening output file!\n"); + close(fd); + goto err; + } + + session = perf_session__new(&data, &c.tool); + if (IS_ERR(session)) { + fprintf(stderr, "Error creating perf session!\n"); + goto err_fclose; + } + if (symbol__init(perf_session__env(session)) < 0) { + fprintf(stderr, "Symbol init error!\n"); + goto err_session_delete; + } + + if (opts->time_str) { + ret = perf_time__parse_for_ranges(opts->time_str, session, + &c.ptime_range, + &c.range_size, + &c.range_num); + if (ret < 0) + goto err_session_delete; + } + + // The opening brace is printed manually because it isn't delimited from a + // previous value (i.e. we don't want a leading newline) + fputc('{', c.out); + + // Version number for future-proofing. Most additions should be able to be + // done in a backwards-compatible way so this should only need to be bumped + // if some major breaking change must be made. + output_json_format(c.out, false, 1, "\"linux-perf-json-version\": 1"); + + // Output headers + output_json_format(c.out, true, 1, "\"headers\": {"); + output_headers(session, &c); + output_json_format(c.out, false, 1, "}"); + + // Output samples + output_json_format(c.out, true, 1, "\"samples\": ["); + perf_session__process_events(session); + output_json_format(c.out, false, 1, "]"); + output_json_format(c.out, false, 0, "}"); + fputc('\n', c.out); + + fprintf(stderr, "[ perf data convert: Converted '%s' into JSON data '%s' ]\n", + data.path, output_name); + + fprintf(stderr, + "[ perf data convert: Converted and wrote %.3f MB (%" PRIu64 " samples) ]\n", + (ftell(c.out)) / 1024.0 / 1024.0, c.events_count); + + if (c.skipped) { + fprintf(stderr, "[ perf data convert: Skipped %" PRIu64 " samples ]\n", + c.skipped); + } + + ret = 0; + + if (c.ptime_range) + zfree(&c.ptime_range); + +err_session_delete: + perf_session__delete(session); +err_fclose: + fclose(c.out); +err: + return ret; +} diff --git a/tools/perf/util/data-convert.h b/tools/perf/util/data-convert.h new file mode 100644 index 000000000..a96240f15 --- /dev/null +++ b/tools/perf/util/data-convert.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __DATA_CONVERT_H +#define __DATA_CONVERT_H + +#include + +struct perf_data_convert_opts { + bool force; + bool all; + bool tod; + const char *time_str; +}; + +#ifdef HAVE_BABELTRACE2_CTF_WRITER_SUPPORT +int bt_convert__perf2ctf(const char *input_name, const char *to_ctf, + struct perf_data_convert_opts *opts); +#endif /* HAVE_BABELTRACE2_CTF_WRITER_SUPPORT */ + +int bt_convert__perf2json(const char *input_name, const char *to_ctf, + struct perf_data_convert_opts *opts); + +#endif /* __DATA_CONVERT_H */ diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c new file mode 100644 index 000000000..94dc534a7 --- /dev/null +++ b/tools/perf/util/data.c @@ -0,0 +1,580 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "data.h" +#include "util.h" // rm_rf_perf_data() +#include "debug.h" +#include "header.h" +#include "rlimit.h" +#include + +static void perf_data_file__close(struct perf_data_file *file) +{ + if (file->use_stdio) { + if (file->fptr) { + fclose(file->fptr); + file->fptr = NULL; + } + } else { + close(file->fd); + file->fd = -1; + } + zfree(&file->path); +} + +static void close_dir(struct perf_data_file *files, int nr) +{ + while (--nr >= 0) + perf_data_file__close(&files[nr]); + + free(files); +} + +void perf_data__close_dir(struct perf_data *data) +{ + close_dir(data->dir.files, data->dir.nr); + data->dir.files = NULL; + data->dir.nr = 0; +} + +int perf_data__create_dir(struct perf_data *data, int nr) +{ + enum rlimit_action set_rlimit = NO_CHANGE; + struct perf_data_file *files = NULL; + int i, ret; + + if (WARN_ON(!data->is_dir)) + return -EINVAL; + + files = calloc(nr, sizeof(*files)); + if (!files) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + struct perf_data_file *file = &files[i]; + + ret = asprintf(&file->path, "%s/data.%d", data->path, i); + if (ret < 0) { + ret = -ENOMEM; + goto out_err; + } + +retry_open: + ret = open(file->path, O_RDWR|O_CREAT|O_TRUNC, S_IRUSR|S_IWUSR); + if (ret < 0) { + /* + * If using parallel threads to collect data, + * perf record needs at least 6 fds per CPU. + * When we run out of them try to increase the limits. + */ + if (errno == EMFILE && rlimit__increase_nofile(&set_rlimit)) + goto retry_open; + + ret = -errno; + goto out_err; + } + set_rlimit = NO_CHANGE; + + file->fd = ret; + } + + data->dir.version = PERF_DIR_VERSION; + data->dir.files = files; + data->dir.nr = nr; + return 0; + +out_err: + close_dir(files, i); + return ret; +} + +int perf_data__open_dir(struct perf_data *data) +{ + struct perf_data_file *files = NULL; + struct dirent *dent; + int ret = -1; + DIR *dir; + int nr = 0; + + /* + * Directory containing a single regular perf data file which is already + * open, means there is nothing more to do here. + */ + if (perf_data__is_single_file(data)) + return 0; + + if (WARN_ON(!data->is_dir)) + return -EINVAL; + + /* The version is provided by DIR_FORMAT feature. */ + if (WARN_ON(data->dir.version != PERF_DIR_VERSION)) + return -1; + + dir = opendir(data->path); + if (!dir) + return -EINVAL; + + while ((dent = readdir(dir)) != NULL) { + struct perf_data_file *file; + char path[PATH_MAX]; + struct stat st; + + snprintf(path, sizeof(path), "%s/%s", data->path, dent->d_name); + if (stat(path, &st)) + continue; + + if (!S_ISREG(st.st_mode) || strncmp(dent->d_name, "data.", 5)) + continue; + + ret = -ENOMEM; + + file = realloc(files, (nr + 1) * sizeof(*files)); + if (!file) + goto out_err; + + files = file; + file = &files[nr++]; + + *file = (struct perf_data_file){ + .path = strdup(path), + .fd = -1, + .size = st.st_size, + .use_stdio = false, + }; + if (!file->path) + goto out_err; + + ret = open(file->path, O_RDONLY); + if (ret < 0) { + ret = -errno; + goto out_err; + } + file->fd = ret; + } + + closedir(dir); + if (!files) + return -EINVAL; + + data->dir.files = files; + data->dir.nr = nr; + return 0; + +out_err: + closedir(dir); + close_dir(files, nr); + return ret; +} + +static bool check_pipe(struct perf_data *data) +{ + struct stat st; + bool is_pipe = false; + int fd = perf_data__is_read(data) ? + STDIN_FILENO : STDOUT_FILENO; + + if (!data->path) { + if (!fstat(fd, &st) && S_ISFIFO(st.st_mode)) + is_pipe = true; + } else { + if (!strcmp(data->path, "-")) + is_pipe = true; + } + + if (is_pipe) { + if (data->file.use_stdio) { + const char *mode; + + mode = perf_data__is_read(data) ? "r" : "w"; + data->file.fptr = fdopen(fd, mode); + + if (data->file.fptr == NULL) { + data->file.fd = fd; + data->file.use_stdio = false; + } + + /* + * When is_pipe and data->file.fd is given, use given fd + * instead of STDIN_FILENO or STDOUT_FILENO + */ + } else if (data->file.fd <= 0) { + data->file.fd = fd; + } + } + + return data->is_pipe = is_pipe; +} + +static int check_backup(struct perf_data *data) +{ + struct stat st; + + if (perf_data__is_read(data)) + return 0; + + if (!stat(data->path, &st) && st.st_size) { + char oldname[PATH_MAX]; + int ret; + + snprintf(oldname, sizeof(oldname), "%s.old", + data->path); + + ret = rm_rf_perf_data(oldname); + if (ret) { + if (ret == -2) + pr_err("Can't remove old data: Unknown file found (%s)\n", oldname); + else + pr_err("Can't remove old data: %m (%s)\n", oldname); + return -1; + } + + if (rename(data->path, oldname)) { + pr_err("Can't move data: %m (%s to %s)\n", data->path, oldname); + return -1; + } + } + + return 0; +} + +static bool is_dir(struct perf_data *data) +{ + struct stat st; + + if (stat(data->path, &st)) + return false; + + return (st.st_mode & S_IFMT) == S_IFDIR; +} + +static int open_file_read(struct perf_data *data) +{ + int flags = data->in_place_update ? O_RDWR : O_RDONLY; + struct stat st; + int fd; + + fd = open(data->file.path, flags); + if (fd < 0) { + int err = errno; + + pr_err("failed to open %s: %m", data->file.path); + if (err == ENOENT && !strcmp(data->file.path, "perf.data")) + pr_err(" (try 'perf record' first)"); + pr_err("\n"); + return -err; + } + + if (fstat(fd, &st) < 0) + goto out_close; + + if (!data->force && st.st_uid && (st.st_uid != geteuid())) { + pr_err("File %s not owned by current user or root (use -f to override)\n", + data->file.path); + goto out_close; + } + + if (!st.st_size) { + pr_info("zero-sized data (%s), nothing to do!\n", + data->file.path); + goto out_close; + } + + data->file.size = st.st_size; + return fd; + + out_close: + close(fd); + return -1; +} + +static int open_file_write(struct perf_data *data) +{ + int fd = open(data->file.path, O_CREAT|O_RDWR|O_TRUNC|O_CLOEXEC, S_IRUSR|S_IWUSR); + + if (fd < 0) + pr_err("failed to open %s : %m\n", data->file.path); + + return fd; +} + +static int open_file(struct perf_data *data) +{ + int fd; + + fd = perf_data__is_read(data) ? + open_file_read(data) : open_file_write(data); + + if (fd < 0) { + zfree(&data->file.path); + return -1; + } + + data->file.fd = fd; + return 0; +} + +static int open_file_dup(struct perf_data *data) +{ + data->file.path = strdup(data->path); + if (!data->file.path) + return -ENOMEM; + + return open_file(data); +} + +static int open_dir(struct perf_data *data) +{ + int ret; + + /* + * So far we open only the header, so we can read the data version and + * layout. + */ + if (asprintf(&data->file.path, "%s/data", data->path) < 0) + return -1; + + if (perf_data__is_write(data) && + mkdir(data->path, S_IRWXU) < 0) + return -1; + + ret = open_file(data); + + /* Cleanup whatever we managed to create so far. */ + if (ret && perf_data__is_write(data)) + rm_rf_perf_data(data->path); + + return ret; +} + +int perf_data__open(struct perf_data *data) +{ + if (check_pipe(data)) + return 0; + + /* currently it allows stdio for pipe only */ + data->file.use_stdio = false; + + if (!data->path) + data->path = "perf.data"; + + if (check_backup(data)) + return -1; + + if (perf_data__is_read(data)) + data->is_dir = is_dir(data); + + return perf_data__is_dir(data) ? + open_dir(data) : open_file_dup(data); +} + +void perf_data__close(struct perf_data *data) +{ + if (perf_data__is_dir(data)) + perf_data__close_dir(data); + + perf_data_file__close(&data->file); +} + +static ssize_t perf_data_file__read(struct perf_data_file *file, void *buf, size_t size) +{ + if (file->use_stdio) { + if (fread(buf, size, 1, file->fptr) == 1) + return size; + return feof(file->fptr) ? 0 : -1; + } + return readn(file->fd, buf, size); +} + +ssize_t perf_data__read(struct perf_data *data, void *buf, size_t size) +{ + return perf_data_file__read(&data->file, buf, size); +} + +ssize_t perf_data_file__write(struct perf_data_file *file, + void *buf, size_t size) +{ + if (file->use_stdio) { + if (fwrite(buf, size, /*nmemb=*/1, file->fptr) == 1) + return size; + return -1; + } + return writen(file->fd, buf, size); +} + +ssize_t perf_data__write(struct perf_data *data, + void *buf, size_t size) +{ + return perf_data_file__write(&data->file, buf, size); +} + +off_t perf_data_file__seek(struct perf_data_file *file, off_t offset, int whence) +{ + if (file->use_stdio) { + off_t res = fseeko(file->fptr, offset, whence); + + return res < 0 ? -1 : ftello(file->fptr); + } + return lseek(file->fd, offset, whence); +} + +off_t perf_data__seek(struct perf_data *data, off_t offset, int whence) +{ + /* Note, a pipe fd will fail with -1 with errno of ESPIPE. */ + return perf_data_file__seek(&data->file, offset, whence); +} + +int perf_data__switch(struct perf_data *data, + const char *postfix, + size_t pos, bool at_exit, + char **new_filepath) +{ + int ret; + + if (perf_data__is_read(data)) + return -EINVAL; + + if (asprintf(new_filepath, "%s.%s", data->path, postfix) < 0) + return -ENOMEM; + + /* + * Only fire a warning, don't return error, continue fill + * original file. + */ + if (rename(data->path, *new_filepath)) + pr_warning("Failed to rename %s to %s\n", data->path, *new_filepath); + + if (!at_exit) { + perf_data_file__close(&data->file); + ret = perf_data__open(data); + if (ret < 0) + goto out; + + if (perf_data__seek(data, pos, SEEK_SET) == (off_t)-1) { + ret = -errno; + pr_debug("Failed to seek to %zu: %m", pos); + goto out; + } + } + ret = perf_data__fd(data); +out: + return ret; +} + +unsigned long perf_data__size(struct perf_data *data) +{ + u64 size = data->file.size; + int i; + + if (perf_data__is_single_file(data)) + return size; + + for (i = 0; i < data->dir.nr; i++) { + struct perf_data_file *file = &data->dir.files[i]; + + size += file->size; + } + + return size; +} + +int perf_data__make_kcore_dir(struct perf_data *data, char *buf, size_t buf_sz) +{ + int ret; + + if (!data->is_dir) + return -1; + + ret = snprintf(buf, buf_sz, "%s/kcore_dir", data->path); + if (ret < 0 || (size_t)ret >= buf_sz) + return -1; + + return mkdir(buf, S_IRWXU); +} + +bool has_kcore_dir(const char *path) +{ + struct dirent *d = ERR_PTR(-EINVAL); + const char *name = "kcore_dir"; + DIR *dir = opendir(path); + size_t n = strlen(name); + bool result = false; + + if (dir) { + while (d && !result) { + d = readdir(dir); + result = d ? strncmp(d->d_name, name, n) : false; + } + closedir(dir); + } + + return result; +} + +char *perf_data__kallsyms_name(struct perf_data *data) +{ + char *kallsyms_name; + struct stat st; + + if (!data->is_dir) + return NULL; + + if (asprintf(&kallsyms_name, "%s/kcore_dir/kallsyms", data->path) < 0) + return NULL; + + if (stat(kallsyms_name, &st)) { + free(kallsyms_name); + return NULL; + } + + return kallsyms_name; +} + +char *perf_data__guest_kallsyms_name(struct perf_data *data, pid_t machine_pid) +{ + char *kallsyms_name; + struct stat st; + + if (!data->is_dir) + return NULL; + + if (asprintf(&kallsyms_name, "%s/kcore_dir__%d/kallsyms", data->path, machine_pid) < 0) + return NULL; + + if (stat(kallsyms_name, &st)) { + free(kallsyms_name); + return NULL; + } + + return kallsyms_name; +} + +bool is_perf_data(const char *path) +{ + bool ret = false; + FILE *file; + u64 magic; + + file = fopen(path, "r"); + if (!file) + return false; + + if (fread(&magic, 1, 8, file) < 8) + goto out; + + ret = is_perf_magic(magic); +out: + fclose(file); + return ret; +} diff --git a/tools/perf/util/data.h b/tools/perf/util/data.h new file mode 100644 index 000000000..8299fb5fa --- /dev/null +++ b/tools/perf/util/data.h @@ -0,0 +1,142 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DATA_H +#define __PERF_DATA_H + +#include +#include +#include +#include + +enum perf_data_mode { + PERF_DATA_MODE_WRITE, + PERF_DATA_MODE_READ, +}; + +enum perf_dir_version { + PERF_DIR_SINGLE_FILE = 0, + PERF_DIR_VERSION = 1, +}; + +/** + * struct perf_data_file: A wrapper around a file used for perf.data reading or writing. Generally + * part of struct perf_data. + */ +struct perf_data_file { + /** + * @path: Path of file. Generally a copy of perf_data.path but for a + * directory it is the file within the directory. + */ + char *path; + union { + /** @fd: File descriptor for read/writes. Valid if use_stdio is false. */ + int fd; + /** + * @fptr: Stdio FILE. Valid if use_stdio is true, currently just + * pipes in perf inject. + */ + FILE *fptr; + }; + /** @size: Size of file when opened. */ + unsigned long size; + /** @use_stdio: Use buffered stdio operations. */ + bool use_stdio; +}; + +/** + * struct perf_data: A wrapper around a file used for perf.data reading or writing. + */ +struct perf_data { + /** @path: Path to open and of the file. NULL implies 'perf.data' will be used. */ + const char *path; + /** @file: Underlying file to be used. */ + struct perf_data_file file; + /** @is_pipe: Underlying file is a pipe. */ + bool is_pipe; + /** @is_dir: Underlying file is a directory. */ + bool is_dir; + /** @force: Ignore opening a file creating created by a different user. */ + bool force; + /** @in_place_update: A file opened for reading but will be written to. */ + bool in_place_update; + /** @mode: Read or write mode. */ + enum perf_data_mode mode; + + struct { + /** @version: perf_dir_version. */ + u64 version; + /** @files: perf data files for the directory. */ + struct perf_data_file *files; + /** @nr: Number of perf data files for the directory. */ + int nr; + } dir; +}; + +static inline int perf_data_file__fd(struct perf_data_file *file) +{ + return file->use_stdio ? fileno(file->fptr) : file->fd; +} + +ssize_t perf_data_file__write(struct perf_data_file *file, + void *buf, size_t size); +off_t perf_data_file__seek(struct perf_data_file *file, off_t offset, int whence); + + +static inline bool perf_data__is_read(struct perf_data *data) +{ + return data->mode == PERF_DATA_MODE_READ; +} + +static inline bool perf_data__is_write(struct perf_data *data) +{ + return data->mode == PERF_DATA_MODE_WRITE; +} + +static inline int perf_data__is_pipe(struct perf_data *data) +{ + return data->is_pipe; +} + +static inline bool perf_data__is_dir(struct perf_data *data) +{ + return data->is_dir; +} + +static inline bool perf_data__is_single_file(struct perf_data *data) +{ + return data->dir.version == PERF_DIR_SINGLE_FILE; +} + +static inline int perf_data__fd(struct perf_data *data) +{ + return perf_data_file__fd(&data->file); +} + +int perf_data__open(struct perf_data *data); +void perf_data__close(struct perf_data *data); +ssize_t perf_data__read(struct perf_data *data, void *buf, size_t size); +ssize_t perf_data__write(struct perf_data *data, + void *buf, size_t size); +off_t perf_data__seek(struct perf_data *data, off_t offset, int whence); +/* + * If at_exit is set, only rename current perf.data to + * perf.data., continue write on original data. + * Set at_exit when flushing the last output. + * + * Return value is fd of new output. + */ +int perf_data__switch(struct perf_data *data, + const char *postfix, + size_t pos, bool at_exit, char **new_filepath); + +int perf_data__create_dir(struct perf_data *data, int nr); +int perf_data__open_dir(struct perf_data *data); +void perf_data__close_dir(struct perf_data *data); +unsigned long perf_data__size(struct perf_data *data); +int perf_data__make_kcore_dir(struct perf_data *data, char *buf, size_t buf_sz); +char *perf_data__kallsyms_name(struct perf_data *data); +char *perf_data__guest_kallsyms_name(struct perf_data *data, pid_t machine_pid); + +bool has_kcore_dir(const char *path); +bool is_perf_data(const char *path); + +#endif /* __PERF_DATA_H */ diff --git a/tools/perf/util/db-export.c b/tools/perf/util/db-export.c new file mode 100644 index 000000000..ba54b1119 --- /dev/null +++ b/tools/perf/util/db-export.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * db-export.c: Support for exporting data suitable for import to a database + * Copyright (c) 2014, Intel Corporation. + */ + +#include +#include + +#include "dso.h" +#include "evsel.h" +#include "machine.h" +#include "thread.h" +#include "comm.h" +#include "symbol.h" +#include "map.h" +#include "event.h" +#include "thread-stack.h" +#include "callchain.h" +#include "call-path.h" +#include "db-export.h" + +int db_export__init(struct db_export *dbe) +{ + memset(dbe, 0, sizeof(struct db_export)); + return 0; +} + +void db_export__exit(struct db_export *dbe) +{ + call_return_processor__free(dbe->crp); + dbe->crp = NULL; +} + +int db_export__evsel(struct db_export *dbe, struct evsel *evsel) +{ + if (evsel->db_id) + return 0; + + evsel->db_id = ++dbe->evsel_last_db_id; + + if (dbe->export_evsel) + return dbe->export_evsel(dbe, evsel); + + return 0; +} + +int db_export__machine(struct db_export *dbe, struct machine *machine) +{ + if (machine->db_id) + return 0; + + machine->db_id = ++dbe->machine_last_db_id; + + if (dbe->export_machine) + return dbe->export_machine(dbe, machine); + + return 0; +} + +int db_export__thread(struct db_export *dbe, struct thread *thread, + struct machine *machine, struct thread *main_thread) +{ + u64 main_thread_db_id = 0; + + if (thread__db_id(thread)) + return 0; + + thread__set_db_id(thread, ++dbe->thread_last_db_id); + + if (main_thread) + main_thread_db_id = thread__db_id(main_thread); + + if (dbe->export_thread) + return dbe->export_thread(dbe, thread, main_thread_db_id, + machine); + + return 0; +} + +static int __db_export__comm(struct db_export *dbe, struct comm *comm, + struct thread *thread) +{ + comm->db_id = ++dbe->comm_last_db_id; + + if (dbe->export_comm) + return dbe->export_comm(dbe, comm, thread); + + return 0; +} + +int db_export__comm(struct db_export *dbe, struct comm *comm, + struct thread *thread) +{ + if (comm->db_id) + return 0; + + return __db_export__comm(dbe, comm, thread); +} + +/* + * Export the "exec" comm. The "exec" comm is the program / application command + * name at the time it first executes. It is used to group threads for the same + * program. Note that the main thread pid (or thread group id tgid) cannot be + * used because it does not change when a new program is exec'ed. + */ +int db_export__exec_comm(struct db_export *dbe, struct comm *comm, + struct thread *main_thread) +{ + int err; + + if (comm->db_id) + return 0; + + err = __db_export__comm(dbe, comm, main_thread); + if (err) + return err; + + /* + * Record the main thread for this comm. Note that the main thread can + * have many "exec" comms because there will be a new one every time it + * exec's. An "exec" comm however will only ever have 1 main thread. + * That is different to any other threads for that same program because + * exec() will effectively kill them, so the relationship between the + * "exec" comm and non-main threads is 1-to-1. That is why + * db_export__comm_thread() is called here for the main thread, but it + * is called for non-main threads when they are exported. + */ + return db_export__comm_thread(dbe, comm, main_thread); +} + +int db_export__comm_thread(struct db_export *dbe, struct comm *comm, + struct thread *thread) +{ + u64 db_id; + + db_id = ++dbe->comm_thread_last_db_id; + + if (dbe->export_comm_thread) + return dbe->export_comm_thread(dbe, db_id, comm, thread); + + return 0; +} + +int db_export__dso(struct db_export *dbe, struct dso *dso, + struct machine *machine) +{ + if (dso__db_id(dso)) + return 0; + + dso__set_db_id(dso, ++dbe->dso_last_db_id); + + if (dbe->export_dso) + return dbe->export_dso(dbe, dso, machine); + + return 0; +} + +int db_export__symbol(struct db_export *dbe, struct symbol *sym, + struct dso *dso) +{ + u64 *sym_db_id = symbol__priv(sym); + + if (*sym_db_id) + return 0; + + *sym_db_id = ++dbe->symbol_last_db_id; + + if (dbe->export_symbol) + return dbe->export_symbol(dbe, sym, dso); + + return 0; +} + +static int db_ids_from_al(struct db_export *dbe, struct addr_location *al, + u64 *dso_db_id, u64 *sym_db_id, u64 *offset) +{ + int err; + + if (al->map) { + struct dso *dso = map__dso(al->map); + + err = db_export__dso(dbe, dso, maps__machine(thread__maps(al->thread))); + if (err) + return err; + *dso_db_id = dso__db_id(dso); + + if (!al->sym) { + al->sym = symbol__new(al->addr, 0, 0, 0, "unknown"); + if (al->sym) + dso__insert_symbol(dso, al->sym); + } + + if (al->sym) { + u64 *db_id = symbol__priv(al->sym); + + err = db_export__symbol(dbe, al->sym, dso); + if (err) + return err; + *sym_db_id = *db_id; + *offset = al->addr - al->sym->start; + } + } + + return 0; +} + +static struct call_path *call_path_from_sample(struct db_export *dbe, + struct machine *machine, + struct thread *thread, + struct perf_sample *sample) +{ + u64 kernel_start = machine__kernel_start(machine); + struct call_path *current = &dbe->cpr->call_path; + enum chain_order saved_order = callchain_param.order; + struct callchain_cursor *cursor; + int err; + + if (!symbol_conf.use_callchain || !sample->callchain) + return NULL; + + /* + * Since the call path tree must be built starting with the root, we + * must use ORDER_CALL for call chain resolution, in order to process + * the callchain starting with the root node and ending with the leaf. + */ + callchain_param.order = ORDER_CALLER; + cursor = get_tls_callchain_cursor(); + err = thread__resolve_callchain(thread, cursor, + sample, NULL, NULL, PERF_MAX_STACK_DEPTH); + if (err) { + callchain_param.order = saved_order; + return NULL; + } + callchain_cursor_commit(cursor); + + while (1) { + struct callchain_cursor_node *node; + struct addr_location al; + u64 dso_db_id = 0, sym_db_id = 0, offset = 0; + + + node = callchain_cursor_current(cursor); + if (!node) + break; + + /* + * Handle export of symbol and dso for this node by + * constructing an addr_location struct and then passing it to + * db_ids_from_al() to perform the export. + */ + addr_location__init(&al); + al.sym = node->ms.sym; + al.map = map__get(node->ms.map); + al.addr = node->ip; + al.thread = thread__get(thread); + + if (al.map && !al.sym) + al.sym = dso__find_symbol(map__dso(al.map), al.addr); + + db_ids_from_al(dbe, &al, &dso_db_id, &sym_db_id, &offset); + + /* add node to the call path tree if it doesn't exist */ + current = call_path__findnew(dbe->cpr, current, + al.sym, node->ip, + kernel_start); + + callchain_cursor_advance(cursor); + addr_location__exit(&al); + } + + /* Reset the callchain order to its prior value. */ + callchain_param.order = saved_order; + + if (current == &dbe->cpr->call_path) { + /* Bail because the callchain was empty. */ + return NULL; + } + + return current; +} + +int db_export__branch_type(struct db_export *dbe, u32 branch_type, + const char *name) +{ + if (dbe->export_branch_type) + return dbe->export_branch_type(dbe, branch_type, name); + + return 0; +} + +static int db_export__threads(struct db_export *dbe, struct thread *thread, + struct thread *main_thread, + struct machine *machine, struct comm **comm_ptr) +{ + struct comm *comm = NULL; + struct comm *curr_comm; + int err; + + if (main_thread) { + /* + * A thread has a reference to the main thread, so export the + * main thread first. + */ + err = db_export__thread(dbe, main_thread, machine, main_thread); + if (err) + return err; + /* + * Export comm before exporting the non-main thread because + * db_export__comm_thread() can be called further below. + */ + comm = machine__thread_exec_comm(machine, main_thread); + if (comm) { + err = db_export__exec_comm(dbe, comm, main_thread); + if (err) + return err; + *comm_ptr = comm; + } + } + + if (thread != main_thread) { + /* + * For a non-main thread, db_export__comm_thread() must be + * called only if thread has not previously been exported. + */ + bool export_comm_thread = comm && !thread__db_id(thread); + + err = db_export__thread(dbe, thread, machine, main_thread); + if (err) + return err; + + if (export_comm_thread) { + err = db_export__comm_thread(dbe, comm, thread); + if (err) + return err; + } + } + + curr_comm = thread__comm(thread); + if (curr_comm) + return db_export__comm(dbe, curr_comm, thread); + + return 0; +} + +int db_export__sample(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, + struct addr_location *al, struct addr_location *addr_al) +{ + struct thread *thread = al->thread; + struct export_sample es = { + .event = event, + .sample = sample, + .al = al, + }; + struct thread *main_thread; + struct comm *comm = NULL; + struct machine *machine = NULL; + int err; + + if (thread__maps(thread)) + machine = maps__machine(thread__maps(thread)); + if (!machine) + return -1; + + err = db_export__evsel(dbe, sample->evsel); + if (err) + return err; + + err = db_export__machine(dbe, machine); + if (err) + return err; + + main_thread = thread__main_thread(machine, thread); + + err = db_export__threads(dbe, thread, main_thread, machine, &comm); + if (err) + goto out_put; + + if (comm) + es.comm_db_id = comm->db_id; + + es.db_id = ++dbe->sample_last_db_id; + + err = db_ids_from_al(dbe, al, &es.dso_db_id, &es.sym_db_id, &es.offset); + if (err) + goto out_put; + + if (dbe->cpr) { + struct call_path *cp = call_path_from_sample(dbe, machine, + thread, sample); + if (cp) { + db_export__call_path(dbe, cp); + es.call_path_id = cp->db_id; + } + } + + if (addr_al) { + err = db_ids_from_al(dbe, addr_al, &es.addr_dso_db_id, + &es.addr_sym_db_id, &es.addr_offset); + if (err) + goto out_put; + if (dbe->crp) { + err = thread_stack__process(thread, comm, sample, al, + addr_al, es.db_id, + dbe->crp); + if (err) + goto out_put; + } + } + + if (dbe->export_sample) + err = dbe->export_sample(dbe, &es); + +out_put: + thread__put(main_thread); + return err; +} + +static struct { + u32 branch_type; + const char *name; +} branch_types[] = { + {0, "no branch"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL, "call"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN, "return"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CONDITIONAL, "conditional jump"}, + {PERF_IP_FLAG_BRANCH, "unconditional jump"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_INTERRUPT, + "software interrupt"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT, + "return from interrupt"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_SYSCALLRET, + "system call"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET, + "return from system call"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_ASYNC, "asynchronous branch"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | + PERF_IP_FLAG_INTERRUPT, "hardware interrupt"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT, "transaction abort"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_BEGIN, "trace begin"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "trace end"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vm entry"}, + {PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vm exit"}, + {0, NULL} +}; + +int db_export__branch_types(struct db_export *dbe) +{ + int i, err = 0; + + for (i = 0; branch_types[i].name ; i++) { + err = db_export__branch_type(dbe, branch_types[i].branch_type, + branch_types[i].name); + if (err) + break; + } + + /* Add trace begin / end variants */ + for (i = 0; branch_types[i].name ; i++) { + const char *name = branch_types[i].name; + u32 type = branch_types[i].branch_type; + char buf[64]; + + if (type == PERF_IP_FLAG_BRANCH || + (type & (PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END))) + continue; + + snprintf(buf, sizeof(buf), "trace begin / %s", name); + err = db_export__branch_type(dbe, type | PERF_IP_FLAG_TRACE_BEGIN, buf); + if (err) + break; + + snprintf(buf, sizeof(buf), "%s / trace end", name); + err = db_export__branch_type(dbe, type | PERF_IP_FLAG_TRACE_END, buf); + if (err) + break; + } + + return err; +} + +int db_export__call_path(struct db_export *dbe, struct call_path *cp) +{ + int err; + + if (cp->db_id) + return 0; + + if (cp->parent) { + err = db_export__call_path(dbe, cp->parent); + if (err) + return err; + } + + cp->db_id = ++dbe->call_path_last_db_id; + + if (dbe->export_call_path) + return dbe->export_call_path(dbe, cp); + + return 0; +} + +int db_export__call_return(struct db_export *dbe, struct call_return *cr, + u64 *parent_db_id) +{ + int err; + + err = db_export__call_path(dbe, cr->cp); + if (err) + return err; + + if (!cr->db_id) + cr->db_id = ++dbe->call_return_last_db_id; + + if (parent_db_id) { + if (!*parent_db_id) + *parent_db_id = ++dbe->call_return_last_db_id; + cr->parent_db_id = *parent_db_id; + } + + if (dbe->export_call_return) + return dbe->export_call_return(dbe, cr); + + return 0; +} + +static int db_export__pid_tid(struct db_export *dbe, struct machine *machine, + pid_t pid, pid_t tid, u64 *db_id, + struct comm **comm_ptr, bool *is_idle) +{ + struct thread *thread = machine__find_thread(machine, pid, tid); + struct thread *main_thread; + int err = 0; + + if (!thread || !thread__comm_set(thread)) + goto out_put; + + *is_idle = !thread__pid(thread) && !thread__tid(thread); + + main_thread = thread__main_thread(machine, thread); + + err = db_export__threads(dbe, thread, main_thread, machine, comm_ptr); + + *db_id = thread__db_id(thread); + + thread__put(main_thread); +out_put: + thread__put(thread); + + return err; +} + +int db_export__switch(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, struct machine *machine) +{ + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + bool out_preempt = out && + (event->header.misc & PERF_RECORD_MISC_SWITCH_OUT_PREEMPT); + int flags = out | (out_preempt << 1); + bool is_idle_a = false, is_idle_b = false; + u64 th_a_id = 0, th_b_id = 0; + u64 comm_out_id, comm_in_id; + struct comm *comm_a = NULL; + struct comm *comm_b = NULL; + u64 th_out_id, th_in_id; + u64 db_id; + int err; + + err = db_export__machine(dbe, machine); + if (err) + return err; + + err = db_export__pid_tid(dbe, machine, sample->pid, sample->tid, + &th_a_id, &comm_a, &is_idle_a); + if (err) + return err; + + if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) { + pid_t pid = event->context_switch.next_prev_pid; + pid_t tid = event->context_switch.next_prev_tid; + + err = db_export__pid_tid(dbe, machine, pid, tid, &th_b_id, + &comm_b, &is_idle_b); + if (err) + return err; + } + + /* + * Do not export if both threads are unknown (i.e. not being traced), + * or one is unknown and the other is the idle task. + */ + if ((!th_a_id || is_idle_a) && (!th_b_id || is_idle_b)) + return 0; + + db_id = ++dbe->context_switch_last_db_id; + + if (out) { + th_out_id = th_a_id; + th_in_id = th_b_id; + comm_out_id = comm_a ? comm_a->db_id : 0; + comm_in_id = comm_b ? comm_b->db_id : 0; + } else { + th_out_id = th_b_id; + th_in_id = th_a_id; + comm_out_id = comm_b ? comm_b->db_id : 0; + comm_in_id = comm_a ? comm_a->db_id : 0; + } + + if (dbe->export_context_switch) + return dbe->export_context_switch(dbe, db_id, machine, sample, + th_out_id, comm_out_id, + th_in_id, comm_in_id, flags); + return 0; +} diff --git a/tools/perf/util/db-export.h b/tools/perf/util/db-export.h new file mode 100644 index 000000000..1abbfd398 --- /dev/null +++ b/tools/perf/util/db-export.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * db-export.h: Support for exporting data suitable for import to a database + * Copyright (c) 2014, Intel Corporation. + */ + +#ifndef __PERF_DB_EXPORT_H +#define __PERF_DB_EXPORT_H + +#include +#include + +struct evsel; +struct machine; +struct thread; +struct comm; +struct dso; +struct perf_sample; +struct addr_location; +struct call_return_processor; +struct call_path_root; +struct call_path; +struct call_return; + +struct export_sample { + union perf_event *event; + struct perf_sample *sample; + struct addr_location *al; + u64 db_id; + u64 comm_db_id; + u64 dso_db_id; + u64 sym_db_id; + u64 offset; /* ip offset from symbol start */ + u64 addr_dso_db_id; + u64 addr_sym_db_id; + u64 addr_offset; /* addr offset from symbol start */ + u64 call_path_id; +}; + +struct db_export { + int (*export_evsel)(struct db_export *dbe, struct evsel *evsel); + int (*export_machine)(struct db_export *dbe, struct machine *machine); + int (*export_thread)(struct db_export *dbe, struct thread *thread, + u64 main_thread_db_id, struct machine *machine); + int (*export_comm)(struct db_export *dbe, struct comm *comm, + struct thread *thread); + int (*export_comm_thread)(struct db_export *dbe, u64 db_id, + struct comm *comm, struct thread *thread); + int (*export_dso)(struct db_export *dbe, struct dso *dso, + struct machine *machine); + int (*export_symbol)(struct db_export *dbe, struct symbol *sym, + struct dso *dso); + int (*export_branch_type)(struct db_export *dbe, u32 branch_type, + const char *name); + int (*export_sample)(struct db_export *dbe, struct export_sample *es); + int (*export_call_path)(struct db_export *dbe, struct call_path *cp); + int (*export_call_return)(struct db_export *dbe, + struct call_return *cr); + int (*export_context_switch)(struct db_export *dbe, u64 db_id, + struct machine *machine, + struct perf_sample *sample, + u64 th_out_id, u64 comm_out_id, + u64 th_in_id, u64 comm_in_id, int flags); + struct call_return_processor *crp; + struct call_path_root *cpr; + u64 evsel_last_db_id; + u64 machine_last_db_id; + u64 thread_last_db_id; + u64 comm_last_db_id; + u64 comm_thread_last_db_id; + u64 dso_last_db_id; + u64 symbol_last_db_id; + u64 sample_last_db_id; + u64 call_path_last_db_id; + u64 call_return_last_db_id; + u64 context_switch_last_db_id; +}; + +int db_export__init(struct db_export *dbe); +void db_export__exit(struct db_export *dbe); +int db_export__evsel(struct db_export *dbe, struct evsel *evsel); +int db_export__machine(struct db_export *dbe, struct machine *machine); +int db_export__thread(struct db_export *dbe, struct thread *thread, + struct machine *machine, struct thread *main_thread); +int db_export__comm(struct db_export *dbe, struct comm *comm, + struct thread *thread); +int db_export__exec_comm(struct db_export *dbe, struct comm *comm, + struct thread *main_thread); +int db_export__comm_thread(struct db_export *dbe, struct comm *comm, + struct thread *thread); +int db_export__dso(struct db_export *dbe, struct dso *dso, + struct machine *machine); +int db_export__symbol(struct db_export *dbe, struct symbol *sym, + struct dso *dso); +int db_export__branch_type(struct db_export *dbe, u32 branch_type, + const char *name); +int db_export__sample(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, + struct addr_location *al, struct addr_location *addr_al); + +int db_export__branch_types(struct db_export *dbe); + +int db_export__call_path(struct db_export *dbe, struct call_path *cp); +int db_export__call_return(struct db_export *dbe, struct call_return *cr, + u64 *parent_db_id); +int db_export__switch(struct db_export *dbe, union perf_event *event, + struct perf_sample *sample, struct machine *machine); + +#endif diff --git a/tools/perf/util/debug.c b/tools/perf/util/debug.c new file mode 100644 index 000000000..6b5ffe81f --- /dev/null +++ b/tools/perf/util/debug.c @@ -0,0 +1,380 @@ +// SPDX-License-Identifier: GPL-2.0 +/* For general debugging purposes */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_BACKTRACE_SUPPORT +#include +#endif +#include "addr_location.h" +#include "color.h" +#include "debug.h" +#include "env.h" +#include "event.h" +#include "machine.h" +#include "map.h" +#include "print_binary.h" +#include "srcline.h" +#include "symbol.h" +#include "synthetic-events.h" +#include "target.h" +#include "thread.h" +#include "trace-event.h" +#include "ui/helpline.h" +#include "ui/ui.h" +#include "util/parse-sublevel-options.h" + +#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#else +#define LIBTRACEEVENT_VERSION 0 +#endif + +int verbose; +int debug_kmaps; +int debug_peo_args; +bool dump_trace = false, quiet = false; +int debug_ordered_events; +static int redirect_to_stderr; +int debug_data_convert; +static FILE *_debug_file; +static bool debug_display_time; +int debug_type_profile; + +FILE *debug_file(void) +{ + if (!_debug_file) { + debug_set_file(stderr); + pr_warning_once("debug_file not set"); + } + return _debug_file; +} + +void debug_set_file(FILE *file) +{ + _debug_file = file; +} + +void debug_set_display_time(bool set) +{ + debug_display_time = set; +} + +static int fprintf_time(FILE *file) +{ + struct timeval tod; + struct tm ltime; + char date[64]; + + if (!debug_display_time) + return 0; + + if (gettimeofday(&tod, NULL) != 0) + return 0; + + if (localtime_r(&tod.tv_sec, <ime) == NULL) + return 0; + + strftime(date, sizeof(date), "%F %H:%M:%S", <ime); + return fprintf(file, "[%s.%06lu] ", date, (long)tod.tv_usec); +} + +int veprintf(int level, int var, const char *fmt, va_list args) +{ + int ret = 0; + + if (var >= level) { + if (use_browser >= 1 && !redirect_to_stderr) { + ui_helpline__vshow(fmt, args); + } else { + ret = fprintf_time(debug_file()); + ret += vfprintf(debug_file(), fmt, args); + } + } + + return ret; +} + +int eprintf(int level, int var, const char *fmt, ...) +{ + va_list args; + int ret; + + va_start(args, fmt); + ret = veprintf(level, var, fmt, args); + va_end(args); + + return ret; +} + +static int veprintf_time(u64 t, const char *fmt, va_list args) +{ + int ret = 0; + u64 secs, usecs, nsecs = t; + + secs = nsecs / NSEC_PER_SEC; + nsecs -= secs * NSEC_PER_SEC; + usecs = nsecs / NSEC_PER_USEC; + + ret = fprintf(debug_file(), "[%13" PRIu64 ".%06" PRIu64 "] ", secs, usecs); + ret += vfprintf(debug_file(), fmt, args); + return ret; +} + +int eprintf_time(int level, int var, u64 t, const char *fmt, ...) +{ + int ret = 0; + va_list args; + + if (var >= level) { + va_start(args, fmt); + ret = veprintf_time(t, fmt, args); + va_end(args); + } + + return ret; +} + +/* + * Overloading libtraceevent standard info print + * function, display with -v in perf. + */ +void pr_stat(const char *fmt, ...) +{ + va_list args; + + va_start(args, fmt); + veprintf(1, verbose, fmt, args); + va_end(args); + eprintf(1, verbose, "\n"); +} + +int dump_printf(const char *fmt, ...) +{ + va_list args; + int ret = 0; + + if (dump_trace) { + va_start(args, fmt); + ret = vprintf(fmt, args); + va_end(args); + } + + return ret; +} + +static int trace_event_printer(enum binary_printer_ops op, + unsigned int val, void *extra, FILE *fp) +{ + const char *color = PERF_COLOR_BLUE; + union perf_event *event = (union perf_event *)extra; + unsigned char ch = (unsigned char)val; + int printed = 0; + + switch (op) { + case BINARY_PRINT_DATA_BEGIN: + printed += fprintf(fp, "."); + printed += color_fprintf(fp, color, "\n. ... raw event: size %d bytes\n", + event->header.size); + break; + case BINARY_PRINT_LINE_BEGIN: + printed += fprintf(fp, "."); + break; + case BINARY_PRINT_ADDR: + printed += color_fprintf(fp, color, " %04x: ", val); + break; + case BINARY_PRINT_NUM_DATA: + printed += color_fprintf(fp, color, " %02x", val); + break; + case BINARY_PRINT_NUM_PAD: + printed += color_fprintf(fp, color, " "); + break; + case BINARY_PRINT_SEP: + printed += color_fprintf(fp, color, " "); + break; + case BINARY_PRINT_CHAR_DATA: + printed += color_fprintf(fp, color, "%c", + isprint(ch) && isascii(ch) ? ch : '.'); + break; + case BINARY_PRINT_CHAR_PAD: + printed += color_fprintf(fp, color, " "); + break; + case BINARY_PRINT_LINE_END: + printed += color_fprintf(fp, color, "\n"); + break; + case BINARY_PRINT_DATA_END: + printed += fprintf(fp, "\n"); + break; + default: + break; + } + + return printed; +} + +void trace_event(union perf_event *event) +{ + unsigned char *raw_event = (void *)event; + + if (!dump_trace) + return; + + print_binary(raw_event, event->header.size, 16, + trace_event_printer, event); +} + +static struct sublevel_option debug_opts[] = { + { .name = "verbose", .value_ptr = &verbose }, + { .name = "ordered-events", .value_ptr = &debug_ordered_events}, + { .name = "stderr", .value_ptr = &redirect_to_stderr}, + { .name = "data-convert", .value_ptr = &debug_data_convert }, + { .name = "perf-event-open", .value_ptr = &debug_peo_args }, + { .name = "kmaps", .value_ptr = &debug_kmaps }, + { .name = "type-profile", .value_ptr = &debug_type_profile }, + { .name = NULL, } +}; + +int perf_debug_option(const char *str) +{ + int ret; + + ret = perf_parse_sublevel_options(str, debug_opts); + if (ret) + return ret; + + /* Allow only verbose value in range (0, 10), otherwise set 0. */ + verbose = (verbose < 0) || (verbose > 10) ? 0 : verbose; + +#if LIBTRACEEVENT_VERSION >= MAKE_LIBTRACEEVENT_VERSION(1, 3, 0) + if (verbose == 1) + tep_set_loglevel(TEP_LOG_INFO); + else if (verbose == 2) + tep_set_loglevel(TEP_LOG_DEBUG); + else if (verbose >= 3) + tep_set_loglevel(TEP_LOG_ALL); +#endif + return 0; +} + +int perf_quiet_option(void) +{ + struct sublevel_option *opt = &debug_opts[0]; + + /* disable all debug messages */ + while (opt->name) { + *opt->value_ptr = -1; + opt++; + } + + /* For debug variables that are used as bool types, set to 0. */ + redirect_to_stderr = 0; + debug_peo_args = 0; + debug_kmaps = 0; + debug_type_profile = 0; + + return 0; +} + +#define DEBUG_WRAPPER(__n, __l) \ +static int pr_ ## __n ## _wrapper(const char *fmt, ...) \ +{ \ + va_list args; \ + int ret; \ + \ + va_start(args, fmt); \ + ret = veprintf(__l, verbose, fmt, args); \ + va_end(args); \ + return ret; \ +} + +DEBUG_WRAPPER(warning, 0); +DEBUG_WRAPPER(debug, 1); + +void perf_debug_setup(void) +{ + debug_set_file(stderr); + libapi_set_print(pr_warning_wrapper, pr_warning_wrapper, pr_debug_wrapper); +} + +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size) +{ + /* TODO: async safety. printf, malloc, etc. aren't safe inside a signal handler. */ + pid_t pid = getpid(); + struct machine *machine; + struct thread *thread = NULL; + struct perf_env host_env; + + perf_env__init(&host_env); + machine = machine__new_live(&host_env, /*kernel_maps=*/false, pid); + + if (machine) + thread = machine__find_thread(machine, pid, pid); + +#ifdef HAVE_BACKTRACE_SUPPORT + if (!machine || !thread) { + /* + * Backtrace functions are async signal safe. Fall back on them + * if machine/thread creation fails. + */ + backtrace_symbols_fd(stackdump, stackdump_size, fileno(file)); + machine__delete(machine); + perf_env__exit(&host_env); + return; + } +#endif + + for (size_t i = 0; i < stackdump_size; i++) { + struct addr_location al; + u64 addr = (u64)(uintptr_t)stackdump[i]; + bool printed = false; + + addr_location__init(&al); + if (thread && thread__find_map(thread, PERF_RECORD_MISC_USER, addr, &al)) { + al.sym = map__find_symbol(al.map, al.addr); + if (al.sym) { + fprintf(file, " #%zd %p in %s ", i, stackdump[i], al.sym->name); + printed = true; + } + } + if (!printed) + fprintf(file, " #%zd %p ", i, stackdump[i]); + + map__fprintf_srcline(al.map, al.addr, "", file); + fprintf(file, "\n"); + addr_location__exit(&al); + } + thread__put(thread); + machine__delete(machine); + perf_env__exit(&host_env); +} + +/* Obtain a backtrace and print it to stdout. */ +#ifdef HAVE_BACKTRACE_SUPPORT +void dump_stack(void) +{ + void *stackdump[32]; + size_t size = backtrace(stackdump, ARRAY_SIZE(stackdump)); + + __dump_stack(stdout, stackdump, size); +} +#else +void dump_stack(void) {} +#endif + +void sighandler_dump_stack(int sig) +{ + psignal(sig, "perf"); + dump_stack(); + signal(sig, SIG_DFL); + raise(sig); +} diff --git a/tools/perf/util/debug.h b/tools/perf/util/debug.h new file mode 100644 index 000000000..6b737e195 --- /dev/null +++ b/tools/perf/util/debug.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* For debugging general purposes */ +#ifndef __PERF_DEBUG_H +#define __PERF_DEBUG_H + +#include +#include +#include +#include + +extern int verbose; +extern int debug_kmaps; +extern int debug_peo_args; +extern bool quiet, dump_trace; +extern int debug_ordered_events; +extern int debug_data_convert; +extern int debug_type_profile; + +#ifndef pr_fmt +#define pr_fmt(fmt) fmt +#endif + +#define pr_err(fmt, ...) \ + eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warning(fmt, ...) \ + eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_warning_once(fmt, ...) ({ \ + static int __warned; \ + if (unlikely(!__warned)) { \ + pr_warning(fmt, ##__VA_ARGS__); \ + __warned = 1; \ + } \ +}) +#define pr_info(fmt, ...) \ + eprintf(0, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug(fmt, ...) \ + eprintf(1, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debugN(n, fmt, ...) \ + eprintf(n, verbose, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug2(fmt, ...) pr_debugN(2, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug3(fmt, ...) pr_debugN(3, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_debug4(fmt, ...) pr_debugN(4, pr_fmt(fmt), ##__VA_ARGS__) + +/* Special macro to print perf_event_open arguments/return value. */ +#define pr_debug2_peo(fmt, ...) { \ + if (debug_peo_args) \ + pr_debugN(0, pr_fmt(fmt), ##__VA_ARGS__); \ + else \ + pr_debugN(2, pr_fmt(fmt), ##__VA_ARGS__); \ +} + +#define pr_time_N(n, var, t, fmt, ...) \ + eprintf_time(n, var, t, fmt, ##__VA_ARGS__) + +#define pr_oe_time(t, fmt, ...) pr_time_N(1, debug_ordered_events, t, pr_fmt(fmt), ##__VA_ARGS__) +#define pr_oe_time2(t, fmt, ...) pr_time_N(2, debug_ordered_events, t, pr_fmt(fmt), ##__VA_ARGS__) + +#define STRERR_BUFSIZE 128 /* For the buffer size of str_error_r */ + +union perf_event; + +int dump_printf(const char *fmt, ...) __printf(1, 2); +void trace_event(union perf_event *event); + +int ui__error(const char *format, ...) __printf(1, 2); +int ui__warning(const char *format, ...) __printf(1, 2); +#define ui__warning_once(format, ...) ({ \ + static int __warned; \ + if (unlikely(!__warned)) { \ + ui__warning(format, ##__VA_ARGS__); \ + __warned = 1; \ + } \ +}) + +void pr_stat(const char *fmt, ...); + +int eprintf(int level, int var, const char *fmt, ...) __printf(3, 4); +int eprintf_time(int level, int var, u64 t, const char *fmt, ...) __printf(4, 5); +int veprintf(int level, int var, const char *fmt, va_list args); + +int perf_debug_option(const char *str); +FILE *debug_file(void); +void debug_set_file(FILE *file); +void debug_set_display_time(bool set); +void perf_debug_setup(void); +int perf_quiet_option(void); + +void __dump_stack(FILE *file, void **stackdump, size_t stackdump_size); +void dump_stack(void); +void sighandler_dump_stack(int sig); + +#endif /* __PERF_DEBUG_H */ diff --git a/tools/perf/util/debuginfo.c b/tools/perf/util/debuginfo.c new file mode 100644 index 000000000..84a78b30c --- /dev/null +++ b/tools/perf/util/debuginfo.c @@ -0,0 +1,215 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * DWARF debug information handling code. Copied from probe-finder.c. + * + * Written by Masami Hiramatsu + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "build-id.h" +#include "dso.h" +#include "debug.h" +#include "debuginfo.h" +#include "symbol.h" + +#ifdef HAVE_DEBUGINFOD_SUPPORT +#include +#endif + +/* Dwarf FL wrappers */ +static char *debuginfo_path; /* Currently dummy */ + +static const Dwfl_Callbacks offline_callbacks = { + .find_debuginfo = dwfl_standard_find_debuginfo, + .debuginfo_path = &debuginfo_path, + + .section_address = dwfl_offline_section_address, + + /* We use this table for core files too. */ + .find_elf = dwfl_build_id_find_elf, +}; + +/* Get a Dwarf from offline image */ +static int debuginfo__init_offline_dwarf(struct debuginfo *dbg, + const char *path) +{ + GElf_Addr dummy; + int fd; + bool fd_consumed = false; + + fd = open(path, O_RDONLY); + if (fd < 0) + return fd; + + dbg->dwfl = dwfl_begin(&offline_callbacks); + if (!dbg->dwfl) + goto error; + + dwfl_report_begin(dbg->dwfl); + dbg->mod = dwfl_report_offline(dbg->dwfl, "", "", fd); + if (!dbg->mod) + goto error; + fd_consumed = true; + + dbg->dbg = dwfl_module_getdwarf(dbg->mod, &dbg->bias); + if (!dbg->dbg) + goto error; + + dwfl_module_build_id(dbg->mod, &dbg->build_id, &dummy); + + if (dwfl_report_end(dbg->dwfl, NULL, NULL) != 0) + goto error; + + return 0; +error: + if (dbg->dwfl) + dwfl_end(dbg->dwfl); + if (!fd_consumed) + close(fd); + memset(dbg, 0, sizeof(*dbg)); + + return -ENOENT; +} + +static struct debuginfo *__debuginfo__new(const char *path) +{ + struct debuginfo *dbg = zalloc(sizeof(*dbg)); + if (!dbg) + return NULL; + + if (debuginfo__init_offline_dwarf(dbg, path) < 0) + zfree(&dbg); + if (dbg) + pr_debug("Open Debuginfo file: %s\n", path); + return dbg; +} + +struct debuginfo *debuginfo__new(const char *path) +{ + static const enum dso_binary_type distro_dwarf_types[] = { + DSO_BINARY_TYPE__FEDORA_DEBUGINFO, + DSO_BINARY_TYPE__UBUNTU_DEBUGINFO, + DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO, + DSO_BINARY_TYPE__BUILDID_DEBUGINFO, + DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO, + DSO_BINARY_TYPE__NOT_FOUND, + }; + const enum dso_binary_type *type; + char buf[PATH_MAX], nil = '\0'; + struct dso *dso; + struct debuginfo *dinfo = NULL; + struct build_id bid = { .size = 0}; + + /* Try to open distro debuginfo files */ + dso = dso__new(path); + if (!dso) + goto out; + + /* + * Set the build id for DSO_BINARY_TYPE__BUILDID_DEBUGINFO. Don't block + * incase the path isn't for a regular file. + */ + assert(!dso__has_build_id(dso)); + if (filename__read_build_id(path, &bid) > 0) + dso__set_build_id(dso, &bid); + + for (type = distro_dwarf_types; + !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND; + type++) { + if (dso__read_binary_type_filename(dso, *type, &nil, + buf, PATH_MAX) < 0) + continue; + dinfo = __debuginfo__new(buf); + } + dso__put(dso); + +out: + if (dinfo) + return dinfo; + + /* if failed to open all distro debuginfo, open given binary */ + symbol__join_symfs(buf, path); + return __debuginfo__new(buf); +} + +void debuginfo__delete(struct debuginfo *dbg) +{ + if (dbg) { + if (dbg->dwfl) + dwfl_end(dbg->dwfl); + free(dbg); + } +} + +/* For the kernel module, we need a special code to get a DIE */ +int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, + bool adjust_offset) +{ + int n, i; + Elf32_Word shndx; + Elf_Scn *scn; + Elf *elf; + GElf_Shdr mem, *shdr; + const char *p; + + elf = dwfl_module_getelf(dbg->mod, &dbg->bias); + if (!elf) + return -EINVAL; + + /* Get the number of relocations */ + n = dwfl_module_relocations(dbg->mod); + if (n < 0) + return -ENOENT; + /* Search the relocation related .text section */ + for (i = 0; i < n; i++) { + p = dwfl_module_relocation_info(dbg->mod, i, &shndx); + if (p && strcmp(p, ".text") == 0) { + /* OK, get the section header */ + scn = elf_getscn(elf, shndx); + if (!scn) + return -ENOENT; + shdr = gelf_getshdr(scn, &mem); + if (!shdr) + return -ENOENT; + *offs = shdr->sh_addr; + if (adjust_offset) + *offs -= shdr->sh_offset; + } + } + return 0; +} + +#ifdef HAVE_DEBUGINFOD_SUPPORT +int get_source_from_debuginfod(const char *raw_path, + const char *sbuild_id, char **new_path) +{ + debuginfod_client *c = debuginfod_begin(); + const char *p = raw_path; + int fd; + + if (!c) + return -ENOMEM; + + fd = debuginfod_find_source(c, (const unsigned char *)sbuild_id, + 0, p, new_path); + pr_debug("Search %s from debuginfod -> %d\n", p, fd); + if (fd >= 0) + close(fd); + debuginfod_end(c); + if (fd < 0) { + pr_debug("Failed to find %s in debuginfod (%s)\n", + raw_path, sbuild_id); + return -ENOENT; + } + pr_debug("Got a source %s\n", *new_path); + + return 0; +} +#endif /* HAVE_DEBUGINFOD_SUPPORT */ diff --git a/tools/perf/util/debuginfo.h b/tools/perf/util/debuginfo.h new file mode 100644 index 000000000..a52d69932 --- /dev/null +++ b/tools/perf/util/debuginfo.h @@ -0,0 +1,66 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _PERF_DEBUGINFO_H +#define _PERF_DEBUGINFO_H + +#include +#include + +#ifdef HAVE_LIBDW_SUPPORT + +#include "dwarf-aux.h" + +/* debug information structure */ +struct debuginfo { + Dwarf *dbg; + Dwfl_Module *mod; + Dwfl *dwfl; + Dwarf_Addr bias; + const unsigned char *build_id; +}; + +/* This also tries to open distro debuginfo */ +struct debuginfo *debuginfo__new(const char *path); +void debuginfo__delete(struct debuginfo *dbg); + +int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, + bool adjust_offset); + +#else /* HAVE_LIBDW_SUPPORT */ + +/* dummy debug information structure */ +struct debuginfo { +}; + +static inline struct debuginfo *debuginfo__new(const char *path __maybe_unused) +{ + return NULL; +} + +static inline void debuginfo__delete(struct debuginfo *dbg __maybe_unused) +{ +} + +typedef void Dwarf_Addr; + +static inline int debuginfo__get_text_offset(struct debuginfo *dbg __maybe_unused, + Dwarf_Addr *offs __maybe_unused, + bool adjust_offset __maybe_unused) +{ + return -EINVAL; +} + +#endif /* HAVE_LIBDW_SUPPORT */ + +#ifdef HAVE_DEBUGINFOD_SUPPORT +int get_source_from_debuginfod(const char *raw_path, const char *sbuild_id, + char **new_path); +#else /* HAVE_DEBUGINFOD_SUPPORT */ +static inline int get_source_from_debuginfod(const char *raw_path __maybe_unused, + const char *sbuild_id __maybe_unused, + char **new_path __maybe_unused) +{ + return -ENOTSUP; +} +#endif /* HAVE_DEBUGINFOD_SUPPORT */ + +#endif /* _PERF_DEBUGINFO_H */ diff --git a/tools/perf/util/demangle-cxx.cpp b/tools/perf/util/demangle-cxx.cpp new file mode 100644 index 000000000..85b706641 --- /dev/null +++ b/tools/perf/util/demangle-cxx.cpp @@ -0,0 +1,49 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "demangle-cxx.h" +#include +#include +#include + +#ifdef HAVE_LIBBFD_SUPPORT +#define PACKAGE 'perf' +#include +#endif + +#ifdef HAVE_CXA_DEMANGLE_SUPPORT +#include +#endif + +#if defined(HAVE_LIBBFD_SUPPORT) || defined(HAVE_CPLUS_DEMANGLE_SUPPORT) +#ifndef DMGL_PARAMS +#define DMGL_PARAMS (1 << 0) /* Include function args */ +#define DMGL_ANSI (1 << 1) /* Include const, volatile, etc */ +#endif +#endif + +/* + * Demangle C++ function signature + * + * Note: caller is responsible for freeing demangled string + */ +extern "C" +char *cxx_demangle_sym(const char *str, bool params __maybe_unused, + bool modifiers __maybe_unused) +{ +#ifdef HAVE_LIBBFD_SUPPORT + int flags = (params ? DMGL_PARAMS : 0) | (modifiers ? DMGL_ANSI : 0); + + return bfd_demangle(NULL, str, flags); +#elif defined(HAVE_CPLUS_DEMANGLE_SUPPORT) + int flags = (params ? DMGL_PARAMS : 0) | (modifiers ? DMGL_ANSI : 0); + + return cplus_demangle(str, flags); +#elif defined(HAVE_CXA_DEMANGLE_SUPPORT) + char *output; + int status; + + output = abi::__cxa_demangle(str, /*output_buffer=*/NULL, /*length=*/NULL, &status); + return output; +#else + return NULL; +#endif +} diff --git a/tools/perf/util/demangle-cxx.h b/tools/perf/util/demangle-cxx.h new file mode 100644 index 000000000..9359937a8 --- /dev/null +++ b/tools/perf/util/demangle-cxx.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DEMANGLE_CXX +#define __PERF_DEMANGLE_CXX 1 + +#include + +#ifdef __cplusplus +extern "C" { +#endif + +char *cxx_demangle_sym(const char *str, bool params, bool modifiers); + +#ifdef __cplusplus +} +#endif + + +#endif /* __PERF_DEMANGLE_CXX */ diff --git a/tools/perf/util/demangle-java.c b/tools/perf/util/demangle-java.c new file mode 100644 index 000000000..c3cb327ed --- /dev/null +++ b/tools/perf/util/demangle-java.c @@ -0,0 +1,203 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include "symbol.h" + +#include "demangle-java.h" + +#include +#include + +enum { + MODE_PREFIX = 0, + MODE_CLASS = 1, + MODE_FUNC = 2, + MODE_TYPE = 3, + MODE_CTYPE = 4, /* class arg */ +}; + +#define BASE_ENT(c, n) [c - 'A']=n +static const char *base_types['Z' - 'A' + 1] = { + BASE_ENT('B', "byte" ), + BASE_ENT('C', "char" ), + BASE_ENT('D', "double" ), + BASE_ENT('F', "float" ), + BASE_ENT('I', "int" ), + BASE_ENT('J', "long" ), + BASE_ENT('S', "short" ), + BASE_ENT('Z', "boolean" ), +}; + +/* + * demangle Java symbol between str and end positions and stores + * up to maxlen characters into buf. The parser starts in mode. + * + * Use MODE_PREFIX to process entire prototype till end position + * Use MODE_TYPE to process return type if str starts on return type char + * + * Return: + * success: buf + * error : NULL + */ +static char * +__demangle_java_sym(const char *str, const char *end, char *buf, int maxlen, int mode) +{ + int rlen = 0; + int array = 0; + int narg = 0; + const char *q; + + if (!end) + end = str + strlen(str); + + for (q = str; q != end; q++) { + + if (rlen == (maxlen - 1)) + break; + + switch (*q) { + case 'L': + if (mode == MODE_PREFIX || mode == MODE_TYPE) { + if (mode == MODE_TYPE) { + if (narg) + rlen += scnprintf(buf + rlen, maxlen - rlen, ", "); + narg++; + } + if (mode == MODE_PREFIX) + mode = MODE_CLASS; + else + mode = MODE_CTYPE; + } else + buf[rlen++] = *q; + break; + case 'B': + case 'C': + case 'D': + case 'F': + case 'I': + case 'J': + case 'S': + case 'Z': + if (mode == MODE_TYPE) { + if (narg) + rlen += scnprintf(buf + rlen, maxlen - rlen, ", "); + rlen += scnprintf(buf + rlen, maxlen - rlen, "%s", base_types[*q - 'A']); + while (array--) + rlen += scnprintf(buf + rlen, maxlen - rlen, "[]"); + array = 0; + narg++; + } else + buf[rlen++] = *q; + break; + case 'V': + if (mode == MODE_TYPE) { + rlen += scnprintf(buf + rlen, maxlen - rlen, "void"); + while (array--) + rlen += scnprintf(buf + rlen, maxlen - rlen, "[]"); + array = 0; + } else + buf[rlen++] = *q; + break; + case '[': + if (mode != MODE_TYPE) + goto error; + array++; + break; + case '(': + if (mode != MODE_FUNC) + goto error; + buf[rlen++] = *q; + mode = MODE_TYPE; + break; + case ')': + if (mode != MODE_TYPE) + goto error; + buf[rlen++] = *q; + narg = 0; + break; + case ';': + if (mode != MODE_CLASS && mode != MODE_CTYPE) + goto error; + /* safe because at least one other char to process */ + if (isalpha(*(q + 1)) && mode == MODE_CLASS) + rlen += scnprintf(buf + rlen, maxlen - rlen, "."); + if (mode == MODE_CLASS) + mode = MODE_FUNC; + else if (mode == MODE_CTYPE) + mode = MODE_TYPE; + break; + case '/': + if (mode != MODE_CLASS && mode != MODE_CTYPE) + goto error; + rlen += scnprintf(buf + rlen, maxlen - rlen, "."); + break; + default : + buf[rlen++] = *q; + } + } + buf[rlen] = '\0'; + return buf; +error: + return NULL; +} + +/* + * Demangle Java function signature (openJDK, not GCJ) + * input: + * str: string to parse. String is not modified + * flags: combination of JAVA_DEMANGLE_* flags to modify demangling + * return: + * if input can be demangled, then a newly allocated string is returned. + * if input cannot be demangled, then NULL is returned + * + * Note: caller is responsible for freeing demangled string + */ +char * +java_demangle_sym(const char *str, int flags) +{ + char *buf, *ptr; + const char *p; + size_t len, l1 = 0; + + if (!str) + return NULL; + + /* find start of return type */ + p = strrchr(str, ')'); + if (!p) + return NULL; + + /* + * expansion factor estimated to 3x + */ + len = strlen(str) * 3 + 1; + buf = malloc(len); + if (!buf) + return NULL; + + buf[0] = '\0'; + if (!(flags & JAVA_DEMANGLE_NORET)) { + /* + * get return type first + */ + ptr = __demangle_java_sym(p + 1, NULL, buf, len, MODE_TYPE); + if (!ptr) + goto error; + + /* add space between return type and function prototype */ + l1 = strlen(buf); + buf[l1++] = ' '; + } + + /* process function up to return type */ + ptr = __demangle_java_sym(str, p + 1, buf + l1, len - l1, MODE_PREFIX); + if (!ptr) + goto error; + + return buf; +error: + free(buf); + return NULL; +} diff --git a/tools/perf/util/demangle-java.h b/tools/perf/util/demangle-java.h new file mode 100644 index 000000000..f936c8eab --- /dev/null +++ b/tools/perf/util/demangle-java.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DEMANGLE_JAVA +#define __PERF_DEMANGLE_JAVA 1 +/* + * demangle function flags + */ +#define JAVA_DEMANGLE_NORET 0x1 /* do not process return type */ + +char * java_demangle_sym(const char *str, int flags); + +#endif /* __PERF_DEMANGLE_JAVA */ diff --git a/tools/perf/util/demangle-ocaml.c b/tools/perf/util/demangle-ocaml.c new file mode 100644 index 000000000..9d707bb60 --- /dev/null +++ b/tools/perf/util/demangle-ocaml.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "util/string2.h" + +#include "demangle-ocaml.h" + +#include + +static const char *caml_prefix = "caml"; +static const size_t caml_prefix_len = 4; + +/* mangled OCaml symbols start with "caml" followed by an upper-case letter */ +static bool +ocaml_is_mangled(const char *sym) +{ + return 0 == strncmp(sym, caml_prefix, caml_prefix_len) + && isupper(sym[caml_prefix_len]); +} + +/* + * input: + * sym: a symbol which may have been mangled by the OCaml compiler + * return: + * if the input doesn't look like a mangled OCaml symbol, NULL is returned + * otherwise, a newly allocated string containing the demangled symbol is returned + */ +char * +ocaml_demangle_sym(const char *sym) +{ + char *result; + int j = 0; + int i; + int len; + + if (!ocaml_is_mangled(sym)) { + return NULL; + } + + len = strlen(sym); + + /* the demangled symbol is always smaller than the mangled symbol */ + result = malloc(len + 1); + if (!result) + return NULL; + + /* skip "caml" prefix */ + i = caml_prefix_len; + + while (i < len) { + if (sym[i] == '_' && sym[i + 1] == '_') { + /* "__" -> "." */ + result[j++] = '.'; + i += 2; + } + else if (sym[i] == '$' && isxdigit(sym[i + 1]) && isxdigit(sym[i + 2])) { + /* "$xx" is a hex-encoded character */ + result[j++] = (hex(sym[i + 1]) << 4) | hex(sym[i + 2]); + i += 3; + } + else { + result[j++] = sym[i++]; + } + } + result[j] = '\0'; + + return result; +} diff --git a/tools/perf/util/demangle-ocaml.h b/tools/perf/util/demangle-ocaml.h new file mode 100644 index 000000000..843cc4fa1 --- /dev/null +++ b/tools/perf/util/demangle-ocaml.h @@ -0,0 +1,7 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DEMANGLE_OCAML +#define __PERF_DEMANGLE_OCAML 1 + +char * ocaml_demangle_sym(const char *str); + +#endif /* __PERF_DEMANGLE_OCAML */ diff --git a/tools/perf/util/demangle-rust-v0.c b/tools/perf/util/demangle-rust-v0.c new file mode 100644 index 000000000..19924d854 --- /dev/null +++ b/tools/perf/util/demangle-rust-v0.c @@ -0,0 +1,2042 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The contents of this file come from the Rust rustc-demangle library, hosted +// in the repository, licensed +// under "Apache-2.0 OR MIT". For copyright details, see +// . +// Please note that the file should be kept as close as possible to upstream. + +// Code for demangling Rust symbols. This code is mostly +// a line-by-line translation of the Rust code in `rustc-demangle`. + +// you can find the latest version of this code in https://github.com/rust-lang/rustc-demangle + +#include +#include +#include +#include +#include +#include + +#include "demangle-rust-v0.h" + +#if defined(__GNUC__) || defined(__clang__) +#define NODISCARD __attribute__((warn_unused_result)) +#else +#define NODISCARD +#endif + +#define MAX_DEPTH 500 + +typedef enum { + DemangleOk, + DemangleInvalid, + DemangleRecursed, + DemangleBug, +} demangle_status; + +struct demangle_v0 { + const char *mangled; + size_t mangled_len; +}; + +struct demangle_legacy { + const char *mangled; + size_t mangled_len; + size_t elements; +}; + +// private version of memrchr to avoid _GNU_SOURCE +static void *demangle_memrchr(const void *s, int c, size_t n) { + const uint8_t *s_ = s; + for (; n != 0; n--) { + if (s_[n-1] == c) { + return (void*)&s_[n-1]; + } + } + return NULL; +} + + +static bool unicode_iscontrol(uint32_t ch) { + // this is *technically* a unicode table, but + // some unicode properties are simpler than you might think + return ch < 0x20 || (ch >= 0x7f && ch < 0xa0); +} + +// "good enough" tables, the only consequence is that when printing +// *constant strings*, some characters are printed as `\u{abcd}` rather than themselves. +// +// I'm leaving these here to allow easily replacing them with actual +// tables if desired. +static bool unicode_isprint(uint32_t ch) { + if (ch < 0x20) { + return false; + } + if (ch < 0x7f) { + return true; + } + return false; +} + +static bool unicode_isgraphemextend(uint32_t ch) { + (void)ch; + return false; +} + +static bool str_isascii(const char *s, size_t s_len) { + for (size_t i = 0; i < s_len; i++) { + if (s[i] & 0x80) { + return false; + } + } + + return true; +} + +typedef enum { + PunycodeOk, + PunycodeError +} punycode_status; + +struct parser { + // the parser assumes that `sym` has a safe "terminating byte". It might be NUL, + // but it might also be something else if a symbol is "truncated". + const char *sym; + size_t sym_len; + size_t next; + uint32_t depth; +}; + +struct printer { + demangle_status status; // if status == 0 parser is valid + struct parser parser; + char *out; // NULL for no output [in which case out_len is not decremented] + size_t out_len; + uint32_t bound_lifetime_depth; + bool alternate; +}; + +static NODISCARD overflow_status printer_print_path(struct printer *printer, bool in_value); +static NODISCARD overflow_status printer_print_type(struct printer *printer); +static NODISCARD overflow_status printer_print_const(struct printer *printer, bool in_value); + +static NODISCARD demangle_status try_parse_path(struct parser *parser) { + struct printer printer = { + DemangleOk, + *parser, + NULL, + SIZE_MAX, + 0, + false + }; + overflow_status ignore = printer_print_path(&printer, false); // can't fail since no output + (void)ignore; + *parser = printer.parser; + return printer.status; +} + +NODISCARD static demangle_status rust_demangle_v0_demangle(const char *s, size_t s_len, struct demangle_v0 *res, const char **rest) { + if (s_len > strlen(s)) { + // s_len only exists to shorten the string, this is not a buffer API + return DemangleInvalid; + } + + const char *inner; + size_t inner_len; + if (s_len >= 2 && !strncmp(s, "_R", strlen("_R"))) { + inner = s+2; + inner_len = s_len - 2; + } else if (s_len >= 1 && !strncmp(s, "R", strlen("R"))) { + // On Windows, dbghelp strips leading underscores, so we accept "R..." + // form too. + inner = s+1; + inner_len = s_len - 1; + } else if (s_len >= 3 && !strncmp(s, "__R", strlen("__R"))) { + // On OSX, symbols are prefixed with an extra _ + inner = s+3; + inner_len = s_len - 3; + } else { + return DemangleInvalid; + } + + // Paths always start with uppercase characters. + if (*inner < 'A' || *inner > 'Z') { + return DemangleInvalid; + } + + if (!str_isascii(inner, inner_len)) { + return DemangleInvalid; + } + + struct parser parser = { inner, inner_len, 0, 0 }; + + demangle_status status = try_parse_path(&parser); + if (status != DemangleOk) return status; + char next = parser.sym[parser.next]; + + // Instantiating crate (paths always start with uppercase characters). + if (parser.next < parser.sym_len && next >= 'A' && next <= 'Z') { + status = try_parse_path(&parser); + if (status != DemangleOk) return status; + } + + res->mangled = inner; + res->mangled_len = inner_len; + if (rest) { + *rest = parser.sym + parser.next; + } + + return DemangleOk; +} + +// This might require `len` to be up to 3 characters bigger than the real output len in case of utf-8 +NODISCARD static overflow_status rust_demangle_v0_display_demangle(struct demangle_v0 res, char *out, size_t len, bool alternate) { + struct printer printer = { + DemangleOk, + { + res.mangled, + res.mangled_len, + 0, + 0 + }, + out, + len, + 0, + alternate + }; + if (printer_print_path(&printer, true) == OverflowOverflow) { + return OverflowOverflow; + } + if (printer.out_len < OVERFLOW_MARGIN) { + return OverflowOverflow; + } + *printer.out = '\0'; + return OverflowOk; +} + +static size_t code_to_utf8(unsigned char *buffer, uint32_t code) +{ + if (code <= 0x7F) { + buffer[0] = code; + return 1; + } + if (code <= 0x7FF) { + buffer[0] = 0xC0 | (code >> 6); /* 110xxxxx */ + buffer[1] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 2; + } + if (code <= 0xFFFF) { + buffer[0] = 0xE0 | (code >> 12); /* 1110xxxx */ + buffer[1] = 0x80 | ((code >> 6) & 0x3F); /* 10xxxxxx */ + buffer[2] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 3; + } + if (code <= 0x10FFFF) { + buffer[0] = 0xF0 | (code >> 18); /* 11110xxx */ + buffer[1] = 0x80 | ((code >> 12) & 0x3F); /* 10xxxxxx */ + buffer[2] = 0x80 | ((code >> 6) & 0x3F); /* 10xxxxxx */ + buffer[3] = 0x80 | (code & 0x3F); /* 10xxxxxx */ + return 4; + } + return 0; +} + + +// return length of char at byte, or SIZE_MAX if invalid. buf should have 4 valid characters +static NODISCARD size_t utf8_next_char(uint8_t *s, uint32_t *ch) { + uint8_t byte = *s; + // UTF8-1 = %x00-7F + // UTF8-2 = %xC2-DF UTF8-tail + // UTF8-3 = %xE0 %xA0-BF UTF8-tail / %xE1-EC 2( UTF8-tail ) / + // %xED %x80-9F UTF8-tail / %xEE-EF 2( UTF8-tail ) + // UTF8-4 = %xF0 %x90-BF 2( UTF8-tail ) / %xF1-F3 3( UTF8-tail ) / + // %xF4 %x80-8F 2( UTF8-tail ) + if (byte < 0x80) { + *ch = byte; + return 1; + } else if (byte < 0xc2) { + return SIZE_MAX; + } else if (byte < 0xe0) { + if (s[1] >= 0x80 && s[1] < 0xc0) { + *ch = ((byte&0x1f)<<6) + (s[1] & 0x3f); + return 2; + } + return SIZE_MAX; + } if (byte < 0xf0) { + if (!(s[1] >= 0x80 && s[1] < 0xc0) || !(s[2] >= 0x80 && s[2] < 0xc0)) { + return SIZE_MAX; // basic validation + } + if (byte == 0xe0 && s[1] < 0xa0) { + return SIZE_MAX; // overshort + } + if (byte == 0xed && s[1] >= 0xa0) { + return SIZE_MAX; // surrogate + } + *ch = ((byte&0x0f)<<12) + ((s[1] & 0x3f)<<6) + (s[2] & 0x3f); + return 3; + } else if (byte < 0xf5) { + if (!(s[1] >= 0x80 && s[1] < 0xc0) || !(s[2] >= 0x80 && s[2] < 0xc0) || !(s[3] >= 0x80 && s[3] < 0xc0)) { + return SIZE_MAX; // basic validation + } + if (byte == 0xf0 && s[1] < 0x90) { + return SIZE_MAX; // overshort + } + if (byte == 0xf4 && s[1] >= 0x90) { + return SIZE_MAX; // over max + } + *ch = ((byte&0x07)<<18) + ((s[1] & 0x3f)<<12) + ((s[2] & 0x3f)<<6) + (s[3]&0x3f); + return 4; + } else { + return SIZE_MAX; + } +} + +static NODISCARD bool validate_char(uint32_t n) { + return ((n ^ 0xd800) - 0x800) < 0x110000 - 0x800; +} + +#define SMALL_PUNYCODE_LEN 128 + +static NODISCARD punycode_status punycode_decode(const char *start, size_t ascii_len, const char *punycode_start, size_t punycode_len, uint32_t (*out_)[SMALL_PUNYCODE_LEN], size_t *out_len) { + uint32_t *out = *out_; + + if (punycode_len == 0) { + return PunycodeError; + } + + if (ascii_len > SMALL_PUNYCODE_LEN) { + return PunycodeError; + } + for (size_t i = 0; i < ascii_len; i++) { + out[i] = start[i]; + } + size_t len = ascii_len; + + size_t base = 36, t_min = 1, t_max = 26, skew = 38, damp = 700, bias = 72, i = 0, n = 0x80; + for (;;) { + size_t delta = 0, w = 1, k = 0; + for (;;) { + k += base; + size_t biased = k < bias ? 0 : k - bias; + size_t t = MIN(MAX(biased, t_min), t_max); + size_t d; + if (punycode_len == 0) { + return PunycodeError; + } + char nx = *punycode_start++; + punycode_len--; + if ('a' <= nx && nx <= 'z') { + d = nx - 'a'; + } else if ('0' <= nx && nx <= '9') { + d = 26 + (nx - '0'); + } else { + return PunycodeError; + } + if (w == 0 || d > SIZE_MAX / w || d*w > SIZE_MAX - delta) { + return PunycodeError; + } + delta += d * w; + if (d < t) { + break; + } + if (base < t || w == 0 || (base - t) > SIZE_MAX / w) { + return PunycodeError; + } + w *= (base - t); + } + + len += 1; + if (i > SIZE_MAX - delta) { + return PunycodeError; + } + i += delta; + if (n > SIZE_MAX - i / len) { + return PunycodeError; + } + n += i / len; + i %= len; + + // char validation + if (n > UINT32_MAX || !validate_char((uint32_t)n)) { + return PunycodeError; + } + + // insert new character + if (len > SMALL_PUNYCODE_LEN) { + return PunycodeError; + } + memmove(out + i + 1, out + i, (len - i - 1) * sizeof(uint32_t)); + out[i] = (uint32_t)n; + + // start i index at incremented position + i++; + + // If there are no more deltas, decoding is complete. + if (punycode_len == 0) { + *out_len = len; + return PunycodeOk; + } + + // Perform bias adaptation. + delta /= damp; + damp = 2; + + delta += delta / len; + k = 0; + while (delta > ((base - t_min) * t_max) / 2) { + delta /= base - t_min; + k += base; + } + bias = k + ((base - t_min + 1) * delta) / (delta + skew); + } +} + +struct ident { + const char *ascii_start; + size_t ascii_len; + const char *punycode_start; + size_t punycode_len; +}; + +static NODISCARD overflow_status display_ident(const char *ascii_start, size_t ascii_len, const char *punycode_start, size_t punycode_len, uint8_t *out, size_t *out_len) { + uint32_t outbuf[SMALL_PUNYCODE_LEN]; + + size_t wide_len; + size_t out_buflen = *out_len; + + if (punycode_len == 0) { + if (ascii_len > out_buflen) { + return OverflowOverflow; + } + memcpy(out, ascii_start, ascii_len); + *out_len = ascii_len; + } else if (punycode_decode(ascii_start, ascii_len, punycode_start, punycode_len, &outbuf, &wide_len) == PunycodeOk) { + size_t narrow_len = 0; + for (size_t i = 0; i < wide_len; i++) { + if (out_buflen - narrow_len < 4) { + return OverflowOverflow; + } + unsigned char *pos = &out[narrow_len]; + narrow_len += code_to_utf8(pos, outbuf[i]); + } + *out_len = narrow_len; + } else { + size_t narrow_len = 0; + if (out_buflen < strlen("punycode{")) { + return OverflowOverflow; + } + memcpy(out, "punycode{", strlen("punycode{")); + narrow_len = strlen("punycode{"); + if (ascii_len > 0) { + if (out_buflen - narrow_len < ascii_len || out_buflen - narrow_len - ascii_len < 1) { + return OverflowOverflow; + } + memcpy(out + narrow_len, ascii_start, ascii_len); + narrow_len += ascii_len; + out[narrow_len] = '-'; + narrow_len++; + } + if (out_buflen - narrow_len < punycode_len || out_buflen - narrow_len - punycode_len < 1) { + return OverflowOverflow; + } + memcpy(out + narrow_len, punycode_start, punycode_len); + narrow_len += punycode_len; + out[narrow_len] = '}'; + narrow_len++; + *out_len = narrow_len; + } + + return OverflowOk; +} + +static NODISCARD bool try_parse_uint(const char *buf, size_t len, uint64_t *result) { + size_t cur = 0; + for(;cur < len && buf[cur] == '0';cur++); + uint64_t result_val = 0; + if (len - cur > 16) return false; + for(;cur < len;cur++) { + char c = buf[cur]; + result_val <<= 4; + if ('0' <= c && c <= '9') { + result_val += c - '0'; + } else if ('a' <= c && c <= 'f') { + result_val += 10 + (c - 'a'); + } else { + return false; + } + } + *result = result_val; + return true; +} + +static NODISCARD bool dinibble2int(const char *buf, uint8_t *result) { + uint8_t result_val = 0; + for (int i = 0; i < 2; i++) { + char c = buf[i]; + result_val <<= 4; + if ('0' <= c && c <= '9') { + result_val += c - '0'; + } else if ('a' <= c && c <= 'f') { + result_val += 10 + (c - 'a'); + } else { + return false; + } + } + *result = result_val; + return true; +} + + +typedef enum { + NtsOk = 0, + NtsOverflow = 1, + NtsInvalid = 2 +} nibbles_to_string_status; + +// '\u{10ffff}', +margin +#define ESCAPED_SIZE 12 + +static NODISCARD size_t char_to_string(uint32_t ch, uint8_t quote, bool first, char (*buf)[ESCAPED_SIZE]) { + // encode the character + char *escaped_buf = *buf; + escaped_buf[0] = '\\'; + size_t escaped_len = 2; + switch (ch) { + case '\0': + escaped_buf[1] = '0'; + break; + case '\t': + escaped_buf[1] = 't'; + break; + case '\r': + escaped_buf[1] = 'r'; + break; + case '\n': + escaped_buf[1] = 'n'; + break; + case '\\': + escaped_buf[1] = '\\'; + break; + default: + if (ch == quote) { + escaped_buf[1] = ch; + } else if (!unicode_isprint(ch) || (first && unicode_isgraphemextend(ch))) { + int hexlen = snprintf(escaped_buf, ESCAPED_SIZE, "\\u{%x}", (unsigned int)ch); + if (hexlen < 0) { + return 0; // (snprintf shouldn't fail!) + } + escaped_len = hexlen; + } else { + // printable character + escaped_buf[0] = ch; + escaped_len = 1; + } + break; + } + + return escaped_len; +} + +// convert nibbles to a single/double-quoted string +static NODISCARD nibbles_to_string_status nibbles_to_string(const char *buf, size_t len, uint8_t *out, size_t *out_len) { + uint8_t quote = '"'; + bool first = true; + + if ((len % 2) != 0) { + return NtsInvalid; // odd number of nibbles + } + + size_t cur_out_len = 0; + + // write starting quote + if (out != NULL) { + cur_out_len = *out_len; + if (cur_out_len == 0) { + return NtsOverflow; + } + *out++ = quote; + cur_out_len--; + } + + uint8_t conv_buf[4] = {0}; + size_t conv_buf_len = 0; + while (len > 1 || conv_buf_len > 0) { + while (len > 1 && conv_buf_len < sizeof(conv_buf)) { + if (!dinibble2int(buf, &conv_buf[conv_buf_len])) { + return NtsInvalid; + } + conv_buf_len++; + buf += 2; + len -= 2; + } + + // conv_buf is full here if possible, process 1 UTF-8 character + uint32_t ch = 0; + size_t consumed = utf8_next_char(conv_buf, &ch); + if (consumed > conv_buf_len) { + // either SIZE_MAX (invalid UTF-8) or finished input buffer and + // there are still bytes remaining, in both cases invalid + return NtsInvalid; + } + + // "consume" the character + memmove(conv_buf, conv_buf+consumed, conv_buf_len-consumed); + conv_buf_len -= consumed; + + char escaped_buf[ESCAPED_SIZE]; + size_t escaped_len = char_to_string(ch, '"', first, &escaped_buf); + if (out != NULL) { + if (cur_out_len < escaped_len) { + return NtsOverflow; + } + memcpy(out, escaped_buf, escaped_len); + out += escaped_len; + cur_out_len -= escaped_len; + } + first = false; + } + + // write ending quote + if (out != NULL) { + if (cur_out_len == 0) { + return NtsOverflow; + } + *out++ = quote; + cur_out_len--; + *out_len -= cur_out_len; // subtract remaining space to get used space + } + + return NtsOk; +} + +static const char* basic_type(uint8_t tag) { + switch(tag) { + case 'b': + return "bool"; + case 'c': + return "char"; + case 'e': + return "str"; + case 'u': + return "()"; + case 'a': + return "i8"; + case 's': + return "i16"; + case 'l': + return "i32"; + case 'x': + return "i64"; + case 'n': + return "i128"; + case 'i': + return "isize"; + case 'h': + return "u8"; + case 't': + return "u16"; + case 'm': + return "u32"; + case 'y': + return "u64"; + case 'o': + return "u128"; + case 'j': + return "usize"; + case 'f': + return "f32"; + case 'd': + return "f64"; + case 'z': + return "!"; + case 'p': + return "_"; + case 'v': + return "..."; + default: + return NULL; + } +} + +static NODISCARD demangle_status parser_push_depth(struct parser *parser) { + parser->depth++; + if (parser->depth > MAX_DEPTH) { + return DemangleRecursed; + } else { + return DemangleOk; + } +} + +static demangle_status parser_pop_depth(struct parser *parser) { + parser->depth--; + return DemangleOk; +} + +static uint8_t parser_peek(struct parser const *parser) { + if (parser->next == parser->sym_len) { + return 0; // add a "pseudo nul terminator" to avoid peeking past the end of a symbol + } else { + return parser->sym[parser->next]; + } +} + +static bool parser_eat(struct parser *parser, uint8_t ch) { + if (parser_peek(parser) == ch) { + if (ch != 0) { // safety: make sure we don't skip past the NUL terminator + parser->next++; + } + return true; + } else { + return false; + } +} + +static uint8_t parser_next(struct parser *parser) { + // don't advance after end of input, and return an imaginary NUL terminator + if (parser->next == parser->sym_len) { + return 0; + } else { + return parser->sym[parser->next++]; + } +} + +static NODISCARD demangle_status parser_ch(struct parser *parser, uint8_t *next) { + // don't advance after end of input + if (parser->next == parser->sym_len) { + return DemangleInvalid; + } else { + *next = parser->sym[parser->next++]; + return DemangleOk; + } +} + +struct buf { + const char *start; + size_t len; +}; + +static NODISCARD demangle_status parser_hex_nibbles(struct parser *parser, struct buf *buf) { + size_t start = parser->next; + for (;;) { + uint8_t ch = parser_next(parser); + if (ch == '_') { + break; + } + if (!(('0' <= ch && ch <= '9') || ('a' <= ch && ch <= 'f'))) { + return DemangleInvalid; + } + } + buf->start = parser->sym + start; + buf->len = parser->next - start - 1; // skip final _ + return DemangleOk; +} + +static NODISCARD demangle_status parser_digit_10(struct parser *parser, uint8_t *out) { + uint8_t ch = parser_peek(parser); + if ('0' <= ch && ch <= '9') { + *out = ch - '0'; + parser->next++; + return DemangleOk; + } else { + return DemangleInvalid; + } +} + +static NODISCARD demangle_status parser_digit_62(struct parser *parser, uint64_t *out) { + uint8_t ch = parser_peek(parser); + if ('0' <= ch && ch <= '9') { + *out = ch - '0'; + parser->next++; + return DemangleOk; + } else if ('a' <= ch && ch <= 'z') { + *out = 10 + (ch - 'a'); + parser->next++; + return DemangleOk; + } else if ('A' <= ch && ch <= 'Z') { + *out = 10 + 26 + (ch - 'A'); + parser->next++; + return DemangleOk; + } else { + return DemangleInvalid; + } +} + +static NODISCARD demangle_status parser_integer_62(struct parser *parser, uint64_t *out) { + if (parser_eat(parser, '_')) { + *out = 0; + return DemangleOk; + } + + uint64_t x = 0; + demangle_status status; + while (!parser_eat(parser, '_')) { + uint64_t d; + if ((status = parser_digit_62(parser, &d)) != DemangleOk) { + return status; + } + if (x > UINT64_MAX / 62) { + return DemangleInvalid; + } + x *= 62; + if (x > UINT64_MAX - d) { + return DemangleInvalid; + } + x += d; + } + if (x == UINT64_MAX) { + return DemangleInvalid; + } + *out = x + 1; + return DemangleOk; +} + +static NODISCARD demangle_status parser_opt_integer_62(struct parser *parser, uint8_t tag, uint64_t *out) { + if (!parser_eat(parser, tag)) { + *out = 0; + return DemangleOk; + } + + demangle_status status; + if ((status = parser_integer_62(parser, out)) != DemangleOk) { + return status; + } + if (*out == UINT64_MAX) { + return DemangleInvalid; + } + *out = *out + 1; + return DemangleOk; +} + +static NODISCARD demangle_status parser_disambiguator(struct parser *parser, uint64_t *out) { + return parser_opt_integer_62(parser, 's', out); +} + +typedef uint8_t parser_namespace_type; + +static NODISCARD demangle_status parser_namespace(struct parser *parser, parser_namespace_type *out) { + uint8_t next = parser_next(parser); + if ('A' <= next && next <= 'Z') { + *out = next; + return DemangleOk; + } else if ('a' <= next && next <= 'z') { + *out = 0; + return DemangleOk; + } else { + return DemangleInvalid; + } +} + +static NODISCARD demangle_status parser_backref(struct parser *parser, struct parser *out) { + size_t start = parser->next; + if (start == 0) { + return DemangleBug; + } + size_t s_start = start - 1; + uint64_t i; + demangle_status status = parser_integer_62(parser, &i); + if (status != DemangleOk) { + return status; + } + if (i >= s_start) { + return DemangleInvalid; + } + struct parser res = { + .sym = parser->sym, + .sym_len = parser->sym_len, + .next = (size_t)i, + .depth = parser->depth + }; + status = parser_push_depth(&res); + if (status != DemangleOk) { + return status; + } + *out = res; + return DemangleOk; +} + +static NODISCARD demangle_status parser_ident(struct parser *parser, struct ident *out) { + bool is_punycode = parser_eat(parser, 'u'); + size_t len; + uint8_t d; + demangle_status status = parser_digit_10(parser, &d); + len = d; + if (status != DemangleOk) { + return status; + } + if (len) { + for (;;) { + status = parser_digit_10(parser, &d); + if (status != DemangleOk) { + break; + } + if (len > SIZE_MAX / 10) { + return DemangleInvalid; + } + len *= 10; + if (len > SIZE_MAX - d) { + return DemangleInvalid; + } + len += d; + } + } + + // Skip past the optional `_` separator. + parser_eat(parser, '_'); + + size_t start = parser->next; + if (parser->sym_len - parser->next < len) { + return DemangleInvalid; + } + parser->next += len; + + const char *ident = &parser->sym[start]; + + if (is_punycode) { + const char *underscore = demangle_memrchr(ident, '_', (size_t)len); + if (underscore == NULL) { + *out = (struct ident){ + .ascii_start="", + .ascii_len=0, + .punycode_start=ident, + .punycode_len=len + }; + } else { + size_t ascii_len = underscore - ident; + // ascii_len <= len - 1 since `_` is in the first len bytes + size_t punycode_len = len - 1 - ascii_len; + *out = (struct ident){ + .ascii_start=ident, + .ascii_len=ascii_len, + .punycode_start=underscore + 1, + .punycode_len=punycode_len + }; + } + if (out->punycode_len == 0) { + return DemangleInvalid; + } + return DemangleOk; + } else { + *out = (struct ident) { + .ascii_start=ident, + .ascii_len=(size_t)len, + .punycode_start="", + .punycode_len=0, + }; + return DemangleOk; + } +} + +#define INVALID_SYNTAX "{invalid syntax}" + +static const char *demangle_error_message(demangle_status status) { + switch (status) { + case DemangleInvalid: + return INVALID_SYNTAX; + case DemangleBug: + return "{bug}"; + case DemangleRecursed: + return "{recursion limit reached}"; + default: + return "{unknown error}"; + } +} + +#define PRINT(print_fn) \ + do { \ + if ((print_fn) == OverflowOverflow) { \ + return OverflowOverflow; \ + } \ + } while(0) + +#define PRINT_CH(printer, s) PRINT(printer_print_ch((printer), (s))) +#define PRINT_STR(printer, s) PRINT(printer_print_str((printer), (s))) +#define PRINT_U64(printer, s) PRINT(printer_print_u64((printer), (s))) +#define PRINT_IDENT(printer, s) PRINT(printer_print_ident((printer), (s))) + +#define INVALID(printer) \ + do { \ + PRINT_STR((printer), INVALID_SYNTAX); \ + (printer)->status = DemangleInvalid; \ + return OverflowOk; \ + } while(0) + +#define PARSE(printer, method, ...) \ + do { \ + if ((printer)->status != DemangleOk) { \ + PRINT_STR((printer), "?"); \ + return OverflowOk; \ + } else { \ + demangle_status _parse_status = method(&(printer)->parser, ## __VA_ARGS__); \ + if (_parse_status != DemangleOk) { \ + PRINT_STR((printer), demangle_error_message(_parse_status)); \ + (printer)->status = _parse_status; \ + return OverflowOk; \ + } \ + } \ + } while(0) + +#define PRINT_SEP_LIST(printer, body, sep) \ + do { \ + size_t _sep_list_i; \ + PRINT_SEP_LIST_COUNT(printer, _sep_list_i, body, sep); \ + } while(0) + +#define PRINT_SEP_LIST_COUNT(printer, count, body, sep) \ + do { \ + count = 0; \ + while ((printer)->status == DemangleOk && !printer_eat((printer), 'E')) { \ + if (count > 0) { PRINT_STR(printer, sep); } \ + body; \ + count++; \ + } \ + } while(0) + +static bool printer_eat(struct printer *printer, uint8_t b) { + if (printer->status != DemangleOk) { + return false; + } + + return parser_eat(&printer->parser, b); +} + +static void printer_pop_depth(struct printer *printer) { + if (printer->status == DemangleOk) { + parser_pop_depth(&printer->parser); + } +} + +static NODISCARD overflow_status printer_print_buf(struct printer *printer, const char *start, size_t len) { + if (printer->out == NULL) { + return OverflowOk; + } + if (printer->out_len < len) { + return OverflowOverflow; + } + + memcpy(printer->out, start, len); + printer->out += len; + printer->out_len -= len; + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_str(struct printer *printer, const char *buf) { + return printer_print_buf(printer, buf, strlen(buf)); +} + +static NODISCARD overflow_status printer_print_ch(struct printer *printer, char ch) { + return printer_print_buf(printer, &ch, 1); +} + +static NODISCARD overflow_status printer_print_u64(struct printer *printer, uint64_t n) { + char buf[32] = {0}; + sprintf(buf, "%llu", (unsigned long long)n); // printing uint64 uses 21 < 32 chars + return printer_print_str(printer, buf); +} + +static NODISCARD overflow_status printer_print_ident(struct printer *printer, struct ident *ident) { + if (printer->out == NULL) { + return OverflowOk; + } + + size_t out_len = printer->out_len; + overflow_status status; + if ((status = display_ident(ident->ascii_start, ident->ascii_len, ident->punycode_start, ident->punycode_len, (uint8_t*)printer->out, &out_len)) != OverflowOk) { + return status; + } + printer->out += out_len; + printer->out_len -= out_len; + return OverflowOk; +} + +typedef overflow_status (*printer_fn)(struct printer *printer); +typedef overflow_status (*backref_fn)(struct printer *printer, bool *arg); + +static NODISCARD overflow_status printer_print_backref(struct printer *printer, backref_fn func, bool *arg) { + struct parser backref; + PARSE(printer, parser_backref, &backref); + + if (printer->out == NULL) { + return OverflowOk; + } + + struct parser orig_parser = printer->parser; + demangle_status orig_status = printer->status; // fixme not sure this is needed match for Ok on the Rust side + printer->parser = backref; + printer->status = DemangleOk; + overflow_status status = func(printer, arg); + printer->parser = orig_parser; + printer->status = orig_status; + + return status; +} + +static NODISCARD overflow_status printer_print_lifetime_from_index(struct printer *printer, uint64_t lt) { + // Bound lifetimes aren't tracked when skipping printing. + if (printer->out == NULL) { + return OverflowOk; + } + + PRINT_STR(printer, "'"); + if (lt == 0) { + PRINT_STR(printer, "_"); + return OverflowOk; + } + + if (printer->bound_lifetime_depth < lt) { + INVALID(printer); + } else { + uint64_t depth = printer->bound_lifetime_depth - lt; + if (depth < 26) { + PRINT_CH(printer, 'a' + depth); + } else { + PRINT_STR(printer, "_"); + PRINT_U64(printer, depth); + } + + return OverflowOk; + } +} + +static NODISCARD overflow_status printer_in_binder(struct printer *printer, printer_fn func) { + uint64_t bound_lifetimes; + PARSE(printer, parser_opt_integer_62, 'G', &bound_lifetimes); + + // Don't track bound lifetimes when skipping printing. + if (printer->out == NULL) { + return func(printer); + } + + if (bound_lifetimes > 0) { + PRINT_STR(printer, "for<"); + for (uint64_t i = 0; i < bound_lifetimes; i++) { + if (i > 0) { + PRINT_STR(printer, ", "); + } + printer->bound_lifetime_depth++; + PRINT(printer_print_lifetime_from_index(printer, 1)); + } + PRINT_STR(printer, "> "); + } + + overflow_status r = func(printer); + printer->bound_lifetime_depth -= bound_lifetimes; + + return r; +} + +static NODISCARD overflow_status printer_print_generic_arg(struct printer *printer) { + if (printer_eat(printer, 'L')) { + uint64_t lt; + PARSE(printer, parser_integer_62, <); + return printer_print_lifetime_from_index(printer, lt); + } else if (printer_eat(printer, 'K')) { + return printer_print_const(printer, false); + } else { + return printer_print_type(printer); + } +} + +static NODISCARD overflow_status printer_print_generic_args(struct printer *printer) { + PRINT_STR(printer, "<"); + PRINT_SEP_LIST(printer, PRINT(printer_print_generic_arg(printer)), ", "); + PRINT_STR(printer, ">"); + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_path_out_of_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_path(printer, false); +} + +static NODISCARD overflow_status printer_print_path_in_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_path(printer, true); +} + +static NODISCARD overflow_status printer_print_path(struct printer *printer, bool in_value) { + PARSE(printer, parser_push_depth); + uint8_t tag; + PARSE(printer, parser_ch, &tag); + + overflow_status st; + uint64_t dis; + struct ident name; + parser_namespace_type ns; + char *orig_out; + + switch(tag) { + case 'C': + PARSE(printer, parser_disambiguator, &dis); + PARSE(printer, parser_ident, &name); + + PRINT_IDENT(printer, &name); + + if (printer->out != NULL && !printer->alternate && dis != 0) { + PRINT_STR(printer, "["); + char buf[24] = {0}; + sprintf(buf, "%llx", (unsigned long long)dis); + PRINT_STR(printer, buf); + PRINT_STR(printer, "]"); + } + break; + case 'N': + PARSE(printer, parser_namespace, &ns); + if ((st = printer_print_path(printer, in_value)) != OverflowOk) { + return st; + } + + // HACK(eddyb) if the parser is already marked as having errored, + // `parse!` below will print a `?` without its preceding `::` + // (because printing the `::` is skipped in certain conditions, + // i.e. a lowercase namespace with an empty identifier), + // so in order to get `::?`, the `::` has to be printed here. + if (printer->status != DemangleOk) { + PRINT_STR(printer, "::"); + } + + PARSE(printer, parser_disambiguator, &dis); + PARSE(printer, parser_ident, &name); + // Special namespace, like closures and shims + if (ns) { + PRINT_STR(printer, "::{"); + if (ns == 'C') { + PRINT_STR(printer, "closure"); + } else if (ns == 'S') { + PRINT_STR(printer, "shim"); + } else { + PRINT_CH(printer, ns); + } + if (name.ascii_len != 0 || name.punycode_len != 0) { + PRINT_STR(printer, ":"); + PRINT_IDENT(printer, &name); + } + PRINT_STR(printer, "#"); + PRINT_U64(printer, dis); + PRINT_STR(printer, "}"); + } else { + // Implementation-specific/unspecified namespaces + if (name.ascii_len != 0 || name.punycode_len != 0) { + PRINT_STR(printer, "::"); + PRINT_IDENT(printer, &name); + } + } + break; + case 'M': + case 'X': + // for impls, ignore the impls own path + PARSE(printer, parser_disambiguator, &dis); + orig_out = printer->out; + printer->out = NULL; + PRINT(printer_print_path(printer, false)); + printer->out = orig_out; + + // fallthru + case 'Y': + PRINT_STR(printer, "<"); + PRINT(printer_print_type(printer)); + if (tag != 'M') { + PRINT_STR(printer, " as "); + PRINT(printer_print_path(printer, false)); + } + PRINT_STR(printer, ">"); + break; + case 'I': + PRINT(printer_print_path(printer, in_value)); + if (in_value) { + PRINT_STR(printer, "::"); + } + PRINT(printer_print_generic_args(printer)); + break; + case 'B': + PRINT(printer_print_backref(printer, in_value ? printer_print_path_in_value : printer_print_path_out_of_value, NULL)); + break; + default: + INVALID(printer); + break; + } + + printer_pop_depth(printer); + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_const_uint(struct printer *printer, uint8_t tag) { + struct buf hex; + PARSE(printer, parser_hex_nibbles, &hex); + + uint64_t val; + if (try_parse_uint(hex.start, hex.len, &val)) { + PRINT_U64(printer, val); + } else { + PRINT_STR(printer, "0x"); + PRINT(printer_print_buf(printer, hex.start, hex.len)); + } + + if (printer->out != NULL && !printer->alternate) { + const char *ty = basic_type(tag); + if (/* safety */ ty != NULL) { + PRINT_STR(printer, ty); + } + } + + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_const_str_literal(struct printer *printer) { + struct buf hex; + PARSE(printer, parser_hex_nibbles, &hex); + + size_t out_len = SIZE_MAX; + nibbles_to_string_status nts_status = nibbles_to_string(hex.start, hex.len, NULL, &out_len); + switch (nts_status) { + case NtsOk: + if (printer->out != NULL) { + out_len = printer->out_len; + nts_status = nibbles_to_string(hex.start, hex.len, (uint8_t*)printer->out, &out_len); + if (nts_status != NtsOk) { + return OverflowOverflow; + } + printer->out += out_len; + printer->out_len -= out_len; + } + return OverflowOk; + case NtsOverflow: + // technically if there is a string of size `SIZE_MAX/6` whose escaped version overflows + // SIZE_MAX but has an invalid char, this will be a "fake" overflow. In practice, + // that is not going to happen and a fuzzer will not generate strings of this length. + return OverflowOverflow; + case NtsInvalid: + default: + INVALID(printer); + } +} + +static NODISCARD overflow_status printer_print_const_struct(struct printer *printer) { + uint64_t dis; + struct ident name; + PARSE(printer, parser_disambiguator, &dis); + PARSE(printer, parser_ident, &name); + PRINT_IDENT(printer, &name); + PRINT_STR(printer, ": "); + return printer_print_const(printer, true); +} + +static NODISCARD overflow_status printer_print_const_out_of_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_const(printer, false); +} + +static NODISCARD overflow_status printer_print_const_in_value(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_const(printer, true); +} + +static NODISCARD overflow_status printer_print_const(struct printer *printer, bool in_value) { + uint8_t tag; + + PARSE(printer, parser_ch, &tag); + PARSE(printer, parser_push_depth); + + struct buf hex; + uint64_t val; + size_t count; + + bool opened_brace = false; +#define OPEN_BRACE_IF_OUTSIDE_EXPR \ + do { if (!in_value) { \ + opened_brace = true; \ + PRINT_STR(printer, "{"); \ + } } while(0) + + switch(tag) { + case 'p': + PRINT_STR(printer, "_"); + break; + // Primitive leaves with hex-encoded values (see `basic_type`). + case 'a': + case 's': + case 'l': + case 'x': + case 'n': + case 'i': + if (printer_eat(printer, 'n')) { + PRINT_STR(printer, "-"); + } + /* fallthrough */ + case 'h': + case 't': + case 'm': + case 'y': + case 'o': + case 'j': + PRINT(printer_print_const_uint(printer, tag)); + break; + case 'b': + PARSE(printer, parser_hex_nibbles, &hex); + if (try_parse_uint(hex.start, hex.len, &val)) { + if (val == 0) { + PRINT_STR(printer, "false"); + } else if (val == 1) { + PRINT_STR(printer, "true"); + } else { + INVALID(printer); + } + } else { + INVALID(printer); + } + break; + case 'c': + PARSE(printer, parser_hex_nibbles, &hex); + if (try_parse_uint(hex.start, hex.len, &val) + && val < UINT32_MAX + && validate_char((uint32_t)val)) + { + char escaped_buf[ESCAPED_SIZE]; + size_t escaped_size = char_to_string((uint32_t)val, '\'', true, &escaped_buf); + + PRINT_STR(printer, "'"); + PRINT(printer_print_buf(printer, escaped_buf, escaped_size)); + PRINT_STR(printer, "'"); + } else { + INVALID(printer); + } + break; + case 'e': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "*"); + PRINT(printer_print_const_str_literal(printer)); + break; + case 'R': + case 'Q': + if (tag == 'R' && printer_eat(printer, 'e')) { + PRINT(printer_print_const_str_literal(printer)); + } else { + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "&"); + if (tag != 'R') { + PRINT_STR(printer, "mut "); + } + PRINT(printer_print_const(printer, true)); + } + break; + case 'A': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "["); + PRINT_SEP_LIST(printer, PRINT(printer_print_const(printer, true)), ", "); + PRINT_STR(printer, "]"); + break; + case 'T': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT_STR(printer, "("); + PRINT_SEP_LIST_COUNT(printer, count, PRINT(printer_print_const(printer, true)), ", "); + if (count == 1) { + PRINT_STR(printer, ","); + } + PRINT_STR(printer, ")"); + break; + case 'V': + OPEN_BRACE_IF_OUTSIDE_EXPR; + PRINT(printer_print_path(printer, true)); + PARSE(printer, parser_ch, &tag); + switch(tag) { + case 'U': + break; + case 'T': + PRINT_STR(printer, "("); + PRINT_SEP_LIST(printer, PRINT(printer_print_const(printer, true)), ", "); + PRINT_STR(printer, ")"); + break; + case 'S': + PRINT_STR(printer, " { "); + PRINT_SEP_LIST(printer, PRINT(printer_print_const_struct(printer)), ", "); + PRINT_STR(printer, " }"); + break; + default: + INVALID(printer); + } + break; + case 'B': + PRINT(printer_print_backref(printer, in_value ? printer_print_const_in_value : printer_print_const_out_of_value, NULL)); + break; + default: + INVALID(printer); + } +#undef OPEN_BRACE_IF_OUTSIDE_EXPR + + if (opened_brace) { + PRINT_STR(printer, "}"); + } + printer_pop_depth(printer); + + return OverflowOk; +} + +/// A trait in a trait object may have some "existential projections" +/// (i.e. associated type bindings) after it, which should be printed +/// in the `<...>` of the trait, e.g. `dyn Trait`. +/// To this end, this method will keep the `<...>` of an 'I' path +/// open, by omitting the `>`, and return `Ok(true)` in that case. +static NODISCARD overflow_status printer_print_maybe_open_generics(struct printer *printer, bool *open) { + if (printer_eat(printer, 'B')) { + // NOTE(eddyb) the closure may not run if printing is being skipped, + // but in that case the returned boolean doesn't matter. + *open = false; + return printer_print_backref(printer, printer_print_maybe_open_generics, open); + } else if(printer_eat(printer, 'I')) { + PRINT(printer_print_path(printer, false)); + PRINT_STR(printer, "<"); + PRINT_SEP_LIST(printer, PRINT(printer_print_generic_arg(printer)), ", "); + *open = true; + return OverflowOk; + } else { + PRINT(printer_print_path(printer, false)); + *open = false; + return OverflowOk; + } +} + +static NODISCARD overflow_status printer_print_dyn_trait(struct printer *printer) { + bool open; + PRINT(printer_print_maybe_open_generics(printer, &open)); + + while (printer_eat(printer, 'p')) { + if (!open) { + PRINT_STR(printer, "<"); + open = true; + } else { + PRINT_STR(printer, ", "); + } + + struct ident name; + PARSE(printer, parser_ident, &name); + + PRINT_IDENT(printer, &name); + PRINT_STR(printer, " = "); + PRINT(printer_print_type(printer)); + } + + if (open) { + PRINT_STR(printer, ">"); + } + + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_object_bounds(struct printer *printer) { + PRINT_SEP_LIST(printer, PRINT(printer_print_dyn_trait(printer)), " + "); + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_function_type(struct printer *printer) { + bool is_unsafe = printer_eat(printer, 'U'); + const char *abi; + size_t abi_len; + if (printer_eat(printer, 'K')) { + if (printer_eat(printer, 'C')) { + abi = "C"; + abi_len = 1; + } else { + struct ident abi_ident; + PARSE(printer, parser_ident, &abi_ident); + if (abi_ident.ascii_len == 0 || abi_ident.punycode_len != 0) { + INVALID(printer); + } + abi = abi_ident.ascii_start; + abi_len = abi_ident.ascii_len; + } + } else { + abi = NULL; + abi_len = 0; + } + + if (is_unsafe) { + PRINT_STR(printer, "unsafe "); + } + + if (abi != NULL) { + PRINT_STR(printer, "extern \""); + + // replace _ with - + while (abi_len > 0) { + const char *minus = memchr(abi, '_', abi_len); + if (minus == NULL) { + PRINT(printer_print_buf(printer, (const char*)abi, abi_len)); + break; + } else { + size_t space_to_minus = minus - abi; + PRINT(printer_print_buf(printer, (const char*)abi, space_to_minus)); + PRINT_STR(printer, "-"); + abi = minus + 1; + abi_len -= (space_to_minus + 1); + } + } + + PRINT_STR(printer, "\" "); + } + + PRINT_STR(printer, "fn("); + PRINT_SEP_LIST(printer, PRINT(printer_print_type(printer)), ", "); + PRINT_STR(printer, ")"); + + if (printer_eat(printer, 'u')) { + // Skip printing the return type if it's 'u', i.e. `()`. + } else { + PRINT_STR(printer, " -> "); + PRINT(printer_print_type(printer)); + } + + return OverflowOk; +} + +static NODISCARD overflow_status printer_print_type_backref(struct printer *printer, bool *_arg) { + (void)_arg; + return printer_print_type(printer); +} + +static NODISCARD overflow_status printer_print_type(struct printer *printer) { + uint8_t tag; + PARSE(printer, parser_ch, &tag); + + const char *basic_ty = basic_type(tag); + if (basic_ty) { + return printer_print_str(printer, basic_ty); + } + + uint64_t count; + uint64_t lt; + + PARSE(printer, parser_push_depth); + switch (tag) { + case 'R': + case 'Q': + PRINT_STR(printer, "&"); + if (printer_eat(printer, 'L')) { + PARSE(printer, parser_integer_62, <); + if (lt != 0) { + PRINT(printer_print_lifetime_from_index(printer, lt)); + PRINT_STR(printer, " "); + } + } + if (tag != 'R') { + PRINT_STR(printer, "mut "); + } + PRINT(printer_print_type(printer)); + break; + case 'P': + case 'O': + PRINT_STR(printer, "*"); + if (tag != 'P') { + PRINT_STR(printer, "mut "); + } else { + PRINT_STR(printer, "const "); + } + PRINT(printer_print_type(printer)); + break; + case 'A': + case 'S': + PRINT_STR(printer, "["); + PRINT(printer_print_type(printer)); + if (tag == 'A') { + PRINT_STR(printer, "; "); + PRINT(printer_print_const(printer, true)); + } + PRINT_STR(printer, "]"); + break; + case 'T': + PRINT_STR(printer, "("); + PRINT_SEP_LIST_COUNT(printer, count, PRINT(printer_print_type(printer)), ", "); + if (count == 1) { + PRINT_STR(printer, ","); + } + PRINT_STR(printer, ")"); + break; + case 'F': + PRINT(printer_in_binder(printer, printer_print_function_type)); + break; + case 'D': + PRINT_STR(printer, "dyn "); + PRINT(printer_in_binder(printer, printer_print_object_bounds)); + + if (!printer_eat(printer, 'L')) { + INVALID(printer); + } + PARSE(printer, parser_integer_62, <); + + if (lt != 0) { + PRINT_STR(printer, " + "); + PRINT(printer_print_lifetime_from_index(printer, lt)); + } + break; + case 'B': + PRINT(printer_print_backref(printer, printer_print_type_backref, NULL)); + break; + default: + // Go back to the tag, so `print_path` also sees it. + if (printer->status == DemangleOk && /* safety */ printer->parser.next > 0) { + printer->parser.next--; + } + PRINT(printer_print_path(printer, false)); + } + + printer_pop_depth(printer); + return OverflowOk; +} + +NODISCARD static demangle_status rust_demangle_legacy_demangle(const char *s, size_t s_len, struct demangle_legacy *res, const char **rest) +{ + if (s_len > strlen(s)) { + // s_len only exists to shorten the string, this is not a buffer API + return DemangleInvalid; + } + + const char *inner; + size_t inner_len; + if (s_len >= 3 && !strncmp(s, "_ZN", 3)) { + inner = s + 3; + inner_len = s_len - 3; + } else if (s_len >= 2 && !strncmp(s, "ZN", 2)) { + // On Windows, dbghelp strips leading underscores, so we accept "ZN...E" + // form too. + inner = s + 2; + inner_len = s_len - 2; + } else if (s_len >= 4 && !strncmp(s, "__ZN", 4)) { + // On OSX, symbols are prefixed with an extra _ + inner = s + 4; + inner_len = s_len - 4; + } else { + return DemangleInvalid; + } + + if (!str_isascii(inner, inner_len)) { + return DemangleInvalid; + } + + size_t elements = 0; + const char *chars = inner; + size_t chars_len = inner_len; + if (chars_len == 0) { + return DemangleInvalid; + } + char c; + while ((c = *chars) != 'E') { + // Decode an identifier element's length + if (c < '0' || c > '9') { + return DemangleInvalid; + } + size_t len = 0; + while (c >= '0' && c <= '9') { + size_t d = c - '0'; + if (len > SIZE_MAX / 10) { + return DemangleInvalid; + } + len *= 10; + if (len > SIZE_MAX - d) { + return DemangleInvalid; + } + len += d; + + chars++; + chars_len--; + if (chars_len == 0) { + return DemangleInvalid; + } + c = *chars; + } + + // Advance by the length + if (chars_len <= len) { + return DemangleInvalid; + } + chars += len; + chars_len -= len; + elements++; + } + *res = (struct demangle_legacy) { inner, inner_len, elements }; + *rest = chars + 1; + return DemangleOk; +} + +static bool is_rust_hash(const char *s, size_t len) { + if (len == 0 || s[0] != 'h') { + return false; + } + + for (size_t i = 1; i < len; i++) { + if (!((s[i] >= '0' && s[i] <= '9') || (s[i] >= 'a' && s[i] <= 'f') || (s[i] >= 'A' && s[i] <= 'F'))) { + return false; + } + } + + return true; +} + +NODISCARD static overflow_status rust_demangle_legacy_display_demangle(struct demangle_legacy res, char *out, size_t len, bool alternate) +{ + struct printer printer = { + // not actually using the parser part of the printer, just keeping it to share the format functions + DemangleOk, + { NULL }, + out, + len, + 0, + alternate + }; + const char *inner = res.mangled; + for (size_t element = 0; element < res.elements; element++) { + size_t i = 0; + const char *rest; + for (rest = inner; rest < res.mangled + res.mangled_len && *rest >= '0' && *rest <= '9'; rest++) { + i *= 10; + i += *rest - '0'; + } + if ((size_t)(res.mangled + res.mangled_len - rest) < i) { + // safety: shouldn't reach this place if the input string is validated. bail out. + // safety: we knwo rest <= res.mangled + res.mangled_len from the for-loop above + break; + } + + size_t len = i; + inner = rest + len; + + // From here on, inner contains a pointer to the next element, rest[:len] to the current one + if (alternate && element + 1 == res.elements && is_rust_hash(rest, i)) { + break; + } + if (element != 0) { + PRINT_STR(&printer, "::"); + } + + if (len >= 2 && !strncmp(rest, "_$", 2)) { + rest++; + len--; + } + + while (len > 0) { + if (rest[0] == '.') { + if (len >= 2 && rest[1] == '.') { + PRINT_STR(&printer, "::"); + rest += 2; + len -= 2; + } else { + PRINT_STR(&printer, "."); + rest += 1; + len -= 1; + } + } else if (rest[0] == '$') { + const char *escape = memchr(rest + 1, '$', len - 1); + if (escape == NULL) { + break; + } + const char *escape_start = rest + 1; + size_t escape_len = escape - (rest + 1); + + size_t next_len = len - (escape + 1 - rest); + const char *next_rest = escape + 1; + + char ch; + if ((escape_len == 2 && escape_start[0] == 'S' && escape_start[1] == 'P')) { + ch = '@'; + } else if ((escape_len == 2 && escape_start[0] == 'B' && escape_start[1] == 'P')) { + ch = '*'; + } else if ((escape_len == 2 && escape_start[0] == 'R' && escape_start[1] == 'F')) { + ch = '&'; + } else if ((escape_len == 2 && escape_start[0] == 'L' && escape_start[1] == 'T')) { + ch = '<'; + } else if ((escape_len == 2 && escape_start[0] == 'G' && escape_start[1] == 'T')) { + ch = '>'; + } else if ((escape_len == 2 && escape_start[0] == 'L' && escape_start[1] == 'P')) { + ch = '('; + } else if ((escape_len == 2 && escape_start[0] == 'R' && escape_start[1] == 'P')) { + ch = ')'; + } else if ((escape_len == 1 && escape_start[0] == 'C')) { + ch = ','; + } else { + if (escape_len > 1 && escape_start[0] == 'u') { + escape_start++; + escape_len--; + uint64_t val; + if (try_parse_uint(escape_start, escape_len, &val) + && val < UINT32_MAX + && validate_char((uint32_t)val)) + { + if (!unicode_iscontrol(val)) { + uint8_t wchr[4]; + size_t wchr_len = code_to_utf8(wchr, (uint32_t)val); + PRINT(printer_print_buf(&printer, (const char*)wchr, wchr_len)); + len = next_len; + rest = next_rest; + continue; + } + } + } + break; // print the rest of this element raw + } + PRINT_CH(&printer, ch); + len = next_len; + rest = next_rest; + } else { + size_t j = 0; + for (;j < len && rest[j] != '$' && rest[j] != '.';j++); + if (j == len) { + break; + } + PRINT(printer_print_buf(&printer, rest, j)); + rest += j; + len -= j; + } + } + PRINT(printer_print_buf(&printer, rest, len)); + } + + if (printer.out_len < OVERFLOW_MARGIN) { + return OverflowOverflow; + } + *printer.out = '\0'; + return OverflowOk; +} + +static bool is_symbol_like(const char *s, size_t len) { + // rust-demangle definition of symbol like: control characters and space are not symbol-like, all else is + for (size_t i = 0; i < len; i++) { + char ch = s[i]; + if (!(ch >= 0x21 && ch <= 0x7e)) { + return false; + } + } + return true; +} + +void rust_demangle_demangle(const char *s, struct demangle *res) +{ + // During ThinLTO LLVM may import and rename internal symbols, so strip out + // those endings first as they're one of the last manglings applied to symbol + // names. + const char *llvm = ".llvm."; + const char *found_llvm = strstr(s, llvm); + size_t s_len = strlen(s); + if (found_llvm) { + const char *all_hex_ptr = found_llvm + strlen(".llvm."); + bool all_hex = true; + for (;*all_hex_ptr;all_hex_ptr++) { + if (!(('0' <= *all_hex_ptr && *all_hex_ptr <= '9') || + ('A' <= *all_hex_ptr && *all_hex_ptr <= 'F') || + *all_hex_ptr == '@')) { + all_hex = false; + break; + } + } + + if (all_hex) { + s_len = found_llvm - s; + } + } + + const char *suffix; + struct demangle_legacy legacy; + demangle_status st = rust_demangle_legacy_demangle(s, s_len, &legacy, &suffix); + if (st == DemangleOk) { + *res = (struct demangle) { + .style=DemangleStyleLegacy, + .mangled=legacy.mangled, + .mangled_len=legacy.mangled_len, + .elements=legacy.elements, + .original=s, + .original_len=s_len, + .suffix=suffix, + .suffix_len=s_len - (suffix - s), + }; + } else { + struct demangle_v0 v0; + st = rust_demangle_v0_demangle(s, s_len, &v0, &suffix); + if (st == DemangleOk) { + *res = (struct demangle) { + .style=DemangleStyleV0, + .mangled=v0.mangled, + .mangled_len=v0.mangled_len, + .elements=0, + .original=s, + .original_len=s_len, + .suffix=suffix, + .suffix_len=s_len - (suffix - s), + }; + } else { + *res = (struct demangle) { + .style=DemangleStyleUnknown, + .mangled=NULL, + .mangled_len=0, + .elements=0, + .original=s, + .original_len=s_len, + .suffix=s, + .suffix_len=0, + }; + } + } + + // Output like LLVM IR adds extra period-delimited words. See if + // we are in that case and save the trailing words if so. + if (res->suffix_len) { + if (res->suffix[0] == '.' && is_symbol_like(res->suffix, res->suffix_len)) { + // Keep the suffix + } else { + // Reset the suffix and invalidate the demangling + res->style = DemangleStyleUnknown; + res->suffix_len = 0; + } + } +} + +bool rust_demangle_is_known(struct demangle *res) { + return res->style != DemangleStyleUnknown; +} + +overflow_status rust_demangle_display_demangle(struct demangle const *res, char *out, size_t len, bool alternate) { + size_t original_len = res->original_len; + size_t out_len; + switch (res->style) { + case DemangleStyleUnknown: + if (len < original_len) { + return OverflowOverflow; + } else { + memcpy(out, res->original, original_len); + out += original_len; + len -= original_len; + break; + } + break; + case DemangleStyleLegacy: { + struct demangle_legacy legacy = { + res->mangled, + res->mangled_len, + res->elements + }; + if (rust_demangle_legacy_display_demangle(legacy, out, len, alternate) == OverflowOverflow) { + return OverflowOverflow; + } + out_len = strlen(out); + out += out_len; + len -= out_len; + break; + } + case DemangleStyleV0: { + struct demangle_v0 v0 = { + res->mangled, + res->mangled_len + }; + if (rust_demangle_v0_display_demangle(v0, out, len, alternate) == OverflowOverflow) { + return OverflowOverflow; + } + out_len = strlen(out); + out += out_len; + len -= out_len; + break; + } + } + size_t suffix_len = res->suffix_len; + if (len < suffix_len || len - suffix_len < OVERFLOW_MARGIN) { + return OverflowOverflow; + } + memcpy(out, res->suffix, suffix_len); + out[suffix_len] = 0; + return OverflowOk; +} diff --git a/tools/perf/util/demangle-rust-v0.h b/tools/perf/util/demangle-rust-v0.h new file mode 100644 index 000000000..d00928186 --- /dev/null +++ b/tools/perf/util/demangle-rust-v0.h @@ -0,0 +1,88 @@ +// SPDX-License-Identifier: Apache-2.0 OR MIT + +// The contents of this file come from the Rust rustc-demangle library, hosted +// in the repository, licensed +// under "Apache-2.0 OR MIT". For copyright details, see +// . +// Please note that the file should be kept as close as possible to upstream. + +#ifndef _H_DEMANGLE_V0_H +#define _H_DEMANGLE_V0_H + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +#if defined(__GNUC__) || defined(__clang__) +#define DEMANGLE_NODISCARD __attribute__((warn_unused_result)) +#else +#define DEMANGLE_NODISCARD +#endif + +typedef enum { + OverflowOk, + OverflowOverflow +} overflow_status; + +enum demangle_style { + DemangleStyleUnknown = 0, + DemangleStyleLegacy, + DemangleStyleV0, +}; + +// Not using a union here to make the struct easier to copy-paste if needed. +struct demangle { + enum demangle_style style; + // points to the "mangled" part of the name, + // not including `ZN` or `R` prefixes. + const char *mangled; + size_t mangled_len; + // In DemangleStyleLegacy, is the number of path elements + size_t elements; + // while it's called "original", it will not contain `.llvm.9D1C9369@@16` suffixes + // that are to be ignored. + const char *original; + size_t original_len; + // Contains the part after the mangled name that is to be outputted, + // which can be `.exit.i.i` suffixes LLVM sometimes adds. + const char *suffix; + size_t suffix_len; +}; + +// if the length of the output buffer is less than `output_len-OVERFLOW_MARGIN`, +// the demangler will return `OverflowOverflow` even if there is no overflow. +#define OVERFLOW_MARGIN 4 + +/// Demangle a C string that refers to a Rust symbol and put the demangle intermediate result in `res`. +/// Beware that `res` contains references into `s`. If `s` is modified (or free'd) before calling +/// `rust_demangle_display_demangle` behavior is undefined. +/// +/// Use `rust_demangle_display_demangle` to convert it to an actual string. +void rust_demangle_demangle(const char *s, struct demangle *res); + +/// Write the string in a `struct demangle` into a buffer. +/// +/// Return `OverflowOk` if the output buffer was sufficiently big, `OverflowOverflow` if it wasn't. +/// This function is `O(n)` in the length of the input + *output* [$], but the demangled output of demangling a symbol can +/// be exponentially[$$] large, therefore it is recommended to have a sane bound (`rust-demangle` +/// uses 1,000,000 bytes) on `len`. +/// +/// `alternate`, if true, uses the less verbose alternate formatting (Rust `{:#}`) is used, which does not show +/// symbol hashes and types of constant ints. +/// +/// [$] It's `O(n * MAX_DEPTH)`, but `MAX_DEPTH` is a constant 300 and therefore it's `O(n)` +/// [$$] Technically, bounded by `O(n^MAX_DEPTH)`, but this is practically exponential. +DEMANGLE_NODISCARD overflow_status rust_demangle_display_demangle(struct demangle const *res, char *out, size_t len, bool alternate); + +/// Returns true if `res` refers to a known valid Rust demangling style, false if it's an unknown style. +bool rust_demangle_is_known(struct demangle *res); + +#undef DEMANGLE_NODISCARD + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/tools/perf/util/disasm.c b/tools/perf/util/disasm.c new file mode 100644 index 000000000..0a1a7e9cf --- /dev/null +++ b/tools/perf/util/disasm.c @@ -0,0 +1,1648 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include +#ifndef EF_CSKY_ABIMASK +#define EF_CSKY_ABIMASK 0XF0000000 +#endif +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "annotate.h" +#include "annotate-data.h" +#include "build-id.h" +#include "capstone.h" +#include "debug.h" +#include "disasm.h" +#include "dso.h" +#include "dwarf-regs.h" +#include "env.h" +#include "evsel.h" +#include "libbfd.h" +#include "llvm.h" +#include "map.h" +#include "maps.h" +#include "namespaces.h" +#include "srcline.h" +#include "symbol.h" +#include "thread.h" +#include "util.h" + +static regex_t file_lineno; + +/* These can be referred from the arch-dependent code */ +const struct ins_ops call_ops; +const struct ins_ops dec_ops; +const struct ins_ops jump_ops; +const struct ins_ops mov_ops; +const struct ins_ops nop_ops; +const struct ins_ops lock_ops; +const struct ins_ops ret_ops; +const struct ins_ops load_store_ops; +const struct ins_ops arithmetic_ops; + +static void ins__sort(struct arch *arch); +static int disasm_line__parse(char *line, const char **namep, char **rawp); +static int disasm_line__parse_powerpc(struct disasm_line *dl, struct annotate_args *args); + +static __attribute__((constructor)) void symbol__init_regexpr(void) +{ + regcomp(&file_lineno, "^/[^:]+:([0-9]+)", REG_EXTENDED); +} + +static int arch__grow_instructions(struct arch *arch) +{ + struct ins *new_instructions; + size_t new_nr_allocated; + + if (arch->nr_instructions_allocated == 0 && arch->instructions) + goto grow_from_non_allocated_table; + + new_nr_allocated = arch->nr_instructions_allocated + 128; + new_instructions = realloc((void *)arch->instructions, + new_nr_allocated * sizeof(struct ins)); + if (new_instructions == NULL) + return -1; + +out_update_instructions: + arch->instructions = new_instructions; + arch->nr_instructions_allocated = new_nr_allocated; + return 0; + +grow_from_non_allocated_table: + new_nr_allocated = arch->nr_instructions + 128; + new_instructions = calloc(new_nr_allocated, sizeof(struct ins)); + if (new_instructions == NULL) + return -1; + + memcpy(new_instructions, arch->instructions, arch->nr_instructions * sizeof(struct ins)); + goto out_update_instructions; +} + +int arch__associate_ins_ops(struct arch *arch, const char *name, const struct ins_ops *ops) +{ + struct ins *ins; + + if (arch->nr_instructions == arch->nr_instructions_allocated && + arch__grow_instructions(arch)) + return -1; + + ins = (struct ins *)&arch->instructions[arch->nr_instructions]; + ins->name = strdup(name); + if (!ins->name) + return -1; + + ins->ops = ops; + arch->nr_instructions++; + + ins__sort(arch); + return 0; +} + +static int e_machine_and_eflags__cmp(const struct e_machine_and_e_flags *val1, + const struct e_machine_and_e_flags *val2) +{ + if (val1->e_machine == val2->e_machine) { + if (val1->e_machine != EM_CSKY) + return 0; + if ((val1->e_flags & EF_CSKY_ABIMASK) < (val2->e_flags & EF_CSKY_ABIMASK)) + return -1; + return (val1->e_flags & EF_CSKY_ABIMASK) > (val2->e_flags & EF_CSKY_ABIMASK); + } + return val1->e_machine < val2->e_machine ? -1 : 1; +} + +static int arch__key_cmp(const void *key, const void *archp) +{ + const struct arch *const *arch = archp; + + return e_machine_and_eflags__cmp(key, &(*arch)->id); +} + +static int arch__cmp(const void *a, const void *b) +{ + const struct arch *const *aa = a; + const struct arch *const *ab = b; + + return e_machine_and_eflags__cmp(&(*aa)->id, &(*ab)->id); +} + +const struct arch *arch__find(uint16_t e_machine, uint32_t e_flags, const char *cpuid) +{ + static const struct arch *(*const arch_new_fn[])(const struct e_machine_and_e_flags *id, + const char *cpuid) = { + [EM_386] = arch__new_x86, + [EM_ARC] = arch__new_arc, + [EM_ARM] = arch__new_arm, + [EM_AARCH64] = arch__new_arm64, + [EM_CSKY] = arch__new_csky, + [EM_LOONGARCH] = arch__new_loongarch, + [EM_MIPS] = arch__new_mips, + [EM_PPC64] = arch__new_powerpc, + [EM_PPC] = arch__new_powerpc, + [EM_RISCV] = arch__new_riscv64, + [EM_S390] = arch__new_s390, + [EM_SPARC] = arch__new_sparc, + [EM_SPARCV9] = arch__new_sparc, + [EM_X86_64] = arch__new_x86, + }; + static const struct arch **archs; + static size_t num_archs; + struct e_machine_and_e_flags key = { + .e_machine = e_machine, + .e_flags = e_flags, + }; + const struct arch *result = NULL, **tmp; + + if (num_archs > 0) { + tmp = bsearch(&key, archs, num_archs, sizeof(*archs), arch__key_cmp); + if (tmp) + result = *tmp; + } + + if (result) + return result; + + if (e_machine >= ARRAY_SIZE(arch_new_fn) || arch_new_fn[e_machine] == NULL) { + errno = ENOTSUP; + return NULL; + } + + tmp = reallocarray(archs, num_archs + 1, sizeof(*archs)); + if (!tmp) + return NULL; + + result = arch_new_fn[e_machine](&key, cpuid); + if (!result) { + pr_err("%s: failed to initialize %s (%u) arch priv area\n", + __func__, result->name, e_machine); + free(tmp); + return NULL; + } + archs = tmp; + archs[num_archs++] = result; + qsort(archs, num_archs, sizeof(*archs), arch__cmp); + return result; +} + +bool arch__is_x86(const struct arch *arch) +{ + return arch->id.e_machine == EM_386 || arch->id.e_machine == EM_X86_64; +} + +bool arch__is_powerpc(const struct arch *arch) +{ + return arch->id.e_machine == EM_PPC || arch->id.e_machine == EM_PPC64; +} + +static void ins_ops__delete(struct ins_operands *ops) +{ + if (ops == NULL) + return; + zfree(&ops->source.raw); + zfree(&ops->source.name); + zfree(&ops->target.raw); + zfree(&ops->target.name); +} + +int ins__raw_scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->raw); +} + +int ins__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + if (ins->ops->scnprintf) + return ins->ops->scnprintf(ins, bf, size, ops, max_ins_name); + + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); +} + +bool ins__is_fused(const struct arch *arch, const char *ins1, const char *ins2) +{ + if (!arch || !arch->ins_is_fused) + return false; + + return arch->ins_is_fused(arch, ins1, ins2); +} + +static int call__parse(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + char *endptr, *tok, *name; + struct map *map = ms->map; + struct addr_map_symbol target; + + ops->target.addr = strtoull(ops->raw, &endptr, 16); + + name = strchr(endptr, '<'); + if (name == NULL) + goto indirect_call; + + name++; + + if (arch->objdump.skip_functions_char && + strchr(name, arch->objdump.skip_functions_char)) + return -1; + + tok = strchr(name, '>'); + if (tok == NULL) + return -1; + + *tok = '\0'; + ops->target.name = strdup(name); + *tok = '>'; + + if (ops->target.name == NULL) + return -1; +find_target: + target = (struct addr_map_symbol) { + .ms = { .map = map__get(map), }, + .addr = map__objdump_2mem(map, ops->target.addr), + }; + + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + return 0; + +indirect_call: + tok = strchr(endptr, '*'); + if (tok != NULL) { + endptr++; + + /* Indirect call can use a non-rip register and offset: callq *0x8(%rbx). + * Do not parse such instruction. */ + if (strstr(endptr, "(%r") == NULL) + ops->target.addr = strtoull(endptr, NULL, 16); + } + goto find_target; +} + +int call__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + if (ops->target.sym) + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->target.sym->name); + + if (ops->target.addr == 0) + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); + + if (ops->target.name) + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->target.name); + + return scnprintf(bf, size, "%-*s *%" PRIx64, max_ins_name, ins->name, ops->target.addr); +} + +const struct ins_ops call_ops = { + .parse = call__parse, + .scnprintf = call__scnprintf, + .is_call = true, +}; + +bool ins__is_call(const struct ins *ins) +{ + return ins->ops && ins->ops->is_call; +} + +/* + * Prevents from matching commas in the comment section, e.g.: + * ffff200008446e70: b.cs ffff2000084470f4 // b.hs, b.nlast + * + * and skip comma as part of function arguments, e.g.: + * 1d8b4ac + */ +static inline const char *validate_comma(const char *c, struct ins_operands *ops) +{ + if (ops->jump.raw_comment && c > ops->jump.raw_comment) + return NULL; + + if (ops->jump.raw_func_start && c > ops->jump.raw_func_start) + return NULL; + + return c; +} + +static int jump__parse(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + struct map *map = ms->map; + struct symbol *sym = ms->sym; + struct addr_map_symbol target = { + .ms = { .map = map__get(map), }, + }; + const char *c = strchr(ops->raw, ','); + u64 start, end; + + ops->jump.raw_comment = strchr(ops->raw, arch->objdump.comment_char); + ops->jump.raw_func_start = strchr(ops->raw, '<'); + + c = validate_comma(c, ops); + + /* + * Examples of lines to parse for the _cpp_lex_token@@Base + * function: + * + * 1159e6c: jne 115aa32 <_cpp_lex_token@@Base+0xf92> + * 1159e8b: jne c469be + * + * The first is a jump to an offset inside the same function, + * the second is to another function, i.e. that 0xa72 is an + * offset in the cpp_named_operator2name@@base function. + */ + /* + * skip over possible up to 2 operands to get to address, e.g.: + * tbnz w0, #26, ffff0000083cd190 + */ + if (c != NULL) { + c++; + ops->target.addr = strtoull(c, NULL, 16); + if (!ops->target.addr) { + c = strchr(c, ','); + c = validate_comma(c, ops); + if (c != NULL) { + c++; + ops->target.addr = strtoull(c, NULL, 16); + } + } + } else { + ops->target.addr = strtoull(ops->raw, NULL, 16); + } + + target.addr = map__objdump_2mem(map, ops->target.addr); + start = map__unmap_ip(map, sym->start); + end = map__unmap_ip(map, sym->end); + + ops->target.outside = target.addr < start || target.addr >= end; + + /* + * FIXME: things like this in _cpp_lex_token (gcc's cc1 program): + + cpp_named_operator2name@@Base+0xa72 + + * Point to a place that is after the cpp_named_operator2name + * boundaries, i.e. in the ELF symbol table for cc1 + * cpp_named_operator2name is marked as being 32-bytes long, but it in + * fact is much larger than that, so we seem to need a symbols__find() + * routine that looks for >= current->start and < next_symbol->start, + * possibly just for C++ objects? + * + * For now lets just make some progress by marking jumps to outside the + * current function as call like. + * + * Actual navigation will come next, with further understanding of how + * the symbol searching and disassembly should be done. + */ + if (maps__find_ams(thread__maps(ms->thread), &target) == 0 && + map__rip_2objdump(target.ms.map, map__map_ip(target.ms.map, target.addr)) == ops->target.addr) + ops->target.sym = target.ms.sym; + + if (!ops->target.outside) { + ops->target.offset = target.addr - start; + ops->target.offset_avail = true; + } else { + ops->target.offset_avail = false; + } + addr_map_symbol__exit(&target); + return 0; +} + +int jump__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + const char *c; + + if (!ops->target.addr || ops->target.offset < 0) + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); + + if (ops->target.outside && ops->target.sym != NULL) + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, ops->target.sym->name); + + c = strchr(ops->raw, ','); + c = validate_comma(c, ops); + + if (c != NULL) { + const char *c2 = strchr(c + 1, ','); + + c2 = validate_comma(c2, ops); + /* check for 3-op insn */ + if (c2 != NULL) + c = c2; + c++; + + /* mirror arch objdump's space-after-comma style */ + if (*c == ' ') + c++; + } + + return scnprintf(bf, size, "%-*s %.*s%" PRIx64, max_ins_name, + ins->name, c ? c - ops->raw : 0, ops->raw, + ops->target.offset); +} + +void jump__delete(struct ins_operands *ops __maybe_unused) +{ + /* + * The ops->jump.raw_comment and ops->jump.raw_func_start belong to the + * raw string, don't free them. + */ +} + +const struct ins_ops jump_ops = { + .free = jump__delete, + .parse = jump__parse, + .scnprintf = jump__scnprintf, + .is_jump = true, +}; + +bool ins__is_jump(const struct ins *ins) +{ + return ins->ops && ins->ops->is_jump; +} + +static int comment__symbol(char *raw, char *comment, u64 *addrp, char **namep) +{ + char *endptr, *name, *t; + + if (strstr(raw, "(%rip)") == NULL) + return 0; + + *addrp = strtoull(comment, &endptr, 16); + if (endptr == comment) + return 0; + name = strchr(endptr, '<'); + if (name == NULL) + return -1; + + name++; + + t = strchr(name, '>'); + if (t == NULL) + return 0; + + *t = '\0'; + *namep = strdup(name); + *t = '>'; + + return 0; +} + +static int lock__parse(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl __maybe_unused) +{ + ops->locked.ops = zalloc(sizeof(*ops->locked.ops)); + if (ops->locked.ops == NULL) + return 0; + + if (disasm_line__parse(ops->raw, &ops->locked.ins.name, &ops->locked.ops->raw) < 0) + goto out_free_ops; + + ops->locked.ins.ops = ins__find(arch, ops->locked.ins.name, 0); + + if (ops->locked.ins.ops == NULL) + goto out_free_ops; + + if (ops->locked.ins.ops->parse && + ops->locked.ins.ops->parse(arch, ops->locked.ops, ms, NULL) < 0) + goto out_free_ops; + + return 0; + +out_free_ops: + zfree(&ops->locked.ops); + return 0; +} + +static int lock__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + int printed; + + if (ops->locked.ins.ops == NULL) + return ins__raw_scnprintf(ins, bf, size, ops, max_ins_name); + + printed = scnprintf(bf, size, "%-*s ", max_ins_name, ins->name); + return printed + ins__scnprintf(&ops->locked.ins, bf + printed, + size - printed, ops->locked.ops, max_ins_name); +} + +static void lock__delete(struct ins_operands *ops) +{ + struct ins *ins = &ops->locked.ins; + + if (ins->ops && ins->ops->free) + ins->ops->free(ops->locked.ops); + else + ins_ops__delete(ops->locked.ops); + + zfree(&ops->locked.ops); + zfree(&ops->locked.ins.name); + zfree(&ops->target.raw); + zfree(&ops->target.name); +} + +const struct ins_ops lock_ops = { + .free = lock__delete, + .parse = lock__parse, + .scnprintf = lock__scnprintf, +}; + +/* + * Check if the operand has more than one registers like x86 SIB addressing: + * 0x1234(%rax, %rbx, 8) + * + * But it doesn't care segment selectors like %gs:0x5678(%rcx), so just check + * the input string after 'memory_ref_char' if exists. + */ +static bool check_multi_regs(const struct arch *arch, const char *op) +{ + int count = 0; + + if (arch->objdump.register_char == 0) + return false; + + if (arch->objdump.memory_ref_char) { + op = strchr(op, arch->objdump.memory_ref_char); + if (op == NULL) + return false; + } + + while ((op = strchr(op, arch->objdump.register_char)) != NULL) { + count++; + op++; + } + + return count > 1; +} + +static int mov__parse(const struct arch *arch, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *s = strchr(ops->raw, ','), *target, *comment, prev; + + if (s == NULL) + return -1; + + *s = '\0'; + + /* + * x86 SIB addressing has something like 0x8(%rax, %rcx, 1) + * then it needs to have the closing parenthesis. + */ + if (strchr(ops->raw, '(')) { + *s = ','; + s = strchr(ops->raw, ')'); + if (s == NULL || s[1] != ',') + return -1; + *++s = '\0'; + } + + ops->source.raw = strdup(ops->raw); + *s = ','; + + if (ops->source.raw == NULL) + return -1; + + ops->source.multi_regs = check_multi_regs(arch, ops->source.raw); + + target = skip_spaces(++s); + comment = strchr(s, arch->objdump.comment_char); + + if (comment != NULL) + s = comment - 1; + else + s = strchr(s, '\0') - 1; + + while (s > target && isspace(s[0])) + --s; + s++; + prev = *s; + *s = '\0'; + + ops->target.raw = strdup(target); + *s = prev; + + if (ops->target.raw == NULL) + goto out_free_source; + + ops->target.multi_regs = check_multi_regs(arch, ops->target.raw); + + if (comment == NULL) + return 0; + + comment = skip_spaces(comment); + comment__symbol(ops->source.raw, comment + 1, &ops->source.addr, &ops->source.name); + comment__symbol(ops->target.raw, comment + 1, &ops->target.addr, &ops->target.name); + + return 0; + +out_free_source: + zfree(&ops->source.raw); + return -1; +} + +int mov__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s,%s", max_ins_name, ins->name, + ops->source.name ?: ops->source.raw, + ops->target.name ?: ops->target.raw); +} + +const struct ins_ops mov_ops = { + .parse = mov__parse, + .scnprintf = mov__scnprintf, +}; + +static int dec__parse(const struct arch *arch __maybe_unused, struct ins_operands *ops, + struct map_symbol *ms __maybe_unused, + struct disasm_line *dl __maybe_unused) +{ + char *target, *comment, *s, prev; + + target = s = ops->raw; + + while (s[0] != '\0' && !isspace(s[0])) + ++s; + prev = *s; + *s = '\0'; + + ops->target.raw = strdup(target); + *s = prev; + + if (ops->target.raw == NULL) + return -1; + + comment = strchr(s, arch->objdump.comment_char); + if (comment == NULL) + return 0; + + comment = skip_spaces(comment); + comment__symbol(ops->target.raw, comment + 1, &ops->target.addr, &ops->target.name); + + return 0; +} + +static int dec__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s %s", max_ins_name, ins->name, + ops->target.name ?: ops->target.raw); +} + +const struct ins_ops dec_ops = { + .parse = dec__parse, + .scnprintf = dec__scnprintf, +}; + +static int nop__scnprintf(const struct ins *ins __maybe_unused, char *bf, size_t size, + struct ins_operands *ops __maybe_unused, int max_ins_name) +{ + return scnprintf(bf, size, "%-*s", max_ins_name, "nop"); +} + +const struct ins_ops nop_ops = { + .scnprintf = nop__scnprintf, +}; + +const struct ins_ops ret_ops = { + .scnprintf = ins__raw_scnprintf, +}; + +static bool ins__is_nop(const struct ins *ins) +{ + return ins->ops == &nop_ops; +} + +bool ins__is_ret(const struct ins *ins) +{ + return ins->ops == &ret_ops; +} + +bool ins__is_lock(const struct ins *ins) +{ + return ins->ops == &lock_ops; +} + +static int ins__key_cmp(const void *name, const void *insp) +{ + const struct ins *ins = insp; + + return strcmp(name, ins->name); +} + +static int ins__cmp(const void *a, const void *b) +{ + const struct ins *ia = a; + const struct ins *ib = b; + + return strcmp(ia->name, ib->name); +} + +static void ins__sort(struct arch *arch) +{ + const int nmemb = arch->nr_instructions; + + qsort((void *)arch->instructions, nmemb, sizeof(struct ins), ins__cmp); +} + +static const struct ins_ops *__ins__find(const struct arch *arch, const char *name, + struct disasm_line *dl) +{ + const struct ins *ins; + const int nmemb = arch->nr_instructions; + + if (arch__is_powerpc(arch)) { + /* + * For powerpc, identify the instruction ops + * from the opcode using raw_insn. + */ + const struct ins_ops *ops; + + ops = check_ppc_insn(dl); + if (ops) + return ops; + } + + if (!arch->sorted_instructions) { + ins__sort((struct arch *)arch); + ((struct arch *)arch)->sorted_instructions = true; + } + + ins = bsearch(name, arch->instructions, nmemb, sizeof(struct ins), ins__key_cmp); + if (ins) + return ins->ops; + + if (arch->insn_suffix) { + char tmp[32]; + char suffix; + size_t len = strlen(name); + + if (len == 0 || len >= sizeof(tmp)) + return NULL; + + suffix = name[len - 1]; + if (strchr(arch->insn_suffix, suffix) == NULL) + return NULL; + + strcpy(tmp, name); + tmp[len - 1] = '\0'; /* remove the suffix and check again */ + + ins = bsearch(tmp, arch->instructions, nmemb, sizeof(struct ins), ins__key_cmp); + } + return ins ? ins->ops : NULL; +} + +const struct ins_ops *ins__find(const struct arch *arch, const char *name, struct disasm_line *dl) +{ + const struct ins_ops *ops = __ins__find(arch, name, dl); + + if (!ops && arch->associate_instruction_ops) + ops = arch->associate_instruction_ops((struct arch *)arch, name); + + return ops; +} + +static void disasm_line__init_ins(struct disasm_line *dl, const struct arch *arch, + struct map_symbol *ms) +{ + dl->ins.ops = ins__find(arch, dl->ins.name, dl); + + if (!dl->ins.ops) + return; + + if (dl->ins.ops->parse && dl->ins.ops->parse(arch, &dl->ops, ms, dl) < 0) + dl->ins.ops = NULL; +} + +static int disasm_line__parse(char *line, const char **namep, char **rawp) +{ + char tmp, *name = skip_spaces(line); + + if (name[0] == '\0') + return -1; + + *rawp = name + 1; + + while ((*rawp)[0] != '\0' && !isspace((*rawp)[0])) + ++*rawp; + + tmp = (*rawp)[0]; + (*rawp)[0] = '\0'; + *namep = strdup(name); + + if (*namep == NULL) + goto out; + + (*rawp)[0] = tmp; + *rawp = strim(*rawp); + + return 0; + +out: + return -1; +} + +/* + * Parses the result captured from symbol__disassemble_* + * Example, line read from DSO file in powerpc: + * line: 38 01 81 e8 + * opcode: fetched from arch specific get_opcode_insn + * rawp_insn: e8810138 + * + * rawp_insn is used later to extract the reg/offset fields + */ +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define RAW_BYTES 11 + +static int disasm_line__parse_powerpc(struct disasm_line *dl, struct annotate_args *args) +{ + char *line = dl->al.line; + const char **namep = &dl->ins.name; + char **rawp = &dl->ops.raw; + char *tmp_raw_insn, *name_raw_insn = skip_spaces(line); + char *name = skip_spaces(name_raw_insn + RAW_BYTES); + int disasm = 0; + int ret = 0; + + if (args->options->disassembler_used) + disasm = 1; + + if (name_raw_insn[0] == '\0') + return -1; + + if (disasm) + ret = disasm_line__parse(name, namep, rawp); + else + *namep = ""; + + tmp_raw_insn = strndup(name_raw_insn, 11); + if (tmp_raw_insn == NULL) + return -1; + + remove_spaces(tmp_raw_insn); + + sscanf(tmp_raw_insn, "%x", &dl->raw.raw_insn); + if (disasm) + dl->raw.raw_insn = be32_to_cpu(dl->raw.raw_insn); + + return ret; +} + +static void annotation_line__init(struct annotation_line *al, + struct annotate_args *args, + int nr) +{ + al->offset = args->offset; + al->line = strdup(args->line); + al->line_nr = args->line_nr; + al->fileloc = args->fileloc ? strdup(args->fileloc) : NULL; + al->data_nr = nr; +} + +static void annotation_line__exit(struct annotation_line *al) +{ + zfree_srcline(&al->path); + zfree(&al->fileloc); + zfree(&al->line); + zfree(&al->cycles); + zfree(&al->br_cntr); +} + +static size_t disasm_line_size(int nr) +{ + struct annotation_line *al; + + return (sizeof(struct disasm_line) + (sizeof(al->data[0]) * nr)); +} + +/* + * Allocating the disasm annotation line data with + * following structure: + * + * ------------------------------------------- + * struct disasm_line | struct annotation_line + * ------------------------------------------- + * + * We have 'struct annotation_line' member as last member + * of 'struct disasm_line' to have an easy access. + */ +struct disasm_line *disasm_line__new(struct annotate_args *args) +{ + struct disasm_line *dl = NULL; + struct annotation *notes = symbol__annotation(args->ms->sym); + int nr = notes->src->nr_events; + + dl = zalloc(disasm_line_size(nr)); + if (!dl) + return NULL; + + annotation_line__init(&dl->al, args, nr); + if (dl->al.line == NULL) + goto out_free_line; + + if (args->offset != -1) { + if (arch__is_powerpc(args->arch)) { + if (disasm_line__parse_powerpc(dl, args) < 0) + goto out_free_line; + } else if (disasm_line__parse(dl->al.line, &dl->ins.name, &dl->ops.raw) < 0) + goto out_free_line; + + disasm_line__init_ins(dl, args->arch, args->ms); + } + + return dl; + +out_free_line: + annotation_line__exit(&dl->al); + free(dl); + return NULL; +} + +void disasm_line__free(struct disasm_line *dl) +{ + if (dl->ins.ops && dl->ins.ops->free) + dl->ins.ops->free(&dl->ops); + else + ins_ops__delete(&dl->ops); + zfree(&dl->ins.name); + annotation_line__exit(&dl->al); + free(dl); +} + +int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, bool raw, int max_ins_name) +{ + if (raw || !dl->ins.ops) + return scnprintf(bf, size, "%-*s %s", max_ins_name, dl->ins.name, dl->ops.raw); + + return ins__scnprintf(&dl->ins, bf, size, &dl->ops, max_ins_name); +} + +/* + * symbol__parse_objdump_line() parses objdump output (with -d --no-show-raw) + * which looks like following + * + * 0000000000415500 <_init>: + * 415500: sub $0x8,%rsp + * 415504: mov 0x2f5ad5(%rip),%rax # 70afe0 <_DYNAMIC+0x2f8> + * 41550b: test %rax,%rax + * 41550e: je 415515 <_init+0x15> + * 415510: callq 416e70 <__gmon_start__@plt> + * 415515: add $0x8,%rsp + * 415519: retq + * + * it will be parsed and saved into struct disasm_line as + * + * + * The offset will be a relative offset from the start of the symbol and -1 + * means that it's not a disassembly line so should be treated differently. + * The ops.raw part will be parsed further according to type of the instruction. + */ +static int symbol__parse_objdump_line(struct symbol *sym, + struct annotate_args *args, + char *parsed_line, int *line_nr, char **fileloc) +{ + struct map *map = args->ms->map; + struct annotation *notes = symbol__annotation(sym); + struct disasm_line *dl; + char *tmp; + s64 line_ip, offset = -1; + regmatch_t match[2]; + + /* /filename:linenr ? Save line number and ignore. */ + if (regexec(&file_lineno, parsed_line, 2, match, 0) == 0) { + *line_nr = atoi(parsed_line + match[1].rm_so); + free(*fileloc); + *fileloc = strdup(parsed_line); + return 0; + } + + /* Process hex address followed by ':'. */ + line_ip = strtoull(parsed_line, &tmp, 16); + if (parsed_line != tmp && tmp[0] == ':' && tmp[1] != '\0') { + u64 start = map__rip_2objdump(map, sym->start), + end = map__rip_2objdump(map, sym->end); + + offset = line_ip - start; + if ((u64)line_ip < start || (u64)line_ip >= end) + offset = -1; + else + parsed_line = tmp + 1; + } + + args->offset = offset; + args->line = parsed_line; + args->line_nr = *line_nr; + args->fileloc = *fileloc; + args->ms->sym = sym; + + dl = disasm_line__new(args); + (*line_nr)++; + + if (dl == NULL) + return -1; + + if (!disasm_line__has_local_offset(dl)) { + dl->ops.target.offset = dl->ops.target.addr - + map__rip_2objdump(map, sym->start); + dl->ops.target.offset_avail = true; + } + + /* kcore has no symbols, so add the call target symbol */ + if (dl->ins.ops && ins__is_call(&dl->ins) && !dl->ops.target.sym) { + struct addr_map_symbol target = { + .addr = dl->ops.target.addr, + .ms = { .map = map__get(map), }, + }; + + if (!maps__find_ams(thread__maps(args->ms->thread), &target) && + target.ms.sym->start == target.al_addr) + dl->ops.target.sym = target.ms.sym; + + addr_map_symbol__exit(&target); + } + + annotation_line__add(&dl->al, ¬es->src->source); + return 0; +} + +static void delete_last_nop(struct symbol *sym) +{ + struct annotation *notes = symbol__annotation(sym); + struct list_head *list = ¬es->src->source; + struct disasm_line *dl; + + while (!list_empty(list)) { + dl = list_entry(list->prev, struct disasm_line, al.node); + + if (dl->ins.ops) { + if (!ins__is_nop(&dl->ins)) + return; + } else { + if (!strstr(dl->al.line, " nop ") && + !strstr(dl->al.line, " nopl ") && + !strstr(dl->al.line, " nopw ")) + return; + } + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } +} + +int symbol__strerror_disassemble(struct map_symbol *ms, int errnum, char *buf, size_t buflen) +{ + struct dso *dso = map__dso(ms->map); + + BUG_ON(buflen == 0); + + if (errnum >= 0) { + str_error_r(errnum, buf, buflen); + return 0; + } + + switch (errnum) { + case SYMBOL_ANNOTATE_ERRNO__NO_VMLINUX: { + char bf[SBUILD_ID_SIZE + 15] = " with build id "; + char *build_id_msg = NULL; + + if (dso__has_build_id(dso)) { + build_id__snprintf(dso__bid(dso), bf + 15, sizeof(bf) - 15); + build_id_msg = bf; + } + scnprintf(buf, buflen, + "No vmlinux file%s\nwas found in the path.\n\n" + "Note that annotation using /proc/kcore requires CAP_SYS_RAWIO capability.\n\n" + "Please use:\n\n" + " perf buildid-cache -vu vmlinux\n\n" + "or:\n\n" + " --vmlinux vmlinux\n", build_id_msg ?: ""); + } + break; + case SYMBOL_ANNOTATE_ERRNO__NO_LIBOPCODES_FOR_BPF: + scnprintf(buf, buflen, "Please link with binutils's libopcode to enable BPF annotation"); + break; + case SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_REGEXP: + scnprintf(buf, buflen, "Problems with arch specific instruction name regular expressions."); + break; + case SYMBOL_ANNOTATE_ERRNO__ARCH_INIT_CPUID_PARSING: + scnprintf(buf, buflen, "Problems while parsing the CPUID in the arch specific initialization."); + break; + case SYMBOL_ANNOTATE_ERRNO__BPF_INVALID_FILE: + scnprintf(buf, buflen, "Invalid BPF file: %s.", dso__long_name(dso)); + break; + case SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF: + scnprintf(buf, buflen, "The %s BPF file has no BTF section, compile with -g or use pahole -J.", + dso__long_name(dso)); + break; + case SYMBOL_ANNOTATE_ERRNO__COULDNT_DETERMINE_FILE_TYPE: + scnprintf(buf, buflen, "Couldn't determine the file %s type.", dso__long_name(dso)); + break; + default: + scnprintf(buf, buflen, "Internal error: Invalid %d error code\n", errnum); + break; + } + + return 0; +} + +static int dso__disassemble_filename(struct dso *dso, char *filename, size_t filename_size) +{ + char linkname[PATH_MAX]; + char *build_id_filename; + char *build_id_path = NULL; + char *pos; + int len; + + if (dso__symtab_type(dso) == DSO_BINARY_TYPE__KALLSYMS && + !dso__is_kcore(dso)) + return SYMBOL_ANNOTATE_ERRNO__NO_VMLINUX; + + build_id_filename = dso__build_id_filename(dso, NULL, 0, false); + if (build_id_filename) { + __symbol__join_symfs(filename, filename_size, build_id_filename); + free(build_id_filename); + } else { + if (dso__has_build_id(dso)) + return ENOMEM; + goto fallback; + } + + build_id_path = strdup(filename); + if (!build_id_path) + return ENOMEM; + + /* + * old style build-id cache has name of XX/XXXXXXX.. while + * new style has XX/XXXXXXX../{elf,kallsyms,vdso}. + * extract the build-id part of dirname in the new style only. + */ + pos = strrchr(build_id_path, '/'); + if (pos && strlen(pos) < SBUILD_ID_SIZE - 2) + dirname(build_id_path); + + if (dso__is_kcore(dso)) + goto fallback; + + len = readlink(build_id_path, linkname, sizeof(linkname) - 1); + if (len < 0) + goto fallback; + + linkname[len] = '\0'; + if (strstr(linkname, DSO__NAME_KALLSYMS) || + access(filename, R_OK)) { +fallback: + /* + * If we don't have build-ids or the build-id file isn't in the + * cache, or is just a kallsyms file, well, lets hope that this + * DSO is the same as when 'perf record' ran. + */ + if (dso__kernel(dso) && dso__long_name(dso)[0] == '/') + snprintf(filename, filename_size, "%s", dso__long_name(dso)); + else + __symbol__join_symfs(filename, filename_size, dso__long_name(dso)); + + mutex_lock(dso__lock(dso)); + if (access(filename, R_OK) && errno == ENOENT && dso__nsinfo(dso)) { + char *new_name = dso__filename_with_chroot(dso, filename); + if (new_name) { + strlcpy(filename, new_name, filename_size); + free(new_name); + } + } + mutex_unlock(dso__lock(dso)); + } else if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) { + dso__set_binary_type(dso, DSO_BINARY_TYPE__BUILD_ID_CACHE); + } + + free(build_id_path); + return 0; +} + +static int symbol__disassemble_raw(char *filename, struct symbol *sym, + struct annotate_args *args) +{ + struct annotation *notes = symbol__annotation(sym); + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + u64 start = map__rip_2objdump(map, sym->start); + u64 end = map__rip_2objdump(map, sym->end); + u64 len = end - start; + u64 offset; + int i, count; + u8 *buf = NULL; + char disasm_buf[512]; + struct disasm_line *dl; + u32 *line; + + /* Return if objdump is specified explicitly */ + if (args->options->objdump_path) + return -1; + + pr_debug("Reading raw instruction from : %s using dso__data_read_offset\n", filename); + + buf = malloc(len); + if (buf == NULL) + goto err; + + count = dso__data_read_offset(dso, NULL, sym->start, buf, len); + + line = (u32 *)buf; + + if ((u64)count != len) + goto err; + + /* add the function address and name */ + scnprintf(disasm_buf, sizeof(disasm_buf), "%#"PRIx64" <%s>:", + start, sym->name); + + args->offset = -1; + args->line = disasm_buf; + args->line_nr = 0; + args->fileloc = NULL; + args->ms->sym = sym; + + dl = disasm_line__new(args); + if (dl == NULL) + goto err; + + annotation_line__add(&dl->al, ¬es->src->source); + + /* Each raw instruction is 4 byte */ + count = len/4; + + for (i = 0, offset = 0; i < count; i++) { + args->offset = offset; + sprintf(args->line, "%x", line[i]); + dl = disasm_line__new(args); + if (dl == NULL) + break; + + annotation_line__add(&dl->al, ¬es->src->source); + offset += 4; + } + + /* It failed in the middle */ + if (offset != len) { + struct list_head *list = ¬es->src->source; + + /* Discard all lines and fallback to objdump */ + while (!list_empty(list)) { + dl = list_first_entry(list, struct disasm_line, al.node); + + list_del_init(&dl->al.node); + disasm_line__free(dl); + } + count = -1; + } + +out: + free(buf); + return count < 0 ? count : 0; + +err: + count = -1; + goto out; +} + +/* + * Possibly create a new version of line with tabs expanded. Returns the + * existing or new line, storage is updated if a new line is allocated. If + * allocation fails then NULL is returned. + */ +char *expand_tabs(char *line, char **storage, size_t *storage_len) +{ + size_t i, src, dst, len, new_storage_len, num_tabs; + char *new_line; + size_t line_len = strlen(line); + + for (num_tabs = 0, i = 0; i < line_len; i++) + if (line[i] == '\t') + num_tabs++; + + if (num_tabs == 0) + return line; + + /* + * Space for the line and '\0', less the leading and trailing + * spaces. Each tab may introduce 7 additional spaces. + */ + new_storage_len = line_len + 1 + (num_tabs * 7); + + new_line = malloc(new_storage_len); + if (new_line == NULL) { + pr_err("Failure allocating memory for tab expansion\n"); + return NULL; + } + + /* + * Copy regions starting at src and expand tabs. If there are two + * adjacent tabs then 'src == i', the memcpy is of size 0 and the spaces + * are inserted. + */ + for (i = 0, src = 0, dst = 0; i < line_len && num_tabs; i++) { + if (line[i] == '\t') { + len = i - src; + memcpy(&new_line[dst], &line[src], len); + dst += len; + new_line[dst++] = ' '; + while (dst % 8 != 0) + new_line[dst++] = ' '; + src = i + 1; + num_tabs--; + } + } + + /* Expand the last region. */ + len = line_len - src; + memcpy(&new_line[dst], &line[src], len); + dst += len; + new_line[dst] = '\0'; + + free(*storage); + *storage = new_line; + *storage_len = new_storage_len; + return new_line; +} + +static int symbol__disassemble_bpf_image(struct symbol *sym, struct annotate_args *args) +{ + struct annotation *notes = symbol__annotation(sym); + struct disasm_line *dl; + + args->offset = -1; + args->line = strdup("to be implemented"); + args->line_nr = 0; + args->fileloc = NULL; + dl = disasm_line__new(args); + if (dl) + annotation_line__add(&dl->al, ¬es->src->source); + + zfree(&args->line); + return 0; +} + +static int symbol__disassemble_objdump(const char *filename, struct symbol *sym, + struct annotate_args *args) +{ + struct annotation_options *opts = &annotate_opts; + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + char *command; + FILE *file; + int lineno = 0; + char *fileloc = NULL; + int nline; + char *line; + size_t line_len; + const char *objdump_argv[] = { + "/bin/sh", + "-c", + NULL, /* Will be the objdump command to run. */ + "--", + NULL, /* Will be the symfs path. */ + NULL, + }; + struct child_process objdump_process; + int err; + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) + return symbol__disassemble_bpf_libbfd(sym, args); + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) + return symbol__disassemble_bpf_image(sym, args); + + err = asprintf(&command, + "%s %s%s --start-address=0x%016" PRIx64 + " --stop-address=0x%016" PRIx64 + " %s -d %s %s %s %c%s%c %s%s -C \"$1\"", + opts->objdump_path ?: "objdump", + opts->disassembler_style ? "-M " : "", + opts->disassembler_style ?: "", + map__rip_2objdump(map, sym->start), + map__rip_2objdump(map, sym->end), + opts->show_linenr ? "-l" : "", + opts->show_asm_raw ? "" : "--no-show-raw-insn", + opts->annotate_src ? "-S" : "", + opts->prefix ? "--prefix " : "", + opts->prefix ? '"' : ' ', + opts->prefix ?: "", + opts->prefix ? '"' : ' ', + opts->prefix_strip ? "--prefix-strip=" : "", + opts->prefix_strip ?: ""); + + if (err < 0) { + pr_err("Failure allocating memory for the command to run\n"); + return err; + } + + pr_debug("Executing: %s\n", command); + + objdump_argv[2] = command; + objdump_argv[4] = filename; + + /* Create a pipe to read from for stdout */ + memset(&objdump_process, 0, sizeof(objdump_process)); + objdump_process.argv = objdump_argv; + objdump_process.out = -1; + objdump_process.err = -1; + objdump_process.no_stderr = 1; + if (start_command(&objdump_process)) { + pr_err("Failure starting to run %s\n", command); + err = -1; + goto out_free_command; + } + + file = fdopen(objdump_process.out, "r"); + if (!file) { + pr_err("Failure creating FILE stream for %s\n", command); + /* + * If we were using debug info should retry with + * original binary. + */ + err = -1; + goto out_close_stdout; + } + + /* Storage for getline. */ + line = NULL; + line_len = 0; + + nline = 0; + while (!feof(file)) { + const char *match; + char *expanded_line; + + if (getline(&line, &line_len, file) < 0 || !line) + break; + + /* Skip lines containing "filename:" */ + match = strstr(line, filename); + if (match && match[strlen(filename)] == ':') + continue; + + expanded_line = strim(line); + expanded_line = expand_tabs(expanded_line, &line, &line_len); + if (!expanded_line) + break; + + /* + * The source code line number (lineno) needs to be kept in + * across calls to symbol__parse_objdump_line(), so that it + * can associate it with the instructions till the next one. + * See disasm_line__new() and struct disasm_line::line_nr. + */ + if (symbol__parse_objdump_line(sym, args, expanded_line, + &lineno, &fileloc) < 0) + break; + nline++; + } + free(line); + free(fileloc); + + err = finish_command(&objdump_process); + if (err) + pr_err("Error running %s\n", command); + + if (nline == 0) { + err = -1; + pr_err("No output from %s\n", command); + } + + /* + * kallsyms does not have symbol sizes so there may a nop at the end. + * Remove it. + */ + if (dso__is_kcore(dso)) + delete_last_nop(sym); + + fclose(file); + +out_close_stdout: + close(objdump_process.out); + +out_free_command: + free(command); + return err; +} + +int symbol__disassemble(struct symbol *sym, struct annotate_args *args) +{ + struct annotation_options *options = args->options; + struct map *map = args->ms->map; + struct dso *dso = map__dso(map); + char symfs_filename[PATH_MAX]; + bool delete_extract = false; + struct kcore_extract kce; + bool decomp = false; + int err = dso__disassemble_filename(dso, symfs_filename, sizeof(symfs_filename)); + + if (err) + return err; + + pr_debug("%s: filename=%s, sym=%s, start=%#" PRIx64 ", end=%#" PRIx64 "\n", __func__, + symfs_filename, sym->name, map__unmap_ip(map, sym->start), + map__unmap_ip(map, sym->end)); + + pr_debug("annotating [%p] %30s : [%p] %30s\n", dso, dso__long_name(dso), sym, sym->name); + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) { + return SYMBOL_ANNOTATE_ERRNO__COULDNT_DETERMINE_FILE_TYPE; + } else if (dso__is_kcore(dso)) { + kce.addr = map__rip_2objdump(map, sym->start); + kce.kcore_filename = symfs_filename; + kce.len = sym->end - sym->start; + kce.offs = sym->start; + + if (!kcore_extract__create(&kce)) { + delete_extract = true; + strlcpy(symfs_filename, kce.extract_filename, sizeof(symfs_filename)); + } + } else if (dso__needs_decompress(dso)) { + char tmp[KMOD_DECOMP_LEN]; + + if (dso__decompress_kmodule_path(dso, symfs_filename, tmp, sizeof(tmp)) < 0) + return -1; + + /* empty pathname means file wasn't actually compressed */ + if (tmp[0] != '\0') { + decomp = true; + strcpy(symfs_filename, tmp); + } + } + + /* + * For powerpc data type profiling, use the dso__data_read_offset to + * read raw instruction directly and interpret the binary code to + * understand instructions and register fields. For sort keys as type + * and typeoff, disassemble to mnemonic notation is not required in + * case of powerpc. + */ + if (arch__is_powerpc(args->arch)) { + extern const char *sort_order; + + if (sort_order && !strstr(sort_order, "sym")) { + err = symbol__disassemble_raw(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + + err = symbol__disassemble_capstone_powerpc(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + } + } + + /* FIXME: LLVM and CAPSTONE should support source code */ + if (options->annotate_src && !options->hide_src_code) { + err = symbol__disassemble_objdump(symfs_filename, sym, args); + if (err == 0) + goto out_remove_tmp; + } + + err = -1; + for (u8 i = 0; i < ARRAY_SIZE(options->disassemblers) && err != 0; i++) { + enum perf_disassembler dis = options->disassemblers[i]; + + switch (dis) { + case PERF_DISASM_LLVM: + args->options->disassembler_used = PERF_DISASM_LLVM; + err = symbol__disassemble_llvm(symfs_filename, sym, args); + break; + case PERF_DISASM_CAPSTONE: + args->options->disassembler_used = PERF_DISASM_CAPSTONE; + err = symbol__disassemble_capstone(symfs_filename, sym, args); + break; + case PERF_DISASM_OBJDUMP: + args->options->disassembler_used = PERF_DISASM_OBJDUMP; + err = symbol__disassemble_objdump(symfs_filename, sym, args); + break; + case PERF_DISASM_UNKNOWN: /* End of disassemblers. */ + default: + args->options->disassembler_used = PERF_DISASM_UNKNOWN; + goto out_remove_tmp; + } + if (err == 0) + pr_debug("Disassembled with %s\n", perf_disassembler__strs[dis]); + } +out_remove_tmp: + if (decomp) + unlink(symfs_filename); + + if (delete_extract) + kcore_extract__delete(&kce); + + return err; +} diff --git a/tools/perf/util/disasm.h b/tools/perf/util/disasm.h new file mode 100644 index 000000000..25756e3f4 --- /dev/null +++ b/tools/perf/util/disasm.h @@ -0,0 +1,170 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __PERF_UTIL_DISASM_H +#define __PERF_UTIL_DISASM_H + +#include "map_symbol.h" + +#ifdef HAVE_LIBDW_SUPPORT +#include "dwarf-aux.h" +#endif + +struct annotation_options; +struct disasm_line; +struct ins; +struct evsel; +struct symbol; +struct data_loc_info; +struct type_state; +struct disasm_line; + +struct e_machine_and_e_flags { + uint32_t e_flags; + uint16_t e_machine; +}; + +struct arch { + /** @name: name such as "x86" or "powerpc". */ + const char *name; + const struct ins *instructions; + size_t nr_instructions; + size_t nr_instructions_allocated; + const char *insn_suffix; + unsigned int model; + unsigned int family; + /** @id: ELF machine and flags associated with arch. */ + struct e_machine_and_e_flags id; + bool sorted_instructions; + struct { + char comment_char; + char skip_functions_char; + char register_char; + char memory_ref_char; + char imm_char; + } objdump; + bool (*ins_is_fused)(const struct arch *arch, const char *ins1, + const char *ins2); + const struct ins_ops *(*associate_instruction_ops)(struct arch *arch, const char *name); +#ifdef HAVE_LIBDW_SUPPORT + void (*update_insn_state)(struct type_state *state, + struct data_loc_info *dloc, Dwarf_Die *cu_die, + struct disasm_line *dl); +#endif +}; + +struct ins { + const char *name; + const struct ins_ops *ops; +}; + +struct ins_operands { + char *raw; + struct { + char *raw; + char *name; + struct symbol *sym; + u64 addr; + s64 offset; + bool offset_avail; + bool outside; + bool multi_regs; + bool mem_ref; + } target; + union { + struct { + char *raw; + char *name; + u64 addr; + bool multi_regs; + bool mem_ref; + } source; + struct { + struct ins ins; + struct ins_operands *ops; + } locked; + struct { + char *raw_comment; + char *raw_func_start; + } jump; + }; +}; + +struct ins_ops { + void (*free)(struct ins_operands *ops); + int (*parse)(const struct arch *arch, struct ins_operands *ops, struct map_symbol *ms, + struct disasm_line *dl); + int (*scnprintf)(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); + bool is_jump; + bool is_call; +}; + +struct annotate_args { + const struct arch *arch; + struct map_symbol *ms; + struct annotation_options *options; + s64 offset; + char *line; + int line_nr; + char *fileloc; +}; + +const struct arch *arch__find(uint16_t e_machine, uint32_t e_flags, const char *cpuid); +bool arch__is_x86(const struct arch *arch); +bool arch__is_powerpc(const struct arch *arch); + +extern const struct ins_ops call_ops; +extern const struct ins_ops dec_ops; +extern const struct ins_ops jump_ops; +extern const struct ins_ops mov_ops; +extern const struct ins_ops nop_ops; +extern const struct ins_ops lock_ops; +extern const struct ins_ops ret_ops; + +int arch__associate_ins_ops(struct arch *arch, const char *name, const struct ins_ops *ops); + +const struct arch *arch__new_arc(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_arm(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_arm64(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_csky(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_loongarch(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_mips(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_powerpc(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_riscv64(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_s390(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_sparc(const struct e_machine_and_e_flags *id, const char *cpuid); +const struct arch *arch__new_x86(const struct e_machine_and_e_flags *id, const char *cpuid); + +const struct ins_ops *ins__find(const struct arch *arch, const char *name, struct disasm_line *dl); + +bool ins__is_call(const struct ins *ins); +bool ins__is_jump(const struct ins *ins); +bool ins__is_fused(const struct arch *arch, const char *ins1, const char *ins2); +bool ins__is_ret(const struct ins *ins); +bool ins__is_lock(const struct ins *ins); + +const struct ins_ops *check_ppc_insn(struct disasm_line *dl); + +struct disasm_line *disasm_line__new(struct annotate_args *args); +void disasm_line__free(struct disasm_line *dl); + +int disasm_line__scnprintf(struct disasm_line *dl, char *bf, size_t size, + bool raw, int max_ins_name); + +int ins__raw_scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int ins__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int call__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int jump__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); +int mov__scnprintf(const struct ins *ins, char *bf, size_t size, + struct ins_operands *ops, int max_ins_name); + +void jump__delete(struct ins_operands *ops); + +int symbol__disassemble(struct symbol *sym, struct annotate_args *args); + +char *expand_tabs(char *line, char **storage, size_t *storage_len); + +#endif /* __PERF_UTIL_DISASM_H */ diff --git a/tools/perf/util/dlfilter.c b/tools/perf/util/dlfilter.c new file mode 100644 index 000000000..e11e144af --- /dev/null +++ b/tools/perf/util/dlfilter.c @@ -0,0 +1,662 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dlfilter.c: Interface to perf script --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "dso.h" +#include "map.h" +#include "thread.h" +#include "trace-event.h" +#include "symbol.h" +#include "srcline.h" +#include "dlfilter.h" +#include "../include/perf/perf_dlfilter.h" + +static void al_to_d_al(struct addr_location *al, struct perf_dlfilter_al *d_al) +{ + struct symbol *sym = al->sym; + + d_al->size = sizeof(*d_al); + if (al->map) { + struct dso *dso = map__dso(al->map); + + if (symbol_conf.show_kernel_path && dso__long_name(dso)) + d_al->dso = dso__long_name(dso); + else + d_al->dso = dso__name(dso); + d_al->is_64_bit = dso__is_64_bit(dso); + d_al->buildid_size = dso__bid(dso)->size; + d_al->buildid = dso__bid(dso)->data; + } else { + d_al->dso = NULL; + d_al->is_64_bit = 0; + d_al->buildid_size = 0; + d_al->buildid = NULL; + } + if (sym) { + d_al->sym = sym->name; + d_al->sym_start = sym->start; + d_al->sym_end = sym->end; + if (al->addr < sym->end) + d_al->symoff = al->addr - sym->start; + else if (al->map) + d_al->symoff = al->addr - map__start(al->map) - sym->start; + else + d_al->symoff = 0; + d_al->sym_binding = symbol__binding(sym); + } else { + d_al->sym = NULL; + d_al->sym_start = 0; + d_al->sym_end = 0; + d_al->symoff = 0; + d_al->sym_binding = 0; + } + d_al->addr = al->addr; + d_al->comm = NULL; + d_al->filtered = 0; + d_al->priv = NULL; +} + +static struct addr_location *get_al(struct dlfilter *d) +{ + struct addr_location *al = d->al; + + if (!al->thread && machine__resolve(d->machine, al, d->sample) < 0) + return NULL; + return al; +} + +static struct thread *get_thread(struct dlfilter *d) +{ + struct addr_location *al = get_al(d); + + return al ? al->thread : NULL; +} + +static const struct perf_dlfilter_al *dlfilter__resolve_ip(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al *d_al = d->d_ip_al; + struct addr_location *al; + + if (!d->ctx_valid) + return NULL; + + /* 'size' is also used to indicate already initialized */ + if (d_al->size) + return d_al; + + al = get_al(d); + if (!al) + return NULL; + + al_to_d_al(al, d_al); + + d_al->is_kernel_ip = machine__kernel_ip(d->machine, d->sample->ip); + d_al->comm = al->thread ? thread__comm_str(al->thread) : ":-1"; + d_al->filtered = al->filtered; + + return d_al; +} + +static const struct perf_dlfilter_al *dlfilter__resolve_addr(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al *d_addr_al = d->d_addr_al; + struct addr_location *addr_al = d->addr_al; + + if (!d->ctx_valid || !d->d_sample->addr_correlates_sym) + return NULL; + + /* 'size' is also used to indicate already initialized */ + if (d_addr_al->size) + return d_addr_al; + + if (!addr_al->thread) { + struct thread *thread = get_thread(d); + + if (!thread) + return NULL; + thread__resolve(thread, addr_al, d->sample); + } + + al_to_d_al(addr_al, d_addr_al); + + d_addr_al->is_kernel_ip = machine__kernel_ip(d->machine, d->sample->addr); + + return d_addr_al; +} + +static char **dlfilter__args(void *ctx, int *dlargc) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (dlargc) + *dlargc = 0; + else + return NULL; + + if (!d->ctx_valid && !d->in_start && !d->in_stop) + return NULL; + + *dlargc = d->dlargc; + return d->dlargv; +} + +static bool has_priv(struct perf_dlfilter_al *d_al_p) +{ + return d_al_p->size >= offsetof(struct perf_dlfilter_al, priv) + sizeof(d_al_p->priv); +} + +static __s32 dlfilter__resolve_address(void *ctx, __u64 address, struct perf_dlfilter_al *d_al_p) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al d_al; + struct addr_location al; + struct thread *thread; + __u32 sz; + + if (!d->ctx_valid || !d_al_p) + return -1; + + thread = get_thread(d); + if (!thread) + return -1; + + addr_location__init(&al); + thread__find_symbol_fb(thread, d->sample->cpumode, address, &al); + + al_to_d_al(&al, &d_al); + + d_al.is_kernel_ip = machine__kernel_ip(d->machine, address); + + sz = d_al_p->size; + memcpy(d_al_p, &d_al, min((size_t)sz, sizeof(d_al))); + d_al_p->size = sz; + + if (has_priv(d_al_p)) + d_al_p->priv = memdup(&al, sizeof(al)); + else /* Avoid leak for v0 API */ + addr_location__exit(&al); + + return 0; +} + +static void dlfilter__al_cleanup(void *ctx __maybe_unused, struct perf_dlfilter_al *d_al_p) +{ + struct addr_location *al; + + /* Ensure backward compatibility */ + if (!has_priv(d_al_p) || !d_al_p->priv) + return; + + al = d_al_p->priv; + + d_al_p->priv = NULL; + + addr_location__exit(al); + + free(al); +} + +static const __u8 *dlfilter__insn(void *ctx, __u32 *len) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (!len) + return NULL; + + *len = 0; + + if (!d->ctx_valid) + return NULL; + + if (d->sample->ip && !d->sample->insn_len) { + struct addr_location *al = d->al; + + if (!al->thread && machine__resolve(d->machine, al, d->sample) < 0) + return NULL; + + if (thread__maps(al->thread)) { + struct machine *machine = maps__machine(thread__maps(al->thread)); + + if (machine) + perf_sample__fetch_insn(d->sample, al->thread, machine); + } + } + + if (!d->sample->insn_len) + return NULL; + + *len = d->sample->insn_len; + + return (__u8 *)d->sample->insn; +} + +static const char *dlfilter__srcline(void *ctx, __u32 *line_no) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct addr_location *al; + unsigned int line = 0; + char *srcfile = NULL; + struct map *map; + struct dso *dso; + u64 addr; + + if (!d->ctx_valid || !line_no) + return NULL; + + al = get_al(d); + if (!al) + return NULL; + + map = al->map; + addr = al->addr; + dso = map ? map__dso(map) : NULL; + + if (dso) + srcfile = get_srcline_split(dso, map__rip_2objdump(map, addr), &line); + + *line_no = line; + return srcfile; +} + +static struct perf_event_attr *dlfilter__attr(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (!d->ctx_valid) + return NULL; + + return &d->evsel->core.attr; +} + +static __s32 code_read(__u64 ip, struct map *map, struct machine *machine, void *buf, __u32 len) +{ + u64 offset = map__map_ip(map, ip); + + if (ip + len >= map__end(map)) + len = map__end(map) - ip; + + return dso__data_read_offset(map__dso(map), machine, offset, buf, len); +} + +static __s32 dlfilter__object_code(void *ctx, __u64 ip, void *buf, __u32 len) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct addr_location *al; + struct addr_location a; + __s32 ret; + + if (!d->ctx_valid) + return -1; + + al = get_al(d); + if (!al) + return -1; + + if (al->map && ip >= map__start(al->map) && ip < map__end(al->map) && + machine__kernel_ip(d->machine, ip) == machine__kernel_ip(d->machine, d->sample->ip)) + return code_read(ip, al->map, d->machine, buf, len); + + addr_location__init(&a); + + thread__find_map_fb(al->thread, d->sample->cpumode, ip, &a); + ret = a.map ? code_read(ip, a.map, d->machine, buf, len) : -1; + + addr_location__exit(&a); + + return ret; +} + +static const struct perf_dlfilter_fns perf_dlfilter_fns = { + .resolve_ip = dlfilter__resolve_ip, + .resolve_addr = dlfilter__resolve_addr, + .args = dlfilter__args, + .resolve_address = dlfilter__resolve_address, + .al_cleanup = dlfilter__al_cleanup, + .insn = dlfilter__insn, + .srcline = dlfilter__srcline, + .attr = dlfilter__attr, + .object_code = dlfilter__object_code, +}; + +static char *find_dlfilter(const char *file) +{ + char path[PATH_MAX]; + char *exec_path; + + if (strchr(file, '/')) + goto out; + + if (!access(file, R_OK)) { + /* + * Prepend "./" so that dlopen will find the file in the + * current directory. + */ + snprintf(path, sizeof(path), "./%s", file); + file = path; + goto out; + } + + exec_path = get_argv_exec_path(); + if (!exec_path) + goto out; + snprintf(path, sizeof(path), "%s/dlfilters/%s", exec_path, file); + free(exec_path); + if (!access(path, R_OK)) + file = path; +out: + return strdup(file); +} + +#define CHECK_FLAG(x) BUILD_BUG_ON((u64)PERF_DLFILTER_FLAG_ ## x != (u64)PERF_IP_FLAG_ ## x) + +static int dlfilter__init(struct dlfilter *d, const char *file, int dlargc, char **dlargv) +{ + CHECK_FLAG(BRANCH); + CHECK_FLAG(CALL); + CHECK_FLAG(RETURN); + CHECK_FLAG(CONDITIONAL); + CHECK_FLAG(SYSCALLRET); + CHECK_FLAG(ASYNC); + CHECK_FLAG(INTERRUPT); + CHECK_FLAG(TX_ABORT); + CHECK_FLAG(TRACE_BEGIN); + CHECK_FLAG(TRACE_END); + CHECK_FLAG(IN_TX); + CHECK_FLAG(VMENTRY); + CHECK_FLAG(VMEXIT); + + memset(d, 0, sizeof(*d)); + d->file = find_dlfilter(file); + if (!d->file) + return -1; + d->dlargc = dlargc; + d->dlargv = dlargv; + return 0; +} + +static void dlfilter__exit(struct dlfilter *d) +{ + zfree(&d->file); +} + +static int dlfilter__open(struct dlfilter *d) +{ + d->handle = dlopen(d->file, RTLD_NOW); + if (!d->handle) { + pr_err("dlopen failed for: '%s'\n", d->file); + return -1; + } + d->start = dlsym(d->handle, "start"); + d->filter_event = dlsym(d->handle, "filter_event"); + d->filter_event_early = dlsym(d->handle, "filter_event_early"); + d->stop = dlsym(d->handle, "stop"); + d->fns = dlsym(d->handle, "perf_dlfilter_fns"); + if (d->fns) + memcpy(d->fns, &perf_dlfilter_fns, sizeof(struct perf_dlfilter_fns)); + return 0; +} + +static int dlfilter__close(struct dlfilter *d) +{ + return dlclose(d->handle); +} + +struct dlfilter *dlfilter__new(const char *file, int dlargc, char **dlargv) +{ + struct dlfilter *d = malloc(sizeof(*d)); + + if (!d) + return NULL; + + if (dlfilter__init(d, file, dlargc, dlargv)) + goto err_free; + + if (dlfilter__open(d)) + goto err_exit; + + return d; + +err_exit: + dlfilter__exit(d); +err_free: + free(d); + return NULL; +} + +static void dlfilter__free(struct dlfilter *d) +{ + if (d) { + dlfilter__exit(d); + free(d); + } +} + +int dlfilter__start(struct dlfilter *d, struct perf_session *session) +{ + if (d) { + d->session = session; + if (d->start) { + int ret; + + d->in_start = true; + ret = d->start(&d->data, d); + d->in_start = false; + return ret; + } + } + return 0; +} + +static int dlfilter__stop(struct dlfilter *d) +{ + if (d && d->stop) { + int ret; + + d->in_stop = true; + ret = d->stop(d->data, d); + d->in_stop = false; + return ret; + } + return 0; +} + +void dlfilter__cleanup(struct dlfilter *d) +{ + if (d) { + dlfilter__stop(d); + dlfilter__close(d); + dlfilter__free(d); + } +} + +#define ASSIGN(x) d_sample.x = sample->x + +int dlfilter__do_filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al, + bool early) +{ + struct perf_dlfilter_sample d_sample; + struct perf_dlfilter_al d_ip_al; + struct perf_dlfilter_al d_addr_al; + int ret; + + d->event = event; + d->sample = sample; + d->evsel = evsel; + d->machine = machine; + d->al = al; + d->addr_al = addr_al; + d->d_sample = &d_sample; + d->d_ip_al = &d_ip_al; + d->d_addr_al = &d_addr_al; + + d_sample.size = sizeof(d_sample); + d_sample.p_stage_cyc = sample->weight3; + d_ip_al.size = 0; /* To indicate d_ip_al is not initialized */ + d_addr_al.size = 0; /* To indicate d_addr_al is not initialized */ + + ASSIGN(ip); + ASSIGN(pid); + ASSIGN(tid); + ASSIGN(time); + ASSIGN(addr); + ASSIGN(id); + ASSIGN(stream_id); + ASSIGN(period); + ASSIGN(weight); + ASSIGN(ins_lat); + ASSIGN(transaction); + ASSIGN(insn_cnt); + ASSIGN(cyc_cnt); + ASSIGN(cpu); + ASSIGN(flags); + ASSIGN(data_src); + ASSIGN(phys_addr); + ASSIGN(data_page_size); + ASSIGN(code_page_size); + ASSIGN(cgroup); + ASSIGN(cpumode); + ASSIGN(misc); + ASSIGN(raw_size); + ASSIGN(raw_data); + ASSIGN(machine_pid); + ASSIGN(vcpu); + + if (sample->branch_stack) { + d_sample.brstack_nr = sample->branch_stack->nr; + d_sample.brstack = (struct perf_branch_entry *)perf_sample__branch_entries(sample); + } else { + d_sample.brstack_nr = 0; + d_sample.brstack = NULL; + } + + if (sample->callchain) { + d_sample.raw_callchain_nr = sample->callchain->nr; + d_sample.raw_callchain = (__u64 *)sample->callchain->ips; + } else { + d_sample.raw_callchain_nr = 0; + d_sample.raw_callchain = NULL; + } + + d_sample.addr_correlates_sym = + (evsel->core.attr.sample_type & PERF_SAMPLE_ADDR) && + sample_addr_correlates_sym(&evsel->core.attr); + + d_sample.event = evsel__name(evsel); + + d->ctx_valid = true; + + if (early) + ret = d->filter_event_early(d->data, &d_sample, d); + else + ret = d->filter_event(d->data, &d_sample, d); + + d->ctx_valid = false; + + return ret; +} + +bool get_filter_desc(const char *dirname, const char *name, char **desc, + char **long_desc) +{ + char path[PATH_MAX]; + void *handle; + const char *(*desc_fn)(const char **long_description); + + snprintf(path, sizeof(path), "%s/%s", dirname, name); + handle = dlopen(path, RTLD_NOW); + if (!handle || !(dlsym(handle, "filter_event") || dlsym(handle, "filter_event_early"))) + return false; + desc_fn = dlsym(handle, "filter_description"); + if (desc_fn) { + const char *dsc; + const char *long_dsc; + + dsc = desc_fn(&long_dsc); + if (dsc) + *desc = strdup(dsc); + if (long_dsc) + *long_desc = strdup(long_dsc); + } + dlclose(handle); + return true; +} + +static void list_filters(const char *dirname) +{ + struct dirent *entry; + DIR *dir; + + dir = opendir(dirname); + if (!dir) + return; + + while ((entry = readdir(dir)) != NULL) + { + size_t n = strlen(entry->d_name); + char *long_desc = NULL; + char *desc = NULL; + + if (entry->d_type == DT_DIR || n < 4 || + strcmp(".so", entry->d_name + n - 3)) + continue; + if (!get_filter_desc(dirname, entry->d_name, &desc, &long_desc)) + continue; + printf(" %-36s %s\n", entry->d_name, desc ? desc : ""); + if (verbose > 0) { + char *p = long_desc; + char *line; + + while ((line = strsep(&p, "\n")) != NULL) + printf("%39s%s\n", "", line); + } + free(long_desc); + free(desc); + } + + closedir(dir); +} + +int list_available_dlfilters(const struct option *opt __maybe_unused, + const char *s __maybe_unused, + int unset __maybe_unused) +{ + char path[PATH_MAX]; + char *exec_path; + + printf("List of available dlfilters:\n"); + + list_filters("."); + + exec_path = get_argv_exec_path(); + if (!exec_path) + goto out; + snprintf(path, sizeof(path), "%s/dlfilters", exec_path); + + list_filters(path); + + free(exec_path); +out: + exit(0); +} diff --git a/tools/perf/util/dlfilter.h b/tools/perf/util/dlfilter.h new file mode 100644 index 000000000..cc4bb9657 --- /dev/null +++ b/tools/perf/util/dlfilter.h @@ -0,0 +1,99 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * dlfilter.h: Interface to perf script --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ + +#ifndef PERF_UTIL_DLFILTER_H +#define PERF_UTIL_DLFILTER_H + +struct perf_session; +union perf_event; +struct perf_sample; +struct evsel; +struct machine; +struct addr_location; +struct perf_dlfilter_fns; +struct perf_dlfilter_sample; +struct perf_dlfilter_al; + +struct dlfilter { + char *file; + void *handle; + void *data; + struct perf_session *session; + bool ctx_valid; + bool in_start; + bool in_stop; + int dlargc; + char **dlargv; + + union perf_event *event; + struct perf_sample *sample; + struct evsel *evsel; + struct machine *machine; + struct addr_location *al; + struct addr_location *addr_al; + struct perf_dlfilter_sample *d_sample; + struct perf_dlfilter_al *d_ip_al; + struct perf_dlfilter_al *d_addr_al; + + int (*start)(void **data, void *ctx); + int (*stop)(void *data, void *ctx); + + int (*filter_event)(void *data, + const struct perf_dlfilter_sample *sample, + void *ctx); + int (*filter_event_early)(void *data, + const struct perf_dlfilter_sample *sample, + void *ctx); + + struct perf_dlfilter_fns *fns; +}; + +struct dlfilter *dlfilter__new(const char *file, int dlargc, char **dlargv); + +int dlfilter__start(struct dlfilter *d, struct perf_session *session); + +int dlfilter__do_filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al, + bool early); + +void dlfilter__cleanup(struct dlfilter *d); + +static inline int dlfilter__filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al) +{ + if (!d || !d->filter_event) + return 0; + return dlfilter__do_filter_event(d, event, sample, evsel, machine, al, addr_al, false); +} + +static inline int dlfilter__filter_event_early(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al) +{ + if (!d || !d->filter_event_early) + return 0; + return dlfilter__do_filter_event(d, event, sample, evsel, machine, al, addr_al, true); +} + +int list_available_dlfilters(const struct option *opt, const char *s, int unset); +bool get_filter_desc(const char *dirname, const char *name, char **desc, + char **long_desc); + +#endif diff --git a/tools/perf/util/drm_pmu.c b/tools/perf/util/drm_pmu.c new file mode 100644 index 000000000..b8badae70 --- /dev/null +++ b/tools/perf/util/drm_pmu.c @@ -0,0 +1,685 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "drm_pmu.h" +#include "counts.h" +#include "cpumap.h" +#include "debug.h" +#include "evsel.h" +#include "pmu.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum drm_pmu_unit { + DRM_PMU_UNIT_BYTES, + DRM_PMU_UNIT_CAPACITY, + DRM_PMU_UNIT_CYCLES, + DRM_PMU_UNIT_HZ, + DRM_PMU_UNIT_NS, + + DRM_PMU_UNIT_MAX, +}; + +struct drm_pmu_event { + const char *name; + const char *desc; + enum drm_pmu_unit unit; +}; + +struct drm_pmu { + struct perf_pmu pmu; + struct drm_pmu_event *events; + int num_events; +}; + +static const char * const drm_pmu_unit_strs[DRM_PMU_UNIT_MAX] = { + "bytes", + "capacity", + "cycles", + "hz", + "ns", +}; + +static const char * const drm_pmu_scale_unit_strs[DRM_PMU_UNIT_MAX] = { + "1bytes", + "1capacity", + "1cycles", + "1hz", + "1ns", +}; + +bool perf_pmu__is_drm(const struct perf_pmu *pmu) +{ + return pmu && pmu->type >= PERF_PMU_TYPE_DRM_START && + pmu->type <= PERF_PMU_TYPE_DRM_END; +} + +bool evsel__is_drm(const struct evsel *evsel) +{ + return perf_pmu__is_drm(evsel->pmu); +} + +static struct drm_pmu *add_drm_pmu(struct list_head *pmus, char *line, size_t line_len) +{ + struct drm_pmu *drm; + struct perf_pmu *pmu; + const char *name; + __u32 max_drm_pmu_type = 0, type; + int i = 12; + + if (line[line_len - 1] == '\n') + line[line_len - 1] = '\0'; + while (isspace(line[i])) + i++; + + line[--i] = '_'; + line[--i] = 'm'; + line[--i] = 'r'; + line[--i] = 'd'; + name = &line[i]; + + list_for_each_entry(pmu, pmus, list) { + if (!perf_pmu__is_drm(pmu)) + continue; + if (pmu->type > max_drm_pmu_type) + max_drm_pmu_type = pmu->type; + if (!strcmp(pmu->name, name)) { + /* PMU already exists. */ + return NULL; + } + } + + if (max_drm_pmu_type != 0) + type = max_drm_pmu_type + 1; + else + type = PERF_PMU_TYPE_DRM_START; + + if (type > PERF_PMU_TYPE_DRM_END) { + zfree(&drm); + pr_err("Unable to encode DRM PMU type for %s\n", name); + return NULL; + } + + drm = zalloc(sizeof(*drm)); + if (!drm) + return NULL; + + if (perf_pmu__init(&drm->pmu, type, name) != 0) { + perf_pmu__delete(&drm->pmu); + return NULL; + } + + drm->pmu.cpus = perf_cpu_map__new_int(0); + if (!drm->pmu.cpus) { + perf_pmu__delete(&drm->pmu); + return NULL; + } + return drm; +} + + +static int add_event(struct drm_pmu_event **events, int *num_events, + const char *line, enum drm_pmu_unit unit, const char *desc) +{ + const char *colon = strchr(line, ':'); + struct drm_pmu_event *tmp; + + if (!colon) + return -EINVAL; + + tmp = reallocarray(*events, *num_events + 1, sizeof(struct drm_pmu_event)); + if (!tmp) + return -ENOMEM; + tmp[*num_events].unit = unit; + tmp[*num_events].desc = desc; + tmp[*num_events].name = strndup(line, colon - line); + if (!tmp[*num_events].name) + return -ENOMEM; + (*num_events)++; + *events = tmp; + return 0; +} + +static int read_drm_pmus_cb(void *args, int fdinfo_dir_fd, const char *fd_name) +{ + struct list_head *pmus = args; + char buf[640]; + struct io io; + char *line = NULL; + size_t line_len; + struct drm_pmu *drm = NULL; + struct drm_pmu_event *events = NULL; + int num_events = 0; + + io__init(&io, openat(fdinfo_dir_fd, fd_name, O_RDONLY), buf, sizeof(buf)); + if (io.fd == -1) { + /* Failed to open file, ignore. */ + return 0; + } + + while (io__getline(&io, &line, &line_len) > 0) { + if (strstarts(line, "drm-driver:")) { + drm = add_drm_pmu(pmus, line, line_len); + if (!drm) + break; + continue; + } + /* + * Note the string matching below is alphabetical, with more + * specific matches appearing before less specific. + */ + if (strstarts(line, "drm-active-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Total memory active in one or more engines"); + continue; + } + if (strstarts(line, "drm-cycles-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_CYCLES, + "Busy cycles"); + continue; + } + if (strstarts(line, "drm-engine-capacity-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_CAPACITY, + "Engine capacity"); + continue; + } + if (strstarts(line, "drm-engine-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_NS, + "Utilization in ns"); + continue; + } + if (strstarts(line, "drm-maxfreq-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_HZ, + "Maximum frequency"); + continue; + } + if (strstarts(line, "drm-purgeable-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of resident and purgeable memory buffers"); + continue; + } + if (strstarts(line, "drm-resident-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of resident memory buffers"); + continue; + } + if (strstarts(line, "drm-shared-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of shared memory buffers"); + continue; + } + if (strstarts(line, "drm-total-cycles-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Total busy cycles"); + continue; + } + if (strstarts(line, "drm-total-")) { + add_event(&events, &num_events, line, DRM_PMU_UNIT_BYTES, + "Size of shared and private memory"); + continue; + } + if (verbose > 1 && strstarts(line, "drm-") && + !strstarts(line, "drm-client-id:") && + !strstarts(line, "drm-pdev:")) + pr_debug("Unhandled DRM PMU fdinfo line match '%s'\n", line); + } + if (drm) { + drm->events = events; + drm->num_events = num_events; + list_add_tail(&drm->pmu.list, pmus); + } + free(line); + if (io.fd != -1) + close(io.fd); + return 0; +} + +void drm_pmu__exit(struct perf_pmu *pmu) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + free(drm->events); +} + +bool drm_pmu__have_event(const struct perf_pmu *pmu, const char *name) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + if (!strstarts(name, "drm-")) + return false; + + for (int i = 0; i < drm->num_events; i++) { + if (!strcasecmp(drm->events[i].name, name)) + return true; + } + return false; +} + +int drm_pmu__for_each_event(const struct perf_pmu *pmu, void *state, pmu_event_callback cb) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + for (int i = 0; i < drm->num_events; i++) { + char encoding_buf[128]; + struct pmu_event_info info = { + .pmu = pmu, + .name = drm->events[i].name, + .alias = NULL, + .scale_unit = drm_pmu_scale_unit_strs[drm->events[i].unit], + .desc = drm->events[i].desc, + .long_desc = NULL, + .encoding_desc = encoding_buf, + .topic = "drm", + .pmu_name = pmu->name, + .event_type_desc = "DRM event", + }; + int ret; + + snprintf(encoding_buf, sizeof(encoding_buf), "%s/config=0x%x/", pmu->name, i); + + ret = cb(state, &info); + if (ret) + return ret; + } + return 0; +} + +size_t drm_pmu__num_events(const struct perf_pmu *pmu) +{ + const struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + + return drm->num_events; +} + +static int drm_pmu__index_for_event(const struct drm_pmu *drm, const char *name) +{ + for (int i = 0; i < drm->num_events; i++) { + if (!strcmp(drm->events[i].name, name)) + return i; + } + return -1; +} + +static int drm_pmu__config_term(const struct drm_pmu *drm, + struct perf_event_attr *attr, + struct parse_events_term *term, + struct parse_events_error *err) +{ + if (term->type_term == PARSE_EVENTS__TERM_TYPE_USER) { + int i = drm_pmu__index_for_event(drm, term->config); + + if (i >= 0) { + attr->config = i; + return 0; + } + } + if (err) { + char *err_str; + + parse_events_error__handle(err, term->err_val, + asprintf(&err_str, + "unexpected drm event term (%s) %s", + parse_events__term_type_str(term->type_term), + term->config) < 0 + ? strdup("unexpected drm event term") + : err_str, + NULL); + } + return -EINVAL; +} + +int drm_pmu__config_terms(const struct perf_pmu *pmu, + struct perf_event_attr *attr, + struct parse_events_terms *terms, + struct parse_events_error *err) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + struct parse_events_term *term; + + list_for_each_entry(term, &terms->terms, list) { + if (drm_pmu__config_term(drm, attr, term, err)) + return -EINVAL; + } + + return 0; +} + +int drm_pmu__check_alias(const struct perf_pmu *pmu, struct parse_events_terms *terms, + struct perf_pmu_info *info, struct parse_events_error *err) +{ + struct drm_pmu *drm = container_of(pmu, struct drm_pmu, pmu); + struct parse_events_term *term = + list_first_entry(&terms->terms, struct parse_events_term, list); + + if (term->type_term == PARSE_EVENTS__TERM_TYPE_USER) { + int i = drm_pmu__index_for_event(drm, term->config); + + if (i >= 0) { + info->unit = drm_pmu_unit_strs[drm->events[i].unit]; + info->scale = 1; + return 0; + } + } + if (err) { + char *err_str; + + parse_events_error__handle(err, term->err_val, + asprintf(&err_str, + "unexpected drm event term (%s) %s", + parse_events__term_type_str(term->type_term), + term->config) < 0 + ? strdup("unexpected drm event term") + : err_str, + NULL); + } + return -EINVAL; +} + +struct minor_info { + unsigned int *minors; + int minors_num, minors_len; +}; + +static int for_each_drm_fdinfo_in_dir(int (*cb)(void *args, int fdinfo_dir_fd, const char *fd_name), + void *args, int proc_dir, const char *pid_name, + struct minor_info *minors) +{ + char buf[256]; + DIR *fd_dir; + struct dirent *fd_entry; + int fd_dir_fd, fdinfo_dir_fd = -1; + + + scnprintf(buf, sizeof(buf), "%s/fd", pid_name); + fd_dir_fd = openat(proc_dir, buf, O_DIRECTORY); + if (fd_dir_fd == -1) + return 0; /* Presumably lost race to open. */ + fd_dir = fdopendir(fd_dir_fd); + if (!fd_dir) { + close(fd_dir_fd); + return -ENOMEM; + } + while ((fd_entry = readdir(fd_dir)) != NULL) { + struct stat stat; + unsigned int minor; + bool is_dup = false; + int ret; + + if (fd_entry->d_type != DT_LNK) + continue; + + if (fstatat(fd_dir_fd, fd_entry->d_name, &stat, 0) != 0) + continue; + + if ((stat.st_mode & S_IFMT) != S_IFCHR || major(stat.st_rdev) != 226) + continue; + + minor = minor(stat.st_rdev); + for (int i = 0; i < minors->minors_num; i++) { + if (minor(stat.st_rdev) == minors->minors[i]) { + is_dup = true; + break; + } + } + if (is_dup) + continue; + + if (minors->minors_num == minors->minors_len) { + unsigned int *tmp = reallocarray(minors->minors, minors->minors_len + 4, + sizeof(unsigned int)); + + if (tmp) { + minors->minors = tmp; + minors->minors_len += 4; + } + } + minors->minors[minors->minors_num++] = minor; + if (fdinfo_dir_fd == -1) { + /* Open fdinfo dir if we have a DRM fd. */ + scnprintf(buf, sizeof(buf), "%s/fdinfo", pid_name); + fdinfo_dir_fd = openat(proc_dir, buf, O_DIRECTORY); + if (fdinfo_dir_fd == -1) + continue; + } + ret = cb(args, fdinfo_dir_fd, fd_entry->d_name); + if (ret) + goto close_fdinfo; + } + +close_fdinfo: + if (fdinfo_dir_fd != -1) + close(fdinfo_dir_fd); + closedir(fd_dir); + return 0; +} + +static int for_each_drm_fdinfo(bool skip_all_duplicates, + int (*cb)(void *args, int fdinfo_dir_fd, const char *fd_name), + void *args) +{ + DIR *proc_dir; + struct dirent *proc_entry; + int ret; + /* + * minors maintains an array of DRM minor device numbers seen for a pid, + * or for all pids if skip_all_duplicates is true, so that duplicates + * are ignored. + */ + struct minor_info minors = { + .minors = NULL, + .minors_num = 0, + .minors_len = 0, + }; + + proc_dir = opendir(procfs__mountpoint()); + if (!proc_dir) + return 0; + + /* Walk through the /proc directory. */ + while ((proc_entry = readdir(proc_dir)) != NULL) { + if (proc_entry->d_type != DT_DIR || + !isdigit(proc_entry->d_name[0])) + continue; + if (!skip_all_duplicates) { + /* Reset the seen minor numbers for each pid. */ + minors.minors_num = 0; + } + ret = for_each_drm_fdinfo_in_dir(cb, args, + dirfd(proc_dir), proc_entry->d_name, + &minors); + if (ret) + break; + } + free(minors.minors); + closedir(proc_dir); + return ret; +} + +int perf_pmus__read_drm_pmus(struct list_head *pmus) +{ + return for_each_drm_fdinfo(/*skip_all_duplicates=*/true, read_drm_pmus_cb, pmus); +} + +int evsel__drm_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx) +{ + (void)evsel; + (void)threads; + (void)start_cpu_map_idx; + (void)end_cpu_map_idx; + return 0; +} + +static uint64_t read_count_and_apply_unit(const char *count_and_unit, enum drm_pmu_unit unit) +{ + char *unit_ptr = NULL; + uint64_t count = strtoul(count_and_unit, &unit_ptr, 10); + + if (!unit_ptr) + return 0; + + while (isblank(*unit_ptr)) + unit_ptr++; + + switch (unit) { + case DRM_PMU_UNIT_BYTES: + if (*unit_ptr == '\0') + assert(count == 0); /* Generally undocumented, happens for 0. */ + else if (!strcmp(unit_ptr, "KiB")) + count *= 1024; + else if (!strcmp(unit_ptr, "MiB")) + count *= 1024 * 1024; + else + pr_err("Unexpected bytes unit '%s'\n", unit_ptr); + break; + case DRM_PMU_UNIT_CAPACITY: + /* No units expected. */ + break; + case DRM_PMU_UNIT_CYCLES: + /* No units expected. */ + break; + case DRM_PMU_UNIT_HZ: + if (!strcmp(unit_ptr, "Hz")) + count *= 1; + else if (!strcmp(unit_ptr, "KHz")) + count *= 1000; + else if (!strcmp(unit_ptr, "MHz")) + count *= 1000000; + else + pr_err("Unexpected hz unit '%s'\n", unit_ptr); + break; + case DRM_PMU_UNIT_NS: + /* Only unit ns expected. */ + break; + case DRM_PMU_UNIT_MAX: + default: + break; + } + return count; +} + +static uint64_t read_drm_event(int fdinfo_dir_fd, const char *fd_name, + const char *match, enum drm_pmu_unit unit) +{ + char buf[640]; + struct io io; + char *line = NULL; + size_t line_len; + uint64_t count = 0; + + io__init(&io, openat(fdinfo_dir_fd, fd_name, O_RDONLY), buf, sizeof(buf)); + if (io.fd == -1) { + /* Failed to open file, ignore. */ + return 0; + } + while (io__getline(&io, &line, &line_len) > 0) { + size_t i = strlen(match); + + if (strncmp(line, match, i)) + continue; + if (line[i] != ':') + continue; + while (isblank(line[++i])) + ; + if (line[line_len - 1] == '\n') + line[line_len - 1] = '\0'; + count = read_count_and_apply_unit(&line[i], unit); + break; + } + free(line); + close(io.fd); + return count; +} + +struct read_drm_event_cb_args { + const char *match; + uint64_t count; + enum drm_pmu_unit unit; +}; + +static int read_drm_event_cb(void *vargs, int fdinfo_dir_fd, const char *fd_name) +{ + struct read_drm_event_cb_args *args = vargs; + + args->count += read_drm_event(fdinfo_dir_fd, fd_name, args->match, args->unit); + return 0; +} + +static uint64_t drm_pmu__read_system_wide(struct drm_pmu *drm, struct evsel *evsel) +{ + struct read_drm_event_cb_args args = { + .count = 0, + .match = drm->events[evsel->core.attr.config].name, + .unit = drm->events[evsel->core.attr.config].unit, + }; + + for_each_drm_fdinfo(/*skip_all_duplicates=*/false, read_drm_event_cb, &args); + return args.count; +} + +static uint64_t drm_pmu__read_for_pid(struct drm_pmu *drm, struct evsel *evsel, int pid) +{ + struct read_drm_event_cb_args args = { + .count = 0, + .match = drm->events[evsel->core.attr.config].name, + .unit = drm->events[evsel->core.attr.config].unit, + }; + struct minor_info minors = { + .minors = NULL, + .minors_num = 0, + .minors_len = 0, + }; + int proc_dir = open(procfs__mountpoint(), O_DIRECTORY); + char pid_name[12]; + int ret; + + if (proc_dir < 0) + return 0; + + snprintf(pid_name, sizeof(pid_name), "%d", pid); + ret = for_each_drm_fdinfo_in_dir(read_drm_event_cb, &args, proc_dir, pid_name, &minors); + free(minors.minors); + close(proc_dir); + return ret == 0 ? args.count : 0; +} + +int evsel__drm_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread) +{ + struct drm_pmu *drm = container_of(evsel->pmu, struct drm_pmu, pmu); + struct perf_counts_values *count, *old_count = NULL; + int pid = perf_thread_map__pid(evsel->core.threads, thread); + uint64_t counter; + + if (pid != -1) + counter = drm_pmu__read_for_pid(drm, evsel, pid); + else + counter = drm_pmu__read_system_wide(drm, evsel); + + if (evsel->prev_raw_counts) + old_count = perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); + + count = perf_counts(evsel->counts, cpu_map_idx, thread); + if (old_count) { + count->val = old_count->val + counter; + count->run = old_count->run + 1; + count->ena = old_count->ena + 1; + } else { + count->val = counter; + count->run++; + count->ena++; + } + return 0; +} diff --git a/tools/perf/util/drm_pmu.h b/tools/perf/util/drm_pmu.h new file mode 100644 index 000000000..e7f366fca --- /dev/null +++ b/tools/perf/util/drm_pmu.h @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ +#ifndef __DRM_PMU_H +#define __DRM_PMU_H +/* + * Linux DRM clients expose information through usage stats as documented in + * Documentation/gpu/drm-usage-stats.rst (available online at + * https://docs.kernel.org/gpu/drm-usage-stats.html). This is a tool like PMU + * that exposes DRM information. + */ + +#include "pmu.h" +#include + +struct list_head; +struct perf_thread_map; + +void drm_pmu__exit(struct perf_pmu *pmu); +bool drm_pmu__have_event(const struct perf_pmu *pmu, const char *name); +int drm_pmu__for_each_event(const struct perf_pmu *pmu, void *state, pmu_event_callback cb); +size_t drm_pmu__num_events(const struct perf_pmu *pmu); +int drm_pmu__config_terms(const struct perf_pmu *pmu, + struct perf_event_attr *attr, + struct parse_events_terms *terms, + struct parse_events_error *err); +int drm_pmu__check_alias(const struct perf_pmu *pmu, struct parse_events_terms *terms, + struct perf_pmu_info *info, struct parse_events_error *err); + + +bool perf_pmu__is_drm(const struct perf_pmu *pmu); +bool evsel__is_drm(const struct evsel *evsel); + +int perf_pmus__read_drm_pmus(struct list_head *pmus); + +int evsel__drm_pmu_open(struct evsel *evsel, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx); +int evsel__drm_pmu_read(struct evsel *evsel, int cpu_map_idx, int thread); + +#endif /* __DRM_PMU_H */ diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c new file mode 100644 index 000000000..2309196d8 --- /dev/null +++ b/tools/perf/util/dso.c @@ -0,0 +1,2045 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBBPF_SUPPORT +#include +#include "bpf-event.h" +#include "bpf-utils.h" +#endif +#include "compress.h" +#include "env.h" +#include "namespaces.h" +#include "path.h" +#include "map.h" +#include "symbol.h" +#include "srcline.h" +#include "dso.h" +#include "dsos.h" +#include "machine.h" +#include "auxtrace.h" +#include "util.h" /* O_CLOEXEC for older systems */ +#include "debug.h" +#include "string2.h" +#include "vdso.h" +#include "annotate-data.h" +#include "libdw.h" + +static const char * const debuglink_paths[] = { + "%.0s%s", + "%s/%s", + "%s/.debug/%s", + "/usr/lib/debug%s/%s" +}; + +void dso__set_nsinfo(struct dso *dso, struct nsinfo *nsi) +{ + nsinfo__put(RC_CHK_ACCESS(dso)->nsinfo); + RC_CHK_ACCESS(dso)->nsinfo = nsi; +} + +char dso__symtab_origin(const struct dso *dso) +{ + static const char origin[] = { + [DSO_BINARY_TYPE__KALLSYMS] = 'k', + [DSO_BINARY_TYPE__VMLINUX] = 'v', + [DSO_BINARY_TYPE__JAVA_JIT] = 'j', + [DSO_BINARY_TYPE__DEBUGLINK] = 'l', + [DSO_BINARY_TYPE__BUILD_ID_CACHE] = 'B', + [DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO] = 'D', + [DSO_BINARY_TYPE__FEDORA_DEBUGINFO] = 'f', + [DSO_BINARY_TYPE__UBUNTU_DEBUGINFO] = 'u', + [DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO] = 'x', + [DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO] = 'o', + [DSO_BINARY_TYPE__BUILDID_DEBUGINFO] = 'b', + [DSO_BINARY_TYPE__SYSTEM_PATH_DSO] = 'd', + [DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE] = 'K', + [DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP] = 'm', + [DSO_BINARY_TYPE__GUEST_KALLSYMS] = 'g', + [DSO_BINARY_TYPE__GUEST_KMODULE] = 'G', + [DSO_BINARY_TYPE__GUEST_KMODULE_COMP] = 'M', + [DSO_BINARY_TYPE__GUEST_VMLINUX] = 'V', + [DSO_BINARY_TYPE__GNU_DEBUGDATA] = 'n', + }; + + if (dso == NULL || dso__symtab_type(dso) == DSO_BINARY_TYPE__NOT_FOUND) + return '!'; + return origin[dso__symtab_type(dso)]; +} + +bool dso__is_object_file(const struct dso *dso) +{ + switch (dso__binary_type(dso)) { + case DSO_BINARY_TYPE__KALLSYMS: + case DSO_BINARY_TYPE__GUEST_KALLSYMS: + case DSO_BINARY_TYPE__JAVA_JIT: + case DSO_BINARY_TYPE__BPF_PROG_INFO: + case DSO_BINARY_TYPE__BPF_IMAGE: + case DSO_BINARY_TYPE__OOL: + return false; + case DSO_BINARY_TYPE__VMLINUX: + case DSO_BINARY_TYPE__GUEST_VMLINUX: + case DSO_BINARY_TYPE__DEBUGLINK: + case DSO_BINARY_TYPE__BUILD_ID_CACHE: + case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: + case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: + case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: + case DSO_BINARY_TYPE__GNU_DEBUGDATA: + case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: + case DSO_BINARY_TYPE__GUEST_KMODULE: + case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: + case DSO_BINARY_TYPE__KCORE: + case DSO_BINARY_TYPE__GUEST_KCORE: + case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: + case DSO_BINARY_TYPE__NOT_FOUND: + default: + return true; + } +} + +int dso__read_binary_type_filename(const struct dso *dso, + enum dso_binary_type type, + const char *root_dir, char *filename, size_t size) +{ + char build_id_hex[SBUILD_ID_SIZE]; + int ret = 0; + size_t len; + + switch (type) { + case DSO_BINARY_TYPE__DEBUGLINK: + { + const char *last_slash; + char dso_dir[PATH_MAX]; + char symfile[PATH_MAX]; + unsigned int i; + + len = __symbol__join_symfs(filename, size, dso__long_name(dso)); + last_slash = filename + len; + while (last_slash != filename && *last_slash != '/') + last_slash--; + + strncpy(dso_dir, filename, last_slash - filename); + dso_dir[last_slash-filename] = '\0'; + + if (!is_regular_file(filename)) { + ret = -1; + break; + } + + ret = filename__read_debuglink(filename, symfile, PATH_MAX); + if (ret) + break; + + /* Check predefined locations where debug file might reside */ + ret = -1; + for (i = 0; i < ARRAY_SIZE(debuglink_paths); i++) { + snprintf(filename, size, + debuglink_paths[i], dso_dir, symfile); + if (is_regular_file(filename)) { + ret = 0; + break; + } + } + + break; + } + case DSO_BINARY_TYPE__BUILD_ID_CACHE: + if (dso__build_id_filename(dso, filename, size, false) == NULL) + ret = -1; + break; + + case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: + if (dso__build_id_filename(dso, filename, size, true) == NULL) + ret = -1; + break; + + case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: + len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); + snprintf(filename + len, size - len, "%s.debug", dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: + len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); + snprintf(filename + len, size - len, "%s", dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: + /* + * Ubuntu can mixup /usr/lib with /lib, putting debuginfo in + * /usr/lib/debug/lib when it is expected to be in + * /usr/lib/debug/usr/lib + */ + if (strlen(dso__long_name(dso)) < 9 || + strncmp(dso__long_name(dso), "/usr/lib/", 9)) { + ret = -1; + break; + } + len = __symbol__join_symfs(filename, size, "/usr/lib/debug"); + snprintf(filename + len, size - len, "%s", dso__long_name(dso) + 4); + break; + + case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: + { + const char *last_slash; + size_t dir_size; + + last_slash = dso__long_name(dso) + dso__long_name_len(dso); + while (last_slash != dso__long_name(dso) && *last_slash != '/') + last_slash--; + + len = __symbol__join_symfs(filename, size, ""); + dir_size = last_slash - dso__long_name(dso) + 2; + if (dir_size > (size - len)) { + ret = -1; + break; + } + len += scnprintf(filename + len, dir_size, "%s", dso__long_name(dso)); + len += scnprintf(filename + len , size - len, ".debug%s", + last_slash); + break; + } + + case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: + if (!dso__has_build_id(dso)) { + ret = -1; + break; + } + + build_id__snprintf(dso__bid(dso), build_id_hex, sizeof(build_id_hex)); + len = __symbol__join_symfs(filename, size, "/usr/lib/debug/.build-id/"); + snprintf(filename + len, size - len, "%.2s/%s.debug", + build_id_hex, build_id_hex + 2); + break; + + case DSO_BINARY_TYPE__VMLINUX: + case DSO_BINARY_TYPE__GUEST_VMLINUX: + case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: + case DSO_BINARY_TYPE__GNU_DEBUGDATA: + __symbol__join_symfs(filename, size, dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__GUEST_KMODULE: + case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: + path__join3(filename, size, symbol_conf.symfs, + root_dir, dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: + __symbol__join_symfs(filename, size, dso__long_name(dso)); + break; + + case DSO_BINARY_TYPE__KCORE: + case DSO_BINARY_TYPE__GUEST_KCORE: + snprintf(filename, size, "%s", dso__long_name(dso)); + break; + + default: + case DSO_BINARY_TYPE__KALLSYMS: + case DSO_BINARY_TYPE__GUEST_KALLSYMS: + case DSO_BINARY_TYPE__JAVA_JIT: + case DSO_BINARY_TYPE__BPF_PROG_INFO: + case DSO_BINARY_TYPE__BPF_IMAGE: + case DSO_BINARY_TYPE__OOL: + case DSO_BINARY_TYPE__NOT_FOUND: + ret = -1; + break; + } + + return ret; +} + +enum { + COMP_ID__NONE = 0, +}; + +static const struct { + const char *fmt; + int (*decompress)(const char *input, int output); + bool (*is_compressed)(const char *input); +} compressions[] = { + [COMP_ID__NONE] = { .fmt = NULL, }, +#ifdef HAVE_ZLIB_SUPPORT + { "gz", gzip_decompress_to_file, gzip_is_compressed }, +#endif +#ifdef HAVE_LZMA_SUPPORT + { "xz", lzma_decompress_to_file, lzma_is_compressed }, +#endif + { NULL, NULL, NULL }, +}; + +static int is_supported_compression(const char *ext) +{ + unsigned i; + + for (i = 1; compressions[i].fmt; i++) { + if (!strcmp(ext, compressions[i].fmt)) + return i; + } + return COMP_ID__NONE; +} + +bool is_kernel_module(const char *pathname, int cpumode) +{ + struct kmod_path m; + int mode = cpumode & PERF_RECORD_MISC_CPUMODE_MASK; + + WARN_ONCE(mode != cpumode, + "Internal error: passing unmasked cpumode (%x) to is_kernel_module", + cpumode); + + switch (mode) { + case PERF_RECORD_MISC_USER: + case PERF_RECORD_MISC_HYPERVISOR: + case PERF_RECORD_MISC_GUEST_USER: + return false; + /* Treat PERF_RECORD_MISC_CPUMODE_UNKNOWN as kernel */ + default: + if (kmod_path__parse(&m, pathname)) { + pr_err("Failed to check whether %s is a kernel module or not. Assume it is.", + pathname); + return true; + } + } + + return m.kmod; +} + +bool dso__needs_decompress(struct dso *dso) +{ + return dso__symtab_type(dso) == DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP || + dso__symtab_type(dso) == DSO_BINARY_TYPE__GUEST_KMODULE_COMP; +} + +int filename__decompress(const char *name, char *pathname, + size_t len, int comp, int *err) +{ + char tmpbuf[] = KMOD_DECOMP_NAME; + int fd = -1; + + /* + * We have proper compression id for DSO and yet the file + * behind the 'name' can still be plain uncompressed object. + * + * The reason is behind the logic we open the DSO object files, + * when we try all possible 'debug' objects until we find the + * data. So even if the DSO is represented by 'krava.xz' module, + * we can end up here opening ~/.debug/....23432432/debug' file + * which is not compressed. + * + * To keep this transparent, we detect this and return the file + * descriptor to the uncompressed file. + */ + if (!compressions[comp].is_compressed(name)) { + fd = open(name, O_RDONLY | O_CLOEXEC); + if (fd < 0) + *err = errno; + if (pathname && len > 0) + pathname[0] = '\0'; + return fd; + } + + fd = mkostemp(tmpbuf, O_CLOEXEC); + if (fd < 0) { + *err = errno; + return -1; + } + + if (compressions[comp].decompress(name, fd)) { + *err = DSO_LOAD_ERRNO__DECOMPRESSION_FAILURE; + close(fd); + fd = -1; + } + + if (!pathname || (fd < 0)) + unlink(tmpbuf); + + if (pathname && (fd >= 0)) + strlcpy(pathname, tmpbuf, len); + + return fd; +} + +static int decompress_kmodule(struct dso *dso, const char *name, + char *pathname, size_t len) +{ + if (!dso__needs_decompress(dso)) + return -1; + + if (dso__comp(dso) == COMP_ID__NONE) + return -1; + + return filename__decompress(name, pathname, len, dso__comp(dso), dso__load_errno(dso)); +} + +int dso__decompress_kmodule_fd(struct dso *dso, const char *name) +{ + return decompress_kmodule(dso, name, NULL, 0); +} + +int dso__decompress_kmodule_path(struct dso *dso, const char *name, + char *pathname, size_t len) +{ + int fd = decompress_kmodule(dso, name, pathname, len); + + close(fd); + return fd >= 0 ? 0 : -1; +} + +/* + * Parses kernel module specified in @path and updates + * @m argument like: + * + * @comp - true if @path contains supported compression suffix, + * false otherwise + * @kmod - true if @path contains '.ko' suffix in right position, + * false otherwise + * @name - if (@alloc_name && @kmod) is true, it contains strdup-ed base name + * of the kernel module without suffixes, otherwise strudup-ed + * base name of @path + * @ext - if (@alloc_ext && @comp) is true, it contains strdup-ed string + * the compression suffix + * + * Returns 0 if there's no strdup error, -ENOMEM otherwise. + */ +int __kmod_path__parse(struct kmod_path *m, const char *path, + bool alloc_name) +{ + const char *name = strrchr(path, '/'); + const char *ext = strrchr(path, '.'); + bool is_simple_name = false; + + memset(m, 0x0, sizeof(*m)); + name = name ? name + 1 : path; + + /* + * '.' is also a valid character for module name. For example: + * [aaa.bbb] is a valid module name. '[' should have higher + * priority than '.ko' suffix. + * + * The kernel names are from machine__mmap_name. Such + * name should belong to kernel itself, not kernel module. + */ + if (name[0] == '[') { + is_simple_name = true; + if ((strncmp(name, "[kernel.kallsyms]", 17) == 0) || + (strncmp(name, "[guest.kernel.kallsyms", 22) == 0) || + (strncmp(name, "[vdso]", 6) == 0) || + (strncmp(name, "[vdso32]", 8) == 0) || + (strncmp(name, "[vdsox32]", 9) == 0) || + (strncmp(name, "[vsyscall]", 10) == 0)) { + m->kmod = false; + + } else + m->kmod = true; + } + + /* No extension, just return name. */ + if ((ext == NULL) || is_simple_name) { + if (alloc_name) { + m->name = strdup(name); + return m->name ? 0 : -ENOMEM; + } + return 0; + } + + m->comp = is_supported_compression(ext + 1); + if (m->comp > COMP_ID__NONE) + ext -= 3; + + /* Check .ko extension only if there's enough name left. */ + if (ext > name) + m->kmod = !strncmp(ext, ".ko", 3); + + if (alloc_name) { + if (m->kmod) { + if (asprintf(&m->name, "[%.*s]", (int) (ext - name), name) == -1) + return -ENOMEM; + } else { + if (asprintf(&m->name, "%s", name) == -1) + return -ENOMEM; + } + + strreplace(m->name, '-', '_'); + } + + return 0; +} + +void dso__set_module_info(struct dso *dso, struct kmod_path *m, + struct machine *machine) +{ + if (machine__is_host(machine)) + dso__set_symtab_type(dso, DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE); + else + dso__set_symtab_type(dso, DSO_BINARY_TYPE__GUEST_KMODULE); + + /* _KMODULE_COMP should be next to _KMODULE */ + if (m->kmod && m->comp) { + dso__set_symtab_type(dso, dso__symtab_type(dso) + 1); + dso__set_comp(dso, m->comp); + } + + dso__set_is_kmod(dso); + dso__set_short_name(dso, strdup(m->name), true); +} + +/* + * Global list of open DSOs and the counter. + */ +struct mutex _dso__data_open_lock; +static LIST_HEAD(dso__data_open); +static long dso__data_open_cnt GUARDED_BY(_dso__data_open_lock); + +static void dso__data_open_lock_init(void) +{ + mutex_init(&_dso__data_open_lock); +} + +static struct mutex *dso__data_open_lock(void) LOCK_RETURNED(_dso__data_open_lock) +{ + static pthread_once_t data_open_lock_once = PTHREAD_ONCE_INIT; + + pthread_once(&data_open_lock_once, dso__data_open_lock_init); + + return &_dso__data_open_lock; +} + +static void dso__list_add(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + list_add_tail(&dso__data(dso)->open_entry, &dso__data_open); +#ifdef REFCNT_CHECKING + dso__data(dso)->dso = dso__get(dso); +#endif + /* Assume the dso is part of dsos, hence the optional reference count above. */ + assert(dso__dsos(dso)); + dso__data_open_cnt++; +} + +static void dso__list_del(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + list_del_init(&dso__data(dso)->open_entry); +#ifdef REFCNT_CHECKING + mutex_unlock(dso__data_open_lock()); + dso__put(dso__data(dso)->dso); + mutex_lock(dso__data_open_lock()); +#endif + WARN_ONCE(dso__data_open_cnt <= 0, + "DSO data fd counter out of bounds."); + dso__data_open_cnt--; +} + +static void close_first_dso(void); + +static int do_open(char *name) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + do { + int fd = open(name, O_RDONLY|O_CLOEXEC); + + if (fd >= 0) + return fd; + + pr_debug("dso open failed: %m\n"); + if (!dso__data_open_cnt || errno != EMFILE) + break; + + close_first_dso(); + } while (1); + + return -1; +} + +char *dso__filename_with_chroot(const struct dso *dso, const char *filename) +{ + return filename_with_chroot(nsinfo__pid(dso__nsinfo_const(dso)), filename); +} + +static char *dso__get_filename(struct dso *dso, const char *root_dir, + bool *decomp) +{ + char *name = malloc(PATH_MAX); + + *decomp = false; + + if (name == NULL) + return NULL; + + if (dso__read_binary_type_filename(dso, dso__binary_type(dso), + root_dir, name, PATH_MAX)) + goto out; + + if (!is_regular_file(name)) { + char *new_name; + + if (errno != ENOENT || dso__nsinfo(dso) == NULL) + goto out; + + new_name = dso__filename_with_chroot(dso, name); + if (!new_name) + goto out; + + free(name); + name = new_name; + } + + if (dso__needs_decompress(dso)) { + char newpath[KMOD_DECOMP_LEN]; + size_t len = sizeof(newpath); + + if (dso__decompress_kmodule_path(dso, name, newpath, len) < 0) { + /* + * Use a standard errno value, not the negative custom + * DSO_LOAD_ERRNO stored in dso__load_errno(dso): + * __open_dso() computes fd = -errno, so a negative + * errno produces a positive fd that looks valid. + */ + errno = EIO; + goto out; + } + + /* empty pathname means file wasn't actually compressed */ + if (newpath[0] != '\0') { + char *tmp = strdup(newpath); + + if (!tmp) { + unlink(newpath); + goto out; + } + free(name); + name = tmp; + *decomp = true; + } + } + return name; + +out: + free(name); + return NULL; +} + +static int __open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd = -EINVAL; + char *name; + bool decomp = false; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, machine ? machine->root_dir : "", &decomp); + if (name) + fd = do_open(name); + else + fd = -errno; + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return fd; +} + +static void check_data_close(void); + +/** + * dso_close - Open DSO data file + * @dso: dso object + * + * Open @dso's data file descriptor and updates + * list/count of open DSO objects. + */ +static int open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + int fd; + struct nscookie nsc; + + if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) { + mutex_lock(dso__lock(dso)); + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + mutex_unlock(dso__lock(dso)); + } + fd = __open_dso(dso, machine); + if (dso__binary_type(dso) != DSO_BINARY_TYPE__BUILD_ID_CACHE) + nsinfo__mountns_exit(&nsc); + + if (fd >= 0) { + dso__list_add(dso); + /* + * Check if we crossed the allowed number + * of opened DSOs and close one if needed. + */ + check_data_close(); + } + + return fd; +} + +static void close_data_fd(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + if (dso__data(dso)->fd >= 0) { + close(dso__data(dso)->fd); + dso__data(dso)->fd = -1; + dso__data(dso)->file_size = 0; + dso__list_del(dso); + } +} + +/** + * dso_close - Close DSO data file + * @dso: dso object + * + * Close @dso's data file descriptor and updates + * list/count of open DSO objects. + */ +static void close_dso(struct dso *dso) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + close_data_fd(dso); +} + +static void close_first_dso(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + struct dso_data *dso_data; + struct dso *dso; + + dso_data = list_first_entry(&dso__data_open, struct dso_data, open_entry); +#ifdef REFCNT_CHECKING + dso = dso_data->dso; +#else + dso = container_of(dso_data, struct dso, data); +#endif + close_dso(dso); +} + +static rlim_t get_fd_limit(void) +{ + struct rlimit l; + rlim_t limit = 0; + + /* Allow half of the current open fd limit. */ + if (getrlimit(RLIMIT_NOFILE, &l) == 0) { + if (l.rlim_cur == RLIM_INFINITY) + limit = l.rlim_cur; + else + limit = l.rlim_cur / 2; + } else { + pr_err("failed to get fd limit\n"); + limit = 1; + } + + return limit; +} + +static rlim_t fd_limit; + +/* + * Used only by tests/dso-data.c to reset the environment + * for tests. I dont expect we should change this during + * standard runtime. + */ +void reset_fd_limit(void) +{ + fd_limit = 0; +} + +static bool may_cache_fd(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + if (!fd_limit) + fd_limit = get_fd_limit(); + + if (fd_limit == RLIM_INFINITY) + return true; + + return fd_limit > (rlim_t) dso__data_open_cnt; +} + +/* + * Check and close LRU dso if we crossed allowed limit + * for opened dso file descriptors. The limit is half + * of the RLIMIT_NOFILE files opened. +*/ +static void check_data_close(void) EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + bool cache_fd = may_cache_fd(); + + if (!cache_fd) + close_first_dso(); +} + +/** + * dso__data_close - Close DSO data file + * @dso: dso object + * + * External interface to close @dso's data file descriptor. + */ +void dso__data_close(struct dso *dso) +{ + mutex_lock(dso__data_open_lock()); + close_dso(dso); + mutex_unlock(dso__data_open_lock()); +} + +static void try_to_open_dso(struct dso *dso, struct machine *machine) + EXCLUSIVE_LOCKS_REQUIRED(_dso__data_open_lock) +{ + enum dso_binary_type binary_type_data[] = { + DSO_BINARY_TYPE__BUILD_ID_CACHE, + DSO_BINARY_TYPE__SYSTEM_PATH_DSO, + DSO_BINARY_TYPE__NOT_FOUND, + }; + int i = 0; + struct dso_data *dso_data = dso__data(dso); + + if (dso_data->fd >= 0) + return; + + if (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND) { + dso_data->fd = open_dso(dso, machine); + goto out; + } + + do { + dso__set_binary_type(dso, binary_type_data[i++]); + + dso_data->fd = open_dso(dso, machine); + if (dso_data->fd >= 0) + goto out; + + } while (dso__binary_type(dso) != DSO_BINARY_TYPE__NOT_FOUND); +out: + if (dso_data->fd >= 0) + dso_data->status = DSO_DATA_STATUS_OK; + else + dso_data->status = DSO_DATA_STATUS_ERROR; +} + +/** + * dso__data_get_fd - Get dso's data file descriptor + * @dso: dso object + * @machine: machine object + * + * External interface to find dso's file, open it and + * returns file descriptor. It should be paired with + * dso__data_put_fd() if it returns non-negative value. + */ +bool dso__data_get_fd(struct dso *dso, struct machine *machine, int *fd) +{ + *fd = -1; + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return false; + + mutex_lock(dso__data_open_lock()); + + try_to_open_dso(dso, machine); + + *fd = dso__data(dso)->fd; + if (*fd >= 0) + return true; + + mutex_unlock(dso__data_open_lock()); + return false; +} + +void dso__data_put_fd(struct dso *dso __maybe_unused) +{ + mutex_unlock(dso__data_open_lock()); +} + +bool dso__data_status_seen(struct dso *dso, enum dso_data_status_seen by) +{ + u32 flag = 1 << by; + + if (dso__data(dso)->status_seen & flag) + return true; + + dso__data(dso)->status_seen |= flag; + + return false; +} + +#ifdef HAVE_LIBBPF_SUPPORT +static ssize_t bpf_read(struct dso *dso, u64 offset, char *data) +{ + struct bpf_prog_info_node *node; + ssize_t size = DSO__DATA_CACHE_SIZE; + struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso); + u64 len; + u8 *buf; + + node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id); + if (!node || !node->info_linear) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + + /* jited_prog_insns is only valid if bpil_offs_to_addr() converted it */ + if (!(node->info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + + len = node->info_linear->info.jited_prog_len; + buf = (u8 *)(uintptr_t)node->info_linear->info.jited_prog_insns; + + if (offset >= len) + return -1; + + size = (ssize_t)min(len - offset, (u64)size); + memcpy(data, buf + offset, size); + return size; +} + +static int bpf_size(struct dso *dso) +{ + struct bpf_prog_info_node *node; + struct dso_bpf_prog *dso_bpf_prog = dso__bpf_prog(dso); + + node = perf_env__find_bpf_prog_info(dso_bpf_prog->env, dso_bpf_prog->id); + if (!node || !node->info_linear) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + return -1; + } + + dso__data(dso)->file_size = node->info_linear->info.jited_prog_len; + return 0; +} +#endif // HAVE_LIBBPF_SUPPORT + +static void +dso_cache__free(struct dso *dso) +{ + struct rb_root *root = &dso__data(dso)->cache; + struct rb_node *next = rb_first(root); + + mutex_lock(dso__lock(dso)); + while (next) { + struct dso_cache *cache; + + cache = rb_entry(next, struct dso_cache, rb_node); + next = rb_next(&cache->rb_node); + rb_erase(&cache->rb_node, root); + free(cache); + } + mutex_unlock(dso__lock(dso)); +} + +static struct dso_cache *__dso_cache__find(struct dso *dso, u64 offset) +{ + const struct rb_root *root = &dso__data(dso)->cache; + struct rb_node * const *p = &root->rb_node; + const struct rb_node *parent = NULL; + struct dso_cache *cache; + + while (*p != NULL) { + u64 end; + + parent = *p; + cache = rb_entry(parent, struct dso_cache, rb_node); + end = cache->offset + DSO__DATA_CACHE_SIZE; + + if (offset < cache->offset) + p = &(*p)->rb_left; + else if (offset >= end) + p = &(*p)->rb_right; + else + return cache; + } + + return NULL; +} + +static struct dso_cache * +dso_cache__insert(struct dso *dso, struct dso_cache *new) +{ + struct rb_root *root = &dso__data(dso)->cache; + struct rb_node **p = &root->rb_node; + struct rb_node *parent = NULL; + struct dso_cache *cache; + u64 offset = new->offset; + + mutex_lock(dso__lock(dso)); + while (*p != NULL) { + u64 end; + + parent = *p; + cache = rb_entry(parent, struct dso_cache, rb_node); + end = cache->offset + DSO__DATA_CACHE_SIZE; + + if (offset < cache->offset) + p = &(*p)->rb_left; + else if (offset >= end) + p = &(*p)->rb_right; + else + goto out; + } + + rb_link_node(&new->rb_node, parent, p); + rb_insert_color(&new->rb_node, root); + + cache = NULL; +out: + mutex_unlock(dso__lock(dso)); + return cache; +} + +static ssize_t dso_cache__memcpy(struct dso_cache *cache, u64 offset, u8 *data, + u64 size, bool out) +{ + u64 cache_offset = offset - cache->offset; + u64 cache_size = min(cache->size - cache_offset, size); + + if (out) + memcpy(data, cache->data + cache_offset, cache_size); + else + memcpy(cache->data + cache_offset, data, cache_size); + return cache_size; +} + +static ssize_t file_read(struct dso *dso, struct machine *machine, + u64 offset, char *data) +{ + ssize_t ret; + + mutex_lock(dso__data_open_lock()); + + /* + * dso__data(dso)->fd might be closed if other thread opened another + * file (dso) due to open file limit (RLIMIT_NOFILE). + */ + try_to_open_dso(dso, machine); + + if (dso__data(dso)->fd < 0) { + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + ret = -errno; + goto out; + } + + ret = pread(dso__data(dso)->fd, data, DSO__DATA_CACHE_SIZE, offset); +out: + mutex_unlock(dso__data_open_lock()); + return ret; +} + +static struct dso_cache *dso_cache__populate(struct dso *dso, + struct machine *machine, + u64 offset, ssize_t *ret) +{ + u64 cache_offset = offset & DSO__DATA_CACHE_MASK; + struct dso_cache *cache; + struct dso_cache *old; + + cache = zalloc(sizeof(*cache) + DSO__DATA_CACHE_SIZE); + if (!cache) { + *ret = -ENOMEM; + return NULL; + } +#ifdef HAVE_LIBBPF_SUPPORT + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) + *ret = bpf_read(dso, cache_offset, cache->data); + else +#endif + if (dso__binary_type(dso) == DSO_BINARY_TYPE__OOL) + *ret = DSO__DATA_CACHE_SIZE; + else + *ret = file_read(dso, machine, cache_offset, cache->data); + + if (*ret <= 0) { + free(cache); + return NULL; + } + + cache->offset = cache_offset; + cache->size = *ret; + + old = dso_cache__insert(dso, cache); + if (old) { + /* we lose the race */ + free(cache); + cache = old; + } + + return cache; +} + +static struct dso_cache *dso_cache__find(struct dso *dso, + struct machine *machine, + u64 offset, + ssize_t *ret) +{ + struct dso_cache *cache = __dso_cache__find(dso, offset); + + return cache ? cache : dso_cache__populate(dso, machine, offset, ret); +} + +static ssize_t dso_cache_io(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size, bool out) +{ + struct dso_cache *cache; + ssize_t ret = 0; + + cache = dso_cache__find(dso, machine, offset, &ret); + if (!cache) + return ret; + + return dso_cache__memcpy(cache, offset, data, size, out); +} + +/* + * Reads and caches dso data DSO__DATA_CACHE_SIZE size chunks + * in the rb_tree. Any read to already cached data is served + * by cached data. Writes update the cache only, not the backing file. + */ +static ssize_t cached_io(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size, bool out) +{ + ssize_t r = 0; + u8 *p = data; + + do { + ssize_t ret; + + ret = dso_cache_io(dso, machine, offset, p, size, out); + if (ret < 0) + return ret; + + /* Reached EOF, return what we have. */ + if (!ret) + break; + + BUG_ON(ret > size); + + r += ret; + p += ret; + offset += ret; + size -= ret; + + } while (size); + + return r; +} + +static int file_size(struct dso *dso, struct machine *machine) +{ + int ret = 0; + struct stat st; + + mutex_lock(dso__data_open_lock()); + + /* + * dso__data(dso)->fd might be closed if other thread opened another + * file (dso) due to open file limit (RLIMIT_NOFILE). + */ + try_to_open_dso(dso, machine); + + if (dso__data(dso)->fd < 0) { + ret = -errno; + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + goto out; + } + + if (fstat(dso__data(dso)->fd, &st) < 0) { + ret = -errno; + pr_err("dso cache fstat failed: %m\n"); + dso__data(dso)->status = DSO_DATA_STATUS_ERROR; + goto out; + } + dso__data(dso)->file_size = st.st_size; + +out: + mutex_unlock(dso__data_open_lock()); + return ret; +} + +int dso__data_file_size(struct dso *dso, struct machine *machine) +{ + if (dso__data(dso)->file_size) + return 0; + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return -1; +#ifdef HAVE_LIBBPF_SUPPORT + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) + return bpf_size(dso); +#endif + return file_size(dso, machine); +} + +/** + * dso__data_size - Return dso data size + * @dso: dso object + * @machine: machine object + * + * Return: dso data size + */ +off_t dso__data_size(struct dso *dso, struct machine *machine) +{ + if (dso__data_file_size(dso, machine)) + return -1; + + /* For now just estimate dso data size is close to file size */ + return dso__data(dso)->file_size; +} + +static ssize_t data_read_write_offset(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size, + bool out) +{ + if (dso__data_file_size(dso, machine)) + return -1; + + /* Check the offset sanity. */ + if (offset > dso__data(dso)->file_size) + return -1; + + if (offset + size < offset) + return -1; + + return cached_io(dso, machine, offset, data, size, out); +} + +/** + * dso__data_read_offset - Read data from dso file offset + * @dso: dso object + * @machine: machine object + * @offset: file offset + * @data: buffer to store data + * @size: size of the @data buffer + * + * External interface to read data from dso file offset. Open + * dso data file and use cached_read to get the data. + */ +ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size) +{ + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return -1; + + return data_read_write_offset(dso, machine, offset, data, size, true); +} + +static enum dso_swap_type dso_swap_type__from_elf_data(unsigned char eidata) +{ + static const unsigned int endian = 1; + + switch (eidata) { + case ELFDATA2LSB: + /* We are big endian, DSO is little endian. */ + return (*(unsigned char const *)&endian != 1) ? DSO_SWAP__YES : DSO_SWAP__NO; + case ELFDATA2MSB: + /* We are little endian, DSO is big endian. */ + return (*(unsigned char const *)&endian != 0) ? DSO_SWAP__YES : DSO_SWAP__NO; + default: + return DSO_SWAP__UNSET; + } +} + +/* Reads e_machine from fd, optionally caching data in dso. */ +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian) +{ + uint16_t e_machine = EM_NONE; + unsigned char e_ident[EI_NIDENT]; + enum dso_swap_type swap_type; + bool need_e_flags; + + if (e_flags) + *e_flags = 0; + + { + _Static_assert(offsetof(Elf32_Ehdr, e_ident) == 0, "Unexpected offset"); + _Static_assert(offsetof(Elf64_Ehdr, e_ident) == 0, "Unexpected offset"); + } + if (pread(fd, &e_ident, sizeof(e_ident), 0) != sizeof(e_ident)) + return EM_NONE; // Read failed. + + if (memcmp(e_ident, ELFMAG, SELFMAG) != 0) + return EM_NONE; // Not an ELF file. + + if (e_ident[EI_CLASS] == ELFCLASSNONE || e_ident[EI_CLASS] >= ELFCLASSNUM) + return EM_NONE; // Bad ELF class (32 or 64-bit objects). + + if (e_ident[EI_VERSION] != EV_CURRENT) + return EM_NONE; // Bad ELF version. + + swap_type = dso_swap_type__from_elf_data(e_ident[EI_DATA]); + if (swap_type == DSO_SWAP__UNSET) + return EM_NONE; // Bad ELF data encoding. + + if (is_big_endian) + *is_big_endian = (e_ident[EI_DATA] == ELFDATA2MSB); + + /* Cache the need for swapping. */ + if (optional_dso) { + assert(dso__needs_swap(optional_dso) == DSO_SWAP__UNSET || + dso__needs_swap(optional_dso) == swap_type); + dso__set_needs_swap(optional_dso, swap_type); + } + + { + _Static_assert(offsetof(Elf32_Ehdr, e_machine) == 18, "Unexpected offset"); + _Static_assert(offsetof(Elf64_Ehdr, e_machine) == 18, "Unexpected offset"); + } + if (pread(fd, &e_machine, sizeof(e_machine), 18) != sizeof(e_machine)) + return EM_NONE; // e_machine read failed. + + e_machine = DSO_SWAP_TYPE__SWAP(swap_type, uint16_t, e_machine); + if (e_machine >= EM_NUM) + return EM_NONE; // Bad ELF machine number. + +#ifdef NDEBUG + /* In production code the e_flags are only needed on CSKY. */ + need_e_flags = e_flags && e_machine == EM_CSKY; +#else + /* Debug code will always read the e_flags. */ + need_e_flags = e_flags != NULL; +#endif + if (need_e_flags) { + off_t offset = e_ident[EI_CLASS] == ELFCLASS32 + ? offsetof(Elf32_Ehdr, e_flags) + : offsetof(Elf64_Ehdr, e_flags); + + if (pread(fd, e_flags, sizeof(*e_flags), offset) != sizeof(*e_flags)) { + *e_flags = 0; + return EM_NONE; // e_flags read failed. + } + } + return e_machine; +} + +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian) +{ + uint16_t e_machine = EM_NONE; + int fd; + + switch (dso__binary_type(dso)) { + case DSO_BINARY_TYPE__KALLSYMS: + case DSO_BINARY_TYPE__GUEST_KALLSYMS: + case DSO_BINARY_TYPE__VMLINUX: + case DSO_BINARY_TYPE__GUEST_VMLINUX: + case DSO_BINARY_TYPE__GUEST_KMODULE: + case DSO_BINARY_TYPE__GUEST_KMODULE_COMP: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: + case DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: + case DSO_BINARY_TYPE__KCORE: + case DSO_BINARY_TYPE__GUEST_KCORE: + case DSO_BINARY_TYPE__BPF_PROG_INFO: + case DSO_BINARY_TYPE__BPF_IMAGE: + case DSO_BINARY_TYPE__OOL: + case DSO_BINARY_TYPE__JAVA_JIT: + if (is_big_endian) { + *is_big_endian = perf_arch_is_big_endian( + machine && machine->env ? perf_env__arch(machine->env) : NULL); + } + return perf_env__e_machine(machine ? machine->env : NULL, e_flags); + case DSO_BINARY_TYPE__DEBUGLINK: + case DSO_BINARY_TYPE__BUILD_ID_CACHE: + case DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: + case DSO_BINARY_TYPE__GNU_DEBUGDATA: + case DSO_BINARY_TYPE__SYSTEM_PATH_DSO: + case DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: + case DSO_BINARY_TYPE__FEDORA_DEBUGINFO: + case DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: + case DSO_BINARY_TYPE__BUILDID_DEBUGINFO: + break; + case DSO_BINARY_TYPE__NOT_FOUND: + default: + if (e_flags) + *e_flags = 0; + return EM_NONE; + } + + mutex_lock(dso__data_open_lock()); + + /* + * dso__data(dso)->fd might be closed if other thread opened another + * file (dso) due to open file limit (RLIMIT_NOFILE). + */ + try_to_open_dso(dso, machine); + fd = dso__data(dso)->fd; + if (fd >= 0) + e_machine = dso__read_e_machine_endian(dso, fd, e_flags, is_big_endian); + else if (e_flags) + *e_flags = 0; + + mutex_unlock(dso__data_open_lock()); + return e_machine; +} + +/** + * dso__data_read_addr - Read data from dso address + * @dso: dso object + * @machine: machine object + * @add: virtual memory address + * @data: buffer to store data + * @size: size of the @data buffer + * + * External interface to read data from dso address. + */ +ssize_t dso__data_read_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + u8 *data, ssize_t size) +{ + u64 offset = map__map_ip(map, addr); + + return dso__data_read_offset(dso, machine, offset, data, size); +} + +/** + * dso__data_write_cache_offs - Write data to dso data cache at file offset + * @dso: dso object + * @machine: machine object + * @offset: file offset + * @data: buffer to write + * @size: size of the @data buffer + * + * Write into the dso file data cache, but do not change the file itself. + */ +ssize_t dso__data_write_cache_offs(struct dso *dso, struct machine *machine, + u64 offset, const u8 *data_in, ssize_t size) +{ + u8 *data = (u8 *)data_in; /* cast away const to use same fns for r/w */ + + if (dso__data(dso)->status == DSO_DATA_STATUS_ERROR) + return -1; + + return data_read_write_offset(dso, machine, offset, data, size, false); +} + +/** + * dso__data_write_cache_addr - Write data to dso data cache at dso address + * @dso: dso object + * @machine: machine object + * @add: virtual memory address + * @data: buffer to write + * @size: size of the @data buffer + * + * External interface to write into the dso file data cache, but do not change + * the file itself. + */ +ssize_t dso__data_write_cache_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + const u8 *data, ssize_t size) +{ + u64 offset = map__map_ip(map, addr); + + return dso__data_write_cache_offs(dso, machine, offset, data, size); +} + +struct map *dso__new_map(const char *name) +{ + struct map *map = NULL; + struct dso *dso = dso__new(name); + + if (dso) { + map = map__new2(0, dso); + dso__put(dso); + } + + return map; +} + +struct dso *machine__findnew_kernel(struct machine *machine, const char *name, + const char *short_name, int dso_type) +{ + /* + * The kernel dso could be created by build_id processing. + */ + struct dso *dso = machine__findnew_dso(machine, name); + + /* + * We need to run this in all cases, since during the build_id + * processing we had no idea this was the kernel dso. + */ + if (dso != NULL) { + dso__set_short_name(dso, short_name, false); + dso__set_kernel(dso, dso_type); + } + + return dso; +} + +static void __dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated) +{ + if (dso__long_name_allocated(dso)) + free((char *)dso__long_name(dso)); + + RC_CHK_ACCESS(dso)->long_name = name; + RC_CHK_ACCESS(dso)->long_name_len = strlen(name); + dso__set_long_name_allocated(dso, name_allocated); +} + +static void dso__set_long_name_id(struct dso *dso, const char *name, bool name_allocated) +{ + struct dsos *dsos = dso__dsos(dso); + + if (name == NULL) + return; + + if (dsos) { + /* + * Need to avoid re-sorting the dsos breaking by non-atomically + * renaming the dso. + */ + down_write(&dsos->lock); + __dso__set_long_name_id(dso, name, name_allocated); + dsos->sorted = false; + up_write(&dsos->lock); + } else { + __dso__set_long_name_id(dso, name, name_allocated); + } +} + +static int __dso_id__cmp(const struct dso_id *a, const struct dso_id *b) +{ + if (a->mmap2_valid && b->mmap2_valid) { + if (a->maj > b->maj) return -1; + if (a->maj < b->maj) return 1; + + if (a->min > b->min) return -1; + if (a->min < b->min) return 1; + + if (a->ino > b->ino) return -1; + if (a->ino < b->ino) return 1; + } + if (a->mmap2_ino_generation_valid && b->mmap2_ino_generation_valid) { + if (a->ino_generation > b->ino_generation) return -1; + if (a->ino_generation < b->ino_generation) return 1; + } + if (build_id__is_defined(&a->build_id) && build_id__is_defined(&b->build_id)) { + if (a->build_id.size != b->build_id.size) + return a->build_id.size < b->build_id.size ? -1 : 1; + return memcmp(a->build_id.data, b->build_id.data, a->build_id.size); + } + return 0; +} + +const struct dso_id dso_id_empty = { + { + .maj = 0, + .min = 0, + .ino = 0, + .ino_generation = 0, + }, + .mmap2_valid = false, + .mmap2_ino_generation_valid = false, + { + .size = 0, + } +}; + +void __dso__improve_id(struct dso *dso, const struct dso_id *id) +{ + struct dsos *dsos = dso__dsos(dso); + struct dso_id *dso_id = dso__id(dso); + bool changed = false; + + /* dsos write lock held by caller. */ + + if (id->mmap2_valid && !dso_id->mmap2_valid) { + dso_id->maj = id->maj; + dso_id->min = id->min; + dso_id->ino = id->ino; + dso_id->mmap2_valid = true; + changed = true; + } + if (id->mmap2_ino_generation_valid && !dso_id->mmap2_ino_generation_valid) { + dso_id->ino_generation = id->ino_generation; + dso_id->mmap2_ino_generation_valid = true; + changed = true; + } + if (build_id__is_defined(&id->build_id) && !build_id__is_defined(&dso_id->build_id)) { + dso_id->build_id = id->build_id; + changed = true; + } + if (changed && dsos) + dsos->sorted = false; +} + +int dso_id__cmp(const struct dso_id *a, const struct dso_id *b) +{ + if (a == &dso_id_empty || b == &dso_id_empty) { + /* There is no valid data to compare so the comparison always returns identical. */ + return 0; + } + + return __dso_id__cmp(a, b); +} + +int dso__cmp_id(struct dso *a, struct dso *b) +{ + return __dso_id__cmp(dso__id(a), dso__id(b)); +} + +void dso__set_long_name(struct dso *dso, const char *name, bool name_allocated) +{ + dso__set_long_name_id(dso, name, name_allocated); +} + +static void __dso__set_short_name(struct dso *dso, const char *name, bool name_allocated) +{ + if (dso__short_name_allocated(dso)) + free((char *)dso__short_name(dso)); + + RC_CHK_ACCESS(dso)->short_name = name; + RC_CHK_ACCESS(dso)->short_name_len = strlen(name); + dso__set_short_name_allocated(dso, name_allocated); +} + +void dso__set_short_name(struct dso *dso, const char *name, bool name_allocated) +{ + struct dsos *dsos = dso__dsos(dso); + + if (name == NULL) + return; + + if (dsos) { + /* + * Need to avoid re-sorting the dsos breaking by non-atomically + * renaming the dso. + */ + down_write(&dsos->lock); + __dso__set_short_name(dso, name, name_allocated); + dsos->sorted = false; + up_write(&dsos->lock); + } else { + __dso__set_short_name(dso, name, name_allocated); + } +} + +int dso__name_len(const struct dso *dso) +{ + if (!dso) + return strlen("[unknown]"); + if (verbose > 0) + return dso__long_name_len(dso); + + return dso__short_name_len(dso); +} + +bool dso__loaded(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->loaded; +} + +bool dso__sorted_by_name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->sorted_by_name; +} + +void dso__set_sorted_by_name(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->sorted_by_name = true; +} + +struct dso *dso__new_id(const char *name, const struct dso_id *id) +{ + RC_STRUCT(dso) *dso = zalloc(sizeof(*dso) + strlen(name) + 1); + struct dso *res; + struct dso_data *data; + + if (!dso) + return NULL; + + if (ADD_RC_CHK(res, dso)) { + strcpy(dso->name, name); + if (id) + dso->id = *id; + dso__set_long_name_id(res, dso->name, false); + dso__set_short_name(res, dso->name, false); + dso->symbols = RB_ROOT_CACHED; + dso->symbol_names = NULL; + dso->symbol_names_len = 0; + dso->inlined_nodes = RB_ROOT_CACHED; + dso->srclines = RB_ROOT_CACHED; + dso->data_types = RB_ROOT; + dso->global_vars = RB_ROOT; + dso->data.fd = -1; + dso->data.status = DSO_DATA_STATUS_UNKNOWN; + dso->symtab_type = DSO_BINARY_TYPE__NOT_FOUND; + dso->binary_type = DSO_BINARY_TYPE__NOT_FOUND; + dso->is_64_bit = (sizeof(void *) == 8); + dso->loaded = 0; + dso->rel = 0; + dso->sorted_by_name = 0; + dso->has_srcline = 1; + dso->a2l_fails = 1; + dso->kernel = DSO_SPACE__USER; + dso->is_kmod = 0; + dso->needs_swap = DSO_SWAP__UNSET; + dso->comp = COMP_ID__NONE; + mutex_init(&dso->lock); + refcount_set(&dso->refcnt, 1); + data = &dso->data; + data->cache = RB_ROOT; + data->fd = -1; + data->status = DSO_DATA_STATUS_UNKNOWN; + INIT_LIST_HEAD(&data->open_entry); +#ifdef REFCNT_CHECKING + data->dso = NULL; /* Set when on the open_entry list. */ +#endif + } + return res; +} + +struct dso *dso__new(const char *name) +{ + return dso__new_id(name, NULL); +} + +void dso__delete(struct dso *dso) +{ + if (dso__dsos(dso)) + pr_err("DSO %s is still in rbtree when being deleted!\n", dso__long_name(dso)); + + /* free inlines first, as they reference symbols */ + inlines__tree_delete(&RC_CHK_ACCESS(dso)->inlined_nodes); + srcline__tree_delete(&RC_CHK_ACCESS(dso)->srclines); + symbols__delete(&RC_CHK_ACCESS(dso)->symbols); + RC_CHK_ACCESS(dso)->symbol_names_len = 0; + zfree(&RC_CHK_ACCESS(dso)->symbol_names); + annotated_data_type__tree_delete(dso__data_types(dso)); + global_var_type__tree_delete(dso__global_vars(dso)); + + if (RC_CHK_ACCESS(dso)->short_name_allocated) { + zfree((char **)&RC_CHK_ACCESS(dso)->short_name); + RC_CHK_ACCESS(dso)->short_name_allocated = false; + } + + if (RC_CHK_ACCESS(dso)->long_name_allocated) { + zfree((char **)&RC_CHK_ACCESS(dso)->long_name); + RC_CHK_ACCESS(dso)->long_name_allocated = false; + } + + dso__data_close(dso); + auxtrace_cache__free(RC_CHK_ACCESS(dso)->auxtrace_cache); + dso_cache__free(dso); + dso__free_a2l(dso); + dso__free_libdw(dso); + dso__free_symsrc_filename(dso); + nsinfo__zput(RC_CHK_ACCESS(dso)->nsinfo); + mutex_destroy(dso__lock(dso)); + RC_CHK_FREE(dso); +} + +struct dso *dso__get(struct dso *dso) +{ + struct dso *result; + + if (RC_CHK_GET(result, dso)) + refcount_inc(&RC_CHK_ACCESS(dso)->refcnt); + + return result; +} + +void dso__put(struct dso *dso) +{ +#ifdef REFCNT_CHECKING + if (dso && dso__data(dso) && refcount_read(&RC_CHK_ACCESS(dso)->refcnt) == 2) + dso__data_close(dso); +#endif + if (dso && refcount_dec_and_test(&RC_CHK_ACCESS(dso)->refcnt)) + dso__delete(dso); + else + RC_CHK_PUT(dso); +} + +int dso__swap_init(struct dso *dso, unsigned char eidata) +{ + enum dso_swap_type type = dso_swap_type__from_elf_data(eidata); + + dso__set_needs_swap(dso, type); + if (type == DSO_SWAP__UNSET) { + pr_err("unrecognized DSO data encoding %d\n", eidata); + return -EINVAL; + } + return 0; +} + +void dso__set_build_id(struct dso *dso, const struct build_id *bid) +{ + dso__id(dso)->build_id = *bid; +} + +bool dso__build_id_equal(const struct dso *dso, const struct build_id *bid) +{ + const struct build_id *dso_bid = dso__bid(dso); + + if (dso_bid->size > bid->size && dso_bid->size == BUILD_ID_SIZE) { + /* + * For the backward compatibility, it allows a build-id has + * trailing zeros. + */ + return !memcmp(dso_bid->data, bid->data, bid->size) && + !memchr_inv(&dso_bid->data[bid->size], 0, + dso_bid->size - bid->size); + } + + return dso_bid->size == bid->size && + memcmp(dso_bid->data, bid->data, dso_bid->size) == 0; +} + +void dso__read_running_kernel_build_id(struct dso *dso, struct machine *machine) +{ + char path[PATH_MAX]; + struct build_id bid = { .size = 0, }; + + if (machine__is_default_guest(machine)) + return; + snprintf(path, sizeof(path), "%s/sys/kernel/notes", machine->root_dir); + sysfs__read_build_id(path, &bid); + dso__set_build_id(dso, &bid); +} + +int dso__kernel_module_get_build_id(struct dso *dso, + const char *root_dir) +{ + char filename[PATH_MAX]; + struct build_id bid = { .size = 0, }; + /* + * kernel module short names are of the form "[module]" and + * we need just "module" here. + */ + const char *name = dso__short_name(dso) + 1; + + snprintf(filename, sizeof(filename), + "%s/sys/module/%.*s/notes/.note.gnu.build-id", + root_dir, (int)strlen(name) - 1, name); + + sysfs__read_build_id(filename, &bid); + dso__set_build_id(dso, &bid); + return 0; +} + +static size_t dso__fprintf_buildid(struct dso *dso, FILE *fp) +{ + char sbuild_id[SBUILD_ID_SIZE]; + + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + return fprintf(fp, "%s", sbuild_id); +} + +size_t dso__fprintf(struct dso *dso, FILE *fp) +{ + struct rb_node *nd; + size_t ret = fprintf(fp, "dso: %s (", dso__short_name(dso)); + + if (dso__short_name(dso) != dso__long_name(dso)) + ret += fprintf(fp, "%s, ", dso__long_name(dso)); + ret += fprintf(fp, "%sloaded, ", dso__loaded(dso) ? "" : "NOT "); + ret += dso__fprintf_buildid(dso, fp); + ret += fprintf(fp, ")\n"); + for (nd = rb_first_cached(dso__symbols(dso)); nd; nd = rb_next(nd)) { + struct symbol *pos = rb_entry(nd, struct symbol, rb_node); + ret += symbol__fprintf(pos, fp); + } + + return ret; +} + +enum dso_type dso__type(struct dso *dso, struct machine *machine) +{ + int fd = -1; + enum dso_type type = DSO__TYPE_UNKNOWN; + + if (dso__data_get_fd(dso, machine, &fd)) { + type = dso__type_fd(fd); + dso__data_put_fd(dso); + } + + return type; +} + +int dso__strerror_load(struct dso *dso, char *buf, size_t buflen) +{ + int idx, errnum = *dso__load_errno(dso); + /* + * This must have a same ordering as the enum dso_load_errno. + */ + static const char *dso_load__error_str[] = { + "Internal tools/perf/ library error", + "Invalid ELF file", + "Can not read build id", + "Mismatching build id", + "Decompression failure", + }; + + BUG_ON(buflen == 0); + + if (errnum >= 0) { + errno = errnum; + scnprintf(buf, buflen, "%m"); + + return 0; + } + + if (errnum < __DSO_LOAD_ERRNO__START || errnum >= __DSO_LOAD_ERRNO__END) + return -1; + + idx = errnum - __DSO_LOAD_ERRNO__START; + scnprintf(buf, buflen, "%s", dso_load__error_str[idx]); + return 0; +} + +bool perf_pid_map_tid(const char *dso_name, int *tid) +{ + return sscanf(dso_name, "/tmp/perf-%d.map", tid) == 1; +} + +bool is_perf_pid_map_name(const char *dso_name) +{ + int tid; + + return perf_pid_map_tid(dso_name, &tid); +} + +struct find_file_offset_data { + u64 ip; + u64 offset; +}; + +/* This will be called for each PHDR in an ELF binary */ +static int find_file_offset(u64 start, u64 len, u64 pgoff, void *arg) +{ + struct find_file_offset_data *data = arg; + + if (start <= data->ip && data->ip < start + len) { + data->offset = pgoff + data->ip - start; + return 1; + } + return 0; +} + +static const u8 *__dso__read_symbol(struct dso *dso, const char *symfs_filename, + u64 start, size_t len, + u8 **out_buf, u64 *out_buf_len, bool *is_64bit) +{ + struct nscookie nsc; + int fd; + ssize_t count; + struct find_file_offset_data data = { + .ip = start, + }; + u8 *code_buf = NULL; + int saved_errno; + + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + fd = open(symfs_filename, O_RDONLY | O_CLOEXEC); + saved_errno = errno; + nsinfo__mountns_exit(&nsc); + if (fd < 0) { + errno = saved_errno; + return NULL; + } + if (file__read_maps(fd, /*exe=*/true, find_file_offset, &data, is_64bit) <= 0) { + close(fd); + errno = ENOENT; + return NULL; + } + code_buf = malloc(len); + if (code_buf == NULL) { + close(fd); + errno = ENOMEM; + return NULL; + } + count = pread(fd, code_buf, len, data.offset); + saved_errno = errno; + close(fd); + if ((u64)count != len) { + free(code_buf); + errno = saved_errno; + return NULL; + } + *out_buf = code_buf; + *out_buf_len = len; + return code_buf; +} + +/* + * Read a symbol into memory for disassembly by a library like capstone of + * libLLVM. If memory is allocated out_buf holds it. + */ +const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, + const struct map *map, const struct symbol *sym, + u8 **out_buf, u64 *out_buf_len, bool *is_64bit) +{ + u64 start = map__rip_2objdump(map, sym->start); + u64 end = map__rip_2objdump(map, sym->end); + size_t len = end - start; + + *out_buf = NULL; + *out_buf_len = 0; + *is_64bit = false; + + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_IMAGE) { + /* + * Note, there is fallback BPF image disassembly in the objdump + * version but it currently does nothing. + */ + errno = EOPNOTSUPP; + return NULL; + } + if (dso__binary_type(dso) == DSO_BINARY_TYPE__BPF_PROG_INFO) { +#ifdef HAVE_LIBBPF_SUPPORT + struct bpf_prog_info_node *info_node; + struct perf_bpil *info_linear; + + *is_64bit = sizeof(void *) == sizeof(u64); + info_node = perf_env__find_bpf_prog_info(dso__bpf_prog(dso)->env, + dso__bpf_prog(dso)->id); + if (!info_node) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } + info_linear = info_node->info_linear; + if (!(info_linear->arrays & (1UL << PERF_BPIL_JITED_INSNS))) { + errno = SYMBOL_ANNOTATE_ERRNO__BPF_MISSING_BTF; + return NULL; + } + assert(len <= info_linear->info.jited_prog_len); + *out_buf_len = len; + return (const u8 *)(uintptr_t)(info_linear->info.jited_prog_insns); +#else + pr_debug("No BPF program disassembly support\n"); + errno = EOPNOTSUPP; + return NULL; +#endif + } + return __dso__read_symbol(dso, symfs_filename, start, len, + out_buf, out_buf_len, is_64bit); +} + +struct debuginfo *dso__debuginfo(struct dso *dso) +{ + char *name; + bool decomp = false; + struct debuginfo *dinfo = NULL; + + mutex_lock(dso__lock(dso)); + + name = dso__get_filename(dso, "", &decomp); + if (name) + dinfo = debuginfo__new(name); + + if (decomp) + unlink(name); + + mutex_unlock(dso__lock(dso)); + free(name); + return dinfo; +} diff --git a/tools/perf/util/dso.h b/tools/perf/util/dso.h new file mode 100644 index 000000000..2916b954a --- /dev/null +++ b/tools/perf/util/dso.h @@ -0,0 +1,954 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DSO +#define __PERF_DSO + +#include +#include +#include +#include +#include +#include +#include +#include "build-id.h" +#include "debuginfo.h" +#include "mutex.h" +#include + +struct machine; +struct map; +struct perf_env; + +#define DSO__NAME_KALLSYMS "[kernel.kallsyms]" +#define DSO__NAME_KCORE "[kernel.kcore]" + +/** + * enum dso_binary_type - The kind of DSO generally associated with a memory + * region (struct map). + */ +enum dso_binary_type { + /** @DSO_BINARY_TYPE__KALLSYMS: Symbols from /proc/kallsyms file. */ + DSO_BINARY_TYPE__KALLSYMS = 0, + /** @DSO_BINARY_TYPE__GUEST_KALLSYMS: Guest /proc/kallsyms file. */ + DSO_BINARY_TYPE__GUEST_KALLSYMS, + /** @DSO_BINARY_TYPE__VMLINUX: Path to kernel /boot/vmlinux file. */ + DSO_BINARY_TYPE__VMLINUX, + /** @DSO_BINARY_TYPE__GUEST_VMLINUX: Path to guest kernel /boot/vmlinux file. */ + DSO_BINARY_TYPE__GUEST_VMLINUX, + /** @DSO_BINARY_TYPE__JAVA_JIT: Symbols from /tmp/perf.map file. */ + DSO_BINARY_TYPE__JAVA_JIT, + /** + * @DSO_BINARY_TYPE__DEBUGLINK: Debug file readable from the file path + * in the .gnu_debuglink ELF section of the dso. + */ + DSO_BINARY_TYPE__DEBUGLINK, + /** + * @DSO_BINARY_TYPE__BUILD_ID_CACHE: File named after buildid located in + * the buildid cache with an elf filename. + */ + DSO_BINARY_TYPE__BUILD_ID_CACHE, + /** + * @DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO: File named after buildid + * located in the buildid cache with a debug filename. + */ + DSO_BINARY_TYPE__BUILD_ID_CACHE_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__FEDORA_DEBUGINFO: Debug file in /usr/lib/debug + * with .debug suffix. + */ + DSO_BINARY_TYPE__FEDORA_DEBUGINFO, + /** @DSO_BINARY_TYPE__UBUNTU_DEBUGINFO: Debug file in /usr/lib/debug. */ + DSO_BINARY_TYPE__UBUNTU_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO: dso__long_name debuginfo + * file in /usr/lib/debug/lib rather than the expected + * /usr/lib/debug/usr/lib. + */ + DSO_BINARY_TYPE__MIXEDUP_UBUNTU_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__BUILDID_DEBUGINFO: File named after buildid located + * in /usr/lib/debug/.build-id/. + */ + DSO_BINARY_TYPE__BUILDID_DEBUGINFO, + /** + * @DSO_BINARY_TYPE__GNU_DEBUGDATA: MiniDebuginfo where a compressed + * ELF file is placed in a .gnu_debugdata section. + */ + DSO_BINARY_TYPE__GNU_DEBUGDATA, + /** @DSO_BINARY_TYPE__SYSTEM_PATH_DSO: A regular executable/shared-object file. */ + DSO_BINARY_TYPE__SYSTEM_PATH_DSO, + /** @DSO_BINARY_TYPE__GUEST_KMODULE: Guest kernel module .ko file. */ + DSO_BINARY_TYPE__GUEST_KMODULE, + /** @DSO_BINARY_TYPE__GUEST_KMODULE_COMP: Guest kernel module .ko.gz file. */ + DSO_BINARY_TYPE__GUEST_KMODULE_COMP, + /** @DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE: Kernel module .ko file. */ + DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE, + /** @DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP: Kernel module .ko.gz file. */ + DSO_BINARY_TYPE__SYSTEM_PATH_KMODULE_COMP, + /** @DSO_BINARY_TYPE__KCORE: /proc/kcore file. */ + DSO_BINARY_TYPE__KCORE, + /** @DSO_BINARY_TYPE__GUEST_KCORE: Guest /proc/kcore file. */ + DSO_BINARY_TYPE__GUEST_KCORE, + /** + * @DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO: Openembedded/Yocto -dbg + * package debug info. + */ + DSO_BINARY_TYPE__OPENEMBEDDED_DEBUGINFO, + /** @DSO_BINARY_TYPE__BPF_PROG_INFO: jitted BPF code. */ + DSO_BINARY_TYPE__BPF_PROG_INFO, + /** @DSO_BINARY_TYPE__BPF_IMAGE: jitted BPF trampoline or dispatcher code. */ + DSO_BINARY_TYPE__BPF_IMAGE, + /** + * @DSO_BINARY_TYPE__OOL: out of line code such as kprobe-replaced + * instructions or optimized kprobes or ftrace trampolines. + */ + DSO_BINARY_TYPE__OOL, + /** @DSO_BINARY_TYPE__NOT_FOUND: Unknown DSO kind. */ + DSO_BINARY_TYPE__NOT_FOUND, +}; + +enum dso_space_type { + DSO_SPACE__USER = 0, + DSO_SPACE__KERNEL, + DSO_SPACE__KERNEL_GUEST +}; + +enum dso_swap_type { + DSO_SWAP__UNSET, + DSO_SWAP__NO, + DSO_SWAP__YES, +}; + +enum dso_data_status { + DSO_DATA_STATUS_ERROR = -1, + DSO_DATA_STATUS_UNKNOWN = 0, + DSO_DATA_STATUS_OK = 1, +}; + +enum dso_data_status_seen { + DSO_DATA_STATUS_SEEN_ITRACE, +}; + +enum dso_type { + DSO__TYPE_UNKNOWN, + DSO__TYPE_64BIT, + DSO__TYPE_32BIT, + DSO__TYPE_X32BIT, +}; + +enum dso_load_errno { + DSO_LOAD_ERRNO__SUCCESS = 0, + + /* + * Choose an arbitrary negative big number not to clash with standard + * errno since SUS requires the errno has distinct positive values. + * See 'Issue 6' in the link below. + * + * http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/errno.h.html + */ + __DSO_LOAD_ERRNO__START = -10000, + + DSO_LOAD_ERRNO__INTERNAL_ERROR = __DSO_LOAD_ERRNO__START, + + /* for symsrc__init() */ + DSO_LOAD_ERRNO__INVALID_ELF, + DSO_LOAD_ERRNO__CANNOT_READ_BUILDID, + DSO_LOAD_ERRNO__MISMATCHING_BUILDID, + + /* for decompress_kmodule */ + DSO_LOAD_ERRNO__DECOMPRESSION_FAILURE, + + __DSO_LOAD_ERRNO__END, +}; + +#define DSO_SWAP_TYPE__SWAP(swap_type, type, val) \ +({ \ + type ____r = val; \ + BUG_ON(swap_type == DSO_SWAP__UNSET); \ + if (swap_type == DSO_SWAP__YES) { \ + switch (sizeof(____r)) { \ + case 2: \ + ____r = bswap_16(val); \ + break; \ + case 4: \ + ____r = bswap_32(val); \ + break; \ + case 8: \ + ____r = bswap_64(val); \ + break; \ + default: \ + BUG_ON(1); \ + } \ + } \ + ____r; \ +}) + +#define DSO__SWAP(dso, type, val) DSO_SWAP_TYPE__SWAP(dso__needs_swap(dso), type, val) + +#define DSO__DATA_CACHE_SIZE 4096 +#define DSO__DATA_CACHE_MASK ~(DSO__DATA_CACHE_SIZE - 1) + +/** + * struct dso_id + * + * Data about backing storage DSO, comes from PERF_RECORD_MMAP2 meta events, + * reading from /proc/pid/maps or synthesis of build_ids from DSOs. Possibly + * incomplete at any particular use. + */ +struct dso_id { + /* Data related to the mmap2 event or read from /proc/pid/maps. */ + struct { + u32 maj; + u32 min; + u64 ino; + u64 ino_generation; + }; + /** @mmap2_valid: Are the maj, min and ino fields valid? */ + bool mmap2_valid; + /** + * @mmap2_ino_generation_valid: Is the ino_generation valid? Generally + * false for /proc/pid/maps mmap event. + */ + bool mmap2_ino_generation_valid; + /** + * @build_id: A possibly populated build_id. build_id__is_defined checks + * whether it is populated. + */ + struct build_id build_id; +}; + +struct dso_cache { + struct rb_node rb_node; + u64 offset; + u64 size; + char data[]; +}; + +struct dso_data { + struct rb_root cache; + struct list_head open_entry; +#ifdef REFCNT_CHECKING + struct dso *dso; +#endif + int fd; + int status; + u32 status_seen; + u64 file_size; +#ifdef HAVE_LIBUNWIND_SUPPORT + u64 elf_base_addr; + u64 debug_frame_offset; + u64 eh_frame_hdr_addr; + u64 eh_frame_hdr_offset; +#endif +}; + +struct dso_bpf_prog { + u32 id; + u32 sub_id; + struct perf_env *env; +}; + +struct auxtrace_cache; + +DECLARE_RC_STRUCT(dso) { + struct mutex lock; + struct dsos *dsos; + struct rb_root_cached symbols; + struct symbol **symbol_names; + size_t symbol_names_len; + struct rb_root_cached inlined_nodes; + struct rb_root_cached srclines; + struct rb_root data_types; + struct rb_root global_vars; + + struct { + u64 addr; + struct symbol *symbol; + } last_find_result; + u64 text_offset; + u64 text_end; + const char *short_name; + const char *long_name; + void *a2l; + void *libdw; + char *symsrc_filename; + struct nsinfo *nsinfo; + struct auxtrace_cache *auxtrace_cache; + union { /* Tool specific area */ + void *priv; + u64 db_id; + }; + /* bpf prog information */ + struct dso_bpf_prog bpf_prog; + /* dso data file */ + struct dso_data data; + struct dso_id id; + unsigned int a2l_fails; + int comp; + refcount_t refcnt; + enum dso_load_errno load_errno; + u16 long_name_len; + u16 short_name_len; + enum dso_binary_type symtab_type:8; + enum dso_binary_type binary_type:8; + enum dso_space_type kernel:2; + enum dso_swap_type needs_swap:2; + bool is_kmod:1; + u8 adjust_symbols:1; + u8 header_build_id:1; + u8 has_srcline:1; + u8 hit:1; + u8 annotate_warned:1; + u8 auxtrace_warned:1; + u8 debuginfo_warned:1; + u8 short_name_allocated:1; + u8 long_name_allocated:1; + u8 is_64_bit:1; + bool sorted_by_name; + bool loaded; + u8 rel; + char name[]; +}; + +extern struct mutex _dso__data_open_lock; +extern const struct dso_id dso_id_empty; + +int dso_id__cmp(const struct dso_id *a, const struct dso_id *b); + +/* dso__for_each_symbol - iterate over the symbols of given type + * + * @dso: the 'struct dso *' in which symbols are iterated + * @pos: the 'struct symbol *' to use as a loop cursor + * @n: the 'struct rb_node *' to use as a temporary storage + */ +#define dso__for_each_symbol(dso, pos, n) \ + symbols__for_each_entry(dso__symbols(dso), pos, n) + +static inline void *dso__a2l(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->a2l; +} + +static inline void dso__set_a2l(struct dso *dso, void *val) +{ + RC_CHK_ACCESS(dso)->a2l = val; +} + +static inline void *dso__libdw(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->libdw; +} + +static inline void dso__set_libdw(struct dso *dso, void *val) +{ + RC_CHK_ACCESS(dso)->libdw = val; +} + +struct Dwfl; +#ifdef HAVE_LIBDW_SUPPORT +struct Dwfl *dso__libdw_dwfl(struct dso *dso); +#else +static inline struct Dwfl *dso__libdw_dwfl(struct dso *dso __maybe_unused) +{ + return NULL; +} +#endif + +static inline unsigned int dso__a2l_fails(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->a2l_fails; +} + +static inline void dso__set_a2l_fails(struct dso *dso, unsigned int val) +{ + RC_CHK_ACCESS(dso)->a2l_fails = val; +} + +static inline bool dso__adjust_symbols(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->adjust_symbols; +} + +static inline void dso__set_adjust_symbols(struct dso *dso, bool val) +{ + RC_CHK_ACCESS(dso)->adjust_symbols = val; +} + +static inline bool dso__annotate_warned(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->annotate_warned; +} + +static inline void dso__set_annotate_warned(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->annotate_warned = 1; +} + +static inline bool dso__debuginfo_warned(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->debuginfo_warned; +} + +static inline void dso__set_debuginfo_warned(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->debuginfo_warned = 1; +} + +static inline bool dso__auxtrace_warned(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->auxtrace_warned; +} + +static inline void dso__set_auxtrace_warned(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->auxtrace_warned = 1; +} + +static inline struct auxtrace_cache *dso__auxtrace_cache(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->auxtrace_cache; +} + +static inline void dso__set_auxtrace_cache(struct dso *dso, struct auxtrace_cache *cache) +{ + RC_CHK_ACCESS(dso)->auxtrace_cache = cache; +} + +static inline struct dso_bpf_prog *dso__bpf_prog(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->bpf_prog; +} + +static inline bool dso__has_srcline(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->has_srcline; +} + +static inline void dso__set_has_srcline(struct dso *dso, bool val) +{ + RC_CHK_ACCESS(dso)->has_srcline = val; +} + +static inline int dso__comp(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->comp; +} + +static inline void dso__set_comp(struct dso *dso, int comp) +{ + RC_CHK_ACCESS(dso)->comp = comp; +} + +static inline struct dso_data *dso__data(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->data; +} + +static inline u64 dso__db_id(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->db_id; +} + +static inline void dso__set_db_id(struct dso *dso, u64 db_id) +{ + RC_CHK_ACCESS(dso)->db_id = db_id; +} + +static inline struct dsos *dso__dsos(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->dsos; +} + +static inline void dso__set_dsos(struct dso *dso, struct dsos *dsos) +{ + RC_CHK_ACCESS(dso)->dsos = dsos; +} + +static inline bool dso__header_build_id(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->header_build_id; +} + +static inline void dso__set_header_build_id(struct dso *dso, bool val) +{ + RC_CHK_ACCESS(dso)->header_build_id = val; +} + +static inline bool dso__hit(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->hit; +} + +static inline void dso__set_hit(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->hit = 1; +} + +static inline struct dso_id *dso__id(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->id; +} + +static inline const struct dso_id *dso__id_const(const struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->id; +} + +static inline const struct build_id *dso__bid(const struct dso *dso) +{ + return &dso__id_const(dso)->build_id; +} + +static inline bool dso__has_build_id(const struct dso *dso) +{ + return build_id__is_defined(dso__bid(dso)); +} + +static inline struct rb_root_cached *dso__inlined_nodes(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->inlined_nodes; +} + +static inline bool dso__is_64_bit(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->is_64_bit; +} + +static inline void dso__set_is_64_bit(struct dso *dso, bool is) +{ + RC_CHK_ACCESS(dso)->is_64_bit = is; +} + +static inline bool dso__is_kmod(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->is_kmod; +} + +static inline void dso__set_is_kmod(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->is_kmod = 1; +} + +static inline enum dso_space_type dso__kernel(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->kernel; +} + +static inline void dso__set_kernel(struct dso *dso, enum dso_space_type kernel) +{ + RC_CHK_ACCESS(dso)->kernel = kernel; +} + +static inline u64 dso__last_find_result_addr(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->last_find_result.addr; +} + +static inline void dso__set_last_find_result_addr(struct dso *dso, u64 addr) +{ + RC_CHK_ACCESS(dso)->last_find_result.addr = addr; +} + +static inline struct symbol *dso__last_find_result_symbol(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->last_find_result.symbol; +} + +static inline void dso__set_last_find_result_symbol(struct dso *dso, struct symbol *symbol) +{ + RC_CHK_ACCESS(dso)->last_find_result.symbol = symbol; +} + +static inline enum dso_load_errno *dso__load_errno(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->load_errno; +} + +static inline void dso__set_loaded(struct dso *dso) +{ + RC_CHK_ACCESS(dso)->loaded = true; +} + +static inline struct mutex *dso__lock(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->lock; +} + +static inline const char *dso__long_name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->long_name; +} + +static inline bool dso__long_name_allocated(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->long_name_allocated; +} + +static inline void dso__set_long_name_allocated(struct dso *dso, bool allocated) +{ + RC_CHK_ACCESS(dso)->long_name_allocated = allocated; +} + +static inline u16 dso__long_name_len(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->long_name_len; +} + +static inline const char *dso__name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->name; +} + +static inline enum dso_swap_type dso__needs_swap(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->needs_swap; +} + +static inline void dso__set_needs_swap(struct dso *dso, enum dso_swap_type type) +{ + RC_CHK_ACCESS(dso)->needs_swap = type; +} + +static inline struct nsinfo *dso__nsinfo(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->nsinfo; +} + +static inline const struct nsinfo *dso__nsinfo_const(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->nsinfo; +} + +static inline struct nsinfo **dso__nsinfo_ptr(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->nsinfo; +} + +void dso__set_nsinfo(struct dso *dso, struct nsinfo *nsi); + +static inline u8 dso__rel(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->rel; +} + +static inline void dso__set_rel(struct dso *dso, u8 rel) +{ + RC_CHK_ACCESS(dso)->rel = rel; +} + +static inline const char *dso__short_name(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->short_name; +} + +static inline bool dso__short_name_allocated(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->short_name_allocated; +} + +static inline void dso__set_short_name_allocated(struct dso *dso, bool allocated) +{ + RC_CHK_ACCESS(dso)->short_name_allocated = allocated; +} + +static inline u16 dso__short_name_len(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->short_name_len; +} + +static inline struct rb_root_cached *dso__srclines(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->srclines; +} + +static inline struct rb_root *dso__data_types(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->data_types; +} + +static inline struct rb_root *dso__global_vars(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->global_vars; +} + +static inline struct rb_root_cached *dso__symbols(struct dso *dso) +{ + return &RC_CHK_ACCESS(dso)->symbols; +} + +static inline struct symbol **dso__symbol_names(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symbol_names; +} + +static inline void dso__set_symbol_names(struct dso *dso, struct symbol **names) +{ + RC_CHK_ACCESS(dso)->symbol_names = names; +} + +static inline size_t dso__symbol_names_len(struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symbol_names_len; +} + +static inline void dso__set_symbol_names_len(struct dso *dso, size_t len) +{ + RC_CHK_ACCESS(dso)->symbol_names_len = len; +} + +static inline const char *dso__symsrc_filename(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symsrc_filename; +} + +static inline void dso__set_symsrc_filename(struct dso *dso, char *val) +{ + RC_CHK_ACCESS(dso)->symsrc_filename = val; +} + +static inline void dso__free_symsrc_filename(struct dso *dso) +{ + zfree(&RC_CHK_ACCESS(dso)->symsrc_filename); +} + +static inline enum dso_binary_type dso__symtab_type(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->symtab_type; +} + +static inline void dso__set_symtab_type(struct dso *dso, enum dso_binary_type bt) +{ + RC_CHK_ACCESS(dso)->symtab_type = bt; +} + +static inline u64 dso__text_end(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->text_end; +} + +static inline void dso__set_text_end(struct dso *dso, u64 val) +{ + RC_CHK_ACCESS(dso)->text_end = val; +} + +static inline u64 dso__text_offset(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->text_offset; +} + +static inline void dso__set_text_offset(struct dso *dso, u64 val) +{ + RC_CHK_ACCESS(dso)->text_offset = val; +} + +struct dso *dso__new_id(const char *name, const struct dso_id *id); +struct dso *dso__new(const char *name); +void dso__delete(struct dso *dso); + +int dso__cmp_id(struct dso *a, struct dso *b); +void dso__set_short_name(struct dso *dso, const char *name, bool name_allocated); +void dso__set_long_name(struct dso *dso, const char *name, bool name_allocated); +void __dso__improve_id(struct dso *dso, const struct dso_id *id); + +int dso__name_len(const struct dso *dso); + +struct dso *dso__get(struct dso *dso); +void dso__put(struct dso *dso) LOCKS_EXCLUDED(_dso__data_open_lock); + +static inline void __dso__zput(struct dso **dso) +{ + dso__put(*dso); + *dso = NULL; +} + +#define dso__zput(dso) __dso__zput(&dso) + +bool dso__loaded(const struct dso *dso); + +static inline bool dso__has_symbols(const struct dso *dso) +{ + return !RB_EMPTY_ROOT(&RC_CHK_ACCESS(dso)->symbols.rb_root); +} + +char *dso__filename_with_chroot(const struct dso *dso, const char *filename); + +bool dso__sorted_by_name(const struct dso *dso); +void dso__set_sorted_by_name(struct dso *dso); +void dso__sort_by_name(struct dso *dso); + +int dso__swap_init(struct dso *dso, unsigned char eidata); + +void dso__set_build_id(struct dso *dso, const struct build_id *bid); +bool dso__build_id_equal(const struct dso *dso, const struct build_id *bid); +void dso__read_running_kernel_build_id(struct dso *dso, + struct machine *machine); +int dso__kernel_module_get_build_id(struct dso *dso, const char *root_dir); + +char dso__symtab_origin(const struct dso *dso); +int dso__read_binary_type_filename(const struct dso *dso, enum dso_binary_type type, + const char *root_dir, char *filename, size_t size); +bool is_kernel_module(const char *pathname, int cpumode); +bool dso__needs_decompress(struct dso *dso); +int dso__decompress_kmodule_fd(struct dso *dso, const char *name); +int dso__decompress_kmodule_path(struct dso *dso, const char *name, + char *pathname, size_t len); +int filename__decompress(const char *name, char *pathname, + size_t len, int comp, int *err); + +#define KMOD_DECOMP_NAME "/tmp/perf-kmod-XXXXXX" +#define KMOD_DECOMP_LEN sizeof(KMOD_DECOMP_NAME) + +struct kmod_path { + char *name; + int comp; + bool kmod; +}; + +int __kmod_path__parse(struct kmod_path *m, const char *path, + bool alloc_name); + +#define kmod_path__parse(__m, __p) __kmod_path__parse(__m, __p, false) +#define kmod_path__parse_name(__m, __p) __kmod_path__parse(__m, __p, true) + +void dso__set_module_info(struct dso *dso, struct kmod_path *m, + struct machine *machine); + +/* + * The dso__data_* external interface provides following functions: + * dso__data_get_fd + * dso__data_put_fd + * dso__data_close + * dso__data_size + * dso__data_read_offset + * dso__data_read_addr + * dso__data_write_cache_offs + * dso__data_write_cache_addr + * + * Please refer to the dso.c object code for each function and + * arguments documentation. Following text tries to explain the + * dso file descriptor caching. + * + * The dso__data* interface allows caching of opened file descriptors + * to speed up the dso data accesses. The idea is to leave the file + * descriptor opened ideally for the whole life of the dso object. + * + * The current usage of the dso__data_* interface is as follows: + * + * Get DSO's fd: + * int fd; + * if (dso__data_get_fd(dso, machine, &fd)) { + * USE 'fd' SOMEHOW + * dso__data_put_fd(dso); + * } + * + * Read DSO's data: + * n = dso__data_read_offset(dso_0, &machine, 0, buf, BUFSIZE); + * n = dso__data_read_addr(dso_0, &machine, 0, buf, BUFSIZE); + * + * Eventually close DSO's fd: + * dso__data_close(dso); + * + * It is not necessary to close the DSO object data file. Each time new + * DSO data file is opened, the limit (RLIMIT_NOFILE/2) is checked. Once + * it is crossed, the oldest opened DSO object is closed. + * + * The dso__delete function calls close_dso function to ensure the + * data file descriptor gets closed/unmapped before the dso object + * is freed. + * + * TODO +*/ +bool dso__data_get_fd(struct dso *dso, struct machine *machine, int *fd) + EXCLUSIVE_TRYLOCK_FUNCTION(true, _dso__data_open_lock); +void dso__data_put_fd(struct dso *dso) UNLOCK_FUNCTION(_dso__data_open_lock); +void dso__data_close(struct dso *dso) LOCKS_EXCLUDED(_dso__data_open_lock); + +int dso__data_file_size(struct dso *dso, struct machine *machine); +off_t dso__data_size(struct dso *dso, struct machine *machine); +ssize_t dso__data_read_offset(struct dso *dso, struct machine *machine, + u64 offset, u8 *data, ssize_t size); +uint16_t dso__read_e_machine_endian(struct dso *optional_dso, int fd, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__read_e_machine(struct dso *optional_dso, int fd, uint32_t *e_flags) +{ + return dso__read_e_machine_endian(optional_dso, fd, e_flags, NULL); +} +uint16_t dso__e_machine_endian(struct dso *dso, struct machine *machine, uint32_t *e_flags, + bool *is_big_endian); +static inline uint16_t dso__e_machine(struct dso *dso, struct machine *machine, uint32_t *e_flags) +{ + return dso__e_machine_endian(dso, machine, e_flags, NULL); +} +ssize_t dso__data_read_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + u8 *data, ssize_t size); +bool dso__data_status_seen(struct dso *dso, enum dso_data_status_seen by); +ssize_t dso__data_write_cache_offs(struct dso *dso, struct machine *machine, + u64 offset, const u8 *data, ssize_t size); +ssize_t dso__data_write_cache_addr(struct dso *dso, struct map *map, + struct machine *machine, u64 addr, + const u8 *data, ssize_t size); + +struct map *dso__new_map(const char *name); +struct dso *machine__findnew_kernel(struct machine *machine, const char *name, + const char *short_name, int dso_type); + +void dso__reset_find_symbol_cache(struct dso *dso); + +size_t dso__fprintf_symbols_by_name(struct dso *dso, FILE *fp); +size_t dso__fprintf(struct dso *dso, FILE *fp); + +static inline enum dso_binary_type dso__binary_type(const struct dso *dso) +{ + return RC_CHK_ACCESS(dso)->binary_type; +} + +static inline void dso__set_binary_type(struct dso *dso, enum dso_binary_type bt) +{ + RC_CHK_ACCESS(dso)->binary_type = bt; +} + +static inline bool dso__is_vmlinux(const struct dso *dso) +{ + enum dso_binary_type bt = dso__binary_type(dso); + + return bt == DSO_BINARY_TYPE__VMLINUX || bt == DSO_BINARY_TYPE__GUEST_VMLINUX; +} + +static inline bool dso__is_kcore(const struct dso *dso) +{ + enum dso_binary_type bt = dso__binary_type(dso); + + return bt == DSO_BINARY_TYPE__KCORE || bt == DSO_BINARY_TYPE__GUEST_KCORE; +} + +static inline bool dso__is_kallsyms(const struct dso *dso) +{ + enum dso_binary_type bt = dso__binary_type(dso); + + return bt == DSO_BINARY_TYPE__KALLSYMS || bt == DSO_BINARY_TYPE__GUEST_KALLSYMS; +} + +bool dso__is_object_file(const struct dso *dso); + +void dso__free_a2l(struct dso *dso); + +enum dso_type dso__type(struct dso *dso, struct machine *machine); + +int dso__strerror_load(struct dso *dso, char *buf, size_t buflen); + +void reset_fd_limit(void); + +u64 dso__find_global_type(struct dso *dso, u64 addr); +u64 dso__findnew_global_type(struct dso *dso, u64 addr, u64 offset); + +/* Check if dso name is of format "/tmp/perf-%d.map" */ +bool perf_pid_map_tid(const char *dso_name, int *tid); +bool is_perf_pid_map_name(const char *dso_name); + +struct debuginfo *dso__debuginfo(struct dso *dso); + +const u8 *dso__read_symbol(struct dso *dso, const char *symfs_filename, + const struct map *map, const struct symbol *sym, + u8 **out_buf, u64 *out_buf_len, bool *is_64bit); + +#endif /* __PERF_DSO */ diff --git a/tools/perf/util/dsos.c b/tools/perf/util/dsos.c new file mode 100644 index 000000000..e927e707a --- /dev/null +++ b/tools/perf/util/dsos.c @@ -0,0 +1,495 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "debug.h" +#include "dsos.h" +#include "dso.h" +#include "util.h" +#include "vdso.h" +#include "namespaces.h" +#include +#include +#include +#include // filename__read_build_id +#include + +void dsos__init(struct dsos *dsos) +{ + init_rwsem(&dsos->lock); + + dsos->cnt = 0; + dsos->allocated = 0; + dsos->dsos = NULL; + dsos->sorted = true; +} + +static void dsos__purge(struct dsos *dsos) +{ + down_write(&dsos->lock); + + for (unsigned int i = 0; i < dsos->cnt; i++) { + struct dso *dso = dsos->dsos[i]; + + dso__set_dsos(dso, NULL); + dso__put(dso); + } + + zfree(&dsos->dsos); + dsos->cnt = 0; + dsos->allocated = 0; + dsos->sorted = true; + + up_write(&dsos->lock); +} + +void dsos__exit(struct dsos *dsos) +{ + dsos__purge(dsos); + exit_rwsem(&dsos->lock); +} + + +static int __dsos__for_each_dso(struct dsos *dsos, + int (*cb)(struct dso *dso, void *data), + void *data) +{ + for (unsigned int i = 0; i < dsos->cnt; i++) { + struct dso *dso = dsos->dsos[i]; + int err; + + err = cb(dso, data); + if (err) + return err; + } + return 0; +} + +struct dsos__read_build_ids_cb_args { + bool with_hits; + bool have_build_id; +}; + +static int dsos__read_build_ids_cb(struct dso *dso, void *data) +{ + struct dsos__read_build_ids_cb_args *args = data; + struct nscookie nsc; + struct build_id bid = { .size = 0, }; + + if (args->with_hits && !dso__hit(dso) && !dso__is_vdso(dso)) + return 0; + if (dso__has_build_id(dso)) { + args->have_build_id = true; + return 0; + } + nsinfo__mountns_enter(dso__nsinfo(dso), &nsc); + if (filename__read_build_id(dso__long_name(dso), &bid) > 0) { + dso__set_build_id(dso, &bid); + args->have_build_id = true; + } else if (errno == ENOENT && dso__nsinfo(dso)) { + char *new_name = dso__filename_with_chroot(dso, dso__long_name(dso)); + + if (new_name && filename__read_build_id(new_name, &bid) > 0) { + dso__set_build_id(dso, &bid); + args->have_build_id = true; + } + free(new_name); + } + nsinfo__mountns_exit(&nsc); + return 0; +} + +bool dsos__read_build_ids(struct dsos *dsos, bool with_hits) +{ + struct dsos__read_build_ids_cb_args args = { + .with_hits = with_hits, + .have_build_id = false, + }; + + dsos__for_each_dso(dsos, dsos__read_build_ids_cb, &args); + return args.have_build_id; +} + +static int __dso__cmp_long_name(const char *long_name, const struct dso_id *id, + const struct dso *b) +{ + int rc = strcmp(long_name, dso__long_name(b)); + return rc ?: dso_id__cmp(id, dso__id_const(b)); +} + +static int __dso__cmp_short_name(const char *short_name, const struct dso_id *id, + const struct dso *b) +{ + int rc = strcmp(short_name, dso__short_name(b)); + return rc ?: dso_id__cmp(id, dso__id_const(b)); +} + +static int dsos__cmp_long_name_id_short_name(const void *va, const void *vb) +{ + const struct dso *a = *((const struct dso **)va); + const struct dso *b = *((const struct dso **)vb); + int rc = strcmp(dso__long_name(a), dso__long_name(b)); + + if (!rc) { + rc = dso_id__cmp(dso__id_const(a), dso__id_const(b)); + if (!rc) + rc = strcmp(dso__short_name(a), dso__short_name(b)); + } + return rc; +} + +struct dsos__key { + const char *long_name; + const struct dso_id *id; +}; + +static int dsos__cmp_key_long_name_id(const void *vkey, const void *vdso) +{ + const struct dsos__key *key = vkey; + const struct dso *dso = *((const struct dso **)vdso); + + return __dso__cmp_long_name(key->long_name, key->id, dso); +} + +/* + * Find a matching entry and/or link current entry to RB tree. + * Either one of the dso or name parameter must be non-NULL or the + * function will not work. + */ +static struct dso *__dsos__find_by_longname_id(struct dsos *dsos, + const char *name, + const struct dso_id *id, + bool write_locked) + SHARED_LOCKS_REQUIRED(dsos->lock) +{ + struct dsos__key key = { + .long_name = name, + .id = id, + }; + struct dso **res; + + if (dsos->dsos == NULL) + return NULL; + + if (!dsos->sorted) { + if (!write_locked) { + struct dso *dso; + + up_read(&dsos->lock); + down_write(&dsos->lock); + dso = __dsos__find_by_longname_id(dsos, name, id, + /*write_locked=*/true); + up_write(&dsos->lock); + down_read(&dsos->lock); + return dso; + } + qsort(dsos->dsos, dsos->cnt, sizeof(struct dso *), + dsos__cmp_long_name_id_short_name); + dsos->sorted = true; + } + + res = bsearch(&key, dsos->dsos, dsos->cnt, sizeof(struct dso *), + dsos__cmp_key_long_name_id); + if (!res) + return NULL; + + return dso__get(*res); +} + +int __dsos__add(struct dsos *dsos, struct dso *dso) +{ + if (!dso) + return -EINVAL; + + if (dsos->cnt == dsos->allocated) { + unsigned int to_allocate = 2; + struct dso **temp; + + if (dsos->allocated > 0) + to_allocate = dsos->allocated * 2; + temp = realloc(dsos->dsos, sizeof(struct dso *) * to_allocate); + if (!temp) + return -ENOMEM; + dsos->dsos = temp; + dsos->allocated = to_allocate; + } + if (!dsos->sorted) { + dsos->dsos[dsos->cnt++] = dso__get(dso); + } else { + int low = 0, high = dsos->cnt - 1; + int insert = dsos->cnt; /* Default to inserting at the end. */ + + while (low <= high) { + int mid = low + (high - low) / 2; + int cmp = dsos__cmp_long_name_id_short_name(&dsos->dsos[mid], &dso); + + if (cmp < 0) { + low = mid + 1; + } else { + high = mid - 1; + insert = mid; + } + } + memmove(&dsos->dsos[insert + 1], &dsos->dsos[insert], + (dsos->cnt - insert) * sizeof(struct dso *)); + dsos->cnt++; + dsos->dsos[insert] = dso__get(dso); + } + dso__set_dsos(dso, dsos); + return 0; +} + +int dsos__add(struct dsos *dsos, struct dso *dso) +{ + int ret; + + down_write(&dsos->lock); + ret = __dsos__add(dsos, dso); + up_write(&dsos->lock); + return ret; +} + +struct dsos__find_id_cb_args { + const char *name; + const struct dso_id *id; + struct dso *res; +}; + +static int dsos__find_id_cb(struct dso *dso, void *data) +{ + struct dsos__find_id_cb_args *args = data; + + if (__dso__cmp_short_name(args->name, args->id, dso) == 0) { + args->res = dso__get(dso); + return 1; + } + return 0; + +} + +static struct dso *__dsos__find_id(struct dsos *dsos, const char *name, const struct dso_id *id, + bool cmp_short, bool write_locked) + SHARED_LOCKS_REQUIRED(dsos->lock) +{ + struct dso *res; + + if (cmp_short) { + struct dsos__find_id_cb_args args = { + .name = name, + .id = id, + .res = NULL, + }; + + __dsos__for_each_dso(dsos, dsos__find_id_cb, &args); + return args.res; + } + res = __dsos__find_by_longname_id(dsos, name, id, write_locked); + return res; +} + +struct dso *dsos__find(struct dsos *dsos, const char *name, bool cmp_short) +{ + struct dso *res; + + down_read(&dsos->lock); + res = __dsos__find_id(dsos, name, &dso_id_empty, cmp_short, /*write_locked=*/false); + up_read(&dsos->lock); + return res; +} + +static void dso__set_basename(struct dso *dso) +{ + bool allocated = false; + const char *base; + int tid; + + if (perf_pid_map_tid(dso__long_name(dso), &tid)) { + char *jitname; + + if (asprintf(&jitname, "[JIT] tid %d", tid) < 0) + return; + allocated = true; + base = jitname; + } else { + base = perf_basename(dso__long_name(dso)); + } + dso__set_short_name(dso, base, allocated); +} + +static struct dso *__dsos__addnew_id(struct dsos *dsos, const char *name, const struct dso_id *id) +{ + struct dso *dso = dso__new_id(name, id); + + if (dso != NULL) { + /* + * The dsos lock is held on entry, so rename the dso before + * adding it to avoid needing to take the dsos lock again to say + * the array isn't sorted. + */ + dso__set_basename(dso); + __dsos__add(dsos, dso); + } + return dso; +} + +static struct dso *__dsos__findnew_id(struct dsos *dsos, const char *name, const struct dso_id *id) + SHARED_LOCKS_REQUIRED(dsos->lock) +{ + struct dso *dso = __dsos__find_id(dsos, name, id, false, /*write_locked=*/true); + + if (dso) + __dso__improve_id(dso, id); + + return dso ? dso : __dsos__addnew_id(dsos, name, id); +} + +struct dso *dsos__findnew_id(struct dsos *dsos, const char *name, const struct dso_id *id) +{ + struct dso *dso; + down_write(&dsos->lock); + dso = __dsos__findnew_id(dsos, name, id); + up_write(&dsos->lock); + return dso; +} + +struct dsos__fprintf_buildid_cb_args { + FILE *fp; + bool (*skip)(struct dso *dso, int parm); + int parm; + size_t ret; +}; + +static int dsos__fprintf_buildid_cb(struct dso *dso, void *data) +{ + struct dsos__fprintf_buildid_cb_args *args = data; + char sbuild_id[SBUILD_ID_SIZE]; + + if (args->skip && args->skip(dso, args->parm)) + return 0; + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + args->ret += fprintf(args->fp, "%-40s %s\n", sbuild_id, dso__long_name(dso)); + return 0; +} + +size_t dsos__fprintf_buildid(struct dsos *dsos, FILE *fp, + bool (*skip)(struct dso *dso, int parm), int parm) +{ + struct dsos__fprintf_buildid_cb_args args = { + .fp = fp, + .skip = skip, + .parm = parm, + .ret = 0, + }; + + dsos__for_each_dso(dsos, dsos__fprintf_buildid_cb, &args); + return args.ret; +} + +struct dsos__fprintf_cb_args { + FILE *fp; + size_t ret; +}; + +static int dsos__fprintf_cb(struct dso *dso, void *data) +{ + struct dsos__fprintf_cb_args *args = data; + + args->ret += dso__fprintf(dso, args->fp); + return 0; +} + +size_t dsos__fprintf(struct dsos *dsos, FILE *fp) +{ + struct dsos__fprintf_cb_args args = { + .fp = fp, + .ret = 0, + }; + + dsos__for_each_dso(dsos, dsos__fprintf_cb, &args); + return args.ret; +} + +static int dsos__hit_all_cb(struct dso *dso, void *data __maybe_unused) +{ + dso__set_hit(dso); + return 0; +} + +int dsos__hit_all(struct dsos *dsos) +{ + return dsos__for_each_dso(dsos, dsos__hit_all_cb, NULL); +} + +struct dso *dsos__findnew_module_dso(struct dsos *dsos, + struct machine *machine, + struct kmod_path *m, + const char *filename) +{ + struct dso *dso; + + down_write(&dsos->lock); + + dso = __dsos__find_id(dsos, m->name, &dso_id_empty, /*cmp_short=*/true, + /*write_locked=*/true); + if (dso) { + up_write(&dsos->lock); + return dso; + } + /* + * Failed to find the dso so create it. Change the name before adding it + * to the array, to avoid unnecessary sorts and potential locking + * issues. + */ + dso = dso__new_id(m->name, /*id=*/NULL); + if (!dso) { + up_write(&dsos->lock); + return NULL; + } + dso__set_basename(dso); + dso__set_module_info(dso, m, machine); + dso__set_long_name(dso, strdup(filename), true); + dso__set_kernel(dso, DSO_SPACE__KERNEL); + __dsos__add(dsos, dso); + + up_write(&dsos->lock); + return dso; +} + +static int dsos__find_kernel_dso_cb(struct dso *dso, void *data) +{ + struct dso **res = data; + /* + * The cpumode passed to is_kernel_module is not the cpumode of *this* + * event. If we insist on passing correct cpumode to is_kernel_module, + * we should record the cpumode when we adding this dso to the linked + * list. + * + * However we don't really need passing correct cpumode. We know the + * correct cpumode must be kernel mode (if not, we should not link it + * onto kernel_dsos list). + * + * Therefore, we pass PERF_RECORD_MISC_CPUMODE_UNKNOWN. + * is_kernel_module() treats it as a kernel cpumode. + */ + if (!dso__kernel(dso) || + is_kernel_module(dso__long_name(dso), PERF_RECORD_MISC_CPUMODE_UNKNOWN)) + return 0; + + *res = dso__get(dso); + return 1; +} + +struct dso *dsos__find_kernel_dso(struct dsos *dsos) +{ + struct dso *res = NULL; + + dsos__for_each_dso(dsos, dsos__find_kernel_dso_cb, &res); + return res; +} + +int dsos__for_each_dso(struct dsos *dsos, int (*cb)(struct dso *dso, void *data), void *data) +{ + int err; + + down_read(&dsos->lock); + err = __dsos__for_each_dso(dsos, cb, data); + up_read(&dsos->lock); + return err; +} diff --git a/tools/perf/util/dsos.h b/tools/perf/util/dsos.h new file mode 100644 index 000000000..a26774950 --- /dev/null +++ b/tools/perf/util/dsos.h @@ -0,0 +1,52 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DSOS +#define __PERF_DSOS + +#include +#include +#include +#include +#include "rwsem.h" + +struct dso; +struct dso_id; +struct kmod_path; +struct machine; + +/* + * Collection of DSOs as an array for iteration speed, but sorted for O(n) + * lookup. + */ +struct dsos { + struct rw_semaphore lock; + struct dso **dsos; + unsigned int cnt; + unsigned int allocated; + bool sorted; +}; + +void dsos__init(struct dsos *dsos); +void dsos__exit(struct dsos *dsos); + +int __dsos__add(struct dsos *dsos, struct dso *dso); +int dsos__add(struct dsos *dsos, struct dso *dso); +struct dso *dsos__find(struct dsos *dsos, const char *name, bool cmp_short); + +struct dso *dsos__findnew_id(struct dsos *dsos, const char *name, const struct dso_id *id); + +bool dsos__read_build_ids(struct dsos *dsos, bool with_hits); + +size_t dsos__fprintf_buildid(struct dsos *dsos, FILE *fp, + bool (skip)(struct dso *dso, int parm), int parm); +size_t dsos__fprintf(struct dsos *dsos, FILE *fp); + +int dsos__hit_all(struct dsos *dsos); + +struct dso *dsos__findnew_module_dso(struct dsos *dsos, struct machine *machine, + struct kmod_path *m, const char *filename); + +struct dso *dsos__find_kernel_dso(struct dsos *dsos); + +int dsos__for_each_dso(struct dsos *dsos, int (*cb)(struct dso *dso, void *data), void *data); + +#endif /* __PERF_DSOS */ diff --git a/tools/perf/util/dump-insn.h b/tools/perf/util/dump-insn.h new file mode 100644 index 000000000..20d4d7bb5 --- /dev/null +++ b/tools/perf/util/dump-insn.h @@ -0,0 +1,26 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_DUMP_INSN_H +#define __PERF_DUMP_INSN_H 1 + +#define MAXINSN 15 + +#include + +struct thread; + +struct perf_insn { + /* Initialized by callers: */ + struct thread *thread; + struct machine *machine; + u8 cpumode; + bool is64bit; + int cpu; + /* Temporary */ + char out[256]; +}; + +const char *dump_insn(struct perf_insn *x, u64 ip, + u8 *inbuf, int inlen, int *lenp); +int arch_is_uncond_branch(const unsigned char *buf, size_t len, int x86_64); + +#endif diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c new file mode 100644 index 000000000..d7160f87a --- /dev/null +++ b/tools/perf/util/dwarf-aux.c @@ -0,0 +1,2182 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dwarf-aux.c : libdw auxiliary interfaces + */ + +#include +#include +#include +#include +#include "debug.h" +#include "dwarf-aux.h" +#include "dwarf-regs.h" +#include "strbuf.h" +#include "string2.h" + +/** + * cu_find_realpath - Find the realpath of the target file + * @cu_die: A DIE(dwarf information entry) of CU(compilation Unit) + * @fname: The tail filename of the target file + * + * Find the real(long) path of @fname in @cu_die. + */ +const char *cu_find_realpath(Dwarf_Die *cu_die, const char *fname) +{ + Dwarf_Files *files; + size_t nfiles, i; + const char *src = NULL; + int ret; + + if (!fname) + return NULL; + + ret = dwarf_getsrcfiles(cu_die, &files, &nfiles); + if (ret != 0) + return NULL; + + for (i = 0; i < nfiles; i++) { + src = dwarf_filesrc(files, i, NULL, NULL); + if (strtailcmp(src, fname) == 0) + break; + } + if (i == nfiles) + return NULL; + return src; +} + +/** + * cu_get_comp_dir - Get the path of compilation directory + * @cu_die: a CU DIE + * + * Get the path of compilation directory of given @cu_die. + * Since this depends on DW_AT_comp_dir, older gcc will not + * embedded it. In that case, this returns NULL. + */ +const char *cu_get_comp_dir(Dwarf_Die *cu_die) +{ + Dwarf_Attribute attr; + if (dwarf_attr(cu_die, DW_AT_comp_dir, &attr) == NULL) + return NULL; + return dwarf_formstring(&attr); +} + +/* Unlike dwarf_getsrc_die(), cu_getsrc_die() only returns statement line */ +static Dwarf_Line *cu_getsrc_die(Dwarf_Die *cu_die, Dwarf_Addr addr) +{ + Dwarf_Addr laddr; + Dwarf_Lines *lines; + Dwarf_Line *line; + size_t nlines, l, u, n; + bool flag; + + if (dwarf_getsrclines(cu_die, &lines, &nlines) != 0 || + nlines == 0) + return NULL; + + /* Lines are sorted by address, use binary search */ + l = 0; u = nlines - 1; + while (l < u) { + n = u - (u - l) / 2; + line = dwarf_onesrcline(lines, n); + if (!line || dwarf_lineaddr(line, &laddr) != 0) + return NULL; + if (addr < laddr) + u = n - 1; + else + l = n; + } + /* Going backward to find the lowest line */ + do { + line = dwarf_onesrcline(lines, --l); + if (!line || dwarf_lineaddr(line, &laddr) != 0) + return NULL; + } while (laddr == addr); + l++; + /* Going forward to find the statement line */ + do { + line = dwarf_onesrcline(lines, l++); + if (!line || dwarf_lineaddr(line, &laddr) != 0 || + dwarf_linebeginstatement(line, &flag) != 0) + return NULL; + if (laddr > addr) + return NULL; + } while (!flag); + + return line; +} + +/** + * cu_find_lineinfo - Get a line number and file name for given address + * @cu_die: a CU DIE + * @addr: An address + * @fname: a pointer which returns the file name string + * @lineno: a pointer which returns the line number + * + * Find a line number and file name for @addr in @cu_die. + */ +int cu_find_lineinfo(Dwarf_Die *cu_die, Dwarf_Addr addr, + const char **fname, int *lineno) +{ + Dwarf_Line *line; + Dwarf_Die die_mem; + Dwarf_Addr faddr; + + if (die_find_realfunc(cu_die, addr, &die_mem) + && die_entrypc(&die_mem, &faddr) == 0 && + faddr == addr) { + *fname = die_get_decl_file(&die_mem); + if (dwarf_decl_line(&die_mem, lineno) != 0) + return -ENOENT; + goto out; + } + + line = cu_getsrc_die(cu_die, addr); + if (line && dwarf_lineno(line, lineno) == 0) { + *fname = dwarf_linesrc(line, NULL, NULL); + if (!*fname) + /* line number is useless without filename */ + *lineno = 0; + } + +out: + return (*lineno && *fname) ? *lineno : -ENOENT; +} + +static int __die_find_inline_cb(Dwarf_Die *die_mem, void *data); + +/** + * cu_walk_functions_at - Walk on function DIEs at given address + * @cu_die: A CU DIE + * @addr: An address + * @callback: A callback which called with found DIEs + * @data: A user data + * + * Walk on function DIEs at given @addr in @cu_die. Passed DIEs + * should be subprogram or inlined-subroutines. + */ +int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, + int (*callback)(Dwarf_Die *, void *), void *data) +{ + Dwarf_Die die_mem; + Dwarf_Die *sc_die; + int ret = -ENOENT; + + /* Inlined function could be recursive. Trace it until fail */ + for (sc_die = die_find_realfunc(cu_die, addr, &die_mem); + sc_die != NULL; + sc_die = die_find_child(sc_die, __die_find_inline_cb, &addr, + &die_mem)) { + ret = callback(sc_die, data); + if (ret) + break; + } + + return ret; +} + +/** + * die_get_linkage_name - Get the linkage name of the object + * @dw_die: A DIE of the object + * + * Get the linkage name attribute of given @dw_die. + * For C++ binary, the linkage name will be the mangled symbol. + */ +const char *die_get_linkage_name(Dwarf_Die *dw_die) +{ + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(dw_die, DW_AT_linkage_name, &attr) == NULL) + return NULL; + return dwarf_formstring(&attr); +} + +/** + * die_compare_name - Compare diename and tname + * @dw_die: a DIE + * @tname: a string of target name + * + * Compare the name of @dw_die and @tname. Return false if @dw_die has no name. + */ +bool die_compare_name(Dwarf_Die *dw_die, const char *tname) +{ + const char *name; + + name = dwarf_diename(dw_die); + return name ? (strcmp(tname, name) == 0) : false; +} + +/** + * die_match_name - Match diename/linkage name and glob + * @dw_die: a DIE + * @glob: a string of target glob pattern + * + * Glob matching the name of @dw_die and @glob. Return false if matching fail. + * This also match linkage name. + */ +bool die_match_name(Dwarf_Die *dw_die, const char *glob) +{ + const char *name; + + name = dwarf_diename(dw_die); + if (name && strglobmatch(name, glob)) + return true; + /* fall back to check linkage name */ + name = die_get_linkage_name(dw_die); + if (name && strglobmatch(name, glob)) + return true; + + return false; +} + +/** + * die_get_call_lineno - Get callsite line number of inline-function instance + * @in_die: a DIE of an inlined function instance + * + * Get call-site line number of @in_die. This means from where the inline + * function is called. + */ +int die_get_call_lineno(Dwarf_Die *in_die) +{ + Dwarf_Attribute attr; + Dwarf_Word ret; + + if (!dwarf_attr(in_die, DW_AT_call_line, &attr)) + return -ENOENT; + + dwarf_formudata(&attr, &ret); + return (int)ret; +} + +/** + * die_get_type - Get type DIE + * @vr_die: a DIE of a variable + * @die_mem: where to store a type DIE + * + * Get a DIE of the type of given variable (@vr_die), and store + * it to die_mem. Return NULL if fails to get a type DIE. + */ +Dwarf_Die *die_get_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem) +{ + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(vr_die, DW_AT_type, &attr) && + dwarf_formref_die(&attr, die_mem)) + return die_mem; + else + return NULL; +} + +/* Get a type die, but skip qualifiers */ +Dwarf_Die *__die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem) +{ + int tag; + + do { + vr_die = die_get_type(vr_die, die_mem); + if (!vr_die) + break; + tag = dwarf_tag(vr_die); + } while (tag == DW_TAG_const_type || + tag == DW_TAG_restrict_type || + tag == DW_TAG_volatile_type || + tag == DW_TAG_shared_type); + + return vr_die; +} + +/** + * die_get_real_type - Get a type die, but skip qualifiers and typedef + * @vr_die: a DIE of a variable + * @die_mem: where to store a type DIE + * + * Get a DIE of the type of given variable (@vr_die), and store + * it to die_mem. Return NULL if fails to get a type DIE. + * If the type is qualifiers (e.g. const) or typedef, this skips it + * and tries to find real type (structure or basic types, e.g. int). + */ +Dwarf_Die *die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem) +{ + do { + vr_die = __die_get_real_type(vr_die, die_mem); + } while (vr_die && dwarf_tag(vr_die) == DW_TAG_typedef); + + return vr_die; +} + +/** + * die_get_pointer_type - Get a pointer/array type die + * @type_die: a DIE of a type + * @die_mem: where to store a type DIE + * + * Get a pointer/array type DIE from @type_die. If the type is a typedef or + * qualifier (const, volatile, etc.), follow the chain to find the underlying + * pointer type. + */ +Dwarf_Die *die_get_pointer_type(Dwarf_Die *type_die, Dwarf_Die *die_mem) +{ + int tag; + + do { + tag = dwarf_tag(type_die); + if (tag == DW_TAG_pointer_type || tag == DW_TAG_array_type) + return type_die; + if (tag != DW_TAG_typedef && tag != DW_TAG_const_type && + tag != DW_TAG_restrict_type && tag != DW_TAG_volatile_type && + tag != DW_TAG_shared_type) + return NULL; + type_die = die_get_type(type_die, die_mem); + } while (type_die); + + return NULL; +} + +/* Get attribute and translate it as a udata */ +static int die_get_attr_udata(Dwarf_Die *tp_die, unsigned int attr_name, + Dwarf_Word *result) +{ + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(tp_die, attr_name, &attr) == NULL || + dwarf_formudata(&attr, result) != 0) + return -ENOENT; + + return 0; +} + +/** + * die_is_signed_type - Check whether a type DIE is signed or not + * @tp_die: a DIE of a type + * + * Get the encoding of @tp_die and return true if the encoding + * is signed. + */ +bool die_is_signed_type(Dwarf_Die *tp_die) +{ + Dwarf_Word ret; + + if (die_get_attr_udata(tp_die, DW_AT_encoding, &ret)) + return false; + + return (ret == DW_ATE_signed_char || ret == DW_ATE_signed || + ret == DW_ATE_signed_fixed); +} + +/** + * die_is_func_def - Ensure that this DIE is a subprogram and definition + * @dw_die: a DIE + * + * Ensure that this DIE is a subprogram and NOT a declaration. This + * returns true if @dw_die is a function definition. + **/ +bool die_is_func_def(Dwarf_Die *dw_die) +{ + Dwarf_Attribute attr; + Dwarf_Addr addr = 0; + + if (dwarf_tag(dw_die) != DW_TAG_subprogram) + return false; + + if (dwarf_attr(dw_die, DW_AT_declaration, &attr)) + return false; + + /* + * DW_AT_declaration can be lost from function declaration + * by gcc's bug #97060. + * So we need to check this subprogram DIE has DW_AT_inline + * or an entry address. + */ + if (!dwarf_attr(dw_die, DW_AT_inline, &attr) && + die_entrypc(dw_die, &addr) < 0) + return false; + + return true; +} + +/** + * die_entrypc - Returns entry PC (the lowest address) of a DIE + * @dw_die: a DIE + * @addr: where to store entry PC + * + * Since dwarf_entrypc() does not return entry PC if the DIE has only address + * range, we have to use this to retrieve the lowest address from the address + * range attribute. + */ +int die_entrypc(Dwarf_Die *dw_die, Dwarf_Addr *addr) +{ + Dwarf_Addr base, end; + Dwarf_Attribute attr; + + if (!addr) + return -EINVAL; + + if (dwarf_entrypc(dw_die, addr) == 0) + return 0; + + /* + * Since the dwarf_ranges() will return 0 if there is no + * DW_AT_ranges attribute, we should check it first. + */ + if (!dwarf_attr(dw_die, DW_AT_ranges, &attr)) + return -ENOENT; + + return dwarf_ranges(dw_die, 0, &base, addr, &end) < 0 ? -ENOENT : 0; +} + +/** + * die_is_func_instance - Ensure that this DIE is an instance of a subprogram + * @dw_die: a DIE + * + * Ensure that this DIE is an instance (which has an entry address). + * This returns true if @dw_die is a function instance. If not, the @dw_die + * must be a prototype. You can use die_walk_instances() to find actual + * instances. + **/ +bool die_is_func_instance(Dwarf_Die *dw_die) +{ + Dwarf_Addr tmp; + Dwarf_Attribute attr_mem; + int tag = dwarf_tag(dw_die); + + if (tag != DW_TAG_subprogram && + tag != DW_TAG_inlined_subroutine) + return false; + + return dwarf_entrypc(dw_die, &tmp) == 0 || + dwarf_attr(dw_die, DW_AT_ranges, &attr_mem) != NULL; +} + +/** + * die_get_data_member_location - Get the data-member offset + * @mb_die: a DIE of a member of a data structure + * @offs: The offset of the member in the data structure + * + * Get the offset of @mb_die in the data structure including @mb_die, and + * stores result offset to @offs. If any error occurs this returns errno. + */ +int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs) +{ + Dwarf_Attribute attr; + Dwarf_Op *expr; + size_t nexpr; + int ret; + + if (dwarf_attr_integrate(mb_die, DW_AT_data_member_location, &attr) == NULL) + return -ENOENT; + + if (dwarf_formudata(&attr, offs) != 0) { + /* DW_AT_data_member_location should be DW_OP_plus_uconst */ + ret = dwarf_getlocation(&attr, &expr, &nexpr); + if (ret < 0 || nexpr == 0) + return -ENOENT; + + if (expr[0].atom != DW_OP_plus_uconst || nexpr != 1) { + pr_debug("Unable to get offset:Unexpected OP %x (%zd)\n", + expr[0].atom, nexpr); + return -ENOTSUP; + } + *offs = (Dwarf_Word)expr[0].number; + } + return 0; +} + +/* Get the call file index number in CU DIE */ +static int die_get_call_fileno(Dwarf_Die *in_die) +{ + Dwarf_Word idx; + + if (die_get_attr_udata(in_die, DW_AT_call_file, &idx) == 0) + return (int)idx; + else + return -ENOENT; +} + +/* Get the declared file index number in CU DIE */ +static int die_get_decl_fileno(Dwarf_Die *pdie) +{ + Dwarf_Word idx; + + if (die_get_attr_udata(pdie, DW_AT_decl_file, &idx) == 0) + return (int)idx; + else + return -ENOENT; +} + +/* Return the file name by index */ +static const char *die_get_file_name(Dwarf_Die *dw_die, int idx) +{ + Dwarf_Die cu_die; + Dwarf_Files *files; + Dwarf_Attribute attr_mem; + + if (idx < 0 || !dwarf_attr_integrate(dw_die, DW_AT_decl_file, &attr_mem) || + !dwarf_cu_die(attr_mem.cu, &cu_die, NULL, NULL, NULL, NULL, NULL, NULL) || + dwarf_getsrcfiles(&cu_die, &files, NULL) != 0) + return NULL; + + return dwarf_filesrc(files, idx, NULL, NULL); +} + +/** + * die_get_call_file - Get callsite file name of inlined function instance + * @in_die: a DIE of an inlined function instance + * + * Get call-site file name of @in_die. This means from which file the inline + * function is called. + */ +const char *die_get_call_file(Dwarf_Die *in_die) +{ + return die_get_file_name(in_die, die_get_call_fileno(in_die)); +} + +/** + * die_get_decl_file - Find the declared file name of this DIE + * @dw_die: a DIE for something declared. + * + * Get declared file name of @dw_die. + * NOTE: Since some version of clang DWARF5 implementation incorrectly uses + * file index 0 for DW_AT_decl_file, die_get_decl_file() will return NULL for + * such cases. Use this function instead. + */ +const char *die_get_decl_file(Dwarf_Die *dw_die) +{ + return die_get_file_name(dw_die, die_get_decl_fileno(dw_die)); +} + +/** + * die_find_child - Generic DIE search function in DIE tree + * @rt_die: a root DIE + * @callback: a callback function + * @data: a user data passed to the callback function + * @die_mem: a buffer for result DIE + * + * Trace DIE tree from @rt_die and call @callback for each child DIE. + * If @callback returns DIE_FIND_CB_END, this stores the DIE into + * @die_mem and returns it. If @callback returns DIE_FIND_CB_CONTINUE, + * this continues to trace the tree. Optionally, @callback can return + * DIE_FIND_CB_CHILD and DIE_FIND_CB_SIBLING, those means trace only + * the children and trace only the siblings respectively. + * Returns NULL if @callback can't find any appropriate DIE. + */ +Dwarf_Die *die_find_child(Dwarf_Die *rt_die, + int (*callback)(Dwarf_Die *, void *), + void *data, Dwarf_Die *die_mem) +{ + Dwarf_Die child_die; + int ret; + + ret = dwarf_child(rt_die, die_mem); + if (ret != 0) + return NULL; + + do { + ret = callback(die_mem, data); + if (ret == DIE_FIND_CB_END) + return die_mem; + + if ((ret & DIE_FIND_CB_CHILD) && + die_find_child(die_mem, callback, data, &child_die)) { + memcpy(die_mem, &child_die, sizeof(Dwarf_Die)); + return die_mem; + } + } while ((ret & DIE_FIND_CB_SIBLING) && + dwarf_siblingof(die_mem, die_mem) == 0); + + return NULL; +} + +struct __addr_die_search_param { + Dwarf_Addr addr; + Dwarf_Die *die_mem; +}; + +static int __die_search_func_tail_cb(Dwarf_Die *fn_die, void *data) +{ + struct __addr_die_search_param *ad = data; + Dwarf_Addr addr = 0; + + if (dwarf_tag(fn_die) == DW_TAG_subprogram && + !dwarf_highpc(fn_die, &addr) && + addr == ad->addr) { + memcpy(ad->die_mem, fn_die, sizeof(Dwarf_Die)); + return DWARF_CB_ABORT; + } + return DWARF_CB_OK; +} + +/** + * die_find_tailfunc - Search for a non-inlined function with tail call at + * given address + * @cu_die: a CU DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search for a non-inlined function DIE with tail call at @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + */ +Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + struct __addr_die_search_param ad; + ad.addr = addr; + ad.die_mem = die_mem; + /* dwarf_getscopes can't find subprogram. */ + if (dwarf_getfuncs(cu_die, __die_search_func_tail_cb, &ad, 0) <= 0) + return NULL; + else + return die_mem; +} + +/* die_find callback for non-inlined function search */ +static int __die_search_func_cb(Dwarf_Die *fn_die, void *data) +{ + struct __addr_die_search_param *ad = data; + + /* + * Since a declaration entry doesn't has given pc, this always returns + * function definition entry. + */ + if (dwarf_tag(fn_die) == DW_TAG_subprogram && + dwarf_haspc(fn_die, ad->addr)) { + memcpy(ad->die_mem, fn_die, sizeof(Dwarf_Die)); + return DWARF_CB_ABORT; + } + return DWARF_CB_OK; +} + +/** + * die_find_realfunc - Search a non-inlined function at given address + * @cu_die: a CU DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search a non-inlined function DIE which includes @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + */ +Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + struct __addr_die_search_param ad; + ad.addr = addr; + ad.die_mem = die_mem; + /* dwarf_getscopes can't find subprogram. */ + if (dwarf_getfuncs(cu_die, __die_search_func_cb, &ad, 0) <= 0) + return NULL; + else + return die_mem; +} + +/* die_find callback for inline function search */ +static int __die_find_inline_cb(Dwarf_Die *die_mem, void *data) +{ + Dwarf_Addr *addr = data; + + if (dwarf_tag(die_mem) == DW_TAG_inlined_subroutine && + dwarf_haspc(die_mem, *addr)) + return DIE_FIND_CB_END; + + return DIE_FIND_CB_CONTINUE; +} + +/** + * die_find_top_inlinefunc - Search the top inlined function at given address + * @sp_die: a subprogram DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search an inlined function DIE which includes @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + * Even if several inlined functions are expanded recursively, this + * doesn't trace it down, and returns the topmost one. + */ +Dwarf_Die *die_find_top_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + return die_find_child(sp_die, __die_find_inline_cb, &addr, die_mem); +} + +/** + * die_find_inlinefunc - Search an inlined function at given address + * @sp_die: a subprogram DIE which including @addr + * @addr: target address + * @die_mem: a buffer for result DIE + * + * Search an inlined function DIE which includes @addr. Stores the + * DIE to @die_mem and returns it if found. Returns NULL if failed. + * If several inlined functions are expanded recursively, this trace + * it down and returns deepest one. + */ +Dwarf_Die *die_find_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem) +{ + Dwarf_Die tmp_die; + + sp_die = die_find_child(sp_die, __die_find_inline_cb, &addr, &tmp_die); + if (!sp_die) + return NULL; + + /* Inlined function could be recursive. Trace it until fail */ + while (sp_die) { + memcpy(die_mem, sp_die, sizeof(Dwarf_Die)); + sp_die = die_find_child(sp_die, __die_find_inline_cb, &addr, + &tmp_die); + } + + return die_mem; +} + +static int __die_find_func_rettype_cb(Dwarf_Die *die_mem, void *data) +{ + const char *func_name; + + if (dwarf_tag(die_mem) != DW_TAG_subprogram) + return DIE_FIND_CB_SIBLING; + + func_name = dwarf_diename(die_mem); + if (func_name && !strcmp(func_name, data)) + return DIE_FIND_CB_END; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_func_rettype - Search a return type of function + * @cu_die: a CU DIE + * @name: target function name + * @die_mem: a buffer for result DIE + * + * Search a non-inlined function which matches to @name and stores the + * return type of the function to @die_mem and returns it if found. + * Returns NULL if failed. Note that it doesn't needs to find a + * definition of the function, so it doesn't match with address. + * Most likely, it can find a declaration at the top level. Thus the + * callback function continues to sibling entries only. + */ +Dwarf_Die *die_find_func_rettype(Dwarf_Die *cu_die, const char *name, + Dwarf_Die *die_mem) +{ + Dwarf_Die tmp_die; + + cu_die = die_find_child(cu_die, __die_find_func_rettype_cb, + (void *)name, &tmp_die); + if (!cu_die) + return NULL; + + if (die_get_real_type(&tmp_die, die_mem) == NULL) + return NULL; + + return die_mem; +} + +struct __instance_walk_param { + void *addr; + int (*callback)(Dwarf_Die *, void *); + void *data; + int retval; +}; + +static int __die_walk_instances_cb(Dwarf_Die *inst, void *data) +{ + struct __instance_walk_param *iwp = data; + Dwarf_Attribute attr_mem; + Dwarf_Die origin_mem; + Dwarf_Attribute *attr; + Dwarf_Die *origin; + int tmp; + + if (!die_is_func_instance(inst)) + return DIE_FIND_CB_CONTINUE; + + attr = dwarf_attr(inst, DW_AT_abstract_origin, &attr_mem); + if (attr == NULL) + return DIE_FIND_CB_CONTINUE; + + origin = dwarf_formref_die(attr, &origin_mem); + if (origin == NULL || origin->addr != iwp->addr) + return DIE_FIND_CB_CONTINUE; + + /* Ignore redundant instances */ + if (dwarf_tag(inst) == DW_TAG_inlined_subroutine) { + if (dwarf_decl_line(origin, &tmp) == 0 && die_get_call_lineno(inst) == tmp) { + tmp = die_get_decl_fileno(origin); + if (die_get_call_fileno(inst) == tmp) + return DIE_FIND_CB_CONTINUE; + } + } + + iwp->retval = iwp->callback(inst, iwp->data); + + return (iwp->retval) ? DIE_FIND_CB_END : DIE_FIND_CB_CONTINUE; +} + +/** + * die_walk_instances - Walk on instances of given DIE + * @or_die: an abstract original DIE + * @callback: a callback function which is called with instance DIE + * @data: user data + * + * Walk on the instances of give @in_die. @in_die must be an inlined function + * declaration. This returns the return value of @callback if it returns + * non-zero value, or -ENOENT if there is no instance. + */ +int die_walk_instances(Dwarf_Die *or_die, int (*callback)(Dwarf_Die *, void *), + void *data) +{ + Dwarf_Die cu_die; + Dwarf_Die die_mem; + struct __instance_walk_param iwp = { + .addr = or_die->addr, + .callback = callback, + .data = data, + .retval = -ENOENT, + }; + + if (dwarf_diecu(or_die, &cu_die, NULL, NULL) == NULL) + return -ENOENT; + + die_find_child(&cu_die, __die_walk_instances_cb, &iwp, &die_mem); + + return iwp.retval; +} + +/* Line walker internal parameters */ +struct __line_walk_param { + bool recursive; + line_walk_callback_t callback; + void *data; + int retval; +}; + +static int __die_walk_funclines_cb(Dwarf_Die *in_die, void *data) +{ + struct __line_walk_param *lw = data; + Dwarf_Addr addr = 0; + const char *fname; + int lineno; + + if (dwarf_tag(in_die) == DW_TAG_inlined_subroutine) { + fname = die_get_call_file(in_die); + lineno = die_get_call_lineno(in_die); + if (fname && lineno > 0 && die_entrypc(in_die, &addr) == 0) { + lw->retval = lw->callback(fname, lineno, addr, lw->data); + if (lw->retval != 0) + return DIE_FIND_CB_END; + } + if (!lw->recursive) + return DIE_FIND_CB_SIBLING; + } + + if (addr) { + fname = die_get_decl_file(in_die); + if (fname && dwarf_decl_line(in_die, &lineno) == 0) { + lw->retval = lw->callback(fname, lineno, addr, lw->data); + if (lw->retval != 0) + return DIE_FIND_CB_END; + } + } + + /* Continue to search nested inlined function call-sites */ + return DIE_FIND_CB_CONTINUE; +} + +/* Walk on lines of blocks included in given DIE */ +static int __die_walk_funclines(Dwarf_Die *sp_die, bool recursive, + line_walk_callback_t callback, void *data) +{ + struct __line_walk_param lw = { + .recursive = recursive, + .callback = callback, + .data = data, + .retval = 0, + }; + Dwarf_Die die_mem; + Dwarf_Addr addr; + const char *fname; + int lineno; + + /* Handle function declaration line */ + fname = die_get_decl_file(sp_die); + if (fname && dwarf_decl_line(sp_die, &lineno) == 0 && + die_entrypc(sp_die, &addr) == 0) { + lw.retval = callback(fname, lineno, addr, data); + if (lw.retval != 0) + goto done; + } + die_find_child(sp_die, __die_walk_funclines_cb, &lw, &die_mem); +done: + return lw.retval; +} + +static int __die_walk_culines_cb(Dwarf_Die *sp_die, void *data) +{ + struct __line_walk_param *lw = data; + + /* + * Since inlined function can include another inlined function in + * the same file, we need to walk in it recursively. + */ + lw->retval = __die_walk_funclines(sp_die, true, lw->callback, lw->data); + if (lw->retval != 0) + return DWARF_CB_ABORT; + + return DWARF_CB_OK; +} + +/** + * die_walk_lines - Walk on lines inside given DIE + * @rt_die: a root DIE (CU, subprogram or inlined_subroutine) + * @callback: callback routine + * @data: user data + * + * Walk on all lines inside given @rt_die and call @callback on each line. + * If the @rt_die is a function, walk only on the lines inside the function, + * otherwise @rt_die must be a CU DIE. + * Note that this walks not only dwarf line list, but also function entries + * and inline call-site. + */ +int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data) +{ + Dwarf_Lines *lines; + Dwarf_Line *line; + Dwarf_Addr addr; + const char *fname, *decf = NULL, *inf = NULL; + int lineno, ret = 0; + int decl = 0, inl; + Dwarf_Die die_mem, *cu_die; + size_t nlines, i; + bool flag; + + /* Get the CU die */ + if (dwarf_tag(rt_die) != DW_TAG_compile_unit) { + cu_die = dwarf_diecu(rt_die, &die_mem, NULL, NULL); + dwarf_decl_line(rt_die, &decl); + decf = die_get_decl_file(rt_die); + } else + cu_die = rt_die; + if (!cu_die) { + pr_debug2("Failed to get CU from given DIE.\n"); + return -EINVAL; + } + + /* Get lines list in the CU */ + if (dwarf_getsrclines(cu_die, &lines, &nlines) != 0) { + pr_debug2("Failed to get source lines on this CU.\n"); + return -ENOENT; + } + pr_debug2("Get %zd lines from this CU\n", nlines); + + /* Walk on the lines on lines list */ + for (i = 0; i < nlines; i++) { + line = dwarf_onesrcline(lines, i); + if (line == NULL || + dwarf_lineno(line, &lineno) != 0 || + dwarf_lineaddr(line, &addr) != 0) { + pr_debug2("Failed to get line info. " + "Possible error in debuginfo.\n"); + continue; + } + /* Skip end-of-sequence */ + if (dwarf_lineendsequence(line, &flag) != 0 || flag) + continue; + /* Skip Non statement line-info */ + if (dwarf_linebeginstatement(line, &flag) != 0 || !flag) + continue; + /* Filter lines based on address */ + if (rt_die != cu_die) { + /* + * Address filtering + * The line is included in given function, and + * no inline block includes it. + */ + if (!dwarf_haspc(rt_die, addr)) + continue; + + if (die_find_inlinefunc(rt_die, addr, &die_mem)) { + /* Call-site check */ + inf = die_get_call_file(&die_mem); + if ((inf == decf || (inf && decf && !strcmp(inf, decf))) && + die_get_call_lineno(&die_mem) == lineno) + goto found; + + if (dwarf_decl_line(&die_mem, &inl) != 0) + inl = 0; + if (inl != decl || + decf != die_get_decl_file(&die_mem)) + continue; + } + } +found: + /* Get source line */ + fname = dwarf_linesrc(line, NULL, NULL); + + ret = callback(fname, lineno, addr, data); + if (ret != 0) + return ret; + } + + /* + * Dwarf lines doesn't include function declarations and inlined + * subroutines. We have to check functions list or given function. + */ + if (rt_die != cu_die) + /* + * Don't need walk inlined functions recursively, because + * inner inlined functions don't have the lines of the + * specified function. + */ + ret = __die_walk_funclines(rt_die, false, callback, data); + else { + struct __line_walk_param param = { + .callback = callback, + .data = data, + .retval = 0, + }; + if (dwarf_getfuncs(cu_die, __die_walk_culines_cb, ¶m, 0) < 0) + ret = -EINVAL; + else + ret = param.retval; + } + + return ret; +} + +struct __find_variable_param { + const char *name; + Dwarf_Addr addr; +}; + +static int __die_find_variable_cb(Dwarf_Die *die_mem, void *data) +{ + struct __find_variable_param *fvp = data; + Dwarf_Attribute attr; + int tag; + + tag = dwarf_tag(die_mem); + if ((tag == DW_TAG_formal_parameter || + tag == DW_TAG_variable) && + die_compare_name(die_mem, fvp->name) && + /* + * Does the DIE have location information or const value + * or external instance? + */ + (dwarf_attr(die_mem, DW_AT_external, &attr) || + dwarf_attr(die_mem, DW_AT_location, &attr) || + dwarf_attr(die_mem, DW_AT_const_value, &attr))) + return DIE_FIND_CB_END; + if (dwarf_haspc(die_mem, fvp->addr)) + return DIE_FIND_CB_CONTINUE; + else + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_variable_at - Find a given name variable at given address + * @sp_die: a function DIE + * @name: variable name + * @addr: address + * @die_mem: a buffer for result DIE + * + * Find a variable DIE called @name at @addr in @sp_die. + */ +Dwarf_Die *die_find_variable_at(Dwarf_Die *sp_die, const char *name, + Dwarf_Addr addr, Dwarf_Die *die_mem) +{ + struct __find_variable_param fvp = { .name = name, .addr = addr}; + + return die_find_child(sp_die, __die_find_variable_cb, (void *)&fvp, + die_mem); +} + +static int __die_find_member_cb(Dwarf_Die *die_mem, void *data) +{ + const char *name = data; + + if (dwarf_tag(die_mem) == DW_TAG_member) { + if (die_compare_name(die_mem, name)) + return DIE_FIND_CB_END; + else if (!dwarf_diename(die_mem)) { /* Unnamed structure */ + Dwarf_Die type_die, tmp_die; + if (die_get_type(die_mem, &type_die) && + die_find_member(&type_die, name, &tmp_die)) + return DIE_FIND_CB_END; + } + } + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_member - Find a given name member in a data structure + * @st_die: a data structure type DIE + * @name: member name + * @die_mem: a buffer for result DIE + * + * Find a member DIE called @name in @st_die. + */ +Dwarf_Die *die_find_member(Dwarf_Die *st_die, const char *name, + Dwarf_Die *die_mem) +{ + return die_find_child(st_die, __die_find_member_cb, (void *)name, + die_mem); +} + +/** + * die_get_typename_from_type - Get the name of given type DIE + * @type_die: a type DIE + * @buf: a strbuf for result type name + * + * Get the name of @type_die and stores it to @buf. Return 0 if succeeded. + * and Return -ENOENT if failed to find type name. + * Note that the result will stores typedef name if possible, and stores + * "*(function_type)" if the type is a function pointer. + */ +int die_get_typename_from_type(Dwarf_Die *type_die, struct strbuf *buf) +{ + int tag, ret; + const char *tmp = ""; + + tag = dwarf_tag(type_die); + if (tag == DW_TAG_pointer_type) + tmp = "*"; + else if (tag == DW_TAG_array_type) + tmp = "[]"; + else if (tag == DW_TAG_subroutine_type) { + /* Function pointer */ + return strbuf_add(buf, "(function_type)", 15); + } else { + const char *name = dwarf_diename(type_die); + + if (tag == DW_TAG_union_type) + tmp = "union "; + else if (tag == DW_TAG_structure_type) + tmp = "struct "; + else if (tag == DW_TAG_enumeration_type) + tmp = "enum "; + else if (name == NULL) + return -ENOENT; + /* Write a base name */ + return strbuf_addf(buf, "%s%s", tmp, name ?: ""); + } + ret = die_get_typename(type_die, buf); + if (ret < 0) { + /* void pointer has no type attribute */ + if (tag == DW_TAG_pointer_type && ret == -ENOENT) + return strbuf_addf(buf, "void*"); + + return ret; + } + return strbuf_addstr(buf, tmp); +} + +/** + * die_get_typename - Get the name of given variable DIE + * @vr_die: a variable DIE + * @buf: a strbuf for result type name + * + * Get the name of @vr_die and stores it to @buf. Return 0 if succeeded. + * and Return -ENOENT if failed to find type name. + * Note that the result will stores typedef name if possible, and stores + * "*(function_type)" if the type is a function pointer. + */ +int die_get_typename(Dwarf_Die *vr_die, struct strbuf *buf) +{ + Dwarf_Die type; + + if (__die_get_real_type(vr_die, &type) == NULL) + return -ENOENT; + + return die_get_typename_from_type(&type, buf); +} + +/** + * die_get_varname - Get the name and type of given variable DIE + * @vr_die: a variable DIE + * @buf: a strbuf for type and variable name + * + * Get the name and type of @vr_die and stores it in @buf as "type\tname". + */ +int die_get_varname(Dwarf_Die *vr_die, struct strbuf *buf) +{ + int ret; + + ret = die_get_typename(vr_die, buf); + if (ret < 0) { + pr_debug("Failed to get type, make it unknown.\n"); + ret = strbuf_add(buf, "(unknown_type)", 14); + } + + return ret < 0 ? ret : strbuf_addf(buf, "\t%s", dwarf_diename(vr_die)); +} + +static int reg_from_dwarf_op(Dwarf_Op *op) +{ + switch (op->atom) { + case DW_OP_reg0 ... DW_OP_reg31: + return op->atom - DW_OP_reg0; + case DW_OP_breg0 ... DW_OP_breg31: + return op->atom - DW_OP_breg0; + case DW_OP_regx: + case DW_OP_bregx: + return op->number; + case DW_OP_fbreg: + return DWARF_REG_FB; + default: + break; + } + return -1; +} + +static int offset_from_dwarf_op(Dwarf_Op *op) +{ + switch (op->atom) { + case DW_OP_reg0 ... DW_OP_reg31: + case DW_OP_regx: + return 0; + case DW_OP_breg0 ... DW_OP_breg31: + case DW_OP_fbreg: + return op->number; + case DW_OP_bregx: + return op->number2; + default: + break; + } + return -1; +} + +static bool check_allowed_ops(Dwarf_Op *ops, size_t nops) +{ + /* The first op is checked separately */ + ops++; + nops--; + + /* + * It needs to make sure if the location expression matches to the given + * register and offset exactly. Thus it rejects any complex expressions + * and only allows a few of selected operators that doesn't change the + * location. + */ + while (nops) { + switch (ops->atom) { + case DW_OP_stack_value: + case DW_OP_deref_size: + case DW_OP_deref: + case DW_OP_piece: + break; + default: + return false; + } + ops++; + nops--; + } + return true; +} + +/** + * die_get_var_innermost_scope - Get innermost scope range of given variable DIE + * @sp_die: a subprogram DIE + * @vr_die: a variable DIE + * @buf: a strbuf for variable byte offset range + * + * Get the innermost scope range of @vr_die and stores it in @buf as + * "@". + */ +static int die_get_var_innermost_scope(Dwarf_Die *sp_die, Dwarf_Die *vr_die, + struct strbuf *buf) +{ + Dwarf_Die *scopes; + int count; + size_t offset = 0; + Dwarf_Addr base; + Dwarf_Addr start, end; + Dwarf_Addr entry; + int ret; + bool first = true; + const char *name; + + ret = die_entrypc(sp_die, &entry); + if (ret) + return ret; + + name = dwarf_diename(sp_die); + if (!name) + return -ENOENT; + + count = dwarf_getscopes_die(vr_die, &scopes); + + /* (*SCOPES)[1] is the DIE for the scope containing that scope */ + if (count <= 1) { + ret = -EINVAL; + goto out; + } + + while ((offset = dwarf_ranges(&scopes[1], offset, &base, + &start, &end)) > 0) { + start -= entry; + end -= entry; + + if (first) { + ret = strbuf_addf(buf, "@<%s+[%" PRIu64 "-%" PRIu64, + name, start, end); + first = false; + } else { + ret = strbuf_addf(buf, ",%" PRIu64 "-%" PRIu64, + start, end); + } + if (ret < 0) + goto out; + } + + if (!first) + ret = strbuf_add(buf, "]>", 2); + +out: + free(scopes); + return ret; +} + +/** + * die_get_var_range - Get byte offset range of given variable DIE + * @sp_die: a subprogram DIE + * @vr_die: a variable DIE + * @buf: a strbuf for type and variable name and byte offset range + * + * Get the byte offset range of @vr_die and stores it in @buf as + * "@". + */ +int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf) +{ + int ret = 0; + Dwarf_Addr base; + Dwarf_Addr start, end; + Dwarf_Addr entry; + Dwarf_Op *op; + size_t nops; + size_t offset = 0; + Dwarf_Attribute attr; + bool first = true; + const char *name; + + ret = die_entrypc(sp_die, &entry); + if (ret) + return ret; + + name = dwarf_diename(sp_die); + if (!name) + return -ENOENT; + + if (dwarf_attr(vr_die, DW_AT_location, &attr) == NULL) + return -EINVAL; + + while ((offset = dwarf_getlocations(&attr, offset, &base, + &start, &end, &op, &nops)) > 0) { + if (start == 0) { + /* Single Location Descriptions */ + ret = die_get_var_innermost_scope(sp_die, vr_die, buf); + goto out; + } + + /* Location Lists */ + start -= entry; + end -= entry; + if (first) { + ret = strbuf_addf(buf, "@<%s+[%" PRIu64 "-%" PRIu64, + name, start, end); + first = false; + } else { + ret = strbuf_addf(buf, ",%" PRIu64 "-%" PRIu64, + start, end); + } + if (ret < 0) + goto out; + } + + if (!first) + ret = strbuf_add(buf, "]>", 2); +out: + return ret; +} + +/* Interval parameters for __die_find_var_reg_cb() */ +struct find_var_data { + /* Target instruction address */ + Dwarf_Addr pc; + /* Target memory address (for global data) */ + Dwarf_Addr addr; + /* Target register */ + unsigned reg; + /* Access data type */ + Dwarf_Die type; + /* Access offset, set for global data */ + int offset; + /* True if the current register is the frame base */ + bool is_fbreg; +}; + +/* Max number of registers DW_OP_regN supports */ +#define DWARF_OP_DIRECT_REGS 32 + +static bool match_var_offset(Dwarf_Die *die_mem, struct find_var_data *data, + s64 addr_offset, s64 addr_type, bool is_pointer) +{ + Dwarf_Word size; + Dwarf_Die ptr_die; + Dwarf_Die *ptr_type; + s64 offset = addr_offset - addr_type; + + if (offset < 0) + return false; + + if (__die_get_real_type(die_mem, &data->type) == NULL) + return false; + + ptr_type = die_get_pointer_type(&data->type, &ptr_die); + if (is_pointer && ptr_type) { + /* Get the target type of the pointer */ + if (__die_get_real_type(ptr_type, &data->type) == NULL) + return false; + } + + if (offset == 0) { + /* Update offset relative to the start of the variable */ + data->offset = 0; + return true; + } + + if (dwarf_aggregate_size(&data->type, &size) < 0) + return false; + + if ((u64)offset >= size) + return false; + + /* Update offset relative to the start of the variable */ + data->offset = offset; + return true; +} + +/** + * is_breg_access_indirect - Check if breg based access implies type + * dereference + * @ops: DWARF operations array + * @nops: Number of operations in @ops + * + * Returns true if the DWARF expression evaluates to the variable's + * value, so the memory access on that register needs type dereference. + * Returns false if the expression evaluates to the variable's address. + * This is called after check_allowed_ops. + */ +static bool is_breg_access_indirect(Dwarf_Op *ops, size_t nops) +{ + /* only the base register */ + if (nops == 1) + return false; + + if (nops == 2 && ops[1].atom == DW_OP_stack_value) + return true; + + if (nops == 3 && (ops[1].atom == DW_OP_deref || + ops[1].atom == DW_OP_deref_size) && + ops[2].atom == DW_OP_stack_value) + return false; + /* unreachable, OP not supported */ + return false; +} + +/* Only checks direct child DIEs in the given scope. */ +static int __die_find_var_reg_cb(Dwarf_Die *die_mem, void *arg) +{ + struct find_var_data *data = arg; + int tag = dwarf_tag(die_mem); + ptrdiff_t off = 0; + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + + if (tag != DW_TAG_variable && tag != DW_TAG_formal_parameter) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + /* Assuming the location list is sorted by address */ + if (end <= data->pc) + continue; + if (start > data->pc) + break; + + /* Local variables accessed using frame base register */ + if (data->is_fbreg && ops->atom == DW_OP_fbreg && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, ops->number, + is_breg_access_indirect(ops, nops))) + return DIE_FIND_CB_END; + + /* Only match with a simple case */ + if (data->reg < DWARF_OP_DIRECT_REGS) { + /* pointer variables saved in a register 0 to 31 */ + if (ops->atom == (DW_OP_reg0 + data->reg) && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, 0, + /*is_pointer=*/true)) + return DIE_FIND_CB_END; + + /* variables accessed by a register + offset */ + if (ops->atom == (DW_OP_breg0 + data->reg) && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, ops->number, + is_breg_access_indirect(ops, nops))) + return DIE_FIND_CB_END; + } else { + /* pointer variables saved in a register 32 or above */ + if (ops->atom == DW_OP_regx && ops->number == data->reg && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, 0, + /*is_pointer=*/true)) + return DIE_FIND_CB_END; + + /* variables accessed by a register + offset */ + if (ops->atom == DW_OP_bregx && data->reg == ops->number && + check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->offset, ops->number2, + is_breg_access_indirect(ops, nops))) + return DIE_FIND_CB_END; + } + } + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_variable_by_reg - Find a variable saved in a register + * @sc_die: a scope DIE + * @pc: the program address to find + * @reg: the register number to find + * @poffset: pointer to offset, will be updated for fbreg case + * @is_fbreg: boolean value if the current register is the frame base + * @die_mem: a buffer to save the resulting DIE + * + * Find the variable DIE accessed by the given register. It'll update the @offset + * when the variable is in the stack. + */ +Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die, Dwarf_Addr pc, int reg, + Dwarf_Die *type_die, int *poffset, bool is_fbreg, + Dwarf_Die *die_mem) +{ + struct find_var_data data = { + .pc = pc, + .reg = reg, + .offset = *poffset, + .is_fbreg = is_fbreg, + }; + Dwarf_Die *result; + + result = die_find_child(sc_die, __die_find_var_reg_cb, &data, die_mem); + if (result) { + *poffset = data.offset; + *type_die = data.type; + } + return result; +} + +/* Only checks direct child DIEs in the given scope */ +static int __die_find_var_addr_cb(Dwarf_Die *die_mem, void *arg) +{ + struct find_var_data *data = arg; + int tag = dwarf_tag(die_mem); + ptrdiff_t off = 0; + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + + if (tag != DW_TAG_variable) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + if (ops->atom != DW_OP_addr) + continue; + + if (check_allowed_ops(ops, nops) && + match_var_offset(die_mem, data, data->addr, ops->number, + /*is_pointer=*/false)) + return DIE_FIND_CB_END; + } + return DIE_FIND_CB_SIBLING; +} + +/** + * die_find_variable_by_addr - Find variable located at given address + * @sc_die: a scope DIE + * @addr: the data address to find + * @die_mem: a buffer to save the resulting DIE + * @offset: the offset in the resulting type + * + * Find the variable DIE located at the given address (in PC-relative mode). + * This is usually for global variables. + */ +Dwarf_Die *die_find_variable_by_addr(Dwarf_Die *sc_die, Dwarf_Addr addr, + Dwarf_Die *die_mem, Dwarf_Die *type_die, + int *offset) +{ + struct find_var_data data = { + .addr = addr, + }; + Dwarf_Die *result; + + result = die_find_child(sc_die, __die_find_var_addr_cb, &data, die_mem); + if (result) { + *offset = data.offset; + *type_die = data.type; + } + return result; +} + +static int __die_collect_vars_cb(Dwarf_Die *die_mem, void *arg) +{ + struct die_var_type **var_types = arg; + Dwarf_Die type_die; + int tag = dwarf_tag(die_mem); + Dwarf_Attribute attr; + Dwarf_Addr base, start, end = 0; + Dwarf_Op *ops; + size_t nops; + struct die_var_type *vt; + ptrdiff_t off; + + if (tag != DW_TAG_variable && tag != DW_TAG_formal_parameter) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + if (__die_get_real_type(die_mem, &type_die) == NULL) + return DIE_FIND_CB_SIBLING; + + /* + * Collect all location entries as variables may have different + * locations across different address ranges. + */ + off = 0; + while ((off = dwarf_getlocations(&attr, off, &base, &start, &end, &ops, &nops)) > 0) { + if (!check_allowed_ops(ops, nops)) + continue; + + vt = malloc(sizeof(*vt)); + if (vt == NULL) + return DIE_FIND_CB_END; + + /* Usually a register holds the value of a variable */ + vt->is_reg_var_addr = false; + + if (((ops->atom >= DW_OP_breg0 && ops->atom <= DW_OP_breg31) || + ops->atom == DW_OP_bregx || ops->atom == DW_OP_fbreg) && + !is_breg_access_indirect(ops, nops)) + /* The register contains an address of the variable. */ + vt->is_reg_var_addr = true; + + vt->die_off = dwarf_dieoffset(&type_die); + vt->addr = start; + vt->end = end; + vt->has_range = (end != 0 || start != 0); + vt->reg = reg_from_dwarf_op(ops); + vt->offset = offset_from_dwarf_op(ops); + vt->next = *var_types; + *var_types = vt; + } + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_collect_vars - Save all variables and parameters + * @sc_die: a scope DIE + * @var_types: a pointer to save the resulting list + * + * Save all variables and parameters in the @sc_die and save them to @var_types. + * The @var_types is a singly-linked list containing type and location info. + * Actual type can be retrieved using dwarf_offdie() with 'die_off' later. + * + * Callers should free @var_types. + */ +void die_collect_vars(Dwarf_Die *sc_die, struct die_var_type **var_types) +{ + Dwarf_Die die_mem; + + die_find_child(sc_die, __die_collect_vars_cb, (void *)var_types, &die_mem); +} + +static int __die_collect_global_vars_cb(Dwarf_Die *die_mem, void *arg) +{ + struct die_var_type **var_types = arg; + Dwarf_Die type_die; + int tag = dwarf_tag(die_mem); + Dwarf_Attribute attr; + Dwarf_Addr base, start, end; + Dwarf_Op *ops; + size_t nops; + struct die_var_type *vt; + + if (tag != DW_TAG_variable) + return DIE_FIND_CB_SIBLING; + + if (dwarf_attr(die_mem, DW_AT_location, &attr) == NULL) + return DIE_FIND_CB_SIBLING; + + /* Only collect the location with an absolute address. */ + if (dwarf_getlocations(&attr, 0, &base, &start, &end, &ops, &nops) <= 0) + return DIE_FIND_CB_SIBLING; + + if (ops->atom != DW_OP_addr) + return DIE_FIND_CB_SIBLING; + + if (!check_allowed_ops(ops, nops)) + return DIE_FIND_CB_SIBLING; + + if (die_get_real_type(die_mem, &type_die) == NULL) + return DIE_FIND_CB_SIBLING; + + vt = malloc(sizeof(*vt)); + if (vt == NULL) + return DIE_FIND_CB_END; + + vt->die_off = dwarf_dieoffset(&type_die); + vt->addr = ops->number; + vt->end = 0; + vt->has_range = false; + vt->reg = -1; + vt->offset = 0; + vt->next = *var_types; + *var_types = vt; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_collect_global_vars - Save all global variables + * @cu_die: a CU DIE + * @var_types: a pointer to save the resulting list + * + * Save all global variables in the @cu_die and save them to @var_types. + * The @var_types is a singly-linked list containing type and location info. + * Actual type can be retrieved using dwarf_offdie() with 'die_off' later. + * + * Callers should free @var_types. + */ +void die_collect_global_vars(Dwarf_Die *cu_die, struct die_var_type **var_types) +{ + Dwarf_Die die_mem; + + die_find_child(cu_die, __die_collect_global_vars_cb, (void *)var_types, &die_mem); +} + +/** + * die_get_cfa - Get frame base information + * @dwarf: a Dwarf info + * @pc: program address + * @preg: pointer for saved register + * @poffset: pointer for saved offset + * + * This function gets register and offset for CFA (Canonical Frame Address) + * by searching the CIE/FDE info. The CFA usually points to the start address + * of the current stack frame and local variables can be located using an offset + * from the CFA. The @preg and @poffset will be updated if it returns 0. + */ +int die_get_cfa(Dwarf *dwarf, u64 pc, int *preg, int *poffset) +{ + Dwarf_CFI *cfi; + Dwarf_Frame *frame = NULL; + Dwarf_Op *ops = NULL; + size_t nops; + + cfi = dwarf_getcfi(dwarf); + if (cfi == NULL) + return -1; + + if (!dwarf_cfi_addrframe(cfi, pc, &frame) && + !dwarf_frame_cfa(frame, &ops, &nops) && + check_allowed_ops(ops, nops)) { + *preg = reg_from_dwarf_op(ops); + *poffset = offset_from_dwarf_op(ops); + return 0; + } + return -1; +} + +/* + * die_has_loclist - Check if DW_AT_location of @vr_die is a location list + * @vr_die: a variable DIE + */ +static bool die_has_loclist(Dwarf_Die *vr_die) +{ + Dwarf_Attribute loc; + int tag = dwarf_tag(vr_die); + + if (tag != DW_TAG_formal_parameter && + tag != DW_TAG_variable) + return false; + + return (dwarf_attr_integrate(vr_die, DW_AT_location, &loc) && + dwarf_whatform(&loc) == DW_FORM_sec_offset); +} + +/* + * die_is_optimized_target - Check if target program is compiled with + * optimization + * @cu_die: a CU DIE + * + * For any object in given CU whose DW_AT_location is a location list, + * target program is compiled with optimization. This is applicable to + * clang as well. + */ +bool die_is_optimized_target(Dwarf_Die *cu_die) +{ + Dwarf_Die tmp_die; + + if (die_has_loclist(cu_die)) + return true; + + if (!dwarf_child(cu_die, &tmp_die) && + die_is_optimized_target(&tmp_die)) + return true; + + if (!dwarf_siblingof(cu_die, &tmp_die) && + die_is_optimized_target(&tmp_die)) + return true; + + return false; +} + +/* + * die_search_idx - Search index of given line address + * @lines: Line records of single CU + * @nr_lines: Number of @lines + * @addr: address we are looking for + * @idx: index to be set by this function (return value) + * + * Search for @addr by looping over every lines of CU. If address + * matches, set index of that line in @idx. Note that single source + * line can have multiple line records. i.e. single source line can + * have multiple index. + */ +static bool die_search_idx(Dwarf_Lines *lines, unsigned long nr_lines, + Dwarf_Addr addr, unsigned long *idx) +{ + unsigned long i; + Dwarf_Addr tmp; + + for (i = 0; i < nr_lines; i++) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i), &tmp)) + return false; + + if (tmp == addr) { + *idx = i; + return true; + } + } + return false; +} + +/* + * die_get_postprologue_addr - Search next address after function prologue + * @entrypc_idx: entrypc index + * @lines: Line records of single CU + * @nr_lines: Number of @lines + * @hignpc: high PC address of function + * @postprologue_addr: Next address after function prologue (return value) + * + * Look for prologue-end marker. If there is no explicit marker, return + * address of next line record or next source line. + */ +static bool die_get_postprologue_addr(unsigned long entrypc_idx, + Dwarf_Lines *lines, + unsigned long nr_lines, + Dwarf_Addr highpc, + Dwarf_Addr *postprologue_addr) +{ + unsigned long i; + int entrypc_lno, lno; + Dwarf_Line *line; + Dwarf_Addr addr; + bool p_end; + + /* entrypc_lno is actual source line number */ + line = dwarf_onesrcline(lines, entrypc_idx); + if (dwarf_lineno(line, &entrypc_lno)) + return false; + + for (i = entrypc_idx; i < nr_lines; i++) { + line = dwarf_onesrcline(lines, i); + + if (dwarf_lineaddr(line, &addr) || + dwarf_lineno(line, &lno) || + dwarf_lineprologueend(line, &p_end)) + return false; + + /* highpc is exclusive. [entrypc,highpc) */ + if (addr >= highpc) + break; + + /* clang supports prologue-end marker */ + if (p_end) + break; + + /* Actual next line in source */ + if (lno != entrypc_lno) + break; + + /* + * Single source line can have multiple line records. + * For Example, + * void foo() { printf("hello\n"); } + * contains two line records. One points to declaration and + * other points to printf() line. Variable 'lno' won't get + * incremented in this case but 'i' will. + */ + if (i != entrypc_idx) + break; + } + + if (dwarf_lineaddr(line, postprologue_addr) != 0) + return false; + if (*postprologue_addr >= highpc) { + if (dwarf_lineaddr(dwarf_onesrcline(lines, i - 1), postprologue_addr) != 0) + return false; + } + + return true; +} + +/* + * die_skip_prologue - Use next address after prologue as probe location + * @sp_die: a subprogram DIE + * @cu_die: a CU DIE + * @entrypc: entrypc of the function + * + * Function prologue prepares stack and registers before executing function + * logic. When target program is compiled without optimization, function + * parameter information is only valid after prologue. When we probe entrypc + * of the function, and try to record function parameter, it contains + * garbage value. + */ +void die_skip_prologue(Dwarf_Die *sp_die, Dwarf_Die *cu_die, + Dwarf_Addr *entrypc) +{ + size_t nr_lines = 0; + unsigned long entrypc_idx = 0; + Dwarf_Lines *lines = NULL; + Dwarf_Addr postprologue_addr; + Dwarf_Addr highpc; + + if (dwarf_highpc(sp_die, &highpc)) + return; + + if (dwarf_getsrclines(cu_die, &lines, &nr_lines)) + return; + + if (!die_search_idx(lines, nr_lines, *entrypc, &entrypc_idx)) + return; + + if (!die_get_postprologue_addr(entrypc_idx, lines, nr_lines, + highpc, &postprologue_addr)) + return; + + *entrypc = postprologue_addr; +} + +/* Internal parameters for __die_find_scope_cb() */ +struct find_scope_data { + /* Target instruction address */ + Dwarf_Addr pc; + /* Number of scopes found [output] */ + int nr; + /* Array of scopes found, 0 for the outermost one. [output] */ + Dwarf_Die *scopes; +}; + +static int __die_find_scope_cb(Dwarf_Die *die_mem, void *arg) +{ + struct find_scope_data *data = arg; + int tag = dwarf_tag(die_mem); + + if (dwarf_haspc(die_mem, data->pc)) { + Dwarf_Die *tmp; + + tmp = realloc(data->scopes, (data->nr + 1) * sizeof(*tmp)); + if (tmp == NULL) + return DIE_FIND_CB_END; + + memcpy(tmp + data->nr, die_mem, sizeof(*die_mem)); + data->scopes = tmp; + data->nr++; + return DIE_FIND_CB_CHILD; + } + + /* + * If the DIE doesn't have the PC, we still need to check its children + * and siblings if it's a container like a namespace. + */ + if (tag == DW_TAG_namespace) + return DIE_FIND_CB_CONTINUE; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_get_scopes - Return a list of scopes including the address + * @cu_die: a compile unit DIE + * @pc: the address to find + * @scopes: the array of DIEs for scopes (result) + * + * This function does the same as the dwarf_getscopes() but doesn't follow + * the origins of inlined functions. It returns the number of scopes saved + * in the @scopes argument. The outer scope will be saved first (index 0) and + * the last one is the innermost scope at the @pc. + */ +int die_get_scopes(Dwarf_Die *cu_die, Dwarf_Addr pc, Dwarf_Die **scopes) +{ + struct find_scope_data data = { + .pc = pc, + }; + Dwarf_Die die_mem; + + die_find_child(cu_die, __die_find_scope_cb, &data, &die_mem); + + *scopes = data.scopes; + return data.nr; +} + +static int __die_find_member_offset_cb(Dwarf_Die *die_mem, void *arg) +{ + Dwarf_Die type_die; + Dwarf_Word size, loc; + Dwarf_Word offset = (long)arg; + int tag = dwarf_tag(die_mem); + + if (tag != DW_TAG_member) + return DIE_FIND_CB_SIBLING; + + /* Unions might not have location */ + if (die_get_data_member_location(die_mem, &loc) < 0) { + Dwarf_Attribute attr; + + if (dwarf_attr_integrate(die_mem, DW_AT_data_bit_offset, &attr) && + dwarf_formudata(&attr, &loc) == 0) + loc /= 8; + else + loc = 0; + } + + if (offset == loc) + return DIE_FIND_CB_END; + + if (die_get_real_type(die_mem, &type_die) == NULL) { + // TODO: add a pr_debug_dtp() later for this unlikely failure + return DIE_FIND_CB_SIBLING; + } + + if (dwarf_aggregate_size(&type_die, &size) < 0) + size = 0; + + if (loc < offset && offset < (loc + size)) + return DIE_FIND_CB_END; + + return DIE_FIND_CB_SIBLING; +} + +/** + * die_get_member_type - Return type info of struct member + * @type_die: a type DIE + * @offset: offset in the type + * @die_mem: a buffer to save the resulting DIE + * + * This function returns a type of a member in @type_die where it's located at + * @offset if it's a struct. For now, it just returns the first matching + * member in a union. For other types, it'd return the given type directly + * if it's within the size of the type or NULL otherwise. + */ +Dwarf_Die *die_get_member_type(Dwarf_Die *type_die, int offset, + Dwarf_Die *die_mem) +{ + Dwarf_Die *member; + Dwarf_Die mb_type; + int tag; + + tag = dwarf_tag(type_die); + /* If it's not a compound type, return the type directly */ + if (tag != DW_TAG_structure_type && tag != DW_TAG_union_type) { + Dwarf_Word size; + + if (dwarf_aggregate_size(type_die, &size) < 0) + size = 0; + + if ((unsigned)offset >= size) + return NULL; + + *die_mem = *type_die; + return die_mem; + } + + mb_type = *type_die; + /* TODO: Handle union types better? */ + while (tag == DW_TAG_structure_type || tag == DW_TAG_union_type) { + member = die_find_child(&mb_type, __die_find_member_offset_cb, + (void *)(long)offset, die_mem); + if (member == NULL) + return NULL; + + if (die_get_real_type(member, &mb_type) == NULL) + return NULL; + + tag = dwarf_tag(&mb_type); + + if (tag == DW_TAG_structure_type || tag == DW_TAG_union_type || + tag == DW_TAG_array_type) { + Dwarf_Word loc; + + /* Update offset for the start of the member struct */ + if (die_get_data_member_location(member, &loc) == 0) + offset -= loc; + } + + /* Handle array types: resolve to the element type by one level */ + if (tag == DW_TAG_array_type) { + Dwarf_Word size; + + if (die_get_real_type(&mb_type, &mb_type) == NULL) + return NULL; + + if (dwarf_aggregate_size(&mb_type, &size) < 0) + return NULL; + + offset = offset % size; + tag = dwarf_tag(&mb_type); + } + } + *die_mem = mb_type; + return die_mem; +} + +/** + * die_deref_ptr_type - Return type info for pointer access + * @ptr_die: a pointer type DIE + * @offset: access offset for the pointer + * @die_mem: a buffer to save the resulting DIE + * + * This function follows the pointer in @ptr_die with given @offset + * and saves the resulting type in @die_mem. If the pointer points + * a struct type, actual member at the offset would be returned. + */ +Dwarf_Die *die_deref_ptr_type(Dwarf_Die *ptr_die, int offset, + Dwarf_Die *die_mem) +{ + Dwarf_Die type_die; + + if (dwarf_tag(ptr_die) != DW_TAG_pointer_type) + return NULL; + + if (die_get_real_type(ptr_die, &type_die) == NULL) + return NULL; + + return die_get_member_type(&type_die, offset, die_mem); +} diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h new file mode 100644 index 000000000..161f0bf98 --- /dev/null +++ b/tools/perf/util/dwarf-aux.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: GPL-2.0-or-later */ +#ifndef _DWARF_AUX_H +#define _DWARF_AUX_H +/* + * dwarf-aux.h : libdw auxiliary interfaces + */ + +#include +#include +#include +#include + +static inline const char *die_name(Dwarf_Die *die) +{ + return dwarf_diename(die) ?: ""; +} + +struct strbuf; + +/* Find the realpath of the target file */ +const char *cu_find_realpath(Dwarf_Die *cu_die, const char *fname); + +/* Get DW_AT_comp_dir (should be NULL with older gcc) */ +const char *cu_get_comp_dir(Dwarf_Die *cu_die); + +/* Get a line number and file name for given address */ +int cu_find_lineinfo(Dwarf_Die *cudie, Dwarf_Addr addr, + const char **fname, int *lineno); + +/* Walk on functions at given address */ +int cu_walk_functions_at(Dwarf_Die *cu_die, Dwarf_Addr addr, + int (*callback)(Dwarf_Die *, void *), void *data); + +/* Get DW_AT_linkage_name (should be NULL for C binary) */ +const char *die_get_linkage_name(Dwarf_Die *dw_die); + +/* Get the lowest PC in DIE (including range list) */ +int die_entrypc(Dwarf_Die *dw_die, Dwarf_Addr *addr); + +/* Ensure that this DIE is a subprogram and definition (not declaration) */ +bool die_is_func_def(Dwarf_Die *dw_die); + +/* Ensure that this DIE is an instance of a subprogram */ +bool die_is_func_instance(Dwarf_Die *dw_die); + +/* Compare diename and tname */ +bool die_compare_name(Dwarf_Die *dw_die, const char *tname); + +/* Matching diename with glob pattern */ +bool die_match_name(Dwarf_Die *dw_die, const char *glob); + +/* Get callsite line number of inline-function instance */ +int die_get_call_lineno(Dwarf_Die *in_die); + +/* Get callsite file name of inlined function instance */ +const char *die_get_call_file(Dwarf_Die *in_die); + +/* Get declared file name of a DIE */ +const char *die_get_decl_file(Dwarf_Die *dw_die); + +/* Get type die */ +Dwarf_Die *die_get_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem); + +/* Get a type die, but skip qualifiers */ +Dwarf_Die *__die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem); +/* Get a type die, but skip qualifiers and typedef */ +Dwarf_Die *die_get_real_type(Dwarf_Die *vr_die, Dwarf_Die *die_mem); +/* Get a pointer/array type, following typedefs/qualifiers */ +Dwarf_Die *die_get_pointer_type(Dwarf_Die *type_die, Dwarf_Die *die_mem); + +/* Check whether the DIE is signed or not */ +bool die_is_signed_type(Dwarf_Die *tp_die); + +/* Get data_member_location offset */ +int die_get_data_member_location(Dwarf_Die *mb_die, Dwarf_Word *offs); + +/* Return values for die_find_child() callbacks */ +enum { + DIE_FIND_CB_END = 0, /* End of Search */ + DIE_FIND_CB_CHILD = 1, /* Search only children */ + DIE_FIND_CB_SIBLING = 2, /* Search only siblings */ + DIE_FIND_CB_CONTINUE = 3, /* Search children and siblings */ +}; + +/* Search child DIEs */ +Dwarf_Die *die_find_child(Dwarf_Die *rt_die, + int (*callback)(Dwarf_Die *, void *), + void *data, Dwarf_Die *die_mem); + +/* Search a non-inlined function including given address */ +Dwarf_Die *die_find_realfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search a non-inlined function with tail call at given address */ +Dwarf_Die *die_find_tailfunc(Dwarf_Die *cu_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search the top inlined function including given address */ +Dwarf_Die *die_find_top_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search the deepest inlined function including given address */ +Dwarf_Die *die_find_inlinefunc(Dwarf_Die *sp_die, Dwarf_Addr addr, + Dwarf_Die *die_mem); + +/* Search a non-inlined function by name and returns its return type */ +Dwarf_Die *die_find_func_rettype(Dwarf_Die *sp_die, const char *name, + Dwarf_Die *die_mem); + +/* Walk on the instances of given DIE */ +int die_walk_instances(Dwarf_Die *in_die, + int (*callback)(Dwarf_Die *, void *), void *data); + +/* Walker on lines (Note: line number will not be sorted) */ +typedef int (* line_walk_callback_t) (const char *fname, int lineno, + Dwarf_Addr addr, void *data); + +/* + * Walk on lines inside given DIE. If the DIE is a subprogram, walk only on + * the lines inside the subprogram, otherwise the DIE must be a CU DIE. + */ +int die_walk_lines(Dwarf_Die *rt_die, line_walk_callback_t callback, void *data); + +/* Find a variable called 'name' at given address */ +Dwarf_Die *die_find_variable_at(Dwarf_Die *sp_die, const char *name, + Dwarf_Addr addr, Dwarf_Die *die_mem); + +/* Find a member called 'name' */ +Dwarf_Die *die_find_member(Dwarf_Die *st_die, const char *name, + Dwarf_Die *die_mem); + +/* Get the name of given type DIE */ +int die_get_typename_from_type(Dwarf_Die *type_die, struct strbuf *buf); + +/* Get the name of given variable DIE */ +int die_get_typename(Dwarf_Die *vr_die, struct strbuf *buf); + +/* Get the name and type of given variable DIE, stored as "type\tname" */ +int die_get_varname(Dwarf_Die *vr_die, struct strbuf *buf); + +/* Check if target program is compiled with optimization */ +bool die_is_optimized_target(Dwarf_Die *cu_die); + +/* Use next address after prologue as probe location */ +void die_skip_prologue(Dwarf_Die *sp_die, Dwarf_Die *cu_die, + Dwarf_Addr *entrypc); + +/* Get the list of including scopes */ +int die_get_scopes(Dwarf_Die *cu_die, Dwarf_Addr pc, Dwarf_Die **scopes); + +/* Variable type information */ +struct die_var_type { + struct die_var_type *next; + u64 die_off; + u64 addr; + u64 end; /* end address of location range */ + int reg; + int offset; + /* Whether the register holds a address to the type */ + bool is_reg_var_addr; + bool has_range; /* whether end is valid */ +}; + +/* Return type info of a member at offset */ +Dwarf_Die *die_get_member_type(Dwarf_Die *type_die, int offset, Dwarf_Die *die_mem); + +/* Return type info where the pointer and offset point to */ +Dwarf_Die *die_deref_ptr_type(Dwarf_Die *ptr_die, int offset, Dwarf_Die *die_mem); + +/* Get byte offset range of given variable DIE */ +int die_get_var_range(Dwarf_Die *sp_die, Dwarf_Die *vr_die, struct strbuf *buf); + +/* Find a variable saved in the 'reg' at given address */ +Dwarf_Die *die_find_variable_by_reg(Dwarf_Die *sc_die, Dwarf_Addr pc, int reg, + Dwarf_Die *type_die, int *poffset, bool is_fbreg, + Dwarf_Die *die_mem); + +/* Find a (global) variable located in the 'addr' */ +Dwarf_Die *die_find_variable_by_addr(Dwarf_Die *sc_die, Dwarf_Addr addr, + Dwarf_Die *die_mem, Dwarf_Die *type_die, + int *offset); + +/* Save all variables and parameters in this scope */ +void die_collect_vars(Dwarf_Die *sc_die, struct die_var_type **var_types); + +/* Save all global variables in this CU */ +void die_collect_global_vars(Dwarf_Die *cu_die, struct die_var_type **var_types); + +/* Get the frame base information from CFA */ +int die_get_cfa(Dwarf *dwarf, u64 pc, int *preg, int *poffset); + +#endif /* _DWARF_AUX_H */ diff --git a/tools/perf/util/dwarf-regs-arch/Build b/tools/perf/util/dwarf-regs-arch/Build new file mode 100644 index 000000000..ceb68ae86 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/Build @@ -0,0 +1,9 @@ +perf-util-$(CONFIG_LIBDW) += dwarf-regs-arm64.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-arm.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-csky.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-loongarch.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-mips.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-powerpc.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-riscv.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-s390.o +perf-util-$(CONFIG_LIBDW) += dwarf-regs-x86.o diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c new file mode 100644 index 000000000..42c6c0635 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/arm/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_arm(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_ARM_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c new file mode 100644 index 000000000..593ca7d4f --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-arm64.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/arm64/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_arm64(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_ARM64_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c new file mode 100644 index 000000000..cb44b774f --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-csky.c @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: GPL-2.0 +// Copyright (C) 2019 Hangzhou C-SKY Microsystems co.,ltd. +// Mapping of DWARF debug register numbers into register names. + +#include +#include +#include +// Ensure the V2 perf reg definitions are included. +#undef __CSKYABIV2__ +#define __CSKYABIV2__ 1 +#include "../../../arch/csky/include/uapi/asm/perf_regs.h" + +#define CSKY_ABIV2_MAX_REGS 73 +static const char * const csky_dwarf_regs_table_abiv2[CSKY_ABIV2_MAX_REGS] = { + /* r0 ~ r8 */ + "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", "%regs2", "%regs3", + /* r9 ~ r15 */ + "%regs4", "%regs5", "%regs6", "%regs7", "%regs8", "%regs9", "%sp", + "%lr", + /* r16 ~ r23 */ + "%exregs0", "%exregs1", "%exregs2", "%exregs3", "%exregs4", + "%exregs5", "%exregs6", "%exregs7", + /* r24 ~ r31 */ + "%exregs8", "%exregs9", "%exregs10", "%exregs11", "%exregs12", + "%exregs13", "%exregs14", "%tls", + "%pc", NULL, NULL, NULL, "%hi", "%lo", NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + "%epc", +}; + +#define CSKY_ABIV1_MAX_REGS 57 +static const char * const csky_dwarf_regs_table_abiv1[CSKY_ABIV1_MAX_REGS] = { + /* r0 ~ r8 */ + "%sp", "%regs9", "%a0", "%a1", "%a2", "%a3", "%regs0", "%regs1", + /* r9 ~ r15 */ + "%regs2", "%regs3", "%regs4", "%regs5", "%regs6", "%regs7", "%regs8", + "%lr", + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, + "%epc", +}; + +const char *__get_csky_regstr(unsigned int n, unsigned int flags) +{ + if (flags & EF_CSKY_ABIV2) + return (n < CSKY_ABIV2_MAX_REGS) ? csky_dwarf_regs_table_abiv2[n] : NULL; + + return (n < CSKY_ABIV1_MAX_REGS) ? csky_dwarf_regs_table_abiv1[n] : NULL; +} + +static int __get_dwarf_regnum(const char *const *regstr, size_t num_regstr, const char *name) +{ + for (size_t i = 0; i < num_regstr; i++) { + if (regstr[i] && !strcmp(regstr[i], name)) + return i; + } + return -ENOENT; +} + +int __get_csky_regnum(const char *name, unsigned int flags) +{ + if (flags & EF_CSKY_ABIV2) + return __get_dwarf_regnum(csky_dwarf_regs_table_abiv2, CSKY_ABIV2_MAX_REGS, name); + + return __get_dwarf_regnum(csky_dwarf_regs_table_abiv1, CSKY_ABIV1_MAX_REGS, name); +} + +int __get_dwarf_regnum_for_perf_regnum_csky(int perf_regnum, unsigned int flags) +{ + static const int dwarf_csky_regnums[][2] = { + [PERF_REG_CSKY_TLS] = {-ENOENT, 31}, + [PERF_REG_CSKY_LR] = {15, 15}, + [PERF_REG_CSKY_PC] = {-ENOENT, 32}, + /* TODO: PERF_REG_CSKY_SR */ + [PERF_REG_CSKY_SP] = {0, 14}, + /* TODO: PERF_REG_CSKY_ORIG_A0 */ + [PERF_REG_CSKY_A0] = {2, 0}, + [PERF_REG_CSKY_A1] = {3, 1}, + [PERF_REG_CSKY_A2] = {4, 2}, + [PERF_REG_CSKY_A3] = {5, 3}, + [PERF_REG_CSKY_REGS0] = {6, 4}, + [PERF_REG_CSKY_REGS1] = {7, 5}, + [PERF_REG_CSKY_REGS2] = {8, 6}, + [PERF_REG_CSKY_REGS3] = {9, 7}, + [PERF_REG_CSKY_REGS4] = {10, 8}, + [PERF_REG_CSKY_REGS5] = {11, 9}, + [PERF_REG_CSKY_REGS6] = {12, 10}, + [PERF_REG_CSKY_REGS7] = {13, 11}, + [PERF_REG_CSKY_REGS8] = {14, 12}, + [PERF_REG_CSKY_REGS9] = {1, 13}, + [PERF_REG_CSKY_EXREGS0] = {-ENOENT, 16}, + [PERF_REG_CSKY_EXREGS1] = {-ENOENT, 17}, + [PERF_REG_CSKY_EXREGS2] = {-ENOENT, 18}, + [PERF_REG_CSKY_EXREGS3] = {-ENOENT, 19}, + [PERF_REG_CSKY_EXREGS4] = {-ENOENT, 20}, + [PERF_REG_CSKY_EXREGS5] = {-ENOENT, 21}, + [PERF_REG_CSKY_EXREGS6] = {-ENOENT, 22}, + [PERF_REG_CSKY_EXREGS7] = {-ENOENT, 23}, + [PERF_REG_CSKY_EXREGS8] = {-ENOENT, 24}, + [PERF_REG_CSKY_EXREGS9] = {-ENOENT, 25}, + [PERF_REG_CSKY_EXREGS10] = {-ENOENT, 26}, + [PERF_REG_CSKY_EXREGS11] = {-ENOENT, 27}, + [PERF_REG_CSKY_EXREGS12] = {-ENOENT, 28}, + [PERF_REG_CSKY_EXREGS13] = {-ENOENT, 29}, + [PERF_REG_CSKY_EXREGS14] = {-ENOENT, 30}, + /* TODO: PERF_REG_CSKY_HI */ + /* TODO: PERF_REG_CSKY_LO */ + /* TODO: PERF_REG_CSKY_DCSR */ + }; + int idx = 0; + + if (flags & EF_CSKY_ABIV2) + idx++; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_csky_regnums) || + dwarf_csky_regnums[perf_regnum][idx] == 0) + return -ENOENT; + + return dwarf_csky_regnums[perf_regnum][idx]; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c new file mode 100644 index 000000000..203077b74 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-loongarch.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/loongarch/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_loongarch(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_LOONGARCH_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c new file mode 100644 index 000000000..3bb916b45 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-mips.c @@ -0,0 +1,14 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/mips/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_mips(int perf_regnum) +{ + if (perf_regnum == PERF_REG_MIPS_PC) + return 37; + if (perf_regnum < 0 || perf_regnum >= PERF_REG_MIPS_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c new file mode 100644 index 000000000..51892a097 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-powerpc.c @@ -0,0 +1,136 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Mapping of DWARF debug register numbers into register names. + * + * Copyright (C) 2010 Ian Munsie, IBM Corporation. + */ +#include +#include +#include "../../../arch/powerpc/include/uapi/asm/perf_regs.h" + +#define PPC_OP(op) (((op) >> 26) & 0x3F) +#define PPC_RA(a) (((a) >> 16) & 0x1f) +#define PPC_RT(t) (((t) >> 21) & 0x1f) +#define PPC_RB(b) (((b) >> 11) & 0x1f) +#define PPC_D(D) ((D) & 0xfffe) +#define PPC_DS(DS) ((DS) & 0xfffc) +#define OP_LD 58 +#define OP_STD 62 + +static int get_source_reg(u32 raw_insn) +{ + return PPC_RA(raw_insn); +} + +static int get_target_reg(u32 raw_insn) +{ + return PPC_RT(raw_insn); +} + +static int get_offset_opcode(u32 raw_insn) +{ + int opcode = PPC_OP(raw_insn); + + /* DS- form */ + if ((opcode == OP_LD) || (opcode == OP_STD)) + return PPC_DS(raw_insn); + else + return PPC_D(raw_insn); +} + +/* + * Fills the required fields for op_loc depending on if it + * is a source or target. + * D form: ins RT,D(RA) -> src_reg1 = RA, offset = D, dst_reg1 = RT + * DS form: ins RT,DS(RA) -> src_reg1 = RA, offset = DS, dst_reg1 = RT + * X form: ins RT,RA,RB -> src_reg1 = RA, src_reg2 = RB, dst_reg1 = RT + */ +void get_powerpc_regs(u32 raw_insn, int is_source, + struct annotated_op_loc *op_loc) +{ + if (is_source) + op_loc->reg1 = get_source_reg(raw_insn); + else + op_loc->reg1 = get_target_reg(raw_insn); + + if (op_loc->multi_regs) + op_loc->reg2 = PPC_RB(raw_insn); + + /* TODO: Implement offset handling for X Form */ + if ((op_loc->mem_ref) && (PPC_OP(raw_insn) != 31)) + op_loc->offset = get_offset_opcode(raw_insn); +} + +int __get_dwarf_regnum_for_perf_regnum_powerpc(int perf_regnum) +{ + static const int dwarf_powerpc_regnums[] = { + [PERF_REG_POWERPC_R0] = 0, + [PERF_REG_POWERPC_R1] = 1, + [PERF_REG_POWERPC_R2] = 2, + [PERF_REG_POWERPC_R3] = 3, + [PERF_REG_POWERPC_R4] = 4, + [PERF_REG_POWERPC_R5] = 5, + [PERF_REG_POWERPC_R6] = 6, + [PERF_REG_POWERPC_R7] = 7, + [PERF_REG_POWERPC_R8] = 8, + [PERF_REG_POWERPC_R9] = 9, + [PERF_REG_POWERPC_R10] = 10, + [PERF_REG_POWERPC_R11] = 11, + [PERF_REG_POWERPC_R12] = 12, + [PERF_REG_POWERPC_R13] = 13, + [PERF_REG_POWERPC_R14] = 14, + [PERF_REG_POWERPC_R15] = 15, + [PERF_REG_POWERPC_R16] = 16, + [PERF_REG_POWERPC_R17] = 17, + [PERF_REG_POWERPC_R18] = 18, + [PERF_REG_POWERPC_R19] = 19, + [PERF_REG_POWERPC_R20] = 20, + [PERF_REG_POWERPC_R21] = 21, + [PERF_REG_POWERPC_R22] = 22, + [PERF_REG_POWERPC_R23] = 23, + [PERF_REG_POWERPC_R24] = 24, + [PERF_REG_POWERPC_R25] = 25, + [PERF_REG_POWERPC_R26] = 26, + [PERF_REG_POWERPC_R27] = 27, + [PERF_REG_POWERPC_R28] = 28, + [PERF_REG_POWERPC_R29] = 29, + [PERF_REG_POWERPC_R30] = 30, + [PERF_REG_POWERPC_R31] = 31, + /* TODO: PERF_REG_POWERPC_NIP */ + [PERF_REG_POWERPC_MSR] = 66, + /* TODO: PERF_REG_POWERPC_ORIG_R3 */ + [PERF_REG_POWERPC_CTR] = 109, + [PERF_REG_POWERPC_LINK] = 108, /* Note, previously in perf encoded as 65? */ + [PERF_REG_POWERPC_XER] = 101, + /* TODO: PERF_REG_POWERPC_CCR */ + /* TODO: PERF_REG_POWERPC_SOFTE */ + /* TODO: PERF_REG_POWERPC_TRAP */ + /* TODO: PERF_REG_POWERPC_DAR */ + /* TODO: PERF_REG_POWERPC_DSISR */ + /* TODO: PERF_REG_POWERPC_SIER */ + /* TODO: PERF_REG_POWERPC_MMCRA */ + /* TODO: PERF_REG_POWERPC_MMCR0 */ + /* TODO: PERF_REG_POWERPC_MMCR1 */ + /* TODO: PERF_REG_POWERPC_MMCR2 */ + /* TODO: PERF_REG_POWERPC_MMCR3 */ + /* TODO: PERF_REG_POWERPC_SIER2 */ + /* TODO: PERF_REG_POWERPC_SIER3 */ + /* TODO: PERF_REG_POWERPC_PMC1 */ + /* TODO: PERF_REG_POWERPC_PMC2 */ + /* TODO: PERF_REG_POWERPC_PMC3 */ + /* TODO: PERF_REG_POWERPC_PMC4 */ + /* TODO: PERF_REG_POWERPC_PMC5 */ + /* TODO: PERF_REG_POWERPC_PMC6 */ + /* TODO: PERF_REG_POWERPC_SDAR */ + /* TODO: PERF_REG_POWERPC_SIAR */ + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_powerpc_regnums) || + dwarf_powerpc_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_powerpc_regnums[perf_regnum]; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c new file mode 100644 index 000000000..090db51ab --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-riscv.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/riscv/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_riscv(int perf_regnum) +{ + if (perf_regnum < 0 || perf_regnum >= PERF_REG_RISCV_MAX) + return -ENOENT; + + return perf_regnum; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c new file mode 100644 index 000000000..310a37451 --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-s390.c @@ -0,0 +1,53 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include "../../../arch/s390/include/uapi/asm/perf_regs.h" + +int __get_dwarf_regnum_for_perf_regnum_s390(int perf_regnum) +{ + static const int dwarf_s390_regnums[] = { + [PERF_REG_S390_R0] = 0, + [PERF_REG_S390_R1] = 1, + [PERF_REG_S390_R2] = 2, + [PERF_REG_S390_R3] = 3, + [PERF_REG_S390_R4] = 4, + [PERF_REG_S390_R5] = 5, + [PERF_REG_S390_R6] = 6, + [PERF_REG_S390_R7] = 7, + [PERF_REG_S390_R8] = 8, + [PERF_REG_S390_R9] = 9, + [PERF_REG_S390_R10] = 10, + [PERF_REG_S390_R11] = 11, + [PERF_REG_S390_R12] = 12, + [PERF_REG_S390_R13] = 13, + [PERF_REG_S390_R14] = 14, + [PERF_REG_S390_R15] = 15, + [PERF_REG_S390_FP0] = 16, + [PERF_REG_S390_FP1] = 20, + [PERF_REG_S390_FP2] = 17, + [PERF_REG_S390_FP3] = 21, + [PERF_REG_S390_FP4] = 18, + [PERF_REG_S390_FP5] = 22, + [PERF_REG_S390_FP6] = 19, + [PERF_REG_S390_FP7] = 23, + [PERF_REG_S390_FP8] = 24, + [PERF_REG_S390_FP9] = 28, + [PERF_REG_S390_FP10] = 25, + [PERF_REG_S390_FP11] = 29, + [PERF_REG_S390_FP12] = 26, + [PERF_REG_S390_FP13] = 30, + [PERF_REG_S390_FP14] = 27, + [PERF_REG_S390_FP15] = 31, + [PERF_REG_S390_MASK] = 64, + [PERF_REG_S390_PC] = 65, + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_s390_regnums) || + dwarf_s390_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_s390_regnums[perf_regnum]; +} diff --git a/tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c b/tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c new file mode 100644 index 000000000..cadef120a --- /dev/null +++ b/tools/perf/util/dwarf-regs-arch/dwarf-regs-x86.c @@ -0,0 +1,260 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. + * Extracted from probe-finder.c + * + * Written by Masami Hiramatsu + */ + +#include /* for EINVAL */ +#include /* for strcmp */ +#include /* for ARRAY_SIZE */ +#include +#include "../../../arch/x86/include/uapi/asm/perf_regs.h" + +struct dwarf_regs_idx { + const char *name; + int dwarf_regnum; +}; + +static const struct dwarf_regs_idx i386_regidx_table[] = { + { "eax", 0 }, { "ax", 0 }, { "al", 0 }, + { "ecx", 1 }, { "cx", 1 }, { "cl", 1 }, + { "edx", 2 }, { "dx", 2 }, { "dl", 2 }, + { "ebx", 3 }, { "bx", 3 }, { "bl", 3 }, + { "esp", 4 }, { "sp", 4 }, { "$stack", 4}, + { "ebp", 5 }, { "bp", 5 }, + { "esi", 6 }, { "si", 6 }, + { "edi", 7 }, { "di", 7 }, + // 8 - Return Address RA + { "eflags", 9}, { "flags", 9}, + // 10 - reserved + { "st0", 11}, + { "st1", 12}, + { "st2", 13}, + { "st3", 14}, + { "st4", 15}, + { "st5", 16}, + { "st6", 17}, + { "st7", 18}, + // 19-20 - reserved + { "xmm0", 21}, + { "xmm1", 22}, + { "xmm2", 23}, + { "xmm3", 24}, + { "xmm4", 25}, + { "xmm5", 26}, + { "xmm6", 27}, + { "xmm7", 28}, + { "mm0", 29}, + { "mm1", 30}, + { "mm2", 31}, + { "mm3", 32}, + { "mm4", 33}, + { "mm5", 34}, + { "mm6", 35}, + { "mm7", 36}, + // 37-38 - unknown + { "mxcsr", 39}, // 128-bit Media Control and Status + { "es", 40}, + { "cs", 41}, + { "ss", 42}, + { "ds", 43}, + { "fs", 44}, + { "gs", 45}, + // 46-47 - reserved + { "tr", 48}, // Task Register + { "ldtr", 49}, // LDT Register + // 50-92 - reserved + { "fs.base", 92}, + { "gs.base", 93}, + // End of regular dwarf registers. + { "eip", DWARF_REG_PC }, { "ip", DWARF_REG_PC }, +}; + +static const struct dwarf_regs_idx x86_64_regidx_table[] = { + { "rax", 0 }, { "eax", 0 }, { "ax", 0 }, { "al", 0 }, + { "rdx", 1 }, { "edx", 1 }, { "dx", 1 }, { "dl", 1 }, + { "rcx", 2 }, { "ecx", 2 }, { "cx", 2 }, { "cl", 2 }, + { "rbx", 3 }, { "edx", 3 }, { "bx", 3 }, { "bl", 3 }, + { "rsi", 4 }, { "esi", 4 }, { "si", 4 }, { "sil", 4 }, + { "rdi", 5 }, { "edi", 5 }, { "di", 5 }, { "dil", 5 }, + { "rbp", 6 }, { "ebp", 6 }, { "bp", 6 }, { "bpl", 6 }, + { "rsp", 7 }, { "esp", 7 }, { "sp", 7 }, { "spl", 7 }, + { "r8", 8 }, { "r8d", 8 }, { "r8w", 8 }, { "r8b", 8 }, + { "r9", 9 }, { "r9d", 9 }, { "r9w", 9 }, { "r9b", 9 }, + { "r10", 10 }, { "r10d", 10 }, { "r10w", 10 }, { "r10b", 10 }, + { "r11", 11 }, { "r11d", 11 }, { "r11w", 11 }, { "r11b", 11 }, + { "r12", 12 }, { "r12d", 12 }, { "r12w", 12 }, { "r12b", 12 }, + { "r13", 13 }, { "r13d", 13 }, { "r13w", 13 }, { "r13b", 13 }, + { "r14", 14 }, { "r14d", 14 }, { "r14w", 14 }, { "r14b", 14 }, + { "r15", 15 }, { "r15d", 15 }, { "r15w", 15 }, { "r15b", 15 }, + // 16 - Return Address RA + { "xmm0", 17}, + { "xmm1", 18}, + { "xmm2", 19}, + { "xmm3", 20}, + { "xmm4", 21}, + { "xmm5", 22}, + { "xmm6", 23}, + { "xmm7", 24}, + { "xmm8", 25}, + { "xmm9", 26}, + { "xmm10", 27}, + { "xmm11", 28}, + { "xmm12", 29}, + { "xmm13", 30}, + { "xmm14", 31}, + { "xmm15", 32}, + { "st0", 33}, + { "st1", 34}, + { "st2", 35}, + { "st3", 36}, + { "st4", 37}, + { "st5", 38}, + { "st6", 39}, + { "st7", 40}, + { "mm0", 41}, + { "mm1", 42}, + { "mm2", 43}, + { "mm3", 44}, + { "mm4", 45}, + { "mm5", 46}, + { "mm6", 47}, + { "mm7", 48}, + { "rflags", 49}, { "eflags", 49}, { "flags", 49}, + { "es", 50}, + { "cs", 51}, + { "ss", 52}, + { "ds", 53}, + { "fs", 54}, + { "gs", 55}, + // 56-47 - reserved + { "fs.base", 58}, + { "gs.base", 59}, + // 60-61 - reserved + { "tr", 62}, // Task Register + { "ldtr", 63}, // LDT Register + { "mxcsr", 64}, // 128-bit Media Control and Status + { "fcw", 65}, // x87 Control Word + { "fsw", 66}, // x87 Status Word + // End of regular dwarf registers. + { "rip", DWARF_REG_PC }, { "eip", DWARF_REG_PC }, { "ip", DWARF_REG_PC }, +}; + +static int get_regnum(const struct dwarf_regs_idx *entries, size_t num_entries, const char *name) +{ + if (*name != '%') + return -EINVAL; + + name++; + for (size_t i = 0; i < num_entries; i++) { + if (!strcmp(entries[i].name, name)) + return entries[i].dwarf_regnum; + } + return -ENOENT; +} + +int __get_dwarf_regnum_i386(const char *name) +{ + return get_regnum(i386_regidx_table, ARRAY_SIZE(i386_regidx_table), name); +} + +int __get_dwarf_regnum_x86_64(const char *name) +{ + return get_regnum(x86_64_regidx_table, ARRAY_SIZE(x86_64_regidx_table), name); +} + +int __get_dwarf_regnum_for_perf_regnum_i386(int perf_regnum) +{ + static const int dwarf_i386_regnums[] = { + [PERF_REG_X86_AX] = 0, + [PERF_REG_X86_BX] = 3, + [PERF_REG_X86_CX] = 1, + [PERF_REG_X86_DX] = 2, + [PERF_REG_X86_SI] = 6, + [PERF_REG_X86_DI] = 7, + [PERF_REG_X86_BP] = 5, + [PERF_REG_X86_SP] = 4, + [PERF_REG_X86_IP] = 8, + [PERF_REG_X86_FLAGS] = 9, + [PERF_REG_X86_CS] = 41, + [PERF_REG_X86_SS] = 42, + [PERF_REG_X86_DS] = 43, + [PERF_REG_X86_ES] = 40, + [PERF_REG_X86_FS] = 44, + [PERF_REG_X86_GS] = 45, + [PERF_REG_X86_XMM0] = 21, + [PERF_REG_X86_XMM1] = 22, + [PERF_REG_X86_XMM2] = 23, + [PERF_REG_X86_XMM3] = 24, + [PERF_REG_X86_XMM4] = 25, + [PERF_REG_X86_XMM5] = 26, + [PERF_REG_X86_XMM6] = 27, + [PERF_REG_X86_XMM7] = 28, + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_i386_regnums) || + dwarf_i386_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_i386_regnums[perf_regnum]; +} + +int __get_dwarf_regnum_for_perf_regnum_x86_64(int perf_regnum) +{ + static const int dwarf_x86_64_regnums[] = { + [PERF_REG_X86_AX] = 0, + [PERF_REG_X86_BX] = 3, + [PERF_REG_X86_CX] = 2, + [PERF_REG_X86_DX] = 1, + [PERF_REG_X86_SI] = 4, + [PERF_REG_X86_DI] = 5, + [PERF_REG_X86_BP] = 6, + [PERF_REG_X86_SP] = 7, + [PERF_REG_X86_IP] = 16, + [PERF_REG_X86_FLAGS] = 49, + [PERF_REG_X86_CS] = 51, + [PERF_REG_X86_SS] = 52, + [PERF_REG_X86_DS] = 53, + [PERF_REG_X86_ES] = 50, + [PERF_REG_X86_FS] = 54, + [PERF_REG_X86_GS] = 55, + [PERF_REG_X86_R8] = 8, + [PERF_REG_X86_R9] = 9, + [PERF_REG_X86_R10] = 10, + [PERF_REG_X86_R11] = 11, + [PERF_REG_X86_R12] = 12, + [PERF_REG_X86_R13] = 13, + [PERF_REG_X86_R14] = 14, + [PERF_REG_X86_R15] = 15, + [PERF_REG_X86_XMM0] = 17, + [PERF_REG_X86_XMM1] = 18, + [PERF_REG_X86_XMM2] = 19, + [PERF_REG_X86_XMM3] = 20, + [PERF_REG_X86_XMM4] = 21, + [PERF_REG_X86_XMM5] = 22, + [PERF_REG_X86_XMM6] = 23, + [PERF_REG_X86_XMM7] = 24, + [PERF_REG_X86_XMM8] = 25, + [PERF_REG_X86_XMM9] = 26, + [PERF_REG_X86_XMM10] = 27, + [PERF_REG_X86_XMM11] = 28, + [PERF_REG_X86_XMM12] = 29, + [PERF_REG_X86_XMM13] = 30, + [PERF_REG_X86_XMM14] = 31, + [PERF_REG_X86_XMM15] = 32, + }; + + if (perf_regnum == 0) + return 0; + + if (perf_regnum < 0 || perf_regnum > (int)ARRAY_SIZE(dwarf_x86_64_regnums) || + dwarf_x86_64_regnums[perf_regnum] == 0) + return -ENOENT; + + return dwarf_x86_64_regnums[perf_regnum]; +} diff --git a/tools/perf/util/dwarf-regs.c b/tools/perf/util/dwarf-regs.c new file mode 100644 index 000000000..797f455eb --- /dev/null +++ b/tools/perf/util/dwarf-regs.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dwarf-regs.c : Mapping of DWARF debug register numbers into register names. + * + * Written by: Masami Hiramatsu + */ + +#include +#include +#include +#include +#include +#include +#include + +/* Define const char * {arch}_register_tbl[] */ +#define DEFINE_DWARF_REGSTR_TABLE +#include "../arch/x86/include/dwarf-regs-table.h" +#include "../arch/arm/include/dwarf-regs-table.h" +#include "../arch/arm64/include/dwarf-regs-table.h" +#include "../arch/sh/include/dwarf-regs-table.h" +#include "../arch/powerpc/include/dwarf-regs-table.h" +#include "../arch/riscv/include/dwarf-regs-table.h" +#include "../arch/s390/include/dwarf-regs-table.h" +#include "../arch/sparc/include/dwarf-regs-table.h" +#include "../arch/xtensa/include/dwarf-regs-table.h" +#include "../arch/mips/include/dwarf-regs-table.h" +#include "../arch/loongarch/include/dwarf-regs-table.h" + +/* Return architecture dependent register string (for kprobe-tracer) */ +const char *get_dwarf_regstr(unsigned int n, unsigned int machine, unsigned int flags) +{ + #define __get_dwarf_regstr(tbl, n) (((n) < ARRAY_SIZE(tbl)) ? (tbl)[(n)] : NULL) + + if (machine == EM_NONE) { + /* Generic arch - use host arch */ + machine = EM_HOST; + } + switch (machine) { + case EM_386: + return __get_dwarf_regstr(x86_32_regstr_tbl, n); + case EM_X86_64: + return __get_dwarf_regstr(x86_64_regstr_tbl, n); + case EM_ARM: + return __get_dwarf_regstr(arm_regstr_tbl, n); + case EM_AARCH64: + return __get_dwarf_regstr(aarch64_regstr_tbl, n); + case EM_CSKY: + return __get_csky_regstr(n, flags); + case EM_SH: + return __get_dwarf_regstr(sh_regstr_tbl, n); + case EM_S390: + return __get_dwarf_regstr(s390_regstr_tbl, n); + case EM_PPC: + case EM_PPC64: + return __get_dwarf_regstr(powerpc_regstr_tbl, n); + case EM_RISCV: + return __get_dwarf_regstr(riscv_regstr_tbl, n); + case EM_SPARC: + case EM_SPARCV9: + return __get_dwarf_regstr(sparc_regstr_tbl, n); + case EM_XTENSA: + return __get_dwarf_regstr(xtensa_regstr_tbl, n); + case EM_MIPS: + return __get_dwarf_regstr(mips_regstr_tbl, n); + case EM_LOONGARCH: + return __get_dwarf_regstr(loongarch_regstr_tbl, n); + default: + pr_err("ELF MACHINE %x is not supported.\n", machine); + } + return NULL; + + #undef __get_dwarf_regstr +} + +static int __get_dwarf_regnum(const char *const *regstr, size_t num_regstr, const char *name) +{ + for (size_t i = 0; i < num_regstr; i++) { + if (regstr[i] && !strcmp(regstr[i], name)) + return i; + } + return -ENOENT; +} + +/* Return DWARF register number from architecture register name */ +int get_dwarf_regnum(const char *name, unsigned int machine, unsigned int flags) +{ + char *regname = strdup(name); + int reg = -1; + char *p; + + #define _get_dwarf_regnum(tbl, name) __get_dwarf_regnum(tbl, ARRAY_SIZE(tbl), name) + + if (regname == NULL) + return -EINVAL; + + /* For convenience, remove trailing characters */ + p = strpbrk(regname, " ,)"); + if (p) + *p = '\0'; + + if (machine == EM_NONE) { + /* Generic arch - use host arch */ + machine = EM_HOST; + } + switch (machine) { + case EM_X86_64: + reg = __get_dwarf_regnum_x86_64(name); + break; + case EM_386: + reg = __get_dwarf_regnum_i386(name); + break; + case EM_ARM: + reg = _get_dwarf_regnum(arm_regstr_tbl, name); + break; + case EM_AARCH64: + reg = _get_dwarf_regnum(aarch64_regstr_tbl, name); + break; + case EM_CSKY: + reg = __get_csky_regnum(name, flags); + break; + case EM_SH: + reg = _get_dwarf_regnum(sh_regstr_tbl, name); + break; + case EM_S390: + reg = _get_dwarf_regnum(s390_regstr_tbl, name); + break; + case EM_PPC: + case EM_PPC64: + reg = _get_dwarf_regnum(powerpc_regstr_tbl, name); + break; + case EM_RISCV: + reg = _get_dwarf_regnum(riscv_regstr_tbl, name); + break; + case EM_SPARC: + case EM_SPARCV9: + reg = _get_dwarf_regnum(sparc_regstr_tbl, name); + break; + case EM_XTENSA: + reg = _get_dwarf_regnum(xtensa_regstr_tbl, name); + break; + case EM_MIPS: + reg = _get_dwarf_regnum(mips_regstr_tbl, name); + break; + case EM_LOONGARCH: + reg = _get_dwarf_regnum(loongarch_regstr_tbl, name); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", machine); + } + free(regname); + return reg; + + #undef _get_dwarf_regnum +} + +static int get_libdw_frame_nregs(unsigned int machine, unsigned int flags __maybe_unused) +{ + switch (machine) { + case EM_X86_64: + return 17; + case EM_386: + return 9; + case EM_ARM: + return 16; + case EM_AARCH64: + return 97; + case EM_CSKY: + return 38; + case EM_S390: + return 32; + case EM_PPC: + case EM_PPC64: + return 145; + case EM_RISCV: + return 66; + case EM_SPARC: + case EM_SPARCV9: + return 103; + case EM_LOONGARCH: + return 74; + case EM_MIPS: + return 71; + default: + return 0; + } +} + +int get_dwarf_regnum_for_perf_regnum(int perf_regnum, unsigned int machine, + unsigned int flags, bool only_libdw_supported) +{ + int reg; + + switch (machine) { + case EM_X86_64: + reg = __get_dwarf_regnum_for_perf_regnum_x86_64(perf_regnum); + break; + case EM_386: + reg = __get_dwarf_regnum_for_perf_regnum_i386(perf_regnum); + break; + case EM_ARM: + reg = __get_dwarf_regnum_for_perf_regnum_arm(perf_regnum); + break; + case EM_AARCH64: + reg = __get_dwarf_regnum_for_perf_regnum_arm64(perf_regnum); + break; + case EM_CSKY: + reg = __get_dwarf_regnum_for_perf_regnum_csky(perf_regnum, flags); + break; + case EM_PPC: + case EM_PPC64: + reg = __get_dwarf_regnum_for_perf_regnum_powerpc(perf_regnum); + break; + case EM_RISCV: + reg = __get_dwarf_regnum_for_perf_regnum_riscv(perf_regnum); + break; + case EM_S390: + reg = __get_dwarf_regnum_for_perf_regnum_s390(perf_regnum); + break; + case EM_LOONGARCH: + reg = __get_dwarf_regnum_for_perf_regnum_loongarch(perf_regnum); + break; + case EM_MIPS: + reg = __get_dwarf_regnum_for_perf_regnum_mips(perf_regnum); + break; + default: + pr_err("ELF MACHINE %x is not supported.\n", machine); + return -ENOENT; + } + if (reg >= 0 && only_libdw_supported) { + int nregs = get_libdw_frame_nregs(machine, flags); + + if (reg >= nregs) + reg = -ENOENT; + } + return reg; +} diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c new file mode 100644 index 000000000..c0e2b9d5f --- /dev/null +++ b/tools/perf/util/env.c @@ -0,0 +1,1087 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "cpumap.h" +#include "dwarf-regs.h" +#include "debug.h" +#include "env.h" +#include "util/header.h" +#include "util/rwsem.h" +#include +#include +#include +#include +#include +#include +#include "cgroup.h" +#include +#include +#include +#include +#include "pmu.h" +#include "pmus.h" +#include "strbuf.h" +#include "trace/beauty/beauty.h" + +#ifdef HAVE_LIBBPF_SUPPORT +#include "bpf-event.h" +#include "bpf-utils.h" +#include + +bool perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node) +{ + bool ret; + + down_write(&env->bpf_progs.lock); + ret = __perf_env__insert_bpf_prog_info(env, info_node); + up_write(&env->bpf_progs.lock); + + return ret; +} + +bool __perf_env__insert_bpf_prog_info(struct perf_env *env, struct bpf_prog_info_node *info_node) +{ + __u32 prog_id = info_node->info_linear->info.id; + struct bpf_prog_info_node *node; + struct rb_node *parent = NULL; + struct rb_node **p; + + p = &env->bpf_progs.infos.rb_node; + + while (*p != NULL) { + parent = *p; + node = rb_entry(parent, struct bpf_prog_info_node, rb_node); + if (prog_id < node->info_linear->info.id) { + p = &(*p)->rb_left; + } else if (prog_id > node->info_linear->info.id) { + p = &(*p)->rb_right; + } else { + pr_debug("duplicated bpf prog info %u\n", prog_id); + return false; + } + } + + rb_link_node(&info_node->rb_node, parent, p); + rb_insert_color(&info_node->rb_node, &env->bpf_progs.infos); + env->bpf_progs.infos_cnt++; + return true; +} + +struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, + __u32 prog_id) +{ + struct bpf_prog_info_node *node = NULL; + struct rb_node *n; + + down_read(&env->bpf_progs.lock); + n = env->bpf_progs.infos.rb_node; + + while (n) { + node = rb_entry(n, struct bpf_prog_info_node, rb_node); + if (prog_id < node->info_linear->info.id) + n = n->rb_left; + else if (prog_id > node->info_linear->info.id) + n = n->rb_right; + else + goto out; + } + node = NULL; + +out: + up_read(&env->bpf_progs.lock); + return node; +} + +void perf_env__iterate_bpf_prog_info(struct perf_env *env, + void (*cb)(struct bpf_prog_info_node *node, + void *data), + void *data) +{ + struct rb_node *first; + + down_read(&env->bpf_progs.lock); + first = rb_first(&env->bpf_progs.infos); + for (struct rb_node *node = first; node != NULL; node = rb_next(node)) + (*cb)(rb_entry(node, struct bpf_prog_info_node, rb_node), data); + up_read(&env->bpf_progs.lock); +} + +bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) +{ + bool ret; + + down_write(&env->bpf_progs.lock); + ret = __perf_env__insert_btf(env, btf_node); + up_write(&env->bpf_progs.lock); + return ret; +} + +bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node) +{ + struct rb_node *parent = NULL; + __u32 btf_id = btf_node->id; + struct btf_node *node; + struct rb_node **p; + + p = &env->bpf_progs.btfs.rb_node; + + while (*p != NULL) { + parent = *p; + node = rb_entry(parent, struct btf_node, rb_node); + if (btf_id < node->id) { + p = &(*p)->rb_left; + } else if (btf_id > node->id) { + p = &(*p)->rb_right; + } else { + pr_debug("duplicated btf %u\n", btf_id); + return false; + } + } + + rb_link_node(&btf_node->rb_node, parent, p); + rb_insert_color(&btf_node->rb_node, &env->bpf_progs.btfs); + env->bpf_progs.btfs_cnt++; + return true; +} + +struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id) +{ + struct btf_node *res; + + down_read(&env->bpf_progs.lock); + res = __perf_env__find_btf(env, btf_id); + up_read(&env->bpf_progs.lock); + return res; +} + +struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id) +{ + struct btf_node *node = NULL; + struct rb_node *n; + + n = env->bpf_progs.btfs.rb_node; + + while (n) { + node = rb_entry(n, struct btf_node, rb_node); + if (btf_id < node->id) + n = n->rb_left; + else if (btf_id > node->id) + n = n->rb_right; + else + return node; + } + return NULL; +} + +/* purge data in bpf_progs.infos tree */ +static void perf_env__purge_bpf(struct perf_env *env) +{ + struct rb_root *root; + struct rb_node *next; + + down_write(&env->bpf_progs.lock); + + root = &env->bpf_progs.infos; + next = rb_first(root); + + while (next) { + struct bpf_prog_info_node *node; + + node = rb_entry(next, struct bpf_prog_info_node, rb_node); + next = rb_next(&node->rb_node); + rb_erase(&node->rb_node, root); + zfree(&node->info_linear); + bpf_metadata_free(node->metadata); + free(node); + } + + env->bpf_progs.infos_cnt = 0; + + root = &env->bpf_progs.btfs; + next = rb_first(root); + + while (next) { + struct btf_node *node; + + node = rb_entry(next, struct btf_node, rb_node); + next = rb_next(&node->rb_node); + rb_erase(&node->rb_node, root); + free(node); + } + + env->bpf_progs.btfs_cnt = 0; + + up_write(&env->bpf_progs.lock); +} +#else // HAVE_LIBBPF_SUPPORT +static void perf_env__purge_bpf(struct perf_env *env __maybe_unused) +{ +} +#endif // HAVE_LIBBPF_SUPPORT + +void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr) +{ + if (!cd_map) + return; + + for (u32 i = 0; i < nr; i++) { + if (!cd_map[i]) + continue; + + for (u32 j = 0; j < cd_map[i]->nr_domains; j++) { + struct domain_info *d_info = cd_map[i]->domains[j]; + + if (!d_info) + continue; + + if (schedstat_version >= 17) + zfree(&d_info->dname); + + zfree(&d_info->cpumask); + zfree(&d_info->cpulist); + zfree(&d_info); + } + zfree(&cd_map[i]->domains); + zfree(&cd_map[i]); + } + zfree(&cd_map); +} + +void perf_env__exit(struct perf_env *env) +{ + int i, j; + + mutex_destroy(&env->lock); + + perf_env__purge_bpf(env); + perf_env__purge_cgroups(env); + zfree(&env->hostname); + zfree(&env->os_release); + zfree(&env->version); + zfree(&env->arch); + zfree(&env->cpu_desc); + zfree(&env->cpuid); + zfree(&env->cmdline); + zfree(&env->cmdline_argv); + zfree(&env->sibling_dies); + zfree(&env->sibling_cores); + zfree(&env->sibling_threads); + zfree(&env->pmu_mappings); + zfree(&env->cpu); + for (i = 0; i < env->nr_cpu_pmu_caps; i++) + zfree(&env->cpu_pmu_caps[i]); + zfree(&env->cpu_pmu_caps); + zfree(&env->numa_map); + + for (i = 0; i < env->nr_numa_nodes; i++) + perf_cpu_map__put(env->numa_nodes[i].map); + zfree(&env->numa_nodes); + + for (i = 0; i < env->caches_cnt; i++) + cpu_cache_level__free(&env->caches[i]); + zfree(&env->caches); + + for (i = 0; i < env->nr_memory_nodes; i++) + zfree(&env->memory_nodes[i].set); + zfree(&env->memory_nodes); + + for (i = 0; i < env->nr_hybrid_nodes; i++) { + zfree(&env->hybrid_nodes[i].pmu_name); + zfree(&env->hybrid_nodes[i].cpus); + } + zfree(&env->hybrid_nodes); + + for (i = 0; i < env->nr_pmus_with_caps; i++) { + for (j = 0; j < env->pmu_caps[i].nr_caps; j++) + zfree(&env->pmu_caps[i].caps[j]); + zfree(&env->pmu_caps[i].caps); + zfree(&env->pmu_caps[i].pmu_name); + } + zfree(&env->pmu_caps); + free_cpu_domain_info(env->cpu_domain, env->schedstat_version, env->nr_cpus_avail); +} + +void perf_env__init(struct perf_env *env) +{ + memset(env, 0, sizeof(*env)); +#ifdef HAVE_LIBBPF_SUPPORT + env->bpf_progs.infos = RB_ROOT; + env->bpf_progs.btfs = RB_ROOT; + init_rwsem(&env->bpf_progs.lock); +#endif + env->kernel_is_64_bit = -1; + mutex_init(&env->lock); +} + +static void perf_env__init_kernel_mode(struct perf_env *env) +{ + const char *arch = env->arch; + + if (!arch) { + static struct utsname uts = { .machine[0] = '\0', }; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + if (arch) { + if (strstr(arch, "64") || strstr(arch, "s390x")) + env->kernel_is_64_bit = 1; + else + env->kernel_is_64_bit = 0; + return; + } + + /* Fallback if completely unresolvable (assume host-bitness) */ + env->kernel_is_64_bit = (sizeof(void *) == 8) ? 1 : 0; +} + +int perf_env__kernel_is_64_bit(struct perf_env *env) +{ + if (env->kernel_is_64_bit == -1) + perf_env__init_kernel_mode(env); + + return env->kernel_is_64_bit; +} + +bool perf_arch_is_big_endian(const char *arch) +{ + if (!arch) + return __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__; + + if (str_ends_with(arch, "_be") || !strcmp(arch, "sparc") || !strcmp(arch, "sparc64") || + !strcmp(arch, "s390") || !strcmp(arch, "s390x") || !strcmp(arch, "powerpc") || + !strcmp(arch, "ppc") || !strcmp(arch, "ppc64") || + !strcmp(arch, "mips") || !strcmp(arch, "mips64") || !strcmp(arch, "parisc") || + !strcmp(arch, "parisc64") || !strcmp(arch, "m68k") || + !strcmp(arch, "armeb") || !strcmp(arch, "mipseb") || !strcmp(arch, "mips64eb")) + return true; + + return false; +} + +const char *perf_env__os_release(struct perf_env *env) +{ + struct utsname uts; + int ret; + const char *release; + + if (!env) + return perf_version_string; + + mutex_lock(&env->lock); + if (env->os_release) { + release = env->os_release; + goto out; + } + + /* + * If env->arch is set, this is an offline target environment. + * If the os_release is not populated in the file, we do not want + * to poison it with the host's release which would break guest checks. + */ + if (env->arch) { + release = NULL; + goto out; + } + + /* + * The os_release is being accessed but wasn't initialized from a data + * file, assume this is 'live' mode and use the release from uname. If + * uname or strdup fails then use the current perf tool version. + */ + ret = uname(&uts); + env->os_release = strdup(ret < 0 ? perf_version_string : uts.release); + release = env->os_release ?: perf_version_string; +out: + mutex_unlock(&env->lock); + return release; +} + +int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]) +{ + int i; + + /* do not include NULL termination */ + env->cmdline_argv = calloc(argc, sizeof(char *)); + if (env->cmdline_argv == NULL) + goto out_enomem; + + /* + * Must copy argv contents because it gets moved around during option + * parsing: + */ + for (i = 0; i < argc ; i++) { + env->cmdline_argv[i] = argv[i]; + if (env->cmdline_argv[i] == NULL) + goto out_free; + } + + env->nr_cmdline = argc; + + return 0; +out_free: + zfree(&env->cmdline_argv); +out_enomem: + return -ENOMEM; +} + +int perf_env__read_cpu_topology_map(struct perf_env *env) +{ + int idx, nr_cpus; + + if (env->cpu != NULL) + return 0; + + if (env->nr_cpus_avail == 0) + env->nr_cpus_avail = cpu__max_present_cpu().cpu; + + nr_cpus = env->nr_cpus_avail; + if (nr_cpus == -1) + return -EINVAL; + + env->cpu = calloc(nr_cpus, sizeof(env->cpu[0])); + if (env->cpu == NULL) + return -ENOMEM; + + for (idx = 0; idx < nr_cpus; ++idx) { + struct perf_cpu cpu = { .cpu = idx }; + int core_id = cpu__get_core_id(cpu); + int socket_id = cpu__get_socket_id(cpu); + int die_id = cpu__get_die_id(cpu); + + env->cpu[idx].core_id = core_id >= 0 ? core_id : -1; + env->cpu[idx].socket_id = socket_id >= 0 ? socket_id : -1; + env->cpu[idx].die_id = die_id >= 0 ? die_id : -1; + } + + env->nr_cpus_avail = nr_cpus; + return 0; +} + +int perf_env__read_pmu_mappings(struct perf_env *env) +{ + struct perf_pmu *pmu = NULL; + u32 pmu_num = 0; + struct strbuf sb; + + while ((pmu = perf_pmus__scan(pmu))) + pmu_num++; + + if (!pmu_num) { + pr_debug("pmu mappings not available\n"); + return -ENOENT; + } + env->nr_pmu_mappings = pmu_num; + + if (strbuf_init(&sb, 128 * pmu_num) < 0) + return -ENOMEM; + + while ((pmu = perf_pmus__scan(pmu))) { + if (strbuf_addf(&sb, "%u:%s", pmu->type, pmu->name) < 0) + goto error; + /* include a NULL character at the end */ + if (strbuf_add(&sb, "", 1) < 0) + goto error; + } + + env->pmu_mappings = strbuf_detach(&sb, NULL); + + return 0; + +error: + strbuf_release(&sb); + return -1; +} + +int perf_env__read_cpuid(struct perf_env *env) +{ + char cpuid[128]; + struct perf_cpu cpu = {-1}; + int err = get_cpuid(cpuid, sizeof(cpuid), cpu); + + if (err) + return err; + + free(env->cpuid); + env->cpuid = strdup(cpuid); + if (env->cpuid == NULL) + return ENOMEM; + return 0; +} + +static int perf_env__read_nr_cpus_avail(struct perf_env *env) +{ + if (env->nr_cpus_avail == 0) + env->nr_cpus_avail = cpu__max_present_cpu().cpu; + + return env->nr_cpus_avail ? 0 : -ENOENT; +} + +static int __perf_env__read_core_pmu_caps(const struct perf_pmu *pmu, + int *nr_caps, char ***caps, + unsigned int *max_branches, + unsigned int *br_cntr_nr, + unsigned int *br_cntr_width) +{ + struct perf_pmu_caps *pcaps = NULL; + char *ptr, **tmp; + int ret = 0; + + *nr_caps = 0; + *caps = NULL; + + if (!pmu->nr_caps) + return 0; + + *caps = calloc(pmu->nr_caps, sizeof(char *)); + if (!*caps) + return -ENOMEM; + + tmp = *caps; + list_for_each_entry(pcaps, &pmu->caps, list) { + if (asprintf(&ptr, "%s=%s", pcaps->name, pcaps->value) < 0) { + ret = -ENOMEM; + goto error; + } + + *tmp++ = ptr; + + if (!strcmp(pcaps->name, "branches")) + *max_branches = atoi(pcaps->value); + else if (!strcmp(pcaps->name, "branch_counter_nr")) + *br_cntr_nr = atoi(pcaps->value); + else if (!strcmp(pcaps->name, "branch_counter_width")) + *br_cntr_width = atoi(pcaps->value); + } + *nr_caps = pmu->nr_caps; + return 0; +error: + while (tmp-- != *caps) + zfree(tmp); + zfree(caps); + *nr_caps = 0; + return ret; +} + +int perf_env__read_core_pmu_caps(struct perf_env *env) +{ + struct pmu_caps *pmu_caps; + struct perf_pmu *pmu = NULL; + int nr_pmu, i = 0, j; + int ret; + + nr_pmu = perf_pmus__num_core_pmus(); + + if (!nr_pmu) + return -ENODEV; + + if (nr_pmu == 1) { + pmu = perf_pmus__find_core_pmu(); + if (!pmu) + return -ENODEV; + ret = perf_pmu__caps_parse(pmu); + if (ret < 0) + return ret; + return __perf_env__read_core_pmu_caps(pmu, &env->nr_cpu_pmu_caps, + &env->cpu_pmu_caps, + &env->max_branches, + &env->br_cntr_nr, + &env->br_cntr_width); + } + + pmu_caps = calloc(nr_pmu, sizeof(*pmu_caps)); + if (!pmu_caps) + return -ENOMEM; + + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + if (perf_pmu__caps_parse(pmu) <= 0) + continue; + ret = __perf_env__read_core_pmu_caps(pmu, &pmu_caps[i].nr_caps, + &pmu_caps[i].caps, + &pmu_caps[i].max_branches, + &pmu_caps[i].br_cntr_nr, + &pmu_caps[i].br_cntr_width); + if (ret) + goto error; + + pmu_caps[i].pmu_name = strdup(pmu->name); + if (!pmu_caps[i].pmu_name) { + ret = -ENOMEM; + goto error; + } + i++; + } + + env->nr_pmus_with_caps = nr_pmu; + env->pmu_caps = pmu_caps; + + return 0; +error: + for (i = 0; i < nr_pmu; i++) { + for (j = 0; j < pmu_caps[i].nr_caps; j++) + zfree(&pmu_caps[i].caps[j]); + zfree(&pmu_caps[i].caps); + zfree(&pmu_caps[i].pmu_name); + } + zfree(&pmu_caps); + return ret; +} + +int perf_env__nr_cpus_avail(struct perf_env *env) +{ + return env && !perf_env__read_nr_cpus_avail(env) ? env->nr_cpus_avail : 0; +} + +void cpu_cache_level__free(struct cpu_cache_level *cache) +{ + zfree(&cache->type); + zfree(&cache->map); + zfree(&cache->size); +} + +struct arch_to_e_machine { + const char *prefix; + uint16_t e_machine; +}; + +/* + * A mapping from an arch prefix string to an ELF machine that can be used in a + * bsearch. Some arch prefixes are shared an need additional processing as + * marked next to the architecture. The prefixes handle both perf's architecture + * naming and those from uname. + */ +static const struct arch_to_e_machine prefix_to_e_machine[] = { + {"aarch64", EM_AARCH64}, + {"alpha", EM_ALPHA}, + {"arc", EM_ARC}, + {"arm", EM_ARM}, /* Check also for EM_AARCH64. */ + {"avr", EM_AVR}, /* Check also for EM_AVR32. */ + {"bfin", EM_BLACKFIN}, + {"blackfin", EM_BLACKFIN}, + {"cris", EM_CRIS}, + {"csky", EM_CSKY}, + {"hppa", EM_PARISC}, + {"i386", EM_386}, + {"i486", EM_386}, + {"i586", EM_386}, + {"i686", EM_386}, + {"loongarch", EM_LOONGARCH}, + {"m32r", EM_M32R}, + {"m68k", EM_68K}, + {"microblaze", EM_MICROBLAZE}, + {"mips", EM_MIPS}, + {"msp430", EM_MSP430}, + {"parisc", EM_PARISC}, + {"powerpc", EM_PPC}, /* Check also for EM_PPC64. */ + {"ppc", EM_PPC}, /* Check also for EM_PPC64. */ + {"riscv", EM_RISCV}, + {"s390", EM_S390}, + {"sa110", EM_ARM}, + {"sh", EM_SH}, + {"sparc", EM_SPARC}, /* Check also for EM_SPARCV9. */ + {"sun4u", EM_SPARC}, + {"x86", EM_X86_64}, /* Check also for EM_386. */ + {"xtensa", EM_XTENSA}, +}; + +static int compare_prefix(const void *key, const void *element) +{ + const char *search_key = key; + const struct arch_to_e_machine *map_element = element; + size_t prefix_len = strlen(map_element->prefix); + + return strncmp(search_key, map_element->prefix, prefix_len); +} + +static uint16_t perf_arch_to_e_machine(const char *perf_arch, int is_64_bit) +{ + /* Binary search for a matching prefix. */ + const struct arch_to_e_machine *result; + + if (!perf_arch) + return EM_HOST; + + result = bsearch(perf_arch, + prefix_to_e_machine, ARRAY_SIZE(prefix_to_e_machine), + sizeof(prefix_to_e_machine[0]), + compare_prefix); + + if (!result) { + pr_debug("Unknown perf arch for ELF machine mapping: %s\n", perf_arch); + return EM_NONE; + } + + /* + * Handle conflicting prefixes. If the is_64_bit is unknown (-1) then + * assume 64-bit. We can't use perf_env__kernel_is_64_bit as that + * depends on the arch string. + */ + switch (result->e_machine) { + case EM_ARM: + return !strcmp(perf_arch, "arm64") || !strcmp(perf_arch, "aarch64") + ? EM_AARCH64 : EM_ARM; + case EM_AVR: + return !strcmp(perf_arch, "avr32") ? EM_AVR32 : EM_AVR; + case EM_PPC: + if (is_64_bit == 1) + return EM_PPC64; + if (is_64_bit == 0) + return EM_PPC; + return strstarts(perf_arch, "ppc64") ? EM_PPC64 : EM_PPC; + case EM_SPARC: + if (is_64_bit == 1) + return EM_SPARCV9; + if (is_64_bit == 0) + return EM_SPARC; + return !strcmp(perf_arch, "sparc64") || !strcmp(perf_arch, "sun4u") + ? EM_SPARCV9 : EM_SPARC; + case EM_X86_64: + if (is_64_bit == 1) + return EM_X86_64; + if (is_64_bit == 0) + return EM_386; + return !strcmp(perf_arch, "x86_64") || !strcmp(perf_arch, "x86") + ? EM_X86_64 : EM_386; + default: + return result->e_machine; + } +} + +static const char *e_machine_to_perf_arch(uint16_t e_machine) +{ + /* + * Table for if either the perf arch string differs from uname or there + * are >1 ELF machine with the prefix. + */ + static const struct arch_to_e_machine extras[] = { + {"arm64", EM_AARCH64}, + {"avr32", EM_AVR32}, + {"powerpc", EM_PPC}, + {"powerpc", EM_PPC64}, + {"sparc", EM_SPARCV9}, + {"x86", EM_386}, + {"x86", EM_X86_64}, + {"none", EM_NONE}, + }; + + for (size_t i = 0; i < ARRAY_SIZE(extras); i++) { + if (extras[i].e_machine == e_machine) + return extras[i].prefix; + } + + for (size_t i = 0; i < ARRAY_SIZE(prefix_to_e_machine); i++) { + if (prefix_to_e_machine[i].e_machine == e_machine) + return prefix_to_e_machine[i].prefix; + + } + return "unknown"; +} + +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags) +{ + uint16_t e_machine = EM_NONE; + const char *arch = NULL; + int is_64_bit = -1; + + if (e_flags) + *e_flags = 0; + + if (env) { + arch = env->arch; + is_64_bit = env->kernel_is_64_bit; + } + + if (!arch) { + static struct utsname uts = { .machine[0] = '\0', }; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') + arch = uts.machine; + } + + e_machine = perf_arch_to_e_machine(arch, is_64_bit); + + if (e_flags) + *e_flags = (e_machine == EM_HOST) ? EF_HOST : 0; + + return e_machine; +} + +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags) +{ + uint16_t e_machine; + uint32_t local_e_flags = 0; + + if (env && env->e_machine != EM_NONE) { + if (e_flags) + *e_flags = env->e_flags; + + return env->e_machine; + } + e_machine = perf_env__e_machine_nocache(env, &local_e_flags); + /* + * Only cache the e_machine in perf_env if env->arch is not NULL. + * If env->arch is NULL, the e_machine is just a fallback to EM_HOST. + * Caching it permanently would prevent dynamic, more accurate + * thread-based session e_machine scanning later in + * perf_session__e_machine(). + */ + if (env && env->arch) { + env->e_machine = e_machine; + env->e_flags = local_e_flags; + } + if (e_flags) + *e_flags = local_e_flags; + + return e_machine; +} + +const char *perf_env__arch(struct perf_env *env) +{ + uint16_t e_machine; + const char *arch; + + if (!env) { + static struct utsname uts = { .machine[0] = '\0', }; + uint16_t host_e_machine; + + if (uts.machine[0] == '\0') + uname(&uts); + if (uts.machine[0] != '\0') { + host_e_machine = perf_arch_to_e_machine(uts.machine, -1); + return e_machine_to_perf_arch(host_e_machine); + } + return e_machine_to_perf_arch(EM_HOST); + } + + /* + * Lazily compute/allocate arch. The e_machine may have been + * read from a data file and so may not be EM_HOST. + */ + e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + arch = e_machine_to_perf_arch(e_machine); + + if (e_machine == EM_RISCV && perf_env__kernel_is_64_bit(env) == 1) + arch = "riscv64"; + else if (e_machine == EM_MIPS && perf_env__kernel_is_64_bit(env) == 1) + arch = "mips64"; + else if (e_machine == EM_PARISC && perf_env__kernel_is_64_bit(env) == 1) + arch = "parisc64"; + + return arch; +} + +const char *arch_syscalls__strerrno(uint16_t e_machine, int err); + +const char *perf_env__arch_strerrno(uint16_t e_machine, int err) +{ + return arch_syscalls__strerrno(e_machine, err); +} + +const char *perf_env__cpuid(struct perf_env *env) +{ + int status; + + if (!env->cpuid) { /* Assume local operation */ + status = perf_env__read_cpuid(env); + if (status) + return NULL; + } + + return env->cpuid; +} + +int perf_env__nr_pmu_mappings(struct perf_env *env) +{ + int status; + + if (!env->nr_pmu_mappings) { /* Assume local operation */ + status = perf_env__read_pmu_mappings(env); + if (status) + return 0; + } + + return env->nr_pmu_mappings; +} + +const char *perf_env__pmu_mappings(struct perf_env *env) +{ + int status; + + if (!env->pmu_mappings) { /* Assume local operation */ + status = perf_env__read_pmu_mappings(env); + if (status) + return NULL; + } + + return env->pmu_mappings; +} + +int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu) +{ + if (!env->nr_numa_map) { + struct numa_node *nn; + int i, nr = 0; + + for (i = 0; i < env->nr_numa_nodes; i++) { + nn = &env->numa_nodes[i]; + nr = max(nr, (int)perf_cpu_map__max(nn->map).cpu); + } + + nr++; + + /* + * We initialize the numa_map array to prepare + * it for missing cpus, which return node -1 + */ + env->numa_map = malloc(nr * sizeof(int)); + if (!env->numa_map) + return -1; + + for (i = 0; i < nr; i++) + env->numa_map[i] = -1; + + env->nr_numa_map = nr; + + for (i = 0; i < env->nr_numa_nodes; i++) { + struct perf_cpu tmp; + unsigned int j; + + nn = &env->numa_nodes[i]; + perf_cpu_map__for_each_cpu(tmp, j, nn->map) + env->numa_map[tmp.cpu] = i; + } + } + + return cpu.cpu >= 0 && cpu.cpu < env->nr_numa_map ? env->numa_map[cpu.cpu] : -1; +} + +bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name) +{ + char *pmu_mapping = env->pmu_mappings, *colon; + + for (int i = 0; i < env->nr_pmu_mappings; ++i) { + if (strtoul(pmu_mapping, &colon, 0) == ULONG_MAX || *colon != ':') + goto out_error; + + pmu_mapping = colon + 1; + if (strcmp(pmu_mapping, pmu_name) == 0) + return true; + + pmu_mapping += strlen(pmu_mapping) + 1; + } +out_error: + return false; +} + +char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name, + const char *cap) +{ + char *cap_eq; + int cap_size; + char **ptr; + int i, j; + + if (!pmu_name || !cap) + return NULL; + + cap_size = strlen(cap); + cap_eq = zalloc(cap_size + 2); + if (!cap_eq) + return NULL; + + memcpy(cap_eq, cap, cap_size); + cap_eq[cap_size] = '='; + + if (!strcmp(pmu_name, "cpu")) { + for (i = 0; i < env->nr_cpu_pmu_caps; i++) { + if (!strncmp(env->cpu_pmu_caps[i], cap_eq, cap_size + 1)) { + free(cap_eq); + return &env->cpu_pmu_caps[i][cap_size + 1]; + } + } + goto out; + } + + for (i = 0; i < env->nr_pmus_with_caps; i++) { + if (strcmp(env->pmu_caps[i].pmu_name, pmu_name)) + continue; + + ptr = env->pmu_caps[i].caps; + + for (j = 0; j < env->pmu_caps[i].nr_caps; j++) { + if (!strncmp(ptr[j], cap_eq, cap_size + 1)) { + free(cap_eq); + return &ptr[j][cap_size + 1]; + } + } + } + +out: + free(cap_eq); + return NULL; +} + +void perf_env__find_br_cntr_info(struct perf_env *env, + unsigned int *nr, + unsigned int *width) +{ + if (nr) { + *nr = env->cpu_pmu_caps ? env->br_cntr_nr : + env->pmu_caps->br_cntr_nr; + } + + if (width) { + *width = env->cpu_pmu_caps ? env->br_cntr_width : + env->pmu_caps->br_cntr_width; + } +} + +bool perf_env__is_x86_amd_cpu(struct perf_env *env) +{ + static int is_amd; /* 0: Uninitialized, 1: Yes, -1: No */ + + if (is_amd == 0) + is_amd = env->cpuid && strstarts(env->cpuid, "AuthenticAMD") ? 1 : -1; + + return is_amd >= 1 ? true : false; +} + +bool x86__is_amd_cpu(void) +{ + struct perf_env env = { .total_mem = 0, }; + bool is_amd; + + perf_env__init(&env); + perf_env__cpuid(&env); + is_amd = perf_env__is_x86_amd_cpu(&env); + perf_env__exit(&env); + + return is_amd; +} + +bool perf_env__is_x86_intel_cpu(struct perf_env *env) +{ + static int is_intel; /* 0: Uninitialized, 1: Yes, -1: No */ + + if (is_intel == 0) + is_intel = env->cpuid && strstarts(env->cpuid, "GenuineIntel") ? 1 : -1; + + return is_intel >= 1 ? true : false; +} + +bool x86__is_intel_cpu(void) +{ + struct perf_env env = { .total_mem = 0, }; + bool is_intel; + + perf_env__init(&env); + perf_env__cpuid(&env); + is_intel = perf_env__is_x86_intel_cpu(&env); + perf_env__exit(&env); + + return is_intel; +} diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h new file mode 100644 index 000000000..7acca39b4 --- /dev/null +++ b/tools/perf/util/env.h @@ -0,0 +1,245 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_ENV_H +#define __PERF_ENV_H + +#include +#include +#include "cpumap.h" +#include "rwsem.h" +#include "mutex.h" + +struct perf_cpu_map; + +struct cpu_topology_map { + int socket_id; + int die_id; + int cluster_id; + int core_id; +}; + +struct cpu_cache_level { + u32 level; + u32 line_size; + u32 sets; + u32 ways; + char *type; + char *size; + char *map; +}; + +struct numa_node { + u32 node; + u64 mem_total; + u64 mem_free; + struct perf_cpu_map *map; +}; + +struct memory_node { + u64 node; + u64 size; + unsigned long *set; +}; + +struct hybrid_node { + char *pmu_name; + char *cpus; +}; + +struct pmu_caps { + int nr_caps; + unsigned int max_branches; + unsigned int br_cntr_nr; + unsigned int br_cntr_width; + + char **caps; + char *pmu_name; +}; + +struct domain_info { + u32 domain; + char *dname; + char *cpumask; + char *cpulist; +}; + +struct cpu_domain_map { + u32 cpu; + u32 nr_domains; + struct domain_info **domains; +}; + +struct perf_env { + char *hostname; + char *os_release; + char *version; + char *arch; + /* e_machine expanded from 16 to 32-bits for alignment. */ + u32 e_machine; + u32 e_flags; + int nr_cpus_online; + int nr_cpus_avail; + char *cpu_desc; + char *cpuid; + unsigned long long total_mem; + unsigned int msr_pmu_type; + unsigned int max_branches; + unsigned int br_cntr_nr; + unsigned int br_cntr_width; + unsigned int schedstat_version; + unsigned int max_sched_domains; + int kernel_is_64_bit; + + int nr_cmdline; + int nr_sibling_cores; + int nr_sibling_dies; + int nr_sibling_threads; + int nr_numa_nodes; + int nr_memory_nodes; + int nr_pmu_mappings; + int nr_groups; + int nr_cpu_pmu_caps; + int nr_hybrid_nodes; + int nr_pmus_with_caps; + char *cmdline; + const char **cmdline_argv; + char *sibling_cores; + char *sibling_dies; + char *sibling_threads; + char *pmu_mappings; + char **cpu_pmu_caps; + struct cpu_topology_map *cpu; + struct cpu_cache_level *caches; + struct cpu_domain_map **cpu_domain; + int caches_cnt; + unsigned int cln_size; + u32 comp_ratio; + u32 comp_ver; + u32 comp_type; + u32 comp_level; + u32 comp_mmap_len; + struct numa_node *numa_nodes; + struct memory_node *memory_nodes; + unsigned long long memory_bsize; + struct hybrid_node *hybrid_nodes; + struct pmu_caps *pmu_caps; +#ifdef HAVE_LIBBPF_SUPPORT + /* + * bpf_info_lock protects bpf rbtrees. This is needed because the + * trees are accessed by different threads in perf-top + */ + struct { + struct rw_semaphore lock; + struct rb_root infos; + u32 infos_cnt; + struct rb_root btfs; + u32 btfs_cnt; + } bpf_progs; +#endif // HAVE_LIBBPF_SUPPORT + /* same reason as above (for perf-top) */ + struct { + struct rw_semaphore lock; + struct rb_root tree; + } cgroups; + + /* For fast cpu to numa node lookup via perf_env__numa_node */ + int *numa_map; + int nr_numa_map; + + /* For real clock time reference. */ + struct { + u64 tod_ns; + u64 clockid_ns; + u64 clockid_res_ns; + int clockid; + /* + * enabled is valid for report mode, and is true if above + * values are set, it's set in process_clock_data + */ + bool enabled; + } clock; + /* Protects lazy environment initialization (e.g. os_release, e_machine). */ + struct mutex lock; +}; + +enum perf_compress_type { + PERF_COMP_NONE = 0, + PERF_COMP_ZSTD, + PERF_COMP_MAX +}; + +struct bpf_prog_info_node; +struct btf_node; + +int perf_env__read_core_pmu_caps(struct perf_env *env); +void free_cpu_domain_info(struct cpu_domain_map **cd_map, u32 schedstat_version, u32 nr); +void perf_env__exit(struct perf_env *env); + +int perf_env__kernel_is_64_bit(struct perf_env *env); +bool perf_arch_is_big_endian(const char *arch); +const char *perf_env__os_release(struct perf_env *env); + +int perf_env__set_cmdline(struct perf_env *env, int argc, const char *argv[]); + +int perf_env__read_cpuid(struct perf_env *env); +int perf_env__read_pmu_mappings(struct perf_env *env); +int perf_env__nr_pmu_mappings(struct perf_env *env); +const char *perf_env__pmu_mappings(struct perf_env *env); + +int perf_env__read_cpu_topology_map(struct perf_env *env); + +/* + * Safe accessor for env->cpu[] topology array. env->cpu can be NULL when + * reading old-format perf.data that predates topology information — + * process_cpu_topology() in header.c frees it while nr_cpus_avail remains + * set, so callers must not index env->cpu[] without this check. + */ +static inline struct cpu_topology_map * +perf_env__get_cpu_topology(struct perf_env *env, struct perf_cpu cpu) +{ + if (env->cpu && cpu.cpu >= 0 && cpu.cpu < env->nr_cpus_avail) + return &env->cpu[cpu.cpu]; + return NULL; +} + +void cpu_cache_level__free(struct cpu_cache_level *cache); + +uint16_t perf_env__e_machine_nocache(struct perf_env *env, uint32_t *e_flags); +uint16_t perf_env__e_machine(struct perf_env *env, uint32_t *e_flags); +const char *perf_env__arch(struct perf_env *env); +const char *perf_env__arch_strerrno(uint16_t e_machine, int err); +const char *perf_env__cpuid(struct perf_env *env); +int perf_env__nr_cpus_avail(struct perf_env *env); + +void perf_env__init(struct perf_env *env); +#ifdef HAVE_LIBBPF_SUPPORT +bool __perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node); +bool perf_env__insert_bpf_prog_info(struct perf_env *env, + struct bpf_prog_info_node *info_node); +struct bpf_prog_info_node *perf_env__find_bpf_prog_info(struct perf_env *env, + __u32 prog_id); +void perf_env__iterate_bpf_prog_info(struct perf_env *env, + void (*cb)(struct bpf_prog_info_node *node, + void *data), + void *data); +bool perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node); +bool __perf_env__insert_btf(struct perf_env *env, struct btf_node *btf_node); +struct btf_node *perf_env__find_btf(struct perf_env *env, __u32 btf_id); +struct btf_node *__perf_env__find_btf(struct perf_env *env, __u32 btf_id); +#endif // HAVE_LIBBPF_SUPPORT + +int perf_env__numa_node(struct perf_env *env, struct perf_cpu cpu); +char *perf_env__find_pmu_cap(struct perf_env *env, const char *pmu_name, + const char *cap); + +bool perf_env__has_pmu_mapping(struct perf_env *env, const char *pmu_name); +void perf_env__find_br_cntr_info(struct perf_env *env, + unsigned int *nr, + unsigned int *width); + +bool x86__is_amd_cpu(void); +bool perf_env__is_x86_amd_cpu(struct perf_env *env); +bool x86__is_intel_cpu(void); +bool perf_env__is_x86_intel_cpu(struct perf_env *env); + +#endif /* __PERF_ENV_H */ diff --git a/tools/perf/util/event.c b/tools/perf/util/event.c new file mode 100644 index 000000000..ea75816d1 --- /dev/null +++ b/tools/perf/util/event.c @@ -0,0 +1,944 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include /* To get things like MAP_HUGETLB even on older libc headers */ +#include +#include "cpumap.h" +#include "dso.h" +#include "env.h" +#include "event.h" +#include "debug.h" +#include "hist.h" +#include "machine.h" +#include "sort.h" +#include "string2.h" +#include "strlist.h" +#include "thread.h" +#include "thread_map.h" +#include "time-utils.h" +#include +#include "map.h" +#include "util/namespaces.h" +#include "symbol.h" +#include "symbol/kallsyms.h" +#include "asm/bug.h" +#include "stat.h" +#include "session.h" +#include "bpf-event.h" +#include "print_binary.h" +#include "tool.h" +#include "util.h" + +static const char *perf_event__names[] = { + [0] = "TOTAL", + [PERF_RECORD_MMAP] = "MMAP", + [PERF_RECORD_MMAP2] = "MMAP2", + [PERF_RECORD_LOST] = "LOST", + [PERF_RECORD_COMM] = "COMM", + [PERF_RECORD_EXIT] = "EXIT", + [PERF_RECORD_THROTTLE] = "THROTTLE", + [PERF_RECORD_UNTHROTTLE] = "UNTHROTTLE", + [PERF_RECORD_FORK] = "FORK", + [PERF_RECORD_READ] = "READ", + [PERF_RECORD_SAMPLE] = "SAMPLE", + [PERF_RECORD_AUX] = "AUX", + [PERF_RECORD_ITRACE_START] = "ITRACE_START", + [PERF_RECORD_LOST_SAMPLES] = "LOST_SAMPLES", + [PERF_RECORD_SWITCH] = "SWITCH", + [PERF_RECORD_SWITCH_CPU_WIDE] = "SWITCH_CPU_WIDE", + [PERF_RECORD_NAMESPACES] = "NAMESPACES", + [PERF_RECORD_KSYMBOL] = "KSYMBOL", + [PERF_RECORD_BPF_EVENT] = "BPF_EVENT", + [PERF_RECORD_CGROUP] = "CGROUP", + [PERF_RECORD_TEXT_POKE] = "TEXT_POKE", + [PERF_RECORD_AUX_OUTPUT_HW_ID] = "AUX_OUTPUT_HW_ID", + [PERF_RECORD_CALLCHAIN_DEFERRED] = "CALLCHAIN_DEFERRED", + [PERF_RECORD_HEADER_ATTR] = "ATTR", + [PERF_RECORD_HEADER_EVENT_TYPE] = "EVENT_TYPE", + [PERF_RECORD_HEADER_TRACING_DATA] = "TRACING_DATA", + [PERF_RECORD_HEADER_BUILD_ID] = "BUILD_ID", + [PERF_RECORD_FINISHED_ROUND] = "FINISHED_ROUND", + [PERF_RECORD_ID_INDEX] = "ID_INDEX", + [PERF_RECORD_AUXTRACE_INFO] = "AUXTRACE_INFO", + [PERF_RECORD_AUXTRACE] = "AUXTRACE", + [PERF_RECORD_AUXTRACE_ERROR] = "AUXTRACE_ERROR", + [PERF_RECORD_THREAD_MAP] = "THREAD_MAP", + [PERF_RECORD_CPU_MAP] = "CPU_MAP", + [PERF_RECORD_STAT_CONFIG] = "STAT_CONFIG", + [PERF_RECORD_STAT] = "STAT", + [PERF_RECORD_STAT_ROUND] = "STAT_ROUND", + [PERF_RECORD_EVENT_UPDATE] = "EVENT_UPDATE", + [PERF_RECORD_TIME_CONV] = "TIME_CONV", + [PERF_RECORD_HEADER_FEATURE] = "FEATURE", + [PERF_RECORD_COMPRESSED] = "COMPRESSED", + [PERF_RECORD_FINISHED_INIT] = "FINISHED_INIT", + [PERF_RECORD_COMPRESSED2] = "COMPRESSED2", + [PERF_RECORD_BPF_METADATA] = "BPF_METADATA", + [PERF_RECORD_SCHEDSTAT_CPU] = "SCHEDSTAT_CPU", + [PERF_RECORD_SCHEDSTAT_DOMAIN] = "SCHEDSTAT_DOMAIN", +}; + +const char *perf_event__name(unsigned int id) +{ + if (id >= ARRAY_SIZE(perf_event__names)) + return "INVALID"; + if (!perf_event__names[id]) + return "UNKNOWN"; + return perf_event__names[id]; +} + +struct process_symbol_args { + const char *name; + u64 start; +}; + +static int find_func_symbol_cb(void *arg, const char *name, char type, + u64 start) +{ + struct process_symbol_args *args = arg; + + /* + * Must be a function or at least an alias, as in PARISC64, where "_text" is + * an 'A' to the same address as "_stext". + */ + if (!(kallsyms__is_function(type) || + type == 'A') || strcmp(name, args->name)) + return 0; + + args->start = start; + return 1; +} + +static int find_any_symbol_cb(void *arg, const char *name, + char type __maybe_unused, u64 start) +{ + struct process_symbol_args *args = arg; + + if (strcmp(name, args->name)) + return 0; + + args->start = start; + return 1; +} + +int kallsyms__get_function_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr) +{ + struct process_symbol_args args = { .name = symbol_name, }; + + if (kallsyms__parse(kallsyms_filename, &args, find_func_symbol_cb) <= 0) + return -1; + + *addr = args.start; + return 0; +} + +int kallsyms__get_symbol_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr) +{ + struct process_symbol_args args = { .name = symbol_name, }; + + if (kallsyms__parse(kallsyms_filename, &args, find_any_symbol_cb) <= 0) + return -1; + + *addr = args.start; + return 0; +} + +void perf_event__read_stat_config(struct perf_stat_config *config, + struct perf_record_stat_config *event) +{ + unsigned i; + + for (i = 0; i < event->nr; i++) { + + switch (event->data[i].tag) { +#define CASE(__term, __val) \ + case PERF_STAT_CONFIG_TERM__##__term: \ + config->__val = event->data[i].val; \ + break; + + CASE(AGGR_MODE, aggr_mode) + CASE(SCALE, scale) + CASE(INTERVAL, interval) + CASE(AGGR_LEVEL, aggr_level) +#undef CASE + default: + pr_warning("unknown stat config term %" PRI_lu64 "\n", + event->data[i].tag); + } + } +} + +size_t perf_event__fprintf_comm(union perf_event *event, FILE *fp) +{ + const char *s; + + if (event->header.misc & PERF_RECORD_MISC_COMM_EXEC) + s = " exec"; + else + s = ""; + + return fprintf(fp, "%s: %s:%d/%d\n", s, event->comm.comm, event->comm.pid, event->comm.tid); +} + +size_t perf_event__fprintf_namespaces(union perf_event *event, FILE *fp) +{ + size_t ret = 0; + struct perf_ns_link_info *ns_link_info; + u32 nr_namespaces, idx; + + ns_link_info = event->namespaces.link_info; + nr_namespaces = event->namespaces.nr_namespaces; + + ret += fprintf(fp, " %d/%d - nr_namespaces: %u\n\t\t[", + event->namespaces.pid, + event->namespaces.tid, + nr_namespaces); + + for (idx = 0; idx < nr_namespaces; idx++) { + if (idx && (idx % 4 == 0)) + ret += fprintf(fp, "\n\t\t "); + + ret += fprintf(fp, "%u/%s: %" PRIu64 "/%#" PRIx64 "%s", idx, + perf_ns__name(idx), (u64)ns_link_info[idx].dev, + (u64)ns_link_info[idx].ino, + ((idx + 1) != nr_namespaces) ? ", " : "]\n"); + } + + return ret; +} + +size_t perf_event__fprintf_cgroup(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " cgroup: %" PRI_lu64 " %s\n", + event->cgroup.id, event->cgroup.path); +} + +int perf_event__process_comm(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_comm_event(machine, event, sample); +} + +int perf_event__process_namespaces(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_namespaces_event(machine, event, sample); +} + +int perf_event__process_cgroup(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_cgroup_event(machine, event, sample); +} + +int perf_event__process_lost(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_lost_event(machine, event, sample); +} + +int perf_event__process_aux(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_aux_event(machine, event); +} + +int perf_event__process_itrace_start(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_itrace_start_event(machine, event); +} + +int perf_event__process_aux_output_hw_id(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_aux_output_hw_id_event(machine, event); +} + +int perf_event__process_lost_samples(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_lost_samples_event(machine, event, sample); +} + +int perf_event__process_switch(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_switch_event(machine, event); +} + +int perf_event__process_ksymbol(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + return machine__process_ksymbol(machine, event, sample); +} + +int perf_event__process_bpf(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_bpf(machine, event, sample); +} + +int perf_event__process_text_poke(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_text_poke(machine, event, sample); +} + +size_t perf_event__fprintf_mmap(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " %d/%d: [%#" PRI_lx64 "(%#" PRI_lx64 ") @ %#" PRI_lx64 "]: %c %s\n", + event->mmap.pid, event->mmap.tid, event->mmap.start, + event->mmap.len, event->mmap.pgoff, + (event->header.misc & PERF_RECORD_MISC_MMAP_DATA) ? 'r' : 'x', + event->mmap.filename); +} + +size_t perf_event__fprintf_mmap2(union perf_event *event, FILE *fp) +{ + if (event->header.misc & PERF_RECORD_MISC_MMAP_BUILD_ID) { + char sbuild_id[SBUILD_ID_SIZE]; + struct build_id bid; + + build_id__init(&bid, event->mmap2.build_id, + event->mmap2.build_id_size); + build_id__snprintf(&bid, sbuild_id, sizeof(sbuild_id)); + + return fprintf(fp, " %d/%d: [%#" PRI_lx64 "(%#" PRI_lx64 ") @ %#" PRI_lx64 + " <%s>]: %c%c%c%c %s\n", + event->mmap2.pid, event->mmap2.tid, event->mmap2.start, + event->mmap2.len, event->mmap2.pgoff, sbuild_id, + (event->mmap2.prot & PROT_READ) ? 'r' : '-', + (event->mmap2.prot & PROT_WRITE) ? 'w' : '-', + (event->mmap2.prot & PROT_EXEC) ? 'x' : '-', + (event->mmap2.flags & MAP_SHARED) ? 's' : 'p', + event->mmap2.filename); + } else { + return fprintf(fp, " %d/%d: [%#" PRI_lx64 "(%#" PRI_lx64 ") @ %#" PRI_lx64 + " %02x:%02x %"PRI_lu64" %"PRI_lu64"]: %c%c%c%c %s\n", + event->mmap2.pid, event->mmap2.tid, event->mmap2.start, + event->mmap2.len, event->mmap2.pgoff, event->mmap2.maj, + event->mmap2.min, event->mmap2.ino, + event->mmap2.ino_generation, + (event->mmap2.prot & PROT_READ) ? 'r' : '-', + (event->mmap2.prot & PROT_WRITE) ? 'w' : '-', + (event->mmap2.prot & PROT_EXEC) ? 'x' : '-', + (event->mmap2.flags & MAP_SHARED) ? 's' : 'p', + event->mmap2.filename); + } +} + +size_t perf_event__fprintf_thread_map(union perf_event *event, FILE *fp) +{ + struct perf_thread_map *threads = thread_map__new_event(&event->thread_map); + size_t ret; + + ret = fprintf(fp, " nr: "); + + if (threads) + ret += thread_map__fprintf(threads, fp); + else + ret += fprintf(fp, "failed to get threads from event\n"); + + perf_thread_map__put(threads); + return ret; +} + +size_t perf_event__fprintf_cpu_map(union perf_event *event, FILE *fp) +{ + struct perf_cpu_map *cpus = cpu_map__new_data(&event->cpu_map.data); + size_t ret; + + ret = fprintf(fp, ": "); + + if (cpus) + ret += cpu_map__fprintf(cpus, fp); + else + ret += fprintf(fp, "failed to get cpumap from event\n"); + + perf_cpu_map__put(cpus); + return ret; +} + +int perf_event__process_mmap(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_mmap_event(machine, event, sample); +} + +int perf_event__process_mmap2(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_mmap2_event(machine, event, sample); +} + +size_t perf_event__fprintf_task(union perf_event *event, FILE *fp) +{ + return fprintf(fp, "(%d:%d):(%d:%d)\n", + event->fork.pid, event->fork.tid, + event->fork.ppid, event->fork.ptid); +} + +int perf_event__process_fork(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_fork_event(machine, event, sample); +} + +int perf_event__process_exit(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_exit_event(machine, event, sample); +} + +int perf_event__exit_del_thread(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample __maybe_unused, + struct machine *machine) +{ + struct thread *thread = machine__findnew_thread(machine, + event->fork.pid, + event->fork.tid); + + dump_printf("(%d:%d):(%d:%d)\n", event->fork.pid, event->fork.tid, + event->fork.ppid, event->fork.ptid); + + if (thread) { + machine__remove_thread(machine, thread); + thread__put(thread); + } + + return 0; +} + +size_t perf_event__fprintf_aux(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " offset: %#"PRI_lx64" size: %#"PRI_lx64" flags: %#"PRI_lx64" [%s%s%s%s]\n", + event->aux.aux_offset, event->aux.aux_size, + event->aux.flags, + event->aux.flags & PERF_AUX_FLAG_TRUNCATED ? "T" : "", + event->aux.flags & PERF_AUX_FLAG_OVERWRITE ? "O" : "", + event->aux.flags & PERF_AUX_FLAG_PARTIAL ? "P" : "", + event->aux.flags & PERF_AUX_FLAG_COLLISION ? "C" : ""); +} + +size_t perf_event__fprintf_itrace_start(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " pid: %u tid: %u\n", + event->itrace_start.pid, event->itrace_start.tid); +} + +size_t perf_event__fprintf_aux_output_hw_id(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " hw_id: %#"PRI_lx64"\n", + event->aux_output_hw_id.hw_id); +} + +size_t perf_event__fprintf_switch(union perf_event *event, FILE *fp) +{ + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + const char *in_out = !out ? "IN " : + !(event->header.misc & PERF_RECORD_MISC_SWITCH_OUT_PREEMPT) ? + "OUT " : "OUT preempt"; + + if (event->header.type == PERF_RECORD_SWITCH) + return fprintf(fp, " %s\n", in_out); + + return fprintf(fp, " %s %s pid/tid: %5d/%-5d\n", + in_out, out ? "next" : "prev", + event->context_switch.next_prev_pid, + event->context_switch.next_prev_tid); +} + +static size_t perf_event__fprintf_lost(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " lost %" PRI_lu64 "\n", event->lost.lost); +} + +size_t perf_event__fprintf_ksymbol(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " addr %" PRI_lx64 " len %u type %u flags 0x%x name %s\n", + event->ksymbol.addr, event->ksymbol.len, + event->ksymbol.ksym_type, + event->ksymbol.flags, event->ksymbol.name); +} + +size_t perf_event__fprintf_bpf(union perf_event *event, FILE *fp) +{ + return fprintf(fp, " type %u, flags %u, id %u\n", + event->bpf.type, event->bpf.flags, event->bpf.id); +} + +size_t perf_event__fprintf_bpf_metadata(union perf_event *event, FILE *fp) +{ + struct perf_record_bpf_metadata *metadata = &event->bpf_metadata; + size_t ret; + + ret = fprintf(fp, " prog %s\n", metadata->prog_name); + for (__u32 i = 0; i < metadata->nr_entries; i++) { + ret += fprintf(fp, " entry %d: %20s = %s\n", i, + metadata->entries[i].key, + metadata->entries[i].value); + } + return ret; +} + +static int text_poke_printer(enum binary_printer_ops op, unsigned int val, + void *extra, FILE *fp) +{ + bool old = *(bool *)extra; + + switch ((int)op) { + case BINARY_PRINT_LINE_BEGIN: + return fprintf(fp, " %s bytes:", old ? "Old" : "New"); + case BINARY_PRINT_NUM_DATA: + return fprintf(fp, " %02x", val); + case BINARY_PRINT_LINE_END: + return fprintf(fp, "\n"); + default: + return 0; + } +} + +size_t perf_event__fprintf_text_poke(union perf_event *event, struct machine *machine, FILE *fp) +{ + struct perf_record_text_poke_event *tp = &event->text_poke; + size_t ret; + bool old; + + ret = fprintf(fp, " %" PRI_lx64 " ", tp->addr); + if (machine) { + struct addr_location al; + + addr_location__init(&al); + al.map = maps__find(machine__kernel_maps(machine), tp->addr); + if (al.map && map__load(al.map) >= 0) { + al.addr = map__map_ip(al.map, tp->addr); + al.sym = map__find_symbol(al.map, al.addr); + if (al.sym) + ret += symbol__fprintf_symname_offs(al.sym, &al, fp); + } + addr_location__exit(&al); + } + ret += fprintf(fp, " old len %u new len %u\n", tp->old_len, tp->new_len); + old = true; + ret += binary__fprintf(tp->bytes, tp->old_len, 16, text_poke_printer, + &old, fp); + old = false; + ret += binary__fprintf(tp->bytes + tp->old_len, tp->new_len, 16, + text_poke_printer, &old, fp); + return ret; +} + +size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp) +{ + struct perf_record_schedstat_cpu *cs = &event->schedstat_cpu; + size_t size = fprintf(fp, "\ncpu%u ", cs->cpu); + __u16 version = cs->version; + +#define CPU_FIELD(_type, _name, _desc, _format, _is_pct, _pct_of, _ver) \ + size += fprintf(fp, "%" PRIu64 " ", (uint64_t)cs->_ver._name) + + if (version == 15) { +#include + return size; + } else if (version == 16) { +#include + return size; + } else if (version == 17) { +#include + return size; + } +#undef CPU_FIELD + + return fprintf(fp, "Unsupported /proc/schedstat version %d.\n", + event->schedstat_cpu.version); +} + +size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp) +{ + struct perf_record_schedstat_domain *ds = &event->schedstat_domain; + __u16 version = ds->version; + size_t size = fprintf(fp, "\ndomain%u ", ds->domain); + +#define DOMAIN_FIELD(_type, _name, _desc, _format, _is_jiffies, _ver) \ + size += fprintf(fp, "%" PRIu64 " ", (uint64_t)ds->_ver._name) + + if (version == 15) { +#include + return size; + } else if (version == 16) { +#include + return size; + } else if (version == 17) { +#include + return size; + } +#undef DOMAIN_FIELD + + return fprintf(fp, "Unsupported /proc/schedstat version %d.\n", + event->schedstat_domain.version); +} + +size_t perf_event__fprintf(union perf_event *event, struct machine *machine, FILE *fp) +{ + size_t ret = fprintf(fp, "PERF_RECORD_%s", + perf_event__name(event->header.type)); + + switch (event->header.type) { + case PERF_RECORD_COMM: + ret += perf_event__fprintf_comm(event, fp); + break; + case PERF_RECORD_FORK: + case PERF_RECORD_EXIT: + ret += perf_event__fprintf_task(event, fp); + break; + case PERF_RECORD_MMAP: + ret += perf_event__fprintf_mmap(event, fp); + break; + case PERF_RECORD_NAMESPACES: + ret += perf_event__fprintf_namespaces(event, fp); + break; + case PERF_RECORD_CGROUP: + ret += perf_event__fprintf_cgroup(event, fp); + break; + case PERF_RECORD_MMAP2: + ret += perf_event__fprintf_mmap2(event, fp); + break; + case PERF_RECORD_AUX: + ret += perf_event__fprintf_aux(event, fp); + break; + case PERF_RECORD_ITRACE_START: + ret += perf_event__fprintf_itrace_start(event, fp); + break; + case PERF_RECORD_SWITCH: + case PERF_RECORD_SWITCH_CPU_WIDE: + ret += perf_event__fprintf_switch(event, fp); + break; + case PERF_RECORD_LOST: + ret += perf_event__fprintf_lost(event, fp); + break; + case PERF_RECORD_KSYMBOL: + ret += perf_event__fprintf_ksymbol(event, fp); + break; + case PERF_RECORD_BPF_EVENT: + ret += perf_event__fprintf_bpf(event, fp); + break; + case PERF_RECORD_TEXT_POKE: + ret += perf_event__fprintf_text_poke(event, machine, fp); + break; + case PERF_RECORD_AUX_OUTPUT_HW_ID: + ret += perf_event__fprintf_aux_output_hw_id(event, fp); + break; + case PERF_RECORD_BPF_METADATA: + ret += perf_event__fprintf_bpf_metadata(event, fp); + break; + default: + ret += fprintf(fp, "\n"); + } + + return ret; +} + +int perf_event__process(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + return machine__process_event(machine, event, sample); +} + +struct map *thread__find_map(struct thread *thread, u8 cpumode, u64 addr, + struct addr_location *al) +{ + struct maps *maps = thread__maps(thread); + struct machine *machine = maps__machine(maps); + bool load_map = false; + + map__zput(al->map); + thread__zput(al->thread); + al->thread = thread__get(thread); + + al->addr = addr; + al->cpumode = cpumode; + al->filtered = 0; + + if (machine == NULL) + return NULL; + + if (cpumode == PERF_RECORD_MISC_KERNEL && perf_host) { + al->level = 'k'; + maps = machine__kernel_maps(machine); + load_map = !symbol_conf.lazy_load_kernel_maps; + } else if (cpumode == PERF_RECORD_MISC_USER && perf_host) { + al->level = '.'; + } else if (cpumode == PERF_RECORD_MISC_GUEST_KERNEL && perf_guest) { + al->level = 'g'; + maps = machine__kernel_maps(machine); + load_map = !symbol_conf.lazy_load_kernel_maps; + } else if (cpumode == PERF_RECORD_MISC_GUEST_USER && perf_guest) { + al->level = 'u'; + } else { + al->level = 'H'; + + if ((cpumode == PERF_RECORD_MISC_GUEST_USER || + cpumode == PERF_RECORD_MISC_GUEST_KERNEL) && + !perf_guest) + al->filtered |= (1 << HIST_FILTER__GUEST); + if ((cpumode == PERF_RECORD_MISC_USER || + cpumode == PERF_RECORD_MISC_KERNEL) && + !perf_host) + al->filtered |= (1 << HIST_FILTER__HOST); + + return NULL; + } + al->map = maps__find(maps, al->addr); + if (al->map != NULL) { + /* + * Kernel maps might be changed when loading symbols so loading + * must be done prior to using kernel maps. + */ + if (load_map) + map__load(al->map); + al->addr = map__map_ip(al->map, al->addr); + } + + return al->map; +} + +/* + * For branch stacks or branch samples, the sample cpumode might not be correct + * because it applies only to the sample 'ip' and not necessary to 'addr' or + * branch stack addresses. If possible, use a fallback to deal with those cases. + */ +struct map *thread__find_map_fb(struct thread *thread, u8 cpumode, u64 addr, + struct addr_location *al) +{ + struct map *map = thread__find_map(thread, cpumode, addr, al); + struct machine *machine = maps__machine(thread__maps(thread)); + u8 addr_cpumode = machine__addr_cpumode(machine, cpumode, addr); + + if (map || addr_cpumode == cpumode) + return map; + + return thread__find_map(thread, addr_cpumode, addr, al); +} + +struct symbol *thread__find_symbol(struct thread *thread, u8 cpumode, + u64 addr, struct addr_location *al) +{ + al->sym = NULL; + if (thread__find_map(thread, cpumode, addr, al)) + al->sym = map__find_symbol(al->map, al->addr); + return al->sym; +} + +struct symbol *thread__find_symbol_fb(struct thread *thread, u8 cpumode, + u64 addr, struct addr_location *al) +{ + al->sym = NULL; + if (thread__find_map_fb(thread, cpumode, addr, al)) + al->sym = map__find_symbol(al->map, al->addr); + return al->sym; +} + +static bool check_address_range(struct intlist *addr_list, int addr_range, + unsigned long addr) +{ + struct int_node *pos; + + intlist__for_each_entry(pos, addr_list) { + if (addr >= pos->i && addr < pos->i + addr_range) + return true; + } + + return false; +} + +/* + * Callers need to drop the reference to al->thread, obtained in + * machine__findnew_thread() + */ +int machine__resolve(struct machine *machine, struct addr_location *al, + struct perf_sample *sample) +{ + struct thread *thread; + struct dso *dso; + + if (symbol_conf.guest_code && !machine__is_host(machine)) + thread = machine__findnew_guest_code(machine, sample->pid); + else + thread = machine__findnew_thread(machine, sample->pid, sample->tid); + if (thread == NULL) + return -1; + + dump_printf(" ... thread: %s:%d\n", thread__comm_str(thread), thread__tid(thread)); + thread__find_map(thread, sample->cpumode, sample->ip, al); + dso = al->map ? map__dso(al->map) : NULL; + dump_printf(" ...... dso: %s\n", + dso + ? dso__long_name(dso) + : (al->level == 'H' ? "[hypervisor]" : "")); + + if (thread__is_filtered(thread)) + al->filtered |= (1 << HIST_FILTER__THREAD); + + thread__put(thread); + thread = NULL; + + al->sym = NULL; + al->cpu = sample->cpu; + al->socket = -1; + al->srcline = NULL; + + if (al->cpu >= 0) { + struct perf_env *env = machine->env; + + /* + * Bounds-check al->cpu (s32) before casting to struct perf_cpu + * (int16_t): without this, e.g. 65536 truncates to 0 and silently + * returns CPU 0's topology. Can go once perf_cpu.cpu is widened. + */ + if (env && al->cpu < env->nr_cpus_avail) { + struct cpu_topology_map *topo; + + topo = perf_env__get_cpu_topology(env, (struct perf_cpu){ al->cpu }); + if (topo) + al->socket = topo->socket_id; + } + } + + /* Account for possible out-of-order switch events. */ + al->parallelism = max(1, min(machine->parallelism, machine__nr_cpus_avail(machine))); + if (test_bit(al->parallelism, symbol_conf.parallelism_filter)) + al->filtered |= (1 << HIST_FILTER__PARALLELISM); + /* + * Multiply it by some const to avoid precision loss or dealing + * with floats. The multiplier does not matter otherwise since + * we only print it as percents. + */ + al->latency = sample->period * 1000 / al->parallelism; + + if (al->map) { + if (symbol_conf.dso_list && + (!dso || !(strlist__has_entry(symbol_conf.dso_list, + dso__short_name(dso)) || + (dso__short_name(dso) != dso__long_name(dso) && + strlist__has_entry(symbol_conf.dso_list, + dso__long_name(dso)))))) { + al->filtered |= (1 << HIST_FILTER__DSO); + } + + al->sym = map__find_symbol(al->map, al->addr); + } else if (symbol_conf.dso_list) { + al->filtered |= (1 << HIST_FILTER__DSO); + } + + if (symbol_conf.sym_list) { + int ret = 0; + char al_addr_str[32]; + size_t sz = sizeof(al_addr_str); + + if (al->sym) { + ret = strlist__has_entry(symbol_conf.sym_list, + al->sym->name); + } + if (!ret && al->sym) { + snprintf(al_addr_str, sz, "0x%"PRIx64, + map__unmap_ip(al->map, al->sym->start)); + ret = strlist__has_entry(symbol_conf.sym_list, + al_addr_str); + } + if (!ret && symbol_conf.addr_list && al->map) { + unsigned long addr = map__unmap_ip(al->map, al->addr); + + ret = intlist__has_entry(symbol_conf.addr_list, addr); + if (!ret && symbol_conf.addr_range) { + ret = check_address_range(symbol_conf.addr_list, + symbol_conf.addr_range, + addr); + } + } + + if (!ret) + al->filtered |= (1 << HIST_FILTER__SYMBOL); + } + + return 0; +} + +bool is_bts_event(struct perf_event_attr *attr) +{ + return attr->type == PERF_TYPE_HARDWARE && + (attr->config & PERF_COUNT_HW_BRANCH_INSTRUCTIONS) && + attr->sample_period == 1; +} + +bool sample_addr_correlates_sym(struct perf_event_attr *attr) +{ + if (attr->type == PERF_TYPE_SOFTWARE && + (attr->config == PERF_COUNT_SW_PAGE_FAULTS || + attr->config == PERF_COUNT_SW_PAGE_FAULTS_MIN || + attr->config == PERF_COUNT_SW_PAGE_FAULTS_MAJ)) + return true; + + if (is_bts_event(attr)) + return true; + + return false; +} + +void thread__resolve(struct thread *thread, struct addr_location *al, + struct perf_sample *sample) +{ + thread__find_map_fb(thread, sample->cpumode, sample->addr, al); + + al->cpu = sample->cpu; + al->sym = NULL; + + if (al->map) + al->sym = map__find_symbol(al->map, al->addr); +} diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h new file mode 100644 index 000000000..2ea83fdf8 --- /dev/null +++ b/tools/perf/util/event.h @@ -0,0 +1,437 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_RECORD_H +#define __PERF_RECORD_H +/* + * The linux/stddef.h isn't need here, but is needed for __always_inline used + * in files included from uapi/linux/perf_event.h such as + * /usr/include/linux/swab.h and /usr/include/linux/byteorder/little_endian.h, + * detected in at least musl libc, used in Alpine Linux. -acme + */ +#include +#include +#include +#include + +struct dso; +struct machine; +struct perf_event_attr; +struct perf_sample; + +#ifdef __LP64__ +/* + * /usr/include/inttypes.h uses just 'lu' for PRIu64, but we end up defining + * __u64 as long long unsigned int, and then -Werror=format= kicks in and + * complains of the mismatched types, so use these two special extra PRI + * macros to overcome that. + */ +#define PRI_lu64 "l" PRIu64 +#define PRI_lx64 "l" PRIx64 +#define PRI_ld64 "l" PRId64 +#else +#define PRI_lu64 PRIu64 +#define PRI_lx64 PRIx64 +#define PRI_ld64 PRId64 +#endif + +#define PERF_SAMPLE_MASK \ + (PERF_SAMPLE_IP | PERF_SAMPLE_TID | \ + PERF_SAMPLE_TIME | PERF_SAMPLE_ADDR | \ + PERF_SAMPLE_ID | PERF_SAMPLE_STREAM_ID | \ + PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD | \ + PERF_SAMPLE_IDENTIFIER) + +/* perf sample has 16 bits size limit */ +#define PERF_SAMPLE_MAX_SIZE (1 << 16) + +struct ip_callchain { + u64 nr; + u64 ips[]; +}; + +struct branch_stack; + +enum { + PERF_IP_FLAG_BRANCH = 1ULL << 0, + PERF_IP_FLAG_CALL = 1ULL << 1, + PERF_IP_FLAG_RETURN = 1ULL << 2, + PERF_IP_FLAG_CONDITIONAL = 1ULL << 3, + PERF_IP_FLAG_SYSCALLRET = 1ULL << 4, + PERF_IP_FLAG_ASYNC = 1ULL << 5, + PERF_IP_FLAG_INTERRUPT = 1ULL << 6, + PERF_IP_FLAG_TX_ABORT = 1ULL << 7, + PERF_IP_FLAG_TRACE_BEGIN = 1ULL << 8, + PERF_IP_FLAG_TRACE_END = 1ULL << 9, + PERF_IP_FLAG_IN_TX = 1ULL << 10, + PERF_IP_FLAG_VMENTRY = 1ULL << 11, + PERF_IP_FLAG_VMEXIT = 1ULL << 12, + PERF_IP_FLAG_INTR_DISABLE = 1ULL << 13, + PERF_IP_FLAG_INTR_TOGGLE = 1ULL << 14, + PERF_IP_FLAG_BRANCH_MISS = 1ULL << 15, + PERF_IP_FLAG_NOT_TAKEN = 1ULL << 16, +}; + +#define PERF_IP_FLAG_CHARS "bcrosyiABExghDtmn" + +#define PERF_ADDITIONAL_STATE_MASK \ + (PERF_IP_FLAG_IN_TX | \ + PERF_IP_FLAG_INTR_DISABLE | \ + PERF_IP_FLAG_INTR_TOGGLE) + +#define PERF_BRANCH_MASK (\ + PERF_IP_FLAG_BRANCH |\ + PERF_IP_FLAG_CALL |\ + PERF_IP_FLAG_RETURN |\ + PERF_IP_FLAG_CONDITIONAL |\ + PERF_IP_FLAG_SYSCALLRET |\ + PERF_IP_FLAG_ASYNC |\ + PERF_IP_FLAG_INTERRUPT |\ + PERF_IP_FLAG_TX_ABORT |\ + PERF_IP_FLAG_TRACE_BEGIN |\ + PERF_IP_FLAG_TRACE_END |\ + PERF_IP_FLAG_VMENTRY |\ + PERF_IP_FLAG_VMEXIT) + +#define PERF_IP_FLAG_BRANCH_EVENT_MASK \ + (PERF_IP_FLAG_BRANCH_MISS | \ + PERF_IP_FLAG_NOT_TAKEN) + +#define PERF_MEM_DATA_SRC_NONE \ + (PERF_MEM_S(OP, NA) |\ + PERF_MEM_S(LVL, NA) |\ + PERF_MEM_S(SNOOP, NA) |\ + PERF_MEM_S(LOCK, NA) |\ + PERF_MEM_S(TLB, NA) |\ + PERF_MEM_S(LVLNUM, NA)) + +/* Attribute type for custom synthesized events */ +#define PERF_TYPE_SYNTH (INT_MAX + 1U) + +/* Attribute config for custom synthesized events */ +enum perf_synth_id { + PERF_SYNTH_INTEL_PTWRITE, + PERF_SYNTH_INTEL_MWAIT, + PERF_SYNTH_INTEL_PWRE, + PERF_SYNTH_INTEL_EXSTOP, + PERF_SYNTH_INTEL_PWRX, + PERF_SYNTH_INTEL_CBR, + PERF_SYNTH_INTEL_PSB, + PERF_SYNTH_INTEL_EVT, + PERF_SYNTH_INTEL_IFLAG_CHG, + PERF_SYNTH_POWERPC_VPA_DTL, +}; + +/* + * Raw data formats for synthesized events. Note that 4 bytes of padding are + * present to match the 'size' member of PERF_SAMPLE_RAW data which is always + * 8-byte aligned. That means we must dereference raw_data with an offset of 4. + * Refer perf_sample__synth_ptr() and perf_synth__raw_data(). It also means the + * structure sizes are 4 bytes bigger than the raw_size, refer + * perf_synth__raw_size(). + */ + +struct perf_synth_intel_ptwrite { + u32 padding; + union { + struct { + u32 ip : 1, + reserved : 31; + }; + u32 flags; + }; + u64 payload; +}; + +struct perf_synth_intel_mwait { + u32 padding; + u32 reserved; + union { + struct { + u64 hints : 8, + reserved1 : 24, + extensions : 2, + reserved2 : 30; + }; + u64 payload; + }; +}; + +struct perf_synth_intel_pwre { + u32 padding; + u32 reserved; + union { + struct { + u64 reserved1 : 7, + hw : 1, + subcstate : 4, + cstate : 4, + reserved2 : 48; + }; + u64 payload; + }; +}; + +struct perf_synth_intel_exstop { + u32 padding; + union { + struct { + u32 ip : 1, + reserved : 31; + }; + u32 flags; + }; +}; + +struct perf_synth_intel_pwrx { + u32 padding; + u32 reserved; + union { + struct { + u64 deepest_cstate : 4, + last_cstate : 4, + wake_reason : 4, + reserved1 : 52; + }; + u64 payload; + }; +}; + +struct perf_synth_intel_cbr { + u32 padding; + union { + struct { + u32 cbr : 8, + reserved1 : 8, + max_nonturbo : 8, + reserved2 : 8; + }; + u32 flags; + }; + u32 freq; + u32 reserved3; +}; + +struct perf_synth_intel_psb { + u32 padding; + u32 reserved; + u64 offset; +}; + +struct perf_synth_intel_evd { + union { + struct { + u8 evd_type; + u8 reserved[7]; + }; + u64 et; + }; + u64 payload; +}; + +/* Intel PT Event Trace */ +struct perf_synth_intel_evt { + u32 padding; + union { + struct { + u32 type : 5, + reserved : 2, + ip : 1, + vector : 8, + evd_cnt : 16; + }; + u32 cfe; + }; + struct perf_synth_intel_evd evd[0]; +}; + +struct perf_synth_intel_iflag_chg { + u32 padding; + union { + struct { + u32 iflag : 1, + via_branch : 1; + }; + u32 flags; + }; + u64 branch_ip; /* If via_branch */ +}; + +/* + * The powerpc VPA DTL entries are of below format + */ +struct powerpc_vpadtl_entry { + u8 dispatch_reason; + u8 preempt_reason; + u16 processor_id; + u32 enqueue_to_dispatch_time; + u32 ready_to_enqueue_time; + u32 waiting_to_ready_time; + u64 timebase; + u64 fault_addr; + u64 srr0; + u64 srr1; +}; + +extern const char *dispatch_reasons[11]; +extern const char *preempt_reasons[10]; + +static inline void *perf_synth__raw_data(void *p) +{ + return p + 4; +} + +#define perf_synth__raw_size(d) (sizeof(d) - 4) + +#define perf_sample__bad_synth_size(s, d) ((s)->raw_size < sizeof(d) - 4) + +enum { + PERF_STAT_ROUND_TYPE__INTERVAL = 0, + PERF_STAT_ROUND_TYPE__FINAL = 1, +}; + +void perf_event__print_totals(void); + +struct perf_cpu_map; +struct perf_record_stat_config; +struct perf_stat_config; +struct perf_tool; + +void perf_event__read_stat_config(struct perf_stat_config *config, + struct perf_record_stat_config *event); + +int perf_event__process_comm(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_lost(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_lost_samples(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_aux(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_itrace_start(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_aux_output_hw_id(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_switch(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_namespaces(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_cgroup(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_mmap(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_mmap2(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_fork(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_exit(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__exit_del_thread(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_ksymbol(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_bpf(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process_text_poke(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); +int perf_event__process(const struct perf_tool *tool, + union perf_event *event, + struct perf_sample *sample, + struct machine *machine); + +bool is_bts_event(struct perf_event_attr *attr); +bool sample_addr_correlates_sym(struct perf_event_attr *attr); + +const char *perf_event__name(unsigned int id); + +size_t perf_event__fprintf_comm(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_mmap(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_mmap2(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_task(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_aux(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_itrace_start(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_aux_output_hw_id(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_switch(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_thread_map(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_cpu_map(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_namespaces(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_cgroup(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_ksymbol(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_bpf(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_bpf_metadata(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_text_poke(union perf_event *event, struct machine *machine,FILE *fp); +size_t perf_event__fprintf_schedstat_cpu(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_schedstat_domain(union perf_event *event, FILE *fp); +size_t perf_event__fprintf(union perf_event *event, struct machine *machine, FILE *fp); + +int kallsyms__get_function_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr); +int kallsyms__get_symbol_start(const char *kallsyms_filename, + const char *symbol_name, u64 *addr); + +void event_attr_init(struct perf_event_attr *attr); + +int perf_event_paranoid(void); +bool perf_event_paranoid_check(int max_level); + +extern int sysctl_perf_event_max_stack; +extern int sysctl_perf_event_max_contexts_per_stack; +extern unsigned int proc_map_timeout; + +#define PAGE_SIZE_NAME_LEN 32 +char *get_page_size_name(u64 size, char *str); + +static inline bool perf_event_header__cpumode_is_guest(u8 cpumode) +{ + return cpumode == PERF_RECORD_MISC_GUEST_KERNEL || + cpumode == PERF_RECORD_MISC_GUEST_USER; +} + +static inline bool perf_event_header__misc_is_guest(u16 misc) +{ + return perf_event_header__cpumode_is_guest(misc & PERF_RECORD_MISC_CPUMODE_MASK); +} + +static inline bool perf_event_header__is_guest(const struct perf_event_header *header) +{ + return perf_event_header__misc_is_guest(header->misc); +} + +static inline bool perf_event__is_guest(const union perf_event *event) +{ + return perf_event_header__is_guest(&event->header); +} + +#endif /* __PERF_RECORD_H */ diff --git a/tools/perf/util/events_stats.h b/tools/perf/util/events_stats.h new file mode 100644 index 000000000..dcff697ed --- /dev/null +++ b/tools/perf/util/events_stats.h @@ -0,0 +1,72 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_EVENTS_STATS_ +#define __PERF_EVENTS_STATS_ + +#include +#include +#include +#include "auxtrace.h" + +/* + * The kernel collects the number of events it couldn't send in a stretch and + * when possible sends this number in a PERF_RECORD_LOST event. The number of + * such "chunks" of lost events is stored in .nr_events[PERF_EVENT_LOST] while + * total_lost tells exactly how many events the kernel in fact lost, i.e. it is + * the sum of all struct perf_record_lost.lost fields reported. + * + * The kernel discards mixed up samples and sends the number in a + * PERF_RECORD_LOST_SAMPLES event. The number of lost-samples events is stored + * in .nr_events[PERF_RECORD_LOST_SAMPLES] while total_lost_samples tells + * exactly how many samples the kernel in fact dropped, i.e. it is the sum of + * all struct perf_record_lost_samples.lost fields reported without setting the + * misc field in the header. + * + * The BPF program can discard samples according to the filter expressions given + * by the user. This number is kept in a BPF map and dumped at the end of perf + * record in a PERF_RECORD_LOST_SAMPLES event. To differentiate it from other + * lost samples, perf tools sets PERF_RECORD_MISC_LOST_SAMPLES_BPF flag in the + * header.misc field. The number of dropped-samples events is stored in + * .nr_events[PERF_RECORD_LOST_SAMPLES] while total_dropped_samples tells + * exactly how many samples the BPF program in fact dropped, i.e. it is the sum + * of all struct perf_record_lost_samples.lost fields reported with the misc + * field set in the header. + * + * The total_period is needed because by default auto-freq is used, so + * multiplying nr_events[PERF_EVENT_SAMPLE] by a frequency isn't possible to get + * the total number of low level events, it is necessary to sum all struct + * perf_record_sample.period and stash the result in total_period. + */ +struct events_stats { + u64 total_lost; + u64 total_lost_samples; + u64 total_dropped_samples; + u64 total_aux_lost; + u64 total_aux_partial; + u64 total_aux_collision; + u64 total_invalid_chains; + u32 nr_events[PERF_RECORD_HEADER_MAX]; + u32 nr_lost_warned; + u32 nr_unknown_events; + u32 nr_invalid_chains; + u32 nr_unknown_id; + u32 nr_unprocessable_samples; + u32 nr_auxtrace_errors[PERF_AUXTRACE_ERROR_MAX]; + u32 nr_proc_map_timeout; +}; + +struct hists_stats { + u64 total_period; + u64 total_non_filtered_period; + u64 total_latency; + u64 total_non_filtered_latency; + u32 nr_samples; + u32 nr_non_filtered_samples; + u32 nr_lost_samples; + u32 nr_dropped_samples; +}; + +void events_stats__inc(struct events_stats *stats, u32 type); + +size_t events_stats__fprintf(struct events_stats *stats, FILE *fp); + +#endif /* __PERF_EVENTS_STATS_ */ diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c new file mode 100644 index 000000000..1a238b245 --- /dev/null +++ b/tools/perf/util/evlist.c @@ -0,0 +1,2701 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo + * + * Parts came from builtin-{top,stat,record}.c, see those files for further + * copyright notes. + */ +#include +#include +#include +#include +#include "cpumap.h" +#include "util/mmap.h" +#include "thread_map.h" +#include "target.h" +#include "dwarf-regs.h" +#include "evlist.h" +#include "evsel.h" +#include "record.h" +#include "debug.h" +#include "units.h" +#include "bpf_counter.h" +#include // page_size +#include "affinity.h" +#include "../perf.h" +#include "asm/bug.h" +#include "bpf-event.h" +#include "util/event.h" +#include "util/string2.h" +#include "util/perf_api_probe.h" +#include "util/evsel_fprintf.h" +#include "util/pmu.h" +#include "util/sample.h" +#include "util/bpf-filter.h" +#include "util/stat.h" +#include "util/util.h" +#include "util/env.h" +#include "util/intel-tpebs.h" +#include "util/metricgroup.h" +#include "util/strbuf.h" +#include +#include +#include +#include + +#include "parse-events.h" +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#ifdef LACKS_SIGQUEUE_PROTOTYPE +int sigqueue(pid_t pid, int sig, const union sigval value); +#endif + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) +#define SID(e, x, y) xyarray__entry(e->core.sample_id, x, y) + +void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + perf_evlist__init(&evlist->core); + perf_evlist__set_maps(&evlist->core, cpus, threads); + evlist->workload.pid = -1; + evlist->bkw_mmap_state = BKW_MMAP_NOTREADY; + evlist->ctl_fd.fd = -1; + evlist->ctl_fd.ack = -1; + evlist->ctl_fd.pos = -1; + evlist->nr_br_cntr = -1; + metricgroup__rblist_init(&evlist->metric_events); + INIT_LIST_HEAD(&evlist->deferred_samples); +} + +struct evlist *evlist__new(void) +{ + struct evlist *evlist = zalloc(sizeof(*evlist)); + + if (evlist != NULL) + evlist__init(evlist, NULL, NULL); + + return evlist; +} + +struct evlist *evlist__new_default(const struct target *target, bool sample_callchains) +{ + struct evlist *evlist = evlist__new(); + bool can_profile_kernel; + struct perf_pmu *pmu = NULL; + struct evsel *evsel; + char buf[256]; + int err; + + if (!evlist) + return NULL; + + can_profile_kernel = perf_event_paranoid_check(1); + + if (EM_HOST == EM_S390 && sample_callchains) { + snprintf(buf, sizeof(buf), "software/%s/%s", + target__has_cpu(target) ? "cpu-clock" : "task-clock", + can_profile_kernel ? "P" : "Pu"); + err = parse_event(evlist, buf); + if (err) + goto out_err; + } else { + while ((pmu = perf_pmus__scan_core(pmu)) != NULL) { + snprintf(buf, sizeof(buf), "%s/cycles/%s", pmu->name, + can_profile_kernel ? "P" : "Pu"); + err = parse_event(evlist, buf); + if (err) + goto out_err; + } + } + + /* If there is only 1 event a sample identifier isn't necessary. */ + if (evlist->core.nr_entries > 1) { + evlist__for_each_entry(evlist, evsel) + evsel__set_sample_id(evsel, /*can_sample_identifier=*/false); + } + + return evlist; +out_err: + evlist__delete(evlist); + return NULL; +} + +struct evlist *evlist__new_dummy(void) +{ + struct evlist *evlist = evlist__new(); + + if (evlist && evlist__add_dummy(evlist)) { + evlist__delete(evlist); + evlist = NULL; + } + + return evlist; +} + +/** + * evlist__set_id_pos - set the positions of event ids. + * @evlist: selected event list + * + * Events with compatible sample types all have the same id_pos + * and is_pos. For convenience, put a copy on evlist. + */ +void evlist__set_id_pos(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist); + + evlist->id_pos = first->id_pos; + evlist->is_pos = first->is_pos; +} + +static void evlist__update_id_pos(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) + evsel__calc_id_pos(evsel); + + evlist__set_id_pos(evlist); +} + +static void evlist__purge(struct evlist *evlist) +{ + struct evsel *pos, *n; + + evlist__for_each_entry_safe(evlist, n, pos) { + list_del_init(&pos->core.node); + pos->evlist = NULL; + evsel__delete(pos); + } + + evlist->core.nr_entries = 0; +} + +void evlist__exit(struct evlist *evlist) +{ + metricgroup__rblist_exit(&evlist->metric_events); + event_enable_timer__exit(&evlist->eet); + zfree(&evlist->mmap); + zfree(&evlist->overwrite_mmap); + perf_evlist__exit(&evlist->core); +} + +void evlist__delete(struct evlist *evlist) +{ + if (evlist == NULL) + return; + + evlist__free_stats(evlist); + evlist__munmap(evlist); + evlist__close(evlist); + evlist__purge(evlist); + evlist__exit(evlist); + free(evlist); +} + +void evlist__add(struct evlist *evlist, struct evsel *entry) +{ + perf_evlist__add(&evlist->core, &entry->core); + entry->evlist = evlist; + entry->tracking = !entry->core.idx; + + if (evlist->core.nr_entries == 1) + evlist__set_id_pos(evlist); +} + +void evlist__remove(struct evlist *evlist, struct evsel *evsel) +{ + evsel->evlist = NULL; + perf_evlist__remove(&evlist->core, &evsel->core); +} + +void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list) +{ + while (!list_empty(list)) { + struct evsel *evsel, *temp, *leader = NULL; + + __evlist__for_each_entry_safe(list, temp, evsel) { + list_del_init(&evsel->core.node); + evlist__add(evlist, evsel); + leader = evsel; + break; + } + + __evlist__for_each_entry_safe(list, temp, evsel) { + if (evsel__has_leader(evsel, leader)) { + list_del_init(&evsel->core.node); + evlist__add(evlist, evsel); + } + } + } +} + +int __evlist__set_tracepoints_handlers(struct evlist *evlist, + const struct evsel_str_handler *assocs, size_t nr_assocs) +{ + size_t i; + int err; + + for (i = 0; i < nr_assocs; i++) { + // Adding a handler for an event not in this evlist, just ignore it. + struct evsel *evsel = evlist__find_tracepoint_by_name(evlist, assocs[i].name); + if (evsel == NULL) + continue; + + err = -EEXIST; + if (evsel->handler != NULL) + goto out; + evsel->handler = assocs[i].handler; + } + + err = 0; +out: + return err; +} + +static void evlist__set_leader(struct evlist *evlist) +{ + perf_evlist__set_leader(&evlist->core); +} + +static struct evsel *evlist__dummy_event(struct evlist *evlist) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_DUMMY, + .size = sizeof(attr), /* to capture ABI version */ + /* Avoid frequency mode for dummy events to avoid associated timers. */ + .freq = 0, + .sample_period = 1, + }; + + return evsel__new_idx(&attr, evlist->core.nr_entries); +} + +int evlist__add_dummy(struct evlist *evlist) +{ + struct evsel *evsel = evlist__dummy_event(evlist); + + if (evsel == NULL) + return -ENOMEM; + + evlist__add(evlist, evsel); + return 0; +} + +struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel = evlist__dummy_event(evlist); + + if (!evsel) + return NULL; + + evsel->core.attr.exclude_kernel = 1; + evsel->core.attr.exclude_guest = 1; + evsel->core.attr.exclude_hv = 1; + evsel->core.system_wide = system_wide; + evsel->no_aux_samples = true; + evsel->name = strdup("dummy:u"); + + evlist__add(evlist, evsel); + return evsel; +} + +#ifdef HAVE_LIBTRACEEVENT +struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel = evsel__newtp_idx("sched", "sched_switch", 0, + /*format=*/true); + + if (IS_ERR(evsel)) + return evsel; + + evsel__set_sample_bit(evsel, CPU); + evsel__set_sample_bit(evsel, TIME); + + evsel->core.system_wide = system_wide; + evsel->no_aux_samples = true; + + evlist__add(evlist, evsel); + return evsel; +} +#endif + +struct evsel *evlist__find_tracepoint_by_name(struct evlist *evlist, const char *name) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if ((evsel->core.attr.type == PERF_TYPE_TRACEPOINT) && + (strcmp(evsel->name, name) == 0)) + return evsel; + } + + return NULL; +} + +#ifdef HAVE_LIBTRACEEVENT +int evlist__add_newtp(struct evlist *evlist, const char *sys, const char *name, void *handler) +{ + struct evsel *evsel = evsel__newtp(sys, name); + + if (IS_ERR(evsel)) + return -1; + + evsel->handler = handler; + evlist__add(evlist, evsel); + return 0; +} +#endif + +/* + * Should sched_setaffinity be used with evlist__for_each_cpu? Determine if + * migrating the thread will avoid possibly numerous IPIs. + */ +static bool evlist__use_affinity(struct evlist *evlist) +{ + struct evsel *pos; + struct perf_cpu_map *used_cpus = NULL; + bool ret = false; + + if (evlist->no_affinity || !evlist->core.user_requested_cpus || + cpu_map__is_dummy(evlist->core.user_requested_cpus)) + return false; + + evlist__for_each_entry(evlist, pos) { + struct perf_cpu_map *intersect; + + if (!perf_pmu__benefits_from_affinity(pos->pmu)) + continue; + + if (evsel__is_dummy_event(pos)) { + /* + * The dummy event is opened on all CPUs so assume >1 + * event with shared CPUs. + */ + ret = true; + break; + } + if (evsel__is_retire_lat(pos)) { + /* + * Retirement latency events are similar to tool ones in + * their implementation, and so don't require affinity. + */ + continue; + } + if (perf_cpu_map__is_empty(used_cpus)) { + /* First benefitting event, we want >1 on a common CPU. */ + used_cpus = perf_cpu_map__get(pos->core.cpus); + continue; + } + if ((pos->core.attr.read_format & PERF_FORMAT_GROUP) && + evsel__leader(pos) != pos) { + /* Skip members of the same sample group. */ + continue; + } + intersect = perf_cpu_map__intersect(used_cpus, pos->core.cpus); + if (!perf_cpu_map__is_empty(intersect)) { + /* >1 event with shared CPUs. */ + perf_cpu_map__put(intersect); + ret = true; + break; + } + perf_cpu_map__put(intersect); + perf_cpu_map__merge(&used_cpus, pos->core.cpus); + } + perf_cpu_map__put(used_cpus); + return ret; +} + +void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *evlist) +{ + *itr = (struct evlist_cpu_iterator){ + .container = evlist, + .evsel = NULL, + .cpu_map_idx = 0, + .evlist_cpu_map_idx = 0, + .evlist_cpu_map_nr = perf_cpu_map__nr(evlist->core.all_cpus), + .cpu = (struct perf_cpu){ .cpu = -1}, + .affinity = NULL, + }; + + if (evlist__empty(evlist)) { + /* Ensure the empty list doesn't iterate. */ + itr->evlist_cpu_map_idx = itr->evlist_cpu_map_nr; + return; + } + + if (evlist__use_affinity(evlist)) { + if (affinity__setup(&itr->saved_affinity) == 0) + itr->affinity = &itr->saved_affinity; + } + itr->evsel = evlist__first(evlist); + itr->cpu = perf_cpu_map__cpu(evlist->core.all_cpus, 0); + if (itr->affinity) + affinity__set(itr->affinity, itr->cpu.cpu); + itr->cpu_map_idx = perf_cpu_map__idx(itr->evsel->core.cpus, itr->cpu); + /* + * If this CPU isn't in the evsel's cpu map then advance + * through the list. + */ + if (itr->cpu_map_idx == -1) + evlist_cpu_iterator__next(itr); +} + +void evlist_cpu_iterator__exit(struct evlist_cpu_iterator *itr) +{ + if (!itr->affinity) + return; + + affinity__cleanup(itr->affinity); + itr->affinity = NULL; +} + +void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr) +{ + while (evlist_cpu_itr->evsel != evlist__last(evlist_cpu_itr->container)) { + evlist_cpu_itr->evsel = evsel__next(evlist_cpu_itr->evsel); + evlist_cpu_itr->cpu_map_idx = + perf_cpu_map__idx(evlist_cpu_itr->evsel->core.cpus, + evlist_cpu_itr->cpu); + if (evlist_cpu_itr->cpu_map_idx != -1) + return; + } + evlist_cpu_itr->evlist_cpu_map_idx++; + if (evlist_cpu_itr->evlist_cpu_map_idx < evlist_cpu_itr->evlist_cpu_map_nr) { + evlist_cpu_itr->evsel = evlist__first(evlist_cpu_itr->container); + evlist_cpu_itr->cpu = + perf_cpu_map__cpu(evlist_cpu_itr->container->core.all_cpus, + evlist_cpu_itr->evlist_cpu_map_idx); + if (evlist_cpu_itr->affinity) + affinity__set(evlist_cpu_itr->affinity, evlist_cpu_itr->cpu.cpu); + evlist_cpu_itr->cpu_map_idx = + perf_cpu_map__idx(evlist_cpu_itr->evsel->core.cpus, + evlist_cpu_itr->cpu); + /* + * If this CPU isn't in the evsel's cpu map then advance through + * the list. + */ + if (evlist_cpu_itr->cpu_map_idx == -1) + evlist_cpu_iterator__next(evlist_cpu_itr); + } else { + evlist_cpu_iterator__exit(evlist_cpu_itr); + } +} + +static int evsel__strcmp(struct evsel *pos, char *evsel_name) +{ + if (!evsel_name) + return 0; + if (evsel__is_dummy_event(pos)) + return 1; + return !evsel__name_is(pos, evsel_name); +} + +static int evlist__is_enabled(struct evlist *evlist) +{ + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) { + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + /* If at least one event is enabled, evlist is enabled. */ + if (!pos->disabled) + return true; + } + return false; +} + +static void __evlist__disable(struct evlist *evlist, char *evsel_name, bool excl_dummy) +{ + struct evsel *pos, *member; + struct evlist_cpu_iterator evlist_cpu_itr; + bool has_imm = false; + + /* Disable 'immediate' events last */ + for (int imm = 0; imm <= 1; imm++) { + evlist__for_each_cpu(evlist_cpu_itr, evlist) { + pos = evlist_cpu_itr.evsel; + if (evsel__strcmp(pos, evsel_name)) + continue; + if (pos->disabled || !evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + if (pos->immediate) + has_imm = true; + if (pos->immediate != imm) + continue; + evsel__disable_cpu(pos, evlist_cpu_itr.cpu_map_idx); + } + if (!has_imm) + break; + } + + evlist__for_each_entry(evlist, pos) { + if (evsel__strcmp(pos, evsel_name)) + continue; + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + pos->disabled = true; + + for_each_group_member(member, pos) + member->disabled = true; + } + + /* + * If we disabled only single event, we need to check + * the enabled state of the evlist manually. + */ + if (evsel_name) + evlist->enabled = evlist__is_enabled(evlist); + else + evlist->enabled = false; +} + +void evlist__disable(struct evlist *evlist) +{ + __evlist__disable(evlist, NULL, false); +} + +void evlist__disable_non_dummy(struct evlist *evlist) +{ + __evlist__disable(evlist, NULL, true); +} + +void evlist__disable_evsel(struct evlist *evlist, char *evsel_name) +{ + __evlist__disable(evlist, evsel_name, false); +} + +static void __evlist__enable(struct evlist *evlist, char *evsel_name, bool excl_dummy) +{ + struct evsel *pos, *member; + struct evlist_cpu_iterator evlist_cpu_itr; + + evlist__for_each_cpu(evlist_cpu_itr, evlist) { + pos = evlist_cpu_itr.evsel; + if (evsel__strcmp(pos, evsel_name)) + continue; + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + evsel__enable_cpu(pos, evlist_cpu_itr.cpu_map_idx); + } + evlist__for_each_entry(evlist, pos) { + if (evsel__strcmp(pos, evsel_name)) + continue; + if (!evsel__is_group_leader(pos) || !pos->core.fd) + continue; + if (excl_dummy && evsel__is_dummy_event(pos)) + continue; + pos->disabled = false; + + for_each_group_member(member, pos) + member->disabled = false; + } + + /* + * Even single event sets the 'enabled' for evlist, + * so the toggle can work properly and toggle to + * 'disabled' state. + */ + evlist->enabled = true; +} + +void evlist__enable(struct evlist *evlist) +{ + __evlist__enable(evlist, NULL, false); +} + +void evlist__enable_non_dummy(struct evlist *evlist) +{ + __evlist__enable(evlist, NULL, true); +} + +void evlist__enable_evsel(struct evlist *evlist, char *evsel_name) +{ + __evlist__enable(evlist, evsel_name, false); +} + +void evlist__toggle_enable(struct evlist *evlist) +{ + (evlist->enabled ? evlist__disable : evlist__enable)(evlist); +} + +int evlist__add_pollfd(struct evlist *evlist, int fd) +{ + return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, fdarray_flag__default); +} + +int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask) +{ + return perf_evlist__filter_pollfd(&evlist->core, revents_and_mask); +} + +#ifdef HAVE_EVENTFD_SUPPORT +int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd) +{ + return perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event); +} +#endif + +int evlist__poll(struct evlist *evlist, int timeout) +{ + return perf_evlist__poll(&evlist->core, timeout); +} + +struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id) +{ + struct hlist_head *head; + struct perf_sample_id *sid; + int hash; + + hash = hash_64(id, PERF_EVLIST__HLIST_BITS); + head = &evlist->core.heads[hash]; + + hlist_for_each_entry(sid, head, node) + if (sid->id == id) + return sid; + + return NULL; +} + +struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id) +{ + struct perf_sample_id *sid; + + if (evlist->core.nr_entries == 1 || !id) + return evlist__first(evlist); + + sid = evlist__id2sid(evlist, id); + if (sid) + return container_of(sid->evsel, struct evsel, core); + + if (!evlist__sample_id_all(evlist)) + return evlist__first(evlist); + + return NULL; +} + +struct evsel *evlist__id2evsel_strict(struct evlist *evlist, u64 id) +{ + struct perf_sample_id *sid; + + if (!id) + return NULL; + + sid = evlist__id2sid(evlist, id); + if (sid) + return container_of(sid->evsel, struct evsel, core); + + return NULL; +} + +static int evlist__event2id(struct evlist *evlist, union perf_event *event, u64 *id) +{ + const __u64 *array = event->sample.array; + ssize_t n; + + n = (event->header.size - sizeof(event->header)) >> 3; + + if (event->header.type == PERF_RECORD_SAMPLE) { + if (evlist->id_pos >= n) + return -1; + *id = array[evlist->id_pos]; + } else { + if (evlist->is_pos > n) + return -1; + n -= evlist->is_pos; + *id = array[n]; + } + return 0; +} + +struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event) +{ + struct evsel *first = evlist__first(evlist); + struct hlist_head *head; + struct perf_sample_id *sid; + int hash; + u64 id; + + if (evlist->core.nr_entries == 1) + return first; + + if (!first->core.attr.sample_id_all && + event->header.type != PERF_RECORD_SAMPLE) + return first; + + if (evlist__event2id(evlist, event, &id)) + return NULL; + + /* Synthesized events have an id of zero */ + if (!id) + return first; + + hash = hash_64(id, PERF_EVLIST__HLIST_BITS); + head = &evlist->core.heads[hash]; + + hlist_for_each_entry(sid, head, node) { + if (sid->id == id) + return container_of(sid->evsel, struct evsel, core); + } + return NULL; +} + +static int evlist__set_paused(struct evlist *evlist, bool value) +{ + int i; + + if (!evlist->overwrite_mmap) + return 0; + + for (i = 0; i < evlist->core.nr_mmaps; i++) { + int fd = evlist->overwrite_mmap[i].core.fd; + int err; + + if (fd < 0) + continue; + err = ioctl(fd, PERF_EVENT_IOC_PAUSE_OUTPUT, value ? 1 : 0); + if (err) + return err; + } + return 0; +} + +static int evlist__pause(struct evlist *evlist) +{ + return evlist__set_paused(evlist, true); +} + +static int evlist__resume(struct evlist *evlist) +{ + return evlist__set_paused(evlist, false); +} + +static void evlist__munmap_nofree(struct evlist *evlist) +{ + int i; + + if (evlist->mmap) + for (i = 0; i < evlist->core.nr_mmaps; i++) + perf_mmap__munmap(&evlist->mmap[i].core); + + if (evlist->overwrite_mmap) + for (i = 0; i < evlist->core.nr_mmaps; i++) + perf_mmap__munmap(&evlist->overwrite_mmap[i].core); +} + +void evlist__munmap(struct evlist *evlist) +{ + evlist__munmap_nofree(evlist); + zfree(&evlist->mmap); + zfree(&evlist->overwrite_mmap); +} + +static void perf_mmap__unmap_cb(struct perf_mmap *map) +{ + struct mmap *m = container_of(map, struct mmap, core); + + mmap__munmap(m); +} + +static struct mmap *evlist__alloc_mmap(struct evlist *evlist, + bool overwrite) +{ + int i; + struct mmap *map = calloc(evlist->core.nr_mmaps, sizeof(struct mmap)); + + if (!map) + return NULL; + + for (i = 0; i < evlist->core.nr_mmaps; i++) { + struct perf_mmap *prev = i ? &map[i - 1].core : NULL; + + /* + * When the perf_mmap() call is made we grab one refcount, plus + * one extra to let perf_mmap__consume() get the last + * events after all real references (perf_mmap__get()) are + * dropped. + * + * Each PERF_EVENT_IOC_SET_OUTPUT points to this mmap and + * thus does perf_mmap__get() on it. + */ + perf_mmap__init(&map[i].core, prev, overwrite, perf_mmap__unmap_cb); + } + + return map; +} + +static void +perf_evlist__mmap_cb_idx(struct perf_evlist *_evlist, + struct perf_evsel *_evsel, + struct perf_mmap_param *_mp, + int idx) +{ + struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct mmap_params *mp = container_of(_mp, struct mmap_params, core); + struct evsel *evsel = container_of(_evsel, struct evsel, core); + + auxtrace_mmap_params__set_idx(&mp->auxtrace_mp, evlist, evsel, idx); +} + +static struct perf_mmap* +perf_evlist__mmap_cb_get(struct perf_evlist *_evlist, bool overwrite, int idx) +{ + struct evlist *evlist = container_of(_evlist, struct evlist, core); + struct mmap *maps; + + maps = overwrite ? evlist->overwrite_mmap : evlist->mmap; + + if (!maps) { + maps = evlist__alloc_mmap(evlist, overwrite); + if (!maps) + return NULL; + + if (overwrite) { + evlist->overwrite_mmap = maps; + if (evlist->bkw_mmap_state == BKW_MMAP_NOTREADY) + evlist__toggle_bkw_mmap(evlist, BKW_MMAP_RUNNING); + } else { + evlist->mmap = maps; + } + } + + return &maps[idx].core; +} + +static int +perf_evlist__mmap_cb_mmap(struct perf_mmap *_map, struct perf_mmap_param *_mp, + int output, struct perf_cpu cpu) +{ + struct mmap *map = container_of(_map, struct mmap, core); + struct mmap_params *mp = container_of(_mp, struct mmap_params, core); + + return mmap__mmap(map, mp, output, cpu); +} + +unsigned long perf_event_mlock_kb_in_pages(void) +{ + unsigned long pages; + int max; + + if (sysctl__read_int("kernel/perf_event_mlock_kb", &max) < 0) { + /* + * Pick a once upon a time good value, i.e. things look + * strange since we can't read a sysctl value, but lets not + * die yet... + */ + max = 512; + } else { + max -= (page_size / 1024); + } + + pages = (max * 1024) / page_size; + if (!is_power_of_2(pages)) + pages = rounddown_pow_of_two(pages); + + return pages; +} + +size_t evlist__mmap_size(unsigned long pages) +{ + if (pages == UINT_MAX) + pages = perf_event_mlock_kb_in_pages(); + else if (!is_power_of_2(pages)) + return 0; + + return (pages + 1) * page_size; +} + +static long parse_pages_arg(const char *str, unsigned long min, + unsigned long max) +{ + unsigned long pages, val; + static struct parse_tag tags[] = { + { .tag = 'B', .mult = 1 }, + { .tag = 'K', .mult = 1 << 10 }, + { .tag = 'M', .mult = 1 << 20 }, + { .tag = 'G', .mult = 1 << 30 }, + { .tag = 0 }, + }; + + if (str == NULL) + return -EINVAL; + + val = parse_tag_value(str, tags); + if (val != (unsigned long) -1) { + /* we got file size value */ + pages = PERF_ALIGN(val, page_size) / page_size; + } else { + /* we got pages count value */ + char *eptr; + pages = strtoul(str, &eptr, 10); + if (*eptr != '\0') + return -EINVAL; + } + + if (pages == 0 && min == 0) { + /* leave number of pages at 0 */ + } else if (!is_power_of_2(pages)) { + char buf[100]; + + /* round pages up to next power of 2 */ + pages = roundup_pow_of_two(pages); + if (!pages) + return -EINVAL; + + unit_number__scnprintf(buf, sizeof(buf), pages * page_size); + pr_info("rounding mmap pages size to %s (%lu pages)\n", + buf, pages); + } + + if (pages > max) + return -EINVAL; + + return pages; +} + +int __evlist__parse_mmap_pages(unsigned int *mmap_pages, const char *str) +{ + unsigned long max = UINT_MAX; + long pages; + + if (max > SIZE_MAX / page_size) + max = SIZE_MAX / page_size; + + pages = parse_pages_arg(str, 1, max); + if (pages < 0) { + pr_err("Invalid argument for --mmap_pages/-m\n"); + return -1; + } + + *mmap_pages = pages; + return 0; +} + +int evlist__parse_mmap_pages(const struct option *opt, const char *str, int unset __maybe_unused) +{ + return __evlist__parse_mmap_pages(opt->value, str); +} + +/** + * evlist__mmap_ex - Create mmaps to receive events. + * @evlist: list of events + * @pages: map length in pages + * @overwrite: overwrite older events? + * @auxtrace_pages - auxtrace map length in pages + * @auxtrace_overwrite - overwrite older auxtrace data? + * + * If @overwrite is %false the user needs to signal event consumption using + * perf_mmap__write_tail(). Using evlist__mmap_read() does this + * automatically. + * + * Similarly, if @auxtrace_overwrite is %false the user needs to signal data + * consumption using auxtrace_mmap__write_tail(). + * + * Return: %0 on success, negative error code otherwise. + */ +int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, + unsigned int auxtrace_pages, + bool auxtrace_overwrite, int nr_cblocks, int affinity, int flush, + int comp_level) +{ + /* + * Delay setting mp.prot: set it before calling perf_mmap__mmap. + * Its value is decided by evsel's write_backward. + * So &mp should not be passed through const pointer. + */ + struct mmap_params mp = { + .nr_cblocks = nr_cblocks, + .affinity = affinity, + .flush = flush, + .comp_level = comp_level + }; + struct perf_evlist_mmap_ops ops = { + .idx = perf_evlist__mmap_cb_idx, + .get = perf_evlist__mmap_cb_get, + .mmap = perf_evlist__mmap_cb_mmap, + }; + + evlist->core.mmap_len = evlist__mmap_size(pages); + pr_debug("mmap size %zuB\n", evlist->core.mmap_len); + + auxtrace_mmap_params__init(&mp.auxtrace_mp, evlist->core.mmap_len, + auxtrace_pages, auxtrace_overwrite); + + return perf_evlist__mmap_ops(&evlist->core, &ops, &mp.core); +} + +int evlist__mmap(struct evlist *evlist, unsigned int pages) +{ + return evlist__mmap_ex(evlist, pages, 0, false, 0, PERF_AFFINITY_SYS, 1, 0); +} + +int evlist__create_maps(struct evlist *evlist, struct target *target) +{ + bool all_threads = (target->per_thread && target->system_wide); + struct perf_cpu_map *cpus; + struct perf_thread_map *threads; + + /* + * If specify '-a' and '--per-thread' to perf record, perf record + * will override '--per-thread'. target->per_thread = false and + * target->system_wide = true. + * + * If specify '--per-thread' only to perf record, + * target->per_thread = true and target->system_wide = false. + * + * So target->per_thread && target->system_wide is false. + * For perf record, thread_map__new_str doesn't call + * thread_map__new_all_cpus. That will keep perf record's + * current behavior. + * + * For perf stat, it allows the case that target->per_thread and + * target->system_wide are all true. It means to collect system-wide + * per-thread data. thread_map__new_str will call + * thread_map__new_all_cpus to enumerate all threads. + */ + threads = thread_map__new_str(target->pid, target->tid, all_threads); + + if (!threads) + return -1; + + if (target__uses_dummy_map(target) && !evlist__has_bpf_output(evlist)) + cpus = perf_cpu_map__new_any_cpu(); + else + cpus = perf_cpu_map__new(target->cpu_list); + + if (!cpus) + goto out_delete_threads; + + evlist->core.has_user_cpus = !!target->cpu_list; + + perf_evlist__set_maps(&evlist->core, cpus, threads); + + /* as evlist now has references, put count here */ + perf_cpu_map__put(cpus); + perf_thread_map__put(threads); + + return 0; + +out_delete_threads: + perf_thread_map__put(threads); + return -1; +} + +int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, + struct target *target) +{ + struct evsel *evsel; + int err = 0; + + evlist__for_each_entry(evlist, evsel) { + /* + * filters only work for tracepoint event, which doesn't have cpu limit. + * So evlist and evsel should always be same. + */ + if (evsel->filter) { + err = perf_evsel__apply_filter(&evsel->core, evsel->filter); + if (err) { + *err_evsel = evsel; + break; + } + } + + /* + * non-tracepoint events can have BPF filters. + */ + if (!list_empty(&evsel->bpf_filters)) { + err = perf_bpf_filter__prepare(evsel, target); + if (err) { + *err_evsel = evsel; + break; + } + } + } + + return err; +} + +int evlist__set_tp_filter(struct evlist *evlist, const char *filter) +{ + struct evsel *evsel; + int err = 0; + + if (filter == NULL) + return -1; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) + continue; + + err = evsel__set_filter(evsel, filter); + if (err) + break; + } + + return err; +} + +int evlist__append_tp_filter(struct evlist *evlist, const char *filter) +{ + struct evsel *evsel; + int err = 0; + + if (filter == NULL) + return -1; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) + continue; + + err = evsel__append_tp_filter(evsel, filter); + if (err) + break; + } + + return err; +} + +char *asprintf__tp_filter_pids(size_t npids, pid_t *pids) +{ + char *filter; + size_t i; + + for (i = 0; i < npids; ++i) { + if (i == 0) { + if (asprintf(&filter, "common_pid != %d", pids[i]) < 0) + return NULL; + } else { + char *tmp; + + if (asprintf(&tmp, "%s && common_pid != %d", filter, pids[i]) < 0) + goto out_free; + + free(filter); + filter = tmp; + } + } + + return filter; +out_free: + free(filter); + return NULL; +} + +int evlist__set_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids) +{ + char *filter = asprintf__tp_filter_pids(npids, pids); + int ret = evlist__set_tp_filter(evlist, filter); + + free(filter); + return ret; +} + +int evlist__append_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids) +{ + char *filter = asprintf__tp_filter_pids(npids, pids); + int ret = evlist__append_tp_filter(evlist, filter); + + free(filter); + return ret; +} + +int evlist__append_tp_filter_pid(struct evlist *evlist, pid_t pid) +{ + return evlist__append_tp_filter_pids(evlist, 1, &pid); +} + +bool evlist__valid_sample_type(struct evlist *evlist) +{ + struct evsel *pos; + + if (evlist->core.nr_entries == 1) + return true; + + if (evlist->id_pos < 0 || evlist->is_pos < 0) + return false; + + evlist__for_each_entry(evlist, pos) { + if (pos->id_pos != evlist->id_pos || + pos->is_pos != evlist->is_pos) + return false; + } + + return true; +} + +u64 __evlist__combined_sample_type(struct evlist *evlist) +{ + struct evsel *evsel; + + if (evlist->combined_sample_type) + return evlist->combined_sample_type; + + evlist__for_each_entry(evlist, evsel) + evlist->combined_sample_type |= evsel->core.attr.sample_type; + + return evlist->combined_sample_type; +} + +u64 evlist__combined_sample_type(struct evlist *evlist) +{ + evlist->combined_sample_type = 0; + return __evlist__combined_sample_type(evlist); +} + +u64 evlist__combined_branch_type(struct evlist *evlist) +{ + struct evsel *evsel; + u64 branch_type = 0; + + evlist__for_each_entry(evlist, evsel) + branch_type |= evsel->core.attr.branch_sample_type; + return branch_type; +} + +static struct evsel * +evlist__find_dup_event_from_prev(struct evlist *evlist, struct evsel *event) +{ + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) { + if (event == pos) + break; + if ((pos->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + !strcmp(pos->name, event->name)) + return pos; + } + return NULL; +} + +#define MAX_NR_ABBR_NAME (26 * 11) + +/* + * The abbr name is from A to Z9. If the number of event + * which requires the branch counter > MAX_NR_ABBR_NAME, + * return NA. + */ +static void evlist__new_abbr_name(char *name) +{ + static int idx; + int i = idx / 26; + + if (idx >= MAX_NR_ABBR_NAME) { + name[0] = 'N'; + name[1] = 'A'; + name[2] = '\0'; + return; + } + + name[0] = 'A' + (idx % 26); + + if (!i) + name[1] = '\0'; + else { + name[1] = '0' + i - 1; + name[2] = '\0'; + } + + idx++; +} + +void evlist__update_br_cntr(struct evlist *evlist) +{ + struct evsel *evsel, *dup; + int i = 0; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) { + evsel->br_cntr_idx = i++; + evsel__leader(evsel)->br_cntr_nr++; + + dup = evlist__find_dup_event_from_prev(evlist, evsel); + if (dup) + memcpy(evsel->abbr_name, dup->abbr_name, 3 * sizeof(char)); + else + evlist__new_abbr_name(evsel->abbr_name); + } + } + evlist->nr_br_cntr = i; +} + +bool evlist__valid_read_format(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist), *pos = first; + u64 read_format = first->core.attr.read_format; + u64 sample_type = first->core.attr.sample_type; + + evlist__for_each_entry(evlist, pos) { + if (read_format != pos->core.attr.read_format) { + pr_debug("Read format differs %#" PRIx64 " vs %#" PRIx64 "\n", + read_format, (u64)pos->core.attr.read_format); + } + } + + /* PERF_SAMPLE_READ implies PERF_FORMAT_ID. */ + if ((sample_type & PERF_SAMPLE_READ) && + !(read_format & PERF_FORMAT_ID)) { + return false; + } + + return true; +} + +u16 evlist__id_hdr_size(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist); + + return first->core.attr.sample_id_all ? evsel__id_hdr_size(first) : 0; +} + +bool evlist__valid_sample_id_all(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist), *pos = first; + + evlist__for_each_entry_continue(evlist, pos) { + if (first->core.attr.sample_id_all != pos->core.attr.sample_id_all) + return false; + } + + return true; +} + +bool evlist__sample_id_all(struct evlist *evlist) +{ + struct evsel *first = evlist__first(evlist); + return first->core.attr.sample_id_all; +} + +void evlist__set_selected(struct evlist *evlist, struct evsel *evsel) +{ + evlist->selected = evsel; +} + +void evlist__close(struct evlist *evlist) +{ + struct evsel *evsel; + struct evlist_cpu_iterator evlist_cpu_itr; + + evlist__for_each_cpu(evlist_cpu_itr, evlist) { + if (evlist_cpu_itr.cpu_map_idx == 0 && evsel__is_retire_lat(evlist_cpu_itr.evsel)) + evsel__tpebs_close(evlist_cpu_itr.evsel); + perf_evsel__close_cpu(&evlist_cpu_itr.evsel->core, + evlist_cpu_itr.cpu_map_idx); + } + + evlist__for_each_entry_reverse(evlist, evsel) { + perf_evsel__free_fd(&evsel->core); + perf_evsel__free_id(&evsel->core); + } + perf_evlist__reset_id_hash(&evlist->core); +} + +static int evlist__create_syswide_maps(struct evlist *evlist) +{ + struct perf_cpu_map *cpus; + struct perf_thread_map *threads; + + /* + * Try reading /sys/devices/system/cpu/online to get + * an all cpus map. + * + * FIXME: -ENOMEM is the best we can do here, the cpu_map + * code needs an overhaul to properly forward the + * error, and we may not want to do that fallback to a + * default cpu identity map :-\ + */ + cpus = perf_cpu_map__new_online_cpus(); + if (!cpus) + return -ENOMEM; + + threads = perf_thread_map__new_dummy(); + if (!threads) { + perf_cpu_map__put(cpus); + return -ENOMEM; + } + + perf_evlist__set_maps(&evlist->core, cpus, threads); + perf_thread_map__put(threads); + perf_cpu_map__put(cpus); + return 0; +} + +int evlist__open(struct evlist *evlist) +{ + struct evsel *evsel; + int err; + + /* + * Default: one fd per CPU, all threads, aka systemwide + * as sys_perf_event_open(cpu = -1, thread = -1) is EINVAL + */ + if (evlist->core.threads == NULL && evlist->core.user_requested_cpus == NULL) { + err = evlist__create_syswide_maps(evlist); + if (err < 0) + goto out_err; + } + + evlist__update_id_pos(evlist); + + evlist__for_each_entry(evlist, evsel) { + err = evsel__open(evsel, evsel->core.cpus, evsel->core.threads); + if (err < 0) + goto out_err; + } + + return 0; +out_err: + evlist__close(evlist); + errno = -err; + return err; +} + +int evlist__prepare_workload(struct evlist *evlist, struct target *target, const char *argv[], + bool pipe_output, void (*exec_error)(int signo, siginfo_t *info, void *ucontext)) +{ + int child_ready_pipe[2], go_pipe[2]; + char bf; + + evlist->workload.cork_fd = -1; + + if (pipe(child_ready_pipe) < 0) { + perror("failed to create 'ready' pipe"); + return -1; + } + + if (pipe(go_pipe) < 0) { + perror("failed to create 'go' pipe"); + goto out_close_ready_pipe; + } + + evlist->workload.pid = fork(); + if (evlist->workload.pid < 0) { + perror("failed to fork"); + goto out_close_pipes; + } + + if (!evlist->workload.pid) { + int ret; + + if (pipe_output) + dup2(2, 1); + + signal(SIGTERM, SIG_DFL); + + close(child_ready_pipe[0]); + close(go_pipe[1]); + fcntl(go_pipe[0], F_SETFD, FD_CLOEXEC); + + /* + * Change the name of this process not to confuse --exclude-perf users + * that sees 'perf' in the window up to the execvp() and thinks that + * perf samples are not being excluded. + */ + prctl(PR_SET_NAME, "perf-exec"); + + /* + * Tell the parent we're ready to go + */ + close(child_ready_pipe[1]); + + /* + * Wait until the parent tells us to go. + */ + ret = read(go_pipe[0], &bf, 1); + /* + * The parent will ask for the execvp() to be performed by + * writing exactly one byte, in workload.cork_fd, usually via + * evlist__start_workload(). + * + * For cancelling the workload without actually running it, + * the parent will just close workload.cork_fd, without writing + * anything, i.e. read will return zero and we just exit() + * here (See evlist__cancel_workload()). + */ + if (ret != 1) { + if (ret == -1) + perror("unable to read pipe"); + exit(ret); + } + + execvp(argv[0], (char **)argv); + + if (exec_error) { + union sigval val; + + val.sival_int = errno; + if (sigqueue(getppid(), SIGUSR1, val)) + perror(argv[0]); + } else + perror(argv[0]); + exit(-1); + } + + if (exec_error) { + struct sigaction act = { + .sa_flags = SA_SIGINFO, + .sa_sigaction = exec_error, + }; + sigaction(SIGUSR1, &act, NULL); + } + + if (target__none(target)) { + if (evlist->core.threads == NULL) { + fprintf(stderr, "FATAL: evlist->threads need to be set at this point (%s:%d).\n", + __func__, __LINE__); + goto out_close_pipes; + } + perf_thread_map__set_pid(evlist->core.threads, 0, evlist->workload.pid); + } + + close(child_ready_pipe[1]); + close(go_pipe[0]); + /* + * wait for child to settle + */ + if (read(child_ready_pipe[0], &bf, 1) == -1) { + perror("unable to read pipe"); + goto out_close_pipes; + } + + fcntl(go_pipe[1], F_SETFD, FD_CLOEXEC); + evlist->workload.cork_fd = go_pipe[1]; + close(child_ready_pipe[0]); + return 0; + +out_close_pipes: + close(go_pipe[0]); + close(go_pipe[1]); +out_close_ready_pipe: + close(child_ready_pipe[0]); + close(child_ready_pipe[1]); + return -1; +} + +int evlist__start_workload(struct evlist *evlist) +{ + if (evlist->workload.cork_fd >= 0) { + char bf = 0; + int ret; + /* + * Remove the cork, let it rip! + */ + ret = write(evlist->workload.cork_fd, &bf, 1); + if (ret < 0) + perror("unable to write to pipe"); + + close(evlist->workload.cork_fd); + evlist->workload.cork_fd = -1; + return ret; + } + + return 0; +} + +void evlist__cancel_workload(struct evlist *evlist) +{ + int status; + + if (evlist->workload.cork_fd >= 0) { + close(evlist->workload.cork_fd); + evlist->workload.cork_fd = -1; + waitpid(evlist->workload.pid, &status, WNOHANG); + } +} + +int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample) +{ + struct evsel *evsel = evlist__event2evsel(evlist, event); + int ret; + + if (!evsel) { + /* Ensure the sample is okay for perf_sample__exit. */ + perf_sample__init(sample, /*all=*/false); + return -EFAULT; + } + ret = evsel__parse_sample(evsel, event, sample); + if (ret) + return ret; + if (perf_guest && sample->id) { + struct perf_sample_id *sid = evlist__id2sid(evlist, sample->id); + + if (sid) { + sample->machine_pid = sid->machine_pid; + sample->vcpu = sid->vcpu.cpu; + } + } + return 0; +} + +int evlist__parse_sample_timestamp(struct evlist *evlist, union perf_event *event, u64 *timestamp) +{ + struct evsel *evsel = evlist__event2evsel(evlist, event); + + if (!evsel) + return -EFAULT; + return evsel__parse_sample_timestamp(evsel, event, timestamp); +} + +int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size) +{ + int printed, value; + + switch (err) { + case EACCES: + case EPERM: + errno = err; + printed = scnprintf(buf, size, + "Error:\t%m.\n" + "Hint:\tCheck /proc/sys/kernel/perf_event_paranoid setting."); + + value = perf_event_paranoid(); + + printed += scnprintf(buf + printed, size - printed, "\nHint:\t"); + + if (value >= 2) { + printed += scnprintf(buf + printed, size - printed, + "For your workloads it needs to be <= 1\nHint:\t"); + } + printed += scnprintf(buf + printed, size - printed, + "For system wide tracing it needs to be set to -1.\n"); + + printed += scnprintf(buf + printed, size - printed, + "Hint:\tTry: 'sudo sh -c \"echo -1 > /proc/sys/kernel/perf_event_paranoid\"'\n" + "Hint:\tThe current value is %d.", value); + break; + case EINVAL: { + struct evsel *first = evlist__first(evlist); + int max_freq; + + if (sysctl__read_int("kernel/perf_event_max_sample_rate", &max_freq) < 0) + goto out_default; + + if (first->core.attr.sample_freq < (u64)max_freq) + goto out_default; + + errno = err; + printed = scnprintf(buf, size, + "Error:\t%m.\n" + "Hint:\tCheck /proc/sys/kernel/perf_event_max_sample_rate.\n" + "Hint:\tThe current value is %d and %" PRIu64 " is being requested.", + max_freq, first->core.attr.sample_freq); + break; + } + default: +out_default: + errno = err; + scnprintf(buf, size, "%m"); + break; + } + + return 0; +} + +int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size) +{ + int pages_attempted = evlist->core.mmap_len / 1024, pages_max_per_user, printed = 0; + + switch (err) { + case EPERM: + sysctl__read_int("kernel/perf_event_mlock_kb", &pages_max_per_user); + errno = err; + printed += scnprintf(buf + printed, size - printed, + "Error:\t%m.\n" + "Hint:\tCheck /proc/sys/kernel/perf_event_mlock_kb (%d kB) setting.\n" + "Hint:\tTried using %zd kB.\n", + pages_max_per_user, pages_attempted); + + if (pages_attempted >= pages_max_per_user) { + printed += scnprintf(buf + printed, size - printed, + "Hint:\tTry 'sudo sh -c \"echo %d > /proc/sys/kernel/perf_event_mlock_kb\"', or\n", + pages_max_per_user + pages_attempted); + } + + printed += scnprintf(buf + printed, size - printed, + "Hint:\tTry using a smaller -m/--mmap-pages value."); + break; + default: + errno = err; + scnprintf(buf, size, "%m"); + break; + } + + return 0; +} + +void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel) +{ + struct evsel *evsel, *n; + LIST_HEAD(move); + + if (move_evsel == evlist__first(evlist)) + return; + + evlist__for_each_entry_safe(evlist, n, evsel) { + if (evsel__leader(evsel) == evsel__leader(move_evsel)) + list_move_tail(&evsel->core.node, &move); + } + + list_splice(&move, &evlist->core.entries); +} + +struct evsel *evlist__get_tracking_event(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->tracking) + return evsel; + } + + return evlist__first(evlist); +} + +void evlist__set_tracking_event(struct evlist *evlist, struct evsel *tracking_evsel) +{ + struct evsel *evsel; + + if (tracking_evsel->tracking) + return; + + evlist__for_each_entry(evlist, evsel) { + if (evsel != tracking_evsel) + evsel->tracking = false; + } + + tracking_evsel->tracking = true; +} + +struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_wide) +{ + struct evsel *evsel; + + evsel = evlist__get_tracking_event(evlist); + if (!evsel__is_dummy_event(evsel)) { + evsel = evlist__add_aux_dummy(evlist, system_wide); + if (!evsel) + return NULL; + + evlist__set_tracking_event(evlist, evsel); + } else if (system_wide) { + perf_evlist__go_system_wide(&evlist->core, &evsel->core); + } + + return evsel; +} + +struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (!evsel->name) + continue; + if (evsel__name_is(evsel, str)) + return evsel; + } + + return NULL; +} + +void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state) +{ + enum bkw_mmap_state old_state = evlist->bkw_mmap_state; + enum action { + NONE, + PAUSE, + RESUME, + } action = NONE; + + if (!evlist->overwrite_mmap) + return; + + switch (old_state) { + case BKW_MMAP_NOTREADY: { + if (state != BKW_MMAP_RUNNING) + goto state_err; + break; + } + case BKW_MMAP_RUNNING: { + if (state != BKW_MMAP_DATA_PENDING) + goto state_err; + action = PAUSE; + break; + } + case BKW_MMAP_DATA_PENDING: { + if (state != BKW_MMAP_EMPTY) + goto state_err; + break; + } + case BKW_MMAP_EMPTY: { + if (state != BKW_MMAP_RUNNING) + goto state_err; + action = RESUME; + break; + } + default: + WARN_ONCE(1, "Shouldn't get there\n"); + } + + evlist->bkw_mmap_state = state; + + switch (action) { + case PAUSE: + evlist__pause(evlist); + break; + case RESUME: + evlist__resume(evlist); + break; + case NONE: + default: + break; + } + +state_err: + return; +} + +bool evlist__exclude_kernel(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (!evsel->core.attr.exclude_kernel) + return false; + } + + return true; +} + +/* + * Events in data file are not collect in groups, but we still want + * the group display. Set the artificial group and set the leader's + * forced_leader flag to notify the display code. + */ +void evlist__force_leader(struct evlist *evlist) +{ + if (evlist__nr_groups(evlist) == 0) { + struct evsel *leader = evlist__first(evlist); + + evlist__set_leader(evlist); + leader->forced_leader = true; + } +} + +struct evsel *evlist__reset_weak_group(struct evlist *evsel_list, struct evsel *evsel, bool close) +{ + struct evsel *c2, *leader; + bool is_open = true; + + leader = evsel__leader(evsel); + + pr_debug("Weak group for %s/%d failed\n", + leader->name, leader->core.nr_members); + + /* + * for_each_group_member doesn't work here because it doesn't + * include the first entry. + */ + evlist__for_each_entry(evsel_list, c2) { + if (c2 == evsel) + is_open = false; + if (evsel__has_leader(c2, leader)) { + if (is_open && close) + perf_evsel__close(&c2->core); + /* + * We want to close all members of the group and reopen + * them. Some events, like Intel topdown, require being + * in a group and so keep these in the group. + */ + evsel__remove_from_group(c2, leader); + + /* + * Set this for all former members of the group + * to indicate they get reopened. + */ + c2->reset_group = true; + } + } + /* Reset the leader count if all entries were removed. */ + if (leader->core.nr_members == 1) + leader->core.nr_members = 0; + return leader; +} + +static int evlist__parse_control_fifo(const char *str, int *ctl_fd, int *ctl_fd_ack, bool *ctl_fd_close) +{ + char *s, *p; + int ret = 0, fd; + + if (strncmp(str, "fifo:", 5)) + return -EINVAL; + + str += 5; + if (!*str || *str == ',') + return -EINVAL; + + s = strdup(str); + if (!s) + return -ENOMEM; + + p = strchr(s, ','); + if (p) + *p = '\0'; + + /* + * O_RDWR avoids POLLHUPs which is necessary to allow the other + * end of a FIFO to be repeatedly opened and closed. + */ + fd = open(s, O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (fd < 0) { + ret = -errno; + pr_err("Failed to open '%s': %m\n", s); + goto out_free; + } + *ctl_fd = fd; + *ctl_fd_close = true; + + if (p && *++p) { + /* O_RDWR | O_NONBLOCK means the other end need not be open */ + fd = open(p, O_RDWR | O_NONBLOCK | O_CLOEXEC); + if (fd < 0) { + pr_err("Failed to open '%s': %m\n", p); + ret = -errno; + goto out_free; + } + *ctl_fd_ack = fd; + } + +out_free: + free(s); + return ret; +} + +int evlist__parse_control(const char *str, int *ctl_fd, int *ctl_fd_ack, bool *ctl_fd_close) +{ + const char *comma = NULL; + char *endptr = NULL; + + *ctl_fd_close = false; + + if (strncmp(str, "fd:", 3)) + return evlist__parse_control_fifo(str, ctl_fd, ctl_fd_ack, ctl_fd_close); + + *ctl_fd = strtoul(&str[3], &endptr, 0); + if (endptr == &str[3]) + return -EINVAL; + + comma = strchr(str, ','); + if (comma) { + if (endptr != comma) + return -EINVAL; + + *ctl_fd_ack = strtoul(comma + 1, &endptr, 0); + if (endptr == comma + 1 || *endptr != '\0') + return -EINVAL; + } + + return 0; +} + +void evlist__close_control(int ctl_fd, int ctl_fd_ack, bool *ctl_fd_close) +{ + if (*ctl_fd_close) { + *ctl_fd_close = false; + close(ctl_fd); + if (ctl_fd_ack >= 0) + close(ctl_fd_ack); + } +} + +int evlist__initialize_ctlfd(struct evlist *evlist, int fd, int ack) +{ + if (fd == -1) { + pr_debug("Control descriptor is not initialized\n"); + return 0; + } + + evlist->ctl_fd.pos = perf_evlist__add_pollfd(&evlist->core, fd, NULL, POLLIN, + fdarray_flag__nonfilterable | + fdarray_flag__non_perf_event); + if (evlist->ctl_fd.pos < 0) { + evlist->ctl_fd.pos = -1; + pr_err("Failed to add ctl fd entry: %m\n"); + return -1; + } + + evlist->ctl_fd.fd = fd; + evlist->ctl_fd.ack = ack; + + return 0; +} + +bool evlist__ctlfd_initialized(struct evlist *evlist) +{ + return evlist->ctl_fd.pos >= 0; +} + +int evlist__finalize_ctlfd(struct evlist *evlist) +{ + struct pollfd *entries = evlist->core.pollfd.entries; + + if (!evlist__ctlfd_initialized(evlist)) + return 0; + + entries[evlist->ctl_fd.pos].fd = -1; + entries[evlist->ctl_fd.pos].events = 0; + entries[evlist->ctl_fd.pos].revents = 0; + + evlist->ctl_fd.pos = -1; + evlist->ctl_fd.ack = -1; + evlist->ctl_fd.fd = -1; + + return 0; +} + +static int evlist__ctlfd_recv(struct evlist *evlist, enum evlist_ctl_cmd *cmd, + char *cmd_data, size_t data_size) +{ + int err; + char c; + size_t bytes_read = 0; + + *cmd = EVLIST_CTL_CMD_UNSUPPORTED; + memset(cmd_data, 0, data_size); + data_size--; + + do { + err = read(evlist->ctl_fd.fd, &c, 1); + if (err > 0) { + if (c == '\n' || c == '\0') + break; + cmd_data[bytes_read++] = c; + if (bytes_read == data_size) + break; + continue; + } else if (err == -1) { + if (errno == EINTR) + continue; + if (errno == EAGAIN || errno == EWOULDBLOCK) + err = 0; + else + pr_err("Failed to read from ctlfd %d: %m\n", evlist->ctl_fd.fd); + } + break; + } while (1); + + pr_debug("Message from ctl_fd: \"%s%s\"\n", cmd_data, + bytes_read == data_size ? "" : c == '\n' ? "\\n" : "\\0"); + + if (bytes_read > 0) { + if (!strncmp(cmd_data, EVLIST_CTL_CMD_ENABLE_TAG, + (sizeof(EVLIST_CTL_CMD_ENABLE_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_ENABLE; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_DISABLE_TAG, + (sizeof(EVLIST_CTL_CMD_DISABLE_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_DISABLE; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_SNAPSHOT_TAG, + (sizeof(EVLIST_CTL_CMD_SNAPSHOT_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_SNAPSHOT; + pr_debug("is snapshot\n"); + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_EVLIST_TAG, + (sizeof(EVLIST_CTL_CMD_EVLIST_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_EVLIST; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_STOP_TAG, + (sizeof(EVLIST_CTL_CMD_STOP_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_STOP; + } else if (!strncmp(cmd_data, EVLIST_CTL_CMD_PING_TAG, + (sizeof(EVLIST_CTL_CMD_PING_TAG)-1))) { + *cmd = EVLIST_CTL_CMD_PING; + } + } + + return bytes_read ? (int)bytes_read : err; +} + +int evlist__ctlfd_ack(struct evlist *evlist) +{ + int err; + + if (evlist->ctl_fd.ack == -1) + return 0; + + err = write(evlist->ctl_fd.ack, EVLIST_CTL_CMD_ACK_TAG, + sizeof(EVLIST_CTL_CMD_ACK_TAG)); + if (err == -1) + pr_err("failed to write to ctl_ack_fd %d: %m\n", evlist->ctl_fd.ack); + + return err; +} + +static int get_cmd_arg(char *cmd_data, size_t cmd_size, char **arg) +{ + char *data = cmd_data + cmd_size; + + /* no argument */ + if (!*data) + return 0; + + /* there's argument */ + if (*data == ' ') { + *arg = data + 1; + return 1; + } + + /* malformed */ + return -1; +} + +static int evlist__ctlfd_enable(struct evlist *evlist, char *cmd_data, bool enable) +{ + struct evsel *evsel; + char *name; + int err; + + err = get_cmd_arg(cmd_data, + enable ? sizeof(EVLIST_CTL_CMD_ENABLE_TAG) - 1 : + sizeof(EVLIST_CTL_CMD_DISABLE_TAG) - 1, + &name); + if (err < 0) { + pr_info("failed: wrong command\n"); + return -1; + } + + if (err) { + evsel = evlist__find_evsel_by_str(evlist, name); + if (evsel) { + if (enable) + evlist__enable_evsel(evlist, name); + else + evlist__disable_evsel(evlist, name); + pr_info("Event %s %s\n", evsel->name, + enable ? "enabled" : "disabled"); + } else { + pr_info("failed: can't find '%s' event\n", name); + } + } else { + if (enable) { + evlist__enable(evlist); + pr_info(EVLIST_ENABLED_MSG); + } else { + evlist__disable(evlist); + pr_info(EVLIST_DISABLED_MSG); + } + } + + return 0; +} + +static int evlist__ctlfd_list(struct evlist *evlist, char *cmd_data) +{ + struct perf_attr_details details = { .verbose = false, }; + struct evsel *evsel; + char *arg; + int err; + + err = get_cmd_arg(cmd_data, + sizeof(EVLIST_CTL_CMD_EVLIST_TAG) - 1, + &arg); + if (err < 0) { + pr_info("failed: wrong command\n"); + return -1; + } + + if (err) { + if (!strcmp(arg, "-v")) { + details.verbose = true; + } else if (!strcmp(arg, "-g")) { + details.event_group = true; + } else if (!strcmp(arg, "-F")) { + details.freq = true; + } else { + pr_info("failed: wrong command\n"); + return -1; + } + } + + evlist__for_each_entry(evlist, evsel) + evsel__fprintf(evsel, &details, stderr); + + return 0; +} + +int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd) +{ + int err = 0; + char cmd_data[EVLIST_CTL_CMD_MAX_LEN]; + int ctlfd_pos = evlist->ctl_fd.pos; + struct pollfd *entries = evlist->core.pollfd.entries; + + if (!evlist__ctlfd_initialized(evlist) || !entries[ctlfd_pos].revents) + return 0; + + if (entries[ctlfd_pos].revents & POLLIN) { + err = evlist__ctlfd_recv(evlist, cmd, cmd_data, + EVLIST_CTL_CMD_MAX_LEN); + if (err > 0) { + switch (*cmd) { + case EVLIST_CTL_CMD_ENABLE: + case EVLIST_CTL_CMD_DISABLE: + err = evlist__ctlfd_enable(evlist, cmd_data, + *cmd == EVLIST_CTL_CMD_ENABLE); + break; + case EVLIST_CTL_CMD_EVLIST: + err = evlist__ctlfd_list(evlist, cmd_data); + break; + case EVLIST_CTL_CMD_SNAPSHOT: + case EVLIST_CTL_CMD_STOP: + case EVLIST_CTL_CMD_PING: + break; + case EVLIST_CTL_CMD_ACK: + case EVLIST_CTL_CMD_UNSUPPORTED: + default: + pr_debug("ctlfd: unsupported %d\n", *cmd); + break; + } + if (!(*cmd == EVLIST_CTL_CMD_ACK || *cmd == EVLIST_CTL_CMD_UNSUPPORTED || + *cmd == EVLIST_CTL_CMD_SNAPSHOT)) + evlist__ctlfd_ack(evlist); + } + } + + if (entries[ctlfd_pos].revents & (POLLHUP | POLLERR)) + evlist__finalize_ctlfd(evlist); + else + entries[ctlfd_pos].revents = 0; + + return err; +} + +/** + * struct event_enable_time - perf record -D/--delay single time range. + * @start: start of time range to enable events in milliseconds + * @end: end of time range to enable events in milliseconds + * + * N.B. this structure is also accessed as an array of int. + */ +struct event_enable_time { + int start; + int end; +}; + +static int parse_event_enable_time(const char *str, struct event_enable_time *range, bool first) +{ + const char *fmt = first ? "%u - %u %n" : " , %u - %u %n"; + int ret, start, end, n; + + ret = sscanf(str, fmt, &start, &end, &n); + if (ret != 2 || end <= start) + return -EINVAL; + if (range) { + range->start = start; + range->end = end; + } + return n; +} + +static ssize_t parse_event_enable_times(const char *str, struct event_enable_time *range) +{ + int incr = !!range; + bool first = true; + ssize_t ret, cnt; + + for (cnt = 0; *str; cnt++) { + ret = parse_event_enable_time(str, range, first); + if (ret < 0) + return ret; + /* Check no overlap */ + if (!first && range && range->start <= range[-1].end) + return -EINVAL; + str += ret; + range += incr; + first = false; + } + return cnt; +} + +/** + * struct event_enable_timer - control structure for perf record -D/--delay. + * @evlist: event list + * @times: time ranges that events are enabled (N.B. this is also accessed as an + * array of int) + * @times_cnt: number of time ranges + * @timerfd: timer file descriptor + * @pollfd_pos: position in @evlist array of file descriptors to poll (fdarray) + * @times_step: current position in (int *)@times)[], + * refer event_enable_timer__process() + * + * Note, this structure is only used when there are time ranges, not when there + * is only an initial delay. + */ +struct event_enable_timer { + struct evlist *evlist; + struct event_enable_time *times; + size_t times_cnt; + int timerfd; + int pollfd_pos; + size_t times_step; +}; + +static int str_to_delay(const char *str) +{ + char *endptr; + long d; + + d = strtol(str, &endptr, 10); + if (*endptr || d > INT_MAX || d < -1) + return 0; + return d; +} + +int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, + const char *str, int unset) +{ + enum fdarray_flags flags = fdarray_flag__nonfilterable | fdarray_flag__non_perf_event; + struct event_enable_timer *eet; + ssize_t times_cnt; + ssize_t ret; + int err; + + if (unset) + return 0; + + opts->target.initial_delay = str_to_delay(str); + if (opts->target.initial_delay) + return 0; + + ret = parse_event_enable_times(str, NULL); + if (ret < 0) + return ret; + + times_cnt = ret; + if (times_cnt == 0) + return -EINVAL; + + eet = zalloc(sizeof(*eet)); + if (!eet) + return -ENOMEM; + + eet->times = calloc(times_cnt, sizeof(*eet->times)); + if (!eet->times) { + err = -ENOMEM; + goto free_eet; + } + + if (parse_event_enable_times(str, eet->times) != times_cnt) { + err = -EINVAL; + goto free_eet_times; + } + + eet->times_cnt = times_cnt; + + eet->timerfd = timerfd_create(CLOCK_MONOTONIC, TFD_CLOEXEC); + if (eet->timerfd == -1) { + err = -errno; + pr_err("timerfd_create failed: %m\n"); + goto free_eet_times; + } + + eet->pollfd_pos = perf_evlist__add_pollfd(&evlist->core, eet->timerfd, NULL, POLLIN, flags); + if (eet->pollfd_pos < 0) { + err = eet->pollfd_pos; + goto close_timerfd; + } + + eet->evlist = evlist; + evlist->eet = eet; + opts->target.initial_delay = eet->times[0].start; + + return 0; + +close_timerfd: + close(eet->timerfd); +free_eet_times: + zfree(&eet->times); +free_eet: + free(eet); + return err; +} + +static int event_enable_timer__set_timer(struct event_enable_timer *eet, int ms) +{ + struct itimerspec its = { + .it_value.tv_sec = ms / MSEC_PER_SEC, + .it_value.tv_nsec = (ms % MSEC_PER_SEC) * NSEC_PER_MSEC, + }; + int err = 0; + + if (timerfd_settime(eet->timerfd, 0, &its, NULL) < 0) { + err = -errno; + pr_err("timerfd_settime failed: %m\n"); + } + return err; +} + +int event_enable_timer__start(struct event_enable_timer *eet) +{ + int ms; + + if (!eet) + return 0; + + ms = eet->times[0].end - eet->times[0].start; + eet->times_step = 1; + + return event_enable_timer__set_timer(eet, ms); +} + +int event_enable_timer__process(struct event_enable_timer *eet) +{ + struct pollfd *entries; + short revents; + + if (!eet) + return 0; + + entries = eet->evlist->core.pollfd.entries; + revents = entries[eet->pollfd_pos].revents; + entries[eet->pollfd_pos].revents = 0; + + if (revents & POLLIN) { + size_t step = eet->times_step; + size_t pos = step / 2; + + if (step & 1) { + evlist__disable_non_dummy(eet->evlist); + pr_info(EVLIST_DISABLED_MSG); + if (pos >= eet->times_cnt - 1) { + /* Disarm timer */ + event_enable_timer__set_timer(eet, 0); + return 1; /* Stop */ + } + } else { + evlist__enable_non_dummy(eet->evlist); + pr_info(EVLIST_ENABLED_MSG); + } + + step += 1; + pos = step / 2; + + if (pos < eet->times_cnt) { + int *times = (int *)eet->times; /* Accessing 'times' as array of int */ + int ms = times[step] - times[step - 1]; + + eet->times_step = step; + return event_enable_timer__set_timer(eet, ms); + } + } + + return 0; +} + +void event_enable_timer__exit(struct event_enable_timer **ep) +{ + if (!ep || !*ep) + return; + zfree(&(*ep)->times); + zfree(ep); +} + +struct evsel *evlist__find_evsel(struct evlist *evlist, int idx) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.idx == idx) + return evsel; + } + return NULL; +} + +void evlist__format_evsels(struct evlist *evlist, struct strbuf *sb, size_t max_length) +{ + struct evsel *evsel, *leader = NULL; + bool first = true; + + evlist__for_each_entry(evlist, evsel) { + struct evsel *new_leader = evsel__leader(evsel); + + if (evsel__is_dummy_event(evsel)) + continue; + + if (leader != new_leader && leader && leader->core.nr_members > 1) + strbuf_addch(sb, '}'); + + if (!first) + strbuf_addch(sb, ','); + + if (sb->len > max_length) { + strbuf_addstr(sb, "..."); + return; + } + if (leader != new_leader && new_leader->core.nr_members > 1) + strbuf_addch(sb, '{'); + + strbuf_addstr(sb, evsel__name(evsel)); + first = false; + leader = new_leader; + } + if (leader && leader->core.nr_members > 1) + strbuf_addch(sb, '}'); +} + +void evlist__check_mem_load_aux(struct evlist *evlist) +{ + struct evsel *leader, *evsel, *pos; + + /* + * For some platforms, the 'mem-loads' event is required to use + * together with 'mem-loads-aux' within a group and 'mem-loads-aux' + * must be the group leader. Now we disable this group before reporting + * because 'mem-loads-aux' is just an auxiliary event. It doesn't carry + * any valid memory load information. + */ + evlist__for_each_entry(evlist, evsel) { + leader = evsel__leader(evsel); + if (leader == evsel) + continue; + + if (leader->name && strstr(leader->name, "mem-loads-aux")) { + for_each_group_evsel(pos, leader) { + evsel__set_leader(pos, pos); + pos->core.nr_members = 0; + } + } + } +} + +/** + * evlist__warn_user_requested_cpus() - Check each evsel against requested CPUs + * and warn if the user CPU list is inapplicable for the event's PMU's + * CPUs. Not core PMUs list a CPU in sysfs, but this may be overwritten by a + * user requested CPU and so any online CPU is applicable. Core PMUs handle + * events on the CPUs in their list and otherwise the event isn't supported. + * @evlist: The list of events being checked. + * @cpu_list: The user provided list of CPUs. + */ +void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_list) +{ + struct perf_cpu_map *user_requested_cpus; + struct evsel *pos; + + if (!cpu_list) + return; + + user_requested_cpus = perf_cpu_map__new(cpu_list); + if (!user_requested_cpus) + return; + + evlist__for_each_entry(evlist, pos) { + evsel__warn_user_requested_cpus(pos, user_requested_cpus); + } + perf_cpu_map__put(user_requested_cpus); +} + +/* Should uniquify be disabled for the evlist? */ +static bool evlist__disable_uniquify(const struct evlist *evlist) +{ + struct evsel *counter; + struct perf_pmu *last_pmu = NULL; + bool first = true; + + evlist__for_each_entry(evlist, counter) { + /* If PMUs vary then uniquify can be useful. */ + if (!first && counter->pmu != last_pmu) + return false; + first = false; + if (counter->pmu) { + /* Allow uniquify for uncore PMUs. */ + if (!counter->pmu->is_core) + return false; + /* Keep hybrid event names uniquified for clarity. */ + if (perf_pmus__num_core_pmus() > 1) + return false; + } + last_pmu = counter->pmu; + } + return true; +} + +static bool evlist__set_needs_uniquify(struct evlist *evlist, const struct perf_stat_config *config) +{ + struct evsel *counter; + bool needs_uniquify = false; + + if (evlist__disable_uniquify(evlist)) { + evlist__for_each_entry(evlist, counter) + counter->uniquified_name = true; + return false; + } + + evlist__for_each_entry(evlist, counter) { + if (evsel__set_needs_uniquify(counter, config)) + needs_uniquify = true; + } + return needs_uniquify; +} + +void evlist__uniquify_evsel_names(struct evlist *evlist, const struct perf_stat_config *config) +{ + if (evlist__set_needs_uniquify(evlist, config)) { + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) + evsel__uniquify_counter(pos); + } +} + +bool evlist__has_bpf_output(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_bpf_output(evsel)) + return true; + } + + return false; +} + +bool evlist__needs_bpf_sb_event(struct evlist *evlist) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_dummy_event(evsel)) + continue; + if (!evsel->core.attr.exclude_kernel) + return true; + } + + return false; +} diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h new file mode 100644 index 000000000..e507f5f20 --- /dev/null +++ b/tools/perf/util/evlist.h @@ -0,0 +1,464 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_EVLIST_H +#define __PERF_EVLIST_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include "affinity.h" +#include "events_stats.h" +#include "evsel.h" +#include "rblist.h" +#include +#include +#include + +struct pollfd; +struct thread_map; +struct perf_cpu_map; +struct perf_stat_config; +struct record_opts; +struct strbuf; +struct target; + +/* + * State machine of bkw_mmap_state: + * + * .________________(forbid)_____________. + * | V + * NOTREADY --(0)--> RUNNING --(1)--> DATA_PENDING --(2)--> EMPTY + * ^ ^ | ^ | + * | |__(forbid)____/ |___(forbid)___/| + * | | + * \_________________(3)_______________/ + * + * NOTREADY : Backward ring buffers are not ready + * RUNNING : Backward ring buffers are recording + * DATA_PENDING : We are required to collect data from backward ring buffers + * EMPTY : We have collected data from backward ring buffers. + * + * (0): Setup backward ring buffer + * (1): Pause ring buffers for reading + * (2): Read from ring buffers + * (3): Resume ring buffers for recording + */ +enum bkw_mmap_state { + BKW_MMAP_NOTREADY, + BKW_MMAP_RUNNING, + BKW_MMAP_DATA_PENDING, + BKW_MMAP_EMPTY, +}; + +struct event_enable_timer; + +struct evlist { + struct perf_evlist core; + bool enabled; + bool no_affinity; + int id_pos; + int is_pos; + int nr_br_cntr; + u64 combined_sample_type; + enum bkw_mmap_state bkw_mmap_state; + struct { + int cork_fd; + pid_t pid; + } workload; + struct mmap *mmap; + struct mmap *overwrite_mmap; + struct evsel *selected; + struct events_stats stats; + struct perf_session *session; + void (*trace_event_sample_raw)(struct evlist *evlist, + union perf_event *event, + struct perf_sample *sample); + u64 first_sample_time; + u64 last_sample_time; + struct { + pthread_t th; + volatile int done; + } thread; + struct { + int fd; /* control file descriptor */ + int ack; /* ack file descriptor for control commands */ + int pos; /* index at evlist core object to check signals */ + } ctl_fd; + struct event_enable_timer *eet; + /** + * @metric_events: A list of struct metric_event which each have a list + * of struct metric_expr. + */ + struct rblist metric_events; + /* samples with deferred_callchain would wait here. */ + struct list_head deferred_samples; +}; + +struct evsel_str_handler { + const char *name; + void *handler; +}; + +struct evlist *evlist__new(void); +struct evlist *evlist__new_default(const struct target *target, bool sample_callchains); +struct evlist *evlist__new_dummy(void); +void evlist__init(struct evlist *evlist, struct perf_cpu_map *cpus, + struct perf_thread_map *threads); +void evlist__exit(struct evlist *evlist); +void evlist__delete(struct evlist *evlist); + +void evlist__add(struct evlist *evlist, struct evsel *entry); +void evlist__remove(struct evlist *evlist, struct evsel *evsel); + +int arch_evlist__cmp(const struct evsel *lhs, const struct evsel *rhs); +int arch_evlist__add_required_events(struct list_head *list); + +int evlist__add_dummy(struct evlist *evlist); +struct evsel *evlist__add_aux_dummy(struct evlist *evlist, bool system_wide); +static inline struct evsel *evlist__add_dummy_on_all_cpus(struct evlist *evlist) +{ + return evlist__add_aux_dummy(evlist, true); +} +#ifdef HAVE_LIBTRACEEVENT +struct evsel *evlist__add_sched_switch(struct evlist *evlist, bool system_wide); +#endif + +int evlist__add_sb_event(struct evlist *evlist, struct perf_event_attr *attr, + evsel__sb_cb_t cb, void *data); +void evlist__set_cb(struct evlist *evlist, evsel__sb_cb_t cb, void *data); +int evlist__start_sb_thread(struct evlist *evlist, struct target *target); +void evlist__stop_sb_thread(struct evlist *evlist); + +#ifdef HAVE_LIBTRACEEVENT +int evlist__add_newtp(struct evlist *evlist, const char *sys, const char *name, void *handler); +#endif + +int __evlist__set_tracepoints_handlers(struct evlist *evlist, + const struct evsel_str_handler *assocs, + size_t nr_assocs); + +#define evlist__set_tracepoints_handlers(evlist, array) \ + __evlist__set_tracepoints_handlers(evlist, array, ARRAY_SIZE(array)) + +int evlist__set_tp_filter(struct evlist *evlist, const char *filter); +int evlist__set_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids); + +int evlist__append_tp_filter(struct evlist *evlist, const char *filter); + +int evlist__append_tp_filter_pid(struct evlist *evlist, pid_t pid); +int evlist__append_tp_filter_pids(struct evlist *evlist, size_t npids, pid_t *pids); + +struct evsel *evlist__find_tracepoint_by_name(struct evlist *evlist, const char *name); + +int evlist__add_pollfd(struct evlist *evlist, int fd); +int evlist__filter_pollfd(struct evlist *evlist, short revents_and_mask); + +#ifdef HAVE_EVENTFD_SUPPORT +int evlist__add_wakeup_eventfd(struct evlist *evlist, int fd); +#endif + +int evlist__poll(struct evlist *evlist, int timeout); + +struct evsel *evlist__id2evsel(struct evlist *evlist, u64 id); +struct evsel *evlist__id2evsel_strict(struct evlist *evlist, u64 id); + +struct perf_sample_id *evlist__id2sid(struct evlist *evlist, u64 id); + +void evlist__toggle_bkw_mmap(struct evlist *evlist, enum bkw_mmap_state state); + +void evlist__mmap_consume(struct evlist *evlist, int idx); + +int evlist__open(struct evlist *evlist); +void evlist__close(struct evlist *evlist); + +struct callchain_param; + +void evlist__set_id_pos(struct evlist *evlist); +void evlist__config(struct evlist *evlist, struct record_opts *opts, struct callchain_param *callchain); +int record_opts__config(struct record_opts *opts); + +int evlist__prepare_workload(struct evlist *evlist, struct target *target, + const char *argv[], bool pipe_output, + void (*exec_error)(int signo, siginfo_t *info, void *ucontext)); +int evlist__start_workload(struct evlist *evlist); +void evlist__cancel_workload(struct evlist *evlist); + +struct option; + +int __evlist__parse_mmap_pages(unsigned int *mmap_pages, const char *str); +int evlist__parse_mmap_pages(const struct option *opt, const char *str, int unset); + +unsigned long perf_event_mlock_kb_in_pages(void); + +int evlist__mmap_ex(struct evlist *evlist, unsigned int pages, + unsigned int auxtrace_pages, + bool auxtrace_overwrite, int nr_cblocks, + int affinity, int flush, int comp_level); +int evlist__mmap(struct evlist *evlist, unsigned int pages); +void evlist__munmap(struct evlist *evlist); + +size_t evlist__mmap_size(unsigned long pages); + +void evlist__disable(struct evlist *evlist); +void evlist__enable(struct evlist *evlist); +void evlist__toggle_enable(struct evlist *evlist); +void evlist__disable_evsel(struct evlist *evlist, char *evsel_name); +void evlist__enable_evsel(struct evlist *evlist, char *evsel_name); +void evlist__disable_non_dummy(struct evlist *evlist); +void evlist__enable_non_dummy(struct evlist *evlist); + +void evlist__set_selected(struct evlist *evlist, struct evsel *evsel); + +int evlist__create_maps(struct evlist *evlist, struct target *target); +int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel, + struct target *target); + +u64 __evlist__combined_sample_type(struct evlist *evlist); +u64 evlist__combined_sample_type(struct evlist *evlist); +u64 evlist__combined_branch_type(struct evlist *evlist); +void evlist__update_br_cntr(struct evlist *evlist); +bool evlist__sample_id_all(struct evlist *evlist); +u16 evlist__id_hdr_size(struct evlist *evlist); + +int evlist__parse_sample(struct evlist *evlist, union perf_event *event, struct perf_sample *sample); +int evlist__parse_sample_timestamp(struct evlist *evlist, union perf_event *event, u64 *timestamp); + +bool evlist__valid_sample_type(struct evlist *evlist); +bool evlist__valid_sample_id_all(struct evlist *evlist); +bool evlist__valid_read_format(struct evlist *evlist); + +void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list); + +static inline bool evlist__empty(struct evlist *evlist) +{ + return list_empty(&evlist->core.entries); +} + +static inline struct evsel *evlist__first(struct evlist *evlist) +{ + struct perf_evsel *evsel = perf_evlist__first(&evlist->core); + + return container_of(evsel, struct evsel, core); +} + +static inline struct evsel *evlist__last(struct evlist *evlist) +{ + struct perf_evsel *evsel = perf_evlist__last(&evlist->core); + + return container_of(evsel, struct evsel, core); +} + +static inline int evlist__nr_groups(struct evlist *evlist) +{ + return perf_evlist__nr_groups(&evlist->core); +} + +int evlist__strerror_open(struct evlist *evlist, int err, char *buf, size_t size); +int evlist__strerror_mmap(struct evlist *evlist, int err, char *buf, size_t size); + +bool evlist__can_select_event(struct evlist *evlist, const char *str); +void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel); + +/** + * __evlist__for_each_entry - iterate thru all the evsels + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry(list, evsel) \ + list_for_each_entry(evsel, list, core.node) + +/** + * evlist__for_each_entry - iterate thru all the evsels + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry(evlist, evsel) \ + __evlist__for_each_entry(&(evlist)->core.entries, evsel) + +/** + * __evlist__for_each_entry_continue - continue iteration thru all the evsels + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_continue(list, evsel) \ + list_for_each_entry_continue(evsel, list, core.node) + +/** + * evlist__for_each_entry_continue - continue iteration thru all the evsels + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry_continue(evlist, evsel) \ + __evlist__for_each_entry_continue(&(evlist)->core.entries, evsel) + +/** + * __evlist__for_each_entry_from - continue iteration from @evsel (included) + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_from(list, evsel) \ + list_for_each_entry_from(evsel, list, core.node) + +/** + * evlist__for_each_entry_from - continue iteration from @evsel (included) + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry_from(evlist, evsel) \ + __evlist__for_each_entry_from(&(evlist)->core.entries, evsel) + +/** + * __evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order + * @list: list_head instance to iterate + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_reverse(list, evsel) \ + list_for_each_entry_reverse(evsel, list, core.node) + +/** + * evlist__for_each_entry_reverse - iterate thru all the evsels in reverse order + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + */ +#define evlist__for_each_entry_reverse(evlist, evsel) \ + __evlist__for_each_entry_reverse(&(evlist)->core.entries, evsel) + +/** + * __evlist__for_each_entry_safe - safely iterate thru all the evsels + * @list: list_head instance to iterate + * @tmp: struct evsel temp iterator + * @evsel: struct evsel iterator + */ +#define __evlist__for_each_entry_safe(list, tmp, evsel) \ + list_for_each_entry_safe(evsel, tmp, list, core.node) + +/** + * evlist__for_each_entry_safe - safely iterate thru all the evsels + * @evlist: evlist instance to iterate + * @evsel: struct evsel iterator + * @tmp: struct evsel temp iterator + */ +#define evlist__for_each_entry_safe(evlist, tmp, evsel) \ + __evlist__for_each_entry_safe(&(evlist)->core.entries, tmp, evsel) + +/** Iterator state for evlist__for_each_cpu */ +struct evlist_cpu_iterator { + /** The list being iterated through. */ + struct evlist *container; + /** The current evsel of the iterator. */ + struct evsel *evsel; + /** The CPU map index corresponding to the evsel->core.cpus for the current CPU. */ + int cpu_map_idx; + /** + * The CPU map index corresponding to evlist->core.all_cpus for the + * current CPU. Distinct from cpu_map_idx as the evsel's cpu map may + * contain fewer entries. + */ + int evlist_cpu_map_idx; + /** The number of CPU map entries in evlist->core.all_cpus. */ + int evlist_cpu_map_nr; + /** The current CPU of the iterator. */ + struct perf_cpu cpu; + /** If present, used to set the affinity when switching between CPUs. */ + struct affinity *affinity; + /** Maybe be used to hold affinity state prior to iterating. */ + struct affinity saved_affinity; +}; + +/** + * evlist__for_each_cpu - without affinity, iterate over the evlist. With + * affinity, iterate over all CPUs and then the evlist + * for each evsel on that CPU. When switching between + * CPUs the affinity is set to the CPU to avoid IPIs + * during syscalls. The affinity is set up and removed + * automatically, if the loop is broken a call to + * evlist_cpu_iterator__exit is necessary. + * @evlist_cpu_itr: the iterator instance. + * @evlist: evlist instance to iterate. + */ +#define evlist__for_each_cpu(evlist_cpu_itr, evlist) \ + for (evlist_cpu_iterator__init(&(evlist_cpu_itr), evlist); \ + !evlist_cpu_iterator__end(&evlist_cpu_itr); \ + evlist_cpu_iterator__next(&evlist_cpu_itr)) + +/** Setup an iterator set to the first CPU/evsel of evlist. */ +void evlist_cpu_iterator__init(struct evlist_cpu_iterator *itr, struct evlist *evlist); +/** + * Cleans up the iterator, automatically done by evlist_cpu_iterator__next when + * the end of the list is reached. Multiple calls are safe. + */ +void evlist_cpu_iterator__exit(struct evlist_cpu_iterator *itr); +/** Move to next element in iterator, updating CPU, evsel and the affinity. */ +void evlist_cpu_iterator__next(struct evlist_cpu_iterator *evlist_cpu_itr); +/** Returns true when iterator is at the end of the CPUs and evlist. */ +static inline bool evlist_cpu_iterator__end(const struct evlist_cpu_iterator *evlist_cpu_itr) +{ + return evlist_cpu_itr->evlist_cpu_map_idx >= evlist_cpu_itr->evlist_cpu_map_nr; +} + +struct evsel *evlist__get_tracking_event(struct evlist *evlist); +void evlist__set_tracking_event(struct evlist *evlist, struct evsel *tracking_evsel); +struct evsel *evlist__findnew_tracking_event(struct evlist *evlist, bool system_wide); + +struct evsel *evlist__find_evsel_by_str(struct evlist *evlist, const char *str); + +struct evsel *evlist__event2evsel(struct evlist *evlist, union perf_event *event); + +bool evlist__exclude_kernel(struct evlist *evlist); + +void evlist__force_leader(struct evlist *evlist); + +struct evsel *evlist__reset_weak_group(struct evlist *evlist, struct evsel *evsel, bool close); + +#define EVLIST_CTL_CMD_ENABLE_TAG "enable" +#define EVLIST_CTL_CMD_DISABLE_TAG "disable" +#define EVLIST_CTL_CMD_ACK_TAG "ack\n" +#define EVLIST_CTL_CMD_SNAPSHOT_TAG "snapshot" +#define EVLIST_CTL_CMD_EVLIST_TAG "evlist" +#define EVLIST_CTL_CMD_STOP_TAG "stop" +#define EVLIST_CTL_CMD_PING_TAG "ping" + +#define EVLIST_CTL_CMD_MAX_LEN 64 + +enum evlist_ctl_cmd { + EVLIST_CTL_CMD_UNSUPPORTED = 0, + EVLIST_CTL_CMD_ENABLE, + EVLIST_CTL_CMD_DISABLE, + EVLIST_CTL_CMD_ACK, + EVLIST_CTL_CMD_SNAPSHOT, + EVLIST_CTL_CMD_EVLIST, + EVLIST_CTL_CMD_STOP, + EVLIST_CTL_CMD_PING, +}; + +int evlist__parse_control(const char *str, int *ctl_fd, int *ctl_fd_ack, bool *ctl_fd_close); +void evlist__close_control(int ctl_fd, int ctl_fd_ack, bool *ctl_fd_close); +int evlist__initialize_ctlfd(struct evlist *evlist, int ctl_fd, int ctl_fd_ack); +int evlist__finalize_ctlfd(struct evlist *evlist); +bool evlist__ctlfd_initialized(struct evlist *evlist); +int evlist__ctlfd_process(struct evlist *evlist, enum evlist_ctl_cmd *cmd); +int evlist__ctlfd_ack(struct evlist *evlist); + +#define EVLIST_ENABLED_MSG "Events enabled\n" +#define EVLIST_DISABLED_MSG "Events disabled\n" + +int evlist__parse_event_enable_time(struct evlist *evlist, struct record_opts *opts, + const char *str, int unset); +int event_enable_timer__start(struct event_enable_timer *eet); +void event_enable_timer__exit(struct event_enable_timer **ep); +int event_enable_timer__process(struct event_enable_timer *eet); + +struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); + +void evlist__format_evsels(struct evlist *evlist, struct strbuf *sb, size_t max_length); +void evlist__check_mem_load_aux(struct evlist *evlist); +void evlist__warn_user_requested_cpus(struct evlist *evlist, const char *cpu_list); +void evlist__uniquify_evsel_names(struct evlist *evlist, const struct perf_stat_config *config); +bool evlist__has_bpf_output(struct evlist *evlist); +bool evlist__needs_bpf_sb_event(struct evlist *evlist); + +#endif /* __PERF_EVLIST_H */ diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c new file mode 100644 index 000000000..ea9fa0442 --- /dev/null +++ b/tools/perf/util/evsel.c @@ -0,0 +1,4667 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2011, Red Hat Inc, Arnaldo Carvalho de Melo + * + * Parts came from builtin-{top,stat,record}.c, see those files for further + * copyright notes. + */ +/* + * Powerpc needs __SANE_USERSPACE_TYPES__ before to select + * 'int-ll64.h' and avoid compile warnings when printing __u64 with %llu. + */ +#define __SANE_USERSPACE_TYPES__ + +#include "evsel.h" + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../perf-sys.h" +#include "asm/bug.h" +#include "bpf-filter.h" +#include "bpf_counter.h" +#include "callchain.h" +#include "cgroup.h" +#include "counts.h" +#include "debug.h" +#include "drm_pmu.h" +#include "dwarf-regs.h" +#include "env.h" +#include "event.h" +#include "evlist.h" +#include "evsel_config.h" +#include "evsel_fprintf.h" +#include "hashmap.h" +#include "hist.h" +#include "hwmon_pmu.h" +#include "intel-tpebs.h" +#include "memswap.h" +#include "off_cpu.h" +#include "parse-branch-options.h" +#include "perf_regs.h" +#include "pmu.h" +#include "pmus.h" +#include "record.h" +#include "rlimit.h" +#include "session.h" +#include "stat.h" +#include "string2.h" +#include "target.h" +#include "thread_map.h" +#include "time-utils.h" +#include "tool_pmu.h" +#include "tp_pmu.h" +#include "trace-event.h" +#include "util.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +struct perf_missing_features perf_missing_features; + +static clockid_t clockid; + +static int evsel__no_extra_init(struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static bool test_attr__enabled(void) +{ + static bool test_attr__enabled; + static bool test_attr__enabled_tested; + + if (!test_attr__enabled_tested) { + char *dir = getenv("PERF_TEST_ATTR"); + + test_attr__enabled = (dir != NULL); + test_attr__enabled_tested = true; + } + return test_attr__enabled; +} + +#define __WRITE_ASS(str, fmt, data) \ +do { \ + if (fprintf(file, #str "=%"fmt "\n", data) < 0) { \ + perror("test attr - failed to write event file"); \ + fclose(file); \ + return -1; \ + } \ +} while (0) + +#define WRITE_ASS(field, fmt) __WRITE_ASS(field, fmt, attr->field) + +static int store_event(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, + int fd, int group_fd, unsigned long flags) +{ + FILE *file; + char path[PATH_MAX]; + char *dir = getenv("PERF_TEST_ATTR"); + + snprintf(path, PATH_MAX, "%s/event-%d-%llu-%d", dir, + attr->type, attr->config, fd); + + file = fopen(path, "w+"); + if (!file) { + perror("test attr - failed to open event file"); + return -1; + } + + if (fprintf(file, "[event-%d-%llu-%d]\n", + attr->type, attr->config, fd) < 0) { + perror("test attr - failed to write event file"); + fclose(file); + return -1; + } + + /* syscall arguments */ + __WRITE_ASS(fd, "d", fd); + __WRITE_ASS(group_fd, "d", group_fd); + __WRITE_ASS(cpu, "d", cpu.cpu); + __WRITE_ASS(pid, "d", pid); + __WRITE_ASS(flags, "lu", flags); + + /* struct perf_event_attr */ + WRITE_ASS(type, PRIu32); + WRITE_ASS(size, PRIu32); + WRITE_ASS(config, "llu"); + WRITE_ASS(sample_period, "llu"); + WRITE_ASS(sample_type, "llu"); + WRITE_ASS(read_format, "llu"); + WRITE_ASS(disabled, "d"); + WRITE_ASS(inherit, "d"); + WRITE_ASS(pinned, "d"); + WRITE_ASS(exclusive, "d"); + WRITE_ASS(exclude_user, "d"); + WRITE_ASS(exclude_kernel, "d"); + WRITE_ASS(exclude_hv, "d"); + WRITE_ASS(exclude_idle, "d"); + WRITE_ASS(mmap, "d"); + WRITE_ASS(comm, "d"); + WRITE_ASS(freq, "d"); + WRITE_ASS(inherit_stat, "d"); + WRITE_ASS(enable_on_exec, "d"); + WRITE_ASS(task, "d"); + WRITE_ASS(watermark, "d"); + WRITE_ASS(precise_ip, "d"); + WRITE_ASS(mmap_data, "d"); + WRITE_ASS(sample_id_all, "d"); + WRITE_ASS(exclude_host, "d"); + WRITE_ASS(exclude_guest, "d"); + WRITE_ASS(exclude_callchain_kernel, "d"); + WRITE_ASS(exclude_callchain_user, "d"); + WRITE_ASS(mmap2, "d"); + WRITE_ASS(comm_exec, "d"); + WRITE_ASS(context_switch, "d"); + WRITE_ASS(write_backward, "d"); + WRITE_ASS(namespaces, "d"); + WRITE_ASS(use_clockid, "d"); + WRITE_ASS(wakeup_events, PRIu32); + WRITE_ASS(bp_type, PRIu32); + WRITE_ASS(config1, "llu"); + WRITE_ASS(config2, "llu"); + WRITE_ASS(branch_sample_type, "llu"); + WRITE_ASS(sample_regs_user, "llu"); + WRITE_ASS(sample_stack_user, PRIu32); + + fclose(file); + return 0; +} + +#undef __WRITE_ASS +#undef WRITE_ASS + +static void test_attr__open(struct perf_event_attr *attr, pid_t pid, struct perf_cpu cpu, + int fd, int group_fd, unsigned long flags) +{ + int errno_saved = errno; + + if ((fd != -1) && store_event(attr, pid, cpu, fd, group_fd, flags)) { + pr_err("test attr FAILED"); + exit(128); + } + + errno = errno_saved; +} + +static void evsel__no_extra_fini(struct evsel *evsel __maybe_unused) +{ +} + +static struct { + size_t size; + int (*init)(struct evsel *evsel); + void (*fini)(struct evsel *evsel); +} perf_evsel__object = { + .size = sizeof(struct evsel), + .init = evsel__no_extra_init, + .fini = evsel__no_extra_fini, +}; + +int evsel__object_config(size_t object_size, int (*init)(struct evsel *evsel), + void (*fini)(struct evsel *evsel)) +{ + + if (object_size == 0) + goto set_methods; + + if (perf_evsel__object.size > object_size) + return -EINVAL; + + perf_evsel__object.size = object_size; + +set_methods: + if (init != NULL) + perf_evsel__object.init = init; + + if (fini != NULL) + perf_evsel__object.fini = fini; + + return 0; +} + +const char *evsel__pmu_name(const struct evsel *evsel) +{ + struct perf_pmu *pmu = evsel__find_pmu(evsel); + + if (pmu) + return pmu->name; + + return event_type(evsel->core.attr.type); +} + +enum evsel_probe_type { + PROBE__UNKNOWN = 0, + PROBE__NOPE = 1, + PROBE__KPROBE = 2, + PROBE__UPROBE = 3, + /* + * Ftrace-based dynamic probes (kprobes/uprobes/fprobes created via + * tracefs) report PMU "tracepoint", not "kprobe"/"uprobe". Detect + * them by the __probe_ip field that the kernel adds to all dynamic + * probe formats. + */ + PROBE__FTRACE = 4, +}; + +static void evsel__resolve_probe_type(struct evsel *evsel) +{ + const char *name = evsel__pmu_name(evsel); + + if (!strcmp(name, "kprobe")) + evsel->probe_type = PROBE__KPROBE; + else if (!strcmp(name, "uprobe")) + evsel->probe_type = PROBE__UPROBE; + else if (!strcmp(name, "tracepoint") && evsel__field(evsel, "__probe_ip")) + evsel->probe_type = PROBE__FTRACE; + else + evsel->probe_type = PROBE__NOPE; +} + +bool evsel__is_probe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type > PROBE__NOPE; +} + +bool evsel__is_kprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__KPROBE; +} + +bool evsel__is_uprobe(struct evsel *evsel) +{ + if (evsel->probe_type == PROBE__UNKNOWN) + evsel__resolve_probe_type(evsel); + + return evsel->probe_type == PROBE__UPROBE; +} + +#define FD(e, x, y) (*(int *)xyarray__entry(e->core.fd, x, y)) + +int __evsel__sample_size(u64 sample_type) +{ + u64 mask = sample_type & PERF_SAMPLE_MASK; + int size = 0; + int i; + + for (i = 0; i < 64; i++) { + if (mask & (1ULL << i)) + size++; + } + + size *= sizeof(u64); + + return size; +} + +/** + * __perf_evsel__calc_id_pos - calculate id_pos. + * @sample_type: sample type + * + * This function returns the position of the event id (PERF_SAMPLE_ID or + * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of struct + * perf_record_sample. + */ +static int __perf_evsel__calc_id_pos(u64 sample_type) +{ + int idx = 0; + + if (sample_type & PERF_SAMPLE_IDENTIFIER) + return 0; + + if (!(sample_type & PERF_SAMPLE_ID)) + return -1; + + if (sample_type & PERF_SAMPLE_IP) + idx += 1; + + if (sample_type & PERF_SAMPLE_TID) + idx += 1; + + if (sample_type & PERF_SAMPLE_TIME) + idx += 1; + + if (sample_type & PERF_SAMPLE_ADDR) + idx += 1; + + return idx; +} + +/** + * __perf_evsel__calc_is_pos - calculate is_pos. + * @sample_type: sample type + * + * This function returns the position (counting backwards) of the event id + * (PERF_SAMPLE_ID or PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if + * sample_id_all is used there is an id sample appended to non-sample events. + */ +static int __perf_evsel__calc_is_pos(u64 sample_type) +{ + int idx = 1; + + if (sample_type & PERF_SAMPLE_IDENTIFIER) + return 1; + + if (!(sample_type & PERF_SAMPLE_ID)) + return -1; + + if (sample_type & PERF_SAMPLE_CPU) + idx += 1; + + if (sample_type & PERF_SAMPLE_STREAM_ID) + idx += 1; + + return idx; +} + +void evsel__calc_id_pos(struct evsel *evsel) +{ + evsel->id_pos = __perf_evsel__calc_id_pos(evsel->core.attr.sample_type); + evsel->is_pos = __perf_evsel__calc_is_pos(evsel->core.attr.sample_type); +} + +void __evsel__set_sample_bit(struct evsel *evsel, + enum perf_event_sample_format bit) +{ + if (!(evsel->core.attr.sample_type & bit)) { + evsel->core.attr.sample_type |= bit; + evsel->sample_size += sizeof(u64); + evsel__calc_id_pos(evsel); + } +} + +void __evsel__reset_sample_bit(struct evsel *evsel, + enum perf_event_sample_format bit) +{ + if (evsel->core.attr.sample_type & bit) { + evsel->core.attr.sample_type &= ~bit; + evsel->sample_size -= sizeof(u64); + evsel__calc_id_pos(evsel); + } +} + +void evsel__set_sample_id(struct evsel *evsel, + bool can_sample_identifier) +{ + if (can_sample_identifier) { + evsel__reset_sample_bit(evsel, ID); + evsel__set_sample_bit(evsel, IDENTIFIER); + } else { + evsel__set_sample_bit(evsel, ID); + } + evsel->core.attr.read_format |= PERF_FORMAT_ID; +} + +/** + * evsel__is_function_event - Return whether given evsel is a function + * trace event + * + * @evsel - evsel selector to be tested + * + * Return %true if event is function trace event + */ +bool evsel__is_function_event(struct evsel *evsel) +{ +#define FUNCTION_EVENT "ftrace:function" + + return evsel->name && + !strncmp(FUNCTION_EVENT, evsel->name, sizeof(FUNCTION_EVENT)); + +#undef FUNCTION_EVENT +} + +void evsel__init(struct evsel *evsel, + struct perf_event_attr *attr, int idx) +{ + perf_evsel__init(&evsel->core, attr, idx); + evsel->tracking = !idx; + evsel->unit = strdup(""); + evsel->scale = 1.0; + evsel->max_events = ULONG_MAX; + evsel->evlist = NULL; + evsel->bpf_obj = NULL; + evsel->bpf_fd = -1; + INIT_LIST_HEAD(&evsel->config_terms); + INIT_LIST_HEAD(&evsel->bpf_counter_list); + INIT_LIST_HEAD(&evsel->bpf_filters); + perf_evsel__object.init(evsel); + evsel->sample_size = __evsel__sample_size(attr->sample_type); + evsel__calc_id_pos(evsel); + evsel->cmdline_group_boundary = false; + evsel->per_pkg_mask = NULL; + evsel->collect_stat = false; + evsel->group_pmu_name = NULL; + evsel->skippable = false; + evsel->supported = true; + evsel->alternate_hw_config = PERF_COUNT_HW_MAX; + evsel->script_output_type = -1; // FIXME: OUTPUT_TYPE_UNSET, see builtin-script.c + evsel->probe_type = PROBE__UNKNOWN; +} + +struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx) +{ + struct evsel *evsel = zalloc(perf_evsel__object.size); + + if (!evsel) + return NULL; + evsel__init(evsel, attr, idx); + + if (evsel__is_bpf_output(evsel) && !attr->sample_type) { + evsel->core.attr.sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME | + PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD), + evsel->core.attr.sample_period = 1; + } + + if (evsel__is_clock(evsel)) { + free((char *)evsel->unit); + evsel->unit = strdup("msec"); + evsel->scale = 1e-6; + } + + return evsel; +} + +int copy_config_terms(struct list_head *dst, struct list_head *src) +{ + struct evsel_config_term *pos, *tmp; + + list_for_each_entry(pos, src, list) { + tmp = malloc(sizeof(*tmp)); + if (tmp == NULL) + return -ENOMEM; + + *tmp = *pos; + if (tmp->free_str) { + tmp->val.str = strdup(pos->val.str); + if (tmp->val.str == NULL) { + free(tmp); + return -ENOMEM; + } + } + list_add_tail(&tmp->list, dst); + } + return 0; +} + +static int evsel__copy_config_terms(struct evsel *dst, struct evsel *src) +{ + return copy_config_terms(&dst->config_terms, &src->config_terms); +} + +/** + * evsel__clone - create a new evsel copied from @orig + * @orig: original evsel + * + * The assumption is that @orig is not configured nor opened yet. + * So we only care about the attributes that can be set while it's parsed. + */ +struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig) +{ + struct evsel *evsel; + + BUG_ON(orig->core.fd); + BUG_ON(orig->counts); + BUG_ON(orig->priv); + BUG_ON(orig->per_pkg_mask); + + /* cannot handle BPF objects for now */ + if (orig->bpf_obj) + return NULL; + + if (dest) + evsel = dest; + else + evsel = evsel__new(&orig->core.attr); + + if (evsel == NULL) + return NULL; + + evsel->core.cpus = perf_cpu_map__get(orig->core.cpus); + evsel->core.pmu_cpus = perf_cpu_map__get(orig->core.pmu_cpus); + evsel->core.threads = perf_thread_map__get(orig->core.threads); + evsel->core.nr_members = orig->core.nr_members; + evsel->core.system_wide = orig->core.system_wide; + evsel->core.requires_cpu = orig->core.requires_cpu; + evsel->core.is_pmu_core = orig->core.is_pmu_core; + + if (orig->name) { + evsel->name = strdup(orig->name); + if (evsel->name == NULL) + goto out_err; + } + if (orig->group_name) { + evsel->group_name = strdup(orig->group_name); + if (evsel->group_name == NULL) + goto out_err; + } + if (orig->group_pmu_name) { + evsel->group_pmu_name = strdup(orig->group_pmu_name); + if (evsel->group_pmu_name == NULL) + goto out_err; + } + if (orig->filter) { + evsel->filter = strdup(orig->filter); + if (evsel->filter == NULL) + goto out_err; + } + if (orig->metric_id) { + evsel->metric_id = strdup(orig->metric_id); + if (evsel->metric_id == NULL) + goto out_err; + } + evsel->cgrp = cgroup__get(orig->cgrp); +#ifdef HAVE_LIBTRACEEVENT + if (orig->tp_sys) { + evsel->tp_sys = strdup(orig->tp_sys); + if (evsel->tp_sys == NULL) + goto out_err; + } + if (orig->tp_name) { + evsel->tp_name = strdup(orig->tp_name); + if (evsel->tp_name == NULL) + goto out_err; + } + evsel->tp_format = orig->tp_format; +#endif + evsel->handler = orig->handler; + evsel->core.leader = orig->core.leader; + evsel->metric_leader = orig->metric_leader; + + evsel->max_events = orig->max_events; + zfree(&evsel->unit); + if (orig->unit) { + evsel->unit = strdup(orig->unit); + if (evsel->unit == NULL) + goto out_err; + } + evsel->scale = orig->scale; + evsel->snapshot = orig->snapshot; + evsel->per_pkg = orig->per_pkg; + evsel->percore = orig->percore; + evsel->precise_max = orig->precise_max; + evsel->is_libpfm_event = orig->is_libpfm_event; + + evsel->exclude_GH = orig->exclude_GH; + evsel->sample_read = orig->sample_read; + evsel->collect_stat = orig->collect_stat; + evsel->weak_group = orig->weak_group; + evsel->use_config_name = orig->use_config_name; + evsel->pmu = orig->pmu; + evsel->first_wildcard_match = orig->first_wildcard_match; + + if (evsel__copy_config_terms(evsel, orig) < 0) + goto out_err; + + evsel->alternate_hw_config = orig->alternate_hw_config; + + return evsel; + +out_err: + evsel__delete(evsel); + return NULL; +} + +/* + * Returns pointer with encoded error via interface. + */ +struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool format) +{ + struct perf_event_attr attr = { + .type = PERF_TYPE_TRACEPOINT, + .sample_type = (PERF_SAMPLE_RAW | PERF_SAMPLE_TIME | + PERF_SAMPLE_CPU | PERF_SAMPLE_PERIOD), + }; + struct evsel *evsel = zalloc(perf_evsel__object.size); + int err = -ENOMEM, id = -1; + + if (evsel == NULL) + goto out_err; + + + if (asprintf(&evsel->name, "%s:%s", sys, name) < 0) + goto out_free; + +#ifdef HAVE_LIBTRACEEVENT + evsel->tp_sys = strdup(sys); + if (!evsel->tp_sys) + goto out_free; + + evsel->tp_name = strdup(name); + if (!evsel->tp_name) + goto out_free; +#endif + + event_attr_init(&attr); + + if (format) { + id = tp_pmu__id(sys, name); + if (id < 0) { + err = id; + goto out_free; + } + } + attr.config = (__u64)id; + attr.sample_period = 1; + evsel__init(evsel, &attr, idx); + return evsel; + +out_free: + zfree(&evsel->name); +#ifdef HAVE_LIBTRACEEVENT + zfree(&evsel->tp_sys); + zfree(&evsel->tp_name); +#endif + free(evsel); +out_err: + return ERR_PTR(err); +} + +#ifdef HAVE_LIBTRACEEVENT +struct tep_event *evsel__tp_format(struct evsel *evsel) +{ + struct tep_event *tp_format = evsel->tp_format; + + if (tp_format) + return tp_format; + + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) + return NULL; + + if (!evsel->tp_sys) + tp_format = trace_event__tp_format_id(evsel->core.attr.config); + else + tp_format = trace_event__tp_format(evsel->tp_sys, evsel->tp_name); + + if (IS_ERR(tp_format)) { + int err = -PTR_ERR(evsel->tp_format); + + errno = err; + pr_err("Error getting tracepoint format '%s': %m\n", + evsel__name(evsel)); + return NULL; + } + evsel->tp_format = tp_format; + return evsel->tp_format; +} +#endif + +const char *const evsel__hw_names[PERF_COUNT_HW_MAX] = { + "cycles", + "instructions", + "cache-references", + "cache-misses", + "branches", + "branch-misses", + "bus-cycles", + "stalled-cycles-frontend", + "stalled-cycles-backend", + "ref-cycles", +}; + +char *evsel__bpf_counter_events; + +bool evsel__match_bpf_counter_events(const char *name) +{ + int name_len; + bool match; + char *ptr; + + if (!evsel__bpf_counter_events) + return false; + + ptr = strstr(evsel__bpf_counter_events, name); + name_len = strlen(name); + + /* check name matches a full token in evsel__bpf_counter_events */ + match = (ptr != NULL) && + ((ptr == evsel__bpf_counter_events) || (*(ptr - 1) == ',')) && + ((*(ptr + name_len) == ',') || (*(ptr + name_len) == '\0')); + + return match; +} + +static const char *__evsel__hw_name(u64 config) +{ + if (config < PERF_COUNT_HW_MAX && evsel__hw_names[config]) + return evsel__hw_names[config]; + + return "unknown-hardware"; +} + +static int evsel__add_modifiers(struct evsel *evsel, char *bf, size_t size) +{ + int colon = 0, r = 0; + struct perf_event_attr *attr = &evsel->core.attr; + +#define MOD_PRINT(context, mod) do { \ + if (!attr->exclude_##context) { \ + if (!colon) colon = ++r; \ + r += scnprintf(bf + r, size - r, "%c", mod); \ + } } while(0) + + if (attr->exclude_kernel || attr->exclude_user || attr->exclude_hv) { + MOD_PRINT(kernel, 'k'); + MOD_PRINT(user, 'u'); + MOD_PRINT(hv, 'h'); + } + + if (attr->precise_ip) { + if (!colon) + colon = ++r; + r += scnprintf(bf + r, size - r, "%.*s", attr->precise_ip, "ppp"); + } + + if (attr->exclude_host || attr->exclude_guest) { + MOD_PRINT(host, 'H'); + MOD_PRINT(guest, 'G'); + } +#undef MOD_PRINT + if (colon) + bf[colon - 1] = ':'; + return r; +} + +int __weak arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size) +{ + return scnprintf(bf, size, "%s", __evsel__hw_name(evsel->core.attr.config)); +} + +static int evsel__hw_name(struct evsel *evsel, char *bf, size_t size) +{ + int r = arch_evsel__hw_name(evsel, bf, size); + return r + evsel__add_modifiers(evsel, bf + r, size - r); +} + +const char *const evsel__sw_names[PERF_COUNT_SW_MAX] = { + "cpu-clock", + "task-clock", + "page-faults", + "context-switches", + "cpu-migrations", + "minor-faults", + "major-faults", + "alignment-faults", + "emulation-faults", + "dummy", +}; + +static const char *__evsel__sw_name(u64 config) +{ + if (config < PERF_COUNT_SW_MAX && evsel__sw_names[config]) + return evsel__sw_names[config]; + return "unknown-software"; +} + +static int evsel__sw_name(struct evsel *evsel, char *bf, size_t size) +{ + int r = scnprintf(bf, size, "%s", __evsel__sw_name(evsel->core.attr.config)); + return r + evsel__add_modifiers(evsel, bf + r, size - r); +} + +static int __evsel__bp_name(char *bf, size_t size, u64 addr, u64 type) +{ + int r; + + r = scnprintf(bf, size, "mem:0x%" PRIx64 ":", addr); + + if (type & HW_BREAKPOINT_R) + r += scnprintf(bf + r, size - r, "r"); + + if (type & HW_BREAKPOINT_W) + r += scnprintf(bf + r, size - r, "w"); + + if (type & HW_BREAKPOINT_X) + r += scnprintf(bf + r, size - r, "x"); + + return r; +} + +static int evsel__bp_name(struct evsel *evsel, char *bf, size_t size) +{ + struct perf_event_attr *attr = &evsel->core.attr; + int r = __evsel__bp_name(bf, size, attr->bp_addr, attr->bp_type); + return r + evsel__add_modifiers(evsel, bf + r, size - r); +} + +const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES] = { + { "L1-dcache", "l1-d", "l1d", "L1-data", }, + { "L1-icache", "l1-i", "l1i", "L1-instruction", }, + { "LLC", "L2", }, + { "dTLB", "d-tlb", "Data-TLB", }, + { "iTLB", "i-tlb", "Instruction-TLB", }, + { "branch", "branches", "bpu", "btb", "bpc", }, + { "node", }, +}; + +const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES] = { + { "load", "loads", "read", }, + { "store", "stores", "write", }, + { "prefetch", "prefetches", "speculative-read", "speculative-load", }, +}; + +const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES] = { + { "refs", "Reference", "ops", "access", }, + { "misses", "miss", }, +}; + +#define C(x) PERF_COUNT_HW_CACHE_##x +#define CACHE_READ (1 << C(OP_READ)) +#define CACHE_WRITE (1 << C(OP_WRITE)) +#define CACHE_PREFETCH (1 << C(OP_PREFETCH)) +#define COP(x) (1 << x) + +/* + * cache operation stat + * L1I : Read and prefetch only + * ITLB and BPU : Read-only + */ +static const unsigned long evsel__hw_cache_stat[C(MAX)] = { + [C(L1D)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), + [C(L1I)] = (CACHE_READ | CACHE_PREFETCH), + [C(LL)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), + [C(DTLB)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), + [C(ITLB)] = (CACHE_READ), + [C(BPU)] = (CACHE_READ), + [C(NODE)] = (CACHE_READ | CACHE_WRITE | CACHE_PREFETCH), +}; + +bool evsel__is_cache_op_valid(u8 type, u8 op) +{ + if (evsel__hw_cache_stat[type] & COP(op)) + return true; /* valid */ + else + return false; /* invalid */ +} + +int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size) +{ + if (result) { + return scnprintf(bf, size, "%s-%s-%s", evsel__hw_cache[type][0], + evsel__hw_cache_op[op][0], + evsel__hw_cache_result[result][0]); + } + + return scnprintf(bf, size, "%s-%s", evsel__hw_cache[type][0], + evsel__hw_cache_op[op][1]); +} + +static int __evsel__hw_cache_name(u64 config, char *bf, size_t size) +{ + u8 op, result, type = (config >> 0) & 0xff; + const char *err = "unknown-ext-hardware-cache-type"; + + if (type >= PERF_COUNT_HW_CACHE_MAX) + goto out_err; + + op = (config >> 8) & 0xff; + err = "unknown-ext-hardware-cache-op"; + if (op >= PERF_COUNT_HW_CACHE_OP_MAX) + goto out_err; + + result = (config >> 16) & 0xff; + err = "unknown-ext-hardware-cache-result"; + if (result >= PERF_COUNT_HW_CACHE_RESULT_MAX) + goto out_err; + + err = "invalid-cache"; + if (!evsel__is_cache_op_valid(type, op)) + goto out_err; + + return __evsel__hw_cache_type_op_res_name(type, op, result, bf, size); +out_err: + return scnprintf(bf, size, "%s", err); +} + +static int evsel__hw_cache_name(struct evsel *evsel, char *bf, size_t size) +{ + int ret = __evsel__hw_cache_name(evsel->core.attr.config, bf, size); + return ret + evsel__add_modifiers(evsel, bf + ret, size - ret); +} + +static int evsel__raw_name(struct evsel *evsel, char *bf, size_t size) +{ + int ret = scnprintf(bf, size, "raw 0x%" PRIx64, evsel->core.attr.config); + return ret + evsel__add_modifiers(evsel, bf + ret, size - ret); +} + +const char *evsel__name(struct evsel *evsel) +{ + char bf[128]; + + if (!evsel) + goto out_unknown; + + if (evsel->name) + return evsel->name; + + switch (evsel->core.attr.type) { + case PERF_TYPE_RAW: + evsel__raw_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_HARDWARE: + evsel__hw_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_HW_CACHE: + evsel__hw_cache_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_SOFTWARE: + evsel__sw_name(evsel, bf, sizeof(bf)); + break; + + case PERF_TYPE_TRACEPOINT: + scnprintf(bf, sizeof(bf), "unknown tracepoint id=%#"PRIx64, + evsel->core.attr.config); + break; + + case PERF_TYPE_BREAKPOINT: + evsel__bp_name(evsel, bf, sizeof(bf)); + break; + + case PERF_PMU_TYPE_TOOL: + scnprintf(bf, sizeof(bf), "%s", evsel__tool_pmu_event_name(evsel)); + break; + + default: + scnprintf(bf, sizeof(bf), "unknown event PMU=%d config=%#"PRIx64, + evsel->core.attr.type, evsel->core.attr.config); + break; + } + + evsel->name = strdup(bf); + + if (evsel->name) + return evsel->name; +out_unknown: + return "unknown"; +} + +bool evsel__name_is(struct evsel *evsel, const char *name) +{ + return !strcmp(evsel__name(evsel), name); +} + +const char *evsel__metric_id(const struct evsel *evsel) +{ + if (evsel->metric_id) + return evsel->metric_id; + + if (evsel__is_tool(evsel)) + return evsel__tool_pmu_event_name(evsel); + + return "unknown"; +} + +const char *evsel__group_name(struct evsel *evsel) +{ + return evsel->group_name ?: "anon group"; +} + +/* + * Returns the group details for the specified leader, + * with following rules. + * + * For record -e '{cycles,instructions}' + * 'anon group { cycles:u, instructions:u }' + * + * For record -e 'cycles,instructions' and report --group + * 'cycles:u, instructions:u' + */ +int evsel__group_desc(struct evsel *evsel, char *buf, size_t size) +{ + int ret = 0; + bool first = true; + struct evsel *pos; + const char *group_name = evsel__group_name(evsel); + + if (!evsel->forced_leader) + ret = scnprintf(buf, size, "%s { ", group_name); + + for_each_group_evsel(pos, evsel) { + if (symbol_conf.skip_empty && + evsel__hists(pos)->stats.nr_samples == 0) + continue; + + ret += scnprintf(buf + ret, size - ret, "%s%s", + first ? "" : ", ", evsel__name(pos)); + first = false; + } + + if (!evsel->forced_leader) + ret += scnprintf(buf + ret, size - ret, " }"); + + return ret; +} + +uint16_t evsel__e_machine(struct evsel *evsel, uint32_t *e_flags) +{ + struct perf_session *session = evsel__session(evsel); + + return perf_session__e_machine(session, e_flags); +} + +static void __evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *param) +{ + bool function = evsel__is_function_event(evsel); + struct perf_event_attr *attr = &evsel->core.attr; + + if (EM_HOST == EM_S390 && param->record_mode == CALLCHAIN_FP) { + pr_warning_once( + "Framepointer unwinding lacks kernel support. Use '--call-graph dwarf'\n"); + } + + evsel__set_sample_bit(evsel, CALLCHAIN); + + attr->sample_max_stack = param->max_stack; + + if (opts->kernel_callchains) + attr->exclude_callchain_user = 1; + if (opts->user_callchains) + attr->exclude_callchain_kernel = 1; + if (param->record_mode == CALLCHAIN_LBR) { + if (!opts->branch_stack) { + if (attr->exclude_user) { + pr_warning("LBR callstack option is only available " + "to get user callchain information. " + "Falling back to framepointers.\n"); + } else { + evsel__set_sample_bit(evsel, BRANCH_STACK); + attr->branch_sample_type = PERF_SAMPLE_BRANCH_USER | + PERF_SAMPLE_BRANCH_CALL_STACK | + PERF_SAMPLE_BRANCH_NO_CYCLES | + PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_HW_INDEX; + } + } else + pr_warning("Cannot use LBR callstack with branch stack. " + "Falling back to framepointers.\n"); + } + + if (param->record_mode == CALLCHAIN_DWARF) { + if (!function) { + uint16_t e_machine = evsel__e_machine(evsel, /*e_flags=*/NULL); + + evsel__set_sample_bit(evsel, REGS_USER); + evsel__set_sample_bit(evsel, STACK_USER); + if (opts->sample_user_regs && + DWARF_MINIMAL_REGS(e_machine) != perf_user_reg_mask(EM_HOST)) { + attr->sample_regs_user |= DWARF_MINIMAL_REGS(e_machine); + pr_warning("WARNING: The use of --call-graph=dwarf may require all the user registers, " + "specifying a subset with --user-regs may render DWARF unwinding unreliable, " + "so the minimal registers set (IP, SP) is explicitly forced.\n"); + } else { + attr->sample_regs_user |= perf_user_reg_mask(EM_HOST); + } + attr->sample_stack_user = param->dump_size; + attr->exclude_callchain_user = 1; + } else { + pr_info("Cannot use DWARF unwind for function trace event," + " falling back to framepointers.\n"); + } + } + + if (function) { + pr_info("Disabling user space callchains for function trace event.\n"); + attr->exclude_callchain_user = 1; + } + + if (param->defer && !attr->exclude_callchain_user) + attr->defer_callchain = 1; +} + +void evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *param) +{ + if (param->enabled) + return __evsel__config_callchain(evsel, opts, param); +} + +static void evsel__reset_callgraph(struct evsel *evsel, const struct callchain_param *param) +{ + struct perf_event_attr *attr = &evsel->core.attr; + + evsel__reset_sample_bit(evsel, CALLCHAIN); + if (param->record_mode == CALLCHAIN_LBR) { + evsel__reset_sample_bit(evsel, BRANCH_STACK); + attr->branch_sample_type &= ~(PERF_SAMPLE_BRANCH_USER | + PERF_SAMPLE_BRANCH_CALL_STACK | + PERF_SAMPLE_BRANCH_HW_INDEX); + } + if (param->record_mode == CALLCHAIN_DWARF) { + evsel__reset_sample_bit(evsel, REGS_USER); + evsel__reset_sample_bit(evsel, STACK_USER); + } +} + +static void evsel__apply_ratio_to_prev(struct evsel *evsel, + struct perf_event_attr *attr, + const struct record_opts *opts, + const char *buf) +{ + struct perf_event_attr *prev_attr = NULL; + struct evsel *evsel_prev = NULL; + u64 type = evsel->core.attr.sample_type; + u64 prev_type = 0; + double rtp; + + rtp = strtod(buf, NULL); + if (rtp <= 0) { + pr_err("Invalid ratio-to-prev value %lf\n", rtp); + return; + } + if (evsel == evsel__leader(evsel)) { + pr_err("Invalid use of ratio-to-prev term without preceding element in group\n"); + return; + } + if (!evsel->pmu->is_core) { + pr_err("Event using ratio-to-prev term must have a core PMU\n"); + return; + } + + evsel_prev = evsel__prev(evsel); + if (!evsel_prev) { + pr_err("Previous event does not exist.\n"); + return; + } + + if (evsel_prev->pmu->type != evsel->pmu->type) { + pr_err("Compared events (\"%s\", \"%s\") must have same PMU\n", + evsel->name, evsel_prev->name); + return; + } + + prev_attr = &evsel_prev->core.attr; + prev_type = evsel_prev->core.attr.sample_type; + + if (!(prev_type & PERF_SAMPLE_PERIOD)) { + attr->sample_period = prev_attr->sample_period * rtp; + attr->freq = 0; + evsel__reset_sample_bit(evsel, PERIOD); + } else if (!(type & PERF_SAMPLE_PERIOD)) { + prev_attr->sample_period = attr->sample_period / rtp; + prev_attr->freq = 0; + evsel__reset_sample_bit(evsel_prev, PERIOD); + } else { + if (opts->user_interval != ULLONG_MAX) { + prev_attr->sample_period = opts->user_interval; + attr->sample_period = prev_attr->sample_period * rtp; + prev_attr->freq = 0; + attr->freq = 0; + evsel__reset_sample_bit(evsel_prev, PERIOD); + evsel__reset_sample_bit(evsel, PERIOD); + } else { + pr_err("Event period term or count (-c) must be set when using ratio-to-prev term.\n"); + return; + } + } + + arch_evsel__apply_ratio_to_prev(evsel, attr); +} + +static void evsel__apply_config_terms(struct evsel *evsel, + const struct record_opts *opts, bool track) +{ + struct evsel_config_term *term; + struct list_head *config_terms = &evsel->config_terms; + struct perf_event_attr *attr = &evsel->core.attr; + /* callgraph default */ + struct callchain_param param = { + .record_mode = callchain_param.record_mode, + }; + u32 dump_size = 0; + int max_stack = 0; + const char *callgraph_buf = NULL; + const char *rtp_buf = NULL; + + list_for_each_entry(term, config_terms, list) { + switch (term->type) { + case EVSEL__CONFIG_TERM_PERIOD: + if (!(term->weak && opts->user_interval != ULLONG_MAX)) { + attr->sample_period = term->val.period; + attr->freq = 0; + evsel__reset_sample_bit(evsel, PERIOD); + } + break; + case EVSEL__CONFIG_TERM_FREQ: + if (!(term->weak && opts->user_freq != UINT_MAX)) { + attr->sample_freq = term->val.freq; + attr->freq = 1; + evsel__set_sample_bit(evsel, PERIOD); + } + break; + case EVSEL__CONFIG_TERM_TIME: + if (term->val.time) + evsel__set_sample_bit(evsel, TIME); + else + evsel__reset_sample_bit(evsel, TIME); + break; + case EVSEL__CONFIG_TERM_CALLGRAPH: + callgraph_buf = term->val.str; + break; + case EVSEL__CONFIG_TERM_BRANCH: + if (term->val.str && strcmp(term->val.str, "no")) { + evsel__set_sample_bit(evsel, BRANCH_STACK); + parse_branch_str(term->val.str, + &attr->branch_sample_type); + } else + evsel__reset_sample_bit(evsel, BRANCH_STACK); + break; + case EVSEL__CONFIG_TERM_STACK_USER: + dump_size = term->val.stack_user; + break; + case EVSEL__CONFIG_TERM_MAX_STACK: + max_stack = term->val.max_stack; + break; + case EVSEL__CONFIG_TERM_MAX_EVENTS: + evsel->max_events = term->val.max_events; + break; + case EVSEL__CONFIG_TERM_INHERIT: + /* + * attr->inherit should has already been set by + * evsel__config. If user explicitly set + * inherit using config terms, override global + * opt->no_inherit setting. + */ + attr->inherit = term->val.inherit ? 1 : 0; + break; + case EVSEL__CONFIG_TERM_OVERWRITE: + attr->write_backward = term->val.overwrite ? 1 : 0; + break; + case EVSEL__CONFIG_TERM_DRV_CFG: + break; + case EVSEL__CONFIG_TERM_PERCORE: + break; + case EVSEL__CONFIG_TERM_AUX_OUTPUT: + attr->aux_output = term->val.aux_output ? 1 : 0; + break; + case EVSEL__CONFIG_TERM_AUX_ACTION: + /* Already applied by auxtrace */ + break; + case EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE: + /* Already applied by auxtrace */ + break; + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG1: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG2: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG3: + case EVSEL__CONFIG_TERM_USR_CHG_CONFIG4: + break; + case EVSEL__CONFIG_TERM_RATIO_TO_PREV: + rtp_buf = term->val.str; + break; + default: + break; + } + } + + /* User explicitly set per-event callgraph, clear the old setting and reset. */ + if ((callgraph_buf != NULL) || (dump_size > 0) || max_stack) { + bool sample_address = false; + + if (max_stack) { + param.max_stack = max_stack; + if (callgraph_buf == NULL) + callgraph_buf = "fp"; + } + + /* parse callgraph parameters */ + if (callgraph_buf != NULL) { + if (!strcmp(callgraph_buf, "no")) { + param.enabled = false; + param.record_mode = CALLCHAIN_NONE; + } else { + param.enabled = true; + if (parse_callchain_record(callgraph_buf, ¶m)) { + pr_err("per-event callgraph setting for %s failed. " + "Apply callgraph global setting for it\n", + evsel->name); + return; + } + if (param.record_mode == CALLCHAIN_DWARF) + sample_address = true; + } + } + if (dump_size > 0) { + dump_size = round_up(dump_size, sizeof(u64)); + param.dump_size = dump_size; + } + + /* If global callgraph set, clear it */ + if (callchain_param.enabled) + evsel__reset_callgraph(evsel, &callchain_param); + + /* set perf-event callgraph */ + if (param.enabled) { + if (sample_address) { + evsel__set_sample_bit(evsel, ADDR); + evsel__set_sample_bit(evsel, DATA_SRC); + evsel->core.attr.mmap_data = track; + } + evsel__config_callchain(evsel, opts, ¶m); + } + } + if (rtp_buf) + evsel__apply_ratio_to_prev(evsel, attr, opts, rtp_buf); +} + +struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type) +{ + struct evsel_config_term *term, *found_term = NULL; + + list_for_each_entry(term, &evsel->config_terms, list) { + if (term->type == type) + found_term = term; + } + + return found_term; +} + +/* + * Set @config_name to @val as long as the user hasn't already set or cleared it + * by passing a config term on the command line. + * + * @val is the value to put into the bits specified by @config_name rather than + * the bit pattern. It is shifted into position by this function, so to set + * something to true, pass 1 for val rather than a pre shifted value. + */ +void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, + u64 val) +{ + u64 user_bits = 0; + struct evsel_config_term *term = evsel__get_config_term(evsel, + USR_CHG_CONFIG); + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, + config_name); + int fbit; + __u64 *vp; + + if (!format) + return; + + switch (format->value) { + case PERF_PMU_FORMAT_VALUE_CONFIG: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG); + vp = &evsel->core.attr.config; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG1: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG1); + vp = &evsel->core.attr.config1; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG2: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG2); + vp = &evsel->core.attr.config2; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG3: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG3); + vp = &evsel->core.attr.config3; + break; + case PERF_PMU_FORMAT_VALUE_CONFIG4: + term = evsel__get_config_term(evsel, USR_CHG_CONFIG4); + vp = &evsel->core.attr.config4; + break; + default: + pr_err("Unknown format value: %d\n", format->value); + return; + } + + if (!format) + return; + + if (term) + user_bits = term->val.cfg_chg; + + /* Do nothing if the user changed the value */ + for_each_set_bit(fbit, format->bits, PERF_PMU_FORMAT_BITS) + if ((1ULL << fbit) & user_bits) + return; + + /* Otherwise replace it */ + perf_pmu__format_pack(format->bits, val, vp, /*zero=*/true); +} + +bool evsel__config_exists(const struct evsel *evsel, const char *config_name) +{ + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); + + return format && !bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS); +} + +int evsel__get_config_val(const struct evsel *evsel, const char *config_name, + u64 *val) +{ + struct perf_pmu_format *format = pmu_find_format(&evsel->pmu->format, config_name); + + if (!format || bitmap_empty(format->bits, PERF_PMU_FORMAT_BITS)) { + pr_err("Unknown/empty format name: %s\n", config_name); + *val = 0; + return -EINVAL; + } + + switch (format->value) { + case PERF_PMU_FORMAT_VALUE_CONFIG: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG1: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config1); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG2: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config2); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG3: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config3); + return 0; + case PERF_PMU_FORMAT_VALUE_CONFIG4: + *val = perf_pmu__format_unpack(format->bits, + evsel->core.attr.config4); + return 0; + default: + pr_err("Unknown format value: %d\n", format->value); + *val = 0; + return -EINVAL; + } +} + +void __weak arch_evsel__set_sample_weight(struct evsel *evsel) +{ + evsel__set_sample_bit(evsel, WEIGHT); +} + +void __weak arch__post_evsel_config(struct evsel *evsel __maybe_unused, + struct perf_event_attr *attr __maybe_unused) +{ +} + +void __weak arch_evsel__apply_ratio_to_prev(struct evsel *evsel __maybe_unused, + struct perf_event_attr *attr __maybe_unused) +{ +} + +static void evsel__set_default_freq_period(const struct record_opts *opts, + struct perf_event_attr *attr) +{ + if (opts->freq) { + attr->freq = 1; + attr->sample_freq = opts->freq; + } else { + attr->sample_period = opts->default_interval; + } +} + +bool evsel__is_offcpu_event(struct evsel *evsel) +{ + return evsel__is_bpf_output(evsel) && evsel__name_is(evsel, OFFCPU_EVENT) && + evsel->core.attr.sample_type & PERF_SAMPLE_RAW; +} + +/* + * The enable_on_exec/disabled value strategy: + * + * 1) For any type of traced program: + * - all independent events and group leaders are disabled + * - all group members are enabled + * + * Group members are ruled by group leaders. They need to + * be enabled, because the group scheduling relies on that. + * + * 2) For traced programs executed by perf: + * - all independent events and group leaders have + * enable_on_exec set + * - we don't specifically enable or disable any event during + * the record command + * + * Independent events and group leaders are initially disabled + * and get enabled by exec. Group members are ruled by group + * leaders as stated in 1). + * + * 3) For traced programs attached by perf (pid/tid): + * - we specifically enable or disable all events during + * the record command + * + * When attaching events to already running traced we + * enable/disable events specifically, as there's no + * initial traced exec call. + */ +void evsel__config(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *callchain) +{ + struct evsel *leader = evsel__leader(evsel); + struct perf_event_attr *attr = &evsel->core.attr; + int track = evsel->tracking; + bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread; + + attr->sample_id_all = perf_missing_features.sample_id_all ? 0 : 1; + attr->inherit = target__has_cpu(&opts->target) ? 0 : !opts->no_inherit; + attr->write_backward = opts->overwrite ? 1 : 0; + attr->read_format = PERF_FORMAT_LOST; + + evsel__set_sample_bit(evsel, IP); + evsel__set_sample_bit(evsel, TID); + + if (evsel->sample_read) { + evsel__set_sample_bit(evsel, READ); + + /* + * We need ID even in case of single event, because + * PERF_SAMPLE_READ process ID specific data. + */ + evsel__set_sample_id(evsel, false); + + /* + * Apply group format only if we belong to group + * with more than one members. + */ + if (leader->core.nr_members > 1) { + attr->read_format |= PERF_FORMAT_GROUP; + } + + /* + * Inherit + SAMPLE_READ requires SAMPLE_TID in the read_format + */ + if (attr->inherit) { + evsel__set_sample_bit(evsel, TID); + evsel->core.attr.read_format |= + PERF_FORMAT_ID; + } + } + + /* + * We default some events to have a default interval. But keep + * it a weak assumption overridable by the user. + */ + if ((evsel->is_libpfm_event && !attr->sample_period) || + (!evsel->is_libpfm_event && (!attr->sample_period || + opts->user_freq != UINT_MAX || + opts->user_interval != ULLONG_MAX))) + evsel__set_default_freq_period(opts, attr); + + /* + * If attr->freq was set (here or earlier), ask for period + * to be sampled. + */ + if (attr->freq) + evsel__set_sample_bit(evsel, PERIOD); + + if (opts->no_samples) + attr->sample_freq = 0; + + if (opts->inherit_stat) { + evsel->core.attr.read_format |= + PERF_FORMAT_TOTAL_TIME_ENABLED | + PERF_FORMAT_TOTAL_TIME_RUNNING | + PERF_FORMAT_ID; + attr->inherit_stat = 1; + } + + if (opts->sample_address) + evsel__set_sample_bit(evsel, ADDR); + + if (opts->record_data_mmap) + attr->mmap_data = track; + + /* + * We don't allow user space callchains for function trace + * event, due to issues with page faults while tracing page + * fault handler and its overall trickiness nature. + */ + if (evsel__is_function_event(evsel)) + evsel->core.attr.exclude_callchain_user = 1; + + if (callchain && callchain->enabled && !evsel->no_aux_samples) + evsel__config_callchain(evsel, opts, callchain); + + if (opts->sample_intr_regs && !evsel->no_aux_samples && + !evsel__is_dummy_event(evsel)) { + attr->sample_regs_intr = opts->sample_intr_regs; + evsel__set_sample_bit(evsel, REGS_INTR); + } + + if (opts->sample_user_regs && !evsel->no_aux_samples && + !evsel__is_dummy_event(evsel)) { + attr->sample_regs_user |= opts->sample_user_regs; + evsel__set_sample_bit(evsel, REGS_USER); + } + + if (target__has_cpu(&opts->target) || opts->sample_cpu) + evsel__set_sample_bit(evsel, CPU); + + /* + * When the user explicitly disabled time don't force it here. + */ + if (opts->sample_time && + (!perf_missing_features.sample_id_all && + (!opts->no_inherit || target__has_cpu(&opts->target) || per_cpu || + opts->sample_time_set))) + evsel__set_sample_bit(evsel, TIME); + + if (opts->raw_samples && !evsel->no_aux_samples) { + evsel__set_sample_bit(evsel, TIME); + evsel__set_sample_bit(evsel, RAW); + evsel__set_sample_bit(evsel, CPU); + } + + if (opts->sample_data_src) + evsel__set_sample_bit(evsel, DATA_SRC); + + if (opts->sample_phys_addr) + evsel__set_sample_bit(evsel, PHYS_ADDR); + + if (opts->no_buffering) { + attr->watermark = 0; + attr->wakeup_events = 1; + } + if (opts->branch_stack && !evsel->no_aux_samples) { + evsel__set_sample_bit(evsel, BRANCH_STACK); + attr->branch_sample_type = opts->branch_stack; + } + + if (opts->sample_weight || evsel->retire_lat) { + arch_evsel__set_sample_weight(evsel); + evsel->retire_lat = false; + } + attr->task = track; + attr->mmap = track; + attr->mmap2 = track && !perf_missing_features.mmap2; + attr->comm = track; + attr->build_id = track && opts->build_id; + attr->defer_output = track && callchain && callchain->defer; + + /* + * ksymbol is tracked separately with text poke because it needs to be + * system wide and enabled immediately. + */ + if (!opts->text_poke) + attr->ksymbol = track && !perf_missing_features.ksymbol; + attr->bpf_event = track && !opts->no_bpf_event && !perf_missing_features.bpf; + + if (opts->record_namespaces) + attr->namespaces = track; + + if (opts->record_cgroup) { + attr->cgroup = track && !perf_missing_features.cgroup; + evsel__set_sample_bit(evsel, CGROUP); + } + + if (opts->sample_data_page_size) + evsel__set_sample_bit(evsel, DATA_PAGE_SIZE); + + if (opts->sample_code_page_size) + evsel__set_sample_bit(evsel, CODE_PAGE_SIZE); + + if (opts->record_switch_events) + attr->context_switch = track; + + if (opts->sample_transaction) + evsel__set_sample_bit(evsel, TRANSACTION); + + if (opts->running_time) { + evsel->core.attr.read_format |= + PERF_FORMAT_TOTAL_TIME_ENABLED | + PERF_FORMAT_TOTAL_TIME_RUNNING; + } + + /* + * XXX see the function comment above + * + * Disabling only independent events or group leaders, + * keeping group members enabled. + */ + if (evsel__is_group_leader(evsel)) + attr->disabled = 1; + + /* + * Setting enable_on_exec for independent events and + * group leaders for traced executed by perf. + */ + if (target__none(&opts->target) && evsel__is_group_leader(evsel) && + !opts->target.initial_delay) + attr->enable_on_exec = 1; + + if (evsel->immediate) { + attr->disabled = 0; + attr->enable_on_exec = 0; + } + + clockid = opts->clockid; + if (opts->use_clockid) { + attr->use_clockid = 1; + attr->clockid = opts->clockid; + } + + if (evsel->precise_max) + attr->precise_ip = 3; + + if (opts->all_user) { + attr->exclude_kernel = 1; + attr->exclude_user = 0; + } + + if (opts->all_kernel) { + attr->exclude_kernel = 0; + attr->exclude_user = 1; + } + + if (evsel->core.pmu_cpus || evsel->unit) + evsel->core.attr.read_format |= PERF_FORMAT_ID; + + /* + * Apply event specific term settings, + * it overloads any global configuration. + */ + evsel__apply_config_terms(evsel, opts, track); + + evsel->ignore_missing_thread = opts->ignore_missing_thread; + + /* The --period option takes the precedence. */ + if (opts->period_set) { + if (opts->period) + evsel__set_sample_bit(evsel, PERIOD); + else + evsel__reset_sample_bit(evsel, PERIOD); + } + + /* + * A dummy event never triggers any actual counter and therefore + * cannot be used with branch_stack. + * + * For initial_delay, a dummy event is added implicitly. + * The software event will trigger -EOPNOTSUPP error out, + * if BRANCH_STACK bit is set. + */ + if (evsel__is_dummy_event(evsel)) + evsel__reset_sample_bit(evsel, BRANCH_STACK); + + if (evsel__is_offcpu_event(evsel)) { + evsel->core.attr.sample_type &= OFFCPU_SAMPLE_TYPES; + attr->inherit = 0; + } + + arch__post_evsel_config(evsel, attr); +} + +int evsel__set_filter(struct evsel *evsel, const char *filter) +{ + char *new_filter = strdup(filter); + + if (new_filter != NULL) { + free(evsel->filter); + evsel->filter = new_filter; + return 0; + } + + return -1; +} + +static int evsel__append_filter(struct evsel *evsel, const char *fmt, const char *filter) +{ + char *new_filter; + + if (evsel->filter == NULL) + return evsel__set_filter(evsel, filter); + + if (asprintf(&new_filter, fmt, evsel->filter, filter) > 0) { + free(evsel->filter); + evsel->filter = new_filter; + return 0; + } + + return -1; +} + +int evsel__append_tp_filter(struct evsel *evsel, const char *filter) +{ + return evsel__append_filter(evsel, "(%s) && (%s)", filter); +} + +int evsel__append_addr_filter(struct evsel *evsel, const char *filter) +{ + return evsel__append_filter(evsel, "%s,%s", filter); +} + +/* Caller has to clear disabled after going through all CPUs. */ +int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__enable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; +} + +int evsel__enable(struct evsel *evsel) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_enable(evsel); + else + err = perf_evsel__enable(&evsel->core); + + if (!err) + evsel->disabled = false; + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel (opened with group_fd = -1) + * and are skipped by the kernel when enabling the + * group leader. We must manually enable them in + * userspace. + */ + int mem_err = evsel__enable(member); + + if (mem_err) + return mem_err; + } + member->disabled = false; + } + } + + return err; +} + +/* Caller has to set disabled after going through all CPUs. */ +int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable_cpu(evsel, cpu_map_idx); + else + err = perf_evsel__disable_cpu(&evsel->core, cpu_map_idx); + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable_cpu(member, cpu_map_idx); + + if (mem_err) + return mem_err; + } + } + } + return err; +} + +int evsel__disable(struct evsel *evsel) +{ + int err; + + if (evsel__is_tool(evsel)) + err = evsel__tool_pmu_disable(evsel); + else + err = perf_evsel__disable(&evsel->core); + + /* + * We mark it disabled here so that tools that disable a event can + * ignore events after they disable it. I.e. the ring buffer may have + * already a few more events queued up before the kernel got the stop + * request. + */ + if (!err) + evsel->disabled = true; + + if (!err && evsel__is_group_leader(evsel)) { + struct evsel *member; + + for_each_group_member(member, evsel) { + if (evsel__is_non_perf_event_open_pmu(evsel) || + evsel__is_non_perf_event_open_pmu(member)) { + /* + * In a mixed PMU group, userspace PMUs are not + * grouped in the kernel and are skipped by the + * kernel when disabling the group leader. We must + * manually disable them in userspace. + */ + int mem_err = evsel__disable(member); + + if (mem_err) + return mem_err; + } + member->disabled = true; + } + } + + return err; +} + +void free_config_terms(struct list_head *config_terms) +{ + struct evsel_config_term *term, *h; + + list_for_each_entry_safe(term, h, config_terms, list) { + list_del_init(&term->list); + if (term->free_str) + zfree(&term->val.str); + free(term); + } +} + +static void evsel__free_config_terms(struct evsel *evsel) +{ + free_config_terms(&evsel->config_terms); +} + +static void (*evsel__priv_destructor)(void *priv); + +void evsel__set_priv_destructor(void (*destructor)(void *priv)) +{ + assert(evsel__priv_destructor == NULL); + + evsel__priv_destructor = destructor; +} + +void evsel__exit(struct evsel *evsel) +{ + assert(list_empty(&evsel->core.node)); + assert(evsel->evlist == NULL); + if (evsel__is_retire_lat(evsel)) + evsel__tpebs_close(evsel); + bpf_counter__destroy(evsel); + perf_bpf_filter__destroy(evsel); + evsel__free_counts(evsel); + perf_evsel__free_fd(&evsel->core); + perf_evsel__free_id(&evsel->core); + evsel__free_config_terms(evsel); + cgroup__put(evsel->cgrp); + perf_evsel__exit(&evsel->core); + zfree(&evsel->group_name); + zfree(&evsel->name); +#ifdef HAVE_LIBTRACEEVENT + zfree(&evsel->tp_sys); + zfree(&evsel->tp_name); +#endif + zfree(&evsel->filter); + zfree(&evsel->group_pmu_name); + zfree(&evsel->unit); + zfree(&evsel->metric_id); + evsel__zero_per_pkg(evsel); + hashmap__free(evsel->per_pkg_mask); + evsel->per_pkg_mask = NULL; + if (evsel__priv_destructor) + evsel__priv_destructor(evsel->priv); + perf_evsel__object.fini(evsel); + if (evsel__tool_event(evsel) == TOOL_PMU__EVENT_SYSTEM_TIME || + evsel__tool_event(evsel) == TOOL_PMU__EVENT_USER_TIME) { + xyarray__delete(evsel->process_time.start_times); + xyarray__delete(evsel->process_time.accumulated_times); + } +} + +void evsel__delete(struct evsel *evsel) +{ + if (!evsel) + return; + + evsel__exit(evsel); + free(evsel); +} + +void evsel__compute_deltas(struct evsel *evsel, int cpu_map_idx, int thread, + struct perf_counts_values *count) +{ + struct perf_counts_values tmp; + + if (!evsel->prev_raw_counts) + return; + + tmp = *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread); + *perf_counts(evsel->prev_raw_counts, cpu_map_idx, thread) = *count; + + count->val = count->val - tmp.val; + count->ena = count->ena - tmp.ena; + count->run = count->run - tmp.run; +} + +static int evsel__read_one(struct evsel *evsel, int cpu_map_idx, int thread) +{ + struct perf_counts_values *count = perf_counts(evsel->counts, cpu_map_idx, thread); + + return perf_evsel__read(&evsel->core, cpu_map_idx, thread, count); +} + +static void evsel__set_count(struct evsel *counter, int cpu_map_idx, int thread, + u64 val, u64 ena, u64 run, u64 lost) +{ + struct perf_counts_values *count; + + count = perf_counts(counter->counts, cpu_map_idx, thread); + + if (evsel__is_retire_lat(counter)) { + evsel__tpebs_read(counter, cpu_map_idx, thread); + perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true); + return; + } + + count->val = val; + count->ena = ena; + count->run = run; + count->lost = lost; + + perf_counts__set_loaded(counter->counts, cpu_map_idx, thread, true); +} + +static bool evsel__group_has_tpebs(struct evsel *leader) +{ + struct evsel *evsel; + + for_each_group_evsel(evsel, leader) { + if (evsel__is_retire_lat(evsel)) + return true; + } + return false; +} + +static u64 evsel__group_read_nr_members(struct evsel *leader) +{ + u64 nr = leader->core.nr_members; + struct evsel *evsel; + + for_each_group_evsel(evsel, leader) { + if (evsel__is_retire_lat(evsel)) + nr--; + } + return nr; +} + +static u64 evsel__group_read_size(struct evsel *leader) +{ + u64 read_format = leader->core.attr.read_format; + int entry = sizeof(u64); /* value */ + int size = 0; + int nr = 1; + + if (!evsel__group_has_tpebs(leader)) + return perf_evsel__read_size(&leader->core); + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + size += sizeof(u64); + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + size += sizeof(u64); + + if (read_format & PERF_FORMAT_ID) + entry += sizeof(u64); + + if (read_format & PERF_FORMAT_LOST) + entry += sizeof(u64); + + if (read_format & PERF_FORMAT_GROUP) { + nr = evsel__group_read_nr_members(leader); + size += sizeof(u64); + } + + size += entry * nr; + return size; +} + +static int evsel__process_group_data(struct evsel *leader, int cpu_map_idx, int thread, u64 *data) +{ + u64 read_format = leader->core.attr.read_format; + struct sample_read_value *v; + u64 nr, ena = 0, run = 0, lost = 0; + + nr = *data++; + + if (nr != evsel__group_read_nr_members(leader)) + return -EINVAL; + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + ena = *data++; + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + run = *data++; + + v = (void *)data; + sample_read_group__for_each(v, nr, read_format) { + struct evsel *counter; + + counter = evlist__id2evsel(leader->evlist, v->id); + if (!counter) + return -EINVAL; + + if (read_format & PERF_FORMAT_LOST) + lost = v->lost; + + evsel__set_count(counter, cpu_map_idx, thread, v->value, ena, run, lost); + } + + return 0; +} + +static int evsel__read_group(struct evsel *leader, int cpu_map_idx, int thread) +{ + struct perf_stat_evsel *ps = leader->stats; + u64 read_format = leader->core.attr.read_format; + int size = evsel__group_read_size(leader); + u64 *data = ps->group_data; + + if (!(read_format & PERF_FORMAT_ID)) + return -EINVAL; + + if (!evsel__is_group_leader(leader)) + return -EINVAL; + + if (!data) { + data = zalloc(size); + if (!data) + return -ENOMEM; + + ps->group_data = data; + } + + if (FD(leader, cpu_map_idx, thread) < 0) + return -EINVAL; + + if (readn(FD(leader, cpu_map_idx, thread), data, size) <= 0) + return -errno; + + return evsel__process_group_data(leader, cpu_map_idx, thread, data); +} + +bool __evsel__match(const struct evsel *evsel, u32 type, u64 config) +{ + + u32 e_type = evsel->core.attr.type; + u64 e_config = evsel->core.attr.config; + + if (e_type == type && e_config == config) + return true; + if (type != PERF_TYPE_HARDWARE && type != PERF_TYPE_HW_CACHE) + return false; + if ((e_type == PERF_TYPE_HARDWARE || e_type == PERF_TYPE_HW_CACHE) && + perf_pmus__supports_extended_type()) + e_config &= PERF_HW_EVENT_MASK; + if (e_type == type && e_config == config) + return true; + if (type == PERF_TYPE_HARDWARE && evsel->pmu && evsel->pmu->is_core && + evsel->alternate_hw_config == config) + return true; + return false; +} + +int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread) +{ + if (evsel__is_tool(evsel)) + return evsel__tool_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_hwmon(evsel)) + return evsel__hwmon_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_drm(evsel)) + return evsel__drm_pmu_read(evsel, cpu_map_idx, thread); + + if (evsel__is_retire_lat(evsel)) + return evsel__tpebs_read(evsel, cpu_map_idx, thread); + + if (evsel->core.attr.read_format & PERF_FORMAT_GROUP) + return evsel__read_group(evsel, cpu_map_idx, thread); + + return evsel__read_one(evsel, cpu_map_idx, thread); +} + +int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale) +{ + struct perf_counts_values count; + size_t nv = scale ? 3 : 1; + + if (FD(evsel, cpu_map_idx, thread) < 0) + return -EINVAL; + + if (evsel->counts == NULL && evsel__alloc_counts(evsel) < 0) + return -ENOMEM; + + if (readn(FD(evsel, cpu_map_idx, thread), &count, nv * sizeof(u64)) <= 0) + return -errno; + + evsel__compute_deltas(evsel, cpu_map_idx, thread, &count); + perf_counts_values__scale(&count, scale, NULL); + *perf_counts(evsel->counts, cpu_map_idx, thread) = count; + return 0; +} + +static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other, + int cpu_map_idx) +{ + struct perf_cpu cpu; + + cpu = perf_cpu_map__cpu(evsel->core.cpus, cpu_map_idx); + return perf_cpu_map__idx(other->core.cpus, cpu); +} + +static int evsel__hybrid_group_cpu_map_idx(struct evsel *evsel, int cpu_map_idx) +{ + struct evsel *leader = evsel__leader(evsel); + + if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) || + (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) { + return evsel__match_other_cpu(evsel, leader, cpu_map_idx); + } + + return cpu_map_idx; +} + +static int get_group_fd(struct evsel *evsel, int cpu_map_idx, int thread) +{ + struct evsel *leader = evsel__leader(evsel); + int fd; + + if (!evsel->supported || evsel__is_group_leader(evsel)) + return -1; + + /* + * Leader must be already processed/open, + * if not it's a bug. + */ + BUG_ON(!leader->core.fd); + + cpu_map_idx = evsel__hybrid_group_cpu_map_idx(evsel, cpu_map_idx); + if (cpu_map_idx == -1) + return -1; + + fd = FD(leader, cpu_map_idx, thread); + BUG_ON(fd == -1 && leader->supported); + + /* + * When the leader has been skipped, return -2 to distinguish from no + * group leader case. + */ + return fd == -1 ? -2 : fd; +} + +static void evsel__remove_fd(struct evsel *pos, int nr_cpus, int nr_threads, int thread_idx) +{ + for (int cpu = 0; cpu < nr_cpus; cpu++) + for (int thread = thread_idx; thread < nr_threads - 1; thread++) + FD(pos, cpu, thread) = FD(pos, cpu, thread + 1); +} + +static int update_fds(struct evsel *evsel, + int nr_cpus, int cpu_map_idx, + int nr_threads, int thread_idx) +{ + struct evsel *pos; + + if (cpu_map_idx >= nr_cpus || thread_idx >= nr_threads) + return -EINVAL; + + evlist__for_each_entry(evsel->evlist, pos) { + nr_cpus = pos != evsel ? nr_cpus : cpu_map_idx; + + evsel__remove_fd(pos, nr_cpus, nr_threads, thread_idx); + + /* + * Since fds for next evsel has not been created, + * there is no need to iterate whole event list. + */ + if (pos == evsel) + break; + } + return 0; +} + +static bool evsel__ignore_missing_thread(struct evsel *evsel, + int nr_cpus, int cpu_map_idx, + struct perf_thread_map *threads, + int thread, int err) +{ + pid_t ignore_pid = perf_thread_map__pid(threads, thread); + + if (!evsel->ignore_missing_thread) + return false; + + /* The system wide setup does not work with threads. */ + if (evsel->core.system_wide) + return false; + + /* The -ESRCH is perf event syscall errno for pid's not found. */ + if (err != -ESRCH) + return false; + + /* If there's only one thread, let it fail. */ + if (threads->nr == 1) + return false; + + /* + * We should remove fd for missing_thread first + * because thread_map__remove() will decrease threads->nr. + */ + if (update_fds(evsel, nr_cpus, cpu_map_idx, threads->nr, thread)) + return false; + + if (thread_map__remove(threads, thread)) + return false; + + pr_warning("WARNING: Ignored open failure for pid %d\n", + ignore_pid); + return true; +} + +static int __open_attr__fprintf(FILE *fp, const char *name, const char *val, + void *priv __maybe_unused) +{ + return fprintf(fp, " %-32s %s\n", name, val); +} + +static void display_attr(struct perf_event_attr *attr) +{ + if (verbose >= 2 || debug_peo_args) { + fprintf(stderr, "%.60s\n", graph_dotted_line); + fprintf(stderr, "perf_event_attr:\n"); + perf_event_attr__fprintf(stderr, attr, __open_attr__fprintf, NULL); + fprintf(stderr, "%.60s\n", graph_dotted_line); + } +} + +bool evsel__precise_ip_fallback(struct evsel *evsel) +{ + /* Do not try less precise if not requested. */ + if (!evsel->precise_max) + return false; + + /* + * We tried all the precise_ip values, and it's + * still failing, so leave it to standard fallback. + */ + if (!evsel->core.attr.precise_ip) { + evsel->core.attr.precise_ip = evsel->precise_ip_original; + return false; + } + + if (!evsel->precise_ip_original) + evsel->precise_ip_original = evsel->core.attr.precise_ip; + + evsel->core.attr.precise_ip--; + pr_debug2_peo("decreasing precise_ip by one (%d)\n", evsel->core.attr.precise_ip); + display_attr(&evsel->core.attr); + return true; +} + +static struct perf_cpu_map *empty_cpu_map; +static struct perf_thread_map *empty_thread_map; + +static int __evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + int ret = 0; + int nthreads = perf_thread_map__nr(threads); + + if ((perf_missing_features.write_backward && evsel->core.attr.write_backward) || + (perf_missing_features.aux_output && evsel->core.attr.aux_output)) + return -EINVAL; + + if (cpus == NULL) { + if (empty_cpu_map == NULL) { + empty_cpu_map = perf_cpu_map__new_any_cpu(); + if (empty_cpu_map == NULL) + return -ENOMEM; + } + + cpus = empty_cpu_map; + } + + if (threads == NULL) { + if (empty_thread_map == NULL) { + empty_thread_map = thread_map__new_by_tid(-1); + if (empty_thread_map == NULL) + return -ENOMEM; + } + + threads = empty_thread_map; + } + + if (evsel->core.fd == NULL && + perf_evsel__alloc_fd(&evsel->core, perf_cpu_map__nr(cpus), nthreads) < 0) + return -ENOMEM; + + if (evsel__is_tool(evsel)) + ret = evsel__tool_pmu_prepare_open(evsel, cpus, nthreads); + + evsel->open_flags = PERF_FLAG_FD_CLOEXEC; + if (evsel->cgrp) + evsel->open_flags |= PERF_FLAG_PID_CGROUP; + + return ret; +} + +static void evsel__disable_missing_features(struct evsel *evsel) +{ + if (perf_missing_features.defer_callchain && evsel->core.attr.defer_callchain) + evsel->core.attr.defer_callchain = 0; + if (perf_missing_features.defer_callchain && evsel->core.attr.defer_output) + evsel->core.attr.defer_output = 0; + if (perf_missing_features.inherit_sample_read && evsel->core.attr.inherit && + (evsel->core.attr.sample_type & PERF_SAMPLE_READ)) + evsel->core.attr.inherit = 0; + if (perf_missing_features.branch_counters) + evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_COUNTERS; + if (perf_missing_features.read_lost) + evsel->core.attr.read_format &= ~PERF_FORMAT_LOST; + if (perf_missing_features.weight_struct) { + evsel__set_sample_bit(evsel, WEIGHT); + evsel__reset_sample_bit(evsel, WEIGHT_STRUCT); + } + if (perf_missing_features.clockid_wrong) + evsel->core.attr.clockid = CLOCK_MONOTONIC; /* should always work */ + if (perf_missing_features.clockid) { + evsel->core.attr.use_clockid = 0; + evsel->core.attr.clockid = 0; + } + if (perf_missing_features.cloexec) + evsel->open_flags &= ~(unsigned long)PERF_FLAG_FD_CLOEXEC; + if (perf_missing_features.mmap2) + evsel->core.attr.mmap2 = 0; + if (evsel->pmu && evsel->pmu->missing_features.exclude_guest) + evsel->core.attr.exclude_guest = evsel->core.attr.exclude_host = 0; + if (perf_missing_features.lbr_flags) + evsel->core.attr.branch_sample_type &= ~(PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_NO_CYCLES); + if (perf_missing_features.group_read && evsel->core.attr.inherit) + evsel->core.attr.read_format &= ~(PERF_FORMAT_GROUP|PERF_FORMAT_ID); + if (perf_missing_features.ksymbol) + evsel->core.attr.ksymbol = 0; + if (perf_missing_features.bpf) + evsel->core.attr.bpf_event = 0; + if (perf_missing_features.branch_hw_idx) + evsel->core.attr.branch_sample_type &= ~PERF_SAMPLE_BRANCH_HW_INDEX; + if (perf_missing_features.sample_id_all) + evsel->core.attr.sample_id_all = 0; +} + +int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + int err; + + err = __evsel__prepare_open(evsel, cpus, threads); + if (err) + return err; + + evsel__disable_missing_features(evsel); + + return err; +} + +static bool __has_attr_feature(struct perf_event_attr *attr, + struct perf_cpu cpu, unsigned long flags) +{ + int fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + + if (fd < 0) { + attr->exclude_kernel = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + } + + if (fd < 0) { + attr->exclude_hv = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + } + + if (fd < 0) { + attr->exclude_guest = 1; + + fd = syscall(SYS_perf_event_open, attr, /*pid=*/0, cpu.cpu, + /*group_fd=*/-1, flags); + close(fd); + } + + attr->exclude_kernel = 0; + attr->exclude_guest = 0; + attr->exclude_hv = 0; + + return fd >= 0; +} + +static bool has_attr_feature(struct perf_event_attr *attr, unsigned long flags) +{ + struct perf_cpu cpu = {.cpu = -1}; + + return __has_attr_feature(attr, cpu, flags); +} + +static void evsel__detect_missing_pmu_features(struct evsel *evsel) +{ + struct perf_event_attr attr = { + .type = evsel->core.attr.type, + .config = evsel->core.attr.config, + .disabled = 1, + }; + struct perf_pmu *pmu = evsel->pmu; + int old_errno; + + old_errno = errno; + + if (pmu == NULL) + pmu = evsel->pmu = evsel__find_pmu(evsel); + + if (pmu == NULL || pmu->missing_features.checked) + goto out; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are kernel core limitation but + * specific to PMUs with branch stack. So we can detect with the given + * hardware event and stop on the first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.exclude_guest = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + pmu->missing_features.exclude_guest = true; + pr_debug2("switching off exclude_guest for PMU %s\n", pmu->name); + +found: + pmu->missing_features.checked = true; +out: + errno = old_errno; +} + +static void evsel__detect_missing_brstack_features(struct evsel *evsel) +{ + static bool detection_done = false; + struct perf_event_attr attr = { + .type = evsel->core.attr.type, + .config = evsel->core.attr.config, + .disabled = 1, + .sample_type = PERF_SAMPLE_BRANCH_STACK, + .sample_period = 1000, + }; + int old_errno; + + if (detection_done) + return; + + old_errno = errno; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are PMU specific limitation + * so we can detect with the given hardware event and stop on the + * first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_COUNTERS; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.branch_counters = true; + pr_debug2("switching off branch counters support\n"); + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_HW_INDEX; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.branch_hw_idx = true; + pr_debug2("switching off branch HW index support\n"); + + attr.branch_sample_type = PERF_SAMPLE_BRANCH_NO_CYCLES | PERF_SAMPLE_BRANCH_NO_FLAGS; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.lbr_flags = true; + pr_debug2_peo("switching off branch sample type no (cycles/flags)\n"); + +found: + detection_done = true; + errno = old_errno; +} + +static bool evsel__probe_aux_action(struct evsel *evsel, struct perf_cpu cpu) +{ + struct perf_event_attr attr = evsel->core.attr; + int old_errno = errno; + + attr.disabled = 1; + attr.aux_start_paused = 1; + + if (__has_attr_feature(&attr, cpu, /*flags=*/0)) { + errno = old_errno; + return true; + } + + /* + * EOPNOTSUPP means the kernel supports the feature but the PMU does + * not, so keep that distinction if possible. + */ + if (errno != EOPNOTSUPP) + errno = old_errno; + + return false; +} + +static void evsel__detect_missing_aux_action_feature(struct evsel *evsel, struct perf_cpu cpu) +{ + static bool detection_done; + struct evsel *leader; + + /* + * Don't bother probing aux_action if it is not being used or has been + * probed before. + */ + if (!evsel->core.attr.aux_action || detection_done) + return; + + detection_done = true; + + /* + * The leader is an AUX area event. If it has failed, assume the feature + * is not supported. + */ + leader = evsel__leader(evsel); + if (evsel == leader) { + perf_missing_features.aux_action = true; + return; + } + + /* + * AUX area event with aux_action must have been opened successfully + * already, so feature is supported. + */ + if (leader->core.attr.aux_action) + return; + + if (!evsel__probe_aux_action(leader, cpu)) + perf_missing_features.aux_action = true; +} + +static bool evsel__detect_missing_features(struct evsel *evsel, struct perf_cpu cpu) +{ + static bool detection_done = false; + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_TASK_CLOCK, + .disabled = 1, + }; + int old_errno; + + evsel__detect_missing_aux_action_feature(evsel, cpu); + + evsel__detect_missing_pmu_features(evsel); + + if (evsel__has_br_stack(evsel)) + evsel__detect_missing_brstack_features(evsel); + + if (detection_done) + goto check; + + old_errno = errno; + + /* + * Must probe features in the order they were added to the + * perf_event_attr interface. These are kernel core limitation + * not PMU-specific so we can detect with a software event and + * stop on the first one succeeded. + */ + + /* Please add new feature detection here. */ + + attr.defer_callchain = true; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.defer_callchain = true; + pr_debug2("switching off deferred callchain support\n"); + attr.defer_callchain = false; + + attr.inherit = true; + attr.sample_type = PERF_SAMPLE_READ | PERF_SAMPLE_TID; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.inherit_sample_read = true; + pr_debug2("Using PERF_SAMPLE_READ / :S modifier is not compatible with inherit, falling back to no-inherit.\n"); + attr.inherit = false; + attr.sample_type = 0; + + attr.read_format = PERF_FORMAT_LOST; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.read_lost = true; + pr_debug2("switching off PERF_FORMAT_LOST support\n"); + attr.read_format = 0; + + attr.sample_type = PERF_SAMPLE_WEIGHT_STRUCT; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.weight_struct = true; + pr_debug2("switching off weight struct support\n"); + attr.sample_type = 0; + + attr.sample_type = PERF_SAMPLE_CODE_PAGE_SIZE; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.code_page_size = true; + pr_debug2_peo("Kernel has no PERF_SAMPLE_CODE_PAGE_SIZE support\n"); + attr.sample_type = 0; + + attr.sample_type = PERF_SAMPLE_DATA_PAGE_SIZE; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.data_page_size = true; + pr_debug2_peo("Kernel has no PERF_SAMPLE_DATA_PAGE_SIZE support\n"); + attr.sample_type = 0; + + attr.cgroup = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.cgroup = true; + pr_debug2_peo("Kernel has no cgroup sampling support\n"); + attr.cgroup = 0; + + attr.aux_output = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.aux_output = true; + pr_debug2_peo("Kernel has no attr.aux_output support\n"); + attr.aux_output = 0; + + attr.bpf_event = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.bpf = true; + pr_debug2_peo("switching off bpf_event\n"); + attr.bpf_event = 0; + + attr.ksymbol = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.ksymbol = true; + pr_debug2_peo("switching off ksymbol\n"); + attr.ksymbol = 0; + + attr.write_backward = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.write_backward = true; + pr_debug2_peo("switching off write_backward\n"); + attr.write_backward = 0; + + attr.use_clockid = 1; + attr.clockid = CLOCK_MONOTONIC; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.clockid = true; + pr_debug2_peo("switching off clockid\n"); + attr.use_clockid = 0; + attr.clockid = 0; + + if (has_attr_feature(&attr, /*flags=*/PERF_FLAG_FD_CLOEXEC)) + goto found; + perf_missing_features.cloexec = true; + pr_debug2_peo("switching off cloexec flag\n"); + + attr.mmap2 = 1; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.mmap2 = true; + pr_debug2_peo("switching off mmap2\n"); + attr.mmap2 = 0; + + /* set this unconditionally? */ + perf_missing_features.sample_id_all = true; + pr_debug2_peo("switching off sample_id_all\n"); + + attr.inherit = 1; + attr.read_format = PERF_FORMAT_GROUP; + if (has_attr_feature(&attr, /*flags=*/0)) + goto found; + perf_missing_features.group_read = true; + pr_debug2_peo("switching off group read\n"); + attr.inherit = 0; + attr.read_format = 0; + +found: + detection_done = true; + errno = old_errno; + +check: + if ((evsel->core.attr.defer_callchain || evsel->core.attr.defer_output) && + perf_missing_features.defer_callchain) + return true; + + if (evsel->core.attr.inherit && + (evsel->core.attr.sample_type & PERF_SAMPLE_READ) && + perf_missing_features.inherit_sample_read) + return true; + + if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_COUNTERS) && + perf_missing_features.branch_counters) + return true; + + if ((evsel->core.attr.read_format & PERF_FORMAT_LOST) && + perf_missing_features.read_lost) + return true; + + if ((evsel->core.attr.sample_type & PERF_SAMPLE_WEIGHT_STRUCT) && + perf_missing_features.weight_struct) + return true; + + if (evsel->core.attr.use_clockid && evsel->core.attr.clockid != CLOCK_MONOTONIC && + !perf_missing_features.clockid) { + perf_missing_features.clockid_wrong = true; + return true; + } + + if (evsel->core.attr.use_clockid && perf_missing_features.clockid) + return true; + + if ((evsel->open_flags & PERF_FLAG_FD_CLOEXEC) && + perf_missing_features.cloexec) + return true; + + if (evsel->core.attr.mmap2 && perf_missing_features.mmap2) + return true; + + if ((evsel->core.attr.branch_sample_type & (PERF_SAMPLE_BRANCH_NO_FLAGS | + PERF_SAMPLE_BRANCH_NO_CYCLES)) && + perf_missing_features.lbr_flags) + return true; + + if (evsel->core.attr.inherit && (evsel->core.attr.read_format & PERF_FORMAT_GROUP) && + perf_missing_features.group_read) + return true; + + if (evsel->core.attr.ksymbol && perf_missing_features.ksymbol) + return true; + + if (evsel->core.attr.bpf_event && perf_missing_features.bpf) + return true; + + if ((evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX) && + perf_missing_features.branch_hw_idx) + return true; + + if (evsel->core.attr.sample_id_all && perf_missing_features.sample_id_all) + return true; + + return false; +} + +static int evsel__open_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads, + int start_cpu_map_idx, int end_cpu_map_idx) +{ + int idx, thread, nthreads; + int pid = -1, err, old_errno; + enum rlimit_action set_rlimit = NO_CHANGE; + struct perf_cpu cpu; + + if (evsel__is_retire_lat(evsel)) { + err = evsel__tpebs_open(evsel); + goto out; + } + + err = __evsel__prepare_open(evsel, cpus, threads); + if (err) + goto out; + + if (cpus == NULL) + cpus = empty_cpu_map; + + if (threads == NULL) + threads = empty_thread_map; + + nthreads = perf_thread_map__nr(threads); + + if (evsel->cgrp) + pid = evsel->cgrp->fd; + +fallback_missing_features: + evsel__disable_missing_features(evsel); + + pr_debug3("Opening: %s\n", evsel__name(evsel)); + display_attr(&evsel->core.attr); + + if (evsel__is_tool(evsel)) { + err = evsel__tool_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + goto out; + } + if (evsel__is_hwmon(evsel)) { + err = evsel__hwmon_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + goto out; + } + if (evsel__is_drm(evsel)) { + err = evsel__drm_pmu_open(evsel, threads, + start_cpu_map_idx, + end_cpu_map_idx); + goto out; + } + + for (idx = start_cpu_map_idx; idx < end_cpu_map_idx; idx++) { + cpu = perf_cpu_map__cpu(cpus, idx); + + for (thread = 0; thread < nthreads; thread++) { + int fd, group_fd; +retry_open: + if (thread >= nthreads) + break; + + if (!evsel->cgrp && !evsel->core.system_wide) + pid = perf_thread_map__pid(threads, thread); + + group_fd = get_group_fd(evsel, idx, thread); + + if (group_fd == -2) { + pr_debug("broken group leader for %s\n", evsel->name); + err = -EINVAL; + goto out_close; + } + + /* Debug message used by test scripts */ + pr_debug2_peo("sys_perf_event_open: pid %d cpu %d group_fd %d flags %#lx", + pid, cpu.cpu, group_fd, evsel->open_flags); + + fd = sys_perf_event_open(&evsel->core.attr, pid, cpu.cpu, + group_fd, evsel->open_flags); + + FD(evsel, idx, thread) = fd; + + if (fd < 0) { + err = -errno; + + pr_debug2_peo("\nsys_perf_event_open failed, error %d\n", + err); + goto try_fallback; + } + + bpf_counter__install_pe(evsel, idx, fd); + + if (unlikely(test_attr__enabled())) { + test_attr__open(&evsel->core.attr, pid, cpu, + fd, group_fd, evsel->open_flags); + } + + /* Debug message used by test scripts */ + pr_debug2_peo(" = %d\n", fd); + + if (evsel->bpf_fd >= 0) { + int evt_fd = fd; + int bpf_fd = evsel->bpf_fd; + + err = ioctl(evt_fd, + PERF_EVENT_IOC_SET_BPF, + bpf_fd); + if (err && errno != EEXIST) { + pr_err("failed to attach bpf fd %d: %m\n", + bpf_fd); + err = -EINVAL; + goto out_close; + } + } + + set_rlimit = NO_CHANGE; + + /* + * If we succeeded but had to kill clockid, fail and + * have evsel__open_strerror() print us a nice error. + */ + if (perf_missing_features.clockid || + perf_missing_features.clockid_wrong) { + err = -EINVAL; + goto out_close; + } + } + } + + err = 0; + goto out; + +try_fallback: + if (evsel__ignore_missing_thread(evsel, perf_cpu_map__nr(cpus), + idx, threads, thread, err)) { + /* We just removed 1 thread, so lower the upper nthreads limit. */ + nthreads--; + + /* ... and pretend like nothing have happened. */ + err = 0; + goto retry_open; + } + /* + * perf stat needs between 5 and 22 fds per CPU. When we run out + * of them try to increase the limits. + */ + if (err == -EMFILE && rlimit__increase_nofile(&set_rlimit)) + goto retry_open; + + if (err == -EINVAL && evsel__detect_missing_features(evsel, cpu)) + goto fallback_missing_features; + + if (evsel__precise_ip_fallback(evsel)) + goto retry_open; + +out_close: + if (err) + threads->err_thread = thread; + + old_errno = errno; + do { + while (--thread >= 0) { + if (FD(evsel, idx, thread) >= 0) + close(FD(evsel, idx, thread)); + FD(evsel, idx, thread) = -1; + } + thread = nthreads; + } while (--idx >= 0); + errno = old_errno; +out: + if (err) + evsel->supported = false; + return err; +} + +int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads) +{ + return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus)); +} + +void evsel__close(struct evsel *evsel) +{ + if (evsel__is_retire_lat(evsel)) + evsel__tpebs_close(evsel); + perf_evsel__close(&evsel->core); + perf_evsel__free_id(&evsel->core); +} + +int evsel__open_per_cpu_and_thread(struct evsel *evsel, + struct perf_cpu_map *cpus, int cpu_map_idx, + struct perf_thread_map *threads) +{ + if (cpu_map_idx == -1) + return evsel__open_cpu(evsel, cpus, threads, 0, perf_cpu_map__nr(cpus)); + + return evsel__open_cpu(evsel, cpus, threads, cpu_map_idx, cpu_map_idx + 1); +} + +int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx) +{ + struct perf_thread_map *threads = thread_map__new_by_tid(-1); + int ret = evsel__open_per_cpu_and_thread(evsel, cpus, cpu_map_idx, threads); + + perf_thread_map__put(threads); + return ret; +} + +int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads) +{ + struct perf_cpu_map *cpus = perf_cpu_map__new_any_cpu(); + int ret = evsel__open_per_cpu_and_thread(evsel, cpus, -1, threads); + + perf_cpu_map__put(cpus); + return ret; +} + +static int perf_evsel__parse_id_sample(const union perf_event *event, + struct perf_sample *sample) +{ + const struct evsel *evsel = sample->evsel; + u64 type = evsel->core.attr.sample_type; + const __u64 *array = event->sample.array; + bool swapped = evsel->needs_swap; + union u64_swap u; + int i = ((event->header.size - sizeof(event->header)) / sizeof(u64)) - 1; + + if (type & PERF_SAMPLE_IDENTIFIER) { + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; + } + + if (type & PERF_SAMPLE_CPU) { + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + } + sample->cpu = u.val32[0]; + } + + if (type & PERF_SAMPLE_STREAM_ID) { + if (i < 0) + return -EFAULT; + + sample->stream_id = array[i--]; + } + + if (type & PERF_SAMPLE_ID) { + if (i < 0) + return -EFAULT; + + sample->id = array[i--]; + } + + if (type & PERF_SAMPLE_TIME) { + if (i < 0) + return -EFAULT; + + sample->time = array[i--]; + } + + if (type & PERF_SAMPLE_TID) { + if (i < 0) + return -EFAULT; + + u.val64 = array[i--]; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + u.val32[1] = bswap_32(u.val32[1]); + } + + sample->pid = u.val32[0]; + sample->tid = u.val32[1]; + } + + return 0; +} + +static inline bool overflow(const void *endp, u16 max_size, const void *offset, + u64 size) +{ + return size > max_size || offset + size > endp; +} + +#define OVERFLOW_CHECK(offset, size, max_size) \ + do { \ + if (overflow(endp, (max_size), (offset), (size))) \ + goto out_efault; \ + } while (0) + +#define OVERFLOW_CHECK_u64(offset) \ + OVERFLOW_CHECK(offset, sizeof(u64), sizeof(u64)) + +static int +perf_event__check_size(union perf_event *event, unsigned int sample_size) +{ + /* + * The evsel's sample_size is based on PERF_SAMPLE_MASK which includes + * up to PERF_SAMPLE_PERIOD. After that overflow() must be used to + * check the format does not go past the end of the event. + */ + if (sample_size + sizeof(event->header) > event->header.size) + return -EFAULT; + + return 0; +} + +static void perf_parse_sample_weight(struct perf_sample *data, const __u64 *array, u64 type) +{ + union perf_sample_weight weight; + + weight.full = *array; + if (type & PERF_SAMPLE_WEIGHT_STRUCT) { + data->weight = weight.var1_dw; + data->ins_lat = weight.var2_w; + data->weight3 = weight.var3_w; + } else { + data->weight = weight.full; + } +} + +u64 evsel__bitfield_swap_branch_flags(u64 value) +{ + u64 new_val = 0; + + /* + * branch_flags + * union { + * u64 values; + * struct { + * mispred:1 //target mispredicted + * predicted:1 //target predicted + * in_tx:1 //in transaction + * abort:1 //transaction abort + * cycles:16 //cycle count to last branch + * type:4 //branch type + * spec:2 //branch speculation info + * new_type:4 //additional branch type + * priv:3 //privilege level + * reserved:31 + * } + * } + * + * Avoid bswap64() the entire branch_flag.value, + * as it has variable bit-field sizes. Instead the + * macro takes the bit-field position/size, + * swaps it based on the host endianness. + */ + if (host_is_bigendian()) { + new_val = bitfield_swap(value, 0, 1); + new_val |= bitfield_swap(value, 1, 1); + new_val |= bitfield_swap(value, 2, 1); + new_val |= bitfield_swap(value, 3, 1); + new_val |= bitfield_swap(value, 4, 16); + new_val |= bitfield_swap(value, 20, 4); + new_val |= bitfield_swap(value, 24, 2); + new_val |= bitfield_swap(value, 26, 4); + new_val |= bitfield_swap(value, 30, 3); + new_val |= bitfield_swap(value, 33, 31); + } else { + new_val = bitfield_swap(value, 63, 1); + new_val |= bitfield_swap(value, 62, 1); + new_val |= bitfield_swap(value, 61, 1); + new_val |= bitfield_swap(value, 60, 1); + new_val |= bitfield_swap(value, 44, 16); + new_val |= bitfield_swap(value, 40, 4); + new_val |= bitfield_swap(value, 38, 2); + new_val |= bitfield_swap(value, 34, 4); + new_val |= bitfield_swap(value, 31, 3); + new_val |= bitfield_swap(value, 0, 31); + } + + return new_val; +} + +static inline bool evsel__has_branch_counters(const struct evsel *evsel) +{ + struct evsel *leader = evsel__leader(evsel); + + /* The branch counters feature only supports group */ + if (!leader || !evsel->evlist) + return false; + + if (evsel->evlist->nr_br_cntr < 0) + evlist__update_br_cntr(evsel->evlist); + + if (leader->br_cntr_nr > 0) + return true; + + return false; +} + +static int __set_offcpu_sample(struct perf_sample *data) +{ + u64 *array = data->raw_data; + u32 max_size = data->raw_size, *p32; + const void *endp = (void *)array + max_size; + + if (array == NULL) + return -EFAULT; + + OVERFLOW_CHECK_u64(array); + p32 = (void *)array++; + data->pid = p32[0]; + data->tid = p32[1]; + + OVERFLOW_CHECK_u64(array); + data->period = *array++; + + OVERFLOW_CHECK_u64(array); + data->callchain = (struct ip_callchain *)array++; + OVERFLOW_CHECK(array, data->callchain->nr * sizeof(u64), max_size); + data->ip = data->callchain->ips[1]; + array += data->callchain->nr; + + OVERFLOW_CHECK_u64(array); + data->cgroup = *array; + + return 0; +out_efault: + return -EFAULT; +} + +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap) +{ + u64 type = evsel->core.attr.sample_type; + bool swapped = needs_swap; + const __u64 *array; + u16 max_size = event->header.size; + const void *endp = (void *)event + max_size; + u64 sz; + + /* + * used for cross-endian analysis. See git commit 65014ab3 + * for why this goofiness is needed. + */ + union u64_swap u; + + perf_sample__init(data, /*all=*/true); + data->evsel = evsel; + data->cpu = data->pid = data->tid = -1; + data->stream_id = data->id = data->time = -1ULL; + data->period = evsel->core.attr.sample_period; + data->cpumode = event->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + data->misc = event->header.misc; + data->data_src = PERF_MEM_DATA_SRC_NONE; + data->vcpu = -1; + + if (event->header.type == PERF_RECORD_CALLCHAIN_DEFERRED) { + const u64 max_callchain_nr = UINT64_MAX / sizeof(u64); + + data->callchain = (struct ip_callchain *)&event->callchain_deferred.nr; + if (data->callchain->nr > max_callchain_nr) + goto out_efault; + + data->deferred_cookie = event->callchain_deferred.cookie; + + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } + return 0; + } + + if (event->header.type != PERF_RECORD_SAMPLE) { + if (evsel->core.attr.sample_id_all) { + if (perf_evsel__parse_id_sample(event, data)) + goto out_efault; + } + return 0; + } + + array = event->sample.array; + + if (perf_event__check_size(event, evsel->sample_size)) + goto out_efault; + + if (type & PERF_SAMPLE_IDENTIFIER) { + data->id = *array; + array++; + } + + if (type & PERF_SAMPLE_IP) { + data->ip = *array; + array++; + } + + if (type & PERF_SAMPLE_TID) { + u.val64 = *array; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + u.val32[1] = bswap_32(u.val32[1]); + } + + data->pid = u.val32[0]; + data->tid = u.val32[1]; + array++; + } + + if (type & PERF_SAMPLE_TIME) { + data->time = *array; + array++; + } + + if (type & PERF_SAMPLE_ADDR) { + data->addr = *array; + array++; + } + + if (type & PERF_SAMPLE_ID) { + data->id = *array; + array++; + } + + if (type & PERF_SAMPLE_STREAM_ID) { + data->stream_id = *array; + array++; + } + + if (type & PERF_SAMPLE_CPU) { + + u.val64 = *array; + if (swapped) { + /* undo swap of u64, then swap on individual u32s */ + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + } + + data->cpu = u.val32[0]; + array++; + } + + if (type & PERF_SAMPLE_PERIOD) { + data->period = *array; + array++; + } + + if (type & PERF_SAMPLE_READ) { + u64 read_format = evsel->core.attr.read_format; + + OVERFLOW_CHECK_u64(array); + if (read_format & PERF_FORMAT_GROUP) + data->read.group.nr = *array; + else + data->read.one.value = *array; + + array++; + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) { + OVERFLOW_CHECK_u64(array); + data->read.time_enabled = *array; + array++; + } + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) { + OVERFLOW_CHECK_u64(array); + data->read.time_running = *array; + array++; + } + + /* PERF_FORMAT_ID is forced for PERF_SAMPLE_READ */ + if (read_format & PERF_FORMAT_GROUP) { + const u64 max_group_nr = UINT64_MAX / + sizeof(struct sample_read_value); + + if (data->read.group.nr > max_group_nr) + goto out_efault; + + sz = data->read.group.nr * sample_read_value_size(read_format); + OVERFLOW_CHECK(array, sz, max_size); + data->read.group.values = + (struct sample_read_value *)array; + array = (void *)array + sz; + } else { + OVERFLOW_CHECK_u64(array); + data->read.one.id = *array; + array++; + + if (read_format & PERF_FORMAT_LOST) { + OVERFLOW_CHECK_u64(array); + data->read.one.lost = *array; + array++; + } + } + } + + if (type & PERF_SAMPLE_CALLCHAIN) { + const u64 max_callchain_nr = UINT64_MAX / sizeof(u64); + u64 callchain_nr; + + OVERFLOW_CHECK_u64(array); + data->callchain = (struct ip_callchain *)array++; + callchain_nr = data->callchain->nr; + if (callchain_nr > max_callchain_nr) + goto out_efault; + sz = callchain_nr * sizeof(u64); + /* + * Save the cookie for the deferred user callchain. The last 2 + * entries in the callchain should be the context marker and the + * cookie. The cookie will be used to match PERF_RECORD_ + * CALLCHAIN_DEFERRED later. + */ + if (evsel->core.attr.defer_callchain && callchain_nr >= 2 && + data->callchain->ips[callchain_nr - 2] == PERF_CONTEXT_USER_DEFERRED) { + data->deferred_cookie = data->callchain->ips[callchain_nr - 1]; + data->deferred_callchain = true; + } + OVERFLOW_CHECK(array, sz, max_size); + array = (void *)array + sz; + } + + if (type & PERF_SAMPLE_RAW) { + OVERFLOW_CHECK_u64(array); + u.val64 = *array; + + /* + * Undo swap of u64, then swap on individual u32s, + * get the size of the raw area and undo all of the + * swap. The pevent interface handles endianness by + * itself. + */ + if (swapped) { + u.val64 = bswap_64(u.val64); + u.val32[0] = bswap_32(u.val32[0]); + u.val32[1] = bswap_32(u.val32[1]); + } + data->raw_size = u.val32[0]; + + /* + * The raw data is aligned on 64bits including the + * u32 size, so it's safe to use mem_bswap_64. + */ + if (swapped) + mem_bswap_64((void *) array, data->raw_size); + + array = (void *)array + sizeof(u32); + + OVERFLOW_CHECK(array, data->raw_size, max_size); + data->raw_data = (void *)array; + array = (void *)array + data->raw_size; + } + + if (type & PERF_SAMPLE_BRANCH_STACK) { + const u64 max_branch_nr = UINT64_MAX / + sizeof(struct branch_entry); + struct branch_entry *e; + unsigned int i; + + OVERFLOW_CHECK_u64(array); + data->branch_stack = (struct branch_stack *)array++; + + if (data->branch_stack->nr > max_branch_nr) + goto out_efault; + + sz = data->branch_stack->nr * sizeof(struct branch_entry); + if (evsel__has_branch_hw_idx(evsel)) { + sz += sizeof(u64); + e = &data->branch_stack->entries[0]; + } else { + data->no_hw_idx = true; + /* + * if the PERF_SAMPLE_BRANCH_HW_INDEX is not applied, + * only nr and entries[] will be output by kernel. + */ + e = (struct branch_entry *)&data->branch_stack->hw_idx; + } + + if (swapped) { + /* + * struct branch_flag does not have endian + * specific bit field definition. And bswap + * will not resolve the issue, since these + * are bit fields. + * + * evsel__bitfield_swap_branch_flags() uses a + * bitfield_swap macro to swap the bit position + * based on the host endians. + */ + for (i = 0; i < data->branch_stack->nr; i++, e++) + e->flags.value = evsel__bitfield_swap_branch_flags(e->flags.value); + } + + OVERFLOW_CHECK(array, sz, max_size); + array = (void *)array + sz; + + if (evsel__has_branch_counters(evsel)) { + data->branch_stack_cntr = (u64 *)array; + sz = data->branch_stack->nr * sizeof(u64); + + OVERFLOW_CHECK(array, sz, max_size); + array = (void *)array + sz; + } + } + + if (type & PERF_SAMPLE_REGS_USER) { + struct regs_dump *regs = perf_sample__user_regs(data); + + OVERFLOW_CHECK_u64(array); + regs->abi = *array; + array++; + + if (regs->abi) { + u64 mask = evsel->core.attr.sample_regs_user; + + sz = hweight64(mask) * sizeof(u64); + OVERFLOW_CHECK(array, sz, max_size); + regs->mask = mask; + regs->regs = (u64 *)array; + array = (void *)array + sz; + } + } + + if (type & PERF_SAMPLE_STACK_USER) { + OVERFLOW_CHECK_u64(array); + sz = *array++; + + data->user_stack.offset = ((char *)(array - 1) + - (char *) event); + + if (!sz) { + data->user_stack.size = 0; + } else { + OVERFLOW_CHECK(array, sz, max_size); + data->user_stack.data = (char *)array; + array = (void *)array + sz; + OVERFLOW_CHECK_u64(array); + data->user_stack.size = *array++; + if (WARN_ONCE(data->user_stack.size > sz, + "user stack dump failure\n")) + goto out_efault; + } + } + + if (type & PERF_SAMPLE_WEIGHT_TYPE) { + OVERFLOW_CHECK_u64(array); + perf_parse_sample_weight(data, array, type); + array++; + } + + if (type & PERF_SAMPLE_DATA_SRC) { + OVERFLOW_CHECK_u64(array); + data->data_src = *array; + array++; + } + + if (type & PERF_SAMPLE_TRANSACTION) { + OVERFLOW_CHECK_u64(array); + data->transaction = *array; + array++; + } + + if (type & PERF_SAMPLE_REGS_INTR) { + struct regs_dump *regs = perf_sample__intr_regs(data); + + OVERFLOW_CHECK_u64(array); + regs->abi = *array; + array++; + + if (regs->abi != PERF_SAMPLE_REGS_ABI_NONE) { + u64 mask = evsel->core.attr.sample_regs_intr; + + sz = hweight64(mask) * sizeof(u64); + OVERFLOW_CHECK(array, sz, max_size); + regs->mask = mask; + regs->regs = (u64 *)array; + array = (void *)array + sz; + } + } + + data->phys_addr = 0; + if (type & PERF_SAMPLE_PHYS_ADDR) { + data->phys_addr = *array; + array++; + } + + data->cgroup = 0; + if (type & PERF_SAMPLE_CGROUP) { + data->cgroup = *array; + array++; + } + + data->data_page_size = 0; + if (type & PERF_SAMPLE_DATA_PAGE_SIZE) { + data->data_page_size = *array; + array++; + } + + data->code_page_size = 0; + if (type & PERF_SAMPLE_CODE_PAGE_SIZE) { + data->code_page_size = *array; + array++; + } + + if (type & PERF_SAMPLE_AUX) { + OVERFLOW_CHECK_u64(array); + sz = *array++; + + OVERFLOW_CHECK(array, sz, max_size); + /* Undo swap of data */ + if (swapped) + mem_bswap_64((char *)array, sz); + data->aux_sample.size = sz; + data->aux_sample.data = (char *)array; + array = (void *)array + sz; + } + + if (evsel__is_offcpu_event(evsel)) { + if (__set_offcpu_sample(data)) + goto out_efault; + } + + return 0; +out_efault: + perf_sample__exit(data); + return -EFAULT; +} + +int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, + u64 *timestamp) +{ + u64 type = evsel->core.attr.sample_type; + const __u64 *array; + + if (!(type & PERF_SAMPLE_TIME)) + return -1; + + if (event->header.type != PERF_RECORD_SAMPLE) { + struct perf_sample data = { + .evsel = evsel, + .time = -1ULL, + }; + + if (!evsel->core.attr.sample_id_all) + return -1; + if (perf_evsel__parse_id_sample(event, &data)) + return -1; + + *timestamp = data.time; + return 0; + } + + array = event->sample.array; + + if (perf_event__check_size(event, evsel->sample_size)) + return -EFAULT; + + if (type & PERF_SAMPLE_IDENTIFIER) + array++; + + if (type & PERF_SAMPLE_IP) + array++; + + if (type & PERF_SAMPLE_TID) + array++; + + if (type & PERF_SAMPLE_TIME) + *timestamp = *array; + + return 0; +} + +u16 evsel__id_hdr_size(const struct evsel *evsel) +{ + u64 sample_type = evsel->core.attr.sample_type; + u16 size = 0; + + if (sample_type & PERF_SAMPLE_TID) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_TIME) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_ID) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_STREAM_ID) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_CPU) + size += sizeof(u64); + + if (sample_type & PERF_SAMPLE_IDENTIFIER) + size += sizeof(u64); + + return size; +} + +#ifdef HAVE_LIBTRACEEVENT +struct tep_format_field *evsel__field(struct evsel *evsel, const char *name) +{ + struct tep_event *tp_format = evsel__tp_format(evsel); + + return tp_format ? tep_find_field(tp_format, name) : NULL; +} + +struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name) +{ + struct tep_event *tp_format = evsel__tp_format(evsel); + + return tp_format ? tep_find_common_field(tp_format, name) : NULL; +} + +static bool out_of_bounds(const struct tep_format_field *field, int offset, int size, u32 raw_size) +{ + if (offset < 0) { + pr_warning("Negative trace point field offset %d in %s\n", + offset, field->name); + return true; + } + if (size < 0) { + pr_warning("Negative trace point field size %d in %s\n", + size, field->name); + return true; + } + if ((u32)offset + (u32)size > raw_size) { + pr_warning("Out of bound tracepoint field (%s) offset %d size %d in %u\n", + field->name, offset, size, raw_size); + return true; + } + return false; +} + +void *perf_sample__rawptr(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__field(sample->evsel, name); + int offset, size; + + if (!field) + return NULL; + + offset = field->offset; + size = field->size; + if (field->flags & TEP_FIELD_IS_DYNAMIC) { + int dynamic_data; + + if (out_of_bounds(field, offset, 4, sample->raw_size)) + return NULL; + + dynamic_data = *(int *)(sample->raw_data + field->offset); + + if (sample->evsel->needs_swap) + dynamic_data = bswap_32(dynamic_data); + + offset = dynamic_data & 0xffff; + size = (dynamic_data >> 16) & 0xffff; + + if (tep_field_is_relative(field->flags)) { + /* + * Newer kernel feature: Relative offsets (__rel_loc). + * If the relative flag is set, the parsed offset is not + * absolute from the start of the record. Instead, it is + * relative to the *end* of the dynamic field descriptor + * itself. + */ + offset += field->offset + field->size; + } + } + if (out_of_bounds(field, offset, size, sample->raw_size)) + return NULL; + + return sample->raw_data + offset; +} + +u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, + bool needs_swap) +{ + u64 value; + void *ptr = sample->raw_data + field->offset; + + if (out_of_bounds(field, field->offset, field->size, sample->raw_size)) + return 0; + + switch (field->size) { + case 1: + return *(u8 *)ptr; + case 2: + value = *(u16 *)ptr; + break; + case 4: + value = *(u32 *)ptr; + break; + case 8: + memcpy(&value, ptr, sizeof(u64)); + break; + default: + return 0; + } + + if (!needs_swap) + return value; + + switch (field->size) { + case 2: + return bswap_16(value); + case 4: + return bswap_32(value); + case 8: + return bswap_64(value); + default: + return 0; + } + + return 0; +} + +u64 perf_sample__intval(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__field(sample->evsel, name); + + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; +} + +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name) +{ + struct tep_format_field *field = evsel__common_field(sample->evsel, name); + + return field ? format_field__intval(field, sample, sample->evsel->needs_swap) : 0; +} + +char perf_sample__taskstate(struct perf_sample *sample, const char *name) +{ + static struct tep_format_field *prev_state_field; + static const char *states; + struct tep_format_field *field; + unsigned long long val; + unsigned int bit; + char state = '?'; /* '?' denotes unknown task state */ + + field = evsel__field(sample->evsel, name); + + if (!field) + return state; + + if (!states || field != prev_state_field) { + states = parse_task_states(field); + if (!states) + return state; + prev_state_field = field; + } + + /* + * Note since the kernel exposes TASK_REPORT_MAX to userspace + * to denote the 'preempted' state, we might as welll report + * 'R' for this case, which make senses to users as well. + * + * We can change this if we have a good reason in the future. + */ + val = perf_sample__intval(sample, name); + bit = val ? ffs(val) : 0; + state = (!bit || bit > strlen(states)) ? 'R' : states[bit-1]; + return state; +} +#endif + +bool evsel__fallback(struct evsel *evsel, struct target *target, int err, + char *msg, size_t msgsize) +{ + int paranoid; + + if ((err == ENODEV || err == ENOENT || err == ENXIO) && + evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) { + /* + * If it's the legacy hardware cycles event fails then fall back + * to hrtimer based cpu-clock sw counter, which is always + * available even if no PMU support. PPC returned ENXIO rather + * than ENODEV or ENOENT until 2.6.37. + */ + evsel->pmu = perf_pmus__find_by_type(PERF_TYPE_SOFTWARE); + assert(evsel->pmu); /* software is a "well-known" and can't fail PMU type. */ + + /* Configure the event. */ + evsel->core.attr.type = PERF_TYPE_SOFTWARE; + evsel->core.attr.config = target__has_cpu(target) + ? PERF_COUNT_SW_CPU_CLOCK + : PERF_COUNT_SW_TASK_CLOCK; + evsel->core.is_pmu_core = false; + + /* Remove excludes for new event. */ + if (evsel->fallenback_eacces) { + evsel->core.attr.exclude_kernel = 0; + evsel->core.attr.exclude_hv = 0; + evsel->fallenback_eacces = false; + } + if (evsel->fallenback_eopnotsupp) { + evsel->core.attr.exclude_guest = 0; + evsel->fallenback_eopnotsupp = false; + } + + /* Name is recomputed by evsel__name. */ + zfree(&evsel->name); + + /* Log message. */ + scnprintf(msg, msgsize, + "The cycles event is not supported, trying to fall back to %s", + evsel__name(evsel)); + return true; + } else if (err == EACCES && !evsel->core.attr.exclude_kernel && + (paranoid = perf_event_paranoid()) > 1) { + const char *name = evsel__name(evsel); + char *new_name; + const char *sep = ":"; + + /* If event has exclude user then don't exclude kernel. */ + if (evsel->core.attr.exclude_user) + goto no_fallback; + + /* Is there already the separator in the name. */ + if (strchr(name, '/') || + (strchr(name, ':') && !evsel->is_libpfm_event)) + sep = ""; + + if (asprintf(&new_name, "%s%su", name, sep) < 0) + goto no_fallback; + + free(evsel->name); + evsel->name = new_name; + scnprintf(msg, msgsize, "kernel.perf_event_paranoid=%d, trying " + "to fall back to excluding kernel and hypervisor " + " samples", paranoid); + evsel->core.attr.exclude_kernel = 1; + evsel->core.attr.exclude_hv = 1; + evsel->fallenback_eacces = true; + return true; + } else if (err == EOPNOTSUPP && !evsel->core.attr.exclude_guest && + !evsel->exclude_GH) { + const char *name = evsel__name(evsel); + char *new_name; + const char *sep = ":"; + + /* Is there already the separator in the name. */ + if (strchr(name, '/') || + (strchr(name, ':') && !evsel->is_libpfm_event)) + sep = ""; + + if (asprintf(&new_name, "%s%sH", name, sep) < 0) + goto no_fallback; + + free(evsel->name); + evsel->name = new_name; + /* Apple M1 requires exclude_guest */ + scnprintf(msg, msgsize, "Trying to fall back to excluding guest samples"); + evsel->core.attr.exclude_guest = 1; + evsel->fallenback_eopnotsupp = true; + return true; + } +no_fallback: + scnprintf(msg, msgsize, "No fallback found for '%s' for error %d", + evsel__name(evsel), err); + return false; +} + +static bool find_process(const char *name) +{ + size_t len = strlen(name); + DIR *dir; + struct dirent *d; + int ret = -1; + + dir = opendir(procfs__mountpoint()); + if (!dir) + return false; + + /* Walk through the directory. */ + while (ret && (d = readdir(dir)) != NULL) { + char path[PATH_MAX]; + char *data; + size_t size; + + if ((d->d_type != DT_DIR) || + !strcmp(".", d->d_name) || + !strcmp("..", d->d_name)) + continue; + + scnprintf(path, sizeof(path), "%s/%s/comm", + procfs__mountpoint(), d->d_name); + + if (filename__read_str(path, &data, &size)) + continue; + + ret = strncmp(name, data, len); + free(data); + } + + closedir(dir); + return ret ? false : true; +} + +static int dump_perf_event_processes(char *msg, size_t size) +{ + DIR *proc_dir; + struct dirent *proc_entry; + int printed = 0; + + proc_dir = opendir(procfs__mountpoint()); + if (!proc_dir) + return 0; + + /* Walk through the /proc directory. */ + while ((proc_entry = readdir(proc_dir)) != NULL) { + char buf[256]; + DIR *fd_dir; + struct dirent *fd_entry; + int fd_dir_fd; + + if (proc_entry->d_type != DT_DIR || + !isdigit(proc_entry->d_name[0]) || + strlen(proc_entry->d_name) > sizeof(buf) - 4) + continue; + + scnprintf(buf, sizeof(buf), "%s/fd", proc_entry->d_name); + fd_dir_fd = openat(dirfd(proc_dir), buf, O_DIRECTORY); + if (fd_dir_fd == -1) + continue; + fd_dir = fdopendir(fd_dir_fd); + if (!fd_dir) { + close(fd_dir_fd); + continue; + } + while ((fd_entry = readdir(fd_dir)) != NULL) { + ssize_t link_size; + + if (fd_entry->d_type != DT_LNK) + continue; + link_size = readlinkat(fd_dir_fd, fd_entry->d_name, buf, sizeof(buf)); + if (link_size < 0) + continue; + /* Take care as readlink doesn't null terminate the string. */ + if (!strncmp(buf, "anon_inode:[perf_event]", link_size)) { + int cmdline_fd; + ssize_t cmdline_size; + + scnprintf(buf, sizeof(buf), "%s/cmdline", proc_entry->d_name); + cmdline_fd = openat(dirfd(proc_dir), buf, O_RDONLY); + if (cmdline_fd == -1) + continue; + cmdline_size = read(cmdline_fd, buf, sizeof(buf) - 1); + close(cmdline_fd); + if (cmdline_size < 0) + continue; + buf[cmdline_size] = '\0'; + for (ssize_t i = 0; i < cmdline_size; i++) { + if (buf[i] == '\0') + buf[i] = ' '; + } + + if (printed == 0) + printed += scnprintf(msg, size, "Possible processes:\n"); + + printed += scnprintf(msg + printed, size - printed, + "%s %s\n", proc_entry->d_name, buf); + break; + } + } + closedir(fd_dir); + } + closedir(proc_dir); + return printed; +} + +int __weak arch_evsel__open_strerror(struct evsel *evsel __maybe_unused, + int err __maybe_unused, + char *msg __maybe_unused, + size_t size __maybe_unused) +{ + return 0; +} + +int evsel__open_strerror(struct evsel *evsel, struct target *target, + int err, char *msg, size_t size) +{ + struct perf_pmu *pmu; + int printed = 0, enforced = 0; + int ret; + + switch (err) { + case EPERM: + case EACCES: + printed += scnprintf(msg + printed, size - printed, + "Access to performance monitoring and observability operations is limited.\n"); + + if (!sysfs__read_int("fs/selinux/enforce", &enforced)) { + if (enforced) { + printed += scnprintf(msg + printed, size - printed, + "Enforced MAC policy settings (SELinux) can limit access to performance\n" + "monitoring and observability operations. Inspect system audit records for\n" + "more perf_event access control information and adjusting the policy.\n"); + } + } + + if (err == EPERM) + printed += scnprintf(msg, size, + "No permission to enable %s event.\n\n", evsel__name(evsel)); + + return printed + scnprintf(msg + printed, size - printed, + "Consider adjusting /proc/sys/kernel/perf_event_paranoid setting to open\n" + "access to performance monitoring and observability operations for processes\n" + "without CAP_PERFMON, CAP_SYS_PTRACE or CAP_SYS_ADMIN Linux capability.\n" + "More information can be found at 'Perf events and tool security' document:\n" + "https://www.kernel.org/doc/html/latest/admin-guide/perf-security.html\n" + "perf_event_paranoid setting is %d:\n" + " -1: Allow use of (almost) all events by all users\n" + " Ignore mlock limit after perf_event_mlock_kb without CAP_IPC_LOCK\n" + ">= 0: Disallow raw and ftrace function tracepoint access\n" + ">= 1: Disallow CPU event access\n" + ">= 2: Disallow kernel profiling\n" + "To make the adjusted perf_event_paranoid setting permanent preserve it\n" + "in /etc/sysctl.conf (e.g. kernel.perf_event_paranoid = )", + perf_event_paranoid()); + case ENOENT: + return scnprintf(msg, size, "The %s event is not supported.", evsel__name(evsel)); + case EMFILE: + return scnprintf(msg, size, "%s", + "Too many events are opened.\n" + "Probably the maximum number of open file descriptors has been reached.\n" + "Hint: Try again after reducing the number of events.\n" + "Hint: Try increasing the limit with 'ulimit -n '"); + case ENOMEM: + if (evsel__has_callchain(evsel) && + access("/proc/sys/kernel/perf_event_max_stack", F_OK) == 0) + return scnprintf(msg, size, + "Not enough memory to setup event with callchain.\n" + "Hint: Try tweaking /proc/sys/kernel/perf_event_max_stack\n" + "Hint: Current value: %d", sysctl__max_stack()); + break; + case ENODEV: + if (target->cpu_list) + return scnprintf(msg, size, "%s", + "No such device - did you specify an out-of-range profile CPU?"); + break; + case EOPNOTSUPP: + if (evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK) + return scnprintf(msg, size, + "%s: PMU Hardware or event type doesn't support branch stack sampling.", + evsel__name(evsel)); + if (evsel->core.attr.aux_output) + return scnprintf(msg, size, + "%s: PMU Hardware doesn't support 'aux_output' feature", + evsel__name(evsel)); + if (evsel->core.attr.aux_action) + return scnprintf(msg, size, + "%s: PMU Hardware doesn't support 'aux_action' feature", + evsel__name(evsel)); + if (evsel->core.attr.sample_period != 0) + return scnprintf(msg, size, + "%s: PMU Hardware doesn't support sampling/overflow-interrupts. Try 'perf stat'", + evsel__name(evsel)); + if (evsel->core.attr.precise_ip) + return scnprintf(msg, size, "%s", + "\'precise\' request may not be supported. Try removing 'p' modifier."); +#if defined(__i386__) || defined(__x86_64__) + if (evsel->core.attr.type == PERF_TYPE_HARDWARE) + return scnprintf(msg, size, "%s", + "No hardware sampling interrupt available.\n"); +#endif + if (!target__has_cpu(target)) + return scnprintf(msg, size, + "Unsupported event (%s) in per-thread mode, enable system wide with '-a'.", + evsel__name(evsel)); + break; + case EBUSY: + if (find_process("oprofiled")) + return scnprintf(msg, size, + "The PMU counters are busy/taken by another profiler.\n" + "We found oprofile daemon running, please stop it and try again."); + printed += scnprintf( + msg, size, + "The PMU %s counters are busy and in use by another process.\n", + evsel->pmu ? evsel->pmu->name : ""); + return printed + dump_perf_event_processes(msg + printed, size - printed); + break; + case EINVAL: + if (evsel->core.attr.sample_type & PERF_SAMPLE_CODE_PAGE_SIZE && perf_missing_features.code_page_size) + return scnprintf(msg, size, "Asking for the code page size isn't supported by this kernel."); + if (evsel->core.attr.sample_type & PERF_SAMPLE_DATA_PAGE_SIZE && perf_missing_features.data_page_size) + return scnprintf(msg, size, "Asking for the data page size isn't supported by this kernel."); + if (evsel->core.attr.write_backward && perf_missing_features.write_backward) + return scnprintf(msg, size, "Reading from overwrite event is not supported by this kernel."); + if (perf_missing_features.clockid) + return scnprintf(msg, size, "clockid feature not supported."); + if (perf_missing_features.clockid_wrong) + return scnprintf(msg, size, "wrong clockid (%d).", clockid); + if (perf_missing_features.aux_action) + return scnprintf(msg, size, "The 'aux_action' feature is not supported, update the kernel."); + if (perf_missing_features.aux_output) + return scnprintf(msg, size, "The 'aux_output' feature is not supported, update the kernel."); + pmu = evsel__find_pmu(evsel); + if (!pmu->is_core && !target__has_cpu(target)) + return scnprintf(msg, size, + "Invalid event (%s) in per-thread mode, enable system wide with '-a'.", + evsel__name(evsel)); + + break; + case ENODATA: + return scnprintf(msg, size, "Cannot collect data source with the load latency event alone. " + "Please add an auxiliary event in front of the load latency event."); + default: + break; + } + + ret = arch_evsel__open_strerror(evsel, err, msg, size); + if (ret) + return ret; + + errno = err; + return scnprintf(msg, size, + "The sys_perf_event_open() syscall failed for event (%s): %m\n" + "\"dmesg | grep -i perf\" may provide additional information.\n", + evsel__name(evsel)); +} + +struct perf_session *evsel__session(struct evsel *evsel) +{ + return evsel && evsel->evlist ? evsel->evlist->session : NULL; +} + +struct perf_env *evsel__env(struct evsel *evsel) +{ + struct perf_session *session = evsel__session(evsel); + + return session ? perf_session__env(session) : NULL; +} + +static int store_evsel_ids(struct evsel *evsel, struct evlist *evlist) +{ + int cpu_map_idx, thread; + + if (evsel__is_retire_lat(evsel)) + return 0; + + if (perf_pmu__kind(evsel->pmu) != PERF_PMU_KIND_PE) + return 0; + + for (cpu_map_idx = 0; cpu_map_idx < xyarray__max_x(evsel->core.fd); cpu_map_idx++) { + for (thread = 0; thread < xyarray__max_y(evsel->core.fd); + thread++) { + int fd = FD(evsel, cpu_map_idx, thread); + + if (perf_evlist__id_add_fd(&evlist->core, &evsel->core, + cpu_map_idx, thread, fd) < 0) + return -1; + } + } + + return 0; +} + +int evsel__store_ids(struct evsel *evsel, struct evlist *evlist) +{ + struct perf_cpu_map *cpus = evsel->core.cpus; + struct perf_thread_map *threads = evsel->core.threads; + + if (perf_evsel__alloc_id(&evsel->core, perf_cpu_map__nr(cpus), threads->nr)) + return -ENOMEM; + + return store_evsel_ids(evsel, evlist); +} + +void evsel__zero_per_pkg(struct evsel *evsel) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (evsel->per_pkg_mask) { + hashmap__for_each_entry(evsel->per_pkg_mask, cur, bkt) + zfree(&cur->pkey); + + hashmap__clear(evsel->per_pkg_mask); + } +} + +/** + * evsel__is_hybrid - does the evsel have a known PMU that is hybrid. Note, this + * will be false on hybrid systems for hardware and legacy + * cache events. + */ +bool evsel__is_hybrid(const struct evsel *evsel) +{ + if (!evsel->core.is_pmu_core) + return false; + + return perf_pmus__num_core_pmus() > 1; +} + +struct evsel *evsel__leader(const struct evsel *evsel) +{ + if (evsel->core.leader == NULL) + return NULL; + return container_of(evsel->core.leader, struct evsel, core); +} + +bool evsel__has_leader(struct evsel *evsel, struct evsel *leader) +{ + return evsel->core.leader == &leader->core; +} + +bool evsel__is_leader(struct evsel *evsel) +{ + return evsel__has_leader(evsel, evsel); +} + +void evsel__set_leader(struct evsel *evsel, struct evsel *leader) +{ + evsel->core.leader = &leader->core; +} + +bool evsel__is_aux_event(const struct evsel *evsel) +{ + struct perf_pmu *pmu; + + if (evsel->needs_auxtrace_mmap) + return true; + + pmu = evsel__find_pmu(evsel); + return pmu && pmu->auxtrace; +} + +int evsel__source_count(const struct evsel *evsel) +{ + struct evsel *pos; + int count = 0; + + evlist__for_each_entry(evsel->evlist, pos) { + if (pos->metric_leader == evsel) + count++; + } + return count; +} + +bool __weak arch_evsel__must_be_in_group(const struct evsel *evsel __maybe_unused) +{ + return false; +} + +/* + * Remove an event from a given group (leader). + * Some events, e.g., perf metrics Topdown events, + * must always be grouped. Ignore the events. + */ +void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader) +{ + if (!arch_evsel__must_be_in_group(evsel) && evsel != leader) { + evsel__set_leader(evsel, evsel); + evsel->core.nr_members = 0; + leader->core.nr_members--; + } +} + +bool evsel__set_needs_uniquify(struct evsel *counter, const struct perf_stat_config *config) +{ + struct evsel *evsel; + + if (counter->needs_uniquify) { + /* Already set. */ + return true; + } + + if (counter->use_config_name || counter->is_libpfm_event) { + /* Original name will be used. */ + return false; + } + + if (!config->hybrid_merge && evsel__is_hybrid(counter)) { + /* Unique hybrid counters necessary. */ + counter->needs_uniquify = true; + return true; + } + + if (counter->core.attr.type < PERF_TYPE_MAX && counter->core.attr.type != PERF_TYPE_RAW) { + /* Legacy event, don't uniquify. */ + return false; + } + + if (counter->pmu && counter->pmu->is_core && + counter->alternate_hw_config != PERF_COUNT_HW_MAX) { + /* A sysfs or json event replacing a legacy event, don't uniquify. */ + return false; + } + + if (config->aggr_mode == AGGR_NONE) { + /* Always unique with no aggregation. */ + counter->needs_uniquify = true; + return true; + } + + if (counter->first_wildcard_match != NULL) { + /* + * If stats are merged then only the first_wildcard_match is + * displayed, there is no need to uniquify this evsel as the + * name won't be shown. + */ + return false; + } + + /* + * Do other non-merged events in the evlist have the same name? If so + * uniquify is necessary. + */ + evlist__for_each_entry(counter->evlist, evsel) { + if (evsel == counter || evsel->first_wildcard_match || evsel->pmu == counter->pmu) + continue; + + if (evsel__name_is(counter, evsel__name(evsel))) { + counter->needs_uniquify = true; + return true; + } + } + return false; +} + +void evsel__uniquify_counter(struct evsel *counter) +{ + const char *name, *pmu_name, *config; + char *new_name; + int len, ret; + + /* No uniquification necessary. */ + if (!counter->needs_uniquify) + return; + + /* The evsel was already uniquified. */ + if (counter->uniquified_name) + return; + + /* Avoid checking to uniquify twice. */ + counter->uniquified_name = true; + + name = evsel__name(counter); + config = strchr(name, '/'); + pmu_name = counter->pmu->name; + + /* Already prefixed by the PMU name? */ + len = pmu_name_len_no_suffix(pmu_name); + + if (!strncmp(name, pmu_name, len)) { + /* + * If the PMU name is there, then there is no sense in not + * having a slash. Do this for robustness. + */ + if (config == NULL) + config = name - 1; + + ret = asprintf(&new_name, "%s/%s", pmu_name, config + 1); + } else if (config) { + len = config - name; + if (config[1] == '/') { + /* case: event// */ + ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 2); + } else { + /* case: event/.../ */ + ret = asprintf(&new_name, "%s/%.*s,%s", pmu_name, len, name, config + 1); + } + } else { + config = strchr(name, ':'); + if (config) { + /* case: event:.. */ + len = config - name; + + ret = asprintf(&new_name, "%s/%.*s/%s", pmu_name, len, name, config + 1); + } else { + /* case: event */ + ret = asprintf(&new_name, "%s/%s/", pmu_name, name); + } + } + if (ret > 0) { + free(counter->name); + counter->name = new_name; + } else { + /* ENOMEM from asprintf. */ + counter->uniquified_name = false; + } +} + +void evsel__warn_user_requested_cpus(struct evsel *evsel, struct perf_cpu_map *user_requested_cpus) +{ + struct perf_cpu_map *intersect, *online = NULL; + const struct perf_pmu *pmu = evsel__find_pmu(evsel); + + if (pmu && pmu->is_core) { + intersect = perf_cpu_map__intersect(pmu->cpus, user_requested_cpus); + } else { + online = cpu_map__online(); + intersect = perf_cpu_map__intersect(online, user_requested_cpus); + } + if (!perf_cpu_map__equal(intersect, user_requested_cpus)) { + char buf1[128]; + char buf2[128]; + + cpu_map__snprint(user_requested_cpus, buf1, sizeof(buf1)); + cpu_map__snprint(online ?: pmu->cpus, buf2, sizeof(buf2)); + pr_warning("WARNING: A requested CPU in '%s' is not supported by PMU '%s' (CPUs %s) for event '%s'\n", + buf1, pmu ? pmu->name : "cpu", buf2, evsel__name(evsel)); + } + perf_cpu_map__put(intersect); + perf_cpu_map__put(online); +} diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h new file mode 100644 index 000000000..163fc2b6a --- /dev/null +++ b/tools/perf/util/evsel.h @@ -0,0 +1,627 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_EVSEL_H +#define __PERF_EVSEL_H 1 + +#include +#include +#include +#include +#include +#include +#include +#include "symbol_conf.h" +#include "pmus.h" +#include "pmu.h" + +struct bpf_object; +struct cgroup; +struct perf_counts; +struct perf_stat_config; +struct perf_stat_evsel; +union perf_event; +struct bpf_counter_ops; +struct target; +struct hashmap; +struct bperf_leader_bpf; +struct bperf_follower_bpf; + +typedef int (evsel__sb_cb_t)(union perf_event *event, void *data); + +/** struct evsel - event selector + * + * @evlist - evlist this evsel is in, if it is in one. + * @core - libperf evsel object + * @name - Can be set to retain the original event name passed by the user, + * so that when showing results in tools such as 'perf stat', we + * show the name used, not some alias. + * @id_pos: the position of the event id (PERF_SAMPLE_ID or + * PERF_SAMPLE_IDENTIFIER) in a sample event i.e. in the array of + * struct perf_record_sample + * @is_pos: the position (counting backwards) of the event id (PERF_SAMPLE_ID or + * PERF_SAMPLE_IDENTIFIER) in a non-sample event i.e. if sample_id_all + * is used there is an id sample appended to non-sample events + * @priv: And what is in its containing unnamed union are tool specific + */ +struct evsel { + struct perf_evsel core; + struct evlist *evlist; + off_t id_offset; + int id_pos; + int is_pos; + unsigned int sample_size; + + /* + * These fields can be set in the parse-events code or similar. + * Please check evsel__clone() to copy them properly so that + * they can be released properly. + */ + struct { + char *name; + char *group_name; + const char *group_pmu_name; +#ifdef HAVE_LIBTRACEEVENT + char *tp_sys; + char *tp_name; + struct tep_event *tp_format; +#endif + char *filter; + unsigned long max_events; + double scale; + const char *unit; + struct cgroup *cgrp; + const char *metric_id; + /* + * This point to the first evsel with the same name, intended to store the + * aggregated counts in aggregation mode. + */ + struct evsel *first_wildcard_match; + /* parse modifier helper */ + int exclude_GH; + int sample_read; + bool snapshot; + bool per_pkg; + bool percore; + bool precise_max; + bool is_libpfm_event; + bool collect_stat; + bool weak_group; + bool bpf_counter; + bool use_config_name; + bool skippable; + bool retire_lat; + bool dont_regroup; + int bpf_fd; + struct bpf_object *bpf_obj; + struct list_head config_terms; + u64 alternate_hw_config; + }; + + /* + * metric fields are similar, but needs more care as they can have + * references to other metric (evsel). + */ + struct evsel *metric_leader; + + void *handler; + struct perf_counts *counts; + struct perf_counts *prev_raw_counts; + unsigned long nr_events_printed; + struct perf_stat_evsel *stats; + void *priv; + u64 db_id; + bool uniquified_name; + bool supported; + bool needs_swap; + bool disabled; + bool no_aux_samples; + bool immediate; + bool tracking; + bool ignore_missing_thread; + bool forced_leader; + bool cmdline_group_boundary; + bool reset_group; + bool needs_auxtrace_mmap; + bool default_metricgroup; /* A member of the Default metricgroup */ + bool default_show_events; /* If a default group member, show the event */ + bool needs_uniquify; + bool fallenback_eacces; + bool fallenback_eopnotsupp; + u8 probe_type:3; + struct hashmap *per_pkg_mask; + int err; + int script_output_type; + struct { + evsel__sb_cb_t *cb; + void *data; + } side_band; + /* + * For reporting purposes, an evsel sample can have a callchain + * synthesized from AUX area data. Keep track of synthesized sample + * types here. Note, the recorded sample_type cannot be changed because + * it is needed to continue to parse events. + * See also evsel__has_callchain(). + */ + __u64 synth_sample_type; + + /* + * Store the branch counter related information. + * br_cntr_idx: The idx of the branch counter event in the evlist + * br_cntr_nr: The number of the branch counter event in the group + * (Only available for the leader event) + * abbr_name: The abbreviation name assigned to an event which is + * logged by the branch counter. + * The abbr name is from A to Z9. NA is applied if out + * of the range. + */ + int br_cntr_idx; + int br_cntr_nr; + char abbr_name[3]; + + /* + * bpf_counter_ops serves two use cases: + * 1. perf-stat -b counting events used byBPF programs + * 2. perf-stat --use-bpf use BPF programs to aggregate counts + */ + struct bpf_counter_ops *bpf_counter_ops; + + struct list_head bpf_counter_list; /* for perf-stat -b */ + struct list_head bpf_filters; /* for perf-record --filter */ + + /* for perf-stat --use-bpf */ + int bperf_leader_prog_fd; + int bperf_leader_link_fd; + union { + struct bperf_leader_bpf *leader_skel; + struct bperf_follower_bpf *follower_skel; + void *bpf_skel; + }; + unsigned long open_flags; + int precise_ip_original; + + /* The PMU the event is from. Used for missing_features, PMU name, etc. */ + struct perf_pmu *pmu; + + /* For tool events */ + /* Beginning time subtracted when the counter is read. */ + union { + /* Defaults for retirement latency events. */ + struct _retirement_latency { + double mean; + double min; + double max; + } retirement_latency; + /* duration_time is a single global time. */ + struct { + __u64 start_time; + __u64 accumulated_time; + } duration_time; + /* + * user_time and system_time read an initial value potentially + * per-CPU or per-pid. + */ + struct { + struct xyarray *start_times; + struct xyarray *accumulated_times; + } process_time; + }; + /* Is the tool's fd for /proc/pid/stat or /proc/stat. */ + bool pid_stat; +}; + +struct perf_missing_features { + bool sample_id_all; + bool exclude_guest; + bool mmap2; + bool cloexec; + bool clockid; + bool clockid_wrong; + bool lbr_flags; + bool write_backward; + bool group_read; + bool ksymbol; + bool bpf; + bool aux_output; + bool branch_hw_idx; + bool cgroup; + bool data_page_size; + bool code_page_size; + bool weight_struct; + bool read_lost; + bool branch_counters; + bool aux_action; + bool inherit_sample_read; + bool defer_callchain; +}; + +extern struct perf_missing_features perf_missing_features; + +struct perf_cpu_map; +struct thread_map; +struct record_opts; + +static inline struct perf_cpu_map *evsel__cpus(struct evsel *evsel) +{ + return perf_evsel__cpus(&evsel->core); +} + +static inline int evsel__nr_cpus(struct evsel *evsel) +{ + return perf_cpu_map__nr(evsel__cpus(evsel)); +} + +void evsel__compute_deltas(struct evsel *evsel, int cpu, int thread, + struct perf_counts_values *count); + +int evsel__object_config(size_t object_size, + int (*init)(struct evsel *evsel), + void (*fini)(struct evsel *evsel)); + +struct perf_pmu *evsel__find_pmu(const struct evsel *evsel); +const char *evsel__pmu_name(const struct evsel *evsel); +bool evsel__is_aux_event(const struct evsel *evsel); + +bool evsel__is_probe(struct evsel *evsel); +bool evsel__is_kprobe(struct evsel *evsel); +bool evsel__is_uprobe(struct evsel *evsel); + +struct evsel *evsel__new_idx(struct perf_event_attr *attr, int idx); + +static inline struct evsel *evsel__new(struct perf_event_attr *attr) +{ + return evsel__new_idx(attr, 0); +} + +struct evsel *evsel__clone(struct evsel *dest, struct evsel *orig); + +int copy_config_terms(struct list_head *dst, struct list_head *src); +void free_config_terms(struct list_head *config_terms); + + +/* + * Returns pointer with encoded error via interface. + */ +struct evsel *evsel__newtp_idx(const char *sys, const char *name, int idx, bool format); +static inline struct evsel *evsel__newtp(const char *sys, const char *name) +{ + return evsel__newtp_idx(sys, name, 0, true); +} + +#ifdef HAVE_LIBTRACEEVENT +struct tep_event *evsel__tp_format(struct evsel *evsel); +#endif + +void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx); +void evsel__exit(struct evsel *evsel); +void evsel__delete(struct evsel *evsel); + +void evsel__set_priv_destructor(void (*destructor)(void *priv)); + +struct callchain_param; + +void evsel__config(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *callchain); +void evsel__config_callchain(struct evsel *evsel, const struct record_opts *opts, + const struct callchain_param *callchain); + +int __evsel__sample_size(u64 sample_type); +void evsel__calc_id_pos(struct evsel *evsel); + +bool evsel__is_cache_op_valid(u8 type, u8 op); + +static inline bool evsel__is_bpf(struct evsel *evsel) +{ + return evsel->bpf_counter_ops != NULL; +} + +static inline bool evsel__is_bperf(struct evsel *evsel) +{ + return evsel->bpf_counter_ops != NULL && list_empty(&evsel->bpf_counter_list); +} + +#define EVSEL__MAX_ALIASES 8 + +extern const char *const evsel__hw_cache[PERF_COUNT_HW_CACHE_MAX][EVSEL__MAX_ALIASES]; +extern const char *const evsel__hw_cache_op[PERF_COUNT_HW_CACHE_OP_MAX][EVSEL__MAX_ALIASES]; +extern const char *const evsel__hw_cache_result[PERF_COUNT_HW_CACHE_RESULT_MAX][EVSEL__MAX_ALIASES]; +extern const char *const evsel__hw_names[PERF_COUNT_HW_MAX]; +extern const char *const evsel__sw_names[PERF_COUNT_SW_MAX]; +extern char *evsel__bpf_counter_events; +bool evsel__match_bpf_counter_events(const char *name); +int arch_evsel__hw_name(struct evsel *evsel, char *bf, size_t size); + +int __evsel__hw_cache_type_op_res_name(u8 type, u8 op, u8 result, char *bf, size_t size); +const char *evsel__name(struct evsel *evsel); +bool evsel__name_is(struct evsel *evsel, const char *name); +const char *evsel__metric_id(const struct evsel *evsel); + +static inline bool evsel__is_retire_lat(const struct evsel *evsel) +{ + return evsel->retire_lat; +} + +const char *evsel__group_name(struct evsel *evsel); +int evsel__group_desc(struct evsel *evsel, char *buf, size_t size); + +void __evsel__set_sample_bit(struct evsel *evsel, enum perf_event_sample_format bit); +void __evsel__reset_sample_bit(struct evsel *evsel, enum perf_event_sample_format bit); + +#define evsel__set_sample_bit(evsel, bit) \ + __evsel__set_sample_bit(evsel, PERF_SAMPLE_##bit) + +#define evsel__reset_sample_bit(evsel, bit) \ + __evsel__reset_sample_bit(evsel, PERF_SAMPLE_##bit) + +void evsel__set_sample_id(struct evsel *evsel, bool use_sample_identifier); + +void arch_evsel__set_sample_weight(struct evsel *evsel); +void arch__post_evsel_config(struct evsel *evsel, struct perf_event_attr *attr); +int arch_evsel__open_strerror(struct evsel *evsel, int err, char *msg, size_t size); +void arch_evsel__apply_ratio_to_prev(struct evsel *evsel, struct perf_event_attr *attr); + +int evsel__set_filter(struct evsel *evsel, const char *filter); +int evsel__append_tp_filter(struct evsel *evsel, const char *filter); +int evsel__append_addr_filter(struct evsel *evsel, const char *filter); +static inline bool evsel__is_non_perf_event_open_pmu(const struct evsel *evsel) +{ + return evsel->pmu && evsel->pmu->type > PERF_PMU_TYPE_PE_END; +} + +int evsel__enable_cpu(struct evsel *evsel, int cpu_map_idx); +int evsel__enable(struct evsel *evsel); +int evsel__disable(struct evsel *evsel); +int evsel__disable_cpu(struct evsel *evsel, int cpu_map_idx); + +int evsel__open_per_cpu_and_thread(struct evsel *evsel, + struct perf_cpu_map *cpus, int cpu_map_idx, + struct perf_thread_map *threads); +int evsel__open_per_cpu(struct evsel *evsel, struct perf_cpu_map *cpus, int cpu_map_idx); +int evsel__open_per_thread(struct evsel *evsel, struct perf_thread_map *threads); +int evsel__open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads); +void evsel__close(struct evsel *evsel); +int evsel__prepare_open(struct evsel *evsel, struct perf_cpu_map *cpus, + struct perf_thread_map *threads); + +bool evsel__precise_ip_fallback(struct evsel *evsel); + +struct perf_sample; + +#ifdef HAVE_LIBTRACEEVENT +void *perf_sample__rawptr(struct perf_sample *sample, const char *name); +u64 perf_sample__intval(struct perf_sample *sample, const char *name); +u64 perf_sample__intval_common(struct perf_sample *sample, const char *name); +char perf_sample__taskstate(struct perf_sample *sample, const char *name); + +static inline char *perf_sample__strval(struct perf_sample *sample, const char *name) +{ + return perf_sample__rawptr(sample, name); +} +#endif + +struct tep_format_field; + +u64 format_field__intval(struct tep_format_field *field, struct perf_sample *sample, bool needs_swap); + +#ifdef HAVE_LIBTRACEEVENT +struct tep_format_field *evsel__field(struct evsel *evsel, const char *name); +struct tep_format_field *evsel__common_field(struct evsel *evsel, const char *name); +#else +static inline struct tep_format_field * +evsel__field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} + +static inline struct tep_format_field * +evsel__common_field(struct evsel *evsel __maybe_unused, const char *name __maybe_unused) +{ + return NULL; +} +#endif + +bool __evsel__match(const struct evsel *evsel, u32 type, u64 config); + +#define evsel__match(evsel, t, c) __evsel__match(evsel, PERF_TYPE_##t, PERF_COUNT_##c) + +int evsel__read_counter(struct evsel *evsel, int cpu_map_idx, int thread); + +int __evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread, bool scale); + +/** + * evsel__read_on_cpu - Read out the results on a CPU and thread + * + * @evsel - event selector to read value + * @cpu_map_idx - CPU of interest + * @thread - thread of interest + */ +static inline int evsel__read_on_cpu(struct evsel *evsel, int cpu_map_idx, int thread) +{ + return __evsel__read_on_cpu(evsel, cpu_map_idx, thread, false); +} + +/** + * evsel__read_on_cpu_scaled - Read out the results on a CPU and thread, scaled + * + * @evsel - event selector to read value + * @cpu_map_idx - CPU of interest + * @thread - thread of interest + */ +static inline int evsel__read_on_cpu_scaled(struct evsel *evsel, int cpu_map_idx, int thread) +{ + return __evsel__read_on_cpu(evsel, cpu_map_idx, thread, true); +} + +int __evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data, bool needs_swap); + +static inline int evsel__parse_sample(struct evsel *evsel, union perf_event *event, + struct perf_sample *data) +{ + return __evsel__parse_sample(evsel, event, data, evsel->needs_swap); +} + +int evsel__parse_sample_timestamp(struct evsel *evsel, union perf_event *event, + u64 *timestamp); + +u16 evsel__id_hdr_size(const struct evsel *evsel); + +static inline struct evsel *evsel__next(struct evsel *evsel) +{ + return list_entry(evsel->core.node.next, struct evsel, core.node); +} + +static inline struct evsel *evsel__prev(struct evsel *evsel) +{ + return list_entry(evsel->core.node.prev, struct evsel, core.node); +} + +/** + * evsel__is_group_leader - Return whether given evsel is a leader event + * + * @evsel - evsel selector to be tested + * + * Return %true if @evsel is a group leader or a stand-alone event + */ +static inline bool evsel__is_group_leader(const struct evsel *evsel) +{ + return evsel->core.leader == &evsel->core; +} + +/** + * evsel__is_group_event - Return whether given evsel is a group event + * + * @evsel - evsel selector to be tested + * + * Return %true iff event group view is enabled and @evsel is a actual group + * leader which has other members in the group + */ +static inline bool evsel__is_group_event(struct evsel *evsel) +{ + if (!symbol_conf.event_group) + return false; + + return evsel__is_group_leader(evsel) && evsel->core.nr_members > 1; +} + +bool evsel__is_function_event(struct evsel *evsel); + +static inline bool evsel__is_bpf_output(struct evsel *evsel) +{ + return evsel__match(evsel, SOFTWARE, SW_BPF_OUTPUT); +} + +static inline bool evsel__is_clock(const struct evsel *evsel) +{ + return evsel__match(evsel, SOFTWARE, SW_CPU_CLOCK) || + evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK); +} + +bool evsel__fallback(struct evsel *evsel, struct target *target, int err, + char *msg, size_t msgsize); +int evsel__open_strerror(struct evsel *evsel, struct target *target, + int err, char *msg, size_t size); + +static inline int evsel__group_idx(struct evsel *evsel) +{ + return evsel->core.idx - evsel->core.leader->idx; +} + +/* Iterates group WITHOUT the leader. */ +#define for_each_group_member_head(_evsel, _leader, _head) \ +for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node); \ + (_evsel) && &(_evsel)->core.node != (_head) && \ + (_evsel)->core.leader == &(_leader)->core; \ + (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) + +#define for_each_group_member(_evsel, _leader) \ + for_each_group_member_head(_evsel, _leader, &(_leader)->evlist->core.entries) + +/* Iterates group WITH the leader. */ +#define for_each_group_evsel_head(_evsel, _leader, _head) \ +for ((_evsel) = _leader; \ + (_evsel) && &(_evsel)->core.node != (_head) && \ + (_evsel)->core.leader == &(_leader)->core; \ + (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) + +#define for_each_group_evsel(_evsel, _leader) \ + for_each_group_evsel_head(_evsel, _leader, &(_leader)->evlist->core.entries) + +static inline bool evsel__has_branch_callstack(const struct evsel *evsel) +{ + return evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_CALL_STACK; +} + +static inline bool evsel__has_branch_hw_idx(const struct evsel *evsel) +{ + return evsel->core.attr.branch_sample_type & PERF_SAMPLE_BRANCH_HW_INDEX; +} + +static inline bool evsel__has_callchain(const struct evsel *evsel) +{ + /* + * For reporting purposes, an evsel sample can have a recorded callchain + * or a callchain synthesized from AUX area data. + */ + return evsel->core.attr.sample_type & PERF_SAMPLE_CALLCHAIN || + evsel->synth_sample_type & PERF_SAMPLE_CALLCHAIN; +} + +static inline bool evsel__has_br_stack(const struct evsel *evsel) +{ + /* + * For reporting purposes, an evsel sample can have a recorded branch + * stack or a branch stack synthesized from AUX area data. + */ + return evsel->core.attr.sample_type & PERF_SAMPLE_BRANCH_STACK || + evsel->synth_sample_type & PERF_SAMPLE_BRANCH_STACK; +} + +static inline bool evsel__is_dummy_event(struct evsel *evsel) +{ + return (evsel->core.attr.type == PERF_TYPE_SOFTWARE) && + (evsel->core.attr.config == PERF_COUNT_SW_DUMMY); +} + +struct perf_session *evsel__session(struct evsel *evsel); +struct perf_env *evsel__env(struct evsel *evsel); +uint16_t evsel__e_machine(struct evsel *evsel, uint32_t *e_flags); + +int evsel__store_ids(struct evsel *evsel, struct evlist *evlist); + +void evsel__zero_per_pkg(struct evsel *evsel); +bool evsel__is_hybrid(const struct evsel *evsel); +struct evsel *evsel__leader(const struct evsel *evsel); +bool evsel__has_leader(struct evsel *evsel, struct evsel *leader); +bool evsel__is_leader(struct evsel *evsel); +void evsel__set_leader(struct evsel *evsel, struct evsel *leader); +int evsel__source_count(const struct evsel *evsel); +void evsel__remove_from_group(struct evsel *evsel, struct evsel *leader); + +bool arch_evsel__must_be_in_group(const struct evsel *evsel); + +bool evsel__set_needs_uniquify(struct evsel *counter, const struct perf_stat_config *config); +void evsel__uniquify_counter(struct evsel *counter); + +/* + * Macro to swap the bit-field postition and size. + * Used when, + * - dont need to swap the entire u64 && + * - when u64 has variable bit-field sizes && + * - when presented in a host endian which is different + * than the source endian of the perf.data file + */ +#define bitfield_swap(src, pos, size) \ + ((((src) >> (pos)) & ((1ull << (size)) - 1)) << (63 - ((pos) + (size) - 1))) + +u64 evsel__bitfield_swap_branch_flags(u64 value); +bool evsel__config_exists(const struct evsel *evsel, const char *config_name); +int evsel__get_config_val(const struct evsel *evsel, const char *config_name, + u64 *val); +void evsel__set_config_if_unset(struct evsel *evsel, const char *config_name, + u64 val); + +bool evsel__is_offcpu_event(struct evsel *evsel); + +void evsel__warn_user_requested_cpus(struct evsel *evsel, struct perf_cpu_map *user_requested_cpus); + +#endif /* __PERF_EVSEL_H */ diff --git a/tools/perf/util/evsel_config.h b/tools/perf/util/evsel_config.h new file mode 100644 index 000000000..7b565d76c --- /dev/null +++ b/tools/perf/util/evsel_config.h @@ -0,0 +1,69 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __PERF_EVSEL_CONFIG_H +#define __PERF_EVSEL_CONFIG_H 1 + +#include +#include + +/* + * The 'struct evsel_config_term' is used to pass event + * specific configuration data to evsel__config routine. + * It is allocated within event parsing and attached to + * evsel::config_terms list head. +*/ +enum evsel_term_type { + EVSEL__CONFIG_TERM_PERIOD, + EVSEL__CONFIG_TERM_FREQ, + EVSEL__CONFIG_TERM_TIME, + EVSEL__CONFIG_TERM_CALLGRAPH, + EVSEL__CONFIG_TERM_STACK_USER, + EVSEL__CONFIG_TERM_INHERIT, + EVSEL__CONFIG_TERM_MAX_STACK, + EVSEL__CONFIG_TERM_MAX_EVENTS, + EVSEL__CONFIG_TERM_OVERWRITE, + EVSEL__CONFIG_TERM_DRV_CFG, + EVSEL__CONFIG_TERM_BRANCH, + EVSEL__CONFIG_TERM_PERCORE, + EVSEL__CONFIG_TERM_AUX_OUTPUT, + EVSEL__CONFIG_TERM_AUX_ACTION, + EVSEL__CONFIG_TERM_AUX_SAMPLE_SIZE, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG1, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG2, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG3, + EVSEL__CONFIG_TERM_USR_CHG_CONFIG4, + EVSEL__CONFIG_TERM_RATIO_TO_PREV, +}; + +struct evsel_config_term { + struct list_head list; + enum evsel_term_type type; + bool free_str; + union { + u64 period; + u64 freq; + bool time; + u64 stack_user; + int max_stack; + bool inherit; + bool overwrite; + unsigned long max_events; + bool percore; + bool aux_output; + u32 aux_sample_size; + u64 cfg_chg; + char *str; + int cpu; + u64 val; + } val; + bool weak; +}; + +struct evsel; + +struct evsel_config_term *__evsel__get_config_term(struct evsel *evsel, enum evsel_term_type type); + +#define evsel__get_config_term(evsel, type) \ + __evsel__get_config_term(evsel, EVSEL__CONFIG_TERM_ ## type) + +#endif // __PERF_EVSEL_CONFIG_H diff --git a/tools/perf/util/evsel_fprintf.c b/tools/perf/util/evsel_fprintf.c new file mode 100644 index 000000000..0f7a25500 --- /dev/null +++ b/tools/perf/util/evsel_fprintf.c @@ -0,0 +1,258 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "util/evlist.h" +#include "evsel.h" +#include "util/evsel_fprintf.h" +#include "util/event.h" +#include "callchain.h" +#include "map.h" +#include "strlist.h" +#include "symbol.h" +#include "srcline.h" +#include "dso.h" + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +static int comma_fprintf(FILE *fp, bool *first, const char *fmt, ...) +{ + va_list args; + int ret = 0; + + if (!*first) { + ret += fprintf(fp, ","); + } else { + ret += fprintf(fp, ":"); + *first = false; + } + + va_start(args, fmt); + ret += vfprintf(fp, fmt, args); + va_end(args); + return ret; +} + +static int __print_attr__fprintf(FILE *fp, const char *name, const char *val, void *priv) +{ + return comma_fprintf(fp, (bool *)priv, " %s: %s", name, val); +} + +int evsel__fprintf(struct evsel *evsel, struct perf_attr_details *details, FILE *fp) +{ + bool first = true; + int printed = 0; + + if (details->event_group) { + struct evsel *pos; + + if (!evsel__is_group_leader(evsel)) + return 0; + + if (evsel->core.nr_members > 1) + printed += fprintf(fp, "%s{", evsel->group_name ?: ""); + + printed += fprintf(fp, "%s", evsel__name(evsel)); + for_each_group_member(pos, evsel) + printed += fprintf(fp, ",%s", evsel__name(pos)); + + if (evsel->core.nr_members > 1) + printed += fprintf(fp, "}"); + goto out; + } + + printed += fprintf(fp, "%s", evsel__name(evsel)); + + if (details->verbose) { + printed += perf_event_attr__fprintf(fp, &evsel->core.attr, + __print_attr__fprintf, &first); + } else if (details->freq) { + const char *term = "sample_freq"; + + if (!evsel->core.attr.freq) + term = "sample_period"; + + printed += comma_fprintf(fp, &first, " %s=%" PRIu64, + term, (u64)evsel->core.attr.sample_freq); + } + +#ifdef HAVE_LIBTRACEEVENT + if (details->trace_fields) { + struct tep_format_field *field; + const struct tep_event *tp_format; + + if (evsel->core.attr.type != PERF_TYPE_TRACEPOINT) { + printed += comma_fprintf(fp, &first, " (not a tracepoint)"); + goto out; + } + + tp_format = evsel__tp_format(evsel); + field = tp_format ? tp_format->format.fields : NULL; + if (field == NULL) { + printed += comma_fprintf(fp, &first, " (no trace field)"); + goto out; + } + + printed += comma_fprintf(fp, &first, " trace_fields: %s", field->name); + + field = field->next; + while (field) { + printed += comma_fprintf(fp, &first, "%s", field->name); + field = field->next; + } + } +#endif +out: + fputc('\n', fp); + return ++printed; +} + +int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, + unsigned int print_opts, struct callchain_cursor *cursor, + struct strlist *bt_stop_list, FILE *fp) +{ + int printed = 0; + struct callchain_cursor_node *node; + int print_ip = print_opts & EVSEL__PRINT_IP; + int print_sym = print_opts & EVSEL__PRINT_SYM; + int print_dso = print_opts & EVSEL__PRINT_DSO; + int print_dsoff = print_opts & EVSEL__PRINT_DSOFF; + int print_symoffset = print_opts & EVSEL__PRINT_SYMOFFSET; + int print_oneline = print_opts & EVSEL__PRINT_ONELINE; + int print_srcline = print_opts & EVSEL__PRINT_SRCLINE; + int print_unknown_as_addr = print_opts & EVSEL__PRINT_UNKNOWN_AS_ADDR; + int print_arrow = print_opts & EVSEL__PRINT_CALLCHAIN_ARROW; + int print_skip_ignored = print_opts & EVSEL__PRINT_SKIP_IGNORED; + char s = print_oneline ? ' ' : '\t'; + bool first = true; + + if (cursor == NULL) + return fprintf(fp, "%s", print_oneline ? "" : "\n"); + + if (sample->callchain) { + callchain_cursor_commit(cursor); + + while (1) { + struct symbol *sym; + struct map *map; + u64 addr = 0; + + node = callchain_cursor_current(cursor); + if (!node) + break; + + sym = node->ms.sym; + map = node->ms.map; + + if (sym && symbol__ignore(sym) && print_skip_ignored) + goto next; + + printed += fprintf(fp, "%-*.*s", left_alignment, left_alignment, " "); + + if (print_arrow && !first) + printed += fprintf(fp, " <-"); + + if (map) + addr = map__map_ip(map, node->ip); + + if (print_ip) + printed += fprintf(fp, "%c%16" PRIx64, s, node->ip); + + if (print_sym) { + struct addr_location node_al; + + addr_location__init(&node_al); + printed += fprintf(fp, " "); + node_al.addr = addr; + node_al.map = map__get(map); + + if (sample->deferred_callchain && + sample->deferred_cookie == node->ip) { + printed += fprintf(fp, "(cookie)"); + } else if (print_symoffset) { + printed += __symbol__fprintf_symname_offs(sym, &node_al, + print_unknown_as_addr, + true, fp); + } else { + printed += __symbol__fprintf_symname(sym, &node_al, + print_unknown_as_addr, fp); + } + addr_location__exit(&node_al); + } + + if (print_dso && (!sym || !symbol__inlined(sym))) + printed += map__fprintf_dsoname_dsoff(map, print_dsoff, addr, fp); + + if (print_srcline) { + if (node->srcline) + printed += fprintf(fp, "\n %s", node->srcline); + else + printed += map__fprintf_srcline(map, addr, "\n ", fp); + } + + if (sym && symbol__inlined(sym)) + printed += fprintf(fp, " (inlined)"); + + if (!print_oneline) + printed += fprintf(fp, "\n"); + + /* Add srccode here too? */ + if (bt_stop_list && sym && + strlist__has_entry(bt_stop_list, sym->name)) { + break; + } + + first = false; +next: + callchain_cursor_advance(cursor); + } + } + + return printed; +} + +int sample__fprintf_sym(struct perf_sample *sample, struct addr_location *al, + int left_alignment, unsigned int print_opts, + struct callchain_cursor *cursor, struct strlist *bt_stop_list, FILE *fp) +{ + int printed = 0; + int print_ip = print_opts & EVSEL__PRINT_IP; + int print_sym = print_opts & EVSEL__PRINT_SYM; + int print_dso = print_opts & EVSEL__PRINT_DSO; + int print_dsoff = print_opts & EVSEL__PRINT_DSOFF; + int print_symoffset = print_opts & EVSEL__PRINT_SYMOFFSET; + int print_srcline = print_opts & EVSEL__PRINT_SRCLINE; + int print_unknown_as_addr = print_opts & EVSEL__PRINT_UNKNOWN_AS_ADDR; + + if (cursor != NULL) { + printed += sample__fprintf_callchain(sample, left_alignment, print_opts, + cursor, bt_stop_list, fp); + } else { + printed += fprintf(fp, "%-*.*s", left_alignment, left_alignment, " "); + + if (print_ip) + printed += fprintf(fp, "%16" PRIx64, sample->ip); + + if (print_sym) { + printed += fprintf(fp, " "); + if (print_symoffset) { + printed += __symbol__fprintf_symname_offs(al->sym, al, + print_unknown_as_addr, + true, fp); + } else { + printed += __symbol__fprintf_symname(al->sym, al, + print_unknown_as_addr, fp); + } + } + + if (print_dso) + printed += map__fprintf_dsoname_dsoff(al->map, print_dsoff, al->addr, fp); + + if (print_srcline) + printed += map__fprintf_srcline(al->map, al->addr, "\n ", fp); + } + + return printed; +} diff --git a/tools/perf/util/evsel_fprintf.h b/tools/perf/util/evsel_fprintf.h new file mode 100644 index 000000000..c8a9fac2f --- /dev/null +++ b/tools/perf/util/evsel_fprintf.h @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0 +#ifndef __PERF_EVSEL_FPRINTF_H +#define __PERF_EVSEL_FPRINTF_H 1 + +#include +#include + +struct evsel; + +struct perf_attr_details { + bool freq; + bool verbose; + bool event_group; + bool force; + bool trace_fields; +}; + +int evsel__fprintf(struct evsel *evsel, struct perf_attr_details *details, FILE *fp); + +#define EVSEL__PRINT_IP (1<<0) +#define EVSEL__PRINT_SYM (1<<1) +#define EVSEL__PRINT_DSO (1<<2) +#define EVSEL__PRINT_SYMOFFSET (1<<3) +#define EVSEL__PRINT_ONELINE (1<<4) +#define EVSEL__PRINT_SRCLINE (1<<5) +#define EVSEL__PRINT_UNKNOWN_AS_ADDR (1<<6) +#define EVSEL__PRINT_CALLCHAIN_ARROW (1<<7) +#define EVSEL__PRINT_SKIP_IGNORED (1<<8) +#define EVSEL__PRINT_DSOFF (1<<9) + +struct addr_location; +struct perf_event_attr; +struct perf_sample; +struct callchain_cursor; +struct strlist; + +int sample__fprintf_callchain(struct perf_sample *sample, int left_alignment, + unsigned int print_opts, struct callchain_cursor *cursor, + struct strlist *bt_stop_list, FILE *fp); + +int sample__fprintf_sym(struct perf_sample *sample, struct addr_location *al, + int left_alignment, unsigned int print_opts, + struct callchain_cursor *cursor, + struct strlist *bt_stop_list, FILE *fp); + +typedef int (*attr__fprintf_f)(FILE *, const char *, const char *, void *); + +int perf_event_attr__fprintf(FILE *fp, struct perf_event_attr *attr, + attr__fprintf_f attr__fprintf, void *priv); +#endif // __PERF_EVSEL_H diff --git a/tools/perf/util/evswitch.c b/tools/perf/util/evswitch.c new file mode 100644 index 000000000..d4c06a3f8 --- /dev/null +++ b/tools/perf/util/evswitch.c @@ -0,0 +1,62 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019, Red Hat Inc, Arnaldo Carvalho de Melo + +#include +#include "evswitch.h" +#include "evlist.h" + +bool evswitch__discard(struct evswitch *evswitch, struct evsel *evsel) +{ + if (evswitch->on && evswitch->discarding) { + if (evswitch->on != evsel) + return true; + + evswitch->discarding = false; + + if (!evswitch->show_on_off_events) + return true; + + return false; + } + + if (evswitch->off && !evswitch->discarding) { + if (evswitch->off != evsel) + return false; + + evswitch->discarding = true; + + if (!evswitch->show_on_off_events) + return true; + } + + return false; +} + +static int evswitch__fprintf_enoent(FILE *fp, const char *evtype, const char *evname) +{ + int printed = fprintf(fp, "ERROR: switch-%s event not found (%s)\n", evtype, evname); + + return printed += fprintf(fp, "HINT: use 'perf evlist' to see the available event names\n"); +} + +int evswitch__init(struct evswitch *evswitch, struct evlist *evlist, FILE *fp) +{ + if (evswitch->on_name) { + evswitch->on = evlist__find_evsel_by_str(evlist, evswitch->on_name); + if (evswitch->on == NULL) { + evswitch__fprintf_enoent(fp, "on", evswitch->on_name); + return -ENOENT; + } + evswitch->discarding = true; + } + + if (evswitch->off_name) { + evswitch->off = evlist__find_evsel_by_str(evlist, evswitch->off_name); + if (evswitch->off == NULL) { + evswitch__fprintf_enoent(fp, "off", evswitch->off_name); + return -ENOENT; + } + } + + return 0; +} diff --git a/tools/perf/util/evswitch.h b/tools/perf/util/evswitch.h new file mode 100644 index 000000000..8ffdbe526 --- /dev/null +++ b/tools/perf/util/evswitch.h @@ -0,0 +1,31 @@ +// SPDX-License-Identifier: GPL-2.0-only +// Copyright (C) 2019, Red Hat Inc, Arnaldo Carvalho de Melo +#ifndef __PERF_EVSWITCH_H +#define __PERF_EVSWITCH_H 1 + +#include +#include + +struct evsel; +struct evlist; + +struct evswitch { + struct evsel *on, *off; + const char *on_name, *off_name; + bool discarding; + bool show_on_off_events; +}; + +int evswitch__init(struct evswitch *evswitch, struct evlist *evlist, FILE *fp); + +bool evswitch__discard(struct evswitch *evswitch, struct evsel *evsel); + +#define OPTS_EVSWITCH(evswitch) \ + OPT_STRING(0, "switch-on", &(evswitch)->on_name, \ + "event", "Consider events after the occurrence of this event"), \ + OPT_STRING(0, "switch-off", &(evswitch)->off_name, \ + "event", "Stop considering events after the occurrence of this event"), \ + OPT_BOOLEAN(0, "show-on-off-events", &(evswitch)->show_on_off_events, \ + "Show the on/off switch events, used with --switch-on and --switch-off") + +#endif /* __PERF_EVSWITCH_H */ diff --git a/tools/perf/util/expr.c b/tools/perf/util/expr.c new file mode 100644 index 000000000..232998fef --- /dev/null +++ b/tools/perf/util/expr.c @@ -0,0 +1,489 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include "metricgroup.h" +#include "debug.h" +#include "evlist.h" +#include "expr.h" +#include "smt.h" +#include "tool_pmu.h" +#include +#include +#include "util/hashmap.h" +#include "util/header.h" +#include "util/pmu.h" +#include +#include +#include +#include +#include +#include + +struct expr_id_data { + union { + struct { + double val; + int source_count; + int aggr_nr; + } val; + struct { + double val; + const char *metric_name; + const char *metric_expr; + } ref; + }; + + enum { + /* Holding a double value. */ + EXPR_ID_DATA__VALUE, + /* Reference to another metric. */ + EXPR_ID_DATA__REF, + /* A reference but the value has been computed. */ + EXPR_ID_DATA__REF_VALUE, + } kind; +}; + +static size_t key_hash(long key, void *ctx __maybe_unused) +{ + const char *str = (const char *)key; + size_t hash = 0; + + while (*str != '\0') { + hash *= 31; + hash += *str; + str++; + } + return hash; +} + +static bool key_equal(long key1, long key2, void *ctx __maybe_unused) +{ + return !strcmp((const char *)key1, (const char *)key2); +} + +struct hashmap *ids__new(void) +{ + struct hashmap *hash; + + hash = hashmap__new(key_hash, key_equal, NULL); + if (IS_ERR(hash)) + return NULL; + return hash; +} + +void ids__free(struct hashmap *ids) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (ids == NULL) + return; + + hashmap__for_each_entry(ids, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + + hashmap__free(ids); +} + +int ids__insert(struct hashmap *ids, const char *id) +{ + struct expr_id_data *data_ptr = NULL, *old_data = NULL; + char *old_key = NULL; + int ret; + + ret = hashmap__set(ids, id, data_ptr, &old_key, &old_data); + if (ret) + free(data_ptr); + free(old_key); + free(old_data); + return ret; +} + +struct hashmap *ids__union(struct hashmap *ids1, struct hashmap *ids2) +{ + size_t bkt; + struct hashmap_entry *cur; + int ret; + struct expr_id_data *old_data = NULL; + char *old_key = NULL; + + if (!ids1) + return ids2; + + if (!ids2) + return ids1; + + if (hashmap__size(ids1) < hashmap__size(ids2)) { + struct hashmap *tmp = ids1; + + ids1 = ids2; + ids2 = tmp; + } + hashmap__for_each_entry(ids2, cur, bkt) { + ret = hashmap__set(ids1, cur->key, cur->value, &old_key, &old_data); + free(old_key); + free(old_data); + + if (ret) { + hashmap__free(ids1); + hashmap__free(ids2); + return NULL; + } + } + hashmap__free(ids2); + return ids1; +} + +/* Caller must make sure id is allocated */ +int expr__add_id(struct expr_parse_ctx *ctx, const char *id) +{ + return ids__insert(ctx->ids, id); +} + +/* Caller must make sure id is allocated */ +int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val) +{ + return expr__add_id_val_source_count(ctx, id, val, /*source_count=*/1); +} + +/* Caller must make sure id is allocated */ +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr) +{ + struct expr_id_data *data_ptr = NULL, *old_data = NULL; + char *old_key = NULL; + int ret; + + data_ptr = malloc(sizeof(*data_ptr)); + if (!data_ptr) + return -ENOMEM; + data_ptr->val.val = val; + data_ptr->val.source_count = source_count; + data_ptr->val.aggr_nr = aggr_nr; + data_ptr->kind = EXPR_ID_DATA__VALUE; + + ret = hashmap__set(ctx->ids, id, data_ptr, &old_key, &old_data); + if (ret) { + free(data_ptr); + } else if (old_data) { + data_ptr->val.val += old_data->val.val; + data_ptr->val.source_count += old_data->val.source_count; + data_ptr->val.aggr_nr += old_data->val.aggr_nr; + } + free(old_key); + free(old_data); + return ret; +} + +/* Caller must make sure id is allocated */ +int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count) +{ + return expr__add_id_val_source_count_aggr_nr(ctx, id, val, source_count, 1); +} + +int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref) +{ + struct expr_id_data *data_ptr = NULL, *old_data = NULL; + char *old_key = NULL; + char *name; + int ret; + + data_ptr = zalloc(sizeof(*data_ptr)); + if (!data_ptr) + return -ENOMEM; + + name = strdup(ref->metric_name); + if (!name) { + free(data_ptr); + return -ENOMEM; + } + + /* + * Intentionally passing just const char pointers, + * originally from 'struct pmu_event' object. + * We don't need to change them, so there's no + * need to create our own copy. + */ + data_ptr->ref.metric_name = ref->metric_name; + data_ptr->ref.metric_expr = ref->metric_expr; + data_ptr->kind = EXPR_ID_DATA__REF; + + ret = hashmap__set(ctx->ids, name, data_ptr, &old_key, &old_data); + if (ret) + free(data_ptr); + + pr_debug2("adding ref metric %s: %s\n", + ref->metric_name, ref->metric_expr); + + free(old_key); + free(old_data); + return ret; +} + +int expr__get_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **data) +{ + if (!ctx || !id) + return -1; + return hashmap__find(ctx->ids, id, data) ? 0 : -1; +} + +bool expr__subset_of_ids(struct expr_parse_ctx *haystack, + struct expr_parse_ctx *needles) +{ + struct hashmap_entry *cur; + size_t bkt; + struct expr_id_data *data; + + hashmap__for_each_entry(needles->ids, cur, bkt) { + if (expr__get_id(haystack, cur->pkey, &data)) + return false; + } + return true; +} + + +int expr__resolve_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **datap) +{ + struct expr_id_data *data; + + if (expr__get_id(ctx, id, datap) || !*datap) { + pr_debug("%s not found\n", id); + return -1; + } + + data = *datap; + + switch (data->kind) { + case EXPR_ID_DATA__VALUE: + pr_debug2("lookup(%s): val %f\n", id, data->val.val); + break; + case EXPR_ID_DATA__REF: + pr_debug2("lookup(%s): ref metric name %s\n", id, + data->ref.metric_name); + pr_debug("processing metric: %s ENTRY\n", id); + data->kind = EXPR_ID_DATA__REF_VALUE; + if (expr__parse(&data->ref.val, ctx, data->ref.metric_expr)) { + pr_debug("%s failed to count\n", id); + return -1; + } + pr_debug("processing metric: %s EXIT: %f\n", id, data->ref.val); + break; + case EXPR_ID_DATA__REF_VALUE: + pr_debug2("lookup(%s): ref val %f metric name %s\n", id, + data->ref.val, data->ref.metric_name); + break; + default: + assert(0); /* Unreachable. */ + } + + return 0; +} + +void expr__del_id(struct expr_parse_ctx *ctx, const char *id) +{ + struct expr_id_data *old_val = NULL; + char *old_key = NULL; + + hashmap__delete(ctx->ids, id, &old_key, &old_val); + free(old_key); + free(old_val); +} + +struct expr_parse_ctx *expr__ctx_new(void) +{ + struct expr_parse_ctx *ctx; + + ctx = calloc(1, sizeof(struct expr_parse_ctx)); + if (!ctx) + return NULL; + + ctx->ids = hashmap__new(key_hash, key_equal, NULL); + if (IS_ERR(ctx->ids)) { + free(ctx); + return NULL; + } + + return ctx; +} + +void expr__ctx_clear(struct expr_parse_ctx *ctx) +{ + struct hashmap_entry *cur; + size_t bkt; + + hashmap__for_each_entry(ctx->ids, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__clear(ctx->ids); +} + +void expr__ctx_free(struct expr_parse_ctx *ctx) +{ + struct hashmap_entry *cur; + size_t bkt; + + if (!ctx) + return; + + zfree(&ctx->sctx.user_requested_cpu_list); + hashmap__for_each_entry(ctx->ids, cur, bkt) { + zfree(&cur->pkey); + zfree(&cur->pvalue); + } + hashmap__free(ctx->ids); + free(ctx); +} + +static int +__expr__parse(double *val, struct expr_parse_ctx *ctx, const char *expr, + bool compute_ids) +{ + YY_BUFFER_STATE buffer; + void *scanner; + int ret; + + pr_debug2("parsing metric: %s\n", expr); + + ret = expr_lex_init_extra(&ctx->sctx, &scanner); + if (ret) + return ret; + + buffer = expr__scan_string(expr, scanner); + +#ifdef PARSER_DEBUG + expr_debug = 1; + expr_set_debug(1, scanner); +#endif + + ret = expr_parse(val, ctx, compute_ids, scanner); + + expr__flush_buffer(buffer, scanner); + expr__delete_buffer(buffer, scanner); + expr_lex_destroy(scanner); + return ret; +} + +int expr__parse(double *final_val, struct expr_parse_ctx *ctx, + const char *expr) +{ + return __expr__parse(final_val, ctx, expr, /*compute_ids=*/false) ? -1 : 0; +} + +int expr__find_ids(const char *expr, const char *one, + struct expr_parse_ctx *ctx) +{ + int ret = __expr__parse(NULL, ctx, expr, /*compute_ids=*/true); + + if (one) + expr__del_id(ctx, one); + + /* A positive value means syntax error, convert to -EINVAL */ + return ret > 0 ? -EINVAL : ret; +} + +double expr_id_data__value(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.val; + assert(data->kind == EXPR_ID_DATA__REF_VALUE); + return data->ref.val; +} + +double expr_id_data__source_count(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.source_count; + return 1.0; +} + +double expr_id_data__aggr_nr(const struct expr_id_data *data) +{ + if (data->kind == EXPR_ID_DATA__VALUE) + return data->val.aggr_nr; + return 1.0; +} + +double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx) +{ + double result = NAN; + enum tool_pmu_event ev = tool_pmu__str_to_event(literal + 1); + + if (ev != TOOL_PMU__EVENT_NONE) { + u64 count; + + if (tool_pmu__read_event(ev, /*evsel=*/NULL, + ctx->system_wide, ctx->user_requested_cpu_list, + &count)) + result = count; + else + pr_err("Failure to read '%s'\n", literal); + } else { + pr_err("Unrecognized literal '%s'\n", literal); + } + + pr_debug2("literal: %s = %f\n", literal, result); + return result; +} + +/* Does the event 'id' parse? Determine via ctx->ids if possible. */ +double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id) +{ + struct evlist *tmp; + double ret; + + if (hashmap__find(ctx->ids, id, /*value=*/NULL)) + return 1.0; + + if (!compute_ids) + return 0.0; + + tmp = evlist__new(); + if (!tmp) + return NAN; + + if (strchr(id, '@')) { + char *tmp_id, *p; + + tmp_id = strdup(id); + if (!tmp_id) { + ret = NAN; + goto out; + } + p = strchr(tmp_id, '@'); + *p = '/'; + p = strrchr(tmp_id, '@'); + *p = '/'; + ret = parse_event(tmp, tmp_id) ? 0 : 1; + free(tmp_id); + } else { + ret = parse_event(tmp, id) ? 0 : 1; + } +out: + evlist__delete(tmp); + return ret; +} + +double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx __maybe_unused, + bool compute_ids __maybe_unused, const char *test_id) +{ + double ret; + struct perf_cpu cpu = {-1}; + char *cpuid = get_cpuid_allow_env_override(cpu); + + if (!cpuid) + return NAN; + + ret = !strcmp_cpuid_str(test_id, cpuid); + + free(cpuid); + return ret; +} diff --git a/tools/perf/util/expr.h b/tools/perf/util/expr.h new file mode 100644 index 000000000..ed12e4007 --- /dev/null +++ b/tools/perf/util/expr.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef PARSE_CTX_H +#define PARSE_CTX_H 1 + +struct hashmap; +struct metric_ref; + +struct expr_scanner_ctx { + char *user_requested_cpu_list; + int runtime; + bool system_wide; + bool is_test; +}; + +struct expr_parse_ctx { + struct hashmap *ids; + struct expr_scanner_ctx sctx; +}; + +struct expr_id_data; + +struct hashmap *ids__new(void); +void ids__free(struct hashmap *ids); +int ids__insert(struct hashmap *ids, const char *id); +/* + * Union two sets of ids (hashmaps) and construct a third, freeing ids1 and + * ids2. + */ +struct hashmap *ids__union(struct hashmap *ids1, struct hashmap *ids2); + +struct expr_parse_ctx *expr__ctx_new(void); +void expr__ctx_clear(struct expr_parse_ctx *ctx); +void expr__ctx_free(struct expr_parse_ctx *ctx); + +void expr__del_id(struct expr_parse_ctx *ctx, const char *id); +int expr__add_id(struct expr_parse_ctx *ctx, const char *id); +int expr__add_id_val(struct expr_parse_ctx *ctx, const char *id, double val); +int expr__add_id_val_source_count(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count); +int expr__add_id_val_source_count_aggr_nr(struct expr_parse_ctx *ctx, const char *id, + double val, int source_count, int aggr_nr); +int expr__add_ref(struct expr_parse_ctx *ctx, struct metric_ref *ref); +int expr__get_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **data); +bool expr__subset_of_ids(struct expr_parse_ctx *haystack, + struct expr_parse_ctx *needles); +int expr__resolve_id(struct expr_parse_ctx *ctx, const char *id, + struct expr_id_data **datap); + +int expr__parse(double *final_val, struct expr_parse_ctx *ctx, + const char *expr); + +int expr__find_ids(const char *expr, const char *one, + struct expr_parse_ctx *ids); + +double expr_id_data__value(const struct expr_id_data *data); +double expr_id_data__source_count(const struct expr_id_data *data); +double expr_id_data__aggr_nr(const struct expr_id_data *data); + +double expr__get_literal(const char *literal, const struct expr_scanner_ctx *ctx); +double expr__has_event(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); +double expr__strcmp_cpuid_str(const struct expr_parse_ctx *ctx, bool compute_ids, const char *id); + +#endif diff --git a/tools/perf/util/expr.l b/tools/perf/util/expr.l new file mode 100644 index 000000000..f16ccff27 --- /dev/null +++ b/tools/perf/util/expr.l @@ -0,0 +1,150 @@ +%option prefix="expr_" +%option reentrant +%option bison-bridge + +%{ +#include +#include "expr.h" +#include "expr-bison.h" +#include + +char *expr_get_text(yyscan_t yyscanner); +YYSTYPE *expr_get_lval(yyscan_t yyscanner); + +static double __value(YYSTYPE *yylval, char *str, int token) +{ + double num; + + errno = 0; + num = strtod(str, NULL); + if (errno) + return EXPR_ERROR; + + yylval->num = num; + return token; +} + +static int value(yyscan_t scanner) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + char *text = expr_get_text(scanner); + + return __value(yylval, text, NUMBER); +} + +/* + * Allow @ instead of / to be able to specify pmu/event/ without + * conflicts with normal division. + */ +static char *normalize(char *str, int runtime) +{ + char *ret = str; + char *dst = str; + + while (*str) { + if (*str == '\\') { + *dst++ = *++str; + if (!*str) + break; + } + else if (*str == '?') { + char *paramval; + int i = 0; + int size = asprintf(¶mval, "%d", runtime); + + if (size < 0) + *dst++ = '0'; + else { + while (i < size) + *dst++ = paramval[i++]; + free(paramval); + } + } + else + *dst++ = *str; + str++; + } + + *dst = 0x0; + return ret; +} + +static int str(yyscan_t scanner, int token, int runtime) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + char *text = expr_get_text(scanner); + + yylval->str = normalize(strdup(text), runtime); + if (!yylval->str) + return EXPR_ERROR; + + yylval->str = normalize(yylval->str, runtime); + return token; +} + +static int literal(yyscan_t scanner, const struct expr_scanner_ctx *sctx) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + + yylval->num = expr__get_literal(expr_get_text(scanner), sctx); + if (isnan(yylval->num)) { + if (!sctx->is_test) + return EXPR_ERROR; + yylval->num = 1; + } + return LITERAL; +} + +static int nan_value(yyscan_t scanner) +{ + YYSTYPE *yylval = expr_get_lval(scanner); + + yylval->num = NAN; + return NUMBER; +} +%} + +number ([0-9]+\.?[0-9]*|[0-9]*\.?[0-9]+)(e-?[0-9]+)? + +sch [-,=] +spec \\{sch} +sym [0-9a-zA-Z_\.:@?]+ +symbol ({spec}|{sym})+ +literal #[0-9a-zA-Z_\.\-]+ + +%% + struct expr_scanner_ctx *sctx = expr_get_extra(yyscanner); + +d_ratio { return D_RATIO; } +max { return MAX; } +min { return MIN; } +if { return IF; } +else { return ELSE; } +source_count { return SOURCE_COUNT; } +aggr_nr { return AGGR_NR; } +has_event { return HAS_EVENT; } +strcmp_cpuid_str { return STRCMP_CPUID_STR; } +NaN { return nan_value(yyscanner); } +{literal} { return literal(yyscanner, sctx); } +{number} { return value(yyscanner); } +{symbol} { return str(yyscanner, ID, sctx->runtime); } +"|" { return '|'; } +"^" { return '^'; } +"&" { return '&'; } +"<" { return '<'; } +">" { return '>'; } +"-" { return '-'; } +"+" { return '+'; } +"*" { return '*'; } +"/" { return '/'; } +"%" { return '%'; } +"(" { return '('; } +")" { return ')'; } +"," { return ','; } +. { } +%% + +int expr_wrap(void *scanner __maybe_unused) +{ + return 1; +} diff --git a/tools/perf/util/expr.y b/tools/perf/util/expr.y new file mode 100644 index 000000000..e20f64935 --- /dev/null +++ b/tools/perf/util/expr.y @@ -0,0 +1,391 @@ +/* Simple expression parser */ +%{ +#ifndef NDEBUG +#define YYDEBUG 1 +#endif +#include +#include +#include +#include "util/debug.h" +#define IN_EXPR_Y 1 +#include "expr.h" +#include "expr-bison.h" +int expr_lex(YYSTYPE * yylval_param , void *yyscanner); +%} + +%define api.pure full + +%parse-param { double *final_val } +%parse-param { struct expr_parse_ctx *ctx } +%parse-param { bool compute_ids } +%parse-param {void *scanner} +%lex-param {void* scanner} + +%union { + double num; + char *str; + struct ids { + /* + * When creating ids, holds the working set of event ids. NULL + * implies the set is empty. + */ + struct hashmap *ids; + /* + * The metric value. When not creating ids this is the value + * read from a counter, a constant or some computed value. When + * creating ids the value is either a constant or BOTTOM. NAN is + * used as the special BOTTOM value, representing a "set of all + * values" case. + */ + double val; + } ids; +} + +%token ID NUMBER MIN MAX IF ELSE LITERAL D_RATIO SOURCE_COUNT AGGR_NR HAS_EVENT STRCMP_CPUID_STR EXPR_ERROR +%left MIN MAX IF +%left '|' +%left '^' +%left '&' +%left '<' '>' +%left '-' '+' +%left '*' '/' '%' +%left NEG NOT +%type NUMBER LITERAL +%type ID +%destructor { free ($$); } +%type expr if_expr +%destructor { ids__free($$.ids); } + +%{ +static void expr_error(double *final_val __maybe_unused, + struct expr_parse_ctx *ctx __maybe_unused, + bool compute_ids __maybe_unused, + void *scanner __maybe_unused, + const char *s) +{ + pr_debug("%s\n", s); +} + +/* + * During compute ids, the special "bottom" value uses NAN to represent the set + * of all values. NAN is selected as it isn't a useful constant value. + */ +#define BOTTOM NAN + +/* During computing ids, does val represent a constant (non-BOTTOM) value? */ +static bool is_const(double val) +{ + return isfinite(val); +} + +static struct ids union_expr(struct ids ids1, struct ids ids2) +{ + struct ids result = { + .val = BOTTOM, + .ids = ids__union(ids1.ids, ids2.ids), + }; + return result; +} + +enum expr_id_kind { + EXPR_ID_KIND__VALUE, + EXPR_ID_KIND__SOURCE_COUNT, + EXPR_ID_KIND__AGGR_NR, +}; + +static struct ids handle_id(struct expr_parse_ctx *ctx, char *id, + bool compute_ids, enum expr_id_kind kind) +{ + struct ids result; + + if (!compute_ids) { + /* + * Compute the event's value from ID. If the ID isn't known then + * it isn't used to compute the formula so set to NAN. + */ + struct expr_id_data *data; + + result.val = NAN; + if (expr__resolve_id(ctx, id, &data) == 0) { + if (kind == EXPR_ID_KIND__SOURCE_COUNT) + result.val = expr_id_data__source_count(data); + else if (kind == EXPR_ID_KIND__AGGR_NR) + result.val = expr_id_data__aggr_nr(data); + else + result.val = expr_id_data__value(data); + } + result.ids = NULL; + free(id); + } else { + /* + * Set the value to BOTTOM to show that any value is possible + * when the event is computed. Create a set of just the ID. + */ + result.val = BOTTOM; + result.ids = ids__new(); + if (!result.ids || ids__insert(result.ids, id)) { + pr_err("Error creating IDs for '%s'", id); + free(id); + } + } + return result; +} + +/* + * If we're not computing ids or $1 and $3 are constants, compute the new + * constant value using OP. Its invariant that there are no ids. If computing + * ids for non-constants union the set of IDs that must be computed. + */ +#define BINARY_OP(RESULT, OP, LHS, RHS) \ + if (!compute_ids || (is_const(LHS.val) && is_const(RHS.val))) { \ + assert(LHS.ids == NULL); \ + assert(RHS.ids == NULL); \ + if (isnan(LHS.val) || isnan(RHS.val)) { \ + RESULT.val = NAN; \ + } else { \ + RESULT.val = LHS.val OP RHS.val; \ + } \ + RESULT.ids = NULL; \ + } else { \ + RESULT = union_expr(LHS, RHS); \ + } + +%} +%% + +start: if_expr +{ + if (compute_ids) + ctx->ids = ids__union($1.ids, ctx->ids); + + if (final_val) + *final_val = $1.val; +} +; + +if_expr: expr IF expr ELSE if_expr +{ + if (fpclassify($3.val) == FP_ZERO) { + /* + * The IF expression evaluated to 0 so treat as false, take the + * ELSE and discard everything else. + */ + $$.val = $5.val; + $$.ids = $5.ids; + ids__free($1.ids); + ids__free($3.ids); + } else if (!compute_ids || is_const($3.val)) { + /* + * If ids aren't computed then treat the expression as true. If + * ids are being computed and the IF expr is a non-zero + * constant, then also evaluate the true case. + */ + $$.val = $1.val; + $$.ids = $1.ids; + ids__free($3.ids); + ids__free($5.ids); + } else if ($1.val == $5.val) { + /* + * LHS == RHS, so both are an identical constant. No need to + * evaluate any events. + */ + $$.val = $1.val; + $$.ids = NULL; + ids__free($1.ids); + ids__free($3.ids); + ids__free($5.ids); + } else { + /* + * Value is either the LHS or RHS and we need the IF expression + * to compute it. + */ + $$ = union_expr($1, union_expr($3, $5)); + } +} +| expr +; + +expr: NUMBER +{ + $$.val = $1; + $$.ids = NULL; +} +| ID { $$ = handle_id(ctx, $1, compute_ids, EXPR_ID_KIND__VALUE); } +| SOURCE_COUNT '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__SOURCE_COUNT); } +| AGGR_NR '(' ID ')' { $$ = handle_id(ctx, $3, compute_ids, EXPR_ID_KIND__AGGR_NR); } +| HAS_EVENT '(' ID ')' +{ + $$.val = expr__has_event(ctx, compute_ids, $3); + $$.ids = NULL; + free($3); +} +| STRCMP_CPUID_STR '(' ID ')' +{ + $$.val = expr__strcmp_cpuid_str(ctx, compute_ids, $3); + $$.ids = NULL; + free($3); +} +| expr '|' expr +{ + if (is_const($1.val) && is_const($3.val)) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.ids = NULL; + $$.val = (fpclassify($1.val) == FP_ZERO && fpclassify($3.val) == FP_ZERO) ? 0 : 1; + } else if (is_const($1.val)) { + assert($1.ids == NULL); + if (fpclassify($1.val) == FP_ZERO) { + $$ = $3; + } else { + $$.val = 1; + $$.ids = NULL; + ids__free($3.ids); + } + } else if (is_const($3.val)) { + assert($3.ids == NULL); + if (fpclassify($3.val) == FP_ZERO) { + $$ = $1; + } else { + $$.val = 1; + $$.ids = NULL; + ids__free($1.ids); + } + } else { + $$ = union_expr($1, $3); + } +} +| expr '&' expr +{ + if (is_const($1.val) && is_const($3.val)) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = (fpclassify($1.val) != FP_ZERO && fpclassify($3.val) != FP_ZERO) ? 1 : 0; + $$.ids = NULL; + } else if (is_const($1.val)) { + assert($1.ids == NULL); + if (fpclassify($1.val) != FP_ZERO) { + $$ = $3; + } else { + $$.val = 0; + $$.ids = NULL; + ids__free($3.ids); + } + } else if (is_const($3.val)) { + assert($3.ids == NULL); + if (fpclassify($3.val) != FP_ZERO) { + $$ = $1; + } else { + $$.val = 0; + $$.ids = NULL; + ids__free($1.ids); + } + } else { + $$ = union_expr($1, $3); + } +} +| expr '^' expr +{ + if (is_const($1.val) && is_const($3.val)) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = (fpclassify($1.val) == FP_ZERO) != (fpclassify($3.val) == FP_ZERO) ? 1 : 0; + $$.ids = NULL; + } else { + $$ = union_expr($1, $3); + } +} +| expr '<' expr { BINARY_OP($$, <, $1, $3); } +| expr '>' expr { BINARY_OP($$, >, $1, $3); } +| expr '+' expr { BINARY_OP($$, +, $1, $3); } +| expr '-' expr { BINARY_OP($$, -, $1, $3); } +| expr '*' expr { BINARY_OP($$, *, $1, $3); } +| expr '/' expr +{ + if (fpclassify($3.val) == FP_ZERO) { + pr_debug("division by zero\n"); + assert($3.ids == NULL); + if (compute_ids) + ids__free($1.ids); + $$.val = NAN; + $$.ids = NULL; + } else if (!compute_ids || (is_const($1.val) && is_const($3.val))) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = $1.val / $3.val; + $$.ids = NULL; + } else { + /* LHS and/or RHS need computing from event IDs so union. */ + $$ = union_expr($1, $3); + } +} +| expr '%' expr +{ + if (fpclassify($3.val) == FP_ZERO) { + pr_debug("division by zero\n"); + YYABORT; + } else if (!compute_ids || (is_const($1.val) && is_const($3.val))) { + assert($1.ids == NULL); + assert($3.ids == NULL); + $$.val = (long)$1.val % (long)$3.val; + $$.ids = NULL; + } else { + /* LHS and/or RHS need computing from event IDs so union. */ + $$ = union_expr($1, $3); + } +} +| D_RATIO '(' expr ',' expr ')' +{ + if (fpclassify($5.val) == FP_ZERO) { + /* + * Division by constant zero always yields zero and no events + * are necessary. + */ + assert($5.ids == NULL); + $$.val = 0.0; + $$.ids = NULL; + ids__free($3.ids); + } else if (!compute_ids || (is_const($3.val) && is_const($5.val))) { + assert($3.ids == NULL); + assert($5.ids == NULL); + $$.val = $3.val / $5.val; + $$.ids = NULL; + } else { + /* LHS and/or RHS need computing from event IDs so union. */ + $$ = union_expr($3, $5); + } +} +| '-' expr %prec NEG +{ + $$.val = -$2.val; + $$.ids = $2.ids; +} +| '(' if_expr ')' +{ + $$ = $2; +} +| MIN '(' expr ',' expr ')' +{ + if (!compute_ids) { + $$.val = $3.val < $5.val ? $3.val : $5.val; + $$.ids = NULL; + } else { + $$ = union_expr($3, $5); + } +} +| MAX '(' expr ',' expr ')' +{ + if (!compute_ids) { + $$.val = $3.val > $5.val ? $3.val : $5.val; + $$.ids = NULL; + } else { + $$ = union_expr($3, $5); + } +} +| LITERAL +{ + $$.val = $1; + $$.ids = NULL; +} +; + +%% diff --git a/tools/perf/util/find-map.c b/tools/perf/util/find-map.c new file mode 100644 index 000000000..7b2300588 --- /dev/null +++ b/tools/perf/util/find-map.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0 +static int find_map(void **start, void **end, const char *name) +{ + FILE *maps; + char line[128]; + int found = 0; + + maps = fopen("/proc/self/maps", "r"); + if (!maps) { + fprintf(stderr, "cannot open maps\n"); + return -1; + } + + while (!found && fgets(line, sizeof(line), maps)) { + int m = -1; + + /* We care only about private r-x mappings. */ + if (2 != sscanf(line, "%p-%p r-xp %*x %*x:%*x %*u %n", + start, end, &m)) + continue; + if (m < 0) + continue; + + if (!strncmp(&line[m], name, strlen(name))) + found = 1; + } + + fclose(maps); + return !found; +} diff --git a/tools/perf/util/fncache.c b/tools/perf/util/fncache.c new file mode 100644 index 000000000..bf9559c55 --- /dev/null +++ b/tools/perf/util/fncache.c @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Manage a cache of file names' existence */ +#include +#include +#include +#include +#include +#include "fncache.h" +#include "hashmap.h" + +static struct hashmap *fncache; + +static size_t fncache__hash(long key, void *ctx __maybe_unused) +{ + return str_hash((const char *)key); +} + +static bool fncache__equal(long key1, long key2, void *ctx __maybe_unused) +{ + return strcmp((const char *)key1, (const char *)key2) == 0; +} + +static void fncache__init(void) +{ + fncache = hashmap__new(fncache__hash, fncache__equal, /*ctx=*/NULL); +} + +static struct hashmap *fncache__get(void) +{ + static pthread_once_t fncache_once = PTHREAD_ONCE_INIT; + + pthread_once(&fncache_once, fncache__init); + + return fncache; +} + +static bool lookup_fncache(const char *name, bool *res) +{ + long val; + + if (!hashmap__find(fncache__get(), name, &val)) + return false; + + *res = (val != 0); + return true; +} + +static void update_fncache(const char *name, bool res) +{ + char *old_key = NULL, *key = strdup(name); + + if (key) { + hashmap__set(fncache__get(), key, res, &old_key, /*old_value*/NULL); + free(old_key); + } +} + +/* No LRU, only use when bounded in some other way. */ +bool file_available(const char *name) +{ + bool res; + + if (lookup_fncache(name, &res)) + return res; + res = access(name, R_OK) == 0; + update_fncache(name, res); + return res; +} diff --git a/tools/perf/util/fncache.h b/tools/perf/util/fncache.h new file mode 100644 index 000000000..b6a0f2094 --- /dev/null +++ b/tools/perf/util/fncache.h @@ -0,0 +1,6 @@ +#ifndef _FCACHE_H +#define _FCACHE_H 1 + +bool file_available(const char *name); + +#endif diff --git a/tools/perf/util/ftrace.h b/tools/perf/util/ftrace.h new file mode 100644 index 000000000..950f2efaf --- /dev/null +++ b/tools/perf/util/ftrace.h @@ -0,0 +1,96 @@ +#ifndef __PERF_FTRACE_H__ +#define __PERF_FTRACE_H__ + +#include + +#include "target.h" + +struct evlist; +struct hashamp; +struct stats; + +struct perf_ftrace { + struct evlist *evlist; + struct target target; + const char *tracer; + struct list_head filters; + struct list_head notrace; + struct list_head graph_funcs; + struct list_head nograph_funcs; + struct list_head event_pair; + struct hashmap *profile_hash; + unsigned long percpu_buffer_size; + bool inherit; + bool use_nsec; + unsigned int bucket_range; + unsigned int min_latency; + unsigned int max_latency; + unsigned int bucket_num; + bool hide_empty; + int graph_depth; + int func_stack_trace; + int func_irq_info; + int graph_args; + int graph_retval; + int graph_retval_hex; + int graph_retaddr; + int graph_nosleep_time; + int graph_noirqs; + int graph_verbose; + int graph_thresh; + int graph_tail; +}; + +struct filter_entry { + struct list_head list; + char name[]; +}; + +#define NUM_BUCKET 22 /* 20 + 2 (for outliers in both direction) */ + +#ifdef HAVE_BPF_SKEL + +int perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace); +int perf_ftrace__latency_start_bpf(struct perf_ftrace *ftrace); +int perf_ftrace__latency_stop_bpf(struct perf_ftrace *ftrace); +int perf_ftrace__latency_read_bpf(struct perf_ftrace *ftrace, + int buckets[], struct stats *stats); +int perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace); + +#else /* !HAVE_BPF_SKEL */ + +static inline int +perf_ftrace__latency_prepare_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_start_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_stop_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_read_bpf(struct perf_ftrace *ftrace __maybe_unused, + int buckets[] __maybe_unused, + struct stats *stats __maybe_unused) +{ + return -1; +} + +static inline int +perf_ftrace__latency_cleanup_bpf(struct perf_ftrace *ftrace __maybe_unused) +{ + return -1; +} + +#endif /* HAVE_BPF_SKEL */ + +#endif /* __PERF_FTRACE_H__ */ diff --git a/tools/perf/util/genelf.c b/tools/perf/util/genelf.c new file mode 100644 index 000000000..14882def9 --- /dev/null +++ b/tools/perf/util/genelf.c @@ -0,0 +1,471 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * genelf.c + * Copyright (C) 2014, Google, Inc + * + * Contributed by: + * Stephane Eranian + */ + +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBDW_SUPPORT +#include +#endif + +#include "blake2s.h" +#include "genelf.h" +#include "../util/jitdump.h" +#include + +#ifndef NT_GNU_BUILD_ID +#define NT_GNU_BUILD_ID 3 +#endif + +typedef struct { + unsigned int namesz; /* Size of entry's owner string */ + unsigned int descsz; /* Size of the note descriptor */ + unsigned int type; /* Interpretation of the descriptor */ + char name[0]; /* Start of the name+desc data */ +} Elf_Note; + +static char shd_string_table[] = { + 0, + '.', 't', 'e', 'x', 't', 0, /* 1 */ + '.', 's', 'h', 's', 't', 'r', 't', 'a', 'b', 0, /* 7 */ + '.', 's', 'y', 'm', 't', 'a', 'b', 0, /* 17 */ + '.', 's', 't', 'r', 't', 'a', 'b', 0, /* 25 */ + '.', 'n', 'o', 't', 'e', '.', 'g', 'n', 'u', '.', 'b', 'u', 'i', 'l', 'd', '-', 'i', 'd', 0, /* 33 */ + '.', 'd', 'e', 'b', 'u', 'g', '_', 'l', 'i', 'n', 'e', 0, /* 52 */ + '.', 'd', 'e', 'b', 'u', 'g', '_', 'i', 'n', 'f', 'o', 0, /* 64 */ + '.', 'd', 'e', 'b', 'u', 'g', '_', 'a', 'b', 'b', 'r', 'e', 'v', 0, /* 76 */ + '.', 'e', 'h', '_', 'f', 'r', 'a', 'm', 'e', '_', 'h', 'd', 'r', 0, /* 90 */ + '.', 'e', 'h', '_', 'f', 'r', 'a', 'm', 'e', 0, /* 104 */ +}; + +static struct buildid_note { + Elf_Note desc; /* descsz: size of build-id, must be multiple of 4 */ + char name[4]; /* GNU\0 */ + u8 build_id[20]; +} bnote; + +static Elf_Sym symtab[]={ + /* symbol 0 MUST be the undefined symbol */ + { .st_name = 0, /* index in sym_string table */ + .st_info = ELF_ST_TYPE(STT_NOTYPE), + .st_shndx = 0, /* for now */ + .st_value = 0x0, + .st_other = ELF_ST_VIS(STV_DEFAULT), + .st_size = 0, + }, + { .st_name = 1, /* index in sym_string table */ + .st_info = ELF_ST_BIND(STB_LOCAL) | ELF_ST_TYPE(STT_FUNC), + .st_shndx = 1, + .st_value = 0, /* for now */ + .st_other = ELF_ST_VIS(STV_DEFAULT), + .st_size = 0, /* for now */ + } +}; + +static int +jit_add_eh_frame_info(Elf *e, void* unwinding, uint64_t unwinding_header_size, + uint64_t unwinding_size, uint64_t base_offset) +{ + Elf_Data *d; + Elf_Scn *scn; + Elf_Shdr *shdr; + uint64_t unwinding_table_size = unwinding_size - unwinding_header_size; + + /* + * setup eh_frame section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + return -1; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + return -1; + } + + d->d_align = 8; + d->d_off = 0LL; + d->d_buf = unwinding; + d->d_type = ELF_T_BYTE; + d->d_size = unwinding_table_size; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + return -1; + } + + shdr->sh_name = 104; + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = base_offset; + shdr->sh_flags = SHF_ALLOC; + shdr->sh_entsize = 0; + + /* + * setup eh_frame_hdr section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + return -1; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + return -1; + } + + d->d_align = 4; + d->d_off = 0LL; + d->d_buf = unwinding + unwinding_table_size; + d->d_type = ELF_T_BYTE; + d->d_size = unwinding_header_size; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + return -1; + } + + shdr->sh_name = 90; + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = base_offset + unwinding_table_size; + shdr->sh_flags = SHF_ALLOC; + shdr->sh_entsize = 0; + + return 0; +} + +enum { + TAG_CODE = 0, + TAG_SYMTAB = 1, + TAG_STRSYM = 2, +}; + +/* + * Update the hash using the given data, also prepending a (tag, len) prefix to + * ensure that distinct input tuples reliably result in distinct hashes. + */ +static void blake2s_update_tagged(struct blake2s_ctx *ctx, int tag, + const void *data, size_t len) +{ + u64 prefix = ((u64)tag << 56) | len; + + blake2s_update(ctx, (const u8 *)&prefix, sizeof(prefix)); + blake2s_update(ctx, data, len); +} + +/* + * fd: file descriptor open for writing for the output file + * load_addr: code load address (could be zero) + * sym: function name (for native code - used as the symbol) + * code: the native code + * csize: the code size in bytes + */ +int +jit_write_elf(int fd, uint64_t load_addr __maybe_unused, const char *sym, + const void *code, int csize, + void *debug __maybe_unused, int nr_debug_entries __maybe_unused, + void *unwinding, uint64_t unwinding_header_size, uint64_t unwinding_size) +{ + Elf *e; + Elf_Data *d; + Elf_Scn *scn; + Elf_Ehdr *ehdr; + Elf_Phdr *phdr; + Elf_Shdr *shdr; + uint64_t eh_frame_base_offset; + char *strsym = NULL; + struct blake2s_ctx ctx; + int symlen; + int retval = -1; + + if (elf_version(EV_CURRENT) == EV_NONE) { + warnx("ELF initialization failed"); + return -1; + } + + e = elf_begin(fd, ELF_C_WRITE, NULL); + if (!e) { + warnx("elf_begin failed"); + goto error; + } + + /* + * setup ELF header + */ + ehdr = elf_newehdr(e); + if (!ehdr) { + warnx("cannot get ehdr"); + goto error; + } + + ehdr->e_ident[EI_DATA] = GEN_ELF_ENDIAN; + ehdr->e_ident[EI_CLASS] = GEN_ELF_CLASS; + ehdr->e_machine = GEN_ELF_ARCH; + ehdr->e_type = ET_DYN; + ehdr->e_entry = GEN_ELF_TEXT_OFFSET; + ehdr->e_version = EV_CURRENT; + ehdr->e_shstrndx= unwinding ? 4 : 2; /* shdr index for section name */ + + /* + * setup program header + */ + phdr = elf_newphdr(e, 1); + phdr[0].p_type = PT_LOAD; + phdr[0].p_offset = GEN_ELF_TEXT_OFFSET; + phdr[0].p_vaddr = GEN_ELF_TEXT_OFFSET; + phdr[0].p_paddr = GEN_ELF_TEXT_OFFSET; + phdr[0].p_filesz = csize; + phdr[0].p_memsz = csize; + phdr[0].p_flags = PF_X | PF_R; + phdr[0].p_align = 8; + + /* + * setup text section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 16; + d->d_off = 0LL; + d->d_buf = (void *)code; + d->d_type = ELF_T_BYTE; + d->d_size = csize; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 1; + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = GEN_ELF_TEXT_OFFSET; + shdr->sh_flags = SHF_EXECINSTR | SHF_ALLOC; + shdr->sh_entsize = 0; + + blake2s_init(&ctx, sizeof(bnote.build_id)); + blake2s_update_tagged(&ctx, TAG_CODE, code, csize); + + /* + * Setup .eh_frame_hdr and .eh_frame + */ + if (unwinding) { + eh_frame_base_offset = ALIGN_8(GEN_ELF_TEXT_OFFSET + csize); + retval = jit_add_eh_frame_info(e, unwinding, + unwinding_header_size, unwinding_size, + eh_frame_base_offset); + if (retval) + goto error; + retval = -1; + } + + /* + * setup section headers string table + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = shd_string_table; + d->d_type = ELF_T_BYTE; + d->d_size = sizeof(shd_string_table); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 7; /* offset of '.shstrtab' in shd_string_table */ + shdr->sh_type = SHT_STRTAB; + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * setup symtab section + */ + symtab[1].st_size = csize; + symtab[1].st_value = GEN_ELF_TEXT_OFFSET; + + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 8; + d->d_off = 0LL; + d->d_buf = symtab; + d->d_type = ELF_T_SYM; + d->d_size = sizeof(symtab); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 17; /* offset of '.symtab' in shd_string_table */ + shdr->sh_type = SHT_SYMTAB; + shdr->sh_flags = 0; + shdr->sh_entsize = sizeof(Elf_Sym); + shdr->sh_link = unwinding ? 6 : 4; /* index of .strtab section */ + + blake2s_update_tagged(&ctx, TAG_SYMTAB, symtab, sizeof(symtab)); + + /* + * setup symbols string table + * 2 = 1 for 0 in 1st entry, 1 for the 0 at end of symbol for 2nd entry + */ + symlen = 2 + strlen(sym); + strsym = calloc(1, symlen); + if (!strsym) { + warnx("cannot allocate strsym"); + goto error; + } + strcpy(strsym + 1, sym); + + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = strsym; + d->d_type = ELF_T_BYTE; + d->d_size = symlen; + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 25; /* offset in shd_string_table */ + shdr->sh_type = SHT_STRTAB; + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + blake2s_update_tagged(&ctx, TAG_STRSYM, strsym, symlen); + + /* + * setup build-id section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto error; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto error; + } + + /* + * build-id generation + */ + blake2s_final(&ctx, bnote.build_id); + bnote.desc.namesz = sizeof(bnote.name); /* must include 0 termination */ + bnote.desc.descsz = sizeof(bnote.build_id); + bnote.desc.type = NT_GNU_BUILD_ID; + strcpy(bnote.name, "GNU"); + + d->d_align = 4; + d->d_off = 0LL; + d->d_buf = &bnote; + d->d_type = ELF_T_BYTE; + d->d_size = sizeof(bnote); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto error; + } + + shdr->sh_name = 33; /* offset in shd_string_table */ + shdr->sh_type = SHT_NOTE; + shdr->sh_addr = 0x0; + shdr->sh_flags = SHF_ALLOC; + shdr->sh_size = sizeof(bnote); + shdr->sh_entsize = 0; + +#ifdef HAVE_LIBDW_SUPPORT + if (debug && nr_debug_entries) { + retval = jit_add_debug_info(e, load_addr, debug, nr_debug_entries); + if (retval) + goto error; + } else +#endif + { + if (elf_update(e, ELF_C_WRITE) < 0) { + warnx("elf_update 4 failed"); + goto error; + } + } + + retval = 0; +error: + (void)elf_end(e); + + free(strsym); + + return retval; +} diff --git a/tools/perf/util/genelf.h b/tools/perf/util/genelf.h new file mode 100644 index 000000000..9f0b875d6 --- /dev/null +++ b/tools/perf/util/genelf.h @@ -0,0 +1,89 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __GENELF_H__ +#define __GENELF_H__ + +#include + +/* genelf.c */ +int jit_write_elf(int fd, uint64_t code_addr, const char *sym, + const void *code, int csize, void *debug, int nr_debug_entries, + void *unwinding, uint64_t unwinding_header_size, uint64_t unwinding_size); +#ifdef HAVE_LIBDW_SUPPORT +/* genelf_debug.c */ +int jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_entries); +#endif + +#if defined(__arm__) +#define GEN_ELF_ARCH EM_ARM +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__aarch64__) +#define GEN_ELF_ARCH EM_AARCH64 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__x86_64__) +#define GEN_ELF_ARCH EM_X86_64 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__i386__) +#define GEN_ELF_ARCH EM_386 +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__powerpc64__) +#define GEN_ELF_ARCH EM_PPC64 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__powerpc__) +#define GEN_ELF_ARCH EM_PPC +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__sparc__) && defined(__arch64__) +#define GEN_ELF_ARCH EM_SPARCV9 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__sparc__) +#define GEN_ELF_ARCH EM_SPARC +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__s390x__) +#define GEN_ELF_ARCH EM_S390 +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__riscv) && __riscv_xlen == 64 +#define GEN_ELF_ARCH EM_RISCV +#define GEN_ELF_CLASS ELFCLASS64 +#elif defined(__riscv) && __riscv_xlen == 32 +#define GEN_ELF_ARCH EM_RISCV +#define GEN_ELF_CLASS ELFCLASS32 +#elif defined(__loongarch__) +#define GEN_ELF_ARCH EM_LOONGARCH +#define GEN_ELF_CLASS ELFCLASS64 +#else +#error "unsupported architecture" +#endif + +#if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ +#define GEN_ELF_ENDIAN ELFDATA2MSB +#else +#define GEN_ELF_ENDIAN ELFDATA2LSB +#endif + +#if GEN_ELF_CLASS == ELFCLASS64 +#define elf_newehdr elf64_newehdr +#define elf_newphdr elf64_newphdr +#define elf_getshdr elf64_getshdr +#define Elf_Ehdr Elf64_Ehdr +#define Elf_Phdr Elf64_Phdr +#define Elf_Shdr Elf64_Shdr +#define Elf_Sym Elf64_Sym +#define ELF_ST_TYPE(a) ELF64_ST_TYPE(a) +#define ELF_ST_BIND(a) ELF64_ST_BIND(a) +#define ELF_ST_VIS(a) ELF64_ST_VISIBILITY(a) +#else +#define elf_newehdr elf32_newehdr +#define elf_newphdr elf32_newphdr +#define elf_getshdr elf32_getshdr +#define Elf_Ehdr Elf32_Ehdr +#define Elf_Phdr Elf32_Phdr +#define Elf_Shdr Elf32_Shdr +#define Elf_Sym Elf32_Sym +#define ELF_ST_TYPE(a) ELF32_ST_TYPE(a) +#define ELF_ST_BIND(a) ELF32_ST_BIND(a) +#define ELF_ST_VIS(a) ELF32_ST_VISIBILITY(a) +#endif + +/* The .text section is directly after the ELF header */ +#define GEN_ELF_TEXT_OFFSET round_up(sizeof(Elf_Ehdr) + sizeof(Elf_Phdr), 16) + +#endif diff --git a/tools/perf/util/genelf_debug.c b/tools/perf/util/genelf_debug.c new file mode 100644 index 000000000..8588b3e35 --- /dev/null +++ b/tools/perf/util/genelf_debug.c @@ -0,0 +1,625 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * genelf_debug.c + * Copyright (C) 2015, Google, Inc + * + * Contributed by: + * Stephane Eranian + * + * based on GPLv2 source code from Oprofile + * @remark Copyright 2007 OProfile authors + * @author Philippe Elie + */ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "genelf.h" +#include "../util/jitdump.h" + +#define BUFFER_EXT_DFL_SIZE (4 * 1024) + +typedef uint32_t uword; +typedef uint16_t uhalf; +typedef int32_t sword; +typedef int16_t shalf; +typedef uint8_t ubyte; +typedef int8_t sbyte; + +struct buffer_ext { + size_t cur_pos; + size_t max_sz; + void *data; +}; + +static void +buffer_ext_dump(struct buffer_ext *be, const char *msg) +{ + size_t i; + warnx("DUMP for %s", msg); + for (i = 0 ; i < be->cur_pos; i++) + warnx("%4zu 0x%02x", i, (((char *)be->data)[i]) & 0xff); +} + +static inline int +buffer_ext_add(struct buffer_ext *be, void *addr, size_t sz) +{ + void *tmp; + size_t be_sz = be->max_sz; + +retry: + if ((be->cur_pos + sz) < be_sz) { + memcpy(be->data + be->cur_pos, addr, sz); + be->cur_pos += sz; + return 0; + } + + if (!be_sz) + be_sz = BUFFER_EXT_DFL_SIZE; + else + be_sz <<= 1; + + tmp = realloc(be->data, be_sz); + if (!tmp) + return -1; + + be->data = tmp; + be->max_sz = be_sz; + + goto retry; +} + +static void +buffer_ext_init(struct buffer_ext *be) +{ + be->data = NULL; + be->cur_pos = 0; + be->max_sz = 0; +} + +static void +buffer_ext_exit(struct buffer_ext *be) +{ + zfree(&be->data); +} + +static inline size_t +buffer_ext_size(struct buffer_ext *be) +{ + return be->cur_pos; +} + +static inline void * +buffer_ext_addr(struct buffer_ext *be) +{ + return be->data; +} + +struct debug_line_header { + // Not counting this field + uword total_length; + // version number (2 currently) + uhalf version; + // relative offset from next field to + // program statement + uword prolog_length; + ubyte minimum_instruction_length; + ubyte default_is_stmt; + // line_base - see DWARF 2 specs + sbyte line_base; + // line_range - see DWARF 2 specs + ubyte line_range; + // number of opcode + 1 + ubyte opcode_base; + /* follow the array of opcode args nr: ubytes [nr_opcode_base] */ + /* follow the search directories index, zero terminated string + * terminated by an empty string. + */ + /* follow an array of { filename, LEB128, LEB128, LEB128 }, first is + * the directory index entry, 0 means current directory, then mtime + * and filesize, last entry is followed by en empty string. + */ + /* follow the first program statement */ +} __packed; + +/* DWARF 2 spec talk only about one possible compilation unit header while + * binutils can handle two flavours of dwarf 2, 32 and 64 bits, this is not + * related to the used arch, an ELF 32 can hold more than 4 Go of debug + * information. For now we handle only DWARF 2 32 bits comp unit. It'll only + * become a problem if we generate more than 4GB of debug information. + */ +struct compilation_unit_header { + uword total_length; + uhalf version; + uword debug_abbrev_offset; + ubyte pointer_size; +} __packed; + +#define DW_LNS_num_opcode (DW_LNS_set_isa + 1) + +/* field filled at run time are marked with -1 */ +static struct debug_line_header const default_debug_line_header = { + .total_length = -1, + .version = 2, + .prolog_length = -1, + .minimum_instruction_length = 1, /* could be better when min instruction size != 1 */ + .default_is_stmt = 1, /* we don't take care about basic block */ + .line_base = -5, /* sensible value for line base ... */ + .line_range = -14, /* ... and line range are guessed statically */ + .opcode_base = DW_LNS_num_opcode +}; + +static ubyte standard_opcode_length[] = +{ + 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1 +}; +#if 0 +{ + [DW_LNS_advance_pc] = 1, + [DW_LNS_advance_line] = 1, + [DW_LNS_set_file] = 1, + [DW_LNS_set_column] = 1, + [DW_LNS_fixed_advance_pc] = 1, + [DW_LNS_set_isa] = 1, +}; +#endif + +/* field filled at run time are marked with -1 */ +static struct compilation_unit_header default_comp_unit_header = { + .total_length = -1, + .version = 2, + .debug_abbrev_offset = 0, /* we reuse the same abbrev entries for all comp unit */ + .pointer_size = sizeof(void *) +}; + +static void emit_uword(struct buffer_ext *be, uword data) +{ + buffer_ext_add(be, &data, sizeof(uword)); +} + +static void emit_string(struct buffer_ext *be, const char *s) +{ + buffer_ext_add(be, (void *)s, strlen(s) + 1); +} + +static void emit_unsigned_LEB128(struct buffer_ext *be, + unsigned long data) +{ + do { + ubyte cur = data & 0x7F; + data >>= 7; + if (data) + cur |= 0x80; + buffer_ext_add(be, &cur, 1); + } while (data); +} + +static void emit_signed_LEB128(struct buffer_ext *be, long data) +{ + int more = 1; + int negative = data < 0; + int size = sizeof(long) * CHAR_BIT; + while (more) { + ubyte cur = data & 0x7F; + data >>= 7; + if (negative) + data |= - (1 << (size - 7)); + if ((data == 0 && !(cur & 0x40)) || + (data == -1l && (cur & 0x40))) + more = 0; + else + cur |= 0x80; + buffer_ext_add(be, &cur, 1); + } +} + +static void emit_extended_opcode(struct buffer_ext *be, ubyte opcode, + void *data, size_t data_len) +{ + buffer_ext_add(be, (char *)"", 1); + + emit_unsigned_LEB128(be, data_len + 1); + + buffer_ext_add(be, &opcode, 1); + buffer_ext_add(be, data, data_len); +} + +static void emit_opcode(struct buffer_ext *be, ubyte opcode) +{ + buffer_ext_add(be, &opcode, 1); +} + +static void emit_opcode_signed(struct buffer_ext *be, + ubyte opcode, long data) +{ + buffer_ext_add(be, &opcode, 1); + emit_signed_LEB128(be, data); +} + +static void emit_opcode_unsigned(struct buffer_ext *be, ubyte opcode, + unsigned long data) +{ + buffer_ext_add(be, &opcode, 1); + emit_unsigned_LEB128(be, data); +} + +static void emit_advance_pc(struct buffer_ext *be, unsigned long delta_pc) +{ + emit_opcode_unsigned(be, DW_LNS_advance_pc, delta_pc); +} + +static void emit_advance_lineno(struct buffer_ext *be, long delta_lineno) +{ + emit_opcode_signed(be, DW_LNS_advance_line, delta_lineno); +} + +static void emit_lne_end_of_sequence(struct buffer_ext *be) +{ + emit_extended_opcode(be, DW_LNE_end_sequence, NULL, 0); +} + +static void emit_set_file(struct buffer_ext *be, unsigned long idx) +{ + emit_opcode_unsigned(be, DW_LNS_set_file, idx); +} + +static void emit_lne_define_filename(struct buffer_ext *be, + const char *filename) +{ + buffer_ext_add(be, (void *)"", 1); + + /* LNE field, strlen(filename) + zero termination, 3 bytes for: the dir entry, timestamp, filesize */ + emit_unsigned_LEB128(be, strlen(filename) + 5); + emit_opcode(be, DW_LNE_define_file); + emit_string(be, filename); + /* directory index 0=do not know */ + emit_unsigned_LEB128(be, 0); + /* last modification date on file 0=do not know */ + emit_unsigned_LEB128(be, 0); + /* filesize 0=do not know */ + emit_unsigned_LEB128(be, 0); +} + +static void emit_lne_set_address(struct buffer_ext *be, + void *address) +{ + emit_extended_opcode(be, DW_LNE_set_address, &address, sizeof(unsigned long)); +} + +static ubyte get_special_opcode(struct debug_entry *ent, + unsigned int last_line, + unsigned long last_vma) +{ + unsigned int temp; + unsigned long delta_addr; + + /* + * delta from line_base + */ + temp = (ent->lineno - last_line) - default_debug_line_header.line_base; + + if (temp >= default_debug_line_header.line_range) + return 0; + + /* + * delta of addresses + */ + delta_addr = (ent->addr - last_vma) / default_debug_line_header.minimum_instruction_length; + + /* This is not sufficient to ensure opcode will be in [0-256] but + * sufficient to ensure when summing with the delta lineno we will + * not overflow the unsigned long opcode */ + + if (delta_addr <= 256 / default_debug_line_header.line_range) { + unsigned long opcode = temp + + (delta_addr * default_debug_line_header.line_range) + + default_debug_line_header.opcode_base; + + return opcode <= 255 ? opcode : 0; + } + return 0; +} + +static void emit_lineno_info(struct buffer_ext *be, + struct debug_entry *ent, size_t nr_entry, + unsigned long code_addr) +{ + size_t i; + + /* as described in the jitdump format */ + const char repeated_name_marker[] = {'\xff', '\0'}; + + /* + * Machine state at start of a statement program + * address = 0 + * file = 1 + * line = 1 + * column = 0 + * is_stmt = default_is_stmt as given in the debug_line_header + * basic block = 0 + * end sequence = 0 + */ + + /* start state of the state machine we take care of */ + unsigned long last_vma = 0; + char const *cur_filename = NULL; + unsigned long cur_file_idx = 0; + int last_line = 1; + + emit_lne_set_address(be, (void *)code_addr); + + for (i = 0; i < nr_entry; i++, ent = debug_entry_next(ent)) { + int need_copy = 0; + ubyte special_opcode; + + /* + * check if filename changed, if so add it + */ + if ((!cur_filename || strcmp(cur_filename, ent->name)) && + strcmp(repeated_name_marker, ent->name)) { + emit_lne_define_filename(be, ent->name); + cur_filename = ent->name; + emit_set_file(be, ++cur_file_idx); + need_copy = 1; + } + + special_opcode = get_special_opcode(ent, last_line, last_vma); + if (special_opcode != 0) { + last_line = ent->lineno; + last_vma = ent->addr; + emit_opcode(be, special_opcode); + } else { + /* + * lines differ, emit line delta + */ + if (last_line != ent->lineno) { + emit_advance_lineno(be, ent->lineno - last_line); + last_line = ent->lineno; + need_copy = 1; + } + /* + * addresses differ, emit address delta + */ + if (last_vma != ent->addr) { + emit_advance_pc(be, ent->addr - last_vma); + last_vma = ent->addr; + need_copy = 1; + } + /* + * add new row to matrix + */ + if (need_copy) + emit_opcode(be, DW_LNS_copy); + } + } +} + +static void add_debug_line(struct buffer_ext *be, + struct debug_entry *ent, size_t nr_entry, + unsigned long code_addr) +{ + struct debug_line_header * dbg_header; + size_t old_size; + + old_size = buffer_ext_size(be); + + buffer_ext_add(be, (void *)&default_debug_line_header, + sizeof(default_debug_line_header)); + + buffer_ext_add(be, &standard_opcode_length, sizeof(standard_opcode_length)); + + // empty directory entry + buffer_ext_add(be, (void *)"", 1); + + // empty filename directory + buffer_ext_add(be, (void *)"", 1); + + dbg_header = buffer_ext_addr(be) + old_size; + dbg_header->prolog_length = (buffer_ext_size(be) - old_size) - + offsetof(struct debug_line_header, minimum_instruction_length); + + emit_lineno_info(be, ent, nr_entry, code_addr); + + emit_lne_end_of_sequence(be); + + dbg_header = buffer_ext_addr(be) + old_size; + dbg_header->total_length = (buffer_ext_size(be) - old_size) - + offsetof(struct debug_line_header, version); +} + +static void +add_debug_abbrev(struct buffer_ext *be) +{ + emit_unsigned_LEB128(be, 1); + emit_unsigned_LEB128(be, DW_TAG_compile_unit); + emit_unsigned_LEB128(be, DW_CHILDREN_yes); + emit_unsigned_LEB128(be, DW_AT_stmt_list); + emit_unsigned_LEB128(be, DW_FORM_data4); + emit_unsigned_LEB128(be, 0); + emit_unsigned_LEB128(be, 0); + emit_unsigned_LEB128(be, 0); +} + +static void +add_compilation_unit(struct buffer_ext *be, + size_t offset_debug_line) +{ + struct compilation_unit_header *comp_unit_header; + size_t old_size = buffer_ext_size(be); + + buffer_ext_add(be, &default_comp_unit_header, + sizeof(default_comp_unit_header)); + + emit_unsigned_LEB128(be, 1); + emit_uword(be, offset_debug_line); + + comp_unit_header = buffer_ext_addr(be) + old_size; + comp_unit_header->total_length = (buffer_ext_size(be) - old_size) - + offsetof(struct compilation_unit_header, version); +} + +static int +jit_process_debug_info(uint64_t code_addr, + void *debug, int nr_debug_entries, + struct buffer_ext *dl, + struct buffer_ext *da, + struct buffer_ext *di) +{ + struct debug_entry *ent = debug; + int i; + + for (i = 0; i < nr_debug_entries; i++) { + ent->addr = ent->addr - code_addr; + ent = debug_entry_next(ent); + } + add_compilation_unit(di, buffer_ext_size(dl)); + add_debug_line(dl, debug, nr_debug_entries, GEN_ELF_TEXT_OFFSET); + add_debug_abbrev(da); + if (0) buffer_ext_dump(da, "abbrev"); + + return 0; +} + +int +jit_add_debug_info(Elf *e, uint64_t code_addr, void *debug, int nr_debug_entries) +{ + Elf_Data *d; + Elf_Scn *scn; + Elf_Shdr *shdr; + struct buffer_ext dl, di, da; + int ret = -1; + + buffer_ext_init(&dl); + buffer_ext_init(&di); + buffer_ext_init(&da); + + if (jit_process_debug_info(code_addr, debug, nr_debug_entries, &dl, &da, &di)) + goto out; + + /* + * setup .debug_line section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto out; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto out; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = buffer_ext_addr(&dl); + d->d_type = ELF_T_BYTE; + d->d_size = buffer_ext_size(&dl); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto out; + } + + shdr->sh_name = 52; /* .debug_line */ + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = 0; /* must be zero or == sh_offset -> dynamic object */ + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * setup .debug_info section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto out; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto out; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = buffer_ext_addr(&di); + d->d_type = ELF_T_BYTE; + d->d_size = buffer_ext_size(&di); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto out; + } + + shdr->sh_name = 64; /* .debug_info */ + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = 0; /* must be zero or == sh_offset -> dynamic object */ + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * setup .debug_abbrev section + */ + scn = elf_newscn(e); + if (!scn) { + warnx("cannot create section"); + goto out; + } + + d = elf_newdata(scn); + if (!d) { + warnx("cannot get new data"); + goto out; + } + + d->d_align = 1; + d->d_off = 0LL; + d->d_buf = buffer_ext_addr(&da); + d->d_type = ELF_T_BYTE; + d->d_size = buffer_ext_size(&da); + d->d_version = EV_CURRENT; + + shdr = elf_getshdr(scn); + if (!shdr) { + warnx("cannot get section header"); + goto out; + } + + shdr->sh_name = 76; /* .debug_info */ + shdr->sh_type = SHT_PROGBITS; + shdr->sh_addr = 0; /* must be zero or == sh_offset -> dynamic object */ + shdr->sh_flags = 0; + shdr->sh_entsize = 0; + + /* + * now we update the ELF image with all the sections + */ + if (elf_update(e, ELF_C_WRITE) < 0) + warnx("elf_update debug failed"); + else + ret = 0; + +out: + buffer_ext_exit(&dl); + buffer_ext_exit(&di); + buffer_ext_exit(&da); + return ret; +} diff --git a/tools/perf/util/hashmap.c b/tools/perf/util/hashmap.c new file mode 100644 index 000000000..140ee4055 --- /dev/null +++ b/tools/perf/util/hashmap.c @@ -0,0 +1,240 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) + +/* + * Generic non-thread safe hash map implementation. + * + * Copyright (c) 2019 Facebook + */ +#include +#include +#include +#include +#include +#include "hashmap.h" + +/* make sure libbpf doesn't use kernel-only integer typedefs */ +#pragma GCC poison u8 u16 u32 u64 s8 s16 s32 s64 + +/* prevent accidental re-addition of reallocarray() */ +#pragma GCC poison reallocarray + +/* start with 4 buckets */ +#define HASHMAP_MIN_CAP_BITS 2 + +static void hashmap_add_entry(struct hashmap_entry **pprev, + struct hashmap_entry *entry) +{ + entry->next = *pprev; + *pprev = entry; +} + +static void hashmap_del_entry(struct hashmap_entry **pprev, + struct hashmap_entry *entry) +{ + *pprev = entry->next; + entry->next = NULL; +} + +void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, void *ctx) +{ + map->hash_fn = hash_fn; + map->equal_fn = equal_fn; + map->ctx = ctx; + + map->buckets = NULL; + map->cap = 0; + map->cap_bits = 0; + map->sz = 0; +} + +struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, + void *ctx) +{ + struct hashmap *map = malloc(sizeof(struct hashmap)); + + if (!map) + return ERR_PTR(-ENOMEM); + hashmap__init(map, hash_fn, equal_fn, ctx); + return map; +} + +void hashmap__clear(struct hashmap *map) +{ + struct hashmap_entry *cur, *tmp; + size_t bkt; + + hashmap__for_each_entry_safe(map, cur, tmp, bkt) { + free(cur); + } + free(map->buckets); + map->buckets = NULL; + map->cap = map->cap_bits = map->sz = 0; +} + +void hashmap__free(struct hashmap *map) +{ + if (IS_ERR_OR_NULL(map)) + return; + + hashmap__clear(map); + free(map); +} + +size_t hashmap__size(const struct hashmap *map) +{ + return map->sz; +} + +size_t hashmap__capacity(const struct hashmap *map) +{ + return map->cap; +} + +static bool hashmap_needs_to_grow(struct hashmap *map) +{ + /* grow if empty or more than 75% filled */ + return (map->cap == 0) || ((map->sz + 1) * 4 / 3 > map->cap); +} + +static int hashmap_grow(struct hashmap *map) +{ + struct hashmap_entry **new_buckets; + struct hashmap_entry *cur, *tmp; + size_t new_cap_bits, new_cap; + size_t h, bkt; + + new_cap_bits = map->cap_bits + 1; + if (new_cap_bits < HASHMAP_MIN_CAP_BITS) + new_cap_bits = HASHMAP_MIN_CAP_BITS; + + new_cap = 1UL << new_cap_bits; + new_buckets = calloc(new_cap, sizeof(new_buckets[0])); + if (!new_buckets) + return -ENOMEM; + + hashmap__for_each_entry_safe(map, cur, tmp, bkt) { + h = hash_bits(map->hash_fn(cur->key, map->ctx), new_cap_bits); + hashmap_add_entry(&new_buckets[h], cur); + } + + map->cap = new_cap; + map->cap_bits = new_cap_bits; + free(map->buckets); + map->buckets = new_buckets; + + return 0; +} + +static bool hashmap_find_entry(const struct hashmap *map, + const long key, size_t hash, + struct hashmap_entry ***pprev, + struct hashmap_entry **entry) +{ + struct hashmap_entry *cur, **prev_ptr; + + if (!map->buckets) + return false; + + for (prev_ptr = &map->buckets[hash], cur = *prev_ptr; + cur; + prev_ptr = &cur->next, cur = cur->next) { + if (map->equal_fn(cur->key, key, map->ctx)) { + if (pprev) + *pprev = prev_ptr; + *entry = cur; + return true; + } + } + + return false; +} + +int hashmap_insert(struct hashmap *map, long key, long value, + enum hashmap_insert_strategy strategy, + long *old_key, long *old_value) +{ + struct hashmap_entry *entry; + size_t h; + int err; + + if (old_key) + *old_key = 0; + if (old_value) + *old_value = 0; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (strategy != HASHMAP_APPEND && + hashmap_find_entry(map, key, h, NULL, &entry)) { + if (old_key) + *old_key = entry->key; + if (old_value) + *old_value = entry->value; + + if (strategy == HASHMAP_SET || strategy == HASHMAP_UPDATE) { + entry->key = key; + entry->value = value; + return 0; + } else if (strategy == HASHMAP_ADD) { + return -EEXIST; + } + } + + if (strategy == HASHMAP_UPDATE) + return -ENOENT; + + if (hashmap_needs_to_grow(map)) { + err = hashmap_grow(map); + if (err) + return err; + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + } + + entry = malloc(sizeof(struct hashmap_entry)); + if (!entry) + return -ENOMEM; + + entry->key = key; + entry->value = value; + hashmap_add_entry(&map->buckets[h], entry); + map->sz++; + + return 0; +} + +bool hashmap_find(const struct hashmap *map, long key, long *value) +{ + struct hashmap_entry *entry; + size_t h; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (!hashmap_find_entry(map, key, h, NULL, &entry)) + return false; + + if (value) + *value = entry->value; + return true; +} + +bool hashmap_delete(struct hashmap *map, long key, + long *old_key, long *old_value) +{ + struct hashmap_entry **pprev, *entry; + size_t h; + + h = hash_bits(map->hash_fn(key, map->ctx), map->cap_bits); + if (!hashmap_find_entry(map, key, h, &pprev, &entry)) + return false; + + if (old_key) + *old_key = entry->key; + if (old_value) + *old_value = entry->value; + + hashmap_del_entry(pprev, entry); + free(entry); + map->sz--; + + return true; +} diff --git a/tools/perf/util/hashmap.h b/tools/perf/util/hashmap.h new file mode 100644 index 000000000..0c4f155e8 --- /dev/null +++ b/tools/perf/util/hashmap.h @@ -0,0 +1,208 @@ +/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ + +/* + * Generic non-thread safe hash map implementation. + * + * Copyright (c) 2019 Facebook + */ +#ifndef __LIBBPF_HASHMAP_H +#define __LIBBPF_HASHMAP_H + +#include +#include +#include + +static inline size_t hash_bits(size_t h, int bits) +{ + /* shuffle bits and return requested number of upper bits */ + if (bits == 0) + return 0; + +#if (__SIZEOF_SIZE_T__ == __SIZEOF_LONG_LONG__) + /* LP64 case */ + return (h * 11400714819323198485llu) >> (__SIZEOF_LONG_LONG__ * 8 - bits); +#elif (__SIZEOF_SIZE_T__ <= __SIZEOF_LONG__) + return (h * 2654435769lu) >> (__SIZEOF_LONG__ * 8 - bits); +#else +# error "Unsupported size_t size" +#endif +} + +/* generic C-string hashing function */ +static inline size_t str_hash(const char *s) +{ + size_t h = 0; + + while (*s) { + h = h * 31 + *s; + s++; + } + return h; +} + +typedef size_t (*hashmap_hash_fn)(long key, void *ctx); +typedef bool (*hashmap_equal_fn)(long key1, long key2, void *ctx); + +/* + * Hashmap interface is polymorphic, keys and values could be either + * long-sized integers or pointers, this is achieved as follows: + * - interface functions that operate on keys and values are hidden + * behind auxiliary macros, e.g. hashmap_insert <-> hashmap__insert; + * - these auxiliary macros cast the key and value parameters as + * long or long *, so the user does not have to specify the casts explicitly; + * - for pointer parameters (e.g. old_key) the size of the pointed + * type is verified by hashmap_cast_ptr using _Static_assert; + * - when iterating using hashmap__for_each_* forms + * hasmap_entry->key should be used for integer keys and + * hasmap_entry->pkey should be used for pointer keys, + * same goes for values. + */ +struct hashmap_entry { + union { + long key; + const void *pkey; + }; + union { + long value; + void *pvalue; + }; + struct hashmap_entry *next; +}; + +struct hashmap { + hashmap_hash_fn hash_fn; + hashmap_equal_fn equal_fn; + void *ctx; + + struct hashmap_entry **buckets; + size_t cap; + size_t cap_bits; + size_t sz; +}; + +void hashmap__init(struct hashmap *map, hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, void *ctx); +struct hashmap *hashmap__new(hashmap_hash_fn hash_fn, + hashmap_equal_fn equal_fn, + void *ctx); +void hashmap__clear(struct hashmap *map); +void hashmap__free(struct hashmap *map); + +size_t hashmap__size(const struct hashmap *map); +size_t hashmap__capacity(const struct hashmap *map); + +/* + * Hashmap insertion strategy: + * - HASHMAP_ADD - only add key/value if key doesn't exist yet; + * - HASHMAP_SET - add key/value pair if key doesn't exist yet; otherwise, + * update value; + * - HASHMAP_UPDATE - update value, if key already exists; otherwise, do + * nothing and return -ENOENT; + * - HASHMAP_APPEND - always add key/value pair, even if key already exists. + * This turns hashmap into a multimap by allowing multiple values to be + * associated with the same key. Most useful read API for such hashmap is + * hashmap__for_each_key_entry() iteration. If hashmap__find() is still + * used, it will return last inserted key/value entry (first in a bucket + * chain). + */ +enum hashmap_insert_strategy { + HASHMAP_ADD, + HASHMAP_SET, + HASHMAP_UPDATE, + HASHMAP_APPEND, +}; + +#define hashmap_cast_ptr(p) ({ \ + _Static_assert((__builtin_constant_p((p)) ? (p) == NULL : 0) || \ + sizeof(*(p)) == sizeof(long), \ + #p " pointee should be a long-sized integer or a pointer"); \ + (long *)(p); \ +}) + +/* + * hashmap__insert() adds key/value entry w/ various semantics, depending on + * provided strategy value. If a given key/value pair replaced already + * existing key/value pair, both old key and old value will be returned + * through old_key and old_value to allow calling code do proper memory + * management. + */ +int hashmap_insert(struct hashmap *map, long key, long value, + enum hashmap_insert_strategy strategy, + long *old_key, long *old_value); + +#define hashmap__insert(map, key, value, strategy, old_key, old_value) \ + hashmap_insert((map), (long)(key), (long)(value), (strategy), \ + hashmap_cast_ptr(old_key), \ + hashmap_cast_ptr(old_value)) + +#define hashmap__add(map, key, value) \ + hashmap__insert((map), (key), (value), HASHMAP_ADD, NULL, NULL) + +#define hashmap__set(map, key, value, old_key, old_value) \ + hashmap__insert((map), (key), (value), HASHMAP_SET, (old_key), (old_value)) + +#define hashmap__update(map, key, value, old_key, old_value) \ + hashmap__insert((map), (key), (value), HASHMAP_UPDATE, (old_key), (old_value)) + +#define hashmap__append(map, key, value) \ + hashmap__insert((map), (key), (value), HASHMAP_APPEND, NULL, NULL) + +bool hashmap_delete(struct hashmap *map, long key, long *old_key, long *old_value); + +#define hashmap__delete(map, key, old_key, old_value) \ + hashmap_delete((map), (long)(key), \ + hashmap_cast_ptr(old_key), \ + hashmap_cast_ptr(old_value)) + +bool hashmap_find(const struct hashmap *map, long key, long *value); + +#define hashmap__find(map, key, value) \ + hashmap_find((map), (long)(key), hashmap_cast_ptr(value)) + +/* + * hashmap__for_each_entry - iterate over all entries in hashmap + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @bkt: integer used as a bucket loop cursor + */ +#define hashmap__for_each_entry(map, cur, bkt) \ + for (bkt = 0; bkt < (map)->cap; bkt++) \ + for (cur = (map)->buckets[bkt]; cur; cur = cur->next) + +/* + * hashmap__for_each_entry_safe - iterate over all entries in hashmap, safe + * against removals + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @tmp: struct hashmap_entry * used as a temporary next cursor storage + * @bkt: integer used as a bucket loop cursor + */ +#define hashmap__for_each_entry_safe(map, cur, tmp, bkt) \ + for (bkt = 0; bkt < (map)->cap; bkt++) \ + for (cur = (map)->buckets[bkt]; \ + cur && ({tmp = cur->next; true; }); \ + cur = tmp) + +/* + * hashmap__for_each_key_entry - iterate over entries associated with given key + * @map: hashmap to iterate + * @cur: struct hashmap_entry * used as a loop cursor + * @key: key to iterate entries for + */ +#define hashmap__for_each_key_entry(map, cur, _key) \ + for (cur = (map)->buckets \ + ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \ + : NULL; \ + cur; \ + cur = cur->next) \ + if ((map)->equal_fn(cur->key, (_key), (map)->ctx)) + +#define hashmap__for_each_key_entry_safe(map, cur, tmp, _key) \ + for (cur = (map)->buckets \ + ? (map)->buckets[hash_bits((map)->hash_fn((_key), (map)->ctx), (map)->cap_bits)] \ + : NULL; \ + cur && ({ tmp = cur->next; true; }); \ + cur = tmp) \ + if ((map)->equal_fn(cur->key, (_key), (map)->ctx)) + +#endif /* __LIBBPF_HASHMAP_H */ diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c new file mode 100644 index 000000000..091d8f7f6 --- /dev/null +++ b/tools/perf/util/header.c @@ -0,0 +1,5799 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include "string2.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef HAVE_LIBBPF_SUPPORT +#include +#endif +#include +#include // reallocarray + +#include "dso.h" +#include "evlist.h" +#include "evsel.h" +#include "util/evsel_fprintf.h" +#include "header.h" +#include "memswap.h" +#include "trace-event.h" +#include "session.h" +#include "symbol.h" +#include "debug.h" +#include "cpumap.h" +#include "pmu.h" +#include "pmus.h" +#include "vdso.h" +#include "strbuf.h" +#include "build-id.h" +#include "data.h" +#include +#include +#include "asm/bug.h" +#include "tool.h" +#include "../perf.h" +#include "time-utils.h" +#include "units.h" +#include "util/util.h" // perf_exe() +#include "cputopo.h" +#include "bpf-event.h" +#include "bpf-utils.h" +#include "clockid.h" +#include "cacheline.h" + +#include +#include + +#ifdef HAVE_LIBTRACEEVENT +#include +#endif + +/* + * nr_ids * sizeof(struct perf_sample_id) must not overflow + * size_t on 32-bit; the struct is ~104 bytes (32-bit) or + * ~184 bytes (64-bit), so 1<<24 (16M) keeps the product + * under 2 GB on 32-bit. + * + * This is a per-attribute cap only — the total across all + * attributes is not capped because legitimate high-core-count + * workloads (e.g. 5000 tracepoints × 4096 CPUs) can exceed + * a single-attribute limit. + */ +#define MAX_IDS_PER_ATTR (1 << 24) +/* + * Cap nr_attrs to prevent resource exhaustion from crafted + * files. 65536 is well beyond any real workload (perf stat + * typically uses < 100 events) but prevents u64-to-int + * truncation on the attr count. + */ +#define MAX_NR_ATTRS (1 << 16) +#define MAX_BPF_DATA_LEN (256 * 1024 * 1024) +#define MAX_BPF_PROGS 131072 +#define MAX_CACHE_ENTRIES 32768 +#define MAX_GROUP_DESC 32768 +#define MAX_NUMA_NODES 4096 +#define MAX_PMU_CAPS 512 +#define MAX_PMU_MAPPINGS 4096 +#define MAX_SCHED_DOMAINS 64 + +/* + * magic2 = "PERFILE2" + * must be a numerical value to let the endianness + * determine the memory layout. That way we are able + * to detect endianness when reading the perf.data file + * back. + * + * we check for legacy (PERFFILE) format. + */ +static const char *__perf_magic1 = "PERFFILE"; +static const u64 __perf_magic2 = 0x32454c4946524550ULL; +static const u64 __perf_magic2_sw = 0x50455246494c4532ULL; + +#define PERF_MAGIC __perf_magic2 +#define DNAME_LEN 16 + +const char perf_version_string[] = PERF_VERSION; + +struct perf_file_attr { + struct perf_event_attr attr; + struct perf_file_section ids; +}; + +void perf_header__set_feat(struct perf_header *header, int feat) +{ + __set_bit(feat, header->adds_features); +} + +void perf_header__clear_feat(struct perf_header *header, int feat) +{ + __clear_bit(feat, header->adds_features); +} + +bool perf_header__has_feat(const struct perf_header *header, int feat) +{ + return test_bit(feat, header->adds_features); +} + +static int __do_write_fd(struct feat_fd *ff, const void *buf, size_t size) +{ + ssize_t ret = writen(ff->fd, buf, size); + + if (ret != (ssize_t)size) + return ret < 0 ? (int)ret : -1; + return 0; +} + +static int __do_write_buf(struct feat_fd *ff, const void *buf, size_t size) +{ + /* struct perf_event_header::size is u16 */ + const size_t max_size = 0xffff - sizeof(struct perf_event_header); + size_t new_size = ff->size; + void *addr; + + if (size + ff->offset > max_size) + return -E2BIG; + + while (size > (new_size - ff->offset)) + new_size <<= 1; + new_size = min(max_size, new_size); + + if (ff->size < new_size) { + addr = realloc(ff->buf, new_size); + if (!addr) + return -ENOMEM; + ff->buf = addr; + ff->size = new_size; + } + + memcpy(ff->buf + ff->offset, buf, size); + ff->offset += size; + + return 0; +} + +/* Return: 0 if succeeded, -ERR if failed. */ +int do_write(struct feat_fd *ff, const void *buf, size_t size) +{ + if (!ff->buf) + return __do_write_fd(ff, buf, size); + return __do_write_buf(ff, buf, size); +} + +/* Return: 0 if succeeded, -ERR if failed. */ +static int do_write_bitmap(struct feat_fd *ff, unsigned long *set, u64 size) +{ + size_t byte_size = BITS_TO_LONGS(size) * sizeof(unsigned long); + int i, ret; + + ret = do_write(ff, &size, sizeof(size)); + if (ret < 0) + return ret; + + /* + * The on-disk format uses u64 elements, but the in-memory bitmap + * uses unsigned long, which is only 4 bytes on 32-bit architectures. + * Copy with bounded size so the last element doesn't read past the + * bitmap allocation when BITS_TO_LONGS(size) is odd. + */ + for (i = 0; (u64) i < BITS_TO_U64(size); i++) { + u64 val = 0; + size_t off = i * sizeof(val); + + memcpy(&val, (char *)set + off, min(sizeof(val), byte_size - off)); + ret = do_write(ff, &val, sizeof(val)); + if (ret < 0) + return ret; + } + + return 0; +} + +/* Return: 0 if succeeded, -ERR if failed. */ +int write_padded(struct feat_fd *ff, const void *bf, + size_t count, size_t count_aligned) +{ + static const char zero_buf[NAME_ALIGN]; + int err = do_write(ff, bf, count); + + if (!err) + err = do_write(ff, zero_buf, count_aligned - count); + + return err; +} + +#define string_size(str) \ + (PERF_ALIGN((strlen(str) + 1), NAME_ALIGN) + sizeof(u32)) + +/* Return: 0 if succeeded, -ERR if failed. */ +static int do_write_string(struct feat_fd *ff, const char *str) +{ + u32 len, olen; + int ret; + + olen = strlen(str) + 1; + len = PERF_ALIGN(olen, NAME_ALIGN); + + /* write len, incl. \0 */ + ret = do_write(ff, &len, sizeof(len)); + if (ret < 0) + return ret; + + return write_padded(ff, str, olen, len); +} + +static int __do_read_fd(struct feat_fd *ff, void *addr, ssize_t size) +{ + ssize_t ret = readn(ff->fd, addr, size); + + if (ret != size) + return ret < 0 ? (int)ret : -1; + ff->offset += size; + return 0; +} + +static int __do_read_buf(struct feat_fd *ff, void *addr, ssize_t size) +{ + memcpy(addr, ff->buf + ff->offset, size); + ff->offset += size; + + return 0; +} + +static int __do_read(struct feat_fd *ff, void *addr, ssize_t size) +{ + /* + * Reject negative sizes, which on 32-bit can occur when a + * u32 >= 0x80000000 is passed as ssize_t. The cast to + * ssize_t is safe because perf_header__process_sections() + * validates that each section fits within the file size + * before any feature callback reaches here, and only + * feature sections (metadata like build IDs, topology, etc.) + * use this path — these cannot legitimately approach 2GB. + */ + if (size < 0 || size > (ssize_t)ff->size - ff->offset) + return -1; + + if (!ff->buf) + return __do_read_fd(ff, addr, size); + return __do_read_buf(ff, addr, size); +} + +static int do_read_u32(struct feat_fd *ff, u32 *addr) +{ + int ret; + + ret = __do_read(ff, addr, sizeof(*addr)); + if (ret) + return ret; + + if (ff->ph->needs_swap) + *addr = bswap_32(*addr); + return 0; +} + +static int do_read_u64(struct feat_fd *ff, u64 *addr) +{ + int ret; + + ret = __do_read(ff, addr, sizeof(*addr)); + if (ret) + return ret; + + if (ff->ph->needs_swap) + *addr = bswap_64(*addr); + return 0; +} + +static char *do_read_string(struct feat_fd *ff) +{ + u32 len; + char *buf; + + if (do_read_u32(ff, &len)) + return NULL; + + /* At least the null terminator. */ + if (len < 1 || len > ff->size - ff->offset) { + pr_debug("do_read_string: invalid length %u (remaining %zu)\n", + len, (size_t)(ff->size - ff->offset)); + return NULL; + } + + buf = malloc(len); + if (!buf) + return NULL; + + if (!__do_read(ff, buf, len)) { + /* + * do_write_string() writes len including the null + * terminator, padded to NAME_ALIGN. Ensure the + * string is always null-terminated even if the file + * data has been tampered with. + */ + buf[len - 1] = '\0'; + return buf; + } + + free(buf); + return NULL; +} + +/* Return: 0 if succeeded, -ERR if failed. */ +static int do_read_bitmap(struct feat_fd *ff, unsigned long **pset, u64 *psize) +{ + unsigned long *set; + u64 size, *p; + int i, ret; + + ret = do_read_u64(ff, &size); + if (ret) + return ret; + + /* Bitmap APIs use int for nbits; reject u64 values that truncate. */ + if (size > INT_MAX || + BITS_TO_U64(size) > (ff->size - ff->offset) / sizeof(u64)) { + pr_debug("do_read_bitmap: size %" PRIu64 " exceeds section bounds\n", size); + return -1; + } + + /* + * bitmap_zalloc() allocates in unsigned long units, which are only + * 4 bytes on 32-bit architectures. The read loop below casts the + * buffer to u64 * and writes 8-byte elements, so allocate in u64 + * units to ensure the buffer is large enough. + */ + set = calloc(BITS_TO_U64(size), sizeof(u64)); + if (!set) + return -ENOMEM; + + p = (u64 *) set; + + for (i = 0; (u64) i < BITS_TO_U64(size); i++) { + ret = do_read_u64(ff, p + i); + if (ret < 0) { + free(set); + return ret; + } + } + + *pset = set; + *psize = size; + return 0; +} + +static int write_tracing_data(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + +#ifdef HAVE_LIBTRACEEVENT + return read_tracing_data(ff->fd, &evlist->core.entries); +#else + pr_err("ERROR: Trying to write tracing data without libtraceevent support.\n"); + return -1; +#endif +} + +static int write_build_id(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_session *session; + int err; + + session = container_of(ff->ph, struct perf_session, header); + + if (!perf_session__read_build_ids(session, true)) + return -1; + + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + + err = perf_session__write_buildid_table(session, ff); + if (err < 0) { + pr_debug("failed to write buildid table\n"); + return err; + } + + return 0; +} + +static int write_hostname(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct utsname uts; + int ret; + + ret = uname(&uts); + if (ret < 0) + return -1; + + return do_write_string(ff, uts.nodename); +} + +static int write_osrelease(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct utsname uts; + const char *release = NULL; + + if (evlist->session) + release = perf_env__os_release(perf_session__env(evlist->session)); + + if (!release) { + int ret = uname(&uts); + + if (ret < 0) + return -1; + release = uts.release; + } + return do_write_string(ff, release); +} + +static int write_arch(struct feat_fd *ff, struct evlist *evlist) +{ + struct utsname uts; + const char *arch = NULL; + + if (evlist->session) + arch = perf_env__arch(perf_session__env(evlist->session)); + + if (!arch) { + int ret = uname(&uts); + + if (ret < 0) + return -1; + arch = uts.machine; + } + return do_write_string(ff, arch); +} + +static int write_e_machine(struct feat_fd *ff, struct evlist *evlist) +{ + /* e_machine expanded from 16 to 32-bits for alignment. */ + uint32_t e_flags; + uint32_t e_machine = perf_session__e_machine(evlist->session, &e_flags); + int ret; + + ret = do_write(ff, &e_machine, sizeof(e_machine)); + if (ret) + return ret; + + return do_write(ff, &e_flags, sizeof(e_flags)); +} + +static int write_version(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + return do_write_string(ff, perf_version_string); +} + +static int __write_cpudesc(struct feat_fd *ff, const char *cpuinfo_proc) +{ + FILE *file; + char *buf = NULL; + char *s, *p; + const char *search = cpuinfo_proc; + size_t len = 0; + int ret = -1; + + if (!search) + return -1; + + file = fopen("/proc/cpuinfo", "r"); + if (!file) + return -1; + + while (getline(&buf, &len, file) > 0) { + ret = strncmp(buf, search, strlen(search)); + if (!ret) + break; + } + + if (ret) { + ret = -1; + goto done; + } + + s = buf; + + p = strchr(buf, ':'); + if (p && *(p+1) == ' ' && *(p+2)) + s = p + 2; + p = strchr(s, '\n'); + if (p) + *p = '\0'; + + /* squash extra space characters (branding string) */ + p = s; + while (*p) { + if (isspace(*p)) { + char *r = p + 1; + char *q = skip_spaces(r); + *p = ' '; + if (q != (p+1)) + while ((*r++ = *q++)); + } + p++; + } + ret = do_write_string(ff, s); +done: + free(buf); + fclose(file); + return ret; +} + +static int write_cpudesc(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ +#if defined(__powerpc__) || defined(__hppa__) || defined(__sparc__) +#define CPUINFO_PROC { "cpu", } +#elif defined(__s390__) +#define CPUINFO_PROC { "vendor_id", } +#elif defined(__sh__) +#define CPUINFO_PROC { "cpu type", } +#elif defined(__alpha__) || defined(__mips__) +#define CPUINFO_PROC { "cpu model", } +#elif defined(__arm__) +#define CPUINFO_PROC { "model name", "Processor", } +#elif defined(__arc__) +#define CPUINFO_PROC { "Processor", } +#elif defined(__xtensa__) +#define CPUINFO_PROC { "core ID", } +#elif defined(__loongarch__) +#define CPUINFO_PROC { "Model Name", } +#else +#define CPUINFO_PROC { "model name", } +#endif + const char *cpuinfo_procs[] = CPUINFO_PROC; +#undef CPUINFO_PROC + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(cpuinfo_procs); i++) { + int ret; + ret = __write_cpudesc(ff, cpuinfo_procs[i]); + if (ret >= 0) + return ret; + } + return -1; +} + + +static int write_nrcpus(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + long nr; + u32 nrc, nra; + int ret; + + nrc = cpu__max_present_cpu().cpu; + + nr = sysconf(_SC_NPROCESSORS_ONLN); + if (nr < 0) + return -1; + + nra = (u32)(nr & UINT_MAX); + + ret = do_write(ff, &nrc, sizeof(nrc)); + if (ret < 0) + return ret; + + return do_write(ff, &nra, sizeof(nra)); +} + +static int write_event_desc(struct feat_fd *ff, + struct evlist *evlist) +{ + struct evsel *evsel; + u32 nre, nri, sz; + int ret; + + nre = evlist->core.nr_entries; + + /* + * write number of events + */ + ret = do_write(ff, &nre, sizeof(nre)); + if (ret < 0) + return ret; + + /* + * size of perf_event_attr struct + */ + sz = (u32)sizeof(evsel->core.attr); + ret = do_write(ff, &sz, sizeof(sz)); + if (ret < 0) + return ret; + + evlist__for_each_entry(evlist, evsel) { + ret = do_write(ff, &evsel->core.attr, sz); + if (ret < 0) + return ret; + /* + * write number of unique id per event + * there is one id per instance of an event + * + * copy into an nri to be independent of the + * type of ids, + */ + nri = evsel->core.ids; + ret = do_write(ff, &nri, sizeof(nri)); + if (ret < 0) + return ret; + + /* + * write event string as passed on cmdline + */ + ret = do_write_string(ff, evsel__name(evsel)); + if (ret < 0) + return ret; + /* + * write unique ids for this event + */ + ret = do_write(ff, evsel->core.id, evsel->core.ids * sizeof(u64)); + if (ret < 0) + return ret; + } + return 0; +} + +static int write_cmdline(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + char pbuf[MAXPATHLEN], *buf; + int i, ret, n; + + /* actual path to perf binary */ + buf = perf_exe(pbuf, MAXPATHLEN); + + /* account for binary path */ + n = env->nr_cmdline + 1; + + ret = do_write(ff, &n, sizeof(n)); + if (ret < 0) + return ret; + + ret = do_write_string(ff, buf); + if (ret < 0) + return ret; + + for (i = 0 ; i < env->nr_cmdline; i++) { + ret = do_write_string(ff, env->cmdline_argv[i]); + if (ret < 0) + return ret; + } + return 0; +} + + +static int write_cpu_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct cpu_topology *tp; + u32 i; + int ret, j; + + tp = cpu_topology__new(); + if (!tp) + return -1; + + ret = do_write(ff, &tp->package_cpus_lists, sizeof(tp->package_cpus_lists)); + if (ret < 0) + goto done; + + for (i = 0; i < tp->package_cpus_lists; i++) { + ret = do_write_string(ff, tp->package_cpus_list[i]); + if (ret < 0) + goto done; + } + ret = do_write(ff, &tp->core_cpus_lists, sizeof(tp->core_cpus_lists)); + if (ret < 0) + goto done; + + for (i = 0; i < tp->core_cpus_lists; i++) { + ret = do_write_string(ff, tp->core_cpus_list[i]); + if (ret < 0) + break; + } + + ret = perf_env__read_cpu_topology_map(env); + if (ret < 0) + goto done; + + for (j = 0; j < env->nr_cpus_avail; j++) { + ret = do_write(ff, &env->cpu[j].core_id, + sizeof(env->cpu[j].core_id)); + if (ret < 0) + return ret; + ret = do_write(ff, &env->cpu[j].socket_id, + sizeof(env->cpu[j].socket_id)); + if (ret < 0) + return ret; + } + + if (!tp->die_cpus_lists) + goto done; + + ret = do_write(ff, &tp->die_cpus_lists, sizeof(tp->die_cpus_lists)); + if (ret < 0) + goto done; + + for (i = 0; i < tp->die_cpus_lists; i++) { + ret = do_write_string(ff, tp->die_cpus_list[i]); + if (ret < 0) + goto done; + } + + for (j = 0; j < env->nr_cpus_avail; j++) { + ret = do_write(ff, &env->cpu[j].die_id, + sizeof(env->cpu[j].die_id)); + if (ret < 0) + return ret; + } + +done: + cpu_topology__delete(tp); + return ret; +} + + + +static int write_total_mem(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + char *buf = NULL; + FILE *fp; + size_t len = 0; + int ret = -1, n; + uint64_t mem; + + fp = fopen("/proc/meminfo", "r"); + if (!fp) + return -1; + + while (getline(&buf, &len, fp) > 0) { + ret = strncmp(buf, "MemTotal:", 9); + if (!ret) + break; + } + if (!ret) { + n = sscanf(buf, "%*s %"PRIu64, &mem); + if (n == 1) + ret = do_write(ff, &mem, sizeof(mem)); + } else + ret = -1; + free(buf); + fclose(fp); + return ret; +} + +static int write_numa_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct numa_topology *tp; + int ret = -1; + u32 i; + + tp = numa_topology__new(); + if (!tp) + return -ENOMEM; + + ret = do_write(ff, &tp->nr, sizeof(u32)); + if (ret < 0) + goto err; + + for (i = 0; i < tp->nr; i++) { + struct numa_topology_node *n = &tp->nodes[i]; + + ret = do_write(ff, &n->node, sizeof(u32)); + if (ret < 0) + goto err; + + ret = do_write(ff, &n->mem_total, sizeof(u64)); + if (ret) + goto err; + + ret = do_write(ff, &n->mem_free, sizeof(u64)); + if (ret) + goto err; + + ret = do_write_string(ff, n->cpus); + if (ret < 0) + goto err; + } + + ret = 0; + +err: + numa_topology__delete(tp); + return ret; +} + +/* + * File format: + * + * struct pmu_mappings { + * u32 pmu_num; + * struct pmu_map { + * u32 type; + * char name[]; + * }[pmu_num]; + * }; + */ + +static int write_pmu_mappings(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *pmu = NULL; + u32 pmu_num = 0; + int ret; + + /* + * Do a first pass to count number of pmu to avoid lseek so this + * works in pipe mode as well. + */ + while ((pmu = perf_pmus__scan(pmu))) + pmu_num++; + + ret = do_write(ff, &pmu_num, sizeof(pmu_num)); + if (ret < 0) + return ret; + + while ((pmu = perf_pmus__scan(pmu))) { + ret = do_write(ff, &pmu->type, sizeof(pmu->type)); + if (ret < 0) + return ret; + + ret = do_write_string(ff, pmu->name); + if (ret < 0) + return ret; + } + + return 0; +} + +/* + * File format: + * + * struct group_descs { + * u32 nr_groups; + * struct group_desc { + * char name[]; + * u32 leader_idx; + * u32 nr_members; + * }[nr_groups]; + * }; + */ +static int write_group_desc(struct feat_fd *ff, + struct evlist *evlist) +{ + u32 nr_groups = evlist__nr_groups(evlist); + struct evsel *evsel; + int ret; + + ret = do_write(ff, &nr_groups, sizeof(nr_groups)); + if (ret < 0) + return ret; + + evlist__for_each_entry(evlist, evsel) { + if (evsel__is_group_leader(evsel) && evsel->core.nr_members > 1) { + const char *name = evsel->group_name ?: "{anon_group}"; + u32 leader_idx = evsel->core.idx; + u32 nr_members = evsel->core.nr_members; + + ret = do_write_string(ff, name); + if (ret < 0) + return ret; + + ret = do_write(ff, &leader_idx, sizeof(leader_idx)); + if (ret < 0) + return ret; + + ret = do_write(ff, &nr_members, sizeof(nr_members)); + if (ret < 0) + return ret; + } + } + return 0; +} + +/* + * Return the CPU id as a raw string. + * + * Each architecture should provide a more precise id string that + * can be use to match the architecture's "mapfile". + */ +char * __weak get_cpuid_str(struct perf_cpu cpu __maybe_unused) +{ + return NULL; +} + +char *get_cpuid_allow_env_override(struct perf_cpu cpu) +{ + char *cpuid; + static bool printed; + + cpuid = getenv("PERF_CPUID"); + if (cpuid) + cpuid = strdup(cpuid); + if (!cpuid) + cpuid = get_cpuid_str(cpu); + if (!cpuid) + return NULL; + + if (!printed) { + pr_debug("Using CPUID %s\n", cpuid); + printed = true; + } + return cpuid; +} + +/* Return zero when the cpuid from the mapfile.csv matches the + * cpuid string generated on this platform. + * Otherwise return non-zero. + */ +int __weak strcmp_cpuid_str(const char *mapcpuid, const char *cpuid) +{ + regex_t re; + regmatch_t pmatch[1]; + int match; + + if (regcomp(&re, mapcpuid, REG_EXTENDED) != 0) { + /* Warn unable to generate match particular string. */ + pr_info("Invalid regular expression %s\n", mapcpuid); + return 1; + } + + match = !regexec(&re, cpuid, 1, pmatch, 0); + regfree(&re); + if (match) { + size_t match_len = (pmatch[0].rm_eo - pmatch[0].rm_so); + + /* Verify the entire string matched. */ + if (match_len == strlen(cpuid)) + return 0; + } + return 1; +} + +/* + * default get_cpuid(): nothing gets recorded + * actual implementation must be in arch/$(SRCARCH)/util/header.c + */ +int __weak get_cpuid(char *buffer __maybe_unused, size_t sz __maybe_unused, + struct perf_cpu cpu __maybe_unused) +{ + return ENOSYS; /* Not implemented */ +} + +static int write_cpuid(struct feat_fd *ff, struct evlist *evlist) +{ + struct perf_cpu cpu = perf_cpu_map__min(evlist->core.all_cpus); + char buffer[64]; + int ret; + + ret = get_cpuid(buffer, sizeof(buffer), cpu); + if (ret) + return -1; + + return do_write_string(ff, buffer); +} + +static int write_branch_stack(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + return 0; +} + +static int write_auxtrace(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_session *session; + int err; + + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + + session = container_of(ff->ph, struct perf_session, header); + + err = auxtrace_index__write(ff->fd, &session->auxtrace_index); + if (err < 0) + pr_err("Failed to write auxtrace index\n"); + return err; +} + +static int write_clockid(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + return do_write(ff, &ff->ph->env.clock.clockid_res_ns, + sizeof(ff->ph->env.clock.clockid_res_ns)); +} + +static int write_clock_data(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + u64 *data64; + u32 data32; + int ret; + + /* version */ + data32 = 1; + + ret = do_write(ff, &data32, sizeof(data32)); + if (ret < 0) + return ret; + + /* clockid */ + data32 = ff->ph->env.clock.clockid; + + ret = do_write(ff, &data32, sizeof(data32)); + if (ret < 0) + return ret; + + /* TOD ref time */ + data64 = &ff->ph->env.clock.tod_ns; + + ret = do_write(ff, data64, sizeof(*data64)); + if (ret < 0) + return ret; + + /* clockid ref time */ + data64 = &ff->ph->env.clock.clockid_ns; + + return do_write(ff, data64, sizeof(*data64)); +} + +static int write_hybrid_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct hybrid_topology *tp; + int ret; + u32 i; + + tp = hybrid_topology__new(); + if (!tp) + return -ENOENT; + + ret = do_write(ff, &tp->nr, sizeof(u32)); + if (ret < 0) + goto err; + + for (i = 0; i < tp->nr; i++) { + struct hybrid_topology_node *n = &tp->nodes[i]; + + ret = do_write_string(ff, n->pmu_name); + if (ret < 0) + goto err; + + ret = do_write_string(ff, n->cpus); + if (ret < 0) + goto err; + } + + ret = 0; + +err: + hybrid_topology__delete(tp); + return ret; +} + +static int write_dir_format(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_session *session; + struct perf_data *data; + + session = container_of(ff->ph, struct perf_session, header); + data = session->data; + + if (WARN_ON(!perf_data__is_dir(data))) + return -1; + + return do_write(ff, &data->dir.version, sizeof(data->dir.version)); +} + +static int write_bpf_prog_info(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + int ret = 0; + + down_read(&env->bpf_progs.lock); + + ret = do_write(ff, &env->bpf_progs.infos_cnt, + sizeof(env->bpf_progs.infos_cnt)); + if (ret < 0 || env->bpf_progs.infos_cnt == 0) + goto out; + + root = &env->bpf_progs.infos; + next = rb_first(root); + while (next) { + struct bpf_prog_info_node *node; + size_t len; + + node = rb_entry(next, struct bpf_prog_info_node, rb_node); + next = rb_next(&node->rb_node); + len = sizeof(struct perf_bpil) + + node->info_linear->data_len; + + /* before writing to file, translate address to offset */ + bpil_addr_to_offs(node->info_linear); + ret = do_write(ff, node->info_linear, len); + /* + * translate back to address even when do_write() fails, + * so that this function never changes the data. + */ + bpil_offs_to_addr(node->info_linear); + if (ret < 0) + goto out; + } +out: + up_read(&env->bpf_progs.lock); + return ret; +#else + pr_err("ERROR: Trying to write bpf_prog_info without libbpf support.\n"); + return -1; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int write_bpf_btf(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + int ret = 0; + + down_read(&env->bpf_progs.lock); + + ret = do_write(ff, &env->bpf_progs.btfs_cnt, + sizeof(env->bpf_progs.btfs_cnt)); + + if (ret < 0 || env->bpf_progs.btfs_cnt == 0) + goto out; + + root = &env->bpf_progs.btfs; + next = rb_first(root); + while (next) { + struct btf_node *node; + + node = rb_entry(next, struct btf_node, rb_node); + next = rb_next(&node->rb_node); + ret = do_write(ff, &node->id, + sizeof(u32) * 2 + node->data_size); + if (ret < 0) + goto out; + } +out: + up_read(&env->bpf_progs.lock); + return ret; +#else + pr_err("ERROR: Trying to write btf data without libbpf support.\n"); + return -1; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int cpu_cache_level__sort(const void *a, const void *b) +{ + struct cpu_cache_level *cache_a = (struct cpu_cache_level *)a; + struct cpu_cache_level *cache_b = (struct cpu_cache_level *)b; + + return cache_a->level - cache_b->level; +} + +static bool cpu_cache_level__cmp(struct cpu_cache_level *a, struct cpu_cache_level *b) +{ + if (a->level != b->level) + return false; + + if (a->line_size != b->line_size) + return false; + + if (a->sets != b->sets) + return false; + + if (a->ways != b->ways) + return false; + + if (strcmp(a->type, b->type)) + return false; + + if (strcmp(a->size, b->size)) + return false; + + if (strcmp(a->map, b->map)) + return false; + + return true; +} + +static int cpu_cache_level__read(struct cpu_cache_level *cache, u32 cpu, u16 level) +{ + char path[PATH_MAX], file[PATH_MAX]; + struct stat st; + size_t len; + + scnprintf(path, PATH_MAX, "devices/system/cpu/cpu%d/cache/index%d/", cpu, level); + scnprintf(file, PATH_MAX, "%s/%s", sysfs__mountpoint(), path); + + if (stat(file, &st)) + return 1; + + scnprintf(file, PATH_MAX, "%s/level", path); + if (sysfs__read_int(file, (int *) &cache->level)) + return -1; + + scnprintf(file, PATH_MAX, "%s/coherency_line_size", path); + if (sysfs__read_int(file, (int *) &cache->line_size)) + return -1; + + scnprintf(file, PATH_MAX, "%s/number_of_sets", path); + if (sysfs__read_int(file, (int *) &cache->sets)) + return -1; + + scnprintf(file, PATH_MAX, "%s/ways_of_associativity", path); + if (sysfs__read_int(file, (int *) &cache->ways)) + return -1; + + scnprintf(file, PATH_MAX, "%s/type", path); + if (sysfs__read_str(file, &cache->type, &len)) + return -1; + + cache->type[len] = 0; + cache->type = strim(cache->type); + + scnprintf(file, PATH_MAX, "%s/size", path); + if (sysfs__read_str(file, &cache->size, &len)) { + zfree(&cache->type); + return -1; + } + + cache->size[len] = 0; + cache->size = strim(cache->size); + + scnprintf(file, PATH_MAX, "%s/shared_cpu_list", path); + if (sysfs__read_str(file, &cache->map, &len)) { + zfree(&cache->size); + zfree(&cache->type); + return -1; + } + + cache->map[len] = 0; + cache->map = strim(cache->map); + return 0; +} + +static void cpu_cache_level__fprintf(FILE *out, struct cpu_cache_level *c) +{ + fprintf(out, "L%d %-15s %8s [%s]\n", c->level, c->type, c->size, c->map); +} + +/* + * Build caches levels for a particular CPU from the data in + * /sys/devices/system/cpu/cpu/cache/ + * The cache level data is stored in caches[] from index at + * *cntp. + */ +int build_caches_for_cpu(u32 cpu, struct cpu_cache_level caches[], u32 *cntp) +{ + u16 level; + + for (level = 0; level < MAX_CACHE_LVL; level++) { + struct cpu_cache_level c; + int err; + u32 i; + + err = cpu_cache_level__read(&c, cpu, level); + if (err < 0) + return err; + + if (err == 1) + break; + + for (i = 0; i < *cntp; i++) { + if (cpu_cache_level__cmp(&c, &caches[i])) + break; + } + + if (i == *cntp) { + caches[*cntp] = c; + *cntp = *cntp + 1; + } else + cpu_cache_level__free(&c); + } + + return 0; +} + +static int build_caches(struct cpu_cache_level caches[], u32 *cntp) +{ + u32 nr, cpu, cnt = 0; + + nr = cpu__max_cpu().cpu; + + for (cpu = 0; cpu < nr; cpu++) { + int ret = build_caches_for_cpu(cpu, caches, &cnt); + + if (ret) + return ret; + } + *cntp = cnt; + return 0; +} + +static int write_cache(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + u32 max_caches = cpu__max_cpu().cpu * MAX_CACHE_LVL; + struct cpu_cache_level caches[max_caches]; + u32 cnt = 0, i, version = 1; + int ret; + + ret = build_caches(caches, &cnt); + if (ret) + goto out; + + qsort(&caches, cnt, sizeof(struct cpu_cache_level), cpu_cache_level__sort); + + ret = do_write(ff, &version, sizeof(u32)); + if (ret < 0) + goto out; + + ret = do_write(ff, &cnt, sizeof(u32)); + if (ret < 0) + goto out; + + for (i = 0; i < cnt; i++) { + struct cpu_cache_level *c = &caches[i]; + + #define _W(v) \ + ret = do_write(ff, &c->v, sizeof(u32)); \ + if (ret < 0) \ + goto out; + + _W(level) + _W(line_size) + _W(sets) + _W(ways) + #undef _W + + #define _W(v) \ + ret = do_write_string(ff, (const char *) c->v); \ + if (ret < 0) \ + goto out; + + _W(type) + _W(size) + _W(map) + #undef _W + } + +out: + for (i = 0; i < cnt; i++) + cpu_cache_level__free(&caches[i]); + return ret; +} + +static int write_cln_size(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + int cln_size = cacheline_size(); + + if (!cln_size) + cln_size = DEFAULT_CACHELINE_SIZE; + + ff->ph->env.cln_size = cln_size; + + return do_write(ff, &cln_size, sizeof(cln_size)); +} + +static int write_stat(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + return 0; +} + +static int write_sample_time(struct feat_fd *ff, + struct evlist *evlist) +{ + int ret; + + ret = do_write(ff, &evlist->first_sample_time, + sizeof(evlist->first_sample_time)); + if (ret < 0) + return ret; + + return do_write(ff, &evlist->last_sample_time, + sizeof(evlist->last_sample_time)); +} + + +static int memory_node__read(struct memory_node *n, unsigned long idx) +{ + unsigned int phys, size = 0; + char path[PATH_MAX]; + struct io_dirent64 *ent; + struct io_dir dir; + +#define for_each_memory(mem, dir) \ + while ((ent = io_dir__readdir(&dir)) != NULL) \ + if (strcmp(ent->d_name, ".") && \ + strcmp(ent->d_name, "..") && \ + sscanf(ent->d_name, "memory%u", &mem) == 1) + + scnprintf(path, PATH_MAX, + "%s/devices/system/node/node%lu", + sysfs__mountpoint(), idx); + + io_dir__init(&dir, open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY)); + if (dir.dirfd < 0) { + pr_warning("failed: can't open memory sysfs data '%s'\n", path); + return -1; + } + + for_each_memory(phys, dir) { + size = max(phys, size); + } + + size++; + + n->set = bitmap_zalloc(size); + if (!n->set) { + close(dir.dirfd); + return -ENOMEM; + } + + n->node = idx; + n->size = size; + + io_dir__rewinddir(&dir); + + for_each_memory(phys, dir) { + __set_bit(phys, n->set); + } + + close(dir.dirfd); + return 0; +} + +static void memory_node__delete_nodes(struct memory_node *nodesp, u64 cnt) +{ + for (u64 i = 0; i < cnt; i++) + bitmap_free(nodesp[i].set); + + free(nodesp); +} + +static int memory_node__sort(const void *a, const void *b) +{ + const struct memory_node *na = a; + const struct memory_node *nb = b; + + return na->node - nb->node; +} + +static int build_mem_topology(struct memory_node **nodesp, u64 *cntp) +{ + char path[PATH_MAX]; + struct io_dirent64 *ent; + struct io_dir dir; + int ret = 0; + size_t cnt = 0, size = 0; + struct memory_node *nodes = NULL; + + scnprintf(path, PATH_MAX, "%s/devices/system/node/", + sysfs__mountpoint()); + + io_dir__init(&dir, open(path, O_CLOEXEC | O_DIRECTORY | O_RDONLY)); + if (dir.dirfd < 0) { + pr_debug2("%s: couldn't read %s, does this arch have topology information?\n", + __func__, path); + return -1; + } + + while (!ret && (ent = io_dir__readdir(&dir))) { + unsigned int idx; + int r; + + if (!strcmp(ent->d_name, ".") || + !strcmp(ent->d_name, "..")) + continue; + + r = sscanf(ent->d_name, "node%u", &idx); + if (r != 1) + continue; + + if (cnt >= size) { + struct memory_node *new_nodes = + reallocarray(nodes, cnt + 4, sizeof(*nodes)); + + if (!new_nodes) { + pr_err("Failed to write MEM_TOPOLOGY, size %zd nodes\n", size); + ret = -ENOMEM; + goto out; + } + nodes = new_nodes; + size += 4; + } + ret = memory_node__read(&nodes[cnt], idx); + if (!ret) + cnt += 1; + } +out: + close(dir.dirfd); + if (!ret) { + *cntp = cnt; + *nodesp = nodes; + qsort(nodes, cnt, sizeof(nodes[0]), memory_node__sort); + } else + memory_node__delete_nodes(nodes, cnt); + + return ret; +} + +/* + * The MEM_TOPOLOGY holds physical memory map for every + * node in system. The format of data is as follows: + * + * 0 - version | for future changes + * 8 - block_size_bytes | /sys/devices/system/memory/block_size_bytes + * 16 - count | number of nodes + * + * For each node we store map of physical indexes for + * each node: + * + * 32 - node id | node index + * 40 - size | size of bitmap + * 48 - bitmap | bitmap of memory indexes that belongs to node + */ +static int write_mem_topology(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + struct memory_node *nodes = NULL; + u64 bsize, version = 1, i, nr = 0; + int ret; + + ret = sysfs__read_xll("devices/system/memory/block_size_bytes", + (unsigned long long *) &bsize); + if (ret) + return ret; + + ret = build_mem_topology(&nodes, &nr); + if (ret) + return ret; + + ret = do_write(ff, &version, sizeof(version)); + if (ret < 0) + goto out; + + ret = do_write(ff, &bsize, sizeof(bsize)); + if (ret < 0) + goto out; + + ret = do_write(ff, &nr, sizeof(nr)); + if (ret < 0) + goto out; + + for (i = 0; i < nr; i++) { + struct memory_node *n = &nodes[i]; + + #define _W(v) \ + ret = do_write(ff, &n->v, sizeof(n->v)); \ + if (ret < 0) \ + goto out; + + _W(node) + _W(size) + + #undef _W + + ret = do_write_bitmap(ff, n->set, n->size); + if (ret < 0) + goto out; + } + +out: + memory_node__delete_nodes(nodes, nr); + return ret; +} + +static int write_compressed(struct feat_fd *ff __maybe_unused, + struct evlist *evlist __maybe_unused) +{ + int ret; + + ret = do_write(ff, &(ff->ph->env.comp_ver), sizeof(ff->ph->env.comp_ver)); + if (ret) + return ret; + + ret = do_write(ff, &(ff->ph->env.comp_type), sizeof(ff->ph->env.comp_type)); + if (ret) + return ret; + + ret = do_write(ff, &(ff->ph->env.comp_level), sizeof(ff->ph->env.comp_level)); + if (ret) + return ret; + + ret = do_write(ff, &(ff->ph->env.comp_ratio), sizeof(ff->ph->env.comp_ratio)); + if (ret) + return ret; + + return do_write(ff, &(ff->ph->env.comp_mmap_len), sizeof(ff->ph->env.comp_mmap_len)); +} + +static int __write_pmu_caps(struct feat_fd *ff, struct perf_pmu *pmu, + bool write_pmu) +{ + struct perf_pmu_caps *caps = NULL; + int ret; + + ret = do_write(ff, &pmu->nr_caps, sizeof(pmu->nr_caps)); + if (ret < 0) + return ret; + + list_for_each_entry(caps, &pmu->caps, list) { + ret = do_write_string(ff, caps->name); + if (ret < 0) + return ret; + + ret = do_write_string(ff, caps->value); + if (ret < 0) + return ret; + } + + if (write_pmu) { + ret = do_write_string(ff, pmu->name); + if (ret < 0) + return ret; + } + + return ret; +} + +static int write_cpu_pmu_caps(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *cpu_pmu = perf_pmus__find_core_pmu(); + int ret; + + if (!cpu_pmu) + return -ENOENT; + + ret = perf_pmu__caps_parse(cpu_pmu); + if (ret < 0) + return ret; + + return __write_pmu_caps(ff, cpu_pmu, false); +} + +static int write_pmu_caps(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *pmu = NULL; + int nr_pmu = 0; + int ret; + + while ((pmu = perf_pmus__scan(pmu))) { + if (!strcmp(pmu->name, "cpu")) { + /* + * The "cpu" PMU is special and covered by + * HEADER_CPU_PMU_CAPS. Note, core PMUs are + * counted/written here for ARM, s390 and Intel hybrid. + */ + continue; + } + if (perf_pmu__caps_parse(pmu) <= 0) + continue; + nr_pmu++; + } + + ret = do_write(ff, &nr_pmu, sizeof(nr_pmu)); + if (ret < 0) + return ret; + + if (!nr_pmu) + return 0; + + /* + * Note older perf tools assume core PMUs come first, this is a property + * of perf_pmus__scan. + */ + pmu = NULL; + while ((pmu = perf_pmus__scan(pmu))) { + if (!strcmp(pmu->name, "cpu")) { + /* Skip as above. */ + continue; + } + if (perf_pmu__caps_parse(pmu) <= 0) + continue; + ret = __write_pmu_caps(ff, pmu, true); + if (ret < 0) + return ret; + } + return 0; +} + +struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains, u32 nr) +{ + char dname[DNAME_LEN], cpumask[MAX_NR_CPUS]; + struct domain_info *domain_info; + struct cpu_domain_map **cd_map; + char cpulist[MAX_NR_CPUS]; + char *line = NULL; + u32 cpu, domain; + u32 dcount = 0; + size_t len; + FILE *fp; + + fp = fopen("/proc/schedstat", "r"); + if (!fp) { + pr_err("Failed to open /proc/schedstat\n"); + return NULL; + } + + cd_map = zalloc(sizeof(*cd_map) * nr); + if (!cd_map) + goto out; + + while (getline(&line, &len, fp) > 0) { + int retval; + + if (strncmp(line, "version", 7) == 0) { + retval = sscanf(line, "version %d\n", schedstat_version); + if (retval != 1) + continue; + + } else if (strncmp(line, "cpu", 3) == 0) { + retval = sscanf(line, "cpu%u %*s", &cpu); + if (retval == 1) { + cd_map[cpu] = zalloc(sizeof(*cd_map[cpu])); + if (!cd_map[cpu]) + goto out_free_line; + cd_map[cpu]->cpu = cpu; + } else + continue; + + dcount = 0; + } else if (strncmp(line, "domain", 6) == 0) { + struct domain_info **temp_domains; + + dcount++; + temp_domains = realloc(cd_map[cpu]->domains, dcount * sizeof(domain_info)); + if (!temp_domains) + goto out_free_line; + else + cd_map[cpu]->domains = temp_domains; + + domain_info = zalloc(sizeof(*domain_info)); + if (!domain_info) + goto out_free_line; + + cd_map[cpu]->domains[dcount - 1] = domain_info; + + if (*schedstat_version >= 17) { + retval = sscanf(line, "domain%u %s %s %*s", &domain, dname, + cpumask); + if (retval != 3) + continue; + + domain_info->dname = strdup(dname); + if (!domain_info->dname) + goto out_free_line; + } else { + retval = sscanf(line, "domain%u %s %*s", &domain, cpumask); + if (retval != 2) + continue; + } + + domain_info->domain = domain; + if (domain > *max_sched_domains) + *max_sched_domains = domain; + + domain_info->cpumask = strdup(cpumask); + if (!domain_info->cpumask) + goto out_free_line; + + cpumask_to_cpulist(cpumask, cpulist); + domain_info->cpulist = strdup(cpulist); + if (!domain_info->cpulist) + goto out_free_line; + + cd_map[cpu]->nr_domains = dcount; + } + } + +out_free_line: + free(line); +out: + fclose(fp); + return cd_map; +} + +static int write_cpu_domain_info(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + u32 max_sched_domains = 0, schedstat_version = 0; + struct cpu_domain_map **cd_map; + u32 i, j, nr, ret; + + nr = cpu__max_present_cpu().cpu; + + cd_map = build_cpu_domain_map(&schedstat_version, &max_sched_domains, nr); + if (!cd_map) + return -1; + + ret = do_write(ff, &schedstat_version, sizeof(u32)); + if (ret < 0) + goto out; + + max_sched_domains += 1; + ret = do_write(ff, &max_sched_domains, sizeof(u32)); + if (ret < 0) + goto out; + + for (i = 0; i < nr; i++) { + if (!cd_map[i]) + continue; + + ret = do_write(ff, &cd_map[i]->cpu, sizeof(u32)); + if (ret < 0) + goto out; + + ret = do_write(ff, &cd_map[i]->nr_domains, sizeof(u32)); + if (ret < 0) + goto out; + + for (j = 0; j < cd_map[i]->nr_domains; j++) { + ret = do_write(ff, &cd_map[i]->domains[j]->domain, sizeof(u32)); + if (ret < 0) + goto out; + if (schedstat_version >= 17) { + ret = do_write_string(ff, cd_map[i]->domains[j]->dname); + if (ret < 0) + goto out; + } + + ret = do_write_string(ff, cd_map[i]->domains[j]->cpumask); + if (ret < 0) + goto out; + + ret = do_write_string(ff, cd_map[i]->domains[j]->cpulist); + if (ret < 0) + goto out; + } + } + +out: + free_cpu_domain_info(cd_map, schedstat_version, nr); + return ret; +} + +static void print_hostname(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname); +} + +static void print_osrelease(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# os release : %s\n", ff->ph->env.os_release); +} + +static void print_arch(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# arch : %s\n", ff->ph->env.arch); +} + +static void print_e_machine(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# e_machine : %u\n", ff->ph->env.e_machine); + fprintf(fp, "# e_flags : %u\n", ff->ph->env.e_flags); +} + +static void print_cpudesc(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# cpudesc : %s\n", ff->ph->env.cpu_desc); +} + +static void print_nrcpus(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# nrcpus online : %u\n", ff->ph->env.nr_cpus_online); + fprintf(fp, "# nrcpus avail : %u\n", ff->ph->env.nr_cpus_avail); +} + +static void print_version(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# perf version : %s\n", ff->ph->env.version); +} + +static void print_cmdline(struct feat_fd *ff, FILE *fp) +{ + int nr, i; + + nr = ff->ph->env.nr_cmdline; + + fprintf(fp, "# cmdline : "); + + for (i = 0; i < nr; i++) { + char *argv_i = strdup(ff->ph->env.cmdline_argv[i]); + if (!argv_i) { + fprintf(fp, "%s ", ff->ph->env.cmdline_argv[i]); + } else { + char *mem = argv_i; + do { + char *quote = strchr(argv_i, '\''); + if (!quote) + break; + *quote++ = '\0'; + fprintf(fp, "%s\\\'", argv_i); + argv_i = quote; + } while (1); + fprintf(fp, "%s ", argv_i); + free(mem); + } + } + fputc('\n', fp); +} + +static void print_cpu_topology(struct feat_fd *ff, FILE *fp) +{ + struct perf_header *ph = ff->ph; + int cpu_nr = ph->env.nr_cpus_avail; + int nr, i; + char *str; + + nr = ph->env.nr_sibling_cores; + str = ph->env.sibling_cores; + + for (i = 0; i < nr; i++) { + fprintf(fp, "# sibling sockets : %s\n", str); + str += strlen(str) + 1; + } + + if (ph->env.nr_sibling_dies) { + nr = ph->env.nr_sibling_dies; + str = ph->env.sibling_dies; + + for (i = 0; i < nr; i++) { + fprintf(fp, "# sibling dies : %s\n", str); + str += strlen(str) + 1; + } + } + + nr = ph->env.nr_sibling_threads; + str = ph->env.sibling_threads; + + for (i = 0; i < nr; i++) { + fprintf(fp, "# sibling threads : %s\n", str); + str += strlen(str) + 1; + } + + if (ph->env.nr_sibling_dies) { + if (ph->env.cpu != NULL) { + for (i = 0; i < cpu_nr; i++) + fprintf(fp, "# CPU %d: Core ID %d, " + "Die ID %d, Socket ID %d\n", + i, ph->env.cpu[i].core_id, + ph->env.cpu[i].die_id, + ph->env.cpu[i].socket_id); + } else + fprintf(fp, "# Core ID, Die ID and Socket ID " + "information is not available\n"); + } else { + if (ph->env.cpu != NULL) { + for (i = 0; i < cpu_nr; i++) + fprintf(fp, "# CPU %d: Core ID %d, " + "Socket ID %d\n", + i, ph->env.cpu[i].core_id, + ph->env.cpu[i].socket_id); + } else + fprintf(fp, "# Core ID and Socket ID " + "information is not available\n"); + } +} + +static void print_clockid(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# clockid frequency: %"PRIu64" MHz\n", + ff->ph->env.clock.clockid_res_ns * 1000); +} + +static void print_clock_data(struct feat_fd *ff, FILE *fp) +{ + struct timespec clockid_ns; + char tstr[64], date[64]; + struct timeval tod_ns; + clockid_t clockid; + struct tm ltime; + u64 ref; + + if (!ff->ph->env.clock.enabled) { + fprintf(fp, "# reference time disabled\n"); + return; + } + + /* Compute TOD time. */ + ref = ff->ph->env.clock.tod_ns; + tod_ns.tv_sec = ref / NSEC_PER_SEC; + ref -= tod_ns.tv_sec * NSEC_PER_SEC; + tod_ns.tv_usec = ref / NSEC_PER_USEC; + + /* Compute clockid time. */ + ref = ff->ph->env.clock.clockid_ns; + clockid_ns.tv_sec = ref / NSEC_PER_SEC; + ref -= clockid_ns.tv_sec * NSEC_PER_SEC; + clockid_ns.tv_nsec = ref; + + clockid = ff->ph->env.clock.clockid; + + if (localtime_r(&tod_ns.tv_sec, <ime) == NULL) + snprintf(tstr, sizeof(tstr), ""); + else { + strftime(date, sizeof(date), "%F %T", <ime); + scnprintf(tstr, sizeof(tstr), "%s.%06d", + date, (int) tod_ns.tv_usec); + } + + fprintf(fp, "# clockid: %s (%u)\n", clockid_name(clockid), clockid); + fprintf(fp, "# reference time: %s = %ld.%06d (TOD) = %ld.%09ld (%s)\n", + tstr, (long) tod_ns.tv_sec, (int) tod_ns.tv_usec, + (long) clockid_ns.tv_sec, clockid_ns.tv_nsec, + clockid_name(clockid)); +} + +static void print_hybrid_topology(struct feat_fd *ff, FILE *fp) +{ + int i; + struct hybrid_node *n; + + fprintf(fp, "# hybrid cpu system:\n"); + for (i = 0; i < ff->ph->env.nr_hybrid_nodes; i++) { + n = &ff->ph->env.hybrid_nodes[i]; + fprintf(fp, "# %s cpu list : %s\n", n->pmu_name, n->cpus); + } +} + +static void print_dir_format(struct feat_fd *ff, FILE *fp) +{ + struct perf_session *session; + struct perf_data *data; + + session = container_of(ff->ph, struct perf_session, header); + data = session->data; + + fprintf(fp, "# directory data version : %"PRIu64"\n", data->dir.version); +} + +static void print_bpf_prog_info(struct feat_fd *ff __maybe_unused, FILE *fp) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + + down_read(&env->bpf_progs.lock); + + root = &env->bpf_progs.infos; + next = rb_first(root); + + if (!next) + fprintf(fp, "# bpf_prog_info empty\n"); + + while (next) { + struct bpf_prog_info_node *node; + + node = rb_entry(next, struct bpf_prog_info_node, rb_node); + next = rb_next(&node->rb_node); + + __bpf_event__print_bpf_prog_info(node->info_linear, env, fp); + } + + up_read(&env->bpf_progs.lock); +#else + fprintf(fp, "# bpf_prog_info missing, no libbpf support\n"); +#endif // HAVE_LIBBPF_SUPPORT +} + +static void print_bpf_btf(struct feat_fd *ff __maybe_unused, FILE *fp) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct rb_root *root; + struct rb_node *next; + + down_read(&env->bpf_progs.lock); + + root = &env->bpf_progs.btfs; + next = rb_first(root); + + if (!next) + printf("# btf info empty\n"); + + while (next) { + struct btf_node *node; + + node = rb_entry(next, struct btf_node, rb_node); + next = rb_next(&node->rb_node); + fprintf(fp, "# btf info of id %u\n", node->id); + } + + up_read(&env->bpf_progs.lock); +#else + fprintf(fp, "# bpf btf data missing, no libbpf support\n"); +#endif // HAVE_LIBBPF_SUPPORT +} + +static void free_event_desc(struct evsel *events) +{ + struct evsel *evsel; + + if (!events) + return; + + for (evsel = events; evsel->core.attr.size; evsel++) { + zfree(&evsel->name); + zfree(&evsel->core.id); + } + + free(events); +} + +static bool perf_attr_check(struct perf_event_attr *attr) +{ + if (attr->__reserved_1 || attr->__reserved_2 || attr->__reserved_3) { + pr_warning("Reserved bits are set unexpectedly. " + "Please update perf tool.\n"); + return false; + } + + if (attr->sample_type & ~(PERF_SAMPLE_MAX-1)) { + pr_warning("Unknown sample type (0x%llx) is detected. " + "Please update perf tool.\n", + attr->sample_type); + return false; + } + + if (attr->read_format & ~(PERF_FORMAT_MAX-1)) { + pr_warning("Unknown read format (0x%llx) is detected. " + "Please update perf tool.\n", + attr->read_format); + return false; + } + + if ((attr->sample_type & PERF_SAMPLE_BRANCH_STACK) && + (attr->branch_sample_type & ~(PERF_SAMPLE_BRANCH_MAX-1))) { + pr_warning("Unknown branch sample type (0x%llx) is detected. " + "Please update perf tool.\n", + attr->branch_sample_type); + + return false; + } + + return true; +} + +static struct evsel *read_event_desc(struct feat_fd *ff) +{ + struct evsel *evsel, *events = NULL; + u64 *id; + void *buf = NULL; + u32 nre, sz, nr, i, j; + size_t msz; + + /* number of events */ + if (do_read_u32(ff, &nre)) + goto error; + + /* Size of each of the nre attributes. */ + if (do_read_u32(ff, &sz)) + goto error; + + /* + * Require at least one event with an attr no smaller than the + * first published struct, and reject sz values where + * sz + sizeof(u32) would overflow size_t (possible on 32-bit) + * or nre == UINT32_MAX where nre + 1 wraps to 0 in the calloc. + * + * The minimum section footprint per event is sz bytes for the + * attr plus a u32 for the id count, check that nre events fit. + */ + if (!nre || sz < PERF_ATTR_SIZE_VER0 || + sz > ff->size || (size_t)sz > SIZE_MAX - sizeof(u32) || + nre == UINT32_MAX || + nre > (ff->size - ff->offset) / (sz + sizeof(u32))) { + pr_err("Invalid HEADER_EVENT_DESC: nre=%u sz=%u (min %d)\n", + nre, sz, PERF_ATTR_SIZE_VER0); + goto error; + } + + /* buffer to hold on file attr struct */ + buf = malloc(sz); + if (!buf) + goto error; + + /* the last event terminates with evsel->core.attr.size == 0: */ + events = calloc(nre + 1, sizeof(*events)); + if (!events) + goto error; + + msz = sizeof(evsel->core.attr); + if (sz < msz) + msz = sz; + + for (i = 0, evsel = events; i < nre; evsel++, i++) { + struct perf_event_attr *attr = buf; + u32 attr_size; + + evsel->core.idx = i; + + /* + * must read entire on-file attr struct to + * sync up with layout. + */ + if (__do_read(ff, buf, sz)) + goto error; + + /* Reject before attr_swap to prevent OOB via bswap_safe() */ + attr_size = ff->ph->needs_swap ? bswap_32(attr->size) : attr->size; + /* ABI0: size == 0 means the producer didn't set it */ + if (!attr_size) { + attr_size = PERF_ATTR_SIZE_VER0; + /* + * Write back so free_event_desc() doesn't + * treat this event as the end-of-array sentinel + * (it iterates while attr.size != 0). + * + * Only for native — the swap path must NOT + * write native-endian VER0 here because + * perf_event__attr_swap() would re-swap it + * to 0x40000000, defeating bswap_safe() bounds. + * perf_event__attr_swap() has its own ABI0 + * fallback that sets VER0 after swapping. + */ + if (!ff->ph->needs_swap) + attr->size = attr_size; + } + if (attr_size < PERF_ATTR_SIZE_VER0 || attr_size > sz) { + pr_err("Event %d attr.size (%u) invalid (min: %d, max: %u)\n", + i, attr_size, PERF_ATTR_SIZE_VER0, sz); + goto error; + } + + if (ff->ph->needs_swap) + perf_event__attr_swap(buf); + + memcpy(&evsel->core.attr, buf, msz); + + if (!perf_attr_check(&evsel->core.attr)) + goto error; + + if (do_read_u32(ff, &nr)) + goto error; + + if (ff->ph->needs_swap) + evsel->needs_swap = true; + + evsel->name = do_read_string(ff); + if (!evsel->name) + goto error; + + if (!nr) + continue; + + /* Prevent oversized allocation from crafted nr */ + if (nr > (ff->size - ff->offset) / sizeof(*id)) { + pr_err("Event %d: id count %u exceeds remaining section\n", i, nr); + goto error; + } + + id = calloc(nr, sizeof(*id)); + if (!id) + goto error; + evsel->core.ids = nr; + evsel->core.id = id; + + for (j = 0 ; j < nr; j++) { + if (do_read_u64(ff, id)) + goto error; + id++; + } + } +out: + free(buf); + return events; +error: + free_event_desc(events); + events = NULL; + goto out; +} + +static int __desc_attr__fprintf(FILE *fp, const char *name, const char *val, + void *priv __maybe_unused) +{ + return fprintf(fp, ", %s = %s", name, val); +} + +static void print_event_desc(struct feat_fd *ff, FILE *fp) +{ + struct evsel *evsel, *events; + u32 j; + u64 *id; + + if (ff->events) + events = ff->events; + else + events = read_event_desc(ff); + + if (!events) { + fprintf(fp, "# event desc: not available or unable to read\n"); + return; + } + + for (evsel = events; evsel->core.attr.size; evsel++) { + fprintf(fp, "# event : name = %s, ", evsel->name); + + if (evsel->core.ids) { + fprintf(fp, ", id = {"); + for (j = 0, id = evsel->core.id; j < evsel->core.ids; j++, id++) { + if (j) + fputc(',', fp); + fprintf(fp, " %"PRIu64, *id); + } + fprintf(fp, " }"); + } + + perf_event_attr__fprintf(fp, &evsel->core.attr, __desc_attr__fprintf, NULL); + + fputc('\n', fp); + } + + free_event_desc(events); + ff->events = NULL; +} + +static void print_total_mem(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# total memory : %llu kB\n", ff->ph->env.total_mem); +} + +static void print_numa_topology(struct feat_fd *ff, FILE *fp) +{ + int i; + struct numa_node *n; + + for (i = 0; i < ff->ph->env.nr_numa_nodes; i++) { + n = &ff->ph->env.numa_nodes[i]; + + fprintf(fp, "# node%u meminfo : total = %"PRIu64" kB," + " free = %"PRIu64" kB\n", + n->node, n->mem_total, n->mem_free); + + fprintf(fp, "# node%u cpu list : ", n->node); + cpu_map__fprintf(n->map, fp); + } +} + +static void print_cpuid(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# cpuid : %s\n", ff->ph->env.cpuid); +} + +static void print_branch_stack(struct feat_fd *ff __maybe_unused, FILE *fp) +{ + fprintf(fp, "# contains samples with branch stack\n"); +} + +static void print_auxtrace(struct feat_fd *ff __maybe_unused, FILE *fp) +{ + fprintf(fp, "# contains AUX area data (e.g. instruction trace)\n"); +} + +static void print_stat(struct feat_fd *ff __maybe_unused, FILE *fp) +{ + fprintf(fp, "# contains stat data\n"); +} + +static void print_cache(struct feat_fd *ff, FILE *fp __maybe_unused) +{ + int i; + + fprintf(fp, "# CPU cache info:\n"); + for (i = 0; i < ff->ph->env.caches_cnt; i++) { + fprintf(fp, "# "); + cpu_cache_level__fprintf(fp, &ff->ph->env.caches[i]); + } +} + +static void print_cln_size(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# cacheline size: %u\n", ff->ph->env.cln_size); +} + +static void print_compressed(struct feat_fd *ff, FILE *fp) +{ + fprintf(fp, "# compressed : %s, level = %d, ratio = %d\n", + ff->ph->env.comp_type == PERF_COMP_ZSTD ? "Zstd" : "Unknown", + ff->ph->env.comp_level, ff->ph->env.comp_ratio); +} + +static void __print_pmu_caps(FILE *fp, int nr_caps, char **caps, char *pmu_name) +{ + const char *delimiter = ""; + int i; + + if (!nr_caps) { + fprintf(fp, "# %s pmu capabilities: not available\n", pmu_name); + return; + } + + fprintf(fp, "# %s pmu capabilities: ", pmu_name); + for (i = 0; i < nr_caps; i++) { + fprintf(fp, "%s%s", delimiter, caps[i]); + delimiter = ", "; + } + + fprintf(fp, "\n"); +} + +static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) +{ + __print_pmu_caps(fp, ff->ph->env.nr_cpu_pmu_caps, + ff->ph->env.cpu_pmu_caps, (char *)"cpu"); +} + +static void print_pmu_caps(struct feat_fd *ff, FILE *fp) +{ + struct perf_env *env = &ff->ph->env; + uint16_t e_machine = perf_env__e_machine(env, /*e_flags=*/NULL); + + for (int i = 0; i < env->nr_pmus_with_caps; i++) { + struct pmu_caps *pmu_caps = &env->pmu_caps[i]; + + __print_pmu_caps(fp, pmu_caps->nr_caps, pmu_caps->caps, + pmu_caps->pmu_name); + } + + if ((e_machine == EM_X86_64 || e_machine == EM_386) && + perf_env__has_pmu_mapping(env, "ibs_op")) { + char *max_precise = perf_env__find_pmu_cap(env, "cpu", "max_precise"); + + if (max_precise != NULL && atoi(max_precise) == 0) + fprintf(fp, "# AMD systems uses ibs_op// PMU for some precise events, e.g.: cycles:p, see the 'perf list' man page for further details.\n"); + } +} + +static void print_pmu_mappings(struct feat_fd *ff, FILE *fp) +{ + struct perf_env *env = &ff->ph->env; + const char *delimiter = "# pmu mappings: "; + char *str, *tmp; + u32 pmu_num; + u32 type; + + pmu_num = env->nr_pmu_mappings; + if (!pmu_num) { + fprintf(fp, "# pmu mappings: not available\n"); + return; + } + + str = env->pmu_mappings; + + while (pmu_num) { + type = strtoul(str, &tmp, 0); + if (*tmp != ':') + goto error; + + str = tmp + 1; + fprintf(fp, "%s%s = %" PRIu32, delimiter, str, type); + + delimiter = ", "; + str += strlen(str) + 1; + pmu_num--; + } + + fprintf(fp, "\n"); + + if (!pmu_num) + return; +error: + fprintf(fp, "# pmu mappings: unable to read\n"); +} + +static void print_group_desc(struct feat_fd *ff, FILE *fp) +{ + struct perf_session *session; + struct evsel *evsel; + u32 nr = 0; + + session = container_of(ff->ph, struct perf_session, header); + + evlist__for_each_entry(session->evlist, evsel) { + if (evsel__is_group_leader(evsel) && evsel->core.nr_members > 1) { + fprintf(fp, "# group: %s{%s", evsel->group_name ?: "", evsel__name(evsel)); + + nr = evsel->core.nr_members - 1; + } else if (nr) { + fprintf(fp, ",%s", evsel__name(evsel)); + + if (--nr == 0) + fprintf(fp, "}\n"); + } + } +} + +static void print_sample_time(struct feat_fd *ff, FILE *fp) +{ + struct perf_session *session; + char time_buf[32]; + double d; + + session = container_of(ff->ph, struct perf_session, header); + + timestamp__scnprintf_usec(session->evlist->first_sample_time, + time_buf, sizeof(time_buf)); + fprintf(fp, "# time of first sample : %s\n", time_buf); + + timestamp__scnprintf_usec(session->evlist->last_sample_time, + time_buf, sizeof(time_buf)); + fprintf(fp, "# time of last sample : %s\n", time_buf); + + d = (double)(session->evlist->last_sample_time - + session->evlist->first_sample_time) / NSEC_PER_MSEC; + + fprintf(fp, "# sample duration : %10.3f ms\n", d); +} + +static void memory_node__fprintf(struct memory_node *n, + unsigned long long bsize, FILE *fp) +{ + char buf_map[100], buf_size[50]; + unsigned long long size; + + size = bsize * bitmap_weight(n->set, n->size); + unit_number__scnprintf(buf_size, 50, size); + + bitmap_scnprintf(n->set, n->size, buf_map, 100); + fprintf(fp, "# %3" PRIu64 " [%s]: %s\n", n->node, buf_size, buf_map); +} + +static void print_mem_topology(struct feat_fd *ff, FILE *fp) +{ + struct perf_env *env = &ff->ph->env; + struct memory_node *nodes; + int i, nr; + + nodes = env->memory_nodes; + nr = env->nr_memory_nodes; + + fprintf(fp, "# memory nodes (nr %d, block size 0x%llx):\n", + nr, env->memory_bsize); + + for (i = 0; i < nr; i++) { + memory_node__fprintf(&nodes[i], env->memory_bsize, fp); + } +} + +static void print_cpu_domain_info(struct feat_fd *ff, FILE *fp) +{ + struct cpu_domain_map **cd_map = ff->ph->env.cpu_domain; + u32 nr = ff->ph->env.nr_cpus_avail; + struct domain_info *d_info; + u32 i, j; + + fprintf(fp, "# schedstat version : %u\n", ff->ph->env.schedstat_version); + fprintf(fp, "# Maximum sched domains : %u\n", ff->ph->env.max_sched_domains); + + for (i = 0; i < nr; i++) { + if (!cd_map[i]) + continue; + + fprintf(fp, "# cpu : %u\n", cd_map[i]->cpu); + fprintf(fp, "# nr_domains : %u\n", cd_map[i]->nr_domains); + + for (j = 0; j < cd_map[i]->nr_domains; j++) { + d_info = cd_map[i]->domains[j]; + if (!d_info) + continue; + + fprintf(fp, "# Domain : %u\n", d_info->domain); + + if (ff->ph->env.schedstat_version >= 17) + fprintf(fp, "# Domain name : %s\n", d_info->dname); + + fprintf(fp, "# Domain cpu map : %s\n", d_info->cpumask); + fprintf(fp, "# Domain cpu list : %s\n", d_info->cpulist); + } + } +} + +static int __event_process_build_id(struct perf_record_header_build_id *bev, + char *filename, + struct perf_session *session) +{ + int err = -1; + struct machine *machine; + u16 cpumode; + struct dso *dso; + enum dso_space_type dso_space; + + machine = perf_session__findnew_machine(session, bev->pid); + if (!machine) + goto out; + + cpumode = bev->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + + switch (cpumode) { + case PERF_RECORD_MISC_KERNEL: + dso_space = DSO_SPACE__KERNEL; + break; + case PERF_RECORD_MISC_GUEST_KERNEL: + dso_space = DSO_SPACE__KERNEL_GUEST; + break; + case PERF_RECORD_MISC_USER: + case PERF_RECORD_MISC_GUEST_USER: + dso_space = DSO_SPACE__USER; + break; + default: + goto out; + } + + dso = machine__findnew_dso(machine, filename); + if (dso != NULL) { + char sbuild_id[SBUILD_ID_SIZE]; + struct build_id bid; + size_t size = BUILD_ID_SIZE; + + if (bev->header.misc & PERF_RECORD_MISC_BUILD_ID_SIZE) + size = bev->size; + + build_id__init(&bid, bev->data, size); + dso__set_build_id(dso, &bid); + dso__set_header_build_id(dso, true); + + if (dso_space != DSO_SPACE__USER) { + struct kmod_path m = { .name = NULL, }; + + if (!kmod_path__parse_name(&m, filename) && m.kmod) + dso__set_module_info(dso, &m, machine); + + dso__set_kernel(dso, dso_space); + free(m.name); + } + + build_id__snprintf(dso__bid(dso), sbuild_id, sizeof(sbuild_id)); + pr_debug("build id event received for %s: %s [%zu]\n", + dso__long_name(dso), sbuild_id, size); + dso__put(dso); + } + + err = 0; +out: + return err; +} + +static int perf_header__read_build_ids_abi_quirk(struct perf_header *header, + int input, u64 offset, u64 size) +{ + struct perf_session *session = container_of(header, struct perf_session, header); + struct { + struct perf_event_header header; + u8 build_id[PERF_ALIGN(BUILD_ID_SIZE, sizeof(u64))]; + char filename[0]; + } old_bev; + struct perf_record_header_build_id bev; + char filename[PATH_MAX]; + u64 limit; + + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; + + while (offset < limit) { + ssize_t len; + + if (readn(input, &old_bev, sizeof(old_bev)) != sizeof(old_bev)) + return -1; + + if (header->needs_swap) + perf_event_header__bswap(&old_bev.header); + + /* size == 0 loops forever; size > remaining reads past section */ + if (old_bev.header.size == 0 || old_bev.header.size > limit - offset) + return -1; + + len = old_bev.header.size - sizeof(old_bev); + if (len < 0 || len >= PATH_MAX) { + pr_warning("invalid build_id filename length %zd\n", len); + return -1; + } + + if (readn(input, filename, len) != len) + return -1; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; + + bev.header = old_bev.header; + + /* + * As the pid is the missing value, we need to fill + * it properly. The header.misc value give us nice hint. + */ + bev.pid = HOST_KERNEL_ID; + if (bev.header.misc == PERF_RECORD_MISC_GUEST_USER || + bev.header.misc == PERF_RECORD_MISC_GUEST_KERNEL) + bev.pid = DEFAULT_GUEST_KERNEL_ID; + + memcpy(bev.build_id, old_bev.build_id, sizeof(bev.build_id)); + __event_process_build_id(&bev, filename, session); + + offset += bev.header.size; + } + + return 0; +} + +static int perf_header__read_build_ids(struct perf_header *header, + int input, u64 offset, u64 size) +{ + struct perf_session *session = container_of(header, struct perf_session, header); + struct perf_record_header_build_id bev; + char filename[PATH_MAX]; + u64 limit, orig_offset = offset; + int err = -1; + + /* Prevent offset + size from wrapping past ULLONG_MAX */ + if (size > ULLONG_MAX - offset) + return -1; + + limit = offset + size; + + while (offset < limit) { + ssize_t len; + + if (readn(input, &bev, sizeof(bev)) != sizeof(bev)) + goto out; + + if (header->needs_swap) { + perf_event_header__bswap(&bev.header); + bev.pid = bswap_32(bev.pid); + } + + /* + * size == 0 would loop forever (offset never advances); + * size > remaining would read past the section boundary. + */ + if (bev.header.size == 0 || bev.header.size > limit - offset) + goto out; + + len = bev.header.size - sizeof(bev); + if (len < 0 || len >= PATH_MAX) { + pr_warning("invalid build_id filename length %zd\n", len); + goto out; + } + + if (readn(input, filename, len) != len) + goto out; + /* + * The file data may lack a null terminator, which could + * indicate a corrupt or crafted perf.data file. Ensure + * filename is always a valid C string before passing it + * to functions like machine__findnew_dso(). + */ + filename[len] = '\0'; + /* + * The a1645ce1 changeset: + * + * "perf: 'perf kvm' tool for monitoring guest performance from host" + * + * Added a field to struct perf_record_header_build_id that broke the file + * format. + * + * Since the kernel build-id is the first entry, process the + * table using the old format if the well known + * '[kernel.kallsyms]' string for the kernel build-id has the + * first 4 characters chopped off (where the pid_t sits). + */ + /* Guard short filenames against memcmp reading past the buffer */ + if (len >= (ssize_t)sizeof("nel.kallsyms]") - 1 && + memcmp(filename, "nel.kallsyms]", sizeof("nel.kallsyms]") - 1) == 0) { + if (lseek(input, orig_offset, SEEK_SET) == (off_t)-1) + return -1; + return perf_header__read_build_ids_abi_quirk(header, input, offset, size); + } + + __event_process_build_id(&bev, filename, session); + + offset += bev.header.size; + } + err = 0; +out: + return err; +} + +/* Macro for features that simply need to read and store a string. */ +#define FEAT_PROCESS_STR_FUN(__feat, __feat_env) \ +static int process_##__feat(struct feat_fd *ff, void *data __maybe_unused) \ +{\ + free(ff->ph->env.__feat_env); \ + ff->ph->env.__feat_env = do_read_string(ff); \ + return ff->ph->env.__feat_env ? 0 : -ENOMEM; \ +} + +FEAT_PROCESS_STR_FUN(hostname, hostname); +FEAT_PROCESS_STR_FUN(osrelease, os_release); +FEAT_PROCESS_STR_FUN(version, version); +FEAT_PROCESS_STR_FUN(cpudesc, cpu_desc); +FEAT_PROCESS_STR_FUN(cpuid, cpuid); + +static int process_arch(struct feat_fd *ff, void *data __maybe_unused) +{ + free(ff->ph->env.arch); + ff->ph->env.arch = do_read_string(ff); + if (!ff->ph->env.arch) + return -ENOMEM; + return 0; +} + +static int process_e_machine(struct feat_fd *ff, void *data __maybe_unused) +{ + int ret; + + ret = do_read_u32(ff, &ff->ph->env.e_machine); + if (ret) + return ret; + + return do_read_u32(ff, &ff->ph->env.e_flags); +} + +static int process_tracing_data(struct feat_fd *ff __maybe_unused, void *data __maybe_unused) +{ +#ifdef HAVE_LIBTRACEEVENT + ssize_t ret = trace_report(ff->fd, data, false); + + return ret < 0 ? -1 : 0; +#else + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("Tracing data present but libtraceevent not available, skipping.\n"); + return 0; +#endif +} + +static int process_build_id(struct feat_fd *ff, void *data __maybe_unused) +{ + /* lseek fails in pipe mode — fall back to ff->offset */ + off_t offset = lseek(ff->fd, 0, SEEK_CUR); + + if (offset == (off_t)-1) + offset = ff->offset; + + if (perf_header__read_build_ids(ff->ph, ff->fd, offset, ff->size)) + pr_debug("Failed to read buildids, continuing...\n"); + return 0; +} + +static int process_nrcpus(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + int ret; + u32 nr_cpus_avail, nr_cpus_online; + + ret = do_read_u32(ff, &nr_cpus_avail); + if (ret) + return ret; + + ret = do_read_u32(ff, &nr_cpus_online); + if (ret) + return ret; + + /* + * Cap at 1M CPUs — generous for any real system but prevents + * stack overflow from VLA allocations sized by nr_cpus_avail + * (e.g. DECLARE_BITMAP in builtin-c2c.c node_entry()). + */ + if (nr_cpus_avail > (1U << 20)) { + pr_err("Invalid HEADER_NRCPUS: nr_cpus_avail (%u) exceeds maximum (%u)\n", + nr_cpus_avail, 1U << 20); + return -1; + } + + if (nr_cpus_online > nr_cpus_avail) { + pr_err("Invalid HEADER_NRCPUS: nr_cpus_online (%u) > nr_cpus_avail (%u)\n", + nr_cpus_online, nr_cpus_avail); + return -1; + } + + env->nr_cpus_avail = (int)nr_cpus_avail; + env->nr_cpus_online = (int)nr_cpus_online; + return 0; +} + +static int process_total_mem(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + u64 total_mem; + int ret; + + ret = do_read_u64(ff, &total_mem); + if (ret) + return -1; + env->total_mem = (unsigned long long)total_mem; + return 0; +} + +static struct evsel *evlist__find_by_index(struct evlist *evlist, int idx) +{ + struct evsel *evsel; + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.idx == idx) + return evsel; + } + + return NULL; +} + +static void evlist__set_event_name(struct evlist *evlist, struct evsel *event) +{ + struct evsel *evsel; + + if (!event->name) + return; + + evsel = evlist__find_by_index(evlist, event->core.idx); + if (!evsel) + return; + + if (evsel->name) + return; + + evsel->name = strdup(event->name); +} + +static int +process_event_desc(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_session *session; + struct evsel *evsel, *events = read_event_desc(ff); + + if (!events) + return 0; + + session = container_of(ff->ph, struct perf_session, header); + + if (session->data->is_pipe) { + /* Save events for reading later by print_event_desc, + * since they can't be read again in pipe mode. */ + ff->events = events; + } + + for (evsel = events; evsel->core.attr.size; evsel++) + evlist__set_event_name(session->evlist, evsel); + + if (!session->data->is_pipe) + free_event_desc(events); + + return 0; +} + +/* + * Some arbitrary max for the number of command line arguments, + * Wildcards can expand and end up with tons of command line args. + */ +#define MAX_CMDLINE_NR 1048576 + +static int process_cmdline(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + char *str, *cmdline = NULL, **argv = NULL; + u32 nr, i, len = 0; + + if (do_read_u32(ff, &nr)) + return -1; + + if (nr > MAX_CMDLINE_NR) + return -1; + + env->nr_cmdline = nr; + + cmdline = zalloc(ff->size + nr + 1); + if (!cmdline) + return -1; + + argv = calloc(nr + 1, sizeof(char *)); + if (!argv) + goto error; + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + argv[i] = cmdline + len; + memcpy(argv[i], str, strlen(str) + 1); + len += strlen(str) + 1; + free(str); + } + env->cmdline = cmdline; + env->cmdline_argv = (const char **) argv; + return 0; + +error: + free(argv); + free(cmdline); + return -1; +} + +static int process_cpu_topology(struct feat_fd *ff, void *data __maybe_unused) +{ + u32 nr, i; + char *str = NULL; + struct strbuf sb; + struct perf_env *env = &ff->ph->env; + int cpu_nr = env->nr_cpus_avail; + u64 size = 0; + + if (cpu_nr == 0) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: missing HEADER_NRCPUS\n"); + return -1; + } + + env->cpu = calloc(cpu_nr, sizeof(*env->cpu)); + if (!env->cpu) + return -1; + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + if (nr > (u32)cpu_nr) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: nr_sibling_cores (%u) > nr_cpus_avail (%d)\n", + nr, cpu_nr); + goto free_cpu; + } + + env->nr_sibling_cores = nr; + size += sizeof(u32); + if (strbuf_init(&sb, 128) < 0) + goto free_cpu; + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + /* include a NULL character at the end */ + if (strbuf_add(&sb, str, strlen(str) + 1) < 0) + goto error; + size += string_size(str); + zfree(&str); + } + env->sibling_cores = strbuf_detach(&sb, NULL); + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + if (nr > (u32)cpu_nr) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: nr_sibling_threads (%u) > nr_cpus_avail (%d)\n", + nr, cpu_nr); + goto free_cpu; + } + + env->nr_sibling_threads = nr; + size += sizeof(u32); + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + /* include a NULL character at the end */ + if (strbuf_add(&sb, str, strlen(str) + 1) < 0) + goto error; + size += string_size(str); + zfree(&str); + } + env->sibling_threads = strbuf_detach(&sb, NULL); + + /* + * The header may be from old perf, + * which doesn't include core id and socket id information. + */ + if (ff->size <= size) { + zfree(&env->cpu); + return 0; + } + + for (i = 0; i < (u32)cpu_nr; i++) { + if (do_read_u32(ff, &nr)) + goto free_cpu; + + env->cpu[i].core_id = nr; + size += sizeof(u32); + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + env->cpu[i].socket_id = nr; + size += sizeof(u32); + } + + /* + * The header may be from old perf, + * which doesn't include die information. + */ + if (ff->size <= size) + return 0; + + if (do_read_u32(ff, &nr)) + goto free_cpu; + + if (nr > (u32)cpu_nr) { + pr_err("Invalid HEADER_CPU_TOPOLOGY: nr_sibling_dies (%u) > nr_cpus_avail (%d)\n", + nr, cpu_nr); + goto free_cpu; + } + + env->nr_sibling_dies = nr; + size += sizeof(u32); + + for (i = 0; i < nr; i++) { + str = do_read_string(ff); + if (!str) + goto error; + + /* include a NULL character at the end */ + if (strbuf_add(&sb, str, strlen(str) + 1) < 0) + goto error; + size += string_size(str); + zfree(&str); + } + env->sibling_dies = strbuf_detach(&sb, NULL); + + for (i = 0; i < (u32)cpu_nr; i++) { + if (do_read_u32(ff, &nr)) + goto free_cpu; + + env->cpu[i].die_id = nr; + } + + return 0; + +error: + strbuf_release(&sb); + zfree(&str); +free_cpu: + zfree(&env->cpu); + return -1; +} + +static int process_numa_topology(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct numa_node *nodes, *n; + u32 nr, i; + char *str; + + /* nr nodes */ + if (do_read_u32(ff, &nr)) + return -1; + + if (nr > MAX_NUMA_NODES) { + pr_err("Invalid HEADER_NUMA_TOPOLOGY: nr_nodes (%u) > %u\n", + nr, MAX_NUMA_NODES); + return -1; + } + + if (ff->size < sizeof(u32) + nr * (sizeof(u32) + 2 * sizeof(u64))) { + pr_err("Invalid HEADER_NUMA_TOPOLOGY: section too small (%zu) for %u nodes\n", + ff->size, nr); + return -1; + } + + nodes = calloc(nr, sizeof(*nodes)); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + n = &nodes[i]; + + /* node number */ + if (do_read_u32(ff, &n->node)) + goto error; + + if (do_read_u64(ff, &n->mem_total)) + goto error; + + if (do_read_u64(ff, &n->mem_free)) + goto error; + + str = do_read_string(ff); + if (!str) + goto error; + + n->map = perf_cpu_map__new(str); + free(str); + if (!n->map) + goto error; + } + env->nr_numa_nodes = nr; + env->numa_nodes = nodes; + return 0; + +error: + free(nodes); + return -1; +} + +static int process_pmu_mappings(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + char *name; + u32 pmu_num; + u32 type; + struct strbuf sb; + + if (do_read_u32(ff, &pmu_num)) + return -1; + + if (!pmu_num) { + pr_debug("pmu mappings not available\n"); + return 0; + } + + if (pmu_num > MAX_PMU_MAPPINGS) { + pr_err("Invalid HEADER_PMU_MAPPINGS: pmu_num (%u) > %u\n", + pmu_num, MAX_PMU_MAPPINGS); + return -1; + } + + if (ff->size < sizeof(u32) + pmu_num * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_PMU_MAPPINGS: section too small (%zu) for %u PMUs\n", + ff->size, pmu_num); + return -1; + } + + env->nr_pmu_mappings = pmu_num; + if (strbuf_init(&sb, 128) < 0) + return -1; + + while (pmu_num) { + if (do_read_u32(ff, &type)) + goto error; + + name = do_read_string(ff); + if (!name) + goto error; + + if (strbuf_addf(&sb, "%u:%s", type, name) < 0) + goto error; + /* include a NULL character at the end */ + if (strbuf_add(&sb, "", 1) < 0) + goto error; + + if (!strcmp(name, "msr")) + env->msr_pmu_type = type; + + free(name); + pmu_num--; + } + /* AMD may set it by evlist__has_amd_ibs() from perf_session__new() */ + free(env->pmu_mappings); + env->pmu_mappings = strbuf_detach(&sb, NULL); + return 0; + +error: + strbuf_release(&sb); + return -1; +} + +static int process_group_desc(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + size_t ret = -1; + u32 i, nr, nr_groups; + struct perf_session *session; + struct evsel *evsel, *leader = NULL; + struct group_desc { + char *name; + u32 leader_idx; + u32 nr_members; + } *desc; + + if (do_read_u32(ff, &nr_groups)) + return -1; + + if (!nr_groups) { + pr_debug("group desc not available\n"); + return 0; + } + + if (nr_groups > MAX_GROUP_DESC) { + pr_err("Invalid HEADER_GROUP_DESC: nr_groups (%u) > %u\n", + nr_groups, MAX_GROUP_DESC); + return -1; + } + + if (ff->size < sizeof(u32) + nr_groups * 3 * sizeof(u32)) { + pr_err("Invalid HEADER_GROUP_DESC: section too small (%zu) for %u groups\n", + ff->size, nr_groups); + return -1; + } + + env->nr_groups = nr_groups; + + desc = calloc(nr_groups, sizeof(*desc)); + if (!desc) + return -1; + + for (i = 0; i < nr_groups; i++) { + desc[i].name = do_read_string(ff); + if (!desc[i].name) + goto out_free; + + if (do_read_u32(ff, &desc[i].leader_idx)) + goto out_free; + + if (do_read_u32(ff, &desc[i].nr_members)) + goto out_free; + } + + /* + * Rebuild group relationship based on the group_desc + */ + session = container_of(ff->ph, struct perf_session, header); + + i = nr = 0; + evlist__for_each_entry(session->evlist, evsel) { + if (i < nr_groups && evsel->core.idx == (int) desc[i].leader_idx) { + evsel__set_leader(evsel, evsel); + /* {anon_group} is a dummy name */ + if (strcmp(desc[i].name, "{anon_group}")) { + evsel->group_name = desc[i].name; + desc[i].name = NULL; + } + evsel->core.nr_members = desc[i].nr_members; + + if (i >= nr_groups || nr > 0) { + pr_debug("invalid group desc\n"); + goto out_free; + } + + leader = evsel; + nr = evsel->core.nr_members - 1; + i++; + } else if (nr) { + /* This is a group member */ + evsel__set_leader(evsel, leader); + + nr--; + } + } + + if (i != nr_groups || nr != 0) { + pr_debug("invalid group desc\n"); + goto out_free; + } + + ret = 0; +out_free: + for (i = 0; i < nr_groups; i++) + zfree(&desc[i].name); + free(desc); + + return ret; +} + +static int process_auxtrace(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_session *session; + int err; + + session = container_of(ff->ph, struct perf_session, header); + + err = auxtrace_index__process(ff->fd, ff->size, session, + ff->ph->needs_swap); + if (err < 0) + pr_err("Failed to process auxtrace index\n"); + return err; +} + +static int process_cache(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct cpu_cache_level *caches; + u32 cnt, i, version; + + if (do_read_u32(ff, &version)) + return -1; + + if (version != 1) + return -1; + + if (do_read_u32(ff, &cnt)) + return -1; + + if (cnt > MAX_CACHE_ENTRIES) { + pr_err("Invalid HEADER_CACHE: cnt (%u) > %u\n", + cnt, MAX_CACHE_ENTRIES); + return -1; + } + + if (ff->size < 2 * sizeof(u32) + cnt * 7 * sizeof(u32)) { + pr_err("Invalid HEADER_CACHE: section too small (%zu) for %u entries\n", + ff->size, cnt); + return -1; + } + + caches = calloc(cnt, sizeof(*caches)); + if (!caches) + return -1; + + for (i = 0; i < cnt; i++) { + struct cpu_cache_level *c = &caches[i]; + + #define _R(v) \ + if (do_read_u32(ff, &c->v)) \ + goto out_free_caches; \ + + _R(level) + _R(line_size) + _R(sets) + _R(ways) + #undef _R + + #define _R(v) \ + c->v = do_read_string(ff); \ + if (!c->v) \ + goto out_free_caches; \ + + _R(type) + _R(size) + _R(map) + #undef _R + } + + env->caches = caches; + env->caches_cnt = cnt; + return 0; +out_free_caches: + for (i = 0; i < cnt; i++) { + free(caches[i].type); + free(caches[i].size); + free(caches[i].map); + } + free(caches); + return -1; +} + +static int process_cln_size(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + + if (do_read_u32(ff, &env->cln_size)) + return -1; + + return 0; +} + +static int process_sample_time(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_session *session; + u64 first_sample_time, last_sample_time; + int ret; + + session = container_of(ff->ph, struct perf_session, header); + + ret = do_read_u64(ff, &first_sample_time); + if (ret) + return -1; + + ret = do_read_u64(ff, &last_sample_time); + if (ret) + return -1; + + session->evlist->first_sample_time = first_sample_time; + session->evlist->last_sample_time = last_sample_time; + return 0; +} + +static int process_mem_topology(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct memory_node *nodes; + u64 version, i, nr, bsize; + int ret = -1; + + if (do_read_u64(ff, &version)) + return -1; + + if (version != 1) + return -1; + + if (do_read_u64(ff, &bsize)) + return -1; + + if (do_read_u64(ff, &nr)) + return -1; + + if (nr > MAX_NUMA_NODES) { + pr_err("Invalid HEADER_MEM_TOPOLOGY: nr_nodes (%llu) > %u\n", + (unsigned long long)nr, MAX_NUMA_NODES); + return -1; + } + + /* Per node: node_id(u64) + mem_size(u64) + bitmap_nr_bits(u64) */ + if (ff->size < 3 * sizeof(u64) + nr * 3 * sizeof(u64)) { + pr_err("Invalid HEADER_MEM_TOPOLOGY: section too small (%zu) for %llu nodes\n", + ff->size, (unsigned long long)nr); + return -1; + } + + nodes = calloc(nr, sizeof(*nodes)); + if (!nodes) + return -1; + + for (i = 0; i < nr; i++) { + struct memory_node n; + + #define _R(v) \ + if (do_read_u64(ff, &n.v)) \ + goto out; \ + + _R(node) + _R(size) + + #undef _R + + if (do_read_bitmap(ff, &n.set, &n.size)) + goto out; + + nodes[i] = n; + } + + env->memory_bsize = bsize; + env->memory_nodes = nodes; + env->nr_memory_nodes = nr; + ret = 0; + +out: + if (ret) + memory_node__delete_nodes(nodes, nr); + return ret; +} + +static int process_clockid(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + + if (do_read_u64(ff, &env->clock.clockid_res_ns)) + return -1; + + return 0; +} + +static int process_clock_data(struct feat_fd *ff, + void *_data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + u32 data32; + u64 data64; + + /* version */ + if (do_read_u32(ff, &data32)) + return -1; + + if (data32 != 1) + return -1; + + /* clockid */ + if (do_read_u32(ff, &data32)) + return -1; + + env->clock.clockid = data32; + + /* TOD ref time */ + if (do_read_u64(ff, &data64)) + return -1; + + env->clock.tod_ns = data64; + + /* clockid ref time */ + if (do_read_u64(ff, &data64)) + return -1; + + env->clock.clockid_ns = data64; + env->clock.enabled = true; + return 0; +} + +static int process_hybrid_topology(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct hybrid_node *nodes, *n; + u32 nr, i; + + /* nr nodes */ + if (do_read_u32(ff, &nr)) + return -1; + + if (nr > MAX_PMU_MAPPINGS) { + pr_err("Invalid HEADER_HYBRID_TOPOLOGY: nr_nodes (%u) > %u\n", + nr, MAX_PMU_MAPPINGS); + return -1; + } + + if (ff->size < sizeof(u32) + nr * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_HYBRID_TOPOLOGY: section too small (%zu) for %u nodes\n", + ff->size, nr); + return -1; + } + + nodes = calloc(nr, sizeof(*nodes)); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + n = &nodes[i]; + + n->pmu_name = do_read_string(ff); + if (!n->pmu_name) + goto error; + + n->cpus = do_read_string(ff); + if (!n->cpus) + goto error; + } + + env->nr_hybrid_nodes = nr; + env->hybrid_nodes = nodes; + return 0; + +error: + for (i = 0; i < nr; i++) { + free(nodes[i].pmu_name); + free(nodes[i].cpus); + } + + free(nodes); + return -1; +} + +static int process_dir_format(struct feat_fd *ff, + void *_data __maybe_unused) +{ + struct perf_session *session; + struct perf_data *data; + + session = container_of(ff->ph, struct perf_session, header); + data = session->data; + + if (WARN_ON(!perf_data__is_dir(data))) + return -1; + + return do_read_u64(ff, &data->dir.version); +} + +static int process_bpf_prog_info(struct feat_fd *ff __maybe_unused, void *data __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct bpf_prog_info_node *info_node; + struct perf_env *env = &ff->ph->env; + struct perf_bpil *info_linear; + u32 count, i; + int err = -1; + + if (ff->ph->needs_swap) { + pr_warning("interpreting bpf_prog_info from systems with endianness is not yet supported\n"); + return 0; + } + + if (do_read_u32(ff, &count)) + return -1; + + if (count > MAX_BPF_PROGS) { + pr_err("Invalid HEADER_BPF_PROG_INFO: count (%u) > %u\n", + count, MAX_BPF_PROGS); + return -1; + } + + if (ff->size < sizeof(u32) + count * (2 * sizeof(u32) + sizeof(u64))) { + pr_err("Invalid HEADER_BPF_PROG_INFO: section too small (%zu) for %u entries\n", + ff->size, count); + return -1; + } + + down_write(&env->bpf_progs.lock); + + for (i = 0; i < count; ++i) { + u32 info_len, data_len; + + info_linear = NULL; + info_node = NULL; + if (do_read_u32(ff, &info_len)) + goto out; + if (do_read_u32(ff, &data_len)) + goto out; + + if (info_len > sizeof(struct bpf_prog_info)) { + pr_warning("detected invalid bpf_prog_info\n"); + goto out; + } + + if (data_len > MAX_BPF_DATA_LEN) { + pr_warning("Invalid HEADER_BPF_PROG_INFO: data_len (%u) too large\n", + data_len); + goto out; + } + + info_linear = malloc(sizeof(struct perf_bpil) + + data_len); + if (!info_linear) + goto out; + info_linear->info_len = sizeof(struct bpf_prog_info); + info_linear->data_len = data_len; + if (do_read_u64(ff, (u64 *)(&info_linear->arrays))) + goto out; + if (__do_read(ff, &info_linear->info, info_len)) + goto out; + if (info_len < sizeof(struct bpf_prog_info)) + memset(((void *)(&info_linear->info)) + info_len, 0, + sizeof(struct bpf_prog_info) - info_len); + + if (__do_read(ff, info_linear->data, data_len)) + goto out; + + info_node = malloc(sizeof(struct bpf_prog_info_node)); + if (!info_node) + goto out; + + /* after reading from file, translate offset to address */ + bpil_offs_to_addr(info_linear); + info_node->info_linear = info_linear; + info_node->metadata = NULL; + if (!__perf_env__insert_bpf_prog_info(env, info_node)) { + free(info_linear); + free(info_node); + } + } + + up_write(&env->bpf_progs.lock); + return 0; +out: + free(info_linear); + free(info_node); + up_write(&env->bpf_progs.lock); + return err; +#else + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BPF prog info present but libbpf not available, skipping.\n"); + return 0; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int process_bpf_btf(struct feat_fd *ff __maybe_unused, void *data __maybe_unused) +{ +#ifdef HAVE_LIBBPF_SUPPORT + struct perf_env *env = &ff->ph->env; + struct btf_node *node = NULL; + u32 count, i; + int err = -1; + + if (ff->ph->needs_swap) { + pr_warning("interpreting btf from systems with endianness is not yet supported\n"); + return 0; + } + + if (do_read_u32(ff, &count)) + return -1; + + if (count > MAX_BPF_PROGS) { + pr_err("bpf btf count %u too large (max %u)\n", count, MAX_BPF_PROGS); + return -1; + } + + if (ff->size < sizeof(u32) + count * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_BPF_BTF: section too small (%zu) for %u entries\n", + ff->size, count); + return -1; + } + + down_write(&env->bpf_progs.lock); + + for (i = 0; i < count; ++i) { + u32 id, data_size; + + if (do_read_u32(ff, &id)) + goto out; + if (do_read_u32(ff, &data_size)) + goto out; + + if (data_size > MAX_BPF_DATA_LEN) { + pr_err("bpf btf data size %u too large (max %u)\n", + data_size, MAX_BPF_DATA_LEN); + goto out; + } + + node = malloc(sizeof(struct btf_node) + data_size); + if (!node) + goto out; + + node->id = id; + node->data_size = data_size; + + if (__do_read(ff, node->data, data_size)) + goto out; + + if (!__perf_env__insert_btf(env, node)) + free(node); + node = NULL; + } + + err = 0; +out: + up_write(&env->bpf_progs.lock); + free(node); + return err; +#else + /* Not an error — the feature is simply unsupported in this build */ + pr_debug("BTF data present but libbpf not available, skipping.\n"); + return 0; +#endif // HAVE_LIBBPF_SUPPORT +} + +static int process_compressed(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + + if (do_read_u32(ff, &(env->comp_ver))) + return -1; + + if (do_read_u32(ff, &(env->comp_type))) + return -1; + + if (do_read_u32(ff, &(env->comp_level))) + return -1; + + if (do_read_u32(ff, &(env->comp_ratio))) + return -1; + + if (do_read_u32(ff, &(env->comp_mmap_len))) + return -1; + + /* + * FIXME: perf.data should record the recording system's page + * size — it affects mmap buffer alignment, sample addresses, + * and data_page_size/code_page_size interpretation. Without + * it we assume 4K (the smallest Linux page size) as a safe + * minimum alignment for comp_mmap_len validation. + * + * No upper-bound cap: perf_session__process_compressed_event() + * checks decomp_len + sizeof(struct decomp) against SIZE_MAX + * before allocating, which handles 32-bit safety. + */ + if (env->comp_mmap_len < 4096 || env->comp_mmap_len % 4096) { + pr_err("Invalid HEADER_COMPRESSED: comp_mmap_len (%u) must be a 4K-aligned value >= 4096\n", + env->comp_mmap_len); + return -1; + } + + return 0; +} + +static int __process_pmu_caps(struct feat_fd *ff, int *nr_caps, + char ***caps, unsigned int *max_branches, + unsigned int *br_cntr_nr, + unsigned int *br_cntr_width) +{ + char *name, *value, *ptr; + u32 nr_pmu_caps, i; + + *nr_caps = 0; + *caps = NULL; + + if (do_read_u32(ff, &nr_pmu_caps)) + return -1; + + if (!nr_pmu_caps) + return 0; + + if (nr_pmu_caps > MAX_PMU_CAPS) { + pr_err("Invalid pmu caps: nr_pmu_caps (%u) > %u\n", + nr_pmu_caps, MAX_PMU_CAPS); + return -1; + } + + *caps = calloc(nr_pmu_caps, sizeof(char *)); + if (!*caps) + return -1; + + for (i = 0; i < nr_pmu_caps; i++) { + name = do_read_string(ff); + if (!name) + goto error; + + value = do_read_string(ff); + if (!value) + goto free_name; + + if (asprintf(&ptr, "%s=%s", name, value) < 0) + goto free_value; + + (*caps)[i] = ptr; + + if (!strcmp(name, "branches")) + *max_branches = atoi(value); + + if (!strcmp(name, "branch_counter_nr")) + *br_cntr_nr = atoi(value); + + if (!strcmp(name, "branch_counter_width")) + *br_cntr_width = atoi(value); + + free(value); + free(name); + } + *nr_caps = nr_pmu_caps; + return 0; + +free_value: + free(value); +free_name: + free(name); +error: + for (; i > 0; i--) + free((*caps)[i - 1]); + free(*caps); + *caps = NULL; + *nr_caps = 0; + return -1; +} + +static int process_cpu_pmu_caps(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + int ret = __process_pmu_caps(ff, &env->nr_cpu_pmu_caps, + &env->cpu_pmu_caps, + &env->max_branches, + &env->br_cntr_nr, + &env->br_cntr_width); + + if (!ret && !env->cpu_pmu_caps) + pr_debug("cpu pmu capabilities not available\n"); + return ret; +} + +static int process_pmu_caps(struct feat_fd *ff, void *data __maybe_unused) +{ + struct perf_env *env = &ff->ph->env; + struct pmu_caps *pmu_caps; + u32 nr_pmu, i; + int ret; + int j; + + if (do_read_u32(ff, &nr_pmu)) + return -1; + + if (!nr_pmu) { + pr_debug("pmu capabilities not available\n"); + return 0; + } + + if (nr_pmu > MAX_PMU_MAPPINGS) { + pr_err("Invalid HEADER_PMU_CAPS: nr_pmu (%u) > %u\n", + nr_pmu, MAX_PMU_MAPPINGS); + return -1; + } + + if (ff->size < sizeof(u32) + nr_pmu * sizeof(u32)) { + pr_err("Invalid HEADER_PMU_CAPS: section too small (%zu) for %u PMUs\n", + ff->size, nr_pmu); + return -1; + } + + pmu_caps = calloc(nr_pmu, sizeof(*pmu_caps)); + if (!pmu_caps) + return -ENOMEM; + + for (i = 0; i < nr_pmu; i++) { + ret = __process_pmu_caps(ff, &pmu_caps[i].nr_caps, + &pmu_caps[i].caps, + &pmu_caps[i].max_branches, + &pmu_caps[i].br_cntr_nr, + &pmu_caps[i].br_cntr_width); + if (ret) + goto err; + + pmu_caps[i].pmu_name = do_read_string(ff); + if (!pmu_caps[i].pmu_name) { + ret = -1; + goto err; + } + if (!pmu_caps[i].nr_caps) { + pr_debug("%s pmu capabilities not available\n", + pmu_caps[i].pmu_name); + } + } + + env->nr_pmus_with_caps = nr_pmu; + env->pmu_caps = pmu_caps; + return 0; + +err: + for (i = 0; i < nr_pmu; i++) { + for (j = 0; j < pmu_caps[i].nr_caps; j++) + free(pmu_caps[i].caps[j]); + free(pmu_caps[i].caps); + free(pmu_caps[i].pmu_name); + } + + free(pmu_caps); + return ret; +} + +static int process_cpu_domain_info(struct feat_fd *ff, void *data __maybe_unused) +{ + u32 schedstat_version, max_sched_domains, cpu, domain, nr_domains; + struct perf_env *env = &ff->ph->env; + char *dname, *cpumask, *cpulist; + struct cpu_domain_map **cd_map; + struct domain_info *d_info; + u32 nra, nr, i, j; + int ret; + + nra = env->nr_cpus_avail; + nr = env->nr_cpus_online; + + if (nra == 0 || nr == 0) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: missing HEADER_NRCPUS\n"); + return -1; + } + + if (ff->size < 2 * sizeof(u32) + nr * 2 * sizeof(u32)) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: section too small (%zu) for %u CPUs\n", + (size_t)ff->size, nr); + return -1; + } + + cd_map = calloc(nra, sizeof(*cd_map)); + if (!cd_map) + return -1; + + env->cpu_domain = cd_map; + + ret = do_read_u32(ff, &schedstat_version); + if (ret) + return ret; + + env->schedstat_version = schedstat_version; + + ret = do_read_u32(ff, &max_sched_domains); + if (ret) + return ret; + + /* + * Sanity check: real systems have at most ~10 sched domain levels + * (SMT, CLS, MC, PKG + NUMA hops). Reject obviously bogus values + * from malformed perf.data files before they cause excessive + * allocation in the per-CPU loop. + */ + if (max_sched_domains > MAX_SCHED_DOMAINS) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: max_sched_domains %u > %u\n", + max_sched_domains, MAX_SCHED_DOMAINS); + return -1; + } + + env->max_sched_domains = max_sched_domains; + + for (i = 0; i < nr; i++) { + if (do_read_u32(ff, &cpu)) + return -1; + + if (cpu >= nra) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: cpu %d >= nr_cpus_avail (%d)\n", cpu, nra); + return -1; + } + + if (cd_map[cpu]) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: duplicate cpu %u\n", cpu); + return -1; + } + + cd_map[cpu] = zalloc(sizeof(*cd_map[cpu])); + if (!cd_map[cpu]) + return -1; + + cd_map[cpu]->cpu = cpu; + + if (do_read_u32(ff, &nr_domains)) + return -1; + + if (nr_domains > max_sched_domains) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: nr_domains %u > max_sched_domains (%u)\n", + nr_domains, max_sched_domains); + return -1; + } + + cd_map[cpu]->nr_domains = nr_domains; + + cd_map[cpu]->domains = calloc(max_sched_domains, sizeof(*d_info)); + if (!cd_map[cpu]->domains) + return -1; + + for (j = 0; j < nr_domains; j++) { + if (do_read_u32(ff, &domain)) + return -1; + + if (domain >= max_sched_domains) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: domain %d >= max_sched_domains (%d)\n", + domain, max_sched_domains); + return -1; + } + + d_info = zalloc(sizeof(*d_info)); + if (!d_info) + return -1; + + if (cd_map[cpu]->domains[domain]) { + pr_err("Invalid HEADER_CPU_DOMAIN_INFO: duplicate domain %u for cpu %u\n", + domain, cpu); + free(d_info); + return -1; + } + + cd_map[cpu]->domains[domain] = d_info; + d_info->domain = domain; + + if (schedstat_version >= 17) { + dname = do_read_string(ff); + if (!dname) + return -1; + + d_info->dname = dname; + } + + cpumask = do_read_string(ff); + if (!cpumask) + return -1; + + d_info->cpumask = cpumask; + + cpulist = do_read_string(ff); + if (!cpulist) + return -1; + + d_info->cpulist = cpulist; + } + } + + return ret; +} + +#define FEAT_OPR(n, func, __full_only) \ + [HEADER_##n] = { \ + .name = __stringify(n), \ + .write = write_##func, \ + .print = print_##func, \ + .full_only = __full_only, \ + .process = process_##func, \ + .synthesize = true \ + } + +#define FEAT_OPN(n, func, __full_only) \ + [HEADER_##n] = { \ + .name = __stringify(n), \ + .write = write_##func, \ + .print = print_##func, \ + .full_only = __full_only, \ + .process = process_##func \ + } + +/* feature_ops not implemented: */ +#define print_tracing_data NULL +#define print_build_id NULL + +#define process_branch_stack NULL +#define process_stat NULL + +// Only used in util/synthetic-events.c +const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE]; + +const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE] = { + FEAT_OPN(TRACING_DATA, tracing_data, false), + FEAT_OPN(BUILD_ID, build_id, false), + FEAT_OPR(HOSTNAME, hostname, false), + FEAT_OPR(OSRELEASE, osrelease, false), + FEAT_OPR(VERSION, version, false), + FEAT_OPR(ARCH, arch, false), + FEAT_OPR(NRCPUS, nrcpus, false), + FEAT_OPR(CPUDESC, cpudesc, false), + FEAT_OPR(CPUID, cpuid, false), + FEAT_OPR(TOTAL_MEM, total_mem, false), + FEAT_OPR(EVENT_DESC, event_desc, false), + FEAT_OPR(CMDLINE, cmdline, false), + FEAT_OPR(CPU_TOPOLOGY, cpu_topology, true), + FEAT_OPR(NUMA_TOPOLOGY, numa_topology, true), + FEAT_OPN(BRANCH_STACK, branch_stack, false), + FEAT_OPR(PMU_MAPPINGS, pmu_mappings, false), + FEAT_OPR(GROUP_DESC, group_desc, false), + FEAT_OPN(AUXTRACE, auxtrace, false), + FEAT_OPN(STAT, stat, false), + FEAT_OPN(CACHE, cache, true), + FEAT_OPR(SAMPLE_TIME, sample_time, false), + FEAT_OPR(MEM_TOPOLOGY, mem_topology, true), + FEAT_OPR(CLOCKID, clockid, false), + FEAT_OPN(DIR_FORMAT, dir_format, false), + FEAT_OPR(BPF_PROG_INFO, bpf_prog_info, false), + FEAT_OPR(BPF_BTF, bpf_btf, false), + FEAT_OPR(COMPRESSED, compressed, false), + FEAT_OPR(CPU_PMU_CAPS, cpu_pmu_caps, false), + FEAT_OPR(CLOCK_DATA, clock_data, false), + FEAT_OPN(HYBRID_TOPOLOGY, hybrid_topology, true), + FEAT_OPR(PMU_CAPS, pmu_caps, false), + FEAT_OPR(CPU_DOMAIN_INFO, cpu_domain_info, true), + FEAT_OPR(E_MACHINE, e_machine, false), + FEAT_OPR(CLN_SIZE, cln_size, false), +}; + +struct header_print_data { + FILE *fp; + bool full; /* extended list of headers */ +}; + +const char *header_feat__name(unsigned int id) +{ + if (id < HEADER_LAST_FEATURE) + return feat_ops[id].name ?: "INVALID"; + return "INVALID"; +} + +static int perf_file_section__fprintf_info(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data) +{ + struct header_print_data *hd = data; + struct feat_fd ff; + + if (lseek(fd, section->offset, SEEK_SET) == (off_t)-1) { + pr_debug("Failed to lseek to %" PRIu64 " offset for feature %s (%d), continuing...\n", + section->offset, header_feat__name(feat), feat); + return 0; + } + if (feat >= ph->last_feat) { + pr_warning("unknown feature %d\n", feat); + return 0; + } + if (!feat_ops[feat].print) + return 0; + + ff = (struct feat_fd) { + .fd = fd, + .ph = ph, + .size = section->size, + }; + + if (!feat_ops[feat].full_only || hd->full) + feat_ops[feat].print(&ff, hd->fp); + else + fprintf(hd->fp, "# %s info available, use -I to display\n", + feat_ops[feat].name); + + return 0; +} + +int perf_header__fprintf_info(struct perf_session *session, FILE *fp, bool full) +{ + struct header_print_data hd; + struct perf_header *header = &session->header; + int fd = perf_data__fd(session->data); + struct stat st; + time_t stctime; + int ret, bit; + + hd.fp = fp; + hd.full = full; + + ret = fstat(fd, &st); + if (ret == -1) + return -1; + + stctime = st.st_mtime; + fprintf(fp, "# captured on : %s", ctime(&stctime)); + + fprintf(fp, "# header version : %u\n", header->version); + fprintf(fp, "# data offset : %" PRIu64 "\n", header->data_offset); + fprintf(fp, "# data size : %" PRIu64 "\n", header->data_size); + fprintf(fp, "# feat offset : %" PRIu64 "\n", header->feat_offset); + + perf_header__process_sections(header, fd, &hd, + perf_file_section__fprintf_info); + + if (session->data->is_pipe) + return 0; + + fprintf(fp, "# missing features: "); + for_each_clear_bit(bit, header->adds_features, header->last_feat) { + if (bit) + fprintf(fp, "%s ", feat_ops[bit].name); + } + + fprintf(fp, "\n"); + return 0; +} + +struct header_fw { + struct feat_writer fw; + struct feat_fd *ff; +}; + +static int feat_writer_cb(struct feat_writer *fw, void *buf, size_t sz) +{ + struct header_fw *h = container_of(fw, struct header_fw, fw); + + return do_write(h->ff, buf, sz); +} + +static int do_write_feat(struct feat_fd *ff, int type, + struct perf_file_section **p, + struct evlist *evlist, + struct feat_copier *fc) +{ + int err; + int ret = 0; + + if (perf_header__has_feat(ff->ph, type)) { + if (!feat_ops[type].write) + return -1; + + if (WARN(ff->buf, "Error: calling %s in pipe-mode.\n", __func__)) + return -1; + + (*p)->offset = lseek(ff->fd, 0, SEEK_CUR); + + /* + * Hook to let perf inject copy features sections from the input + * file. + */ + if (fc && fc->copy) { + struct header_fw h = { + .fw.write = feat_writer_cb, + .ff = ff, + }; + + /* ->copy() returns 0 if the feature was not copied */ + err = fc->copy(fc, type, &h.fw); + } else { + err = 0; + } + if (!err) + err = feat_ops[type].write(ff, evlist); + if (err < 0) { + pr_debug("failed to write feature %s\n", feat_ops[type].name); + + /* undo anything written */ + lseek(ff->fd, (*p)->offset, SEEK_SET); + + return -1; + } + (*p)->size = lseek(ff->fd, 0, SEEK_CUR) - (*p)->offset; + (*p)++; + } + return ret; +} + +static int perf_header__adds_write(struct perf_header *header, + struct evlist *evlist, int fd, + struct feat_copier *fc) +{ + int nr_sections; + struct feat_fd ff = { + .fd = fd, + .ph = header, + }; + struct perf_file_section *feat_sec, *p; + int sec_size; + u64 sec_start; + int feat; + int err; + + nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); + if (!nr_sections) + return 0; + + feat_sec = p = calloc(nr_sections, sizeof(*feat_sec)); + if (feat_sec == NULL) + return -ENOMEM; + + sec_size = sizeof(*feat_sec) * nr_sections; + + sec_start = header->feat_offset; + lseek(fd, sec_start + sec_size, SEEK_SET); + + for_each_set_bit(feat, header->adds_features, HEADER_FEAT_BITS) { + if (do_write_feat(&ff, feat, &p, evlist, fc)) + perf_header__clear_feat(header, feat); + } + + lseek(fd, sec_start, SEEK_SET); + /* + * may write more than needed due to dropped feature, but + * this is okay, reader will skip the missing entries + */ + err = do_write(&ff, feat_sec, sec_size); + if (err < 0) + pr_debug("failed to write feature section\n"); + free(ff.buf); /* TODO: added to silence clang-tidy. */ + free(feat_sec); + return err; +} + +int perf_header__write_pipe(int fd) +{ + struct perf_pipe_file_header f_header; + struct feat_fd ff = { + .fd = fd, + }; + int err; + + f_header = (struct perf_pipe_file_header){ + .magic = PERF_MAGIC, + .size = sizeof(f_header), + }; + + err = do_write(&ff, &f_header, sizeof(f_header)); + if (err < 0) { + pr_debug("failed to write perf pipe header\n"); + return err; + } + free(ff.buf); + return 0; +} + +static int perf_session__do_write_header(struct perf_session *session, + struct evlist *evlist, + int fd, bool at_exit, + struct feat_copier *fc, + bool write_attrs_after_data) +{ + struct perf_file_header f_header; + struct perf_header *header = &session->header; + struct evsel *evsel; + struct feat_fd ff = { + .ph = header, + .fd = fd, + }; + u64 attr_offset = sizeof(f_header), attr_size = 0; + int err; + + if (write_attrs_after_data && at_exit) { + /* + * Write features at the end of the file first so that + * attributes may come after them. + */ + if (!header->data_offset && header->data_size) { + pr_err("File contains data but offset unknown\n"); + err = -1; + goto err_out; + } + header->feat_offset = header->data_offset + header->data_size; + err = perf_header__adds_write(header, evlist, fd, fc); + if (err < 0) + goto err_out; + attr_offset = lseek(fd, 0, SEEK_CUR); + } else { + lseek(fd, attr_offset, SEEK_SET); + } + + evlist__for_each_entry(session->evlist, evsel) { + evsel->id_offset = attr_offset; + /* Avoid writing at the end of the file until the session is exiting. */ + if (!write_attrs_after_data || at_exit) { + err = do_write(&ff, evsel->core.id, evsel->core.ids * sizeof(u64)); + if (err < 0) { + pr_debug("failed to write perf header\n"); + goto err_out; + } + } + attr_offset += evsel->core.ids * sizeof(u64); + } + + evlist__for_each_entry(evlist, evsel) { + if (evsel->core.attr.size < sizeof(evsel->core.attr)) { + /* + * We are likely in "perf inject" and have read + * from an older file. Update attr size so that + * reader gets the right offset to the ids. + */ + evsel->core.attr.size = sizeof(evsel->core.attr); + } + /* Avoid writing at the end of the file until the session is exiting. */ + if (!write_attrs_after_data || at_exit) { + struct perf_file_attr f_attr = { + .attr = evsel->core.attr, + .ids = { + .offset = evsel->id_offset, + .size = evsel->core.ids * sizeof(u64), + } + }; + err = do_write(&ff, &f_attr, sizeof(f_attr)); + if (err < 0) { + pr_debug("failed to write perf header attribute\n"); + goto err_out; + } + } + attr_size += sizeof(struct perf_file_attr); + } + + if (!header->data_offset) { + if (write_attrs_after_data) + header->data_offset = sizeof(f_header); + else + header->data_offset = attr_offset + attr_size; + } + header->feat_offset = header->data_offset + header->data_size; + + if (!write_attrs_after_data && at_exit) { + /* Write features now feat_offset is known. */ + err = perf_header__adds_write(header, evlist, fd, fc); + if (err < 0) + goto err_out; + } + + f_header = (struct perf_file_header){ + .magic = PERF_MAGIC, + .size = sizeof(f_header), + .attr_size = sizeof(struct perf_file_attr), + .attrs = { + .offset = attr_offset, + .size = attr_size, + }, + .data = { + .offset = header->data_offset, + .size = header->data_size, + }, + /* event_types is ignored, store zeros */ + }; + + memcpy(&f_header.adds_features, &header->adds_features, sizeof(header->adds_features)); + + lseek(fd, 0, SEEK_SET); + err = do_write(&ff, &f_header, sizeof(f_header)); + if (err < 0) { + pr_debug("failed to write perf header\n"); + goto err_out; + } else { + lseek(fd, 0, SEEK_END); + err = 0; + } +err_out: + free(ff.buf); + return err; +} + +int perf_session__write_header(struct perf_session *session, + struct evlist *evlist, + int fd, bool at_exit) +{ + return perf_session__do_write_header(session, evlist, fd, at_exit, /*fc=*/NULL, + /*write_attrs_after_data=*/false); +} + +size_t perf_session__data_offset(const struct evlist *evlist) +{ + struct evsel *evsel; + size_t data_offset; + + data_offset = sizeof(struct perf_file_header); + evlist__for_each_entry(evlist, evsel) { + data_offset += evsel->core.ids * sizeof(u64); + } + data_offset += evlist->core.nr_entries * sizeof(struct perf_file_attr); + + return data_offset; +} + +int perf_session__inject_header(struct perf_session *session, + struct evlist *evlist, + int fd, + struct feat_copier *fc, + bool write_attrs_after_data) +{ + return perf_session__do_write_header(session, evlist, fd, true, fc, + write_attrs_after_data); +} + +static int perf_header__getbuffer64(struct perf_header *header, + int fd, void *buf, size_t size) +{ + ssize_t n = readn(fd, buf, size); + + if (n <= 0) { + if (n == 0) + errno = EIO; + return -1; + } + + if (header->needs_swap) + mem_bswap_64(buf, size); + + return 0; +} + +int perf_header__process_sections(struct perf_header *header, int fd, + void *data, + int (*process)(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data)) +{ + struct perf_file_section *feat_sec, *sec; + int nr_sections; + int sec_size; + int feat; + int err; + struct stat st; + + nr_sections = bitmap_weight(header->adds_features, HEADER_FEAT_BITS); + if (!nr_sections) + return 0; + + feat_sec = sec = calloc(nr_sections, sizeof(*feat_sec)); + if (!feat_sec) + return -1; + + sec_size = sizeof(*feat_sec) * nr_sections; + + lseek(fd, header->feat_offset, SEEK_SET); + + err = perf_header__getbuffer64(header, fd, feat_sec, sec_size); + if (err < 0) + goto out_free; + + if (fstat(fd, &st) < 0) { + pr_err("Failed to stat the perf data file\n"); + err = -1; + goto out_free; + } + + for_each_set_bit(feat, header->adds_features, header->last_feat) { + /* + * FIXME: block devices have st_size == 0, so we skip + * bounds checking entirely. Historically perf never + * prevented using a block device as input, but it + * probably should — there's no valid use case for it + * and it bypasses all file-size validation. + */ + if (S_ISREG(st.st_mode) && + (sec->offset > (u64)st.st_size || + sec->size > (u64)st.st_size - sec->offset)) { + pr_err("Feature %s (%d) section extends past EOF (offset=%" PRIu64 ", size=%" PRIu64 ", file=%" PRIu64 ")\n", + header_feat__name(feat), feat, + sec->offset, sec->size, (u64)st.st_size); + err = -1; + goto out_free; + } + err = process(sec++, header, feat, fd, data); + if (err < 0) + goto out_free; + } + err = 0; +out_free: + free(feat_sec); + return err; +} + +static const int attr_file_abi_sizes[] = { + [0] = PERF_ATTR_SIZE_VER0, + [1] = PERF_ATTR_SIZE_VER1, + [2] = PERF_ATTR_SIZE_VER2, + [3] = PERF_ATTR_SIZE_VER3, + [4] = PERF_ATTR_SIZE_VER4, + 0, +}; + +/* + * In the legacy file format, the magic number is not used to encode endianness. + * hdr_sz was used to encode endianness. But given that hdr_sz can vary based + * on ABI revisions, we need to try all combinations for all endianness to + * detect the endianness. + */ +static int try_all_file_abis(uint64_t hdr_sz, struct perf_header *ph) +{ + uint64_t ref_size, attr_size; + int i; + + for (i = 0 ; attr_file_abi_sizes[i]; i++) { + ref_size = attr_file_abi_sizes[i] + + sizeof(struct perf_file_section); + if (hdr_sz != ref_size) { + attr_size = bswap_64(hdr_sz); + if (attr_size != ref_size) + continue; + + ph->needs_swap = true; + } + pr_debug("ABI%d perf.data file detected, need_swap=%d\n", + i, + ph->needs_swap); + return 0; + } + /* could not determine endianness */ + return -1; +} + +#define PERF_PIPE_HDR_VER0 16 + +static const size_t attr_pipe_abi_sizes[] = { + [0] = PERF_PIPE_HDR_VER0, + 0, +}; + +/* + * In the legacy pipe format, there is an implicit assumption that endianness + * between host recording the samples, and host parsing the samples is the + * same. This is not always the case given that the pipe output may always be + * redirected into a file and analyzed on a different machine with possibly a + * different endianness and perf_event ABI revisions in the perf tool itself. + */ +static int try_all_pipe_abis(uint64_t hdr_sz, struct perf_header *ph) +{ + u64 attr_size; + int i; + + for (i = 0 ; attr_pipe_abi_sizes[i]; i++) { + if (hdr_sz != attr_pipe_abi_sizes[i]) { + attr_size = bswap_64(hdr_sz); + if (attr_size != hdr_sz) + continue; + + ph->needs_swap = true; + } + pr_debug("Pipe ABI%d perf.data file detected\n", i); + return 0; + } + return -1; +} + +bool is_perf_magic(u64 magic) +{ + if (!memcmp(&magic, __perf_magic1, sizeof(magic)) + || magic == __perf_magic2 + || magic == __perf_magic2_sw) + return true; + + return false; +} + +static int check_magic_endian(u64 magic, uint64_t hdr_sz, + bool is_pipe, struct perf_header *ph) +{ + int ret; + + /* check for legacy format */ + ret = memcmp(&magic, __perf_magic1, sizeof(magic)); + if (ret == 0) { + ph->version = PERF_HEADER_VERSION_1; + pr_debug("legacy perf.data format\n"); + if (is_pipe) + return try_all_pipe_abis(hdr_sz, ph); + + return try_all_file_abis(hdr_sz, ph); + } + /* + * the new magic number serves two purposes: + * - unique number to identify actual perf.data files + * - encode endianness of file + */ + ph->version = PERF_HEADER_VERSION_2; + + /* check magic number with one endianness */ + if (magic == __perf_magic2) + return 0; + + /* check magic number with opposite endianness */ + if (magic != __perf_magic2_sw) + return -1; + + ph->needs_swap = true; + + return 0; +} + +int perf_file_header__read(struct perf_file_header *header, + struct perf_header *ph, int fd) +{ + ssize_t ret; + + lseek(fd, 0, SEEK_SET); + + ret = readn(fd, header, sizeof(*header)); + if (ret <= 0) + return -1; + + if (check_magic_endian(header->magic, + header->attr_size, false, ph) < 0) { + pr_debug("magic/endian check failed\n"); + return -1; + } + + if (ph->needs_swap) { + mem_bswap_64(header, offsetof(struct perf_file_header, + adds_features)); + } + + if (header->size > header->attrs.offset) { + pr_err("Perf file header corrupt: header overlaps attrs\n"); + return -1; + } + + if (header->size > header->data.offset) { + pr_err("Perf file header corrupt: header overlaps data\n"); + return -1; + } + + if ((header->attrs.offset <= header->data.offset && + header->attrs.offset + header->attrs.size > header->data.offset) || + (header->attrs.offset > header->data.offset && + header->data.offset + header->data.size > header->attrs.offset)) { + pr_err("Perf file header corrupt: Attributes and data overlap\n"); + return -1; + } + + if (header->size != sizeof(*header)) { + /* Support the previous format */ + if (header->size == offsetof(typeof(*header), adds_features)) + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + else + return -1; + } else if (ph->needs_swap) { + /* + * feature bitmap is declared as an array of unsigned longs -- + * not good since its size can differ between the host that + * generated the data file and the host analyzing the file. + * + * We need to handle endianness, but we don't know the size of + * the unsigned long where the file was generated. Take a best + * guess at determining it: try 64-bit swap first (ie., file + * created on a 64-bit host), and check if the hostname feature + * bit is set (this feature bit is forced on as of fbe96f2). + * If the bit is not, undo the 64-bit swap and try a 32-bit + * swap. If the hostname bit is still not set (e.g., older data + * file), punt and fallback to the original behavior -- + * clearing all feature bits and setting buildid. + */ + mem_bswap_64(&header->adds_features, + BITS_TO_U64(HEADER_FEAT_BITS)); + + if (!test_bit(HEADER_HOSTNAME, header->adds_features)) { + /* unswap as u64 */ + mem_bswap_64(&header->adds_features, + BITS_TO_U64(HEADER_FEAT_BITS)); + + /* unswap as u32 */ + mem_bswap_32(&header->adds_features, + BITS_TO_U32(HEADER_FEAT_BITS)); + } + + if (!test_bit(HEADER_HOSTNAME, header->adds_features)) { + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + __set_bit(HEADER_BUILD_ID, header->adds_features); + } + } + + memcpy(&ph->adds_features, &header->adds_features, + sizeof(ph->adds_features)); + + ph->data_offset = header->data.offset; + ph->data_size = header->data.size; + ph->feat_offset = header->data.offset + header->data.size; + ph->last_feat = HEADER_LAST_FEATURE; + return 0; +} + +static int perf_file_section__process(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data) +{ + struct feat_fd fdd = { + .fd = fd, + .ph = ph, + .size = section->size, + .offset = 0, + }; + + if (lseek(fd, section->offset, SEEK_SET) == (off_t)-1) { + pr_debug("Failed to lseek to %" PRIu64 " offset for feature %s (%d), continuing...\n", + section->offset, header_feat__name(feat), feat); + return 0; + } + + if (feat >= HEADER_LAST_FEATURE) { + pr_debug("unknown feature %d, continuing...\n", feat); + return 0; + } + + if (!feat_ops[feat].process) + return 0; + + return feat_ops[feat].process(&fdd, data); +} + +static int perf_file_header__read_pipe(struct perf_pipe_file_header *header, + struct perf_header *ph, + struct perf_data *data) +{ + ssize_t ret; + + ret = perf_data__read(data, header, sizeof(*header)); + if (ret <= 0) + return -1; + + if (check_magic_endian(header->magic, header->size, true, ph) < 0) { + pr_debug("endian/magic failed\n"); + return -1; + } + + if (ph->needs_swap) + header->size = bswap_64(header->size); + + /* The last feature is written out as a 0 sized event and will update this value. */ + ph->last_feat = 0; + return 0; +} + +static int perf_header__read_pipe(struct perf_session *session) +{ + struct perf_header *header = &session->header; + struct perf_pipe_file_header f_header; + + if (perf_file_header__read_pipe(&f_header, header, session->data) < 0) { + pr_debug("incompatible file format\n"); + return -EINVAL; + } + + return f_header.size == sizeof(f_header) ? 0 : -1; +} + +static int read_attr(int fd, struct perf_header *ph, + struct perf_file_attr *f_attr) +{ + struct perf_event_attr *attr = &f_attr->attr; + size_t sz, left; + size_t our_sz = sizeof(f_attr->attr); + ssize_t ret; + + memset(f_attr, 0, sizeof(*f_attr)); + + /* read minimal guaranteed structure */ + ret = readn(fd, attr, PERF_ATTR_SIZE_VER0); + if (ret <= 0) { + pr_debug("cannot read %d bytes of header attr\n", + PERF_ATTR_SIZE_VER0); + if (ret == 0) + errno = EIO; + return -1; + } + + /* on file perf_event_attr size */ + sz = attr->size; + + if (ph->needs_swap) + sz = bswap_32(sz); + + if (sz == 0) { + /* assume ABI0 */ + sz = PERF_ATTR_SIZE_VER0; + } else if (sz < PERF_ATTR_SIZE_VER0) { + pr_debug("bad attr size %zu, expected at least %d\n", + sz, PERF_ATTR_SIZE_VER0); + errno = EINVAL; + return -1; + } else if (sz > our_sz) { + pr_debug("file uses a more recent and unsupported ABI" + " (%zu bytes extra)\n", sz - our_sz); + errno = EINVAL; + return -1; + } + /* what we have not yet read and that we know about */ + left = sz - PERF_ATTR_SIZE_VER0; + if (left) { + void *ptr = attr; + ptr += PERF_ATTR_SIZE_VER0; + + ret = readn(fd, ptr, left); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } + } + /* read perf_file_section, ids are read in caller */ + ret = readn(fd, &f_attr->ids, sizeof(f_attr->ids)); + if (ret <= 0) { + if (ret == 0) + errno = EIO; + return -1; + } + + return 0; +} + +#ifdef HAVE_LIBTRACEEVENT +static int evsel__prepare_tracepoint_event(struct evsel *evsel, struct tep_handle *pevent) +{ + struct tep_event *event; + char bf[128]; + + /* already prepared */ + if (evsel->tp_format) + return 0; + + if (pevent == NULL) { + pr_debug("broken or missing trace data\n"); + return -1; + } + + event = tep_find_event(pevent, evsel->core.attr.config); + if (event == NULL) { + pr_debug("cannot find event format for %d\n", (int)evsel->core.attr.config); + return -1; + } + + if (!evsel->name) { + snprintf(bf, sizeof(bf), "%s:%s", event->system, event->name); + evsel->name = strdup(bf); + if (evsel->name == NULL) + return -1; + } + + evsel->tp_format = event; + return 0; +} + +static int evlist__prepare_tracepoint_events(struct evlist *evlist, struct tep_handle *pevent) +{ + struct evsel *pos; + + evlist__for_each_entry(evlist, pos) { + if (pos->core.attr.type == PERF_TYPE_TRACEPOINT && + evsel__prepare_tracepoint_event(pos, pevent)) + return -1; + } + + return 0; +} +#endif + +int perf_session__read_header(struct perf_session *session) +{ + struct perf_data *data = session->data; + struct perf_header *header = &session->header; + struct perf_file_header f_header; + struct perf_file_attr f_attr; + u64 f_id; + struct stat input_stat; + int nr_attrs, nr_ids, i, j, err = -ENOMEM; + int fd = perf_data__fd(data); + + session->evlist = evlist__new(); + if (session->evlist == NULL) + return -ENOMEM; + + session->evlist->session = session; + session->machines.host.env = &header->env; + + /* + * We can read 'pipe' data event from regular file, + * check for the pipe header regardless of source. + */ + err = perf_header__read_pipe(session); + if (!err || perf_data__is_pipe(data)) { + data->is_pipe = true; + return err; + } + + err = -ENOMEM; + if (perf_file_header__read(&f_header, header, fd) < 0) + return -EINVAL; + + if (header->needs_swap && data->in_place_update) { + pr_err("In-place update not supported when byte-swapping is required\n"); + return -EINVAL; + } + + /* + * Sanity check that perf.data was written cleanly; data size is + * initialized to 0 and updated only if the on_exit function is run. + * If data size is still 0 then the file contains only partial + * information. Just warn user and process it as much as it can. + */ + if (f_header.data.size == 0) { + pr_warning("WARNING: The %s file's data size field is 0 which is unexpected.\n" + "Was the 'perf record' command properly terminated?\n", + data->file.path); + } + + if (f_header.attr_size == 0) { + pr_err("ERROR: The %s file's attr size field is 0 which is unexpected.\n" + "Was the 'perf record' command properly terminated?\n", + data->file.path); + return -EINVAL; + } + + if (fstat(fd, &input_stat) < 0) + return -errno; + + /* Check before assigning to int to avoid u64-to-int truncation */ + if (f_header.attrs.size / f_header.attr_size > MAX_NR_ATTRS) { + pr_err("Too many attributes: %" PRIu64 " (max %d)\n", + f_header.attrs.size / f_header.attr_size, MAX_NR_ATTRS); + return -EINVAL; + } + nr_attrs = f_header.attrs.size / f_header.attr_size; + lseek(fd, f_header.attrs.offset, SEEK_SET); + + for (i = 0; i < nr_attrs; i++) { + struct evsel *evsel; + off_t tmp; + + if (read_attr(fd, header, &f_attr) < 0) + goto out_errno; + + if (header->needs_swap) { + f_attr.ids.size = bswap_64(f_attr.ids.size); + f_attr.ids.offset = bswap_64(f_attr.ids.offset); + perf_event__attr_swap(&f_attr.attr); + } + + /* + * Validate ids section: must be aligned to u64, and + * the count must fit in an int to avoid truncation in + * nr_ids and size_t overflow in perf_evsel__alloc_id() + * on 32-bit architectures. + */ + if (f_attr.ids.size % sizeof(u64)) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, not aligned to u64\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_delete_evlist; + } + + /* + * Cap the ID count to avoid int truncation of nr_ids + * on 64-bit and size_t overflow in the allocation + * paths (nr_ids * sizeof(u64), nr_ids * + * sizeof(struct perf_sample_id)) on 32-bit. + */ + if (f_attr.ids.size / sizeof(u64) > MAX_IDS_PER_ATTR) { + pr_err("Invalid ids section size %" PRIu64 " for attr %d, too many IDs\n", + f_attr.ids.size, i); + err = -EINVAL; + goto out_delete_evlist; + } + + /* + * FIXME: see perf_header__process_sections() — block + * devices bypass this check because st_size is 0. + */ + if (S_ISREG(input_stat.st_mode) && + (f_attr.ids.offset > (u64)input_stat.st_size || + f_attr.ids.size > (u64)input_stat.st_size - f_attr.ids.offset)) { + pr_err("Invalid ids section for attr %d: offset=%" PRIu64 " size=%" PRIu64 " exceeds file size %" PRIu64 "\n", + i, f_attr.ids.offset, f_attr.ids.size, (u64)input_stat.st_size); + err = -EINVAL; + goto out_delete_evlist; + } + + tmp = lseek(fd, 0, SEEK_CUR); + evsel = evsel__new(&f_attr.attr); + + if (evsel == NULL) + goto out_delete_evlist; + + evsel->needs_swap = header->needs_swap; + /* + * Do it before so that if perf_evsel__alloc_id fails, this + * entry gets purged too at evlist__delete(). + */ + evlist__add(session->evlist, evsel); + + nr_ids = f_attr.ids.size / sizeof(u64); + /* + * We don't have the cpu and thread maps on the header, so + * for allocating the perf_sample_id table we fake 1 cpu and + * hattr->ids threads. + */ + if (perf_evsel__alloc_id(&evsel->core, 1, nr_ids)) + goto out_delete_evlist; + + lseek(fd, f_attr.ids.offset, SEEK_SET); + + for (j = 0; j < nr_ids; j++) { + if (perf_header__getbuffer64(header, fd, &f_id, sizeof(f_id))) + goto out_errno; + + perf_evlist__id_add(&session->evlist->core, &evsel->core, 0, j, f_id); + } + + lseek(fd, tmp, SEEK_SET); + } + + /* + * Skip feature section processing for truncated files + * (data.size == 0 means recording was interrupted). The + * section table is unreliable in that case, and the event + * data can still be processed without the feature headers. + * Clear the bitmap so has_feat() returns false and tools + * use their "feature not present" fallbacks instead of + * accessing uninitialized env fields. + */ + if (f_header.data.size == 0) { + bitmap_zero(header->adds_features, HEADER_FEAT_BITS); + } else { +#ifdef HAVE_LIBTRACEEVENT + err = perf_header__process_sections(header, fd, &session->tevent, + perf_file_section__process); + if (err < 0) + goto out_delete_evlist; + + if (evlist__prepare_tracepoint_events(session->evlist, + session->tevent.pevent)) { + err = -ENOMEM; + goto out_delete_evlist; + } +#else + err = perf_header__process_sections(header, fd, NULL, + perf_file_section__process); + if (err < 0) + goto out_delete_evlist; +#endif + } + + /* + * Without nr_cpus_avail the sample CPU bounds check in + * perf_session__deliver_event() is bypassed, allowing crafted + * CPU IDs to reach downstream consumers that index fixed-size + * arrays (timechart, kwork, sched — all sized MAX_NR_CPUS). + * + * This can happen with truncated files (interrupted recording + * loses all feature sections), very old files that predate + * HEADER_NRCPUS, or crafted files that omit it. Fall back to + * MAX_NR_CPUS so the bounds check is still effective — any + * CPU ID below that limit is safe for all downstream arrays. + */ + if (header->env.nr_cpus_avail == 0) { + header->env.nr_cpus_avail = MAX_NR_CPUS; + pr_warning("WARNING: perf.data is missing HEADER_NRCPUS, using MAX_NR_CPUS (%d) as CPU bound\n", + MAX_NR_CPUS); + } + + return 0; +out_errno: + return -errno; + +out_delete_evlist: + evlist__delete(session->evlist); + session->evlist = NULL; + return err; +} + +int perf_event__process_feature(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + struct feat_fd ff = { .fd = 0 }; + struct perf_record_header_feature *fe = (struct perf_record_header_feature *)event; + struct perf_header *header = &session->header; + int type = fe->header.type; + int feat = (int)fe->feat_id; + int ret = 0; + bool print = dump_trace; + bool last_feature_mark = false; + + if (type < 0 || type >= PERF_RECORD_HEADER_MAX) { + pr_warning("invalid record type %d in pipe-mode\n", type); + return 0; + } + if (feat == HEADER_RESERVED) { + pr_warning("invalid reserved record type in pipe-mode\n"); + return -1; + } + if (feat < 0 || feat == INT_MAX) { + pr_warning("invalid value for feature type %x\n", feat); + return -1; + } + if (feat >= header->last_feat) { + if (event->header.size == sizeof(*fe)) { + /* + * Either an unexpected zero size feature or the + * HEADER_LAST_FEATURE mark. + */ + if (feat > header->last_feat) + header->last_feat = min(feat, HEADER_LAST_FEATURE); + last_feature_mark = true; + } else { + /* + * A feature but beyond what is known as in + * bounds. Assume the last feature is 1 beyond this + * feature. + */ + session->header.last_feat = min(feat + 1, HEADER_LAST_FEATURE); + } + } + if (feat >= HEADER_LAST_FEATURE) { + if (!last_feature_mark) { + pr_warning("unknown feature %d for data file version (%s) in this version of perf (%s)\n", + feat, header->env.version, perf_version_string); + } + return 0; + } + if (event->header.size < sizeof(*fe)) { + pr_warning("feature header size too small\n"); + return -1; + } + ff.buf = (void *)fe->data; + ff.size = event->header.size - sizeof(*fe); + ff.ph = header; + + if (feat_ops[feat].process && feat_ops[feat].process(&ff, NULL)) { + // Processing failed, ignore when this is the last feature mark. + if (!last_feature_mark) + ret = -1; + goto out; + } + + if (session->tool->show_feat_hdr) { + if (!feat_ops[feat].full_only || + session->tool->show_feat_hdr >= SHOW_FEAT_HEADER_FULL_INFO) { + print = true; + } else { + fprintf(stdout, "# %s info available, use -I to display\n", + feat_ops[feat].name); + } + } + + if (dump_trace) + printf(", "); + + if (print) { + if (feat_ops[feat].print) + feat_ops[feat].print(&ff, stdout); + else + printf("# %s", feat_ops[feat].name); + } + +out: + free_event_desc(ff.events); + return ret; +} + +size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp) +{ + struct perf_record_event_update *ev = &event->event_update; + struct perf_cpu_map *map; + size_t ret; + + ret = fprintf(fp, "\n... id: %" PRI_lu64 "\n", ev->id); + + switch (ev->type) { + case PERF_EVENT_UPDATE__SCALE: + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + ret += fprintf(fp, "... scale: (truncated)\n"); + break; + } + ret += fprintf(fp, "... scale: %f\n", ev->scale.scale); + break; + case PERF_EVENT_UPDATE__UNIT: + case PERF_EVENT_UPDATE__NAME: { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + ret += fprintf(fp, "... %s: (unterminated)\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + break; + } + ret += fprintf(fp, "... %s: %s\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name", + ev->unit); + break; + } + case PERF_EVENT_UPDATE__CPUS: { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + u32 cpus_payload; + + if (event->header.size < cpus_off + sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map)) { + ret += fprintf(fp, "... cpus: (truncated)\n"); + break; + } + + /* + * Validate nr against payload — this function may be + * called from the stub handler (dump_trace path) which + * bypasses perf_event__process_event_update() validation. + */ + cpus_payload = event->header.size - cpus_off; + if (ev->cpus.cpus.type == PERF_CPU_MAP__CPUS) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, cpus_data.cpu) || + ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + ret += fprintf(fp, "... cpus: nr %u exceeds payload\n", + ev->cpus.cpus.cpus_data.nr); + break; + } + } else if (ev->cpus.cpus.type == PERF_CPU_MAP__MASK) { + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask32_data.nr); + break; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + ret += fprintf(fp, "... cpus: mask nr %u exceeds payload\n", + ev->cpus.cpus.mask64_data.nr); + break; + } + } + } + + ret += fprintf(fp, "... "); + + map = cpu_map__new_data(&ev->cpus.cpus); + if (map) { + ret += cpu_map__fprintf(map, fp); + perf_cpu_map__put(map); + } else + ret += fprintf(fp, "failed to get cpus\n"); + break; + } + default: + ret += fprintf(fp, "... unknown type\n"); + break; + } + + return ret; +} + +size_t perf_event__fprintf_attr(union perf_event *event, FILE *fp) +{ + return perf_event_attr__fprintf(fp, &event->attr.attr, __desc_attr__fprintf, NULL); +} + +int perf_event__process_attr(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct evlist **pevlist) +{ + struct perf_event_attr attr; + u32 i, n_ids, raw_attr_size; + u64 *ids; + size_t attr_size, copy_size; + struct evsel *evsel; + struct evlist *evlist = *pevlist; + + /* + * HEADER_ATTR event layout (pipe/inject mode): + * + * [header (8 bytes)] [attr (attr_size bytes)] [id0 id1 ... idN] + * |<------------------ header.size --------------------------->| + * + * attr_size varies across perf versions: VER0 = 64 bytes, + * current sizeof(struct perf_event_attr) = larger. A newer + * producer may emit a larger attr than we understand. + * + * attr.size == 0 (ABI0) means the producer didn't set it + * (e.g., bench/inject-buildid, older perf). Treat as VER0. + * + * Require 8-byte alignment so the u64 ID array is aligned + * and attr.size fits cleanly within the payload. + * + * Read attr.size once — the event may be on a shared mmap + * and re-reading could yield a different value. + */ + raw_attr_size = event->attr.attr.size; + if (event->header.size < sizeof(event->header) + PERF_ATTR_SIZE_VER0 || + (raw_attr_size && (raw_attr_size < PERF_ATTR_SIZE_VER0 || + raw_attr_size % sizeof(u64) || + raw_attr_size > event->header.size - sizeof(event->header)))) { + pr_err("PERF_RECORD_HEADER_ATTR: invalid attr.size %u (event size %u, min %d)\n", + raw_attr_size, event->header.size, PERF_ATTR_SIZE_VER0); + return -EINVAL; + } + + if (dump_trace) + perf_event__fprintf_attr(event, stdout); + + if (evlist == NULL) { + *pevlist = evlist = evlist__new(); + if (evlist == NULL) + return -ENOMEM; + } + + /* + * attr_size = footprint of the attr in the event — determines + * where the ID array starts. For ABI0, assume VER0 (64 bytes). + * + * copy_size = how much we copy into our local struct, capped at + * sizeof(attr) so a newer producer's larger attr doesn't + * overflow. Fields beyond copy_size are zeroed. + * + * Do NOT write attr_size back to the event — native-endian + * files use MAP_SHARED (read-only), writing would SIGSEGV. + * The swap path handles ABI0 in perf_event__attr_swap() + * which writes to the writable MAP_PRIVATE copy instead. + */ + attr_size = raw_attr_size ?: PERF_ATTR_SIZE_VER0; + copy_size = min(attr_size, sizeof(attr)); + memcpy(&attr, &event->attr.attr, copy_size); + if (copy_size < sizeof(attr)) + memset((void *)&attr + copy_size, 0, sizeof(attr) - copy_size); + + /* + * Normalize ABI0: the swap path sets attr.size = VER0 on the + * event, but the native path leaves it as 0. Set it on the + * local copy so perf inject re-synthesizes with consistent + * layout regardless of endianness. + */ + attr.size = attr_size; + + evsel = evsel__new(&attr); + if (evsel == NULL) + return -ENOMEM; + + evlist__add(evlist, evsel); + + /* + * IDs occupy the remainder after header + attr. Use attr_size + * (not copy_size) — even if the producer's attr is larger than + * our struct, the IDs start after attr_size bytes in the event. + * Validation above guarantees attr_size <= payload size. + */ + n_ids = event->header.size - sizeof(event->header) - attr_size; + n_ids = n_ids / sizeof(u64); + /* + * We don't have the cpu and thread maps on the header, so + * for allocating the perf_sample_id table we fake 1 cpu and + * hattr->ids threads. + */ + if (perf_evsel__alloc_id(&evsel->core, 1, n_ids)) + return -ENOMEM; + + /* + * Locate IDs at attr_size bytes past the attr start in the + * event. Cannot use perf_record_header_attr_id() — that + * macro reads event->attr.attr.size, which is 0 for ABI0 + * on the native-endian path (no swap handler to fix it up). + */ + ids = (void *)&event->attr.attr + attr_size; + for (i = 0; i < n_ids; i++) { + perf_evlist__id_add(&evlist->core, &evsel->core, 0, i, ids[i]); + } + + return 0; +} + +int perf_event__process_event_update(const struct perf_tool *tool __maybe_unused, + union perf_event *event, + struct evlist **pevlist) +{ + struct perf_record_event_update *ev = &event->event_update; + struct evlist *evlist; + struct evsel *evsel; + struct perf_cpu_map *map; + + /* + * Validate payload before dump_trace or processing — both + * paths access variant-specific fields without further checks. + */ + if (ev->type == PERF_EVENT_UPDATE__UNIT || + ev->type == PERF_EVENT_UPDATE__NAME) { + size_t str_off = offsetof(struct perf_record_event_update, unit); + size_t max_len = event->header.size > str_off ? + event->header.size - str_off : 0; + + if (max_len == 0 || strnlen(ev->unit, max_len) == max_len) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: %s not null-terminated, skipping\n", + ev->type == PERF_EVENT_UPDATE__UNIT ? "unit" : "name"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__SCALE) { + if (event->header.size < offsetof(struct perf_record_event_update, scale) + + sizeof(ev->scale)) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: SCALE payload too small, skipping\n"); + return 0; + } + } else if (ev->type == PERF_EVENT_UPDATE__CPUS) { + size_t cpus_off = offsetof(struct perf_record_event_update, cpus); + size_t min_cpus = sizeof(__u16) + + sizeof(struct perf_record_range_cpu_map); + u32 cpus_payload; + + if (event->header.size < cpus_off + min_cpus) { + pr_warning("WARNING: PERF_RECORD_EVENT_UPDATE: CPUS payload too small, skipping\n"); + return 0; + } + + /* + * Validate per-variant nr against the remaining + * payload on the native path — the swap path clamps + * nr in perf_event__event_update_swap(), but native + * events are read-only and cannot be clamped in place. + * cpu_map__new_data() trusts nr for allocation and + * iteration, so unchecked values cause OOB reads. + */ + cpus_payload = event->header.size - cpus_off; + switch (ev->cpus.cpus.type) { + case PERF_CPU_MAP__CPUS: + if (ev->cpus.cpus.cpus_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, cpus_data.cpu)) / + sizeof(ev->cpus.cpus.cpus_data.cpu[0])) { + pr_warning("WARNING: EVENT_UPDATE CPUS: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.cpus_data.nr); + return 0; + } + break; + case PERF_CPU_MAP__MASK: + if (ev->cpus.cpus.mask32_data.long_size == 4) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask32_data.mask) || + ev->cpus.cpus.mask32_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask32_data.mask)) / + sizeof(ev->cpus.cpus.mask32_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask32_data.nr); + return 0; + } + } else if (ev->cpus.cpus.mask64_data.long_size == 8) { + if (cpus_payload < offsetof(struct perf_record_cpu_map_data, mask64_data.mask) || + ev->cpus.cpus.mask64_data.nr > + (cpus_payload - offsetof(struct perf_record_cpu_map_data, mask64_data.mask)) / + sizeof(ev->cpus.cpus.mask64_data.mask[0])) { + pr_warning("WARNING: EVENT_UPDATE MASK: nr %u exceeds payload, skipping\n", + ev->cpus.cpus.mask64_data.nr); + return 0; + } + } + break; + default: + break; + } + } + + if (dump_trace) + perf_event__fprintf_event_update(event, stdout); + + if (!pevlist || *pevlist == NULL) + return -EINVAL; + + evlist = *pevlist; + + evsel = evlist__id2evsel(evlist, ev->id); + if (evsel == NULL) + return -EINVAL; + + switch (ev->type) { + case PERF_EVENT_UPDATE__UNIT: + free((char *)evsel->unit); + evsel->unit = strdup(ev->unit); + break; + case PERF_EVENT_UPDATE__NAME: + free(evsel->name); + evsel->name = strdup(ev->name); + break; + case PERF_EVENT_UPDATE__SCALE: + evsel->scale = ev->scale.scale; + break; + case PERF_EVENT_UPDATE__CPUS: + map = cpu_map__new_data(&ev->cpus.cpus); + if (map) { + perf_cpu_map__put(evsel->core.pmu_cpus); + evsel->core.pmu_cpus = map; + } else + pr_err("failed to get event_update cpus\n"); + break; + default: + break; + } + + return 0; +} + +#ifdef HAVE_LIBTRACEEVENT +int perf_event__process_tracing_data(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + ssize_t size_read, padding, size = event->tracing_data.size; + int fd = perf_data__fd(session->data); + char buf[BUFSIZ]; + + /* + * The pipe fd is already in proper place and in any case + * we can't move it, and we'd screw the case where we read + * 'pipe' data from regular file. The trace_report reads + * data from 'fd' so we need to set it directly behind the + * event, where the tracing data starts. + */ + if (!perf_data__is_pipe(session->data)) { + off_t offset = lseek(fd, 0, SEEK_CUR); + + /* setup for reading amidst mmap */ + lseek(fd, offset + sizeof(struct perf_record_header_tracing_data), + SEEK_SET); + } + + size_read = trace_report(fd, &session->tevent, session->trace_event_repipe); + padding = PERF_ALIGN(size_read, sizeof(u64)) - size_read; + + if (readn(fd, buf, padding) < 0) { + pr_err("%s: reading input file", __func__); + return -1; + } + if (session->trace_event_repipe) { + int retw = write(STDOUT_FILENO, buf, padding); + if (retw <= 0 || retw != padding) { + pr_err("%s: repiping tracing data padding", __func__); + return -1; + } + } + + if (size_read + padding != size) { + pr_err("%s: tracing data size mismatch", __func__); + return -1; + } + + evlist__prepare_tracepoint_events(session->evlist, session->tevent.pevent); + + return size_read + padding; +} +#endif + +int perf_event__process_build_id(const struct perf_tool *tool __maybe_unused, + struct perf_session *session, + union perf_event *event) +{ + __event_process_build_id(&event->build_id, + event->build_id.filename, + session); + return 0; +} diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h new file mode 100644 index 000000000..86b1a7202 --- /dev/null +++ b/tools/perf/util/header.h @@ -0,0 +1,225 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_HEADER_H +#define __PERF_HEADER_H + +#include +#include +#include +#include // FILE +#include +#include +#include +#include "env.h" +#include + +struct evlist; +union perf_event; +struct perf_header; +struct perf_session; +struct perf_tool; + +enum { + HEADER_RESERVED = 0, /* always cleared */ + HEADER_FIRST_FEATURE = 1, + HEADER_TRACING_DATA = 1, + HEADER_BUILD_ID, + + HEADER_HOSTNAME, + HEADER_OSRELEASE, + HEADER_VERSION, + HEADER_ARCH, + HEADER_NRCPUS, + HEADER_CPUDESC, + HEADER_CPUID, + HEADER_TOTAL_MEM, + HEADER_CMDLINE, + HEADER_EVENT_DESC, + HEADER_CPU_TOPOLOGY, + HEADER_NUMA_TOPOLOGY, + HEADER_BRANCH_STACK, + HEADER_PMU_MAPPINGS, + HEADER_GROUP_DESC, + HEADER_AUXTRACE, + HEADER_STAT, + HEADER_CACHE, + HEADER_SAMPLE_TIME, + HEADER_MEM_TOPOLOGY, + HEADER_CLOCKID, + HEADER_DIR_FORMAT, + HEADER_BPF_PROG_INFO, + HEADER_BPF_BTF, + HEADER_COMPRESSED, + HEADER_CPU_PMU_CAPS, + HEADER_CLOCK_DATA, + HEADER_HYBRID_TOPOLOGY, + HEADER_PMU_CAPS, + HEADER_CPU_DOMAIN_INFO, + HEADER_E_MACHINE, + HEADER_CLN_SIZE, + HEADER_LAST_FEATURE, + HEADER_FEAT_BITS = 256, +}; + +enum perf_header_version { + PERF_HEADER_VERSION_1, + PERF_HEADER_VERSION_2, +}; + +struct perf_file_section { + u64 offset; + u64 size; +}; + +/** + * struct perf_file_header: Header representation on disk. + */ +struct perf_file_header { + /** @magic: Holds "PERFILE2". */ + u64 magic; + /** @size: Size of this header - sizeof(struct perf_file_header). */ + u64 size; + /** + * @attr_size: Size of attrs entries - sizeof(struct perf_event_attr) + + * sizeof(struct perf_file_section). + */ + u64 attr_size; + /** @attrs: Offset and size of file section holding attributes. */ + struct perf_file_section attrs; + /** @data: Offset and size of file section holding regular event data. */ + struct perf_file_section data; + /** @event_types: Ignored. */ + struct perf_file_section event_types; + /** + * @adds_features: Bitmap of features. The features are immediately after the data section. + */ + DECLARE_BITMAP(adds_features, HEADER_FEAT_BITS); +}; + +struct perf_pipe_file_header { + u64 magic; + u64 size; +}; + +int perf_file_header__read(struct perf_file_header *header, + struct perf_header *ph, int fd); + +struct perf_header { + enum perf_header_version version; + bool needs_swap; + u64 data_offset; + u64 data_size; + u64 feat_offset; + DECLARE_BITMAP(adds_features, HEADER_FEAT_BITS); + int last_feat; + struct perf_env env; +}; + +struct feat_fd { + struct perf_header *ph; + int fd; + void *buf; /* Either buf != NULL or fd >= 0 */ + ssize_t offset; + size_t size; + struct evsel *events; +}; + +struct perf_header_feature_ops { + int (*write)(struct feat_fd *ff, struct evlist *evlist); + void (*print)(struct feat_fd *ff, FILE *fp); + int (*process)(struct feat_fd *ff, void *data); + const char *name; + bool full_only; + bool synthesize; +}; + +extern const char perf_version_string[]; + +const char *header_feat__name(unsigned int id); + +int perf_session__read_header(struct perf_session *session); +int perf_session__write_header(struct perf_session *session, + struct evlist *evlist, + int fd, bool at_exit); +int perf_header__write_pipe(int fd); + +/* feat_writer writes a feature section to output */ +struct feat_writer { + int (*write)(struct feat_writer *fw, void *buf, size_t sz); +}; + +/* feat_copier copies a feature section using feat_writer to output */ +struct feat_copier { + int (*copy)(struct feat_copier *fc, int feat, struct feat_writer *fw); +}; + +int perf_session__inject_header(struct perf_session *session, + struct evlist *evlist, + int fd, + struct feat_copier *fc, + bool write_attrs_after_data); + +size_t perf_session__data_offset(const struct evlist *evlist); + +void perf_header__set_feat(struct perf_header *header, int feat); +void perf_header__clear_feat(struct perf_header *header, int feat); +bool perf_header__has_feat(const struct perf_header *header, int feat); + +int perf_header__set_cmdline(int argc, const char **argv); + +int perf_header__process_sections(struct perf_header *header, int fd, + void *data, + int (*process)(struct perf_file_section *section, + struct perf_header *ph, + int feat, int fd, void *data)); + +int perf_header__fprintf_info(struct perf_session *s, FILE *fp, bool full); + +int perf_event__process_feature(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +int perf_event__process_attr(const struct perf_tool *tool, union perf_event *event, + struct evlist **pevlist); +int perf_event__process_event_update(const struct perf_tool *tool, + union perf_event *event, + struct evlist **pevlist); +size_t perf_event__fprintf_attr(union perf_event *event, FILE *fp); +size_t perf_event__fprintf_event_update(union perf_event *event, FILE *fp); +#ifdef HAVE_LIBTRACEEVENT +int perf_event__process_tracing_data(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +#endif +int perf_event__process_build_id(const struct perf_tool *tool, + struct perf_session *session, + union perf_event *event); +bool is_perf_magic(u64 magic); + +#define NAME_ALIGN 64 + +struct feat_fd; + +int do_write(struct feat_fd *fd, const void *buf, size_t size); + +int write_padded(struct feat_fd *fd, const void *bf, + size_t count, size_t count_aligned); + +#define MAX_CACHE_LVL 4 + +int build_caches_for_cpu(u32 cpu, struct cpu_cache_level caches[], u32 *cntp); + +#define DEFAULT_CACHELINE_SIZE 64 + +/* + * arch specific callback + */ +int get_cpuid(char *buffer, size_t sz, struct perf_cpu cpu); + +char *get_cpuid_str(struct perf_cpu cpu); + +char *get_cpuid_allow_env_override(struct perf_cpu cpu); + +int strcmp_cpuid_str(const char *s1, const char *s2); + +struct cpu_domain_map **build_cpu_domain_map(u32 *schedstat_version, u32 *max_sched_domains, + u32 nr); +#endif /* __PERF_HEADER_H */ diff --git a/tools/perf/util/help-unknown-cmd.c b/tools/perf/util/help-unknown-cmd.c new file mode 100644 index 000000000..a0a46e34f --- /dev/null +++ b/tools/perf/util/help-unknown-cmd.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "cache.h" +#include "config.h" +#include +#include +#include +#include +#include "../builtin.h" +#include "levenshtein.h" +#include + +static int autocorrect; + +static int perf_unknown_cmd_config(const char *var, const char *value, + void *cb __maybe_unused) +{ + if (!strcmp(var, "help.autocorrect")) + return perf_config_int(&autocorrect, var,value); + + return 0; +} + +static int levenshtein_compare(const void *p1, const void *p2) +{ + const struct cmdname *const *c1 = p1, *const *c2 = p2; + const char *s1 = (*c1)->name, *s2 = (*c2)->name; + int l1 = (*c1)->len; + int l2 = (*c2)->len; + return l1 != l2 ? l1 - l2 : strcmp(s1, s2); +} + +static int add_cmd_list(struct cmdnames *cmds, struct cmdnames *old) +{ + unsigned int i, nr = cmds->cnt + old->cnt; + void *tmp; + + if (nr > cmds->alloc) { + /* Choose bigger one to alloc */ + if (alloc_nr(cmds->alloc) < nr) + cmds->alloc = nr; + else + cmds->alloc = alloc_nr(cmds->alloc); + tmp = realloc(cmds->names, cmds->alloc * sizeof(*cmds->names)); + if (!tmp) + return -1; + cmds->names = tmp; + } + for (i = 0; i < old->cnt; i++) + cmds->names[cmds->cnt++] = old->names[i]; + zfree(&old->names); + old->cnt = 0; + return 0; +} + +const char *help_unknown_cmd(const char *cmd, struct cmdnames *main_cmds) +{ + unsigned int i, n = 0, best_similarity = 0; + struct cmdnames other_cmds; + + memset(&other_cmds, 0, sizeof(other_cmds)); + + perf_config(perf_unknown_cmd_config, NULL); + + load_command_list("perf-", main_cmds, &other_cmds); + + if (add_cmd_list(main_cmds, &other_cmds) < 0) { + fprintf(stderr, "ERROR: Failed to allocate command list for unknown command.\n"); + goto end; + } + qsort(main_cmds->names, main_cmds->cnt, + sizeof(main_cmds->names), cmdname_compare); + uniq(main_cmds); + + if (main_cmds->cnt) { + /* This reuses cmdname->len for similarity index */ + for (i = 0; i < main_cmds->cnt; ++i) { + main_cmds->names[i]->len = + levenshtein(cmd, main_cmds->names[i]->name, + /*swap_penalty=*/0, + /*substition_penality=*/2, + /*insertion_penality=*/1, + /*deletion_penalty=*/1); + } + qsort(main_cmds->names, main_cmds->cnt, + sizeof(*main_cmds->names), levenshtein_compare); + + best_similarity = main_cmds->names[0]->len; + n = 1; + while (n < main_cmds->cnt && best_similarity == main_cmds->names[n]->len) + ++n; + } + + if (autocorrect && n == 1) { + const char *assumed = main_cmds->names[0]->name; + + main_cmds->names[0] = NULL; + clean_cmdnames(&other_cmds); + fprintf(stderr, "WARNING: You called a perf program named '%s', " + "which does not exist.\n" + "Continuing under the assumption that you meant '%s'\n", + cmd, assumed); + if (autocorrect > 0) { + fprintf(stderr, "in %0.1f seconds automatically...\n", + (float)autocorrect/10.0); + poll(NULL, 0, autocorrect * 100); + } + return assumed; + } + + fprintf(stderr, "perf: '%s' is not a perf-command. See 'perf --help'.\n", cmd); + + if (main_cmds->cnt && best_similarity < 6) { + fprintf(stderr, "\nDid you mean %s?\n", + n < 2 ? "this": "one of these"); + + for (i = 0; i < n; i++) + fprintf(stderr, "\t%s\n", main_cmds->names[i]->name); + } +end: + clean_cmdnames(&other_cmds); + return NULL; +} diff --git a/tools/perf/util/help-unknown-cmd.h b/tools/perf/util/help-unknown-cmd.h new file mode 100644 index 000000000..e69de29bb diff --git a/tools/perf/util/hisi-ptt-decoder/Build b/tools/perf/util/hisi-ptt-decoder/Build new file mode 100644 index 000000000..2ee0eb731 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/Build @@ -0,0 +1 @@ +perf-util-y += hisi-ptt-pkt-decoder.o diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c new file mode 100644 index 000000000..a17c423a5 --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.c @@ -0,0 +1,164 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include + +#include "../color.h" +#include "hisi-ptt-pkt-decoder.h" + +/* + * For 8DW format, the bit[31:11] of DW0 is always 0x1fffff, which can be + * used to distinguish the data format. + * 8DW format is like: + * bits [ 31:11 ][ 10:0 ] + * |---------------------------------------|-------------------| + * DW0 [ 0x1fffff ][ Reserved (0x7ff) ] + * DW1 [ Prefix ] + * DW2 [ Header DW0 ] + * DW3 [ Header DW1 ] + * DW4 [ Header DW2 ] + * DW5 [ Header DW3 ] + * DW6 [ Reserved (0x0) ] + * DW7 [ Time ] + * + * 4DW format is like: + * bits [31:30] [ 29:25 ][24][23][22][21][ 20:11 ][ 10:0 ] + * |-----|---------|---|---|---|---|-------------|-------------| + * DW0 [ Fmt ][ Type ][T9][T8][TH][SO][ Length ][ Time ] + * DW1 [ Header DW1 ] + * DW2 [ Header DW2 ] + * DW3 [ Header DW3 ] + */ + +enum hisi_ptt_8dw_pkt_field_type { + HISI_PTT_8DW_CHK_AND_RSV0, + HISI_PTT_8DW_PREFIX, + HISI_PTT_8DW_HEAD0, + HISI_PTT_8DW_HEAD1, + HISI_PTT_8DW_HEAD2, + HISI_PTT_8DW_HEAD3, + HISI_PTT_8DW_RSV1, + HISI_PTT_8DW_TIME, + HISI_PTT_8DW_TYPE_MAX +}; + +enum hisi_ptt_4dw_pkt_field_type { + HISI_PTT_4DW_HEAD1, + HISI_PTT_4DW_HEAD2, + HISI_PTT_4DW_HEAD3, + HISI_PTT_4DW_TYPE_MAX +}; + +static const char * const hisi_ptt_8dw_pkt_field_name[] = { + [HISI_PTT_8DW_PREFIX] = "Prefix", + [HISI_PTT_8DW_HEAD0] = "Header DW0", + [HISI_PTT_8DW_HEAD1] = "Header DW1", + [HISI_PTT_8DW_HEAD2] = "Header DW2", + [HISI_PTT_8DW_HEAD3] = "Header DW3", + [HISI_PTT_8DW_TIME] = "Time" +}; + +static const char * const hisi_ptt_4dw_pkt_field_name[] = { + [HISI_PTT_4DW_HEAD1] = "Header DW1", + [HISI_PTT_4DW_HEAD2] = "Header DW2", + [HISI_PTT_4DW_HEAD3] = "Header DW3", +}; + +union hisi_ptt_4dw { + struct { + uint32_t format : 2; + uint32_t type : 5; + uint32_t t9 : 1; + uint32_t t8 : 1; + uint32_t th : 1; + uint32_t so : 1; + uint32_t len : 10; + uint32_t time : 11; + }; + uint32_t value; +}; + +static void hisi_ptt_print_pkt(const unsigned char *buf, int pos, const char *desc) +{ + const char *color = PERF_COLOR_BLUE; + int i; + + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) + color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + color_fprintf(stdout, color, " %s\n", desc); +} + +static int hisi_ptt_8dw_kpt_desc(const unsigned char *buf, int pos) +{ + int i; + + for (i = 0; i < HISI_PTT_8DW_TYPE_MAX; i++) { + /* Do not show 8DW check field and reserved fields */ + if (i == HISI_PTT_8DW_CHK_AND_RSV0 || i == HISI_PTT_8DW_RSV1) { + pos += HISI_PTT_FIELD_LENTH; + continue; + } + + hisi_ptt_print_pkt(buf, pos, hisi_ptt_8dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_8DW_PKT]; +} + +static void hisi_ptt_4dw_print_dw0(const unsigned char *buf, int pos) +{ + const char *color = PERF_COLOR_BLUE; + union hisi_ptt_4dw dw0; + int i; + + dw0.value = *(uint32_t *)(buf + pos); + printf("."); + color_fprintf(stdout, color, " %08x: ", pos); + for (i = 0; i < HISI_PTT_FIELD_LENTH; i++) + color_fprintf(stdout, color, "%02x ", buf[pos + i]); + for (i = 0; i < HISI_PTT_MAX_SPACE_LEN; i++) + color_fprintf(stdout, color, " "); + + color_fprintf(stdout, color, + " %s %x %s %x %s %x %s %x %s %x %s %x %s %x %s %x\n", + "Format", dw0.format, "Type", dw0.type, "T9", dw0.t9, + "T8", dw0.t8, "TH", dw0.th, "SO", dw0.so, "Length", + dw0.len, "Time", dw0.time); +} + +static int hisi_ptt_4dw_kpt_desc(const unsigned char *buf, int pos) +{ + int i; + + hisi_ptt_4dw_print_dw0(buf, pos); + pos += HISI_PTT_FIELD_LENTH; + + for (i = 0; i < HISI_PTT_4DW_TYPE_MAX; i++) { + hisi_ptt_print_pkt(buf, pos, hisi_ptt_4dw_pkt_field_name[i]); + pos += HISI_PTT_FIELD_LENTH; + } + + return hisi_ptt_pkt_size[HISI_PTT_4DW_PKT]; +} + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type) +{ + if (type == HISI_PTT_8DW_PKT) + return hisi_ptt_8dw_kpt_desc(buf, pos); + + return hisi_ptt_4dw_kpt_desc(buf, pos); +} diff --git a/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h new file mode 100644 index 000000000..e78f1b5bc --- /dev/null +++ b/tools/perf/util/hisi-ptt-decoder/hisi-ptt-pkt-decoder.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__HISI_PTT_PKT_DECODER_H__ +#define INCLUDE__HISI_PTT_PKT_DECODER_H__ + +#include +#include + +#define HISI_PTT_8DW_CHECK_MASK GENMASK(31, 11) +#define HISI_PTT_IS_8DW_PKT GENMASK(31, 11) +#define HISI_PTT_MAX_SPACE_LEN 10 +#define HISI_PTT_FIELD_LENTH 4 + +enum hisi_ptt_pkt_type { + HISI_PTT_4DW_PKT, + HISI_PTT_8DW_PKT, + HISI_PTT_PKT_MAX +}; + +static int hisi_ptt_pkt_size[] = { + [HISI_PTT_4DW_PKT] = 16, + [HISI_PTT_8DW_PKT] = 32, +}; + +int hisi_ptt_pkt_desc(const unsigned char *buf, int pos, enum hisi_ptt_pkt_type type); + +#endif diff --git a/tools/perf/util/hisi-ptt.c b/tools/perf/util/hisi-ptt.c new file mode 100644 index 000000000..e4cc4785f --- /dev/null +++ b/tools/perf/util/hisi-ptt.c @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "auxtrace.h" +#include "color.h" +#include "debug.h" +#include "evsel.h" +#include "hisi-ptt.h" +#include "hisi-ptt-decoder/hisi-ptt-pkt-decoder.h" +#include "machine.h" +#include "session.h" +#include "tool.h" +#include + +struct hisi_ptt { + struct auxtrace auxtrace; + u32 auxtrace_type; + struct perf_session *session; + struct machine *machine; + u32 pmu_type; +}; + +static enum hisi_ptt_pkt_type hisi_ptt_check_packet_type(unsigned char *buf) +{ + uint32_t head = *(uint32_t *)buf; + + if ((HISI_PTT_8DW_CHECK_MASK & head) == HISI_PTT_IS_8DW_PKT) + return HISI_PTT_8DW_PKT; + + return HISI_PTT_4DW_PKT; +} + +static void hisi_ptt_dump(struct hisi_ptt *ptt __maybe_unused, + unsigned char *buf, size_t len) +{ + const char *color = PERF_COLOR_BLUE; + enum hisi_ptt_pkt_type type; + size_t pos = 0; + int pkt_len; + + type = hisi_ptt_check_packet_type(buf); + len = round_down(len, hisi_ptt_pkt_size[type]); + color_fprintf(stdout, color, ". ... HISI PTT data: size %zu bytes\n", + len); + + while (len > 0) { + pkt_len = hisi_ptt_pkt_desc(buf, pos, type); + if (!pkt_len) + color_fprintf(stdout, color, " Bad packet!\n"); + + pos += pkt_len; + len -= pkt_len; + } +} + +static void hisi_ptt_dump_event(struct hisi_ptt *ptt, unsigned char *buf, + size_t len) +{ + printf(".\n"); + + hisi_ptt_dump(ptt, buf, len); +} + +static int hisi_ptt_process_event(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused, + struct perf_sample *sample __maybe_unused, + const struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static int hisi_ptt_process_auxtrace_event(struct perf_session *session, + union perf_event *event, + const struct perf_tool *tool __maybe_unused) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + int fd = perf_data__fd(session->data); + int size = event->auxtrace.size; + void *data = malloc(size); + off_t data_offset; + int err; + + if (!data) + return -errno; + + if (perf_data__is_pipe(session->data)) { + data_offset = 0; + } else { + data_offset = lseek(fd, 0, SEEK_CUR); + if (data_offset == -1) { + free(data); + return -errno; + } + } + + err = readn(fd, data, size); + if (err != (ssize_t)size) { + free(data); + return -errno; + } + + if (dump_trace) + hisi_ptt_dump_event(ptt, data, size); + + free(data); + return 0; +} + +static int hisi_ptt_flush(struct perf_session *session __maybe_unused, + const struct perf_tool *tool __maybe_unused) +{ + return 0; +} + +static void hisi_ptt_free_events(struct perf_session *session __maybe_unused) +{ +} + +static void hisi_ptt_free(struct perf_session *session) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, + auxtrace); + + session->auxtrace = NULL; + free(ptt); +} + +static bool hisi_ptt_evsel_is_auxtrace(struct perf_session *session, + struct evsel *evsel) +{ + struct hisi_ptt *ptt = container_of(session->auxtrace, struct hisi_ptt, auxtrace); + + return evsel->core.attr.type == ptt->pmu_type; +} + +static void hisi_ptt_print_info(__u64 type) +{ + if (!dump_trace) + return; + + fprintf(stdout, " PMU Type %" PRId64 "\n", (s64) type); +} + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session) +{ + struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; + struct hisi_ptt *ptt; + + if (auxtrace_info->header.size < HISI_PTT_AUXTRACE_PRIV_SIZE + + sizeof(struct perf_record_auxtrace_info)) + return -EINVAL; + + ptt = zalloc(sizeof(*ptt)); + if (!ptt) + return -ENOMEM; + + ptt->session = session; + ptt->machine = &session->machines.host; /* No kvm support */ + ptt->auxtrace_type = auxtrace_info->type; + ptt->pmu_type = auxtrace_info->priv[0]; + + ptt->auxtrace.process_event = hisi_ptt_process_event; + ptt->auxtrace.process_auxtrace_event = hisi_ptt_process_auxtrace_event; + ptt->auxtrace.flush_events = hisi_ptt_flush; + ptt->auxtrace.free_events = hisi_ptt_free_events; + ptt->auxtrace.free = hisi_ptt_free; + ptt->auxtrace.evsel_is_auxtrace = hisi_ptt_evsel_is_auxtrace; + session->auxtrace = &ptt->auxtrace; + + hisi_ptt_print_info(auxtrace_info->priv[0]); + + return 0; +} diff --git a/tools/perf/util/hisi-ptt.h b/tools/perf/util/hisi-ptt.h new file mode 100644 index 000000000..2db9b4056 --- /dev/null +++ b/tools/perf/util/hisi-ptt.h @@ -0,0 +1,19 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * HiSilicon PCIe Trace and Tuning (PTT) support + * Copyright (c) 2022 HiSilicon Technologies Co., Ltd. + */ + +#ifndef INCLUDE__PERF_HISI_PTT_H__ +#define INCLUDE__PERF_HISI_PTT_H__ + +#define HISI_PTT_PMU_NAME "hisi_ptt" +#define HISI_PTT_AUXTRACE_PRIV_SIZE sizeof(u64) + +struct auxtrace_record *hisi_ptt_recording_init(int *err, + struct perf_pmu *hisi_ptt_pmu); + +int hisi_ptt_process_auxtrace_info(union perf_event *event, + struct perf_session *session); + +#endif diff --git a/tools/perf/util/hist.c b/tools/perf/util/hist.c new file mode 100644 index 000000000..c93915625 --- /dev/null +++ b/tools/perf/util/hist.c @@ -0,0 +1,3099 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "callchain.h" +#include "debug.h" +#include "dso.h" +#include "build-id.h" +#include "hist.h" +#include "kvm-stat.h" +#include "map.h" +#include "map_symbol.h" +#include "branch.h" +#include "mem-events.h" +#include "mem-info.h" +#include "session.h" +#include "namespaces.h" +#include "cgroup.h" +#include "sort.h" +#include "units.h" +#include "evlist.h" +#include "evsel.h" +#include "annotate.h" +#include "srcline.h" +#include "symbol.h" +#include "thread.h" +#include "block-info.h" +#include "ui/progress.h" +#include +#include +#include +#include +#include +#include +#include +#include + +static int64_t hist_entry__cmp(struct hist_entry *left, struct hist_entry *right); +static int64_t hist_entry__collapse(struct hist_entry *left, struct hist_entry *right); + +static bool hists__filter_entry_by_dso(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_thread(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_symbol(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_socket(struct hists *hists, + struct hist_entry *he); +static bool hists__filter_entry_by_parallelism(struct hists *hists, + struct hist_entry *he); + +u16 hists__col_len(struct hists *hists, enum hist_column col) +{ + return hists->col_len[col]; +} + +void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len) +{ + hists->col_len[col] = len; +} + +bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len) +{ + if (len > hists__col_len(hists, col)) { + hists__set_col_len(hists, col, len); + return true; + } + return false; +} + +void hists__reset_col_len(struct hists *hists) +{ + enum hist_column col; + + for (col = 0; col < HISTC_NR_COLS; ++col) + hists__set_col_len(hists, col, 0); +} + +static void hists__set_unres_dso_col_len(struct hists *hists, int dso) +{ + const unsigned int unresolved_col_width = BITS_PER_LONG / 4; + + if (hists__col_len(hists, dso) < unresolved_col_width && + !symbol_conf.col_width_list_str && !symbol_conf.field_sep && + !symbol_conf.dso_list) + hists__set_col_len(hists, dso, unresolved_col_width); +} + +void hists__calc_col_len(struct hists *hists, struct hist_entry *h) +{ + const unsigned int unresolved_col_width = BITS_PER_LONG / 4; + int symlen; + u16 len; + + if (h->block_info) + return; + /* + * +4 accounts for '[x] ' priv level info + * +2 accounts for 0x prefix on raw addresses + * +3 accounts for ' y ' symtab origin info + */ + if (h->ms.sym) { + symlen = h->ms.sym->namelen + 4; + if (verbose > 0) + symlen += BITS_PER_LONG / 4 + 2 + 3; + hists__new_col_len(hists, HISTC_SYMBOL, symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_SYMBOL, symlen); + hists__set_unres_dso_col_len(hists, HISTC_DSO); + } + + len = thread__comm_len(h->thread); + if (hists__new_col_len(hists, HISTC_COMM, len)) + hists__set_col_len(hists, HISTC_THREAD, len + 8); + if (hists->hpp_list->comm_nodigit) + hists__new_col_len(hists, HISTC_COMM_NODIGIT, + (u16) sort__comm_nodigit_len(h)); + + if (h->ms.map) { + len = dso__name_len(map__dso(h->ms.map)); + hists__new_col_len(hists, HISTC_DSO, len); + } + + if (h->parent) + hists__new_col_len(hists, HISTC_PARENT, h->parent->namelen); + + if (h->branch_info) { + if (h->branch_info->from.ms.sym) { + symlen = (int)h->branch_info->from.ms.sym->namelen + 4; + if (verbose > 0) + symlen += BITS_PER_LONG / 4 + 2 + 3; + hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen); + + symlen = dso__name_len(map__dso(h->branch_info->from.ms.map)); + hists__new_col_len(hists, HISTC_DSO_FROM, symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_SYMBOL_FROM, symlen); + hists__new_col_len(hists, HISTC_ADDR_FROM, symlen); + hists__set_unres_dso_col_len(hists, HISTC_DSO_FROM); + } + + if (h->branch_info->to.ms.sym) { + symlen = (int)h->branch_info->to.ms.sym->namelen + 4; + if (verbose > 0) + symlen += BITS_PER_LONG / 4 + 2 + 3; + hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen); + + symlen = dso__name_len(map__dso(h->branch_info->to.ms.map)); + hists__new_col_len(hists, HISTC_DSO_TO, symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_SYMBOL_TO, symlen); + hists__new_col_len(hists, HISTC_ADDR_TO, symlen); + hists__set_unres_dso_col_len(hists, HISTC_DSO_TO); + } + + if (h->branch_info->srcline_from) + hists__new_col_len(hists, HISTC_SRCLINE_FROM, + strlen(h->branch_info->srcline_from)); + if (h->branch_info->srcline_to) + hists__new_col_len(hists, HISTC_SRCLINE_TO, + strlen(h->branch_info->srcline_to)); + } + + if (h->mem_info) { + if (mem_info__daddr(h->mem_info)->ms.sym) { + symlen = (int)mem_info__daddr(h->mem_info)->ms.sym->namelen + 4 + + unresolved_col_width + 2; + hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, + symlen); + hists__new_col_len(hists, HISTC_MEM_DCACHELINE, + symlen + 1); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, + symlen); + hists__new_col_len(hists, HISTC_MEM_DCACHELINE, + symlen); + } + + if (mem_info__iaddr(h->mem_info)->ms.sym) { + symlen = (int)mem_info__iaddr(h->mem_info)->ms.sym->namelen + 4 + + unresolved_col_width + 2; + hists__new_col_len(hists, HISTC_MEM_IADDR_SYMBOL, + symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_MEM_IADDR_SYMBOL, + symlen); + } + + if (mem_info__daddr(h->mem_info)->ms.map) { + symlen = dso__name_len(map__dso(mem_info__daddr(h->mem_info)->ms.map)); + hists__new_col_len(hists, HISTC_MEM_DADDR_DSO, + symlen); + } else { + symlen = unresolved_col_width + 4 + 2; + hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO); + } + + hists__new_col_len(hists, HISTC_MEM_PHYS_DADDR, + unresolved_col_width + 4 + 2); + + hists__new_col_len(hists, HISTC_MEM_DATA_PAGE_SIZE, + unresolved_col_width + 4 + 2); + + } else { + symlen = unresolved_col_width + 4 + 2; + hists__new_col_len(hists, HISTC_MEM_DADDR_SYMBOL, symlen); + hists__new_col_len(hists, HISTC_MEM_IADDR_SYMBOL, symlen); + hists__set_unres_dso_col_len(hists, HISTC_MEM_DADDR_DSO); + } + + hists__new_col_len(hists, HISTC_CGROUP, 6); + hists__new_col_len(hists, HISTC_CGROUP_ID, 20); + hists__new_col_len(hists, HISTC_PARALLELISM, 11); + hists__new_col_len(hists, HISTC_CPU, 3); + hists__new_col_len(hists, HISTC_SOCKET, 6); + hists__new_col_len(hists, HISTC_MEM_LOCKED, 6); + hists__new_col_len(hists, HISTC_MEM_TLB, 22); + hists__new_col_len(hists, HISTC_MEM_SNOOP, 12); + hists__new_col_len(hists, HISTC_MEM_LVL, 36 + 3); + hists__new_col_len(hists, HISTC_LOCAL_WEIGHT, 12); + hists__new_col_len(hists, HISTC_GLOBAL_WEIGHT, 12); + hists__new_col_len(hists, HISTC_MEM_BLOCKED, 10); + hists__new_col_len(hists, HISTC_LOCAL_INS_LAT, 13); + hists__new_col_len(hists, HISTC_GLOBAL_INS_LAT, 13); + hists__new_col_len(hists, HISTC_LOCAL_P_STAGE_CYC, 13); + hists__new_col_len(hists, HISTC_GLOBAL_P_STAGE_CYC, 13); + hists__new_col_len(hists, HISTC_ADDR, BITS_PER_LONG / 4 + 2); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_PREDICTED, 9); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_ABORT, 5); + hists__new_col_len(hists, HISTC_CALLCHAIN_BRANCH_CYCLES, 6); + + if (symbol_conf.nanosecs) + hists__new_col_len(hists, HISTC_TIME, 16); + else + hists__new_col_len(hists, HISTC_TIME, 12); + hists__new_col_len(hists, HISTC_CODE_PAGE_SIZE, 6); + + if (h->srcline) { + len = MAX(strlen(h->srcline), strlen(sort_srcline.se_header)); + hists__new_col_len(hists, HISTC_SRCLINE, len); + } + + if (h->srcfile) + hists__new_col_len(hists, HISTC_SRCFILE, strlen(h->srcfile)); + + if (h->transaction) + hists__new_col_len(hists, HISTC_TRANSACTION, + hist_entry__transaction_len()); + + if (h->trace_output) + hists__new_col_len(hists, HISTC_TRACE, strlen(h->trace_output)); + + if (h->cgroup) { + const char *cgrp_name = "unknown"; + struct cgroup *cgrp = cgroup__find(maps__machine(thread__maps(h->ms.thread))->env, + h->cgroup); + if (cgrp != NULL) + cgrp_name = cgrp->name; + + hists__new_col_len(hists, HISTC_CGROUP, strlen(cgrp_name)); + } +} + +void hists__output_recalc_col_len(struct hists *hists, int max_rows) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + int row = 0; + + hists__reset_col_len(hists); + + while (next && row++ < max_rows) { + n = rb_entry(next, struct hist_entry, rb_node); + if (!n->filtered) + hists__calc_col_len(hists, n); + next = rb_next(&n->rb_node); + } +} + +static void he_stat__add_cpumode_period(struct he_stat *he_stat, + unsigned int cpumode, u64 period) +{ + switch (cpumode) { + case PERF_RECORD_MISC_KERNEL: + he_stat->period_sys += period; + break; + case PERF_RECORD_MISC_USER: + he_stat->period_us += period; + break; + case PERF_RECORD_MISC_GUEST_KERNEL: + he_stat->period_guest_sys += period; + break; + case PERF_RECORD_MISC_GUEST_USER: + he_stat->period_guest_us += period; + break; + default: + break; + } +} + +static long hist_time(unsigned long htime) +{ + unsigned long time_quantum = symbol_conf.time_quantum; + if (time_quantum) + return (htime / time_quantum) * time_quantum; + return htime; +} + +static void he_stat__add_period(struct he_stat *he_stat, u64 period, u64 latency) +{ + he_stat->period += period; + he_stat->latency += latency; + he_stat->nr_events += 1; +} + +static void he_stat__add_stat(struct he_stat *dest, struct he_stat *src) +{ + dest->period += src->period; + dest->period_sys += src->period_sys; + dest->period_us += src->period_us; + dest->period_guest_sys += src->period_guest_sys; + dest->period_guest_us += src->period_guest_us; + dest->weight1 += src->weight1; + dest->weight2 += src->weight2; + dest->weight3 += src->weight3; + dest->nr_events += src->nr_events; + dest->latency += src->latency; +} + +static void he_stat__decay(struct he_stat *he_stat) +{ + he_stat->period = (he_stat->period * 7) / 8; + he_stat->nr_events = (he_stat->nr_events * 7) / 8; + he_stat->weight1 = (he_stat->weight1 * 7) / 8; + he_stat->weight2 = (he_stat->weight2 * 7) / 8; + he_stat->weight3 = (he_stat->weight3 * 7) / 8; + he_stat->latency = (he_stat->latency * 7) / 8; +} + +static int hists__update_mem_stat(struct hists *hists, struct hist_entry *he, + struct mem_info *mi, u64 period) +{ + if (hists->nr_mem_stats == 0) + return 0; + + if (he->mem_stat == NULL) { + he->mem_stat = calloc(hists->nr_mem_stats, sizeof(*he->mem_stat)); + if (he->mem_stat == NULL) + return -1; + } + + for (int i = 0; i < hists->nr_mem_stats; i++) { + int idx = mem_stat_index(hists->mem_stat_types[i], + mem_info__const_data_src(mi)->val); + + assert(0 <= idx && idx < MEM_STAT_LEN); + he->mem_stat[i].entries[idx] += period; + hists->mem_stat_total[i].entries[idx] += period; + } + return 0; +} + +static void hists__add_mem_stat(struct hists *hists, struct hist_entry *dst, + struct hist_entry *src) +{ + if (hists->nr_mem_stats == 0) + return; + + for (int i = 0; i < hists->nr_mem_stats; i++) { + for (int k = 0; k < MEM_STAT_LEN; k++) + dst->mem_stat[i].entries[k] += src->mem_stat[i].entries[k]; + } +} + +static int hists__clone_mem_stat(struct hists *hists, struct hist_entry *dst, + struct hist_entry *src) +{ + if (hists->nr_mem_stats == 0) + return 0; + + dst->mem_stat = calloc(hists->nr_mem_stats, sizeof(*dst->mem_stat)); + if (dst->mem_stat == NULL) + return -1; + + for (int i = 0; i < hists->nr_mem_stats; i++) { + for (int k = 0; k < MEM_STAT_LEN; k++) + dst->mem_stat[i].entries[k] = src->mem_stat[i].entries[k]; + } + return 0; +} + +static void hists__decay_mem_stat(struct hists *hists, struct hist_entry *he) +{ + if (hists->nr_mem_stats == 0) + return; + + for (int i = 0; i < hists->nr_mem_stats; i++) { + for (int k = 0; k < MEM_STAT_LEN; k++) + he->mem_stat[i].entries[k] = (he->mem_stat[i].entries[k] * 7) / 8; + } +} + +static void hists__delete_entry(struct hists *hists, struct hist_entry *he); + +static bool hists__decay_entry(struct hists *hists, struct hist_entry *he) +{ + u64 prev_period = he->stat.period; + u64 prev_latency = he->stat.latency; + + if (prev_period == 0) + return true; + + he_stat__decay(&he->stat); + if (symbol_conf.cumulate_callchain) + he_stat__decay(he->stat_acc); + decay_callchain(he->callchain); + hists__decay_mem_stat(hists, he); + + if (!he->depth) { + u64 period_diff = prev_period - he->stat.period; + u64 latency_diff = prev_latency - he->stat.latency; + + hists->stats.total_period -= period_diff; + hists->stats.total_latency -= latency_diff; + if (!he->filtered) { + hists->stats.total_non_filtered_period -= period_diff; + hists->stats.total_non_filtered_latency -= latency_diff; + } + } + + if (!he->leaf) { + struct hist_entry *child; + struct rb_node *node = rb_first_cached(&he->hroot_out); + while (node) { + child = rb_entry(node, struct hist_entry, rb_node); + node = rb_next(node); + + if (hists__decay_entry(hists, child)) + hists__delete_entry(hists, child); + } + } + + return he->stat.period == 0 && he->stat.latency == 0; +} + +static void hists__delete_entry(struct hists *hists, struct hist_entry *he) +{ + struct rb_root_cached *root_in; + struct rb_root_cached *root_out; + + if (he->parent_he) { + root_in = &he->parent_he->hroot_in; + root_out = &he->parent_he->hroot_out; + } else { + if (hists__has(hists, need_collapse)) + root_in = &hists->entries_collapsed; + else + root_in = hists->entries_in; + root_out = &hists->entries; + } + + rb_erase_cached(&he->rb_node_in, root_in); + rb_erase_cached(&he->rb_node, root_out); + + --hists->nr_entries; + if (!he->filtered) + --hists->nr_non_filtered_entries; + + hist_entry__delete(he); +} + +void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node); + next = rb_next(&n->rb_node); + if (((zap_user && n->level == '.') || + (zap_kernel && n->level != '.') || + hists__decay_entry(hists, n))) { + hists__delete_entry(hists, n); + } + } +} + +void hists__delete_entries(struct hists *hists) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node); + next = rb_next(&n->rb_node); + + hists__delete_entry(hists, n); + } +} + +struct hist_entry *hists__get_entry(struct hists *hists, int idx) +{ + struct rb_node *next = rb_first_cached(&hists->entries); + struct hist_entry *n; + int i = 0; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node); + if (i == idx) + return n; + + next = rb_next(&n->rb_node); + i++; + } + + return NULL; +} + +/* + * histogram, sorted on item, collects periods + */ + +static int hist_entry__init(struct hist_entry *he, + struct hist_entry *template, + bool sample_self, + size_t callchain_size) +{ + *he = *template; + he->callchain_size = callchain_size; + + if (symbol_conf.cumulate_callchain) { + he->stat_acc = malloc(sizeof(he->stat)); + if (he->stat_acc == NULL) + return -ENOMEM; + memcpy(he->stat_acc, &he->stat, sizeof(he->stat)); + if (!sample_self) + memset(&he->stat, 0, sizeof(he->stat)); + } + + he->ms.thread = thread__get(he->ms.thread); + he->ms.map = map__get(he->ms.map); + + if (he->branch_info) { + /* + * This branch info is (a part of) allocated from + * sample__resolve_bstack() and will be freed after + * adding new entries. So we need to save a copy. + */ + he->branch_info = malloc(sizeof(*he->branch_info)); + if (he->branch_info == NULL) + goto err; + + memcpy(he->branch_info, template->branch_info, + sizeof(*he->branch_info)); + + he->branch_info->from.ms.thread = thread__get(he->branch_info->from.ms.thread); + he->branch_info->from.ms.map = map__get(he->branch_info->from.ms.map); + he->branch_info->to.ms.thread = thread__get(he->branch_info->to.ms.thread); + he->branch_info->to.ms.map = map__get(he->branch_info->to.ms.map); + } + + if (he->mem_info) { + he->mem_info = mem_info__clone(template->mem_info); + if (he->mem_info == NULL) + goto err_infos; + } + + if (hist_entry__has_callchains(he) && symbol_conf.use_callchain) + callchain_init(he->callchain); + + if (he->raw_data) { + he->raw_data = memdup(he->raw_data, he->raw_size); + if (he->raw_data == NULL) + goto err_infos; + } + + if (he->srcline && he->srcline != SRCLINE_UNKNOWN) { + he->srcline = strdup(he->srcline); + if (he->srcline == NULL) + goto err_rawdata; + } + + if (symbol_conf.res_sample) { + he->res_samples = calloc(symbol_conf.res_sample, + sizeof(struct res_sample)); + if (!he->res_samples) + goto err_srcline; + } + + INIT_LIST_HEAD(&he->pairs.node); + he->thread = thread__get(he->thread); + he->hroot_in = RB_ROOT_CACHED; + he->hroot_out = RB_ROOT_CACHED; + + if (!symbol_conf.report_hierarchy) + he->leaf = true; + + return 0; + +err_srcline: + zfree(&he->srcline); + +err_rawdata: + zfree(&he->raw_data); + +err_infos: + if (he->branch_info) { + map_symbol__exit(&he->branch_info->from.ms); + map_symbol__exit(&he->branch_info->to.ms); + zfree(&he->branch_info); + } + if (he->mem_info) + mem_info__zput(he->mem_info); +err: + map_symbol__exit(&he->ms); + zfree(&he->stat_acc); + return -ENOMEM; +} + +static void *hist_entry__zalloc(size_t size) +{ + return zalloc(size + sizeof(struct hist_entry)); +} + +static void hist_entry__free(void *ptr) +{ + free(ptr); +} + +static struct hist_entry_ops default_ops = { + .new = hist_entry__zalloc, + .free = hist_entry__free, +}; + +static struct hist_entry *hist_entry__new(struct hist_entry *template, + bool sample_self) +{ + struct hist_entry_ops *ops = template->ops; + size_t callchain_size = 0; + struct hist_entry *he; + int err = 0; + + if (!ops) + ops = template->ops = &default_ops; + + if (symbol_conf.use_callchain) + callchain_size = sizeof(struct callchain_root); + + he = ops->new(callchain_size); + if (he) { + err = hist_entry__init(he, template, sample_self, callchain_size); + if (err) { + ops->free(he); + he = NULL; + } + } + return he; +} + +static filter_mask_t symbol__parent_filter(const struct symbol *parent) +{ + if (symbol_conf.exclude_other && parent == NULL) + return 1 << HIST_FILTER__PARENT; + return 0; +} + +static void hist_entry__add_callchain_period(struct hist_entry *he, u64 period, u64 latency) +{ + if (!hist_entry__has_callchains(he) || !symbol_conf.use_callchain) + return; + + he->hists->callchain_period += period; + he->hists->callchain_latency += latency; + if (!he->filtered) { + he->hists->callchain_non_filtered_period += period; + he->hists->callchain_non_filtered_latency += latency; + } +} + +static struct hist_entry *hists__findnew_entry(struct hists *hists, + struct hist_entry *entry, + const struct addr_location *al, + bool sample_self) +{ + struct rb_node **p; + struct rb_node *parent = NULL; + struct hist_entry *he; + int64_t cmp; + u64 period = entry->stat.period; + u64 latency = entry->stat.latency; + bool leftmost = true; + + p = &hists->entries_in->rb_root.rb_node; + + while (*p != NULL) { + parent = *p; + he = rb_entry(parent, struct hist_entry, rb_node_in); + + /* + * Make sure that it receives arguments in a same order as + * hist_entry__collapse() so that we can use an appropriate + * function when searching an entry regardless which sort + * keys were used. + */ + cmp = hist_entry__cmp(he, entry); + if (!cmp) { + if (sample_self) { + he_stat__add_stat(&he->stat, &entry->stat); + hist_entry__add_callchain_period(he, period, latency); + } + if (symbol_conf.cumulate_callchain) + he_stat__add_period(he->stat_acc, period, latency); + + block_info__delete(entry->block_info); + + kvm_info__zput(entry->kvm_info); + + /* If the map of an existing hist_entry has + * become out-of-date due to an exec() or + * similar, update it. Otherwise we will + * mis-adjust symbol addresses when computing + * the history counter to increment. + */ + if (hists__has(hists, sym) && he->ms.map != entry->ms.map) { + if (he->ms.sym) { + u64 addr = he->ms.sym->start; + he->ms.sym = map__find_symbol(entry->ms.map, addr); + } + + map__put(he->ms.map); + he->ms.map = map__get(entry->ms.map); + } + goto out; + } + + if (cmp < 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + he = hist_entry__new(entry, sample_self); + if (!he) + return NULL; + + if (sample_self) + hist_entry__add_callchain_period(he, period, latency); + hists->nr_entries++; + + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, hists->entries_in, leftmost); +out: + if (sample_self) + he_stat__add_cpumode_period(&he->stat, al->cpumode, period); + if (symbol_conf.cumulate_callchain) + he_stat__add_cpumode_period(he->stat_acc, al->cpumode, period); + if (hists__update_mem_stat(hists, he, entry->mem_info, period) < 0) { + hist_entry__delete(he); + return NULL; + } + return he; +} + +static unsigned random_max(unsigned high) +{ + unsigned thresh = -high % high; + for (;;) { + unsigned r = random(); + if (r >= thresh) + return r % high; + } +} + +static void hists__res_sample(struct hist_entry *he, struct perf_sample *sample) +{ + struct res_sample *r; + int j; + + if (he->num_res < symbol_conf.res_sample) { + j = he->num_res++; + } else { + j = random_max(symbol_conf.res_sample); + } + r = &he->res_samples[j]; + r->time = sample->time; + r->cpu = sample->cpu; + r->tid = sample->tid; +} + +static struct hist_entry* +__hists__add_entry(struct hists *hists, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct block_info *block_info, + struct perf_sample *sample, + bool sample_self, + struct hist_entry_ops *ops) +{ + struct namespaces *ns = thread__namespaces(al->thread); + struct hist_entry entry = { + .thread = al->thread, + .comm = thread__comm(al->thread), + .cgroup_id = { + .dev = ns ? ns->link_info[CGROUP_NS_INDEX].dev : 0, + .ino = ns ? ns->link_info[CGROUP_NS_INDEX].ino : 0, + }, + .cgroup = sample->cgroup, + .ms = { + .thread = al->thread, + .map = al->map, + .sym = al->sym, + }, + .srcline = (char *) al->srcline, + .socket = al->socket, + .cpu = al->cpu, + .cpumode = al->cpumode, + .ip = al->addr, + .level = al->level, + .code_page_size = sample->code_page_size, + .parallelism = al->parallelism, + .stat = { + .nr_events = 1, + .period = sample->period, + .weight1 = sample->weight, + .weight2 = sample->ins_lat, + .weight3 = sample->weight3, + .latency = al->latency, + }, + .parent = sym_parent, + .filtered = symbol__parent_filter(sym_parent) | al->filtered, + .hists = hists, + .branch_info = bi, + .mem_info = mi, + .kvm_info = ki, + .block_info = block_info, + .transaction = sample->transaction, + .raw_data = sample->raw_data, + .raw_size = sample->raw_size, + .ops = ops, + .time = hist_time(sample->time), + .weight = sample->weight, + .ins_lat = sample->ins_lat, + .weight3 = sample->weight3, + .simd_flags = sample->simd_flags, + }, *he = hists__findnew_entry(hists, &entry, al, sample_self); + + if (!hists->has_callchains && he && he->callchain_size != 0) + hists->has_callchains = true; + if (he && symbol_conf.res_sample) + hists__res_sample(he, sample); + return he; +} + +struct hist_entry *hists__add_entry(struct hists *hists, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self) +{ + return __hists__add_entry(hists, al, sym_parent, bi, mi, ki, NULL, + sample, sample_self, NULL); +} + +struct hist_entry *hists__add_entry_ops(struct hists *hists, + struct hist_entry_ops *ops, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self) +{ + return __hists__add_entry(hists, al, sym_parent, bi, mi, ki, NULL, + sample, sample_self, ops); +} + +struct hist_entry *hists__add_entry_block(struct hists *hists, + struct addr_location *al, + struct block_info *block_info) +{ + struct hist_entry entry = { + .block_info = block_info, + .hists = hists, + .ms = { + .thread = al->thread, + .map = al->map, + .sym = al->sym, + }, + }, *he = hists__findnew_entry(hists, &entry, al, false); + + return he; +} + +static int +iter_next_nop_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_add_next_nop_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_prepare_mem_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct mem_info *mi; + + mi = sample__resolve_mem(sample, al); + if (mi == NULL) + return -ENOMEM; + + iter->mi = mi; + return 0; +} + +static int +iter_add_single_mem_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + u64 cost; + struct mem_info *mi = iter->mi; + struct perf_sample *sample = iter->sample; + struct hists *hists = evsel__hists(sample->evsel); + struct hist_entry *he; + + if (mi == NULL) + return -EINVAL; + + cost = sample->weight; + if (!cost) + cost = 1; + + /* + * must pass period=weight in order to get the correct + * sorting from hists__collapse_resort() which is solely + * based on periods. We want sorting be done on nr_events * weight + * and this is indirectly achieved by passing period=weight here + * and the he_stat__add_period() function. + */ + sample->period = cost; + + he = hists__add_entry(hists, al, iter->parent, NULL, mi, NULL, + sample, true); + if (!he) + return -ENOMEM; + + iter->he = he; + return 0; +} + +static int +iter_finish_mem_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct evsel *evsel = iter->sample->evsel; + struct hists *hists = evsel__hists(evsel); + struct hist_entry *he = iter->he; + int err = -EINVAL; + + if (he == NULL) + goto out; + + hists__inc_nr_samples(hists, he->filtered); + + err = hist_entry__append_callchain(he, iter->sample); + +out: + mem_info__zput(iter->mi); + + iter->he = NULL; + return err; +} + +static int +iter_prepare_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct branch_info *bi; + struct perf_sample *sample = iter->sample; + + bi = sample__resolve_bstack(sample, al); + if (!bi) + return -ENOMEM; + + iter->curr = 0; + iter->total = sample->branch_stack->nr; + + iter->bi = bi; + return 0; +} + +static int +iter_add_single_branch_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct branch_info *bi = iter->bi; + int i = iter->curr; + + if (bi == NULL) + return 0; + + if (iter->curr >= iter->total) + return 0; + + thread__put(al->thread); + al->thread = thread__get(bi[i].to.ms.thread); + map__put(al->map); + al->map = map__get(bi[i].to.ms.map); + al->sym = bi[i].to.ms.sym; + al->addr = bi[i].to.addr; + return 1; +} + +static int +iter_add_next_branch_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct branch_info *bi; + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); + struct hist_entry *he = NULL; + int i = iter->curr; + int err = 0; + + bi = iter->bi; + + if (iter->hide_unresolved && !(bi[i].from.ms.sym && bi[i].to.ms.sym)) + goto out; + + /* + * The report shows the percentage of total branches captured + * and not events sampled. Thus we use a pseudo period of 1. + */ + sample->period = 1; + sample->weight = bi->flags.cycles ? bi->flags.cycles : 1; + + he = hists__add_entry(hists, al, iter->parent, &bi[i], NULL, NULL, + sample, true); + if (he == NULL) + return -ENOMEM; + +out: + iter->he = he; + iter->curr++; + return err; +} + +static void branch_info__exit(struct branch_info *bi) +{ + map_symbol__exit(&bi->from.ms); + map_symbol__exit(&bi->to.ms); + zfree_srcline(&bi->srcline_from); + zfree_srcline(&bi->srcline_to); +} + +static int +iter_finish_branch_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct evsel *evsel = iter->sample->evsel; + struct hists *hists = evsel__hists(evsel); + + for (int i = 0; i < iter->total; i++) + branch_info__exit(&iter->bi[i]); + + if (iter->he) + hists__inc_nr_samples(hists, iter->he->filtered); + + zfree(&iter->bi); + iter->he = NULL; + + return iter->curr >= iter->total ? 0 : -1; +} + +static int +iter_prepare_normal_entry(struct hist_entry_iter *iter __maybe_unused, + struct addr_location *al __maybe_unused) +{ + return 0; +} + +static int +iter_add_single_normal_entry(struct hist_entry_iter *iter, struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hist_entry *he; + + he = hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, + NULL, sample, true); + if (he == NULL) + return -ENOMEM; + + iter->he = he; + return 0; +} + +static int +iter_finish_normal_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct hist_entry *he = iter->he; + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + + if (he == NULL) + return 0; + + iter->he = NULL; + + hists__inc_nr_samples(evsel__hists(evsel), he->filtered); + + return hist_entry__append_callchain(he, sample); +} + +static int +iter_prepare_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + struct hist_entry **he_cache; + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + + if (cursor == NULL) + return -ENOMEM; + + callchain_cursor_commit(cursor); + + /* + * This is for detecting cycles or recursions so that they're + * cumulated only one time to prevent entries more than 100% + * overhead. + */ + he_cache = calloc(cursor->nr + 1, sizeof(*he_cache)); + if (he_cache == NULL) + return -ENOMEM; + + iter->he_cache = he_cache; + iter->curr = 0; + + return 0; +} + +static int +iter_add_single_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hists *hists = evsel__hists(evsel); + struct hist_entry **he_cache = iter->he_cache; + struct hist_entry *he; + int err = 0; + + he = hists__add_entry(hists, al, iter->parent, NULL, NULL, NULL, + sample, true); + if (he == NULL) + return -ENOMEM; + + iter->he = he; + he_cache[iter->curr++] = he; + + hist_entry__append_callchain(he, sample); + + /* + * We need to re-initialize the cursor since callchain_append() + * advanced the cursor to the end. + */ + callchain_cursor_commit(get_tls_callchain_cursor()); + + hists__inc_nr_samples(hists, he->filtered); + + return err; +} + +static int +iter_next_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al) +{ + struct callchain_cursor_node *node; + + node = callchain_cursor_current(get_tls_callchain_cursor()); + if (node == NULL) + return 0; + + return fill_callchain_info(al, node, iter->hide_unresolved); +} + +static bool +hist_entry__fast__sym_diff(struct hist_entry *left, + struct hist_entry *right) +{ + struct symbol *sym_l = left->ms.sym; + struct symbol *sym_r = right->ms.sym; + + if (!sym_l && !sym_r) + return left->ip != right->ip; + + return !!_sort__sym_cmp(sym_l, sym_r); +} + + +static int +iter_add_next_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al) +{ + struct perf_sample *sample = iter->sample; + struct evsel *evsel = sample->evsel; + struct hist_entry **he_cache = iter->he_cache; + struct hist_entry *he; + struct hist_entry he_tmp = { + .hists = evsel__hists(evsel), + .cpu = al->cpu, + .thread = al->thread, + .comm = thread__comm(al->thread), + .ip = al->addr, + .ms = { + .thread = al->thread, + .map = al->map, + .sym = al->sym, + }, + .srcline = (char *) al->srcline, + .parent = iter->parent, + .raw_data = sample->raw_data, + .raw_size = sample->raw_size, + }; + int i; + struct callchain_cursor cursor, *tls_cursor = get_tls_callchain_cursor(); + bool fast = hists__has(he_tmp.hists, sym); + + if (tls_cursor == NULL) + return -ENOMEM; + + callchain_cursor_snapshot(&cursor, tls_cursor); + + callchain_cursor_advance(tls_cursor); + + /* + * Check if there's duplicate entries in the callchain. + * It's possible that it has cycles or recursive calls. + */ + for (i = 0; i < iter->curr; i++) { + /* + * For most cases, there are no duplicate entries in callchain. + * The symbols are usually different. Do a quick check for + * symbols first. + */ + if (fast && hist_entry__fast__sym_diff(he_cache[i], &he_tmp)) + continue; + + if (hist_entry__cmp(he_cache[i], &he_tmp) == 0) { + /* to avoid calling callback function */ + iter->he = NULL; + return 0; + } + } + + he = hists__add_entry(evsel__hists(evsel), al, iter->parent, NULL, NULL, + NULL, sample, false); + if (he == NULL) + return -ENOMEM; + + iter->he = he; + he_cache[iter->curr++] = he; + + if (hist_entry__has_callchains(he) && symbol_conf.use_callchain) + callchain_append(he->callchain, &cursor, sample->period); + return 0; +} + +static int +iter_finish_cumulative_entry(struct hist_entry_iter *iter, + struct addr_location *al __maybe_unused) +{ + mem_info__zput(iter->mi); + zfree(&iter->bi); + zfree(&iter->he_cache); + iter->he = NULL; + + return 0; +} + +const struct hist_iter_ops hist_iter_mem = { + .prepare_entry = iter_prepare_mem_entry, + .add_single_entry = iter_add_single_mem_entry, + .next_entry = iter_next_nop_entry, + .add_next_entry = iter_add_next_nop_entry, + .finish_entry = iter_finish_mem_entry, +}; + +const struct hist_iter_ops hist_iter_branch = { + .prepare_entry = iter_prepare_branch_entry, + .add_single_entry = iter_add_single_branch_entry, + .next_entry = iter_next_branch_entry, + .add_next_entry = iter_add_next_branch_entry, + .finish_entry = iter_finish_branch_entry, +}; + +const struct hist_iter_ops hist_iter_normal = { + .prepare_entry = iter_prepare_normal_entry, + .add_single_entry = iter_add_single_normal_entry, + .next_entry = iter_next_nop_entry, + .add_next_entry = iter_add_next_nop_entry, + .finish_entry = iter_finish_normal_entry, +}; + +const struct hist_iter_ops hist_iter_cumulative = { + .prepare_entry = iter_prepare_cumulative_entry, + .add_single_entry = iter_add_single_cumulative_entry, + .next_entry = iter_next_cumulative_entry, + .add_next_entry = iter_add_next_cumulative_entry, + .finish_entry = iter_finish_cumulative_entry, +}; + +int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, + int max_stack_depth, void *arg) +{ + int err, err2; + struct map *alm = NULL; + + if (al) + alm = map__get(al->map); + + err = sample__resolve_callchain(iter->sample, get_tls_callchain_cursor(), &iter->parent, + al, max_stack_depth); + if (err) { + map__put(alm); + return err; + } + + err = iter->ops->prepare_entry(iter, al); + if (err) + goto out; + + err = iter->ops->add_single_entry(iter, al); + if (err) + goto out; + + if (iter->he && iter->add_entry_cb) { + err = iter->add_entry_cb(iter, al, true, arg); + if (err) + goto out; + } + + while (iter->ops->next_entry(iter, al)) { + err = iter->ops->add_next_entry(iter, al); + if (err) + break; + + if (iter->he && iter->add_entry_cb) { + err = iter->add_entry_cb(iter, al, false, arg); + if (err) + goto out; + } + } + +out: + err2 = iter->ops->finish_entry(iter, al); + if (!err) + err = err2; + + map__put(alm); + + return err; +} + +static int64_t +hist_entry__cmp_impl(struct perf_hpp_list *hpp_list, struct hist_entry *left, + struct hist_entry *right, unsigned long fn_offset, + bool ignore_dynamic, bool ignore_skipped) +{ + struct hists *hists = left->hists; + struct perf_hpp_fmt *fmt; + perf_hpp_fmt_cmp_t *fn; + int64_t cmp; + + /* + * Never collapse filtered and non-filtered entries. + * Note this is not the same as having an extra (invisible) fmt + * that corresponds to the filtered status. + */ + cmp = (int64_t)!!left->filtered - (int64_t)!!right->filtered; + if (cmp) + return cmp; + + perf_hpp_list__for_each_sort_list(hpp_list, fmt) { + if (ignore_dynamic && perf_hpp__is_dynamic_entry(fmt) && + !perf_hpp__defined_dynamic_entry(fmt, hists)) + continue; + + if (ignore_skipped && perf_hpp__should_skip(fmt, hists)) + continue; + + fn = (void *)fmt + fn_offset; + cmp = (*fn)(fmt, left, right); + if (cmp) + break; + } + + return cmp; +} + +int64_t +hist_entry__cmp(struct hist_entry *left, struct hist_entry *right) +{ + return hist_entry__cmp_impl(left->hists->hpp_list, left, right, + offsetof(struct perf_hpp_fmt, cmp), true, false); +} + +static int64_t +hist_entry__sort(struct hist_entry *left, struct hist_entry *right) +{ + return hist_entry__cmp_impl(left->hists->hpp_list, left, right, + offsetof(struct perf_hpp_fmt, sort), false, true); +} + +int64_t +hist_entry__collapse(struct hist_entry *left, struct hist_entry *right) +{ + return hist_entry__cmp_impl(left->hists->hpp_list, left, right, + offsetof(struct perf_hpp_fmt, collapse), true, false); +} + +static int64_t +hist_entry__collapse_hierarchy(struct perf_hpp_list *hpp_list, + struct hist_entry *left, + struct hist_entry *right) +{ + return hist_entry__cmp_impl(hpp_list, left, right, + offsetof(struct perf_hpp_fmt, collapse), false, false); +} + +void hist_entry__delete(struct hist_entry *he) +{ + struct hist_entry_ops *ops = he->ops; + + if (symbol_conf.report_hierarchy) { + struct rb_root *root = &he->hroot_out.rb_root; + struct hist_entry *child, *tmp; + + rbtree_postorder_for_each_entry_safe(child, tmp, root, rb_node) + hist_entry__delete(child); + + *root = RB_ROOT; + } + + thread__zput(he->thread); + map_symbol__exit(&he->ms); + + if (he->branch_info) { + branch_info__exit(he->branch_info); + zfree(&he->branch_info); + } + + if (he->mem_info) { + map_symbol__exit(&mem_info__iaddr(he->mem_info)->ms); + map_symbol__exit(&mem_info__daddr(he->mem_info)->ms); + mem_info__zput(he->mem_info); + } + + if (he->block_info) + block_info__delete(he->block_info); + + if (he->kvm_info) + kvm_info__zput(he->kvm_info); + + zfree(&he->res_samples); + zfree(&he->stat_acc); + zfree_srcline(&he->srcline); + if (he->srcfile && he->srcfile[0]) + zfree(&he->srcfile); + free_callchain(he->callchain); + zfree(&he->trace_output); + zfree(&he->raw_data); + zfree(&he->mem_stat); + ops->free(he); +} + +/* + * If this is not the last column, then we need to pad it according to the + * pre-calculated max length for this column, otherwise don't bother adding + * spaces because that would break viewing this with, for instance, 'less', + * that would show tons of trailing spaces when a long C++ demangled method + * names is sampled. +*/ +int hist_entry__snprintf_alignment(struct hist_entry *he, struct perf_hpp *hpp, + struct perf_hpp_fmt *fmt, int printed) +{ + if (!list_is_last(&fmt->list, &he->hists->hpp_list->fields)) { + const int width = fmt->width(fmt, hpp, he->hists); + if (printed < width) { + advance_hpp(hpp, printed); + printed = scnprintf(hpp->buf, hpp->size, "%-*s", width - printed, " "); + } + } + + return printed; +} + +/* + * collapse the histogram + */ + +static void hists__apply_filters(struct hists *hists, struct hist_entry *he); +static void hists__remove_entry_filter(struct hists *hists, struct hist_entry *he, + enum hist_filter type); + +typedef bool (*fmt_chk_fn)(struct perf_hpp_fmt *fmt); + +static bool check_thread_entry(struct perf_hpp_fmt *fmt) +{ + return perf_hpp__is_thread_entry(fmt) || perf_hpp__is_comm_entry(fmt); +} + +static void hist_entry__check_and_remove_filter(struct hist_entry *he, + enum hist_filter type, + fmt_chk_fn check) +{ + struct perf_hpp_fmt *fmt; + bool type_match = false; + struct hist_entry *parent = he->parent_he; + + switch (type) { + case HIST_FILTER__THREAD: + if (symbol_conf.comm_list == NULL && + symbol_conf.pid_list == NULL && + symbol_conf.tid_list == NULL) + return; + break; + case HIST_FILTER__DSO: + if (symbol_conf.dso_list == NULL) + return; + break; + case HIST_FILTER__SYMBOL: + if (symbol_conf.sym_list == NULL) + return; + break; + case HIST_FILTER__PARALLELISM: + if (__bitmap_weight(symbol_conf.parallelism_filter, MAX_NR_CPUS + 1) == 0) + return; + break; + case HIST_FILTER__PARENT: + case HIST_FILTER__GUEST: + case HIST_FILTER__HOST: + case HIST_FILTER__SOCKET: + case HIST_FILTER__C2C: + default: + return; + } + + /* if it's filtered by own fmt, it has to have filter bits */ + perf_hpp_list__for_each_format(he->hpp_list, fmt) { + if (check(fmt)) { + type_match = true; + break; + } + } + + if (type_match) { + /* + * If the filter is for current level entry, propagate + * filter marker to parents. The marker bit was + * already set by default so it only needs to clear + * non-filtered entries. + */ + if (!(he->filtered & (1 << type))) { + while (parent) { + parent->filtered &= ~(1 << type); + parent = parent->parent_he; + } + } + } else { + /* + * If current entry doesn't have matching formats, set + * filter marker for upper level entries. it will be + * cleared if its lower level entries is not filtered. + * + * For lower-level entries, it inherits parent's + * filter bit so that lower level entries of a + * non-filtered entry won't set the filter marker. + */ + if (parent == NULL) + he->filtered |= (1 << type); + else + he->filtered |= (parent->filtered & (1 << type)); + } +} + +static void hist_entry__apply_hierarchy_filters(struct hist_entry *he) +{ + hist_entry__check_and_remove_filter(he, HIST_FILTER__THREAD, + check_thread_entry); + + hist_entry__check_and_remove_filter(he, HIST_FILTER__DSO, + perf_hpp__is_dso_entry); + + hist_entry__check_and_remove_filter(he, HIST_FILTER__SYMBOL, + perf_hpp__is_sym_entry); + + hist_entry__check_and_remove_filter(he, HIST_FILTER__PARALLELISM, + perf_hpp__is_parallelism_entry); + + hists__apply_filters(he->hists, he); +} + +static struct hist_entry *hierarchy_insert_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *he, + struct hist_entry *parent_he, + struct perf_hpp_list *hpp_list) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter, *new; + struct perf_hpp_fmt *fmt; + int64_t cmp; + bool leftmost = true; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node_in); + cmp = hist_entry__collapse_hierarchy(hpp_list, iter, he); + if (!cmp) { + he_stat__add_stat(&iter->stat, &he->stat); + hists__add_mem_stat(hists, iter, he); + return iter; + } + + if (cmp < 0) + p = &parent->rb_left; + else { + p = &parent->rb_right; + leftmost = false; + } + } + + new = hist_entry__new(he, true); + if (new == NULL) + return NULL; + + hists->nr_entries++; + + /* save related format list for output */ + new->hpp_list = hpp_list; + new->parent_he = parent_he; + + hist_entry__apply_hierarchy_filters(new); + + /* some fields are now passed to 'new' */ + perf_hpp_list__for_each_sort_list(hpp_list, fmt) { + if (perf_hpp__is_trace_entry(fmt) || perf_hpp__is_dynamic_entry(fmt)) + he->trace_output = NULL; + else + new->trace_output = NULL; + + if (perf_hpp__is_srcline_entry(fmt)) + he->srcline = NULL; + else + new->srcline = NULL; + + if (perf_hpp__is_srcfile_entry(fmt)) + he->srcfile = NULL; + else + new->srcfile = NULL; + } + + if (hists__clone_mem_stat(hists, new, he) < 0) { + hist_entry__delete(new); + return NULL; + } + + rb_link_node(&new->rb_node_in, parent, p); + rb_insert_color_cached(&new->rb_node_in, root, leftmost); + return new; +} + +static int hists__hierarchy_insert_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *he) +{ + struct perf_hpp_list_node *node; + struct hist_entry *new_he = NULL; + struct hist_entry *parent = NULL; + int depth = 0; + int ret = 0; + + list_for_each_entry(node, &hists->hpp_formats, list) { + /* skip period (overhead) and elided columns */ + if (node->level == 0 || node->skip) + continue; + + /* insert copy of 'he' for each fmt into the hierarchy */ + new_he = hierarchy_insert_entry(hists, root, he, parent, &node->hpp); + if (new_he == NULL) { + ret = -1; + break; + } + + root = &new_he->hroot_in; + new_he->depth = depth++; + parent = new_he; + } + + if (new_he) { + new_he->leaf = true; + + if (hist_entry__has_callchains(new_he) && + symbol_conf.use_callchain) { + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + + if (cursor == NULL) + return -1; + + callchain_cursor_reset(cursor); + if (callchain_merge(cursor, + new_he->callchain, + he->callchain) < 0) + ret = -1; + } + } + + /* 'he' is no longer used */ + hist_entry__delete(he); + + /* return 0 (or -1) since it already applied filters */ + return ret; +} + +static int hists__collapse_insert_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + int64_t cmp; + bool leftmost = true; + + if (symbol_conf.report_hierarchy) + return hists__hierarchy_insert_entry(hists, root, he); + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node_in); + + cmp = hist_entry__collapse(iter, he); + + if (!cmp) { + int ret = 0; + + he_stat__add_stat(&iter->stat, &he->stat); + if (symbol_conf.cumulate_callchain) + he_stat__add_stat(iter->stat_acc, he->stat_acc); + hists__add_mem_stat(hists, iter, he); + + if (hist_entry__has_callchains(he) && symbol_conf.use_callchain) { + struct callchain_cursor *cursor = get_tls_callchain_cursor(); + + if (cursor != NULL) { + callchain_cursor_reset(cursor); + if (callchain_merge(cursor, iter->callchain, he->callchain) < 0) + ret = -1; + } else { + ret = 0; + } + } + hist_entry__delete(he); + return ret; + } + + if (cmp < 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + hists->nr_entries++; + + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, root, leftmost); + return 1; +} + +struct rb_root_cached *hists__get_rotate_entries_in(struct hists *hists) +{ + struct rb_root_cached *root; + + mutex_lock(&hists->lock); + + root = hists->entries_in; + if (++hists->entries_in > &hists->entries_in_array[1]) + hists->entries_in = &hists->entries_in_array[0]; + + mutex_unlock(&hists->lock); + + return root; +} + +static void hists__apply_filters(struct hists *hists, struct hist_entry *he) +{ + hists__filter_entry_by_dso(hists, he); + hists__filter_entry_by_thread(hists, he); + hists__filter_entry_by_symbol(hists, he); + hists__filter_entry_by_socket(hists, he); + hists__filter_entry_by_parallelism(hists, he); +} + +int hists__collapse_resort(struct hists *hists, struct ui_progress *prog) +{ + struct rb_root_cached *root; + struct rb_node *next; + struct hist_entry *n; + int ret; + + if (!hists__has(hists, need_collapse)) + return 0; + + hists->nr_entries = 0; + + root = hists__get_rotate_entries_in(hists); + + next = rb_first_cached(root); + + while (next) { + if (session_done()) + break; + n = rb_entry(next, struct hist_entry, rb_node_in); + next = rb_next(&n->rb_node_in); + + rb_erase_cached(&n->rb_node_in, root); + ret = hists__collapse_insert_entry(hists, &hists->entries_collapsed, n); + if (ret < 0) + return -1; + + if (ret) { + /* + * If it wasn't combined with one of the entries already + * collapsed, we need to apply the filters that may have + * been set by, say, the hist_browser. + */ + hists__apply_filters(hists, n); + } + if (prog) + ui_progress__update(prog, 1); + } + return 0; +} + +static void hists__reset_filter_stats(struct hists *hists) +{ + hists->nr_non_filtered_entries = 0; + hists->stats.total_non_filtered_period = 0; + hists->stats.total_non_filtered_latency = 0; +} + +void hists__reset_stats(struct hists *hists) +{ + hists->nr_entries = 0; + hists->stats.total_period = 0; + hists->stats.total_latency = 0; + + hists__reset_filter_stats(hists); +} + +static void hists__inc_filter_stats(struct hists *hists, struct hist_entry *h) +{ + hists->nr_non_filtered_entries++; + hists->stats.total_non_filtered_period += h->stat.period; + hists->stats.total_non_filtered_latency += h->stat.latency; +} + +void hists__inc_stats(struct hists *hists, struct hist_entry *h) +{ + if (!h->filtered) + hists__inc_filter_stats(hists, h); + + hists->nr_entries++; + hists->stats.total_period += h->stat.period; + hists->stats.total_latency += h->stat.latency; +} + +static void hierarchy_recalc_total_periods(struct hists *hists) +{ + struct rb_node *node; + struct hist_entry *he; + + node = rb_first_cached(&hists->entries); + + hists->stats.total_period = 0; + hists->stats.total_non_filtered_period = 0; + hists->stats.total_latency = 0; + hists->stats.total_non_filtered_latency = 0; + + /* + * recalculate total period using top-level entries only + * since lower level entries only see non-filtered entries + * but upper level entries have sum of both entries. + */ + while (node) { + he = rb_entry(node, struct hist_entry, rb_node); + node = rb_next(node); + + hists->stats.total_period += he->stat.period; + hists->stats.total_latency += he->stat.latency; + if (!he->filtered) { + hists->stats.total_non_filtered_period += he->stat.period; + hists->stats.total_non_filtered_latency += he->stat.latency; + } + } +} + +static void hierarchy_insert_output_entry(struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + struct perf_hpp_fmt *fmt; + bool leftmost = true; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node); + + if (hist_entry__sort(he, iter) > 0) + p = &parent->rb_left; + else { + p = &parent->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, root, leftmost); + + /* update column width of dynamic entry */ + perf_hpp_list__for_each_sort_list(he->hpp_list, fmt) { + if (fmt->init) + fmt->init(fmt, he); + } +} + +static void hists__hierarchy_output_resort(struct hists *hists, + struct ui_progress *prog, + struct rb_root_cached *root_in, + struct rb_root_cached *root_out, + u64 min_callchain_hits, + bool use_callchain) +{ + struct rb_node *node; + struct hist_entry *he; + + *root_out = RB_ROOT_CACHED; + node = rb_first_cached(root_in); + + while (node) { + he = rb_entry(node, struct hist_entry, rb_node_in); + node = rb_next(node); + + hierarchy_insert_output_entry(root_out, he); + + if (prog) + ui_progress__update(prog, 1); + + hists->nr_entries++; + if (!he->filtered) { + hists->nr_non_filtered_entries++; + hists__calc_col_len(hists, he); + } + + if (!he->leaf) { + hists__hierarchy_output_resort(hists, prog, + &he->hroot_in, + &he->hroot_out, + min_callchain_hits, + use_callchain); + continue; + } + + if (!use_callchain) + continue; + + if (callchain_param.mode == CHAIN_GRAPH_REL) { + u64 total = he->stat.period; + + if (symbol_conf.cumulate_callchain) + total = he->stat_acc->period; + + min_callchain_hits = total * (callchain_param.min_percent / 100); + } + + callchain_param.sort(&he->sorted_chain, he->callchain, + min_callchain_hits, &callchain_param); + } +} + +static void __hists__insert_output_entry(struct rb_root_cached *entries, + struct hist_entry *he, + u64 min_callchain_hits, + bool use_callchain) +{ + struct rb_node **p = &entries->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + struct perf_hpp_fmt *fmt; + bool leftmost = true; + + if (use_callchain) { + if (callchain_param.mode == CHAIN_GRAPH_REL) { + u64 total = he->stat.period; + + if (symbol_conf.cumulate_callchain) + total = he->stat_acc->period; + + min_callchain_hits = total * (callchain_param.min_percent / 100); + } + callchain_param.sort(&he->sorted_chain, he->callchain, + min_callchain_hits, &callchain_param); + } + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node); + + if (hist_entry__sort(he, iter) > 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, entries, leftmost); + + /* update column width of dynamic entries */ + perf_hpp_list__for_each_sort_list(&perf_hpp_list, fmt) { + if (fmt->init) + fmt->init(fmt, he); + } +} + +static void output_resort(struct hists *hists, struct ui_progress *prog, + bool use_callchain, hists__resort_cb_t cb, + void *cb_arg) +{ + struct rb_root_cached *root; + struct rb_node *next; + struct hist_entry *n; + u64 callchain_total; + u64 min_callchain_hits; + + callchain_total = hists->callchain_period; + if (symbol_conf.filter_relative) + callchain_total = hists->callchain_non_filtered_period; + + min_callchain_hits = callchain_total * (callchain_param.min_percent / 100); + + hists__reset_stats(hists); + hists__reset_col_len(hists); + + if (symbol_conf.report_hierarchy) { + hists__hierarchy_output_resort(hists, prog, + &hists->entries_collapsed, + &hists->entries, + min_callchain_hits, + use_callchain); + hierarchy_recalc_total_periods(hists); + return; + } + + if (hists__has(hists, need_collapse)) + root = &hists->entries_collapsed; + else + root = hists->entries_in; + + next = rb_first_cached(root); + hists->entries = RB_ROOT_CACHED; + + while (next) { + n = rb_entry(next, struct hist_entry, rb_node_in); + next = rb_next(&n->rb_node_in); + + if (cb && cb(n, cb_arg)) + continue; + + __hists__insert_output_entry(&hists->entries, n, min_callchain_hits, use_callchain); + hists__inc_stats(hists, n); + + if (!n->filtered) + hists__calc_col_len(hists, n); + + if (prog) + ui_progress__update(prog, 1); + } +} + +void evsel__output_resort_cb(struct evsel *evsel, struct ui_progress *prog, + hists__resort_cb_t cb, void *cb_arg) +{ + bool use_callchain; + + if (evsel && symbol_conf.use_callchain && !symbol_conf.show_ref_callgraph) + use_callchain = evsel__has_callchain(evsel); + else + use_callchain = symbol_conf.use_callchain; + + use_callchain |= symbol_conf.show_branchflag_count; + + output_resort(evsel__hists(evsel), prog, use_callchain, cb, cb_arg); +} + +void evsel__output_resort(struct evsel *evsel, struct ui_progress *prog) +{ + return evsel__output_resort_cb(evsel, prog, NULL, NULL); +} + +void hists__output_resort(struct hists *hists, struct ui_progress *prog) +{ + output_resort(hists, prog, symbol_conf.use_callchain, NULL, NULL); +} + +void hists__output_resort_cb(struct hists *hists, struct ui_progress *prog, + hists__resort_cb_t cb) +{ + output_resort(hists, prog, symbol_conf.use_callchain, cb, NULL); +} + +static bool can_goto_child(struct hist_entry *he, enum hierarchy_move_dir hmd) +{ + if (he->leaf || hmd == HMD_FORCE_SIBLING) + return false; + + if (he->unfolded || hmd == HMD_FORCE_CHILD) + return true; + + return false; +} + +struct rb_node *rb_hierarchy_last(struct rb_node *node) +{ + struct hist_entry *he = rb_entry(node, struct hist_entry, rb_node); + + while (can_goto_child(he, HMD_NORMAL)) { + node = rb_last(&he->hroot_out.rb_root); + he = rb_entry(node, struct hist_entry, rb_node); + } + return node; +} + +struct rb_node *__rb_hierarchy_next(struct rb_node *node, enum hierarchy_move_dir hmd) +{ + struct hist_entry *he = rb_entry(node, struct hist_entry, rb_node); + + if (can_goto_child(he, hmd)) + node = rb_first_cached(&he->hroot_out); + else + node = rb_next(node); + + while (node == NULL) { + he = he->parent_he; + if (he == NULL) + break; + + node = rb_next(&he->rb_node); + } + return node; +} + +struct rb_node *rb_hierarchy_prev(struct rb_node *node) +{ + struct hist_entry *he = rb_entry(node, struct hist_entry, rb_node); + + node = rb_prev(node); + if (node) + return rb_hierarchy_last(node); + + he = he->parent_he; + if (he == NULL) + return NULL; + + return &he->rb_node; +} + +bool hist_entry__has_hierarchy_children(struct hist_entry *he, float limit) +{ + struct rb_node *node; + struct hist_entry *child; + float percent; + + if (he->leaf) + return false; + + node = rb_first_cached(&he->hroot_out); + child = rb_entry(node, struct hist_entry, rb_node); + + while (node && child->filtered) { + node = rb_next(node); + child = rb_entry(node, struct hist_entry, rb_node); + } + + if (node) + percent = hist_entry__get_percent_limit(child); + else + percent = 0; + + return node && percent >= limit; +} + +static void hists__remove_entry_filter(struct hists *hists, struct hist_entry *h, + enum hist_filter filter) +{ + h->filtered &= ~(1 << filter); + + if (symbol_conf.report_hierarchy) { + struct hist_entry *parent = h->parent_he; + + while (parent) { + he_stat__add_stat(&parent->stat, &h->stat); + + parent->filtered &= ~(1 << filter); + + if (parent->filtered) + goto next; + + /* force fold unfiltered entry for simplicity */ + parent->unfolded = false; + parent->has_no_entry = false; + parent->row_offset = 0; + parent->nr_rows = 0; +next: + parent = parent->parent_he; + } + } + + if (h->filtered) + return; + + /* force fold unfiltered entry for simplicity */ + h->unfolded = false; + h->has_no_entry = false; + h->row_offset = 0; + h->nr_rows = 0; + + hists->stats.nr_non_filtered_samples += h->stat.nr_events; + + hists__inc_filter_stats(hists, h); + hists__calc_col_len(hists, h); +} + + +static bool hists__filter_entry_by_dso(struct hists *hists, + struct hist_entry *he) +{ + if (hists->dso_filter != NULL && + (he->ms.map == NULL || !RC_CHK_EQUAL(map__dso(he->ms.map), hists->dso_filter))) { + he->filtered |= (1 << HIST_FILTER__DSO); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_thread(struct hists *hists, + struct hist_entry *he) +{ + if (hists->thread_filter != NULL && + !RC_CHK_EQUAL(he->thread, hists->thread_filter)) { + he->filtered |= (1 << HIST_FILTER__THREAD); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_symbol(struct hists *hists, + struct hist_entry *he) +{ + if (hists->symbol_filter_str != NULL && + (!he->ms.sym || strstr(he->ms.sym->name, + hists->symbol_filter_str) == NULL)) { + he->filtered |= (1 << HIST_FILTER__SYMBOL); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_socket(struct hists *hists, + struct hist_entry *he) +{ + if ((hists->socket_filter > -1) && + (he->socket != hists->socket_filter)) { + he->filtered |= (1 << HIST_FILTER__SOCKET); + return true; + } + + return false; +} + +static bool hists__filter_entry_by_parallelism(struct hists *hists, + struct hist_entry *he) +{ + if (test_bit(he->parallelism, hists->parallelism_filter)) { + he->filtered |= (1 << HIST_FILTER__PARALLELISM); + return true; + } + return false; +} + +typedef bool (*filter_fn_t)(struct hists *hists, struct hist_entry *he); + +static void hists__filter_by_type(struct hists *hists, int type, filter_fn_t filter) +{ + struct rb_node *nd; + + hists->stats.nr_non_filtered_samples = 0; + + hists__reset_filter_stats(hists); + hists__reset_col_len(hists); + + for (nd = rb_first_cached(&hists->entries); nd; nd = rb_next(nd)) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + + if (filter(hists, h)) + continue; + + hists__remove_entry_filter(hists, h, type); + } +} + +static void resort_filtered_entry(struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node **p = &root->rb_root.rb_node; + struct rb_node *parent = NULL; + struct hist_entry *iter; + struct rb_root_cached new_root = RB_ROOT_CACHED; + struct rb_node *nd; + bool leftmost = true; + + while (*p != NULL) { + parent = *p; + iter = rb_entry(parent, struct hist_entry, rb_node); + + if (hist_entry__sort(he, iter) > 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + rb_link_node(&he->rb_node, parent, p); + rb_insert_color_cached(&he->rb_node, root, leftmost); + + if (he->leaf || he->filtered) + return; + + nd = rb_first_cached(&he->hroot_out); + while (nd) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + + nd = rb_next(nd); + rb_erase_cached(&h->rb_node, &he->hroot_out); + + resort_filtered_entry(&new_root, h); + } + + he->hroot_out = new_root; +} + +static void hists__filter_hierarchy(struct hists *hists, int type, const void *arg) +{ + struct rb_node *nd; + struct rb_root_cached new_root = RB_ROOT_CACHED; + + hists->stats.nr_non_filtered_samples = 0; + + hists__reset_filter_stats(hists); + hists__reset_col_len(hists); + + nd = rb_first_cached(&hists->entries); + while (nd) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + int ret; + + ret = hist_entry__filter(h, type, arg); + + /* + * case 1. non-matching type + * zero out the period, set filter marker and move to child + */ + if (ret < 0) { + memset(&h->stat, 0, sizeof(h->stat)); + h->filtered |= (1 << type); + + nd = __rb_hierarchy_next(&h->rb_node, HMD_FORCE_CHILD); + } + /* + * case 2. matched type (filter out) + * set filter marker and move to next + */ + else if (ret == 1) { + h->filtered |= (1 << type); + + nd = __rb_hierarchy_next(&h->rb_node, HMD_FORCE_SIBLING); + } + /* + * case 3. ok (not filtered) + * add period to hists and parents, erase the filter marker + * and move to next sibling + */ + else { + hists__remove_entry_filter(hists, h, type); + + nd = __rb_hierarchy_next(&h->rb_node, HMD_FORCE_SIBLING); + } + } + + hierarchy_recalc_total_periods(hists); + + /* + * resort output after applying a new filter since filter in a lower + * hierarchy can change periods in a upper hierarchy. + */ + nd = rb_first_cached(&hists->entries); + while (nd) { + struct hist_entry *h = rb_entry(nd, struct hist_entry, rb_node); + + nd = rb_next(nd); + rb_erase_cached(&h->rb_node, &hists->entries); + + resort_filtered_entry(&new_root, h); + } + + hists->entries = new_root; +} + +void hists__filter_by_thread(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__THREAD, + hists->thread_filter); + else + hists__filter_by_type(hists, HIST_FILTER__THREAD, + hists__filter_entry_by_thread); +} + +void hists__filter_by_dso(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__DSO, + hists->dso_filter); + else + hists__filter_by_type(hists, HIST_FILTER__DSO, + hists__filter_entry_by_dso); +} + +void hists__filter_by_symbol(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__SYMBOL, + hists->symbol_filter_str); + else + hists__filter_by_type(hists, HIST_FILTER__SYMBOL, + hists__filter_entry_by_symbol); +} + +void hists__filter_by_socket(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__SOCKET, + &hists->socket_filter); + else + hists__filter_by_type(hists, HIST_FILTER__SOCKET, + hists__filter_entry_by_socket); +} + +void hists__filter_by_parallelism(struct hists *hists) +{ + if (symbol_conf.report_hierarchy) + hists__filter_hierarchy(hists, HIST_FILTER__PARALLELISM, + hists->parallelism_filter); + else + hists__filter_by_type(hists, HIST_FILTER__PARALLELISM, + hists__filter_entry_by_parallelism); +} + +void events_stats__inc(struct events_stats *stats, u32 type) +{ + ++stats->nr_events[0]; + ++stats->nr_events[type]; +} + +static void hists_stats__inc(struct hists_stats *stats) +{ + ++stats->nr_samples; +} + +void hists__inc_nr_events(struct hists *hists) +{ + hists_stats__inc(&hists->stats); +} + +void hists__inc_nr_samples(struct hists *hists, bool filtered) +{ + hists_stats__inc(&hists->stats); + if (!filtered) + hists->stats.nr_non_filtered_samples++; +} + +void hists__inc_nr_lost_samples(struct hists *hists, u32 lost) +{ + hists->stats.nr_lost_samples += lost; +} + +void hists__inc_nr_dropped_samples(struct hists *hists, u32 lost) +{ + hists->stats.nr_dropped_samples += lost; +} + +static struct hist_entry *hists__add_dummy_entry(struct hists *hists, + struct hist_entry *pair) +{ + struct rb_root_cached *root; + struct rb_node **p; + struct rb_node *parent = NULL; + struct hist_entry *he; + int64_t cmp; + bool leftmost = true; + + if (hists__has(hists, need_collapse)) + root = &hists->entries_collapsed; + else + root = hists->entries_in; + + p = &root->rb_root.rb_node; + + while (*p != NULL) { + parent = *p; + he = rb_entry(parent, struct hist_entry, rb_node_in); + + cmp = hist_entry__collapse(he, pair); + + if (!cmp) + goto out; + + if (cmp < 0) + p = &(*p)->rb_left; + else { + p = &(*p)->rb_right; + leftmost = false; + } + } + + he = hist_entry__new(pair, true); + if (he) { + memset(&he->stat, 0, sizeof(he->stat)); + he->hists = hists; + if (symbol_conf.cumulate_callchain) + memset(he->stat_acc, 0, sizeof(he->stat)); + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, root, leftmost); + hists__inc_stats(hists, he); + he->dummy = true; + } +out: + return he; +} + +static struct hist_entry *add_dummy_hierarchy_entry(struct hists *hists, + struct rb_root_cached *root, + struct hist_entry *pair) +{ + struct rb_node **p; + struct rb_node *parent = NULL; + struct hist_entry *he; + bool leftmost = true; + + p = &root->rb_root.rb_node; + while (*p != NULL) { + int64_t cmp; + + parent = *p; + he = rb_entry(parent, struct hist_entry, rb_node_in); + cmp = hist_entry__collapse_hierarchy(he->hpp_list, he, pair); + if (!cmp) + goto out; + + if (cmp < 0) + p = &parent->rb_left; + else { + p = &parent->rb_right; + leftmost = false; + } + } + + he = hist_entry__new(pair, true); + if (he) { + rb_link_node(&he->rb_node_in, parent, p); + rb_insert_color_cached(&he->rb_node_in, root, leftmost); + + he->dummy = true; + he->hists = hists; + memset(&he->stat, 0, sizeof(he->stat)); + hists__inc_stats(hists, he); + } +out: + return he; +} + +static struct hist_entry *hists__find_entry(struct hists *hists, + struct hist_entry *he) +{ + struct rb_node *n; + + if (hists__has(hists, need_collapse)) + n = hists->entries_collapsed.rb_root.rb_node; + else + n = hists->entries_in->rb_root.rb_node; + + while (n) { + struct hist_entry *iter = rb_entry(n, struct hist_entry, rb_node_in); + int64_t cmp = hist_entry__collapse(iter, he); + + if (cmp < 0) + n = n->rb_left; + else if (cmp > 0) + n = n->rb_right; + else + return iter; + } + + return NULL; +} + +static struct hist_entry *hists__find_hierarchy_entry(struct rb_root_cached *root, + struct hist_entry *he) +{ + struct rb_node *n = root->rb_root.rb_node; + + while (n) { + struct hist_entry *iter; + int64_t cmp; + + iter = rb_entry(n, struct hist_entry, rb_node_in); + cmp = hist_entry__collapse_hierarchy(he->hpp_list, iter, he); + if (cmp < 0) + n = n->rb_left; + else if (cmp > 0) + n = n->rb_right; + else + return iter; + } + + return NULL; +} + +static void hists__match_hierarchy(struct rb_root_cached *leader_root, + struct rb_root_cached *other_root) +{ + struct rb_node *nd; + struct hist_entry *pos, *pair; + + for (nd = rb_first_cached(leader_root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + pair = hists__find_hierarchy_entry(other_root, pos); + + if (pair) { + hist_entry__add_pair(pair, pos); + hists__match_hierarchy(&pos->hroot_in, &pair->hroot_in); + } + } +} + +/* + * Look for pairs to link to the leader buckets (hist_entries): + */ +void hists__match(struct hists *leader, struct hists *other) +{ + struct rb_root_cached *root; + struct rb_node *nd; + struct hist_entry *pos, *pair; + + if (symbol_conf.report_hierarchy) { + /* hierarchy report always collapses entries */ + return hists__match_hierarchy(&leader->entries_collapsed, + &other->entries_collapsed); + } + + if (hists__has(leader, need_collapse)) + root = &leader->entries_collapsed; + else + root = leader->entries_in; + + for (nd = rb_first_cached(root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + pair = hists__find_entry(other, pos); + + if (pair) + hist_entry__add_pair(pair, pos); + } +} + +static int hists__link_hierarchy(struct hists *leader_hists, + struct hist_entry *parent, + struct rb_root_cached *leader_root, + struct rb_root_cached *other_root) +{ + struct rb_node *nd; + struct hist_entry *pos, *leader; + + for (nd = rb_first_cached(other_root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + + if (hist_entry__has_pairs(pos)) { + bool found = false; + + list_for_each_entry(leader, &pos->pairs.head, pairs.node) { + if (leader->hists == leader_hists) { + found = true; + break; + } + } + if (!found) + return -1; + } else { + leader = add_dummy_hierarchy_entry(leader_hists, + leader_root, pos); + if (leader == NULL) + return -1; + + /* do not point parent in the pos */ + leader->parent_he = parent; + + hist_entry__add_pair(pos, leader); + } + + if (!pos->leaf) { + if (hists__link_hierarchy(leader_hists, leader, + &leader->hroot_in, + &pos->hroot_in) < 0) + return -1; + } + } + return 0; +} + +/* + * Look for entries in the other hists that are not present in the leader, if + * we find them, just add a dummy entry on the leader hists, with period=0, + * nr_events=0, to serve as the list header. + */ +int hists__link(struct hists *leader, struct hists *other) +{ + struct rb_root_cached *root; + struct rb_node *nd; + struct hist_entry *pos, *pair; + + if (symbol_conf.report_hierarchy) { + /* hierarchy report always collapses entries */ + return hists__link_hierarchy(leader, NULL, + &leader->entries_collapsed, + &other->entries_collapsed); + } + + if (hists__has(other, need_collapse)) + root = &other->entries_collapsed; + else + root = other->entries_in; + + for (nd = rb_first_cached(root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + + if (!hist_entry__has_pairs(pos)) { + pair = hists__add_dummy_entry(leader, pos); + if (pair == NULL) + return -1; + hist_entry__add_pair(pos, pair); + } + } + + return 0; +} + +int hists__unlink(struct hists *hists) +{ + struct rb_root_cached *root; + struct rb_node *nd; + struct hist_entry *pos; + + if (hists__has(hists, need_collapse)) + root = &hists->entries_collapsed; + else + root = hists->entries_in; + + for (nd = rb_first_cached(root); nd; nd = rb_next(nd)) { + pos = rb_entry(nd, struct hist_entry, rb_node_in); + list_del_init(&pos->pairs.node); + } + + return 0; +} + +void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, + struct perf_sample *sample, bool nonany_branch_mode, + u64 *total_cycles) +{ + struct branch_info *bi; + struct branch_entry *entries = perf_sample__branch_entries(sample); + + /* If we have branch cycles always annotate them. */ + if (bs && bs->nr && entries[0].flags.cycles) { + bi = sample__resolve_bstack(sample, al); + if (bi) { + struct addr_map_symbol *prev = NULL; + + /* + * Ignore errors, still want to process the + * other entries. + * + * For non standard branch modes always + * force no IPC (prev == NULL) + * + * Note that perf stores branches reversed from + * program order! + */ + for (int i = bs->nr - 1; i >= 0; i--) { + addr_map_symbol__account_cycles(&bi[i].from, + nonany_branch_mode ? NULL : prev, + bi[i].flags.cycles, sample->evsel, + bi[i].branch_stack_cntr); + prev = &bi[i].to; + + if (total_cycles) + *total_cycles += bi[i].flags.cycles; + } + for (unsigned int i = 0; i < bs->nr; i++) { + map_symbol__exit(&bi[i].to.ms); + map_symbol__exit(&bi[i].from.ms); + } + free(bi); + } + } +} + +size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp) +{ + struct evsel *pos; + size_t ret = 0; + + evlist__for_each_entry(evlist, pos) { + struct hists *hists = evsel__hists(pos); + u64 total_samples = hists->stats.nr_samples; + + total_samples += hists->stats.nr_lost_samples; + total_samples += hists->stats.nr_dropped_samples; + + if (symbol_conf.skip_empty && total_samples == 0) + continue; + + ret += fprintf(fp, "%s stats:\n", evsel__name(pos)); + if (hists->stats.nr_samples) + ret += fprintf(fp, "%20s events: %10d\n", + "SAMPLE", hists->stats.nr_samples); + if (hists->stats.nr_lost_samples) + ret += fprintf(fp, "%20s events: %10d\n", + "LOST_SAMPLES", hists->stats.nr_lost_samples); + if (hists->stats.nr_dropped_samples) + ret += fprintf(fp, "%20s events: %10d\n", + "LOST_SAMPLES (BPF)", hists->stats.nr_dropped_samples); + } + + return ret; +} + + +u64 hists__total_period(struct hists *hists) +{ + return symbol_conf.filter_relative ? hists->stats.total_non_filtered_period : + hists->stats.total_period; +} + +u64 hists__total_latency(struct hists *hists) +{ + return symbol_conf.filter_relative ? hists->stats.total_non_filtered_latency : + hists->stats.total_latency; +} + +int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool show_freq) +{ + char unit; + int printed; + const struct dso *dso = hists->dso_filter; + struct thread *thread = hists->thread_filter; + int socket_id = hists->socket_filter; + unsigned long nr_samples = hists->stats.nr_samples; + u64 nr_events = hists->stats.total_period; + struct evsel *evsel = hists_to_evsel(hists); + const char *ev_name = evsel__name(evsel); + char buf[512], sample_freq_str[64] = ""; + size_t buflen = sizeof(buf); + char ref[30] = " show reference callgraph, "; + bool enable_ref = false; + + if (symbol_conf.filter_relative) { + nr_samples = hists->stats.nr_non_filtered_samples; + nr_events = hists->stats.total_non_filtered_period; + } + + if (evsel__is_group_event(evsel)) { + struct evsel *pos; + + evsel__group_desc(evsel, buf, buflen); + ev_name = buf; + + for_each_group_member(pos, evsel) { + struct hists *pos_hists = evsel__hists(pos); + + if (symbol_conf.filter_relative) { + nr_samples += pos_hists->stats.nr_non_filtered_samples; + nr_events += pos_hists->stats.total_non_filtered_period; + } else { + nr_samples += pos_hists->stats.nr_samples; + nr_events += pos_hists->stats.total_period; + } + } + } + + if (symbol_conf.show_ref_callgraph && + strstr(ev_name, "call-graph=no")) + enable_ref = true; + + if (show_freq) + scnprintf(sample_freq_str, sizeof(sample_freq_str), " %d Hz,", evsel->core.attr.sample_freq); + + nr_samples = convert_unit(nr_samples, &unit); + printed = scnprintf(bf, size, + "Samples: %lu%c of event%s '%s',%s%sEvent count (approx.): %" PRIu64, + nr_samples, unit, evsel->core.nr_members > 1 ? "s" : "", + ev_name, sample_freq_str, enable_ref ? ref : " ", nr_events); + + + if (hists->uid_filter_str) { + printed += scnprintf(bf + printed, size - printed, + ", UID: %s", hists->uid_filter_str); + } + if (thread) { + if (hists__has(hists, thread)) { + printed += scnprintf(bf + printed, size - printed, + ", Thread: %s(%d)", + (thread__comm_set(thread) ? thread__comm_str(thread) : ""), + thread__tid(thread)); + } else { + printed += scnprintf(bf + printed, size - printed, + ", Thread: %s", + (thread__comm_set(thread) ? thread__comm_str(thread) : "")); + } + } + if (dso) + printed += scnprintf(bf + printed, size - printed, + ", DSO: %s", dso__short_name(dso)); + if (socket_id > -1) + printed += scnprintf(bf + printed, size - printed, + ", Processor Socket: %d", socket_id); + + return printed; +} + +int parse_filter_percentage(const struct option *opt __maybe_unused, + const char *arg, int unset __maybe_unused) +{ + if (!strcmp(arg, "relative")) + symbol_conf.filter_relative = true; + else if (!strcmp(arg, "absolute")) + symbol_conf.filter_relative = false; + else { + pr_debug("Invalid percentage: %s\n", arg); + return -1; + } + + return 0; +} + +int perf_hist_config(const char *var, const char *value) +{ + if (!strcmp(var, "hist.percentage")) + return parse_filter_percentage(NULL, value, 0); + + return 0; +} + +int __hists__init(struct hists *hists, struct perf_hpp_list *hpp_list) +{ + memset(hists, 0, sizeof(*hists)); + hists->entries_in_array[0] = hists->entries_in_array[1] = RB_ROOT_CACHED; + hists->entries_in = &hists->entries_in_array[0]; + hists->entries_collapsed = RB_ROOT_CACHED; + hists->entries = RB_ROOT_CACHED; + mutex_init(&hists->lock); + hists->socket_filter = -1; + hists->parallelism_filter = symbol_conf.parallelism_filter; + hists->hpp_list = hpp_list; + INIT_LIST_HEAD(&hists->hpp_formats); + return 0; +} + +static void hists__delete_remaining_entries(struct rb_root_cached *root) +{ + struct rb_node *node; + struct hist_entry *he; + + while (!RB_EMPTY_ROOT(&root->rb_root)) { + node = rb_first_cached(root); + rb_erase_cached(node, root); + + he = rb_entry(node, struct hist_entry, rb_node_in); + hist_entry__delete(he); + } +} + +void hists__delete_all_entries(struct hists *hists) +{ + hists__delete_entries(hists); + hists__delete_remaining_entries(&hists->entries_in_array[0]); + hists__delete_remaining_entries(&hists->entries_in_array[1]); + hists__delete_remaining_entries(&hists->entries_collapsed); +} + +static void hists_evsel__exit(struct evsel *evsel) +{ + struct hists *hists = evsel__hists(evsel); + struct perf_hpp_fmt *fmt, *pos; + struct perf_hpp_list_node *node, *tmp; + + hists__delete_all_entries(hists); + zfree(&hists->mem_stat_types); + zfree(&hists->mem_stat_total); + + list_for_each_entry_safe(node, tmp, &hists->hpp_formats, list) { + perf_hpp_list__for_each_format_safe(&node->hpp, fmt, pos) { + list_del_init(&fmt->list); + free(fmt); + } + list_del_init(&node->list); + free(node); + } +} + +static int hists_evsel__init(struct evsel *evsel) +{ + struct hists *hists = evsel__hists(evsel); + + __hists__init(hists, &perf_hpp_list); + return 0; +} + +/* + * XXX We probably need a hists_evsel__exit() to free the hist_entries + * stored in the rbtree... + */ + +int hists__init(void) +{ + int err = evsel__object_config(sizeof(struct hists_evsel), + hists_evsel__init, hists_evsel__exit); + if (err) + fputs("FATAL ERROR: Couldn't setup hists class\n", stderr); + + return err; +} + +void perf_hpp_list__init(struct perf_hpp_list *list) +{ + INIT_LIST_HEAD(&list->fields); + INIT_LIST_HEAD(&list->sorts); +} diff --git a/tools/perf/util/hist.h b/tools/perf/util/hist.h new file mode 100644 index 000000000..b830cbe7f --- /dev/null +++ b/tools/perf/util/hist.h @@ -0,0 +1,841 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __PERF_HIST_H +#define __PERF_HIST_H + +#include +#include +#include "callchain.h" +#include "color.h" +#include "events_stats.h" +#include "evsel.h" +#include "map_symbol.h" +#include "mem-events.h" +#include "mutex.h" +#include "sample.h" +#include "spark.h" +#include "stat.h" + +struct addr_location; +struct mem_info; +struct kvm_info; +struct branch_info; +struct branch_stack; +struct block_info; +struct ui_progress; + +enum hist_filter { + HIST_FILTER__DSO, + HIST_FILTER__THREAD, + HIST_FILTER__PARENT, + HIST_FILTER__SYMBOL, + HIST_FILTER__GUEST, + HIST_FILTER__HOST, + HIST_FILTER__SOCKET, + HIST_FILTER__C2C, + HIST_FILTER__PARALLELISM, +}; + +typedef u16 filter_mask_t; + +enum hist_column { + HISTC_SYMBOL, + HISTC_TIME, + HISTC_DSO, + HISTC_THREAD, + HISTC_TGID, + HISTC_COMM, + HISTC_COMM_NODIGIT, + HISTC_CGROUP_ID, + HISTC_CGROUP, + HISTC_PARENT, + HISTC_PARALLELISM, + HISTC_CPU, + HISTC_SOCKET, + HISTC_SRCLINE, + HISTC_SRCFILE, + HISTC_MISPREDICT, + HISTC_IN_TX, + HISTC_ABORT, + HISTC_SYMBOL_FROM, + HISTC_SYMBOL_TO, + HISTC_DSO_FROM, + HISTC_DSO_TO, + HISTC_LOCAL_WEIGHT, + HISTC_GLOBAL_WEIGHT, + HISTC_CODE_PAGE_SIZE, + HISTC_MEM_DADDR_SYMBOL, + HISTC_MEM_DADDR_DSO, + HISTC_MEM_PHYS_DADDR, + HISTC_MEM_DATA_PAGE_SIZE, + HISTC_MEM_LOCKED, + HISTC_MEM_TLB, + HISTC_MEM_LVL, + HISTC_MEM_SNOOP, + HISTC_MEM_DCACHELINE, + HISTC_MEM_IADDR_SYMBOL, + HISTC_TRANSACTION, + HISTC_CYCLES, + HISTC_SRCLINE_FROM, + HISTC_SRCLINE_TO, + HISTC_TRACE, + HISTC_SYM_SIZE, + HISTC_DSO_SIZE, + HISTC_SYMBOL_IPC, + HISTC_MEM_BLOCKED, + HISTC_LOCAL_INS_LAT, + HISTC_GLOBAL_INS_LAT, + HISTC_LOCAL_P_STAGE_CYC, + HISTC_GLOBAL_P_STAGE_CYC, + HISTC_ADDR_FROM, + HISTC_ADDR_TO, + HISTC_ADDR, + HISTC_SIMD, + HISTC_TYPE, + HISTC_TYPE_OFFSET, + HISTC_SYMBOL_OFFSET, + HISTC_TYPE_CACHELINE, + HISTC_CALLCHAIN_BRANCH_PREDICTED, + HISTC_CALLCHAIN_BRANCH_ABORT, + HISTC_CALLCHAIN_BRANCH_CYCLES, + HISTC_NR_COLS, /* Last entry */ +}; + +struct thread; +struct dso; + +#define MEM_STAT_LEN 8 + +struct he_mem_stat { + /* meaning of entries depends on enum mem_stat_type */ + u64 entries[MEM_STAT_LEN]; +}; + +struct hists { + struct rb_root_cached entries_in_array[2]; + struct rb_root_cached *entries_in; + struct rb_root_cached entries; + struct rb_root_cached entries_collapsed; + u64 nr_entries; + u64 nr_non_filtered_entries; + u64 callchain_period; + u64 callchain_non_filtered_period; + u64 callchain_latency; + u64 callchain_non_filtered_latency; + struct thread *thread_filter; + const struct dso *dso_filter; + const char *uid_filter_str; + const char *symbol_filter_str; + unsigned long *parallelism_filter; + struct mutex lock; + struct hists_stats stats; + u64 event_stream; + u16 col_len[HISTC_NR_COLS]; + bool has_callchains; + int socket_filter; + struct perf_hpp_list *hpp_list; + struct list_head hpp_formats; + int nr_hpp_node; + int nr_mem_stats; + enum mem_stat_type *mem_stat_types; + struct he_mem_stat *mem_stat_total; +}; + +#define hists__has(__h, __f) (__h)->hpp_list->__f + +struct hist_entry_iter; + +struct hist_iter_ops { + int (*prepare_entry)(struct hist_entry_iter *, struct addr_location *); + int (*add_single_entry)(struct hist_entry_iter *, struct addr_location *); + int (*next_entry)(struct hist_entry_iter *, struct addr_location *); + int (*add_next_entry)(struct hist_entry_iter *, struct addr_location *); + int (*finish_entry)(struct hist_entry_iter *, struct addr_location *); +}; + +struct hist_entry_iter { + int total; + int curr; + + struct perf_sample *sample; + struct hist_entry *he; + struct symbol *parent; + + struct mem_info *mi; + struct branch_info *bi; + struct hist_entry **he_cache; + + const struct hist_iter_ops *ops; + /* user-defined callback function (optional) */ + int (*add_entry_cb)(struct hist_entry_iter *iter, + struct addr_location *al, bool single, void *arg); + bool hide_unresolved; +}; + +extern const struct hist_iter_ops hist_iter_normal; +extern const struct hist_iter_ops hist_iter_branch; +extern const struct hist_iter_ops hist_iter_mem; +extern const struct hist_iter_ops hist_iter_cumulative; + +struct res_sample { + u64 time; + int cpu; + int tid; +}; + +struct he_stat { + u64 period; + /* + * Period re-scaled from CPU time to wall-clock time (divided by the + * parallelism at the time of the sample). This represents effect of + * the event on latency rather than CPU consumption. + */ + u64 latency; + u64 period_sys; + u64 period_us; + u64 period_guest_sys; + u64 period_guest_us; + u64 weight1; + u64 weight2; + u64 weight3; + u32 nr_events; +}; + +struct namespace_id { + u64 dev; + u64 ino; +}; + +struct hist_entry_diff { + bool computed; + union { + /* PERF_HPP__DELTA */ + double period_ratio_delta; + + /* PERF_HPP__RATIO */ + double period_ratio; + + /* HISTC_WEIGHTED_DIFF */ + s64 wdiff; + + /* PERF_HPP_DIFF__CYCLES */ + s64 cycles; + }; + struct stats stats; + unsigned long svals[NUM_SPARKS]; +}; + +struct hist_entry_ops { + void *(*new)(size_t size); + void (*free)(void *ptr); +}; + +/** + * struct hist_entry - histogram entry + * + * @row_offset - offset from the first callchain expanded to appear on screen + * @nr_rows - rows expanded in callchain, recalculated on folding/unfolding + */ +struct hist_entry { + struct rb_node rb_node_in; + struct rb_node rb_node; + union { + struct list_head node; + struct list_head head; + } pairs; + struct he_stat stat; + struct he_stat *stat_acc; + struct he_mem_stat *mem_stat; + struct map_symbol ms; + struct thread *thread; + struct comm *comm; + struct namespace_id cgroup_id; + u64 cgroup; + u64 ip; + u64 transaction; + u64 code_page_size; + u64 weight; + u64 ins_lat; + /** @weight3: On x86 holds retire_lat, on powerpc holds p_stage_cyc. */ + u64 weight3; + s32 socket; + s32 cpu; + int parallelism; + int mem_type_off; + u8 cpumode; + u8 depth; + struct simd_flags simd_flags; + + /* We are added by hists__add_dummy_entry. */ + bool dummy; + bool leaf; + + char level; + filter_mask_t filtered; + + u16 callchain_size; + union { + /* + * Since perf diff only supports the stdio output, TUI + * fields are only accessed from perf report (or perf + * top). So make it a union to reduce memory usage. + */ + struct hist_entry_diff diff; + struct /* for TUI */ { + u16 row_offset; + u16 nr_rows; + bool init_have_children; + bool unfolded; + bool has_children; + bool has_no_entry; + }; + }; + char *srcline; + char *srcfile; + struct symbol *parent; + struct branch_info *branch_info; + long time; + struct hists *hists; + struct mem_info *mem_info; + struct block_info *block_info; + struct kvm_info *kvm_info; + void *raw_data; + u32 raw_size; + int num_res; + struct res_sample *res_samples; + void *trace_output; + struct perf_hpp_list *hpp_list; + struct hist_entry *parent_he; + struct hist_entry_ops *ops; + struct annotated_data_type *mem_type; + union { + /* this is for hierarchical entry structure */ + struct { + struct rb_root_cached hroot_in; + struct rb_root_cached hroot_out; + }; /* non-leaf entries */ + struct rb_root sorted_chain; /* leaf entry has callchains */ + }; + struct callchain_root callchain[0]; /* must be last member */ +}; + +static __pure inline bool hist_entry__has_callchains(struct hist_entry *he) +{ + return he->callchain_size != 0; +} + +static inline bool hist_entry__has_pairs(struct hist_entry *he) +{ + return !list_empty(&he->pairs.node); +} + +static inline struct hist_entry *hist_entry__next_pair(struct hist_entry *he) +{ + if (hist_entry__has_pairs(he)) + return list_entry(he->pairs.node.next, struct hist_entry, pairs.node); + return NULL; +} + +static inline void hist_entry__add_pair(struct hist_entry *pair, + struct hist_entry *he) +{ + list_add_tail(&pair->pairs.node, &he->pairs.head); +} + +struct hist_entry *hists__add_entry(struct hists *hists, + struct addr_location *al, + struct symbol *parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self); + +struct hist_entry *hists__add_entry_ops(struct hists *hists, + struct hist_entry_ops *ops, + struct addr_location *al, + struct symbol *sym_parent, + struct branch_info *bi, + struct mem_info *mi, + struct kvm_info *ki, + struct perf_sample *sample, + bool sample_self); + +struct hist_entry *hists__add_entry_block(struct hists *hists, + struct addr_location *al, + struct block_info *bi); + +int hist_entry_iter__add(struct hist_entry_iter *iter, struct addr_location *al, + int max_stack_depth, void *arg); + +struct perf_hpp; +struct perf_hpp_fmt; + +int hist_entry__transaction_len(void); +int hist_entry__sort_snprintf(struct hist_entry *he, char *bf, size_t size, + struct hists *hists); +int hist_entry__snprintf_alignment(struct hist_entry *he, struct perf_hpp *hpp, + struct perf_hpp_fmt *fmt, int printed); +int hist_entry__sym_snprintf(struct hist_entry *he, char *bf, size_t size, + unsigned int width); +void hist_entry__delete(struct hist_entry *he); + +typedef int (*hists__resort_cb_t)(struct hist_entry *he, void *arg); + +void evsel__output_resort_cb(struct evsel *evsel, struct ui_progress *prog, + hists__resort_cb_t cb, void *cb_arg); +void evsel__output_resort(struct evsel *evsel, struct ui_progress *prog); +void hists__output_resort(struct hists *hists, struct ui_progress *prog); +void hists__output_resort_cb(struct hists *hists, struct ui_progress *prog, + hists__resort_cb_t cb); +int hists__collapse_resort(struct hists *hists, struct ui_progress *prog); + +void hists__decay_entries(struct hists *hists, bool zap_user, bool zap_kernel); +void hists__delete_entries(struct hists *hists); +void hists__delete_all_entries(struct hists *hists); +void hists__output_recalc_col_len(struct hists *hists, int max_rows); + +struct hist_entry *hists__get_entry(struct hists *hists, int idx); + +u64 hists__total_period(struct hists *hists); +u64 hists__total_latency(struct hists *hists); +void hists__reset_stats(struct hists *hists); +void hists__inc_stats(struct hists *hists, struct hist_entry *h); +void hists__inc_nr_events(struct hists *hists); +void hists__inc_nr_samples(struct hists *hists, bool filtered); +void hists__inc_nr_lost_samples(struct hists *hists, u32 lost); +void hists__inc_nr_dropped_samples(struct hists *hists, u32 lost); + +size_t hists__fprintf(struct hists *hists, bool show_header, int max_rows, + int max_cols, float min_pcnt, FILE *fp, + bool ignore_callchains); +size_t evlist__fprintf_nr_events(struct evlist *evlist, FILE *fp); + +void hists__filter_by_dso(struct hists *hists); +void hists__filter_by_thread(struct hists *hists); +void hists__filter_by_symbol(struct hists *hists); +void hists__filter_by_socket(struct hists *hists); +void hists__filter_by_parallelism(struct hists *hists); + +static inline bool hists__has_filter(struct hists *hists) +{ + return hists->thread_filter || hists->dso_filter || + hists->symbol_filter_str || (hists->socket_filter > -1) || + hists->parallelism_filter; +} + +u16 hists__col_len(struct hists *hists, enum hist_column col); +void hists__set_col_len(struct hists *hists, enum hist_column col, u16 len); +bool hists__new_col_len(struct hists *hists, enum hist_column col, u16 len); +void hists__reset_col_len(struct hists *hists); +void hists__calc_col_len(struct hists *hists, struct hist_entry *he); + +void hists__match(struct hists *leader, struct hists *other); +int hists__link(struct hists *leader, struct hists *other); +int hists__unlink(struct hists *hists); + +static inline float hist_entry__get_percent_limit(struct hist_entry *he) +{ + u64 period = he->stat.period; + u64 total_period = hists__total_period(he->hists); + + if (unlikely(total_period == 0)) + return 0; + + if (symbol_conf.cumulate_callchain) + period = he->stat_acc->period; + + return period * 100.0 / total_period; +} + +struct hists_evsel { + struct evsel evsel; + struct hists hists; +}; + +static inline struct evsel *hists_to_evsel(struct hists *hists) +{ + struct hists_evsel *hevsel = container_of(hists, struct hists_evsel, hists); + return &hevsel->evsel; +} + +static inline struct hists *evsel__hists(struct evsel *evsel) +{ + struct hists_evsel *hevsel = (struct hists_evsel *)evsel; + return &hevsel->hists; +} + +static __pure inline bool hists__has_callchains(struct hists *hists) +{ + return hists->has_callchains; +} + +int hists__init(void); +int __hists__init(struct hists *hists, struct perf_hpp_list *hpp_list); + +struct rb_root_cached *hists__get_rotate_entries_in(struct hists *hists); + +struct perf_hpp { + char *buf; + size_t size; + const char *sep; + void *ptr; + bool skip; +}; + +typedef int64_t (*perf_hpp_fmt_cmp_t)( + struct perf_hpp_fmt *, struct hist_entry *, struct hist_entry *); + +struct perf_hpp_fmt { + const char *name; + int (*header)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hists *hists, int line, int *span); + int (*width)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hists *hists); + void (*init)(struct perf_hpp_fmt *fmt, struct hist_entry *he); + int (*color)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he); + int (*entry)(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he); + perf_hpp_fmt_cmp_t cmp; + perf_hpp_fmt_cmp_t collapse; + perf_hpp_fmt_cmp_t sort; + bool (*equal)(struct perf_hpp_fmt *a, struct perf_hpp_fmt *b); + void (*free)(struct perf_hpp_fmt *fmt); + + struct list_head list; + struct list_head sort_list; + bool elide; + int len; + int user_len; + int idx; + int level; +}; + +struct perf_hpp_list { + struct list_head fields; + struct list_head sorts; + + int nr_header_lines; + int need_collapse; + int parent; + int sym; + int dso; + int socket; + int thread; + int comm; + int comm_nodigit; +}; + +extern struct perf_hpp_list perf_hpp_list; + +struct perf_hpp_list_node { + struct list_head list; + struct perf_hpp_list hpp; + int level; + bool skip; +}; + +void perf_hpp_list__column_register(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); +void perf_hpp_list__register_sort_field(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); +void perf_hpp_list__prepend_sort_field(struct perf_hpp_list *list, + struct perf_hpp_fmt *format); + +static inline void perf_hpp__column_register(struct perf_hpp_fmt *format) +{ + perf_hpp_list__column_register(&perf_hpp_list, format); +} + +static inline void perf_hpp__register_sort_field(struct perf_hpp_fmt *format) +{ + perf_hpp_list__register_sort_field(&perf_hpp_list, format); +} + +static inline void perf_hpp__prepend_sort_field(struct perf_hpp_fmt *format) +{ + perf_hpp_list__prepend_sort_field(&perf_hpp_list, format); +} + +#define perf_hpp_list__for_each_format(_list, format) \ + list_for_each_entry(format, &(_list)->fields, list) + +#define perf_hpp_list__for_each_format_safe(_list, format, tmp) \ + list_for_each_entry_safe(format, tmp, &(_list)->fields, list) + +#define perf_hpp_list__for_each_sort_list(_list, format) \ + list_for_each_entry(format, &(_list)->sorts, sort_list) + +#define perf_hpp_list__for_each_sort_list_safe(_list, format, tmp) \ + list_for_each_entry_safe(format, tmp, &(_list)->sorts, sort_list) + +#define hists__for_each_format(hists, format) \ + perf_hpp_list__for_each_format((hists)->hpp_list, format) + +#define hists__for_each_sort_list(hists, format) \ + perf_hpp_list__for_each_sort_list((hists)->hpp_list, format) + +extern struct perf_hpp_fmt perf_hpp__format[]; + +enum { + /* Matches perf_hpp__format array. */ + PERF_HPP__OVERHEAD, + PERF_HPP__LATENCY, + PERF_HPP__OVERHEAD_SYS, + PERF_HPP__OVERHEAD_US, + PERF_HPP__OVERHEAD_GUEST_SYS, + PERF_HPP__OVERHEAD_GUEST_US, + PERF_HPP__OVERHEAD_ACC, + PERF_HPP__LATENCY_ACC, + PERF_HPP__SAMPLES, + PERF_HPP__PERIOD, + PERF_HPP__WEIGHT1, + PERF_HPP__WEIGHT2, + PERF_HPP__WEIGHT3, + PERF_HPP__MEM_STAT_OP, + PERF_HPP__MEM_STAT_CACHE, + PERF_HPP__MEM_STAT_MEMORY, + PERF_HPP__MEM_STAT_SNOOP, + PERF_HPP__MEM_STAT_DTLB, + + PERF_HPP__MAX_INDEX +}; + +void perf_hpp__init(void); +void perf_hpp__cancel_cumulate(struct evlist *evlist); +void perf_hpp__cancel_latency(struct evlist *evlist); +void perf_hpp__setup_output_field(struct perf_hpp_list *list); +void perf_hpp__reset_output_field(struct perf_hpp_list *list); +void perf_hpp__append_sort_keys(struct perf_hpp_list *list); +int perf_hpp__setup_hists_formats(struct perf_hpp_list *list, + struct evlist *evlist); +int perf_hpp__alloc_mem_stats(struct perf_hpp_list *list, + struct evlist *evlist); + + +bool perf_hpp__is_sort_entry(struct perf_hpp_fmt *format); +bool perf_hpp__is_dynamic_entry(struct perf_hpp_fmt *format); +bool perf_hpp__defined_dynamic_entry(struct perf_hpp_fmt *fmt, struct hists *hists); +bool perf_hpp__is_trace_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_srcline_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_srcfile_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_thread_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_comm_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_dso_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_sym_entry(struct perf_hpp_fmt *fmt); +bool perf_hpp__is_parallelism_entry(struct perf_hpp_fmt *fmt); + +struct perf_hpp_fmt *perf_hpp_fmt__dup(struct perf_hpp_fmt *fmt); + +int hist_entry__filter(struct hist_entry *he, int type, const void *arg); + +static inline bool perf_hpp__should_skip(struct perf_hpp_fmt *format, + struct hists *hists) +{ + if (format->elide) + return true; + + if (perf_hpp__is_dynamic_entry(format) && + !perf_hpp__defined_dynamic_entry(format, hists)) + return true; + + return false; +} + +void perf_hpp__reset_width(struct perf_hpp_fmt *fmt, struct hists *hists); +void perf_hpp__reset_sort_width(struct perf_hpp_fmt *fmt, struct hists *hists); +void perf_hpp__set_user_width(const char *width_list_str); +void hists__reset_column_width(struct hists *hists); + +enum perf_hpp_fmt_type { + PERF_HPP_FMT_TYPE__RAW, + PERF_HPP_FMT_TYPE__PERCENT, + PERF_HPP_FMT_TYPE__LATENCY, + PERF_HPP_FMT_TYPE__AVERAGE, +}; + +typedef u64 (*hpp_field_fn)(struct hist_entry *he); +typedef int (*hpp_callback_fn)(struct perf_hpp *hpp, bool front); +typedef int (*hpp_snprint_fn)(struct perf_hpp *hpp, const char *fmt, ...); + +int hpp__fmt(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he, hpp_field_fn get_field, + const char *fmtstr, hpp_snprint_fn print_fn, + enum perf_hpp_fmt_type fmtype); +int hpp__fmt_acc(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he, hpp_field_fn get_field, + const char *fmtstr, hpp_snprint_fn print_fn, + enum perf_hpp_fmt_type fmtype); +int hpp__fmt_mem_stat(struct perf_hpp_fmt *fmt, struct perf_hpp *hpp, + struct hist_entry *he, enum mem_stat_type mst, + const char *fmtstr, hpp_snprint_fn print_fn); + +static inline void advance_hpp(struct perf_hpp *hpp, int inc) +{ + hpp->buf += inc; + hpp->size -= inc; +} + +static inline size_t perf_hpp__use_color(void) +{ + return !symbol_conf.field_sep; +} + +static inline size_t perf_hpp__color_overhead(void) +{ + return perf_hpp__use_color() ? + (COLOR_MAXLEN + sizeof(PERF_COLOR_RESET)) * PERF_HPP__MAX_INDEX + : 0; +} + +struct evlist; + +struct hist_browser_timer { + void (*timer)(void *arg); + void *arg; + int refresh; +}; + +enum rstype { + A_NORMAL, + A_ASM, + A_SOURCE +}; + +struct block_hist { + struct hists block_hists; + struct perf_hpp_list block_list; + struct perf_hpp_fmt block_fmt; + int block_idx; + bool valid; + struct hist_entry he; +}; + +#define NO_ADDR 0 + +#ifdef HAVE_SLANG_SUPPORT +#include "../ui/keysyms.h" +void attr_to_script(char *buf, struct perf_event_attr *attr); + +int __hist_entry__tui_annotate(struct hist_entry *he, struct map_symbol *ms, + struct evsel *evsel, + struct hist_browser_timer *hbt, u64 al_addr); + +int hist_entry__tui_annotate(struct hist_entry *he, struct evsel *evsel, + struct hist_browser_timer *hbt, u64 al_addr); + +int evlist__tui_browse_hists(struct evlist *evlist, const char *help, struct hist_browser_timer *hbt, + float min_pcnt, struct perf_env *env, bool warn_lost_event); + +int script_browse(const char *script_opt, struct evsel *evsel); + +void run_script(char *cmd); +int res_sample_browse(struct res_sample *res_samples, int num_res, + struct evsel *evsel, enum rstype rstype); +void res_sample_init(void); + +int block_hists_tui_browse(struct block_hist *bh, struct evsel *evsel, + float min_percent, struct perf_env *env); +#else +static inline +int evlist__tui_browse_hists(struct evlist *evlist __maybe_unused, + const char *help __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused, + float min_pcnt __maybe_unused, + struct perf_env *env __maybe_unused, + bool warn_lost_event __maybe_unused) +{ + return 0; +} +static inline int __hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, + struct map_symbol *ms __maybe_unused, + struct evsel *evsel __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused, + u64 al_addr __maybe_unused) +{ + return 0; +} + +static inline int hist_entry__tui_annotate(struct hist_entry *he __maybe_unused, + struct evsel *evsel __maybe_unused, + struct hist_browser_timer *hbt __maybe_unused, + u64 al_addr __maybe_unused) +{ + return 0; +} + +static inline int script_browse(const char *script_opt __maybe_unused, + struct evsel *evsel __maybe_unused) +{ + return 0; +} + +static inline int res_sample_browse(struct res_sample *res_samples __maybe_unused, + int num_res __maybe_unused, + struct evsel *evsel __maybe_unused, + enum rstype rstype __maybe_unused) +{ + return 0; +} + +static inline void res_sample_init(void) {} + +static inline int block_hists_tui_browse(struct block_hist *bh __maybe_unused, + struct evsel *evsel __maybe_unused, + float min_percent __maybe_unused, + struct perf_env *env __maybe_unused) +{ + return 0; +} + +#define K_LEFT -1000 +#define K_RIGHT -2000 +#define K_SWITCH_INPUT_DATA -3000 +#define K_RELOAD -4000 +#endif + +unsigned int hists__sort_list_width(struct hists *hists); +unsigned int hists__overhead_width(struct hists *hists); + +void hist__account_cycles(struct branch_stack *bs, struct addr_location *al, + struct perf_sample *sample, bool nonany_branch_mode, + u64 *total_cycles); + +struct option; +int parse_filter_percentage(const struct option *opt, const char *arg, int unset); +int perf_hist_config(const char *var, const char *value); + +void perf_hpp_list__init(struct perf_hpp_list *list); + +enum hierarchy_move_dir { + HMD_NORMAL, + HMD_FORCE_SIBLING, + HMD_FORCE_CHILD, +}; + +struct rb_node *rb_hierarchy_last(struct rb_node *node); +struct rb_node *__rb_hierarchy_next(struct rb_node *node, + enum hierarchy_move_dir hmd); +struct rb_node *rb_hierarchy_prev(struct rb_node *node); + +static inline struct rb_node *rb_hierarchy_next(struct rb_node *node) +{ + return __rb_hierarchy_next(node, HMD_NORMAL); +} + +#define HIERARCHY_INDENT 3 + +bool hist_entry__has_hierarchy_children(struct hist_entry *he, float limit); +int hpp_color_scnprintf(struct perf_hpp *hpp, const char *fmt, ...); +int __hpp__slsmg_color_printf(struct perf_hpp *hpp, const char *fmt, ...); +int __hist_entry__snprintf(struct hist_entry *he, struct perf_hpp *hpp, + struct perf_hpp_list *hpp_list); +int hists__fprintf_headers(struct hists *hists, FILE *fp); +int __hists__scnprintf_title(struct hists *hists, char *bf, size_t size, bool show_freq); + +static inline int hists__scnprintf_title(struct hists *hists, char *bf, size_t size) +{ + return __hists__scnprintf_title(hists, bf, size, true); +} + +#endif /* __PERF_HIST_H */ diff --git a/tools/perf/util/hwmon_pmu.c b/tools/perf/util/hwmon_pmu.c new file mode 100644 index 000000000..ed544dca7 --- /dev/null +++ b/tools/perf/util/hwmon_pmu.c @@ -0,0 +1,839 @@ +// SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "hashmap.h" +#include "hwmon_pmu.h" +#include "pmu.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** Strings that correspond to enum hwmon_type. */ +static const char * const hwmon_type_strs[HWMON_TYPE_MAX] = { + NULL, + "cpu", + "curr", + "energy", + "fan", + "humidity", + "in", + "intrusion", + "power", + "pwm", + "temp", +}; +#define LONGEST_HWMON_TYPE_STR "intrusion" + +/** Strings that correspond to enum hwmon_item. */ +static const char * const hwmon_item_strs[HWMON_ITEM__MAX] = { + NULL, + "accuracy", + "alarm", + "auto_channels_temp", + "average", + "average_highest", + "average_interval", + "average_interval_max", + "average_interval_min", + "average_lowest", + "average_max", + "average_min", + "beep", + "cap", + "cap_hyst", + "cap_max", + "cap_min", + "crit", + "crit_hyst", + "div", + "emergency", + "emergency_hist", + "enable", + "fault", + "freq", + "highest", + "input", + "label", + "lcrit", + "lcrit_hyst", + "lowest", + "max", + "max_hyst", + "min", + "min_hyst", + "mod", + "offset", + "pulses", + "rated_max", + "rated_min", + "reset_history", + "target", + "type", + "vid", +}; +#define LONGEST_HWMON_ITEM_STR "average_interval_max" + +static const char *const hwmon_units[HWMON_TYPE_MAX] = { + NULL, + "V", /* cpu */ + "A", /* curr */ + "J", /* energy */ + "rpm", /* fan */ + "%", /* humidity */ + "V", /* in */ + "", /* intrusion */ + "W", /* power */ + "Hz", /* pwm */ + "'C", /* temp */ +}; + +struct hwmon_pmu { + struct perf_pmu pmu; + struct hashmap events; + char *hwmon_dir; +}; + +/** + * struct hwmon_pmu_event_value: Value in hwmon_pmu->events. + * + * Hwmon files are of the form _ and may have a suffix + * _alarm. + */ +struct hwmon_pmu_event_value { + /** @items: which item files are present. */ + DECLARE_BITMAP(items, HWMON_ITEM__MAX); + /** @alarm_items: which item files are present. */ + DECLARE_BITMAP(alarm_items, HWMON_ITEM__MAX); + /** @label: contents of _label if present. */ + char *label; + /** @name: name computed from label of the form _