From 13f0645c940b426d9a41715e8e77f67bf38337ad Mon Sep 17 00:00:00 2001 From: "Raziel K. Crowe" <101006007+RazielCrowe@users.noreply.github.com> Date: Fri, 9 Sep 2022 14:21:57 +0500 Subject: [PATCH] update --- sound/soc/qcom/qdsp6/q6apm.c | 1 + sound/soc/qcom/qdsp6/q6asm-dai.c | 23 +- sound/soc/qcom/qdsp6/q6asm.c | 2 +- sound/soc/rockchip/rk3288_hdmi_analog.c | 12 +- sound/soc/rockchip/rk3399_gru_sound.c | 16 +- sound/soc/rockchip/rockchip_i2s.c | 180 +- sound/soc/rockchip/rockchip_i2s_tdm.c | 13 +- sound/soc/rockchip/rockchip_max98090.c | 16 +- sound/soc/rockchip/rockchip_pdm.c | 7 +- sound/soc/rockchip/rockchip_rt5645.c | 2 +- sound/soc/rockchip/rockchip_spdif.c | 1 + sound/soc/samsung/Kconfig | 20 +- sound/soc/samsung/aries_wm8994.c | 13 +- sound/soc/samsung/bells.c | 4 +- sound/soc/samsung/h1940_uda1380.c | 5 +- sound/soc/samsung/i2s.c | 8 +- sound/soc/samsung/littlemill.c | 6 +- sound/soc/samsung/lowland.c | 13 +- sound/soc/samsung/midas_wm1811.c | 2 +- sound/soc/samsung/neo1973_wm8753.c | 2 +- sound/soc/samsung/pcm.c | 7 +- sound/soc/samsung/rx1950_uda1380.c | 7 +- sound/soc/samsung/s3c-i2s-v2.c | 17 +- sound/soc/samsung/s3c2412-i2s.c | 7 +- sound/soc/samsung/s3c24xx-i2s.c | 14 +- sound/soc/samsung/smartq_wm8987.c | 8 +- sound/soc/samsung/snow.c | 2 +- sound/soc/samsung/spdif.c | 10 +- sound/soc/samsung/speyside.c | 14 +- sound/soc/samsung/tobermory.c | 12 +- sound/soc/sh/Kconfig | 2 +- sound/soc/sh/fsi.c | 6 +- sound/soc/sh/hac.c | 3 +- sound/soc/sh/rcar/core.c | 47 +- sound/soc/sh/rcar/dma.c | 9 +- sound/soc/sh/rcar/rsnd.h | 2 +- sound/soc/sh/rcar/src.c | 7 +- sound/soc/sh/rcar/ssi.c | 14 +- sound/soc/sh/rcar/ssiu.c | 14 +- sound/soc/sh/rz-ssi.c | 57 +- sound/soc/sh/siu_pcm.c | 17 +- sound/soc/sh/ssi.c | 13 +- sound/soc/sof/Kconfig | 7 + sound/soc/sof/Makefile | 14 +- sound/soc/sof/amd/Kconfig | 1 + sound/soc/sof/amd/acp-dsp-offset.h | 2 + sound/soc/sof/amd/acp-trace.c | 4 +- sound/soc/sof/amd/acp.c | 102 +- sound/soc/sof/amd/acp.h | 14 +- sound/soc/sof/amd/pci-rn.c | 19 +- sound/soc/sof/amd/renoir.c | 10 +- sound/soc/sof/compress.c | 75 +- sound/soc/sof/control.c | 36 +- sound/soc/sof/core.c | 35 +- sound/soc/sof/debug.c | 20 +- sound/soc/sof/imx/Kconfig | 1 + sound/soc/sof/imx/imx8.c | 40 +- sound/soc/sof/imx/imx8m.c | 20 +- sound/soc/sof/intel/Kconfig | 29 +- sound/soc/sof/intel/Makefile | 4 +- sound/soc/sof/intel/apl.c | 128 +- sound/soc/sof/intel/atom.c | 16 +- sound/soc/sof/intel/bdw.c | 28 +- sound/soc/sof/intel/byt.c | 61 +- sound/soc/sof/intel/cnl.c | 243 +- sound/soc/sof/intel/hda-dai.c | 642 +- sound/soc/sof/intel/hda-dsp.c | 86 +- sound/soc/sof/intel/hda-ipc.c | 107 +- sound/soc/sof/intel/hda-loader.c | 148 +- sound/soc/sof/intel/hda-pcm.c | 74 +- sound/soc/sof/intel/hda-probes.c | 16 +- sound/soc/sof/intel/hda-stream.c | 115 +- sound/soc/sof/intel/hda-trace.c | 4 +- sound/soc/sof/intel/hda.c | 269 +- sound/soc/sof/intel/hda.h | 124 +- sound/soc/sof/intel/icl.c | 122 +- sound/soc/sof/intel/pci-apl.c | 37 +- sound/soc/sof/intel/pci-cnl.c | 55 +- sound/soc/sof/intel/pci-icl.c | 37 +- sound/soc/sof/intel/pci-tgl.c | 98 +- sound/soc/sof/intel/pci-tng.c | 27 +- sound/soc/sof/intel/shim.h | 18 +- sound/soc/sof/intel/tgl.c | 176 +- sound/soc/sof/ipc.c | 995 +- sound/soc/sof/ipc3-control.c | 159 +- sound/soc/sof/ipc3-pcm.c | 21 +- sound/soc/sof/ipc3-topology.c | 179 +- sound/soc/sof/ipc3.c | 1058 +- sound/soc/sof/loader.c | 698 +- sound/soc/sof/mediatek/Kconfig | 12 + sound/soc/sof/mediatek/Makefile | 2 + sound/soc/sof/mediatek/adsp_helper.h | 21 +- sound/soc/sof/mediatek/mt8195/mt8195-clk.c | 7 + sound/soc/sof/mediatek/mt8195/mt8195-loader.c | 13 +- sound/soc/sof/mediatek/mt8195/mt8195.c | 242 +- sound/soc/sof/mediatek/mt8195/mt8195.h | 5 + sound/soc/sof/ops.c | 2 +- sound/soc/sof/ops.h | 40 +- sound/soc/sof/pcm.c | 43 +- sound/soc/sof/pm.c | 43 +- sound/soc/sof/sof-acpi-dev.c | 6 +- sound/soc/sof/sof-audio.c | 398 +- sound/soc/sof/sof-audio.h | 92 +- sound/soc/sof/sof-client-ipc-flood-test.c | 3 +- sound/soc/sof/sof-client-ipc-msg-injector.c | 190 +- sound/soc/sof/sof-client-probes.c | 27 +- sound/soc/sof/sof-client-probes.h | 8 +- sound/soc/sof/sof-client.c | 66 +- sound/soc/sof/sof-client.h | 2 + sound/soc/sof/sof-of-dev.c | 12 +- sound/soc/sof/sof-of-dev.h | 1 + sound/soc/sof/sof-pci-dev.c | 96 +- sound/soc/sof/sof-priv.h | 147 +- sound/soc/sof/topology.c | 296 +- sound/soc/sof/trace.c | 602 +- sound/soc/spear/spdif_in.c | 3 +- sound/soc/spear/spdif_out.c | 3 +- sound/soc/sti/sti_uniperif.c | 3 +- sound/soc/stm/stm32_adfsdm.c | 3 +- sound/soc/stm/stm32_i2s.c | 9 +- sound/soc/stm/stm32_sai_sub.c | 11 +- sound/soc/stm/stm32_spdifrx.c | 1 + sound/soc/sunxi/sun4i-codec.c | 85 +- sound/soc/sunxi/sun4i-i2s.c | 23 +- sound/soc/sunxi/sun4i-spdif.c | 3 +- sound/soc/sunxi/sun50i-codec-analog.c | 8 + sound/soc/sunxi/sun8i-codec.c | 7 +- sound/soc/tegra/Kconfig | 21 + sound/soc/tegra/Makefile | 4 + sound/soc/tegra/tegra20_ac97.c | 5 +- sound/soc/tegra/tegra20_das.c | 204 +- sound/soc/tegra/tegra20_i2s.c | 9 +- sound/soc/tegra/tegra20_spdif.c | 1 + sound/soc/tegra/tegra210_adx.c | 2 +- sound/soc/tegra/tegra210_ahub.c | 121 +- sound/soc/tegra/tegra210_i2s.c | 7 +- sound/soc/tegra/tegra30_i2s.c | 9 +- sound/soc/tegra/tegra_asoc_machine.c | 30 +- sound/soc/tegra/tegra_wm8903.c | 10 +- sound/soc/ti/ams-delta.c | 4 +- sound/soc/ti/davinci-i2s.c | 35 +- sound/soc/ti/davinci-mcasp.c | 18 +- sound/soc/ti/davinci-vcif.c | 3 +- sound/soc/ti/j721e-evm.c | 44 +- sound/soc/ti/omap-abe-twl6040.c | 8 +- sound/soc/ti/omap-dmic.c | 3 +- sound/soc/ti/omap-hdmi.c | 1 + sound/soc/ti/omap-mcbsp-priv.h | 2 - sound/soc/ti/omap-mcbsp-st.c | 14 +- sound/soc/ti/omap-mcbsp.c | 34 +- sound/soc/ti/omap-mcpdm.c | 7 +- sound/soc/ti/omap-twl4030.c | 8 +- sound/soc/ti/osk5912.c | 4 +- sound/soc/ti/rx51.c | 2 +- sound/soc/uniphier/aio-compress.c | 7 +- sound/soc/uniphier/evea.c | 1 - sound/soc/ux500/mop500.c | 2 - sound/soc/ux500/mop500_ab8500.c | 11 +- sound/soc/ux500/mop500_ab8500.h | 2 - sound/soc/ux500/ux500_msp_dai.c | 41 +- sound/soc/ux500/ux500_msp_dai.h | 2 - sound/soc/ux500/ux500_msp_i2s.c | 2 - sound/soc/ux500/ux500_msp_i2s.h | 2 - sound/soc/ux500/ux500_pcm.c | 2 - sound/soc/ux500/ux500_pcm.h | 2 - sound/soc/xilinx/xlnx_formatter_pcm.c | 18 +- sound/soc/xilinx/xlnx_i2s.c | 1 + sound/soc/xilinx/xlnx_spdif.c | 1 + sound/soc/xtensa/xtfpga-i2s.c | 19 +- sound/spi/Kconfig | 2 +- sound/usb/6fire/pcm.c | 2 +- sound/usb/bcd2000/bcd2000.c | 3 +- sound/usb/card.c | 9 + sound/usb/card.h | 3 +- sound/usb/clock.c | 11 + sound/usb/endpoint.c | 92 +- sound/usb/hiface/pcm.c | 2 +- sound/usb/implicit.c | 10 +- sound/usb/line6/pcm.c | 4 +- sound/usb/line6/pod.c | 8 +- sound/usb/line6/podhd.c | 4 +- sound/usb/midi.c | 7 +- sound/usb/mixer_maps.c | 64 +- sound/usb/mixer_quirks.c | 188 +- sound/usb/mixer_scarlett_gen2.c | 91 +- sound/usb/mixer_us16x08.c | 6 +- sound/usb/pcm.c | 24 +- sound/usb/quirks-table.h | 267 +- sound/usb/quirks.c | 25 + sound/usb/usbaudio.h | 7 + sound/usb/usx2y/usb_stream.c | 6 +- sound/usb/usx2y/usbusx2yaudio.c | 2 +- sound/usb/usx2y/usx2yhwdeppcm.c | 2 +- sound/x86/intel_hdmi_audio.c | 15 +- sound/xen/xen_snd_front_evtchnl.c | 44 +- sound/xen/xen_snd_front_evtchnl.h | 9 - tools/Makefile | 43 +- tools/accounting/.gitignore | 1 + tools/accounting/Makefile | 2 +- tools/accounting/getdelays.c | 12 +- tools/arch/arm64/include/asm/cputype.h | 12 +- tools/arch/arm64/include/uapi/asm/kvm.h | 42 +- tools/arch/arm64/include/uapi/asm/perf_regs.h | 7 +- tools/arch/s390/include/uapi/asm/kvm.h | 1 + tools/arch/x86/include/asm/amd-ibs.h | 18 +- tools/arch/x86/include/asm/cpufeatures.h | 26 +- .../arch/x86/include/asm/disabled-features.h | 37 +- tools/arch/x86/include/asm/msr-index.h | 69 + tools/arch/x86/include/asm/rmwcc.h | 21 - tools/arch/x86/include/uapi/asm/kvm.h | 33 +- tools/arch/x86/include/uapi/asm/svm.h | 13 + tools/arch/x86/include/uapi/asm/vmx.h | 4 +- tools/bpf/Makefile | 7 +- tools/bpf/bpf_jit_disasm.c | 5 +- .../bpftool/Documentation/bpftool-cgroup.rst | 16 +- .../bpftool/Documentation/bpftool-feature.rst | 12 + .../bpftool/Documentation/bpftool-prog.rst | 5 +- tools/bpf/bpftool/Makefile | 17 +- tools/bpf/bpftool/bash-completion/bpftool | 30 +- tools/bpf/bpftool/btf.c | 119 +- tools/bpf/bpftool/btf_dumper.c | 29 + tools/bpf/bpftool/cgroup.c | 162 +- tools/bpf/bpftool/common.c | 154 +- tools/bpf/bpftool/feature.c | 170 +- tools/bpf/bpftool/gen.c | 120 +- tools/bpf/bpftool/jit_disasm.c | 42 +- tools/bpf/bpftool/link.c | 57 +- tools/bpf/bpftool/main.c | 4 +- tools/bpf/bpftool/main.h | 22 +- tools/bpf/bpftool/map.c | 82 +- tools/bpf/bpftool/perf.c | 116 +- tools/bpf/bpftool/prog.c | 77 +- tools/bpf/bpftool/tracelog.c | 2 +- tools/bpf/resolve_btfids/main.c | 40 +- tools/bpf/runqslower/Makefile | 7 +- tools/bpf/runqslower/runqslower.c | 18 +- tools/build/Makefile.feature | 9 +- tools/build/feature/Makefile | 40 +- tools/build/feature/test-all.c | 4 + tools/build/feature/test-libcrypto.c | 15 +- tools/gpio/gpio-event-mon.c | 6 +- tools/include/asm-generic/bitops/non-atomic.h | 34 +- tools/include/linux/bitmap.h | 29 +- tools/include/linux/bitops.h | 16 + tools/include/linux/btf_ids.h | 35 +- tools/include/linux/compiler_types.h | 4 + tools/include/linux/list.h | 11 + tools/include/linux/objtool.h | 27 +- tools/include/linux/sched/mm.h | 2 + tools/include/nolibc/nolibc.h | 2540 +- tools/include/uapi/asm-generic/fcntl.h | 32 +- tools/include/uapi/asm-generic/socket.h | 2 + tools/include/uapi/asm-generic/unistd.h | 4 +- tools/include/uapi/asm/bitsperlong.h | 2 + tools/include/uapi/asm/bpf_perf_event.h | 2 + tools/include/uapi/drm/i915_drm.h | 756 +- tools/include/uapi/linux/bpf.h | 224 +- tools/include/uapi/linux/btf.h | 21 +- tools/include/uapi/linux/fs.h | 2 +- tools/include/uapi/linux/fscrypt.h | 3 +- tools/include/uapi/linux/if_link.h | 3 + tools/include/uapi/linux/if_tun.h | 2 +- tools/include/uapi/linux/kvm.h | 181 +- tools/include/uapi/linux/perf_event.h | 9 +- tools/include/uapi/linux/pkt_cls.h | 4 +- tools/include/uapi/linux/prctl.h | 9 + tools/include/uapi/linux/seg6.h | 4 +- tools/include/uapi/linux/usbdevice_fs.h | 4 +- tools/include/uapi/linux/vhost.h | 35 +- tools/include/uapi/sound/asound.h | 2 +- tools/kvm/kvm_stat/kvm_stat | 3 +- tools/lib/bitmap.c | 26 +- tools/lib/bpf/Build | 5 +- tools/lib/bpf/Makefile | 6 +- tools/lib/bpf/bpf.c | 365 +- tools/lib/bpf/bpf.h | 155 +- tools/lib/bpf/bpf_core_read.h | 48 +- tools/lib/bpf/bpf_helpers.h | 39 + tools/lib/bpf/bpf_tracing.h | 83 +- tools/lib/bpf/btf.c | 427 +- tools/lib/bpf/btf.h | 118 +- tools/lib/bpf/btf_dump.c | 160 +- tools/lib/bpf/gen_loader.c | 2 +- tools/lib/bpf/libbpf.c | 3295 +- tools/lib/bpf/libbpf.h | 834 +- tools/lib/bpf/libbpf.map | 138 +- tools/lib/bpf/libbpf_common.h | 16 +- tools/lib/bpf/libbpf_internal.h | 70 +- tools/lib/bpf/libbpf_legacy.h | 28 +- tools/lib/bpf/libbpf_probes.c | 125 +- tools/lib/bpf/libbpf_version.h | 4 +- tools/lib/bpf/linker.c | 7 +- tools/lib/bpf/netlink.c | 62 +- tools/lib/bpf/relo_core.c | 581 +- tools/lib/bpf/relo_core.h | 16 +- tools/lib/bpf/skel_internal.h | 4 +- tools/lib/perf/cpumap.c | 2 +- tools/lib/perf/evlist.c | 84 +- tools/lib/perf/evsel.c | 130 +- tools/lib/perf/include/internal/evlist.h | 3 +- tools/lib/perf/include/internal/evsel.h | 15 + tools/lib/perf/include/internal/lib.h | 2 + tools/lib/perf/include/perf/cpumap.h | 5 +- tools/lib/perf/include/perf/event.h | 60 +- tools/lib/perf/include/perf/evsel.h | 5 +- tools/lib/perf/lib.c | 20 + tools/lib/perf/tests/test-evsel.c | 161 + tools/lib/subcmd/parse-options.c | 17 +- tools/memory-model/README | 3 +- tools/objtool/Build | 12 +- tools/objtool/Makefile | 14 +- tools/objtool/arch/x86/decode.c | 7 +- tools/objtool/arch/x86/special.c | 2 +- tools/objtool/builtin-check.c | 169 +- tools/objtool/check.c | 847 +- tools/objtool/elf.c | 217 +- tools/objtool/include/objtool/arch.h | 1 + tools/objtool/include/objtool/builtin.h | 36 +- tools/objtool/include/objtool/check.h | 24 +- tools/objtool/include/objtool/elf.h | 17 +- tools/objtool/include/objtool/objtool.h | 1 + tools/objtool/include/objtool/warn.h | 35 +- tools/objtool/objtool.c | 104 +- tools/objtool/weak.c | 9 +- tools/perf/.gitignore | 1 - tools/perf/Build | 1 + tools/perf/Documentation/intel-hybrid.txt | 10 - tools/perf/Documentation/perf-annotate.txt | 5 + .../perf/Documentation/perf-buildid-list.txt | 4 + tools/perf/Documentation/perf-c2c.txt | 39 +- tools/perf/Documentation/perf-dlfilter.txt | 22 + tools/perf/Documentation/perf-inject.txt | 21 + tools/perf/Documentation/perf-intel-pt.txt | 346 +- tools/perf/Documentation/perf-kvm.txt | 22 +- tools/perf/Documentation/perf-lock.txt | 76 +- tools/perf/Documentation/perf-record.txt | 29 +- tools/perf/Documentation/perf-script.txt | 14 +- tools/perf/Documentation/perf-stat.txt | 33 +- tools/perf/Documentation/perf-top.txt | 2 +- .../Documentation/perf.data-file-format.txt | 20 +- tools/perf/Documentation/perf.txt | 2 +- tools/perf/Makefile.config | 110 +- tools/perf/Makefile.perf | 26 +- tools/perf/arch/arm/util/cs-etm.c | 3 +- tools/perf/arch/arm64/util/arm-spe.c | 3 +- tools/perf/arch/arm64/util/mem-events.c | 6 +- tools/perf/arch/arm64/util/perf_regs.c | 38 + tools/perf/arch/arm64/util/pmu.c | 4 +- tools/perf/arch/arm64/util/unwind-libunwind.c | 73 +- tools/perf/arch/riscv/Makefile | 1 + tools/perf/arch/s390/util/auxtrace.c | 1 + tools/perf/arch/x86/tests/Build | 1 - tools/perf/arch/x86/tests/arch-tests.c | 2 - tools/perf/arch/x86/tests/intel-cqm.c | 2 +- tools/perf/arch/x86/util/evlist.c | 73 +- tools/perf/arch/x86/util/evsel.c | 104 + tools/perf/arch/x86/util/header.c | 27 +- tools/perf/arch/x86/util/intel-bts.c | 3 +- tools/perf/arch/x86/util/intel-pt.c | 34 +- tools/perf/arch/x86/util/iostat.c | 2 +- tools/perf/arch/x86/util/perf_regs.c | 12 + tools/perf/arch/x86/util/topdown.c | 97 +- tools/perf/arch/x86/util/tsc.c | 77 + tools/perf/bench/Build | 1 + tools/perf/bench/bench.h | 2 + tools/perf/bench/numa.c | 4 +- tools/perf/builtin-annotate.c | 32 + tools/perf/builtin-bench.c | 8 + tools/perf/builtin-buildid-list.c | 39 +- tools/perf/builtin-c2c.c | 550 +- tools/perf/builtin-inject.c | 1208 +- tools/perf/builtin-kvm.c | 10 +- tools/perf/builtin-list.c | 2 +- tools/perf/builtin-lock.c | 1046 +- tools/perf/builtin-record.c | 132 +- tools/perf/builtin-report.c | 6 + tools/perf/builtin-sched.c | 26 +- tools/perf/builtin-script.c | 64 +- tools/perf/builtin-stat.c | 135 +- tools/perf/builtin-timechart.c | 1 + tools/perf/builtin-trace.c | 5 +- tools/perf/builtin-version.c | 1 + tools/perf/builtin.h | 1 + tools/perf/command-list.txt | 1 + tools/perf/perf.c | 1 + tools/perf/pmu-events/Build | 21 +- .../arch/arm64/common-and-microarch.json | 66 + tools/perf/pmu-events/arch/arm64/mapfile.csv | 15 + .../pmu-events/arch/s390/cf_z10/basic.json | 48 +- .../pmu-events/arch/s390/cf_z10/crypto.json | 64 +- .../pmu-events/arch/s390/cf_z10/extended.json | 36 +- .../pmu-events/arch/s390/cf_z13/basic.json | 48 +- .../pmu-events/arch/s390/cf_z13/crypto.json | 64 +- .../pmu-events/arch/s390/cf_z13/extended.json | 100 +- .../pmu-events/arch/s390/cf_z14/basic.json | 32 +- .../pmu-events/arch/s390/cf_z14/crypto.json | 64 +- .../pmu-events/arch/s390/cf_z14/extended.json | 102 +- .../pmu-events/arch/s390/cf_z15/basic.json | 32 +- .../pmu-events/arch/s390/cf_z15/crypto6.json | 112 + .../pmu-events/arch/s390/cf_z15/extended.json | 108 +- .../pmu-events/arch/s390/cf_z196/basic.json | 48 +- .../pmu-events/arch/s390/cf_z196/crypto.json | 64 +- .../arch/s390/cf_z196/extended.json | 44 +- .../pmu-events/arch/s390/cf_zec12/basic.json | 48 +- .../pmu-events/arch/s390/cf_zec12/crypto.json | 64 +- .../arch/s390/cf_zec12/extended.json | 66 +- tools/perf/pmu-events/arch/s390/mapfile.csv | 1 + .../pmu-events/arch/x86/alderlake/cache.json | 178 +- .../arch/x86/alderlake/floating-point.json | 19 +- .../arch/x86/alderlake/frontend.json | 38 +- .../pmu-events/arch/x86/alderlake/memory.json | 40 +- .../pmu-events/arch/x86/alderlake/other.json | 97 +- .../arch/x86/alderlake/pipeline.json | 507 +- .../arch/x86/alderlake/uncore-other.json | 2 +- .../arch/x86/alderlake/virtual-memory.json | 63 +- .../pmu-events/arch/x86/bonnell/cache.json | 2 +- .../arch/x86/bonnell/floating-point.json | 2 +- .../pmu-events/arch/x86/bonnell/frontend.json | 2 +- .../pmu-events/arch/x86/bonnell/memory.json | 2 +- .../pmu-events/arch/x86/bonnell/other.json | 2 +- .../pmu-events/arch/x86/bonnell/pipeline.json | 2 +- .../arch/x86/bonnell/virtual-memory.json | 2 +- .../arch/x86/broadwell/bdw-metrics.json | 130 +- .../pmu-events/arch/x86/broadwell/cache.json | 2 +- .../arch/x86/broadwell/floating-point.json | 2 +- .../arch/x86/broadwell/frontend.json | 2 +- .../pmu-events/arch/x86/broadwell/memory.json | 2 +- .../pmu-events/arch/x86/broadwell/other.json | 2 +- .../arch/x86/broadwell/pipeline.json | 2 +- .../arch/x86/broadwell/virtual-memory.json | 2 +- .../arch/x86/broadwellde/bdwde-metrics.json | 142 +- .../arch/x86/broadwellde/cache.json | 2 +- .../arch/x86/broadwellde/floating-point.json | 2 +- .../arch/x86/broadwellde/frontend.json | 2 +- .../arch/x86/broadwellde/memory.json | 2 +- .../arch/x86/broadwellde/other.json | 2 +- .../arch/x86/broadwellde/pipeline.json | 2 +- .../arch/x86/broadwellde/uncore-cache.json | 3727 +- .../arch/x86/broadwellde/uncore-memory.json | 2887 +- .../arch/x86/broadwellde/uncore-power.json | 492 +- .../arch/x86/broadwellde/virtual-memory.json | 2 +- .../arch/x86/broadwellx/bdx-metrics.json | 576 +- .../pmu-events/arch/x86/broadwellx/cache.json | 22 +- .../arch/x86/broadwellx/floating-point.json | 9 +- .../arch/x86/broadwellx/frontend.json | 2 +- .../arch/x86/broadwellx/memory.json | 39 +- .../pmu-events/arch/x86/broadwellx/other.json | 2 +- .../arch/x86/broadwellx/pipeline.json | 4 +- .../arch/x86/broadwellx/uncore-cache.json | 3501 +- .../x86/broadwellx/uncore-interconnect.json | 1428 +- .../arch/x86/broadwellx/uncore-memory.json | 2849 +- .../arch/x86/broadwellx/uncore-power.json | 437 +- .../arch/x86/broadwellx/virtual-memory.json | 2 +- .../arch/x86/cascadelakex/cache.json | 1172 +- .../arch/x86/cascadelakex/clx-metrics.json | 730 +- .../arch/x86/cascadelakex/floating-point.json | 2 +- .../arch/x86/cascadelakex/frontend.json | 2 +- .../arch/x86/cascadelakex/memory.json | 702 +- .../arch/x86/cascadelakex/other.json | 219 +- .../arch/x86/cascadelakex/pipeline.json | 25 +- .../arch/x86/cascadelakex/uncore-memory.json | 4464 +- .../arch/x86/cascadelakex/uncore-other.json | 23677 +++++++++- .../arch/x86/cascadelakex/virtual-memory.json | 2 +- .../arch/x86/elkhartlake/cache.json | 956 +- .../arch/x86/elkhartlake/floating-point.json | 19 +- .../arch/x86/elkhartlake/frontend.json | 34 +- .../arch/x86/elkhartlake/memory.json | 388 +- .../arch/x86/elkhartlake/other.json | 540 +- .../arch/x86/elkhartlake/pipeline.json | 214 +- .../arch/x86/elkhartlake/virtual-memory.json | 151 +- .../pmu-events/arch/x86/goldmont/cache.json | 2 +- .../arch/x86/goldmont/floating-point.json | 2 +- .../arch/x86/goldmont/frontend.json | 2 +- .../pmu-events/arch/x86/goldmont/memory.json | 2 +- .../pmu-events/arch/x86/goldmont/other.json | 31 +- .../arch/x86/goldmont/pipeline.json | 33 +- .../arch/x86/goldmont/virtual-memory.json | 2 +- .../arch/x86/goldmontplus/cache.json | 2 +- .../arch/x86/goldmontplus/floating-point.json | 2 +- .../arch/x86/goldmontplus/frontend.json | 2 +- .../arch/x86/goldmontplus/memory.json | 2 +- .../arch/x86/goldmontplus/other.json | 37 +- .../arch/x86/goldmontplus/pipeline.json | 39 +- .../arch/x86/goldmontplus/virtual-memory.json | 2 +- .../pmu-events/arch/x86/haswell/cache.json | 78 +- .../arch/x86/haswell/floating-point.json | 2 +- .../pmu-events/arch/x86/haswell/frontend.json | 2 +- .../arch/x86/haswell/hsw-metrics.json | 85 +- .../pmu-events/arch/x86/haswell/memory.json | 75 +- .../pmu-events/arch/x86/haswell/other.json | 2 +- .../pmu-events/arch/x86/haswell/pipeline.json | 9 +- .../arch/x86/haswell/uncore-other.json | 7 +- .../arch/x86/haswell/virtual-memory.json | 2 +- .../pmu-events/arch/x86/haswellx/cache.json | 44 +- .../arch/x86/haswellx/floating-point.json | 2 +- .../arch/x86/haswellx/frontend.json | 2 +- .../arch/x86/haswellx/hsx-metrics.json | 496 +- .../pmu-events/arch/x86/haswellx/memory.json | 52 +- .../pmu-events/arch/x86/haswellx/other.json | 2 +- .../arch/x86/haswellx/pipeline.json | 9 +- .../arch/x86/haswellx/uncore-cache.json | 3490 +- .../x86/haswellx/uncore-interconnect.json | 1428 +- .../arch/x86/haswellx/uncore-memory.json | 2839 +- .../arch/x86/haswellx/uncore-power.json | 477 +- .../arch/x86/haswellx/virtual-memory.json | 2 +- .../pmu-events/arch/x86/icelake/cache.json | 49 +- .../arch/x86/icelake/floating-point.json | 2 +- .../pmu-events/arch/x86/icelake/frontend.json | 2 +- .../arch/x86/icelake/icl-metrics.json | 162 +- .../pmu-events/arch/x86/icelake/memory.json | 10 +- .../pmu-events/arch/x86/icelake/other.json | 38 +- .../pmu-events/arch/x86/icelake/pipeline.json | 26 +- .../arch/x86/icelake/virtual-memory.json | 2 +- .../pmu-events/arch/x86/icelakex/cache.json | 59 +- .../arch/x86/icelakex/floating-point.json | 2 +- .../arch/x86/icelakex/frontend.json | 2 +- .../arch/x86/icelakex/icx-metrics.json | 717 +- .../pmu-events/arch/x86/icelakex/memory.json | 27 +- .../pmu-events/arch/x86/icelakex/other.json | 119 +- .../arch/x86/icelakex/pipeline.json | 26 +- .../arch/x86/icelakex/uncore-memory.json | 1523 + .../arch/x86/icelakex/uncore-other.json | 37834 +++++++++++++++- .../arch/x86/icelakex/uncore-power.json | 225 + .../arch/x86/icelakex/virtual-memory.json | 2 +- .../pmu-events/arch/x86/ivybridge/cache.json | 2 +- .../arch/x86/ivybridge/floating-point.json | 2 +- .../arch/x86/ivybridge/frontend.json | 2 +- .../arch/x86/ivybridge/ivb-metrics.json | 94 +- .../pmu-events/arch/x86/ivybridge/memory.json | 2 +- .../pmu-events/arch/x86/ivybridge/other.json | 2 +- .../arch/x86/ivybridge/pipeline.json | 4 +- .../arch/x86/ivybridge/uncore-other.json | 2 +- .../arch/x86/ivybridge/virtual-memory.json | 2 +- .../pmu-events/arch/x86/ivytown/cache.json | 2 +- .../arch/x86/ivytown/floating-point.json | 2 +- .../pmu-events/arch/x86/ivytown/frontend.json | 2 +- .../arch/x86/ivytown/ivt-metrics.json | 100 +- .../pmu-events/arch/x86/ivytown/memory.json | 2 +- .../pmu-events/arch/x86/ivytown/other.json | 2 +- .../pmu-events/arch/x86/ivytown/pipeline.json | 4 +- .../arch/x86/ivytown/uncore-cache.json | 3447 +- .../arch/x86/ivytown/uncore-interconnect.json | 1761 +- .../arch/x86/ivytown/uncore-memory.json | 1774 +- .../arch/x86/ivytown/uncore-power.json | 779 +- .../arch/x86/ivytown/virtual-memory.json | 2 +- .../pmu-events/arch/x86/jaketown/cache.json | 2 +- .../arch/x86/jaketown/floating-point.json | 2 +- .../arch/x86/jaketown/frontend.json | 2 +- .../arch/x86/jaketown/jkt-metrics.json | 17 +- .../pmu-events/arch/x86/jaketown/memory.json | 2 +- .../pmu-events/arch/x86/jaketown/other.json | 2 +- .../arch/x86/jaketown/pipeline.json | 16 +- .../arch/x86/jaketown/uncore-cache.json | 1938 +- .../x86/jaketown/uncore-interconnect.json | 844 +- .../arch/x86/jaketown/uncore-memory.json | 515 +- .../arch/x86/jaketown/uncore-power.json | 483 +- .../arch/x86/jaketown/virtual-memory.json | 2 +- .../arch/x86/knightslanding/cache.json | 2 +- .../x86/knightslanding/floating-point.json | 2 +- .../arch/x86/knightslanding/frontend.json | 2 +- .../arch/x86/knightslanding/memory.json | 2 +- .../arch/x86/knightslanding/pipeline.json | 2 +- .../x86/knightslanding/virtual-memory.json | 2 +- tools/perf/pmu-events/arch/x86/mapfile.csv | 73 +- .../pmu-events/arch/x86/nehalemep/cache.json | 14 +- .../arch/x86/nehalemep/floating-point.json | 2 +- .../arch/x86/nehalemep/frontend.json | 2 +- .../pmu-events/arch/x86/nehalemep/memory.json | 6 +- .../pmu-events/arch/x86/nehalemep/other.json | 66 +- .../arch/x86/nehalemep/pipeline.json | 66 +- .../arch/x86/nehalemep/virtual-memory.json | 2 +- .../pmu-events/arch/x86/nehalemex/cache.json | 3012 +- .../arch/x86/nehalemex/floating-point.json | 232 +- .../arch/x86/nehalemex/frontend.json | 20 +- .../pmu-events/arch/x86/nehalemex/memory.json | 672 +- .../pmu-events/arch/x86/nehalemex/other.json | 170 +- .../arch/x86/nehalemex/pipeline.json | 906 +- .../arch/x86/nehalemex/virtual-memory.json | 92 +- .../arch/x86/sandybridge/cache.json | 2 +- .../arch/x86/sandybridge/floating-point.json | 2 +- .../arch/x86/sandybridge/frontend.json | 4 +- .../arch/x86/sandybridge/memory.json | 2 +- .../arch/x86/sandybridge/other.json | 2 +- .../arch/x86/sandybridge/pipeline.json | 10 +- .../arch/x86/sandybridge/snb-metrics.json | 11 +- .../arch/x86/sandybridge/uncore-other.json | 2 +- .../arch/x86/sandybridge/virtual-memory.json | 2 +- .../pmu-events/arch/x86/silvermont/cache.json | 2 +- .../arch/x86/silvermont/floating-point.json | 2 +- .../arch/x86/silvermont/frontend.json | 2 +- .../arch/x86/silvermont/memory.json | 2 +- .../pmu-events/arch/x86/silvermont/other.json | 2 +- .../arch/x86/silvermont/pipeline.json | 2 +- .../arch/x86/silvermont/virtual-memory.json | 2 +- .../pmu-events/arch/x86/skylake/cache.json | 174 +- .../arch/x86/skylake/floating-point.json | 2 +- .../pmu-events/arch/x86/skylake/frontend.json | 2 +- .../pmu-events/arch/x86/skylake/memory.json | 90 +- .../pmu-events/arch/x86/skylake/other.json | 2 +- .../pmu-events/arch/x86/skylake/pipeline.json | 14 +- .../arch/x86/skylake/skl-metrics.json | 178 +- .../arch/x86/skylake/virtual-memory.json | 2 +- .../pmu-events/arch/x86/skylakex/cache.json | 74 +- .../arch/x86/skylakex/floating-point.json | 2 +- .../arch/x86/skylakex/frontend.json | 2 +- .../pmu-events/arch/x86/skylakex/memory.json | 74 +- .../pmu-events/arch/x86/skylakex/other.json | 66 +- .../arch/x86/skylakex/pipeline.json | 25 +- .../arch/x86/skylakex/skx-metrics.json | 673 +- .../arch/x86/skylakex/uncore-memory.json | 3627 +- .../arch/x86/skylakex/uncore-other.json | 23466 +++++++++- .../arch/x86/skylakex/virtual-memory.json | 2 +- .../pmu-events/arch/x86/tigerlake/cache.json | 48 +- .../arch/x86/tigerlake/floating-point.json | 2 +- .../arch/x86/tigerlake/frontend.json | 2 +- .../pmu-events/arch/x86/tigerlake/memory.json | 2 +- .../pmu-events/arch/x86/tigerlake/other.json | 14 +- .../arch/x86/tigerlake/pipeline.json | 17 +- .../arch/x86/tigerlake/tgl-metrics.json | 396 +- .../arch/x86/tigerlake/virtual-memory.json | 2 +- .../arch/x86/westmereep-dp/cache.json | 2 +- .../x86/westmereep-dp/floating-point.json | 2 +- .../arch/x86/westmereep-dp/frontend.json | 2 +- .../arch/x86/westmereep-dp/memory.json | 2 +- .../arch/x86/westmereep-dp/other.json | 66 +- .../arch/x86/westmereep-dp/pipeline.json | 66 +- .../x86/westmereep-dp/virtual-memory.json | 2 +- .../arch/x86/westmereep-sp/cache.json | 14 +- .../x86/westmereep-sp/floating-point.json | 2 +- .../arch/x86/westmereep-sp/frontend.json | 2 +- .../arch/x86/westmereep-sp/memory.json | 6 +- .../arch/x86/westmereep-sp/other.json | 66 +- .../arch/x86/westmereep-sp/pipeline.json | 66 +- .../x86/westmereep-sp/virtual-memory.json | 2 +- .../pmu-events/arch/x86/westmereex/cache.json | 14 +- .../arch/x86/westmereex/floating-point.json | 2 +- .../arch/x86/westmereex/frontend.json | 2 +- .../arch/x86/westmereex/memory.json | 6 +- .../pmu-events/arch/x86/westmereex/other.json | 66 +- .../arch/x86/westmereex/pipeline.json | 66 +- .../arch/x86/westmereex/virtual-memory.json | 2 +- tools/perf/pmu-events/pmu-events.h | 40 +- tools/perf/scripts/perl/Perf-Trace-Util/Build | 6 +- tools/perf/scripts/python/intel-pt-events.py | 40 +- tools/perf/tests/Build | 1 + tools/perf/tests/bp_account.c | 16 +- tools/perf/tests/bpf-script-example.c | 35 +- tools/perf/tests/bpf.c | 10 +- tools/perf/tests/builtin-test.c | 156 +- tools/perf/tests/code-reading.c | 2 +- tools/perf/tests/cpumap.c | 19 +- tools/perf/tests/event-times.c | 2 +- tools/perf/tests/evsel-roundtrip-name.c | 6 +- tools/perf/tests/expand-cgroup.c | 25 +- tools/perf/tests/expr.c | 15 + tools/perf/tests/hists_cumulate.c | 2 +- tools/perf/tests/hists_filter.c | 4 +- tools/perf/tests/hists_link.c | 4 +- tools/perf/tests/hists_output.c | 2 +- tools/perf/tests/keep-tracking.c | 4 +- tools/perf/tests/mmap-basic.c | 143 +- tools/perf/tests/openat-syscall-all-cpus.c | 23 +- tools/perf/tests/openat-syscall.c | 20 +- tools/perf/tests/parse-events.c | 500 +- tools/perf/tests/parse-metric.c | 77 +- tools/perf/tests/perf-record.c | 18 +- tools/perf/tests/perf-time-to-tsc.c | 31 +- tools/perf/tests/pmu-events.c | 506 +- tools/perf/tests/sample-parsing.c | 14 +- tools/perf/tests/shell/stat_all_metrics.sh | 47 +- tools/perf/tests/shell/stat_all_pmu.sh | 10 + .../perf/tests/shell/test_arm_callgraph_fp.sh | 2 +- tools/perf/tests/shell/test_arm_coresight.sh | 1 - tools/perf/tests/shell/test_arm_spe.sh | 30 +- tools/perf/tests/switch-tracking.c | 24 +- tools/perf/tests/topology.c | 11 + tools/perf/tests/vmlinux-kallsyms.c | 12 +- tools/perf/trace/beauty/arch_errno_names.sh | 14 +- .../perf/trace/beauty/include/linux/socket.h | 23 +- tools/perf/util/Build | 6 + tools/perf/util/amd-sample-raw.c | 68 +- tools/perf/util/annotate.c | 7 +- .../util/arm-spe-decoder/arm-spe-decoder.c | 1 + .../util/arm-spe-decoder/arm-spe-decoder.h | 12 + tools/perf/util/arm-spe.c | 140 +- tools/perf/util/auxtrace.c | 61 +- tools/perf/util/auxtrace.h | 17 +- tools/perf/util/bpf-event.c | 28 +- tools/perf/util/bpf-loader.c | 251 +- tools/perf/util/bpf-utils.c | 5 +- tools/perf/util/bpf_counter.c | 67 +- tools/perf/util/bpf_counter_cgroup.c | 42 +- tools/perf/util/build-id.c | 132 +- tools/perf/util/build-id.h | 16 +- tools/perf/util/callchain.c | 18 +- tools/perf/util/cpumap.c | 80 +- tools/perf/util/cpumap.h | 4 +- tools/perf/util/cs-etm.c | 2 +- tools/perf/util/data-convert-json.c | 5 +- tools/perf/util/data.c | 39 + tools/perf/util/data.h | 4 + tools/perf/util/dlfilter.c | 2 + tools/perf/util/dso.h | 8 + tools/perf/util/dsos.c | 15 +- tools/perf/util/env.c | 62 +- tools/perf/util/env.h | 14 +- tools/perf/util/event.c | 8 +- tools/perf/util/event.h | 48 +- tools/perf/util/events_stats.h | 2 +- tools/perf/util/evlist.c | 176 +- tools/perf/util/evlist.h | 18 +- tools/perf/util/evsel.c | 163 +- tools/perf/util/evsel.h | 33 +- tools/perf/util/expr.c | 13 + tools/perf/util/expr.l | 2 +- tools/perf/util/genelf.c | 6 +- tools/perf/util/genelf.h | 3 + tools/perf/util/header.c | 268 +- tools/perf/util/header.h | 19 +- .../util/intel-pt-decoder/intel-pt-decoder.c | 99 +- .../util/intel-pt-decoder/intel-pt-decoder.h | 1 + .../intel-pt-decoder/intel-pt-insn-decoder.c | 1 + .../intel-pt-decoder/intel-pt-insn-decoder.h | 1 + tools/perf/util/intel-pt.c | 236 +- tools/perf/util/jitdump.c | 9 +- tools/perf/util/libunwind/arm64.c | 2 +- tools/perf/util/llvm-utils.c | 2 +- tools/perf/util/machine.c | 158 +- tools/perf/util/machine.h | 12 +- tools/perf/util/mem-events.c | 78 +- tools/perf/util/mem-events.h | 4 + tools/perf/util/metricgroup.c | 408 +- tools/perf/util/metricgroup.h | 5 +- tools/perf/util/mmap.c | 4 +- tools/perf/util/ordered-events.h | 6 + tools/perf/util/parse-events.c | 716 +- tools/perf/util/parse-events.h | 35 +- tools/perf/util/parse-events.l | 2 + tools/perf/util/path.c | 14 +- tools/perf/util/path.h | 1 + tools/perf/util/perf_api_probe.c | 2 +- tools/perf/util/perf_regs.c | 2 + tools/perf/util/pmu.c | 156 +- tools/perf/util/pmu.h | 12 +- tools/perf/util/probe-event.c | 8 +- tools/perf/util/python-ext-sources | 1 + tools/perf/util/record.c | 4 +- tools/perf/util/record.h | 1 + tools/perf/util/s390-sample-raw.c | 50 +- tools/perf/util/scripting-engines/Build | 4 +- .../scripting-engines/trace-event-python.c | 106 +- tools/perf/util/session.c | 213 +- tools/perf/util/session.h | 4 + tools/perf/util/setup.py | 12 +- tools/perf/util/stat-display.c | 435 +- tools/perf/util/stat-shadow.c | 53 +- tools/perf/util/stat.c | 32 +- tools/perf/util/stat.h | 24 + tools/perf/util/symbol-elf.c | 71 +- tools/perf/util/symbol.c | 6 +- tools/perf/util/symbol_conf.h | 3 +- tools/perf/util/synthetic-events.c | 215 +- tools/perf/util/synthetic-events.h | 4 +- tools/perf/util/thread.c | 1 + tools/perf/util/thread.h | 1 + tools/perf/util/tool.h | 3 +- tools/perf/util/topdown.c | 24 +- tools/perf/util/topdown.h | 6 +- tools/perf/util/trace-event-info.c | 96 + tools/perf/util/tsc.h | 1 + tools/perf/util/unwind-libunwind-local.c | 105 +- tools/perf/util/util.c | 75 +- tools/perf/util/util.h | 15 + tools/power/acpi/common/cmfsize.c | 2 +- tools/power/acpi/common/getopt.c | 2 +- .../os_specific/service_layers/oslinuxtbl.c | 2 +- .../os_specific/service_layers/osunixdir.c | 2 +- .../os_specific/service_layers/osunixmap.c | 2 +- .../os_specific/service_layers/osunixxf.c | 2 +- tools/power/acpi/tools/acpidump/acpidump.h | 2 +- tools/power/acpi/tools/acpidump/apdump.c | 2 +- tools/power/acpi/tools/acpidump/apfiles.c | 2 +- tools/power/acpi/tools/acpidump/apmain.c | 2 +- tools/power/cpupower/debug/i386/dump_psb.c | 6 +- tools/power/pm-graph/README | 6 +- tools/power/pm-graph/bootgraph.py | 20 +- .../config/custom-timeline-functions.cfg | 2 +- tools/power/pm-graph/sleepgraph.py | 518 +- .../power/x86/intel-speed-select/hfi-events.c | 4 +- .../x86/intel-speed-select/isst-config.c | 6 + .../x86/intel-speed-select/isst-daemon.c | 2 +- tools/power/x86/turbostat/Makefile | 2 +- tools/power/x86/turbostat/turbostat.8 | 202 +- tools/power/x86/turbostat/turbostat.c | 828 +- tools/spi/spidev_test.c | 11 + tools/testing/cxl/Kbuild | 4 +- tools/testing/cxl/test/cxl.c | 131 +- tools/testing/cxl/test/mem.c | 76 +- tools/testing/cxl/test/mock.c | 31 + tools/testing/kunit/kunit.py | 142 +- tools/testing/kunit/kunit_config.py | 73 +- tools/testing/kunit/kunit_json.py | 66 +- tools/testing/kunit/kunit_kernel.py | 183 +- tools/testing/kunit/kunit_parser.py | 191 +- tools/testing/kunit/kunit_tool_test.py | 254 +- tools/testing/kunit/qemu_config.py | 17 +- tools/testing/kunit/qemu_configs/alpha.py | 2 +- tools/testing/kunit/qemu_configs/arm.py | 2 +- tools/testing/kunit/qemu_configs/arm64.py | 2 +- tools/testing/kunit/qemu_configs/i386.py | 4 +- tools/testing/kunit/qemu_configs/powerpc.py | 2 +- tools/testing/kunit/qemu_configs/riscv.py | 7 +- tools/testing/kunit/qemu_configs/s390.py | 4 +- tools/testing/kunit/qemu_configs/sparc.py | 2 +- tools/testing/kunit/qemu_configs/x86_64.py | 2 +- tools/testing/kunit/run_checks.py | 2 +- .../test_is_test_passed-no_tests_no_plan.log | 2 +- tools/testing/memblock/Makefile | 3 +- tools/testing/memblock/README | 17 +- tools/testing/memblock/TODO | 17 +- tools/testing/memblock/internal.h | 11 + tools/testing/memblock/linux/kmemleak.h | 2 +- tools/testing/memblock/linux/memory_hotplug.h | 8 +- tools/testing/memblock/main.c | 2 + .../testing/memblock/scripts/Makefile.include | 10 +- tools/testing/memblock/tests/alloc_api.c | 225 +- .../memblock/tests/alloc_helpers_api.c | 129 +- tools/testing/memblock/tests/alloc_nid_api.c | 351 +- tools/testing/memblock/tests/basic_api.c | 729 +- tools/testing/memblock/tests/common.c | 118 + tools/testing/memblock/tests/common.h | 54 + tools/testing/nvdimm/pmem-dax.c | 4 +- tools/testing/nvdimm/test/iomap.c | 18 +- tools/testing/nvdimm/test/nfit.c | 3 - tools/testing/selftests/Makefile | 35 +- tools/testing/selftests/alsa/Makefile | 3 + tools/testing/selftests/alsa/mixer-test.c | 41 +- tools/testing/selftests/arm64/Makefile | 11 +- tools/testing/selftests/arm64/abi/.gitignore | 1 + tools/testing/selftests/arm64/abi/Makefile | 9 +- .../selftests/arm64/abi/syscall-abi-asm.S | 79 +- .../testing/selftests/arm64/abi/syscall-abi.c | 204 +- tools/testing/selftests/arm64/bti/Makefile | 6 +- tools/testing/selftests/arm64/fp/.gitignore | 5 + tools/testing/selftests/arm64/fp/Makefile | 46 +- tools/testing/selftests/arm64/fp/rdvl.S | 10 + tools/testing/selftests/arm64/fp/rdvl.h | 1 + tools/testing/selftests/arm64/fp/sve-ptrace.c | 175 +- tools/testing/selftests/arm64/fp/sve-test.S | 20 + tools/testing/selftests/arm64/fp/vec-syscfg.c | 10 + tools/testing/selftests/arm64/fp/vlset.c | 10 +- tools/testing/selftests/arm64/mte/.gitignore | 1 + tools/testing/selftests/arm64/mte/Makefile | 1 - .../selftests/arm64/mte/check_child_memory.c | 8 +- .../arm64/mte/check_tags_inclusion.c | 54 +- .../selftests/arm64/mte/mte_common_util.c | 42 +- .../selftests/arm64/mte/mte_common_util.h | 15 +- .../testing/selftests/arm64/signal/.gitignore | 3 + tools/testing/selftests/arm64/signal/Makefile | 1 - .../selftests/arm64/signal/test_signals.h | 8 +- .../arm64/signal/test_signals_utils.c | 6 + .../testcases/fake_sigreturn_sve_change_vl.c | 2 + .../arm64/signal/testcases/testcases.c | 36 + .../arm64/signal/testcases/testcases.h | 3 +- tools/testing/selftests/bpf/.gitignore | 3 +- tools/testing/selftests/bpf/Makefile | 65 +- tools/testing/selftests/bpf/bench.c | 100 +- tools/testing/selftests/bpf/bench.h | 16 + .../selftests/bpf/benchs/run_common.sh | 17 + tools/testing/selftests/bpf/bpf_legacy.h | 9 - tools/testing/selftests/bpf/bpf_tcp_helpers.h | 13 + .../selftests/bpf/bpf_testmod/bpf_testmod.c | 10 +- tools/testing/selftests/bpf/btf_helpers.c | 25 +- tools/testing/selftests/bpf/config | 91 +- .../selftests/bpf/flow_dissector_load.c | 6 +- .../selftests/bpf/get_cgroup_id_user.c | 4 +- tools/testing/selftests/bpf/network_helpers.c | 42 +- tools/testing/selftests/bpf/network_helpers.h | 2 + .../selftests/bpf/prog_tests/attach_probe.c | 144 +- .../selftests/bpf/prog_tests/bpf_cookie.c | 167 +- .../selftests/bpf/prog_tests/bpf_iter.c | 397 +- .../selftests/bpf/prog_tests/bpf_loop.c | 62 + .../selftests/bpf/prog_tests/bpf_mod_race.c | 4 +- .../testing/selftests/bpf/prog_tests/bpf_nf.c | 64 +- .../selftests/bpf/prog_tests/bpf_tcp_ca.c | 67 +- tools/testing/selftests/bpf/prog_tests/btf.c | 257 +- .../selftests/bpf/prog_tests/btf_write.c | 126 +- .../selftests/bpf/prog_tests/core_autosize.c | 2 +- .../selftests/bpf/prog_tests/core_extern.c | 17 +- .../selftests/bpf/prog_tests/core_reloc.c | 153 +- .../selftests/bpf/prog_tests/core_retro.c | 17 +- .../selftests/bpf/prog_tests/fexit_bpf2bpf.c | 109 + .../selftests/bpf/prog_tests/fexit_stress.c | 36 +- .../selftests/bpf/prog_tests/for_each.c | 42 +- .../bpf/prog_tests/helper_restricted.c | 10 +- .../bpf/prog_tests/kprobe_multi_test.c | 164 +- .../selftests/bpf/prog_tests/ksyms_btf.c | 17 +- .../selftests/bpf/prog_tests/linked_funcs.c | 6 + .../bpf/prog_tests/lookup_and_delete.c | 15 +- .../testing/selftests/bpf/prog_tests/netcnt.c | 2 +- .../selftests/bpf/prog_tests/probe_user.c | 35 +- .../bpf/prog_tests/reference_tracking.c | 23 +- .../selftests/bpf/prog_tests/resolve_btfids.c | 2 +- .../selftests/bpf/prog_tests/ringbuf_multi.c | 21 +- .../selftests/bpf/prog_tests/send_signal.c | 2 +- .../selftests/bpf/prog_tests/skeleton.c | 2 + .../selftests/bpf/prog_tests/snprintf.c | 4 +- .../selftests/bpf/prog_tests/sock_fields.c | 1 - .../selftests/bpf/prog_tests/sockmap_ktls.c | 84 +- .../bpf/prog_tests/stacktrace_build_id.c | 8 +- .../bpf/prog_tests/stacktrace_build_id_nmi.c | 11 +- .../selftests/bpf/prog_tests/tailcalls.c | 55 + .../selftests/bpf/prog_tests/tc_redirect.c | 9 +- .../bpf/prog_tests/test_global_funcs.c | 1 + .../selftests/bpf/prog_tests/test_strncmp.c | 25 +- .../selftests/bpf/prog_tests/timer_mim.c | 2 +- .../bpf/prog_tests/trampoline_count.c | 136 +- tools/testing/selftests/bpf/progs/bpf_iter.h | 14 + .../bpf/progs/bpf_iter_bpf_hash_map.c | 9 + .../bpf/progs/bpf_iter_bpf_sk_storage_map.c | 22 +- tools/testing/selftests/bpf/progs/bpf_loop.c | 114 + .../selftests/bpf/progs/bpf_syscall_macro.c | 6 +- .../selftests/bpf/progs/bpf_tracing_net.h | 1 + .../bpf/progs/btf_dump_test_case_syntax.c | 2 +- .../selftests/bpf/progs/core_reloc_types.h | 208 +- .../selftests/bpf/progs/exhandler_kern.c | 13 +- .../selftests/bpf/progs/kprobe_multi.c | 38 +- .../selftests/bpf/progs/linked_funcs1.c | 15 +- .../selftests/bpf/progs/linked_funcs2.c | 15 +- tools/testing/selftests/bpf/progs/loop5.c | 1 - .../selftests/bpf/progs/perf_event_stackmap.c | 4 +- .../selftests/bpf/progs/profiler.inc.h | 5 +- tools/testing/selftests/bpf/progs/profiler1.c | 1 - tools/testing/selftests/bpf/progs/pyperf.h | 6 +- tools/testing/selftests/bpf/progs/pyperf600.c | 11 +- .../selftests/bpf/progs/strncmp_test.c | 8 +- .../selftests/bpf/progs/test_attach_probe.c | 127 +- .../selftests/bpf/progs/test_bpf_cookie.c | 56 +- .../testing/selftests/bpf/progs/test_bpf_nf.c | 85 +- .../selftests/bpf/progs/test_btf_newkv.c | 18 - .../selftests/bpf/progs/test_core_extern.c | 3 + .../bpf/progs/test_core_reloc_existence.c | 11 +- .../bpf/progs/test_core_reloc_kernel.c | 19 + .../bpf/progs/test_core_reloc_size.c | 31 +- .../bpf/progs/test_core_reloc_type_based.c | 49 +- .../bpf/progs/test_helper_restricted.c | 16 +- .../bpf/progs/test_ksyms_btf_write_check.c | 18 +- .../selftests/bpf/progs/test_l4lb_noinline.c | 2 +- .../selftests/bpf/progs/test_module_attach.c | 2 +- .../selftests/bpf/progs/test_pkt_access.c | 2 - .../selftests/bpf/progs/test_probe_user.c | 52 +- .../selftests/bpf/progs/test_ringbuf_multi.c | 2 + .../selftests/bpf/progs/test_sk_assign.c | 4 +- .../selftests/bpf/progs/test_sk_lookup_kern.c | 18 +- .../selftests/bpf/progs/test_skeleton.c | 4 + .../bpf/progs/test_stacktrace_build_id.c | 2 +- .../selftests/bpf/progs/test_subprogs.c | 8 + .../selftests/bpf/progs/test_task_pt_regs.c | 2 +- .../selftests/bpf/progs/test_tc_dtime.c | 53 +- .../bpf/progs/test_trampoline_count.c | 18 +- .../selftests/bpf/progs/test_tunnel_kern.c | 433 +- .../testing/selftests/bpf/progs/test_varlen.c | 8 +- .../selftests/bpf/progs/test_xdp_noinline.c | 42 +- .../selftests/bpf/progs/trigger_bench.c | 2 +- .../selftests/bpf/test_bpftool_synctypes.py | 184 +- tools/testing/selftests/bpf/test_btf.h | 3 + .../selftests/bpf/test_cgroup_storage.c | 7 +- tools/testing/selftests/bpf/test_dev_cgroup.c | 4 +- tools/testing/selftests/bpf/test_lpm_map.c | 43 +- tools/testing/selftests/bpf/test_lru_map.c | 70 +- tools/testing/selftests/bpf/test_offload.py | 2 +- tools/testing/selftests/bpf/test_progs.c | 1011 +- tools/testing/selftests/bpf/test_progs.h | 89 +- .../selftests/bpf/test_skb_cgroup_id_user.c | 4 +- tools/testing/selftests/bpf/test_sock.c | 6 +- tools/testing/selftests/bpf/test_sock_addr.c | 4 +- tools/testing/selftests/bpf/test_sockmap.c | 5 +- tools/testing/selftests/bpf/test_sysctl.c | 6 +- tools/testing/selftests/bpf/test_tag.c | 4 +- .../bpf/test_tcp_check_syncookie_user.c | 4 +- .../selftests/bpf/test_tcpnotify_user.c | 1 - tools/testing/selftests/bpf/test_tunnel.sh | 124 +- tools/testing/selftests/bpf/test_verifier.c | 420 +- .../testing/selftests/bpf/test_verifier_log.c | 5 +- tools/testing/selftests/bpf/test_xdp_veth.sh | 6 +- tools/testing/selftests/bpf/test_xdping.sh | 4 + tools/testing/selftests/bpf/test_xsk.sh | 57 +- tools/testing/selftests/bpf/testing_helpers.c | 91 +- tools/testing/selftests/bpf/testing_helpers.h | 8 + tools/testing/selftests/bpf/trace_helpers.c | 9 +- tools/testing/selftests/bpf/urandom_read.c | 63 +- tools/testing/selftests/bpf/verifier/calls.c | 73 + tools/testing/selftests/bpf/verifier/jmp32.c | 21 + tools/testing/selftests/bpf/verifier/jump.c | 22 + .../testing/selftests/bpf/verifier/precise.c | 25 + .../selftests/bpf/verifier/ref_tracking.c | 2 +- tools/testing/selftests/bpf/verifier/sock.c | 6 +- tools/testing/selftests/bpf/vmtest.sh | 53 +- .../selftests/bpf/xdp_redirect_multi.c | 1 - tools/testing/selftests/bpf/xdping.c | 8 +- tools/testing/selftests/bpf/xsk_prereqs.sh | 51 +- tools/testing/selftests/cgroup/.gitignore | 1 + tools/testing/selftests/cgroup/Makefile | 2 + tools/testing/selftests/cgroup/cgroup_util.c | 64 +- tools/testing/selftests/cgroup/cgroup_util.h | 5 + .../selftests/cgroup/test_memcontrol.c | 414 +- tools/testing/selftests/cgroup/test_stress.sh | 2 +- .../selftests/damon/_chk_dependency.sh | 10 + tools/testing/selftests/damon/sysfs.sh | 1 + tools/testing/selftests/dma/Makefile | 1 + .../testing/selftests/dma/dma_map_benchmark.c | 2 +- tools/testing/selftests/drivers/.gitignore | 1 + .../selftests/drivers/dma-buf/udmabuf.c | 3 +- tools/testing/selftests/drivers/gpu/drm_mm.sh | 4 +- .../drivers/net/mlxsw/qos_headroom.sh | 4 +- .../selftests/drivers/net/mlxsw/qos_pfc.sh | 4 +- .../drivers/net/mlxsw/sch_red_ets.sh | 5 +- .../drivers/net/mlxsw/sch_red_root.sh | 5 +- .../net/mlxsw/spectrum-2/resource_scale.sh | 31 +- .../net/mlxsw/spectrum-2/tc_flower_scale.sh | 15 +- .../net/mlxsw/spectrum/resource_scale.sh | 29 +- .../drivers/net/mlxsw/tc_flower_scale.sh | 17 + .../selftests/drivers/net/netdevsim/fib.sh | 45 + .../drivers/net/netdevsim/hw_stats_l3.sh | 4 +- .../drivers/net/ocelot/tc_flower_chains.sh | 202 +- .../filesystems/binderfs/binderfs_test.c | 4 +- .../selftests/filesystems/binderfs/config | 1 - tools/testing/selftests/firmware/Makefile | 2 +- tools/testing/selftests/firmware/config | 1 + .../selftests/firmware/fw_filesystem.sh | 162 +- tools/testing/selftests/firmware/fw_lib.sh | 19 +- .../selftests/firmware/fw_run_tests.sh | 4 + .../test.d/dynevent/add_remove_eprobe.tc | 9 +- .../test.d/dynevent/add_remove_kprobe.tc | 7 + .../test.d/kprobe/kprobe_args_string.tc | 3 + .../test.d/kprobe/kprobe_args_syntax.tc | 4 + .../test.d/kprobe/kprobe_syntax_errors.tc | 2 +- .../ftrace/test.d/kprobe/multiple_kprobes.tc | 2 + .../selftests/futex/functional/Makefile | 1 - tools/testing/selftests/gpio/Makefile | 2 +- tools/testing/selftests/ir/ir_loopback.sh | 2 +- tools/testing/selftests/kcmp/kcmp_test.c | 6 +- .../selftests/kexec/kexec_common_lib.sh | 36 +- tools/testing/selftests/kselftest.h | 15 + tools/testing/selftests/kselftest_deps.sh | 2 +- tools/testing/selftests/kselftest_module.h | 4 + tools/testing/selftests/kvm/.gitignore | 16 +- tools/testing/selftests/kvm/Makefile | 81 +- .../selftests/kvm/aarch64/arch_timer.c | 88 +- .../selftests/kvm/aarch64/debug-exceptions.c | 26 +- .../selftests/kvm/aarch64/get-reg-list.c | 38 +- .../selftests/kvm/aarch64/vcpu_width_config.c | 71 +- .../testing/selftests/kvm/aarch64/vgic_init.c | 446 +- .../testing/selftests/kvm/aarch64/vgic_irq.c | 44 +- .../selftests/kvm/access_tracking_perf_test.c | 92 +- .../selftests/kvm/demand_paging_test.c | 49 +- .../selftests/kvm/dirty_log_perf_test.c | 89 +- tools/testing/selftests/kvm/dirty_log_test.c | 95 +- .../selftests/kvm/hardware_disable_test.c | 29 +- .../selftests/kvm/include/aarch64/processor.h | 50 +- .../selftests/kvm/include/aarch64/vgic.h | 6 +- .../selftests/kvm/include/kvm_util_base.h | 819 +- .../selftests/kvm/include/perf_test_util.h | 14 +- .../selftests/kvm/include/riscv/processor.h | 28 +- .../testing/selftests/kvm/include/test_util.h | 7 + .../selftests/kvm/include/ucall_common.h | 65 +- .../selftests/kvm/include/x86_64/apic.h | 1 + .../selftests/kvm/include/x86_64/evmcs.h | 2 +- .../selftests/kvm/include/x86_64/processor.h | 503 +- .../selftests/kvm/include/x86_64/svm.h | 2 - .../selftests/kvm/include/x86_64/svm_util.h | 27 +- .../selftests/kvm/include/x86_64/vmx.h | 8 +- .../selftests/kvm/kvm_binary_stats_test.c | 181 +- .../selftests/kvm/kvm_create_max_vcpus.c | 10 +- .../selftests/kvm/kvm_page_table_test.c | 66 +- .../selftests/kvm/lib/aarch64/processor.c | 106 +- .../testing/selftests/kvm/lib/aarch64/ucall.c | 22 +- .../testing/selftests/kvm/lib/aarch64/vgic.c | 54 +- tools/testing/selftests/kvm/lib/elf.c | 1 - tools/testing/selftests/kvm/lib/guest_modes.c | 6 +- tools/testing/selftests/kvm/lib/kvm_util.c | 1211 +- .../selftests/kvm/lib/perf_test_util.c | 125 +- .../selftests/kvm/lib/riscv/processor.c | 120 +- tools/testing/selftests/kvm/lib/riscv/ucall.c | 37 +- .../kvm/lib/s390x/diag318_test_handler.c | 11 +- .../selftests/kvm/lib/s390x/processor.c | 44 +- tools/testing/selftests/kvm/lib/s390x/ucall.c | 10 +- .../selftests/kvm/lib/x86_64/processor.c | 852 +- tools/testing/selftests/kvm/lib/x86_64/svm.c | 17 - .../testing/selftests/kvm/lib/x86_64/ucall.c | 12 +- tools/testing/selftests/kvm/lib/x86_64/vmx.c | 169 +- .../selftests/kvm/max_guest_memory_test.c | 55 +- .../kvm/memslot_modification_stress_test.c | 13 +- .../testing/selftests/kvm/memslot_perf_test.c | 32 +- tools/testing/selftests/kvm/rseq_test.c | 88 +- tools/testing/selftests/kvm/s390x/memop.c | 226 +- tools/testing/selftests/kvm/s390x/resets.c | 178 +- .../selftests/kvm/s390x/sync_regs_test.c | 121 +- tools/testing/selftests/kvm/s390x/tprot.c | 68 +- .../selftests/kvm/set_memory_region_test.c | 46 +- tools/testing/selftests/kvm/steal_time.c | 158 +- .../kvm/system_counter_offset_test.c | 38 +- tools/testing/selftests/kvm/x86_64/amx_test.c | 91 +- .../testing/selftests/kvm/x86_64/cpuid_test.c | 109 +- .../kvm/x86_64/cr4_cpuid_sync_test.c | 43 +- .../testing/selftests/kvm/x86_64/debug_regs.c | 77 +- .../kvm/x86_64/emulator_error_test.c | 85 +- .../testing/selftests/kvm/x86_64/evmcs_test.c | 65 +- .../kvm/x86_64/get_msr_index_features.c | 123 +- .../selftests/kvm/x86_64/hyperv_clock.c | 38 +- .../selftests/kvm/x86_64/hyperv_cpuid.c | 48 +- .../selftests/kvm/x86_64/hyperv_features.c | 404 +- .../selftests/kvm/x86_64/hyperv_svm_test.c | 28 +- .../selftests/kvm/x86_64/kvm_clock_test.c | 32 +- .../selftests/kvm/x86_64/kvm_pv_test.c | 119 +- .../selftests/kvm/x86_64/mmio_warning_test.c | 16 +- .../selftests/kvm/x86_64/platform_info_test.c | 51 +- .../kvm/x86_64/pmu_event_filter_test.c | 148 +- .../selftests/kvm/x86_64/set_boot_cpu_id.c | 95 +- .../selftests/kvm/x86_64/set_sregs_test.c | 75 +- .../selftests/kvm/x86_64/sev_migrate_tests.c | 131 +- tools/testing/selftests/kvm/x86_64/smm_test.c | 46 +- .../testing/selftests/kvm/x86_64/state_test.c | 39 +- .../selftests/kvm/x86_64/svm_int_ctl_test.c | 25 +- .../selftests/kvm/x86_64/svm_vmcall_test.c | 20 +- .../selftests/kvm/x86_64/sync_regs_test.c | 62 +- .../selftests/kvm/x86_64/tsc_msrs_test.c | 39 +- .../selftests/kvm/x86_64/userspace_io_test.c | 22 +- .../kvm/x86_64/userspace_msr_exit_test.c | 188 +- .../kvm/x86_64/vmx_apic_access_test.c | 32 +- .../kvm/x86_64/vmx_close_while_nested_test.c | 21 +- .../selftests/kvm/x86_64/vmx_dirty_log_test.c | 18 +- .../vmx_exception_with_invalid_guest_state.c | 68 +- .../x86_64/vmx_invalid_nested_guest_state.c | 22 +- .../kvm/x86_64/vmx_nested_tsc_scaling_test.c | 33 +- .../kvm/x86_64/vmx_preemption_timer_test.c | 38 +- .../kvm/x86_64/vmx_set_nested_state_test.c | 105 +- .../kvm/x86_64/vmx_tsc_adjust_test.c | 17 +- .../selftests/kvm/x86_64/xapic_ipi_test.c | 48 +- .../selftests/kvm/x86_64/xapic_state_test.c | 82 +- .../selftests/kvm/x86_64/xen_shinfo_test.c | 409 +- .../selftests/kvm/x86_64/xen_vmcall_test.c | 27 +- .../selftests/kvm/x86_64/xss_msr_test.c | 56 +- tools/testing/selftests/landlock/Makefile | 5 +- tools/testing/selftests/landlock/base_test.c | 189 +- tools/testing/selftests/landlock/common.h | 66 +- tools/testing/selftests/landlock/fs_test.c | 1654 +- .../testing/selftests/landlock/ptrace_test.c | 40 +- tools/testing/selftests/lib.mk | 64 +- tools/testing/selftests/lkdtm/config | 4 + tools/testing/selftests/lkdtm/tests.txt | 9 +- .../testing/selftests/mqueue/mq_perf_tests.c | 4 +- tools/testing/selftests/net/.gitignore | 50 +- tools/testing/selftests/net/Makefile | 15 +- tools/testing/selftests/net/af_unix/Makefile | 3 +- tools/testing/selftests/net/cmsg_sender.c | 2 +- tools/testing/selftests/net/fcnal-test.sh | 73 +- tools/testing/selftests/net/fib_nexthops.sh | 53 +- tools/testing/selftests/net/fib_rule_tests.sh | 35 +- .../testing/selftests/net/forwarding/Makefile | 7 +- .../net/forwarding/custom_multipath_hash.sh | 24 +- .../net/forwarding/ethtool_extended_state.sh | 43 +- .../forwarding/gre_custom_multipath_hash.sh | 24 +- .../selftests/net/forwarding/hw_stats_l3.sh | 16 +- .../ip6gre_custom_multipath_hash.sh | 24 +- tools/testing/selftests/net/forwarding/lib.sh | 150 +- .../forwarding/mirror_gre_bridge_1q_lag.sh | 7 +- .../selftests/net/forwarding/router.sh | 18 + .../selftests/net/forwarding/router_vid_1.sh | 27 +- .../selftests/net/forwarding/tc_actions.sh | 2 +- .../net/forwarding/vxlan_asymmetric.sh | 2 +- tools/testing/selftests/net/ioam6.sh | 12 +- tools/testing/selftests/net/ipv6_flowlabel.c | 75 +- tools/testing/selftests/net/ipv6_flowlabel.sh | 16 + tools/testing/selftests/net/mptcp/Makefile | 3 +- tools/testing/selftests/net/mptcp/config | 8 + tools/testing/selftests/net/mptcp/diag.sh | 86 +- .../selftests/net/mptcp/mptcp_connect.c | 28 +- tools/testing/selftests/net/mptcp/mptcp_inq.c | 2 +- .../testing/selftests/net/mptcp/mptcp_join.sh | 407 +- .../selftests/net/mptcp/mptcp_sockopt.c | 2 +- tools/testing/selftests/net/mptcp/pm_nl_ctl.c | 720 +- .../selftests/net/mptcp/simult_flows.sh | 14 +- tools/testing/selftests/net/psock_snd.c | 2 + tools/testing/selftests/net/tls.c | 124 +- tools/testing/selftests/net/udpgro.sh | 2 +- tools/testing/selftests/net/udpgro_bench.sh | 2 +- tools/testing/selftests/net/udpgro_fwd.sh | 2 +- tools/testing/selftests/net/udpgso_bench.sh | 2 +- tools/testing/selftests/net/veth.sh | 6 +- .../selftests/net/vrf_strict_mode_test.sh | 48 +- .../selftests/netfilter/nft_concat_range.sh | 2 +- tools/testing/selftests/netfilter/nft_fib.sh | 50 + .../selftests/netfilter/nft_flowtable.sh | 363 +- tools/testing/selftests/netfilter/nft_nat.sh | 43 + .../selftests/netfilter/nft_trans_stress.sh | 81 +- .../selftests/powerpc/copyloops/copy_mc_64.S | 242 - .../selftests/powerpc/copyloops/copyuser_64.S | 564 - .../powerpc/copyloops/copyuser_power7.S | 695 - .../selftests/powerpc/copyloops/memcpy_64.S | 230 - .../powerpc/copyloops/memcpy_power7.S | 641 - .../selftests/powerpc/include/basic_asm.h | 63 +- tools/testing/selftests/powerpc/include/reg.h | 73 +- .../testing/selftests/powerpc/include/utils.h | 15 + tools/testing/selftests/powerpc/lib/reg.S | 107 +- .../testing/selftests/powerpc/math/.gitignore | 1 + tools/testing/selftests/powerpc/math/Makefile | 4 +- tools/testing/selftests/powerpc/mce/vas-api.h | 28 - tools/testing/selftests/powerpc/mm/.gitignore | 1 + tools/testing/selftests/powerpc/mm/Makefile | 4 +- .../powerpc/nx-gzip/include/vas-api.h | 28 - .../powerpc/papr_attributes/attr_test.c | 30 +- tools/testing/selftests/powerpc/pmu/Makefile | 11 +- .../selftests/powerpc/pmu/ebb/.gitignore | 1 + .../powerpc/pmu/ebb/cycles_with_mmcr2_test.c | 1 + .../powerpc/pmu/sampling_tests/.gitignore | 18 +- .../powerpc/pmu/sampling_tests/Makefile | 7 +- .../powerpc/pmu/sampling_tests/misc.c | 137 +- .../powerpc/pmu/sampling_tests/misc.h | 9 +- .../powerpc/primitives/asm/asm-compat.h | 64 - .../powerpc/primitives/asm/asm-const.h | 15 - .../powerpc/primitives/asm/extable.h | 44 - .../powerpc/primitives/asm/feature-fixups.h | 298 - .../powerpc/primitives/asm/ppc_asm.h | 781 - .../powerpc/primitives/word-at-a-time.h | 206 - .../testing/selftests/powerpc/ptrace/Makefile | 40 +- .../selftests/powerpc/ptrace/ptrace-gpr.c | 125 +- .../selftests/powerpc/ptrace/ptrace-gpr.h | 14 +- .../selftests/powerpc/ptrace/ptrace-tm-gpr.c | 18 +- .../powerpc/ptrace/ptrace-tm-spd-gpr.c | 20 +- .../testing/selftests/powerpc/ptrace/ptrace.h | 79 +- .../selftests/powerpc/security/.gitignore | 1 + .../selftests/powerpc/security/spectre_v2.c | 32 +- .../selftests/powerpc/stringloops/memcmp_32.S | 45 - .../selftests/powerpc/stringloops/memcmp_64.S | 638 - .../selftests/powerpc/stringloops/strlen_32.S | 78 - .../selftests/powerpc/vphn/asm/lppaca.h | 152 - tools/testing/selftests/powerpc/vphn/vphn.c | 90 - tools/testing/selftests/proc/proc-pid-vm.c | 75 +- .../selftests/rcutorture/bin/functions.sh | 2 +- .../rcutorture/bin/kvm-check-branches.sh | 11 +- .../rcutorture/bin/kvm-find-errors.sh | 2 +- .../selftests/rcutorture/bin/kvm-recheck.sh | 7 +- .../selftests/rcutorture/bin/kvm-remote.sh | 15 +- tools/testing/selftests/rcutorture/bin/kvm.sh | 16 +- .../selftests/rcutorture/bin/torture.sh | 29 +- .../selftests/rcutorture/configs/rcu/RUDE01 | 2 + .../selftests/rcutorture/configs/rcu/SRCU-N | 2 + .../selftests/rcutorture/configs/rcu/TASKS01 | 1 + .../selftests/rcutorture/configs/rcu/TASKS02 | 4 + .../rcutorture/configs/rcu/TASKS02.boot | 1 + .../selftests/rcutorture/configs/rcu/TASKS03 | 2 + .../selftests/rcutorture/configs/rcu/TRACE01 | 3 + .../selftests/rcutorture/configs/rcu/TRACE02 | 2 + .../selftests/rcutorture/configs/rcu/TREE04 | 5 +- .../selftests/rcutorture/configs/rcu/TREE07 | 1 + .../selftests/rcutorture/configs/rcu/TREE09 | 2 + .../selftests/rcutorture/configs/rcu/TREE10 | 1 + .../rcutorture/configs/rcu/ver_functions.sh | 16 +- .../rcutorture/configs/rcuscale/CFcommon | 7 +- .../rcutorture/configs/rcuscale/TREE | 2 + .../rcutorture/configs/refscale/CFcommon | 4 + .../rcutorture/configs/refscale/NOPREEMPT | 2 + .../rcutorture/configs/scf/NOPREEMPT | 2 + .../selftests/rcutorture/configs/scf/PREEMPT | 1 + .../rcutorture/configs/scf/ver_functions.sh | 3 +- tools/testing/selftests/resctrl/Makefile | 19 +- tools/testing/selftests/resctrl/README | 39 +- tools/testing/selftests/resctrl/cat_test.c | 2 +- tools/testing/selftests/resctrl/fill_buf.c | 4 +- tools/testing/selftests/resctrl/resctrl.h | 5 +- .../testing/selftests/resctrl/resctrl_tests.c | 49 +- tools/testing/selftests/resctrl/resctrl_val.c | 1 + tools/testing/selftests/resctrl/resctrlfs.c | 2 +- tools/testing/selftests/rseq/rseq-riscv.h | 50 +- tools/testing/selftests/rseq/rseq.c | 3 +- tools/testing/selftests/safesetid/Makefile | 2 +- .../selftests/safesetid/safesetid-test.c | 295 +- tools/testing/selftests/seccomp/Makefile | 1 + tools/testing/selftests/seccomp/seccomp_bpf.c | 429 +- tools/testing/selftests/sgx/defines.h | 23 + tools/testing/selftests/sgx/load.c | 41 + tools/testing/selftests/sgx/main.c | 1435 + tools/testing/selftests/sgx/main.h | 1 + tools/testing/selftests/sgx/sigstruct.c | 6 + tools/testing/selftests/sgx/test_encl.c | 68 + .../selftests/sgx/test_encl_bootstrap.S | 6 + tools/testing/selftests/sync/config | 1 - tools/testing/selftests/sysctl/sysctl.sh | 23 + tools/testing/selftests/tc-testing/.gitignore | 1 - tools/testing/selftests/tc-testing/Makefile | 1 - .../selftests/tc-testing/plugins/__init__.py | 0 .../tc-testing/tc-tests/actions/gact.json | 77 + tools/testing/selftests/timens/Makefile | 2 +- tools/testing/selftests/timers/adjtick.c | 2 +- .../selftests/timers/alarmtimer-suspend.c | 2 +- tools/testing/selftests/timers/change_skew.c | 2 +- .../selftests/timers/clocksource-switch.c | 71 +- .../selftests/timers/inconsistency-check.c | 32 +- tools/testing/selftests/timers/nanosleep.c | 18 +- tools/testing/selftests/timers/raw_skew.c | 2 +- .../selftests/timers/skew_consistency.c | 2 +- .../testing/selftests/timers/valid-adjtimex.c | 2 +- tools/testing/selftests/vm/.gitignore | 3 + tools/testing/selftests/vm/Makefile | 26 +- tools/testing/selftests/vm/config | 2 + tools/testing/selftests/vm/gup_test.c | 26 +- tools/testing/selftests/vm/hmm-tests.c | 325 +- tools/testing/selftests/vm/hugepage-mremap.c | 8 +- tools/testing/selftests/vm/hugetlb-madvise.c | 5 +- tools/testing/selftests/vm/ksm_tests.c | 11 +- tools/testing/selftests/vm/madv_populate.c | 34 +- tools/testing/selftests/vm/pkey-x86.h | 21 +- tools/testing/selftests/vm/protection_keys.c | 2 +- tools/testing/selftests/vm/run_vmtests.sh | 526 +- .../selftests/vm/split_huge_page_test.c | 79 +- tools/testing/selftests/vm/test_hmm.sh | 24 +- tools/testing/selftests/vm/userfaultfd.c | 19 +- tools/testing/selftests/vm/va_128TBswitch.c | 8 +- .../testing/selftests/wireguard/qemu/Makefile | 61 +- .../selftests/wireguard/qemu/arch/arm.config | 1 + .../wireguard/qemu/arch/armeb.config | 1 + .../selftests/wireguard/qemu/arch/i686.config | 8 +- .../selftests/wireguard/qemu/arch/m68k.config | 10 +- .../selftests/wireguard/qemu/arch/mips.config | 1 + .../wireguard/qemu/arch/mipsel.config | 1 + .../wireguard/qemu/arch/powerpc.config | 1 + .../wireguard/qemu/arch/riscv32.config | 1 + .../wireguard/qemu/arch/x86_64.config | 7 +- .../selftests/wireguard/qemu/debug.config | 5 - tools/testing/selftests/wireguard/qemu/init.c | 14 + .../selftests/wireguard/qemu/kernel.config | 5 +- tools/testing/selftests/x86/amx.c | 24 +- .../selftests/x86/corrupt_xstate_header.c | 16 +- tools/thermal/tmon/pid.c | 2 +- tools/thermal/tmon/sysfs.c | 24 +- tools/thermal/tmon/tmon.h | 3 + tools/tracing/rtla/Makefile | 64 +- tools/tracing/rtla/README.txt | 13 +- tools/tracing/rtla/src/osnoise_hist.c | 5 +- tools/tracing/rtla/src/osnoise_top.c | 9 +- tools/tracing/rtla/src/timerlat_hist.c | 13 +- tools/tracing/rtla/src/timerlat_top.c | 13 +- tools/tracing/rtla/src/trace.c | 9 +- tools/tracing/rtla/src/utils.c | 117 +- tools/tracing/rtla/src/utils.h | 3 +- tools/usb/testusb.c | 20 +- tools/virtio/linux/kernel.h | 2 +- tools/virtio/linux/vringh.h | 1 + tools/virtio/virtio_test.c | 4 +- tools/vm/page-types.c | 8 +- tools/vm/page_owner_sort.c | 414 +- tools/vm/slabinfo.c | 58 +- usr/gen_init_cpio.c | 92 +- usr/include/Makefile | 14 +- 1355 files changed, 181039 insertions(+), 43440 deletions(-) create mode 100644 tools/testing/selftests/tc-testing/plugins/__init__.py diff --git a/sound/soc/qcom/qdsp6/q6apm.c b/sound/soc/qcom/qdsp6/q6apm.c index f424d7aa38..794019286c 100644 --- a/sound/soc/qcom/qdsp6/q6apm.c +++ b/sound/soc/qcom/qdsp6/q6apm.c @@ -75,6 +75,7 @@ static struct audioreach_graph *q6apm_get_audioreach_graph(struct q6apm *apm, ui id = idr_alloc(&apm->graph_idr, graph, graph_id, graph_id + 1, GFP_KERNEL); if (id < 0) { dev_err(apm->dev, "Unable to allocate graph id (%d)\n", graph_id); + kfree(graph->graph); kfree(graph); mutex_unlock(&apm->lock); return ERR_PTR(id); diff --git a/sound/soc/qcom/qdsp6/q6asm-dai.c b/sound/soc/qcom/qdsp6/q6asm-dai.c index b74b67720e..5fc8088e63 100644 --- a/sound/soc/qcom/qdsp6/q6asm-dai.c +++ b/sound/soc/qcom/qdsp6/q6asm-dai.c @@ -1205,17 +1205,18 @@ static const struct snd_soc_dapm_widget q6asm_dapm_widgets[] = { }; static const struct snd_soc_component_driver q6asm_fe_dai_component = { - .name = DRV_NAME, - .open = q6asm_dai_open, - .hw_params = q6asm_dai_hw_params, - .close = q6asm_dai_close, - .prepare = q6asm_dai_prepare, - .trigger = q6asm_dai_trigger, - .pointer = q6asm_dai_pointer, - .pcm_construct = q6asm_dai_pcm_new, - .compress_ops = &q6asm_dai_compress_ops, - .dapm_widgets = q6asm_dapm_widgets, - .num_dapm_widgets = ARRAY_SIZE(q6asm_dapm_widgets), + .name = DRV_NAME, + .open = q6asm_dai_open, + .hw_params = q6asm_dai_hw_params, + .close = q6asm_dai_close, + .prepare = q6asm_dai_prepare, + .trigger = q6asm_dai_trigger, + .pointer = q6asm_dai_pointer, + .pcm_construct = q6asm_dai_pcm_new, + .compress_ops = &q6asm_dai_compress_ops, + .dapm_widgets = q6asm_dapm_widgets, + .num_dapm_widgets = ARRAY_SIZE(q6asm_dapm_widgets), + .legacy_dai_naming = 1, }; static struct snd_soc_dai_driver q6asm_fe_dais_template[] = { diff --git a/sound/soc/qcom/qdsp6/q6asm.c b/sound/soc/qcom/qdsp6/q6asm.c index 9251d85489..195780f75d 100644 --- a/sound/soc/qcom/qdsp6/q6asm.c +++ b/sound/soc/qcom/qdsp6/q6asm.c @@ -513,7 +513,7 @@ int q6asm_map_memory_regions(unsigned int dir, struct audio_client *ac, return 0; } - buf = kzalloc(((sizeof(struct audio_buffer)) * periods), GFP_ATOMIC); + buf = kcalloc(periods, sizeof(*buf), GFP_ATOMIC); if (!buf) { spin_unlock_irqrestore(&ac->lock, flags); return -ENOMEM; diff --git a/sound/soc/rockchip/rk3288_hdmi_analog.c b/sound/soc/rockchip/rk3288_hdmi_analog.c index b052642ea6..0c6bd9a019 100644 --- a/sound/soc/rockchip/rk3288_hdmi_analog.c +++ b/sound/soc/rockchip/rk3288_hdmi_analog.c @@ -124,10 +124,10 @@ static int rk_init(struct snd_soc_pcm_runtime *runtime) /* Enable Headset Jack detection */ if (gpio_is_valid(machine->gpio_hp_det)) { - snd_soc_card_jack_new(runtime->card, "Headphone Jack", - SND_JACK_HEADPHONE, &headphone_jack, - headphone_jack_pins, - ARRAY_SIZE(headphone_jack_pins)); + snd_soc_card_jack_new_pins(runtime->card, "Headphone Jack", + SND_JACK_HEADPHONE, &headphone_jack, + headphone_jack_pins, + ARRAY_SIZE(headphone_jack_pins)); rk_hp_jack_gpio.gpio = machine->gpio_hp_det; snd_soc_jack_add_gpios(&headphone_jack, 1, &rk_hp_jack_gpio); } @@ -169,7 +169,7 @@ static struct snd_soc_card snd_soc_card_rk = { static int snd_rk_mc_probe(struct platform_device *pdev) { - int ret = 0; + int ret; struct snd_soc_card *card = &snd_soc_card_rk; struct device_node *np = pdev->dev.of_node; struct rk_drvdata *machine; @@ -253,7 +253,7 @@ static int snd_rk_mc_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "Soc register card failed\n"); - return ret; + return 0; } static const struct of_device_id rockchip_sound_of_match[] = { diff --git a/sound/soc/rockchip/rk3399_gru_sound.c b/sound/soc/rockchip/rk3399_gru_sound.c index eeef3ed700..2540b9ba37 100644 --- a/sound/soc/rockchip/rk3399_gru_sound.c +++ b/sound/soc/rockchip/rk3399_gru_sound.c @@ -174,7 +174,7 @@ static int rockchip_sound_cdndp_init(struct snd_soc_pcm_runtime *rtd) /* Enable jack detection. */ ret = snd_soc_card_jack_new(card, "DP Jack", SND_JACK_LINEOUT, - &cdn_dp_card_jack, NULL, 0); + &cdn_dp_card_jack); if (ret) { dev_err(card->dev, "Can't create DP Jack %d\n", ret); return ret; @@ -204,13 +204,13 @@ static int rockchip_sound_da7219_init(struct snd_soc_pcm_runtime *rtd) } /* Enable Headset and 4 Buttons Jack detection */ - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", - SND_JACK_HEADSET | SND_JACK_LINEOUT | - SND_JACK_BTN_0 | SND_JACK_BTN_1 | - SND_JACK_BTN_2 | SND_JACK_BTN_3, - &rockchip_sound_jack, - rockchip_sound_jack_pins, - ARRAY_SIZE(rockchip_sound_jack_pins)); + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", + SND_JACK_HEADSET | SND_JACK_LINEOUT | + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3, + &rockchip_sound_jack, + rockchip_sound_jack_pins, + ARRAY_SIZE(rockchip_sound_jack_pins)); if (ret) { dev_err(rtd->card->dev, "New Headset Jack failed! (%d)\n", ret); diff --git a/sound/soc/rockchip/rockchip_i2s.c b/sound/soc/rockchip/rockchip_i2s.c index 4ce5d25793..f5f3540a9e 100644 --- a/sound/soc/rockchip/rockchip_i2s.c +++ b/sound/soc/rockchip/rockchip_i2s.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -54,8 +55,38 @@ struct rk_i2s_dev { const struct rk_i2s_pins *pins; unsigned int bclk_ratio; spinlock_t lock; /* tx/rx lock */ + struct pinctrl *pinctrl; + struct pinctrl_state *bclk_on; + struct pinctrl_state *bclk_off; }; +static int i2s_pinctrl_select_bclk_on(struct rk_i2s_dev *i2s) +{ + int ret = 0; + + if (!IS_ERR(i2s->pinctrl) && !IS_ERR_OR_NULL(i2s->bclk_on)) + ret = pinctrl_select_state(i2s->pinctrl, i2s->bclk_on); + + if (ret) + dev_err(i2s->dev, "bclk enable failed %d\n", ret); + + return ret; +} + +static int i2s_pinctrl_select_bclk_off(struct rk_i2s_dev *i2s) +{ + + int ret = 0; + + if (!IS_ERR(i2s->pinctrl) && !IS_ERR_OR_NULL(i2s->bclk_off)) + ret = pinctrl_select_state(i2s->pinctrl, i2s->bclk_off); + + if (ret) + dev_err(i2s->dev, "bclk disable failed %d\n", ret); + + return ret; +} + static int i2s_runtime_suspend(struct device *dev) { struct rk_i2s_dev *i2s = dev_get_drvdata(dev); @@ -92,39 +123,46 @@ static inline struct rk_i2s_dev *to_info(struct snd_soc_dai *dai) return snd_soc_dai_get_drvdata(dai); } -static void rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on) +static int rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on) { unsigned int val = 0; int retry = 10; + int ret = 0; spin_lock(&i2s->lock); if (on) { - regmap_update_bits(i2s->regmap, I2S_DMACR, - I2S_DMACR_TDE_ENABLE, I2S_DMACR_TDE_ENABLE); - - regmap_update_bits(i2s->regmap, I2S_XFER, - I2S_XFER_TXS_START | I2S_XFER_RXS_START, - I2S_XFER_TXS_START | I2S_XFER_RXS_START); - + ret = regmap_update_bits(i2s->regmap, I2S_DMACR, + I2S_DMACR_TDE_ENABLE, + I2S_DMACR_TDE_ENABLE); + if (ret < 0) + goto end; + ret = regmap_update_bits(i2s->regmap, I2S_XFER, + I2S_XFER_TXS_START | I2S_XFER_RXS_START, + I2S_XFER_TXS_START | I2S_XFER_RXS_START); + if (ret < 0) + goto end; i2s->tx_start = true; } else { i2s->tx_start = false; - regmap_update_bits(i2s->regmap, I2S_DMACR, - I2S_DMACR_TDE_ENABLE, I2S_DMACR_TDE_DISABLE); + ret = regmap_update_bits(i2s->regmap, I2S_DMACR, + I2S_DMACR_TDE_ENABLE, + I2S_DMACR_TDE_DISABLE); + if (ret < 0) + goto end; if (!i2s->rx_start) { - regmap_update_bits(i2s->regmap, I2S_XFER, - I2S_XFER_TXS_START | - I2S_XFER_RXS_START, - I2S_XFER_TXS_STOP | - I2S_XFER_RXS_STOP); - + ret = regmap_update_bits(i2s->regmap, I2S_XFER, + I2S_XFER_TXS_START | I2S_XFER_RXS_START, + I2S_XFER_TXS_STOP | I2S_XFER_RXS_STOP); + if (ret < 0) + goto end; udelay(150); - regmap_update_bits(i2s->regmap, I2S_CLR, - I2S_CLR_TXC | I2S_CLR_RXC, - I2S_CLR_TXC | I2S_CLR_RXC); - + ret = regmap_update_bits(i2s->regmap, I2S_CLR, + I2S_CLR_TXC | I2S_CLR_RXC, + I2S_CLR_TXC | I2S_CLR_RXC); + if (ret < 0) + goto end; regmap_read(i2s->regmap, I2S_CLR, &val); /* Should wait for clear operation to finish */ @@ -133,61 +171,80 @@ static void rockchip_snd_txctrl(struct rk_i2s_dev *i2s, int on) retry--; if (!retry) { dev_warn(i2s->dev, "fail to clear\n"); + ret = -EBUSY; break; } } } } +end: spin_unlock(&i2s->lock); + if (ret < 0) + dev_err(i2s->dev, "lrclk update failed\n"); + + return ret; } -static void rockchip_snd_rxctrl(struct rk_i2s_dev *i2s, int on) +static int rockchip_snd_rxctrl(struct rk_i2s_dev *i2s, int on) { unsigned int val = 0; int retry = 10; + int ret = 0; spin_lock(&i2s->lock); if (on) { - regmap_update_bits(i2s->regmap, I2S_DMACR, - I2S_DMACR_RDE_ENABLE, I2S_DMACR_RDE_ENABLE); - - regmap_update_bits(i2s->regmap, I2S_XFER, - I2S_XFER_TXS_START | I2S_XFER_RXS_START, - I2S_XFER_TXS_START | I2S_XFER_RXS_START); + ret = regmap_update_bits(i2s->regmap, I2S_DMACR, + I2S_DMACR_RDE_ENABLE, + I2S_DMACR_RDE_ENABLE); + if (ret < 0) + goto end; + ret = regmap_update_bits(i2s->regmap, I2S_XFER, + I2S_XFER_TXS_START | I2S_XFER_RXS_START, + I2S_XFER_TXS_START | I2S_XFER_RXS_START); + if (ret < 0) + goto end; i2s->rx_start = true; } else { i2s->rx_start = false; - regmap_update_bits(i2s->regmap, I2S_DMACR, - I2S_DMACR_RDE_ENABLE, I2S_DMACR_RDE_DISABLE); + ret = regmap_update_bits(i2s->regmap, I2S_DMACR, + I2S_DMACR_RDE_ENABLE, + I2S_DMACR_RDE_DISABLE); + if (ret < 0) + goto end; if (!i2s->tx_start) { - regmap_update_bits(i2s->regmap, I2S_XFER, - I2S_XFER_TXS_START | - I2S_XFER_RXS_START, - I2S_XFER_TXS_STOP | - I2S_XFER_RXS_STOP); - + ret = regmap_update_bits(i2s->regmap, I2S_XFER, + I2S_XFER_TXS_START | I2S_XFER_RXS_START, + I2S_XFER_TXS_STOP | I2S_XFER_RXS_STOP); + if (ret < 0) + goto end; udelay(150); - regmap_update_bits(i2s->regmap, I2S_CLR, - I2S_CLR_TXC | I2S_CLR_RXC, - I2S_CLR_TXC | I2S_CLR_RXC); - + ret = regmap_update_bits(i2s->regmap, I2S_CLR, + I2S_CLR_TXC | I2S_CLR_RXC, + I2S_CLR_TXC | I2S_CLR_RXC); + if (ret < 0) + goto end; regmap_read(i2s->regmap, I2S_CLR, &val); - /* Should wait for clear operation to finish */ while (val) { regmap_read(i2s->regmap, I2S_CLR, &val); retry--; if (!retry) { dev_warn(i2s->dev, "fail to clear\n"); + ret = -EBUSY; break; } } } } +end: spin_unlock(&i2s->lock); + if (ret < 0) + dev_err(i2s->dev, "lrclk update failed\n"); + + return ret; } static int rockchip_i2s_set_fmt(struct snd_soc_dai *cpu_dai, @@ -199,13 +256,13 @@ static int rockchip_i2s_set_fmt(struct snd_soc_dai *cpu_dai, pm_runtime_get_sync(cpu_dai->dev); mask = I2S_CKR_MSS_MASK; - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* Set source clock in Master mode */ val = I2S_CKR_MSS_MASTER; i2s->is_master_mode = true; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: val = I2S_CKR_MSS_SLAVE; i2s->is_master_mode = false; break; @@ -425,17 +482,25 @@ static int rockchip_i2s_trigger(struct snd_pcm_substream *substream, case SNDRV_PCM_TRIGGER_RESUME: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) - rockchip_snd_rxctrl(i2s, 1); + ret = rockchip_snd_rxctrl(i2s, 1); else - rockchip_snd_txctrl(i2s, 1); + ret = rockchip_snd_txctrl(i2s, 1); + if (ret < 0) + return ret; + i2s_pinctrl_select_bclk_on(i2s); break; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_STOP: case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) - rockchip_snd_rxctrl(i2s, 0); - else - rockchip_snd_txctrl(i2s, 0); + if (substream->stream == SNDRV_PCM_STREAM_CAPTURE) { + if (!i2s->tx_start) + i2s_pinctrl_select_bclk_off(i2s); + ret = rockchip_snd_rxctrl(i2s, 0); + } else { + if (!i2s->rx_start) + i2s_pinctrl_select_bclk_off(i2s); + ret = rockchip_snd_txctrl(i2s, 0); + } break; default: ret = -EINVAL; @@ -498,6 +563,7 @@ static struct snd_soc_dai_driver rockchip_i2s_dai = { static const struct snd_soc_component_driver rockchip_i2s_component = { .name = DRV_NAME, + .legacy_dai_naming = 1, }; static bool rockchip_i2s_wr_reg(struct device *dev, unsigned int reg) @@ -736,6 +802,22 @@ static int rockchip_i2s_probe(struct platform_device *pdev) } i2s->bclk_ratio = 64; + i2s->pinctrl = devm_pinctrl_get(&pdev->dev); + if (!IS_ERR(i2s->pinctrl)) { + i2s->bclk_on = pinctrl_lookup_state(i2s->pinctrl, "bclk_on"); + if (!IS_ERR_OR_NULL(i2s->bclk_on)) { + i2s->bclk_off = pinctrl_lookup_state(i2s->pinctrl, "bclk_off"); + if (IS_ERR_OR_NULL(i2s->bclk_off)) { + dev_err(&pdev->dev, "failed to find i2s bclk_off\n"); + ret = -EINVAL; + goto err_clk; + } + } + } else { + dev_dbg(&pdev->dev, "failed to find i2s pinctrl\n"); + } + + i2s_pinctrl_select_bclk_off(i2s); dev_set_drvdata(&pdev->dev, i2s); diff --git a/sound/soc/rockchip/rockchip_i2s_tdm.c b/sound/soc/rockchip/rockchip_i2s_tdm.c index 98700e75b8..2550bd2a5e 100644 --- a/sound/soc/rockchip/rockchip_i2s_tdm.c +++ b/sound/soc/rockchip/rockchip_i2s_tdm.c @@ -404,19 +404,17 @@ static int rockchip_i2s_tdm_set_fmt(struct snd_soc_dai *cpu_dai, int ret; bool is_tdm = i2s_tdm->tdm_mode; - ret = pm_runtime_get_sync(cpu_dai->dev); - if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(cpu_dai->dev); + ret = pm_runtime_resume_and_get(cpu_dai->dev); + if (ret < 0 && ret != -EACCES) return ret; - } mask = I2S_CKR_MSS_MASK; - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBC_CFC: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: val = I2S_CKR_MSS_MASTER; i2s_tdm->is_master_mode = true; break; - case SND_SOC_DAIFMT_CBP_CFP: + case SND_SOC_DAIFMT_BC_FC: val = I2S_CKR_MSS_SLAVE; i2s_tdm->is_master_mode = false; break; @@ -1120,6 +1118,7 @@ static const struct snd_soc_dai_ops rockchip_i2s_tdm_dai_ops = { static const struct snd_soc_component_driver rockchip_i2s_tdm_component = { .name = DRV_NAME, + .legacy_dai_naming = 1, }; static bool rockchip_i2s_tdm_wr_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/rockchip/rockchip_max98090.c b/sound/soc/rockchip/rockchip_max98090.c index c8f1a28a92..150ac524a5 100644 --- a/sound/soc/rockchip/rockchip_max98090.c +++ b/sound/soc/rockchip/rockchip_max98090.c @@ -231,7 +231,7 @@ static int rk_hdmi_init(struct snd_soc_pcm_runtime *runtime) /* enable jack detection */ ret = snd_soc_card_jack_new(card, "HDMI Jack", SND_JACK_LINEOUT, - &rk_hdmi_jack, NULL, 0); + &rk_hdmi_jack); if (ret) { dev_err(card->dev, "Can't new HDMI Jack %d\n", ret); return ret; @@ -345,13 +345,13 @@ static int rk_98090_headset_init(struct snd_soc_component *component) int ret; /* Enable Headset and 4 Buttons Jack detection */ - ret = snd_soc_card_jack_new(component->card, "Headset Jack", - SND_JACK_HEADSET | - SND_JACK_BTN_0 | SND_JACK_BTN_1 | - SND_JACK_BTN_2 | SND_JACK_BTN_3, - &headset_jack, - headset_jack_pins, - ARRAY_SIZE(headset_jack_pins)); + ret = snd_soc_card_jack_new_pins(component->card, "Headset Jack", + SND_JACK_HEADSET | + SND_JACK_BTN_0 | SND_JACK_BTN_1 | + SND_JACK_BTN_2 | SND_JACK_BTN_3, + &headset_jack, + headset_jack_pins, + ARRAY_SIZE(headset_jack_pins)); if (ret) return ret; diff --git a/sound/soc/rockchip/rockchip_pdm.c b/sound/soc/rockchip/rockchip_pdm.c index 64d9891b64..a7549f8272 100644 --- a/sound/soc/rockchip/rockchip_pdm.c +++ b/sound/soc/rockchip/rockchip_pdm.c @@ -405,6 +405,7 @@ static struct snd_soc_dai_driver rockchip_pdm_dai = { static const struct snd_soc_component_driver rockchip_pdm_component = { .name = "rockchip-pdm", + .legacy_dai_naming = 1, }; static int rockchip_pdm_runtime_suspend(struct device *dev) @@ -688,11 +689,9 @@ static int rockchip_pdm_resume(struct device *dev) struct rk_pdm_dev *pdm = dev_get_drvdata(dev); int ret; - ret = pm_runtime_get_sync(dev); - if (ret < 0) { - pm_runtime_put(dev); + ret = pm_runtime_resume_and_get(dev); + if (ret < 0) return ret; - } ret = regcache_sync(pdm->regmap); diff --git a/sound/soc/rockchip/rockchip_rt5645.c b/sound/soc/rockchip/rockchip_rt5645.c index 16ca2ad924..d07cc5c813 100644 --- a/sound/soc/rockchip/rockchip_rt5645.c +++ b/sound/soc/rockchip/rockchip_rt5645.c @@ -107,7 +107,7 @@ static int rk_init(struct snd_soc_pcm_runtime *runtime) SND_JACK_HEADPHONE | SND_JACK_MICROPHONE | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3, - &headset_jack, NULL, 0); + &headset_jack); if (ret) { dev_err(card->dev, "New Headset Jack failed! (%d)\n", ret); return ret; diff --git a/sound/soc/rockchip/rockchip_spdif.c b/sound/soc/rockchip/rockchip_spdif.c index d027ca4b17..8bef572d3c 100644 --- a/sound/soc/rockchip/rockchip_spdif.c +++ b/sound/soc/rockchip/rockchip_spdif.c @@ -225,6 +225,7 @@ static struct snd_soc_dai_driver rk_spdif_dai = { static const struct snd_soc_component_driver rk_spdif_component = { .name = "rockchip-spdif", + .legacy_dai_naming = 1, }; static bool rk_spdif_wr_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/samsung/Kconfig b/sound/soc/samsung/Kconfig index a2221ebb1b..2a61e620cd 100644 --- a/sound/soc/samsung/Kconfig +++ b/sound/soc/samsung/Kconfig @@ -33,7 +33,8 @@ config SND_SAMSUNG_I2S config SND_SOC_SAMSUNG_NEO1973_WM8753 tristate "Audio support for Openmoko Neo1973 Smartphones (GTA02)" - depends on MACH_NEO1973_GTA02 + depends on MACH_NEO1973_GTA02 || COMPILE_TEST + depends on SND_SOC_I2C_AND_SPI select SND_S3C24XX_I2S select SND_SOC_WM8753 select SND_SOC_BT_SCO @@ -43,7 +44,8 @@ config SND_SOC_SAMSUNG_NEO1973_WM8753 config SND_SOC_SAMSUNG_JIVE_WM8750 tristate "SoC I2S Audio support for Jive" - depends on MACH_JIVE && I2C + depends on MACH_JIVE && I2C || COMPILE_TEST && ARM + depends on SND_SOC_I2C_AND_SPI select SND_SOC_WM8750 select SND_S3C2412_SOC_I2S help @@ -69,7 +71,7 @@ config SND_SOC_SAMSUNG_SMDK_WM8994 config SND_SOC_SAMSUNG_S3C24XX_UDA134X tristate "SoC I2S Audio support UDA134X wired to a S3C24XX" - depends on ARCH_S3C24XX + depends on ARCH_S3C24XX || COMPILE_TEST select SND_S3C24XX_I2S select SND_SOC_L3 select SND_SOC_UDA134X @@ -81,21 +83,24 @@ config SND_SOC_SAMSUNG_SIMTEC config SND_SOC_SAMSUNG_SIMTEC_TLV320AIC23 tristate "SoC I2S Audio support for TLV320AIC23 on Simtec boards" - depends on ARCH_S3C24XX && I2C + depends on ARCH_S3C24XX || COMPILE_TEST + depends on I2C select SND_S3C24XX_I2S select SND_SOC_TLV320AIC23_I2C select SND_SOC_SAMSUNG_SIMTEC config SND_SOC_SAMSUNG_SIMTEC_HERMES tristate "SoC I2S Audio support for Simtec Hermes board" - depends on ARCH_S3C24XX && I2C + depends on ARCH_S3C24XX || COMPILE_TEST + depends on I2C select SND_S3C24XX_I2S select SND_SOC_TLV320AIC3X select SND_SOC_SAMSUNG_SIMTEC config SND_SOC_SAMSUNG_H1940_UDA1380 tristate "Audio support for the HP iPAQ H1940" - depends on ARCH_H1940 && I2C + depends on ARCH_H1940 || COMPILE_TEST + depends on I2C select SND_S3C24XX_I2S select SND_SOC_UDA1380 help @@ -103,7 +108,8 @@ config SND_SOC_SAMSUNG_H1940_UDA1380 config SND_SOC_SAMSUNG_RX1950_UDA1380 tristate "Audio support for the HP iPAQ RX1950" - depends on MACH_RX1950 && I2C + depends on MACH_RX1950 || COMPILE_TEST + depends on I2C select SND_S3C24XX_I2S select SND_SOC_UDA1380 help diff --git a/sound/soc/samsung/aries_wm8994.c b/sound/soc/samsung/aries_wm8994.c index 5265e546b1..e7d52d2713 100644 --- a/sound/soc/samsung/aries_wm8994.c +++ b/sound/soc/samsung/aries_wm8994.c @@ -343,7 +343,7 @@ static int aries_late_probe(struct snd_soc_card *card) struct aries_wm8994_data *priv = snd_soc_card_get_drvdata(card); int ret, irq; - ret = snd_soc_card_jack_new(card, "Dock", SND_JACK_LINEOUT, + ret = snd_soc_card_jack_new_pins(card, "Dock", SND_JACK_LINEOUT, &aries_dock, dock_pins, ARRAY_SIZE(dock_pins)); if (ret) return ret; @@ -361,7 +361,7 @@ static int aries_late_probe(struct snd_soc_card *card) else snd_soc_jack_report(&aries_dock, 0, SND_JACK_LINEOUT); - ret = snd_soc_card_jack_new(card, "Headset", + ret = snd_soc_card_jack_new_pins(card, "Headset", SND_JACK_HEADSET | SND_JACK_BTN_0, &aries_headset, jack_pins, ARRAY_SIZE(jack_pins)); @@ -432,7 +432,6 @@ static const struct snd_soc_component_driver aries_component = { .idle_bias_on = 1, .use_pmdown_time = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; static struct snd_soc_dai_driver aries_ext_dai[] = { @@ -585,10 +584,10 @@ static int aries_audio_probe(struct platform_device *pdev) extcon_np = of_parse_phandle(np, "extcon", 0); priv->usb_extcon = extcon_find_edev_by_node(extcon_np); + of_node_put(extcon_np); if (IS_ERR(priv->usb_extcon)) return dev_err_probe(dev, PTR_ERR(priv->usb_extcon), "Failed to get extcon device"); - of_node_put(extcon_np); priv->adc = devm_iio_channel_get(dev, "headset-detect"); if (IS_ERR(priv->adc)) @@ -628,8 +627,10 @@ static int aries_audio_probe(struct platform_device *pdev) return -EINVAL; codec = of_get_child_by_name(dev->of_node, "codec"); - if (!codec) - return -EINVAL; + if (!codec) { + ret = -EINVAL; + goto out; + } for_each_card_prelinks(card, i, dai_link) { dai_link->codecs->of_node = of_parse_phandle(codec, diff --git a/sound/soc/samsung/bells.c b/sound/soc/samsung/bells.c index 8b83f39c3a..76998a4a4c 100644 --- a/sound/soc/samsung/bells.c +++ b/sound/soc/samsung/bells.c @@ -386,11 +386,11 @@ static struct snd_soc_codec_conf bells_codec_conf[] = { }, }; -static struct snd_soc_dapm_widget bells_widgets[] = { +static const struct snd_soc_dapm_widget bells_widgets[] = { SND_SOC_DAPM_MIC("DMIC", NULL), }; -static struct snd_soc_dapm_route bells_routes[] = { +static const struct snd_soc_dapm_route bells_routes[] = { { "Sub CLK_SYS", NULL, "OPCLK" }, { "CLKIN", NULL, "OPCLK" }, diff --git a/sound/soc/samsung/h1940_uda1380.c b/sound/soc/samsung/h1940_uda1380.c index c994e67d1e..fa45a54ab1 100644 --- a/sound/soc/samsung/h1940_uda1380.c +++ b/sound/soc/samsung/h1940_uda1380.c @@ -8,7 +8,7 @@ // Based on version from Arnaud Patard #include -#include +#include #include #include @@ -151,7 +151,8 @@ static const struct snd_soc_dapm_route audio_map[] = { static int h1940_uda1380_init(struct snd_soc_pcm_runtime *rtd) { - snd_soc_card_jack_new(rtd->card, "Headphone Jack", SND_JACK_HEADPHONE, + snd_soc_card_jack_new_pins(rtd->card, "Headphone Jack", + SND_JACK_HEADPHONE, &hp_jack, hp_jack_pins, ARRAY_SIZE(hp_jack_pins)); snd_soc_jack_add_gpios(&hp_jack, ARRAY_SIZE(hp_jack_gpios), diff --git a/sound/soc/samsung/i2s.c b/sound/soc/samsung/i2s.c index 70c827162b..9505200f3d 100644 --- a/sound/soc/samsung/i2s.c +++ b/sound/soc/samsung/i2s.c @@ -671,11 +671,11 @@ static int i2s_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) return -EINVAL; } - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: tmp |= mod_slave; break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: /* * Set default source clock in Master mode, only when the * CLK_I2S_RCLK_SRC clock is not exposed so we ensure any @@ -1143,6 +1143,8 @@ static const struct snd_soc_component_driver samsung_i2s_component = { .suspend = i2s_suspend, .resume = i2s_resume, + + .legacy_dai_naming = 1, }; #define SAMSUNG_I2S_FMTS (SNDRV_PCM_FMTBIT_S8 | SNDRV_PCM_FMTBIT_S16_LE | \ diff --git a/sound/soc/samsung/littlemill.c b/sound/soc/samsung/littlemill.c index 34067cc314..411e25cec5 100644 --- a/sound/soc/samsung/littlemill.c +++ b/sound/soc/samsung/littlemill.c @@ -228,7 +228,7 @@ static const struct snd_kcontrol_new controls[] = { SOC_DAPM_PIN_SWITCH("WM1250 Output"), }; -static struct snd_soc_dapm_widget widgets[] = { +static const struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_HP("Headphone", NULL), SND_SOC_DAPM_MIC("AMIC", NULL), @@ -239,7 +239,7 @@ static struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_PRE_PMU | SND_SOC_DAPM_POST_PMD), }; -static struct snd_soc_dapm_route audio_paths[] = { +static const struct snd_soc_dapm_route audio_paths[] = { { "Headphone", NULL, "HPOUT1L" }, { "Headphone", NULL, "HPOUT1R" }, @@ -285,7 +285,7 @@ static int littlemill_late_probe(struct snd_soc_card *card) SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3 | SND_JACK_BTN_4 | SND_JACK_BTN_5, - &littlemill_headset, NULL, 0); + &littlemill_headset); if (ret) return ret; diff --git a/sound/soc/samsung/lowland.c b/sound/soc/samsung/lowland.c index 7b12ccd2a9..b44f5e9222 100644 --- a/sound/soc/samsung/lowland.c +++ b/sound/soc/samsung/lowland.c @@ -51,10 +51,11 @@ static int lowland_wm5100_init(struct snd_soc_pcm_runtime *rtd) return ret; } - ret = snd_soc_card_jack_new(rtd->card, "Headset", SND_JACK_LINEOUT | - SND_JACK_HEADSET | SND_JACK_BTN_0, - &lowland_headset, lowland_headset_pins, - ARRAY_SIZE(lowland_headset_pins)); + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset", + SND_JACK_LINEOUT | SND_JACK_HEADSET | + SND_JACK_BTN_0, + &lowland_headset, lowland_headset_pins, + ARRAY_SIZE(lowland_headset_pins)); if (ret) return ret; @@ -140,7 +141,7 @@ static const struct snd_kcontrol_new controls[] = { SOC_DAPM_PIN_SWITCH("Headphone"), }; -static struct snd_soc_dapm_widget widgets[] = { +static const struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_HP("Headphone", NULL), SND_SOC_DAPM_MIC("Headset Mic", NULL), @@ -150,7 +151,7 @@ static struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_MIC("Main DMIC", NULL), }; -static struct snd_soc_dapm_route audio_paths[] = { +static const struct snd_soc_dapm_route audio_paths[] = { { "Sub IN1", NULL, "HPOUT2L" }, { "Sub IN2", NULL, "HPOUT2R" }, diff --git a/sound/soc/samsung/midas_wm1811.c b/sound/soc/samsung/midas_wm1811.c index 5e9dc18687..6931b9a45b 100644 --- a/sound/soc/samsung/midas_wm1811.c +++ b/sound/soc/samsung/midas_wm1811.c @@ -309,7 +309,7 @@ static int midas_late_probe(struct snd_soc_card *card) SND_JACK_HEADSET | SND_JACK_MECHANICAL | SND_JACK_BTN_0 | SND_JACK_BTN_1 | SND_JACK_BTN_2 | SND_JACK_BTN_3 | SND_JACK_BTN_4 | SND_JACK_BTN_5, - &priv->headset_jack, NULL, 0); + &priv->headset_jack); if (ret) return ret; diff --git a/sound/soc/samsung/neo1973_wm8753.c b/sound/soc/samsung/neo1973_wm8753.c index c98b68567a..e9f2334028 100644 --- a/sound/soc/samsung/neo1973_wm8753.c +++ b/sound/soc/samsung/neo1973_wm8753.c @@ -344,7 +344,7 @@ static int neo1973_probe(struct platform_device *pdev) return devm_snd_soc_register_card(dev, &neo1973); } -struct platform_driver neo1973_audio = { +static struct platform_driver neo1973_audio = { .driver = { .name = "neo1973-audio", .pm = &snd_soc_pm_ops, diff --git a/sound/soc/samsung/pcm.c b/sound/soc/samsung/pcm.c index 4c4dfde056..e859252ae5 100644 --- a/sound/soc/samsung/pcm.c +++ b/sound/soc/samsung/pcm.c @@ -340,8 +340,8 @@ static int s3c_pcm_set_fmt(struct snd_soc_dai *cpu_dai, goto exit; } - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* Nothing to do, Master by default */ break; default: @@ -480,7 +480,8 @@ static struct snd_soc_dai_driver s3c_pcm_dai[] = { }; static const struct snd_soc_component_driver s3c_pcm_component = { - .name = "s3c-pcm", + .name = "s3c-pcm", + .legacy_dai_naming = 1, }; static int s3c_pcm_dev_probe(struct platform_device *pdev) diff --git a/sound/soc/samsung/rx1950_uda1380.c b/sound/soc/samsung/rx1950_uda1380.c index 6ea1c8cc91..abf28321f7 100644 --- a/sound/soc/samsung/rx1950_uda1380.c +++ b/sound/soc/samsung/rx1950_uda1380.c @@ -128,7 +128,7 @@ static int rx1950_startup(struct snd_pcm_substream *substream) &hw_rates); } -struct gpio_desc *gpiod_speaker_power; +static struct gpio_desc *gpiod_speaker_power; static int rx1950_spk_power(struct snd_soc_dapm_widget *w, struct snd_kcontrol *kcontrol, int event) @@ -201,7 +201,8 @@ static int rx1950_hw_params(struct snd_pcm_substream *substream, static int rx1950_uda1380_init(struct snd_soc_pcm_runtime *rtd) { - snd_soc_card_jack_new(rtd->card, "Headphone Jack", SND_JACK_HEADPHONE, + snd_soc_card_jack_new_pins(rtd->card, "Headphone Jack", + SND_JACK_HEADPHONE, &hp_jack, hp_jack_pins, ARRAY_SIZE(hp_jack_pins)); snd_soc_jack_add_gpios(&hp_jack, ARRAY_SIZE(hp_jack_gpios), @@ -227,7 +228,7 @@ static int rx1950_probe(struct platform_device *pdev) return devm_snd_soc_register_card(dev, &rx1950_asoc); } -struct platform_driver rx1950_audio = { +static struct platform_driver rx1950_audio = { .driver = { .name = "rx1950-audio", .pm = &snd_soc_pm_ops, diff --git a/sound/soc/samsung/s3c-i2s-v2.c b/sound/soc/samsung/s3c-i2s-v2.c index de66cc422e..2b221cb0ed 100644 --- a/sound/soc/samsung/s3c-i2s-v2.c +++ b/sound/soc/samsung/s3c-i2s-v2.c @@ -21,17 +21,6 @@ #include "regs-i2s-v2.h" #include "s3c-i2s-v2.h" -#undef S3C_IIS_V2_SUPPORTED - -#if defined(CONFIG_CPU_S3C2412) \ - || defined(CONFIG_ARCH_S3C64XX) || defined(CONFIG_CPU_S5PV210) -#define S3C_IIS_V2_SUPPORTED -#endif - -#ifndef S3C_IIS_V2_SUPPORTED -#error Unsupported CPU model -#endif - #define S3C2412_I2S_DEBUG_CON 0 static inline struct s3c_i2sv2_info *to_info(struct snd_soc_dai *cpu_dai) @@ -252,12 +241,12 @@ static int s3c2412_i2s_set_fmt(struct snd_soc_dai *cpu_dai, iismod = readl(i2s->regs + S3C2412_IISMOD); pr_debug("hw_params r: IISMOD: %x \n", iismod); - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: i2s->master = 0; iismod |= S3C2412_IISMOD_SLAVE; break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: i2s->master = 1; iismod &= ~S3C2412_IISMOD_SLAVE; break; diff --git a/sound/soc/samsung/s3c2412-i2s.c b/sound/soc/samsung/s3c2412-i2s.c index ec1c6f9d76..0579a35296 100644 --- a/sound/soc/samsung/s3c2412-i2s.c +++ b/sound/soc/samsung/s3c2412-i2s.c @@ -192,9 +192,10 @@ static struct snd_soc_dai_driver s3c2412_i2s_dai = { }; static const struct snd_soc_component_driver s3c2412_i2s_component = { - .name = "s3c2412-i2s", - .suspend = s3c2412_i2s_suspend, - .resume = s3c2412_i2s_resume, + .name = "s3c2412-i2s", + .suspend = s3c2412_i2s_suspend, + .resume = s3c2412_i2s_resume, + .legacy_dai_naming = 1, }; static int s3c2412_iis_dev_probe(struct platform_device *pdev) diff --git a/sound/soc/samsung/s3c24xx-i2s.c b/sound/soc/samsung/s3c24xx-i2s.c index 0f46304eaa..7b7bbe007a 100644 --- a/sound/soc/samsung/s3c24xx-i2s.c +++ b/sound/soc/samsung/s3c24xx-i2s.c @@ -12,7 +12,6 @@ #include #include #include -#include #include #include @@ -169,11 +168,11 @@ static int s3c24xx_i2s_set_fmt(struct snd_soc_dai *cpu_dai, iismod = readl(s3c24xx_i2s.regs + S3C2410_IISMOD); pr_debug("hw_params r: IISMOD: %x \n", iismod); - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: iismod |= S3C2410_IISMOD_SLAVE; break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: iismod &= ~S3C2410_IISMOD_SLAVE; break; default: @@ -415,9 +414,10 @@ static struct snd_soc_dai_driver s3c24xx_i2s_dai = { }; static const struct snd_soc_component_driver s3c24xx_i2s_component = { - .name = "s3c24xx-i2s", - .suspend = s3c24xx_i2s_suspend, - .resume = s3c24xx_i2s_resume, + .name = "s3c24xx-i2s", + .suspend = s3c24xx_i2s_suspend, + .resume = s3c24xx_i2s_resume, + .legacy_dai_naming = 1, }; static int s3c24xx_iis_dev_probe(struct platform_device *pdev) diff --git a/sound/soc/samsung/smartq_wm8987.c b/sound/soc/samsung/smartq_wm8987.c index cee39ad166..29bf917242 100644 --- a/sound/soc/samsung/smartq_wm8987.c +++ b/sound/soc/samsung/smartq_wm8987.c @@ -139,10 +139,10 @@ static int smartq_wm8987_init(struct snd_soc_pcm_runtime *rtd) snd_soc_dapm_nc_pin(dapm, "ROUT1"); /* Headphone jack detection */ - err = snd_soc_card_jack_new(rtd->card, "Headphone Jack", - SND_JACK_HEADPHONE, &smartq_jack, - smartq_jack_pins, - ARRAY_SIZE(smartq_jack_pins)); + err = snd_soc_card_jack_new_pins(rtd->card, "Headphone Jack", + SND_JACK_HEADPHONE, &smartq_jack, + smartq_jack_pins, + ARRAY_SIZE(smartq_jack_pins)); if (err) return err; diff --git a/sound/soc/samsung/snow.c b/sound/soc/samsung/snow.c index 02372109c2..da342da038 100644 --- a/sound/soc/samsung/snow.c +++ b/sound/soc/samsung/snow.c @@ -216,7 +216,7 @@ static int snow_probe(struct platform_device *pdev) return dev_err_probe(&pdev->dev, ret, "snd_soc_register_card failed\n"); - return ret; + return 0; } static int snow_remove(struct platform_device *pdev) diff --git a/sound/soc/samsung/spdif.c b/sound/soc/samsung/spdif.c index 226c359892..7d815e237e 100644 --- a/sound/soc/samsung/spdif.c +++ b/sound/soc/samsung/spdif.c @@ -352,9 +352,10 @@ static struct snd_soc_dai_driver samsung_spdif_dai = { }; static const struct snd_soc_component_driver samsung_spdif_component = { - .name = "samsung-spdif", - .suspend = spdif_suspend, - .resume = spdif_resume, + .name = "samsung-spdif", + .suspend = spdif_suspend, + .resume = spdif_resume, + .legacy_dai_naming = 1, }; static int spdif_probe(struct platform_device *pdev) @@ -467,8 +468,7 @@ static int spdif_remove(struct platform_device *pdev) iounmap(spdif->regs); mem_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); - if (mem_res) - release_mem_region(mem_res->start, resource_size(mem_res)); + release_mem_region(mem_res->start, resource_size(mem_res)); clk_disable_unprepare(spdif->sclk); clk_disable_unprepare(spdif->pclk); diff --git a/sound/soc/samsung/speyside.c b/sound/soc/samsung/speyside.c index 37b1f4f60b..69d7b0115b 100644 --- a/sound/soc/samsung/speyside.c +++ b/sound/soc/samsung/speyside.c @@ -156,10 +156,12 @@ static int speyside_wm8996_init(struct snd_soc_pcm_runtime *rtd) pr_err("Failed to request HP_SEL GPIO: %d\n", ret); gpio_direction_output(WM8996_HPSEL_GPIO, speyside_jack_polarity); - ret = snd_soc_card_jack_new(rtd->card, "Headset", SND_JACK_LINEOUT | - SND_JACK_HEADSET | SND_JACK_BTN_0, - &speyside_headset, speyside_headset_pins, - ARRAY_SIZE(speyside_headset_pins)); + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset", + SND_JACK_LINEOUT | SND_JACK_HEADSET | + SND_JACK_BTN_0, + &speyside_headset, + speyside_headset_pins, + ARRAY_SIZE(speyside_headset_pins)); if (ret) return ret; @@ -261,7 +263,7 @@ static const struct snd_kcontrol_new controls[] = { SOC_DAPM_PIN_SWITCH("Headphone"), }; -static struct snd_soc_dapm_widget widgets[] = { +static const struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_HP("Headphone", NULL), SND_SOC_DAPM_MIC("Headset Mic", NULL), @@ -271,7 +273,7 @@ static struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_MIC("Main DMIC", NULL), }; -static struct snd_soc_dapm_route audio_paths[] = { +static const struct snd_soc_dapm_route audio_paths[] = { { "IN1RN", NULL, "MICB1" }, { "IN1RP", NULL, "MICB1" }, { "IN1RN", NULL, "MICB2" }, diff --git a/sound/soc/samsung/tobermory.c b/sound/soc/samsung/tobermory.c index 8d3149a47a..9287a1d0ee 100644 --- a/sound/soc/samsung/tobermory.c +++ b/sound/soc/samsung/tobermory.c @@ -130,7 +130,7 @@ static const struct snd_kcontrol_new controls[] = { SOC_DAPM_PIN_SWITCH("DMIC"), }; -static struct snd_soc_dapm_widget widgets[] = { +static const struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_HP("Headphone", NULL), SND_SOC_DAPM_MIC("Headset Mic", NULL), @@ -140,7 +140,7 @@ static struct snd_soc_dapm_widget widgets[] = { SND_SOC_DAPM_SPK("Main Speaker", NULL), }; -static struct snd_soc_dapm_route audio_paths[] = { +static const struct snd_soc_dapm_route audio_paths[] = { { "Headphone", NULL, "HPOUTL" }, { "Headphone", NULL, "HPOUTR" }, @@ -189,10 +189,10 @@ static int tobermory_late_probe(struct snd_soc_card *card) if (ret < 0) return ret; - ret = snd_soc_card_jack_new(card, "Headset", SND_JACK_HEADSET | - SND_JACK_BTN_0, &tobermory_headset, - tobermory_headset_pins, - ARRAY_SIZE(tobermory_headset_pins)); + ret = snd_soc_card_jack_new_pins(card, "Headset", SND_JACK_HEADSET | + SND_JACK_BTN_0, &tobermory_headset, + tobermory_headset_pins, + ARRAY_SIZE(tobermory_headset_pins)); if (ret) return ret; diff --git a/sound/soc/sh/Kconfig b/sound/soc/sh/Kconfig index ae46f187cc..97916e3ca9 100644 --- a/sound/soc/sh/Kconfig +++ b/sound/soc/sh/Kconfig @@ -47,7 +47,7 @@ config SND_SOC_RCAR config SND_SOC_RZ tristate "RZ/G2L series SSIF-2 support" - depends on ARCH_R9A07G044 || COMPILE_TEST + depends on ARCH_RZG2L || COMPILE_TEST help This option enables RZ/G2L SSIF-2 sound support. diff --git a/sound/soc/sh/fsi.c b/sound/soc/sh/fsi.c index e9a1eb6bdf..f3edc2e3d9 100644 --- a/sound/soc/sh/fsi.c +++ b/sound/soc/sh/fsi.c @@ -1646,10 +1646,10 @@ static int fsi_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) int ret; /* set clock master audio interface */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: fsi->clk_master = 1; /* cpu is master */ break; default: diff --git a/sound/soc/sh/hac.c b/sound/soc/sh/hac.c index 475fc984f8..46d145cbaf 100644 --- a/sound/soc/sh/hac.c +++ b/sound/soc/sh/hac.c @@ -307,7 +307,8 @@ static struct snd_soc_dai_driver sh4_hac_dai[] = { }; static const struct snd_soc_component_driver sh4_hac_component = { - .name = "sh4-hac", + .name = "sh4-hac", + .legacy_dai_naming = 1, }; static int hac_soc_platform_probe(struct platform_device *pdev) diff --git a/sound/soc/sh/rcar/core.c b/sound/soc/sh/rcar/core.c index 6a8fe0da76..7e380d71b0 100644 --- a/sound/soc/sh/rcar/core.c +++ b/sound/soc/sh/rcar/core.c @@ -755,11 +755,11 @@ static int rsnd_soc_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) struct rsnd_dai *rdai = rsnd_dai_to_rdai(dai); /* set clock master for audio interface */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBP_CFP: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: rdai->clk_master = 0; break; - case SND_SOC_DAIFMT_CBC_CFC: + case SND_SOC_DAIFMT_BP_FP: rdai->clk_master = 1; /* cpu is master */ break; default: @@ -1159,6 +1159,7 @@ void rsnd_parse_connect_common(struct rsnd_dai *rdai, char *name, struct device_node *capture) { struct rsnd_priv *priv = rsnd_rdai_to_priv(rdai); + struct device *dev = rsnd_priv_to_dev(priv); struct device_node *np; int i; @@ -1169,7 +1170,11 @@ void rsnd_parse_connect_common(struct rsnd_dai *rdai, char *name, for_each_child_of_node(node, np) { struct rsnd_mod *mod; - i = rsnd_node_fixed_index(np, name, i); + i = rsnd_node_fixed_index(dev, np, name, i); + if (i < 0) { + of_node_put(np); + break; + } mod = mod_get(priv, i); @@ -1183,7 +1188,7 @@ void rsnd_parse_connect_common(struct rsnd_dai *rdai, char *name, of_node_put(node); } -int rsnd_node_fixed_index(struct device_node *node, char *name, int idx) +int rsnd_node_fixed_index(struct device *dev, struct device_node *node, char *name, int idx) { char node_name[16]; @@ -1210,6 +1215,8 @@ int rsnd_node_fixed_index(struct device_node *node, char *name, int idx) return idx; } + dev_err(dev, "strange node numbering (%s)", + of_node_full_name(node)); return -EINVAL; } @@ -1221,10 +1228,8 @@ int rsnd_node_count(struct rsnd_priv *priv, struct device_node *node, char *name i = 0; for_each_child_of_node(node, np) { - i = rsnd_node_fixed_index(np, name, i); + i = rsnd_node_fixed_index(dev, np, name, i); if (i < 0) { - dev_err(dev, "strange node numbering (%s)", - of_node_full_name(node)); of_node_put(np); return 0; } @@ -1808,11 +1813,12 @@ int rsnd_kctrl_new(struct rsnd_mod *mod, * snd_soc_component */ static const struct snd_soc_component_driver rsnd_soc_component = { - .name = "rsnd", - .probe = rsnd_debugfs_probe, - .hw_params = rsnd_hw_params, - .hw_free = rsnd_hw_free, - .pointer = rsnd_pointer, + .name = "rsnd", + .probe = rsnd_debugfs_probe, + .hw_params = rsnd_hw_params, + .hw_free = rsnd_hw_free, + .pointer = rsnd_pointer, + .legacy_dai_naming = 1, }; static int rsnd_rdai_continuance_probe(struct rsnd_priv *priv, @@ -1963,19 +1969,26 @@ static int rsnd_remove(struct platform_device *pdev) rsnd_cmd_remove, rsnd_adg_remove, }; - int ret = 0, i; + int i; pm_runtime_disable(&pdev->dev); for_each_rsnd_dai(rdai, priv, i) { - ret |= rsnd_dai_call(remove, &rdai->playback, priv); - ret |= rsnd_dai_call(remove, &rdai->capture, priv); + int ret; + + ret = rsnd_dai_call(remove, &rdai->playback, priv); + if (ret) + dev_warn(&pdev->dev, "Failed to remove playback dai #%d\n", i); + + ret = rsnd_dai_call(remove, &rdai->capture, priv); + if (ret) + dev_warn(&pdev->dev, "Failed to remove capture dai #%d\n", i); } for (i = 0; i < ARRAY_SIZE(remove_func); i++) remove_func[i](priv); - return ret; + return 0; } static int __maybe_unused rsnd_suspend(struct device *dev) diff --git a/sound/soc/sh/rcar/dma.c b/sound/soc/sh/rcar/dma.c index 03e0d4eca7..463ab237d7 100644 --- a/sound/soc/sh/rcar/dma.c +++ b/sound/soc/sh/rcar/dma.c @@ -240,12 +240,19 @@ static int rsnd_dmaen_start(struct rsnd_mod *mod, struct dma_chan *rsnd_dma_request_channel(struct device_node *of_node, char *name, struct rsnd_mod *mod, char *x) { + struct rsnd_priv *priv = rsnd_mod_to_priv(mod); + struct device *dev = rsnd_priv_to_dev(priv); struct dma_chan *chan = NULL; struct device_node *np; int i = 0; for_each_child_of_node(of_node, np) { - i = rsnd_node_fixed_index(np, name, i); + i = rsnd_node_fixed_index(dev, np, name, i); + if (i < 0) { + chan = NULL; + of_node_put(np); + break; + } if (i == rsnd_mod_id_raw(mod) && (!chan)) chan = of_dma_request_slave_channel(np, x); diff --git a/sound/soc/sh/rcar/rsnd.h b/sound/soc/sh/rcar/rsnd.h index 6580bab0e2..d9cd190d7e 100644 --- a/sound/soc/sh/rcar/rsnd.h +++ b/sound/soc/sh/rcar/rsnd.h @@ -460,7 +460,7 @@ void rsnd_parse_connect_common(struct rsnd_dai *rdai, char *name, struct device_node *playback, struct device_node *capture); int rsnd_node_count(struct rsnd_priv *priv, struct device_node *node, char *name); -int rsnd_node_fixed_index(struct device_node *node, char *name, int idx); +int rsnd_node_fixed_index(struct device *dev, struct device_node *node, char *name, int idx); int rsnd_channel_normalization(int chan); #define rsnd_runtime_channel_original(io) \ diff --git a/sound/soc/sh/rcar/src.c b/sound/soc/sh/rcar/src.c index 42a100c630..0ea84ae57c 100644 --- a/sound/soc/sh/rcar/src.c +++ b/sound/soc/sh/rcar/src.c @@ -676,7 +676,12 @@ int rsnd_src_probe(struct rsnd_priv *priv) if (!of_device_is_available(np)) goto skip; - i = rsnd_node_fixed_index(np, SRC_NAME, i); + i = rsnd_node_fixed_index(dev, np, SRC_NAME, i); + if (i < 0) { + ret = -EINVAL; + of_node_put(np); + goto rsnd_src_probe_done; + } src = rsnd_src_get(priv, i); diff --git a/sound/soc/sh/rcar/ssi.c b/sound/soc/sh/rcar/ssi.c index 87e606f688..43c5e27dc5 100644 --- a/sound/soc/sh/rcar/ssi.c +++ b/sound/soc/sh/rcar/ssi.c @@ -1105,6 +1105,7 @@ void rsnd_parse_connect_ssi(struct rsnd_dai *rdai, struct device_node *capture) { struct rsnd_priv *priv = rsnd_rdai_to_priv(rdai); + struct device *dev = rsnd_priv_to_dev(priv); struct device_node *node; struct device_node *np; int i; @@ -1117,7 +1118,11 @@ void rsnd_parse_connect_ssi(struct rsnd_dai *rdai, for_each_child_of_node(node, np) { struct rsnd_mod *mod; - i = rsnd_node_fixed_index(np, SSI_NAME, i); + i = rsnd_node_fixed_index(dev, np, SSI_NAME, i); + if (i < 0) { + of_node_put(np); + break; + } mod = rsnd_ssi_mod_get(priv, i); @@ -1182,7 +1187,12 @@ int rsnd_ssi_probe(struct rsnd_priv *priv) if (!of_device_is_available(np)) goto skip; - i = rsnd_node_fixed_index(np, SSI_NAME, i); + i = rsnd_node_fixed_index(dev, np, SSI_NAME, i); + if (i < 0) { + ret = -EINVAL; + of_node_put(np); + goto rsnd_ssi_probe_done; + } ssi = rsnd_ssi_get(priv, i); diff --git a/sound/soc/sh/rcar/ssiu.c b/sound/soc/sh/rcar/ssiu.c index 0d8f97633d..281bc20d4c 100644 --- a/sound/soc/sh/rcar/ssiu.c +++ b/sound/soc/sh/rcar/ssiu.c @@ -67,6 +67,8 @@ static void rsnd_ssiu_busif_err_irq_ctrl(struct rsnd_mod *mod, int enable) shift = 1; offset = 1; break; + default: + return; } for (i = 0; i < 4; i++) { @@ -102,6 +104,8 @@ bool rsnd_ssiu_busif_err_status_clear(struct rsnd_mod *mod) shift = 1; offset = 1; break; + default: + goto out; } for (i = 0; i < 4; i++) { @@ -120,7 +124,7 @@ bool rsnd_ssiu_busif_err_status_clear(struct rsnd_mod *mod) } rsnd_mod_write(mod, reg, val); } - +out: return error; } @@ -415,6 +419,7 @@ static struct rsnd_mod_ops rsnd_ssiu_ops_gen2 = { .name = SSIU_NAME, .dma_req = rsnd_ssiu_dma_req, .init = rsnd_ssiu_init_gen2, + .quit = rsnd_ssiu_quit, .start = rsnd_ssiu_start_gen2, .stop = rsnd_ssiu_stop_gen2, .get_status = rsnd_ssiu_get_status, @@ -460,6 +465,7 @@ void rsnd_parse_connect_ssiu(struct rsnd_dai *rdai, struct device_node *capture) { struct rsnd_priv *priv = rsnd_rdai_to_priv(rdai); + struct device *dev = rsnd_priv_to_dev(priv); struct device_node *node = rsnd_ssiu_of_node(priv); struct rsnd_dai_stream *io_p = &rdai->playback; struct rsnd_dai_stream *io_c = &rdai->capture; @@ -472,7 +478,11 @@ void rsnd_parse_connect_ssiu(struct rsnd_dai *rdai, for_each_child_of_node(node, np) { struct rsnd_mod *mod; - i = rsnd_node_fixed_index(np, SSIU_NAME, i); + i = rsnd_node_fixed_index(dev, np, SSIU_NAME, i); + if (i < 0) { + of_node_put(np); + break; + } mod = rsnd_ssiu_mod_get(priv, i); diff --git a/sound/soc/sh/rz-ssi.c b/sound/soc/sh/rz-ssi.c index e8edaed05d..7ace0c0db5 100644 --- a/sound/soc/sh/rz-ssi.c +++ b/sound/soc/sh/rz-ssi.c @@ -59,9 +59,7 @@ #define SSIFSR_RDC_MASK 0x3f #define SSIFSR_RDC_SHIFT 8 -#define SSIFSR_TDC(x) (((x) & 0x1f) << 24) #define SSIFSR_TDE BIT(16) -#define SSIFSR_RDC(x) (((x) & 0x1f) << 8) #define SSIFSR_RDF BIT(0) #define SSIOFR_LRCONT BIT(8) @@ -769,7 +767,7 @@ static int rz_ssi_dai_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) struct rz_ssi_priv *ssi = snd_soc_dai_get_drvdata(dai); switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { - case SND_SOC_DAIFMT_CBC_CFC: + case SND_SOC_DAIFMT_BP_FP: break; default: dev_err(ssi->dev, "Codec should be clk and frame consumer\n"); @@ -908,10 +906,11 @@ static struct snd_soc_dai_driver rz_ssi_soc_dai[] = { }; static const struct snd_soc_component_driver rz_ssi_soc_component = { - .name = "rz-ssi", - .open = rz_ssi_pcm_open, - .pointer = rz_ssi_pcm_pointer, - .pcm_construct = rz_ssi_pcm_new, + .name = "rz-ssi", + .open = rz_ssi_pcm_open, + .pointer = rz_ssi_pcm_pointer, + .pcm_construct = rz_ssi_pcm_new, + .legacy_dai_naming = 1, }; static int rz_ssi_probe(struct platform_device *pdev) @@ -978,22 +977,24 @@ static int rz_ssi_probe(struct platform_device *pdev) /* Error Interrupt */ ssi->irq_int = platform_get_irq_byname(pdev, "int_req"); - if (ssi->irq_int < 0) - return dev_err_probe(&pdev->dev, -ENODEV, - "Unable to get SSI int_req IRQ\n"); + if (ssi->irq_int < 0) { + rz_ssi_release_dma_channels(ssi); + return ssi->irq_int; + } ret = devm_request_irq(&pdev->dev, ssi->irq_int, &rz_ssi_interrupt, 0, dev_name(&pdev->dev), ssi); - if (ret < 0) + if (ret < 0) { + rz_ssi_release_dma_channels(ssi); return dev_err_probe(&pdev->dev, ret, "irq request error (int_req)\n"); + } if (!rz_ssi_is_dma_enabled(ssi)) { /* Tx and Rx interrupts (pio only) */ ssi->irq_tx = platform_get_irq_byname(pdev, "dma_tx"); if (ssi->irq_tx < 0) - return dev_err_probe(&pdev->dev, -ENODEV, - "Unable to get SSI dma_tx IRQ\n"); + return ssi->irq_tx; ret = devm_request_irq(&pdev->dev, ssi->irq_tx, &rz_ssi_interrupt, 0, @@ -1004,8 +1005,7 @@ static int rz_ssi_probe(struct platform_device *pdev) ssi->irq_rx = platform_get_irq_byname(pdev, "dma_rx"); if (ssi->irq_rx < 0) - return dev_err_probe(&pdev->dev, -ENODEV, - "Unable to get SSI dma_rx IRQ\n"); + return ssi->irq_rx; ret = devm_request_irq(&pdev->dev, ssi->irq_rx, &rz_ssi_interrupt, 0, @@ -1016,30 +1016,37 @@ static int rz_ssi_probe(struct platform_device *pdev) } ssi->rstc = devm_reset_control_get_exclusive(&pdev->dev, NULL); - if (IS_ERR(ssi->rstc)) - return PTR_ERR(ssi->rstc); + if (IS_ERR(ssi->rstc)) { + ret = PTR_ERR(ssi->rstc); + goto err_reset; + } reset_control_deassert(ssi->rstc); pm_runtime_enable(&pdev->dev); ret = pm_runtime_resume_and_get(&pdev->dev); if (ret < 0) { - pm_runtime_disable(ssi->dev); - reset_control_assert(ssi->rstc); - return dev_err_probe(ssi->dev, ret, "pm_runtime_resume_and_get failed\n"); + dev_err(&pdev->dev, "pm_runtime_resume_and_get failed\n"); + goto err_pm; } ret = devm_snd_soc_register_component(&pdev->dev, &rz_ssi_soc_component, rz_ssi_soc_dai, ARRAY_SIZE(rz_ssi_soc_dai)); if (ret < 0) { - rz_ssi_release_dma_channels(ssi); - - pm_runtime_put(ssi->dev); - pm_runtime_disable(ssi->dev); - reset_control_assert(ssi->rstc); dev_err(&pdev->dev, "failed to register snd component\n"); + goto err_snd_soc; } + return 0; + +err_snd_soc: + pm_runtime_put(ssi->dev); +err_pm: + pm_runtime_disable(ssi->dev); + reset_control_assert(ssi->rstc); +err_reset: + rz_ssi_release_dma_channels(ssi); + return ret; } diff --git a/sound/soc/sh/siu_pcm.c b/sound/soc/sh/siu_pcm.c index 0a8a3c314a..f15ff36e79 100644 --- a/sound/soc/sh/siu_pcm.c +++ b/sound/soc/sh/siu_pcm.c @@ -540,13 +540,14 @@ static void siu_pcm_free(struct snd_soc_component *component, } const struct snd_soc_component_driver siu_component = { - .name = DRV_NAME, - .open = siu_pcm_open, - .close = siu_pcm_close, - .prepare = siu_pcm_prepare, - .trigger = siu_pcm_trigger, - .pointer = siu_pcm_pointer_dma, - .pcm_construct = siu_pcm_new, - .pcm_destruct = siu_pcm_free, + .name = DRV_NAME, + .open = siu_pcm_open, + .close = siu_pcm_close, + .prepare = siu_pcm_prepare, + .trigger = siu_pcm_trigger, + .pointer = siu_pcm_pointer_dma, + .pcm_construct = siu_pcm_new, + .pcm_destruct = siu_pcm_free, + .legacy_dai_naming = 1, }; EXPORT_SYMBOL_GPL(siu_component); diff --git a/sound/soc/sh/ssi.c b/sound/soc/sh/ssi.c index 15b01bcefc..96cf523c22 100644 --- a/sound/soc/sh/ssi.c +++ b/sound/soc/sh/ssi.c @@ -291,16 +291,16 @@ static int ssi_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) return -EINVAL; } - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: break; - case SND_SOC_DAIFMT_CBS_CFM: + case SND_SOC_DAIFMT_BP_FC: ssicr |= CR_SCK_MASTER; break; - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FP: ssicr |= CR_SWS_MASTER; break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: ssicr |= CR_SWS_MASTER | CR_SCK_MASTER; break; default: @@ -377,7 +377,8 @@ static struct snd_soc_dai_driver sh4_ssi_dai[] = { }; static const struct snd_soc_component_driver sh4_ssi_component = { - .name = "sh4-ssi", + .name = "sh4-ssi", + .legacy_dai_naming = 1, }; static int sh4_soc_dai_probe(struct platform_device *pdev) diff --git a/sound/soc/sof/Kconfig b/sound/soc/sof/Kconfig index 4542868cd7..e90f173d06 100644 --- a/sound/soc/sof/Kconfig +++ b/sound/soc/sof/Kconfig @@ -252,6 +252,13 @@ config SND_SOC_SOF_PROBE_WORK_QUEUE When selected, the probe is handled in two steps, for example to avoid lockdeps if request_module is used in the probe. +# Supported IPC versions +config SND_SOC_SOF_IPC3 + bool + +config SND_SOC_SOF_INTEL_IPC4 + bool + source "sound/soc/sof/amd/Kconfig" source "sound/soc/sof/imx/Kconfig" source "sound/soc/sof/intel/Kconfig" diff --git a/sound/soc/sof/Makefile b/sound/soc/sof/Makefile index 18acbc001b..9a74ed116e 100644 --- a/sound/soc/sof/Makefile +++ b/sound/soc/sof/Makefile @@ -1,8 +1,18 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) snd-sof-objs := core.o ops.o loader.o ipc.o pcm.o pm.o debug.o topology.o\ - control.o trace.o iomem-utils.o sof-audio.o stream-ipc.o\ - ipc3-topology.o ipc3.o ipc3-control.o ipc3-pcm.o + control.o trace.o iomem-utils.o sof-audio.o stream-ipc.o + +# IPC implementations +ifneq ($(CONFIG_SND_SOC_SOF_IPC3),) +snd-sof-objs += ipc3.o ipc3-loader.o ipc3-topology.o ipc3-control.o ipc3-pcm.o\ + ipc3-dtrace.o +endif +ifneq ($(CONFIG_SND_SOC_SOF_INTEL_IPC4),) +snd-sof-objs += ipc4.o ipc4-loader.o ipc4-topology.o ipc4-control.o ipc4-pcm.o +endif + +# SOF client support ifneq ($(CONFIG_SND_SOC_SOF_CLIENT),) snd-sof-objs += sof-client.o endif diff --git a/sound/soc/sof/amd/Kconfig b/sound/soc/sof/amd/Kconfig index 085232e045..190c85d570 100644 --- a/sound/soc/sof/amd/Kconfig +++ b/sound/soc/sof/amd/Kconfig @@ -17,6 +17,7 @@ if SND_SOC_SOF_AMD_TOPLEVEL config SND_SOC_SOF_AMD_COMMON tristate select SND_SOC_SOF + select SND_SOC_SOF_IPC3 select SND_SOC_SOF_PCI_DEV select SND_AMD_ACP_CONFIG select SND_SOC_ACPI if ACPI diff --git a/sound/soc/sof/amd/acp-dsp-offset.h b/sound/soc/sof/amd/acp-dsp-offset.h index 40fbf11fac..56cefd4a84 100644 --- a/sound/soc/sof/amd/acp-dsp-offset.h +++ b/sound/soc/sof/amd/acp-dsp-offset.h @@ -46,12 +46,14 @@ #define ACPAXI2AXI_ATU_BASE_ADDR_GRP_8 0xC3C #define ACPAXI2AXI_ATU_CTRL 0xC40 #define ACP_SOFT_RESET 0x1000 +#define ACP_CONTROL 0x1004 #define ACP_I2S_PIN_CONFIG 0x1400 /* Registers from ACP_PGFSM block */ #define ACP_PGFSM_CONTROL 0x141C #define ACP_PGFSM_STATUS 0x1420 +#define ACP_CLKMUX_SEL 0x1424 /* Registers from ACP_INTR block */ #define ACP_EXTERNAL_INTR_ENB 0x1800 diff --git a/sound/soc/sof/amd/acp-trace.c b/sound/soc/sof/amd/acp-trace.c index 903b6cc3dd..c9482b27cb 100644 --- a/sound/soc/sof/amd/acp-trace.c +++ b/sound/soc/sof/amd/acp-trace.c @@ -34,7 +34,7 @@ int acp_sof_trace_release(struct snd_sof_dev *sdev) } EXPORT_SYMBOL_NS(acp_sof_trace_release, SND_SOC_SOF_AMD_COMMON); -int acp_sof_trace_init(struct snd_sof_dev *sdev, +int acp_sof_trace_init(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, struct sof_ipc_dma_trace_params_ext *dtrace_params) { struct acp_dsp_stream *stream; @@ -46,7 +46,7 @@ int acp_sof_trace_init(struct snd_sof_dev *sdev, if (!stream) return -ENODEV; - stream->dmab = &sdev->dmatb; + stream->dmab = dmab; stream->num_pages = NUM_PAGES; ret = acp_dsp_stream_config(sdev, stream); diff --git a/sound/soc/sof/amd/acp.c b/sound/soc/sof/amd/acp.c index 71d71c1523..c40d2900dd 100644 --- a/sound/soc/sof/amd/acp.c +++ b/sound/soc/sof/amd/acp.c @@ -138,23 +138,75 @@ int configure_and_run_dma(struct acp_dev_data *adata, unsigned int src_addr, return ret; } -static int psp_fw_validate(struct acp_dev_data *adata) +/* + * psp_mbox_ready- function to poll ready bit of psp mbox + * @adata: acp device data + * @ack: bool variable to check ready bit status or psp ack + */ + +static int psp_mbox_ready(struct acp_dev_data *adata, bool ack) { struct snd_sof_dev *sdev = adata->dev; int timeout; u32 data; - smn_write(adata->smn_dev, MP0_C2PMSG_26_REG, MBOX_ACP_SHA_DMA_COMMAND); - for (timeout = ACP_PSP_TIMEOUT_COUNTER; timeout > 0; timeout--) { msleep(20); - smn_read(adata->smn_dev, MP0_C2PMSG_26_REG, &data); + smn_read(adata->smn_dev, MP0_C2PMSG_114_REG, &data); if (data & MBOX_READY_MASK) return 0; } - dev_err(sdev->dev, "FW validation timedout: status %x\n", data & MBOX_STATUS_MASK); - return -ETIMEDOUT; + dev_err(sdev->dev, "PSP error status %x\n", data & MBOX_STATUS_MASK); + + if (ack) + return -ETIMEDOUT; + + return -EBUSY; +} + +/* + * psp_send_cmd - function to send psp command over mbox + * @adata: acp device data + * @cmd: non zero integer value for command type + */ + +static int psp_send_cmd(struct acp_dev_data *adata, int cmd) +{ + struct snd_sof_dev *sdev = adata->dev; + int ret, timeout; + u32 data; + + if (!cmd) + return -EINVAL; + + /* Get a non-zero Doorbell value from PSP */ + for (timeout = ACP_PSP_TIMEOUT_COUNTER; timeout > 0; timeout--) { + msleep(MBOX_DELAY); + smn_read(adata->smn_dev, MP0_C2PMSG_73_REG, &data); + if (data) + break; + } + + if (!timeout) { + dev_err(sdev->dev, "Failed to get Doorbell from MBOX %x\n", MP0_C2PMSG_73_REG); + return -EINVAL; + } + + /* Check if PSP is ready for new command */ + ret = psp_mbox_ready(adata, 0); + if (ret) + return ret; + + smn_write(adata->smn_dev, MP0_C2PMSG_114_REG, cmd); + + /* Ring the Doorbell for PSP */ + smn_write(adata->smn_dev, MP0_C2PMSG_73_REG, data); + + /* Check MBOX ready as PSP ack */ + ret = psp_mbox_ready(adata, 1); + + return ret; } int configure_and_run_sha_dma(struct acp_dev_data *adata, void *image_addr, @@ -196,7 +248,7 @@ int configure_and_run_sha_dma(struct acp_dev_data *adata, void *image_addr, return ret; } - ret = psp_fw_validate(adata); + ret = psp_send_cmd(adata, MBOX_ACP_SHA_DMA_COMMAND); if (ret) return ret; @@ -361,10 +413,46 @@ static int acp_init(struct snd_sof_dev *sdev) dev_err(sdev->dev, "ACP power on failed\n"); return ret; } + + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CONTROL, 0x01); /* Reset */ return acp_reset(sdev); } +int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state) +{ + int ret; + + ret = acp_reset(sdev); + if (ret) { + dev_err(sdev->dev, "ACP Reset failed\n"); + return ret; + } + + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CONTROL, 0x00); + + return 0; +} +EXPORT_SYMBOL_NS(amd_sof_acp_suspend, SND_SOC_SOF_AMD_COMMON); + +int amd_sof_acp_resume(struct snd_sof_dev *sdev) +{ + int ret; + + ret = acp_init(sdev); + if (ret) { + dev_err(sdev->dev, "ACP Init failed\n"); + return ret; + } + + snd_sof_dsp_write(sdev, ACP_DSP_BAR, ACP_CLKMUX_SEL, 0x03); + + ret = acp_memory_init(sdev); + + return ret; +} +EXPORT_SYMBOL_NS(amd_sof_acp_resume, SND_SOC_SOF_AMD_COMMON); + int amd_sof_acp_probe(struct snd_sof_dev *sdev) { struct pci_dev *pci = to_pci_dev(sdev->dev); diff --git a/sound/soc/sof/amd/acp.h b/sound/soc/sof/amd/acp.h index 35e46fe667..4c42b8fd6a 100644 --- a/sound/soc/sof/amd/acp.h +++ b/sound/soc/sof/amd/acp.h @@ -57,8 +57,10 @@ #define ACP_SHA_STAT 0x8000 #define ACP_PSP_TIMEOUT_COUNTER 5 #define ACP_EXT_INTR_ERROR_STAT 0x20000000 -#define MP0_C2PMSG_26_REG 0x03810570 -#define MBOX_ACP_SHA_DMA_COMMAND 0x330000 +#define MP0_C2PMSG_114_REG 0x3810AC8 +#define MP0_C2PMSG_73_REG 0x3810A24 +#define MBOX_ACP_SHA_DMA_COMMAND 0x70000 +#define MBOX_DELAY 1000 #define MBOX_READY_MASK 0x80000000 #define MBOX_STATUS_MASK 0xFFFF @@ -204,16 +206,20 @@ int acp_pcm_hw_params(struct snd_sof_dev *sdev, struct snd_pcm_substream *substr struct snd_pcm_hw_params *params, struct snd_sof_platform_stream_params *platform_params); -extern const struct snd_sof_dsp_ops sof_renoir_ops; +extern struct snd_sof_dsp_ops sof_renoir_ops; /* Machine configuration */ int snd_amd_acp_find_config(struct pci_dev *pci); /* Trace */ -int acp_sof_trace_init(struct snd_sof_dev *sdev, +int acp_sof_trace_init(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, struct sof_ipc_dma_trace_params_ext *dtrace_params); int acp_sof_trace_release(struct snd_sof_dev *sdev); +/* PM Callbacks */ +int amd_sof_acp_suspend(struct snd_sof_dev *sdev, u32 target_state); +int amd_sof_acp_resume(struct snd_sof_dev *sdev); + struct sof_amd_acp_desc { unsigned int host_bridge_id; }; diff --git a/sound/soc/sof/amd/pci-rn.c b/sound/soc/sof/amd/pci-rn.c index 392ffbdf64..3a7fed25a2 100644 --- a/sound/soc/sof/amd/pci-rn.c +++ b/sound/soc/sof/amd/pci-rn.c @@ -49,14 +49,23 @@ static const struct sof_amd_acp_desc renoir_chip_info = { static const struct sof_dev_desc renoir_desc = { .machines = snd_soc_acpi_amd_sof_machines, + .use_acpi_target_states = true, .resindex_lpe_base = 0, .resindex_pcicfg_base = -1, .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &renoir_chip_info, - .default_fw_path = "amd/sof", - .default_tplg_path = "amd/sof-tplg", - .default_fw_filename = "sof-rn.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "amd/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "amd/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-rn.ri", + }, .nocodec_tplg_filename = "sof-acp.tplg", .ops = &sof_renoir_ops, }; @@ -93,6 +102,7 @@ static int acp_pci_rn_probe(struct pci_dev *pci, const struct pci_device_id *pci res = devm_kzalloc(&pci->dev, sizeof(struct resource) * ARRAY_SIZE(renoir_res), GFP_KERNEL); if (!res) { sof_pci_remove(pci); + platform_device_unregister(dmic_dev); return -ENOMEM; } @@ -157,6 +167,9 @@ static struct pci_driver snd_sof_pci_amd_rn_driver = { .id_table = rn_pci_ids, .probe = acp_pci_rn_probe, .remove = acp_pci_rn_remove, + .driver = { + .pm = &sof_pci_pm, + }, }; module_pci_driver(snd_sof_pci_amd_rn_driver); diff --git a/sound/soc/sof/amd/renoir.c b/sound/soc/sof/amd/renoir.c index 409fd57448..9261c8bc22 100644 --- a/sound/soc/sof/amd/renoir.c +++ b/sound/soc/sof/amd/renoir.c @@ -123,7 +123,7 @@ static struct snd_soc_acpi_mach *amd_sof_machine_select(struct snd_sof_dev *sdev } /* AMD Renoir DSP ops */ -const struct snd_sof_dsp_ops sof_renoir_ops = { +struct snd_sof_dsp_ops sof_renoir_ops = { /* probe and remove */ .probe = amd_sof_acp_probe, .remove = amd_sof_acp_remove, @@ -136,9 +136,6 @@ const struct snd_sof_dsp_ops sof_renoir_ops = { .block_read = acp_dsp_block_read, .block_write = acp_dsp_block_write, - /* Module loading */ - .load_module = snd_sof_parse_module_memcpy, - /*Firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, .pre_fw_run = acp_dsp_pre_fw_run, @@ -152,7 +149,6 @@ const struct snd_sof_dsp_ops sof_renoir_ops = { .ipc_msg_data = acp_sof_ipc_msg_data, .get_mailbox_offset = acp_sof_ipc_get_mailbox_offset, .irq_thread = acp_sof_ipc_irq_thread, - .fw_ready = sof_fw_ready, /* DAI drivers */ .drv = renoir_sof_dai, @@ -177,6 +173,10 @@ const struct snd_sof_dsp_ops sof_renoir_ops = { /* Trace Logger */ .trace_init = acp_sof_trace_init, .trace_release = acp_sof_trace_release, + + /* PM */ + .suspend = amd_sof_acp_suspend, + .resume = amd_sof_acp_resume, }; EXPORT_SYMBOL(sof_renoir_ops); diff --git a/sound/soc/sof/compress.c b/sound/soc/sof/compress.c index a8e908e501..67139e15f8 100644 --- a/sound/soc/sof/compress.c +++ b/sound/soc/sof/compress.c @@ -147,8 +147,7 @@ static int sof_compr_free(struct snd_soc_component *component, stream.comp_id = spcm->stream[cstream->direction].comp_id; if (spcm->prepared[cstream->direction]) { - ret = sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, - &stream, sizeof(stream), + ret = sof_ipc_tx_message(sdev->ipc, &stream, sizeof(stream), &reply, sizeof(reply)); if (!ret) spcm->prepared[cstream->direction] = false; @@ -168,11 +167,23 @@ static int sof_compr_set_params(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd = cstream->private_data; struct snd_compr_runtime *crtd = cstream->runtime; struct sof_ipc_pcm_params_reply ipc_params_reply; + struct sof_ipc_fw_ready *ready = &sdev->fw_ready; + struct sof_ipc_fw_version *v = &ready->version; struct snd_compr_tstamp *tstamp; - struct sof_ipc_pcm_params pcm; + struct sof_ipc_pcm_params *pcm; struct snd_sof_pcm *spcm; + size_t ext_data_size; int ret; + if (v->abi_version < SOF_ABI_VER(3, 22, 0)) { + dev_err(component->dev, + "Compress params not supported with FW ABI version %d:%d:%d\n", + SOF_ABI_VERSION_MAJOR(v->abi_version), + SOF_ABI_VERSION_MINOR(v->abi_version), + SOF_ABI_VERSION_PATCH(v->abi_version)); + return -EINVAL; + } + tstamp = crtd->private_data; spcm = snd_sof_find_spcm_dai(component, rtd); @@ -180,40 +191,50 @@ static int sof_compr_set_params(struct snd_soc_component *component, if (!spcm) return -EINVAL; + ext_data_size = sizeof(params->codec); + + if (sizeof(*pcm) + ext_data_size > sdev->ipc->max_payload_size) + return -EINVAL; + + pcm = kzalloc(sizeof(*pcm) + ext_data_size, GFP_KERNEL); + if (!pcm) + return -ENOMEM; + cstream->dma_buffer.dev.type = SNDRV_DMA_TYPE_DEV_SG; cstream->dma_buffer.dev.dev = sdev->dev; ret = snd_compr_malloc_pages(cstream, crtd->buffer_size); if (ret < 0) - return ret; + goto out; ret = create_page_table(component, cstream, crtd->dma_area, crtd->dma_bytes); if (ret < 0) - return ret; + goto out; - memset(&pcm, 0, sizeof(pcm)); + pcm->params.buffer.pages = PFN_UP(crtd->dma_bytes); + pcm->hdr.size = sizeof(*pcm) + ext_data_size; + pcm->hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_PCM_PARAMS; - pcm.params.buffer.pages = PFN_UP(crtd->dma_bytes); - pcm.hdr.size = sizeof(pcm); - pcm.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_PCM_PARAMS; - - pcm.comp_id = spcm->stream[cstream->direction].comp_id; - pcm.params.hdr.size = sizeof(pcm.params); - pcm.params.buffer.phy_addr = spcm->stream[cstream->direction].page_table.addr; - pcm.params.buffer.size = crtd->dma_bytes; - pcm.params.direction = cstream->direction; - pcm.params.channels = params->codec.ch_out; - pcm.params.rate = params->codec.sample_rate; - pcm.params.buffer_fmt = SOF_IPC_BUFFER_INTERLEAVED; - pcm.params.frame_fmt = SOF_IPC_FRAME_S32_LE; - pcm.params.sample_container_bytes = + pcm->comp_id = spcm->stream[cstream->direction].comp_id; + pcm->params.hdr.size = sizeof(pcm->params) + ext_data_size; + pcm->params.buffer.phy_addr = spcm->stream[cstream->direction].page_table.addr; + pcm->params.buffer.size = crtd->dma_bytes; + pcm->params.direction = cstream->direction; + pcm->params.channels = params->codec.ch_out; + pcm->params.rate = params->codec.sample_rate; + pcm->params.buffer_fmt = SOF_IPC_BUFFER_INTERLEAVED; + pcm->params.frame_fmt = SOF_IPC_FRAME_S32_LE; + pcm->params.sample_container_bytes = snd_pcm_format_physical_width(SNDRV_PCM_FORMAT_S32) >> 3; - pcm.params.host_period_bytes = params->buffer.fragment_size; + pcm->params.host_period_bytes = params->buffer.fragment_size; + pcm->params.ext_data_length = ext_data_size; - ret = sof_ipc_tx_message(sdev->ipc, pcm.hdr.cmd, &pcm, sizeof(pcm), + memcpy((u8 *)pcm->params.ext_data, ¶ms->codec, ext_data_size); + + ret = sof_ipc_tx_message(sdev->ipc, pcm, sizeof(*pcm) + ext_data_size, &ipc_params_reply, sizeof(ipc_params_reply)); if (ret < 0) { dev_err(component->dev, "error ipc failed\n"); - return ret; + goto out; } tstamp->byte_offset = sdev->stream_box.offset + ipc_params_reply.posn_offset; @@ -221,7 +242,10 @@ static int sof_compr_set_params(struct snd_soc_component *component, spcm->prepared[cstream->direction] = true; - return 0; +out: + kfree(pcm); + + return ret; } static int sof_compr_get_params(struct snd_soc_component *component, @@ -268,8 +292,7 @@ static int sof_compr_trigger(struct snd_soc_component *component, break; } - return sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, - &stream, sizeof(stream), + return sof_ipc_tx_message(sdev->ipc, &stream, sizeof(stream), &reply, sizeof(reply)); } diff --git a/sound/soc/sof/control.c b/sound/soc/sof/control.c index de1778c400..e0e9efd25d 100644 --- a/sound/soc/sof/control.c +++ b/sound/soc/sof/control.c @@ -15,36 +15,6 @@ #include "sof-priv.h" #include "sof-audio.h" -static void update_mute_led(struct snd_sof_control *scontrol, - struct snd_kcontrol *kcontrol, - struct snd_ctl_elem_value *ucontrol) -{ - int temp = 0; - int mask; - int i; - - mask = 1U << snd_ctl_get_ioffidx(kcontrol, &ucontrol->id); - - for (i = 0; i < scontrol->num_channels; i++) { - if (ucontrol->value.integer.value[i]) { - temp |= mask; - break; - } - } - - if (temp == scontrol->led_ctl.led_value) - return; - - scontrol->led_ctl.led_value = temp; - -#if IS_REACHABLE(CONFIG_LEDS_TRIGGER_AUDIO) - if (!scontrol->led_ctl.direction) - ledtrig_audio_set(LED_AUDIO_MUTE, temp ? LED_OFF : LED_ON); - else - ledtrig_audio_set(LED_AUDIO_MICMUTE, temp ? LED_OFF : LED_ON); -#endif -} - int snd_sof_volume_get(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_value *ucontrol) { @@ -121,9 +91,6 @@ int snd_sof_switch_put(struct snd_kcontrol *kcontrol, struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; - if (scontrol->led_ctl.use_led) - update_mute_led(scontrol, kcontrol, ucontrol); - if (tplg_ops->control->switch_put) return tplg_ops->control->switch_put(scontrol, ucontrol); @@ -220,10 +187,9 @@ int snd_sof_bytes_ext_volatile_get(struct snd_kcontrol *kcontrol, unsigned int _ const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; int ret, err; - ret = pm_runtime_get_sync(scomp->dev); + ret = pm_runtime_resume_and_get(scomp->dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(scomp->dev, "%s: failed to resume %d\n", __func__, ret); - pm_runtime_put_noidle(scomp->dev); return ret; } diff --git a/sound/soc/sof/core.c b/sound/soc/sof/core.c index e91631618b..c99b5e6c02 100644 --- a/sound/soc/sof/core.c +++ b/sound/soc/sof/core.c @@ -189,7 +189,7 @@ static int sof_probe_continue(struct snd_sof_dev *sdev) ret = snd_sof_probe(sdev); if (ret < 0) { dev_err(sdev->dev, "error: failed to probe DSP %d\n", ret); - return ret; + goto probe_err; } sof_set_fw_state(sdev, SOF_FW_BOOT_PREPARE); @@ -250,14 +250,13 @@ static int sof_probe_continue(struct snd_sof_dev *sdev) } if (sof_debug_check_flag(SOF_DBG_ENABLE_TRACE)) { - sdev->dtrace_is_supported = true; + sdev->fw_trace_is_supported = true; - /* init DMA trace */ - ret = snd_sof_init_trace(sdev); + /* init firmware tracing */ + ret = sof_fw_trace_init(sdev); if (ret < 0) { /* non fatal */ - dev_warn(sdev->dev, - "warning: failed to initialize trace %d\n", + dev_warn(sdev->dev, "failed to initialize firmware tracing %d\n", ret); } } else { @@ -308,7 +307,7 @@ static int sof_probe_continue(struct snd_sof_dev *sdev) sof_machine_err: snd_sof_machine_unregister(sdev, plat_data); fw_trace_err: - snd_sof_free_trace(sdev); + sof_fw_trace_free(sdev); fw_run_err: snd_sof_fw_unload(sdev); fw_load_err: @@ -318,6 +317,8 @@ static int sof_probe_continue(struct snd_sof_dev *sdev) snd_sof_free_debug(sdev); dsp_err: snd_sof_remove(sdev); +probe_err: + sof_ops_free(sdev); /* all resources freed, update state to match */ sof_set_fw_state(sdev, SOF_FW_BOOT_NOT_STARTED); @@ -342,6 +343,7 @@ static void sof_probe_work(struct work_struct *work) int snd_sof_device_probe(struct device *dev, struct snd_sof_pdata *plat_data) { struct snd_sof_dev *sdev; + int ret; sdev = devm_kzalloc(dev, sizeof(*sdev), GFP_KERNEL); if (!sdev) @@ -357,11 +359,24 @@ int snd_sof_device_probe(struct device *dev, struct snd_sof_pdata *plat_data) sdev->first_boot = true; dev_set_drvdata(dev, sdev); + /* check IPC support */ + if (!(BIT(plat_data->ipc_type) & plat_data->desc->ipc_supported_mask)) { + dev_err(dev, "ipc_type %d is not supported on this platform, mask is %#x\n", + plat_data->ipc_type, plat_data->desc->ipc_supported_mask); + return -EINVAL; + } + + /* init ops, if necessary */ + ret = sof_ops_init(sdev); + if (ret < 0) + return ret; + /* check all mandatory ops */ if (!sof_ops(sdev) || !sof_ops(sdev)->probe || !sof_ops(sdev)->run || !sof_ops(sdev)->block_read || !sof_ops(sdev)->block_write || !sof_ops(sdev)->send_msg || !sof_ops(sdev)->load_firmware || - !sof_ops(sdev)->ipc_msg_data || !sof_ops(sdev)->fw_ready) { + !sof_ops(sdev)->ipc_msg_data) { + sof_ops_free(sdev); dev_err(dev, "error: missing mandatory ops\n"); return -EINVAL; } @@ -434,7 +449,7 @@ int snd_sof_device_remove(struct device *dev) snd_sof_machine_unregister(sdev, pdata); if (sdev->fw_state > SOF_FW_BOOT_NOT_STARTED) { - snd_sof_free_trace(sdev); + sof_fw_trace_free(sdev); ret = snd_sof_dsp_power_down_notify(sdev); if (ret < 0) dev_warn(dev, "error: %d failed to prepare DSP for device removal", @@ -445,6 +460,8 @@ int snd_sof_device_remove(struct device *dev) snd_sof_remove(sdev); } + sof_ops_free(sdev); + /* release firmware */ snd_sof_fw_unload(sdev); diff --git a/sound/soc/sof/debug.c b/sound/soc/sof/debug.c index 7b1139961a..d9a3ce7b69 100644 --- a/sound/soc/sof/debug.c +++ b/sound/soc/sof/debug.c @@ -229,14 +229,13 @@ static int memory_info_update(struct snd_sof_dev *sdev, char *buf, size_t buff_s if (!reply) return -ENOMEM; - ret = pm_runtime_get_sync(sdev->dev); + ret = pm_runtime_resume_and_get(sdev->dev); if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(sdev->dev); dev_err(sdev->dev, "error: enabling device failed: %d\n", ret); goto error; } - ret = sof_ipc_tx_message(sdev->ipc, msg.cmd, &msg, msg.size, reply, SOF_IPC_MSG_MAX_SIZE); + ret = sof_ipc_tx_message(sdev->ipc, &msg, msg.size, reply, SOF_IPC_MSG_MAX_SIZE); pm_runtime_mark_last_busy(sdev->dev); pm_runtime_put_autosuspend(sdev->dev); if (ret < 0 || reply->rhdr.error < 0) { @@ -253,9 +252,9 @@ static int memory_info_update(struct snd_sof_dev *sdev, char *buf, size_t buff_s } for (i = 0, len = 0; i < reply->num_elems; i++) { - ret = snprintf(buf + len, buff_size - len, "zone %d.%d used %#8x free %#8x\n", - reply->elems[i].zone, reply->elems[i].id, - reply->elems[i].used, reply->elems[i].free); + ret = scnprintf(buf + len, buff_size - len, "zone %d.%d used %#8x free %#8x\n", + reply->elems[i].zone, reply->elems[i].id, + reply->elems[i].used, reply->elems[i].free); if (ret < 0) goto error; len += ret; @@ -331,7 +330,7 @@ EXPORT_SYMBOL_GPL(snd_sof_dbg_memory_info_init); int snd_sof_dbg_init(struct snd_sof_dev *sdev) { - const struct snd_sof_dsp_ops *ops = sof_ops(sdev); + struct snd_sof_dsp_ops *ops = sof_ops(sdev); const struct snd_sof_debugfs_map *map; int i; int err; @@ -429,7 +428,7 @@ static void snd_sof_ipc_dump(struct snd_sof_dev *sdev) } } -void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev) +void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev, const char *msg) { if (IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_RETAIN_DSP_CONTEXT) || sof_debug_check_flag(SOF_DBG_RETAIN_CTX)) { @@ -442,8 +441,7 @@ void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev) /* dump vital information to the logs */ snd_sof_ipc_dump(sdev); - snd_sof_dsp_dbg_dump(sdev, "Firmware exception", - SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX); - snd_sof_trace_notify_for_error(sdev); + snd_sof_dsp_dbg_dump(sdev, msg, SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX); + sof_fw_trace_fw_crashed(sdev); } EXPORT_SYMBOL(snd_sof_handle_fw_exception); diff --git a/sound/soc/sof/imx/Kconfig b/sound/soc/sof/imx/Kconfig index 9b8d5bb1e4..cc6e695f91 100644 --- a/sound/soc/sof/imx/Kconfig +++ b/sound/soc/sof/imx/Kconfig @@ -15,6 +15,7 @@ config SND_SOC_SOF_IMX_COMMON tristate select SND_SOC_SOF_OF_DEV select SND_SOC_SOF + select SND_SOC_SOF_IPC3 select SND_SOC_SOF_XTENSA select SND_SOC_SOF_COMPRESS help diff --git a/sound/soc/sof/imx/imx8.c b/sound/soc/sof/imx/imx8.c index 825bd2b9b7..2844d9a804 100644 --- a/sound/soc/sof/imx/imx8.c +++ b/sound/soc/sof/imx/imx8.c @@ -487,7 +487,7 @@ static int imx8_dsp_set_power_state(struct snd_sof_dev *sdev, } /* i.MX8 ops */ -static const struct snd_sof_dsp_ops sof_imx8_ops = { +static struct snd_sof_dsp_ops sof_imx8_ops = { /* probe and remove */ .probe = imx8_probe, .remove = imx8_remove, @@ -504,16 +504,14 @@ static const struct snd_sof_dsp_ops sof_imx8_ops = { /* ipc */ .send_msg = imx8_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = imx8_get_mailbox_offset, .get_window_offset = imx8_get_window_offset, .ipc_msg_data = sof_ipc_msg_data, .set_stream_data_offset = sof_set_stream_data_offset, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, .get_bar_index = imx8_get_bar_index, + /* firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -550,7 +548,7 @@ static const struct snd_sof_dsp_ops sof_imx8_ops = { }; /* i.MX8X ops */ -static const struct snd_sof_dsp_ops sof_imx8x_ops = { +static struct snd_sof_dsp_ops sof_imx8x_ops = { /* probe and remove */ .probe = imx8_probe, .remove = imx8_remove, @@ -567,16 +565,14 @@ static const struct snd_sof_dsp_ops sof_imx8x_ops = { /* ipc */ .send_msg = imx8_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = imx8_get_mailbox_offset, .get_window_offset = imx8_get_window_offset, .ipc_msg_data = sof_ipc_msg_data, .set_stream_data_offset = sof_set_stream_data_offset, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, .get_bar_index = imx8_get_bar_index, + /* firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -613,17 +609,33 @@ static const struct snd_sof_dsp_ops sof_imx8x_ops = { }; static struct sof_dev_desc sof_of_imx8qxp_desc = { - .default_fw_path = "imx/sof", - .default_tplg_path = "imx/sof-tplg", - .default_fw_filename = "sof-imx8x.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "imx/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "imx/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-imx8x.ri", + }, .nocodec_tplg_filename = "sof-imx8-nocodec.tplg", .ops = &sof_imx8x_ops, }; static struct sof_dev_desc sof_of_imx8qm_desc = { - .default_fw_path = "imx/sof", - .default_tplg_path = "imx/sof-tplg", - .default_fw_filename = "sof-imx8.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "imx/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "imx/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-imx8.ri", + }, .nocodec_tplg_filename = "sof-imx8-nocodec.tplg", .ops = &sof_imx8_ops, }; diff --git a/sound/soc/sof/imx/imx8m.c b/sound/soc/sof/imx/imx8m.c index 803d6be6b4..1243f8a614 100644 --- a/sound/soc/sof/imx/imx8m.c +++ b/sound/soc/sof/imx/imx8m.c @@ -412,7 +412,7 @@ static int imx8m_dsp_suspend(struct snd_sof_dev *sdev, unsigned int target_state } /* i.MX8 ops */ -static const struct snd_sof_dsp_ops sof_imx8m_ops = { +static struct snd_sof_dsp_ops sof_imx8m_ops = { /* probe and remove */ .probe = imx8m_probe, .remove = imx8m_remove, @@ -430,16 +430,14 @@ static const struct snd_sof_dsp_ops sof_imx8m_ops = { /* ipc */ .send_msg = imx8m_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = imx8m_get_mailbox_offset, .get_window_offset = imx8m_get_window_offset, .ipc_msg_data = sof_ipc_msg_data, .set_stream_data_offset = sof_set_stream_data_offset, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, .get_bar_index = imx8m_get_bar_index, + /* firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -473,9 +471,17 @@ static const struct snd_sof_dsp_ops sof_imx8m_ops = { }; static struct sof_dev_desc sof_of_imx8mp_desc = { - .default_fw_path = "imx/sof", - .default_tplg_path = "imx/sof-tplg", - .default_fw_filename = "sof-imx8m.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "imx/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "imx/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-imx8m.ri", + }, .nocodec_tplg_filename = "sof-imx8-nocodec.tplg", .ops = &sof_imx8m_ops, }; diff --git a/sound/soc/sof/intel/Kconfig b/sound/soc/sof/intel/Kconfig index 172419392b..3f54678e81 100644 --- a/sound/soc/sof/intel/Kconfig +++ b/sound/soc/sof/intel/Kconfig @@ -40,6 +40,7 @@ if SND_SOC_SOF_ACPI config SND_SOC_SOF_BAYTRAIL tristate "SOF support for Baytrail, Braswell and Cherrytrail" default SND_SOC_SOF_ACPI + select SND_SOC_SOF_IPC3 select SND_SOC_SOF_INTEL_COMMON select SND_SOC_SOF_INTEL_ATOM_HIFI_EP select SND_SOC_SOF_ACPI_DEV @@ -60,6 +61,7 @@ config SND_SOC_SOF_BAYTRAIL config SND_SOC_SOF_BROADWELL tristate "SOF support for Broadwell" default SND_SOC_SOF_ACPI + select SND_SOC_SOF_IPC3 select SND_SOC_SOF_INTEL_COMMON select SND_SOC_SOF_INTEL_HIFI_EP_IPC select SND_SOC_SOF_ACPI_DEV @@ -85,6 +87,7 @@ config SND_SOC_SOF_MERRIFIELD tristate "SOF support for Tangier/Merrifield" default SND_SOC_SOF_PCI select SND_SOC_SOF_PCI_DEV + select SND_SOC_SOF_IPC3 select SND_SOC_SOF_INTEL_ATOM_HIFI_EP help This adds support for Sound Open Firmware for Intel(R) platforms @@ -95,6 +98,8 @@ config SND_SOC_SOF_MERRIFIELD config SND_SOC_SOF_INTEL_APL tristate select SND_SOC_SOF_HDA_COMMON + select SND_SOC_SOF_IPC3 + select SND_SOC_SOF_INTEL_IPC4 config SND_SOC_SOF_APOLLOLAKE tristate "SOF support for Apollolake" @@ -120,6 +125,8 @@ config SND_SOC_SOF_INTEL_CNL tristate select SND_SOC_SOF_HDA_COMMON select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE + select SND_SOC_SOF_IPC3 + select SND_SOC_SOF_INTEL_IPC4 config SND_SOC_SOF_CANNONLAKE tristate "SOF support for Cannonlake" @@ -154,6 +161,8 @@ config SND_SOC_SOF_INTEL_ICL tristate select SND_SOC_SOF_HDA_COMMON select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE + select SND_SOC_SOF_IPC3 + select SND_SOC_SOF_INTEL_IPC4 config SND_SOC_SOF_ICELAKE tristate "SOF support for Icelake" @@ -179,6 +188,8 @@ config SND_SOC_SOF_INTEL_TGL tristate select SND_SOC_SOF_HDA_COMMON select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE + select SND_SOC_SOF_IPC3 + select SND_SOC_SOF_INTEL_IPC4 config SND_SOC_SOF_TIGERLAKE tristate "SOF support for Tigerlake" @@ -210,6 +221,22 @@ config SND_SOC_SOF_ALDERLAKE Say Y if you have such a device. If unsure select "N". +config SND_SOC_SOF_INTEL_MTL + tristate + select SND_SOC_SOF_HDA_COMMON + select SND_SOC_SOF_INTEL_SOUNDWIRE_LINK_BASELINE + select SND_SOC_SOF_INTEL_IPC4 + +config SND_SOC_SOF_METEORLAKE + tristate "SOF support for Meteorlake" + default SND_SOC_SOF_PCI + select SND_SOC_SOF_INTEL_MTL + help + This adds support for Sound Open Firmware for Intel(R) platforms + using the Meteorlake processors. + Say Y if you have such a device. + If unsure select "N". + config SND_SOC_SOF_HDA_COMMON tristate select SND_SOC_SOF_INTEL_COMMON @@ -260,7 +287,7 @@ config SND_SOC_SOF_HDA 'select' statements at a higher level. config SND_SOC_SOF_HDA_PROBES - bool + tristate select SND_SOC_SOF_DEBUG_PROBES help The option enables the data probing for Intel(R) Skylake and newer diff --git a/sound/soc/sof/intel/Makefile b/sound/soc/sof/intel/Makefile index 1f473d4d84..a079159bb2 100644 --- a/sound/soc/sof/intel/Makefile +++ b/sound/soc/sof/intel/Makefile @@ -6,7 +6,7 @@ snd-sof-acpi-intel-bdw-objs := bdw.o snd-sof-intel-hda-common-objs := hda.o hda-loader.o hda-stream.o hda-trace.o \ hda-dsp.o hda-ipc.o hda-ctrl.o hda-pcm.o \ hda-dai.o hda-bus.o \ - apl.o cnl.o tgl.o icl.o + apl.o cnl.o tgl.o icl.o mtl.o hda-common-ops.o snd-sof-intel-hda-common-$(CONFIG_SND_SOC_SOF_HDA_PROBES) += hda-probes.o snd-sof-intel-hda-objs := hda-codec.o @@ -24,9 +24,11 @@ snd-sof-pci-intel-apl-objs := pci-apl.o snd-sof-pci-intel-cnl-objs := pci-cnl.o snd-sof-pci-intel-icl-objs := pci-icl.o snd-sof-pci-intel-tgl-objs := pci-tgl.o +snd-sof-pci-intel-mtl-objs := pci-mtl.o obj-$(CONFIG_SND_SOC_SOF_MERRIFIELD) += snd-sof-pci-intel-tng.o obj-$(CONFIG_SND_SOC_SOF_INTEL_APL) += snd-sof-pci-intel-apl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_CNL) += snd-sof-pci-intel-cnl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_ICL) += snd-sof-pci-intel-icl.o obj-$(CONFIG_SND_SOC_SOF_INTEL_TGL) += snd-sof-pci-intel-tgl.o +obj-$(CONFIG_SND_SOC_SOF_INTEL_MTL) += snd-sof-pci-intel-mtl.o diff --git a/sound/soc/sof/intel/apl.c b/sound/soc/sof/intel/apl.c index 6721c8f951..084c245a95 100644 --- a/sound/soc/sof/intel/apl.c +++ b/sound/soc/sof/intel/apl.c @@ -15,6 +15,8 @@ * Hardware interface for audio DSP on Apollolake and GeminiLake */ +#include +#include "../ipc4-priv.h" #include "../sof-priv.h" #include "hda.h" #include "../sof-audio.h" @@ -26,108 +28,62 @@ static const struct snd_sof_debugfs_map apl_dsp_debugfs[] = { }; /* apollolake ops */ -const struct snd_sof_dsp_ops sof_apl_ops = { +struct snd_sof_dsp_ops sof_apl_ops; +EXPORT_SYMBOL_NS(sof_apl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); + +int sof_apl_ops_init(struct snd_sof_dev *sdev) +{ + /* common defaults */ + memcpy(&sof_apl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops)); + /* probe/remove/shutdown */ - .probe = hda_dsp_probe, - .remove = hda_dsp_remove, - .shutdown = hda_dsp_shutdown, + sof_apl_ops.shutdown = hda_dsp_shutdown; - /* Register IO */ - .write = sof_io_write, - .read = sof_io_read, - .write64 = sof_io_write64, - .read64 = sof_io_read64, + if (sdev->pdata->ipc_type == SOF_IPC) { + /* doorbell */ + sof_apl_ops.irq_thread = hda_dsp_ipc_irq_thread; - /* Block IO */ - .block_read = sof_block_read, - .block_write = sof_block_write, + /* ipc */ + sof_apl_ops.send_msg = hda_dsp_ipc_send_msg; + } - /* Mailbox IO */ - .mailbox_read = sof_mailbox_read, - .mailbox_write = sof_mailbox_write, + if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_fw_data *ipc4_data; - /* doorbell */ - .irq_thread = hda_dsp_ipc_irq_thread, + sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL); + if (!sdev->private) + return -ENOMEM; - /* ipc */ - .send_msg = hda_dsp_ipc_send_msg, - .fw_ready = sof_fw_ready, - .get_mailbox_offset = hda_dsp_ipc_get_mailbox_offset, - .get_window_offset = hda_dsp_ipc_get_window_offset, + ipc4_data = sdev->private; + ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; - .ipc_msg_data = hda_ipc_msg_data, - .set_stream_data_offset = hda_set_stream_data_offset, + /* doorbell */ + sof_apl_ops.irq_thread = hda_dsp_ipc4_irq_thread; - /* machine driver */ - .machine_select = hda_machine_select, - .machine_register = sof_machine_register, - .machine_unregister = sof_machine_unregister, - .set_mach_params = hda_set_mach_params, + /* ipc */ + sof_apl_ops.send_msg = hda_dsp_ipc4_send_msg; + } + + /* set DAI driver ops */ + hda_set_dai_drv_ops(sdev, &sof_apl_ops); /* debug */ - .debug_map = apl_dsp_debugfs, - .debug_map_count = ARRAY_SIZE(apl_dsp_debugfs), - .dbg_dump = hda_dsp_dump, - .ipc_dump = hda_ipc_dump, - .debugfs_add_region_item = snd_sof_debugfs_add_region_item_iomem, - - /* stream callbacks */ - .pcm_open = hda_dsp_pcm_open, - .pcm_close = hda_dsp_pcm_close, - .pcm_hw_params = hda_dsp_pcm_hw_params, - .pcm_hw_free = hda_dsp_stream_hw_free, - .pcm_trigger = hda_dsp_pcm_trigger, - .pcm_pointer = hda_dsp_pcm_pointer, - .pcm_ack = hda_dsp_pcm_ack, - - /* firmware loading */ - .load_firmware = snd_sof_load_firmware_raw, + sof_apl_ops.debug_map = apl_dsp_debugfs; + sof_apl_ops.debug_map_count = ARRAY_SIZE(apl_dsp_debugfs); + sof_apl_ops.ipc_dump = hda_ipc_dump; /* firmware run */ - .run = hda_dsp_cl_boot_firmware, + sof_apl_ops.run = hda_dsp_cl_boot_firmware; /* pre/post fw run */ - .pre_fw_run = hda_dsp_pre_fw_run, - .post_fw_run = hda_dsp_post_fw_run, - - /* parse platform specific extended manifest */ - .parse_platform_ext_manifest = hda_dsp_ext_man_get_cavs_config_data, + sof_apl_ops.post_fw_run = hda_dsp_post_fw_run; /* dsp core get/put */ - .core_get = hda_dsp_core_get, + sof_apl_ops.core_get = hda_dsp_core_get; - /* trace callback */ - .trace_init = hda_dsp_trace_init, - .trace_release = hda_dsp_trace_release, - .trace_trigger = hda_dsp_trace_trigger, - - /* client ops */ - .register_ipc_clients = hda_register_clients, - .unregister_ipc_clients = hda_unregister_clients, - - /* DAI drivers */ - .drv = skl_dai, - .num_drv = SOF_SKL_NUM_DAIS, - - /* PM */ - .suspend = hda_dsp_suspend, - .resume = hda_dsp_resume, - .runtime_suspend = hda_dsp_runtime_suspend, - .runtime_resume = hda_dsp_runtime_resume, - .runtime_idle = hda_dsp_runtime_idle, - .set_hw_params_upon_resume = hda_dsp_set_hw_params_upon_resume, - .set_power_state = hda_dsp_set_power_state, - - /* ALSA HW info flags */ - .hw_info = SNDRV_PCM_INFO_MMAP | - SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED | - SNDRV_PCM_INFO_PAUSE | - SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, - - .dsp_arch_ops = &sof_xtensa_arch_ops, + return 0; }; -EXPORT_SYMBOL_NS(sof_apl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); +EXPORT_SYMBOL_NS(sof_apl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON); const struct sof_intel_dsp_desc apl_chip_info = { /* Apollolake */ @@ -139,9 +95,13 @@ const struct sof_intel_dsp_desc apl_chip_info = { .ipc_ack = HDA_DSP_REG_HIPCIE, .ipc_ack_mask = HDA_DSP_REG_HIPCIE_DONE, .ipc_ctl = HDA_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 150, .ssp_count = APL_SSP_COUNT, .ssp_base_offset = APL_SSP_BASE_OFFSET, .quirks = SOF_INTEL_PROCEN_FMT_QUIRK, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_1_5_PLUS, }; EXPORT_SYMBOL_NS(apl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/atom.c b/sound/soc/sof/intel/atom.c index ff5900b155..bd9789b483 100644 --- a/sound/soc/sof/intel/atom.c +++ b/sound/soc/sof/intel/atom.c @@ -274,22 +274,22 @@ static const char *fixup_tplg_name(struct snd_sof_dev *sdev, const char *ssp_str) { const char *tplg_filename = NULL; - char *filename; - char *split_ext; + const char *split_ext; + char *filename, *tmp; - filename = devm_kstrdup(sdev->dev, sof_tplg_filename, GFP_KERNEL); + filename = kstrdup(sof_tplg_filename, GFP_KERNEL); if (!filename) return NULL; /* this assumes a .tplg extension */ - split_ext = strsep(&filename, "."); - if (split_ext) { + tmp = filename; + split_ext = strsep(&tmp, "."); + if (split_ext) tplg_filename = devm_kasprintf(sdev->dev, GFP_KERNEL, "%s-%s.tplg", split_ext, ssp_str); - if (!tplg_filename) - return NULL; - } + kfree(filename); + return tplg_filename; } diff --git a/sound/soc/sof/intel/bdw.c b/sound/soc/sof/intel/bdw.c index fb9682b2fe..a446154f28 100644 --- a/sound/soc/sof/intel/bdw.c +++ b/sound/soc/sof/intel/bdw.c @@ -567,7 +567,7 @@ static struct snd_soc_dai_driver bdw_dai[] = { }; /* broadwell ops */ -static const struct snd_sof_dsp_ops sof_bdw_ops = { +static struct snd_sof_dsp_ops sof_bdw_ops = { /*Device init */ .probe = bdw_probe, @@ -591,7 +591,6 @@ static const struct snd_sof_dsp_ops sof_bdw_ops = { /* ipc */ .send_msg = bdw_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = bdw_get_mailbox_offset, .get_window_offset = bdw_get_window_offset, @@ -614,9 +613,6 @@ static const struct snd_sof_dsp_ops sof_bdw_ops = { .pcm_open = sof_stream_pcm_open, .pcm_close = sof_stream_pcm_close, - /* Module loading */ - .load_module = snd_sof_parse_module_memcpy, - /*Firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -637,6 +633,7 @@ static const struct snd_sof_dsp_ops sof_bdw_ops = { static const struct sof_intel_dsp_desc bdw_chip_info = { .cores_num = 1, .host_managed_cores_mask = 1, + .hw_ip_version = SOF_INTEL_BROADWELL, }; static const struct sof_dev_desc sof_acpi_broadwell_desc = { @@ -646,9 +643,17 @@ static const struct sof_dev_desc sof_acpi_broadwell_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = 0, .chip_info = &bdw_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-bdw.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-bdw.ri", + }, .nocodec_tplg_filename = "sof-bdw-nocodec.tplg", .ops = &sof_bdw_ops, }; @@ -676,11 +681,8 @@ static int sof_broadwell_probe(struct platform_device *pdev) return -ENODEV; } - desc = device_get_match_data(dev); - if (!desc) - return -ENODEV; - - return sof_acpi_probe(pdev, device_get_match_data(dev)); + desc = (const struct sof_dev_desc *)id->driver_data; + return sof_acpi_probe(pdev, desc); } /* acpi_driver definition */ diff --git a/sound/soc/sof/intel/byt.c b/sound/soc/sof/intel/byt.c index bb84a4aa58..e6dc4ff531 100644 --- a/sound/soc/sof/intel/byt.c +++ b/sound/soc/sof/intel/byt.c @@ -216,7 +216,7 @@ static int byt_acpi_probe(struct snd_sof_dev *sdev) } /* baytrail ops */ -static const struct snd_sof_dsp_ops sof_byt_ops = { +static struct snd_sof_dsp_ops sof_byt_ops = { /* device init */ .probe = byt_acpi_probe, .remove = byt_remove, @@ -245,7 +245,6 @@ static const struct snd_sof_dsp_ops sof_byt_ops = { /* ipc */ .send_msg = atom_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = atom_get_mailbox_offset, .get_window_offset = atom_get_window_offset, @@ -268,9 +267,6 @@ static const struct snd_sof_dsp_ops sof_byt_ops = { .pcm_open = sof_stream_pcm_open, .pcm_close = sof_stream_pcm_close, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, - /*Firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -295,10 +291,11 @@ static const struct snd_sof_dsp_ops sof_byt_ops = { static const struct sof_intel_dsp_desc byt_chip_info = { .cores_num = 1, .host_managed_cores_mask = 1, + .hw_ip_version = SOF_INTEL_BAYTRAIL, }; /* cherrytrail and braswell ops */ -static const struct snd_sof_dsp_ops sof_cht_ops = { +static struct snd_sof_dsp_ops sof_cht_ops = { /* device init */ .probe = byt_acpi_probe, .remove = byt_remove, @@ -327,7 +324,6 @@ static const struct snd_sof_dsp_ops sof_cht_ops = { /* ipc */ .send_msg = atom_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = atom_get_mailbox_offset, .get_window_offset = atom_get_window_offset, @@ -350,9 +346,6 @@ static const struct snd_sof_dsp_ops sof_cht_ops = { .pcm_open = sof_stream_pcm_open, .pcm_close = sof_stream_pcm_close, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, - /*Firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -378,6 +371,7 @@ static const struct snd_sof_dsp_ops sof_cht_ops = { static const struct sof_intel_dsp_desc cht_chip_info = { .cores_num = 1, .host_managed_cores_mask = 1, + .hw_ip_version = SOF_INTEL_BAYTRAIL, }; /* BYTCR uses different IRQ index */ @@ -388,9 +382,17 @@ static const struct sof_dev_desc sof_acpi_baytrailcr_desc = { .resindex_imr_base = 2, .irqindex_host_ipc = 0, .chip_info = &byt_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-byt.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-byt.ri", + }, .nocodec_tplg_filename = "sof-byt-nocodec.tplg", .ops = &sof_byt_ops, }; @@ -402,9 +404,17 @@ static const struct sof_dev_desc sof_acpi_baytrail_desc = { .resindex_imr_base = 2, .irqindex_host_ipc = 5, .chip_info = &byt_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-byt.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-byt.ri", + }, .nocodec_tplg_filename = "sof-byt-nocodec.tplg", .ops = &sof_byt_ops, }; @@ -416,9 +426,17 @@ static const struct sof_dev_desc sof_acpi_cherrytrail_desc = { .resindex_imr_base = 2, .irqindex_host_ipc = 5, .chip_info = &cht_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-cht.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-cht.ri", + }, .nocodec_tplg_filename = "sof-cht-nocodec.tplg", .ops = &sof_cht_ops, }; @@ -447,10 +465,7 @@ static int sof_baytrail_probe(struct platform_device *pdev) return -ENODEV; } - desc = device_get_match_data(&pdev->dev); - if (!desc) - return -ENODEV; - + desc = (const struct sof_dev_desc *)id->driver_data; if (desc == &sof_acpi_baytrail_desc && soc_intel_is_byt_cr(pdev)) desc = &sof_acpi_baytrailcr_desc; diff --git a/sound/soc/sof/intel/cnl.c b/sound/soc/sof/intel/cnl.c index 6a96470b96..a064453f0b 100644 --- a/sound/soc/sof/intel/cnl.c +++ b/sound/soc/sof/intel/cnl.c @@ -15,6 +15,9 @@ * Hardware interface for audio DSP on Cannonlake. */ +#include +#include +#include "../ipc4-priv.h" #include "../ops.h" #include "hda.h" #include "hda-ipc.h" @@ -29,6 +32,74 @@ static const struct snd_sof_debugfs_map cnl_dsp_debugfs[] = { static void cnl_ipc_host_done(struct snd_sof_dev *sdev); static void cnl_ipc_dsp_done(struct snd_sof_dev *sdev); +irqreturn_t cnl_ipc4_irq_thread(int irq, void *context) +{ + struct sof_ipc4_msg notification_data = {{ 0 }}; + struct snd_sof_dev *sdev = context; + bool ipc_irq = false; + u32 hipcida, hipctdr; + + hipcida = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDA); + if (hipcida & CNL_DSP_REG_HIPCIDA_DONE) { + /* DSP received the message */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, + CNL_DSP_REG_HIPCCTL, + CNL_DSP_REG_HIPCCTL_DONE, 0); + cnl_ipc_dsp_done(sdev); + + ipc_irq = true; + } + + hipctdr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCTDR); + if (hipctdr & CNL_DSP_REG_HIPCTDR_BUSY) { + /* Message from DSP (reply or notification) */ + u32 hipctdd = snd_sof_dsp_read(sdev, HDA_DSP_BAR, + CNL_DSP_REG_HIPCTDD); + u32 primary = hipctdr & CNL_DSP_REG_HIPCTDR_MSG_MASK; + u32 extension = hipctdd & CNL_DSP_REG_HIPCTDD_MSG_MASK; + + if (primary & SOF_IPC4_MSG_DIR_MASK) { + /* Reply received */ + if (likely(sdev->fw_state == SOF_FW_BOOT_COMPLETE)) { + struct sof_ipc4_msg *data = sdev->ipc->msg.reply_data; + + data->primary = primary; + data->extension = extension; + + spin_lock_irq(&sdev->ipc_lock); + + snd_sof_ipc_get_reply(sdev); + snd_sof_ipc_reply(sdev, data->primary); + + spin_unlock_irq(&sdev->ipc_lock); + } else { + dev_dbg_ratelimited(sdev->dev, + "IPC reply before FW_READY: %#x|%#x\n", + primary, extension); + } + } else { + /* Notification received */ + notification_data.primary = primary; + notification_data.extension = extension; + + sdev->ipc->msg.rx_data = ¬ification_data; + snd_sof_ipc_msgs_rx(sdev); + sdev->ipc->msg.rx_data = NULL; + } + + /* Let DSP know that we have finished processing the message */ + cnl_ipc_host_done(sdev); + + ipc_irq = true; + } + + if (!ipc_irq) + /* This interrupt is not shared so no need to return IRQ_NONE. */ + dev_dbg_ratelimited(sdev->dev, "nothing to do in IPC IRQ thread\n"); + + return IRQ_HANDLED; +} + irqreturn_t cnl_ipc_irq_thread(int irq, void *context) { struct snd_sof_dev *sdev = context; @@ -59,15 +130,20 @@ irqreturn_t cnl_ipc_irq_thread(int irq, void *context) CNL_DSP_REG_HIPCCTL, CNL_DSP_REG_HIPCCTL_DONE, 0); - spin_lock_irq(&sdev->ipc_lock); + if (likely(sdev->fw_state == SOF_FW_BOOT_COMPLETE)) { + spin_lock_irq(&sdev->ipc_lock); - /* handle immediate reply from DSP core */ - hda_dsp_ipc_get_reply(sdev); - snd_sof_ipc_reply(sdev, msg); + /* handle immediate reply from DSP core */ + hda_dsp_ipc_get_reply(sdev); + snd_sof_ipc_reply(sdev, msg); - cnl_ipc_dsp_done(sdev); + cnl_ipc_dsp_done(sdev); - spin_unlock_irq(&sdev->ipc_lock); + spin_unlock_irq(&sdev->ipc_lock); + } else { + dev_dbg_ratelimited(sdev->dev, "IPC reply before FW_READY: %#x\n", + msg); + } ipc_irq = true; } @@ -176,6 +252,22 @@ static bool cnl_compact_ipc_compress(struct snd_sof_ipc_msg *msg, return false; } +int cnl_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) +{ + struct sof_ipc4_msg *msg_data = msg->msg_data; + + /* send the message via mailbox */ + if (msg_data->data_size) + sof_mailbox_write(sdev, sdev->host_box.offset, msg_data->data_ptr, + msg_data->data_size); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDD, msg_data->extension); + snd_sof_dsp_write(sdev, HDA_DSP_BAR, CNL_DSP_REG_HIPCIDR, + msg_data->primary | CNL_DSP_REG_HIPCIDR_BUSY); + + return 0; +} + int cnl_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) { struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; @@ -244,108 +336,63 @@ void cnl_ipc_dump(struct snd_sof_dev *sdev) } /* cannonlake ops */ -const struct snd_sof_dsp_ops sof_cnl_ops = { +struct snd_sof_dsp_ops sof_cnl_ops; +EXPORT_SYMBOL_NS(sof_cnl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); + +int sof_cnl_ops_init(struct snd_sof_dev *sdev) +{ + /* common defaults */ + memcpy(&sof_cnl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops)); + /* probe/remove/shutdown */ - .probe = hda_dsp_probe, - .remove = hda_dsp_remove, - .shutdown = hda_dsp_shutdown, - - /* Register IO */ - .write = sof_io_write, - .read = sof_io_read, - .write64 = sof_io_write64, - .read64 = sof_io_read64, - - /* Block IO */ - .block_read = sof_block_read, - .block_write = sof_block_write, - - /* Mailbox IO */ - .mailbox_read = sof_mailbox_read, - .mailbox_write = sof_mailbox_write, - - /* doorbell */ - .irq_thread = cnl_ipc_irq_thread, + sof_cnl_ops.shutdown = hda_dsp_shutdown; /* ipc */ - .send_msg = cnl_ipc_send_msg, - .fw_ready = sof_fw_ready, - .get_mailbox_offset = hda_dsp_ipc_get_mailbox_offset, - .get_window_offset = hda_dsp_ipc_get_window_offset, + if (sdev->pdata->ipc_type == SOF_IPC) { + /* doorbell */ + sof_cnl_ops.irq_thread = cnl_ipc_irq_thread; - .ipc_msg_data = hda_ipc_msg_data, - .set_stream_data_offset = hda_set_stream_data_offset, + /* ipc */ + sof_cnl_ops.send_msg = cnl_ipc_send_msg; + } - /* machine driver */ - .machine_select = hda_machine_select, - .machine_register = sof_machine_register, - .machine_unregister = sof_machine_unregister, - .set_mach_params = hda_set_mach_params, + if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_fw_data *ipc4_data; + + sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL); + if (!sdev->private) + return -ENOMEM; + + ipc4_data = sdev->private; + ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; + + /* doorbell */ + sof_cnl_ops.irq_thread = cnl_ipc4_irq_thread; + + /* ipc */ + sof_cnl_ops.send_msg = cnl_ipc4_send_msg; + } + + /* set DAI driver ops */ + hda_set_dai_drv_ops(sdev, &sof_cnl_ops); /* debug */ - .debug_map = cnl_dsp_debugfs, - .debug_map_count = ARRAY_SIZE(cnl_dsp_debugfs), - .dbg_dump = hda_dsp_dump, - .ipc_dump = cnl_ipc_dump, - .debugfs_add_region_item = snd_sof_debugfs_add_region_item_iomem, - - /* stream callbacks */ - .pcm_open = hda_dsp_pcm_open, - .pcm_close = hda_dsp_pcm_close, - .pcm_hw_params = hda_dsp_pcm_hw_params, - .pcm_hw_free = hda_dsp_stream_hw_free, - .pcm_trigger = hda_dsp_pcm_trigger, - .pcm_pointer = hda_dsp_pcm_pointer, - .pcm_ack = hda_dsp_pcm_ack, - - /* firmware loading */ - .load_firmware = snd_sof_load_firmware_raw, + sof_cnl_ops.debug_map = cnl_dsp_debugfs; + sof_cnl_ops.debug_map_count = ARRAY_SIZE(cnl_dsp_debugfs); + sof_cnl_ops.ipc_dump = cnl_ipc_dump; /* pre/post fw run */ - .pre_fw_run = hda_dsp_pre_fw_run, - .post_fw_run = hda_dsp_post_fw_run, - - /* parse platform specific extended manifest */ - .parse_platform_ext_manifest = hda_dsp_ext_man_get_cavs_config_data, - - /* dsp core get/put */ - .core_get = hda_dsp_core_get, + sof_cnl_ops.post_fw_run = hda_dsp_post_fw_run; /* firmware run */ - .run = hda_dsp_cl_boot_firmware, + sof_cnl_ops.run = hda_dsp_cl_boot_firmware; - /* trace callback */ - .trace_init = hda_dsp_trace_init, - .trace_release = hda_dsp_trace_release, - .trace_trigger = hda_dsp_trace_trigger, + /* dsp core get/put */ + sof_cnl_ops.core_get = hda_dsp_core_get; - /* client ops */ - .register_ipc_clients = hda_register_clients, - .unregister_ipc_clients = hda_unregister_clients, - - /* DAI drivers */ - .drv = skl_dai, - .num_drv = SOF_SKL_NUM_DAIS, - - /* PM */ - .suspend = hda_dsp_suspend, - .resume = hda_dsp_resume, - .runtime_suspend = hda_dsp_runtime_suspend, - .runtime_resume = hda_dsp_runtime_resume, - .runtime_idle = hda_dsp_runtime_idle, - .set_hw_params_upon_resume = hda_dsp_set_hw_params_upon_resume, - .set_power_state = hda_dsp_set_power_state, - - /* ALSA HW info flags */ - .hw_info = SNDRV_PCM_INFO_MMAP | - SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED | - SNDRV_PCM_INFO_PAUSE | - SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, - - .dsp_arch_ops = &sof_xtensa_arch_ops, + return 0; }; -EXPORT_SYMBOL_NS(sof_cnl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); +EXPORT_SYMBOL_NS(sof_cnl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON); const struct sof_intel_dsp_desc cnl_chip_info = { /* Cannonlake */ @@ -357,12 +404,16 @@ const struct sof_intel_dsp_desc cnl_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = CNL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_1_8, }; EXPORT_SYMBOL_NS(cnl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -383,11 +434,15 @@ const struct sof_intel_dsp_desc jsl_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = ICL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_2_0, }; EXPORT_SYMBOL_NS(jsl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/hda-dai.c b/sound/soc/sof/intel/hda-dai.c index f9cb9f1f02..556e883a32 100644 --- a/sound/soc/sof/intel/hda-dai.c +++ b/sound/soc/sof/intel/hda-dai.c @@ -10,10 +10,23 @@ #include #include +#include +#include +#include +#include "../ipc4-priv.h" +#include "../ipc4-topology.h" #include "../sof-priv.h" #include "../sof-audio.h" #include "hda.h" +/* + * The default method is to fetch NHLT from BIOS. With this parameter set + * it is possible to override that with NHLT in the SOF topology manifest. + */ +static bool hda_use_tplg_nhlt; +module_param_named(sof_use_tplg_nhlt, hda_use_tplg_nhlt, bool, 0444); +MODULE_PARM_DESC(sof_use_tplg_nhlt, "SOF topology nhlt override"); + #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) struct hda_pipe_params { @@ -50,8 +63,8 @@ static bool hda_check_fes(struct snd_soc_pcm_runtime *rtd, } static struct hdac_ext_stream * - hda_link_stream_assign(struct hdac_bus *bus, - struct snd_pcm_substream *substream) +hda_link_stream_assign(struct hdac_bus *bus, + struct snd_pcm_substream *substream) { struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); struct sof_intel_hda_stream *hda_stream; @@ -113,12 +126,8 @@ static struct hdac_ext_stream * } if (res) { - /* - * Decouple host and link DMA. The decoupled flag - * is updated in snd_hdac_ext_stream_decouple(). - */ - if (!res->decoupled) - snd_hdac_ext_stream_decouple_locked(bus, res, true); + /* Make sure that host and link DMA is decoupled. */ + snd_hdac_ext_stream_decouple_locked(bus, res, true); res->link_locked = 1; res->link_substream = substream; @@ -128,6 +137,40 @@ static struct hdac_ext_stream * return res; } +static int hda_link_dma_cleanup(struct snd_pcm_substream *substream, + struct hdac_stream *hstream, + struct snd_soc_dai *cpu_dai, + struct snd_soc_dai *codec_dai, + bool trigger_suspend_stop) +{ + struct hdac_ext_stream *hext_stream = snd_soc_dai_get_dma_data(cpu_dai, substream); + struct hdac_bus *bus = hstream->bus; + struct sof_intel_hda_stream *hda_stream; + struct hdac_ext_link *link; + int stream_tag; + + link = snd_hdac_ext_bus_get_link(bus, codec_dai->component->name); + if (!link) + return -EINVAL; + + if (trigger_suspend_stop) + snd_hdac_ext_link_stream_clear(hext_stream); + + if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { + stream_tag = hdac_stream(hext_stream)->stream_tag; + snd_hdac_ext_link_clear_stream_id(link, stream_tag); + } + snd_soc_dai_set_dma_data(cpu_dai, substream, NULL); + snd_hdac_ext_stream_release(hext_stream, HDAC_EXT_STREAM_TYPE_LINK); + hext_stream->link_prepared = 0; + + /* free the host DMA channel reserved by hostless streams */ + hda_stream = hstream_to_sof_hda_stream(hext_stream); + hda_stream->host_reserved = 0; + + return 0; +} + static int hda_link_dma_params(struct hdac_ext_stream *hext_stream, struct hda_pipe_params *params) { @@ -137,7 +180,6 @@ static int hda_link_dma_params(struct hdac_ext_stream *hext_stream, struct hdac_ext_link *link; unsigned int format_val; - snd_hdac_ext_stream_decouple(bus, hext_stream, true); snd_hdac_ext_link_stream_reset(hext_stream); format_val = snd_hdac_calc_stream_format(params->s_freq, params->ch, @@ -162,61 +204,27 @@ static int hda_link_dma_params(struct hdac_ext_stream *hext_stream, return 0; } -static int hda_link_dai_widget_update(struct sof_intel_hda_stream *hda_stream, - struct snd_soc_dapm_widget *w, - int channel, bool widget_setup) -{ - struct snd_sof_dai_config_data data; - - data.dai_data = channel; - - /* set up/free DAI widget and send DAI_CONFIG IPC */ - if (widget_setup) - return hda_ctrl_dai_widget_setup(w, SOF_DAI_CONFIG_FLAGS_2_STEP_STOP, &data); - - return hda_ctrl_dai_widget_free(w, SOF_DAI_CONFIG_FLAGS_NONE, &data); -} - -static int hda_link_hw_params(struct snd_pcm_substream *substream, - struct snd_pcm_hw_params *params, - struct snd_soc_dai *dai) +static int hda_link_dma_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params) { struct hdac_stream *hstream = substream->runtime->private_data; - struct hdac_bus *bus = hstream->bus; struct hdac_ext_stream *hext_stream; struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); - struct sof_intel_hda_stream *hda_stream; struct hda_pipe_params p_params = {0}; - struct snd_soc_dapm_widget *w; + struct hdac_bus *bus = hstream->bus; struct hdac_ext_link *link; - int stream_tag; - int ret; - /* get stored dma data if resuming from system suspend */ - hext_stream = snd_soc_dai_get_dma_data(dai, substream); + hext_stream = snd_soc_dai_get_dma_data(cpu_dai, substream); if (!hext_stream) { hext_stream = hda_link_stream_assign(bus, substream); if (!hext_stream) return -EBUSY; - snd_soc_dai_set_dma_data(dai, substream, (void *)hext_stream); + snd_soc_dai_set_dma_data(cpu_dai, substream, (void *)hext_stream); } - stream_tag = hdac_stream(hext_stream)->stream_tag; - - hda_stream = hstream_to_sof_hda_stream(hext_stream); - - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; - - /* set up the DAI widget and send the DAI_CONFIG with the new tag */ - ret = hda_link_dai_widget_update(hda_stream, w, stream_tag - 1, true); - if (ret < 0) - return ret; - link = snd_hdac_ext_bus_get_link(bus, codec_dai->component->name); if (!link) return -EINVAL; @@ -239,26 +247,117 @@ static int hda_link_hw_params(struct snd_pcm_substream *substream, return hda_link_dma_params(hext_stream, &p_params); } -static int hda_link_pcm_prepare(struct snd_pcm_substream *substream, - struct snd_soc_dai *dai) +static int hda_link_dma_prepare(struct snd_pcm_substream *substream) { - struct hdac_ext_stream *hext_stream = - snd_soc_dai_get_dma_data(dai, substream); - struct snd_sof_dev *sdev = - snd_soc_component_get_drvdata(dai->component); struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); int stream = substream->stream; - if (hext_stream->link_prepared) - return 0; - - dev_dbg(sdev->dev, "hda: prepare stream dir %d\n", substream->stream); - - return hda_link_hw_params(substream, &rtd->dpcm[stream].hw_params, - dai); + return hda_link_dma_hw_params(substream, &rtd->dpcm[stream].hw_params); } -static int hda_link_dai_config_pause_push_ipc(struct snd_soc_dapm_widget *w) +static int hda_link_dma_trigger(struct snd_pcm_substream *substream, int cmd) +{ + struct hdac_stream *hstream = substream->runtime->private_data; + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct hdac_ext_stream *hext_stream = snd_soc_dai_get_dma_data(cpu_dai, substream); + int ret; + + if (!hext_stream) + return 0; + + switch (cmd) { + case SNDRV_PCM_TRIGGER_START: + case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: + snd_hdac_ext_link_stream_start(hext_stream); + break; + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_STOP: + ret = hda_link_dma_cleanup(substream, hstream, cpu_dai, codec_dai, true); + if (ret < 0) + return ret; + + break; + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + snd_hdac_ext_link_stream_clear(hext_stream); + + break; + default: + return -EINVAL; + } + return 0; +} + +static int hda_link_dma_hw_free(struct snd_pcm_substream *substream) +{ + struct hdac_stream *hstream = substream->runtime->private_data; + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + struct snd_soc_dai *cpu_dai = asoc_rtd_to_cpu(rtd, 0); + struct snd_soc_dai *codec_dai = asoc_rtd_to_codec(rtd, 0); + struct hdac_ext_stream *hext_stream; + + hext_stream = snd_soc_dai_get_dma_data(cpu_dai, substream); + if (!hext_stream) + return 0; + + return hda_link_dma_cleanup(substream, hstream, cpu_dai, codec_dai, false); +} + +static int hda_dai_widget_update(struct snd_soc_dapm_widget *w, + int channel, bool widget_setup) +{ + struct snd_sof_dai_config_data data; + + data.dai_data = channel; + + /* set up/free DAI widget and send DAI_CONFIG IPC */ + if (widget_setup) + return hda_ctrl_dai_widget_setup(w, SOF_DAI_CONFIG_FLAGS_2_STEP_STOP, &data); + + return hda_ctrl_dai_widget_free(w, SOF_DAI_CONFIG_FLAGS_NONE, &data); +} + +static int hda_dai_hw_params_update(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *hext_stream; + struct snd_soc_dapm_widget *w; + int stream_tag; + + hext_stream = snd_soc_dai_get_dma_data(dai, substream); + if (!hext_stream) + return -EINVAL; + + stream_tag = hdac_stream(hext_stream)->stream_tag; + + w = snd_soc_dai_get_widget(dai, substream->stream); + + /* set up the DAI widget and send the DAI_CONFIG with the new tag */ + return hda_dai_widget_update(w, stream_tag - 1, true); +} + +static int hda_dai_hw_params(struct snd_pcm_substream *substream, + struct snd_pcm_hw_params *params, + struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *hext_stream = + snd_soc_dai_get_dma_data(dai, substream); + int ret; + + if (hext_stream && hext_stream->link_prepared) + return 0; + + ret = hda_link_dma_hw_params(substream, params); + if (ret < 0) + return ret; + + return hda_dai_hw_params_update(substream, params, dai); +} + + +static int hda_dai_config_pause_push_ipc(struct snd_soc_dapm_widget *w) { struct snd_sof_widget *swidget = w->dobj.private; struct snd_soc_component *component = swidget->scomp; @@ -276,34 +375,104 @@ static int hda_link_dai_config_pause_push_ipc(struct snd_soc_dapm_widget *w) return ret; } -static int hda_link_pcm_trigger(struct snd_pcm_substream *substream, - int cmd, struct snd_soc_dai *dai) +static int hda_dai_prepare(struct snd_pcm_substream *substream, struct snd_soc_dai *dai) { struct hdac_ext_stream *hext_stream = snd_soc_dai_get_dma_data(dai, substream); - struct sof_intel_hda_stream *hda_stream; - struct snd_soc_pcm_runtime *rtd; - struct snd_soc_dapm_widget *w; - struct hdac_ext_link *link; - struct hdac_stream *hstream; - struct hdac_bus *bus; - int stream_tag; + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(dai->component); + struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); + int stream = substream->stream; int ret; - hstream = substream->runtime->private_data; - bus = hstream->bus; - rtd = asoc_substream_to_rtd(substream); + if (hext_stream && hext_stream->link_prepared) + return 0; - link = snd_hdac_ext_bus_get_link(bus, asoc_rtd_to_codec(rtd, 0)->component->name); - if (!link) - return -EINVAL; + dev_dbg(sdev->dev, "prepare stream dir %d\n", substream->stream); - hda_stream = hstream_to_sof_hda_stream(hext_stream); + ret = hda_link_dma_prepare(substream); + if (ret < 0) + return ret; - dev_dbg(dai->dev, "In %s cmd=%d\n", __func__, cmd); + return hda_dai_hw_params_update(substream, &rtd->dpcm[stream].hw_params, dai); +} + +static int hda_dai_hw_free_ipc(int stream, /* direction */ + struct snd_soc_dai *dai) +{ + struct snd_soc_dapm_widget *w; + + w = snd_soc_dai_get_widget(dai, stream); + + /* free the link DMA channel in the FW and the DAI widget */ + return hda_dai_widget_update(w, DMA_CHAN_INVALID, false); +} + +static int ipc3_hda_dai_trigger(struct snd_pcm_substream *substream, + int cmd, struct snd_soc_dai *dai) +{ + struct snd_soc_dapm_widget *w; + int ret; + + dev_dbg(dai->dev, "cmd=%d dai %s direction %d\n", cmd, + dai->name, substream->stream); + + ret = hda_link_dma_trigger(substream, cmd); + if (ret < 0) + return ret; w = snd_soc_dai_get_widget(dai, substream->stream); + switch (cmd) { + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_STOP: + /* + * free DAI widget during stop/suspend to keep widget use_count's balanced. + */ + ret = hda_dai_hw_free_ipc(substream->stream, dai); + if (ret < 0) + return ret; + + break; + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + ret = hda_dai_config_pause_push_ipc(w); + if (ret < 0) + return ret; + break; + + default: + break; + } + return 0; +} + +/* + * In contrast to IPC3, the dai trigger in IPC4 mixes pipeline state changes + * (over IPC channel) and DMA state change (direct host register changes). + */ +static int ipc4_hda_dai_trigger(struct snd_pcm_substream *substream, + int cmd, struct snd_soc_dai *dai) +{ + struct hdac_ext_stream *hext_stream = snd_soc_dai_get_dma_data(dai, substream); + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(dai->component); + struct snd_soc_pcm_runtime *rtd; + struct snd_sof_widget *swidget; + struct snd_soc_dapm_widget *w; + struct snd_soc_dai *codec_dai; + struct hdac_stream *hstream; + struct snd_soc_dai *cpu_dai; + int ret; + + dev_dbg(dai->dev, "cmd=%d dai %s direction %d\n", cmd, + dai->name, substream->stream); + + hstream = substream->runtime->private_data; + rtd = asoc_substream_to_rtd(substream); + cpu_dai = asoc_rtd_to_cpu(rtd, 0); + codec_dai = asoc_rtd_to_codec(rtd, 0); + + w = snd_soc_dai_get_widget(dai, substream->stream); + swidget = w->dobj.private; + switch (cmd) { case SNDRV_PCM_TRIGGER_START: case SNDRV_PCM_TRIGGER_PAUSE_RELEASE: @@ -311,95 +480,121 @@ static int hda_link_pcm_trigger(struct snd_pcm_substream *substream, break; case SNDRV_PCM_TRIGGER_SUSPEND: case SNDRV_PCM_TRIGGER_STOP: - snd_hdac_ext_link_stream_clear(hext_stream); + { + struct snd_sof_widget *pipe_widget = swidget->pipe_widget; + struct sof_ipc4_pipeline *pipeline = pipe_widget->private; - /* - * free DAI widget during stop/suspend to keep widget use_count's balanced. - */ - ret = hda_link_dai_widget_update(hda_stream, w, DMA_CHAN_INVALID, false); + ret = sof_ipc4_set_pipeline_state(sdev, swidget->pipeline_id, + SOF_IPC4_PIPE_PAUSED); if (ret < 0) return ret; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - stream_tag = hdac_stream(hext_stream)->stream_tag; - snd_hdac_ext_link_clear_stream_id(link, stream_tag); + pipeline->state = SOF_IPC4_PIPE_PAUSED; + + snd_hdac_ext_link_stream_clear(hext_stream); + + ret = sof_ipc4_set_pipeline_state(sdev, swidget->pipeline_id, + SOF_IPC4_PIPE_RESET); + if (ret < 0) + return ret; + + pipeline->state = SOF_IPC4_PIPE_RESET; + + ret = hda_link_dma_cleanup(substream, hstream, cpu_dai, codec_dai, false); + if (ret < 0) { + dev_err(sdev->dev, "%s: failed to clean up link DMA\n", __func__); + return ret; } - - hext_stream->link_prepared = 0; break; + } case SNDRV_PCM_TRIGGER_PAUSE_PUSH: - snd_hdac_ext_link_stream_clear(hext_stream); + { + struct snd_sof_widget *pipe_widget = swidget->pipe_widget; + struct sof_ipc4_pipeline *pipeline = pipe_widget->private; - ret = hda_link_dai_config_pause_push_ipc(w); + ret = sof_ipc4_set_pipeline_state(sdev, swidget->pipeline_id, + SOF_IPC4_PIPE_PAUSED); if (ret < 0) return ret; + + pipeline->state = SOF_IPC4_PIPE_PAUSED; + + snd_hdac_ext_link_stream_clear(hext_stream); break; + } default: + dev_err(sdev->dev, "%s: unknown trigger command %d\n", __func__, cmd); return -EINVAL; } + return 0; } -static int hda_link_hw_free(struct snd_pcm_substream *substream, - struct snd_soc_dai *dai) +static int hda_dai_hw_free(struct snd_pcm_substream *substream, + struct snd_soc_dai *dai) { - unsigned int stream_tag; - struct sof_intel_hda_stream *hda_stream; - struct hdac_bus *bus; - struct hdac_ext_link *link; - struct hdac_stream *hstream; - struct snd_soc_pcm_runtime *rtd; - struct hdac_ext_stream *hext_stream; - struct snd_soc_dapm_widget *w; int ret; - hstream = substream->runtime->private_data; - bus = hstream->bus; - rtd = asoc_substream_to_rtd(substream); - hext_stream = snd_soc_dai_get_dma_data(dai, substream); - - if (!hext_stream) { - dev_dbg(dai->dev, - "%s: hext_stream is not assigned\n", __func__); - return -EINVAL; - } - - hda_stream = hstream_to_sof_hda_stream(hext_stream); - - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; - - /* free the link DMA channel in the FW and the DAI widget */ - ret = hda_link_dai_widget_update(hda_stream, w, DMA_CHAN_INVALID, false); + ret = hda_link_dma_hw_free(substream); if (ret < 0) return ret; - link = snd_hdac_ext_bus_get_link(bus, asoc_rtd_to_codec(rtd, 0)->component->name); - if (!link) - return -EINVAL; + return hda_dai_hw_free_ipc(substream->stream, dai); +} - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - stream_tag = hdac_stream(hext_stream)->stream_tag; - snd_hdac_ext_link_clear_stream_id(link, stream_tag); +static const struct snd_soc_dai_ops ipc3_hda_dai_ops = { + .hw_params = hda_dai_hw_params, + .hw_free = hda_dai_hw_free, + .trigger = ipc3_hda_dai_trigger, + .prepare = hda_dai_prepare, +}; + +static int hda_dai_suspend(struct hdac_bus *bus) +{ + struct snd_soc_pcm_runtime *rtd; + struct hdac_ext_stream *hext_stream; + struct hdac_stream *s; + int ret; + + /* set internal flag for BE */ + list_for_each_entry(s, &bus->stream_list, list) { + + hext_stream = stream_to_hdac_ext_stream(s); + + /* + * clear stream. This should already be taken care for running + * streams when the SUSPEND trigger is called. But paused + * streams do not get suspended, so this needs to be done + * explicitly during suspend. + */ + if (hext_stream->link_substream) { + struct snd_soc_dai *cpu_dai; + struct snd_soc_dai *codec_dai; + + rtd = asoc_substream_to_rtd(hext_stream->link_substream); + cpu_dai = asoc_rtd_to_cpu(rtd, 0); + codec_dai = asoc_rtd_to_codec(rtd, 0); + + ret = hda_link_dma_cleanup(hext_stream->link_substream, s, + cpu_dai, codec_dai, false); + if (ret < 0) + return ret; + + /* for consistency with TRIGGER_SUSPEND we free DAI resources */ + ret = hda_dai_hw_free_ipc(hdac_stream(hext_stream)->direction, cpu_dai); + if (ret < 0) + return ret; + } } - snd_soc_dai_set_dma_data(dai, substream, NULL); - snd_hdac_ext_stream_release(hext_stream, HDAC_EXT_STREAM_TYPE_LINK); - hext_stream->link_prepared = 0; - - /* free the host DMA channel reserved by hostless streams */ - hda_stream->host_reserved = 0; - return 0; } -static const struct snd_soc_dai_ops hda_link_dai_ops = { - .hw_params = hda_link_hw_params, - .hw_free = hda_link_hw_free, - .trigger = hda_link_pcm_trigger, - .prepare = hda_link_pcm_prepare, +static const struct snd_soc_dai_ops ipc4_hda_dai_ops = { + .hw_params = hda_dai_hw_params, + .hw_free = hda_dai_hw_free, + .trigger = ipc4_hda_dai_trigger, + .prepare = hda_dai_prepare, }; #endif @@ -414,10 +609,7 @@ static int ssp_dai_setup_or_free(struct snd_pcm_substream *substream, struct snd { struct snd_soc_dapm_widget *w; - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) - w = dai->playback_widget; - else - w = dai->capture_widget; + w = snd_soc_dai_get_widget(dai, substream->stream); if (setup) return hda_ctrl_dai_widget_setup(w, SOF_DAI_CONFIG_FLAGS_NONE, NULL); @@ -478,8 +670,8 @@ static int ssp_dai_prepare(struct snd_pcm_substream *substream, return ssp_dai_setup(substream, dai, true); } -static int ssp_dai_trigger(struct snd_pcm_substream *substream, - int cmd, struct snd_soc_dai *dai) +static int ipc3_ssp_dai_trigger(struct snd_pcm_substream *substream, + int cmd, struct snd_soc_dai *dai) { if (cmd != SNDRV_PCM_TRIGGER_SUSPEND) return 0; @@ -507,15 +699,137 @@ static void ssp_dai_shutdown(struct snd_pcm_substream *substream, kfree(dma_data); } -static const struct snd_soc_dai_ops ssp_dai_ops = { +static const struct snd_soc_dai_ops ipc3_ssp_dai_ops = { .startup = ssp_dai_startup, .hw_params = ssp_dai_hw_params, .prepare = ssp_dai_prepare, - .trigger = ssp_dai_trigger, + .trigger = ipc3_ssp_dai_trigger, .hw_free = ssp_dai_hw_free, .shutdown = ssp_dai_shutdown, }; +static int ipc4_be_dai_common_trigger(struct snd_soc_dai *dai, int cmd, int stream) +{ + struct snd_sof_widget *pipe_widget; + struct sof_ipc4_pipeline *pipeline; + struct snd_sof_widget *swidget; + struct snd_soc_dapm_widget *w; + struct snd_sof_dev *sdev; + int ret; + + w = snd_soc_dai_get_widget(dai, stream); + swidget = w->dobj.private; + pipe_widget = swidget->pipe_widget; + pipeline = pipe_widget->private; + sdev = snd_soc_component_get_drvdata(swidget->scomp); + + switch (cmd) { + case SNDRV_PCM_TRIGGER_SUSPEND: + case SNDRV_PCM_TRIGGER_STOP: + ret = sof_ipc4_set_pipeline_state(sdev, swidget->pipeline_id, + SOF_IPC4_PIPE_PAUSED); + if (ret < 0) + return ret; + pipeline->state = SOF_IPC4_PIPE_PAUSED; + + ret = sof_ipc4_set_pipeline_state(sdev, swidget->pipeline_id, + SOF_IPC4_PIPE_RESET); + if (ret < 0) + return ret; + pipeline->state = SOF_IPC4_PIPE_RESET; + break; + case SNDRV_PCM_TRIGGER_PAUSE_PUSH: + ret = sof_ipc4_set_pipeline_state(sdev, swidget->pipeline_id, + SOF_IPC4_PIPE_PAUSED); + if (ret < 0) + return ret; + pipeline->state = SOF_IPC4_PIPE_PAUSED; + break; + default: + break; + } + + return 0; +} + +static int ipc4_be_dai_trigger(struct snd_pcm_substream *substream, + int cmd, struct snd_soc_dai *dai) +{ + return ipc4_be_dai_common_trigger(dai, cmd, substream->stream); +} + +static const struct snd_soc_dai_ops ipc4_dmic_dai_ops = { + .trigger = ipc4_be_dai_trigger, +}; + +static const struct snd_soc_dai_ops ipc4_ssp_dai_ops = { + .trigger = ipc4_be_dai_trigger, +}; + +void hda_set_dai_drv_ops(struct snd_sof_dev *sdev, struct snd_sof_dsp_ops *ops) +{ + int i; + + switch (sdev->pdata->ipc_type) { + case SOF_IPC: + for (i = 0; i < ops->num_drv; i++) { + if (strstr(ops->drv[i].name, "SSP")) { + ops->drv[i].ops = &ipc3_ssp_dai_ops; + continue; + } +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + if (strstr(ops->drv[i].name, "iDisp") || + strstr(ops->drv[i].name, "Analog") || + strstr(ops->drv[i].name, "Digital")) + ops->drv[i].ops = &ipc3_hda_dai_ops; +#endif + } + break; + case SOF_INTEL_IPC4: + { + struct sof_ipc4_fw_data *ipc4_data = sdev->private; + + for (i = 0; i < ops->num_drv; i++) { + if (strstr(ops->drv[i].name, "DMIC")) { + ops->drv[i].ops = &ipc4_dmic_dai_ops; + continue; + } + if (strstr(ops->drv[i].name, "SSP")) { + ops->drv[i].ops = &ipc4_ssp_dai_ops; + continue; + } +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + if (strstr(ops->drv[i].name, "iDisp") || + strstr(ops->drv[i].name, "Analog") || + strstr(ops->drv[i].name, "Digital")) + ops->drv[i].ops = &ipc4_hda_dai_ops; +#endif + } + + if (!hda_use_tplg_nhlt) + ipc4_data->nhlt = intel_nhlt_init(sdev->dev); + + if (IS_ENABLED(CONFIG_SND_SOC_SOF_INTEL_SOUNDWIRE)) + sdw_callback.trigger = ipc4_be_dai_common_trigger; + + break; + } + default: + break; + } +} + +void hda_ops_free(struct snd_sof_dev *sdev) +{ + if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_fw_data *ipc4_data = sdev->private; + + if (!hda_use_tplg_nhlt) + intel_nhlt_free(ipc4_data->nhlt); + } +} +EXPORT_SYMBOL_NS(hda_ops_free, SND_SOC_SOF_INTEL_HDA_COMMON); + /* * common dai driver for skl+ platforms. * some products who use this DAI array only physically have a subset of @@ -524,7 +838,6 @@ static const struct snd_soc_dai_ops ssp_dai_ops = { struct snd_soc_dai_driver skl_dai[] = { { .name = "SSP0 Pin", - .ops = &ssp_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -536,7 +849,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "SSP1 Pin", - .ops = &ssp_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -548,7 +860,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "SSP2 Pin", - .ops = &ssp_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -560,7 +871,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "SSP3 Pin", - .ops = &ssp_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -572,7 +882,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "SSP4 Pin", - .ops = &ssp_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -584,7 +893,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "SSP5 Pin", - .ops = &ssp_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -611,7 +919,6 @@ struct snd_soc_dai_driver skl_dai[] = { #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) { .name = "iDisp1 Pin", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -619,7 +926,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "iDisp2 Pin", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -627,7 +933,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "iDisp3 Pin", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -635,7 +940,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "iDisp4 Pin", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 8, @@ -643,7 +947,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "Analog CPU DAI", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 16, @@ -655,7 +958,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "Digital CPU DAI", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 16, @@ -667,7 +969,6 @@ struct snd_soc_dai_driver skl_dai[] = { }, { .name = "Alt Analog CPU DAI", - .ops = &hda_link_dai_ops, .playback = { .channels_min = 1, .channels_max = 16, @@ -679,3 +980,22 @@ struct snd_soc_dai_driver skl_dai[] = { }, #endif }; + +int hda_dsp_dais_suspend(struct snd_sof_dev *sdev) +{ + /* + * In the corner case where a SUSPEND happens during a PAUSE, the ALSA core + * does not throw the TRIGGER_SUSPEND. This leaves the DAIs in an unbalanced state. + * Since the component suspend is called last, we can trap this corner case + * and force the DAIs to release their resources. + */ +#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) + int ret; + + ret = hda_dai_suspend(sof_to_bus(sdev)); + if (ret < 0) + return ret; +#endif + + return 0; +} diff --git a/sound/soc/sof/intel/hda-dsp.c b/sound/soc/sof/intel/hda-dsp.c index 8ddde60c56..eddfd77ad9 100644 --- a/sound/soc/sof/intel/hda-dsp.c +++ b/sound/soc/sof/intel/hda-dsp.c @@ -181,12 +181,20 @@ int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask) * Power Management. */ -static int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask) +int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask) { + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; unsigned int cpa; u32 adspcs; int ret; + /* restrict core_mask to host managed cores mask */ + core_mask &= chip->host_managed_cores_mask; + /* return if core_mask is not valid */ + if (!core_mask) + return 0; + /* update bits */ snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_ADSPCS, HDA_DSP_ADSPCS_SPA_MASK(core_mask), @@ -363,9 +371,8 @@ static int hda_dsp_send_pm_gate_ipc(struct snd_sof_dev *sdev, u32 flags) pm_gate.flags = flags; /* send pm_gate ipc to dsp */ - return sof_ipc_tx_message_no_pm(sdev->ipc, pm_gate.hdr.cmd, - &pm_gate, sizeof(pm_gate), &reply, - sizeof(reply)); + return sof_ipc_tx_message_no_pm(sdev->ipc, &pm_gate, sizeof(pm_gate), + &reply, sizeof(reply)); } static int hda_dsp_update_d0i3c_register(struct snd_sof_dev *sdev, u8 value) @@ -433,7 +440,7 @@ static int hda_dsp_set_D0_state(struct snd_sof_dev *sdev, * when the DSP enters D0I3 while the system is in S0 * for debug purpose. */ - if (!sdev->dtrace_is_supported || + if (!sdev->fw_trace_is_supported || !hda_enable_trace_D0I3_S0 || sdev->system_suspend_target != SOF_SUSPEND_NONE) flags = HDA_PM_NO_DMA_TRACE; @@ -610,6 +617,13 @@ static int hda_suspend(struct snd_sof_dev *sdev, bool runtime_suspend) #endif int ret, j; + /* + * The memory used for IMR boot loses its content in deeper than S3 state + * We must not try IMR boot on next power up (as it will fail). + */ + if (sdev->system_suspend_target > SOF_SUSPEND_S3) + hda->skip_imr_boot = true; + hda_sdw_int_enable(sdev, false); /* disable IPC interrupts */ @@ -736,7 +750,7 @@ int hda_dsp_resume(struct snd_sof_dev *sdev) if (hlink->ref_count) { ret = snd_hdac_ext_bus_link_power_up(hlink); if (ret < 0) { - dev_dbg(sdev->dev, + dev_err(sdev->dev, "error %d in %s: failed to power up links", ret, __func__); return ret; @@ -864,7 +878,7 @@ int hda_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state) /* no link can be powered in s0ix state */ ret = snd_hdac_ext_bus_link_power_down_all(bus); if (ret < 0) { - dev_dbg(sdev->dev, + dev_err(sdev->dev, "error %d in %s: failed to power down links", ret, __func__); return ret; @@ -895,44 +909,14 @@ int hda_dsp_shutdown(struct snd_sof_dev *sdev) int hda_dsp_set_hw_params_upon_resume(struct snd_sof_dev *sdev) { -#if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA) - struct hdac_bus *bus = sof_to_bus(sdev); - struct snd_soc_pcm_runtime *rtd; - struct hdac_ext_stream *hext_stream; - struct hdac_ext_link *link; - struct hdac_stream *s; - const char *name; - int stream_tag; + int ret; - /* set internal flag for BE */ - list_for_each_entry(s, &bus->stream_list, list) { - hext_stream = stream_to_hdac_ext_stream(s); + /* make sure all DAI resources are freed */ + ret = hda_dsp_dais_suspend(sdev); + if (ret < 0) + dev_warn(sdev->dev, "%s: failure in hda_dsp_dais_suspend\n", __func__); - /* - * clear stream. This should already be taken care for running - * streams when the SUSPEND trigger is called. But paused - * streams do not get suspended, so this needs to be done - * explicitly during suspend. - */ - if (hext_stream->link_substream) { - rtd = asoc_substream_to_rtd(hext_stream->link_substream); - name = asoc_rtd_to_codec(rtd, 0)->component->name; - link = snd_hdac_ext_bus_get_link(bus, name); - if (!link) - return -EINVAL; - - hext_stream->link_prepared = 0; - - if (hdac_stream(hext_stream)->direction == - SNDRV_PCM_STREAM_CAPTURE) - continue; - - stream_tag = hdac_stream(hext_stream)->stream_tag; - snd_hdac_ext_link_clear_stream_id(link, stream_tag); - } - } -#endif - return 0; + return ret; } void hda_dsp_d0i3_work(struct work_struct *work) @@ -963,13 +947,7 @@ void hda_dsp_d0i3_work(struct work_struct *work) int hda_dsp_core_get(struct snd_sof_dev *sdev, int core) { - struct sof_ipc_pm_core_config pm_core_config = { - .hdr = { - .cmd = SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CORE_ENABLE, - .size = sizeof(pm_core_config), - }, - .enable_mask = sdev->enabled_cores_mask | BIT(core), - }; + const struct sof_ipc_pm_ops *pm_ops = sdev->ipc->ops->pm; int ret, ret1; /* power up core */ @@ -984,10 +962,12 @@ int hda_dsp_core_get(struct snd_sof_dev *sdev, int core) if (sdev->fw_state != SOF_FW_BOOT_COMPLETE || core == SOF_DSP_PRIMARY_CORE) return 0; + /* No need to continue the set_core_state ops is not available */ + if (!pm_ops->set_core_state) + return 0; + /* Now notify DSP for secondary cores */ - ret = sof_ipc_tx_message(sdev->ipc, pm_core_config.hdr.cmd, - &pm_core_config, sizeof(pm_core_config), - &pm_core_config, sizeof(pm_core_config)); + ret = pm_ops->set_core_state(sdev, core, true); if (ret < 0) { dev_err(sdev->dev, "failed to enable secondary core '%d' failed with %d\n", core, ret); diff --git a/sound/soc/sof/intel/hda-ipc.c b/sound/soc/sof/intel/hda-ipc.c index 0395638c43..65e688f749 100644 --- a/sound/soc/sof/intel/hda-ipc.c +++ b/sound/soc/sof/intel/hda-ipc.c @@ -15,6 +15,7 @@ * Hardware interface for generic Intel audio DSP HDA IP */ +#include #include "../ops.h" #include "hda.h" @@ -65,6 +66,22 @@ int hda_dsp_ipc_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) return 0; } +int hda_dsp_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg) +{ + struct sof_ipc4_msg *msg_data = msg->msg_data; + + /* send the message via mailbox */ + if (msg_data->data_size) + sof_mailbox_write(sdev, sdev->host_box.offset, msg_data->data_ptr, + msg_data->data_size); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCIE, msg_data->extension); + snd_sof_dsp_write(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCI, + msg_data->primary | HDA_DSP_REG_HIPCI_BUSY); + + return 0; +} + void hda_dsp_ipc_get_reply(struct snd_sof_dev *sdev) { struct snd_sof_ipc_msg *msg = sdev->msg; @@ -100,6 +117,77 @@ void hda_dsp_ipc_get_reply(struct snd_sof_dev *sdev) } } +irqreturn_t hda_dsp_ipc4_irq_thread(int irq, void *context) +{ + struct sof_ipc4_msg notification_data = {{ 0 }}; + struct snd_sof_dev *sdev = context; + bool ipc_irq = false; + u32 hipcie, hipct; + + hipcie = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCIE); + if (hipcie & HDA_DSP_REG_HIPCIE_DONE) { + /* DSP received the message */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCCTL, + HDA_DSP_REG_HIPCCTL_DONE, 0); + hda_dsp_ipc_dsp_done(sdev); + + ipc_irq = true; + } + + hipct = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCT); + if (hipct & HDA_DSP_REG_HIPCT_BUSY) { + /* Message from DSP (reply or notification) */ + u32 hipcte = snd_sof_dsp_read(sdev, HDA_DSP_BAR, + HDA_DSP_REG_HIPCTE); + u32 primary = hipct & HDA_DSP_REG_HIPCT_MSG_MASK; + u32 extension = hipcte & HDA_DSP_REG_HIPCTE_MSG_MASK; + + /* mask BUSY interrupt */ + snd_sof_dsp_update_bits(sdev, HDA_DSP_BAR, HDA_DSP_REG_HIPCCTL, + HDA_DSP_REG_HIPCCTL_BUSY, 0); + + if (primary & SOF_IPC4_MSG_DIR_MASK) { + /* Reply received */ + if (likely(sdev->fw_state == SOF_FW_BOOT_COMPLETE)) { + struct sof_ipc4_msg *data = sdev->ipc->msg.reply_data; + + data->primary = primary; + data->extension = extension; + + spin_lock_irq(&sdev->ipc_lock); + + snd_sof_ipc_get_reply(sdev); + snd_sof_ipc_reply(sdev, data->primary); + + spin_unlock_irq(&sdev->ipc_lock); + } else { + dev_dbg_ratelimited(sdev->dev, + "IPC reply before FW_READY: %#x|%#x\n", + primary, extension); + } + } else { + /* Notification received */ + + notification_data.primary = primary; + notification_data.extension = extension; + sdev->ipc->msg.rx_data = ¬ification_data; + snd_sof_ipc_msgs_rx(sdev); + sdev->ipc->msg.rx_data = NULL; + } + + /* Let DSP know that we have finished processing the message */ + hda_dsp_ipc_host_done(sdev); + + ipc_irq = true; + } + + if (!ipc_irq) + /* This interrupt is not shared so no need to return IRQ_NONE. */ + dev_dbg_ratelimited(sdev->dev, "nothing to do in IPC IRQ thread\n"); + + return IRQ_HANDLED; +} + /* IPC handler thread */ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) { @@ -143,16 +231,21 @@ irqreturn_t hda_dsp_ipc_irq_thread(int irq, void *context) * place, the message might not yet be marked as expecting a * reply. */ - spin_lock_irq(&sdev->ipc_lock); + if (likely(sdev->fw_state == SOF_FW_BOOT_COMPLETE)) { + spin_lock_irq(&sdev->ipc_lock); - /* handle immediate reply from DSP core */ - hda_dsp_ipc_get_reply(sdev); - snd_sof_ipc_reply(sdev, msg); + /* handle immediate reply from DSP core */ + hda_dsp_ipc_get_reply(sdev); + snd_sof_ipc_reply(sdev, msg); - /* set the done bit */ - hda_dsp_ipc_dsp_done(sdev); + /* set the done bit */ + hda_dsp_ipc_dsp_done(sdev); - spin_unlock_irq(&sdev->ipc_lock); + spin_unlock_irq(&sdev->ipc_lock); + } else { + dev_dbg_ratelimited(sdev->dev, "IPC reply before FW_READY: %#x\n", + msg); + } ipc_irq = true; } diff --git a/sound/soc/sof/intel/hda-loader.c b/sound/soc/sof/intel/hda-loader.c index 2ac5d9d071..eb22eb3f6f 100644 --- a/sound/soc/sof/intel/hda-loader.c +++ b/sound/soc/sof/intel/hda-loader.c @@ -24,8 +24,6 @@ #include "../sof-priv.h" #include "hda.h" -#define HDA_CL_STREAM_FORMAT 0x40 - static void hda_ssp_set_cbp_cfp(struct snd_sof_dev *sdev) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; @@ -43,9 +41,9 @@ static void hda_ssp_set_cbp_cfp(struct snd_sof_dev *sdev) } } -static struct hdac_ext_stream *cl_stream_prepare(struct snd_sof_dev *sdev, unsigned int format, - unsigned int size, struct snd_dma_buffer *dmab, - int direction) +struct hdac_ext_stream *hda_cl_stream_prepare(struct snd_sof_dev *sdev, unsigned int format, + unsigned int size, struct snd_dma_buffer *dmab, + int direction) { struct hdac_ext_stream *hext_stream; struct hdac_stream *hstream; @@ -97,22 +95,22 @@ static struct hdac_ext_stream *cl_stream_prepare(struct snd_sof_dev *sdev, unsig } /* - * first boot sequence has some extra steps. core 0 waits for power - * status on core 1, so power up core 1 also momentarily, keep it in - * reset/stall and then turn it off + * first boot sequence has some extra steps. + * power on all host managed cores and only unstall/run the boot core to boot the + * DSP then turn off all non boot cores (if any) is powered on. */ -static int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag) +int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; const struct sof_intel_dsp_desc *chip = hda->desc; - unsigned int status; + unsigned int status, target_status; + u32 flags, ipc_hdr, j; unsigned long mask; char *dump_msg; - u32 flags, j; int ret; /* step 1: power up corex */ - ret = hda_dsp_enable_core(sdev, chip->host_managed_cores_mask); + ret = hda_dsp_core_power_up(sdev, chip->host_managed_cores_mask); if (ret < 0) { if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS) dev_err(sdev->dev, "error: dsp core 0/1 power up failed\n"); @@ -121,13 +119,15 @@ static int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag) hda_ssp_set_cbp_cfp(sdev); - /* step 2: purge FW request */ - snd_sof_dsp_write(sdev, HDA_DSP_BAR, chip->ipc_req, - chip->ipc_req_mask | (HDA_DSP_IPC_PURGE_FW | - ((stream_tag - 1) << 9))); + /* step 2: Send ROM_CONTROL command (stream_tag is ignored for IMR boot) */ + ipc_hdr = chip->ipc_req_mask | HDA_DSP_ROM_IPC_CONTROL; + if (!imr_boot) + ipc_hdr |= HDA_DSP_ROM_IPC_PURGE_FW | ((stream_tag - 1) << 9); + + snd_sof_dsp_write(sdev, HDA_DSP_BAR, chip->ipc_req, ipc_hdr); /* step 3: unset core 0 reset state & unstall/run core 0 */ - ret = hda_dsp_core_run(sdev, BIT(0)); + ret = hda_dsp_core_run(sdev, chip->init_core_mask); if (ret < 0) { if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS) dev_err(sdev->dev, @@ -171,11 +171,20 @@ static int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag) /* step 6: enable IPC interrupts */ hda_dsp_ipc_int_enable(sdev); - /* step 7: wait for ROM init */ + /* + * step 7: + * - Cold/Full boot: wait for ROM init to proceed to download the firmware + * - IMR boot: wait for ROM firmware entered (firmware booted up from IMR) + */ + if (imr_boot) + target_status = HDA_DSP_ROM_FW_ENTERED; + else + target_status = HDA_DSP_ROM_INIT; + ret = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, - HDA_DSP_SRAM_REG_ROM_STATUS, status, + chip->rom_status_reg, status, ((status & HDA_DSP_ROM_STS_MASK) - == HDA_DSP_ROM_INIT), + == target_status), HDA_DSP_REG_POLL_INTERVAL_US, chip->rom_init_timeout * USEC_PER_MSEC); @@ -190,8 +199,8 @@ static int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag) if (hda->boot_iteration == HDA_FW_BOOT_ATTEMPTS) dev_err(sdev->dev, - "error: %s: timeout HDA_DSP_SRAM_REG_ROM_STATUS read\n", - __func__); + "%s: timeout with rom_status_reg (%#x) read\n", + __func__, chip->rom_status_reg); err: flags = SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX | SOF_DBG_DUMP_OPTIONAL; @@ -236,8 +245,8 @@ static int cl_trigger(struct snd_sof_dev *sdev, } } -static int cl_cleanup(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, - struct hdac_ext_stream *hext_stream) +int hda_cl_cleanup(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, + struct hdac_ext_stream *hext_stream) { struct hdac_stream *hstream = &hext_stream->hstream; int sd_offset = SOF_STREAM_SD_OFFSET(hstream); @@ -268,8 +277,10 @@ static int cl_cleanup(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, return ret; } -static int cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_stream) +int hda_cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_stream) { + struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; + const struct sof_intel_dsp_desc *chip = hda->desc; unsigned int reg; int ret, status; @@ -280,7 +291,7 @@ static int cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_str } status = snd_sof_dsp_read_poll_timeout(sdev, HDA_DSP_BAR, - HDA_DSP_SRAM_REG_ROM_STATUS, reg, + chip->rom_status_reg, reg, ((reg & HDA_DSP_ROM_STS_MASK) == HDA_DSP_ROM_FW_ENTERED), HDA_DSP_REG_POLL_INTERVAL_US, @@ -293,8 +304,8 @@ static int cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_str if (status < 0) { dev_err(sdev->dev, - "error: %s: timeout HDA_DSP_SRAM_REG_ROM_STATUS read\n", - __func__); + "%s: timeout with rom_status_reg (%#x) read\n", + __func__, chip->rom_status_reg); } ret = cl_trigger(sdev, hext_stream, SNDRV_PCM_TRIGGER_STOP); @@ -313,6 +324,7 @@ int hda_dsp_cl_boot_firmware_iccmax(struct snd_sof_dev *sdev) struct hdac_ext_stream *iccmax_stream; struct hdac_bus *bus = sof_to_bus(sdev); struct firmware stripped_firmware; + struct snd_dma_buffer dmab_bdl; int ret, ret1; u8 original_gb; @@ -327,8 +339,8 @@ int hda_dsp_cl_boot_firmware_iccmax(struct snd_sof_dev *sdev) stripped_firmware.size = plat_data->fw->size - plat_data->fw_offset; /* prepare capture stream for ICCMAX */ - iccmax_stream = cl_stream_prepare(sdev, HDA_CL_STREAM_FORMAT, stripped_firmware.size, - &sdev->dmab_bdl, SNDRV_PCM_STREAM_CAPTURE); + iccmax_stream = hda_cl_stream_prepare(sdev, HDA_CL_STREAM_FORMAT, stripped_firmware.size, + &dmab_bdl, SNDRV_PCM_STREAM_CAPTURE); if (IS_ERR(iccmax_stream)) { dev_err(sdev->dev, "error: dma prepare for ICCMAX stream failed\n"); return PTR_ERR(iccmax_stream); @@ -340,7 +352,7 @@ int hda_dsp_cl_boot_firmware_iccmax(struct snd_sof_dev *sdev) * Perform iccmax stream cleanup. This should be done even if firmware loading fails. * If the cleanup also fails, we return the initial error */ - ret1 = cl_cleanup(sdev, &sdev->dmab_bdl, iccmax_stream); + ret1 = hda_cl_cleanup(sdev, &dmab_bdl, iccmax_stream); if (ret1 < 0) { dev_err(sdev->dev, "error: ICCMAX stream cleanup failed\n"); @@ -357,32 +369,17 @@ int hda_dsp_cl_boot_firmware_iccmax(struct snd_sof_dev *sdev) static int hda_dsp_boot_imr(struct snd_sof_dev *sdev) { - struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; - const struct sof_intel_dsp_desc *chip = hda->desc; - unsigned long mask; - u32 j; + const struct sof_intel_dsp_desc *chip_info; int ret; - /* power up & unstall/run the cores to run the firmware */ - ret = hda_dsp_enable_core(sdev, chip->init_core_mask); - if (ret < 0) { - dev_err(sdev->dev, "dsp core start failed %d\n", ret); - return -EIO; - } + chip_info = get_chip_info(sdev->pdata); + if (chip_info->cl_init) + ret = chip_info->cl_init(sdev, 0, true); + else + ret = -EINVAL; - /* set enabled cores mask and increment ref count for cores in init_core_mask */ - sdev->enabled_cores_mask |= chip->init_core_mask; - mask = sdev->enabled_cores_mask; - for_each_set_bit(j, &mask, SOF_MAX_DSP_NUM_CORES) - sdev->dsp_core_ref_count[j]++; - - hda_ssp_set_cbp_cfp(sdev); - - /* enable IPC interrupts */ - hda_dsp_ipc_int_enable(sdev); - - /* process wakes */ - hda_sdw_process_wakeen(sdev); + if (!ret) + hda_sdw_process_wakeen(sdev); return ret; } @@ -395,13 +392,17 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev) const struct sof_intel_dsp_desc *chip_info; struct hdac_ext_stream *hext_stream; struct firmware stripped_firmware; + struct snd_dma_buffer dmab; int ret, ret1, i; - if ((sdev->fw_ready.flags & SOF_IPC_INFO_D3_PERSISTENT) && - !(sof_debug_check_flag(SOF_DBG_IGNORE_D3_PERSISTENT)) && - !sdev->first_boot) { + if (hda->imrboot_supported && !sdev->first_boot && !hda->skip_imr_boot) { dev_dbg(sdev->dev, "IMR restore supported, booting from IMR directly\n"); - return hda_dsp_boot_imr(sdev); + hda->boot_iteration = 0; + ret = hda_dsp_boot_imr(sdev); + if (!ret) + return 0; + + dev_warn(sdev->dev, "IMR restore failed, trying to cold boot\n"); } chip_info = desc->chip_info; @@ -418,14 +419,15 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev) init_waitqueue_head(&sdev->boot_wait); /* prepare DMA for code loader stream */ - hext_stream = cl_stream_prepare(sdev, HDA_CL_STREAM_FORMAT, stripped_firmware.size, - &sdev->dmab, SNDRV_PCM_STREAM_PLAYBACK); + hext_stream = hda_cl_stream_prepare(sdev, HDA_CL_STREAM_FORMAT, + stripped_firmware.size, + &dmab, SNDRV_PCM_STREAM_PLAYBACK); if (IS_ERR(hext_stream)) { dev_err(sdev->dev, "error: dma prepare for fw loading failed\n"); return PTR_ERR(hext_stream); } - memcpy(sdev->dmab.area, stripped_firmware.data, + memcpy(dmab.area, stripped_firmware.data, stripped_firmware.size); /* try ROM init a few times before giving up */ @@ -434,7 +436,10 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev) "Attempting iteration %d of Core En/ROM load...\n", i); hda->boot_iteration = i + 1; - ret = cl_dsp_init(sdev, hext_stream->hstream.stream_tag); + if (chip_info->cl_init) + ret = chip_info->cl_init(sdev, hext_stream->hstream.stream_tag, false); + else + ret = -EINVAL; /* don't retry anymore if successful */ if (!ret) @@ -473,12 +478,15 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev) * Continue with code loading and firmware boot */ hda->boot_iteration = HDA_FW_BOOT_ATTEMPTS; - ret = cl_copy_fw(sdev, hext_stream); - if (!ret) + ret = hda_cl_copy_fw(sdev, hext_stream); + if (!ret) { dev_dbg(sdev->dev, "Firmware download successful, booting...\n"); - else + hda->skip_imr_boot = false; + } else { snd_sof_dsp_dbg_dump(sdev, "Firmware download failed", SOF_DBG_DUMP_PCI | SOF_DBG_DUMP_MBOX); + hda->skip_imr_boot = true; + } cleanup: /* @@ -486,7 +494,7 @@ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev) * This should be done even if firmware loading fails. * If the cleanup also fails, we return the initial error */ - ret1 = cl_cleanup(sdev, &sdev->dmab, hext_stream); + ret1 = hda_cl_cleanup(sdev, &dmab, hext_stream); if (ret1 < 0) { dev_err(sdev->dev, "error: Code loader DSP cleanup failed\n"); @@ -522,12 +530,20 @@ int hda_dsp_post_fw_run(struct snd_sof_dev *sdev) int ret; if (sdev->first_boot) { + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; + ret = hda_sdw_startup(sdev); if (ret < 0) { dev_err(sdev->dev, "error: could not startup SoundWire links\n"); return ret; } + + /* Check if IMR boot is usable */ + if (!sof_debug_check_flag(SOF_DBG_IGNORE_D3_PERSISTENT) && + (sdev->fw_ready.flags & SOF_IPC_INFO_D3_PERSISTENT || + sdev->pdata->ipc_type == SOF_INTEL_IPC4)) + hdev->imrboot_supported = true; } hda_sdw_int_enable(sdev, true); diff --git a/sound/soc/sof/intel/hda-pcm.c b/sound/soc/sof/intel/hda-pcm.c index dc1f743730..6888e0a466 100644 --- a/sound/soc/sof/intel/hda-pcm.c +++ b/sound/soc/sof/intel/hda-pcm.c @@ -192,79 +192,7 @@ snd_pcm_uframes_t hda_dsp_pcm_pointer(struct snd_sof_dev *sdev, goto found; } - switch (sof_hda_position_quirk) { - case SOF_HDA_POSITION_QUIRK_USE_SKYLAKE_LEGACY: - /* - * This legacy code, inherited from the Skylake driver, - * mixes DPIB registers and DPIB DDR updates and - * does not seem to follow any known hardware recommendations. - * It's not clear e.g. why there is a different flow - * for capture and playback, the only information that matters is - * what traffic class is used, and on all SOF-enabled platforms - * only VC0 is supported so the work-around was likely not necessary - * and quite possibly wrong. - */ - - /* DPIB/posbuf position mode: - * For Playback, Use DPIB register from HDA space which - * reflects the actual data transferred. - * For Capture, Use the position buffer for pointer, as DPIB - * is not accurate enough, its update may be completed - * earlier than the data written to DDR. - */ - if (substream->stream == SNDRV_PCM_STREAM_PLAYBACK) { - pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, - AZX_REG_VS_SDXDPIB_XBASE + - (AZX_REG_VS_SDXDPIB_XINTERVAL * - hstream->index)); - } else { - /* - * For capture stream, we need more workaround to fix the - * position incorrect issue: - * - * 1. Wait at least 20us before reading position buffer after - * the interrupt generated(IOC), to make sure position update - * happens on frame boundary i.e. 20.833uSec for 48KHz. - * 2. Perform a dummy Read to DPIB register to flush DMA - * position value. - * 3. Read the DMA Position from posbuf. Now the readback - * value should be >= period boundary. - */ - usleep_range(20, 21); - snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, - AZX_REG_VS_SDXDPIB_XBASE + - (AZX_REG_VS_SDXDPIB_XINTERVAL * - hstream->index)); - pos = snd_hdac_stream_get_pos_posbuf(hstream); - } - break; - case SOF_HDA_POSITION_QUIRK_USE_DPIB_REGISTERS: - /* - * In case VC1 traffic is disabled this is the recommended option - */ - pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, - AZX_REG_VS_SDXDPIB_XBASE + - (AZX_REG_VS_SDXDPIB_XINTERVAL * - hstream->index)); - break; - case SOF_HDA_POSITION_QUIRK_USE_DPIB_DDR_UPDATE: - /* - * This is the recommended option when VC1 is enabled. - * While this isn't needed for SOF platforms it's added for - * consistency and debug. - */ - pos = snd_hdac_stream_get_pos_posbuf(hstream); - break; - default: - dev_err_once(sdev->dev, "hda_position_quirk value %d not supported\n", - sof_hda_position_quirk); - pos = 0; - break; - } - - if (pos >= hstream->bufsize) - pos = 0; - + pos = hda_dsp_stream_get_position(hstream, substream->stream, true); found: pos = bytes_to_frames(substream->runtime, pos); diff --git a/sound/soc/sof/intel/hda-probes.c b/sound/soc/sof/intel/hda-probes.c index 31e85d4aae..56a533c63c 100644 --- a/sound/soc/sof/intel/hda-probes.c +++ b/sound/soc/sof/intel/hda-probes.c @@ -25,9 +25,9 @@ hda_compr_get_stream(struct snd_compr_stream *cstream) return cstream->runtime->private_data; } -static int hda_probes_compr_assign(struct sof_client_dev *cdev, - struct snd_compr_stream *cstream, - struct snd_soc_dai *dai, u32 *stream_id) +static int hda_probes_compr_startup(struct sof_client_dev *cdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai, u32 *stream_id) { struct snd_sof_dev *sdev = sof_client_dev_to_sof_dev(cdev); struct hdac_ext_stream *hext_stream; @@ -45,9 +45,9 @@ static int hda_probes_compr_assign(struct sof_client_dev *cdev, return 0; } -static int hda_probes_compr_free(struct sof_client_dev *cdev, - struct snd_compr_stream *cstream, - struct snd_soc_dai *dai) +static int hda_probes_compr_shutdown(struct sof_client_dev *cdev, + struct snd_compr_stream *cstream, + struct snd_soc_dai *dai) { struct hdac_ext_stream *hext_stream = hda_compr_get_stream(cstream); struct snd_sof_dev *sdev = sof_client_dev_to_sof_dev(cdev); @@ -127,8 +127,8 @@ static int hda_probes_compr_pointer(struct sof_client_dev *cdev, /* SOF client implementation */ static const struct sof_probes_host_ops hda_probes_ops = { - .assign = hda_probes_compr_assign, - .free = hda_probes_compr_free, + .startup = hda_probes_compr_startup, + .shutdown = hda_probes_compr_shutdown, .set_params = hda_probes_compr_set_params, .trigger = hda_probes_compr_trigger, .pointer = hda_probes_compr_pointer, diff --git a/sound/soc/sof/intel/hda-stream.c b/sound/soc/sof/intel/hda-stream.c index daeb64c495..b58662faa4 100644 --- a/sound/soc/sof/intel/hda-stream.c +++ b/sound/soc/sof/intel/hda-stream.c @@ -116,13 +116,13 @@ int hda_dsp_stream_setup_bdl(struct snd_sof_dev *sdev, int remain, ioc; period_bytes = hstream->period_bytes; - dev_dbg(sdev->dev, "%s: period_bytes:0x%x\n", __func__, period_bytes); + dev_dbg(sdev->dev, "period_bytes:0x%x\n", period_bytes); if (!period_bytes) period_bytes = hstream->bufsize; periods = hstream->bufsize / period_bytes; - dev_dbg(sdev->dev, "%s: periods:%d\n", __func__, periods); + dev_dbg(sdev->dev, "periods:%d\n", periods); remain = hstream->bufsize % period_bytes; if (remain) @@ -271,7 +271,7 @@ int hda_dsp_stream_put(struct snd_sof_dev *sdev, int direction, int stream_tag) HDA_VS_INTEL_EM2_L1SEN, HDA_VS_INTEL_EM2_L1SEN); if (!found) { - dev_dbg(sdev->dev, "%s: stream_tag %d not opened!\n", + dev_err(sdev->dev, "%s: stream_tag %d not opened!\n", __func__, stream_tag); return -ENODEV; } @@ -411,6 +411,11 @@ int hda_dsp_iccmax_stream_hw_params(struct snd_sof_dev *sdev, struct hdac_ext_st return -ENODEV; } + if (!dmab) { + dev_err(sdev->dev, "error: no dma buffer allocated!\n"); + return -ENODEV; + } + if (hstream->posbuf) *hstream->posbuf = 0; @@ -485,16 +490,16 @@ int hda_dsp_stream_hw_params(struct snd_sof_dev *sdev, return -ENODEV; } - /* decouple host and link DMA */ - mask = 0x1 << hstream->index; - snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL, - mask, mask); - if (!dmab) { dev_err(sdev->dev, "error: no dma buffer allocated!\n"); return -ENODEV; } + /* decouple host and link DMA */ + mask = 0x1 << hstream->index; + snd_sof_dsp_update_bits(sdev, HDA_DSP_PP_BAR, SOF_HDA_REG_PP_PPCTL, + mask, mask); + /* clear stream status */ snd_sof_dsp_update_bits(sdev, HDA_DSP_HDA_BAR, sd_offset, SOF_HDA_CL_DMA_SD_INT_MASK | @@ -707,12 +712,13 @@ bool hda_dsp_check_stream_irq(struct snd_sof_dev *sdev) } static void -hda_dsp_set_bytes_transferred(struct hdac_stream *hstream, u64 buffer_size) +hda_dsp_compr_bytes_transferred(struct hdac_stream *hstream, int direction) { + u64 buffer_size = hstream->bufsize; u64 prev_pos, pos, num_bytes; div64_u64_rem(hstream->curr_pos, buffer_size, &prev_pos); - pos = snd_hdac_stream_get_pos_posbuf(hstream); + pos = hda_dsp_stream_get_position(hstream, direction, false); if (pos < prev_pos) num_bytes = (buffer_size - prev_pos) + pos; @@ -748,8 +754,7 @@ static bool hda_dsp_stream_check(struct hdac_bus *bus, u32 status) if (s->substream && sof_hda->no_ipc_position) { snd_sof_pcm_period_elapsed(s->substream); } else if (s->cstream) { - hda_dsp_set_bytes_transferred(s, - s->cstream->runtime->buffer_size); + hda_dsp_compr_bytes_transferred(s, s->cstream->direction); snd_compr_fragment_elapsed(s->cstream); } } @@ -1009,3 +1014,89 @@ void hda_dsp_stream_free(struct snd_sof_dev *sdev) devm_kfree(sdev->dev, hda_stream); } } + +snd_pcm_uframes_t hda_dsp_stream_get_position(struct hdac_stream *hstream, + int direction, bool can_sleep) +{ + struct hdac_ext_stream *hext_stream = stream_to_hdac_ext_stream(hstream); + struct sof_intel_hda_stream *hda_stream = hstream_to_sof_hda_stream(hext_stream); + struct snd_sof_dev *sdev = hda_stream->sdev; + snd_pcm_uframes_t pos; + + switch (sof_hda_position_quirk) { + case SOF_HDA_POSITION_QUIRK_USE_SKYLAKE_LEGACY: + /* + * This legacy code, inherited from the Skylake driver, + * mixes DPIB registers and DPIB DDR updates and + * does not seem to follow any known hardware recommendations. + * It's not clear e.g. why there is a different flow + * for capture and playback, the only information that matters is + * what traffic class is used, and on all SOF-enabled platforms + * only VC0 is supported so the work-around was likely not necessary + * and quite possibly wrong. + */ + + /* DPIB/posbuf position mode: + * For Playback, Use DPIB register from HDA space which + * reflects the actual data transferred. + * For Capture, Use the position buffer for pointer, as DPIB + * is not accurate enough, its update may be completed + * earlier than the data written to DDR. + */ + if (direction == SNDRV_PCM_STREAM_PLAYBACK) { + pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, + AZX_REG_VS_SDXDPIB_XBASE + + (AZX_REG_VS_SDXDPIB_XINTERVAL * + hstream->index)); + } else { + /* + * For capture stream, we need more workaround to fix the + * position incorrect issue: + * + * 1. Wait at least 20us before reading position buffer after + * the interrupt generated(IOC), to make sure position update + * happens on frame boundary i.e. 20.833uSec for 48KHz. + * 2. Perform a dummy Read to DPIB register to flush DMA + * position value. + * 3. Read the DMA Position from posbuf. Now the readback + * value should be >= period boundary. + */ + if (can_sleep) + usleep_range(20, 21); + + snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, + AZX_REG_VS_SDXDPIB_XBASE + + (AZX_REG_VS_SDXDPIB_XINTERVAL * + hstream->index)); + pos = snd_hdac_stream_get_pos_posbuf(hstream); + } + break; + case SOF_HDA_POSITION_QUIRK_USE_DPIB_REGISTERS: + /* + * In case VC1 traffic is disabled this is the recommended option + */ + pos = snd_sof_dsp_read(sdev, HDA_DSP_HDA_BAR, + AZX_REG_VS_SDXDPIB_XBASE + + (AZX_REG_VS_SDXDPIB_XINTERVAL * + hstream->index)); + break; + case SOF_HDA_POSITION_QUIRK_USE_DPIB_DDR_UPDATE: + /* + * This is the recommended option when VC1 is enabled. + * While this isn't needed for SOF platforms it's added for + * consistency and debug. + */ + pos = snd_hdac_stream_get_pos_posbuf(hstream); + break; + default: + dev_err_once(sdev->dev, "hda_position_quirk value %d not supported\n", + sof_hda_position_quirk); + pos = 0; + break; + } + + if (pos >= hstream->bufsize) + pos = 0; + + return pos; +} diff --git a/sound/soc/sof/intel/hda-trace.c b/sound/soc/sof/intel/hda-trace.c index 755ef1d835..cbb9bd7770 100644 --- a/sound/soc/sof/intel/hda-trace.c +++ b/sound/soc/sof/intel/hda-trace.c @@ -36,7 +36,7 @@ static int hda_dsp_trace_prepare(struct snd_sof_dev *sdev, struct snd_dma_buffer return ret; } -int hda_dsp_trace_init(struct snd_sof_dev *sdev, +int hda_dsp_trace_init(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, struct sof_ipc_dma_trace_params_ext *dtrace_params) { struct sof_intel_hda_dev *hda = sdev->pdata->hw_pdata; @@ -57,7 +57,7 @@ int hda_dsp_trace_init(struct snd_sof_dev *sdev, * initialize capture stream, set BDL address and return corresponding * stream tag which will be sent to the firmware by IPC message. */ - ret = hda_dsp_trace_prepare(sdev, &sdev->dmatb); + ret = hda_dsp_trace_prepare(sdev, dmab); if (ret < 0) { dev_err(sdev->dev, "error: hdac trace init failed: %d\n", ret); hda_dsp_stream_put(sdev, SNDRV_PCM_STREAM_CAPTURE, diff --git a/sound/soc/sof/intel/hda.c b/sound/soc/sof/intel/hda.c index 9c97c80a7f..6d4ecbe14a 100644 --- a/sound/soc/sof/intel/hda.c +++ b/sound/soc/sof/intel/hda.c @@ -147,7 +147,7 @@ static int sdw_free_stream(struct device *dev, return hda_ctrl_dai_widget_free(w, SOF_DAI_CONFIG_FLAGS_NONE, &data); } -static const struct sdw_intel_ops sdw_callback = { +struct sdw_intel_ops sdw_callback = { .params_stream = sdw_params_stream, .free_stream = sdw_free_stream, }; @@ -353,7 +353,7 @@ static inline bool hda_sdw_check_wakeen_irq(struct snd_sof_dev *sdev) struct hda_dsp_msg_code { u32 code; - const char *msg; + const char *text; }; #if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG) @@ -382,10 +382,7 @@ module_param_named(use_common_hdmi, hda_codec_use_common_hdmi, bool, 0444); MODULE_PARM_DESC(use_common_hdmi, "SOF HDA use common HDMI codec driver"); #endif -static const struct hda_dsp_msg_code hda_dsp_rom_msg[] = { - {HDA_DSP_ROM_FW_MANIFEST_LOADED, "status: manifest loaded"}, - {HDA_DSP_ROM_FW_FW_LOADED, "status: fw loaded"}, - {HDA_DSP_ROM_FW_ENTERED, "status: fw entered"}, +static const struct hda_dsp_msg_code hda_dsp_rom_fw_error_texts[] = { {HDA_DSP_ROM_CSE_ERROR, "error: cse error"}, {HDA_DSP_ROM_CSE_WRONG_RESPONSE, "error: cse wrong response"}, {HDA_DSP_ROM_IMR_TO_SMALL, "error: IMR too small"}, @@ -404,24 +401,136 @@ static const struct hda_dsp_msg_code hda_dsp_rom_msg[] = { {HDA_DSP_ROM_NULL_FW_ENTRY, "error: null FW entry point"}, }; -static void hda_dsp_get_status(struct snd_sof_dev *sdev, const char *level) +#define FSR_ROM_STATE_ENTRY(state) {FSR_STATE_ROM_##state, #state} +static const struct hda_dsp_msg_code fsr_rom_state_names[] = { + FSR_ROM_STATE_ENTRY(INIT), + FSR_ROM_STATE_ENTRY(INIT_DONE), + FSR_ROM_STATE_ENTRY(CSE_MANIFEST_LOADED), + FSR_ROM_STATE_ENTRY(FW_MANIFEST_LOADED), + FSR_ROM_STATE_ENTRY(FW_FW_LOADED), + FSR_ROM_STATE_ENTRY(FW_ENTERED), + FSR_ROM_STATE_ENTRY(VERIFY_FEATURE_MASK), + FSR_ROM_STATE_ENTRY(GET_LOAD_OFFSET), + FSR_ROM_STATE_ENTRY(FETCH_ROM_EXT), + FSR_ROM_STATE_ENTRY(FETCH_ROM_EXT_DONE), + /* CSE states */ + FSR_ROM_STATE_ENTRY(CSE_IMR_REQUEST), + FSR_ROM_STATE_ENTRY(CSE_IMR_GRANTED), + FSR_ROM_STATE_ENTRY(CSE_VALIDATE_IMAGE_REQUEST), + FSR_ROM_STATE_ENTRY(CSE_IMAGE_VALIDATED), + FSR_ROM_STATE_ENTRY(CSE_IPC_IFACE_INIT), + FSR_ROM_STATE_ENTRY(CSE_IPC_RESET_PHASE_1), + FSR_ROM_STATE_ENTRY(CSE_IPC_OPERATIONAL_ENTRY), + FSR_ROM_STATE_ENTRY(CSE_IPC_OPERATIONAL), + FSR_ROM_STATE_ENTRY(CSE_IPC_DOWN), +}; + +#define FSR_BRINGUP_STATE_ENTRY(state) {FSR_STATE_BRINGUP_##state, #state} +static const struct hda_dsp_msg_code fsr_bringup_state_names[] = { + FSR_BRINGUP_STATE_ENTRY(INIT), + FSR_BRINGUP_STATE_ENTRY(INIT_DONE), + FSR_BRINGUP_STATE_ENTRY(HPSRAM_LOAD), + FSR_BRINGUP_STATE_ENTRY(UNPACK_START), + FSR_BRINGUP_STATE_ENTRY(IMR_RESTORE), + FSR_BRINGUP_STATE_ENTRY(FW_ENTERED), +}; + +#define FSR_WAIT_STATE_ENTRY(state) {FSR_WAIT_FOR_##state, #state} +static const struct hda_dsp_msg_code fsr_wait_state_names[] = { + FSR_WAIT_STATE_ENTRY(IPC_BUSY), + FSR_WAIT_STATE_ENTRY(IPC_DONE), + FSR_WAIT_STATE_ENTRY(CACHE_INVALIDATION), + FSR_WAIT_STATE_ENTRY(LP_SRAM_OFF), + FSR_WAIT_STATE_ENTRY(DMA_BUFFER_FULL), + FSR_WAIT_STATE_ENTRY(CSE_CSR), +}; + +#define FSR_MODULE_NAME_ENTRY(mod) [FSR_MOD_##mod] = #mod +static const char * const fsr_module_names[] = { + FSR_MODULE_NAME_ENTRY(ROM), + FSR_MODULE_NAME_ENTRY(ROM_BYP), + FSR_MODULE_NAME_ENTRY(BASE_FW), + FSR_MODULE_NAME_ENTRY(LP_BOOT), + FSR_MODULE_NAME_ENTRY(BRNGUP), + FSR_MODULE_NAME_ENTRY(ROM_EXT), +}; + +static const char * +hda_dsp_get_state_text(u32 code, const struct hda_dsp_msg_code *msg_code, + size_t array_size) { - u32 status; int i; - status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, - HDA_DSP_SRAM_REG_ROM_STATUS); - - for (i = 0; i < ARRAY_SIZE(hda_dsp_rom_msg); i++) { - if (status == hda_dsp_rom_msg[i].code) { - dev_printk(level, sdev->dev, "%s - code %8.8x\n", - hda_dsp_rom_msg[i].msg, status); - return; - } + for (i = 0; i < array_size; i++) { + if (code == msg_code[i].code) + return msg_code[i].text; } + return NULL; +} + +static void hda_dsp_get_state(struct snd_sof_dev *sdev, const char *level) +{ + const struct sof_intel_dsp_desc *chip = get_chip_info(sdev->pdata); + const char *state_text, *error_text, *module_text; + u32 fsr, state, wait_state, module, error_code; + + fsr = snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg); + state = FSR_TO_STATE_CODE(fsr); + wait_state = FSR_TO_WAIT_STATE_CODE(fsr); + module = FSR_TO_MODULE_CODE(fsr); + + if (module > FSR_MOD_ROM_EXT) + module_text = "unknown"; + else + module_text = fsr_module_names[module]; + + if (module == FSR_MOD_BRNGUP) + state_text = hda_dsp_get_state_text(state, fsr_bringup_state_names, + ARRAY_SIZE(fsr_bringup_state_names)); + else + state_text = hda_dsp_get_state_text(state, fsr_rom_state_names, + ARRAY_SIZE(fsr_rom_state_names)); + /* not for us, must be generic sof message */ - dev_dbg(sdev->dev, "unknown ROM status value %8.8x\n", status); + if (!state_text) { + dev_printk(level, sdev->dev, "%#010x: unknown ROM status value\n", fsr); + return; + } + + if (wait_state) { + const char *wait_state_text; + + wait_state_text = hda_dsp_get_state_text(wait_state, fsr_wait_state_names, + ARRAY_SIZE(fsr_wait_state_names)); + if (!wait_state_text) + wait_state_text = "unknown"; + + dev_printk(level, sdev->dev, + "%#010x: module: %s, state: %s, waiting for: %s, %s\n", + fsr, module_text, state_text, wait_state_text, + fsr & FSR_HALTED ? "not running" : "running"); + } else { + dev_printk(level, sdev->dev, "%#010x: module: %s, state: %s, %s\n", + fsr, module_text, state_text, + fsr & FSR_HALTED ? "not running" : "running"); + } + + error_code = snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg + 4); + if (!error_code) + return; + + error_text = hda_dsp_get_state_text(error_code, hda_dsp_rom_fw_error_texts, + ARRAY_SIZE(hda_dsp_rom_fw_error_texts)); + if (!error_text) + error_text = "unknown"; + + if (state == FSR_STATE_FW_ENTERED) + dev_printk(level, sdev->dev, "status code: %#x (%s)\n", error_code, + error_text); + else + dev_printk(level, sdev->dev, "error code: %#x (%s)\n", error_code, + error_text); } static void hda_dsp_get_registers(struct snd_sof_dev *sdev, @@ -456,14 +565,16 @@ static void hda_dsp_get_registers(struct snd_sof_dev *sdev, static void hda_dsp_dump_ext_rom_status(struct snd_sof_dev *sdev, const char *level, u32 flags) { + const struct sof_intel_dsp_desc *chip; char msg[128]; int len = 0; u32 value; int i; + chip = get_chip_info(sdev->pdata); for (i = 0; i < HDA_EXT_ROM_STATUS_SIZE; i++) { - value = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_ROM_STATUS + i * 0x4); - len += snprintf(msg + len, sizeof(msg) - len, " 0x%x", value); + value = snd_sof_dsp_read(sdev, HDA_DSP_BAR, chip->rom_status_reg + i * 0x4); + len += scnprintf(msg + len, sizeof(msg) - len, " 0x%x", value); } dev_printk(level, sdev->dev, "extended rom status: %s", msg); @@ -478,7 +589,7 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags) u32 stack[HDA_DSP_STACK_DUMP_SIZE]; /* print ROM/FW status */ - hda_dsp_get_status(sdev, level); + hda_dsp_get_state(sdev, level); if (flags & SOF_DBG_DUMP_REGS) { u32 status = snd_sof_dsp_read(sdev, HDA_DSP_BAR, HDA_DSP_SRAM_REG_FW_STATUS); @@ -493,6 +604,17 @@ void hda_dsp_dump(struct snd_sof_dev *sdev, u32 flags) } } +static bool hda_check_ipc_irq(struct snd_sof_dev *sdev) +{ + const struct sof_intel_dsp_desc *chip; + + chip = get_chip_info(sdev->pdata); + if (chip && chip->check_ipc_irq) + return chip->check_ipc_irq(sdev); + + return false; +} + void hda_ipc_irq_dump(struct snd_sof_dev *sdev) { struct hdac_bus *bus = sof_to_bus(sdev); @@ -584,14 +706,13 @@ static int hda_init(struct snd_sof_dev *sdev) static int check_dmic_num(struct snd_sof_dev *sdev) { + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; struct nhlt_acpi_table *nhlt; int dmic_num = 0; - nhlt = intel_nhlt_init(sdev->dev); - if (nhlt) { + nhlt = hdev->nhlt; + if (nhlt) dmic_num = intel_nhlt_get_dmic_geo(sdev->dev, nhlt); - intel_nhlt_free(nhlt); - } /* allow for module parameter override */ if (dmic_num_override != -1) { @@ -611,10 +732,11 @@ static int check_dmic_num(struct snd_sof_dev *sdev) static int check_nhlt_ssp_mask(struct snd_sof_dev *sdev) { + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; struct nhlt_acpi_table *nhlt; int ssp_mask = 0; - nhlt = intel_nhlt_init(sdev->dev); + nhlt = hdev->nhlt; if (!nhlt) return ssp_mask; @@ -623,7 +745,6 @@ static int check_nhlt_ssp_mask(struct snd_sof_dev *sdev) if (ssp_mask) dev_info(sdev->dev, "NHLT_DEVICE_I2S detected, ssp_mask %#x\n", ssp_mask); } - intel_nhlt_free(nhlt); return ssp_mask; } @@ -655,13 +776,12 @@ static const char *fixup_tplg_name(struct snd_sof_dev *sdev, return tplg_filename; } -static int dmic_topology_fixup(struct snd_sof_dev *sdev, - const char **tplg_filename, - const char *idisp_str, - int *dmic_found) +static int dmic_detect_topology_fixup(struct snd_sof_dev *sdev, + const char **tplg_filename, + const char *idisp_str, + int *dmic_found, + bool tplg_fixup) { - const char *default_tplg_filename = *tplg_filename; - const char *fixed_tplg_filename; const char *dmic_str; int dmic_num; @@ -687,14 +807,19 @@ static int dmic_topology_fixup(struct snd_sof_dev *sdev, break; } - fixed_tplg_filename = fixup_tplg_name(sdev, default_tplg_filename, - idisp_str, dmic_str); - if (!fixed_tplg_filename) - return -ENOMEM; + if (tplg_fixup) { + const char *default_tplg_filename = *tplg_filename; + const char *fixed_tplg_filename; + + fixed_tplg_filename = fixup_tplg_name(sdev, default_tplg_filename, + idisp_str, dmic_str); + if (!fixed_tplg_filename) + return -ENOMEM; + *tplg_filename = fixed_tplg_filename; + } dev_info(sdev->dev, "DMICs detected in NHLT tables: %d\n", dmic_num); *dmic_found = dmic_num; - *tplg_filename = fixed_tplg_filename; return 0; } @@ -816,7 +941,7 @@ static irqreturn_t hda_dsp_interrupt_thread(int irq, void *context) if (hda_dsp_check_stream_irq(sdev)) hda_dsp_stream_threaded_handler(irq, sdev); - if (hda_dsp_check_ipc_irq(sdev)) + if (hda_check_ipc_irq(sdev)) sof_ops(sdev)->irq_thread(irq, sdev); if (hda_dsp_check_sdw_irq(sdev)) @@ -984,6 +1109,8 @@ int hda_dsp_probe(struct snd_sof_dev *sdev) INIT_DELAYED_WORK(&hdev->d0i3_work, hda_dsp_d0i3_work); + hdev->nhlt = intel_nhlt_init(sdev->dev); + return 0; free_ipc_irq: @@ -1009,6 +1136,10 @@ int hda_dsp_remove(struct snd_sof_dev *sdev) const struct sof_intel_dsp_desc *chip = hda->desc; struct hdac_bus *bus = sof_to_bus(sdev); struct pci_dev *pci = to_pci_dev(sdev->dev); + struct nhlt_acpi_table *nhlt = hda->nhlt; + + if (nhlt) + intel_nhlt_free(nhlt); /* cancel any attempt for DSP D0I3 */ cancel_delayed_work_sync(&hda->d0i3_work); @@ -1094,6 +1225,8 @@ static void hda_generic_machine_select(struct snd_sof_dev *sdev, * - one external HDAudio codec */ if (!*mach && codec_num <= 2) { + bool tplg_fixup; + hda_mach = snd_soc_acpi_intel_hda_machines; dev_info(bus->dev, "using HDA machine driver %s now\n", @@ -1105,8 +1238,15 @@ static void hda_generic_machine_select(struct snd_sof_dev *sdev, idisp_str = ""; /* topology: use the info from hda_machines */ - tplg_filename = hda_mach->sof_tplg_filename; - ret = dmic_topology_fixup(sdev, &tplg_filename, idisp_str, &dmic_num); + if (pdata->tplg_filename) { + tplg_fixup = false; + tplg_filename = pdata->tplg_filename; + } else { + tplg_fixup = true; + tplg_filename = hda_mach->sof_tplg_filename; + } + ret = dmic_detect_topology_fixup(sdev, &tplg_filename, idisp_str, &dmic_num, + tplg_fixup); if (ret < 0) return; @@ -1270,30 +1410,37 @@ static struct snd_soc_acpi_mach *hda_sdw_machine_select(struct snd_sof_dev *sdev } if (mach && mach->link_mask) { int dmic_num = 0; + bool tplg_fixup; + const char *tplg_filename; mach->mach_params.links = mach->links; mach->mach_params.link_mask = mach->link_mask; mach->mach_params.platform = dev_name(sdev->dev); - pdata->fw_filename = pdata->desc->default_fw_filename; - pdata->tplg_filename = mach->sof_tplg_filename; + + if (pdata->tplg_filename) { + tplg_fixup = false; + } else { + tplg_fixup = true; + tplg_filename = mach->sof_tplg_filename; + } /* * DMICs use up to 4 pins and are typically pin-muxed with SoundWire - * link 2 and 3, thus we only try to enable dmics if all conditions - * are true: - * a) link 2 and 3 are not used by SoundWire + * link 2 and 3, or link 1 and 2, thus we only try to enable dmics + * if all conditions are true: + * a) 2 or fewer links are used by SoundWire * b) the NHLT table reports the presence of microphones */ - if (!(mach->link_mask & GENMASK(3, 2))) { - const char *tplg_filename = mach->sof_tplg_filename; + if (hweight_long(mach->link_mask) <= 2) { int ret; - ret = dmic_topology_fixup(sdev, &tplg_filename, "", &dmic_num); + ret = dmic_detect_topology_fixup(sdev, &tplg_filename, "", + &dmic_num, tplg_fixup); if (ret < 0) return NULL; - - pdata->tplg_filename = tplg_filename; } + if (tplg_fixup) + pdata->tplg_filename = tplg_filename; mach->mach_params.dmic_num = dmic_num; dev_dbg(sdev->dev, @@ -1339,18 +1486,22 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) mach = snd_soc_acpi_find_machine(desc->machines); if (mach) { bool add_extension = false; + bool tplg_fixup = false; /* * If tplg file name is overridden, use it instead of * the one set in mach table */ - if (!sof_pdata->tplg_filename) + if (!sof_pdata->tplg_filename) { sof_pdata->tplg_filename = mach->sof_tplg_filename; + tplg_fixup = true; + } /* report to machine driver if any DMICs are found */ mach->mach_params.dmic_num = check_dmic_num(sdev); - if (mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_DMIC_NUMBER && + if (tplg_fixup && + mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_DMIC_NUMBER && mach->mach_params.dmic_num) { tplg_filename = devm_kasprintf(sdev->dev, GFP_KERNEL, "%s%s%d%s", @@ -1373,8 +1524,10 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) /* report SSP link mask to machine driver */ mach->mach_params.i2s_link_mask = check_nhlt_ssp_mask(sdev); - if (mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_SSP_NUMBER && + if (tplg_fixup && + mach->tplg_quirk_mask & SND_SOC_ACPI_TPLG_INTEL_SSP_NUMBER && mach->mach_params.i2s_link_mask) { + const struct sof_intel_dsp_desc *chip = get_chip_info(sdev->pdata); int ssp_num; if (hweight_long(mach->mach_params.i2s_link_mask) > 1 && @@ -1384,6 +1537,12 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) /* fls returns 1-based results, SSPs indices are 0-based */ ssp_num = fls(mach->mach_params.i2s_link_mask) - 1; + if (ssp_num >= chip->ssp_count) { + dev_err(sdev->dev, "Invalid SSP %d, max on this platform is %d\n", + ssp_num, chip->ssp_count); + return NULL; + } + tplg_filename = devm_kasprintf(sdev->dev, GFP_KERNEL, "%s%s%d", sof_pdata->tplg_filename, @@ -1396,7 +1555,7 @@ struct snd_soc_acpi_mach *hda_machine_select(struct snd_sof_dev *sdev) add_extension = true; } - if (add_extension) { + if (tplg_fixup && add_extension) { tplg_filename = devm_kasprintf(sdev->dev, GFP_KERNEL, "%s%s", sof_pdata->tplg_filename, diff --git a/sound/soc/sof/intel/hda.h b/sound/soc/sof/intel/hda.h index 05e5e15861..5ef3e8775e 100644 --- a/sound/soc/sof/intel/hda.h +++ b/sound/soc/sof/intel/hda.h @@ -187,6 +187,69 @@ #define HDA_DSP_STACK_DUMP_SIZE 32 +/* ROM/FW status register */ +#define FSR_STATE_MASK GENMASK(23, 0) +#define FSR_WAIT_STATE_MASK GENMASK(27, 24) +#define FSR_MODULE_MASK GENMASK(30, 28) +#define FSR_HALTED BIT(31) +#define FSR_TO_STATE_CODE(x) ((x) & FSR_STATE_MASK) +#define FSR_TO_WAIT_STATE_CODE(x) (((x) & FSR_WAIT_STATE_MASK) >> 24) +#define FSR_TO_MODULE_CODE(x) (((x) & FSR_MODULE_MASK) >> 28) + +/* Wait states */ +#define FSR_WAIT_FOR_IPC_BUSY 0x1 +#define FSR_WAIT_FOR_IPC_DONE 0x2 +#define FSR_WAIT_FOR_CACHE_INVALIDATION 0x3 +#define FSR_WAIT_FOR_LP_SRAM_OFF 0x4 +#define FSR_WAIT_FOR_DMA_BUFFER_FULL 0x5 +#define FSR_WAIT_FOR_CSE_CSR 0x6 + +/* Module codes */ +#define FSR_MOD_ROM 0x0 +#define FSR_MOD_ROM_BYP 0x1 +#define FSR_MOD_BASE_FW 0x2 +#define FSR_MOD_LP_BOOT 0x3 +#define FSR_MOD_BRNGUP 0x4 +#define FSR_MOD_ROM_EXT 0x5 + +/* State codes (module dependent) */ +/* Module independent states */ +#define FSR_STATE_INIT 0x0 +#define FSR_STATE_INIT_DONE 0x1 +#define FSR_STATE_FW_ENTERED 0x5 + +/* ROM states */ +#define FSR_STATE_ROM_INIT FSR_STATE_INIT +#define FSR_STATE_ROM_INIT_DONE FSR_STATE_INIT_DONE +#define FSR_STATE_ROM_CSE_MANIFEST_LOADED 0x2 +#define FSR_STATE_ROM_FW_MANIFEST_LOADED 0x3 +#define FSR_STATE_ROM_FW_FW_LOADED 0x4 +#define FSR_STATE_ROM_FW_ENTERED FSR_STATE_FW_ENTERED +#define FSR_STATE_ROM_VERIFY_FEATURE_MASK 0x6 +#define FSR_STATE_ROM_GET_LOAD_OFFSET 0x7 +#define FSR_STATE_ROM_FETCH_ROM_EXT 0x8 +#define FSR_STATE_ROM_FETCH_ROM_EXT_DONE 0x9 + +/* (ROM) CSE states */ +#define FSR_STATE_ROM_CSE_IMR_REQUEST 0x10 +#define FSR_STATE_ROM_CSE_IMR_GRANTED 0x11 +#define FSR_STATE_ROM_CSE_VALIDATE_IMAGE_REQUEST 0x12 +#define FSR_STATE_ROM_CSE_IMAGE_VALIDATED 0x13 + +#define FSR_STATE_ROM_CSE_IPC_IFACE_INIT 0x20 +#define FSR_STATE_ROM_CSE_IPC_RESET_PHASE_1 0x21 +#define FSR_STATE_ROM_CSE_IPC_OPERATIONAL_ENTRY 0x22 +#define FSR_STATE_ROM_CSE_IPC_OPERATIONAL 0x23 +#define FSR_STATE_ROM_CSE_IPC_DOWN 0x24 + +/* BRINGUP (or BRNGUP) states */ +#define FSR_STATE_BRINGUP_INIT FSR_STATE_INIT +#define FSR_STATE_BRINGUP_INIT_DONE FSR_STATE_INIT_DONE +#define FSR_STATE_BRINGUP_HPSRAM_LOAD 0x2 +#define FSR_STATE_BRINGUP_UNPACK_START 0X3 +#define FSR_STATE_BRINGUP_IMR_RESTORE 0x4 +#define FSR_STATE_BRINGUP_FW_ENTERED FSR_STATE_FW_ENTERED + /* ROM status/error values */ #define HDA_DSP_ROM_STS_MASK GENMASK(23, 0) #define HDA_DSP_ROM_INIT 0x1 @@ -210,7 +273,9 @@ #define HDA_DSP_ROM_USER_EXCEPTION 0xBEEF0000 #define HDA_DSP_ROM_UNEXPECTED_RESET 0xDECAF000 #define HDA_DSP_ROM_NULL_FW_ENTRY 0x4c4c4e55 -#define HDA_DSP_IPC_PURGE_FW 0x01004000 + +#define HDA_DSP_ROM_IPC_CONTROL 0x01000000 +#define HDA_DSP_ROM_IPC_PURGE_FW 0x00004000 /* various timeout values */ #define HDA_DSP_PU_TIMEOUT 50 @@ -223,8 +288,8 @@ #define HDA_DSP_REG_POLL_INTERVAL_US 500 /* 0.5 msec */ #define HDA_DSP_REG_POLL_RETRY_COUNT 50 -#define HDA_DSP_ADSPIC_IPC 1 -#define HDA_DSP_ADSPIS_IPC 1 +#define HDA_DSP_ADSPIC_IPC BIT(0) +#define HDA_DSP_ADSPIS_IPC BIT(0) /* Intel HD Audio General DSP Registers */ #define HDA_DSP_GEN_BASE 0x0 @@ -268,8 +333,8 @@ /* HIPCTE */ #define HDA_DSP_REG_HIPCTE_MSG_MASK 0x3FFFFFFF -#define HDA_DSP_ADSPIC_CL_DMA 0x2 -#define HDA_DSP_ADSPIS_CL_DMA 0x2 +#define HDA_DSP_ADSPIC_CL_DMA BIT(1) +#define HDA_DSP_ADSPIS_CL_DMA BIT(1) /* Delay before scheduling D0i3 entry */ #define BXT_D0I3_DELAY 5000 @@ -416,6 +481,9 @@ enum sof_hda_D0_substate { /* represents DSP HDA controller frontend - i.e. host facing control */ struct sof_intel_hda_dev { + bool imrboot_supported; + bool skip_imr_boot; + int boot_iteration; struct hda_bus hbus; @@ -449,6 +517,9 @@ struct sof_intel_hda_dev { /* FW clock config, 0:HPRO, 1:LPRO */ bool clk_config_lpro; + + /* Intel NHLT information */ + struct nhlt_acpi_table *nhlt; }; static inline struct hdac_bus *sof_to_bus(struct snd_sof_dev *s) @@ -490,6 +561,7 @@ struct sof_intel_hda_stream { */ int hda_dsp_probe(struct snd_sof_dev *sdev); int hda_dsp_remove(struct snd_sof_dev *sdev); +int hda_dsp_core_power_up(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_core_run(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_enable_core(struct snd_sof_dev *sdev, unsigned int core_mask); int hda_dsp_core_reset_power_down(struct snd_sof_dev *sdev, @@ -557,6 +629,9 @@ int hda_dsp_stream_setup_bdl(struct snd_sof_dev *sdev, bool hda_dsp_check_ipc_irq(struct snd_sof_dev *sdev); bool hda_dsp_check_stream_irq(struct snd_sof_dev *sdev); +snd_pcm_uframes_t hda_dsp_stream_get_position(struct hdac_stream *hstream, + int direction, bool can_sleep); + struct hdac_ext_stream * hda_dsp_stream_get(struct snd_sof_dev *sdev, int direction, u32 flags); int hda_dsp_stream_put(struct snd_sof_dev *sdev, int direction, int stream_tag); @@ -588,6 +663,14 @@ int hda_dsp_ipc_cmd_done(struct snd_sof_dev *sdev, int dir); */ int hda_dsp_cl_boot_firmware(struct snd_sof_dev *sdev); int hda_dsp_cl_boot_firmware_iccmax(struct snd_sof_dev *sdev); +int hda_cl_copy_fw(struct snd_sof_dev *sdev, struct hdac_ext_stream *hext_stream); +struct hdac_ext_stream *hda_cl_stream_prepare(struct snd_sof_dev *sdev, unsigned int format, + unsigned int size, struct snd_dma_buffer *dmab, + int direction); +int hda_cl_cleanup(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, + struct hdac_ext_stream *hext_stream); +int cl_dsp_init(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot); +#define HDA_CL_STREAM_FORMAT 0x40 /* pre and post fw run ops */ int hda_dsp_pre_fw_run(struct snd_sof_dev *sdev); @@ -644,7 +727,7 @@ static inline int hda_codec_i915_exit(struct snd_sof_dev *sdev) { return 0; } /* * Trace Control. */ -int hda_dsp_trace_init(struct snd_sof_dev *sdev, +int hda_dsp_trace_init(struct snd_sof_dev *sdev, struct snd_dma_buffer *dmab, struct sof_ipc_dma_trace_params_ext *dtrace_params); int hda_dsp_trace_release(struct snd_sof_dev *sdev); int hda_dsp_trace_trigger(struct snd_sof_dev *sdev, int cmd); @@ -683,24 +766,33 @@ static inline bool hda_common_check_sdw_irq(struct snd_sof_dev *sdev) /* common dai driver */ extern struct snd_soc_dai_driver skl_dai[]; +int hda_dsp_dais_suspend(struct snd_sof_dev *sdev); /* * Platform Specific HW abstraction Ops. */ -extern const struct snd_sof_dsp_ops sof_apl_ops; -extern const struct snd_sof_dsp_ops sof_cnl_ops; -extern const struct snd_sof_dsp_ops sof_tgl_ops; -extern const struct snd_sof_dsp_ops sof_icl_ops; +extern struct snd_sof_dsp_ops sof_hda_common_ops; + +extern struct snd_sof_dsp_ops sof_apl_ops; +int sof_apl_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_cnl_ops; +int sof_cnl_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_tgl_ops; +int sof_tgl_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_icl_ops; +int sof_icl_ops_init(struct snd_sof_dev *sdev); +extern struct snd_sof_dsp_ops sof_mtl_ops; +int sof_mtl_ops_init(struct snd_sof_dev *sdev); extern const struct sof_intel_dsp_desc apl_chip_info; extern const struct sof_intel_dsp_desc cnl_chip_info; -extern const struct sof_intel_dsp_desc skl_chip_info; extern const struct sof_intel_dsp_desc icl_chip_info; extern const struct sof_intel_dsp_desc tgl_chip_info; extern const struct sof_intel_dsp_desc tglh_chip_info; extern const struct sof_intel_dsp_desc ehl_chip_info; extern const struct sof_intel_dsp_desc jsl_chip_info; extern const struct sof_intel_dsp_desc adls_chip_info; +extern const struct sof_intel_dsp_desc mtl_chip_info; /* Probes support */ #if IS_ENABLED(CONFIG_SND_SOC_SOF_HDA_PROBES) @@ -742,4 +834,14 @@ int hda_ctrl_dai_widget_free(struct snd_soc_dapm_widget *w, unsigned int quirk_f extern int sof_hda_position_quirk; +void hda_set_dai_drv_ops(struct snd_sof_dev *sdev, struct snd_sof_dsp_ops *ops); +void hda_ops_free(struct snd_sof_dev *sdev); + +/* IPC4 */ +irqreturn_t cnl_ipc4_irq_thread(int irq, void *context); +int cnl_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg); +irqreturn_t hda_dsp_ipc4_irq_thread(int irq, void *context); +int hda_dsp_ipc4_send_msg(struct snd_sof_dev *sdev, struct snd_sof_ipc_msg *msg); +extern struct sdw_intel_ops sdw_callback; + #endif diff --git a/sound/soc/sof/intel/icl.c b/sound/soc/sof/intel/icl.c index b44a649bfc..4e37b7fe06 100644 --- a/sound/soc/sof/intel/icl.c +++ b/sound/soc/sof/intel/icl.c @@ -56,11 +56,18 @@ static int icl_dsp_post_fw_run(struct snd_sof_dev *sdev) int ret; if (sdev->first_boot) { + struct sof_intel_hda_dev *hdev = sdev->pdata->hw_pdata; + ret = hda_sdw_startup(sdev); if (ret < 0) { dev_err(sdev->dev, "error: could not startup SoundWire links\n"); return ret; } + + /* Check if IMR boot is usable */ + if (!sof_debug_check_flag(SOF_DBG_IGNORE_D3_PERSISTENT) && + sdev->fw_ready.flags & SOF_IPC_INFO_D3_PERSISTENT) + hdev->imrboot_supported = true; } hda_sdw_int_enable(sdev, true); @@ -88,109 +95,44 @@ static int icl_dsp_post_fw_run(struct snd_sof_dev *sdev) } /* Icelake ops */ -const struct snd_sof_dsp_ops sof_icl_ops = { +struct snd_sof_dsp_ops sof_icl_ops; +EXPORT_SYMBOL_NS(sof_icl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); + +int sof_icl_ops_init(struct snd_sof_dev *sdev) +{ + /* common defaults */ + memcpy(&sof_icl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops)); + /* probe/remove/shutdown */ - .probe = hda_dsp_probe, - .remove = hda_dsp_remove, - .shutdown = hda_dsp_shutdown, - - /* Register IO */ - .write = sof_io_write, - .read = sof_io_read, - .write64 = sof_io_write64, - .read64 = sof_io_read64, - - /* Block IO */ - .block_read = sof_block_read, - .block_write = sof_block_write, - - /* Mailbox IO */ - .mailbox_read = sof_mailbox_read, - .mailbox_write = sof_mailbox_write, + sof_icl_ops.shutdown = hda_dsp_shutdown; /* doorbell */ - .irq_thread = cnl_ipc_irq_thread, + sof_icl_ops.irq_thread = cnl_ipc_irq_thread; /* ipc */ - .send_msg = cnl_ipc_send_msg, - .fw_ready = sof_fw_ready, - .get_mailbox_offset = hda_dsp_ipc_get_mailbox_offset, - .get_window_offset = hda_dsp_ipc_get_window_offset, - - .ipc_msg_data = hda_ipc_msg_data, - .set_stream_data_offset = hda_set_stream_data_offset, - - /* machine driver */ - .machine_select = hda_machine_select, - .machine_register = sof_machine_register, - .machine_unregister = sof_machine_unregister, - .set_mach_params = hda_set_mach_params, + sof_icl_ops.send_msg = cnl_ipc_send_msg; /* debug */ - .debug_map = icl_dsp_debugfs, - .debug_map_count = ARRAY_SIZE(icl_dsp_debugfs), - .dbg_dump = hda_dsp_dump, - .ipc_dump = cnl_ipc_dump, - .debugfs_add_region_item = snd_sof_debugfs_add_region_item_iomem, - - /* stream callbacks */ - .pcm_open = hda_dsp_pcm_open, - .pcm_close = hda_dsp_pcm_close, - .pcm_hw_params = hda_dsp_pcm_hw_params, - .pcm_hw_free = hda_dsp_stream_hw_free, - .pcm_trigger = hda_dsp_pcm_trigger, - .pcm_pointer = hda_dsp_pcm_pointer, - .pcm_ack = hda_dsp_pcm_ack, - - /* firmware loading */ - .load_firmware = snd_sof_load_firmware_raw, + sof_icl_ops.debug_map = icl_dsp_debugfs; + sof_icl_ops.debug_map_count = ARRAY_SIZE(icl_dsp_debugfs); + sof_icl_ops.ipc_dump = cnl_ipc_dump; /* pre/post fw run */ - .pre_fw_run = hda_dsp_pre_fw_run, - .post_fw_run = icl_dsp_post_fw_run, - - /* parse platform specific extended manifest */ - .parse_platform_ext_manifest = hda_dsp_ext_man_get_cavs_config_data, - - /* dsp core get/put */ - .core_get = hda_dsp_core_get, + sof_icl_ops.post_fw_run = icl_dsp_post_fw_run; /* firmware run */ - .run = hda_dsp_cl_boot_firmware_iccmax, - .stall = icl_dsp_core_stall, + sof_icl_ops.run = hda_dsp_cl_boot_firmware_iccmax; + sof_icl_ops.stall = icl_dsp_core_stall; - /* trace callback */ - .trace_init = hda_dsp_trace_init, - .trace_release = hda_dsp_trace_release, - .trace_trigger = hda_dsp_trace_trigger, + /* dsp core get/put */ + sof_icl_ops.core_get = hda_dsp_core_get; - /* client ops */ - .register_ipc_clients = hda_register_clients, - .unregister_ipc_clients = hda_unregister_clients, + /* set DAI driver ops */ + hda_set_dai_drv_ops(sdev, &sof_icl_ops); - /* DAI drivers */ - .drv = skl_dai, - .num_drv = SOF_SKL_NUM_DAIS, - - /* PM */ - .suspend = hda_dsp_suspend, - .resume = hda_dsp_resume, - .runtime_suspend = hda_dsp_runtime_suspend, - .runtime_resume = hda_dsp_runtime_resume, - .runtime_idle = hda_dsp_runtime_idle, - .set_hw_params_upon_resume = hda_dsp_set_hw_params_upon_resume, - .set_power_state = hda_dsp_set_power_state, - - /* ALSA HW info flags */ - .hw_info = SNDRV_PCM_INFO_MMAP | - SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED | - SNDRV_PCM_INFO_PAUSE | - SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, - - .dsp_arch_ops = &sof_xtensa_arch_ops, + return 0; }; -EXPORT_SYMBOL_NS(sof_icl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); +EXPORT_SYMBOL_NS(sof_icl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON); const struct sof_intel_dsp_desc icl_chip_info = { /* Icelake */ @@ -202,11 +144,15 @@ const struct sof_intel_dsp_desc icl_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = ICL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_2_0, }; EXPORT_SYMBOL_NS(icl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/intel/pci-apl.c b/sound/soc/sof/intel/pci-apl.c index a023b3cc0a..998e219011 100644 --- a/sound/soc/sof/intel/pci-apl.c +++ b/sound/soc/sof/intel/pci-apl.c @@ -27,11 +27,24 @@ static const struct sof_dev_desc bxt_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &apl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-apl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/apl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-apl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-apl-nocodec.tplg", .ops = &sof_apl_ops, + .ops_init = sof_apl_ops_init, + .ops_free = hda_ops_free, }; static const struct sof_dev_desc glk_desc = { @@ -42,11 +55,23 @@ static const struct sof_dev_desc glk_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &apl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-glk.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/glk", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-glk.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-glk-nocodec.tplg", .ops = &sof_apl_ops, + .ops_init = sof_apl_ops_init, }; /* PCI IDs */ diff --git a/sound/soc/sof/intel/pci-cnl.c b/sound/soc/sof/intel/pci-cnl.c index 40cf1cd000..c797356f70 100644 --- a/sound/soc/sof/intel/pci-cnl.c +++ b/sound/soc/sof/intel/pci-cnl.c @@ -28,11 +28,23 @@ static const struct sof_dev_desc cnl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &cnl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-cnl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/cnl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-cnl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-cnl-nocodec.tplg", .ops = &sof_cnl_ops, + .ops_init = sof_cnl_ops_init, }; static const struct sof_dev_desc cfl_desc = { @@ -44,11 +56,24 @@ static const struct sof_dev_desc cfl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &cnl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-cfl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/cnl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-cfl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-cnl-nocodec.tplg", .ops = &sof_cnl_ops, + .ops_init = sof_cnl_ops_init, + .ops_free = hda_ops_free, }; static const struct sof_dev_desc cml_desc = { @@ -60,11 +85,23 @@ static const struct sof_dev_desc cml_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &cnl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-cml.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/cnl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-cml.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-cnl-nocodec.tplg", .ops = &sof_cnl_ops, + .ops_init = sof_cnl_ops_init, }; /* PCI IDs */ diff --git a/sound/soc/sof/intel/pci-icl.c b/sound/soc/sof/intel/pci-icl.c index 39c84121b3..48f24f8ace 100644 --- a/sound/soc/sof/intel/pci-icl.c +++ b/sound/soc/sof/intel/pci-icl.c @@ -28,11 +28,24 @@ static const struct sof_dev_desc icl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &icl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-icl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/icl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-icl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-icl-nocodec.tplg", .ops = &sof_icl_ops, + .ops_init = sof_icl_ops_init, + .ops_free = hda_ops_free, }; static const struct sof_dev_desc jsl_desc = { @@ -43,11 +56,23 @@ static const struct sof_dev_desc jsl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &jsl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-jsl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/jsl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-jsl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-jsl-nocodec.tplg", .ops = &sof_cnl_ops, + .ops_init = sof_cnl_ops_init, }; /* PCI IDs */ diff --git a/sound/soc/sof/intel/pci-tgl.c b/sound/soc/sof/intel/pci-tgl.c index feaec251ad..ccc44ba3ad 100644 --- a/sound/soc/sof/intel/pci-tgl.c +++ b/sound/soc/sof/intel/pci-tgl.c @@ -28,11 +28,23 @@ static const struct sof_dev_desc tgl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &tgl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-tgl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/tgl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-tgl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-tgl-nocodec.tplg", .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, }; static const struct sof_dev_desc tglh_desc = { @@ -44,11 +56,24 @@ static const struct sof_dev_desc tglh_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &tglh_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-tgl-h.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/tgl-h", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-tgl-h.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-tgl-nocodec.tplg", .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, + .ops_free = hda_ops_free, }; static const struct sof_dev_desc ehl_desc = { @@ -59,11 +84,23 @@ static const struct sof_dev_desc ehl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &ehl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-ehl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/ehl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-ehl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-ehl-nocodec.tplg", .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, }; static const struct sof_dev_desc adls_desc = { @@ -75,11 +112,23 @@ static const struct sof_dev_desc adls_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &adls_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-adl-s.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/adl-s", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-adl-s.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-adl-nocodec.tplg", .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, }; static const struct sof_dev_desc adl_desc = { @@ -91,11 +140,23 @@ static const struct sof_dev_desc adl_desc = { .resindex_imr_base = -1, .irqindex_host_ipc = -1, .chip_info = &tgl_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-adl.ri", + .ipc_supported_mask = BIT(SOF_IPC) | BIT(SOF_INTEL_IPC4), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + [SOF_INTEL_IPC4] = "intel/avs/adl", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + [SOF_INTEL_IPC4] = "intel/avs-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-adl.ri", + [SOF_INTEL_IPC4] = "dsp_basefw.bin", + }, .nocodec_tplg_filename = "sof-adl-nocodec.tplg", .ops = &sof_tgl_ops, + .ops_init = sof_tgl_ops_init, }; /* PCI IDs */ @@ -116,6 +177,12 @@ static const struct pci_device_id sof_pci_ids[] = { .driver_data = (unsigned long)&adl_desc}, { PCI_DEVICE(0x8086, 0x51cd), /* ADL-P */ .driver_data = (unsigned long)&adl_desc}, + { PCI_DEVICE(0x8086, 0x51c9), /* ADL-PS */ + .driver_data = (unsigned long)&adl_desc}, + { PCI_DEVICE(0x8086, 0x51ca), /* RPL-P */ + .driver_data = (unsigned long)&adl_desc}, + { PCI_DEVICE(0x8086, 0x51cb), /* RPL-P */ + .driver_data = (unsigned long)&adl_desc}, { PCI_DEVICE(0x8086, 0x51cc), /* ADL-M */ .driver_data = (unsigned long)&adl_desc}, { PCI_DEVICE(0x8086, 0x54c8), /* ADL-N */ @@ -140,4 +207,3 @@ module_pci_driver(snd_sof_pci_intel_tgl_driver); MODULE_LICENSE("Dual BSD/GPL"); MODULE_IMPORT_NS(SND_SOC_SOF_INTEL_HDA_COMMON); MODULE_IMPORT_NS(SND_SOC_SOF_PCI_DEV); - diff --git a/sound/soc/sof/intel/pci-tng.c b/sound/soc/sof/intel/pci-tng.c index 6efef22597..f0f6d9ba88 100644 --- a/sound/soc/sof/intel/pci-tng.c +++ b/sound/soc/sof/intel/pci-tng.c @@ -75,7 +75,11 @@ static int tangier_pci_probe(struct snd_sof_dev *sdev) /* LPE base */ base = pci_resource_start(pci, desc->resindex_lpe_base) - IRAM_OFFSET; - size = PCI_BAR_SIZE; + size = pci_resource_len(pci, desc->resindex_lpe_base); + if (size < PCI_BAR_SIZE) { + dev_err(sdev->dev, "error: I/O region is too small.\n"); + return -ENODEV; + } dev_dbg(sdev->dev, "LPE PHY base at 0x%x size 0x%x", base, size); sdev->bar[DSP_BAR] = devm_ioremap(sdev->dev, base, size); @@ -132,7 +136,7 @@ static int tangier_pci_probe(struct snd_sof_dev *sdev) return ret; } -const struct snd_sof_dsp_ops sof_tng_ops = { +struct snd_sof_dsp_ops sof_tng_ops = { /* device init */ .probe = tangier_pci_probe, @@ -160,7 +164,6 @@ const struct snd_sof_dsp_ops sof_tng_ops = { /* ipc */ .send_msg = atom_send_msg, - .fw_ready = sof_fw_ready, .get_mailbox_offset = atom_get_mailbox_offset, .get_window_offset = atom_get_window_offset, @@ -183,9 +186,6 @@ const struct snd_sof_dsp_ops sof_tng_ops = { .pcm_open = sof_stream_pcm_open, .pcm_close = sof_stream_pcm_close, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, - /*Firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, @@ -206,6 +206,7 @@ const struct snd_sof_dsp_ops sof_tng_ops = { const struct sof_intel_dsp_desc tng_chip_info = { .cores_num = 1, .host_managed_cores_mask = 1, + .hw_ip_version = SOF_INTEL_TANGIER, }; static const struct sof_dev_desc tng_desc = { @@ -215,9 +216,17 @@ static const struct sof_dev_desc tng_desc = { .resindex_imr_base = 0, .irqindex_host_ipc = -1, .chip_info = &tng_chip_info, - .default_fw_path = "intel/sof", - .default_tplg_path = "intel/sof-tplg", - .default_fw_filename = "sof-byt.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "intel/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "intel/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-byt.ri", + }, .nocodec_tplg_filename = "sof-byt.tplg", .ops = &sof_tng_ops, }; diff --git a/sound/soc/sof/intel/shim.h b/sound/soc/sof/intel/shim.h index f36cd9d5eb..638159bee8 100644 --- a/sound/soc/sof/intel/shim.h +++ b/sound/soc/sof/intel/shim.h @@ -11,6 +11,18 @@ #ifndef __SOF_INTEL_SHIM_H #define __SOF_INTEL_SHIM_H +enum sof_intel_hw_ip_version { + SOF_INTEL_TANGIER, + SOF_INTEL_BAYTRAIL, + SOF_INTEL_BROADWELL, + SOF_INTEL_CAVS_1_5, /* SkyLake, KabyLake, AmberLake */ + SOF_INTEL_CAVS_1_5_PLUS,/* ApolloLake, GeminiLake */ + SOF_INTEL_CAVS_1_8, /* CannonLake, CometLake, CoffeeLake */ + SOF_INTEL_CAVS_2_0, /* IceLake, JasperLake */ + SOF_INTEL_CAVS_2_5, /* TigerLake, AlderLake */ + SOF_INTEL_ACE_1_0, /* MeteorLake */ +}; + /* * SHIM registers for BYT, BSW, CHT, BDW */ @@ -164,16 +176,20 @@ struct sof_intel_dsp_desc { int ipc_ack; int ipc_ack_mask; int ipc_ctl; + int rom_status_reg; int rom_init_timeout; int ssp_count; /* ssp count of the platform */ int ssp_base_offset; /* base address of the SSPs */ u32 sdw_shim_base; u32 sdw_alh_base; u32 quirks; + enum sof_intel_hw_ip_version hw_ip_version; bool (*check_sdw_irq)(struct snd_sof_dev *sdev); + bool (*check_ipc_irq)(struct snd_sof_dev *sdev); + int (*cl_init)(struct snd_sof_dev *sdev, int stream_tag, bool imr_boot); }; -extern const struct snd_sof_dsp_ops sof_tng_ops; +extern struct snd_sof_dsp_ops sof_tng_ops; extern const struct sof_intel_dsp_desc tng_chip_info; diff --git a/sound/soc/sof/intel/tgl.c b/sound/soc/sof/intel/tgl.c index cb1c319d5b..6dfb4786c7 100644 --- a/sound/soc/sof/intel/tgl.c +++ b/sound/soc/sof/intel/tgl.c @@ -9,6 +9,8 @@ * Hardware interface for audio DSP on Tigerlake. */ +#include +#include "../ipc4-priv.h" #include "../ops.h" #include "hda.h" #include "hda-ipc.h" @@ -22,148 +24,90 @@ static const struct snd_sof_debugfs_map tgl_dsp_debugfs[] = { static int tgl_dsp_core_get(struct snd_sof_dev *sdev, int core) { - struct sof_ipc_pm_core_config pm_core_config = { - .hdr = { - .cmd = SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CORE_ENABLE, - .size = sizeof(pm_core_config), - }, - .enable_mask = sdev->enabled_cores_mask | BIT(core), - }; + const struct sof_ipc_pm_ops *pm_ops = sdev->ipc->ops->pm; /* power up primary core if not already powered up and return */ if (core == SOF_DSP_PRIMARY_CORE) return hda_dsp_enable_core(sdev, BIT(core)); - /* notify DSP for secondary cores */ - return sof_ipc_tx_message(sdev->ipc, pm_core_config.hdr.cmd, - &pm_core_config, sizeof(pm_core_config), - &pm_core_config, sizeof(pm_core_config)); + if (pm_ops->set_core_state) + return pm_ops->set_core_state(sdev, core, true); + + return 0; } static int tgl_dsp_core_put(struct snd_sof_dev *sdev, int core) { - struct sof_ipc_pm_core_config pm_core_config = { - .hdr = { - .cmd = SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CORE_ENABLE, - .size = sizeof(pm_core_config), - }, - .enable_mask = sdev->enabled_cores_mask & ~BIT(core), - }; + const struct sof_ipc_pm_ops *pm_ops = sdev->ipc->ops->pm; /* power down primary core and return */ if (core == SOF_DSP_PRIMARY_CORE) return hda_dsp_core_reset_power_down(sdev, BIT(core)); - /* notify DSP for secondary cores */ - return sof_ipc_tx_message(sdev->ipc, pm_core_config.hdr.cmd, - &pm_core_config, sizeof(pm_core_config), - &pm_core_config, sizeof(pm_core_config)); + if (pm_ops->set_core_state) + return pm_ops->set_core_state(sdev, core, false); + + return 0; } /* Tigerlake ops */ -const struct snd_sof_dsp_ops sof_tgl_ops = { +struct snd_sof_dsp_ops sof_tgl_ops; +EXPORT_SYMBOL_NS(sof_tgl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); + +int sof_tgl_ops_init(struct snd_sof_dev *sdev) +{ + /* common defaults */ + memcpy(&sof_tgl_ops, &sof_hda_common_ops, sizeof(struct snd_sof_dsp_ops)); + /* probe/remove/shutdown */ - .probe = hda_dsp_probe, - .remove = hda_dsp_remove, - .shutdown = hda_dsp_shutdown, + sof_tgl_ops.shutdown = hda_dsp_shutdown; - /* Register IO */ - .write = sof_io_write, - .read = sof_io_read, - .write64 = sof_io_write64, - .read64 = sof_io_read64, + if (sdev->pdata->ipc_type == SOF_IPC) { + /* doorbell */ + sof_tgl_ops.irq_thread = cnl_ipc_irq_thread; - /* Block IO */ - .block_read = sof_block_read, - .block_write = sof_block_write, + /* ipc */ + sof_tgl_ops.send_msg = cnl_ipc_send_msg; + } - /* Mailbox IO */ - .mailbox_read = sof_mailbox_read, - .mailbox_write = sof_mailbox_write, + if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_fw_data *ipc4_data; - /* doorbell */ - .irq_thread = cnl_ipc_irq_thread, + sdev->private = devm_kzalloc(sdev->dev, sizeof(*ipc4_data), GFP_KERNEL); + if (!sdev->private) + return -ENOMEM; - /* ipc */ - .send_msg = cnl_ipc_send_msg, - .fw_ready = sof_fw_ready, - .get_mailbox_offset = hda_dsp_ipc_get_mailbox_offset, - .get_window_offset = hda_dsp_ipc_get_window_offset, + ipc4_data = sdev->private; + ipc4_data->manifest_fw_hdr_offset = SOF_MAN4_FW_HDR_OFFSET; - .ipc_msg_data = hda_ipc_msg_data, - .set_stream_data_offset = hda_set_stream_data_offset, + /* doorbell */ + sof_tgl_ops.irq_thread = cnl_ipc4_irq_thread; - /* machine driver */ - .machine_select = hda_machine_select, - .machine_register = sof_machine_register, - .machine_unregister = sof_machine_unregister, - .set_mach_params = hda_set_mach_params, + /* ipc */ + sof_tgl_ops.send_msg = cnl_ipc4_send_msg; + } + + /* set DAI driver ops */ + hda_set_dai_drv_ops(sdev, &sof_tgl_ops); /* debug */ - .debug_map = tgl_dsp_debugfs, - .debug_map_count = ARRAY_SIZE(tgl_dsp_debugfs), - .dbg_dump = hda_dsp_dump, - .ipc_dump = cnl_ipc_dump, - .debugfs_add_region_item = snd_sof_debugfs_add_region_item_iomem, - - /* stream callbacks */ - .pcm_open = hda_dsp_pcm_open, - .pcm_close = hda_dsp_pcm_close, - .pcm_hw_params = hda_dsp_pcm_hw_params, - .pcm_hw_free = hda_dsp_stream_hw_free, - .pcm_trigger = hda_dsp_pcm_trigger, - .pcm_pointer = hda_dsp_pcm_pointer, - .pcm_ack = hda_dsp_pcm_ack, - - /* firmware loading */ - .load_firmware = snd_sof_load_firmware_raw, + sof_tgl_ops.debug_map = tgl_dsp_debugfs; + sof_tgl_ops.debug_map_count = ARRAY_SIZE(tgl_dsp_debugfs); + sof_tgl_ops.ipc_dump = cnl_ipc_dump; /* pre/post fw run */ - .pre_fw_run = hda_dsp_pre_fw_run, - .post_fw_run = hda_dsp_post_fw_run, - - /* parse platform specific extended manifest */ - .parse_platform_ext_manifest = hda_dsp_ext_man_get_cavs_config_data, - - /* dsp core get/put */ - .core_get = tgl_dsp_core_get, - .core_put = tgl_dsp_core_put, + sof_tgl_ops.post_fw_run = hda_dsp_post_fw_run; /* firmware run */ - .run = hda_dsp_cl_boot_firmware_iccmax, + sof_tgl_ops.run = hda_dsp_cl_boot_firmware_iccmax; - /* trace callback */ - .trace_init = hda_dsp_trace_init, - .trace_release = hda_dsp_trace_release, - .trace_trigger = hda_dsp_trace_trigger, + /* dsp core get/put */ + sof_tgl_ops.core_get = tgl_dsp_core_get; + sof_tgl_ops.core_put = tgl_dsp_core_put; - /* client ops */ - .register_ipc_clients = hda_register_clients, - .unregister_ipc_clients = hda_unregister_clients, - - /* DAI drivers */ - .drv = skl_dai, - .num_drv = SOF_SKL_NUM_DAIS, - - /* PM */ - .suspend = hda_dsp_suspend, - .resume = hda_dsp_resume, - .runtime_suspend = hda_dsp_runtime_suspend, - .runtime_resume = hda_dsp_runtime_resume, - .runtime_idle = hda_dsp_runtime_idle, - .set_hw_params_upon_resume = hda_dsp_set_hw_params_upon_resume, - .set_power_state = hda_dsp_set_power_state, - - /* ALSA HW info flags */ - .hw_info = SNDRV_PCM_INFO_MMAP | - SNDRV_PCM_INFO_MMAP_VALID | - SNDRV_PCM_INFO_INTERLEAVED | - SNDRV_PCM_INFO_PAUSE | - SNDRV_PCM_INFO_NO_PERIOD_WAKEUP, - - .dsp_arch_ops = &sof_xtensa_arch_ops, + return 0; }; -EXPORT_SYMBOL_NS(sof_tgl_ops, SND_SOC_SOF_INTEL_HDA_COMMON); +EXPORT_SYMBOL_NS(sof_tgl_ops_init, SND_SOC_SOF_INTEL_HDA_COMMON); const struct sof_intel_dsp_desc tgl_chip_info = { /* Tigerlake , Alderlake */ @@ -175,12 +119,16 @@ const struct sof_intel_dsp_desc tgl_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = ICL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(tgl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -194,12 +142,16 @@ const struct sof_intel_dsp_desc tglh_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = ICL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(tglh_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -213,12 +165,16 @@ const struct sof_intel_dsp_desc ehl_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = ICL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(ehl_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); @@ -232,11 +188,15 @@ const struct sof_intel_dsp_desc adls_chip_info = { .ipc_ack = CNL_DSP_REG_HIPCIDA, .ipc_ack_mask = CNL_DSP_REG_HIPCIDA_DONE, .ipc_ctl = CNL_DSP_REG_HIPCCTL, + .rom_status_reg = HDA_DSP_SRAM_REG_ROM_STATUS, .rom_init_timeout = 300, .ssp_count = ICL_SSP_COUNT, .ssp_base_offset = CNL_SSP_BASE_OFFSET, .sdw_shim_base = SDW_SHIM_BASE, .sdw_alh_base = SDW_ALH_BASE, .check_sdw_irq = hda_common_check_sdw_irq, + .check_ipc_irq = hda_dsp_check_ipc_irq, + .cl_init = cl_dsp_init, + .hw_ip_version = SOF_INTEL_CAVS_2_5, }; EXPORT_SYMBOL_NS(adls_chip_info, SND_SOC_SOF_INTEL_HDA_COMMON); diff --git a/sound/soc/sof/ipc.c b/sound/soc/sof/ipc.c index 5f5753608c..6ed3f9b6a0 100644 --- a/sound/soc/sof/ipc.c +++ b/sound/soc/sof/ipc.c @@ -17,294 +17,35 @@ #include "sof-priv.h" #include "sof-audio.h" #include "ops.h" -#include "ipc3-ops.h" -typedef void (*ipc_rx_callback)(struct snd_sof_dev *sdev, void *msg_buf); - -static void ipc_trace_message(struct snd_sof_dev *sdev, void *msg_buf); -static void ipc_stream_message(struct snd_sof_dev *sdev, void *msg_buf); - -/* - * IPC message Tx/Rx message handling. +/** + * sof_ipc_send_msg - generic function to prepare and send one IPC message + * @sdev: pointer to SOF core device struct + * @msg_data: pointer to a message to send + * @msg_bytes: number of bytes in the message + * @reply_bytes: number of bytes available for the reply. + * The buffer for the reply data is not passed to this + * function, the available size is an information for the + * reply handling functions. + * + * On success the function returns 0, otherwise negative error number. + * + * Note: higher level sdev->ipc->tx_mutex must be held to make sure that + * transfers are synchronized. */ - -struct sof_ipc_ctrl_data_params { - size_t msg_bytes; - size_t hdr_bytes; - size_t pl_size; - size_t elems; - u32 num_msg; - u8 *src; - u8 *dst; -}; - -#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC) -static void ipc_log_header(struct device *dev, u8 *text, u32 cmd) +int sof_ipc_send_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes, + size_t reply_bytes) { - u8 *str; - u8 *str2 = NULL; - u32 glb; - u32 type; - bool vdbg = false; - - glb = cmd & SOF_GLB_TYPE_MASK; - type = cmd & SOF_CMD_TYPE_MASK; - - switch (glb) { - case SOF_IPC_GLB_REPLY: - str = "GLB_REPLY"; break; - case SOF_IPC_GLB_COMPOUND: - str = "GLB_COMPOUND"; break; - case SOF_IPC_GLB_TPLG_MSG: - str = "GLB_TPLG_MSG"; - switch (type) { - case SOF_IPC_TPLG_COMP_NEW: - str2 = "COMP_NEW"; break; - case SOF_IPC_TPLG_COMP_FREE: - str2 = "COMP_FREE"; break; - case SOF_IPC_TPLG_COMP_CONNECT: - str2 = "COMP_CONNECT"; break; - case SOF_IPC_TPLG_PIPE_NEW: - str2 = "PIPE_NEW"; break; - case SOF_IPC_TPLG_PIPE_FREE: - str2 = "PIPE_FREE"; break; - case SOF_IPC_TPLG_PIPE_CONNECT: - str2 = "PIPE_CONNECT"; break; - case SOF_IPC_TPLG_PIPE_COMPLETE: - str2 = "PIPE_COMPLETE"; break; - case SOF_IPC_TPLG_BUFFER_NEW: - str2 = "BUFFER_NEW"; break; - case SOF_IPC_TPLG_BUFFER_FREE: - str2 = "BUFFER_FREE"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_PM_MSG: - str = "GLB_PM_MSG"; - switch (type) { - case SOF_IPC_PM_CTX_SAVE: - str2 = "CTX_SAVE"; break; - case SOF_IPC_PM_CTX_RESTORE: - str2 = "CTX_RESTORE"; break; - case SOF_IPC_PM_CTX_SIZE: - str2 = "CTX_SIZE"; break; - case SOF_IPC_PM_CLK_SET: - str2 = "CLK_SET"; break; - case SOF_IPC_PM_CLK_GET: - str2 = "CLK_GET"; break; - case SOF_IPC_PM_CLK_REQ: - str2 = "CLK_REQ"; break; - case SOF_IPC_PM_CORE_ENABLE: - str2 = "CORE_ENABLE"; break; - case SOF_IPC_PM_GATE: - str2 = "GATE"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_COMP_MSG: - str = "GLB_COMP_MSG"; - switch (type) { - case SOF_IPC_COMP_SET_VALUE: - str2 = "SET_VALUE"; break; - case SOF_IPC_COMP_GET_VALUE: - str2 = "GET_VALUE"; break; - case SOF_IPC_COMP_SET_DATA: - str2 = "SET_DATA"; break; - case SOF_IPC_COMP_GET_DATA: - str2 = "GET_DATA"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_STREAM_MSG: - str = "GLB_STREAM_MSG"; - switch (type) { - case SOF_IPC_STREAM_PCM_PARAMS: - str2 = "PCM_PARAMS"; break; - case SOF_IPC_STREAM_PCM_PARAMS_REPLY: - str2 = "PCM_REPLY"; break; - case SOF_IPC_STREAM_PCM_FREE: - str2 = "PCM_FREE"; break; - case SOF_IPC_STREAM_TRIG_START: - str2 = "TRIG_START"; break; - case SOF_IPC_STREAM_TRIG_STOP: - str2 = "TRIG_STOP"; break; - case SOF_IPC_STREAM_TRIG_PAUSE: - str2 = "TRIG_PAUSE"; break; - case SOF_IPC_STREAM_TRIG_RELEASE: - str2 = "TRIG_RELEASE"; break; - case SOF_IPC_STREAM_TRIG_DRAIN: - str2 = "TRIG_DRAIN"; break; - case SOF_IPC_STREAM_TRIG_XRUN: - str2 = "TRIG_XRUN"; break; - case SOF_IPC_STREAM_POSITION: - vdbg = true; - str2 = "POSITION"; break; - case SOF_IPC_STREAM_VORBIS_PARAMS: - str2 = "VORBIS_PARAMS"; break; - case SOF_IPC_STREAM_VORBIS_FREE: - str2 = "VORBIS_FREE"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_FW_READY: - str = "FW_READY"; break; - case SOF_IPC_GLB_DAI_MSG: - str = "GLB_DAI_MSG"; - switch (type) { - case SOF_IPC_DAI_CONFIG: - str2 = "CONFIG"; break; - case SOF_IPC_DAI_LOOPBACK: - str2 = "LOOPBACK"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_TRACE_MSG: - str = "GLB_TRACE_MSG"; - switch (type) { - case SOF_IPC_TRACE_DMA_PARAMS: - str2 = "DMA_PARAMS"; break; - case SOF_IPC_TRACE_DMA_POSITION: - str2 = "DMA_POSITION"; break; - case SOF_IPC_TRACE_DMA_PARAMS_EXT: - str2 = "DMA_PARAMS_EXT"; break; - case SOF_IPC_TRACE_FILTER_UPDATE: - str2 = "FILTER_UPDATE"; break; - case SOF_IPC_TRACE_DMA_FREE: - str2 = "DMA_FREE"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_TEST_MSG: - str = "GLB_TEST_MSG"; - switch (type) { - case SOF_IPC_TEST_IPC_FLOOD: - str2 = "IPC_FLOOD"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_DEBUG: - str = "GLB_DEBUG"; - switch (type) { - case SOF_IPC_DEBUG_MEM_USAGE: - str2 = "MEM_USAGE"; break; - default: - str2 = "unknown type"; break; - } - break; - case SOF_IPC_GLB_PROBE: - str = "GLB_PROBE"; - switch (type) { - case SOF_IPC_PROBE_INIT: - str2 = "INIT"; break; - case SOF_IPC_PROBE_DEINIT: - str2 = "DEINIT"; break; - case SOF_IPC_PROBE_DMA_ADD: - str2 = "DMA_ADD"; break; - case SOF_IPC_PROBE_DMA_INFO: - str2 = "DMA_INFO"; break; - case SOF_IPC_PROBE_DMA_REMOVE: - str2 = "DMA_REMOVE"; break; - case SOF_IPC_PROBE_POINT_ADD: - str2 = "POINT_ADD"; break; - case SOF_IPC_PROBE_POINT_INFO: - str2 = "POINT_INFO"; break; - case SOF_IPC_PROBE_POINT_REMOVE: - str2 = "POINT_REMOVE"; break; - default: - str2 = "unknown type"; break; - } - break; - default: - str = "unknown GLB command"; break; - } - - if (str2) { - if (vdbg) - dev_vdbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2); - else - dev_dbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2); - } else { - dev_dbg(dev, "%s: 0x%x: %s\n", text, cmd, str); - } -} -#else -static inline void ipc_log_header(struct device *dev, u8 *text, u32 cmd) -{ - if ((cmd & SOF_GLB_TYPE_MASK) != SOF_IPC_GLB_TRACE_MSG) - dev_dbg(dev, "%s: 0x%x\n", text, cmd); -} -#endif - -/* wait for IPC message reply */ -static int tx_wait_done(struct snd_sof_ipc *ipc, struct snd_sof_ipc_msg *msg, - void *reply_data) -{ - struct snd_sof_dev *sdev = ipc->sdev; - struct sof_ipc_cmd_hdr *hdr = msg->msg_data; - int ret; - - /* wait for DSP IPC completion */ - ret = wait_event_timeout(msg->waitq, msg->ipc_complete, - msecs_to_jiffies(sdev->ipc_timeout)); - - if (ret == 0) { - dev_err(sdev->dev, - "ipc tx timed out for %#x (msg/reply size: %d/%zu)\n", - hdr->cmd, hdr->size, msg->reply_size); - snd_sof_handle_fw_exception(ipc->sdev); - ret = -ETIMEDOUT; - } else { - ret = msg->reply_error; - if (ret < 0) { - dev_err(sdev->dev, - "ipc tx error for %#x (msg/reply size: %d/%zu): %d\n", - hdr->cmd, hdr->size, msg->reply_size, ret); - } else { - ipc_log_header(sdev->dev, "ipc tx succeeded", hdr->cmd); - if (msg->reply_size) - /* copy the data returned from DSP */ - memcpy(reply_data, msg->reply_data, - msg->reply_size); - } - - /* re-enable dumps after successful IPC tx */ - if (sdev->ipc_dump_printed) { - sdev->dbg_dump_printed = false; - sdev->ipc_dump_printed = false; - } - } - - return ret; -} - -/* send IPC message from host to DSP */ -static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc, - void *msg_data, size_t msg_bytes, - void *reply_data, size_t reply_bytes) -{ - struct sof_ipc_cmd_hdr *hdr = msg_data; - struct snd_sof_dev *sdev = ipc->sdev; + struct snd_sof_ipc *ipc = sdev->ipc; struct snd_sof_ipc_msg *msg; int ret; - if (!msg_data || msg_bytes < sizeof(*hdr)) { - dev_err_ratelimited(sdev->dev, "No IPC message to send\n"); - return -EINVAL; - } - if (ipc->disable_ipc_tx || sdev->fw_state != SOF_FW_BOOT_COMPLETE) return -ENODEV; /* - * The spin-lock is also still needed to protect message objects against - * other atomic contexts. + * The spin-lock is needed to protect message objects against other + * atomic contexts. */ spin_lock_irq(&sdev->ipc_lock); @@ -327,38 +68,19 @@ static int sof_ipc_tx_message_unlocked(struct snd_sof_ipc *ipc, spin_unlock_irq(&sdev->ipc_lock); - if (ret) { - dev_err_ratelimited(sdev->dev, - "error: ipc tx failed with error %d\n", - ret); - return ret; - } - - ipc_log_header(sdev->dev, "ipc tx", hdr->cmd); - - /* now wait for completion */ - return tx_wait_done(ipc, msg, reply_data); + return ret; } /* send IPC message from host to DSP */ -int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header, - void *msg_data, size_t msg_bytes, void *reply_data, - size_t reply_bytes) +int sof_ipc_tx_message(struct snd_sof_ipc *ipc, void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes) { - const struct sof_dsp_power_state target_state = { - .state = SOF_DSP_PM_D0, - }; - int ret; + if (msg_bytes > ipc->max_payload_size || + reply_bytes > ipc->max_payload_size) + return -ENOBUFS; - /* ensure the DSP is in D0 before sending a new IPC */ - ret = snd_sof_dsp_set_power_state(ipc->sdev, &target_state); - if (ret < 0) { - dev_err(ipc->sdev->dev, "error: resuming DSP %d\n", ret); - return ret; - } - - return sof_ipc_tx_message_no_pm(ipc, header, msg_data, msg_bytes, - reply_data, reply_bytes); + return ipc->ops->tx_msg(ipc->sdev, msg_data, msg_bytes, reply_data, + reply_bytes, false); } EXPORT_SYMBOL(sof_ipc_tx_message); @@ -367,86 +89,32 @@ EXPORT_SYMBOL(sof_ipc_tx_message); * This will be used for IPC's that can be handled by the DSP * even in a low-power D0 substate. */ -int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, u32 header, - void *msg_data, size_t msg_bytes, +int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, void *msg_data, size_t msg_bytes, void *reply_data, size_t reply_bytes) { - int ret; - - if (msg_bytes > SOF_IPC_MSG_MAX_SIZE || - reply_bytes > SOF_IPC_MSG_MAX_SIZE) + if (msg_bytes > ipc->max_payload_size || + reply_bytes > ipc->max_payload_size) return -ENOBUFS; - /* Serialise IPC TX */ - mutex_lock(&ipc->tx_mutex); - - ret = sof_ipc_tx_message_unlocked(ipc, msg_data, msg_bytes, - reply_data, reply_bytes); - - mutex_unlock(&ipc->tx_mutex); - - return ret; + return ipc->ops->tx_msg(ipc->sdev, msg_data, msg_bytes, reply_data, + reply_bytes, true); } EXPORT_SYMBOL(sof_ipc_tx_message_no_pm); /* Generic helper function to retrieve the reply */ void snd_sof_ipc_get_reply(struct snd_sof_dev *sdev) { - struct snd_sof_ipc_msg *msg = sdev->msg; - struct sof_ipc_reply reply; - int ret = 0; - /* * Sometimes, there is unexpected reply ipc arriving. The reply * ipc belongs to none of the ipcs sent from driver. * In this case, the driver must ignore the ipc. */ - if (!msg) { + if (!sdev->msg) { dev_warn(sdev->dev, "unexpected ipc interrupt raised!\n"); return; } - /* get the generic reply */ - snd_sof_dsp_mailbox_read(sdev, sdev->host_box.offset, &reply, - sizeof(reply)); - - if (reply.error < 0) { - memcpy(msg->reply_data, &reply, sizeof(reply)); - ret = reply.error; - } else if (!reply.hdr.size) { - /* Reply should always be >= sizeof(struct sof_ipc_reply) */ - if (msg->reply_size) - dev_err(sdev->dev, - "empty reply received, expected %zu bytes\n", - msg->reply_size); - else - dev_err(sdev->dev, "empty reply received\n"); - - ret = -EINVAL; - } else if (msg->reply_size > 0) { - if (reply.hdr.size == msg->reply_size) { - ret = 0; - } else if (reply.hdr.size < msg->reply_size) { - dev_dbg(sdev->dev, - "reply size (%u) is less than expected (%zu)\n", - reply.hdr.size, msg->reply_size); - - msg->reply_size = reply.hdr.size; - ret = 0; - } else { - dev_err(sdev->dev, - "reply size (%u) exceeds the buffer size (%zu)\n", - reply.hdr.size, msg->reply_size); - ret = -EINVAL; - } - - /* get the full message if reply.hdr.size <= msg->reply_size */ - if (!ret) - snd_sof_dsp_mailbox_read(sdev, sdev->host_box.offset, - msg->reply_data, msg->reply_size); - } - - msg->reply_error = ret; + sdev->msg->reply_error = sdev->ipc->ops->get_reply(sdev); } EXPORT_SYMBOL(snd_sof_ipc_get_reply); @@ -468,550 +136,11 @@ void snd_sof_ipc_reply(struct snd_sof_dev *sdev, u32 msg_id) } EXPORT_SYMBOL(snd_sof_ipc_reply); -static void ipc_comp_notification(struct snd_sof_dev *sdev, void *msg_buf) -{ - const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; - struct sof_ipc_cmd_hdr *hdr = msg_buf; - u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK; - - switch (msg_type) { - case SOF_IPC_COMP_GET_VALUE: - case SOF_IPC_COMP_GET_DATA: - break; - default: - dev_err(sdev->dev, "error: unhandled component message %#x\n", msg_type); - return; - } - - if (tplg_ops->control->update) - tplg_ops->control->update(sdev, msg_buf); -} - -/* DSP firmware has sent host a message */ -void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev) -{ - ipc_rx_callback rx_callback = NULL; - struct sof_ipc_cmd_hdr hdr; - void *msg_buf; - u32 cmd; - int err; - - /* read back header */ - err = snd_sof_ipc_msg_data(sdev, NULL, &hdr, sizeof(hdr)); - if (err < 0) { - dev_warn(sdev->dev, "failed to read IPC header: %d\n", err); - return; - } - - if (hdr.size < sizeof(hdr)) { - dev_err(sdev->dev, "The received message size is invalid\n"); - return; - } - - ipc_log_header(sdev->dev, "ipc rx", hdr.cmd); - - cmd = hdr.cmd & SOF_GLB_TYPE_MASK; - - /* check message type */ - switch (cmd) { - case SOF_IPC_GLB_REPLY: - dev_err(sdev->dev, "error: ipc reply unknown\n"); - break; - case SOF_IPC_FW_READY: - /* check for FW boot completion */ - if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) { - err = sof_ops(sdev)->fw_ready(sdev, cmd); - if (err < 0) - sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED); - else - sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK); - - /* wake up firmware loader */ - wake_up(&sdev->boot_wait); - } - break; - case SOF_IPC_GLB_COMPOUND: - case SOF_IPC_GLB_TPLG_MSG: - case SOF_IPC_GLB_PM_MSG: - break; - case SOF_IPC_GLB_COMP_MSG: - rx_callback = ipc_comp_notification; - break; - case SOF_IPC_GLB_STREAM_MSG: - rx_callback = ipc_stream_message; - break; - case SOF_IPC_GLB_TRACE_MSG: - rx_callback = ipc_trace_message; - break; - default: - dev_err(sdev->dev, "%s: Unknown DSP message: 0x%x\n", __func__, cmd); - break; - } - - /* read the full message */ - msg_buf = kmalloc(hdr.size, GFP_KERNEL); - if (!msg_buf) - return; - - err = snd_sof_ipc_msg_data(sdev, NULL, msg_buf, hdr.size); - if (err < 0) { - dev_err(sdev->dev, "%s: Failed to read message: %d\n", __func__, err); - } else { - /* Call local handler for the message */ - if (rx_callback) - rx_callback(sdev, msg_buf); - - /* Notify registered clients */ - sof_client_ipc_rx_dispatcher(sdev, msg_buf); - } - - kfree(msg_buf); - - ipc_log_header(sdev->dev, "ipc rx done", hdr.cmd); -} -EXPORT_SYMBOL(snd_sof_ipc_msgs_rx); - -/* - * IPC trace mechanism. - */ - -static void ipc_trace_message(struct snd_sof_dev *sdev, void *msg_buf) -{ - struct sof_ipc_cmd_hdr *hdr = msg_buf; - u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK; - - switch (msg_type) { - case SOF_IPC_TRACE_DMA_POSITION: - snd_sof_trace_update_pos(sdev, msg_buf); - break; - default: - dev_err(sdev->dev, "error: unhandled trace message %#x\n", msg_type); - break; - } -} - -/* - * IPC stream position. - */ - -static void ipc_period_elapsed(struct snd_sof_dev *sdev, u32 msg_id) -{ - struct snd_soc_component *scomp = sdev->component; - struct snd_sof_pcm_stream *stream; - struct sof_ipc_stream_posn posn; - struct snd_sof_pcm *spcm; - int direction, ret; - - spcm = snd_sof_find_spcm_comp(scomp, msg_id, &direction); - if (!spcm) { - dev_err(sdev->dev, - "error: period elapsed for unknown stream, msg_id %d\n", - msg_id); - return; - } - - stream = &spcm->stream[direction]; - ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn)); - if (ret < 0) { - dev_warn(sdev->dev, "failed to read stream position: %d\n", ret); - return; - } - - dev_vdbg(sdev->dev, "posn : host 0x%llx dai 0x%llx wall 0x%llx\n", - posn.host_posn, posn.dai_posn, posn.wallclock); - - memcpy(&stream->posn, &posn, sizeof(posn)); - - if (spcm->pcm.compress) - snd_sof_compr_fragment_elapsed(stream->cstream); - else if (stream->substream->runtime && - !stream->substream->runtime->no_period_wakeup) - /* only inform ALSA for period_wakeup mode */ - snd_sof_pcm_period_elapsed(stream->substream); -} - -/* DSP notifies host of an XRUN within FW */ -static void ipc_xrun(struct snd_sof_dev *sdev, u32 msg_id) -{ - struct snd_soc_component *scomp = sdev->component; - struct snd_sof_pcm_stream *stream; - struct sof_ipc_stream_posn posn; - struct snd_sof_pcm *spcm; - int direction, ret; - - spcm = snd_sof_find_spcm_comp(scomp, msg_id, &direction); - if (!spcm) { - dev_err(sdev->dev, "error: XRUN for unknown stream, msg_id %d\n", - msg_id); - return; - } - - stream = &spcm->stream[direction]; - ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn)); - if (ret < 0) { - dev_warn(sdev->dev, "failed to read overrun position: %d\n", ret); - return; - } - - dev_dbg(sdev->dev, "posn XRUN: host %llx comp %d size %d\n", - posn.host_posn, posn.xrun_comp_id, posn.xrun_size); - -#if defined(CONFIG_SND_SOC_SOF_DEBUG_XRUN_STOP) - /* stop PCM on XRUN - used for pipeline debug */ - memcpy(&stream->posn, &posn, sizeof(posn)); - snd_pcm_stop_xrun(stream->substream); -#endif -} - -/* stream notifications from DSP FW */ -static void ipc_stream_message(struct snd_sof_dev *sdev, void *msg_buf) -{ - struct sof_ipc_cmd_hdr *hdr = msg_buf; - u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK; - u32 msg_id = SOF_IPC_MESSAGE_ID(hdr->cmd); - - switch (msg_type) { - case SOF_IPC_STREAM_POSITION: - ipc_period_elapsed(sdev, msg_id); - break; - case SOF_IPC_STREAM_TRIG_XRUN: - ipc_xrun(sdev, msg_id); - break; - default: - dev_err(sdev->dev, "error: unhandled stream message %#x\n", - msg_id); - break; - } -} - -/* get stream position IPC - use faster MMIO method if available on platform */ -int snd_sof_ipc_stream_posn(struct snd_soc_component *scomp, - struct snd_sof_pcm *spcm, int direction, - struct sof_ipc_stream_posn *posn) -{ - struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); - struct sof_ipc_stream stream; - int err; - - /* read position via slower IPC */ - stream.hdr.size = sizeof(stream); - stream.hdr.cmd = SOF_IPC_GLB_STREAM_MSG | SOF_IPC_STREAM_POSITION; - stream.comp_id = spcm->stream[direction].comp_id; - - /* send IPC to the DSP */ - err = sof_ipc_tx_message(sdev->ipc, - stream.hdr.cmd, &stream, sizeof(stream), posn, - sizeof(*posn)); - if (err < 0) { - dev_err(sdev->dev, "error: failed to get stream %d position\n", - stream.comp_id); - return err; - } - - return 0; -} -EXPORT_SYMBOL(snd_sof_ipc_stream_posn); - -static int sof_get_ctrl_copy_params(enum sof_ipc_ctrl_type ctrl_type, - struct sof_ipc_ctrl_data *src, - struct sof_ipc_ctrl_data *dst, - struct sof_ipc_ctrl_data_params *sparams) -{ - switch (ctrl_type) { - case SOF_CTRL_TYPE_VALUE_CHAN_GET: - case SOF_CTRL_TYPE_VALUE_CHAN_SET: - sparams->src = (u8 *)src->chanv; - sparams->dst = (u8 *)dst->chanv; - break; - case SOF_CTRL_TYPE_DATA_GET: - case SOF_CTRL_TYPE_DATA_SET: - sparams->src = (u8 *)src->data->data; - sparams->dst = (u8 *)dst->data->data; - break; - default: - return -EINVAL; - } - - /* calculate payload size and number of messages */ - sparams->pl_size = SOF_IPC_MSG_MAX_SIZE - sparams->hdr_bytes; - sparams->num_msg = DIV_ROUND_UP(sparams->msg_bytes, sparams->pl_size); - - return 0; -} - -static int sof_set_get_large_ctrl_data(struct snd_sof_dev *sdev, - struct sof_ipc_ctrl_data *cdata, - struct sof_ipc_ctrl_data_params *sparams, - bool set) -{ - struct sof_ipc_ctrl_data *partdata; - size_t send_bytes; - size_t offset = 0; - size_t msg_bytes; - size_t pl_size; - int err; - int i; - - /* allocate max ipc size because we have at least one */ - partdata = kzalloc(SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); - if (!partdata) - return -ENOMEM; - - if (set) - err = sof_get_ctrl_copy_params(cdata->type, cdata, partdata, - sparams); - else - err = sof_get_ctrl_copy_params(cdata->type, partdata, cdata, - sparams); - if (err < 0) { - kfree(partdata); - return err; - } - - msg_bytes = sparams->msg_bytes; - pl_size = sparams->pl_size; - - /* copy the header data */ - memcpy(partdata, cdata, sparams->hdr_bytes); - - /* Serialise IPC TX */ - mutex_lock(&sdev->ipc->tx_mutex); - - /* copy the payload data in a loop */ - for (i = 0; i < sparams->num_msg; i++) { - send_bytes = min(msg_bytes, pl_size); - partdata->num_elems = send_bytes; - partdata->rhdr.hdr.size = sparams->hdr_bytes + send_bytes; - partdata->msg_index = i; - msg_bytes -= send_bytes; - partdata->elems_remaining = msg_bytes; - - if (set) - memcpy(sparams->dst, sparams->src + offset, send_bytes); - - err = sof_ipc_tx_message_unlocked(sdev->ipc, - partdata, - partdata->rhdr.hdr.size, - partdata, - partdata->rhdr.hdr.size); - if (err < 0) - break; - - if (!set) - memcpy(sparams->dst + offset, sparams->src, send_bytes); - - offset += pl_size; - } - - mutex_unlock(&sdev->ipc->tx_mutex); - - kfree(partdata); - return err; -} - -/* - * IPC get()/set() for kcontrols. - */ -int snd_sof_ipc_set_get_comp_data(struct snd_sof_control *scontrol, bool set) -{ - struct snd_soc_component *scomp = scontrol->scomp; - struct sof_ipc_ctrl_data *cdata = scontrol->ipc_control_data; - struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); - struct sof_ipc_fw_ready *ready = &sdev->fw_ready; - struct sof_ipc_fw_version *v = &ready->version; - struct sof_ipc_ctrl_data_params sparams; - enum sof_ipc_ctrl_type ctrl_type; - struct snd_sof_widget *swidget; - bool widget_found = false; - size_t send_bytes; - u32 ipc_cmd; - int err; - - list_for_each_entry(swidget, &sdev->widget_list, list) { - if (swidget->comp_id == scontrol->comp_id) { - widget_found = true; - break; - } - } - - if (!widget_found) { - dev_err(sdev->dev, "error: can't find widget with id %d\n", scontrol->comp_id); - return -EINVAL; - } - - /* - * Volatile controls should always be part of static pipelines and the widget use_count - * would always be > 0 in this case. For the others, just return the cached value if the - * widget is not set up. - */ - if (!swidget->use_count) - return 0; - - /* read or write firmware volume */ - if (scontrol->readback_offset != 0) { - /* write/read value header via mmaped region */ - send_bytes = sizeof(struct sof_ipc_ctrl_value_chan) * - cdata->num_elems; - if (set) - err = snd_sof_dsp_block_write(sdev, SOF_FW_BLK_TYPE_IRAM, - scontrol->readback_offset, - cdata->chanv, send_bytes); - - else - err = snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_IRAM, - scontrol->readback_offset, - cdata->chanv, send_bytes); - - if (err) - dev_err_once(sdev->dev, "error: %s TYPE_IRAM failed\n", - set ? "write to" : "read from"); - return err; - } - - /* - * Select the IPC cmd and the ctrl_type based on the ctrl_cmd and the - * direction - * Note: SOF_CTRL_TYPE_VALUE_COMP_* is not used and supported currently - * for ctrl_type - */ - if (cdata->cmd == SOF_CTRL_CMD_BINARY) { - ipc_cmd = set ? SOF_IPC_COMP_SET_DATA : SOF_IPC_COMP_GET_DATA; - ctrl_type = set ? SOF_CTRL_TYPE_DATA_SET : SOF_CTRL_TYPE_DATA_GET; - } else { - ipc_cmd = set ? SOF_IPC_COMP_SET_VALUE : SOF_IPC_COMP_GET_VALUE; - ctrl_type = set ? SOF_CTRL_TYPE_VALUE_CHAN_SET : SOF_CTRL_TYPE_VALUE_CHAN_GET; - } - - cdata->rhdr.hdr.cmd = SOF_IPC_GLB_COMP_MSG | ipc_cmd; - cdata->type = ctrl_type; - cdata->comp_id = scontrol->comp_id; - cdata->msg_index = 0; - - /* calculate header and data size */ - switch (cdata->type) { - case SOF_CTRL_TYPE_VALUE_CHAN_GET: - case SOF_CTRL_TYPE_VALUE_CHAN_SET: - sparams.msg_bytes = scontrol->num_channels * - sizeof(struct sof_ipc_ctrl_value_chan); - sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data); - sparams.elems = scontrol->num_channels; - break; - case SOF_CTRL_TYPE_DATA_GET: - case SOF_CTRL_TYPE_DATA_SET: - sparams.msg_bytes = cdata->data->size; - sparams.hdr_bytes = sizeof(struct sof_ipc_ctrl_data) + - sizeof(struct sof_abi_hdr); - sparams.elems = cdata->data->size; - break; - default: - return -EINVAL; - } - - cdata->rhdr.hdr.size = sparams.msg_bytes + sparams.hdr_bytes; - cdata->num_elems = sparams.elems; - cdata->elems_remaining = 0; - - /* send normal size ipc in one part */ - if (cdata->rhdr.hdr.size <= SOF_IPC_MSG_MAX_SIZE) { - err = sof_ipc_tx_message(sdev->ipc, cdata->rhdr.hdr.cmd, cdata, - cdata->rhdr.hdr.size, cdata, - cdata->rhdr.hdr.size); - - if (err < 0) - dev_err(sdev->dev, "error: set/get ctrl ipc comp %d\n", - cdata->comp_id); - - return err; - } - - /* data is bigger than max ipc size, chop into smaller pieces */ - dev_dbg(sdev->dev, "large ipc size %u, control size %u\n", - cdata->rhdr.hdr.size, scontrol->size); - - /* large messages is only supported from ABI 3.3.0 onwards */ - if (v->abi_version < SOF_ABI_VER(3, 3, 0)) { - dev_err(sdev->dev, "error: incompatible FW ABI version\n"); - return -EINVAL; - } - - err = sof_set_get_large_ctrl_data(sdev, cdata, &sparams, set); - - if (err < 0) - dev_err(sdev->dev, "error: set/get large ctrl ipc comp %d\n", - cdata->comp_id); - - return err; -} -EXPORT_SYMBOL(snd_sof_ipc_set_get_comp_data); - -int snd_sof_ipc_valid(struct snd_sof_dev *sdev) -{ - struct sof_ipc_fw_ready *ready = &sdev->fw_ready; - struct sof_ipc_fw_version *v = &ready->version; - - dev_info(sdev->dev, - "Firmware info: version %d:%d:%d-%s\n", v->major, v->minor, - v->micro, v->tag); - dev_info(sdev->dev, - "Firmware: ABI %d:%d:%d Kernel ABI %d:%d:%d\n", - SOF_ABI_VERSION_MAJOR(v->abi_version), - SOF_ABI_VERSION_MINOR(v->abi_version), - SOF_ABI_VERSION_PATCH(v->abi_version), - SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH); - - if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, v->abi_version)) { - dev_err(sdev->dev, "error: incompatible FW ABI version\n"); - return -EINVAL; - } - - if (SOF_ABI_VERSION_MINOR(v->abi_version) > SOF_ABI_MINOR) { - if (!IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS)) { - dev_warn(sdev->dev, "warn: FW ABI is more recent than kernel\n"); - } else { - dev_err(sdev->dev, "error: FW ABI is more recent than kernel\n"); - return -EINVAL; - } - } - - if (ready->flags & SOF_IPC_INFO_BUILD) { - dev_info(sdev->dev, - "Firmware debug build %d on %s-%s - options:\n" - " GDB: %s\n" - " lock debug: %s\n" - " lock vdebug: %s\n", - v->build, v->date, v->time, - (ready->flags & SOF_IPC_INFO_GDB) ? - "enabled" : "disabled", - (ready->flags & SOF_IPC_INFO_LOCKS) ? - "enabled" : "disabled", - (ready->flags & SOF_IPC_INFO_LOCKSV) ? - "enabled" : "disabled"); - } - - /* copy the fw_version into debugfs at first boot */ - memcpy(&sdev->fw_version, v, sizeof(*v)); - - return 0; -} -EXPORT_SYMBOL(snd_sof_ipc_valid); - -int sof_ipc_init_msg_memory(struct snd_sof_dev *sdev) -{ - struct snd_sof_ipc_msg *msg; - - msg = &sdev->ipc->msg; - - msg->reply_data = devm_kzalloc(sdev->dev, SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); - if (!msg->reply_data) - return -ENOMEM; - - return 0; -} - struct snd_sof_ipc *snd_sof_ipc_init(struct snd_sof_dev *sdev) { struct snd_sof_ipc *ipc; struct snd_sof_ipc_msg *msg; + const struct sof_ipc_ops *ops; ipc = devm_kzalloc(sdev->dev, sizeof(*ipc), GFP_KERNEL); if (!ipc) @@ -1026,19 +155,53 @@ struct snd_sof_ipc *snd_sof_ipc_init(struct snd_sof_dev *sdev) init_waitqueue_head(&msg->waitq); - /* - * Use IPC3 ops as it is the only available version now. With the addition of new IPC - * versions, this will need to be modified to use the selected version at runtime. - */ - ipc->ops = &ipc3_ops; - - /* check for mandatory ops */ - if (!ipc->ops->pcm || !ipc->ops->tplg || !ipc->ops->tplg->widget || - !ipc->ops->tplg->control) { - dev_err(sdev->dev, "Invalid IPC ops\n"); + switch (sdev->pdata->ipc_type) { +#if defined(CONFIG_SND_SOC_SOF_IPC3) + case SOF_IPC: + ops = &ipc3_ops; + break; +#endif +#if defined(CONFIG_SND_SOC_SOF_INTEL_IPC4) + case SOF_INTEL_IPC4: + ops = &ipc4_ops; + break; +#endif + default: + dev_err(sdev->dev, "Not supported IPC version: %d\n", + sdev->pdata->ipc_type); return NULL; } + /* check for mandatory ops */ + if (!ops->tx_msg || !ops->rx_msg || !ops->set_get_data || !ops->get_reply) { + dev_err(sdev->dev, "Missing IPC message handling ops\n"); + return NULL; + } + + if (!ops->fw_loader || !ops->fw_loader->validate || + !ops->fw_loader->parse_ext_manifest) { + dev_err(sdev->dev, "Missing IPC firmware loading ops\n"); + return NULL; + } + + if (!ops->pcm) { + dev_err(sdev->dev, "Missing IPC PCM ops\n"); + return NULL; + } + + if (!ops->tplg || !ops->tplg->widget || !ops->tplg->control) { + dev_err(sdev->dev, "Missing IPC topology ops\n"); + return NULL; + } + + if (ops->fw_tracing && (!ops->fw_tracing->init || !ops->fw_tracing->suspend || + !ops->fw_tracing->resume)) { + dev_err(sdev->dev, "Missing firmware tracing ops\n"); + return NULL; + } + + ipc->ops = ops; + return ipc; } EXPORT_SYMBOL(snd_sof_ipc_init); diff --git a/sound/soc/sof/ipc3-control.c b/sound/soc/sof/ipc3-control.c index cdd5ad860a..3fdc0d854e 100644 --- a/sound/soc/sof/ipc3-control.c +++ b/sound/soc/sof/ipc3-control.c @@ -9,26 +9,85 @@ #include "sof-priv.h" #include "sof-audio.h" -#include "ipc3-ops.h" +#include "ipc3-priv.h" -static inline u32 mixer_to_ipc(unsigned int value, u32 *volume_map, int size) +/* IPC set()/get() for kcontrols. */ +static int sof_ipc3_set_get_kcontrol_data(struct snd_sof_control *scontrol, bool set) { - if (value >= size) - return volume_map[size - 1]; + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scontrol->scomp); + struct sof_ipc_ctrl_data *cdata = scontrol->ipc_control_data; + const struct sof_ipc_ops *iops = sdev->ipc->ops; + enum sof_ipc_ctrl_type ctrl_type; + struct snd_sof_widget *swidget; + bool widget_found = false; + u32 ipc_cmd, msg_bytes; - return volume_map[value]; -} - -static inline u32 ipc_to_mixer(u32 value, u32 *volume_map, int size) -{ - int i; - - for (i = 0; i < size; i++) { - if (volume_map[i] >= value) - return i; + list_for_each_entry(swidget, &sdev->widget_list, list) { + if (swidget->comp_id == scontrol->comp_id) { + widget_found = true; + break; + } } - return i - 1; + if (!widget_found) { + dev_err(sdev->dev, "%s: can't find widget with id %d\n", __func__, + scontrol->comp_id); + return -EINVAL; + } + + /* + * Volatile controls should always be part of static pipelines and the widget use_count + * would always be > 0 in this case. For the others, just return the cached value if the + * widget is not set up. + */ + if (!swidget->use_count) + return 0; + + /* + * Select the IPC cmd and the ctrl_type based on the ctrl_cmd and the + * direction + * Note: SOF_CTRL_TYPE_VALUE_COMP_* is not used and supported currently + * for ctrl_type + */ + if (cdata->cmd == SOF_CTRL_CMD_BINARY) { + ipc_cmd = set ? SOF_IPC_COMP_SET_DATA : SOF_IPC_COMP_GET_DATA; + ctrl_type = set ? SOF_CTRL_TYPE_DATA_SET : SOF_CTRL_TYPE_DATA_GET; + } else { + ipc_cmd = set ? SOF_IPC_COMP_SET_VALUE : SOF_IPC_COMP_GET_VALUE; + ctrl_type = set ? SOF_CTRL_TYPE_VALUE_CHAN_SET : SOF_CTRL_TYPE_VALUE_CHAN_GET; + } + + cdata->rhdr.hdr.cmd = SOF_IPC_GLB_COMP_MSG | ipc_cmd; + cdata->type = ctrl_type; + cdata->comp_id = scontrol->comp_id; + cdata->msg_index = 0; + + /* calculate header and data size */ + switch (cdata->type) { + case SOF_CTRL_TYPE_VALUE_CHAN_GET: + case SOF_CTRL_TYPE_VALUE_CHAN_SET: + cdata->num_elems = scontrol->num_channels; + + msg_bytes = scontrol->num_channels * + sizeof(struct sof_ipc_ctrl_value_chan); + msg_bytes += sizeof(struct sof_ipc_ctrl_data); + break; + case SOF_CTRL_TYPE_DATA_GET: + case SOF_CTRL_TYPE_DATA_SET: + cdata->num_elems = cdata->data->size; + + msg_bytes = cdata->data->size; + msg_bytes += sizeof(struct sof_ipc_ctrl_data) + + sizeof(struct sof_abi_hdr); + break; + default: + return -EINVAL; + } + + cdata->rhdr.hdr.size = msg_bytes; + cdata->elems_remaining = 0; + + return iops->set_get_data(sdev, cdata, cdata->rhdr.hdr.size, set); } static void snd_sof_refresh_control(struct snd_sof_control *scontrol) @@ -49,7 +108,7 @@ static void snd_sof_refresh_control(struct snd_sof_control *scontrol) /* refresh the component data from DSP */ scontrol->comp_data_dirty = false; - ret = snd_sof_ipc_set_get_comp_data(scontrol, false); + ret = sof_ipc3_set_get_kcontrol_data(scontrol, false); if (ret < 0) { dev_err(scomp->dev, "Failed to get control data: %d\n", ret); @@ -97,7 +156,7 @@ static bool sof_ipc3_volume_put(struct snd_sof_control *scontrol, /* notify DSP of mixer updates */ if (pm_runtime_active(scomp->dev)) { - int ret = snd_sof_ipc_set_get_comp_data(scontrol, true); + int ret = sof_ipc3_set_get_kcontrol_data(scontrol, true); if (ret < 0) { dev_err(scomp->dev, "Failed to set mixer updates for %s\n", @@ -145,7 +204,7 @@ static bool sof_ipc3_switch_put(struct snd_sof_control *scontrol, /* notify DSP of mixer updates */ if (pm_runtime_active(scomp->dev)) { - int ret = snd_sof_ipc_set_get_comp_data(scontrol, true); + int ret = sof_ipc3_set_get_kcontrol_data(scontrol, true); if (ret < 0) { dev_err(scomp->dev, "Failed to set mixer updates for %s\n", @@ -193,7 +252,7 @@ static bool sof_ipc3_enum_put(struct snd_sof_control *scontrol, /* notify DSP of enum updates */ if (pm_runtime_active(scomp->dev)) { - int ret = snd_sof_ipc_set_get_comp_data(scontrol, true); + int ret = sof_ipc3_set_get_kcontrol_data(scontrol, true); if (ret < 0) { dev_err(scomp->dev, "Failed to set enum updates for %s\n", @@ -265,7 +324,7 @@ static int sof_ipc3_bytes_put(struct snd_sof_control *scontrol, /* notify DSP of byte control updates */ if (pm_runtime_active(scomp->dev)) - return snd_sof_ipc_set_get_comp_data(scontrol, true); + return sof_ipc3_set_get_kcontrol_data(scontrol, true); return 0; } @@ -379,7 +438,7 @@ static int sof_ipc3_bytes_ext_put(struct snd_sof_control *scontrol, /* notify DSP of byte control updates */ if (pm_runtime_active(scomp->dev)) - return snd_sof_ipc_set_get_comp_data(scontrol, true); + return sof_ipc3_set_get_kcontrol_data(scontrol, true); return 0; } @@ -409,7 +468,7 @@ static int sof_ipc3_bytes_ext_volatile_get(struct snd_sof_control *scontrol, cdata->data->abi = SOF_ABI_VERSION; /* get all the component data from DSP */ - ret = snd_sof_ipc_set_get_comp_data(scontrol, false); + ret = sof_ipc3_set_get_kcontrol_data(scontrol, false); if (ret < 0) return ret; @@ -578,6 +637,60 @@ static void sof_ipc3_control_update(struct snd_sof_dev *sdev, void *ipc_control_ snd_ctl_notify_one(swidget->scomp->card->snd_card, SNDRV_CTL_EVENT_MASK_VALUE, kc, 0); } +static int sof_ipc3_widget_kcontrol_setup(struct snd_sof_dev *sdev, + struct snd_sof_widget *swidget) +{ + struct snd_sof_control *scontrol; + int ret; + + /* set up all controls for the widget */ + list_for_each_entry(scontrol, &sdev->kcontrol_list, list) + if (scontrol->comp_id == swidget->comp_id) { + /* set kcontrol data in DSP */ + ret = sof_ipc3_set_get_kcontrol_data(scontrol, true); + if (ret < 0) { + dev_err(sdev->dev, + "kcontrol %d set up failed for widget %s\n", + scontrol->comp_id, swidget->widget->name); + return ret; + } + + /* + * Read back the data from the DSP for static widgets. + * This is particularly useful for binary kcontrols + * associated with static pipeline widgets to initialize + * the data size to match that in the DSP. + */ + if (swidget->dynamic_pipeline_widget) + continue; + + ret = sof_ipc3_set_get_kcontrol_data(scontrol, false); + if (ret < 0) + dev_warn(sdev->dev, + "kcontrol %d read failed for widget %s\n", + scontrol->comp_id, swidget->widget->name); + } + + return 0; +} + +static int +sof_ipc3_set_up_volume_table(struct snd_sof_control *scontrol, int tlv[SOF_TLV_ITEMS], int size) +{ + int i; + + /* init the volume table */ + scontrol->volume_table = kcalloc(size, sizeof(u32), GFP_KERNEL); + if (!scontrol->volume_table) + return -ENOMEM; + + /* populate the volume table */ + for (i = 0; i < size ; i++) + scontrol->volume_table[i] = vol_compute_gain(i, tlv); + + return 0; +} + const struct sof_ipc_tplg_control_ops tplg_ipc3_control_ops = { .volume_put = sof_ipc3_volume_put, .volume_get = sof_ipc3_volume_get, @@ -591,4 +704,6 @@ const struct sof_ipc_tplg_control_ops tplg_ipc3_control_ops = { .bytes_ext_get = sof_ipc3_bytes_ext_get, .bytes_ext_volatile_get = sof_ipc3_bytes_ext_volatile_get, .update = sof_ipc3_control_update, + .widget_kcontrol_setup = sof_ipc3_widget_kcontrol_setup, + .set_up_volume_table = sof_ipc3_set_up_volume_table, }; diff --git a/sound/soc/sof/ipc3-pcm.c b/sound/soc/sof/ipc3-pcm.c index 58b75943cf..9c6a84bdec 100644 --- a/sound/soc/sof/ipc3-pcm.c +++ b/sound/soc/sof/ipc3-pcm.c @@ -8,7 +8,7 @@ // #include -#include "ipc3-ops.h" +#include "ipc3-priv.h" #include "ops.h" #include "sof-priv.h" #include "sof-audio.h" @@ -34,8 +34,7 @@ static int sof_ipc3_pcm_hw_free(struct snd_soc_component *component, stream.comp_id = spcm->stream[substream->stream].comp_id; /* send IPC to the DSP */ - return sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream, - sizeof(stream), &reply, sizeof(reply)); + return sof_ipc_tx_message(sdev->ipc, &stream, sizeof(stream), &reply, sizeof(reply)); } static int sof_ipc3_pcm_hw_params(struct snd_soc_component *component, @@ -116,10 +115,13 @@ static int sof_ipc3_pcm_hw_params(struct snd_soc_component *component, pcm.params.no_stream_position = 1; } + if (platform_params->cont_update_posn) + pcm.params.cont_update_posn = 1; + dev_dbg(component->dev, "stream_tag %d", pcm.params.stream_tag); /* send hw_params IPC to the DSP */ - ret = sof_ipc_tx_message(sdev->ipc, pcm.hdr.cmd, &pcm, sizeof(pcm), + ret = sof_ipc_tx_message(sdev->ipc, &pcm, sizeof(pcm), &ipc_params_reply, sizeof(ipc_params_reply)); if (ret < 0) { dev_err(component->dev, "HW params ipc failed for stream %d\n", @@ -175,8 +177,7 @@ static int sof_ipc3_pcm_trigger(struct snd_soc_component *component, } /* send IPC to the DSP */ - return sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream, - sizeof(stream), &reply, sizeof(reply)); + return sof_ipc_tx_message(sdev->ipc, &stream, sizeof(stream), &reply, sizeof(reply)); } static void ssp_dai_config_pcm_params_match(struct snd_sof_dev *sdev, const char *link_name, @@ -346,10 +347,10 @@ static int sof_ipc3_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd, channels->min, channels->max); break; case SOF_DAI_AMD_DMIC: - rate->min = private->dai_config->acpdmic.fsync_rate; - rate->max = private->dai_config->acpdmic.fsync_rate; - channels->min = private->dai_config->acpdmic.tdm_slots; - channels->max = private->dai_config->acpdmic.tdm_slots; + rate->min = private->dai_config->acpdmic.pdm_rate; + rate->max = private->dai_config->acpdmic.pdm_rate; + channels->min = private->dai_config->acpdmic.pdm_ch; + channels->max = private->dai_config->acpdmic.pdm_ch; dev_dbg(component->dev, "AMD_DMIC rate_min: %d rate_max: %d\n", rate->min, rate->max); diff --git a/sound/soc/sof/ipc3-topology.c b/sound/soc/sof/ipc3-topology.c index 2f8450a8c0..65923e7a59 100644 --- a/sound/soc/sof/ipc3-topology.c +++ b/sound/soc/sof/ipc3-topology.c @@ -11,16 +11,20 @@ #include #include "sof-priv.h" #include "sof-audio.h" -#include "ipc3-ops.h" +#include "ipc3-priv.h" #include "ops.h" /* Full volume for default values */ #define VOL_ZERO_DB BIT(VOLUME_FWL) +/* size of tplg ABI in bytes */ +#define SOF_IPC3_TPLG_ABI_SIZE 3 + struct sof_widget_data { int ctrl_type; int ipc_cmd; - struct sof_abi_hdr *pdata; + void *pdata; + size_t pdata_size; struct snd_sof_control *control; }; @@ -262,6 +266,16 @@ static const struct sof_topology_token afe_tokens[] = { offsetof(struct sof_ipc_dai_mtk_afe_params, format)}, }; +/* ACPDMIC */ +static const struct sof_topology_token acpdmic_tokens[] = { + {SOF_TKN_AMD_ACPDMIC_RATE, + SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, + offsetof(struct sof_ipc_dai_acpdmic_params, pdm_rate)}, + {SOF_TKN_AMD_ACPDMIC_CH, + SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, + offsetof(struct sof_ipc_dai_acpdmic_params, pdm_ch)}, +}; + /* Core tokens */ static const struct sof_topology_token core_tokens[] = { {SOF_TKN_COMP_CORE_ID, SND_SOC_TPLG_TUPLE_TYPE_WORD, get_token_u32, @@ -296,6 +310,7 @@ static const struct sof_token_info ipc3_token_list[SOF_TOKEN_COUNT] = { [SOF_ESAI_TOKENS] = {"ESAI tokens", esai_tokens, ARRAY_SIZE(esai_tokens)}, [SOF_SAI_TOKENS] = {"SAI tokens", sai_tokens, ARRAY_SIZE(sai_tokens)}, [SOF_AFE_TOKENS] = {"AFE tokens", afe_tokens, ARRAY_SIZE(afe_tokens)}, + [SOF_ACPDMIC_TOKENS] = {"ACPDMIC tokens", acpdmic_tokens, ARRAY_SIZE(acpdmic_tokens)}, }; /** @@ -784,16 +799,26 @@ static int sof_get_control_data(struct snd_soc_component *scomp, } cdata = wdata[i].control->ipc_control_data; - wdata[i].pdata = cdata->data; - if (!wdata[i].pdata) - return -EINVAL; - /* make sure data is valid - data can be updated at runtime */ - if (widget->dobj.widget.kcontrol_type[i] == SND_SOC_TPLG_TYPE_BYTES && - wdata[i].pdata->magic != SOF_ABI_MAGIC) - return -EINVAL; + if (widget->dobj.widget.kcontrol_type[i] == SND_SOC_TPLG_TYPE_BYTES) { + /* make sure data is valid - data can be updated at runtime */ + if (cdata->data->magic != SOF_ABI_MAGIC) + return -EINVAL; - *size += wdata[i].pdata->size; + wdata[i].pdata = cdata->data->data; + wdata[i].pdata_size = cdata->data->size; + } else { + /* points to the control data union */ + wdata[i].pdata = cdata->chanv; + /* + * wdata[i].control->size is calculated with struct_size + * and includes the size of struct sof_ipc_ctrl_data + */ + wdata[i].pdata_size = wdata[i].control->size - + sizeof(struct sof_ipc_ctrl_data); + } + + *size += wdata[i].pdata_size; /* get data type */ switch (cdata->cmd) { @@ -876,10 +901,12 @@ static int sof_process_load(struct snd_soc_component *scomp, */ if (ipc_data_size) { for (i = 0; i < widget->num_kcontrols; i++) { - memcpy(&process->data[offset], - wdata[i].pdata->data, - wdata[i].pdata->size); - offset += wdata[i].pdata->size; + if (!wdata[i].pdata_size) + continue; + + memcpy(&process->data[offset], wdata[i].pdata, + wdata[i].pdata_size); + offset += wdata[i].pdata_size; } } @@ -1104,20 +1131,22 @@ static int sof_link_acp_dmic_load(struct snd_soc_component *scomp, struct snd_so struct snd_soc_tplg_hw_config *hw_config = slink->hw_configs; struct sof_dai_private_data *private = dai->private; u32 size = sizeof(*config); + int ret; /* handle master/slave and inverted clocks */ sof_dai_set_format(hw_config, config); - /* init IPC */ - memset(&config->acpdmic, 0, sizeof(config->acpdmic)); config->hdr.size = size; - config->acpdmic.fsync_rate = le32_to_cpu(hw_config->fsync_rate); - config->acpdmic.tdm_slots = le32_to_cpu(hw_config->tdm_slots); + /* parse the required set of ACPDMIC tokens based on num_hw_cfgs */ + ret = sof_update_ipc_object(scomp, &config->acpdmic, SOF_ACPDMIC_TOKENS, slink->tuples, + slink->num_tuples, size, slink->num_hw_configs); + if (ret < 0) + return ret; dev_info(scomp->dev, "ACP_DMIC config ACP%d channel %d rate %d\n", - config->dai_index, config->acpdmic.tdm_slots, - config->acpdmic.fsync_rate); + config->dai_index, config->acpdmic.pdm_ch, + config->acpdmic.pdm_rate); dai->number_configs = 1; dai->current_config = 0; @@ -1423,8 +1452,8 @@ static int sof_ipc3_widget_setup_comp_dai(struct snd_sof_widget *swidget) if (ret < 0) goto free; - dev_dbg(scomp->dev, "%s dai %s: type %d index %d\n", - __func__, swidget->widget->name, comp_dai->type, comp_dai->dai_index); + dev_dbg(scomp->dev, "dai %s: type %d index %d\n", + swidget->widget->name, comp_dai->type, comp_dai->dai_index); sof_dbg_comp_config(scomp, &comp_dai->config); /* now update DAI config */ @@ -1551,8 +1580,7 @@ static int sof_ipc3_route_setup(struct snd_sof_dev *sdev, struct snd_sof_route * sroute->sink_widget->widget->name); /* send ipc */ - ret = sof_ipc_tx_message(sdev->ipc, connect.hdr.cmd, &connect, sizeof(connect), - &reply, sizeof(reply)); + ret = sof_ipc_tx_message(sdev->ipc, &connect, sizeof(connect), &reply, sizeof(reply)); if (ret < 0) dev_err(sdev->dev, "%s: route %s -> %s failed\n", __func__, sroute->src_widget->widget->name, sroute->sink_widget->widget->name); @@ -1565,24 +1593,23 @@ static int sof_ipc3_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ struct sof_ipc_ctrl_data *cdata; int ret; + if (scontrol->max_size < (sizeof(*cdata) + sizeof(struct sof_abi_hdr))) { + dev_err(sdev->dev, "%s: insufficient size for a bytes control: %zu.\n", + __func__, scontrol->max_size); + return -EINVAL; + } + + if (scontrol->priv_size > scontrol->max_size - sizeof(*cdata)) { + dev_err(sdev->dev, + "%s: bytes data size %zu exceeds max %zu.\n", __func__, + scontrol->priv_size, scontrol->max_size - sizeof(*cdata)); + return -EINVAL; + } + scontrol->ipc_control_data = kzalloc(scontrol->max_size, GFP_KERNEL); if (!scontrol->ipc_control_data) return -ENOMEM; - if (scontrol->max_size < sizeof(*cdata) || - scontrol->max_size < sizeof(struct sof_abi_hdr)) { - ret = -EINVAL; - goto err; - } - - /* init the get/put bytes data */ - if (scontrol->priv_size > scontrol->max_size - sizeof(*cdata)) { - dev_err(sdev->dev, "err: bytes data size %zu exceeds max %zu.\n", - scontrol->priv_size, scontrol->max_size - sizeof(*cdata)); - ret = -EINVAL; - goto err; - } - scontrol->size = sizeof(struct sof_ipc_ctrl_data) + scontrol->priv_size; cdata = scontrol->ipc_control_data; @@ -1592,6 +1619,7 @@ static int sof_ipc3_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ if (scontrol->priv_size > 0) { memcpy(cdata->data, scontrol->priv, scontrol->priv_size); kfree(scontrol->priv); + scontrol->priv = NULL; if (cdata->data->magic != SOF_ABI_MAGIC) { dev_err(sdev->dev, "Wrong ABI magic 0x%08x.\n", cdata->data->magic); @@ -1616,6 +1644,7 @@ static int sof_ipc3_control_load_bytes(struct snd_sof_dev *sdev, struct snd_sof_ return 0; err: kfree(scontrol->ipc_control_data); + scontrol->ipc_control_data = NULL; return ret; } @@ -1697,7 +1726,7 @@ static int sof_ipc3_control_free(struct snd_sof_dev *sdev, struct snd_sof_contro fcomp.id = scontrol->comp_id; /* send IPC to the DSP */ - return sof_ipc_tx_message(sdev->ipc, fcomp.hdr.cmd, &fcomp, sizeof(fcomp), NULL, 0); + return sof_ipc_tx_message(sdev->ipc, &fcomp, sizeof(fcomp), NULL, 0); } /* send pcm params ipc */ @@ -1749,7 +1778,7 @@ static int sof_ipc3_keyword_detect_pcm_params(struct snd_sof_widget *swidget, in } /* send IPC to the DSP */ - ret = sof_ipc_tx_message(sdev->ipc, pcm.hdr.cmd, &pcm, sizeof(pcm), + ret = sof_ipc_tx_message(sdev->ipc, &pcm, sizeof(pcm), &ipc_params_reply, sizeof(ipc_params_reply)); if (ret < 0) dev_err(scomp->dev, "%s: PCM params failed for %s\n", __func__, @@ -1773,8 +1802,7 @@ static int sof_ipc3_keyword_detect_trigger(struct snd_sof_widget *swidget, int c stream.comp_id = swidget->comp_id; /* send IPC to the DSP */ - ret = sof_ipc_tx_message(sdev->ipc, stream.hdr.cmd, &stream, - sizeof(stream), &reply, sizeof(reply)); + ret = sof_ipc_tx_message(sdev->ipc, &stream, sizeof(stream), &reply, sizeof(reply)); if (ret < 0) dev_err(scomp->dev, "%s: Failed to trigger %s\n", __func__, swidget->widget->name); @@ -1902,8 +1930,7 @@ static int sof_ipc3_complete_pipeline(struct snd_sof_dev *sdev, struct snd_sof_w ready.hdr.cmd = SOF_IPC_GLB_TPLG_MSG | SOF_IPC_TPLG_PIPE_COMPLETE; ready.comp_id = swidget->comp_id; - ret = sof_ipc_tx_message(sdev->ipc, ready.hdr.cmd, &ready, sizeof(ready), &reply, - sizeof(reply)); + ret = sof_ipc_tx_message(sdev->ipc, &ready, sizeof(ready), &reply, sizeof(reply)); if (ret < 0) return ret; @@ -1939,7 +1966,7 @@ static int sof_ipc3_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget break; } - ret = sof_ipc_tx_message(sdev->ipc, ipc_free.hdr.cmd, &ipc_free, sizeof(ipc_free), + ret = sof_ipc_tx_message(sdev->ipc, &ipc_free, sizeof(ipc_free), &reply, sizeof(reply)); if (ret < 0) dev_err(sdev->dev, "failed to free widget %s\n", swidget->widget->name); @@ -2003,7 +2030,7 @@ static int sof_ipc3_dai_config(struct snd_sof_dev *sdev, struct snd_sof_widget * /* only send the IPC if the widget is set up in the DSP */ if (swidget->use_count > 0) { - ret = sof_ipc_tx_message(sdev->ipc, config->hdr.cmd, config, config->hdr.size, + ret = sof_ipc_tx_message(sdev->ipc, config, config->hdr.size, &reply, sizeof(reply)); if (ret < 0) dev_err(sdev->dev, "Failed to set dai config for %s\n", dai->name); @@ -2028,7 +2055,7 @@ static int sof_ipc3_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget struct sof_dai_private_data *dai_data = dai->private; struct sof_ipc_comp *comp = &dai_data->comp_dai->comp; - ret = sof_ipc_tx_message(sdev->ipc, comp->hdr.cmd, dai_data->comp_dai, + ret = sof_ipc_tx_message(sdev->ipc, dai_data->comp_dai, comp->hdr.size, &reply, sizeof(reply)); break; } @@ -2037,8 +2064,8 @@ static int sof_ipc3_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget struct sof_ipc_pipe_new *pipeline; pipeline = swidget->private; - ret = sof_ipc_tx_message(sdev->ipc, pipeline->hdr.cmd, pipeline, - sizeof(*pipeline), &reply, sizeof(reply)); + ret = sof_ipc_tx_message(sdev->ipc, pipeline, sizeof(*pipeline), + &reply, sizeof(reply)); break; } default: @@ -2046,7 +2073,7 @@ static int sof_ipc3_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget struct sof_ipc_cmd_hdr *hdr; hdr = swidget->private; - ret = sof_ipc_tx_message(sdev->ipc, hdr->cmd, swidget->private, hdr->size, + ret = sof_ipc_tx_message(sdev->ipc, swidget->private, hdr->size, &reply, sizeof(reply)); break; } @@ -2249,6 +2276,18 @@ static int sof_ipc3_tear_down_all_pipelines(struct snd_sof_dev *sdev, bool verif list_for_each_entry(sroute, &sdev->route_list, list) sroute->setup = false; + /* + * before suspending, make sure the refcounts are all zeroed out. There's no way + * to recover at this point but this will help root cause bad sequences leading to + * more issues on resume + */ + list_for_each_entry(swidget, &sdev->widget_list, list) { + if (swidget->use_count != 0) { + dev_err(sdev->dev, "%s: widget %s is still in use: count %d\n", + __func__, swidget->widget->name, swidget->use_count); + } + } + return 0; } @@ -2280,6 +2319,45 @@ static int sof_ipc3_dai_get_clk(struct snd_sof_dev *sdev, struct snd_sof_dai *da return -EINVAL; } +static int sof_ipc3_parse_manifest(struct snd_soc_component *scomp, int index, + struct snd_soc_tplg_manifest *man) +{ + u32 size = le32_to_cpu(man->priv.size); + u32 abi_version; + + /* backward compatible with tplg without ABI info */ + if (!size) { + dev_dbg(scomp->dev, "No topology ABI info\n"); + return 0; + } + + if (size != SOF_IPC3_TPLG_ABI_SIZE) { + dev_err(scomp->dev, "%s: Invalid topology ABI size: %u\n", + __func__, size); + return -EINVAL; + } + + dev_info(scomp->dev, + "Topology: ABI %d:%d:%d Kernel ABI %d:%d:%d\n", + man->priv.data[0], man->priv.data[1], man->priv.data[2], + SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH); + + abi_version = SOF_ABI_VER(man->priv.data[0], man->priv.data[1], man->priv.data[2]); + + if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, abi_version)) { + dev_err(scomp->dev, "%s: Incompatible topology ABI version\n", __func__); + return -EINVAL; + } + + if (IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS) && + SOF_ABI_VERSION_MINOR(abi_version) > SOF_ABI_MINOR) { + dev_err(scomp->dev, "%s: Topology ABI is more recent than kernel\n", __func__); + return -EINVAL; + } + + return 0; +} + /* token list for each topology object */ static enum sof_tokens host_token_list[] = { SOF_CORE_TOKENS, @@ -2390,4 +2468,5 @@ const struct sof_ipc_tplg_ops ipc3_tplg_ops = { .dai_get_clk = sof_ipc3_dai_get_clk, .set_up_all_pipelines = sof_ipc3_set_up_all_pipelines, .tear_down_all_pipelines = sof_ipc3_tear_down_all_pipelines, + .parse_manifest = sof_ipc3_parse_manifest, }; diff --git a/sound/soc/sof/ipc3.c b/sound/soc/sof/ipc3.c index 03e914b627..82fa320253 100644 --- a/sound/soc/sof/ipc3.c +++ b/sound/soc/sof/ipc3.c @@ -7,8 +7,1052 @@ // // +#include +#include #include "sof-priv.h" -#include "ipc3-ops.h" +#include "sof-audio.h" +#include "ipc3-priv.h" +#include "ops.h" + +typedef void (*ipc3_rx_callback)(struct snd_sof_dev *sdev, void *msg_buf); + +#if IS_ENABLED(CONFIG_SND_SOC_SOF_DEBUG_VERBOSE_IPC) +static void ipc3_log_header(struct device *dev, u8 *text, u32 cmd) +{ + u8 *str; + u8 *str2 = NULL; + u32 glb; + u32 type; + bool vdbg = false; + + glb = cmd & SOF_GLB_TYPE_MASK; + type = cmd & SOF_CMD_TYPE_MASK; + + switch (glb) { + case SOF_IPC_GLB_REPLY: + str = "GLB_REPLY"; break; + case SOF_IPC_GLB_COMPOUND: + str = "GLB_COMPOUND"; break; + case SOF_IPC_GLB_TPLG_MSG: + str = "GLB_TPLG_MSG"; + switch (type) { + case SOF_IPC_TPLG_COMP_NEW: + str2 = "COMP_NEW"; break; + case SOF_IPC_TPLG_COMP_FREE: + str2 = "COMP_FREE"; break; + case SOF_IPC_TPLG_COMP_CONNECT: + str2 = "COMP_CONNECT"; break; + case SOF_IPC_TPLG_PIPE_NEW: + str2 = "PIPE_NEW"; break; + case SOF_IPC_TPLG_PIPE_FREE: + str2 = "PIPE_FREE"; break; + case SOF_IPC_TPLG_PIPE_CONNECT: + str2 = "PIPE_CONNECT"; break; + case SOF_IPC_TPLG_PIPE_COMPLETE: + str2 = "PIPE_COMPLETE"; break; + case SOF_IPC_TPLG_BUFFER_NEW: + str2 = "BUFFER_NEW"; break; + case SOF_IPC_TPLG_BUFFER_FREE: + str2 = "BUFFER_FREE"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_PM_MSG: + str = "GLB_PM_MSG"; + switch (type) { + case SOF_IPC_PM_CTX_SAVE: + str2 = "CTX_SAVE"; break; + case SOF_IPC_PM_CTX_RESTORE: + str2 = "CTX_RESTORE"; break; + case SOF_IPC_PM_CTX_SIZE: + str2 = "CTX_SIZE"; break; + case SOF_IPC_PM_CLK_SET: + str2 = "CLK_SET"; break; + case SOF_IPC_PM_CLK_GET: + str2 = "CLK_GET"; break; + case SOF_IPC_PM_CLK_REQ: + str2 = "CLK_REQ"; break; + case SOF_IPC_PM_CORE_ENABLE: + str2 = "CORE_ENABLE"; break; + case SOF_IPC_PM_GATE: + str2 = "GATE"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_COMP_MSG: + str = "GLB_COMP_MSG"; + switch (type) { + case SOF_IPC_COMP_SET_VALUE: + str2 = "SET_VALUE"; break; + case SOF_IPC_COMP_GET_VALUE: + str2 = "GET_VALUE"; break; + case SOF_IPC_COMP_SET_DATA: + str2 = "SET_DATA"; break; + case SOF_IPC_COMP_GET_DATA: + str2 = "GET_DATA"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_STREAM_MSG: + str = "GLB_STREAM_MSG"; + switch (type) { + case SOF_IPC_STREAM_PCM_PARAMS: + str2 = "PCM_PARAMS"; break; + case SOF_IPC_STREAM_PCM_PARAMS_REPLY: + str2 = "PCM_REPLY"; break; + case SOF_IPC_STREAM_PCM_FREE: + str2 = "PCM_FREE"; break; + case SOF_IPC_STREAM_TRIG_START: + str2 = "TRIG_START"; break; + case SOF_IPC_STREAM_TRIG_STOP: + str2 = "TRIG_STOP"; break; + case SOF_IPC_STREAM_TRIG_PAUSE: + str2 = "TRIG_PAUSE"; break; + case SOF_IPC_STREAM_TRIG_RELEASE: + str2 = "TRIG_RELEASE"; break; + case SOF_IPC_STREAM_TRIG_DRAIN: + str2 = "TRIG_DRAIN"; break; + case SOF_IPC_STREAM_TRIG_XRUN: + str2 = "TRIG_XRUN"; break; + case SOF_IPC_STREAM_POSITION: + vdbg = true; + str2 = "POSITION"; break; + case SOF_IPC_STREAM_VORBIS_PARAMS: + str2 = "VORBIS_PARAMS"; break; + case SOF_IPC_STREAM_VORBIS_FREE: + str2 = "VORBIS_FREE"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_FW_READY: + str = "FW_READY"; break; + case SOF_IPC_GLB_DAI_MSG: + str = "GLB_DAI_MSG"; + switch (type) { + case SOF_IPC_DAI_CONFIG: + str2 = "CONFIG"; break; + case SOF_IPC_DAI_LOOPBACK: + str2 = "LOOPBACK"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_TRACE_MSG: + str = "GLB_TRACE_MSG"; + switch (type) { + case SOF_IPC_TRACE_DMA_PARAMS: + str2 = "DMA_PARAMS"; break; + case SOF_IPC_TRACE_DMA_POSITION: + if (!sof_debug_check_flag(SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS)) + return; + str2 = "DMA_POSITION"; break; + case SOF_IPC_TRACE_DMA_PARAMS_EXT: + str2 = "DMA_PARAMS_EXT"; break; + case SOF_IPC_TRACE_FILTER_UPDATE: + str2 = "FILTER_UPDATE"; break; + case SOF_IPC_TRACE_DMA_FREE: + str2 = "DMA_FREE"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_TEST_MSG: + str = "GLB_TEST_MSG"; + switch (type) { + case SOF_IPC_TEST_IPC_FLOOD: + str2 = "IPC_FLOOD"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_DEBUG: + str = "GLB_DEBUG"; + switch (type) { + case SOF_IPC_DEBUG_MEM_USAGE: + str2 = "MEM_USAGE"; break; + default: + str2 = "unknown type"; break; + } + break; + case SOF_IPC_GLB_PROBE: + str = "GLB_PROBE"; + switch (type) { + case SOF_IPC_PROBE_INIT: + str2 = "INIT"; break; + case SOF_IPC_PROBE_DEINIT: + str2 = "DEINIT"; break; + case SOF_IPC_PROBE_DMA_ADD: + str2 = "DMA_ADD"; break; + case SOF_IPC_PROBE_DMA_INFO: + str2 = "DMA_INFO"; break; + case SOF_IPC_PROBE_DMA_REMOVE: + str2 = "DMA_REMOVE"; break; + case SOF_IPC_PROBE_POINT_ADD: + str2 = "POINT_ADD"; break; + case SOF_IPC_PROBE_POINT_INFO: + str2 = "POINT_INFO"; break; + case SOF_IPC_PROBE_POINT_REMOVE: + str2 = "POINT_REMOVE"; break; + default: + str2 = "unknown type"; break; + } + break; + default: + str = "unknown GLB command"; break; + } + + if (str2) { + if (vdbg) + dev_vdbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2); + else + dev_dbg(dev, "%s: 0x%x: %s: %s\n", text, cmd, str, str2); + } else { + dev_dbg(dev, "%s: 0x%x: %s\n", text, cmd, str); + } +} +#else +static inline void ipc3_log_header(struct device *dev, u8 *text, u32 cmd) +{ + if ((cmd & SOF_GLB_TYPE_MASK) != SOF_IPC_GLB_TRACE_MSG) + dev_dbg(dev, "%s: 0x%x\n", text, cmd); +} +#endif + +static int sof_ipc3_get_reply(struct snd_sof_dev *sdev) +{ + struct snd_sof_ipc_msg *msg = sdev->msg; + struct sof_ipc_reply *reply; + int ret = 0; + + /* get the generic reply */ + reply = msg->reply_data; + snd_sof_dsp_mailbox_read(sdev, sdev->host_box.offset, reply, sizeof(*reply)); + + if (reply->error < 0) + return reply->error; + + if (!reply->hdr.size) { + /* Reply should always be >= sizeof(struct sof_ipc_reply) */ + if (msg->reply_size) + dev_err(sdev->dev, + "empty reply received, expected %zu bytes\n", + msg->reply_size); + else + dev_err(sdev->dev, "empty reply received\n"); + + return -EINVAL; + } + + if (msg->reply_size > 0) { + if (reply->hdr.size == msg->reply_size) { + ret = 0; + } else if (reply->hdr.size < msg->reply_size) { + dev_dbg(sdev->dev, + "reply size (%u) is less than expected (%zu)\n", + reply->hdr.size, msg->reply_size); + + msg->reply_size = reply->hdr.size; + ret = 0; + } else { + dev_err(sdev->dev, + "reply size (%u) exceeds the buffer size (%zu)\n", + reply->hdr.size, msg->reply_size); + ret = -EINVAL; + } + + /* + * get the full message if reply->hdr.size <= msg->reply_size + * and the reply->hdr.size > sizeof(struct sof_ipc_reply) + */ + if (!ret && msg->reply_size > sizeof(*reply)) + snd_sof_dsp_mailbox_read(sdev, sdev->host_box.offset, + msg->reply_data, msg->reply_size); + } + + return ret; +} + +/* wait for IPC message reply */ +static int ipc3_wait_tx_done(struct snd_sof_ipc *ipc, void *reply_data) +{ + struct snd_sof_ipc_msg *msg = &ipc->msg; + struct sof_ipc_cmd_hdr *hdr = msg->msg_data; + struct snd_sof_dev *sdev = ipc->sdev; + int ret; + + /* wait for DSP IPC completion */ + ret = wait_event_timeout(msg->waitq, msg->ipc_complete, + msecs_to_jiffies(sdev->ipc_timeout)); + + if (ret == 0) { + dev_err(sdev->dev, + "ipc tx timed out for %#x (msg/reply size: %d/%zu)\n", + hdr->cmd, hdr->size, msg->reply_size); + snd_sof_handle_fw_exception(ipc->sdev, "IPC timeout"); + ret = -ETIMEDOUT; + } else { + ret = msg->reply_error; + if (ret < 0) { + dev_err(sdev->dev, + "ipc tx error for %#x (msg/reply size: %d/%zu): %d\n", + hdr->cmd, hdr->size, msg->reply_size, ret); + } else { + if (sof_debug_check_flag(SOF_DBG_PRINT_IPC_SUCCESS_LOGS)) + ipc3_log_header(sdev->dev, "ipc tx succeeded", hdr->cmd); + if (msg->reply_size) + /* copy the data returned from DSP */ + memcpy(reply_data, msg->reply_data, + msg->reply_size); + } + + /* re-enable dumps after successful IPC tx */ + if (sdev->ipc_dump_printed) { + sdev->dbg_dump_printed = false; + sdev->ipc_dump_printed = false; + } + } + + return ret; +} + +/* send IPC message from host to DSP */ +static int ipc3_tx_msg_unlocked(struct snd_sof_ipc *ipc, + void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes) +{ + struct sof_ipc_cmd_hdr *hdr = msg_data; + struct snd_sof_dev *sdev = ipc->sdev; + int ret; + + ret = sof_ipc_send_msg(sdev, msg_data, msg_bytes, reply_bytes); + + if (ret) { + dev_err_ratelimited(sdev->dev, + "%s: ipc message send for %#x failed: %d\n", + __func__, hdr->cmd, ret); + return ret; + } + + ipc3_log_header(sdev->dev, "ipc tx", hdr->cmd); + + /* now wait for completion */ + return ipc3_wait_tx_done(ipc, reply_data); +} + +static int sof_ipc3_tx_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes, bool no_pm) +{ + struct snd_sof_ipc *ipc = sdev->ipc; + int ret; + + if (!msg_data || msg_bytes < sizeof(struct sof_ipc_cmd_hdr)) { + dev_err_ratelimited(sdev->dev, "No IPC message to send\n"); + return -EINVAL; + } + + if (!no_pm) { + const struct sof_dsp_power_state target_state = { + .state = SOF_DSP_PM_D0, + }; + + /* ensure the DSP is in D0 before sending a new IPC */ + ret = snd_sof_dsp_set_power_state(sdev, &target_state); + if (ret < 0) { + dev_err(sdev->dev, "%s: resuming DSP failed: %d\n", + __func__, ret); + return ret; + } + } + + /* Serialise IPC TX */ + mutex_lock(&ipc->tx_mutex); + + ret = ipc3_tx_msg_unlocked(ipc, msg_data, msg_bytes, reply_data, reply_bytes); + + mutex_unlock(&ipc->tx_mutex); + + return ret; +} + +static int sof_ipc3_set_get_data(struct snd_sof_dev *sdev, void *data, size_t data_bytes, + bool set) +{ + size_t msg_bytes, hdr_bytes, payload_size, send_bytes; + struct sof_ipc_ctrl_data *cdata = data; + struct sof_ipc_ctrl_data *cdata_chunk; + struct snd_sof_ipc *ipc = sdev->ipc; + size_t offset = 0; + u8 *src, *dst; + u32 num_msg; + int ret = 0; + int i; + + if (!cdata || data_bytes < sizeof(*cdata)) + return -EINVAL; + + if ((cdata->rhdr.hdr.cmd & SOF_GLB_TYPE_MASK) != SOF_IPC_GLB_COMP_MSG) { + dev_err(sdev->dev, "%s: Not supported message type of %#x\n", + __func__, cdata->rhdr.hdr.cmd); + return -EINVAL; + } + + /* send normal size ipc in one part */ + if (cdata->rhdr.hdr.size <= ipc->max_payload_size) + return sof_ipc3_tx_msg(sdev, cdata, cdata->rhdr.hdr.size, + cdata, cdata->rhdr.hdr.size, false); + + cdata_chunk = kzalloc(ipc->max_payload_size, GFP_KERNEL); + if (!cdata_chunk) + return -ENOMEM; + + switch (cdata->type) { + case SOF_CTRL_TYPE_VALUE_CHAN_GET: + case SOF_CTRL_TYPE_VALUE_CHAN_SET: + hdr_bytes = sizeof(struct sof_ipc_ctrl_data); + if (set) { + src = (u8 *)cdata->chanv; + dst = (u8 *)cdata_chunk->chanv; + } else { + src = (u8 *)cdata_chunk->chanv; + dst = (u8 *)cdata->chanv; + } + break; + case SOF_CTRL_TYPE_DATA_GET: + case SOF_CTRL_TYPE_DATA_SET: + hdr_bytes = sizeof(struct sof_ipc_ctrl_data) + sizeof(struct sof_abi_hdr); + if (set) { + src = (u8 *)cdata->data->data; + dst = (u8 *)cdata_chunk->data->data; + } else { + src = (u8 *)cdata_chunk->data->data; + dst = (u8 *)cdata->data->data; + } + break; + default: + kfree(cdata_chunk); + return -EINVAL; + } + + msg_bytes = cdata->rhdr.hdr.size - hdr_bytes; + payload_size = ipc->max_payload_size - hdr_bytes; + num_msg = DIV_ROUND_UP(msg_bytes, payload_size); + + /* copy the header data */ + memcpy(cdata_chunk, cdata, hdr_bytes); + + /* Serialise IPC TX */ + mutex_lock(&sdev->ipc->tx_mutex); + + /* copy the payload data in a loop */ + for (i = 0; i < num_msg; i++) { + send_bytes = min(msg_bytes, payload_size); + cdata_chunk->num_elems = send_bytes; + cdata_chunk->rhdr.hdr.size = hdr_bytes + send_bytes; + cdata_chunk->msg_index = i; + msg_bytes -= send_bytes; + cdata_chunk->elems_remaining = msg_bytes; + + if (set) + memcpy(dst, src + offset, send_bytes); + + ret = ipc3_tx_msg_unlocked(sdev->ipc, + cdata_chunk, cdata_chunk->rhdr.hdr.size, + cdata_chunk, cdata_chunk->rhdr.hdr.size); + if (ret < 0) + break; + + if (!set) + memcpy(dst + offset, src, send_bytes); + + offset += payload_size; + } + + mutex_unlock(&sdev->ipc->tx_mutex); + + kfree(cdata_chunk); + + return ret; +} + +int sof_ipc3_get_ext_windows(struct snd_sof_dev *sdev, + const struct sof_ipc_ext_data_hdr *ext_hdr) +{ + const struct sof_ipc_window *w = + container_of(ext_hdr, struct sof_ipc_window, ext_hdr); + + if (w->num_windows == 0 || w->num_windows > SOF_IPC_MAX_ELEMS) + return -EINVAL; + + if (sdev->info_window) { + if (memcmp(sdev->info_window, w, ext_hdr->hdr.size)) { + dev_err(sdev->dev, "mismatch between window descriptor from extended manifest and mailbox"); + return -EINVAL; + } + return 0; + } + + /* keep a local copy of the data */ + sdev->info_window = devm_kmemdup(sdev->dev, w, ext_hdr->hdr.size, GFP_KERNEL); + if (!sdev->info_window) + return -ENOMEM; + + return 0; +} + +int sof_ipc3_get_cc_info(struct snd_sof_dev *sdev, + const struct sof_ipc_ext_data_hdr *ext_hdr) +{ + int ret; + + const struct sof_ipc_cc_version *cc = + container_of(ext_hdr, struct sof_ipc_cc_version, ext_hdr); + + if (sdev->cc_version) { + if (memcmp(sdev->cc_version, cc, cc->ext_hdr.hdr.size)) { + dev_err(sdev->dev, + "Receive diverged cc_version descriptions"); + return -EINVAL; + } + return 0; + } + + dev_dbg(sdev->dev, + "Firmware info: used compiler %s %d:%d:%d%s used optimization flags %s\n", + cc->name, cc->major, cc->minor, cc->micro, cc->desc, cc->optim); + + /* create read-only cc_version debugfs to store compiler version info */ + /* use local copy of the cc_version to prevent data corruption */ + if (sdev->first_boot) { + sdev->cc_version = devm_kmalloc(sdev->dev, cc->ext_hdr.hdr.size, + GFP_KERNEL); + + if (!sdev->cc_version) + return -ENOMEM; + + memcpy(sdev->cc_version, cc, cc->ext_hdr.hdr.size); + ret = snd_sof_debugfs_buf_item(sdev, sdev->cc_version, + cc->ext_hdr.hdr.size, + "cc_version", 0444); + + /* errors are only due to memory allocation, not debugfs */ + if (ret < 0) { + dev_err(sdev->dev, "snd_sof_debugfs_buf_item failed\n"); + return ret; + } + } + + return 0; +} + +/* parse the extended FW boot data structures from FW boot message */ +static int ipc3_fw_parse_ext_data(struct snd_sof_dev *sdev, u32 offset) +{ + struct sof_ipc_ext_data_hdr *ext_hdr; + void *ext_data; + int ret = 0; + + ext_data = kzalloc(PAGE_SIZE, GFP_KERNEL); + if (!ext_data) + return -ENOMEM; + + /* get first header */ + snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, offset, ext_data, + sizeof(*ext_hdr)); + ext_hdr = ext_data; + + while (ext_hdr->hdr.cmd == SOF_IPC_FW_READY) { + /* read in ext structure */ + snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, + offset + sizeof(*ext_hdr), + (void *)((u8 *)ext_data + sizeof(*ext_hdr)), + ext_hdr->hdr.size - sizeof(*ext_hdr)); + + dev_dbg(sdev->dev, "found ext header type %d size 0x%x\n", + ext_hdr->type, ext_hdr->hdr.size); + + /* process structure data */ + switch (ext_hdr->type) { + case SOF_IPC_EXT_WINDOW: + ret = sof_ipc3_get_ext_windows(sdev, ext_hdr); + break; + case SOF_IPC_EXT_CC_INFO: + ret = sof_ipc3_get_cc_info(sdev, ext_hdr); + break; + case SOF_IPC_EXT_UNUSED: + case SOF_IPC_EXT_PROBE_INFO: + case SOF_IPC_EXT_USER_ABI_INFO: + /* They are supported but we don't do anything here */ + break; + default: + dev_info(sdev->dev, "unknown ext header type %d size 0x%x\n", + ext_hdr->type, ext_hdr->hdr.size); + ret = 0; + break; + } + + if (ret < 0) { + dev_err(sdev->dev, "Failed to parse ext data type %d\n", + ext_hdr->type); + break; + } + + /* move to next header */ + offset += ext_hdr->hdr.size; + snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, offset, ext_data, + sizeof(*ext_hdr)); + ext_hdr = ext_data; + } + + kfree(ext_data); + return ret; +} + +static void ipc3_get_windows(struct snd_sof_dev *sdev) +{ + struct sof_ipc_window_elem *elem; + u32 outbox_offset = 0; + u32 stream_offset = 0; + u32 inbox_offset = 0; + u32 outbox_size = 0; + u32 stream_size = 0; + u32 inbox_size = 0; + u32 debug_size = 0; + u32 debug_offset = 0; + int window_offset; + int i; + + if (!sdev->info_window) { + dev_err(sdev->dev, "%s: No window info present\n", __func__); + return; + } + + for (i = 0; i < sdev->info_window->num_windows; i++) { + elem = &sdev->info_window->window[i]; + + window_offset = snd_sof_dsp_get_window_offset(sdev, elem->id); + if (window_offset < 0) { + dev_warn(sdev->dev, "No offset for window %d\n", elem->id); + continue; + } + + switch (elem->type) { + case SOF_IPC_REGION_UPBOX: + inbox_offset = window_offset + elem->offset; + inbox_size = elem->size; + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + inbox_offset, + elem->size, "inbox", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + case SOF_IPC_REGION_DOWNBOX: + outbox_offset = window_offset + elem->offset; + outbox_size = elem->size; + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + outbox_offset, + elem->size, "outbox", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + case SOF_IPC_REGION_TRACE: + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + window_offset + elem->offset, + elem->size, "etrace", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + case SOF_IPC_REGION_DEBUG: + debug_offset = window_offset + elem->offset; + debug_size = elem->size; + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + window_offset + elem->offset, + elem->size, "debug", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + case SOF_IPC_REGION_STREAM: + stream_offset = window_offset + elem->offset; + stream_size = elem->size; + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + stream_offset, + elem->size, "stream", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + case SOF_IPC_REGION_REGS: + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + window_offset + elem->offset, + elem->size, "regs", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + case SOF_IPC_REGION_EXCEPTION: + sdev->dsp_oops_offset = window_offset + elem->offset; + snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, + window_offset + elem->offset, + elem->size, "exception", + SOF_DEBUGFS_ACCESS_D0_ONLY); + break; + default: + dev_err(sdev->dev, "%s: Illegal window info: %u\n", + __func__, elem->type); + return; + } + } + + if (outbox_size == 0 || inbox_size == 0) { + dev_err(sdev->dev, "%s: Illegal mailbox window\n", __func__); + return; + } + + sdev->dsp_box.offset = inbox_offset; + sdev->dsp_box.size = inbox_size; + + sdev->host_box.offset = outbox_offset; + sdev->host_box.size = outbox_size; + + sdev->stream_box.offset = stream_offset; + sdev->stream_box.size = stream_size; + + sdev->debug_box.offset = debug_offset; + sdev->debug_box.size = debug_size; + + dev_dbg(sdev->dev, " mailbox upstream 0x%x - size 0x%x\n", + inbox_offset, inbox_size); + dev_dbg(sdev->dev, " mailbox downstream 0x%x - size 0x%x\n", + outbox_offset, outbox_size); + dev_dbg(sdev->dev, " stream region 0x%x - size 0x%x\n", + stream_offset, stream_size); + dev_dbg(sdev->dev, " debug region 0x%x - size 0x%x\n", + debug_offset, debug_size); +} + +static int ipc3_init_reply_data_buffer(struct snd_sof_dev *sdev) +{ + struct snd_sof_ipc_msg *msg = &sdev->ipc->msg; + + msg->reply_data = devm_kzalloc(sdev->dev, SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); + if (!msg->reply_data) + return -ENOMEM; + + sdev->ipc->max_payload_size = SOF_IPC_MSG_MAX_SIZE; + + return 0; +} + +int sof_ipc3_validate_fw_version(struct snd_sof_dev *sdev) +{ + struct sof_ipc_fw_ready *ready = &sdev->fw_ready; + struct sof_ipc_fw_version *v = &ready->version; + + dev_info(sdev->dev, + "Firmware info: version %d:%d:%d-%s\n", v->major, v->minor, + v->micro, v->tag); + dev_info(sdev->dev, + "Firmware: ABI %d:%d:%d Kernel ABI %d:%d:%d\n", + SOF_ABI_VERSION_MAJOR(v->abi_version), + SOF_ABI_VERSION_MINOR(v->abi_version), + SOF_ABI_VERSION_PATCH(v->abi_version), + SOF_ABI_MAJOR, SOF_ABI_MINOR, SOF_ABI_PATCH); + + if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, v->abi_version)) { + dev_err(sdev->dev, "incompatible FW ABI version\n"); + return -EINVAL; + } + + if (IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS) && + SOF_ABI_VERSION_MINOR(v->abi_version) > SOF_ABI_MINOR) { + dev_err(sdev->dev, "FW ABI is more recent than kernel\n"); + return -EINVAL; + } + + if (ready->flags & SOF_IPC_INFO_BUILD) { + dev_info(sdev->dev, + "Firmware debug build %d on %s-%s - options:\n" + " GDB: %s\n" + " lock debug: %s\n" + " lock vdebug: %s\n", + v->build, v->date, v->time, + (ready->flags & SOF_IPC_INFO_GDB) ? + "enabled" : "disabled", + (ready->flags & SOF_IPC_INFO_LOCKS) ? + "enabled" : "disabled", + (ready->flags & SOF_IPC_INFO_LOCKSV) ? + "enabled" : "disabled"); + } + + /* copy the fw_version into debugfs at first boot */ + memcpy(&sdev->fw_version, v, sizeof(*v)); + + return 0; +} + +static int ipc3_fw_ready(struct snd_sof_dev *sdev, u32 cmd) +{ + struct sof_ipc_fw_ready *fw_ready = &sdev->fw_ready; + int offset; + int ret; + + /* mailbox must be on 4k boundary */ + offset = snd_sof_dsp_get_mailbox_offset(sdev); + if (offset < 0) { + dev_err(sdev->dev, "%s: no mailbox offset\n", __func__); + return offset; + } + + dev_dbg(sdev->dev, "DSP is ready 0x%8.8x offset 0x%x\n", cmd, offset); + + /* no need to re-check version/ABI for subsequent boots */ + if (!sdev->first_boot) + return 0; + + /* + * copy data from the DSP FW ready offset + * Subsequent error handling is not needed for BLK_TYPE_SRAM + */ + ret = snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, offset, fw_ready, + sizeof(*fw_ready)); + if (ret) { + dev_err(sdev->dev, + "Unable to read fw_ready, read from TYPE_SRAM failed\n"); + return ret; + } + + /* make sure ABI version is compatible */ + ret = sof_ipc3_validate_fw_version(sdev); + if (ret < 0) + return ret; + + /* now check for extended data */ + ipc3_fw_parse_ext_data(sdev, offset + sizeof(struct sof_ipc_fw_ready)); + + ipc3_get_windows(sdev); + + return ipc3_init_reply_data_buffer(sdev); +} + +/* IPC stream position. */ +static void ipc3_period_elapsed(struct snd_sof_dev *sdev, u32 msg_id) +{ + struct snd_soc_component *scomp = sdev->component; + struct snd_sof_pcm_stream *stream; + struct sof_ipc_stream_posn posn; + struct snd_sof_pcm *spcm; + int direction, ret; + + spcm = snd_sof_find_spcm_comp(scomp, msg_id, &direction); + if (!spcm) { + dev_err(sdev->dev, "period elapsed for unknown stream, msg_id %d\n", + msg_id); + return; + } + + stream = &spcm->stream[direction]; + ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn)); + if (ret < 0) { + dev_warn(sdev->dev, "failed to read stream position: %d\n", ret); + return; + } + + dev_vdbg(sdev->dev, "posn : host 0x%llx dai 0x%llx wall 0x%llx\n", + posn.host_posn, posn.dai_posn, posn.wallclock); + + memcpy(&stream->posn, &posn, sizeof(posn)); + + if (spcm->pcm.compress) + snd_sof_compr_fragment_elapsed(stream->cstream); + else if (stream->substream->runtime && + !stream->substream->runtime->no_period_wakeup) + /* only inform ALSA for period_wakeup mode */ + snd_sof_pcm_period_elapsed(stream->substream); +} + +/* DSP notifies host of an XRUN within FW */ +static void ipc3_xrun(struct snd_sof_dev *sdev, u32 msg_id) +{ + struct snd_soc_component *scomp = sdev->component; + struct snd_sof_pcm_stream *stream; + struct sof_ipc_stream_posn posn; + struct snd_sof_pcm *spcm; + int direction, ret; + + spcm = snd_sof_find_spcm_comp(scomp, msg_id, &direction); + if (!spcm) { + dev_err(sdev->dev, "XRUN for unknown stream, msg_id %d\n", + msg_id); + return; + } + + stream = &spcm->stream[direction]; + ret = snd_sof_ipc_msg_data(sdev, stream->substream, &posn, sizeof(posn)); + if (ret < 0) { + dev_warn(sdev->dev, "failed to read overrun position: %d\n", ret); + return; + } + + dev_dbg(sdev->dev, "posn XRUN: host %llx comp %d size %d\n", + posn.host_posn, posn.xrun_comp_id, posn.xrun_size); + +#if defined(CONFIG_SND_SOC_SOF_DEBUG_XRUN_STOP) + /* stop PCM on XRUN - used for pipeline debug */ + memcpy(&stream->posn, &posn, sizeof(posn)); + snd_pcm_stop_xrun(stream->substream); +#endif +} + +/* stream notifications from firmware */ +static void ipc3_stream_message(struct snd_sof_dev *sdev, void *msg_buf) +{ + struct sof_ipc_cmd_hdr *hdr = msg_buf; + u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK; + u32 msg_id = SOF_IPC_MESSAGE_ID(hdr->cmd); + + switch (msg_type) { + case SOF_IPC_STREAM_POSITION: + ipc3_period_elapsed(sdev, msg_id); + break; + case SOF_IPC_STREAM_TRIG_XRUN: + ipc3_xrun(sdev, msg_id); + break; + default: + dev_err(sdev->dev, "unhandled stream message %#x\n", + msg_id); + break; + } +} + +/* component notifications from firmware */ +static void ipc3_comp_notification(struct snd_sof_dev *sdev, void *msg_buf) +{ + const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; + struct sof_ipc_cmd_hdr *hdr = msg_buf; + u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK; + + switch (msg_type) { + case SOF_IPC_COMP_GET_VALUE: + case SOF_IPC_COMP_GET_DATA: + break; + default: + dev_err(sdev->dev, "unhandled component message %#x\n", msg_type); + return; + } + + if (tplg_ops->control->update) + tplg_ops->control->update(sdev, msg_buf); +} + +static void ipc3_trace_message(struct snd_sof_dev *sdev, void *msg_buf) +{ + struct sof_ipc_cmd_hdr *hdr = msg_buf; + u32 msg_type = hdr->cmd & SOF_CMD_TYPE_MASK; + + switch (msg_type) { + case SOF_IPC_TRACE_DMA_POSITION: + ipc3_dtrace_posn_update(sdev, msg_buf); + break; + default: + dev_err(sdev->dev, "unhandled trace message %#x\n", msg_type); + break; + } +} + +/* DSP firmware has sent host a message */ +static void sof_ipc3_rx_msg(struct snd_sof_dev *sdev) +{ + ipc3_rx_callback rx_callback = NULL; + struct sof_ipc_cmd_hdr hdr; + void *msg_buf; + u32 cmd; + int err; + + /* read back header */ + err = snd_sof_ipc_msg_data(sdev, NULL, &hdr, sizeof(hdr)); + if (err < 0) { + dev_warn(sdev->dev, "failed to read IPC header: %d\n", err); + return; + } + + if (hdr.size < sizeof(hdr)) { + dev_err(sdev->dev, "The received message size is invalid\n"); + return; + } + + ipc3_log_header(sdev->dev, "ipc rx", hdr.cmd); + + cmd = hdr.cmd & SOF_GLB_TYPE_MASK; + + /* check message type */ + switch (cmd) { + case SOF_IPC_GLB_REPLY: + dev_err(sdev->dev, "ipc reply unknown\n"); + break; + case SOF_IPC_FW_READY: + /* check for FW boot completion */ + if (sdev->fw_state == SOF_FW_BOOT_IN_PROGRESS) { + err = ipc3_fw_ready(sdev, cmd); + if (err < 0) + sof_set_fw_state(sdev, SOF_FW_BOOT_READY_FAILED); + else + sof_set_fw_state(sdev, SOF_FW_BOOT_READY_OK); + + /* wake up firmware loader */ + wake_up(&sdev->boot_wait); + } + break; + case SOF_IPC_GLB_COMPOUND: + case SOF_IPC_GLB_TPLG_MSG: + case SOF_IPC_GLB_PM_MSG: + break; + case SOF_IPC_GLB_COMP_MSG: + rx_callback = ipc3_comp_notification; + break; + case SOF_IPC_GLB_STREAM_MSG: + rx_callback = ipc3_stream_message; + break; + case SOF_IPC_GLB_TRACE_MSG: + rx_callback = ipc3_trace_message; + break; + default: + dev_err(sdev->dev, "%s: Unknown DSP message: 0x%x\n", __func__, cmd); + break; + } + + /* read the full message */ + msg_buf = kmalloc(hdr.size, GFP_KERNEL); + if (!msg_buf) + return; + + err = snd_sof_ipc_msg_data(sdev, NULL, msg_buf, hdr.size); + if (err < 0) { + dev_err(sdev->dev, "%s: Failed to read message: %d\n", __func__, err); + } else { + /* Call local handler for the message */ + if (rx_callback) + rx_callback(sdev, msg_buf); + + /* Notify registered clients */ + sof_client_ipc_rx_dispatcher(sdev, msg_buf); + } + + kfree(msg_buf); + + ipc3_log_header(sdev->dev, "ipc rx done", hdr.cmd); +} + +static int sof_ipc3_set_core_state(struct snd_sof_dev *sdev, int core_idx, bool on) +{ + struct sof_ipc_pm_core_config core_cfg = { + .hdr.size = sizeof(core_cfg), + .hdr.cmd = SOF_IPC_GLB_PM_MSG | SOF_IPC_PM_CORE_ENABLE, + }; + struct sof_ipc_reply reply; + + if (on) + core_cfg.enable_mask = sdev->enabled_cores_mask | BIT(core_idx); + else + core_cfg.enable_mask = sdev->enabled_cores_mask & ~BIT(core_idx); + + return sof_ipc3_tx_msg(sdev, &core_cfg, sizeof(core_cfg), + &reply, sizeof(reply), false); +} static int sof_ipc3_ctx_ipc(struct snd_sof_dev *sdev, int cmd) { @@ -19,8 +1063,8 @@ static int sof_ipc3_ctx_ipc(struct snd_sof_dev *sdev, int cmd) struct sof_ipc_reply reply; /* send ctx save ipc to dsp */ - return sof_ipc_tx_message(sdev->ipc, pm_ctx.hdr.cmd, &pm_ctx, - sizeof(pm_ctx), &reply, sizeof(reply)); + return sof_ipc3_tx_msg(sdev, &pm_ctx, sizeof(pm_ctx), + &reply, sizeof(reply), false); } static int sof_ipc3_ctx_save(struct snd_sof_dev *sdev) @@ -36,10 +1080,18 @@ static int sof_ipc3_ctx_restore(struct snd_sof_dev *sdev) static const struct sof_ipc_pm_ops ipc3_pm_ops = { .ctx_save = sof_ipc3_ctx_save, .ctx_restore = sof_ipc3_ctx_restore, + .set_core_state = sof_ipc3_set_core_state, }; const struct sof_ipc_ops ipc3_ops = { .tplg = &ipc3_tplg_ops, .pm = &ipc3_pm_ops, .pcm = &ipc3_pcm_ops, + .fw_loader = &ipc3_loader_ops, + .fw_tracing = &ipc3_dtrace_ops, + + .tx_msg = sof_ipc3_tx_msg, + .rx_msg = sof_ipc3_rx_msg, + .set_get_data = sof_ipc3_set_get_data, + .get_reply = sof_ipc3_get_reply, }; diff --git a/sound/soc/sof/loader.c b/sound/soc/sof/loader.c index 697f03565a..5f51d936b3 100644 --- a/sound/soc/sof/loader.c +++ b/sound/soc/sof/loader.c @@ -11,690 +11,9 @@ // #include -#include -#include #include "sof-priv.h" #include "ops.h" -static int get_ext_windows(struct snd_sof_dev *sdev, - const struct sof_ipc_ext_data_hdr *ext_hdr) -{ - const struct sof_ipc_window *w = - container_of(ext_hdr, struct sof_ipc_window, ext_hdr); - - if (w->num_windows == 0 || w->num_windows > SOF_IPC_MAX_ELEMS) - return -EINVAL; - - if (sdev->info_window) { - if (memcmp(sdev->info_window, w, ext_hdr->hdr.size)) { - dev_err(sdev->dev, "error: mismatch between window descriptor from extended manifest and mailbox"); - return -EINVAL; - } - return 0; - } - - /* keep a local copy of the data */ - sdev->info_window = devm_kmemdup(sdev->dev, w, ext_hdr->hdr.size, - GFP_KERNEL); - if (!sdev->info_window) - return -ENOMEM; - - return 0; -} - -static int get_cc_info(struct snd_sof_dev *sdev, - const struct sof_ipc_ext_data_hdr *ext_hdr) -{ - int ret; - - const struct sof_ipc_cc_version *cc = - container_of(ext_hdr, struct sof_ipc_cc_version, ext_hdr); - - if (sdev->cc_version) { - if (memcmp(sdev->cc_version, cc, cc->ext_hdr.hdr.size)) { - dev_err(sdev->dev, "error: receive diverged cc_version descriptions"); - return -EINVAL; - } - return 0; - } - - dev_dbg(sdev->dev, "Firmware info: used compiler %s %d:%d:%d%s used optimization flags %s\n", - cc->name, cc->major, cc->minor, cc->micro, cc->desc, - cc->optim); - - /* create read-only cc_version debugfs to store compiler version info */ - /* use local copy of the cc_version to prevent data corruption */ - if (sdev->first_boot) { - sdev->cc_version = devm_kmalloc(sdev->dev, cc->ext_hdr.hdr.size, - GFP_KERNEL); - - if (!sdev->cc_version) - return -ENOMEM; - - memcpy(sdev->cc_version, cc, cc->ext_hdr.hdr.size); - ret = snd_sof_debugfs_buf_item(sdev, sdev->cc_version, - cc->ext_hdr.hdr.size, - "cc_version", 0444); - - /* errors are only due to memory allocation, not debugfs */ - if (ret < 0) { - dev_err(sdev->dev, "error: snd_sof_debugfs_buf_item failed\n"); - return ret; - } - } - - return 0; -} - -/* parse the extended FW boot data structures from FW boot message */ -static int snd_sof_fw_parse_ext_data(struct snd_sof_dev *sdev, u32 offset) -{ - struct sof_ipc_ext_data_hdr *ext_hdr; - void *ext_data; - int ret = 0; - - ext_data = kzalloc(PAGE_SIZE, GFP_KERNEL); - if (!ext_data) - return -ENOMEM; - - /* get first header */ - snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, offset, ext_data, - sizeof(*ext_hdr)); - ext_hdr = ext_data; - - while (ext_hdr->hdr.cmd == SOF_IPC_FW_READY) { - /* read in ext structure */ - snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, - offset + sizeof(*ext_hdr), - (void *)((u8 *)ext_data + sizeof(*ext_hdr)), - ext_hdr->hdr.size - sizeof(*ext_hdr)); - - dev_dbg(sdev->dev, "found ext header type %d size 0x%x\n", - ext_hdr->type, ext_hdr->hdr.size); - - /* process structure data */ - switch (ext_hdr->type) { - case SOF_IPC_EXT_WINDOW: - ret = get_ext_windows(sdev, ext_hdr); - break; - case SOF_IPC_EXT_CC_INFO: - ret = get_cc_info(sdev, ext_hdr); - break; - case SOF_IPC_EXT_UNUSED: - case SOF_IPC_EXT_PROBE_INFO: - case SOF_IPC_EXT_USER_ABI_INFO: - /* They are supported but we don't do anything here */ - break; - default: - dev_info(sdev->dev, "unknown ext header type %d size 0x%x\n", - ext_hdr->type, ext_hdr->hdr.size); - ret = 0; - break; - } - - if (ret < 0) { - dev_err(sdev->dev, "error: failed to parse ext data type %d\n", - ext_hdr->type); - break; - } - - /* move to next header */ - offset += ext_hdr->hdr.size; - snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, offset, ext_data, - sizeof(*ext_hdr)); - ext_hdr = ext_data; - } - - kfree(ext_data); - return ret; -} - -static int ext_man_get_fw_version(struct snd_sof_dev *sdev, - const struct sof_ext_man_elem_header *hdr) -{ - const struct sof_ext_man_fw_version *v = - container_of(hdr, struct sof_ext_man_fw_version, hdr); - - memcpy(&sdev->fw_ready.version, &v->version, sizeof(v->version)); - sdev->fw_ready.flags = v->flags; - - /* log ABI versions and check FW compatibility */ - return snd_sof_ipc_valid(sdev); -} - -static int ext_man_get_windows(struct snd_sof_dev *sdev, - const struct sof_ext_man_elem_header *hdr) -{ - const struct sof_ext_man_window *w; - - w = container_of(hdr, struct sof_ext_man_window, hdr); - - return get_ext_windows(sdev, &w->ipc_window.ext_hdr); -} - -static int ext_man_get_cc_info(struct snd_sof_dev *sdev, - const struct sof_ext_man_elem_header *hdr) -{ - const struct sof_ext_man_cc_version *cc; - - cc = container_of(hdr, struct sof_ext_man_cc_version, hdr); - - return get_cc_info(sdev, &cc->cc_version.ext_hdr); -} - -static int ext_man_get_dbg_abi_info(struct snd_sof_dev *sdev, - const struct sof_ext_man_elem_header *hdr) -{ - const struct ext_man_dbg_abi *dbg_abi = - container_of(hdr, struct ext_man_dbg_abi, hdr); - - if (sdev->first_boot) - dev_dbg(sdev->dev, - "Firmware: DBG_ABI %d:%d:%d\n", - SOF_ABI_VERSION_MAJOR(dbg_abi->dbg_abi.abi_dbg_version), - SOF_ABI_VERSION_MINOR(dbg_abi->dbg_abi.abi_dbg_version), - SOF_ABI_VERSION_PATCH(dbg_abi->dbg_abi.abi_dbg_version)); - - return 0; -} - -static int ext_man_get_config_data(struct snd_sof_dev *sdev, - const struct sof_ext_man_elem_header *hdr) -{ - const struct sof_ext_man_config_data *config = - container_of(hdr, struct sof_ext_man_config_data, hdr); - const struct sof_config_elem *elem; - int elems_counter; - int elems_size; - int ret = 0; - int i; - - /* calculate elements counter */ - elems_size = config->hdr.size - sizeof(struct sof_ext_man_elem_header); - elems_counter = elems_size / sizeof(struct sof_config_elem); - - dev_dbg(sdev->dev, "%s can hold up to %d config elements\n", - __func__, elems_counter); - - for (i = 0; i < elems_counter; ++i) { - elem = &config->elems[i]; - dev_dbg(sdev->dev, "%s get index %d token %d val %d\n", - __func__, i, elem->token, elem->value); - switch (elem->token) { - case SOF_EXT_MAN_CONFIG_EMPTY: - /* unused memory space is zero filled - mapped to EMPTY elements */ - break; - case SOF_EXT_MAN_CONFIG_IPC_MSG_SIZE: - /* TODO: use ipc msg size from config data */ - break; - case SOF_EXT_MAN_CONFIG_MEMORY_USAGE_SCAN: - if (sdev->first_boot && elem->value) - ret = snd_sof_dbg_memory_info_init(sdev); - break; - default: - dev_info(sdev->dev, "Unknown firmware configuration token %d value %d", - elem->token, elem->value); - break; - } - if (ret < 0) { - dev_err(sdev->dev, "error: processing sof_ext_man_config_data failed for token %d value 0x%x, %d\n", - elem->token, elem->value, ret); - return ret; - } - } - - return 0; -} - -static ssize_t snd_sof_ext_man_size(const struct firmware *fw) -{ - const struct sof_ext_man_header *head; - - head = (struct sof_ext_man_header *)fw->data; - - /* - * assert fw size is big enough to contain extended manifest header, - * it prevents from reading unallocated memory from `head` in following - * step. - */ - if (fw->size < sizeof(*head)) - return -EINVAL; - - /* - * When fw points to extended manifest, - * then first u32 must be equal SOF_EXT_MAN_MAGIC_NUMBER. - */ - if (head->magic == SOF_EXT_MAN_MAGIC_NUMBER) - return head->full_size; - - /* otherwise given fw don't have an extended manifest */ - return 0; -} - -/* parse extended FW manifest data structures */ -static int snd_sof_fw_ext_man_parse(struct snd_sof_dev *sdev, - const struct firmware *fw) -{ - const struct sof_ext_man_elem_header *elem_hdr; - const struct sof_ext_man_header *head; - ssize_t ext_man_size; - ssize_t remaining; - uintptr_t iptr; - int ret = 0; - - head = (struct sof_ext_man_header *)fw->data; - remaining = head->full_size - head->header_size; - ext_man_size = snd_sof_ext_man_size(fw); - - /* Assert firmware starts with extended manifest */ - if (ext_man_size <= 0) - return ext_man_size; - - /* incompatible version */ - if (SOF_EXT_MAN_VERSION_INCOMPATIBLE(SOF_EXT_MAN_VERSION, - head->header_version)) { - dev_err(sdev->dev, "error: extended manifest version 0x%X differ from used 0x%X\n", - head->header_version, SOF_EXT_MAN_VERSION); - return -EINVAL; - } - - /* get first extended manifest element header */ - iptr = (uintptr_t)fw->data + head->header_size; - - while (remaining > sizeof(*elem_hdr)) { - elem_hdr = (struct sof_ext_man_elem_header *)iptr; - - dev_dbg(sdev->dev, "found sof_ext_man header type %d size 0x%X\n", - elem_hdr->type, elem_hdr->size); - - if (elem_hdr->size < sizeof(*elem_hdr) || - elem_hdr->size > remaining) { - dev_err(sdev->dev, "error: invalid sof_ext_man header size, type %d size 0x%X\n", - elem_hdr->type, elem_hdr->size); - return -EINVAL; - } - - /* process structure data */ - switch (elem_hdr->type) { - case SOF_EXT_MAN_ELEM_FW_VERSION: - ret = ext_man_get_fw_version(sdev, elem_hdr); - break; - case SOF_EXT_MAN_ELEM_WINDOW: - ret = ext_man_get_windows(sdev, elem_hdr); - break; - case SOF_EXT_MAN_ELEM_CC_VERSION: - ret = ext_man_get_cc_info(sdev, elem_hdr); - break; - case SOF_EXT_MAN_ELEM_DBG_ABI: - ret = ext_man_get_dbg_abi_info(sdev, elem_hdr); - break; - case SOF_EXT_MAN_ELEM_CONFIG_DATA: - ret = ext_man_get_config_data(sdev, elem_hdr); - break; - case SOF_EXT_MAN_ELEM_PLATFORM_CONFIG_DATA: - ret = snd_sof_dsp_parse_platform_ext_manifest(sdev, elem_hdr); - break; - default: - dev_info(sdev->dev, "unknown sof_ext_man header type %d size 0x%X\n", - elem_hdr->type, elem_hdr->size); - break; - } - - if (ret < 0) { - dev_err(sdev->dev, "error: failed to parse sof_ext_man header type %d size 0x%X\n", - elem_hdr->type, elem_hdr->size); - return ret; - } - - remaining -= elem_hdr->size; - iptr += elem_hdr->size; - } - - if (remaining) { - dev_err(sdev->dev, "error: sof_ext_man header is inconsistent\n"); - return -EINVAL; - } - - return ext_man_size; -} - -/* - * IPC Firmware ready. - */ -static void sof_get_windows(struct snd_sof_dev *sdev) -{ - struct sof_ipc_window_elem *elem; - u32 outbox_offset = 0; - u32 stream_offset = 0; - u32 inbox_offset = 0; - u32 outbox_size = 0; - u32 stream_size = 0; - u32 inbox_size = 0; - u32 debug_size = 0; - u32 debug_offset = 0; - int window_offset; - int i; - - if (!sdev->info_window) { - dev_err(sdev->dev, "error: have no window info\n"); - return; - } - - for (i = 0; i < sdev->info_window->num_windows; i++) { - elem = &sdev->info_window->window[i]; - - window_offset = snd_sof_dsp_get_window_offset(sdev, elem->id); - if (window_offset < 0) { - dev_warn(sdev->dev, "warn: no offset for window %d\n", - elem->id); - continue; - } - - switch (elem->type) { - case SOF_IPC_REGION_UPBOX: - inbox_offset = window_offset + elem->offset; - inbox_size = elem->size; - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - inbox_offset, - elem->size, "inbox", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - case SOF_IPC_REGION_DOWNBOX: - outbox_offset = window_offset + elem->offset; - outbox_size = elem->size; - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - outbox_offset, - elem->size, "outbox", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - case SOF_IPC_REGION_TRACE: - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - window_offset + elem->offset, - elem->size, "etrace", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - case SOF_IPC_REGION_DEBUG: - debug_offset = window_offset + elem->offset; - debug_size = elem->size; - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - window_offset + elem->offset, - elem->size, "debug", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - case SOF_IPC_REGION_STREAM: - stream_offset = window_offset + elem->offset; - stream_size = elem->size; - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - stream_offset, - elem->size, "stream", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - case SOF_IPC_REGION_REGS: - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - window_offset + elem->offset, - elem->size, "regs", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - case SOF_IPC_REGION_EXCEPTION: - sdev->dsp_oops_offset = window_offset + elem->offset; - snd_sof_debugfs_add_region_item(sdev, SOF_FW_BLK_TYPE_SRAM, - window_offset + elem->offset, - elem->size, "exception", - SOF_DEBUGFS_ACCESS_D0_ONLY); - break; - default: - dev_err(sdev->dev, "error: get illegal window info\n"); - return; - } - } - - if (outbox_size == 0 || inbox_size == 0) { - dev_err(sdev->dev, "error: get illegal mailbox window\n"); - return; - } - - sdev->dsp_box.offset = inbox_offset; - sdev->dsp_box.size = inbox_size; - - sdev->host_box.offset = outbox_offset; - sdev->host_box.size = outbox_size; - - sdev->stream_box.offset = stream_offset; - sdev->stream_box.size = stream_size; - - sdev->debug_box.offset = debug_offset; - sdev->debug_box.size = debug_size; - - dev_dbg(sdev->dev, " mailbox upstream 0x%x - size 0x%x\n", - inbox_offset, inbox_size); - dev_dbg(sdev->dev, " mailbox downstream 0x%x - size 0x%x\n", - outbox_offset, outbox_size); - dev_dbg(sdev->dev, " stream region 0x%x - size 0x%x\n", - stream_offset, stream_size); - dev_dbg(sdev->dev, " debug region 0x%x - size 0x%x\n", - debug_offset, debug_size); -} - -/* check for ABI compatibility and create memory windows on first boot */ -int sof_fw_ready(struct snd_sof_dev *sdev, u32 msg_id) -{ - struct sof_ipc_fw_ready *fw_ready = &sdev->fw_ready; - int offset; - int ret; - - /* mailbox must be on 4k boundary */ - offset = snd_sof_dsp_get_mailbox_offset(sdev); - if (offset < 0) { - dev_err(sdev->dev, "error: have no mailbox offset\n"); - return offset; - } - - dev_dbg(sdev->dev, "ipc: DSP is ready 0x%8.8x offset 0x%x\n", - msg_id, offset); - - /* no need to re-check version/ABI for subsequent boots */ - if (!sdev->first_boot) - return 0; - - /* - * copy data from the DSP FW ready offset - * Subsequent error handling is not needed for BLK_TYPE_SRAM - */ - ret = snd_sof_dsp_block_read(sdev, SOF_FW_BLK_TYPE_SRAM, offset, fw_ready, - sizeof(*fw_ready)); - if (ret) { - dev_err(sdev->dev, - "error: unable to read fw_ready, read from TYPE_SRAM failed\n"); - return ret; - } - - /* make sure ABI version is compatible */ - ret = snd_sof_ipc_valid(sdev); - if (ret < 0) - return ret; - - /* now check for extended data */ - snd_sof_fw_parse_ext_data(sdev, offset + sizeof(struct sof_ipc_fw_ready)); - - sof_get_windows(sdev); - - return sof_ipc_init_msg_memory(sdev); -} -EXPORT_SYMBOL(sof_fw_ready); - -/* generic module parser for mmaped DSPs */ -int snd_sof_parse_module_memcpy(struct snd_sof_dev *sdev, - struct snd_sof_mod_hdr *module) -{ - struct snd_sof_blk_hdr *block; - int count, ret; - u32 offset; - size_t remaining; - - dev_dbg(sdev->dev, "new module size 0x%x blocks 0x%x type 0x%x\n", - module->size, module->num_blocks, module->type); - - block = (struct snd_sof_blk_hdr *)((u8 *)module + sizeof(*module)); - - /* module->size doesn't include header size */ - remaining = module->size; - for (count = 0; count < module->num_blocks; count++) { - /* check for wrap */ - if (remaining < sizeof(*block)) { - dev_err(sdev->dev, "error: not enough data remaining\n"); - return -EINVAL; - } - - /* minus header size of block */ - remaining -= sizeof(*block); - - if (block->size == 0) { - dev_warn(sdev->dev, - "warning: block %d size zero\n", count); - dev_warn(sdev->dev, " type 0x%x offset 0x%x\n", - block->type, block->offset); - continue; - } - - switch (block->type) { - case SOF_FW_BLK_TYPE_RSRVD0: - case SOF_FW_BLK_TYPE_ROM...SOF_FW_BLK_TYPE_RSRVD14: - continue; /* not handled atm */ - case SOF_FW_BLK_TYPE_IRAM: - case SOF_FW_BLK_TYPE_DRAM: - case SOF_FW_BLK_TYPE_SRAM: - offset = block->offset; - break; - default: - dev_err(sdev->dev, "error: bad type 0x%x for block 0x%x\n", - block->type, count); - return -EINVAL; - } - - dev_dbg(sdev->dev, - "block %d type 0x%x size 0x%x ==> offset 0x%x\n", - count, block->type, block->size, offset); - - /* checking block->size to avoid unaligned access */ - if (block->size % sizeof(u32)) { - dev_err(sdev->dev, "error: invalid block size 0x%x\n", - block->size); - return -EINVAL; - } - ret = snd_sof_dsp_block_write(sdev, block->type, offset, - block + 1, block->size); - if (ret < 0) { - dev_err(sdev->dev, "error: write to block type 0x%x failed\n", - block->type); - return ret; - } - - if (remaining < block->size) { - dev_err(sdev->dev, "error: not enough data remaining\n"); - return -EINVAL; - } - - /* minus body size of block */ - remaining -= block->size; - /* next block */ - block = (struct snd_sof_blk_hdr *)((u8 *)block + sizeof(*block) - + block->size); - } - - return 0; -} -EXPORT_SYMBOL(snd_sof_parse_module_memcpy); - -static int check_header(struct snd_sof_dev *sdev, const struct firmware *fw, - size_t fw_offset) -{ - struct snd_sof_fw_header *header; - size_t fw_size = fw->size - fw_offset; - - if (fw->size <= fw_offset) { - dev_err(sdev->dev, "error: firmware size must be greater than firmware offset\n"); - return -EINVAL; - } - - /* Read the header information from the data pointer */ - header = (struct snd_sof_fw_header *)(fw->data + fw_offset); - - /* verify FW sig */ - if (strncmp(header->sig, SND_SOF_FW_SIG, SND_SOF_FW_SIG_SIZE) != 0) { - dev_err(sdev->dev, "error: invalid firmware signature\n"); - return -EINVAL; - } - - /* check size is valid */ - if (fw_size != header->file_size + sizeof(*header)) { - dev_err(sdev->dev, "error: invalid filesize mismatch got 0x%zx expected 0x%zx\n", - fw_size, header->file_size + sizeof(*header)); - return -EINVAL; - } - - dev_dbg(sdev->dev, "header size=0x%x modules=0x%x abi=0x%x size=%zu\n", - header->file_size, header->num_modules, - header->abi, sizeof(*header)); - - return 0; -} - -static int load_modules(struct snd_sof_dev *sdev, const struct firmware *fw, - size_t fw_offset) -{ - struct snd_sof_fw_header *header; - struct snd_sof_mod_hdr *module; - int (*load_module)(struct snd_sof_dev *sof_dev, - struct snd_sof_mod_hdr *hdr); - int ret, count; - size_t remaining; - - header = (struct snd_sof_fw_header *)(fw->data + fw_offset); - load_module = sof_ops(sdev)->load_module; - if (!load_module) - return -EINVAL; - - /* parse each module */ - module = (struct snd_sof_mod_hdr *)(fw->data + fw_offset + - sizeof(*header)); - remaining = fw->size - sizeof(*header) - fw_offset; - /* check for wrap */ - if (remaining > fw->size) { - dev_err(sdev->dev, "error: fw size smaller than header size\n"); - return -EINVAL; - } - - for (count = 0; count < header->num_modules; count++) { - /* check for wrap */ - if (remaining < sizeof(*module)) { - dev_err(sdev->dev, "error: not enough data remaining\n"); - return -EINVAL; - } - - /* minus header size of module */ - remaining -= sizeof(*module); - - /* module */ - ret = load_module(sdev, module); - if (ret < 0) { - dev_err(sdev->dev, "error: invalid module %d\n", count); - return ret; - } - - if (remaining < module->size) { - dev_err(sdev->dev, "error: not enough data remaining\n"); - return -EINVAL; - } - - /* minus body size of module */ - remaining -= module->size; - module = (struct snd_sof_mod_hdr *)((u8 *)module - + sizeof(*module) + module->size); - } - - return 0; -} - int snd_sof_load_firmware_raw(struct snd_sof_dev *sdev) { struct snd_sof_pdata *plat_data = sdev->pdata; @@ -726,7 +45,7 @@ int snd_sof_load_firmware_raw(struct snd_sof_dev *sdev) } /* check for extended manifest */ - ext_man_size = snd_sof_fw_ext_man_parse(sdev, plat_data->fw); + ext_man_size = sdev->ipc->ops->fw_loader->parse_ext_manifest(sdev); if (ext_man_size > 0) { /* when no error occurred, drop extended manifest */ plat_data->fw_offset = ext_man_size; @@ -756,7 +75,7 @@ int snd_sof_load_firmware_memcpy(struct snd_sof_dev *sdev) return ret; /* make sure the FW header and file is valid */ - ret = check_header(sdev, plat_data->fw, plat_data->fw_offset); + ret = sdev->ipc->ops->fw_loader->validate(sdev); if (ret < 0) { dev_err(sdev->dev, "error: invalid FW header\n"); goto error; @@ -770,10 +89,12 @@ int snd_sof_load_firmware_memcpy(struct snd_sof_dev *sdev) } /* parse and load firmware modules to DSP */ - ret = load_modules(sdev, plat_data->fw, plat_data->fw_offset); - if (ret < 0) { - dev_err(sdev->dev, "error: invalid FW modules\n"); - goto error; + if (sdev->ipc->ops->fw_loader->load_fw_to_dsp) { + ret = sdev->ipc->ops->fw_loader->load_fw_to_dsp(sdev); + if (ret < 0) { + dev_err(sdev->dev, "Firmware loading failed\n"); + goto error; + } } return 0; @@ -854,6 +175,9 @@ int snd_sof_run_firmware(struct snd_sof_dev *sdev) dev_dbg(sdev->dev, "firmware boot complete\n"); sof_set_fw_state(sdev, SOF_FW_BOOT_COMPLETE); + if (sdev->first_boot && sdev->ipc->ops->fw_loader->query_fw_configuration) + return sdev->ipc->ops->fw_loader->query_fw_configuration(sdev); + return 0; } EXPORT_SYMBOL(snd_sof_run_firmware); diff --git a/sound/soc/sof/mediatek/Kconfig b/sound/soc/sof/mediatek/Kconfig index aeacf0e5bf..4a2eddf600 100644 --- a/sound/soc/sof/mediatek/Kconfig +++ b/sound/soc/sof/mediatek/Kconfig @@ -15,15 +15,27 @@ config SND_SOC_SOF_MTK_COMMON tristate select SND_SOC_SOF_OF_DEV select SND_SOC_SOF + select SND_SOC_SOF_IPC3 select SND_SOC_SOF_XTENSA select SND_SOC_SOF_COMPRESS help This option is not user-selectable but automagically handled by 'select' statements at a higher level +config SND_SOC_SOF_MT8186 + tristate "SOF support for MT8186 audio DSP" + select SND_SOC_SOF_MTK_COMMON + depends on MTK_ADSP_IPC + help + This adds support for Sound Open Firmware for Mediatek platforms + using the mt8186 processors. + Say Y if you have such a device. + If unsure select "N". + config SND_SOC_SOF_MT8195 tristate "SOF support for MT8195 audio DSP" select SND_SOC_SOF_MTK_COMMON + depends on MTK_ADSP_IPC help This adds support for Sound Open Firmware for Mediatek platforms using the mt8195 processors. diff --git a/sound/soc/sof/mediatek/Makefile b/sound/soc/sof/mediatek/Makefile index e8ec6da981..29c5afb2f3 100644 --- a/sound/soc/sof/mediatek/Makefile +++ b/sound/soc/sof/mediatek/Makefile @@ -1,2 +1,4 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) +obj-$(CONFIG_SND_SOC_SOF_MTK_COMMON) += mtk-adsp-common.o obj-$(CONFIG_SND_SOC_SOF_MT8195) += mt8195/ +obj-$(CONFIG_SND_SOC_SOF_MT8186) += mt8186/ diff --git a/sound/soc/sof/mediatek/adsp_helper.h b/sound/soc/sof/mediatek/adsp_helper.h index 6734e2c0c6..d41e904e66 100644 --- a/sound/soc/sof/mediatek/adsp_helper.h +++ b/sound/soc/sof/mediatek/adsp_helper.h @@ -7,37 +7,42 @@ #ifndef __MTK_ADSP_HELPER_H__ #define __MTK_ADSP_HELPER_H__ +#include + /* * Global important adsp data structure. */ -#define DSP_MBOX_NUM 3 - struct mtk_adsp_chip_info { phys_addr_t pa_sram; phys_addr_t pa_dram; /* adsp dram physical base */ phys_addr_t pa_shared_dram; /* adsp dram physical base */ phys_addr_t pa_cfgreg; - phys_addr_t pa_mboxreg[DSP_MBOX_NUM]; u32 sramsize; u32 dramsize; u32 cfgregsize; + u32 shared_size; void __iomem *va_sram; /* corresponding to pa_sram */ void __iomem *va_dram; /* corresponding to pa_dram */ void __iomem *va_cfgreg; - void __iomem *va_mboxreg[DSP_MBOX_NUM]; void __iomem *shared_sram; /* part of va_sram */ void __iomem *shared_dram; /* part of va_dram */ phys_addr_t adsp_bootup_addr; int dram_offset; /*dram offset between system and dsp view*/ + + phys_addr_t pa_secreg; + u32 secregsize; + void __iomem *va_secreg; + + phys_addr_t pa_busreg; + u32 busregsize; + void __iomem *va_busreg; }; struct adsp_priv { struct device *dev; struct snd_sof_dev *sdev; - - /* DSP IPC handler */ - struct mbox_controller *adsp_mbox; - + struct mtk_adsp_ipc *dsp_ipc; + struct platform_device *ipc_dev; struct mtk_adsp_chip_info *adsp; struct clk **clk; u32 (*ap2adsp_addr)(u32 addr, void *data); diff --git a/sound/soc/sof/mediatek/mt8195/mt8195-clk.c b/sound/soc/sof/mediatek/mt8195/mt8195-clk.c index 6bcb4b9b00..9ef08e43aa 100644 --- a/sound/soc/sof/mediatek/mt8195/mt8195-clk.c +++ b/sound/soc/sof/mediatek/mt8195/mt8195-clk.c @@ -132,6 +132,13 @@ static int adsp_default_clk_init(struct snd_sof_dev *sdev, bool enable) return ret; } + ret = clk_set_parent(priv->clk[CLK_TOP_AUDIO_H], + priv->clk[CLK_TOP_CLK26M]); + if (ret) { + dev_err(dev, "set audio_h_sel failed %d\n", ret); + return ret; + } + ret = adsp_enable_all_clock(sdev); if (ret) { dev_err(dev, "failed to adsp_enable_clock: %d\n", ret); diff --git a/sound/soc/sof/mediatek/mt8195/mt8195-loader.c b/sound/soc/sof/mediatek/mt8195/mt8195-loader.c index ed18d6379e..4be99ff4eb 100644 --- a/sound/soc/sof/mediatek/mt8195/mt8195-loader.c +++ b/sound/soc/sof/mediatek/mt8195/mt8195-loader.c @@ -21,7 +21,7 @@ void sof_hifixdsp_boot_sequence(struct snd_sof_dev *sdev, u32 boot_addr) /* pull high StatVectorSel to use AltResetVec (set bit4 to 1) */ snd_sof_dsp_update_bits(sdev, DSP_REG_BAR, DSP_RESET_SW, - DSP_RESET_SW, DSP_RESET_SW); + STATVECTOR_SEL, STATVECTOR_SEL); /* toggle DReset & BReset */ /* pull high DReset & BReset */ @@ -29,6 +29,9 @@ void sof_hifixdsp_boot_sequence(struct snd_sof_dev *sdev, u32 boot_addr) ADSP_BRESET_SW | ADSP_DRESET_SW, ADSP_BRESET_SW | ADSP_DRESET_SW); + /* delay 10 DSP cycles at 26M about 1us by IP vendor's suggestion */ + udelay(1); + /* pull low DReset & BReset */ snd_sof_dsp_update_bits(sdev, DSP_REG_BAR, DSP_RESET_SW, ADSP_BRESET_SW | ADSP_DRESET_SW, @@ -46,11 +49,13 @@ void sof_hifixdsp_boot_sequence(struct snd_sof_dev *sdev, u32 boot_addr) void sof_hifixdsp_shutdown(struct snd_sof_dev *sdev) { - /* Clear to 0 firstly */ - snd_sof_dsp_write(sdev, DSP_REG_BAR, DSP_RESET_SW, 0x0); - /* RUN_STALL pull high again to reset */ snd_sof_dsp_update_bits(sdev, DSP_REG_BAR, DSP_RESET_SW, ADSP_RUNSTALL, ADSP_RUNSTALL); + + /* pull high DReset & BReset */ + snd_sof_dsp_update_bits(sdev, DSP_REG_BAR, DSP_RESET_SW, + ADSP_BRESET_SW | ADSP_DRESET_SW, + ADSP_BRESET_SW | ADSP_DRESET_SW); } diff --git a/sound/soc/sof/mediatek/mt8195/mt8195.c b/sound/soc/sof/mediatek/mt8195/mt8195.c index 3ab12f3529..9c146015cd 100644 --- a/sound/soc/sof/mediatek/mt8195/mt8195.c +++ b/sound/soc/sof/mediatek/mt8195/mt8195.c @@ -24,9 +24,103 @@ #include "../../sof-of-dev.h" #include "../../sof-audio.h" #include "../adsp_helper.h" +#include "../mtk-adsp-common.h" #include "mt8195.h" #include "mt8195-clk.h" +static int mt8195_get_mailbox_offset(struct snd_sof_dev *sdev) +{ + return MBOX_OFFSET; +} + +static int mt8195_get_window_offset(struct snd_sof_dev *sdev, u32 id) +{ + return MBOX_OFFSET; +} + +static int mt8195_send_msg(struct snd_sof_dev *sdev, + struct snd_sof_ipc_msg *msg) +{ + struct adsp_priv *priv = sdev->pdata->hw_pdata; + + sof_mailbox_write(sdev, sdev->host_box.offset, msg->msg_data, + msg->msg_size); + + return mtk_adsp_ipc_send(priv->dsp_ipc, MTK_ADSP_IPC_REQ, MTK_ADSP_IPC_OP_REQ); +} + +static void mt8195_get_reply(struct snd_sof_dev *sdev) +{ + struct snd_sof_ipc_msg *msg = sdev->msg; + struct sof_ipc_reply reply; + int ret = 0; + + if (!msg) { + dev_warn(sdev->dev, "unexpected ipc interrupt\n"); + return; + } + + /* get reply */ + sof_mailbox_read(sdev, sdev->host_box.offset, &reply, sizeof(reply)); + if (reply.error < 0) { + memcpy(msg->reply_data, &reply, sizeof(reply)); + ret = reply.error; + } else { + /* reply has correct size? */ + if (reply.hdr.size != msg->reply_size) { + dev_err(sdev->dev, "error: reply expected %zu got %u bytes\n", + msg->reply_size, reply.hdr.size); + ret = -EINVAL; + } + + /* read the message */ + if (msg->reply_size > 0) + sof_mailbox_read(sdev, sdev->host_box.offset, + msg->reply_data, msg->reply_size); + } + + msg->reply_error = ret; +} + +static void mt8195_dsp_handle_reply(struct mtk_adsp_ipc *ipc) +{ + struct adsp_priv *priv = mtk_adsp_ipc_get_data(ipc); + unsigned long flags; + + spin_lock_irqsave(&priv->sdev->ipc_lock, flags); + mt8195_get_reply(priv->sdev); + snd_sof_ipc_reply(priv->sdev, 0); + spin_unlock_irqrestore(&priv->sdev->ipc_lock, flags); +} + +static void mt8195_dsp_handle_request(struct mtk_adsp_ipc *ipc) +{ + struct adsp_priv *priv = mtk_adsp_ipc_get_data(ipc); + u32 p; /* panic code */ + int ret; + + /* Read the message from the debug box. */ + sof_mailbox_read(priv->sdev, priv->sdev->debug_box.offset + 4, + &p, sizeof(p)); + + /* Check to see if the message is a panic code 0x0dead*** */ + if ((p & SOF_IPC_PANIC_MAGIC_MASK) == SOF_IPC_PANIC_MAGIC) { + snd_sof_dsp_panic(priv->sdev, p, true); + } else { + snd_sof_ipc_msgs_rx(priv->sdev); + + /* tell DSP cmd is done */ + ret = mtk_adsp_ipc_send(priv->dsp_ipc, MTK_ADSP_IPC_RSP, MTK_ADSP_IPC_OP_RSP); + if (ret) + dev_err(priv->dev, "request send ipc failed"); + } +} + +static struct mtk_adsp_ipc_ops dsp_ops = { + .handle_reply = mt8195_dsp_handle_reply, + .handle_request = mt8195_dsp_handle_request, +}; + static int platform_parse_resource(struct platform_device *pdev, void *data) { struct resource *mmio; @@ -51,6 +145,14 @@ static int platform_parse_resource(struct platform_device *pdev, void *data) dev_dbg(dev, "DMA %pR\n", &res); + adsp->pa_shared_dram = (phys_addr_t)res.start; + adsp->shared_size = resource_size(&res); + if (adsp->pa_shared_dram & DRAM_REMAP_MASK) { + dev_err(dev, "adsp shared dma memory(%#x) is not 4K-aligned\n", + (u32)adsp->pa_shared_dram); + return -EINVAL; + } + ret = of_reserved_mem_device_init(dev); if (ret) { dev_err(dev, "of_reserved_mem_device_init failed\n"); @@ -179,23 +281,18 @@ static int adsp_shared_base_ioremap(struct platform_device *pdev, void *data) { struct device *dev = &pdev->dev; struct mtk_adsp_chip_info *adsp = data; - u32 shared_size; /* remap shared-dram base to be non-cachable */ - shared_size = TOTAL_SIZE_SHARED_DRAM_FROM_TAIL; - adsp->pa_shared_dram = adsp->pa_dram + adsp->dramsize - shared_size; - if (adsp->va_dram) { - adsp->shared_dram = adsp->va_dram + DSP_DRAM_SIZE - shared_size; - } else { - adsp->shared_dram = devm_ioremap(dev, adsp->pa_shared_dram, - shared_size); - if (!adsp->shared_dram) { - dev_err(dev, "ioremap failed for shared DRAM\n"); - return -ENOMEM; - } + adsp->shared_dram = devm_ioremap(dev, adsp->pa_shared_dram, + adsp->shared_size); + if (!adsp->shared_dram) { + dev_err(dev, "failed to ioremap base %pa size %#x\n", + adsp->shared_dram, adsp->shared_size); + return -ENOMEM; } + dev_dbg(dev, "shared-dram vbase=%p, phy addr :%pa, size=%#x\n", - adsp->shared_dram, &adsp->pa_shared_dram, shared_size); + adsp->shared_dram, &adsp->pa_shared_dram, adsp->shared_size); return 0; } @@ -267,9 +364,11 @@ static int mt8195_dsp_probe(struct snd_sof_dev *sdev) goto err_adsp_sram_power_off; } - sdev->bar[SOF_FW_BLK_TYPE_SRAM] = devm_ioremap_wc(sdev->dev, - priv->adsp->pa_dram, - priv->adsp->dramsize); + priv->adsp->va_sram = sdev->bar[SOF_FW_BLK_TYPE_IRAM]; + + sdev->bar[SOF_FW_BLK_TYPE_SRAM] = devm_ioremap(sdev->dev, + priv->adsp->pa_dram, + priv->adsp->dramsize); if (!sdev->bar[SOF_FW_BLK_TYPE_SRAM]) { dev_err(sdev->dev, "failed to ioremap base %pa size %#x\n", &priv->adsp->pa_dram, priv->adsp->dramsize); @@ -285,15 +384,36 @@ static int mt8195_dsp_probe(struct snd_sof_dev *sdev) } sdev->bar[DSP_REG_BAR] = priv->adsp->va_cfgreg; - sdev->bar[DSP_MBOX0_BAR] = priv->adsp->va_mboxreg[0]; - sdev->bar[DSP_MBOX1_BAR] = priv->adsp->va_mboxreg[1]; - sdev->bar[DSP_MBOX2_BAR] = priv->adsp->va_mboxreg[2]; sdev->mmio_bar = SOF_FW_BLK_TYPE_SRAM; sdev->mailbox_bar = SOF_FW_BLK_TYPE_SRAM; + /* set default mailbox offset for FW ready message */ + sdev->dsp_box.offset = mt8195_get_mailbox_offset(sdev); + + priv->ipc_dev = platform_device_register_data(&pdev->dev, "mtk-adsp-ipc", + PLATFORM_DEVID_NONE, + pdev, sizeof(*pdev)); + if (IS_ERR(priv->ipc_dev)) { + ret = PTR_ERR(priv->ipc_dev); + dev_err(sdev->dev, "failed to register mtk-adsp-ipc device\n"); + goto err_adsp_sram_power_off; + } + + priv->dsp_ipc = dev_get_drvdata(&priv->ipc_dev->dev); + if (!priv->dsp_ipc) { + ret = -EPROBE_DEFER; + dev_err(sdev->dev, "failed to get drvdata\n"); + goto exit_pdev_unregister; + } + + mtk_adsp_ipc_set_data(priv->dsp_ipc, priv); + priv->dsp_ipc->ops = &dsp_ops; + return 0; +exit_pdev_unregister: + platform_device_unregister(priv->ipc_dev); err_adsp_sram_power_off: adsp_sram_power_on(&pdev->dev, false); exit_clk_disable: @@ -302,10 +422,17 @@ static int mt8195_dsp_probe(struct snd_sof_dev *sdev) return ret; } +static int mt8195_dsp_shutdown(struct snd_sof_dev *sdev) +{ + return snd_sof_suspend(sdev->dev); +} + static int mt8195_dsp_remove(struct snd_sof_dev *sdev) { struct platform_device *pdev = container_of(sdev->dev, struct platform_device, dev); + struct adsp_priv *priv = sdev->pdata->hw_pdata; + platform_device_unregister(priv->ipc_dev); adsp_sram_power_on(&pdev->dev, false); adsp_clock_off(sdev); @@ -316,6 +443,19 @@ static int mt8195_dsp_suspend(struct snd_sof_dev *sdev, u32 target_state) { struct platform_device *pdev = container_of(sdev->dev, struct platform_device, dev); int ret; + u32 reset_sw, dbg_pc; + + /* wait dsp enter idle, timeout is 1 second */ + ret = snd_sof_dsp_read_poll_timeout(sdev, DSP_REG_BAR, + DSP_RESET_SW, reset_sw, + ((reset_sw & ADSP_PWAIT) == ADSP_PWAIT), + SUSPEND_DSP_IDLE_POLL_INTERVAL_US, + SUSPEND_DSP_IDLE_TIMEOUT_US); + if (ret < 0) { + dbg_pc = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGPC); + dev_warn(sdev->dev, "dsp not idle, powering off anyway : swrest %#x, pc %#x, ret %d\n", + reset_sw, dbg_pc, ret); + } /* stall and reset dsp */ sof_hifixdsp_shutdown(sdev); @@ -356,6 +496,39 @@ static int mt8195_get_bar_index(struct snd_sof_dev *sdev, u32 type) return type; } +static int mt8195_ipc_msg_data(struct snd_sof_dev *sdev, + struct snd_pcm_substream *substream, + void *p, size_t sz) +{ + sof_mailbox_read(sdev, sdev->dsp_box.offset, p, sz); + return 0; +} + +static void mt8195_adsp_dump(struct snd_sof_dev *sdev, u32 flags) +{ + u32 dbg_pc, dbg_data, dbg_bus0, dbg_bus1, dbg_inst; + u32 dbg_ls0stat, dbg_ls1stat, faultbus, faultinfo, swrest; + + /* dump debug registers */ + dbg_pc = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGPC); + dbg_data = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGDATA); + dbg_bus0 = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGBUS0); + dbg_bus1 = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGBUS1); + dbg_inst = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGINST); + dbg_ls0stat = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGLS0STAT); + dbg_ls1stat = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PDEBUGLS1STAT); + faultbus = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PFAULTBUS); + faultinfo = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_PFAULTINFO); + swrest = snd_sof_dsp_read(sdev, DSP_REG_BAR, DSP_RESET_SW); + + dev_info(sdev->dev, "adsp dump : pc %#x, data %#x, bus0 %#x, bus1 %#x, swrest %#x", + dbg_pc, dbg_data, dbg_bus0, dbg_bus1, swrest); + dev_info(sdev->dev, "dbg_inst %#x, ls0stat %#x, ls1stat %#x, faultbus %#x, faultinfo %#x", + dbg_inst, dbg_ls0stat, dbg_ls1stat, faultbus, faultinfo); + + mtk_adsp_dump(sdev, flags); +} + static struct snd_soc_dai_driver mt8195_dai[] = { { .name = "SOF_DL2", @@ -388,10 +561,11 @@ static struct snd_soc_dai_driver mt8195_dai[] = { }; /* mt8195 ops */ -static const struct snd_sof_dsp_ops sof_mt8195_ops = { +static struct snd_sof_dsp_ops sof_mt8195_ops = { /* probe and remove */ .probe = mt8195_dsp_probe, .remove = mt8195_dsp_remove, + .shutdown = mt8195_dsp_shutdown, /* DSP core boot */ .run = mt8195_run, @@ -406,17 +580,25 @@ static const struct snd_sof_dsp_ops sof_mt8195_ops = { .write64 = sof_io_write64, .read64 = sof_io_read64, + /* ipc */ + .send_msg = mt8195_send_msg, + .get_mailbox_offset = mt8195_get_mailbox_offset, + .get_window_offset = mt8195_get_window_offset, + .ipc_msg_data = mt8195_ipc_msg_data, + .set_stream_data_offset = sof_set_stream_data_offset, + /* misc */ .get_bar_index = mt8195_get_bar_index, - /* module loading */ - .load_module = snd_sof_parse_module_memcpy, /* firmware loading */ .load_firmware = snd_sof_load_firmware_memcpy, /* Firmware ops */ .dsp_arch_ops = &sof_xtensa_arch_ops, + /* Debug information */ + .dbg_dump = mt8195_adsp_dump, + /* DAI drivers */ .drv = mt8195_dai, .num_drv = ARRAY_SIZE(mt8195_dai), @@ -434,11 +616,20 @@ static const struct snd_sof_dsp_ops sof_mt8195_ops = { }; static const struct sof_dev_desc sof_of_mt8195_desc = { - .default_fw_path = "mediatek/sof", - .default_tplg_path = "mediatek/sof-tplg", - .default_fw_filename = "sof-mt8195.ri", + .ipc_supported_mask = BIT(SOF_IPC), + .ipc_default = SOF_IPC, + .default_fw_path = { + [SOF_IPC] = "mediatek/sof", + }, + .default_tplg_path = { + [SOF_IPC] = "mediatek/sof-tplg", + }, + .default_fw_filename = { + [SOF_IPC] = "sof-mt8195.ri", + }, .nocodec_tplg_filename = "sof-mt8195-nocodec.tplg", .ops = &sof_mt8195_ops, + .ipc_timeout = 1000, }; static const struct of_device_id sof_of_mt8195_ids[] = { @@ -451,6 +642,7 @@ MODULE_DEVICE_TABLE(of, sof_of_mt8195_ids); static struct platform_driver snd_sof_of_mt8195_driver = { .probe = sof_of_probe, .remove = sof_of_remove, + .shutdown = sof_of_shutdown, .driver = { .name = "sof-audio-of-mt8195", .pm = &sof_of_pm, diff --git a/sound/soc/sof/mediatek/mt8195/mt8195.h b/sound/soc/sof/mediatek/mt8195/mt8195.h index 9294241823..7ffd523f93 100644 --- a/sound/soc/sof/mediatek/mt8195/mt8195.h +++ b/sound/soc/sof/mediatek/mt8195/mt8195.h @@ -34,6 +34,7 @@ struct snd_sof_dev; #define ADSP_DRESET_SW BIT(1) #define ADSP_RUNSTALL BIT(3) #define STATVECTOR_SEL BIT(4) +#define ADSP_PWAIT BIT(16) #define DSP_PFAULTBUS 0x0028 #define DSP_PFAULTINFO 0x002c #define DSP_GPR00 0x0030 @@ -153,6 +154,10 @@ struct snd_sof_dev; #define DRAM_REMAP_SHIFT 12 #define DRAM_REMAP_MASK (BIT(DRAM_REMAP_SHIFT) - 1) +/* suspend dsp idle check interval and timeout */ +#define SUSPEND_DSP_IDLE_TIMEOUT_US 1000000 /* timeout to wait dsp idle, 1 sec */ +#define SUSPEND_DSP_IDLE_POLL_INTERVAL_US 500 /* 0.5 msec */ + void sof_hifixdsp_boot_sequence(struct snd_sof_dev *sdev, u32 boot_addr); void sof_hifixdsp_shutdown(struct snd_sof_dev *sdev); #endif diff --git a/sound/soc/sof/ops.c b/sound/soc/sof/ops.c index 235e2ef721..ff066de4ce 100644 --- a/sound/soc/sof/ops.c +++ b/sound/soc/sof/ops.c @@ -177,7 +177,7 @@ void snd_sof_dsp_panic(struct snd_sof_dev *sdev, u32 offset, bool non_recoverabl snd_sof_dsp_dbg_dump(sdev, "DSP panic!", SOF_DBG_DUMP_REGS | SOF_DBG_DUMP_MBOX); sof_set_fw_state(sdev, SOF_FW_CRASHED); - snd_sof_trace_notify_for_error(sdev); + sof_fw_trace_fw_crashed(sdev); } else { snd_sof_dsp_dbg_dump(sdev, "DSP panic (recovery will be attempted)", diff --git a/sound/soc/sof/ops.h b/sound/soc/sof/ops.h index a194746637..55d43adb6a 100644 --- a/sound/soc/sof/ops.h +++ b/sound/soc/sof/ops.h @@ -21,6 +21,20 @@ #define sof_ops(sdev) \ ((sdev)->pdata->desc->ops) +static inline int sof_ops_init(struct snd_sof_dev *sdev) +{ + if (sdev->pdata->desc->ops_init) + return sdev->pdata->desc->ops_init(sdev); + + return 0; +} + +static inline void sof_ops_free(struct snd_sof_dev *sdev) +{ + if (sdev->pdata->desc->ops_free) + sdev->pdata->desc->ops_free(sdev); +} + /* Mandatory operations are verified during probing */ /* init */ @@ -367,32 +381,6 @@ static inline int snd_sof_dsp_send_msg(struct snd_sof_dev *sdev, return sof_ops(sdev)->send_msg(sdev, msg); } -/* host DMA trace */ -static inline int snd_sof_dma_trace_init(struct snd_sof_dev *sdev, - struct sof_ipc_dma_trace_params_ext *dtrace_params) -{ - if (sof_ops(sdev)->trace_init) - return sof_ops(sdev)->trace_init(sdev, dtrace_params); - - return 0; -} - -static inline int snd_sof_dma_trace_release(struct snd_sof_dev *sdev) -{ - if (sof_ops(sdev)->trace_release) - return sof_ops(sdev)->trace_release(sdev); - - return 0; -} - -static inline int snd_sof_dma_trace_trigger(struct snd_sof_dev *sdev, int cmd) -{ - if (sof_ops(sdev)->trace_trigger) - return sof_ops(sdev)->trace_trigger(sdev, cmd); - - return 0; -} - /* host PCM ops */ static inline int snd_sof_pcm_platform_open(struct snd_sof_dev *sdev, diff --git a/sound/soc/sof/pcm.c b/sound/soc/sof/pcm.c index 658cd8966c..6cb6a432be 100644 --- a/sound/soc/sof/pcm.c +++ b/sound/soc/sof/pcm.c @@ -82,8 +82,10 @@ void snd_sof_pcm_period_elapsed(struct snd_pcm_substream *substream) } EXPORT_SYMBOL(snd_sof_pcm_period_elapsed); -int sof_pcm_setup_connected_widgets(struct snd_sof_dev *sdev, struct snd_soc_pcm_runtime *rtd, - struct snd_sof_pcm *spcm, int dir) +static int +sof_pcm_setup_connected_widgets(struct snd_sof_dev *sdev, struct snd_soc_pcm_runtime *rtd, + struct snd_sof_pcm *spcm, struct snd_pcm_hw_params *params, + struct snd_sof_platform_stream_params *platform_params, int dir) { struct snd_soc_dai *dai; int ret, j; @@ -102,7 +104,7 @@ int sof_pcm_setup_connected_widgets(struct snd_sof_dev *sdev, struct snd_soc_pcm spcm->stream[dir].list = list; - ret = sof_widget_list_setup(sdev, spcm, dir); + ret = sof_widget_list_setup(sdev, spcm, params, platform_params, dir); if (ret < 0) { dev_err(sdev->dev, "error: failed widget list set up for pcm %d dir %d\n", spcm->pcm.pcm_id, dir); @@ -150,9 +152,16 @@ static int sof_pcm_hw_params(struct snd_soc_component *component, dev_dbg(component->dev, "pcm: hw params stream %d dir %d\n", spcm->pcm.pcm_id, substream->stream); + ret = snd_sof_pcm_platform_hw_params(sdev, substream, params, &platform_params); + if (ret < 0) { + dev_err(component->dev, "platform hw params failed\n"); + return ret; + } + /* if this is a repeated hw_params without hw_free, skip setting up widgets */ if (!spcm->stream[substream->stream].list) { - ret = sof_pcm_setup_connected_widgets(sdev, rtd, spcm, substream->stream); + ret = sof_pcm_setup_connected_widgets(sdev, rtd, spcm, params, &platform_params, + substream->stream); if (ret < 0) return ret; } @@ -166,12 +175,6 @@ static int sof_pcm_hw_params(struct snd_soc_component *component, return ret; } - ret = snd_sof_pcm_platform_hw_params(sdev, substream, params, &platform_params); - if (ret < 0) { - dev_err(component->dev, "platform hw params failed\n"); - return ret; - } - if (pcm_ops->hw_params) { ret = pcm_ops->hw_params(component, substream, params, &platform_params); if (ret < 0) @@ -347,12 +350,9 @@ static int sof_pcm_trigger(struct snd_soc_component *component, snd_sof_pcm_platform_trigger(sdev, substream, cmd); /* free PCM if reset_hw_params is set and the STOP IPC is successful */ - if (!ret && reset_hw_params) { + if (!ret && reset_hw_params) ret = sof_pcm_stream_free(sdev, substream, spcm, substream->stream, free_widget_list); - if (ret < 0) - return ret; - } return ret; } @@ -396,7 +396,7 @@ static int sof_pcm_open(struct snd_soc_component *component, struct snd_soc_pcm_runtime *rtd = asoc_substream_to_rtd(substream); struct snd_pcm_runtime *runtime = substream->runtime; struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(component); - const struct snd_sof_dsp_ops *ops = sof_ops(sdev); + struct snd_sof_dsp_ops *ops = sof_ops(sdev); struct snd_sof_pcm *spcm; struct snd_soc_tplg_stream_caps *caps; int ret; @@ -604,6 +604,14 @@ static int sof_pcm_probe(struct snd_soc_component *component) const char *tplg_filename; int ret; + /* + * make sure the device is pm_runtime_active before loading the + * topology and initiating IPC or bus transactions + */ + ret = pm_runtime_resume_and_get(component->dev); + if (ret < 0 && ret != -EACCES) + return ret; + /* load the default topology */ sdev->component = component; @@ -621,6 +629,9 @@ static int sof_pcm_probe(struct snd_soc_component *component) return ret; } + pm_runtime_mark_last_busy(component->dev); + pm_runtime_put_autosuspend(component->dev); + return ret; } @@ -671,4 +682,6 @@ void snd_sof_new_platform_drv(struct snd_sof_dev *sdev) /* increment module refcount when a pcm is opened */ pd->module_get_upon_open = 1; + + pd->legacy_dai_naming = 1; } diff --git a/sound/soc/sof/pm.c b/sound/soc/sof/pm.c index 1c319582ca..df740be645 100644 --- a/sound/soc/sof/pm.c +++ b/sound/soc/sof/pm.c @@ -23,6 +23,9 @@ static u32 snd_sof_dsp_power_target(struct snd_sof_dev *sdev) u32 target_dsp_state; switch (sdev->system_suspend_target) { + case SOF_SUSPEND_S5: + case SOF_SUSPEND_S4: + /* DSP should be in D3 if the system is suspending to S3+ */ case SOF_SUSPEND_S3: /* DSP should be in D3 if the system is suspending to S3 */ target_dsp_state = SOF_DSP_PM_D3; @@ -102,11 +105,18 @@ static int sof_resume(struct device *dev, bool runtime_resume) /* * Nothing further to be done for platforms that support the low power - * D0 substate. + * D0 substate. Resume trace and return when resuming from + * low-power D0 substate */ if (!runtime_resume && sof_ops(sdev)->set_power_state && - old_state == SOF_DSP_PM_D0) + old_state == SOF_DSP_PM_D0) { + ret = sof_fw_trace_resume(sdev); + if (ret < 0) + /* non fatal */ + dev_warn(sdev->dev, + "failed to enable trace after resume %d\n", ret); return 0; + } sof_set_fw_state(sdev, SOF_FW_BOOT_PREPARE); @@ -135,8 +145,8 @@ static int sof_resume(struct device *dev, bool runtime_resume) return ret; } - /* resume DMA trace, only need send ipc */ - ret = snd_sof_init_trace_ipc(sdev); + /* resume DMA trace */ + ret = sof_fw_trace_resume(sdev); if (ret < 0) { /* non fatal */ dev_warn(sdev->dev, @@ -187,7 +197,7 @@ static int sof_suspend(struct device *dev, bool runtime_suspend) /* prepare for streams to be resumed properly upon resume */ if (!runtime_suspend) { - ret = sof_set_hw_params_upon_resume(sdev->dev); + ret = snd_sof_dsp_hw_params_upon_resume(sdev); if (ret < 0) { dev_err(sdev->dev, "error: setting hw_params flag during suspend %d\n", @@ -201,6 +211,7 @@ static int sof_suspend(struct device *dev, bool runtime_suspend) /* Skip to platform-specific suspend if DSP is entering D0 */ if (target_state == SOF_DSP_PM_D0) { + sof_fw_trace_suspend(sdev, pm_state); /* Notify clients not managed by pm framework about core suspend */ sof_suspend_clients(sdev, pm_state); goto suspend; @@ -209,8 +220,8 @@ static int sof_suspend(struct device *dev, bool runtime_suspend) if (tplg_ops->tear_down_all_pipelines) tplg_ops->tear_down_all_pipelines(sdev, false); - /* release trace */ - snd_sof_release_trace(sdev); + /* suspend DMA trace */ + sof_fw_trace_suspend(sdev, pm_state); /* Notify clients not managed by pm framework about core suspend */ sof_suspend_clients(sdev, pm_state); @@ -327,8 +338,24 @@ int snd_sof_prepare(struct device *dev) return 0; #if defined(CONFIG_ACPI) - if (acpi_target_system_state() == ACPI_STATE_S0) + switch (acpi_target_system_state()) { + case ACPI_STATE_S0: sdev->system_suspend_target = SOF_SUSPEND_S0IX; + break; + case ACPI_STATE_S1: + case ACPI_STATE_S2: + case ACPI_STATE_S3: + sdev->system_suspend_target = SOF_SUSPEND_S3; + break; + case ACPI_STATE_S4: + sdev->system_suspend_target = SOF_SUSPEND_S4; + break; + case ACPI_STATE_S5: + sdev->system_suspend_target = SOF_SUSPEND_S5; + break; + default: + break; + } #endif return 0; diff --git a/sound/soc/sof/sof-acpi-dev.c b/sound/soc/sof/sof-acpi-dev.c index 74982c0449..1b04dcb332 100644 --- a/sound/soc/sof/sof-acpi-dev.c +++ b/sound/soc/sof/sof-acpi-dev.c @@ -74,20 +74,20 @@ int sof_acpi_probe(struct platform_device *pdev, const struct sof_dev_desc *desc sof_pdata->desc = desc; sof_pdata->dev = &pdev->dev; - sof_pdata->fw_filename = desc->default_fw_filename; + sof_pdata->fw_filename = desc->default_fw_filename[SOF_IPC]; /* alternate fw and tplg filenames ? */ if (fw_path) sof_pdata->fw_filename_prefix = fw_path; else sof_pdata->fw_filename_prefix = - sof_pdata->desc->default_fw_path; + sof_pdata->desc->default_fw_path[SOF_IPC]; if (tplg_path) sof_pdata->tplg_filename_prefix = tplg_path; else sof_pdata->tplg_filename_prefix = - sof_pdata->desc->default_tplg_path; + sof_pdata->desc->default_tplg_path[SOF_IPC]; /* set callback to be called on successful device probe to enable runtime_pm */ sof_pdata->sof_probe_complete = sof_acpi_probe_complete; diff --git a/sound/soc/sof/sof-audio.c b/sound/soc/sof/sof-audio.c index b2f009a0c5..28976098a8 100644 --- a/sound/soc/sof/sof-audio.c +++ b/sound/soc/sof/sof-audio.c @@ -12,61 +12,18 @@ #include "sof-audio.h" #include "ops.h" -static int sof_kcontrol_setup(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol) -{ - int ret; - - /* reset readback offset for scontrol */ - scontrol->readback_offset = 0; - - ret = snd_sof_ipc_set_get_comp_data(scontrol, true); - if (ret < 0) - dev_err(sdev->dev, "error: failed kcontrol value set for widget: %d\n", - scontrol->comp_id); - - return ret; -} - -static int sof_widget_kcontrol_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) -{ - struct snd_sof_control *scontrol; - int ret; - - /* set up all controls for the widget */ - list_for_each_entry(scontrol, &sdev->kcontrol_list, list) - if (scontrol->comp_id == swidget->comp_id) { - /* set kcontrol data in DSP */ - ret = sof_kcontrol_setup(sdev, scontrol); - if (ret < 0) { - dev_err(sdev->dev, "error: fail to set up kcontrols for widget %s\n", - swidget->widget->name); - return ret; - } - - /* - * Read back the data from the DSP for static widgets. This is particularly - * useful for binary kcontrols associated with static pipeline widgets to - * initialize the data size to match that in the DSP. - */ - if (swidget->dynamic_pipeline_widget) - continue; - - ret = snd_sof_ipc_set_get_comp_data(scontrol, false); - if (ret < 0) - dev_warn(sdev->dev, "Failed kcontrol get for control in widget %s\n", - swidget->widget->name); - } - - return 0; -} - static void sof_reset_route_setup_status(struct snd_sof_dev *sdev, struct snd_sof_widget *widget) { + const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; struct snd_sof_route *sroute; list_for_each_entry(sroute, &sdev->route_list, list) - if (sroute->src_widget == widget || sroute->sink_widget == widget) + if (sroute->src_widget == widget || sroute->sink_widget == widget) { + if (sroute->setup && tplg_ops->route_free) + tplg_ops->route_free(sdev, sroute); + sroute->setup = false; + } } int sof_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) @@ -82,6 +39,9 @@ int sof_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) if (--swidget->use_count) return 0; + /* reset route setup status for all routes that contain this widget */ + sof_reset_route_setup_status(sdev, swidget); + /* continue to disable core even if IPC fails */ if (tplg_ops->widget_free) err = tplg_ops->widget_free(sdev, swidget); @@ -98,10 +58,6 @@ int sof_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) err = ret; } - /* reset route setup status for all routes that contain this widget */ - sof_reset_route_setup_status(sdev, swidget); - swidget->complete = 0; - /* * free the scheduler widget (same as pipe_widget) associated with the current swidget. * skip for static pipelines @@ -110,6 +66,7 @@ int sof_widget_free(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) ret = sof_widget_free(sdev, swidget->pipe_widget); if (ret < 0 && !err) err = ret; + swidget->pipe_widget->complete = 0; } if (!err) @@ -179,11 +136,10 @@ int sof_widget_setup(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget) } /* restore kcontrols for widget */ - ret = sof_widget_kcontrol_setup(sdev, swidget); - if (ret < 0) { - dev_err(sdev->dev, "error: failed to restore kcontrols for widget %s\n", - swidget->widget->name); - goto widget_free; + if (tplg_ops->control->widget_kcontrol_setup) { + ret = tplg_ops->control->widget_kcontrol_setup(sdev, swidget); + if (ret < 0) + goto widget_free; } dev_dbg(sdev->dev, "widget %s setup complete\n", swidget->widget->name); @@ -301,28 +257,249 @@ static int sof_setup_pipeline_connections(struct snd_sof_dev *sdev, return 0; } -int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, int dir) +static void +sof_unprepare_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dapm_widget *widget) +{ + const struct sof_ipc_tplg_ops *ipc_tplg_ops = sdev->ipc->ops->tplg; + const struct sof_ipc_tplg_widget_ops *widget_ops = ipc_tplg_ops->widget; + struct snd_sof_widget *swidget = widget->dobj.private; + struct snd_soc_dapm_path *p; + + if (!widget_ops[widget->id].ipc_unprepare || !swidget->prepared) + goto sink_unprepare; + + /* unprepare the source widget */ + widget_ops[widget->id].ipc_unprepare(swidget); + swidget->prepared = false; + +sink_unprepare: + /* unprepare all widgets in the sink paths */ + snd_soc_dapm_widget_for_each_sink_path(widget, p) { + if (!p->walking && p->sink->dobj.private) { + p->walking = true; + sof_unprepare_widgets_in_path(sdev, p->sink); + p->walking = false; + } + } +} + +static int +sof_prepare_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dapm_widget *widget, + struct snd_pcm_hw_params *fe_params, + struct snd_sof_platform_stream_params *platform_params, + struct snd_pcm_hw_params *pipeline_params, int dir) +{ + const struct sof_ipc_tplg_ops *ipc_tplg_ops = sdev->ipc->ops->tplg; + const struct sof_ipc_tplg_widget_ops *widget_ops = ipc_tplg_ops->widget; + struct snd_sof_widget *swidget = widget->dobj.private; + struct snd_soc_dapm_path *p; + int ret; + + if (!widget_ops[widget->id].ipc_prepare || swidget->prepared) + goto sink_prepare; + + /* prepare the source widget */ + ret = widget_ops[widget->id].ipc_prepare(swidget, fe_params, platform_params, + pipeline_params, dir); + if (ret < 0) { + dev_err(sdev->dev, "failed to prepare widget %s\n", widget->name); + return ret; + } + + swidget->prepared = true; + +sink_prepare: + /* prepare all widgets in the sink paths */ + snd_soc_dapm_widget_for_each_sink_path(widget, p) { + if (!p->walking && p->sink->dobj.private) { + p->walking = true; + ret = sof_prepare_widgets_in_path(sdev, p->sink, fe_params, + platform_params, pipeline_params, dir); + p->walking = false; + if (ret < 0) { + /* unprepare the source widget */ + if (widget_ops[widget->id].ipc_unprepare && swidget->prepared) { + widget_ops[widget->id].ipc_unprepare(swidget); + swidget->prepared = false; + } + return ret; + } + } + } + + return 0; +} + +/* + * free all widgets in the sink path starting from the source widget + * (DAI type for capture, AIF type for playback) + */ +static int sof_free_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dapm_widget *widget, + int dir) +{ + struct snd_soc_dapm_path *p; + int err; + int ret = 0; + + /* free all widgets even in case of error to keep use counts balanced */ + snd_soc_dapm_widget_for_each_sink_path(widget, p) { + if (!p->walking && p->sink->dobj.private && widget->dobj.private) { + p->walking = true; + if (WIDGET_IS_AIF_OR_DAI(widget->id)) { + err = sof_widget_free(sdev, widget->dobj.private); + if (err < 0) + ret = err; + } + + err = sof_widget_free(sdev, p->sink->dobj.private); + if (err < 0) + ret = err; + + err = sof_free_widgets_in_path(sdev, p->sink, dir); + if (err < 0) + ret = err; + p->walking = false; + } + } + + return ret; +} + +/* + * set up all widgets in the sink path starting from the source widget + * (DAI type for capture, AIF type for playback). + * The error path in this function ensures that all successfully set up widgets getting freed. + */ +static int sof_set_up_widgets_in_path(struct snd_sof_dev *sdev, struct snd_soc_dapm_widget *widget, + int dir) +{ + struct snd_soc_dapm_path *p; + int ret; + + snd_soc_dapm_widget_for_each_sink_path(widget, p) { + if (!p->walking && p->sink->dobj.private && widget->dobj.private) { + p->walking = true; + if (WIDGET_IS_AIF_OR_DAI(widget->id)) { + ret = sof_widget_setup(sdev, widget->dobj.private); + if (ret < 0) + goto out; + } + + ret = sof_widget_setup(sdev, p->sink->dobj.private); + if (ret < 0) { + if (WIDGET_IS_AIF_OR_DAI(widget->id)) + sof_widget_free(sdev, widget->dobj.private); + goto out; + } + + ret = sof_set_up_widgets_in_path(sdev, p->sink, dir); + if (ret < 0) { + if (WIDGET_IS_AIF_OR_DAI(widget->id)) + sof_widget_free(sdev, widget->dobj.private); + sof_widget_free(sdev, p->sink->dobj.private); + } +out: + p->walking = false; + if (ret < 0) + return ret; + } + } + + return 0; +} + +static int +sof_walk_widgets_in_order(struct snd_sof_dev *sdev, struct snd_soc_dapm_widget_list *list, + struct snd_pcm_hw_params *fe_params, + struct snd_sof_platform_stream_params *platform_params, int dir, + enum sof_widget_op op) +{ + struct snd_soc_dapm_widget *widget; + char *str; + int ret = 0; + int i; + + for_each_dapm_widgets(list, i, widget) { + /* starting widget for playback is AIF type */ + if (dir == SNDRV_PCM_STREAM_PLAYBACK && !WIDGET_IS_AIF(widget->id)) + continue; + + /* starting widget for capture is DAI type */ + if (dir == SNDRV_PCM_STREAM_CAPTURE && !WIDGET_IS_DAI(widget->id)) + continue; + + switch (op) { + case SOF_WIDGET_SETUP: + ret = sof_set_up_widgets_in_path(sdev, widget, dir); + str = "set up"; + break; + case SOF_WIDGET_FREE: + ret = sof_free_widgets_in_path(sdev, widget, dir); + str = "free"; + break; + case SOF_WIDGET_PREPARE: + { + struct snd_pcm_hw_params pipeline_params; + + str = "prepare"; + /* + * When walking the list of connected widgets, the pipeline_params for each + * widget is modified by the source widget in the path. Use a local + * copy of the runtime params as the pipeline_params so that the runtime + * params does not get overwritten. + */ + memcpy(&pipeline_params, fe_params, sizeof(*fe_params)); + + ret = sof_prepare_widgets_in_path(sdev, widget, fe_params, + platform_params, &pipeline_params, dir); + break; + } + case SOF_WIDGET_UNPREPARE: + sof_unprepare_widgets_in_path(sdev, widget); + break; + default: + dev_err(sdev->dev, "Invalid widget op %d\n", op); + return -EINVAL; + } + if (ret < 0) { + dev_err(sdev->dev, "Failed to %s connected widgets\n", str); + return ret; + } + } + + return 0; +} + +int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, + struct snd_pcm_hw_params *fe_params, + struct snd_sof_platform_stream_params *platform_params, + int dir) { const struct sof_ipc_tplg_ops *ipc_tplg_ops = sdev->ipc->ops->tplg; struct snd_soc_dapm_widget_list *list = spcm->stream[dir].list; struct snd_soc_dapm_widget *widget; - int i, ret, num_widgets; + int i, ret; /* nothing to set up */ if (!list) return 0; - /* set up widgets in the list */ - for_each_dapm_widgets(list, num_widgets, widget) { - struct snd_sof_widget *swidget = widget->dobj.private; + /* + * Prepare widgets for set up. The prepare step is used to allocate memory, assign + * instance ID and pick the widget configuration based on the runtime PCM params. + */ + ret = sof_walk_widgets_in_order(sdev, list, fe_params, platform_params, + dir, SOF_WIDGET_PREPARE); + if (ret < 0) + return ret; - if (!swidget) - continue; - - /* set up the widget */ - ret = sof_widget_setup(sdev, swidget); - if (ret < 0) - goto widget_free; + /* Set up is used to send the IPC to the DSP to create the widget */ + ret = sof_walk_widgets_in_order(sdev, list, fe_params, platform_params, + dir, SOF_WIDGET_SETUP); + if (ret < 0) { + ret = sof_walk_widgets_in_order(sdev, list, fe_params, platform_params, + dir, SOF_WIDGET_UNPREPARE); + return ret; } /* @@ -364,18 +541,9 @@ int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, in return 0; widget_free: - /* free all widgets that have been set up successfully */ - for_each_dapm_widgets(list, i, widget) { - struct snd_sof_widget *swidget = widget->dobj.private; - - if (!swidget) - continue; - - if (!num_widgets--) - break; - - sof_widget_free(sdev, swidget); - } + sof_walk_widgets_in_order(sdev, list, fe_params, platform_params, dir, + SOF_WIDGET_FREE); + sof_walk_widgets_in_order(sdev, list, NULL, NULL, dir, SOF_WIDGET_UNPREPARE); return ret; } @@ -383,37 +551,22 @@ int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, in int sof_widget_list_free(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, int dir) { struct snd_soc_dapm_widget_list *list = spcm->stream[dir].list; - struct snd_soc_dapm_widget *widget; - int i, ret; - int ret1 = 0; + int ret; /* nothing to free */ if (!list) return 0; - /* - * Free widgets in the list. This can fail but continue freeing other widgets to keep - * use_counts balanced. - */ - for_each_dapm_widgets(list, i, widget) { - struct snd_sof_widget *swidget = widget->dobj.private; + /* send IPC to free widget in the DSP */ + ret = sof_walk_widgets_in_order(sdev, list, NULL, NULL, dir, SOF_WIDGET_FREE); - if (!swidget) - continue; - - /* - * free widget and its pipe_widget. Either of these can fail, but free as many as - * possible before freeing the list and returning the error. - */ - ret = sof_widget_free(sdev, swidget); - if (ret < 0) - ret1 = ret; - } + /* unprepare the widget */ + sof_walk_widgets_in_order(sdev, list, NULL, NULL, dir, SOF_WIDGET_UNPREPARE); snd_soc_dapm_dai_free_widgets(&list); spcm->stream[dir].list = NULL; - return ret1; + return ret; } /* @@ -462,42 +615,6 @@ bool snd_sof_stream_suspend_ignored(struct snd_sof_dev *sdev) return false; } -int sof_set_hw_params_upon_resume(struct device *dev) -{ - struct snd_sof_dev *sdev = dev_get_drvdata(dev); - struct snd_pcm_substream *substream; - struct snd_sof_pcm *spcm; - snd_pcm_state_t state; - int dir; - - /* - * SOF requires hw_params to be set-up internally upon resume. - * So, set the flag to indicate this for those streams that - * have been suspended. - */ - list_for_each_entry(spcm, &sdev->pcm_list, list) { - for_each_pcm_streams(dir) { - /* - * do not reset hw_params upon resume for streams that - * were kept running during suspend - */ - if (spcm->stream[dir].suspend_ignored) - continue; - - substream = spcm->stream[dir].substream; - if (!substream || !substream->runtime) - continue; - - state = substream->runtime->status->state; - if (state == SNDRV_PCM_STATE_SUSPENDED) - spcm->prepared[dir] = false; - } - } - - /* set internal flag for BE */ - return snd_sof_dsp_hw_params_upon_resume(sdev); -} - int sof_pcm_stream_free(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, struct snd_sof_pcm *spcm, int dir, bool free_widget_list) { @@ -701,7 +818,8 @@ int sof_machine_check(struct snd_sof_dev *sdev) return -ENOMEM; mach->drv_name = "sof-nocodec"; - sof_pdata->tplg_filename = desc->nocodec_tplg_filename; + if (!sof_pdata->tplg_filename) + sof_pdata->tplg_filename = desc->nocodec_tplg_filename; sof_pdata->machine = mach; snd_sof_set_mach_params(mach, sdev); diff --git a/sound/soc/sof/sof-audio.h b/sound/soc/sof/sof-audio.h index 7f15b3bc81..4284ea2f3a 100644 --- a/sound/soc/sof/sof-audio.h +++ b/sound/soc/sof/sof-audio.h @@ -29,16 +29,47 @@ #define DMA_CHAN_INVALID 0xFFFFFFFF #define WIDGET_IS_DAI(id) ((id) == snd_soc_dapm_dai_in || (id) == snd_soc_dapm_dai_out) +#define WIDGET_IS_AIF(id) ((id) == snd_soc_dapm_aif_in || (id) == snd_soc_dapm_aif_out) +#define WIDGET_IS_AIF_OR_DAI(id) (WIDGET_IS_DAI(id) || WIDGET_IS_AIF(id)) #define SOF_DAI_CLK_INTEL_SSP_MCLK 0 #define SOF_DAI_CLK_INTEL_SSP_BCLK 1 +enum sof_widget_op { + SOF_WIDGET_PREPARE, + SOF_WIDGET_SETUP, + SOF_WIDGET_FREE, + SOF_WIDGET_UNPREPARE, +}; + /* * Volume fractional word length define to 16 sets * the volume linear gain value to use Qx.16 format */ #define VOLUME_FWL 16 +#define SOF_TLV_ITEMS 3 + +static inline u32 mixer_to_ipc(unsigned int value, u32 *volume_map, int size) +{ + if (value >= size) + return volume_map[size - 1]; + + return volume_map[value]; +} + +static inline u32 ipc_to_mixer(u32 value, u32 *volume_map, int size) +{ + int i; + + for (i = 0; i < size; i++) { + if (volume_map[i] >= value) + return i; + } + + return i - 1; +} + struct snd_sof_widget; struct snd_sof_route; struct snd_sof_control; @@ -86,6 +117,11 @@ struct sof_ipc_tplg_control_ops { const unsigned int __user *binary_data, unsigned int size); /* update control data based on notification from the DSP */ void (*update)(struct snd_sof_dev *sdev, void *ipc_control_message); + /* Optional callback to setup kcontrols associated with an swidget */ + int (*widget_kcontrol_setup)(struct snd_sof_dev *sdev, struct snd_sof_widget *swidget); + /* mandatory callback to set up volume table for volume kcontrols */ + int (*set_up_volume_table)(struct snd_sof_control *scontrol, int tlv[SOF_TLV_ITEMS], + int size); }; /** @@ -95,6 +131,8 @@ struct sof_ipc_tplg_control_ops { * @token_list: List of token ID's that should be parsed for the widget * @token_list_size: number of elements in token_list * @bind_event: Function pointer for binding events to the widget + * @ipc_prepare: Optional op for preparing a widget for set up + * @ipc_unprepare: Optional op for unpreparing a widget */ struct sof_ipc_tplg_widget_ops { int (*ipc_setup)(struct snd_sof_widget *swidget); @@ -103,6 +141,11 @@ struct sof_ipc_tplg_widget_ops { int token_list_size; int (*bind_event)(struct snd_soc_component *scomp, struct snd_sof_widget *swidget, u16 event_type); + int (*ipc_prepare)(struct snd_sof_widget *swidget, + struct snd_pcm_hw_params *fe_params, + struct snd_sof_platform_stream_params *platform_params, + struct snd_pcm_hw_params *source_params, int dir); + void (*ipc_unprepare)(struct snd_sof_widget *swidget); }; /** @@ -112,6 +155,7 @@ struct sof_ipc_tplg_widget_ops { * initialized to 0. * @control: Pointer to the IPC-specific ops for topology kcontrol IO * @route_setup: Function pointer for setting up pipeline connections + * @route_free: Optional op for freeing pipeline connections. * @token_list: List of all tokens supported by the IPC version. The size of the token_list * array should be SOF_TOKEN_COUNT. The unused elements in the array will be * initialized to 0. @@ -124,11 +168,13 @@ struct sof_ipc_tplg_widget_ops { * @dai_get_clk: Function pointer for getting the DAI clock setting * @set_up_all_pipelines: Function pointer for setting up all topology pipelines * @tear_down_all_pipelines: Function pointer for tearing down all topology pipelines + * @parse_manifest: Optional function pointer for ipc4 specific parsing of topology manifest */ struct sof_ipc_tplg_ops { const struct sof_ipc_tplg_widget_ops *widget; const struct sof_ipc_tplg_control_ops *control; int (*route_setup)(struct snd_sof_dev *sdev, struct snd_sof_route *sroute); + int (*route_free)(struct snd_sof_dev *sdev, struct snd_sof_route *sroute); const struct sof_token_info *token_list; int (*control_setup)(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol); int (*control_free)(struct snd_sof_dev *sdev, struct snd_sof_control *scontrol); @@ -140,6 +186,8 @@ struct sof_ipc_tplg_ops { int (*dai_get_clk)(struct snd_sof_dev *sdev, struct snd_sof_dai *dai, int clk_type); int (*set_up_all_pipelines)(struct snd_sof_dev *sdev, bool verify); int (*tear_down_all_pipelines)(struct snd_sof_dev *sdev, bool verify); + int (*parse_manifest)(struct snd_soc_component *scomp, int index, + struct snd_soc_tplg_manifest *man); }; /** struct snd_sof_tuple - Tuple info @@ -180,6 +228,15 @@ enum sof_tokens { SOF_AFE_TOKENS, SOF_CORE_TOKENS, SOF_COMP_EXT_TOKENS, + SOF_IN_AUDIO_FORMAT_TOKENS, + SOF_OUT_AUDIO_FORMAT_TOKENS, + SOF_AUDIO_FORMAT_BUFFER_SIZE_TOKENS, + SOF_COPIER_GATEWAY_CFG_TOKENS, + SOF_COPIER_TOKENS, + SOF_AUDIO_FMT_NUM_TOKENS, + SOF_COPIER_FORMAT_TOKENS, + SOF_GAIN_TOKENS, + SOF_ACPDMIC_TOKENS, /* this should be the last */ SOF_TOKEN_COUNT, @@ -247,7 +304,6 @@ struct snd_sof_control { int max_volume_step; /* max volume step for volume_table */ int num_channels; unsigned int access; - u32 readback_offset; /* offset to mmapped data if used */ int info_type; int index; /* pipeline ID */ void *priv; /* private data copied from topology */ @@ -292,10 +348,24 @@ struct snd_sof_widget { struct snd_soc_component *scomp; int comp_id; int pipeline_id; + /* + * complete flag is used to indicate that pipeline set up is complete for scheduler type + * widgets. It is unused for all other widget types. + */ int complete; + /* + * the prepared flag is used to indicate that a widget has been prepared for getting set + * up in the DSP. + */ + bool prepared; int use_count; /* use_count will be protected by the PCM mutex held by the core */ int core; - int id; + int id; /* id is the DAPM widget type */ + /* + * Instance ID is set dynamically when the widget gets set up in the FW. It should be + * unique for each module type across all pipelines. This will not be used in SOF_IPC. + */ + int instance_id; /* * Flag indicating if the widget should be set up dynamically when a PCM is opened. @@ -310,6 +380,7 @@ struct snd_sof_widget { struct snd_soc_dapm_widget *widget; struct list_head list; /* list in sdev widget list */ struct snd_sof_widget *pipe_widget; + void *module_info; const guid_t uuid; @@ -403,8 +474,7 @@ struct snd_sof_pcm *snd_sof_find_spcm_dai(struct snd_soc_component *scomp, struct snd_soc_pcm_runtime *rtd) { struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); - - struct snd_sof_pcm *spcm = NULL; + struct snd_sof_pcm *spcm; list_for_each_entry(spcm, &sdev->pcm_list, list) { if (le32_to_cpu(spcm->pcm.dai_id) == rtd->dai_link->id) @@ -430,16 +500,10 @@ static inline void snd_sof_compr_fragment_elapsed(struct snd_compr_stream *cstre static inline void snd_sof_compr_init_elapsed_work(struct work_struct *work) { } #endif -/* - * Mixer IPC - */ -int snd_sof_ipc_set_get_comp_data(struct snd_sof_control *scontrol, bool set); - /* DAI link fixup */ int sof_pcm_dai_link_fixup(struct snd_soc_pcm_runtime *rtd, struct snd_pcm_hw_params *params); /* PM */ -int sof_set_hw_params_upon_resume(struct device *dev); bool snd_sof_stream_suspend_ignored(struct snd_sof_dev *sdev); bool snd_sof_dsp_only_d0i3_compatible_stream_active(struct snd_sof_dev *sdev); @@ -453,7 +517,10 @@ int sof_route_setup(struct snd_sof_dev *sdev, struct snd_soc_dapm_widget *wsourc struct snd_soc_dapm_widget *wsink); /* PCM */ -int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, int dir); +int sof_widget_list_setup(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, + struct snd_pcm_hw_params *fe_params, + struct snd_sof_platform_stream_params *platform_params, + int dir); int sof_widget_list_free(struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm, int dir); int sof_pcm_dsp_pcm_free(struct snd_pcm_substream *substream, struct snd_sof_dev *sdev, struct snd_sof_pcm *spcm); @@ -467,6 +534,5 @@ int get_token_uuid(void *elem, void *object, u32 offset); int sof_update_ipc_object(struct snd_soc_component *scomp, void *object, enum sof_tokens token_id, struct snd_sof_tuple *tuples, int num_tuples, size_t object_size, int token_instance_num); -int sof_pcm_setup_connected_widgets(struct snd_sof_dev *sdev, struct snd_soc_pcm_runtime *rtd, - struct snd_sof_pcm *spcm, int dir); +u32 vol_compute_gain(u32 value, int *tlv); #endif diff --git a/sound/soc/sof/sof-client-ipc-flood-test.c b/sound/soc/sof/sof-client-ipc-flood-test.c index db3a052c5d..4bdecd8024 100644 --- a/sound/soc/sof/sof-client-ipc-flood-test.c +++ b/sound/soc/sof/sof-client-ipc-flood-test.c @@ -217,10 +217,9 @@ static ssize_t sof_ipc_flood_dfs_write(struct file *file, const char __user *buf ipc_count = MAX_IPC_FLOOD_COUNT; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(dev, "debugfs write failed to resume %d\n", ret); - pm_runtime_put_noidle(dev); goto out; } diff --git a/sound/soc/sof/sof-client-ipc-msg-injector.c b/sound/soc/sof/sof-client-ipc-msg-injector.c index dba6cfd7db..752d532068 100644 --- a/sound/soc/sof/sof-client-ipc-msg-injector.c +++ b/sound/soc/sof/sof-client-ipc-msg-injector.c @@ -15,6 +15,7 @@ #include #include #include +#include #include "sof-client.h" @@ -22,6 +23,8 @@ struct sof_msg_inject_priv { struct dentry *dfs_file; + size_t max_msg_size; + enum sof_ipc_type ipc_type; void *tx_buffer; void *rx_buffer; @@ -66,44 +69,156 @@ static ssize_t sof_msg_inject_dfs_read(struct file *file, char __user *buffer, return count; } -static ssize_t sof_msg_inject_dfs_write(struct file *file, const char __user *buffer, - size_t count, loff_t *ppos) +static ssize_t sof_msg_inject_ipc4_dfs_read(struct file *file, + char __user *buffer, + size_t count, loff_t *ppos) { struct sof_client_dev *cdev = file->private_data; struct sof_msg_inject_priv *priv = cdev->data; - struct device *dev = &cdev->auxdev.dev; - int ret, err; - size_t size; + struct sof_ipc4_msg *ipc4_msg = priv->rx_buffer; + size_t header_size = sizeof(ipc4_msg->header_u64); + size_t remaining; - if (*ppos) + if (!ipc4_msg->header_u64 || !count || *ppos) return 0; - size = simple_write_to_buffer(priv->tx_buffer, SOF_IPC_MSG_MAX_SIZE, - ppos, buffer, count); - if (size != count) - return size > 0 ? -EFAULT : size; + /* we need space for the header at minimum (u64) */ + if (count < header_size) + return -ENOSPC; - ret = pm_runtime_get_sync(dev); + remaining = header_size; + + /* Only get large config have payload */ + if (SOF_IPC4_MSG_IS_MODULE_MSG(ipc4_msg->primary) && + (SOF_IPC4_MSG_TYPE_GET(ipc4_msg->primary) == SOF_IPC4_MOD_LARGE_CONFIG_GET)) + remaining += ipc4_msg->data_size; + + if (count > remaining) + count = remaining; + else if (count < remaining) + remaining = count; + + /* copy the header first */ + if (copy_to_user(buffer, &ipc4_msg->header_u64, header_size)) + return -EFAULT; + + *ppos += header_size; + remaining -= header_size; + + if (!remaining) + return count; + + if (remaining > ipc4_msg->data_size) + remaining = ipc4_msg->data_size; + + /* Copy the payload */ + if (copy_to_user(buffer + *ppos, ipc4_msg->data_ptr, remaining)) + return -EFAULT; + + *ppos += remaining; + return count; +} + +static int sof_msg_inject_send_message(struct sof_client_dev *cdev) +{ + struct sof_msg_inject_priv *priv = cdev->data; + struct device *dev = &cdev->auxdev.dev; + int ret, err; + + ret = pm_runtime_resume_and_get(dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(dev, "debugfs write failed to resume %d\n", ret); - pm_runtime_put_noidle(dev); return ret; } /* send the message */ - memset(priv->rx_buffer, 0, SOF_IPC_MSG_MAX_SIZE); ret = sof_client_ipc_tx_message(cdev, priv->tx_buffer, priv->rx_buffer, - SOF_IPC_MSG_MAX_SIZE); + priv->max_msg_size); + if (ret) + dev_err(dev, "IPC message send failed: %d\n", ret); + pm_runtime_mark_last_busy(dev); err = pm_runtime_put_autosuspend(dev); if (err < 0) dev_err_ratelimited(dev, "debugfs write failed to idle %d\n", err); - /* return size if test is successful */ - if (ret >= 0) - ret = size; - return ret; +} + +static ssize_t sof_msg_inject_dfs_write(struct file *file, const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct sof_client_dev *cdev = file->private_data; + struct sof_msg_inject_priv *priv = cdev->data; + ssize_t size; + int ret; + + if (*ppos) + return 0; + + size = simple_write_to_buffer(priv->tx_buffer, priv->max_msg_size, + ppos, buffer, count); + if (size < 0) + return size; + if (size != count) + return -EFAULT; + + memset(priv->rx_buffer, 0, priv->max_msg_size); + + ret = sof_msg_inject_send_message(cdev); + + /* return the error code if test failed */ + if (ret < 0) + size = ret; + + return size; +}; + +static ssize_t sof_msg_inject_ipc4_dfs_write(struct file *file, + const char __user *buffer, + size_t count, loff_t *ppos) +{ + struct sof_client_dev *cdev = file->private_data; + struct sof_msg_inject_priv *priv = cdev->data; + struct sof_ipc4_msg *ipc4_msg = priv->tx_buffer; + size_t data_size; + int ret; + + if (*ppos) + return 0; + + if (count < sizeof(ipc4_msg->header_u64)) + return -EINVAL; + + /* copy the header first */ + if (copy_from_user(&ipc4_msg->header_u64, buffer, + sizeof(ipc4_msg->header_u64))) + return -EFAULT; + + data_size = count - sizeof(ipc4_msg->header_u64); + if (data_size > priv->max_msg_size) + return -EINVAL; + + /* Copy the payload */ + if (copy_from_user(ipc4_msg->data_ptr, + buffer + sizeof(ipc4_msg->header_u64), data_size)) + return -EFAULT; + + ipc4_msg->data_size = data_size; + + /* Initialize the reply storage */ + ipc4_msg = priv->rx_buffer; + ipc4_msg->header_u64 = 0; + ipc4_msg->data_size = priv->max_msg_size; + memset(ipc4_msg->data_ptr, 0, priv->max_msg_size); + + ret = sof_msg_inject_send_message(cdev); + + /* return the error code if test failed */ + if (ret < 0) + return ret; + + return count; }; static int sof_msg_inject_dfs_release(struct inode *inode, struct file *file) @@ -123,28 +238,61 @@ static const struct file_operations sof_msg_inject_fops = { .owner = THIS_MODULE, }; +static const struct file_operations sof_msg_inject_ipc4_fops = { + .open = sof_msg_inject_dfs_open, + .read = sof_msg_inject_ipc4_dfs_read, + .write = sof_msg_inject_ipc4_dfs_write, + .llseek = default_llseek, + .release = sof_msg_inject_dfs_release, + + .owner = THIS_MODULE, +}; + static int sof_msg_inject_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id) { struct sof_client_dev *cdev = auxiliary_dev_to_sof_client_dev(auxdev); struct dentry *debugfs_root = sof_client_get_debugfs_root(cdev); + static const struct file_operations *fops; struct device *dev = &auxdev->dev; struct sof_msg_inject_priv *priv; + size_t alloc_size; /* allocate memory for client data */ priv = devm_kzalloc(&auxdev->dev, sizeof(*priv), GFP_KERNEL); if (!priv) return -ENOMEM; - priv->tx_buffer = devm_kmalloc(dev, SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); - priv->rx_buffer = devm_kzalloc(dev, SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); + priv->ipc_type = sof_client_get_ipc_type(cdev); + priv->max_msg_size = sof_client_get_ipc_max_payload_size(cdev); + alloc_size = priv->max_msg_size; + + if (priv->ipc_type == SOF_INTEL_IPC4) + alloc_size += sizeof(struct sof_ipc4_msg); + + priv->tx_buffer = devm_kmalloc(dev, alloc_size, GFP_KERNEL); + priv->rx_buffer = devm_kzalloc(dev, alloc_size, GFP_KERNEL); if (!priv->tx_buffer || !priv->rx_buffer) return -ENOMEM; + if (priv->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_msg *ipc4_msg; + + ipc4_msg = priv->tx_buffer; + ipc4_msg->data_ptr = priv->tx_buffer + sizeof(struct sof_ipc4_msg); + + ipc4_msg = priv->rx_buffer; + ipc4_msg->data_ptr = priv->rx_buffer + sizeof(struct sof_ipc4_msg); + + fops = &sof_msg_inject_ipc4_fops; + } else { + fops = &sof_msg_inject_fops; + } + cdev->data = priv; priv->dfs_file = debugfs_create_file("ipc_msg_inject", 0644, debugfs_root, - cdev, &sof_msg_inject_fops); + cdev, fops); /* enable runtime PM */ pm_runtime_set_autosuspend_delay(dev, SOF_IPC_CLIENT_SUSPEND_DELAY_MS); diff --git a/sound/soc/sof/sof-client-probes.c b/sound/soc/sof/sof-client-probes.c index 797dedb261..eb246b8234 100644 --- a/sound/soc/sof/sof-client-probes.c +++ b/sound/soc/sof/sof-client-probes.c @@ -132,6 +132,7 @@ static int sof_probes_deinit(struct sof_client_dev *cdev) static int sof_probes_info(struct sof_client_dev *cdev, unsigned int cmd, void **params, size_t *num_params) { + size_t max_msg_size = sof_client_get_ipc_max_payload_size(cdev); struct sof_ipc_probe_info_params msg = {{{0}}}; struct sof_ipc_probe_info_params *reply; size_t bytes; @@ -140,13 +141,13 @@ static int sof_probes_info(struct sof_client_dev *cdev, unsigned int cmd, *params = NULL; *num_params = 0; - reply = kzalloc(SOF_IPC_MSG_MAX_SIZE, GFP_KERNEL); + reply = kzalloc(max_msg_size, GFP_KERNEL); if (!reply) return -ENOMEM; msg.rhdr.hdr.size = sizeof(msg); msg.rhdr.hdr.cmd = SOF_IPC_GLB_PROBE | cmd; - ret = sof_client_ipc_tx_message(cdev, &msg, reply, SOF_IPC_MSG_MAX_SIZE); + ret = sof_client_ipc_tx_message(cdev, &msg, reply, max_msg_size); if (ret < 0 || reply->rhdr.error < 0) goto exit; @@ -269,9 +270,9 @@ static int sof_probes_compr_startup(struct snd_compr_stream *cstream, if (ret) return ret; - ret = ops->assign(cdev, cstream, dai, &priv->extractor_stream_tag); + ret = ops->startup(cdev, cstream, dai, &priv->extractor_stream_tag); if (ret) { - dev_err(dai->dev, "Failed to assign probe stream: %d\n", ret); + dev_err(dai->dev, "Failed to startup probe stream: %d\n", ret); priv->extractor_stream_tag = SOF_PROBES_INVALID_NODE_ID; sof_client_core_module_put(cdev); } @@ -309,7 +310,7 @@ static int sof_probes_compr_shutdown(struct snd_compr_stream *cstream, priv->extractor_stream_tag = SOF_PROBES_INVALID_NODE_ID; snd_compr_free_pages(cstream); - ret = ops->free(cdev, cstream, dai); + ret = ops->shutdown(cdev, cstream, dai); sof_client_core_module_put(cdev); @@ -503,10 +504,9 @@ static ssize_t sof_probes_dfs_points_read(struct file *file, char __user *to, if (!buf) return -ENOMEM; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(dev, "debugfs read failed to resume %d\n", ret); - pm_runtime_put_noidle(dev); goto exit; } @@ -568,10 +568,9 @@ sof_probes_dfs_points_write(struct file *file, const char __user *from, desc = (struct sof_probe_point_desc *)tkns; - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0 && ret != -EACCES) { dev_err_ratelimited(dev, "debugfs write failed to resume %d\n", ret); - pm_runtime_put_noidle(dev); goto exit; } @@ -621,10 +620,9 @@ sof_probes_dfs_points_remove_write(struct file *file, const char __user *from, goto exit; } - ret = pm_runtime_get_sync(dev); + ret = pm_runtime_resume_and_get(dev); if (ret < 0) { dev_err_ratelimited(dev, "debugfs write failed to resume %d\n", ret); - pm_runtime_put_noidle(dev); goto exit; } @@ -669,6 +667,7 @@ static const struct snd_soc_component_driver sof_probes_component = { .name = "sof-probes-component", .compress_ops = &sof_probes_compressed_ops, .module_get_upon_open = 1, + .legacy_dai_naming = 1, }; SND_SOC_DAILINK_DEF(dummy, DAILINK_COMP_ARRAY(COMP_DUMMY())); @@ -695,6 +694,10 @@ static int sof_probes_client_probe(struct auxiliary_device *auxdev, if (!sof_probes_enabled) return -ENXIO; + /* only ipc3 is supported */ + if (sof_client_get_ipc_type(cdev) != SOF_IPC) + return -ENXIO; + if (!dev->platform_data) { dev_err(dev, "missing platform data\n"); return -ENODEV; @@ -706,7 +709,7 @@ static int sof_probes_client_probe(struct auxiliary_device *auxdev, ops = dev->platform_data; - if (!ops->assign || !ops->free || !ops->set_params || !ops->trigger || + if (!ops->startup || !ops->shutdown || !ops->set_params || !ops->trigger || !ops->pointer) { dev_err(dev, "missing platform callback(s)\n"); return -ENODEV; diff --git a/sound/soc/sof/sof-client-probes.h b/sound/soc/sof/sof-client-probes.h index 0f9ed4569f..9e43f3c444 100644 --- a/sound/soc/sof/sof-client-probes.h +++ b/sound/soc/sof/sof-client-probes.h @@ -14,10 +14,10 @@ struct snd_soc_dai; * DSP and host, like HDA. */ struct sof_probes_host_ops { - int (*assign)(struct sof_client_dev *cdev, struct snd_compr_stream *cstream, - struct snd_soc_dai *dai, u32 *stream_id); - int (*free)(struct sof_client_dev *cdev, struct snd_compr_stream *cstream, - struct snd_soc_dai *dai); + int (*startup)(struct sof_client_dev *cdev, struct snd_compr_stream *cstream, + struct snd_soc_dai *dai, u32 *stream_id); + int (*shutdown)(struct sof_client_dev *cdev, struct snd_compr_stream *cstream, + struct snd_soc_dai *dai); int (*set_params)(struct sof_client_dev *cdev, struct snd_compr_stream *cstream, struct snd_compr_params *params, struct snd_soc_dai *dai); diff --git a/sound/soc/sof/sof-client.c b/sound/soc/sof/sof-client.c index 686ad0c3bb..125aa21371 100644 --- a/sound/soc/sof/sof-client.c +++ b/sound/soc/sof/sof-client.c @@ -12,6 +12,7 @@ #include #include #include +#include #include "ops.h" #include "sof-client.h" #include "sof-priv.h" @@ -72,6 +73,9 @@ static int sof_register_ipc_flood_test(struct snd_sof_dev *sdev) int ret = 0; int i; + if (sdev->pdata->ipc_type != SOF_IPC) + return 0; + for (i = 0; i < CONFIG_SND_SOC_SOF_DEBUG_IPC_FLOOD_TEST_NUM; i++) { ret = sof_client_dev_register(sdev, "ipc_flood", i, NULL, 0); if (ret < 0) @@ -245,10 +249,19 @@ EXPORT_SYMBOL_NS_GPL(sof_client_dev_unregister, SND_SOC_SOF_CLIENT); int sof_client_ipc_tx_message(struct sof_client_dev *cdev, void *ipc_msg, void *reply_data, size_t reply_bytes) { - struct sof_ipc_cmd_hdr *hdr = ipc_msg; + if (cdev->sdev->pdata->ipc_type == SOF_IPC) { + struct sof_ipc_cmd_hdr *hdr = ipc_msg; - return sof_ipc_tx_message(cdev->sdev->ipc, hdr->cmd, ipc_msg, hdr->size, - reply_data, reply_bytes); + return sof_ipc_tx_message(cdev->sdev->ipc, ipc_msg, hdr->size, + reply_data, reply_bytes); + } else if (cdev->sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_msg *msg = ipc_msg; + + return sof_ipc_tx_message(cdev->sdev->ipc, ipc_msg, msg->data_size, + reply_data, reply_bytes); + } + + return -EINVAL; } EXPORT_SYMBOL_NS_GPL(sof_client_ipc_tx_message, SND_SOC_SOF_CLIENT); @@ -319,6 +332,22 @@ const struct sof_ipc_fw_version *sof_client_get_fw_version(struct sof_client_dev } EXPORT_SYMBOL_NS_GPL(sof_client_get_fw_version, SND_SOC_SOF_CLIENT); +size_t sof_client_get_ipc_max_payload_size(struct sof_client_dev *cdev) +{ + struct snd_sof_dev *sdev = sof_client_dev_to_sof_dev(cdev); + + return sdev->ipc->max_payload_size; +} +EXPORT_SYMBOL_NS_GPL(sof_client_get_ipc_max_payload_size, SND_SOC_SOF_CLIENT); + +enum sof_ipc_type sof_client_get_ipc_type(struct sof_client_dev *cdev) +{ + struct snd_sof_dev *sdev = sof_client_dev_to_sof_dev(cdev); + + return sdev->pdata->ipc_type; +} +EXPORT_SYMBOL_NS_GPL(sof_client_get_ipc_type, SND_SOC_SOF_CLIENT); + /* module refcount management of SOF core */ int sof_client_core_module_get(struct sof_client_dev *cdev) { @@ -342,9 +371,22 @@ EXPORT_SYMBOL_NS_GPL(sof_client_core_module_put, SND_SOC_SOF_CLIENT); /* IPC event handling */ void sof_client_ipc_rx_dispatcher(struct snd_sof_dev *sdev, void *msg_buf) { - struct sof_ipc_cmd_hdr *hdr = msg_buf; - u32 msg_type = hdr->cmd & SOF_GLB_TYPE_MASK; struct sof_ipc_event_entry *event; + u32 msg_type; + + if (sdev->pdata->ipc_type == SOF_IPC) { + struct sof_ipc_cmd_hdr *hdr = msg_buf; + + msg_type = hdr->cmd & SOF_GLB_TYPE_MASK; + } else if (sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + struct sof_ipc4_msg *msg = msg_buf; + + msg_type = SOF_IPC4_NOTIFICATION_TYPE_GET(msg->primary); + } else { + dev_dbg_once(sdev->dev, "Not supported IPC version: %d\n", + sdev->pdata->ipc_type); + return; + } mutex_lock(&sdev->client_event_handler_mutex); @@ -363,9 +405,21 @@ int sof_client_register_ipc_rx_handler(struct sof_client_dev *cdev, struct snd_sof_dev *sdev = sof_client_dev_to_sof_dev(cdev); struct sof_ipc_event_entry *event; - if (!callback || !(ipc_msg_type & SOF_GLB_TYPE_MASK)) + if (!callback) return -EINVAL; + if (cdev->sdev->pdata->ipc_type == SOF_IPC) { + if (!(ipc_msg_type & SOF_GLB_TYPE_MASK)) + return -EINVAL; + } else if (cdev->sdev->pdata->ipc_type == SOF_INTEL_IPC4) { + if (!(ipc_msg_type & SOF_IPC4_NOTIFICATION_TYPE_MASK)) + return -EINVAL; + } else { + dev_warn(sdev->dev, "%s: Not supported IPC version: %d\n", + __func__, sdev->pdata->ipc_type); + return -EINVAL; + } + event = kmalloc(sizeof(*event), GFP_KERNEL); if (!event) return -ENOMEM; diff --git a/sound/soc/sof/sof-client.h b/sound/soc/sof/sof-client.h index 4b6394b4c6..46b215d920 100644 --- a/sound/soc/sof/sof-client.h +++ b/sound/soc/sof/sof-client.h @@ -41,6 +41,8 @@ int sof_client_ipc_tx_message(struct sof_client_dev *cdev, void *ipc_msg, struct dentry *sof_client_get_debugfs_root(struct sof_client_dev *cdev); struct device *sof_client_get_dma_dev(struct sof_client_dev *cdev); const struct sof_ipc_fw_version *sof_client_get_fw_version(struct sof_client_dev *cdev); +size_t sof_client_get_ipc_max_payload_size(struct sof_client_dev *cdev); +enum sof_ipc_type sof_client_get_ipc_type(struct sof_client_dev *cdev); /* module refcount management of SOF core */ int sof_client_core_module_get(struct sof_client_dev *cdev); diff --git a/sound/soc/sof/sof-of-dev.c b/sound/soc/sof/sof-of-dev.c index e3718638f9..53faeccedd 100644 --- a/sound/soc/sof/sof-of-dev.c +++ b/sound/soc/sof/sof-of-dev.c @@ -64,17 +64,17 @@ int sof_of_probe(struct platform_device *pdev) sof_pdata->desc = desc; sof_pdata->dev = &pdev->dev; - sof_pdata->fw_filename = desc->default_fw_filename; + sof_pdata->fw_filename = desc->default_fw_filename[SOF_IPC]; if (fw_path) sof_pdata->fw_filename_prefix = fw_path; else - sof_pdata->fw_filename_prefix = sof_pdata->desc->default_fw_path; + sof_pdata->fw_filename_prefix = sof_pdata->desc->default_fw_path[SOF_IPC]; if (tplg_path) sof_pdata->tplg_filename_prefix = tplg_path; else - sof_pdata->tplg_filename_prefix = sof_pdata->desc->default_tplg_path; + sof_pdata->tplg_filename_prefix = sof_pdata->desc->default_tplg_path[SOF_IPC]; /* set callback to be called on successful device probe to enable runtime_pm */ sof_pdata->sof_probe_complete = sof_of_probe_complete; @@ -95,4 +95,10 @@ int sof_of_remove(struct platform_device *pdev) } EXPORT_SYMBOL(sof_of_remove); +void sof_of_shutdown(struct platform_device *pdev) +{ + snd_sof_device_shutdown(&pdev->dev); +} +EXPORT_SYMBOL(sof_of_shutdown); + MODULE_LICENSE("Dual BSD/GPL"); diff --git a/sound/soc/sof/sof-of-dev.h b/sound/soc/sof/sof-of-dev.h index 4e0f6588da..fd950a222b 100644 --- a/sound/soc/sof/sof-of-dev.h +++ b/sound/soc/sof/sof-of-dev.h @@ -13,5 +13,6 @@ extern const struct dev_pm_ops sof_of_pm; int sof_of_probe(struct platform_device *pdev); int sof_of_remove(struct platform_device *pdev); +void sof_of_shutdown(struct platform_device *pdev); #endif diff --git a/sound/soc/sof/sof-pci-dev.c b/sound/soc/sof/sof-pci-dev.c index 7fa2649e56..d627092b39 100644 --- a/sound/soc/sof/sof-pci-dev.c +++ b/sound/soc/sof/sof-pci-dev.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -23,21 +24,34 @@ static char *fw_path; module_param(fw_path, charp, 0444); MODULE_PARM_DESC(fw_path, "alternate path for SOF firmware."); +static char *fw_filename; +module_param(fw_filename, charp, 0444); +MODULE_PARM_DESC(fw_filename, "alternate filename for SOF firmware."); + static char *tplg_path; module_param(tplg_path, charp, 0444); MODULE_PARM_DESC(tplg_path, "alternate path for SOF topology."); +static char *tplg_filename; +module_param(tplg_filename, charp, 0444); +MODULE_PARM_DESC(tplg_filename, "alternate filename for SOF topology."); + static int sof_pci_debug; module_param_named(sof_pci_debug, sof_pci_debug, int, 0444); MODULE_PARM_DESC(sof_pci_debug, "SOF PCI debug options (0x0 all off)"); -static const char *sof_override_tplg_name; +static int sof_pci_ipc_type = -1; +module_param_named(ipc_type, sof_pci_ipc_type, int, 0444); +MODULE_PARM_DESC(ipc_type, "SOF IPC type (0): SOF, (1) Intel CAVS"); + +static const char *sof_dmi_override_tplg_name; +static bool sof_dmi_use_community_key; #define SOF_PCI_DISABLE_PM_RUNTIME BIT(0) static int sof_tplg_cb(const struct dmi_system_id *id) { - sof_override_tplg_name = id->driver_data; + sof_dmi_override_tplg_name = id->driver_data; return 1; } @@ -94,15 +108,35 @@ static const struct dmi_system_id sof_tplg_table[] = { {} }; +/* all Up boards use the community key */ +static int up_use_community_key(const struct dmi_system_id *id) +{ + sof_dmi_use_community_key = true; + return 1; +} + +/* + * For ApolloLake Chromebooks we want to force the use of the Intel production key. + * All newer platforms use the community key + */ +static int chromebook_use_community_key(const struct dmi_system_id *id) +{ + if (!soc_intel_is_apl()) + sof_dmi_use_community_key = true; + return 1; +} + static const struct dmi_system_id community_key_platforms[] = { { .ident = "Up boards", + .callback = up_use_community_key, .matches = { DMI_MATCH(DMI_SYS_VENDOR, "AAEON"), } }, { .ident = "Google Chromebooks", + .callback = chromebook_use_community_key, .matches = { DMI_MATCH(DMI_SYS_VENDOR, "Google"), } @@ -178,7 +212,36 @@ int sof_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) sof_pdata->name = pci_name(pci); sof_pdata->desc = desc; sof_pdata->dev = dev; - sof_pdata->fw_filename = desc->default_fw_filename; + + sof_pdata->ipc_type = desc->ipc_default; + + if (sof_pci_ipc_type < 0) { + sof_pdata->ipc_type = desc->ipc_default; + } else { + dev_info(dev, "overriding default IPC %d to requested %d\n", + desc->ipc_default, sof_pci_ipc_type); + if (sof_pci_ipc_type >= SOF_IPC_TYPE_COUNT) { + dev_err(dev, "invalid request value %d\n", sof_pci_ipc_type); + ret = -EINVAL; + goto out; + } + if (!(BIT(sof_pci_ipc_type) & desc->ipc_supported_mask)) { + dev_err(dev, "invalid request value %d, supported mask is %#x\n", + sof_pci_ipc_type, desc->ipc_supported_mask); + ret = -EINVAL; + goto out; + } + sof_pdata->ipc_type = sof_pci_ipc_type; + } + + if (fw_filename) { + sof_pdata->fw_filename = fw_filename; + + dev_dbg(dev, "Module parameter used, changed fw filename to %s\n", + sof_pdata->fw_filename); + } else { + sof_pdata->fw_filename = desc->default_fw_filename[sof_pdata->ipc_type]; + } /* * for platforms using the SOF community key, change the @@ -195,10 +258,10 @@ int sof_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) "Module parameter used, changed fw path to %s\n", sof_pdata->fw_filename_prefix); - } else if (dmi_check_system(community_key_platforms)) { + } else if (dmi_check_system(community_key_platforms) && sof_dmi_use_community_key) { sof_pdata->fw_filename_prefix = devm_kasprintf(dev, GFP_KERNEL, "%s/%s", - sof_pdata->desc->default_fw_path, + sof_pdata->desc->default_fw_path[sof_pdata->ipc_type], "community"); dev_dbg(dev, @@ -206,24 +269,37 @@ int sof_pci_probe(struct pci_dev *pci, const struct pci_device_id *pci_id) sof_pdata->fw_filename_prefix); } else { sof_pdata->fw_filename_prefix = - sof_pdata->desc->default_fw_path; + sof_pdata->desc->default_fw_path[sof_pdata->ipc_type]; } if (tplg_path) sof_pdata->tplg_filename_prefix = tplg_path; else sof_pdata->tplg_filename_prefix = - sof_pdata->desc->default_tplg_path; + sof_pdata->desc->default_tplg_path[sof_pdata->ipc_type]; - dmi_check_system(sof_tplg_table); - if (sof_override_tplg_name) - sof_pdata->tplg_filename = sof_override_tplg_name; + /* + * the topology filename will be provided in the machine descriptor, unless + * it is overridden by a module parameter or DMI quirk. + */ + if (tplg_filename) { + sof_pdata->tplg_filename = tplg_filename; + + dev_dbg(dev, "Module parameter used, changed tplg filename to %s\n", + sof_pdata->tplg_filename); + } else { + dmi_check_system(sof_tplg_table); + if (sof_dmi_override_tplg_name) + sof_pdata->tplg_filename = sof_dmi_override_tplg_name; + } /* set callback to be called on successful device probe to enable runtime_pm */ sof_pdata->sof_probe_complete = sof_pci_probe_complete; /* call sof helper for DSP hardware probe */ ret = snd_sof_device_probe(dev, sof_pdata); + +out: if (ret) pci_release_regions(pci); diff --git a/sound/soc/sof/sof-priv.h b/sound/soc/sof/sof-priv.h index 0d9b640ae2..8235830862 100644 --- a/sound/soc/sof/sof-priv.h +++ b/sound/soc/sof/sof-priv.h @@ -37,6 +37,12 @@ #define SOF_DBG_IGNORE_D3_PERSISTENT BIT(7) /* ignore the DSP D3 persistent capability * and always download firmware upon D3 exit */ +#define SOF_DBG_PRINT_DMA_POSITION_UPDATE_LOGS BIT(8) /* print DMA position updates + * in dmesg logs + */ +#define SOF_DBG_PRINT_IPC_SUCCESS_LOGS BIT(9) /* print IPC success + * in dmesg logs + */ /* Flag definitions used for controlling the DSP dump behavior */ #define SOF_DBG_DUMP_REGS BIT(0) @@ -85,6 +91,8 @@ enum sof_system_suspend_state { SOF_SUSPEND_NONE = 0, SOF_SUSPEND_S0IX, SOF_SUSPEND_S3, + SOF_SUSPEND_S4, + SOF_SUSPEND_S5, }; enum sof_dfsentry_type { @@ -118,6 +126,7 @@ struct snd_sof_platform_stream_params { bool use_phy_address; u32 phy_addr; bool no_ipc_position; + bool cont_update_posn; }; /* @@ -181,11 +190,6 @@ struct snd_sof_dsp_ops { int (*load_firmware)(struct snd_sof_dev *sof_dev); /* mandatory */ int (*load_module)(struct snd_sof_dev *sof_dev, struct snd_sof_mod_hdr *hdr); /* optional */ - /* - * FW ready checks for ABI compatibility and creates - * memory windows at first boot - */ - int (*fw_ready)(struct snd_sof_dev *sdev, u32 msg_id); /* mandatory */ /* connect pcm substream to a host stream */ int (*pcm_open)(struct snd_sof_dev *sdev, @@ -259,8 +263,9 @@ struct snd_sof_dsp_ops { size_t size, const char *name, enum sof_debugfs_access_type access_type); /* optional */ - /* host DMA trace initialization */ + /* host DMA trace (IPC3) */ int (*trace_init)(struct snd_sof_dev *sdev, + struct snd_dma_buffer *dmatb, struct sof_ipc_dma_trace_params_ext *dtrace_params); /* optional */ int (*trace_release)(struct snd_sof_dev *sdev); /* optional */ int (*trace_trigger)(struct snd_sof_dev *sdev, @@ -349,25 +354,64 @@ struct snd_sof_mailbox { /* IPC message descriptor for host <-> DSP IO */ struct snd_sof_ipc_msg { /* message data */ - u32 header; void *msg_data; void *reply_data; size_t msg_size; size_t reply_size; int reply_error; + /* notification, firmware initiated messages */ + void *rx_data; + wait_queue_head_t waitq; bool ipc_complete; }; +/** + * struct sof_ipc_fw_tracing_ops - IPC-specific firmware tracing ops + * @init: Function pointer for initialization of the tracing + * @free: Optional function pointer for freeing of the tracing + * @fw_crashed: Optional function pointer to notify the tracing of a firmware crash + * @suspend: Function pointer for system/runtime suspend + * @resume: Function pointer for system/runtime resume + */ +struct sof_ipc_fw_tracing_ops { + int (*init)(struct snd_sof_dev *sdev); + void (*free)(struct snd_sof_dev *sdev); + void (*fw_crashed)(struct snd_sof_dev *sdev); + void (*suspend)(struct snd_sof_dev *sdev, pm_message_t pm_state); + int (*resume)(struct snd_sof_dev *sdev); +}; + /** * struct sof_ipc_pm_ops - IPC-specific PM ops - * @ctx_save: Function pointer for context save - * @ctx_restore: Function pointer for context restore + * @ctx_save: Optional function pointer for context save + * @ctx_restore: Optional function pointer for context restore + * @set_core_state: Optional function pointer for turning on/off a DSP core */ struct sof_ipc_pm_ops { int (*ctx_save)(struct snd_sof_dev *sdev); int (*ctx_restore)(struct snd_sof_dev *sdev); + int (*set_core_state)(struct snd_sof_dev *sdev, int core_idx, bool on); +}; + +/** + * struct sof_ipc_fw_loader_ops - IPC/FW-specific loader ops + * @validate: Function pointer for validating the firmware image + * @parse_ext_manifest: Function pointer for parsing the manifest of the firmware + * @load_fw_to_dsp: Optional function pointer for loading the firmware to the + * DSP. + * The function implements generic, hardware independent way + * of loading the initial firmware and its modules (if any). + * @query_fw_configuration: Optional function pointer to query information and + * configuration from the booted firmware. + * Executed after the first successful firmware boot. + */ +struct sof_ipc_fw_loader_ops { + int (*validate)(struct snd_sof_dev *sdev); + size_t (*parse_ext_manifest)(struct snd_sof_dev *sdev); + int (*load_fw_to_dsp)(struct snd_sof_dev *sdev); + int (*query_fw_configuration)(struct snd_sof_dev *sdev); }; struct sof_ipc_tplg_ops; @@ -378,11 +422,36 @@ struct sof_ipc_pcm_ops; * @tplg: Pointer to IPC-specific topology ops * @pm: Pointer to PM ops * @pcm: Pointer to PCM ops + * @fw_loader: Pointer to Firmware Loader ops + * @fw_tracing: Pointer to Firmware tracing ops + * + * @tx_msg: Function pointer for sending a 'short' IPC message + * @set_get_data: Function pointer for set/get data ('large' IPC message). This + * function may split up the 'large' message and use the @tx_msg + * path to transfer individual chunks, or use other means to transfer + * the message. + * @get_reply: Function pointer for fetching the reply to + * sdev->ipc->msg.reply_data + * @rx_msg: Function pointer for handling a received message + * + * Note: both @tx_msg and @set_get_data considered as TX functions and they are + * serialized for the duration of the instructed transfer. A large message sent + * via @set_get_data is a single transfer even if at the hardware level it is + * handled with multiple chunks. */ struct sof_ipc_ops { const struct sof_ipc_tplg_ops *tplg; const struct sof_ipc_pm_ops *pm; const struct sof_ipc_pcm_ops *pcm; + const struct sof_ipc_fw_loader_ops *fw_loader; + const struct sof_ipc_fw_tracing_ops *fw_tracing; + + int (*tx_msg)(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes, bool no_pm); + int (*set_get_data)(struct snd_sof_dev *sdev, void *data, size_t data_bytes, + bool set); + int (*get_reply)(struct snd_sof_dev *sdev); + void (*rx_msg)(struct snd_sof_dev *sdev); }; /* SOF generic IPC data */ @@ -394,6 +463,9 @@ struct snd_sof_ipc { /* disables further sending of ipc's */ bool disable_ipc_tx; + /* Maximum allowed size of a single IPC message/reply */ + size_t max_payload_size; + struct snd_sof_ipc_msg msg; /* IPC ops based on version */ @@ -457,8 +529,6 @@ struct snd_sof_dev { bool ipc_dump_printed; /* firmware loader */ - struct snd_dma_buffer dmab; - struct snd_dma_buffer dmab_bdl; struct sof_ipc_fw_ready fw_ready; struct sof_ipc_fw_version fw_version; struct sof_ipc_cc_version *cc_version; @@ -473,6 +543,7 @@ struct snd_sof_dev { struct list_head route_list; struct snd_soc_component *component; u32 enabled_cores_mask; /* keep track of enabled cores */ + bool led_present; /* FW configuration */ struct sof_ipc_window *info_window; @@ -481,16 +552,9 @@ struct snd_sof_dev { int ipc_timeout; int boot_timeout; - /* DMA for Trace */ - struct snd_dma_buffer dmatb; - struct snd_dma_buffer dmatp; - int dma_trace_pages; - wait_queue_head_t trace_sleep; - u32 host_offset; - bool dtrace_is_supported; /* set with Kconfig or module parameter */ - bool dtrace_is_enabled; - bool dtrace_error; - bool dtrace_draining; + /* firmwre tracing */ + bool fw_trace_is_supported; /* set with Kconfig or module parameter */ + void *fw_trace_data; /* private data used by firmware tracing implementation */ bool msi_enabled; @@ -564,8 +628,6 @@ extern struct snd_compress_ops sof_compressed_ops; int snd_sof_load_firmware_raw(struct snd_sof_dev *sdev); int snd_sof_load_firmware_memcpy(struct snd_sof_dev *sdev); int snd_sof_run_firmware(struct snd_sof_dev *sdev); -int snd_sof_parse_module_memcpy(struct snd_sof_dev *sdev, - struct snd_sof_mod_hdr *module); void snd_sof_fw_unload(struct snd_sof_dev *sdev); /* @@ -575,15 +637,17 @@ struct snd_sof_ipc *snd_sof_ipc_init(struct snd_sof_dev *sdev); void snd_sof_ipc_free(struct snd_sof_dev *sdev); void snd_sof_ipc_get_reply(struct snd_sof_dev *sdev); void snd_sof_ipc_reply(struct snd_sof_dev *sdev, u32 msg_id); -void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev); -int snd_sof_ipc_valid(struct snd_sof_dev *sdev); -int sof_ipc_tx_message(struct snd_sof_ipc *ipc, u32 header, - void *msg_data, size_t msg_bytes, void *reply_data, - size_t reply_bytes); -int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, u32 header, - void *msg_data, size_t msg_bytes, +static inline void snd_sof_ipc_msgs_rx(struct snd_sof_dev *sdev) +{ + sdev->ipc->ops->rx_msg(sdev); +} +int sof_ipc_tx_message(struct snd_sof_ipc *ipc, void *msg_data, size_t msg_bytes, + void *reply_data, size_t reply_bytes); +int sof_ipc_tx_message_no_pm(struct snd_sof_ipc *ipc, void *msg_data, size_t msg_bytes, void *reply_data, size_t reply_bytes); -int sof_ipc_init_msg_memory(struct snd_sof_dev *sdev); +int sof_ipc_send_msg(struct snd_sof_dev *sdev, void *msg_data, size_t msg_bytes, + size_t reply_bytes); + static inline void snd_sof_ipc_process_reply(struct snd_sof_dev *sdev, u32 msg_id) { snd_sof_ipc_get_reply(sdev); @@ -593,27 +657,26 @@ static inline void snd_sof_ipc_process_reply(struct snd_sof_dev *sdev, u32 msg_i /* * Trace/debug */ -int snd_sof_init_trace(struct snd_sof_dev *sdev); -void snd_sof_release_trace(struct snd_sof_dev *sdev); -void snd_sof_free_trace(struct snd_sof_dev *sdev); int snd_sof_dbg_init(struct snd_sof_dev *sdev); void snd_sof_free_debug(struct snd_sof_dev *sdev); int snd_sof_debugfs_buf_item(struct snd_sof_dev *sdev, void *base, size_t size, const char *name, mode_t mode); -int snd_sof_trace_update_pos(struct snd_sof_dev *sdev, - struct sof_ipc_dma_trace_posn *posn); -void snd_sof_trace_notify_for_error(struct snd_sof_dev *sdev); void sof_print_oops_and_stack(struct snd_sof_dev *sdev, const char *level, u32 panic_code, u32 tracep_code, void *oops, struct sof_ipc_panic_info *panic_info, void *stack, size_t stack_words); -int snd_sof_init_trace_ipc(struct snd_sof_dev *sdev); -void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev); +void snd_sof_handle_fw_exception(struct snd_sof_dev *sdev, const char *msg); int snd_sof_dbg_memory_info_init(struct snd_sof_dev *sdev); int snd_sof_debugfs_add_region_item_iomem(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_type, u32 offset, size_t size, const char *name, enum sof_debugfs_access_type access_type); +/* Firmware tracing */ +int sof_fw_trace_init(struct snd_sof_dev *sdev); +void sof_fw_trace_free(struct snd_sof_dev *sdev); +void sof_fw_trace_fw_crashed(struct snd_sof_dev *sdev); +void sof_fw_trace_suspend(struct snd_sof_dev *sdev, pm_message_t pm_state); +int sof_fw_trace_resume(struct snd_sof_dev *sdev); /* * DSP Architectures. @@ -654,8 +717,6 @@ int sof_block_write(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_type, int sof_block_read(struct snd_sof_dev *sdev, enum snd_sof_fw_blk_type blk_type, u32 offset, void *dest, size_t size); -int sof_fw_ready(struct snd_sof_dev *sdev, u32 msg_id); - int sof_ipc_msg_data(struct snd_sof_dev *sdev, struct snd_pcm_substream *substream, void *p, size_t sz); @@ -721,4 +782,8 @@ static inline int sof_resume_clients(struct snd_sof_dev *sdev) } #endif /* CONFIG_SND_SOC_SOF_CLIENT */ +/* Main ops for IPC implementations */ +extern const struct sof_ipc_ops ipc3_ops; +extern const struct sof_ipc_ops ipc4_ops; + #endif diff --git a/sound/soc/sof/topology.c b/sound/soc/sof/topology.c index 3e5b319b44..9273a70fec 100644 --- a/sound/soc/sof/topology.c +++ b/sound/soc/sof/topology.c @@ -32,14 +32,10 @@ #define VOL_HALF_DB_STEP 50 /* TLV data items */ -#define TLV_ITEMS 3 #define TLV_MIN 0 #define TLV_STEP 1 #define TLV_MUTE 2 -/* size of tplg abi in byte */ -#define SOF_TPLG_ABI_SIZE 3 - /** * sof_update_ipc_object - Parse multiple sets of tokens within the token array associated with the * token ID. @@ -134,7 +130,7 @@ int sof_update_ipc_object(struct snd_soc_component *scomp, void *object, enum so return 0; } -static inline int get_tlv_data(const int *p, int tlv[TLV_ITEMS]) +static inline int get_tlv_data(const int *p, int tlv[SOF_TLV_ITEMS]) { /* we only support dB scale TLV type at the moment */ if ((int)p[SNDRV_CTL_TLVO_TYPE] != SNDRV_CTL_TLVT_DB_SCALE) @@ -224,7 +220,7 @@ static u32 vol_pow32(u32 a, int exp, u32 fwl) * Function to calculate volume gain from TLV data. * This function can only handle gain steps that are multiples of 0.5 dB */ -static u32 vol_compute_gain(u32 value, int *tlv) +u32 vol_compute_gain(u32 value, int *tlv) { int dB_gain; u32 linear_gain; @@ -263,20 +259,17 @@ static u32 vol_compute_gain(u32 value, int *tlv) * "size" specifies the number of entries in the table */ static int set_up_volume_table(struct snd_sof_control *scontrol, - int tlv[TLV_ITEMS], int size) + int tlv[SOF_TLV_ITEMS], int size) { - int j; + struct snd_soc_component *scomp = scontrol->scomp; + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); + const struct sof_ipc_tplg_ops *tplg_ops = sdev->ipc->ops->tplg; - /* init the volume table */ - scontrol->volume_table = kcalloc(size, sizeof(u32), GFP_KERNEL); - if (!scontrol->volume_table) - return -ENOMEM; + if (tplg_ops->control->set_up_volume_table) + return tplg_ops->control->set_up_volume_table(scontrol, tlv, size); - /* populate the volume table */ - for (j = 0; j < size ; j++) - scontrol->volume_table[j] = vol_compute_gain(j, tlv); - - return 0; + dev_err(scomp->dev, "Mandatory op %s not set\n", __func__); + return -EINVAL; } struct sof_dai_types { @@ -772,7 +765,8 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); struct snd_soc_tplg_mixer_control *mc = container_of(hdr, struct snd_soc_tplg_mixer_control, hdr); - int tlv[TLV_ITEMS]; + int tlv[SOF_TLV_ITEMS]; + unsigned int mask; int ret; /* validate topology data */ @@ -821,6 +815,16 @@ static int sof_control_load_volume(struct snd_soc_component *scomp, goto err; } + if (scontrol->led_ctl.use_led) { + mask = scontrol->led_ctl.direction ? SNDRV_CTL_ELEM_ACCESS_MIC_LED : + SNDRV_CTL_ELEM_ACCESS_SPK_LED; + scontrol->access &= ~SNDRV_CTL_ELEM_ACCESS_LED_MASK; + scontrol->access |= mask; + kc->access &= ~SNDRV_CTL_ELEM_ACCESS_LED_MASK; + kc->access |= mask; + sdev->led_present = true; + } + dev_dbg(scomp->dev, "tplg: load kcontrol index %d chans %d\n", scontrol->comp_id, scontrol->num_channels); @@ -1001,15 +1005,18 @@ static int sof_connect_dai_widget(struct snd_soc_component *scomp, struct snd_soc_dai *cpu_dai; int i; + if (!w->sname) { + dev_err(scomp->dev, "Widget %s does not have stream\n", w->name); + return -EINVAL; + } + list_for_each_entry(rtd, &card->rtd_list, list) { dev_vdbg(scomp->dev, "tplg: check widget: %s stream: %s dai stream: %s\n", w->name, w->sname, rtd->dai_link->stream_name); - if (!w->sname || !rtd->dai_link->stream_name) - continue; - /* does stream match DAI link ? */ - if (strcmp(w->sname, rtd->dai_link->stream_name)) + if (!rtd->dai_link->stream_name || + strcmp(w->sname, rtd->dai_link->stream_name)) continue; switch (w->id) { @@ -1131,75 +1138,6 @@ static int spcm_bind(struct snd_soc_component *scomp, struct snd_sof_pcm *spcm, return 0; } -static int sof_widget_parse_tokens(struct snd_soc_component *scomp, struct snd_sof_widget *swidget, - struct snd_soc_tplg_dapm_widget *tw, - enum sof_tokens *object_token_list, int count) -{ - struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); - const struct sof_ipc_tplg_ops *ipc_tplg_ops = sdev->ipc->ops->tplg; - const struct sof_token_info *token_list = ipc_tplg_ops->token_list; - struct snd_soc_tplg_private *private = &tw->priv; - int num_tuples = 0; - size_t size; - int ret, i; - - if (count > 0 && !object_token_list) { - dev_err(scomp->dev, "No token list for widget %s\n", swidget->widget->name); - return -EINVAL; - } - - /* calculate max size of tuples array */ - for (i = 0; i < count; i++) - num_tuples += token_list[object_token_list[i]].count; - - /* allocate memory for tuples array */ - size = sizeof(struct snd_sof_tuple) * num_tuples; - swidget->tuples = kzalloc(size, GFP_KERNEL); - if (!swidget->tuples) - return -ENOMEM; - - /* parse token list for widget */ - for (i = 0; i < count; i++) { - if (object_token_list[i] >= SOF_TOKEN_COUNT) { - dev_err(scomp->dev, "Invalid token id %d for widget %s\n", - object_token_list[i], swidget->widget->name); - ret = -EINVAL; - goto err; - } - - /* parse and save UUID in swidget */ - if (object_token_list[i] == SOF_COMP_EXT_TOKENS) { - ret = sof_parse_tokens(scomp, swidget, - token_list[object_token_list[i]].tokens, - token_list[object_token_list[i]].count, - private->array, le32_to_cpu(private->size)); - if (ret < 0) { - dev_err(scomp->dev, "Failed parsing %s for widget %s\n", - token_list[object_token_list[i]].name, - swidget->widget->name); - goto err; - } - - continue; - } - - /* copy one set of tuples per token ID into swidget->tuples */ - ret = sof_copy_tuples(sdev, private->array, le32_to_cpu(private->size), - object_token_list[i], 1, swidget->tuples, - num_tuples, &swidget->num_tuples); - if (ret < 0) { - dev_err(scomp->dev, "Failed parsing %s for widget %s err: %d\n", - token_list[object_token_list[i]].name, swidget->widget->name, ret); - goto err; - } - } - - return 0; -err: - kfree(swidget->tuples); - return ret; -} - static int sof_get_token_value(u32 token_id, struct snd_sof_tuple *tuples, int num_tuples) { int i; @@ -1215,6 +1153,106 @@ static int sof_get_token_value(u32 token_id, struct snd_sof_tuple *tuples, int n return -EINVAL; } +static int sof_widget_parse_tokens(struct snd_soc_component *scomp, struct snd_sof_widget *swidget, + struct snd_soc_tplg_dapm_widget *tw, + enum sof_tokens *object_token_list, int count) +{ + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); + const struct sof_ipc_tplg_ops *ipc_tplg_ops = sdev->ipc->ops->tplg; + const struct sof_token_info *token_list = ipc_tplg_ops->token_list; + struct snd_soc_tplg_private *private = &tw->priv; + int num_tuples = 0; + int ret, i; + + if (count > 0 && !object_token_list) { + dev_err(scomp->dev, "No token list for widget %s\n", swidget->widget->name); + return -EINVAL; + } + + /* calculate max size of tuples array */ + for (i = 0; i < count; i++) + num_tuples += token_list[object_token_list[i]].count; + + /* allocate memory for tuples array */ + swidget->tuples = kcalloc(num_tuples, sizeof(*swidget->tuples), GFP_KERNEL); + if (!swidget->tuples) + return -ENOMEM; + + /* parse token list for widget */ + for (i = 0; i < count; i++) { + int num_sets = 1; + + if (object_token_list[i] >= SOF_TOKEN_COUNT) { + dev_err(scomp->dev, "Invalid token id %d for widget %s\n", + object_token_list[i], swidget->widget->name); + ret = -EINVAL; + goto err; + } + + switch (object_token_list[i]) { + case SOF_COMP_EXT_TOKENS: + /* parse and save UUID in swidget */ + ret = sof_parse_tokens(scomp, swidget, + token_list[object_token_list[i]].tokens, + token_list[object_token_list[i]].count, + private->array, le32_to_cpu(private->size)); + if (ret < 0) { + dev_err(scomp->dev, "Failed parsing %s for widget %s\n", + token_list[object_token_list[i]].name, + swidget->widget->name); + goto err; + } + + continue; + case SOF_IN_AUDIO_FORMAT_TOKENS: + case SOF_OUT_AUDIO_FORMAT_TOKENS: + case SOF_COPIER_GATEWAY_CFG_TOKENS: + case SOF_AUDIO_FORMAT_BUFFER_SIZE_TOKENS: + num_sets = sof_get_token_value(SOF_TKN_COMP_NUM_AUDIO_FORMATS, + swidget->tuples, swidget->num_tuples); + + if (num_sets < 0) { + dev_err(sdev->dev, "Invalid audio format count for %s\n", + swidget->widget->name); + ret = num_sets; + goto err; + } + + if (num_sets > 1) { + struct snd_sof_tuple *new_tuples; + + num_tuples += token_list[object_token_list[i]].count * num_sets; + new_tuples = krealloc(swidget->tuples, + sizeof(*new_tuples) * num_tuples, GFP_KERNEL); + if (!new_tuples) { + ret = -ENOMEM; + goto err; + } + + swidget->tuples = new_tuples; + } + break; + default: + break; + } + + /* copy one set of tuples per token ID into swidget->tuples */ + ret = sof_copy_tuples(sdev, private->array, le32_to_cpu(private->size), + object_token_list[i], num_sets, swidget->tuples, + num_tuples, &swidget->num_tuples); + if (ret < 0) { + dev_err(scomp->dev, "Failed parsing %s for widget %s err: %d\n", + token_list[object_token_list[i]].name, swidget->widget->name, ret); + goto err; + } + } + + return 0; +err: + kfree(swidget->tuples); + return ret; +} + /* external widget init - used for any driver specific init */ static int sof_widget_ready(struct snd_soc_component *scomp, int index, struct snd_soc_dapm_widget *w, @@ -1381,7 +1419,6 @@ static int sof_widget_unload(struct snd_soc_component *scomp, struct soc_bytes_ext *sbe; struct snd_sof_dai *dai; struct soc_enum *se; - int ret = 0; int i; swidget = dobj->private; @@ -1442,7 +1479,7 @@ static int sof_widget_unload(struct snd_soc_component *scomp, list_del(&swidget->list); kfree(swidget); - return ret; + return 0; } /* @@ -1595,7 +1632,6 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_ const struct sof_token_info *token_list = ipc_tplg_ops->token_list; struct snd_soc_tplg_private *private = &cfg->priv; struct snd_sof_dai_link *slink; - size_t size; u32 token_id = 0; int num_tuples = 0; int ret, num_sets; @@ -1702,27 +1738,32 @@ static int sof_link_load(struct snd_soc_component *scomp, int index, struct snd_ token_id = SOF_AFE_TOKENS; num_tuples += token_list[SOF_AFE_TOKENS].count; break; + case SOF_DAI_AMD_DMIC: + token_id = SOF_ACPDMIC_TOKENS; + num_tuples += token_list[SOF_ACPDMIC_TOKENS].count; + break; default: break; } /* allocate memory for tuples array */ - size = sizeof(struct snd_sof_tuple) * num_tuples; - slink->tuples = kzalloc(size, GFP_KERNEL); + slink->tuples = kcalloc(num_tuples, sizeof(*slink->tuples), GFP_KERNEL); if (!slink->tuples) { kfree(slink->hw_configs); kfree(slink); return -ENOMEM; } - /* parse one set of DAI link tokens */ - ret = sof_copy_tuples(sdev, private->array, le32_to_cpu(private->size), - SOF_DAI_LINK_TOKENS, 1, slink->tuples, - num_tuples, &slink->num_tuples); - if (ret < 0) { - dev_err(scomp->dev, "failed to parse %s for dai link %s\n", - token_list[SOF_DAI_LINK_TOKENS].name, link->name); - goto err; + if (token_list[SOF_DAI_LINK_TOKENS].tokens) { + /* parse one set of DAI link tokens */ + ret = sof_copy_tuples(sdev, private->array, le32_to_cpu(private->size), + SOF_DAI_LINK_TOKENS, 1, slink->tuples, + num_tuples, &slink->num_tuples); + if (ret < 0) { + dev_err(scomp->dev, "failed to parse %s for dai link %s\n", + token_list[SOF_DAI_LINK_TOKENS].name, link->name); + goto err; + } } /* nothing more to do if there are no DAI type-specific tokens defined */ @@ -1979,45 +2020,11 @@ static int sof_complete(struct snd_soc_component *scomp) static int sof_manifest(struct snd_soc_component *scomp, int index, struct snd_soc_tplg_manifest *man) { - u32 size; - u32 abi_version; + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); + const struct sof_ipc_tplg_ops *ipc_tplg_ops = sdev->ipc->ops->tplg; - size = le32_to_cpu(man->priv.size); - - /* backward compatible with tplg without ABI info */ - if (!size) { - dev_dbg(scomp->dev, "No topology ABI info\n"); - return 0; - } - - if (size != SOF_TPLG_ABI_SIZE) { - dev_err(scomp->dev, "error: invalid topology ABI size\n"); - return -EINVAL; - } - - dev_info(scomp->dev, - "Topology: ABI %d:%d:%d Kernel ABI %d:%d:%d\n", - man->priv.data[0], man->priv.data[1], - man->priv.data[2], SOF_ABI_MAJOR, SOF_ABI_MINOR, - SOF_ABI_PATCH); - - abi_version = SOF_ABI_VER(man->priv.data[0], - man->priv.data[1], - man->priv.data[2]); - - if (SOF_ABI_VERSION_INCOMPATIBLE(SOF_ABI_VERSION, abi_version)) { - dev_err(scomp->dev, "error: incompatible topology ABI version\n"); - return -EINVAL; - } - - if (SOF_ABI_VERSION_MINOR(abi_version) > SOF_ABI_MINOR) { - if (!IS_ENABLED(CONFIG_SND_SOC_SOF_STRICT_ABI_CHECKS)) { - dev_warn(scomp->dev, "warn: topology ABI is more recent than kernel\n"); - } else { - dev_err(scomp->dev, "error: topology ABI is more recent than kernel\n"); - return -EINVAL; - } - } + if (ipc_tplg_ops->parse_manifest) + return ipc_tplg_ops->parse_manifest(scomp, index, man); return 0; } @@ -2075,6 +2082,7 @@ static struct snd_soc_tplg_ops sof_tplg_ops = { int snd_sof_load_topology(struct snd_soc_component *scomp, const char *file) { + struct snd_sof_dev *sdev = snd_soc_component_get_drvdata(scomp); const struct firmware *fw; int ret; @@ -2097,6 +2105,10 @@ int snd_sof_load_topology(struct snd_soc_component *scomp, const char *file) } release_firmware(fw); + + if (ret >= 0 && sdev->led_present) + ret = snd_ctl_led_request(); + return ret; } EXPORT_SYMBOL(snd_sof_load_topology); diff --git a/sound/soc/sof/trace.c b/sound/soc/sof/trace.c index ea8e4506d0..6f662642d6 100644 --- a/sound/soc/sof/trace.c +++ b/sound/soc/sof/trace.c @@ -1,601 +1,51 @@ -// SPDX-License-Identifier: (GPL-2.0-only OR BSD-3-Clause) -// -// This file is provided under a dual BSD/GPLv2 license. When using or -// redistributing this file, you may do so under either license. -// -// Copyright(c) 2018 Intel Corporation. All rights reserved. -// -// Author: Liam Girdwood +// SPDX-License-Identifier: GPL-2.0-only // +// Copyright(c) 2022 Intel Corporation. All rights reserved. -#include -#include #include "sof-priv.h" -#include "ops.h" -#include "sof-utils.h" -#define TRACE_FILTER_ELEMENTS_PER_ENTRY 4 -#define TRACE_FILTER_MAX_CONFIG_STRING_LENGTH 1024 - -static int trace_filter_append_elem(struct snd_sof_dev *sdev, uint32_t key, uint32_t value, - struct sof_ipc_trace_filter_elem *elem_list, - int capacity, int *counter) +int sof_fw_trace_init(struct snd_sof_dev *sdev) { - if (*counter >= capacity) - return -ENOMEM; + if (!sdev->ipc->ops->fw_tracing) { + dev_info(sdev->dev, "Firmware tracing is not available\n"); + sdev->fw_trace_is_supported = false; - elem_list[*counter].key = key; - elem_list[*counter].value = value; - ++*counter; - - return 0; -} - -static int trace_filter_parse_entry(struct snd_sof_dev *sdev, const char *line, - struct sof_ipc_trace_filter_elem *elem, - int capacity, int *counter) -{ - int len = strlen(line); - int cnt = *counter; - uint32_t uuid_id; - int log_level; - int pipe_id; - int comp_id; - int read; - int ret; - - /* ignore empty content */ - ret = sscanf(line, " %n", &read); - if (!ret && read == len) - return len; - - ret = sscanf(line, " %d %x %d %d %n", &log_level, &uuid_id, &pipe_id, &comp_id, &read); - if (ret != TRACE_FILTER_ELEMENTS_PER_ENTRY || read != len) { - dev_err(sdev->dev, "error: invalid trace filter entry '%s'\n", line); - return -EINVAL; - } - - if (uuid_id > 0) { - ret = trace_filter_append_elem(sdev, SOF_IPC_TRACE_FILTER_ELEM_BY_UUID, - uuid_id, elem, capacity, &cnt); - if (ret) - return ret; - } - if (pipe_id >= 0) { - ret = trace_filter_append_elem(sdev, SOF_IPC_TRACE_FILTER_ELEM_BY_PIPE, - pipe_id, elem, capacity, &cnt); - if (ret) - return ret; - } - if (comp_id >= 0) { - ret = trace_filter_append_elem(sdev, SOF_IPC_TRACE_FILTER_ELEM_BY_COMP, - comp_id, elem, capacity, &cnt); - if (ret) - return ret; - } - - ret = trace_filter_append_elem(sdev, SOF_IPC_TRACE_FILTER_ELEM_SET_LEVEL | - SOF_IPC_TRACE_FILTER_ELEM_FIN, - log_level, elem, capacity, &cnt); - if (ret) - return ret; - - /* update counter only when parsing whole entry passed */ - *counter = cnt; - - return len; -} - -static int trace_filter_parse(struct snd_sof_dev *sdev, char *string, - int *out_elem_cnt, - struct sof_ipc_trace_filter_elem **out) -{ - static const char entry_delimiter[] = ";"; - char *entry = string; - int capacity = 0; - int entry_len; - int cnt = 0; - - /* - * Each entry contains at least 1, up to TRACE_FILTER_ELEMENTS_PER_ENTRY - * IPC elements, depending on content. Calculate IPC elements capacity - * for the input string where each element is set. - */ - while (entry) { - capacity += TRACE_FILTER_ELEMENTS_PER_ENTRY; - entry = strchr(entry + 1, entry_delimiter[0]); - } - *out = kmalloc(capacity * sizeof(**out), GFP_KERNEL); - if (!*out) - return -ENOMEM; - - /* split input string by ';', and parse each entry separately in trace_filter_parse_entry */ - while ((entry = strsep(&string, entry_delimiter))) { - entry_len = trace_filter_parse_entry(sdev, entry, *out, capacity, &cnt); - if (entry_len < 0) { - dev_err(sdev->dev, "error: %s failed for '%s', %d\n", __func__, entry, - entry_len); - return -EINVAL; - } - } - - *out_elem_cnt = cnt; - - return 0; -} - -static int sof_ipc_trace_update_filter(struct snd_sof_dev *sdev, int num_elems, - struct sof_ipc_trace_filter_elem *elems) -{ - struct sof_ipc_trace_filter *msg; - struct sof_ipc_reply reply; - size_t size; - int ret; - - size = struct_size(msg, elems, num_elems); - if (size > SOF_IPC_MSG_MAX_SIZE) - return -EINVAL; - - msg = kmalloc(size, GFP_KERNEL); - if (!msg) - return -ENOMEM; - - msg->hdr.size = size; - msg->hdr.cmd = SOF_IPC_GLB_TRACE_MSG | SOF_IPC_TRACE_FILTER_UPDATE; - msg->elem_cnt = num_elems; - memcpy(&msg->elems[0], elems, num_elems * sizeof(*elems)); - - ret = pm_runtime_get_sync(sdev->dev); - if (ret < 0 && ret != -EACCES) { - pm_runtime_put_noidle(sdev->dev); - dev_err(sdev->dev, "error: enabling device failed: %d\n", ret); - goto error; - } - ret = sof_ipc_tx_message(sdev->ipc, msg->hdr.cmd, msg, msg->hdr.size, - &reply, sizeof(reply)); - pm_runtime_mark_last_busy(sdev->dev); - pm_runtime_put_autosuspend(sdev->dev); - -error: - kfree(msg); - return ret ? ret : reply.error; -} - -static ssize_t sof_dfsentry_trace_filter_write(struct file *file, const char __user *from, - size_t count, loff_t *ppos) -{ - struct snd_sof_dfsentry *dfse = file->private_data; - struct sof_ipc_trace_filter_elem *elems = NULL; - struct snd_sof_dev *sdev = dfse->sdev; - loff_t pos = 0; - int num_elems; - char *string; - int ret; - - if (count > TRACE_FILTER_MAX_CONFIG_STRING_LENGTH) { - dev_err(sdev->dev, "%s too long input, %zu > %d\n", __func__, count, - TRACE_FILTER_MAX_CONFIG_STRING_LENGTH); - return -EINVAL; - } - - string = kmalloc(count + 1, GFP_KERNEL); - if (!string) - return -ENOMEM; - - /* assert null termination */ - string[count] = 0; - ret = simple_write_to_buffer(string, count, &pos, from, count); - if (ret < 0) - goto error; - - ret = trace_filter_parse(sdev, string, &num_elems, &elems); - if (ret < 0) { - dev_err(sdev->dev, "error: fail in trace_filter_parse, %d\n", ret); - goto error; - } - - if (num_elems) { - ret = sof_ipc_trace_update_filter(sdev, num_elems, elems); - if (ret < 0) { - dev_err(sdev->dev, "error: fail in sof_ipc_trace_update_filter %d\n", ret); - goto error; - } - } - ret = count; -error: - kfree(string); - kfree(elems); - return ret; -} - -static const struct file_operations sof_dfs_trace_filter_fops = { - .open = simple_open, - .write = sof_dfsentry_trace_filter_write, - .llseek = default_llseek, -}; - -static int trace_debugfs_filter_create(struct snd_sof_dev *sdev) -{ - struct snd_sof_dfsentry *dfse; - - dfse = devm_kzalloc(sdev->dev, sizeof(*dfse), GFP_KERNEL); - if (!dfse) - return -ENOMEM; - - dfse->sdev = sdev; - dfse->type = SOF_DFSENTRY_TYPE_BUF; - - debugfs_create_file("filter", 0200, sdev->debugfs_root, dfse, - &sof_dfs_trace_filter_fops); - /* add to dfsentry list */ - list_add(&dfse->list, &sdev->dfsentry_list); - - return 0; -} - -static size_t sof_trace_avail(struct snd_sof_dev *sdev, - loff_t pos, size_t buffer_size) -{ - loff_t host_offset = READ_ONCE(sdev->host_offset); - - /* - * If host offset is less than local pos, it means write pointer of - * host DMA buffer has been wrapped. We should output the trace data - * at the end of host DMA buffer at first. - */ - if (host_offset < pos) - return buffer_size - pos; - - /* If there is available trace data now, it is unnecessary to wait. */ - if (host_offset > pos) - return host_offset - pos; - - return 0; -} - -static size_t sof_wait_trace_avail(struct snd_sof_dev *sdev, - loff_t pos, size_t buffer_size) -{ - wait_queue_entry_t wait; - size_t ret = sof_trace_avail(sdev, pos, buffer_size); - - /* data immediately available */ - if (ret) - return ret; - - if (!sdev->dtrace_is_enabled && sdev->dtrace_draining) { - /* - * tracing has ended and all traces have been - * read by client, return EOF - */ - sdev->dtrace_draining = false; return 0; } - /* wait for available trace data from FW */ - init_waitqueue_entry(&wait, current); - set_current_state(TASK_INTERRUPTIBLE); - add_wait_queue(&sdev->trace_sleep, &wait); - - if (!signal_pending(current)) { - /* set timeout to max value, no error code */ - schedule_timeout(MAX_SCHEDULE_TIMEOUT); - } - remove_wait_queue(&sdev->trace_sleep, &wait); - - return sof_trace_avail(sdev, pos, buffer_size); + return sdev->ipc->ops->fw_tracing->init(sdev); } -static ssize_t sof_dfsentry_trace_read(struct file *file, char __user *buffer, - size_t count, loff_t *ppos) +void sof_fw_trace_free(struct snd_sof_dev *sdev) { - struct snd_sof_dfsentry *dfse = file->private_data; - struct snd_sof_dev *sdev = dfse->sdev; - unsigned long rem; - loff_t lpos = *ppos; - size_t avail, buffer_size = dfse->size; - u64 lpos_64; - - /* make sure we know about any failures on the DSP side */ - sdev->dtrace_error = false; - - /* check pos and count */ - if (lpos < 0) - return -EINVAL; - if (!count) - return 0; - - /* check for buffer wrap and count overflow */ - lpos_64 = lpos; - lpos = do_div(lpos_64, buffer_size); - - /* get available count based on current host offset */ - avail = sof_wait_trace_avail(sdev, lpos, buffer_size); - if (sdev->dtrace_error) { - dev_err(sdev->dev, "error: trace IO error\n"); - return -EIO; - } - - /* make sure count is <= avail */ - if (count > avail) - count = avail; - - /* - * make sure that all trace data is available for the CPU as the trace - * data buffer might be allocated from non consistent memory. - * Note: snd_dma_buffer_sync() is called for normal audio playback and - * capture streams also. - */ - snd_dma_buffer_sync(&sdev->dmatb, SNDRV_DMA_SYNC_CPU); - /* copy available trace data to debugfs */ - rem = copy_to_user(buffer, ((u8 *)(dfse->buf) + lpos), count); - if (rem) - return -EFAULT; - - *ppos += count; - - /* move debugfs reading position */ - return count; -} - -static int sof_dfsentry_trace_release(struct inode *inode, struct file *file) -{ - struct snd_sof_dfsentry *dfse = inode->i_private; - struct snd_sof_dev *sdev = dfse->sdev; - - /* avoid duplicate traces at next open */ - if (!sdev->dtrace_is_enabled) - sdev->host_offset = 0; - - return 0; -} - -static const struct file_operations sof_dfs_trace_fops = { - .open = simple_open, - .read = sof_dfsentry_trace_read, - .llseek = default_llseek, - .release = sof_dfsentry_trace_release, -}; - -static int trace_debugfs_create(struct snd_sof_dev *sdev) -{ - struct snd_sof_dfsentry *dfse; - int ret; - - if (!sdev) - return -EINVAL; - - ret = trace_debugfs_filter_create(sdev); - if (ret < 0) - dev_err(sdev->dev, "error: fail in %s, %d", __func__, ret); - - dfse = devm_kzalloc(sdev->dev, sizeof(*dfse), GFP_KERNEL); - if (!dfse) - return -ENOMEM; - - dfse->type = SOF_DFSENTRY_TYPE_BUF; - dfse->buf = sdev->dmatb.area; - dfse->size = sdev->dmatb.bytes; - dfse->sdev = sdev; - - debugfs_create_file("trace", 0444, sdev->debugfs_root, dfse, - &sof_dfs_trace_fops); - - return 0; -} - -int snd_sof_init_trace_ipc(struct snd_sof_dev *sdev) -{ - struct sof_ipc_fw_ready *ready = &sdev->fw_ready; - struct sof_ipc_fw_version *v = &ready->version; - struct sof_ipc_dma_trace_params_ext params; - struct sof_ipc_reply ipc_reply; - int ret; - - if (!sdev->dtrace_is_supported) - return 0; - - if (sdev->dtrace_is_enabled || !sdev->dma_trace_pages) - return -EINVAL; - - /* set IPC parameters */ - params.hdr.cmd = SOF_IPC_GLB_TRACE_MSG; - /* PARAMS_EXT is only supported from ABI 3.7.0 onwards */ - if (v->abi_version >= SOF_ABI_VER(3, 7, 0)) { - params.hdr.size = sizeof(struct sof_ipc_dma_trace_params_ext); - params.hdr.cmd |= SOF_IPC_TRACE_DMA_PARAMS_EXT; - params.timestamp_ns = ktime_get(); /* in nanosecond */ - } else { - params.hdr.size = sizeof(struct sof_ipc_dma_trace_params); - params.hdr.cmd |= SOF_IPC_TRACE_DMA_PARAMS; - } - params.buffer.phy_addr = sdev->dmatp.addr; - params.buffer.size = sdev->dmatb.bytes; - params.buffer.pages = sdev->dma_trace_pages; - params.stream_tag = 0; - - sdev->host_offset = 0; - sdev->dtrace_draining = false; - - ret = snd_sof_dma_trace_init(sdev, ¶ms); - if (ret < 0) { - dev_err(sdev->dev, - "error: fail in snd_sof_dma_trace_init %d\n", ret); - return ret; - } - dev_dbg(sdev->dev, "%s: stream_tag: %d\n", __func__, params.stream_tag); - - /* send IPC to the DSP */ - ret = sof_ipc_tx_message(sdev->ipc, - params.hdr.cmd, ¶ms, sizeof(params), - &ipc_reply, sizeof(ipc_reply)); - if (ret < 0) { - dev_err(sdev->dev, - "error: can't set params for DMA for trace %d\n", ret); - goto trace_release; - } - - ret = snd_sof_dma_trace_trigger(sdev, SNDRV_PCM_TRIGGER_START); - if (ret < 0) { - dev_err(sdev->dev, - "error: snd_sof_dma_trace_trigger: start: %d\n", ret); - goto trace_release; - } - - sdev->dtrace_is_enabled = true; - - return 0; - -trace_release: - snd_sof_dma_trace_release(sdev); - return ret; -} - -int snd_sof_init_trace(struct snd_sof_dev *sdev) -{ - int ret; - - if (!sdev->dtrace_is_supported) - return 0; - - /* set false before start initialization */ - sdev->dtrace_is_enabled = false; - - /* allocate trace page table buffer */ - ret = snd_dma_alloc_pages(SNDRV_DMA_TYPE_DEV, sdev->dev, - PAGE_SIZE, &sdev->dmatp); - if (ret < 0) { - dev_err(sdev->dev, - "error: can't alloc page table for trace %d\n", ret); - return ret; - } - - /* allocate trace data buffer */ - ret = snd_dma_alloc_dir_pages(SNDRV_DMA_TYPE_DEV_SG, sdev->dev, - DMA_FROM_DEVICE, DMA_BUF_SIZE_FOR_TRACE, - &sdev->dmatb); - if (ret < 0) { - dev_err(sdev->dev, - "error: can't alloc buffer for trace %d\n", ret); - goto page_err; - } - - /* create compressed page table for audio firmware */ - ret = snd_sof_create_page_table(sdev->dev, &sdev->dmatb, - sdev->dmatp.area, sdev->dmatb.bytes); - if (ret < 0) - goto table_err; - - sdev->dma_trace_pages = ret; - dev_dbg(sdev->dev, "%s: dma_trace_pages: %d\n", - __func__, sdev->dma_trace_pages); - - if (sdev->first_boot) { - ret = trace_debugfs_create(sdev); - if (ret < 0) - goto table_err; - } - - init_waitqueue_head(&sdev->trace_sleep); - - ret = snd_sof_init_trace_ipc(sdev); - if (ret < 0) - goto table_err; - - return 0; -table_err: - sdev->dma_trace_pages = 0; - snd_dma_free_pages(&sdev->dmatb); -page_err: - snd_dma_free_pages(&sdev->dmatp); - return ret; -} -EXPORT_SYMBOL(snd_sof_init_trace); - -int snd_sof_trace_update_pos(struct snd_sof_dev *sdev, - struct sof_ipc_dma_trace_posn *posn) -{ - if (!sdev->dtrace_is_supported) - return 0; - - if (sdev->dtrace_is_enabled && sdev->host_offset != posn->host_offset) { - sdev->host_offset = posn->host_offset; - wake_up(&sdev->trace_sleep); - } - - if (posn->overflow != 0) - dev_err(sdev->dev, - "error: DSP trace buffer overflow %u bytes. Total messages %d\n", - posn->overflow, posn->messages); - - return 0; -} - -/* an error has occurred within the DSP that prevents further trace */ -void snd_sof_trace_notify_for_error(struct snd_sof_dev *sdev) -{ - if (!sdev->dtrace_is_supported) + if (!sdev->fw_trace_is_supported || !sdev->ipc->ops->fw_tracing) return; - if (sdev->dtrace_is_enabled) { - sdev->dtrace_error = true; - wake_up(&sdev->trace_sleep); - } + if (sdev->ipc->ops->fw_tracing->free) + sdev->ipc->ops->fw_tracing->free(sdev); } -EXPORT_SYMBOL(snd_sof_trace_notify_for_error); -void snd_sof_release_trace(struct snd_sof_dev *sdev) +void sof_fw_trace_fw_crashed(struct snd_sof_dev *sdev) { - struct sof_ipc_fw_ready *ready = &sdev->fw_ready; - struct sof_ipc_fw_version *v = &ready->version; - struct sof_ipc_cmd_hdr hdr; - struct sof_ipc_reply ipc_reply; - int ret; - - if (!sdev->dtrace_is_supported || !sdev->dtrace_is_enabled) + if (!sdev->fw_trace_is_supported) return; - ret = snd_sof_dma_trace_trigger(sdev, SNDRV_PCM_TRIGGER_STOP); - if (ret < 0) - dev_err(sdev->dev, - "error: snd_sof_dma_trace_trigger: stop: %d\n", ret); - - /* - * stop and free trace DMA in the DSP. TRACE_DMA_FREE is only supported from - * ABI 3.20.0 onwards - */ - if (v->abi_version >= SOF_ABI_VER(3, 20, 0)) { - hdr.size = sizeof(hdr); - hdr.cmd = SOF_IPC_GLB_TRACE_MSG | SOF_IPC_TRACE_DMA_FREE; - - ret = sof_ipc_tx_message(sdev->ipc, hdr.cmd, &hdr, hdr.size, - &ipc_reply, sizeof(ipc_reply)); - if (ret < 0) - dev_err(sdev->dev, "DMA_TRACE_FREE failed with error: %d\n", ret); - } - - ret = snd_sof_dma_trace_release(sdev); - if (ret < 0) - dev_err(sdev->dev, - "error: fail in snd_sof_dma_trace_release %d\n", ret); - - sdev->dtrace_is_enabled = false; - sdev->dtrace_draining = true; - wake_up(&sdev->trace_sleep); + if (sdev->ipc->ops->fw_tracing->fw_crashed) + sdev->ipc->ops->fw_tracing->fw_crashed(sdev); } -EXPORT_SYMBOL(snd_sof_release_trace); -void snd_sof_free_trace(struct snd_sof_dev *sdev) +void sof_fw_trace_suspend(struct snd_sof_dev *sdev, pm_message_t pm_state) { - if (!sdev->dtrace_is_supported) + if (!sdev->fw_trace_is_supported) return; - snd_sof_release_trace(sdev); - - if (sdev->dma_trace_pages) { - snd_dma_free_pages(&sdev->dmatb); - snd_dma_free_pages(&sdev->dmatp); - sdev->dma_trace_pages = 0; - } + sdev->ipc->ops->fw_tracing->suspend(sdev, pm_state); +} + +int sof_fw_trace_resume(struct snd_sof_dev *sdev) +{ + if (!sdev->fw_trace_is_supported) + return 0; + + return sdev->ipc->ops->fw_tracing->resume(sdev); } -EXPORT_SYMBOL(snd_sof_free_trace); diff --git a/sound/soc/spear/spdif_in.c b/sound/soc/spear/spdif_in.c index 4b68d6ee75..4ad8b1fc71 100644 --- a/sound/soc/spear/spdif_in.c +++ b/sound/soc/spear/spdif_in.c @@ -172,7 +172,8 @@ static struct snd_soc_dai_driver spdif_in_dai = { }; static const struct snd_soc_component_driver spdif_in_component = { - .name = "spdif-in", + .name = "spdif-in", + .legacy_dai_naming = 1, }; static irqreturn_t spdif_in_irq(int irq, void *arg) diff --git a/sound/soc/spear/spdif_out.c b/sound/soc/spear/spdif_out.c index 549295a6ed..fb107c5790 100644 --- a/sound/soc/spear/spdif_out.c +++ b/sound/soc/spear/spdif_out.c @@ -273,7 +273,8 @@ static struct snd_soc_dai_driver spdif_out_dai = { }; static const struct snd_soc_component_driver spdif_out_component = { - .name = "spdif-out", + .name = "spdif-out", + .legacy_dai_naming = 1, }; static int spdif_out_probe(struct platform_device *pdev) diff --git a/sound/soc/sti/sti_uniperif.c b/sound/soc/sti/sti_uniperif.c index 34668fe390..a4d74d1e3c 100644 --- a/sound/soc/sti/sti_uniperif.c +++ b/sound/soc/sti/sti_uniperif.c @@ -376,7 +376,8 @@ static const struct snd_soc_dai_driver sti_uniperiph_dai_template = { static const struct snd_soc_component_driver sti_uniperiph_dai_component = { .name = "sti_cpu_dai", .suspend = sti_uniperiph_suspend, - .resume = sti_uniperiph_resume + .resume = sti_uniperiph_resume, + .legacy_dai_naming = 1, }; static int sti_uniperiph_cpu_dai_of(struct device_node *node, diff --git a/sound/soc/stm/stm32_adfsdm.c b/sound/soc/stm/stm32_adfsdm.c index 6ee714542b..04f2912e14 100644 --- a/sound/soc/stm/stm32_adfsdm.c +++ b/sound/soc/stm/stm32_adfsdm.c @@ -149,6 +149,7 @@ static const struct snd_soc_dai_driver stm32_adfsdm_dai = { static const struct snd_soc_component_driver stm32_adfsdm_dai_component = { .name = "stm32_dfsdm_audio", + .legacy_dai_naming = 1, }; static void stm32_memcpy_32to16(void *dest, const void *src, size_t n) @@ -296,7 +297,7 @@ static int stm32_adfsdm_pcm_new(struct snd_soc_component *component, static int stm32_adfsdm_dummy_cb(const void *data, void *private) { /* - * This dummmy callback is requested by iio_channel_get_all_cb() API, + * This dummy callback is requested by iio_channel_get_all_cb() API, * but the stm32_dfsdm_get_buff_cb() API is used instead, to optimize * DMA transfers. */ diff --git a/sound/soc/stm/stm32_i2s.c b/sound/soc/stm/stm32_i2s.c index ac5dff4d16..6aafe793ee 100644 --- a/sound/soc/stm/stm32_i2s.c +++ b/sound/soc/stm/stm32_i2s.c @@ -593,16 +593,16 @@ static int stm32_i2s_set_dai_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) } /* DAI clock master masks */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: i2s->ms_flg = I2S_MS_SLAVE; break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: i2s->ms_flg = I2S_MS_MASTER; break; default: dev_err(cpu_dai->dev, "Unsupported mode %#x\n", - fmt & SND_SOC_DAIFMT_MASTER_MASK); + fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK); return -EINVAL; } @@ -978,6 +978,7 @@ static const struct snd_dmaengine_pcm_config stm32_i2s_pcm_config = { static const struct snd_soc_component_driver stm32_i2s_component = { .name = "stm32-i2s", + .legacy_dai_naming = 1, }; static void stm32_i2s_dai_init(struct snd_soc_pcm_stream *stream, diff --git a/sound/soc/stm/stm32_sai_sub.c b/sound/soc/stm/stm32_sai_sub.c index dd636af81c..eb31b49e65 100644 --- a/sound/soc/stm/stm32_sai_sub.c +++ b/sound/soc/stm/stm32_sai_sub.c @@ -45,8 +45,6 @@ #define STM_SAI_B_ID 0x1 #define STM_SAI_IS_SUB_A(x) ((x)->id == STM_SAI_A_ID) -#define STM_SAI_IS_SUB_B(x) ((x)->id == STM_SAI_B_ID) -#define STM_SAI_BLOCK_NAME(x) (((x)->id == STM_SAI_A_ID) ? "A" : "B") #define SAI_SYNC_NONE 0x0 #define SAI_SYNC_INTERNAL 0x1 @@ -719,18 +717,18 @@ static int stm32_sai_set_dai_fmt(struct snd_soc_dai *cpu_dai, unsigned int fmt) stm32_sai_sub_reg_up(sai, STM_SAI_FRCR_REGX, frcr_mask, frcr); /* DAI clock master masks */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: /* codec is master */ cr1 |= SAI_XCR1_SLAVE; sai->master = false; break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: sai->master = true; break; default: dev_err(cpu_dai->dev, "Unsupported mode %#x\n", - fmt & SND_SOC_DAIFMT_MASTER_MASK); + fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK); return -EINVAL; } @@ -1338,6 +1336,7 @@ static const struct snd_dmaengine_pcm_config stm32_sai_pcm_config_spdif = { static const struct snd_soc_component_driver stm32_component = { .name = "stm32-sai", + .legacy_dai_naming = 1, }; static const struct of_device_id stm32_sai_sub_ids[] = { diff --git a/sound/soc/stm/stm32_spdifrx.c b/sound/soc/stm/stm32_spdifrx.c index 6f7882c4fe..0f71467567 100644 --- a/sound/soc/stm/stm32_spdifrx.c +++ b/sound/soc/stm/stm32_spdifrx.c @@ -888,6 +888,7 @@ static const struct snd_pcm_hardware stm32_spdifrx_pcm_hw = { static const struct snd_soc_component_driver stm32_spdifrx_component = { .name = "stm32-spdifrx", + .legacy_dai_naming = 1, }; static const struct snd_dmaengine_pcm_config stm32_spdifrx_pcm_config = { diff --git a/sound/soc/sunxi/sun4i-codec.c b/sound/soc/sunxi/sun4i-codec.c index 60712f24ad..830beb38bf 100644 --- a/sound/soc/sunxi/sun4i-codec.c +++ b/sound/soc/sunxi/sun4i-codec.c @@ -250,37 +250,33 @@ struct sun4i_codec { static void sun4i_codec_start_playback(struct sun4i_codec *scodec) { /* Flush TX FIFO */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_FIFO_FLUSH), - BIT(SUN4I_CODEC_DAC_FIFOC_FIFO_FLUSH)); + regmap_set_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, + BIT(SUN4I_CODEC_DAC_FIFOC_FIFO_FLUSH)); /* Enable DAC DRQ */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_DAC_DRQ_EN), - BIT(SUN4I_CODEC_DAC_FIFOC_DAC_DRQ_EN)); + regmap_set_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, + BIT(SUN4I_CODEC_DAC_FIFOC_DAC_DRQ_EN)); } static void sun4i_codec_stop_playback(struct sun4i_codec *scodec) { /* Disable DAC DRQ */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_DAC_DRQ_EN), - 0); + regmap_clear_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, + BIT(SUN4I_CODEC_DAC_FIFOC_DAC_DRQ_EN)); } static void sun4i_codec_start_capture(struct sun4i_codec *scodec) { /* Enable ADC DRQ */ - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN), - BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN)); + regmap_field_set_bits(scodec->reg_adc_fifoc, + BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN)); } static void sun4i_codec_stop_capture(struct sun4i_codec *scodec) { /* Disable ADC DRQ */ - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN), 0); + regmap_field_clear_bits(scodec->reg_adc_fifoc, + BIT(SUN4I_CODEC_ADC_FIFOC_ADC_DRQ_EN)); } static int sun4i_codec_trigger(struct snd_pcm_substream *substream, int cmd, @@ -323,8 +319,7 @@ static int sun4i_codec_prepare_capture(struct snd_pcm_substream *substream, /* Flush RX FIFO */ - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH), + regmap_field_set_bits(scodec->reg_adc_fifoc, BIT(SUN4I_CODEC_ADC_FIFOC_FIFO_FLUSH)); @@ -365,8 +360,7 @@ static int sun4i_codec_prepare_playback(struct snd_pcm_substream *substream, u32 val; /* Flush the TX FIFO */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_FIFO_FLUSH), + regmap_set_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, BIT(SUN4I_CODEC_DAC_FIFOC_FIFO_FLUSH)); /* Set TX FIFO Empty Trigger Level */ @@ -386,9 +380,8 @@ static int sun4i_codec_prepare_playback(struct snd_pcm_substream *substream, val); /* Send zeros when we have an underrun */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_SEND_LASAT), - 0); + regmap_clear_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, + BIT(SUN4I_CODEC_DAC_FIFOC_SEND_LASAT)); return 0; }; @@ -485,33 +478,27 @@ static int sun4i_codec_hw_params_capture(struct sun4i_codec *scodec, /* Set the number of channels we want to use */ if (params_channels(params) == 1) - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN), + regmap_field_set_bits(scodec->reg_adc_fifoc, BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN)); else - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN), - 0); + regmap_field_clear_bits(scodec->reg_adc_fifoc, + BIT(SUN4I_CODEC_ADC_FIFOC_MONO_EN)); /* Set the number of sample bits to either 16 or 24 bits */ if (hw_param_interval(params, SNDRV_PCM_HW_PARAM_SAMPLE_BITS)->min == 32) { - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_RX_SAMPLE_BITS), + regmap_field_set_bits(scodec->reg_adc_fifoc, BIT(SUN4I_CODEC_ADC_FIFOC_RX_SAMPLE_BITS)); - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE), - 0); + regmap_field_clear_bits(scodec->reg_adc_fifoc, + BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE)); scodec->capture_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; } else { - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_RX_SAMPLE_BITS), - 0); + regmap_field_clear_bits(scodec->reg_adc_fifoc, + BIT(SUN4I_CODEC_ADC_FIFOC_RX_SAMPLE_BITS)); /* Fill most significant bits with valid data MSB */ - regmap_field_update_bits(scodec->reg_adc_fifoc, - BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE), + regmap_field_set_bits(scodec->reg_adc_fifoc, BIT(SUN4I_CODEC_ADC_FIFOC_RX_FIFO_MODE)); scodec->capture_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; @@ -543,24 +530,20 @@ static int sun4i_codec_hw_params_playback(struct sun4i_codec *scodec, /* Set the number of sample bits to either 16 or 24 bits */ if (hw_param_interval(params, SNDRV_PCM_HW_PARAM_SAMPLE_BITS)->min == 32) { - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_TX_SAMPLE_BITS), + regmap_set_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, BIT(SUN4I_CODEC_DAC_FIFOC_TX_SAMPLE_BITS)); /* Set TX FIFO mode to padding the LSBs with 0 */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_TX_FIFO_MODE), - 0); + regmap_clear_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, + BIT(SUN4I_CODEC_DAC_FIFOC_TX_FIFO_MODE)); scodec->playback_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; } else { - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_TX_SAMPLE_BITS), - 0); + regmap_clear_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, + BIT(SUN4I_CODEC_DAC_FIFOC_TX_SAMPLE_BITS)); /* Set TX FIFO mode to repeat the MSB */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - BIT(SUN4I_CODEC_DAC_FIFOC_TX_FIFO_MODE), + regmap_set_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, BIT(SUN4I_CODEC_DAC_FIFOC_TX_FIFO_MODE)); scodec->playback_dma_data.addr_width = DMA_SLAVE_BUSWIDTH_2_BYTES; @@ -624,8 +607,7 @@ static int sun4i_codec_startup(struct snd_pcm_substream *substream, * Stop issuing DRQ when we have room for less than 16 samples * in our TX FIFO */ - regmap_update_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, - 3 << SUN4I_CODEC_DAC_FIFOC_DRQ_CLR_CNT, + regmap_set_bits(scodec->regmap, SUN4I_CODEC_DAC_FIFOC, 3 << SUN4I_CODEC_DAC_FIFOC_DRQ_CLR_CNT); return clk_prepare_enable(scodec->clk_module); @@ -899,7 +881,6 @@ static const struct snd_soc_component_driver sun4i_codec_codec = { .idle_bias_on = 1, .use_pmdown_time = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; static const struct snd_soc_component_driver sun7i_codec_codec = { @@ -912,7 +893,6 @@ static const struct snd_soc_component_driver sun7i_codec_codec = { .idle_bias_on = 1, .use_pmdown_time = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; /*** sun6i Codec ***/ @@ -1220,7 +1200,6 @@ static const struct snd_soc_component_driver sun6i_codec_codec = { .idle_bias_on = 1, .use_pmdown_time = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; /* sun8i A23 codec */ @@ -1248,11 +1227,11 @@ static const struct snd_soc_component_driver sun8i_a23_codec_codec = { .idle_bias_on = 1, .use_pmdown_time = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; static const struct snd_soc_component_driver sun4i_codec_component = { - .name = "sun4i-codec", + .name = "sun4i-codec", + .legacy_dai_naming = 1, }; #define SUN4I_CODEC_RATES SNDRV_PCM_RATE_CONTINUOUS diff --git a/sound/soc/sunxi/sun4i-i2s.c b/sound/soc/sunxi/sun4i-i2s.c index 7047f71629..6028871825 100644 --- a/sound/soc/sunxi/sun4i-i2s.c +++ b/sound/soc/sunxi/sun4i-i2s.c @@ -161,6 +161,8 @@ struct sun4i_i2s; * @field_clkdiv_mclk_en: regmap field to enable mclk output. * @field_fmt_wss: regmap field to set word select size. * @field_fmt_sr: regmap field to set sample resolution. + * @num_din_pins: input pins + * @num_dout_pins: output pins (currently set but unused) * @bclk_dividers: bit clock dividers array * @num_bclk_dividers: number of bit clock dividers * @mclk_dividers: mclk dividers array @@ -702,13 +704,13 @@ static int sun4i_i2s_set_soc_fmt(const struct sun4i_i2s *i2s, SUN4I_I2S_FMT0_FMT_MASK, val); /* DAI clock master masks */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* BCLK and LRCLK master */ val = SUN4I_I2S_CTRL_MODE_MASTER; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* BCLK and LRCLK slave */ val = SUN4I_I2S_CTRL_MODE_SLAVE; break; @@ -802,13 +804,13 @@ static int sun8i_i2s_set_soc_fmt(const struct sun4i_i2s *i2s, SUN8I_I2S_TX_CHAN_OFFSET(offset)); /* DAI clock master masks */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* BCLK and LRCLK master */ val = SUN8I_I2S_CTRL_BCLK_OUT | SUN8I_I2S_CTRL_LRCK_OUT; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* BCLK and LRCLK slave */ val = 0; break; @@ -909,13 +911,13 @@ static int sun50i_h6_i2s_set_soc_fmt(const struct sun4i_i2s *i2s, SUN50I_H6_I2S_TX_CHAN_SEL_OFFSET(offset)); /* DAI clock master masks */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* BCLK and LRCLK master */ val = SUN8I_I2S_CTRL_BCLK_OUT | SUN8I_I2S_CTRL_LRCK_OUT; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* BCLK and LRCLK slave */ val = 0; break; @@ -1123,7 +1125,8 @@ static struct snd_soc_dai_driver sun4i_i2s_dai = { }; static const struct snd_soc_component_driver sun4i_i2s_component = { - .name = "sun4i-dai", + .name = "sun4i-dai", + .legacy_dai_naming = 1, }; static bool sun4i_i2s_rd_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/sunxi/sun4i-spdif.c b/sound/soc/sunxi/sun4i-spdif.c index 17090f4315..bcceebca91 100644 --- a/sound/soc/sunxi/sun4i-spdif.c +++ b/sound/soc/sunxi/sun4i-spdif.c @@ -583,7 +583,8 @@ static const struct of_device_id sun4i_spdif_of_match[] = { MODULE_DEVICE_TABLE(of, sun4i_spdif_of_match); static const struct snd_soc_component_driver sun4i_spdif_component = { - .name = "sun4i-spdif", + .name = "sun4i-spdif", + .legacy_dai_naming = 1, }; static int sun4i_spdif_runtime_suspend(struct device *dev) diff --git a/sound/soc/sunxi/sun50i-codec-analog.c b/sound/soc/sunxi/sun50i-codec-analog.c index a41e25ad0a..e1e5e8de01 100644 --- a/sound/soc/sunxi/sun50i-codec-analog.c +++ b/sound/soc/sunxi/sun50i-codec-analog.c @@ -117,6 +117,7 @@ #define SUN50I_ADDA_HS_MBIAS_CTRL_MMICBIASEN 7 #define SUN50I_ADDA_JACK_MIC_CTRL 0x1d +#define SUN50I_ADDA_JACK_MIC_CTRL_INNERRESEN 6 #define SUN50I_ADDA_JACK_MIC_CTRL_HMICBIASEN 5 /* mixer controls */ @@ -507,6 +508,7 @@ static int sun50i_codec_analog_probe(struct platform_device *pdev) { struct regmap *regmap; void __iomem *base; + bool enable; base = devm_platform_ioremap_resource(pdev, 0); if (IS_ERR(base)) { @@ -520,6 +522,12 @@ static int sun50i_codec_analog_probe(struct platform_device *pdev) return PTR_ERR(regmap); } + enable = device_property_read_bool(&pdev->dev, + "allwinner,internal-bias-resistor"); + regmap_update_bits(regmap, SUN50I_ADDA_JACK_MIC_CTRL, + BIT(SUN50I_ADDA_JACK_MIC_CTRL_INNERRESEN), + enable << SUN50I_ADDA_JACK_MIC_CTRL_INNERRESEN); + return devm_snd_soc_register_component(&pdev->dev, &sun50i_codec_analog_cmpnt_drv, NULL, 0); diff --git a/sound/soc/sunxi/sun8i-codec.c b/sound/soc/sunxi/sun8i-codec.c index 0bea2162f6..9844978d91 100644 --- a/sound/soc/sunxi/sun8i-codec.c +++ b/sound/soc/sunxi/sun8i-codec.c @@ -286,11 +286,11 @@ static int sun8i_codec_set_fmt(struct snd_soc_dai *dai, unsigned int fmt) u32 dsp_format, format, invert, value; /* clock masters */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: /* Codec slave, DAI master */ + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_CBC_CFC: /* Codec slave, DAI master */ value = 0x1; break; - case SND_SOC_DAIFMT_CBM_CFM: /* Codec Master, DAI slave */ + case SND_SOC_DAIFMT_CBP_CFP: /* Codec Master, DAI slave */ value = 0x0; break; default: @@ -1278,7 +1278,6 @@ static const struct snd_soc_component_driver sun8i_soc_component = { .probe = sun8i_codec_component_probe, .idle_bias_on = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; static const struct regmap_config sun8i_codec_regmap_config = { diff --git a/sound/soc/tegra/Kconfig b/sound/soc/tegra/Kconfig index cd454871d6..b6712a3d1f 100644 --- a/sound/soc/tegra/Kconfig +++ b/sound/soc/tegra/Kconfig @@ -85,6 +85,27 @@ config SND_SOC_TEGRA210_I2S compatible devices. Say Y or M if you want to add support for Tegra210 I2S module. +config SND_SOC_TEGRA210_OPE + tristate "Tegra210 OPE module" + help + Config to enable the Output Processing Engine (OPE) which includes + Parametric Equalizer (PEQ) and Multi Band Dynamic Range Compressor + (MBDRC) sub blocks for data processing. It can support up to 8 + channels. + Say Y or M if you want to add support for Tegra210 OPE module. + +config SND_SOC_TEGRA186_ASRC + tristate "Tegra186 ASRC module" + help + Config to enable the Asynchronous Sample Rate Converter (ASRC), + which converts the sampling frequency of the input signal from + one frequency to another. It can handle over a wide range of + sample rate ratios (freq_in/freq_out) from 1:24 to 24:1. + ASRC has two modes of operation. One where ratio can be programmed + in SW and the other where it gets information from ratio estimator + module. + Say Y or M if you want to add support for Tegra186 ASRC module. + config SND_SOC_TEGRA186_DSPK tristate "Tegra186 DSPK module" help diff --git a/sound/soc/tegra/Makefile b/sound/soc/tegra/Makefile index f19d56690a..b723c78e66 100644 --- a/sound/soc/tegra/Makefile +++ b/sound/soc/tegra/Makefile @@ -11,6 +11,7 @@ snd-soc-tegra30-i2s-objs := tegra30_i2s.o snd-soc-tegra210-ahub-objs := tegra210_ahub.o snd-soc-tegra210-dmic-objs := tegra210_dmic.o snd-soc-tegra210-i2s-objs := tegra210_i2s.o +snd-soc-tegra186-asrc-objs := tegra186_asrc.o snd-soc-tegra186-dspk-objs := tegra186_dspk.o snd-soc-tegra210-admaif-objs := tegra210_admaif.o snd-soc-tegra210-mvc-objs := tegra210_mvc.o @@ -18,6 +19,7 @@ snd-soc-tegra210-sfc-objs := tegra210_sfc.o snd-soc-tegra210-amx-objs := tegra210_amx.o snd-soc-tegra210-adx-objs := tegra210_adx.o snd-soc-tegra210-mixer-objs := tegra210_mixer.o +snd-soc-tegra210-ope-objs := tegra210_ope.o tegra210_mbdrc.o tegra210_peq.o obj-$(CONFIG_SND_SOC_TEGRA) += snd-soc-tegra-pcm.o obj-$(CONFIG_SND_SOC_TEGRA20_AC97) += snd-soc-tegra20-ac97.o @@ -29,6 +31,7 @@ obj-$(CONFIG_SND_SOC_TEGRA30_I2S) += snd-soc-tegra30-i2s.o obj-$(CONFIG_SND_SOC_TEGRA210_DMIC) += snd-soc-tegra210-dmic.o obj-$(CONFIG_SND_SOC_TEGRA210_AHUB) += snd-soc-tegra210-ahub.o obj-$(CONFIG_SND_SOC_TEGRA210_I2S) += snd-soc-tegra210-i2s.o +obj-$(CONFIG_SND_SOC_TEGRA186_ASRC) += snd-soc-tegra186-asrc.o obj-$(CONFIG_SND_SOC_TEGRA186_DSPK) += snd-soc-tegra186-dspk.o obj-$(CONFIG_SND_SOC_TEGRA210_ADMAIF) += snd-soc-tegra210-admaif.o obj-$(CONFIG_SND_SOC_TEGRA210_MVC) += snd-soc-tegra210-mvc.o @@ -36,6 +39,7 @@ obj-$(CONFIG_SND_SOC_TEGRA210_SFC) += snd-soc-tegra210-sfc.o obj-$(CONFIG_SND_SOC_TEGRA210_AMX) += snd-soc-tegra210-amx.o obj-$(CONFIG_SND_SOC_TEGRA210_ADX) += snd-soc-tegra210-adx.o obj-$(CONFIG_SND_SOC_TEGRA210_MIXER) += snd-soc-tegra210-mixer.o +obj-$(CONFIG_SND_SOC_TEGRA210_OPE) += snd-soc-tegra210-ope.o # Tegra machine Support snd-soc-tegra-wm8903-objs := tegra_wm8903.o diff --git a/sound/soc/tegra/tegra20_ac97.c b/sound/soc/tegra/tegra20_ac97.c index c454a34c15..87facfbcdd 100644 --- a/sound/soc/tegra/tegra20_ac97.c +++ b/sound/soc/tegra/tegra20_ac97.c @@ -239,7 +239,8 @@ static struct snd_soc_dai_driver tegra20_ac97_dai = { }; static const struct snd_soc_component_driver tegra20_ac97_component = { - .name = DRV_NAME, + .name = DRV_NAME, + .legacy_dai_naming = 1, }; static bool tegra20_ac97_wr_rd_reg(struct device *dev, unsigned int reg) @@ -353,6 +354,7 @@ static int tegra20_ac97_platform_probe(struct platform_device *pdev) } } else { dev_err(&pdev->dev, "no codec-reset GPIO supplied\n"); + ret = -EINVAL; goto err_clk_put; } @@ -360,6 +362,7 @@ static int tegra20_ac97_platform_probe(struct platform_device *pdev) "nvidia,codec-sync-gpio", 0); if (!gpio_is_valid(ac97->sync_gpio)) { dev_err(&pdev->dev, "no codec-sync GPIO supplied\n"); + ret = -EINVAL; goto err_clk_put; } diff --git a/sound/soc/tegra/tegra20_das.c b/sound/soc/tegra/tegra20_das.c index 69c651274c..c620ab0c60 100644 --- a/sound/soc/tegra/tegra20_das.c +++ b/sound/soc/tegra/tegra20_das.c @@ -13,84 +13,119 @@ #include #include #include -#include "tegra20_das.h" #define DRV_NAME "tegra20-das" -static struct tegra20_das *das; +/* Register TEGRA20_DAS_DAP_CTRL_SEL */ +#define TEGRA20_DAS_DAP_CTRL_SEL 0x00 +#define TEGRA20_DAS_DAP_CTRL_SEL_COUNT 5 +#define TEGRA20_DAS_DAP_CTRL_SEL_STRIDE 4 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_MS_SEL_P 31 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_MS_SEL_S 1 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_SDATA1_TX_RX_P 30 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_SDATA1_TX_RX_S 1 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_SDATA2_TX_RX_P 29 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_SDATA2_TX_RX_S 1 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_CTRL_SEL_P 0 +#define TEGRA20_DAS_DAP_CTRL_SEL_DAP_CTRL_SEL_S 5 -static inline void tegra20_das_write(u32 reg, u32 val) +/* Values for field TEGRA20_DAS_DAP_CTRL_SEL_DAP_CTRL_SEL */ +#define TEGRA20_DAS_DAP_SEL_DAC1 0 +#define TEGRA20_DAS_DAP_SEL_DAC2 1 +#define TEGRA20_DAS_DAP_SEL_DAC3 2 +#define TEGRA20_DAS_DAP_SEL_DAP1 16 +#define TEGRA20_DAS_DAP_SEL_DAP2 17 +#define TEGRA20_DAS_DAP_SEL_DAP3 18 +#define TEGRA20_DAS_DAP_SEL_DAP4 19 +#define TEGRA20_DAS_DAP_SEL_DAP5 20 + +/* Register TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL */ +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL 0x40 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_COUNT 3 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_STRIDE 4 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA2_SEL_P 28 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA2_SEL_S 4 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA1_SEL_P 24 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA1_SEL_S 4 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_CLK_SEL_P 0 +#define TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_CLK_SEL_S 4 + +/* + * Values for: + * TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA2_SEL + * TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA1_SEL + * TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_CLK_SEL + */ +#define TEGRA20_DAS_DAC_SEL_DAP1 0 +#define TEGRA20_DAS_DAC_SEL_DAP2 1 +#define TEGRA20_DAS_DAC_SEL_DAP3 2 +#define TEGRA20_DAS_DAC_SEL_DAP4 3 +#define TEGRA20_DAS_DAC_SEL_DAP5 4 + +/* + * Names/IDs of the DACs/DAPs. + */ + +#define TEGRA20_DAS_DAP_ID_1 0 +#define TEGRA20_DAS_DAP_ID_2 1 +#define TEGRA20_DAS_DAP_ID_3 2 +#define TEGRA20_DAS_DAP_ID_4 3 +#define TEGRA20_DAS_DAP_ID_5 4 + +#define TEGRA20_DAS_DAC_ID_1 0 +#define TEGRA20_DAS_DAC_ID_2 1 +#define TEGRA20_DAS_DAC_ID_3 2 + +struct tegra20_das { + struct regmap *regmap; +}; + +/* + * Terminology: + * DAS: Digital audio switch (HW module controlled by this driver) + * DAP: Digital audio port (port/pins on Tegra device) + * DAC: Digital audio controller (e.g. I2S or AC97 controller elsewhere) + * + * The Tegra DAS is a mux/cross-bar which can connect each DAP to a specific + * DAC, or another DAP. When DAPs are connected, one must be the master and + * one the slave. Each DAC allows selection of a specific DAP for input, to + * cater for the case where N DAPs are connected to 1 DAC for broadcast + * output. + * + * This driver is dumb; no attempt is made to ensure that a valid routing + * configuration is programmed. + */ + +static inline void tegra20_das_write(struct tegra20_das *das, u32 reg, u32 val) { regmap_write(das->regmap, reg, val); } -static inline u32 tegra20_das_read(u32 reg) -{ - u32 val; - - regmap_read(das->regmap, reg, &val); - return val; -} - -int tegra20_das_connect_dap_to_dac(int dap, int dac) +static void tegra20_das_connect_dap_to_dac(struct tegra20_das *das, int dap, int dac) { u32 addr; u32 reg; - if (!das) - return -ENODEV; - addr = TEGRA20_DAS_DAP_CTRL_SEL + (dap * TEGRA20_DAS_DAP_CTRL_SEL_STRIDE); reg = dac << TEGRA20_DAS_DAP_CTRL_SEL_DAP_CTRL_SEL_P; - tegra20_das_write(addr, reg); - - return 0; + tegra20_das_write(das, addr, reg); } -EXPORT_SYMBOL_GPL(tegra20_das_connect_dap_to_dac); -int tegra20_das_connect_dap_to_dap(int dap, int otherdap, int master, - int sdata1rx, int sdata2rx) +static void tegra20_das_connect_dac_to_dap(struct tegra20_das *das, int dac, int dap) { u32 addr; u32 reg; - if (!das) - return -ENODEV; - - addr = TEGRA20_DAS_DAP_CTRL_SEL + - (dap * TEGRA20_DAS_DAP_CTRL_SEL_STRIDE); - reg = (otherdap << TEGRA20_DAS_DAP_CTRL_SEL_DAP_CTRL_SEL_P) | - (!!sdata2rx << TEGRA20_DAS_DAP_CTRL_SEL_DAP_SDATA2_TX_RX_P) | - (!!sdata1rx << TEGRA20_DAS_DAP_CTRL_SEL_DAP_SDATA1_TX_RX_P) | - (!!master << TEGRA20_DAS_DAP_CTRL_SEL_DAP_MS_SEL_P); - - tegra20_das_write(addr, reg); - - return 0; -} -EXPORT_SYMBOL_GPL(tegra20_das_connect_dap_to_dap); - -int tegra20_das_connect_dac_to_dap(int dac, int dap) -{ - u32 addr; - u32 reg; - - if (!das) - return -ENODEV; - addr = TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL + (dac * TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_STRIDE); reg = dap << TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_CLK_SEL_P | dap << TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA1_SEL_P | dap << TEGRA20_DAS_DAC_INPUT_DATA_CLK_SEL_DAC_SDATA2_SEL_P; - tegra20_das_write(addr, reg); - - return 0; + tegra20_das_write(das, addr, reg); } -EXPORT_SYMBOL_GPL(tegra20_das_connect_dac_to_dap); #define LAST_REG(name) \ (TEGRA20_DAS_##name + \ @@ -120,73 +155,31 @@ static const struct regmap_config tegra20_das_regmap_config = { static int tegra20_das_probe(struct platform_device *pdev) { void __iomem *regs; - int ret = 0; - - if (das) - return -ENODEV; + struct tegra20_das *das; das = devm_kzalloc(&pdev->dev, sizeof(struct tegra20_das), GFP_KERNEL); - if (!das) { - ret = -ENOMEM; - goto err; - } - das->dev = &pdev->dev; + if (!das) + return -ENOMEM; regs = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(regs)) { - ret = PTR_ERR(regs); - goto err; - } + if (IS_ERR(regs)) + return PTR_ERR(regs); das->regmap = devm_regmap_init_mmio(&pdev->dev, regs, &tegra20_das_regmap_config); if (IS_ERR(das->regmap)) { dev_err(&pdev->dev, "regmap init failed\n"); - ret = PTR_ERR(das->regmap); - goto err; + return PTR_ERR(das->regmap); } - ret = tegra20_das_connect_dap_to_dac(TEGRA20_DAS_DAP_ID_1, - TEGRA20_DAS_DAP_SEL_DAC1); - if (ret) { - dev_err(&pdev->dev, "Can't set up DAS DAP connection\n"); - goto err; - } - ret = tegra20_das_connect_dac_to_dap(TEGRA20_DAS_DAC_ID_1, - TEGRA20_DAS_DAC_SEL_DAP1); - if (ret) { - dev_err(&pdev->dev, "Can't set up DAS DAC connection\n"); - goto err; - } - - ret = tegra20_das_connect_dap_to_dac(TEGRA20_DAS_DAP_ID_3, - TEGRA20_DAS_DAP_SEL_DAC3); - if (ret) { - dev_err(&pdev->dev, "Can't set up DAS DAP connection\n"); - goto err; - } - ret = tegra20_das_connect_dac_to_dap(TEGRA20_DAS_DAC_ID_3, - TEGRA20_DAS_DAC_SEL_DAP3); - if (ret) { - dev_err(&pdev->dev, "Can't set up DAS DAC connection\n"); - goto err; - } - - platform_set_drvdata(pdev, das); - - return 0; - -err: - das = NULL; - return ret; -} - -static int tegra20_das_remove(struct platform_device *pdev) -{ - if (!das) - return -ENODEV; - - das = NULL; + tegra20_das_connect_dap_to_dac(das, TEGRA20_DAS_DAP_ID_1, + TEGRA20_DAS_DAP_SEL_DAC1); + tegra20_das_connect_dac_to_dap(das, TEGRA20_DAS_DAC_ID_1, + TEGRA20_DAS_DAC_SEL_DAP1); + tegra20_das_connect_dap_to_dac(das, TEGRA20_DAS_DAP_ID_3, + TEGRA20_DAS_DAP_SEL_DAC3); + tegra20_das_connect_dac_to_dap(das, TEGRA20_DAS_DAC_ID_3, + TEGRA20_DAS_DAC_SEL_DAP3); return 0; } @@ -198,7 +191,6 @@ static const struct of_device_id tegra20_das_of_match[] = { static struct platform_driver tegra20_das_driver = { .probe = tegra20_das_probe, - .remove = tegra20_das_remove, .driver = { .name = DRV_NAME, .of_match_table = tegra20_das_of_match, diff --git a/sound/soc/tegra/tegra20_i2s.c b/sound/soc/tegra/tegra20_i2s.c index 27365a877e..fff0cd6588 100644 --- a/sound/soc/tegra/tegra20_i2s.c +++ b/sound/soc/tegra/tegra20_i2s.c @@ -95,11 +95,11 @@ static int tegra20_i2s_set_fmt(struct snd_soc_dai *dai, } mask |= TEGRA20_I2S_CTRL_MASTER_ENABLE; - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: val |= TEGRA20_I2S_CTRL_MASTER_ENABLE; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: break; default: return -EINVAL; @@ -338,7 +338,8 @@ static const struct snd_soc_dai_driver tegra20_i2s_dai_template = { }; static const struct snd_soc_component_driver tegra20_i2s_component = { - .name = DRV_NAME, + .name = DRV_NAME, + .legacy_dai_naming = 1, }; static bool tegra20_i2s_wr_rd_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/tegra/tegra20_spdif.c b/sound/soc/tegra/tegra20_spdif.c index 64c2f304f2..ca7b222e07 100644 --- a/sound/soc/tegra/tegra20_spdif.c +++ b/sound/soc/tegra/tegra20_spdif.c @@ -264,6 +264,7 @@ static struct snd_soc_dai_driver tegra20_spdif_dai = { static const struct snd_soc_component_driver tegra20_spdif_component = { .name = "tegra20-spdif", + .legacy_dai_naming = 1, }; static bool tegra20_spdif_wr_rd_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/tegra/tegra210_adx.c b/sound/soc/tegra/tegra210_adx.c index 3785cade2d..49691d2cce 100644 --- a/sound/soc/tegra/tegra210_adx.c +++ b/sound/soc/tegra/tegra210_adx.c @@ -191,7 +191,7 @@ static int tegra210_adx_put_byte_map(struct snd_kcontrol *kcontrol, unsigned char *bytes_map = (unsigned char *)&adx->map; int value = ucontrol->value.integer.value[0]; struct soc_mixer_control *mc = - (struct soc_mixer_control *)kcontrol->private_value;; + (struct soc_mixer_control *)kcontrol->private_value; if (value == bytes_map[mc->reg]) return 0; diff --git a/sound/soc/tegra/tegra210_ahub.c b/sound/soc/tegra/tegra210_ahub.c index bccf8b8ca7..b38d205b69 100644 --- a/sound/soc/tegra/tegra210_ahub.c +++ b/sound/soc/tegra/tegra210_ahub.c @@ -170,6 +170,11 @@ static struct snd_soc_dai_driver tegra210_ahub_dais[] = { DAI(MIXER1 TX3), DAI(MIXER1 TX4), DAI(MIXER1 TX5), + /* XBAR -> OPE -> XBAR */ + DAI(OPE1 RX), + DAI(OPE1 TX), + DAI(OPE2 RX), + DAI(OPE2 TX), }; static struct snd_soc_dai_driver tegra186_ahub_dais[] = { @@ -280,6 +285,23 @@ static struct snd_soc_dai_driver tegra186_ahub_dais[] = { DAI(MIXER1 TX3), DAI(MIXER1 TX4), DAI(MIXER1 TX5), + /* XBAR -> ASRC -> XBAR */ + DAI(ASRC1 RX1), + DAI(ASRC1 TX1), + DAI(ASRC1 RX2), + DAI(ASRC1 TX2), + DAI(ASRC1 RX3), + DAI(ASRC1 TX3), + DAI(ASRC1 RX4), + DAI(ASRC1 TX4), + DAI(ASRC1 RX5), + DAI(ASRC1 TX5), + DAI(ASRC1 RX6), + DAI(ASRC1 TX6), + DAI(ASRC1 RX7), + /* XBAR -> OPE -> XBAR */ + DAI(OPE1 RX), + DAI(OPE1 TX), }; static const char * const tegra210_ahub_mux_texts[] = { @@ -323,6 +345,8 @@ static const char * const tegra210_ahub_mux_texts[] = { "MIXER1 TX3", "MIXER1 TX4", "MIXER1 TX5", + "OPE1", + "OPE2", }; static const char * const tegra186_ahub_mux_texts[] = { @@ -388,6 +412,13 @@ static const char * const tegra186_ahub_mux_texts[] = { "MIXER1 TX3", "MIXER1 TX4", "MIXER1 TX5", + "ASRC1 TX1", + "ASRC1 TX2", + "ASRC1 TX3", + "ASRC1 TX4", + "ASRC1 TX5", + "ASRC1 TX6", + "OPE1", }; static const unsigned int tegra210_ahub_mux_values[] = { @@ -439,6 +470,9 @@ static const unsigned int tegra210_ahub_mux_values[] = { MUX_VALUE(1, 2), MUX_VALUE(1, 3), MUX_VALUE(1, 4), + /* OPE */ + MUX_VALUE(2, 0), + MUX_VALUE(2, 1), }; static const unsigned int tegra186_ahub_mux_values[] = { @@ -513,6 +547,15 @@ static const unsigned int tegra186_ahub_mux_values[] = { MUX_VALUE(1, 2), MUX_VALUE(1, 3), MUX_VALUE(1, 4), + /* ASRC */ + MUX_VALUE(3, 24), + MUX_VALUE(3, 25), + MUX_VALUE(3, 26), + MUX_VALUE(3, 27), + MUX_VALUE(3, 28), + MUX_VALUE(3, 29), + /* OPE */ + MUX_VALUE(2, 0), }; /* Controls for t210 */ @@ -557,6 +600,8 @@ MUX_ENUM_CTRL_DECL(t210_mixer17_tx, 0x26); MUX_ENUM_CTRL_DECL(t210_mixer18_tx, 0x27); MUX_ENUM_CTRL_DECL(t210_mixer19_tx, 0x28); MUX_ENUM_CTRL_DECL(t210_mixer110_tx, 0x29); +MUX_ENUM_CTRL_DECL(t210_ope1_tx, 0x40); +MUX_ENUM_CTRL_DECL(t210_ope2_tx, 0x41); /* Controls for t186 */ MUX_ENUM_CTRL_DECL_186(t186_admaif1_tx, 0x00); @@ -623,6 +668,14 @@ MUX_ENUM_CTRL_DECL_186(t186_mixer17_tx, 0x26); MUX_ENUM_CTRL_DECL_186(t186_mixer18_tx, 0x27); MUX_ENUM_CTRL_DECL_186(t186_mixer19_tx, 0x28); MUX_ENUM_CTRL_DECL_186(t186_mixer110_tx, 0x29); +MUX_ENUM_CTRL_DECL_186(t186_asrc11_tx, 0x6c); +MUX_ENUM_CTRL_DECL_186(t186_asrc12_tx, 0x6d); +MUX_ENUM_CTRL_DECL_186(t186_asrc13_tx, 0x6e); +MUX_ENUM_CTRL_DECL_186(t186_asrc14_tx, 0x6f); +MUX_ENUM_CTRL_DECL_186(t186_asrc15_tx, 0x70); +MUX_ENUM_CTRL_DECL_186(t186_asrc16_tx, 0x71); +MUX_ENUM_CTRL_DECL_186(t186_asrc17_tx, 0x72); +MUX_ENUM_CTRL_DECL_186(t186_ope1_tx, 0x40); /* Controls for t234 */ MUX_ENUM_CTRL_DECL_234(t234_mvc1_tx, 0x44); @@ -651,6 +704,13 @@ MUX_ENUM_CTRL_DECL_234(t234_admaif17_tx, 0x60); MUX_ENUM_CTRL_DECL_234(t234_admaif18_tx, 0x61); MUX_ENUM_CTRL_DECL_234(t234_admaif19_tx, 0x62); MUX_ENUM_CTRL_DECL_234(t234_admaif20_tx, 0x63); +MUX_ENUM_CTRL_DECL_234(t234_asrc11_tx, 0x64); +MUX_ENUM_CTRL_DECL_234(t234_asrc12_tx, 0x65); +MUX_ENUM_CTRL_DECL_234(t234_asrc13_tx, 0x66); +MUX_ENUM_CTRL_DECL_234(t234_asrc14_tx, 0x67); +MUX_ENUM_CTRL_DECL_234(t234_asrc15_tx, 0x68); +MUX_ENUM_CTRL_DECL_234(t234_asrc16_tx, 0x69); +MUX_ENUM_CTRL_DECL_234(t234_asrc17_tx, 0x6a); /* * The number of entries in, and order of, this array is closely tied to the @@ -717,6 +777,8 @@ static const struct snd_soc_dapm_widget tegra210_ahub_widgets[] = { TX_WIDGETS("MIXER1 TX3"), TX_WIDGETS("MIXER1 TX4"), TX_WIDGETS("MIXER1 TX5"), + WIDGETS("OPE1", t210_ope1_tx), + WIDGETS("OPE2", t210_ope2_tx), }; static const struct snd_soc_dapm_widget tegra186_ahub_widgets[] = { @@ -813,6 +875,20 @@ static const struct snd_soc_dapm_widget tegra186_ahub_widgets[] = { TX_WIDGETS("MIXER1 TX3"), TX_WIDGETS("MIXER1 TX4"), TX_WIDGETS("MIXER1 TX5"), + WIDGETS("ASRC1 RX1", t186_asrc11_tx), + WIDGETS("ASRC1 RX2", t186_asrc12_tx), + WIDGETS("ASRC1 RX3", t186_asrc13_tx), + WIDGETS("ASRC1 RX4", t186_asrc14_tx), + WIDGETS("ASRC1 RX5", t186_asrc15_tx), + WIDGETS("ASRC1 RX6", t186_asrc16_tx), + WIDGETS("ASRC1 RX7", t186_asrc17_tx), + TX_WIDGETS("ASRC1 TX1"), + TX_WIDGETS("ASRC1 TX2"), + TX_WIDGETS("ASRC1 TX3"), + TX_WIDGETS("ASRC1 TX4"), + TX_WIDGETS("ASRC1 TX5"), + TX_WIDGETS("ASRC1 TX6"), + WIDGETS("OPE1", t186_ope1_tx), }; static const struct snd_soc_dapm_widget tegra234_ahub_widgets[] = { @@ -909,6 +985,20 @@ static const struct snd_soc_dapm_widget tegra234_ahub_widgets[] = { TX_WIDGETS("MIXER1 TX3"), TX_WIDGETS("MIXER1 TX4"), TX_WIDGETS("MIXER1 TX5"), + WIDGETS("ASRC1 RX1", t234_asrc11_tx), + WIDGETS("ASRC1 RX2", t234_asrc12_tx), + WIDGETS("ASRC1 RX3", t234_asrc13_tx), + WIDGETS("ASRC1 RX4", t234_asrc14_tx), + WIDGETS("ASRC1 RX5", t234_asrc15_tx), + WIDGETS("ASRC1 RX6", t234_asrc16_tx), + WIDGETS("ASRC1 RX7", t234_asrc17_tx), + TX_WIDGETS("ASRC1 TX1"), + TX_WIDGETS("ASRC1 TX2"), + TX_WIDGETS("ASRC1 TX3"), + TX_WIDGETS("ASRC1 TX4"), + TX_WIDGETS("ASRC1 TX5"), + TX_WIDGETS("ASRC1 TX6"), + WIDGETS("OPE1", t186_ope1_tx), }; #define TEGRA_COMMON_MUX_ROUTES(name) \ @@ -951,7 +1041,11 @@ static const struct snd_soc_dapm_widget tegra234_ahub_widgets[] = { { name " Mux", "MIXER1 TX2", "MIXER1 TX2 XBAR-RX" }, \ { name " Mux", "MIXER1 TX3", "MIXER1 TX3 XBAR-RX" }, \ { name " Mux", "MIXER1 TX4", "MIXER1 TX4 XBAR-RX" }, \ - { name " Mux", "MIXER1 TX5", "MIXER1 TX5 XBAR-RX" }, + { name " Mux", "MIXER1 TX5", "MIXER1 TX5 XBAR-RX" }, \ + { name " Mux", "OPE1", "OPE1 XBAR-RX" }, + +#define TEGRA210_ONLY_MUX_ROUTES(name) \ + { name " Mux", "OPE2", "OPE2 XBAR-RX" }, #define TEGRA186_ONLY_MUX_ROUTES(name) \ { name " Mux", "ADMAIF11", "ADMAIF11 XBAR-RX" }, \ @@ -975,12 +1069,19 @@ static const struct snd_soc_dapm_widget tegra234_ahub_widgets[] = { { name " Mux", "ADX4 TX1", "ADX4 TX1 XBAR-RX" }, \ { name " Mux", "ADX4 TX2", "ADX4 TX2 XBAR-RX" }, \ { name " Mux", "ADX4 TX3", "ADX4 TX3 XBAR-RX" }, \ - { name " Mux", "ADX4 TX4", "ADX4 TX4 XBAR-RX" }, + { name " Mux", "ADX4 TX4", "ADX4 TX4 XBAR-RX" }, \ + { name " Mux", "ASRC1 TX1", "ASRC1 TX1 XBAR-RX" }, \ + { name " Mux", "ASRC1 TX2", "ASRC1 TX2 XBAR-RX" }, \ + { name " Mux", "ASRC1 TX3", "ASRC1 TX3 XBAR-RX" }, \ + { name " Mux", "ASRC1 TX4", "ASRC1 TX4 XBAR-RX" }, \ + { name " Mux", "ASRC1 TX5", "ASRC1 TX5 XBAR-RX" }, \ + { name " Mux", "ASRC1 TX6", "ASRC1 TX6 XBAR-RX" }, -#define TEGRA210_MUX_ROUTES(name) \ - TEGRA_COMMON_MUX_ROUTES(name) +#define TEGRA210_MUX_ROUTES(name) \ + TEGRA_COMMON_MUX_ROUTES(name) \ + TEGRA210_ONLY_MUX_ROUTES(name) -#define TEGRA186_MUX_ROUTES(name) \ +#define TEGRA186_MUX_ROUTES(name) \ TEGRA_COMMON_MUX_ROUTES(name) \ TEGRA186_ONLY_MUX_ROUTES(name) @@ -1048,6 +1149,8 @@ static const struct snd_soc_dapm_route tegra210_ahub_routes[] = { TEGRA210_MUX_ROUTES("MIXER1 RX8") TEGRA210_MUX_ROUTES("MIXER1 RX9") TEGRA210_MUX_ROUTES("MIXER1 RX10") + TEGRA210_MUX_ROUTES("OPE1") + TEGRA210_MUX_ROUTES("OPE2") }; static const struct snd_soc_dapm_route tegra186_ahub_routes[] = { @@ -1135,6 +1238,14 @@ static const struct snd_soc_dapm_route tegra186_ahub_routes[] = { TEGRA186_MUX_ROUTES("MIXER1 RX8") TEGRA186_MUX_ROUTES("MIXER1 RX9") TEGRA186_MUX_ROUTES("MIXER1 RX10") + TEGRA186_MUX_ROUTES("ASRC1 RX1") + TEGRA186_MUX_ROUTES("ASRC1 RX2") + TEGRA186_MUX_ROUTES("ASRC1 RX3") + TEGRA186_MUX_ROUTES("ASRC1 RX4") + TEGRA186_MUX_ROUTES("ASRC1 RX5") + TEGRA186_MUX_ROUTES("ASRC1 RX6") + TEGRA186_MUX_ROUTES("ASRC1 RX7") + TEGRA186_MUX_ROUTES("OPE1") }; static const struct snd_soc_component_driver tegra210_ahub_component = { diff --git a/sound/soc/tegra/tegra210_i2s.c b/sound/soc/tegra/tegra210_i2s.c index 9552bbb939..39ffa4d76b 100644 --- a/sound/soc/tegra/tegra210_i2s.c +++ b/sound/soc/tegra/tegra210_i2s.c @@ -214,11 +214,11 @@ static int tegra210_i2s_set_fmt(struct snd_soc_dai *dai, unsigned int mask, val; mask = I2S_CTRL_MASTER_EN_MASK; - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: val = 0; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BP_FP: val = I2S_CTRL_MASTER_EN; break; default: @@ -803,7 +803,6 @@ static const struct snd_soc_component_driver tegra210_i2s_cmpnt = { .num_dapm_routes = ARRAY_SIZE(tegra210_i2s_routes), .controls = tegra210_i2s_controls, .num_controls = ARRAY_SIZE(tegra210_i2s_controls), - .non_legacy_dai_naming = 1, }; static bool tegra210_i2s_wr_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/tegra/tegra30_i2s.c b/sound/soc/tegra/tegra30_i2s.c index 084a533bf4..10cd37096f 100644 --- a/sound/soc/tegra/tegra30_i2s.c +++ b/sound/soc/tegra/tegra30_i2s.c @@ -87,11 +87,11 @@ static int tegra30_i2s_set_fmt(struct snd_soc_dai *dai, } mask |= TEGRA30_I2S_CTRL_MASTER_ENABLE; - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: val |= TEGRA30_I2S_CTRL_MASTER_ENABLE; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: break; default: return -EINVAL; @@ -331,7 +331,8 @@ static const struct snd_soc_dai_driver tegra30_i2s_dai_template = { }; static const struct snd_soc_component_driver tegra30_i2s_component = { - .name = DRV_NAME, + .name = DRV_NAME, + .legacy_dai_naming = 1, }; static bool tegra30_i2s_wr_rd_reg(struct device *dev, unsigned int reg) diff --git a/sound/soc/tegra/tegra_asoc_machine.c b/sound/soc/tegra/tegra_asoc_machine.c index a73404879a..78faa8bcae 100644 --- a/sound/soc/tegra/tegra_asoc_machine.c +++ b/sound/soc/tegra/tegra_asoc_machine.c @@ -133,11 +133,11 @@ int tegra_asoc_machine_init(struct snd_soc_pcm_runtime *rtd) else jack_name = "Headphones Jack"; - err = snd_soc_card_jack_new(card, jack_name, - SND_JACK_HEADPHONE, - &tegra_machine_hp_jack, - tegra_machine_hp_jack_pins, - ARRAY_SIZE(tegra_machine_hp_jack_pins)); + err = snd_soc_card_jack_new_pins(card, jack_name, + SND_JACK_HEADPHONE, + &tegra_machine_hp_jack, + tegra_machine_hp_jack_pins, + ARRAY_SIZE(tegra_machine_hp_jack_pins)); if (err) { dev_err(rtd->dev, "Headphones Jack creation failed: %d\n", err); @@ -153,11 +153,11 @@ int tegra_asoc_machine_init(struct snd_soc_pcm_runtime *rtd) } if (machine->gpiod_hp_det && machine->asoc->add_headset_jack) { - err = snd_soc_card_jack_new(card, "Headset Jack", - SND_JACK_HEADSET, - &tegra_machine_headset_jack, - tegra_machine_headset_jack_pins, - ARRAY_SIZE(tegra_machine_headset_jack_pins)); + err = snd_soc_card_jack_new_pins(card, "Headset Jack", + SND_JACK_HEADSET, + &tegra_machine_headset_jack, + tegra_machine_headset_jack_pins, + ARRAY_SIZE(tegra_machine_headset_jack_pins)); if (err) { dev_err(rtd->dev, "Headset Jack creation failed: %d\n", err); @@ -173,11 +173,11 @@ int tegra_asoc_machine_init(struct snd_soc_pcm_runtime *rtd) } if (machine->gpiod_mic_det && machine->asoc->add_mic_jack) { - err = snd_soc_card_jack_new(rtd->card, "Mic Jack", - SND_JACK_MICROPHONE, - &tegra_machine_mic_jack, - tegra_machine_mic_jack_pins, - ARRAY_SIZE(tegra_machine_mic_jack_pins)); + err = snd_soc_card_jack_new_pins(rtd->card, "Mic Jack", + SND_JACK_MICROPHONE, + &tegra_machine_mic_jack, + tegra_machine_mic_jack_pins, + ARRAY_SIZE(tegra_machine_mic_jack_pins)); if (err) { dev_err(rtd->dev, "Mic Jack creation failed: %d\n", err); return err; diff --git a/sound/soc/tegra/tegra_wm8903.c b/sound/soc/tegra/tegra_wm8903.c index 5751fb398c..b3cd0a34da 100644 --- a/sound/soc/tegra/tegra_wm8903.c +++ b/sound/soc/tegra/tegra_wm8903.c @@ -79,11 +79,11 @@ static int tegra_wm8903_init(struct snd_soc_pcm_runtime *rtd) struct snd_soc_component *component = codec_dai->component; int shrt = 0; - err = snd_soc_card_jack_new(rtd->card, "Mic Jack", - SND_JACK_MICROPHONE, - machine->mic_jack, - tegra_wm8903_mic_jack_pins, - ARRAY_SIZE(tegra_wm8903_mic_jack_pins)); + err = snd_soc_card_jack_new_pins(rtd->card, "Mic Jack", + SND_JACK_MICROPHONE, + machine->mic_jack, + tegra_wm8903_mic_jack_pins, + ARRAY_SIZE(tegra_wm8903_mic_jack_pins)); if (err) { dev_err(rtd->dev, "Mic Jack creation failed: %d\n", err); return err; diff --git a/sound/soc/ti/ams-delta.c b/sound/soc/ti/ams-delta.c index b1a32545ba..438e2fa843 100644 --- a/sound/soc/ti/ams-delta.c +++ b/sound/soc/ti/ams-delta.c @@ -471,8 +471,8 @@ static int ams_delta_cx20442_init(struct snd_soc_pcm_runtime *rtd) /* Add hook switch - can be used to control the codec from userspace * even if line discipline fails */ - ret = snd_soc_card_jack_new(card, "hook_switch", SND_JACK_HEADSET, - &ams_delta_hook_switch, NULL, 0); + ret = snd_soc_card_jack_new_pins(card, "hook_switch", SND_JACK_HEADSET, + &ams_delta_hook_switch, NULL, 0); if (ret) dev_warn(card->dev, "Failed to allocate resources for hook switch, " diff --git a/sound/soc/ti/davinci-i2s.c b/sound/soc/ti/davinci-i2s.c index 0363a088d2..e6e77a5f3c 100644 --- a/sound/soc/ti/davinci-i2s.c +++ b/sound/soc/ti/davinci-i2s.c @@ -230,15 +230,15 @@ static int davinci_i2s_set_dai_fmt(struct snd_soc_dai *cpu_dai, dev->fmt = fmt; /* set master/slave audio interface */ - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* cpu is master */ pcr = DAVINCI_MCBSP_PCR_FSXM | DAVINCI_MCBSP_PCR_FSRM | DAVINCI_MCBSP_PCR_CLKXM | DAVINCI_MCBSP_PCR_CLKRM; break; - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FP: pcr = DAVINCI_MCBSP_PCR_FSRM | DAVINCI_MCBSP_PCR_FSXM; /* * Selection of the clock input pin that is the @@ -260,7 +260,7 @@ static int davinci_i2s_set_dai_fmt(struct snd_soc_dai *cpu_dai, } break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* codec is master */ pcr = 0; break; @@ -395,12 +395,12 @@ static int davinci_i2s_hw_params(struct snd_pcm_substream *substream, davinci_mcbsp_write_reg(dev, DAVINCI_MCBSP_SPCR_REG, spcr); } - master = dev->fmt & SND_SOC_DAIFMT_MASTER_MASK; + master = dev->fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK; fmt = params_format(params); mcbsp_word_length = asp_word_length[fmt]; switch (master) { - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: freq = clk_get_rate(dev->clk); srgr = DAVINCI_MCBSP_SRGR_FSGM | DAVINCI_MCBSP_SRGR_CLKSM; @@ -426,7 +426,7 @@ static int davinci_i2s_hw_params(struct snd_pcm_substream *substream, clk_div &= 0xFF; srgr |= clk_div; break; - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FP: srgr = DAVINCI_MCBSP_SRGR_FSGM; clk_div = dev->clk_div - 1; srgr |= DAVINCI_MCBSP_SRGR_FWID(mcbsp_word_length * 8 - 1); @@ -434,7 +434,7 @@ static int davinci_i2s_hw_params(struct snd_pcm_substream *substream, clk_div &= 0xFF; srgr |= clk_div; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* Clock and frame sync given from external sources */ i = hw_param_interval(params, SNDRV_PCM_HW_PARAM_SAMPLE_BITS); srgr = DAVINCI_MCBSP_SRGR_FSGM; @@ -473,15 +473,15 @@ static int davinci_i2s_hw_params(struct snd_pcm_substream *substream, fmt = double_fmt[fmt]; } switch (master) { - case SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_CBS_CFM: + case SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_BP_FC: rcr |= DAVINCI_MCBSP_RCR_RFRLEN2(0); xcr |= DAVINCI_MCBSP_XCR_XFRLEN2(0); rcr |= DAVINCI_MCBSP_RCR_RPHASE; xcr |= DAVINCI_MCBSP_XCR_XPHASE; break; - case SND_SOC_DAIFMT_CBM_CFM: - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FC: + case SND_SOC_DAIFMT_BC_FP: rcr |= DAVINCI_MCBSP_RCR_RFRLEN2(element_cnt - 1); xcr |= DAVINCI_MCBSP_XCR_XFRLEN2(element_cnt - 1); break; @@ -492,13 +492,13 @@ static int davinci_i2s_hw_params(struct snd_pcm_substream *substream, mcbsp_word_length = asp_word_length[fmt]; switch (master) { - case SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_CBS_CFM: + case SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_BP_FC: rcr |= DAVINCI_MCBSP_RCR_RFRLEN1(0); xcr |= DAVINCI_MCBSP_XCR_XFRLEN1(0); break; - case SND_SOC_DAIFMT_CBM_CFM: - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FC: + case SND_SOC_DAIFMT_BC_FP: rcr |= DAVINCI_MCBSP_RCR_RFRLEN1(element_cnt - 1); xcr |= DAVINCI_MCBSP_XCR_XFRLEN1(element_cnt - 1); break; @@ -640,7 +640,8 @@ static struct snd_soc_dai_driver davinci_i2s_dai = { }; static const struct snd_soc_component_driver davinci_i2s_component = { - .name = DRV_NAME, + .name = DRV_NAME, + .legacy_dai_naming = 1, }; static int davinci_i2s_probe(struct platform_device *pdev) diff --git a/sound/soc/ti/davinci-mcasp.c b/sound/soc/ti/davinci-mcasp.c index 2c146b91fc..ca5d1bb6ac 100644 --- a/sound/soc/ti/davinci-mcasp.c +++ b/sound/soc/ti/davinci-mcasp.c @@ -492,8 +492,8 @@ static int davinci_mcasp_set_dai_fmt(struct snd_soc_dai *cpu_dai, mcasp_mod_bits(mcasp, DAVINCI_MCASP_RXFMT_REG, FSRDLY(data_delay), FSRDLY(3)); - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* codec is clock and frame slave */ mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE); mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE); @@ -510,7 +510,7 @@ static int davinci_mcasp_set_dai_fmt(struct snd_soc_dai *cpu_dai, mcasp->bclk_master = 1; break; - case SND_SOC_DAIFMT_CBS_CFM: + case SND_SOC_DAIFMT_BP_FC: /* codec is clock slave and frame master */ mcasp_set_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE); mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE); @@ -527,7 +527,7 @@ static int davinci_mcasp_set_dai_fmt(struct snd_soc_dai *cpu_dai, mcasp->bclk_master = 1; break; - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FP: /* codec is clock master and frame slave */ mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE); mcasp_set_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE); @@ -544,7 +544,7 @@ static int davinci_mcasp_set_dai_fmt(struct snd_soc_dai *cpu_dai, mcasp->bclk_master = 0; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* codec is clock and frame master */ mcasp_clr_bits(mcasp, DAVINCI_MCASP_ACLKXCTL_REG, ACLKXE); mcasp_clr_bits(mcasp, DAVINCI_MCASP_TXFMCTL_REG, AFSXE); @@ -1765,7 +1765,8 @@ static struct snd_soc_dai_driver davinci_mcasp_dai[] = { }; static const struct snd_soc_component_driver davinci_mcasp_component = { - .name = "davinci-mcasp", + .name = "davinci-mcasp", + .legacy_dai_naming = 1, }; /* Some HW specific values and defaults. The rest is filled in from DT. */ @@ -2047,6 +2048,8 @@ static int davinci_mcasp_get_dma_type(struct davinci_mcasp *mcasp) return PCM_SDMA; else if (strstr(tmp, "udmap")) return PCM_UDMA; + else if (strstr(tmp, "bcdma")) + return PCM_UDMA; return PCM_EDMA; } @@ -2109,8 +2112,7 @@ static int davinci_mcasp_gpio_request(struct gpio_chip *chip, unsigned offset) } /* Do not change the PIN yet */ - - return pm_runtime_get_sync(mcasp->dev); + return pm_runtime_resume_and_get(mcasp->dev); } static void davinci_mcasp_gpio_free(struct gpio_chip *chip, unsigned offset) diff --git a/sound/soc/ti/davinci-vcif.c b/sound/soc/ti/davinci-vcif.c index f810123cc4..36fa97e2b9 100644 --- a/sound/soc/ti/davinci-vcif.c +++ b/sound/soc/ti/davinci-vcif.c @@ -185,7 +185,8 @@ static struct snd_soc_dai_driver davinci_vcif_dai = { }; static const struct snd_soc_component_driver davinci_vcif_component = { - .name = "davinci-vcif", + .name = "davinci-vcif", + .legacy_dai_naming = 1, }; static int davinci_vcif_probe(struct platform_device *pdev) diff --git a/sound/soc/ti/j721e-evm.c b/sound/soc/ti/j721e-evm.c index 4077e15ec4..6a969874c9 100644 --- a/sound/soc/ti/j721e-evm.c +++ b/sound/soc/ti/j721e-evm.c @@ -630,17 +630,18 @@ static int j721e_soc_probe_cpb(struct j721e_priv *priv, int *link_idx, codec_node = of_parse_phandle(node, "ti,cpb-codec", 0); if (!codec_node) { dev_err(priv->dev, "CPB codec node is not provided\n"); - return -EINVAL; + ret = -EINVAL; + goto put_dai_node; } domain = &priv->audio_domains[J721E_AUDIO_DOMAIN_CPB]; ret = j721e_get_clocks(priv->dev, &domain->codec, "cpb-codec-scki"); if (ret) - return ret; + goto put_codec_node; ret = j721e_get_clocks(priv->dev, &domain->mcasp, "cpb-mcasp-auxclk"); if (ret) - return ret; + goto put_codec_node; /* * Common Processor Board, two links @@ -650,8 +651,10 @@ static int j721e_soc_probe_cpb(struct j721e_priv *priv, int *link_idx, comp_count = 6; compnent = devm_kzalloc(priv->dev, comp_count * sizeof(*compnent), GFP_KERNEL); - if (!compnent) - return -ENOMEM; + if (!compnent) { + ret = -ENOMEM; + goto put_codec_node; + } comp_idx = 0; priv->dai_links[*link_idx].cpus = &compnent[comp_idx++]; @@ -702,6 +705,12 @@ static int j721e_soc_probe_cpb(struct j721e_priv *priv, int *link_idx, (*conf_idx)++; return 0; + +put_codec_node: + of_node_put(codec_node); +put_dai_node: + of_node_put(dai_node); + return ret; } static int j721e_soc_probe_ivi(struct j721e_priv *priv, int *link_idx, @@ -726,23 +735,25 @@ static int j721e_soc_probe_ivi(struct j721e_priv *priv, int *link_idx, codeca_node = of_parse_phandle(node, "ti,ivi-codec-a", 0); if (!codeca_node) { dev_err(priv->dev, "IVI codec-a node is not provided\n"); - return -EINVAL; + ret = -EINVAL; + goto put_dai_node; } codecb_node = of_parse_phandle(node, "ti,ivi-codec-b", 0); if (!codecb_node) { dev_warn(priv->dev, "IVI codec-b node is not provided\n"); - return 0; + ret = 0; + goto put_codeca_node; } domain = &priv->audio_domains[J721E_AUDIO_DOMAIN_IVI]; ret = j721e_get_clocks(priv->dev, &domain->codec, "ivi-codec-scki"); if (ret) - return ret; + goto put_codecb_node; ret = j721e_get_clocks(priv->dev, &domain->mcasp, "ivi-mcasp-auxclk"); if (ret) - return ret; + goto put_codecb_node; /* * IVI extension, two links @@ -754,8 +765,10 @@ static int j721e_soc_probe_ivi(struct j721e_priv *priv, int *link_idx, comp_count = 8; compnent = devm_kzalloc(priv->dev, comp_count * sizeof(*compnent), GFP_KERNEL); - if (!compnent) - return -ENOMEM; + if (!compnent) { + ret = -ENOMEM; + goto put_codecb_node; + } comp_idx = 0; priv->dai_links[*link_idx].cpus = &compnent[comp_idx++]; @@ -816,6 +829,15 @@ static int j721e_soc_probe_ivi(struct j721e_priv *priv, int *link_idx, (*conf_idx)++; return 0; + + +put_codecb_node: + of_node_put(codecb_node); +put_codeca_node: + of_node_put(codeca_node); +put_dai_node: + of_node_put(dai_node); + return ret; } static int j721e_soc_probe(struct platform_device *pdev) diff --git a/sound/soc/ti/omap-abe-twl6040.c b/sound/soc/ti/omap-abe-twl6040.c index da809c7f25..805ffbf890 100644 --- a/sound/soc/ti/omap-abe-twl6040.c +++ b/sound/soc/ti/omap-abe-twl6040.c @@ -182,10 +182,10 @@ static int omap_abe_twl6040_init(struct snd_soc_pcm_runtime *rtd) /* Headset jack detection only if it is supported */ if (priv->jack_detection) { - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", - SND_JACK_HEADSET, &hs_jack, - hs_jack_pins, - ARRAY_SIZE(hs_jack_pins)); + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", + SND_JACK_HEADSET, &hs_jack, + hs_jack_pins, + ARRAY_SIZE(hs_jack_pins)); if (ret) return ret; diff --git a/sound/soc/ti/omap-dmic.c b/sound/soc/ti/omap-dmic.c index f3eed20611..825c70a443 100644 --- a/sound/soc/ti/omap-dmic.c +++ b/sound/soc/ti/omap-dmic.c @@ -453,7 +453,8 @@ static struct snd_soc_dai_driver omap_dmic_dai = { }; static const struct snd_soc_component_driver omap_dmic_component = { - .name = "omap-dmic", + .name = "omap-dmic", + .legacy_dai_naming = 1, }; static int asoc_dmic_probe(struct platform_device *pdev) diff --git a/sound/soc/ti/omap-hdmi.c b/sound/soc/ti/omap-hdmi.c index 3328c02f93..0dc0475670 100644 --- a/sound/soc/ti/omap-hdmi.c +++ b/sound/soc/ti/omap-hdmi.c @@ -275,6 +275,7 @@ static const struct snd_soc_dai_ops hdmi_dai_ops = { static const struct snd_soc_component_driver omap_hdmi_component = { .name = "omapdss_hdmi", + .legacy_dai_naming = 1, }; static struct snd_soc_dai_driver omap5_hdmi_dai = { diff --git a/sound/soc/ti/omap-mcbsp-priv.h b/sound/soc/ti/omap-mcbsp-priv.h index 7865cda4bf..da519ea1f3 100644 --- a/sound/soc/ti/omap-mcbsp-priv.h +++ b/sound/soc/ti/omap-mcbsp-priv.h @@ -316,8 +316,6 @@ static inline int omap_mcbsp_read(struct omap_mcbsp *mcbsp, u16 reg, /* Sidetone specific API */ int omap_mcbsp_st_init(struct platform_device *pdev); -void omap_mcbsp_st_cleanup(struct platform_device *pdev); - int omap_mcbsp_st_start(struct omap_mcbsp *mcbsp); int omap_mcbsp_st_stop(struct omap_mcbsp *mcbsp); diff --git a/sound/soc/ti/omap-mcbsp-st.c b/sound/soc/ti/omap-mcbsp-st.c index 0bc7d26c66..7e8179cae9 100644 --- a/sound/soc/ti/omap-mcbsp-st.c +++ b/sound/soc/ti/omap-mcbsp-st.c @@ -347,7 +347,7 @@ int omap_mcbsp_st_init(struct platform_device *pdev) if (!st_data) return -ENOMEM; - st_data->mcbsp_iclk = clk_get(mcbsp->dev, "ick"); + st_data->mcbsp_iclk = devm_clk_get(mcbsp->dev, "ick"); if (IS_ERR(st_data->mcbsp_iclk)) { dev_warn(mcbsp->dev, "Failed to get ick, sidetone might be broken\n"); @@ -359,7 +359,7 @@ int omap_mcbsp_st_init(struct platform_device *pdev) if (!st_data->io_base_st) return -ENOMEM; - ret = sysfs_create_group(&mcbsp->dev->kobj, &sidetone_attr_group); + ret = devm_device_add_group(mcbsp->dev, &sidetone_attr_group); if (ret) return ret; @@ -368,16 +368,6 @@ int omap_mcbsp_st_init(struct platform_device *pdev) return 0; } -void omap_mcbsp_st_cleanup(struct platform_device *pdev) -{ - struct omap_mcbsp *mcbsp = platform_get_drvdata(pdev); - - if (mcbsp->st_data) { - sysfs_remove_group(&mcbsp->dev->kobj, &sidetone_attr_group); - clk_put(mcbsp->st_data->mcbsp_iclk); - } -} - static int omap_mcbsp_st_info_volsw(struct snd_kcontrol *kcontrol, struct snd_ctl_elem_info *uinfo) { diff --git a/sound/soc/ti/omap-mcbsp.c b/sound/soc/ti/omap-mcbsp.c index 4479d74f0a..c4ac1f30b9 100644 --- a/sound/soc/ti/omap-mcbsp.c +++ b/sound/soc/ti/omap-mcbsp.c @@ -702,8 +702,7 @@ static int omap_mcbsp_init(struct platform_device *pdev) mcbsp->max_tx_thres = max_thres(mcbsp) - 0x10; mcbsp->max_rx_thres = max_thres(mcbsp) - 0x10; - ret = sysfs_create_group(&mcbsp->dev->kobj, - &additional_attr_group); + ret = devm_device_add_group(mcbsp->dev, &additional_attr_group); if (ret) { dev_err(mcbsp->dev, "Unable to create additional controls\n"); @@ -711,16 +710,7 @@ static int omap_mcbsp_init(struct platform_device *pdev) } } - ret = omap_mcbsp_st_init(pdev); - if (ret) - goto err_st; - - return 0; - -err_st: - if (mcbsp->pdata->buffer_size) - sysfs_remove_group(&mcbsp->dev->kobj, &additional_attr_group); - return ret; + return omap_mcbsp_st_init(pdev); } /* @@ -1036,8 +1026,8 @@ static int omap_mcbsp_dai_hw_params(struct snd_pcm_substream *substream, /* In McBSP master modes, FRAME (i.e. sample rate) is generated * by _counting_ BCLKs. Calculate frame size in BCLKs */ - master = mcbsp->fmt & SND_SOC_DAIFMT_MASTER_MASK; - if (master == SND_SOC_DAIFMT_CBS_CFS) { + master = mcbsp->fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK; + if (master == SND_SOC_DAIFMT_BP_FP) { div = mcbsp->clk_div ? mcbsp->clk_div : 1; framesize = (mcbsp->in_freq / div) / params_rate(params); @@ -1136,20 +1126,20 @@ static int omap_mcbsp_dai_set_dai_fmt(struct snd_soc_dai *cpu_dai, return -EINVAL; } - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBS_CFS: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BP_FP: /* McBSP master. Set FS and bit clocks as outputs */ regs->pcr0 |= FSXM | FSRM | CLKXM | CLKRM; /* Sample rate generator drives the FS */ regs->srgr2 |= FSGM; break; - case SND_SOC_DAIFMT_CBM_CFS: + case SND_SOC_DAIFMT_BC_FP: /* McBSP slave. FS clock as output */ regs->srgr2 |= FSGM; regs->pcr0 |= FSXM | FSRM; break; - case SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_BC_FC: /* McBSP slave */ break; default: @@ -1317,7 +1307,8 @@ static struct snd_soc_dai_driver omap_mcbsp_dai = { }; static const struct snd_soc_component_driver omap_mcbsp_component = { - .name = "omap-mcbsp", + .name = "omap-mcbsp", + .legacy_dai_naming = 1, }; static struct omap_mcbsp_platform_data omap2420_pdata = { @@ -1431,11 +1422,6 @@ static int asoc_mcbsp_remove(struct platform_device *pdev) if (cpu_latency_qos_request_active(&mcbsp->pm_qos_req)) cpu_latency_qos_remove_request(&mcbsp->pm_qos_req); - if (mcbsp->pdata->buffer_size) - sysfs_remove_group(&mcbsp->dev->kobj, &additional_attr_group); - - omap_mcbsp_st_cleanup(pdev); - return 0; } diff --git a/sound/soc/ti/omap-mcpdm.c b/sound/soc/ti/omap-mcpdm.c index fafb2998ad..0b18a7bfd3 100644 --- a/sound/soc/ti/omap-mcpdm.c +++ b/sound/soc/ti/omap-mcpdm.c @@ -524,9 +524,10 @@ static struct snd_soc_dai_driver omap_mcpdm_dai = { }; static const struct snd_soc_component_driver omap_mcpdm_component = { - .name = "omap-mcpdm", - .suspend = omap_mcpdm_suspend, - .resume = omap_mcpdm_resume, + .name = "omap-mcpdm", + .suspend = omap_mcpdm_suspend, + .resume = omap_mcpdm_resume, + .legacy_dai_naming = 1, }; void omap_mcpdm_configure_dn_offsets(struct snd_soc_pcm_runtime *rtd, diff --git a/sound/soc/ti/omap-twl4030.c b/sound/soc/ti/omap-twl4030.c index 1da05a6cdc..950eec4450 100644 --- a/sound/soc/ti/omap-twl4030.c +++ b/sound/soc/ti/omap-twl4030.c @@ -155,10 +155,10 @@ static int omap_twl4030_init(struct snd_soc_pcm_runtime *rtd) if (priv->jack_detect > 0) { hs_jack_gpios[0].gpio = priv->jack_detect; - ret = snd_soc_card_jack_new(rtd->card, "Headset Jack", - SND_JACK_HEADSET, &priv->hs_jack, - hs_jack_pins, - ARRAY_SIZE(hs_jack_pins)); + ret = snd_soc_card_jack_new_pins(rtd->card, "Headset Jack", + SND_JACK_HEADSET, + &priv->hs_jack, hs_jack_pins, + ARRAY_SIZE(hs_jack_pins)); if (ret) return ret; diff --git a/sound/soc/ti/osk5912.c b/sound/soc/ti/osk5912.c index 40e29dda7e..2790c8915f 100644 --- a/sound/soc/ti/osk5912.c +++ b/sound/soc/ti/osk5912.c @@ -27,12 +27,12 @@ static struct clk *tlv320aic23_mclk; static int osk_startup(struct snd_pcm_substream *substream) { - return clk_enable(tlv320aic23_mclk); + return clk_prepare_enable(tlv320aic23_mclk); } static void osk_shutdown(struct snd_pcm_substream *substream) { - clk_disable(tlv320aic23_mclk); + clk_disable_unprepare(tlv320aic23_mclk); } static int osk_hw_params(struct snd_pcm_substream *substream, diff --git a/sound/soc/ti/rx51.c b/sound/soc/ti/rx51.c index a2629ccc1d..322c398d20 100644 --- a/sound/soc/ti/rx51.c +++ b/sound/soc/ti/rx51.c @@ -277,7 +277,7 @@ static int rx51_aic34_init(struct snd_soc_pcm_runtime *rtd) /* AV jack detection */ err = snd_soc_card_jack_new(rtd->card, "AV Jack", SND_JACK_HEADSET | SND_JACK_VIDEOOUT, - &rx51_av_jack, NULL, 0); + &rx51_av_jack); if (err) { dev_err(card->dev, "Failed to add AV Jack\n"); return err; diff --git a/sound/soc/uniphier/aio-compress.c b/sound/soc/uniphier/aio-compress.c index 0f76bc601c..7d1492c15b 100644 --- a/sound/soc/uniphier/aio-compress.c +++ b/sound/soc/uniphier/aio-compress.c @@ -139,7 +139,6 @@ static int uniphier_aio_compr_set_params(struct snd_soc_component *component, struct uniphier_aio *aio = uniphier_priv(asoc_rtd_to_cpu(rtd, 0)); struct uniphier_aio_sub *sub = &aio->sub[cstream->direction]; struct device *dev = &aio->chip->pdev->dev; - int ret; if (params->codec.id != SND_AUDIOCODEC_IEC61937) { dev_err(dev, "Codec ID is not supported(%d)\n", @@ -161,11 +160,7 @@ static int uniphier_aio_compr_set_params(struct snd_soc_component *component, aio_port_reset(sub); aio_src_reset(sub); - ret = uniphier_aio_compr_prepare(component, cstream); - if (ret) - return ret; - - return 0; + return uniphier_aio_compr_prepare(component, cstream); } static int uniphier_aio_compr_hw_free(struct snd_soc_component *component, diff --git a/sound/soc/uniphier/evea.c b/sound/soc/uniphier/evea.c index 96343d19a1..42403ae8e3 100644 --- a/sound/soc/uniphier/evea.c +++ b/sound/soc/uniphier/evea.c @@ -397,7 +397,6 @@ static struct snd_soc_component_driver soc_codec_evea = { .idle_bias_on = 1, .use_pmdown_time = 1, .endianness = 1, - .non_legacy_dai_naming = 1, }; static struct snd_soc_dai_driver soc_dai_evea[] = { diff --git a/sound/soc/ux500/mop500.c b/sound/soc/ux500/mop500.c index 4f41bb0ab2..fdd55d772b 100644 --- a/sound/soc/ux500/mop500.c +++ b/sound/soc/ux500/mop500.c @@ -4,8 +4,6 @@ * * Author: Ola Lilja (ola.o.lilja@stericsson.com) * for ST-Ericsson. - * - * License terms: */ #include diff --git a/sound/soc/ux500/mop500_ab8500.c b/sound/soc/ux500/mop500_ab8500.c index 3e654e708f..e5e73a2bd9 100644 --- a/sound/soc/ux500/mop500_ab8500.c +++ b/sound/soc/ux500/mop500_ab8500.c @@ -5,8 +5,6 @@ * Author: Ola Lilja , * Kristoffer Karlsson * for ST-Ericsson. - * - * License terms: */ #include @@ -433,12 +431,9 @@ void mop500_ab8500_remove(struct snd_soc_card *card) { struct mop500_ab8500_drvdata *drvdata = snd_soc_card_get_drvdata(card); - if (drvdata->clk_ptr_sysclk != NULL) - clk_put(drvdata->clk_ptr_sysclk); - if (drvdata->clk_ptr_ulpclk != NULL) - clk_put(drvdata->clk_ptr_ulpclk); - if (drvdata->clk_ptr_intclk != NULL) - clk_put(drvdata->clk_ptr_intclk); + clk_put(drvdata->clk_ptr_sysclk); + clk_put(drvdata->clk_ptr_ulpclk); + clk_put(drvdata->clk_ptr_intclk); snd_soc_card_set_drvdata(card, drvdata); } diff --git a/sound/soc/ux500/mop500_ab8500.h b/sound/soc/ux500/mop500_ab8500.h index 087ef246d8..98de80a9cc 100644 --- a/sound/soc/ux500/mop500_ab8500.h +++ b/sound/soc/ux500/mop500_ab8500.h @@ -4,8 +4,6 @@ * * Author: Ola Lilja * for ST-Ericsson. - * - * License terms: */ #ifndef MOP500_AB8500_H diff --git a/sound/soc/ux500/ux500_msp_dai.c b/sound/soc/ux500/ux500_msp_dai.c index 21052378a3..9d99ea6d7f 100644 --- a/sound/soc/ux500/ux500_msp_dai.c +++ b/sound/soc/ux500/ux500_msp_dai.c @@ -5,8 +5,6 @@ * Author: Ola Lilja , * Roger Nilsson * for ST-Ericsson. - * - * License terms: */ #include @@ -191,8 +189,8 @@ static int setup_clocking(struct snd_soc_dai *dai, return -EINVAL; } - switch (fmt & SND_SOC_DAIFMT_MASTER_MASK) { - case SND_SOC_DAIFMT_CBM_CFM: + switch (fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) { + case SND_SOC_DAIFMT_BC_FC: dev_dbg(dai->dev, "%s: Codec is master.\n", __func__); msp_config->iodelay = 0x20; @@ -204,7 +202,7 @@ static int setup_clocking(struct snd_soc_dai *dai, break; - case SND_SOC_DAIFMT_CBS_CFS: + case SND_SOC_DAIFMT_BP_FP: dev_dbg(dai->dev, "%s: Codec is slave.\n", __func__); msp_config->tx_clk_sel = TX_CLK_SEL_SRG; @@ -328,15 +326,15 @@ static int setup_msp_config(struct snd_pcm_substream *substream, dev_dbg(dai->dev, "%s: rate: %u, channels: %d.\n", __func__, runtime->rate, runtime->channels); switch (fmt & - (SND_SOC_DAIFMT_FORMAT_MASK | SND_SOC_DAIFMT_MASTER_MASK)) { - case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBS_CFS: + (SND_SOC_DAIFMT_FORMAT_MASK | SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK)) { + case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_BP_FP: dev_dbg(dai->dev, "%s: SND_SOC_DAIFMT_I2S.\n", __func__); msp_config->default_protdesc = 1; msp_config->protocol = MSP_I2S_PROTOCOL; break; - case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_BC_FC: dev_dbg(dai->dev, "%s: SND_SOC_DAIFMT_I2S.\n", __func__); msp_config->data_size = MSP_DATA_BITS_16; @@ -348,10 +346,10 @@ static int setup_msp_config(struct snd_pcm_substream *substream, break; - case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBM_CFM: - case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM: + case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_BC_FC: + case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_BC_FC: dev_dbg(dai->dev, "%s: PCM format.\n", __func__); msp_config->data_size = MSP_DATA_BITS_16; @@ -477,7 +475,7 @@ static int ux500_msp_dai_prepare(struct snd_pcm_substream *substream, } /* Set OPP-level */ - if ((drvdata->fmt & SND_SOC_DAIFMT_MASTER_MASK) && + if ((drvdata->fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) && (drvdata->msp->f_bitclk > 19200000)) { /* If the bit-clock is higher than 19.2MHz, Vape should be * run in 100% OPP. Only when bit-clock is used (MSP master) @@ -544,13 +542,13 @@ static int ux500_msp_dai_set_dai_fmt(struct snd_soc_dai *dai, dev_dbg(dai->dev, "%s: MSP %d: Enter.\n", __func__, dai->id); switch (fmt & (SND_SOC_DAIFMT_FORMAT_MASK | - SND_SOC_DAIFMT_MASTER_MASK)) { - case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_CBM_CFM: - case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_CBM_CFM: - case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBS_CFS: - case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_CBM_CFM: + SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK)) { + case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_I2S | SND_SOC_DAIFMT_BC_FC: + case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_DSP_B | SND_SOC_DAIFMT_BC_FC: + case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_BP_FP: + case SND_SOC_DAIFMT_DSP_A | SND_SOC_DAIFMT_BC_FC: break; default: @@ -731,7 +729,8 @@ static struct snd_soc_dai_driver ux500_msp_dai_drv = { }; static const struct snd_soc_component_driver ux500_msp_component = { - .name = "ux500-msp", + .name = "ux500-msp", + .legacy_dai_naming = 1, }; diff --git a/sound/soc/ux500/ux500_msp_dai.h b/sound/soc/ux500/ux500_msp_dai.h index fcd4b26f5d..30bf708381 100644 --- a/sound/soc/ux500/ux500_msp_dai.h +++ b/sound/soc/ux500/ux500_msp_dai.h @@ -5,8 +5,6 @@ * Author: Ola Lilja , * Roger Nilsson * for ST-Ericsson. - * - * License terms: */ #ifndef UX500_msp_dai_H diff --git a/sound/soc/ux500/ux500_msp_i2s.c b/sound/soc/ux500/ux500_msp_i2s.c index fd0b88bb79..d113411a19 100644 --- a/sound/soc/ux500/ux500_msp_i2s.c +++ b/sound/soc/ux500/ux500_msp_i2s.c @@ -6,8 +6,6 @@ * Roger Nilsson , * Sandeep Kaushik * for ST-Ericsson. - * - * License terms: */ #include diff --git a/sound/soc/ux500/ux500_msp_i2s.h b/sound/soc/ux500/ux500_msp_i2s.h index 756b3973af..d45b5e2831 100644 --- a/sound/soc/ux500/ux500_msp_i2s.h +++ b/sound/soc/ux500/ux500_msp_i2s.h @@ -4,8 +4,6 @@ * * Author: Ola Lilja , * for ST-Ericsson. - * - * License terms: */ diff --git a/sound/soc/ux500/ux500_pcm.c b/sound/soc/ux500/ux500_pcm.c index 18191084b8..d3802e5ef1 100644 --- a/sound/soc/ux500/ux500_pcm.c +++ b/sound/soc/ux500/ux500_pcm.c @@ -5,8 +5,6 @@ * Author: Ola Lilja , * Roger Nilsson * for ST-Ericsson. - * - * License terms: */ #include diff --git a/sound/soc/ux500/ux500_pcm.h b/sound/soc/ux500/ux500_pcm.h index ff3ef7223d..bd4348ebf9 100644 --- a/sound/soc/ux500/ux500_pcm.h +++ b/sound/soc/ux500/ux500_pcm.h @@ -5,8 +5,6 @@ * Author: Ola Lilja , * Roger Nilsson * for ST-Ericsson. - * - * License terms: */ #ifndef UX500_PCM_H #define UX500_PCM_H diff --git a/sound/soc/xilinx/xlnx_formatter_pcm.c b/sound/soc/xilinx/xlnx_formatter_pcm.c index 5c4158069a..ff1fe62fea 100644 --- a/sound/soc/xilinx/xlnx_formatter_pcm.c +++ b/sound/soc/xilinx/xlnx_formatter_pcm.c @@ -575,14 +575,14 @@ static int xlnx_formatter_pcm_new(struct snd_soc_component *component, } static const struct snd_soc_component_driver xlnx_asoc_component = { - .name = DRV_NAME, - .set_sysclk = xlnx_formatter_set_sysclk, - .open = xlnx_formatter_pcm_open, - .close = xlnx_formatter_pcm_close, - .hw_params = xlnx_formatter_pcm_hw_params, - .trigger = xlnx_formatter_pcm_trigger, - .pointer = xlnx_formatter_pcm_pointer, - .pcm_construct = xlnx_formatter_pcm_new, + .name = DRV_NAME, + .set_sysclk = xlnx_formatter_set_sysclk, + .open = xlnx_formatter_pcm_open, + .close = xlnx_formatter_pcm_close, + .hw_params = xlnx_formatter_pcm_hw_params, + .trigger = xlnx_formatter_pcm_trigger, + .pointer = xlnx_formatter_pcm_pointer, + .pcm_construct = xlnx_formatter_pcm_new, }; static int xlnx_formatter_pcm_probe(struct platform_device *pdev) @@ -703,7 +703,7 @@ static int xlnx_formatter_pcm_remove(struct platform_device *pdev) dev_err(&pdev->dev, "audio formatter reset failed\n"); clk_disable_unprepare(adata->axi_clk); - return ret; + return 0; } static const struct of_device_id xlnx_formatter_pcm_of_match[] = { diff --git a/sound/soc/xilinx/xlnx_i2s.c b/sound/soc/xilinx/xlnx_i2s.c index 4cc6ee7c81..9de92d35e3 100644 --- a/sound/soc/xilinx/xlnx_i2s.c +++ b/sound/soc/xilinx/xlnx_i2s.c @@ -158,6 +158,7 @@ static const struct snd_soc_dai_ops xlnx_i2s_dai_ops = { static const struct snd_soc_component_driver xlnx_i2s_component = { .name = DRV_NAME, + .legacy_dai_naming = 1, }; static const struct of_device_id xlnx_i2s_of_match[] = { diff --git a/sound/soc/xilinx/xlnx_spdif.c b/sound/soc/xilinx/xlnx_spdif.c index cba0e868a7..7342048e98 100644 --- a/sound/soc/xilinx/xlnx_spdif.c +++ b/sound/soc/xilinx/xlnx_spdif.c @@ -226,6 +226,7 @@ static struct snd_soc_dai_driver xlnx_spdif_rx_dai = { static const struct snd_soc_component_driver xlnx_spdif_component = { .name = "xlnx-spdif", + .legacy_dai_naming = 1, }; static const struct of_device_id xlnx_spdif_of_match[] = { diff --git a/sound/soc/xtensa/xtfpga-i2s.c b/sound/soc/xtensa/xtfpga-i2s.c index aeb4b2c4d1..a8f156540b 100644 --- a/sound/soc/xtensa/xtfpga-i2s.c +++ b/sound/soc/xtensa/xtfpga-i2s.c @@ -339,7 +339,7 @@ static int xtfpga_i2s_set_fmt(struct snd_soc_dai *cpu_dai, { if ((fmt & SND_SOC_DAIFMT_INV_MASK) != SND_SOC_DAIFMT_NB_NF) return -EINVAL; - if ((fmt & SND_SOC_DAIFMT_MASTER_MASK) != SND_SOC_DAIFMT_CBS_CFS) + if ((fmt & SND_SOC_DAIFMT_CLOCK_PROVIDER_MASK) != SND_SOC_DAIFMT_BP_FP) return -EINVAL; if ((fmt & SND_SOC_DAIFMT_FORMAT_MASK) != SND_SOC_DAIFMT_I2S) return -EINVAL; @@ -475,19 +475,20 @@ static int xtfpga_pcm_new(struct snd_soc_component *component, } static const struct snd_soc_component_driver xtfpga_i2s_component = { - .name = DRV_NAME, - .open = xtfpga_pcm_open, - .close = xtfpga_pcm_close, - .hw_params = xtfpga_pcm_hw_params, - .trigger = xtfpga_pcm_trigger, - .pointer = xtfpga_pcm_pointer, - .pcm_construct = xtfpga_pcm_new, + .name = DRV_NAME, + .open = xtfpga_pcm_open, + .close = xtfpga_pcm_close, + .hw_params = xtfpga_pcm_hw_params, + .trigger = xtfpga_pcm_trigger, + .pointer = xtfpga_pcm_pointer, + .pcm_construct = xtfpga_pcm_new, + .legacy_dai_naming = 1, }; static const struct snd_soc_dai_ops xtfpga_i2s_dai_ops = { .startup = xtfpga_i2s_startup, .hw_params = xtfpga_i2s_hw_params, - .set_fmt = xtfpga_i2s_set_fmt, + .set_fmt = xtfpga_i2s_set_fmt, }; static struct snd_soc_dai_driver xtfpga_i2s_dai[] = { diff --git a/sound/spi/Kconfig b/sound/spi/Kconfig index 44d39fa635..f407c37c37 100644 --- a/sound/spi/Kconfig +++ b/sound/spi/Kconfig @@ -19,7 +19,7 @@ config SND_AT73C213 DAC can be found on Atmel development boards. This driver requires the Atmel SSC driver for sound sink, a - peripheral found on most AT91 and AVR32 microprocessors. + peripheral found on most AT91 microprocessors. To compile this driver as a module, choose M here: the module will be called snd-at73c213. diff --git a/sound/usb/6fire/pcm.c b/sound/usb/6fire/pcm.c index 7168f1c6a3..32c39d8bd2 100644 --- a/sound/usb/6fire/pcm.c +++ b/sound/usb/6fire/pcm.c @@ -175,7 +175,7 @@ static int usb6fire_pcm_stream_start(struct pcm_runtime *rt) } } - /* wait for first out urb to return (sent in in urb handler) */ + /* wait for first out urb to return (sent in urb handler) */ wait_event_timeout(rt->stream_wait_queue, rt->stream_wait_cond, HZ); if (rt->stream_wait_cond) diff --git a/sound/usb/bcd2000/bcd2000.c b/sound/usb/bcd2000/bcd2000.c index cd4a0bc6d2..7aec0a95c6 100644 --- a/sound/usb/bcd2000/bcd2000.c +++ b/sound/usb/bcd2000/bcd2000.c @@ -348,7 +348,8 @@ static int bcd2000_init_midi(struct bcd2000 *bcd2k) static void bcd2000_free_usb_related_resources(struct bcd2000 *bcd2k, struct usb_interface *interface) { - /* usb_kill_urb not necessary, urb is aborted automatically */ + usb_kill_urb(bcd2k->midi_out_urb); + usb_kill_urb(bcd2k->midi_in_urb); usb_free_urb(bcd2k->midi_out_urb); usb_free_urb(bcd2k->midi_in_urb); diff --git a/sound/usb/card.c b/sound/usb/card.c index 2a1f76ff85..991bd68a9f 100644 --- a/sound/usb/card.c +++ b/sound/usb/card.c @@ -387,6 +387,14 @@ static const struct usb_audio_device_name usb_audio_names[] = { DEVICE_NAME(0x05e1, 0x0408, "Syntek", "STK1160"), DEVICE_NAME(0x05e1, 0x0480, "Hauppauge", "Woodbury"), + /* ASUS ROG Zenith II: this machine has also two devices, one for + * the front headphone and another for the rest + */ + PROFILE_NAME(0x0b05, 0x1915, "ASUS", "Zenith II Front Headphone", + "Zenith-II-Front-Headphone"), + PROFILE_NAME(0x0b05, 0x1916, "ASUS", "Zenith II Main Audio", + "Zenith-II-Main-Audio"), + /* ASUS ROG Strix */ PROFILE_NAME(0x0b05, 0x1917, "Realtek", "ALC1220-VB-DT", "Realtek-ALC1220-VB-Desktop"), @@ -635,6 +643,7 @@ static int snd_usb_audio_create(struct usb_interface *intf, INIT_LIST_HEAD(&chip->pcm_list); INIT_LIST_HEAD(&chip->ep_list); INIT_LIST_HEAD(&chip->iface_ref_list); + INIT_LIST_HEAD(&chip->clock_ref_list); INIT_LIST_HEAD(&chip->midi_list); INIT_LIST_HEAD(&chip->mixer_list); diff --git a/sound/usb/card.h b/sound/usb/card.h index 87f042d06c..ca75f22061 100644 --- a/sound/usb/card.h +++ b/sound/usb/card.h @@ -44,6 +44,7 @@ struct audioformat { struct snd_usb_substream; struct snd_usb_iface_ref; +struct snd_usb_clock_ref; struct snd_usb_endpoint; struct snd_usb_power_domain; @@ -62,6 +63,7 @@ struct snd_urb_ctx { struct snd_usb_endpoint { struct snd_usb_audio *chip; struct snd_usb_iface_ref *iface_ref; + struct snd_usb_clock_ref *clock_ref; int opened; /* open refcount; protect with chip->mutex */ atomic_t running; /* running status */ @@ -138,7 +140,6 @@ struct snd_usb_endpoint { unsigned int cur_period_frames; unsigned int cur_period_bytes; unsigned int cur_buffer_periods; - unsigned char cur_clock; spinlock_t lock; struct list_head list; diff --git a/sound/usb/clock.c b/sound/usb/clock.c index 4dfe764167..33db334e65 100644 --- a/sound/usb/clock.c +++ b/sound/usb/clock.c @@ -572,6 +572,17 @@ static int set_sample_rate_v2v3(struct snd_usb_audio *chip, /* continue processing */ } + /* FIXME - TEAC devices require the immediate interface setup */ + if (USB_ID_VENDOR(chip->usb_id) == 0x0644) { + bool cur_base_48k = (rate % 48000 == 0); + bool prev_base_48k = (prev_rate % 48000 == 0); + if (cur_base_48k != prev_base_48k) { + usb_set_interface(chip->dev, fmt->iface, fmt->altsetting); + if (chip->quirk_flags & QUIRK_FLAG_IFACE_DELAY) + msleep(50); + } + } + validation: /* validate clock after rate change */ if (!uac_clock_source_is_valid(chip, fmt, clock)) diff --git a/sound/usb/endpoint.c b/sound/usb/endpoint.c index 743b8287cf..0d7b73bf79 100644 --- a/sound/usb/endpoint.c +++ b/sound/usb/endpoint.c @@ -35,6 +35,14 @@ struct snd_usb_iface_ref { struct list_head list; }; +/* clock refcounting */ +struct snd_usb_clock_ref { + unsigned char clock; + atomic_t locked; + int rate; + struct list_head list; +}; + /* * snd_usb_endpoint is a model that abstracts everything related to an * USB endpoint and its streaming. @@ -125,7 +133,7 @@ static inline bool ep_state_running(struct snd_usb_endpoint *ep) static inline bool ep_state_update(struct snd_usb_endpoint *ep, int old, int new) { - return atomic_cmpxchg(&ep->state, old, new) == old; + return atomic_try_cmpxchg(&ep->state, &old, new); } /** @@ -591,6 +599,25 @@ iface_ref_find(struct snd_usb_audio *chip, int iface) return ip; } +/* Similarly, a refcount object for clock */ +static struct snd_usb_clock_ref * +clock_ref_find(struct snd_usb_audio *chip, int clock) +{ + struct snd_usb_clock_ref *ref; + + list_for_each_entry(ref, &chip->clock_ref_list, list) + if (ref->clock == clock) + return ref; + + ref = kzalloc(sizeof(*ref), GFP_KERNEL); + if (!ref) + return NULL; + ref->clock = clock; + atomic_set(&ref->locked, 0); + list_add_tail(&ref->list, &chip->clock_ref_list); + return ref; +} + /* * Get the existing endpoint object corresponding EP * Returns NULL if not present. @@ -768,6 +795,14 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip, goto unlock; } + if (fp->protocol != UAC_VERSION_1) { + ep->clock_ref = clock_ref_find(chip, fp->clock); + if (!ep->clock_ref) { + ep = NULL; + goto unlock; + } + } + ep->cur_audiofmt = fp; ep->cur_channels = fp->channels; ep->cur_rate = params_rate(params); @@ -777,7 +812,6 @@ snd_usb_endpoint_open(struct snd_usb_audio *chip, ep->cur_period_frames = params_period_size(params); ep->cur_period_bytes = ep->cur_period_frames * ep->cur_frame_bytes; ep->cur_buffer_periods = params_periods(params); - ep->cur_clock = fp->clock; if (ep->type == SND_USB_ENDPOINT_TYPE_SYNC) endpoint_set_syncinterval(chip, ep); @@ -894,8 +928,8 @@ void snd_usb_endpoint_close(struct snd_usb_audio *chip, ep->altsetting = 0; ep->cur_audiofmt = NULL; ep->cur_rate = 0; - ep->cur_clock = 0; ep->iface_ref = NULL; + ep->clock_ref = NULL; usb_audio_dbg(chip, "EP 0x%x closed\n", ep->ep_num); } mutex_unlock(&chip->mutex); @@ -907,6 +941,8 @@ void snd_usb_endpoint_suspend(struct snd_usb_endpoint *ep) ep->need_setup = true; if (ep->iface_ref) ep->iface_ref->need_setup = true; + if (ep->clock_ref) + ep->clock_ref->rate = 0; } /* @@ -1314,6 +1350,33 @@ static int snd_usb_endpoint_set_params(struct snd_usb_audio *chip, return 0; } +static int init_sample_rate(struct snd_usb_audio *chip, + struct snd_usb_endpoint *ep) +{ + struct snd_usb_clock_ref *clock = ep->clock_ref; + int err; + + if (clock) { + if (atomic_read(&clock->locked)) + return 0; + if (clock->rate == ep->cur_rate) + return 0; + if (clock->rate && clock->rate != ep->cur_rate) { + usb_audio_dbg(chip, "Mismatched sample rate %d vs %d for EP 0x%x\n", + clock->rate, ep->cur_rate, ep->ep_num); + return -EINVAL; + } + } + + err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, ep->cur_rate); + if (err < 0) + return err; + + if (clock) + clock->rate = ep->cur_rate; + return 0; +} + /* * snd_usb_endpoint_configure: Configure the endpoint * @@ -1343,8 +1406,7 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, * to update at each EP configuration */ if (ep->cur_audiofmt->protocol == UAC_VERSION_1) { - err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, - ep->cur_rate); + err = init_sample_rate(chip, ep); if (err < 0) goto unlock; } @@ -1374,7 +1436,7 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, if (err < 0) goto unlock; - err = snd_usb_init_sample_rate(chip, ep->cur_audiofmt, ep->cur_rate); + err = init_sample_rate(chip, ep); if (err < 0) goto unlock; @@ -1407,15 +1469,15 @@ int snd_usb_endpoint_configure(struct snd_usb_audio *chip, /* get the current rate set to the given clock by any endpoint */ int snd_usb_endpoint_get_clock_rate(struct snd_usb_audio *chip, int clock) { - struct snd_usb_endpoint *ep; + struct snd_usb_clock_ref *ref; int rate = 0; if (!clock) return 0; mutex_lock(&chip->mutex); - list_for_each_entry(ep, &chip->ep_list, list) { - if (ep->cur_clock == clock && ep->cur_rate) { - rate = ep->cur_rate; + list_for_each_entry(ref, &chip->clock_ref_list, list) { + if (ref->clock == clock) { + rate = ref->rate; break; } } @@ -1456,6 +1518,9 @@ int snd_usb_endpoint_start(struct snd_usb_endpoint *ep) if (atomic_inc_return(&ep->running) != 1) return 0; + if (ep->clock_ref) + atomic_inc(&ep->clock_ref->locked); + ep->active_mask = 0; ep->unlink_mask = 0; ep->phase = 0; @@ -1565,6 +1630,9 @@ void snd_usb_endpoint_stop(struct snd_usb_endpoint *ep, bool keep_pending) if (ep->sync_source) WRITE_ONCE(ep->sync_source->sync_sink, NULL); stop_urbs(ep, false, keep_pending); + if (ep->clock_ref) + if (!atomic_dec_return(&ep->clock_ref->locked)) + ep->clock_ref->rate = 0; } } @@ -1591,12 +1659,16 @@ void snd_usb_endpoint_free_all(struct snd_usb_audio *chip) { struct snd_usb_endpoint *ep, *en; struct snd_usb_iface_ref *ip, *in; + struct snd_usb_clock_ref *cp, *cn; list_for_each_entry_safe(ep, en, &chip->ep_list, list) kfree(ep); list_for_each_entry_safe(ip, in, &chip->iface_ref_list, list) kfree(ip); + + list_for_each_entry_safe(cp, cn, &chip->clock_ref_list, list) + kfree(cp); } /* diff --git a/sound/usb/hiface/pcm.c b/sound/usb/hiface/pcm.c index 71f17f02f3..cf650fab54 100644 --- a/sound/usb/hiface/pcm.c +++ b/sound/usb/hiface/pcm.c @@ -225,7 +225,7 @@ static int hiface_pcm_stream_start(struct pcm_runtime *rt) } } - /* wait for first out urb to return (sent in in urb handler) */ + /* wait for first out urb to return (sent in urb handler) */ wait_event_timeout(rt->stream_wait_queue, rt->stream_wait_cond, HZ); if (rt->stream_wait_cond) { diff --git a/sound/usb/implicit.c b/sound/usb/implicit.c index 2d444ec742..e1bf1b5da4 100644 --- a/sound/usb/implicit.c +++ b/sound/usb/implicit.c @@ -45,11 +45,6 @@ struct snd_usb_implicit_fb_match { /* Implicit feedback quirk table for playback */ static const struct snd_usb_implicit_fb_match playback_implicit_fb_quirks[] = { - /* Generic matching */ - IMPLICIT_FB_GENERIC_DEV(0x0499, 0x1509), /* Steinberg UR22 */ - IMPLICIT_FB_GENERIC_DEV(0x0763, 0x2030), /* M-Audio Fast Track C400 */ - IMPLICIT_FB_GENERIC_DEV(0x0763, 0x2031), /* M-Audio Fast Track C600 */ - /* Fixed EP */ /* FIXME: check the availability of generic matching */ IMPLICIT_FB_FIXED_DEV(0x0763, 0x2080, 0x81, 2), /* M-Audio FastTrack Ultra */ @@ -350,7 +345,8 @@ static int audioformat_implicit_fb_quirk(struct snd_usb_audio *chip, } /* Try the generic implicit fb if available */ - if (chip->generic_implicit_fb) + if (chip->generic_implicit_fb || + (chip->quirk_flags & QUIRK_FLAG_GENERIC_IMPLICIT_FB)) return add_generic_implicit_fb(chip, fmt, alts); /* No quirk */ @@ -387,6 +383,8 @@ int snd_usb_parse_implicit_fb_quirk(struct snd_usb_audio *chip, struct audioformat *fmt, struct usb_host_interface *alts) { + if (chip->quirk_flags & QUIRK_FLAG_SKIP_IMPLICIT_FB) + return 0; if (fmt->endpoint & USB_DIR_IN) return audioformat_capture_quirk(chip, fmt, alts); else diff --git a/sound/usb/line6/pcm.c b/sound/usb/line6/pcm.c index fdbdfb7bce..6a4af725ae 100644 --- a/sound/usb/line6/pcm.c +++ b/sound/usb/line6/pcm.c @@ -552,10 +552,10 @@ int line6_init_pcm(struct usb_line6 *line6, line6pcm->max_packet_size_in = usb_maxpacket(line6->usbdev, - usb_rcvisocpipe(line6->usbdev, ep_read), 0); + usb_rcvisocpipe(line6->usbdev, ep_read)); line6pcm->max_packet_size_out = usb_maxpacket(line6->usbdev, - usb_sndisocpipe(line6->usbdev, ep_write), 1); + usb_sndisocpipe(line6->usbdev, ep_write)); if (!line6pcm->max_packet_size_in || !line6pcm->max_packet_size_out) { dev_err(line6pcm->line6->ifcdev, "cannot get proper max packet size\n"); diff --git a/sound/usb/line6/pod.c b/sound/usb/line6/pod.c index 16e644330c..cd41aa7f03 100644 --- a/sound/usb/line6/pod.c +++ b/sound/usb/line6/pod.c @@ -235,7 +235,7 @@ static ssize_t serial_number_show(struct device *dev, struct snd_card *card = dev_to_snd_card(dev); struct usb_line6_pod *pod = card->private_data; - return sprintf(buf, "%u\n", pod->serial_number); + return sysfs_emit(buf, "%u\n", pod->serial_number); } /* @@ -247,8 +247,8 @@ static ssize_t firmware_version_show(struct device *dev, struct snd_card *card = dev_to_snd_card(dev); struct usb_line6_pod *pod = card->private_data; - return sprintf(buf, "%d.%02d\n", pod->firmware_version / 100, - pod->firmware_version % 100); + return sysfs_emit(buf, "%d.%02d\n", pod->firmware_version / 100, + pod->firmware_version % 100); } /* @@ -260,7 +260,7 @@ static ssize_t device_id_show(struct device *dev, struct snd_card *card = dev_to_snd_card(dev); struct usb_line6_pod *pod = card->private_data; - return sprintf(buf, "%d\n", pod->device_id); + return sysfs_emit(buf, "%d\n", pod->device_id); } /* diff --git a/sound/usb/line6/podhd.c b/sound/usb/line6/podhd.c index b24bc82f89..ffd8c157a2 100644 --- a/sound/usb/line6/podhd.c +++ b/sound/usb/line6/podhd.c @@ -146,7 +146,7 @@ static ssize_t serial_number_show(struct device *dev, struct snd_card *card = dev_to_snd_card(dev); struct usb_line6_podhd *pod = card->private_data; - return sprintf(buf, "%u\n", pod->serial_number); + return sysfs_emit(buf, "%u\n", pod->serial_number); } static ssize_t firmware_version_show(struct device *dev, @@ -155,7 +155,7 @@ static ssize_t firmware_version_show(struct device *dev, struct snd_card *card = dev_to_snd_card(dev); struct usb_line6_podhd *pod = card->private_data; - return sprintf(buf, "%06x\n", pod->firmware_version); + return sysfs_emit(buf, "%06x\n", pod->firmware_version); } static DEVICE_ATTR_RO(firmware_version); diff --git a/sound/usb/midi.c b/sound/usb/midi.c index 7c6ca2b433..bbff0923d2 100644 --- a/sound/usb/midi.c +++ b/sound/usb/midi.c @@ -1145,6 +1145,9 @@ static int snd_usbmidi_output_open(struct snd_rawmidi_substream *substream) static int snd_usbmidi_output_close(struct snd_rawmidi_substream *substream) { + struct usbmidi_out_port *port = substream->runtime->private_data; + + cancel_work_sync(&port->ep->work); return substream_open(substream, 0, 0); } @@ -1286,7 +1289,7 @@ static int snd_usbmidi_in_endpoint_create(struct snd_usb_midi *umidi, pipe = usb_rcvintpipe(umidi->dev, ep_info->in_ep); else pipe = usb_rcvbulkpipe(umidi->dev, ep_info->in_ep); - length = usb_maxpacket(umidi->dev, pipe, 0); + length = usb_maxpacket(umidi->dev, pipe); for (i = 0; i < INPUT_URBS; ++i) { buffer = usb_alloc_coherent(umidi->dev, length, GFP_KERNEL, &ep->urbs[i]->transfer_dma); @@ -1375,7 +1378,7 @@ static int snd_usbmidi_out_endpoint_create(struct snd_usb_midi *umidi, pipe = usb_sndbulkpipe(umidi->dev, ep_info->out_ep); switch (umidi->usb_id) { default: - ep->max_transfer = usb_maxpacket(umidi->dev, pipe, 1); + ep->max_transfer = usb_maxpacket(umidi->dev, pipe); break; /* * Various chips declare a packet size larger than 4 bytes, but diff --git a/sound/usb/mixer_maps.c b/sound/usb/mixer_maps.c index 7ef7a8abcc..f4bd1e8ae4 100644 --- a/sound/usb/mixer_maps.c +++ b/sound/usb/mixer_maps.c @@ -374,13 +374,28 @@ static const struct usbmix_name_map corsair_virtuoso_map[] = { { 0 } }; -/* Some mobos shipped with a dummy HD-audio show the invalid GET_MIN/GET_MAX - * response for Input Gain Pad (id=19, control=12) and the connector status - * for SPDIF terminal (id=18). Skip them. - */ -static const struct usbmix_name_map asus_rog_map[] = { - { 18, NULL }, /* OT, connector control */ - { 19, NULL, 12 }, /* FU, Input Gain Pad */ +/* ASUS ROG Zenith II with Realtek ALC1220-VB */ +static const struct usbmix_name_map asus_zenith_ii_map[] = { + { 19, NULL, 12 }, /* FU, Input Gain Pad - broken response, disabled */ + { 16, "Speaker" }, /* OT */ + { 22, "Speaker Playback" }, /* FU */ + { 7, "Line" }, /* IT */ + { 19, "Line Capture" }, /* FU */ + { 8, "Mic" }, /* IT */ + { 20, "Mic Capture" }, /* FU */ + { 9, "Front Mic" }, /* IT */ + { 21, "Front Mic Capture" }, /* FU */ + { 17, "IEC958" }, /* OT */ + { 23, "IEC958 Playback" }, /* FU */ + {} +}; + +static const struct usbmix_connector_map asus_zenith_ii_connector_map[] = { + { 10, 16 }, /* (Back) Speaker */ + { 11, 17 }, /* SPDIF */ + { 13, 7 }, /* Line */ + { 14, 8 }, /* Mic */ + { 15, 9 }, /* Front Mic */ {} }; @@ -439,6 +454,31 @@ static const struct usbmix_name_map msi_mpg_x570s_carbon_max_wifi_alc4080_map[] {} }; +/* Gigabyte B450/550 Mobo */ +static const struct usbmix_name_map gigabyte_b450_map[] = { + { 24, NULL }, /* OT, IEC958?, disabled */ + { 21, "Speaker" }, /* OT */ + { 29, "Speaker Playback" }, /* FU */ + { 22, "Headphone" }, /* OT */ + { 30, "Headphone Playback" }, /* FU */ + { 11, "Line" }, /* IT */ + { 27, "Line Capture" }, /* FU */ + { 12, "Mic" }, /* IT */ + { 28, "Mic Capture" }, /* FU */ + { 9, "Front Mic" }, /* IT */ + { 25, "Front Mic Capture" }, /* FU */ + {} +}; + +static const struct usbmix_connector_map gigabyte_b450_connector_map[] = { + { 13, 21 }, /* Speaker */ + { 14, 22 }, /* Headphone */ + { 19, 11 }, /* Line */ + { 20, 12 }, /* Mic */ + { 17, 9 }, /* Front Mic */ + {} +}; + /* * Control map entries */ @@ -581,9 +621,15 @@ static const struct usbmix_ctl_map usbmix_ctl_maps[] = { .map = trx40_mobo_map, .connector_map = trx40_mobo_connector_map, }, - { /* ASUS ROG Zenith II */ + { /* Gigabyte B450/550 Mobo */ + .id = USB_ID(0x0414, 0xa00d), + .map = gigabyte_b450_map, + .connector_map = gigabyte_b450_connector_map, + }, + { /* ASUS ROG Zenith II (main audio) */ .id = USB_ID(0x0b05, 0x1916), - .map = asus_rog_map, + .map = asus_zenith_ii_map, + .connector_map = asus_zenith_ii_connector_map, }, { /* ASUS ROG Strix */ .id = USB_ID(0x0b05, 0x1917), diff --git a/sound/usb/mixer_quirks.c b/sound/usb/mixer_quirks.c index d35cf54cab..ab0d459f42 100644 --- a/sound/usb/mixer_quirks.c +++ b/sound/usb/mixer_quirks.c @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -1934,13 +1935,194 @@ static int snd_soundblaster_e1_switch_create(struct usb_mixer_interface *mixer) NULL); } +/* + * Dell WD15 dock jack detection + * + * The WD15 contains an ALC4020 USB audio controller and ALC3263 audio codec + * from Realtek. It is a UAC 1 device, and UAC 1 does not support jack + * detection. Instead, jack detection works by sending HD Audio commands over + * vendor-type USB messages. + */ + +#define HDA_VERB_CMD(V, N, D) (((N) << 20) | ((V) << 8) | (D)) + +#define REALTEK_HDA_VALUE 0x0038 + +#define REALTEK_HDA_SET 62 +#define REALTEK_MANUAL_MODE 72 +#define REALTEK_HDA_GET_OUT 88 +#define REALTEK_HDA_GET_IN 89 + +#define REALTEK_AUDIO_FUNCTION_GROUP 0x01 +#define REALTEK_LINE1 0x1a +#define REALTEK_VENDOR_REGISTERS 0x20 +#define REALTEK_HP_OUT 0x21 + +#define REALTEK_CBJ_CTRL2 0x50 + +#define REALTEK_JACK_INTERRUPT_NODE 5 + +#define REALTEK_MIC_FLAG 0x100 + +static int realtek_hda_set(struct snd_usb_audio *chip, u32 cmd) +{ + struct usb_device *dev = chip->dev; + __be32 buf = cpu_to_be32(cmd); + + return snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), REALTEK_HDA_SET, + USB_RECIP_DEVICE | USB_TYPE_VENDOR | USB_DIR_OUT, + REALTEK_HDA_VALUE, 0, &buf, sizeof(buf)); +} + +static int realtek_hda_get(struct snd_usb_audio *chip, u32 cmd, u32 *value) +{ + struct usb_device *dev = chip->dev; + int err; + __be32 buf = cpu_to_be32(cmd); + + err = snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), REALTEK_HDA_GET_OUT, + USB_RECIP_DEVICE | USB_TYPE_VENDOR | USB_DIR_OUT, + REALTEK_HDA_VALUE, 0, &buf, sizeof(buf)); + if (err < 0) + return err; + err = snd_usb_ctl_msg(dev, usb_rcvctrlpipe(dev, 0), REALTEK_HDA_GET_IN, + USB_RECIP_DEVICE | USB_TYPE_VENDOR | USB_DIR_IN, + REALTEK_HDA_VALUE, 0, &buf, sizeof(buf)); + if (err < 0) + return err; + + *value = be32_to_cpu(buf); + return 0; +} + +static int realtek_ctl_connector_get(struct snd_kcontrol *kcontrol, + struct snd_ctl_elem_value *ucontrol) +{ + struct usb_mixer_elem_info *cval = kcontrol->private_data; + struct snd_usb_audio *chip = cval->head.mixer->chip; + u32 pv = kcontrol->private_value; + u32 node_id = pv & 0xff; + u32 sense; + u32 cbj_ctrl2; + bool presence; + int err; + + err = snd_usb_lock_shutdown(chip); + if (err < 0) + return err; + err = realtek_hda_get(chip, + HDA_VERB_CMD(AC_VERB_GET_PIN_SENSE, node_id, 0), + &sense); + if (err < 0) + goto err; + if (pv & REALTEK_MIC_FLAG) { + err = realtek_hda_set(chip, + HDA_VERB_CMD(AC_VERB_SET_COEF_INDEX, + REALTEK_VENDOR_REGISTERS, + REALTEK_CBJ_CTRL2)); + if (err < 0) + goto err; + err = realtek_hda_get(chip, + HDA_VERB_CMD(AC_VERB_GET_PROC_COEF, + REALTEK_VENDOR_REGISTERS, 0), + &cbj_ctrl2); + if (err < 0) + goto err; + } +err: + snd_usb_unlock_shutdown(chip); + if (err < 0) + return err; + + presence = sense & AC_PINSENSE_PRESENCE; + if (pv & REALTEK_MIC_FLAG) + presence = presence && (cbj_ctrl2 & 0x0070) == 0x0070; + ucontrol->value.integer.value[0] = presence; + return 0; +} + +static const struct snd_kcontrol_new realtek_connector_ctl_ro = { + .iface = SNDRV_CTL_ELEM_IFACE_CARD, + .name = "", /* will be filled later manually */ + .access = SNDRV_CTL_ELEM_ACCESS_READ, + .info = snd_ctl_boolean_mono_info, + .get = realtek_ctl_connector_get, +}; + +static int realtek_resume_jack(struct usb_mixer_elem_list *list) +{ + snd_ctl_notify(list->mixer->chip->card, SNDRV_CTL_EVENT_MASK_VALUE, + &list->kctl->id); + return 0; +} + +static int realtek_add_jack(struct usb_mixer_interface *mixer, + char *name, u32 val) +{ + struct usb_mixer_elem_info *cval; + struct snd_kcontrol *kctl; + + cval = kzalloc(sizeof(*cval), GFP_KERNEL); + if (!cval) + return -ENOMEM; + snd_usb_mixer_elem_init_std(&cval->head, mixer, + REALTEK_JACK_INTERRUPT_NODE); + cval->head.resume = realtek_resume_jack; + cval->val_type = USB_MIXER_BOOLEAN; + cval->channels = 1; + cval->min = 0; + cval->max = 1; + kctl = snd_ctl_new1(&realtek_connector_ctl_ro, cval); + if (!kctl) { + kfree(cval); + return -ENOMEM; + } + kctl->private_value = val; + strscpy(kctl->id.name, name, sizeof(kctl->id.name)); + kctl->private_free = snd_usb_mixer_elem_free; + return snd_usb_mixer_add_control(&cval->head, kctl); +} + +static int dell_dock_mixer_create(struct usb_mixer_interface *mixer) +{ + int err; + struct usb_device *dev = mixer->chip->dev; + + /* Power down the audio codec to avoid loud pops in the next step. */ + realtek_hda_set(mixer->chip, + HDA_VERB_CMD(AC_VERB_SET_POWER_STATE, + REALTEK_AUDIO_FUNCTION_GROUP, + AC_PWRST_D3)); + + /* + * Turn off 'manual mode' in case it was enabled. This removes the need + * to power cycle the dock after it was attached to a Windows machine. + */ + snd_usb_ctl_msg(dev, usb_sndctrlpipe(dev, 0), REALTEK_MANUAL_MODE, + USB_RECIP_DEVICE | USB_TYPE_VENDOR | USB_DIR_OUT, + 0, 0, NULL, 0); + + err = realtek_add_jack(mixer, "Line Out Jack", REALTEK_LINE1); + if (err < 0) + return err; + err = realtek_add_jack(mixer, "Headphone Jack", REALTEK_HP_OUT); + if (err < 0) + return err; + err = realtek_add_jack(mixer, "Headset Mic Jack", + REALTEK_HP_OUT | REALTEK_MIC_FLAG); + if (err < 0) + return err; + return 0; +} + static void dell_dock_init_vol(struct snd_usb_audio *chip, int ch, int id) { u16 buf = 0; snd_usb_ctl_msg(chip->dev, usb_sndctrlpipe(chip->dev, 0), UAC_SET_CUR, USB_RECIP_INTERFACE | USB_TYPE_CLASS | USB_DIR_OUT, - ch, snd_usb_ctrl_intf(chip) | (id << 8), + (UAC_FU_VOLUME << 8) | ch, + snd_usb_ctrl_intf(chip) | (id << 8), &buf, 2); } @@ -3238,6 +3420,7 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) case USB_ID(0x1235, 0x8213): /* Focusrite Scarlett 8i6 3rd Gen */ case USB_ID(0x1235, 0x8214): /* Focusrite Scarlett 18i8 3rd Gen */ case USB_ID(0x1235, 0x8215): /* Focusrite Scarlett 18i20 3rd Gen */ + case USB_ID(0x1235, 0x820c): /* Focusrite Clarett+ 8Pre */ err = snd_scarlett_gen2_init(mixer); break; @@ -3245,6 +3428,9 @@ int snd_usb_mixer_apply_create_quirk(struct usb_mixer_interface *mixer) err = snd_soundblaster_e1_switch_create(mixer); break; case USB_ID(0x0bda, 0x4014): /* Dell WD15 dock */ + err = dell_dock_mixer_create(mixer); + if (err < 0) + break; err = dell_dock_mixer_init(mixer); break; diff --git a/sound/usb/mixer_scarlett_gen2.c b/sound/usb/mixer_scarlett_gen2.c index 69a2cd429e..9d11bb0866 100644 --- a/sound/usb/mixer_scarlett_gen2.c +++ b/sound/usb/mixer_scarlett_gen2.c @@ -1,13 +1,15 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Focusrite Scarlett Gen 2/3 Driver for ALSA + * Focusrite Scarlett Gen 2/3 and Clarett+ Driver for ALSA * * Supported models: * - 6i6/18i8/18i20 Gen 2 * - Solo/2i2/4i4/8i6/18i8/18i20 Gen 3 + * - Clarett+ 8Pre * * Copyright (c) 2018-2022 by Geoffrey D. Bennett * Copyright (c) 2020-2021 by Vladimir Sadovnikov + * Copyright (c) 2022 by Christian Colglazier * * Based on the Scarlett (Gen 1) Driver for ALSA: * @@ -51,6 +53,9 @@ * Support for phantom power, direct monitoring, speaker switching, * and talkback added in May-June 2021. * + * Support for Clarett+ 8Pre added in Aug 2022 by Christian + * Colglazier. + * * This ALSA mixer gives access to (model-dependent): * - input, output, mixer-matrix muxes * - mixer-matrix gain stages @@ -203,7 +208,8 @@ enum { SCARLETT2_CONFIG_SET_NO_MIXER = 0, SCARLETT2_CONFIG_SET_GEN_2 = 1, SCARLETT2_CONFIG_SET_GEN_3 = 2, - SCARLETT2_CONFIG_SET_COUNT = 3 + SCARLETT2_CONFIG_SET_CLARETT = 3, + SCARLETT2_CONFIG_SET_COUNT = 4 }; /* Hardware port types: @@ -841,6 +847,61 @@ static const struct scarlett2_device_info s18i20_gen3_info = { } }, }; +static const struct scarlett2_device_info clarett_8pre_info = { + .usb_id = USB_ID(0x1235, 0x820c), + + .config_set = SCARLETT2_CONFIG_SET_CLARETT, + .line_out_hw_vol = 1, + .level_input_count = 2, + .air_input_count = 8, + + .line_out_descrs = { + "Monitor L", + "Monitor R", + NULL, + NULL, + NULL, + NULL, + "Headphones 1 L", + "Headphones 1 R", + "Headphones 2 L", + "Headphones 2 R", + }, + + .port_count = { + [SCARLETT2_PORT_TYPE_NONE] = { 1, 0 }, + [SCARLETT2_PORT_TYPE_ANALOGUE] = { 8, 10 }, + [SCARLETT2_PORT_TYPE_SPDIF] = { 2, 2 }, + [SCARLETT2_PORT_TYPE_ADAT] = { 8, 8 }, + [SCARLETT2_PORT_TYPE_MIX] = { 10, 18 }, + [SCARLETT2_PORT_TYPE_PCM] = { 20, 18 }, + }, + + .mux_assignment = { { + { SCARLETT2_PORT_TYPE_PCM, 0, 18 }, + { SCARLETT2_PORT_TYPE_ANALOGUE, 0, 10 }, + { SCARLETT2_PORT_TYPE_SPDIF, 0, 2 }, + { SCARLETT2_PORT_TYPE_ADAT, 0, 8 }, + { SCARLETT2_PORT_TYPE_MIX, 0, 18 }, + { SCARLETT2_PORT_TYPE_NONE, 0, 8 }, + { 0, 0, 0 }, + }, { + { SCARLETT2_PORT_TYPE_PCM, 0, 14 }, + { SCARLETT2_PORT_TYPE_ANALOGUE, 0, 10 }, + { SCARLETT2_PORT_TYPE_SPDIF, 0, 2 }, + { SCARLETT2_PORT_TYPE_ADAT, 0, 4 }, + { SCARLETT2_PORT_TYPE_MIX, 0, 18 }, + { SCARLETT2_PORT_TYPE_NONE, 0, 8 }, + { 0, 0, 0 }, + }, { + { SCARLETT2_PORT_TYPE_PCM, 0, 12 }, + { SCARLETT2_PORT_TYPE_ANALOGUE, 0, 10 }, + { SCARLETT2_PORT_TYPE_SPDIF, 0, 2 }, + { SCARLETT2_PORT_TYPE_NONE, 0, 22 }, + { 0, 0, 0 }, + } }, +}; + static const struct scarlett2_device_info *scarlett2_devices[] = { /* Supported Gen 2 devices */ &s6i6_gen2_info, @@ -855,6 +916,9 @@ static const struct scarlett2_device_info *scarlett2_devices[] = { &s18i8_gen3_info, &s18i20_gen3_info, + /* Supported Clarett+ devices */ + &clarett_8pre_info, + /* End of list */ NULL }; @@ -1047,6 +1111,29 @@ static const struct scarlett2_config [SCARLETT2_CONFIG_TALKBACK_MAP] = { .offset = 0xb0, .size = 16, .activate = 10 }, + +/* Clarett+ 8Pre */ +}, { + [SCARLETT2_CONFIG_DIM_MUTE] = { + .offset = 0x31, .size = 8, .activate = 2 }, + + [SCARLETT2_CONFIG_LINE_OUT_VOLUME] = { + .offset = 0x34, .size = 16, .activate = 1 }, + + [SCARLETT2_CONFIG_MUTE_SWITCH] = { + .offset = 0x5c, .size = 8, .activate = 1 }, + + [SCARLETT2_CONFIG_SW_HW_SWITCH] = { + .offset = 0x66, .size = 8, .activate = 3 }, + + [SCARLETT2_CONFIG_LEVEL_SWITCH] = { + .offset = 0x7c, .size = 8, .activate = 7 }, + + [SCARLETT2_CONFIG_AIR_SWITCH] = { + .offset = 0x95, .size = 8, .activate = 8 }, + + [SCARLETT2_CONFIG_STANDALONE_SWITCH] = { + .offset = 0x8d, .size = 8, .activate = 6 }, } }; /* proprietary request/response format */ diff --git a/sound/usb/mixer_us16x08.c b/sound/usb/mixer_us16x08.c index b7b6f3834e..6eb7d93b35 100644 --- a/sound/usb/mixer_us16x08.c +++ b/sound/usb/mixer_us16x08.c @@ -637,10 +637,10 @@ static int snd_get_meter_comp_index(struct snd_us16x08_meter_store *store) } } else { /* skip channels with no compressor active */ - while (!store->comp_store->val[ + while (store->comp_index <= SND_US16X08_MAX_CHANNELS + && !store->comp_store->val[ COMP_STORE_IDX(SND_US16X08_ID_COMP_SWITCH)] - [store->comp_index - 1] - && store->comp_index <= SND_US16X08_MAX_CHANNELS) { + [store->comp_index - 1]) { store->comp_index++; } ret = store->comp_index++; diff --git a/sound/usb/pcm.c b/sound/usb/pcm.c index 6d699065e8..d45d1d7e66 100644 --- a/sound/usb/pcm.c +++ b/sound/usb/pcm.c @@ -291,6 +291,9 @@ int snd_usb_audioformat_set_sync_ep(struct snd_usb_audio *chip, bool is_playback; int err; + if (fmt->sync_ep) + return 0; /* already set up */ + alts = snd_usb_get_host_interface(chip, fmt->iface, fmt->altsetting); if (!alts) return 0; @@ -304,7 +307,7 @@ int snd_usb_audioformat_set_sync_ep(struct snd_usb_audio *chip, * Generic sync EP handling */ - if (altsd->bNumEndpoints < 2) + if (fmt->ep_idx > 0 || altsd->bNumEndpoints < 2) return 0; is_playback = !(get_endpoint(alts, 0)->bEndpointAddress & USB_DIR_IN); @@ -439,16 +442,21 @@ static int configure_endpoints(struct snd_usb_audio *chip, /* stop any running stream beforehand */ if (stop_endpoints(subs, false)) sync_pending_stops(subs); + if (subs->sync_endpoint) { + err = snd_usb_endpoint_configure(chip, subs->sync_endpoint); + if (err < 0) + return err; + } err = snd_usb_endpoint_configure(chip, subs->data_endpoint); if (err < 0) return err; snd_usb_set_format_quirk(subs, subs->cur_audiofmt); - } - - if (subs->sync_endpoint) { - err = snd_usb_endpoint_configure(chip, subs->sync_endpoint); - if (err < 0) - return err; + } else { + if (subs->sync_endpoint) { + err = snd_usb_endpoint_configure(chip, subs->sync_endpoint); + if (err < 0) + return err; + } } return 0; @@ -1261,7 +1269,7 @@ static inline void fill_playback_urb_dsd_dop(struct snd_usb_substream *subs, unsigned int wrap = subs->buffer_bytes; u8 *dst = urb->transfer_buffer; u8 *src = runtime->dma_area; - u8 marker[] = { 0x05, 0xfa }; + static const u8 marker[] = { 0x05, 0xfa }; unsigned int queued = 0; /* diff --git a/sound/usb/quirks-table.h b/sound/usb/quirks-table.h index 0ea39565e6..f93201a830 100644 --- a/sound/usb/quirks-table.h +++ b/sound/usb/quirks-table.h @@ -2658,7 +2658,12 @@ YAMAHA_DEVICE(0x7010, "UB99"), .nr_rates = 2, .rate_table = (unsigned int[]) { 44100, 48000 - } + }, + .sync_ep = 0x82, + .sync_iface = 0, + .sync_altsetting = 1, + .sync_ep_idx = 1, + .implicit_fb = 1, } }, { @@ -2672,6 +2677,7 @@ YAMAHA_DEVICE(0x7010, "UB99"), .altset_idx = 1, .attributes = 0, .endpoint = 0x82, + .ep_idx = 1, .ep_attr = USB_ENDPOINT_XFER_ISOC, .datainterval = 1, .maxpacksize = 0x0126, @@ -2875,6 +2881,7 @@ YAMAHA_DEVICE(0x7010, "UB99"), .altset_idx = 1, .attributes = 0x4, .endpoint = 0x81, + .ep_idx = 1, .ep_attr = USB_ENDPOINT_XFER_ISOC | USB_ENDPOINT_SYNC_ASYNC, .maxpacksize = 0x130, @@ -3235,6 +3242,15 @@ YAMAHA_DEVICE(0x7010, "UB99"), } }, +/* Rane SL-1 */ +{ + USB_DEVICE(0x13e5, 0x0001), + .driver_info = (unsigned long) & (const struct snd_usb_audio_quirk) { + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_AUDIO_STANDARD_INTERFACE + } +}, + /* disabled due to regression for other devices; * see https://bugzilla.kernel.org/show_bug.cgi?id=199905 */ @@ -3382,6 +3398,7 @@ YAMAHA_DEVICE(0x7010, "UB99"), .altset_idx = 1, .attributes = 0, .endpoint = 0x03, + .ep_idx = 1, .rates = SNDRV_PCM_RATE_96000, .ep_attr = USB_ENDPOINT_XFER_ISOC | USB_ENDPOINT_SYNC_ASYNC, @@ -3785,6 +3802,54 @@ YAMAHA_DEVICE(0x7010, "UB99"), } }, +/* + * MacroSilicon MS2100/MS2106 based AV capture cards + * + * These claim 96kHz 1ch in the descriptors, but are actually 48kHz 2ch. + * They also need QUIRK_FLAG_ALIGN_TRANSFER, which makes one wonder if + * they pretend to be 96kHz mono as a workaround for stereo being broken + * by that... + * + * They also have an issue with initial stream alignment that causes the + * channels to be swapped and out of phase, which is dealt with in quirks.c. + */ +{ + USB_AUDIO_DEVICE(0x534d, 0x0021), + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .vendor_name = "MacroSilicon", + .product_name = "MS210x", + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_COMPOSITE, + .data = &(const struct snd_usb_audio_quirk[]) { + { + .ifnum = 2, + .type = QUIRK_AUDIO_STANDARD_MIXER, + }, + { + .ifnum = 3, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S16_LE, + .channels = 2, + .iface = 3, + .altsetting = 1, + .altset_idx = 1, + .attributes = 0, + .endpoint = 0x82, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .rates = SNDRV_PCM_RATE_CONTINUOUS, + .rate_min = 48000, + .rate_max = 48000, + } + }, + { + .ifnum = -1 + } + } + } +}, + /* * MacroSilicon MS2109 based HDMI capture cards * @@ -4102,6 +4167,206 @@ YAMAHA_DEVICE(0x7010, "UB99"), } } }, +{ + /* + * Fiero SC-01 (firmware v1.0.0 @ 48 kHz) + */ + USB_DEVICE(0x2b53, 0x0023), + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .vendor_name = "Fiero", + .product_name = "SC-01", + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_COMPOSITE, + .data = &(const struct snd_usb_audio_quirk[]) { + { + .ifnum = 0, + .type = QUIRK_AUDIO_STANDARD_INTERFACE + }, + /* Playback */ + { + .ifnum = 1, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels = 2, + .fmt_bits = 24, + .iface = 1, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x01, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { 48000 }, + .clock = 0x29 + } + }, + /* Capture */ + { + .ifnum = 2, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels = 2, + .fmt_bits = 24, + .iface = 2, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x82, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC | + USB_ENDPOINT_USAGE_IMPLICIT_FB, + .rates = SNDRV_PCM_RATE_48000, + .rate_min = 48000, + .rate_max = 48000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { 48000 }, + .clock = 0x29 + } + }, + { + .ifnum = -1 + } + } + } +}, +{ + /* + * Fiero SC-01 (firmware v1.0.0 @ 96 kHz) + */ + USB_DEVICE(0x2b53, 0x0024), + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .vendor_name = "Fiero", + .product_name = "SC-01", + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_COMPOSITE, + .data = &(const struct snd_usb_audio_quirk[]) { + { + .ifnum = 0, + .type = QUIRK_AUDIO_STANDARD_INTERFACE + }, + /* Playback */ + { + .ifnum = 1, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels = 2, + .fmt_bits = 24, + .iface = 1, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x01, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .rates = SNDRV_PCM_RATE_96000, + .rate_min = 96000, + .rate_max = 96000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { 96000 }, + .clock = 0x29 + } + }, + /* Capture */ + { + .ifnum = 2, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels = 2, + .fmt_bits = 24, + .iface = 2, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x82, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC | + USB_ENDPOINT_USAGE_IMPLICIT_FB, + .rates = SNDRV_PCM_RATE_96000, + .rate_min = 96000, + .rate_max = 96000, + .nr_rates = 1, + .rate_table = (unsigned int[]) { 96000 }, + .clock = 0x29 + } + }, + { + .ifnum = -1 + } + } + } +}, +{ + /* + * Fiero SC-01 (firmware v1.1.0) + */ + USB_DEVICE(0x2b53, 0x0031), + .driver_info = (unsigned long) &(const struct snd_usb_audio_quirk) { + .vendor_name = "Fiero", + .product_name = "SC-01", + .ifnum = QUIRK_ANY_INTERFACE, + .type = QUIRK_COMPOSITE, + .data = &(const struct snd_usb_audio_quirk[]) { + { + .ifnum = 0, + .type = QUIRK_AUDIO_STANDARD_INTERFACE + }, + /* Playback */ + { + .ifnum = 1, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels = 2, + .fmt_bits = 24, + .iface = 1, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x01, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC, + .rates = SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_96000, + .rate_min = 48000, + .rate_max = 96000, + .nr_rates = 2, + .rate_table = (unsigned int[]) { 48000, 96000 }, + .clock = 0x29 + } + }, + /* Capture */ + { + .ifnum = 2, + .type = QUIRK_AUDIO_FIXED_ENDPOINT, + .data = &(const struct audioformat) { + .formats = SNDRV_PCM_FMTBIT_S32_LE, + .channels = 2, + .fmt_bits = 24, + .iface = 2, + .altsetting = 1, + .altset_idx = 1, + .endpoint = 0x82, + .ep_attr = USB_ENDPOINT_XFER_ISOC | + USB_ENDPOINT_SYNC_ASYNC | + USB_ENDPOINT_USAGE_IMPLICIT_FB, + .rates = SNDRV_PCM_RATE_48000 | + SNDRV_PCM_RATE_96000, + .rate_min = 48000, + .rate_max = 96000, + .nr_rates = 2, + .rate_table = (unsigned int[]) { 48000, 96000 }, + .clock = 0x29 + } + }, + { + .ifnum = -1 + } + } + } +}, #undef USB_DEVICE_VENDOR_SPEC #undef USB_AUDIO_DEVICE diff --git a/sound/usb/quirks.c b/sound/usb/quirks.c index edfdc002d5..b5b0998d2c 100644 --- a/sound/usb/quirks.c +++ b/sound/usb/quirks.c @@ -1478,6 +1478,7 @@ void snd_usb_set_format_quirk(struct snd_usb_substream *subs, case USB_ID(0x041e, 0x3f19): /* E-Mu 0204 USB */ set_format_emu_quirk(subs, fmt); break; + case USB_ID(0x534d, 0x0021): /* MacroSilicon MS2100/MS2106 */ case USB_ID(0x534d, 0x2109): /* MacroSilicon MS2109 */ subs->stream_offset_adj = 2; break; @@ -1793,6 +1794,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_CTL_MSG_DELAY_1M | QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x046d, 0x09a4, /* Logitech QuickCam E 3500 */ QUIRK_FLAG_CTL_MSG_DELAY_1M | QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x0499, 0x1509, /* Steinberg UR22 */ + QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x04d8, 0xfeea, /* Benchmark DAC1 Pre */ QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x04e8, 0xa051, /* Samsung USBC Headset (AKG) */ @@ -1822,8 +1825,14 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x06f8, 0xd002, /* Hercules DJ Console (Macintosh Edition) */ QUIRK_FLAG_IGNORE_CTL_ERROR), + DEVICE_FLG(0x0711, 0x5800, /* MCT Trigger 5 USB-to-HDMI */ + QUIRK_FLAG_GET_SAMPLE_RATE), DEVICE_FLG(0x074d, 0x3553, /* Outlaw RR2150 (Micronas UAC3553B) */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x0763, 0x2030, /* M-Audio Fast Track C400 */ + QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x0763, 0x2031, /* M-Audio Fast Track C600 */ + QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x08bb, 0x2702, /* LineX FM Transmitter */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x0951, 0x16ad, /* Kingston HyperX */ @@ -1834,6 +1843,12 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x1395, 0x740a, /* Sennheiser DECT */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x1397, 0x0507, /* Behringer UMC202HD */ + QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x1397, 0x0508, /* Behringer UMC204HD */ + QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x1397, 0x0509, /* Behringer UMC404HD */ + QUIRK_FLAG_PLAYBACK_FIRST | QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x13e5, 0x0001, /* Serato Phono */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x154e, 0x1002, /* Denon DCD-1500RE */ @@ -1888,6 +1903,8 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_SHARE_MEDIA_DEVICE | QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x21b4, 0x0081, /* AudioQuest DragonFly */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x2522, 0x0007, /* LH Labs Geek Out HD Audio 1V5 */ + QUIRK_FLAG_SET_IFACE_FIRST), DEVICE_FLG(0x2708, 0x0002, /* Audient iD14 */ QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x2912, 0x30c8, /* Audioengine D1 */ @@ -1896,10 +1913,18 @@ static const struct usb_audio_quirk_flags_table quirk_flags_table[] = { QUIRK_FLAG_IGNORE_CTL_ERROR), DEVICE_FLG(0x413c, 0xa506, /* Dell AE515 sound bar */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x534d, 0x0021, /* MacroSilicon MS2100/MS2106 */ + QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x534d, 0x2109, /* MacroSilicon MS2109 */ QUIRK_FLAG_ALIGN_TRANSFER), DEVICE_FLG(0x1224, 0x2a25, /* Jieli Technology USB PHY 2.0 */ QUIRK_FLAG_GET_SAMPLE_RATE), + DEVICE_FLG(0x2b53, 0x0023, /* Fiero SC-01 (firmware v1.0.0 @ 48 kHz) */ + QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x2b53, 0x0024, /* Fiero SC-01 (firmware v1.0.0 @ 96 kHz) */ + QUIRK_FLAG_GENERIC_IMPLICIT_FB), + DEVICE_FLG(0x2b53, 0x0031, /* Fiero SC-01 (firmware v1.1.0) */ + QUIRK_FLAG_GENERIC_IMPLICIT_FB), DEVICE_FLG(0x09da, 0x2695, /* A4Tech FHD 1080p webcam */ QUIRK_FLAG_DISABLE_AUTOSUSPEND | QUIRK_FLAG_GET_SAMPLE_RATE), diff --git a/sound/usb/usbaudio.h b/sound/usb/usbaudio.h index b8359a0aa0..ffbb4b0d09 100644 --- a/sound/usb/usbaudio.h +++ b/sound/usb/usbaudio.h @@ -45,6 +45,7 @@ struct snd_usb_audio { struct list_head pcm_list; /* list of pcm streams */ struct list_head ep_list; /* list of audio-related endpoints */ struct list_head iface_ref_list; /* list of interface refcounts */ + struct list_head clock_ref_list; /* list of clock refcounts */ int pcm_devs; struct list_head midi_list; /* list of midi interfaces */ @@ -164,6 +165,10 @@ extern bool snd_usb_skip_validation; * Support generic DSD raw U32_BE format * QUIRK_FLAG_SET_IFACE_FIRST: * Set up the interface at first like UAC1 + * QUIRK_FLAG_GENERIC_IMPLICIT_FB + * Apply the generic implicit feedback sync mode (same as implicit_fb=1 option) + * QUIRK_FLAG_SKIP_IMPLICIT_FB + * Don't apply implicit feedback sync mode */ #define QUIRK_FLAG_GET_SAMPLE_RATE (1U << 0) @@ -183,5 +188,7 @@ extern bool snd_usb_skip_validation; #define QUIRK_FLAG_IGNORE_CTL_ERROR (1U << 14) #define QUIRK_FLAG_DSD_RAW (1U << 15) #define QUIRK_FLAG_SET_IFACE_FIRST (1U << 16) +#define QUIRK_FLAG_GENERIC_IMPLICIT_FB (1U << 17) +#define QUIRK_FLAG_SKIP_IMPLICIT_FB (1U << 18) #endif /* __USBAUDIO_H */ diff --git a/sound/usb/usx2y/usb_stream.c b/sound/usb/usx2y/usb_stream.c index 9d0e447938..a4d32e8a1d 100644 --- a/sound/usb/usx2y/usb_stream.c +++ b/sound/usb/usx2y/usb_stream.c @@ -51,7 +51,7 @@ static int init_pipe_urbs(struct usb_stream_kernel *sk, { int u, p; int maxpacket = use_packsize ? - use_packsize : usb_maxpacket(dev, pipe, usb_pipeout(pipe)); + use_packsize : usb_maxpacket(dev, pipe); int transfer_length = maxpacket * sk->n_o_ps; for (u = 0; u < USB_STREAM_NURBS; @@ -171,7 +171,7 @@ struct usb_stream *usb_stream_new(struct usb_stream_kernel *sk, out_pipe = usb_sndisocpipe(dev, out_endpoint); max_packsize = use_packsize ? - use_packsize : usb_maxpacket(dev, in_pipe, 0); + use_packsize : usb_maxpacket(dev, in_pipe); /* t_period = period_frames / sample_rate @@ -187,7 +187,7 @@ struct usb_stream *usb_stream_new(struct usb_stream_kernel *sk, read_size += packets * USB_STREAM_URBDEPTH * (max_packsize + sizeof(struct usb_stream_packet)); - max_packsize = usb_maxpacket(dev, out_pipe, 1); + max_packsize = usb_maxpacket(dev, out_pipe); write_size = max_packsize * packets * USB_STREAM_URBDEPTH; if (read_size >= 256*PAGE_SIZE || write_size >= 256*PAGE_SIZE) { diff --git a/sound/usb/usx2y/usbusx2yaudio.c b/sound/usb/usx2y/usbusx2yaudio.c index cfc1ea5397..9cd5e3aae4 100644 --- a/sound/usb/usx2y/usbusx2yaudio.c +++ b/sound/usb/usx2y/usbusx2yaudio.c @@ -421,7 +421,7 @@ static int usx2y_urbs_allocate(struct snd_usx2y_substream *subs) pipe = is_playback ? usb_sndisocpipe(dev, subs->endpoint) : usb_rcvisocpipe(dev, subs->endpoint); - subs->maxpacksize = usb_maxpacket(dev, pipe, is_playback); + subs->maxpacksize = usb_maxpacket(dev, pipe); if (!subs->maxpacksize) return -EINVAL; diff --git a/sound/usb/usx2y/usx2yhwdeppcm.c b/sound/usb/usx2y/usx2yhwdeppcm.c index db83522c1b..240349b644 100644 --- a/sound/usb/usx2y/usx2yhwdeppcm.c +++ b/sound/usb/usx2y/usx2yhwdeppcm.c @@ -321,7 +321,7 @@ static int usx2y_usbpcm_urbs_allocate(struct snd_usx2y_substream *subs) pipe = is_playback ? usb_sndisocpipe(dev, subs->endpoint) : usb_rcvisocpipe(dev, subs->endpoint); - subs->maxpacksize = usb_maxpacket(dev, pipe, is_playback); + subs->maxpacksize = usb_maxpacket(dev, pipe); if (!subs->maxpacksize) return -EINVAL; diff --git a/sound/x86/intel_hdmi_audio.c b/sound/x86/intel_hdmi_audio.c index 0d828e35b4..ab95fb34a6 100644 --- a/sound/x86/intel_hdmi_audio.c +++ b/sound/x86/intel_hdmi_audio.c @@ -33,6 +33,8 @@ #include #include "intel_hdmi_audio.h" +#define INTEL_HDMI_AUDIO_SUSPEND_DELAY_MS 5000 + #define for_each_pipe(card_ctx, pipe) \ for ((pipe) = 0; (pipe) < (card_ctx)->num_pipes; (pipe)++) #define for_each_port(card_ctx, port) \ @@ -1066,7 +1068,9 @@ static int had_pcm_open(struct snd_pcm_substream *substream) intelhaddata = snd_pcm_substream_chip(substream); runtime = substream->runtime; - pm_runtime_get_sync(intelhaddata->dev); + retval = pm_runtime_resume_and_get(intelhaddata->dev); + if (retval < 0) + return retval; /* set the runtime hw parameter with local snd_pcm_hardware struct */ runtime->hw = had_pcm_hardware; @@ -1534,8 +1538,12 @@ static void had_audio_wq(struct work_struct *work) container_of(work, struct snd_intelhad, hdmi_audio_wq); struct intel_hdmi_lpe_audio_pdata *pdata = ctx->dev->platform_data; struct intel_hdmi_lpe_audio_port_pdata *ppdata = &pdata->port[ctx->port]; + int ret; + + ret = pm_runtime_resume_and_get(ctx->dev); + if (ret < 0) + return; - pm_runtime_get_sync(ctx->dev); mutex_lock(&ctx->mutex); if (ppdata->pipe < 0) { dev_dbg(ctx->dev, "%s: Event: HAD_NOTIFY_HOT_UNPLUG : port = %d\n", @@ -1802,8 +1810,11 @@ static int __hdmi_lpe_audio_probe(struct platform_device *pdev) pdata->notify_audio_lpe = notify_audio_lpe; spin_unlock_irq(&pdata->lpe_audio_slock); + pm_runtime_set_autosuspend_delay(&pdev->dev, INTEL_HDMI_AUDIO_SUSPEND_DELAY_MS); pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); pm_runtime_mark_last_busy(&pdev->dev); + pm_runtime_idle(&pdev->dev); dev_dbg(&pdev->dev, "%s: handle pending notification\n", __func__); for_each_port(card_ctx, port) { diff --git a/sound/xen/xen_snd_front_evtchnl.c b/sound/xen/xen_snd_front_evtchnl.c index ecbc294fc5..26d1b39878 100644 --- a/sound/xen/xen_snd_front_evtchnl.c +++ b/sound/xen/xen_snd_front_evtchnl.c @@ -143,12 +143,12 @@ void xen_snd_front_evtchnl_flush(struct xen_snd_front_evtchnl *channel) static void evtchnl_free(struct xen_snd_front_info *front_info, struct xen_snd_front_evtchnl *channel) { - unsigned long page = 0; + void *page = NULL; if (channel->type == EVTCHNL_TYPE_REQ) - page = (unsigned long)channel->u.req.ring.sring; + page = channel->u.req.ring.sring; else if (channel->type == EVTCHNL_TYPE_EVT) - page = (unsigned long)channel->u.evt.page; + page = channel->u.evt.page; if (!page) return; @@ -167,10 +167,7 @@ static void evtchnl_free(struct xen_snd_front_info *front_info, xenbus_free_evtchn(front_info->xb_dev, channel->port); /* End access and free the page. */ - if (channel->gref != GRANT_INVALID_REF) - gnttab_end_foreign_access(channel->gref, page); - else - free_page(page); + xenbus_teardown_ring(&page, 1, &channel->gref); memset(channel, 0, sizeof(*channel)); } @@ -196,8 +193,7 @@ static int evtchnl_alloc(struct xen_snd_front_info *front_info, int index, enum xen_snd_front_evtchnl_type type) { struct xenbus_device *xb_dev = front_info->xb_dev; - unsigned long page; - grant_ref_t gref; + void *page; irq_handler_t handler; char *handler_name = NULL; int ret; @@ -207,12 +203,9 @@ static int evtchnl_alloc(struct xen_snd_front_info *front_info, int index, channel->index = index; channel->front_info = front_info; channel->state = EVTCHNL_STATE_DISCONNECTED; - channel->gref = GRANT_INVALID_REF; - page = get_zeroed_page(GFP_KERNEL); - if (!page) { - ret = -ENOMEM; + ret = xenbus_setup_ring(xb_dev, GFP_KERNEL, &page, 1, &channel->gref); + if (ret) goto fail; - } handler_name = kasprintf(GFP_KERNEL, "%s-%s", XENSND_DRIVER_NAME, type == EVTCHNL_TYPE_REQ ? @@ -226,33 +219,18 @@ static int evtchnl_alloc(struct xen_snd_front_info *front_info, int index, mutex_init(&channel->ring_io_lock); if (type == EVTCHNL_TYPE_REQ) { - struct xen_sndif_sring *sring = (struct xen_sndif_sring *)page; + struct xen_sndif_sring *sring = page; init_completion(&channel->u.req.completion); mutex_init(&channel->u.req.req_io_lock); - SHARED_RING_INIT(sring); - FRONT_RING_INIT(&channel->u.req.ring, sring, XEN_PAGE_SIZE); - - ret = xenbus_grant_ring(xb_dev, sring, 1, &gref); - if (ret < 0) { - channel->u.req.ring.sring = NULL; - goto fail; - } + XEN_FRONT_RING_INIT(&channel->u.req.ring, sring, XEN_PAGE_SIZE); handler = evtchnl_interrupt_req; } else { - ret = gnttab_grant_foreign_access(xb_dev->otherend_id, - virt_to_gfn((void *)page), 0); - if (ret < 0) - goto fail; - - channel->u.evt.page = (struct xensnd_event_page *)page; - gref = ret; + channel->u.evt.page = page; handler = evtchnl_interrupt_evt; } - channel->gref = gref; - ret = xenbus_alloc_evtchn(xb_dev, &channel->port); if (ret < 0) goto fail; @@ -279,8 +257,6 @@ static int evtchnl_alloc(struct xen_snd_front_info *front_info, int index, return 0; fail: - if (page) - free_page(page); kfree(handler_name); dev_err(&xb_dev->dev, "Failed to allocate ring: %d\n", ret); return ret; diff --git a/sound/xen/xen_snd_front_evtchnl.h b/sound/xen/xen_snd_front_evtchnl.h index cbe51fd1ec..3675fba705 100644 --- a/sound/xen/xen_snd_front_evtchnl.h +++ b/sound/xen/xen_snd_front_evtchnl.h @@ -15,15 +15,6 @@ struct xen_snd_front_info; -#ifndef GRANT_INVALID_REF -/* - * FIXME: usage of grant reference 0 as invalid grant reference: - * grant reference 0 is valid, but never exposed to a PV driver, - * because of the fact it is already in use/reserved by the PV console. - */ -#define GRANT_INVALID_REF 0 -#endif - /* Timeout in ms to wait for backend to respond. */ #define VSND_WAIT_BACK_MS 3000 diff --git a/tools/Makefile b/tools/Makefile index db2f7b8ebe..e497875fc7 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -24,6 +24,7 @@ help: @echo ' intel-speed-select - Intel Speed Select tool' @echo ' kvm_stat - top-like utility for displaying kvm statistics' @echo ' leds - LEDs tools' + @echo ' nolibc - nolibc headers testing and installation' @echo ' objtool - an ELF object analysis tool' @echo ' pci - PCI tools' @echo ' perf - Linux performance measurement and analysis tool' @@ -31,6 +32,9 @@ help: @echo ' bootconfig - boot config tool' @echo ' spi - spi tools' @echo ' tmon - thermal monitoring and tuning tool' + @echo ' thermometer - temperature capture tool' + @echo ' thermal-engine - thermal monitoring tool' + @echo ' thermal - thermal library' @echo ' tracing - misc tracing tools' @echo ' turbostat - Intel CPU idle stats and freq reporting tool' @echo ' usb - USB testing tools' @@ -74,6 +78,12 @@ bpf/%: FORCE libapi: FORCE $(call descend,lib/api) +nolibc: FORCE + $(call descend,include/nolibc) + +nolibc_%: FORCE + $(call descend,include/nolibc,$(patsubst nolibc_%,%,$@)) + # The perf build does not follow the descend function setup, # invoking it via it's own make rule. PERF_O = $(if $(O),$(O)/tools/perf,) @@ -85,12 +95,21 @@ perf: FORCE selftests: FORCE $(call descend,testing/$@) +thermal: FORCE + $(call descend,lib/$@) + turbostat x86_energy_perf_policy intel-speed-select: FORCE $(call descend,power/x86/$@) tmon: FORCE $(call descend,thermal/$@) +thermometer: FORCE + $(call descend,thermal/$@) + +thermal-engine: FORCE thermal + $(call descend,thermal/$@) + freefall: FORCE $(call descend,laptop/$@) @@ -101,7 +120,7 @@ all: acpi cgroup counter cpupower gpio hv firewire \ perf selftests bootconfig spi turbostat usb \ virtio vm bpf x86_energy_perf_policy \ tmon freefall iio objtool kvm_stat wmi \ - pci debugging tracing + pci debugging tracing thermal thermometer thermal-engine acpi_install: $(call descend,power/$(@:_install=),install) @@ -115,12 +134,21 @@ cgroup_install counter_install firewire_install gpio_install hv_install iio_inst selftests_install: $(call descend,testing/$(@:_install=),install) +thermal_install: + $(call descend,lib/$(@:_install=),install) + turbostat_install x86_energy_perf_policy_install intel-speed-select_install: $(call descend,power/x86/$(@:_install=),install) tmon_install: $(call descend,thermal/$(@:_install=),install) +thermometer_install: + $(call descend,thermal/$(@:_install=),install) + +thermal-engine_install: + $(call descend,thermal/$(@:_install=),install) + freefall_install: $(call descend,laptop/$(@:_install=),install) @@ -133,7 +161,7 @@ install: acpi_install cgroup_install counter_install cpupower_install gpio_insta virtio_install vm_install bpf_install x86_energy_perf_policy_install \ tmon_install freefall_install objtool_install kvm_stat_install \ wmi_install pci_install debugging_install intel-speed-select_install \ - tracing_install + tracing_install thermometer_install thermal-engine_install acpi_clean: $(call descend,power/acpi,clean) @@ -160,9 +188,18 @@ perf_clean: selftests_clean: $(call descend,testing/$(@:_clean=),clean) +thermal_clean: + $(call descend,lib/thermal,clean) + turbostat_clean x86_energy_perf_policy_clean intel-speed-select_clean: $(call descend,power/x86/$(@:_clean=),clean) +thermometer_clean: + $(call descend,thermal/thermometer,clean) + +thermal-engine_clean: + $(call descend,thermal/thermal-engine,clean) + tmon_clean: $(call descend,thermal/tmon,clean) @@ -177,6 +214,6 @@ clean: acpi_clean cgroup_clean counter_clean cpupower_clean hv_clean firewire_cl vm_clean bpf_clean iio_clean x86_energy_perf_policy_clean tmon_clean \ freefall_clean build_clean libbpf_clean libsubcmd_clean \ gpio_clean objtool_clean leds_clean wmi_clean pci_clean firmware_clean debugging_clean \ - intel-speed-select_clean tracing_clean + intel-speed-select_clean tracing_clean thermal_clean thermometer_clean thermal-engine_clean .PHONY: FORCE diff --git a/tools/accounting/.gitignore b/tools/accounting/.gitignore index c45fb4ed43..522a690aaf 100644 --- a/tools/accounting/.gitignore +++ b/tools/accounting/.gitignore @@ -1,2 +1,3 @@ # SPDX-License-Identifier: GPL-2.0-only getdelays +procacct diff --git a/tools/accounting/Makefile b/tools/accounting/Makefile index 03687f19cb..11def1ad04 100644 --- a/tools/accounting/Makefile +++ b/tools/accounting/Makefile @@ -2,7 +2,7 @@ CC := $(CROSS_COMPILE)gcc CFLAGS := -I../../usr/include -PROGS := getdelays +PROGS := getdelays procacct all: $(PROGS) diff --git a/tools/accounting/getdelays.c b/tools/accounting/getdelays.c index 11e8673945..938dec0dfa 100644 --- a/tools/accounting/getdelays.c +++ b/tools/accounting/getdelays.c @@ -45,7 +45,6 @@ exit(code); \ } while (0) -int done; int rcvbufsz; char name[100]; int dbg; @@ -207,6 +206,8 @@ static void print_delayacct(struct taskstats *t) "THRASHING%12s%15s%15s\n" " %15llu%15llu%15llums\n" "COMPACT %12s%15s%15s\n" + " %15llu%15llu%15llums\n" + "WPCOPY %12s%15s%15s\n" " %15llu%15llu%15llums\n", "count", "real total", "virtual total", "delay total", "delay average", @@ -234,7 +235,11 @@ static void print_delayacct(struct taskstats *t) "count", "delay total", "delay average", (unsigned long long)t->compact_count, (unsigned long long)t->compact_delay_total, - average_ms(t->compact_delay_total, t->compact_count)); + average_ms(t->compact_delay_total, t->compact_count), + "count", "delay total", "delay average", + (unsigned long long)t->wpcopy_count, + (unsigned long long)t->wpcopy_delay_total, + average_ms(t->wpcopy_delay_total, t->wpcopy_count)); } static void task_context_switch_counts(struct taskstats *t) @@ -279,7 +284,6 @@ int main(int argc, char *argv[]) pid_t rtid = 0; int fd = 0; - int count = 0; int write_file = 0; int maskset = 0; char *logfile = NULL; @@ -489,7 +493,6 @@ int main(int argc, char *argv[]) len2 = 0; /* For nested attributes, na follows */ na = (struct nlattr *) NLA_DATA(na); - done = 0; while (len2 < aggr_len) { switch (na->nla_type) { case TASKSTATS_TYPE_PID: @@ -503,7 +506,6 @@ int main(int argc, char *argv[]) printf("TGID\t%d\n", rtid); break; case TASKSTATS_TYPE_STATS: - count++; if (print_delays) print_delayacct((struct taskstats *) NLA_DATA(na)); if (print_io_accounting) diff --git a/tools/arch/arm64/include/asm/cputype.h b/tools/arch/arm64/include/asm/cputype.h index e09d6908a2..8aa0d276a6 100644 --- a/tools/arch/arm64/include/asm/cputype.h +++ b/tools/arch/arm64/include/asm/cputype.h @@ -36,7 +36,7 @@ #define MIDR_VARIANT(midr) \ (((midr) & MIDR_VARIANT_MASK) >> MIDR_VARIANT_SHIFT) #define MIDR_IMPLEMENTOR_SHIFT 24 -#define MIDR_IMPLEMENTOR_MASK (0xff << MIDR_IMPLEMENTOR_SHIFT) +#define MIDR_IMPLEMENTOR_MASK (0xffU << MIDR_IMPLEMENTOR_SHIFT) #define MIDR_IMPLEMENTOR(midr) \ (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) @@ -118,6 +118,10 @@ #define APPLE_CPU_PART_M1_ICESTORM 0x022 #define APPLE_CPU_PART_M1_FIRESTORM 0x023 +#define APPLE_CPU_PART_M1_ICESTORM_PRO 0x024 +#define APPLE_CPU_PART_M1_FIRESTORM_PRO 0x025 +#define APPLE_CPU_PART_M1_ICESTORM_MAX 0x028 +#define APPLE_CPU_PART_M1_FIRESTORM_MAX 0x029 #define MIDR_CORTEX_A53 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A53) #define MIDR_CORTEX_A57 MIDR_CPU_MODEL(ARM_CPU_IMP_ARM, ARM_CPU_PART_CORTEX_A57) @@ -164,6 +168,10 @@ #define MIDR_HISI_TSV110 MIDR_CPU_MODEL(ARM_CPU_IMP_HISI, HISI_CPU_PART_TSV110) #define MIDR_APPLE_M1_ICESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM) #define MIDR_APPLE_M1_FIRESTORM MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM) +#define MIDR_APPLE_M1_ICESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_PRO) +#define MIDR_APPLE_M1_FIRESTORM_PRO MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_PRO) +#define MIDR_APPLE_M1_ICESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_ICESTORM_MAX) +#define MIDR_APPLE_M1_FIRESTORM_MAX MIDR_CPU_MODEL(ARM_CPU_IMP_APPLE, APPLE_CPU_PART_M1_FIRESTORM_MAX) /* Fujitsu Erratum 010001 affects A64FX 1.0 and 1.1, (v0r0 and v1r0) */ #define MIDR_FUJITSU_ERRATUM_010001 MIDR_FUJITSU_A64FX @@ -172,7 +180,7 @@ #ifndef __ASSEMBLY__ -#include "sysreg.h" +#include #define read_cpuid(reg) read_sysreg_s(SYS_ ## reg) diff --git a/tools/arch/arm64/include/uapi/asm/kvm.h b/tools/arch/arm64/include/uapi/asm/kvm.h index c1b6ddc02d..316917b987 100644 --- a/tools/arch/arm64/include/uapi/asm/kvm.h +++ b/tools/arch/arm64/include/uapi/asm/kvm.h @@ -75,9 +75,11 @@ struct kvm_regs { /* KVM_ARM_SET_DEVICE_ADDR ioctl id encoding */ #define KVM_ARM_DEVICE_TYPE_SHIFT 0 -#define KVM_ARM_DEVICE_TYPE_MASK (0xffff << KVM_ARM_DEVICE_TYPE_SHIFT) +#define KVM_ARM_DEVICE_TYPE_MASK GENMASK(KVM_ARM_DEVICE_TYPE_SHIFT + 15, \ + KVM_ARM_DEVICE_TYPE_SHIFT) #define KVM_ARM_DEVICE_ID_SHIFT 16 -#define KVM_ARM_DEVICE_ID_MASK (0xffff << KVM_ARM_DEVICE_ID_SHIFT) +#define KVM_ARM_DEVICE_ID_MASK GENMASK(KVM_ARM_DEVICE_ID_SHIFT + 15, \ + KVM_ARM_DEVICE_ID_SHIFT) /* Supported device IDs */ #define KVM_ARM_DEVICE_VGIC_V2 0 @@ -139,8 +141,10 @@ struct kvm_guest_debug_arch { __u64 dbg_wvr[KVM_ARM_MAX_DBG_REGS]; }; +#define KVM_DEBUG_ARCH_HSR_HIGH_VALID (1 << 0) struct kvm_debug_exit_arch { __u32 hsr; + __u32 hsr_high; /* ESR_EL2[61:32] */ __u64 far; /* used for watchpoints */ }; @@ -332,6 +336,40 @@ struct kvm_arm_copy_mte_tags { #define KVM_ARM64_SVE_VLS_WORDS \ ((KVM_ARM64_SVE_VQ_MAX - KVM_ARM64_SVE_VQ_MIN) / 64 + 1) +/* Bitmap feature firmware registers */ +#define KVM_REG_ARM_FW_FEAT_BMAP (0x0016 << KVM_REG_ARM_COPROC_SHIFT) +#define KVM_REG_ARM_FW_FEAT_BMAP_REG(r) (KVM_REG_ARM64 | KVM_REG_SIZE_U64 | \ + KVM_REG_ARM_FW_FEAT_BMAP | \ + ((r) & 0xffff)) + +#define KVM_REG_ARM_STD_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(0) + +enum { + KVM_REG_ARM_STD_BIT_TRNG_V1_0 = 0, +#ifdef __KERNEL__ + KVM_REG_ARM_STD_BMAP_BIT_COUNT, +#endif +}; + +#define KVM_REG_ARM_STD_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(1) + +enum { + KVM_REG_ARM_STD_HYP_BIT_PV_TIME = 0, +#ifdef __KERNEL__ + KVM_REG_ARM_STD_HYP_BMAP_BIT_COUNT, +#endif +}; + +#define KVM_REG_ARM_VENDOR_HYP_BMAP KVM_REG_ARM_FW_FEAT_BMAP_REG(2) + +enum { + KVM_REG_ARM_VENDOR_HYP_BIT_FUNC_FEAT = 0, + KVM_REG_ARM_VENDOR_HYP_BIT_PTP = 1, +#ifdef __KERNEL__ + KVM_REG_ARM_VENDOR_HYP_BMAP_BIT_COUNT, +#endif +}; + /* Device Control API: ARM VGIC */ #define KVM_DEV_ARM_VGIC_GRP_ADDR 0 #define KVM_DEV_ARM_VGIC_GRP_DIST_REGS 1 diff --git a/tools/arch/arm64/include/uapi/asm/perf_regs.h b/tools/arch/arm64/include/uapi/asm/perf_regs.h index d54daafa89..fd157f4672 100644 --- a/tools/arch/arm64/include/uapi/asm/perf_regs.h +++ b/tools/arch/arm64/include/uapi/asm/perf_regs.h @@ -36,6 +36,11 @@ enum perf_event_arm_regs { PERF_REG_ARM64_LR, PERF_REG_ARM64_SP, PERF_REG_ARM64_PC, - PERF_REG_ARM64_MAX, + + /* Extended/pseudo registers */ + PERF_REG_ARM64_VG = 46, // SVE Vector Granule + + PERF_REG_ARM64_MAX = PERF_REG_ARM64_PC + 1, + PERF_REG_ARM64_EXTENDED_MAX = PERF_REG_ARM64_VG + 1 }; #endif /* _ASM_ARM64_PERF_REGS_H */ diff --git a/tools/arch/s390/include/uapi/asm/kvm.h b/tools/arch/s390/include/uapi/asm/kvm.h index 7a6b14874d..a73cf01a16 100644 --- a/tools/arch/s390/include/uapi/asm/kvm.h +++ b/tools/arch/s390/include/uapi/asm/kvm.h @@ -74,6 +74,7 @@ struct kvm_s390_io_adapter_req { #define KVM_S390_VM_CRYPTO 2 #define KVM_S390_VM_CPU_MODEL 3 #define KVM_S390_VM_MIGRATION 4 +#define KVM_S390_VM_CPU_TOPOLOGY 5 /* kvm attributes for mem_ctrl */ #define KVM_S390_VM_MEM_ENABLE_CMMA 0 diff --git a/tools/arch/x86/include/asm/amd-ibs.h b/tools/arch/x86/include/asm/amd-ibs.h index 174e7d83fc..9a3312e12e 100644 --- a/tools/arch/x86/include/asm/amd-ibs.h +++ b/tools/arch/x86/include/asm/amd-ibs.h @@ -29,7 +29,10 @@ union ibs_fetch_ctl { rand_en:1, /* 57: random tagging enable */ fetch_l2_miss:1,/* 58: L2 miss for sampled fetch * (needs IbsFetchComp) */ - reserved:5; /* 59-63: reserved */ + l3_miss_only:1, /* 59: Collect L3 miss samples only */ + fetch_oc_miss:1,/* 60: Op cache miss for the sampled fetch */ + fetch_l3_miss:1,/* 61: L3 cache miss for the sampled fetch */ + reserved:2; /* 62-63: reserved */ }; }; @@ -38,18 +41,18 @@ union ibs_op_ctl { __u64 val; struct { __u64 opmaxcnt:16, /* 0-15: periodic op max. count */ - reserved0:1, /* 16: reserved */ + l3_miss_only:1, /* 16: Collect L3 miss samples only */ op_en:1, /* 17: op sampling enable */ op_val:1, /* 18: op sample valid */ cnt_ctl:1, /* 19: periodic op counter control */ opmaxcnt_ext:7, /* 20-26: upper 7 bits of periodic op maximum count */ - reserved1:5, /* 27-31: reserved */ + reserved0:5, /* 27-31: reserved */ opcurcnt:27, /* 32-58: periodic op counter current count */ - reserved2:5; /* 59-63: reserved */ + reserved1:5; /* 59-63: reserved */ }; }; -/* MSR 0xc0011035: IBS Op Data 2 */ +/* MSR 0xc0011035: IBS Op Data 1 */ union ibs_op_data { __u64 val; struct { @@ -71,11 +74,12 @@ union ibs_op_data { union ibs_op_data2 { __u64 val; struct { - __u64 data_src:3, /* 0-2: data source */ + __u64 data_src_lo:3, /* 0-2: data source low */ reserved0:1, /* 3: reserved */ rmt_node:1, /* 4: destination node */ cache_hit_st:1, /* 5: cache hit state */ - reserved1:57; /* 5-63: reserved */ + data_src_hi:2, /* 6-7: data source high */ + reserved1:56; /* 8-63: reserved */ }; }; diff --git a/tools/arch/x86/include/asm/cpufeatures.h b/tools/arch/x86/include/asm/cpufeatures.h index 73e643ae94..235dc85c91 100644 --- a/tools/arch/x86/include/asm/cpufeatures.h +++ b/tools/arch/x86/include/asm/cpufeatures.h @@ -201,17 +201,17 @@ #define X86_FEATURE_INVPCID_SINGLE ( 7*32+ 7) /* Effectively INVPCID && CR4.PCIDE=1 */ #define X86_FEATURE_HW_PSTATE ( 7*32+ 8) /* AMD HW-PState */ #define X86_FEATURE_PROC_FEEDBACK ( 7*32+ 9) /* AMD ProcFeedbackInterface */ -/* FREE! ( 7*32+10) */ +#define X86_FEATURE_XCOMPACTED ( 7*32+10) /* "" Use compacted XSTATE (XSAVES or XSAVEC) */ #define X86_FEATURE_PTI ( 7*32+11) /* Kernel Page Table Isolation enabled */ -#define X86_FEATURE_RETPOLINE ( 7*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ -#define X86_FEATURE_RETPOLINE_LFENCE ( 7*32+13) /* "" Use LFENCE for Spectre variant 2 */ +#define X86_FEATURE_KERNEL_IBRS ( 7*32+12) /* "" Set/clear IBRS on kernel entry/exit */ +#define X86_FEATURE_RSB_VMEXIT ( 7*32+13) /* "" Fill RSB on VM-Exit */ #define X86_FEATURE_INTEL_PPIN ( 7*32+14) /* Intel Processor Inventory Number */ #define X86_FEATURE_CDP_L2 ( 7*32+15) /* Code and Data Prioritization L2 */ #define X86_FEATURE_MSR_SPEC_CTRL ( 7*32+16) /* "" MSR SPEC_CTRL is implemented */ #define X86_FEATURE_SSBD ( 7*32+17) /* Speculative Store Bypass Disable */ #define X86_FEATURE_MBA ( 7*32+18) /* Memory Bandwidth Allocation */ #define X86_FEATURE_RSB_CTXSW ( 7*32+19) /* "" Fill RSB on context switches */ -/* FREE! ( 7*32+20) */ +#define X86_FEATURE_PERFMON_V2 ( 7*32+20) /* AMD Performance Monitoring Version 2 */ #define X86_FEATURE_USE_IBPB ( 7*32+21) /* "" Indirect Branch Prediction Barrier enabled */ #define X86_FEATURE_USE_IBRS_FW ( 7*32+22) /* "" Use IBRS during runtime firmware calls */ #define X86_FEATURE_SPEC_STORE_BYPASS_DISABLE ( 7*32+23) /* "" Disable Speculative Store Bypass. */ @@ -219,7 +219,7 @@ #define X86_FEATURE_IBRS ( 7*32+25) /* Indirect Branch Restricted Speculation */ #define X86_FEATURE_IBPB ( 7*32+26) /* Indirect Branch Prediction Barrier */ #define X86_FEATURE_STIBP ( 7*32+27) /* Single Thread Indirect Branch Predictors */ -#define X86_FEATURE_ZEN ( 7*32+28) /* "" CPU is AMD family 0x17 or above (Zen) */ +#define X86_FEATURE_ZEN (7*32+28) /* "" CPU based on Zen microarchitecture */ #define X86_FEATURE_L1TF_PTEINV ( 7*32+29) /* "" L1TF workaround PTE inversion */ #define X86_FEATURE_IBRS_ENHANCED ( 7*32+30) /* Enhanced IBRS */ #define X86_FEATURE_MSR_IA32_FEAT_CTL ( 7*32+31) /* "" MSR IA32_FEAT_CTL configured */ @@ -238,6 +238,7 @@ #define X86_FEATURE_VMW_VMMCALL ( 8*32+19) /* "" VMware prefers VMMCALL hypercall instruction */ #define X86_FEATURE_PVUNLOCK ( 8*32+20) /* "" PV unlock function */ #define X86_FEATURE_VCPUPREEMPT ( 8*32+21) /* "" PV vcpu_is_preempted function */ +#define X86_FEATURE_TDX_GUEST ( 8*32+22) /* Intel Trust Domain Extensions Guest */ /* Intel-defined CPU features, CPUID level 0x00000007:0 (EBX), word 9 */ #define X86_FEATURE_FSGSBASE ( 9*32+ 0) /* RDFSBASE, WRFSBASE, RDGSBASE, WRGSBASE instructions*/ @@ -295,6 +296,14 @@ #define X86_FEATURE_PER_THREAD_MBA (11*32+ 7) /* "" Per-thread Memory Bandwidth Allocation */ #define X86_FEATURE_SGX1 (11*32+ 8) /* "" Basic SGX */ #define X86_FEATURE_SGX2 (11*32+ 9) /* "" SGX Enclave Dynamic Memory Management (EDMM) */ +#define X86_FEATURE_ENTRY_IBPB (11*32+10) /* "" Issue an IBPB on kernel entry */ +#define X86_FEATURE_RRSBA_CTRL (11*32+11) /* "" RET prediction control */ +#define X86_FEATURE_RETPOLINE (11*32+12) /* "" Generic Retpoline mitigation for Spectre variant 2 */ +#define X86_FEATURE_RETPOLINE_LFENCE (11*32+13) /* "" Use LFENCE for Spectre variant 2 */ +#define X86_FEATURE_RETHUNK (11*32+14) /* "" Use REturn THUNK */ +#define X86_FEATURE_UNRET (11*32+15) /* "" AMD BTB untrain return */ +#define X86_FEATURE_USE_IBPB_FW (11*32+16) /* "" Use IBPB during runtime firmware calls */ +#define X86_FEATURE_RSB_VMEXIT_LITE (11*32+17) /* "" Fill RSB on VM exit when EIBRS is enabled */ /* Intel-defined CPU features, CPUID level 0x00000007:1 (EAX), word 12 */ #define X86_FEATURE_AVX_VNNI (12*32+ 4) /* AVX VNNI instructions */ @@ -315,6 +324,8 @@ #define X86_FEATURE_VIRT_SSBD (13*32+25) /* Virtualized Speculative Store Bypass Disable */ #define X86_FEATURE_AMD_SSB_NO (13*32+26) /* "" Speculative Store Bypass is fixed in hardware. */ #define X86_FEATURE_CPPC (13*32+27) /* Collaborative Processor Performance Control */ +#define X86_FEATURE_BTC_NO (13*32+29) /* "" Not vulnerable to Branch Type Confusion */ +#define X86_FEATURE_BRS (13*32+31) /* Branch Sampling available */ /* Thermal and Power Management Leaf, CPUID level 0x00000006 (EAX), word 14 */ #define X86_FEATURE_DTHERM (14*32+ 0) /* Digital Thermal Sensor */ @@ -343,6 +354,7 @@ #define X86_FEATURE_AVIC (15*32+13) /* Virtual Interrupt Controller */ #define X86_FEATURE_V_VMSAVE_VMLOAD (15*32+15) /* Virtual VMSAVE VMLOAD */ #define X86_FEATURE_VGIF (15*32+16) /* Virtual GIF */ +#define X86_FEATURE_X2AVIC (15*32+18) /* Virtual x2apic */ #define X86_FEATURE_V_SPEC_CTRL (15*32+20) /* Virtual SPEC_CTRL */ #define X86_FEATURE_SVME_ADDR_CHK (15*32+28) /* "" SVME addr check */ @@ -405,6 +417,7 @@ #define X86_FEATURE_SEV (19*32+ 1) /* AMD Secure Encrypted Virtualization */ #define X86_FEATURE_VM_PAGE_FLUSH (19*32+ 2) /* "" VM Page Flush MSR is supported */ #define X86_FEATURE_SEV_ES (19*32+ 3) /* AMD Secure Encrypted Virtualization - Encrypted State */ +#define X86_FEATURE_V_TSC_AUX (19*32+ 9) /* "" Virtual TSC_AUX */ #define X86_FEATURE_SME_COHERENT (19*32+10) /* "" AMD hardware-enforced cache coherency */ /* @@ -443,5 +456,8 @@ #define X86_BUG_TAA X86_BUG(22) /* CPU is affected by TSX Async Abort(TAA) */ #define X86_BUG_ITLB_MULTIHIT X86_BUG(23) /* CPU may incur MCE during certain page attribute changes */ #define X86_BUG_SRBDS X86_BUG(24) /* CPU may leak RNG bits if not mitigated */ +#define X86_BUG_MMIO_STALE_DATA X86_BUG(25) /* CPU is affected by Processor MMIO Stale Data vulnerabilities */ +#define X86_BUG_RETBLEED X86_BUG(26) /* CPU is affected by RETBleed */ +#define X86_BUG_EIBRS_PBRSB X86_BUG(27) /* EIBRS is vulnerable to Post Barrier RSB Predictions */ #endif /* _ASM_X86_CPUFEATURES_H */ diff --git a/tools/arch/x86/include/asm/disabled-features.h b/tools/arch/x86/include/asm/disabled-features.h index 1231d63f83..33d2cd04d2 100644 --- a/tools/arch/x86/include/asm/disabled-features.h +++ b/tools/arch/x86/include/asm/disabled-features.h @@ -10,12 +10,6 @@ * cpu_feature_enabled(). */ -#ifdef CONFIG_X86_SMAP -# define DISABLE_SMAP 0 -#else -# define DISABLE_SMAP (1<<(X86_FEATURE_SMAP & 31)) -#endif - #ifdef CONFIG_X86_UMIP # define DISABLE_UMIP 0 #else @@ -56,6 +50,25 @@ # define DISABLE_PTI (1 << (X86_FEATURE_PTI & 31)) #endif +#ifdef CONFIG_RETPOLINE +# define DISABLE_RETPOLINE 0 +#else +# define DISABLE_RETPOLINE ((1 << (X86_FEATURE_RETPOLINE & 31)) | \ + (1 << (X86_FEATURE_RETPOLINE_LFENCE & 31))) +#endif + +#ifdef CONFIG_RETHUNK +# define DISABLE_RETHUNK 0 +#else +# define DISABLE_RETHUNK (1 << (X86_FEATURE_RETHUNK & 31)) +#endif + +#ifdef CONFIG_CPU_UNRET_ENTRY +# define DISABLE_UNRET 0 +#else +# define DISABLE_UNRET (1 << (X86_FEATURE_UNRET & 31)) +#endif + #ifdef CONFIG_INTEL_IOMMU_SVM # define DISABLE_ENQCMD 0 #else @@ -68,6 +81,12 @@ # define DISABLE_SGX (1 << (X86_FEATURE_SGX & 31)) #endif +#ifdef CONFIG_INTEL_TDX_GUEST +# define DISABLE_TDX_GUEST 0 +#else +# define DISABLE_TDX_GUEST (1 << (X86_FEATURE_TDX_GUEST & 31)) +#endif + /* * Make sure to add features to the correct mask */ @@ -79,10 +98,10 @@ #define DISABLED_MASK5 0 #define DISABLED_MASK6 0 #define DISABLED_MASK7 (DISABLE_PTI) -#define DISABLED_MASK8 0 -#define DISABLED_MASK9 (DISABLE_SMAP|DISABLE_SGX) +#define DISABLED_MASK8 (DISABLE_TDX_GUEST) +#define DISABLED_MASK9 (DISABLE_SGX) #define DISABLED_MASK10 0 -#define DISABLED_MASK11 0 +#define DISABLED_MASK11 (DISABLE_RETPOLINE|DISABLE_RETHUNK|DISABLE_UNRET) #define DISABLED_MASK12 0 #define DISABLED_MASK13 0 #define DISABLED_MASK14 0 diff --git a/tools/arch/x86/include/asm/msr-index.h b/tools/arch/x86/include/asm/msr-index.h index ee15311b6b..6674bdb096 100644 --- a/tools/arch/x86/include/asm/msr-index.h +++ b/tools/arch/x86/include/asm/msr-index.h @@ -51,6 +51,8 @@ #define SPEC_CTRL_STIBP BIT(SPEC_CTRL_STIBP_SHIFT) /* STIBP mask */ #define SPEC_CTRL_SSBD_SHIFT 2 /* Speculative Store Bypass Disable bit */ #define SPEC_CTRL_SSBD BIT(SPEC_CTRL_SSBD_SHIFT) /* Speculative Store Bypass Disable */ +#define SPEC_CTRL_RRSBA_DIS_S_SHIFT 6 /* Disable RRSBA behavior */ +#define SPEC_CTRL_RRSBA_DIS_S BIT(SPEC_CTRL_RRSBA_DIS_S_SHIFT) #define MSR_IA32_PRED_CMD 0x00000049 /* Prediction Command */ #define PRED_CMD_IBPB BIT(0) /* Indirect Branch Prediction Barrier */ @@ -76,6 +78,8 @@ /* Abbreviated from Intel SDM name IA32_CORE_CAPABILITIES */ #define MSR_IA32_CORE_CAPS 0x000000cf +#define MSR_IA32_CORE_CAPS_INTEGRITY_CAPS_BIT 2 +#define MSR_IA32_CORE_CAPS_INTEGRITY_CAPS BIT(MSR_IA32_CORE_CAPS_INTEGRITY_CAPS_BIT) #define MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT_BIT 5 #define MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT BIT(MSR_IA32_CORE_CAPS_SPLIT_LOCK_DETECT_BIT) @@ -91,6 +95,7 @@ #define MSR_IA32_ARCH_CAPABILITIES 0x0000010a #define ARCH_CAP_RDCL_NO BIT(0) /* Not susceptible to Meltdown */ #define ARCH_CAP_IBRS_ALL BIT(1) /* Enhanced IBRS support */ +#define ARCH_CAP_RSBA BIT(2) /* RET may use alternative branch predictors */ #define ARCH_CAP_SKIP_VMENTRY_L1DFLUSH BIT(3) /* Skip L1D flush on vmentry */ #define ARCH_CAP_SSB_NO BIT(4) /* * Not susceptible to Speculative Store Bypass @@ -114,6 +119,41 @@ * Not susceptible to * TSX Async Abort (TAA) vulnerabilities. */ +#define ARCH_CAP_SBDR_SSDP_NO BIT(13) /* + * Not susceptible to SBDR and SSDP + * variants of Processor MMIO stale data + * vulnerabilities. + */ +#define ARCH_CAP_FBSDP_NO BIT(14) /* + * Not susceptible to FBSDP variant of + * Processor MMIO stale data + * vulnerabilities. + */ +#define ARCH_CAP_PSDP_NO BIT(15) /* + * Not susceptible to PSDP variant of + * Processor MMIO stale data + * vulnerabilities. + */ +#define ARCH_CAP_FB_CLEAR BIT(17) /* + * VERW clears CPU fill buffer + * even on MDS_NO CPUs. + */ +#define ARCH_CAP_FB_CLEAR_CTRL BIT(18) /* + * MSR_IA32_MCU_OPT_CTRL[FB_CLEAR_DIS] + * bit available to control VERW + * behavior. + */ +#define ARCH_CAP_RRSBA BIT(19) /* + * Indicates RET may use predictors + * other than the RSB. With eIBRS + * enabled predictions in kernel mode + * are restricted to targets in + * kernel. + */ +#define ARCH_CAP_PBRSB_NO BIT(24) /* + * Not susceptible to Post-Barrier + * Return Stack Buffer Predictions. + */ #define MSR_IA32_FLUSH_CMD 0x0000010b #define L1D_FLUSH BIT(0) /* @@ -131,6 +171,7 @@ #define MSR_IA32_MCU_OPT_CTRL 0x00000123 #define RNGDS_MITG_DIS BIT(0) /* SRBDS support */ #define RTM_ALLOW BIT(1) /* TSX development mode */ +#define FB_CLEAR_DIS BIT(3) /* CPU Fill buffer clear disable */ #define MSR_IA32_SYSENTER_CS 0x00000174 #define MSR_IA32_SYSENTER_ESP 0x00000175 @@ -154,6 +195,11 @@ #define MSR_IA32_POWER_CTL 0x000001fc #define MSR_IA32_POWER_CTL_BIT_EE 19 +/* Abbreviated from Intel SDM name IA32_INTEGRITY_CAPABILITIES */ +#define MSR_INTEGRITY_CAPS 0x000002d9 +#define MSR_INTEGRITY_CAPS_PERIODIC_BIST_BIT 4 +#define MSR_INTEGRITY_CAPS_PERIODIC_BIST BIT(MSR_INTEGRITY_CAPS_PERIODIC_BIST_BIT) + #define MSR_LBR_NHM_FROM 0x00000680 #define MSR_LBR_NHM_TO 0x000006c0 #define MSR_LBR_CORE_FROM 0x00000040 @@ -189,6 +235,12 @@ #define PERF_CAP_PT_IDX 16 #define MSR_PEBS_LD_LAT_THRESHOLD 0x000003f6 +#define PERF_CAP_PEBS_TRAP BIT_ULL(6) +#define PERF_CAP_ARCH_REG BIT_ULL(7) +#define PERF_CAP_PEBS_FORMAT 0xf00 +#define PERF_CAP_PEBS_BASELINE BIT_ULL(14) +#define PERF_CAP_PEBS_MASK (PERF_CAP_PEBS_TRAP | PERF_CAP_ARCH_REG | \ + PERF_CAP_PEBS_FORMAT | PERF_CAP_PEBS_BASELINE) #define MSR_IA32_RTIT_CTL 0x00000570 #define RTIT_CTL_TRACEEN BIT(0) @@ -312,6 +364,7 @@ /* Run Time Average Power Limiting (RAPL) Interface */ +#define MSR_VR_CURRENT_CONFIG 0x00000601 #define MSR_RAPL_POWER_UNIT 0x00000606 #define MSR_PKG_POWER_LIMIT 0x00000610 @@ -345,6 +398,7 @@ #define MSR_TURBO_ACTIVATION_RATIO 0x0000064C #define MSR_PLATFORM_ENERGY_STATUS 0x0000064D +#define MSR_SECONDARY_TURBO_RATIO_LIMIT 0x00000650 #define MSR_PKG_WEIGHTED_CORE_C0_RES 0x00000658 #define MSR_PKG_ANY_CORE_C0_RES 0x00000659 @@ -502,8 +556,10 @@ #define MSR_AMD64_SEV 0xc0010131 #define MSR_AMD64_SEV_ENABLED_BIT 0 #define MSR_AMD64_SEV_ES_ENABLED_BIT 1 +#define MSR_AMD64_SEV_SNP_ENABLED_BIT 2 #define MSR_AMD64_SEV_ENABLED BIT_ULL(MSR_AMD64_SEV_ENABLED_BIT) #define MSR_AMD64_SEV_ES_ENABLED BIT_ULL(MSR_AMD64_SEV_ES_ENABLED_BIT) +#define MSR_AMD64_SEV_SNP_ENABLED BIT_ULL(MSR_AMD64_SEV_SNP_ENABLED_BIT) #define MSR_AMD64_VIRT_SPEC_CTRL 0xc001011f @@ -524,9 +580,17 @@ #define AMD_CPPC_DES_PERF(x) (((x) & 0xff) << 16) #define AMD_CPPC_ENERGY_PERF_PREF(x) (((x) & 0xff) << 24) +/* AMD Performance Counter Global Status and Control MSRs */ +#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS 0xc0000300 +#define MSR_AMD64_PERF_CNTR_GLOBAL_CTL 0xc0000301 +#define MSR_AMD64_PERF_CNTR_GLOBAL_STATUS_CLR 0xc0000302 + /* Fam 17h MSRs */ #define MSR_F17H_IRPERF 0xc00000e9 +#define MSR_ZEN2_SPECTRAL_CHICKEN 0xc00110e3 +#define MSR_ZEN2_SPECTRAL_CHICKEN_BIT BIT_ULL(1) + /* Fam 16h MSRs */ #define MSR_F16H_L2I_PERF_CTL 0xc0010230 #define MSR_F16H_L2I_PERF_CTR 0xc0010231 @@ -688,6 +752,10 @@ #define MSR_IA32_PERF_CTL 0x00000199 #define INTEL_PERF_CTL_MASK 0xffff +/* AMD Branch Sampling configuration */ +#define MSR_AMD_DBG_EXTN_CFG 0xc000010f +#define MSR_AMD_SAMP_BR_FROM 0xc0010300 + #define MSR_IA32_MPERF 0x000000e7 #define MSR_IA32_APERF 0x000000e8 @@ -961,6 +1029,7 @@ #define MSR_IA32_VMX_TRUE_EXIT_CTLS 0x0000048f #define MSR_IA32_VMX_TRUE_ENTRY_CTLS 0x00000490 #define MSR_IA32_VMX_VMFUNC 0x00000491 +#define MSR_IA32_VMX_PROCBASED_CTLS3 0x00000492 /* VMX_BASIC bits and bitmasks */ #define VMX_BASIC_VMCS_SIZE_SHIFT 32 diff --git a/tools/arch/x86/include/asm/rmwcc.h b/tools/arch/x86/include/asm/rmwcc.h index fee7983a90..11ff975242 100644 --- a/tools/arch/x86/include/asm/rmwcc.h +++ b/tools/arch/x86/include/asm/rmwcc.h @@ -2,8 +2,6 @@ #ifndef _TOOLS_LINUX_ASM_X86_RMWcc #define _TOOLS_LINUX_ASM_X86_RMWcc -#ifdef CONFIG_CC_HAS_ASM_GOTO - #define __GEN_RMWcc(fullop, var, cc, ...) \ do { \ asm_volatile_goto (fullop "; j" cc " %l[cc_label]" \ @@ -20,23 +18,4 @@ cc_label: \ #define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ __GEN_RMWcc(op " %1, " arg0, var, cc, vcon (val)) -#else /* !CONFIG_CC_HAS_ASM_GOTO */ - -#define __GEN_RMWcc(fullop, var, cc, ...) \ -do { \ - char c; \ - asm volatile (fullop "; set" cc " %1" \ - : "+m" (var), "=qm" (c) \ - : __VA_ARGS__ : "memory"); \ - return c != 0; \ -} while (0) - -#define GEN_UNARY_RMWcc(op, var, arg0, cc) \ - __GEN_RMWcc(op " " arg0, var, cc) - -#define GEN_BINARY_RMWcc(op, var, vcon, val, arg0, cc) \ - __GEN_RMWcc(op " %2, " arg0, var, cc, vcon (val)) - -#endif /* CONFIG_CC_HAS_ASM_GOTO */ - #endif /* _TOOLS_LINUX_ASM_X86_RMWcc */ diff --git a/tools/arch/x86/include/uapi/asm/kvm.h b/tools/arch/x86/include/uapi/asm/kvm.h index bf6e96011d..46de10a809 100644 --- a/tools/arch/x86/include/uapi/asm/kvm.h +++ b/tools/arch/x86/include/uapi/asm/kvm.h @@ -198,13 +198,13 @@ struct kvm_msrs { __u32 nmsrs; /* number of msrs in entries */ __u32 pad; - struct kvm_msr_entry entries[0]; + struct kvm_msr_entry entries[]; }; /* for KVM_GET_MSR_INDEX_LIST */ struct kvm_msr_list { __u32 nmsrs; /* number of msrs in entries */ - __u32 indices[0]; + __u32 indices[]; }; /* Maximum size of any access bitmap in bytes */ @@ -241,7 +241,7 @@ struct kvm_cpuid_entry { struct kvm_cpuid { __u32 nent; __u32 padding; - struct kvm_cpuid_entry entries[0]; + struct kvm_cpuid_entry entries[]; }; struct kvm_cpuid_entry2 { @@ -263,7 +263,7 @@ struct kvm_cpuid_entry2 { struct kvm_cpuid2 { __u32 nent; __u32 padding; - struct kvm_cpuid_entry2 entries[0]; + struct kvm_cpuid_entry2 entries[]; }; /* for KVM_GET_PIT and KVM_SET_PIT */ @@ -306,7 +306,8 @@ struct kvm_pit_state { struct kvm_pit_channel_state channels[3]; }; -#define KVM_PIT_FLAGS_HPET_LEGACY 0x00000001 +#define KVM_PIT_FLAGS_HPET_LEGACY 0x00000001 +#define KVM_PIT_FLAGS_SPEAKER_DATA_ON 0x00000002 struct kvm_pit_state2 { struct kvm_pit_channel_state channels[3]; @@ -325,6 +326,7 @@ struct kvm_reinject_control { #define KVM_VCPUEVENT_VALID_SHADOW 0x00000004 #define KVM_VCPUEVENT_VALID_SMM 0x00000008 #define KVM_VCPUEVENT_VALID_PAYLOAD 0x00000010 +#define KVM_VCPUEVENT_VALID_TRIPLE_FAULT 0x00000020 /* Interrupt shadow states */ #define KVM_X86_SHADOW_INT_MOV_SS 0x01 @@ -359,7 +361,10 @@ struct kvm_vcpu_events { __u8 smm_inside_nmi; __u8 latched_init; } smi; - __u8 reserved[27]; + struct { + __u8 pending; + } triple_fault; + __u8 reserved[26]; __u8 exception_has_payload; __u64 exception_payload; }; @@ -389,7 +394,7 @@ struct kvm_xsave { * the contents of CPUID leaf 0xD on the host. */ __u32 region[1024]; - __u32 extra[0]; + __u32 extra[]; }; #define KVM_MAX_XCRS 16 @@ -428,11 +433,13 @@ struct kvm_sync_regs { struct kvm_vcpu_events events; }; -#define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0) -#define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1) -#define KVM_X86_QUIRK_LAPIC_MMIO_HOLE (1 << 2) -#define KVM_X86_QUIRK_OUT_7E_INC_RIP (1 << 3) -#define KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT (1 << 4) +#define KVM_X86_QUIRK_LINT0_REENABLED (1 << 0) +#define KVM_X86_QUIRK_CD_NW_CLEARED (1 << 1) +#define KVM_X86_QUIRK_LAPIC_MMIO_HOLE (1 << 2) +#define KVM_X86_QUIRK_OUT_7E_INC_RIP (1 << 3) +#define KVM_X86_QUIRK_MISC_ENABLE_NO_MWAIT (1 << 4) +#define KVM_X86_QUIRK_FIX_HYPERCALL_INSN (1 << 5) +#define KVM_X86_QUIRK_MWAIT_NEVER_UD_FAULTS (1 << 6) #define KVM_STATE_NESTED_FORMAT_VMX 0 #define KVM_STATE_NESTED_FORMAT_SVM 1 @@ -515,7 +522,7 @@ struct kvm_pmu_event_filter { __u32 fixed_counter_bitmap; __u32 flags; __u32 pad[4]; - __u64 events[0]; + __u64 events[]; }; #define KVM_PMU_EVENT_ALLOW 0 diff --git a/tools/arch/x86/include/uapi/asm/svm.h b/tools/arch/x86/include/uapi/asm/svm.h index efa969325e..f69c168391 100644 --- a/tools/arch/x86/include/uapi/asm/svm.h +++ b/tools/arch/x86/include/uapi/asm/svm.h @@ -108,6 +108,14 @@ #define SVM_VMGEXIT_AP_JUMP_TABLE 0x80000005 #define SVM_VMGEXIT_SET_AP_JUMP_TABLE 0 #define SVM_VMGEXIT_GET_AP_JUMP_TABLE 1 +#define SVM_VMGEXIT_PSC 0x80000010 +#define SVM_VMGEXIT_GUEST_REQUEST 0x80000011 +#define SVM_VMGEXIT_EXT_GUEST_REQUEST 0x80000012 +#define SVM_VMGEXIT_AP_CREATION 0x80000013 +#define SVM_VMGEXIT_AP_CREATE_ON_INIT 0 +#define SVM_VMGEXIT_AP_CREATE 1 +#define SVM_VMGEXIT_AP_DESTROY 2 +#define SVM_VMGEXIT_HV_FEATURES 0x8000fffd #define SVM_VMGEXIT_UNSUPPORTED_EVENT 0x8000ffff /* Exit code reserved for hypervisor/software use */ @@ -218,6 +226,11 @@ { SVM_VMGEXIT_NMI_COMPLETE, "vmgexit_nmi_complete" }, \ { SVM_VMGEXIT_AP_HLT_LOOP, "vmgexit_ap_hlt_loop" }, \ { SVM_VMGEXIT_AP_JUMP_TABLE, "vmgexit_ap_jump_table" }, \ + { SVM_VMGEXIT_PSC, "vmgexit_page_state_change" }, \ + { SVM_VMGEXIT_GUEST_REQUEST, "vmgexit_guest_request" }, \ + { SVM_VMGEXIT_EXT_GUEST_REQUEST, "vmgexit_ext_guest_request" }, \ + { SVM_VMGEXIT_AP_CREATION, "vmgexit_ap_creation" }, \ + { SVM_VMGEXIT_HV_FEATURES, "vmgexit_hypervisor_feature" }, \ { SVM_EXIT_ERR, "invalid_guest_state" } diff --git a/tools/arch/x86/include/uapi/asm/vmx.h b/tools/arch/x86/include/uapi/asm/vmx.h index 946d761adb..a5faf6d88f 100644 --- a/tools/arch/x86/include/uapi/asm/vmx.h +++ b/tools/arch/x86/include/uapi/asm/vmx.h @@ -91,6 +91,7 @@ #define EXIT_REASON_UMWAIT 67 #define EXIT_REASON_TPAUSE 68 #define EXIT_REASON_BUS_LOCK 74 +#define EXIT_REASON_NOTIFY 75 #define VMX_EXIT_REASONS \ { EXIT_REASON_EXCEPTION_NMI, "EXCEPTION_NMI" }, \ @@ -153,7 +154,8 @@ { EXIT_REASON_XRSTORS, "XRSTORS" }, \ { EXIT_REASON_UMWAIT, "UMWAIT" }, \ { EXIT_REASON_TPAUSE, "TPAUSE" }, \ - { EXIT_REASON_BUS_LOCK, "BUS_LOCK" } + { EXIT_REASON_BUS_LOCK, "BUS_LOCK" }, \ + { EXIT_REASON_NOTIFY, "NOTIFY" } #define VMX_EXIT_REASON_FLAGS \ { VMX_EXIT_REASONS_FAILED_VMENTRY, "FAILED_VMENTRY" } diff --git a/tools/bpf/Makefile b/tools/bpf/Makefile index b11cfc86a3..243b79f2b4 100644 --- a/tools/bpf/Makefile +++ b/tools/bpf/Makefile @@ -34,8 +34,8 @@ else endif FEATURE_USER = .bpf -FEATURE_TESTS = libbfd disassembler-four-args -FEATURE_DISPLAY = libbfd disassembler-four-args +FEATURE_TESTS = libbfd disassembler-four-args disassembler-init-styled +FEATURE_DISPLAY = libbfd check_feat := 1 NON_CHECK_FEAT_TARGETS := clean bpftool_clean runqslower_clean resolve_btfids_clean @@ -56,6 +56,9 @@ endif ifeq ($(feature-disassembler-four-args), 1) CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE endif +ifeq ($(feature-disassembler-init-styled), 1) +CFLAGS += -DDISASM_INIT_STYLED +endif $(OUTPUT)%.yacc.c: $(srctree)/tools/bpf/%.y $(QUIET_BISON)$(YACC) -o $@ -d $< diff --git a/tools/bpf/bpf_jit_disasm.c b/tools/bpf/bpf_jit_disasm.c index c8ae958047..a90a5d110f 100644 --- a/tools/bpf/bpf_jit_disasm.c +++ b/tools/bpf/bpf_jit_disasm.c @@ -28,6 +28,7 @@ #include #include #include +#include #define CMD_ACTION_SIZE_BUFFER 10 #define CMD_ACTION_READ_ALL 3 @@ -64,7 +65,9 @@ static void get_asm_insns(uint8_t *image, size_t len, int opcodes) assert(bfdf); assert(bfd_check_format(bfdf, bfd_object)); - init_disassemble_info(&info, stdout, (fprintf_ftype) fprintf); + init_disassemble_info_compat(&info, stdout, + (fprintf_ftype) fprintf, + fprintf_styled); info.arch = bfd_get_arch(bfdf); info.mach = bfd_get_mach(bfdf); info.buffer = image; diff --git a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst index a17e9aa314..bd015ec984 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-cgroup.rst @@ -31,11 +31,17 @@ CGROUP COMMANDS | **bpftool** **cgroup help** | | *PROG* := { **id** *PROG_ID* | **pinned** *FILE* | **tag** *PROG_TAG* } -| *ATTACH_TYPE* := { **ingress** | **egress** | **sock_create** | **sock_ops** | **device** | -| **bind4** | **bind6** | **post_bind4** | **post_bind6** | **connect4** | **connect6** | -| **getpeername4** | **getpeername6** | **getsockname4** | **getsockname6** | **sendmsg4** | -| **sendmsg6** | **recvmsg4** | **recvmsg6** | **sysctl** | **getsockopt** | **setsockopt** | -| **sock_release** } +| *ATTACH_TYPE* := { **cgroup_inet_ingress** | **cgroup_inet_egress** | +| **cgroup_inet_sock_create** | **cgroup_sock_ops** | +| **cgroup_device** | **cgroup_inet4_bind** | **cgroup_inet6_bind** | +| **cgroup_inet4_post_bind** | **cgroup_inet6_post_bind** | +| **cgroup_inet4_connect** | **cgroup_inet6_connect** | +| **cgroup_inet4_getpeername** | **cgroup_inet6_getpeername** | +| **cgroup_inet4_getsockname** | **cgroup_inet6_getsockname** | +| **cgroup_udp4_sendmsg** | **cgroup_udp6_sendmsg** | +| **cgroup_udp4_recvmsg** | **cgroup_udp6_recvmsg** | +| **cgroup_sysctl** | **cgroup_getsockopt** | **cgroup_setsockopt** | +| **cgroup_inet_sock_release** } | *ATTACH_FLAGS* := { **multi** | **override** } DESCRIPTION diff --git a/tools/bpf/bpftool/Documentation/bpftool-feature.rst b/tools/bpf/bpftool/Documentation/bpftool-feature.rst index 4ce9a77bc1..e44039f89b 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-feature.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-feature.rst @@ -24,9 +24,11 @@ FEATURE COMMANDS ================ | **bpftool** **feature probe** [*COMPONENT*] [**full**] [**unprivileged**] [**macros** [**prefix** *PREFIX*]] +| **bpftool** **feature list_builtins** *GROUP* | **bpftool** **feature help** | | *COMPONENT* := { **kernel** | **dev** *NAME* } +| *GROUP* := { **prog_types** | **map_types** | **attach_types** | **link_types** | **helpers** } DESCRIPTION =========== @@ -70,6 +72,16 @@ DESCRIPTION The keywords **full**, **macros** and **prefix** have the same role as when probing the kernel. + **bpftool feature list_builtins** *GROUP* + List items known to bpftool. These can be BPF program types + (**prog_types**), BPF map types (**map_types**), attach types + (**attach_types**), link types (**link_types**), or BPF helper + functions (**helpers**). The command does not probe the system, but + simply lists the elements that bpftool knows from compilation time, + as provided from libbpf (for all object types) or from the BPF UAPI + header (list of helpers). This can be used in scripts to iterate over + BPF types or helpers. + **bpftool feature help** Print short help message. diff --git a/tools/bpf/bpftool/Documentation/bpftool-prog.rst b/tools/bpf/bpftool/Documentation/bpftool-prog.rst index a2e9359e55..eb1b2a254e 100644 --- a/tools/bpf/bpftool/Documentation/bpftool-prog.rst +++ b/tools/bpf/bpftool/Documentation/bpftool-prog.rst @@ -53,8 +53,9 @@ PROG COMMANDS | **cgroup/getsockopt** | **cgroup/setsockopt** | **cgroup/sock_release** | | **struct_ops** | **fentry** | **fexit** | **freplace** | **sk_lookup** | } -| *ATTACH_TYPE* := { -| **msg_verdict** | **skb_verdict** | **stream_verdict** | **stream_parser** | **flow_dissector** +| *ATTACH_TYPE* := { +| **sk_msg_verdict** | **sk_skb_verdict** | **sk_skb_stream_verdict** | +| **sk_skb_stream_parser** | **flow_dissector** | } | *METRICs* := { | **cycles** | **instructions** | **l1d_loads** | **llc_misses** | diff --git a/tools/bpf/bpftool/Makefile b/tools/bpf/bpftool/Makefile index c6d2c77d02..4a95c017ad 100644 --- a/tools/bpf/bpftool/Makefile +++ b/tools/bpf/bpftool/Makefile @@ -53,7 +53,7 @@ $(LIBBPF_INTERNAL_HDRS): $(LIBBPF_HDRS_DIR)/%.h: $(BPF_DIR)/%.h | $(LIBBPF_HDRS_ $(LIBBPF_BOOTSTRAP): $(wildcard $(BPF_DIR)/*.[ch] $(BPF_DIR)/Makefile) | $(LIBBPF_BOOTSTRAP_OUTPUT) $(Q)$(MAKE) -C $(BPF_DIR) OUTPUT=$(LIBBPF_BOOTSTRAP_OUTPUT) \ DESTDIR=$(LIBBPF_BOOTSTRAP_DESTDIR:/=) prefix= \ - ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) $@ install_headers + ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) AR=$(HOSTAR) $@ install_headers $(LIBBPF_BOOTSTRAP_INTERNAL_HDRS): $(LIBBPF_BOOTSTRAP_HDRS_DIR)/%.h: $(BPF_DIR)/%.h | $(LIBBPF_BOOTSTRAP_HDRS_DIR) $(call QUIET_INSTALL, $@) @@ -93,10 +93,11 @@ INSTALL ?= install RM ?= rm -f FEATURE_USER = .bpftool -FEATURE_TESTS = libbfd disassembler-four-args zlib libcap \ - clang-bpf-co-re -FEATURE_DISPLAY = libbfd disassembler-four-args zlib libcap \ +FEATURE_TESTS = libbfd libbfd-liberty libbfd-liberty-z \ + disassembler-four-args disassembler-init-styled libcap \ clang-bpf-co-re +FEATURE_DISPLAY = libbfd libbfd-liberty libbfd-liberty-z \ + libcap clang-bpf-co-re check_feat := 1 NON_CHECK_FEAT_TARGETS := clean uninstall doc doc-clean doc-install doc-uninstall @@ -117,6 +118,9 @@ endif ifeq ($(feature-disassembler-four-args), 1) CFLAGS += -DDISASM_FOUR_ARGS_SIGNATURE endif +ifeq ($(feature-disassembler-init-styled), 1) + CFLAGS += -DDISASM_INIT_STYLED +endif LIBS = $(LIBBPF) -lelf -lz LIBS_BOOTSTRAP = $(LIBBPF_BOOTSTRAP) -lelf -lz @@ -204,11 +208,6 @@ $(BOOTSTRAP_OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c $(OUTPUT)disasm.o: $(srctree)/kernel/bpf/disasm.c $(QUIET_CC)$(CC) $(CFLAGS) -c -MMD $< -o $@ -$(OUTPUT)feature.o: -ifneq ($(feature-zlib), 1) - $(error "No zlib found") -endif - $(BPFTOOL_BOOTSTRAP): $(BOOTSTRAP_OBJS) $(LIBBPF_BOOTSTRAP) $(QUIET_LINK)$(HOSTCC) $(HOST_CFLAGS) $(LDFLAGS) $(BOOTSTRAP_OBJS) $(LIBS_BOOTSTRAP) -o $@ diff --git a/tools/bpf/bpftool/bash-completion/bpftool b/tools/bpf/bpftool/bash-completion/bpftool index 5df8d72c51..dc1641e367 100644 --- a/tools/bpf/bpftool/bash-completion/bpftool +++ b/tools/bpf/bpftool/bash-completion/bpftool @@ -407,8 +407,8 @@ _bpftool() return 0 ;; 5) - local BPFTOOL_PROG_ATTACH_TYPES='msg_verdict \ - skb_verdict stream_verdict stream_parser \ + local BPFTOOL_PROG_ATTACH_TYPES='sk_msg_verdict \ + sk_skb_verdict sk_skb_stream_verdict sk_skb_stream_parser \ flow_dissector' COMPREPLY=( $( compgen -W "$BPFTOOL_PROG_ATTACH_TYPES" -- "$cur" ) ) return 0 @@ -703,15 +703,8 @@ _bpftool() return 0 ;; type) - local BPFTOOL_MAP_CREATE_TYPES='hash array \ - prog_array perf_event_array percpu_hash \ - percpu_array stack_trace cgroup_array lru_hash \ - lru_percpu_hash lpm_trie array_of_maps \ - hash_of_maps devmap devmap_hash sockmap cpumap \ - xskmap sockhash cgroup_storage reuseport_sockarray \ - percpu_cgroup_storage queue stack sk_storage \ - struct_ops ringbuf inode_storage task_storage \ - bloom_filter' + local BPFTOOL_MAP_CREATE_TYPES="$(bpftool feature list_builtins map_types 2>/dev/null | \ + grep -v '^unspec$')" COMPREPLY=( $( compgen -W "$BPFTOOL_MAP_CREATE_TYPES" -- "$cur" ) ) return 0 ;; @@ -1039,12 +1032,8 @@ _bpftool() return 0 ;; attach|detach) - local BPFTOOL_CGROUP_ATTACH_TYPES='ingress egress \ - sock_create sock_ops device \ - bind4 bind6 post_bind4 post_bind6 connect4 connect6 \ - getpeername4 getpeername6 getsockname4 getsockname6 \ - sendmsg4 sendmsg6 recvmsg4 recvmsg6 sysctl getsockopt \ - setsockopt sock_release' + local BPFTOOL_CGROUP_ATTACH_TYPES="$(bpftool feature list_builtins attach_types 2>/dev/null | \ + grep '^cgroup_')" local ATTACH_FLAGS='multi override' local PROG_TYPE='id pinned tag name' # Check for $prev = $command first @@ -1173,9 +1162,14 @@ _bpftool() _bpftool_once_attr 'full unprivileged' return 0 ;; + list_builtins) + [[ $prev != "$command" ]] && return 0 + COMPREPLY=( $( compgen -W 'prog_types map_types \ + attach_types link_types helpers' -- "$cur" ) ) + ;; *) [[ $prev == $object ]] && \ - COMPREPLY=( $( compgen -W 'help probe' -- "$cur" ) ) + COMPREPLY=( $( compgen -W 'help list_builtins probe' -- "$cur" ) ) ;; esac ;; diff --git a/tools/bpf/bpftool/btf.c b/tools/bpf/bpftool/btf.c index a2c665beda..0744bd1150 100644 --- a/tools/bpf/bpftool/btf.c +++ b/tools/bpf/bpftool/btf.c @@ -40,6 +40,7 @@ static const char * const btf_kind_str[NR_BTF_KINDS] = { [BTF_KIND_FLOAT] = "FLOAT", [BTF_KIND_DECL_TAG] = "DECL_TAG", [BTF_KIND_TYPE_TAG] = "TYPE_TAG", + [BTF_KIND_ENUM64] = "ENUM64", }; struct btf_attach_point { @@ -212,15 +213,18 @@ static int dump_btf_type(const struct btf *btf, __u32 id, case BTF_KIND_ENUM: { const struct btf_enum *v = (const void *)(t + 1); __u16 vlen = BTF_INFO_VLEN(t->info); + const char *encoding; int i; + encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED"; if (json_output) { + jsonw_string_field(w, "encoding", encoding); jsonw_uint_field(w, "size", t->size); jsonw_uint_field(w, "vlen", vlen); jsonw_name(w, "values"); jsonw_start_array(w); } else { - printf(" size=%u vlen=%u", t->size, vlen); + printf(" encoding=%s size=%u vlen=%u", encoding, t->size, vlen); } for (i = 0; i < vlen; i++, v++) { const char *name = btf_str(btf, v->name_off); @@ -228,10 +232,57 @@ static int dump_btf_type(const struct btf *btf, __u32 id, if (json_output) { jsonw_start_object(w); jsonw_string_field(w, "name", name); - jsonw_uint_field(w, "val", v->val); + if (btf_kflag(t)) + jsonw_int_field(w, "val", v->val); + else + jsonw_uint_field(w, "val", v->val); jsonw_end_object(w); } else { - printf("\n\t'%s' val=%u", name, v->val); + if (btf_kflag(t)) + printf("\n\t'%s' val=%d", name, v->val); + else + printf("\n\t'%s' val=%u", name, v->val); + } + } + if (json_output) + jsonw_end_array(w); + break; + } + case BTF_KIND_ENUM64: { + const struct btf_enum64 *v = btf_enum64(t); + __u16 vlen = btf_vlen(t); + const char *encoding; + int i; + + encoding = btf_kflag(t) ? "SIGNED" : "UNSIGNED"; + if (json_output) { + jsonw_string_field(w, "encoding", encoding); + jsonw_uint_field(w, "size", t->size); + jsonw_uint_field(w, "vlen", vlen); + jsonw_name(w, "values"); + jsonw_start_array(w); + } else { + printf(" encoding=%s size=%u vlen=%u", encoding, t->size, vlen); + } + for (i = 0; i < vlen; i++, v++) { + const char *name = btf_str(btf, v->name_off); + __u64 val = ((__u64)v->val_hi32 << 32) | v->val_lo32; + + if (json_output) { + jsonw_start_object(w); + jsonw_string_field(w, "name", name); + if (btf_kflag(t)) + jsonw_int_field(w, "val", val); + else + jsonw_uint_field(w, "val", val); + jsonw_end_object(w); + } else { + if (btf_kflag(t)) + printf("\n\t'%s' val=%lldLL", name, + (unsigned long long)val); + else + printf("\n\t'%s' val=%lluULL", name, + (unsigned long long)val); } } if (json_output) @@ -459,6 +510,51 @@ static int dump_btf_c(const struct btf *btf, return err; } +static const char sysfs_vmlinux[] = "/sys/kernel/btf/vmlinux"; + +static struct btf *get_vmlinux_btf_from_sysfs(void) +{ + struct btf *base; + + base = btf__parse(sysfs_vmlinux, NULL); + if (libbpf_get_error(base)) { + p_err("failed to parse vmlinux BTF at '%s': %ld\n", + sysfs_vmlinux, libbpf_get_error(base)); + base = NULL; + } + + return base; +} + +#define BTF_NAME_BUFF_LEN 64 + +static bool btf_is_kernel_module(__u32 btf_id) +{ + struct bpf_btf_info btf_info = {}; + char btf_name[BTF_NAME_BUFF_LEN]; + int btf_fd; + __u32 len; + int err; + + btf_fd = bpf_btf_get_fd_by_id(btf_id); + if (btf_fd < 0) { + p_err("can't get BTF object by id (%u): %s", btf_id, strerror(errno)); + return false; + } + + len = sizeof(btf_info); + btf_info.name = ptr_to_u64(btf_name); + btf_info.name_len = sizeof(btf_name); + err = bpf_obj_get_info_by_fd(btf_fd, &btf_info, &len); + close(btf_fd); + if (err) { + p_err("can't get BTF (ID %u) object info: %s", btf_id, strerror(errno)); + return false; + } + + return btf_info.kernel_btf && strncmp(btf_name, "vmlinux", sizeof(btf_name)) != 0; +} + static int do_dump(int argc, char **argv) { struct btf *btf = NULL, *base = NULL; @@ -536,18 +632,11 @@ static int do_dump(int argc, char **argv) NEXT_ARG(); } else if (is_prefix(src, "file")) { const char sysfs_prefix[] = "/sys/kernel/btf/"; - const char sysfs_vmlinux[] = "/sys/kernel/btf/vmlinux"; if (!base_btf && strncmp(*argv, sysfs_prefix, sizeof(sysfs_prefix) - 1) == 0 && - strcmp(*argv, sysfs_vmlinux) != 0) { - base = btf__parse(sysfs_vmlinux, NULL); - if (libbpf_get_error(base)) { - p_err("failed to parse vmlinux BTF at '%s': %ld\n", - sysfs_vmlinux, libbpf_get_error(base)); - base = NULL; - } - } + strcmp(*argv, sysfs_vmlinux) != 0) + base = get_vmlinux_btf_from_sysfs(); btf = btf__parse_split(*argv, base ?: base_btf); err = libbpf_get_error(btf); @@ -591,6 +680,12 @@ static int do_dump(int argc, char **argv) } if (!btf) { + if (!base_btf && btf_is_kernel_module(btf_id)) { + p_info("Warning: valid base BTF was not specified with -B option, falling back to standard base BTF (%s)", + sysfs_vmlinux); + base_btf = get_vmlinux_btf_from_sysfs(); + } + btf = btf__load_from_kernel_by_id_split(btf_id, base_btf); err = libbpf_get_error(btf); if (err) { diff --git a/tools/bpf/bpftool/btf_dumper.c b/tools/bpf/bpftool/btf_dumper.c index f5dddf8ef4..125798b0bc 100644 --- a/tools/bpf/bpftool/btf_dumper.c +++ b/tools/bpf/bpftool/btf_dumper.c @@ -182,6 +182,32 @@ static int btf_dumper_enum(const struct btf_dumper *d, return 0; } +static int btf_dumper_enum64(const struct btf_dumper *d, + const struct btf_type *t, + const void *data) +{ + const struct btf_enum64 *enums = btf_enum64(t); + __u32 val_lo32, val_hi32; + __u64 value; + __u16 i; + + value = *(__u64 *)data; + val_lo32 = (__u32)value; + val_hi32 = value >> 32; + + for (i = 0; i < btf_vlen(t); i++) { + if (val_lo32 == enums[i].val_lo32 && val_hi32 == enums[i].val_hi32) { + jsonw_string(d->jw, + btf__name_by_offset(d->btf, + enums[i].name_off)); + return 0; + } + } + + jsonw_int(d->jw, value); + return 0; +} + static bool is_str_array(const struct btf *btf, const struct btf_array *arr, const char *s) { @@ -542,6 +568,8 @@ static int btf_dumper_do_type(const struct btf_dumper *d, __u32 type_id, return btf_dumper_array(d, type_id, data); case BTF_KIND_ENUM: return btf_dumper_enum(d, t, data); + case BTF_KIND_ENUM64: + return btf_dumper_enum64(d, t, data); case BTF_KIND_PTR: btf_dumper_ptr(d, t, data); return 0; @@ -618,6 +646,7 @@ static int __btf_dumper_type_only(const struct btf *btf, __u32 type_id, btf__name_by_offset(btf, t->name_off)); break; case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: BTF_PRINT_ARG("enum %s ", btf__name_by_offset(btf, t->name_off)); break; diff --git a/tools/bpf/bpftool/cgroup.c b/tools/bpf/bpftool/cgroup.c index effe136119..cced668fb2 100644 --- a/tools/bpf/bpftool/cgroup.c +++ b/tools/bpf/bpftool/cgroup.c @@ -15,43 +15,92 @@ #include #include +#include #include "main.h" #define HELP_SPEC_ATTACH_FLAGS \ "ATTACH_FLAGS := { multi | override }" -#define HELP_SPEC_ATTACH_TYPES \ - " ATTACH_TYPE := { ingress | egress | sock_create |\n" \ - " sock_ops | device | bind4 | bind6 |\n" \ - " post_bind4 | post_bind6 | connect4 |\n" \ - " connect6 | getpeername4 | getpeername6 |\n" \ - " getsockname4 | getsockname6 | sendmsg4 |\n" \ - " sendmsg6 | recvmsg4 | recvmsg6 |\n" \ - " sysctl | getsockopt | setsockopt |\n" \ - " sock_release }" +#define HELP_SPEC_ATTACH_TYPES \ + " ATTACH_TYPE := { cgroup_inet_ingress | cgroup_inet_egress |\n" \ + " cgroup_inet_sock_create | cgroup_sock_ops |\n" \ + " cgroup_device | cgroup_inet4_bind |\n" \ + " cgroup_inet6_bind | cgroup_inet4_post_bind |\n" \ + " cgroup_inet6_post_bind | cgroup_inet4_connect |\n" \ + " cgroup_inet6_connect | cgroup_inet4_getpeername |\n" \ + " cgroup_inet6_getpeername | cgroup_inet4_getsockname |\n" \ + " cgroup_inet6_getsockname | cgroup_udp4_sendmsg |\n" \ + " cgroup_udp6_sendmsg | cgroup_udp4_recvmsg |\n" \ + " cgroup_udp6_recvmsg | cgroup_sysctl |\n" \ + " cgroup_getsockopt | cgroup_setsockopt |\n" \ + " cgroup_inet_sock_release }" static unsigned int query_flags; +static struct btf *btf_vmlinux; +static __u32 btf_vmlinux_id; static enum bpf_attach_type parse_attach_type(const char *str) { + const char *attach_type_str; enum bpf_attach_type type; - for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) { - if (attach_type_name[type] && - is_prefix(str, attach_type_name[type])) + for (type = 0; ; type++) { + attach_type_str = libbpf_bpf_attach_type_str(type); + if (!attach_type_str) + break; + if (!strcmp(str, attach_type_str)) + return type; + } + + /* Also check traditionally used attach type strings. For these we keep + * allowing prefixed usage. + */ + for (type = 0; ; type++) { + attach_type_str = bpf_attach_type_input_str(type); + if (!attach_type_str) + break; + if (is_prefix(str, attach_type_str)) return type; } return __MAX_BPF_ATTACH_TYPE; } +static void guess_vmlinux_btf_id(__u32 attach_btf_obj_id) +{ + struct bpf_btf_info btf_info = {}; + __u32 btf_len = sizeof(btf_info); + char name[16] = {}; + int err; + int fd; + + btf_info.name = ptr_to_u64(name); + btf_info.name_len = sizeof(name); + + fd = bpf_btf_get_fd_by_id(attach_btf_obj_id); + if (fd < 0) + return; + + err = bpf_obj_get_info_by_fd(fd, &btf_info, &btf_len); + if (err) + goto out; + + if (btf_info.kernel_btf && strncmp(name, "vmlinux", sizeof(name)) == 0) + btf_vmlinux_id = btf_info.id; + +out: + close(fd); +} + static int show_bpf_prog(int id, enum bpf_attach_type attach_type, const char *attach_flags_str, int level) { char prog_name[MAX_PROG_FULL_NAME]; + const char *attach_btf_name = NULL; struct bpf_prog_info info = {}; + const char *attach_type_str; __u32 info_len = sizeof(info); int prog_fd; @@ -64,26 +113,50 @@ static int show_bpf_prog(int id, enum bpf_attach_type attach_type, return -1; } + attach_type_str = libbpf_bpf_attach_type_str(attach_type); + + if (btf_vmlinux) { + if (!btf_vmlinux_id) + guess_vmlinux_btf_id(info.attach_btf_obj_id); + + if (btf_vmlinux_id == info.attach_btf_obj_id && + info.attach_btf_id < btf__type_cnt(btf_vmlinux)) { + const struct btf_type *t = + btf__type_by_id(btf_vmlinux, info.attach_btf_id); + attach_btf_name = + btf__name_by_offset(btf_vmlinux, t->name_off); + } + } + get_prog_full_name(&info, prog_fd, prog_name, sizeof(prog_name)); if (json_output) { jsonw_start_object(json_wtr); jsonw_uint_field(json_wtr, "id", info.id); - if (attach_type < ARRAY_SIZE(attach_type_name)) - jsonw_string_field(json_wtr, "attach_type", - attach_type_name[attach_type]); + if (attach_type_str) + jsonw_string_field(json_wtr, "attach_type", attach_type_str); else jsonw_uint_field(json_wtr, "attach_type", attach_type); jsonw_string_field(json_wtr, "attach_flags", attach_flags_str); jsonw_string_field(json_wtr, "name", prog_name); + if (attach_btf_name) + jsonw_string_field(json_wtr, "attach_btf_name", attach_btf_name); + jsonw_uint_field(json_wtr, "attach_btf_obj_id", info.attach_btf_obj_id); + jsonw_uint_field(json_wtr, "attach_btf_id", info.attach_btf_id); jsonw_end_object(json_wtr); } else { printf("%s%-8u ", level ? " " : "", info.id); - if (attach_type < ARRAY_SIZE(attach_type_name)) - printf("%-15s", attach_type_name[attach_type]); + if (attach_type_str) + printf("%-15s", attach_type_str); else printf("type %-10u", attach_type); - printf(" %-15s %-15s\n", attach_flags_str, prog_name); + printf(" %-15s %-15s", attach_flags_str, prog_name); + if (attach_btf_name) + printf(" %-15s", attach_btf_name); + else if (info.attach_btf_id) + printf(" attach_btf_obj_id=%d attach_btf_id=%d", + info.attach_btf_obj_id, info.attach_btf_id); + printf("\n"); } close(prog_fd); @@ -125,40 +198,49 @@ static int cgroup_has_attached_progs(int cgroup_fd) static int show_attached_bpf_progs(int cgroup_fd, enum bpf_attach_type type, int level) { + LIBBPF_OPTS(bpf_prog_query_opts, p); + __u32 prog_attach_flags[1024] = {0}; const char *attach_flags_str; __u32 prog_ids[1024] = {0}; - __u32 prog_cnt, iter; - __u32 attach_flags; char buf[32]; + __u32 iter; int ret; - prog_cnt = ARRAY_SIZE(prog_ids); - ret = bpf_prog_query(cgroup_fd, type, query_flags, &attach_flags, - prog_ids, &prog_cnt); + p.query_flags = query_flags; + p.prog_cnt = ARRAY_SIZE(prog_ids); + p.prog_ids = prog_ids; + p.prog_attach_flags = prog_attach_flags; + + ret = bpf_prog_query_opts(cgroup_fd, type, &p); if (ret) return ret; - if (prog_cnt == 0) + if (p.prog_cnt == 0) return 0; - switch (attach_flags) { - case BPF_F_ALLOW_MULTI: - attach_flags_str = "multi"; - break; - case BPF_F_ALLOW_OVERRIDE: - attach_flags_str = "override"; - break; - case 0: - attach_flags_str = ""; - break; - default: - snprintf(buf, sizeof(buf), "unknown(%x)", attach_flags); - attach_flags_str = buf; - } + for (iter = 0; iter < p.prog_cnt; iter++) { + __u32 attach_flags; + + attach_flags = prog_attach_flags[iter] ?: p.attach_flags; + + switch (attach_flags) { + case BPF_F_ALLOW_MULTI: + attach_flags_str = "multi"; + break; + case BPF_F_ALLOW_OVERRIDE: + attach_flags_str = "override"; + break; + case 0: + attach_flags_str = ""; + break; + default: + snprintf(buf, sizeof(buf), "unknown(%x)", attach_flags); + attach_flags_str = buf; + } - for (iter = 0; iter < prog_cnt; iter++) show_bpf_prog(prog_ids[iter], type, attach_flags_str, level); + } return 0; } @@ -214,6 +296,7 @@ static int do_show(int argc, char **argv) printf("%-8s %-15s %-15s %-15s\n", "ID", "AttachType", "AttachFlags", "Name"); + btf_vmlinux = libbpf_find_kernel_btf(); for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) { /* * Not all attach types may be supported, so it's expected, @@ -277,6 +360,7 @@ static int do_show_tree_fn(const char *fpath, const struct stat *sb, printf("%s\n", fpath); } + btf_vmlinux = libbpf_find_kernel_btf(); for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) show_attached_bpf_progs(cgroup_fd, type, ftw->level); diff --git a/tools/bpf/bpftool/common.c b/tools/bpf/bpftool/common.c index 0c1e06cf50..067e9ea59e 100644 --- a/tools/bpf/bpftool/common.c +++ b/tools/bpf/bpftool/common.c @@ -13,14 +13,17 @@ #include #include #include -#include -#include #include #include #include #include #include +#include +#include +#include +#include + #include #include #include /* libbpf_num_possible_cpus */ @@ -32,52 +35,6 @@ #define BPF_FS_MAGIC 0xcafe4a11 #endif -const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE] = { - [BPF_CGROUP_INET_INGRESS] = "ingress", - [BPF_CGROUP_INET_EGRESS] = "egress", - [BPF_CGROUP_INET_SOCK_CREATE] = "sock_create", - [BPF_CGROUP_INET_SOCK_RELEASE] = "sock_release", - [BPF_CGROUP_SOCK_OPS] = "sock_ops", - [BPF_CGROUP_DEVICE] = "device", - [BPF_CGROUP_INET4_BIND] = "bind4", - [BPF_CGROUP_INET6_BIND] = "bind6", - [BPF_CGROUP_INET4_CONNECT] = "connect4", - [BPF_CGROUP_INET6_CONNECT] = "connect6", - [BPF_CGROUP_INET4_POST_BIND] = "post_bind4", - [BPF_CGROUP_INET6_POST_BIND] = "post_bind6", - [BPF_CGROUP_INET4_GETPEERNAME] = "getpeername4", - [BPF_CGROUP_INET6_GETPEERNAME] = "getpeername6", - [BPF_CGROUP_INET4_GETSOCKNAME] = "getsockname4", - [BPF_CGROUP_INET6_GETSOCKNAME] = "getsockname6", - [BPF_CGROUP_UDP4_SENDMSG] = "sendmsg4", - [BPF_CGROUP_UDP6_SENDMSG] = "sendmsg6", - [BPF_CGROUP_SYSCTL] = "sysctl", - [BPF_CGROUP_UDP4_RECVMSG] = "recvmsg4", - [BPF_CGROUP_UDP6_RECVMSG] = "recvmsg6", - [BPF_CGROUP_GETSOCKOPT] = "getsockopt", - [BPF_CGROUP_SETSOCKOPT] = "setsockopt", - [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", - [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", - [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", - [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", - [BPF_LIRC_MODE2] = "lirc_mode2", - [BPF_FLOW_DISSECTOR] = "flow_dissector", - [BPF_TRACE_RAW_TP] = "raw_tp", - [BPF_TRACE_FENTRY] = "fentry", - [BPF_TRACE_FEXIT] = "fexit", - [BPF_MODIFY_RETURN] = "mod_ret", - [BPF_LSM_MAC] = "lsm_mac", - [BPF_SK_LOOKUP] = "sk_lookup", - [BPF_TRACE_ITER] = "trace_iter", - [BPF_XDP_DEVMAP] = "xdp_devmap", - [BPF_XDP_CPUMAP] = "xdp_cpumap", - [BPF_XDP] = "xdp", - [BPF_SK_REUSEPORT_SELECT] = "sk_skb_reuseport_select", - [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_skb_reuseport_select_or_migrate", - [BPF_PERF_EVENT] = "perf_event", - [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", -}; - void p_err(const char *fmt, ...) { va_list ap; @@ -119,11 +76,73 @@ static bool is_bpffs(char *path) return (unsigned long)st_fs.f_type == BPF_FS_MAGIC; } +/* Probe whether kernel switched from memlock-based (RLIMIT_MEMLOCK) to + * memcg-based memory accounting for BPF maps and programs. This was done in + * commit 97306be45fbe ("Merge branch 'switch to memcg-based memory + * accounting'"), in Linux 5.11. + * + * Libbpf also offers to probe for memcg-based accounting vs rlimit, but does + * so by checking for the availability of a given BPF helper and this has + * failed on some kernels with backports in the past, see commit 6b4384ff1088 + * ("Revert "bpftool: Use libbpf 1.0 API mode instead of RLIMIT_MEMLOCK""). + * Instead, we can probe by lowering the process-based rlimit to 0, trying to + * load a BPF object, and resetting the rlimit. If the load succeeds then + * memcg-based accounting is supported. + * + * This would be too dangerous to do in the library, because multithreaded + * applications might attempt to load items while the rlimit is at 0. Given + * that bpftool is single-threaded, this is fine to do here. + */ +static bool known_to_need_rlimit(void) +{ + struct rlimit rlim_init, rlim_cur_zero = {}; + struct bpf_insn insns[] = { + BPF_MOV64_IMM(BPF_REG_0, 0), + BPF_EXIT_INSN(), + }; + size_t insn_cnt = ARRAY_SIZE(insns); + union bpf_attr attr; + int prog_fd, err; + + memset(&attr, 0, sizeof(attr)); + attr.prog_type = BPF_PROG_TYPE_SOCKET_FILTER; + attr.insns = ptr_to_u64(insns); + attr.insn_cnt = insn_cnt; + attr.license = ptr_to_u64("GPL"); + + if (getrlimit(RLIMIT_MEMLOCK, &rlim_init)) + return false; + + /* Drop the soft limit to zero. We maintain the hard limit to its + * current value, because lowering it would be a permanent operation + * for unprivileged users. + */ + rlim_cur_zero.rlim_max = rlim_init.rlim_max; + if (setrlimit(RLIMIT_MEMLOCK, &rlim_cur_zero)) + return false; + + /* Do not use bpf_prog_load() from libbpf here, because it calls + * bump_rlimit_memlock(), interfering with the current probe. + */ + prog_fd = syscall(__NR_bpf, BPF_PROG_LOAD, &attr, sizeof(attr)); + err = errno; + + /* reset soft rlimit to its initial value */ + setrlimit(RLIMIT_MEMLOCK, &rlim_init); + + if (prog_fd < 0) + return err == EPERM; + + close(prog_fd); + return false; +} + void set_max_rlimit(void) { struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; - setrlimit(RLIMIT_MEMLOCK, &rinf); + if (known_to_need_rlimit()) + setrlimit(RLIMIT_MEMLOCK, &rinf); } static int @@ -297,6 +316,7 @@ const char *get_fd_type_name(enum bpf_obj_type type) [BPF_OBJ_UNKNOWN] = "unknown", [BPF_OBJ_PROG] = "prog", [BPF_OBJ_MAP] = "map", + [BPF_OBJ_LINK] = "link", }; if (type < 0 || type >= ARRAY_SIZE(names) || !names[type]) @@ -1017,3 +1037,39 @@ bool equal_fn_for_key_as_id(const void *k1, const void *k2, void *ctx) { return k1 == k2; } + +const char *bpf_attach_type_input_str(enum bpf_attach_type t) +{ + switch (t) { + case BPF_CGROUP_INET_INGRESS: return "ingress"; + case BPF_CGROUP_INET_EGRESS: return "egress"; + case BPF_CGROUP_INET_SOCK_CREATE: return "sock_create"; + case BPF_CGROUP_INET_SOCK_RELEASE: return "sock_release"; + case BPF_CGROUP_SOCK_OPS: return "sock_ops"; + case BPF_CGROUP_DEVICE: return "device"; + case BPF_CGROUP_INET4_BIND: return "bind4"; + case BPF_CGROUP_INET6_BIND: return "bind6"; + case BPF_CGROUP_INET4_CONNECT: return "connect4"; + case BPF_CGROUP_INET6_CONNECT: return "connect6"; + case BPF_CGROUP_INET4_POST_BIND: return "post_bind4"; + case BPF_CGROUP_INET6_POST_BIND: return "post_bind6"; + case BPF_CGROUP_INET4_GETPEERNAME: return "getpeername4"; + case BPF_CGROUP_INET6_GETPEERNAME: return "getpeername6"; + case BPF_CGROUP_INET4_GETSOCKNAME: return "getsockname4"; + case BPF_CGROUP_INET6_GETSOCKNAME: return "getsockname6"; + case BPF_CGROUP_UDP4_SENDMSG: return "sendmsg4"; + case BPF_CGROUP_UDP6_SENDMSG: return "sendmsg6"; + case BPF_CGROUP_SYSCTL: return "sysctl"; + case BPF_CGROUP_UDP4_RECVMSG: return "recvmsg4"; + case BPF_CGROUP_UDP6_RECVMSG: return "recvmsg6"; + case BPF_CGROUP_GETSOCKOPT: return "getsockopt"; + case BPF_CGROUP_SETSOCKOPT: return "setsockopt"; + case BPF_TRACE_RAW_TP: return "raw_tp"; + case BPF_TRACE_FENTRY: return "fentry"; + case BPF_TRACE_FEXIT: return "fexit"; + case BPF_MODIFY_RETURN: return "mod_ret"; + case BPF_SK_REUSEPORT_SELECT: return "sk_skb_reuseport_select"; + case BPF_SK_REUSEPORT_SELECT_OR_MIGRATE: return "sk_skb_reuseport_select_or_migrate"; + default: return libbpf_bpf_attach_type_str(t); + } +} diff --git a/tools/bpf/bpftool/feature.c b/tools/bpf/bpftool/feature.c index 290998c82d..7ecabf7947 100644 --- a/tools/bpf/bpftool/feature.c +++ b/tools/bpf/bpftool/feature.c @@ -548,8 +548,8 @@ static bool probe_prog_type_ifindex(enum bpf_prog_type prog_type, __u32 ifindex) } static void -probe_prog_type(enum bpf_prog_type prog_type, bool *supported_types, - const char *define_prefix, __u32 ifindex) +probe_prog_type(enum bpf_prog_type prog_type, const char *prog_type_str, + bool *supported_types, const char *define_prefix, __u32 ifindex) { char feat_name[128], plain_desc[128], define_name[128]; const char *plain_comment = "eBPF program_type "; @@ -567,7 +567,7 @@ probe_prog_type(enum bpf_prog_type prog_type, bool *supported_types, res = probe_prog_type_ifindex(prog_type, ifindex); } else { - res = libbpf_probe_bpf_prog_type(prog_type, NULL); + res = libbpf_probe_bpf_prog_type(prog_type, NULL) > 0; } #ifdef USE_LIBCAP @@ -580,20 +580,16 @@ probe_prog_type(enum bpf_prog_type prog_type, bool *supported_types, supported_types[prog_type] |= res; - if (!prog_type_name[prog_type]) { - p_info("program type name not found (type %d)", prog_type); - return; - } maxlen = sizeof(plain_desc) - strlen(plain_comment) - 1; - if (strlen(prog_type_name[prog_type]) > maxlen) { + if (strlen(prog_type_str) > maxlen) { p_info("program type name too long"); return; } - sprintf(feat_name, "have_%s_prog_type", prog_type_name[prog_type]); - sprintf(define_name, "%s_prog_type", prog_type_name[prog_type]); + sprintf(feat_name, "have_%s_prog_type", prog_type_str); + sprintf(define_name, "%s_prog_type", prog_type_str); uppercase(define_name, sizeof(define_name)); - sprintf(plain_desc, "%s%s", plain_comment, prog_type_name[prog_type]); + sprintf(plain_desc, "%s%s", plain_comment, prog_type_str); print_bool_feature(feat_name, plain_desc, define_name, res, define_prefix); } @@ -619,8 +615,8 @@ static bool probe_map_type_ifindex(enum bpf_map_type map_type, __u32 ifindex) } static void -probe_map_type(enum bpf_map_type map_type, const char *define_prefix, - __u32 ifindex) +probe_map_type(enum bpf_map_type map_type, char const *map_type_str, + const char *define_prefix, __u32 ifindex) { char feat_name[128], plain_desc[128], define_name[128]; const char *plain_comment = "eBPF map_type "; @@ -638,27 +634,23 @@ probe_map_type(enum bpf_map_type map_type, const char *define_prefix, res = probe_map_type_ifindex(map_type, ifindex); } else { - res = libbpf_probe_bpf_map_type(map_type, NULL); + res = libbpf_probe_bpf_map_type(map_type, NULL) > 0; } /* Probe result depends on the success of map creation, no additional * check required for unprivileged users */ - if (!map_type_name[map_type]) { - p_info("map type name not found (type %d)", map_type); - return; - } maxlen = sizeof(plain_desc) - strlen(plain_comment) - 1; - if (strlen(map_type_name[map_type]) > maxlen) { + if (strlen(map_type_str) > maxlen) { p_info("map type name too long"); return; } - sprintf(feat_name, "have_%s_map_type", map_type_name[map_type]); - sprintf(define_name, "%s_map_type", map_type_name[map_type]); + sprintf(feat_name, "have_%s_map_type", map_type_str); + sprintf(define_name, "%s_map_type", map_type_str); uppercase(define_name, sizeof(define_name)); - sprintf(plain_desc, "%s%s", plain_comment, map_type_name[map_type]); + sprintf(plain_desc, "%s%s", plain_comment, map_type_str); print_bool_feature(feat_name, plain_desc, define_name, res, define_prefix); } @@ -690,7 +682,7 @@ probe_helper_ifindex(enum bpf_func_id id, enum bpf_prog_type prog_type, return res; } -static void +static bool probe_helper_for_progtype(enum bpf_prog_type prog_type, bool supported_type, const char *define_prefix, unsigned int id, const char *ptype_name, __u32 ifindex) @@ -701,7 +693,7 @@ probe_helper_for_progtype(enum bpf_prog_type prog_type, bool supported_type, if (ifindex) res = probe_helper_ifindex(id, prog_type, ifindex); else - res = libbpf_probe_bpf_helper(prog_type, id, NULL); + res = libbpf_probe_bpf_helper(prog_type, id, NULL) > 0; #ifdef USE_LIBCAP /* Probe may succeed even if program load fails, for * unprivileged users check that we did not fail because of @@ -723,15 +715,18 @@ probe_helper_for_progtype(enum bpf_prog_type prog_type, bool supported_type, if (res) printf("\n\t- %s", helper_name[id]); } + + return res; } static void -probe_helpers_for_progtype(enum bpf_prog_type prog_type, bool supported_type, +probe_helpers_for_progtype(enum bpf_prog_type prog_type, + const char *prog_type_str, bool supported_type, const char *define_prefix, __u32 ifindex) { - const char *ptype_name = prog_type_name[prog_type]; char feat_name[128]; unsigned int id; + bool probe_res = false; if (ifindex) /* Only test helpers for offload-able program types */ @@ -744,12 +739,12 @@ probe_helpers_for_progtype(enum bpf_prog_type prog_type, bool supported_type, } if (json_output) { - sprintf(feat_name, "%s_available_helpers", ptype_name); + sprintf(feat_name, "%s_available_helpers", prog_type_str); jsonw_name(json_wtr, feat_name); jsonw_start_array(json_wtr); } else if (!define_prefix) { printf("eBPF helpers supported for program type %s:", - ptype_name); + prog_type_str); } for (id = 1; id < ARRAY_SIZE(helper_name); id++) { @@ -764,16 +759,25 @@ probe_helpers_for_progtype(enum bpf_prog_type prog_type, bool supported_type, continue; /* fallthrough */ default: - probe_helper_for_progtype(prog_type, supported_type, - define_prefix, id, ptype_name, + probe_res |= probe_helper_for_progtype(prog_type, supported_type, + define_prefix, id, prog_type_str, ifindex); } } if (json_output) jsonw_end_array(json_wtr); - else if (!define_prefix) + else if (!define_prefix) { printf("\n"); + if (!probe_res) { + if (!supported_type) + printf("\tProgram type not supported\n"); + else + printf("\tCould not determine which helpers are available\n"); + } + } + + } static void @@ -931,30 +935,47 @@ static void section_program_types(bool *supported_types, const char *define_prefix, __u32 ifindex) { - unsigned int i; + unsigned int prog_type = BPF_PROG_TYPE_UNSPEC; + const char *prog_type_str; print_start_section("program_types", "Scanning eBPF program types...", "/*** eBPF program types ***/", define_prefix); - for (i = BPF_PROG_TYPE_UNSPEC + 1; i < prog_type_name_size; i++) - probe_prog_type(i, supported_types, define_prefix, ifindex); + while (true) { + prog_type++; + prog_type_str = libbpf_bpf_prog_type_str(prog_type); + /* libbpf will return NULL for variants unknown to it. */ + if (!prog_type_str) + break; + + probe_prog_type(prog_type, prog_type_str, supported_types, define_prefix, + ifindex); + } print_end_section(); } static void section_map_types(const char *define_prefix, __u32 ifindex) { - unsigned int i; + unsigned int map_type = BPF_MAP_TYPE_UNSPEC; + const char *map_type_str; print_start_section("map_types", "Scanning eBPF map types...", "/*** eBPF map types ***/", define_prefix); - for (i = BPF_MAP_TYPE_UNSPEC + 1; i < map_type_name_size; i++) - probe_map_type(i, define_prefix, ifindex); + while (true) { + map_type++; + map_type_str = libbpf_bpf_map_type_str(map_type); + /* libbpf will return NULL for variants unknown to it. */ + if (!map_type_str) + break; + + probe_map_type(map_type, map_type_str, define_prefix, ifindex); + } print_end_section(); } @@ -962,7 +983,8 @@ static void section_map_types(const char *define_prefix, __u32 ifindex) static void section_helpers(bool *supported_types, const char *define_prefix, __u32 ifindex) { - unsigned int i; + unsigned int prog_type = BPF_PROG_TYPE_UNSPEC; + const char *prog_type_str; print_start_section("helpers", "Scanning eBPF helper functions...", @@ -984,9 +1006,18 @@ section_helpers(bool *supported_types, const char *define_prefix, __u32 ifindex) " %sBPF__PROG_TYPE_ ## prog_type ## __HELPER_ ## helper\n", define_prefix, define_prefix, define_prefix, define_prefix); - for (i = BPF_PROG_TYPE_UNSPEC + 1; i < prog_type_name_size; i++) - probe_helpers_for_progtype(i, supported_types[i], define_prefix, + while (true) { + prog_type++; + prog_type_str = libbpf_bpf_prog_type_str(prog_type); + /* libbpf will return NULL for variants unknown to it. */ + if (!prog_type_str) + break; + + probe_helpers_for_progtype(prog_type, prog_type_str, + supported_types[prog_type], + define_prefix, ifindex); + } print_end_section(); } @@ -1227,6 +1258,58 @@ static int do_probe(int argc, char **argv) return 0; } +static const char *get_helper_name(unsigned int id) +{ + if (id >= ARRAY_SIZE(helper_name)) + return NULL; + + return helper_name[id]; +} + +static int do_list_builtins(int argc, char **argv) +{ + const char *(*get_name)(unsigned int id); + unsigned int id = 0; + + if (argc < 1) + usage(); + + if (is_prefix(*argv, "prog_types")) { + get_name = (const char *(*)(unsigned int))libbpf_bpf_prog_type_str; + } else if (is_prefix(*argv, "map_types")) { + get_name = (const char *(*)(unsigned int))libbpf_bpf_map_type_str; + } else if (is_prefix(*argv, "attach_types")) { + get_name = (const char *(*)(unsigned int))libbpf_bpf_attach_type_str; + } else if (is_prefix(*argv, "link_types")) { + get_name = (const char *(*)(unsigned int))libbpf_bpf_link_type_str; + } else if (is_prefix(*argv, "helpers")) { + get_name = get_helper_name; + } else { + p_err("expected 'prog_types', 'map_types', 'attach_types', 'link_types' or 'helpers', got: %s", *argv); + return -1; + } + + if (json_output) + jsonw_start_array(json_wtr); /* root array */ + + while (true) { + const char *name; + + name = get_name(id++); + if (!name) + break; + if (json_output) + jsonw_string(json_wtr, name); + else + printf("%s\n", name); + } + + if (json_output) + jsonw_end_array(json_wtr); /* root array */ + + return 0; +} + static int do_help(int argc, char **argv) { if (json_output) { @@ -1236,9 +1319,11 @@ static int do_help(int argc, char **argv) fprintf(stderr, "Usage: %1$s %2$s probe [COMPONENT] [full] [unprivileged] [macros [prefix PREFIX]]\n" + " %1$s %2$s list_builtins GROUP\n" " %1$s %2$s help\n" "\n" " COMPONENT := { kernel | dev NAME }\n" + " GROUP := { prog_types | map_types | attach_types | link_types | helpers }\n" " " HELP_SPEC_OPTIONS " }\n" "", bin_name, argv[-2]); @@ -1247,8 +1332,9 @@ static int do_help(int argc, char **argv) } static const struct cmd cmds[] = { - { "probe", do_probe }, - { "help", do_help }, + { "probe", do_probe }, + { "list_builtins", do_list_builtins }, + { "help", do_help }, { 0 } }; diff --git a/tools/bpf/bpftool/gen.c b/tools/bpf/bpftool/gen.c index 7678af3647..7070dcffa8 100644 --- a/tools/bpf/bpftool/gen.c +++ b/tools/bpf/bpftool/gen.c @@ -474,6 +474,9 @@ static void codegen_asserts(struct bpf_object *obj, const char *obj_name) const struct btf_type *sec; char map_ident[256], var_ident[256]; + if (!btf) + return; + codegen("\ \n\ __attribute__((unused)) static void \n\ @@ -549,6 +552,7 @@ static void codegen_attach_detach(struct bpf_object *obj, const char *obj_name) printf("\tint fd = skel_raw_tracepoint_open(\"%s\", prog_fd);\n", tp_name); break; case BPF_PROG_TYPE_TRACING: + case BPF_PROG_TYPE_LSM: if (bpf_program__expected_attach_type(prog) == BPF_TRACE_ITER) printf("\tint fd = skel_link_create(prog_fd, 0, BPF_TRACE_ITER);\n"); else @@ -999,7 +1003,7 @@ static int do_skeleton(int argc, char **argv) codegen("\ \n\ /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ \n\ - /* THIS FILE IS AUTOGENERATED! */ \n\ + /* THIS FILE IS AUTOGENERATED BY BPFTOOL! */ \n\ #ifndef %2$s \n\ #define %2$s \n\ \n\ @@ -1015,7 +1019,7 @@ static int do_skeleton(int argc, char **argv) \n\ /* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ \n\ \n\ - /* THIS FILE IS AUTOGENERATED! */ \n\ + /* THIS FILE IS AUTOGENERATED BY BPFTOOL! */ \n\ #ifndef %2$s \n\ #define %2$s \n\ \n\ @@ -1171,7 +1175,7 @@ static int do_skeleton(int argc, char **argv) static inline void \n\ %1$s__detach(struct %1$s *obj) \n\ { \n\ - return bpf_object__detach_skeleton(obj->skeleton); \n\ + bpf_object__detach_skeleton(obj->skeleton); \n\ } \n\ ", obj_name @@ -1746,6 +1750,7 @@ btfgen_mark_type(struct btfgen_info *info, unsigned int type_id, bool follow_poi case BTF_KIND_INT: case BTF_KIND_FLOAT: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_STRUCT: case BTF_KIND_UNION: break; @@ -1757,6 +1762,7 @@ btfgen_mark_type(struct btfgen_info *info, unsigned int type_id, bool follow_poi } break; case BTF_KIND_CONST: + case BTF_KIND_RESTRICT: case BTF_KIND_VOLATILE: case BTF_KIND_TYPEDEF: err = btfgen_mark_type(info, btf_type->type, follow_pointers); @@ -1851,6 +1857,112 @@ static int btfgen_record_field_relo(struct btfgen_info *info, struct bpf_core_sp return 0; } +/* Mark types, members, and member types. Compared to btfgen_record_field_relo, + * this function does not rely on the target spec for inferring members, but + * uses the associated BTF. + * + * The `behind_ptr` argument is used to stop marking of composite types reached + * through a pointer. This way, we can keep BTF size in check while providing + * reasonable match semantics. + */ +static int btfgen_mark_type_match(struct btfgen_info *info, __u32 type_id, bool behind_ptr) +{ + const struct btf_type *btf_type; + struct btf *btf = info->src_btf; + struct btf_type *cloned_type; + int i, err; + + if (type_id == 0) + return 0; + + btf_type = btf__type_by_id(btf, type_id); + /* mark type on cloned BTF as used */ + cloned_type = (struct btf_type *)btf__type_by_id(info->marked_btf, type_id); + cloned_type->name_off = MARKED; + + switch (btf_kind(btf_type)) { + case BTF_KIND_UNKN: + case BTF_KIND_INT: + case BTF_KIND_FLOAT: + case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: + break; + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: { + struct btf_member *m = btf_members(btf_type); + __u16 vlen = btf_vlen(btf_type); + + if (behind_ptr) + break; + + for (i = 0; i < vlen; i++, m++) { + /* mark member */ + btfgen_mark_member(info, type_id, i); + + /* mark member's type */ + err = btfgen_mark_type_match(info, m->type, false); + if (err) + return err; + } + break; + } + case BTF_KIND_CONST: + case BTF_KIND_FWD: + case BTF_KIND_RESTRICT: + case BTF_KIND_TYPEDEF: + case BTF_KIND_VOLATILE: + return btfgen_mark_type_match(info, btf_type->type, behind_ptr); + case BTF_KIND_PTR: + return btfgen_mark_type_match(info, btf_type->type, true); + case BTF_KIND_ARRAY: { + struct btf_array *array; + + array = btf_array(btf_type); + /* mark array type */ + err = btfgen_mark_type_match(info, array->type, false); + /* mark array's index type */ + err = err ? : btfgen_mark_type_match(info, array->index_type, false); + if (err) + return err; + break; + } + case BTF_KIND_FUNC_PROTO: { + __u16 vlen = btf_vlen(btf_type); + struct btf_param *param; + + /* mark ret type */ + err = btfgen_mark_type_match(info, btf_type->type, false); + if (err) + return err; + + /* mark parameters types */ + param = btf_params(btf_type); + for (i = 0; i < vlen; i++) { + err = btfgen_mark_type_match(info, param->type, false); + if (err) + return err; + param++; + } + break; + } + /* tells if some other type needs to be handled */ + default: + p_err("unsupported kind: %s (%d)", btf_kind_str(btf_type), type_id); + return -EINVAL; + } + + return 0; +} + +/* Mark types, members, and member types. Compared to btfgen_record_field_relo, + * this function does not rely on the target spec for inferring members, but + * uses the associated BTF. + */ +static int btfgen_record_type_match_relo(struct btfgen_info *info, struct bpf_core_spec *targ_spec) +{ + return btfgen_mark_type_match(info, targ_spec->root_type_id, false); +} + static int btfgen_record_type_relo(struct btfgen_info *info, struct bpf_core_spec *targ_spec) { return btfgen_mark_type(info, targ_spec->root_type_id, true); @@ -1877,6 +1989,8 @@ static int btfgen_record_reloc(struct btfgen_info *info, struct bpf_core_spec *r case BPF_CORE_TYPE_EXISTS: case BPF_CORE_TYPE_SIZE: return btfgen_record_type_relo(info, res); + case BPF_CORE_TYPE_MATCHES: + return btfgen_record_type_match_relo(info, res); case BPF_CORE_ENUMVAL_EXISTS: case BPF_CORE_ENUMVAL_VALUE: return btfgen_record_enumval_relo(info, res); diff --git a/tools/bpf/bpftool/jit_disasm.c b/tools/bpf/bpftool/jit_disasm.c index 24734f2249..aaf99a0168 100644 --- a/tools/bpf/bpftool/jit_disasm.c +++ b/tools/bpf/bpftool/jit_disasm.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "json_writer.h" #include "main.h" @@ -39,15 +40,12 @@ static void get_exec_path(char *tpath, size_t size) } static int oper_count; -static int fprintf_json(void *out, const char *fmt, ...) +static int printf_json(void *out, const char *fmt, va_list ap) { - va_list ap; char *s; int err; - va_start(ap, fmt); err = vasprintf(&s, fmt, ap); - va_end(ap); if (err < 0) return -1; @@ -73,6 +71,32 @@ static int fprintf_json(void *out, const char *fmt, ...) return 0; } +static int fprintf_json(void *out, const char *fmt, ...) +{ + va_list ap; + int r; + + va_start(ap, fmt); + r = printf_json(out, fmt, ap); + va_end(ap); + + return r; +} + +static int fprintf_json_styled(void *out, + enum disassembler_style style __maybe_unused, + const char *fmt, ...) +{ + va_list ap; + int r; + + va_start(ap, fmt); + r = printf_json(out, fmt, ap); + va_end(ap); + + return r; +} + void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, const char *arch, const char *disassembler_options, const struct btf *btf, @@ -99,11 +123,13 @@ void disasm_print_insn(unsigned char *image, ssize_t len, int opcodes, assert(bfd_check_format(bfdf, bfd_object)); if (json_output) - init_disassemble_info(&info, stdout, - (fprintf_ftype) fprintf_json); + init_disassemble_info_compat(&info, stdout, + (fprintf_ftype) fprintf_json, + fprintf_json_styled); else - init_disassemble_info(&info, stdout, - (fprintf_ftype) fprintf); + init_disassemble_info_compat(&info, stdout, + (fprintf_ftype) fprintf, + fprintf_styled); /* Update architecture info for offload. */ if (arch) { diff --git a/tools/bpf/bpftool/link.c b/tools/bpf/bpftool/link.c index 97dec81950..7a20931c32 100644 --- a/tools/bpf/bpftool/link.c +++ b/tools/bpf/bpftool/link.c @@ -13,15 +13,6 @@ #include "json_writer.h" #include "main.h" -static const char * const link_type_name[] = { - [BPF_LINK_TYPE_UNSPEC] = "unspec", - [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", - [BPF_LINK_TYPE_TRACING] = "tracing", - [BPF_LINK_TYPE_CGROUP] = "cgroup", - [BPF_LINK_TYPE_ITER] = "iter", - [BPF_LINK_TYPE_NETNS] = "netns", -}; - static struct hashmap *link_table; static int link_parse_fd(int *argc, char ***argv) @@ -63,9 +54,12 @@ static int link_parse_fd(int *argc, char ***argv) static void show_link_header_json(struct bpf_link_info *info, json_writer_t *wtr) { + const char *link_type_str; + jsonw_uint_field(wtr, "id", info->id); - if (info->type < ARRAY_SIZE(link_type_name)) - jsonw_string_field(wtr, "type", link_type_name[info->type]); + link_type_str = libbpf_bpf_link_type_str(info->type); + if (link_type_str) + jsonw_string_field(wtr, "type", link_type_str); else jsonw_uint_field(wtr, "type", info->type); @@ -74,9 +68,11 @@ show_link_header_json(struct bpf_link_info *info, json_writer_t *wtr) static void show_link_attach_type_json(__u32 attach_type, json_writer_t *wtr) { - if (attach_type < ARRAY_SIZE(attach_type_name)) - jsonw_string_field(wtr, "attach_type", - attach_type_name[attach_type]); + const char *attach_type_str; + + attach_type_str = libbpf_bpf_attach_type_str(attach_type); + if (attach_type_str) + jsonw_string_field(wtr, "attach_type", attach_type_str); else jsonw_uint_field(wtr, "attach_type", attach_type); } @@ -117,6 +113,7 @@ static int get_prog_info(int prog_id, struct bpf_prog_info *info) static int show_link_close_json(int fd, struct bpf_link_info *info) { struct bpf_prog_info prog_info; + const char *prog_type_str; int err; jsonw_start_object(json_wtr); @@ -133,12 +130,12 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) if (err) return err; - if (prog_info.type < prog_type_name_size) - jsonw_string_field(json_wtr, "prog_type", - prog_type_name[prog_info.type]); + prog_type_str = libbpf_bpf_prog_type_str(prog_info.type); + /* libbpf will return NULL for variants unknown to it. */ + if (prog_type_str) + jsonw_string_field(json_wtr, "prog_type", prog_type_str); else - jsonw_uint_field(json_wtr, "prog_type", - prog_info.type); + jsonw_uint_field(json_wtr, "prog_type", prog_info.type); show_link_attach_type_json(info->tracing.attach_type, json_wtr); @@ -180,9 +177,12 @@ static int show_link_close_json(int fd, struct bpf_link_info *info) static void show_link_header_plain(struct bpf_link_info *info) { + const char *link_type_str; + printf("%u: ", info->id); - if (info->type < ARRAY_SIZE(link_type_name)) - printf("%s ", link_type_name[info->type]); + link_type_str = libbpf_bpf_link_type_str(info->type); + if (link_type_str) + printf("%s ", link_type_str); else printf("type %u ", info->type); @@ -191,8 +191,11 @@ static void show_link_header_plain(struct bpf_link_info *info) static void show_link_attach_type_plain(__u32 attach_type) { - if (attach_type < ARRAY_SIZE(attach_type_name)) - printf("attach_type %s ", attach_type_name[attach_type]); + const char *attach_type_str; + + attach_type_str = libbpf_bpf_attach_type_str(attach_type); + if (attach_type_str) + printf("attach_type %s ", attach_type_str); else printf("attach_type %u ", attach_type); } @@ -210,6 +213,7 @@ static void show_iter_plain(struct bpf_link_info *info) static int show_link_close_plain(int fd, struct bpf_link_info *info) { struct bpf_prog_info prog_info; + const char *prog_type_str; int err; show_link_header_plain(info); @@ -224,9 +228,10 @@ static int show_link_close_plain(int fd, struct bpf_link_info *info) if (err) return err; - if (prog_info.type < prog_type_name_size) - printf("\n\tprog_type %s ", - prog_type_name[prog_info.type]); + prog_type_str = libbpf_bpf_prog_type_str(prog_info.type); + /* libbpf will return NULL for variants unknown to it. */ + if (prog_type_str) + printf("\n\tprog_type %s ", prog_type_str); else printf("\n\tprog_type %u ", prog_info.type); diff --git a/tools/bpf/bpftool/main.c b/tools/bpf/bpftool/main.c index e81227761f..451cefc2d0 100644 --- a/tools/bpf/bpftool/main.c +++ b/tools/bpf/bpftool/main.c @@ -507,9 +507,7 @@ int main(int argc, char **argv) * It will still be rejected if users use LIBBPF_STRICT_ALL * mode for loading generated skeleton. */ - ret = libbpf_set_strict_mode(LIBBPF_STRICT_ALL & ~LIBBPF_STRICT_MAP_DEFINITIONS); - if (ret) - p_err("failed to enable libbpf strict mode: %d", ret); + libbpf_set_strict_mode(LIBBPF_STRICT_ALL & ~LIBBPF_STRICT_MAP_DEFINITIONS); } argc -= optind; diff --git a/tools/bpf/bpftool/main.h b/tools/bpf/bpftool/main.h index 6e9277ffc6..5e5060c2ac 100644 --- a/tools/bpf/bpftool/main.h +++ b/tools/bpf/bpftool/main.h @@ -63,14 +63,6 @@ static inline void *u64_to_ptr(__u64 ptr) #define HELP_SPEC_LINK \ "LINK := { id LINK_ID | pinned FILE }" -extern const char * const prog_type_name[]; -extern const size_t prog_type_name_size; - -extern const char * const attach_type_name[__MAX_BPF_ATTACH_TYPE]; - -extern const char * const map_type_name[]; -extern const size_t map_type_name_size; - /* keep in sync with the definition in skeleton/pid_iter.bpf.c */ enum bpf_obj_type { BPF_OBJ_UNKNOWN, @@ -251,6 +243,20 @@ int print_all_levels(__maybe_unused enum libbpf_print_level level, size_t hash_fn_for_key_as_id(const void *key, void *ctx); bool equal_fn_for_key_as_id(const void *k1, const void *k2, void *ctx); +/* bpf_attach_type_input_str - convert the provided attach type value into a + * textual representation that we accept for input purposes. + * + * This function is similar in nature to libbpf_bpf_attach_type_str, but + * recognizes some attach type names that have been used by the program in the + * past and which do not follow the string inference scheme that libbpf uses. + * These textual representations should only be used for user input. + * + * @t: The attach type + * Returns a pointer to a static string identifying the attach type. NULL is + * returned for unknown bpf_attach_type values. + */ +const char *bpf_attach_type_input_str(enum bpf_attach_type t); + static inline void *u32_as_hash_field(__u32 x) { return (void *)(uintptr_t)x; diff --git a/tools/bpf/bpftool/map.c b/tools/bpf/bpftool/map.c index c26378f208..38b6bc9c26 100644 --- a/tools/bpf/bpftool/map.c +++ b/tools/bpf/bpftool/map.c @@ -22,42 +22,6 @@ #include "json_writer.h" #include "main.h" -const char * const map_type_name[] = { - [BPF_MAP_TYPE_UNSPEC] = "unspec", - [BPF_MAP_TYPE_HASH] = "hash", - [BPF_MAP_TYPE_ARRAY] = "array", - [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", - [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", - [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", - [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", - [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", - [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", - [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", - [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", - [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", - [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", - [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", - [BPF_MAP_TYPE_DEVMAP] = "devmap", - [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", - [BPF_MAP_TYPE_SOCKMAP] = "sockmap", - [BPF_MAP_TYPE_CPUMAP] = "cpumap", - [BPF_MAP_TYPE_XSKMAP] = "xskmap", - [BPF_MAP_TYPE_SOCKHASH] = "sockhash", - [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", - [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", - [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", - [BPF_MAP_TYPE_QUEUE] = "queue", - [BPF_MAP_TYPE_STACK] = "stack", - [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", - [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", - [BPF_MAP_TYPE_RINGBUF] = "ringbuf", - [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", - [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", - [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", -}; - -const size_t map_type_name_size = ARRAY_SIZE(map_type_name); - static struct hashmap *map_table; static bool map_is_per_cpu(__u32 type) @@ -81,12 +45,18 @@ static bool map_is_map_of_progs(__u32 type) static int map_type_from_str(const char *type) { + const char *map_type_str; unsigned int i; - for (i = 0; i < ARRAY_SIZE(map_type_name); i++) + for (i = 0; ; i++) { + map_type_str = libbpf_bpf_map_type_str(i); + if (!map_type_str) + break; + /* Don't allow prefixing in case of possible future shadowing */ - if (map_type_name[i] && !strcmp(map_type_name[i], type)) + if (!strcmp(map_type_str, type)) return i; + } return -1; } @@ -472,9 +442,12 @@ static int parse_elem(char **argv, struct bpf_map_info *info, static void show_map_header_json(struct bpf_map_info *info, json_writer_t *wtr) { + const char *map_type_str; + jsonw_uint_field(wtr, "id", info->id); - if (info->type < ARRAY_SIZE(map_type_name)) - jsonw_string_field(wtr, "type", map_type_name[info->type]); + map_type_str = libbpf_bpf_map_type_str(info->type); + if (map_type_str) + jsonw_string_field(wtr, "type", map_type_str); else jsonw_uint_field(wtr, "type", info->type); @@ -513,10 +486,12 @@ static int show_map_close_json(int fd, struct bpf_map_info *info) if (owner_prog_type) { unsigned int prog_type = atoi(owner_prog_type); + const char *prog_type_str; - if (prog_type < prog_type_name_size) + prog_type_str = libbpf_bpf_prog_type_str(prog_type); + if (prog_type_str) jsonw_string_field(json_wtr, "owner_prog_type", - prog_type_name[prog_type]); + prog_type_str); else jsonw_uint_field(json_wtr, "owner_prog_type", prog_type); @@ -559,9 +534,13 @@ static int show_map_close_json(int fd, struct bpf_map_info *info) static void show_map_header_plain(struct bpf_map_info *info) { + const char *map_type_str; + printf("%u: ", info->id); - if (info->type < ARRAY_SIZE(map_type_name)) - printf("%s ", map_type_name[info->type]); + + map_type_str = libbpf_bpf_map_type_str(info->type); + if (map_type_str) + printf("%s ", map_type_str); else printf("type %u ", info->type); @@ -597,10 +576,11 @@ static int show_map_close_plain(int fd, struct bpf_map_info *info) printf("\n\t"); if (owner_prog_type) { unsigned int prog_type = atoi(owner_prog_type); + const char *prog_type_str; - if (prog_type < prog_type_name_size) - printf("owner_prog_type %s ", - prog_type_name[prog_type]); + prog_type_str = libbpf_bpf_prog_type_str(prog_type); + if (prog_type_str) + printf("owner_prog_type %s ", prog_type_str); else printf("owner_prog_type %d ", prog_type); } @@ -876,9 +856,13 @@ map_dump(int fd, struct bpf_map_info *info, json_writer_t *wtr, } if (info->type == BPF_MAP_TYPE_REUSEPORT_SOCKARRAY && - info->value_size != 8) + info->value_size != 8) { + const char *map_type_str; + + map_type_str = libbpf_bpf_map_type_str(info->type); p_info("Warning: cannot read values from %s map with value_size != 8", - map_type_name[info->type]); + map_type_str); + } while (true) { err = bpf_map_get_next_key(fd, prev_key, key); if (err) { diff --git a/tools/bpf/bpftool/perf.c b/tools/bpf/bpftool/perf.c index 50de087b0d..226ec2c390 100644 --- a/tools/bpf/bpftool/perf.c +++ b/tools/bpf/bpftool/perf.c @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include @@ -147,81 +147,83 @@ static void print_perf_plain(int pid, int fd, __u32 prog_id, __u32 fd_type, } } -static int show_proc(const char *fpath, const struct stat *sb, - int tflag, struct FTW *ftwbuf) +static int show_proc(void) { + struct dirent *proc_de, *pid_fd_de; __u64 probe_offset, probe_addr; __u32 len, prog_id, fd_type; - int err, pid = 0, fd = 0; + DIR *proc, *pid_fd; + int err, pid, fd; const char *pch; char buf[4096]; - /* prefix always /proc */ - pch = fpath + 5; - if (*pch == '\0') - return 0; + proc = opendir("/proc"); + if (!proc) + return -1; - /* pid should be all numbers */ - pch++; - while (isdigit(*pch)) { - pid = pid * 10 + *pch - '0'; - pch++; + while ((proc_de = readdir(proc))) { + pid = 0; + pch = proc_de->d_name; + + /* pid should be all numbers */ + while (isdigit(*pch)) { + pid = pid * 10 + *pch - '0'; + pch++; + } + if (*pch != '\0') + continue; + + err = snprintf(buf, sizeof(buf), "/proc/%s/fd", proc_de->d_name); + if (err < 0 || err >= (int)sizeof(buf)) + continue; + + pid_fd = opendir(buf); + if (!pid_fd) + continue; + + while ((pid_fd_de = readdir(pid_fd))) { + fd = 0; + pch = pid_fd_de->d_name; + + /* fd should be all numbers */ + while (isdigit(*pch)) { + fd = fd * 10 + *pch - '0'; + pch++; + } + if (*pch != '\0') + continue; + + /* query (pid, fd) for potential perf events */ + len = sizeof(buf); + err = bpf_task_fd_query(pid, fd, 0, buf, &len, + &prog_id, &fd_type, + &probe_offset, &probe_addr); + if (err < 0) + continue; + + if (json_output) + print_perf_json(pid, fd, prog_id, fd_type, buf, + probe_offset, probe_addr); + else + print_perf_plain(pid, fd, prog_id, fd_type, buf, + probe_offset, probe_addr); + } + closedir(pid_fd); } - if (*pch == '\0') - return 0; - if (*pch != '/') - return FTW_SKIP_SUBTREE; - - /* check /proc//fd directory */ - pch++; - if (strncmp(pch, "fd", 2)) - return FTW_SKIP_SUBTREE; - pch += 2; - if (*pch == '\0') - return 0; - if (*pch != '/') - return FTW_SKIP_SUBTREE; - - /* check /proc//fd/ */ - pch++; - while (isdigit(*pch)) { - fd = fd * 10 + *pch - '0'; - pch++; - } - if (*pch != '\0') - return FTW_SKIP_SUBTREE; - - /* query (pid, fd) for potential perf events */ - len = sizeof(buf); - err = bpf_task_fd_query(pid, fd, 0, buf, &len, &prog_id, &fd_type, - &probe_offset, &probe_addr); - if (err < 0) - return 0; - - if (json_output) - print_perf_json(pid, fd, prog_id, fd_type, buf, probe_offset, - probe_addr); - else - print_perf_plain(pid, fd, prog_id, fd_type, buf, probe_offset, - probe_addr); - + closedir(proc); return 0; } static int do_show(int argc, char **argv) { - int flags = FTW_ACTIONRETVAL | FTW_PHYS; - int err = 0, nopenfd = 16; + int err; if (!has_perf_query_support()) return -1; if (json_output) jsonw_start_array(json_wtr); - if (nftw("/proc", show_proc, nopenfd, flags) == -1) { - p_err("%s", strerror(errno)); - err = -1; - } + err = show_proc(); if (json_output) jsonw_end_array(json_wtr); diff --git a/tools/bpf/bpftool/prog.c b/tools/bpf/bpftool/prog.c index bc4e05542c..c81362a001 100644 --- a/tools/bpf/bpftool/prog.c +++ b/tools/bpf/bpftool/prog.c @@ -36,53 +36,28 @@ #define BPF_METADATA_PREFIX "bpf_metadata_" #define BPF_METADATA_PREFIX_LEN (sizeof(BPF_METADATA_PREFIX) - 1) -const char * const prog_type_name[] = { - [BPF_PROG_TYPE_UNSPEC] = "unspec", - [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", - [BPF_PROG_TYPE_KPROBE] = "kprobe", - [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", - [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", - [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", - [BPF_PROG_TYPE_XDP] = "xdp", - [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", - [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", - [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", - [BPF_PROG_TYPE_LWT_IN] = "lwt_in", - [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", - [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", - [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", - [BPF_PROG_TYPE_SK_SKB] = "sk_skb", - [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", - [BPF_PROG_TYPE_SK_MSG] = "sk_msg", - [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", - [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", - [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", - [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", - [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", - [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", - [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", - [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", - [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", - [BPF_PROG_TYPE_TRACING] = "tracing", - [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", - [BPF_PROG_TYPE_EXT] = "ext", - [BPF_PROG_TYPE_LSM] = "lsm", - [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", -}; - -const size_t prog_type_name_size = ARRAY_SIZE(prog_type_name); - enum dump_mode { DUMP_JITED, DUMP_XLATED, }; +static const bool attach_types[] = { + [BPF_SK_SKB_STREAM_PARSER] = true, + [BPF_SK_SKB_STREAM_VERDICT] = true, + [BPF_SK_SKB_VERDICT] = true, + [BPF_SK_MSG_VERDICT] = true, + [BPF_FLOW_DISSECTOR] = true, + [__MAX_BPF_ATTACH_TYPE] = false, +}; + +/* Textual representations traditionally used by the program and kept around + * for the sake of backwards compatibility. + */ static const char * const attach_type_strings[] = { [BPF_SK_SKB_STREAM_PARSER] = "stream_parser", [BPF_SK_SKB_STREAM_VERDICT] = "stream_verdict", [BPF_SK_SKB_VERDICT] = "skb_verdict", [BPF_SK_MSG_VERDICT] = "msg_verdict", - [BPF_FLOW_DISSECTOR] = "flow_dissector", [__MAX_BPF_ATTACH_TYPE] = NULL, }; @@ -93,6 +68,14 @@ static enum bpf_attach_type parse_attach_type(const char *str) enum bpf_attach_type type; for (type = 0; type < __MAX_BPF_ATTACH_TYPE; type++) { + if (attach_types[type]) { + const char *attach_type_str; + + attach_type_str = libbpf_bpf_attach_type_str(type); + if (!strcmp(str, attach_type_str)) + return type; + } + if (attach_type_strings[type] && is_prefix(str, attach_type_strings[type])) return type; @@ -427,12 +410,14 @@ static void show_prog_metadata(int fd, __u32 num_maps) static void print_prog_header_json(struct bpf_prog_info *info, int fd) { + const char *prog_type_str; char prog_name[MAX_PROG_FULL_NAME]; jsonw_uint_field(json_wtr, "id", info->id); - if (info->type < ARRAY_SIZE(prog_type_name)) - jsonw_string_field(json_wtr, "type", - prog_type_name[info->type]); + prog_type_str = libbpf_bpf_prog_type_str(info->type); + + if (prog_type_str) + jsonw_string_field(json_wtr, "type", prog_type_str); else jsonw_uint_field(json_wtr, "type", info->type); @@ -514,11 +499,13 @@ static void print_prog_json(struct bpf_prog_info *info, int fd) static void print_prog_header_plain(struct bpf_prog_info *info, int fd) { + const char *prog_type_str; char prog_name[MAX_PROG_FULL_NAME]; printf("%u: ", info->id); - if (info->type < ARRAY_SIZE(prog_type_name)) - printf("%s ", prog_type_name[info->type]); + prog_type_str = libbpf_bpf_prog_type_str(info->type); + if (prog_type_str) + printf("%s ", prog_type_str); else printf("type %u ", info->type); @@ -1975,7 +1962,7 @@ static int profile_parse_metrics(int argc, char **argv) int selected_cnt = 0; unsigned int i; - metric_cnt = sizeof(metrics) / sizeof(struct profile_metric); + metric_cnt = ARRAY_SIZE(metrics); while (argc > 0) { for (i = 0; i < metric_cnt; i++) { @@ -2376,8 +2363,8 @@ static int do_help(int argc, char **argv) " cgroup/sendmsg6 | cgroup/recvmsg4 | cgroup/recvmsg6 |\n" " cgroup/getsockopt | cgroup/setsockopt | cgroup/sock_release |\n" " struct_ops | fentry | fexit | freplace | sk_lookup }\n" - " ATTACH_TYPE := { msg_verdict | skb_verdict | stream_verdict |\n" - " stream_parser | flow_dissector }\n" + " ATTACH_TYPE := { sk_msg_verdict | sk_skb_verdict | sk_skb_stream_verdict |\n" + " sk_skb_stream_parser | flow_dissector }\n" " METRIC := { cycles | instructions | l1d_loads | llc_misses | itlb_misses | dtlb_misses }\n" " " HELP_SPEC_OPTIONS " |\n" " {-f|--bpffs} | {-m|--mapcompat} | {-n|--nomount} |\n" diff --git a/tools/bpf/bpftool/tracelog.c b/tools/bpf/bpftool/tracelog.c index e80a5c79b3..bf1f022127 100644 --- a/tools/bpf/bpftool/tracelog.c +++ b/tools/bpf/bpftool/tracelog.c @@ -9,7 +9,7 @@ #include #include #include -#include +#include #include #include "main.h" diff --git a/tools/bpf/resolve_btfids/main.c b/tools/bpf/resolve_btfids/main.c index 5d26f3c6f9..80cd7843c6 100644 --- a/tools/bpf/resolve_btfids/main.c +++ b/tools/bpf/resolve_btfids/main.c @@ -45,6 +45,19 @@ * .zero 4 * __BTF_ID__func__vfs_fallocate__4: * .zero 4 + * + * set8 - store symbol size into first 4 bytes and sort following + * ID list + * + * __BTF_ID__set8__list: + * .zero 8 + * list: + * __BTF_ID__func__vfs_getattr__3: + * .zero 4 + * .word (1 << 0) | (1 << 2) + * __BTF_ID__func__vfs_fallocate__5: + * .zero 4 + * .word (1 << 3) | (1 << 1) | (1 << 2) */ #define _GNU_SOURCE @@ -72,6 +85,7 @@ #define BTF_TYPEDEF "typedef" #define BTF_FUNC "func" #define BTF_SET "set" +#define BTF_SET8 "set8" #define ADDR_CNT 100 @@ -84,6 +98,7 @@ struct btf_id { }; int addr_cnt; bool is_set; + bool is_set8; Elf64_Addr addr[ADDR_CNT]; }; @@ -231,14 +246,14 @@ static char *get_id(const char *prefix_end) return id; } -static struct btf_id *add_set(struct object *obj, char *name) +static struct btf_id *add_set(struct object *obj, char *name, bool is_set8) { /* * __BTF_ID__set__name * name = ^ * id = ^ */ - char *id = name + sizeof(BTF_SET "__") - 1; + char *id = name + (is_set8 ? sizeof(BTF_SET8 "__") : sizeof(BTF_SET "__")) - 1; int len = strlen(name); if (id >= name + len) { @@ -444,9 +459,21 @@ static int symbols_collect(struct object *obj) } else if (!strncmp(prefix, BTF_FUNC, sizeof(BTF_FUNC) - 1)) { obj->nr_funcs++; id = add_symbol(&obj->funcs, prefix, sizeof(BTF_FUNC) - 1); + /* set8 */ + } else if (!strncmp(prefix, BTF_SET8, sizeof(BTF_SET8) - 1)) { + id = add_set(obj, prefix, true); + /* + * SET8 objects store list's count, which is encoded + * in symbol's size, together with 'cnt' field hence + * that - 1. + */ + if (id) { + id->cnt = sym.st_size / sizeof(uint64_t) - 1; + id->is_set8 = true; + } /* set */ } else if (!strncmp(prefix, BTF_SET, sizeof(BTF_SET) - 1)) { - id = add_set(obj, prefix); + id = add_set(obj, prefix, false); /* * SET objects store list's count, which is encoded * in symbol's size, together with 'cnt' field hence @@ -571,7 +598,8 @@ static int id_patch(struct object *obj, struct btf_id *id) int *ptr = data->d_buf; int i; - if (!id->id && !id->is_set) + /* For set, set8, id->id may be 0 */ + if (!id->id && !id->is_set && !id->is_set8) pr_err("WARN: resolve_btfids: unresolved symbol %s\n", id->name); for (i = 0; i < id->addr_cnt; i++) { @@ -643,13 +671,13 @@ static int sets_patch(struct object *obj) } idx = idx / sizeof(int); - base = &ptr[idx] + 1; + base = &ptr[idx] + (id->is_set8 ? 2 : 1); cnt = ptr[idx]; pr_debug("sorting addr %5lu: cnt %6d [%s]\n", (idx + 1) * sizeof(int), cnt, id->name); - qsort(base, cnt, sizeof(int), cmp_id); + qsort(base, cnt, id->is_set8 ? sizeof(uint64_t) : sizeof(int), cmp_id); next = rb_next(next); } diff --git a/tools/bpf/runqslower/Makefile b/tools/bpf/runqslower/Makefile index da6de16a3d..8b3d87b82b 100644 --- a/tools/bpf/runqslower/Makefile +++ b/tools/bpf/runqslower/Makefile @@ -4,7 +4,7 @@ include ../../scripts/Makefile.include OUTPUT ?= $(abspath .output)/ BPFTOOL_OUTPUT := $(OUTPUT)bpftool/ -DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bpftool +DEFAULT_BPFTOOL := $(BPFTOOL_OUTPUT)bootstrap/bpftool BPFTOOL ?= $(DEFAULT_BPFTOOL) LIBBPF_SRC := $(abspath ../../lib/bpf) BPFOBJ_OUTPUT := $(OUTPUT)libbpf/ @@ -86,6 +86,5 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(BPFOBJ_OU $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(BPFOBJ_OUTPUT) \ DESTDIR=$(BPFOBJ_OUTPUT) prefix= $(abspath $@) install_headers -$(DEFAULT_BPFTOOL): $(BPFOBJ) | $(BPFTOOL_OUTPUT) - $(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) \ - ARCH= CROSS_COMPILE= CC=$(HOSTCC) LD=$(HOSTLD) +$(DEFAULT_BPFTOOL): | $(BPFTOOL_OUTPUT) + $(Q)$(MAKE) $(submake_extras) -C ../bpftool OUTPUT=$(BPFTOOL_OUTPUT) bootstrap diff --git a/tools/bpf/runqslower/runqslower.c b/tools/bpf/runqslower/runqslower.c index d78f414859..83c5993a13 100644 --- a/tools/bpf/runqslower/runqslower.c +++ b/tools/bpf/runqslower/runqslower.c @@ -4,7 +4,6 @@ #include #include #include -#include #include #include #include @@ -88,16 +87,6 @@ int libbpf_print_fn(enum libbpf_print_level level, return vfprintf(stderr, format, args); } -static int bump_memlock_rlimit(void) -{ - struct rlimit rlim_new = { - .rlim_cur = RLIM_INFINITY, - .rlim_max = RLIM_INFINITY, - }; - - return setrlimit(RLIMIT_MEMLOCK, &rlim_new); -} - void handle_event(void *ctx, int cpu, void *data, __u32 data_sz) { const struct runq_event *e = data; @@ -133,11 +122,8 @@ int main(int argc, char **argv) libbpf_set_print(libbpf_print_fn); - err = bump_memlock_rlimit(); - if (err) { - fprintf(stderr, "failed to increase rlimit: %d", err); - return 1; - } + /* Use libbpf 1.0 API mode */ + libbpf_set_strict_mode(LIBBPF_STRICT_ALL); obj = runqslower_bpf__open(); if (!obj) { diff --git a/tools/build/Makefile.feature b/tools/build/Makefile.feature index ae61f46404..fc6ce0b253 100644 --- a/tools/build/Makefile.feature +++ b/tools/build/Makefile.feature @@ -70,6 +70,7 @@ FEATURE_TESTS_BASIC := \ libaio \ libzstd \ disassembler-four-args \ + disassembler-init-styled \ file-handle # FEATURE_TESTS_BASIC + FEATURE_TESTS_EXTRA is the complete list @@ -98,6 +99,11 @@ FEATURE_TESTS_EXTRA := \ llvm-version \ clang \ libbpf \ + libbpf-btf__load_from_kernel_by_id \ + libbpf-bpf_prog_load \ + libbpf-bpf_object__next_program \ + libbpf-bpf_object__next_map \ + libbpf-bpf_create_map \ libpfm4 \ libdebuginfod \ clang-bpf-co-re @@ -129,8 +135,7 @@ FEATURE_DISPLAY ?= \ get_cpuid \ bpf \ libaio \ - libzstd \ - disassembler-four-args + libzstd # Set FEATURE_CHECK_(C|LD)FLAGS-all for all FEATURE_TESTS features. # If in the future we need per-feature checks/flags for features not diff --git a/tools/build/feature/Makefile b/tools/build/feature/Makefile index de66e1cc07..04b07ff882 100644 --- a/tools/build/feature/Makefile +++ b/tools/build/feature/Makefile @@ -18,6 +18,7 @@ FILES= \ test-libbfd.bin \ test-libbfd-buildid.bin \ test-disassembler-four-args.bin \ + test-disassembler-init-styled.bin \ test-reallocarray.bin \ test-libbfd-liberty.bin \ test-libbfd-liberty-z.bin \ @@ -57,6 +58,12 @@ FILES= \ test-lzma.bin \ test-bpf.bin \ test-libbpf.bin \ + test-libbpf-btf__load_from_kernel_by_id.bin \ + test-libbpf-bpf_prog_load.bin \ + test-libbpf-bpf_map_create.bin \ + test-libbpf-bpf_object__next_program.bin \ + test-libbpf-bpf_object__next_map.bin \ + test-libbpf-btf__raw_data.bin \ test-get_cpuid.bin \ test-sdt.bin \ test-cxx.bin \ @@ -83,6 +90,8 @@ all: $(FILES) __BUILD = $(CC) $(CFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.c,$(@F)) $(LDFLAGS) BUILD = $(__BUILD) > $(@:.bin=.make.output) 2>&1 + BUILD_BFD = $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl + BUILD_ALL = $(BUILD) -fstack-protector-all -O2 -D_FORTIFY_SOURCE=2 -ldw -lelf -lnuma -lelf -lslang $(FLAGS_PERL_EMBED) $(FLAGS_PYTHON_EMBED) -DPACKAGE='"perf"' -lbfd -ldl -lz -llzma -lzstd -lcap __BUILDXX = $(CXX) $(CXXFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.cpp,$(@F)) $(LDFLAGS) BUILDXX = $(__BUILDXX) > $(@:.bin=.make.output) 2>&1 @@ -90,7 +99,7 @@ __BUILDXX = $(CXX) $(CXXFLAGS) -MD -Wall -Werror -o $@ $(patsubst %.bin,%.cpp,$( ############################### $(OUTPUT)test-all.bin: - $(BUILD) -fstack-protector-all -O2 -D_FORTIFY_SOURCE=2 -ldw -lelf -lnuma -lelf -lslang $(FLAGS_PERL_EMBED) $(FLAGS_PYTHON_EMBED) -DPACKAGE='"perf"' -lbfd -ldl -lz -llzma -lzstd -lcap + $(BUILD_ALL) || $(BUILD_ALL) -lopcodes -liberty $(OUTPUT)test-hello.bin: $(BUILD) @@ -234,13 +243,18 @@ $(OUTPUT)test-libpython.bin: $(BUILD) $(FLAGS_PYTHON_EMBED) $(OUTPUT)test-libbfd.bin: - $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl + $(BUILD_BFD) $(OUTPUT)test-libbfd-buildid.bin: - $(BUILD) -DPACKAGE='"perf"' -lbfd -ldl + $(BUILD_BFD) || $(BUILD_BFD) -liberty || $(BUILD_BFD) -liberty -lz $(OUTPUT)test-disassembler-four-args.bin: - $(BUILD) -DPACKAGE='"perf"' -lbfd -lopcodes + $(BUILD_BFD) -lopcodes || $(BUILD_BFD) -lopcodes -liberty || \ + $(BUILD_BFD) -lopcodes -liberty -lz + +$(OUTPUT)test-disassembler-init-styled.bin: + $(BUILD_BFD) -lopcodes || $(BUILD_BFD) -lopcodes -liberty || \ + $(BUILD_BFD) -lopcodes -liberty -lz $(OUTPUT)test-reallocarray.bin: $(BUILD) @@ -287,6 +301,24 @@ $(OUTPUT)test-bpf.bin: $(OUTPUT)test-libbpf.bin: $(BUILD) -lbpf +$(OUTPUT)test-libbpf-btf__load_from_kernel_by_id.bin: + $(BUILD) -lbpf + +$(OUTPUT)test-libbpf-bpf_prog_load.bin: + $(BUILD) -lbpf + +$(OUTPUT)test-libbpf-bpf_map_create.bin: + $(BUILD) -lbpf + +$(OUTPUT)test-libbpf-bpf_object__next_program.bin: + $(BUILD) -lbpf + +$(OUTPUT)test-libbpf-bpf_object__next_map.bin: + $(BUILD) -lbpf + +$(OUTPUT)test-libbpf-btf__raw_data.bin: + $(BUILD) -lbpf + $(OUTPUT)test-sdt.bin: $(BUILD) diff --git a/tools/build/feature/test-all.c b/tools/build/feature/test-all.c index 5ffafb967b..957c02c7b1 100644 --- a/tools/build/feature/test-all.c +++ b/tools/build/feature/test-all.c @@ -166,6 +166,10 @@ # include "test-disassembler-four-args.c" #undef main +#define main main_test_disassembler_init_styled +# include "test-disassembler-init-styled.c" +#undef main + #define main main_test_libzstd # include "test-libzstd.c" #undef main diff --git a/tools/build/feature/test-libcrypto.c b/tools/build/feature/test-libcrypto.c index a98174e056..bc34a5bbb5 100644 --- a/tools/build/feature/test-libcrypto.c +++ b/tools/build/feature/test-libcrypto.c @@ -1,16 +1,23 @@ // SPDX-License-Identifier: GPL-2.0 +#include #include #include int main(void) { - MD5_CTX context; + EVP_MD_CTX *mdctx; unsigned char md[MD5_DIGEST_LENGTH + SHA_DIGEST_LENGTH]; unsigned char dat[] = "12345"; + unsigned int digest_len; - MD5_Init(&context); - MD5_Update(&context, &dat[0], sizeof(dat)); - MD5_Final(&md[0], &context); + mdctx = EVP_MD_CTX_new(); + if (!mdctx) + return 0; + + EVP_DigestInit_ex(mdctx, EVP_md5(), NULL); + EVP_DigestUpdate(mdctx, &dat[0], sizeof(dat)); + EVP_DigestFinal_ex(mdctx, &md[0], &digest_len); + EVP_MD_CTX_free(mdctx); SHA1(&dat[0], sizeof(dat), &md[0]); diff --git a/tools/gpio/gpio-event-mon.c b/tools/gpio/gpio-event-mon.c index a2b233fdb5..6c122952c5 100644 --- a/tools/gpio/gpio-event-mon.c +++ b/tools/gpio/gpio-event-mon.c @@ -149,6 +149,7 @@ void print_usage(void) " -r Listen for rising edges\n" " -f Listen for falling edges\n" " -w Report the wall-clock time for events\n" + " -t Report the hardware timestamp for events\n" " -b Debounce the line with period n microseconds\n" " [-c ] Do loops (optional, infinite loop if not stated)\n" " -? This helptext\n" @@ -174,7 +175,7 @@ int main(int argc, char **argv) memset(&config, 0, sizeof(config)); config.flags = GPIO_V2_LINE_FLAG_INPUT; - while ((c = getopt(argc, argv, "c:n:o:b:dsrfw?")) != -1) { + while ((c = getopt(argc, argv, "c:n:o:b:dsrfwt?")) != -1) { switch (c) { case 'c': loops = strtoul(optarg, NULL, 10); @@ -208,6 +209,9 @@ int main(int argc, char **argv) case 'w': config.flags |= GPIO_V2_LINE_FLAG_EVENT_CLOCK_REALTIME; break; + case 't': + config.flags |= GPIO_V2_LINE_FLAG_EVENT_CLOCK_HTE; + break; case '?': print_usage(); return -1; diff --git a/tools/include/asm-generic/bitops/non-atomic.h b/tools/include/asm-generic/bitops/non-atomic.h index 7e10c4b50c..0c472a8334 100644 --- a/tools/include/asm-generic/bitops/non-atomic.h +++ b/tools/include/asm-generic/bitops/non-atomic.h @@ -2,10 +2,10 @@ #ifndef _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ #define _ASM_GENERIC_BITOPS_NON_ATOMIC_H_ -#include +#include /** - * __set_bit - Set a bit in memory + * ___set_bit - Set a bit in memory * @nr: the bit to set * @addr: the address to start counting from * @@ -13,7 +13,8 @@ * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __set_bit(int nr, volatile unsigned long *addr) +static __always_inline void +___set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -21,7 +22,8 @@ static inline void __set_bit(int nr, volatile unsigned long *addr) *p |= mask; } -static inline void __clear_bit(int nr, volatile unsigned long *addr) +static __always_inline void +___clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -30,7 +32,7 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) } /** - * __change_bit - Toggle a bit in memory + * ___change_bit - Toggle a bit in memory * @nr: the bit to change * @addr: the address to start counting from * @@ -38,7 +40,8 @@ static inline void __clear_bit(int nr, volatile unsigned long *addr) * If it's called on the same region of memory simultaneously, the effect * may be that only one operation succeeds. */ -static inline void __change_bit(int nr, volatile unsigned long *addr) +static __always_inline void +___change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -47,7 +50,7 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_set_bit - Set a bit and return its old value + * ___test_and_set_bit - Set a bit and return its old value * @nr: Bit to set * @addr: Address to count from * @@ -55,7 +58,8 @@ static inline void __change_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_set_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -66,7 +70,7 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) } /** - * __test_and_clear_bit - Clear a bit and return its old value + * ___test_and_clear_bit - Clear a bit and return its old value * @nr: Bit to clear * @addr: Address to count from * @@ -74,7 +78,8 @@ static inline int __test_and_set_bit(int nr, volatile unsigned long *addr) * If two examples of this operation race, one can appear to succeed * but actually fail. You must protect multiple accesses with a lock. */ -static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) +static __always_inline bool +___test_and_clear_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -85,8 +90,8 @@ static inline int __test_and_clear_bit(int nr, volatile unsigned long *addr) } /* WARNING: non atomic and it can be reordered! */ -static inline int __test_and_change_bit(int nr, - volatile unsigned long *addr) +static __always_inline bool +___test_and_change_bit(unsigned long nr, volatile unsigned long *addr) { unsigned long mask = BIT_MASK(nr); unsigned long *p = ((unsigned long *)addr) + BIT_WORD(nr); @@ -97,11 +102,12 @@ static inline int __test_and_change_bit(int nr, } /** - * test_bit - Determine whether a bit is set + * _test_bit - Determine whether a bit is set * @nr: bit number to test * @addr: Address to start counting from */ -static inline int test_bit(int nr, const volatile unsigned long *addr) +static __always_inline bool +_test_bit(unsigned long nr, const volatile unsigned long *addr) { return 1UL & (addr[BIT_WORD(nr)] >> (nr & (BITS_PER_LONG-1))); } diff --git a/tools/include/linux/bitmap.h b/tools/include/linux/bitmap.h index ea97804d04..65d0747c52 100644 --- a/tools/include/linux/bitmap.h +++ b/tools/include/linux/bitmap.h @@ -11,16 +11,16 @@ #define DECLARE_BITMAP(name,bits) \ unsigned long name[BITS_TO_LONGS(bits)] -int __bitmap_weight(const unsigned long *bitmap, int bits); +unsigned int __bitmap_weight(const unsigned long *bitmap, int bits); void __bitmap_or(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, int bits); -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits); -int __bitmap_equal(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits); +bool __bitmap_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits); void bitmap_clear(unsigned long *map, unsigned int start, int len); -int __bitmap_intersects(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits); +bool __bitmap_intersects(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits); #define BITMAP_FIRST_WORD_MASK(start) (~0UL << ((start) & (BITS_PER_LONG - 1))) #define BITMAP_LAST_WORD_MASK(nbits) (~0UL >> (-(nbits) & (BITS_PER_LONG - 1))) @@ -45,7 +45,7 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits) dst[nlongs - 1] = BITMAP_LAST_WORD_MASK(nbits); } -static inline int bitmap_empty(const unsigned long *src, unsigned nbits) +static inline bool bitmap_empty(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -53,7 +53,7 @@ static inline int bitmap_empty(const unsigned long *src, unsigned nbits) return find_first_bit(src, nbits) == nbits; } -static inline int bitmap_full(const unsigned long *src, unsigned int nbits) +static inline bool bitmap_full(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits)); @@ -61,7 +61,7 @@ static inline int bitmap_full(const unsigned long *src, unsigned int nbits) return find_first_zero_bit(src, nbits) == nbits; } -static inline int bitmap_weight(const unsigned long *src, unsigned int nbits) +static inline unsigned int bitmap_weight(const unsigned long *src, unsigned int nbits) { if (small_const_nbits(nbits)) return hweight_long(*src & BITMAP_LAST_WORD_MASK(nbits)); @@ -146,7 +146,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, * @src2: operand 2 * @nbits: size of bitmap */ -static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, +static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1, const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) @@ -162,8 +162,8 @@ static inline int bitmap_and(unsigned long *dst, const unsigned long *src1, #define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1) #define IS_ALIGNED(x, a) (((x) & ((typeof(x))(a) - 1)) == 0) -static inline int bitmap_equal(const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static inline bool bitmap_equal(const unsigned long *src1, + const unsigned long *src2, unsigned int nbits) { if (small_const_nbits(nbits)) return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits)); @@ -173,8 +173,9 @@ static inline int bitmap_equal(const unsigned long *src1, return __bitmap_equal(src1, src2, nbits); } -static inline int bitmap_intersects(const unsigned long *src1, - const unsigned long *src2, unsigned int nbits) +static inline bool bitmap_intersects(const unsigned long *src1, + const unsigned long *src2, + unsigned int nbits) { if (small_const_nbits(nbits)) return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0; diff --git a/tools/include/linux/bitops.h b/tools/include/linux/bitops.h index 5fca38fe1b..f18683b95e 100644 --- a/tools/include/linux/bitops.h +++ b/tools/include/linux/bitops.h @@ -25,6 +25,22 @@ extern unsigned int __sw_hweight16(unsigned int w); extern unsigned int __sw_hweight32(unsigned int w); extern unsigned long __sw_hweight64(__u64 w); +/* + * Defined here because those may be needed by architecture-specific static + * inlines. + */ + +#define bitop(op, nr, addr) \ + op(nr, addr) + +#define __set_bit(nr, addr) bitop(___set_bit, nr, addr) +#define __clear_bit(nr, addr) bitop(___clear_bit, nr, addr) +#define __change_bit(nr, addr) bitop(___change_bit, nr, addr) +#define __test_and_set_bit(nr, addr) bitop(___test_and_set_bit, nr, addr) +#define __test_and_clear_bit(nr, addr) bitop(___test_and_clear_bit, nr, addr) +#define __test_and_change_bit(nr, addr) bitop(___test_and_change_bit, nr, addr) +#define test_bit(nr, addr) bitop(_test_bit, nr, addr) + /* * Include this here because some architectures need generic_ffs/fls in * scope diff --git a/tools/include/linux/btf_ids.h b/tools/include/linux/btf_ids.h index 57890b357f..71e54b1e37 100644 --- a/tools/include/linux/btf_ids.h +++ b/tools/include/linux/btf_ids.h @@ -73,7 +73,7 @@ asm( \ __BTF_ID_LIST(name, local) \ extern u32 name[]; -#define BTF_ID_LIST_GLOBAL(name) \ +#define BTF_ID_LIST_GLOBAL(name, n) \ __BTF_ID_LIST(name, globl) /* The BTF_ID_LIST_SINGLE macro defines a BTF_ID_LIST with @@ -82,6 +82,9 @@ __BTF_ID_LIST(name, globl) #define BTF_ID_LIST_SINGLE(name, prefix, typename) \ BTF_ID_LIST(name) \ BTF_ID(prefix, typename) +#define BTF_ID_LIST_GLOBAL_SINGLE(name, prefix, typename) \ + BTF_ID_LIST_GLOBAL(name, 1) \ + BTF_ID(prefix, typename) /* * The BTF_ID_UNUSED macro defines 4 zero bytes. @@ -143,13 +146,14 @@ extern struct btf_id_set name; #else -#define BTF_ID_LIST(name) static u32 name[5]; +#define BTF_ID_LIST(name) static u32 __maybe_unused name[5]; #define BTF_ID(prefix, name) #define BTF_ID_UNUSED -#define BTF_ID_LIST_GLOBAL(name) u32 name[1]; -#define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 name[1]; -#define BTF_SET_START(name) static struct btf_id_set name = { 0 }; -#define BTF_SET_START_GLOBAL(name) static struct btf_id_set name = { 0 }; +#define BTF_ID_LIST_GLOBAL(name, n) u32 __maybe_unused name[n]; +#define BTF_ID_LIST_SINGLE(name, prefix, typename) static u32 __maybe_unused name[1]; +#define BTF_ID_LIST_GLOBAL_SINGLE(name, prefix, typename) u32 __maybe_unused name[1]; +#define BTF_SET_START(name) static struct btf_id_set __maybe_unused name = { 0 }; +#define BTF_SET_START_GLOBAL(name) static struct btf_id_set __maybe_unused name = { 0 }; #define BTF_SET_END(name) #endif /* CONFIG_DEBUG_INFO_BTF */ @@ -172,7 +176,10 @@ extern struct btf_id_set name; BTF_SOCK_TYPE(BTF_SOCK_TYPE_TCP_TW, tcp_timewait_sock) \ BTF_SOCK_TYPE(BTF_SOCK_TYPE_TCP6, tcp6_sock) \ BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP, udp_sock) \ - BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP6, udp6_sock) + BTF_SOCK_TYPE(BTF_SOCK_TYPE_UDP6, udp6_sock) \ + BTF_SOCK_TYPE(BTF_SOCK_TYPE_UNIX, unix_sock) \ + BTF_SOCK_TYPE(BTF_SOCK_TYPE_MPTCP, mptcp_sock) \ + BTF_SOCK_TYPE(BTF_SOCK_TYPE_SOCKET, socket) enum { #define BTF_SOCK_TYPE(name, str) name, @@ -184,4 +191,18 @@ MAX_BTF_SOCK_TYPE, extern u32 btf_sock_ids[]; #endif +#define BTF_TRACING_TYPE_xxx \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_TASK, task_struct) \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_FILE, file) \ + BTF_TRACING_TYPE(BTF_TRACING_TYPE_VMA, vm_area_struct) + +enum { +#define BTF_TRACING_TYPE(name, type) name, +BTF_TRACING_TYPE_xxx +#undef BTF_TRACING_TYPE +MAX_BTF_TRACING_TYPE, +}; + +extern u32 btf_tracing_ids[]; + #endif diff --git a/tools/include/linux/compiler_types.h b/tools/include/linux/compiler_types.h index 24ae3054f3..1bdd834bdd 100644 --- a/tools/include/linux/compiler_types.h +++ b/tools/include/linux/compiler_types.h @@ -36,4 +36,8 @@ #include #endif +#ifndef asm_volatile_goto +#define asm_volatile_goto(x...) asm goto(x) +#endif + #endif /* __LINUX_COMPILER_TYPES_H */ diff --git a/tools/include/linux/list.h b/tools/include/linux/list.h index b2fc48d547..a4dfb6a7cc 100644 --- a/tools/include/linux/list.h +++ b/tools/include/linux/list.h @@ -384,6 +384,17 @@ static inline void list_splice_tail_init(struct list_head *list, #define list_first_entry_or_null(ptr, type, member) \ (!list_empty(ptr) ? list_first_entry(ptr, type, member) : NULL) +/** + * list_last_entry_or_null - get the last element from a list + * @ptr: the list head to take the element from. + * @type: the type of the struct this is embedded in. + * @member: the name of the list_head within the struct. + * + * Note that if the list is empty, it returns NULL. + */ +#define list_last_entry_or_null(ptr, type, member) \ + (!list_empty(ptr) ? list_last_entry(ptr, type, member) : NULL) + /** * list_next_entry - get the next element in list * @pos: the type * to cursor diff --git a/tools/include/linux/objtool.h b/tools/include/linux/objtool.h index 586d35720f..62c54ffbee 100644 --- a/tools/include/linux/objtool.h +++ b/tools/include/linux/objtool.h @@ -32,13 +32,20 @@ struct unwind_hint { * * UNWIND_HINT_FUNC: Generate the unwind metadata of a callable function. * Useful for code which doesn't have an ELF function annotation. + * + * UNWIND_HINT_ENTRY: machine entry without stack, SYSCALL/SYSENTER etc. */ #define UNWIND_HINT_TYPE_CALL 0 #define UNWIND_HINT_TYPE_REGS 1 #define UNWIND_HINT_TYPE_REGS_PARTIAL 2 #define UNWIND_HINT_TYPE_FUNC 3 +#define UNWIND_HINT_TYPE_ENTRY 4 +#define UNWIND_HINT_TYPE_SAVE 5 +#define UNWIND_HINT_TYPE_RESTORE 6 -#ifdef CONFIG_STACK_VALIDATION +#ifdef CONFIG_OBJTOOL + +#include #ifndef __ASSEMBLY__ @@ -60,7 +67,7 @@ struct unwind_hint { * It should only be used in special cases where you're 100% sure it won't * affect the reliability of frame pointers and kernel stack traces. * - * For more information, see tools/objtool/Documentation/stack-validation.txt. + * For more information, see tools/objtool/Documentation/objtool.txt. */ #define STACK_FRAME_NON_STANDARD(func) \ static void __used __section(".discard.func_stack_frame_non_standard") \ @@ -122,7 +129,7 @@ struct unwind_hint { * the debuginfo as necessary. It will also warn if it sees any * inconsistencies. */ -.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 end=0 .Lunwind_hint_ip_\@: .pushsection .discard.unwind_hints /* struct unwind_hint */ @@ -137,10 +144,16 @@ struct unwind_hint { .macro STACK_FRAME_NON_STANDARD func:req .pushsection .discard.func_stack_frame_non_standard, "aw" - .long \func - . + _ASM_PTR \func .popsection .endm +.macro STACK_FRAME_NON_STANDARD_FP func:req +#ifdef CONFIG_FRAME_POINTER + STACK_FRAME_NON_STANDARD \func +#endif +.endm + .macro ANNOTATE_NOENDBR .Lhere_\@: .pushsection .discard.noendbr @@ -157,7 +170,7 @@ struct unwind_hint { #endif /* __ASSEMBLY__ */ -#else /* !CONFIG_STACK_VALIDATION */ +#else /* !CONFIG_OBJTOOL */ #ifndef __ASSEMBLY__ @@ -169,7 +182,7 @@ struct unwind_hint { #define ASM_REACHABLE #else #define ANNOTATE_INTRA_FUNCTION_CALL -.macro UNWIND_HINT sp_reg:req sp_offset=0 type:req end=0 +.macro UNWIND_HINT type:req sp_reg=0 sp_offset=0 end=0 .endm .macro STACK_FRAME_NON_STANDARD func:req .endm @@ -179,6 +192,6 @@ struct unwind_hint { .endm #endif -#endif /* CONFIG_STACK_VALIDATION */ +#endif /* CONFIG_OBJTOOL */ #endif /* _LINUX_OBJTOOL_H */ diff --git a/tools/include/linux/sched/mm.h b/tools/include/linux/sched/mm.h index c8d9f19c1f..967294b8ed 100644 --- a/tools/include/linux/sched/mm.h +++ b/tools/include/linux/sched/mm.h @@ -1,4 +1,6 @@ #ifndef _TOOLS_PERF_LINUX_SCHED_MM_H #define _TOOLS_PERF_LINUX_SCHED_MM_H +#define might_alloc(gfp) do { } while (0) + #endif /* _TOOLS_PERF_LINUX_SCHED_MM_H */ diff --git a/tools/include/nolibc/nolibc.h b/tools/include/nolibc/nolibc.h index c1c285fe49..b2bc48d3cf 100644 --- a/tools/include/nolibc/nolibc.h +++ b/tools/include/nolibc/nolibc.h @@ -57,22 +57,32 @@ * having to specify anything. * * Finally some very common libc-level functions are provided. It is the case - * for a few functions usually found in string.h, ctype.h, or stdlib.h. Nothing - * is currently provided regarding stdio emulation. + * for a few functions usually found in string.h, ctype.h, or stdlib.h. * - * The macro NOLIBC is always defined, so that it is possible for a program to - * check this macro to know if it is being built against and decide to disable - * some features or simply not to include some standard libc files. - * - * Ideally this file should be split in multiple files for easier long term - * maintenance, but provided as a single file as it is now, it's quite - * convenient to use. Maybe some variations involving a set of includes at the - * top could work. + * The nolibc.h file is only a convenient entry point which includes all other + * files. It also defines the NOLIBC macro, so that it is possible for a + * program to check this macro to know if it is being built against and decide + * to disable some features or simply not to include some standard libc files. * * A simple static executable may be built this way : * $ gcc -fno-asynchronous-unwind-tables -fno-ident -s -Os -nostdlib \ * -static -include nolibc.h -o hello hello.c -lgcc * + * Simple programs meant to be reasonably portable to various libc and using + * only a few common includes, may also be built by simply making the include + * path point to the nolibc directory: + * $ gcc -fno-asynchronous-unwind-tables -fno-ident -s -Os -nostdlib \ + * -I../nolibc -o hello hello.c -lgcc + * + * The available standard (but limited) include files are: + * ctype.h, errno.h, signal.h, stdio.h, stdlib.h, string.h, time.h + * + * In addition, the following ones are expected to be provided by the compiler: + * float.h, stdarg.h, stddef.h + * + * The following ones which are part to the C standard are not provided: + * assert.h, locale.h, math.h, setjmp.h, limits.h + * * A very useful calling convention table may be found here : * http://man7.org/linux/man-pages/man2/syscall.2.html * @@ -80,2502 +90,22 @@ * https://w3challs.com/syscalls/ * */ +#ifndef _NOLIBC_H +#define _NOLIBC_H -#include -#include -#include -#include -#include -#include +#include "std.h" +#include "arch.h" +#include "types.h" +#include "sys.h" +#include "ctype.h" +#include "signal.h" +#include "stdio.h" +#include "stdlib.h" +#include "string.h" +#include "time.h" +#include "unistd.h" +/* Used by programs to avoid std includes */ #define NOLIBC -/* this way it will be removed if unused */ -static int errno; - -#ifndef NOLIBC_IGNORE_ERRNO -#define SET_ERRNO(v) do { errno = (v); } while (0) -#else -#define SET_ERRNO(v) do { } while (0) -#endif - -/* errno codes all ensure that they will not conflict with a valid pointer - * because they all correspond to the highest addressable memory page. - */ -#define MAX_ERRNO 4095 - -/* Declare a few quite common macros and types that usually are in stdlib.h, - * stdint.h, ctype.h, unistd.h and a few other common locations. - */ - -#define NULL ((void *)0) - -/* stdint types */ -typedef unsigned char uint8_t; -typedef signed char int8_t; -typedef unsigned short uint16_t; -typedef signed short int16_t; -typedef unsigned int uint32_t; -typedef signed int int32_t; -typedef unsigned long long uint64_t; -typedef signed long long int64_t; -typedef unsigned long size_t; -typedef signed long ssize_t; -typedef unsigned long uintptr_t; -typedef signed long intptr_t; -typedef signed long ptrdiff_t; - -/* for stat() */ -typedef unsigned int dev_t; -typedef unsigned long ino_t; -typedef unsigned int mode_t; -typedef signed int pid_t; -typedef unsigned int uid_t; -typedef unsigned int gid_t; -typedef unsigned long nlink_t; -typedef signed long off_t; -typedef signed long blksize_t; -typedef signed long blkcnt_t; -typedef signed long time_t; - -/* for poll() */ -struct pollfd { - int fd; - short int events; - short int revents; -}; - -/* for getdents64() */ -struct linux_dirent64 { - uint64_t d_ino; - int64_t d_off; - unsigned short d_reclen; - unsigned char d_type; - char d_name[]; -}; - -/* commonly an fd_set represents 256 FDs */ -#define FD_SETSIZE 256 -typedef struct { uint32_t fd32[FD_SETSIZE/32]; } fd_set; - -/* needed by wait4() */ -struct rusage { - struct timeval ru_utime; - struct timeval ru_stime; - long ru_maxrss; - long ru_ixrss; - long ru_idrss; - long ru_isrss; - long ru_minflt; - long ru_majflt; - long ru_nswap; - long ru_inblock; - long ru_oublock; - long ru_msgsnd; - long ru_msgrcv; - long ru_nsignals; - long ru_nvcsw; - long ru_nivcsw; -}; - -/* stat flags (WARNING, octal here) */ -#define S_IFDIR 0040000 -#define S_IFCHR 0020000 -#define S_IFBLK 0060000 -#define S_IFREG 0100000 -#define S_IFIFO 0010000 -#define S_IFLNK 0120000 -#define S_IFSOCK 0140000 -#define S_IFMT 0170000 - -#define S_ISDIR(mode) (((mode) & S_IFDIR) == S_IFDIR) -#define S_ISCHR(mode) (((mode) & S_IFCHR) == S_IFCHR) -#define S_ISBLK(mode) (((mode) & S_IFBLK) == S_IFBLK) -#define S_ISREG(mode) (((mode) & S_IFREG) == S_IFREG) -#define S_ISFIFO(mode) (((mode) & S_IFIFO) == S_IFIFO) -#define S_ISLNK(mode) (((mode) & S_IFLNK) == S_IFLNK) -#define S_ISSOCK(mode) (((mode) & S_IFSOCK) == S_IFSOCK) - -#define DT_UNKNOWN 0 -#define DT_FIFO 1 -#define DT_CHR 2 -#define DT_DIR 4 -#define DT_BLK 6 -#define DT_REG 8 -#define DT_LNK 10 -#define DT_SOCK 12 - -/* all the *at functions */ -#ifndef AT_FDCWD -#define AT_FDCWD -100 -#endif - -/* lseek */ -#define SEEK_SET 0 -#define SEEK_CUR 1 -#define SEEK_END 2 - -/* reboot */ -#define LINUX_REBOOT_MAGIC1 0xfee1dead -#define LINUX_REBOOT_MAGIC2 0x28121969 -#define LINUX_REBOOT_CMD_HALT 0xcdef0123 -#define LINUX_REBOOT_CMD_POWER_OFF 0x4321fedc -#define LINUX_REBOOT_CMD_RESTART 0x01234567 -#define LINUX_REBOOT_CMD_SW_SUSPEND 0xd000fce2 - - -/* The format of the struct as returned by the libc to the application, which - * significantly differs from the format returned by the stat() syscall flavours. - */ -struct stat { - dev_t st_dev; /* ID of device containing file */ - ino_t st_ino; /* inode number */ - mode_t st_mode; /* protection */ - nlink_t st_nlink; /* number of hard links */ - uid_t st_uid; /* user ID of owner */ - gid_t st_gid; /* group ID of owner */ - dev_t st_rdev; /* device ID (if special file) */ - off_t st_size; /* total size, in bytes */ - blksize_t st_blksize; /* blocksize for file system I/O */ - blkcnt_t st_blocks; /* number of 512B blocks allocated */ - time_t st_atime; /* time of last access */ - time_t st_mtime; /* time of last modification */ - time_t st_ctime; /* time of last status change */ -}; - -#define WEXITSTATUS(status) (((status) & 0xff00) >> 8) -#define WIFEXITED(status) (((status) & 0x7f) == 0) - -/* for SIGCHLD */ -#include - -/* Below comes the architecture-specific code. For each architecture, we have - * the syscall declarations and the _start code definition. This is the only - * global part. On all architectures the kernel puts everything in the stack - * before jumping to _start just above us, without any return address (_start - * is not a function but an entry pint). So at the stack pointer we find argc. - * Then argv[] begins, and ends at the first NULL. Then we have envp which - * starts and ends with a NULL as well. So envp=argv+argc+1. - */ - -#if defined(__x86_64__) -/* Syscalls for x86_64 : - * - registers are 64-bit - * - syscall number is passed in rax - * - arguments are in rdi, rsi, rdx, r10, r8, r9 respectively - * - the system call is performed by calling the syscall instruction - * - syscall return comes in rax - * - rcx and r11 are clobbered, others are preserved. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - the syscall number is always specified last in order to allow to force - * some registers before (gcc refuses a %-register at the last position). - * - see also x86-64 ABI section A.2 AMD64 Linux Kernel Conventions, A.2.1 - * Calling Conventions. - * - * Link x86-64 ABI: https://gitlab.com/x86-psABIs/x86-64-ABI/-/wikis/x86-64-psABI - * - */ - -#define my_syscall0(num) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - register long _arg1 asm("rdi") = (long)(arg1); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - register long _arg1 asm("rdi") = (long)(arg1); \ - register long _arg2 asm("rsi") = (long)(arg2); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - register long _arg1 asm("rdi") = (long)(arg1); \ - register long _arg2 asm("rsi") = (long)(arg2); \ - register long _arg3 asm("rdx") = (long)(arg3); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - register long _arg1 asm("rdi") = (long)(arg1); \ - register long _arg2 asm("rsi") = (long)(arg2); \ - register long _arg3 asm("rdx") = (long)(arg3); \ - register long _arg4 asm("r10") = (long)(arg4); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - register long _arg1 asm("rdi") = (long)(arg1); \ - register long _arg2 asm("rsi") = (long)(arg2); \ - register long _arg3 asm("rdx") = (long)(arg3); \ - register long _arg4 asm("r10") = (long)(arg4); \ - register long _arg5 asm("r8") = (long)(arg5); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - long _ret; \ - register long _num asm("rax") = (num); \ - register long _arg1 asm("rdi") = (long)(arg1); \ - register long _arg2 asm("rsi") = (long)(arg2); \ - register long _arg3 asm("rdx") = (long)(arg3); \ - register long _arg4 asm("r10") = (long)(arg4); \ - register long _arg5 asm("r8") = (long)(arg5); \ - register long _arg6 asm("r9") = (long)(arg6); \ - \ - asm volatile ( \ - "syscall\n" \ - : "=a"(_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_arg6), "0"(_num) \ - : "rcx", "r11", "memory", "cc" \ - ); \ - _ret; \ -}) - -/* startup code */ -/* - * x86-64 System V ABI mandates: - * 1) %rsp must be 16-byte aligned right before the function call. - * 2) The deepest stack frame should be zero (the %rbp). - * - */ -asm(".section .text\n" - ".global _start\n" - "_start:\n" - "pop %rdi\n" // argc (first arg, %rdi) - "mov %rsp, %rsi\n" // argv[] (second arg, %rsi) - "lea 8(%rsi,%rdi,8),%rdx\n" // then a NULL then envp (third arg, %rdx) - "xor %ebp, %ebp\n" // zero the stack frame - "and $-16, %rsp\n" // x86 ABI : esp must be 16-byte aligned before call - "call main\n" // main() returns the status code, we'll exit with it. - "mov %eax, %edi\n" // retrieve exit code (32 bit) - "mov $60, %eax\n" // NR_exit == 60 - "syscall\n" // really exit - "hlt\n" // ensure it does not return - ""); - -/* fcntl / open */ -#define O_RDONLY 0 -#define O_WRONLY 1 -#define O_RDWR 2 -#define O_CREAT 0x40 -#define O_EXCL 0x80 -#define O_NOCTTY 0x100 -#define O_TRUNC 0x200 -#define O_APPEND 0x400 -#define O_NONBLOCK 0x800 -#define O_DIRECTORY 0x10000 - -/* The struct returned by the stat() syscall, equivalent to stat64(). The - * syscall returns 116 bytes and stops in the middle of __unused. - */ -struct sys_stat_struct { - unsigned long st_dev; - unsigned long st_ino; - unsigned long st_nlink; - unsigned int st_mode; - unsigned int st_uid; - - unsigned int st_gid; - unsigned int __pad0; - unsigned long st_rdev; - long st_size; - long st_blksize; - - long st_blocks; - unsigned long st_atime; - unsigned long st_atime_nsec; - unsigned long st_mtime; - - unsigned long st_mtime_nsec; - unsigned long st_ctime; - unsigned long st_ctime_nsec; - long __unused[3]; -}; - -#elif defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) -/* Syscalls for i386 : - * - mostly similar to x86_64 - * - registers are 32-bit - * - syscall number is passed in eax - * - arguments are in ebx, ecx, edx, esi, edi, ebp respectively - * - all registers are preserved (except eax of course) - * - the system call is performed by calling int $0x80 - * - syscall return comes in eax - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - the syscall number is always specified last in order to allow to force - * some registers before (gcc refuses a %-register at the last position). - * - * Also, i386 supports the old_select syscall if newselect is not available - */ -#define __ARCH_WANT_SYS_OLD_SELECT - -#define my_syscall0(num) \ -({ \ - long _ret; \ - register long _num asm("eax") = (num); \ - \ - asm volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - long _ret; \ - register long _num asm("eax") = (num); \ - register long _arg1 asm("ebx") = (long)(arg1); \ - \ - asm volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - long _ret; \ - register long _num asm("eax") = (num); \ - register long _arg1 asm("ebx") = (long)(arg1); \ - register long _arg2 asm("ecx") = (long)(arg2); \ - \ - asm volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - long _ret; \ - register long _num asm("eax") = (num); \ - register long _arg1 asm("ebx") = (long)(arg1); \ - register long _arg2 asm("ecx") = (long)(arg2); \ - register long _arg3 asm("edx") = (long)(arg3); \ - \ - asm volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - long _ret; \ - register long _num asm("eax") = (num); \ - register long _arg1 asm("ebx") = (long)(arg1); \ - register long _arg2 asm("ecx") = (long)(arg2); \ - register long _arg3 asm("edx") = (long)(arg3); \ - register long _arg4 asm("esi") = (long)(arg4); \ - \ - asm volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - long _ret; \ - register long _num asm("eax") = (num); \ - register long _arg1 asm("ebx") = (long)(arg1); \ - register long _arg2 asm("ecx") = (long)(arg2); \ - register long _arg3 asm("edx") = (long)(arg3); \ - register long _arg4 asm("esi") = (long)(arg4); \ - register long _arg5 asm("edi") = (long)(arg5); \ - \ - asm volatile ( \ - "int $0x80\n" \ - : "=a" (_ret) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "0"(_num) \ - : "memory", "cc" \ - ); \ - _ret; \ -}) - -/* startup code */ -/* - * i386 System V ABI mandates: - * 1) last pushed argument must be 16-byte aligned. - * 2) The deepest stack frame should be set to zero - * - */ -asm(".section .text\n" - ".global _start\n" - "_start:\n" - "pop %eax\n" // argc (first arg, %eax) - "mov %esp, %ebx\n" // argv[] (second arg, %ebx) - "lea 4(%ebx,%eax,4),%ecx\n" // then a NULL then envp (third arg, %ecx) - "xor %ebp, %ebp\n" // zero the stack frame - "and $-16, %esp\n" // x86 ABI : esp must be 16-byte aligned before - "sub $4, %esp\n" // the call instruction (args are aligned) - "push %ecx\n" // push all registers on the stack so that we - "push %ebx\n" // support both regparm and plain stack modes - "push %eax\n" - "call main\n" // main() returns the status code in %eax - "mov %eax, %ebx\n" // retrieve exit code (32-bit int) - "movl $1, %eax\n" // NR_exit == 1 - "int $0x80\n" // exit now - "hlt\n" // ensure it does not - ""); - -/* fcntl / open */ -#define O_RDONLY 0 -#define O_WRONLY 1 -#define O_RDWR 2 -#define O_CREAT 0x40 -#define O_EXCL 0x80 -#define O_NOCTTY 0x100 -#define O_TRUNC 0x200 -#define O_APPEND 0x400 -#define O_NONBLOCK 0x800 -#define O_DIRECTORY 0x10000 - -/* The struct returned by the stat() syscall, 32-bit only, the syscall returns - * exactly 56 bytes (stops before the unused array). - */ -struct sys_stat_struct { - unsigned long st_dev; - unsigned long st_ino; - unsigned short st_mode; - unsigned short st_nlink; - unsigned short st_uid; - unsigned short st_gid; - - unsigned long st_rdev; - unsigned long st_size; - unsigned long st_blksize; - unsigned long st_blocks; - - unsigned long st_atime; - unsigned long st_atime_nsec; - unsigned long st_mtime; - unsigned long st_mtime_nsec; - - unsigned long st_ctime; - unsigned long st_ctime_nsec; - unsigned long __unused[2]; -}; - -#elif defined(__ARM_EABI__) -/* Syscalls for ARM in ARM or Thumb modes : - * - registers are 32-bit - * - stack is 8-byte aligned - * ( http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka4127.html) - * - syscall number is passed in r7 - * - arguments are in r0, r1, r2, r3, r4, r5 - * - the system call is performed by calling svc #0 - * - syscall return comes in r0. - * - only lr is clobbered. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - the syscall number is always specified last in order to allow to force - * some registers before (gcc refuses a %-register at the last position). - * - * Also, ARM supports the old_select syscall if newselect is not available - */ -#define __ARCH_WANT_SYS_OLD_SELECT - -#define my_syscall0(num) \ -({ \ - register long _num asm("r7") = (num); \ - register long _arg1 asm("r0"); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - register long _num asm("r7") = (num); \ - register long _arg1 asm("r0") = (long)(arg1); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - register long _num asm("r7") = (num); \ - register long _arg1 asm("r0") = (long)(arg1); \ - register long _arg2 asm("r1") = (long)(arg2); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - register long _num asm("r7") = (num); \ - register long _arg1 asm("r0") = (long)(arg1); \ - register long _arg2 asm("r1") = (long)(arg2); \ - register long _arg3 asm("r2") = (long)(arg3); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - register long _num asm("r7") = (num); \ - register long _arg1 asm("r0") = (long)(arg1); \ - register long _arg2 asm("r1") = (long)(arg2); \ - register long _arg3 asm("r2") = (long)(arg3); \ - register long _arg4 asm("r3") = (long)(arg4); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - register long _num asm("r7") = (num); \ - register long _arg1 asm("r0") = (long)(arg1); \ - register long _arg2 asm("r1") = (long)(arg2); \ - register long _arg3 asm("r2") = (long)(arg3); \ - register long _arg4 asm("r3") = (long)(arg4); \ - register long _arg5 asm("r4") = (long)(arg5); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r" (_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_num) \ - : "memory", "cc", "lr" \ - ); \ - _arg1; \ -}) - -/* startup code */ -asm(".section .text\n" - ".global _start\n" - "_start:\n" -#if defined(__THUMBEB__) || defined(__THUMBEL__) - /* We enter here in 32-bit mode but if some previous functions were in - * 16-bit mode, the assembler cannot know, so we need to tell it we're in - * 32-bit now, then switch to 16-bit (is there a better way to do it than - * adding 1 by hand ?) and tell the asm we're now in 16-bit mode so that - * it generates correct instructions. Note that we do not support thumb1. - */ - ".code 32\n" - "add r0, pc, #1\n" - "bx r0\n" - ".code 16\n" -#endif - "pop {%r0}\n" // argc was in the stack - "mov %r1, %sp\n" // argv = sp - "add %r2, %r1, %r0, lsl #2\n" // envp = argv + 4*argc ... - "add %r2, %r2, $4\n" // ... + 4 - "and %r3, %r1, $-8\n" // AAPCS : sp must be 8-byte aligned in the - "mov %sp, %r3\n" // callee, an bl doesn't push (lr=pc) - "bl main\n" // main() returns the status code, we'll exit with it. - "movs r7, $1\n" // NR_exit == 1 - "svc $0x00\n" - ""); - -/* fcntl / open */ -#define O_RDONLY 0 -#define O_WRONLY 1 -#define O_RDWR 2 -#define O_CREAT 0x40 -#define O_EXCL 0x80 -#define O_NOCTTY 0x100 -#define O_TRUNC 0x200 -#define O_APPEND 0x400 -#define O_NONBLOCK 0x800 -#define O_DIRECTORY 0x4000 - -/* The struct returned by the stat() syscall, 32-bit only, the syscall returns - * exactly 56 bytes (stops before the unused array). In big endian, the format - * differs as devices are returned as short only. - */ -struct sys_stat_struct { -#if defined(__ARMEB__) - unsigned short st_dev; - unsigned short __pad1; -#else - unsigned long st_dev; -#endif - unsigned long st_ino; - unsigned short st_mode; - unsigned short st_nlink; - unsigned short st_uid; - unsigned short st_gid; -#if defined(__ARMEB__) - unsigned short st_rdev; - unsigned short __pad2; -#else - unsigned long st_rdev; -#endif - unsigned long st_size; - unsigned long st_blksize; - unsigned long st_blocks; - unsigned long st_atime; - unsigned long st_atime_nsec; - unsigned long st_mtime; - unsigned long st_mtime_nsec; - unsigned long st_ctime; - unsigned long st_ctime_nsec; - unsigned long __unused[2]; -}; - -#elif defined(__aarch64__) -/* Syscalls for AARCH64 : - * - registers are 64-bit - * - stack is 16-byte aligned - * - syscall number is passed in x8 - * - arguments are in x0, x1, x2, x3, x4, x5 - * - the system call is performed by calling svc 0 - * - syscall return comes in x0. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - * - * On aarch64, select() is not implemented so we have to use pselect6(). - */ -#define __ARCH_WANT_SYS_PSELECT6 - -#define my_syscall0(num) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0"); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0") = (long)(arg1); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0") = (long)(arg1); \ - register long _arg2 asm("x1") = (long)(arg2); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0") = (long)(arg1); \ - register long _arg2 asm("x1") = (long)(arg2); \ - register long _arg3 asm("x2") = (long)(arg3); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0") = (long)(arg1); \ - register long _arg2 asm("x1") = (long)(arg2); \ - register long _arg3 asm("x2") = (long)(arg3); \ - register long _arg4 asm("x3") = (long)(arg4); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r"(_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0") = (long)(arg1); \ - register long _arg2 asm("x1") = (long)(arg2); \ - register long _arg3 asm("x2") = (long)(arg3); \ - register long _arg4 asm("x3") = (long)(arg4); \ - register long _arg5 asm("x4") = (long)(arg5); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r" (_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - register long _num asm("x8") = (num); \ - register long _arg1 asm("x0") = (long)(arg1); \ - register long _arg2 asm("x1") = (long)(arg2); \ - register long _arg3 asm("x2") = (long)(arg3); \ - register long _arg4 asm("x3") = (long)(arg4); \ - register long _arg5 asm("x4") = (long)(arg5); \ - register long _arg6 asm("x5") = (long)(arg6); \ - \ - asm volatile ( \ - "svc #0\n" \ - : "=r" (_arg1) \ - : "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_arg6), "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -/* startup code */ -asm(".section .text\n" - ".global _start\n" - "_start:\n" - "ldr x0, [sp]\n" // argc (x0) was in the stack - "add x1, sp, 8\n" // argv (x1) = sp - "lsl x2, x0, 3\n" // envp (x2) = 8*argc ... - "add x2, x2, 8\n" // + 8 (skip null) - "add x2, x2, x1\n" // + argv - "and sp, x1, -16\n" // sp must be 16-byte aligned in the callee - "bl main\n" // main() returns the status code, we'll exit with it. - "mov x8, 93\n" // NR_exit == 93 - "svc #0\n" - ""); - -/* fcntl / open */ -#define O_RDONLY 0 -#define O_WRONLY 1 -#define O_RDWR 2 -#define O_CREAT 0x40 -#define O_EXCL 0x80 -#define O_NOCTTY 0x100 -#define O_TRUNC 0x200 -#define O_APPEND 0x400 -#define O_NONBLOCK 0x800 -#define O_DIRECTORY 0x4000 - -/* The struct returned by the newfstatat() syscall. Differs slightly from the - * x86_64's stat one by field ordering, so be careful. - */ -struct sys_stat_struct { - unsigned long st_dev; - unsigned long st_ino; - unsigned int st_mode; - unsigned int st_nlink; - unsigned int st_uid; - unsigned int st_gid; - - unsigned long st_rdev; - unsigned long __pad1; - long st_size; - int st_blksize; - int __pad2; - - long st_blocks; - long st_atime; - unsigned long st_atime_nsec; - long st_mtime; - - unsigned long st_mtime_nsec; - long st_ctime; - unsigned long st_ctime_nsec; - unsigned int __unused[2]; -}; - -#elif defined(__mips__) && defined(_ABIO32) -/* Syscalls for MIPS ABI O32 : - * - WARNING! there's always a delayed slot! - * - WARNING again, the syntax is different, registers take a '$' and numbers - * do not. - * - registers are 32-bit - * - stack is 8-byte aligned - * - syscall number is passed in v0 (starts at 0xfa0). - * - arguments are in a0, a1, a2, a3, then the stack. The caller needs to - * leave some room in the stack for the callee to save a0..a3 if needed. - * - Many registers are clobbered, in fact only a0..a2 and s0..s8 are - * preserved. See: https://www.linux-mips.org/wiki/Syscall as well as - * scall32-o32.S in the kernel sources. - * - the system call is performed by calling "syscall" - * - syscall return comes in v0, and register a3 needs to be checked to know - * if an error occurred, in which case errno is in v0. - * - the arguments are cast to long and assigned into the target registers - * which are then simply passed as registers to the asm code, so that we - * don't have to experience issues with register constraints. - */ - -#define my_syscall0(num) \ -({ \ - register long _num asm("v0") = (num); \ - register long _arg4 asm("a3"); \ - \ - asm volatile ( \ - "addiu $sp, $sp, -32\n" \ - "syscall\n" \ - "addiu $sp, $sp, 32\n" \ - : "=r"(_num), "=r"(_arg4) \ - : "r"(_num) \ - : "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \ - ); \ - _arg4 ? -_num : _num; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - register long _num asm("v0") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg4 asm("a3"); \ - \ - asm volatile ( \ - "addiu $sp, $sp, -32\n" \ - "syscall\n" \ - "addiu $sp, $sp, 32\n" \ - : "=r"(_num), "=r"(_arg4) \ - : "0"(_num), \ - "r"(_arg1) \ - : "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \ - ); \ - _arg4 ? -_num : _num; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - register long _num asm("v0") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg4 asm("a3"); \ - \ - asm volatile ( \ - "addiu $sp, $sp, -32\n" \ - "syscall\n" \ - "addiu $sp, $sp, 32\n" \ - : "=r"(_num), "=r"(_arg4) \ - : "0"(_num), \ - "r"(_arg1), "r"(_arg2) \ - : "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \ - ); \ - _arg4 ? -_num : _num; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - register long _num asm("v0") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - register long _arg4 asm("a3"); \ - \ - asm volatile ( \ - "addiu $sp, $sp, -32\n" \ - "syscall\n" \ - "addiu $sp, $sp, 32\n" \ - : "=r"(_num), "=r"(_arg4) \ - : "0"(_num), \ - "r"(_arg1), "r"(_arg2), "r"(_arg3) \ - : "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \ - ); \ - _arg4 ? -_num : _num; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - register long _num asm("v0") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - register long _arg4 asm("a3") = (long)(arg4); \ - \ - asm volatile ( \ - "addiu $sp, $sp, -32\n" \ - "syscall\n" \ - "addiu $sp, $sp, 32\n" \ - : "=r" (_num), "=r"(_arg4) \ - : "0"(_num), \ - "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4) \ - : "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \ - ); \ - _arg4 ? -_num : _num; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - register long _num asm("v0") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - register long _arg4 asm("a3") = (long)(arg4); \ - register long _arg5 = (long)(arg5); \ - \ - asm volatile ( \ - "addiu $sp, $sp, -32\n" \ - "sw %7, 16($sp)\n" \ - "syscall\n " \ - "addiu $sp, $sp, 32\n" \ - : "=r" (_num), "=r"(_arg4) \ - : "0"(_num), \ - "r"(_arg1), "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5) \ - : "memory", "cc", "at", "v1", "hi", "lo", \ - "t0", "t1", "t2", "t3", "t4", "t5", "t6", "t7", "t8", "t9" \ - ); \ - _arg4 ? -_num : _num; \ -}) - -/* startup code, note that it's called __start on MIPS */ -asm(".section .text\n" - ".set nomips16\n" - ".global __start\n" - ".set noreorder\n" - ".option pic0\n" - ".ent __start\n" - "__start:\n" - "lw $a0,($sp)\n" // argc was in the stack - "addiu $a1, $sp, 4\n" // argv = sp + 4 - "sll $a2, $a0, 2\n" // a2 = argc * 4 - "add $a2, $a2, $a1\n" // envp = argv + 4*argc ... - "addiu $a2, $a2, 4\n" // ... + 4 - "li $t0, -8\n" - "and $sp, $sp, $t0\n" // sp must be 8-byte aligned - "addiu $sp,$sp,-16\n" // the callee expects to save a0..a3 there! - "jal main\n" // main() returns the status code, we'll exit with it. - "nop\n" // delayed slot - "move $a0, $v0\n" // retrieve 32-bit exit code from v0 - "li $v0, 4001\n" // NR_exit == 4001 - "syscall\n" - ".end __start\n" - ""); - -/* fcntl / open */ -#define O_RDONLY 0 -#define O_WRONLY 1 -#define O_RDWR 2 -#define O_APPEND 0x0008 -#define O_NONBLOCK 0x0080 -#define O_CREAT 0x0100 -#define O_TRUNC 0x0200 -#define O_EXCL 0x0400 -#define O_NOCTTY 0x0800 -#define O_DIRECTORY 0x10000 - -/* The struct returned by the stat() syscall. 88 bytes are returned by the - * syscall. - */ -struct sys_stat_struct { - unsigned int st_dev; - long st_pad1[3]; - unsigned long st_ino; - unsigned int st_mode; - unsigned int st_nlink; - unsigned int st_uid; - unsigned int st_gid; - unsigned int st_rdev; - long st_pad2[2]; - long st_size; - long st_pad3; - long st_atime; - long st_atime_nsec; - long st_mtime; - long st_mtime_nsec; - long st_ctime; - long st_ctime_nsec; - long st_blksize; - long st_blocks; - long st_pad4[14]; -}; - -#elif defined(__riscv) - -#if __riscv_xlen == 64 -#define PTRLOG "3" -#define SZREG "8" -#elif __riscv_xlen == 32 -#define PTRLOG "2" -#define SZREG "4" -#endif - -/* Syscalls for RISCV : - * - stack is 16-byte aligned - * - syscall number is passed in a7 - * - arguments are in a0, a1, a2, a3, a4, a5 - * - the system call is performed by calling ecall - * - syscall return comes in a0 - * - the arguments are cast to long and assigned into the target - * registers which are then simply passed as registers to the asm code, - * so that we don't have to experience issues with register constraints. - */ - -#define my_syscall0(num) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0"); \ - \ - asm volatile ( \ - "ecall\n\t" \ - : "=r"(_arg1) \ - : "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall1(num, arg1) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - \ - asm volatile ( \ - "ecall\n" \ - : "+r"(_arg1) \ - : "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall2(num, arg1, arg2) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - \ - asm volatile ( \ - "ecall\n" \ - : "+r"(_arg1) \ - : "r"(_arg2), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall3(num, arg1, arg2, arg3) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - \ - asm volatile ( \ - "ecall\n\t" \ - : "+r"(_arg1) \ - : "r"(_arg2), "r"(_arg3), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall4(num, arg1, arg2, arg3, arg4) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - register long _arg4 asm("a3") = (long)(arg4); \ - \ - asm volatile ( \ - "ecall\n" \ - : "+r"(_arg1) \ - : "r"(_arg2), "r"(_arg3), "r"(_arg4), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall5(num, arg1, arg2, arg3, arg4, arg5) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - register long _arg4 asm("a3") = (long)(arg4); \ - register long _arg5 asm("a4") = (long)(arg5); \ - \ - asm volatile ( \ - "ecall\n" \ - : "+r"(_arg1) \ - : "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -#define my_syscall6(num, arg1, arg2, arg3, arg4, arg5, arg6) \ -({ \ - register long _num asm("a7") = (num); \ - register long _arg1 asm("a0") = (long)(arg1); \ - register long _arg2 asm("a1") = (long)(arg2); \ - register long _arg3 asm("a2") = (long)(arg3); \ - register long _arg4 asm("a3") = (long)(arg4); \ - register long _arg5 asm("a4") = (long)(arg5); \ - register long _arg6 asm("a5") = (long)(arg6); \ - \ - asm volatile ( \ - "ecall\n" \ - : "+r"(_arg1) \ - : "r"(_arg2), "r"(_arg3), "r"(_arg4), "r"(_arg5), "r"(_arg6), \ - "r"(_num) \ - : "memory", "cc" \ - ); \ - _arg1; \ -}) - -/* startup code */ -asm(".section .text\n" - ".global _start\n" - "_start:\n" - ".option push\n" - ".option norelax\n" - "lla gp, __global_pointer$\n" - ".option pop\n" - "ld a0, 0(sp)\n" // argc (a0) was in the stack - "add a1, sp, "SZREG"\n" // argv (a1) = sp - "slli a2, a0, "PTRLOG"\n" // envp (a2) = SZREG*argc ... - "add a2, a2, "SZREG"\n" // + SZREG (skip null) - "add a2,a2,a1\n" // + argv - "andi sp,a1,-16\n" // sp must be 16-byte aligned - "call main\n" // main() returns the status code, we'll exit with it. - "li a7, 93\n" // NR_exit == 93 - "ecall\n" - ""); - -/* fcntl / open */ -#define O_RDONLY 0 -#define O_WRONLY 1 -#define O_RDWR 2 -#define O_CREAT 0x100 -#define O_EXCL 0x200 -#define O_NOCTTY 0x400 -#define O_TRUNC 0x1000 -#define O_APPEND 0x2000 -#define O_NONBLOCK 0x4000 -#define O_DIRECTORY 0x200000 - -struct sys_stat_struct { - unsigned long st_dev; /* Device. */ - unsigned long st_ino; /* File serial number. */ - unsigned int st_mode; /* File mode. */ - unsigned int st_nlink; /* Link count. */ - unsigned int st_uid; /* User ID of the file's owner. */ - unsigned int st_gid; /* Group ID of the file's group. */ - unsigned long st_rdev; /* Device number, if device. */ - unsigned long __pad1; - long st_size; /* Size of file, in bytes. */ - int st_blksize; /* Optimal block size for I/O. */ - int __pad2; - long st_blocks; /* Number 512-byte blocks allocated. */ - long st_atime; /* Time of last access. */ - unsigned long st_atime_nsec; - long st_mtime; /* Time of last modification. */ - unsigned long st_mtime_nsec; - long st_ctime; /* Time of last status change. */ - unsigned long st_ctime_nsec; - unsigned int __unused4; - unsigned int __unused5; -}; - -#endif - - -/* Below are the C functions used to declare the raw syscalls. They try to be - * architecture-agnostic, and return either a success or -errno. Declaring them - * static will lead to them being inlined in most cases, but it's still possible - * to reference them by a pointer if needed. - */ -static __attribute__((unused)) -void *sys_brk(void *addr) -{ - return (void *)my_syscall1(__NR_brk, addr); -} - -static __attribute__((noreturn,unused)) -void sys_exit(int status) -{ - my_syscall1(__NR_exit, status & 255); - while(1); // shut the "noreturn" warnings. -} - -static __attribute__((unused)) -int sys_chdir(const char *path) -{ - return my_syscall1(__NR_chdir, path); -} - -static __attribute__((unused)) -int sys_chmod(const char *path, mode_t mode) -{ -#ifdef __NR_fchmodat - return my_syscall4(__NR_fchmodat, AT_FDCWD, path, mode, 0); -#elif defined(__NR_chmod) - return my_syscall2(__NR_chmod, path, mode); -#else -#error Neither __NR_fchmodat nor __NR_chmod defined, cannot implement sys_chmod() -#endif -} - -static __attribute__((unused)) -int sys_chown(const char *path, uid_t owner, gid_t group) -{ -#ifdef __NR_fchownat - return my_syscall5(__NR_fchownat, AT_FDCWD, path, owner, group, 0); -#elif defined(__NR_chown) - return my_syscall3(__NR_chown, path, owner, group); -#else -#error Neither __NR_fchownat nor __NR_chown defined, cannot implement sys_chown() -#endif -} - -static __attribute__((unused)) -int sys_chroot(const char *path) -{ - return my_syscall1(__NR_chroot, path); -} - -static __attribute__((unused)) -int sys_close(int fd) -{ - return my_syscall1(__NR_close, fd); -} - -static __attribute__((unused)) -int sys_dup(int fd) -{ - return my_syscall1(__NR_dup, fd); -} - -#ifdef __NR_dup3 -static __attribute__((unused)) -int sys_dup3(int old, int new, int flags) -{ - return my_syscall3(__NR_dup3, old, new, flags); -} -#endif - -static __attribute__((unused)) -int sys_dup2(int old, int new) -{ -#ifdef __NR_dup3 - return my_syscall3(__NR_dup3, old, new, 0); -#elif defined(__NR_dup2) - return my_syscall2(__NR_dup2, old, new); -#else -#error Neither __NR_dup3 nor __NR_dup2 defined, cannot implement sys_dup2() -#endif -} - -static __attribute__((unused)) -int sys_execve(const char *filename, char *const argv[], char *const envp[]) -{ - return my_syscall3(__NR_execve, filename, argv, envp); -} - -static __attribute__((unused)) -pid_t sys_fork(void) -{ -#ifdef __NR_clone - /* note: some archs only have clone() and not fork(). Different archs - * have a different API, but most archs have the flags on first arg and - * will not use the rest with no other flag. - */ - return my_syscall5(__NR_clone, SIGCHLD, 0, 0, 0, 0); -#elif defined(__NR_fork) - return my_syscall0(__NR_fork); -#else -#error Neither __NR_clone nor __NR_fork defined, cannot implement sys_fork() -#endif -} - -static __attribute__((unused)) -int sys_fsync(int fd) -{ - return my_syscall1(__NR_fsync, fd); -} - -static __attribute__((unused)) -int sys_getdents64(int fd, struct linux_dirent64 *dirp, int count) -{ - return my_syscall3(__NR_getdents64, fd, dirp, count); -} - -static __attribute__((unused)) -pid_t sys_getpgid(pid_t pid) -{ - return my_syscall1(__NR_getpgid, pid); -} - -static __attribute__((unused)) -pid_t sys_getpgrp(void) -{ - return sys_getpgid(0); -} - -static __attribute__((unused)) -pid_t sys_getpid(void) -{ - return my_syscall0(__NR_getpid); -} - -static __attribute__((unused)) -pid_t sys_gettid(void) -{ - return my_syscall0(__NR_gettid); -} - -static __attribute__((unused)) -int sys_gettimeofday(struct timeval *tv, struct timezone *tz) -{ - return my_syscall2(__NR_gettimeofday, tv, tz); -} - -static __attribute__((unused)) -int sys_ioctl(int fd, unsigned long req, void *value) -{ - return my_syscall3(__NR_ioctl, fd, req, value); -} - -static __attribute__((unused)) -int sys_kill(pid_t pid, int signal) -{ - return my_syscall2(__NR_kill, pid, signal); -} - -static __attribute__((unused)) -int sys_link(const char *old, const char *new) -{ -#ifdef __NR_linkat - return my_syscall5(__NR_linkat, AT_FDCWD, old, AT_FDCWD, new, 0); -#elif defined(__NR_link) - return my_syscall2(__NR_link, old, new); -#else -#error Neither __NR_linkat nor __NR_link defined, cannot implement sys_link() -#endif -} - -static __attribute__((unused)) -off_t sys_lseek(int fd, off_t offset, int whence) -{ - return my_syscall3(__NR_lseek, fd, offset, whence); -} - -static __attribute__((unused)) -int sys_mkdir(const char *path, mode_t mode) -{ -#ifdef __NR_mkdirat - return my_syscall3(__NR_mkdirat, AT_FDCWD, path, mode); -#elif defined(__NR_mkdir) - return my_syscall2(__NR_mkdir, path, mode); -#else -#error Neither __NR_mkdirat nor __NR_mkdir defined, cannot implement sys_mkdir() -#endif -} - -static __attribute__((unused)) -long sys_mknod(const char *path, mode_t mode, dev_t dev) -{ -#ifdef __NR_mknodat - return my_syscall4(__NR_mknodat, AT_FDCWD, path, mode, dev); -#elif defined(__NR_mknod) - return my_syscall3(__NR_mknod, path, mode, dev); -#else -#error Neither __NR_mknodat nor __NR_mknod defined, cannot implement sys_mknod() -#endif -} - -static __attribute__((unused)) -int sys_mount(const char *src, const char *tgt, const char *fst, - unsigned long flags, const void *data) -{ - return my_syscall5(__NR_mount, src, tgt, fst, flags, data); -} - -static __attribute__((unused)) -int sys_open(const char *path, int flags, mode_t mode) -{ -#ifdef __NR_openat - return my_syscall4(__NR_openat, AT_FDCWD, path, flags, mode); -#elif defined(__NR_open) - return my_syscall3(__NR_open, path, flags, mode); -#else -#error Neither __NR_openat nor __NR_open defined, cannot implement sys_open() -#endif -} - -static __attribute__((unused)) -int sys_pivot_root(const char *new, const char *old) -{ - return my_syscall2(__NR_pivot_root, new, old); -} - -static __attribute__((unused)) -int sys_poll(struct pollfd *fds, int nfds, int timeout) -{ -#if defined(__NR_ppoll) - struct timespec t; - - if (timeout >= 0) { - t.tv_sec = timeout / 1000; - t.tv_nsec = (timeout % 1000) * 1000000; - } - return my_syscall4(__NR_ppoll, fds, nfds, (timeout >= 0) ? &t : NULL, NULL); -#elif defined(__NR_poll) - return my_syscall3(__NR_poll, fds, nfds, timeout); -#else -#error Neither __NR_ppoll nor __NR_poll defined, cannot implement sys_poll() -#endif -} - -static __attribute__((unused)) -ssize_t sys_read(int fd, void *buf, size_t count) -{ - return my_syscall3(__NR_read, fd, buf, count); -} - -static __attribute__((unused)) -ssize_t sys_reboot(int magic1, int magic2, int cmd, void *arg) -{ - return my_syscall4(__NR_reboot, magic1, magic2, cmd, arg); -} - -static __attribute__((unused)) -int sys_sched_yield(void) -{ - return my_syscall0(__NR_sched_yield); -} - -static __attribute__((unused)) -int sys_select(int nfds, fd_set *rfds, fd_set *wfds, fd_set *efds, struct timeval *timeout) -{ -#if defined(__ARCH_WANT_SYS_OLD_SELECT) && !defined(__NR__newselect) - struct sel_arg_struct { - unsigned long n; - fd_set *r, *w, *e; - struct timeval *t; - } arg = { .n = nfds, .r = rfds, .w = wfds, .e = efds, .t = timeout }; - return my_syscall1(__NR_select, &arg); -#elif defined(__ARCH_WANT_SYS_PSELECT6) && defined(__NR_pselect6) - struct timespec t; - - if (timeout) { - t.tv_sec = timeout->tv_sec; - t.tv_nsec = timeout->tv_usec * 1000; - } - return my_syscall6(__NR_pselect6, nfds, rfds, wfds, efds, timeout ? &t : NULL, NULL); -#elif defined(__NR__newselect) || defined(__NR_select) -#ifndef __NR__newselect -#define __NR__newselect __NR_select -#endif - return my_syscall5(__NR__newselect, nfds, rfds, wfds, efds, timeout); -#else -#error None of __NR_select, __NR_pselect6, nor __NR__newselect defined, cannot implement sys_select() -#endif -} - -static __attribute__((unused)) -int sys_setpgid(pid_t pid, pid_t pgid) -{ - return my_syscall2(__NR_setpgid, pid, pgid); -} - -static __attribute__((unused)) -pid_t sys_setsid(void) -{ - return my_syscall0(__NR_setsid); -} - -static __attribute__((unused)) -int sys_stat(const char *path, struct stat *buf) -{ - struct sys_stat_struct stat; - long ret; - -#ifdef __NR_newfstatat - /* only solution for arm64 */ - ret = my_syscall4(__NR_newfstatat, AT_FDCWD, path, &stat, 0); -#elif defined(__NR_stat) - ret = my_syscall2(__NR_stat, path, &stat); -#else -#error Neither __NR_newfstatat nor __NR_stat defined, cannot implement sys_stat() -#endif - buf->st_dev = stat.st_dev; - buf->st_ino = stat.st_ino; - buf->st_mode = stat.st_mode; - buf->st_nlink = stat.st_nlink; - buf->st_uid = stat.st_uid; - buf->st_gid = stat.st_gid; - buf->st_rdev = stat.st_rdev; - buf->st_size = stat.st_size; - buf->st_blksize = stat.st_blksize; - buf->st_blocks = stat.st_blocks; - buf->st_atime = stat.st_atime; - buf->st_mtime = stat.st_mtime; - buf->st_ctime = stat.st_ctime; - return ret; -} - - -static __attribute__((unused)) -int sys_symlink(const char *old, const char *new) -{ -#ifdef __NR_symlinkat - return my_syscall3(__NR_symlinkat, old, AT_FDCWD, new); -#elif defined(__NR_symlink) - return my_syscall2(__NR_symlink, old, new); -#else -#error Neither __NR_symlinkat nor __NR_symlink defined, cannot implement sys_symlink() -#endif -} - -static __attribute__((unused)) -mode_t sys_umask(mode_t mode) -{ - return my_syscall1(__NR_umask, mode); -} - -static __attribute__((unused)) -int sys_umount2(const char *path, int flags) -{ - return my_syscall2(__NR_umount2, path, flags); -} - -static __attribute__((unused)) -int sys_unlink(const char *path) -{ -#ifdef __NR_unlinkat - return my_syscall3(__NR_unlinkat, AT_FDCWD, path, 0); -#elif defined(__NR_unlink) - return my_syscall1(__NR_unlink, path); -#else -#error Neither __NR_unlinkat nor __NR_unlink defined, cannot implement sys_unlink() -#endif -} - -static __attribute__((unused)) -pid_t sys_wait4(pid_t pid, int *status, int options, struct rusage *rusage) -{ - return my_syscall4(__NR_wait4, pid, status, options, rusage); -} - -static __attribute__((unused)) -pid_t sys_waitpid(pid_t pid, int *status, int options) -{ - return sys_wait4(pid, status, options, 0); -} - -static __attribute__((unused)) -pid_t sys_wait(int *status) -{ - return sys_waitpid(-1, status, 0); -} - -static __attribute__((unused)) -ssize_t sys_write(int fd, const void *buf, size_t count) -{ - return my_syscall3(__NR_write, fd, buf, count); -} - - -/* Below are the libc-compatible syscalls which return x or -1 and set errno. - * They rely on the functions above. Similarly they're marked static so that it - * is possible to assign pointers to them if needed. - */ - -static __attribute__((unused)) -int brk(void *addr) -{ - void *ret = sys_brk(addr); - - if (!ret) { - SET_ERRNO(ENOMEM); - return -1; - } - return 0; -} - -static __attribute__((noreturn,unused)) -void exit(int status) -{ - sys_exit(status); -} - -static __attribute__((unused)) -int chdir(const char *path) -{ - int ret = sys_chdir(path); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int chmod(const char *path, mode_t mode) -{ - int ret = sys_chmod(path, mode); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int chown(const char *path, uid_t owner, gid_t group) -{ - int ret = sys_chown(path, owner, group); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int chroot(const char *path) -{ - int ret = sys_chroot(path); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int close(int fd) -{ - int ret = sys_close(fd); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int dup(int fd) -{ - int ret = sys_dup(fd); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int dup2(int old, int new) -{ - int ret = sys_dup2(old, new); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -#ifdef __NR_dup3 -static __attribute__((unused)) -int dup3(int old, int new, int flags) -{ - int ret = sys_dup3(old, new, flags); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} -#endif - -static __attribute__((unused)) -int execve(const char *filename, char *const argv[], char *const envp[]) -{ - int ret = sys_execve(filename, argv, envp); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t fork(void) -{ - pid_t ret = sys_fork(); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int fsync(int fd) -{ - int ret = sys_fsync(fd); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int getdents64(int fd, struct linux_dirent64 *dirp, int count) -{ - int ret = sys_getdents64(fd, dirp, count); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t getpgid(pid_t pid) -{ - pid_t ret = sys_getpgid(pid); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t getpgrp(void) -{ - pid_t ret = sys_getpgrp(); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t getpid(void) -{ - pid_t ret = sys_getpid(); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t gettid(void) -{ - pid_t ret = sys_gettid(); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int gettimeofday(struct timeval *tv, struct timezone *tz) -{ - int ret = sys_gettimeofday(tv, tz); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int ioctl(int fd, unsigned long req, void *value) -{ - int ret = sys_ioctl(fd, req, value); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int kill(pid_t pid, int signal) -{ - int ret = sys_kill(pid, signal); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int link(const char *old, const char *new) -{ - int ret = sys_link(old, new); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -off_t lseek(int fd, off_t offset, int whence) -{ - off_t ret = sys_lseek(fd, offset, whence); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int mkdir(const char *path, mode_t mode) -{ - int ret = sys_mkdir(path, mode); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int mknod(const char *path, mode_t mode, dev_t dev) -{ - int ret = sys_mknod(path, mode, dev); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int mount(const char *src, const char *tgt, - const char *fst, unsigned long flags, - const void *data) -{ - int ret = sys_mount(src, tgt, fst, flags, data); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int open(const char *path, int flags, mode_t mode) -{ - int ret = sys_open(path, flags, mode); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int pivot_root(const char *new, const char *old) -{ - int ret = sys_pivot_root(new, old); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int poll(struct pollfd *fds, int nfds, int timeout) -{ - int ret = sys_poll(fds, nfds, timeout); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -ssize_t read(int fd, void *buf, size_t count) -{ - ssize_t ret = sys_read(fd, buf, count); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int reboot(int cmd) -{ - int ret = sys_reboot(LINUX_REBOOT_MAGIC1, LINUX_REBOOT_MAGIC2, cmd, 0); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -void *sbrk(intptr_t inc) -{ - void *ret; - - /* first call to find current end */ - if ((ret = sys_brk(0)) && (sys_brk(ret + inc) == ret + inc)) - return ret + inc; - - SET_ERRNO(ENOMEM); - return (void *)-1; -} - -static __attribute__((unused)) -int sched_yield(void) -{ - int ret = sys_sched_yield(); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int select(int nfds, fd_set *rfds, fd_set *wfds, fd_set *efds, struct timeval *timeout) -{ - int ret = sys_select(nfds, rfds, wfds, efds, timeout); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int setpgid(pid_t pid, pid_t pgid) -{ - int ret = sys_setpgid(pid, pgid); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t setsid(void) -{ - pid_t ret = sys_setsid(); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -unsigned int sleep(unsigned int seconds) -{ - struct timeval my_timeval = { seconds, 0 }; - - if (sys_select(0, 0, 0, 0, &my_timeval) < 0) - return my_timeval.tv_sec + !!my_timeval.tv_usec; - else - return 0; -} - -static __attribute__((unused)) -int msleep(unsigned int msecs) -{ - struct timeval my_timeval = { msecs / 1000, (msecs % 1000) * 1000 }; - - if (sys_select(0, 0, 0, 0, &my_timeval) < 0) - return (my_timeval.tv_sec * 1000) + - (my_timeval.tv_usec / 1000) + - !!(my_timeval.tv_usec % 1000); - else - return 0; -} - -static __attribute__((unused)) -int stat(const char *path, struct stat *buf) -{ - int ret = sys_stat(path, buf); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int symlink(const char *old, const char *new) -{ - int ret = sys_symlink(old, new); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int tcsetpgrp(int fd, pid_t pid) -{ - return ioctl(fd, TIOCSPGRP, &pid); -} - -static __attribute__((unused)) -mode_t umask(mode_t mode) -{ - return sys_umask(mode); -} - -static __attribute__((unused)) -int umount2(const char *path, int flags) -{ - int ret = sys_umount2(path, flags); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -int unlink(const char *path) -{ - int ret = sys_unlink(path); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t wait4(pid_t pid, int *status, int options, struct rusage *rusage) -{ - pid_t ret = sys_wait4(pid, status, options, rusage); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t waitpid(pid_t pid, int *status, int options) -{ - pid_t ret = sys_waitpid(pid, status, options); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -pid_t wait(int *status) -{ - pid_t ret = sys_wait(status); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -static __attribute__((unused)) -ssize_t write(int fd, const void *buf, size_t count) -{ - ssize_t ret = sys_write(fd, buf, count); - - if (ret < 0) { - SET_ERRNO(-ret); - ret = -1; - } - return ret; -} - -/* some size-optimized reimplementations of a few common str* and mem* - * functions. They're marked static, except memcpy() and raise() which are used - * by libgcc on ARM, so they are marked weak instead in order not to cause an - * error when building a program made of multiple files (not recommended). - */ - -static __attribute__((unused)) -void *memmove(void *dst, const void *src, size_t len) -{ - ssize_t pos = (dst <= src) ? -1 : (long)len; - void *ret = dst; - - while (len--) { - pos += (dst <= src) ? 1 : -1; - ((char *)dst)[pos] = ((char *)src)[pos]; - } - return ret; -} - -static __attribute__((unused)) -void *memset(void *dst, int b, size_t len) -{ - char *p = dst; - - while (len--) - *(p++) = b; - return dst; -} - -static __attribute__((unused)) -int memcmp(const void *s1, const void *s2, size_t n) -{ - size_t ofs = 0; - char c1 = 0; - - while (ofs < n && !(c1 = ((char *)s1)[ofs] - ((char *)s2)[ofs])) { - ofs++; - } - return c1; -} - -static __attribute__((unused)) -char *strcpy(char *dst, const char *src) -{ - char *ret = dst; - - while ((*dst++ = *src++)); - return ret; -} - -static __attribute__((unused)) -char *strchr(const char *s, int c) -{ - while (*s) { - if (*s == (char)c) - return (char *)s; - s++; - } - return NULL; -} - -static __attribute__((unused)) -char *strrchr(const char *s, int c) -{ - const char *ret = NULL; - - while (*s) { - if (*s == (char)c) - ret = s; - s++; - } - return (char *)ret; -} - -static __attribute__((unused)) -size_t nolibc_strlen(const char *str) -{ - size_t len; - - for (len = 0; str[len]; len++); - return len; -} - -#define strlen(str) ({ \ - __builtin_constant_p((str)) ? \ - __builtin_strlen((str)) : \ - nolibc_strlen((str)); \ -}) - -static __attribute__((unused)) -int isdigit(int c) -{ - return (unsigned int)(c - '0') <= 9; -} - -static __attribute__((unused)) -long atol(const char *s) -{ - unsigned long ret = 0; - unsigned long d; - int neg = 0; - - if (*s == '-') { - neg = 1; - s++; - } - - while (1) { - d = (*s++) - '0'; - if (d > 9) - break; - ret *= 10; - ret += d; - } - - return neg ? -ret : ret; -} - -static __attribute__((unused)) -int atoi(const char *s) -{ - return atol(s); -} - -static __attribute__((unused)) -const char *ltoa(long in) -{ - /* large enough for -9223372036854775808 */ - static char buffer[21]; - char *pos = buffer + sizeof(buffer) - 1; - int neg = in < 0; - unsigned long n = neg ? -in : in; - - *pos-- = '\0'; - do { - *pos-- = '0' + n % 10; - n /= 10; - if (pos < buffer) - return pos + 1; - } while (n); - - if (neg) - *pos-- = '-'; - return pos + 1; -} - -__attribute__((weak,unused)) -void *memcpy(void *dst, const void *src, size_t len) -{ - return memmove(dst, src, len); -} - -/* needed by libgcc for divide by zero */ -__attribute__((weak,unused)) -int raise(int signal) -{ - return kill(getpid(), signal); -} - -/* Here come a few helper functions */ - -static __attribute__((unused)) -void FD_ZERO(fd_set *set) -{ - memset(set, 0, sizeof(*set)); -} - -static __attribute__((unused)) -void FD_SET(int fd, fd_set *set) -{ - if (fd < 0 || fd >= FD_SETSIZE) - return; - set->fd32[fd / 32] |= 1 << (fd & 31); -} - -/* WARNING, it only deals with the 4096 first majors and 256 first minors */ -static __attribute__((unused)) -dev_t makedev(unsigned int major, unsigned int minor) -{ - return ((major & 0xfff) << 8) | (minor & 0xff); -} +#endif /* _NOLIBC_H */ diff --git a/tools/include/uapi/asm-generic/fcntl.h b/tools/include/uapi/asm-generic/fcntl.h index ac190958c9..b02c8e0f40 100644 --- a/tools/include/uapi/asm-generic/fcntl.h +++ b/tools/include/uapi/asm-generic/fcntl.h @@ -1,3 +1,4 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ #ifndef _ASM_GENERIC_FCNTL_H #define _ASM_GENERIC_FCNTL_H @@ -90,7 +91,7 @@ /* a horrid kludge trying to make sure that this will fail on old kernels */ #define O_TMPFILE (__O_TMPFILE | O_DIRECTORY) -#define O_TMPFILE_MASK (__O_TMPFILE | O_DIRECTORY | O_CREAT) +#define O_TMPFILE_MASK (__O_TMPFILE | O_DIRECTORY | O_CREAT) #ifndef O_NDELAY #define O_NDELAY O_NONBLOCK @@ -115,13 +116,13 @@ #define F_GETSIG 11 /* for sockets. */ #endif -#ifndef CONFIG_64BIT +#if __BITS_PER_LONG == 32 || defined(__KERNEL__) #ifndef F_GETLK64 #define F_GETLK64 12 /* using 'struct flock64' */ #define F_SETLK64 13 #define F_SETLKW64 14 #endif -#endif +#endif /* __BITS_PER_LONG == 32 || defined(__KERNEL__) */ #ifndef F_SETOWN_EX #define F_SETOWN_EX 15 @@ -142,7 +143,7 @@ * record locks, but are "owned" by the open file description, not the * process. This means that they are inherited across fork() like BSD (flock) * locks, and they are only released automatically when the last reference to - * the the open file against which they were acquired is put. + * the open file against which they were acquired is put. */ #define F_OFD_GETLK 36 #define F_OFD_SETLK 37 @@ -180,6 +181,10 @@ struct f_owner_ex { blocking */ #define LOCK_UN 8 /* remove lock */ +/* + * LOCK_MAND support has been removed from the kernel. We leave the symbols + * here to not break legacy builds, but these should not be used in new code. + */ #define LOCK_MAND 32 /* This is a mandatory flock ... */ #define LOCK_READ 64 /* which allows concurrent read operations */ #define LOCK_WRITE 128 /* which allows concurrent write operations */ @@ -188,24 +193,19 @@ struct f_owner_ex { #define F_LINUX_SPECIFIC_BASE 1024 #ifndef HAVE_ARCH_STRUCT_FLOCK -#ifndef __ARCH_FLOCK_PAD -#define __ARCH_FLOCK_PAD -#endif - struct flock { short l_type; short l_whence; __kernel_off_t l_start; __kernel_off_t l_len; __kernel_pid_t l_pid; +#ifdef __ARCH_FLOCK_EXTRA_SYSID + __ARCH_FLOCK_EXTRA_SYSID +#endif +#ifdef __ARCH_FLOCK_PAD __ARCH_FLOCK_PAD +#endif }; -#endif - -#ifndef HAVE_ARCH_STRUCT_FLOCK64 -#ifndef __ARCH_FLOCK64_PAD -#define __ARCH_FLOCK64_PAD -#endif struct flock64 { short l_type; @@ -213,8 +213,10 @@ struct flock64 { __kernel_loff_t l_start; __kernel_loff_t l_len; __kernel_pid_t l_pid; +#ifdef __ARCH_FLOCK64_PAD __ARCH_FLOCK64_PAD -}; #endif +}; +#endif /* HAVE_ARCH_STRUCT_FLOCK */ #endif /* _ASM_GENERIC_FCNTL_H */ diff --git a/tools/include/uapi/asm-generic/socket.h b/tools/include/uapi/asm-generic/socket.h index 77f7c1638e..8756df13be 100644 --- a/tools/include/uapi/asm-generic/socket.h +++ b/tools/include/uapi/asm-generic/socket.h @@ -119,6 +119,8 @@ #define SO_DETACH_REUSEPORT_BPF 68 +#define SO_RCVMARK 75 + #if !defined(__KERNEL__) #if __BITS_PER_LONG == 64 || (defined(__x86_64__) && defined(__ILP32__)) diff --git a/tools/include/uapi/asm-generic/unistd.h b/tools/include/uapi/asm-generic/unistd.h index 1c48b0ae3b..45fa180cc5 100644 --- a/tools/include/uapi/asm-generic/unistd.h +++ b/tools/include/uapi/asm-generic/unistd.h @@ -383,7 +383,7 @@ __SYSCALL(__NR_syslog, sys_syslog) /* kernel/ptrace.c */ #define __NR_ptrace 117 -__SYSCALL(__NR_ptrace, sys_ptrace) +__SC_COMP(__NR_ptrace, sys_ptrace, compat_sys_ptrace) /* kernel/sched/core.c */ #define __NR_sched_setparam 118 @@ -779,7 +779,7 @@ __SYSCALL(__NR_rseq, sys_rseq) #define __NR_kexec_file_load 294 __SYSCALL(__NR_kexec_file_load, sys_kexec_file_load) /* 295 through 402 are unassigned to sync up with generic numbers, don't use */ -#if __BITS_PER_LONG == 32 +#if defined(__SYSCALL_COMPAT) || __BITS_PER_LONG == 32 #define __NR_clock_gettime64 403 __SYSCALL(__NR_clock_gettime64, sys_clock_gettime) #define __NR_clock_settime64 404 diff --git a/tools/include/uapi/asm/bitsperlong.h b/tools/include/uapi/asm/bitsperlong.h index edba4d93e9..da52065171 100644 --- a/tools/include/uapi/asm/bitsperlong.h +++ b/tools/include/uapi/asm/bitsperlong.h @@ -17,6 +17,8 @@ #include "../../../arch/riscv/include/uapi/asm/bitsperlong.h" #elif defined(__alpha__) #include "../../../arch/alpha/include/uapi/asm/bitsperlong.h" +#elif defined(__loongarch__) +#include "../../../arch/loongarch/include/uapi/asm/bitsperlong.h" #else #include #endif diff --git a/tools/include/uapi/asm/bpf_perf_event.h b/tools/include/uapi/asm/bpf_perf_event.h index 39acc149d8..d7dfeab0d7 100644 --- a/tools/include/uapi/asm/bpf_perf_event.h +++ b/tools/include/uapi/asm/bpf_perf_event.h @@ -1,5 +1,7 @@ #if defined(__aarch64__) #include "../../arch/arm64/include/uapi/asm/bpf_perf_event.h" +#elif defined(__arc__) +#include "../../arch/arc/include/uapi/asm/bpf_perf_event.h" #elif defined(__s390__) #include "../../arch/s390/include/uapi/asm/bpf_perf_event.h" #elif defined(__riscv) diff --git a/tools/include/uapi/drm/i915_drm.h b/tools/include/uapi/drm/i915_drm.h index 05c3642aae..520ad2691a 100644 --- a/tools/include/uapi/drm/i915_drm.h +++ b/tools/include/uapi/drm/i915_drm.h @@ -154,25 +154,77 @@ enum i915_mocs_table_index { I915_MOCS_CACHED, }; -/* +/** + * enum drm_i915_gem_engine_class - uapi engine type enumeration + * * Different engines serve different roles, and there may be more than one - * engine serving each role. enum drm_i915_gem_engine_class provides a - * classification of the role of the engine, which may be used when requesting - * operations to be performed on a certain subset of engines, or for providing - * information about that group. + * engine serving each role. This enum provides a classification of the role + * of the engine, which may be used when requesting operations to be performed + * on a certain subset of engines, or for providing information about that + * group. */ enum drm_i915_gem_engine_class { + /** + * @I915_ENGINE_CLASS_RENDER: + * + * Render engines support instructions used for 3D, Compute (GPGPU), + * and programmable media workloads. These instructions fetch data and + * dispatch individual work items to threads that operate in parallel. + * The threads run small programs (called "kernels" or "shaders") on + * the GPU's execution units (EUs). + */ I915_ENGINE_CLASS_RENDER = 0, + + /** + * @I915_ENGINE_CLASS_COPY: + * + * Copy engines (also referred to as "blitters") support instructions + * that move blocks of data from one location in memory to another, + * or that fill a specified location of memory with fixed data. + * Copy engines can perform pre-defined logical or bitwise operations + * on the source, destination, or pattern data. + */ I915_ENGINE_CLASS_COPY = 1, + + /** + * @I915_ENGINE_CLASS_VIDEO: + * + * Video engines (also referred to as "bit stream decode" (BSD) or + * "vdbox") support instructions that perform fixed-function media + * decode and encode. + */ I915_ENGINE_CLASS_VIDEO = 2, + + /** + * @I915_ENGINE_CLASS_VIDEO_ENHANCE: + * + * Video enhancement engines (also referred to as "vebox") support + * instructions related to image enhancement. + */ I915_ENGINE_CLASS_VIDEO_ENHANCE = 3, - /* should be kept compact */ + /** + * @I915_ENGINE_CLASS_COMPUTE: + * + * Compute engines support a subset of the instructions available + * on render engines: compute engines support Compute (GPGPU) and + * programmable media workloads, but do not support the 3D pipeline. + */ + I915_ENGINE_CLASS_COMPUTE = 4, + /* Values in this enum should be kept compact. */ + + /** + * @I915_ENGINE_CLASS_INVALID: + * + * Placeholder value to represent an invalid engine class assignment. + */ I915_ENGINE_CLASS_INVALID = -1 }; -/* +/** + * struct i915_engine_class_instance - Engine class/instance identifier + * * There may be more than one engine fulfilling any role within the system. * Each engine of a class is given a unique instance number and therefore * any engine can be specified by its class:instance tuplet. APIs that allow @@ -180,10 +232,21 @@ enum drm_i915_gem_engine_class { * for this identification. */ struct i915_engine_class_instance { - __u16 engine_class; /* see enum drm_i915_gem_engine_class */ - __u16 engine_instance; + /** + * @engine_class: + * + * Engine class from enum drm_i915_gem_engine_class + */ + __u16 engine_class; #define I915_ENGINE_CLASS_INVALID_NONE -1 #define I915_ENGINE_CLASS_INVALID_VIRTUAL -2 + + /** + * @engine_instance: + * + * Engine instance. + */ + __u16 engine_instance; }; /** @@ -688,14 +751,27 @@ typedef struct drm_i915_irq_wait { /* Must be kept compact -- no holes and well documented */ -typedef struct drm_i915_getparam { +/** + * struct drm_i915_getparam - Driver parameter query structure. + */ +struct drm_i915_getparam { + /** @param: Driver parameter to query. */ __s32 param; - /* + + /** + * @value: Address of memory where queried value should be put. + * * WARNING: Using pointers instead of fixed-size u64 means we need to write * compat32 code. Don't repeat this mistake. */ int __user *value; -} drm_i915_getparam_t; +}; + +/** + * typedef drm_i915_getparam_t - Driver parameter query structure. + * See struct drm_i915_getparam. + */ +typedef struct drm_i915_getparam drm_i915_getparam_t; /* Ioctl to set kernel params: */ @@ -1176,76 +1252,119 @@ struct drm_i915_gem_exec_object2 { __u64 rsvd2; }; +/** + * struct drm_i915_gem_exec_fence - An input or output fence for the execbuf + * ioctl. + * + * The request will wait for input fence to signal before submission. + * + * The returned output fence will be signaled after the completion of the + * request. + */ struct drm_i915_gem_exec_fence { - /** - * User's handle for a drm_syncobj to wait on or signal. - */ + /** @handle: User's handle for a drm_syncobj to wait on or signal. */ __u32 handle; + /** + * @flags: Supported flags are: + * + * I915_EXEC_FENCE_WAIT: + * Wait for the input fence before request submission. + * + * I915_EXEC_FENCE_SIGNAL: + * Return request completion fence as output + */ + __u32 flags; #define I915_EXEC_FENCE_WAIT (1<<0) #define I915_EXEC_FENCE_SIGNAL (1<<1) #define __I915_EXEC_FENCE_UNKNOWN_FLAGS (-(I915_EXEC_FENCE_SIGNAL << 1)) - __u32 flags; }; -/* - * See drm_i915_gem_execbuffer_ext_timeline_fences. - */ -#define DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES 0 - -/* +/** + * struct drm_i915_gem_execbuffer_ext_timeline_fences - Timeline fences + * for execbuf ioctl. + * * This structure describes an array of drm_syncobj and associated points for * timeline variants of drm_syncobj. It is invalid to append this structure to * the execbuf if I915_EXEC_FENCE_ARRAY is set. */ struct drm_i915_gem_execbuffer_ext_timeline_fences { +#define DRM_I915_GEM_EXECBUFFER_EXT_TIMELINE_FENCES 0 + /** @base: Extension link. See struct i915_user_extension. */ struct i915_user_extension base; /** - * Number of element in the handles_ptr & value_ptr arrays. + * @fence_count: Number of elements in the @handles_ptr & @value_ptr + * arrays. */ __u64 fence_count; /** - * Pointer to an array of struct drm_i915_gem_exec_fence of length - * fence_count. + * @handles_ptr: Pointer to an array of struct drm_i915_gem_exec_fence + * of length @fence_count. */ __u64 handles_ptr; /** - * Pointer to an array of u64 values of length fence_count. Values - * must be 0 for a binary drm_syncobj. A Value of 0 for a timeline - * drm_syncobj is invalid as it turns a drm_syncobj into a binary one. + * @values_ptr: Pointer to an array of u64 values of length + * @fence_count. + * Values must be 0 for a binary drm_syncobj. A Value of 0 for a + * timeline drm_syncobj is invalid as it turns a drm_syncobj into a + * binary one. */ __u64 values_ptr; }; +/** + * struct drm_i915_gem_execbuffer2 - Structure for DRM_I915_GEM_EXECBUFFER2 + * ioctl. + */ struct drm_i915_gem_execbuffer2 { - /** - * List of gem_exec_object2 structs - */ + /** @buffers_ptr: Pointer to a list of gem_exec_object2 structs */ __u64 buffers_ptr; + + /** @buffer_count: Number of elements in @buffers_ptr array */ __u32 buffer_count; - /** Offset in the batchbuffer to start execution from. */ - __u32 batch_start_offset; - /** Bytes used in batchbuffer from batch_start_offset */ - __u32 batch_len; - __u32 DR1; - __u32 DR4; - __u32 num_cliprects; /** - * This is a struct drm_clip_rect *cliprects if I915_EXEC_FENCE_ARRAY - * & I915_EXEC_USE_EXTENSIONS are not set. + * @batch_start_offset: Offset in the batchbuffer to start execution + * from. + */ + __u32 batch_start_offset; + + /** + * @batch_len: Length in bytes of the batch buffer, starting from the + * @batch_start_offset. If 0, length is assumed to be the batch buffer + * object size. + */ + __u32 batch_len; + + /** @DR1: deprecated */ + __u32 DR1; + + /** @DR4: deprecated */ + __u32 DR4; + + /** @num_cliprects: See @cliprects_ptr */ + __u32 num_cliprects; + + /** + * @cliprects_ptr: Kernel clipping was a DRI1 misfeature. + * + * It is invalid to use this field if I915_EXEC_FENCE_ARRAY or + * I915_EXEC_USE_EXTENSIONS flags are not set. * * If I915_EXEC_FENCE_ARRAY is set, then this is a pointer to an array - * of struct drm_i915_gem_exec_fence and num_cliprects is the length - * of the array. + * of &drm_i915_gem_exec_fence and @num_cliprects is the length of the + * array. * * If I915_EXEC_USE_EXTENSIONS is set, then this is a pointer to a - * single struct i915_user_extension and num_cliprects is 0. + * single &i915_user_extension and num_cliprects is 0. */ __u64 cliprects_ptr; + + /** @flags: Execbuf flags */ + __u64 flags; #define I915_EXEC_RING_MASK (0x3f) #define I915_EXEC_DEFAULT (0<<0) #define I915_EXEC_RENDER (1<<0) @@ -1263,10 +1382,6 @@ struct drm_i915_gem_execbuffer2 { #define I915_EXEC_CONSTANTS_REL_GENERAL (0<<6) /* default */ #define I915_EXEC_CONSTANTS_ABSOLUTE (1<<6) #define I915_EXEC_CONSTANTS_REL_SURFACE (2<<6) /* gen4/5 only */ - __u64 flags; - __u64 rsvd1; /* now used for context info */ - __u64 rsvd2; -}; /** Resets the SO write offset registers for transform feedback on gen7. */ #define I915_EXEC_GEN7_SOL_RESET (1<<8) @@ -1369,9 +1484,23 @@ struct drm_i915_gem_execbuffer2 { * drm_i915_gem_execbuffer_ext enum. */ #define I915_EXEC_USE_EXTENSIONS (1 << 21) - #define __I915_EXEC_UNKNOWN_FLAGS (-(I915_EXEC_USE_EXTENSIONS << 1)) + /** @rsvd1: Context id */ + __u64 rsvd1; + + /** + * @rsvd2: in and out sync_file file descriptors. + * + * When I915_EXEC_FENCE_IN or I915_EXEC_FENCE_SUBMIT flag is set, the + * lower 32 bits of this field will have the in sync_file fd (input). + * + * When I915_EXEC_FENCE_OUT flag is set, the upper 32 bits of this + * field will have the out sync_file fd (output). + */ + __u64 rsvd2; +}; + #define I915_EXEC_CONTEXT_ID_MASK (0xffffffff) #define i915_execbuffer2_set_context_id(eb2, context) \ (eb2).rsvd1 = context & I915_EXEC_CONTEXT_ID_MASK @@ -1751,19 +1880,58 @@ struct drm_i915_gem_context_create { __u32 pad; }; +/** + * struct drm_i915_gem_context_create_ext - Structure for creating contexts. + */ struct drm_i915_gem_context_create_ext { - __u32 ctx_id; /* output: id of new context*/ + /** @ctx_id: Id of the created context (output) */ + __u32 ctx_id; + + /** + * @flags: Supported flags are: + * + * I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS: + * + * Extensions may be appended to this structure and driver must check + * for those. See @extensions. + * + * I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE + * + * Created context will have single timeline. + */ __u32 flags; #define I915_CONTEXT_CREATE_FLAGS_USE_EXTENSIONS (1u << 0) #define I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE (1u << 1) #define I915_CONTEXT_CREATE_FLAGS_UNKNOWN \ (-(I915_CONTEXT_CREATE_FLAGS_SINGLE_TIMELINE << 1)) + + /** + * @extensions: Zero-terminated chain of extensions. + * + * I915_CONTEXT_CREATE_EXT_SETPARAM: + * Context parameter to set or query during context creation. + * See struct drm_i915_gem_context_create_ext_setparam. + * + * I915_CONTEXT_CREATE_EXT_CLONE: + * This extension has been removed. On the off chance someone somewhere + * has attempted to use it, never re-use this extension number. + */ __u64 extensions; +#define I915_CONTEXT_CREATE_EXT_SETPARAM 0 +#define I915_CONTEXT_CREATE_EXT_CLONE 1 }; +/** + * struct drm_i915_gem_context_param - Context parameter to set or query. + */ struct drm_i915_gem_context_param { + /** @ctx_id: Context id */ __u32 ctx_id; + + /** @size: Size of the parameter @value */ __u32 size; + + /** @param: Parameter to set or query */ __u64 param; #define I915_CONTEXT_PARAM_BAN_PERIOD 0x1 /* I915_CONTEXT_PARAM_NO_ZEROMAP has been removed. On the off chance @@ -1910,6 +2078,7 @@ struct drm_i915_gem_context_param { #define I915_CONTEXT_PARAM_PROTECTED_CONTENT 0xd /* Must be kept compact -- no holes and well documented */ + /** @value: Context parameter value to be set or queried */ __u64 value; }; @@ -2060,7 +2229,7 @@ struct i915_context_engines_load_balance { __u64 mbz64; /* reserved for future use; must be zero */ - struct i915_engine_class_instance engines[0]; + struct i915_engine_class_instance engines[]; } __attribute__((packed)); #define I915_DEFINE_CONTEXT_ENGINES_LOAD_BALANCE(name__, N__) struct { \ @@ -2098,7 +2267,7 @@ struct i915_context_engines_bond { __u64 flags; /* all undefined flags must be zero */ __u64 mbz64[4]; /* reserved for future use; must be zero */ - struct i915_engine_class_instance engines[0]; + struct i915_engine_class_instance engines[]; } __attribute__((packed)); #define I915_DEFINE_CONTEXT_ENGINES_BOND(name__, N__) struct { \ @@ -2225,7 +2394,7 @@ struct i915_context_engines_parallel_submit { * length = width (i) * num_siblings (j) * index = j + i * num_siblings */ - struct i915_engine_class_instance engines[0]; + struct i915_engine_class_instance engines[]; } __packed; @@ -2308,23 +2477,29 @@ struct i915_context_param_engines { struct i915_engine_class_instance engines[N__]; \ } __attribute__((packed)) name__ +/** + * struct drm_i915_gem_context_create_ext_setparam - Context parameter + * to set or query during context creation. + */ struct drm_i915_gem_context_create_ext_setparam { -#define I915_CONTEXT_CREATE_EXT_SETPARAM 0 + /** @base: Extension link. See struct i915_user_extension. */ struct i915_user_extension base; + + /** + * @param: Context parameter to set or query. + * See struct drm_i915_gem_context_param. + */ struct drm_i915_gem_context_param param; }; -/* This API has been removed. On the off chance someone somewhere has - * attempted to use it, never re-use this extension number. - */ -#define I915_CONTEXT_CREATE_EXT_CLONE 1 - struct drm_i915_gem_context_destroy { __u32 ctx_id; __u32 pad; }; -/* +/** + * struct drm_i915_gem_vm_control - Structure to create or destroy VM. + * * DRM_I915_GEM_VM_CREATE - * * Create a new virtual memory address space (ppGTT) for use within a context @@ -2334,20 +2509,23 @@ struct drm_i915_gem_context_destroy { * The id of new VM (bound to the fd) for use with I915_CONTEXT_PARAM_VM is * returned in the outparam @id. * - * No flags are defined, with all bits reserved and must be zero. - * * An extension chain maybe provided, starting with @extensions, and terminated * by the @next_extension being 0. Currently, no extensions are defined. * * DRM_I915_GEM_VM_DESTROY - * - * Destroys a previously created VM id, specified in @id. + * Destroys a previously created VM id, specified in @vm_id. * * No extensions or flags are allowed currently, and so must be zero. */ struct drm_i915_gem_vm_control { + /** @extensions: Zero-terminated chain of extensions. */ __u64 extensions; + + /** @flags: reserved for future usage, currently MBZ */ __u32 flags; + + /** @vm_id: Id of the VM created or to be destroyed */ __u32 vm_id; }; @@ -2657,24 +2835,65 @@ enum drm_i915_perf_record_type { DRM_I915_PERF_RECORD_MAX /* non-ABI */ }; -/* +/** + * struct drm_i915_perf_oa_config + * * Structure to upload perf dynamic configuration into the kernel. */ struct drm_i915_perf_oa_config { - /** String formatted like "%08x-%04x-%04x-%04x-%012x" */ + /** + * @uuid: + * + * String formatted like "%\08x-%\04x-%\04x-%\04x-%\012x" + */ char uuid[36]; + /** + * @n_mux_regs: + * + * Number of mux regs in &mux_regs_ptr. + */ __u32 n_mux_regs; + + /** + * @n_boolean_regs: + * + * Number of boolean regs in &boolean_regs_ptr. + */ __u32 n_boolean_regs; + + /** + * @n_flex_regs: + * + * Number of flex regs in &flex_regs_ptr. + */ __u32 n_flex_regs; - /* - * These fields are pointers to tuples of u32 values (register address, - * value). For example the expected length of the buffer pointed by - * mux_regs_ptr is (2 * sizeof(u32) * n_mux_regs). + /** + * @mux_regs_ptr: + * + * Pointer to tuples of u32 values (register address, value) for mux + * registers. Expected length of buffer is (2 * sizeof(u32) * + * &n_mux_regs). */ __u64 mux_regs_ptr; + + /** + * @boolean_regs_ptr: + * + * Pointer to tuples of u32 values (register address, value) for mux + * registers. Expected length of buffer is (2 * sizeof(u32) * + * &n_boolean_regs). + */ __u64 boolean_regs_ptr; + + /** + * @flex_regs_ptr: + * + * Pointer to tuples of u32 values (register address, value) for mux + * registers. Expected length of buffer is (2 * sizeof(u32) * + * &n_flex_regs). + */ __u64 flex_regs_ptr; }; @@ -2685,12 +2904,24 @@ struct drm_i915_perf_oa_config { * @data_ptr is also depends on the specific @query_id. */ struct drm_i915_query_item { - /** @query_id: The id for this query */ + /** + * @query_id: + * + * The id for this query. Currently accepted query IDs are: + * - %DRM_I915_QUERY_TOPOLOGY_INFO (see struct drm_i915_query_topology_info) + * - %DRM_I915_QUERY_ENGINE_INFO (see struct drm_i915_engine_info) + * - %DRM_I915_QUERY_PERF_CONFIG (see struct drm_i915_query_perf_config) + * - %DRM_I915_QUERY_MEMORY_REGIONS (see struct drm_i915_query_memory_regions) + * - %DRM_I915_QUERY_HWCONFIG_BLOB (see `GuC HWCONFIG blob uAPI`) + * - %DRM_I915_QUERY_GEOMETRY_SUBSLICES (see struct drm_i915_query_topology_info) + */ __u64 query_id; -#define DRM_I915_QUERY_TOPOLOGY_INFO 1 -#define DRM_I915_QUERY_ENGINE_INFO 2 -#define DRM_I915_QUERY_PERF_CONFIG 3 -#define DRM_I915_QUERY_MEMORY_REGIONS 4 +#define DRM_I915_QUERY_TOPOLOGY_INFO 1 +#define DRM_I915_QUERY_ENGINE_INFO 2 +#define DRM_I915_QUERY_PERF_CONFIG 3 +#define DRM_I915_QUERY_MEMORY_REGIONS 4 +#define DRM_I915_QUERY_HWCONFIG_BLOB 5 +#define DRM_I915_QUERY_GEOMETRY_SUBSLICES 6 /* Must be kept compact -- no holes and well documented */ /** @@ -2706,14 +2937,17 @@ struct drm_i915_query_item { /** * @flags: * - * When query_id == DRM_I915_QUERY_TOPOLOGY_INFO, must be 0. + * When &query_id == %DRM_I915_QUERY_TOPOLOGY_INFO, must be 0. * - * When query_id == DRM_I915_QUERY_PERF_CONFIG, must be one of the + * When &query_id == %DRM_I915_QUERY_PERF_CONFIG, must be one of the * following: * - * - DRM_I915_QUERY_PERF_CONFIG_LIST - * - DRM_I915_QUERY_PERF_CONFIG_DATA_FOR_UUID - * - DRM_I915_QUERY_PERF_CONFIG_FOR_UUID + * - %DRM_I915_QUERY_PERF_CONFIG_LIST + * - %DRM_I915_QUERY_PERF_CONFIG_DATA_FOR_UUID + * - %DRM_I915_QUERY_PERF_CONFIG_FOR_UUID + * + * When &query_id == %DRM_I915_QUERY_GEOMETRY_SUBSLICES must contain + * a struct i915_engine_class_instance that references a render engine. */ __u32 flags; #define DRM_I915_QUERY_PERF_CONFIG_LIST 1 @@ -2771,66 +3005,112 @@ struct drm_i915_query { __u64 items_ptr; }; -/* - * Data written by the kernel with query DRM_I915_QUERY_TOPOLOGY_INFO : +/** + * struct drm_i915_query_topology_info * - * data: contains the 3 pieces of information : - * - * - the slice mask with one bit per slice telling whether a slice is - * available. The availability of slice X can be queried with the following - * formula : - * - * (data[X / 8] >> (X % 8)) & 1 - * - * - the subslice mask for each slice with one bit per subslice telling - * whether a subslice is available. Gen12 has dual-subslices, which are - * similar to two gen11 subslices. For gen12, this array represents dual- - * subslices. The availability of subslice Y in slice X can be queried - * with the following formula : - * - * (data[subslice_offset + - * X * subslice_stride + - * Y / 8] >> (Y % 8)) & 1 - * - * - the EU mask for each subslice in each slice with one bit per EU telling - * whether an EU is available. The availability of EU Z in subslice Y in - * slice X can be queried with the following formula : - * - * (data[eu_offset + - * (X * max_subslices + Y) * eu_stride + - * Z / 8] >> (Z % 8)) & 1 + * Describes slice/subslice/EU information queried by + * %DRM_I915_QUERY_TOPOLOGY_INFO */ struct drm_i915_query_topology_info { - /* + /** + * @flags: + * * Unused for now. Must be cleared to zero. */ __u16 flags; + /** + * @max_slices: + * + * The number of bits used to express the slice mask. + */ __u16 max_slices; + + /** + * @max_subslices: + * + * The number of bits used to express the subslice mask. + */ __u16 max_subslices; + + /** + * @max_eus_per_subslice: + * + * The number of bits in the EU mask that correspond to a single + * subslice's EUs. + */ __u16 max_eus_per_subslice; - /* + /** + * @subslice_offset: + * * Offset in data[] at which the subslice masks are stored. */ __u16 subslice_offset; - /* + /** + * @subslice_stride: + * * Stride at which each of the subslice masks for each slice are * stored. */ __u16 subslice_stride; - /* + /** + * @eu_offset: + * * Offset in data[] at which the EU masks are stored. */ __u16 eu_offset; - /* + /** + * @eu_stride: + * * Stride at which each of the EU masks for each subslice are stored. */ __u16 eu_stride; + /** + * @data: + * + * Contains 3 pieces of information : + * + * - The slice mask with one bit per slice telling whether a slice is + * available. The availability of slice X can be queried with the + * following formula : + * + * .. code:: c + * + * (data[X / 8] >> (X % 8)) & 1 + * + * Starting with Xe_HP platforms, Intel hardware no longer has + * traditional slices so i915 will always report a single slice + * (hardcoded slicemask = 0x1) which contains all of the platform's + * subslices. I.e., the mask here does not reflect any of the newer + * hardware concepts such as "gslices" or "cslices" since userspace + * is capable of inferring those from the subslice mask. + * + * - The subslice mask for each slice with one bit per subslice telling + * whether a subslice is available. Starting with Gen12 we use the + * term "subslice" to refer to what the hardware documentation + * describes as a "dual-subslices." The availability of subslice Y + * in slice X can be queried with the following formula : + * + * .. code:: c + * + * (data[subslice_offset + X * subslice_stride + Y / 8] >> (Y % 8)) & 1 + * + * - The EU mask for each subslice in each slice, with one bit per EU + * telling whether an EU is available. The availability of EU Z in + * subslice Y in slice X can be queried with the following formula : + * + * .. code:: c + * + * (data[eu_offset + + * (X * max_subslices + Y) * eu_stride + + * Z / 8 + * ] >> (Z % 8)) & 1 + */ __u8 data[]; }; @@ -2951,52 +3231,68 @@ struct drm_i915_query_engine_info { struct drm_i915_engine_info engines[]; }; -/* - * Data written by the kernel with query DRM_I915_QUERY_PERF_CONFIG. +/** + * struct drm_i915_query_perf_config + * + * Data written by the kernel with query %DRM_I915_QUERY_PERF_CONFIG and + * %DRM_I915_QUERY_GEOMETRY_SUBSLICES. */ struct drm_i915_query_perf_config { union { - /* - * When query_item.flags == DRM_I915_QUERY_PERF_CONFIG_LIST, i915 sets - * this fields to the number of configurations available. + /** + * @n_configs: + * + * When &drm_i915_query_item.flags == + * %DRM_I915_QUERY_PERF_CONFIG_LIST, i915 sets this fields to + * the number of configurations available. */ __u64 n_configs; - /* - * When query_id == DRM_I915_QUERY_PERF_CONFIG_DATA_FOR_ID, - * i915 will use the value in this field as configuration - * identifier to decide what data to write into config_ptr. + /** + * @config: + * + * When &drm_i915_query_item.flags == + * %DRM_I915_QUERY_PERF_CONFIG_DATA_FOR_ID, i915 will use the + * value in this field as configuration identifier to decide + * what data to write into config_ptr. */ __u64 config; - /* - * When query_id == DRM_I915_QUERY_PERF_CONFIG_DATA_FOR_UUID, - * i915 will use the value in this field as configuration - * identifier to decide what data to write into config_ptr. + /** + * @uuid: + * + * When &drm_i915_query_item.flags == + * %DRM_I915_QUERY_PERF_CONFIG_DATA_FOR_UUID, i915 will use the + * value in this field as configuration identifier to decide + * what data to write into config_ptr. * * String formatted like "%08x-%04x-%04x-%04x-%012x" */ char uuid[36]; }; - /* + /** + * @flags: + * * Unused for now. Must be cleared to zero. */ __u32 flags; - /* - * When query_item.flags == DRM_I915_QUERY_PERF_CONFIG_LIST, i915 will - * write an array of __u64 of configuration identifiers. + /** + * @data: * - * When query_item.flags == DRM_I915_QUERY_PERF_CONFIG_DATA, i915 will - * write a struct drm_i915_perf_oa_config. If the following fields of - * drm_i915_perf_oa_config are set not set to 0, i915 will write into - * the associated pointers the values of submitted when the + * When &drm_i915_query_item.flags == %DRM_I915_QUERY_PERF_CONFIG_LIST, + * i915 will write an array of __u64 of configuration identifiers. + * + * When &drm_i915_query_item.flags == %DRM_I915_QUERY_PERF_CONFIG_DATA, + * i915 will write a struct drm_i915_perf_oa_config. If the following + * fields of struct drm_i915_perf_oa_config are not set to 0, i915 will + * write into the associated pointers the values of submitted when the * configuration was created : * - * - n_mux_regs - * - n_boolean_regs - * - n_flex_regs + * - &drm_i915_perf_oa_config.n_mux_regs + * - &drm_i915_perf_oa_config.n_boolean_regs + * - &drm_i915_perf_oa_config.n_flex_regs */ __u8 data[]; }; @@ -3026,36 +3322,6 @@ struct drm_i915_gem_memory_class_instance { * struct drm_i915_memory_region_info - Describes one region as known to the * driver. * - * Note that we reserve some stuff here for potential future work. As an example - * we might want expose the capabilities for a given region, which could include - * things like if the region is CPU mappable/accessible, what are the supported - * mapping types etc. - * - * Note that to extend struct drm_i915_memory_region_info and struct - * drm_i915_query_memory_regions in the future the plan is to do the following: - * - * .. code-block:: C - * - * struct drm_i915_memory_region_info { - * struct drm_i915_gem_memory_class_instance region; - * union { - * __u32 rsvd0; - * __u32 new_thing1; - * }; - * ... - * union { - * __u64 rsvd1[8]; - * struct { - * __u64 new_thing2; - * __u64 new_thing3; - * ... - * }; - * }; - * }; - * - * With this things should remain source compatible between versions for - * userspace, even as we add new fields. - * * Note this is using both struct drm_i915_query_item and struct drm_i915_query. * For this new query we are adding the new query id DRM_I915_QUERY_MEMORY_REGIONS * at &drm_i915_query_item.query_id. @@ -3067,14 +3333,81 @@ struct drm_i915_memory_region_info { /** @rsvd0: MBZ */ __u32 rsvd0; - /** @probed_size: Memory probed by the driver (-1 = unknown) */ + /** + * @probed_size: Memory probed by the driver + * + * Note that it should not be possible to ever encounter a zero value + * here, also note that no current region type will ever return -1 here. + * Although for future region types, this might be a possibility. The + * same applies to the other size fields. + */ __u64 probed_size; - /** @unallocated_size: Estimate of memory remaining (-1 = unknown) */ + /** + * @unallocated_size: Estimate of memory remaining + * + * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable accounting. + * Without this (or if this is an older kernel) the value here will + * always equal the @probed_size. Note this is only currently tracked + * for I915_MEMORY_CLASS_DEVICE regions (for other types the value here + * will always equal the @probed_size). + */ __u64 unallocated_size; - /** @rsvd1: MBZ */ - __u64 rsvd1[8]; + union { + /** @rsvd1: MBZ */ + __u64 rsvd1[8]; + struct { + /** + * @probed_cpu_visible_size: Memory probed by the driver + * that is CPU accessible. + * + * This will be always be <= @probed_size, and the + * remainder (if there is any) will not be CPU + * accessible. + * + * On systems without small BAR, the @probed_size will + * always equal the @probed_cpu_visible_size, since all + * of it will be CPU accessible. + * + * Note this is only tracked for + * I915_MEMORY_CLASS_DEVICE regions (for other types the + * value here will always equal the @probed_size). + * + * Note that if the value returned here is zero, then + * this must be an old kernel which lacks the relevant + * small-bar uAPI support (including + * I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS), but on + * such systems we should never actually end up with a + * small BAR configuration, assuming we are able to load + * the kernel module. Hence it should be safe to treat + * this the same as when @probed_cpu_visible_size == + * @probed_size. + */ + __u64 probed_cpu_visible_size; + + /** + * @unallocated_cpu_visible_size: Estimate of CPU + * visible memory remaining. + * + * Note this is only tracked for + * I915_MEMORY_CLASS_DEVICE regions (for other types the + * value here will always equal the + * @probed_cpu_visible_size). + * + * Requires CAP_PERFMON or CAP_SYS_ADMIN to get reliable + * accounting. Without this the value here will always + * equal the @probed_cpu_visible_size. Note this is only + * currently tracked for I915_MEMORY_CLASS_DEVICE + * regions (for other types the value here will also + * always equal the @probed_cpu_visible_size). + * + * If this is an older kernel the value here will be + * zero, see also @probed_cpu_visible_size. + */ + __u64 unallocated_cpu_visible_size; + }; + }; }; /** @@ -3134,15 +3467,25 @@ struct drm_i915_query_memory_regions { struct drm_i915_memory_region_info regions[]; }; +/** + * DOC: GuC HWCONFIG blob uAPI + * + * The GuC produces a blob with information about the current device. + * i915 reads this blob from GuC and makes it available via this uAPI. + * + * The format and meaning of the blob content are documented in the + * Programmer's Reference Manual. + */ + /** * struct drm_i915_gem_create_ext - Existing gem_create behaviour, with added * extension support using struct i915_user_extension. * - * Note that in the future we want to have our buffer flags here, at least for - * the stuff that is immutable. Previously we would have two ioctls, one to - * create the object with gem_create, and another to apply various parameters, - * however this creates some ambiguity for the params which are considered - * immutable. Also in general we're phasing out the various SET/GET ioctls. + * Note that new buffer flags should be added here, at least for the stuff that + * is immutable. Previously we would have two ioctls, one to create the object + * with gem_create, and another to apply various parameters, however this + * creates some ambiguity for the params which are considered immutable. Also in + * general we're phasing out the various SET/GET ioctls. */ struct drm_i915_gem_create_ext { /** @@ -3150,7 +3493,6 @@ struct drm_i915_gem_create_ext { * * The (page-aligned) allocated size for the object will be returned. * - * * DG2 64K min page size implications: * * On discrete platforms, starting from DG2, we have to contend with GTT @@ -3162,7 +3504,9 @@ struct drm_i915_gem_create_ext { * * Note that the returned size here will always reflect any required * rounding up done by the kernel, i.e 4K will now become 64K on devices - * such as DG2. + * such as DG2. The kernel will always select the largest minimum + * page-size for the set of possible placements as the value to use when + * rounding up the @size. * * Special DG2 GTT address alignment requirement: * @@ -3186,14 +3530,58 @@ struct drm_i915_gem_create_ext { * is deemed to be a good compromise. */ __u64 size; + /** * @handle: Returned handle for the object. * * Object handles are nonzero. */ __u32 handle; - /** @flags: MBZ */ + + /** + * @flags: Optional flags. + * + * Supported values: + * + * I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS - Signal to the kernel that + * the object will need to be accessed via the CPU. + * + * Only valid when placing objects in I915_MEMORY_CLASS_DEVICE, and only + * strictly required on configurations where some subset of the device + * memory is directly visible/mappable through the CPU (which we also + * call small BAR), like on some DG2+ systems. Note that this is quite + * undesirable, but due to various factors like the client CPU, BIOS etc + * it's something we can expect to see in the wild. See + * &drm_i915_memory_region_info.probed_cpu_visible_size for how to + * determine if this system applies. + * + * Note that one of the placements MUST be I915_MEMORY_CLASS_SYSTEM, to + * ensure the kernel can always spill the allocation to system memory, + * if the object can't be allocated in the mappable part of + * I915_MEMORY_CLASS_DEVICE. + * + * Also note that since the kernel only supports flat-CCS on objects + * that can *only* be placed in I915_MEMORY_CLASS_DEVICE, we therefore + * don't support I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS together with + * flat-CCS. + * + * Without this hint, the kernel will assume that non-mappable + * I915_MEMORY_CLASS_DEVICE is preferred for this object. Note that the + * kernel can still migrate the object to the mappable part, as a last + * resort, if userspace ever CPU faults this object, but this might be + * expensive, and so ideally should be avoided. + * + * On older kernels which lack the relevant small-bar uAPI support (see + * also &drm_i915_memory_region_info.probed_cpu_visible_size), + * usage of the flag will result in an error, but it should NEVER be + * possible to end up with a small BAR configuration, assuming we can + * also successfully load the i915 kernel module. In such cases the + * entire I915_MEMORY_CLASS_DEVICE region will be CPU accessible, and as + * such there are zero restrictions on where the object can be placed. + */ +#define I915_GEM_CREATE_EXT_FLAG_NEEDS_CPU_ACCESS (1 << 0) __u32 flags; + /** * @extensions: The chain of extensions to apply to this object. * @@ -3252,6 +3640,22 @@ struct drm_i915_gem_create_ext { * At which point we get the object handle in &drm_i915_gem_create_ext.handle, * along with the final object size in &drm_i915_gem_create_ext.size, which * should account for any rounding up, if required. + * + * Note that userspace has no means of knowing the current backing region + * for objects where @num_regions is larger than one. The kernel will only + * ensure that the priority order of the @regions array is honoured, either + * when initially placing the object, or when moving memory around due to + * memory pressure + * + * On Flat-CCS capable HW, compression is supported for the objects residing + * in I915_MEMORY_CLASS_DEVICE. When such objects (compressed) have other + * memory class in @regions and migrated (by i915, due to memory + * constraints) to the non I915_MEMORY_CLASS_DEVICE region, then i915 needs to + * decompress the content. But i915 doesn't have the required information to + * decompress the userspace compressed objects. + * + * So i915 supports Flat-CCS, on the objects which can reside only on + * I915_MEMORY_CLASS_DEVICE regions. */ struct drm_i915_gem_create_ext_memory_regions { /** @base: Extension link. See struct i915_user_extension. */ diff --git a/tools/include/uapi/linux/bpf.h b/tools/include/uapi/linux/bpf.h index d14b10b85e..59a217ca2d 100644 --- a/tools/include/uapi/linux/bpf.h +++ b/tools/include/uapi/linux/bpf.h @@ -998,6 +998,7 @@ enum bpf_attach_type { BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, BPF_PERF_EVENT, BPF_TRACE_KPROBE_MULTI, + BPF_LSM_CGROUP, __MAX_BPF_ATTACH_TYPE }; @@ -1013,6 +1014,7 @@ enum bpf_link_type { BPF_LINK_TYPE_XDP = 6, BPF_LINK_TYPE_PERF_EVENT = 7, BPF_LINK_TYPE_KPROBE_MULTI = 8, + BPF_LINK_TYPE_STRUCT_OPS = 9, MAX_BPF_LINK_TYPE, }; @@ -1430,6 +1432,7 @@ union bpf_attr { __u32 attach_flags; __aligned_u64 prog_ids; __u32 prog_cnt; + __aligned_u64 prog_attach_flags; /* output: per-program attach_flags */ } query; struct { /* anonymous struct used by BPF_RAW_TRACEPOINT_OPEN command */ @@ -1489,6 +1492,15 @@ union bpf_attr { __aligned_u64 addrs; __aligned_u64 cookies; } kprobe_multi; + struct { + /* this is overlaid with the target_btf_id above. */ + __u32 target_btf_id; + /* black box user-provided value passed through + * to BPF program at the execution time and + * accessible through bpf_get_attach_cookie() BPF helper + */ + __u64 cookie; + } tracing; }; } link_create; @@ -2349,7 +2361,8 @@ union bpf_attr { * Pull in non-linear data in case the *skb* is non-linear and not * all of *len* are part of the linear section. Make *len* bytes * from *skb* readable and writable. If a zero value is passed for - * *len*, then the whole length of the *skb* is pulled. + * *len*, then all bytes in the linear part of *skb* will be made + * readable and writable. * * This helper is only needed for reading and writing with direct * packet access. @@ -3587,10 +3600,11 @@ union bpf_attr { * * *iph* points to the start of the IPv4 or IPv6 header, while * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ip6hdr**). + * **sizeof**\ (**struct ipv6hdr**). * * *th* points to the start of the TCP header, while *th_len* - * contains **sizeof**\ (**struct tcphdr**). + * contains the length of the TCP header (at least + * **sizeof**\ (**struct tcphdr**)). * Return * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative * error otherwise. @@ -3773,10 +3787,11 @@ union bpf_attr { * * *iph* points to the start of the IPv4 or IPv6 header, while * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ip6hdr**). + * **sizeof**\ (**struct ipv6hdr**). * * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header. + * contains the length of the TCP header with options (at least + * **sizeof**\ (**struct tcphdr**)). * Return * On success, lower 32 bits hold the generated SYN cookie in * followed by 16 bits which hold the MSS value for that cookie, @@ -5143,6 +5158,179 @@ union bpf_attr { * The **hash_algo** is returned on success, * **-EOPNOTSUP** if the hash calculation failed or **-EINVAL** if * invalid arguments are passed. + * + * void *bpf_kptr_xchg(void *map_value, void *ptr) + * Description + * Exchange kptr at pointer *map_value* with *ptr*, and return the + * old value. *ptr* can be NULL, otherwise it must be a referenced + * pointer which will be released when this helper is called. + * Return + * The old value of kptr (which can be NULL). The returned pointer + * if not NULL, is a reference which must be released using its + * corresponding release function, or moved into a BPF map before + * program exit. + * + * void *bpf_map_lookup_percpu_elem(struct bpf_map *map, const void *key, u32 cpu) + * Description + * Perform a lookup in *percpu map* for an entry associated to + * *key* on *cpu*. + * Return + * Map value associated to *key* on *cpu*, or **NULL** if no entry + * was found or *cpu* is invalid. + * + * struct mptcp_sock *bpf_skc_to_mptcp_sock(void *sk) + * Description + * Dynamically cast a *sk* pointer to a *mptcp_sock* pointer. + * Return + * *sk* if casting is valid, or **NULL** otherwise. + * + * long bpf_dynptr_from_mem(void *data, u32 size, u64 flags, struct bpf_dynptr *ptr) + * Description + * Get a dynptr to local memory *data*. + * + * *data* must be a ptr to a map value. + * The maximum *size* supported is DYNPTR_MAX_SIZE. + * *flags* is currently unused. + * Return + * 0 on success, -E2BIG if the size exceeds DYNPTR_MAX_SIZE, + * -EINVAL if flags is not 0. + * + * long bpf_ringbuf_reserve_dynptr(void *ringbuf, u32 size, u64 flags, struct bpf_dynptr *ptr) + * Description + * Reserve *size* bytes of payload in a ring buffer *ringbuf* + * through the dynptr interface. *flags* must be 0. + * + * Please note that a corresponding bpf_ringbuf_submit_dynptr or + * bpf_ringbuf_discard_dynptr must be called on *ptr*, even if the + * reservation fails. This is enforced by the verifier. + * Return + * 0 on success, or a negative error in case of failure. + * + * void bpf_ringbuf_submit_dynptr(struct bpf_dynptr *ptr, u64 flags) + * Description + * Submit reserved ring buffer sample, pointed to by *data*, + * through the dynptr interface. This is a no-op if the dynptr is + * invalid/null. + * + * For more information on *flags*, please see + * 'bpf_ringbuf_submit'. + * Return + * Nothing. Always succeeds. + * + * void bpf_ringbuf_discard_dynptr(struct bpf_dynptr *ptr, u64 flags) + * Description + * Discard reserved ring buffer sample through the dynptr + * interface. This is a no-op if the dynptr is invalid/null. + * + * For more information on *flags*, please see + * 'bpf_ringbuf_discard'. + * Return + * Nothing. Always succeeds. + * + * long bpf_dynptr_read(void *dst, u32 len, struct bpf_dynptr *src, u32 offset, u64 flags) + * Description + * Read *len* bytes from *src* into *dst*, starting from *offset* + * into *src*. + * *flags* is currently unused. + * Return + * 0 on success, -E2BIG if *offset* + *len* exceeds the length + * of *src*'s data, -EINVAL if *src* is an invalid dynptr or if + * *flags* is not 0. + * + * long bpf_dynptr_write(struct bpf_dynptr *dst, u32 offset, void *src, u32 len, u64 flags) + * Description + * Write *len* bytes from *src* into *dst*, starting from *offset* + * into *dst*. + * *flags* is currently unused. + * Return + * 0 on success, -E2BIG if *offset* + *len* exceeds the length + * of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst* + * is a read-only dynptr or if *flags* is not 0. + * + * void *bpf_dynptr_data(struct bpf_dynptr *ptr, u32 offset, u32 len) + * Description + * Get a pointer to the underlying dynptr data. + * + * *len* must be a statically known value. The returned data slice + * is invalidated whenever the dynptr is invalidated. + * Return + * Pointer to the underlying dynptr data, NULL if the dynptr is + * read-only, if the dynptr is invalid, or if the offset and length + * is out of bounds. + * + * s64 bpf_tcp_raw_gen_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th, u32 th_len) + * Description + * Try to issue a SYN cookie for the packet with corresponding + * IPv4/TCP headers, *iph* and *th*, without depending on a + * listening socket. + * + * *iph* points to the IPv4 header. + * + * *th* points to the start of the TCP header, while *th_len* + * contains the length of the TCP header (at least + * **sizeof**\ (**struct tcphdr**)). + * Return + * On success, lower 32 bits hold the generated SYN cookie in + * followed by 16 bits which hold the MSS value for that cookie, + * and the top 16 bits are unused. + * + * On failure, the returned value is one of the following: + * + * **-EINVAL** if *th_len* is invalid. + * + * s64 bpf_tcp_raw_gen_syncookie_ipv6(struct ipv6hdr *iph, struct tcphdr *th, u32 th_len) + * Description + * Try to issue a SYN cookie for the packet with corresponding + * IPv6/TCP headers, *iph* and *th*, without depending on a + * listening socket. + * + * *iph* points to the IPv6 header. + * + * *th* points to the start of the TCP header, while *th_len* + * contains the length of the TCP header (at least + * **sizeof**\ (**struct tcphdr**)). + * Return + * On success, lower 32 bits hold the generated SYN cookie in + * followed by 16 bits which hold the MSS value for that cookie, + * and the top 16 bits are unused. + * + * On failure, the returned value is one of the following: + * + * **-EINVAL** if *th_len* is invalid. + * + * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. + * + * long bpf_tcp_raw_check_syncookie_ipv4(struct iphdr *iph, struct tcphdr *th) + * Description + * Check whether *iph* and *th* contain a valid SYN cookie ACK + * without depending on a listening socket. + * + * *iph* points to the IPv4 header. + * + * *th* points to the TCP header. + * Return + * 0 if *iph* and *th* are a valid SYN cookie ACK. + * + * On failure, the returned value is one of the following: + * + * **-EACCES** if the SYN cookie is not valid. + * + * long bpf_tcp_raw_check_syncookie_ipv6(struct ipv6hdr *iph, struct tcphdr *th) + * Description + * Check whether *iph* and *th* contain a valid SYN cookie ACK + * without depending on a listening socket. + * + * *iph* points to the IPv6 header. + * + * *th* points to the TCP header. + * Return + * 0 if *iph* and *th* are a valid SYN cookie ACK. + * + * On failure, the returned value is one of the following: + * + * **-EACCES** if the SYN cookie is not valid. + * + * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. */ #define __BPF_FUNC_MAPPER(FN) \ FN(unspec), \ @@ -5339,6 +5527,20 @@ union bpf_attr { FN(copy_from_user_task), \ FN(skb_set_tstamp), \ FN(ima_file_hash), \ + FN(kptr_xchg), \ + FN(map_lookup_percpu_elem), \ + FN(skc_to_mptcp_sock), \ + FN(dynptr_from_mem), \ + FN(ringbuf_reserve_dynptr), \ + FN(ringbuf_submit_dynptr), \ + FN(ringbuf_discard_dynptr), \ + FN(dynptr_read), \ + FN(dynptr_write), \ + FN(dynptr_data), \ + FN(tcp_raw_gen_syncookie_ipv4), \ + FN(tcp_raw_gen_syncookie_ipv6), \ + FN(tcp_raw_check_syncookie_ipv4), \ + FN(tcp_raw_check_syncookie_ipv6), \ /* */ /* integer value in 'imm' field of BPF_CALL instruction selects which helper @@ -5592,6 +5794,10 @@ struct bpf_tunnel_key { __u8 tunnel_ttl; __u16 tunnel_ext; /* Padding, future use. */ __u32 tunnel_label; + union { + __u32 local_ipv4; + __u32 local_ipv6[4]; + }; }; /* user accessible mirror of in-kernel xfrm_state. @@ -5875,6 +6081,8 @@ struct bpf_prog_info { __u64 run_cnt; __u64 recursion_misses; __u32 verified_insns; + __u32 attach_btf_obj_id; + __u32 attach_btf_id; } __attribute__((aligned(8))); struct bpf_map_info { @@ -6486,6 +6694,11 @@ struct bpf_timer { __u64 :64; } __attribute__((aligned(8))); +struct bpf_dynptr { + __u64 :64; + __u64 :64; +} __attribute__((aligned(8))); + struct bpf_sysctl { __u32 write; /* Sysctl is being read (= 0) or written (= 1). * Allows 1,2,4-byte read, but no write. @@ -6577,6 +6790,7 @@ enum bpf_core_relo_kind { BPF_CORE_TYPE_SIZE = 9, /* type size in bytes */ BPF_CORE_ENUMVAL_EXISTS = 10, /* enum value existence in target kernel */ BPF_CORE_ENUMVAL_VALUE = 11, /* enum value integer value */ + BPF_CORE_TYPE_MATCHES = 12, /* type match in target kernel */ }; /* diff --git a/tools/include/uapi/linux/btf.h b/tools/include/uapi/linux/btf.h index b0d8fea195..ec1798b6d3 100644 --- a/tools/include/uapi/linux/btf.h +++ b/tools/include/uapi/linux/btf.h @@ -33,13 +33,13 @@ struct btf_type { /* "info" bits arrangement * bits 0-15: vlen (e.g. # of struct's members) * bits 16-23: unused - * bits 24-27: kind (e.g. int, ptr, array...etc) - * bits 28-30: unused + * bits 24-28: kind (e.g. int, ptr, array...etc) + * bits 29-30: unused * bit 31: kind_flag, currently used by - * struct, union and fwd + * struct, union, enum, fwd and enum64 */ __u32 info; - /* "size" is used by INT, ENUM, STRUCT, UNION and DATASEC. + /* "size" is used by INT, ENUM, STRUCT, UNION, DATASEC and ENUM64. * "size" tells the size of the type it is describing. * * "type" is used by PTR, TYPEDEF, VOLATILE, CONST, RESTRICT, @@ -63,7 +63,7 @@ enum { BTF_KIND_ARRAY = 3, /* Array */ BTF_KIND_STRUCT = 4, /* Struct */ BTF_KIND_UNION = 5, /* Union */ - BTF_KIND_ENUM = 6, /* Enumeration */ + BTF_KIND_ENUM = 6, /* Enumeration up to 32-bit values */ BTF_KIND_FWD = 7, /* Forward */ BTF_KIND_TYPEDEF = 8, /* Typedef */ BTF_KIND_VOLATILE = 9, /* Volatile */ @@ -76,6 +76,7 @@ enum { BTF_KIND_FLOAT = 16, /* Floating point */ BTF_KIND_DECL_TAG = 17, /* Decl Tag */ BTF_KIND_TYPE_TAG = 18, /* Type Tag */ + BTF_KIND_ENUM64 = 19, /* Enumeration up to 64-bit values */ NR_BTF_KINDS, BTF_KIND_MAX = NR_BTF_KINDS - 1, @@ -186,4 +187,14 @@ struct btf_decl_tag { __s32 component_idx; }; +/* BTF_KIND_ENUM64 is followed by multiple "struct btf_enum64". + * The exact number of btf_enum64 is stored in the vlen (of the + * info in "struct btf_type"). + */ +struct btf_enum64 { + __u32 name_off; + __u32 val_lo32; + __u32 val_hi32; +}; + #endif /* _UAPI__LINUX_BTF_H__ */ diff --git a/tools/include/uapi/linux/fs.h b/tools/include/uapi/linux/fs.h index bdf7b404b3..b7b5687102 100644 --- a/tools/include/uapi/linux/fs.h +++ b/tools/include/uapi/linux/fs.h @@ -90,7 +90,7 @@ struct file_dedupe_range { __u16 dest_count; /* in - total elements in info array */ __u16 reserved1; /* must be zero */ __u32 reserved2; /* must be zero */ - struct file_dedupe_range_info info[0]; + struct file_dedupe_range_info info[]; }; /* And dynamically-tunable limits and defaults: */ diff --git a/tools/include/uapi/linux/fscrypt.h b/tools/include/uapi/linux/fscrypt.h index 9f4428be3e..a756b29afc 100644 --- a/tools/include/uapi/linux/fscrypt.h +++ b/tools/include/uapi/linux/fscrypt.h @@ -27,7 +27,8 @@ #define FSCRYPT_MODE_AES_128_CBC 5 #define FSCRYPT_MODE_AES_128_CTS 6 #define FSCRYPT_MODE_ADIANTUM 9 -/* If adding a mode number > 9, update FSCRYPT_MODE_MAX in fscrypt_private.h */ +#define FSCRYPT_MODE_AES_256_HCTR2 10 +/* If adding a mode number > 10, update FSCRYPT_MODE_MAX in fscrypt_private.h */ /* * Legacy policy version; ad-hoc KDF and no key verification. diff --git a/tools/include/uapi/linux/if_link.h b/tools/include/uapi/linux/if_link.h index e1ba2d51b7..0242f31e33 100644 --- a/tools/include/uapi/linux/if_link.h +++ b/tools/include/uapi/linux/if_link.h @@ -348,6 +348,8 @@ enum { IFLA_PARENT_DEV_NAME, IFLA_PARENT_DEV_BUS_NAME, IFLA_GRO_MAX_SIZE, + IFLA_TSO_MAX_SIZE, + IFLA_TSO_MAX_SEGS, __IFLA_MAX }; @@ -888,6 +890,7 @@ enum { IFLA_BOND_SLAVE_AD_AGGREGATOR_ID, IFLA_BOND_SLAVE_AD_ACTOR_OPER_PORT_STATE, IFLA_BOND_SLAVE_AD_PARTNER_OPER_PORT_STATE, + IFLA_BOND_SLAVE_PRIO, __IFLA_BOND_SLAVE_MAX, }; diff --git a/tools/include/uapi/linux/if_tun.h b/tools/include/uapi/linux/if_tun.h index 454ae31b93..2ec07de1d7 100644 --- a/tools/include/uapi/linux/if_tun.h +++ b/tools/include/uapi/linux/if_tun.h @@ -108,7 +108,7 @@ struct tun_pi { struct tun_filter { __u16 flags; /* TUN_FLT_ flags see above */ __u16 count; /* Number of addresses */ - __u8 addr[0][ETH_ALEN]; + __u8 addr[][ETH_ALEN]; }; #endif /* _UAPI__IF_TUN_H */ diff --git a/tools/include/uapi/linux/kvm.h b/tools/include/uapi/linux/kvm.h index 91a6fe4e02..eed0315a77 100644 --- a/tools/include/uapi/linux/kvm.h +++ b/tools/include/uapi/linux/kvm.h @@ -270,6 +270,8 @@ struct kvm_xen_exit { #define KVM_EXIT_X86_BUS_LOCK 33 #define KVM_EXIT_XEN 34 #define KVM_EXIT_RISCV_SBI 35 +#define KVM_EXIT_RISCV_CSR 36 +#define KVM_EXIT_NOTIFY 37 /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -444,8 +446,17 @@ struct kvm_run { #define KVM_SYSTEM_EVENT_SHUTDOWN 1 #define KVM_SYSTEM_EVENT_RESET 2 #define KVM_SYSTEM_EVENT_CRASH 3 +#define KVM_SYSTEM_EVENT_WAKEUP 4 +#define KVM_SYSTEM_EVENT_SUSPEND 5 +#define KVM_SYSTEM_EVENT_SEV_TERM 6 __u32 type; - __u64 flags; + __u32 ndata; + union { +#ifndef __KERNEL__ + __u64 flags; +#endif + __u64 data[16]; + }; } system_event; /* KVM_EXIT_S390_STSI */ struct { @@ -487,6 +498,18 @@ struct kvm_run { unsigned long args[6]; unsigned long ret[2]; } riscv_sbi; + /* KVM_EXIT_RISCV_CSR */ + struct { + unsigned long csr_num; + unsigned long new_value; + unsigned long write_mask; + unsigned long ret_value; + } riscv_csr; + /* KVM_EXIT_NOTIFY */ + struct { +#define KVM_NOTIFY_CONTEXT_INVALID (1 << 0) + __u32 flags; + } notify; /* Fix the size of the union. */ char padding[256]; }; @@ -533,7 +556,7 @@ struct kvm_coalesced_mmio { struct kvm_coalesced_mmio_ring { __u32 first, last; - struct kvm_coalesced_mmio coalesced_mmio[0]; + struct kvm_coalesced_mmio coalesced_mmio[]; }; #define KVM_COALESCED_MMIO_MAX \ @@ -612,7 +635,7 @@ struct kvm_clear_dirty_log { /* for KVM_SET_SIGNAL_MASK */ struct kvm_signal_mask { __u32 len; - __u8 sigset[0]; + __u8 sigset[]; }; /* for KVM_TPR_ACCESS_REPORTING */ @@ -640,6 +663,7 @@ struct kvm_vapic_addr { #define KVM_MP_STATE_OPERATING 7 #define KVM_MP_STATE_LOAD 8 #define KVM_MP_STATE_AP_RESET_HOLD 9 +#define KVM_MP_STATE_SUSPENDED 10 struct kvm_mp_state { __u32 mp_state; @@ -1144,6 +1168,15 @@ struct kvm_ppc_resize_hpt { #define KVM_CAP_S390_MEM_OP_EXTENSION 211 #define KVM_CAP_PMU_CAPABILITY 212 #define KVM_CAP_DISABLE_QUIRKS2 213 +#define KVM_CAP_VM_TSC_CONTROL 214 +#define KVM_CAP_SYSTEM_EVENT_DATA 215 +#define KVM_CAP_ARM_SYSTEM_SUSPEND 216 +#define KVM_CAP_S390_PROTECTED_DUMP 217 +#define KVM_CAP_X86_TRIPLE_FAULT_EVENT 218 +#define KVM_CAP_X86_NOTIFY_VMEXIT 219 +#define KVM_CAP_VM_DISABLE_NX_HUGE_PAGES 220 +#define KVM_CAP_S390_ZPCI_OP 221 +#define KVM_CAP_S390_CPU_TOPOLOGY 222 #ifdef KVM_CAP_IRQ_ROUTING @@ -1208,7 +1241,7 @@ struct kvm_irq_routing_entry { struct kvm_irq_routing { __u32 nr; __u32 flags; - struct kvm_irq_routing_entry entries[0]; + struct kvm_irq_routing_entry entries[]; }; #endif @@ -1232,6 +1265,7 @@ struct kvm_x86_mce { #define KVM_XEN_HVM_CONFIG_SHARED_INFO (1 << 2) #define KVM_XEN_HVM_CONFIG_RUNSTATE (1 << 3) #define KVM_XEN_HVM_CONFIG_EVTCHN_2LEVEL (1 << 4) +#define KVM_XEN_HVM_CONFIG_EVTCHN_SEND (1 << 5) struct kvm_xen_hvm_config { __u32 flags; @@ -1327,7 +1361,7 @@ struct kvm_dirty_tlb { struct kvm_reg_list { __u64 n; /* number of regs */ - __u64 reg[0]; + __u64 reg[]; }; struct kvm_one_reg { @@ -1470,7 +1504,8 @@ struct kvm_s390_ucas_mapping { #define KVM_SET_PIT2 _IOW(KVMIO, 0xa0, struct kvm_pit_state2) /* Available with KVM_CAP_PPC_GET_PVINFO */ #define KVM_PPC_GET_PVINFO _IOW(KVMIO, 0xa1, struct kvm_ppc_pvinfo) -/* Available with KVM_CAP_TSC_CONTROL */ +/* Available with KVM_CAP_TSC_CONTROL for a vCPU, or with +* KVM_CAP_VM_TSC_CONTROL to set defaults for a VM */ #define KVM_SET_TSC_KHZ _IO(KVMIO, 0xa2) #define KVM_GET_TSC_KHZ _IO(KVMIO, 0xa3) /* Available with KVM_CAP_PCI_2_3 */ @@ -1645,6 +1680,55 @@ struct kvm_s390_pv_unp { __u64 tweak; }; +enum pv_cmd_dmp_id { + KVM_PV_DUMP_INIT, + KVM_PV_DUMP_CONFIG_STOR_STATE, + KVM_PV_DUMP_COMPLETE, + KVM_PV_DUMP_CPU, +}; + +struct kvm_s390_pv_dmp { + __u64 subcmd; + __u64 buff_addr; + __u64 buff_len; + __u64 gaddr; /* For dump storage state */ + __u64 reserved[4]; +}; + +enum pv_cmd_info_id { + KVM_PV_INFO_VM, + KVM_PV_INFO_DUMP, +}; + +struct kvm_s390_pv_info_dump { + __u64 dump_cpu_buffer_len; + __u64 dump_config_mem_buffer_per_1m; + __u64 dump_config_finalize_len; +}; + +struct kvm_s390_pv_info_vm { + __u64 inst_calls_list[4]; + __u64 max_cpus; + __u64 max_guests; + __u64 max_guest_addr; + __u64 feature_indication; +}; + +struct kvm_s390_pv_info_header { + __u32 id; + __u32 len_max; + __u32 len_written; + __u32 reserved; +}; + +struct kvm_s390_pv_info { + struct kvm_s390_pv_info_header header; + union { + struct kvm_s390_pv_info_dump dump; + struct kvm_s390_pv_info_vm vm; + }; +}; + enum pv_cmd_id { KVM_PV_ENABLE, KVM_PV_DISABLE, @@ -1653,6 +1737,8 @@ enum pv_cmd_id { KVM_PV_VERIFY, KVM_PV_PREP_RESET, KVM_PV_UNSHARE_ALL, + KVM_PV_INFO, + KVM_PV_DUMP, }; struct kvm_pv_cmd { @@ -1686,6 +1772,32 @@ struct kvm_xen_hvm_attr { struct { __u64 gfn; } shared_info; + struct { + __u32 send_port; + __u32 type; /* EVTCHNSTAT_ipi / EVTCHNSTAT_interdomain */ + __u32 flags; +#define KVM_XEN_EVTCHN_DEASSIGN (1 << 0) +#define KVM_XEN_EVTCHN_UPDATE (1 << 1) +#define KVM_XEN_EVTCHN_RESET (1 << 2) + /* + * Events sent by the guest are either looped back to + * the guest itself (potentially on a different port#) + * or signalled via an eventfd. + */ + union { + struct { + __u32 port; + __u32 vcpu; + __u32 priority; + } port; + struct { + __u32 port; /* Zero for eventfd */ + __s32 fd; + } eventfd; + __u32 padding[4]; + } deliver; + } evtchn; + __u32 xen_version; __u64 pad[8]; } u; }; @@ -1694,11 +1806,17 @@ struct kvm_xen_hvm_attr { #define KVM_XEN_ATTR_TYPE_LONG_MODE 0x0 #define KVM_XEN_ATTR_TYPE_SHARED_INFO 0x1 #define KVM_XEN_ATTR_TYPE_UPCALL_VECTOR 0x2 +/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ +#define KVM_XEN_ATTR_TYPE_EVTCHN 0x3 +#define KVM_XEN_ATTR_TYPE_XEN_VERSION 0x4 /* Per-vCPU Xen attributes */ #define KVM_XEN_VCPU_GET_ATTR _IOWR(KVMIO, 0xca, struct kvm_xen_vcpu_attr) #define KVM_XEN_VCPU_SET_ATTR _IOW(KVMIO, 0xcb, struct kvm_xen_vcpu_attr) +/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ +#define KVM_XEN_HVM_EVTCHN_SEND _IOW(KVMIO, 0xd0, struct kvm_irq_routing_xen_evtchn) + #define KVM_GET_SREGS2 _IOR(KVMIO, 0xcc, struct kvm_sregs2) #define KVM_SET_SREGS2 _IOW(KVMIO, 0xcd, struct kvm_sregs2) @@ -1716,6 +1834,13 @@ struct kvm_xen_vcpu_attr { __u64 time_blocked; __u64 time_offline; } runstate; + __u32 vcpu_id; + struct { + __u32 port; + __u32 priority; + __u64 expires_ns; + } timer; + __u8 vector; } u; }; @@ -1726,6 +1851,10 @@ struct kvm_xen_vcpu_attr { #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_CURRENT 0x3 #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_DATA 0x4 #define KVM_XEN_VCPU_ATTR_TYPE_RUNSTATE_ADJUST 0x5 +/* Available with KVM_CAP_XEN_HVM / KVM_XEN_HVM_CONFIG_EVTCHN_SEND */ +#define KVM_XEN_VCPU_ATTR_TYPE_VCPU_ID 0x6 +#define KVM_XEN_VCPU_ATTR_TYPE_TIMER 0x7 +#define KVM_XEN_VCPU_ATTR_TYPE_UPCALL_VECTOR 0x8 /* Secure Encrypted Virtualization command */ enum sev_cmd_id { @@ -2025,7 +2154,8 @@ struct kvm_stats_header { #define KVM_STATS_UNIT_BYTES (0x1 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_SECONDS (0x2 << KVM_STATS_UNIT_SHIFT) #define KVM_STATS_UNIT_CYCLES (0x3 << KVM_STATS_UNIT_SHIFT) -#define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_CYCLES +#define KVM_STATS_UNIT_BOOLEAN (0x4 << KVM_STATS_UNIT_SHIFT) +#define KVM_STATS_UNIT_MAX KVM_STATS_UNIT_BOOLEAN #define KVM_STATS_BASE_SHIFT 8 #define KVM_STATS_BASE_MASK (0xF << KVM_STATS_BASE_SHIFT) @@ -2060,4 +2190,41 @@ struct kvm_stats_desc { /* Available with KVM_CAP_XSAVE2 */ #define KVM_GET_XSAVE2 _IOR(KVMIO, 0xcf, struct kvm_xsave) +/* Available with KVM_CAP_S390_PROTECTED_DUMP */ +#define KVM_S390_PV_CPU_COMMAND _IOWR(KVMIO, 0xd0, struct kvm_pv_cmd) + +/* Available with KVM_CAP_X86_NOTIFY_VMEXIT */ +#define KVM_X86_NOTIFY_VMEXIT_ENABLED (1ULL << 0) +#define KVM_X86_NOTIFY_VMEXIT_USER (1ULL << 1) + +/* Available with KVM_CAP_S390_ZPCI_OP */ +#define KVM_S390_ZPCI_OP _IOW(KVMIO, 0xd1, struct kvm_s390_zpci_op) + +struct kvm_s390_zpci_op { + /* in */ + __u32 fh; /* target device */ + __u8 op; /* operation to perform */ + __u8 pad[3]; + union { + /* for KVM_S390_ZPCIOP_REG_AEN */ + struct { + __u64 ibv; /* Guest addr of interrupt bit vector */ + __u64 sb; /* Guest addr of summary bit */ + __u32 flags; + __u32 noi; /* Number of interrupts */ + __u8 isc; /* Guest interrupt subclass */ + __u8 sbo; /* Offset of guest summary bit vector */ + __u16 pad; + } reg_aen; + __u64 reserved[8]; + } u; +}; + +/* types for kvm_s390_zpci_op->op */ +#define KVM_S390_ZPCIOP_REG_AEN 0 +#define KVM_S390_ZPCIOP_DEREG_AEN 1 + +/* flags for kvm_s390_zpci_op->u.reg_aen.flags */ +#define KVM_S390_ZPCIOP_REGAEN_HOST (1 << 0) + #endif /* __LINUX_KVM_H */ diff --git a/tools/include/uapi/linux/perf_event.h b/tools/include/uapi/linux/perf_event.h index d37629dbad..581ed4bdc0 100644 --- a/tools/include/uapi/linux/perf_event.h +++ b/tools/include/uapi/linux/perf_event.h @@ -301,6 +301,7 @@ enum { * { u64 time_enabled; } && PERF_FORMAT_TOTAL_TIME_ENABLED * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING * { u64 id; } && PERF_FORMAT_ID + * { u64 lost; } && PERF_FORMAT_LOST * } && !PERF_FORMAT_GROUP * * { u64 nr; @@ -308,6 +309,7 @@ enum { * { u64 time_running; } && PERF_FORMAT_TOTAL_TIME_RUNNING * { u64 value; * { u64 id; } && PERF_FORMAT_ID + * { u64 lost; } && PERF_FORMAT_LOST * } cntr[nr]; * } && PERF_FORMAT_GROUP * }; @@ -317,8 +319,9 @@ enum perf_event_read_format { PERF_FORMAT_TOTAL_TIME_RUNNING = 1U << 1, PERF_FORMAT_ID = 1U << 2, PERF_FORMAT_GROUP = 1U << 3, + PERF_FORMAT_LOST = 1U << 4, - PERF_FORMAT_MAX = 1U << 4, /* non-ABI */ + PERF_FORMAT_MAX = 1U << 5, /* non-ABI */ }; #define PERF_ATTR_SIZE_VER0 64 /* sizeof first published struct */ @@ -491,7 +494,7 @@ struct perf_event_query_bpf { /* * User provided buffer to store program ids */ - __u32 ids[0]; + __u32 ids[]; }; /* @@ -1310,7 +1313,7 @@ union perf_mem_data_src { #define PERF_MEM_SNOOP_SHIFT 19 #define PERF_MEM_SNOOPX_FWD 0x01 /* forward */ -/* 1 free */ +#define PERF_MEM_SNOOPX_PEER 0x02 /* xfer from peer */ #define PERF_MEM_SNOOPX_SHIFT 38 /* locked instruction */ diff --git a/tools/include/uapi/linux/pkt_cls.h b/tools/include/uapi/linux/pkt_cls.h index 1215377139..3faee0199a 100644 --- a/tools/include/uapi/linux/pkt_cls.h +++ b/tools/include/uapi/linux/pkt_cls.h @@ -180,7 +180,7 @@ struct tc_u32_sel { short hoff; __be32 hmask; - struct tc_u32_key keys[0]; + struct tc_u32_key keys[]; }; struct tc_u32_mark { @@ -192,7 +192,7 @@ struct tc_u32_mark { struct tc_u32_pcnt { __u64 rcnt; __u64 rhit; - __u64 kcnts[0]; + __u64 kcnts[]; }; /* Flags */ diff --git a/tools/include/uapi/linux/prctl.h b/tools/include/uapi/linux/prctl.h index e998764f02..a5e06dcbba 100644 --- a/tools/include/uapi/linux/prctl.h +++ b/tools/include/uapi/linux/prctl.h @@ -272,6 +272,15 @@ struct prctl_mm_map { # define PR_SCHED_CORE_SCOPE_THREAD_GROUP 1 # define PR_SCHED_CORE_SCOPE_PROCESS_GROUP 2 +/* arm64 Scalable Matrix Extension controls */ +/* Flag values must be in sync with SVE versions */ +#define PR_SME_SET_VL 63 /* set task vector length */ +# define PR_SME_SET_VL_ONEXEC (1 << 18) /* defer effect until exec */ +#define PR_SME_GET_VL 64 /* get task vector length */ +/* Bits common to PR_SME_SET_VL and PR_SME_GET_VL */ +# define PR_SME_VL_LEN_MASK 0xffff +# define PR_SME_VL_INHERIT (1 << 17) /* inherit across exec */ + #define PR_SET_VMA 0x53564d41 # define PR_SET_VMA_ANON_NAME 0 diff --git a/tools/include/uapi/linux/seg6.h b/tools/include/uapi/linux/seg6.h index 286e8d6a8e..f94baf154c 100644 --- a/tools/include/uapi/linux/seg6.h +++ b/tools/include/uapi/linux/seg6.h @@ -30,7 +30,7 @@ struct ipv6_sr_hdr { __u8 flags; __u16 tag; - struct in6_addr segments[0]; + struct in6_addr segments[]; }; #define SR6_FLAG1_PROTECTED (1 << 6) @@ -49,7 +49,7 @@ struct ipv6_sr_hdr { struct sr6_tlv { __u8 type; __u8 len; - __u8 data[0]; + __u8 data[]; }; #endif diff --git a/tools/include/uapi/linux/usbdevice_fs.h b/tools/include/uapi/linux/usbdevice_fs.h index cf525cddeb..74a84e0242 100644 --- a/tools/include/uapi/linux/usbdevice_fs.h +++ b/tools/include/uapi/linux/usbdevice_fs.h @@ -131,7 +131,7 @@ struct usbdevfs_urb { unsigned int signr; /* signal to be sent on completion, or 0 if none should be sent. */ void __user *usercontext; - struct usbdevfs_iso_packet_desc iso_frame_desc[0]; + struct usbdevfs_iso_packet_desc iso_frame_desc[]; }; /* ioctls for talking directly to drivers */ @@ -176,7 +176,7 @@ struct usbdevfs_disconnect_claim { struct usbdevfs_streams { unsigned int num_streams; /* Not used by USBDEVFS_FREE_STREAMS */ unsigned int num_eps; - unsigned char eps[0]; + unsigned char eps[]; }; /* diff --git a/tools/include/uapi/linux/vhost.h b/tools/include/uapi/linux/vhost.h index 5d99e7c242..f9f115a7c7 100644 --- a/tools/include/uapi/linux/vhost.h +++ b/tools/include/uapi/linux/vhost.h @@ -89,11 +89,6 @@ /* Set or get vhost backend capability */ -/* Use message type V2 */ -#define VHOST_BACKEND_F_IOTLB_MSG_V2 0x1 -/* IOTLB can accept batching hints */ -#define VHOST_BACKEND_F_IOTLB_BATCH 0x2 - #define VHOST_SET_BACKEND_FEATURES _IOW(VHOST_VIRTIO, 0x25, __u64) #define VHOST_GET_BACKEND_FEATURES _IOR(VHOST_VIRTIO, 0x26, __u64) @@ -150,11 +145,39 @@ /* Get the valid iova range */ #define VHOST_VDPA_GET_IOVA_RANGE _IOR(VHOST_VIRTIO, 0x78, \ struct vhost_vdpa_iova_range) - /* Get the config size */ #define VHOST_VDPA_GET_CONFIG_SIZE _IOR(VHOST_VIRTIO, 0x79, __u32) /* Get the count of all virtqueues */ #define VHOST_VDPA_GET_VQS_COUNT _IOR(VHOST_VIRTIO, 0x80, __u32) +/* Get the number of virtqueue groups. */ +#define VHOST_VDPA_GET_GROUP_NUM _IOR(VHOST_VIRTIO, 0x81, __u32) + +/* Get the number of address spaces. */ +#define VHOST_VDPA_GET_AS_NUM _IOR(VHOST_VIRTIO, 0x7A, unsigned int) + +/* Get the group for a virtqueue: read index, write group in num, + * The virtqueue index is stored in the index field of + * vhost_vring_state. The group for this specific virtqueue is + * returned via num field of vhost_vring_state. + */ +#define VHOST_VDPA_GET_VRING_GROUP _IOWR(VHOST_VIRTIO, 0x7B, \ + struct vhost_vring_state) +/* Set the ASID for a virtqueue group. The group index is stored in + * the index field of vhost_vring_state, the ASID associated with this + * group is stored at num field of vhost_vring_state. + */ +#define VHOST_VDPA_SET_GROUP_ASID _IOW(VHOST_VIRTIO, 0x7C, \ + struct vhost_vring_state) + +/* Suspend a device so it does not process virtqueue requests anymore + * + * After the return of ioctl the device must preserve all the necessary state + * (the virtqueue vring base plus the possible device specific states) that is + * required for restoring in the future. The device must not change its + * configuration after that point. + */ +#define VHOST_VDPA_SUSPEND _IO(VHOST_VIRTIO, 0x7D) + #endif diff --git a/tools/include/uapi/sound/asound.h b/tools/include/uapi/sound/asound.h index 2d3e5df39a..3974a2a911 100644 --- a/tools/include/uapi/sound/asound.h +++ b/tools/include/uapi/sound/asound.h @@ -1106,7 +1106,7 @@ struct snd_ctl_elem_value { struct snd_ctl_tlv { unsigned int numid; /* control element numeric identification */ unsigned int length; /* in bytes aligned to 4 */ - unsigned int tlv[0]; /* first TLV */ + unsigned int tlv[]; /* first TLV */ }; #define SNDRV_CTL_IOCTL_PVERSION _IOR('U', 0x00, int) diff --git a/tools/kvm/kvm_stat/kvm_stat b/tools/kvm/kvm_stat/kvm_stat index 5a5bd74f55..9c366b3a67 100644 --- a/tools/kvm/kvm_stat/kvm_stat +++ b/tools/kvm/kvm_stat/kvm_stat @@ -1646,7 +1646,8 @@ Press any other key to refresh statistics immediately. .format(values)) if len(pids) > 1: sys.exit('Error: Multiple processes found (pids: {}). Use "-p"' - ' to specify the desired pid'.format(" ".join(pids))) + ' to specify the desired pid' + .format(" ".join(map(str, pids)))) namespace.pid = pids[0] argparser = argparse.ArgumentParser(description=description_text, diff --git a/tools/lib/bitmap.c b/tools/lib/bitmap.c index db466ef7be..c3e4871967 100644 --- a/tools/lib/bitmap.c +++ b/tools/lib/bitmap.c @@ -5,9 +5,9 @@ */ #include -int __bitmap_weight(const unsigned long *bitmap, int bits) +unsigned int __bitmap_weight(const unsigned long *bitmap, int bits) { - int k, w = 0, lim = bits/BITS_PER_LONG; + unsigned int k, w = 0, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; k++) w += hweight_long(bitmap[k]); @@ -57,7 +57,7 @@ size_t bitmap_scnprintf(unsigned long *bitmap, unsigned int nbits, return ret; } -int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, +bool __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, const unsigned long *bitmap2, unsigned int bits) { unsigned int k; @@ -72,31 +72,31 @@ int __bitmap_and(unsigned long *dst, const unsigned long *bitmap1, return result != 0; } -int __bitmap_equal(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_equal(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] != bitmap2[k]) - return 0; + return false; if (bits % BITS_PER_LONG) if ((bitmap1[k] ^ bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 0; + return false; - return 1; + return true; } -int __bitmap_intersects(const unsigned long *bitmap1, - const unsigned long *bitmap2, unsigned int bits) +bool __bitmap_intersects(const unsigned long *bitmap1, + const unsigned long *bitmap2, unsigned int bits) { unsigned int k, lim = bits/BITS_PER_LONG; for (k = 0; k < lim; ++k) if (bitmap1[k] & bitmap2[k]) - return 1; + return true; if (bits % BITS_PER_LONG) if ((bitmap1[k] & bitmap2[k]) & BITMAP_LAST_WORD_MASK(bits)) - return 1; - return 0; + return true; + return false; } diff --git a/tools/lib/bpf/Build b/tools/lib/bpf/Build index 94f0a146bb..5a3dfb56d7 100644 --- a/tools/lib/bpf/Build +++ b/tools/lib/bpf/Build @@ -1,3 +1,4 @@ libbpf-y := libbpf.o bpf.o nlattr.o btf.o libbpf_errno.o str_error.o \ - netlink.o bpf_prog_linfo.o libbpf_probes.o xsk.o hashmap.o \ - btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o + netlink.o bpf_prog_linfo.o libbpf_probes.o hashmap.o \ + btf_dump.o ringbuf.o strset.o linker.o gen_loader.o relo_core.o \ + usdt.o diff --git a/tools/lib/bpf/Makefile b/tools/lib/bpf/Makefile index 064c89e315..4c904ef0b4 100644 --- a/tools/lib/bpf/Makefile +++ b/tools/lib/bpf/Makefile @@ -127,7 +127,7 @@ TAGS_PROG := $(if $(shell which etags 2>/dev/null),etags,ctags) GLOBAL_SYM_COUNT = $(shell readelf -s --wide $(BPF_IN_SHARED) | \ cut -d "@" -f1 | sed 's/_v[0-9]_[0-9]_[0-9].*//' | \ sed 's/\[.*\]//' | \ - awk '/GLOBAL/ && /DEFAULT/ && !/UND/ {print $$NF}' | \ + awk '/GLOBAL/ && /DEFAULT/ && !/UND|ABS/ {print $$NF}' | \ sort -u | wc -l) VERSIONED_SYM_COUNT = $(shell readelf --dyn-syms --wide $(OUTPUT)libbpf.so | \ sed 's/\[.*\]//' | \ @@ -237,9 +237,9 @@ install_lib: all_cmd $(call do_install_mkdir,$(libdir_SQ)); \ cp -fpR $(LIB_FILE) $(DESTDIR)$(libdir_SQ) -SRC_HDRS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h xsk.h \ +SRC_HDRS := bpf.h libbpf.h btf.h libbpf_common.h libbpf_legacy.h \ bpf_helpers.h bpf_tracing.h bpf_endian.h bpf_core_read.h \ - skel_internal.h libbpf_version.h + skel_internal.h libbpf_version.h usdt.bpf.h GEN_HDRS := $(BPF_GENERATED) INSTALL_PFX := $(DESTDIR)$(prefix)/include/bpf diff --git a/tools/lib/bpf/bpf.c b/tools/lib/bpf/bpf.c index cf27251adb..efcc06dafb 100644 --- a/tools/lib/bpf/bpf.c +++ b/tools/lib/bpf/bpf.c @@ -147,10 +147,6 @@ int bump_rlimit_memlock(void) { struct rlimit rlim; - /* this the default in libbpf 1.0, but for now user has to opt-in explicitly */ - if (!(libbpf_mode & LIBBPF_STRICT_AUTO_RLIMIT_MEMLOCK)) - return 0; - /* if kernel supports memcg-based accounting, skip bumping RLIMIT_MEMLOCK */ if (memlock_bumped || kernel_supports(NULL, FEAT_MEMCG_ACCOUNT)) return 0; @@ -208,86 +204,6 @@ int bpf_map_create(enum bpf_map_type map_type, return libbpf_err_errno(fd); } -int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr) -{ - LIBBPF_OPTS(bpf_map_create_opts, p); - - p.map_flags = create_attr->map_flags; - p.numa_node = create_attr->numa_node; - p.btf_fd = create_attr->btf_fd; - p.btf_key_type_id = create_attr->btf_key_type_id; - p.btf_value_type_id = create_attr->btf_value_type_id; - p.map_ifindex = create_attr->map_ifindex; - if (create_attr->map_type == BPF_MAP_TYPE_STRUCT_OPS) - p.btf_vmlinux_value_type_id = create_attr->btf_vmlinux_value_type_id; - else - p.inner_map_fd = create_attr->inner_map_fd; - - return bpf_map_create(create_attr->map_type, create_attr->name, - create_attr->key_size, create_attr->value_size, - create_attr->max_entries, &p); -} - -int bpf_create_map_node(enum bpf_map_type map_type, const char *name, - int key_size, int value_size, int max_entries, - __u32 map_flags, int node) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts); - - opts.map_flags = map_flags; - if (node >= 0) { - opts.numa_node = node; - opts.map_flags |= BPF_F_NUMA_NODE; - } - - return bpf_map_create(map_type, name, key_size, value_size, max_entries, &opts); -} - -int bpf_create_map(enum bpf_map_type map_type, int key_size, - int value_size, int max_entries, __u32 map_flags) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); - - return bpf_map_create(map_type, NULL, key_size, value_size, max_entries, &opts); -} - -int bpf_create_map_name(enum bpf_map_type map_type, const char *name, - int key_size, int value_size, int max_entries, - __u32 map_flags) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, .map_flags = map_flags); - - return bpf_map_create(map_type, name, key_size, value_size, max_entries, &opts); -} - -int bpf_create_map_in_map_node(enum bpf_map_type map_type, const char *name, - int key_size, int inner_map_fd, int max_entries, - __u32 map_flags, int node) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts); - - opts.inner_map_fd = inner_map_fd; - opts.map_flags = map_flags; - if (node >= 0) { - opts.map_flags |= BPF_F_NUMA_NODE; - opts.numa_node = node; - } - - return bpf_map_create(map_type, name, key_size, 4, max_entries, &opts); -} - -int bpf_create_map_in_map(enum bpf_map_type map_type, const char *name, - int key_size, int inner_map_fd, int max_entries, - __u32 map_flags) -{ - LIBBPF_OPTS(bpf_map_create_opts, opts, - .inner_map_fd = inner_map_fd, - .map_flags = map_flags, - ); - - return bpf_map_create(map_type, name, key_size, 4, max_entries, &opts); -} - static void * alloc_zero_tailing_info(const void *orecord, __u32 cnt, __u32 actual_rec_size, __u32 expected_rec_size) @@ -313,11 +229,10 @@ alloc_zero_tailing_info(const void *orecord, __u32 cnt, return info; } -DEFAULT_VERSION(bpf_prog_load_v0_6_0, bpf_prog_load, LIBBPF_0.6.0) -int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, - const char *prog_name, const char *license, - const struct bpf_insn *insns, size_t insn_cnt, - const struct bpf_prog_load_opts *opts) +int bpf_prog_load(enum bpf_prog_type prog_type, + const char *prog_name, const char *license, + const struct bpf_insn *insns, size_t insn_cnt, + const struct bpf_prog_load_opts *opts) { void *finfo = NULL, *linfo = NULL; const char *func_info, *line_info; @@ -464,94 +379,6 @@ int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, return libbpf_err_errno(fd); } -__attribute__((alias("bpf_load_program_xattr2"))) -int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, - char *log_buf, size_t log_buf_sz); - -static int bpf_load_program_xattr2(const struct bpf_load_program_attr *load_attr, - char *log_buf, size_t log_buf_sz) -{ - LIBBPF_OPTS(bpf_prog_load_opts, p); - - if (!load_attr || !log_buf != !log_buf_sz) - return libbpf_err(-EINVAL); - - p.expected_attach_type = load_attr->expected_attach_type; - switch (load_attr->prog_type) { - case BPF_PROG_TYPE_STRUCT_OPS: - case BPF_PROG_TYPE_LSM: - p.attach_btf_id = load_attr->attach_btf_id; - break; - case BPF_PROG_TYPE_TRACING: - case BPF_PROG_TYPE_EXT: - p.attach_btf_id = load_attr->attach_btf_id; - p.attach_prog_fd = load_attr->attach_prog_fd; - break; - default: - p.prog_ifindex = load_attr->prog_ifindex; - p.kern_version = load_attr->kern_version; - } - p.log_level = load_attr->log_level; - p.log_buf = log_buf; - p.log_size = log_buf_sz; - p.prog_btf_fd = load_attr->prog_btf_fd; - p.func_info_rec_size = load_attr->func_info_rec_size; - p.func_info_cnt = load_attr->func_info_cnt; - p.func_info = load_attr->func_info; - p.line_info_rec_size = load_attr->line_info_rec_size; - p.line_info_cnt = load_attr->line_info_cnt; - p.line_info = load_attr->line_info; - p.prog_flags = load_attr->prog_flags; - - return bpf_prog_load(load_attr->prog_type, load_attr->name, load_attr->license, - load_attr->insns, load_attr->insns_cnt, &p); -} - -int bpf_load_program(enum bpf_prog_type type, const struct bpf_insn *insns, - size_t insns_cnt, const char *license, - __u32 kern_version, char *log_buf, - size_t log_buf_sz) -{ - struct bpf_load_program_attr load_attr; - - memset(&load_attr, 0, sizeof(struct bpf_load_program_attr)); - load_attr.prog_type = type; - load_attr.expected_attach_type = 0; - load_attr.name = NULL; - load_attr.insns = insns; - load_attr.insns_cnt = insns_cnt; - load_attr.license = license; - load_attr.kern_version = kern_version; - - return bpf_load_program_xattr2(&load_attr, log_buf, log_buf_sz); -} - -int bpf_verify_program(enum bpf_prog_type type, const struct bpf_insn *insns, - size_t insns_cnt, __u32 prog_flags, const char *license, - __u32 kern_version, char *log_buf, size_t log_buf_sz, - int log_level) -{ - union bpf_attr attr; - int fd; - - bump_rlimit_memlock(); - - memset(&attr, 0, sizeof(attr)); - attr.prog_type = type; - attr.insn_cnt = (__u32)insns_cnt; - attr.insns = ptr_to_u64(insns); - attr.license = ptr_to_u64(license); - attr.log_buf = ptr_to_u64(log_buf); - attr.log_size = log_buf_sz; - attr.log_level = log_level; - log_buf[0] = 0; - attr.kern_version = kern_version; - attr.prog_flags = prog_flags; - - fd = sys_bpf_prog_load(&attr, sizeof(attr), PROG_LOAD_ATTEMPTS); - return libbpf_err_errno(fd); -} - int bpf_map_update_elem(int fd, const void *key, const void *value, __u64 flags) { @@ -639,6 +466,20 @@ int bpf_map_delete_elem(int fd, const void *key) return libbpf_err_errno(ret); } +int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags) +{ + union bpf_attr attr; + int ret; + + memset(&attr, 0, sizeof(attr)); + attr.map_fd = fd; + attr.key = ptr_to_u64(key); + attr.flags = flags; + + ret = sys_bpf(BPF_MAP_DELETE_ELEM, &attr, sizeof(attr)); + return libbpf_err_errno(ret); +} + int bpf_map_get_next_key(int fd, const void *key, void *next_key) { union bpf_attr attr; @@ -737,12 +578,21 @@ int bpf_obj_pin(int fd, const char *pathname) } int bpf_obj_get(const char *pathname) +{ + return bpf_obj_get_opts(pathname, NULL); +} + +int bpf_obj_get_opts(const char *pathname, const struct bpf_obj_get_opts *opts) { union bpf_attr attr; int fd; + if (!OPTS_VALID(opts, bpf_obj_get_opts)) + return libbpf_err(-EINVAL); + memset(&attr, 0, sizeof(attr)); attr.pathname = ptr_to_u64((void *)pathname); + attr.file_flags = OPTS_GET(opts, file_flags, 0); fd = sys_bpf_fd(BPF_OBJ_GET, &attr, sizeof(attr)); return libbpf_err_errno(fd); @@ -817,7 +667,7 @@ int bpf_link_create(int prog_fd, int target_fd, { __u32 target_btf_id, iter_info_len; union bpf_attr attr; - int fd; + int fd, err; if (!OPTS_VALID(opts, bpf_link_create_opts)) return libbpf_err(-EINVAL); @@ -863,6 +713,14 @@ int bpf_link_create(int prog_fd, int target_fd, if (!OPTS_ZEROED(opts, kprobe_multi)) return libbpf_err(-EINVAL); break; + case BPF_TRACE_FENTRY: + case BPF_TRACE_FEXIT: + case BPF_MODIFY_RETURN: + case BPF_LSM_MAC: + attr.link_create.tracing.cookie = OPTS_GET(opts, tracing.cookie, 0); + if (!OPTS_ZEROED(opts, tracing)) + return libbpf_err(-EINVAL); + break; default: if (!OPTS_ZEROED(opts, flags)) return libbpf_err(-EINVAL); @@ -870,7 +728,37 @@ int bpf_link_create(int prog_fd, int target_fd, } proceed: fd = sys_bpf_fd(BPF_LINK_CREATE, &attr, sizeof(attr)); - return libbpf_err_errno(fd); + if (fd >= 0) + return fd; + /* we'll get EINVAL if LINK_CREATE doesn't support attaching fentry + * and other similar programs + */ + err = -errno; + if (err != -EINVAL) + return libbpf_err(err); + + /* if user used features not supported by + * BPF_RAW_TRACEPOINT_OPEN command, then just give up immediately + */ + if (attr.link_create.target_fd || attr.link_create.target_btf_id) + return libbpf_err(err); + if (!OPTS_ZEROED(opts, sz)) + return libbpf_err(err); + + /* otherwise, for few select kinds of programs that can be + * attached using BPF_RAW_TRACEPOINT_OPEN command, try that as + * a fallback for older kernels + */ + switch (attach_type) { + case BPF_TRACE_RAW_TP: + case BPF_LSM_MAC: + case BPF_TRACE_FENTRY: + case BPF_TRACE_FEXIT: + case BPF_MODIFY_RETURN: + return bpf_raw_tracepoint_open(NULL, prog_fd); + default: + return libbpf_err(err); + } } int bpf_link_detach(int link_fd) @@ -916,80 +804,48 @@ int bpf_iter_create(int link_fd) return libbpf_err_errno(fd); } -int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, - __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt) +int bpf_prog_query_opts(int target_fd, + enum bpf_attach_type type, + struct bpf_prog_query_opts *opts) { union bpf_attr attr; int ret; - memset(&attr, 0, sizeof(attr)); - attr.query.target_fd = target_fd; - attr.query.attach_type = type; - attr.query.query_flags = query_flags; - attr.query.prog_cnt = *prog_cnt; - attr.query.prog_ids = ptr_to_u64(prog_ids); - - ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr)); - - if (attach_flags) - *attach_flags = attr.query.attach_flags; - *prog_cnt = attr.query.prog_cnt; - - return libbpf_err_errno(ret); -} - -int bpf_prog_test_run(int prog_fd, int repeat, void *data, __u32 size, - void *data_out, __u32 *size_out, __u32 *retval, - __u32 *duration) -{ - union bpf_attr attr; - int ret; - - memset(&attr, 0, sizeof(attr)); - attr.test.prog_fd = prog_fd; - attr.test.data_in = ptr_to_u64(data); - attr.test.data_out = ptr_to_u64(data_out); - attr.test.data_size_in = size; - attr.test.repeat = repeat; - - ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); - - if (size_out) - *size_out = attr.test.data_size_out; - if (retval) - *retval = attr.test.retval; - if (duration) - *duration = attr.test.duration; - - return libbpf_err_errno(ret); -} - -int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr) -{ - union bpf_attr attr; - int ret; - - if (!test_attr->data_out && test_attr->data_size_out > 0) + if (!OPTS_VALID(opts, bpf_prog_query_opts)) return libbpf_err(-EINVAL); memset(&attr, 0, sizeof(attr)); - attr.test.prog_fd = test_attr->prog_fd; - attr.test.data_in = ptr_to_u64(test_attr->data_in); - attr.test.data_out = ptr_to_u64(test_attr->data_out); - attr.test.data_size_in = test_attr->data_size_in; - attr.test.data_size_out = test_attr->data_size_out; - attr.test.ctx_in = ptr_to_u64(test_attr->ctx_in); - attr.test.ctx_out = ptr_to_u64(test_attr->ctx_out); - attr.test.ctx_size_in = test_attr->ctx_size_in; - attr.test.ctx_size_out = test_attr->ctx_size_out; - attr.test.repeat = test_attr->repeat; - ret = sys_bpf(BPF_PROG_TEST_RUN, &attr, sizeof(attr)); + attr.query.target_fd = target_fd; + attr.query.attach_type = type; + attr.query.query_flags = OPTS_GET(opts, query_flags, 0); + attr.query.prog_cnt = OPTS_GET(opts, prog_cnt, 0); + attr.query.prog_ids = ptr_to_u64(OPTS_GET(opts, prog_ids, NULL)); + attr.query.prog_attach_flags = ptr_to_u64(OPTS_GET(opts, prog_attach_flags, NULL)); - test_attr->data_size_out = attr.test.data_size_out; - test_attr->ctx_size_out = attr.test.ctx_size_out; - test_attr->retval = attr.test.retval; - test_attr->duration = attr.test.duration; + ret = sys_bpf(BPF_PROG_QUERY, &attr, sizeof(attr)); + + OPTS_SET(opts, attach_flags, attr.query.attach_flags); + OPTS_SET(opts, prog_cnt, attr.query.prog_cnt); + + return libbpf_err_errno(ret); +} + +int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, + __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt) +{ + LIBBPF_OPTS(bpf_prog_query_opts, opts); + int ret; + + opts.query_flags = query_flags; + opts.prog_ids = prog_ids; + opts.prog_cnt = *prog_cnt; + + ret = bpf_prog_query_opts(target_fd, type, &opts); + + if (attach_flags) + *attach_flags = opts.attach_flags; + *prog_cnt = opts.prog_cnt; return libbpf_err_errno(ret); } @@ -1190,27 +1046,6 @@ int bpf_btf_load(const void *btf_data, size_t btf_size, const struct bpf_btf_loa return libbpf_err_errno(fd); } -int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, __u32 log_buf_size, bool do_log) -{ - LIBBPF_OPTS(bpf_btf_load_opts, opts); - int fd; - -retry: - if (do_log && log_buf && log_buf_size) { - opts.log_buf = log_buf; - opts.log_size = log_buf_size; - opts.log_level = 1; - } - - fd = bpf_btf_load(btf, btf_size, &opts); - if (fd < 0 && !do_log && log_buf && log_buf_size) { - do_log = true; - goto retry; - } - - return libbpf_err_errno(fd); -} - int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, __u64 *probe_offset, __u64 *probe_addr) diff --git a/tools/lib/bpf/bpf.h b/tools/lib/bpf/bpf.h index f4b4afb6d4..9c50beabdd 100644 --- a/tools/lib/bpf/bpf.h +++ b/tools/lib/bpf/bpf.h @@ -61,48 +61,6 @@ LIBBPF_API int bpf_map_create(enum bpf_map_type map_type, __u32 max_entries, const struct bpf_map_create_opts *opts); -struct bpf_create_map_attr { - const char *name; - enum bpf_map_type map_type; - __u32 map_flags; - __u32 key_size; - __u32 value_size; - __u32 max_entries; - __u32 numa_node; - __u32 btf_fd; - __u32 btf_key_type_id; - __u32 btf_value_type_id; - __u32 map_ifindex; - union { - __u32 inner_map_fd; - __u32 btf_vmlinux_value_type_id; - }; -}; - -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_map_create() instead") -LIBBPF_API int bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_map_create() instead") -LIBBPF_API int bpf_create_map_node(enum bpf_map_type map_type, const char *name, - int key_size, int value_size, - int max_entries, __u32 map_flags, int node); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_map_create() instead") -LIBBPF_API int bpf_create_map_name(enum bpf_map_type map_type, const char *name, - int key_size, int value_size, - int max_entries, __u32 map_flags); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_map_create() instead") -LIBBPF_API int bpf_create_map(enum bpf_map_type map_type, int key_size, - int value_size, int max_entries, __u32 map_flags); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_map_create() instead") -LIBBPF_API int bpf_create_map_in_map_node(enum bpf_map_type map_type, - const char *name, int key_size, - int inner_map_fd, int max_entries, - __u32 map_flags, int node); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_map_create() instead") -LIBBPF_API int bpf_create_map_in_map(enum bpf_map_type map_type, - const char *name, int key_size, - int inner_map_fd, int max_entries, - __u32 map_flags); - struct bpf_prog_load_opts { size_t sz; /* size of this struct for forward/backward compatibility */ @@ -145,54 +103,6 @@ LIBBPF_API int bpf_prog_load(enum bpf_prog_type prog_type, const char *prog_name, const char *license, const struct bpf_insn *insns, size_t insn_cnt, const struct bpf_prog_load_opts *opts); -/* this "specialization" should go away in libbpf 1.0 */ -LIBBPF_API int bpf_prog_load_v0_6_0(enum bpf_prog_type prog_type, - const char *prog_name, const char *license, - const struct bpf_insn *insns, size_t insn_cnt, - const struct bpf_prog_load_opts *opts); - -/* This is an elaborate way to not conflict with deprecated bpf_prog_load() - * API, defined in libbpf.h. Once we hit libbpf 1.0, all this will be gone. - * With this approach, if someone is calling bpf_prog_load() with - * 4 arguments, they will use the deprecated API, which keeps backwards - * compatibility (both source code and binary). If bpf_prog_load() is called - * with 6 arguments, though, it gets redirected to __bpf_prog_load. - * So looking forward to libbpf 1.0 when this hack will be gone and - * __bpf_prog_load() will be called just bpf_prog_load(). - */ -#ifndef bpf_prog_load -#define bpf_prog_load(...) ___libbpf_overload(___bpf_prog_load, __VA_ARGS__) -#define ___bpf_prog_load4(file, type, pobj, prog_fd) \ - bpf_prog_load_deprecated(file, type, pobj, prog_fd) -#define ___bpf_prog_load6(prog_type, prog_name, license, insns, insn_cnt, opts) \ - bpf_prog_load(prog_type, prog_name, license, insns, insn_cnt, opts) -#endif /* bpf_prog_load */ - -struct bpf_load_program_attr { - enum bpf_prog_type prog_type; - enum bpf_attach_type expected_attach_type; - const char *name; - const struct bpf_insn *insns; - size_t insns_cnt; - const char *license; - union { - __u32 kern_version; - __u32 attach_prog_fd; - }; - union { - __u32 prog_ifindex; - __u32 attach_btf_id; - }; - __u32 prog_btf_fd; - __u32 func_info_rec_size; - const void *func_info; - __u32 func_info_cnt; - __u32 line_info_rec_size; - const void *line_info; - __u32 line_info_cnt; - __u32 log_level; - __u32 prog_flags; -}; /* Flags to direct loading requirements */ #define MAPS_RELAX_COMPAT 0x01 @@ -200,22 +110,6 @@ struct bpf_load_program_attr { /* Recommended log buffer size */ #define BPF_LOG_BUF_SIZE (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */ -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") -LIBBPF_API int bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, - char *log_buf, size_t log_buf_sz); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") -LIBBPF_API int bpf_load_program(enum bpf_prog_type type, - const struct bpf_insn *insns, size_t insns_cnt, - const char *license, __u32 kern_version, - char *log_buf, size_t log_buf_sz); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_load() instead") -LIBBPF_API int bpf_verify_program(enum bpf_prog_type type, - const struct bpf_insn *insns, - size_t insns_cnt, __u32 prog_flags, - const char *license, __u32 kern_version, - char *log_buf, size_t log_buf_sz, - int log_level); - struct bpf_btf_load_opts { size_t sz; /* size of this struct for forward/backward compatibility */ @@ -229,10 +123,6 @@ struct bpf_btf_load_opts { LIBBPF_API int bpf_btf_load(const void *btf_data, size_t btf_size, const struct bpf_btf_load_opts *opts); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_btf_load() instead") -LIBBPF_API int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, - __u32 log_buf_size, bool do_log); - LIBBPF_API int bpf_map_update_elem(int fd, const void *key, const void *value, __u64 flags); @@ -244,6 +134,7 @@ LIBBPF_API int bpf_map_lookup_and_delete_elem(int fd, const void *key, LIBBPF_API int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, void *value, __u64 flags); LIBBPF_API int bpf_map_delete_elem(int fd, const void *key); +LIBBPF_API int bpf_map_delete_elem_flags(int fd, const void *key, __u64 flags); LIBBPF_API int bpf_map_get_next_key(int fd, const void *key, void *next_key); LIBBPF_API int bpf_map_freeze(int fd); @@ -379,8 +270,19 @@ LIBBPF_API int bpf_map_update_batch(int fd, const void *keys, const void *values __u32 *count, const struct bpf_map_batch_opts *opts); +struct bpf_obj_get_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + + __u32 file_flags; + + size_t :0; +}; +#define bpf_obj_get_opts__last_field file_flags + LIBBPF_API int bpf_obj_pin(int fd, const char *pathname); LIBBPF_API int bpf_obj_get(const char *pathname); +LIBBPF_API int bpf_obj_get_opts(const char *pathname, + const struct bpf_obj_get_opts *opts); struct bpf_prog_attach_opts { size_t sz; /* size of this struct for forward/backward compatibility */ @@ -394,10 +296,6 @@ LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd, LIBBPF_API int bpf_prog_attach_opts(int prog_fd, int attachable_fd, enum bpf_attach_type type, const struct bpf_prog_attach_opts *opts); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_prog_attach_opts() instead") -LIBBPF_API int bpf_prog_attach_xattr(int prog_fd, int attachable_fd, - enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts); LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type); LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd, enum bpf_attach_type type); @@ -420,6 +318,9 @@ struct bpf_link_create_opts { const unsigned long *addrs; const __u64 *cookies; } kprobe_multi; + struct { + __u64 cookie; + } tracing; }; size_t :0; }; @@ -460,17 +361,6 @@ struct bpf_prog_test_run_attr { * out: length of cxt_out */ }; -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_test_run_opts() instead") -LIBBPF_API int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr); - -/* - * bpf_prog_test_run does not check that data_out is large enough. Consider - * using bpf_prog_test_run_opts instead. - */ -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_prog_test_run_opts() instead") -LIBBPF_API int bpf_prog_test_run(int prog_fd, int repeat, void *data, - __u32 size, void *data_out, __u32 *size_out, - __u32 *retval, __u32 *duration); LIBBPF_API int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id); LIBBPF_API int bpf_map_get_next_id(__u32 start_id, __u32 *next_id); LIBBPF_API int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id); @@ -480,9 +370,24 @@ LIBBPF_API int bpf_map_get_fd_by_id(__u32 id); LIBBPF_API int bpf_btf_get_fd_by_id(__u32 id); LIBBPF_API int bpf_link_get_fd_by_id(__u32 id); LIBBPF_API int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len); + +struct bpf_prog_query_opts { + size_t sz; /* size of this struct for forward/backward compatibility */ + __u32 query_flags; + __u32 attach_flags; /* output argument */ + __u32 *prog_ids; + __u32 prog_cnt; /* input+output argument */ + __u32 *prog_attach_flags; +}; +#define bpf_prog_query_opts__last_field prog_attach_flags + +LIBBPF_API int bpf_prog_query_opts(int target_fd, + enum bpf_attach_type type, + struct bpf_prog_query_opts *opts); LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, __u32 query_flags, __u32 *attach_flags, __u32 *prog_ids, __u32 *prog_cnt); + LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd); LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, diff --git a/tools/lib/bpf/bpf_core_read.h b/tools/lib/bpf/bpf_core_read.h index e4aa9996a5..496e6a8ee0 100644 --- a/tools/lib/bpf/bpf_core_read.h +++ b/tools/lib/bpf/bpf_core_read.h @@ -29,6 +29,7 @@ enum bpf_type_id_kind { enum bpf_type_info_kind { BPF_TYPE_EXISTS = 0, /* type existence in target kernel */ BPF_TYPE_SIZE = 1, /* type size in target kernel */ + BPF_TYPE_MATCHES = 2, /* type match in target kernel */ }; /* second argument to __builtin_preserve_enum_value() built-in */ @@ -110,21 +111,50 @@ enum bpf_enum_value_kind { val; \ }) +#define ___bpf_field_ref1(field) (field) +#define ___bpf_field_ref2(type, field) (((typeof(type) *)0)->field) +#define ___bpf_field_ref(args...) \ + ___bpf_apply(___bpf_field_ref, ___bpf_narg(args))(args) + /* * Convenience macro to check that field actually exists in target kernel's. * Returns: * 1, if matching field is present in target kernel; * 0, if no matching field found. + * + * Supports two forms: + * - field reference through variable access: + * bpf_core_field_exists(p->my_field); + * - field reference through type and field names: + * bpf_core_field_exists(struct my_type, my_field). */ -#define bpf_core_field_exists(field) \ - __builtin_preserve_field_info(field, BPF_FIELD_EXISTS) +#define bpf_core_field_exists(field...) \ + __builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_EXISTS) /* * Convenience macro to get the byte size of a field. Works for integers, * struct/unions, pointers, arrays, and enums. + * + * Supports two forms: + * - field reference through variable access: + * bpf_core_field_size(p->my_field); + * - field reference through type and field names: + * bpf_core_field_size(struct my_type, my_field). */ -#define bpf_core_field_size(field) \ - __builtin_preserve_field_info(field, BPF_FIELD_BYTE_SIZE) +#define bpf_core_field_size(field...) \ + __builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_BYTE_SIZE) + +/* + * Convenience macro to get field's byte offset. + * + * Supports two forms: + * - field reference through variable access: + * bpf_core_field_offset(p->my_field); + * - field reference through type and field names: + * bpf_core_field_offset(struct my_type, my_field). + */ +#define bpf_core_field_offset(field...) \ + __builtin_preserve_field_info(___bpf_field_ref(field), BPF_FIELD_BYTE_OFFSET) /* * Convenience macro to get BTF type ID of a specified type, using a local BTF @@ -154,6 +184,16 @@ enum bpf_enum_value_kind { #define bpf_core_type_exists(type) \ __builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_EXISTS) +/* + * Convenience macro to check that provided named type + * (struct/union/enum/typedef) "matches" that in a target kernel. + * Returns: + * 1, if the type matches in the target kernel's BTF; + * 0, if the type does not match any in the target kernel + */ +#define bpf_core_type_matches(type) \ + __builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_MATCHES) + /* * Convenience macro to get the byte size of a provided named type * (struct/union/enum/typedef) in a target kernel. diff --git a/tools/lib/bpf/bpf_helpers.h b/tools/lib/bpf/bpf_helpers.h index 44df982d2a..7349b16b8e 100644 --- a/tools/lib/bpf/bpf_helpers.h +++ b/tools/lib/bpf/bpf_helpers.h @@ -22,12 +22,25 @@ * To allow use of SEC() with externs (e.g., for extern .maps declarations), * make sure __attribute__((unused)) doesn't trigger compilation warning. */ +#if __GNUC__ && !__clang__ + +/* + * Pragma macros are broken on GCC + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578 + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400 + */ +#define SEC(name) __attribute__((section(name), used)) + +#else + #define SEC(name) \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \ __attribute__((section(name), used)) \ _Pragma("GCC diagnostic pop") \ +#endif + /* Avoid 'linux/stddef.h' definition of '__always_inline'. */ #undef __always_inline #define __always_inline inline __attribute__((always_inline)) @@ -75,6 +88,30 @@ }) #endif +/* + * Compiler (optimization) barrier. + */ +#ifndef barrier +#define barrier() asm volatile("" ::: "memory") +#endif + +/* Variable-specific compiler (optimization) barrier. It's a no-op which makes + * compiler believe that there is some black box modification of a given + * variable and thus prevents compiler from making extra assumption about its + * value and potential simplifications and optimizations on this variable. + * + * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of + * a variable, making some code patterns unverifiable. Putting barrier_var() + * in place will ensure that cast is performed before the barrier_var() + * invocation, because compiler has to pessimistically assume that embedded + * asm section might perform some extra operations on that variable. + * + * This is a variable-specific variant of more global barrier(). + */ +#ifndef barrier_var +#define barrier_var(var) asm volatile("" : "=r"(var) : "0"(var)) +#endif + /* * Helper macro to throw a compilation error if __bpf_unreachable() gets * built into the resulting code. This works given BPF back end does not @@ -149,6 +186,8 @@ enum libbpf_tristate { #define __kconfig __attribute__((section(".kconfig"))) #define __ksym __attribute__((section(".ksyms"))) +#define __kptr __attribute__((btf_type_tag("kptr"))) +#define __kptr_ref __attribute__((btf_type_tag("kptr_ref"))) #ifndef ___bpf_concat #define ___bpf_concat(a, b) a ## b diff --git a/tools/lib/bpf/bpf_tracing.h b/tools/lib/bpf/bpf_tracing.h index e3a8c947e8..43ca3aff22 100644 --- a/tools/lib/bpf/bpf_tracing.h +++ b/tools/lib/bpf/bpf_tracing.h @@ -2,6 +2,8 @@ #ifndef __BPF_TRACING_H__ #define __BPF_TRACING_H__ +#include + /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ #if defined(__TARGET_ARCH_x86) #define bpf_target_x86 @@ -27,6 +29,9 @@ #elif defined(__TARGET_ARCH_riscv) #define bpf_target_riscv #define bpf_target_defined +#elif defined(__TARGET_ARCH_arc) + #define bpf_target_arc + #define bpf_target_defined #else /* Fall back to what the compiler says */ @@ -54,6 +59,9 @@ #elif defined(__riscv) && __riscv_xlen == 64 #define bpf_target_riscv #define bpf_target_defined +#elif defined(__arc__) + #define bpf_target_arc + #define bpf_target_defined #endif /* no compiler target */ #endif @@ -134,7 +142,7 @@ struct pt_regs___s390 { #define __PT_RC_REG gprs[2] #define __PT_SP_REG gprs[15] #define __PT_IP_REG psw.addr -#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; }) +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) #define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___s390 *)(x), orig_gpr2) #elif defined(bpf_target_arm) @@ -168,7 +176,7 @@ struct pt_regs___arm64 { #define __PT_RC_REG regs[0] #define __PT_SP_REG sp #define __PT_IP_REG pc -#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma("GCC error \"use PT_REGS_PARM1_CORE_SYSCALL() instead\""); 0l; }) +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) #define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ((const struct pt_regs___arm64 *)(x), orig_x0) #elif defined(bpf_target_mips) @@ -227,12 +235,29 @@ struct pt_regs___arm64 { #define __PT_PARM5_REG a4 #define __PT_RET_REG ra #define __PT_FP_REG s0 -#define __PT_RC_REG a5 +#define __PT_RC_REG a0 #define __PT_SP_REG sp #define __PT_IP_REG pc /* riscv does not select ARCH_HAS_SYSCALL_WRAPPER. */ #define PT_REGS_SYSCALL_REGS(ctx) ctx +#elif defined(bpf_target_arc) + +/* arc provides struct user_pt_regs instead of struct pt_regs to userspace */ +#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) +#define __PT_PARM1_REG scratch.r0 +#define __PT_PARM2_REG scratch.r1 +#define __PT_PARM3_REG scratch.r2 +#define __PT_PARM4_REG scratch.r3 +#define __PT_PARM5_REG scratch.r4 +#define __PT_RET_REG scratch.blink +#define __PT_FP_REG __unsupported__ +#define __PT_RC_REG scratch.r0 +#define __PT_SP_REG scratch.sp +#define __PT_IP_REG scratch.ret +/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ctx + #endif #if defined(bpf_target_defined) @@ -470,39 +495,69 @@ typeof(name(0)) name(struct pt_regs *ctx) \ } \ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) +/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */ #define ___bpf_syscall_args0() ctx -#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) -#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) -#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) -#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) -#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs) +#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs) +#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs) +#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs) +#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs) #define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) +/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ +#define ___bpf_syswrap_args0() ctx +#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) + /* - * BPF_KPROBE_SYSCALL is a variant of BPF_KPROBE, which is intended for + * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for * tracing syscall functions, like __x64_sys_close. It hides the underlying * platform-specific low-level way of getting syscall input arguments from * struct pt_regs, and provides a familiar typed and named function arguments * syntax and semantics of accessing syscall input parameters. * - * Original struct pt_regs* context is preserved as 'ctx' argument. This might + * Original struct pt_regs * context is preserved as 'ctx' argument. This might * be necessary when using BPF helpers like bpf_perf_event_output(). * - * This macro relies on BPF CO-RE support. + * At the moment BPF_KSYSCALL does not transparently handle all the calling + * convention quirks for the following syscalls: + * + * - mmap(): __ARCH_WANT_SYS_OLD_MMAP. + * - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and + * CONFIG_CLONE_BACKWARDS3. + * - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL. + * - compat syscalls. + * + * This may or may not change in the future. User needs to take extra measures + * to handle such quirks explicitly, if necessary. + * + * This macro relies on BPF CO-RE support and virtual __kconfig externs. */ -#define BPF_KPROBE_SYSCALL(name, args...) \ +#define BPF_KSYSCALL(name, args...) \ name(struct pt_regs *ctx); \ +extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ static __attribute__((always_inline)) typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ - struct pt_regs *regs = PT_REGS_SYSCALL_REGS(ctx); \ + struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \ + ? (struct pt_regs *)PT_REGS_PARM1(ctx) \ + : ctx; \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ - return ____##name(___bpf_syscall_args(args)); \ + if (LINUX_HAS_SYSCALL_WRAPPER) \ + return ____##name(___bpf_syswrap_args(args)); \ + else \ + return ____##name(___bpf_syscall_args(args)); \ _Pragma("GCC diagnostic pop") \ } \ static __attribute__((always_inline)) typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) +#define BPF_KPROBE_SYSCALL BPF_KSYSCALL + #endif diff --git a/tools/lib/bpf/btf.c b/tools/lib/bpf/btf.c index 1383e26c5d..2d14f1a52d 100644 --- a/tools/lib/bpf/btf.c +++ b/tools/lib/bpf/btf.c @@ -130,7 +130,7 @@ static inline __u64 ptr_to_u64(const void *ptr) /* Ensure given dynamically allocated memory region pointed to by *data* with * capacity of *cap_cnt* elements each taking *elem_sz* bytes has enough - * memory to accomodate *add_cnt* new elements, assuming *cur_cnt* elements + * memory to accommodate *add_cnt* new elements, assuming *cur_cnt* elements * are already used. At most *max_cnt* elements can be ever allocated. * If necessary, memory is reallocated and all existing data is copied over, * new pointer to the memory region is stored at *data, new memory region @@ -305,6 +305,8 @@ static int btf_type_size(const struct btf_type *t) return base_size + sizeof(__u32); case BTF_KIND_ENUM: return base_size + vlen * sizeof(struct btf_enum); + case BTF_KIND_ENUM64: + return base_size + vlen * sizeof(struct btf_enum64); case BTF_KIND_ARRAY: return base_size + sizeof(struct btf_array); case BTF_KIND_STRUCT: @@ -334,6 +336,7 @@ static void btf_bswap_type_base(struct btf_type *t) static int btf_bswap_type_rest(struct btf_type *t) { struct btf_var_secinfo *v; + struct btf_enum64 *e64; struct btf_member *m; struct btf_array *a; struct btf_param *p; @@ -361,6 +364,13 @@ static int btf_bswap_type_rest(struct btf_type *t) e->val = bswap_32(e->val); } return 0; + case BTF_KIND_ENUM64: + for (i = 0, e64 = btf_enum64(t); i < vlen; i++, e64++) { + e64->name_off = bswap_32(e64->name_off); + e64->val_lo32 = bswap_32(e64->val_lo32); + e64->val_hi32 = bswap_32(e64->val_hi32); + } + return 0; case BTF_KIND_ARRAY: a = btf_array(t); a->type = bswap_32(a->type); @@ -438,11 +448,6 @@ static int btf_parse_type_sec(struct btf *btf) return 0; } -__u32 btf__get_nr_types(const struct btf *btf) -{ - return btf->start_id + btf->nr_types - 1; -} - __u32 btf__type_cnt(const struct btf *btf) { return btf->start_id + btf->nr_types; @@ -472,9 +477,22 @@ const struct btf_type *btf__type_by_id(const struct btf *btf, __u32 type_id) static int determine_ptr_size(const struct btf *btf) { + static const char * const long_aliases[] = { + "long", + "long int", + "int long", + "unsigned long", + "long unsigned", + "unsigned long int", + "unsigned int long", + "long unsigned int", + "long int unsigned", + "int unsigned long", + "int long unsigned", + }; const struct btf_type *t; const char *name; - int i, n; + int i, j, n; if (btf->base_btf && btf->base_btf->ptr_sz > 0) return btf->base_btf->ptr_sz; @@ -485,15 +503,16 @@ static int determine_ptr_size(const struct btf *btf) if (!btf_is_int(t)) continue; + if (t->size != 4 && t->size != 8) + continue; + name = btf__name_by_offset(btf, t->name_off); if (!name) continue; - if (strcmp(name, "long int") == 0 || - strcmp(name, "long unsigned int") == 0) { - if (t->size != 4 && t->size != 8) - continue; - return t->size; + for (j = 0; j < ARRAY_SIZE(long_aliases); j++) { + if (strcmp(name, long_aliases[j]) == 0) + return t->size; } } @@ -597,6 +616,7 @@ __s64 btf__resolve_size(const struct btf *btf, __u32 type_id) case BTF_KIND_STRUCT: case BTF_KIND_UNION: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_DATASEC: case BTF_KIND_FLOAT: size = t->size; @@ -644,6 +664,7 @@ int btf__align_of(const struct btf *btf, __u32 id) switch (kind) { case BTF_KIND_INT: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_FLOAT: return min(btf_ptr_sz(btf), (size_t)t->size); case BTF_KIND_PTR: @@ -1382,92 +1403,6 @@ struct btf *btf__load_from_kernel_by_id(__u32 id) return btf__load_from_kernel_by_id_split(id, NULL); } -int btf__get_from_id(__u32 id, struct btf **btf) -{ - struct btf *res; - int err; - - *btf = NULL; - res = btf__load_from_kernel_by_id(id); - err = libbpf_get_error(res); - - if (err) - return libbpf_err(err); - - *btf = res; - return 0; -} - -int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, - __u32 expected_key_size, __u32 expected_value_size, - __u32 *key_type_id, __u32 *value_type_id) -{ - const struct btf_type *container_type; - const struct btf_member *key, *value; - const size_t max_name = 256; - char container_name[max_name]; - __s64 key_size, value_size; - __s32 container_id; - - if (snprintf(container_name, max_name, "____btf_map_%s", map_name) == max_name) { - pr_warn("map:%s length of '____btf_map_%s' is too long\n", - map_name, map_name); - return libbpf_err(-EINVAL); - } - - container_id = btf__find_by_name(btf, container_name); - if (container_id < 0) { - pr_debug("map:%s container_name:%s cannot be found in BTF. Missing BPF_ANNOTATE_KV_PAIR?\n", - map_name, container_name); - return libbpf_err(container_id); - } - - container_type = btf__type_by_id(btf, container_id); - if (!container_type) { - pr_warn("map:%s cannot find BTF type for container_id:%u\n", - map_name, container_id); - return libbpf_err(-EINVAL); - } - - if (!btf_is_struct(container_type) || btf_vlen(container_type) < 2) { - pr_warn("map:%s container_name:%s is an invalid container struct\n", - map_name, container_name); - return libbpf_err(-EINVAL); - } - - key = btf_members(container_type); - value = key + 1; - - key_size = btf__resolve_size(btf, key->type); - if (key_size < 0) { - pr_warn("map:%s invalid BTF key_type_size\n", map_name); - return libbpf_err(key_size); - } - - if (expected_key_size != key_size) { - pr_warn("map:%s btf_key_type_size:%u != map_def_key_size:%u\n", - map_name, (__u32)key_size, expected_key_size); - return libbpf_err(-EINVAL); - } - - value_size = btf__resolve_size(btf, value->type); - if (value_size < 0) { - pr_warn("map:%s invalid BTF value_type_size\n", map_name); - return libbpf_err(value_size); - } - - if (expected_value_size != value_size) { - pr_warn("map:%s btf_value_type_size:%u != map_def_value_size:%u\n", - map_name, (__u32)value_size, expected_value_size); - return libbpf_err(-EINVAL); - } - - *key_type_id = key->type; - *value_type_id = value->type; - - return 0; -} - static void btf_invalidate_raw_data(struct btf *btf) { if (btf->raw_data) { @@ -2115,20 +2050,8 @@ int btf__add_field(struct btf *btf, const char *name, int type_id, return 0; } -/* - * Append new BTF_KIND_ENUM type with: - * - *name* - name of the enum, can be NULL or empty for anonymous enums; - * - *byte_sz* - size of the enum, in bytes. - * - * Enum initially has no enum values in it (and corresponds to enum forward - * declaration). Enumerator values can be added by btf__add_enum_value() - * immediately after btf__add_enum() succeeds. - * - * Returns: - * - >0, type ID of newly added BTF type; - * - <0, on error. - */ -int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) +static int btf_add_enum_common(struct btf *btf, const char *name, __u32 byte_sz, + bool is_signed, __u8 kind) { struct btf_type *t; int sz, name_off = 0; @@ -2153,12 +2076,34 @@ int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) /* start out with vlen=0; it will be adjusted when adding enum values */ t->name_off = name_off; - t->info = btf_type_info(BTF_KIND_ENUM, 0, 0); + t->info = btf_type_info(kind, 0, is_signed); t->size = byte_sz; return btf_commit_type(btf, sz); } +/* + * Append new BTF_KIND_ENUM type with: + * - *name* - name of the enum, can be NULL or empty for anonymous enums; + * - *byte_sz* - size of the enum, in bytes. + * + * Enum initially has no enum values in it (and corresponds to enum forward + * declaration). Enumerator values can be added by btf__add_enum_value() + * immediately after btf__add_enum() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_enum(struct btf *btf, const char *name, __u32 byte_sz) +{ + /* + * set the signedness to be unsigned, it will change to signed + * if any later enumerator is negative. + */ + return btf_add_enum_common(btf, name, byte_sz, false, BTF_KIND_ENUM); +} + /* * Append new enum value for the current ENUM type with: * - *name* - name of the enumerator value, can't be NULL or empty; @@ -2206,6 +2151,82 @@ int btf__add_enum_value(struct btf *btf, const char *name, __s64 value) t = btf_last_type(btf); btf_type_inc_vlen(t); + /* if negative value, set signedness to signed */ + if (value < 0) + t->info = btf_type_info(btf_kind(t), btf_vlen(t), true); + + btf->hdr->type_len += sz; + btf->hdr->str_off += sz; + return 0; +} + +/* + * Append new BTF_KIND_ENUM64 type with: + * - *name* - name of the enum, can be NULL or empty for anonymous enums; + * - *byte_sz* - size of the enum, in bytes. + * - *is_signed* - whether the enum values are signed or not; + * + * Enum initially has no enum values in it (and corresponds to enum forward + * declaration). Enumerator values can be added by btf__add_enum64_value() + * immediately after btf__add_enum64() succeeds. + * + * Returns: + * - >0, type ID of newly added BTF type; + * - <0, on error. + */ +int btf__add_enum64(struct btf *btf, const char *name, __u32 byte_sz, + bool is_signed) +{ + return btf_add_enum_common(btf, name, byte_sz, is_signed, + BTF_KIND_ENUM64); +} + +/* + * Append new enum value for the current ENUM64 type with: + * - *name* - name of the enumerator value, can't be NULL or empty; + * - *value* - integer value corresponding to enum value *name*; + * Returns: + * - 0, on success; + * - <0, on error. + */ +int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value) +{ + struct btf_enum64 *v; + struct btf_type *t; + int sz, name_off; + + /* last type should be BTF_KIND_ENUM64 */ + if (btf->nr_types == 0) + return libbpf_err(-EINVAL); + t = btf_last_type(btf); + if (!btf_is_enum64(t)) + return libbpf_err(-EINVAL); + + /* non-empty name */ + if (!name || !name[0]) + return libbpf_err(-EINVAL); + + /* decompose and invalidate raw data */ + if (btf_ensure_modifiable(btf)) + return libbpf_err(-ENOMEM); + + sz = sizeof(struct btf_enum64); + v = btf_add_type_mem(btf, sz); + if (!v) + return libbpf_err(-ENOMEM); + + name_off = btf__add_str(btf, name); + if (name_off < 0) + return name_off; + + v->name_off = name_off; + v->val_lo32 = (__u32)value; + v->val_hi32 = value >> 32; + + /* update parent type's vlen */ + t = btf_last_type(btf); + btf_type_inc_vlen(t); + btf->hdr->type_len += sz; btf->hdr->str_off += sz; return 0; @@ -2626,6 +2647,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, const struct btf_ext_info_sec *sinfo; struct btf_ext_info *ext_info; __u32 info_left, record_size; + size_t sec_cnt = 0; /* The start of the info sec (including the __u32 record_size). */ void *info; @@ -2689,8 +2711,7 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, return -EINVAL; } - total_record_size = sec_hdrlen + - (__u64)num_records * record_size; + total_record_size = sec_hdrlen + (__u64)num_records * record_size; if (info_left < total_record_size) { pr_debug("%s section has incorrect num_records in .BTF.ext\n", ext_sec->desc); @@ -2699,12 +2720,14 @@ static int btf_ext_setup_info(struct btf_ext *btf_ext, info_left -= total_record_size; sinfo = (void *)sinfo + total_record_size; + sec_cnt++; } ext_info = ext_sec->ext_info; ext_info->len = ext_sec->len - sizeof(__u32); ext_info->rec_size = record_size; ext_info->info = info + sizeof(__u32); + ext_info->sec_cnt = sec_cnt; return 0; } @@ -2788,6 +2811,9 @@ void btf_ext__free(struct btf_ext *btf_ext) { if (IS_ERR_OR_NULL(btf_ext)) return; + free(btf_ext->func_info.sec_idxs); + free(btf_ext->line_info.sec_idxs); + free(btf_ext->core_relo_info.sec_idxs); free(btf_ext->data); free(btf_ext); } @@ -2826,10 +2852,8 @@ struct btf_ext *btf_ext__new(const __u8 *data, __u32 size) if (err) goto done; - if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) { - err = -EINVAL; - goto done; - } + if (btf_ext->hdr->hdr_len < offsetofend(struct btf_ext_header, core_relo_len)) + goto done; /* skip core relos parsing */ err = btf_ext_setup_core_relos(btf_ext); if (err) @@ -2850,81 +2874,6 @@ const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, __u32 *size) return btf_ext->data; } -static int btf_ext_reloc_info(const struct btf *btf, - const struct btf_ext_info *ext_info, - const char *sec_name, __u32 insns_cnt, - void **info, __u32 *cnt) -{ - __u32 sec_hdrlen = sizeof(struct btf_ext_info_sec); - __u32 i, record_size, existing_len, records_len; - struct btf_ext_info_sec *sinfo; - const char *info_sec_name; - __u64 remain_len; - void *data; - - record_size = ext_info->rec_size; - sinfo = ext_info->info; - remain_len = ext_info->len; - while (remain_len > 0) { - records_len = sinfo->num_info * record_size; - info_sec_name = btf__name_by_offset(btf, sinfo->sec_name_off); - if (strcmp(info_sec_name, sec_name)) { - remain_len -= sec_hdrlen + records_len; - sinfo = (void *)sinfo + sec_hdrlen + records_len; - continue; - } - - existing_len = (*cnt) * record_size; - data = realloc(*info, existing_len + records_len); - if (!data) - return libbpf_err(-ENOMEM); - - memcpy(data + existing_len, sinfo->data, records_len); - /* adjust insn_off only, the rest data will be passed - * to the kernel. - */ - for (i = 0; i < sinfo->num_info; i++) { - __u32 *insn_off; - - insn_off = data + existing_len + (i * record_size); - *insn_off = *insn_off / sizeof(struct bpf_insn) + insns_cnt; - } - *info = data; - *cnt += sinfo->num_info; - return 0; - } - - return libbpf_err(-ENOENT); -} - -int btf_ext__reloc_func_info(const struct btf *btf, - const struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **func_info, __u32 *cnt) -{ - return btf_ext_reloc_info(btf, &btf_ext->func_info, sec_name, - insns_cnt, func_info, cnt); -} - -int btf_ext__reloc_line_info(const struct btf *btf, - const struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **line_info, __u32 *cnt) -{ - return btf_ext_reloc_info(btf, &btf_ext->line_info, sec_name, - insns_cnt, line_info, cnt); -} - -__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext) -{ - return btf_ext->func_info.rec_size; -} - -__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext) -{ - return btf_ext->line_info.rec_size; -} - struct btf_dedup; static struct btf_dedup *btf_dedup_new(struct btf *btf, const struct btf_dedup_opts *opts); @@ -3074,9 +3023,7 @@ static int btf_dedup_remap_types(struct btf_dedup *d); * deduplicating structs/unions is described in greater details in comments for * `btf_dedup_is_equiv` function. */ - -DEFAULT_VERSION(btf__dedup_v0_6_0, btf__dedup, LIBBPF_0.6.0) -int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts) +int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts) { struct btf_dedup *d; int err; @@ -3136,19 +3083,6 @@ int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts) return libbpf_err(err); } -COMPAT_VERSION(btf__dedup_deprecated, btf__dedup, LIBBPF_0.0.2) -int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *unused_opts) -{ - LIBBPF_OPTS(btf_dedup_opts, opts, .btf_ext = btf_ext); - - if (unused_opts) { - pr_warn("please use new version of btf__dedup() that supports options\n"); - return libbpf_err(-ENOTSUP); - } - - return btf__dedup(btf, &opts); -} - #define BTF_UNPROCESSED_ID ((__u32)-1) #define BTF_IN_PROGRESS_ID ((__u32)-2) @@ -3467,7 +3401,7 @@ static bool btf_equal_int_tag(struct btf_type *t1, struct btf_type *t2) return info1 == info2; } -/* Calculate type signature hash of ENUM. */ +/* Calculate type signature hash of ENUM/ENUM64. */ static long btf_hash_enum(struct btf_type *t) { long h; @@ -3501,9 +3435,31 @@ static bool btf_equal_enum(struct btf_type *t1, struct btf_type *t2) return true; } +static bool btf_equal_enum64(struct btf_type *t1, struct btf_type *t2) +{ + const struct btf_enum64 *m1, *m2; + __u16 vlen; + int i; + + if (!btf_equal_common(t1, t2)) + return false; + + vlen = btf_vlen(t1); + m1 = btf_enum64(t1); + m2 = btf_enum64(t2); + for (i = 0; i < vlen; i++) { + if (m1->name_off != m2->name_off || m1->val_lo32 != m2->val_lo32 || + m1->val_hi32 != m2->val_hi32) + return false; + m1++; + m2++; + } + return true; +} + static inline bool btf_is_enum_fwd(struct btf_type *t) { - return btf_is_enum(t) && btf_vlen(t) == 0; + return btf_is_any_enum(t) && btf_vlen(t) == 0; } static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) @@ -3516,6 +3472,17 @@ static bool btf_compat_enum(struct btf_type *t1, struct btf_type *t2) t1->size == t2->size; } +static bool btf_compat_enum64(struct btf_type *t1, struct btf_type *t2) +{ + if (!btf_is_enum_fwd(t1) && !btf_is_enum_fwd(t2)) + return btf_equal_enum64(t1, t2); + + /* ignore vlen when comparing */ + return t1->name_off == t2->name_off && + (t1->info & ~0xffff) == (t2->info & ~0xffff) && + t1->size == t2->size; +} + /* * Calculate type signature hash of STRUCT/UNION, ignoring referenced type IDs, * as referenced type IDs equivalence is established separately during type @@ -3728,6 +3695,7 @@ static int btf_dedup_prep(struct btf_dedup *d) h = btf_hash_int_decl_tag(t); break; case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: h = btf_hash_enum(t); break; case BTF_KIND_STRUCT: @@ -3817,6 +3785,27 @@ static int btf_dedup_prim_type(struct btf_dedup *d, __u32 type_id) } break; + case BTF_KIND_ENUM64: + h = btf_hash_enum(t); + for_each_dedup_cand(d, hash_entry, h) { + cand_id = (__u32)(long)hash_entry->value; + cand = btf_type_by_id(d->btf, cand_id); + if (btf_equal_enum64(t, cand)) { + new_id = cand_id; + break; + } + if (btf_compat_enum64(t, cand)) { + if (btf_is_enum_fwd(t)) { + /* resolve fwd to full enum */ + new_id = cand_id; + break; + } + /* resolve canonical enum fwd to full enum */ + d->map[cand_id] = type_id; + } + } + break; + case BTF_KIND_FWD: case BTF_KIND_FLOAT: h = btf_hash_common(t); @@ -4112,6 +4101,9 @@ static int btf_dedup_is_equiv(struct btf_dedup *d, __u32 cand_id, case BTF_KIND_ENUM: return btf_compat_enum(cand_type, canon_type); + case BTF_KIND_ENUM64: + return btf_compat_enum64(cand_type, canon_type); + case BTF_KIND_FWD: case BTF_KIND_FLOAT: return btf_equal_common(cand_type, canon_type); @@ -4714,6 +4706,7 @@ int btf_type_visit_type_ids(struct btf_type *t, type_id_visit_fn visit, void *ct case BTF_KIND_INT: case BTF_KIND_FLOAT: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: return 0; case BTF_KIND_FWD: @@ -4808,6 +4801,16 @@ int btf_type_visit_str_offs(struct btf_type *t, str_off_visit_fn visit, void *ct } break; } + case BTF_KIND_ENUM64: { + struct btf_enum64 *m = btf_enum64(t); + + for (i = 0, n = btf_vlen(t); i < n; i++, m++) { + err = visit(&m->name_off, ctx); + if (err) + return err; + } + break; + } case BTF_KIND_FUNC_PROTO: { struct btf_param *m = btf_params(t); diff --git a/tools/lib/bpf/btf.h b/tools/lib/bpf/btf.h index 951ac74757..583760df83 100644 --- a/tools/lib/bpf/btf.h +++ b/tools/lib/bpf/btf.h @@ -120,20 +120,12 @@ LIBBPF_API struct btf *libbpf_find_kernel_btf(void); LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id); LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf); -LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_from_kernel_by_id instead") -LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); -LIBBPF_DEPRECATED_SINCE(0, 6, "intended for internal libbpf use only") -LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf); -LIBBPF_DEPRECATED_SINCE(0, 6, "use btf__load_into_kernel instead") -LIBBPF_API int btf__load(struct btf *btf); LIBBPF_API int btf__load_into_kernel(struct btf *btf); LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, const char *type_name); LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf, const char *type_name, __u32 kind); -LIBBPF_DEPRECATED_SINCE(0, 7, "use btf__type_cnt() instead; note that btf__get_nr_types() == btf__type_cnt() - 1") -LIBBPF_API __u32 btf__get_nr_types(const struct btf *btf); LIBBPF_API __u32 btf__type_cnt(const struct btf *btf); LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf); LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, @@ -150,29 +142,10 @@ LIBBPF_API void btf__set_fd(struct btf *btf, int fd); LIBBPF_API const void *btf__raw_data(const struct btf *btf, __u32 *size); LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); -LIBBPF_DEPRECATED_SINCE(0, 7, "this API is not necessary when BTF-defined maps are used") -LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, - __u32 expected_key_size, - __u32 expected_value_size, - __u32 *key_type_id, __u32 *value_type_id); LIBBPF_API struct btf_ext *btf_ext__new(const __u8 *data, __u32 size); LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext); LIBBPF_API const void *btf_ext__raw_data(const struct btf_ext *btf_ext, __u32 *size); -LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions") -int btf_ext__reloc_func_info(const struct btf *btf, - const struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **func_info, __u32 *cnt); -LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_line_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions") -int btf_ext__reloc_line_info(const struct btf *btf, - const struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **line_info, __u32 *cnt); -LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info is deprecated; write custom func_info parsing to fetch rec_size") -__u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); -LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_line_info is deprecated; write custom line_info parsing to fetch rec_size") -__u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); LIBBPF_API int btf__find_str(struct btf *btf, const char *s); LIBBPF_API int btf__add_str(struct btf *btf, const char *s); @@ -215,6 +188,8 @@ LIBBPF_API int btf__add_field(struct btf *btf, const char *name, int field_type_ /* enum construction APIs */ LIBBPF_API int btf__add_enum(struct btf *btf, const char *name, __u32 bytes_sz); LIBBPF_API int btf__add_enum_value(struct btf *btf, const char *name, __s64 value); +LIBBPF_API int btf__add_enum64(struct btf *btf, const char *name, __u32 bytes_sz, bool is_signed); +LIBBPF_API int btf__add_enum64_value(struct btf *btf, const char *name, __u64 value); enum btf_fwd_kind { BTF_FWD_STRUCT = 0, @@ -257,22 +232,12 @@ struct btf_dedup_opts { LIBBPF_API int btf__dedup(struct btf *btf, const struct btf_dedup_opts *opts); -LIBBPF_API int btf__dedup_v0_6_0(struct btf *btf, const struct btf_dedup_opts *opts); - -LIBBPF_DEPRECATED_SINCE(0, 7, "use btf__dedup() instead") -LIBBPF_API int btf__dedup_deprecated(struct btf *btf, struct btf_ext *btf_ext, const void *opts); -#define btf__dedup(...) ___libbpf_overload(___btf_dedup, __VA_ARGS__) -#define ___btf_dedup3(btf, btf_ext, opts) btf__dedup_deprecated(btf, btf_ext, opts) -#define ___btf_dedup2(btf, opts) btf__dedup(btf, opts) - struct btf_dump; struct btf_dump_opts { - union { - size_t sz; - void *ctx; /* DEPRECATED: will be gone in v1.0 */ - }; + size_t sz; }; +#define btf_dump_opts__last_field sz typedef void (*btf_dump_printf_fn_t)(void *ctx, const char *fmt, va_list args); @@ -281,51 +246,6 @@ LIBBPF_API struct btf_dump *btf_dump__new(const struct btf *btf, void *ctx, const struct btf_dump_opts *opts); -LIBBPF_API struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf, - btf_dump_printf_fn_t printf_fn, - void *ctx, - const struct btf_dump_opts *opts); - -LIBBPF_API struct btf_dump *btf_dump__new_deprecated(const struct btf *btf, - const struct btf_ext *btf_ext, - const struct btf_dump_opts *opts, - btf_dump_printf_fn_t printf_fn); - -/* Choose either btf_dump__new() or btf_dump__new_deprecated() based on the - * type of 4th argument. If it's btf_dump's print callback, use deprecated - * API; otherwise, choose the new btf_dump__new(). ___libbpf_override() - * doesn't work here because both variants have 4 input arguments. - * - * (void *) casts are necessary to avoid compilation warnings about type - * mismatches, because even though __builtin_choose_expr() only ever evaluates - * one side the other side still has to satisfy type constraints (this is - * compiler implementation limitation which might be lifted eventually, - * according to the documentation). So passing struct btf_ext in place of - * btf_dump_printf_fn_t would be generating compilation warning. Casting to - * void * avoids this issue. - * - * Also, two type compatibility checks for a function and function pointer are - * required because passing function reference into btf_dump__new() as - * btf_dump__new(..., my_callback, ...) and as btf_dump__new(..., - * &my_callback, ...) (not explicit ampersand in the latter case) actually - * differs as far as __builtin_types_compatible_p() is concerned. Thus two - * checks are combined to detect callback argument. - * - * The rest works just like in case of ___libbpf_override() usage with symbol - * versioning. - * - * C++ compilers don't support __builtin_types_compatible_p(), so at least - * don't screw up compilation for them and let C++ users pick btf_dump__new - * vs btf_dump__new_deprecated explicitly. - */ -#ifndef __cplusplus -#define btf_dump__new(a1, a2, a3, a4) __builtin_choose_expr( \ - __builtin_types_compatible_p(typeof(a4), btf_dump_printf_fn_t) || \ - __builtin_types_compatible_p(typeof(a4), void(void *, const char *, va_list)), \ - btf_dump__new_deprecated((void *)a1, (void *)a2, (void *)a3, (void *)a4), \ - btf_dump__new((void *)a1, (void *)a2, (void *)a3, (void *)a4)) -#endif - LIBBPF_API void btf_dump__free(struct btf_dump *d); LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id); @@ -393,9 +313,10 @@ btf_dump__dump_type_data(struct btf_dump *d, __u32 id, #ifndef BTF_KIND_FLOAT #define BTF_KIND_FLOAT 16 /* Floating point */ #endif -/* The kernel header switched to enums, so these two were never #defined */ +/* The kernel header switched to enums, so the following were never #defined */ #define BTF_KIND_DECL_TAG 17 /* Decl Tag */ #define BTF_KIND_TYPE_TAG 18 /* Type Tag */ +#define BTF_KIND_ENUM64 19 /* Enum for up-to 64bit values */ static inline __u16 btf_kind(const struct btf_type *t) { @@ -454,6 +375,11 @@ static inline bool btf_is_enum(const struct btf_type *t) return btf_kind(t) == BTF_KIND_ENUM; } +static inline bool btf_is_enum64(const struct btf_type *t) +{ + return btf_kind(t) == BTF_KIND_ENUM64; +} + static inline bool btf_is_fwd(const struct btf_type *t) { return btf_kind(t) == BTF_KIND_FWD; @@ -524,6 +450,18 @@ static inline bool btf_is_type_tag(const struct btf_type *t) return btf_kind(t) == BTF_KIND_TYPE_TAG; } +static inline bool btf_is_any_enum(const struct btf_type *t) +{ + return btf_is_enum(t) || btf_is_enum64(t); +} + +static inline bool btf_kind_core_compat(const struct btf_type *t1, + const struct btf_type *t2) +{ + return btf_kind(t1) == btf_kind(t2) || + (btf_is_any_enum(t1) && btf_is_any_enum(t2)); +} + static inline __u8 btf_int_encoding(const struct btf_type *t) { return BTF_INT_ENCODING(*(__u32 *)(t + 1)); @@ -549,6 +487,16 @@ static inline struct btf_enum *btf_enum(const struct btf_type *t) return (struct btf_enum *)(t + 1); } +static inline struct btf_enum64 *btf_enum64(const struct btf_type *t) +{ + return (struct btf_enum64 *)(t + 1); +} + +static inline __u64 btf_enum64_value(const struct btf_enum64 *e) +{ + return ((__u64)e->val_hi32 << 32) | e->val_lo32; +} + static inline struct btf_member *btf_members(const struct btf_type *t) { return (struct btf_member *)(t + 1); diff --git a/tools/lib/bpf/btf_dump.c b/tools/lib/bpf/btf_dump.c index 6b1bc1f437..627edb5bb6 100644 --- a/tools/lib/bpf/btf_dump.c +++ b/tools/lib/bpf/btf_dump.c @@ -144,15 +144,17 @@ static void btf_dump_printf(const struct btf_dump *d, const char *fmt, ...) static int btf_dump_mark_referenced(struct btf_dump *d); static int btf_dump_resize(struct btf_dump *d); -DEFAULT_VERSION(btf_dump__new_v0_6_0, btf_dump__new, LIBBPF_0.6.0) -struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf, - btf_dump_printf_fn_t printf_fn, - void *ctx, - const struct btf_dump_opts *opts) +struct btf_dump *btf_dump__new(const struct btf *btf, + btf_dump_printf_fn_t printf_fn, + void *ctx, + const struct btf_dump_opts *opts) { struct btf_dump *d; int err; + if (!OPTS_VALID(opts, btf_dump_opts)) + return libbpf_err_ptr(-EINVAL); + if (!printf_fn) return libbpf_err_ptr(-EINVAL); @@ -188,17 +190,6 @@ struct btf_dump *btf_dump__new_v0_6_0(const struct btf *btf, return libbpf_err_ptr(err); } -COMPAT_VERSION(btf_dump__new_deprecated, btf_dump__new, LIBBPF_0.0.4) -struct btf_dump *btf_dump__new_deprecated(const struct btf *btf, - const struct btf_ext *btf_ext, - const struct btf_dump_opts *opts, - btf_dump_printf_fn_t printf_fn) -{ - if (!printf_fn) - return libbpf_err_ptr(-EINVAL); - return btf_dump__new_v0_6_0(btf, printf_fn, opts ? opts->ctx : NULL, opts); -} - static int btf_dump_resize(struct btf_dump *d) { int err, last_id = btf__type_cnt(d->btf) - 1; @@ -318,6 +309,7 @@ static int btf_dump_mark_referenced(struct btf_dump *d) switch (btf_kind(t)) { case BTF_KIND_INT: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_FWD: case BTF_KIND_FLOAT: break; @@ -538,6 +530,7 @@ static int btf_dump_order_type(struct btf_dump *d, __u32 id, bool through_ptr) return 1; } case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_FWD: /* * non-anonymous or non-referenced enums are top-level @@ -739,6 +732,7 @@ static void btf_dump_emit_type(struct btf_dump *d, __u32 id, __u32 cont_id) tstate->emit_state = EMITTED; break; case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: if (top_level_def) { btf_dump_emit_enum_def(d, id, t, 0); btf_dump_printf(d, ";\n\n"); @@ -989,38 +983,81 @@ static void btf_dump_emit_enum_fwd(struct btf_dump *d, __u32 id, btf_dump_printf(d, "enum %s", btf_dump_type_name(d, id)); } +static void btf_dump_emit_enum32_val(struct btf_dump *d, + const struct btf_type *t, + int lvl, __u16 vlen) +{ + const struct btf_enum *v = btf_enum(t); + bool is_signed = btf_kflag(t); + const char *fmt_str; + const char *name; + size_t dup_cnt; + int i; + + for (i = 0; i < vlen; i++, v++) { + name = btf_name_of(d, v->name_off); + /* enumerators share namespace with typedef idents */ + dup_cnt = btf_dump_name_dups(d, d->ident_names, name); + if (dup_cnt > 1) { + fmt_str = is_signed ? "\n%s%s___%zd = %d," : "\n%s%s___%zd = %u,"; + btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, dup_cnt, v->val); + } else { + fmt_str = is_signed ? "\n%s%s = %d," : "\n%s%s = %u,"; + btf_dump_printf(d, fmt_str, pfx(lvl + 1), name, v->val); + } + } +} + +static void btf_dump_emit_enum64_val(struct btf_dump *d, + const struct btf_type *t, + int lvl, __u16 vlen) +{ + const struct btf_enum64 *v = btf_enum64(t); + bool is_signed = btf_kflag(t); + const char *fmt_str; + const char *name; + size_t dup_cnt; + __u64 val; + int i; + + for (i = 0; i < vlen; i++, v++) { + name = btf_name_of(d, v->name_off); + dup_cnt = btf_dump_name_dups(d, d->ident_names, name); + val = btf_enum64_value(v); + if (dup_cnt > 1) { + fmt_str = is_signed ? "\n%s%s___%zd = %lldLL," + : "\n%s%s___%zd = %lluULL,"; + btf_dump_printf(d, fmt_str, + pfx(lvl + 1), name, dup_cnt, + (unsigned long long)val); + } else { + fmt_str = is_signed ? "\n%s%s = %lldLL," + : "\n%s%s = %lluULL,"; + btf_dump_printf(d, fmt_str, + pfx(lvl + 1), name, + (unsigned long long)val); + } + } +} static void btf_dump_emit_enum_def(struct btf_dump *d, __u32 id, const struct btf_type *t, int lvl) { - const struct btf_enum *v = btf_enum(t); __u16 vlen = btf_vlen(t); - const char *name; - size_t dup_cnt; - int i; btf_dump_printf(d, "enum%s%s", t->name_off ? " " : "", btf_dump_type_name(d, id)); - if (vlen) { - btf_dump_printf(d, " {"); - for (i = 0; i < vlen; i++, v++) { - name = btf_name_of(d, v->name_off); - /* enumerators share namespace with typedef idents */ - dup_cnt = btf_dump_name_dups(d, d->ident_names, name); - if (dup_cnt > 1) { - btf_dump_printf(d, "\n%s%s___%zu = %u,", - pfx(lvl + 1), name, dup_cnt, - (__u32)v->val); - } else { - btf_dump_printf(d, "\n%s%s = %u,", - pfx(lvl + 1), name, - (__u32)v->val); - } - } - btf_dump_printf(d, "\n%s}", pfx(lvl)); - } + if (!vlen) + return; + + btf_dump_printf(d, " {"); + if (btf_is_enum(t)) + btf_dump_emit_enum32_val(d, t, lvl, vlen); + else + btf_dump_emit_enum64_val(d, t, lvl, vlen); + btf_dump_printf(d, "\n%s}", pfx(lvl)); } static void btf_dump_emit_fwd_def(struct btf_dump *d, __u32 id, @@ -1178,6 +1215,7 @@ static void btf_dump_emit_type_decl(struct btf_dump *d, __u32 id, break; case BTF_KIND_INT: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_FWD: case BTF_KIND_STRUCT: case BTF_KIND_UNION: @@ -1312,6 +1350,7 @@ static void btf_dump_emit_type_chain(struct btf_dump *d, btf_dump_emit_struct_fwd(d, id, t); break; case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: btf_dump_emit_mods(d, decls); /* inline anonymous enum */ if (t->name_off == 0 && !d->skip_anon_defs) @@ -1988,7 +2027,8 @@ static int btf_dump_get_enum_value(struct btf_dump *d, __u32 id, __s64 *value) { - /* handle unaligned enum value */ + bool is_signed = btf_kflag(t); + if (!ptr_is_aligned(d->btf, id, data)) { __u64 val; int err; @@ -2005,13 +2045,13 @@ static int btf_dump_get_enum_value(struct btf_dump *d, *value = *(__s64 *)data; return 0; case 4: - *value = *(__s32 *)data; + *value = is_signed ? (__s64)*(__s32 *)data : *(__u32 *)data; return 0; case 2: - *value = *(__s16 *)data; + *value = is_signed ? *(__s16 *)data : *(__u16 *)data; return 0; case 1: - *value = *(__s8 *)data; + *value = is_signed ? *(__s8 *)data : *(__u8 *)data; return 0; default: pr_warn("unexpected size %d for enum, id:[%u]\n", t->size, id); @@ -2024,7 +2064,7 @@ static int btf_dump_enum_data(struct btf_dump *d, __u32 id, const void *data) { - const struct btf_enum *e; + bool is_signed; __s64 value; int i, err; @@ -2032,14 +2072,31 @@ static int btf_dump_enum_data(struct btf_dump *d, if (err) return err; - for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) { - if (value != e->val) - continue; - btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off)); - return 0; - } + is_signed = btf_kflag(t); + if (btf_is_enum(t)) { + const struct btf_enum *e; - btf_dump_type_values(d, "%d", value); + for (i = 0, e = btf_enum(t); i < btf_vlen(t); i++, e++) { + if (value != e->val) + continue; + btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off)); + return 0; + } + + btf_dump_type_values(d, is_signed ? "%d" : "%u", value); + } else { + const struct btf_enum64 *e; + + for (i = 0, e = btf_enum64(t); i < btf_vlen(t); i++, e++) { + if (value != btf_enum64_value(e)) + continue; + btf_dump_type_values(d, "%s", btf_name_of(d, e->name_off)); + return 0; + } + + btf_dump_type_values(d, is_signed ? "%lldLL" : "%lluULL", + (unsigned long long)value); + } return 0; } @@ -2099,6 +2156,7 @@ static int btf_dump_type_data_check_overflow(struct btf_dump *d, case BTF_KIND_FLOAT: case BTF_KIND_PTR: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: if (data + bits_offset / 8 + size > d->typed_dump->data_end) return -E2BIG; break; @@ -2203,6 +2261,7 @@ static int btf_dump_type_data_check_zero(struct btf_dump *d, return -ENODATA; } case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: err = btf_dump_get_enum_value(d, t, data, id, &value); if (err) return err; @@ -2275,6 +2334,7 @@ static int btf_dump_dump_type_data(struct btf_dump *d, err = btf_dump_struct_data(d, t, id, data); break; case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: /* handle bitfield and int enum values */ if (bit_sz) { __u64 print_num; diff --git a/tools/lib/bpf/gen_loader.c b/tools/lib/bpf/gen_loader.c index 927745b080..23f5c46708 100644 --- a/tools/lib/bpf/gen_loader.c +++ b/tools/lib/bpf/gen_loader.c @@ -533,7 +533,7 @@ void bpf_gen__record_attach_target(struct bpf_gen *gen, const char *attach_name, gen->attach_kind = kind; ret = snprintf(gen->attach_target, sizeof(gen->attach_target), "%s%s", prefix, attach_name); - if (ret == sizeof(gen->attach_target)) + if (ret >= sizeof(gen->attach_target)) gen->error = -ENOSPC; } diff --git a/tools/lib/bpf/libbpf.c b/tools/lib/bpf/libbpf.c index 809fe209cd..50d41815f4 100644 --- a/tools/lib/bpf/libbpf.c +++ b/tools/lib/bpf/libbpf.c @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -72,6 +71,135 @@ static struct bpf_map *bpf_object__add_map(struct bpf_object *obj); static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog); +static const char * const attach_type_name[] = { + [BPF_CGROUP_INET_INGRESS] = "cgroup_inet_ingress", + [BPF_CGROUP_INET_EGRESS] = "cgroup_inet_egress", + [BPF_CGROUP_INET_SOCK_CREATE] = "cgroup_inet_sock_create", + [BPF_CGROUP_INET_SOCK_RELEASE] = "cgroup_inet_sock_release", + [BPF_CGROUP_SOCK_OPS] = "cgroup_sock_ops", + [BPF_CGROUP_DEVICE] = "cgroup_device", + [BPF_CGROUP_INET4_BIND] = "cgroup_inet4_bind", + [BPF_CGROUP_INET6_BIND] = "cgroup_inet6_bind", + [BPF_CGROUP_INET4_CONNECT] = "cgroup_inet4_connect", + [BPF_CGROUP_INET6_CONNECT] = "cgroup_inet6_connect", + [BPF_CGROUP_INET4_POST_BIND] = "cgroup_inet4_post_bind", + [BPF_CGROUP_INET6_POST_BIND] = "cgroup_inet6_post_bind", + [BPF_CGROUP_INET4_GETPEERNAME] = "cgroup_inet4_getpeername", + [BPF_CGROUP_INET6_GETPEERNAME] = "cgroup_inet6_getpeername", + [BPF_CGROUP_INET4_GETSOCKNAME] = "cgroup_inet4_getsockname", + [BPF_CGROUP_INET6_GETSOCKNAME] = "cgroup_inet6_getsockname", + [BPF_CGROUP_UDP4_SENDMSG] = "cgroup_udp4_sendmsg", + [BPF_CGROUP_UDP6_SENDMSG] = "cgroup_udp6_sendmsg", + [BPF_CGROUP_SYSCTL] = "cgroup_sysctl", + [BPF_CGROUP_UDP4_RECVMSG] = "cgroup_udp4_recvmsg", + [BPF_CGROUP_UDP6_RECVMSG] = "cgroup_udp6_recvmsg", + [BPF_CGROUP_GETSOCKOPT] = "cgroup_getsockopt", + [BPF_CGROUP_SETSOCKOPT] = "cgroup_setsockopt", + [BPF_SK_SKB_STREAM_PARSER] = "sk_skb_stream_parser", + [BPF_SK_SKB_STREAM_VERDICT] = "sk_skb_stream_verdict", + [BPF_SK_SKB_VERDICT] = "sk_skb_verdict", + [BPF_SK_MSG_VERDICT] = "sk_msg_verdict", + [BPF_LIRC_MODE2] = "lirc_mode2", + [BPF_FLOW_DISSECTOR] = "flow_dissector", + [BPF_TRACE_RAW_TP] = "trace_raw_tp", + [BPF_TRACE_FENTRY] = "trace_fentry", + [BPF_TRACE_FEXIT] = "trace_fexit", + [BPF_MODIFY_RETURN] = "modify_return", + [BPF_LSM_MAC] = "lsm_mac", + [BPF_LSM_CGROUP] = "lsm_cgroup", + [BPF_SK_LOOKUP] = "sk_lookup", + [BPF_TRACE_ITER] = "trace_iter", + [BPF_XDP_DEVMAP] = "xdp_devmap", + [BPF_XDP_CPUMAP] = "xdp_cpumap", + [BPF_XDP] = "xdp", + [BPF_SK_REUSEPORT_SELECT] = "sk_reuseport_select", + [BPF_SK_REUSEPORT_SELECT_OR_MIGRATE] = "sk_reuseport_select_or_migrate", + [BPF_PERF_EVENT] = "perf_event", + [BPF_TRACE_KPROBE_MULTI] = "trace_kprobe_multi", +}; + +static const char * const link_type_name[] = { + [BPF_LINK_TYPE_UNSPEC] = "unspec", + [BPF_LINK_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", + [BPF_LINK_TYPE_TRACING] = "tracing", + [BPF_LINK_TYPE_CGROUP] = "cgroup", + [BPF_LINK_TYPE_ITER] = "iter", + [BPF_LINK_TYPE_NETNS] = "netns", + [BPF_LINK_TYPE_XDP] = "xdp", + [BPF_LINK_TYPE_PERF_EVENT] = "perf_event", + [BPF_LINK_TYPE_KPROBE_MULTI] = "kprobe_multi", + [BPF_LINK_TYPE_STRUCT_OPS] = "struct_ops", +}; + +static const char * const map_type_name[] = { + [BPF_MAP_TYPE_UNSPEC] = "unspec", + [BPF_MAP_TYPE_HASH] = "hash", + [BPF_MAP_TYPE_ARRAY] = "array", + [BPF_MAP_TYPE_PROG_ARRAY] = "prog_array", + [BPF_MAP_TYPE_PERF_EVENT_ARRAY] = "perf_event_array", + [BPF_MAP_TYPE_PERCPU_HASH] = "percpu_hash", + [BPF_MAP_TYPE_PERCPU_ARRAY] = "percpu_array", + [BPF_MAP_TYPE_STACK_TRACE] = "stack_trace", + [BPF_MAP_TYPE_CGROUP_ARRAY] = "cgroup_array", + [BPF_MAP_TYPE_LRU_HASH] = "lru_hash", + [BPF_MAP_TYPE_LRU_PERCPU_HASH] = "lru_percpu_hash", + [BPF_MAP_TYPE_LPM_TRIE] = "lpm_trie", + [BPF_MAP_TYPE_ARRAY_OF_MAPS] = "array_of_maps", + [BPF_MAP_TYPE_HASH_OF_MAPS] = "hash_of_maps", + [BPF_MAP_TYPE_DEVMAP] = "devmap", + [BPF_MAP_TYPE_DEVMAP_HASH] = "devmap_hash", + [BPF_MAP_TYPE_SOCKMAP] = "sockmap", + [BPF_MAP_TYPE_CPUMAP] = "cpumap", + [BPF_MAP_TYPE_XSKMAP] = "xskmap", + [BPF_MAP_TYPE_SOCKHASH] = "sockhash", + [BPF_MAP_TYPE_CGROUP_STORAGE] = "cgroup_storage", + [BPF_MAP_TYPE_REUSEPORT_SOCKARRAY] = "reuseport_sockarray", + [BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE] = "percpu_cgroup_storage", + [BPF_MAP_TYPE_QUEUE] = "queue", + [BPF_MAP_TYPE_STACK] = "stack", + [BPF_MAP_TYPE_SK_STORAGE] = "sk_storage", + [BPF_MAP_TYPE_STRUCT_OPS] = "struct_ops", + [BPF_MAP_TYPE_RINGBUF] = "ringbuf", + [BPF_MAP_TYPE_INODE_STORAGE] = "inode_storage", + [BPF_MAP_TYPE_TASK_STORAGE] = "task_storage", + [BPF_MAP_TYPE_BLOOM_FILTER] = "bloom_filter", +}; + +static const char * const prog_type_name[] = { + [BPF_PROG_TYPE_UNSPEC] = "unspec", + [BPF_PROG_TYPE_SOCKET_FILTER] = "socket_filter", + [BPF_PROG_TYPE_KPROBE] = "kprobe", + [BPF_PROG_TYPE_SCHED_CLS] = "sched_cls", + [BPF_PROG_TYPE_SCHED_ACT] = "sched_act", + [BPF_PROG_TYPE_TRACEPOINT] = "tracepoint", + [BPF_PROG_TYPE_XDP] = "xdp", + [BPF_PROG_TYPE_PERF_EVENT] = "perf_event", + [BPF_PROG_TYPE_CGROUP_SKB] = "cgroup_skb", + [BPF_PROG_TYPE_CGROUP_SOCK] = "cgroup_sock", + [BPF_PROG_TYPE_LWT_IN] = "lwt_in", + [BPF_PROG_TYPE_LWT_OUT] = "lwt_out", + [BPF_PROG_TYPE_LWT_XMIT] = "lwt_xmit", + [BPF_PROG_TYPE_SOCK_OPS] = "sock_ops", + [BPF_PROG_TYPE_SK_SKB] = "sk_skb", + [BPF_PROG_TYPE_CGROUP_DEVICE] = "cgroup_device", + [BPF_PROG_TYPE_SK_MSG] = "sk_msg", + [BPF_PROG_TYPE_RAW_TRACEPOINT] = "raw_tracepoint", + [BPF_PROG_TYPE_CGROUP_SOCK_ADDR] = "cgroup_sock_addr", + [BPF_PROG_TYPE_LWT_SEG6LOCAL] = "lwt_seg6local", + [BPF_PROG_TYPE_LIRC_MODE2] = "lirc_mode2", + [BPF_PROG_TYPE_SK_REUSEPORT] = "sk_reuseport", + [BPF_PROG_TYPE_FLOW_DISSECTOR] = "flow_dissector", + [BPF_PROG_TYPE_CGROUP_SYSCTL] = "cgroup_sysctl", + [BPF_PROG_TYPE_RAW_TRACEPOINT_WRITABLE] = "raw_tracepoint_writable", + [BPF_PROG_TYPE_CGROUP_SOCKOPT] = "cgroup_sockopt", + [BPF_PROG_TYPE_TRACING] = "tracing", + [BPF_PROG_TYPE_STRUCT_OPS] = "struct_ops", + [BPF_PROG_TYPE_EXT] = "ext", + [BPF_PROG_TYPE_LSM] = "lsm", + [BPF_PROG_TYPE_SK_LOOKUP] = "sk_lookup", + [BPF_PROG_TYPE_SYSCALL] = "syscall", +}; + static int __base_pr(enum libbpf_print_level level, const char *format, va_list args) { @@ -151,12 +279,9 @@ static inline __u64 ptr_to_u64(const void *ptr) return (__u64) (unsigned long) ptr; } -/* this goes away in libbpf 1.0 */ -enum libbpf_strict_mode libbpf_mode = LIBBPF_STRICT_NONE; - int libbpf_set_strict_mode(enum libbpf_strict_mode mode) { - libbpf_mode = mode; + /* as of v1.0 libbpf_set_strict_mode() is a no-op */ return 0; } @@ -219,12 +344,8 @@ enum sec_def_flags { SEC_ATTACH_BTF = 4, /* BPF program type allows sleeping/blocking in kernel */ SEC_SLEEPABLE = 8, - /* allow non-strict prefix matching */ - SEC_SLOPPY_PFX = 16, /* BPF program support non-linear XDP buffer */ - SEC_XDP_FRAGS = 32, - /* deprecated sec definitions not supposed to be used */ - SEC_DEPRECATED = 64, + SEC_XDP_FRAGS = 16, }; struct bpf_sec_def { @@ -244,9 +365,10 @@ struct bpf_sec_def { * linux/filter.h. */ struct bpf_program { - const struct bpf_sec_def *sec_def; + char *name; char *sec_name; size_t sec_idx; + const struct bpf_sec_def *sec_def; /* this program's instruction offset (in number of instructions) * within its containing ELF section */ @@ -266,12 +388,6 @@ struct bpf_program { */ size_t sub_insn_off; - char *name; - /* name with / replaced by _; makes recursive pinning - * in bpf_object__pin_programs easier - */ - char *pin_name; - /* instructions that belong to BPF program; insns[0] is located at * sec_insn_off instruction within its ELF section in ELF file, so * when mapping ELF file instruction index to the local instruction, @@ -292,24 +408,19 @@ struct bpf_program { size_t log_size; __u32 log_level; - struct { - int nr; - int *fds; - } instances; - bpf_program_prep_t preprocessor; - struct bpf_object *obj; - void *priv; - bpf_program_clear_priv_t clear_priv; - bool load; + int fd; + bool autoload; bool mark_btf_static; enum bpf_prog_type type; enum bpf_attach_type expected_attach_type; + int prog_ifindex; __u32 attach_btf_obj_fd; __u32 attach_btf_id; __u32 attach_prog_fd; + void *func_info; __u32 func_info_rec_size; __u32 func_info_cnt; @@ -356,7 +467,16 @@ enum libbpf_map_type { LIBBPF_MAP_KCONFIG, }; +struct bpf_map_def { + unsigned int type; + unsigned int key_size; + unsigned int value_size; + unsigned int max_entries; + unsigned int map_flags; +}; + struct bpf_map { + struct bpf_object *obj; char *name; /* real_name is defined for special internal maps (.rodata*, * .data*, .bss, .kconfig) and preserves their original ELF section @@ -375,8 +495,6 @@ struct bpf_map { __u32 btf_key_type_id; __u32 btf_value_type_id; __u32 btf_vmlinux_value_type_id; - void *priv; - bpf_map_clear_priv_t clear_priv; enum libbpf_map_type libbpf_type; void *mmaped; struct bpf_struct_ops *st_ops; @@ -386,7 +504,7 @@ struct bpf_map { char *pin_path; bool pinned; bool reused; - bool skipped; + bool autocreate; __u64 map_extra; }; @@ -439,8 +557,6 @@ struct extern_desc { }; }; -static LIST_HEAD(bpf_objects_list); - struct module_btf { struct btf *btf; char *name; @@ -483,6 +599,8 @@ struct elf_state { int st_ops_shndx; }; +struct usdt_manager; + struct bpf_object { char name[BPF_OBJ_NAME_LEN]; char license[64]; @@ -507,12 +625,6 @@ struct bpf_object { /* Information when doing ELF related work. Only valid if efile.elf is not NULL */ struct elf_state efile; - /* - * All loaded bpf_object are linked in a list, which is - * hidden to caller. bpf_objects__ handlers deal with - * all objects. - */ - struct list_head list; struct btf *btf; struct btf_ext *btf_ext; @@ -538,13 +650,12 @@ struct bpf_object { size_t log_size; __u32 log_level; - void *priv; - bpf_object_clear_priv_t clear_priv; - int *fd_array; size_t fd_array_cap; size_t fd_array_cnt; + struct usdt_manager *usdt_man; + char path[]; }; @@ -560,25 +671,10 @@ static Elf64_Rel *elf_rel_by_idx(Elf_Data *data, size_t idx); void bpf_program__unload(struct bpf_program *prog) { - int i; - if (!prog) return; - /* - * If the object is opened but the program was never loaded, - * it is possible that prog->instances.nr == -1. - */ - if (prog->instances.nr > 0) { - for (i = 0; i < prog->instances.nr; i++) - zclose(prog->instances.fds[i]); - } else if (prog->instances.nr != -1) { - pr_warn("Internal error: instances.nr is %d\n", - prog->instances.nr); - } - - prog->instances.nr = -1; - zfree(&prog->instances.fds); + zclose(prog->fd); zfree(&prog->func_info); zfree(&prog->line_info); @@ -589,16 +685,9 @@ static void bpf_program__exit(struct bpf_program *prog) if (!prog) return; - if (prog->clear_priv) - prog->clear_priv(prog, prog->priv); - - prog->priv = NULL; - prog->clear_priv = NULL; - bpf_program__unload(prog); zfree(&prog->name); zfree(&prog->sec_name); - zfree(&prog->pin_name); zfree(&prog->insns); zfree(&prog->reloc_desc); @@ -607,26 +696,6 @@ static void bpf_program__exit(struct bpf_program *prog) prog->sec_idx = -1; } -static char *__bpf_program__pin_name(struct bpf_program *prog) -{ - char *name, *p; - - if (libbpf_mode & LIBBPF_STRICT_SEC_NAME) - name = strdup(prog->name); - else - name = strdup(prog->sec_name); - - if (!name) - return NULL; - - p = name; - - while ((p = strchr(p, '/'))) - *p = '_'; - - return name; -} - static bool insn_is_subprog_call(const struct bpf_insn *insn) { return BPF_CLASS(insn->code) == BPF_JMP && @@ -668,10 +737,19 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, prog->insns_cnt = prog->sec_insn_cnt; prog->type = BPF_PROG_TYPE_UNSPEC; - prog->load = true; + prog->fd = -1; - prog->instances.fds = NULL; - prog->instances.nr = -1; + /* libbpf's convention for SEC("?abc...") is that it's just like + * SEC("abc...") but the corresponding bpf_program starts out with + * autoload set to false. + */ + if (sec_name[0] == '?') { + prog->autoload = false; + /* from now on forget there was ? in section name */ + sec_name++; + } else { + prog->autoload = true; + } /* inherit object's log_level */ prog->log_level = obj->log_level; @@ -684,10 +762,6 @@ bpf_object__init_prog(struct bpf_object *obj, struct bpf_program *prog, if (!prog->name) goto errout; - prog->pin_name = __bpf_program__pin_name(prog); - if (!prog->pin_name) - goto errout; - prog->insns = malloc(insn_data_sz); if (!prog->insns) goto errout; @@ -1169,7 +1243,6 @@ static struct bpf_object *bpf_object__new(const char *path, size_t obj_buf_sz, const char *obj_name) { - bool strict = (libbpf_mode & LIBBPF_STRICT_NO_OBJECT_LIST); struct bpf_object *obj; char *end; @@ -1207,9 +1280,6 @@ static struct bpf_object *bpf_object__new(const char *path, obj->kern_version = get_kernel_version(); obj->loaded = false; - INIT_LIST_HEAD(&obj->list); - if (!strict) - list_add(&obj->list, &bpf_objects_list); return obj; } @@ -1218,10 +1288,8 @@ static void bpf_object__elf_finish(struct bpf_object *obj) if (!obj->efile.elf) return; - if (obj->efile.elf) { - elf_end(obj->efile.elf); - obj->efile.elf = NULL; - } + elf_end(obj->efile.elf); + obj->efile.elf = NULL; obj->efile.symbols = NULL; obj->efile.st_ops_data = NULL; @@ -1244,10 +1312,7 @@ static int bpf_object__elf_init(struct bpf_object *obj) } if (obj->efile.obj_buf_sz > 0) { - /* - * obj_buf should have been validated by - * bpf_object__open_buffer(). - */ + /* obj_buf should have been validated by bpf_object__open_mem(). */ elf = elf_memory((char *)obj->efile.obj_buf, obj->efile.obj_buf_sz); } else { obj->efile.fd = open(obj->path, O_RDONLY | O_CLOEXEC); @@ -1397,8 +1462,11 @@ static int find_elf_var_offset(const struct bpf_object *obj, const char *name, _ for (si = 0; si < symbols->d_size / sizeof(Elf64_Sym); si++) { Elf64_Sym *sym = elf_sym_by_idx(obj, si); - if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL || - ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) + if (ELF64_ST_TYPE(sym->st_info) != STT_OBJECT) + continue; + + if (ELF64_ST_BIND(sym->st_info) != STB_GLOBAL && + ELF64_ST_BIND(sym->st_info) != STB_WEAK) continue; sname = elf_sym_str(obj, sym->st_name); @@ -1417,36 +1485,21 @@ static int find_elf_var_offset(const struct bpf_object *obj, const char *name, _ static struct bpf_map *bpf_object__add_map(struct bpf_object *obj) { - struct bpf_map *new_maps; - size_t new_cap; - int i; + struct bpf_map *map; + int err; - if (obj->nr_maps < obj->maps_cap) - return &obj->maps[obj->nr_maps++]; + err = libbpf_ensure_mem((void **)&obj->maps, &obj->maps_cap, + sizeof(*obj->maps), obj->nr_maps + 1); + if (err) + return ERR_PTR(err); - new_cap = max((size_t)4, obj->maps_cap * 3 / 2); - new_maps = libbpf_reallocarray(obj->maps, new_cap, sizeof(*obj->maps)); - if (!new_maps) { - pr_warn("alloc maps for object failed\n"); - return ERR_PTR(-ENOMEM); - } + map = &obj->maps[obj->nr_maps++]; + map->obj = obj; + map->fd = -1; + map->inner_map_fd = -1; + map->autocreate = true; - obj->maps_cap = new_cap; - obj->maps = new_maps; - - /* zero out new maps */ - memset(obj->maps + obj->nr_maps, 0, - (obj->maps_cap - obj->nr_maps) * sizeof(*obj->maps)); - /* - * fill all fd with -1 so won't close incorrect fd (fd=0 is stdin) - * when failure (zclose won't close negative fd)). - */ - for (i = obj->nr_maps; i < obj->maps_cap; i++) { - obj->maps[i].fd = -1; - obj->maps[i].inner_map_fd = -1; - } - - return &obj->maps[obj->nr_maps++]; + return map; } static size_t bpf_map_mmap_sz(const struct bpf_map *map) @@ -1641,7 +1694,7 @@ static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, switch (ext->kcfg.type) { case KCFG_BOOL: if (value == 'm') { - pr_warn("extern (kcfg) %s=%c should be tristate or char\n", + pr_warn("extern (kcfg) '%s': value '%c' implies tristate or char type\n", ext->name, value); return -EINVAL; } @@ -1662,7 +1715,7 @@ static int set_kcfg_value_tri(struct extern_desc *ext, void *ext_val, case KCFG_INT: case KCFG_CHAR_ARR: default: - pr_warn("extern (kcfg) %s=%c should be bool, tristate, or char\n", + pr_warn("extern (kcfg) '%s': value '%c' implies bool, tristate, or char type\n", ext->name, value); return -EINVAL; } @@ -1676,7 +1729,8 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, size_t len; if (ext->kcfg.type != KCFG_CHAR_ARR) { - pr_warn("extern (kcfg) %s=%s should be char array\n", ext->name, value); + pr_warn("extern (kcfg) '%s': value '%s' implies char array type\n", + ext->name, value); return -EINVAL; } @@ -1690,7 +1744,7 @@ static int set_kcfg_value_str(struct extern_desc *ext, char *ext_val, /* strip quotes */ len -= 2; if (len >= ext->kcfg.sz) { - pr_warn("extern (kcfg) '%s': long string config %s of (%zu bytes) truncated to %d bytes\n", + pr_warn("extern (kcfg) '%s': long string '%s' of (%zu bytes) truncated to %d bytes\n", ext->name, value, len, ext->kcfg.sz - 1); len = ext->kcfg.sz - 1; } @@ -1747,13 +1801,20 @@ static bool is_kcfg_value_in_range(const struct extern_desc *ext, __u64 v) static int set_kcfg_value_num(struct extern_desc *ext, void *ext_val, __u64 value) { - if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { - pr_warn("extern (kcfg) %s=%llu should be integer\n", + if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR && + ext->kcfg.type != KCFG_BOOL) { + pr_warn("extern (kcfg) '%s': value '%llu' implies integer, char, or boolean type\n", ext->name, (unsigned long long)value); return -EINVAL; } + if (ext->kcfg.type == KCFG_BOOL && value > 1) { + pr_warn("extern (kcfg) '%s': value '%llu' isn't boolean compatible\n", + ext->name, (unsigned long long)value); + return -EINVAL; + + } if (!is_kcfg_value_in_range(ext, value)) { - pr_warn("extern (kcfg) %s=%llu value doesn't fit in %d bytes\n", + pr_warn("extern (kcfg) '%s': value '%llu' doesn't fit in %d bytes\n", ext->name, (unsigned long long)value, ext->kcfg.sz); return -ERANGE; } @@ -1817,16 +1878,19 @@ static int bpf_object__process_kconfig_line(struct bpf_object *obj, /* assume integer */ err = parse_u64(value, &num); if (err) { - pr_warn("extern (kcfg) %s=%s should be integer\n", - ext->name, value); + pr_warn("extern (kcfg) '%s': value '%s' isn't a valid integer\n", ext->name, value); return err; } + if (ext->kcfg.type != KCFG_INT && ext->kcfg.type != KCFG_CHAR) { + pr_warn("extern (kcfg) '%s': value '%s' implies integer type\n", ext->name, value); + return -EINVAL; + } err = set_kcfg_value_num(ext, ext_val, num); break; } if (err) return err; - pr_debug("extern (kcfg) %s=%s\n", ext->name, value); + pr_debug("extern (kcfg) '%s': set to %s\n", ext->name, value); return 0; } @@ -1922,143 +1986,6 @@ static int bpf_object__init_kconfig_map(struct bpf_object *obj) return 0; } -static int bpf_object__init_user_maps(struct bpf_object *obj, bool strict) -{ - Elf_Data *symbols = obj->efile.symbols; - int i, map_def_sz = 0, nr_maps = 0, nr_syms; - Elf_Data *data = NULL; - Elf_Scn *scn; - - if (obj->efile.maps_shndx < 0) - return 0; - - if (libbpf_mode & LIBBPF_STRICT_MAP_DEFINITIONS) { - pr_warn("legacy map definitions in SEC(\"maps\") are not supported\n"); - return -EOPNOTSUPP; - } - - if (!symbols) - return -EINVAL; - - scn = elf_sec_by_idx(obj, obj->efile.maps_shndx); - data = elf_sec_data(obj, scn); - if (!scn || !data) { - pr_warn("elf: failed to get legacy map definitions for %s\n", - obj->path); - return -EINVAL; - } - - /* - * Count number of maps. Each map has a name. - * Array of maps is not supported: only the first element is - * considered. - * - * TODO: Detect array of map and report error. - */ - nr_syms = symbols->d_size / sizeof(Elf64_Sym); - for (i = 0; i < nr_syms; i++) { - Elf64_Sym *sym = elf_sym_by_idx(obj, i); - - if (sym->st_shndx != obj->efile.maps_shndx) - continue; - if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) - continue; - nr_maps++; - } - /* Assume equally sized map definitions */ - pr_debug("elf: found %d legacy map definitions (%zd bytes) in %s\n", - nr_maps, data->d_size, obj->path); - - if (!data->d_size || nr_maps == 0 || (data->d_size % nr_maps) != 0) { - pr_warn("elf: unable to determine legacy map definition size in %s\n", - obj->path); - return -EINVAL; - } - map_def_sz = data->d_size / nr_maps; - - /* Fill obj->maps using data in "maps" section. */ - for (i = 0; i < nr_syms; i++) { - Elf64_Sym *sym = elf_sym_by_idx(obj, i); - const char *map_name; - struct bpf_map_def *def; - struct bpf_map *map; - - if (sym->st_shndx != obj->efile.maps_shndx) - continue; - if (ELF64_ST_TYPE(sym->st_info) == STT_SECTION) - continue; - - map = bpf_object__add_map(obj); - if (IS_ERR(map)) - return PTR_ERR(map); - - map_name = elf_sym_str(obj, sym->st_name); - if (!map_name) { - pr_warn("failed to get map #%d name sym string for obj %s\n", - i, obj->path); - return -LIBBPF_ERRNO__FORMAT; - } - - pr_warn("map '%s' (legacy): legacy map definitions are deprecated, use BTF-defined maps instead\n", map_name); - - if (ELF64_ST_BIND(sym->st_info) == STB_LOCAL) { - pr_warn("map '%s' (legacy): static maps are not supported\n", map_name); - return -ENOTSUP; - } - - map->libbpf_type = LIBBPF_MAP_UNSPEC; - map->sec_idx = sym->st_shndx; - map->sec_offset = sym->st_value; - pr_debug("map '%s' (legacy): at sec_idx %d, offset %zu.\n", - map_name, map->sec_idx, map->sec_offset); - if (sym->st_value + map_def_sz > data->d_size) { - pr_warn("corrupted maps section in %s: last map \"%s\" too small\n", - obj->path, map_name); - return -EINVAL; - } - - map->name = strdup(map_name); - if (!map->name) { - pr_warn("map '%s': failed to alloc map name\n", map_name); - return -ENOMEM; - } - pr_debug("map %d is \"%s\"\n", i, map->name); - def = (struct bpf_map_def *)(data->d_buf + sym->st_value); - /* - * If the definition of the map in the object file fits in - * bpf_map_def, copy it. Any extra fields in our version - * of bpf_map_def will default to zero as a result of the - * calloc above. - */ - if (map_def_sz <= sizeof(struct bpf_map_def)) { - memcpy(&map->def, def, map_def_sz); - } else { - /* - * Here the map structure being read is bigger than what - * we expect, truncate if the excess bits are all zero. - * If they are not zero, reject this map as - * incompatible. - */ - char *b; - - for (b = ((char *)def) + sizeof(struct bpf_map_def); - b < ((char *)def) + map_def_sz; b++) { - if (*b != 0) { - pr_warn("maps section in %s: \"%s\" has unrecognized, non-zero options\n", - obj->path, map_name); - if (strict) - return -EINVAL; - } - } - memcpy(&map->def, def, sizeof(struct bpf_map_def)); - } - - /* btf info may not exist but fill it in if it does exist */ - (void) bpf_map_find_btf_info(obj, map); - } - return 0; -} - const struct btf_type * skip_mods_and_typedefs(const struct btf *btf, __u32 id, __u32 *res_id) { @@ -2112,6 +2039,7 @@ static const char *__btf_kind_str(__u16 kind) case BTF_KIND_FLOAT: return "float"; case BTF_KIND_DECL_TAG: return "decl_tag"; case BTF_KIND_TYPE_TAG: return "type_tag"; + case BTF_KIND_ENUM64: return "enum64"; default: return "unknown"; } } @@ -2175,6 +2103,13 @@ static int build_map_pin_path(struct bpf_map *map, const char *path) return bpf_map__set_pin_path(map, buf); } +/* should match definition in bpf_helpers.h */ +enum libbpf_pin_type { + LIBBPF_PIN_NONE, + /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ + LIBBPF_PIN_BY_NAME, +}; + int parse_btf_map_def(const char *map_name, struct btf *btf, const struct btf_type *def_t, bool strict, struct btf_map_def *map_def, struct btf_map_def *inner_def) @@ -2396,6 +2331,37 @@ int parse_btf_map_def(const char *map_name, struct btf *btf, return 0; } +static size_t adjust_ringbuf_sz(size_t sz) +{ + __u32 page_sz = sysconf(_SC_PAGE_SIZE); + __u32 mul; + + /* if user forgot to set any size, make sure they see error */ + if (sz == 0) + return 0; + /* Kernel expects BPF_MAP_TYPE_RINGBUF's max_entries to be + * a power-of-2 multiple of kernel's page size. If user diligently + * satisified these conditions, pass the size through. + */ + if ((sz % page_sz) == 0 && is_pow_of_2(sz / page_sz)) + return sz; + + /* Otherwise find closest (page_sz * power_of_2) product bigger than + * user-set size to satisfy both user size request and kernel + * requirements and substitute correct max_entries for map creation. + */ + for (mul = 1; mul <= UINT_MAX / page_sz; mul <<= 1) { + if (mul * page_sz > sz) + return mul * page_sz; + } + + /* if it's impossible to satisfy the conditions (i.e., user size is + * very close to UINT_MAX but is not a power-of-2 multiple of + * page_size) then just return original size and let kernel reject it + */ + return sz; +} + static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def) { map->def.type = def->map_type; @@ -2409,6 +2375,10 @@ static void fill_map_from_def(struct bpf_map *map, const struct btf_map_def *def map->btf_key_type_id = def->key_type_id; map->btf_value_type_id = def->value_type_id; + /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ + if (map->def.type == BPF_MAP_TYPE_RINGBUF) + map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); + if (def->parts & MAP_DEF_MAP_TYPE) pr_debug("map '%s': found type = %u.\n", map->name, def->map_type); @@ -2607,12 +2577,11 @@ static int bpf_object__init_maps(struct bpf_object *obj, { const char *pin_root_path; bool strict; - int err; + int err = 0; strict = !OPTS_GET(opts, relaxed_maps, false); pin_root_path = OPTS_GET(opts, pin_root_path, NULL); - err = bpf_object__init_user_maps(obj, strict); err = err ?: bpf_object__init_user_btf_maps(obj, strict, pin_root_path); err = err ?: bpf_object__init_global_data_maps(obj); err = err ?: bpf_object__init_kconfig_map(obj); @@ -2640,12 +2609,13 @@ static bool btf_needs_sanitization(struct bpf_object *obj) bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); + bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); return !has_func || !has_datasec || !has_func_global || !has_float || - !has_decl_tag || !has_type_tag; + !has_decl_tag || !has_type_tag || !has_enum64; } -static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) +static int bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) { bool has_func_global = kernel_supports(obj, FEAT_BTF_GLOBAL_FUNC); bool has_datasec = kernel_supports(obj, FEAT_BTF_DATASEC); @@ -2653,6 +2623,8 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) bool has_func = kernel_supports(obj, FEAT_BTF_FUNC); bool has_decl_tag = kernel_supports(obj, FEAT_BTF_DECL_TAG); bool has_type_tag = kernel_supports(obj, FEAT_BTF_TYPE_TAG); + bool has_enum64 = kernel_supports(obj, FEAT_BTF_ENUM64); + int enum64_placeholder_id = 0; struct btf_type *t; int i, j, vlen; @@ -2715,8 +2687,32 @@ static void bpf_object__sanitize_btf(struct bpf_object *obj, struct btf *btf) /* replace TYPE_TAG with a CONST */ t->name_off = 0; t->info = BTF_INFO_ENC(BTF_KIND_CONST, 0, 0); - } + } else if (!has_enum64 && btf_is_enum(t)) { + /* clear the kflag */ + t->info = btf_type_info(btf_kind(t), btf_vlen(t), false); + } else if (!has_enum64 && btf_is_enum64(t)) { + /* replace ENUM64 with a union */ + struct btf_member *m; + + if (enum64_placeholder_id == 0) { + enum64_placeholder_id = btf__add_int(btf, "enum64_placeholder", 1, 0); + if (enum64_placeholder_id < 0) + return enum64_placeholder_id; + + t = (struct btf_type *)btf__type_by_id(btf, i); + } + + m = btf_members(t); + vlen = btf_vlen(t); + t->info = BTF_INFO_ENC(BTF_KIND_UNION, 0, vlen); + for (j = 0; j < vlen; j++, m++) { + m->type = enum64_placeholder_id; + m->offset = 0; + } + } } + + return 0; } static bool libbpf_needs_btf(const struct bpf_object *obj) @@ -2749,6 +2745,9 @@ static int bpf_object__init_btf(struct bpf_object *obj, btf__set_pointer_size(obj->btf, 8); } if (btf_ext_data) { + struct btf_ext_info *ext_segs[3]; + int seg_num, sec_num; + if (!obj->btf) { pr_debug("Ignore ELF section %s because its depending ELF section %s is not found.\n", BTF_EXT_ELF_SEC, BTF_ELF_SEC); @@ -2762,6 +2761,43 @@ static int bpf_object__init_btf(struct bpf_object *obj, obj->btf_ext = NULL; goto out; } + + /* setup .BTF.ext to ELF section mapping */ + ext_segs[0] = &obj->btf_ext->func_info; + ext_segs[1] = &obj->btf_ext->line_info; + ext_segs[2] = &obj->btf_ext->core_relo_info; + for (seg_num = 0; seg_num < ARRAY_SIZE(ext_segs); seg_num++) { + struct btf_ext_info *seg = ext_segs[seg_num]; + const struct btf_ext_info_sec *sec; + const char *sec_name; + Elf_Scn *scn; + + if (seg->sec_cnt == 0) + continue; + + seg->sec_idxs = calloc(seg->sec_cnt, sizeof(*seg->sec_idxs)); + if (!seg->sec_idxs) { + err = -ENOMEM; + goto out; + } + + sec_num = 0; + for_each_btf_ext_sec(seg, sec) { + /* preventively increment index to avoid doing + * this before every continue below + */ + sec_num++; + + sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); + if (str_is_empty(sec_name)) + continue; + scn = elf_sec_by_name(obj, sec_name); + if (!scn) + continue; + + seg->sec_idxs[sec_num - 1] = elf_ndxscn(scn); + } + } } out: if (err && libbpf_needs_btf(obj)) { @@ -2863,11 +2899,6 @@ static int btf_finalize_data(struct bpf_object *obj, struct btf *btf) return libbpf_err(err); } -int btf__finalize_data(struct bpf_object *obj, struct btf *btf) -{ - return btf_finalize_data(obj, btf); -} - static int bpf_object__finalize_btf(struct bpf_object *obj) { int err; @@ -2920,7 +2951,7 @@ static bool obj_needs_vmlinux_btf(const struct bpf_object *obj) } bpf_object__for_each_program(prog, obj) { - if (!prog->load) + if (!prog->autoload) continue; if (prog_needs_vmlinux_btf(prog)) return true; @@ -3014,7 +3045,9 @@ static int bpf_object__sanitize_and_load_btf(struct bpf_object *obj) /* enforce 8-byte pointers for BPF-targeted BTFs */ btf__set_pointer_size(obj->btf, 8); - bpf_object__sanitize_btf(obj, kern_btf); + err = bpf_object__sanitize_btf(obj, kern_btf); + if (err) + return err; } if (obj->gen_loader) { @@ -3521,6 +3554,10 @@ static enum kcfg_type find_kcfg_type(const struct btf *btf, int id, if (strcmp(name, "libbpf_tristate")) return KCFG_UNKNOWN; return KCFG_TRISTATE; + case BTF_KIND_ENUM64: + if (strcmp(name, "libbpf_tristate")) + return KCFG_UNKNOWN; + return KCFG_TRISTATE; case BTF_KIND_ARRAY: if (btf_array(t)->nelems == 0) return KCFG_UNKNOWN; @@ -3696,7 +3733,7 @@ static int bpf_object__collect_externs(struct bpf_object *obj) ext->kcfg.type = find_kcfg_type(obj->btf, t->type, &ext->kcfg.is_signed); if (ext->kcfg.type == KCFG_UNKNOWN) { - pr_warn("extern (kcfg) '%s' type is unsupported\n", ext_name); + pr_warn("extern (kcfg) '%s': type is unsupported\n", ext_name); return -ENOTSUP; } } else if (strcmp(sec_name, KSYMS_SEC) == 0) { @@ -3818,41 +3855,8 @@ static int bpf_object__collect_externs(struct bpf_object *obj) return 0; } -struct bpf_program * -bpf_object__find_program_by_title(const struct bpf_object *obj, - const char *title) +static bool prog_is_subprog(const struct bpf_object *obj, const struct bpf_program *prog) { - struct bpf_program *pos; - - bpf_object__for_each_program(pos, obj) { - if (pos->sec_name && !strcmp(pos->sec_name, title)) - return pos; - } - return errno = ENOENT, NULL; -} - -static bool prog_is_subprog(const struct bpf_object *obj, - const struct bpf_program *prog) -{ - /* For legacy reasons, libbpf supports an entry-point BPF programs - * without SEC() attribute, i.e., those in the .text section. But if - * there are 2 or more such programs in the .text section, they all - * must be subprograms called from entry-point BPF programs in - * designated SEC()'tions, otherwise there is no way to distinguish - * which of those programs should be loaded vs which are a subprogram. - * Similarly, if there is a function/program in .text and at least one - * other BPF program with custom SEC() attribute, then we just assume - * .text programs are subprograms (even if they are not called from - * other programs), because libbpf never explicitly supported mixing - * SEC()-designated BPF programs and .text entry-point BPF programs. - * - * In libbpf 1.0 strict mode, we always consider .text - * programs to be subprograms. - */ - - if (libbpf_mode & LIBBPF_STRICT_SEC_NAME) - return prog->sec_idx == obj->efile.text_shndx; - return prog->sec_idx == obj->efile.text_shndx && obj->nr_programs > 1; } @@ -4193,9 +4197,7 @@ bpf_object__collect_prog_relos(struct bpf_object *obj, Elf64_Shdr *shdr, Elf_Dat static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) { - struct bpf_map_def *def = &map->def; - __u32 key_type_id = 0, value_type_id = 0; - int ret; + int id; if (!obj->btf) return -ENOENT; @@ -4204,31 +4206,22 @@ static int bpf_map_find_btf_info(struct bpf_object *obj, struct bpf_map *map) * For struct_ops map, it does not need btf_key_type_id and * btf_value_type_id. */ - if (map->sec_idx == obj->efile.btf_maps_shndx || - bpf_map__is_struct_ops(map)) + if (map->sec_idx == obj->efile.btf_maps_shndx || bpf_map__is_struct_ops(map)) return 0; - if (!bpf_map__is_internal(map)) { - pr_warn("Use of BPF_ANNOTATE_KV_PAIR is deprecated, use BTF-defined maps in .maps section instead\n"); -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - ret = btf__get_map_kv_tids(obj->btf, map->name, def->key_size, - def->value_size, &key_type_id, - &value_type_id); -#pragma GCC diagnostic pop - } else { - /* - * LLVM annotates global data differently in BTF, that is, - * only as '.data', '.bss' or '.rodata'. - */ - ret = btf__find_by_name(obj->btf, map->real_name); - } - if (ret < 0) - return ret; + /* + * LLVM annotates global data differently in BTF, that is, + * only as '.data', '.bss' or '.rodata'. + */ + if (!bpf_map__is_internal(map)) + return -ENOENT; - map->btf_key_type_id = key_type_id; - map->btf_value_type_id = bpf_map__is_internal(map) ? - ret : value_type_id; + id = btf__find_by_name(obj->btf, map->real_name); + if (id < 0) + return id; + + map->btf_key_type_id = 0; + map->btf_value_type_id = id; return 0; } @@ -4268,10 +4261,24 @@ static int bpf_get_map_info_from_fdinfo(int fd, struct bpf_map_info *info) return 0; } +bool bpf_map__autocreate(const struct bpf_map *map) +{ + return map->autocreate; +} + +int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate) +{ + if (map->obj->loaded) + return libbpf_err(-EBUSY); + + map->autocreate = autocreate; + return 0; +} + int bpf_map__reuse_fd(struct bpf_map *map, int fd) { struct bpf_map_info info = {}; - __u32 len = sizeof(info); + __u32 len = sizeof(info), name_len; int new_fd, err; char *new_name; @@ -4281,7 +4288,12 @@ int bpf_map__reuse_fd(struct bpf_map *map, int fd) if (err) return libbpf_err(err); - new_name = strdup(info.name); + name_len = strlen(info.name); + if (name_len == BPF_OBJ_NAME_LEN - 1 && strncmp(map->name, info.name, name_len) == 0) + new_name = strdup(map->name); + else + new_name = strdup(info.name); + if (!new_name) return libbpf_err(-errno); @@ -4340,20 +4352,18 @@ struct bpf_map *bpf_map__inner_map(struct bpf_map *map) int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries) { - if (map->fd >= 0) + if (map->obj->loaded) return libbpf_err(-EBUSY); + map->def.max_entries = max_entries; + + /* auto-adjust BPF ringbuf map max_entries to be a multiple of page size */ + if (map->def.type == BPF_MAP_TYPE_RINGBUF) + map->def.max_entries = adjust_ringbuf_sz(map->def.max_entries); + return 0; } -int bpf_map__resize(struct bpf_map *map, __u32 max_entries) -{ - if (!map || !max_entries) - return libbpf_err(-EINVAL); - - return bpf_map__set_max_entries(map, max_entries); -} - static int bpf_object__probe_loading(struct bpf_object *obj) { @@ -4587,7 +4597,7 @@ static int probe_kern_probe_read_kernel(void) }; int fd, insn_cnt = ARRAY_SIZE(insns); - fd = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); + fd = bpf_prog_load(BPF_PROG_TYPE_TRACEPOINT, NULL, "GPL", insns, insn_cnt, NULL); return probe_fd(fd); } @@ -4678,6 +4688,31 @@ static int probe_perf_link(void) return link_fd < 0 && err == -EBADF; } +static int probe_kern_bpf_cookie(void) +{ + struct bpf_insn insns[] = { + BPF_RAW_INSN(BPF_JMP | BPF_CALL, 0, 0, 0, BPF_FUNC_get_attach_cookie), + BPF_EXIT_INSN(), + }; + int ret, insn_cnt = ARRAY_SIZE(insns); + + ret = bpf_prog_load(BPF_PROG_TYPE_KPROBE, NULL, "GPL", insns, insn_cnt, NULL); + return probe_fd(ret); +} + +static int probe_kern_btf_enum64(void) +{ + static const char strs[] = "\0enum64"; + __u32 types[] = { + BTF_TYPE_ENC(1, BTF_INFO_ENC(BTF_KIND_ENUM64, 0, 0), 8), + }; + + return probe_fd(libbpf__load_raw_btf((char *)types, sizeof(types), + strs, sizeof(strs))); +} + +static int probe_kern_syscall_wrapper(void); + enum kern_feature_result { FEAT_UNKNOWN = 0, FEAT_SUPPORTED = 1, @@ -4740,6 +4775,15 @@ static struct kern_feature_desc { [FEAT_MEMCG_ACCOUNT] = { "memcg-based memory accounting", probe_memcg_account, }, + [FEAT_BPF_COOKIE] = { + "BPF cookie support", probe_kern_bpf_cookie, + }, + [FEAT_BTF_ENUM64] = { + "BTF_KIND_ENUM64 support", probe_kern_btf_enum64, + }, + [FEAT_SYSCALL_WRAPPER] = { + "Kernel using syscall wrapper", probe_kern_syscall_wrapper, + }, }; bool kernel_supports(const struct bpf_object *obj, enum kern_feature_id feat_id) @@ -4923,7 +4967,6 @@ static int bpf_object__create_map(struct bpf_object *obj, struct bpf_map *map, b case BPF_MAP_TYPE_SOCKHASH: case BPF_MAP_TYPE_QUEUE: case BPF_MAP_TYPE_STACK: - case BPF_MAP_TYPE_RINGBUF: create_attr.btf_fd = 0; create_attr.btf_key_type_id = 0; create_attr.btf_value_type_id = 0; @@ -5109,9 +5152,11 @@ bpf_object__create_maps(struct bpf_object *obj) * bpf_object loading will succeed just fine even on old * kernels. */ - if (bpf_map__is_internal(map) && - !kernel_supports(obj, FEAT_GLOBAL_DATA)) { - map->skipped = true; + if (bpf_map__is_internal(map) && !kernel_supports(obj, FEAT_GLOBAL_DATA)) + map->autocreate = false; + + if (!map->autocreate) { + pr_debug("map '%s': skipped auto-creating...\n", map->name); continue; } @@ -5242,7 +5287,7 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, n = btf__type_cnt(targ_btf); for (i = targ_start_id; i < n; i++) { t = btf__type_by_id(targ_btf, i); - if (btf_kind(t) != btf_kind(local_t)) + if (!btf_kind_core_compat(t, local_t)) continue; targ_name = btf__name_by_offset(targ_btf, t->name_off); @@ -5450,76 +5495,13 @@ bpf_core_find_cands(struct bpf_object *obj, const struct btf *local_btf, __u32 l int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, __u32 targ_id) { - const struct btf_type *local_type, *targ_type; - int depth = 32; /* max recursion depth */ + return __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, 32); +} - /* caller made sure that names match (ignoring flavor suffix) */ - local_type = btf__type_by_id(local_btf, local_id); - targ_type = btf__type_by_id(targ_btf, targ_id); - if (btf_kind(local_type) != btf_kind(targ_type)) - return 0; - -recur: - depth--; - if (depth < 0) - return -EINVAL; - - local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); - targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); - if (!local_type || !targ_type) - return -EINVAL; - - if (btf_kind(local_type) != btf_kind(targ_type)) - return 0; - - switch (btf_kind(local_type)) { - case BTF_KIND_UNKN: - case BTF_KIND_STRUCT: - case BTF_KIND_UNION: - case BTF_KIND_ENUM: - case BTF_KIND_FWD: - return 1; - case BTF_KIND_INT: - /* just reject deprecated bitfield-like integers; all other - * integers are by default compatible between each other - */ - return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; - case BTF_KIND_PTR: - local_id = local_type->type; - targ_id = targ_type->type; - goto recur; - case BTF_KIND_ARRAY: - local_id = btf_array(local_type)->type; - targ_id = btf_array(targ_type)->type; - goto recur; - case BTF_KIND_FUNC_PROTO: { - struct btf_param *local_p = btf_params(local_type); - struct btf_param *targ_p = btf_params(targ_type); - __u16 local_vlen = btf_vlen(local_type); - __u16 targ_vlen = btf_vlen(targ_type); - int i, err; - - if (local_vlen != targ_vlen) - return 0; - - for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { - skip_mods_and_typedefs(local_btf, local_p->type, &local_id); - skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); - err = bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id); - if (err <= 0) - return err; - } - - /* tail recurse for return type check */ - skip_mods_and_typedefs(local_btf, local_type->type, &local_id); - skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); - goto recur; - } - default: - pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", - btf_kind_str(local_type), local_id, targ_id); - return 0; - } +int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, + const struct btf *targ_btf, __u32 targ_id) +{ + return __bpf_core_types_match(local_btf, local_id, targ_btf, targ_id, false, 32); } static size_t bpf_core_hash_fn(const void *key, void *ctx) @@ -5555,6 +5537,22 @@ static int record_relo_core(struct bpf_program *prog, return 0; } +static const struct bpf_core_relo *find_relo_core(struct bpf_program *prog, int insn_idx) +{ + struct reloc_desc *relo; + int i; + + for (i = 0; i < prog->nr_reloc; i++) { + relo = &prog->reloc_desc[i]; + if (relo->type != RELO_CORE || relo->insn_idx != insn_idx) + continue; + + return relo->core_relo; + } + + return NULL; +} + static int bpf_core_resolve_relo(struct bpf_program *prog, const struct bpf_core_relo *relo, int relo_idx, @@ -5611,7 +5609,7 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) struct bpf_program *prog; struct bpf_insn *insn; const char *sec_name; - int i, err = 0, insn_idx, sec_idx; + int i, err = 0, insn_idx, sec_idx, sec_num; if (obj->btf_ext->core_relo_info.len == 0) return 0; @@ -5632,32 +5630,18 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) } seg = &obj->btf_ext->core_relo_info; + sec_num = 0; for_each_btf_ext_sec(seg, sec) { + sec_idx = seg->sec_idxs[sec_num]; + sec_num++; + sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); if (str_is_empty(sec_name)) { err = -EINVAL; goto out; } - /* bpf_object's ELF is gone by now so it's not easy to find - * section index by section name, but we can find *any* - * bpf_program within desired section name and use it's - * prog->sec_idx to do a proper search by section index and - * instruction offset - */ - prog = NULL; - for (i = 0; i < obj->nr_programs; i++) { - prog = &obj->programs[i]; - if (strcmp(prog->sec_name, sec_name) == 0) - break; - } - if (!prog) { - pr_warn("sec '%s': failed to find a BPF program\n", sec_name); - return -ENOENT; - } - sec_idx = prog->sec_idx; - pr_debug("sec '%s': found %d CO-RE relocations\n", - sec_name, sec->num_info); + pr_debug("sec '%s': found %d CO-RE relocations\n", sec_name, sec->num_info); for_each_btf_ext_rec(seg, sec, i, rec) { if (rec->insn_off % BPF_INSN_SZ) @@ -5665,15 +5649,22 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) insn_idx = rec->insn_off / BPF_INSN_SZ; prog = find_prog_by_sec_insn(obj, sec_idx, insn_idx); if (!prog) { - pr_warn("sec '%s': failed to find program at insn #%d for CO-RE offset relocation #%d\n", - sec_name, insn_idx, i); - err = -EINVAL; - goto out; + /* When __weak subprog is "overridden" by another instance + * of the subprog from a different object file, linker still + * appends all the .BTF.ext info that used to belong to that + * eliminated subprogram. + * This is similar to what x86-64 linker does for relocations. + * So just ignore such relocations just like we ignore + * subprog instructions when discovering subprograms. + */ + pr_debug("sec '%s': skipping CO-RE relocation #%d for insn #%d belonging to eliminated weak subprogram\n", + sec_name, i, insn_idx); + continue; } /* no need to apply CO-RE relocation if the program is * not going to be loaded */ - if (!prog->load) + if (!prog->autoload) continue; /* adjust insn_idx from section frame of reference to the local @@ -5685,16 +5676,16 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) return -EINVAL; insn = &prog->insns[insn_idx]; - if (prog->obj->gen_loader) { - err = record_relo_core(prog, rec, insn_idx); - if (err) { - pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n", - prog->name, i, err); - goto out; - } - continue; + err = record_relo_core(prog, rec, insn_idx); + if (err) { + pr_warn("prog '%s': relo #%d: failed to record relocation: %d\n", + prog->name, i, err); + goto out; } + if (prog->obj->gen_loader) + continue; + err = bpf_core_resolve_relo(prog, rec, i, obj->btf, cand_cache, &targ_res); if (err) { pr_warn("prog '%s': relo #%d: failed to relocate: %d\n", @@ -5725,6 +5716,36 @@ bpf_object__relocate_core(struct bpf_object *obj, const char *targ_btf_path) return err; } +/* base map load ldimm64 special constant, used also for log fixup logic */ +#define MAP_LDIMM64_POISON_BASE 2001000000 +#define MAP_LDIMM64_POISON_PFX "200100" + +static void poison_map_ldimm64(struct bpf_program *prog, int relo_idx, + int insn_idx, struct bpf_insn *insn, + int map_idx, const struct bpf_map *map) +{ + int i; + + pr_debug("prog '%s': relo #%d: poisoning insn #%d that loads map #%d '%s'\n", + prog->name, relo_idx, insn_idx, map_idx, map->name); + + /* we turn single ldimm64 into two identical invalid calls */ + for (i = 0; i < 2; i++) { + insn->code = BPF_JMP | BPF_CALL; + insn->dst_reg = 0; + insn->src_reg = 0; + insn->off = 0; + /* if this instruction is reachable (not a dead code), + * verifier will complain with something like: + * invalid func unknown#2001000123 + * where lower 123 is map index into obj->maps[] array + */ + insn->imm = MAP_LDIMM64_POISON_BASE + map_idx; + + insn++; + } +} + /* Relocate data references within program code: * - map references; * - global variable references; @@ -5738,33 +5759,35 @@ bpf_object__relocate_data(struct bpf_object *obj, struct bpf_program *prog) for (i = 0; i < prog->nr_reloc; i++) { struct reloc_desc *relo = &prog->reloc_desc[i]; struct bpf_insn *insn = &prog->insns[relo->insn_idx]; + const struct bpf_map *map; struct extern_desc *ext; switch (relo->type) { case RELO_LD64: + map = &obj->maps[relo->map_idx]; if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX; insn[0].imm = relo->map_idx; - } else { + } else if (map->autocreate) { insn[0].src_reg = BPF_PSEUDO_MAP_FD; - insn[0].imm = obj->maps[relo->map_idx].fd; + insn[0].imm = map->fd; + } else { + poison_map_ldimm64(prog, i, relo->insn_idx, insn, + relo->map_idx, map); } break; case RELO_DATA: + map = &obj->maps[relo->map_idx]; insn[1].imm = insn[0].imm + relo->sym_off; if (obj->gen_loader) { insn[0].src_reg = BPF_PSEUDO_MAP_IDX_VALUE; insn[0].imm = relo->map_idx; - } else { - const struct bpf_map *map = &obj->maps[relo->map_idx]; - - if (map->skipped) { - pr_warn("prog '%s': relo #%d: kernel doesn't support global data\n", - prog->name, i); - return -ENOTSUP; - } + } else if (map->autocreate) { insn[0].src_reg = BPF_PSEUDO_MAP_VALUE; - insn[0].imm = obj->maps[relo->map_idx].fd; + insn[0].imm = map->fd; + } else { + poison_map_ldimm64(prog, i, relo->insn_idx, insn, + relo->map_idx, map); } break; case RELO_EXTERN_VAR: @@ -5834,14 +5857,13 @@ static int adjust_prog_btf_ext_info(const struct bpf_object *obj, void *rec, *rec_end, *new_prog_info; const struct btf_ext_info_sec *sec; size_t old_sz, new_sz; - const char *sec_name; - int i, off_adj; + int i, sec_num, sec_idx, off_adj; + sec_num = 0; for_each_btf_ext_sec(ext_info, sec) { - sec_name = btf__name_by_offset(obj->btf, sec->sec_name_off); - if (!sec_name) - return -EINVAL; - if (strcmp(sec_name, prog->sec_name) != 0) + sec_idx = ext_info->sec_idxs[sec_num]; + sec_num++; + if (prog->sec_idx != sec_idx) continue; for_each_btf_ext_rec(ext_info, sec, i, rec) { @@ -6236,7 +6258,6 @@ bpf_object__relocate_calls(struct bpf_object *obj, struct bpf_program *prog) if (err) return err; - return 0; } @@ -6297,8 +6318,7 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) err); return err; } - if (obj->gen_loader) - bpf_object__sort_relos(obj); + bpf_object__sort_relos(obj); } /* Before relocating calls pre-process relocations and mark @@ -6334,7 +6354,7 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) */ if (prog_is_subprog(obj, prog)) continue; - if (!prog->load) + if (!prog->autoload) continue; err = bpf_object__relocate_calls(obj, prog); @@ -6349,7 +6369,7 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) prog = &obj->programs[i]; if (prog_is_subprog(obj, prog)) continue; - if (!prog->load) + if (!prog->autoload) continue; err = bpf_object__relocate_data(obj, prog); if (err) { @@ -6358,8 +6378,7 @@ bpf_object__relocate(struct bpf_object *obj, const char *targ_btf_path) return err; } } - if (!obj->gen_loader) - bpf_object__free_relocs(obj); + return 0; } @@ -6606,17 +6625,27 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, if (prog->type == BPF_PROG_TYPE_XDP && (def & SEC_XDP_FRAGS)) opts->prog_flags |= BPF_F_XDP_HAS_FRAGS; - if (def & SEC_DEPRECATED) - pr_warn("SEC(\"%s\") is deprecated, please see https://github.com/libbpf/libbpf/wiki/Libbpf-1.0-migration-guide#bpf-program-sec-annotation-deprecations for details\n", - prog->sec_name); - - if ((prog->type == BPF_PROG_TYPE_TRACING || - prog->type == BPF_PROG_TYPE_LSM || - prog->type == BPF_PROG_TYPE_EXT) && !prog->attach_btf_id) { + if ((def & SEC_ATTACH_BTF) && !prog->attach_btf_id) { int btf_obj_fd = 0, btf_type_id = 0, err; const char *attach_name; - attach_name = strchr(prog->sec_name, '/') + 1; + attach_name = strchr(prog->sec_name, '/'); + if (!attach_name) { + /* if BPF program is annotated with just SEC("fentry") + * (or similar) without declaratively specifying + * target, then it is expected that target will be + * specified with bpf_program__set_attach_target() at + * runtime before BPF object load step. If not, then + * there is nothing to load into the kernel as BPF + * verifier won't be able to validate BPF program + * correctness anyways. + */ + pr_warn("prog '%s': no BTF-based attach target is specified, use bpf_program__set_attach_target()\n", + prog->name); + return -EINVAL; + } + attach_name++; /* skip over / */ + err = libbpf_find_attach_btf_id(prog, attach_name, &btf_obj_fd, &btf_type_id); if (err) return err; @@ -6636,10 +6665,11 @@ static int libbpf_prepare_prog_load(struct bpf_program *prog, return 0; } -static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_program *prog, - struct bpf_insn *insns, int insns_cnt, - const char *license, __u32 kern_version, - int *prog_fd) +static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz); + +static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, + struct bpf_insn *insns, int insns_cnt, + const char *license, __u32 kern_version, int *prog_fd) { LIBBPF_OPTS(bpf_prog_load_opts, load_attr); const char *prog_name = NULL; @@ -6695,6 +6725,8 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog prog->name, err); return err; } + insns = prog->insns; + insns_cnt = prog->insns_cnt; } if (obj->gen_loader) { @@ -6706,7 +6738,7 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog } retry_load: - /* if log_level is zero, we don't request logs initiallly even if + /* if log_level is zero, we don't request logs initially even if * custom log_buf is specified; if the program load fails, then we'll * bump log_level to 1 and use either custom log_buf or we'll allocate * our own and retry the load to get details on what failed @@ -6782,6 +6814,10 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog goto retry_load; ret = -errno; + + /* post-process verifier log to improve error descriptions */ + fixup_verifier_log(prog, log_buf, log_buf_size); + cp = libbpf_strerror_r(errno, errmsg, sizeof(errmsg)); pr_warn("prog '%s': BPF program load failed: %s\n", prog->name, cp); pr_perm_msg(ret); @@ -6790,10 +6826,6 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog pr_warn("prog '%s': -- BEGIN PROG LOAD LOG --\n%s-- END PROG LOAD LOG --\n", prog->name, log_buf); } - if (insns_cnt >= BPF_MAXINSNS) { - pr_warn("prog '%s': program too large (%d insns), at most %d insns\n", - prog->name, insns_cnt, BPF_MAXINSNS); - } out: if (own_log_buf) @@ -6801,6 +6833,169 @@ static int bpf_object_load_prog_instance(struct bpf_object *obj, struct bpf_prog return ret; } +static char *find_prev_line(char *buf, char *cur) +{ + char *p; + + if (cur == buf) /* end of a log buf */ + return NULL; + + p = cur - 1; + while (p - 1 >= buf && *(p - 1) != '\n') + p--; + + return p; +} + +static void patch_log(char *buf, size_t buf_sz, size_t log_sz, + char *orig, size_t orig_sz, const char *patch) +{ + /* size of the remaining log content to the right from the to-be-replaced part */ + size_t rem_sz = (buf + log_sz) - (orig + orig_sz); + size_t patch_sz = strlen(patch); + + if (patch_sz != orig_sz) { + /* If patch line(s) are longer than original piece of verifier log, + * shift log contents by (patch_sz - orig_sz) bytes to the right + * starting from after to-be-replaced part of the log. + * + * If patch line(s) are shorter than original piece of verifier log, + * shift log contents by (orig_sz - patch_sz) bytes to the left + * starting from after to-be-replaced part of the log + * + * We need to be careful about not overflowing available + * buf_sz capacity. If that's the case, we'll truncate the end + * of the original log, as necessary. + */ + if (patch_sz > orig_sz) { + if (orig + patch_sz >= buf + buf_sz) { + /* patch is big enough to cover remaining space completely */ + patch_sz -= (orig + patch_sz) - (buf + buf_sz) + 1; + rem_sz = 0; + } else if (patch_sz - orig_sz > buf_sz - log_sz) { + /* patch causes part of remaining log to be truncated */ + rem_sz -= (patch_sz - orig_sz) - (buf_sz - log_sz); + } + } + /* shift remaining log to the right by calculated amount */ + memmove(orig + patch_sz, orig + orig_sz, rem_sz); + } + + memcpy(orig, patch, patch_sz); +} + +static void fixup_log_failed_core_relo(struct bpf_program *prog, + char *buf, size_t buf_sz, size_t log_sz, + char *line1, char *line2, char *line3) +{ + /* Expected log for failed and not properly guarded CO-RE relocation: + * line1 -> 123: (85) call unknown#195896080 + * line2 -> invalid func unknown#195896080 + * line3 -> + * + * "123" is the index of the instruction that was poisoned. We extract + * instruction index to find corresponding CO-RE relocation and + * replace this part of the log with more relevant information about + * failed CO-RE relocation. + */ + const struct bpf_core_relo *relo; + struct bpf_core_spec spec; + char patch[512], spec_buf[256]; + int insn_idx, err, spec_len; + + if (sscanf(line1, "%d: (%*d) call unknown#195896080\n", &insn_idx) != 1) + return; + + relo = find_relo_core(prog, insn_idx); + if (!relo) + return; + + err = bpf_core_parse_spec(prog->name, prog->obj->btf, relo, &spec); + if (err) + return; + + spec_len = bpf_core_format_spec(spec_buf, sizeof(spec_buf), &spec); + snprintf(patch, sizeof(patch), + "%d: \n" + "failed to resolve CO-RE relocation %s%s\n", + insn_idx, spec_buf, spec_len >= sizeof(spec_buf) ? "..." : ""); + + patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); +} + +static void fixup_log_missing_map_load(struct bpf_program *prog, + char *buf, size_t buf_sz, size_t log_sz, + char *line1, char *line2, char *line3) +{ + /* Expected log for failed and not properly guarded CO-RE relocation: + * line1 -> 123: (85) call unknown#2001000345 + * line2 -> invalid func unknown#2001000345 + * line3 -> + * + * "123" is the index of the instruction that was poisoned. + * "345" in "2001000345" are map index in obj->maps to fetch map name. + */ + struct bpf_object *obj = prog->obj; + const struct bpf_map *map; + int insn_idx, map_idx; + char patch[128]; + + if (sscanf(line1, "%d: (%*d) call unknown#%d\n", &insn_idx, &map_idx) != 2) + return; + + map_idx -= MAP_LDIMM64_POISON_BASE; + if (map_idx < 0 || map_idx >= obj->nr_maps) + return; + map = &obj->maps[map_idx]; + + snprintf(patch, sizeof(patch), + "%d: \n" + "BPF map '%s' is referenced but wasn't created\n", + insn_idx, map->name); + + patch_log(buf, buf_sz, log_sz, line1, line3 - line1, patch); +} + +static void fixup_verifier_log(struct bpf_program *prog, char *buf, size_t buf_sz) +{ + /* look for familiar error patterns in last N lines of the log */ + const size_t max_last_line_cnt = 10; + char *prev_line, *cur_line, *next_line; + size_t log_sz; + int i; + + if (!buf) + return; + + log_sz = strlen(buf) + 1; + next_line = buf + log_sz - 1; + + for (i = 0; i < max_last_line_cnt; i++, next_line = cur_line) { + cur_line = find_prev_line(buf, next_line); + if (!cur_line) + return; + + /* failed CO-RE relocation case */ + if (str_has_pfx(cur_line, "invalid func unknown#195896080\n")) { + prev_line = find_prev_line(buf, cur_line); + if (!prev_line) + continue; + + fixup_log_failed_core_relo(prog, buf, buf_sz, log_sz, + prev_line, cur_line, next_line); + return; + } else if (str_has_pfx(cur_line, "invalid func unknown#"MAP_LDIMM64_POISON_PFX)) { + prev_line = find_prev_line(buf, cur_line); + if (!prev_line) + continue; + + fixup_log_missing_map_load(prog, buf, buf_sz, log_sz, + prev_line, cur_line, next_line); + return; + } + } +} + static int bpf_program_record_relos(struct bpf_program *prog) { struct bpf_object *obj = prog->obj; @@ -6841,93 +7036,6 @@ static int bpf_program_record_relos(struct bpf_program *prog) return 0; } -static int bpf_object_load_prog(struct bpf_object *obj, struct bpf_program *prog, - const char *license, __u32 kern_ver) -{ - int err = 0, fd, i; - - if (obj->loaded) { - pr_warn("prog '%s': can't load after object was loaded\n", prog->name); - return libbpf_err(-EINVAL); - } - - if (prog->instances.nr < 0 || !prog->instances.fds) { - if (prog->preprocessor) { - pr_warn("Internal error: can't load program '%s'\n", - prog->name); - return libbpf_err(-LIBBPF_ERRNO__INTERNAL); - } - - prog->instances.fds = malloc(sizeof(int)); - if (!prog->instances.fds) { - pr_warn("Not enough memory for BPF fds\n"); - return libbpf_err(-ENOMEM); - } - prog->instances.nr = 1; - prog->instances.fds[0] = -1; - } - - if (!prog->preprocessor) { - if (prog->instances.nr != 1) { - pr_warn("prog '%s': inconsistent nr(%d) != 1\n", - prog->name, prog->instances.nr); - } - if (obj->gen_loader) - bpf_program_record_relos(prog); - err = bpf_object_load_prog_instance(obj, prog, - prog->insns, prog->insns_cnt, - license, kern_ver, &fd); - if (!err) - prog->instances.fds[0] = fd; - goto out; - } - - for (i = 0; i < prog->instances.nr; i++) { - struct bpf_prog_prep_result result; - bpf_program_prep_t preprocessor = prog->preprocessor; - - memset(&result, 0, sizeof(result)); - err = preprocessor(prog, i, prog->insns, - prog->insns_cnt, &result); - if (err) { - pr_warn("Preprocessing the %dth instance of program '%s' failed\n", - i, prog->name); - goto out; - } - - if (!result.new_insn_ptr || !result.new_insn_cnt) { - pr_debug("Skip loading the %dth instance of program '%s'\n", - i, prog->name); - prog->instances.fds[i] = -1; - if (result.pfd) - *result.pfd = -1; - continue; - } - - err = bpf_object_load_prog_instance(obj, prog, - result.new_insn_ptr, result.new_insn_cnt, - license, kern_ver, &fd); - if (err) { - pr_warn("Loading the %dth instance of program '%s' failed\n", - i, prog->name); - goto out; - } - - if (result.pfd) - *result.pfd = fd; - prog->instances.fds[i] = fd; - } -out: - if (err) - pr_warn("failed to load program '%s'\n", prog->name); - return libbpf_err(err); -} - -int bpf_program__load(struct bpf_program *prog, const char *license, __u32 kern_ver) -{ - return bpf_object_load_prog(prog->obj, prog, license, kern_ver); -} - static int bpf_object__load_progs(struct bpf_object *obj, int log_level) { @@ -6946,17 +7054,24 @@ bpf_object__load_progs(struct bpf_object *obj, int log_level) prog = &obj->programs[i]; if (prog_is_subprog(obj, prog)) continue; - if (!prog->load) { + if (!prog->autoload) { pr_debug("prog '%s': skipped loading\n", prog->name); continue; } prog->log_level |= log_level; - err = bpf_object_load_prog(obj, prog, obj->license, obj->kern_version); - if (err) + + if (obj->gen_loader) + bpf_program_record_relos(prog); + + err = bpf_object_load_prog(obj, prog, prog->insns, prog->insns_cnt, + obj->license, obj->kern_version, &prog->fd); + if (err) { + pr_warn("prog '%s': failed to load: %d\n", prog->name, err); return err; + } } - if (obj->gen_loader) - bpf_object__free_relocs(obj); + + bpf_object__free_relocs(obj); return 0; } @@ -6976,15 +7091,8 @@ static int bpf_object_init_progs(struct bpf_object *obj, const struct bpf_object continue; } - bpf_program__set_type(prog, prog->sec_def->prog_type); - bpf_program__set_expected_attach_type(prog, prog->sec_def->expected_attach_type); - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wdeprecated-declarations" - if (prog->sec_def->prog_type == BPF_PROG_TYPE_TRACING || - prog->sec_def->prog_type == BPF_PROG_TYPE_EXT) - prog->attach_prog_fd = OPTS_GET(opts, attach_prog_fd, 0); -#pragma GCC diagnostic pop + prog->type = prog->sec_def->prog_type; + prog->expected_attach_type = prog->sec_def->expected_attach_type; /* sec_def can have custom callback which should be called * after bpf_program is initialized to adjust its properties @@ -7091,36 +7199,6 @@ static struct bpf_object *bpf_object_open(const char *path, const void *obj_buf, return ERR_PTR(err); } -static struct bpf_object * -__bpf_object__open_xattr(struct bpf_object_open_attr *attr, int flags) -{ - DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, - .relaxed_maps = flags & MAPS_RELAX_COMPAT, - ); - - /* param validation */ - if (!attr->file) - return NULL; - - pr_debug("loading %s\n", attr->file); - return bpf_object_open(attr->file, NULL, 0, &opts); -} - -struct bpf_object *bpf_object__open_xattr(struct bpf_object_open_attr *attr) -{ - return libbpf_ptr(__bpf_object__open_xattr(attr, 0)); -} - -struct bpf_object *bpf_object__open(const char *path) -{ - struct bpf_object_open_attr attr = { - .file = path, - .prog_type = BPF_PROG_TYPE_UNSPEC, - }; - - return libbpf_ptr(__bpf_object__open_xattr(&attr, 0)); -} - struct bpf_object * bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) { @@ -7132,6 +7210,11 @@ bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts) return libbpf_ptr(bpf_object_open(path, NULL, 0, opts)); } +struct bpf_object *bpf_object__open(const char *path) +{ + return bpf_object__open_file(path, NULL); +} + struct bpf_object * bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, const struct bpf_object_open_opts *opts) @@ -7142,23 +7225,6 @@ bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, opts)); } -struct bpf_object * -bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, - const char *name) -{ - DECLARE_LIBBPF_OPTS(bpf_object_open_opts, opts, - .object_name = name, - /* wrong default, but backwards-compatible */ - .relaxed_maps = true, - ); - - /* returning NULL is wrong, but backwards-compatible */ - if (!obj_buf || obj_buf_sz == 0) - return errno = EINVAL, NULL; - - return libbpf_ptr(bpf_object_open(NULL, obj_buf, obj_buf_sz, &opts)); -} - static int bpf_object_unload(struct bpf_object *obj) { size_t i; @@ -7244,14 +7310,14 @@ static int kallsyms_cb(unsigned long long sym_addr, char sym_type, return 0; if (ext->is_set && ext->ksym.addr != sym_addr) { - pr_warn("extern (ksym) '%s' resolution is ambiguous: 0x%llx or 0x%llx\n", + pr_warn("extern (ksym) '%s': resolution is ambiguous: 0x%llx or 0x%llx\n", sym_name, ext->ksym.addr, sym_addr); return -EINVAL; } if (!ext->is_set) { ext->is_set = true; ext->ksym.addr = sym_addr; - pr_debug("extern (ksym) %s=0x%llx\n", sym_name, sym_addr); + pr_debug("extern (ksym) '%s': set to 0x%llx\n", sym_name, sym_addr); } return 0; } @@ -7455,28 +7521,52 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, for (i = 0; i < obj->nr_extern; i++) { ext = &obj->externs[i]; - if (ext->type == EXT_KCFG && - strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { - void *ext_val = kcfg_data + ext->kcfg.data_off; - __u32 kver = get_kernel_version(); - - if (!kver) { - pr_warn("failed to get kernel version\n"); - return -EINVAL; - } - err = set_kcfg_value_num(ext, ext_val, kver); - if (err) - return err; - pr_debug("extern (kcfg) %s=0x%x\n", ext->name, kver); - } else if (ext->type == EXT_KCFG && str_has_pfx(ext->name, "CONFIG_")) { - need_config = true; - } else if (ext->type == EXT_KSYM) { + if (ext->type == EXT_KSYM) { if (ext->ksym.type_id) need_vmlinux_btf = true; else need_kallsyms = true; + continue; + } else if (ext->type == EXT_KCFG) { + void *ext_ptr = kcfg_data + ext->kcfg.data_off; + __u64 value = 0; + + /* Kconfig externs need actual /proc/config.gz */ + if (str_has_pfx(ext->name, "CONFIG_")) { + need_config = true; + continue; + } + + /* Virtual kcfg externs are customly handled by libbpf */ + if (strcmp(ext->name, "LINUX_KERNEL_VERSION") == 0) { + value = get_kernel_version(); + if (!value) { + pr_warn("extern (kcfg) '%s': failed to get kernel version\n", ext->name); + return -EINVAL; + } + } else if (strcmp(ext->name, "LINUX_HAS_BPF_COOKIE") == 0) { + value = kernel_supports(obj, FEAT_BPF_COOKIE); + } else if (strcmp(ext->name, "LINUX_HAS_SYSCALL_WRAPPER") == 0) { + value = kernel_supports(obj, FEAT_SYSCALL_WRAPPER); + } else if (!str_has_pfx(ext->name, "LINUX_") || !ext->is_weak) { + /* Currently libbpf supports only CONFIG_ and LINUX_ prefixed + * __kconfig externs, where LINUX_ ones are virtual and filled out + * customly by libbpf (their values don't come from Kconfig). + * If LINUX_xxx variable is not recognized by libbpf, but is marked + * __weak, it defaults to zero value, just like for CONFIG_xxx + * externs. + */ + pr_warn("extern (kcfg) '%s': unrecognized virtual extern\n", ext->name); + return -EINVAL; + } + + err = set_kcfg_value_num(ext, ext_ptr, value); + if (err) + return err; + pr_debug("extern (kcfg) '%s': set to 0x%llx\n", + ext->name, (long long)value); } else { - pr_warn("unrecognized extern '%s'\n", ext->name); + pr_warn("extern '%s': unrecognized extern kind\n", ext->name); return -EINVAL; } } @@ -7512,10 +7602,10 @@ static int bpf_object__resolve_externs(struct bpf_object *obj, ext = &obj->externs[i]; if (!ext->is_set && !ext->is_weak) { - pr_warn("extern %s (strong) not resolved\n", ext->name); + pr_warn("extern '%s' (strong): not resolved\n", ext->name); return -ESRCH; } else if (!ext->is_set) { - pr_debug("extern %s (weak) not resolved, defaulting to zero\n", + pr_debug("extern '%s' (weak): not resolved, defaulting to zero\n", ext->name); } } @@ -7591,11 +7681,6 @@ static int bpf_object_load(struct bpf_object *obj, int extra_log_level, const ch return libbpf_err(err); } -int bpf_object__load_xattr(struct bpf_object_load_attr *attr) -{ - return bpf_object_load(attr->obj, attr->log_level, attr->target_btf_path); -} - int bpf_object__load(struct bpf_object *obj) { return bpf_object_load(obj, 0, NULL); @@ -7653,11 +7738,16 @@ static int check_path(const char *path) return err; } -static int bpf_program_pin_instance(struct bpf_program *prog, const char *path, int instance) +int bpf_program__pin(struct bpf_program *prog, const char *path) { char *cp, errmsg[STRERR_BUFSIZE]; int err; + if (prog->fd < 0) { + pr_warn("prog '%s': can't pin program that wasn't loaded\n", prog->name); + return libbpf_err(-EINVAL); + } + err = make_parent_dir(path); if (err) return libbpf_err(err); @@ -7666,170 +7756,35 @@ static int bpf_program_pin_instance(struct bpf_program *prog, const char *path, if (err) return libbpf_err(err); - if (prog == NULL) { - pr_warn("invalid program pointer\n"); - return libbpf_err(-EINVAL); - } - - if (instance < 0 || instance >= prog->instances.nr) { - pr_warn("invalid prog instance %d of prog %s (max %d)\n", - instance, prog->name, prog->instances.nr); - return libbpf_err(-EINVAL); - } - - if (bpf_obj_pin(prog->instances.fds[instance], path)) { + if (bpf_obj_pin(prog->fd, path)) { err = -errno; cp = libbpf_strerror_r(err, errmsg, sizeof(errmsg)); - pr_warn("failed to pin program: %s\n", cp); + pr_warn("prog '%s': failed to pin at '%s': %s\n", prog->name, path, cp); return libbpf_err(err); } - pr_debug("pinned program '%s'\n", path); + pr_debug("prog '%s': pinned at '%s'\n", prog->name, path); return 0; } -static int bpf_program_unpin_instance(struct bpf_program *prog, const char *path, int instance) -{ - int err; - - err = check_path(path); - if (err) - return libbpf_err(err); - - if (prog == NULL) { - pr_warn("invalid program pointer\n"); - return libbpf_err(-EINVAL); - } - - if (instance < 0 || instance >= prog->instances.nr) { - pr_warn("invalid prog instance %d of prog %s (max %d)\n", - instance, prog->name, prog->instances.nr); - return libbpf_err(-EINVAL); - } - - err = unlink(path); - if (err != 0) - return libbpf_err(-errno); - - pr_debug("unpinned program '%s'\n", path); - - return 0; -} - -__attribute__((alias("bpf_program_pin_instance"))) -int bpf_object__pin_instance(struct bpf_program *prog, const char *path, int instance); - -__attribute__((alias("bpf_program_unpin_instance"))) -int bpf_program__unpin_instance(struct bpf_program *prog, const char *path, int instance); - -int bpf_program__pin(struct bpf_program *prog, const char *path) -{ - int i, err; - - err = make_parent_dir(path); - if (err) - return libbpf_err(err); - - err = check_path(path); - if (err) - return libbpf_err(err); - - if (prog == NULL) { - pr_warn("invalid program pointer\n"); - return libbpf_err(-EINVAL); - } - - if (prog->instances.nr <= 0) { - pr_warn("no instances of prog %s to pin\n", prog->name); - return libbpf_err(-EINVAL); - } - - if (prog->instances.nr == 1) { - /* don't create subdirs when pinning single instance */ - return bpf_program_pin_instance(prog, path, 0); - } - - for (i = 0; i < prog->instances.nr; i++) { - char buf[PATH_MAX]; - int len; - - len = snprintf(buf, PATH_MAX, "%s/%d", path, i); - if (len < 0) { - err = -EINVAL; - goto err_unpin; - } else if (len >= PATH_MAX) { - err = -ENAMETOOLONG; - goto err_unpin; - } - - err = bpf_program_pin_instance(prog, buf, i); - if (err) - goto err_unpin; - } - - return 0; - -err_unpin: - for (i = i - 1; i >= 0; i--) { - char buf[PATH_MAX]; - int len; - - len = snprintf(buf, PATH_MAX, "%s/%d", path, i); - if (len < 0) - continue; - else if (len >= PATH_MAX) - continue; - - bpf_program_unpin_instance(prog, buf, i); - } - - rmdir(path); - - return libbpf_err(err); -} - int bpf_program__unpin(struct bpf_program *prog, const char *path) { - int i, err; + int err; + + if (prog->fd < 0) { + pr_warn("prog '%s': can't unpin program that wasn't loaded\n", prog->name); + return libbpf_err(-EINVAL); + } err = check_path(path); if (err) return libbpf_err(err); - if (prog == NULL) { - pr_warn("invalid program pointer\n"); - return libbpf_err(-EINVAL); - } - - if (prog->instances.nr <= 0) { - pr_warn("no instances of prog %s to pin\n", prog->name); - return libbpf_err(-EINVAL); - } - - if (prog->instances.nr == 1) { - /* don't create subdirs when pinning single instance */ - return bpf_program_unpin_instance(prog, path, 0); - } - - for (i = 0; i < prog->instances.nr; i++) { - char buf[PATH_MAX]; - int len; - - len = snprintf(buf, PATH_MAX, "%s/%d", path, i); - if (len < 0) - return libbpf_err(-EINVAL); - else if (len >= PATH_MAX) - return libbpf_err(-ENAMETOOLONG); - - err = bpf_program_unpin_instance(prog, buf, i); - if (err) - return err; - } - - err = rmdir(path); + err = unlink(path); if (err) return libbpf_err(-errno); + pr_debug("prog '%s': unpinned from '%s'\n", prog->name, path); return 0; } @@ -7985,7 +7940,7 @@ int bpf_object__pin_maps(struct bpf_object *obj, const char *path) char *pin_path = NULL; char buf[PATH_MAX]; - if (map->skipped) + if (!map->autocreate) continue; if (path) { @@ -8076,8 +8031,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path) char buf[PATH_MAX]; int len; - len = snprintf(buf, PATH_MAX, "%s/%s", path, - prog->pin_name); + len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name); if (len < 0) { err = -EINVAL; goto err_unpin_programs; @@ -8098,8 +8052,7 @@ int bpf_object__pin_programs(struct bpf_object *obj, const char *path) char buf[PATH_MAX]; int len; - len = snprintf(buf, PATH_MAX, "%s/%s", path, - prog->pin_name); + len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name); if (len < 0) continue; else if (len >= PATH_MAX) @@ -8123,8 +8076,7 @@ int bpf_object__unpin_programs(struct bpf_object *obj, const char *path) char buf[PATH_MAX]; int len; - len = snprintf(buf, PATH_MAX, "%s/%s", path, - prog->pin_name); + len = snprintf(buf, PATH_MAX, "%s/%s", path, prog->name); if (len < 0) return libbpf_err(-EINVAL); else if (len >= PATH_MAX) @@ -8157,11 +8109,6 @@ int bpf_object__pin(struct bpf_object *obj, const char *path) static void bpf_map__destroy(struct bpf_map *map) { - if (map->clear_priv) - map->clear_priv(map, map->priv); - map->priv = NULL; - map->clear_priv = NULL; - if (map->inner_map) { bpf_map__destroy(map->inner_map); zfree(&map->inner_map); @@ -8197,8 +8144,8 @@ void bpf_object__close(struct bpf_object *obj) if (IS_ERR_OR_NULL(obj)) return; - if (obj->clear_priv) - obj->clear_priv(obj, obj->priv); + usdt_manager_free(obj->usdt_man); + obj->usdt_man = NULL; bpf_gen__free(obj->gen_loader); bpf_object__elf_finish(obj); @@ -8223,33 +8170,9 @@ void bpf_object__close(struct bpf_object *obj) } zfree(&obj->programs); - list_del(&obj->list); free(obj); } -struct bpf_object * -bpf_object__next(struct bpf_object *prev) -{ - struct bpf_object *next; - bool strict = (libbpf_mode & LIBBPF_STRICT_NO_OBJECT_LIST); - - if (strict) - return NULL; - - if (!prev) - next = list_first_entry(&bpf_objects_list, - struct bpf_object, - list); - else - next = list_next_entry(prev, list); - - /* Empty list is noticed here so don't need checking on entry. */ - if (&next->list == &bpf_objects_list) - return NULL; - - return next; -} - const char *bpf_object__name(const struct bpf_object *obj) { return obj ? obj->name : libbpf_err_ptr(-EINVAL); @@ -8280,22 +8203,6 @@ int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version) return 0; } -int bpf_object__set_priv(struct bpf_object *obj, void *priv, - bpf_object_clear_priv_t clear_priv) -{ - if (obj->priv && obj->clear_priv) - obj->clear_priv(obj, obj->priv); - - obj->priv = priv; - obj->clear_priv = clear_priv; - return 0; -} - -void *bpf_object__priv(const struct bpf_object *obj) -{ - return obj ? obj->priv : libbpf_err_ptr(-EINVAL); -} - int bpf_object__gen_loader(struct bpf_object *obj, struct gen_loader_opts *opts) { struct bpf_gen *gen; @@ -8338,12 +8245,6 @@ __bpf_program__iter(const struct bpf_program *p, const struct bpf_object *obj, return &obj->programs[idx]; } -struct bpf_program * -bpf_program__next(struct bpf_program *prev, const struct bpf_object *obj) -{ - return bpf_object__next_program(obj, prev); -} - struct bpf_program * bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) { @@ -8356,12 +8257,6 @@ bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prev) return prog; } -struct bpf_program * -bpf_program__prev(struct bpf_program *next, const struct bpf_object *obj) -{ - return bpf_object__prev_program(obj, next); -} - struct bpf_program * bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) { @@ -8374,22 +8269,6 @@ bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *next) return prog; } -int bpf_program__set_priv(struct bpf_program *prog, void *priv, - bpf_program_clear_priv_t clear_priv) -{ - if (prog->priv && prog->clear_priv) - prog->clear_priv(prog, prog->priv); - - prog->priv = priv; - prog->clear_priv = clear_priv; - return 0; -} - -void *bpf_program__priv(const struct bpf_program *prog) -{ - return prog ? prog->priv : libbpf_err_ptr(-EINVAL); -} - void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex) { prog->prog_ifindex = ifindex; @@ -8405,25 +8284,9 @@ const char *bpf_program__section_name(const struct bpf_program *prog) return prog->sec_name; } -const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy) -{ - const char *title; - - title = prog->sec_name; - if (needs_copy) { - title = strdup(title); - if (!title) { - pr_warn("failed to strdup program title\n"); - return libbpf_err_ptr(-ENOMEM); - } - } - - return title; -} - bool bpf_program__autoload(const struct bpf_program *prog) { - return prog->load; + return prog->autoload; } int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) @@ -8431,22 +8294,10 @@ int bpf_program__set_autoload(struct bpf_program *prog, bool autoload) if (prog->obj->loaded) return libbpf_err(-EINVAL); - prog->load = autoload; + prog->autoload = autoload; return 0; } -static int bpf_program_nth_fd(const struct bpf_program *prog, int n); - -int bpf_program__fd(const struct bpf_program *prog) -{ - return bpf_program_nth_fd(prog, 0); -} - -size_t bpf_program__size(const struct bpf_program *prog) -{ - return prog->insns_cnt * BPF_INSN_SZ; -} - const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog) { return prog->insns; @@ -8457,58 +8308,35 @@ size_t bpf_program__insn_cnt(const struct bpf_program *prog) return prog->insns_cnt; } -int bpf_program__set_prep(struct bpf_program *prog, int nr_instances, - bpf_program_prep_t prep) +int bpf_program__set_insns(struct bpf_program *prog, + struct bpf_insn *new_insns, size_t new_insn_cnt) { - int *instances_fds; + struct bpf_insn *insns; - if (nr_instances <= 0 || !prep) - return libbpf_err(-EINVAL); + if (prog->obj->loaded) + return -EBUSY; - if (prog->instances.nr > 0 || prog->instances.fds) { - pr_warn("Can't set pre-processor after loading\n"); - return libbpf_err(-EINVAL); + insns = libbpf_reallocarray(prog->insns, new_insn_cnt, sizeof(*insns)); + if (!insns) { + pr_warn("prog '%s': failed to realloc prog code\n", prog->name); + return -ENOMEM; } + memcpy(insns, new_insns, new_insn_cnt * sizeof(*insns)); - instances_fds = malloc(sizeof(int) * nr_instances); - if (!instances_fds) { - pr_warn("alloc memory failed for fds\n"); - return libbpf_err(-ENOMEM); - } - - /* fill all fd with -1 */ - memset(instances_fds, -1, sizeof(int) * nr_instances); - - prog->instances.nr = nr_instances; - prog->instances.fds = instances_fds; - prog->preprocessor = prep; + prog->insns = insns; + prog->insns_cnt = new_insn_cnt; return 0; } -__attribute__((alias("bpf_program_nth_fd"))) -int bpf_program__nth_fd(const struct bpf_program *prog, int n); - -static int bpf_program_nth_fd(const struct bpf_program *prog, int n) +int bpf_program__fd(const struct bpf_program *prog) { - int fd; - if (!prog) return libbpf_err(-EINVAL); - if (n >= prog->instances.nr || n < 0) { - pr_warn("Can't get the %dth fd from program %s: only %d instances\n", - n, prog->name, prog->instances.nr); - return libbpf_err(-EINVAL); - } - - fd = prog->instances.fds[n]; - if (fd < 0) { - pr_warn("%dth instance of program '%s' is invalid\n", - n, prog->name); + if (prog->fd < 0) return libbpf_err(-ENOENT); - } - return fd; + return prog->fd; } __alias(bpf_program__type) @@ -8519,45 +8347,15 @@ enum bpf_prog_type bpf_program__type(const struct bpf_program *prog) return prog->type; } -void bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) +int bpf_program__set_type(struct bpf_program *prog, enum bpf_prog_type type) { + if (prog->obj->loaded) + return libbpf_err(-EBUSY); + prog->type = type; + return 0; } -static bool bpf_program__is_type(const struct bpf_program *prog, - enum bpf_prog_type type) -{ - return prog ? (prog->type == type) : false; -} - -#define BPF_PROG_TYPE_FNS(NAME, TYPE) \ -int bpf_program__set_##NAME(struct bpf_program *prog) \ -{ \ - if (!prog) \ - return libbpf_err(-EINVAL); \ - bpf_program__set_type(prog, TYPE); \ - return 0; \ -} \ - \ -bool bpf_program__is_##NAME(const struct bpf_program *prog) \ -{ \ - return bpf_program__is_type(prog, TYPE); \ -} \ - -BPF_PROG_TYPE_FNS(socket_filter, BPF_PROG_TYPE_SOCKET_FILTER); -BPF_PROG_TYPE_FNS(lsm, BPF_PROG_TYPE_LSM); -BPF_PROG_TYPE_FNS(kprobe, BPF_PROG_TYPE_KPROBE); -BPF_PROG_TYPE_FNS(sched_cls, BPF_PROG_TYPE_SCHED_CLS); -BPF_PROG_TYPE_FNS(sched_act, BPF_PROG_TYPE_SCHED_ACT); -BPF_PROG_TYPE_FNS(tracepoint, BPF_PROG_TYPE_TRACEPOINT); -BPF_PROG_TYPE_FNS(raw_tracepoint, BPF_PROG_TYPE_RAW_TRACEPOINT); -BPF_PROG_TYPE_FNS(xdp, BPF_PROG_TYPE_XDP); -BPF_PROG_TYPE_FNS(perf_event, BPF_PROG_TYPE_PERF_EVENT); -BPF_PROG_TYPE_FNS(tracing, BPF_PROG_TYPE_TRACING); -BPF_PROG_TYPE_FNS(struct_ops, BPF_PROG_TYPE_STRUCT_OPS); -BPF_PROG_TYPE_FNS(extension, BPF_PROG_TYPE_EXT); -BPF_PROG_TYPE_FNS(sk_lookup, BPF_PROG_TYPE_SK_LOOKUP); - __alias(bpf_program__expected_attach_type) enum bpf_attach_type bpf_program__get_expected_attach_type(const struct bpf_program *prog); @@ -8566,10 +8364,14 @@ enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program return prog->expected_attach_type; } -void bpf_program__set_expected_attach_type(struct bpf_program *prog, +int bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type) { + if (prog->obj->loaded) + return libbpf_err(-EBUSY); + prog->expected_attach_type = type; + return 0; } __u32 bpf_program__flags(const struct bpf_program *prog) @@ -8630,6 +8432,9 @@ int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log } static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); +static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link); +static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link); +static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link); static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link); @@ -8638,83 +8443,87 @@ static int attach_lsm(const struct bpf_program *prog, long cookie, struct bpf_li static int attach_iter(const struct bpf_program *prog, long cookie, struct bpf_link **link); static const struct bpf_sec_def section_defs[] = { - SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("kprobe/", KPROBE, 0, SEC_NONE, attach_kprobe), - SEC_DEF("uprobe/", KPROBE, 0, SEC_NONE), - SEC_DEF("kretprobe/", KPROBE, 0, SEC_NONE, attach_kprobe), - SEC_DEF("uretprobe/", KPROBE, 0, SEC_NONE), - SEC_DEF("kprobe.multi/", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), - SEC_DEF("kretprobe.multi/", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), + SEC_DEF("socket", SOCKET_FILTER, 0, SEC_NONE), + SEC_DEF("sk_reuseport/migrate", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT_OR_MIGRATE, SEC_ATTACHABLE), + SEC_DEF("sk_reuseport", SK_REUSEPORT, BPF_SK_REUSEPORT_SELECT, SEC_ATTACHABLE), + SEC_DEF("kprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), + SEC_DEF("uprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), + SEC_DEF("uprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), + SEC_DEF("kretprobe+", KPROBE, 0, SEC_NONE, attach_kprobe), + SEC_DEF("uretprobe+", KPROBE, 0, SEC_NONE, attach_uprobe), + SEC_DEF("uretprobe.s+", KPROBE, 0, SEC_SLEEPABLE, attach_uprobe), + SEC_DEF("kprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), + SEC_DEF("kretprobe.multi+", KPROBE, BPF_TRACE_KPROBE_MULTI, SEC_NONE, attach_kprobe_multi), + SEC_DEF("ksyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), + SEC_DEF("kretsyscall+", KPROBE, 0, SEC_NONE, attach_ksyscall), + SEC_DEF("usdt+", KPROBE, 0, SEC_NONE, attach_usdt), SEC_DEF("tc", SCHED_CLS, 0, SEC_NONE), - SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE | SEC_SLOPPY_PFX | SEC_DEPRECATED), - SEC_DEF("action", SCHED_ACT, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("tracepoint/", TRACEPOINT, 0, SEC_NONE, attach_tp), - SEC_DEF("tp/", TRACEPOINT, 0, SEC_NONE, attach_tp), - SEC_DEF("raw_tracepoint/", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("raw_tp/", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("raw_tracepoint.w/", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("raw_tp.w/", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), - SEC_DEF("tp_btf/", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fentry/", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fmod_ret/", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fexit/", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("fentry.s/", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), - SEC_DEF("fmod_ret.s/", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), - SEC_DEF("fexit.s/", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), - SEC_DEF("freplace/", EXT, 0, SEC_ATTACH_BTF, attach_trace), - SEC_DEF("lsm/", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), - SEC_DEF("lsm.s/", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), - SEC_DEF("iter/", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), - SEC_DEF("iter.s/", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), + SEC_DEF("classifier", SCHED_CLS, 0, SEC_NONE), + SEC_DEF("action", SCHED_ACT, 0, SEC_NONE), + SEC_DEF("tracepoint+", TRACEPOINT, 0, SEC_NONE, attach_tp), + SEC_DEF("tp+", TRACEPOINT, 0, SEC_NONE, attach_tp), + SEC_DEF("raw_tracepoint+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), + SEC_DEF("raw_tp+", RAW_TRACEPOINT, 0, SEC_NONE, attach_raw_tp), + SEC_DEF("raw_tracepoint.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), + SEC_DEF("raw_tp.w+", RAW_TRACEPOINT_WRITABLE, 0, SEC_NONE, attach_raw_tp), + SEC_DEF("tp_btf+", TRACING, BPF_TRACE_RAW_TP, SEC_ATTACH_BTF, attach_trace), + SEC_DEF("fentry+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF, attach_trace), + SEC_DEF("fmod_ret+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF, attach_trace), + SEC_DEF("fexit+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF, attach_trace), + SEC_DEF("fentry.s+", TRACING, BPF_TRACE_FENTRY, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), + SEC_DEF("fmod_ret.s+", TRACING, BPF_MODIFY_RETURN, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), + SEC_DEF("fexit.s+", TRACING, BPF_TRACE_FEXIT, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_trace), + SEC_DEF("freplace+", EXT, 0, SEC_ATTACH_BTF, attach_trace), + SEC_DEF("lsm+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF, attach_lsm), + SEC_DEF("lsm.s+", LSM, BPF_LSM_MAC, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_lsm), + SEC_DEF("lsm_cgroup+", LSM, BPF_LSM_CGROUP, SEC_ATTACH_BTF), + SEC_DEF("iter+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF, attach_iter), + SEC_DEF("iter.s+", TRACING, BPF_TRACE_ITER, SEC_ATTACH_BTF | SEC_SLEEPABLE, attach_iter), SEC_DEF("syscall", SYSCALL, 0, SEC_SLEEPABLE), SEC_DEF("xdp.frags/devmap", XDP, BPF_XDP_DEVMAP, SEC_XDP_FRAGS), SEC_DEF("xdp/devmap", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE), - SEC_DEF("xdp_devmap/", XDP, BPF_XDP_DEVMAP, SEC_ATTACHABLE | SEC_DEPRECATED), SEC_DEF("xdp.frags/cpumap", XDP, BPF_XDP_CPUMAP, SEC_XDP_FRAGS), SEC_DEF("xdp/cpumap", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE), - SEC_DEF("xdp_cpumap/", XDP, BPF_XDP_CPUMAP, SEC_ATTACHABLE | SEC_DEPRECATED), SEC_DEF("xdp.frags", XDP, BPF_XDP, SEC_XDP_FRAGS), - SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE | SEC_SLOPPY_PFX), - SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE | SEC_SLOPPY_PFX), - SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE | SEC_SLOPPY_PFX), + SEC_DEF("xdp", XDP, BPF_XDP, SEC_ATTACHABLE_OPT), + SEC_DEF("perf_event", PERF_EVENT, 0, SEC_NONE), + SEC_DEF("lwt_in", LWT_IN, 0, SEC_NONE), + SEC_DEF("lwt_out", LWT_OUT, 0, SEC_NONE), + SEC_DEF("lwt_xmit", LWT_XMIT, 0, SEC_NONE), + SEC_DEF("lwt_seg6local", LWT_SEG6LOCAL, 0, SEC_NONE), + SEC_DEF("sockops", SOCK_OPS, BPF_CGROUP_SOCK_OPS, SEC_ATTACHABLE_OPT), + SEC_DEF("sk_skb/stream_parser", SK_SKB, BPF_SK_SKB_STREAM_PARSER, SEC_ATTACHABLE_OPT), + SEC_DEF("sk_skb/stream_verdict",SK_SKB, BPF_SK_SKB_STREAM_VERDICT, SEC_ATTACHABLE_OPT), + SEC_DEF("sk_skb", SK_SKB, 0, SEC_NONE), + SEC_DEF("sk_msg", SK_MSG, BPF_SK_MSG_VERDICT, SEC_ATTACHABLE_OPT), + SEC_DEF("lirc_mode2", LIRC_MODE2, BPF_LIRC_MODE2, SEC_ATTACHABLE_OPT), + SEC_DEF("flow_dissector", FLOW_DISSECTOR, BPF_FLOW_DISSECTOR, SEC_ATTACHABLE_OPT), + SEC_DEF("cgroup_skb/ingress", CGROUP_SKB, BPF_CGROUP_INET_INGRESS, SEC_ATTACHABLE_OPT), + SEC_DEF("cgroup_skb/egress", CGROUP_SKB, BPF_CGROUP_INET_EGRESS, SEC_ATTACHABLE_OPT), + SEC_DEF("cgroup/skb", CGROUP_SKB, 0, SEC_NONE), + SEC_DEF("cgroup/sock_create", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE), + SEC_DEF("cgroup/sock_release", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_RELEASE, SEC_ATTACHABLE), + SEC_DEF("cgroup/sock", CGROUP_SOCK, BPF_CGROUP_INET_SOCK_CREATE, SEC_ATTACHABLE_OPT), + SEC_DEF("cgroup/post_bind4", CGROUP_SOCK, BPF_CGROUP_INET4_POST_BIND, SEC_ATTACHABLE), + SEC_DEF("cgroup/post_bind6", CGROUP_SOCK, BPF_CGROUP_INET6_POST_BIND, SEC_ATTACHABLE), + SEC_DEF("cgroup/bind4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_BIND, SEC_ATTACHABLE), + SEC_DEF("cgroup/bind6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_BIND, SEC_ATTACHABLE), + SEC_DEF("cgroup/connect4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_CONNECT, SEC_ATTACHABLE), + SEC_DEF("cgroup/connect6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_CONNECT, SEC_ATTACHABLE), + SEC_DEF("cgroup/sendmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_SENDMSG, SEC_ATTACHABLE), + SEC_DEF("cgroup/sendmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_SENDMSG, SEC_ATTACHABLE), + SEC_DEF("cgroup/recvmsg4", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP4_RECVMSG, SEC_ATTACHABLE), + SEC_DEF("cgroup/recvmsg6", CGROUP_SOCK_ADDR, BPF_CGROUP_UDP6_RECVMSG, SEC_ATTACHABLE), + SEC_DEF("cgroup/getpeername4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETPEERNAME, SEC_ATTACHABLE), + SEC_DEF("cgroup/getpeername6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETPEERNAME, SEC_ATTACHABLE), + SEC_DEF("cgroup/getsockname4", CGROUP_SOCK_ADDR, BPF_CGROUP_INET4_GETSOCKNAME, SEC_ATTACHABLE), + SEC_DEF("cgroup/getsockname6", CGROUP_SOCK_ADDR, BPF_CGROUP_INET6_GETSOCKNAME, SEC_ATTACHABLE), + SEC_DEF("cgroup/sysctl", CGROUP_SYSCTL, BPF_CGROUP_SYSCTL, SEC_ATTACHABLE), + SEC_DEF("cgroup/getsockopt", CGROUP_SOCKOPT, BPF_CGROUP_GETSOCKOPT, SEC_ATTACHABLE), + SEC_DEF("cgroup/setsockopt", CGROUP_SOCKOPT, BPF_CGROUP_SETSOCKOPT, SEC_ATTACHABLE), + SEC_DEF("cgroup/dev", CGROUP_DEVICE, BPF_CGROUP_DEVICE, SEC_ATTACHABLE_OPT), SEC_DEF("struct_ops+", STRUCT_OPS, 0, SEC_NONE), - SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE | SEC_SLOPPY_PFX), + SEC_DEF("sk_lookup", SK_LOOKUP, BPF_SK_LOOKUP, SEC_ATTACHABLE), }; static size_t custom_sec_def_cnt; @@ -8809,8 +8618,7 @@ int libbpf_unregister_prog_handler(int handler_id) return 0; } -static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name, - bool allow_sloppy) +static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_name) { size_t len = strlen(sec_def->sec); @@ -8835,17 +8643,6 @@ static bool sec_def_matches(const struct bpf_sec_def *sec_def, const char *sec_n return false; } - /* SEC_SLOPPY_PFX definitions are allowed to be just prefix - * matches, unless strict section name mode - * (LIBBPF_STRICT_SEC_NAME) is enabled, in which case the - * match has to be exact. - */ - if (allow_sloppy && str_has_pfx(sec_name, sec_def->sec)) - return true; - - /* Definitions not marked SEC_SLOPPY_PFX (e.g., - * SEC("syscall")) are exact matches in both modes. - */ return strcmp(sec_name, sec_def->sec) == 0; } @@ -8853,20 +8650,18 @@ static const struct bpf_sec_def *find_sec_def(const char *sec_name) { const struct bpf_sec_def *sec_def; int i, n; - bool strict = libbpf_mode & LIBBPF_STRICT_SEC_NAME, allow_sloppy; n = custom_sec_def_cnt; for (i = 0; i < n; i++) { sec_def = &custom_sec_defs[i]; - if (sec_def_matches(sec_def, sec_name, false)) + if (sec_def_matches(sec_def, sec_name)) return sec_def; } n = ARRAY_SIZE(section_defs); for (i = 0; i < n; i++) { sec_def = §ion_defs[i]; - allow_sloppy = (sec_def->cookie & SEC_SLOPPY_PFX) && !strict; - if (sec_def_matches(sec_def, sec_name, allow_sloppy)) + if (sec_def_matches(sec_def, sec_name)) return sec_def; } @@ -8937,6 +8732,38 @@ int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, return libbpf_err(-ESRCH); } +const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t) +{ + if (t < 0 || t >= ARRAY_SIZE(attach_type_name)) + return NULL; + + return attach_type_name[t]; +} + +const char *libbpf_bpf_link_type_str(enum bpf_link_type t) +{ + if (t < 0 || t >= ARRAY_SIZE(link_type_name)) + return NULL; + + return link_type_name[t]; +} + +const char *libbpf_bpf_map_type_str(enum bpf_map_type t) +{ + if (t < 0 || t >= ARRAY_SIZE(map_type_name)) + return NULL; + + return map_type_name[t]; +} + +const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t) +{ + if (t < 0 || t >= ARRAY_SIZE(prog_type_name)) + return NULL; + + return prog_type_name[t]; +} + static struct bpf_map *find_struct_ops_map_by_offset(struct bpf_object *obj, size_t offset) { @@ -9087,6 +8914,7 @@ void btf_get_kernel_prefix_kind(enum bpf_attach_type attach_type, *kind = BTF_KIND_TYPEDEF; break; case BPF_LSM_MAC: + case BPF_LSM_CGROUP: *prefix = BTF_LSM_PREFIX; *kind = BTF_KIND_FUNC; break; @@ -9290,11 +9118,6 @@ int bpf_map__fd(const struct bpf_map *map) return map ? map->fd : libbpf_err(-EINVAL); } -const struct bpf_map_def *bpf_map__def(const struct bpf_map *map) -{ - return map ? &map->def : libbpf_err_ptr(-EINVAL); -} - static bool map_uses_real_name(const struct bpf_map *map) { /* Since libbpf started to support custom .data.* and .rodata.* maps, @@ -9409,27 +9232,6 @@ __u32 bpf_map__btf_value_type_id(const struct bpf_map *map) return map ? map->btf_value_type_id : 0; } -int bpf_map__set_priv(struct bpf_map *map, void *priv, - bpf_map_clear_priv_t clear_priv) -{ - if (!map) - return libbpf_err(-EINVAL); - - if (map->priv) { - if (map->clear_priv) - map->clear_priv(map, map->priv); - } - - map->priv = priv; - map->clear_priv = clear_priv; - return 0; -} - -void *bpf_map__priv(const struct bpf_map *map) -{ - return map ? map->priv : libbpf_err_ptr(-EINVAL); -} - int bpf_map__set_initial_value(struct bpf_map *map, const void *data, size_t size) { @@ -9449,11 +9251,6 @@ const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize) return map->mmaped; } -bool bpf_map__is_offload_neutral(const struct bpf_map *map) -{ - return map->def.type == BPF_MAP_TYPE_PERF_EVENT_ARRAY; -} - bool bpf_map__is_internal(const struct bpf_map *map) { return map->libbpf_type != LIBBPF_MAP_UNSPEC; @@ -9514,12 +9311,6 @@ __bpf_map__iter(const struct bpf_map *m, const struct bpf_object *obj, int i) return &obj->maps[idx]; } -struct bpf_map * -bpf_map__next(const struct bpf_map *prev, const struct bpf_object *obj) -{ - return bpf_object__next_map(obj, prev); -} - struct bpf_map * bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) { @@ -9529,12 +9320,6 @@ bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *prev) return __bpf_map__iter(prev, obj, 1); } -struct bpf_map * -bpf_map__prev(const struct bpf_map *next, const struct bpf_object *obj) -{ - return bpf_object__prev_map(obj, next); -} - struct bpf_map * bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *next) { @@ -9580,10 +9365,108 @@ bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name) return bpf_map__fd(bpf_object__find_map_by_name(obj, name)); } -struct bpf_map * -bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset) +static int validate_map_op(const struct bpf_map *map, size_t key_sz, + size_t value_sz, bool check_value_sz) { - return libbpf_err_ptr(-ENOTSUP); + if (map->fd <= 0) + return -ENOENT; + + if (map->def.key_size != key_sz) { + pr_warn("map '%s': unexpected key size %zu provided, expected %u\n", + map->name, key_sz, map->def.key_size); + return -EINVAL; + } + + if (!check_value_sz) + return 0; + + switch (map->def.type) { + case BPF_MAP_TYPE_PERCPU_ARRAY: + case BPF_MAP_TYPE_PERCPU_HASH: + case BPF_MAP_TYPE_LRU_PERCPU_HASH: + case BPF_MAP_TYPE_PERCPU_CGROUP_STORAGE: { + int num_cpu = libbpf_num_possible_cpus(); + size_t elem_sz = roundup(map->def.value_size, 8); + + if (value_sz != num_cpu * elem_sz) { + pr_warn("map '%s': unexpected value size %zu provided for per-CPU map, expected %d * %zu = %zd\n", + map->name, value_sz, num_cpu, elem_sz, num_cpu * elem_sz); + return -EINVAL; + } + break; + } + default: + if (map->def.value_size != value_sz) { + pr_warn("map '%s': unexpected value size %zu provided, expected %u\n", + map->name, value_sz, map->def.value_size); + return -EINVAL; + } + break; + } + return 0; +} + +int bpf_map__lookup_elem(const struct bpf_map *map, + const void *key, size_t key_sz, + void *value, size_t value_sz, __u64 flags) +{ + int err; + + err = validate_map_op(map, key_sz, value_sz, true); + if (err) + return libbpf_err(err); + + return bpf_map_lookup_elem_flags(map->fd, key, value, flags); +} + +int bpf_map__update_elem(const struct bpf_map *map, + const void *key, size_t key_sz, + const void *value, size_t value_sz, __u64 flags) +{ + int err; + + err = validate_map_op(map, key_sz, value_sz, true); + if (err) + return libbpf_err(err); + + return bpf_map_update_elem(map->fd, key, value, flags); +} + +int bpf_map__delete_elem(const struct bpf_map *map, + const void *key, size_t key_sz, __u64 flags) +{ + int err; + + err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); + if (err) + return libbpf_err(err); + + return bpf_map_delete_elem_flags(map->fd, key, flags); +} + +int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, + const void *key, size_t key_sz, + void *value, size_t value_sz, __u64 flags) +{ + int err; + + err = validate_map_op(map, key_sz, value_sz, true); + if (err) + return libbpf_err(err); + + return bpf_map_lookup_and_delete_elem_flags(map->fd, key, value, flags); +} + +int bpf_map__get_next_key(const struct bpf_map *map, + const void *cur_key, void *next_key, size_t key_sz) +{ + int err; + + err = validate_map_op(map, key_sz, 0, false /* check_value_sz */); + if (err) + return libbpf_err(err); + + return bpf_map_get_next_key(map->fd, cur_key, next_key); } long libbpf_get_error(const void *ptr) @@ -9602,104 +9485,6 @@ long libbpf_get_error(const void *ptr) return -errno; } -__attribute__((alias("bpf_prog_load_xattr2"))) -int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, - struct bpf_object **pobj, int *prog_fd); - -static int bpf_prog_load_xattr2(const struct bpf_prog_load_attr *attr, - struct bpf_object **pobj, int *prog_fd) -{ - struct bpf_object_open_attr open_attr = {}; - struct bpf_program *prog, *first_prog = NULL; - struct bpf_object *obj; - struct bpf_map *map; - int err; - - if (!attr) - return libbpf_err(-EINVAL); - if (!attr->file) - return libbpf_err(-EINVAL); - - open_attr.file = attr->file; - open_attr.prog_type = attr->prog_type; - - obj = __bpf_object__open_xattr(&open_attr, 0); - err = libbpf_get_error(obj); - if (err) - return libbpf_err(-ENOENT); - - bpf_object__for_each_program(prog, obj) { - enum bpf_attach_type attach_type = attr->expected_attach_type; - /* - * to preserve backwards compatibility, bpf_prog_load treats - * attr->prog_type, if specified, as an override to whatever - * bpf_object__open guessed - */ - if (attr->prog_type != BPF_PROG_TYPE_UNSPEC) { - bpf_program__set_type(prog, attr->prog_type); - bpf_program__set_expected_attach_type(prog, - attach_type); - } - if (bpf_program__type(prog) == BPF_PROG_TYPE_UNSPEC) { - /* - * we haven't guessed from section name and user - * didn't provide a fallback type, too bad... - */ - bpf_object__close(obj); - return libbpf_err(-EINVAL); - } - - prog->prog_ifindex = attr->ifindex; - prog->log_level = attr->log_level; - prog->prog_flags |= attr->prog_flags; - if (!first_prog) - first_prog = prog; - } - - bpf_object__for_each_map(map, obj) { - if (map->def.type != BPF_MAP_TYPE_PERF_EVENT_ARRAY) - map->map_ifindex = attr->ifindex; - } - - if (!first_prog) { - pr_warn("object file doesn't contain bpf program\n"); - bpf_object__close(obj); - return libbpf_err(-ENOENT); - } - - err = bpf_object__load(obj); - if (err) { - bpf_object__close(obj); - return libbpf_err(err); - } - - *pobj = obj; - *prog_fd = bpf_program__fd(first_prog); - return 0; -} - -COMPAT_VERSION(bpf_prog_load_deprecated, bpf_prog_load, LIBBPF_0.0.1) -int bpf_prog_load_deprecated(const char *file, enum bpf_prog_type type, - struct bpf_object **pobj, int *prog_fd) -{ - struct bpf_prog_load_attr attr; - - memset(&attr, 0, sizeof(struct bpf_prog_load_attr)); - attr.file = file; - attr.prog_type = type; - attr.expected_attach_type = 0; - - return bpf_prog_load_xattr2(&attr, pobj, prog_fd); -} - -struct bpf_link { - int (*detach)(struct bpf_link *link); - void (*dealloc)(struct bpf_link *link); - char *pin_path; /* NULL, if not pinned */ - int fd; /* hook FD, -1 if not applicable */ - bool disconnected; -}; - /* Replace link's underlying BPF program with the new one */ int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog) { @@ -10027,7 +9812,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, { struct perf_event_attr attr = {}; char errmsg[STRERR_BUFSIZE]; - int type, pfd, err; + int type, pfd; if (ref_ctr_off >= (1ULL << PERF_UPROBE_REF_CTR_OFFSET_BITS)) return -EINVAL; @@ -10063,14 +9848,7 @@ static int perf_event_open_probe(bool uprobe, bool retprobe, const char *name, pid < 0 ? -1 : pid /* pid */, pid == -1 ? 0 : -1 /* cpu */, -1 /* group_fd */, PERF_FLAG_FD_CLOEXEC); - if (pfd < 0) { - err = -errno; - pr_warn("%s perf_event_open() failed: %s\n", - uprobe ? "uprobe" : "kprobe", - libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - return err; - } - return pfd; + return pfd >= 0 ? pfd : -errno; } static int append_to_file(const char *file, const char *fmt, ...) @@ -10093,6 +9871,34 @@ static int append_to_file(const char *file, const char *fmt, ...) return err; } +#define DEBUGFS "/sys/kernel/debug/tracing" +#define TRACEFS "/sys/kernel/tracing" + +static bool use_debugfs(void) +{ + static int has_debugfs = -1; + + if (has_debugfs < 0) + has_debugfs = access(DEBUGFS, F_OK) == 0; + + return has_debugfs == 1; +} + +static const char *tracefs_path(void) +{ + return use_debugfs() ? DEBUGFS : TRACEFS; +} + +static const char *tracefs_kprobe_events(void) +{ + return use_debugfs() ? DEBUGFS"/kprobe_events" : TRACEFS"/kprobe_events"; +} + +static const char *tracefs_uprobe_events(void) +{ + return use_debugfs() ? DEBUGFS"/uprobe_events" : TRACEFS"/uprobe_events"; +} + static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, const char *kfunc_name, size_t offset) { @@ -10105,9 +9911,7 @@ static void gen_kprobe_legacy_event_name(char *buf, size_t buf_sz, static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, const char *kfunc_name, size_t offset) { - const char *file = "/sys/kernel/debug/tracing/kprobe_events"; - - return append_to_file(file, "%c:%s/%s %s+0x%zx", + return append_to_file(tracefs_kprobe_events(), "%c:%s/%s %s+0x%zx", retprobe ? 'r' : 'p', retprobe ? "kretprobes" : "kprobes", probe_name, kfunc_name, offset); @@ -10115,18 +9919,16 @@ static int add_kprobe_event_legacy(const char *probe_name, bool retprobe, static int remove_kprobe_event_legacy(const char *probe_name, bool retprobe) { - const char *file = "/sys/kernel/debug/tracing/kprobe_events"; - - return append_to_file(file, "-:%s/%s", retprobe ? "kretprobes" : "kprobes", probe_name); + return append_to_file(tracefs_kprobe_events(), "-:%s/%s", + retprobe ? "kretprobes" : "kprobes", probe_name); } static int determine_kprobe_perf_type_legacy(const char *probe_name, bool retprobe) { char file[256]; - snprintf(file, sizeof(file), - "/sys/kernel/debug/tracing/events/%s/%s/id", - retprobe ? "kretprobes" : "kprobes", probe_name); + snprintf(file, sizeof(file), "%s/events/%s/%s/id", + tracefs_path(), retprobe ? "kretprobes" : "kprobes", probe_name); return parse_uint_from_file(file, "%d\n"); } @@ -10147,10 +9949,11 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, } type = determine_kprobe_perf_type_legacy(probe_name, retprobe); if (type < 0) { + err = type; pr_warn("failed to determine legacy kprobe event id for '%s+0x%zx': %s\n", kfunc_name, offset, - libbpf_strerror_r(type, errmsg, sizeof(errmsg))); - return type; + libbpf_strerror_r(err, errmsg, sizeof(errmsg))); + goto err_clean_legacy; } attr.size = sizeof(attr); attr.config = type; @@ -10164,9 +9967,72 @@ static int perf_event_kprobe_open_legacy(const char *probe_name, bool retprobe, err = -errno; pr_warn("legacy kprobe perf_event_open() failed: %s\n", libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - return err; + goto err_clean_legacy; } return pfd; + +err_clean_legacy: + /* Clear the newly added legacy kprobe_event */ + remove_kprobe_event_legacy(probe_name, retprobe); + return err; +} + +static const char *arch_specific_syscall_pfx(void) +{ +#if defined(__x86_64__) + return "x64"; +#elif defined(__i386__) + return "ia32"; +#elif defined(__s390x__) + return "s390x"; +#elif defined(__s390__) + return "s390"; +#elif defined(__arm__) + return "arm"; +#elif defined(__aarch64__) + return "arm64"; +#elif defined(__mips__) + return "mips"; +#elif defined(__riscv) + return "riscv"; +#elif defined(__powerpc__) + return "powerpc"; +#elif defined(__powerpc64__) + return "powerpc64"; +#else + return NULL; +#endif +} + +static int probe_kern_syscall_wrapper(void) +{ + char syscall_name[64]; + const char *ksys_pfx; + + ksys_pfx = arch_specific_syscall_pfx(); + if (!ksys_pfx) + return 0; + + snprintf(syscall_name, sizeof(syscall_name), "__%s_sys_bpf", ksys_pfx); + + if (determine_kprobe_perf_type() >= 0) { + int pfd; + + pfd = perf_event_open_probe(false, false, syscall_name, 0, getpid(), 0); + if (pfd >= 0) + close(pfd); + + return pfd >= 0 ? 1 : 0; + } else { /* legacy mode */ + char probe_name[128]; + + gen_kprobe_legacy_event_name(probe_name, sizeof(probe_name), syscall_name, 0); + if (add_kprobe_event_legacy(probe_name, false, syscall_name, 0) < 0) + return 0; + + (void)remove_kprobe_event_legacy(probe_name, false); + return 1; + } } struct bpf_link * @@ -10223,7 +10089,7 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, prog->name, retprobe ? "kretprobe" : "kprobe", func_name, offset, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_out; + goto err_clean_legacy; } if (legacy) { struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); @@ -10234,6 +10100,10 @@ bpf_program__attach_kprobe_opts(const struct bpf_program *prog, } return link; + +err_clean_legacy: + if (legacy) + remove_kprobe_event_legacy(legacy_probe, retprobe); err_out: free(legacy_probe); return libbpf_err_ptr(err); @@ -10250,6 +10120,34 @@ struct bpf_link *bpf_program__attach_kprobe(const struct bpf_program *prog, return bpf_program__attach_kprobe_opts(prog, func_name, &opts); } +struct bpf_link *bpf_program__attach_ksyscall(const struct bpf_program *prog, + const char *syscall_name, + const struct bpf_ksyscall_opts *opts) +{ + LIBBPF_OPTS(bpf_kprobe_opts, kprobe_opts); + char func_name[128]; + + if (!OPTS_VALID(opts, bpf_ksyscall_opts)) + return libbpf_err_ptr(-EINVAL); + + if (kernel_supports(prog->obj, FEAT_SYSCALL_WRAPPER)) { + /* arch_specific_syscall_pfx() should never return NULL here + * because it is guarded by kernel_supports(). However, since + * compiler does not know that we have an explicit conditional + * as well. + */ + snprintf(func_name, sizeof(func_name), "__%s_sys_%s", + arch_specific_syscall_pfx() ? : "", syscall_name); + } else { + snprintf(func_name, sizeof(func_name), "__se_sys_%s", syscall_name); + } + + kprobe_opts.retprobe = OPTS_GET(opts, retprobe, false); + kprobe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); + + return bpf_program__attach_kprobe_opts(prog, func_name, &kprobe_opts); +} + /* Adapted from perf/util/string.c */ static bool glob_match(const char *str, const char *pat) { @@ -10391,6 +10289,12 @@ static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf char *func; int n; + *link = NULL; + + /* no auto-attach for SEC("kprobe") and SEC("kretprobe") */ + if (strcmp(prog->sec_name, "kprobe") == 0 || strcmp(prog->sec_name, "kretprobe") == 0) + return 0; + opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe/"); if (opts.retprobe) func_name = prog->sec_name + sizeof("kretprobe/") - 1; @@ -10414,6 +10318,27 @@ static int attach_kprobe(const struct bpf_program *prog, long cookie, struct bpf return libbpf_get_error(*link); } +static int attach_ksyscall(const struct bpf_program *prog, long cookie, struct bpf_link **link) +{ + LIBBPF_OPTS(bpf_ksyscall_opts, opts); + const char *syscall_name; + + *link = NULL; + + /* no auto-attach for SEC("ksyscall") and SEC("kretsyscall") */ + if (strcmp(prog->sec_name, "ksyscall") == 0 || strcmp(prog->sec_name, "kretsyscall") == 0) + return 0; + + opts.retprobe = str_has_pfx(prog->sec_name, "kretsyscall/"); + if (opts.retprobe) + syscall_name = prog->sec_name + sizeof("kretsyscall/") - 1; + else + syscall_name = prog->sec_name + sizeof("ksyscall/") - 1; + + *link = bpf_program__attach_ksyscall(prog, syscall_name, &opts); + return *link ? 0 : -errno; +} + static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, struct bpf_link **link) { LIBBPF_OPTS(bpf_kprobe_multi_opts, opts); @@ -10421,6 +10346,13 @@ static int attach_kprobe_multi(const struct bpf_program *prog, long cookie, stru char *pattern; int n; + *link = NULL; + + /* no auto-attach for SEC("kprobe.multi") and SEC("kretprobe.multi") */ + if (strcmp(prog->sec_name, "kprobe.multi") == 0 || + strcmp(prog->sec_name, "kretprobe.multi") == 0) + return 0; + opts.retprobe = str_has_pfx(prog->sec_name, "kretprobe.multi/"); if (opts.retprobe) spec = prog->sec_name + sizeof("kretprobe.multi/") - 1; @@ -10455,9 +10387,7 @@ static void gen_uprobe_legacy_event_name(char *buf, size_t buf_sz, static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, const char *binary_path, size_t offset) { - const char *file = "/sys/kernel/debug/tracing/uprobe_events"; - - return append_to_file(file, "%c:%s/%s %s:0x%zx", + return append_to_file(tracefs_uprobe_events(), "%c:%s/%s %s:0x%zx", retprobe ? 'r' : 'p', retprobe ? "uretprobes" : "uprobes", probe_name, binary_path, offset); @@ -10465,18 +10395,16 @@ static inline int add_uprobe_event_legacy(const char *probe_name, bool retprobe, static inline int remove_uprobe_event_legacy(const char *probe_name, bool retprobe) { - const char *file = "/sys/kernel/debug/tracing/uprobe_events"; - - return append_to_file(file, "-:%s/%s", retprobe ? "uretprobes" : "uprobes", probe_name); + return append_to_file(tracefs_uprobe_events(), "-:%s/%s", + retprobe ? "uretprobes" : "uprobes", probe_name); } static int determine_uprobe_perf_type_legacy(const char *probe_name, bool retprobe) { char file[512]; - snprintf(file, sizeof(file), - "/sys/kernel/debug/tracing/events/%s/%s/id", - retprobe ? "uretprobes" : "uprobes", probe_name); + snprintf(file, sizeof(file), "%s/events/%s/%s/id", + tracefs_path(), retprobe ? "uretprobes" : "uprobes", probe_name); return parse_uint_from_file(file, "%d\n"); } @@ -10495,9 +10423,10 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, } type = determine_uprobe_perf_type_legacy(probe_name, retprobe); if (type < 0) { + err = type; pr_warn("failed to determine legacy uprobe event id for %s:0x%zx: %d\n", binary_path, offset, err); - return type; + goto err_clean_legacy; } memset(&attr, 0, sizeof(attr)); @@ -10512,9 +10441,262 @@ static int perf_event_uprobe_open_legacy(const char *probe_name, bool retprobe, if (pfd < 0) { err = -errno; pr_warn("legacy uprobe perf_event_open() failed: %d\n", err); - return err; + goto err_clean_legacy; } return pfd; + +err_clean_legacy: + /* Clear the newly added legacy uprobe_event */ + remove_uprobe_event_legacy(probe_name, retprobe); + return err; +} + +/* Return next ELF section of sh_type after scn, or first of that type if scn is NULL. */ +static Elf_Scn *elf_find_next_scn_by_type(Elf *elf, int sh_type, Elf_Scn *scn) +{ + while ((scn = elf_nextscn(elf, scn)) != NULL) { + GElf_Shdr sh; + + if (!gelf_getshdr(scn, &sh)) + continue; + if (sh.sh_type == sh_type) + return scn; + } + return NULL; +} + +/* Find offset of function name in object specified by path. "name" matches + * symbol name or name@@LIB for library functions. + */ +static long elf_find_func_offset(const char *binary_path, const char *name) +{ + int fd, i, sh_types[2] = { SHT_DYNSYM, SHT_SYMTAB }; + bool is_shared_lib, is_name_qualified; + char errmsg[STRERR_BUFSIZE]; + long ret = -ENOENT; + size_t name_len; + GElf_Ehdr ehdr; + Elf *elf; + + fd = open(binary_path, O_RDONLY | O_CLOEXEC); + if (fd < 0) { + ret = -errno; + pr_warn("failed to open %s: %s\n", binary_path, + libbpf_strerror_r(ret, errmsg, sizeof(errmsg))); + return ret; + } + elf = elf_begin(fd, ELF_C_READ_MMAP, NULL); + if (!elf) { + pr_warn("elf: could not read elf from %s: %s\n", binary_path, elf_errmsg(-1)); + close(fd); + return -LIBBPF_ERRNO__FORMAT; + } + if (!gelf_getehdr(elf, &ehdr)) { + pr_warn("elf: failed to get ehdr from %s: %s\n", binary_path, elf_errmsg(-1)); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + /* for shared lib case, we do not need to calculate relative offset */ + is_shared_lib = ehdr.e_type == ET_DYN; + + name_len = strlen(name); + /* Does name specify "@@LIB"? */ + is_name_qualified = strstr(name, "@@") != NULL; + + /* Search SHT_DYNSYM, SHT_SYMTAB for symbol. This search order is used because if + * a binary is stripped, it may only have SHT_DYNSYM, and a fully-statically + * linked binary may not have SHT_DYMSYM, so absence of a section should not be + * reported as a warning/error. + */ + for (i = 0; i < ARRAY_SIZE(sh_types); i++) { + size_t nr_syms, strtabidx, idx; + Elf_Data *symbols = NULL; + Elf_Scn *scn = NULL; + int last_bind = -1; + const char *sname; + GElf_Shdr sh; + + scn = elf_find_next_scn_by_type(elf, sh_types[i], NULL); + if (!scn) { + pr_debug("elf: failed to find symbol table ELF sections in '%s'\n", + binary_path); + continue; + } + if (!gelf_getshdr(scn, &sh)) + continue; + strtabidx = sh.sh_link; + symbols = elf_getdata(scn, 0); + if (!symbols) { + pr_warn("elf: failed to get symbols for symtab section in '%s': %s\n", + binary_path, elf_errmsg(-1)); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } + nr_syms = symbols->d_size / sh.sh_entsize; + + for (idx = 0; idx < nr_syms; idx++) { + int curr_bind; + GElf_Sym sym; + Elf_Scn *sym_scn; + GElf_Shdr sym_sh; + + if (!gelf_getsym(symbols, idx, &sym)) + continue; + + if (GELF_ST_TYPE(sym.st_info) != STT_FUNC) + continue; + + sname = elf_strptr(elf, strtabidx, sym.st_name); + if (!sname) + continue; + + curr_bind = GELF_ST_BIND(sym.st_info); + + /* User can specify func, func@@LIB or func@@LIB_VERSION. */ + if (strncmp(sname, name, name_len) != 0) + continue; + /* ...but we don't want a search for "foo" to match 'foo2" also, so any + * additional characters in sname should be of the form "@@LIB". + */ + if (!is_name_qualified && sname[name_len] != '\0' && sname[name_len] != '@') + continue; + + if (ret >= 0) { + /* handle multiple matches */ + if (last_bind != STB_WEAK && curr_bind != STB_WEAK) { + /* Only accept one non-weak bind. */ + pr_warn("elf: ambiguous match for '%s', '%s' in '%s'\n", + sname, name, binary_path); + ret = -LIBBPF_ERRNO__FORMAT; + goto out; + } else if (curr_bind == STB_WEAK) { + /* already have a non-weak bind, and + * this is a weak bind, so ignore. + */ + continue; + } + } + + /* Transform symbol's virtual address (absolute for + * binaries and relative for shared libs) into file + * offset, which is what kernel is expecting for + * uprobe/uretprobe attachment. + * See Documentation/trace/uprobetracer.rst for more + * details. + * This is done by looking up symbol's containing + * section's header and using it's virtual address + * (sh_addr) and corresponding file offset (sh_offset) + * to transform sym.st_value (virtual address) into + * desired final file offset. + */ + sym_scn = elf_getscn(elf, sym.st_shndx); + if (!sym_scn) + continue; + if (!gelf_getshdr(sym_scn, &sym_sh)) + continue; + + ret = sym.st_value - sym_sh.sh_addr + sym_sh.sh_offset; + last_bind = curr_bind; + } + if (ret > 0) + break; + } + + if (ret > 0) { + pr_debug("elf: symbol address match for '%s' in '%s': 0x%lx\n", name, binary_path, + ret); + } else { + if (ret == 0) { + pr_warn("elf: '%s' is 0 in symtab for '%s': %s\n", name, binary_path, + is_shared_lib ? "should not be 0 in a shared library" : + "try using shared library path instead"); + ret = -ENOENT; + } else { + pr_warn("elf: failed to find symbol '%s' in '%s'\n", name, binary_path); + } + } +out: + elf_end(elf); + close(fd); + return ret; +} + +static const char *arch_specific_lib_paths(void) +{ + /* + * Based on https://packages.debian.org/sid/libc6. + * + * Assume that the traced program is built for the same architecture + * as libbpf, which should cover the vast majority of cases. + */ +#if defined(__x86_64__) + return "/lib/x86_64-linux-gnu"; +#elif defined(__i386__) + return "/lib/i386-linux-gnu"; +#elif defined(__s390x__) + return "/lib/s390x-linux-gnu"; +#elif defined(__s390__) + return "/lib/s390-linux-gnu"; +#elif defined(__arm__) && defined(__SOFTFP__) + return "/lib/arm-linux-gnueabi"; +#elif defined(__arm__) && !defined(__SOFTFP__) + return "/lib/arm-linux-gnueabihf"; +#elif defined(__aarch64__) + return "/lib/aarch64-linux-gnu"; +#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 64 + return "/lib/mips64el-linux-gnuabi64"; +#elif defined(__mips__) && defined(__MIPSEL__) && _MIPS_SZLONG == 32 + return "/lib/mipsel-linux-gnu"; +#elif defined(__powerpc64__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + return "/lib/powerpc64le-linux-gnu"; +#elif defined(__sparc__) && defined(__arch64__) + return "/lib/sparc64-linux-gnu"; +#elif defined(__riscv) && __riscv_xlen == 64 + return "/lib/riscv64-linux-gnu"; +#else + return NULL; +#endif +} + +/* Get full path to program/shared library. */ +static int resolve_full_path(const char *file, char *result, size_t result_sz) +{ + const char *search_paths[3] = {}; + int i; + + if (str_has_sfx(file, ".so") || strstr(file, ".so.")) { + search_paths[0] = getenv("LD_LIBRARY_PATH"); + search_paths[1] = "/usr/lib64:/usr/lib"; + search_paths[2] = arch_specific_lib_paths(); + } else { + search_paths[0] = getenv("PATH"); + search_paths[1] = "/usr/bin:/usr/sbin"; + } + + for (i = 0; i < ARRAY_SIZE(search_paths); i++) { + const char *s; + + if (!search_paths[i]) + continue; + for (s = search_paths[i]; s != NULL; s = strchr(s, ':')) { + char *next_path; + int seg_len; + + if (s[0] == ':') + s++; + next_path = strchr(s, ':'); + seg_len = next_path ? next_path - s : strlen(s); + if (!seg_len) + continue; + snprintf(result, result_sz, "%.*s/%s", seg_len, s, file); + /* ensure it is an executable file/link */ + if (access(result, R_OK | X_OK) < 0) + continue; + pr_debug("resolved '%s' to '%s'\n", file, result); + return 0; + } + } + return -ENOENT; } LIBBPF_API struct bpf_link * @@ -10524,10 +10706,12 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, { DECLARE_LIBBPF_OPTS(bpf_perf_event_opts, pe_opts); char errmsg[STRERR_BUFSIZE], *legacy_probe = NULL; + char full_binary_path[PATH_MAX]; struct bpf_link *link; size_t ref_ctr_off; int pfd, err; bool retprobe, legacy; + const char *func_name; if (!OPTS_VALID(opts, bpf_uprobe_opts)) return libbpf_err_ptr(-EINVAL); @@ -10536,12 +10720,35 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, ref_ctr_off = OPTS_GET(opts, ref_ctr_offset, 0); pe_opts.bpf_cookie = OPTS_GET(opts, bpf_cookie, 0); + if (!binary_path) + return libbpf_err_ptr(-EINVAL); + + if (!strchr(binary_path, '/')) { + err = resolve_full_path(binary_path, full_binary_path, + sizeof(full_binary_path)); + if (err) { + pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", + prog->name, binary_path, err); + return libbpf_err_ptr(err); + } + binary_path = full_binary_path; + } + func_name = OPTS_GET(opts, func_name, NULL); + if (func_name) { + long sym_off; + + sym_off = elf_find_func_offset(binary_path, func_name); + if (sym_off < 0) + return libbpf_err_ptr(sym_off); + func_offset += sym_off; + } + legacy = determine_uprobe_perf_type() < 0; if (!legacy) { pfd = perf_event_open_probe(true /* uprobe */, retprobe, binary_path, func_offset, pid, ref_ctr_off); } else { - char probe_name[512]; + char probe_name[PATH_MAX + 64]; if (ref_ctr_off) return libbpf_err_ptr(-EINVAL); @@ -10573,7 +10780,7 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, prog->name, retprobe ? "uretprobe" : "uprobe", binary_path, func_offset, libbpf_strerror_r(err, errmsg, sizeof(errmsg))); - goto err_out; + goto err_clean_legacy; } if (legacy) { struct bpf_link_perf *perf_link = container_of(link, struct bpf_link_perf, link); @@ -10583,10 +10790,68 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, perf_link->legacy_is_retprobe = retprobe; } return link; + +err_clean_legacy: + if (legacy) + remove_uprobe_event_legacy(legacy_probe, retprobe); err_out: free(legacy_probe); return libbpf_err_ptr(err); +} +/* Format of u[ret]probe section definition supporting auto-attach: + * u[ret]probe/binary:function[+offset] + * + * binary can be an absolute/relative path or a filename; the latter is resolved to a + * full binary path via bpf_program__attach_uprobe_opts. + * + * Specifying uprobe+ ensures we carry out strict matching; either "uprobe" must be + * specified (and auto-attach is not possible) or the above format is specified for + * auto-attach. + */ +static int attach_uprobe(const struct bpf_program *prog, long cookie, struct bpf_link **link) +{ + DECLARE_LIBBPF_OPTS(bpf_uprobe_opts, opts); + char *probe_type = NULL, *binary_path = NULL, *func_name = NULL; + int n, ret = -EINVAL; + long offset = 0; + + *link = NULL; + + n = sscanf(prog->sec_name, "%m[^/]/%m[^:]:%m[a-zA-Z0-9_.]+%li", + &probe_type, &binary_path, &func_name, &offset); + switch (n) { + case 1: + /* handle SEC("u[ret]probe") - format is valid, but auto-attach is impossible. */ + ret = 0; + break; + case 2: + pr_warn("prog '%s': section '%s' missing ':function[+offset]' specification\n", + prog->name, prog->sec_name); + break; + case 3: + case 4: + opts.retprobe = strcmp(probe_type, "uretprobe") == 0 || + strcmp(probe_type, "uretprobe.s") == 0; + if (opts.retprobe && offset != 0) { + pr_warn("prog '%s': uretprobes do not support offset specification\n", + prog->name); + break; + } + opts.func_name = func_name; + *link = bpf_program__attach_uprobe_opts(prog, -1, binary_path, offset, &opts); + ret = libbpf_get_error(*link); + break; + default: + pr_warn("prog '%s': invalid format of section definition '%s'\n", prog->name, + prog->sec_name); + break; + } + free(probe_type); + free(binary_path); + free(func_name); + + return ret; } struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, @@ -10599,15 +10864,96 @@ struct bpf_link *bpf_program__attach_uprobe(const struct bpf_program *prog, return bpf_program__attach_uprobe_opts(prog, pid, binary_path, func_offset, &opts); } +struct bpf_link *bpf_program__attach_usdt(const struct bpf_program *prog, + pid_t pid, const char *binary_path, + const char *usdt_provider, const char *usdt_name, + const struct bpf_usdt_opts *opts) +{ + char resolved_path[512]; + struct bpf_object *obj = prog->obj; + struct bpf_link *link; + __u64 usdt_cookie; + int err; + + if (!OPTS_VALID(opts, bpf_uprobe_opts)) + return libbpf_err_ptr(-EINVAL); + + if (bpf_program__fd(prog) < 0) { + pr_warn("prog '%s': can't attach BPF program w/o FD (did you load it?)\n", + prog->name); + return libbpf_err_ptr(-EINVAL); + } + + if (!binary_path) + return libbpf_err_ptr(-EINVAL); + + if (!strchr(binary_path, '/')) { + err = resolve_full_path(binary_path, resolved_path, sizeof(resolved_path)); + if (err) { + pr_warn("prog '%s': failed to resolve full path for '%s': %d\n", + prog->name, binary_path, err); + return libbpf_err_ptr(err); + } + binary_path = resolved_path; + } + + /* USDT manager is instantiated lazily on first USDT attach. It will + * be destroyed together with BPF object in bpf_object__close(). + */ + if (IS_ERR(obj->usdt_man)) + return libbpf_ptr(obj->usdt_man); + if (!obj->usdt_man) { + obj->usdt_man = usdt_manager_new(obj); + if (IS_ERR(obj->usdt_man)) + return libbpf_ptr(obj->usdt_man); + } + + usdt_cookie = OPTS_GET(opts, usdt_cookie, 0); + link = usdt_manager_attach_usdt(obj->usdt_man, prog, pid, binary_path, + usdt_provider, usdt_name, usdt_cookie); + err = libbpf_get_error(link); + if (err) + return libbpf_err_ptr(err); + return link; +} + +static int attach_usdt(const struct bpf_program *prog, long cookie, struct bpf_link **link) +{ + char *path = NULL, *provider = NULL, *name = NULL; + const char *sec_name; + int n, err; + + sec_name = bpf_program__section_name(prog); + if (strcmp(sec_name, "usdt") == 0) { + /* no auto-attach for just SEC("usdt") */ + *link = NULL; + return 0; + } + + n = sscanf(sec_name, "usdt/%m[^:]:%m[^:]:%m[^:]", &path, &provider, &name); + if (n != 3) { + pr_warn("invalid section '%s', expected SEC(\"usdt/::\")\n", + sec_name); + err = -EINVAL; + } else { + *link = bpf_program__attach_usdt(prog, -1 /* any process */, path, + provider, name, NULL); + err = libbpf_get_error(*link); + } + free(path); + free(provider); + free(name); + return err; +} + static int determine_tracepoint_id(const char *tp_category, const char *tp_name) { char file[PATH_MAX]; int ret; - ret = snprintf(file, sizeof(file), - "/sys/kernel/debug/tracing/events/%s/%s/id", - tp_category, tp_name); + ret = snprintf(file, sizeof(file), "%s/events/%s/%s/id", + tracefs_path(), tp_category, tp_name); if (ret < 0) return -errno; if (ret >= sizeof(file)) { @@ -10694,6 +11040,12 @@ static int attach_tp(const struct bpf_program *prog, long cookie, struct bpf_lin { char *sec_name, *tp_cat, *tp_name; + *link = NULL; + + /* no auto-attach for SEC("tp") or SEC("tracepoint") */ + if (strcmp(prog->sec_name, "tp") == 0 || strcmp(prog->sec_name, "tracepoint") == 0) + return 0; + sec_name = strdup(prog->sec_name); if (!sec_name) return -ENOMEM; @@ -10749,20 +11101,34 @@ struct bpf_link *bpf_program__attach_raw_tracepoint(const struct bpf_program *pr static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf_link **link) { static const char *const prefixes[] = { - "raw_tp/", - "raw_tracepoint/", - "raw_tp.w/", - "raw_tracepoint.w/", + "raw_tp", + "raw_tracepoint", + "raw_tp.w", + "raw_tracepoint.w", }; size_t i; const char *tp_name = NULL; + *link = NULL; + for (i = 0; i < ARRAY_SIZE(prefixes); i++) { - if (str_has_pfx(prog->sec_name, prefixes[i])) { - tp_name = prog->sec_name + strlen(prefixes[i]); - break; - } + size_t pfx_len; + + if (!str_has_pfx(prog->sec_name, prefixes[i])) + continue; + + pfx_len = strlen(prefixes[i]); + /* no auto-attach case of, e.g., SEC("raw_tp") */ + if (prog->sec_name[pfx_len] == '\0') + return 0; + + if (prog->sec_name[pfx_len] != '/') + continue; + + tp_name = prog->sec_name + pfx_len + 1; + break; } + if (!tp_name) { pr_warn("prog '%s': invalid section name '%s'\n", prog->name, prog->sec_name); @@ -10774,12 +11140,17 @@ static int attach_raw_tp(const struct bpf_program *prog, long cookie, struct bpf } /* Common logic for all BPF program types that attach to a btf_id */ -static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog) +static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *prog, + const struct bpf_trace_opts *opts) { + LIBBPF_OPTS(bpf_link_create_opts, link_opts); char errmsg[STRERR_BUFSIZE]; struct bpf_link *link; int prog_fd, pfd; + if (!OPTS_VALID(opts, bpf_trace_opts)) + return libbpf_err_ptr(-EINVAL); + prog_fd = bpf_program__fd(prog); if (prog_fd < 0) { pr_warn("prog '%s': can't attach before loaded\n", prog->name); @@ -10791,7 +11162,9 @@ static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *pro return libbpf_err_ptr(-ENOMEM); link->detach = &bpf_link__detach_fd; - pfd = bpf_raw_tracepoint_open(NULL, prog_fd); + /* libbpf is smart enough to redirect to BPF_RAW_TRACEPOINT_OPEN on old kernels */ + link_opts.tracing.cookie = OPTS_GET(opts, cookie, 0); + pfd = bpf_link_create(prog_fd, 0, bpf_program__expected_attach_type(prog), &link_opts); if (pfd < 0) { pfd = -errno; free(link); @@ -10800,17 +11173,23 @@ static struct bpf_link *bpf_program__attach_btf_id(const struct bpf_program *pro return libbpf_err_ptr(pfd); } link->fd = pfd; - return (struct bpf_link *)link; + return link; } struct bpf_link *bpf_program__attach_trace(const struct bpf_program *prog) { - return bpf_program__attach_btf_id(prog); + return bpf_program__attach_btf_id(prog, NULL); +} + +struct bpf_link *bpf_program__attach_trace_opts(const struct bpf_program *prog, + const struct bpf_trace_opts *opts) +{ + return bpf_program__attach_btf_id(prog, opts); } struct bpf_link *bpf_program__attach_lsm(const struct bpf_program *prog) { - return bpf_program__attach_btf_id(prog); + return bpf_program__attach_btf_id(prog, NULL); } static int attach_trace(const struct bpf_program *prog, long cookie, struct bpf_link **link) @@ -11031,6 +11410,9 @@ struct bpf_link *bpf_map__attach_struct_ops(const struct bpf_map *map) return link; } +typedef enum bpf_perf_event_ret (*bpf_perf_event_print_t)(struct perf_event_header *hdr, + void *private_data); + static enum bpf_perf_event_ret perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, void **copy_mem, size_t *copy_size, @@ -11079,12 +11461,6 @@ perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, return libbpf_err(ret); } -__attribute__((alias("perf_event_read_simple"))) -enum bpf_perf_event_ret -bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, - void **copy_mem, size_t *copy_size, - bpf_perf_event_print_t fn, void *private_data); - struct perf_buffer; struct perf_buffer_params { @@ -11218,12 +11594,11 @@ perf_buffer__open_cpu_buf(struct perf_buffer *pb, struct perf_event_attr *attr, static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, struct perf_buffer_params *p); -DEFAULT_VERSION(perf_buffer__new_v0_6_0, perf_buffer__new, LIBBPF_0.6.0) -struct perf_buffer *perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt, - perf_buffer_sample_fn sample_cb, - perf_buffer_lost_fn lost_cb, - void *ctx, - const struct perf_buffer_opts *opts) +struct perf_buffer *perf_buffer__new(int map_fd, size_t page_cnt, + perf_buffer_sample_fn sample_cb, + perf_buffer_lost_fn lost_cb, + void *ctx, + const struct perf_buffer_opts *opts) { struct perf_buffer_params p = {}; struct perf_event_attr attr = {}; @@ -11245,22 +11620,10 @@ struct perf_buffer *perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt, return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); } -COMPAT_VERSION(perf_buffer__new_deprecated, perf_buffer__new, LIBBPF_0.0.4) -struct perf_buffer *perf_buffer__new_deprecated(int map_fd, size_t page_cnt, - const struct perf_buffer_opts *opts) -{ - return perf_buffer__new_v0_6_0(map_fd, page_cnt, - opts ? opts->sample_cb : NULL, - opts ? opts->lost_cb : NULL, - opts ? opts->ctx : NULL, - NULL); -} - -DEFAULT_VERSION(perf_buffer__new_raw_v0_6_0, perf_buffer__new_raw, LIBBPF_0.6.0) -struct perf_buffer *perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt, - struct perf_event_attr *attr, - perf_buffer_event_fn event_cb, void *ctx, - const struct perf_buffer_raw_opts *opts) +struct perf_buffer *perf_buffer__new_raw(int map_fd, size_t page_cnt, + struct perf_event_attr *attr, + perf_buffer_event_fn event_cb, void *ctx, + const struct perf_buffer_raw_opts *opts) { struct perf_buffer_params p = {}; @@ -11280,20 +11643,6 @@ struct perf_buffer *perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt, return libbpf_ptr(__perf_buffer__new(map_fd, page_cnt, &p)); } -COMPAT_VERSION(perf_buffer__new_raw_deprecated, perf_buffer__new_raw, LIBBPF_0.0.4) -struct perf_buffer *perf_buffer__new_raw_deprecated(int map_fd, size_t page_cnt, - const struct perf_buffer_raw_opts *opts) -{ - LIBBPF_OPTS(perf_buffer_raw_opts, inner_opts, - .cpu_cnt = opts->cpu_cnt, - .cpus = opts->cpus, - .map_keys = opts->map_keys, - ); - - return perf_buffer__new_raw_v0_6_0(map_fd, page_cnt, opts->attr, - opts->event_cb, opts->ctx, &inner_opts); -} - static struct perf_buffer *__perf_buffer__new(int map_fd, size_t page_cnt, struct perf_buffer_params *p) { @@ -11554,6 +11903,22 @@ int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx) return cpu_buf->fd; } +int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, size_t *buf_size) +{ + struct perf_cpu_buf *cpu_buf; + + if (buf_idx >= pb->cpu_cnt) + return libbpf_err(-EINVAL); + + cpu_buf = pb->cpu_bufs[buf_idx]; + if (!cpu_buf) + return libbpf_err(-ENOENT); + + *buf = cpu_buf->base; + *buf_size = pb->mmap_size; + return 0; +} + /* * Consume data from perf ring buffer corresponding to slot *buf_idx* in * PERF_EVENT_ARRAY BPF map without waiting/polling. If there is no data to @@ -11595,254 +11960,6 @@ int perf_buffer__consume(struct perf_buffer *pb) return 0; } -struct bpf_prog_info_array_desc { - int array_offset; /* e.g. offset of jited_prog_insns */ - int count_offset; /* e.g. offset of jited_prog_len */ - int size_offset; /* > 0: offset of rec size, - * < 0: fix size of -size_offset - */ -}; - -static struct bpf_prog_info_array_desc bpf_prog_info_array_desc[] = { - [BPF_PROG_INFO_JITED_INSNS] = { - offsetof(struct bpf_prog_info, jited_prog_insns), - offsetof(struct bpf_prog_info, jited_prog_len), - -1, - }, - [BPF_PROG_INFO_XLATED_INSNS] = { - offsetof(struct bpf_prog_info, xlated_prog_insns), - offsetof(struct bpf_prog_info, xlated_prog_len), - -1, - }, - [BPF_PROG_INFO_MAP_IDS] = { - offsetof(struct bpf_prog_info, map_ids), - offsetof(struct bpf_prog_info, nr_map_ids), - -(int)sizeof(__u32), - }, - [BPF_PROG_INFO_JITED_KSYMS] = { - offsetof(struct bpf_prog_info, jited_ksyms), - offsetof(struct bpf_prog_info, nr_jited_ksyms), - -(int)sizeof(__u64), - }, - [BPF_PROG_INFO_JITED_FUNC_LENS] = { - offsetof(struct bpf_prog_info, jited_func_lens), - offsetof(struct bpf_prog_info, nr_jited_func_lens), - -(int)sizeof(__u32), - }, - [BPF_PROG_INFO_FUNC_INFO] = { - offsetof(struct bpf_prog_info, func_info), - offsetof(struct bpf_prog_info, nr_func_info), - offsetof(struct bpf_prog_info, func_info_rec_size), - }, - [BPF_PROG_INFO_LINE_INFO] = { - offsetof(struct bpf_prog_info, line_info), - offsetof(struct bpf_prog_info, nr_line_info), - offsetof(struct bpf_prog_info, line_info_rec_size), - }, - [BPF_PROG_INFO_JITED_LINE_INFO] = { - offsetof(struct bpf_prog_info, jited_line_info), - offsetof(struct bpf_prog_info, nr_jited_line_info), - offsetof(struct bpf_prog_info, jited_line_info_rec_size), - }, - [BPF_PROG_INFO_PROG_TAGS] = { - offsetof(struct bpf_prog_info, prog_tags), - offsetof(struct bpf_prog_info, nr_prog_tags), - -(int)sizeof(__u8) * BPF_TAG_SIZE, - }, - -}; - -static __u32 bpf_prog_info_read_offset_u32(struct bpf_prog_info *info, - int offset) -{ - __u32 *array = (__u32 *)info; - - if (offset >= 0) - return array[offset / sizeof(__u32)]; - return -(int)offset; -} - -static __u64 bpf_prog_info_read_offset_u64(struct bpf_prog_info *info, - int offset) -{ - __u64 *array = (__u64 *)info; - - if (offset >= 0) - return array[offset / sizeof(__u64)]; - return -(int)offset; -} - -static void bpf_prog_info_set_offset_u32(struct bpf_prog_info *info, int offset, - __u32 val) -{ - __u32 *array = (__u32 *)info; - - if (offset >= 0) - array[offset / sizeof(__u32)] = val; -} - -static void bpf_prog_info_set_offset_u64(struct bpf_prog_info *info, int offset, - __u64 val) -{ - __u64 *array = (__u64 *)info; - - if (offset >= 0) - array[offset / sizeof(__u64)] = val; -} - -struct bpf_prog_info_linear * -bpf_program__get_prog_info_linear(int fd, __u64 arrays) -{ - struct bpf_prog_info_linear *info_linear; - struct bpf_prog_info info = {}; - __u32 info_len = sizeof(info); - __u32 data_len = 0; - int i, err; - void *ptr; - - if (arrays >> BPF_PROG_INFO_LAST_ARRAY) - return libbpf_err_ptr(-EINVAL); - - /* step 1: get array dimensions */ - err = bpf_obj_get_info_by_fd(fd, &info, &info_len); - if (err) { - pr_debug("can't get prog info: %s", strerror(errno)); - return libbpf_err_ptr(-EFAULT); - } - - /* step 2: calculate total size of all arrays */ - for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { - bool include_array = (arrays & (1UL << i)) > 0; - struct bpf_prog_info_array_desc *desc; - __u32 count, size; - - desc = bpf_prog_info_array_desc + i; - - /* kernel is too old to support this field */ - if (info_len < desc->array_offset + sizeof(__u32) || - info_len < desc->count_offset + sizeof(__u32) || - (desc->size_offset > 0 && info_len < desc->size_offset)) - include_array = false; - - if (!include_array) { - arrays &= ~(1UL << i); /* clear the bit */ - continue; - } - - count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); - size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); - - data_len += count * size; - } - - /* step 3: allocate continuous memory */ - data_len = roundup(data_len, sizeof(__u64)); - info_linear = malloc(sizeof(struct bpf_prog_info_linear) + data_len); - if (!info_linear) - return libbpf_err_ptr(-ENOMEM); - - /* step 4: fill data to info_linear->info */ - info_linear->arrays = arrays; - memset(&info_linear->info, 0, sizeof(info)); - ptr = info_linear->data; - - for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { - struct bpf_prog_info_array_desc *desc; - __u32 count, size; - - if ((arrays & (1UL << i)) == 0) - continue; - - desc = bpf_prog_info_array_desc + i; - count = bpf_prog_info_read_offset_u32(&info, desc->count_offset); - size = bpf_prog_info_read_offset_u32(&info, desc->size_offset); - bpf_prog_info_set_offset_u32(&info_linear->info, - desc->count_offset, count); - bpf_prog_info_set_offset_u32(&info_linear->info, - desc->size_offset, size); - bpf_prog_info_set_offset_u64(&info_linear->info, - desc->array_offset, - ptr_to_u64(ptr)); - ptr += count * size; - } - - /* step 5: call syscall again to get required arrays */ - err = bpf_obj_get_info_by_fd(fd, &info_linear->info, &info_len); - if (err) { - pr_debug("can't get prog info: %s", strerror(errno)); - free(info_linear); - return libbpf_err_ptr(-EFAULT); - } - - /* step 6: verify the data */ - for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { - struct bpf_prog_info_array_desc *desc; - __u32 v1, v2; - - if ((arrays & (1UL << i)) == 0) - continue; - - desc = bpf_prog_info_array_desc + i; - v1 = bpf_prog_info_read_offset_u32(&info, desc->count_offset); - v2 = bpf_prog_info_read_offset_u32(&info_linear->info, - desc->count_offset); - if (v1 != v2) - pr_warn("%s: mismatch in element count\n", __func__); - - v1 = bpf_prog_info_read_offset_u32(&info, desc->size_offset); - v2 = bpf_prog_info_read_offset_u32(&info_linear->info, - desc->size_offset); - if (v1 != v2) - pr_warn("%s: mismatch in rec size\n", __func__); - } - - /* step 7: update info_len and data_len */ - info_linear->info_len = sizeof(struct bpf_prog_info); - info_linear->data_len = data_len; - - return info_linear; -} - -void bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear) -{ - int i; - - for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { - struct bpf_prog_info_array_desc *desc; - __u64 addr, offs; - - if ((info_linear->arrays & (1UL << i)) == 0) - continue; - - desc = bpf_prog_info_array_desc + i; - addr = bpf_prog_info_read_offset_u64(&info_linear->info, - desc->array_offset); - offs = addr - ptr_to_u64(info_linear->data); - bpf_prog_info_set_offset_u64(&info_linear->info, - desc->array_offset, offs); - } -} - -void bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear) -{ - int i; - - for (i = BPF_PROG_INFO_FIRST_ARRAY; i < BPF_PROG_INFO_LAST_ARRAY; ++i) { - struct bpf_prog_info_array_desc *desc; - __u64 addr, offs; - - if ((info_linear->arrays & (1UL << i)) == 0) - continue; - - desc = bpf_prog_info_array_desc + i; - offs = bpf_prog_info_read_offset_u64(&info_linear->info, - desc->array_offset); - addr = offs + ptr_to_u64(info_linear->data); - bpf_prog_info_set_offset_u64(&info_linear->info, - desc->array_offset, addr); - } -} - int bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, const char *attach_func_name) @@ -12211,7 +12328,7 @@ int bpf_object__attach_skeleton(struct bpf_object_skeleton *s) struct bpf_program *prog = *s->progs[i].prog; struct bpf_link **link = s->progs[i].link; - if (!prog->load) + if (!prog->autoload) continue; /* auto-attaching not supported for this program */ diff --git a/tools/lib/bpf/libbpf.h b/tools/lib/bpf/libbpf.h index 05dde85e19..61493c4cdd 100644 --- a/tools/lib/bpf/libbpf.h +++ b/tools/lib/bpf/libbpf.h @@ -51,6 +51,42 @@ enum libbpf_errno { LIBBPF_API int libbpf_strerror(int err, char *buf, size_t size); +/** + * @brief **libbpf_bpf_attach_type_str()** converts the provided attach type + * value into a textual representation. + * @param t The attach type. + * @return Pointer to a static string identifying the attach type. NULL is + * returned for unknown **bpf_attach_type** values. + */ +LIBBPF_API const char *libbpf_bpf_attach_type_str(enum bpf_attach_type t); + +/** + * @brief **libbpf_bpf_link_type_str()** converts the provided link type value + * into a textual representation. + * @param t The link type. + * @return Pointer to a static string identifying the link type. NULL is + * returned for unknown **bpf_link_type** values. + */ +LIBBPF_API const char *libbpf_bpf_link_type_str(enum bpf_link_type t); + +/** + * @brief **libbpf_bpf_map_type_str()** converts the provided map type value + * into a textual representation. + * @param t The map type. + * @return Pointer to a static string identifying the map type. NULL is + * returned for unknown **bpf_map_type** values. + */ +LIBBPF_API const char *libbpf_bpf_map_type_str(enum bpf_map_type t); + +/** + * @brief **libbpf_bpf_prog_type_str()** converts the provided program type + * value into a textual representation. + * @param t The program type. + * @return Pointer to a static string identifying the program type. NULL is + * returned for unknown **bpf_prog_type** values. + */ +LIBBPF_API const char *libbpf_bpf_prog_type_str(enum bpf_prog_type t); + enum libbpf_print_level { LIBBPF_WARN, LIBBPF_INFO, @@ -65,13 +101,8 @@ LIBBPF_API libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn); /* Hide internal to user */ struct bpf_object; -struct bpf_object_open_attr { - const char *file; - enum bpf_prog_type prog_type; -}; - struct bpf_object_open_opts { - /* size of this struct, for forward/backward compatiblity */ + /* size of this struct, for forward/backward compatibility */ size_t sz; /* object name override, if provided: * - for object open from file, this will override setting object @@ -82,21 +113,12 @@ struct bpf_object_open_opts { const char *object_name; /* parse map definitions non-strictly, allowing extra attributes/data */ bool relaxed_maps; - /* DEPRECATED: handle CO-RE relocations non-strictly, allowing failures. - * Value is ignored. Relocations always are processed non-strictly. - * Non-relocatable instructions are replaced with invalid ones to - * prevent accidental errors. - * */ - LIBBPF_DEPRECATED_SINCE(0, 6, "field has no effect") - bool relaxed_core_relocs; /* maps that set the 'pinning' attribute in their definition will have * their pin_path attribute set to a file in this directory, and be * auto-pinned to that path on load; defaults to "/sys/fs/bpf". */ const char *pin_root_path; - - LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__set_attach_target() on each individual bpf_program") - __u32 attach_prog_fd; + long :0; /* Additional kernel config content that augments and overrides * system Kconfig for CONFIG_xxx externs. */ @@ -179,20 +201,10 @@ LIBBPF_API struct bpf_object * bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, const struct bpf_object_open_opts *opts); -/* deprecated bpf_object__open variants */ -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open_mem() instead") -LIBBPF_API struct bpf_object * -bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, - const char *name); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open_file() instead") -LIBBPF_API struct bpf_object * -bpf_object__open_xattr(struct bpf_object_open_attr *attr); +/* Load/unload object into/from kernel */ +LIBBPF_API int bpf_object__load(struct bpf_object *obj); -enum libbpf_pin_type { - LIBBPF_PIN_NONE, - /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ - LIBBPF_PIN_BY_NAME, -}; +LIBBPF_API void bpf_object__close(struct bpf_object *object); /* pin_maps and unpin_maps can both be called with a NULL path, in which case * they will use the pin_path attribute of each map (and ignore all maps that @@ -206,20 +218,6 @@ LIBBPF_API int bpf_object__pin_programs(struct bpf_object *obj, LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj, const char *path); LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path); -LIBBPF_API void bpf_object__close(struct bpf_object *object); - -struct bpf_object_load_attr { - struct bpf_object *obj; - int log_level; - const char *target_btf_path; -}; - -/* Load/unload object into/from kernel */ -LIBBPF_API int bpf_object__load(struct bpf_object *obj); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__load() instead") -LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr); -LIBBPF_DEPRECATED_SINCE(0, 6, "bpf_object__unload() is deprecated, use bpf_object__close() instead") -LIBBPF_API int bpf_object__unload(struct bpf_object *obj); LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj); LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj); @@ -229,29 +227,10 @@ struct btf; LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj); LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__find_program_by_name() instead") -LIBBPF_API struct bpf_program * -bpf_object__find_program_by_title(const struct bpf_object *obj, - const char *title); LIBBPF_API struct bpf_program * bpf_object__find_program_by_name(const struct bpf_object *obj, const char *name); -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "track bpf_objects in application code instead") -struct bpf_object *bpf_object__next(struct bpf_object *prev); -#define bpf_object__for_each_safe(pos, tmp) \ - for ((pos) = bpf_object__next(NULL), \ - (tmp) = bpf_object__next(pos); \ - (pos) != NULL; \ - (pos) = (tmp), (tmp) = bpf_object__next(tmp)) - -typedef void (*bpf_object_clear_priv_t)(struct bpf_object *, void *); -LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") -LIBBPF_API int bpf_object__set_priv(struct bpf_object *obj, void *priv, - bpf_object_clear_priv_t clear_priv); -LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") -LIBBPF_API void *bpf_object__priv(const struct bpf_object *prog); - LIBBPF_API int libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, enum bpf_attach_type *expected_attach_type); @@ -262,9 +241,7 @@ LIBBPF_API int libbpf_find_vmlinux_btf_id(const char *name, /* Accessors of bpf_program */ struct bpf_program; -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_program() instead") -struct bpf_program *bpf_program__next(struct bpf_program *prog, - const struct bpf_object *obj); + LIBBPF_API struct bpf_program * bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prog); @@ -273,33 +250,17 @@ bpf_object__next_program(const struct bpf_object *obj, struct bpf_program *prog) (pos) != NULL; \ (pos) = bpf_object__next_program((obj), (pos))) -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__prev_program() instead") -struct bpf_program *bpf_program__prev(struct bpf_program *prog, - const struct bpf_object *obj); LIBBPF_API struct bpf_program * bpf_object__prev_program(const struct bpf_object *obj, struct bpf_program *prog); -typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *); - -LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") -LIBBPF_API int bpf_program__set_priv(struct bpf_program *prog, void *priv, - bpf_program_clear_priv_t clear_priv); -LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") -LIBBPF_API void *bpf_program__priv(const struct bpf_program *prog); LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog, __u32 ifindex); LIBBPF_API const char *bpf_program__name(const struct bpf_program *prog); LIBBPF_API const char *bpf_program__section_name(const struct bpf_program *prog); -LIBBPF_API LIBBPF_DEPRECATED("BPF program title is confusing term; please use bpf_program__section_name() instead") -const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy); LIBBPF_API bool bpf_program__autoload(const struct bpf_program *prog); LIBBPF_API int bpf_program__set_autoload(struct bpf_program *prog, bool autoload); -/* returns program size in bytes */ -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__insn_cnt() instead") -LIBBPF_API size_t bpf_program__size(const struct bpf_program *prog); - struct bpf_insn; /** @@ -323,6 +284,24 @@ struct bpf_insn; * different. */ LIBBPF_API const struct bpf_insn *bpf_program__insns(const struct bpf_program *prog); + +/** + * @brief **bpf_program__set_insns()** can set BPF program's underlying + * BPF instructions. + * + * WARNING: This is a very advanced libbpf API and users need to know + * what they are doing. This should be used from prog_prepare_load_fn + * callback only. + * + * @param prog BPF program for which to return instructions + * @param new_insns a pointer to an array of BPF instructions + * @param new_insn_cnt number of `struct bpf_insn`'s that form + * specified BPF program + * @return 0, on success; negative error code, otherwise + */ +LIBBPF_API int bpf_program__set_insns(struct bpf_program *prog, + struct bpf_insn *new_insns, size_t new_insn_cnt); + /** * @brief **bpf_program__insn_cnt()** returns number of `struct bpf_insn`'s * that form specified BPF program. @@ -334,17 +313,7 @@ LIBBPF_API const struct bpf_insn *bpf_program__insns(const struct bpf_program *p */ LIBBPF_API size_t bpf_program__insn_cnt(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 6, "use bpf_object__load() instead") -LIBBPF_API int bpf_program__load(struct bpf_program *prog, const char *license, __u32 kern_version); LIBBPF_API int bpf_program__fd(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated") -LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog, - const char *path, - int instance); -LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated") -LIBBPF_API int bpf_program__unpin_instance(struct bpf_program *prog, - const char *path, - int instance); /** * @brief **bpf_program__pin()** pins the BPF program to a file @@ -378,7 +347,31 @@ struct bpf_link; LIBBPF_API struct bpf_link *bpf_link__open(const char *path); LIBBPF_API int bpf_link__fd(const struct bpf_link *link); LIBBPF_API const char *bpf_link__pin_path(const struct bpf_link *link); +/** + * @brief **bpf_link__pin()** pins the BPF link to a file + * in the BPF FS specified by a path. This increments the links + * reference count, allowing it to stay loaded after the process + * which loaded it has exited. + * + * @param link BPF link to pin, must already be loaded + * @param path file path in a BPF file system + * @return 0, on success; negative error code, otherwise + */ + LIBBPF_API int bpf_link__pin(struct bpf_link *link, const char *path); + +/** + * @brief **bpf_link__unpin()** unpins the BPF link from a file + * in the BPFFS specified by a path. This decrements the links + * reference count. + * + * The file pinning the BPF link can also be unlinked by a different + * process in which case this function will return an error. + * + * @param prog BPF program to unpin + * @param path file path to the pin in a BPF file system + * @return 0, on success; negative error code, otherwise + */ LIBBPF_API int bpf_link__unpin(struct bpf_link *link); LIBBPF_API int bpf_link__update_program(struct bpf_link *link, struct bpf_program *prog); @@ -386,6 +379,22 @@ LIBBPF_API void bpf_link__disconnect(struct bpf_link *link); LIBBPF_API int bpf_link__detach(struct bpf_link *link); LIBBPF_API int bpf_link__destroy(struct bpf_link *link); +/** + * @brief **bpf_program__attach()** is a generic function for attaching + * a BPF program based on auto-detection of program type, attach type, + * and extra paremeters, where applicable. + * + * @param prog BPF program to attach + * @return Reference to the newly created BPF link; or NULL is returned on error, + * error code is stored in errno + * + * This is supported for: + * - kprobe/kretprobe (depends on SEC() definition) + * - uprobe/uretprobe (depends on SEC() definition) + * - tracepoint + * - raw tracepoint + * - tracing programs (typed raw TP/fentry/fexit/fmod_ret) + */ LIBBPF_API struct bpf_link * bpf_program__attach(const struct bpf_program *prog); @@ -448,6 +457,52 @@ bpf_program__attach_kprobe_multi_opts(const struct bpf_program *prog, const char *pattern, const struct bpf_kprobe_multi_opts *opts); +struct bpf_ksyscall_opts { + /* size of this struct, for forward/backward compatiblity */ + size_t sz; + /* custom user-provided value fetchable through bpf_get_attach_cookie() */ + __u64 bpf_cookie; + /* attach as return probe? */ + bool retprobe; + size_t :0; +}; +#define bpf_ksyscall_opts__last_field retprobe + +/** + * @brief **bpf_program__attach_ksyscall()** attaches a BPF program + * to kernel syscall handler of a specified syscall. Optionally it's possible + * to request to install retprobe that will be triggered at syscall exit. It's + * also possible to associate BPF cookie (though options). + * + * Libbpf automatically will determine correct full kernel function name, + * which depending on system architecture and kernel version/configuration + * could be of the form ___sys_ or __se_sys_, and will + * attach specified program using kprobe/kretprobe mechanism. + * + * **bpf_program__attach_ksyscall()** is an API counterpart of declarative + * **SEC("ksyscall/")** annotation of BPF programs. + * + * At the moment **SEC("ksyscall")** and **bpf_program__attach_ksyscall()** do + * not handle all the calling convention quirks for mmap(), clone() and compat + * syscalls. It also only attaches to "native" syscall interfaces. If host + * system supports compat syscalls or defines 32-bit syscalls in 64-bit + * kernel, such syscall interfaces won't be attached to by libbpf. + * + * These limitations may or may not change in the future. Therefore it is + * recommended to use SEC("kprobe") for these syscalls or if working with + * compat and 32-bit interfaces is required. + * + * @param prog BPF program to attach + * @param syscall_name Symbolic name of the syscall (e.g., "bpf") + * @param opts Additional options (see **struct bpf_ksyscall_opts**) + * @return Reference to the newly created BPF link; or NULL is returned on + * error, error code is stored in errno + */ +LIBBPF_API struct bpf_link * +bpf_program__attach_ksyscall(const struct bpf_program *prog, + const char *syscall_name, + const struct bpf_ksyscall_opts *opts); + struct bpf_uprobe_opts { /* size of this struct, for forward/backward compatiblity */ size_t sz; @@ -459,9 +514,17 @@ struct bpf_uprobe_opts { __u64 bpf_cookie; /* uprobe is return probe, invoked at function return time */ bool retprobe; + /* Function name to attach to. Could be an unqualified ("abc") or library-qualified + * "abc@LIBXYZ" name. To specify function entry, func_name should be set while + * func_offset argument to bpf_prog__attach_uprobe_opts() should be 0. To trace an + * offset within a function, specify func_name and use func_offset argument to specify + * offset within the function. Shared library functions must specify the shared library + * binary_path. + */ + const char *func_name; size_t :0; }; -#define bpf_uprobe_opts__last_field retprobe +#define bpf_uprobe_opts__last_field func_name /** * @brief **bpf_program__attach_uprobe()** attaches a BPF program @@ -503,6 +566,37 @@ bpf_program__attach_uprobe_opts(const struct bpf_program *prog, pid_t pid, const char *binary_path, size_t func_offset, const struct bpf_uprobe_opts *opts); +struct bpf_usdt_opts { + /* size of this struct, for forward/backward compatibility */ + size_t sz; + /* custom user-provided value accessible through usdt_cookie() */ + __u64 usdt_cookie; + size_t :0; +}; +#define bpf_usdt_opts__last_field usdt_cookie + +/** + * @brief **bpf_program__attach_usdt()** is just like + * bpf_program__attach_uprobe_opts() except it covers USDT (User-space + * Statically Defined Tracepoint) attachment, instead of attaching to + * user-space function entry or exit. + * + * @param prog BPF program to attach + * @param pid Process ID to attach the uprobe to, 0 for self (own process), + * -1 for all processes + * @param binary_path Path to binary that contains provided USDT probe + * @param usdt_provider USDT provider name + * @param usdt_name USDT probe name + * @param opts Options for altering program attachment + * @return Reference to the newly created BPF link; or NULL is returned on error, + * error code is stored in errno + */ +LIBBPF_API struct bpf_link * +bpf_program__attach_usdt(const struct bpf_program *prog, + pid_t pid, const char *binary_path, + const char *usdt_provider, const char *usdt_name, + const struct bpf_usdt_opts *opts); + struct bpf_tracepoint_opts { /* size of this struct, for forward/backward compatiblity */ size_t sz; @@ -524,8 +618,20 @@ bpf_program__attach_tracepoint_opts(const struct bpf_program *prog, LIBBPF_API struct bpf_link * bpf_program__attach_raw_tracepoint(const struct bpf_program *prog, const char *tp_name); + +struct bpf_trace_opts { + /* size of this struct, for forward/backward compatibility */ + size_t sz; + /* custom user-provided value fetchable through bpf_get_attach_cookie() */ + __u64 cookie; +}; +#define bpf_trace_opts__last_field cookie + LIBBPF_API struct bpf_link * bpf_program__attach_trace(const struct bpf_program *prog); +LIBBPF_API struct bpf_link * +bpf_program__attach_trace_opts(const struct bpf_program *prog, const struct bpf_trace_opts *opts); + LIBBPF_API struct bpf_link * bpf_program__attach_lsm(const struct bpf_program *prog); LIBBPF_API struct bpf_link * @@ -553,106 +659,38 @@ LIBBPF_API struct bpf_link * bpf_program__attach_iter(const struct bpf_program *prog, const struct bpf_iter_attach_opts *opts); -/* - * Libbpf allows callers to adjust BPF programs before being loaded - * into kernel. One program in an object file can be transformed into - * multiple variants to be attached to different hooks. - * - * bpf_program_prep_t, bpf_program__set_prep and bpf_program__nth_fd - * form an API for this purpose. - * - * - bpf_program_prep_t: - * Defines a 'preprocessor', which is a caller defined function - * passed to libbpf through bpf_program__set_prep(), and will be - * called before program is loaded. The processor should adjust - * the program one time for each instance according to the instance id - * passed to it. - * - * - bpf_program__set_prep: - * Attaches a preprocessor to a BPF program. The number of instances - * that should be created is also passed through this function. - * - * - bpf_program__nth_fd: - * After the program is loaded, get resulting FD of a given instance - * of the BPF program. - * - * If bpf_program__set_prep() is not used, the program would be loaded - * without adjustment during bpf_object__load(). The program has only - * one instance. In this case bpf_program__fd(prog) is equal to - * bpf_program__nth_fd(prog, 0). - */ -struct bpf_prog_prep_result { - /* - * If not NULL, load new instruction array. - * If set to NULL, don't load this instance. - */ - struct bpf_insn *new_insn_ptr; - int new_insn_cnt; - - /* If not NULL, result FD is written to it. */ - int *pfd; -}; - -/* - * Parameters of bpf_program_prep_t: - * - prog: The bpf_program being loaded. - * - n: Index of instance being generated. - * - insns: BPF instructions array. - * - insns_cnt:Number of instructions in insns. - * - res: Output parameter, result of transformation. - * - * Return value: - * - Zero: pre-processing success. - * - Non-zero: pre-processing error, stop loading. - */ -typedef int (*bpf_program_prep_t)(struct bpf_program *prog, int n, - struct bpf_insn *insns, int insns_cnt, - struct bpf_prog_prep_result *res); - -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_program__insns() for getting bpf_program instructions") -LIBBPF_API int bpf_program__set_prep(struct bpf_program *prog, int nr_instance, - bpf_program_prep_t prep); - -LIBBPF_DEPRECATED_SINCE(0, 7, "multi-instance bpf_program support is deprecated") -LIBBPF_API int bpf_program__nth_fd(const struct bpf_program *prog, int n); - -/* - * Adjust type of BPF program. Default is kprobe. - */ -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_socket_filter(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_tracepoint(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_raw_tracepoint(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_kprobe(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_lsm(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_sched_cls(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__set_type() instead") -LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog); - LIBBPF_API enum bpf_prog_type bpf_program__type(const struct bpf_program *prog); -LIBBPF_API void bpf_program__set_type(struct bpf_program *prog, - enum bpf_prog_type type); + +/** + * @brief **bpf_program__set_type()** sets the program + * type of the passed BPF program. + * @param prog BPF program to set the program type for + * @param type program type to set the BPF map to have + * @return error code; or 0 if no error. An error occurs + * if the object is already loaded. + * + * This must be called before the BPF object is loaded, + * otherwise it has no effect and an error is returned. + */ +LIBBPF_API int bpf_program__set_type(struct bpf_program *prog, + enum bpf_prog_type type); LIBBPF_API enum bpf_attach_type bpf_program__expected_attach_type(const struct bpf_program *prog); -LIBBPF_API void + +/** + * @brief **bpf_program__set_expected_attach_type()** sets the + * attach type of the passed BPF program. This is used for + * auto-detection of attachment when programs are loaded. + * @param prog BPF program to set the attach type for + * @param type attach type to set the BPF map to have + * @return error code; or 0 if no error. An error occurs + * if the object is already loaded. + * + * This must be called before the BPF object is loaded, + * otherwise it has no effect and an error is returned. + */ +LIBBPF_API int bpf_program__set_expected_attach_type(struct bpf_program *prog, enum bpf_attach_type type); @@ -668,51 +706,21 @@ LIBBPF_API int bpf_program__set_log_level(struct bpf_program *prog, __u32 log_le LIBBPF_API const char *bpf_program__log_buf(const struct bpf_program *prog, size_t *log_size); LIBBPF_API int bpf_program__set_log_buf(struct bpf_program *prog, char *log_buf, size_t log_size); +/** + * @brief **bpf_program__set_attach_target()** sets BTF-based attach target + * for supported BPF program types: + * - BTF-aware raw tracepoints (tp_btf); + * - fentry/fexit/fmod_ret; + * - lsm; + * - freplace. + * @param prog BPF program to set the attach type for + * @param type attach type to set the BPF map to have + * @return error code; or 0 if no error occurred. + */ LIBBPF_API int bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, const char *attach_func_name); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_socket_filter(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_tracepoint(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_raw_tracepoint(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_kprobe(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_lsm(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_struct_ops(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_extension(const struct bpf_program *prog); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_program__type() instead") -LIBBPF_API bool bpf_program__is_sk_lookup(const struct bpf_program *prog); - -/* - * No need for __attribute__((packed)), all members of 'bpf_map_def' - * are all aligned. In addition, using __attribute__((packed)) - * would trigger a -Wpacked warning message, and lead to an error - * if -Werror is set. - */ -struct bpf_map_def { - unsigned int type; - unsigned int key_size; - unsigned int value_size; - unsigned int max_entries; - unsigned int map_flags; -}; - /** * @brief **bpf_object__find_map_by_name()** returns BPF map of * the given name, if it exists within the passed BPF object @@ -727,16 +735,6 @@ bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name); LIBBPF_API int bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name); -/* - * Get bpf_map through the offset of corresponding struct bpf_map_def - * in the BPF object file. - */ -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__find_map_by_name() instead") -struct bpf_map * -bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset); - -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__next_map() instead") -struct bpf_map *bpf_map__next(const struct bpf_map *map, const struct bpf_object *obj); LIBBPF_API struct bpf_map * bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *map); @@ -746,11 +744,31 @@ bpf_object__next_map(const struct bpf_object *obj, const struct bpf_map *map); (pos) = bpf_object__next_map((obj), (pos))) #define bpf_map__for_each bpf_object__for_each_map -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__prev_map() instead") -struct bpf_map *bpf_map__prev(const struct bpf_map *map, const struct bpf_object *obj); LIBBPF_API struct bpf_map * bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map); +/** + * @brief **bpf_map__set_autocreate()** sets whether libbpf has to auto-create + * BPF map during BPF object load phase. + * @param map the BPF map instance + * @param autocreate whether to create BPF map during BPF object load + * @return 0 on success; -EBUSY if BPF object was already loaded + * + * **bpf_map__set_autocreate()** allows to opt-out from libbpf auto-creating + * BPF map. By default, libbpf will attempt to create every single BPF map + * defined in BPF object file using BPF_MAP_CREATE command of bpf() syscall + * and fill in map FD in BPF instructions. + * + * This API allows to opt-out of this process for specific map instance. This + * can be useful if host kernel doesn't support such BPF map type or used + * combination of flags and user application wants to avoid creating such + * a map in the first place. User is still responsible to make sure that their + * BPF-side code that expects to use such missing BPF map is recognized by BPF + * verifier as dead code, otherwise BPF verifier will reject such BPF program. + */ +LIBBPF_API int bpf_map__set_autocreate(struct bpf_map *map, bool autocreate); +LIBBPF_API bool bpf_map__autocreate(const struct bpf_map *map); + /** * @brief **bpf_map__fd()** gets the file descriptor of the passed * BPF map @@ -759,9 +777,6 @@ bpf_object__prev_map(const struct bpf_object *obj, const struct bpf_map *map); */ LIBBPF_API int bpf_map__fd(const struct bpf_map *map); LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd); -/* get map definition */ -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 8, "use appropriate getters or setters instead") -const struct bpf_map_def *bpf_map__def(const struct bpf_map *map); /* get map name */ LIBBPF_API const char *bpf_map__name(const struct bpf_map *map); /* get/set map type */ @@ -770,8 +785,6 @@ LIBBPF_API int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type); /* get/set map size (max_entries) */ LIBBPF_API __u32 bpf_map__max_entries(const struct bpf_map *map); LIBBPF_API int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_map__set_max_entries() instead") -LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries); /* get/set map flags */ LIBBPF_API __u32 bpf_map__map_flags(const struct bpf_map *map); LIBBPF_API int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags); @@ -794,17 +807,9 @@ LIBBPF_API int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex); LIBBPF_API __u64 bpf_map__map_extra(const struct bpf_map *map); LIBBPF_API int bpf_map__set_map_extra(struct bpf_map *map, __u64 map_extra); -typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *); -LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") -LIBBPF_API int bpf_map__set_priv(struct bpf_map *map, void *priv, - bpf_map_clear_priv_t clear_priv); -LIBBPF_DEPRECATED_SINCE(0, 7, "storage via set_priv/priv is deprecated") -LIBBPF_API void *bpf_map__priv(const struct bpf_map *map); LIBBPF_API int bpf_map__set_initial_value(struct bpf_map *map, const void *data, size_t size); LIBBPF_API const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_map__type() instead") -LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map); /** * @brief **bpf_map__is_internal()** tells the caller whether or not the @@ -824,63 +829,108 @@ LIBBPF_API int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd); LIBBPF_API struct bpf_map *bpf_map__inner_map(struct bpf_map *map); /** - * @brief **libbpf_get_error()** extracts the error code from the passed - * pointer - * @param ptr pointer returned from libbpf API function - * @return error code; or 0 if no error occured + * @brief **bpf_map__lookup_elem()** allows to lookup BPF map value + * corresponding to provided key. + * @param map BPF map to lookup element in + * @param key pointer to memory containing bytes of the key used for lookup + * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** + * @param value pointer to memory in which looked up value will be stored + * @param value_sz size in byte of value data memory; it has to match BPF map + * definition's **value_size**. For per-CPU BPF maps value size has to be + * a product of BPF map value size and number of possible CPUs in the system + * (could be fetched with **libbpf_num_possible_cpus()**). Note also that for + * per-CPU values value size has to be aligned up to closest 8 bytes for + * alignment reasons, so expected size is: `round_up(value_size, 8) + * * libbpf_num_possible_cpus()`. + * @flags extra flags passed to kernel for this operation + * @return 0, on success; negative error, otherwise * - * Many libbpf API functions which return pointers have logic to encode error - * codes as pointers, and do not return NULL. Meaning **libbpf_get_error()** - * should be used on the return value from these functions immediately after - * calling the API function, with no intervening calls that could clobber the - * `errno` variable. Consult the individual functions documentation to verify - * if this logic applies should be used. - * - * For these API functions, if `libbpf_set_strict_mode(LIBBPF_STRICT_CLEAN_PTRS)` - * is enabled, NULL is returned on error instead. - * - * If ptr is NULL, then errno should be already set by the failing - * API, because libbpf never returns NULL on success and it now always - * sets errno on error. - * - * Example usage: - * - * struct perf_buffer *pb; - * - * pb = perf_buffer__new(bpf_map__fd(obj->maps.events), PERF_BUFFER_PAGES, &opts); - * err = libbpf_get_error(pb); - * if (err) { - * pb = NULL; - * fprintf(stderr, "failed to open perf buffer: %d\n", err); - * goto cleanup; - * } + * **bpf_map__lookup_elem()** is high-level equivalent of + * **bpf_map_lookup_elem()** API with added check for key and value size. */ -LIBBPF_API long libbpf_get_error(const void *ptr); +LIBBPF_API int bpf_map__lookup_elem(const struct bpf_map *map, + const void *key, size_t key_sz, + void *value, size_t value_sz, __u64 flags); -struct bpf_prog_load_attr { - const char *file; - enum bpf_prog_type prog_type; - enum bpf_attach_type expected_attach_type; - int ifindex; - int log_level; - int prog_flags; -}; +/** + * @brief **bpf_map__update_elem()** allows to insert or update value in BPF + * map that corresponds to provided key. + * @param map BPF map to insert to or update element in + * @param key pointer to memory containing bytes of the key + * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** + * @param value pointer to memory containing bytes of the value + * @param value_sz size in byte of value data memory; it has to match BPF map + * definition's **value_size**. For per-CPU BPF maps value size has to be + * a product of BPF map value size and number of possible CPUs in the system + * (could be fetched with **libbpf_num_possible_cpus()**). Note also that for + * per-CPU values value size has to be aligned up to closest 8 bytes for + * alignment reasons, so expected size is: `round_up(value_size, 8) + * * libbpf_num_possible_cpus()`. + * @flags extra flags passed to kernel for this operation + * @return 0, on success; negative error, otherwise + * + * **bpf_map__update_elem()** is high-level equivalent of + * **bpf_map_update_elem()** API with added check for key and value size. + */ +LIBBPF_API int bpf_map__update_elem(const struct bpf_map *map, + const void *key, size_t key_sz, + const void *value, size_t value_sz, __u64 flags); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_object__open() and bpf_object__load() instead") -LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, - struct bpf_object **pobj, int *prog_fd); -LIBBPF_DEPRECATED_SINCE(0, 7, "use bpf_object__open() and bpf_object__load() instead") -LIBBPF_API int bpf_prog_load_deprecated(const char *file, enum bpf_prog_type type, - struct bpf_object **pobj, int *prog_fd); +/** + * @brief **bpf_map__delete_elem()** allows to delete element in BPF map that + * corresponds to provided key. + * @param map BPF map to delete element from + * @param key pointer to memory containing bytes of the key + * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** + * @flags extra flags passed to kernel for this operation + * @return 0, on success; negative error, otherwise + * + * **bpf_map__delete_elem()** is high-level equivalent of + * **bpf_map_delete_elem()** API with added check for key size. + */ +LIBBPF_API int bpf_map__delete_elem(const struct bpf_map *map, + const void *key, size_t key_sz, __u64 flags); -/* XDP related API */ -struct xdp_link_info { - __u32 prog_id; - __u32 drv_prog_id; - __u32 hw_prog_id; - __u32 skb_prog_id; - __u8 attach_mode; -}; +/** + * @brief **bpf_map__lookup_and_delete_elem()** allows to lookup BPF map value + * corresponding to provided key and atomically delete it afterwards. + * @param map BPF map to lookup element in + * @param key pointer to memory containing bytes of the key used for lookup + * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** + * @param value pointer to memory in which looked up value will be stored + * @param value_sz size in byte of value data memory; it has to match BPF map + * definition's **value_size**. For per-CPU BPF maps value size has to be + * a product of BPF map value size and number of possible CPUs in the system + * (could be fetched with **libbpf_num_possible_cpus()**). Note also that for + * per-CPU values value size has to be aligned up to closest 8 bytes for + * alignment reasons, so expected size is: `round_up(value_size, 8) + * * libbpf_num_possible_cpus()`. + * @flags extra flags passed to kernel for this operation + * @return 0, on success; negative error, otherwise + * + * **bpf_map__lookup_and_delete_elem()** is high-level equivalent of + * **bpf_map_lookup_and_delete_elem()** API with added check for key and value size. + */ +LIBBPF_API int bpf_map__lookup_and_delete_elem(const struct bpf_map *map, + const void *key, size_t key_sz, + void *value, size_t value_sz, __u64 flags); + +/** + * @brief **bpf_map__get_next_key()** allows to iterate BPF map keys by + * fetching next key that follows current key. + * @param map BPF map to fetch next key from + * @param cur_key pointer to memory containing bytes of current key or NULL to + * fetch the first key + * @param next_key pointer to memory to write next key into + * @param key_sz size in bytes of key data, needs to match BPF map definition's **key_size** + * @return 0, on success; -ENOENT if **cur_key** is the last key in BPF map; + * negative error, otherwise + * + * **bpf_map__get_next_key()** is high-level equivalent of + * **bpf_map_get_next_key()** API with added check for key size. + */ +LIBBPF_API int bpf_map__get_next_key(const struct bpf_map *map, + const void *cur_key, void *next_key, size_t key_sz); struct bpf_xdp_set_link_opts { size_t sz; @@ -889,17 +939,6 @@ struct bpf_xdp_set_link_opts { }; #define bpf_xdp_set_link_opts__last_field old_fd -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_attach() instead") -LIBBPF_API int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_attach() instead") -LIBBPF_API int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags, - const struct bpf_xdp_set_link_opts *opts); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_query_id() instead") -LIBBPF_API int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags); -LIBBPF_DEPRECATED_SINCE(0, 8, "use bpf_xdp_query() instead") -LIBBPF_API int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, - size_t info_size, __u32 flags); - struct bpf_xdp_attach_opts { size_t sz; int old_prog_fd; @@ -998,17 +1037,7 @@ typedef void (*perf_buffer_lost_fn)(void *ctx, int cpu, __u64 cnt); /* common use perf buffer options */ struct perf_buffer_opts { - union { - size_t sz; - struct { /* DEPRECATED: will be removed in v1.0 */ - /* if specified, sample_cb is called for each sample */ - perf_buffer_sample_fn sample_cb; - /* if specified, lost_cb is called for each batch of lost samples */ - perf_buffer_lost_fn lost_cb; - /* ctx is provided to sample_cb and lost_cb */ - void *ctx; - }; - }; + size_t sz; }; #define perf_buffer_opts__last_field sz @@ -1029,21 +1058,6 @@ perf_buffer__new(int map_fd, size_t page_cnt, perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx, const struct perf_buffer_opts *opts); -LIBBPF_API struct perf_buffer * -perf_buffer__new_v0_6_0(int map_fd, size_t page_cnt, - perf_buffer_sample_fn sample_cb, perf_buffer_lost_fn lost_cb, void *ctx, - const struct perf_buffer_opts *opts); - -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use new variant of perf_buffer__new() instead") -struct perf_buffer *perf_buffer__new_deprecated(int map_fd, size_t page_cnt, - const struct perf_buffer_opts *opts); - -#define perf_buffer__new(...) ___libbpf_overload(___perf_buffer_new, __VA_ARGS__) -#define ___perf_buffer_new6(map_fd, page_cnt, sample_cb, lost_cb, ctx, opts) \ - perf_buffer__new(map_fd, page_cnt, sample_cb, lost_cb, ctx, opts) -#define ___perf_buffer_new3(map_fd, page_cnt, opts) \ - perf_buffer__new_deprecated(map_fd, page_cnt, opts) - enum bpf_perf_event_ret { LIBBPF_PERF_EVENT_DONE = 0, LIBBPF_PERF_EVENT_ERROR = -1, @@ -1057,21 +1071,9 @@ typedef enum bpf_perf_event_ret /* raw perf buffer options, giving most power and control */ struct perf_buffer_raw_opts { - union { - struct { - size_t sz; - long :0; - long :0; - }; - struct { /* DEPRECATED: will be removed in v1.0 */ - /* perf event attrs passed directly into perf_event_open() */ - struct perf_event_attr *attr; - /* raw event callback */ - perf_buffer_event_fn event_cb; - /* ctx is provided to event_cb */ - void *ctx; - }; - }; + size_t sz; + long :0; + long :0; /* if cpu_cnt == 0, open all on all possible CPUs (up to the number of * max_entries of given PERF_EVENT_ARRAY map) */ @@ -1083,26 +1085,13 @@ struct perf_buffer_raw_opts { }; #define perf_buffer_raw_opts__last_field map_keys +struct perf_event_attr; + LIBBPF_API struct perf_buffer * perf_buffer__new_raw(int map_fd, size_t page_cnt, struct perf_event_attr *attr, perf_buffer_event_fn event_cb, void *ctx, const struct perf_buffer_raw_opts *opts); -LIBBPF_API struct perf_buffer * -perf_buffer__new_raw_v0_6_0(int map_fd, size_t page_cnt, struct perf_event_attr *attr, - perf_buffer_event_fn event_cb, void *ctx, - const struct perf_buffer_raw_opts *opts); - -LIBBPF_API LIBBPF_DEPRECATED_SINCE(0, 7, "use new variant of perf_buffer__new_raw() instead") -struct perf_buffer *perf_buffer__new_raw_deprecated(int map_fd, size_t page_cnt, - const struct perf_buffer_raw_opts *opts); - -#define perf_buffer__new_raw(...) ___libbpf_overload(___perf_buffer_new_raw, __VA_ARGS__) -#define ___perf_buffer_new_raw6(map_fd, page_cnt, attr, event_cb, ctx, opts) \ - perf_buffer__new_raw(map_fd, page_cnt, attr, event_cb, ctx, opts) -#define ___perf_buffer_new_raw3(map_fd, page_cnt, opts) \ - perf_buffer__new_raw_deprecated(map_fd, page_cnt, opts) - LIBBPF_API void perf_buffer__free(struct perf_buffer *pb); LIBBPF_API int perf_buffer__epoll_fd(const struct perf_buffer *pb); LIBBPF_API int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms); @@ -1110,15 +1099,22 @@ LIBBPF_API int perf_buffer__consume(struct perf_buffer *pb); LIBBPF_API int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx); LIBBPF_API size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb); LIBBPF_API int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx); - -typedef enum bpf_perf_event_ret - (*bpf_perf_event_print_t)(struct perf_event_header *hdr, - void *private_data); -LIBBPF_DEPRECATED_SINCE(0, 8, "use perf_buffer__poll() or perf_buffer__consume() instead") -LIBBPF_API enum bpf_perf_event_ret -bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, - void **copy_mem, size_t *copy_size, - bpf_perf_event_print_t fn, void *private_data); +/** + * @brief **perf_buffer__buffer()** returns the per-cpu raw mmap()'ed underlying + * memory region of the ring buffer. + * This ring buffer can be used to implement a custom events consumer. + * The ring buffer starts with the *struct perf_event_mmap_page*, which + * holds the ring buffer managment fields, when accessing the header + * structure it's important to be SMP aware. + * You can refer to *perf_event_read_simple* for a simple example. + * @param pb the perf buffer structure + * @param buf_idx the buffer index to retreive + * @param buf (out) gets the base pointer of the mmap()'ed memory + * @param buf_size (out) gets the size of the mmap()'ed region + * @return 0 on success, negative error code for failure + */ +LIBBPF_API int perf_buffer__buffer(struct perf_buffer *pb, int buf_idx, void **buf, + size_t *buf_size); struct bpf_prog_linfo; struct bpf_prog_info; @@ -1141,14 +1137,6 @@ bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, * user, causing subsequent probes to fail. In this case, the caller may want * to adjust that limit with setrlimit(). */ -LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_prog_type() instead") -LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex); -LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_map_type() instead") -LIBBPF_API bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex); -LIBBPF_DEPRECATED_SINCE(0, 8, "use libbpf_probe_bpf_helper() instead") -LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, __u32 ifindex); -LIBBPF_DEPRECATED_SINCE(0, 8, "implement your own or use bpftool for feature detection") -LIBBPF_API bool bpf_probe_large_insn_limit(__u32 ifindex); /** * @brief **libbpf_probe_bpf_prog_type()** detects if host kernel supports @@ -1192,72 +1180,6 @@ LIBBPF_API int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void LIBBPF_API int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helper_id, const void *opts); -/* - * Get bpf_prog_info in continuous memory - * - * struct bpf_prog_info has multiple arrays. The user has option to choose - * arrays to fetch from kernel. The following APIs provide an uniform way to - * fetch these data. All arrays in bpf_prog_info are stored in a single - * continuous memory region. This makes it easy to store the info in a - * file. - * - * Before writing bpf_prog_info_linear to files, it is necessary to - * translate pointers in bpf_prog_info to offsets. Helper functions - * bpf_program__bpil_addr_to_offs() and bpf_program__bpil_offs_to_addr() - * are introduced to switch between pointers and offsets. - * - * Examples: - * # To fetch map_ids and prog_tags: - * __u64 arrays = (1UL << BPF_PROG_INFO_MAP_IDS) | - * (1UL << BPF_PROG_INFO_PROG_TAGS); - * struct bpf_prog_info_linear *info_linear = - * bpf_program__get_prog_info_linear(fd, arrays); - * - * # To save data in file - * bpf_program__bpil_addr_to_offs(info_linear); - * write(f, info_linear, sizeof(*info_linear) + info_linear->data_len); - * - * # To read data from file - * read(f, info_linear, ); - * bpf_program__bpil_offs_to_addr(info_linear); - */ -enum bpf_prog_info_array { - BPF_PROG_INFO_FIRST_ARRAY = 0, - BPF_PROG_INFO_JITED_INSNS = 0, - BPF_PROG_INFO_XLATED_INSNS, - BPF_PROG_INFO_MAP_IDS, - BPF_PROG_INFO_JITED_KSYMS, - BPF_PROG_INFO_JITED_FUNC_LENS, - BPF_PROG_INFO_FUNC_INFO, - BPF_PROG_INFO_LINE_INFO, - BPF_PROG_INFO_JITED_LINE_INFO, - BPF_PROG_INFO_PROG_TAGS, - BPF_PROG_INFO_LAST_ARRAY, -}; - -struct bpf_prog_info_linear { - /* size of struct bpf_prog_info, when the tool is compiled */ - __u32 info_len; - /* total bytes allocated for data, round up to 8 bytes */ - __u32 data_len; - /* which arrays are included in data */ - __u64 arrays; - struct bpf_prog_info info; - __u8 data[]; -}; - -LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper") -LIBBPF_API struct bpf_prog_info_linear * -bpf_program__get_prog_info_linear(int fd, __u64 arrays); - -LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper") -LIBBPF_API void -bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear); - -LIBBPF_DEPRECATED_SINCE(0, 6, "use a custom linear prog_info wrapper") -LIBBPF_API void -bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear); - /** * @brief **libbpf_num_possible_cpus()** is a helper function to get the * number of possible CPUs that the host kernel supports and expects. diff --git a/tools/lib/bpf/libbpf.map b/tools/lib/bpf/libbpf.map index dd35ee58bf..119e6e1ea7 100644 --- a/tools/lib/bpf/libbpf.map +++ b/tools/lib/bpf/libbpf.map @@ -1,29 +1,14 @@ LIBBPF_0.0.1 { global: bpf_btf_get_fd_by_id; - bpf_create_map; - bpf_create_map_in_map; - bpf_create_map_in_map_node; - bpf_create_map_name; - bpf_create_map_node; - bpf_create_map_xattr; - bpf_load_btf; - bpf_load_program; - bpf_load_program_xattr; bpf_map__btf_key_type_id; bpf_map__btf_value_type_id; - bpf_map__def; bpf_map__fd; - bpf_map__is_offload_neutral; bpf_map__name; - bpf_map__next; bpf_map__pin; - bpf_map__prev; - bpf_map__priv; bpf_map__reuse_fd; bpf_map__set_ifindex; bpf_map__set_inner_map_fd; - bpf_map__set_priv; bpf_map__unpin; bpf_map_delete_elem; bpf_map_get_fd_by_id; @@ -38,79 +23,37 @@ LIBBPF_0.0.1 { bpf_object__btf_fd; bpf_object__close; bpf_object__find_map_by_name; - bpf_object__find_map_by_offset; - bpf_object__find_program_by_title; bpf_object__kversion; bpf_object__load; bpf_object__name; - bpf_object__next; bpf_object__open; - bpf_object__open_buffer; - bpf_object__open_xattr; bpf_object__pin; bpf_object__pin_maps; bpf_object__pin_programs; - bpf_object__priv; - bpf_object__set_priv; - bpf_object__unload; bpf_object__unpin_maps; bpf_object__unpin_programs; - bpf_perf_event_read_simple; bpf_prog_attach; bpf_prog_detach; bpf_prog_detach2; bpf_prog_get_fd_by_id; bpf_prog_get_next_id; - bpf_prog_load; - bpf_prog_load_xattr; bpf_prog_query; - bpf_prog_test_run; - bpf_prog_test_run_xattr; bpf_program__fd; - bpf_program__is_kprobe; - bpf_program__is_perf_event; - bpf_program__is_raw_tracepoint; - bpf_program__is_sched_act; - bpf_program__is_sched_cls; - bpf_program__is_socket_filter; - bpf_program__is_tracepoint; - bpf_program__is_xdp; - bpf_program__load; - bpf_program__next; - bpf_program__nth_fd; bpf_program__pin; - bpf_program__pin_instance; - bpf_program__prev; - bpf_program__priv; bpf_program__set_expected_attach_type; bpf_program__set_ifindex; - bpf_program__set_kprobe; - bpf_program__set_perf_event; - bpf_program__set_prep; - bpf_program__set_priv; - bpf_program__set_raw_tracepoint; - bpf_program__set_sched_act; - bpf_program__set_sched_cls; - bpf_program__set_socket_filter; - bpf_program__set_tracepoint; bpf_program__set_type; - bpf_program__set_xdp; - bpf_program__title; bpf_program__unload; bpf_program__unpin; - bpf_program__unpin_instance; bpf_prog_linfo__free; bpf_prog_linfo__new; bpf_prog_linfo__lfind_addr_func; bpf_prog_linfo__lfind; bpf_raw_tracepoint_open; - bpf_set_link_xdp_fd; bpf_task_fd_query; - bpf_verify_program; btf__fd; btf__find_by_name; btf__free; - btf__get_from_id; btf__name_by_offset; btf__new; btf__resolve_size; @@ -127,48 +70,24 @@ LIBBPF_0.0.1 { LIBBPF_0.0.2 { global: - bpf_probe_helper; - bpf_probe_map_type; - bpf_probe_prog_type; - bpf_map__resize; bpf_map_lookup_elem_flags; bpf_object__btf; bpf_object__find_map_fd_by_name; - bpf_get_link_xdp_id; - btf__dedup; - btf__get_map_kv_tids; - btf__get_nr_types; btf__get_raw_data; - btf__load; btf_ext__free; - btf_ext__func_info_rec_size; btf_ext__get_raw_data; - btf_ext__line_info_rec_size; btf_ext__new; - btf_ext__reloc_func_info; - btf_ext__reloc_line_info; - xsk_umem__create; - xsk_socket__create; - xsk_umem__delete; - xsk_socket__delete; - xsk_umem__fd; - xsk_socket__fd; - bpf_program__get_prog_info_linear; - bpf_program__bpil_addr_to_offs; - bpf_program__bpil_offs_to_addr; } LIBBPF_0.0.1; LIBBPF_0.0.3 { global: bpf_map__is_internal; bpf_map_freeze; - btf__finalize_data; } LIBBPF_0.0.2; LIBBPF_0.0.4 { global: bpf_link__destroy; - bpf_object__load_xattr; bpf_program__attach_kprobe; bpf_program__attach_perf_event; bpf_program__attach_raw_tracepoint; @@ -176,14 +95,10 @@ LIBBPF_0.0.4 { bpf_program__attach_uprobe; btf_dump__dump_type; btf_dump__free; - btf_dump__new; btf__parse_elf; libbpf_num_possible_cpus; perf_buffer__free; - perf_buffer__new; - perf_buffer__new_raw; perf_buffer__poll; - xsk_umem__create; } LIBBPF_0.0.3; LIBBPF_0.0.5 { @@ -193,7 +108,6 @@ LIBBPF_0.0.5 { LIBBPF_0.0.6 { global: - bpf_get_link_xdp_info; bpf_map__get_pin_path; bpf_map__is_pinned; bpf_map__set_pin_path; @@ -202,9 +116,6 @@ LIBBPF_0.0.6 { bpf_program__attach_trace; bpf_program__get_expected_attach_type; bpf_program__get_type; - bpf_program__is_tracing; - bpf_program__set_tracing; - bpf_program__size; btf__find_by_name_kind; libbpf_find_vmlinux_btf_id; } LIBBPF_0.0.5; @@ -224,14 +135,8 @@ LIBBPF_0.0.7 { bpf_object__detach_skeleton; bpf_object__load_skeleton; bpf_object__open_skeleton; - bpf_probe_large_insn_limit; - bpf_prog_attach_xattr; bpf_program__attach; bpf_program__name; - bpf_program__is_extension; - bpf_program__is_struct_ops; - bpf_program__set_extension; - bpf_program__set_struct_ops; btf__align_of; libbpf_find_kernel_btf; } LIBBPF_0.0.6; @@ -250,10 +155,7 @@ LIBBPF_0.0.8 { bpf_prog_attach_opts; bpf_program__attach_cgroup; bpf_program__attach_lsm; - bpf_program__is_lsm; bpf_program__set_attach_target; - bpf_program__set_lsm; - bpf_set_link_xdp_fd_opts; } LIBBPF_0.0.7; LIBBPF_0.0.9 { @@ -291,9 +193,7 @@ LIBBPF_0.1.0 { bpf_map__value_size; bpf_program__attach_xdp; bpf_program__autoload; - bpf_program__is_sk_lookup; bpf_program__set_autoload; - bpf_program__set_sk_lookup; btf__parse; btf__parse_raw; btf__pointer_size; @@ -336,7 +236,6 @@ LIBBPF_0.2.0 { perf_buffer__buffer_fd; perf_buffer__epoll_fd; perf_buffer__consume_buffer; - xsk_socket__create_shared; } LIBBPF_0.1.0; LIBBPF_0.3.0 { @@ -348,8 +247,6 @@ LIBBPF_0.3.0 { btf__new_empty_split; btf__new_split; ring_buffer__epoll_fd; - xsk_setup_xdp_prog; - xsk_socket__update_xskmap; } LIBBPF_0.2.0; LIBBPF_0.4.0 { @@ -397,7 +294,6 @@ LIBBPF_0.6.0 { bpf_object__next_program; bpf_object__prev_map; bpf_object__prev_program; - bpf_prog_load_deprecated; bpf_prog_load; bpf_program__flags; bpf_program__insn_cnt; @@ -407,18 +303,14 @@ LIBBPF_0.6.0 { btf__add_decl_tag; btf__add_type_tag; btf__dedup; - btf__dedup_deprecated; btf__raw_data; btf__type_cnt; btf_dump__new; - btf_dump__new_deprecated; libbpf_major_version; libbpf_minor_version; libbpf_version_string; perf_buffer__new; - perf_buffer__new_deprecated; perf_buffer__new_raw; - perf_buffer__new_raw_deprecated; } LIBBPF_0.5.0; LIBBPF_0.7.0 { @@ -434,17 +326,43 @@ LIBBPF_0.7.0 { bpf_xdp_detach; bpf_xdp_query; bpf_xdp_query_id; + btf_ext__raw_data; libbpf_probe_bpf_helper; libbpf_probe_bpf_map_type; libbpf_probe_bpf_prog_type; - libbpf_set_memlock_rlim_max; + libbpf_set_memlock_rlim; } LIBBPF_0.6.0; LIBBPF_0.8.0 { global: + bpf_map__autocreate; + bpf_map__get_next_key; + bpf_map__delete_elem; + bpf_map__lookup_and_delete_elem; + bpf_map__lookup_elem; + bpf_map__set_autocreate; + bpf_map__update_elem; + bpf_map_delete_elem_flags; bpf_object__destroy_subskeleton; bpf_object__open_subskeleton; + bpf_program__attach_kprobe_multi_opts; + bpf_program__attach_trace_opts; + bpf_program__attach_usdt; + bpf_program__set_insns; libbpf_register_prog_handler; libbpf_unregister_prog_handler; - bpf_program__attach_kprobe_multi_opts; } LIBBPF_0.7.0; + +LIBBPF_1.0.0 { + global: + bpf_obj_get_opts; + bpf_prog_query_opts; + bpf_program__attach_ksyscall; + btf__add_enum64; + btf__add_enum64_value; + libbpf_bpf_attach_type_str; + libbpf_bpf_link_type_str; + libbpf_bpf_map_type_str; + libbpf_bpf_prog_type_str; + perf_buffer__buffer; +}; diff --git a/tools/lib/bpf/libbpf_common.h b/tools/lib/bpf/libbpf_common.h index 000e37798f..9a7937f339 100644 --- a/tools/lib/bpf/libbpf_common.h +++ b/tools/lib/bpf/libbpf_common.h @@ -30,20 +30,10 @@ /* Add checks for other versions below when planning deprecation of API symbols * with the LIBBPF_DEPRECATED_SINCE macro. */ -#if __LIBBPF_CURRENT_VERSION_GEQ(0, 6) -#define __LIBBPF_MARK_DEPRECATED_0_6(X) X +#if __LIBBPF_CURRENT_VERSION_GEQ(1, 0) +#define __LIBBPF_MARK_DEPRECATED_1_0(X) X #else -#define __LIBBPF_MARK_DEPRECATED_0_6(X) -#endif -#if __LIBBPF_CURRENT_VERSION_GEQ(0, 7) -#define __LIBBPF_MARK_DEPRECATED_0_7(X) X -#else -#define __LIBBPF_MARK_DEPRECATED_0_7(X) -#endif -#if __LIBBPF_CURRENT_VERSION_GEQ(0, 8) -#define __LIBBPF_MARK_DEPRECATED_0_8(X) X -#else -#define __LIBBPF_MARK_DEPRECATED_0_8(X) +#define __LIBBPF_MARK_DEPRECATED_1_0(X) #endif /* This set of internal macros allows to do "function overloading" based on diff --git a/tools/lib/bpf/libbpf_internal.h b/tools/lib/bpf/libbpf_internal.h index b6247dc7f8..4135ae0a2b 100644 --- a/tools/lib/bpf/libbpf_internal.h +++ b/tools/lib/bpf/libbpf_internal.h @@ -15,7 +15,6 @@ #include #include #include -#include "libbpf_legacy.h" #include "relo_core.h" /* make sure libbpf doesn't use kernel-only integer typedefs */ @@ -103,6 +102,17 @@ #define str_has_pfx(str, pfx) \ (strncmp(str, pfx, __builtin_constant_p(pfx) ? sizeof(pfx) - 1 : strlen(pfx)) == 0) +/* suffix check */ +static inline bool str_has_sfx(const char *str, const char *sfx) +{ + size_t str_len = strlen(str); + size_t sfx_len = strlen(sfx); + + if (sfx_len > str_len) + return false; + return strcmp(str + str_len - sfx_len, sfx) == 0; +} + /* Symbol versioning is different between static and shared library. * Properly versioned symbols are needed for shared library, but * only the symbol of the new version is needed for static library. @@ -148,6 +158,15 @@ do { \ #ifndef __has_builtin #define __has_builtin(x) 0 #endif + +struct bpf_link { + int (*detach)(struct bpf_link *link); + void (*dealloc)(struct bpf_link *link); + char *pin_path; /* NULL, if not pinned */ + int fd; /* hook FD, -1 if not applicable */ + bool disconnected; +}; + /* * Re-implement glibc's reallocarray() for libbpf internal-only use. * reallocarray(), unfortunately, is not available in all versions of glibc, @@ -329,6 +348,12 @@ enum kern_feature_id { FEAT_BTF_TYPE_TAG, /* memcg-based accounting for BPF maps and progs */ FEAT_MEMCG_ACCOUNT, + /* BPF cookie (bpf_get_attach_cookie() BPF helper) support */ + FEAT_BPF_COOKIE, + /* BTF_KIND_ENUM64 support and BTF_KIND_ENUM kflag support */ + FEAT_BTF_ENUM64, + /* Kernel uses syscall wrapper (CONFIG_ARCH_HAS_SYSCALL_WRAPPER) */ + FEAT_SYSCALL_WRAPPER, __FEAT_CNT, }; @@ -354,6 +379,13 @@ struct btf_ext_info { void *info; __u32 rec_size; __u32 len; + /* optional (maintained internally by libbpf) mapping between .BTF.ext + * section and corresponding ELF section. This is used to join + * information like CO-RE relocation records with corresponding BPF + * programs defined in ELF sections + */ + __u32 *sec_idxs; + int sec_cnt; }; #define for_each_btf_ext_sec(seg, sec) \ @@ -447,8 +479,6 @@ int btf_ext_visit_str_offs(struct btf_ext *btf_ext, str_off_visit_fn visit, void __s32 btf__find_by_name_kind_own(const struct btf *btf, const char *type_name, __u32 kind); -extern enum libbpf_strict_mode libbpf_mode; - typedef int (*kallsyms_cb_t)(unsigned long long sym_addr, char sym_type, const char *sym_name, void *ctx); @@ -467,12 +497,8 @@ static inline int libbpf_err(int ret) */ static inline int libbpf_err_errno(int ret) { - if (libbpf_mode & LIBBPF_STRICT_DIRECT_ERRS) - /* errno is already assumed to be set on error */ - return ret < 0 ? -errno : ret; - - /* legacy: on error return -1 directly and don't touch errno */ - return ret; + /* errno is already assumed to be set on error */ + return ret < 0 ? -errno : ret; } /* handle error for pointer-returning APIs, err is assumed to be < 0 always */ @@ -480,12 +506,7 @@ static inline void *libbpf_err_ptr(int err) { /* set errno on error, this doesn't break anything */ errno = -err; - - if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS) - return NULL; - - /* legacy: encode err as ptr */ - return ERR_PTR(err); + return NULL; } /* handle pointer-returning APIs' error handling */ @@ -495,11 +516,7 @@ static inline void *libbpf_ptr(void *ret) if (IS_ERR(ret)) errno = -PTR_ERR(ret); - if (libbpf_mode & LIBBPF_STRICT_CLEAN_PTRS) - return IS_ERR(ret) ? NULL : ret; - - /* legacy: pass-through original pointer */ - return ret; + return IS_ERR(ret) ? NULL : ret; } static inline bool str_is_empty(const char *s) @@ -543,4 +560,17 @@ int bpf_core_add_cands(struct bpf_core_cand *local_cand, struct bpf_core_cand_list *cands); void bpf_core_free_cands(struct bpf_core_cand_list *cands); +struct usdt_manager *usdt_manager_new(struct bpf_object *obj); +void usdt_manager_free(struct usdt_manager *man); +struct bpf_link * usdt_manager_attach_usdt(struct usdt_manager *man, + const struct bpf_program *prog, + pid_t pid, const char *path, + const char *usdt_provider, const char *usdt_name, + __u64 usdt_cookie); + +static inline bool is_pow_of_2(size_t x) +{ + return x && (x & (x - 1)) == 0; +} + #endif /* __LIBBPF_LIBBPF_INTERNAL_H */ diff --git a/tools/lib/bpf/libbpf_legacy.h b/tools/lib/bpf/libbpf_legacy.h index d7bcbd01f6..5b7e0155db 100644 --- a/tools/lib/bpf/libbpf_legacy.h +++ b/tools/lib/bpf/libbpf_legacy.h @@ -20,6 +20,11 @@ extern "C" { #endif +/* As of libbpf 1.0 libbpf_set_strict_mode() and enum libbpf_struct_mode have + * no effect. But they are left in libbpf_legacy.h so that applications that + * prepared for libbpf 1.0 before final release by using + * libbpf_set_strict_mode() still work with libbpf 1.0+ without any changes. + */ enum libbpf_strict_mode { /* Turn on all supported strict features of libbpf to simulate libbpf * v1.0 behavior. @@ -71,8 +76,8 @@ enum libbpf_strict_mode { * first BPF program or map creation operation. This is done only if * kernel is too old to support memcg-based memory accounting for BPF * subsystem. By default, RLIMIT_MEMLOCK limit is set to RLIM_INFINITY, - * but it can be overriden with libbpf_set_memlock_rlim_max() API. - * Note that libbpf_set_memlock_rlim_max() needs to be called before + * but it can be overriden with libbpf_set_memlock_rlim() API. + * Note that libbpf_set_memlock_rlim() needs to be called before * the very first bpf_prog_load(), bpf_map_create() or bpf_object__load() * operation. */ @@ -88,6 +93,25 @@ enum libbpf_strict_mode { LIBBPF_API int libbpf_set_strict_mode(enum libbpf_strict_mode mode); +/** + * @brief **libbpf_get_error()** extracts the error code from the passed + * pointer + * @param ptr pointer returned from libbpf API function + * @return error code; or 0 if no error occured + * + * Note, as of libbpf 1.0 this function is not necessary and not recommended + * to be used. Libbpf doesn't return error code embedded into the pointer + * itself. Instead, NULL is returned on error and error code is passed through + * thread-local errno variable. **libbpf_get_error()** is just returning -errno + * value if it receives NULL, which is correct only if errno hasn't been + * modified between libbpf API call and corresponding **libbpf_get_error()** + * call. Prefer to check return for NULL and use errno directly. + * + * This API is left in libbpf 1.0 to allow applications that were 1.0-ready + * before final libbpf 1.0 without needing to change them. + */ +LIBBPF_API long libbpf_get_error(const void *ptr); + #define DECLARE_LIBBPF_OPTS LIBBPF_OPTS /* "Discouraged" APIs which don't follow consistent libbpf naming patterns. diff --git a/tools/lib/bpf/libbpf_probes.c b/tools/lib/bpf/libbpf_probes.c index 97b06cede5..0b5398786b 100644 --- a/tools/lib/bpf/libbpf_probes.c +++ b/tools/lib/bpf/libbpf_probes.c @@ -17,47 +17,14 @@ #include "libbpf.h" #include "libbpf_internal.h" -static bool grep(const char *buffer, const char *pattern) -{ - return !!strstr(buffer, pattern); -} - -static int get_vendor_id(int ifindex) -{ - char ifname[IF_NAMESIZE], path[64], buf[8]; - ssize_t len; - int fd; - - if (!if_indextoname(ifindex, ifname)) - return -1; - - snprintf(path, sizeof(path), "/sys/class/net/%s/device/vendor", ifname); - - fd = open(path, O_RDONLY | O_CLOEXEC); - if (fd < 0) - return -1; - - len = read(fd, buf, sizeof(buf)); - close(fd); - if (len < 0) - return -1; - if (len >= (ssize_t)sizeof(buf)) - return -1; - buf[len] = '\0'; - - return strtol(buf, NULL, 0); -} - static int probe_prog_load(enum bpf_prog_type prog_type, const struct bpf_insn *insns, size_t insns_cnt, - char *log_buf, size_t log_buf_sz, - __u32 ifindex) + char *log_buf, size_t log_buf_sz) { LIBBPF_OPTS(bpf_prog_load_opts, opts, .log_buf = log_buf, .log_size = log_buf_sz, .log_level = log_buf ? 1 : 0, - .prog_ifindex = ifindex, ); int fd, err, exp_err = 0; const char *exp_msg = NULL; @@ -161,31 +128,10 @@ int libbpf_probe_bpf_prog_type(enum bpf_prog_type prog_type, const void *opts) if (opts) return libbpf_err(-EINVAL); - ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0, 0); + ret = probe_prog_load(prog_type, insns, insn_cnt, NULL, 0); return libbpf_err(ret); } -bool bpf_probe_prog_type(enum bpf_prog_type prog_type, __u32 ifindex) -{ - struct bpf_insn insns[2] = { - BPF_MOV64_IMM(BPF_REG_0, 0), - BPF_EXIT_INSN() - }; - - /* prefer libbpf_probe_bpf_prog_type() unless offload is requested */ - if (ifindex == 0) - return libbpf_probe_bpf_prog_type(prog_type, NULL) == 1; - - if (ifindex && prog_type == BPF_PROG_TYPE_SCHED_CLS) - /* nfp returns -EINVAL on exit(0) with TC offload */ - insns[0].imm = 2; - - errno = 0; - probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), NULL, 0, ifindex); - - return errno != EINVAL && errno != EOPNOTSUPP; -} - int libbpf__load_raw_btf(const char *raw_types, size_t types_len, const char *str_sec, size_t str_len) { @@ -242,15 +188,13 @@ static int load_local_storage_btf(void) strs, sizeof(strs)); } -static int probe_map_create(enum bpf_map_type map_type, __u32 ifindex) +static int probe_map_create(enum bpf_map_type map_type) { LIBBPF_OPTS(bpf_map_create_opts, opts); int key_size, value_size, max_entries; __u32 btf_key_type_id = 0, btf_value_type_id = 0; int fd = -1, btf_fd = -1, fd_inner = -1, exp_err = 0, err; - opts.map_ifindex = ifindex; - key_size = sizeof(__u32); value_size = sizeof(__u32); max_entries = 1; @@ -326,12 +270,6 @@ static int probe_map_create(enum bpf_map_type map_type, __u32 ifindex) if (map_type == BPF_MAP_TYPE_ARRAY_OF_MAPS || map_type == BPF_MAP_TYPE_HASH_OF_MAPS) { - /* TODO: probe for device, once libbpf has a function to create - * map-in-map for offload - */ - if (ifindex) - goto cleanup; - fd_inner = bpf_map_create(BPF_MAP_TYPE_HASH, NULL, sizeof(__u32), sizeof(__u32), 1, NULL); if (fd_inner < 0) @@ -370,15 +308,10 @@ int libbpf_probe_bpf_map_type(enum bpf_map_type map_type, const void *opts) if (opts) return libbpf_err(-EINVAL); - ret = probe_map_create(map_type, 0); + ret = probe_map_create(map_type); return libbpf_err(ret); } -bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex) -{ - return probe_map_create(map_type, ifindex) == 1; -} - int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helper_id, const void *opts) { @@ -407,7 +340,7 @@ int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helpe } buf[0] = '\0'; - ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf), 0); + ret = probe_prog_load(prog_type, insns, insn_cnt, buf, sizeof(buf)); if (ret < 0) return libbpf_err(ret); @@ -427,51 +360,3 @@ int libbpf_probe_bpf_helper(enum bpf_prog_type prog_type, enum bpf_func_id helpe return 0; return 1; /* assume supported */ } - -bool bpf_probe_helper(enum bpf_func_id id, enum bpf_prog_type prog_type, - __u32 ifindex) -{ - struct bpf_insn insns[2] = { - BPF_EMIT_CALL(id), - BPF_EXIT_INSN() - }; - char buf[4096] = {}; - bool res; - - probe_prog_load(prog_type, insns, ARRAY_SIZE(insns), buf, sizeof(buf), ifindex); - res = !grep(buf, "invalid func ") && !grep(buf, "unknown func "); - - if (ifindex) { - switch (get_vendor_id(ifindex)) { - case 0x19ee: /* Netronome specific */ - res = res && !grep(buf, "not supported by FW") && - !grep(buf, "unsupported function id"); - break; - default: - break; - } - } - - return res; -} - -/* - * Probe for availability of kernel commit (5.3): - * - * c04c0d2b968a ("bpf: increase complexity limit and maximum program size") - */ -bool bpf_probe_large_insn_limit(__u32 ifindex) -{ - struct bpf_insn insns[BPF_MAXINSNS + 1]; - int i; - - for (i = 0; i < BPF_MAXINSNS; i++) - insns[i] = BPF_MOV64_IMM(BPF_REG_0, 1); - insns[BPF_MAXINSNS] = BPF_EXIT_INSN(); - - errno = 0; - probe_prog_load(BPF_PROG_TYPE_SCHED_CLS, insns, ARRAY_SIZE(insns), NULL, 0, - ifindex); - - return errno != E2BIG && errno != EINVAL; -} diff --git a/tools/lib/bpf/libbpf_version.h b/tools/lib/bpf/libbpf_version.h index 61f2039404..2fb2f42900 100644 --- a/tools/lib/bpf/libbpf_version.h +++ b/tools/lib/bpf/libbpf_version.h @@ -3,7 +3,7 @@ #ifndef __LIBBPF_VERSION_H #define __LIBBPF_VERSION_H -#define LIBBPF_MAJOR_VERSION 0 -#define LIBBPF_MINOR_VERSION 8 +#define LIBBPF_MAJOR_VERSION 1 +#define LIBBPF_MINOR_VERSION 0 #endif /* __LIBBPF_VERSION_H */ diff --git a/tools/lib/bpf/linker.c b/tools/lib/bpf/linker.c index 9aa016fb55..4ac02c28e1 100644 --- a/tools/lib/bpf/linker.c +++ b/tools/lib/bpf/linker.c @@ -697,11 +697,6 @@ static int linker_load_obj_file(struct bpf_linker *linker, const char *filename, return err; } -static bool is_pow_of_2(size_t x) -{ - return x && (x & (x - 1)) == 0; -} - static int linker_sanity_check_elf(struct src_obj *obj) { struct src_sec *sec; @@ -1340,6 +1335,7 @@ static bool glob_sym_btf_matches(const char *sym_name, bool exact, case BTF_KIND_STRUCT: case BTF_KIND_UNION: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: case BTF_KIND_FWD: case BTF_KIND_FUNC: case BTF_KIND_VAR: @@ -1362,6 +1358,7 @@ static bool glob_sym_btf_matches(const char *sym_name, bool exact, case BTF_KIND_INT: case BTF_KIND_FLOAT: case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: /* ignore encoding for int and enum values for enum */ if (t1->size != t2->size) { pr_warn("global '%s': incompatible %s '%s' size %u and %u\n", diff --git a/tools/lib/bpf/netlink.c b/tools/lib/bpf/netlink.c index cbc8967d54..6c01316803 100644 --- a/tools/lib/bpf/netlink.c +++ b/tools/lib/bpf/netlink.c @@ -27,6 +27,14 @@ typedef int (*libbpf_dump_nlmsg_t)(void *cookie, void *msg, struct nlattr **tb); typedef int (*__dump_nlmsg_t)(struct nlmsghdr *nlmsg, libbpf_dump_nlmsg_t, void *cookie); +struct xdp_link_info { + __u32 prog_id; + __u32 drv_prog_id; + __u32 hw_prog_id; + __u32 skb_prog_id; + __u8 attach_mode; +}; + struct xdp_id_md { int ifindex; __u32 flags; @@ -288,31 +296,6 @@ int bpf_xdp_detach(int ifindex, __u32 flags, const struct bpf_xdp_attach_opts *o return bpf_xdp_attach(ifindex, -1, flags, opts); } -int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags, - const struct bpf_xdp_set_link_opts *opts) -{ - int old_fd = -1, ret; - - if (!OPTS_VALID(opts, bpf_xdp_set_link_opts)) - return libbpf_err(-EINVAL); - - if (OPTS_HAS(opts, old_fd)) { - old_fd = OPTS_GET(opts, old_fd, -1); - flags |= XDP_FLAGS_REPLACE; - } - - ret = __bpf_set_link_xdp_fd_replace(ifindex, fd, old_fd, flags); - return libbpf_err(ret); -} - -int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags) -{ - int ret; - - ret = __bpf_set_link_xdp_fd_replace(ifindex, fd, 0, flags); - return libbpf_err(ret); -} - static int __dump_link_nlmsg(struct nlmsghdr *nlh, libbpf_dump_nlmsg_t dump_link_nlmsg, void *cookie) { @@ -413,30 +396,6 @@ int bpf_xdp_query(int ifindex, int xdp_flags, struct bpf_xdp_query_opts *opts) return 0; } -int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, - size_t info_size, __u32 flags) -{ - LIBBPF_OPTS(bpf_xdp_query_opts, opts); - size_t sz; - int err; - - if (!info_size) - return libbpf_err(-EINVAL); - - err = bpf_xdp_query(ifindex, flags, &opts); - if (err) - return libbpf_err(err); - - /* struct xdp_link_info field layout matches struct bpf_xdp_query_opts - * layout after sz field - */ - sz = min(info_size, offsetofend(struct xdp_link_info, attach_mode)); - memcpy(info, &opts.prog_id, sz); - memset((void *)info + sz, 0, info_size - sz); - - return 0; -} - int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id) { LIBBPF_OPTS(bpf_xdp_query_opts, opts); @@ -463,11 +422,6 @@ int bpf_xdp_query_id(int ifindex, int flags, __u32 *prog_id) } -int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags) -{ - return bpf_xdp_query_id(ifindex, flags, prog_id); -} - typedef int (*qdisc_config_t)(struct libbpf_nla_req *req); static int clsact_config(struct libbpf_nla_req *req) diff --git a/tools/lib/bpf/relo_core.c b/tools/lib/bpf/relo_core.c index f946f23eab..c4b0e81ae2 100644 --- a/tools/lib/bpf/relo_core.c +++ b/tools/lib/bpf/relo_core.c @@ -95,6 +95,7 @@ static const char *core_relo_kind_str(enum bpf_core_relo_kind kind) case BPF_CORE_TYPE_ID_LOCAL: return "local_type_id"; case BPF_CORE_TYPE_ID_TARGET: return "target_type_id"; case BPF_CORE_TYPE_EXISTS: return "type_exists"; + case BPF_CORE_TYPE_MATCHES: return "type_matches"; case BPF_CORE_TYPE_SIZE: return "type_size"; case BPF_CORE_ENUMVAL_EXISTS: return "enumval_exists"; case BPF_CORE_ENUMVAL_VALUE: return "enumval_value"; @@ -123,6 +124,7 @@ static bool core_relo_is_type_based(enum bpf_core_relo_kind kind) case BPF_CORE_TYPE_ID_LOCAL: case BPF_CORE_TYPE_ID_TARGET: case BPF_CORE_TYPE_EXISTS: + case BPF_CORE_TYPE_MATCHES: case BPF_CORE_TYPE_SIZE: return true; default: @@ -141,6 +143,86 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) } } +int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, + const struct btf *targ_btf, __u32 targ_id, int level) +{ + const struct btf_type *local_type, *targ_type; + int depth = 32; /* max recursion depth */ + + /* caller made sure that names match (ignoring flavor suffix) */ + local_type = btf_type_by_id(local_btf, local_id); + targ_type = btf_type_by_id(targ_btf, targ_id); + if (!btf_kind_core_compat(local_type, targ_type)) + return 0; + +recur: + depth--; + if (depth < 0) + return -EINVAL; + + local_type = skip_mods_and_typedefs(local_btf, local_id, &local_id); + targ_type = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); + if (!local_type || !targ_type) + return -EINVAL; + + if (!btf_kind_core_compat(local_type, targ_type)) + return 0; + + switch (btf_kind(local_type)) { + case BTF_KIND_UNKN: + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + case BTF_KIND_ENUM: + case BTF_KIND_FWD: + case BTF_KIND_ENUM64: + return 1; + case BTF_KIND_INT: + /* just reject deprecated bitfield-like integers; all other + * integers are by default compatible between each other + */ + return btf_int_offset(local_type) == 0 && btf_int_offset(targ_type) == 0; + case BTF_KIND_PTR: + local_id = local_type->type; + targ_id = targ_type->type; + goto recur; + case BTF_KIND_ARRAY: + local_id = btf_array(local_type)->type; + targ_id = btf_array(targ_type)->type; + goto recur; + case BTF_KIND_FUNC_PROTO: { + struct btf_param *local_p = btf_params(local_type); + struct btf_param *targ_p = btf_params(targ_type); + __u16 local_vlen = btf_vlen(local_type); + __u16 targ_vlen = btf_vlen(targ_type); + int i, err; + + if (local_vlen != targ_vlen) + return 0; + + for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { + if (level <= 0) + return -EINVAL; + + skip_mods_and_typedefs(local_btf, local_p->type, &local_id); + skip_mods_and_typedefs(targ_btf, targ_p->type, &targ_id); + err = __bpf_core_types_are_compat(local_btf, local_id, targ_btf, targ_id, + level - 1); + if (err <= 0) + return err; + } + + /* tail recurse for return type check */ + skip_mods_and_typedefs(local_btf, local_type->type, &local_id); + skip_mods_and_typedefs(targ_btf, targ_type->type, &targ_id); + goto recur; + } + default: + pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + btf_kind_str(local_type), local_id, targ_id); + return 0; + } +} + /* * Turn bpf_core_relo into a low- and high-level spec representation, * validating correctness along the way, as well as calculating resulting @@ -167,40 +249,39 @@ static bool core_relo_is_enumval_based(enum bpf_core_relo_kind kind) * just a parsed access string representation): [0, 1, 2, 3]. * * High-level spec will capture only 3 points: - * - intial zero-index access by pointer (&s->... is the same as &s[0]...); + * - initial zero-index access by pointer (&s->... is the same as &s[0]...); * - field 'a' access (corresponds to '2' in low-level spec); * - array element #3 access (corresponds to '3' in low-level spec). * - * Type-based relocations (TYPE_EXISTS/TYPE_SIZE, + * Type-based relocations (TYPE_EXISTS/TYPE_MATCHES/TYPE_SIZE, * TYPE_ID_LOCAL/TYPE_ID_TARGET) don't capture any field information. Their * spec and raw_spec are kept empty. * * Enum value-based relocations (ENUMVAL_EXISTS/ENUMVAL_VALUE) use access * string to specify enumerator's value index that need to be relocated. */ -static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, - __u32 type_id, - const char *spec_str, - enum bpf_core_relo_kind relo_kind, - struct bpf_core_spec *spec) +int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, + const struct bpf_core_relo *relo, + struct bpf_core_spec *spec) { int access_idx, parsed_len, i; struct bpf_core_accessor *acc; const struct btf_type *t; - const char *name; - __u32 id; + const char *name, *spec_str; + __u32 id, name_off; __s64 sz; + spec_str = btf__name_by_offset(btf, relo->access_str_off); if (str_is_empty(spec_str) || *spec_str == ':') return -EINVAL; memset(spec, 0, sizeof(*spec)); spec->btf = btf; - spec->root_type_id = type_id; - spec->relo_kind = relo_kind; + spec->root_type_id = relo->type_id; + spec->relo_kind = relo->kind; /* type-based relocations don't have a field access string */ - if (core_relo_is_type_based(relo_kind)) { + if (core_relo_is_type_based(relo->kind)) { if (strcmp(spec_str, "0")) return -EINVAL; return 0; @@ -221,7 +302,7 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, if (spec->raw_len == 0) return -EINVAL; - t = skip_mods_and_typedefs(btf, type_id, &id); + t = skip_mods_and_typedefs(btf, relo->type_id, &id); if (!t) return -EINVAL; @@ -231,16 +312,18 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, acc->idx = access_idx; spec->len++; - if (core_relo_is_enumval_based(relo_kind)) { - if (!btf_is_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t)) + if (core_relo_is_enumval_based(relo->kind)) { + if (!btf_is_any_enum(t) || spec->raw_len > 1 || access_idx >= btf_vlen(t)) return -EINVAL; /* record enumerator name in a first accessor */ - acc->name = btf__name_by_offset(btf, btf_enum(t)[access_idx].name_off); + name_off = btf_is_enum(t) ? btf_enum(t)[access_idx].name_off + : btf_enum64(t)[access_idx].name_off; + acc->name = btf__name_by_offset(btf, name_off); return 0; } - if (!core_relo_is_field_based(relo_kind)) + if (!core_relo_is_field_based(relo->kind)) return -EINVAL; sz = btf__resolve_size(btf, id); @@ -301,7 +384,7 @@ static int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, spec->bit_offset += access_idx * sz * 8; } else { pr_warn("prog '%s': relo for [%u] %s (at idx %d) captures type [%d] of unexpected kind %s\n", - prog_name, type_id, spec_str, i, id, btf_kind_str(t)); + prog_name, relo->type_id, spec_str, i, id, btf_kind_str(t)); return -EINVAL; } } @@ -341,7 +424,7 @@ static int bpf_core_fields_are_compat(const struct btf *local_btf, if (btf_is_composite(local_type) && btf_is_composite(targ_type)) return 1; - if (btf_kind(local_type) != btf_kind(targ_type)) + if (!btf_kind_core_compat(local_type, targ_type)) return 0; switch (btf_kind(local_type)) { @@ -349,6 +432,7 @@ static int bpf_core_fields_are_compat(const struct btf *local_btf, case BTF_KIND_FLOAT: return 1; case BTF_KIND_FWD: + case BTF_KIND_ENUM64: case BTF_KIND_ENUM: { const char *local_name, *targ_name; size_t local_len, targ_len; @@ -478,6 +562,7 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, const struct bpf_core_accessor *local_acc; struct bpf_core_accessor *targ_acc; int i, sz, matched; + __u32 name_off; memset(targ_spec, 0, sizeof(*targ_spec)); targ_spec->btf = targ_btf; @@ -485,9 +570,14 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, targ_spec->relo_kind = local_spec->relo_kind; if (core_relo_is_type_based(local_spec->relo_kind)) { - return bpf_core_types_are_compat(local_spec->btf, - local_spec->root_type_id, - targ_btf, targ_id); + if (local_spec->relo_kind == BPF_CORE_TYPE_MATCHES) + return bpf_core_types_match(local_spec->btf, + local_spec->root_type_id, + targ_btf, targ_id); + else + return bpf_core_types_are_compat(local_spec->btf, + local_spec->root_type_id, + targ_btf, targ_id); } local_acc = &local_spec->spec[0]; @@ -495,18 +585,22 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, if (core_relo_is_enumval_based(local_spec->relo_kind)) { size_t local_essent_len, targ_essent_len; - const struct btf_enum *e; const char *targ_name; /* has to resolve to an enum */ targ_type = skip_mods_and_typedefs(targ_spec->btf, targ_id, &targ_id); - if (!btf_is_enum(targ_type)) + if (!btf_is_any_enum(targ_type)) return 0; local_essent_len = bpf_core_essential_name_len(local_acc->name); - for (i = 0, e = btf_enum(targ_type); i < btf_vlen(targ_type); i++, e++) { - targ_name = btf__name_by_offset(targ_spec->btf, e->name_off); + for (i = 0; i < btf_vlen(targ_type); i++) { + if (btf_is_enum(targ_type)) + name_off = btf_enum(targ_type)[i].name_off; + else + name_off = btf_enum64(targ_type)[i].name_off; + + targ_name = btf__name_by_offset(targ_spec->btf, name_off); targ_essent_len = bpf_core_essential_name_len(targ_name); if (targ_essent_len != local_essent_len) continue; @@ -584,7 +678,7 @@ static int bpf_core_spec_match(struct bpf_core_spec *local_spec, static int bpf_core_calc_field_relo(const char *prog_name, const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, - __u32 *val, __u32 *field_sz, __u32 *type_id, + __u64 *val, __u32 *field_sz, __u32 *type_id, bool *validate) { const struct bpf_core_accessor *acc; @@ -681,8 +775,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, *val = byte_sz; break; case BPF_CORE_FIELD_SIGNED: - /* enums will be assumed unsigned */ - *val = btf_is_enum(mt) || + *val = (btf_is_any_enum(mt) && BTF_INFO_KFLAG(mt->info)) || (btf_int_encoding(mt) & BTF_INT_SIGNED); if (validate) *validate = true; /* signedness is never ambiguous */ @@ -709,7 +802,7 @@ static int bpf_core_calc_field_relo(const char *prog_name, static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, - __u32 *val, bool *validate) + __u64 *val, bool *validate) { __s64 sz; @@ -733,6 +826,7 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, *validate = false; break; case BPF_CORE_TYPE_EXISTS: + case BPF_CORE_TYPE_MATCHES: *val = 1; break; case BPF_CORE_TYPE_SIZE: @@ -752,10 +846,9 @@ static int bpf_core_calc_type_relo(const struct bpf_core_relo *relo, static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, const struct bpf_core_spec *spec, - __u32 *val) + __u64 *val) { const struct btf_type *t; - const struct btf_enum *e; switch (relo->kind) { case BPF_CORE_ENUMVAL_EXISTS: @@ -765,8 +858,10 @@ static int bpf_core_calc_enumval_relo(const struct bpf_core_relo *relo, if (!spec) return -EUCLEAN; /* request instruction poisoning */ t = btf_type_by_id(spec->btf, spec->spec[0].type_id); - e = btf_enum(t) + spec->spec[0].idx; - *val = e->val; + if (btf_is_enum(t)) + *val = btf_enum(t)[spec->spec[0].idx].val; + else + *val = btf_enum64_value(btf_enum64(t) + spec->spec[0].idx); break; default: return -EOPNOTSUPP; @@ -930,7 +1025,7 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, int insn_idx, const struct bpf_core_relo *relo, int relo_idx, const struct bpf_core_relo_res *res) { - __u32 orig_val, new_val; + __u64 orig_val, new_val; __u8 class; class = BPF_CLASS(insn->code); @@ -955,28 +1050,30 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, if (BPF_SRC(insn->code) != BPF_K) return -EINVAL; if (res->validate && insn->imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %u -> %u\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (ALU/ALU64) value: got %u, exp %llu -> %llu\n", prog_name, relo_idx, - insn_idx, insn->imm, orig_val, new_val); + insn_idx, insn->imm, (unsigned long long)orig_val, + (unsigned long long)new_val); return -EINVAL; } orig_val = insn->imm; insn->imm = new_val; - pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %u -> %u\n", + pr_debug("prog '%s': relo #%d: patched insn #%d (ALU/ALU64) imm %llu -> %llu\n", prog_name, relo_idx, insn_idx, - orig_val, new_val); + (unsigned long long)orig_val, (unsigned long long)new_val); break; case BPF_LDX: case BPF_ST: case BPF_STX: if (res->validate && insn->off != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %u -> %u\n", - prog_name, relo_idx, insn_idx, insn->off, orig_val, new_val); + pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDX/ST/STX) value: got %u, exp %llu -> %llu\n", + prog_name, relo_idx, insn_idx, insn->off, (unsigned long long)orig_val, + (unsigned long long)new_val); return -EINVAL; } if (new_val > SHRT_MAX) { - pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %u\n", - prog_name, relo_idx, insn_idx, new_val); + pr_warn("prog '%s': relo #%d: insn #%d (LDX/ST/STX) value too big: %llu\n", + prog_name, relo_idx, insn_idx, (unsigned long long)new_val); return -ERANGE; } if (res->fail_memsz_adjust) { @@ -988,8 +1085,9 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, orig_val = insn->off; insn->off = new_val; - pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %u -> %u\n", - prog_name, relo_idx, insn_idx, orig_val, new_val); + pr_debug("prog '%s': relo #%d: patched insn #%d (LDX/ST/STX) off %llu -> %llu\n", + prog_name, relo_idx, insn_idx, (unsigned long long)orig_val, + (unsigned long long)new_val); if (res->new_sz != res->orig_sz) { int insn_bytes_sz, insn_bpf_sz; @@ -1025,20 +1123,20 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, return -EINVAL; } - imm = insn[0].imm + ((__u64)insn[1].imm << 32); + imm = (__u32)insn[0].imm | ((__u64)insn[1].imm << 32); if (res->validate && imm != orig_val) { - pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %u -> %u\n", + pr_warn("prog '%s': relo #%d: unexpected insn #%d (LDIMM64) value: got %llu, exp %llu -> %llu\n", prog_name, relo_idx, insn_idx, (unsigned long long)imm, - orig_val, new_val); + (unsigned long long)orig_val, (unsigned long long)new_val); return -EINVAL; } insn[0].imm = new_val; - insn[1].imm = 0; /* currently only 32-bit values are supported */ - pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %u\n", + insn[1].imm = new_val >> 32; + pr_debug("prog '%s': relo #%d: patched insn #%d (LDIMM64) imm64 %llu -> %llu\n", prog_name, relo_idx, insn_idx, - (unsigned long long)imm, new_val); + (unsigned long long)imm, (unsigned long long)new_val); break; } default: @@ -1055,51 +1153,78 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, * [] () + => @, * where is a C-syntax view of recorded field access, e.g.: x.a[3].b */ -static void bpf_core_dump_spec(const char *prog_name, int level, const struct bpf_core_spec *spec) +int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec) { const struct btf_type *t; - const struct btf_enum *e; const char *s; __u32 type_id; - int i; + int i, len = 0; + +#define append_buf(fmt, args...) \ + ({ \ + int r; \ + r = snprintf(buf, buf_sz, fmt, ##args); \ + len += r; \ + if (r >= buf_sz) \ + r = buf_sz; \ + buf += r; \ + buf_sz -= r; \ + }) type_id = spec->root_type_id; t = btf_type_by_id(spec->btf, type_id); s = btf__name_by_offset(spec->btf, t->name_off); - libbpf_print(level, "[%u] %s %s", type_id, btf_kind_str(t), str_is_empty(s) ? "" : s); + append_buf("<%s> [%u] %s %s", + core_relo_kind_str(spec->relo_kind), + type_id, btf_kind_str(t), str_is_empty(s) ? "" : s); if (core_relo_is_type_based(spec->relo_kind)) - return; + return len; if (core_relo_is_enumval_based(spec->relo_kind)) { t = skip_mods_and_typedefs(spec->btf, type_id, NULL); - e = btf_enum(t) + spec->raw_spec[0]; - s = btf__name_by_offset(spec->btf, e->name_off); + if (btf_is_enum(t)) { + const struct btf_enum *e; + const char *fmt_str; - libbpf_print(level, "::%s = %u", s, e->val); - return; + e = btf_enum(t) + spec->raw_spec[0]; + s = btf__name_by_offset(spec->btf, e->name_off); + fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %d" : "::%s = %u"; + append_buf(fmt_str, s, e->val); + } else { + const struct btf_enum64 *e; + const char *fmt_str; + + e = btf_enum64(t) + spec->raw_spec[0]; + s = btf__name_by_offset(spec->btf, e->name_off); + fmt_str = BTF_INFO_KFLAG(t->info) ? "::%s = %lld" : "::%s = %llu"; + append_buf(fmt_str, s, (unsigned long long)btf_enum64_value(e)); + } + return len; } if (core_relo_is_field_based(spec->relo_kind)) { for (i = 0; i < spec->len; i++) { if (spec->spec[i].name) - libbpf_print(level, ".%s", spec->spec[i].name); + append_buf(".%s", spec->spec[i].name); else if (i > 0 || spec->spec[i].idx > 0) - libbpf_print(level, "[%u]", spec->spec[i].idx); + append_buf("[%u]", spec->spec[i].idx); } - libbpf_print(level, " ("); + append_buf(" ("); for (i = 0; i < spec->raw_len; i++) - libbpf_print(level, "%s%d", i == 0 ? "" : ":", spec->raw_spec[i]); + append_buf("%s%d", i == 0 ? "" : ":", spec->raw_spec[i]); if (spec->bit_offset % 8) - libbpf_print(level, " @ offset %u.%u)", - spec->bit_offset / 8, spec->bit_offset % 8); + append_buf(" @ offset %u.%u)", spec->bit_offset / 8, spec->bit_offset % 8); else - libbpf_print(level, " @ offset %u)", spec->bit_offset / 8); - return; + append_buf(" @ offset %u)", spec->bit_offset / 8); + return len; } + + return len; +#undef append_buf } /* @@ -1134,11 +1259,11 @@ static void bpf_core_dump_spec(const char *prog_name, int level, const struct bp * 3. It is supported and expected that there might be multiple flavors * matching the spec. As long as all the specs resolve to the same set of * offsets across all candidates, there is no error. If there is any - * ambiguity, CO-RE relocation will fail. This is necessary to accomodate - * imprefection of BTF deduplication, which can cause slight duplication of + * ambiguity, CO-RE relocation will fail. This is necessary to accommodate + * imperfection of BTF deduplication, which can cause slight duplication of * the same BTF type, if some directly or indirectly referenced (by * pointer) type gets resolved to different actual types in different - * object files. If such situation occurs, deduplicated BTF will end up + * object files. If such a situation occurs, deduplicated BTF will end up * with two (or more) structurally identical types, which differ only in * types they refer to through pointer. This should be OK in most cases and * is not an error. @@ -1167,7 +1292,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, const struct btf_type *local_type; const char *local_name; __u32 local_id; - const char *spec_str; + char spec_buf[256]; int i, j, err; local_id = relo->type_id; @@ -1176,24 +1301,20 @@ int bpf_core_calc_relo_insn(const char *prog_name, if (!local_name) return -EINVAL; - spec_str = btf__name_by_offset(local_btf, relo->access_str_off); - if (str_is_empty(spec_str)) - return -EINVAL; - - err = bpf_core_parse_spec(prog_name, local_btf, local_id, spec_str, - relo->kind, local_spec); + err = bpf_core_parse_spec(prog_name, local_btf, relo, local_spec); if (err) { + const char *spec_str; + + spec_str = btf__name_by_offset(local_btf, relo->access_str_off); pr_warn("prog '%s': relo #%d: parsing [%d] %s %s + %s failed: %d\n", prog_name, relo_idx, local_id, btf_kind_str(local_type), str_is_empty(local_name) ? "" : local_name, - spec_str, err); + spec_str ?: "", err); return -EINVAL; } - pr_debug("prog '%s': relo #%d: kind <%s> (%d), spec is ", prog_name, - relo_idx, core_relo_kind_str(relo->kind), relo->kind); - bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, local_spec); - libbpf_print(LIBBPF_DEBUG, "\n"); + bpf_core_format_spec(spec_buf, sizeof(spec_buf), local_spec); + pr_debug("prog '%s': relo #%d: %s\n", prog_name, relo_idx, spec_buf); /* TYPE_ID_LOCAL relo is special and doesn't need candidate search */ if (relo->kind == BPF_CORE_TYPE_ID_LOCAL) { @@ -1207,7 +1328,7 @@ int bpf_core_calc_relo_insn(const char *prog_name, } /* libbpf doesn't support candidate search for anonymous types */ - if (str_is_empty(spec_str)) { + if (str_is_empty(local_name)) { pr_warn("prog '%s': relo #%d: <%s> (%d) relocation doesn't support anonymous types\n", prog_name, relo_idx, core_relo_kind_str(relo->kind), relo->kind); return -EOPNOTSUPP; @@ -1217,17 +1338,15 @@ int bpf_core_calc_relo_insn(const char *prog_name, err = bpf_core_spec_match(local_spec, cands->cands[i].btf, cands->cands[i].id, cand_spec); if (err < 0) { - pr_warn("prog '%s': relo #%d: error matching candidate #%d ", - prog_name, relo_idx, i); - bpf_core_dump_spec(prog_name, LIBBPF_WARN, cand_spec); - libbpf_print(LIBBPF_WARN, ": %d\n", err); + bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec); + pr_warn("prog '%s': relo #%d: error matching candidate #%d %s: %d\n ", + prog_name, relo_idx, i, spec_buf, err); return err; } - pr_debug("prog '%s': relo #%d: %s candidate #%d ", prog_name, - relo_idx, err == 0 ? "non-matching" : "matching", i); - bpf_core_dump_spec(prog_name, LIBBPF_DEBUG, cand_spec); - libbpf_print(LIBBPF_DEBUG, "\n"); + bpf_core_format_spec(spec_buf, sizeof(spec_buf), cand_spec); + pr_debug("prog '%s': relo #%d: %s candidate #%d %s\n", prog_name, + relo_idx, err == 0 ? "non-matching" : "matching", i, spec_buf); if (err == 0) continue; @@ -1253,10 +1372,12 @@ int bpf_core_calc_relo_insn(const char *prog_name, * decision and value, otherwise it's dangerous to * proceed due to ambiguity */ - pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %u != %s %u\n", + pr_warn("prog '%s': relo #%d: relocation decision ambiguity: %s %llu != %s %llu\n", prog_name, relo_idx, - cand_res.poison ? "failure" : "success", cand_res.new_val, - targ_res->poison ? "failure" : "success", targ_res->new_val); + cand_res.poison ? "failure" : "success", + (unsigned long long)cand_res.new_val, + targ_res->poison ? "failure" : "success", + (unsigned long long)targ_res->new_val); return -EINVAL; } @@ -1297,3 +1418,273 @@ int bpf_core_calc_relo_insn(const char *prog_name, return 0; } + +static bool bpf_core_names_match(const struct btf *local_btf, size_t local_name_off, + const struct btf *targ_btf, size_t targ_name_off) +{ + const char *local_n, *targ_n; + size_t local_len, targ_len; + + local_n = btf__name_by_offset(local_btf, local_name_off); + targ_n = btf__name_by_offset(targ_btf, targ_name_off); + + if (str_is_empty(targ_n)) + return str_is_empty(local_n); + + targ_len = bpf_core_essential_name_len(targ_n); + local_len = bpf_core_essential_name_len(local_n); + + return targ_len == local_len && strncmp(local_n, targ_n, local_len) == 0; +} + +static int bpf_core_enums_match(const struct btf *local_btf, const struct btf_type *local_t, + const struct btf *targ_btf, const struct btf_type *targ_t) +{ + __u16 local_vlen = btf_vlen(local_t); + __u16 targ_vlen = btf_vlen(targ_t); + int i, j; + + if (local_t->size != targ_t->size) + return 0; + + if (local_vlen > targ_vlen) + return 0; + + /* iterate over the local enum's variants and make sure each has + * a symbolic name correspondent in the target + */ + for (i = 0; i < local_vlen; i++) { + bool matched = false; + __u32 local_n_off, targ_n_off; + + local_n_off = btf_is_enum(local_t) ? btf_enum(local_t)[i].name_off : + btf_enum64(local_t)[i].name_off; + + for (j = 0; j < targ_vlen; j++) { + targ_n_off = btf_is_enum(targ_t) ? btf_enum(targ_t)[j].name_off : + btf_enum64(targ_t)[j].name_off; + + if (bpf_core_names_match(local_btf, local_n_off, targ_btf, targ_n_off)) { + matched = true; + break; + } + } + + if (!matched) + return 0; + } + return 1; +} + +static int bpf_core_composites_match(const struct btf *local_btf, const struct btf_type *local_t, + const struct btf *targ_btf, const struct btf_type *targ_t, + bool behind_ptr, int level) +{ + const struct btf_member *local_m = btf_members(local_t); + __u16 local_vlen = btf_vlen(local_t); + __u16 targ_vlen = btf_vlen(targ_t); + int i, j, err; + + if (local_vlen > targ_vlen) + return 0; + + /* check that all local members have a match in the target */ + for (i = 0; i < local_vlen; i++, local_m++) { + const struct btf_member *targ_m = btf_members(targ_t); + bool matched = false; + + for (j = 0; j < targ_vlen; j++, targ_m++) { + if (!bpf_core_names_match(local_btf, local_m->name_off, + targ_btf, targ_m->name_off)) + continue; + + err = __bpf_core_types_match(local_btf, local_m->type, targ_btf, + targ_m->type, behind_ptr, level - 1); + if (err < 0) + return err; + if (err > 0) { + matched = true; + break; + } + } + + if (!matched) + return 0; + } + return 1; +} + +/* Check that two types "match". This function assumes that root types were + * already checked for name match. + * + * The matching relation is defined as follows: + * - modifiers and typedefs are stripped (and, hence, effectively ignored) + * - generally speaking types need to be of same kind (struct vs. struct, union + * vs. union, etc.) + * - exceptions are struct/union behind a pointer which could also match a + * forward declaration of a struct or union, respectively, and enum vs. + * enum64 (see below) + * Then, depending on type: + * - integers: + * - match if size and signedness match + * - arrays & pointers: + * - target types are recursively matched + * - structs & unions: + * - local members need to exist in target with the same name + * - for each member we recursively check match unless it is already behind a + * pointer, in which case we only check matching names and compatible kind + * - enums: + * - local variants have to have a match in target by symbolic name (but not + * numeric value) + * - size has to match (but enum may match enum64 and vice versa) + * - function pointers: + * - number and position of arguments in local type has to match target + * - for each argument and the return value we recursively check match + */ +int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, + __u32 targ_id, bool behind_ptr, int level) +{ + const struct btf_type *local_t, *targ_t; + int depth = 32; /* max recursion depth */ + __u16 local_k, targ_k; + + if (level <= 0) + return -EINVAL; + + local_t = btf_type_by_id(local_btf, local_id); + targ_t = btf_type_by_id(targ_btf, targ_id); + +recur: + depth--; + if (depth < 0) + return -EINVAL; + + local_t = skip_mods_and_typedefs(local_btf, local_id, &local_id); + targ_t = skip_mods_and_typedefs(targ_btf, targ_id, &targ_id); + if (!local_t || !targ_t) + return -EINVAL; + + /* While the name check happens after typedefs are skipped, root-level + * typedefs would still be name-matched as that's the contract with + * callers. + */ + if (!bpf_core_names_match(local_btf, local_t->name_off, targ_btf, targ_t->name_off)) + return 0; + + local_k = btf_kind(local_t); + targ_k = btf_kind(targ_t); + + switch (local_k) { + case BTF_KIND_UNKN: + return local_k == targ_k; + case BTF_KIND_FWD: { + bool local_f = BTF_INFO_KFLAG(local_t->info); + + if (behind_ptr) { + if (local_k == targ_k) + return local_f == BTF_INFO_KFLAG(targ_t->info); + + /* for forward declarations kflag dictates whether the + * target is a struct (0) or union (1) + */ + return (targ_k == BTF_KIND_STRUCT && !local_f) || + (targ_k == BTF_KIND_UNION && local_f); + } else { + if (local_k != targ_k) + return 0; + + /* match if the forward declaration is for the same kind */ + return local_f == BTF_INFO_KFLAG(targ_t->info); + } + } + case BTF_KIND_ENUM: + case BTF_KIND_ENUM64: + if (!btf_is_any_enum(targ_t)) + return 0; + + return bpf_core_enums_match(local_btf, local_t, targ_btf, targ_t); + case BTF_KIND_STRUCT: + case BTF_KIND_UNION: + if (behind_ptr) { + bool targ_f = BTF_INFO_KFLAG(targ_t->info); + + if (local_k == targ_k) + return 1; + + if (targ_k != BTF_KIND_FWD) + return 0; + + return (local_k == BTF_KIND_UNION) == targ_f; + } else { + if (local_k != targ_k) + return 0; + + return bpf_core_composites_match(local_btf, local_t, targ_btf, targ_t, + behind_ptr, level); + } + case BTF_KIND_INT: { + __u8 local_sgn; + __u8 targ_sgn; + + if (local_k != targ_k) + return 0; + + local_sgn = btf_int_encoding(local_t) & BTF_INT_SIGNED; + targ_sgn = btf_int_encoding(targ_t) & BTF_INT_SIGNED; + + return local_t->size == targ_t->size && local_sgn == targ_sgn; + } + case BTF_KIND_PTR: + if (local_k != targ_k) + return 0; + + behind_ptr = true; + + local_id = local_t->type; + targ_id = targ_t->type; + goto recur; + case BTF_KIND_ARRAY: { + const struct btf_array *local_array = btf_array(local_t); + const struct btf_array *targ_array = btf_array(targ_t); + + if (local_k != targ_k) + return 0; + + if (local_array->nelems != targ_array->nelems) + return 0; + + local_id = local_array->type; + targ_id = targ_array->type; + goto recur; + } + case BTF_KIND_FUNC_PROTO: { + struct btf_param *local_p = btf_params(local_t); + struct btf_param *targ_p = btf_params(targ_t); + __u16 local_vlen = btf_vlen(local_t); + __u16 targ_vlen = btf_vlen(targ_t); + int i, err; + + if (local_k != targ_k) + return 0; + + if (local_vlen != targ_vlen) + return 0; + + for (i = 0; i < local_vlen; i++, local_p++, targ_p++) { + err = __bpf_core_types_match(local_btf, local_p->type, targ_btf, + targ_p->type, behind_ptr, level - 1); + if (err <= 0) + return err; + } + + /* tail recurse for return type check */ + local_id = local_t->type; + targ_id = targ_t->type; + goto recur; + } + default: + pr_warn("unexpected kind %s relocated, local [%d], target [%d]\n", + btf_kind_str(local_t), local_id, targ_id); + return 0; + } +} diff --git a/tools/lib/bpf/relo_core.h b/tools/lib/bpf/relo_core.h index a28bf3711c..1c0566daf8 100644 --- a/tools/lib/bpf/relo_core.h +++ b/tools/lib/bpf/relo_core.h @@ -46,9 +46,9 @@ struct bpf_core_spec { struct bpf_core_relo_res { /* expected value in the instruction, unless validate == false */ - __u32 orig_val; + __u64 orig_val; /* new value that needs to be patched up to */ - __u32 new_val; + __u64 new_val; /* relocation unsuccessful, poison instruction, but don't fail load */ bool poison; /* some relocations can't be validated against orig_val */ @@ -68,8 +68,14 @@ struct bpf_core_relo_res { __u32 new_type_id; }; +int __bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, + const struct btf *targ_btf, __u32 targ_id, int level); int bpf_core_types_are_compat(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, __u32 targ_id); +int __bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, + __u32 targ_id, bool behind_ptr, int level); +int bpf_core_types_match(const struct btf *local_btf, __u32 local_id, const struct btf *targ_btf, + __u32 targ_id); size_t bpf_core_essential_name_len(const char *name); @@ -84,4 +90,10 @@ int bpf_core_patch_insn(const char *prog_name, struct bpf_insn *insn, int insn_idx, const struct bpf_core_relo *relo, int relo_idx, const struct bpf_core_relo_res *res); +int bpf_core_parse_spec(const char *prog_name, const struct btf *btf, + const struct bpf_core_relo *relo, + struct bpf_core_spec *spec); + +int bpf_core_format_spec(char *buf, size_t buf_sz, const struct bpf_core_spec *spec); + #endif diff --git a/tools/lib/bpf/skel_internal.h b/tools/lib/bpf/skel_internal.h index bd6f4505e7..70adf7b119 100644 --- a/tools/lib/bpf/skel_internal.h +++ b/tools/lib/bpf/skel_internal.h @@ -66,13 +66,13 @@ struct bpf_load_and_run_opts { const char *errstr; }; -long bpf_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); +long kern_sys_bpf(__u32 cmd, void *attr, __u32 attr_size); static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, unsigned int size) { #ifdef __KERNEL__ - return bpf_sys_bpf(cmd, attr, size); + return kern_sys_bpf(cmd, attr, size); #else return syscall(__NR_bpf, cmd, attr, size); #endif diff --git a/tools/lib/perf/cpumap.c b/tools/lib/perf/cpumap.c index 384d5e076e..6cd0be7c1b 100644 --- a/tools/lib/perf/cpumap.c +++ b/tools/lib/perf/cpumap.c @@ -309,7 +309,7 @@ bool perf_cpu_map__has(const struct perf_cpu_map *cpus, struct perf_cpu cpu) return perf_cpu_map__idx(cpus, cpu) != -1; } -struct perf_cpu perf_cpu_map__max(struct perf_cpu_map *map) +struct perf_cpu perf_cpu_map__max(const struct perf_cpu_map *map) { struct perf_cpu result = { .cpu = -1 diff --git a/tools/lib/perf/evlist.c b/tools/lib/perf/evlist.c index a09315538a..e6c98a6e39 100644 --- a/tools/lib/perf/evlist.c +++ b/tools/lib/perf/evlist.c @@ -23,6 +23,7 @@ #include #include #include +#include "internal.h" void perf_evlist__init(struct perf_evlist *evlist) { @@ -39,10 +40,11 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, * We already have cpus for evsel (via PMU sysfs) so * keep it, if there's no target cpu list defined. */ - if (!evsel->own_cpus || evlist->has_user_cpus) { - perf_cpu_map__put(evsel->cpus); - evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus); - } else if (!evsel->system_wide && perf_cpu_map__empty(evlist->user_requested_cpus)) { + if (!evsel->own_cpus || + (!evsel->system_wide && evlist->has_user_cpus) || + (!evsel->system_wide && + !evsel->requires_cpu && + perf_cpu_map__empty(evlist->user_requested_cpus))) { perf_cpu_map__put(evsel->cpus); evsel->cpus = perf_cpu_map__get(evlist->user_requested_cpus); } else if (evsel->cpus != evsel->own_cpus) { @@ -50,8 +52,11 @@ static void __perf_evlist__propagate_maps(struct perf_evlist *evlist, evsel->cpus = perf_cpu_map__get(evsel->own_cpus); } - perf_thread_map__put(evsel->threads); - evsel->threads = perf_thread_map__get(evlist->threads); + if (!evsel->system_wide) { + perf_thread_map__put(evsel->threads); + evsel->threads = perf_thread_map__get(evlist->threads); + } + evlist->all_cpus = perf_cpu_map__merge(evlist->all_cpus, evsel->cpus); } @@ -59,6 +64,10 @@ static void perf_evlist__propagate_maps(struct perf_evlist *evlist) { struct perf_evsel *evsel; + /* Recomputing all_cpus, so start with a blank slate. */ + perf_cpu_map__put(evlist->all_cpus); + evlist->all_cpus = NULL; + perf_evlist__for_each_evsel(evlist, evsel) __perf_evlist__propagate_maps(evlist, evsel); } @@ -294,7 +303,7 @@ int perf_evlist__id_add_fd(struct perf_evlist *evlist, int perf_evlist__alloc_pollfd(struct perf_evlist *evlist) { - int nr_cpus = perf_cpu_map__nr(evlist->user_requested_cpus); + int nr_cpus = perf_cpu_map__nr(evlist->all_cpus); int nr_threads = perf_thread_map__nr(evlist->threads); int nfds = 0; struct perf_evsel *evsel; @@ -424,9 +433,9 @@ static void perf_evlist__set_mmap_first(struct perf_evlist *evlist, struct perf_ static int mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, int idx, struct perf_mmap_param *mp, int cpu_idx, - int thread, int *_output, int *_output_overwrite) + int thread, int *_output, int *_output_overwrite, int *nr_mmaps) { - struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->user_requested_cpus, cpu_idx); + struct perf_cpu evlist_cpu = perf_cpu_map__cpu(evlist->all_cpus, cpu_idx); struct perf_evsel *evsel; int revent; @@ -474,9 +483,14 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, */ refcount_set(&map->refcnt, 2); + if (ops->idx) + ops->idx(evlist, evsel, mp, idx); + if (ops->mmap(map, mp, *output, evlist_cpu) < 0) return -1; + *nr_mmaps += 1; + if (!idx) perf_evlist__set_mmap_first(evlist, map, overwrite); } else { @@ -505,54 +519,29 @@ mmap_per_evsel(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, return 0; } -static int -mmap_per_thread(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, - struct perf_mmap_param *mp) -{ - int thread; - int nr_threads = perf_thread_map__nr(evlist->threads); - - for (thread = 0; thread < nr_threads; thread++) { - int output = -1; - int output_overwrite = -1; - - if (ops->idx) - ops->idx(evlist, mp, thread, false); - - if (mmap_per_evsel(evlist, ops, thread, mp, 0, thread, - &output, &output_overwrite)) - goto out_unmap; - } - - return 0; - -out_unmap: - perf_evlist__munmap(evlist); - return -1; -} - static int mmap_per_cpu(struct perf_evlist *evlist, struct perf_evlist_mmap_ops *ops, struct perf_mmap_param *mp) { int nr_threads = perf_thread_map__nr(evlist->threads); - int nr_cpus = perf_cpu_map__nr(evlist->user_requested_cpus); + int nr_cpus = perf_cpu_map__nr(evlist->all_cpus); + int nr_mmaps = 0; int cpu, thread; for (cpu = 0; cpu < nr_cpus; cpu++) { int output = -1; int output_overwrite = -1; - if (ops->idx) - ops->idx(evlist, mp, cpu, true); - for (thread = 0; thread < nr_threads; thread++) { if (mmap_per_evsel(evlist, ops, cpu, mp, cpu, - thread, &output, &output_overwrite)) + thread, &output, &output_overwrite, &nr_mmaps)) goto out_unmap; } } + if (nr_mmaps != evlist->nr_mmaps) + pr_err("Miscounted nr_mmaps %d vs %d\n", nr_mmaps, evlist->nr_mmaps); + return 0; out_unmap: @@ -564,9 +553,14 @@ static int perf_evlist__nr_mmaps(struct perf_evlist *evlist) { int nr_mmaps; - nr_mmaps = perf_cpu_map__nr(evlist->user_requested_cpus); - if (perf_cpu_map__empty(evlist->user_requested_cpus)) - nr_mmaps = perf_thread_map__nr(evlist->threads); + /* One for each CPU */ + nr_mmaps = perf_cpu_map__nr(evlist->all_cpus); + if (perf_cpu_map__empty(evlist->all_cpus)) { + /* Plus one for each thread */ + nr_mmaps += perf_thread_map__nr(evlist->threads); + /* Minus the per-thread CPU (-1) */ + nr_mmaps -= 1; + } return nr_mmaps; } @@ -576,7 +570,6 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist, struct perf_mmap_param *mp) { struct perf_evsel *evsel; - const struct perf_cpu_map *cpus = evlist->user_requested_cpus; if (!ops || !ops->get || !ops->mmap) return -EINVAL; @@ -595,9 +588,6 @@ int perf_evlist__mmap_ops(struct perf_evlist *evlist, if (evlist->pollfd.entries == NULL && perf_evlist__alloc_pollfd(evlist) < 0) return -ENOMEM; - if (perf_cpu_map__empty(cpus)) - return mmap_per_thread(evlist, ops, mp); - return mmap_per_cpu(evlist, ops, mp); } diff --git a/tools/lib/perf/evsel.c b/tools/lib/perf/evsel.c index 210ea7c06c..8ce5bbd096 100644 --- a/tools/lib/perf/evsel.c +++ b/tools/lib/perf/evsel.c @@ -149,23 +149,30 @@ int perf_evsel__open(struct perf_evsel *evsel, struct perf_cpu_map *cpus, int fd, group_fd, *evsel_fd; evsel_fd = FD(evsel, idx, thread); - if (evsel_fd == NULL) - return -EINVAL; + if (evsel_fd == NULL) { + err = -EINVAL; + goto out; + } err = get_group_fd(evsel, idx, thread, &group_fd); if (err < 0) - return err; + goto out; fd = sys_perf_event_open(&evsel->attr, threads->map[thread].pid, cpu, group_fd, 0); - if (fd < 0) - return -errno; + if (fd < 0) { + err = -errno; + goto out; + } *evsel_fd = fd; } } +out: + if (err) + perf_evsel__close(evsel); return err; } @@ -298,6 +305,9 @@ int perf_evsel__read_size(struct perf_evsel *evsel) if (read_format & PERF_FORMAT_ID) entry += sizeof(u64); + if (read_format & PERF_FORMAT_LOST) + entry += sizeof(u64); + if (read_format & PERF_FORMAT_GROUP) { nr = evsel->nr_members; size += sizeof(u64); @@ -307,27 +317,112 @@ int perf_evsel__read_size(struct perf_evsel *evsel) return size; } +/* This only reads values for the leader */ +static int perf_evsel__read_group(struct perf_evsel *evsel, int cpu_map_idx, + int thread, struct perf_counts_values *count) +{ + size_t size = perf_evsel__read_size(evsel); + int *fd = FD(evsel, cpu_map_idx, thread); + u64 read_format = evsel->attr.read_format; + u64 *data; + int idx = 1; + + if (fd == NULL || *fd < 0) + return -EINVAL; + + data = calloc(1, size); + if (data == NULL) + return -ENOMEM; + + if (readn(*fd, data, size) <= 0) { + free(data); + return -errno; + } + + /* + * This reads only the leader event intentionally since we don't have + * perf counts values for sibling events. + */ + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + count->ena = data[idx++]; + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + count->run = data[idx++]; + + /* value is always available */ + count->val = data[idx++]; + if (read_format & PERF_FORMAT_ID) + count->id = data[idx++]; + if (read_format & PERF_FORMAT_LOST) + count->lost = data[idx++]; + + free(data); + return 0; +} + +/* + * The perf read format is very flexible. It needs to set the proper + * values according to the read format. + */ +static void perf_evsel__adjust_values(struct perf_evsel *evsel, u64 *buf, + struct perf_counts_values *count) +{ + u64 read_format = evsel->attr.read_format; + int n = 0; + + count->val = buf[n++]; + + if (read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + count->ena = buf[n++]; + + if (read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + count->run = buf[n++]; + + if (read_format & PERF_FORMAT_ID) + count->id = buf[n++]; + + if (read_format & PERF_FORMAT_LOST) + count->lost = buf[n++]; +} + int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int thread, struct perf_counts_values *count) { size_t size = perf_evsel__read_size(evsel); int *fd = FD(evsel, cpu_map_idx, thread); + u64 read_format = evsel->attr.read_format; + struct perf_counts_values buf; memset(count, 0, sizeof(*count)); if (fd == NULL || *fd < 0) return -EINVAL; + if (read_format & PERF_FORMAT_GROUP) + return perf_evsel__read_group(evsel, cpu_map_idx, thread, count); + if (MMAP(evsel, cpu_map_idx, thread) && + !(read_format & (PERF_FORMAT_ID | PERF_FORMAT_LOST)) && !perf_mmap__read_self(MMAP(evsel, cpu_map_idx, thread), count)) return 0; - if (readn(*fd, count->values, size) <= 0) + if (readn(*fd, buf.values, size) <= 0) return -errno; + perf_evsel__adjust_values(evsel, buf.values, count); return 0; } +static int perf_evsel__ioctl(struct perf_evsel *evsel, int ioc, void *arg, + int cpu_map_idx, int thread) +{ + int *fd = FD(evsel, cpu_map_idx, thread); + + if (fd == NULL || *fd < 0) + return -1; + + return ioctl(*fd, ioc, arg); +} + static int perf_evsel__run_ioctl(struct perf_evsel *evsel, int ioc, void *arg, int cpu_map_idx) @@ -335,13 +430,7 @@ static int perf_evsel__run_ioctl(struct perf_evsel *evsel, int thread; for (thread = 0; thread < xyarray__max_y(evsel->fd); thread++) { - int err; - int *fd = FD(evsel, cpu_map_idx, thread); - - if (fd == NULL || *fd < 0) - return -1; - - err = ioctl(*fd, ioc, arg); + int err = perf_evsel__ioctl(evsel, ioc, arg, cpu_map_idx, thread); if (err) return err; @@ -355,6 +444,21 @@ int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx) return perf_evsel__run_ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, cpu_map_idx); } +int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread) +{ + struct perf_cpu cpu __maybe_unused; + int idx; + int err; + + perf_cpu_map__for_each_cpu(cpu, idx, evsel->cpus) { + err = perf_evsel__ioctl(evsel, PERF_EVENT_IOC_ENABLE, NULL, idx, thread); + if (err) + return err; + } + + return 0; +} + int perf_evsel__enable(struct perf_evsel *evsel) { int i; diff --git a/tools/lib/perf/include/internal/evlist.h b/tools/lib/perf/include/internal/evlist.h index e3e64f37db..6f89aec3e6 100644 --- a/tools/lib/perf/include/internal/evlist.h +++ b/tools/lib/perf/include/internal/evlist.h @@ -38,7 +38,8 @@ struct perf_evlist { }; typedef void -(*perf_evlist_mmap__cb_idx_t)(struct perf_evlist*, struct perf_mmap_param*, int, bool); +(*perf_evlist_mmap__cb_idx_t)(struct perf_evlist*, struct perf_evsel*, + struct perf_mmap_param*, int); typedef struct perf_mmap* (*perf_evlist_mmap__cb_get_t)(struct perf_evlist*, bool, int); typedef int diff --git a/tools/lib/perf/include/internal/evsel.h b/tools/lib/perf/include/internal/evsel.h index cfc9ebd796..a99a75d9e7 100644 --- a/tools/lib/perf/include/internal/evsel.h +++ b/tools/lib/perf/include/internal/evsel.h @@ -30,6 +30,10 @@ struct perf_sample_id { struct perf_cpu cpu; pid_t tid; + /* Guest machine pid and VCPU, valid only if machine_pid is non-zero */ + pid_t machine_pid; + struct perf_cpu vcpu; + /* Holds total ID period value for PERF_SAMPLE_READ processing. */ u64 period; }; @@ -49,7 +53,18 @@ struct perf_evsel { /* parse modifier helper */ int nr_members; + /* + * system_wide is for events that need to be on every CPU, irrespective + * of user requested CPUs or threads. Map propagation will set cpus to + * this event's own_cpus, whereby they will contribute to evlist + * all_cpus. + */ bool system_wide; + /* + * Some events, for example uncore events, require a CPU. + * i.e. it cannot be the 'any CPU' value of -1. + */ + bool requires_cpu; int idx; }; diff --git a/tools/lib/perf/include/internal/lib.h b/tools/lib/perf/include/internal/lib.h index 5175d491b2..85471a4b90 100644 --- a/tools/lib/perf/include/internal/lib.h +++ b/tools/lib/perf/include/internal/lib.h @@ -9,4 +9,6 @@ extern unsigned int page_size; ssize_t readn(int fd, void *buf, size_t n); ssize_t writen(int fd, const void *buf, size_t n); +ssize_t preadn(int fd, void *buf, size_t n, off_t offs); + #endif /* __LIBPERF_INTERNAL_CPUMAP_H */ diff --git a/tools/lib/perf/include/perf/cpumap.h b/tools/lib/perf/include/perf/cpumap.h index 4a2edbdb5e..03aceb72a7 100644 --- a/tools/lib/perf/include/perf/cpumap.h +++ b/tools/lib/perf/include/perf/cpumap.h @@ -23,7 +23,7 @@ LIBPERF_API void perf_cpu_map__put(struct perf_cpu_map *map); LIBPERF_API struct perf_cpu perf_cpu_map__cpu(const struct perf_cpu_map *cpus, int idx); LIBPERF_API int perf_cpu_map__nr(const struct perf_cpu_map *cpus); LIBPERF_API bool perf_cpu_map__empty(const struct perf_cpu_map *map); -LIBPERF_API struct perf_cpu perf_cpu_map__max(struct perf_cpu_map *map); +LIBPERF_API struct perf_cpu perf_cpu_map__max(const struct perf_cpu_map *map); LIBPERF_API bool perf_cpu_map__has(const struct perf_cpu_map *map, struct perf_cpu cpu); #define perf_cpu_map__for_each_cpu(cpu, idx, cpus) \ @@ -31,4 +31,7 @@ LIBPERF_API bool perf_cpu_map__has(const struct perf_cpu_map *map, struct perf_c (idx) < perf_cpu_map__nr(cpus); \ (idx)++, (cpu) = perf_cpu_map__cpu(cpus, idx)) +#define perf_cpu_map__for_each_idx(idx, cpus) \ + for ((idx) = 0; (idx) < perf_cpu_map__nr(cpus); (idx)++) + #endif /* __LIBPERF_CPUMAP_H */ diff --git a/tools/lib/perf/include/perf/event.h b/tools/lib/perf/include/perf/event.h index e7758707ca..93bf93a59c 100644 --- a/tools/lib/perf/include/perf/event.h +++ b/tools/lib/perf/include/perf/event.h @@ -6,6 +6,7 @@ #include #include #include +#include #include /* pid_t */ #define event_contains(obj, mem) ((obj).header.size > offsetof(typeof(obj), mem)) @@ -76,7 +77,7 @@ struct perf_record_lost_samples { }; /* - * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID + * PERF_FORMAT_ENABLED | PERF_FORMAT_RUNNING | PERF_FORMAT_ID | PERF_FORMAT_LOST */ struct perf_record_read { struct perf_event_header header; @@ -85,6 +86,7 @@ struct perf_record_read { __u64 time_enabled; __u64 time_running; __u64 id; + __u64 lost; }; struct perf_record_throttle { @@ -153,22 +155,60 @@ enum { PERF_CPU_MAP__MASK = 1, }; +/* + * Array encoding of a perf_cpu_map where nr is the number of entries in cpu[] + * and each entry is a value for a CPU in the map. + */ struct cpu_map_entries { __u16 nr; __u16 cpu[]; }; -struct perf_record_record_cpu_map { +/* Bitmap encoding of a perf_cpu_map where bitmap entries are 32-bit. */ +struct perf_record_mask_cpu_map32 { + /* Number of mask values. */ __u16 nr; + /* Constant 4. */ __u16 long_size; - unsigned long mask[]; + /* Bitmap data. */ + __u32 mask[]; }; -struct perf_record_cpu_map_data { - __u16 type; - char data[]; +/* Bitmap encoding of a perf_cpu_map where bitmap entries are 64-bit. */ +struct perf_record_mask_cpu_map64 { + /* Number of mask values. */ + __u16 nr; + /* Constant 8. */ + __u16 long_size; + /* Legacy padding. */ + char __pad[4]; + /* Bitmap data. */ + __u64 mask[]; }; +/* + * 'struct perf_record_cpu_map_data' is packed as unfortunately an earlier + * version had unaligned data and we wish to retain file format compatibility. + * -irogers + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpacked" +#pragma GCC diagnostic ignored "-Wattributes" + +struct __packed perf_record_cpu_map_data { + __u16 type; + union { + /* Used when type == PERF_CPU_MAP__CPUS. */ + struct cpu_map_entries cpus_data; + /* Used when type == PERF_CPU_MAP__MASK and long_size == 4. */ + struct perf_record_mask_cpu_map32 mask32_data; + /* Used when type == PERF_CPU_MAP__MASK and long_size == 8. */ + struct perf_record_mask_cpu_map64 mask64_data; + }; +}; + +#pragma GCC diagnostic pop + struct perf_record_cpu_map { struct perf_event_header header; struct perf_record_cpu_map_data data; @@ -237,6 +277,11 @@ struct id_index_entry { __u64 tid; }; +struct id_index_entry_2 { + __u64 machine_pid; + __u64 vcpu; +}; + struct perf_record_id_index { struct perf_event_header header; __u64 nr; @@ -274,6 +319,8 @@ struct perf_record_auxtrace_error { __u64 ip; __u64 time; char msg[MAX_AUXTRACE_ERROR_MSG]; + __u32 machine_pid; + __u32 vcpu; }; struct perf_record_aux { @@ -389,6 +436,7 @@ enum perf_user_event_type { /* above any possible kernel type */ PERF_RECORD_TIME_CONV = 79, PERF_RECORD_HEADER_FEATURE = 80, PERF_RECORD_COMPRESSED = 81, + PERF_RECORD_FINISHED_INIT = 82, PERF_RECORD_HEADER_MAX }; diff --git a/tools/lib/perf/include/perf/evsel.h b/tools/lib/perf/include/perf/evsel.h index 2a9516b42d..6f92204075 100644 --- a/tools/lib/perf/include/perf/evsel.h +++ b/tools/lib/perf/include/perf/evsel.h @@ -18,8 +18,10 @@ struct perf_counts_values { uint64_t val; uint64_t ena; uint64_t run; + uint64_t id; + uint64_t lost; }; - uint64_t values[3]; + uint64_t values[5]; }; }; @@ -36,6 +38,7 @@ LIBPERF_API int perf_evsel__read(struct perf_evsel *evsel, int cpu_map_idx, int struct perf_counts_values *count); LIBPERF_API int perf_evsel__enable(struct perf_evsel *evsel); LIBPERF_API int perf_evsel__enable_cpu(struct perf_evsel *evsel, int cpu_map_idx); +LIBPERF_API int perf_evsel__enable_thread(struct perf_evsel *evsel, int thread); LIBPERF_API int perf_evsel__disable(struct perf_evsel *evsel); LIBPERF_API int perf_evsel__disable_cpu(struct perf_evsel *evsel, int cpu_map_idx); LIBPERF_API struct perf_cpu_map *perf_evsel__cpus(struct perf_evsel *evsel); diff --git a/tools/lib/perf/lib.c b/tools/lib/perf/lib.c index 18658931fc..696fb0ea67 100644 --- a/tools/lib/perf/lib.c +++ b/tools/lib/perf/lib.c @@ -38,6 +38,26 @@ ssize_t readn(int fd, void *buf, size_t n) return ion(true, fd, buf, n); } +ssize_t preadn(int fd, void *buf, size_t n, off_t offs) +{ + size_t left = n; + + while (left) { + ssize_t ret = pread(fd, buf, left, offs); + + if (ret < 0 && errno == EINTR) + continue; + if (ret <= 0) + return ret; + + left -= ret; + buf += ret; + offs += ret; + } + + return n; +} + /* * Write exactly 'n' bytes or return an error. */ diff --git a/tools/lib/perf/tests/test-evsel.c b/tools/lib/perf/tests/test-evsel.c index 89be89afb2..a11fc51bfb 100644 --- a/tools/lib/perf/tests/test-evsel.c +++ b/tools/lib/perf/tests/test-evsel.c @@ -1,10 +1,13 @@ // SPDX-License-Identifier: GPL-2.0 #include #include +#include #include +#include #include #include #include +#include #include #include "tests.h" @@ -189,6 +192,163 @@ static int test_stat_user_read(int event) return 0; } +static int test_stat_read_format_single(struct perf_event_attr *attr, struct perf_thread_map *threads) +{ + struct perf_evsel *evsel; + struct perf_counts_values counts; + volatile int count = 0x100000; + int err; + + evsel = perf_evsel__new(attr); + __T("failed to create evsel", evsel); + + /* skip old kernels that don't support the format */ + err = perf_evsel__open(evsel, NULL, threads); + if (err < 0) + return 0; + + while (count--) ; + + memset(&counts, -1, sizeof(counts)); + perf_evsel__read(evsel, 0, 0, &counts); + + __T("failed to read value", counts.val); + if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + __T("failed to read TOTAL_TIME_ENABLED", counts.ena); + if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + __T("failed to read TOTAL_TIME_RUNNING", counts.run); + if (attr->read_format & PERF_FORMAT_ID) + __T("failed to read ID", counts.id); + if (attr->read_format & PERF_FORMAT_LOST) + __T("failed to read LOST", counts.lost == 0); + + perf_evsel__close(evsel); + perf_evsel__delete(evsel); + return 0; +} + +static int test_stat_read_format_group(struct perf_event_attr *attr, struct perf_thread_map *threads) +{ + struct perf_evsel *leader, *member; + struct perf_counts_values counts; + volatile int count = 0x100000; + int err; + + attr->read_format |= PERF_FORMAT_GROUP; + leader = perf_evsel__new(attr); + __T("failed to create leader", leader); + + attr->read_format &= ~PERF_FORMAT_GROUP; + member = perf_evsel__new(attr); + __T("failed to create member", member); + + member->leader = leader; + leader->nr_members = 2; + + /* skip old kernels that don't support the format */ + err = perf_evsel__open(leader, NULL, threads); + if (err < 0) + return 0; + err = perf_evsel__open(member, NULL, threads); + if (err < 0) + return 0; + + while (count--) ; + + memset(&counts, -1, sizeof(counts)); + perf_evsel__read(leader, 0, 0, &counts); + + __T("failed to read leader value", counts.val); + if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + __T("failed to read leader TOTAL_TIME_ENABLED", counts.ena); + if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + __T("failed to read leader TOTAL_TIME_RUNNING", counts.run); + if (attr->read_format & PERF_FORMAT_ID) + __T("failed to read leader ID", counts.id); + if (attr->read_format & PERF_FORMAT_LOST) + __T("failed to read leader LOST", counts.lost == 0); + + memset(&counts, -1, sizeof(counts)); + perf_evsel__read(member, 0, 0, &counts); + + __T("failed to read member value", counts.val); + if (attr->read_format & PERF_FORMAT_TOTAL_TIME_ENABLED) + __T("failed to read member TOTAL_TIME_ENABLED", counts.ena); + if (attr->read_format & PERF_FORMAT_TOTAL_TIME_RUNNING) + __T("failed to read member TOTAL_TIME_RUNNING", counts.run); + if (attr->read_format & PERF_FORMAT_ID) + __T("failed to read member ID", counts.id); + if (attr->read_format & PERF_FORMAT_LOST) + __T("failed to read member LOST", counts.lost == 0); + + perf_evsel__close(member); + perf_evsel__close(leader); + perf_evsel__delete(member); + perf_evsel__delete(leader); + return 0; +} + +static int test_stat_read_format(void) +{ + struct perf_thread_map *threads; + struct perf_event_attr attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_TASK_CLOCK, + }; + int err, i; + +#define FMT(_fmt) PERF_FORMAT_ ## _fmt +#define FMT_TIME (FMT(TOTAL_TIME_ENABLED) | FMT(TOTAL_TIME_RUNNING)) + + uint64_t test_formats [] = { + 0, + FMT_TIME, + FMT(ID), + FMT(LOST), + FMT_TIME | FMT(ID), + FMT_TIME | FMT(LOST), + FMT_TIME | FMT(ID) | FMT(LOST), + FMT(ID) | FMT(LOST), + }; + +#undef FMT +#undef FMT_TIME + + threads = perf_thread_map__new_dummy(); + __T("failed to create threads", threads); + + perf_thread_map__set_pid(threads, 0, 0); + + for (i = 0; i < (int)ARRAY_SIZE(test_formats); i++) { + attr.read_format = test_formats[i]; + __T_VERBOSE("testing single read with read_format: %lx\n", + (unsigned long)test_formats[i]); + + err = test_stat_read_format_single(&attr, threads); + __T("failed to read single format", err == 0); + } + + perf_thread_map__put(threads); + + threads = perf_thread_map__new_array(2, NULL); + __T("failed to create threads", threads); + + perf_thread_map__set_pid(threads, 0, 0); + perf_thread_map__set_pid(threads, 1, 0); + + for (i = 0; i < (int)ARRAY_SIZE(test_formats); i++) { + attr.read_format = test_formats[i]; + __T_VERBOSE("testing group read with read_format: %lx\n", + (unsigned long)test_formats[i]); + + err = test_stat_read_format_group(&attr, threads); + __T("failed to read group format", err == 0); + } + + perf_thread_map__put(threads); + return 0; +} + int test_evsel(int argc, char **argv) { __T_START; @@ -200,6 +360,7 @@ int test_evsel(int argc, char **argv) test_stat_thread_enable(); test_stat_user_read(PERF_COUNT_HW_INSTRUCTIONS); test_stat_user_read(PERF_COUNT_HW_CPU_CYCLES); + test_stat_read_format(); __T_END; return tests_failed == 0 ? 0 : -1; diff --git a/tools/lib/subcmd/parse-options.c b/tools/lib/subcmd/parse-options.c index 39ebf61920..9fa75943f2 100644 --- a/tools/lib/subcmd/parse-options.c +++ b/tools/lib/subcmd/parse-options.c @@ -806,9 +806,9 @@ static int option__cmp(const void *va, const void *vb) static struct option *options__order(const struct option *opts) { - int nr_opts = 0, len; + int nr_opts = 0, nr_group = 0, len; const struct option *o = opts; - struct option *ordered; + struct option *opt, *ordered, *group; for (o = opts; o->type != OPTION_END; o++) ++nr_opts; @@ -819,7 +819,18 @@ static struct option *options__order(const struct option *opts) goto out; memcpy(ordered, opts, len); - qsort(ordered, nr_opts, sizeof(*o), option__cmp); + /* sort each option group individually */ + for (opt = group = ordered; opt->type != OPTION_END; opt++) { + if (opt->type == OPTION_GROUP) { + qsort(group, nr_group, sizeof(*opt), option__cmp); + group = opt + 1; + nr_group = 0; + continue; + } + nr_group++; + } + qsort(group, nr_group, sizeof(*opt), option__cmp); + out: return ordered; } diff --git a/tools/memory-model/README b/tools/memory-model/README index 9edd402704..dab3890420 100644 --- a/tools/memory-model/README +++ b/tools/memory-model/README @@ -54,7 +54,8 @@ klitmus7 Compatibility Table -- 4.14 7.48 -- 4.15 -- 4.19 7.49 -- 4.20 -- 5.5 7.54 -- - 5.6 -- 7.56 -- + 5.6 -- 5.16 7.56 -- + 5.17 -- 7.56.1 -- ============ ========== diff --git a/tools/objtool/Build b/tools/objtool/Build index b7222d5cc7..33f2ee5a46 100644 --- a/tools/objtool/Build +++ b/tools/objtool/Build @@ -2,17 +2,15 @@ objtool-y += arch/$(SRCARCH)/ objtool-y += weak.o -objtool-$(SUBCMD_CHECK) += check.o -objtool-$(SUBCMD_CHECK) += special.o -objtool-$(SUBCMD_ORC) += check.o -objtool-$(SUBCMD_ORC) += orc_gen.o -objtool-$(SUBCMD_ORC) += orc_dump.o - +objtool-y += check.o +objtool-y += special.o objtool-y += builtin-check.o -objtool-y += builtin-orc.o objtool-y += elf.o objtool-y += objtool.o +objtool-$(BUILD_ORC) += orc_gen.o +objtool-$(BUILD_ORC) += orc_dump.o + objtool-y += libstring.o objtool-y += libctype.o objtool-y += str_error_r.o diff --git a/tools/objtool/Makefile b/tools/objtool/Makefile index 0dbd397f31..a3a9cc24e0 100644 --- a/tools/objtool/Makefile +++ b/tools/objtool/Makefile @@ -19,8 +19,8 @@ LIBSUBCMD = $(LIBSUBCMD_OUTPUT)libsubcmd.a OBJTOOL := $(OUTPUT)objtool OBJTOOL_IN := $(OBJTOOL)-in.o -LIBELF_FLAGS := $(shell pkg-config libelf --cflags 2>/dev/null) -LIBELF_LIBS := $(shell pkg-config libelf --libs 2>/dev/null || echo -lelf) +LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null) +LIBELF_LIBS := $(shell $(HOSTPKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf) all: $(OBJTOOL) @@ -39,15 +39,13 @@ CFLAGS += $(if $(elfshdr),,-DLIBELF_USE_DEPRECATED) AWK = awk -SUBCMD_CHECK := n -SUBCMD_ORC := n +BUILD_ORC := n ifeq ($(SRCARCH),x86) - SUBCMD_CHECK := y - SUBCMD_ORC := y + BUILD_ORC := y endif -export SUBCMD_CHECK SUBCMD_ORC +export BUILD_ORC export srctree OUTPUT CFLAGS SRCARCH AWK include $(srctree)/tools/build/Makefile.include @@ -65,7 +63,7 @@ $(LIBSUBCMD): fixdep FORCE clean: $(call QUIET_CLEAN, objtool) $(RM) $(OBJTOOL) $(Q)find $(OUTPUT) -name '*.o' -delete -o -name '\.*.cmd' -delete -o -name '\.*.d' -delete - $(Q)$(RM) $(OUTPUT)arch/x86/inat-tables.c $(OUTPUT)fixdep + $(Q)$(RM) $(OUTPUT)arch/x86/lib/inat-tables.c $(OUTPUT)fixdep $(LIBSUBCMD) FORCE: diff --git a/tools/objtool/arch/x86/decode.c b/tools/objtool/arch/x86/decode.c index 943cb41cdd..c260006106 100644 --- a/tools/objtool/arch/x86/decode.c +++ b/tools/objtool/arch/x86/decode.c @@ -581,7 +581,7 @@ int arch_decode_instruction(struct objtool_file *file, const struct section *sec break; case 0xc7: /* mov imm, r/m */ - if (!noinstr) + if (!opts.noinstr) break; if (insn.length == 3+4+4 && !strncmp(sec->name, ".init.text", 10)) { @@ -787,3 +787,8 @@ bool arch_is_retpoline(struct symbol *sym) { return !strncmp(sym->name, "__x86_indirect_", 15); } + +bool arch_is_rethunk(struct symbol *sym) +{ + return !strcmp(sym->name, "__x86_return_thunk"); +} diff --git a/tools/objtool/arch/x86/special.c b/tools/objtool/arch/x86/special.c index e707d9bcd1..7c97b73912 100644 --- a/tools/objtool/arch/x86/special.c +++ b/tools/objtool/arch/x86/special.c @@ -20,7 +20,7 @@ void arch_handle_alternative(unsigned short feature, struct special_alt *alt) * find paths that see the STAC but take the NOP instead of * CLAC and the other way around. */ - if (uaccess) + if (opts.uaccess) alt->skip_orig = true; else alt->skip_alt = true; diff --git a/tools/objtool/builtin-check.c b/tools/objtool/builtin-check.c index fc6975ab8b..24fbe803a0 100644 --- a/tools/objtool/builtin-check.c +++ b/tools/objtool/builtin-check.c @@ -3,28 +3,21 @@ * Copyright (C) 2015-2017 Josh Poimboeuf */ -/* - * objtool check: - * - * This command analyzes every .o file and ensures the validity of its stack - * trace metadata. It enforces a set of rules on asm code and C inline - * assembly code so that stack traces can be reliable. - * - * For more information, see tools/objtool/Documentation/stack-validation.txt. - */ - #include #include #include #include #include -bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - lto, vmlinux, mcount, noinstr, backup, sls, dryrun, - ibt; +#define ERROR(format, ...) \ + fprintf(stderr, \ + "error: objtool: " format "\n", \ + ##__VA_ARGS__) + +struct opts opts; static const char * const check_usage[] = { - "objtool check [] file.o", + "objtool [] file.o", NULL, }; @@ -33,22 +26,66 @@ static const char * const env_usage[] = { NULL, }; +static int parse_dump(const struct option *opt, const char *str, int unset) +{ + if (!str || !strcmp(str, "orc")) { + opts.dump_orc = true; + return 0; + } + + return -1; +} + +static int parse_hacks(const struct option *opt, const char *str, int unset) +{ + bool found = false; + + /* + * Use strstr() as a lazy method of checking for comma-separated + * options. + * + * No string provided == enable all options. + */ + + if (!str || strstr(str, "jump_label")) { + opts.hack_jump_label = true; + found = true; + } + + if (!str || strstr(str, "noinstr")) { + opts.hack_noinstr = true; + found = true; + } + + return found ? 0 : -1; +} + const struct option check_options[] = { - OPT_BOOLEAN('f', "no-fp", &no_fp, "Skip frame pointer validation"), - OPT_BOOLEAN('u', "no-unreachable", &no_unreachable, "Skip 'unreachable instruction' warnings"), - OPT_BOOLEAN('r', "retpoline", &retpoline, "Validate retpoline assumptions"), - OPT_BOOLEAN('m', "module", &module, "Indicates the object will be part of a kernel module"), - OPT_BOOLEAN('b', "backtrace", &backtrace, "unwind on error"), - OPT_BOOLEAN('a', "uaccess", &uaccess, "enable uaccess checking"), - OPT_BOOLEAN('s', "stats", &stats, "print statistics"), - OPT_BOOLEAN(0, "lto", <o, "whole-archive like runs"), - OPT_BOOLEAN('n', "noinstr", &noinstr, "noinstr validation for vmlinux.o"), - OPT_BOOLEAN('l', "vmlinux", &vmlinux, "vmlinux.o validation"), - OPT_BOOLEAN('M', "mcount", &mcount, "generate __mcount_loc"), - OPT_BOOLEAN('B', "backup", &backup, "create .orig files before modification"), - OPT_BOOLEAN('S', "sls", &sls, "validate straight-line-speculation"), - OPT_BOOLEAN(0, "dry-run", &dryrun, "don't write the modifications"), - OPT_BOOLEAN(0, "ibt", &ibt, "validate ENDBR placement"), + OPT_GROUP("Actions:"), + OPT_CALLBACK_OPTARG('h', "hacks", NULL, NULL, "jump_label,noinstr", "patch toolchain bugs/limitations", parse_hacks), + OPT_BOOLEAN('i', "ibt", &opts.ibt, "validate and annotate IBT"), + OPT_BOOLEAN('m', "mcount", &opts.mcount, "annotate mcount/fentry calls for ftrace"), + OPT_BOOLEAN('n', "noinstr", &opts.noinstr, "validate noinstr rules"), + OPT_BOOLEAN('o', "orc", &opts.orc, "generate ORC metadata"), + OPT_BOOLEAN('r', "retpoline", &opts.retpoline, "validate and annotate retpoline usage"), + OPT_BOOLEAN(0, "rethunk", &opts.rethunk, "validate and annotate rethunk usage"), + OPT_BOOLEAN(0, "unret", &opts.unret, "validate entry unret placement"), + OPT_BOOLEAN('l', "sls", &opts.sls, "validate straight-line-speculation mitigations"), + OPT_BOOLEAN('s', "stackval", &opts.stackval, "validate frame pointer rules"), + OPT_BOOLEAN('t', "static-call", &opts.static_call, "annotate static calls"), + OPT_BOOLEAN('u', "uaccess", &opts.uaccess, "validate uaccess rules for SMAP"), + OPT_CALLBACK_OPTARG(0, "dump", NULL, NULL, "orc", "dump metadata", parse_dump), + + OPT_GROUP("Options:"), + OPT_BOOLEAN(0, "backtrace", &opts.backtrace, "unwind on error"), + OPT_BOOLEAN(0, "backup", &opts.backup, "create .orig files before modification"), + OPT_BOOLEAN(0, "dry-run", &opts.dryrun, "don't write modifications"), + OPT_BOOLEAN(0, "link", &opts.link, "object is a linked object"), + OPT_BOOLEAN(0, "module", &opts.module, "object is part of a kernel module"), + OPT_BOOLEAN(0, "no-unreachable", &opts.no_unreachable, "skip 'unreachable instruction' warnings"), + OPT_BOOLEAN(0, "sec-address", &opts.sec_address, "print section addresses in warnings"), + OPT_BOOLEAN(0, "stats", &opts.stats, "print statistics"), + OPT_END(), }; @@ -79,7 +116,70 @@ int cmd_parse_options(int argc, const char **argv, const char * const usage[]) return argc; } -int cmd_check(int argc, const char **argv) +static bool opts_valid(void) +{ + if (opts.hack_jump_label || + opts.hack_noinstr || + opts.ibt || + opts.mcount || + opts.noinstr || + opts.orc || + opts.retpoline || + opts.rethunk || + opts.sls || + opts.stackval || + opts.static_call || + opts.uaccess) { + if (opts.dump_orc) { + ERROR("--dump can't be combined with other options"); + return false; + } + + return true; + } + + if (opts.unret && !opts.rethunk) { + ERROR("--unret requires --rethunk"); + return false; + } + + if (opts.dump_orc) + return true; + + ERROR("At least one command required"); + return false; +} + +static bool link_opts_valid(struct objtool_file *file) +{ + if (opts.link) + return true; + + if (has_multiple_files(file->elf)) { + ERROR("Linked object detected, forcing --link"); + opts.link = true; + return true; + } + + if (opts.noinstr) { + ERROR("--noinstr requires --link"); + return false; + } + + if (opts.ibt) { + ERROR("--ibt requires --link"); + return false; + } + + if (opts.unret) { + ERROR("--unret requires --link"); + return false; + } + + return true; +} + +int objtool_run(int argc, const char **argv) { const char *objname; struct objtool_file *file; @@ -88,10 +188,19 @@ int cmd_check(int argc, const char **argv) argc = cmd_parse_options(argc, argv, check_usage); objname = argv[0]; + if (!opts_valid()) + return 1; + + if (opts.dump_orc) + return orc_dump(objname); + file = objtool_open_read(objname); if (!file) return 1; + if (!link_opts_valid(file)) + return 1; + ret = check(file); if (ret) return ret; diff --git a/tools/objtool/check.c b/tools/objtool/check.c index ca5b746030..e55fdf952a 100644 --- a/tools/objtool/check.c +++ b/tools/objtool/check.c @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -161,30 +162,34 @@ static bool __dead_end_function(struct objtool_file *file, struct symbol *func, /* * Unfortunately these have to be hard coded because the noreturn - * attribute isn't provided in ELF data. + * attribute isn't provided in ELF data. Keep 'em sorted. */ static const char * const global_noreturns[] = { - "__stack_chk_fail", - "panic", - "do_exit", - "do_task_dead", - "kthread_exit", - "make_task_dead", - "__module_put_and_kthread_exit", - "kthread_complete_and_exit", - "__reiserfs_panic", - "lbug_with_loc", - "fortify_panic", - "usercopy_abort", - "machine_real_restart", - "rewind_stack_and_make_dead", - "kunit_try_catch_throw", - "xen_start_kernel", - "cpu_bringup_and_idle", - "do_group_exit", - "stop_this_cpu", "__invalid_creds", - "cpu_startup_entry", + "__module_put_and_kthread_exit", + "__reiserfs_panic", + "__stack_chk_fail", + "__ubsan_handle_builtin_unreachable", + "cpu_bringup_and_idle", + "cpu_startup_entry", + "do_exit", + "do_group_exit", + "do_task_dead", + "ex_handler_msr_mce", + "fortify_panic", + "kthread_complete_and_exit", + "kthread_exit", + "kunit_try_catch_throw", + "lbug_with_loc", + "machine_real_restart", + "make_task_dead", + "panic", + "rewind_stack_and_make_dead", + "sev_es_terminate", + "snp_abort", + "stop_this_cpu", + "usercopy_abort", + "xen_start_kernel", }; if (!func) @@ -263,7 +268,8 @@ static void init_cfi_state(struct cfi_state *cfi) cfi->drap_offset = -1; } -static void init_insn_state(struct insn_state *state, struct section *sec) +static void init_insn_state(struct objtool_file *file, struct insn_state *state, + struct section *sec) { memset(state, 0, sizeof(*state)); init_cfi_state(&state->cfi); @@ -273,7 +279,7 @@ static void init_insn_state(struct insn_state *state, struct section *sec) * not correctly determine insn->call_dest->sec (external symbols do * not have a section). */ - if (vmlinux && noinstr && sec) + if (opts.link && opts.noinstr && sec) state->noinstr = sec->noinstr; } @@ -339,7 +345,7 @@ static void *cfi_hash_alloc(unsigned long size) if (cfi_hash == (void *)-1L) { WARN("mmap fail cfi_hash"); cfi_hash = NULL; - } else if (stats) { + } else if (opts.stats) { printf("cfi_bits: %d\n", cfi_bits); } @@ -372,7 +378,8 @@ static int decode_instructions(struct objtool_file *file) sec->text = true; if (!strcmp(sec->name, ".noinstr.text") || - !strcmp(sec->name, ".entry.text")) + !strcmp(sec->name, ".entry.text") || + !strncmp(sec->name, ".text.__x86.", 12)) sec->noinstr = true; for (offset = 0; offset < sec->sh.sh_size; offset += insn->len) { @@ -434,7 +441,7 @@ static int decode_instructions(struct objtool_file *file) } } - if (stats) + if (opts.stats) printf("nr_insns: %lu\n", nr_insns); return 0; @@ -497,7 +504,7 @@ static int init_pv_ops(struct objtool_file *file) struct symbol *sym; int idx, nr; - if (!noinstr) + if (!opts.noinstr) return 0; file->pv_ops = NULL; @@ -560,12 +567,12 @@ static int add_dead_ends(struct objtool_file *file) else if (reloc->addend == reloc->sym->sec->sh.sh_size) { insn = find_last_insn(file, reloc->sym->sec); if (!insn) { - WARN("can't find unreachable insn at %s+0x%lx", + WARN("can't find unreachable insn at %s+0x%" PRIx64, reloc->sym->sec->name, reloc->addend); return -1; } } else { - WARN("can't find unreachable insn at %s+0x%lx", + WARN("can't find unreachable insn at %s+0x%" PRIx64, reloc->sym->sec->name, reloc->addend); return -1; } @@ -595,12 +602,12 @@ static int add_dead_ends(struct objtool_file *file) else if (reloc->addend == reloc->sym->sec->sh.sh_size) { insn = find_last_insn(file, reloc->sym->sec); if (!insn) { - WARN("can't find reachable insn at %s+0x%lx", + WARN("can't find reachable insn at %s+0x%" PRIx64, reloc->sym->sec->name, reloc->addend); return -1; } } else { - WARN("can't find reachable insn at %s+0x%lx", + WARN("can't find reachable insn at %s+0x%" PRIx64, reloc->sym->sec->name, reloc->addend); return -1; } @@ -668,7 +675,7 @@ static int create_static_call_sections(struct objtool_file *file) key_sym = find_symbol_by_name(file->elf, tmp); if (!key_sym) { - if (!module) { + if (!opts.module) { WARN("static_call: can't find static_call_key symbol: %s", tmp); return -1; } @@ -745,6 +752,52 @@ static int create_retpoline_sites_sections(struct objtool_file *file) return 0; } +static int create_return_sites_sections(struct objtool_file *file) +{ + struct instruction *insn; + struct section *sec; + int idx; + + sec = find_section_by_name(file->elf, ".return_sites"); + if (sec) { + WARN("file already has .return_sites, skipping"); + return 0; + } + + idx = 0; + list_for_each_entry(insn, &file->return_thunk_list, call_node) + idx++; + + if (!idx) + return 0; + + sec = elf_create_section(file->elf, ".return_sites", 0, + sizeof(int), idx); + if (!sec) { + WARN("elf_create_section: .return_sites"); + return -1; + } + + idx = 0; + list_for_each_entry(insn, &file->return_thunk_list, call_node) { + + int *site = (int *)sec->data->d_buf + idx; + *site = 0; + + if (elf_add_reloc_to_insn(file->elf, sec, + idx * sizeof(int), + R_X86_64_PC32, + insn->sec, insn->offset)) { + WARN("elf_add_reloc_to_insn: .return_sites"); + return -1; + } + + idx++; + } + + return 0; +} + static int create_ibt_endbr_seal_sections(struct objtool_file *file) { struct instruction *insn; @@ -761,7 +814,7 @@ static int create_ibt_endbr_seal_sections(struct objtool_file *file) list_for_each_entry(insn, &file->endbr_list, call_node) idx++; - if (stats) { + if (opts.stats) { printf("ibt: ENDBR at function start: %d\n", file->nr_endbr); printf("ibt: ENDBR inside functions: %d\n", file->nr_endbr_int); printf("ibt: superfluous ENDBR: %d\n", idx); @@ -1028,7 +1081,7 @@ static void add_uaccess_safe(struct objtool_file *file) struct symbol *func; const char **name; - if (!uaccess) + if (!opts.uaccess) return; for (name = uaccess_safe_builtin; *name; name++) { @@ -1079,6 +1132,11 @@ __weak bool arch_is_retpoline(struct symbol *sym) return false; } +__weak bool arch_is_rethunk(struct symbol *sym) +{ + return false; +} + #define NEGATIVE_RELOC ((void *)-1L) static struct reloc *insn_reloc(struct objtool_file *file, struct instruction *insn) @@ -1144,7 +1202,7 @@ static void annotate_call_site(struct objtool_file *file, * attribute so they need a little help, NOP out any such calls from * noinstr text. */ - if (insn->sec->noinstr && sym->profiling_func) { + if (opts.hack_noinstr && insn->sec->noinstr && sym->profiling_func) { if (reloc) { reloc->type = R_NONE; elf_write_reloc(file->elf, reloc); @@ -1170,7 +1228,7 @@ static void annotate_call_site(struct objtool_file *file, return; } - if (mcount && sym->fentry) { + if (opts.mcount && sym->fentry) { if (sibling) WARN_FUNC("Tail call to __fentry__ !?!?", insn->sec, insn->offset); @@ -1246,6 +1304,19 @@ static void add_retpoline_call(struct objtool_file *file, struct instruction *in annotate_call_site(file, insn, false); } +static void add_return_call(struct objtool_file *file, struct instruction *insn, bool add) +{ + /* + * Return thunk tail calls are really just returns in disguise, + * so convert them accordingly. + */ + insn->type = INSN_RETURN; + insn->retpoline_safe = true; + + if (add) + list_add_tail(&insn->call_node, &file->return_thunk_list); +} + static bool same_function(struct instruction *insn1, struct instruction *insn2) { return insn1->func->pfunc == insn2->func->pfunc; @@ -1256,7 +1327,7 @@ static bool is_first_func_insn(struct objtool_file *file, struct instruction *in if (insn->offset == insn->func->offset) return true; - if (ibt) { + if (opts.ibt) { struct instruction *prev = prev_insn_same_sym(file, insn); if (prev && prev->type == INSN_ENDBR && @@ -1298,6 +1369,9 @@ static int add_jump_destinations(struct objtool_file *file) } else if (reloc->sym->retpoline_thunk) { add_retpoline_call(file, insn); continue; + } else if (reloc->sym->return_thunk) { + add_return_call(file, insn, true); + continue; } else if (insn->func) { /* * External sibling call or internal sibling call with @@ -1316,6 +1390,21 @@ static int add_jump_destinations(struct objtool_file *file) jump_dest = find_insn(file, dest_sec, dest_off); if (!jump_dest) { + struct symbol *sym = find_symbol_by_offset(dest_sec, dest_off); + + /* + * This is a special case for zen_untrain_ret(). + * It jumps to __x86_return_thunk(), but objtool + * can't find the thunk's starting RET + * instruction, because the RET is also in the + * middle of another instruction. Objtool only + * knows about the outer instruction. + */ + if (sym && sym->return_thunk) { + add_return_call(file, insn, false); + continue; + } + WARN_FUNC("can't find jump dest instruction at %s+0x%lx", insn->sec, insn->offset, dest_sec->name, dest_off); @@ -1592,7 +1681,7 @@ static int handle_jump_alt(struct objtool_file *file, return -1; } - if (special_alt->key_addend & 2) { + if (opts.hack_jump_label && special_alt->key_addend & 2) { struct reloc *reloc = insn_reloc(file, orig_insn); if (reloc) { @@ -1699,7 +1788,7 @@ static int add_special_section_alts(struct objtool_file *file) free(special_alt); } - if (stats) { + if (opts.stats) { printf("jl\\\tNOP\tJMP\n"); printf("short:\t%ld\t%ld\n", file->jl_nop_short, file->jl_short); printf("long:\t%ld\t%ld\n", file->jl_nop_long, file->jl_long); @@ -1945,16 +2034,35 @@ static int read_unwind_hints(struct objtool_file *file) insn->hint = true; - if (ibt && hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { + if (hint->type == UNWIND_HINT_TYPE_SAVE) { + insn->hint = false; + insn->save = true; + continue; + } + + if (hint->type == UNWIND_HINT_TYPE_RESTORE) { + insn->restore = true; + continue; + } + + if (hint->type == UNWIND_HINT_TYPE_REGS_PARTIAL) { struct symbol *sym = find_symbol_by_offset(insn->sec, insn->offset); - if (sym && sym->bind == STB_GLOBAL && - insn->type != INSN_ENDBR && !insn->noendbr) { - WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", - insn->sec, insn->offset); + if (sym && sym->bind == STB_GLOBAL) { + if (opts.ibt && insn->type != INSN_ENDBR && !insn->noendbr) { + WARN_FUNC("UNWIND_HINT_IRET_REGS without ENDBR", + insn->sec, insn->offset); + } + + insn->entry = 1; } } + if (hint->type == UNWIND_HINT_TYPE_ENTRY) { + hint->type = UNWIND_HINT_TYPE_CALL; + insn->entry = 1; + } + if (hint->type == UNWIND_HINT_TYPE_FUNC) { insn->cfi = &func_cfi; continue; @@ -2028,8 +2136,10 @@ static int read_retpoline_hints(struct objtool_file *file) } if (insn->type != INSN_JUMP_DYNAMIC && - insn->type != INSN_CALL_DYNAMIC) { - WARN_FUNC("retpoline_safe hint not an indirect jump/call", + insn->type != INSN_CALL_DYNAMIC && + insn->type != INSN_RETURN && + insn->type != INSN_NOP) { + WARN_FUNC("retpoline_safe hint not an indirect jump/call/ret/nop", insn->sec, insn->offset); return -1; } @@ -2180,6 +2290,9 @@ static int classify_symbols(struct objtool_file *file) if (arch_is_retpoline(func)) func->retpoline_thunk = true; + if (arch_is_rethunk(func)) + func->return_thunk = true; + if (!strcmp(func->name, "__fentry__")) func->fentry = true; @@ -2806,7 +2919,7 @@ static int update_cfi_state(struct instruction *insn, } /* detect when asm code uses rbp as a scratch register */ - if (!no_fp && insn->func && op->src.reg == CFI_BP && + if (opts.stackval && insn->func && op->src.reg == CFI_BP && cfa->base != CFI_BP) cfi->bp_scratch = true; break; @@ -3182,119 +3295,11 @@ static struct instruction *next_insn_to_validate(struct objtool_file *file, return next_insn_same_sec(file, insn); } -static struct instruction * -validate_ibt_reloc(struct objtool_file *file, struct reloc *reloc) -{ - struct instruction *dest; - struct section *sec; - unsigned long off; - - sec = reloc->sym->sec; - off = reloc->sym->offset; - - if ((reloc->sec->base->sh.sh_flags & SHF_EXECINSTR) && - (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32)) - off += arch_dest_reloc_offset(reloc->addend); - else - off += reloc->addend; - - dest = find_insn(file, sec, off); - if (!dest) - return NULL; - - if (dest->type == INSN_ENDBR) { - if (!list_empty(&dest->call_node)) - list_del_init(&dest->call_node); - - return NULL; - } - - if (reloc->sym->static_call_tramp) - return NULL; - - return dest; -} - -static void warn_noendbr(const char *msg, struct section *sec, unsigned long offset, - struct instruction *dest) -{ - WARN_FUNC("%srelocation to !ENDBR: %s", sec, offset, msg, - offstr(dest->sec, dest->offset)); -} - -static void validate_ibt_dest(struct objtool_file *file, struct instruction *insn, - struct instruction *dest) -{ - if (dest->func && dest->func == insn->func) { - /* - * Anything from->to self is either _THIS_IP_ or IRET-to-self. - * - * There is no sane way to annotate _THIS_IP_ since the compiler treats the - * relocation as a constant and is happy to fold in offsets, skewing any - * annotation we do, leading to vast amounts of false-positives. - * - * There's also compiler generated _THIS_IP_ through KCOV and - * such which we have no hope of annotating. - * - * As such, blanket accept self-references without issue. - */ - return; - } - - if (dest->noendbr) - return; - - warn_noendbr("", insn->sec, insn->offset, dest); -} - -static void validate_ibt_insn(struct objtool_file *file, struct instruction *insn) -{ - struct instruction *dest; - struct reloc *reloc; - - switch (insn->type) { - case INSN_CALL: - case INSN_CALL_DYNAMIC: - case INSN_JUMP_CONDITIONAL: - case INSN_JUMP_UNCONDITIONAL: - case INSN_JUMP_DYNAMIC: - case INSN_JUMP_DYNAMIC_CONDITIONAL: - case INSN_RETURN: - /* - * We're looking for code references setting up indirect code - * flow. As such, ignore direct code flow and the actual - * dynamic branches. - */ - return; - - case INSN_NOP: - /* - * handle_group_alt() will create INSN_NOP instruction that - * don't belong to any section, ignore all NOP since they won't - * carry a (useful) relocation anyway. - */ - return; - - default: - break; - } - - for (reloc = insn_reloc(file, insn); - reloc; - reloc = find_reloc_by_dest_range(file->elf, insn->sec, - reloc->offset + 1, - (insn->offset + insn->len) - (reloc->offset + 1))) { - dest = validate_ibt_reloc(file, reloc); - if (dest) - validate_ibt_dest(file, insn, dest); - } -} - /* * Follow the branch starting at the given instruction, and recursively follow * any other branches (jumps). Meanwhile, track the frame pointer state at * each instruction and validate all the rules described in - * tools/objtool/Documentation/stack-validation.txt. + * tools/objtool/Documentation/objtool.txt. */ static int validate_branch(struct objtool_file *file, struct symbol *func, struct instruction *insn, struct insn_state state) @@ -3322,8 +3327,8 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, return 1; } - visited = 1 << state.uaccess; - if (insn->visited) { + visited = VISITED_BRANCH << state.uaccess; + if (insn->visited & VISITED_BRANCH_MASK) { if (!insn->hint && !insn_cfi_match(insn, &state.cfi)) return 1; @@ -3337,6 +3342,35 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, state.instr += insn->instr; if (insn->hint) { + if (insn->restore) { + struct instruction *save_insn, *i; + + i = insn; + save_insn = NULL; + + sym_for_each_insn_continue_reverse(file, func, i) { + if (i->save) { + save_insn = i; + break; + } + } + + if (!save_insn) { + WARN_FUNC("no corresponding CFI save for CFI restore", + sec, insn->offset); + return 1; + } + + if (!save_insn->visited) { + WARN_FUNC("objtool isn't smart enough to handle this CFI save/restore combo", + sec, insn->offset); + return 1; + } + + insn->cfi = save_insn->cfi; + nr_cfi_reused++; + } + state.cfi = *insn->cfi; } else { /* XXX track if we actually changed state.cfi */ @@ -3363,7 +3397,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, ret = validate_branch(file, func, alt->insn, state); if (ret) { - if (backtrace) + if (opts.backtrace) BT_FUNC("(alt)", insn); return ret; } @@ -3379,11 +3413,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, switch (insn->type) { case INSN_RETURN: - if (sls && !insn->retpoline_safe && - next_insn && next_insn->type != INSN_TRAP) { - WARN_FUNC("missing int3 after ret", - insn->sec, insn->offset); - } return validate_return(func, insn, &state); case INSN_CALL: @@ -3392,7 +3421,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, if (ret) return ret; - if (!no_fp && func && !is_fentry_call(insn) && + if (opts.stackval && func && !is_fentry_call(insn) && !has_valid_stack_frame(&state)) { WARN_FUNC("call without frame pointer save/setup", sec, insn->offset); @@ -3415,7 +3444,7 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, ret = validate_branch(file, func, insn->jump_dest, state); if (ret) { - if (backtrace) + if (opts.backtrace) BT_FUNC("(branch)", insn); return ret; } @@ -3427,13 +3456,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, break; case INSN_JUMP_DYNAMIC: - if (sls && !insn->retpoline_safe && - next_insn && next_insn->type != INSN_TRAP) { - WARN_FUNC("missing int3 after indirect jump", - insn->sec, insn->offset); - } - - /* fallthrough */ case INSN_JUMP_DYNAMIC_CONDITIONAL: if (is_sibling_call(insn)) { ret = validate_sibling_call(file, insn, &state); @@ -3499,9 +3521,6 @@ static int validate_branch(struct objtool_file *file, struct symbol *func, break; } - if (ibt) - validate_ibt_insn(file, insn); - if (insn->dead_end) return 0; @@ -3528,7 +3547,7 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec) if (!file->hints) return 0; - init_insn_state(&state, sec); + init_insn_state(file, &state, sec); if (sec) { insn = find_insn(file, sec, 0); @@ -3541,7 +3560,7 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec) while (&insn->list != &file->insn_list && (!sec || insn->sec == sec)) { if (insn->hint && !insn->visited && !insn->ignore) { ret = validate_branch(file, insn->func, insn, state); - if (ret && backtrace) + if (ret && opts.backtrace) BT_FUNC("<=== (hint)", insn); warnings += ret; } @@ -3552,6 +3571,145 @@ static int validate_unwind_hints(struct objtool_file *file, struct section *sec) return warnings; } +/* + * Validate rethunk entry constraint: must untrain RET before the first RET. + * + * Follow every branch (intra-function) and ensure ANNOTATE_UNRET_END comes + * before an actual RET instruction. + */ +static int validate_entry(struct objtool_file *file, struct instruction *insn) +{ + struct instruction *next, *dest; + int ret, warnings = 0; + + for (;;) { + next = next_insn_to_validate(file, insn); + + if (insn->visited & VISITED_ENTRY) + return 0; + + insn->visited |= VISITED_ENTRY; + + if (!insn->ignore_alts && !list_empty(&insn->alts)) { + struct alternative *alt; + bool skip_orig = false; + + list_for_each_entry(alt, &insn->alts, list) { + if (alt->skip_orig) + skip_orig = true; + + ret = validate_entry(file, alt->insn); + if (ret) { + if (opts.backtrace) + BT_FUNC("(alt)", insn); + return ret; + } + } + + if (skip_orig) + return 0; + } + + switch (insn->type) { + + case INSN_CALL_DYNAMIC: + case INSN_JUMP_DYNAMIC: + case INSN_JUMP_DYNAMIC_CONDITIONAL: + WARN_FUNC("early indirect call", insn->sec, insn->offset); + return 1; + + case INSN_JUMP_UNCONDITIONAL: + case INSN_JUMP_CONDITIONAL: + if (!is_sibling_call(insn)) { + if (!insn->jump_dest) { + WARN_FUNC("unresolved jump target after linking?!?", + insn->sec, insn->offset); + return -1; + } + ret = validate_entry(file, insn->jump_dest); + if (ret) { + if (opts.backtrace) { + BT_FUNC("(branch%s)", insn, + insn->type == INSN_JUMP_CONDITIONAL ? "-cond" : ""); + } + return ret; + } + + if (insn->type == INSN_JUMP_UNCONDITIONAL) + return 0; + + break; + } + + /* fallthrough */ + case INSN_CALL: + dest = find_insn(file, insn->call_dest->sec, + insn->call_dest->offset); + if (!dest) { + WARN("Unresolved function after linking!?: %s", + insn->call_dest->name); + return -1; + } + + ret = validate_entry(file, dest); + if (ret) { + if (opts.backtrace) + BT_FUNC("(call)", insn); + return ret; + } + /* + * If a call returns without error, it must have seen UNTRAIN_RET. + * Therefore any non-error return is a success. + */ + return 0; + + case INSN_RETURN: + WARN_FUNC("RET before UNTRAIN", insn->sec, insn->offset); + return 1; + + case INSN_NOP: + if (insn->retpoline_safe) + return 0; + break; + + default: + break; + } + + if (!next) { + WARN_FUNC("teh end!", insn->sec, insn->offset); + return -1; + } + insn = next; + } + + return warnings; +} + +/* + * Validate that all branches starting at 'insn->entry' encounter UNRET_END + * before RET. + */ +static int validate_unret(struct objtool_file *file) +{ + struct instruction *insn; + int ret, warnings = 0; + + for_each_insn(file, insn) { + if (!insn->entry) + continue; + + ret = validate_entry(file, insn); + if (ret < 0) { + WARN_FUNC("Failed UNRET validation", insn->sec, insn->offset); + return ret; + } + warnings += ret; + } + + return warnings; +} + static int validate_retpoline(struct objtool_file *file) { struct instruction *insn; @@ -3559,7 +3717,8 @@ static int validate_retpoline(struct objtool_file *file) for_each_insn(file, insn) { if (insn->type != INSN_JUMP_DYNAMIC && - insn->type != INSN_CALL_DYNAMIC) + insn->type != INSN_CALL_DYNAMIC && + insn->type != INSN_RETURN) continue; if (insn->retpoline_safe) @@ -3571,12 +3730,20 @@ static int validate_retpoline(struct objtool_file *file) * loaded late, they very much do need retpoline in their * .init.text */ - if (!strcmp(insn->sec->name, ".init.text") && !module) + if (!strcmp(insn->sec->name, ".init.text") && !opts.module) continue; - WARN_FUNC("indirect %s found in RETPOLINE build", - insn->sec, insn->offset, - insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); + if (insn->type == INSN_RETURN) { + if (opts.rethunk) { + WARN_FUNC("'naked' return found in RETHUNK build", + insn->sec, insn->offset); + } else + continue; + } else { + WARN_FUNC("indirect %s found in RETPOLINE build", + insn->sec, insn->offset, + insn->type == INSN_JUMP_DYNAMIC ? "jump" : "call"); + } warnings++; } @@ -3614,14 +3781,14 @@ static bool ignore_unreachable_insn(struct objtool_file *file, struct instructio return true; /* - * Whole archive runs might encounder dead code from weak symbols. + * Whole archive runs might encounter dead code from weak symbols. * This is where the linker will have dropped the weak symbol in * favour of a regular symbol, but leaves the code in place. * * In this case we'll find a piece of code (whole function) that is not * covered by a !section symbol. Ignore them. */ - if (!insn->func && lto) { + if (opts.link && !insn->func) { int size = find_symbol_hole_containing(insn->sec, insn->offset); unsigned long end = insn->offset + size; @@ -3728,7 +3895,7 @@ static int validate_symbol(struct objtool_file *file, struct section *sec, state->uaccess = sym->uaccess_safe; ret = validate_branch(file, insn->func, insn, *state); - if (ret && backtrace) + if (ret && opts.backtrace) BT_FUNC("<=== (sym)", insn); return ret; } @@ -3743,7 +3910,7 @@ static int validate_section(struct objtool_file *file, struct section *sec) if (func->type != STT_FUNC) continue; - init_insn_state(&state, sec); + init_insn_state(file, &state, sec); set_func_state(&state.cfi); warnings += validate_symbol(file, sec, func, &state); @@ -3752,7 +3919,7 @@ static int validate_section(struct objtool_file *file, struct section *sec) return warnings; } -static int validate_vmlinux_functions(struct objtool_file *file) +static int validate_noinstr_sections(struct objtool_file *file) { struct section *sec; int warnings = 0; @@ -3787,48 +3954,208 @@ static int validate_functions(struct objtool_file *file) return warnings; } +static void mark_endbr_used(struct instruction *insn) +{ + if (!list_empty(&insn->call_node)) + list_del_init(&insn->call_node); +} + +static int validate_ibt_insn(struct objtool_file *file, struct instruction *insn) +{ + struct instruction *dest; + struct reloc *reloc; + unsigned long off; + int warnings = 0; + + /* + * Looking for function pointer load relocations. Ignore + * direct/indirect branches: + */ + switch (insn->type) { + case INSN_CALL: + case INSN_CALL_DYNAMIC: + case INSN_JUMP_CONDITIONAL: + case INSN_JUMP_UNCONDITIONAL: + case INSN_JUMP_DYNAMIC: + case INSN_JUMP_DYNAMIC_CONDITIONAL: + case INSN_RETURN: + case INSN_NOP: + return 0; + default: + break; + } + + for (reloc = insn_reloc(file, insn); + reloc; + reloc = find_reloc_by_dest_range(file->elf, insn->sec, + reloc->offset + 1, + (insn->offset + insn->len) - (reloc->offset + 1))) { + + /* + * static_call_update() references the trampoline, which + * doesn't have (or need) ENDBR. Skip warning in that case. + */ + if (reloc->sym->static_call_tramp) + continue; + + off = reloc->sym->offset; + if (reloc->type == R_X86_64_PC32 || reloc->type == R_X86_64_PLT32) + off += arch_dest_reloc_offset(reloc->addend); + else + off += reloc->addend; + + dest = find_insn(file, reloc->sym->sec, off); + if (!dest) + continue; + + if (dest->type == INSN_ENDBR) { + mark_endbr_used(dest); + continue; + } + + if (dest->func && dest->func == insn->func) { + /* + * Anything from->to self is either _THIS_IP_ or + * IRET-to-self. + * + * There is no sane way to annotate _THIS_IP_ since the + * compiler treats the relocation as a constant and is + * happy to fold in offsets, skewing any annotation we + * do, leading to vast amounts of false-positives. + * + * There's also compiler generated _THIS_IP_ through + * KCOV and such which we have no hope of annotating. + * + * As such, blanket accept self-references without + * issue. + */ + continue; + } + + if (dest->noendbr) + continue; + + WARN_FUNC("relocation to !ENDBR: %s", + insn->sec, insn->offset, + offstr(dest->sec, dest->offset)); + + warnings++; + } + + return warnings; +} + +static int validate_ibt_data_reloc(struct objtool_file *file, + struct reloc *reloc) +{ + struct instruction *dest; + + dest = find_insn(file, reloc->sym->sec, + reloc->sym->offset + reloc->addend); + if (!dest) + return 0; + + if (dest->type == INSN_ENDBR) { + mark_endbr_used(dest); + return 0; + } + + if (dest->noendbr) + return 0; + + WARN_FUNC("data relocation to !ENDBR: %s", + reloc->sec->base, reloc->offset, + offstr(dest->sec, dest->offset)); + + return 1; +} + +/* + * Validate IBT rules and remove used ENDBR instructions from the seal list. + * Unused ENDBR instructions will be annotated for sealing (i.e., replaced with + * NOPs) later, in create_ibt_endbr_seal_sections(). + */ static int validate_ibt(struct objtool_file *file) { struct section *sec; struct reloc *reloc; + struct instruction *insn; + int warnings = 0; + + for_each_insn(file, insn) + warnings += validate_ibt_insn(file, insn); for_each_sec(file, sec) { - bool is_data; - /* already done in validate_branch() */ + /* Already done by validate_ibt_insn() */ if (sec->sh.sh_flags & SHF_EXECINSTR) continue; if (!sec->reloc) continue; - if (!strncmp(sec->name, ".orc", 4)) + /* + * These sections can reference text addresses, but not with + * the intent to indirect branch to them. + */ + if ((!strncmp(sec->name, ".discard", 8) && + strcmp(sec->name, ".discard.ibt_endbr_noseal")) || + !strncmp(sec->name, ".debug", 6) || + !strcmp(sec->name, ".altinstructions") || + !strcmp(sec->name, ".ibt_endbr_seal") || + !strcmp(sec->name, ".orc_unwind_ip") || + !strcmp(sec->name, ".parainstructions") || + !strcmp(sec->name, ".retpoline_sites") || + !strcmp(sec->name, ".smp_locks") || + !strcmp(sec->name, ".static_call_sites") || + !strcmp(sec->name, "_error_injection_whitelist") || + !strcmp(sec->name, "_kprobe_blacklist") || + !strcmp(sec->name, "__bug_table") || + !strcmp(sec->name, "__ex_table") || + !strcmp(sec->name, "__jump_table") || + !strcmp(sec->name, "__mcount_loc")) continue; - if (!strncmp(sec->name, ".discard", 8)) + list_for_each_entry(reloc, &sec->reloc->reloc_list, list) + warnings += validate_ibt_data_reloc(file, reloc); + } + + return warnings; +} + +static int validate_sls(struct objtool_file *file) +{ + struct instruction *insn, *next_insn; + int warnings = 0; + + for_each_insn(file, insn) { + next_insn = next_insn_same_sec(file, insn); + + if (insn->retpoline_safe) continue; - if (!strncmp(sec->name, ".debug", 6)) - continue; + switch (insn->type) { + case INSN_RETURN: + if (!next_insn || next_insn->type != INSN_TRAP) { + WARN_FUNC("missing int3 after ret", + insn->sec, insn->offset); + warnings++; + } - if (!strcmp(sec->name, "_error_injection_whitelist")) - continue; - - if (!strcmp(sec->name, "_kprobe_blacklist")) - continue; - - is_data = strstr(sec->name, ".data") || strstr(sec->name, ".rodata"); - - list_for_each_entry(reloc, &sec->reloc->reloc_list, list) { - struct instruction *dest; - - dest = validate_ibt_reloc(file, reloc); - if (is_data && dest && !dest->noendbr) - warn_noendbr("data ", sec, reloc->offset, dest); + break; + case INSN_JUMP_DYNAMIC: + if (!next_insn || next_insn->type != INSN_TRAP) { + WARN_FUNC("missing int3 after indirect jump", + insn->sec, insn->offset); + warnings++; + } + break; + default: + break; } } - return 0; + return warnings; } static int validate_reachable_instructions(struct objtool_file *file) @@ -3853,16 +4180,6 @@ int check(struct objtool_file *file) { int ret, warnings = 0; - if (lto && !(vmlinux || module)) { - fprintf(stderr, "--lto requires: --vmlinux or --module\n"); - return 1; - } - - if (ibt && !lto) { - fprintf(stderr, "--ibt requires: --lto\n"); - return 1; - } - arch_initial_func_cfi_state(&initial_func_cfi); init_cfi_state(&init_cfi); init_cfi_state(&func_cfi); @@ -3883,73 +4200,107 @@ int check(struct objtool_file *file) if (list_empty(&file->insn_list)) goto out; - if (vmlinux && !lto) { - ret = validate_vmlinux_functions(file); - if (ret < 0) - goto out; - - warnings += ret; - goto out; - } - - if (retpoline) { + if (opts.retpoline) { ret = validate_retpoline(file); if (ret < 0) return ret; warnings += ret; } - ret = validate_functions(file); - if (ret < 0) - goto out; - warnings += ret; + if (opts.stackval || opts.orc || opts.uaccess) { + ret = validate_functions(file); + if (ret < 0) + goto out; + warnings += ret; - ret = validate_unwind_hints(file, NULL); - if (ret < 0) - goto out; - warnings += ret; + ret = validate_unwind_hints(file, NULL); + if (ret < 0) + goto out; + warnings += ret; - if (ibt) { + if (!warnings) { + ret = validate_reachable_instructions(file); + if (ret < 0) + goto out; + warnings += ret; + } + + } else if (opts.noinstr) { + ret = validate_noinstr_sections(file); + if (ret < 0) + goto out; + warnings += ret; + } + + if (opts.unret) { + /* + * Must be after validate_branch() and friends, it plays + * further games with insn->visited. + */ + ret = validate_unret(file); + if (ret < 0) + return ret; + warnings += ret; + } + + if (opts.ibt) { ret = validate_ibt(file); if (ret < 0) goto out; warnings += ret; } - if (!warnings) { - ret = validate_reachable_instructions(file); + if (opts.sls) { + ret = validate_sls(file); if (ret < 0) goto out; warnings += ret; } - ret = create_static_call_sections(file); - if (ret < 0) - goto out; - warnings += ret; + if (opts.static_call) { + ret = create_static_call_sections(file); + if (ret < 0) + goto out; + warnings += ret; + } - if (retpoline) { + if (opts.retpoline) { ret = create_retpoline_sites_sections(file); if (ret < 0) goto out; warnings += ret; } - if (mcount) { + if (opts.rethunk) { + ret = create_return_sites_sections(file); + if (ret < 0) + goto out; + warnings += ret; + } + + if (opts.mcount) { ret = create_mcount_loc_sections(file); if (ret < 0) goto out; warnings += ret; } - if (ibt) { + if (opts.ibt) { ret = create_ibt_endbr_seal_sections(file); if (ret < 0) goto out; warnings += ret; } - if (stats) { + if (opts.orc && !list_empty(&file->insn_list)) { + ret = orc_create(file); + if (ret < 0) + goto out; + warnings += ret; + } + + + if (opts.stats) { printf("nr_insns_visited: %ld\n", nr_insns_visited); printf("nr_cfi: %ld\n", nr_cfi); printf("nr_cfi_reused: %ld\n", nr_cfi_reused); diff --git a/tools/objtool/elf.c b/tools/objtool/elf.c index ebf2ba5755..c25e957c1e 100644 --- a/tools/objtool/elf.c +++ b/tools/objtool/elf.c @@ -355,7 +355,7 @@ static int read_sections(struct elf *elf) elf_hash_add(section_name, &sec->name_hash, str_hash(sec->name)); } - if (stats) { + if (opts.stats) { printf("nr_sections: %lu\n", (unsigned long)sections_nr); printf("section_bits: %d\n", elf->section_bits); } @@ -374,9 +374,15 @@ static void elf_add_symbol(struct elf *elf, struct symbol *sym) struct list_head *entry; struct rb_node *pnode; + INIT_LIST_HEAD(&sym->pv_target); + sym->alias = sym; + sym->type = GELF_ST_TYPE(sym->sym.st_info); sym->bind = GELF_ST_BIND(sym->sym.st_info); + if (sym->type == STT_FILE) + elf->num_files++; + sym->offset = sym->sym.st_value; sym->len = sym->sym.st_size; @@ -435,8 +441,6 @@ static int read_symbols(struct elf *elf) return -1; } memset(sym, 0, sizeof(*sym)); - INIT_LIST_HEAD(&sym->pv_target); - sym->alias = sym; sym->idx = i; @@ -475,7 +479,7 @@ static int read_symbols(struct elf *elf) elf_add_symbol(elf, sym); } - if (stats) { + if (opts.stats) { printf("nr_symbols: %lu\n", (unsigned long)symbols_nr); printf("symbol_bits: %d\n", elf->symbol_bits); } @@ -546,7 +550,7 @@ static struct section *elf_create_reloc_section(struct elf *elf, int reltype); int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset, - unsigned int type, struct symbol *sym, long addend) + unsigned int type, struct symbol *sym, s64 addend) { struct reloc *reloc; @@ -600,24 +604,21 @@ static void elf_dirty_reloc_sym(struct elf *elf, struct symbol *sym) } /* - * Move the first global symbol, as per sh_info, into a new, higher symbol - * index. This fees up the shndx for a new local symbol. + * The libelf API is terrible; gelf_update_sym*() takes a data block relative + * index value, *NOT* the symbol index. As such, iterate the data blocks and + * adjust index until it fits. + * + * If no data block is found, allow adding a new data block provided the index + * is only one past the end. */ -static int elf_move_global_symbol(struct elf *elf, struct section *symtab, - struct section *symtab_shndx) +static int elf_update_symbol(struct elf *elf, struct section *symtab, + struct section *symtab_shndx, struct symbol *sym) { - Elf_Data *data, *shndx_data = NULL; - Elf32_Word first_non_local; - struct symbol *sym; - Elf_Scn *s; - - first_non_local = symtab->sh.sh_info; - - sym = find_symbol_by_index(elf, first_non_local); - if (!sym) { - WARN("no non-local symbols !?"); - return first_non_local; - } + Elf32_Word shndx = sym->sec ? sym->sec->idx : SHN_UNDEF; + Elf_Data *symtab_data = NULL, *shndx_data = NULL; + Elf64_Xword entsize = symtab->sh.sh_entsize; + int max_idx, idx = sym->idx; + Elf_Scn *s, *t = NULL; s = elf_getscn(elf->elf, symtab->idx); if (!s) { @@ -625,79 +626,124 @@ static int elf_move_global_symbol(struct elf *elf, struct section *symtab, return -1; } - data = elf_newdata(s); - if (!data) { - WARN_ELF("elf_newdata"); - return -1; - } - - data->d_buf = &sym->sym; - data->d_size = sizeof(sym->sym); - data->d_align = 1; - data->d_type = ELF_T_SYM; - - sym->idx = symtab->sh.sh_size / sizeof(sym->sym); - elf_dirty_reloc_sym(elf, sym); - - symtab->sh.sh_info += 1; - symtab->sh.sh_size += data->d_size; - symtab->changed = true; - if (symtab_shndx) { - s = elf_getscn(elf->elf, symtab_shndx->idx); - if (!s) { + t = elf_getscn(elf->elf, symtab_shndx->idx); + if (!t) { WARN_ELF("elf_getscn"); return -1; } + } - shndx_data = elf_newdata(s); - if (!shndx_data) { - WARN_ELF("elf_newshndx_data"); + for (;;) { + /* get next data descriptor for the relevant sections */ + symtab_data = elf_getdata(s, symtab_data); + if (t) + shndx_data = elf_getdata(t, shndx_data); + + /* end-of-list */ + if (!symtab_data) { + void *buf; + + if (idx) { + /* we don't do holes in symbol tables */ + WARN("index out of range"); + return -1; + } + + /* if @idx == 0, it's the next contiguous entry, create it */ + symtab_data = elf_newdata(s); + if (t) + shndx_data = elf_newdata(t); + + buf = calloc(1, entsize); + if (!buf) { + WARN("malloc"); + return -1; + } + + symtab_data->d_buf = buf; + symtab_data->d_size = entsize; + symtab_data->d_align = 1; + symtab_data->d_type = ELF_T_SYM; + + symtab->sh.sh_size += entsize; + symtab->changed = true; + + if (t) { + shndx_data->d_buf = &sym->sec->idx; + shndx_data->d_size = sizeof(Elf32_Word); + shndx_data->d_align = sizeof(Elf32_Word); + shndx_data->d_type = ELF_T_WORD; + + symtab_shndx->sh.sh_size += sizeof(Elf32_Word); + symtab_shndx->changed = true; + } + + break; + } + + /* empty blocks should not happen */ + if (!symtab_data->d_size) { + WARN("zero size data"); return -1; } - shndx_data->d_buf = &sym->sec->idx; - shndx_data->d_size = sizeof(Elf32_Word); - shndx_data->d_align = 4; - shndx_data->d_type = ELF_T_WORD; + /* is this the right block? */ + max_idx = symtab_data->d_size / entsize; + if (idx < max_idx) + break; - symtab_shndx->sh.sh_size += 4; - symtab_shndx->changed = true; + /* adjust index and try again */ + idx -= max_idx; } - return first_non_local; + /* something went side-ways */ + if (idx < 0) { + WARN("negative index"); + return -1; + } + + /* setup extended section index magic and write the symbol */ + if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) { + sym->sym.st_shndx = shndx; + if (!shndx_data) + shndx = 0; + } else { + sym->sym.st_shndx = SHN_XINDEX; + if (!shndx_data) { + WARN("no .symtab_shndx"); + return -1; + } + } + + if (!gelf_update_symshndx(symtab_data, shndx_data, idx, &sym->sym, shndx)) { + WARN_ELF("gelf_update_symshndx"); + return -1; + } + + return 0; } static struct symbol * elf_create_section_symbol(struct elf *elf, struct section *sec) { struct section *symtab, *symtab_shndx; - Elf_Data *shndx_data = NULL; - struct symbol *sym; - Elf32_Word shndx; + Elf32_Word first_non_local, new_idx; + struct symbol *sym, *old; symtab = find_section_by_name(elf, ".symtab"); if (symtab) { symtab_shndx = find_section_by_name(elf, ".symtab_shndx"); - if (symtab_shndx) - shndx_data = symtab_shndx->data; } else { WARN("no .symtab"); return NULL; } - sym = malloc(sizeof(*sym)); + sym = calloc(1, sizeof(*sym)); if (!sym) { perror("malloc"); return NULL; } - memset(sym, 0, sizeof(*sym)); - - sym->idx = elf_move_global_symbol(elf, symtab, symtab_shndx); - if (sym->idx < 0) { - WARN("elf_move_global_symbol"); - return NULL; - } sym->name = sec->name; sym->sec = sec; @@ -707,24 +753,41 @@ elf_create_section_symbol(struct elf *elf, struct section *sec) // st_other 0 // st_value 0 // st_size 0 - shndx = sec->idx; - if (shndx >= SHN_UNDEF && shndx < SHN_LORESERVE) { - sym->sym.st_shndx = shndx; - if (!shndx_data) - shndx = 0; - } else { - sym->sym.st_shndx = SHN_XINDEX; - if (!shndx_data) { - WARN("no .symtab_shndx"); + + /* + * Move the first global symbol, as per sh_info, into a new, higher + * symbol index. This fees up a spot for a new local symbol. + */ + first_non_local = symtab->sh.sh_info; + new_idx = symtab->sh.sh_size / symtab->sh.sh_entsize; + old = find_symbol_by_index(elf, first_non_local); + if (old) { + old->idx = new_idx; + + hlist_del(&old->hash); + elf_hash_add(symbol, &old->hash, old->idx); + + elf_dirty_reloc_sym(elf, old); + + if (elf_update_symbol(elf, symtab, symtab_shndx, old)) { + WARN("elf_update_symbol move"); return NULL; } + + new_idx = first_non_local; } - if (!gelf_update_symshndx(symtab->data, shndx_data, sym->idx, &sym->sym, shndx)) { - WARN_ELF("gelf_update_symshndx"); + sym->idx = new_idx; + if (elf_update_symbol(elf, symtab, symtab_shndx, sym)) { + WARN("elf_update_symbol"); return NULL; } + /* + * Either way, we added a LOCAL symbol. + */ + symtab->sh.sh_info += 1; + elf_add_symbol(elf, sym); return sym; @@ -843,7 +906,7 @@ static int read_relocs(struct elf *elf) tot_reloc += nr_reloc; } - if (stats) { + if (opts.stats) { printf("max_reloc: %lu\n", max_reloc); printf("tot_reloc: %lu\n", tot_reloc); printf("reloc_bits: %d\n", elf->reloc_bits); @@ -1222,7 +1285,7 @@ int elf_write(struct elf *elf) struct section *sec; Elf_Scn *s; - if (dryrun) + if (opts.dryrun) return 0; /* Update changed relocation sections and section headers: */ diff --git a/tools/objtool/include/objtool/arch.h b/tools/objtool/include/objtool/arch.h index 9b19cc3041..beb2f3aa94 100644 --- a/tools/objtool/include/objtool/arch.h +++ b/tools/objtool/include/objtool/arch.h @@ -89,6 +89,7 @@ const char *arch_ret_insn(int len); int arch_decode_hint_reg(u8 sp_reg, int *base); bool arch_is_retpoline(struct symbol *sym); +bool arch_is_rethunk(struct symbol *sym); int arch_rewrite_retpolines(struct objtool_file *file); diff --git a/tools/objtool/include/objtool/builtin.h b/tools/objtool/include/objtool/builtin.h index c39dbfaef6..42a52f1a0a 100644 --- a/tools/objtool/include/objtool/builtin.h +++ b/tools/objtool/include/objtool/builtin.h @@ -8,13 +8,39 @@ #include extern const struct option check_options[]; -extern bool no_fp, no_unreachable, retpoline, module, backtrace, uaccess, stats, - lto, vmlinux, mcount, noinstr, backup, sls, dryrun, - ibt; + +struct opts { + /* actions: */ + bool dump_orc; + bool hack_jump_label; + bool hack_noinstr; + bool ibt; + bool mcount; + bool noinstr; + bool orc; + bool retpoline; + bool rethunk; + bool unret; + bool sls; + bool stackval; + bool static_call; + bool uaccess; + + /* options: */ + bool backtrace; + bool backup; + bool dryrun; + bool link; + bool module; + bool no_unreachable; + bool sec_address; + bool stats; +}; + +extern struct opts opts; extern int cmd_parse_options(int argc, const char **argv, const char * const usage[]); -extern int cmd_check(int argc, const char **argv); -extern int cmd_orc(int argc, const char **argv); +extern int objtool_run(int argc, const char **argv); #endif /* _BUILTIN_H */ diff --git a/tools/objtool/include/objtool/check.h b/tools/objtool/include/objtool/check.h index f10d7374f3..036129cebe 100644 --- a/tools/objtool/include/objtool/check.h +++ b/tools/objtool/include/objtool/check.h @@ -46,16 +46,19 @@ struct instruction { enum insn_type type; unsigned long immediate; - u8 dead_end : 1, - ignore : 1, - ignore_alts : 1, - hint : 1, - retpoline_safe : 1, - noendbr : 1; - /* 2 bit hole */ + u16 dead_end : 1, + ignore : 1, + ignore_alts : 1, + hint : 1, + save : 1, + restore : 1, + retpoline_safe : 1, + noendbr : 1, + entry : 1; + /* 7 bit hole */ + s8 instr; u8 visited; - /* u8 hole */ struct alt_group *alt_group; struct symbol *call_dest; @@ -69,6 +72,11 @@ struct instruction { struct cfi_state *cfi; }; +#define VISITED_BRANCH 0x01 +#define VISITED_BRANCH_UACCESS 0x02 +#define VISITED_BRANCH_MASK 0x03 +#define VISITED_ENTRY 0x04 + static inline bool is_static_jump(struct instruction *insn) { return insn->type == INSN_JUMP_CONDITIONAL || diff --git a/tools/objtool/include/objtool/elf.h b/tools/objtool/include/objtool/elf.h index 9b36802ed8..16f4067b82 100644 --- a/tools/objtool/include/objtool/elf.h +++ b/tools/objtool/include/objtool/elf.h @@ -57,6 +57,7 @@ struct symbol { u8 uaccess_safe : 1; u8 static_call_tramp : 1; u8 retpoline_thunk : 1; + u8 return_thunk : 1; u8 fentry : 1; u8 profiling_func : 1; struct list_head pv_target; @@ -73,7 +74,7 @@ struct reloc { struct symbol *sym; unsigned long offset; unsigned int type; - long addend; + s64 addend; int idx; bool jump_table_start; }; @@ -86,7 +87,7 @@ struct elf { int fd; bool changed; char *name; - unsigned int text_size; + unsigned int text_size, num_files; struct list_head sections; int symbol_bits; @@ -131,11 +132,21 @@ static inline u32 reloc_hash(struct reloc *reloc) return sec_offset_hash(reloc->sec, reloc->offset); } +/* + * Try to see if it's a whole archive (vmlinux.o or module). + * + * Note this will miss the case where a module only has one source file. + */ +static inline bool has_multiple_files(struct elf *elf) +{ + return elf->num_files > 1; +} + struct elf *elf_open_read(const char *name, int flags); struct section *elf_create_section(struct elf *elf, const char *name, unsigned int sh_flags, size_t entsize, int nr); int elf_add_reloc(struct elf *elf, struct section *sec, unsigned long offset, - unsigned int type, struct symbol *sym, long addend); + unsigned int type, struct symbol *sym, s64 addend); int elf_add_reloc_to_insn(struct elf *elf, struct section *sec, unsigned long offset, unsigned int type, struct section *insn_sec, unsigned long insn_off); diff --git a/tools/objtool/include/objtool/objtool.h b/tools/objtool/include/objtool/objtool.h index a6e72d9168..7f2d1b0953 100644 --- a/tools/objtool/include/objtool/objtool.h +++ b/tools/objtool/include/objtool/objtool.h @@ -24,6 +24,7 @@ struct objtool_file { struct list_head insn_list; DECLARE_HASHTABLE(insn_hash, 20); struct list_head retpoline_call_list; + struct list_head return_thunk_list; struct list_head static_call_list; struct list_head mcount_loc_list; struct list_head endbr_list; diff --git a/tools/objtool/include/objtool/warn.h b/tools/objtool/include/objtool/warn.h index 802cfda0a6..a3e79ae75f 100644 --- a/tools/objtool/include/objtool/warn.h +++ b/tools/objtool/include/objtool/warn.h @@ -11,34 +11,33 @@ #include #include #include +#include #include extern const char *objname; static inline char *offstr(struct section *sec, unsigned long offset) { - struct symbol *func; - char *name, *str; - unsigned long name_off; + bool is_text = (sec->sh.sh_flags & SHF_EXECINSTR); + struct symbol *sym = NULL; + char *str; + int len; - func = find_func_containing(sec, offset); - if (!func) - func = find_symbol_containing(sec, offset); - if (func) { - name = func->name; - name_off = offset - func->offset; + if (is_text) + sym = find_func_containing(sec, offset); + if (!sym) + sym = find_symbol_containing(sec, offset); + + if (sym) { + str = malloc(strlen(sym->name) + strlen(sec->name) + 40); + len = sprintf(str, "%s+0x%lx", sym->name, offset - sym->offset); + if (opts.sec_address) + sprintf(str+len, " (%s+0x%lx)", sec->name, offset); } else { - name = sec->name; - name_off = offset; + str = malloc(strlen(sec->name) + 20); + sprintf(str, "%s+0x%lx", sec->name, offset); } - str = malloc(strlen(name) + 20); - - if (func) - sprintf(str, "%s()+0x%lx", name, name_off); - else - sprintf(str, "%s+0x%lx", name, name_off); - return str; } diff --git a/tools/objtool/objtool.c b/tools/objtool/objtool.c index 843ff3c2f2..a7ecc32e35 100644 --- a/tools/objtool/objtool.c +++ b/tools/objtool/objtool.c @@ -3,16 +3,6 @@ * Copyright (C) 2015 Josh Poimboeuf */ -/* - * objtool: - * - * The 'check' subcmd analyzes every .o file and ensures the validity of its - * stack trace metadata. It enforces a set of rules on asm code and C inline - * assembly code so that stack traces can be reliable. - * - * For more information, see tools/objtool/Documentation/stack-validation.txt. - */ - #include #include #include @@ -26,20 +16,6 @@ #include #include -struct cmd_struct { - const char *name; - int (*fn)(int, const char **); - const char *help; -}; - -static const char objtool_usage_string[] = - "objtool COMMAND [ARGS]"; - -static struct cmd_struct objtool_cmds[] = { - {"check", cmd_check, "Perform stack metadata validation on an object file" }, - {"orc", cmd_orc, "Generate in-place ORC unwind tables for an object file" }, -}; - bool help; const char *objname; @@ -118,7 +94,7 @@ struct objtool_file *objtool_open_read(const char *_objname) if (!file.elf) return NULL; - if (backup && !objtool_create_backup(objname)) { + if (opts.backup && !objtool_create_backup(objname)) { WARN("can't create backup file"); return NULL; } @@ -126,10 +102,11 @@ struct objtool_file *objtool_open_read(const char *_objname) INIT_LIST_HEAD(&file.insn_list); hash_init(file.insn_hash); INIT_LIST_HEAD(&file.retpoline_call_list); + INIT_LIST_HEAD(&file.return_thunk_list); INIT_LIST_HEAD(&file.static_call_list); INIT_LIST_HEAD(&file.mcount_loc_list); INIT_LIST_HEAD(&file.endbr_list); - file.ignore_unreachables = no_unreachable; + file.ignore_unreachables = opts.no_unreachable; file.hints = false; return &file; @@ -137,7 +114,7 @@ struct objtool_file *objtool_open_read(const char *_objname) void objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func) { - if (!noinstr) + if (!opts.noinstr) return; if (!f->pv_ops) { @@ -161,70 +138,6 @@ void objtool_pv_add(struct objtool_file *f, int idx, struct symbol *func) f->pv_ops[idx].clean = false; } -static void cmd_usage(void) -{ - unsigned int i, longest = 0; - - printf("\n usage: %s\n\n", objtool_usage_string); - - for (i = 0; i < ARRAY_SIZE(objtool_cmds); i++) { - if (longest < strlen(objtool_cmds[i].name)) - longest = strlen(objtool_cmds[i].name); - } - - puts(" Commands:"); - for (i = 0; i < ARRAY_SIZE(objtool_cmds); i++) { - printf(" %-*s ", longest, objtool_cmds[i].name); - puts(objtool_cmds[i].help); - } - - printf("\n"); - - if (!help) - exit(129); - exit(0); -} - -static void handle_options(int *argc, const char ***argv) -{ - while (*argc > 0) { - const char *cmd = (*argv)[0]; - - if (cmd[0] != '-') - break; - - if (!strcmp(cmd, "--help") || !strcmp(cmd, "-h")) { - help = true; - break; - } else { - fprintf(stderr, "Unknown option: %s\n", cmd); - cmd_usage(); - } - - (*argv)++; - (*argc)--; - } -} - -static void handle_internal_command(int argc, const char **argv) -{ - const char *cmd = argv[0]; - unsigned int i, ret; - - for (i = 0; i < ARRAY_SIZE(objtool_cmds); i++) { - struct cmd_struct *p = objtool_cmds+i; - - if (strcmp(p->name, cmd)) - continue; - - ret = p->fn(argc, argv); - - exit(ret); - } - - cmd_usage(); -} - int main(int argc, const char **argv) { static const char *UNUSED = "OBJTOOL_NOT_IMPLEMENTED"; @@ -233,14 +146,7 @@ int main(int argc, const char **argv) exec_cmd_init("objtool", UNUSED, UNUSED, UNUSED); pager_init(UNUSED); - argv++; - argc--; - handle_options(&argc, &argv); - - if (!argc || help) - cmd_usage(); - - handle_internal_command(argc, argv); + objtool_run(argc, argv); return 0; } diff --git a/tools/objtool/weak.c b/tools/objtool/weak.c index 8314e824db..d83f607733 100644 --- a/tools/objtool/weak.c +++ b/tools/objtool/weak.c @@ -15,17 +15,12 @@ return ENOSYS; \ }) -int __weak check(struct objtool_file *file) -{ - UNSUPPORTED("check subcommand"); -} - int __weak orc_dump(const char *_objname) { - UNSUPPORTED("orc"); + UNSUPPORTED("ORC"); } int __weak orc_create(struct objtool_file *file) { - UNSUPPORTED("orc"); + UNSUPPORTED("ORC"); } diff --git a/tools/perf/.gitignore b/tools/perf/.gitignore index 20b8ab984d..4b9c71faa0 100644 --- a/tools/perf/.gitignore +++ b/tools/perf/.gitignore @@ -19,7 +19,6 @@ perf.data perf.data.old output.svg perf-archive -perf-with-kcore perf-iostat tags TAGS diff --git a/tools/perf/Build b/tools/perf/Build index db61dbe2b5..496b096153 100644 --- a/tools/perf/Build +++ b/tools/perf/Build @@ -25,6 +25,7 @@ perf-y += builtin-data.o perf-y += builtin-version.o perf-y += builtin-c2c.o perf-y += builtin-daemon.o +perf-y += builtin-kwork.o perf-$(CONFIG_TRACE) += builtin-trace.o perf-$(CONFIG_LIBELF) += builtin-probe.o diff --git a/tools/perf/Documentation/intel-hybrid.txt b/tools/perf/Documentation/intel-hybrid.txt index c9302096dc..e7a776ad25 100644 --- a/tools/perf/Documentation/intel-hybrid.txt +++ b/tools/perf/Documentation/intel-hybrid.txt @@ -21,11 +21,6 @@ cat /sys/devices/cpu_atom/cpus It indicates cpu0-cpu15 are core cpus and cpu16-cpu23 are atom cpus. -Quickstart - -List hybrid event ------------------ - As before, use perf-list to list the symbolic event. perf list @@ -40,7 +35,6 @@ the event is belong to. Same event name but with different pmu can be supported. Enable hybrid event with a specific pmu ---------------------------------------- To enable a core only event or atom only event, following syntax is supported: @@ -53,7 +47,6 @@ For example, count the 'cycles' event on core cpus. perf stat -e cpu_core/cycles/ Create two events for one hardware event automatically ------------------------------------------------------- When creating one event and the event is available on both atom and core, two events are created automatically. One is for atom, the other is for @@ -132,7 +125,6 @@ For perf-stat result, it displays two events: The first 'cycles' is core event, the second 'cycles' is atom event. Thread mode example: --------------------- perf-stat reports the scaled counts for hybrid event and with a percentage displayed. The percentage is the event's running time/enabling time. @@ -176,14 +168,12 @@ perf_event_attr: 604,097,080 cpu_atom/cycles/ (99.57%) perf-record: ------------- If there is no '-e' specified in perf record, on hybrid platform, it creates two default 'cycles' and adds them to event list. One is for core, the other is for atom. perf-stat: ----------- If there is no '-e' specified in perf stat, on hybrid platform, besides of software events, following events are created and diff --git a/tools/perf/Documentation/perf-annotate.txt b/tools/perf/Documentation/perf-annotate.txt index 33c2521cba..18fcc52809 100644 --- a/tools/perf/Documentation/perf-annotate.txt +++ b/tools/perf/Documentation/perf-annotate.txt @@ -147,6 +147,11 @@ include::itrace.txt[] The period/hits keywords set the base the percentage is computed on - the samples period or the number of samples (hits). +--percent-limit:: + Do not show functions which have an overhead under that percent on + stdio or stdio2 (Default: 0). Note that this is about selection of + functions to display, not about lines within the function. + SEE ALSO -------- linkperf:perf-record[1], linkperf:perf-report[1] diff --git a/tools/perf/Documentation/perf-buildid-list.txt b/tools/perf/Documentation/perf-buildid-list.txt index 25c52efcc7..e1e8fdbe06 100644 --- a/tools/perf/Documentation/perf-buildid-list.txt +++ b/tools/perf/Documentation/perf-buildid-list.txt @@ -33,6 +33,10 @@ OPTIONS -k:: --kernel:: Show running kernel build id. +-m:: +--kernel-maps:: + Show buildid, start/end text address, and path of running kernel and + its modules. -v:: --verbose:: Be more verbose. diff --git a/tools/perf/Documentation/perf-c2c.txt b/tools/perf/Documentation/perf-c2c.txt index 3b6a2c84ea..f1f7ae6b08 100644 --- a/tools/perf/Documentation/perf-c2c.txt +++ b/tools/perf/Documentation/perf-c2c.txt @@ -109,7 +109,9 @@ REPORT OPTIONS -d:: --display:: - Switch to HITM type (rmt, lcl) to display and sort on. Total HITMs as default. + Switch to HITM type (rmt, lcl) or peer snooping type (peer) to display + and sort on. Total HITMs (tot) as default, except Arm64 uses peer mode + as default. --stitch-lbr:: Show callgraph with stitched LBRs, which may have more complete @@ -174,12 +176,18 @@ For each cacheline in the 1) list we display following data: Cacheline - cacheline address (hex number) - Rmt/Lcl Hitm + Rmt/Lcl Hitm (Display with HITM types) - cacheline percentage of all Remote/Local HITM accesses - LLC Load Hitm - Total, LclHitm, RmtHitm + Peer Snoop (Display with peer type) + - cacheline percentage of all peer accesses + + LLC Load Hitm - Total, LclHitm, RmtHitm (For display with HITM types) - count of Total/Local/Remote load HITMs + Load Peer - Total, Local, Remote (For display with peer type) + - count of Total/Local/Remote load from peer cache or DRAM + Total records - sum of all cachelines accesses @@ -189,9 +197,10 @@ For each cacheline in the 1) list we display following data: Total stores - sum of all store accesses - Store Reference - L1Hit, L1Miss + Store Reference - L1Hit, L1Miss, N/A L1Hit - store accesses that hit L1 L1Miss - store accesses that missed L1 + N/A - store accesses with memory level is not available Core Load Hit - FB, L1, L2 - count of load hits in FB (Fill Buffer), L1 and L2 cache @@ -200,18 +209,24 @@ For each cacheline in the 1) list we display following data: - count of LLC load accesses, includes LLC hits and LLC HITMs RMT Load Hit - RmtHit, RmtHitm - - count of remote load accesses, includes remote hits and remote HITMs + - count of remote load accesses, includes remote hits and remote HITMs; + on Arm neoverse cores, RmtHit is used to account remote accesses, + includes remote DRAM or any upward cache level in remote node Load Dram - Lcl, Rmt - count of local and remote DRAM accesses For each offset in the 2) list we display following data: - HITM - Rmt, Lcl + HITM - Rmt, Lcl (Display with HITM types) - % of Remote/Local HITM accesses for given offset within cacheline - Store Refs - L1 Hit, L1 Miss - - % of store accesses that hit/missed L1 for given offset within cacheline + Peer Snoop - Rmt, Lcl (Display with peer type) + - % of Remote/Local peer accesses for given offset within cacheline + + Store Refs - L1 Hit, L1 Miss, N/A + - % of store accesses that hit L1, missed L1 and N/A (no available) memory + level for given offset within cacheline Data address - Offset - offset address @@ -225,9 +240,12 @@ For each offset in the 2) list we display following data: Code address - code address responsible for the accesses - cycles - rmt hitm, lcl hitm, load + cycles - rmt hitm, lcl hitm, load (Display with HITM types) - sum of cycles for given accesses - Remote/Local HITM and generic load + cycles - rmt peer, lcl peer, load (Display with peer type) + - sum of cycles for given accesses - Remote/Local peer load and generic load + cpu cnt - number of cpus that participated on the access @@ -249,7 +267,8 @@ The 'Node' field displays nodes that accesses given cacheline offset. Its output comes in 3 flavors: - node IDs separated by ',' - node IDs with stats for each ID, in following format: - Node{cpus %hitms %stores} + Node{cpus %hitms %stores} (Display with HITM types) + Node{cpus %peers %stores} (Display with peer type) - node IDs with list of affected CPUs in following format: Node{cpu list} diff --git a/tools/perf/Documentation/perf-dlfilter.txt b/tools/perf/Documentation/perf-dlfilter.txt index 594f5a5a0c..fb22e3b31d 100644 --- a/tools/perf/Documentation/perf-dlfilter.txt +++ b/tools/perf/Documentation/perf-dlfilter.txt @@ -107,9 +107,31 @@ struct perf_dlfilter_sample { __u64 raw_callchain_nr; /* Number of raw_callchain entries */ const __u64 *raw_callchain; /* Refer */ const char *event; + __s32 machine_pid; + __s32 vcpu; }; ---- +Note: 'machine_pid' and 'vcpu' are not original members, but were added together later. +'size' can be used to determine their presence at run time. +PERF_DLFILTER_HAS_MACHINE_PID will be defined if they are present at compile time. +For example: +[source,c] +---- +#include +#include +#include + +static inline bool have_machine_pid(const struct perf_dlfilter_sample *sample) +{ +#ifdef PERF_DLFILTER_HAS_MACHINE_PID + return sample->size >= offsetof(struct perf_dlfilter_sample, vcpu) + sizeof(sample->vcpu); +#else + return false; +#endif +} +---- + The perf_dlfilter_fns structure ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/tools/perf/Documentation/perf-inject.txt b/tools/perf/Documentation/perf-inject.txt index 0570a1ccd3..ffc293fdf6 100644 --- a/tools/perf/Documentation/perf-inject.txt +++ b/tools/perf/Documentation/perf-inject.txt @@ -85,6 +85,27 @@ include::itrace.txt[] without updating it. Currently this option is supported only by Intel PT, refer linkperf:perf-intel-pt[1] +--guest-data=,[,