KVM: SEV: Do cache maintenance on the source VM during intra-host migrationgrafted

Manually perform cache maintenance on the source VM during intra-host
migration to ensure no stale data is left in CPU caches after the VM is
destroyed.  Because the source VM is "converted" to a non-SEV VM, KVM's
memory reclaim flows won't trigger cache maintenance, e.g. when all guest
memory is reclaimed in response to detaching from the mmu_notifier.

Note, relying on the destination VM to do cache maintenance isn't an option
as KVM doesn't require identical guest memory configurations, i.e. the
source VM may have access to memory that the destination VM does not.
Enforcing equivalent memory configurations is infeasible, as it would
require a *deep* comparison of memslots, e.g. to verify that not only are
the memslot identical, but what the memslots point at is also identical.

Fixes: b56639318bb2 ("KVM: SEV: Add support for SEV intra host migration")
Cc: stable@vger.kernel.org
Reported-by: Stefan Teodorescu <fane@google.com>
Signed-off-by: Sean Christopherson <seanjc@google.com>
Message-ID: <20260923163721.1584779-3-seanjc@google.com>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

1 files changed, 336 insertions, 0 deletions

diff --git a/scripts/atomic/gen-atomic-fallback.sh b/scripts/atomic/gen-atomic-fallback.sh
new file mode 100755
index 000000000..f80d69cfe
--- /dev/null
+++ b/scripts/atomic/gen-atomic-fallback.sh
@@ -0,0 +1,336 @@
+#!/bin/sh
+# SPDX-License-Identifier: GPL-2.0
+
+ATOMICDIR=$(dirname $0)
+
+. ${ATOMICDIR}/atomic-tbl.sh
+
+#gen_template_fallback(template, meta, pfx, name, sfx, order, atomic, int, args...)
+gen_template_fallback()
+{
+	local template="$1"; shift
+	local meta="$1"; shift
+	local pfx="$1"; shift
+	local name="$1"; shift
+	local sfx="$1"; shift
+	local order="$1"; shift
+	local atomic="$1"; shift
+	local int="$1"; shift
+
+	local ret="$(gen_ret_type "${meta}" "${int}")"
+	local retstmt="$(gen_ret_stmt "${meta}")"
+	local params="$(gen_params "${int}" "${atomic}" "$@")"
+	local args="$(gen_args "$@")"
+
+	. ${template}
+}
+
+#gen_order_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
+gen_order_fallback()
+{
+	local meta="$1"; shift
+	local pfx="$1"; shift
+	local name="$1"; shift
+	local sfx="$1"; shift
+	local order="$1"; shift
+
+	local tmpl_order=${order#_}
+	local tmpl="${ATOMICDIR}/fallbacks/${tmpl_order:-fence}"
+	gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
+}
+
+#gen_proto_fallback(meta, pfx, name, sfx, order, atomic, int, args...)
+gen_proto_fallback()
+{
+	local meta="$1"; shift
+	local pfx="$1"; shift
+	local name="$1"; shift
+	local sfx="$1"; shift
+	local order="$1"; shift
+
+	local tmpl="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
+	gen_template_fallback "${tmpl}" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "$@"
+}
+
+#gen_proto_order_variant(meta, pfx, name, sfx, order, atomic, int, args...)
+gen_proto_order_variant()
+{
+	local meta="$1"; shift
+	local pfx="$1"; shift
+	local name="$1"; shift
+	local sfx="$1"; shift
+	local order="$1"; shift
+	local atomic="$1"; shift
+	local int="$1"; shift
+
+	local atomicname="${atomic}_${pfx}${name}${sfx}${order}"
+	local basename="${atomic}_${pfx}${name}${sfx}"
+
+	local template="$(find_fallback_template "${pfx}" "${name}" "${sfx}" "${order}")"
+
+	local ret="$(gen_ret_type "${meta}" "${int}")"
+	local retstmt="$(gen_ret_stmt "${meta}")"
+	local params="$(gen_params "${int}" "${atomic}" "$@")"
+	local args="$(gen_args "$@")"
+
+	gen_kerneldoc "raw_" "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
+
+	printf "static __always_inline ${ret}\n"
+	printf "raw_${atomicname}(${params})\n"
+	printf "{\n"
+
+	# Where there is no possible fallback, this order variant is mandatory
+	# and must be provided by arch code. Add a comment to the header to
+	# make this obvious.
+	#
+	# Ideally we'd error on a missing definition, but arch code might
+	# define this order variant as a C function without a preprocessor
+	# symbol.
+	if [ -z ${template} ] && [ -z "${order}" ] && ! meta_has_relaxed "${meta}"; then
+		printf "\t${retstmt}arch_${atomicname}(${args});\n"
+		printf "}\n\n"
+		return
+	fi
+
+	printf "#if defined(arch_${atomicname})\n"
+	printf "\t${retstmt}arch_${atomicname}(${args});\n"
+
+	# Allow FULL/ACQUIRE/RELEASE ops to be defined in terms of RELAXED ops
+	if [ "${order}" != "_relaxed" ] && meta_has_relaxed "${meta}"; then
+		printf "#elif defined(arch_${basename}_relaxed)\n"
+		gen_order_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
+	fi
+
+	# Allow ACQUIRE/RELEASE/RELAXED ops to be defined in terms of FULL ops
+	if [ ! -z "${order}" ] && ! meta_is_implicitly_relaxed "${meta}"; then
+		printf "#elif defined(arch_${basename})\n"
+		printf "\t${retstmt}arch_${basename}(${args});\n"
+	fi
+
+	printf "#else\n"
+	if [ ! -z "${template}" ]; then
+		gen_proto_fallback "${meta}" "${pfx}" "${name}" "${sfx}" "${order}" "${atomic}" "${int}" "$@"
+	else
+		printf "#error \"Unable to define raw_${atomicname}\"\n"
+	fi
+
+	printf "#endif\n"
+	printf "}\n\n"
+}
+
+
+#gen_proto_order_variants(meta, pfx, name, sfx, atomic, int, args...)
+gen_proto_order_variants()
+{
+	local meta="$1"; shift
+	local pfx="$1"; shift
+	local name="$1"; shift
+	local sfx="$1"; shift
+	local atomic="$1"
+
+	gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "" "$@"
+
+	if meta_has_acquire "${meta}"; then
+		gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_acquire" "$@"
+	fi
+
+	if meta_has_release "${meta}"; then
+		gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_release" "$@"
+	fi
+
+	if meta_has_relaxed "${meta}"; then
+		gen_proto_order_variant "${meta}" "${pfx}" "${name}" "${sfx}" "_relaxed" "$@"
+	fi
+}
+
+#gen_basic_fallbacks(basename)
+gen_basic_fallbacks()
+{
+	local basename="$1"; shift
+cat << EOF
+#define raw_${basename}_acquire arch_${basename}
+#define raw_${basename}_release arch_${basename}
+#define raw_${basename}_relaxed arch_${basename}
+EOF
+}
+
+gen_order_fallbacks()
+{
+	local xchg="$1"; shift
+
+cat <<EOF
+
+#define raw_${xchg}_relaxed arch_${xchg}_relaxed
+
+#ifdef arch_${xchg}_acquire
+#define raw_${xchg}_acquire arch_${xchg}_acquire
+#else
+#define raw_${xchg}_acquire(...) \\
+	__atomic_op_acquire(arch_${xchg}, __VA_ARGS__)
+#endif
+
+#ifdef arch_${xchg}_release
+#define raw_${xchg}_release arch_${xchg}_release
+#else
+#define raw_${xchg}_release(...) \\
+	__atomic_op_release(arch_${xchg}, __VA_ARGS__)
+#endif
+
+#ifdef arch_${xchg}
+#define raw_${xchg} arch_${xchg}
+#else
+#define raw_${xchg}(...) \\
+	__atomic_op_fence(arch_${xchg}, __VA_ARGS__)
+#endif
+
+EOF
+}
+
+gen_xchg_order_fallback()
+{
+	local xchg="$1"; shift
+	local order="$1"; shift
+	local forder="${order:-_fence}"
+
+	printf "#if defined(arch_${xchg}${order})\n"
+	printf "#define raw_${xchg}${order} arch_${xchg}${order}\n"
+
+	if [ "${order}" != "_relaxed" ]; then
+		printf "#elif defined(arch_${xchg}_relaxed)\n"
+		printf "#define raw_${xchg}${order}(...) \\\\\n"
+		printf "	__atomic_op${forder}(arch_${xchg}, __VA_ARGS__)\n"
+	fi
+
+	if [ ! -z "${order}" ]; then
+		printf "#elif defined(arch_${xchg})\n"
+		printf "#define raw_${xchg}${order} arch_${xchg}\n"
+	fi
+
+	printf "#else\n"
+	printf "extern void raw_${xchg}${order}_not_implemented(void);\n"
+	printf "#define raw_${xchg}${order}(...) raw_${xchg}${order}_not_implemented()\n"
+	printf "#endif\n\n"
+}
+
+gen_xchg_fallbacks()
+{
+	local xchg="$1"; shift
+
+	for order in "" "_acquire" "_release" "_relaxed"; do
+		gen_xchg_order_fallback "${xchg}" "${order}"
+	done
+}
+
+gen_try_cmpxchg_fallback()
+{
+	local prefix="$1"; shift
+	local cmpxchg="$1"; shift;
+	local suffix="$1"; shift;
+
+cat <<EOF
+#define raw_${prefix}try_${cmpxchg}${suffix}(_ptr, _oldp, _new) \\
+({ \\
+	typeof(*(_ptr)) *___op = (_oldp), ___o = *___op, ___r; \\
+	___r = raw_${prefix}${cmpxchg}${suffix}((_ptr), ___o, (_new)); \\
+	if (unlikely(___r != ___o)) \\
+		*___op = ___r; \\
+	likely(___r == ___o); \\
+})
+EOF
+}
+
+gen_try_cmpxchg_order_fallback()
+{
+	local cmpxchg="$1"; shift
+	local order="$1"; shift
+	local forder="${order:-_fence}"
+
+	printf "#if defined(arch_try_${cmpxchg}${order})\n"
+	printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}${order}\n"
+
+	if [ "${order}" != "_relaxed" ]; then
+		printf "#elif defined(arch_try_${cmpxchg}_relaxed)\n"
+		printf "#define raw_try_${cmpxchg}${order}(...) \\\\\n"
+		printf "	__atomic_op${forder}(arch_try_${cmpxchg}, __VA_ARGS__)\n"
+	fi
+
+	if [ ! -z "${order}" ]; then
+		printf "#elif defined(arch_try_${cmpxchg})\n"
+		printf "#define raw_try_${cmpxchg}${order} arch_try_${cmpxchg}\n"
+	fi
+
+	printf "#else\n"
+	gen_try_cmpxchg_fallback "" "${cmpxchg}" "${order}"
+	printf "#endif\n\n"
+}
+
+gen_try_cmpxchg_order_fallbacks()
+{
+	local cmpxchg="$1"; shift;
+
+	for order in "" "_acquire" "_release" "_relaxed"; do
+		gen_try_cmpxchg_order_fallback "${cmpxchg}" "${order}"
+	done
+}
+
+gen_def_and_try_cmpxchg_fallback()
+{
+	local prefix="$1"; shift
+	local cmpxchg="$1"; shift
+	local suffix="$1"; shift
+
+	printf "#define raw_${prefix}${cmpxchg}${suffix} arch_${prefix}${cmpxchg}${suffix}\n\n"
+	printf "#ifdef arch_${prefix}try_${cmpxchg}${suffix}\n"
+	printf "#define raw_${prefix}try_${cmpxchg}${suffix} arch_${prefix}try_${cmpxchg}${suffix}\n"
+	printf "#else\n"
+	gen_try_cmpxchg_fallback "${prefix}" "${cmpxchg}" "${suffix}"
+	printf "#endif\n\n"
+}
+
+cat << EOF
+// SPDX-License-Identifier: GPL-2.0
+
+// Generated by $0
+// DO NOT MODIFY THIS FILE DIRECTLY
+
+#ifndef _LINUX_ATOMIC_FALLBACK_H
+#define _LINUX_ATOMIC_FALLBACK_H
+
+#include <linux/compiler.h>
+
+EOF
+
+for xchg in "xchg" "cmpxchg" "cmpxchg64" "cmpxchg128"; do
+	gen_xchg_fallbacks "${xchg}"
+done
+
+for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
+	gen_try_cmpxchg_order_fallbacks "${cmpxchg}"
+done
+
+for cmpxchg in "cmpxchg" "cmpxchg64" "cmpxchg128"; do
+	gen_def_and_try_cmpxchg_fallback "" "${cmpxchg}" "_local"
+done
+
+for cmpxchg in "cmpxchg"; do
+	gen_def_and_try_cmpxchg_fallback "sync_" "${cmpxchg}" ""
+done
+
+grep '^[a-z]' "$1" | while read name meta args; do
+	gen_proto "${meta}" "${name}" "atomic" "int" ${args}
+done
+
+cat <<EOF
+#ifdef CONFIG_GENERIC_ATOMIC64
+#include <asm-generic/atomic64.h>
+#endif
+
+EOF
+
+grep '^[a-z]' "$1" | while read name meta args; do
+	gen_proto "${meta}" "${name}" "atomic64" "s64" ${args}
+done
+
+cat <<EOF
+#endif /* _LINUX_ATOMIC_FALLBACK_H */
+EOF