summaryrefslogtreecommitdiffstats
path: root/arch/x86/virt/hw.c
diff options
context:
space:
mode:
authorSang-Heon Jeon <ekffu200098@gmail.com>2026-08-24 01:18:14 +0900
committerJulia Lawall <Julia.Lawall@inria.fr>2026-08-23 22:12:11 +0200
commit2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c (patch)
treedcc4da1d99f17541c5590ac63c77da4bf372a4ab /arch/x86/virt/hw.c
downloadlinux-stable-2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c.tar.gz
linux-stable-2bd30b8dc75b823dd7bbef9d7bc68e1fd7ecee5c.zip
coccinelle: alloc_cast: drop removed allocatorsgrafted
- pci_alloc_consistent() and pci_zalloc_consistent() were removed by commit 7968778914e5 ("PCI: Remove the deprecated "pci-dma-compat.h" API") - kmem_alloc() was removed by commit f078d4ea8276 ("xfs: convert kmem_alloc() to kmalloc()") - kmem_zalloc() was removed by commit 10634530f7ba ("xfs: convert kmem_zalloc() to kzalloc()") - kmem_zone_alloc() and kmem_zone_zalloc() were removed by commit bae633a4a283 ("xfs: remove xfs_zone_{alloc,zalloc} helpers") So drop them from the rules. No functional change. Signed-off-by: Sang-Heon Jeon <ekffu200098@gmail.com> Signed-off-by: Julia Lawall <Julia.Lawall@inria.fr>
Diffstat (limited to 'arch/x86/virt/hw.c')
-rw-r--r--arch/x86/virt/hw.c373
1 files changed, 373 insertions, 0 deletions
diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c
new file mode 100644
index 000000000..a236447ac
--- /dev/null
+++ b/arch/x86/virt/hw.c
@@ -0,0 +1,373 @@
+// SPDX-License-Identifier: GPL-2.0-only
+#include <linux/cpu.h>
+#include <linux/cpumask.h>
+#include <linux/errno.h>
+#include <linux/kvm_types.h>
+#include <linux/list.h>
+#include <linux/percpu.h>
+
+#include <asm/perf_event.h>
+#include <asm/processor.h>
+#include <asm/virt.h>
+#include <asm/vmx.h>
+
+struct x86_virt_ops {
+ int feature;
+ int (*enable_virtualization_cpu)(void);
+ int (*disable_virtualization_cpu)(void);
+ void (*emergency_disable_virtualization_cpu)(void);
+};
+static struct x86_virt_ops virt_ops __ro_after_init;
+
+__visible bool virt_rebooting;
+EXPORT_SYMBOL_FOR_KVM(virt_rebooting);
+
+static DEFINE_PER_CPU(int, virtualization_nr_users);
+
+static cpu_emergency_virt_cb __rcu *kvm_emergency_callback;
+
+void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback)
+{
+ if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback)))
+ return;
+
+ rcu_assign_pointer(kvm_emergency_callback, callback);
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_register_emergency_callback);
+
+void x86_virt_unregister_emergency_callback(cpu_emergency_virt_cb *callback)
+{
+ if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback) != callback))
+ return;
+
+ rcu_assign_pointer(kvm_emergency_callback, NULL);
+ synchronize_rcu();
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_unregister_emergency_callback);
+
+static void x86_virt_invoke_kvm_emergency_callback(void)
+{
+ cpu_emergency_virt_cb *kvm_callback;
+
+ /*
+ * RCU may not be watching the crashing CPU here, so rcu_dereference()
+ * triggers a suspicious-RCU-usage splat. In principle, a concurrent
+ * KVM module unload could race with this read; see commit 2baa33a8ddd6
+ * ("KVM: x86: Leave user-return notifier registered on reboot/shutdown")
+ * which notes that nothing prevents module unload during panic/reboot.
+ *
+ * However, taking a lock here would be riskier than the current race:
+ * the system is going down via NMI shootdown, and any lock could be
+ * held by an already-stopped CPU. Use rcu_dereference_raw() to silence
+ * the lockdep splat and accept the comically small remaining race;
+ * panic context inherently cannot guarantee complete correctness.
+ */
+ kvm_callback = rcu_dereference_raw(kvm_emergency_callback);
+ if (kvm_callback)
+ kvm_callback();
+}
+
+#if IS_ENABLED(CONFIG_KVM_INTEL)
+static DEFINE_PER_CPU(struct vmcs *, root_vmcs);
+
+static int x86_virt_cpu_vmxon(void)
+{
+ u64 vmxon_pointer = __pa(per_cpu(root_vmcs, raw_smp_processor_id()));
+ u64 msr;
+
+ cr4_set_bits(X86_CR4_VMXE);
+
+ asm goto("1: vmxon %[vmxon_pointer]\n\t"
+ _ASM_EXTABLE(1b, %l[fault])
+ : : [vmxon_pointer] "m"(vmxon_pointer)
+ : : fault);
+ return 0;
+
+fault:
+ WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n",
+ rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr);
+ cr4_clear_bits(X86_CR4_VMXE);
+
+ return -EFAULT;
+}
+
+static int x86_vmx_enable_virtualization_cpu(void)
+{
+ int r;
+
+ if (cr4_read_shadow() & X86_CR4_VMXE)
+ return -EBUSY;
+
+ intel_pt_handle_vmx(1);
+
+ r = x86_virt_cpu_vmxon();
+ if (r) {
+ intel_pt_handle_vmx(0);
+ return r;
+ }
+
+ return 0;
+}
+
+/*
+ * Disable VMX and clear CR4.VMXE (even if VMXOFF faults)
+ *
+ * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to
+ * atomically track post-VMXON state, e.g. this may be called in NMI context.
+ * Eat all faults as all other faults on VMXOFF faults are mode related, i.e.
+ * faults are guaranteed to be due to the !post-VMXON check unless the CPU is
+ * magically in RM, VM86, compat mode, or at CPL>0.
+ */
+static int x86_vmx_disable_virtualization_cpu(void)
+{
+ int r = -EIO;
+
+ asm goto("1: vmxoff\n\t"
+ _ASM_EXTABLE(1b, %l[fault])
+ ::: "cc", "memory" : fault);
+ r = 0;
+
+fault:
+ cr4_clear_bits(X86_CR4_VMXE);
+ intel_pt_handle_vmx(0);
+ return r;
+}
+
+static void x86_vmx_emergency_disable_virtualization_cpu(void)
+{
+ virt_rebooting = true;
+
+ /*
+ * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be
+ * set in task context. If this races with _another_ emergency call
+ * from NMI context, VMCLEAR (in KVM) and VMXOFF may #UD, but KVM and
+ * the kernel will eat those faults due to virt_rebooting being set by
+ * the interrupting NMI callback.
+ */
+ if (!(__read_cr4() & X86_CR4_VMXE))
+ return;
+
+ x86_virt_invoke_kvm_emergency_callback();
+
+ x86_vmx_disable_virtualization_cpu();
+}
+
+static __init void x86_vmx_exit(void)
+{
+ int cpu;
+
+ for_each_possible_cpu(cpu) {
+ free_page((unsigned long)per_cpu(root_vmcs, cpu));
+ per_cpu(root_vmcs, cpu) = NULL;
+ }
+}
+
+static __init int __x86_vmx_init(void)
+{
+ const struct x86_virt_ops vmx_ops = {
+ .feature = X86_FEATURE_VMX,
+ .enable_virtualization_cpu = x86_vmx_enable_virtualization_cpu,
+ .disable_virtualization_cpu = x86_vmx_disable_virtualization_cpu,
+ .emergency_disable_virtualization_cpu = x86_vmx_emergency_disable_virtualization_cpu,
+ };
+
+ u64 basic_msr;
+ u32 rev_id;
+ int cpu;
+
+ if (!cpu_feature_enabled(X86_FEATURE_VMX))
+ return -EOPNOTSUPP;
+
+ rdmsrq(MSR_IA32_VMX_BASIC, basic_msr);
+
+ /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */
+ if (WARN_ON_ONCE(vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE))
+ return -EIO;
+
+ /*
+ * Even if eVMCS is enabled (or will be enabled?), and even though not
+ * explicitly documented by TLFS, the root VMCS passed to VMXON should
+ * still be marked with the revision_id reported by the physical CPU.
+ */
+ rev_id = vmx_basic_vmcs_revision_id(basic_msr);
+
+ for_each_possible_cpu(cpu) {
+ int node = cpu_to_node(cpu);
+ struct page *page;
+ struct vmcs *vmcs;
+
+ page = alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
+ if (WARN_ON_ONCE(!page)) {
+ x86_vmx_exit();
+ return -ENOMEM;
+ }
+
+ vmcs = page_address(page);
+ vmcs->hdr.revision_id = rev_id;
+ per_cpu(root_vmcs, cpu) = vmcs;
+ }
+
+ memcpy(&virt_ops, &vmx_ops, sizeof(virt_ops));
+ return 0;
+}
+
+static __init int x86_vmx_init(void)
+{
+ int r;
+
+ r = __x86_vmx_init();
+ if (r)
+ setup_clear_cpu_cap(X86_FEATURE_VMX);
+ return r;
+}
+#else
+static __init int x86_vmx_init(void) { return -EOPNOTSUPP; }
+static __init void x86_vmx_exit(void) { }
+#endif
+
+#if IS_ENABLED(CONFIG_KVM_AMD)
+static int x86_svm_enable_virtualization_cpu(void)
+{
+ u64 efer;
+
+ rdmsrq(MSR_EFER, efer);
+ if (efer & EFER_SVME)
+ return -EBUSY;
+
+ wrmsrq(MSR_EFER, efer | EFER_SVME);
+ return 0;
+}
+
+static int x86_svm_disable_virtualization_cpu(void)
+{
+ int r = -EIO;
+ u64 efer;
+
+ /*
+ * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and
+ * NMI aren't blocked.
+ */
+ asm goto("1: stgi\n\t"
+ _ASM_EXTABLE(1b, %l[fault])
+ ::: "memory" : fault);
+ r = 0;
+
+fault:
+ rdmsrq(MSR_EFER, efer);
+ wrmsrq(MSR_EFER, efer & ~EFER_SVME);
+ return r;
+}
+
+static void x86_svm_emergency_disable_virtualization_cpu(void)
+{
+ u64 efer;
+
+ virt_rebooting = true;
+
+ rdmsrq(MSR_EFER, efer);
+ if (!(efer & EFER_SVME))
+ return;
+
+ x86_virt_invoke_kvm_emergency_callback();
+
+ x86_svm_disable_virtualization_cpu();
+}
+
+static __init int x86_svm_init(void)
+{
+ const struct x86_virt_ops svm_ops = {
+ .feature = X86_FEATURE_SVM,
+ .enable_virtualization_cpu = x86_svm_enable_virtualization_cpu,
+ .disable_virtualization_cpu = x86_svm_disable_virtualization_cpu,
+ .emergency_disable_virtualization_cpu = x86_svm_emergency_disable_virtualization_cpu,
+ };
+
+ if (!cpu_feature_enabled(X86_FEATURE_SVM) ||
+ cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT))
+ return -EOPNOTSUPP;
+
+ memcpy(&virt_ops, &svm_ops, sizeof(virt_ops));
+ return 0;
+}
+#else
+static __init int x86_svm_init(void) { return -EOPNOTSUPP; }
+#endif
+
+int x86_virt_get_ref(int feat)
+{
+ int r;
+
+ /* Ensure the !feature check can't get false positives. */
+ BUILD_BUG_ON(!X86_FEATURE_SVM || !X86_FEATURE_VMX);
+
+ if (!virt_ops.feature || virt_ops.feature != feat)
+ return -EOPNOTSUPP;
+
+ guard(preempt)();
+
+ if (this_cpu_inc_return(virtualization_nr_users) > 1)
+ return 0;
+
+ r = virt_ops.enable_virtualization_cpu();
+ if (r)
+ WARN_ON_ONCE(this_cpu_dec_return(virtualization_nr_users));
+
+ return r;
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_get_ref);
+
+void x86_virt_put_ref(int feat)
+{
+ guard(preempt)();
+
+ if (WARN_ON_ONCE(!this_cpu_read(virtualization_nr_users)) ||
+ this_cpu_dec_return(virtualization_nr_users))
+ return;
+
+ BUG_ON(virt_ops.disable_virtualization_cpu() && !virt_rebooting);
+}
+EXPORT_SYMBOL_FOR_KVM(x86_virt_put_ref);
+
+/*
+ * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during
+ * reboot. VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if
+ * GIF=0, i.e. if the crash occurred between CLGI and STGI.
+ */
+int x86_virt_emergency_disable_virtualization_cpu(void)
+{
+ if (!virt_ops.feature)
+ return -EOPNOTSUPP;
+
+ /*
+ * IRQs must be disabled as virtualization is enabled in hardware via
+ * function call IPIs, i.e. IRQs need to be disabled to guarantee
+ * virtualization stays disabled.
+ */
+ lockdep_assert_irqs_disabled();
+
+ /*
+ * Do the NMI shootdown even if virtualization is off on _this_ CPU, as
+ * other CPUs may have virtualization enabled.
+ *
+ * TODO: Track whether or not virtualization might be enabled on other
+ * CPUs? May not be worth avoiding the NMI shootdown...
+ */
+ virt_ops.emergency_disable_virtualization_cpu();
+ return 0;
+}
+
+void __init x86_virt_init(void)
+{
+ /*
+ * Attempt to initialize both SVM and VMX, and simply use whichever one
+ * is present. Rsefuse to enable/use SVM or VMX if both are somehow
+ * supported. No known CPU supports both SVM and VMX.
+ */
+ bool has_vmx = !x86_vmx_init();
+ bool has_svm = !x86_svm_init();
+
+ if (WARN_ON_ONCE(has_vmx && has_svm)) {
+ x86_vmx_exit();
+ memset(&virt_ops, 0, sizeof(virt_ops));
+ }
+}