diff options
Diffstat (limited to 'arch/x86/virt')
| -rw-r--r-- | arch/x86/virt/Makefile | 4 | ||||
| -rw-r--r-- | arch/x86/virt/hw.c | 373 | ||||
| -rw-r--r-- | arch/x86/virt/svm/Makefile | 4 | ||||
| -rw-r--r-- | arch/x86/virt/svm/cmdline.c | 45 | ||||
| -rw-r--r-- | arch/x86/virt/svm/sev.c | 1102 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/Makefile | 2 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/Makefile | 2 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/seamcall.S | 64 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/seamcall_internal.h | 109 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/seamldr.c | 368 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/tdx.c | 2034 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/tdx.h | 116 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/tdx_global_metadata.c | 133 | ||||
| -rw-r--r-- | arch/x86/virt/vmx/tdx/tdxcall.S | 220 |
14 files changed, 4576 insertions, 0 deletions
diff --git a/arch/x86/virt/Makefile b/arch/x86/virt/Makefile new file mode 100644 index 000000000..6e4857516 --- /dev/null +++ b/arch/x86/virt/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-y += svm/ vmx/ + +obj-$(subst m,y,$(CONFIG_KVM_X86)) += hw.o
\ No newline at end of file diff --git a/arch/x86/virt/hw.c b/arch/x86/virt/hw.c new file mode 100644 index 000000000..7e9091c64 --- /dev/null +++ b/arch/x86/virt/hw.c @@ -0,0 +1,373 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include <linux/cpu.h> +#include <linux/cpumask.h> +#include <linux/errno.h> +#include <linux/kvm_types.h> +#include <linux/list.h> +#include <linux/percpu.h> + +#include <asm/perf_event.h> +#include <asm/processor.h> +#include <asm/virt.h> +#include <asm/vmx.h> + +struct x86_virt_ops { + int feature; + int (*enable_virtualization_cpu)(void); + int (*disable_virtualization_cpu)(void); + void (*emergency_disable_virtualization_cpu)(void); +}; +static struct x86_virt_ops virt_ops __ro_after_init; + +__visible bool virt_rebooting; +EXPORT_SYMBOL_FOR_KVM(virt_rebooting); + +static DEFINE_PER_CPU(int, virtualization_nr_users); + +static cpu_emergency_virt_cb __rcu *kvm_emergency_callback; + +void x86_virt_register_emergency_callback(cpu_emergency_virt_cb *callback) +{ + if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback))) + return; + + rcu_assign_pointer(kvm_emergency_callback, callback); +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_register_emergency_callback); + +void x86_virt_unregister_emergency_callback(cpu_emergency_virt_cb *callback) +{ + if (WARN_ON_ONCE(rcu_access_pointer(kvm_emergency_callback) != callback)) + return; + + rcu_assign_pointer(kvm_emergency_callback, NULL); + synchronize_rcu(); +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_unregister_emergency_callback); + +static void x86_virt_invoke_kvm_emergency_callback(void) +{ + cpu_emergency_virt_cb *kvm_callback; + + /* + * RCU may not be watching the crashing CPU here, so rcu_dereference() + * triggers a suspicious-RCU-usage splat. In principle, a concurrent + * KVM module unload could race with this read; see commit 2baa33a8ddd6 + * ("KVM: x86: Leave user-return notifier registered on reboot/shutdown") + * which notes that nothing prevents module unload during panic/reboot. + * + * However, taking a lock here would be riskier than the current race: + * the system is going down via NMI shootdown, and any lock could be + * held by an already-stopped CPU. Use rcu_dereference_raw() to silence + * the lockdep splat and accept the comically small remaining race; + * panic context inherently cannot guarantee complete correctness. + */ + kvm_callback = rcu_dereference_raw(kvm_emergency_callback); + if (kvm_callback) + kvm_callback(); +} + +#if IS_ENABLED(CONFIG_KVM_INTEL) +static DEFINE_PER_CPU(struct vmcs *, root_vmcs); + +static int x86_virt_cpu_vmxon(void) +{ + u64 vmxon_pointer = __pa(per_cpu(root_vmcs, raw_smp_processor_id())); + u64 msr; + + cr4_set_bits(X86_CR4_VMXE); + + asm goto("1: vmxon %[vmxon_pointer]\n\t" + _ASM_EXTABLE(1b, %l[fault]) + : : [vmxon_pointer] "m"(vmxon_pointer) + : : fault); + return 0; + +fault: + WARN_ONCE(1, "VMXON faulted, MSR_IA32_FEAT_CTL (0x3a) = 0x%llx\n", + rdmsrq_safe(MSR_IA32_FEAT_CTL, &msr) ? 0xdeadbeef : msr); + cr4_clear_bits(X86_CR4_VMXE); + + return -EFAULT; +} + +static int x86_vmx_enable_virtualization_cpu(void) +{ + int r; + + if (cr4_read_shadow() & X86_CR4_VMXE) + return -EBUSY; + + intel_pt_handle_vmx(1); + + r = x86_virt_cpu_vmxon(); + if (r) { + intel_pt_handle_vmx(0); + return r; + } + + return 0; +} + +/* + * Disable VMX and clear CR4.VMXE (even if VMXOFF faults) + * + * Note, VMXOFF causes a #UD if the CPU is !post-VMXON, but it's impossible to + * atomically track post-VMXON state, e.g. this may be called in NMI context. + * Eat all faults as all other faults on VMXOFF faults are mode related, i.e. + * faults are guaranteed to be due to the !post-VMXON check unless the CPU is + * magically in RM, VM86, compat mode, or at CPL>0. + */ +static int x86_vmx_disable_virtualization_cpu(void) +{ + int r = -EIO; + + asm goto("1: vmxoff\n\t" + _ASM_EXTABLE(1b, %l[fault]) + ::: "cc", "memory" : fault); + r = 0; + +fault: + cr4_clear_bits(X86_CR4_VMXE); + intel_pt_handle_vmx(0); + return r; +} + +static void x86_vmx_emergency_disable_virtualization_cpu(void) +{ + virt_rebooting = true; + + /* + * Note, CR4.VMXE can be _cleared_ in NMI context, but it can only be + * set in task context. If this races with _another_ emergency call + * from NMI context, VMCLEAR (in KVM) and VMXOFF may #UD, but KVM and + * the kernel will eat those faults due to virt_rebooting being set by + * the interrupting NMI callback. + */ + if (!(__read_cr4() & X86_CR4_VMXE)) + return; + + x86_virt_invoke_kvm_emergency_callback(); + + x86_vmx_disable_virtualization_cpu(); +} + +static __init void x86_vmx_exit(void) +{ + int cpu; + + for_each_possible_cpu(cpu) { + free_page((unsigned long)per_cpu(root_vmcs, cpu)); + per_cpu(root_vmcs, cpu) = NULL; + } +} + +static __init int __x86_vmx_init(void) +{ + const struct x86_virt_ops vmx_ops = { + .feature = X86_FEATURE_VMX, + .enable_virtualization_cpu = x86_vmx_enable_virtualization_cpu, + .disable_virtualization_cpu = x86_vmx_disable_virtualization_cpu, + .emergency_disable_virtualization_cpu = x86_vmx_emergency_disable_virtualization_cpu, + }; + + u64 basic_msr; + u32 rev_id; + int cpu; + + if (!cpu_feature_enabled(X86_FEATURE_VMX)) + return -EOPNOTSUPP; + + rdmsrq(MSR_IA32_VMX_BASIC, basic_msr); + + /* IA-32 SDM Vol 3B: VMCS size is never greater than 4kB. */ + if (WARN_ON_ONCE(vmx_basic_vmcs_size(basic_msr) > PAGE_SIZE)) + return -EIO; + + /* + * Even if eVMCS is enabled (or will be enabled?), and even though not + * explicitly documented by TLFS, the root VMCS passed to VMXON should + * still be marked with the revision_id reported by the physical CPU. + */ + rev_id = vmx_basic_vmcs_revision_id(basic_msr); + + for_each_possible_cpu(cpu) { + int node = cpu_to_node(cpu); + struct page *page; + struct vmcs *vmcs; + + page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0); + if (WARN_ON_ONCE(!page)) { + x86_vmx_exit(); + return -ENOMEM; + } + + vmcs = page_address(page); + vmcs->hdr.revision_id = rev_id; + per_cpu(root_vmcs, cpu) = vmcs; + } + + memcpy(&virt_ops, &vmx_ops, sizeof(virt_ops)); + return 0; +} + +static __init int x86_vmx_init(void) +{ + int r; + + r = __x86_vmx_init(); + if (r) + setup_clear_cpu_cap(X86_FEATURE_VMX); + return r; +} +#else +static __init int x86_vmx_init(void) { return -EOPNOTSUPP; } +static __init void x86_vmx_exit(void) { } +#endif + +#if IS_ENABLED(CONFIG_KVM_AMD) +static int x86_svm_enable_virtualization_cpu(void) +{ + u64 efer; + + rdmsrq(MSR_EFER, efer); + if (efer & EFER_SVME) + return -EBUSY; + + wrmsrq(MSR_EFER, efer | EFER_SVME); + return 0; +} + +static int x86_svm_disable_virtualization_cpu(void) +{ + int r = -EIO; + u64 efer; + + /* + * Force GIF=1 prior to disabling SVM, e.g. to ensure INIT and + * NMI aren't blocked. + */ + asm goto("1: stgi\n\t" + _ASM_EXTABLE(1b, %l[fault]) + ::: "memory" : fault); + r = 0; + +fault: + rdmsrq(MSR_EFER, efer); + wrmsrq(MSR_EFER, efer & ~EFER_SVME); + return r; +} + +static void x86_svm_emergency_disable_virtualization_cpu(void) +{ + u64 efer; + + virt_rebooting = true; + + rdmsrq(MSR_EFER, efer); + if (!(efer & EFER_SVME)) + return; + + x86_virt_invoke_kvm_emergency_callback(); + + x86_svm_disable_virtualization_cpu(); +} + +static __init int x86_svm_init(void) +{ + const struct x86_virt_ops svm_ops = { + .feature = X86_FEATURE_SVM, + .enable_virtualization_cpu = x86_svm_enable_virtualization_cpu, + .disable_virtualization_cpu = x86_svm_disable_virtualization_cpu, + .emergency_disable_virtualization_cpu = x86_svm_emergency_disable_virtualization_cpu, + }; + + if (!cpu_feature_enabled(X86_FEATURE_SVM) || + cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) + return -EOPNOTSUPP; + + memcpy(&virt_ops, &svm_ops, sizeof(virt_ops)); + return 0; +} +#else +static __init int x86_svm_init(void) { return -EOPNOTSUPP; } +#endif + +int x86_virt_get_ref(int feat) +{ + int r; + + /* Ensure the !feature check can't get false positives. */ + BUILD_BUG_ON(!X86_FEATURE_SVM || !X86_FEATURE_VMX); + + if (!virt_ops.feature || virt_ops.feature != feat) + return -EOPNOTSUPP; + + guard(preempt)(); + + if (this_cpu_inc_return(virtualization_nr_users) > 1) + return 0; + + r = virt_ops.enable_virtualization_cpu(); + if (r) + WARN_ON_ONCE(this_cpu_dec_return(virtualization_nr_users)); + + return r; +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_get_ref); + +void x86_virt_put_ref(int feat) +{ + guard(preempt)(); + + if (WARN_ON_ONCE(!this_cpu_read(virtualization_nr_users)) || + this_cpu_dec_return(virtualization_nr_users)) + return; + + BUG_ON(virt_ops.disable_virtualization_cpu() && !virt_rebooting); +} +EXPORT_SYMBOL_FOR_KVM(x86_virt_put_ref); + +/* + * Disable virtualization, i.e. VMX or SVM, to ensure INIT is recognized during + * reboot. VMX blocks INIT if the CPU is post-VMXON, and SVM blocks INIT if + * GIF=0, i.e. if the crash occurred between CLGI and STGI. + */ +int x86_virt_emergency_disable_virtualization_cpu(void) +{ + if (!virt_ops.feature) + return -EOPNOTSUPP; + + /* + * IRQs must be disabled as virtualization is enabled in hardware via + * function call IPIs, i.e. IRQs need to be disabled to guarantee + * virtualization stays disabled. + */ + lockdep_assert_irqs_disabled(); + + /* + * Do the NMI shootdown even if virtualization is off on _this_ CPU, as + * other CPUs may have virtualization enabled. + * + * TODO: Track whether or not virtualization might be enabled on other + * CPUs? May not be worth avoiding the NMI shootdown... + */ + virt_ops.emergency_disable_virtualization_cpu(); + return 0; +} + +void __init x86_virt_init(void) +{ + /* + * Attempt to initialize both SVM and VMX, and simply use whichever one + * is present. Rsefuse to enable/use SVM or VMX if both are somehow + * supported. No known CPU supports both SVM and VMX. + */ + bool has_vmx = !x86_vmx_init(); + bool has_svm = !x86_svm_init(); + + if (WARN_ON_ONCE(has_vmx && has_svm)) { + x86_vmx_exit(); + memset(&virt_ops, 0, sizeof(virt_ops)); + } +} diff --git a/arch/x86/virt/svm/Makefile b/arch/x86/virt/svm/Makefile new file mode 100644 index 000000000..eca6d7135 --- /dev/null +++ b/arch/x86/virt/svm/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 + +obj-$(CONFIG_KVM_AMD_SEV) += sev.o +obj-$(CONFIG_CPU_SUP_AMD) += cmdline.o diff --git a/arch/x86/virt/svm/cmdline.c b/arch/x86/virt/svm/cmdline.c new file mode 100644 index 000000000..affa2759f --- /dev/null +++ b/arch/x86/virt/svm/cmdline.c @@ -0,0 +1,45 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD SVM-SEV command line parsing support + * + * Copyright (C) 2023 - 2024 Advanced Micro Devices, Inc. + * + * Author: Michael Roth <michael.roth@amd.com> + */ + +#include <linux/string.h> +#include <linux/printk.h> +#include <linux/cache.h> +#include <linux/cpufeature.h> + +#include <asm/sev-common.h> + +struct sev_config sev_cfg __read_mostly; + +static int __init init_sev_config(char *str) +{ + char *s; + + while ((s = strsep(&str, ","))) { + if (!strcmp(s, "debug")) { + sev_cfg.debug = true; + continue; + } + + if (!strcmp(s, "nosnp")) { + if (!cpu_feature_enabled(X86_FEATURE_HYPERVISOR)) { + setup_clear_cpu_cap(X86_FEATURE_SEV_SNP); + cc_platform_clear(CC_ATTR_HOST_SEV_SNP); + continue; + } else { + goto warn; + } + } + +warn: + pr_info("SEV command-line option '%s' was not recognized\n", s); + } + + return 1; +} +__setup("sev=", init_sev_config); diff --git a/arch/x86/virt/svm/sev.c b/arch/x86/virt/svm/sev.c new file mode 100644 index 000000000..8bcdce98f --- /dev/null +++ b/arch/x86/virt/svm/sev.c @@ -0,0 +1,1102 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * AMD SVM-SEV Host Support. + * + * Copyright (C) 2023 Advanced Micro Devices, Inc. + * + * Author: Ashish Kalra <ashish.kalra@amd.com> + * + */ + +#include <linux/cc_platform.h> +#include <linux/printk.h> +#include <linux/mm_types.h> +#include <linux/set_memory.h> +#include <linux/memblock.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/cpumask.h> +#include <linux/iommu.h> +#include <linux/amd-iommu.h> +#include <linux/nospec.h> + +#include <asm/sev.h> +#include <asm/processor.h> +#include <asm/setup.h> +#include <asm/svm.h> +#include <asm/smp.h> +#include <asm/cpu.h> +#include <asm/apic.h> +#include <asm/cpuid/api.h> +#include <asm/cmdline.h> +#include <asm/iommu.h> +#include <asm/msr.h> + +/* + * The RMP entry information as returned by the RMPREAD instruction. + */ +struct rmpentry { + u64 gpa; + u8 assigned :1, + rsvd1 :7; + u8 pagesize :1, + hpage_region_status :1, + rsvd2 :6; + u8 immutable :1, + rsvd3 :7; + u8 rsvd4; + u32 asid; +} __packed; + +/* + * The raw RMP entry format is not architectural. The format is defined in PPR + * Family 19h Model 01h, Rev B1 processor. This format represents the actual + * entry in the RMP table memory. The bitfield definitions are used for machines + * without the RMPREAD instruction (Zen3 and Zen4), otherwise the "hi" and "lo" + * fields are only used for dumping the raw data. + */ +struct rmpentry_raw { + union { + struct { + u64 assigned : 1, + pagesize : 1, + immutable : 1, + rsvd1 : 9, + gpa : 39, + asid : 10, + vmsa : 1, + validated : 1, + rsvd2 : 1; + }; + u64 lo; + }; + u64 hi; +} __packed; + +/* + * The first 16KB from the RMP_BASE is used by the processor for the + * bookkeeping, the range needs to be added during the RMP entry lookup. + */ +#define RMPTABLE_CPU_BOOKKEEPING_SZ 0x4000 + +/* + * For a non-segmented RMP table, use the maximum physical addressing as the + * segment size in order to always arrive at index 0 in the table. + */ +#define RMPTABLE_NON_SEGMENTED_SHIFT 52 + +struct rmp_segment_desc { + struct rmpentry_raw *rmp_entry; + u64 max_index; + u64 size; +}; + +/* + * Segmented RMP Table support. + * - The segment size is used for two purposes: + * - Identify the amount of memory covered by an RMP segment + * - Quickly locate an RMP segment table entry for a physical address + * + * - The RMP segment table contains pointers to an RMP table that covers + * a specific portion of memory. There can be up to 512 8-byte entries, + * one pages worth. + */ +#define RST_ENTRY_MAPPED_SIZE(x) ((x) & GENMASK_ULL(19, 0)) +#define RST_ENTRY_SEGMENT_BASE(x) ((x) & GENMASK_ULL(51, 20)) + +#define RST_SIZE SZ_4K +static struct rmp_segment_desc **rmp_segment_table __ro_after_init; +static unsigned int rst_max_index __ro_after_init = 512; + +static unsigned int rmp_segment_shift; +static u64 rmp_segment_size; +static u64 rmp_segment_mask; + +#define RST_ENTRY_INDEX(x) ((x) >> rmp_segment_shift) +#define RMP_ENTRY_INDEX(x) ((u64)(PHYS_PFN((x) & rmp_segment_mask))) + +static u64 rmp_cfg; + +static void *rmp_bookkeeping __ro_after_init; + +/* Mask to apply to a PFN to get the first PFN of a 2MB page */ +#define PFN_PMD_MASK GENMASK_ULL(63, PMD_SHIFT - PAGE_SHIFT) + +static u64 probed_rmp_base, probed_rmp_size; + +static LIST_HEAD(snp_leaked_pages_list); +static DEFINE_SPINLOCK(snp_leaked_pages_list_lock); + +static unsigned long snp_nr_leaked_pages; + +#undef pr_fmt +#define pr_fmt(fmt) "SEV-SNP: " fmt + +static void mfd_reconfigure(void *arg) +{ + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return; + + if (arg) + msr_set_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT); + else + msr_clear_bit(MSR_AMD64_SYSCFG, MSR_AMD64_SYSCFG_MFDM_BIT); +} + +static void snp_enable(void *arg) +{ + u64 val; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return; + + rdmsrq(MSR_AMD64_SYSCFG, val); + + val |= MSR_AMD64_SYSCFG_SNP_EN; + val |= MSR_AMD64_SYSCFG_SNP_VMPL_EN; + + wrmsrq(MSR_AMD64_SYSCFG, val); +} + +static void __init __snp_fixup_e820_tables(u64 pa) +{ + if (IS_ALIGNED(pa, PMD_SIZE)) + return; + + /* + * Handle cases where the RMP table placement by the BIOS is not + * 2M aligned and the kexec kernel could try to allocate + * from within that chunk which then causes a fatal RMP fault. + * + * The e820_table needs to be updated as it is converted to + * kernel memory resources and used by KEXEC_FILE_LOAD syscall + * to load kexec segments. + * + * The e820_table_firmware needs to be updated as it is exposed + * to sysfs and used by the KEXEC_LOAD syscall to load kexec + * segments. + * + * The e820_table_kexec needs to be updated as it passed to + * the kexec-ed kernel. + */ + pa = ALIGN_DOWN(pa, PMD_SIZE); + if (e820__mapped_any(pa, pa + PMD_SIZE, E820_TYPE_RAM)) { + pr_info("Reserving start/end of RMP table on a 2MB boundary [0x%016llx]\n", pa); + e820__range_update(pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED); + e820__range_update_table(e820_table_kexec, pa, PMD_SIZE, E820_TYPE_RAM, E820_TYPE_RESERVED); + if (!memblock_is_region_reserved(pa, PMD_SIZE)) + memblock_reserve(pa, PMD_SIZE); + } +} + +static void __init fixup_e820_tables_for_segmented_rmp(void) +{ + u64 pa, *rst, size, mapped_size; + unsigned int i; + + __snp_fixup_e820_tables(probed_rmp_base); + + pa = probed_rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ; + + __snp_fixup_e820_tables(pa + RST_SIZE); + + rst = early_memremap(pa, RST_SIZE); + if (!rst) + return; + + for (i = 0; i < rst_max_index; i++) { + pa = RST_ENTRY_SEGMENT_BASE(rst[i]); + mapped_size = RST_ENTRY_MAPPED_SIZE(rst[i]); + if (!mapped_size) + continue; + + __snp_fixup_e820_tables(pa); + + /* + * Mapped size in GB. Mapped size is allowed to exceed + * the segment coverage size, but gets reduced to the + * segment coverage size. + */ + mapped_size <<= 30; + if (mapped_size > rmp_segment_size) + mapped_size = rmp_segment_size; + + /* Calculate the RMP segment size (16 bytes/page mapped) */ + size = PHYS_PFN(mapped_size) << 4; + + __snp_fixup_e820_tables(pa + size); + } + + early_memunmap(rst, RST_SIZE); +} + +static void __init fixup_e820_tables_for_contiguous_rmp(void) +{ + __snp_fixup_e820_tables(probed_rmp_base); + __snp_fixup_e820_tables(probed_rmp_base + probed_rmp_size); +} + +void __init snp_fixup_e820_tables(void) +{ + if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) { + fixup_e820_tables_for_segmented_rmp(); + } else { + fixup_e820_tables_for_contiguous_rmp(); + } +} + +static void clear_rmp(void) +{ + unsigned int i; + u64 val; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return; + + /* Clearing the RMP while SNP is enabled will cause an exception */ + rdmsrq(MSR_AMD64_SYSCFG, val); + if (WARN_ON_ONCE(val & MSR_AMD64_SYSCFG_SNP_EN)) + return; + + memset(rmp_bookkeeping, 0, RMPTABLE_CPU_BOOKKEEPING_SZ); + + for (i = 0; i < rst_max_index; i++) { + struct rmp_segment_desc *desc; + + desc = rmp_segment_table[i]; + if (!desc) + continue; + + memset(desc->rmp_entry, 0, desc->size); + } +} + +static bool __init alloc_rmp_segment_desc(u64 segment_pa, u64 segment_size, u64 pa) +{ + u64 rst_index, rmp_segment_size_max; + struct rmp_segment_desc *desc; + void *rmp_segment; + + /* Calculate the maximum size an RMP can be (16 bytes/page mapped) */ + rmp_segment_size_max = PHYS_PFN(rmp_segment_size) << 4; + + /* Validate the RMP segment size */ + if (segment_size > rmp_segment_size_max) { + pr_err("Invalid RMP size 0x%llx for configured segment size 0x%llx\n", + segment_size, rmp_segment_size_max); + return false; + } + + /* Validate the RMP segment table index */ + rst_index = RST_ENTRY_INDEX(pa); + if (rst_index >= rst_max_index) { + pr_err("Invalid RMP segment base address 0x%llx for configured segment size 0x%llx\n", + pa, rmp_segment_size); + return false; + } + + if (rmp_segment_table[rst_index]) { + pr_err("RMP segment descriptor already exists at index %llu\n", rst_index); + return false; + } + + rmp_segment = memremap(segment_pa, segment_size, MEMREMAP_WB); + if (!rmp_segment) { + pr_err("Failed to map RMP segment addr 0x%llx size 0x%llx\n", + segment_pa, segment_size); + return false; + } + + desc = kzalloc_obj(*desc); + if (!desc) { + memunmap(rmp_segment); + return false; + } + + desc->rmp_entry = rmp_segment; + desc->max_index = segment_size / sizeof(*desc->rmp_entry); + desc->size = segment_size; + + rmp_segment_table[rst_index] = desc; + + return true; +} + +static void __init free_rmp_segment_table(void) +{ + unsigned int i; + + for (i = 0; i < rst_max_index; i++) { + struct rmp_segment_desc *desc; + + desc = rmp_segment_table[i]; + if (!desc) + continue; + + memunmap(desc->rmp_entry); + + kfree(desc); + } + + free_page((unsigned long)rmp_segment_table); + + rmp_segment_table = NULL; +} + +/* Allocate the table used to index into the RMP segments */ +static bool __init alloc_rmp_segment_table(void) +{ + struct page *page; + + page = alloc_page(__GFP_ZERO); + if (!page) + return false; + + rmp_segment_table = page_address(page); + + return true; +} + +static bool __init setup_contiguous_rmptable(void) +{ + u64 max_rmp_pfn, calc_rmp_sz, rmptable_segment, rmptable_size, rmp_end; + + if (!probed_rmp_size) + return false; + + rmp_end = probed_rmp_base + probed_rmp_size - 1; + + /* + * Calculate the amount of memory that must be reserved by the BIOS to + * address the whole RAM, including the bookkeeping area. The RMP itself + * must also be covered. + */ + max_rmp_pfn = max_pfn; + if (PFN_UP(rmp_end) > max_pfn) + max_rmp_pfn = PFN_UP(rmp_end); + + calc_rmp_sz = (max_rmp_pfn << 4) + RMPTABLE_CPU_BOOKKEEPING_SZ; + if (calc_rmp_sz > probed_rmp_size) { + pr_err("Memory reserved for the RMP table does not cover full system RAM (expected 0x%llx got 0x%llx)\n", + calc_rmp_sz, probed_rmp_size); + return false; + } + + if (!alloc_rmp_segment_table()) + return false; + + /* Map only the RMP entries */ + rmptable_segment = probed_rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ; + rmptable_size = probed_rmp_size - RMPTABLE_CPU_BOOKKEEPING_SZ; + + if (!alloc_rmp_segment_desc(rmptable_segment, rmptable_size, 0)) { + free_rmp_segment_table(); + return false; + } + + return true; +} + +static bool __init setup_segmented_rmptable(void) +{ + u64 rst_pa, *rst, pa, ram_pa_end, ram_pa_max; + unsigned int i, max_index; + + if (!probed_rmp_base) + return false; + + if (!alloc_rmp_segment_table()) + return false; + + rst_pa = probed_rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ; + rst = memremap(rst_pa, RST_SIZE, MEMREMAP_WB); + if (!rst) { + pr_err("Failed to map RMP segment table addr 0x%llx\n", rst_pa); + goto e_free; + } + + pr_info("Segmented RMP using %lluGB segments\n", rmp_segment_size >> 30); + + ram_pa_max = max_pfn << PAGE_SHIFT; + + max_index = 0; + ram_pa_end = 0; + for (i = 0; i < rst_max_index; i++) { + u64 rmp_segment, rmp_size, mapped_size; + + mapped_size = RST_ENTRY_MAPPED_SIZE(rst[i]); + if (!mapped_size) + continue; + + max_index = i; + + /* + * Mapped size in GB. Mapped size is allowed to exceed the + * segment coverage size, but gets reduced to the segment + * coverage size. + */ + mapped_size <<= 30; + if (mapped_size > rmp_segment_size) { + pr_info("RMP segment %u mapped size (0x%llx) reduced to 0x%llx\n", + i, mapped_size, rmp_segment_size); + mapped_size = rmp_segment_size; + } + + rmp_segment = RST_ENTRY_SEGMENT_BASE(rst[i]); + + /* Calculate the RMP segment size (16 bytes/page mapped) */ + rmp_size = PHYS_PFN(mapped_size) << 4; + + pa = (u64)i << rmp_segment_shift; + + /* + * Some segments may be for MMIO mapped above system RAM. These + * segments are used for Trusted I/O. + */ + if (pa < ram_pa_max) + ram_pa_end = pa + mapped_size; + + if (!alloc_rmp_segment_desc(rmp_segment, rmp_size, pa)) + goto e_unmap; + + pr_info("RMP segment %u physical address [0x%llx - 0x%llx] covering [0x%llx - 0x%llx]\n", + i, rmp_segment, rmp_segment + rmp_size - 1, pa, pa + mapped_size - 1); + } + + if (ram_pa_max > ram_pa_end) { + pr_err("Segmented RMP does not cover full system RAM (expected 0x%llx got 0x%llx)\n", + ram_pa_max, ram_pa_end); + goto e_unmap; + } + + /* Adjust the maximum index based on the found segments */ + rst_max_index = max_index + 1; + + memunmap(rst); + + return true; + +e_unmap: + memunmap(rst); + +e_free: + free_rmp_segment_table(); + + return false; +} + +static bool __init setup_rmptable(void) +{ + if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) { + if (!setup_segmented_rmptable()) + return false; + } else { + if (!setup_contiguous_rmptable()) + return false; + } + + rmp_bookkeeping = memremap(probed_rmp_base, RMPTABLE_CPU_BOOKKEEPING_SZ, MEMREMAP_WB); + if (!rmp_bookkeeping) { + pr_err("Failed to map RMP bookkeeping area\n"); + free_rmp_segment_table(); + + return false; + } + + return true; +} + +static void clear_hsave_pa(void *arg) +{ + wrmsrq(MSR_VM_HSAVE_PA, 0); +} + +int snp_prepare(void) +{ + int ret; + u64 val; + + /* + * Check if SEV-SNP is already enabled, this can happen in case of + * kexec boot. + */ + rdmsrq(MSR_AMD64_SYSCFG, val); + if (val & MSR_AMD64_SYSCFG_SNP_EN) + return 0; + + clear_rmp(); + + cpus_read_lock(); + + if (!cpumask_equal(cpu_online_mask, cpu_present_mask)) { + ret = -EOPNOTSUPP; + pr_warn("SNP init failed: not all CPUs online. (%*pbl online <-> %*pbl present masks).\n", + cpumask_pr_args(cpu_online_mask), + cpumask_pr_args(cpu_present_mask)); + goto unlock; + } + + /* + * MtrrFixDramModEn is not shared between threads on a core, + * therefore it must be set on all CPUs prior to enabling SNP. + */ + on_each_cpu(mfd_reconfigure, (void *)1, 1); + on_each_cpu(snp_enable, NULL, 1); + + /* SNP_INIT requires MSR_VM_HSAVE_PA to be cleared on all CPUs. */ + on_each_cpu(clear_hsave_pa, NULL, 1); + + ret = 0; + +unlock: + cpus_read_unlock(); + + return ret; +} +EXPORT_SYMBOL_FOR_MODULES(snp_prepare, "ccp"); + +void snp_shutdown(void) +{ + u64 syscfg; + + rdmsrq(MSR_AMD64_SYSCFG, syscfg); + if (syscfg & MSR_AMD64_SYSCFG_SNP_EN) + return; + + clear_rmp(); + on_each_cpu(mfd_reconfigure, NULL, 1); +} +EXPORT_SYMBOL_FOR_MODULES(snp_shutdown, "ccp"); + +/* + * Do the necessary preparations which are verified by the firmware as + * described in the SNP_INIT_EX firmware command description in the SNP + * firmware ABI spec. + */ +int __init snp_rmptable_init(void) +{ + if (WARN_ON_ONCE(!cc_platform_has(CC_ATTR_HOST_SEV_SNP))) + return -ENOSYS; + + if (WARN_ON_ONCE(!amd_iommu_snp_en)) + return -ENOSYS; + + if (!setup_rmptable()) + return -ENOSYS; + + /* + * Setting crash_kexec_post_notifiers to 'true' to ensure that SNP panic + * notifier is invoked to do SNP IOMMU shutdown before kdump. + */ + crash_kexec_post_notifiers = true; + + return 0; +} + +static void set_rmp_segment_info(unsigned int segment_shift) +{ + rmp_segment_shift = segment_shift; + rmp_segment_size = 1ULL << rmp_segment_shift; + rmp_segment_mask = rmp_segment_size - 1; +} + +#define RMP_ADDR_MASK GENMASK_ULL(51, 13) + +static bool probe_contiguous_rmptable_info(void) +{ + u64 rmp_sz, rmp_base, rmp_end; + + rdmsrq(MSR_AMD64_RMP_BASE, rmp_base); + rdmsrq(MSR_AMD64_RMP_END, rmp_end); + + if (!(rmp_base & RMP_ADDR_MASK) || !(rmp_end & RMP_ADDR_MASK)) { + pr_err("Memory for the RMP table has not been reserved by BIOS\n"); + return false; + } + + if (rmp_base > rmp_end) { + pr_err("RMP configuration not valid: base=%#llx, end=%#llx\n", rmp_base, rmp_end); + return false; + } + + rmp_sz = rmp_end - rmp_base + 1; + + /* Treat the contiguous RMP table as a single segment */ + rst_max_index = 1; + + set_rmp_segment_info(RMPTABLE_NON_SEGMENTED_SHIFT); + + probed_rmp_base = rmp_base; + probed_rmp_size = rmp_sz; + + pr_info("RMP table physical range [0x%016llx - 0x%016llx]\n", + rmp_base, rmp_end); + + return true; +} + +static bool probe_segmented_rmptable_info(void) +{ + unsigned int eax, ebx, segment_shift, segment_shift_min, segment_shift_max; + u64 rmp_base, rmp_end; + + rdmsrq(MSR_AMD64_RMP_BASE, rmp_base); + if (!(rmp_base & RMP_ADDR_MASK)) { + pr_err("Memory for the RMP table has not been reserved by BIOS\n"); + return false; + } + + rdmsrq(MSR_AMD64_RMP_END, rmp_end); + WARN_ONCE(rmp_end & RMP_ADDR_MASK, + "Segmented RMP enabled but RMP_END MSR is non-zero\n"); + + /* Obtain the min and max supported RMP segment size */ + eax = cpuid_eax(0x80000025); + segment_shift_min = eax & GENMASK(5, 0); + segment_shift_max = (eax & GENMASK(11, 6)) >> 6; + + /* Verify the segment size is within the supported limits */ + segment_shift = MSR_AMD64_RMP_SEGMENT_SHIFT(rmp_cfg); + if (segment_shift > segment_shift_max || segment_shift < segment_shift_min) { + pr_err("RMP segment size (%u) is not within advertised bounds (min=%u, max=%u)\n", + segment_shift, segment_shift_min, segment_shift_max); + return false; + } + + /* Override the max supported RST index if a hardware limit exists */ + ebx = cpuid_ebx(0x80000025); + if (ebx & BIT(10)) + rst_max_index = ebx & GENMASK(9, 0); + + set_rmp_segment_info(segment_shift); + + probed_rmp_base = rmp_base; + probed_rmp_size = 0; + + pr_info("Segmented RMP base table physical range [0x%016llx - 0x%016llx]\n", + rmp_base, rmp_base + RMPTABLE_CPU_BOOKKEEPING_SZ + RST_SIZE); + + return true; +} + +bool snp_probe_rmptable_info(void) +{ + if (cpu_feature_enabled(X86_FEATURE_SEGMENTED_RMP)) + rdmsrq(MSR_AMD64_RMP_CFG, rmp_cfg); + + if (rmp_cfg & MSR_AMD64_SEG_RMP_ENABLED) + return probe_segmented_rmptable_info(); + else + return probe_contiguous_rmptable_info(); +} + +/* + * About the array_index_nospec() usage below: + * + * This function can get called by exported functions like + * snp_lookup_rmpentry(), which is used by the KVM #PF handler, among + * others, and since the @pfn passed in cannot always be trusted, + * speculation should be stopped as a protective measure. + */ +static struct rmpentry_raw *get_raw_rmpentry(u64 pfn) +{ + u64 paddr, rst_index, segment_index; + struct rmp_segment_desc *desc; + + if (!rmp_segment_table) + return ERR_PTR(-ENODEV); + + paddr = pfn << PAGE_SHIFT; + + rst_index = RST_ENTRY_INDEX(paddr); + if (unlikely(rst_index >= rst_max_index)) + return ERR_PTR(-EFAULT); + + rst_index = array_index_nospec(rst_index, rst_max_index); + + desc = rmp_segment_table[rst_index]; + if (unlikely(!desc)) + return ERR_PTR(-EFAULT); + + segment_index = RMP_ENTRY_INDEX(paddr); + if (unlikely(segment_index >= desc->max_index)) + return ERR_PTR(-EFAULT); + + segment_index = array_index_nospec(segment_index, desc->max_index); + + return desc->rmp_entry + segment_index; +} + +static int get_rmpentry(u64 pfn, struct rmpentry *e) +{ + struct rmpentry_raw *e_raw; + + if (cpu_feature_enabled(X86_FEATURE_RMPREAD)) { + int ret; + + /* Binutils version 2.44 supports the RMPREAD mnemonic. */ + asm volatile(".byte 0xf2, 0x0f, 0x01, 0xfd" + : "=a" (ret) + : "a" (pfn << PAGE_SHIFT), "c" (e) + : "memory", "cc"); + + return ret; + } + + e_raw = get_raw_rmpentry(pfn); + if (IS_ERR(e_raw)) + return PTR_ERR(e_raw); + + /* + * Map the raw RMP table entry onto the RMPREAD output format. + * The 2MB region status indicator (hpage_region_status field) is not + * calculated, since the overhead could be significant and the field + * is not used. + */ + memset(e, 0, sizeof(*e)); + e->gpa = e_raw->gpa << PAGE_SHIFT; + e->asid = e_raw->asid; + e->assigned = e_raw->assigned; + e->pagesize = e_raw->pagesize; + e->immutable = e_raw->immutable; + + return 0; +} + +static int __snp_lookup_rmpentry(u64 pfn, struct rmpentry *e, int *level) +{ + struct rmpentry e_large; + int ret; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return -ENODEV; + + ret = get_rmpentry(pfn, e); + if (ret) + return ret; + + /* + * Find the authoritative RMP entry for a PFN. This can be either a 4K + * RMP entry or a special large RMP entry that is authoritative for a + * whole 2M area. + */ + ret = get_rmpentry(pfn & PFN_PMD_MASK, &e_large); + if (ret) + return ret; + + *level = RMP_TO_PG_LEVEL(e_large.pagesize); + + return 0; +} + +int snp_lookup_rmpentry(u64 pfn, bool *assigned, int *level) +{ + struct rmpentry e; + int ret; + + ret = __snp_lookup_rmpentry(pfn, &e, level); + if (ret) + return ret; + + *assigned = !!e.assigned; + return 0; +} +EXPORT_SYMBOL_GPL(snp_lookup_rmpentry); + +/* + * Dump the raw RMP entry for a particular PFN. These bits are documented in the + * PPR for a particular CPU model and provide useful information about how a + * particular PFN is being utilized by the kernel/firmware at the time certain + * unexpected events occur, such as RMP faults. + */ +static void dump_rmpentry(u64 pfn) +{ + struct rmpentry_raw *e_raw; + u64 pfn_i, pfn_end; + struct rmpentry e; + int level, ret; + + ret = __snp_lookup_rmpentry(pfn, &e, &level); + if (ret) { + pr_err("Failed to read RMP entry for PFN 0x%llx, error %d\n", + pfn, ret); + return; + } + + if (e.assigned) { + e_raw = get_raw_rmpentry(pfn); + if (IS_ERR(e_raw)) { + pr_err("Failed to read RMP contents for PFN 0x%llx, error %ld\n", + pfn, PTR_ERR(e_raw)); + return; + } + + pr_info("PFN 0x%llx, RMP entry: [0x%016llx - 0x%016llx]\n", + pfn, e_raw->lo, e_raw->hi); + return; + } + + /* + * If the RMP entry for a particular PFN is not in an assigned state, + * then it is sometimes useful to get an idea of whether or not any RMP + * entries for other PFNs within the same 2MB region are assigned, since + * those too can affect the ability to access a particular PFN in + * certain situations, such as when the PFN is being accessed via a 2MB + * mapping in the host page table. + */ + pfn_i = ALIGN_DOWN(pfn, PTRS_PER_PMD); + pfn_end = pfn_i + PTRS_PER_PMD; + + pr_info("PFN 0x%llx unassigned, dumping non-zero entries in 2M PFN region: [0x%llx - 0x%llx]\n", + pfn, pfn_i, pfn_end); + + while (pfn_i < pfn_end) { + e_raw = get_raw_rmpentry(pfn_i); + if (IS_ERR(e_raw)) { + pr_err("Error %ld reading RMP contents for PFN 0x%llx\n", + PTR_ERR(e_raw), pfn_i); + pfn_i++; + continue; + } + + if (e_raw->lo || e_raw->hi) + pr_info("PFN: 0x%llx, [0x%016llx - 0x%016llx]\n", pfn_i, e_raw->lo, e_raw->hi); + pfn_i++; + } +} + +void snp_dump_hva_rmpentry(unsigned long hva) +{ + unsigned long paddr; + unsigned int level; + pgd_t *pgd; + pte_t *pte; + + pgd = __va(read_cr3_pa()); + pgd += pgd_index(hva); + pte = lookup_address_in_pgd(pgd, hva, &level); + + if (!pte) { + pr_err("Can't dump RMP entry for HVA %lx: no PTE/PFN found\n", hva); + return; + } + + paddr = PFN_PHYS(pte_pfn(*pte)) | (hva & ~page_level_mask(level)); + dump_rmpentry(PHYS_PFN(paddr)); +} + +/* + * PSMASH a 2MB aligned page into 4K pages in the RMP table while preserving the + * Validated bit. + */ +int psmash(u64 pfn) +{ + unsigned long paddr = pfn << PAGE_SHIFT; + int ret; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return -ENODEV; + + if (!pfn_valid(pfn)) + return -EINVAL; + + /* Binutils version 2.36 supports the PSMASH mnemonic. */ + asm volatile(".byte 0xF3, 0x0F, 0x01, 0xFF" + : "=a" (ret) + : "a" (paddr) + : "memory", "cc"); + + return ret; +} +EXPORT_SYMBOL_GPL(psmash); + +/* + * If the kernel uses a 2MB or larger directmap mapping to write to an address, + * and that mapping contains any 4KB pages that are set to private in the RMP + * table, an RMP #PF will trigger and cause a host crash. Hypervisor code that + * owns the PFNs being transitioned will never attempt such a write, but other + * kernel tasks writing to other PFNs in the range may trigger these checks + * inadvertently due a large directmap mapping that happens to overlap such a + * PFN. + * + * Prevent this by splitting any 2MB+ mappings that might end up containing a + * mix of private/shared PFNs as a result of a subsequent RMPUPDATE for the + * PFN/rmp_level passed in. + * + * Note that there is no attempt here to scan all the RMP entries for the 2MB + * physical range, since it would only be worthwhile in determining if a + * subsequent RMPUPDATE for a 4KB PFN would result in all the entries being of + * the same shared/private state, thus avoiding the need to split the mapping. + * But that would mean the entries are currently in a mixed state, and so the + * mapping would have already been split as a result of prior transitions. + * And since the 4K split is only done if the mapping is 2MB+, and there isn't + * currently a mechanism in place to restore 2MB+ mappings, such a check would + * not provide any usable benefit. + * + * More specifics on how these checks are carried out can be found in APM + * Volume 2, "RMP and VMPL Access Checks". + */ +static int adjust_direct_map(u64 pfn, int rmp_level) +{ + unsigned long vaddr; + unsigned int level; + int npages, ret; + pte_t *pte; + + /* + * pfn_to_kaddr() will return a vaddr only within the direct + * map range. + */ + vaddr = (unsigned long)pfn_to_kaddr(pfn); + + /* Only 4KB/2MB RMP entries are supported by current hardware. */ + if (WARN_ON_ONCE(rmp_level > PG_LEVEL_2M)) + return -EINVAL; + + if (!pfn_valid(pfn)) + return -EINVAL; + + if (rmp_level == PG_LEVEL_2M && + (!IS_ALIGNED(pfn, PTRS_PER_PMD) || !pfn_valid(pfn + PTRS_PER_PMD - 1))) + return -EINVAL; + + /* + * If an entire 2MB physical range is being transitioned, then there is + * no risk of RMP #PFs due to write accesses from overlapping mappings, + * since even accesses from 1GB mappings will be treated as 2MB accesses + * as far as RMP table checks are concerned. + */ + if (rmp_level == PG_LEVEL_2M) + return 0; + + pte = lookup_address(vaddr, &level); + if (!pte || pte_none(*pte)) + return 0; + + if (level == PG_LEVEL_4K) + return 0; + + npages = page_level_size(rmp_level) / PAGE_SIZE; + ret = set_memory_4k(vaddr, npages); + if (ret) + pr_warn("Failed to split direct map for PFN 0x%llx, ret: %d\n", + pfn, ret); + + return ret; +} + +/* + * It is expected that those operations are seldom enough so that no mutual + * exclusion of updaters is needed and thus the overlap error condition below + * should happen very rarely and would get resolved relatively quickly by + * the firmware. + * + * If not, one could consider introducing a mutex or so here to sync concurrent + * RMP updates and thus diminish the amount of cases where firmware needs to + * lock 2M ranges to protect against concurrent updates. + * + * The optimal solution would be range locking to avoid locking disjoint + * regions unnecessarily but there's no support for that yet. + */ +static int rmpupdate(u64 pfn, struct rmp_state *state) +{ + unsigned long paddr = pfn << PAGE_SHIFT; + int ret, level; + + if (!cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + return -ENODEV; + + level = RMP_TO_PG_LEVEL(state->pagesize); + + if (adjust_direct_map(pfn, level)) + return -EFAULT; + + do { + /* Binutils version 2.36 supports the RMPUPDATE mnemonic. */ + asm volatile(".byte 0xF2, 0x0F, 0x01, 0xFE" + : "=a" (ret) + : "a" (paddr), "c" ((unsigned long)state) + : "memory", "cc"); + } while (ret == RMPUPDATE_FAIL_OVERLAP); + + if (ret) { + pr_err("RMPUPDATE failed for PFN %llx, pg_level: %d, ret: %d\n", + pfn, level, ret); + dump_rmpentry(pfn); + dump_stack(); + return -EFAULT; + } + + return 0; +} + +/* Transition a page to guest-owned/private state in the RMP table. */ +int rmp_make_private(u64 pfn, u64 gpa, enum pg_level level, u32 asid, bool immutable) +{ + struct rmp_state state; + + memset(&state, 0, sizeof(state)); + state.assigned = 1; + state.asid = asid; + state.immutable = immutable; + state.gpa = gpa; + state.pagesize = PG_LEVEL_TO_RMP(level); + + return rmpupdate(pfn, &state); +} +EXPORT_SYMBOL_GPL(rmp_make_private); + +/* Transition a page to hypervisor-owned/shared state in the RMP table. */ +int rmp_make_shared(u64 pfn, enum pg_level level) +{ + struct rmp_state state; + + memset(&state, 0, sizeof(state)); + state.pagesize = PG_LEVEL_TO_RMP(level); + + return rmpupdate(pfn, &state); +} +EXPORT_SYMBOL_GPL(rmp_make_shared); + +void __snp_leak_pages(u64 pfn, unsigned int npages, bool dump_rmp) +{ + struct page *page = pfn_to_page(pfn); + + pr_warn("Leaking PFN range 0x%llx-0x%llx\n", pfn, pfn + npages); + + spin_lock(&snp_leaked_pages_list_lock); + while (npages--) { + + /* + * Reuse the page's buddy list for chaining into the leaked + * pages list. This page should not be on a free list currently + * and is also unsafe to be added to a free list. + */ + if (likely(!PageCompound(page)) || + + /* + * Skip inserting tail pages of compound page as + * page->buddy_list of tail pages is not usable. + */ + (PageHead(page) && compound_nr(page) <= npages)) + list_add_tail(&page->buddy_list, &snp_leaked_pages_list); + + if (dump_rmp) + dump_rmpentry(pfn); + snp_nr_leaked_pages++; + pfn++; + page++; + } + spin_unlock(&snp_leaked_pages_list_lock); +} +EXPORT_SYMBOL_GPL(__snp_leak_pages); + +void kdump_sev_callback(void) +{ + /* + * Do wbinvd() on remote CPUs when SNP is enabled in order to + * safely do SNP_SHUTDOWN on the local CPU. + */ + if (cc_platform_has(CC_ATTR_HOST_SEV_SNP)) + wbinvd(); +} diff --git a/arch/x86/virt/vmx/Makefile b/arch/x86/virt/vmx/Makefile new file mode 100644 index 000000000..feebda21d --- /dev/null +++ b/arch/x86/virt/vmx/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_INTEL_TDX_HOST) += tdx/ diff --git a/arch/x86/virt/vmx/tdx/Makefile b/arch/x86/virt/vmx/tdx/Makefile new file mode 100644 index 000000000..d1dbc5cc5 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/Makefile @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-y += seamcall.o seamldr.o tdx.o diff --git a/arch/x86/virt/vmx/tdx/seamcall.S b/arch/x86/virt/vmx/tdx/seamcall.S new file mode 100644 index 000000000..6854c52c3 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/seamcall.S @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <linux/linkage.h> +#include <asm/frame.h> + +#include "tdxcall.S" + +/* + * __seamcall() - Host-side interface functions to SEAM software + * (the P-SEAMLDR or the TDX module). + * + * __seamcall() function ABI: + * + * @fn (RDI) - SEAMCALL Leaf number, moved to RAX + * @args (RSI) - struct tdx_module_args for input + * + * Only RCX/RDX/R8-R11 are used as input registers. + * + * Return (via RAX) TDX_SEAMCALL_VMFAILINVALID if the SEAMCALL itself + * fails, or the completion status of the SEAMCALL leaf function. + */ +SYM_FUNC_START(__seamcall) + TDX_MODULE_CALL host=1 +SYM_FUNC_END(__seamcall) + +/* + * __seamcall_ret() - Host-side interface functions to SEAM software + * (the P-SEAMLDR or the TDX module), with saving output registers to + * the 'struct tdx_module_args' used as input. + * + * __seamcall_ret() function ABI: + * + * @fn (RDI) - SEAMCALL Leaf number, moved to RAX + * @args (RSI) - struct tdx_module_args for input and output + * + * Only RCX/RDX/R8-R11 are used as input/output registers. + * + * Return (via RAX) TDX_SEAMCALL_VMFAILINVALID if the SEAMCALL itself + * fails, or the completion status of the SEAMCALL leaf function. + */ +SYM_FUNC_START(__seamcall_ret) + TDX_MODULE_CALL host=1 ret=1 +SYM_FUNC_END(__seamcall_ret) + +/* KVM requires non-instrumentable __seamcall_saved_ret() for TDH.VP.ENTER */ +.section .noinstr.text, "ax" + +/* + * __seamcall_saved_ret() - Host-side interface functions to SEAM software + * (the P-SEAMLDR or the TDX module), with saving output registers to the + * 'struct tdx_module_args' used as input. + * + * __seamcall_saved_ret() function ABI: + * + * @fn (RDI) - SEAMCALL Leaf number, moved to RAX + * @args (RSI) - struct tdx_module_args for input and output + * + * All registers in @args are used as input/output registers. + * + * Return (via RAX) TDX_SEAMCALL_VMFAILINVALID if the SEAMCALL itself + * fails, or the completion status of the SEAMCALL leaf function. + */ +SYM_FUNC_START(__seamcall_saved_ret) + TDX_MODULE_CALL host=1 ret=1 saved=1 +SYM_FUNC_END(__seamcall_saved_ret) diff --git a/arch/x86/virt/vmx/tdx/seamcall_internal.h b/arch/x86/virt/vmx/tdx/seamcall_internal.h new file mode 100644 index 000000000..be5f44646 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/seamcall_internal.h @@ -0,0 +1,109 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * SEAMCALL utilities for TDX host-side operations. + * + * Provides convenient wrappers around SEAMCALL assembly with retry logic, + * error reporting and cache coherency tracking. + * + * Copyright (C) 2021-2023 Intel Corporation + */ + +#ifndef _X86_VIRT_SEAMCALL_INTERNAL_H +#define _X86_VIRT_SEAMCALL_INTERNAL_H + +#include <linux/printk.h> +#include <linux/types.h> +#include <asm/archrandom.h> +#include <asm/processor.h> +#include <asm/tdx.h> + +u64 __seamcall(u64 fn, struct tdx_module_args *args); +u64 __seamcall_ret(u64 fn, struct tdx_module_args *args); +u64 __seamcall_saved_ret(u64 fn, struct tdx_module_args *args); + +typedef u64 (*sc_func_t)(u64 fn, struct tdx_module_args *args); + +static __always_inline u64 __seamcall_dirty_cache(sc_func_t func, u64 fn, + struct tdx_module_args *args) +{ + lockdep_assert_preemption_disabled(); + + /* + * SEAMCALLs are made to the TDX module and can generate dirty + * cachelines of TDX private memory. Mark cache state incoherent + * so that the cache can be flushed during kexec. + * + * This needs to be done before actually making the SEAMCALL, + * because kexec-ing CPU could send NMI to stop remote CPUs, + * in which case even disabling IRQ won't help here. + */ + this_cpu_write(cache_state_incoherent, true); + + return func(fn, args); +} + +static __always_inline u64 sc_retry(sc_func_t func, u64 fn, + struct tdx_module_args *args) +{ + int retry = RDRAND_RETRY_LOOPS; + u64 ret; + + do { + preempt_disable(); + ret = __seamcall_dirty_cache(func, fn, args); + preempt_enable(); + } while (ret == TDX_RND_NO_ENTROPY && --retry); + + return ret; +} + +#define seamcall(_fn, _args) sc_retry(__seamcall, (_fn), (_args)) +#define seamcall_ret(_fn, _args) sc_retry(__seamcall_ret, (_fn), (_args)) +#define seamcall_saved_ret(_fn, _args) sc_retry(__seamcall_saved_ret, (_fn), (_args)) + +typedef void (*sc_err_func_t)(u64 fn, u64 err, struct tdx_module_args *args); + +static inline void seamcall_err(u64 fn, u64 err, struct tdx_module_args *args) +{ + pr_err("SEAMCALL (0x%016llx) failed: 0x%016llx\n", fn, err); +} + +static inline void seamcall_err_ret(u64 fn, u64 err, + struct tdx_module_args *args) +{ + seamcall_err(fn, err, args); + pr_err("RCX 0x%016llx RDX 0x%016llx R08 0x%016llx\n", + args->rcx, args->rdx, args->r8); + pr_err("R09 0x%016llx R10 0x%016llx R11 0x%016llx\n", + args->r9, args->r10, args->r11); +} + +static __always_inline int sc_retry_prerr(sc_func_t func, + sc_err_func_t err_func, + u64 fn, struct tdx_module_args *args) +{ + u64 sret = sc_retry(func, fn, args); + + if (sret == TDX_SUCCESS) + return 0; + + if (sret == TDX_SEAMCALL_VMFAILINVALID) + return -ENODEV; + + if (sret == TDX_SEAMCALL_GP) + return -EOPNOTSUPP; + + if (sret == TDX_SEAMCALL_UD) + return -EACCES; + + err_func(fn, sret, args); + return -EIO; +} + +#define seamcall_prerr(__fn, __args) \ + sc_retry_prerr(__seamcall, seamcall_err, (__fn), (__args)) + +#define seamcall_prerr_ret(__fn, __args) \ + sc_retry_prerr(__seamcall_ret, seamcall_err_ret, (__fn), (__args)) + +#endif /* _X86_VIRT_SEAMCALL_INTERNAL_H */ diff --git a/arch/x86/virt/vmx/tdx/seamldr.c b/arch/x86/virt/vmx/tdx/seamldr.c new file mode 100644 index 000000000..b1137ca61 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/seamldr.c @@ -0,0 +1,368 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * P-SEAMLDR support for TDX module management features like runtime updates + * + * Copyright (C) 2025 Intel Corporation + */ +#define pr_fmt(fmt) "seamldr: " fmt + +#include <linux/bug.h> +#include <linux/mm.h> +#include <linux/nmi.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/stop_machine.h> + +#include <asm/cpufeature.h> +#include <asm/cpufeatures.h> +#include <asm/seamldr.h> + +#include "seamcall_internal.h" +#include "tdx.h" + +/* P-SEAMLDR SEAMCALL leaf function */ +#define P_SEAMLDR_INFO 0x8000000000000000 +#define P_SEAMLDR_INSTALL 0x8000000000000001 + +#define SEAMLDR_MAX_NR_MODULE_PAGES 496 +#define SEAMLDR_MAX_NR_SIG_PAGES 1 + +/* + * The seamldr_params "scenario" field specifies the operation mode: + * 0: Install TDX module from scratch (not used by kernel) + * 1: Update existing TDX module to a compatible version + */ +#define SEAMLDR_SCENARIO_UPDATE 1 + +/* + * This is the "SEAMLDR_PARAMS" data structure defined in the + * "SEAM Loader (SEAMLDR) Interface Specification". + * + * It is the in-memory ABI that the kernel passes to the P-SEAMLDR + * to update the TDX module. It breaks the TDX module image up in + * page-size pieces. + */ +struct seamldr_params { + u32 version; + u32 scenario; + u64 sigstruct_pages_pa_list[SEAMLDR_MAX_NR_SIG_PAGES]; + u8 reserved[104]; + u64 module_nr_pages; + u64 module_pages_pa_list[SEAMLDR_MAX_NR_MODULE_PAGES]; +} __packed; + +static_assert(sizeof(struct seamldr_params) == 4096); + +/* + * Serialize P-SEAMLDR calls since the hardware only allows a single CPU to + * interact with P-SEAMLDR simultaneously. Use raw version as the calls can + * be made with interrupts disabled, where plain spinlocks are prohibited in + * PREEMPT_RT kernels as they become sleeping locks. + */ +static DEFINE_RAW_SPINLOCK(seamldr_lock); + +static int seamldr_call(u64 fn, struct tdx_module_args *args) +{ + /* + * With this bug, P-SEAMLDR calls corrupt the VMCS + * pointer and must be avoided. This path should be + * unreachable since sysfs hides the ABIs. + */ + if (boot_cpu_has_bug(X86_BUG_SEAMRET_INVD_VMCS)) { + WARN_ON(1); + return -EINVAL; + } + + guard(raw_spinlock)(&seamldr_lock); + return seamcall_prerr(fn, args); +} + +int seamldr_get_info(struct seamldr_info *seamldr_info) +{ + struct tdx_module_args args = {}; + + /* + * Use slow_virt_to_phys() since @seamldr_info may be allocated on + * the stack. + */ + args.rcx = slow_virt_to_phys(seamldr_info); + return seamldr_call(P_SEAMLDR_INFO, &args); +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_get_info, "tdx-host"); + +/* Call into P-SEAMLDR to install a TDX module update */ +static int seamldr_install(const struct seamldr_params *params) +{ + struct tdx_module_args args = {}; + + args.rcx = __pa(params); + return seamldr_call(P_SEAMLDR_INSTALL, &args); +} + +#define TDX_IMAGE_VERSION_2 0x200 + +/* First page of the on-disk module update image: */ +struct tdx_image_header { + u16 version; + u16 checksum; + u8 signature[8]; + u32 sigstruct_nr_pages; + u32 module_nr_pages; + u8 reserved[4076]; +} __packed; + +#define TDX_IMAGE_HEADER_SIZE sizeof(struct tdx_image_header) +static_assert(TDX_IMAGE_HEADER_SIZE == 4096); + +/* + * Intel TDX module update ABI structure. aka. "TDX module blob". + * This is the on-disk format that fw_upload lands in a kernel + * buffer. + * + * @payload contains sigstruct pages followed by module pages. + */ +struct tdx_image { + struct tdx_image_header header; + u8 payload[]; +}; + +/* + * Given a vmalloc() allocation, write all of the backing physical + * addresses to pa_list[]. Caller guarantees that the array is big + * enough. + */ +static void populate_pa_list(u64 *pa_list, const u8 *vmalloc_addr, u32 vmalloc_len_pages) +{ + int i; + + for (i = 0; i < vmalloc_len_pages; i++) { + unsigned long offset = i * PAGE_SIZE; + unsigned long pfn = vmalloc_to_pfn(&vmalloc_addr[offset]); + + pa_list[i] = pfn << PAGE_SHIFT; + } +} + +static void populate_seamldr_params(struct seamldr_params *params, + const u8 *sig, u32 sig_nr_pages, + const u8 *mod, u32 mod_nr_pages) +{ + params->version = 0; + params->scenario = SEAMLDR_SCENARIO_UPDATE; + params->module_nr_pages = mod_nr_pages; + + populate_pa_list(params->sigstruct_pages_pa_list, sig, sig_nr_pages); + populate_pa_list(params->module_pages_pa_list, mod, mod_nr_pages); +} + +/* + * @image points to a vmalloc()'d 'struct tdx_image'. Transform + * it into @params which is the P-SEAMLDR ABI format. + */ +static int init_seamldr_params(struct seamldr_params *params, + const struct tdx_image *image, + u32 image_len) +{ + const struct tdx_image_header *header = &image->header; + + u32 sigstruct_len = header->sigstruct_nr_pages * PAGE_SIZE; + u32 module_len = header->module_nr_pages * PAGE_SIZE; + + u8 *header_start = (u8 *)header; + u8 *header_end = header_start + TDX_IMAGE_HEADER_SIZE; + + u8 *sigstruct_start = header_end; + u8 *sigstruct_end = sigstruct_start + sigstruct_len; + + u8 *module_start = sigstruct_end; + + /* Check the calculated payload size against the image size. */ + if (TDX_IMAGE_HEADER_SIZE + sigstruct_len + module_len != image_len) + return -EINVAL; + + /* Reject unsupported tdx_image ABI versions. */ + if (header->version != TDX_IMAGE_VERSION_2) + return -EINVAL; + + if (header->sigstruct_nr_pages > SEAMLDR_MAX_NR_SIG_PAGES || + header->module_nr_pages > SEAMLDR_MAX_NR_MODULE_PAGES) + return -EINVAL; + + if (memcmp(header->signature, "TDX-BLOB", sizeof(header->signature))) + return -EINVAL; + + if (memchr_inv(header->reserved, 0, sizeof(header->reserved))) + return -EINVAL; + + populate_seamldr_params(params, sigstruct_start, header->sigstruct_nr_pages, + module_start, header->module_nr_pages); + return 0; +} + +/* + * During a TDX module update, all CPUs start from MODULE_UPDATE_START and + * progress to MODULE_UPDATE_DONE. Each state is associated with certain + * work. For some states, just one CPU needs to perform the work, while + * other CPUs just wait during those states. + */ +enum module_update_state { + MODULE_UPDATE_START, + MODULE_UPDATE_SHUTDOWN, + MODULE_UPDATE_CPU_INSTALL, + MODULE_UPDATE_CPU_INIT, + MODULE_UPDATE_RUN_UPDATE, + MODULE_UPDATE_DONE, +}; + +static struct update_ctrl { + enum module_update_state state; + int num_ack; + int num_failed; + /* + * Protect update_ctrl. Raw spinlock as it will be acquired from + * interrupt-disabled contexts. + */ + raw_spinlock_t lock; +} update_ctrl; + +/* Called with ctrl->lock held or during initialization. */ +static void __set_target_state(struct update_ctrl *ctrl, + enum module_update_state newstate) +{ + /* Reset ack counter. */ + ctrl->num_ack = 0; + ctrl->state = newstate; +} + +/* Last one to ack a state moves to the next state. */ +static void ack_state(struct update_ctrl *ctrl, int result) +{ + raw_spin_lock(&ctrl->lock); + + ctrl->num_failed += !!result; + ctrl->num_ack++; + if (ctrl->num_ack == num_online_cpus() && !ctrl->num_failed) + __set_target_state(ctrl, ctrl->state + 1); + + raw_spin_unlock(&ctrl->lock); +} + +static void init_state(struct update_ctrl *ctrl) +{ + raw_spin_lock_init(&ctrl->lock); + __set_target_state(ctrl, MODULE_UPDATE_START + 1); + ctrl->num_failed = 0; +} + +/* + * See multi_cpu_stop() from where this multi-cpu state-machine was + * adopted. + */ +static int do_seamldr_install_module(void *seamldr_params) +{ + enum module_update_state curstate = MODULE_UPDATE_START; + enum module_update_state newstate; + bool is_lead_cpu = false; + int ret = 0; + + /* + * Some steps must be run on exactly one CPU. Pick a "lead" CPU to + * execute those steps. Use CPU 0 because it is always online. + */ + if (smp_processor_id() == 0) + is_lead_cpu = true; + + do { + newstate = READ_ONCE(update_ctrl.state); + + if (curstate == newstate) { + cpu_relax(); + continue; + } + + curstate = newstate; + switch (curstate) { + case MODULE_UPDATE_SHUTDOWN: + if (is_lead_cpu) + ret = tdx_module_shutdown(); + break; + case MODULE_UPDATE_CPU_INSTALL: + ret = seamldr_install(seamldr_params); + break; + case MODULE_UPDATE_CPU_INIT: + ret = tdx_cpu_enable(); + break; + case MODULE_UPDATE_RUN_UPDATE: + if (is_lead_cpu) + ret = tdx_module_run_update(); + break; + default: + break; + } + + ack_state(&update_ctrl, ret); + } while (curstate != MODULE_UPDATE_DONE && + !READ_ONCE(update_ctrl.num_failed)); + + return ret; +} + +/** + * seamldr_install_module - Install a new TDX module. + * @data: Pointer to the TDX module image. + * @data_len: Size of the TDX module image. + * + * Returns 0 on success, negative error code on failure. + */ +int seamldr_install_module(const u8 *data, u32 data_len) +{ + struct seamldr_params *params; + const struct tdx_image *image; + int ret; + + /* + * init_seamldr_params() reads the header early. + * Ensure there is enough data to do at least that. + */ + if (data_len < TDX_IMAGE_HEADER_SIZE) + return -EINVAL; + + image = (const struct tdx_image *)data; + + params = kzalloc_obj(*params); + if (!params) + return -ENOMEM; + + /* Populate 'params' from 'image'. */ + ret = init_seamldr_params(params, image, data_len); + if (ret) + goto out; + + /* Ensure a stable set of online CPUs for the update process. */ + cpus_read_lock(); + init_state(&update_ctrl); + ret = stop_machine_cpuslocked(do_seamldr_install_module, params, + cpu_online_mask); + cpus_read_unlock(); + +out: + kfree(params); + return ret; +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_install_module, "tdx-host"); + +/* + * stop_machine() does not interrupt preemption-disabled regions. + * Simply disabling preempt prevents updates. + */ +void seamldr_lock_module_update(void) +{ + preempt_disable(); +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_lock_module_update, "tdx-host"); + +void seamldr_unlock_module_update(void) +{ + preempt_enable(); +} +EXPORT_SYMBOL_FOR_MODULES(seamldr_unlock_module_update, "tdx-host"); diff --git a/arch/x86/virt/vmx/tdx/tdx.c b/arch/x86/virt/vmx/tdx/tdx.c new file mode 100644 index 000000000..42df8ea46 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/tdx.c @@ -0,0 +1,2034 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright(c) 2023 Intel Corporation. + * + * Intel Trusted Domain Extensions (TDX) support + */ + +#include "asm/page_types.h" +#define pr_fmt(fmt) "virt/tdx: " fmt + +#include <linux/types.h> +#include <linux/cache.h> +#include <linux/init.h> +#include <linux/errno.h> +#include <linux/printk.h> +#include <linux/cpu.h> +#include <linux/spinlock.h> +#include <linux/percpu-defs.h> +#include <linux/mutex.h> +#include <linux/list.h> +#include <linux/memblock.h> +#include <linux/memory.h> +#include <linux/minmax.h> +#include <linux/sizes.h> +#include <linux/pfn.h> +#include <linux/align.h> +#include <linux/sort.h> +#include <linux/log2.h> +#include <linux/acpi.h> +#include <linux/suspend.h> +#include <linux/syscore_ops.h> +#include <linux/idr.h> +#include <asm/page.h> +#include <asm/special_insns.h> +#include <asm/msr-index.h> +#include <asm/msr.h> +#include <asm/cpufeature.h> +#include <asm/tdx.h> +#include <asm/shared/tdx_errno.h> +#include <asm/cpu_device_id.h> +#include <asm/processor.h> +#include <asm/mce.h> +#include <asm/virt.h> +#include <asm/vmx.h> + +#include "seamcall_internal.h" +#include "tdx.h" + +struct tdx_module_state { + bool initialized; + bool sysinit_done; + int sysinit_ret; +}; + +static struct tdx_module_state tdx_module_state; +static u32 tdx_global_keyid __ro_after_init; +static u32 tdx_guest_keyid_start __ro_after_init; +static u32 tdx_nr_guest_keyids __ro_after_init; + +static DEFINE_IDA(tdx_guest_keyid_pool); + +static DEFINE_PER_CPU(bool, tdx_lp_initialized); + +static struct tdmr_info_list tdx_tdmr_list; + +/* All TDX-usable memory regions. Protected by mem_hotplug_lock. */ +static LIST_HEAD(tdx_memlist); + +static struct tdx_sys_info tdx_sysinfo; + +static DEFINE_RAW_SPINLOCK(sysinit_lock); + +/* + * Do the module global initialization once and return its result. + * It can be done on any cpu, and from task or IRQ context. + */ +static int try_init_module_global(void) +{ + struct tdx_module_args args = {}; + int ret; + + raw_spin_lock(&sysinit_lock); + + /* Return the "cached" return code. */ + if (tdx_module_state.sysinit_done) { + ret = tdx_module_state.sysinit_ret; + goto out; + } + + /* RCX is module attributes and all bits are reserved */ + args.rcx = 0; + ret = seamcall_prerr(TDH_SYS_INIT, &args); + + /* + * The first SEAMCALL also detects the TDX module, thus + * it can fail due to the TDX module is not loaded. + * Dump message to let the user know. + */ + if (ret == -ENODEV) + pr_err("module not loaded\n"); + + /* Save the return code for later callers. */ + tdx_module_state.sysinit_done = true; + tdx_module_state.sysinit_ret = ret; +out: + raw_spin_unlock(&sysinit_lock); + return ret; +} + +/** + * Enable VMXON and then do one-time TDX module per-cpu initialization SEAMCALL + * (and TDX module global initialization SEAMCALL if not done) on local cpu to + * make this cpu be ready to run any other SEAMCALLs. + */ +int tdx_cpu_enable(void) +{ + struct tdx_module_args args = {}; + int ret; + + if (__this_cpu_read(tdx_lp_initialized)) + return 0; + + /* + * The TDX module global initialization is the very first step + * to enable TDX. Need to do it first (if hasn't been done) + * before the per-cpu initialization. + */ + ret = try_init_module_global(); + if (ret) + return ret; + + ret = seamcall_prerr(TDH_SYS_LP_INIT, &args); + if (ret) + return ret; + + __this_cpu_write(tdx_lp_initialized, true); + + return 0; +} + +static int tdx_online_cpu(unsigned int cpu) +{ + int ret; + + ret = x86_virt_get_ref(X86_FEATURE_VMX); + if (ret) + return ret; + + ret = tdx_cpu_enable(); + if (ret) + x86_virt_put_ref(X86_FEATURE_VMX); + + return ret; +} + +static void tdx_cpu_flush_cache(void) +{ + lockdep_assert_preemption_disabled(); + + if (!this_cpu_read(cache_state_incoherent)) + return; + + wbinvd(); + this_cpu_write(cache_state_incoherent, false); +} + +static int tdx_offline_cpu(unsigned int cpu) +{ + int i; + + /* No TD is running. Allow any cpu to be offline. */ + if (ida_is_empty(&tdx_guest_keyid_pool)) + goto done; + + /* + * In order to reclaim TDX HKID, (i.e. when deleting guest TD), need to + * call TDH.PHYMEM.PAGE.WBINVD on all packages to program all memory + * controller with pconfig. If we have active TDX HKID, refuse to + * offline the last online cpu. + */ + for_each_online_cpu(i) { + /* + * Found another online cpu on the same package. + * Allow to offline. + */ + if (i != cpu && topology_physical_package_id(i) == + topology_physical_package_id(cpu)) + goto done; + } + + /* + * This is the last cpu of this package. Don't offline it. + * + * Because it's hard for human operator to understand the + * reason, warn it. + */ +#define MSG_ALLPKG_ONLINE \ + "TDX requires all packages to have an online CPU. Delete all TDs in order to offline all CPUs of a package.\n" + pr_warn_ratelimited(MSG_ALLPKG_ONLINE); + return -EBUSY; + +done: + /* + * Flush cache on the CPU going offline to ensure no dirty + * cachelines of TDX private memory remain. This may be + * redundant with WBINVD done elsewhere during CPU offline + * (e.g. hlt_play_dead()), but do it explicitly for safety. + */ + tdx_cpu_flush_cache(); + x86_virt_put_ref(X86_FEATURE_VMX); + return 0; +} + +static void tdx_shutdown_cpu(void *ign) +{ + /* + * Flush cache in preparation for kexec - this is necessary to avoid + * having dirty private memory cachelines when the new kernel boots, + * but WBINVD is a relatively expensive operation and doing it during + * kexec can exacerbate races in native_stop_other_cpus(). Do it + * now, since this is a safe moment and there is going to be no more + * TDX activity on this CPU from this point on. + */ + tdx_cpu_flush_cache(); + x86_virt_put_ref(X86_FEATURE_VMX); +} + +static void tdx_shutdown(void *ign) +{ + tdx_sys_disable(); + on_each_cpu(tdx_shutdown_cpu, NULL, 1); +} + +static int tdx_suspend(void *ign) +{ + x86_virt_put_ref(X86_FEATURE_VMX); + return 0; +} + +static void tdx_resume(void *ign) +{ + WARN_ON_ONCE(x86_virt_get_ref(X86_FEATURE_VMX)); +} + +static const struct syscore_ops tdx_syscore_ops = { + .suspend = tdx_suspend, + .resume = tdx_resume, + .shutdown = tdx_shutdown, +}; + +static struct syscore tdx_syscore = { + .ops = &tdx_syscore_ops, +}; + +/* + * Add a memory region as a TDX memory block. The caller must make sure + * all memory regions are added in address ascending order and don't + * overlap. + */ +static __init int add_tdx_memblock(struct list_head *tmb_list, + unsigned long start_pfn, + unsigned long end_pfn, int nid) +{ + struct tdx_memblock *tmb; + + tmb = kmalloc_obj(*tmb); + if (!tmb) + return -ENOMEM; + + INIT_LIST_HEAD(&tmb->list); + tmb->start_pfn = start_pfn; + tmb->end_pfn = end_pfn; + tmb->nid = nid; + + /* @tmb_list is protected by mem_hotplug_lock */ + list_add_tail(&tmb->list, tmb_list); + return 0; +} + +static __init void free_tdx_memlist(struct list_head *tmb_list) +{ + /* @tmb_list is protected by mem_hotplug_lock */ + while (!list_empty(tmb_list)) { + struct tdx_memblock *tmb = list_first_entry(tmb_list, + struct tdx_memblock, list); + + list_del(&tmb->list); + kfree(tmb); + } +} + +/* + * Ensure that all memblock memory regions are convertible to TDX + * memory. Once this has been established, stash the memblock + * ranges off in a secondary structure because memblock is modified + * in memory hotplug while TDX memory regions are fixed. + */ +static __init int build_tdx_memlist(struct list_head *tmb_list) +{ + unsigned long start_pfn, end_pfn; + int i, nid, ret; + + for_each_mem_pfn_range(i, MAX_NUMNODES, &start_pfn, &end_pfn, &nid) { + /* + * The first 1MB is not reported as TDX convertible memory. + * Although the first 1MB is always reserved and won't end up + * to the page allocator, it is still in memblock's memory + * regions. Skip them manually to exclude them as TDX memory. + */ + start_pfn = max(start_pfn, PHYS_PFN(SZ_1M)); + if (start_pfn >= end_pfn) + continue; + + /* + * Add the memory regions as TDX memory. The regions in + * memblock has already guaranteed they are in address + * ascending order and don't overlap. + */ + ret = add_tdx_memblock(tmb_list, start_pfn, end_pfn, nid); + if (ret) + goto err; + } + + return 0; +err: + free_tdx_memlist(tmb_list); + return ret; +} + +static int read_sys_metadata_field(u64 field_id, u64 *data) +{ + struct tdx_module_args args = {}; + int ret; + + /* + * TDH.SYS.RD -- reads one global metadata field + * - RDX (in): the field to read + * - R8 (out): the field data + */ + args.rdx = field_id; + ret = seamcall_prerr_ret(TDH_SYS_RD, &args); + if (ret) + return ret; + + *data = args.r8; + + return 0; +} + +#include "tdx_global_metadata.c" + +static __init int check_features(struct tdx_sys_info *sysinfo) +{ + u64 tdx_features0 = sysinfo->features.tdx_features0; + + if (!(tdx_features0 & TDX_FEATURES0_NO_RBP_MOD)) { + pr_err("frame pointer (RBP) clobber bug present, upgrade TDX module\n"); + return -EINVAL; + } + + return 0; +} + +/* Calculate the actual TDMR size */ +static __init int tdmr_size_single(u16 max_reserved_per_tdmr) +{ + int tdmr_sz; + + /* + * The actual size of TDMR depends on the maximum + * number of reserved areas. + */ + tdmr_sz = sizeof(struct tdmr_info); + tdmr_sz += sizeof(struct tdmr_reserved_area) * max_reserved_per_tdmr; + + return ALIGN(tdmr_sz, TDMR_INFO_ALIGNMENT); +} + +static __init int alloc_tdmr_list(struct tdmr_info_list *tdmr_list, + struct tdx_sys_info_tdmr *sysinfo_tdmr) +{ + size_t tdmr_sz, tdmr_array_sz; + void *tdmr_array; + + tdmr_sz = tdmr_size_single(sysinfo_tdmr->max_reserved_per_tdmr); + tdmr_array_sz = tdmr_sz * sysinfo_tdmr->max_tdmrs; + + /* + * To keep things simple, allocate all TDMRs together. + * The buffer needs to be physically contiguous to make + * sure each TDMR is physically contiguous. + */ + tdmr_array = alloc_pages_exact(tdmr_array_sz, + GFP_KERNEL | __GFP_ZERO); + if (!tdmr_array) + return -ENOMEM; + + tdmr_list->tdmrs = tdmr_array; + + /* + * Keep the size of TDMR to find the target TDMR + * at a given index in the TDMR list. + */ + tdmr_list->tdmr_sz = tdmr_sz; + tdmr_list->max_tdmrs = sysinfo_tdmr->max_tdmrs; + tdmr_list->nr_consumed_tdmrs = 0; + + return 0; +} + +static __init void free_tdmr_list(struct tdmr_info_list *tdmr_list) +{ + free_pages_exact(tdmr_list->tdmrs, + tdmr_list->max_tdmrs * tdmr_list->tdmr_sz); +} + +/* Get the TDMR from the list at the given index. */ +static struct tdmr_info *tdmr_entry(struct tdmr_info_list *tdmr_list, + int idx) +{ + int tdmr_info_offset = tdmr_list->tdmr_sz * idx; + + return (void *)tdmr_list->tdmrs + tdmr_info_offset; +} + +#define TDMR_ALIGNMENT SZ_1G +#define TDMR_ALIGN_DOWN(_addr) ALIGN_DOWN((_addr), TDMR_ALIGNMENT) +#define TDMR_ALIGN_UP(_addr) ALIGN((_addr), TDMR_ALIGNMENT) + +static inline u64 tdmr_end(struct tdmr_info *tdmr) +{ + return tdmr->base + tdmr->size; +} + +/* + * Take the memory referenced in @tmb_list and populate the + * preallocated @tdmr_list, following all the special alignment + * and size rules for TDMR. + */ +static __init int fill_out_tdmrs(struct list_head *tmb_list, + struct tdmr_info_list *tdmr_list) +{ + struct tdx_memblock *tmb; + int tdmr_idx = 0; + + /* + * Loop over TDX memory regions and fill out TDMRs to cover them. + * To keep it simple, always try to use one TDMR to cover one + * memory region. + * + * In practice TDX supports at least 64 TDMRs. A 2-socket system + * typically only consumes less than 10 of those. This code is + * dumb and simple and may use more TMDRs than is strictly + * required. + */ + list_for_each_entry(tmb, tmb_list, list) { + struct tdmr_info *tdmr = tdmr_entry(tdmr_list, tdmr_idx); + u64 start, end; + + start = TDMR_ALIGN_DOWN(PFN_PHYS(tmb->start_pfn)); + end = TDMR_ALIGN_UP(PFN_PHYS(tmb->end_pfn)); + + /* + * A valid size indicates the current TDMR has already + * been filled out to cover the previous memory region(s). + */ + if (tdmr->size) { + /* + * Loop to the next if the current memory region + * has already been fully covered. + */ + if (end <= tdmr_end(tdmr)) + continue; + + /* Otherwise, skip the already covered part. */ + if (start < tdmr_end(tdmr)) + start = tdmr_end(tdmr); + + /* + * Create a new TDMR to cover the current memory + * region, or the remaining part of it. + */ + tdmr_idx++; + if (tdmr_idx >= tdmr_list->max_tdmrs) { + pr_warn("initialization failed: TDMRs exhausted.\n"); + return -ENOSPC; + } + + tdmr = tdmr_entry(tdmr_list, tdmr_idx); + } + + tdmr->base = start; + tdmr->size = end - start; + } + + /* @tdmr_idx is always the index of the last valid TDMR. */ + tdmr_list->nr_consumed_tdmrs = tdmr_idx + 1; + + /* + * Warn early that kernel is about to run out of TDMRs. + * + * This is an indication that TDMR allocation has to be + * reworked to be smarter to not run into an issue. + */ + if (tdmr_list->max_tdmrs - tdmr_list->nr_consumed_tdmrs < TDMR_NR_WARN) + pr_warn("consumed TDMRs reaching limit: %d used out of %d\n", + tdmr_list->nr_consumed_tdmrs, + tdmr_list->max_tdmrs); + + return 0; +} + +/* + * Calculate PAMT size given a TDMR and a page size. The returned + * PAMT size is always aligned up to 4K page boundary. + */ +static __init unsigned long tdmr_get_pamt_sz(struct tdmr_info *tdmr, int pgsz, + u16 pamt_entry_size) +{ + unsigned long pamt_sz, nr_pamt_entries; + + switch (pgsz) { + case TDX_PS_4K: + nr_pamt_entries = tdmr->size >> PAGE_SHIFT; + break; + case TDX_PS_2M: + nr_pamt_entries = tdmr->size >> PMD_SHIFT; + break; + case TDX_PS_1G: + nr_pamt_entries = tdmr->size >> PUD_SHIFT; + break; + default: + WARN_ON_ONCE(1); + return 0; + } + + pamt_sz = nr_pamt_entries * pamt_entry_size; + /* TDX requires PAMT size must be 4K aligned */ + pamt_sz = ALIGN(pamt_sz, PAGE_SIZE); + + return pamt_sz; +} + +/* + * Locate a NUMA node which should hold the allocation of the @tdmr + * PAMT. This node will have some memory covered by the TDMR. The + * relative amount of memory covered is not considered. + */ +static __init int tdmr_get_nid(struct tdmr_info *tdmr, struct list_head *tmb_list) +{ + struct tdx_memblock *tmb; + + /* + * A TDMR must cover at least part of one TMB. That TMB will end + * after the TDMR begins. But, that TMB may have started before + * the TDMR. Find the next 'tmb' that _ends_ after this TDMR + * begins. Ignore 'tmb' start addresses. They are irrelevant. + */ + list_for_each_entry(tmb, tmb_list, list) { + if (tmb->end_pfn > PHYS_PFN(tdmr->base)) + return tmb->nid; + } + + /* + * Fall back to allocating the TDMR's metadata from node 0 when + * no TDX memory block can be found. This should never happen + * since TDMRs originate from TDX memory blocks. + */ + pr_warn("TDMR [0x%llx, 0x%llx): unable to find local NUMA node for PAMT allocation, fallback to use node 0.\n", + tdmr->base, tdmr_end(tdmr)); + return 0; +} + +/* + * Allocate PAMTs from the local NUMA node of some memory in @tmb_list + * within @tdmr, and set up PAMTs for @tdmr. + */ +static __init int tdmr_set_up_pamt(struct tdmr_info *tdmr, + struct list_head *tmb_list, + u16 pamt_entry_size[]) +{ + unsigned long pamt_base[TDX_PS_NR]; + unsigned long pamt_size[TDX_PS_NR]; + unsigned long tdmr_pamt_base; + unsigned long tdmr_pamt_size; + struct page *pamt; + int pgsz, nid; + + nid = tdmr_get_nid(tdmr, tmb_list); + + /* + * Calculate the PAMT size for each TDX supported page size + * and the total PAMT size. + */ + tdmr_pamt_size = 0; + for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) { + pamt_size[pgsz] = tdmr_get_pamt_sz(tdmr, pgsz, + pamt_entry_size[pgsz]); + tdmr_pamt_size += pamt_size[pgsz]; + } + + /* + * Allocate one chunk of physically contiguous memory for all + * PAMTs. This helps minimize the PAMT's use of reserved areas + * in overlapped TDMRs. + */ + pamt = alloc_contig_pages(tdmr_pamt_size >> PAGE_SHIFT, GFP_KERNEL, + nid, &node_online_map); + if (!pamt) + return -ENOMEM; + + /* + * Break the contiguous allocation back up into the + * individual PAMTs for each page size. + */ + tdmr_pamt_base = page_to_pfn(pamt) << PAGE_SHIFT; + for (pgsz = TDX_PS_4K; pgsz < TDX_PS_NR; pgsz++) { + pamt_base[pgsz] = tdmr_pamt_base; + tdmr_pamt_base += pamt_size[pgsz]; + } + + tdmr->pamt_4k_base = pamt_base[TDX_PS_4K]; + tdmr->pamt_4k_size = pamt_size[TDX_PS_4K]; + tdmr->pamt_2m_base = pamt_base[TDX_PS_2M]; + tdmr->pamt_2m_size = pamt_size[TDX_PS_2M]; + tdmr->pamt_1g_base = pamt_base[TDX_PS_1G]; + tdmr->pamt_1g_size = pamt_size[TDX_PS_1G]; + + return 0; +} + +static void tdmr_get_pamt(struct tdmr_info *tdmr, unsigned long *pamt_base, + unsigned long *pamt_size) +{ + unsigned long pamt_bs, pamt_sz; + + /* + * The PAMT was allocated in one contiguous unit. The 4K PAMT + * should always point to the beginning of that allocation. + */ + pamt_bs = tdmr->pamt_4k_base; + pamt_sz = tdmr->pamt_4k_size + tdmr->pamt_2m_size + tdmr->pamt_1g_size; + + WARN_ON_ONCE((pamt_bs & ~PAGE_MASK) || (pamt_sz & ~PAGE_MASK)); + + *pamt_base = pamt_bs; + *pamt_size = pamt_sz; +} + +static __init void tdmr_do_pamt_func(struct tdmr_info *tdmr, + void (*pamt_func)(unsigned long base, unsigned long size)) +{ + unsigned long pamt_base, pamt_size; + + tdmr_get_pamt(tdmr, &pamt_base, &pamt_size); + + /* Do nothing if PAMT hasn't been allocated for this TDMR */ + if (!pamt_size) + return; + + if (WARN_ON_ONCE(!pamt_base)) + return; + + pamt_func(pamt_base, pamt_size); +} + +static __init void free_pamt(unsigned long pamt_base, unsigned long pamt_size) +{ + free_contig_range(pamt_base >> PAGE_SHIFT, pamt_size >> PAGE_SHIFT); +} + +static __init void tdmr_free_pamt(struct tdmr_info *tdmr) +{ + tdmr_do_pamt_func(tdmr, free_pamt); +} + +static __init void tdmrs_free_pamt_all(struct tdmr_info_list *tdmr_list) +{ + int i; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) + tdmr_free_pamt(tdmr_entry(tdmr_list, i)); +} + +/* Allocate and set up PAMTs for all TDMRs */ +static __init int tdmrs_set_up_pamt_all(struct tdmr_info_list *tdmr_list, + struct list_head *tmb_list, + u16 pamt_entry_size[]) +{ + int i, ret = 0; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { + ret = tdmr_set_up_pamt(tdmr_entry(tdmr_list, i), tmb_list, + pamt_entry_size); + if (ret) + goto err; + } + + return 0; +err: + tdmrs_free_pamt_all(tdmr_list); + return ret; +} + +/* + * Convert TDX private pages back to normal by using MOVDIR64B to clear these + * pages. Typically, any write to the page will convert it from TDX private back + * to normal kernel memory. Systems with the X86_BUG_TDX_PW_MCE erratum need to + * do the conversion explicitly via MOVDIR64B. + */ +void tdx_quirk_reset_paddr(unsigned long base, unsigned long size) +{ + const void *zero_page = (const void *)page_address(ZERO_PAGE(0)); + unsigned long phys, end; + + if (!boot_cpu_has_bug(X86_BUG_TDX_PW_MCE)) + return; + + end = base + size; + for (phys = base; phys < end; phys += 64) + movdir64b(__va(phys), zero_page); + + /* + * MOVDIR64B uses WC protocol. Use memory barrier to + * make sure any later user of these pages sees the + * updated data. + */ + mb(); +} +EXPORT_SYMBOL_FOR_KVM(tdx_quirk_reset_paddr); + +static __init void tdmr_quirk_reset_pamt(struct tdmr_info *tdmr) + +{ + tdmr_do_pamt_func(tdmr, tdx_quirk_reset_paddr); +} + +static __init void tdmrs_quirk_reset_pamt_all(struct tdmr_info_list *tdmr_list) +{ + int i; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) + tdmr_quirk_reset_pamt(tdmr_entry(tdmr_list, i)); +} + +static __init unsigned long tdmrs_count_pamt_kb(struct tdmr_info_list *tdmr_list) +{ + unsigned long pamt_size = 0; + int i; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { + unsigned long base, size; + + tdmr_get_pamt(tdmr_entry(tdmr_list, i), &base, &size); + pamt_size += size; + } + + return pamt_size / 1024; +} + +static __init int tdmr_add_rsvd_area(struct tdmr_info *tdmr, int *p_idx, + u64 addr, u64 size, u16 max_reserved_per_tdmr) +{ + struct tdmr_reserved_area *rsvd_areas = tdmr->reserved_areas; + int idx = *p_idx; + + /* Reserved area must be 4K aligned in offset and size */ + if (WARN_ON(addr & ~PAGE_MASK || size & ~PAGE_MASK)) + return -EINVAL; + + if (idx >= max_reserved_per_tdmr) { + pr_warn("initialization failed: TDMR [0x%llx, 0x%llx): reserved areas exhausted.\n", + tdmr->base, tdmr_end(tdmr)); + return -ENOSPC; + } + + /* + * Consume one reserved area per call. Make no effort to + * optimize or reduce the number of reserved areas which are + * consumed by contiguous reserved areas, for instance. + */ + rsvd_areas[idx].offset = addr - tdmr->base; + rsvd_areas[idx].size = size; + + *p_idx = idx + 1; + + return 0; +} + +/* + * Go through @tmb_list to find holes between memory areas. If any of + * those holes fall within @tdmr, set up a TDMR reserved area to cover + * the hole. + */ +static __init int tdmr_populate_rsvd_holes(struct list_head *tmb_list, + struct tdmr_info *tdmr, + int *rsvd_idx, + u16 max_reserved_per_tdmr) +{ + struct tdx_memblock *tmb; + u64 prev_end; + int ret; + + /* + * Start looking for reserved blocks at the + * beginning of the TDMR. + */ + prev_end = tdmr->base; + list_for_each_entry(tmb, tmb_list, list) { + u64 start, end; + + start = PFN_PHYS(tmb->start_pfn); + end = PFN_PHYS(tmb->end_pfn); + + /* Break if this region is after the TDMR */ + if (start >= tdmr_end(tdmr)) + break; + + /* Exclude regions before this TDMR */ + if (end < tdmr->base) + continue; + + /* + * Skip over memory areas that + * have already been dealt with. + */ + if (start <= prev_end) { + prev_end = end; + continue; + } + + /* Add the hole before this region */ + ret = tdmr_add_rsvd_area(tdmr, rsvd_idx, prev_end, + start - prev_end, + max_reserved_per_tdmr); + if (ret) + return ret; + + prev_end = end; + } + + /* Add the hole after the last region if it exists. */ + if (prev_end < tdmr_end(tdmr)) { + ret = tdmr_add_rsvd_area(tdmr, rsvd_idx, prev_end, + tdmr_end(tdmr) - prev_end, + max_reserved_per_tdmr); + if (ret) + return ret; + } + + return 0; +} + +/* + * Go through @tdmr_list to find all PAMTs. If any of those PAMTs + * overlaps with @tdmr, set up a TDMR reserved area to cover the + * overlapping part. + */ +static __init int tdmr_populate_rsvd_pamts(struct tdmr_info_list *tdmr_list, + struct tdmr_info *tdmr, + int *rsvd_idx, + u16 max_reserved_per_tdmr) +{ + int i, ret; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { + struct tdmr_info *tmp = tdmr_entry(tdmr_list, i); + unsigned long pamt_base, pamt_size, pamt_end; + + tdmr_get_pamt(tmp, &pamt_base, &pamt_size); + /* Each TDMR must already have PAMT allocated */ + WARN_ON_ONCE(!pamt_size || !pamt_base); + + pamt_end = pamt_base + pamt_size; + /* Skip PAMTs outside of the given TDMR */ + if ((pamt_end <= tdmr->base) || + (pamt_base >= tdmr_end(tdmr))) + continue; + + /* Only mark the part within the TDMR as reserved */ + if (pamt_base < tdmr->base) + pamt_base = tdmr->base; + if (pamt_end > tdmr_end(tdmr)) + pamt_end = tdmr_end(tdmr); + + ret = tdmr_add_rsvd_area(tdmr, rsvd_idx, pamt_base, + pamt_end - pamt_base, + max_reserved_per_tdmr); + if (ret) + return ret; + } + + return 0; +} + +/* Compare function called by sort() for TDMR reserved areas */ +static __init int rsvd_area_cmp_func(const void *a, const void *b) +{ + struct tdmr_reserved_area *r1 = (struct tdmr_reserved_area *)a; + struct tdmr_reserved_area *r2 = (struct tdmr_reserved_area *)b; + + if (r1->offset + r1->size <= r2->offset) + return -1; + if (r1->offset >= r2->offset + r2->size) + return 1; + + /* Reserved areas cannot overlap. The caller must guarantee. */ + WARN_ON_ONCE(1); + return -1; +} + +/* + * Populate reserved areas for the given @tdmr, including memory holes + * (via @tmb_list) and PAMTs (via @tdmr_list). + */ +static __init int tdmr_populate_rsvd_areas(struct tdmr_info *tdmr, + struct list_head *tmb_list, + struct tdmr_info_list *tdmr_list, + u16 max_reserved_per_tdmr) +{ + int ret, rsvd_idx = 0; + + ret = tdmr_populate_rsvd_holes(tmb_list, tdmr, &rsvd_idx, + max_reserved_per_tdmr); + if (ret) + return ret; + + ret = tdmr_populate_rsvd_pamts(tdmr_list, tdmr, &rsvd_idx, + max_reserved_per_tdmr); + if (ret) + return ret; + + /* TDX requires reserved areas listed in address ascending order */ + sort(tdmr->reserved_areas, rsvd_idx, sizeof(struct tdmr_reserved_area), + rsvd_area_cmp_func, NULL); + + return 0; +} + +/* + * Populate reserved areas for all TDMRs in @tdmr_list, including memory + * holes (via @tmb_list) and PAMTs. + */ +static __init int tdmrs_populate_rsvd_areas_all(struct tdmr_info_list *tdmr_list, + struct list_head *tmb_list, + u16 max_reserved_per_tdmr) +{ + int i; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { + int ret; + + ret = tdmr_populate_rsvd_areas(tdmr_entry(tdmr_list, i), + tmb_list, tdmr_list, max_reserved_per_tdmr); + if (ret) + return ret; + } + + return 0; +} + +/* + * Construct a list of TDMRs on the preallocated space in @tdmr_list + * to cover all TDX memory regions in @tmb_list based on the TDX module + * TDMR global information in @sysinfo_tdmr. + */ +static __init int construct_tdmrs(struct list_head *tmb_list, + struct tdmr_info_list *tdmr_list, + struct tdx_sys_info_tdmr *sysinfo_tdmr) +{ + u16 pamt_entry_size[TDX_PS_NR] = { + sysinfo_tdmr->pamt_4k_entry_size, + sysinfo_tdmr->pamt_2m_entry_size, + sysinfo_tdmr->pamt_1g_entry_size, + }; + int ret; + + ret = fill_out_tdmrs(tmb_list, tdmr_list); + if (ret) + return ret; + + ret = tdmrs_set_up_pamt_all(tdmr_list, tmb_list, pamt_entry_size); + if (ret) + return ret; + + ret = tdmrs_populate_rsvd_areas_all(tdmr_list, tmb_list, + sysinfo_tdmr->max_reserved_per_tdmr); + if (ret) + tdmrs_free_pamt_all(tdmr_list); + + /* + * The tdmr_info_list is read-only from here on out. + * Ensure that these writes are seen by other CPUs. + * Pairs with a smp_rmb() in is_pamt_page(). + */ + smp_wmb(); + + return ret; +} + +static __init int config_tdx_module(struct tdmr_info_list *tdmr_list, + u64 global_keyid) +{ + struct tdx_module_args args = {}; + u64 *tdmr_pa_array; + size_t array_sz; + int i, ret; + + /* + * TDMRs are passed to the TDX module via an array of physical + * addresses of each TDMR. The array itself also has certain + * alignment requirement. + */ + array_sz = tdmr_list->nr_consumed_tdmrs * sizeof(u64); + array_sz = roundup_pow_of_two(array_sz); + if (array_sz < TDMR_INFO_PA_ARRAY_ALIGNMENT) + array_sz = TDMR_INFO_PA_ARRAY_ALIGNMENT; + + tdmr_pa_array = kzalloc(array_sz, GFP_KERNEL); + if (!tdmr_pa_array) + return -ENOMEM; + + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) + tdmr_pa_array[i] = __pa(tdmr_entry(tdmr_list, i)); + + args.rcx = __pa(tdmr_pa_array); + args.rdx = tdmr_list->nr_consumed_tdmrs; + args.r8 = global_keyid; + ret = seamcall_prerr(TDH_SYS_CONFIG, &args); + + /* Free the array as it is not required anymore. */ + kfree(tdmr_pa_array); + + return ret; +} + +static __init int do_global_key_config(void *unused) +{ + struct tdx_module_args args = {}; + + return seamcall_prerr(TDH_SYS_KEY_CONFIG, &args); +} + +/* + * Attempt to configure the global KeyID on all physical packages. + * + * This requires running code on at least one CPU in each package. + * TDMR initialization) will fail will fail if any package in the + * system has no online CPUs. + * + * This code takes no affirmative steps to online CPUs. Callers (aka. + * KVM) can ensure success by ensuring sufficient CPUs are online and + * can run SEAMCALLs. + */ +static __init int config_global_keyid(void) +{ + cpumask_var_t packages; + int cpu, ret = -EINVAL; + + if (!zalloc_cpumask_var(&packages, GFP_KERNEL)) + return -ENOMEM; + + /* + * Hardware doesn't guarantee cache coherency across different + * KeyIDs. The kernel needs to flush PAMT's dirty cachelines + * (associated with KeyID 0) before the TDX module can use the + * global KeyID to access the PAMT. Given PAMTs are potentially + * large (~1/256th of system RAM), just use WBINVD. + */ + wbinvd_on_all_cpus(); + + for_each_online_cpu(cpu) { + /* + * The key configuration only needs to be done once per + * package and will return an error if configured more + * than once. Avoid doing it multiple times per package. + */ + if (cpumask_test_and_set_cpu(topology_physical_package_id(cpu), + packages)) + continue; + + /* + * TDH.SYS.KEY.CONFIG cannot run concurrently on + * different cpus. Do it one by one. + */ + ret = smp_call_on_cpu(cpu, do_global_key_config, NULL, true); + if (ret) + break; + } + + free_cpumask_var(packages); + return ret; +} + +static __init int init_tdmr(struct tdmr_info *tdmr) +{ + u64 next; + + /* + * Initializing a TDMR can be time consuming. To avoid long + * SEAMCALLs, the TDX module may only initialize a part of the + * TDMR in each call. + */ + do { + struct tdx_module_args args = { + .rcx = tdmr->base, + }; + int ret; + + ret = seamcall_prerr_ret(TDH_SYS_TDMR_INIT, &args); + if (ret) + return ret; + /* + * RDX contains 'next-to-initialize' address if + * TDH.SYS.TDMR.INIT did not fully complete and + * should be retried. + */ + next = args.rdx; + cond_resched(); + /* Keep making SEAMCALLs until the TDMR is done */ + } while (next < tdmr->base + tdmr->size); + + return 0; +} + +static __init int init_tdmrs(struct tdmr_info_list *tdmr_list) +{ + int i; + + /* + * This operation is costly. It can be parallelized, + * but keep it simple for now. + */ + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { + int ret; + + ret = init_tdmr(tdmr_entry(tdmr_list, i)); + if (ret) + return ret; + } + + return 0; +} + +static __init int init_tdx_module(void) +{ + int ret; + + ret = get_tdx_sys_info(&tdx_sysinfo); + if (ret) + return ret; + + /* Check whether the kernel can support this module */ + ret = check_features(&tdx_sysinfo); + if (ret) + return ret; + + /* + * To keep things simple, assume that all TDX-protected memory + * will come from the page allocator. Make sure all pages in the + * page allocator are TDX-usable memory. + * + * Build the list of "TDX-usable" memory regions which cover all + * pages in the page allocator to guarantee that. Do it while + * holding mem_hotplug_lock read-lock as the memory hotplug code + * path reads the @tdx_memlist to reject any new memory. + */ + get_online_mems(); + + ret = build_tdx_memlist(&tdx_memlist); + if (ret) + goto out_put_tdxmem; + + /* Allocate enough space for constructing TDMRs */ + ret = alloc_tdmr_list(&tdx_tdmr_list, &tdx_sysinfo.tdmr); + if (ret) + goto err_free_tdxmem; + + /* Cover all TDX-usable memory regions in TDMRs */ + ret = construct_tdmrs(&tdx_memlist, &tdx_tdmr_list, &tdx_sysinfo.tdmr); + if (ret) + goto err_free_tdmrs; + + /* Pass the TDMRs and the global KeyID to the TDX module */ + ret = config_tdx_module(&tdx_tdmr_list, tdx_global_keyid); + if (ret) + goto err_free_pamts; + + /* Config the key of global KeyID on all packages */ + ret = config_global_keyid(); + if (ret) + goto err_reset_pamts; + + /* Initialize TDMRs to complete the TDX module initialization */ + ret = init_tdmrs(&tdx_tdmr_list); + if (ret) + goto err_reset_pamts; + + pr_info("%lu KB allocated for PAMT\n", tdmrs_count_pamt_kb(&tdx_tdmr_list)); + +out_put_tdxmem: + /* + * @tdx_memlist is written here and read at memory hotplug time. + * Lock out memory hotplug code while building it. + */ + put_online_mems(); + return ret; + +err_reset_pamts: + /* + * Part of PAMTs may already have been initialized by the + * TDX module. Flush cache before returning PAMTs back + * to the kernel. + */ + wbinvd_on_all_cpus(); + tdmrs_quirk_reset_pamt_all(&tdx_tdmr_list); +err_free_pamts: + tdmrs_free_pamt_all(&tdx_tdmr_list); +err_free_tdmrs: + free_tdmr_list(&tdx_tdmr_list); +err_free_tdxmem: + free_tdx_memlist(&tdx_memlist); + goto out_put_tdxmem; +} + +static __init int tdx_enable(void) +{ + enum cpuhp_state state; + int ret; + + if (!cpu_feature_enabled(X86_FEATURE_TDX_HOST_PLATFORM)) { + pr_err("TDX not supported by the host platform\n"); + return -ENODEV; + } + + if (!cpu_feature_enabled(X86_FEATURE_XSAVE)) { + pr_err("XSAVE is required for TDX\n"); + return -EINVAL; + } + + if (!cpu_feature_enabled(X86_FEATURE_MOVDIR64B)) { + pr_err("MOVDIR64B is required for TDX\n"); + return -EINVAL; + } + + if (!cpu_feature_enabled(X86_FEATURE_SELFSNOOP)) { + pr_err("Self-snoop is required for TDX\n"); + return -ENODEV; + } + + state = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "virt/tdx:online", + tdx_online_cpu, tdx_offline_cpu); + if (state < 0) + return state; + + ret = init_tdx_module(); + if (ret) { + pr_err("TDX-Module initialization failed (%d)\n", ret); + cpuhp_remove_state(state); + return ret; + } + + register_syscore(&tdx_syscore); + + tdx_module_state.initialized = true; + pr_info("TDX-Module initialized\n"); + return 0; +} +subsys_initcall(tdx_enable); + +int tdx_module_shutdown(void) +{ + struct tdx_sys_info_handoff handoff = {}; + struct tdx_module_args args = {}; + int ret; + int cpu; + + ret = get_tdx_sys_info_handoff(&handoff); + /* + * Handoff information is required for proper + * shutdown. Refuse to shut down without it. + */ + if (ret) + return ret; + + /* + * Use the module's handoff version as it is the highest the + * module can produce and most likely supported by newer modules. + */ + args.rcx = handoff.module_hv; + + ret = seamcall_prerr(TDH_SYS_SHUTDOWN, &args); + if (ret) + return ret; + + /* + * Clear global and per-CPU initialization flags so the new module + * can be fully re-initialized after a successful update. + * + * No locks needed as no concurrent accesses can occur here. + */ + memset(&tdx_module_state, 0, sizeof(tdx_module_state)); + for_each_possible_cpu(cpu) + per_cpu(tdx_lp_initialized, cpu) = false; + + return 0; +} + +int tdx_module_run_update(void) +{ + struct tdx_module_args args = {}; + int ret; + + ret = seamcall_prerr(TDH_SYS_UPDATE, &args); + if (ret) + return ret; + + ret = get_tdx_sys_info_version(&tdx_sysinfo.version); + /* + * Only fails if there is something unexpected + * and severely wrong with the module. + */ + WARN_ON_ONCE(ret); + + tdx_module_state.initialized = true; + return 0; +} + +static bool is_pamt_page(unsigned long phys) +{ + struct tdmr_info_list *tdmr_list = &tdx_tdmr_list; + int i; + + /* Ensure that all remote 'tdmr_list' writes are visible: */ + smp_rmb(); + + /* + * The TDX module is no longer returning TDX_SYS_NOT_READY and + * is initialized. The 'tdmr_list' was initialized long ago + * and is now read-only. + */ + for (i = 0; i < tdmr_list->nr_consumed_tdmrs; i++) { + unsigned long base, size; + + tdmr_get_pamt(tdmr_entry(tdmr_list, i), &base, &size); + + if (phys >= base && phys < (base + size)) + return true; + } + + return false; +} + +/* + * Return whether the memory page at the given physical address is TDX + * private memory or not. + * + * This can be imprecise for two known reasons: + * 1. PAMTs are private memory and exist before the TDX module is + * ready and TDH_PHYMEM_PAGE_RDMD works. This is a relatively + * short window that occurs once per boot. + * 2. TDH_PHYMEM_PAGE_RDMD reflects the TDX module's knowledge of the + * page. However, the page can still cause #MC until it has been + * fully converted to shared using 64-byte writes like MOVDIR64B. + * Buggy hosts might still leave #MC-causing memory in place which + * this function can not detect. + */ +static bool paddr_is_tdx_private(unsigned long phys) +{ + struct tdx_module_args args = { + .rcx = phys & PAGE_MASK, + }; + u64 sret; + + if (!boot_cpu_has(X86_FEATURE_TDX_HOST_PLATFORM)) + return false; + + /* Get page type from the TDX module */ + sret = __seamcall_dirty_cache(__seamcall_ret, TDH_PHYMEM_PAGE_RDMD, &args); + + /* + * The SEAMCALL will not return success unless there is a + * working, "ready" TDX module. Assume an absence of TDX + * private pages until SEAMCALL is working. + */ + if (sret) + return false; + + /* + * SEAMCALL was successful -- read page type (via RCX): + * + * - PT_NDA: Page is not used by the TDX module + * - PT_RSVD: Reserved for Non-TDX use + * - Others: Page is used by the TDX module + * + * Note PAMT pages are marked as PT_RSVD but they are also TDX + * private memory. + */ + switch (args.rcx) { + case PT_NDA: + return false; + case PT_RSVD: + return is_pamt_page(phys); + default: + return true; + } +} + +/* + * Some TDX-capable CPUs have an erratum. A write to TDX private + * memory poisons that memory, and a subsequent read of that memory + * triggers #MC. + * + * Help distinguish erratum-triggered #MCs from a normal hardware one. + * Just print additional message to show such #MC may be result of the + * erratum. + */ +const char *tdx_dump_mce_info(struct mce *m) +{ + if (!m || !mce_is_memory_error(m) || !mce_usable_address(m)) + return NULL; + + if (!paddr_is_tdx_private(m->addr)) + return NULL; + + return "TDX private memory error. Possible kernel bug."; +} + +static __init int record_keyid_partitioning(u32 *tdx_keyid_start, + u32 *nr_tdx_keyids) +{ + u32 _nr_mktme_keyids, _tdx_keyid_start, _nr_tdx_keyids; + int ret; + + /* + * IA32_MKTME_KEYID_PARTIONING: + * Bit [31:0]: Number of MKTME KeyIDs. + * Bit [63:32]: Number of TDX private KeyIDs. + */ + ret = rdmsr_safe(MSR_IA32_MKTME_KEYID_PARTITIONING, &_nr_mktme_keyids, + &_nr_tdx_keyids); + if (ret || !_nr_tdx_keyids) + return -EINVAL; + + /* TDX KeyIDs start after the last MKTME KeyID. */ + _tdx_keyid_start = _nr_mktme_keyids + 1; + + *tdx_keyid_start = _tdx_keyid_start; + *nr_tdx_keyids = _nr_tdx_keyids; + + return 0; +} + +static bool is_tdx_memory(unsigned long start_pfn, unsigned long end_pfn) +{ + struct tdx_memblock *tmb; + + /* + * This check assumes that the start_pfn<->end_pfn range does not + * cross multiple @tdx_memlist entries. A single memory online + * event across multiple memblocks (from which @tdx_memlist + * entries are derived at the time of module initialization) is + * not possible. This is because memory offline/online is done + * on granularity of 'struct memory_block', and the hotpluggable + * memory region (one memblock) must be multiple of memory_block. + */ + list_for_each_entry(tmb, &tdx_memlist, list) { + if (start_pfn >= tmb->start_pfn && end_pfn <= tmb->end_pfn) + return true; + } + return false; +} + +static int tdx_memory_notifier(struct notifier_block *nb, unsigned long action, + void *v) +{ + struct memory_notify *mn = v; + + if (action != MEM_GOING_ONLINE) + return NOTIFY_OK; + + /* + * Empty list means TDX isn't enabled. Allow any memory + * to go online. + */ + if (list_empty(&tdx_memlist)) + return NOTIFY_OK; + + /* + * The TDX memory configuration is static and can not be + * changed. Reject onlining any memory which is outside of + * the static configuration whether it supports TDX or not. + */ + if (is_tdx_memory(mn->start_pfn, mn->start_pfn + mn->nr_pages)) + return NOTIFY_OK; + + return NOTIFY_BAD; +} + +static struct notifier_block tdx_memory_nb = { + .notifier_call = tdx_memory_notifier, +}; + +static void __init check_tdx_erratum(void) +{ + u64 basic_msr; + + /* + * These CPUs have an erratum. A partial write from non-TD + * software (e.g. via MOVNTI variants or UC/WC mapping) to TDX + * private memory poisons that memory, and a subsequent read of + * that memory triggers #MC. + */ + switch (boot_cpu_data.x86_vfm) { + case INTEL_SAPPHIRERAPIDS_X: + case INTEL_EMERALDRAPIDS_X: + setup_force_cpu_bug(X86_BUG_TDX_PW_MCE); + } + + /* + * Some TDX-capable CPUs have an erratum where the current VMCS is + * cleared after calling into P-SEAMLDR. + */ + rdmsrq(MSR_IA32_VMX_BASIC, basic_msr); + if (!(basic_msr & VMX_BASIC_NO_SEAMRET_INVD_VMCS)) + setup_force_cpu_bug(X86_BUG_SEAMRET_INVD_VMCS); +} + +void __init tdx_init(void) +{ + u32 tdx_keyid_start, nr_tdx_keyids; + int err; + + err = record_keyid_partitioning(&tdx_keyid_start, &nr_tdx_keyids); + if (err) + return; + + pr_info("BIOS enabled: private KeyID range [%u, %u)\n", + tdx_keyid_start, tdx_keyid_start + nr_tdx_keyids); + + /* + * The TDX module itself requires one 'global KeyID' to protect + * its metadata. If there's only one TDX KeyID, there won't be + * any left for TDX guests thus there's no point to enable TDX + * at all. + */ + if (nr_tdx_keyids < 2) { + pr_err("initialization failed: too few private KeyIDs available.\n"); + return; + } + + /* + * At this point, hibernation_available() indicates whether or + * not hibernation support has been permanently disabled. + */ + if (hibernation_available()) { + pr_err("initialization failed: Hibernation support is enabled\n"); + return; + } + + err = register_memory_notifier(&tdx_memory_nb); + if (err) { + pr_err("initialization failed: register_memory_notifier() failed (%d)\n", + err); + return; + } + +#if defined(CONFIG_ACPI) && defined(CONFIG_SUSPEND) + pr_info("Disable ACPI S3. Turn off TDX in the BIOS to use ACPI S3.\n"); + acpi_suspend_lowlevel = NULL; +#endif + + /* + * Just use the first TDX KeyID as the 'global KeyID' and + * leave the rest for TDX guests. + */ + tdx_global_keyid = tdx_keyid_start; + tdx_guest_keyid_start = tdx_keyid_start + 1; + tdx_nr_guest_keyids = nr_tdx_keyids - 1; + + setup_force_cpu_cap(X86_FEATURE_TDX_HOST_PLATFORM); + + check_tdx_erratum(); +} + +const struct tdx_sys_info *tdx_get_sysinfo(void) +{ + if (!tdx_module_state.initialized) + return NULL; + + return (const struct tdx_sys_info *)&tdx_sysinfo; +} +EXPORT_SYMBOL_FOR_MODULES(tdx_get_sysinfo, "kvm-intel,tdx-host"); + +u32 tdx_get_nr_guest_keyids(void) +{ + return tdx_nr_guest_keyids; +} +EXPORT_SYMBOL_FOR_KVM(tdx_get_nr_guest_keyids); + +int tdx_guest_keyid_alloc(void) +{ + return ida_alloc_range(&tdx_guest_keyid_pool, tdx_guest_keyid_start, + tdx_guest_keyid_start + tdx_nr_guest_keyids - 1, + GFP_KERNEL); +} +EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_alloc); + +void tdx_guest_keyid_free(unsigned int keyid) +{ + ida_free(&tdx_guest_keyid_pool, keyid); +} +EXPORT_SYMBOL_FOR_KVM(tdx_guest_keyid_free); + +static inline u64 tdx_tdr_pa(struct tdx_td *td) +{ + return page_to_phys(td->tdr_page); +} + +/* + * The TDX module exposes a CLFLUSH_BEFORE_ALLOC bit to specify whether + * a CLFLUSH of pages is required before handing them to the TDX module. + * Be conservative and make the code simpler by doing the CLFLUSH + * unconditionally. + */ +static void tdx_clflush_page(struct page *page) +{ + clflush_cache_range(page_to_virt(page), PAGE_SIZE); +} + +static void tdx_clflush_pfn(kvm_pfn_t pfn) +{ + clflush_cache_range(__va(PFN_PHYS(pfn)), PAGE_SIZE); +} + +static int pg_level_to_tdx_sept_level(enum pg_level level) +{ + WARN_ON_ONCE(level == PG_LEVEL_NONE); + return level - 1; +} + +noinstr u64 tdh_vp_enter(struct tdx_vp *td, struct tdx_module_args *args) +{ + args->rcx = td->tdvpr_pa; + + return __seamcall_dirty_cache(__seamcall_saved_ret, TDH_VP_ENTER, args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_enter); + +u64 tdh_mng_addcx(struct tdx_td *td, struct page *tdcs_page) +{ + struct tdx_module_args args = { + .rcx = page_to_phys(tdcs_page), + .rdx = tdx_tdr_pa(td), + }; + + tdx_clflush_page(tdcs_page); + return seamcall(TDH_MNG_ADDCX, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_addcx); + +u64 tdh_mem_page_add(struct tdx_td *td, u64 gpa, kvm_pfn_t pfn, struct page *source, + u64 *ext_err1, u64 *ext_err2) +{ + struct tdx_module_args args = { + .rcx = gpa, + .rdx = tdx_tdr_pa(td), + .r8 = PFN_PHYS(pfn), + .r9 = page_to_phys(source), + }; + u64 ret; + + tdx_clflush_pfn(pfn); + ret = seamcall_ret(TDH_MEM_PAGE_ADD, &args); + + *ext_err1 = args.rcx; + *ext_err2 = args.rdx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_add); + +u64 tdh_mem_sept_add(struct tdx_td *td, u64 gpa, enum pg_level level, + struct page *page, u64 *ext_err1, u64 *ext_err2) +{ + struct tdx_module_args args = { + .rcx = gpa | pg_level_to_tdx_sept_level(level), + .rdx = tdx_tdr_pa(td), + .r8 = page_to_phys(page), + }; + u64 ret; + + tdx_clflush_page(page); + ret = seamcall_ret(TDH_MEM_SEPT_ADD, &args); + + *ext_err1 = args.rcx; + *ext_err2 = args.rdx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mem_sept_add); + +u64 tdh_vp_addcx(struct tdx_vp *vp, struct page *tdcx_page) +{ + struct tdx_module_args args = { + .rcx = page_to_phys(tdcx_page), + .rdx = vp->tdvpr_pa, + }; + + tdx_clflush_page(tdcx_page); + return seamcall(TDH_VP_ADDCX, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_addcx); + +u64 tdh_mem_page_aug(struct tdx_td *td, u64 gpa, enum pg_level level, + kvm_pfn_t pfn, u64 *ext_err1, u64 *ext_err2) +{ + struct tdx_module_args args = { + .rcx = gpa | pg_level_to_tdx_sept_level(level), + .rdx = tdx_tdr_pa(td), + .r8 = PFN_PHYS(pfn), + }; + u64 ret; + + tdx_clflush_pfn(pfn); + ret = seamcall_ret(TDH_MEM_PAGE_AUG, &args); + + *ext_err1 = args.rcx; + *ext_err2 = args.rdx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_aug); + +u64 tdh_mem_range_block(struct tdx_td *td, u64 gpa, enum pg_level level, + u64 *ext_err1, u64 *ext_err2) +{ + struct tdx_module_args args = { + .rcx = gpa | pg_level_to_tdx_sept_level(level), + .rdx = tdx_tdr_pa(td), + }; + u64 ret; + + ret = seamcall_ret(TDH_MEM_RANGE_BLOCK, &args); + + *ext_err1 = args.rcx; + *ext_err2 = args.rdx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mem_range_block); + +u64 tdh_mng_key_config(struct tdx_td *td) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + }; + + return seamcall(TDH_MNG_KEY_CONFIG, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_key_config); + +u64 tdh_mng_create(struct tdx_td *td, u16 hkid) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + .rdx = hkid, + }; + + tdx_clflush_page(td->tdr_page); + return seamcall(TDH_MNG_CREATE, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_create); + +u64 tdh_vp_create(struct tdx_td *td, struct tdx_vp *vp) +{ + struct tdx_module_args args = { + .rcx = vp->tdvpr_pa, + .rdx = tdx_tdr_pa(td), + }; + + tdx_clflush_page(vp->tdvpr_page); + return seamcall(TDH_VP_CREATE, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_create); + +u64 tdh_mng_rd(struct tdx_td *td, u64 field, u64 *data) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + .rdx = field, + }; + u64 ret; + + ret = seamcall_ret(TDH_MNG_RD, &args); + + /* R8: Content of the field, or 0 in case of error. */ + *data = args.r8; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_rd); + +u64 tdh_mr_extend(struct tdx_td *td, u64 gpa, u64 *ext_err1, u64 *ext_err2) +{ + struct tdx_module_args args = { + .rcx = gpa, + .rdx = tdx_tdr_pa(td), + }; + u64 ret; + + ret = seamcall_ret(TDH_MR_EXTEND, &args); + + *ext_err1 = args.rcx; + *ext_err2 = args.rdx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mr_extend); + +u64 tdh_mr_finalize(struct tdx_td *td) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + }; + + return seamcall(TDH_MR_FINALIZE, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mr_finalize); + +u64 tdh_vp_flush(struct tdx_vp *vp) +{ + struct tdx_module_args args = { + .rcx = vp->tdvpr_pa, + }; + + return seamcall(TDH_VP_FLUSH, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_flush); + +u64 tdh_mng_vpflushdone(struct tdx_td *td) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + }; + + return seamcall(TDH_MNG_VPFLUSHDONE, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_vpflushdone); + +u64 tdh_mng_key_freeid(struct tdx_td *td) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + }; + + return seamcall(TDH_MNG_KEY_FREEID, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_key_freeid); + +u64 tdh_mng_init(struct tdx_td *td, u64 td_params, u64 *extended_err) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + .rdx = td_params, + }; + u64 ret; + + ret = seamcall_ret(TDH_MNG_INIT, &args); + + *extended_err = args.rcx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mng_init); + +u64 tdh_vp_rd(struct tdx_vp *vp, u64 field, u64 *data) +{ + struct tdx_module_args args = { + .rcx = vp->tdvpr_pa, + .rdx = field, + }; + u64 ret; + + ret = seamcall_ret(TDH_VP_RD, &args); + + /* R8: Content of the field, or 0 in case of error. */ + *data = args.r8; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_rd); + +u64 tdh_vp_wr(struct tdx_vp *vp, u64 field, u64 data, u64 mask) +{ + struct tdx_module_args args = { + .rcx = vp->tdvpr_pa, + .rdx = field, + .r8 = data, + .r9 = mask, + }; + + return seamcall(TDH_VP_WR, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_wr); + +u64 tdh_vp_init(struct tdx_vp *vp, u64 initial_rcx, u32 x2apicid) +{ + struct tdx_module_args args = { + .rcx = vp->tdvpr_pa, + .rdx = initial_rcx, + .r8 = x2apicid, + }; + + /* apicid requires version == 1. */ + return seamcall(TDH_VP_INIT | (1ULL << TDX_VERSION_SHIFT), &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_vp_init); + +/* + * TDX ABI defines output operands as PT, OWNER and SIZE. These are TDX defined fomats. + * So despite the names, they must be interpted specially as described by the spec. Return + * them only for error reporting purposes. + */ +u64 tdh_phymem_page_reclaim(struct page *page, u64 *tdx_pt, u64 *tdx_owner, u64 *tdx_size) +{ + struct tdx_module_args args = { + .rcx = page_to_phys(page), + }; + u64 ret; + + ret = seamcall_ret(TDH_PHYMEM_PAGE_RECLAIM, &args); + + *tdx_pt = args.rcx; + *tdx_owner = args.rdx; + *tdx_size = args.r8; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_reclaim); + +u64 tdh_mem_track(struct tdx_td *td) +{ + struct tdx_module_args args = { + .rcx = tdx_tdr_pa(td), + }; + + return seamcall(TDH_MEM_TRACK, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_mem_track); + +u64 tdh_mem_page_remove(struct tdx_td *td, u64 gpa, enum pg_level level, + u64 *ext_err1, u64 *ext_err2) +{ + struct tdx_module_args args = { + .rcx = gpa | pg_level_to_tdx_sept_level(level), + .rdx = tdx_tdr_pa(td), + }; + u64 ret; + + ret = seamcall_ret(TDH_MEM_PAGE_REMOVE, &args); + + *ext_err1 = args.rcx; + *ext_err2 = args.rdx; + + return ret; +} +EXPORT_SYMBOL_FOR_KVM(tdh_mem_page_remove); + +u64 tdh_phymem_cache_wb(bool resume) +{ + struct tdx_module_args args = { + .rcx = resume ? 1 : 0, + }; + + return seamcall(TDH_PHYMEM_CACHE_WB, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_cache_wb); + +static inline u64 mk_keyed_paddr(u16 hkid, kvm_pfn_t pfn) +{ + /* KeyID bits are just above the physical address bits. */ + return PFN_PHYS(pfn) | ((u64)hkid << boot_cpu_data.x86_phys_bits); +} + +u64 tdh_phymem_page_wbinvd_tdr(struct tdx_td *td) +{ + struct tdx_module_args args = {}; + + args.rcx = mk_keyed_paddr(tdx_global_keyid, page_to_pfn(td->tdr_page)); + + return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_tdr); + +u64 tdh_phymem_page_wbinvd_hkid(u64 hkid, kvm_pfn_t pfn) +{ + struct tdx_module_args args = {}; + + args.rcx = mk_keyed_paddr(hkid, pfn); + + return seamcall(TDH_PHYMEM_PAGE_WBINVD, &args); +} +EXPORT_SYMBOL_FOR_KVM(tdh_phymem_page_wbinvd_hkid); + +void tdx_sys_disable(void) +{ + struct tdx_module_args args = {}; + u64 ret; + + /* + * Don't loop forever. + * + * - TDX_INTERRUPTED_RESUMABLE guarantees forward progress between + * calls. + * + * - TDX_SYS_BUSY could be returned due to contention with other + * TDH.SYS.* SEAMCALLs, but will lock out *new* TDH.SYS.* SEAMCALLs, + * so that SYS.DISABLE can eventually make progress. + * + * This is a 'destructive' SEAMCALL, in that no other SEAMCALL can be + * run after this until a full reinitialization is done. + */ + do { + ret = seamcall(TDH_SYS_DISABLE, &args); + } while (ret == TDX_INTERRUPTED_RESUMABLE || ret == TDX_SYS_BUSY); + + /* + * Print SEAMCALL failures, but not SW-defined error codes + * (SEAMCALL faulted with #GP/#UD, TDX not supported). + */ + if (ret && (ret & TDX_SW_ERROR) != TDX_SW_ERROR) + pr_err("TDH.SYS.DISABLE failed: 0x%016llx\n", ret); +} diff --git a/arch/x86/virt/vmx/tdx/tdx.h b/arch/x86/virt/vmx/tdx/tdx.h new file mode 100644 index 000000000..bdfd0e1e3 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/tdx.h @@ -0,0 +1,116 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _X86_VIRT_TDX_H +#define _X86_VIRT_TDX_H + +#include <linux/bits.h> + +/* + * This file contains both macros and data structures defined by the TDX + * architecture and Linux defined software data structures and functions. + * The two should not be mixed together for better readability. The + * architectural definitions come first. + */ + +/* + * TDX module SEAMCALL leaf functions + */ +#define TDH_VP_ENTER 0 +#define TDH_MNG_ADDCX 1 +#define TDH_MEM_PAGE_ADD 2 +#define TDH_MEM_SEPT_ADD 3 +#define TDH_VP_ADDCX 4 +#define TDH_MEM_PAGE_AUG 6 +#define TDH_MEM_RANGE_BLOCK 7 +#define TDH_MNG_KEY_CONFIG 8 +#define TDH_MNG_CREATE 9 +#define TDH_MNG_RD 11 +#define TDH_MR_EXTEND 16 +#define TDH_MR_FINALIZE 17 +#define TDH_VP_FLUSH 18 +#define TDH_MNG_VPFLUSHDONE 19 +#define TDH_VP_CREATE 10 +#define TDH_MNG_KEY_FREEID 20 +#define TDH_MNG_INIT 21 +#define TDH_VP_INIT 22 +#define TDH_PHYMEM_PAGE_RDMD 24 +#define TDH_VP_RD 26 +#define TDH_PHYMEM_PAGE_RECLAIM 28 +#define TDH_MEM_PAGE_REMOVE 29 +#define TDH_SYS_KEY_CONFIG 31 +#define TDH_SYS_INIT 33 +#define TDH_SYS_RD 34 +#define TDH_SYS_LP_INIT 35 +#define TDH_SYS_TDMR_INIT 36 +#define TDH_MEM_TRACK 38 +#define TDH_PHYMEM_CACHE_WB 40 +#define TDH_PHYMEM_PAGE_WBINVD 41 +#define TDH_VP_WR 43 +#define TDH_SYS_CONFIG 45 +#define TDH_SYS_SHUTDOWN 52 +#define TDH_SYS_UPDATE 53 +#define TDH_SYS_DISABLE 69 + +/* + * SEAMCALL leaf: + * + * Bit 15:0 Leaf number + * Bit 23:16 Version number + */ +#define TDX_VERSION_SHIFT 16 + +/* TDX page types */ +#define PT_NDA 0x0 +#define PT_RSVD 0x1 + +struct tdmr_reserved_area { + u64 offset; + u64 size; +} __packed; + +#define TDMR_INFO_ALIGNMENT 512 +#define TDMR_INFO_PA_ARRAY_ALIGNMENT 512 + +struct tdmr_info { + u64 base; + u64 size; + u64 pamt_1g_base; + u64 pamt_1g_size; + u64 pamt_2m_base; + u64 pamt_2m_size; + u64 pamt_4k_base; + u64 pamt_4k_size; + /* + * The actual number of reserved areas depends on the value of + * field MD_FIELD_ID_MAX_RESERVED_PER_TDMR in the TDX module + * global metadata. + */ + DECLARE_FLEX_ARRAY(struct tdmr_reserved_area, reserved_areas); +} __packed __aligned(TDMR_INFO_ALIGNMENT); + +/* + * Do not put any hardware-defined TDX structure representations below + * this comment! + */ +struct tdx_memblock { + struct list_head list; + unsigned long start_pfn; + unsigned long end_pfn; + int nid; +}; + +/* Warn if kernel has less than TDMR_NR_WARN TDMRs after allocation */ +#define TDMR_NR_WARN 4 + +struct tdmr_info_list { + void *tdmrs; /* Flexible array to hold 'tdmr_info's */ + int nr_consumed_tdmrs; /* How many 'tdmr_info's are in use */ + + /* Metadata for finding target 'tdmr_info' and freeing @tdmrs */ + int tdmr_sz; /* Size of one 'tdmr_info' */ + int max_tdmrs; /* How many 'tdmr_info's are allocated */ +}; + +int tdx_module_shutdown(void); +int tdx_module_run_update(void); + +#endif diff --git a/arch/x86/virt/vmx/tdx/tdx_global_metadata.c b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c new file mode 100644 index 000000000..e49c300f2 --- /dev/null +++ b/arch/x86/virt/vmx/tdx/tdx_global_metadata.c @@ -0,0 +1,133 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Automatically generated functions to read TDX global metadata. + * + * This file doesn't compile on its own as it lacks of inclusion + * of SEAMCALL wrapper primitive which reads global metadata. + * Include this file to other C file instead. + */ + +static int get_tdx_sys_info_version(struct tdx_sys_info_version *sysinfo_version) +{ + int ret = 0; + u64 val; + + if (!ret && !(ret = read_sys_metadata_field(0x0800000100000003, &val))) + sysinfo_version->minor_version = val; + if (!ret && !(ret = read_sys_metadata_field(0x0800000100000004, &val))) + sysinfo_version->major_version = val; + if (!ret && !(ret = read_sys_metadata_field(0x0800000100000005, &val))) + sysinfo_version->update_version = val; + + return ret; +} + +static __init int get_tdx_sys_info_features(struct tdx_sys_info_features *sysinfo_features) +{ + int ret = 0; + u64 val; + + if (!ret && !(ret = read_sys_metadata_field(0x0A00000300000008, &val))) + sysinfo_features->tdx_features0 = val; + + return ret; +} + +static __init int get_tdx_sys_info_tdmr(struct tdx_sys_info_tdmr *sysinfo_tdmr) +{ + int ret = 0; + u64 val; + + if (!ret && !(ret = read_sys_metadata_field(0x9100000100000008, &val))) + sysinfo_tdmr->max_tdmrs = val; + if (!ret && !(ret = read_sys_metadata_field(0x9100000100000009, &val))) + sysinfo_tdmr->max_reserved_per_tdmr = val; + if (!ret && !(ret = read_sys_metadata_field(0x9100000100000010, &val))) + sysinfo_tdmr->pamt_4k_entry_size = val; + if (!ret && !(ret = read_sys_metadata_field(0x9100000100000011, &val))) + sysinfo_tdmr->pamt_2m_entry_size = val; + if (!ret && !(ret = read_sys_metadata_field(0x9100000100000012, &val))) + sysinfo_tdmr->pamt_1g_entry_size = val; + + return ret; +} + +static __init int get_tdx_sys_info_td_ctrl(struct tdx_sys_info_td_ctrl *sysinfo_td_ctrl) +{ + int ret = 0; + u64 val; + + if (!ret && !(ret = read_sys_metadata_field(0x9800000100000000, &val))) + sysinfo_td_ctrl->tdr_base_size = val; + if (!ret && !(ret = read_sys_metadata_field(0x9800000100000100, &val))) + sysinfo_td_ctrl->tdcs_base_size = val; + if (!ret && !(ret = read_sys_metadata_field(0x9800000100000200, &val))) + sysinfo_td_ctrl->tdvps_base_size = val; + + return ret; +} + +static __init int get_tdx_sys_info_td_conf(struct tdx_sys_info_td_conf *sysinfo_td_conf) +{ + int ret = 0; + u64 val; + int i, j; + + if (!ret && !(ret = read_sys_metadata_field(0x1900000300000000, &val))) + sysinfo_td_conf->attributes_fixed0 = val; + if (!ret && !(ret = read_sys_metadata_field(0x1900000300000001, &val))) + sysinfo_td_conf->attributes_fixed1 = val; + if (!ret && !(ret = read_sys_metadata_field(0x1900000300000002, &val))) + sysinfo_td_conf->xfam_fixed0 = val; + if (!ret && !(ret = read_sys_metadata_field(0x1900000300000003, &val))) + sysinfo_td_conf->xfam_fixed1 = val; + if (!ret && !(ret = read_sys_metadata_field(0x9900000100000004, &val))) + sysinfo_td_conf->num_cpuid_config = val; + if (!ret && !(ret = read_sys_metadata_field(0x9900000100000008, &val))) + sysinfo_td_conf->max_vcpus_per_td = val; + if (sysinfo_td_conf->num_cpuid_config > ARRAY_SIZE(sysinfo_td_conf->cpuid_config_leaves)) + return -EINVAL; + for (i = 0; i < sysinfo_td_conf->num_cpuid_config; i++) + if (!ret && !(ret = read_sys_metadata_field(0x9900000300000400 + i, &val))) + sysinfo_td_conf->cpuid_config_leaves[i] = val; + if (sysinfo_td_conf->num_cpuid_config > ARRAY_SIZE(sysinfo_td_conf->cpuid_config_values)) + return -EINVAL; + for (i = 0; i < sysinfo_td_conf->num_cpuid_config; i++) + for (j = 0; j < 2; j++) + if (!ret && !(ret = read_sys_metadata_field(0x9900000300000500 + i * 2 + j, &val))) + sysinfo_td_conf->cpuid_config_values[i][j] = val; + + return ret; +} + +static int get_tdx_sys_info_handoff(struct tdx_sys_info_handoff *sysinfo_handoff) +{ + int ret; + u64 val; + + ret = read_sys_metadata_field(0x8900000100000000, &val); + if (ret) + return ret; + + sysinfo_handoff->module_hv = val; + return 0; +} + +static __init int get_tdx_sys_info(struct tdx_sys_info *sysinfo) +{ + int ret = 0; + + ret = ret ?: get_tdx_sys_info_version(&sysinfo->version); + + pr_info("Module version: " TDX_VERSION_FMT "\n", + sysinfo->version.major_version, + sysinfo->version.minor_version, + sysinfo->version.update_version); + + ret = ret ?: get_tdx_sys_info_features(&sysinfo->features); + ret = ret ?: get_tdx_sys_info_tdmr(&sysinfo->tdmr); + ret = ret ?: get_tdx_sys_info_td_ctrl(&sysinfo->td_ctrl); + ret = ret ?: get_tdx_sys_info_td_conf(&sysinfo->td_conf); + + return ret; +} diff --git a/arch/x86/virt/vmx/tdx/tdxcall.S b/arch/x86/virt/vmx/tdx/tdxcall.S new file mode 100644 index 000000000..016a2a1ec --- /dev/null +++ b/arch/x86/virt/vmx/tdx/tdxcall.S @@ -0,0 +1,220 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#include <asm/asm-offsets.h> +#include <asm/frame.h> +#include <asm/asm.h> +#include <asm/tdx.h> + +/* + * TDCALL and SEAMCALL are supported in Binutils >= 2.36. + */ +#define tdcall .byte 0x66,0x0f,0x01,0xcc +#define seamcall .byte 0x66,0x0f,0x01,0xcf + +/* + * TDX_MODULE_CALL - common helper macro for both + * TDCALL and SEAMCALL instructions. + * + * TDCALL - used by TDX guests to make requests to the + * TDX module and hypercalls to the VMM. + * SEAMCALL - used by TDX hosts to make requests to the + * TDX module. + * + *------------------------------------------------------------------------- + * TDCALL/SEAMCALL ABI: + *------------------------------------------------------------------------- + * Input Registers: + * + * RAX - TDCALL/SEAMCALL Leaf number. + * RCX,RDX,RDI,RSI,RBX,R8-R15 - TDCALL/SEAMCALL Leaf specific input registers. + * + * Output Registers: + * + * RAX - TDCALL/SEAMCALL instruction error code. + * RCX,RDX,RDI,RSI,RBX,R8-R15 - TDCALL/SEAMCALL Leaf specific output registers. + * + *------------------------------------------------------------------------- + * + * So while the common core (RAX,RCX,RDX,R8-R11) fits nicely in the + * callee-clobbered registers and even leaves RDI,RSI free to act as a + * base pointer, some leafs (e.g., VP.ENTER) make a giant mess of things. + * + * For simplicity, assume that anything that needs the callee-saved regs + * also tramples on RDI,RSI. This isn't strictly true, see for example + * TDH.EXPORT.MEM. + */ +.macro TDX_MODULE_CALL host:req ret=0 saved=0 + FRAME_BEGIN + + /* Move Leaf ID to RAX */ + mov %rdi, %rax + + /* Move other input regs from 'struct tdx_module_args' */ + movq TDX_MODULE_rcx(%rsi), %rcx + movq TDX_MODULE_rdx(%rsi), %rdx + movq TDX_MODULE_r8(%rsi), %r8 + movq TDX_MODULE_r9(%rsi), %r9 + movq TDX_MODULE_r10(%rsi), %r10 + movq TDX_MODULE_r11(%rsi), %r11 + +.if \saved + /* + * Move additional input regs from the structure. For simplicity + * assume that anything needs the callee-saved regs also tramples + * on RDI/RSI (see VP.ENTER). + */ + /* Save those callee-saved GPRs as mandated by the x86_64 ABI */ + pushq %rbx + pushq %r12 + pushq %r13 + pushq %r14 + pushq %r15 + + movq TDX_MODULE_r12(%rsi), %r12 + movq TDX_MODULE_r13(%rsi), %r13 + movq TDX_MODULE_r14(%rsi), %r14 + movq TDX_MODULE_r15(%rsi), %r15 + movq TDX_MODULE_rbx(%rsi), %rbx + +.if \ret + /* Save the structure pointer as RSI is about to be clobbered */ + pushq %rsi +.endif + + movq TDX_MODULE_rdi(%rsi), %rdi + /* RSI needs to be done at last */ + movq TDX_MODULE_rsi(%rsi), %rsi +.endif /* \saved */ + +.if \host +.Lseamcall\@: + seamcall + /* + * SEAMCALL instruction is essentially a VMExit from VMX root + * mode to SEAM VMX root mode. VMfailInvalid (CF=1) indicates + * that the targeted SEAM firmware is not loaded or disabled, + * or P-SEAMLDR is busy with another SEAMCALL. %rax is not + * changed in this case. + * + * Set %rax to TDX_SEAMCALL_VMFAILINVALID for VMfailInvalid. + * This value will never be used as actual SEAMCALL error code as + * it is from the Reserved status code class. + */ + jc .Lseamcall_vmfailinvalid\@ +.else + tdcall +.endif + +.if \ret +.if \saved + /* + * Restore the structure from stack to save the output registers + * + * In case of VP.ENTER returns due to TDVMCALL, all registers are + * valid thus no register can be used as spare to restore the + * structure from the stack (see "TDH.VP.ENTER Output Operands + * Definition on TDCALL(TDG.VP.VMCALL) Following a TD Entry"). + * For this case, need to make one register as spare by saving it + * to the stack and then manually load the structure pointer to + * the spare register. + * + * Note for other TDCALLs/SEAMCALLs there are spare registers + * thus no need for such hack but just use this for all. + */ + pushq %rax /* save the TDCALL/SEAMCALL return code */ + movq 8(%rsp), %rax /* restore the structure pointer */ + movq %rsi, TDX_MODULE_rsi(%rax) /* save RSI */ + popq %rax /* restore the return code */ + popq %rsi /* pop the structure pointer */ + + /* Copy additional output regs to the structure */ + movq %r12, TDX_MODULE_r12(%rsi) + movq %r13, TDX_MODULE_r13(%rsi) + movq %r14, TDX_MODULE_r14(%rsi) + movq %r15, TDX_MODULE_r15(%rsi) + movq %rbx, TDX_MODULE_rbx(%rsi) + movq %rdi, TDX_MODULE_rdi(%rsi) +.endif /* \saved */ + + /* Copy output registers to the structure */ + movq %rcx, TDX_MODULE_rcx(%rsi) + movq %rdx, TDX_MODULE_rdx(%rsi) + movq %r8, TDX_MODULE_r8(%rsi) + movq %r9, TDX_MODULE_r9(%rsi) + movq %r10, TDX_MODULE_r10(%rsi) + movq %r11, TDX_MODULE_r11(%rsi) +.endif /* \ret */ + +.if \saved && \ret + /* + * Clear registers shared by guest for VP.VMCALL/VP.ENTER to prevent + * speculative use of guest's/VMM's values, including those are + * restored from the stack. + * + * See arch/x86/kvm/vmx/vmenter.S: + * + * In theory, a L1 cache miss when restoring register from stack + * could lead to speculative execution with guest's values. + * + * Note: RBP/RSP are not used as shared register. RSI has been + * restored already. + * + * XOR is cheap, thus unconditionally do for all leafs. + */ + xorl %ecx, %ecx + xorl %edx, %edx + xorl %r8d, %r8d + xorl %r9d, %r9d + xorl %r10d, %r10d + xorl %r11d, %r11d + xorl %r12d, %r12d + xorl %r13d, %r13d + xorl %r14d, %r14d + xorl %r15d, %r15d + xorl %ebx, %ebx + xorl %edi, %edi +.endif /* \ret && \host */ + +.if \host +.Lout\@: +.endif + +.if \saved + /* Restore callee-saved GPRs as mandated by the x86_64 ABI */ + popq %r15 + popq %r14 + popq %r13 + popq %r12 + popq %rbx +.endif /* \saved */ + + FRAME_END + RET + +.if \host +.Lseamcall_vmfailinvalid\@: + mov $TDX_SEAMCALL_VMFAILINVALID, %rax + jmp .Lseamcall_fail\@ + +.Lseamcall_trap\@: + /* + * SEAMCALL caused #GP or #UD. By reaching here RAX contains + * the trap number. Convert the trap number to the TDX error + * code by setting TDX_SW_ERROR to the high 32-bits of RAX. + * + * Note cannot OR TDX_SW_ERROR directly to RAX as OR instruction + * only accepts 32-bit immediate at most. + */ + movq $TDX_SW_ERROR, %rdi + orq %rdi, %rax + +.Lseamcall_fail\@: +.if \ret && \saved + /* pop the unused structure pointer back to RSI */ + popq %rsi +.endif + jmp .Lout\@ + + _ASM_EXTABLE_FAULT(.Lseamcall\@, .Lseamcall_trap\@) +.endif /* \host */ + +.endm |