diff options
Diffstat (limited to 'drivers/hv')
38 files changed, 24915 insertions, 0 deletions
diff --git a/drivers/hv/Kconfig b/drivers/hv/Kconfig new file mode 100644 index 000000000..aa11bcefd --- /dev/null +++ b/drivers/hv/Kconfig @@ -0,0 +1,113 @@ +# SPDX-License-Identifier: GPL-2.0 + +menu "Microsoft Hyper-V guest support" + +config HYPERV + bool "Microsoft Hyper-V core hypervisor support" + depends on (X86 && X86_LOCAL_APIC && HYPERVISOR_GUEST) \ + || (ARM64 && !CPU_BIG_ENDIAN) + select PARAVIRT + select X86_HV_CALLBACK_VECTOR if X86 + select OF_EARLY_FLATTREE if OF + select IRQ_MSI_LIB if X86 + help + Select this option to run Linux as a Hyper-V client operating + system. + +config HYPERV_VTL_MODE + bool "Enable Linux to boot in VTL context" + depends on (X86_64 && HAVE_STATIC_CALL) || ARM64 + depends on HYPERV + depends on SMP + default n + help + Virtual Secure Mode (VSM) is a set of hypervisor capabilities and + enlightenments offered to host and guest partitions which enables + the creation and management of new security boundaries within + operating system software. + + VSM achieves and maintains isolation through Virtual Trust Levels + (VTLs). Virtual Trust Levels are hierarchical, with higher levels + being more privileged than lower levels. VTL0 is the least privileged + level, and currently only other level supported is VTL2. + + Select this option to build a Linux kernel to run at a VTL other than + the normal VTL0, which currently is only VTL2. This option + initializes the kernel to run in VTL2, and adds the ability to boot + secondary CPUs directly into 64-bit context as required for VTLs other + than 0. A kernel built with this option must run at VTL2, and will + not run as a normal guest. + + If unsure, say N + +config HYPERV_TIMER + def_bool HYPERV && X86 + +config HYPERV_UTILS + tristate "Microsoft Hyper-V Utilities driver" + depends on HYPERV_VMBUS && CONNECTOR && NLS + depends on PTP_1588_CLOCK_OPTIONAL + help + Select this option to enable the Hyper-V Utilities. + +config HYPERV_BALLOON + tristate "Microsoft Hyper-V Balloon driver" + depends on HYPERV_VMBUS + select PAGE_REPORTING + help + Select this option to enable Hyper-V Balloon driver. + +config HYPERV_VMBUS + tristate "Microsoft Hyper-V VMBus driver" + depends on HYPERV + default HYPERV + select SYSFB if EFI && !HYPERV_VTL_MODE + help + Select this option to enable Hyper-V Vmbus driver. + +config MSHV_ROOT + tristate "Microsoft Hyper-V root partition support" + depends on HYPERV && (X86_64 || ARM64) + depends on !HYPERV_VTL_MODE + # The hypervisor interface operates on 4k pages. Enforcing it here + # simplifies many assumptions in the root partition code. + # e.g. When withdrawing memory, the hypervisor gives back 4k pages in + # no particular order, making it impossible to reassemble larger pages + depends on PAGE_SIZE_4KB + depends on HYPERV_VMBUS if HYPERV_VMBUS + select EVENTFD + select VIRT_XFER_TO_GUEST_WORK + select HMM_MIRROR + select MMU_NOTIFIER + default n + help + Select this option to enable support for booting and running as root + partition on Microsoft Hyper-V. + + If unsure, say N. + +config MSHV_VTL + tristate "Microsoft Hyper-V VTL driver" + depends on X86_64 && HYPERV_VTL_MODE + depends on HYPERV_VMBUS + # Mapping VTL0 memory to a userspace process in VTL2 is supported in OpenHCL. + # VTL2 for OpenHCL makes use of Huge Pages to improve performance on VMs, + # specially with large memory requirements. + depends on TRANSPARENT_HUGEPAGE + # MTRRs are controlled by VTL0, and are not specific to individual VTLs. + # Therefore, do not attempt to access or modify MTRRs here. + depends on !MTRR + select CPUMASK_OFFSTACK + select VIRT_XFER_TO_GUEST_WORK + default n + help + Select this option to enable Hyper-V VTL driver support. + This driver provides interfaces for Virtual Machine Manager (VMM) running in VTL2 + userspace to create VTLs and partitions, setup and manage VTL0 memory and + allow userspace to make direct hypercalls. This also allows to map VTL0's address + space to a usermode process in VTL2 and supports getting new VMBus messages and channel + events in VTL2. + + If unsure, say N. + +endmenu diff --git a/drivers/hv/Makefile b/drivers/hv/Makefile new file mode 100644 index 000000000..888a748cc --- /dev/null +++ b/drivers/hv/Makefile @@ -0,0 +1,27 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_HYPERV_VMBUS) += hv_vmbus.o +obj-$(CONFIG_HYPERV_UTILS) += hv_utils.o +obj-$(CONFIG_HYPERV_BALLOON) += hv_balloon.o +obj-$(CONFIG_MSHV_ROOT) += mshv_root.o +obj-$(CONFIG_MSHV_VTL) += mshv_vtl.o + +CFLAGS_hv_trace.o = -I$(src) +CFLAGS_hv_balloon.o = -I$(src) + +hv_vmbus-y := vmbus_drv.o \ + hv.o connection.o channel.o \ + channel_mgmt.o ring_buffer.o hv_trace.o +hv_vmbus-$(CONFIG_HYPERV_TESTING) += hv_debugfs.o +hv_utils-y := hv_util.o hv_kvp.o hv_snapshot.o hv_utils_transport.o +mshv_root-y := mshv_root_main.o mshv_synic.o mshv_eventfd.o mshv_irq.o \ + mshv_root_hv_call.o mshv_portid_table.o mshv_regions.o +mshv_root-$(CONFIG_DEBUG_FS) += mshv_debugfs.o +mshv_root-$(CONFIG_TRACEPOINTS) += mshv_trace.o +mshv_vtl-y := mshv_vtl_main.o + +# Code that must be built-in +obj-$(CONFIG_HYPERV) += hv_common.o +obj-$(subst m,y,$(CONFIG_MSHV_ROOT)) += hv_proc.o +ifneq ($(CONFIG_MSHV_ROOT)$(CONFIG_MSHV_VTL),) + obj-y += mshv_common.o +endif diff --git a/drivers/hv/channel.c b/drivers/hv/channel.c new file mode 100644 index 000000000..6821f2252 --- /dev/null +++ b/drivers/hv/channel.c @@ -0,0 +1,1298 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/wait.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/module.h> +#include <linux/hyperv.h> +#include <linux/uio.h> +#include <linux/interrupt.h> +#include <linux/set_memory.h> +#include <linux/export.h> +#include <asm/page.h> +#include <asm/mshyperv.h> + +#include "hyperv_vmbus.h" + +/* + * hv_gpadl_size - Return the real size of a gpadl, the size that Hyper-V uses + * + * For BUFFER gpadl, Hyper-V uses the exact same size as the guest does. + * + * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the header + * (because of the alignment requirement), however, the hypervisor only + * uses the first HV_HYP_PAGE_SIZE as the header, therefore leaving a + * (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. And since there are two rings in a + * ringbuffer, the total size for a RING gpadl that Hyper-V uses is the + * total size that the guest uses minus twice of the gap size. + */ +static inline u32 hv_gpadl_size(enum hv_gpadl_type type, u32 size) +{ + switch (type) { + case HV_GPADL_BUFFER: + return size; + case HV_GPADL_RING: + /* The size of a ringbuffer must be page-aligned */ + BUG_ON(size % PAGE_SIZE); + /* + * Two things to notice here: + * 1) We're processing two ring buffers as a unit + * 2) We're skipping any space larger than HV_HYP_PAGE_SIZE in + * the first guest-size page of each of the two ring buffers. + * So we effectively subtract out two guest-size pages, and add + * back two Hyper-V size pages. + */ + return size - 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE); + } + BUG(); + return 0; +} + +/* + * hv_ring_gpadl_send_hvpgoffset - Calculate the send offset (in unit of + * HV_HYP_PAGE) in a ring gpadl based on the + * offset in the guest + * + * @offset: the offset (in bytes) where the send ringbuffer starts in the + * virtual address space of the guest + */ +static inline u32 hv_ring_gpadl_send_hvpgoffset(u32 offset) +{ + + /* + * For RING gpadl, in each ring, the guest uses one PAGE_SIZE as the + * header (because of the alignment requirement), however, the + * hypervisor only uses the first HV_HYP_PAGE_SIZE as the header, + * therefore leaving a (PAGE_SIZE - HV_HYP_PAGE_SIZE) gap. + * + * And to calculate the effective send offset in gpadl, we need to + * substract this gap. + */ + return (offset - (PAGE_SIZE - HV_HYP_PAGE_SIZE)) >> HV_HYP_PAGE_SHIFT; +} + +/* + * hv_gpadl_hvpfn - Return the Hyper-V page PFN of the @i th Hyper-V page in + * the gpadl + * + * @type: the type of the gpadl + * @kbuffer: the pointer to the gpadl in the guest + * @size: the total size (in bytes) of the gpadl + * @send_offset: the offset (in bytes) where the send ringbuffer starts in the + * virtual address space of the guest + * @i: the index + */ +static inline u64 hv_gpadl_hvpfn(enum hv_gpadl_type type, void *kbuffer, + u32 size, u32 send_offset, int i) +{ + int send_idx = hv_ring_gpadl_send_hvpgoffset(send_offset); + unsigned long delta = 0UL; + + switch (type) { + case HV_GPADL_BUFFER: + break; + case HV_GPADL_RING: + if (i == 0) + delta = 0; + else if (i <= send_idx) + delta = PAGE_SIZE - HV_HYP_PAGE_SIZE; + else + delta = 2 * (PAGE_SIZE - HV_HYP_PAGE_SIZE); + break; + default: + BUG(); + break; + } + + return virt_to_hvpfn(kbuffer + delta + (HV_HYP_PAGE_SIZE * i)); +} + +/* + * vmbus_setevent- Trigger an event notification on the specified + * channel. + */ +void vmbus_setevent(struct vmbus_channel *channel) +{ + struct hv_monitor_page *monitorpage; + + trace_vmbus_setevent(channel); + + /* + * For channels marked as in "low latency" mode + * bypass the monitor page mechanism. + */ + if (channel->offermsg.monitor_allocated && !channel->low_latency) { + vmbus_send_interrupt(channel->offermsg.child_relid); + + /* Get the child to parent monitor page */ + monitorpage = vmbus_connection.monitor_pages[1]; + + sync_set_bit(channel->monitor_bit, + (unsigned long *)&monitorpage->trigger_group + [channel->monitor_grp].pending); + + } else { + vmbus_set_event(channel); + } +} +EXPORT_SYMBOL_GPL(vmbus_setevent); + +/* vmbus_free_ring - drop mapping of ring buffer */ +void vmbus_free_ring(struct vmbus_channel *channel) +{ + hv_ringbuffer_cleanup(&channel->outbound); + hv_ringbuffer_cleanup(&channel->inbound); + + if (channel->ringbuffer_page) { + /* In a CoCo VM leak the memory if it didn't get re-encrypted */ + if (!channel->ringbuffer_gpadlhandle.decrypted) + __free_pages(channel->ringbuffer_page, + get_order(channel->ringbuffer_pagecount + << PAGE_SHIFT)); + channel->ringbuffer_page = NULL; + } +} +EXPORT_SYMBOL_GPL(vmbus_free_ring); + +/* vmbus_alloc_ring - allocate and map pages for ring buffer */ +int vmbus_alloc_ring(struct vmbus_channel *newchannel, + u32 send_size, u32 recv_size) +{ + struct page *page; + int order; + + if (send_size % PAGE_SIZE || recv_size % PAGE_SIZE) + return -EINVAL; + + /* Allocate the ring buffer */ + order = get_order(send_size + recv_size); + page = alloc_pages_node(cpu_to_node(newchannel->target_cpu), + GFP_KERNEL|__GFP_ZERO, order); + + if (!page) + page = alloc_pages(GFP_KERNEL|__GFP_ZERO, order); + + if (!page) + return -ENOMEM; + + newchannel->ringbuffer_page = page; + newchannel->ringbuffer_pagecount = (send_size + recv_size) >> PAGE_SHIFT; + newchannel->ringbuffer_send_offset = send_size >> PAGE_SHIFT; + + return 0; +} +EXPORT_SYMBOL_GPL(vmbus_alloc_ring); + +/* Used for Hyper-V Socket: a guest client's connect() to the host */ +int vmbus_send_tl_connect_request(const guid_t *shv_guest_servie_id, + const guid_t *shv_host_servie_id) +{ + struct vmbus_channel_tl_connect_request conn_msg; + int ret; + + memset(&conn_msg, 0, sizeof(conn_msg)); + conn_msg.header.msgtype = CHANNELMSG_TL_CONNECT_REQUEST; + conn_msg.guest_endpoint_id = *shv_guest_servie_id; + conn_msg.host_service_id = *shv_host_servie_id; + + ret = vmbus_post_msg(&conn_msg, sizeof(conn_msg), true); + + trace_vmbus_send_tl_connect_request(&conn_msg, ret); + + return ret; +} +EXPORT_SYMBOL_GPL(vmbus_send_tl_connect_request); + +static int send_modifychannel_without_ack(struct vmbus_channel *channel, u32 target_vp) +{ + struct vmbus_channel_modifychannel msg; + int ret; + + memset(&msg, 0, sizeof(msg)); + msg.header.msgtype = CHANNELMSG_MODIFYCHANNEL; + msg.child_relid = channel->offermsg.child_relid; + msg.target_vp = target_vp; + + ret = vmbus_post_msg(&msg, sizeof(msg), true); + trace_vmbus_send_modifychannel(&msg, ret); + + return ret; +} + +static int send_modifychannel_with_ack(struct vmbus_channel *channel, u32 target_vp) +{ + struct vmbus_channel_modifychannel *msg; + struct vmbus_channel_msginfo *info; + unsigned long flags; + int ret; + + info = kzalloc(sizeof(struct vmbus_channel_msginfo) + + sizeof(struct vmbus_channel_modifychannel), + GFP_KERNEL); + if (!info) + return -ENOMEM; + + init_completion(&info->waitevent); + info->waiting_channel = channel; + + msg = (struct vmbus_channel_modifychannel *)info->msg; + msg->header.msgtype = CHANNELMSG_MODIFYCHANNEL; + msg->child_relid = channel->offermsg.child_relid; + msg->target_vp = target_vp; + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_add_tail(&info->msglistentry, &vmbus_connection.chn_msg_list); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + ret = vmbus_post_msg(msg, sizeof(*msg), true); + trace_vmbus_send_modifychannel(msg, ret); + if (ret != 0) { + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + goto free_info; + } + + /* + * Release channel_mutex; otherwise, vmbus_onoffer_rescind() could block on + * the mutex and be unable to signal the completion. + * + * See the caller target_cpu_store() for information about the usage of the + * mutex. + */ + mutex_unlock(&vmbus_connection.channel_mutex); + wait_for_completion(&info->waitevent); + mutex_lock(&vmbus_connection.channel_mutex); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (info->response.modify_response.status) + ret = -EAGAIN; + +free_info: + kfree(info); + return ret; +} + +/* + * Set/change the vCPU (@target_vp) the channel (@child_relid) will interrupt. + * + * CHANNELMSG_MODIFYCHANNEL messages are aynchronous. When VMbus version 5.3 + * or later is negotiated, Hyper-V always sends an ACK in response to such a + * message. For VMbus version 5.2 and earlier, it never sends an ACK. With- + * out an ACK, we can not know when the host will stop interrupting the "old" + * vCPU and start interrupting the "new" vCPU for the given channel. + * + * The CHANNELMSG_MODIFYCHANNEL message type is supported since VMBus version + * VERSION_WIN10_V4_1. + */ +int vmbus_send_modifychannel(struct vmbus_channel *channel, u32 target_vp) +{ + if (vmbus_proto_version >= VERSION_WIN10_V5_3) + return send_modifychannel_with_ack(channel, target_vp); + return send_modifychannel_without_ack(channel, target_vp); +} +EXPORT_SYMBOL_GPL(vmbus_send_modifychannel); + +/* + * create_gpadl_header - Creates a gpadl for the specified buffer + */ +static int create_gpadl_header(enum hv_gpadl_type type, void *kbuffer, + u32 size, u32 send_offset, + struct vmbus_channel_msginfo **msginfo) +{ + int i; + int pagecount; + struct vmbus_channel_gpadl_header *gpadl_header; + struct vmbus_channel_gpadl_body *gpadl_body; + struct vmbus_channel_msginfo *msgheader; + struct vmbus_channel_msginfo *msgbody = NULL; + u32 msgsize; + + int pfnsum, pfncount, pfnleft, pfncurr, pfnsize; + + pagecount = hv_gpadl_size(type, size) >> HV_HYP_PAGE_SHIFT; + + pfnsize = MAX_SIZE_CHANNEL_MESSAGE - + sizeof(struct vmbus_channel_gpadl_header) - + sizeof(struct gpa_range); + pfncount = umin(pagecount, pfnsize / sizeof(u64)); + + msgsize = sizeof(struct vmbus_channel_msginfo) + + sizeof(struct vmbus_channel_gpadl_header) + + sizeof(struct gpa_range) + pfncount * sizeof(u64); + msgheader = kzalloc(msgsize, GFP_KERNEL); + if (!msgheader) + return -ENOMEM; + + INIT_LIST_HEAD(&msgheader->submsglist); + msgheader->msgsize = msgsize; + + gpadl_header = (struct vmbus_channel_gpadl_header *) + msgheader->msg; + gpadl_header->rangecount = 1; + gpadl_header->range_buflen = sizeof(struct gpa_range) + + pagecount * sizeof(u64); + gpadl_header->range[0].byte_offset = 0; + gpadl_header->range[0].byte_count = hv_gpadl_size(type, size); + for (i = 0; i < pfncount; i++) + gpadl_header->range[0].pfn_array[i] = hv_gpadl_hvpfn( + type, kbuffer, size, send_offset, i); + *msginfo = msgheader; + + pfnsum = pfncount; + pfnleft = pagecount - pfncount; + + /* how many pfns can we fit in a body message */ + pfnsize = MAX_SIZE_CHANNEL_MESSAGE - + sizeof(struct vmbus_channel_gpadl_body); + pfncount = pfnsize / sizeof(u64); + + /* + * If pfnleft is zero, everything fits in the header and no body + * messages are needed + */ + while (pfnleft) { + pfncurr = umin(pfncount, pfnleft); + msgsize = sizeof(struct vmbus_channel_msginfo) + + sizeof(struct vmbus_channel_gpadl_body) + + pfncurr * sizeof(u64); + msgbody = kzalloc(msgsize, GFP_KERNEL); + + if (!msgbody) { + struct vmbus_channel_msginfo *pos = NULL; + struct vmbus_channel_msginfo *tmp = NULL; + /* + * Free up all the allocated messages. + */ + list_for_each_entry_safe(pos, tmp, + &msgheader->submsglist, + msglistentry) { + + list_del(&pos->msglistentry); + kfree(pos); + } + kfree(msgheader); + return -ENOMEM; + } + + msgbody->msgsize = msgsize; + gpadl_body = (struct vmbus_channel_gpadl_body *)msgbody->msg; + + /* + * Gpadl is u32 and we are using a pointer which could + * be 64-bit + * This is governed by the guest/host protocol and + * so the hypervisor guarantees that this is ok. + */ + for (i = 0; i < pfncurr; i++) + gpadl_body->pfn[i] = hv_gpadl_hvpfn(type, + kbuffer, size, send_offset, pfnsum + i); + + /* add to msg header */ + list_add_tail(&msgbody->msglistentry, &msgheader->submsglist); + pfnsum += pfncurr; + pfnleft -= pfncurr; + } + + return 0; +} + +static void vmbus_free_channel_msginfo(struct vmbus_channel_msginfo *msginfo) +{ + struct vmbus_channel_msginfo *submsginfo, *tmp; + + if (!msginfo) + return; + + list_for_each_entry_safe(submsginfo, tmp, &msginfo->submsglist, + msglistentry) { + kfree(submsginfo); + } + + kfree(msginfo); +} + +/* + * __vmbus_establish_gpadl - Establish a GPADL for a buffer or ringbuffer + * + * @channel: a channel + * @type: the type of the corresponding GPADL, only meaningful for the guest. + * @kbuffer: from kmalloc or vmalloc + * @size: page-size multiple + * @send_offset: the offset (in bytes) where the send ring buffer starts, + * should be 0 for BUFFER type gpadl + * @gpadl_handle: some funky thing + */ +static int __vmbus_establish_gpadl(struct vmbus_channel *channel, + enum hv_gpadl_type type, void *kbuffer, + u32 size, u32 send_offset, + struct vmbus_gpadl *gpadl) +{ + struct vmbus_channel_gpadl_header *gpadlmsg; + struct vmbus_channel_gpadl_body *gpadl_body; + struct vmbus_channel_msginfo *msginfo = NULL; + struct vmbus_channel_msginfo *submsginfo; + struct list_head *curr; + u32 next_gpadl_handle; + unsigned long flags; + int ret = 0; + + next_gpadl_handle = + (atomic_inc_return(&vmbus_connection.next_gpadl_handle) - 1); + + ret = create_gpadl_header(type, kbuffer, size, send_offset, &msginfo); + if (ret) { + gpadl->decrypted = false; + return ret; + } + + gpadl->decrypted = !((channel->co_external_memory && type == HV_GPADL_BUFFER) || + (channel->co_ring_buffer && type == HV_GPADL_RING)); + if (gpadl->decrypted) { + /* + * The "decrypted" flag being true assumes that set_memory_decrypted() succeeds. + * But if it fails, the encryption state of the memory is unknown. In that case, + * leave "decrypted" as true to ensure the memory is leaked instead of going back + * on the free list. + */ + ret = set_memory_decrypted((unsigned long)kbuffer, + PFN_UP(size)); + if (ret) { + dev_warn(&channel->device_obj->device, + "Failed to set host visibility for new GPADL %d.\n", + ret); + vmbus_free_channel_msginfo(msginfo); + return ret; + } + } + + init_completion(&msginfo->waitevent); + msginfo->waiting_channel = channel; + + gpadlmsg = (struct vmbus_channel_gpadl_header *)msginfo->msg; + gpadlmsg->header.msgtype = CHANNELMSG_GPADL_HEADER; + gpadlmsg->child_relid = channel->offermsg.child_relid; + gpadlmsg->gpadl = next_gpadl_handle; + + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_add_tail(&msginfo->msglistentry, + &vmbus_connection.chn_msg_list); + + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (channel->rescind) { + ret = -ENODEV; + goto cleanup; + } + + ret = vmbus_post_msg(gpadlmsg, msginfo->msgsize - + sizeof(*msginfo), true); + + trace_vmbus_establish_gpadl_header(gpadlmsg, ret); + + if (ret != 0) + goto cleanup; + + list_for_each(curr, &msginfo->submsglist) { + submsginfo = (struct vmbus_channel_msginfo *)curr; + gpadl_body = + (struct vmbus_channel_gpadl_body *)submsginfo->msg; + + gpadl_body->header.msgtype = + CHANNELMSG_GPADL_BODY; + gpadl_body->gpadl = next_gpadl_handle; + + ret = vmbus_post_msg(gpadl_body, + submsginfo->msgsize - sizeof(*submsginfo), + true); + + trace_vmbus_establish_gpadl_body(gpadl_body, ret); + + if (ret != 0) + goto cleanup; + + } + wait_for_completion(&msginfo->waitevent); + + if (msginfo->response.gpadl_created.creation_status != 0) { + pr_err("Failed to establish GPADL: err = 0x%x\n", + msginfo->response.gpadl_created.creation_status); + + ret = -EDQUOT; + goto cleanup; + } + + if (channel->rescind) { + ret = -ENODEV; + goto cleanup; + } + + /* At this point, we received the gpadl created msg */ + gpadl->gpadl_handle = gpadlmsg->gpadl; + gpadl->buffer = kbuffer; + gpadl->size = size; + + +cleanup: + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&msginfo->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + vmbus_free_channel_msginfo(msginfo); + + if (ret) { + /* + * If set_memory_encrypted() fails, the decrypted flag is + * left as true so the memory is leaked instead of being + * put back on the free list. + */ + if (gpadl->decrypted) { + if (!set_memory_encrypted((unsigned long)kbuffer, PFN_UP(size))) + gpadl->decrypted = false; + } + } + + return ret; +} + +/* + * vmbus_establish_gpadl - Establish a GPADL for the specified buffer + * + * @channel: a channel + * @kbuffer: from kmalloc or vmalloc + * @size: page-size multiple + * @gpadl_handle: some funky thing + */ +int vmbus_establish_gpadl(struct vmbus_channel *channel, void *kbuffer, + u32 size, struct vmbus_gpadl *gpadl) +{ + return __vmbus_establish_gpadl(channel, HV_GPADL_BUFFER, kbuffer, size, + 0U, gpadl); +} +EXPORT_SYMBOL_GPL(vmbus_establish_gpadl); + +/** + * request_arr_init - Allocates memory for the requestor array. Each slot + * keeps track of the next available slot in the array. Initially, each + * slot points to the next one (as in a Linked List). The last slot + * does not point to anything, so its value is U64_MAX by default. + * @size: The size of the array + */ +static u64 *request_arr_init(u32 size) +{ + int i; + u64 *req_arr; + + req_arr = kcalloc(size, sizeof(u64), GFP_KERNEL); + if (!req_arr) + return NULL; + + for (i = 0; i < size - 1; i++) + req_arr[i] = i + 1; + + /* Last slot (no more available slots) */ + req_arr[i] = U64_MAX; + + return req_arr; +} + +/* + * vmbus_alloc_requestor - Initializes @rqstor's fields. + * Index 0 is the first free slot + * @size: Size of the requestor array + */ +static int vmbus_alloc_requestor(struct vmbus_requestor *rqstor, u32 size) +{ + u64 *rqst_arr; + unsigned long *bitmap; + + rqst_arr = request_arr_init(size); + if (!rqst_arr) + return -ENOMEM; + + bitmap = bitmap_zalloc(size, GFP_KERNEL); + if (!bitmap) { + kfree(rqst_arr); + return -ENOMEM; + } + + rqstor->req_arr = rqst_arr; + rqstor->req_bitmap = bitmap; + rqstor->size = size; + rqstor->next_request_id = 0; + spin_lock_init(&rqstor->req_lock); + + return 0; +} + +/* + * vmbus_free_requestor - Frees memory allocated for @rqstor + * @rqstor: Pointer to the requestor struct + */ +static void vmbus_free_requestor(struct vmbus_requestor *rqstor) +{ + kfree(rqstor->req_arr); + bitmap_free(rqstor->req_bitmap); +} + +static int __vmbus_open(struct vmbus_channel *newchannel, + void *userdata, u32 userdatalen, + void (*onchannelcallback)(void *context), void *context) +{ + struct vmbus_channel_open_channel *open_msg; + struct vmbus_channel_msginfo *open_info = NULL; + struct page *page = newchannel->ringbuffer_page; + u32 send_pages, recv_pages; + unsigned long flags; + int err; + + if (userdatalen > MAX_USER_DEFINED_BYTES) + return -EINVAL; + + send_pages = newchannel->ringbuffer_send_offset; + recv_pages = newchannel->ringbuffer_pagecount - send_pages; + + if (newchannel->state != CHANNEL_OPEN_STATE) + return -EINVAL; + + /* Create and init requestor */ + if (newchannel->rqstor_size) { + if (vmbus_alloc_requestor(&newchannel->requestor, newchannel->rqstor_size)) + return -ENOMEM; + } + + newchannel->state = CHANNEL_OPENING_STATE; + newchannel->onchannel_callback = onchannelcallback; + newchannel->channel_callback_context = context; + + if (!newchannel->max_pkt_size) + newchannel->max_pkt_size = VMBUS_DEFAULT_MAX_PKT_SIZE; + + /* Establish the gpadl for the ring buffer */ + newchannel->ringbuffer_gpadlhandle.gpadl_handle = 0; + + err = __vmbus_establish_gpadl(newchannel, HV_GPADL_RING, + page_address(newchannel->ringbuffer_page), + (send_pages + recv_pages) << PAGE_SHIFT, + newchannel->ringbuffer_send_offset << PAGE_SHIFT, + &newchannel->ringbuffer_gpadlhandle); + if (err) + goto error_clean_ring; + + err = hv_ringbuffer_init(&newchannel->outbound, + page, send_pages, 0, newchannel->co_ring_buffer); + if (err) + goto error_free_gpadl; + + err = hv_ringbuffer_init(&newchannel->inbound, &page[send_pages], + recv_pages, newchannel->max_pkt_size, + newchannel->co_ring_buffer); + if (err) + goto error_free_gpadl; + + /* Create and init the channel open message */ + open_info = kzalloc(sizeof(*open_info) + + sizeof(struct vmbus_channel_open_channel), + GFP_KERNEL); + if (!open_info) { + err = -ENOMEM; + goto error_free_gpadl; + } + + init_completion(&open_info->waitevent); + open_info->waiting_channel = newchannel; + + open_msg = (struct vmbus_channel_open_channel *)open_info->msg; + open_msg->header.msgtype = CHANNELMSG_OPENCHANNEL; + open_msg->openid = newchannel->offermsg.child_relid; + open_msg->child_relid = newchannel->offermsg.child_relid; + open_msg->ringbuffer_gpadlhandle + = newchannel->ringbuffer_gpadlhandle.gpadl_handle; + /* + * The unit of ->downstream_ringbuffer_pageoffset is HV_HYP_PAGE and + * the unit of ->ringbuffer_send_offset (i.e. send_pages) is PAGE, so + * here we calculate it into HV_HYP_PAGE. + */ + open_msg->downstream_ringbuffer_pageoffset = + hv_ring_gpadl_send_hvpgoffset(send_pages << PAGE_SHIFT); + open_msg->target_vp = hv_cpu_number_to_vp_number(newchannel->target_cpu); + + if (userdatalen) + memcpy(open_msg->userdata, userdata, userdatalen); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_add_tail(&open_info->msglistentry, + &vmbus_connection.chn_msg_list); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (newchannel->rescind) { + err = -ENODEV; + goto error_clean_msglist; + } + + err = vmbus_post_msg(open_msg, + sizeof(struct vmbus_channel_open_channel), true); + + trace_vmbus_open(open_msg, err); + + if (err != 0) + goto error_clean_msglist; + + wait_for_completion(&open_info->waitevent); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&open_info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (newchannel->rescind) { + err = -ENODEV; + goto error_free_info; + } + + if (open_info->response.open_result.status) { + err = -EAGAIN; + goto error_free_info; + } + + newchannel->state = CHANNEL_OPENED_STATE; + kfree(open_info); + return 0; + +error_clean_msglist: + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&open_info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +error_free_info: + kfree(open_info); +error_free_gpadl: + vmbus_teardown_gpadl(newchannel, &newchannel->ringbuffer_gpadlhandle); +error_clean_ring: + hv_ringbuffer_cleanup(&newchannel->outbound); + hv_ringbuffer_cleanup(&newchannel->inbound); + vmbus_free_requestor(&newchannel->requestor); + newchannel->state = CHANNEL_OPEN_STATE; + return err; +} + +/* + * vmbus_connect_ring - Open the channel but reuse ring buffer + */ +int vmbus_connect_ring(struct vmbus_channel *newchannel, + void (*onchannelcallback)(void *context), void *context) +{ + return __vmbus_open(newchannel, NULL, 0, onchannelcallback, context); +} +EXPORT_SYMBOL_GPL(vmbus_connect_ring); + +/* + * vmbus_open - Open the specified channel. + */ +int vmbus_open(struct vmbus_channel *newchannel, + u32 send_ringbuffer_size, u32 recv_ringbuffer_size, + void *userdata, u32 userdatalen, + void (*onchannelcallback)(void *context), void *context) +{ + int err; + + err = vmbus_alloc_ring(newchannel, send_ringbuffer_size, + recv_ringbuffer_size); + if (err) + return err; + + err = __vmbus_open(newchannel, userdata, userdatalen, + onchannelcallback, context); + if (err) + vmbus_free_ring(newchannel); + + return err; +} +EXPORT_SYMBOL_GPL(vmbus_open); + +/* + * vmbus_teardown_gpadl -Teardown the specified GPADL handle + */ +int vmbus_teardown_gpadl(struct vmbus_channel *channel, struct vmbus_gpadl *gpadl) +{ + struct vmbus_channel_gpadl_teardown *msg; + struct vmbus_channel_msginfo *info; + unsigned long flags; + int ret; + + info = kzalloc(sizeof(*info) + + sizeof(struct vmbus_channel_gpadl_teardown), GFP_KERNEL); + if (!info) + return -ENOMEM; + + init_completion(&info->waitevent); + info->waiting_channel = channel; + + msg = (struct vmbus_channel_gpadl_teardown *)info->msg; + + msg->header.msgtype = CHANNELMSG_GPADL_TEARDOWN; + msg->child_relid = channel->offermsg.child_relid; + msg->gpadl = gpadl->gpadl_handle; + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_add_tail(&info->msglistentry, + &vmbus_connection.chn_msg_list); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + if (channel->rescind) + goto post_msg_err; + + ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_gpadl_teardown), + true); + + trace_vmbus_teardown_gpadl(msg, ret); + + if (ret) + goto post_msg_err; + + wait_for_completion(&info->waitevent); + + gpadl->gpadl_handle = 0; + +post_msg_err: + /* + * If the channel has been rescinded; + * we will be awakened by the rescind + * handler; set the error code to zero so we don't leak memory. + */ + if (channel->rescind) + ret = 0; + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&info->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + kfree(info); + + if (gpadl->decrypted) + ret = set_memory_encrypted((unsigned long)gpadl->buffer, + PFN_UP(gpadl->size)); + else + ret = 0; + if (ret) + pr_warn("Fail to set mem host visibility in GPADL teardown %d.\n", ret); + + gpadl->decrypted = ret; + + return ret; +} +EXPORT_SYMBOL_GPL(vmbus_teardown_gpadl); + +void vmbus_reset_channel_cb(struct vmbus_channel *channel) +{ + unsigned long flags; + + /* + * vmbus_on_event(), running in the per-channel tasklet, can race + * with vmbus_close_internal() in the case of SMP guest, e.g., when + * the former is accessing channel->inbound.ring_buffer, the latter + * could be freeing the ring_buffer pages, so here we must stop it + * first. + * + * vmbus_chan_sched() might call the netvsc driver callback function + * that ends up scheduling NAPI work that accesses the ring buffer. + * At this point, we have to ensure that any such work is completed + * and that the channel ring buffer is no longer being accessed, cf. + * the calls to napi_disable() in netvsc_device_remove(). + */ + tasklet_disable(&channel->callback_event); + + /* See the inline comments in vmbus_chan_sched(). */ + spin_lock_irqsave(&channel->sched_lock, flags); + channel->onchannel_callback = NULL; + spin_unlock_irqrestore(&channel->sched_lock, flags); + + channel->sc_creation_callback = NULL; + + /* Re-enable tasklet for use on re-open */ + tasklet_enable(&channel->callback_event); +} + +static int vmbus_close_internal(struct vmbus_channel *channel) +{ + struct vmbus_channel_close_channel *msg; + int ret; + + vmbus_reset_channel_cb(channel); + + /* + * In case a device driver's probe() fails (e.g., + * util_probe() -> vmbus_open() returns -ENOMEM) and the device is + * rescinded later (e.g., we dynamically disable an Integrated Service + * in Hyper-V Manager), the driver's remove() invokes vmbus_close(): + * here we should skip most of the below cleanup work. + */ + if (channel->state != CHANNEL_OPENED_STATE) + return -EINVAL; + + channel->state = CHANNEL_OPEN_STATE; + + /* Send a closing message */ + + msg = &channel->close_msg; + + msg->header.msgtype = CHANNELMSG_CLOSECHANNEL; + msg->child_relid = channel->offermsg.child_relid; + + ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_close_channel), + true); + + trace_vmbus_close_internal(msg, ret); + + if (ret) { + pr_err("Close failed: close post msg return is %d\n", ret); + /* + * If we failed to post the close msg, + * it is perhaps better to leak memory. + */ + } + + /* Tear down the gpadl for the channel's ring buffer */ + else if (channel->ringbuffer_gpadlhandle.gpadl_handle) { + ret = vmbus_teardown_gpadl(channel, &channel->ringbuffer_gpadlhandle); + if (ret) { + pr_err("Close failed: teardown gpadl return %d\n", ret); + /* + * If we failed to teardown gpadl, + * it is perhaps better to leak memory. + */ + } + } + + if (!ret) + vmbus_free_requestor(&channel->requestor); + + return ret; +} + +/* disconnect ring - close all channels */ +int vmbus_disconnect_ring(struct vmbus_channel *channel) +{ + struct vmbus_channel *cur_channel, *tmp; + int ret; + + if (channel->primary_channel != NULL) + return -EINVAL; + + list_for_each_entry_safe(cur_channel, tmp, &channel->sc_list, sc_list) { + if (cur_channel->rescind) + wait_for_completion(&cur_channel->rescind_event); + + mutex_lock(&vmbus_connection.channel_mutex); + if (vmbus_close_internal(cur_channel) == 0) { + vmbus_free_ring(cur_channel); + + if (cur_channel->rescind) + hv_process_channel_removal(cur_channel); + } + mutex_unlock(&vmbus_connection.channel_mutex); + } + + /* + * Now close the primary. + */ + mutex_lock(&vmbus_connection.channel_mutex); + ret = vmbus_close_internal(channel); + mutex_unlock(&vmbus_connection.channel_mutex); + + return ret; +} +EXPORT_SYMBOL_GPL(vmbus_disconnect_ring); + +/* + * vmbus_close - Close the specified channel + */ +void vmbus_close(struct vmbus_channel *channel) +{ + if (vmbus_disconnect_ring(channel) == 0) + vmbus_free_ring(channel); +} +EXPORT_SYMBOL_GPL(vmbus_close); + +/** + * vmbus_sendpacket_getid() - Send the specified buffer on the given channel + * @channel: Pointer to vmbus_channel structure + * @buffer: Pointer to the buffer you want to send the data from. + * @bufferlen: Maximum size of what the buffer holds. + * @requestid: Identifier of the request + * @trans_id: Identifier of the transaction associated to this request, if + * the send is successful; undefined, otherwise. + * @type: Type of packet that is being sent e.g. negotiate, time + * packet etc. + * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED + * + * Sends data in @buffer directly to Hyper-V via the vmbus. + * This will send the data unparsed to Hyper-V. + * + * Mainly used by Hyper-V drivers. + */ +int vmbus_sendpacket_getid(struct vmbus_channel *channel, void *buffer, + u32 bufferlen, u64 requestid, u64 *trans_id, + enum vmbus_packet_type type, u32 flags) +{ + struct vmpacket_descriptor desc; + u32 packetlen = sizeof(struct vmpacket_descriptor) + bufferlen; + u32 packetlen_aligned = ALIGN(packetlen, sizeof(u64)); + struct kvec bufferlist[3]; + u64 aligned_data = 0; + int num_vecs = ((bufferlen != 0) ? 3 : 1); + + + /* Setup the descriptor */ + desc.type = type; /* VmbusPacketTypeDataInBand; */ + desc.flags = flags; /* VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; */ + /* in 8-bytes granularity */ + desc.offset8 = sizeof(struct vmpacket_descriptor) >> 3; + desc.len8 = (u16)(packetlen_aligned >> 3); + desc.trans_id = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ + + bufferlist[0].iov_base = &desc; + bufferlist[0].iov_len = sizeof(struct vmpacket_descriptor); + bufferlist[1].iov_base = buffer; + bufferlist[1].iov_len = bufferlen; + bufferlist[2].iov_base = &aligned_data; + bufferlist[2].iov_len = (packetlen_aligned - packetlen); + + return hv_ringbuffer_write(channel, bufferlist, num_vecs, requestid, trans_id); +} +EXPORT_SYMBOL(vmbus_sendpacket_getid); + +/** + * vmbus_sendpacket() - Send the specified buffer on the given channel + * @channel: Pointer to vmbus_channel structure + * @buffer: Pointer to the buffer you want to send the data from. + * @bufferlen: Maximum size of what the buffer holds. + * @requestid: Identifier of the request + * @type: Type of packet that is being sent e.g. negotiate, time + * packet etc. + * @flags: 0 or VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED + * + * Sends data in @buffer directly to Hyper-V via the vmbus. + * This will send the data unparsed to Hyper-V. + * + * Mainly used by Hyper-V drivers. + */ +int vmbus_sendpacket(struct vmbus_channel *channel, void *buffer, + u32 bufferlen, u64 requestid, + enum vmbus_packet_type type, u32 flags) +{ + return vmbus_sendpacket_getid(channel, buffer, bufferlen, + requestid, NULL, type, flags); +} +EXPORT_SYMBOL(vmbus_sendpacket); + +/* + * vmbus_sendpacket_mpb_desc - Send one or more multi-page buffer packets + * using a GPADL Direct packet type. + * The desc argument must include space for the VMBus descriptor. The + * rangecount field must already be set. + */ +int vmbus_sendpacket_mpb_desc(struct vmbus_channel *channel, + struct vmbus_packet_mpb_array *desc, + u32 desc_size, + void *buffer, u32 bufferlen, u64 requestid) +{ + u32 packetlen; + u32 packetlen_aligned; + struct kvec bufferlist[3]; + u64 aligned_data = 0; + + packetlen = desc_size + bufferlen; + packetlen_aligned = ALIGN(packetlen, sizeof(u64)); + + /* Setup the descriptor */ + desc->type = VM_PKT_DATA_USING_GPA_DIRECT; + desc->flags = VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED; + desc->dataoffset8 = desc_size >> 3; /* in 8-bytes granularity */ + desc->length8 = (u16)(packetlen_aligned >> 3); + desc->transactionid = VMBUS_RQST_ERROR; /* will be updated in hv_ringbuffer_write() */ + desc->reserved = 0; + + bufferlist[0].iov_base = desc; + bufferlist[0].iov_len = desc_size; + bufferlist[1].iov_base = buffer; + bufferlist[1].iov_len = bufferlen; + bufferlist[2].iov_base = &aligned_data; + bufferlist[2].iov_len = (packetlen_aligned - packetlen); + + return hv_ringbuffer_write(channel, bufferlist, 3, requestid, NULL); +} +EXPORT_SYMBOL_GPL(vmbus_sendpacket_mpb_desc); + +/** + * __vmbus_recvpacket() - Retrieve the user packet on the specified channel + * @channel: Pointer to vmbus_channel structure + * @buffer: Pointer to the buffer you want to receive the data into. + * @bufferlen: Maximum size of what the buffer can hold. + * @buffer_actual_len: The actual size of the data after it was received. + * @requestid: Identifier of the request + * @raw: true means keep the vmpacket_descriptor header in the received data. + * + * Receives directly from the hyper-v vmbus and puts the data it received + * into Buffer. This will receive the data unparsed from hyper-v. + * + * Mainly used by Hyper-V drivers. + */ +static inline int +__vmbus_recvpacket(struct vmbus_channel *channel, void *buffer, + u32 bufferlen, u32 *buffer_actual_len, u64 *requestid, + bool raw) +{ + return hv_ringbuffer_read(channel, buffer, bufferlen, + buffer_actual_len, requestid, raw); + +} + +int vmbus_recvpacket(struct vmbus_channel *channel, void *buffer, + u32 bufferlen, u32 *buffer_actual_len, + u64 *requestid) +{ + return __vmbus_recvpacket(channel, buffer, bufferlen, + buffer_actual_len, requestid, false); +} +EXPORT_SYMBOL(vmbus_recvpacket); + +/* + * vmbus_recvpacket_raw - Retrieve the raw packet on the specified channel + */ +int vmbus_recvpacket_raw(struct vmbus_channel *channel, void *buffer, + u32 bufferlen, u32 *buffer_actual_len, + u64 *requestid) +{ + return __vmbus_recvpacket(channel, buffer, bufferlen, + buffer_actual_len, requestid, true); +} +EXPORT_SYMBOL_GPL(vmbus_recvpacket_raw); + +/* + * vmbus_next_request_id - Returns a new request id. It is also + * the index at which the guest memory address is stored. + * Uses a spin lock to avoid race conditions. + * @channel: Pointer to the VMbus channel struct + * @rqst_add: Guest memory address to be stored in the array + */ +u64 vmbus_next_request_id(struct vmbus_channel *channel, u64 rqst_addr) +{ + struct vmbus_requestor *rqstor = &channel->requestor; + unsigned long flags; + u64 current_id; + + /* Check rqstor has been initialized */ + if (!channel->rqstor_size) + return VMBUS_NO_RQSTOR; + + lock_requestor(channel, flags); + current_id = rqstor->next_request_id; + + /* Requestor array is full */ + if (current_id >= rqstor->size) { + unlock_requestor(channel, flags); + return VMBUS_RQST_ERROR; + } + + rqstor->next_request_id = rqstor->req_arr[current_id]; + rqstor->req_arr[current_id] = rqst_addr; + + /* The already held spin lock provides atomicity */ + bitmap_set(rqstor->req_bitmap, current_id, 1); + + unlock_requestor(channel, flags); + + /* + * Cannot return an ID of 0, which is reserved for an unsolicited + * message from Hyper-V; Hyper-V does not acknowledge (respond to) + * VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED requests with ID of + * 0 sent by the guest. + */ + return current_id + 1; +} +EXPORT_SYMBOL_GPL(vmbus_next_request_id); + +/* As in vmbus_request_addr_match() but without the requestor lock */ +u64 __vmbus_request_addr_match(struct vmbus_channel *channel, u64 trans_id, + u64 rqst_addr) +{ + struct vmbus_requestor *rqstor = &channel->requestor; + u64 req_addr; + + /* Check rqstor has been initialized */ + if (!channel->rqstor_size) + return VMBUS_NO_RQSTOR; + + /* Hyper-V can send an unsolicited message with ID of 0 */ + if (!trans_id) + return VMBUS_RQST_ERROR; + + /* Data corresponding to trans_id is stored at trans_id - 1 */ + trans_id--; + + /* Invalid trans_id */ + if (trans_id >= rqstor->size || !test_bit(trans_id, rqstor->req_bitmap)) + return VMBUS_RQST_ERROR; + + req_addr = rqstor->req_arr[trans_id]; + if (rqst_addr == VMBUS_RQST_ADDR_ANY || req_addr == rqst_addr) { + rqstor->req_arr[trans_id] = rqstor->next_request_id; + rqstor->next_request_id = trans_id; + + /* The already held spin lock provides atomicity */ + bitmap_clear(rqstor->req_bitmap, trans_id, 1); + } + + return req_addr; +} +EXPORT_SYMBOL_GPL(__vmbus_request_addr_match); + +/* + * vmbus_request_addr_match - Clears/removes @trans_id from the @channel's + * requestor, provided the memory address stored at @trans_id equals @rqst_addr + * (or provided @rqst_addr matches the sentinel value VMBUS_RQST_ADDR_ANY). + * + * Returns the memory address stored at @trans_id, or VMBUS_RQST_ERROR if + * @trans_id is not contained in the requestor. + * + * Acquires and releases the requestor spin lock. + */ +u64 vmbus_request_addr_match(struct vmbus_channel *channel, u64 trans_id, + u64 rqst_addr) +{ + unsigned long flags; + u64 req_addr; + + lock_requestor(channel, flags); + req_addr = __vmbus_request_addr_match(channel, trans_id, rqst_addr); + unlock_requestor(channel, flags); + + return req_addr; +} +EXPORT_SYMBOL_GPL(vmbus_request_addr_match); + +/* + * vmbus_request_addr - Returns the memory address stored at @trans_id + * in @rqstor. Uses a spin lock to avoid race conditions. + * @channel: Pointer to the VMbus channel struct + * @trans_id: Request id sent back from Hyper-V. Becomes the requestor's + * next request id. + */ +u64 vmbus_request_addr(struct vmbus_channel *channel, u64 trans_id) +{ + return vmbus_request_addr_match(channel, trans_id, VMBUS_RQST_ADDR_ANY); +} +EXPORT_SYMBOL_GPL(vmbus_request_addr); diff --git a/drivers/hv/channel_mgmt.c b/drivers/hv/channel_mgmt.c new file mode 100644 index 000000000..89d214dda --- /dev/null +++ b/drivers/hv/channel_mgmt.c @@ -0,0 +1,1688 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/interrupt.h> +#include <linux/sched.h> +#include <linux/wait.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/completion.h> +#include <linux/delay.h> +#include <linux/cpu.h> +#include <linux/hyperv.h> +#include <linux/export.h> +#include <asm/mshyperv.h> +#include <linux/sched/isolation.h> + +#include "hyperv_vmbus.h" + +static void init_vp_index(struct vmbus_channel *channel); + +const struct vmbus_device vmbus_devs[] = { + /* IDE */ + { .dev_type = HV_IDE, + HV_IDE_GUID, + .perf_device = true, + .allowed_in_isolated = false, + }, + + /* SCSI */ + { .dev_type = HV_SCSI, + HV_SCSI_GUID, + .perf_device = true, + .allowed_in_isolated = true, + }, + + /* Fibre Channel */ + { .dev_type = HV_FC, + HV_SYNTHFC_GUID, + .perf_device = true, + .allowed_in_isolated = false, + }, + + /* Synthetic NIC */ + { .dev_type = HV_NIC, + HV_NIC_GUID, + .perf_device = true, + .allowed_in_isolated = true, + }, + + /* Network Direct */ + { .dev_type = HV_ND, + HV_ND_GUID, + .perf_device = true, + .allowed_in_isolated = false, + }, + + /* PCIE */ + { .dev_type = HV_PCIE, + HV_PCIE_GUID, + .perf_device = false, + .allowed_in_isolated = true, + }, + + /* Synthetic Frame Buffer */ + { .dev_type = HV_FB, + HV_SYNTHVID_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* Synthetic Keyboard */ + { .dev_type = HV_KBD, + HV_KBD_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* Synthetic MOUSE */ + { .dev_type = HV_MOUSE, + HV_MOUSE_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* KVP */ + { .dev_type = HV_KVP, + HV_KVP_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* Time Synch */ + { .dev_type = HV_TS, + HV_TS_GUID, + .perf_device = false, + .allowed_in_isolated = true, + }, + + /* Heartbeat */ + { .dev_type = HV_HB, + HV_HEART_BEAT_GUID, + .perf_device = false, + .allowed_in_isolated = true, + }, + + /* Shutdown */ + { .dev_type = HV_SHUTDOWN, + HV_SHUTDOWN_GUID, + .perf_device = false, + .allowed_in_isolated = true, + }, + + /* File copy */ + /* fcopy always uses 16KB ring buffer size and is working well for last many years */ + { .pref_ring_size = 0x4000, + .dev_type = HV_FCOPY, + HV_FCOPY_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* Backup */ + { .dev_type = HV_BACKUP, + HV_VSS_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* Dynamic Memory */ + { .dev_type = HV_DM, + HV_DM_GUID, + .perf_device = false, + .allowed_in_isolated = false, + }, + + /* + * Unknown GUID + * 64 KB ring buffer + 4 KB header should be sufficient size for any Hyper-V device apart + * from HV_NIC and HV_SCSI. This case avoid the fallback for unknown devices to allocate + * much bigger (2 MB) of ring size. + */ + { .pref_ring_size = 0x11000, + .dev_type = HV_UNKNOWN, + .perf_device = false, + .allowed_in_isolated = false, + }, +}; +EXPORT_SYMBOL_GPL(vmbus_devs); + +static const struct { + guid_t guid; +} vmbus_unsupported_devs[] = { + { HV_AVMA1_GUID }, + { HV_AVMA2_GUID }, + { HV_RDV_GUID }, + { HV_IMC_GUID }, +}; + +/* + * The rescinded channel may be blocked waiting for a response from the host; + * take care of that. + */ +static void vmbus_rescind_cleanup(struct vmbus_channel *channel) +{ + struct vmbus_channel_msginfo *msginfo; + unsigned long flags; + + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + channel->rescind = true; + list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, + msglistentry) { + + if (msginfo->waiting_channel == channel) { + complete(&msginfo->waitevent); + break; + } + } + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +} + +static bool is_unsupported_vmbus_devs(const guid_t *guid) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(vmbus_unsupported_devs); i++) + if (guid_equal(guid, &vmbus_unsupported_devs[i].guid)) + return true; + return false; +} + +static u16 hv_get_dev_type(const struct vmbus_channel *channel) +{ + const guid_t *guid = &channel->offermsg.offer.if_type; + u16 i; + + if (is_hvsock_channel(channel) || is_unsupported_vmbus_devs(guid)) + return HV_UNKNOWN; + + for (i = HV_IDE; i < HV_UNKNOWN; i++) { + if (guid_equal(guid, &vmbus_devs[i].guid)) + return i; + } + pr_info("Unknown GUID: %pUl\n", guid); + return i; +} + +/** + * vmbus_prep_negotiate_resp() - Create default response for Negotiate message + * @icmsghdrp: Pointer to msg header structure + * @buf: Raw buffer channel data + * @buflen: Length of the raw buffer channel data. + * @fw_version: The framework versions we can support. + * @fw_vercnt: The size of @fw_version. + * @srv_version: The service versions we can support. + * @srv_vercnt: The size of @srv_version. + * @nego_fw_version: The selected framework version. + * @nego_srv_version: The selected service version. + * + * Note: Versions are given in decreasing order. + * + * Set up and fill in default negotiate response message. + * Mainly used by Hyper-V drivers. + */ +bool vmbus_prep_negotiate_resp(struct icmsg_hdr *icmsghdrp, u8 *buf, + u32 buflen, const int *fw_version, int fw_vercnt, + const int *srv_version, int srv_vercnt, + int *nego_fw_version, int *nego_srv_version) +{ + int icframe_major, icframe_minor; + int icmsg_major, icmsg_minor; + int fw_major, fw_minor; + int srv_major, srv_minor; + int i, j; + bool found_match = false; + struct icmsg_negotiate *negop; + + /* Check that there's enough space for icframe_vercnt, icmsg_vercnt */ + if (buflen < ICMSG_HDR + offsetof(struct icmsg_negotiate, reserved)) { + pr_err_ratelimited("Invalid icmsg negotiate\n"); + return false; + } + + icmsghdrp->icmsgsize = 0x10; + negop = (struct icmsg_negotiate *)&buf[ICMSG_HDR]; + + icframe_major = negop->icframe_vercnt; + icframe_minor = 0; + + icmsg_major = negop->icmsg_vercnt; + icmsg_minor = 0; + + /* Validate negop packet */ + if (icframe_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || + icmsg_major > IC_VERSION_NEGOTIATION_MAX_VER_COUNT || + ICMSG_NEGOTIATE_PKT_SIZE(icframe_major, icmsg_major) > buflen) { + pr_err_ratelimited("Invalid icmsg negotiate - icframe_major: %u, icmsg_major: %u\n", + icframe_major, icmsg_major); + goto fw_error; + } + + /* + * Select the framework version number we will + * support. + */ + + for (i = 0; i < fw_vercnt; i++) { + fw_major = (fw_version[i] >> 16); + fw_minor = (fw_version[i] & 0xFFFF); + + for (j = 0; j < negop->icframe_vercnt; j++) { + if ((negop->icversion_data[j].major == fw_major) && + (negop->icversion_data[j].minor == fw_minor)) { + icframe_major = negop->icversion_data[j].major; + icframe_minor = negop->icversion_data[j].minor; + found_match = true; + break; + } + } + + if (found_match) + break; + } + + if (!found_match) + goto fw_error; + + found_match = false; + + for (i = 0; i < srv_vercnt; i++) { + srv_major = (srv_version[i] >> 16); + srv_minor = (srv_version[i] & 0xFFFF); + + for (j = negop->icframe_vercnt; + (j < negop->icframe_vercnt + negop->icmsg_vercnt); + j++) { + + if ((negop->icversion_data[j].major == srv_major) && + (negop->icversion_data[j].minor == srv_minor)) { + + icmsg_major = negop->icversion_data[j].major; + icmsg_minor = negop->icversion_data[j].minor; + found_match = true; + break; + } + } + + if (found_match) + break; + } + + /* + * Respond with the framework and service + * version numbers we can support. + */ + +fw_error: + if (!found_match) { + negop->icframe_vercnt = 0; + negop->icmsg_vercnt = 0; + } else { + negop->icframe_vercnt = 1; + negop->icmsg_vercnt = 1; + } + + if (nego_fw_version) + *nego_fw_version = (icframe_major << 16) | icframe_minor; + + if (nego_srv_version) + *nego_srv_version = (icmsg_major << 16) | icmsg_minor; + + negop->icversion_data[0].major = icframe_major; + negop->icversion_data[0].minor = icframe_minor; + negop->icversion_data[1].major = icmsg_major; + negop->icversion_data[1].minor = icmsg_minor; + return found_match; +} +EXPORT_SYMBOL_GPL(vmbus_prep_negotiate_resp); + +/* + * alloc_channel - Allocate and initialize a vmbus channel object + */ +static struct vmbus_channel *alloc_channel(void) +{ + struct vmbus_channel *channel; + + channel = kzalloc_obj(*channel, GFP_ATOMIC); + if (!channel) + return NULL; + + spin_lock_init(&channel->sched_lock); + init_completion(&channel->rescind_event); + + INIT_LIST_HEAD(&channel->sc_list); + + tasklet_init(&channel->callback_event, + vmbus_on_event, (unsigned long)channel); + + hv_ringbuffer_pre_init(channel); + + return channel; +} + +/* + * free_channel - Release the resources used by the vmbus channel object + */ +static void free_channel(struct vmbus_channel *channel) +{ + tasklet_kill(&channel->callback_event); + vmbus_remove_channel_attr_group(channel); + + kobject_put(&channel->kobj); +} + +void vmbus_channel_map_relid(struct vmbus_channel *channel) +{ + u32 new_relid = channel->offermsg.child_relid; + + if (WARN_ON(new_relid >= MAX_CHANNEL_RELIDS)) + return; + + /* + * This function is always called in the tasklet for the connect CPU. + * So updating the relid hiwater mark does not need to be atomic. + */ + if (new_relid > READ_ONCE(vmbus_connection.relid_hiwater)) + WRITE_ONCE(vmbus_connection.relid_hiwater, new_relid); + + /* + * The mapping of the channel's relid is visible from the CPUs that + * execute vmbus_chan_sched() by the time that vmbus_chan_sched() will + * execute: + * + * (a) In the "normal (i.e., not resuming from hibernation)" path, + * the full barrier in virt_store_mb() guarantees that the store + * is propagated to all CPUs before the add_channel_work work + * is queued. In turn, add_channel_work is queued before the + * channel's ring buffer is allocated/initialized and the + * OPENCHANNEL message for the channel is sent in vmbus_open(). + * Hyper-V won't start sending the interrupts for the channel + * before the OPENCHANNEL message is acked. The memory barrier + * in vmbus_chan_sched() -> sync_test_and_clear_bit() ensures + * that vmbus_chan_sched() must find the channel's relid in + * recv_int_page before retrieving the channel pointer from the + * array of channels. + * + * (b) In the "resuming from hibernation" path, the virt_store_mb() + * guarantees that the store is propagated to all CPUs before + * the VMBus connection is marked as ready for the resume event + * (cf. check_ready_for_resume_event()). The interrupt handler + * of the VMBus driver and vmbus_chan_sched() can not run before + * vmbus_bus_resume() has completed execution (cf. resume_noirq). + */ + virt_store_mb(vmbus_connection.channels[new_relid], channel); +} + +void vmbus_channel_unmap_relid(struct vmbus_channel *channel) +{ + if (WARN_ON(channel->offermsg.child_relid >= MAX_CHANNEL_RELIDS)) + return; + WRITE_ONCE( + vmbus_connection.channels[channel->offermsg.child_relid], + NULL); +} + +static void vmbus_release_relid(u32 relid) +{ + struct vmbus_channel_relid_released msg; + int ret; + + memset(&msg, 0, sizeof(struct vmbus_channel_relid_released)); + msg.child_relid = relid; + msg.header.msgtype = CHANNELMSG_RELID_RELEASED; + ret = vmbus_post_msg(&msg, sizeof(struct vmbus_channel_relid_released), + true); + + trace_vmbus_release_relid(&msg, ret); +} + +void hv_process_channel_removal(struct vmbus_channel *channel) +{ + lockdep_assert_held(&vmbus_connection.channel_mutex); + BUG_ON(!channel->rescind); + + /* + * hv_process_channel_removal() could find INVALID_RELID only for + * hv_sock channels. See the inline comments in vmbus_onoffer(). + */ + WARN_ON(channel->offermsg.child_relid == INVALID_RELID && + !is_hvsock_channel(channel)); + + /* + * Upon suspend, an in-use hv_sock channel is removed from the array of + * channels and the relid is invalidated. After hibernation, when the + * user-space application destroys the channel, it's unnecessary and + * unsafe to remove the channel from the array of channels. See also + * the inline comments before the call of vmbus_release_relid() below. + */ + if (channel->offermsg.child_relid != INVALID_RELID) + vmbus_channel_unmap_relid(channel); + + if (channel->primary_channel == NULL) + list_del(&channel->listentry); + else + list_del(&channel->sc_list); + + /* + * If this is a "perf" channel, updates the hv_numa_map[] masks so that + * init_vp_index() can (re-)use the CPU. + */ + if (hv_is_perf_channel(channel)) + hv_clear_allocated_cpu(channel->target_cpu); + + /* + * Upon suspend, an in-use hv_sock channel is marked as "rescinded" and + * the relid is invalidated; after hibernation, when the user-space app + * destroys the channel, the relid is INVALID_RELID, and in this case + * it's unnecessary and unsafe to release the old relid, since the same + * relid can refer to a completely different channel now. + */ + if (channel->offermsg.child_relid != INVALID_RELID) + vmbus_release_relid(channel->offermsg.child_relid); + + free_channel(channel); +} + +void vmbus_free_channels(void) +{ + struct vmbus_channel *channel, *tmp; + + list_for_each_entry_safe(channel, tmp, &vmbus_connection.chn_list, + listentry) { + /* hv_process_channel_removal() needs this */ + channel->rescind = true; + + vmbus_device_unregister(channel->device_obj); + } +} + +/* Note: the function can run concurrently for primary/sub channels. */ +static void vmbus_add_channel_work(struct work_struct *work) +{ + struct vmbus_channel *newchannel = + container_of(work, struct vmbus_channel, add_channel_work); + struct vmbus_channel *primary_channel = newchannel->primary_channel; + int ret; + + /* + * This state is used to indicate a successful open + * so that when we do close the channel normally, we + * can cleanup properly. + */ + newchannel->state = CHANNEL_OPEN_STATE; + + if (primary_channel != NULL) { + /* newchannel is a sub-channel. */ + struct hv_device *dev = primary_channel->device_obj; + + if (vmbus_add_channel_kobj(dev, newchannel)) + goto err_deq_chan; + + if (primary_channel->sc_creation_callback != NULL) + primary_channel->sc_creation_callback(newchannel); + + newchannel->probe_done = true; + return; + } + + /* + * Start the process of binding the primary channel to the driver + */ + newchannel->device_obj = vmbus_device_create( + &newchannel->offermsg.offer.if_type, + &newchannel->offermsg.offer.if_instance, + newchannel); + if (!newchannel->device_obj) + goto err_deq_chan; + + newchannel->device_obj->device_id = newchannel->device_id; + /* + * Add the new device to the bus. This will kick off device-driver + * binding which eventually invokes the device driver's AddDevice() + * method. + * + * If vmbus_device_register() fails, the 'device_obj' is freed in + * vmbus_device_release() as called by device_unregister() in the + * error path of vmbus_device_register(). In the outside error + * path, there's no need to free it. + */ + ret = vmbus_device_register(newchannel->device_obj); + + if (ret != 0) { + pr_err("unable to add child device object (relid %d)\n", + newchannel->offermsg.child_relid); + goto err_deq_chan; + } + + newchannel->probe_done = true; + return; + +err_deq_chan: + mutex_lock(&vmbus_connection.channel_mutex); + + /* + * We need to set the flag, otherwise + * vmbus_onoffer_rescind() can be blocked. + */ + newchannel->probe_done = true; + + if (primary_channel == NULL) + list_del(&newchannel->listentry); + else + list_del(&newchannel->sc_list); + + /* vmbus_process_offer() has mapped the channel. */ + vmbus_channel_unmap_relid(newchannel); + + mutex_unlock(&vmbus_connection.channel_mutex); + + vmbus_release_relid(newchannel->offermsg.child_relid); + + free_channel(newchannel); +} + +/* + * vmbus_process_offer - Process the offer by creating a channel/device + * associated with this offer + */ +static void vmbus_process_offer(struct vmbus_channel *newchannel) +{ + struct vmbus_channel *channel; + struct workqueue_struct *wq; + bool fnew = true; + + /* + * Synchronize vmbus_process_offer() and CPU hotplugging: + * + * CPU1 CPU2 + * + * [vmbus_process_offer()] [Hot removal of the CPU] + * + * CPU_READ_LOCK CPUS_WRITE_LOCK + * LOAD cpu_online_mask SEARCH chn_list + * STORE target_cpu LOAD target_cpu + * INSERT chn_list STORE cpu_online_mask + * CPUS_READ_UNLOCK CPUS_WRITE_UNLOCK + * + * Forbids: CPU1's LOAD from *not* seing CPU2's STORE && + * CPU2's SEARCH from *not* seeing CPU1's INSERT + * + * Forbids: CPU2's SEARCH from seeing CPU1's INSERT && + * CPU2's LOAD from *not* seing CPU1's STORE + */ + cpus_read_lock(); + + /* + * Serializes the modifications of the chn_list list as well as + * the accesses to next_numa_node_id in init_vp_index(). + */ + mutex_lock(&vmbus_connection.channel_mutex); + + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (guid_equal(&channel->offermsg.offer.if_type, + &newchannel->offermsg.offer.if_type) && + guid_equal(&channel->offermsg.offer.if_instance, + &newchannel->offermsg.offer.if_instance)) { + fnew = false; + newchannel->primary_channel = channel; + break; + } + } + + init_vp_index(newchannel); + + /* Remember the channels that should be cleaned up upon suspend. */ + if (is_hvsock_channel(newchannel) || is_sub_channel(newchannel)) + atomic_inc(&vmbus_connection.nr_chan_close_on_suspend); + + /* + * Now that we have acquired the channel_mutex, + * we can release the potentially racing rescind thread. + */ + atomic_dec(&vmbus_connection.offer_in_progress); + + if (fnew) { + list_add_tail(&newchannel->listentry, + &vmbus_connection.chn_list); + } else { + /* + * Check to see if this is a valid sub-channel. + */ + if (newchannel->offermsg.offer.sub_channel_index == 0) { + mutex_unlock(&vmbus_connection.channel_mutex); + cpus_read_unlock(); + /* + * Don't call free_channel(), because newchannel->kobj + * is not initialized yet. + */ + kfree(newchannel); + WARN_ON_ONCE(1); + return; + } + /* + * Process the sub-channel. + */ + list_add_tail(&newchannel->sc_list, &channel->sc_list); + } + + vmbus_channel_map_relid(newchannel); + + mutex_unlock(&vmbus_connection.channel_mutex); + cpus_read_unlock(); + + /* + * vmbus_process_offer() mustn't call channel->sc_creation_callback() + * directly for sub-channels, because sc_creation_callback() -> + * vmbus_open() may never get the host's response to the + * OPEN_CHANNEL message (the host may rescind a channel at any time, + * e.g. in the case of hot removing a NIC), and vmbus_onoffer_rescind() + * may not wake up the vmbus_open() as it's blocked due to a non-zero + * vmbus_connection.offer_in_progress, and finally we have a deadlock. + * + * The above is also true for primary channels, if the related device + * drivers use sync probing mode by default. + * + * And, usually the handling of primary channels and sub-channels can + * depend on each other, so we should offload them to different + * workqueues to avoid possible deadlock, e.g. in sync-probing mode, + * NIC1's netvsc_subchan_work() can race with NIC2's netvsc_probe() -> + * rtnl_lock(), and causes deadlock: the former gets the rtnl_lock + * and waits for all the sub-channels to appear, but the latter + * can't get the rtnl_lock and this blocks the handling of + * sub-channels. + */ + INIT_WORK(&newchannel->add_channel_work, vmbus_add_channel_work); + wq = fnew ? vmbus_connection.handle_primary_chan_wq : + vmbus_connection.handle_sub_chan_wq; + queue_work(wq, &newchannel->add_channel_work); +} + +/* + * Check if CPUs used by other channels of the same device. + * It should only be called by init_vp_index(). + */ +static bool hv_cpuself_used(u32 cpu, struct vmbus_channel *chn) +{ + struct vmbus_channel *primary = chn->primary_channel; + struct vmbus_channel *sc; + + lockdep_assert_held(&vmbus_connection.channel_mutex); + + if (!primary) + return false; + + if (primary->target_cpu == cpu) + return true; + + list_for_each_entry(sc, &primary->sc_list, sc_list) + if (sc != chn && sc->target_cpu == cpu) + return true; + + return false; +} + +/* + * We use this state to statically distribute the channel interrupt load. + */ +static int next_numa_node_id; + +/* + * We can statically distribute the incoming channel interrupt load + * by binding a channel to VCPU. + * + * For non-performance critical channels we assign the VMBUS_CONNECT_CPU. + * Performance critical channels will be distributed evenly among all + * the available NUMA nodes. Once the node is assigned, we will assign + * the CPU based on a simple round robin scheme. + */ +static void init_vp_index(struct vmbus_channel *channel) +{ + bool perf_chn = hv_is_perf_channel(channel); + u32 i, ncpu = num_online_cpus(); + cpumask_var_t available_mask; + struct cpumask *allocated_mask; + const struct cpumask *hk_mask = housekeeping_cpumask(HK_TYPE_MANAGED_IRQ); + u32 target_cpu; + int numa_node; + + if (!perf_chn || + !alloc_cpumask_var(&available_mask, GFP_KERNEL) || + cpumask_empty(hk_mask)) { + /* + * If the channel is not a performance critical + * channel, bind it to VMBUS_CONNECT_CPU. + * In case alloc_cpumask_var() fails, bind it to + * VMBUS_CONNECT_CPU. + * If all the cpus are isolated, bind it to + * VMBUS_CONNECT_CPU. + */ + channel->target_cpu = VMBUS_CONNECT_CPU; + if (perf_chn) + hv_set_allocated_cpu(VMBUS_CONNECT_CPU); + return; + } + + for (i = 1; i <= ncpu + 1; i++) { + while (true) { + numa_node = next_numa_node_id++; + if (numa_node == nr_node_ids) { + next_numa_node_id = 0; + continue; + } + if (cpumask_empty(cpumask_of_node(numa_node))) + continue; + break; + } + allocated_mask = &hv_context.hv_numa_map[numa_node]; + +retry: + cpumask_xor(available_mask, allocated_mask, cpumask_of_node(numa_node)); + cpumask_and(available_mask, available_mask, hk_mask); + + if (cpumask_empty(available_mask)) { + /* + * We have cycled through all the CPUs in the node; + * reset the allocated map. + */ + cpumask_clear(allocated_mask); + goto retry; + } + + target_cpu = cpumask_first(available_mask); + cpumask_set_cpu(target_cpu, allocated_mask); + + if (channel->offermsg.offer.sub_channel_index >= ncpu || + i > ncpu || !hv_cpuself_used(target_cpu, channel)) + break; + } + + channel->target_cpu = target_cpu; + + free_cpumask_var(available_mask); +} + +#define UNLOAD_DELAY_UNIT_MS 10 /* 10 milliseconds */ +#define UNLOAD_WAIT_MS (100*1000) /* 100 seconds */ +#define UNLOAD_WAIT_LOOPS (UNLOAD_WAIT_MS/UNLOAD_DELAY_UNIT_MS) +#define UNLOAD_MSG_MS (5*1000) /* Every 5 seconds */ +#define UNLOAD_MSG_LOOPS (UNLOAD_MSG_MS/UNLOAD_DELAY_UNIT_MS) + +static void vmbus_wait_for_unload(void) +{ + int cpu; + void *page_addr; + struct hv_message *msg; + struct vmbus_channel_message_header *hdr; + u32 message_type, i; + + /* + * CHANNELMSG_UNLOAD_RESPONSE is always delivered to the CPU which was + * used for initial contact or to CPU0 depending on host version. When + * we're crashing on a different CPU let's hope that IRQ handler on + * the cpu which receives CHANNELMSG_UNLOAD_RESPONSE is still + * functional and vmbus_unload_response() will complete + * vmbus_connection.unload_event. If not, the last thing we can do is + * read message pages for all CPUs directly. + * + * Wait up to 100 seconds since an Azure host must writeback any dirty + * data in its disk cache before the VMbus UNLOAD request will + * complete. This flushing has been empirically observed to take up + * to 50 seconds in cases with a lot of dirty data, so allow additional + * leeway and for inaccuracies in mdelay(). But eventually time out so + * that the panic path can't get hung forever in case the response + * message isn't seen. + */ + for (i = 1; i <= UNLOAD_WAIT_LOOPS; i++) { + if (completion_done(&vmbus_connection.unload_event)) + goto completed; + + for_each_present_cpu(cpu) { + struct hv_per_cpu_context *hv_cpu + = per_cpu_ptr(hv_context.cpu_context, cpu); + + /* + * In a CoCo VM the hyp_synic_message_page is not allocated + * in hv_synic_alloc(). Instead it is set/cleared in + * hv_hyp_synic_enable_regs() and hv_hyp_synic_disable_regs() + * such that it is set only when the CPU is online. If + * not all present CPUs are online, the message page + * might be NULL, so skip such CPUs. + */ + page_addr = hv_cpu->hyp_synic_message_page; + if (!page_addr) + continue; + + msg = (struct hv_message *)page_addr + + VMBUS_MESSAGE_SINT; + + message_type = READ_ONCE(msg->header.message_type); + if (message_type == HVMSG_NONE) + continue; + + hdr = (struct vmbus_channel_message_header *) + msg->u.payload; + + if (hdr->msgtype == CHANNELMSG_UNLOAD_RESPONSE) + complete(&vmbus_connection.unload_event); + + vmbus_signal_eom(msg, message_type); + } + + /* + * Give a notice periodically so someone watching the + * serial output won't think it is completely hung. + */ + if (!(i % UNLOAD_MSG_LOOPS)) + pr_notice("Waiting for VMBus UNLOAD to complete\n"); + + mdelay(UNLOAD_DELAY_UNIT_MS); + } + pr_err("Continuing even though VMBus UNLOAD did not complete\n"); + +completed: + /* + * We're crashing and already got the UNLOAD_RESPONSE, cleanup all + * maybe-pending messages on all CPUs to be able to receive new + * messages after we reconnect. + */ + for_each_present_cpu(cpu) { + struct hv_per_cpu_context *hv_cpu + = per_cpu_ptr(hv_context.cpu_context, cpu); + + page_addr = hv_cpu->hyp_synic_message_page; + if (!page_addr) + continue; + + msg = (struct hv_message *)page_addr + VMBUS_MESSAGE_SINT; + msg->header.message_type = HVMSG_NONE; + } +} + +/* + * vmbus_unload_response - Handler for the unload response. + */ +static void vmbus_unload_response(struct vmbus_channel_message_header *hdr) +{ + /* + * This is a global event; just wakeup the waiting thread. + * Once we successfully unload, we can cleanup the monitor state. + * + * NB. A malicious or compromised Hyper-V could send a spurious + * message of type CHANNELMSG_UNLOAD_RESPONSE, and trigger a call + * of the complete() below. Make sure that unload_event has been + * initialized by the time this complete() is executed. + */ + complete(&vmbus_connection.unload_event); +} + +void vmbus_initiate_unload(bool crash) +{ + struct vmbus_channel_message_header hdr; + + if (xchg(&vmbus_connection.conn_state, DISCONNECTED) == DISCONNECTED) + return; + + /* Pre-Win2012R2 hosts don't support reconnect */ + if (vmbus_proto_version < VERSION_WIN8_1) + return; + + reinit_completion(&vmbus_connection.unload_event); + memset(&hdr, 0, sizeof(struct vmbus_channel_message_header)); + hdr.msgtype = CHANNELMSG_UNLOAD; + vmbus_post_msg(&hdr, sizeof(struct vmbus_channel_message_header), + !crash); + + /* + * vmbus_initiate_unload() is also called on crash and the crash can be + * happening in an interrupt context, where scheduling is impossible. + */ + if (!crash) + wait_for_completion(&vmbus_connection.unload_event); + else + vmbus_wait_for_unload(); +} +EXPORT_SYMBOL_GPL(vmbus_initiate_unload); + +static void vmbus_setup_channel_state(struct vmbus_channel *channel, + struct vmbus_channel_offer_channel *offer) +{ + /* + * Setup state for signalling the host. + */ + channel->sig_event = VMBUS_EVENT_CONNECTION_ID; + + channel->is_dedicated_interrupt = + (offer->is_dedicated_interrupt != 0); + channel->sig_event = offer->connection_id; + + memcpy(&channel->offermsg, offer, + sizeof(struct vmbus_channel_offer_channel)); + channel->monitor_grp = (u8)offer->monitorid / 32; + channel->monitor_bit = (u8)offer->monitorid % 32; + channel->device_id = hv_get_dev_type(channel); +} + +/* + * find_primary_channel_by_offer - Get the channel object given the new offer. + * This is only used in the resume path of hibernation. + */ +static struct vmbus_channel * +find_primary_channel_by_offer(const struct vmbus_channel_offer_channel *offer) +{ + struct vmbus_channel *channel = NULL, *iter; + const guid_t *inst1, *inst2; + + /* Ignore sub-channel offers. */ + if (offer->offer.sub_channel_index != 0) + return NULL; + + mutex_lock(&vmbus_connection.channel_mutex); + + list_for_each_entry(iter, &vmbus_connection.chn_list, listentry) { + inst1 = &iter->offermsg.offer.if_instance; + inst2 = &offer->offer.if_instance; + + if (guid_equal(inst1, inst2)) { + channel = iter; + break; + } + } + + mutex_unlock(&vmbus_connection.channel_mutex); + + return channel; +} + +static bool vmbus_is_valid_offer(const struct vmbus_channel_offer_channel *offer) +{ + const guid_t *guid = &offer->offer.if_type; + u16 i; + + if (!hv_is_isolation_supported()) + return true; + + if (is_hvsock_offer(offer)) + return true; + + for (i = 0; i < ARRAY_SIZE(vmbus_devs); i++) { + if (guid_equal(guid, &vmbus_devs[i].guid)) + return vmbus_devs[i].allowed_in_isolated; + } + return false; +} + +/* + * vmbus_onoffer - Handler for channel offers from vmbus in parent partition. + * + */ +static void vmbus_onoffer(struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_offer_channel *offer; + struct vmbus_channel *oldchannel, *newchannel; + size_t offer_sz; + bool co_ring_buffer, co_external_memory; + + offer = (struct vmbus_channel_offer_channel *)hdr; + + trace_vmbus_onoffer(offer); + + if (!vmbus_is_valid_offer(offer)) { + pr_err_ratelimited("Invalid offer %d from the host supporting isolation\n", + offer->child_relid); + atomic_dec(&vmbus_connection.offer_in_progress); + return; + } + + co_ring_buffer = is_co_ring_buffer(offer); + co_external_memory = is_co_external_memory(offer); + if (!co_ring_buffer && co_external_memory) { + pr_err("Invalid offer relid=%d: the ring buffer isn't encrypted\n", + offer->child_relid); + return; + } + if (co_ring_buffer || co_external_memory) { + if (vmbus_proto_version < VERSION_WIN10_V6_0 || !vmbus_is_confidential()) { + pr_err("Invalid offer relid=%d: no support for confidential VMBus\n", + offer->child_relid); + atomic_dec(&vmbus_connection.offer_in_progress); + return; + } + } + + oldchannel = find_primary_channel_by_offer(offer); + + if (oldchannel != NULL) { + /* + * We're resuming from hibernation: all the sub-channel and + * hv_sock channels we had before the hibernation should have + * been cleaned up, and now we must be seeing a re-offered + * primary channel that we had before the hibernation. + */ + + /* + * { Initially: channel relid = INVALID_RELID, + * channels[valid_relid] = NULL } + * + * CPU1 CPU2 + * + * [vmbus_onoffer()] [vmbus_device_release()] + * + * LOCK channel_mutex LOCK channel_mutex + * STORE channel relid = valid_relid LOAD r1 = channel relid + * MAP_RELID channel if (r1 != INVALID_RELID) + * UNLOCK channel_mutex UNMAP_RELID channel + * UNLOCK channel_mutex + * + * Forbids: r1 == valid_relid && + * channels[valid_relid] == channel + * + * Note. r1 can be INVALID_RELID only for an hv_sock channel. + * None of the hv_sock channels which were present before the + * suspend are re-offered upon the resume. See the WARN_ON() + * in hv_process_channel_removal(). + */ + mutex_lock(&vmbus_connection.channel_mutex); + + atomic_dec(&vmbus_connection.offer_in_progress); + + WARN_ON(oldchannel->offermsg.child_relid != INVALID_RELID); + /* Fix up the relid. */ + oldchannel->offermsg.child_relid = offer->child_relid; + + offer_sz = sizeof(*offer); + if (memcmp(offer, &oldchannel->offermsg, offer_sz) != 0) { + /* + * This is not an error, since the host can also change + * the other field(s) of the offer, e.g. on WS RS5 + * (Build 17763), the offer->connection_id of the + * Mellanox VF vmbus device can change when the host + * reoffers the device upon resume. + */ + pr_debug("vmbus offer changed: relid=%d\n", + offer->child_relid); + + print_hex_dump_debug("Old vmbus offer: ", + DUMP_PREFIX_OFFSET, 16, 4, + &oldchannel->offermsg, offer_sz, + false); + print_hex_dump_debug("New vmbus offer: ", + DUMP_PREFIX_OFFSET, 16, 4, + offer, offer_sz, false); + + /* Fix up the old channel. */ + vmbus_setup_channel_state(oldchannel, offer); + } + + /* Add the channel back to the array of channels. */ + vmbus_channel_map_relid(oldchannel); + mutex_unlock(&vmbus_connection.channel_mutex); + return; + } + + /* Allocate the channel object and save this offer. */ + newchannel = alloc_channel(); + if (!newchannel) { + vmbus_release_relid(offer->child_relid); + atomic_dec(&vmbus_connection.offer_in_progress); + pr_err("Unable to allocate channel object\n"); + return; + } + newchannel->co_ring_buffer = co_ring_buffer; + newchannel->co_external_memory = co_external_memory; + + vmbus_setup_channel_state(newchannel, offer); + + vmbus_process_offer(newchannel); +} + +static void check_ready_for_suspend_event(void) +{ + /* + * If all the sub-channels or hv_sock channels have been cleaned up, + * then it's safe to suspend. + */ + if (atomic_dec_and_test(&vmbus_connection.nr_chan_close_on_suspend)) + complete(&vmbus_connection.ready_for_suspend_event); +} + +/* + * vmbus_onoffer_rescind - Rescind offer handler. + * + * We queue a work item to process this offer synchronously + */ +static void vmbus_onoffer_rescind(struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_rescind_offer *rescind; + struct vmbus_channel *channel; + struct device *dev; + bool clean_up_chan_for_suspend; + + rescind = (struct vmbus_channel_rescind_offer *)hdr; + + trace_vmbus_onoffer_rescind(rescind); + + /* + * The offer msg and the corresponding rescind msg + * from the host are guranteed to be ordered - + * offer comes in first and then the rescind. + * Since we process these events in work elements, + * and with preemption, we may end up processing + * the events out of order. We rely on the synchronization + * provided by offer_in_progress and by channel_mutex for + * ordering these events: + * + * { Initially: offer_in_progress = 1 } + * + * CPU1 CPU2 + * + * [vmbus_onoffer()] [vmbus_onoffer_rescind()] + * + * LOCK channel_mutex WAIT_ON offer_in_progress == 0 + * DECREMENT offer_in_progress LOCK channel_mutex + * STORE channels[] LOAD channels[] + * UNLOCK channel_mutex UNLOCK channel_mutex + * + * Forbids: CPU2's LOAD from *not* seeing CPU1's STORE + */ + + while (atomic_read(&vmbus_connection.offer_in_progress) != 0) { + /* + * We wait here until any channel offer is currently + * being processed. + */ + msleep(1); + } + + mutex_lock(&vmbus_connection.channel_mutex); + channel = relid2channel(rescind->child_relid); + if (channel != NULL) { + /* + * Guarantee that no other instance of vmbus_onoffer_rescind() + * has got a reference to the channel object. Synchronize on + * &vmbus_connection.channel_mutex. + */ + if (channel->rescind_ref) { + mutex_unlock(&vmbus_connection.channel_mutex); + return; + } + channel->rescind_ref = true; + } + mutex_unlock(&vmbus_connection.channel_mutex); + + if (channel == NULL) { + /* + * We failed in processing the offer message; + * we would have cleaned up the relid in that + * failure path. + */ + return; + } + + clean_up_chan_for_suspend = is_hvsock_channel(channel) || + is_sub_channel(channel); + /* + * Before setting channel->rescind in vmbus_rescind_cleanup(), we + * should make sure the channel callback is not running any more. + */ + vmbus_reset_channel_cb(channel); + + /* + * Now wait for offer handling to complete. + */ + vmbus_rescind_cleanup(channel); + while (READ_ONCE(channel->probe_done) == false) { + /* + * We wait here until any channel offer is currently + * being processed. + */ + msleep(1); + } + + /* + * At this point, the rescind handling can proceed safely. + */ + + if (channel->device_obj) { + if (channel->chn_rescind_callback) { + channel->chn_rescind_callback(channel); + + if (clean_up_chan_for_suspend) + check_ready_for_suspend_event(); + + return; + } + /* + * We will have to unregister this device from the + * driver core. + */ + dev = get_device(&channel->device_obj->device); + if (dev) { + vmbus_device_unregister(channel->device_obj); + put_device(dev); + } + } else if (channel->primary_channel != NULL) { + /* + * Sub-channel is being rescinded. Following is the channel + * close sequence when initiated from the driveri (refer to + * vmbus_close() for details): + * 1. Close all sub-channels first + * 2. Then close the primary channel. + */ + mutex_lock(&vmbus_connection.channel_mutex); + if (channel->state == CHANNEL_OPEN_STATE) { + /* + * The channel is currently not open; + * it is safe for us to cleanup the channel. + */ + hv_process_channel_removal(channel); + } else { + complete(&channel->rescind_event); + } + mutex_unlock(&vmbus_connection.channel_mutex); + } + + /* The "channel" may have been freed. Do not access it any longer. */ + + if (clean_up_chan_for_suspend) + check_ready_for_suspend_event(); +} + +void vmbus_hvsock_device_unregister(struct vmbus_channel *channel) +{ + BUG_ON(!is_hvsock_channel(channel)); + + /* We always get a rescind msg when a connection is closed. */ + while (!READ_ONCE(channel->probe_done) || !READ_ONCE(channel->rescind)) + msleep(1); + + vmbus_device_unregister(channel->device_obj); +} +EXPORT_SYMBOL_GPL(vmbus_hvsock_device_unregister); + + +/* + * vmbus_onoffers_delivered - + * The CHANNELMSG_ALLOFFERS_DELIVERED message arrives after all + * boot-time offers are delivered. A boot-time offer is for the primary + * channel for any virtual hardware configured in the VM at the time it boots. + * Boot-time offers include offers for physical devices assigned to the VM + * via Hyper-V's Discrete Device Assignment (DDA) functionality that are + * handled as virtual PCI devices in Linux (e.g., NVMe devices and GPUs). + * Boot-time offers do not include offers for VMBus sub-channels. Because + * devices can be hot-added to the VM after it is booted, additional channel + * offers that aren't boot-time offers can be received at any time after the + * all-offers-delivered message. + * + * SR-IOV NIC Virtual Functions (VFs) assigned to a VM are not considered + * to be assigned to the VM at boot-time, and offers for VFs may occur after + * the all-offers-delivered message. VFs are optional accelerators to the + * synthetic VMBus NIC and are effectively hot-added only after the VMBus + * NIC channel is opened (once it knows the guest can support it, via the + * sriov bit in the netvsc protocol). + */ +static void vmbus_onoffers_delivered( + struct vmbus_channel_message_header *hdr) +{ + complete(&vmbus_connection.all_offers_delivered_event); +} + +/* + * vmbus_onopen_result - Open result handler. + * + * This is invoked when we received a response to our channel open request. + * Find the matching request, copy the response and signal the requesting + * thread. + */ +static void vmbus_onopen_result(struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_open_result *result; + struct vmbus_channel_msginfo *msginfo; + struct vmbus_channel_message_header *requestheader; + struct vmbus_channel_open_channel *openmsg; + unsigned long flags; + + result = (struct vmbus_channel_open_result *)hdr; + + trace_vmbus_onopen_result(result); + + /* + * Find the open msg, copy the result and signal/unblock the wait event + */ + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + + list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, + msglistentry) { + requestheader = + (struct vmbus_channel_message_header *)msginfo->msg; + + if (requestheader->msgtype == CHANNELMSG_OPENCHANNEL) { + openmsg = + (struct vmbus_channel_open_channel *)msginfo->msg; + if (openmsg->child_relid == result->child_relid && + openmsg->openid == result->openid) { + memcpy(&msginfo->response.open_result, + result, + sizeof( + struct vmbus_channel_open_result)); + complete(&msginfo->waitevent); + break; + } + } + } + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +} + +/* + * vmbus_ongpadl_created - GPADL created handler. + * + * This is invoked when we received a response to our gpadl create request. + * Find the matching request, copy the response and signal the requesting + * thread. + */ +static void vmbus_ongpadl_created(struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_gpadl_created *gpadlcreated; + struct vmbus_channel_msginfo *msginfo; + struct vmbus_channel_message_header *requestheader; + struct vmbus_channel_gpadl_header *gpadlheader; + unsigned long flags; + + gpadlcreated = (struct vmbus_channel_gpadl_created *)hdr; + + trace_vmbus_ongpadl_created(gpadlcreated); + + /* + * Find the establish msg, copy the result and signal/unblock the wait + * event + */ + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + + list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, + msglistentry) { + requestheader = + (struct vmbus_channel_message_header *)msginfo->msg; + + if (requestheader->msgtype == CHANNELMSG_GPADL_HEADER) { + gpadlheader = + (struct vmbus_channel_gpadl_header *)requestheader; + + if ((gpadlcreated->child_relid == + gpadlheader->child_relid) && + (gpadlcreated->gpadl == gpadlheader->gpadl)) { + memcpy(&msginfo->response.gpadl_created, + gpadlcreated, + sizeof( + struct vmbus_channel_gpadl_created)); + complete(&msginfo->waitevent); + break; + } + } + } + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +} + +/* + * vmbus_onmodifychannel_response - Modify Channel response handler. + * + * This is invoked when we received a response to our channel modify request. + * Find the matching request, copy the response and signal the requesting thread. + */ +static void vmbus_onmodifychannel_response(struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_modifychannel_response *response; + struct vmbus_channel_msginfo *msginfo; + unsigned long flags; + + response = (struct vmbus_channel_modifychannel_response *)hdr; + + trace_vmbus_onmodifychannel_response(response); + + /* + * Find the modify msg, copy the response and signal/unblock the wait event. + */ + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + + list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, msglistentry) { + struct vmbus_channel_message_header *responseheader = + (struct vmbus_channel_message_header *)msginfo->msg; + + if (responseheader->msgtype == CHANNELMSG_MODIFYCHANNEL) { + struct vmbus_channel_modifychannel *modifymsg; + + modifymsg = (struct vmbus_channel_modifychannel *)msginfo->msg; + if (modifymsg->child_relid == response->child_relid) { + memcpy(&msginfo->response.modify_response, response, + sizeof(*response)); + complete(&msginfo->waitevent); + break; + } + } + } + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +} + +/* + * vmbus_ongpadl_torndown - GPADL torndown handler. + * + * This is invoked when we received a response to our gpadl teardown request. + * Find the matching request, copy the response and signal the requesting + * thread. + */ +static void vmbus_ongpadl_torndown( + struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_gpadl_torndown *gpadl_torndown; + struct vmbus_channel_msginfo *msginfo; + struct vmbus_channel_message_header *requestheader; + struct vmbus_channel_gpadl_teardown *gpadl_teardown; + unsigned long flags; + + gpadl_torndown = (struct vmbus_channel_gpadl_torndown *)hdr; + + trace_vmbus_ongpadl_torndown(gpadl_torndown); + + /* + * Find the open msg, copy the result and signal/unblock the wait event + */ + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + + list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, + msglistentry) { + requestheader = + (struct vmbus_channel_message_header *)msginfo->msg; + + if (requestheader->msgtype == CHANNELMSG_GPADL_TEARDOWN) { + gpadl_teardown = + (struct vmbus_channel_gpadl_teardown *)requestheader; + + if (gpadl_torndown->gpadl == gpadl_teardown->gpadl) { + memcpy(&msginfo->response.gpadl_torndown, + gpadl_torndown, + sizeof( + struct vmbus_channel_gpadl_torndown)); + complete(&msginfo->waitevent); + break; + } + } + } + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +} + +/* + * vmbus_onversion_response - Version response handler + * + * This is invoked when we received a response to our initiate contact request. + * Find the matching request, copy the response and signal the requesting + * thread. + */ +static void vmbus_onversion_response( + struct vmbus_channel_message_header *hdr) +{ + struct vmbus_channel_msginfo *msginfo; + struct vmbus_channel_message_header *requestheader; + struct vmbus_channel_version_response *version_response; + unsigned long flags; + + version_response = (struct vmbus_channel_version_response *)hdr; + + trace_vmbus_onversion_response(version_response); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + + list_for_each_entry(msginfo, &vmbus_connection.chn_msg_list, + msglistentry) { + requestheader = + (struct vmbus_channel_message_header *)msginfo->msg; + + if (requestheader->msgtype == + CHANNELMSG_INITIATE_CONTACT) { + memcpy(&msginfo->response.version_response, + version_response, + sizeof(struct vmbus_channel_version_response)); + complete(&msginfo->waitevent); + } + } + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); +} + +/* Channel message dispatch table */ +const struct vmbus_channel_message_table_entry +channel_message_table[CHANNELMSG_COUNT] = { + { CHANNELMSG_INVALID, 0, NULL, 0}, + { CHANNELMSG_OFFERCHANNEL, 0, vmbus_onoffer, + sizeof(struct vmbus_channel_offer_channel)}, + { CHANNELMSG_RESCIND_CHANNELOFFER, 0, vmbus_onoffer_rescind, + sizeof(struct vmbus_channel_rescind_offer) }, + { CHANNELMSG_REQUESTOFFERS, 0, NULL, 0}, + { CHANNELMSG_ALLOFFERS_DELIVERED, 1, vmbus_onoffers_delivered, 0}, + { CHANNELMSG_OPENCHANNEL, 0, NULL, 0}, + { CHANNELMSG_OPENCHANNEL_RESULT, 1, vmbus_onopen_result, + sizeof(struct vmbus_channel_open_result)}, + { CHANNELMSG_CLOSECHANNEL, 0, NULL, 0}, + { CHANNELMSG_GPADL_HEADER, 0, NULL, 0}, + { CHANNELMSG_GPADL_BODY, 0, NULL, 0}, + { CHANNELMSG_GPADL_CREATED, 1, vmbus_ongpadl_created, + sizeof(struct vmbus_channel_gpadl_created)}, + { CHANNELMSG_GPADL_TEARDOWN, 0, NULL, 0}, + { CHANNELMSG_GPADL_TORNDOWN, 1, vmbus_ongpadl_torndown, + sizeof(struct vmbus_channel_gpadl_torndown) }, + { CHANNELMSG_RELID_RELEASED, 0, NULL, 0}, + { CHANNELMSG_INITIATE_CONTACT, 0, NULL, 0}, + { CHANNELMSG_VERSION_RESPONSE, 1, vmbus_onversion_response, + sizeof(struct vmbus_channel_version_response)}, + { CHANNELMSG_UNLOAD, 0, NULL, 0}, + { CHANNELMSG_UNLOAD_RESPONSE, 1, vmbus_unload_response, 0}, + { CHANNELMSG_18, 0, NULL, 0}, + { CHANNELMSG_19, 0, NULL, 0}, + { CHANNELMSG_20, 0, NULL, 0}, + { CHANNELMSG_TL_CONNECT_REQUEST, 0, NULL, 0}, + { CHANNELMSG_MODIFYCHANNEL, 0, NULL, 0}, + { CHANNELMSG_TL_CONNECT_RESULT, 0, NULL, 0}, + { CHANNELMSG_MODIFYCHANNEL_RESPONSE, 1, vmbus_onmodifychannel_response, + sizeof(struct vmbus_channel_modifychannel_response)}, +}; + +/* + * vmbus_onmessage - Handler for channel protocol messages. + * + * This is invoked in the vmbus worker thread context. + */ +void vmbus_onmessage(struct vmbus_channel_message_header *hdr) +{ + trace_vmbus_on_message(hdr); + + /* + * vmbus_on_msg_dpc() makes sure the hdr->msgtype here can not go + * out of bound and the message_handler pointer can not be NULL. + */ + channel_message_table[hdr->msgtype].message_handler(hdr); +} + +/* + * vmbus_request_offers - Send a request to get all our pending offers + * and wait for all boot-time offers to arrive. + */ +int vmbus_request_offers(void) +{ + struct vmbus_channel_message_header *msg; + struct vmbus_channel_msginfo *msginfo; + int ret; + + msginfo = kzalloc(sizeof(*msginfo) + + sizeof(struct vmbus_channel_message_header), + GFP_KERNEL); + if (!msginfo) + return -ENOMEM; + + msg = (struct vmbus_channel_message_header *)msginfo->msg; + + msg->msgtype = CHANNELMSG_REQUESTOFFERS; + + /* + * This REQUESTOFFERS message will result in the host sending an all + * offers delivered message after all the boot-time offers are sent. + */ + ret = vmbus_post_msg(msg, sizeof(struct vmbus_channel_message_header), + true); + + trace_vmbus_request_offers(ret); + + if (ret != 0) { + pr_err("Unable to request offers - %d\n", ret); + + goto cleanup; + } + + /* + * Wait for the host to send all boot-time offers. + * Keeping it as a best-effort mechanism, where a warning is + * printed if a timeout occurs, and execution is resumed. + */ + if (!wait_for_completion_timeout(&vmbus_connection.all_offers_delivered_event, + secs_to_jiffies(60))) { + pr_warn("timed out waiting for all boot-time offers to be delivered.\n"); + } + + /* + * Flush handling of offer messages (which may initiate work on + * other work queues). + */ + flush_workqueue(vmbus_connection.work_queue); + + /* + * Flush workqueue for processing the incoming offers. Subchannel + * offers and their processing can happen later, so there is no need to + * flush that workqueue here. + */ + flush_workqueue(vmbus_connection.handle_primary_chan_wq); + +cleanup: + kfree(msginfo); + + return ret; +} + +void vmbus_set_sc_create_callback(struct vmbus_channel *primary_channel, + void (*sc_cr_cb)(struct vmbus_channel *new_sc)) +{ + primary_channel->sc_creation_callback = sc_cr_cb; +} +EXPORT_SYMBOL_GPL(vmbus_set_sc_create_callback); + +void vmbus_set_chn_rescind_callback(struct vmbus_channel *channel, + void (*chn_rescind_cb)(struct vmbus_channel *)) +{ + channel->chn_rescind_callback = chn_rescind_cb; +} +EXPORT_SYMBOL_GPL(vmbus_set_chn_rescind_callback); diff --git a/drivers/hv/connection.c b/drivers/hv/connection.c new file mode 100644 index 000000000..b5b322ce1 --- /dev/null +++ b/drivers/hv/connection.c @@ -0,0 +1,531 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/sched.h> +#include <linux/wait.h> +#include <linux/delay.h> +#include <linux/mm.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/hyperv.h> +#include <linux/export.h> +#include <linux/io.h> +#include <linux/set_memory.h> +#include <asm/mshyperv.h> + +#include "hyperv_vmbus.h" + + +struct vmbus_connection vmbus_connection = { + .conn_state = DISCONNECTED, + .unload_event = COMPLETION_INITIALIZER( + vmbus_connection.unload_event), + .next_gpadl_handle = ATOMIC_INIT(0xE1E10), + + .ready_for_suspend_event = COMPLETION_INITIALIZER( + vmbus_connection.ready_for_suspend_event), + .all_offers_delivered_event = COMPLETION_INITIALIZER( + vmbus_connection.all_offers_delivered_event), +}; +EXPORT_SYMBOL_GPL(vmbus_connection); + +/* + * Negotiated protocol version with the host. + */ +__u32 vmbus_proto_version; +EXPORT_SYMBOL_GPL(vmbus_proto_version); + +/* + * Table of VMBus versions listed from newest to oldest. + * VERSION_WIN7 and VERSION_WS2008 are no longer supported in + * Linux guests and are not listed. + */ +static __u32 vmbus_versions[] = { + VERSION_WIN10_V6_0, + VERSION_WIN10_V5_3, + VERSION_WIN10_V5_2, + VERSION_WIN10_V5_1, + VERSION_WIN10_V5, + VERSION_WIN10_V4_1, + VERSION_WIN10, + VERSION_WIN8_1, + VERSION_WIN8 +}; + +/* + * Maximal VMBus protocol version guests can negotiate. Useful to cap the + * VMBus version for testing and debugging purpose. + */ +static uint max_version = VERSION_WIN10_V6_0; + +module_param(max_version, uint, S_IRUGO); +MODULE_PARM_DESC(max_version, + "Maximal VMBus protocol version which can be negotiated"); + +int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version) +{ + int ret = 0; + struct vmbus_channel_initiate_contact *msg; + unsigned long flags; + + init_completion(&msginfo->waitevent); + + msg = (struct vmbus_channel_initiate_contact *)msginfo->msg; + + memset(msg, 0, sizeof(*msg)); + msg->header.msgtype = CHANNELMSG_INITIATE_CONTACT; + msg->vmbus_version_requested = version; + + /* + * VMBus protocol 5.0 (VERSION_WIN10_V5) and higher require that we must + * use VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate Contact Message, + * and for subsequent messages, we must use the Message Connection ID + * field in the host-returned Version Response Message. And, with + * VERSION_WIN10_V5 and higher, we don't use msg->interrupt_page, but we + * tell the host explicitly that we still use VMBUS_MESSAGE_SINT(2) for + * compatibility. + * + * On old hosts, we should always use VMBUS_MESSAGE_CONNECTION_ID (1). + */ + if (version >= VERSION_WIN10_V5) { + msg->msg_sint = VMBUS_MESSAGE_SINT; + msg->msg_vtl = ms_hyperv.vtl; + vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID_4; + } else { + msg->interrupt_page = virt_to_phys(vmbus_connection.int_page); + vmbus_connection.msg_conn_id = VMBUS_MESSAGE_CONNECTION_ID; + } + + if (vmbus_is_confidential() && version >= VERSION_WIN10_V6_0) + msg->feature_flags = VMBUS_FEATURE_FLAG_CONFIDENTIAL_CHANNELS; + + /* + * shared_gpa_boundary is zero in non-SNP VMs, so it's safe to always + * bitwise OR it + */ + msg->monitor_page1 = virt_to_phys(vmbus_connection.monitor_pages[0]) | + ms_hyperv.shared_gpa_boundary; + msg->monitor_page2 = virt_to_phys(vmbus_connection.monitor_pages[1]) | + ms_hyperv.shared_gpa_boundary; + + msg->target_vcpu = hv_cpu_number_to_vp_number(VMBUS_CONNECT_CPU); + + /* + * Add to list before we send the request since we may + * receive the response before returning from this routine + */ + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_add_tail(&msginfo->msglistentry, + &vmbus_connection.chn_msg_list); + + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + ret = vmbus_post_msg(msg, + sizeof(struct vmbus_channel_initiate_contact), + true); + + trace_vmbus_negotiate_version(msg, ret); + + if (ret != 0) { + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&msginfo->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, + flags); + return ret; + } + + /* Wait for the connection response */ + wait_for_completion(&msginfo->waitevent); + + spin_lock_irqsave(&vmbus_connection.channelmsg_lock, flags); + list_del(&msginfo->msglistentry); + spin_unlock_irqrestore(&vmbus_connection.channelmsg_lock, flags); + + /* Check if successful */ + if (msginfo->response.version_response.version_supported) { + vmbus_connection.conn_state = CONNECTED; + + if (version >= VERSION_WIN10_V5) + vmbus_connection.msg_conn_id = + msginfo->response.version_response.msg_conn_id; + } else { + return -ECONNREFUSED; + } + + return ret; +} + +/* + * vmbus_connect - Sends a connect request on the partition service connection + */ +int vmbus_connect(void) +{ + struct vmbus_channel_msginfo *msginfo = NULL; + int i, ret = 0; + __u32 version; + + /* Initialize the vmbus connection */ + vmbus_connection.conn_state = CONNECTING; + vmbus_connection.work_queue = create_workqueue("hv_vmbus_con"); + if (!vmbus_connection.work_queue) { + ret = -ENOMEM; + goto cleanup; + } + + vmbus_connection.rescind_work_queue = + create_workqueue("hv_vmbus_rescind"); + if (!vmbus_connection.rescind_work_queue) { + ret = -ENOMEM; + goto cleanup; + } + vmbus_connection.ignore_any_offer_msg = false; + + vmbus_connection.handle_primary_chan_wq = + create_workqueue("hv_pri_chan"); + if (!vmbus_connection.handle_primary_chan_wq) { + ret = -ENOMEM; + goto cleanup; + } + + vmbus_connection.handle_sub_chan_wq = + create_workqueue("hv_sub_chan"); + if (!vmbus_connection.handle_sub_chan_wq) { + ret = -ENOMEM; + goto cleanup; + } + + INIT_LIST_HEAD(&vmbus_connection.chn_msg_list); + spin_lock_init(&vmbus_connection.channelmsg_lock); + + INIT_LIST_HEAD(&vmbus_connection.chn_list); + mutex_init(&vmbus_connection.channel_mutex); + + /* + * The following Hyper-V interrupt and monitor pages can be used by + * UIO for mapping to user-space, so they should always be allocated on + * system page boundaries. The system page size must be >= the Hyper-V + * page size. + */ + BUILD_BUG_ON(PAGE_SIZE < HV_HYP_PAGE_SIZE); + + /* + * Setup the vmbus event connection for channel interrupt + * abstraction stuff + */ + vmbus_connection.int_page = + (void *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + if (vmbus_connection.int_page == NULL) { + ret = -ENOMEM; + goto cleanup; + } + + vmbus_connection.recv_int_page = vmbus_connection.int_page; + vmbus_connection.send_int_page = + (void *)((unsigned long)vmbus_connection.int_page + + (HV_HYP_PAGE_SIZE >> 1)); + + /* + * Setup the monitor notification facility. The 1st page for + * parent->child and the 2nd page for child->parent + */ + vmbus_connection.monitor_pages[0] = (void *)__get_free_page(GFP_KERNEL); + vmbus_connection.monitor_pages[1] = (void *)__get_free_page(GFP_KERNEL); + if ((vmbus_connection.monitor_pages[0] == NULL) || + (vmbus_connection.monitor_pages[1] == NULL)) { + ret = -ENOMEM; + goto cleanup; + } + + ret = set_memory_decrypted((unsigned long) + vmbus_connection.monitor_pages[0], 1); + ret |= set_memory_decrypted((unsigned long) + vmbus_connection.monitor_pages[1], 1); + if (ret) { + /* + * If set_memory_decrypted() fails, the encryption state + * of the memory is unknown. So leak the memory instead + * of risking returning decrypted memory to the free list. + * For simplicity, always handle both pages the same. + */ + vmbus_connection.monitor_pages[0] = NULL; + vmbus_connection.monitor_pages[1] = NULL; + goto cleanup; + } + + /* + * Set_memory_decrypted() will change the memory contents if + * decryption occurs, so zero monitor pages here. + */ + memset(vmbus_connection.monitor_pages[0], 0x00, HV_HYP_PAGE_SIZE); + memset(vmbus_connection.monitor_pages[1], 0x00, HV_HYP_PAGE_SIZE); + + msginfo = kzalloc(sizeof(*msginfo) + + sizeof(struct vmbus_channel_initiate_contact), + GFP_KERNEL); + if (msginfo == NULL) { + ret = -ENOMEM; + goto cleanup; + } + + /* + * Negotiate a compatible VMBUS version number with the + * host. We start with the highest number we can support + * and work our way down until we negotiate a compatible + * version. + */ + + for (i = 0; ; i++) { + if (i == ARRAY_SIZE(vmbus_versions)) { + ret = -EDOM; + goto cleanup; + } + + version = vmbus_versions[i]; + if (version > max_version) + continue; + + ret = vmbus_negotiate_version(msginfo, version); + if (ret == -ETIMEDOUT) + goto cleanup; + + if (vmbus_connection.conn_state == CONNECTED) + break; + } + + if (hv_is_isolation_supported() && version < VERSION_WIN10_V5_2) { + pr_err("Invalid VMBus version %d.%d (expected >= %d.%d) from the host supporting isolation\n", + version >> 16, version & 0xFFFF, VERSION_WIN10_V5_2 >> 16, VERSION_WIN10_V5_2 & 0xFFFF); + ret = -EINVAL; + goto cleanup; + } + + vmbus_proto_version = version; + pr_info("Vmbus version:%d.%d\n", + version >> 16, version & 0xFFFF); + + vmbus_connection.channels = kzalloc_objs(struct vmbus_channel *, + MAX_CHANNEL_RELIDS); + if (vmbus_connection.channels == NULL) { + ret = -ENOMEM; + goto cleanup; + } + + kfree(msginfo); + return 0; + +cleanup: + pr_err("Unable to connect to host\n"); + + vmbus_connection.conn_state = DISCONNECTED; + vmbus_disconnect(); + + kfree(msginfo); + + return ret; +} + +void vmbus_disconnect(void) +{ + /* + * First send the unload request to the host. + */ + vmbus_initiate_unload(false); + + if (vmbus_connection.handle_sub_chan_wq) + destroy_workqueue(vmbus_connection.handle_sub_chan_wq); + + if (vmbus_connection.handle_primary_chan_wq) + destroy_workqueue(vmbus_connection.handle_primary_chan_wq); + + if (vmbus_connection.rescind_work_queue) + destroy_workqueue(vmbus_connection.rescind_work_queue); + + if (vmbus_connection.work_queue) + destroy_workqueue(vmbus_connection.work_queue); + + if (vmbus_connection.int_page) { + free_page((unsigned long)vmbus_connection.int_page); + vmbus_connection.int_page = NULL; + } + + if (vmbus_connection.monitor_pages[0]) { + if (!set_memory_encrypted( + (unsigned long)vmbus_connection.monitor_pages[0], 1)) + free_page((unsigned long) + vmbus_connection.monitor_pages[0]); + vmbus_connection.monitor_pages[0] = NULL; + } + + if (vmbus_connection.monitor_pages[1]) { + if (!set_memory_encrypted( + (unsigned long)vmbus_connection.monitor_pages[1], 1)) + free_page((unsigned long) + vmbus_connection.monitor_pages[1]); + vmbus_connection.monitor_pages[1] = NULL; + } +} + +/* + * relid2channel - Get the channel object given its + * child relative id (ie channel id) + */ +struct vmbus_channel *relid2channel(u32 relid) +{ + if (vmbus_connection.channels == NULL) { + pr_warn_once("relid2channel: relid=%d: No channels mapped!\n", relid); + return NULL; + } + if (WARN_ON(relid >= MAX_CHANNEL_RELIDS)) + return NULL; + return READ_ONCE(vmbus_connection.channels[relid]); +} + +/* + * vmbus_on_event - Process a channel event notification + * + * For batched channels (default) optimize host to guest signaling + * by ensuring: + * 1. While reading the channel, we disable interrupts from host. + * 2. Ensure that we process all posted messages from the host + * before returning from this callback. + * 3. Once we return, enable signaling from the host. Once this + * state is set we check to see if additional packets are + * available to read. In this case we repeat the process. + * If this tasklet has been running for a long time + * then reschedule ourselves. + */ +void vmbus_on_event(unsigned long data) +{ + struct vmbus_channel *channel = (void *) data; + void (*callback_fn)(void *context); + + trace_vmbus_on_event(channel); + + hv_debug_delay_test(channel, INTERRUPT_DELAY); + + /* A channel once created is persistent even when + * there is no driver handling the device. An + * unloading driver sets the onchannel_callback to NULL. + */ + callback_fn = READ_ONCE(channel->onchannel_callback); + if (unlikely(!callback_fn)) + return; + + (*callback_fn)(channel->channel_callback_context); + + if (channel->callback_mode != HV_CALL_BATCHED) + return; + + if (likely(hv_end_read(&channel->inbound) == 0)) + return; + + hv_begin_read(&channel->inbound); + tasklet_schedule(&channel->callback_event); +} + +/* + * vmbus_post_msg - Send a msg on the vmbus's message connection + */ +int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep) +{ + struct vmbus_channel_message_header *hdr; + union hv_connection_id conn_id; + int ret = 0; + int retries = 0; + u32 usec = 1; + + conn_id.asu32 = 0; + conn_id.u.id = vmbus_connection.msg_conn_id; + + /* + * hv_post_message() can have transient failures because of + * insufficient resources. Retry the operation a couple of + * times before giving up. + */ + while (retries < 100) { + ret = hv_post_message(conn_id, 1, buffer, buflen); + + switch (ret) { + case HV_STATUS_INVALID_CONNECTION_ID: + /* + * See vmbus_negotiate_version(): VMBus protocol 5.0 + * and higher require that we must use + * VMBUS_MESSAGE_CONNECTION_ID_4 for the Initiate + * Contact message, but on old hosts that only + * support VMBus protocol 4.0 or lower, here we get + * HV_STATUS_INVALID_CONNECTION_ID and we should + * return an error immediately without retrying. + */ + hdr = buffer; + if (hdr->msgtype == CHANNELMSG_INITIATE_CONTACT) + return -EINVAL; + /* + * We could get this if we send messages too + * frequently. + */ + ret = -EAGAIN; + break; + case HV_STATUS_INSUFFICIENT_MEMORY: + case HV_STATUS_INSUFFICIENT_BUFFERS: + ret = -ENOBUFS; + break; + case HV_STATUS_SUCCESS: + return ret; + default: + pr_err("hv_post_msg() failed; error code:%d\n", ret); + return -EINVAL; + } + + retries++; + if (can_sleep && usec > 1000) + msleep(usec / 1000); + else if (usec < MAX_UDELAY_MS * 1000) + udelay(usec); + else + mdelay(usec / 1000); + + if (retries < 22) + usec *= 2; + } + return ret; +} + +/* + * vmbus_set_event - Send an event notification to the parent + */ +void vmbus_set_event(struct vmbus_channel *channel) +{ + u32 child_relid = channel->offermsg.child_relid; + + if (!channel->is_dedicated_interrupt) + vmbus_send_interrupt(child_relid); + + ++channel->sig_events; + + if (ms_hyperv.paravisor_present) { + if (hv_isolation_type_snp()) + hv_ghcb_hypercall(HVCALL_SIGNAL_EVENT, &channel->sig_event, + NULL, sizeof(channel->sig_event)); + else if (hv_isolation_type_tdx()) + hv_tdx_hypercall(HVCALL_SIGNAL_EVENT | HV_HYPERCALL_FAST_BIT, + channel->sig_event, 0); + else + WARN_ON_ONCE(1); + } else { + u64 control = HVCALL_SIGNAL_EVENT; + + control |= hv_nested ? HV_HYPERCALL_NESTED : 0; + hv_do_fast_hypercall8(control, channel->sig_event); + } +} +EXPORT_SYMBOL_GPL(vmbus_set_event); diff --git a/drivers/hv/hv.c b/drivers/hv/hv.c new file mode 100644 index 000000000..ef4b1b033 --- /dev/null +++ b/drivers/hv/hv.c @@ -0,0 +1,658 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/io.h> +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/vmalloc.h> +#include <linux/hyperv.h> +#include <linux/random.h> +#include <linux/clockchips.h> +#include <linux/delay.h> +#include <linux/interrupt.h> +#include <linux/export.h> +#include <clocksource/hyperv_timer.h> +#include <asm/mshyperv.h> +#include <linux/set_memory.h> +#include "hyperv_vmbus.h" + +/* The one and only */ +struct hv_context hv_context; +EXPORT_SYMBOL_FOR_MODULES(hv_context, "mshv_vtl"); + +/* + * hv_init - Main initialization routine. + * + * This routine must be called before any other routines in here are called + */ +int hv_init(void) +{ + hv_context.cpu_context = alloc_percpu(struct hv_per_cpu_context); + if (!hv_context.cpu_context) + return -ENOMEM; + return 0; +} + +/* + * hv_post_message - Post a message using the hypervisor message IPC. + * + * This involves a hypercall. + */ +int hv_post_message(union hv_connection_id connection_id, + enum hv_message_type message_type, + void *payload, size_t payload_size) +{ + struct hv_input_post_message *aligned_msg; + unsigned long flags; + u64 status; + + if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) + return -EMSGSIZE; + + local_irq_save(flags); + + /* + * A TDX VM with the paravisor must use the decrypted post_msg_page: see + * the comment in struct hv_per_cpu_context. A SNP VM with the paravisor + * can use the encrypted hyperv_pcpu_input_arg because it copies the + * input into the GHCB page, which has been decrypted by the paravisor. + */ + if (hv_isolation_type_tdx() && ms_hyperv.paravisor_present) + aligned_msg = this_cpu_ptr(hv_context.cpu_context)->post_msg_page; + else + aligned_msg = *this_cpu_ptr(hyperv_pcpu_input_arg); + + aligned_msg->connectionid = connection_id; + aligned_msg->reserved = 0; + aligned_msg->message_type = message_type; + aligned_msg->payload_size = payload_size; + memcpy((void *)aligned_msg->payload, payload, payload_size); + + if (ms_hyperv.paravisor_present && !vmbus_is_confidential()) { + /* + * If the VMBus isn't confidential, use the CoCo-specific + * mechanism to communicate with the hypervisor. + */ + if (hv_isolation_type_tdx()) + status = hv_tdx_hypercall(HVCALL_POST_MESSAGE, + virt_to_phys(aligned_msg), 0); + else if (hv_isolation_type_snp()) + status = hv_ghcb_hypercall(HVCALL_POST_MESSAGE, + aligned_msg, NULL, + sizeof(*aligned_msg)); + else + status = HV_STATUS_INVALID_PARAMETER; + } else { + u64 control = HVCALL_POST_MESSAGE; + + control |= hv_nested ? HV_HYPERCALL_NESTED : 0; + /* + * If there is no paravisor, this will go to the hypervisor. + * In the Confidential VMBus case, there is the paravisor + * to which this will trap. + */ + status = hv_do_hypercall(control, aligned_msg, NULL); + } + + local_irq_restore(flags); + + return hv_result(status); +} +EXPORT_SYMBOL_FOR_MODULES(hv_post_message, "mshv_vtl"); + +static int hv_alloc_page(void **page, bool decrypt, const char *note) +{ + int ret = 0; + + /* + * After the page changes its encryption status, its contents might + * appear scrambled on some hardware. Thus `get_zeroed_page` would + * zero the page out in vain, so do that explicitly exactly once. + * + * By default, the page is allocated encrypted in a CoCo VM. + */ + *page = (void *)__get_free_page(GFP_KERNEL); + if (!*page) + return -ENOMEM; + + if (decrypt) + ret = set_memory_decrypted((unsigned long)*page, 1); + if (ret) + goto failed; + + memset(*page, 0, PAGE_SIZE); + return 0; + +failed: + /* + * Report the failure but don't put the page back on the free list as + * its encryption status is unknown. + */ + pr_err("allocation failed for %s page, error %d, decrypted %d\n", + note, ret, decrypt); + *page = NULL; + return ret; +} + +static int hv_free_page(void **page, bool encrypt, const char *note) +{ + int ret = 0; + + if (!*page) + return 0; + + if (encrypt) + ret = set_memory_encrypted((unsigned long)*page, 1); + + /* + * In the case of the failure, the page is leaked. Something is wrong, + * prefer to lose the page with the unknown encryption status and stay afloat. + */ + if (ret) + pr_err("deallocation failed for %s page, error %d, encrypt %d\n", + note, ret, encrypt); + else + free_page((unsigned long)*page); + + *page = NULL; + + return ret; +} + +int hv_synic_alloc(void) +{ + int cpu, ret = -ENOMEM; + struct hv_per_cpu_context *hv_cpu; + const bool decrypt = !vmbus_is_confidential(); + + /* + * First, zero all per-cpu memory areas so hv_synic_free() can + * detect what memory has been allocated and cleanup properly + * after any failures. + */ + for_each_present_cpu(cpu) { + hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); + memset(hv_cpu, 0, sizeof(*hv_cpu)); + } + + hv_context.hv_numa_map = kzalloc_objs(struct cpumask, nr_node_ids); + if (!hv_context.hv_numa_map) { + pr_err("Unable to allocate NUMA map\n"); + goto err; + } + + for_each_present_cpu(cpu) { + hv_cpu = per_cpu_ptr(hv_context.cpu_context, cpu); + + tasklet_init(&hv_cpu->msg_dpc, + vmbus_on_msg_dpc, (unsigned long)hv_cpu); + + if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) { + ret = hv_alloc_page(&hv_cpu->post_msg_page, + decrypt, "post msg"); + if (ret) + goto err; + } + + /* + * If these SynIC pages are not allocated, SIEF and SIM pages + * are configured using what the root partition or the paravisor + * provides upon reading the SIEFP and SIMP registers. + */ + if (!ms_hyperv.paravisor_present && !hv_root_partition()) { + ret = hv_alloc_page(&hv_cpu->hyp_synic_message_page, + decrypt, "hypervisor SynIC msg"); + if (ret) + goto err; + ret = hv_alloc_page(&hv_cpu->hyp_synic_event_page, + decrypt, "hypervisor SynIC event"); + if (ret) + goto err; + } + + if (vmbus_is_confidential()) { + ret = hv_alloc_page(&hv_cpu->para_synic_message_page, + false, "paravisor SynIC msg"); + if (ret) + goto err; + ret = hv_alloc_page(&hv_cpu->para_synic_event_page, + false, "paravisor SynIC event"); + if (ret) + goto err; + } + } + + return 0; + +err: + /* + * Any memory allocations that succeeded will be freed when + * the caller cleans up by calling hv_synic_free() + */ + return ret; +} + +void hv_synic_free(void) +{ + int cpu; + const bool encrypt = !vmbus_is_confidential(); + + for_each_present_cpu(cpu) { + struct hv_per_cpu_context *hv_cpu = + per_cpu_ptr(hv_context.cpu_context, cpu); + + if (ms_hyperv.paravisor_present && hv_isolation_type_tdx()) + hv_free_page(&hv_cpu->post_msg_page, + encrypt, "post msg"); + if (!ms_hyperv.paravisor_present && !hv_root_partition()) { + hv_free_page(&hv_cpu->hyp_synic_event_page, + encrypt, "hypervisor SynIC event"); + hv_free_page(&hv_cpu->hyp_synic_message_page, + encrypt, "hypervisor SynIC msg"); + } + if (vmbus_is_confidential()) { + hv_free_page(&hv_cpu->para_synic_event_page, + false, "paravisor SynIC event"); + hv_free_page(&hv_cpu->para_synic_message_page, + false, "paravisor SynIC msg"); + } + } + + kfree(hv_context.hv_numa_map); +} + +/* + * hv_hyp_synic_enable_regs - Initialize the Synthetic Interrupt Controller + * with the hypervisor. + * + * Note: When MSHV is present, mshv_synic_cpu_init() intializes further + * registers later. + */ +void hv_hyp_synic_enable_regs(unsigned int cpu) +{ + struct hv_per_cpu_context *hv_cpu = + per_cpu_ptr(hv_context.cpu_context, cpu); + union hv_synic_simp simp; + union hv_synic_siefp siefp; + union hv_synic_sint shared_sint; + + /* Setup the Synic's message page with the hypervisor. */ + simp.as_uint64 = hv_get_msr(HV_MSR_SIMP); + simp.simp_enabled = 1; + + if (ms_hyperv.paravisor_present || hv_root_partition()) { + /* Mask out vTOM bit and map as decrypted */ + u64 base = (simp.base_simp_gpa << HV_HYP_PAGE_SHIFT) & + ~ms_hyperv.shared_gpa_boundary; + hv_cpu->hyp_synic_message_page = + memremap(base, HV_HYP_PAGE_SIZE, MEMREMAP_WB | MEMREMAP_DEC); + if (!hv_cpu->hyp_synic_message_page) + pr_err("Fail to map synic message page.\n"); + } else { + simp.base_simp_gpa = virt_to_phys(hv_cpu->hyp_synic_message_page) + >> HV_HYP_PAGE_SHIFT; + } + + hv_set_msr(HV_MSR_SIMP, simp.as_uint64); + + /* Setup the Synic's event page with the hypervisor. */ + siefp.as_uint64 = hv_get_msr(HV_MSR_SIEFP); + siefp.siefp_enabled = 1; + + if (ms_hyperv.paravisor_present || hv_root_partition()) { + /* Mask out vTOM bit and map as decrypted */ + u64 base = (siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT) & + ~ms_hyperv.shared_gpa_boundary; + hv_cpu->hyp_synic_event_page = + memremap(base, HV_HYP_PAGE_SIZE, MEMREMAP_WB | MEMREMAP_DEC); + if (!hv_cpu->hyp_synic_event_page) + pr_err("Fail to map synic event page.\n"); + } else { + siefp.base_siefp_gpa = virt_to_phys(hv_cpu->hyp_synic_event_page) + >> HV_HYP_PAGE_SHIFT; + } + + hv_set_msr(HV_MSR_SIEFP, siefp.as_uint64); + hv_enable_coco_interrupt(cpu, vmbus_interrupt, true); + + /* Setup the shared SINT. */ + if (vmbus_irq != -1) + enable_percpu_irq(vmbus_irq, 0); + shared_sint.as_uint64 = hv_get_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT); + + shared_sint.vector = vmbus_interrupt; + shared_sint.masked = false; + shared_sint.auto_eoi = hv_recommend_using_aeoi(); + hv_set_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); +} + +static void hv_hyp_synic_enable_interrupts(void) +{ + union hv_synic_scontrol sctrl; + + /* Enable the global synic bit */ + sctrl.as_uint64 = hv_get_msr(HV_MSR_SCONTROL); + sctrl.enable = 1; + + hv_set_msr(HV_MSR_SCONTROL, sctrl.as_uint64); +} + +static void hv_para_synic_enable_regs(unsigned int cpu) +{ + union hv_synic_simp simp; + union hv_synic_siefp siefp; + struct hv_per_cpu_context *hv_cpu + = per_cpu_ptr(hv_context.cpu_context, cpu); + + /* Setup the Synic's message page with the paravisor. */ + simp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIMP); + simp.simp_enabled = 1; + simp.base_simp_gpa = virt_to_phys(hv_cpu->para_synic_message_page) + >> HV_HYP_PAGE_SHIFT; + hv_para_set_synic_register(HV_MSR_SIMP, simp.as_uint64); + + /* Setup the Synic's event page with the paravisor. */ + siefp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIEFP); + siefp.siefp_enabled = 1; + siefp.base_siefp_gpa = virt_to_phys(hv_cpu->para_synic_event_page) + >> HV_HYP_PAGE_SHIFT; + hv_para_set_synic_register(HV_MSR_SIEFP, siefp.as_uint64); +} + +static void hv_para_synic_enable_interrupts(void) +{ + union hv_synic_scontrol sctrl; + + /* Enable the global synic bit */ + sctrl.as_uint64 = hv_para_get_synic_register(HV_MSR_SCONTROL); + sctrl.enable = 1; + hv_para_set_synic_register(HV_MSR_SCONTROL, sctrl.as_uint64); +} + +int hv_synic_init(unsigned int cpu) +{ + if (vmbus_is_confidential()) + hv_para_synic_enable_regs(cpu); + + /* + * The SINT is set in hv_hyp_synic_enable_regs() by calling + * hv_set_msr(). hv_set_msr() in turn has special case code for the + * SINT MSRs that write to the hypervisor version of the MSR *and* + * the paravisor version of the MSR (but *without* the proxy bit when + * VMBus is confidential). + * + * Then enable interrupts via the paravisor if VMBus is confidential, + * and otherwise via the hypervisor. + */ + + hv_hyp_synic_enable_regs(cpu); + if (vmbus_is_confidential()) + hv_para_synic_enable_interrupts(); + else + hv_hyp_synic_enable_interrupts(); + + hv_stimer_legacy_init(cpu, VMBUS_MESSAGE_SINT); + + return 0; +} + +void hv_hyp_synic_disable_regs(unsigned int cpu) +{ + struct hv_per_cpu_context *hv_cpu = + per_cpu_ptr(hv_context.cpu_context, cpu); + union hv_synic_sint shared_sint; + union hv_synic_simp simp; + union hv_synic_siefp siefp; + + shared_sint.as_uint64 = hv_get_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT); + + shared_sint.masked = 1; + + /* Need to correctly cleanup in the case of SMP!!! */ + /* Disable the interrupt */ + hv_set_msr(HV_MSR_SINT0 + VMBUS_MESSAGE_SINT, shared_sint.as_uint64); + hv_enable_coco_interrupt(cpu, vmbus_interrupt, false); + + simp.as_uint64 = hv_get_msr(HV_MSR_SIMP); + /* + * In Isolation VM, simp and sief pages are allocated by + * paravisor. These pages also will be used by kdump + * kernel. So just reset enable bit here and keep page + * addresses. + */ + simp.simp_enabled = 0; + if (ms_hyperv.paravisor_present || hv_root_partition()) { + if (hv_cpu->hyp_synic_message_page) { + memunmap(hv_cpu->hyp_synic_message_page); + hv_cpu->hyp_synic_message_page = NULL; + } + } else { + simp.base_simp_gpa = 0; + } + + hv_set_msr(HV_MSR_SIMP, simp.as_uint64); + + siefp.as_uint64 = hv_get_msr(HV_MSR_SIEFP); + siefp.siefp_enabled = 0; + + if (ms_hyperv.paravisor_present || hv_root_partition()) { + if (hv_cpu->hyp_synic_event_page) { + memunmap(hv_cpu->hyp_synic_event_page); + hv_cpu->hyp_synic_event_page = NULL; + } + } else { + siefp.base_siefp_gpa = 0; + } + + hv_set_msr(HV_MSR_SIEFP, siefp.as_uint64); +} + +static void hv_hyp_synic_disable_interrupts(void) +{ + union hv_synic_scontrol sctrl; + + /* Disable the global synic bit */ + sctrl.as_uint64 = hv_get_msr(HV_MSR_SCONTROL); + sctrl.enable = 0; + hv_set_msr(HV_MSR_SCONTROL, sctrl.as_uint64); +} + +static void hv_para_synic_disable_regs(unsigned int cpu) +{ + union hv_synic_simp simp; + union hv_synic_siefp siefp; + + /* Disable SynIC's message page in the paravisor. */ + simp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIMP); + simp.simp_enabled = 0; + hv_para_set_synic_register(HV_MSR_SIMP, simp.as_uint64); + + /* Disable SynIC's event page in the paravisor. */ + siefp.as_uint64 = hv_para_get_synic_register(HV_MSR_SIEFP); + siefp.siefp_enabled = 0; + hv_para_set_synic_register(HV_MSR_SIEFP, siefp.as_uint64); +} + +static void hv_para_synic_disable_interrupts(void) +{ + union hv_synic_scontrol sctrl; + + /* Disable the global synic bit */ + sctrl.as_uint64 = hv_para_get_synic_register(HV_MSR_SCONTROL); + sctrl.enable = 0; + hv_para_set_synic_register(HV_MSR_SCONTROL, sctrl.as_uint64); +} + +#define HV_MAX_TRIES 3 +/* + * Scan the event flags page of 'this' CPU looking for any bit that is set. If we find one + * bit set, then wait for a few milliseconds. Repeat these steps for a maximum of 3 times. + * Return 'true', if there is still any set bit after this operation; 'false', otherwise. + * + * If a bit is set, that means there is a pending channel interrupt. The expectation is + * that the normal interrupt handling mechanism will find and process the channel interrupt + * "very soon", and in the process clear the bit. + */ +static bool __hv_synic_event_pending(union hv_synic_event_flags *event, int sint) +{ + unsigned long *recv_int_page; + bool pending; + u32 relid; + int tries = 0; + + if (!event) + return false; + + event += sint; + recv_int_page = event->flags; /* assumes VMBus version >= VERSION_WIN8 */ +retry: + pending = false; + for_each_set_bit(relid, recv_int_page, HV_EVENT_FLAGS_COUNT) { + /* Special case - VMBus channel protocol messages */ + if (relid == 0) + continue; + pending = true; + break; + } + if (pending && tries++ < HV_MAX_TRIES) { + usleep_range(10000, 20000); + goto retry; + } + return pending; +} + +static bool hv_synic_event_pending(void) +{ + struct hv_per_cpu_context *hv_cpu = this_cpu_ptr(hv_context.cpu_context); + union hv_synic_event_flags *hyp_synic_event_page = hv_cpu->hyp_synic_event_page; + union hv_synic_event_flags *para_synic_event_page = hv_cpu->para_synic_event_page; + + return + __hv_synic_event_pending(hyp_synic_event_page, VMBUS_MESSAGE_SINT) || + __hv_synic_event_pending(para_synic_event_page, VMBUS_MESSAGE_SINT); +} + +static int hv_pick_new_cpu(struct vmbus_channel *channel) +{ + int ret = -EBUSY; + int start; + int cpu; + + lockdep_assert_cpus_held(); + lockdep_assert_held(&vmbus_connection.channel_mutex); + + /* + * We can't assume that the relevant interrupts will be sent before + * the cpu is offlined on older versions of hyperv. + */ + if (vmbus_proto_version < VERSION_WIN10_V5_3) + return -EBUSY; + + start = get_random_u32_below(nr_cpu_ids); + + for_each_cpu_wrap(cpu, cpu_online_mask, start) { + if (channel->target_cpu == cpu || + channel->target_cpu == VMBUS_CONNECT_CPU) + continue; + + ret = vmbus_channel_set_cpu(channel, cpu); + if (!ret) + break; + } + + if (ret) + ret = vmbus_channel_set_cpu(channel, VMBUS_CONNECT_CPU); + + return ret; +} + +/* + * hv_synic_cleanup - Cleanup routine for hv_synic_init(). + */ +int hv_synic_cleanup(unsigned int cpu) +{ + struct vmbus_channel *channel, *sc; + int ret = 0; + + if (vmbus_connection.conn_state != CONNECTED) + goto always_cleanup; + + /* + * Hyper-V does not provide a way to change the connect CPU once + * it is set; we must prevent the connect CPU from going offline + * while the VM is running normally. But in the panic or kexec() + * path where the vmbus is already disconnected, the CPU must be + * allowed to shut down. + */ + if (cpu == VMBUS_CONNECT_CPU) + return -EBUSY; + + /* + * Search for channels which are bound to the CPU we're about to + * cleanup. + */ + mutex_lock(&vmbus_connection.channel_mutex); + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (channel->target_cpu == cpu) { + ret = hv_pick_new_cpu(channel); + if (ret) { + mutex_unlock(&vmbus_connection.channel_mutex); + return ret; + } + } + list_for_each_entry(sc, &channel->sc_list, sc_list) { + if (sc->target_cpu == cpu) { + ret = hv_pick_new_cpu(sc); + if (ret) { + mutex_unlock(&vmbus_connection.channel_mutex); + return ret; + } + } + } + } + mutex_unlock(&vmbus_connection.channel_mutex); + + /* + * Scan the event flags page looking for bits that are set and waiting + * with a timeout for vmbus_chan_sched() to process such bits. If bits + * are still set after this operation and VMBus is connected, fail the + * CPU offlining operation. + */ + if (vmbus_proto_version >= VERSION_WIN10_V4_1 && hv_synic_event_pending()) + return -EBUSY; + +always_cleanup: + hv_stimer_legacy_cleanup(cpu); + + /* + * First, disable the event and message pages + * used for communicating with the host, and then + * disable the host interrupts if VMBus is not + * confidential. + */ + hv_hyp_synic_disable_regs(cpu); + if (!vmbus_is_confidential()) + hv_hyp_synic_disable_interrupts(); + + /* + * Perform the same steps for the Confidential VMBus. + * The sequencing provides the guarantee that no data + * may be posted for processing before disabling interrupts. + */ + if (vmbus_is_confidential()) { + hv_para_synic_disable_regs(cpu); + hv_para_synic_disable_interrupts(); + } + if (vmbus_irq != -1) + disable_percpu_irq(vmbus_irq); + + return ret; +} diff --git a/drivers/hv/hv_balloon.c b/drivers/hv/hv_balloon.c new file mode 100644 index 000000000..42ce27be3 --- /dev/null +++ b/drivers/hv/hv_balloon.c @@ -0,0 +1,2138 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2012, Microsoft Corporation. + * + * Author: + * K. Y. Srinivasan <kys@microsoft.com> + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/cleanup.h> +#include <linux/kernel.h> +#include <linux/jiffies.h> +#include <linux/mman.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/kthread.h> +#include <linux/completion.h> +#include <linux/count_zeros.h> +#include <linux/memory_hotplug.h> +#include <linux/memory.h> +#include <linux/notifier.h> +#include <linux/percpu_counter.h> +#include <linux/page_reporting.h> +#include <linux/sizes.h> + +#include <linux/hyperv.h> +#include <hyperv/hvhdk.h> + +#include <asm/mshyperv.h> + +#define CREATE_TRACE_POINTS +#include "hv_trace_balloon.h" + +/* + * We begin with definitions supporting the Dynamic Memory protocol + * with the host. + * + * Begin protocol definitions. + */ + +/* + * Protocol versions. The low word is the minor version, the high word the major + * version. + * + * History: + * Initial version 1.0 + * Changed to 0.1 on 2009/03/25 + * Changes to 0.2 on 2009/05/14 + * Changes to 0.3 on 2009/12/03 + * Changed to 1.0 on 2011/04/05 + */ + +#define DYNMEM_MAKE_VERSION(Major, Minor) ((__u32)(((Major) << 16) | (Minor))) +#define DYNMEM_MAJOR_VERSION(Version) ((__u32)(Version) >> 16) +#define DYNMEM_MINOR_VERSION(Version) ((__u32)(Version) & 0xff) + +enum { + DYNMEM_PROTOCOL_VERSION_1 = DYNMEM_MAKE_VERSION(0, 3), + DYNMEM_PROTOCOL_VERSION_2 = DYNMEM_MAKE_VERSION(1, 0), + DYNMEM_PROTOCOL_VERSION_3 = DYNMEM_MAKE_VERSION(2, 0), + + DYNMEM_PROTOCOL_VERSION_WIN7 = DYNMEM_PROTOCOL_VERSION_1, + DYNMEM_PROTOCOL_VERSION_WIN8 = DYNMEM_PROTOCOL_VERSION_2, + DYNMEM_PROTOCOL_VERSION_WIN10 = DYNMEM_PROTOCOL_VERSION_3, + + DYNMEM_PROTOCOL_VERSION_CURRENT = DYNMEM_PROTOCOL_VERSION_WIN10 +}; + +/* + * Message Types + */ + +enum dm_message_type { + /* + * Version 0.3 + */ + DM_ERROR = 0, + DM_VERSION_REQUEST = 1, + DM_VERSION_RESPONSE = 2, + DM_CAPABILITIES_REPORT = 3, + DM_CAPABILITIES_RESPONSE = 4, + DM_STATUS_REPORT = 5, + DM_BALLOON_REQUEST = 6, + DM_BALLOON_RESPONSE = 7, + DM_UNBALLOON_REQUEST = 8, + DM_UNBALLOON_RESPONSE = 9, + DM_MEM_HOT_ADD_REQUEST = 10, + DM_MEM_HOT_ADD_RESPONSE = 11, + DM_VERSION_03_MAX = 11, + /* + * Version 1.0. + */ + DM_INFO_MESSAGE = 12, + DM_VERSION_1_MAX = 12 +}; + +/* + * Structures defining the dynamic memory management + * protocol. + */ + +union dm_version { + struct { + __u16 minor_version; + __u16 major_version; + }; + __u32 version; +} __packed; + +union dm_caps { + struct { + __u64 balloon:1; + __u64 hot_add:1; + /* + * To support guests that may have alignment + * limitations on hot-add, the guest can specify + * its alignment requirements; a value of n + * represents an alignment of 2^n in mega bytes. + */ + __u64 hot_add_alignment:4; + __u64 reservedz:58; + } cap_bits; + __u64 caps; +} __packed; + +union dm_mem_page_range { + struct { + /* + * The PFN number of the first page in the range. + * 40 bits is the architectural limit of a PFN + * number for AMD64. + */ + __u64 start_page:40; + /* + * The number of pages in the range. + */ + __u64 page_cnt:24; + } finfo; + __u64 page_range; +} __packed; + +/* + * The header for all dynamic memory messages: + * + * type: Type of the message. + * size: Size of the message in bytes; including the header. + * trans_id: The guest is responsible for manufacturing this ID. + */ + +struct dm_header { + __u16 type; + __u16 size; + __u32 trans_id; +} __packed; + +/* + * A generic message format for dynamic memory. + * Specific message formats are defined later in the file. + */ + +struct dm_message { + struct dm_header hdr; + __u8 data[]; /* enclosed message */ +} __packed; + +/* + * Specific message types supporting the dynamic memory protocol. + */ + +/* + * Version negotiation message. Sent from the guest to the host. + * The guest is free to try different versions until the host + * accepts the version. + * + * dm_version: The protocol version requested. + * is_last_attempt: If TRUE, this is the last version guest will request. + * reservedz: Reserved field, set to zero. + */ + +struct dm_version_request { + struct dm_header hdr; + union dm_version version; + __u32 is_last_attempt:1; + __u32 reservedz:31; +} __packed; + +/* + * Version response message; Host to Guest and indicates + * if the host has accepted the version sent by the guest. + * + * is_accepted: If TRUE, host has accepted the version and the guest + * should proceed to the next stage of the protocol. FALSE indicates that + * guest should re-try with a different version. + * + * reservedz: Reserved field, set to zero. + */ + +struct dm_version_response { + struct dm_header hdr; + __u64 is_accepted:1; + __u64 reservedz:63; +} __packed; + +/* + * Message reporting capabilities. This is sent from the guest to the + * host. + */ + +struct dm_capabilities { + struct dm_header hdr; + union dm_caps caps; + __u64 min_page_cnt; + __u64 max_page_number; +} __packed; + +/* + * Response to the capabilities message. This is sent from the host to the + * guest. This message notifies if the host has accepted the guest's + * capabilities. If the host has not accepted, the guest must shutdown + * the service. + * + * is_accepted: Indicates if the host has accepted guest's capabilities. + * reservedz: Must be 0. + */ + +struct dm_capabilities_resp_msg { + struct dm_header hdr; + __u64 is_accepted:1; + __u64 reservedz:63; +} __packed; + +/* + * This message is used to report memory pressure from the guest. + * This message is not part of any transaction and there is no + * response to this message. + * + * num_avail: Available memory in pages. + * num_committed: Committed memory in pages. + * page_file_size: The accumulated size of all page files + * in the system in pages. + * zero_free: The number of zero and free pages. + * page_file_writes: The writes to the page file in pages. + * io_diff: An indicator of file cache efficiency or page file activity, + * calculated as File Cache Page Fault Count - Page Read Count. + * This value is in pages. + * + * Some of these metrics are Windows specific and fortunately + * the algorithm on the host side that computes the guest memory + * pressure only uses num_committed value. + */ + +struct dm_status { + struct dm_header hdr; + __u64 num_avail; + __u64 num_committed; + __u64 page_file_size; + __u64 zero_free; + __u32 page_file_writes; + __u32 io_diff; +} __packed; + +/* + * Message to ask the guest to allocate memory - balloon up message. + * This message is sent from the host to the guest. The guest may not be + * able to allocate as much memory as requested. + * + * num_pages: number of pages to allocate. + */ + +struct dm_balloon { + struct dm_header hdr; + __u32 num_pages; + __u32 reservedz; +} __packed; + +/* + * Balloon response message; this message is sent from the guest + * to the host in response to the balloon message. + * + * reservedz: Reserved; must be set to zero. + * more_pages: If FALSE, this is the last message of the transaction. + * if TRUE there will be at least one more message from the guest. + * + * range_count: The number of ranges in the range array. + * + * range_array: An array of page ranges returned to the host. + * + */ + +struct dm_balloon_response { + struct dm_header hdr; + __u32 reservedz; + __u32 more_pages:1; + __u32 range_count:31; + union dm_mem_page_range range_array[]; +} __packed; + +/* + * Un-balloon message; this message is sent from the host + * to the guest to give guest more memory. + * + * more_pages: If FALSE, this is the last message of the transaction. + * if TRUE there will be at least one more message from the guest. + * + * reservedz: Reserved; must be set to zero. + * + * range_count: The number of ranges in the range array. + * + * range_array: An array of page ranges returned to the host. + * + */ + +struct dm_unballoon_request { + struct dm_header hdr; + __u32 more_pages:1; + __u32 reservedz:31; + __u32 range_count; + union dm_mem_page_range range_array[]; +} __packed; + +/* + * Un-balloon response message; this message is sent from the guest + * to the host in response to an unballoon request. + * + */ + +struct dm_unballoon_response { + struct dm_header hdr; +} __packed; + +/* + * Hot add request message. Message sent from the host to the guest. + * + * mem_range: Memory range to hot add. + * + */ + +struct dm_hot_add { + struct dm_header hdr; + union dm_mem_page_range range; +} __packed; + +/* + * Hot add response message. + * This message is sent by the guest to report the status of a hot add request. + * If page_count is less than the requested page count, then the host should + * assume all further hot add requests will fail, since this indicates that + * the guest has hit an upper physical memory barrier. + * + * Hot adds may also fail due to low resources; in this case, the guest must + * not complete this message until the hot add can succeed, and the host must + * not send a new hot add request until the response is sent. + * If VSC fails to hot add memory DYNMEM_NUMBER_OF_UNSUCCESSFUL_HOTADD_ATTEMPTS + * times it fails the request. + * + * + * page_count: number of pages that were successfully hot added. + * + * result: result of the operation 1: success, 0: failure. + * + */ + +struct dm_hot_add_response { + struct dm_header hdr; + __u32 page_count; + __u32 result; +} __packed; + +/* + * Types of information sent from host to the guest. + */ + +enum dm_info_type { + INFO_TYPE_MAX_PAGE_CNT = 0, + MAX_INFO_TYPE +}; + +/* + * Header for the information message. + */ + +struct dm_info_header { + enum dm_info_type type; + __u32 data_size; +} __packed; + +/* + * This message is sent from the host to the guest to pass + * some relevant information (win8 addition). + * + * reserved: no used. + * info_size: size of the information blob. + * info: information blob. + */ + +struct dm_info_msg { + struct dm_header hdr; + __u32 reserved; + __u32 info_size; + __u8 info[]; +}; + +/* + * End protocol definitions. + */ + +/* + * State to manage hot adding memory into the guest. + * The range start_pfn : end_pfn specifies the range + * that the host has asked us to hot add. The range + * start_pfn : ha_end_pfn specifies the range that we have + * currently hot added. We hot add in chunks equal to the + * memory block size; it is possible that we may not be able + * to bring online all the pages in the region. The range + * covered_start_pfn:covered_end_pfn defines the pages that can + * be brought online. + */ + +struct hv_hotadd_state { + struct list_head list; + unsigned long start_pfn; + unsigned long covered_start_pfn; + unsigned long covered_end_pfn; + unsigned long ha_end_pfn; + unsigned long end_pfn; + /* + * A list of gaps. + */ + struct list_head gap_list; +}; + +struct hv_hotadd_gap { + struct list_head list; + unsigned long start_pfn; + unsigned long end_pfn; +}; + +struct balloon_state { + __u32 num_pages; + struct work_struct wrk; +}; + +struct hot_add_wrk { + union dm_mem_page_range ha_page_range; + union dm_mem_page_range ha_region_range; + struct work_struct wrk; +}; + +static bool allow_hibernation; +static bool hot_add = true; +static bool do_hot_add; +/* + * Delay reporting memory pressure by + * the specified number of seconds. + */ +static uint pressure_report_delay = 45; +extern unsigned int page_reporting_order; +#define HV_MAX_FAILURES 2 + +/* + * The last time we posted a pressure report to host. + */ +static unsigned long last_post_time; + +static int hv_hypercall_multi_failure; + +module_param(hot_add, bool, 0644); +MODULE_PARM_DESC(hot_add, "If set attempt memory hot_add"); + +module_param(pressure_report_delay, uint, 0644); +MODULE_PARM_DESC(pressure_report_delay, "Delay in secs in reporting pressure"); +static atomic_t trans_id = ATOMIC_INIT(0); + +static int dm_ring_size = VMBUS_RING_SIZE(16 * 1024); + +/* + * Driver specific state. + */ + +enum hv_dm_state { + DM_INITIALIZING = 0, + DM_INITIALIZED, + DM_BALLOON_UP, + DM_BALLOON_DOWN, + DM_HOT_ADD, + DM_INIT_ERROR +}; + +static __u8 recv_buffer[HV_HYP_PAGE_SIZE]; +static __u8 balloon_up_send_buffer[HV_HYP_PAGE_SIZE]; + +static unsigned long ha_pages_in_chunk; +#define HA_BYTES_IN_CHUNK (ha_pages_in_chunk << PAGE_SHIFT) + +#define PAGES_IN_2M (2 * 1024 * 1024 / PAGE_SIZE) + +struct hv_dynmem_device { + struct hv_device *dev; + enum hv_dm_state state; + struct completion host_event; + struct completion config_event; + + /* + * Number of pages we have currently ballooned out. + */ + unsigned int num_pages_ballooned; + unsigned int num_pages_onlined; + unsigned int num_pages_added; + + /* + * State to manage the ballooning (up) operation. + */ + struct balloon_state balloon_wrk; + + /* + * State to execute the "hot-add" operation. + */ + struct hot_add_wrk ha_wrk; + + /* + * This state tracks if the host has specified a hot-add + * region. + */ + bool host_specified_ha_region; + + /* + * State to synchronize hot-add. + */ + struct completion ol_waitevent; + /* + * This thread handles hot-add + * requests from the host as well as notifying + * the host with regards to memory pressure in + * the guest. + */ + struct task_struct *thread; + + /* + * Protects ha_region_list, num_pages_onlined counter and individual + * regions from ha_region_list. + */ + spinlock_t ha_lock; + + /* + * A list of hot-add regions. + */ + struct list_head ha_region_list; + + /* + * We start with the highest version we can support + * and downgrade based on the host; we save here the + * next version to try. + */ + __u32 next_version; + + /* + * The negotiated version agreed by host. + */ + __u32 version; + + struct page_reporting_dev_info pr_dev_info; + + /* + * Maximum number of pages that can be hot_add-ed + */ + __u64 max_dynamic_page_count; +}; + +static struct hv_dynmem_device dm_device; + +static void post_status(struct hv_dynmem_device *dm); + +static void enable_page_reporting(void); + +static void disable_page_reporting(void); + +#ifdef CONFIG_MEMORY_HOTPLUG +static inline bool has_pfn_is_backed(struct hv_hotadd_state *has, + unsigned long pfn) +{ + struct hv_hotadd_gap *gap; + + /* The page is not backed. */ + if (pfn < has->covered_start_pfn || pfn >= has->covered_end_pfn) + return false; + + /* Check for gaps. */ + list_for_each_entry(gap, &has->gap_list, list) { + if (pfn >= gap->start_pfn && pfn < gap->end_pfn) + return false; + } + + return true; +} + +static unsigned long hv_page_offline_check(unsigned long start_pfn, + unsigned long nr_pages) +{ + unsigned long pfn = start_pfn, count = 0; + struct hv_hotadd_state *has; + bool found; + + while (pfn < start_pfn + nr_pages) { + /* + * Search for HAS which covers the pfn and when we find one + * count how many consequitive PFNs are covered. + */ + found = false; + list_for_each_entry(has, &dm_device.ha_region_list, list) { + while ((pfn >= has->start_pfn) && + (pfn < has->end_pfn) && + (pfn < start_pfn + nr_pages)) { + found = true; + if (has_pfn_is_backed(has, pfn)) + count++; + pfn++; + } + } + + /* + * This PFN is not in any HAS (e.g. we're offlining a region + * which was present at boot), no need to account for it. Go + * to the next one. + */ + if (!found) + pfn++; + } + + return count; +} + +static int hv_memory_notifier(struct notifier_block *nb, unsigned long val, + void *v) +{ + struct memory_notify *mem = (struct memory_notify *)v; + unsigned long pfn_count; + + switch (val) { + case MEM_ONLINE: + case MEM_CANCEL_ONLINE: + complete(&dm_device.ol_waitevent); + break; + + case MEM_OFFLINE: + scoped_guard(spinlock_irqsave, &dm_device.ha_lock) { + pfn_count = hv_page_offline_check(mem->start_pfn, + mem->nr_pages); + if (pfn_count <= dm_device.num_pages_onlined) { + dm_device.num_pages_onlined -= pfn_count; + } else { + /* + * We're offlining more pages than we + * managed to online. This is + * unexpected. In any case don't let + * num_pages_onlined wrap around zero. + */ + WARN_ON_ONCE(1); + dm_device.num_pages_onlined = 0; + } + } + break; + case MEM_GOING_ONLINE: + case MEM_GOING_OFFLINE: + case MEM_CANCEL_OFFLINE: + break; + } + return NOTIFY_OK; +} + +static struct notifier_block hv_memory_nb = { + .notifier_call = hv_memory_notifier, + .priority = 0 +}; + +/* Check if the particular page is backed and can be onlined and online it. */ +static void hv_page_online_one(struct hv_hotadd_state *has, struct page *pg) +{ + if (!has_pfn_is_backed(has, page_to_pfn(pg))) { + if (!PageOffline(pg)) + __SetPageOffline(pg); + return; + } else if (!PageOffline(pg)) + return; + + /* This frame is currently backed; online the page. */ + generic_online_page(pg, 0); + + lockdep_assert_held(&dm_device.ha_lock); + dm_device.num_pages_onlined++; +} + +static void hv_bring_pgs_online(struct hv_hotadd_state *has, + unsigned long start_pfn, unsigned long size) +{ + int i; + + pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn); + for (i = 0; i < size; i++) + hv_page_online_one(has, pfn_to_page(start_pfn + i)); +} + +static void hv_mem_hot_add(unsigned long start, unsigned long size, + unsigned long pfn_count, + struct hv_hotadd_state *has) +{ + int ret = 0; + int i, nid; + unsigned long start_pfn; + unsigned long processed_pfn; + unsigned long total_pfn = pfn_count; + + for (i = 0; i < (size/ha_pages_in_chunk); i++) { + start_pfn = start + (i * ha_pages_in_chunk); + + scoped_guard(spinlock_irqsave, &dm_device.ha_lock) { + has->ha_end_pfn += ha_pages_in_chunk; + processed_pfn = umin(total_pfn, ha_pages_in_chunk); + total_pfn -= processed_pfn; + has->covered_end_pfn += processed_pfn; + } + + reinit_completion(&dm_device.ol_waitevent); + + nid = memory_add_physaddr_to_nid(PFN_PHYS(start_pfn)); + ret = add_memory(nid, PFN_PHYS((start_pfn)), + HA_BYTES_IN_CHUNK, MHP_MERGE_RESOURCE); + + if (ret) { + pr_err("hot_add memory failed error is %d\n", ret); + if (ret == -EEXIST) { + /* + * This error indicates that the error + * is not a transient failure. This is the + * case where the guest's physical address map + * precludes hot adding memory. Stop all further + * memory hot-add. + */ + do_hot_add = false; + } + scoped_guard(spinlock_irqsave, &dm_device.ha_lock) { + has->ha_end_pfn -= ha_pages_in_chunk; + has->covered_end_pfn -= processed_pfn; + } + break; + } + + /* + * Wait for memory to get onlined. If the kernel onlined the + * memory when adding it, this will return directly. Otherwise, + * it will wait for user space to online the memory. This helps + * to avoid adding memory faster than it is getting onlined. As + * adding succeeded, it is ok to proceed even if the memory was + * not onlined in time. + */ + wait_for_completion_timeout(&dm_device.ol_waitevent, secs_to_jiffies(5)); + post_status(&dm_device); + } +} + +static void hv_online_page(struct page *pg, unsigned int order) +{ + struct hv_hotadd_state *has; + unsigned long pfn = page_to_pfn(pg); + + scoped_guard(spinlock_irqsave, &dm_device.ha_lock) { + list_for_each_entry(has, &dm_device.ha_region_list, list) { + /* The page belongs to a different HAS. */ + if (pfn < has->start_pfn || + (pfn + (1UL << order) > has->end_pfn)) + continue; + + hv_bring_pgs_online(has, pfn, 1UL << order); + return; + } + } + generic_online_page(pg, order); +} + +static int pfn_covered(unsigned long start_pfn, unsigned long pfn_cnt) +{ + struct hv_hotadd_state *has; + struct hv_hotadd_gap *gap; + unsigned long residual; + int ret = 0; + + guard(spinlock_irqsave)(&dm_device.ha_lock); + list_for_each_entry(has, &dm_device.ha_region_list, list) { + /* + * If the pfn range we are dealing with is not in the current + * "hot add block", move on. + */ + if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn) + continue; + + /* + * If the current start pfn is not where the covered_end + * is, create a gap and update covered_end_pfn. + */ + if (has->covered_end_pfn != start_pfn) { + gap = kzalloc_obj(struct hv_hotadd_gap, GFP_ATOMIC); + if (!gap) { + ret = -ENOMEM; + break; + } + + INIT_LIST_HEAD(&gap->list); + gap->start_pfn = has->covered_end_pfn; + gap->end_pfn = start_pfn; + list_add_tail(&gap->list, &has->gap_list); + + has->covered_end_pfn = start_pfn; + } + + /* + * If the current hot add-request extends beyond + * our current limit; extend it. + */ + if ((start_pfn + pfn_cnt) > has->end_pfn) { + /* Extend the region by multiples of ha_pages_in_chunk */ + residual = (start_pfn + pfn_cnt - has->end_pfn); + has->end_pfn += ALIGN(residual, ha_pages_in_chunk); + } + + ret = 1; + break; + } + + return ret; +} + +static unsigned long handle_pg_range(unsigned long pg_start, + unsigned long pg_count) +{ + unsigned long start_pfn = pg_start; + unsigned long pfn_cnt = pg_count; + unsigned long size; + struct hv_hotadd_state *has; + unsigned long pgs_ol = 0; + unsigned long old_covered_state; + unsigned long res = 0, flags; + + pr_debug("Hot adding %lu pages starting at pfn 0x%lx.\n", pg_count, + pg_start); + + spin_lock_irqsave(&dm_device.ha_lock, flags); + list_for_each_entry(has, &dm_device.ha_region_list, list) { + /* + * If the pfn range we are dealing with is not in the current + * "hot add block", move on. + */ + if (start_pfn < has->start_pfn || start_pfn >= has->end_pfn) + continue; + + old_covered_state = has->covered_end_pfn; + + if (start_pfn < has->ha_end_pfn) { + /* + * This is the case where we are backing pages + * in an already hot added region. Bring + * these pages online first. + */ + pgs_ol = has->ha_end_pfn - start_pfn; + if (pgs_ol > pfn_cnt) + pgs_ol = pfn_cnt; + + has->covered_end_pfn += pgs_ol; + pfn_cnt -= pgs_ol; + /* + * Check if the corresponding memory block is already + * online. It is possible to observe struct pages still + * being uninitialized here so check section instead. + * In case the section is online we need to bring the + * rest of pfns (which were not backed previously) + * online too. + */ + if (start_pfn > has->start_pfn && + online_section_nr(pfn_to_section_nr(start_pfn))) + hv_bring_pgs_online(has, start_pfn, pgs_ol); + } + + if (has->ha_end_pfn < has->end_pfn && pfn_cnt > 0) { + /* + * We have some residual hot add range + * that needs to be hot added; hot add + * it now. Hot add a multiple of + * ha_pages_in_chunk that fully covers the pages + * we have. + */ + size = (has->end_pfn - has->ha_end_pfn); + if (pfn_cnt <= size) { + size = ALIGN(pfn_cnt, ha_pages_in_chunk); + } else { + pfn_cnt = size; + } + spin_unlock_irqrestore(&dm_device.ha_lock, flags); + hv_mem_hot_add(has->ha_end_pfn, size, pfn_cnt, has); + spin_lock_irqsave(&dm_device.ha_lock, flags); + } + /* + * If we managed to online any pages that were given to us, + * we declare success. + */ + res = has->covered_end_pfn - old_covered_state; + break; + } + spin_unlock_irqrestore(&dm_device.ha_lock, flags); + + return res; +} + +static unsigned long process_hot_add(unsigned long pg_start, + unsigned long pfn_cnt, + unsigned long rg_start, + unsigned long rg_size) +{ + struct hv_hotadd_state *ha_region = NULL; + int covered; + + if (pfn_cnt == 0) + return 0; + + if (!dm_device.host_specified_ha_region) { + covered = pfn_covered(pg_start, pfn_cnt); + if (covered < 0) + return 0; + + if (covered) + goto do_pg_range; + } + + /* + * If the host has specified a hot-add range; deal with it first. + */ + + if (rg_size != 0) { + ha_region = kzalloc(sizeof(struct hv_hotadd_state), GFP_KERNEL); + if (!ha_region) + return 0; + + INIT_LIST_HEAD(&ha_region->list); + INIT_LIST_HEAD(&ha_region->gap_list); + + ha_region->start_pfn = rg_start; + ha_region->ha_end_pfn = rg_start; + ha_region->covered_start_pfn = pg_start; + ha_region->covered_end_pfn = pg_start; + ha_region->end_pfn = rg_start + rg_size; + + scoped_guard(spinlock_irqsave, &dm_device.ha_lock) { + list_add_tail(&ha_region->list, &dm_device.ha_region_list); + } + } + +do_pg_range: + /* + * Process the page range specified; bringing them + * online if possible. + */ + return handle_pg_range(pg_start, pfn_cnt); +} + +#endif + +static void hot_add_req(struct work_struct *dummy) +{ + struct dm_hot_add_response resp; +#ifdef CONFIG_MEMORY_HOTPLUG + unsigned long pg_start, pfn_cnt; + unsigned long rg_start, rg_sz; +#endif + struct hv_dynmem_device *dm = &dm_device; + + memset(&resp, 0, sizeof(struct dm_hot_add_response)); + resp.hdr.type = DM_MEM_HOT_ADD_RESPONSE; + resp.hdr.size = sizeof(struct dm_hot_add_response); + +#ifdef CONFIG_MEMORY_HOTPLUG + pg_start = dm->ha_wrk.ha_page_range.finfo.start_page; + pfn_cnt = dm->ha_wrk.ha_page_range.finfo.page_cnt; + + rg_start = dm->ha_wrk.ha_region_range.finfo.start_page; + rg_sz = dm->ha_wrk.ha_region_range.finfo.page_cnt; + + if (rg_start == 0 && !dm->host_specified_ha_region) { + /* + * The host has not specified the hot-add region. + * Based on the hot-add page range being specified, + * compute a hot-add region that can cover the pages + * that need to be hot-added while ensuring the alignment + * and size requirements of Linux as it relates to hot-add. + */ + rg_start = ALIGN_DOWN(pg_start, ha_pages_in_chunk); + rg_sz = ALIGN(pfn_cnt, ha_pages_in_chunk); + } + + if (do_hot_add) + resp.page_count = process_hot_add(pg_start, pfn_cnt, + rg_start, rg_sz); + + dm->num_pages_added += resp.page_count; +#endif + /* + * The result field of the response structure has the + * following semantics: + * + * 1. If all or some pages hot-added: Guest should return success. + * + * 2. If no pages could be hot-added: + * + * If the guest returns success, then the host + * will not attempt any further hot-add operations. This + * signifies a permanent failure. + * + * If the guest returns failure, then this failure will be + * treated as a transient failure and the host may retry the + * hot-add operation after some delay. + */ + if (resp.page_count > 0) + resp.result = 1; + else if (!do_hot_add) + resp.result = 1; + else + resp.result = 0; + + if (!do_hot_add || resp.page_count == 0) { + if (!allow_hibernation) + pr_err("Memory hot add failed\n"); + else + pr_info("Ignore hot-add request!\n"); + } + + dm->state = DM_INITIALIZED; + resp.hdr.trans_id = atomic_inc_return(&trans_id); + vmbus_sendpacket(dm->dev->channel, &resp, + sizeof(struct dm_hot_add_response), + (unsigned long)NULL, + VM_PKT_DATA_INBAND, 0); +} + +static void process_info(struct hv_dynmem_device *dm, struct dm_info_msg *msg) +{ + struct dm_info_header *info_hdr; + + info_hdr = (struct dm_info_header *)msg->info; + + switch (info_hdr->type) { + case INFO_TYPE_MAX_PAGE_CNT: + if (info_hdr->data_size == sizeof(__u64)) { + __u64 *max_page_count = (__u64 *)&info_hdr[1]; + + pr_info("Max. dynamic memory size: %llu MB\n", + (*max_page_count) >> (20 - HV_HYP_PAGE_SHIFT)); + dm->max_dynamic_page_count = *max_page_count; + } + + break; + default: + pr_warn("Received Unknown type: %d\n", info_hdr->type); + } +} + +static unsigned long compute_balloon_floor(void) +{ + unsigned long min_pages; + unsigned long nr_pages = totalram_pages(); +#define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT)) + /* Simple continuous piecewiese linear function: + * max MiB -> min MiB gradient + * 0 0 + * 16 16 + * 32 24 + * 128 72 (1/2) + * 512 168 (1/4) + * 2048 360 (1/8) + * 8192 744 (1/16) + * 32768 1512 (1/32) + */ + if (nr_pages < MB2PAGES(128)) + min_pages = MB2PAGES(8) + (nr_pages >> 1); + else if (nr_pages < MB2PAGES(512)) + min_pages = MB2PAGES(40) + (nr_pages >> 2); + else if (nr_pages < MB2PAGES(2048)) + min_pages = MB2PAGES(104) + (nr_pages >> 3); + else if (nr_pages < MB2PAGES(8192)) + min_pages = MB2PAGES(232) + (nr_pages >> 4); + else + min_pages = MB2PAGES(488) + (nr_pages >> 5); +#undef MB2PAGES + return min_pages; +} + +/* + * Compute total committed memory pages + */ + +static unsigned long get_pages_committed(struct hv_dynmem_device *dm) +{ + return vm_memory_committed() + + dm->num_pages_ballooned + + (dm->num_pages_added > dm->num_pages_onlined ? + dm->num_pages_added - dm->num_pages_onlined : 0) + + compute_balloon_floor(); +} + +/* + * Post our status as it relates memory pressure to the + * host. Host expects the guests to post this status + * periodically at 1 second intervals. + * + * The metrics specified in this protocol are very Windows + * specific and so we cook up numbers here to convey our memory + * pressure. + */ + +static void post_status(struct hv_dynmem_device *dm) +{ + struct dm_status status; + unsigned long now = jiffies; + unsigned long last_post = last_post_time; + unsigned long num_pages_avail, num_pages_committed; + + if (pressure_report_delay > 0) { + --pressure_report_delay; + return; + } + + if (!time_after(now, (last_post_time + HZ))) + return; + + memset(&status, 0, sizeof(struct dm_status)); + status.hdr.type = DM_STATUS_REPORT; + status.hdr.size = sizeof(struct dm_status); + status.hdr.trans_id = atomic_inc_return(&trans_id); + + /* + * The host expects the guest to report free and committed memory. + * Furthermore, the host expects the pressure information to include + * the ballooned out pages. For a given amount of memory that we are + * managing we need to compute a floor below which we should not + * balloon. Compute this and add it to the pressure report. + * We also need to report all offline pages (num_pages_added - + * num_pages_onlined) as committed to the host, otherwise it can try + * asking us to balloon them out. + */ + num_pages_avail = si_mem_available(); + num_pages_committed = get_pages_committed(dm); + + trace_balloon_status(num_pages_avail, num_pages_committed, + vm_memory_committed(), dm->num_pages_ballooned, + dm->num_pages_added, dm->num_pages_onlined); + + /* Convert numbers of pages into numbers of HV_HYP_PAGEs. */ + status.num_avail = num_pages_avail * NR_HV_HYP_PAGES_IN_PAGE; + status.num_committed = num_pages_committed * NR_HV_HYP_PAGES_IN_PAGE; + + /* + * If our transaction ID is no longer current, just don't + * send the status. This can happen if we were interrupted + * after we picked our transaction ID. + */ + if (status.hdr.trans_id != atomic_read(&trans_id)) + return; + + /* + * If the last post time that we sampled has changed, + * we have raced, don't post the status. + */ + if (last_post != last_post_time) + return; + + last_post_time = jiffies; + vmbus_sendpacket(dm->dev->channel, &status, + sizeof(struct dm_status), + (unsigned long)NULL, + VM_PKT_DATA_INBAND, 0); +} + +static void free_balloon_pages(struct hv_dynmem_device *dm, + union dm_mem_page_range *range_array) +{ + int num_pages = range_array->finfo.page_cnt; + __u64 start_frame = range_array->finfo.start_page; + struct page *pg; + int i; + + for (i = 0; i < num_pages; i++) { + pg = pfn_to_page(i + start_frame); + __ClearPageOffline(pg); + __free_page(pg); + dm->num_pages_ballooned--; + mod_node_page_state(page_pgdat(pg), NR_BALLOON_PAGES, -1); + adjust_managed_page_count(pg, 1); + } +} + +static unsigned int alloc_balloon_pages(struct hv_dynmem_device *dm, + unsigned int num_pages, + struct dm_balloon_response *bl_resp, + int alloc_unit) +{ + unsigned int i, j; + struct page *pg; + + for (i = 0; i < num_pages / alloc_unit; i++) { + if (bl_resp->hdr.size + sizeof(union dm_mem_page_range) > + HV_HYP_PAGE_SIZE) + return i * alloc_unit; + + /* + * We execute this code in a thread context. Furthermore, + * we don't want the kernel to try too hard. + */ + pg = alloc_pages(GFP_HIGHUSER | __GFP_NORETRY | + __GFP_NOMEMALLOC | __GFP_NOWARN, + get_order(alloc_unit << PAGE_SHIFT)); + + if (!pg) + return i * alloc_unit; + + dm->num_pages_ballooned += alloc_unit; + mod_node_page_state(page_pgdat(pg), NR_BALLOON_PAGES, alloc_unit); + + /* + * If we allocatted 2M pages; split them so we + * can free them in any order we get. + */ + + if (alloc_unit != 1) + split_page(pg, get_order(alloc_unit << PAGE_SHIFT)); + + /* mark all pages offline */ + for (j = 0; j < alloc_unit; j++) { + __SetPageOffline(pg + j); + adjust_managed_page_count(pg + j, -1); + } + + bl_resp->range_count++; + bl_resp->range_array[i].finfo.start_page = + page_to_pfn(pg); + bl_resp->range_array[i].finfo.page_cnt = alloc_unit; + bl_resp->hdr.size += sizeof(union dm_mem_page_range); + } + + return i * alloc_unit; +} + +static void balloon_up(struct work_struct *dummy) +{ + unsigned int num_pages = dm_device.balloon_wrk.num_pages; + unsigned int num_ballooned = 0; + struct dm_balloon_response *bl_resp; + int alloc_unit; + int ret; + bool done = false; + int i; + long avail_pages; + unsigned long floor; + + /* + * We will attempt 2M allocations. However, if we fail to + * allocate 2M chunks, we will go back to PAGE_SIZE allocations. + */ + alloc_unit = PAGES_IN_2M; + + avail_pages = si_mem_available(); + floor = compute_balloon_floor(); + + /* Refuse to balloon below the floor. */ + if (avail_pages < num_pages || avail_pages - num_pages < floor) { + pr_info("Balloon request will be partially fulfilled. %s\n", + avail_pages < num_pages ? "Not enough memory." : + "Balloon floor reached."); + + num_pages = avail_pages > floor ? (avail_pages - floor) : 0; + } + + while (!done) { + memset(balloon_up_send_buffer, 0, HV_HYP_PAGE_SIZE); + bl_resp = (struct dm_balloon_response *)balloon_up_send_buffer; + bl_resp->hdr.type = DM_BALLOON_RESPONSE; + bl_resp->hdr.size = sizeof(struct dm_balloon_response); + bl_resp->more_pages = 1; + + num_pages -= num_ballooned; + num_ballooned = alloc_balloon_pages(&dm_device, num_pages, + bl_resp, alloc_unit); + + if (alloc_unit != 1 && num_ballooned == 0) { + alloc_unit = 1; + continue; + } + + if (num_ballooned == 0 || num_ballooned == num_pages) { + pr_debug("Ballooned %u out of %u requested pages.\n", + num_pages, dm_device.balloon_wrk.num_pages); + + bl_resp->more_pages = 0; + done = true; + dm_device.state = DM_INITIALIZED; + } + + /* + * We are pushing a lot of data through the channel; + * deal with transient failures caused because of the + * lack of space in the ring buffer. + */ + + do { + bl_resp->hdr.trans_id = atomic_inc_return(&trans_id); + ret = vmbus_sendpacket(dm_device.dev->channel, + bl_resp, + bl_resp->hdr.size, + (unsigned long)NULL, + VM_PKT_DATA_INBAND, 0); + + if (ret == -EAGAIN) + msleep(20); + post_status(&dm_device); + } while (ret == -EAGAIN); + + if (ret) { + /* + * Free up the memory we allocatted. + */ + pr_err("Balloon response failed\n"); + + for (i = 0; i < bl_resp->range_count; i++) + free_balloon_pages(&dm_device, + &bl_resp->range_array[i]); + + done = true; + } + } +} + +static void balloon_down(struct hv_dynmem_device *dm, + struct dm_unballoon_request *req) +{ + union dm_mem_page_range *range_array = req->range_array; + int range_count = req->range_count; + struct dm_unballoon_response resp; + int i; + unsigned int prev_pages_ballooned = dm->num_pages_ballooned; + + for (i = 0; i < range_count; i++) { + free_balloon_pages(dm, &range_array[i]); + complete(&dm_device.config_event); + } + + pr_debug("Freed %u ballooned pages.\n", + prev_pages_ballooned - dm->num_pages_ballooned); + + if (req->more_pages == 1) + return; + + memset(&resp, 0, sizeof(struct dm_unballoon_response)); + resp.hdr.type = DM_UNBALLOON_RESPONSE; + resp.hdr.trans_id = atomic_inc_return(&trans_id); + resp.hdr.size = sizeof(struct dm_unballoon_response); + + vmbus_sendpacket(dm_device.dev->channel, &resp, + sizeof(struct dm_unballoon_response), + (unsigned long)NULL, + VM_PKT_DATA_INBAND, 0); + + dm->state = DM_INITIALIZED; +} + +static void balloon_onchannelcallback(void *context); + +static int dm_thread_func(void *dm_dev) +{ + struct hv_dynmem_device *dm = dm_dev; + + while (!kthread_should_stop()) { + wait_for_completion_interruptible_timeout(&dm_device.config_event, + secs_to_jiffies(1)); + /* + * The host expects us to post information on the memory + * pressure every second. + */ + reinit_completion(&dm_device.config_event); + post_status(dm); + /* + * disable free page reporting if multiple hypercall + * failure flag set. It is not done in the page_reporting + * callback context as that causes a deadlock between + * page_reporting_process() and page_reporting_unregister() + */ + if (hv_hypercall_multi_failure >= HV_MAX_FAILURES) { + pr_err("Multiple failures in cold memory discard hypercall, disabling page reporting\n"); + disable_page_reporting(); + /* Reset the flag after disabling reporting */ + hv_hypercall_multi_failure = 0; + } + } + + return 0; +} + +static void version_resp(struct hv_dynmem_device *dm, + struct dm_version_response *vresp) +{ + struct dm_version_request version_req; + int ret; + + if (vresp->is_accepted) { + /* + * We are done; wakeup the + * context waiting for version + * negotiation. + */ + complete(&dm->host_event); + return; + } + /* + * If there are more versions to try, continue + * with negotiations; if not + * shutdown the service since we are not able + * to negotiate a suitable version number + * with the host. + */ + if (dm->next_version == 0) + goto version_error; + + memset(&version_req, 0, sizeof(struct dm_version_request)); + version_req.hdr.type = DM_VERSION_REQUEST; + version_req.hdr.size = sizeof(struct dm_version_request); + version_req.hdr.trans_id = atomic_inc_return(&trans_id); + version_req.version.version = dm->next_version; + dm->version = version_req.version.version; + + /* + * Set the next version to try in case current version fails. + * Win7 protocol ought to be the last one to try. + */ + switch (version_req.version.version) { + case DYNMEM_PROTOCOL_VERSION_WIN8: + dm->next_version = DYNMEM_PROTOCOL_VERSION_WIN7; + version_req.is_last_attempt = 0; + break; + default: + dm->next_version = 0; + version_req.is_last_attempt = 1; + } + + ret = vmbus_sendpacket(dm->dev->channel, &version_req, + sizeof(struct dm_version_request), + (unsigned long)NULL, + VM_PKT_DATA_INBAND, 0); + + if (ret) + goto version_error; + + return; + +version_error: + dm->state = DM_INIT_ERROR; + complete(&dm->host_event); +} + +static void cap_resp(struct hv_dynmem_device *dm, + struct dm_capabilities_resp_msg *cap_resp) +{ + if (!cap_resp->is_accepted) { + pr_err("Capabilities not accepted by host\n"); + dm->state = DM_INIT_ERROR; + } + complete(&dm->host_event); +} + +static void balloon_onchannelcallback(void *context) +{ + struct hv_device *dev = context; + u32 recvlen; + u64 requestid; + struct dm_message *dm_msg; + struct dm_header *dm_hdr; + struct hv_dynmem_device *dm = hv_get_drvdata(dev); + struct dm_balloon *bal_msg; + struct dm_hot_add *ha_msg; + union dm_mem_page_range *ha_pg_range; + union dm_mem_page_range *ha_region; + + memset(recv_buffer, 0, sizeof(recv_buffer)); + vmbus_recvpacket(dev->channel, recv_buffer, + HV_HYP_PAGE_SIZE, &recvlen, &requestid); + + if (recvlen > 0) { + dm_msg = (struct dm_message *)recv_buffer; + dm_hdr = &dm_msg->hdr; + + switch (dm_hdr->type) { + case DM_VERSION_RESPONSE: + version_resp(dm, + (struct dm_version_response *)dm_msg); + break; + + case DM_CAPABILITIES_RESPONSE: + cap_resp(dm, + (struct dm_capabilities_resp_msg *)dm_msg); + break; + + case DM_BALLOON_REQUEST: + if (allow_hibernation) { + pr_info("Ignore balloon-up request!\n"); + break; + } + + if (dm->state == DM_BALLOON_UP) + pr_warn("Currently ballooning\n"); + bal_msg = (struct dm_balloon *)recv_buffer; + dm->state = DM_BALLOON_UP; + dm_device.balloon_wrk.num_pages = bal_msg->num_pages; + schedule_work(&dm_device.balloon_wrk.wrk); + break; + + case DM_UNBALLOON_REQUEST: + if (allow_hibernation) { + pr_info("Ignore balloon-down request!\n"); + break; + } + + dm->state = DM_BALLOON_DOWN; + balloon_down(dm, + (struct dm_unballoon_request *)recv_buffer); + break; + + case DM_MEM_HOT_ADD_REQUEST: + if (dm->state == DM_HOT_ADD) + pr_warn("Currently hot-adding\n"); + dm->state = DM_HOT_ADD; + ha_msg = (struct dm_hot_add *)recv_buffer; + if (ha_msg->hdr.size == sizeof(struct dm_hot_add)) { + /* + * This is a normal hot-add request specifying + * hot-add memory. + */ + dm->host_specified_ha_region = false; + ha_pg_range = &ha_msg->range; + dm->ha_wrk.ha_page_range = *ha_pg_range; + dm->ha_wrk.ha_region_range.page_range = 0; + } else { + /* + * Host is specifying that we first hot-add + * a region and then partially populate this + * region. + */ + dm->host_specified_ha_region = true; + ha_pg_range = &ha_msg->range; + ha_region = &ha_pg_range[1]; + dm->ha_wrk.ha_page_range = *ha_pg_range; + dm->ha_wrk.ha_region_range = *ha_region; + } + schedule_work(&dm_device.ha_wrk.wrk); + break; + + case DM_INFO_MESSAGE: + process_info(dm, (struct dm_info_msg *)dm_msg); + break; + + default: + pr_warn_ratelimited("Unhandled message: type: %d\n", dm_hdr->type); + } + } +} + +#define HV_LARGE_REPORTING_ORDER 9 +#define HV_LARGE_REPORTING_LEN (HV_HYP_PAGE_SIZE << \ + HV_LARGE_REPORTING_ORDER) +static int hv_free_page_report(struct page_reporting_dev_info *pr_dev_info, + struct scatterlist *sgl, unsigned int nents) +{ + unsigned long flags; + struct hv_memory_hint *hint; + int i, order; + u64 status; + struct scatterlist *sg; + + WARN_ON_ONCE(nents > HV_MEMORY_HINT_MAX_GPA_PAGE_RANGES); + WARN_ON_ONCE(sgl->length < (HV_HYP_PAGE_SIZE << page_reporting_order)); + local_irq_save(flags); + hint = *this_cpu_ptr(hyperv_pcpu_input_arg); + if (!hint) { + local_irq_restore(flags); + return -ENOSPC; + } + + hint->heat_type = HV_EXTMEM_HEAT_HINT_COLD_DISCARD; + hint->reserved = 0; + for_each_sg(sgl, sg, nents, i) { + union hv_gpa_page_range *range; + + range = &hint->ranges[i]; + range->address_space = 0; + order = get_order(sg->length); + /* + * Hyper-V expects the additional_pages field in the units + * of one of these 3 sizes, 4Kbytes, 2Mbytes or 1Gbytes. + * This is dictated by the values of the fields page.largesize + * and page_size. + * This code however, only uses 4Kbytes and 2Mbytes units + * and not 1Gbytes unit. + */ + + /* page reporting for pages 2MB or higher */ + if (order >= HV_LARGE_REPORTING_ORDER) { + range->page.largepage = 1; + range->page_size = HV_GPA_PAGE_RANGE_PAGE_SIZE_2MB; + range->base_large_pfn = page_to_hvpfn( + sg_page(sg)) >> HV_LARGE_REPORTING_ORDER; + range->page.additional_pages = + (sg->length / HV_LARGE_REPORTING_LEN) - 1; + } else { + /* Page reporting for pages below 2MB */ + range->page.basepfn = page_to_hvpfn(sg_page(sg)); + range->page.largepage = false; + range->page.additional_pages = + (sg->length / HV_HYP_PAGE_SIZE) - 1; + } + } + + status = hv_do_rep_hypercall(HV_EXT_CALL_MEMORY_HEAT_HINT, nents, 0, + hint, NULL); + local_irq_restore(flags); + if (!hv_result_success(status)) { + pr_err("Cold memory discard hypercall failed with status %llx\n", + status); + if (hv_hypercall_multi_failure > 0) + hv_hypercall_multi_failure++; + + if (hv_result(status) == HV_STATUS_INVALID_PARAMETER) { + pr_err("Underlying Hyper-V does not support order less than 9. Hypercall failed\n"); + pr_err("Defaulting to page_reporting_order %d\n", + pageblock_order); + page_reporting_order = pageblock_order; + hv_hypercall_multi_failure++; + return -EINVAL; + } + + return -EINVAL; + } + + return 0; +} + +static void enable_page_reporting(void) +{ + int ret; + + if (!hv_query_ext_cap(HV_EXT_CAPABILITY_MEMORY_COLD_DISCARD_HINT)) { + pr_debug("Cold memory discard hint not supported by Hyper-V\n"); + return; + } + + BUILD_BUG_ON(PAGE_REPORTING_CAPACITY > HV_MEMORY_HINT_MAX_GPA_PAGE_RANGES); + dm_device.pr_dev_info.report = hv_free_page_report; + /* + * We let the page_reporting_order parameter decide the order + * in the page_reporting code + */ + dm_device.pr_dev_info.order = PAGE_REPORTING_ORDER_UNSPECIFIED; + ret = page_reporting_register(&dm_device.pr_dev_info); + if (ret < 0) { + dm_device.pr_dev_info.report = NULL; + pr_err("Failed to enable cold memory discard: %d\n", ret); + } else { + pr_info("Cold memory discard hint enabled with order %d\n", + page_reporting_order); + } +} + +static void disable_page_reporting(void) +{ + if (dm_device.pr_dev_info.report) { + page_reporting_unregister(&dm_device.pr_dev_info); + dm_device.pr_dev_info.report = NULL; + } +} + +static int ballooning_enabled(void) +{ + /* + * Disable ballooning if the page size is not 4k (HV_HYP_PAGE_SIZE), + * since currently it's unclear to us whether an unballoon request can + * make sure all page ranges are guest page size aligned. + */ + if (PAGE_SIZE != HV_HYP_PAGE_SIZE) { + pr_info("Ballooning disabled because page size is not 4096 bytes\n"); + return 0; + } + + return 1; +} + +static int hot_add_enabled(void) +{ + /* + * Disable hot add on ARM64, because we currently rely on + * memory_add_physaddr_to_nid() to get a node id of a hot add range, + * however ARM64's memory_add_physaddr_to_nid() always return 0 and + * DM_MEM_HOT_ADD_REQUEST doesn't have the NUMA node information for + * add_memory(). + */ + if (IS_ENABLED(CONFIG_ARM64)) { + pr_info("Memory hot add disabled on ARM64\n"); + return 0; + } + + return 1; +} + +static int balloon_connect_vsp(struct hv_device *dev) +{ + struct dm_version_request version_req; + struct dm_capabilities cap_msg; + unsigned long t; + int ret; + + /* + * max_pkt_size should be large enough for one vmbus packet header plus + * our receive buffer size. Hyper-V sends messages up to + * HV_HYP_PAGE_SIZE bytes long on balloon channel. + */ + dev->channel->max_pkt_size = HV_HYP_PAGE_SIZE * 2; + + ret = vmbus_open(dev->channel, dm_ring_size, dm_ring_size, NULL, 0, + balloon_onchannelcallback, dev); + if (ret) + return ret; + + /* + * Initiate the hand shake with the host and negotiate + * a version that the host can support. We start with the + * highest version number and go down if the host cannot + * support it. + */ + memset(&version_req, 0, sizeof(struct dm_version_request)); + version_req.hdr.type = DM_VERSION_REQUEST; + version_req.hdr.size = sizeof(struct dm_version_request); + version_req.hdr.trans_id = atomic_inc_return(&trans_id); + version_req.version.version = DYNMEM_PROTOCOL_VERSION_WIN10; + version_req.is_last_attempt = 0; + dm_device.version = version_req.version.version; + + ret = vmbus_sendpacket(dev->channel, &version_req, + sizeof(struct dm_version_request), + (unsigned long)NULL, VM_PKT_DATA_INBAND, 0); + if (ret) + goto out; + + t = wait_for_completion_timeout(&dm_device.host_event, secs_to_jiffies(5)); + if (t == 0) { + ret = -ETIMEDOUT; + goto out; + } + + /* + * If we could not negotiate a compatible version with the host + * fail the probe function. + */ + if (dm_device.state == DM_INIT_ERROR) { + ret = -EPROTO; + goto out; + } + + pr_info("Using Dynamic Memory protocol version %u.%u\n", + DYNMEM_MAJOR_VERSION(dm_device.version), + DYNMEM_MINOR_VERSION(dm_device.version)); + + /* + * Now submit our capabilities to the host. + */ + memset(&cap_msg, 0, sizeof(struct dm_capabilities)); + cap_msg.hdr.type = DM_CAPABILITIES_REPORT; + cap_msg.hdr.size = sizeof(struct dm_capabilities); + cap_msg.hdr.trans_id = atomic_inc_return(&trans_id); + + /* + * When hibernation (i.e. virtual ACPI S4 state) is enabled, the host + * currently still requires the bits to be set, so we have to add code + * to fail the host's hot-add and balloon up/down requests, if any. + */ + cap_msg.caps.cap_bits.balloon = ballooning_enabled(); + cap_msg.caps.cap_bits.hot_add = hot_add_enabled(); + + /* + * Specify our alignment requirements for memory hot-add. The value is + * the log base 2 of the number of megabytes in a chunk. For example, + * with 256 MiB chunks, the value is 8. The number of MiB in a chunk + * must be a power of 2. + */ + cap_msg.caps.cap_bits.hot_add_alignment = + ilog2(HA_BYTES_IN_CHUNK / SZ_1M); + + /* + * Currently the host does not use these + * values and we set them to what is done in the + * Windows driver. + */ + cap_msg.min_page_cnt = 0; + cap_msg.max_page_number = -1; + + ret = vmbus_sendpacket(dev->channel, &cap_msg, + sizeof(struct dm_capabilities), + (unsigned long)NULL, VM_PKT_DATA_INBAND, 0); + if (ret) + goto out; + + t = wait_for_completion_timeout(&dm_device.host_event, secs_to_jiffies(5)); + if (t == 0) { + ret = -ETIMEDOUT; + goto out; + } + + /* + * If the host does not like our capabilities, + * fail the probe function. + */ + if (dm_device.state == DM_INIT_ERROR) { + ret = -EPROTO; + goto out; + } + + return 0; +out: + vmbus_close(dev->channel); + return ret; +} + +/* + * DEBUGFS Interface + */ +#ifdef CONFIG_DEBUG_FS + +/** + * hv_balloon_debug_show - shows statistics of balloon operations. + * @f: pointer to the &struct seq_file. + * @offset: ignored. + * + * Provides the statistics that can be accessed in hv-balloon in the debugfs. + * + * Return: zero on success or an error code. + */ +static int hv_balloon_debug_show(struct seq_file *f, void *offset) +{ + struct hv_dynmem_device *dm = f->private; + char *sname; + + seq_printf(f, "%-22s: %u.%u\n", "host_version", + DYNMEM_MAJOR_VERSION(dm->version), + DYNMEM_MINOR_VERSION(dm->version)); + + seq_printf(f, "%-22s:", "capabilities"); + if (ballooning_enabled()) + seq_puts(f, " enabled"); + + if (hot_add_enabled()) + seq_puts(f, " hot_add"); + + seq_printf(f, "\n%-22s: %u", "state", dm->state); + switch (dm->state) { + case DM_INITIALIZING: + sname = "Initializing"; + break; + case DM_INITIALIZED: + sname = "Initialized"; + break; + case DM_BALLOON_UP: + sname = "Balloon Up"; + break; + case DM_BALLOON_DOWN: + sname = "Balloon Down"; + break; + case DM_HOT_ADD: + sname = "Hot Add"; + break; + case DM_INIT_ERROR: + sname = "Error"; + break; + default: + sname = "Unknown"; + } + seq_printf(f, " (%s)\n", sname); + + /* HV Page Size */ + seq_printf(f, "%-22s: %ld\n", "page_size", HV_HYP_PAGE_SIZE); + + /* Pages added with hot_add */ + seq_printf(f, "%-22s: %u\n", "pages_added", dm->num_pages_added); + + /* pages that are "onlined"/used from pages_added */ + seq_printf(f, "%-22s: %u\n", "pages_onlined", dm->num_pages_onlined); + + /* pages we have given back to host */ + seq_printf(f, "%-22s: %u\n", "pages_ballooned", dm->num_pages_ballooned); + + seq_printf(f, "%-22s: %lu\n", "total_pages_committed", + get_pages_committed(dm)); + + seq_printf(f, "%-22s: %llu\n", "max_dynamic_page_count", + dm->max_dynamic_page_count); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hv_balloon_debug); + +static void hv_balloon_debugfs_init(struct hv_dynmem_device *b) +{ + debugfs_create_file("hv-balloon", 0444, NULL, b, + &hv_balloon_debug_fops); +} + +static void hv_balloon_debugfs_exit(struct hv_dynmem_device *b) +{ + debugfs_lookup_and_remove("hv-balloon", NULL); +} + +#else + +static inline void hv_balloon_debugfs_init(struct hv_dynmem_device *b) +{ +} + +static inline void hv_balloon_debugfs_exit(struct hv_dynmem_device *b) +{ +} + +#endif /* CONFIG_DEBUG_FS */ + +static int balloon_probe(struct hv_device *dev, + const struct hv_vmbus_device_id *dev_id) +{ + int ret; + + allow_hibernation = hv_is_hibernation_supported(); + if (allow_hibernation) + hot_add = false; + +#ifdef CONFIG_MEMORY_HOTPLUG + /* + * Hot-add must operate in chunks that are of size equal to the + * memory block size because that's what the core add_memory() + * interface requires. The Hyper-V interface requires that the memory + * block size be a power of 2, which is guaranteed by the check in + * memory_dev_init(). + */ + ha_pages_in_chunk = memory_block_size_bytes() / PAGE_SIZE; + do_hot_add = hot_add; +#else + /* + * Without MEMORY_HOTPLUG, the guest returns a failure status for all + * hot add requests from Hyper-V, and the chunk size is used only to + * specify alignment to Hyper-V as required by the host/guest protocol. + * Somewhat arbitrarily, use 128 MiB. + */ + ha_pages_in_chunk = SZ_128M / PAGE_SIZE; + do_hot_add = false; +#endif + dm_device.dev = dev; + dm_device.state = DM_INITIALIZING; + dm_device.next_version = DYNMEM_PROTOCOL_VERSION_WIN8; + init_completion(&dm_device.host_event); + init_completion(&dm_device.config_event); + INIT_LIST_HEAD(&dm_device.ha_region_list); + spin_lock_init(&dm_device.ha_lock); + INIT_WORK(&dm_device.balloon_wrk.wrk, balloon_up); + INIT_WORK(&dm_device.ha_wrk.wrk, hot_add_req); + dm_device.host_specified_ha_region = false; + +#ifdef CONFIG_MEMORY_HOTPLUG + set_online_page_callback(&hv_online_page); + init_completion(&dm_device.ol_waitevent); + register_memory_notifier(&hv_memory_nb); +#endif + + hv_set_drvdata(dev, &dm_device); + + ret = balloon_connect_vsp(dev); + if (ret != 0) + goto connect_error; + + enable_page_reporting(); + dm_device.state = DM_INITIALIZED; + + dm_device.thread = + kthread_run(dm_thread_func, &dm_device, "hv_balloon"); + if (IS_ERR(dm_device.thread)) { + ret = PTR_ERR(dm_device.thread); + goto probe_error; + } + + hv_balloon_debugfs_init(&dm_device); + + return 0; + +probe_error: + dm_device.state = DM_INIT_ERROR; + dm_device.thread = NULL; + disable_page_reporting(); + vmbus_close(dev->channel); +connect_error: +#ifdef CONFIG_MEMORY_HOTPLUG + unregister_memory_notifier(&hv_memory_nb); + restore_online_page_callback(&hv_online_page); +#endif + return ret; +} + +static void balloon_remove(struct hv_device *dev) +{ + struct hv_dynmem_device *dm = hv_get_drvdata(dev); + struct hv_hotadd_state *has, *tmp; + struct hv_hotadd_gap *gap, *tmp_gap; + + if (dm->num_pages_ballooned != 0) + pr_warn("Ballooned pages: %d\n", dm->num_pages_ballooned); + + hv_balloon_debugfs_exit(dm); + + cancel_work_sync(&dm->balloon_wrk.wrk); + cancel_work_sync(&dm->ha_wrk.wrk); + + kthread_stop(dm->thread); + + /* + * This is to handle the case when balloon_resume() + * call has failed and some cleanup has been done as + * a part of the error handling. + */ + if (dm_device.state != DM_INIT_ERROR) { + disable_page_reporting(); + vmbus_close(dev->channel); +#ifdef CONFIG_MEMORY_HOTPLUG + unregister_memory_notifier(&hv_memory_nb); + restore_online_page_callback(&hv_online_page); +#endif + } + + guard(spinlock_irqsave)(&dm_device.ha_lock); + list_for_each_entry_safe(has, tmp, &dm->ha_region_list, list) { + list_for_each_entry_safe(gap, tmp_gap, &has->gap_list, list) { + list_del(&gap->list); + kfree(gap); + } + list_del(&has->list); + kfree(has); + } +} + +static int balloon_suspend(struct hv_device *hv_dev) +{ + struct hv_dynmem_device *dm = hv_get_drvdata(hv_dev); + + tasklet_disable(&hv_dev->channel->callback_event); + + cancel_work_sync(&dm->balloon_wrk.wrk); + cancel_work_sync(&dm->ha_wrk.wrk); + + if (dm->thread) { + kthread_stop(dm->thread); + dm->thread = NULL; + vmbus_close(hv_dev->channel); + } + + tasklet_enable(&hv_dev->channel->callback_event); + + return 0; +} + +static int balloon_resume(struct hv_device *dev) +{ + int ret; + + dm_device.state = DM_INITIALIZING; + + ret = balloon_connect_vsp(dev); + + if (ret != 0) + goto out; + + dm_device.thread = + kthread_run(dm_thread_func, &dm_device, "hv_balloon"); + if (IS_ERR(dm_device.thread)) { + ret = PTR_ERR(dm_device.thread); + dm_device.thread = NULL; + goto close_channel; + } + + dm_device.state = DM_INITIALIZED; + return 0; +close_channel: + vmbus_close(dev->channel); +out: + dm_device.state = DM_INIT_ERROR; + disable_page_reporting(); +#ifdef CONFIG_MEMORY_HOTPLUG + unregister_memory_notifier(&hv_memory_nb); + restore_online_page_callback(&hv_online_page); +#endif + return ret; +} + +static const struct hv_vmbus_device_id id_table[] = { + /* Dynamic Memory Class ID */ + /* 525074DC-8985-46e2-8057-A307DC18A502 */ + { HV_DM_GUID, }, + { }, +}; + +MODULE_DEVICE_TABLE(vmbus, id_table); + +static struct hv_driver balloon_drv = { + .name = "hv_balloon", + .id_table = id_table, + .probe = balloon_probe, + .remove = balloon_remove, + .suspend = balloon_suspend, + .resume = balloon_resume, + .driver = { + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +static int __init init_balloon_drv(void) +{ + return vmbus_driver_register(&balloon_drv); +} + +module_init(init_balloon_drv); + +MODULE_DESCRIPTION("Hyper-V Balloon"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hv/hv_common.c b/drivers/hv/hv_common.c new file mode 100644 index 000000000..6b67ac616 --- /dev/null +++ b/drivers/hv/hv_common.c @@ -0,0 +1,865 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* + * Architecture neutral utility routines for interacting with + * Hyper-V. This file is specifically for code that must be + * built-in to the kernel image when CONFIG_HYPERV is set + * (vs. being in a module) because it is called from architecture + * specific code under arch/. + * + * Copyright (C) 2021, Microsoft, Inc. + * + * Author : Michael Kelley <mikelley@microsoft.com> + */ + +#include <linux/types.h> +#include <linux/acpi.h> +#include <linux/export.h> +#include <linux/bitfield.h> +#include <linux/cpumask.h> +#include <linux/sched/task_stack.h> +#include <linux/panic_notifier.h> +#include <linux/ptrace.h> +#include <linux/random.h> +#include <linux/efi.h> +#include <linux/kdebug.h> +#include <linux/kmsg_dump.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/dma-map-ops.h> +#include <linux/set_memory.h> +#include <hyperv/hvhdk.h> +#include <asm/mshyperv.h> + +u64 hv_current_partition_id = HV_PARTITION_ID_SELF; +EXPORT_SYMBOL_GPL(hv_current_partition_id); + +enum hv_partition_type hv_curr_partition_type; +EXPORT_SYMBOL_GPL(hv_curr_partition_type); + +/* + * ms_hyperv and hv_nested are defined here with other + * Hyper-V specific globals so they are shared across all architectures and are + * built only when CONFIG_HYPERV is defined. But on x86, + * ms_hyperv_init_platform() is built even when CONFIG_HYPERV is not + * defined, and it uses these three variables. So mark them as __weak + * here, allowing for an overriding definition in the module containing + * ms_hyperv_init_platform(). + */ +bool __weak hv_nested; +EXPORT_SYMBOL_GPL(hv_nested); + +struct ms_hyperv_info __weak ms_hyperv; +EXPORT_SYMBOL_GPL(ms_hyperv); + +u32 *hv_vp_index; +EXPORT_SYMBOL_GPL(hv_vp_index); + +u32 hv_max_vp_index; +EXPORT_SYMBOL_GPL(hv_max_vp_index); + +void * __percpu *hyperv_pcpu_input_arg; +EXPORT_SYMBOL_GPL(hyperv_pcpu_input_arg); + +void * __percpu *hyperv_pcpu_output_arg; +EXPORT_SYMBOL_GPL(hyperv_pcpu_output_arg); + +static void hv_kmsg_dump_unregister(void); + +static struct ctl_table_header *hv_ctl_table_hdr; + +/* + * Per-cpu array holding the tail pointer for the SynIC event ring buffer + * for each SINT. + * + * We cannot maintain this in mshv driver because the tail pointer should + * persist even if the mshv driver is unloaded. + */ +u8 * __percpu *hv_synic_eventring_tail; +EXPORT_SYMBOL_GPL(hv_synic_eventring_tail); + +/* + * Hyper-V specific initialization and shutdown code that is + * common across all architectures. Called from architecture + * specific initialization functions. + */ + +void __init hv_common_free(void) +{ + unregister_sysctl_table(hv_ctl_table_hdr); + hv_ctl_table_hdr = NULL; + + if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) + hv_kmsg_dump_unregister(); + + kfree(hv_vp_index); + hv_vp_index = NULL; + + free_percpu(hyperv_pcpu_output_arg); + hyperv_pcpu_output_arg = NULL; + + free_percpu(hyperv_pcpu_input_arg); + hyperv_pcpu_input_arg = NULL; + + free_percpu(hv_synic_eventring_tail); + hv_synic_eventring_tail = NULL; +} + +static void *hv_panic_page; + +/* + * Boolean to control whether to report panic messages over Hyper-V. + * + * It can be set via /proc/sys/kernel/hyperv_record_panic_msg + */ +static int sysctl_record_panic_msg = 1; + +/* + * sysctl option to allow the user to control whether kmsg data should be + * reported to Hyper-V on panic. + */ +static const struct ctl_table hv_ctl_table[] = { + { + .procname = "hyperv_record_panic_msg", + .data = &sysctl_record_panic_msg, + .maxlen = sizeof(int), + .mode = 0644, + .proc_handler = proc_dointvec_minmax, + .extra1 = SYSCTL_ZERO, + .extra2 = SYSCTL_ONE + }, +}; + +static int hv_die_panic_notify_crash(struct notifier_block *self, + unsigned long val, void *args); + +static struct notifier_block hyperv_die_report_block = { + .notifier_call = hv_die_panic_notify_crash, +}; + +static struct notifier_block hyperv_panic_report_block = { + .notifier_call = hv_die_panic_notify_crash, +}; + +/* + * The following callback works both as die and panic notifier; its + * goal is to provide panic information to the hypervisor unless the + * kmsg dumper is used [see hv_kmsg_dump()], which provides more + * information but isn't always available. + * + * Notice that both the panic/die report notifiers are registered only + * if we have the capability HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE set. + */ +static int hv_die_panic_notify_crash(struct notifier_block *self, + unsigned long val, void *args) +{ + struct pt_regs *regs; + bool is_die; + + /* Don't notify Hyper-V unless we have a die oops event or panic. */ + if (self == &hyperv_panic_report_block) { + is_die = false; + regs = current_pt_regs(); + } else { /* die event */ + if (val != DIE_OOPS) + return NOTIFY_DONE; + + is_die = true; + regs = ((struct die_args *)args)->regs; + } + + /* + * Hyper-V should be notified only once about a panic/die. If we will + * be calling hv_kmsg_dump() later with kmsg data, don't do the + * notification here. + */ + if (!sysctl_record_panic_msg || !hv_panic_page) + hyperv_report_panic(regs, val, is_die); + + return NOTIFY_DONE; +} + +/* + * Callback from kmsg_dump. Grab as much as possible from the end of the kmsg + * buffer and call into Hyper-V to transfer the data. + */ +static void hv_kmsg_dump(struct kmsg_dumper *dumper, + struct kmsg_dump_detail *detail) +{ + struct kmsg_dump_iter iter; + size_t bytes_written; + + /* We are only interested in panics. */ + if (detail->reason != KMSG_DUMP_PANIC || !sysctl_record_panic_msg) + return; + + /* + * Write dump contents to the page. No need to synchronize; panic should + * be single-threaded. Ignore failures from kmsg_dump_get_buffer() since + * panic notification should be done even if there is no message data. + * Don't assume bytes_written is set in case of failure, so initialize it. + */ + kmsg_dump_rewind(&iter); + bytes_written = 0; + (void)kmsg_dump_get_buffer(&iter, false, hv_panic_page, HV_HYP_PAGE_SIZE, + &bytes_written); + + /* + * P3 to contain the physical address of the panic page & P4 to + * contain the size of the panic data in that page. Rest of the + * registers are no-op when the NOTIFY_MSG flag is set. + */ + hv_set_msr(HV_MSR_CRASH_P0, 0); + hv_set_msr(HV_MSR_CRASH_P1, 0); + hv_set_msr(HV_MSR_CRASH_P2, 0); + hv_set_msr(HV_MSR_CRASH_P3, bytes_written ? virt_to_phys(hv_panic_page) : 0); + hv_set_msr(HV_MSR_CRASH_P4, bytes_written); + + /* + * Let Hyper-V know there is crash data available along with + * the panic message. + */ + hv_set_msr(HV_MSR_CRASH_CTL, + (HV_CRASH_CTL_CRASH_NOTIFY | + HV_CRASH_CTL_CRASH_NOTIFY_MSG)); +} + +static struct kmsg_dumper hv_kmsg_dumper = { + .dump = hv_kmsg_dump, +}; + +static void hv_kmsg_dump_unregister(void) +{ + kmsg_dump_unregister(&hv_kmsg_dumper); + unregister_die_notifier(&hyperv_die_report_block); + atomic_notifier_chain_unregister(&panic_notifier_list, + &hyperv_panic_report_block); + + kfree(hv_panic_page); + hv_panic_page = NULL; +} + +static void hv_kmsg_dump_register(void) +{ + int ret; + + hv_panic_page = kzalloc(HV_HYP_PAGE_SIZE, GFP_KERNEL); + if (!hv_panic_page) { + pr_err("Hyper-V: panic message page memory allocation failed\n"); + return; + } + + ret = kmsg_dump_register(&hv_kmsg_dumper); + if (ret) { + pr_err("Hyper-V: kmsg dump register error 0x%x\n", ret); + kfree(hv_panic_page); + hv_panic_page = NULL; + } +} + +static inline bool hv_output_page_exists(void) +{ + return hv_parent_partition() || IS_ENABLED(CONFIG_HYPERV_VTL_MODE); +} + +void __init hv_get_partition_id(void) +{ + struct hv_output_get_partition_id *output; + unsigned long flags; + u64 status, pt_id; + + local_irq_save(flags); + output = *this_cpu_ptr(hyperv_pcpu_input_arg); + status = hv_do_hypercall(HVCALL_GET_PARTITION_ID, NULL, output); + pt_id = output->partition_id; + local_irq_restore(flags); + + if (hv_result_success(status)) + hv_current_partition_id = pt_id; + else + pr_err("Hyper-V: failed to get partition ID: %#x\n", + hv_result(status)); +} +#if IS_ENABLED(CONFIG_HYPERV_VTL_MODE) +u8 __init get_vtl(void) +{ + u64 control = HV_HYPERCALL_REP_COMP_1 | HVCALL_GET_VP_REGISTERS; + struct hv_input_get_vp_registers *input; + struct hv_output_get_vp_registers *output; + unsigned long flags; + u64 ret; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + memset(input, 0, struct_size(input, names, 1)); + input->partition_id = HV_PARTITION_ID_SELF; + input->vp_index = HV_VP_INDEX_SELF; + input->input_vtl.as_uint8 = 0; + input->names[0] = HV_REGISTER_VSM_VP_STATUS; + + ret = hv_do_hypercall(control, input, output); + if (hv_result_success(ret)) { + ret = output->values[0].reg8 & HV_VTL_MASK; + } else { + pr_err("Failed to get VTL(error: %lld) exiting...\n", ret); + BUG(); + } + + local_irq_restore(flags); + return ret; +} +#endif + +int __init hv_common_init(void) +{ + int i; + union hv_hypervisor_version_info version; + + /* Get information about the Microsoft Hypervisor version */ + if (!hv_get_hypervisor_version(&version)) + pr_info("Hyper-V: Hypervisor Build %d.%d.%d.%d-%d-%d\n", + version.major_version, version.minor_version, + version.build_number, version.service_number, + version.service_pack, version.service_branch); + + if (hv_is_isolation_supported()) + sysctl_record_panic_msg = 0; + + /* + * Hyper-V expects to get crash register data or kmsg when + * crash enlightment is available and system crashes. Set + * crash_kexec_post_notifiers to be true to make sure that + * calling crash enlightment interface before running kdump + * kernel. + */ + if (ms_hyperv.misc_features & HV_FEATURE_GUEST_CRASH_MSR_AVAILABLE) { + u64 hyperv_crash_ctl; + + crash_kexec_post_notifiers = true; + pr_info("Hyper-V: enabling crash_kexec_post_notifiers\n"); + + /* + * Panic message recording (sysctl_record_panic_msg) + * is enabled by default in non-isolated guests and + * disabled by default in isolated guests; the panic + * message recording won't be available in isolated + * guests should the following registration fail. + */ + hv_ctl_table_hdr = register_sysctl("kernel", hv_ctl_table); + if (!hv_ctl_table_hdr) + pr_err("Hyper-V: sysctl table register error"); + + /* + * Register for panic kmsg callback only if the right + * capability is supported by the hypervisor. + */ + hyperv_crash_ctl = hv_get_msr(HV_MSR_CRASH_CTL); + if (hyperv_crash_ctl & HV_CRASH_CTL_CRASH_NOTIFY_MSG) + hv_kmsg_dump_register(); + + register_die_notifier(&hyperv_die_report_block); + atomic_notifier_chain_register(&panic_notifier_list, + &hyperv_panic_report_block); + } + + /* + * Allocate the per-CPU state for the hypercall input arg. + * If this allocation fails, we will not be able to setup + * (per-CPU) hypercall input page and thus this failure is + * fatal on Hyper-V. + */ + hyperv_pcpu_input_arg = alloc_percpu(void *); + BUG_ON(!hyperv_pcpu_input_arg); + + /* Allocate the per-CPU state for output arg for root */ + if (hv_output_page_exists()) { + hyperv_pcpu_output_arg = alloc_percpu(void *); + BUG_ON(!hyperv_pcpu_output_arg); + } + + if (hv_parent_partition()) { + hv_synic_eventring_tail = alloc_percpu(u8 *); + BUG_ON(!hv_synic_eventring_tail); + } + + hv_vp_index = kmalloc_array(nr_cpu_ids, sizeof(*hv_vp_index), + GFP_KERNEL); + if (!hv_vp_index) { + hv_common_free(); + return -ENOMEM; + } + + for (i = 0; i < nr_cpu_ids; i++) + hv_vp_index[i] = VP_INVAL; + + return 0; +} + +void __init ms_hyperv_late_init(void) +{ + struct acpi_table_header *header; + acpi_status status; + u8 *randomdata; + u32 length, i; + + /* + * Seed the Linux random number generator with entropy provided by + * the Hyper-V host in ACPI table OEM0. + */ + if (!IS_ENABLED(CONFIG_ACPI)) + return; + + status = acpi_get_table("OEM0", 0, &header); + if (ACPI_FAILURE(status) || !header) + return; + + /* + * Since the "OEM0" table name is for OEM specific usage, verify + * that what we're seeing purports to be from Microsoft. + */ + if (strncmp(header->oem_table_id, "MICROSFT", 8)) + goto error; + + /* + * Ensure the length is reasonable. Requiring at least 8 bytes and + * no more than 4K bytes is somewhat arbitrary and just protects + * against a malformed table. Hyper-V currently provides 64 bytes, + * but allow for a change in a later version. + */ + if (header->length < sizeof(*header) + 8 || + header->length > sizeof(*header) + SZ_4K) + goto error; + + length = header->length - sizeof(*header); + randomdata = (u8 *)(header + 1); + + pr_debug("Hyper-V: Seeding rng with %d random bytes from ACPI table OEM0\n", + length); + + add_bootloader_randomness(randomdata, length); + + /* + * To prevent the seed data from being visible in /sys/firmware/acpi, + * zero out the random data in the ACPI table and fixup the checksum. + * The zero'ing is done out of an abundance of caution in avoiding + * potential security risks to the rng. Similarly, reset the table + * length to just the header size so that a subsequent kexec doesn't + * try to use the zero'ed out random data. + */ + for (i = 0; i < length; i++) { + header->checksum += randomdata[i]; + randomdata[i] = 0; + } + + for (i = 0; i < sizeof(header->length); i++) + header->checksum += ((u8 *)&header->length)[i]; + header->length = sizeof(*header); + for (i = 0; i < sizeof(header->length); i++) + header->checksum -= ((u8 *)&header->length)[i]; + +error: + acpi_put_table(header); +} + +/* + * Hyper-V specific initialization and die code for + * individual CPUs that is common across all architectures. + * Called by the CPU hotplug mechanism. + */ + +int hv_common_cpu_init(unsigned int cpu) +{ + void **inputarg, **outputarg; + u8 **synic_eventring_tail; + u64 msr_vp_index; + gfp_t flags; + const int pgcount = hv_output_page_exists() ? 2 : 1; + void *mem; + int ret = 0; + + /* hv_cpu_init() can be called with IRQs disabled from hv_resume() */ + flags = irqs_disabled() ? GFP_ATOMIC : GFP_KERNEL; + + inputarg = (void **)this_cpu_ptr(hyperv_pcpu_input_arg); + + /* + * The per-cpu memory is already allocated if this CPU was previously + * online and then taken offline + */ + if (!*inputarg) { + mem = kmalloc_array(pgcount, HV_HYP_PAGE_SIZE, flags); + if (!mem) + return -ENOMEM; + + if (hv_output_page_exists()) { + outputarg = (void **)this_cpu_ptr(hyperv_pcpu_output_arg); + *outputarg = (char *)mem + HV_HYP_PAGE_SIZE; + } + + if (!ms_hyperv.paravisor_present && + (hv_isolation_type_snp() || hv_isolation_type_tdx())) { + ret = set_memory_decrypted((unsigned long)mem, pgcount); + if (ret) { + /* It may be unsafe to free 'mem' */ + return ret; + } + + memset(mem, 0x00, pgcount * HV_HYP_PAGE_SIZE); + } + + /* + * In a fully enlightened TDX/SNP VM with more than 64 VPs, if + * hyperv_pcpu_input_arg is not NULL, set_memory_decrypted() -> + * ... -> cpa_flush()-> ... -> __send_ipi_mask_ex() tries to + * use hyperv_pcpu_input_arg as the hypercall input page, which + * must be a decrypted page in such a VM, but the page is still + * encrypted before set_memory_decrypted() returns. Fix this by + * setting *inputarg after the above set_memory_decrypted(): if + * hyperv_pcpu_input_arg is NULL, __send_ipi_mask_ex() returns + * HV_STATUS_INVALID_PARAMETER immediately, and the function + * hv_send_ipi_mask() falls back to orig_apic.send_IPI_mask(), + * which may be slightly slower than the hypercall, but still + * works correctly in such a VM. + */ + *inputarg = mem; + } + + msr_vp_index = hv_get_msr(HV_MSR_VP_INDEX); + + hv_vp_index[cpu] = msr_vp_index; + + if (msr_vp_index > hv_max_vp_index) + hv_max_vp_index = msr_vp_index; + + if (hv_parent_partition()) { + synic_eventring_tail = (u8 **)this_cpu_ptr(hv_synic_eventring_tail); + *synic_eventring_tail = kcalloc(HV_SYNIC_SINT_COUNT, + sizeof(u8), flags); + /* No need to unwind any of the above on failure here */ + if (unlikely(!*synic_eventring_tail)) + ret = -ENOMEM; + } + + return ret; +} + +int hv_common_cpu_die(unsigned int cpu) +{ + u8 **synic_eventring_tail; + /* + * The hyperv_pcpu_input_arg and hyperv_pcpu_output_arg memory + * is not freed when the CPU goes offline as the hyperv_pcpu_input_arg + * may be used by the Hyper-V vPCI driver in reassigning interrupts + * as part of the offlining process. The interrupt reassignment + * happens *after* the CPUHP_AP_HYPERV_ONLINE state has run and + * called this function. + * + * If a previously offlined CPU is brought back online again, the + * originally allocated memory is reused in hv_common_cpu_init(). + */ + + if (hv_parent_partition()) { + synic_eventring_tail = this_cpu_ptr(hv_synic_eventring_tail); + kfree(*synic_eventring_tail); + *synic_eventring_tail = NULL; + } + + return 0; +} + +/* Bit mask of the extended capability to query: see HV_EXT_CAPABILITY_xxx */ +bool hv_query_ext_cap(u64 cap_query) +{ + /* + * The address of the 'hv_extended_cap' variable will be used as an + * output parameter to the hypercall below and so it should be + * compatible with 'virt_to_phys'. Which means, it's address should be + * directly mapped. Use 'static' to keep it compatible; stack variables + * can be virtually mapped, making them incompatible with + * 'virt_to_phys'. + * Hypercall input/output addresses should also be 8-byte aligned. + */ + static u64 hv_extended_cap __aligned(8); + static bool hv_extended_cap_queried; + u64 status; + + /* + * Querying extended capabilities is an extended hypercall. Check if the + * partition supports extended hypercall, first. + */ + if (!(ms_hyperv.priv_high & HV_ENABLE_EXTENDED_HYPERCALLS)) + return false; + + /* Extended capabilities do not change at runtime. */ + if (hv_extended_cap_queried) + return hv_extended_cap & cap_query; + + status = hv_do_hypercall(HV_EXT_CALL_QUERY_CAPABILITIES, NULL, + &hv_extended_cap); + + /* + * The query extended capabilities hypercall should not fail under + * any normal circumstances. Avoid repeatedly making the hypercall, on + * error. + */ + hv_extended_cap_queried = true; + if (!hv_result_success(status)) { + pr_err("Hyper-V: Extended query capabilities hypercall failed 0x%llx\n", + status); + return false; + } + + return hv_extended_cap & cap_query; +} +EXPORT_SYMBOL_GPL(hv_query_ext_cap); + +void hv_setup_dma_ops(struct device *dev, bool coherent) +{ + arch_setup_dma_ops(dev, coherent); +} +EXPORT_SYMBOL_GPL(hv_setup_dma_ops); + +bool hv_is_hibernation_supported(void) +{ + return !hv_root_partition() && acpi_sleep_state_supported(ACPI_STATE_S4); +} +EXPORT_SYMBOL_GPL(hv_is_hibernation_supported); + +/* + * Default function to read the Hyper-V reference counter, independent + * of whether Hyper-V enlightened clocks/timers are being used. But on + * architectures where it is used, Hyper-V enlightenment code in + * hyperv_timer.c may override this function. + */ +static u64 __hv_read_ref_counter(void) +{ + return hv_get_msr(HV_MSR_TIME_REF_COUNT); +} + +u64 (*hv_read_reference_counter)(void) = __hv_read_ref_counter; +EXPORT_SYMBOL_GPL(hv_read_reference_counter); + +/* These __weak functions provide default "no-op" behavior and + * may be overridden by architecture specific versions. Architectures + * for which the default "no-op" behavior is sufficient can leave + * them unimplemented and not be cluttered with a bunch of stub + * functions in arch-specific code. + */ + +bool __weak hv_is_isolation_supported(void) +{ + return false; +} +EXPORT_SYMBOL_GPL(hv_is_isolation_supported); + +bool __weak hv_isolation_type_snp(void) +{ + return false; +} +EXPORT_SYMBOL_GPL(hv_isolation_type_snp); + +bool __weak hv_isolation_type_tdx(void) +{ + return false; +} +EXPORT_SYMBOL_GPL(hv_isolation_type_tdx); + +void __weak hv_setup_vmbus_handler(void (*handler)(void)) +{ +} +EXPORT_SYMBOL_GPL(hv_setup_vmbus_handler); + +void __weak hv_remove_vmbus_handler(void) +{ +} +EXPORT_SYMBOL_GPL(hv_remove_vmbus_handler); + +void __weak hv_setup_mshv_handler(void (*handler)(void)) +{ +} +EXPORT_SYMBOL_GPL(hv_setup_mshv_handler); + +void __weak hv_setup_kexec_handler(void (*handler)(void)) +{ +} +EXPORT_SYMBOL_GPL(hv_setup_kexec_handler); + +void __weak hv_remove_kexec_handler(void) +{ +} +EXPORT_SYMBOL_GPL(hv_remove_kexec_handler); + +void __weak hv_setup_crash_handler(void (*handler)(struct pt_regs *regs)) +{ +} +EXPORT_SYMBOL_GPL(hv_setup_crash_handler); + +void __weak hv_remove_crash_handler(void) +{ +} +EXPORT_SYMBOL_GPL(hv_remove_crash_handler); + +void __weak hyperv_cleanup(void) +{ +} +EXPORT_SYMBOL_GPL(hyperv_cleanup); + +u64 __weak hv_ghcb_hypercall(u64 control, void *input, void *output, u32 input_size) +{ + return HV_STATUS_INVALID_PARAMETER; +} +EXPORT_SYMBOL_GPL(hv_ghcb_hypercall); + +u64 __weak hv_tdx_hypercall(u64 control, u64 param1, u64 param2) +{ + return HV_STATUS_INVALID_PARAMETER; +} +EXPORT_SYMBOL_GPL(hv_tdx_hypercall); + +void __weak hv_enable_coco_interrupt(unsigned int cpu, unsigned int vector, bool set) +{ +} +EXPORT_SYMBOL_GPL(hv_enable_coco_interrupt); + +void __weak hv_para_set_sint_proxy(bool enable) +{ +} +EXPORT_SYMBOL_GPL(hv_para_set_sint_proxy); + +u64 __weak hv_para_get_synic_register(unsigned int reg) +{ + return ~0ULL; +} +EXPORT_SYMBOL_GPL(hv_para_get_synic_register); + +void __weak hv_para_set_synic_register(unsigned int reg, u64 val) +{ +} +EXPORT_SYMBOL_GPL(hv_para_set_synic_register); + +void hv_identify_partition_type(void) +{ + /* Assume guest role */ + hv_curr_partition_type = HV_PARTITION_TYPE_GUEST; + /* + * Check partition creation and cpu management privileges + * + * Hyper-V should never specify running as root and as a Confidential + * VM. But to protect against a compromised/malicious Hyper-V trying + * to exploit root behavior to expose Confidential VM memory, ignore + * the root partition setting if also a Confidential VM. + */ + if ((ms_hyperv.priv_high & HV_CREATE_PARTITIONS) && + !(ms_hyperv.priv_high & HV_ISOLATION)) { + + if (!IS_ENABLED(CONFIG_MSHV_ROOT)) { + pr_crit("Hyper-V: CONFIG_MSHV_ROOT not enabled!\n"); + } else if (ms_hyperv.priv_high & HV_CPU_MANAGEMENT) { + pr_info("Hyper-V: running as root partition\n"); + hv_curr_partition_type = HV_PARTITION_TYPE_ROOT; + } else { + pr_info("Hyper-V: running as L1VH partition\n"); + hv_curr_partition_type = HV_PARTITION_TYPE_L1VH; + } + } +} + +struct hv_status_info { + char *string; + int errno; + u16 code; +}; + +/* + * Note on the errno mappings: + * A failed hypercall is usually only recoverable (or loggable) near + * the call site where the HV_STATUS_* code is known. So the errno + * it gets converted to is not too useful further up the stack. + * Provide a few mappings that could be useful, and revert to -EIO + * as a fallback. + */ +static const struct hv_status_info hv_status_infos[] = { +#define _STATUS_INFO(status, errno) { #status, (errno), (status) } + _STATUS_INFO(HV_STATUS_SUCCESS, 0), + _STATUS_INFO(HV_STATUS_INVALID_HYPERCALL_CODE, -EINVAL), + _STATUS_INFO(HV_STATUS_INVALID_HYPERCALL_INPUT, -EINVAL), + _STATUS_INFO(HV_STATUS_INVALID_ALIGNMENT, -EIO), + _STATUS_INFO(HV_STATUS_INVALID_PARAMETER, -EINVAL), + _STATUS_INFO(HV_STATUS_ACCESS_DENIED, -EIO), + _STATUS_INFO(HV_STATUS_INVALID_PARTITION_STATE, -EIO), + _STATUS_INFO(HV_STATUS_OPERATION_DENIED, -EIO), + _STATUS_INFO(HV_STATUS_UNKNOWN_PROPERTY, -EIO), + _STATUS_INFO(HV_STATUS_PROPERTY_VALUE_OUT_OF_RANGE, -EIO), + _STATUS_INFO(HV_STATUS_INSUFFICIENT_MEMORY, -ENOMEM), + _STATUS_INFO(HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY, -ENOMEM), + _STATUS_INFO(HV_STATUS_INSUFFICIENT_ROOT_MEMORY, -ENOMEM), + _STATUS_INFO(HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY, -ENOMEM), + _STATUS_INFO(HV_STATUS_INVALID_PARTITION_ID, -EINVAL), + _STATUS_INFO(HV_STATUS_INVALID_VP_INDEX, -EINVAL), + _STATUS_INFO(HV_STATUS_NOT_FOUND, -EIO), + _STATUS_INFO(HV_STATUS_INVALID_PORT_ID, -EINVAL), + _STATUS_INFO(HV_STATUS_INVALID_CONNECTION_ID, -EINVAL), + _STATUS_INFO(HV_STATUS_INSUFFICIENT_BUFFERS, -EIO), + _STATUS_INFO(HV_STATUS_NOT_ACKNOWLEDGED, -EIO), + _STATUS_INFO(HV_STATUS_INVALID_VP_STATE, -EIO), + _STATUS_INFO(HV_STATUS_NO_RESOURCES, -EIO), + _STATUS_INFO(HV_STATUS_PROCESSOR_FEATURE_NOT_SUPPORTED, -EIO), + _STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EINVAL), + _STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EINVAL), + _STATUS_INFO(HV_STATUS_INVALID_LP_INDEX, -EIO), + _STATUS_INFO(HV_STATUS_INVALID_REGISTER_VALUE, -EIO), + _STATUS_INFO(HV_STATUS_OPERATION_FAILED, -EIO), + _STATUS_INFO(HV_STATUS_TIME_OUT, -EIO), + _STATUS_INFO(HV_STATUS_CALL_PENDING, -EIO), + _STATUS_INFO(HV_STATUS_VTL_ALREADY_ENABLED, -EIO), +#undef _STATUS_INFO +}; + +static inline const struct hv_status_info *find_hv_status_info(u64 hv_status) +{ + int i; + u16 code = hv_result(hv_status); + + for (i = 0; i < ARRAY_SIZE(hv_status_infos); ++i) { + const struct hv_status_info *info = &hv_status_infos[i]; + + if (info->code == code) + return info; + } + + return NULL; +} + +/* Convert a hypercall result into a linux-friendly error code. */ +int hv_result_to_errno(u64 status) +{ + const struct hv_status_info *info; + + /* hv_do_hypercall() may return U64_MAX, hypercalls aren't possible */ + if (unlikely(status == U64_MAX)) + return -EOPNOTSUPP; + + info = find_hv_status_info(status); + if (info) + return info->errno; + + return -EIO; +} +EXPORT_SYMBOL_GPL(hv_result_to_errno); + +const char *hv_result_to_string(u64 status) +{ + const struct hv_status_info *info; + + if (unlikely(status == U64_MAX)) + return "Hypercall page missing!"; + + info = find_hv_status_info(status); + if (info) + return info->string; + + return "Unknown"; +} +EXPORT_SYMBOL_GPL(hv_result_to_string); diff --git a/drivers/hv/hv_debugfs.c b/drivers/hv/hv_debugfs.c new file mode 100644 index 000000000..ccf752b66 --- /dev/null +++ b/drivers/hv/hv_debugfs.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Authors: + * Branden Bonaby <brandonbonaby94@gmail.com> + */ + +#include <linux/hyperv.h> +#include <linux/debugfs.h> +#include <linux/delay.h> +#include <linux/err.h> + +#include "hyperv_vmbus.h" + +static struct dentry *hv_debug_root; + +static int hv_debugfs_delay_get(void *data, u64 *val) +{ + *val = *(u32 *)data; + return 0; +} + +static int hv_debugfs_delay_set(void *data, u64 val) +{ + if (val > 1000) + return -EINVAL; + *(u32 *)data = val; + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(hv_debugfs_delay_fops, hv_debugfs_delay_get, + hv_debugfs_delay_set, "%llu\n"); + +static int hv_debugfs_state_get(void *data, u64 *val) +{ + *val = *(bool *)data; + return 0; +} + +static int hv_debugfs_state_set(void *data, u64 val) +{ + if (val == 1) + *(bool *)data = true; + else if (val == 0) + *(bool *)data = false; + else + return -EINVAL; + return 0; +} + +DEFINE_DEBUGFS_ATTRIBUTE(hv_debugfs_state_fops, hv_debugfs_state_get, + hv_debugfs_state_set, "%llu\n"); + +/* Setup delay files to store test values */ +static int hv_debug_delay_files(struct hv_device *dev, struct dentry *root) +{ + struct vmbus_channel *channel = dev->channel; + char *buffer = "fuzz_test_buffer_interrupt_delay"; + char *message = "fuzz_test_message_delay"; + int *buffer_val = &channel->fuzz_testing_interrupt_delay; + int *message_val = &channel->fuzz_testing_message_delay; + struct dentry *buffer_file, *message_file; + + buffer_file = debugfs_create_file(buffer, 0644, root, + buffer_val, + &hv_debugfs_delay_fops); + if (IS_ERR(buffer_file)) { + pr_debug("debugfs_hyperv: file %s not created\n", buffer); + return PTR_ERR(buffer_file); + } + + message_file = debugfs_create_file(message, 0644, root, + message_val, + &hv_debugfs_delay_fops); + if (IS_ERR(message_file)) { + pr_debug("debugfs_hyperv: file %s not created\n", message); + return PTR_ERR(message_file); + } + + return 0; +} + +/* Setup test state value for vmbus device */ +static int hv_debug_set_test_state(struct hv_device *dev, struct dentry *root) +{ + struct vmbus_channel *channel = dev->channel; + bool *state = &channel->fuzz_testing_state; + char *status = "fuzz_test_state"; + struct dentry *test_state; + + test_state = debugfs_create_file(status, 0644, root, + state, + &hv_debugfs_state_fops); + if (IS_ERR(test_state)) { + pr_debug("debugfs_hyperv: file %s not created\n", status); + return PTR_ERR(test_state); + } + + return 0; +} + +/* Bind hv device to a dentry for debugfs */ +static void hv_debug_set_dir_dentry(struct hv_device *dev, struct dentry *root) +{ + if (hv_debug_root) + dev->debug_dir = root; +} + +/* Create all test dentry's and names for fuzz testing */ +int hv_debug_add_dev_dir(struct hv_device *dev) +{ + const char *device = dev_name(&dev->device); + char *delay_name = "delay"; + struct dentry *delay, *dev_root; + int ret; + + if (!IS_ERR(hv_debug_root)) { + dev_root = debugfs_create_dir(device, hv_debug_root); + if (IS_ERR(dev_root)) { + pr_debug("debugfs_hyperv: hyperv/%s/ not created\n", + device); + return PTR_ERR(dev_root); + } + hv_debug_set_test_state(dev, dev_root); + hv_debug_set_dir_dentry(dev, dev_root); + delay = debugfs_create_dir(delay_name, dev_root); + + if (IS_ERR(delay)) { + pr_debug("debugfs_hyperv: hyperv/%s/%s/ not created\n", + device, delay_name); + return PTR_ERR(delay); + } + ret = hv_debug_delay_files(dev, delay); + + return ret; + } + pr_debug("debugfs_hyperv: hyperv/ not in root debugfs path\n"); + return PTR_ERR(hv_debug_root); +} + +/* Remove dentry associated with released hv device */ +void hv_debug_rm_dev_dir(struct hv_device *dev) +{ + if (!IS_ERR(hv_debug_root)) + debugfs_remove_recursive(dev->debug_dir); +} + +/* Remove all dentrys associated with vmbus testing */ +void hv_debug_rm_all_dir(void) +{ + debugfs_remove_recursive(hv_debug_root); +} + +/* Delay buffer/message reads on a vmbus channel */ +void hv_debug_delay_test(struct vmbus_channel *channel, enum delay delay_type) +{ + struct vmbus_channel *test_channel = channel->primary_channel ? + channel->primary_channel : + channel; + bool state = test_channel->fuzz_testing_state; + + if (state) { + if (delay_type == 0) + udelay(test_channel->fuzz_testing_interrupt_delay); + else + udelay(test_channel->fuzz_testing_message_delay); + } +} + +/* Initialize top dentry for vmbus testing */ +int hv_debug_init(void) +{ + hv_debug_root = debugfs_create_dir("hyperv", NULL); + if (IS_ERR(hv_debug_root)) { + pr_debug("debugfs_hyperv: hyperv/ not created\n"); + return PTR_ERR(hv_debug_root); + } + return 0; +} diff --git a/drivers/hv/hv_kvp.c b/drivers/hv/hv_kvp.c new file mode 100644 index 000000000..336b278b2 --- /dev/null +++ b/drivers/hv/hv_kvp.c @@ -0,0 +1,829 @@ +/* + * An implementation of key value pair (KVP) functionality for Linux. + * + * + * Copyright (C) 2010, Novell, Inc. + * Author : K. Y. Srinivasan <ksrinivasan@novell.com> + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published + * by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or + * NON INFRINGEMENT. See the GNU General Public License for more + * details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/net.h> +#include <linux/nls.h> +#include <linux/connector.h> +#include <linux/workqueue.h> +#include <linux/hyperv.h> +#include <linux/string.h> +#include <hyperv/hvhdk.h> + +#include "hyperv_vmbus.h" +#include "hv_utils_transport.h" + +/* + * Pre win8 version numbers used in ws2008 and ws 2008 r2 (win7) + */ +#define WS2008_SRV_MAJOR 1 +#define WS2008_SRV_MINOR 0 +#define WS2008_SRV_VERSION (WS2008_SRV_MAJOR << 16 | WS2008_SRV_MINOR) + +#define WIN7_SRV_MAJOR 3 +#define WIN7_SRV_MINOR 0 +#define WIN7_SRV_VERSION (WIN7_SRV_MAJOR << 16 | WIN7_SRV_MINOR) + +#define WIN8_SRV_MAJOR 4 +#define WIN8_SRV_MINOR 0 +#define WIN8_SRV_VERSION (WIN8_SRV_MAJOR << 16 | WIN8_SRV_MINOR) + +#define KVP_VER_COUNT 3 +static const int kvp_versions[] = { + WIN8_SRV_VERSION, + WIN7_SRV_VERSION, + WS2008_SRV_VERSION +}; + +#define FW_VER_COUNT 2 +static const int fw_versions[] = { + UTIL_FW_VERSION, + UTIL_WS2K8_FW_VERSION +}; + +/* + * Global state maintained for transaction that is being processed. For a class + * of integration services, including the "KVP service", the specified protocol + * is a "request/response" protocol which means that there can only be single + * outstanding transaction from the host at any given point in time. We use + * this to simplify memory management in this driver - we cache and process + * only one message at a time. + * + * While the request/response protocol is guaranteed by the host, we further + * ensure this by serializing packet processing in this driver - we do not + * read additional packets from the VMBUS until the current packet is fully + * handled. + */ + +static struct { + int state; /* hvutil_device_state */ + int recv_len; /* number of bytes received. */ + struct hv_kvp_msg *kvp_msg; /* current message */ + struct vmbus_channel *recv_channel; /* chn we got the request */ + u64 recv_req_id; /* request ID. */ +} kvp_transaction; + +/* + * This state maintains the version number registered by the daemon. + */ +static int dm_reg_value; + +static void kvp_send_key(struct work_struct *dummy); + + +static void kvp_respond_to_host(struct hv_kvp_msg *msg, int error); +static void kvp_timeout_func(struct work_struct *dummy); +static void kvp_host_handshake_func(struct work_struct *dummy); +static int kvp_register(int); + +static DECLARE_DELAYED_WORK(kvp_timeout_work, kvp_timeout_func); +static DECLARE_DELAYED_WORK(kvp_host_handshake_work, kvp_host_handshake_func); +static DECLARE_WORK(kvp_sendkey_work, kvp_send_key); + +static const char kvp_devname[] = "vmbus/hv_kvp"; +static u8 *recv_buffer; +static struct hvutil_transport *hvt; +/* + * Register the kernel component with the user-level daemon. + * As part of this registration, pass the LIC version number. + * This number has no meaning, it satisfies the registration protocol. + */ +#define HV_DRV_VERSION "3.1" + +static void kvp_poll_wrapper(void *channel) +{ + /* Transaction is finished, reset the state here to avoid races. */ + kvp_transaction.state = HVUTIL_READY; + tasklet_schedule(&((struct vmbus_channel *)channel)->callback_event); +} + +static void kvp_register_done(void) +{ + /* + * If we're still negotiating with the host cancel the timeout + * work to not poll the channel twice. + */ + pr_debug("KVP: userspace daemon registered\n"); + cancel_delayed_work_sync(&kvp_host_handshake_work); + hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper); +} + +static int +kvp_register(int reg_value) +{ + struct hv_kvp_msg *kvp_msg; + int ret; + + kvp_msg = kzalloc_obj(*kvp_msg); + if (!kvp_msg) + return -ENOMEM; + + kvp_msg->kvp_hdr.operation = reg_value; + strscpy(kvp_msg->body.kvp_register.version, HV_DRV_VERSION); + + ret = hvutil_transport_send(hvt, kvp_msg, sizeof(*kvp_msg), + kvp_register_done); + kfree(kvp_msg); + return ret; +} + +static void kvp_timeout_func(struct work_struct *dummy) +{ + /* + * If the timer fires, the user-mode component has not responded; + * process the pending transaction. + */ + kvp_respond_to_host(NULL, HV_E_FAIL); + + hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper); +} + +static void kvp_host_handshake_func(struct work_struct *dummy) +{ + tasklet_schedule(&kvp_transaction.recv_channel->callback_event); +} + +static int kvp_handle_handshake(struct hv_kvp_msg *msg) +{ + switch (msg->kvp_hdr.operation) { + case KVP_OP_REGISTER: + dm_reg_value = KVP_OP_REGISTER; + pr_info("KVP: IP injection functionality not available\n"); + pr_info("KVP: Upgrade the KVP daemon\n"); + break; + case KVP_OP_REGISTER1: + dm_reg_value = KVP_OP_REGISTER1; + break; + default: + pr_info("KVP: incompatible daemon\n"); + pr_info("KVP: KVP version: %d, Daemon version: %d\n", + KVP_OP_REGISTER1, msg->kvp_hdr.operation); + return -EINVAL; + } + + /* + * We have a compatible daemon; complete the handshake. + */ + pr_debug("KVP: userspace daemon ver. %d connected\n", + msg->kvp_hdr.operation); + + return kvp_register(dm_reg_value); +} + + +/* + * Callback when data is received from user mode. + */ + +static int kvp_on_msg(void *msg, int len) +{ + struct hv_kvp_msg *message = (struct hv_kvp_msg *)msg; + struct hv_kvp_msg_enumerate *data; + int error = 0; + + if (len < sizeof(*message)) + return -EINVAL; + + /* + * If we are negotiating the version information + * with the daemon; handle that first. + */ + + if (kvp_transaction.state < HVUTIL_READY) { + return kvp_handle_handshake(message); + } + + /* We didn't send anything to userspace so the reply is spurious */ + if (kvp_transaction.state < HVUTIL_USERSPACE_REQ) + return -EINVAL; + + kvp_transaction.state = HVUTIL_USERSPACE_RECV; + + /* + * Based on the version of the daemon, we propagate errors from the + * daemon differently. + */ + + data = &message->body.kvp_enum_data; + + switch (dm_reg_value) { + case KVP_OP_REGISTER: + /* + * Null string is used to pass back error condition. + */ + if (data->data.key[0] == 0) + error = HV_S_CONT; + break; + + case KVP_OP_REGISTER1: + /* + * We use the message header information from + * the user level daemon to transmit errors. + */ + error = message->error; + break; + } + + /* + * Complete the transaction by forwarding the key value + * to the host. But first, cancel the timeout. + */ + if (cancel_delayed_work_sync(&kvp_timeout_work)) { + kvp_respond_to_host(message, error); + hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper); + } + + return 0; +} + + +static int process_ob_ipinfo(void *in_msg, void *out_msg, int op) +{ + struct hv_kvp_msg *in = in_msg; + struct hv_kvp_ip_msg *out = out_msg; + int len; + + switch (op) { + case KVP_OP_GET_IP_INFO: + /* + * Transform all parameters into utf16 encoding. + */ + len = utf8s_to_utf16s((char *)in->body.kvp_ip_val.ip_addr, + strlen((char *)in->body.kvp_ip_val.ip_addr), + UTF16_HOST_ENDIAN, + (wchar_t *)out->kvp_ip_val.ip_addr, + MAX_IP_ADDR_SIZE); + if (len < 0) + return len; + + len = utf8s_to_utf16s((char *)in->body.kvp_ip_val.sub_net, + strlen((char *)in->body.kvp_ip_val.sub_net), + UTF16_HOST_ENDIAN, + (wchar_t *)out->kvp_ip_val.sub_net, + MAX_IP_ADDR_SIZE); + if (len < 0) + return len; + + len = utf8s_to_utf16s((char *)in->body.kvp_ip_val.gate_way, + strlen((char *)in->body.kvp_ip_val.gate_way), + UTF16_HOST_ENDIAN, + (wchar_t *)out->kvp_ip_val.gate_way, + MAX_GATEWAY_SIZE); + if (len < 0) + return len; + + len = utf8s_to_utf16s((char *)in->body.kvp_ip_val.dns_addr, + strlen((char *)in->body.kvp_ip_val.dns_addr), + UTF16_HOST_ENDIAN, + (wchar_t *)out->kvp_ip_val.dns_addr, + MAX_IP_ADDR_SIZE); + if (len < 0) + return len; + + len = utf8s_to_utf16s((char *)in->body.kvp_ip_val.adapter_id, + strlen((char *)in->body.kvp_ip_val.adapter_id), + UTF16_HOST_ENDIAN, + (wchar_t *)out->kvp_ip_val.adapter_id, + MAX_ADAPTER_ID_SIZE); + if (len < 0) + return len; + + out->kvp_ip_val.dhcp_enabled = + in->body.kvp_ip_val.dhcp_enabled; + out->kvp_ip_val.addr_family = + in->body.kvp_ip_val.addr_family; + } + + return 0; +} + +static void process_ib_ipinfo(void *in_msg, void *out_msg, int op) +{ + struct hv_kvp_ip_msg *in = in_msg; + struct hv_kvp_msg *out = out_msg; + + switch (op) { + case KVP_OP_SET_IP_INFO: + /* + * Transform all parameters into utf8 encoding. + */ + utf16s_to_utf8s((wchar_t *)in->kvp_ip_val.ip_addr, + MAX_IP_ADDR_SIZE, + UTF16_LITTLE_ENDIAN, + (__u8 *)out->body.kvp_ip_val.ip_addr, + MAX_IP_ADDR_SIZE); + + utf16s_to_utf8s((wchar_t *)in->kvp_ip_val.sub_net, + MAX_IP_ADDR_SIZE, + UTF16_LITTLE_ENDIAN, + (__u8 *)out->body.kvp_ip_val.sub_net, + MAX_IP_ADDR_SIZE); + + utf16s_to_utf8s((wchar_t *)in->kvp_ip_val.gate_way, + MAX_GATEWAY_SIZE, + UTF16_LITTLE_ENDIAN, + (__u8 *)out->body.kvp_ip_val.gate_way, + MAX_GATEWAY_SIZE); + + utf16s_to_utf8s((wchar_t *)in->kvp_ip_val.dns_addr, + MAX_IP_ADDR_SIZE, + UTF16_LITTLE_ENDIAN, + (__u8 *)out->body.kvp_ip_val.dns_addr, + MAX_IP_ADDR_SIZE); + + out->body.kvp_ip_val.dhcp_enabled = in->kvp_ip_val.dhcp_enabled; + + fallthrough; + + case KVP_OP_GET_IP_INFO: + utf16s_to_utf8s((wchar_t *)in->kvp_ip_val.adapter_id, + MAX_ADAPTER_ID_SIZE, + UTF16_LITTLE_ENDIAN, + (__u8 *)out->body.kvp_ip_val.adapter_id, + MAX_ADAPTER_ID_SIZE); + + out->body.kvp_ip_val.addr_family = in->kvp_ip_val.addr_family; + } +} + + + + +static void +kvp_send_key(struct work_struct *dummy) +{ + struct hv_kvp_msg *message; + struct hv_kvp_msg *in_msg; + __u8 operation = kvp_transaction.kvp_msg->kvp_hdr.operation; + __u8 pool = kvp_transaction.kvp_msg->kvp_hdr.pool; + __u32 val32; + __u64 val64; + int rc; + + /* The transaction state is wrong. */ + if (kvp_transaction.state != HVUTIL_HOSTMSG_RECEIVED) + return; + + message = kzalloc_obj(*message); + if (!message) + return; + + message->kvp_hdr.operation = operation; + message->kvp_hdr.pool = pool; + in_msg = kvp_transaction.kvp_msg; + + /* + * The key/value strings sent from the host are encoded + * in utf16; convert it to utf8 strings. + * The host assures us that the utf16 strings will not exceed + * the max lengths specified. We will however, reserve room + * for the string terminating character - in the utf16s_utf8s() + * function we limit the size of the buffer where the converted + * string is placed to HV_KVP_EXCHANGE_MAX_*_SIZE -1 to guarantee + * that the strings can be properly terminated! + */ + + switch (message->kvp_hdr.operation) { + case KVP_OP_SET_IP_INFO: + process_ib_ipinfo(in_msg, message, KVP_OP_SET_IP_INFO); + break; + case KVP_OP_GET_IP_INFO: + /* + * We only need to pass on the info of operation, adapter_id + * and addr_family to the userland kvp daemon. + */ + process_ib_ipinfo(in_msg, message, KVP_OP_GET_IP_INFO); + break; + case KVP_OP_SET: + switch (in_msg->body.kvp_set.data.value_type) { + case REG_SZ: + /* + * The value is a string - utf16 encoding. + */ + message->body.kvp_set.data.value_size = + utf16s_to_utf8s( + (wchar_t *)in_msg->body.kvp_set.data.value, + in_msg->body.kvp_set.data.value_size, + UTF16_LITTLE_ENDIAN, + message->body.kvp_set.data.value, + HV_KVP_EXCHANGE_MAX_VALUE_SIZE - 1) + 1; + break; + + case REG_U32: + /* + * The value is a 32 bit scalar. + * We save this as a utf8 string. + */ + val32 = in_msg->body.kvp_set.data.value_u32; + message->body.kvp_set.data.value_size = + sprintf(message->body.kvp_set.data.value, + "%u", val32) + 1; + break; + + case REG_U64: + /* + * The value is a 64 bit scalar. + * We save this as a utf8 string. + */ + val64 = in_msg->body.kvp_set.data.value_u64; + message->body.kvp_set.data.value_size = + sprintf(message->body.kvp_set.data.value, + "%llu", val64) + 1; + break; + + } + + /* + * The key is always a string - utf16 encoding. + */ + message->body.kvp_set.data.key_size = + utf16s_to_utf8s( + (wchar_t *)in_msg->body.kvp_set.data.key, + in_msg->body.kvp_set.data.key_size, + UTF16_LITTLE_ENDIAN, + message->body.kvp_set.data.key, + HV_KVP_EXCHANGE_MAX_KEY_SIZE - 1) + 1; + + break; + + case KVP_OP_GET: + message->body.kvp_get.data.key_size = + utf16s_to_utf8s( + (wchar_t *)in_msg->body.kvp_get.data.key, + in_msg->body.kvp_get.data.key_size, + UTF16_LITTLE_ENDIAN, + message->body.kvp_get.data.key, + HV_KVP_EXCHANGE_MAX_KEY_SIZE - 1) + 1; + break; + + case KVP_OP_DELETE: + message->body.kvp_delete.key_size = + utf16s_to_utf8s( + (wchar_t *)in_msg->body.kvp_delete.key, + in_msg->body.kvp_delete.key_size, + UTF16_LITTLE_ENDIAN, + message->body.kvp_delete.key, + HV_KVP_EXCHANGE_MAX_KEY_SIZE - 1) + 1; + break; + + case KVP_OP_ENUMERATE: + message->body.kvp_enum_data.index = + in_msg->body.kvp_enum_data.index; + break; + } + + kvp_transaction.state = HVUTIL_USERSPACE_REQ; + rc = hvutil_transport_send(hvt, message, sizeof(*message), NULL); + if (rc) { + pr_debug("KVP: failed to communicate to the daemon: %d\n", rc); + if (cancel_delayed_work_sync(&kvp_timeout_work)) { + kvp_respond_to_host(message, HV_E_FAIL); + kvp_transaction.state = HVUTIL_READY; + } + } + + kfree(message); +} + +/* + * Send a response back to the host. + */ + +static void +kvp_respond_to_host(struct hv_kvp_msg *msg_to_host, int error) +{ + struct hv_kvp_msg *kvp_msg; + struct hv_kvp_exchg_msg_value *kvp_data; + char *key_name; + char *value; + struct icmsg_hdr *icmsghdrp; + int keylen = 0; + int valuelen = 0; + u32 buf_len; + struct vmbus_channel *channel; + u64 req_id; + int ret; + + /* + * Copy the global state for completing the transaction. Note that + * only one transaction can be active at a time. + */ + + buf_len = kvp_transaction.recv_len; + channel = kvp_transaction.recv_channel; + req_id = kvp_transaction.recv_req_id; + + icmsghdrp = (struct icmsg_hdr *) + &recv_buffer[sizeof(struct vmbuspipe_hdr)]; + + if (channel->onchannel_callback == NULL) + /* + * We have raced with util driver being unloaded; + * silently return. + */ + return; + + icmsghdrp->status = error; + + /* + * If the error parameter is set, terminate the host's enumeration + * on this pool. + */ + if (error) { + /* + * Something failed or we have timed out; + * terminate the current host-side iteration. + */ + goto response_done; + } + + kvp_msg = (struct hv_kvp_msg *) + &recv_buffer[sizeof(struct vmbuspipe_hdr) + + sizeof(struct icmsg_hdr)]; + + switch (kvp_transaction.kvp_msg->kvp_hdr.operation) { + case KVP_OP_GET_IP_INFO: + ret = process_ob_ipinfo(msg_to_host, + (struct hv_kvp_ip_msg *)kvp_msg, + KVP_OP_GET_IP_INFO); + if (ret < 0) + icmsghdrp->status = HV_E_FAIL; + + goto response_done; + case KVP_OP_SET_IP_INFO: + goto response_done; + case KVP_OP_GET: + kvp_data = &kvp_msg->body.kvp_get.data; + goto copy_value; + + case KVP_OP_SET: + case KVP_OP_DELETE: + goto response_done; + + default: + break; + } + + kvp_data = &kvp_msg->body.kvp_enum_data.data; + key_name = msg_to_host->body.kvp_enum_data.data.key; + + /* + * The windows host expects the key/value pair to be encoded + * in utf16. Ensure that the key/value size reported to the host + * will be less than or equal to the MAX size (including the + * terminating character). + */ + keylen = utf8s_to_utf16s(key_name, strlen(key_name), UTF16_HOST_ENDIAN, + (wchar_t *) kvp_data->key, + (HV_KVP_EXCHANGE_MAX_KEY_SIZE / 2) - 2); + kvp_data->key_size = 2*(keylen + 1); /* utf16 encoding */ + +copy_value: + value = msg_to_host->body.kvp_enum_data.data.value; + valuelen = utf8s_to_utf16s(value, strlen(value), UTF16_HOST_ENDIAN, + (wchar_t *) kvp_data->value, + (HV_KVP_EXCHANGE_MAX_VALUE_SIZE / 2) - 2); + kvp_data->value_size = 2*(valuelen + 1); /* utf16 encoding */ + + /* + * If the utf8s to utf16s conversion failed; notify host + * of the error. + */ + if ((keylen < 0) || (valuelen < 0)) + icmsghdrp->status = HV_E_FAIL; + + kvp_data->value_type = REG_SZ; /* all our values are strings */ + +response_done: + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, recv_buffer, buf_len, req_id, + VM_PKT_DATA_INBAND, 0); +} + +/* + * This callback is invoked when we get a KVP message from the host. + * The host ensures that only one KVP transaction can be active at a time. + * KVP implementation in Linux needs to forward the key to a user-mde + * component to retrieve the corresponding value. Consequently, we cannot + * respond to the host in the context of this callback. Since the host + * guarantees that at most only one transaction can be active at a time, + * we stash away the transaction state in a set of global variables. + */ + +void hv_kvp_onchannelcallback(void *context) +{ + struct vmbus_channel *channel = context; + u32 recvlen; + u64 requestid; + + struct hv_kvp_msg *kvp_msg; + + struct icmsg_hdr *icmsghdrp; + int kvp_srv_version; + static enum {NEGO_NOT_STARTED, + NEGO_IN_PROGRESS, + NEGO_FINISHED} host_negotiatied = NEGO_NOT_STARTED; + + if (kvp_transaction.state < HVUTIL_READY) { + /* + * If userspace daemon is not connected and host is asking + * us to negotiate we need to delay to not lose messages. + * This is important for Failover IP setting. + */ + if (host_negotiatied == NEGO_NOT_STARTED) { + host_negotiatied = NEGO_IN_PROGRESS; + schedule_delayed_work(&kvp_host_handshake_work, + secs_to_jiffies(HV_UTIL_NEGO_TIMEOUT)); + } + return; + } + if (kvp_transaction.state > HVUTIL_READY) + return; + + if (vmbus_recvpacket(channel, recv_buffer, HV_HYP_PAGE_SIZE * 4, &recvlen, &requestid)) { + pr_err_ratelimited("KVP request received. Could not read into recv buf\n"); + return; + } + + if (!recvlen) + return; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("KVP request received. Packet length too small: %d\n", + recvlen); + return; + } + + icmsghdrp = (struct icmsg_hdr *)&recv_buffer[sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + recv_buffer, recvlen, + fw_versions, FW_VER_COUNT, + kvp_versions, KVP_VER_COUNT, + NULL, &kvp_srv_version)) { + pr_info("KVP IC version %d.%d\n", + kvp_srv_version >> 16, + kvp_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_KVPEXCHANGE) { + /* + * recvlen is not checked against sizeof(struct kvp_msg) because kvp_msg contains + * a union of structs and the msg type received is not known. Code using this + * struct should provide validation when accessing its fields. + */ + kvp_msg = (struct hv_kvp_msg *)&recv_buffer[ICMSG_HDR]; + + /* + * Stash away this global state for completing the + * transaction; note transactions are serialized. + */ + + kvp_transaction.recv_len = recvlen; + kvp_transaction.recv_req_id = requestid; + kvp_transaction.kvp_msg = kvp_msg; + + if (kvp_transaction.state < HVUTIL_READY) { + /* Userspace is not registered yet */ + kvp_respond_to_host(NULL, HV_E_FAIL); + return; + } + kvp_transaction.state = HVUTIL_HOSTMSG_RECEIVED; + + /* + * Get the information from the + * user-mode component. + * component. This transaction will be + * completed when we get the value from + * the user-mode component. + * Set a timeout to deal with + * user-mode not responding. + */ + schedule_work(&kvp_sendkey_work); + schedule_delayed_work(&kvp_timeout_work, + secs_to_jiffies(HV_UTIL_TIMEOUT)); + + return; + + } else { + pr_err_ratelimited("KVP request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + return; + } + + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION + | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, recv_buffer, + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); + + host_negotiatied = NEGO_FINISHED; + hv_poll_channel(kvp_transaction.recv_channel, kvp_poll_wrapper); +} + +static void kvp_on_reset(void) +{ + if (cancel_delayed_work_sync(&kvp_timeout_work)) + kvp_respond_to_host(NULL, HV_E_FAIL); + kvp_transaction.state = HVUTIL_DEVICE_INIT; +} + +int +hv_kvp_init(struct hv_util_service *srv) +{ + recv_buffer = srv->recv_buffer; + kvp_transaction.recv_channel = srv->channel; + kvp_transaction.recv_channel->max_pkt_size = HV_HYP_PAGE_SIZE * 4; + + /* + * When this driver loads, the user level daemon that + * processes the host requests may not yet be running. + * Defer processing channel callbacks until the daemon + * has registered. + */ + kvp_transaction.state = HVUTIL_DEVICE_INIT; + + return 0; +} + +int +hv_kvp_init_transport(void) +{ + hvt = hvutil_transport_init(kvp_devname, CN_KVP_IDX, CN_KVP_VAL, + kvp_on_msg, kvp_on_reset); + if (!hvt) + return -EFAULT; + + return 0; +} + +static void hv_kvp_cancel_work(void) +{ + cancel_delayed_work_sync(&kvp_host_handshake_work); + cancel_delayed_work_sync(&kvp_timeout_work); + cancel_work_sync(&kvp_sendkey_work); +} + +int hv_kvp_pre_suspend(void) +{ + struct vmbus_channel *channel = kvp_transaction.recv_channel; + + tasklet_disable(&channel->callback_event); + + /* + * If there is a pending transtion, it's unnecessary to tell the host + * that the transaction will fail, because that is implied when + * util_suspend() calls vmbus_close() later. + */ + hv_kvp_cancel_work(); + + /* + * Forece the state to READY to handle the ICMSGTYPE_NEGOTIATE message + * later. The user space daemon may go out of order and its write() + * may fail with EINVAL: this doesn't matter since the daemon will + * reset the device by closing and re-opening it. + */ + kvp_transaction.state = HVUTIL_READY; + return 0; +} + +int hv_kvp_pre_resume(void) +{ + struct vmbus_channel *channel = kvp_transaction.recv_channel; + + tasklet_enable(&channel->callback_event); + + return 0; +} + +void hv_kvp_deinit(void) +{ + kvp_transaction.state = HVUTIL_DEVICE_DYING; + + hv_kvp_cancel_work(); + + hvutil_transport_destroy(hvt); +} diff --git a/drivers/hv/hv_proc.c b/drivers/hv/hv_proc.c new file mode 100644 index 000000000..57b2c6419 --- /dev/null +++ b/drivers/hv/hv_proc.c @@ -0,0 +1,288 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/types.h> +#include <linux/vmalloc.h> +#include <linux/mm.h> +#include <linux/clockchips.h> +#include <linux/slab.h> +#include <linux/cpuhotplug.h> +#include <linux/minmax.h> +#include <linux/export.h> +#include <asm/mshyperv.h> + +/* + * See struct hv_deposit_memory. The first u64 is partition ID, the rest + * are GPAs. + */ +#define HV_DEPOSIT_MAX (HV_HYP_PAGE_SIZE / sizeof(u64) - 1) + +/* Deposits exact number of pages. Must be called with interrupts enabled. */ +int hv_call_deposit_pages(int node, u64 partition_id, u32 num_pages) +{ + struct page **pages, *page; + int *counts; + int num_allocations; + int i, j, page_count; + int order; + u64 status; + int ret; + u64 base_pfn; + struct hv_deposit_memory *input_page; + unsigned long flags; + + if (num_pages > HV_DEPOSIT_MAX) + return -E2BIG; + if (!num_pages) + return 0; + + /* One buffer for page pointers and counts */ + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + pages = page_address(page); + + counts = kzalloc_objs(int, HV_DEPOSIT_MAX); + if (!counts) { + free_page((unsigned long)pages); + return -ENOMEM; + } + + /* Allocate all the pages before disabling interrupts */ + i = 0; + + while (num_pages) { + /* Find highest order we can actually allocate */ + order = 31 - __builtin_clz(num_pages); + + while (1) { + pages[i] = alloc_pages_node(node, GFP_KERNEL, order); + if (pages[i]) + break; + if (!order) { + ret = -ENOMEM; + num_allocations = i; + goto err_free_allocations; + } + --order; + } + + split_page(pages[i], order); + counts[i] = 1 << order; + num_pages -= counts[i]; + i++; + } + num_allocations = i; + + local_irq_save(flags); + + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + + input_page->partition_id = partition_id; + + /* Populate gpa_page_list - these will fit on the input page */ + for (i = 0, page_count = 0; i < num_allocations; ++i) { + base_pfn = page_to_pfn(pages[i]); + for (j = 0; j < counts[i]; ++j, ++page_count) + input_page->gpa_page_list[page_count] = base_pfn + j; + } + status = hv_do_rep_hypercall(HVCALL_DEPOSIT_MEMORY, + page_count, 0, input_page, NULL); + local_irq_restore(flags); + if (!hv_result_success(status)) { + hv_status_err(status, "\n"); + ret = hv_result_to_errno(status); + goto err_free_allocations; + } + + ret = 0; + goto free_buf; + +err_free_allocations: + for (i = 0; i < num_allocations; ++i) { + base_pfn = page_to_pfn(pages[i]); + for (j = 0; j < counts[i]; ++j) + __free_page(pfn_to_page(base_pfn + j)); + } + +free_buf: + free_page((unsigned long)pages); + kfree(counts); + return ret; +} +EXPORT_SYMBOL_GPL(hv_call_deposit_pages); + +int hv_deposit_memory_node(int node, u64 partition_id, + u64 hv_status) +{ + u32 num_pages = 1; + + switch (hv_result(hv_status)) { + case HV_STATUS_INSUFFICIENT_MEMORY: + break; + case HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY: + num_pages = HV_MAX_CONTIGUOUS_ALLOCATION_PAGES; + break; + + case HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY: + num_pages = HV_MAX_CONTIGUOUS_ALLOCATION_PAGES; + fallthrough; + case HV_STATUS_INSUFFICIENT_ROOT_MEMORY: + if (!hv_root_partition()) { + hv_status_err(hv_status, "Unexpected root memory deposit\n"); + return -ENOMEM; + } + partition_id = HV_PARTITION_ID_SELF; + break; + + default: + hv_status_err(hv_status, "Unexpected!\n"); + return -ENOMEM; + } + return hv_call_deposit_pages(node, partition_id, num_pages); +} +EXPORT_SYMBOL_GPL(hv_deposit_memory_node); + +bool hv_result_needs_memory(u64 status) +{ + switch (hv_result(status)) { + case HV_STATUS_INSUFFICIENT_MEMORY: + case HV_STATUS_INSUFFICIENT_CONTIGUOUS_MEMORY: + case HV_STATUS_INSUFFICIENT_ROOT_MEMORY: + case HV_STATUS_INSUFFICIENT_CONTIGUOUS_ROOT_MEMORY: + return true; + } + return false; +} +EXPORT_SYMBOL_GPL(hv_result_needs_memory); + +int hv_call_add_logical_proc(int node, u32 lp_index, u32 apic_id) +{ + struct hv_input_add_logical_processor *input; + struct hv_output_add_logical_processor *output; + u64 status; + unsigned long flags; + int ret = 0; + + /* + * When adding a logical processor, the hypervisor may return + * HV_STATUS_INSUFFICIENT_MEMORY. When that happens, we deposit more + * pages and retry. + */ + do { + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + /* We don't do anything with the output right now */ + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + input->lp_index = lp_index; + input->apic_id = apic_id; + input->proximity_domain_info = hv_numa_node_to_pxm_info(node); + status = hv_do_hypercall(HVCALL_ADD_LOGICAL_PROCESSOR, + input, output); + local_irq_restore(flags); + + if (!hv_result_needs_memory(status)) { + if (!hv_result_success(status)) { + hv_status_err(status, "cpu %u apic ID: %u\n", + lp_index, apic_id); + ret = hv_result_to_errno(status); + } + break; + } + ret = hv_deposit_memory_node(node, hv_current_partition_id, + status); + } while (!ret); + + return ret; +} + +int hv_call_create_vp(int node, u64 partition_id, u32 vp_index, u32 flags) +{ + struct hv_create_vp *input; + u64 status; + unsigned long irq_flags; + int ret = 0; + + /* Root VPs don't seem to need pages deposited */ + if (partition_id != hv_current_partition_id) { + /* The value 90 is empirically determined. It may change. */ + ret = hv_call_deposit_pages(node, partition_id, 90); + if (ret) + return ret; + } + + do { + local_irq_save(irq_flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + + input->partition_id = partition_id; + input->vp_index = vp_index; + input->flags = flags; + input->subnode_type = HV_SUBNODE_ANY; + input->proximity_domain_info = hv_numa_node_to_pxm_info(node); + status = hv_do_hypercall(HVCALL_CREATE_VP, input, NULL); + local_irq_restore(irq_flags); + + if (!hv_result_needs_memory(status)) { + if (!hv_result_success(status)) { + hv_status_err(status, "vcpu: %u, lp: %u\n", + vp_index, flags); + ret = hv_result_to_errno(status); + } + break; + } + ret = hv_deposit_memory_node(node, partition_id, status); + + } while (!ret); + + return ret; +} +EXPORT_SYMBOL_GPL(hv_call_create_vp); + +int hv_call_notify_all_processors_started(void) +{ + struct hv_input_notify_partition_event *input; + u64 status; + unsigned long irq_flags; + int ret = 0; + + local_irq_save(irq_flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->event = HV_PARTITION_ALL_LOGICAL_PROCESSORS_STARTED; + status = hv_do_hypercall(HVCALL_NOTIFY_PARTITION_EVENT, + input, NULL); + local_irq_restore(irq_flags); + + if (!hv_result_success(status)) { + hv_status_err(status, "\n"); + ret = hv_result_to_errno(status); + } + return ret; +} + +bool hv_lp_exists(u32 lp_index) +{ + struct hv_input_get_logical_processor_run_time *input; + struct hv_output_get_logical_processor_run_time *output; + unsigned long flags; + u64 status; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + input->lp_index = lp_index; + status = hv_do_hypercall(HVCALL_GET_LOGICAL_PROCESSOR_RUN_TIME, + input, output); + local_irq_restore(flags); + + if (!hv_result_success(status) && + hv_result(status) != HV_STATUS_INVALID_LP_INDEX) { + hv_status_err(status, "\n"); + BUG(); + } + + return hv_result_success(status); +} diff --git a/drivers/hv/hv_snapshot.c b/drivers/hv/hv_snapshot.c new file mode 100644 index 000000000..506871aea --- /dev/null +++ b/drivers/hv/hv_snapshot.c @@ -0,0 +1,465 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * An implementation of host initiated guest snapshot. + * + * Copyright (C) 2013, Microsoft, Inc. + * Author : K. Y. Srinivasan <kys@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/net.h> +#include <linux/nls.h> +#include <linux/connector.h> +#include <linux/workqueue.h> +#include <linux/hyperv.h> +#include <hyperv/hvhdk.h> + +#include "hyperv_vmbus.h" +#include "hv_utils_transport.h" + +#define VSS_MAJOR 5 +#define VSS_MINOR 0 +#define VSS_VERSION (VSS_MAJOR << 16 | VSS_MINOR) + +#define VSS_VER_COUNT 1 +static const int vss_versions[] = { + VSS_VERSION +}; + +#define FW_VER_COUNT 1 +static const int fw_versions[] = { + UTIL_FW_VERSION +}; + +/* See comment with struct hv_vss_msg regarding the max VMbus packet size */ +#define VSS_MAX_PKT_SIZE (HV_HYP_PAGE_SIZE * 2) + +/* + * Timeout values are based on expecations from host + */ +#define VSS_FREEZE_TIMEOUT (15 * 60) + +/* + * Global state maintained for transaction that is being processed. For a class + * of integration services, including the "VSS service", the specified protocol + * is a "request/response" protocol which means that there can only be single + * outstanding transaction from the host at any given point in time. We use + * this to simplify memory management in this driver - we cache and process + * only one message at a time. + * + * While the request/response protocol is guaranteed by the host, we further + * ensure this by serializing packet processing in this driver - we do not + * read additional packets from the VMBUs until the current packet is fully + * handled. + */ + +static struct { + int state; /* hvutil_device_state */ + int recv_len; /* number of bytes received. */ + struct vmbus_channel *recv_channel; /* chn we got the request */ + u64 recv_req_id; /* request ID. */ + struct hv_vss_msg *msg; /* current message */ +} vss_transaction; + + +static void vss_respond_to_host(int error); + +/* + * This state maintains the version number registered by the daemon. + */ +static int dm_reg_value; + +static const char vss_devname[] = "vmbus/hv_vss"; +static __u8 *recv_buffer; +static struct hvutil_transport *hvt; + +static void vss_timeout_func(struct work_struct *dummy); +static void vss_handle_request(struct work_struct *dummy); + +static DECLARE_DELAYED_WORK(vss_timeout_work, vss_timeout_func); +static DECLARE_WORK(vss_handle_request_work, vss_handle_request); + +static void vss_poll_wrapper(void *channel) +{ + /* Transaction is finished, reset the state here to avoid races. */ + vss_transaction.state = HVUTIL_READY; + tasklet_schedule(&((struct vmbus_channel *)channel)->callback_event); +} + +/* + * Callback when data is received from user mode. + */ + +static void vss_timeout_func(struct work_struct *dummy) +{ + /* + * Timeout waiting for userspace component to reply happened. + */ + pr_warn("VSS: timeout waiting for daemon to reply\n"); + vss_respond_to_host(HV_E_FAIL); + + hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper); +} + +static void vss_register_done(void) +{ + hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper); + pr_debug("VSS: userspace daemon registered\n"); +} + +static int vss_handle_handshake(struct hv_vss_msg *vss_msg) +{ + u32 our_ver = VSS_OP_REGISTER1; + + switch (vss_msg->vss_hdr.operation) { + case VSS_OP_REGISTER: + /* Daemon doesn't expect us to reply */ + dm_reg_value = VSS_OP_REGISTER; + break; + case VSS_OP_REGISTER1: + /* Daemon expects us to reply with our own version */ + if (hvutil_transport_send(hvt, &our_ver, sizeof(our_ver), + vss_register_done)) + return -EFAULT; + dm_reg_value = VSS_OP_REGISTER1; + break; + default: + return -EINVAL; + } + pr_info("VSS: userspace daemon ver. %d connected\n", dm_reg_value); + return 0; +} + +static int vss_on_msg(void *msg, int len) +{ + struct hv_vss_msg *vss_msg = (struct hv_vss_msg *)msg; + + if (len != sizeof(*vss_msg)) { + pr_debug("VSS: Message size does not match length\n"); + return -EINVAL; + } + + if (vss_msg->vss_hdr.operation == VSS_OP_REGISTER || + vss_msg->vss_hdr.operation == VSS_OP_REGISTER1) { + /* + * Don't process registration messages if we're in the middle + * of a transaction processing. + */ + if (vss_transaction.state > HVUTIL_READY) { + pr_debug("VSS: Got unexpected registration request\n"); + return -EINVAL; + } + + return vss_handle_handshake(vss_msg); + } else if (vss_transaction.state == HVUTIL_USERSPACE_REQ) { + vss_transaction.state = HVUTIL_USERSPACE_RECV; + + if (vss_msg->vss_hdr.operation == VSS_OP_HOT_BACKUP) + vss_transaction.msg->vss_cf.flags = + VSS_HBU_NO_AUTO_RECOVERY; + + if (cancel_delayed_work_sync(&vss_timeout_work)) { + vss_respond_to_host(vss_msg->error); + /* Transaction is finished, reset the state. */ + hv_poll_channel(vss_transaction.recv_channel, + vss_poll_wrapper); + } + } else { + /* This is a spurious call! */ + pr_debug("VSS: Transaction not active\n"); + return -EINVAL; + } + return 0; +} + +static void vss_send_op(void) +{ + int op = vss_transaction.msg->vss_hdr.operation; + int rc; + struct hv_vss_msg *vss_msg; + + /* The transaction state is wrong. */ + if (vss_transaction.state != HVUTIL_HOSTMSG_RECEIVED) { + pr_debug("VSS: Unexpected attempt to send to daemon\n"); + return; + } + + vss_msg = kzalloc_obj(*vss_msg); + if (!vss_msg) + return; + + vss_msg->vss_hdr.operation = op; + + vss_transaction.state = HVUTIL_USERSPACE_REQ; + + schedule_delayed_work(&vss_timeout_work, op == VSS_OP_FREEZE ? + secs_to_jiffies(VSS_FREEZE_TIMEOUT) : + secs_to_jiffies(HV_UTIL_TIMEOUT)); + + rc = hvutil_transport_send(hvt, vss_msg, sizeof(*vss_msg), NULL); + if (rc) { + pr_warn("VSS: failed to communicate to the daemon: %d\n", rc); + if (cancel_delayed_work_sync(&vss_timeout_work)) { + vss_respond_to_host(HV_E_FAIL); + vss_transaction.state = HVUTIL_READY; + } + } + + kfree(vss_msg); +} + +static void vss_handle_request(struct work_struct *dummy) +{ + switch (vss_transaction.msg->vss_hdr.operation) { + /* + * Initiate a "freeze/thaw" operation in the guest. + * We respond to the host once the operation is complete. + * + * We send the message to the user space daemon and the operation is + * performed in the daemon. + */ + case VSS_OP_THAW: + case VSS_OP_FREEZE: + case VSS_OP_HOT_BACKUP: + if (vss_transaction.state < HVUTIL_READY) { + /* Userspace is not registered yet */ + pr_debug("VSS: Not ready for request.\n"); + vss_respond_to_host(HV_E_FAIL); + return; + } + + pr_debug("VSS: Received request for op code: %d\n", + vss_transaction.msg->vss_hdr.operation); + vss_transaction.state = HVUTIL_HOSTMSG_RECEIVED; + vss_send_op(); + return; + case VSS_OP_GET_DM_INFO: + vss_transaction.msg->dm_info.flags = 0; + break; + default: + break; + } + + vss_respond_to_host(0); + hv_poll_channel(vss_transaction.recv_channel, vss_poll_wrapper); +} + +/* + * Send a response back to the host. + */ + +static void +vss_respond_to_host(int error) +{ + struct icmsg_hdr *icmsghdrp; + u32 buf_len; + struct vmbus_channel *channel; + u64 req_id; + + /* + * Copy the global state for completing the transaction. Note that + * only one transaction can be active at a time. + */ + + buf_len = vss_transaction.recv_len; + channel = vss_transaction.recv_channel; + req_id = vss_transaction.recv_req_id; + + icmsghdrp = (struct icmsg_hdr *) + &recv_buffer[sizeof(struct vmbuspipe_hdr)]; + + if (channel->onchannel_callback == NULL) + /* + * We have raced with util driver being unloaded; + * silently return. + */ + return; + + icmsghdrp->status = error; + + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, recv_buffer, buf_len, req_id, + VM_PKT_DATA_INBAND, 0); + +} + +/* + * This callback is invoked when we get a VSS message from the host. + * The host ensures that only one VSS transaction can be active at a time. + */ + +void hv_vss_onchannelcallback(void *context) +{ + struct vmbus_channel *channel = context; + u32 recvlen; + u64 requestid; + struct hv_vss_msg *vss_msg; + int vss_srv_version; + + struct icmsg_hdr *icmsghdrp; + + if (vss_transaction.state > HVUTIL_READY) + return; + + if (vmbus_recvpacket(channel, recv_buffer, VSS_MAX_PKT_SIZE, &recvlen, &requestid)) { + pr_err_ratelimited("VSS request received. Could not read into recv buf\n"); + return; + } + + if (!recvlen) + return; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("VSS request received. Packet length too small: %d\n", + recvlen); + return; + } + + icmsghdrp = (struct icmsg_hdr *)&recv_buffer[sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + recv_buffer, recvlen, + fw_versions, FW_VER_COUNT, + vss_versions, VSS_VER_COUNT, + NULL, &vss_srv_version)) { + + pr_info("VSS IC version %d.%d\n", + vss_srv_version >> 16, + vss_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_VSS) { + /* Ensure recvlen is big enough to contain hv_vss_msg */ + if (recvlen < ICMSG_HDR + sizeof(struct hv_vss_msg)) { + pr_err_ratelimited("Invalid VSS msg. Packet length too small: %u\n", + recvlen); + return; + } + vss_msg = (struct hv_vss_msg *)&recv_buffer[ICMSG_HDR]; + + /* + * Stash away this global state for completing the + * transaction; note transactions are serialized. + */ + + vss_transaction.recv_len = recvlen; + vss_transaction.recv_req_id = requestid; + vss_transaction.msg = (struct hv_vss_msg *)vss_msg; + + schedule_work(&vss_handle_request_work); + return; + } else { + pr_err_ratelimited("VSS request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + return; + } + + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION | + ICMSGHDRFLAG_RESPONSE; + vmbus_sendpacket(channel, recv_buffer, recvlen, requestid, + VM_PKT_DATA_INBAND, 0); +} + +static void vss_on_reset(void) +{ + if (cancel_delayed_work_sync(&vss_timeout_work)) + vss_respond_to_host(HV_E_FAIL); + vss_transaction.state = HVUTIL_DEVICE_INIT; +} + +int +hv_vss_init(struct hv_util_service *srv) +{ + if (vmbus_proto_version < VERSION_WIN8_1) { + pr_warn("Integration service 'Backup (volume snapshot)'" + " not supported on this host version.\n"); + return -ENOTSUPP; + } + recv_buffer = srv->recv_buffer; + vss_transaction.recv_channel = srv->channel; + vss_transaction.recv_channel->max_pkt_size = VSS_MAX_PKT_SIZE; + + /* + * When this driver loads, the user level daemon that + * processes the host requests may not yet be running. + * Defer processing channel callbacks until the daemon + * has registered. + */ + vss_transaction.state = HVUTIL_DEVICE_INIT; + + return 0; +} + +int +hv_vss_init_transport(void) +{ + hvt = hvutil_transport_init(vss_devname, CN_VSS_IDX, CN_VSS_VAL, + vss_on_msg, vss_on_reset); + if (!hvt) { + pr_warn("VSS: Failed to initialize transport\n"); + return -EFAULT; + } + + return 0; +} + +static void hv_vss_cancel_work(void) +{ + cancel_delayed_work_sync(&vss_timeout_work); + cancel_work_sync(&vss_handle_request_work); +} + +int hv_vss_pre_suspend(void) +{ + struct vmbus_channel *channel = vss_transaction.recv_channel; + struct hv_vss_msg *vss_msg; + + /* + * Fake a THAW message for the user space daemon in case the daemon + * has frozen the file systems. It doesn't matter if there is already + * a message pending to be delivered to the user space since we force + * vss_transaction.state to be HVUTIL_READY, so the user space daemon's + * write() will fail with EINVAL (see vss_on_msg()), and the daemon + * will reset the device by closing and re-opening it. + */ + vss_msg = kzalloc_obj(*vss_msg); + if (!vss_msg) + return -ENOMEM; + + tasklet_disable(&channel->callback_event); + + vss_msg->vss_hdr.operation = VSS_OP_THAW; + + /* Cancel any possible pending work. */ + hv_vss_cancel_work(); + + /* We don't care about the return value. */ + hvutil_transport_send(hvt, vss_msg, sizeof(*vss_msg), NULL); + + kfree(vss_msg); + + vss_transaction.state = HVUTIL_READY; + + /* tasklet_enable() will be called in hv_vss_pre_resume(). */ + return 0; +} + +int hv_vss_pre_resume(void) +{ + struct vmbus_channel *channel = vss_transaction.recv_channel; + + tasklet_enable(&channel->callback_event); + + return 0; +} + +void hv_vss_deinit(void) +{ + vss_transaction.state = HVUTIL_DEVICE_DYING; + + hv_vss_cancel_work(); + + hvutil_transport_destroy(hvt); +} diff --git a/drivers/hv/hv_trace.c b/drivers/hv/hv_trace.c new file mode 100644 index 000000000..38d359cf1 --- /dev/null +++ b/drivers/hv/hv_trace.c @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: GPL-2.0 + +#include "hyperv_vmbus.h" + +#define CREATE_TRACE_POINTS +#include "hv_trace.h" diff --git a/drivers/hv/hv_trace.h b/drivers/hv/hv_trace.h new file mode 100644 index 000000000..c02a1719e --- /dev/null +++ b/drivers/hv/hv_trace.h @@ -0,0 +1,361 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM hyperv + +#if !defined(_HV_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _HV_TRACE_H + +#include <linux/tracepoint.h> + +DECLARE_EVENT_CLASS(vmbus_hdr_msg, + TP_PROTO(const struct vmbus_channel_message_header *hdr), + TP_ARGS(hdr), + TP_STRUCT__entry(__field(unsigned int, msgtype)), + TP_fast_assign(__entry->msgtype = hdr->msgtype;), + TP_printk("msgtype=%u", __entry->msgtype) +); + +DEFINE_EVENT(vmbus_hdr_msg, vmbus_on_msg_dpc, + TP_PROTO(const struct vmbus_channel_message_header *hdr), + TP_ARGS(hdr) +); + +DEFINE_EVENT(vmbus_hdr_msg, vmbus_on_message, + TP_PROTO(const struct vmbus_channel_message_header *hdr), + TP_ARGS(hdr) +); + +TRACE_EVENT(vmbus_onoffer, + TP_PROTO(const struct vmbus_channel_offer_channel *offer), + TP_ARGS(offer), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u8, monitorid) + __field(u16, is_ddc_int) + __field(u32, connection_id) + __array(char, if_type, 16) + __array(char, if_instance, 16) + __field(u16, chn_flags) + __field(u16, mmio_mb) + __field(u16, sub_idx) + ), + TP_fast_assign(__entry->child_relid = offer->child_relid; + __entry->monitorid = offer->monitorid; + __entry->is_ddc_int = offer->is_dedicated_interrupt; + __entry->connection_id = offer->connection_id; + export_guid(__entry->if_type, &offer->offer.if_type); + export_guid(__entry->if_instance, &offer->offer.if_instance); + __entry->chn_flags = offer->offer.chn_flags; + __entry->mmio_mb = offer->offer.mmio_megabytes; + __entry->sub_idx = offer->offer.sub_channel_index; + ), + TP_printk("child_relid 0x%x, monitorid 0x%x, is_dedicated %d, " + "connection_id 0x%x, if_type %pUl, if_instance %pUl, " + "chn_flags 0x%x, mmio_megabytes %d, sub_channel_index %d", + __entry->child_relid, __entry->monitorid, + __entry->is_ddc_int, __entry->connection_id, + __entry->if_type, __entry->if_instance, + __entry->chn_flags, __entry->mmio_mb, + __entry->sub_idx + ) + ); + +TRACE_EVENT(vmbus_onoffer_rescind, + TP_PROTO(const struct vmbus_channel_rescind_offer *offer), + TP_ARGS(offer), + TP_STRUCT__entry(__field(u32, child_relid)), + TP_fast_assign(__entry->child_relid = offer->child_relid), + TP_printk("child_relid 0x%x", __entry->child_relid) + ); + +TRACE_EVENT(vmbus_onopen_result, + TP_PROTO(const struct vmbus_channel_open_result *result), + TP_ARGS(result), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, openid) + __field(u32, status) + ), + TP_fast_assign(__entry->child_relid = result->child_relid; + __entry->openid = result->openid; + __entry->status = result->status; + ), + TP_printk("child_relid 0x%x, openid %d, status %d", + __entry->child_relid, __entry->openid, __entry->status + ) + ); + +TRACE_EVENT(vmbus_ongpadl_created, + TP_PROTO(const struct vmbus_channel_gpadl_created *gpadlcreated), + TP_ARGS(gpadlcreated), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, gpadl) + __field(u32, status) + ), + TP_fast_assign(__entry->child_relid = gpadlcreated->child_relid; + __entry->gpadl = gpadlcreated->gpadl; + __entry->status = gpadlcreated->creation_status; + ), + TP_printk("child_relid 0x%x, gpadl 0x%x, creation_status %d", + __entry->child_relid, __entry->gpadl, __entry->status + ) + ); + +TRACE_EVENT(vmbus_onmodifychannel_response, + TP_PROTO(const struct vmbus_channel_modifychannel_response *response), + TP_ARGS(response), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, status) + ), + TP_fast_assign(__entry->child_relid = response->child_relid; + __entry->status = response->status; + ), + TP_printk("child_relid 0x%x, status %d", + __entry->child_relid, __entry->status + ) + ); + +TRACE_EVENT(vmbus_ongpadl_torndown, + TP_PROTO(const struct vmbus_channel_gpadl_torndown *gpadltorndown), + TP_ARGS(gpadltorndown), + TP_STRUCT__entry(__field(u32, gpadl)), + TP_fast_assign(__entry->gpadl = gpadltorndown->gpadl), + TP_printk("gpadl 0x%x", __entry->gpadl) + ); + +TRACE_EVENT(vmbus_onversion_response, + TP_PROTO(const struct vmbus_channel_version_response *response), + TP_ARGS(response), + TP_STRUCT__entry( + __field(u8, ver) + ), + TP_fast_assign(__entry->ver = response->version_supported; + ), + TP_printk("version_supported %d", __entry->ver) + ); + +TRACE_EVENT(vmbus_request_offers, + TP_PROTO(int ret), + TP_ARGS(ret), + TP_STRUCT__entry(__field(int, ret)), + TP_fast_assign(__entry->ret = ret), + TP_printk("sending ret %d", __entry->ret) + ); + +TRACE_EVENT(vmbus_open, + TP_PROTO(const struct vmbus_channel_open_channel *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, openid) + __field(u32, gpadlhandle) + __field(u32, target_vp) + __field(u32, offset) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->openid = msg->openid; + __entry->gpadlhandle = msg->ringbuffer_gpadlhandle; + __entry->target_vp = msg->target_vp; + __entry->offset = msg->downstream_ringbuffer_pageoffset; + __entry->ret = ret; + ), + TP_printk("sending child_relid 0x%x, openid %d, " + "gpadlhandle 0x%x, target_vp 0x%x, offset 0x%x, ret %d", + __entry->child_relid, __entry->openid, + __entry->gpadlhandle, __entry->target_vp, + __entry->offset, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_close_internal, + TP_PROTO(const struct vmbus_channel_close_channel *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->ret = ret; + ), + TP_printk("sending child_relid 0x%x, ret %d", __entry->child_relid, + __entry->ret) + ); + +TRACE_EVENT(vmbus_establish_gpadl_header, + TP_PROTO(const struct vmbus_channel_gpadl_header *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, gpadl) + __field(u16, range_buflen) + __field(u16, rangecount) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->gpadl = msg->gpadl; + __entry->range_buflen = msg->range_buflen; + __entry->rangecount = msg->rangecount; + __entry->ret = ret; + ), + TP_printk("sending child_relid 0x%x, gpadl 0x%x, range_buflen %d " + "rangecount %d, ret %d", + __entry->child_relid, __entry->gpadl, + __entry->range_buflen, __entry->rangecount, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_establish_gpadl_body, + TP_PROTO(const struct vmbus_channel_gpadl_body *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, msgnumber) + __field(u32, gpadl) + __field(int, ret) + ), + TP_fast_assign( + __entry->msgnumber = msg->msgnumber; + __entry->gpadl = msg->gpadl; + __entry->ret = ret; + ), + TP_printk("sending msgnumber %d, gpadl 0x%x, ret %d", + __entry->msgnumber, __entry->gpadl, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_teardown_gpadl, + TP_PROTO(const struct vmbus_channel_gpadl_teardown *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, gpadl) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->gpadl = msg->gpadl; + __entry->ret = ret; + ), + TP_printk("sending child_relid 0x%x, gpadl 0x%x, ret %d", + __entry->child_relid, __entry->gpadl, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_negotiate_version, + TP_PROTO(const struct vmbus_channel_initiate_contact *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, ver) + __field(u32, target_vcpu) + __field(int, ret) + __field(u64, int_page) + __field(u64, mon_page1) + __field(u64, mon_page2) + ), + TP_fast_assign( + __entry->ver = msg->vmbus_version_requested; + __entry->target_vcpu = msg->target_vcpu; + __entry->int_page = msg->interrupt_page; + __entry->mon_page1 = msg->monitor_page1; + __entry->mon_page2 = msg->monitor_page2; + __entry->ret = ret; + ), + TP_printk("sending vmbus_version_requested %d, target_vcpu 0x%x, " + "pages %llx:%llx:%llx, ret %d", + __entry->ver, __entry->target_vcpu, __entry->int_page, + __entry->mon_page1, __entry->mon_page2, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_release_relid, + TP_PROTO(const struct vmbus_channel_relid_released *msg, int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->ret = ret; + ), + TP_printk("sending child_relid 0x%x, ret %d", + __entry->child_relid, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_send_tl_connect_request, + TP_PROTO(const struct vmbus_channel_tl_connect_request *msg, + int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __array(char, guest_id, 16) + __array(char, host_id, 16) + __field(int, ret) + ), + TP_fast_assign( + export_guid(__entry->guest_id, &msg->guest_endpoint_id); + export_guid(__entry->host_id, &msg->host_service_id); + __entry->ret = ret; + ), + TP_printk("sending guest_endpoint_id %pUl, host_service_id %pUl, " + "ret %d", + __entry->guest_id, __entry->host_id, __entry->ret + ) + ); + +TRACE_EVENT(vmbus_send_modifychannel, + TP_PROTO(const struct vmbus_channel_modifychannel *msg, + int ret), + TP_ARGS(msg, ret), + TP_STRUCT__entry( + __field(u32, child_relid) + __field(u32, target_vp) + __field(int, ret) + ), + TP_fast_assign( + __entry->child_relid = msg->child_relid; + __entry->target_vp = msg->target_vp; + __entry->ret = ret; + ), + TP_printk("binding child_relid 0x%x to target_vp 0x%x, ret %d", + __entry->child_relid, __entry->target_vp, __entry->ret + ) + ); + +DECLARE_EVENT_CLASS(vmbus_channel, + TP_PROTO(const struct vmbus_channel *channel), + TP_ARGS(channel), + TP_STRUCT__entry(__field(u32, relid)), + TP_fast_assign(__entry->relid = channel->offermsg.child_relid), + TP_printk("relid 0x%x", __entry->relid) +); + +DEFINE_EVENT(vmbus_channel, vmbus_chan_sched, + TP_PROTO(const struct vmbus_channel *channel), + TP_ARGS(channel) +); + +DEFINE_EVENT(vmbus_channel, vmbus_setevent, + TP_PROTO(const struct vmbus_channel *channel), + TP_ARGS(channel) +); + +DEFINE_EVENT(vmbus_channel, vmbus_on_event, + TP_PROTO(const struct vmbus_channel *channel), + TP_ARGS(channel) +); + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE hv_trace +#endif /* _HV_TRACE_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/drivers/hv/hv_trace_balloon.h b/drivers/hv/hv_trace_balloon.h new file mode 100644 index 000000000..93082888a --- /dev/null +++ b/drivers/hv/hv_trace_balloon.h @@ -0,0 +1,48 @@ +#undef TRACE_SYSTEM +#define TRACE_SYSTEM hyperv + +#if !defined(_HV_TRACE_BALLOON_H) || defined(TRACE_HEADER_MULTI_READ) +#define _HV_TRACE_BALLOON_H + +#include <linux/tracepoint.h> + +TRACE_EVENT(balloon_status, + TP_PROTO(u64 available, u64 committed, + unsigned long vm_memory_committed, + unsigned long pages_ballooned, + unsigned long pages_added, + unsigned long pages_onlined), + TP_ARGS(available, committed, vm_memory_committed, + pages_ballooned, pages_added, pages_onlined), + TP_STRUCT__entry( + __field(u64, available) + __field(u64, committed) + __field(unsigned long, vm_memory_committed) + __field(unsigned long, pages_ballooned) + __field(unsigned long, pages_added) + __field(unsigned long, pages_onlined) + ), + TP_fast_assign( + __entry->available = available; + __entry->committed = committed; + __entry->vm_memory_committed = vm_memory_committed; + __entry->pages_ballooned = pages_ballooned; + __entry->pages_added = pages_added; + __entry->pages_onlined = pages_onlined; + ), + TP_printk("available %lld, committed %lld; vm_memory_committed %ld;" + " pages_ballooned %ld, pages_added %ld, pages_onlined %ld", + __entry->available, __entry->committed, + __entry->vm_memory_committed, __entry->pages_ballooned, + __entry->pages_added, __entry->pages_onlined + ) + ); + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE hv_trace_balloon +#endif /* _HV_TRACE_BALLOON_H */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/drivers/hv/hv_util.c b/drivers/hv/hv_util.c new file mode 100644 index 000000000..7e9c8e169 --- /dev/null +++ b/drivers/hv/hv_util.c @@ -0,0 +1,815 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2010, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/init.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <linux/sysctl.h> +#include <linux/reboot.h> +#include <linux/hyperv.h> +#include <linux/clockchips.h> +#include <linux/ptp_clock_kernel.h> +#include <asm/mshyperv.h> + +#include "hyperv_vmbus.h" + +#define SD_MAJOR 3 +#define SD_MINOR 0 +#define SD_MINOR_1 1 +#define SD_MINOR_2 2 +#define SD_VERSION_3_1 (SD_MAJOR << 16 | SD_MINOR_1) +#define SD_VERSION_3_2 (SD_MAJOR << 16 | SD_MINOR_2) +#define SD_VERSION (SD_MAJOR << 16 | SD_MINOR) + +#define SD_MAJOR_1 1 +#define SD_VERSION_1 (SD_MAJOR_1 << 16 | SD_MINOR) + +#define TS_MAJOR 4 +#define TS_MINOR 0 +#define TS_VERSION (TS_MAJOR << 16 | TS_MINOR) + +#define TS_MAJOR_1 1 +#define TS_VERSION_1 (TS_MAJOR_1 << 16 | TS_MINOR) + +#define TS_MAJOR_3 3 +#define TS_VERSION_3 (TS_MAJOR_3 << 16 | TS_MINOR) + +#define HB_MAJOR 3 +#define HB_MINOR 0 +#define HB_VERSION (HB_MAJOR << 16 | HB_MINOR) + +#define HB_MAJOR_1 1 +#define HB_VERSION_1 (HB_MAJOR_1 << 16 | HB_MINOR) + +static int sd_srv_version; +static int ts_srv_version; +static int hb_srv_version; + +#define SD_VER_COUNT 4 +static const int sd_versions[] = { + SD_VERSION_3_2, + SD_VERSION_3_1, + SD_VERSION, + SD_VERSION_1 +}; + +#define TS_VER_COUNT 3 +static const int ts_versions[] = { + TS_VERSION, + TS_VERSION_3, + TS_VERSION_1 +}; + +#define HB_VER_COUNT 2 +static const int hb_versions[] = { + HB_VERSION, + HB_VERSION_1 +}; + +#define FW_VER_COUNT 2 +static const int fw_versions[] = { + UTIL_FW_VERSION, + UTIL_WS2K8_FW_VERSION +}; + +/* + * Send the "hibernate" udev event in a thread context. + */ +struct hibernate_work_context { + struct work_struct work; + struct hv_device *dev; +}; + +static struct hibernate_work_context hibernate_context; +static bool hibernation_supported; + +static void send_hibernate_uevent(struct work_struct *work) +{ + char *uevent_env[2] = { "EVENT=hibernate", NULL }; + struct hibernate_work_context *ctx; + + ctx = container_of(work, struct hibernate_work_context, work); + + kobject_uevent_env(&ctx->dev->device.kobj, KOBJ_CHANGE, uevent_env); + + pr_info("Sent hibernation uevent\n"); +} + +static int hv_shutdown_init(struct hv_util_service *srv) +{ + struct vmbus_channel *channel = srv->channel; + + INIT_WORK(&hibernate_context.work, send_hibernate_uevent); + hibernate_context.dev = channel->device_obj; + + hibernation_supported = hv_is_hibernation_supported(); + + return 0; +} + +static void shutdown_onchannelcallback(void *context); +static struct hv_util_service util_shutdown = { + .util_cb = shutdown_onchannelcallback, + .util_init = hv_shutdown_init, +}; + +static int hv_timesync_init(struct hv_util_service *srv); +static int hv_timesync_pre_suspend(void); +static void hv_timesync_deinit(void); + +static void timesync_onchannelcallback(void *context); +static struct hv_util_service util_timesynch = { + .util_cb = timesync_onchannelcallback, + .util_init = hv_timesync_init, + .util_pre_suspend = hv_timesync_pre_suspend, + .util_deinit = hv_timesync_deinit, +}; + +static void heartbeat_onchannelcallback(void *context); +static struct hv_util_service util_heartbeat = { + .util_cb = heartbeat_onchannelcallback, +}; + +static struct hv_util_service util_kvp = { + .util_cb = hv_kvp_onchannelcallback, + .util_init = hv_kvp_init, + .util_init_transport = hv_kvp_init_transport, + .util_pre_suspend = hv_kvp_pre_suspend, + .util_pre_resume = hv_kvp_pre_resume, + .util_deinit = hv_kvp_deinit, +}; + +static struct hv_util_service util_vss = { + .util_cb = hv_vss_onchannelcallback, + .util_init = hv_vss_init, + .util_init_transport = hv_vss_init_transport, + .util_pre_suspend = hv_vss_pre_suspend, + .util_pre_resume = hv_vss_pre_resume, + .util_deinit = hv_vss_deinit, +}; + +static void perform_shutdown(struct work_struct *dummy) +{ + orderly_poweroff(true); +} + +static void perform_restart(struct work_struct *dummy) +{ + orderly_reboot(); +} + +/* + * Perform the shutdown operation in a thread context. + */ +static DECLARE_WORK(shutdown_work, perform_shutdown); + +/* + * Perform the restart operation in a thread context. + */ +static DECLARE_WORK(restart_work, perform_restart); + +static void shutdown_onchannelcallback(void *context) +{ + struct vmbus_channel *channel = context; + struct work_struct *work = NULL; + u32 recvlen; + u64 requestid; + u8 *shut_txf_buf = util_shutdown.recv_buffer; + + struct shutdown_msg_data *shutdown_msg; + + struct icmsg_hdr *icmsghdrp; + + if (vmbus_recvpacket(channel, shut_txf_buf, HV_HYP_PAGE_SIZE, &recvlen, &requestid)) { + pr_err_ratelimited("Shutdown request received. Could not read into shut txf buf\n"); + return; + } + + if (!recvlen) + return; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Shutdown request received. Packet length too small: %d\n", + recvlen); + return; + } + + icmsghdrp = (struct icmsg_hdr *)&shut_txf_buf[sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + shut_txf_buf, recvlen, + fw_versions, FW_VER_COUNT, + sd_versions, SD_VER_COUNT, + NULL, &sd_srv_version)) { + pr_info("Shutdown IC version %d.%d\n", + sd_srv_version >> 16, + sd_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_SHUTDOWN) { + /* Ensure recvlen is big enough to contain shutdown_msg_data struct */ + if (recvlen < ICMSG_HDR + sizeof(struct shutdown_msg_data)) { + pr_err_ratelimited("Invalid shutdown msg data. Packet length too small: %u\n", + recvlen); + return; + } + + shutdown_msg = (struct shutdown_msg_data *)&shut_txf_buf[ICMSG_HDR]; + + /* + * shutdown_msg->flags can be 0(shut down), 2(reboot), + * or 4(hibernate). It may bitwise-OR 1, which means + * performing the request by force. Linux always tries + * to perform the request by force. + */ + switch (shutdown_msg->flags) { + case 0: + case 1: + icmsghdrp->status = HV_S_OK; + work = &shutdown_work; + pr_info("Shutdown request received - graceful shutdown initiated\n"); + break; + case 2: + case 3: + icmsghdrp->status = HV_S_OK; + work = &restart_work; + pr_info("Restart request received - graceful restart initiated\n"); + break; + case 4: + case 5: + pr_info("Hibernation request received\n"); + icmsghdrp->status = hibernation_supported ? + HV_S_OK : HV_E_FAIL; + if (hibernation_supported) + work = &hibernate_context.work; + break; + default: + icmsghdrp->status = HV_E_FAIL; + pr_info("Shutdown request received - Invalid request\n"); + break; + } + } else { + icmsghdrp->status = HV_E_FAIL; + pr_err_ratelimited("Shutdown request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + } + + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION + | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, shut_txf_buf, + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); + + if (work) + schedule_work(work); +} + +/* + * Set the host time in a process context. + */ +static struct work_struct adj_time_work; + +/* + * The last time sample, received from the host. PTP device responds to + * requests by using this data and the current partition-wide time reference + * count. + */ +static struct { + u64 host_time; + u64 ref_time; + spinlock_t lock; +} host_ts; + +static bool timesync_implicit; + +module_param(timesync_implicit, bool, 0644); +MODULE_PARM_DESC(timesync_implicit, "If set treat SAMPLE as SYNC when clock is behind"); + +static inline u64 reftime_to_ns(u64 reftime) +{ + return (reftime - WLTIMEDELTA) * 100; +} + +/* + * Hard coded threshold for host timesync delay: 600 seconds + */ +static const u64 HOST_TIMESYNC_DELAY_THRESH = 600 * (u64)NSEC_PER_SEC; + +static int hv_get_adj_host_time(struct timespec64 *ts) +{ + u64 newtime, reftime, timediff_adj; + unsigned long flags; + int ret = 0; + + spin_lock_irqsave(&host_ts.lock, flags); + reftime = hv_read_reference_counter(); + + /* + * We need to let the caller know that last update from host + * is older than the max allowable threshold. clock_gettime() + * and PTP ioctl do not have a documented error that we could + * return for this specific case. Use ESTALE to report this. + */ + timediff_adj = reftime - host_ts.ref_time; + if (timediff_adj * 100 > HOST_TIMESYNC_DELAY_THRESH) { + pr_warn_once("TIMESYNC IC: Stale time stamp, %llu nsecs old\n", + (timediff_adj * 100)); + ret = -ESTALE; + } + + newtime = host_ts.host_time + timediff_adj; + *ts = ns_to_timespec64(reftime_to_ns(newtime)); + spin_unlock_irqrestore(&host_ts.lock, flags); + + return ret; +} + +static void hv_set_host_time(struct work_struct *work) +{ + + struct timespec64 ts; + + if (!hv_get_adj_host_time(&ts)) + do_settimeofday64(&ts); +} + +/* + * Due to a bug on Hyper-V hosts, the sync flag may not always be sent on resume. + * Force a sync if the guest is behind. + */ +static inline bool hv_implicit_sync(u64 host_time) +{ + struct timespec64 new_ts; + struct timespec64 threshold_ts; + + new_ts = ns_to_timespec64(reftime_to_ns(host_time)); + ktime_get_real_ts64(&threshold_ts); + + threshold_ts.tv_sec += 5; + + /* + * If guest behind the host by 5 or more seconds. + */ + if (timespec64_compare(&new_ts, &threshold_ts) >= 0) + return true; + + return false; +} + +/* + * Synchronize time with host after reboot, restore, etc. + * + * ICTIMESYNCFLAG_SYNC flag bit indicates reboot, restore events of the VM. + * After reboot the flag ICTIMESYNCFLAG_SYNC is included in the first time + * message after the timesync channel is opened. Since the hv_utils module is + * loaded after hv_vmbus, the first message is usually missed. This bit is + * considered a hard request to discipline the clock. + * + * ICTIMESYNCFLAG_SAMPLE bit indicates a time sample from host. This is + * typically used as a hint to the guest. The guest is under no obligation + * to discipline the clock. + */ +static inline void adj_guesttime(u64 hosttime, u64 reftime, u8 adj_flags) +{ + unsigned long flags; + u64 cur_reftime; + + /* + * Save the adjusted time sample from the host and the snapshot + * of the current system time. + */ + spin_lock_irqsave(&host_ts.lock, flags); + + cur_reftime = hv_read_reference_counter(); + host_ts.host_time = hosttime; + host_ts.ref_time = cur_reftime; + + /* + * TimeSync v4 messages contain reference time (guest's Hyper-V + * clocksource read when the time sample was generated), we can + * improve the precision by adding the delta between now and the + * time of generation. For older protocols we set + * reftime == cur_reftime on call. + */ + host_ts.host_time += (cur_reftime - reftime); + + spin_unlock_irqrestore(&host_ts.lock, flags); + + /* Schedule work to do do_settimeofday64() */ + if ((adj_flags & ICTIMESYNCFLAG_SYNC) || + (timesync_implicit && hv_implicit_sync(host_ts.host_time))) + schedule_work(&adj_time_work); +} + +/* + * Time Sync Channel message handler. + */ +static void timesync_onchannelcallback(void *context) +{ + struct vmbus_channel *channel = context; + u32 recvlen; + u64 requestid; + struct icmsg_hdr *icmsghdrp; + struct ictimesync_data *timedatap; + struct ictimesync_ref_data *refdata; + u8 *time_txf_buf = util_timesynch.recv_buffer; + + /* + * Drain the ring buffer and use the last packet to update + * host_ts + */ + while (1) { + int ret = vmbus_recvpacket(channel, time_txf_buf, + HV_HYP_PAGE_SIZE, &recvlen, + &requestid); + if (ret) { + pr_err_ratelimited("TimeSync IC pkt recv failed (Err: %d)\n", + ret); + break; + } + + if (!recvlen) + break; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Timesync request received. Packet length too small: %d\n", + recvlen); + break; + } + + icmsghdrp = (struct icmsg_hdr *)&time_txf_buf[ + sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + time_txf_buf, recvlen, + fw_versions, FW_VER_COUNT, + ts_versions, TS_VER_COUNT, + NULL, &ts_srv_version)) { + pr_info("TimeSync IC version %d.%d\n", + ts_srv_version >> 16, + ts_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_TIMESYNC) { + if (ts_srv_version > TS_VERSION_3) { + /* Ensure recvlen is big enough to read ictimesync_ref_data */ + if (recvlen < ICMSG_HDR + sizeof(struct ictimesync_ref_data)) { + pr_err_ratelimited("Invalid ictimesync ref data. Length too small: %u\n", + recvlen); + break; + } + refdata = (struct ictimesync_ref_data *)&time_txf_buf[ICMSG_HDR]; + + adj_guesttime(refdata->parenttime, + refdata->vmreferencetime, + refdata->flags); + } else { + /* Ensure recvlen is big enough to read ictimesync_data */ + if (recvlen < ICMSG_HDR + sizeof(struct ictimesync_data)) { + pr_err_ratelimited("Invalid ictimesync data. Length too small: %u\n", + recvlen); + break; + } + timedatap = (struct ictimesync_data *)&time_txf_buf[ICMSG_HDR]; + + adj_guesttime(timedatap->parenttime, + hv_read_reference_counter(), + timedatap->flags); + } + } else { + icmsghdrp->status = HV_E_FAIL; + pr_err_ratelimited("Timesync request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + } + + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION + | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, time_txf_buf, + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); + } +} + +/* + * Heartbeat functionality. + * Every two seconds, Hyper-V send us a heartbeat request message. + * we respond to this message, and Hyper-V knows we are alive. + */ +static void heartbeat_onchannelcallback(void *context) +{ + struct vmbus_channel *channel = context; + u32 recvlen; + u64 requestid; + struct icmsg_hdr *icmsghdrp; + struct heartbeat_msg_data *heartbeat_msg; + u8 *hbeat_txf_buf = util_heartbeat.recv_buffer; + + while (1) { + + if (vmbus_recvpacket(channel, hbeat_txf_buf, HV_HYP_PAGE_SIZE, + &recvlen, &requestid)) { + pr_err_ratelimited("Heartbeat request received. Could not read into hbeat txf buf\n"); + return; + } + + if (!recvlen) + break; + + /* Ensure recvlen is big enough to read header data */ + if (recvlen < ICMSG_HDR) { + pr_err_ratelimited("Heartbeat request received. Packet length too small: %d\n", + recvlen); + break; + } + + icmsghdrp = (struct icmsg_hdr *)&hbeat_txf_buf[ + sizeof(struct vmbuspipe_hdr)]; + + if (icmsghdrp->icmsgtype == ICMSGTYPE_NEGOTIATE) { + if (vmbus_prep_negotiate_resp(icmsghdrp, + hbeat_txf_buf, recvlen, + fw_versions, FW_VER_COUNT, + hb_versions, HB_VER_COUNT, + NULL, &hb_srv_version)) { + + pr_info("Heartbeat IC version %d.%d\n", + hb_srv_version >> 16, + hb_srv_version & 0xFFFF); + } + } else if (icmsghdrp->icmsgtype == ICMSGTYPE_HEARTBEAT) { + /* + * Ensure recvlen is big enough to read seq_num. Reserved area is not + * included in the check as the host may not fill it up entirely + */ + if (recvlen < ICMSG_HDR + sizeof(u64)) { + pr_err_ratelimited("Invalid heartbeat msg data. Length too small: %u\n", + recvlen); + break; + } + heartbeat_msg = (struct heartbeat_msg_data *)&hbeat_txf_buf[ICMSG_HDR]; + + heartbeat_msg->seq_num += 1; + } else { + icmsghdrp->status = HV_E_FAIL; + pr_err_ratelimited("Heartbeat request received. Invalid msg type: %d\n", + icmsghdrp->icmsgtype); + } + + icmsghdrp->icflags = ICMSGHDRFLAG_TRANSACTION + | ICMSGHDRFLAG_RESPONSE; + + vmbus_sendpacket(channel, hbeat_txf_buf, + recvlen, requestid, + VM_PKT_DATA_INBAND, 0); + } +} + +#define HV_UTIL_RING_SEND_SIZE VMBUS_RING_SIZE(3 * HV_HYP_PAGE_SIZE) +#define HV_UTIL_RING_RECV_SIZE VMBUS_RING_SIZE(3 * HV_HYP_PAGE_SIZE) + +static int util_probe(struct hv_device *dev, + const struct hv_vmbus_device_id *dev_id) +{ + struct hv_util_service *srv = + (struct hv_util_service *)dev_id->driver_data; + int ret; + + srv->recv_buffer = kmalloc_array(4, HV_HYP_PAGE_SIZE, GFP_KERNEL); + if (!srv->recv_buffer) + return -ENOMEM; + srv->channel = dev->channel; + if (srv->util_init) { + ret = srv->util_init(srv); + if (ret) + goto error1; + } + + /* + * The set of services managed by the util driver are not performance + * critical and do not need batched reading. Furthermore, some services + * such as KVP can only handle one message from the host at a time. + * Turn off batched reading for all util drivers before we open the + * channel. + */ + set_channel_read_mode(dev->channel, HV_CALL_DIRECT); + + hv_set_drvdata(dev, srv); + + ret = vmbus_open(dev->channel, HV_UTIL_RING_SEND_SIZE, + HV_UTIL_RING_RECV_SIZE, NULL, 0, srv->util_cb, + dev->channel); + if (ret) + goto error; + + if (srv->util_init_transport) { + ret = srv->util_init_transport(); + if (ret) { + vmbus_close(dev->channel); + goto error; + } + } + return 0; + +error: + if (srv->util_deinit) + srv->util_deinit(); +error1: + kfree(srv->recv_buffer); + return ret; +} + +static void util_remove(struct hv_device *dev) +{ + struct hv_util_service *srv = hv_get_drvdata(dev); + + if (srv->util_deinit) + srv->util_deinit(); + vmbus_close(dev->channel); + kfree(srv->recv_buffer); +} + +/* + * When we're in util_suspend(), all the userspace processes have been frozen + * (refer to hibernate() -> freeze_processes()). The userspace is thawed only + * after the whole resume procedure, including util_resume(), finishes. + */ +static int util_suspend(struct hv_device *dev) +{ + struct hv_util_service *srv = hv_get_drvdata(dev); + int ret = 0; + + if (srv->util_pre_suspend) { + ret = srv->util_pre_suspend(); + if (ret) + return ret; + } + + vmbus_close(dev->channel); + + return 0; +} + +static int util_resume(struct hv_device *dev) +{ + struct hv_util_service *srv = hv_get_drvdata(dev); + int ret = 0; + + if (srv->util_pre_resume) { + ret = srv->util_pre_resume(); + if (ret) + return ret; + } + + ret = vmbus_open(dev->channel, HV_UTIL_RING_SEND_SIZE, + HV_UTIL_RING_RECV_SIZE, NULL, 0, srv->util_cb, + dev->channel); + return ret; +} + +static const struct hv_vmbus_device_id id_table[] = { + /* Shutdown guid */ + { HV_SHUTDOWN_GUID, + .driver_data = (unsigned long)&util_shutdown + }, + /* Time synch guid */ + { HV_TS_GUID, + .driver_data = (unsigned long)&util_timesynch + }, + /* Heartbeat guid */ + { HV_HEART_BEAT_GUID, + .driver_data = (unsigned long)&util_heartbeat + }, + /* KVP guid */ + { HV_KVP_GUID, + .driver_data = (unsigned long)&util_kvp + }, + /* VSS GUID */ + { HV_VSS_GUID, + .driver_data = (unsigned long)&util_vss + }, + { }, +}; + +MODULE_DEVICE_TABLE(vmbus, id_table); + +/* The one and only one */ +static struct hv_driver util_drv = { + .name = "hv_utils", + .id_table = id_table, + .probe = util_probe, + .remove = util_remove, + .suspend = util_suspend, + .resume = util_resume, + .driver = { + .probe_type = PROBE_PREFER_ASYNCHRONOUS, + }, +}; + +static int hv_ptp_enable(struct ptp_clock_info *info, + struct ptp_clock_request *request, int on) +{ + return -EOPNOTSUPP; +} + +static int hv_ptp_settime(struct ptp_clock_info *p, const struct timespec64 *ts) +{ + return -EOPNOTSUPP; +} + +static int hv_ptp_adjfine(struct ptp_clock_info *ptp, long delta) +{ + return -EOPNOTSUPP; +} +static int hv_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + return -EOPNOTSUPP; +} + +static int hv_ptp_gettime(struct ptp_clock_info *info, struct timespec64 *ts) +{ + return hv_get_adj_host_time(ts); +} + +static struct ptp_clock_info ptp_hyperv_info = { + .name = "hyperv", + .enable = hv_ptp_enable, + .adjtime = hv_ptp_adjtime, + .adjfine = hv_ptp_adjfine, + .gettime64 = hv_ptp_gettime, + .settime64 = hv_ptp_settime, + .owner = THIS_MODULE, +}; + +static struct ptp_clock *hv_ptp_clock; + +static int hv_timesync_init(struct hv_util_service *srv) +{ + spin_lock_init(&host_ts.lock); + + INIT_WORK(&adj_time_work, hv_set_host_time); + + /* + * ptp_clock_register() returns NULL when CONFIG_PTP_1588_CLOCK is + * disabled but the driver is still useful without the PTP device + * as it still handles the ICTIMESYNCFLAG_SYNC case. + */ + hv_ptp_clock = ptp_clock_register(&ptp_hyperv_info, NULL); + if (IS_ERR_OR_NULL(hv_ptp_clock)) { + pr_err("cannot register PTP clock: %d\n", + PTR_ERR_OR_ZERO(hv_ptp_clock)); + hv_ptp_clock = NULL; + } + + return 0; +} + +static void hv_timesync_cancel_work(void) +{ + cancel_work_sync(&adj_time_work); +} + +static int hv_timesync_pre_suspend(void) +{ + hv_timesync_cancel_work(); + return 0; +} + +static void hv_timesync_deinit(void) +{ + if (hv_ptp_clock) + ptp_clock_unregister(hv_ptp_clock); + + hv_timesync_cancel_work(); +} + +static int __init init_hyperv_utils(void) +{ + pr_info("Registering HyperV Utility Driver\n"); + + return vmbus_driver_register(&util_drv); +} + +static void exit_hyperv_utils(void) +{ + pr_info("De-Registered HyperV Utility Driver\n"); + + vmbus_driver_unregister(&util_drv); +} + +module_init(init_hyperv_utils); +module_exit(exit_hyperv_utils); + +MODULE_DESCRIPTION("Hyper-V Utilities"); +MODULE_LICENSE("GPL"); diff --git a/drivers/hv/hv_utils_transport.c b/drivers/hv/hv_utils_transport.c new file mode 100644 index 000000000..c3be5c570 --- /dev/null +++ b/drivers/hv/hv_utils_transport.c @@ -0,0 +1,350 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Kernel/userspace transport abstraction for Hyper-V util driver. + * + * Copyright (C) 2015, Vitaly Kuznetsov <vkuznets@redhat.com> + */ + +#include <linux/slab.h> +#include <linux/fs.h> +#include <linux/poll.h> + +#include "hyperv_vmbus.h" +#include "hv_utils_transport.h" + +static DEFINE_SPINLOCK(hvt_list_lock); +static LIST_HEAD(hvt_list); + +static void hvt_reset(struct hvutil_transport *hvt) +{ + kfree(hvt->outmsg); + hvt->outmsg = NULL; + hvt->outmsg_len = 0; + if (hvt->on_reset) + hvt->on_reset(); +} + +static ssize_t hvt_op_read(struct file *file, char __user *buf, + size_t count, loff_t *ppos) +{ + struct hvutil_transport *hvt; + int ret; + + hvt = container_of(file->f_op, struct hvutil_transport, fops); + + if (wait_event_interruptible(hvt->outmsg_q, hvt->outmsg_len > 0 || + hvt->mode != HVUTIL_TRANSPORT_CHARDEV)) + return -EINTR; + + mutex_lock(&hvt->lock); + + if (hvt->mode == HVUTIL_TRANSPORT_DESTROY) { + ret = -EBADF; + goto out_unlock; + } + + if (!hvt->outmsg) { + ret = -EAGAIN; + goto out_unlock; + } + + if (count < hvt->outmsg_len) { + ret = -EINVAL; + goto out_unlock; + } + + if (!copy_to_user(buf, hvt->outmsg, hvt->outmsg_len)) + ret = hvt->outmsg_len; + else + ret = -EFAULT; + + kfree(hvt->outmsg); + hvt->outmsg = NULL; + hvt->outmsg_len = 0; + + if (hvt->on_read) + hvt->on_read(); + hvt->on_read = NULL; + +out_unlock: + mutex_unlock(&hvt->lock); + return ret; +} + +static ssize_t hvt_op_write(struct file *file, const char __user *buf, + size_t count, loff_t *ppos) +{ + struct hvutil_transport *hvt; + u8 *inmsg; + int ret; + + hvt = container_of(file->f_op, struct hvutil_transport, fops); + + inmsg = memdup_user(buf, count); + if (IS_ERR(inmsg)) + return PTR_ERR(inmsg); + + if (hvt->mode == HVUTIL_TRANSPORT_DESTROY) + ret = -EBADF; + else + ret = hvt->on_msg(inmsg, count); + + kfree(inmsg); + + return ret ? ret : count; +} + +static __poll_t hvt_op_poll(struct file *file, poll_table *wait) +{ + struct hvutil_transport *hvt; + + hvt = container_of(file->f_op, struct hvutil_transport, fops); + + poll_wait(file, &hvt->outmsg_q, wait); + + if (hvt->mode == HVUTIL_TRANSPORT_DESTROY) + return EPOLLERR | EPOLLHUP; + + if (hvt->outmsg_len > 0) + return EPOLLIN | EPOLLRDNORM; + + return 0; +} + +static int hvt_op_open(struct inode *inode, struct file *file) +{ + struct hvutil_transport *hvt; + int ret = 0; + bool issue_reset = false; + + hvt = container_of(file->f_op, struct hvutil_transport, fops); + + mutex_lock(&hvt->lock); + + if (hvt->mode == HVUTIL_TRANSPORT_DESTROY) { + ret = -EBADF; + } else if (hvt->mode == HVUTIL_TRANSPORT_INIT) { + /* + * Switching to CHARDEV mode. We switch bach to INIT when + * device gets released. + */ + hvt->mode = HVUTIL_TRANSPORT_CHARDEV; + } else if (hvt->mode == HVUTIL_TRANSPORT_NETLINK) { + /* + * We're switching from netlink communication to using char + * device. Issue the reset first. + */ + issue_reset = true; + hvt->mode = HVUTIL_TRANSPORT_CHARDEV; + } else { + ret = -EBUSY; + } + + if (issue_reset) + hvt_reset(hvt); + + mutex_unlock(&hvt->lock); + + return ret; +} + +static void hvt_transport_free(struct hvutil_transport *hvt) +{ + misc_deregister(&hvt->mdev); + kfree(hvt->outmsg); + kfree(hvt); +} + +static int hvt_op_release(struct inode *inode, struct file *file) +{ + struct hvutil_transport *hvt; + int mode_old; + + hvt = container_of(file->f_op, struct hvutil_transport, fops); + + mutex_lock(&hvt->lock); + mode_old = hvt->mode; + if (hvt->mode != HVUTIL_TRANSPORT_DESTROY) + hvt->mode = HVUTIL_TRANSPORT_INIT; + /* + * Cleanup message buffers to avoid spurious messages when the daemon + * connects back. + */ + hvt_reset(hvt); + + if (mode_old == HVUTIL_TRANSPORT_DESTROY) + complete(&hvt->release); + + mutex_unlock(&hvt->lock); + + return 0; +} + +static void hvt_cn_callback(struct cn_msg *msg, struct netlink_skb_parms *nsp) +{ + struct hvutil_transport *hvt, *hvt_found = NULL; + + spin_lock(&hvt_list_lock); + list_for_each_entry(hvt, &hvt_list, list) { + if (hvt->cn_id.idx == msg->id.idx && + hvt->cn_id.val == msg->id.val) { + hvt_found = hvt; + break; + } + } + spin_unlock(&hvt_list_lock); + if (!hvt_found) { + pr_warn("%s: spurious message received!\n", __func__); + return; + } + + /* + * Switching to NETLINK mode. Switching to CHARDEV happens when someone + * opens the device. + */ + mutex_lock(&hvt->lock); + if (hvt->mode == HVUTIL_TRANSPORT_INIT) + hvt->mode = HVUTIL_TRANSPORT_NETLINK; + + if (hvt->mode == HVUTIL_TRANSPORT_NETLINK) + hvt_found->on_msg(msg->data, msg->len); + else + pr_warn("%s: unexpected netlink message!\n", __func__); + mutex_unlock(&hvt->lock); +} + +int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len, + void (*on_read_cb)(void)) +{ + struct cn_msg *cn_msg; + int ret = 0; + + if (hvt->mode == HVUTIL_TRANSPORT_INIT || + hvt->mode == HVUTIL_TRANSPORT_DESTROY) { + return -EINVAL; + } else if (hvt->mode == HVUTIL_TRANSPORT_NETLINK) { + cn_msg = kzalloc(sizeof(*cn_msg) + len, GFP_ATOMIC); + if (!cn_msg) + return -ENOMEM; + cn_msg->id.idx = hvt->cn_id.idx; + cn_msg->id.val = hvt->cn_id.val; + cn_msg->len = len; + memcpy(cn_msg->data, msg, len); + ret = cn_netlink_send(cn_msg, 0, 0, GFP_ATOMIC); + kfree(cn_msg); + /* + * We don't know when netlink messages are delivered but unlike + * in CHARDEV mode we're not blocked and we can send next + * messages right away. + */ + if (on_read_cb) + on_read_cb(); + return ret; + } + /* HVUTIL_TRANSPORT_CHARDEV */ + mutex_lock(&hvt->lock); + if (hvt->mode != HVUTIL_TRANSPORT_CHARDEV) { + ret = -EINVAL; + goto out_unlock; + } + + if (hvt->outmsg) { + /* Previous message wasn't received */ + ret = -EFAULT; + goto out_unlock; + } + hvt->outmsg = kzalloc(len, GFP_KERNEL); + if (hvt->outmsg) { + memcpy(hvt->outmsg, msg, len); + hvt->outmsg_len = len; + hvt->on_read = on_read_cb; + wake_up_interruptible(&hvt->outmsg_q); + } else { + ret = -ENOMEM; + } +out_unlock: + mutex_unlock(&hvt->lock); + return ret; +} + +struct hvutil_transport *hvutil_transport_init(const char *name, + u32 cn_idx, u32 cn_val, + int (*on_msg)(void *, int), + void (*on_reset)(void)) +{ + struct hvutil_transport *hvt; + + hvt = kzalloc_obj(*hvt); + if (!hvt) + return NULL; + + hvt->cn_id.idx = cn_idx; + hvt->cn_id.val = cn_val; + + hvt->mdev.minor = MISC_DYNAMIC_MINOR; + hvt->mdev.name = name; + + hvt->fops.owner = THIS_MODULE; + hvt->fops.read = hvt_op_read; + hvt->fops.write = hvt_op_write; + hvt->fops.poll = hvt_op_poll; + hvt->fops.open = hvt_op_open; + hvt->fops.release = hvt_op_release; + + hvt->mdev.fops = &hvt->fops; + + init_waitqueue_head(&hvt->outmsg_q); + mutex_init(&hvt->lock); + init_completion(&hvt->release); + + spin_lock(&hvt_list_lock); + list_add(&hvt->list, &hvt_list); + spin_unlock(&hvt_list_lock); + + hvt->on_msg = on_msg; + hvt->on_reset = on_reset; + + if (misc_register(&hvt->mdev)) + goto err_free_hvt; + + /* Use cn_id.idx/cn_id.val to determine if we need to setup netlink */ + if (hvt->cn_id.idx > 0 && hvt->cn_id.val > 0 && + cn_add_callback(&hvt->cn_id, name, hvt_cn_callback)) + goto err_free_hvt; + + return hvt; + +err_free_hvt: + spin_lock(&hvt_list_lock); + list_del(&hvt->list); + spin_unlock(&hvt_list_lock); + kfree(hvt); + return NULL; +} + +void hvutil_transport_destroy(struct hvutil_transport *hvt) +{ + int mode_old; + + mutex_lock(&hvt->lock); + mode_old = hvt->mode; + hvt->mode = HVUTIL_TRANSPORT_DESTROY; + wake_up_interruptible(&hvt->outmsg_q); + mutex_unlock(&hvt->lock); + + /* + * In case we were in 'chardev' mode we still have an open fd so we + * have to defer freeing the device. Netlink interface can be freed + * now. + */ + spin_lock(&hvt_list_lock); + list_del(&hvt->list); + spin_unlock(&hvt_list_lock); + if (hvt->cn_id.idx > 0 && hvt->cn_id.val > 0) + cn_del_callback(&hvt->cn_id); + + if (mode_old == HVUTIL_TRANSPORT_CHARDEV) + wait_for_completion(&hvt->release); + + hvt_transport_free(hvt); +} diff --git a/drivers/hv/hv_utils_transport.h b/drivers/hv/hv_utils_transport.h new file mode 100644 index 000000000..1c162393c --- /dev/null +++ b/drivers/hv/hv_utils_transport.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Kernel/userspace transport abstraction for Hyper-V util driver. + * + * Copyright (C) 2015, Vitaly Kuznetsov <vkuznets@redhat.com> + */ + +#ifndef _HV_UTILS_TRANSPORT_H +#define _HV_UTILS_TRANSPORT_H + +#include <linux/connector.h> +#include <linux/miscdevice.h> + +enum hvutil_transport_mode { + HVUTIL_TRANSPORT_INIT = 0, + HVUTIL_TRANSPORT_NETLINK, + HVUTIL_TRANSPORT_CHARDEV, + HVUTIL_TRANSPORT_DESTROY, +}; + +struct hvutil_transport { + int mode; /* hvutil_transport_mode */ + struct file_operations fops; /* file operations */ + struct miscdevice mdev; /* misc device */ + struct cb_id cn_id; /* CN_*_IDX/CN_*_VAL */ + struct list_head list; /* hvt_list */ + int (*on_msg)(void *, int); /* callback on new user message */ + void (*on_reset)(void); /* callback when userspace drops */ + void (*on_read)(void); /* callback on message read */ + u8 *outmsg; /* message to the userspace */ + int outmsg_len; /* its length */ + wait_queue_head_t outmsg_q; /* poll/read wait queue */ + struct mutex lock; /* protects struct members */ + struct completion release; /* synchronize with fd release */ +}; + +struct hvutil_transport *hvutil_transport_init(const char *name, + u32 cn_idx, u32 cn_val, + int (*on_msg)(void *, int), + void (*on_reset)(void)); +int hvutil_transport_send(struct hvutil_transport *hvt, void *msg, int len, + void (*on_read_cb)(void)); +void hvutil_transport_destroy(struct hvutil_transport *hvt); + +#endif /* _HV_UTILS_TRANSPORT_H */ diff --git a/drivers/hv/hyperv_vmbus.h b/drivers/hv/hyperv_vmbus.h new file mode 100644 index 000000000..eb8bdd8bb --- /dev/null +++ b/drivers/hv/hyperv_vmbus.h @@ -0,0 +1,552 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * + * Copyright (c) 2011, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + * K. Y. Srinivasan <kys@microsoft.com> + */ + +#ifndef _HYPERV_VMBUS_H +#define _HYPERV_VMBUS_H + +#include <linux/list.h> +#include <linux/bitops.h> +#include <asm/sync_bitops.h> +#include <asm/mshyperv.h> +#include <linux/atomic.h> +#include <linux/hyperv.h> +#include <linux/interrupt.h> +#include <hyperv/hvhdk.h> + +#include "hv_trace.h" + +/* + * Timeout for services such as KVP and fcopy. + */ +#define HV_UTIL_TIMEOUT 30 + +/* + * Timeout for guest-host handshake for services. + */ +#define HV_UTIL_NEGO_TIMEOUT 55 + +void vmbus_isr(void); + +/* Definitions for the monitored notification facility */ +union hv_monitor_trigger_group { + u64 as_uint64; + struct { + u32 pending; + u32 armed; + }; +}; + +struct hv_monitor_parameter { + union hv_connection_id connectionid; + u16 flagnumber; + u16 rsvdz; +}; + +union hv_monitor_trigger_state { + u32 asu32; + + struct { + u32 group_enable:4; + u32 rsvdz:28; + }; +}; + +/* struct hv_monitor_page Layout */ +/* ------------------------------------------------------ */ +/* | 0 | TriggerState (4 bytes) | Rsvd1 (4 bytes) | */ +/* | 8 | TriggerGroup[0] | */ +/* | 10 | TriggerGroup[1] | */ +/* | 18 | TriggerGroup[2] | */ +/* | 20 | TriggerGroup[3] | */ +/* | 28 | Rsvd2[0] | */ +/* | 30 | Rsvd2[1] | */ +/* | 38 | Rsvd2[2] | */ +/* | 40 | NextCheckTime[0][0] | NextCheckTime[0][1] | */ +/* | ... | */ +/* | 240 | Latency[0][0..3] | */ +/* | 340 | Rsvz3[0] | */ +/* | 440 | Parameter[0][0] | */ +/* | 448 | Parameter[0][1] | */ +/* | ... | */ +/* | 840 | Rsvd4[0] | */ +/* ------------------------------------------------------ */ +struct hv_monitor_page { + union hv_monitor_trigger_state trigger_state; + u32 rsvdz1; + + union hv_monitor_trigger_group trigger_group[4]; + u64 rsvdz2[3]; + + s32 next_checktime[4][32]; + + u16 latency[4][32]; + u64 rsvdz3[32]; + + struct hv_monitor_parameter parameter[4][32]; + + u8 rsvdz4[1984]; +}; + +#define HV_HYPERCALL_PARAM_ALIGN sizeof(u64) + +/* Definition of the hv_post_message hypercall input structure. */ +struct hv_input_post_message { + union hv_connection_id connectionid; + u32 reserved; + u32 message_type; + u32 payload_size; + u64 payload[HV_MESSAGE_PAYLOAD_QWORD_COUNT]; +}; + + +enum { + VMBUS_MESSAGE_CONNECTION_ID = 1, + VMBUS_MESSAGE_CONNECTION_ID_4 = 4, + VMBUS_MESSAGE_PORT_ID = 1, + VMBUS_EVENT_CONNECTION_ID = 2, + VMBUS_EVENT_PORT_ID = 2, + VMBUS_MONITOR_CONNECTION_ID = 3, + VMBUS_MONITOR_PORT_ID = 3, + VMBUS_MESSAGE_SINT = 2, +}; + +/* + * Per cpu state for channel handling + */ +struct hv_per_cpu_context { + /* + * SynIC pages for communicating with the host. + * + * These pages are accessible to the host partition and the hypervisor. + * They may be used for exchanging data with the host partition and the + * hypervisor even when they aren't trusted yet the guest partition + * must be prepared to handle the malicious behavior. + */ + void *hyp_synic_message_page; + void *hyp_synic_event_page; + /* + * SynIC pages for communicating with the paravisor. + * + * These pages may be accessed from within the guest partition only in + * CoCo VMs. Neither the host partition nor the hypervisor can access + * these pages in that case; they are used for exchanging data with the + * paravisor. + */ + void *para_synic_message_page; + void *para_synic_event_page; + + /* + * The page is only used in hv_post_message() for a TDX VM (with the + * paravisor) to post a messages to Hyper-V: when such a VM calls + * HVCALL_POST_MESSAGE, it can't use the hyperv_pcpu_input_arg (which + * is encrypted in such a VM) as the hypercall input page, because + * the input page for HVCALL_POST_MESSAGE must be decrypted in such a + * VM, so post_msg_page (which is decrypted in hv_synic_alloc()) is + * introduced for this purpose. See hyperv_init() for more comments. + */ + void *post_msg_page; + + /* + * Starting with win8, we can take channel interrupts on any CPU; + * we will manage the tasklet that handles events messages on a per CPU + * basis. + */ + struct tasklet_struct msg_dpc; +}; + +struct hv_context { + /* We only support running on top of Hyper-V + * So at this point this really can only contain the Hyper-V ID + */ + u64 guestid; + + struct hv_per_cpu_context __percpu *cpu_context; + + /* + * To manage allocations in a NUMA node. + * Array indexed by numa node ID. + */ + struct cpumask *hv_numa_map; +}; + +extern struct hv_context hv_context; + +/* Hv Interface */ + +extern int hv_init(void); + +extern int hv_post_message(union hv_connection_id connection_id, + enum hv_message_type message_type, + void *payload, size_t payload_size); + +extern int hv_synic_alloc(void); + +extern void hv_synic_free(void); + +extern void hv_hyp_synic_enable_regs(unsigned int cpu); +extern int hv_synic_init(unsigned int cpu); + +extern void hv_hyp_synic_disable_regs(unsigned int cpu); +extern int hv_synic_cleanup(unsigned int cpu); + +/* Interface */ + +void hv_ringbuffer_pre_init(struct vmbus_channel *channel); + +int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, + struct page *pages, u32 pagecnt, u32 max_pkt_size, + bool confidential); + +void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info); + +int hv_ringbuffer_write(struct vmbus_channel *channel, + const struct kvec *kv_list, u32 kv_count, + u64 requestid, u64 *trans_id); + +int hv_ringbuffer_read(struct vmbus_channel *channel, + void *buffer, u32 buflen, u32 *buffer_actual_len, + u64 *requestid, bool raw); + +/* + * The Maximum number of channels (16384) is determined by the size of the + * interrupt page, which is HV_HYP_PAGE_SIZE. 1/2 of HV_HYP_PAGE_SIZE is to + * send endpoint interrupts, and the other is to receive endpoint interrupts. + */ +#define MAX_NUM_CHANNELS ((HV_HYP_PAGE_SIZE >> 1) << 3) + +/* The value here must be in multiple of 32 */ +#define MAX_NUM_CHANNELS_SUPPORTED 256 + +#define MAX_CHANNEL_RELIDS \ + max(MAX_NUM_CHANNELS_SUPPORTED, HV_EVENT_FLAGS_COUNT) + +enum vmbus_connect_state { + DISCONNECTED, + CONNECTING, + CONNECTED, + DISCONNECTING +}; + +#define MAX_SIZE_CHANNEL_MESSAGE HV_MESSAGE_PAYLOAD_BYTE_COUNT + +/* + * The CPU that Hyper-V will interrupt for VMBUS messages, such as + * CHANNELMSG_OFFERCHANNEL and CHANNELMSG_RESCIND_CHANNELOFFER. + */ +#define VMBUS_CONNECT_CPU 0 + +struct vmbus_connection { + u32 msg_conn_id; + + atomic_t offer_in_progress; + + enum vmbus_connect_state conn_state; + + atomic_t next_gpadl_handle; + + struct completion unload_event; + /* + * Represents channel interrupts. Each bit position represents a + * channel. When a channel sends an interrupt via VMBUS, it finds its + * bit in the sendInterruptPage, set it and calls Hv to generate a port + * event. The other end receives the port event and parse the + * recvInterruptPage to see which bit is set + */ + void *int_page; + void *send_int_page; + void *recv_int_page; + + /* + * 2 pages - 1st page for parent->child notification and 2nd + * is child->parent notification + */ + struct hv_monitor_page *monitor_pages[2]; + struct list_head chn_msg_list; + spinlock_t channelmsg_lock; + + /* List of channels */ + struct list_head chn_list; + struct mutex channel_mutex; + + /* Array of channel pointers, indexed by relid */ + struct vmbus_channel **channels; + u32 relid_hiwater; + + /* + * An offer message is handled first on the work_queue, and then + * is further handled on handle_primary_chan_wq or + * handle_sub_chan_wq. + */ + struct workqueue_struct *work_queue; + struct workqueue_struct *handle_primary_chan_wq; + struct workqueue_struct *handle_sub_chan_wq; + struct workqueue_struct *rescind_work_queue; + + /* + * On suspension of the vmbus, the accumulated offer messages + * must be dropped. + */ + bool ignore_any_offer_msg; + + /* + * The number of sub-channels and hv_sock channels that should be + * cleaned up upon suspend: sub-channels will be re-created upon + * resume, and hv_sock channels should not survive suspend. + */ + atomic_t nr_chan_close_on_suspend; + /* + * vmbus_bus_suspend() waits for "nr_chan_close_on_suspend" to + * drop to zero. + */ + struct completion ready_for_suspend_event; + + /* + * Completed once the host has offered all boot-time channels. + * Note that some channels may still be under process on a workqueue. + */ + struct completion all_offers_delivered_event; +}; + + +struct vmbus_msginfo { + /* Bookkeeping stuff */ + struct list_head msglist_entry; + + /* The message itself */ + unsigned char msg[]; +}; + + +extern struct vmbus_connection vmbus_connection; + +int vmbus_negotiate_version(struct vmbus_channel_msginfo *msginfo, u32 version); + +static inline void vmbus_send_interrupt(u32 relid) +{ + sync_set_bit(relid, vmbus_connection.send_int_page); +} + +enum vmbus_message_handler_type { + /* The related handler can sleep. */ + VMHT_BLOCKING = 0, + + /* The related handler must NOT sleep. */ + VMHT_NON_BLOCKING = 1, +}; + +struct vmbus_channel_message_table_entry { + enum vmbus_channel_message_type message_type; + enum vmbus_message_handler_type handler_type; + void (*message_handler)(struct vmbus_channel_message_header *msg); + u32 min_payload_len; +}; + +extern const struct vmbus_channel_message_table_entry + channel_message_table[CHANNELMSG_COUNT]; + + +/* General vmbus interface */ + +bool vmbus_is_confidential(void); + +#if IS_ENABLED(CONFIG_HYPERV_VMBUS) +/* Free the message slot and signal end-of-message if required */ +static inline void vmbus_signal_eom(struct hv_message *msg, u32 old_msg_type) +{ + /* + * On crash we're reading some other CPU's message page and we need + * to be careful: this other CPU may already had cleared the header + * and the host may already had delivered some other message there. + * In case we blindly write msg->header.message_type we're going + * to lose it. We can still lose a message of the same type but + * we count on the fact that there can only be one + * CHANNELMSG_UNLOAD_RESPONSE and we don't care about other messages + * on crash. + */ + if (!try_cmpxchg(&msg->header.message_type, + &old_msg_type, HVMSG_NONE)) + return; + + /* + * The cmpxchg() above does an implicit memory barrier to + * ensure the write to MessageType (ie set to + * HVMSG_NONE) happens before we read the + * MessagePending and EOMing. Otherwise, the EOMing + * will not deliver any more messages since there is + * no empty slot + */ + if (msg->header.message_flags.msg_pending) { + /* + * This will cause message queue rescan to + * possibly deliver another msg from the + * hypervisor + */ + if (vmbus_is_confidential()) + hv_para_set_synic_register(HV_MSR_EOM, 0); + else + hv_set_msr(HV_MSR_EOM, 0); + } +} + +extern int vmbus_interrupt; +extern int vmbus_irq; +#endif /* CONFIG_HYPERV_VMBUS */ + +struct hv_device *vmbus_device_create(const guid_t *type, + const guid_t *instance, + struct vmbus_channel *channel); + +int vmbus_device_register(struct hv_device *child_device_obj); +void vmbus_device_unregister(struct hv_device *device_obj); +int vmbus_add_channel_kobj(struct hv_device *device_obj, + struct vmbus_channel *channel); + +void vmbus_remove_channel_attr_group(struct vmbus_channel *channel); + +void vmbus_channel_map_relid(struct vmbus_channel *channel); +void vmbus_channel_unmap_relid(struct vmbus_channel *channel); + +struct vmbus_channel *relid2channel(u32 relid); + +void vmbus_free_channels(void); + +/* Connection interface */ + +int vmbus_connect(void); +void vmbus_disconnect(void); + +int vmbus_post_msg(void *buffer, size_t buflen, bool can_sleep); + +void vmbus_on_event(unsigned long data); +void vmbus_on_msg_dpc(unsigned long data); + +int hv_kvp_init(struct hv_util_service *srv); +int hv_kvp_init_transport(void); +void hv_kvp_deinit(void); +int hv_kvp_pre_suspend(void); +int hv_kvp_pre_resume(void); +void hv_kvp_onchannelcallback(void *context); + +int hv_vss_init(struct hv_util_service *srv); +int hv_vss_init_transport(void); +void hv_vss_deinit(void); +int hv_vss_pre_suspend(void); +int hv_vss_pre_resume(void); +void hv_vss_onchannelcallback(void *context); + +static inline void hv_poll_channel(struct vmbus_channel *channel, + void (*cb)(void *)) +{ + if (!channel) + return; + cb(channel); +} + +enum hvutil_device_state { + HVUTIL_DEVICE_INIT = 0, /* driver is loaded, waiting for userspace */ + HVUTIL_READY, /* userspace is registered */ + HVUTIL_HOSTMSG_RECEIVED, /* message from the host was received */ + HVUTIL_USERSPACE_REQ, /* request to userspace was sent */ + HVUTIL_USERSPACE_RECV, /* reply from userspace was received */ + HVUTIL_DEVICE_DYING, /* driver unload is in progress */ +}; + +enum delay { + INTERRUPT_DELAY = 0, + MESSAGE_DELAY = 1, +}; + +extern const struct vmbus_device vmbus_devs[]; + +static inline bool hv_is_perf_channel(struct vmbus_channel *channel) +{ + return vmbus_devs[channel->device_id].perf_device; +} + +static inline size_t hv_dev_ring_size(struct vmbus_channel *channel) +{ + return vmbus_devs[channel->device_id].pref_ring_size; +} + +static inline bool hv_is_allocated_cpu(unsigned int cpu) +{ + struct vmbus_channel *channel, *sc; + + lockdep_assert_held(&vmbus_connection.channel_mutex); + /* + * List additions/deletions as well as updates of the target CPUs are + * protected by channel_mutex. + */ + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (!hv_is_perf_channel(channel)) + continue; + if (channel->target_cpu == cpu) + return true; + list_for_each_entry(sc, &channel->sc_list, sc_list) { + if (sc->target_cpu == cpu) + return true; + } + } + return false; +} + +static inline void hv_set_allocated_cpu(unsigned int cpu) +{ + cpumask_set_cpu(cpu, &hv_context.hv_numa_map[cpu_to_node(cpu)]); +} + +static inline void hv_clear_allocated_cpu(unsigned int cpu) +{ + if (hv_is_allocated_cpu(cpu)) + return; + cpumask_clear_cpu(cpu, &hv_context.hv_numa_map[cpu_to_node(cpu)]); +} + +static inline void hv_update_allocated_cpus(unsigned int old_cpu, + unsigned int new_cpu) +{ + hv_set_allocated_cpu(new_cpu); + hv_clear_allocated_cpu(old_cpu); +} + +#ifdef CONFIG_HYPERV_TESTING + +int hv_debug_add_dev_dir(struct hv_device *dev); +void hv_debug_rm_dev_dir(struct hv_device *dev); +void hv_debug_rm_all_dir(void); +int hv_debug_init(void); +void hv_debug_delay_test(struct vmbus_channel *channel, enum delay delay_type); + +#else /* CONFIG_HYPERV_TESTING */ + +static inline void hv_debug_rm_dev_dir(struct hv_device *dev) {}; +static inline void hv_debug_rm_all_dir(void) {}; +static inline void hv_debug_delay_test(struct vmbus_channel *channel, + enum delay delay_type) {}; +static inline int hv_debug_init(void) +{ + return -1; +} + +static inline int hv_debug_add_dev_dir(struct hv_device *dev) +{ + return -1; +} + +#endif /* CONFIG_HYPERV_TESTING */ + +/* Create and remove sysfs entry for memory mapped ring buffers for a channel */ +int hv_create_ring_sysfs(struct vmbus_channel *channel, + int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel, + struct vm_area_desc *desc)); +int hv_remove_ring_sysfs(struct vmbus_channel *channel); + +#endif /* _HYPERV_VMBUS_H */ diff --git a/drivers/hv/mshv.h b/drivers/hv/mshv.h new file mode 100644 index 000000000..d4813df92 --- /dev/null +++ b/drivers/hv/mshv.h @@ -0,0 +1,28 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2023, Microsoft Corporation. + */ + +#ifndef _MSHV_H_ +#define _MSHV_H_ + +#include <linux/stddef.h> +#include <linux/string.h> +#include <hyperv/hvhdk.h> + +#define mshv_field_nonzero(STRUCT, MEMBER) \ + memchr_inv(&((STRUCT).MEMBER), \ + 0, sizeof_field(typeof(STRUCT), MEMBER)) + +int hv_call_get_vp_registers(u32 vp_index, u64 partition_id, u16 count, + union hv_input_vtl input_vtl, + struct hv_register_assoc *registers); + +int hv_call_set_vp_registers(u32 vp_index, u64 partition_id, u16 count, + union hv_input_vtl input_vtl, + struct hv_register_assoc *registers); + +int hv_call_get_partition_property(u64 partition_id, u64 property_code, + u64 *property_value); + +#endif /* _MSHV_H */ diff --git a/drivers/hv/mshv_common.c b/drivers/hv/mshv_common.c new file mode 100644 index 000000000..63f09bb51 --- /dev/null +++ b/drivers/hv/mshv_common.c @@ -0,0 +1,241 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Microsoft Corporation. + * + * This file contains functions that will be called from one or more modules. + * If any of these modules are configured to build, this file is built and just + * statically linked in. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <asm/mshyperv.h> +#include <linux/resume_user_mode.h> +#include <linux/export.h> +#include <linux/acpi.h> +#include <linux/notifier.h> +#include <linux/reboot.h> + +#include "mshv.h" + +#define HV_GET_REGISTER_BATCH_SIZE \ + (HV_HYP_PAGE_SIZE / sizeof(union hv_register_value)) +#define HV_SET_REGISTER_BATCH_SIZE \ + ((HV_HYP_PAGE_SIZE - sizeof(struct hv_input_set_vp_registers)) \ + / sizeof(struct hv_register_assoc)) + +int hv_call_get_vp_registers(u32 vp_index, u64 partition_id, u16 count, + union hv_input_vtl input_vtl, + struct hv_register_assoc *registers) +{ + struct hv_input_get_vp_registers *input_page; + union hv_register_value *output_page; + u16 completed = 0; + unsigned long remaining = count; + int rep_count, i; + u64 status = HV_STATUS_SUCCESS; + unsigned long flags; + + local_irq_save(flags); + + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + output_page = *this_cpu_ptr(hyperv_pcpu_output_arg); + + input_page->partition_id = partition_id; + input_page->vp_index = vp_index; + input_page->input_vtl.as_uint8 = input_vtl.as_uint8; + input_page->rsvd_z8 = 0; + input_page->rsvd_z16 = 0; + + while (remaining) { + rep_count = min(remaining, HV_GET_REGISTER_BATCH_SIZE); + for (i = 0; i < rep_count; ++i) + input_page->names[i] = registers[i].name; + + status = hv_do_rep_hypercall(HVCALL_GET_VP_REGISTERS, rep_count, + 0, input_page, output_page); + if (!hv_result_success(status)) + break; + + completed = hv_repcomp(status); + for (i = 0; i < completed; ++i) + registers[i].value = output_page[i]; + + registers += completed; + remaining -= completed; + } + local_irq_restore(flags); + + return hv_result_to_errno(status); +} +EXPORT_SYMBOL_GPL(hv_call_get_vp_registers); + +int hv_call_set_vp_registers(u32 vp_index, u64 partition_id, u16 count, + union hv_input_vtl input_vtl, + struct hv_register_assoc *registers) +{ + struct hv_input_set_vp_registers *input_page; + u16 completed = 0; + unsigned long remaining = count; + int rep_count; + u64 status = HV_STATUS_SUCCESS; + unsigned long flags; + + local_irq_save(flags); + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + + input_page->partition_id = partition_id; + input_page->vp_index = vp_index; + input_page->input_vtl.as_uint8 = input_vtl.as_uint8; + input_page->rsvd_z8 = 0; + input_page->rsvd_z16 = 0; + + while (remaining) { + rep_count = min(remaining, HV_SET_REGISTER_BATCH_SIZE); + memcpy(input_page->elements, registers, + sizeof(struct hv_register_assoc) * rep_count); + + status = hv_do_rep_hypercall(HVCALL_SET_VP_REGISTERS, rep_count, + 0, input_page, NULL); + if (!hv_result_success(status)) + break; + + completed = hv_repcomp(status); + registers += completed; + remaining -= completed; + } + + local_irq_restore(flags); + + return hv_result_to_errno(status); +} +EXPORT_SYMBOL_GPL(hv_call_set_vp_registers); + +int hv_call_get_partition_property(u64 partition_id, + u64 property_code, + u64 *property_value) +{ + u64 status; + unsigned long flags; + struct hv_input_get_partition_property *input; + struct hv_output_get_partition_property *output; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + memset(input, 0, sizeof(*input)); + input->partition_id = partition_id; + input->property_code = property_code; + status = hv_do_hypercall(HVCALL_GET_PARTITION_PROPERTY, input, output); + + if (!hv_result_success(status)) { + local_irq_restore(flags); + return hv_result_to_errno(status); + } + *property_value = output->property_value; + + local_irq_restore(flags); + + return 0; +} +EXPORT_SYMBOL_GPL(hv_call_get_partition_property); + +#ifdef CONFIG_X86 +/* + * Corresponding sleep states have to be initialized in order for a subsequent + * HVCALL_ENTER_SLEEP_STATE call to succeed. Currently only S5 state as per + * ACPI 6.4 chapter 7.4.2 is relevant, while S1, S2 and S3 can be supported. + * + * In order to pass proper PM values to mshv, ACPI should be initialized and + * should support S5 sleep state when this method is invoked. + */ +static int hv_initialize_sleep_states(void) +{ + u64 status; + unsigned long flags; + struct hv_input_set_system_property *in; + acpi_status acpi_status; + u8 sleep_type_a, sleep_type_b; + + if (!acpi_sleep_state_supported(ACPI_STATE_S5)) { + pr_err("%s: S5 sleep state not supported.\n", __func__); + return -ENODEV; + } + + acpi_status = acpi_get_sleep_type_data(ACPI_STATE_S5, &sleep_type_a, + &sleep_type_b); + if (ACPI_FAILURE(acpi_status)) + return -ENODEV; + + local_irq_save(flags); + in = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(in, 0, sizeof(*in)); + + in->property_id = HV_SYSTEM_PROPERTY_SLEEP_STATE; + in->set_sleep_state_info.sleep_state = HV_SLEEP_STATE_S5; + in->set_sleep_state_info.pm1a_slp_typ = sleep_type_a; + in->set_sleep_state_info.pm1b_slp_typ = sleep_type_b; + + status = hv_do_hypercall(HVCALL_SET_SYSTEM_PROPERTY, in, NULL); + local_irq_restore(flags); + + if (!hv_result_success(status)) { + hv_status_err(status, "\n"); + return hv_result_to_errno(status); + } + + return 0; +} + +/* + * This notifier initializes sleep states in mshv hypervisor which will be + * used during power off. + */ +static int hv_reboot_notifier_handler(struct notifier_block *this, + unsigned long code, void *another) +{ + int ret = 0; + + if (code == SYS_HALT || code == SYS_POWER_OFF) + ret = hv_initialize_sleep_states(); + + return ret ? NOTIFY_DONE : NOTIFY_OK; +} + +static struct notifier_block hv_reboot_notifier = { + .notifier_call = hv_reboot_notifier_handler, +}; + +void hv_sleep_notifiers_register(void) +{ + int ret; + + ret = register_reboot_notifier(&hv_reboot_notifier); + if (ret) + pr_err("%s: cannot register reboot notifier %d\n", __func__, + ret); +} + +/* + * Power off the machine by entering S5 sleep state via Hyper-V hypercall. + * This call does not return if successful. + */ +void hv_machine_power_off(void) +{ + unsigned long flags; + struct hv_input_enter_sleep_state *in; + + local_irq_save(flags); + in = *this_cpu_ptr(hyperv_pcpu_input_arg); + in->sleep_state = HV_SLEEP_STATE_S5; + + (void)hv_do_hypercall(HVCALL_ENTER_SLEEP_STATE, in, NULL); + local_irq_restore(flags); + + /* should never reach here */ + BUG(); + +} +#endif diff --git a/drivers/hv/mshv_debugfs.c b/drivers/hv/mshv_debugfs.c new file mode 100644 index 000000000..3c3e02237 --- /dev/null +++ b/drivers/hv/mshv_debugfs.c @@ -0,0 +1,729 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2026, Microsoft Corporation. + * + * The /sys/kernel/debug/mshv directory contents. + * Contains various statistics data, provided by the hypervisor. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/debugfs.h> +#include <linux/stringify.h> +#include <asm/mshyperv.h> +#include <linux/slab.h> + +#include "mshv.h" +#include "mshv_root.h" + +/* Ensure this file is not used elsewhere by accident */ +#define MSHV_DEBUGFS_C +#include "mshv_debugfs_counters.c" + +#define U32_BUF_SZ 11 +#define U64_BUF_SZ 21 +/* Only support SELF and PARENT areas */ +#define NUM_STATS_AREAS 2 +static_assert(HV_STATS_AREA_SELF == 0 && HV_STATS_AREA_PARENT == 1, + "SELF and PARENT areas must be usable as indices into an array of size NUM_STATS_AREAS"); +/* HV_HYPERVISOR_COUNTER */ +#define HV_HYPERVISOR_COUNTER_LOGICAL_PROCESSORS 1 + +static struct dentry *mshv_debugfs; +static struct dentry *mshv_debugfs_partition; +static struct dentry *mshv_debugfs_lp; +static struct dentry **parent_vp_stats; +static struct dentry *parent_partition_stats; + +static u64 mshv_lps_count; +static struct hv_stats_page **mshv_lps_stats; + +static int lp_stats_show(struct seq_file *m, void *v) +{ + const struct hv_stats_page *stats = m->private; + int idx; + + for (idx = 0; idx < ARRAY_SIZE(hv_lp_counters); idx++) { + char *name = hv_lp_counters[idx]; + + if (!name) + continue; + seq_printf(m, "%-32s: %llu\n", name, stats->data[idx]); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(lp_stats); + +static void mshv_lp_stats_unmap(u32 lp_index) +{ + union hv_stats_object_identity identity = { + .lp.lp_index = lp_index, + .lp.stats_area_type = HV_STATS_AREA_SELF, + }; + int err; + + err = hv_unmap_stats_page(HV_STATS_OBJECT_LOGICAL_PROCESSOR, + mshv_lps_stats[lp_index], &identity); + if (err) + pr_err("%s: failed to unmap logical processor %u stats, err: %d\n", + __func__, lp_index, err); + + mshv_lps_stats[lp_index] = NULL; +} + +static struct hv_stats_page * __init mshv_lp_stats_map(u32 lp_index) +{ + union hv_stats_object_identity identity = { + .lp.lp_index = lp_index, + .lp.stats_area_type = HV_STATS_AREA_SELF, + }; + struct hv_stats_page *stats; + int err; + + err = hv_map_stats_page(HV_STATS_OBJECT_LOGICAL_PROCESSOR, &identity, + &stats); + if (err) { + pr_err("%s: failed to map logical processor %u stats, err: %d\n", + __func__, lp_index, err); + return ERR_PTR(err); + } + mshv_lps_stats[lp_index] = stats; + + return stats; +} + +static struct hv_stats_page * __init lp_debugfs_stats_create(u32 lp_index, + struct dentry *parent) +{ + struct dentry *dentry; + struct hv_stats_page *stats; + + stats = mshv_lp_stats_map(lp_index); + if (IS_ERR(stats)) + return stats; + + dentry = debugfs_create_file("stats", 0400, parent, + stats, &lp_stats_fops); + if (IS_ERR(dentry)) { + mshv_lp_stats_unmap(lp_index); + return ERR_CAST(dentry); + } + return stats; +} + +static int __init lp_debugfs_create(u32 lp_index, struct dentry *parent) +{ + struct dentry *idx; + char lp_idx_str[U32_BUF_SZ]; + struct hv_stats_page *stats; + int err; + + sprintf(lp_idx_str, "%u", lp_index); + + idx = debugfs_create_dir(lp_idx_str, parent); + if (IS_ERR(idx)) + return PTR_ERR(idx); + + stats = lp_debugfs_stats_create(lp_index, idx); + if (IS_ERR(stats)) { + err = PTR_ERR(stats); + goto remove_debugfs_lp_idx; + } + + return 0; + +remove_debugfs_lp_idx: + debugfs_remove_recursive(idx); + return err; +} + +static void mshv_debugfs_lp_remove(void) +{ + int lp_index; + + debugfs_remove_recursive(mshv_debugfs_lp); + + for (lp_index = 0; lp_index < mshv_lps_count; lp_index++) + mshv_lp_stats_unmap(lp_index); + + kfree(mshv_lps_stats); + mshv_lps_stats = NULL; +} + +static int __init mshv_debugfs_lp_create(struct dentry *parent) +{ + struct dentry *lp_dir; + int err, lp_index; + + mshv_lps_stats = kzalloc_objs(*mshv_lps_stats, mshv_lps_count, + GFP_KERNEL_ACCOUNT); + + if (!mshv_lps_stats) + return -ENOMEM; + + lp_dir = debugfs_create_dir("lp", parent); + if (IS_ERR(lp_dir)) { + err = PTR_ERR(lp_dir); + goto free_lp_stats; + } + + for (lp_index = 0; lp_index < mshv_lps_count; lp_index++) { + err = lp_debugfs_create(lp_index, lp_dir); + if (err) + goto remove_debugfs_lps; + } + + mshv_debugfs_lp = lp_dir; + + return 0; + +remove_debugfs_lps: + for (lp_index -= 1; lp_index >= 0; lp_index--) + mshv_lp_stats_unmap(lp_index); + debugfs_remove_recursive(lp_dir); +free_lp_stats: + kfree(mshv_lps_stats); + mshv_lps_stats = NULL; + + return err; +} + +static int vp_stats_show(struct seq_file *m, void *v) +{ + const struct hv_stats_page **pstats = m->private; + u64 parent_val, self_val; + int idx; + + /* + * For VP and partition stats, there may be two stats areas mapped, + * SELF and PARENT. These refer to the privilege level of the data in + * each page. Some fields may be 0 in SELF and nonzero in PARENT, or + * vice versa. + * + * Hence, prioritize printing from the PARENT page (more privileged + * data), but use the value from the SELF page if the PARENT value is + * 0. + */ + + for (idx = 0; idx < ARRAY_SIZE(hv_vp_counters); idx++) { + char *name = hv_vp_counters[idx]; + + if (!name) + continue; + + parent_val = pstats[HV_STATS_AREA_PARENT]->data[idx]; + self_val = pstats[HV_STATS_AREA_SELF]->data[idx]; + seq_printf(m, "%-43s: %llu\n", name, + parent_val ? parent_val : self_val); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(vp_stats); + +static void vp_debugfs_remove(struct dentry *vp_stats) +{ + debugfs_remove_recursive(vp_stats->d_parent); +} + +static int vp_debugfs_create(u64 partition_id, u32 vp_index, + struct hv_stats_page **pstats, + struct dentry **vp_stats_ptr, + struct dentry *parent) +{ + struct dentry *vp_idx_dir, *d; + char vp_idx_str[U32_BUF_SZ]; + int err; + + sprintf(vp_idx_str, "%u", vp_index); + + vp_idx_dir = debugfs_create_dir(vp_idx_str, parent); + if (IS_ERR(vp_idx_dir)) + return PTR_ERR(vp_idx_dir); + + d = debugfs_create_file("stats", 0400, vp_idx_dir, + pstats, &vp_stats_fops); + if (IS_ERR(d)) { + err = PTR_ERR(d); + goto remove_debugfs_vp_idx; + } + + *vp_stats_ptr = d; + + return 0; + +remove_debugfs_vp_idx: + debugfs_remove_recursive(vp_idx_dir); + return err; +} + +static int partition_stats_show(struct seq_file *m, void *v) +{ + const struct hv_stats_page **pstats = m->private; + u64 parent_val, self_val; + int idx; + + for (idx = 0; idx < ARRAY_SIZE(hv_partition_counters); idx++) { + char *name = hv_partition_counters[idx]; + + if (!name) + continue; + + parent_val = pstats[HV_STATS_AREA_PARENT]->data[idx]; + self_val = pstats[HV_STATS_AREA_SELF]->data[idx]; + seq_printf(m, "%-37s: %llu\n", name, + parent_val ? parent_val : self_val); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(partition_stats); + +static void mshv_partition_stats_unmap(u64 partition_id, + struct hv_stats_page *stats_page, + enum hv_stats_area_type stats_area_type) +{ + union hv_stats_object_identity identity = { + .partition.partition_id = partition_id, + .partition.stats_area_type = stats_area_type, + }; + int err; + + err = hv_unmap_stats_page(HV_STATS_OBJECT_PARTITION, stats_page, + &identity); + if (err) + pr_err("%s: failed to unmap partition %lld %s stats, err: %d\n", + __func__, partition_id, + (stats_area_type == HV_STATS_AREA_SELF) ? "self" : "parent", + err); +} + +static struct hv_stats_page *mshv_partition_stats_map(u64 partition_id, + enum hv_stats_area_type stats_area_type) +{ + union hv_stats_object_identity identity = { + .partition.partition_id = partition_id, + .partition.stats_area_type = stats_area_type, + }; + struct hv_stats_page *stats; + int err; + + err = hv_map_stats_page(HV_STATS_OBJECT_PARTITION, &identity, &stats); + if (err) { + pr_err("%s: failed to map partition %lld %s stats, err: %d\n", + __func__, partition_id, + (stats_area_type == HV_STATS_AREA_SELF) ? "self" : "parent", + err); + return ERR_PTR(err); + } + return stats; +} + +static int mshv_debugfs_partition_stats_create(u64 partition_id, + struct dentry **partition_stats_ptr, + struct dentry *parent) +{ + struct dentry *dentry; + struct hv_stats_page **pstats; + int err; + + pstats = kzalloc_objs(struct hv_stats_page *, NUM_STATS_AREAS, + GFP_KERNEL_ACCOUNT); + if (!pstats) + return -ENOMEM; + + pstats[HV_STATS_AREA_SELF] = mshv_partition_stats_map(partition_id, + HV_STATS_AREA_SELF); + if (IS_ERR(pstats[HV_STATS_AREA_SELF])) { + err = PTR_ERR(pstats[HV_STATS_AREA_SELF]); + goto cleanup; + } + + /* + * L1VH partition cannot access its partition stats in parent area. + */ + if (is_l1vh_parent(partition_id)) { + pstats[HV_STATS_AREA_PARENT] = pstats[HV_STATS_AREA_SELF]; + } else { + pstats[HV_STATS_AREA_PARENT] = mshv_partition_stats_map(partition_id, + HV_STATS_AREA_PARENT); + if (IS_ERR(pstats[HV_STATS_AREA_PARENT])) { + err = PTR_ERR(pstats[HV_STATS_AREA_PARENT]); + goto unmap_self; + } + if (!pstats[HV_STATS_AREA_PARENT]) + pstats[HV_STATS_AREA_PARENT] = pstats[HV_STATS_AREA_SELF]; + } + + dentry = debugfs_create_file("stats", 0400, parent, + pstats, &partition_stats_fops); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + goto unmap_partition_stats; + } + + *partition_stats_ptr = dentry; + return 0; + +unmap_partition_stats: + if (pstats[HV_STATS_AREA_PARENT] != pstats[HV_STATS_AREA_SELF]) + mshv_partition_stats_unmap(partition_id, pstats[HV_STATS_AREA_PARENT], + HV_STATS_AREA_PARENT); +unmap_self: + mshv_partition_stats_unmap(partition_id, pstats[HV_STATS_AREA_SELF], + HV_STATS_AREA_SELF); +cleanup: + kfree(pstats); + return err; +} + +static void partition_debugfs_remove(u64 partition_id, struct dentry *dentry) +{ + struct hv_stats_page **pstats = NULL; + + pstats = dentry->d_inode->i_private; + + debugfs_remove_recursive(dentry->d_parent); + + if (pstats[HV_STATS_AREA_PARENT] != pstats[HV_STATS_AREA_SELF]) { + mshv_partition_stats_unmap(partition_id, + pstats[HV_STATS_AREA_PARENT], + HV_STATS_AREA_PARENT); + } + + mshv_partition_stats_unmap(partition_id, + pstats[HV_STATS_AREA_SELF], + HV_STATS_AREA_SELF); + + kfree(pstats); +} + +static int partition_debugfs_create(u64 partition_id, + struct dentry **vp_dir_ptr, + struct dentry **partition_stats_ptr, + struct dentry *parent) +{ + char part_id_str[U64_BUF_SZ]; + struct dentry *part_id_dir, *vp_dir; + int err; + + if (is_l1vh_parent(partition_id)) + sprintf(part_id_str, "self"); + else + sprintf(part_id_str, "%llu", partition_id); + + part_id_dir = debugfs_create_dir(part_id_str, parent); + if (IS_ERR(part_id_dir)) + return PTR_ERR(part_id_dir); + + vp_dir = debugfs_create_dir("vp", part_id_dir); + if (IS_ERR(vp_dir)) { + err = PTR_ERR(vp_dir); + goto remove_debugfs_partition_id; + } + + err = mshv_debugfs_partition_stats_create(partition_id, + partition_stats_ptr, + part_id_dir); + if (err) + goto remove_debugfs_partition_id; + + *vp_dir_ptr = vp_dir; + + return 0; + +remove_debugfs_partition_id: + debugfs_remove_recursive(part_id_dir); + return err; +} + +static void parent_vp_debugfs_remove(u32 vp_index, + struct dentry *vp_stats_ptr) +{ + struct hv_stats_page **pstats; + + pstats = vp_stats_ptr->d_inode->i_private; + vp_debugfs_remove(vp_stats_ptr); + mshv_vp_stats_unmap(hv_current_partition_id, vp_index, pstats); + kfree(pstats); +} + +static void mshv_debugfs_parent_partition_remove(void) +{ + int idx; + + for_each_online_cpu(idx) + parent_vp_debugfs_remove(hv_vp_index[idx], + parent_vp_stats[idx]); + + partition_debugfs_remove(hv_current_partition_id, + parent_partition_stats); + kfree(parent_vp_stats); + parent_vp_stats = NULL; + parent_partition_stats = NULL; +} + +static int __init parent_vp_debugfs_create(u32 vp_index, + struct dentry **vp_stats_ptr, + struct dentry *parent) +{ + struct hv_stats_page **pstats; + int err; + + pstats = kzalloc_objs(struct hv_stats_page *, NUM_STATS_AREAS, + GFP_KERNEL_ACCOUNT); + if (!pstats) + return -ENOMEM; + + err = mshv_vp_stats_map(hv_current_partition_id, vp_index, pstats); + if (err) + goto cleanup; + + err = vp_debugfs_create(hv_current_partition_id, vp_index, pstats, + vp_stats_ptr, parent); + if (err) + goto unmap_vp_stats; + + return 0; + +unmap_vp_stats: + mshv_vp_stats_unmap(hv_current_partition_id, vp_index, pstats); +cleanup: + kfree(pstats); + return err; +} + +static int __init mshv_debugfs_parent_partition_create(void) +{ + struct dentry *vp_dir; + int err, idx, i; + + mshv_debugfs_partition = debugfs_create_dir("partition", + mshv_debugfs); + if (IS_ERR(mshv_debugfs_partition)) + return PTR_ERR(mshv_debugfs_partition); + + err = partition_debugfs_create(hv_current_partition_id, + &vp_dir, + &parent_partition_stats, + mshv_debugfs_partition); + if (err) + goto remove_debugfs_partition; + + parent_vp_stats = kzalloc_objs(*parent_vp_stats, nr_cpu_ids); + if (!parent_vp_stats) { + err = -ENOMEM; + goto remove_debugfs_partition; + } + + for_each_online_cpu(idx) { + err = parent_vp_debugfs_create(hv_vp_index[idx], + &parent_vp_stats[idx], + vp_dir); + if (err) + goto remove_debugfs_partition_vp; + } + + return 0; + +remove_debugfs_partition_vp: + for_each_online_cpu(i) { + if (i >= idx) + break; + parent_vp_debugfs_remove(i, parent_vp_stats[i]); + } + partition_debugfs_remove(hv_current_partition_id, + parent_partition_stats); + + kfree(parent_vp_stats); + parent_vp_stats = NULL; + parent_partition_stats = NULL; + +remove_debugfs_partition: + debugfs_remove_recursive(mshv_debugfs_partition); + mshv_debugfs_partition = NULL; + return err; +} + +static int hv_stats_show(struct seq_file *m, void *v) +{ + const struct hv_stats_page *stats = m->private; + int idx; + + for (idx = 0; idx < ARRAY_SIZE(hv_hypervisor_counters); idx++) { + char *name = hv_hypervisor_counters[idx]; + + if (!name) + continue; + seq_printf(m, "%-27s: %llu\n", name, stats->data[idx]); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(hv_stats); + +static void mshv_hv_stats_unmap(void) +{ + union hv_stats_object_identity identity = { + .hv.stats_area_type = HV_STATS_AREA_SELF, + }; + int err; + + err = hv_unmap_stats_page(HV_STATS_OBJECT_HYPERVISOR, NULL, &identity); + if (err) + pr_err("%s: failed to unmap hypervisor stats: %d\n", + __func__, err); +} + +static void * __init mshv_hv_stats_map(void) +{ + union hv_stats_object_identity identity = { + .hv.stats_area_type = HV_STATS_AREA_SELF, + }; + struct hv_stats_page *stats; + int err; + + err = hv_map_stats_page(HV_STATS_OBJECT_HYPERVISOR, &identity, &stats); + if (err) { + pr_err("%s: failed to map hypervisor stats: %d\n", + __func__, err); + return ERR_PTR(err); + } + return stats; +} + +static int __init mshv_debugfs_hv_stats_create(struct dentry *parent) +{ + struct dentry *dentry; + u64 *stats; + int err; + + stats = mshv_hv_stats_map(); + if (IS_ERR(stats)) + return PTR_ERR(stats); + + dentry = debugfs_create_file("stats", 0400, parent, + stats, &hv_stats_fops); + if (IS_ERR(dentry)) { + err = PTR_ERR(dentry); + pr_err("%s: failed to create hypervisor stats dentry: %d\n", + __func__, err); + goto unmap_hv_stats; + } + + mshv_lps_count = stats[HV_HYPERVISOR_COUNTER_LOGICAL_PROCESSORS]; + + return 0; + +unmap_hv_stats: + mshv_hv_stats_unmap(); + return err; +} + +int mshv_debugfs_vp_create(struct mshv_vp *vp) +{ + struct mshv_partition *p = vp->vp_partition; + + if (!mshv_debugfs) + return 0; + + return vp_debugfs_create(p->pt_id, vp->vp_index, + vp->vp_stats_pages, + &vp->vp_stats_dentry, + p->pt_vp_dentry); +} + +void mshv_debugfs_vp_remove(struct mshv_vp *vp) +{ + if (!mshv_debugfs) + return; + + vp_debugfs_remove(vp->vp_stats_dentry); +} + +int mshv_debugfs_partition_create(struct mshv_partition *partition) +{ + int err; + + if (!mshv_debugfs) + return 0; + + err = partition_debugfs_create(partition->pt_id, + &partition->pt_vp_dentry, + &partition->pt_stats_dentry, + mshv_debugfs_partition); + if (err) + return err; + + return 0; +} + +void mshv_debugfs_partition_remove(struct mshv_partition *partition) +{ + if (!mshv_debugfs) + return; + + partition_debugfs_remove(partition->pt_id, + partition->pt_stats_dentry); +} + +int __init mshv_debugfs_init(void) +{ + int err; + + mshv_debugfs = debugfs_create_dir("mshv", NULL); + if (IS_ERR(mshv_debugfs)) { + err = PTR_ERR(mshv_debugfs); + mshv_debugfs = NULL; + pr_err("%s: failed to create debugfs directory\n", __func__); + return err; + } + + if (hv_root_partition()) { + err = mshv_debugfs_hv_stats_create(mshv_debugfs); + if (err) + goto remove_mshv_dir; + + err = mshv_debugfs_lp_create(mshv_debugfs); + if (err) + goto unmap_hv_stats; + } + + err = mshv_debugfs_parent_partition_create(); + if (err) + goto unmap_lp_stats; + + return 0; + +unmap_lp_stats: + if (hv_root_partition()) { + mshv_debugfs_lp_remove(); + mshv_debugfs_lp = NULL; + } +unmap_hv_stats: + if (hv_root_partition()) + mshv_hv_stats_unmap(); +remove_mshv_dir: + debugfs_remove_recursive(mshv_debugfs); + mshv_debugfs = NULL; + return err; +} + +void mshv_debugfs_exit(void) +{ + if (!mshv_debugfs) + return; + + mshv_debugfs_parent_partition_remove(); + + if (hv_root_partition()) { + mshv_debugfs_lp_remove(); + mshv_debugfs_lp = NULL; + mshv_hv_stats_unmap(); + } + + debugfs_remove_recursive(mshv_debugfs); + mshv_debugfs = NULL; + mshv_debugfs_partition = NULL; +} diff --git a/drivers/hv/mshv_debugfs_counters.c b/drivers/hv/mshv_debugfs_counters.c new file mode 100644 index 000000000..978536ba6 --- /dev/null +++ b/drivers/hv/mshv_debugfs_counters.c @@ -0,0 +1,490 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2026, Microsoft Corporation. + * + * Data for printing stats page counters via debugfs. + * + * Authors: Microsoft Linux virtualization team + */ + +/* + * For simplicity, this file is included directly in mshv_debugfs.c. + * If these are ever needed elsewhere they should be compiled separately. + * Ensure this file is not used twice by accident. + */ +#ifndef MSHV_DEBUGFS_C +#error "This file should only be included in mshv_debugfs.c" +#endif + +/* HV_HYPERVISOR_COUNTER */ +static char *hv_hypervisor_counters[] = { + [1] = "HvLogicalProcessors", + [2] = "HvPartitions", + [3] = "HvTotalPages", + [4] = "HvVirtualProcessors", + [5] = "HvMonitoredNotifications", + [6] = "HvModernStandbyEntries", + [7] = "HvPlatformIdleTransitions", + [8] = "HvHypervisorStartupCost", + + [10] = "HvIOSpacePages", + [11] = "HvNonEssentialPagesForDump", + [12] = "HvSubsumedPages", +}; + +/* HV_CPU_COUNTER */ +static char *hv_lp_counters[] = { + [1] = "LpGlobalTime", + [2] = "LpTotalRunTime", + [3] = "LpHypervisorRunTime", + [4] = "LpHardwareInterrupts", + [5] = "LpContextSwitches", + [6] = "LpInterProcessorInterrupts", + [7] = "LpSchedulerInterrupts", + [8] = "LpTimerInterrupts", + [9] = "LpInterProcessorInterruptsSent", + [10] = "LpProcessorHalts", + [11] = "LpMonitorTransitionCost", + [12] = "LpContextSwitchTime", + [13] = "LpC1TransitionsCount", + [14] = "LpC1RunTime", + [15] = "LpC2TransitionsCount", + [16] = "LpC2RunTime", + [17] = "LpC3TransitionsCount", + [18] = "LpC3RunTime", + [19] = "LpRootVpIndex", + [20] = "LpIdleSequenceNumber", + [21] = "LpGlobalTscCount", + [22] = "LpActiveTscCount", + [23] = "LpIdleAccumulation", + [24] = "LpReferenceCycleCount0", + [25] = "LpActualCycleCount0", + [26] = "LpReferenceCycleCount1", + [27] = "LpActualCycleCount1", + [28] = "LpProximityDomainId", + [29] = "LpPostedInterruptNotifications", + [30] = "LpBranchPredictorFlushes", +#if IS_ENABLED(CONFIG_X86_64) + [31] = "LpL1DataCacheFlushes", + [32] = "LpImmediateL1DataCacheFlushes", + [33] = "LpMbFlushes", + [34] = "LpCounterRefreshSequenceNumber", + [35] = "LpCounterRefreshReferenceTime", + [36] = "LpIdleAccumulationSnapshot", + [37] = "LpActiveTscCountSnapshot", + [38] = "LpHwpRequestContextSwitches", + [39] = "LpPlaceholder1", + [40] = "LpPlaceholder2", + [41] = "LpPlaceholder3", + [42] = "LpPlaceholder4", + [43] = "LpPlaceholder5", + [44] = "LpPlaceholder6", + [45] = "LpPlaceholder7", + [46] = "LpPlaceholder8", + [47] = "LpPlaceholder9", + [48] = "LpSchLocalRunListSize", + [49] = "LpReserveGroupId", + [50] = "LpRunningPriority", + [51] = "LpPerfmonInterruptCount", +#elif IS_ENABLED(CONFIG_ARM64) + [31] = "LpCounterRefreshSequenceNumber", + [32] = "LpCounterRefreshReferenceTime", + [33] = "LpIdleAccumulationSnapshot", + [34] = "LpActiveTscCountSnapshot", + [35] = "LpHwpRequestContextSwitches", + [36] = "LpPlaceholder2", + [37] = "LpPlaceholder3", + [38] = "LpPlaceholder4", + [39] = "LpPlaceholder5", + [40] = "LpPlaceholder6", + [41] = "LpPlaceholder7", + [42] = "LpPlaceholder8", + [43] = "LpPlaceholder9", + [44] = "LpSchLocalRunListSize", + [45] = "LpReserveGroupId", + [46] = "LpRunningPriority", +#endif +}; + +/* HV_PROCESS_COUNTER */ +static char *hv_partition_counters[] = { + [1] = "PtVirtualProcessors", + + [3] = "PtTlbSize", + [4] = "PtAddressSpaces", + [5] = "PtDepositedPages", + [6] = "PtGpaPages", + [7] = "PtGpaSpaceModifications", + [8] = "PtVirtualTlbFlushEntires", + [9] = "PtRecommendedTlbSize", + [10] = "PtGpaPages4K", + [11] = "PtGpaPages2M", + [12] = "PtGpaPages1G", + [13] = "PtGpaPages512G", + [14] = "PtDevicePages4K", + [15] = "PtDevicePages2M", + [16] = "PtDevicePages1G", + [17] = "PtDevicePages512G", + [18] = "PtAttachedDevices", + [19] = "PtDeviceInterruptMappings", + [20] = "PtIoTlbFlushes", + [21] = "PtIoTlbFlushCost", + [22] = "PtDeviceInterruptErrors", + [23] = "PtDeviceDmaErrors", + [24] = "PtDeviceInterruptThrottleEvents", + [25] = "PtSkippedTimerTicks", + [26] = "PtPartitionId", +#if IS_ENABLED(CONFIG_X86_64) + [27] = "PtNestedTlbSize", + [28] = "PtRecommendedNestedTlbSize", + [29] = "PtNestedTlbFreeListSize", + [30] = "PtNestedTlbTrimmedPages", + [31] = "PtPagesShattered", + [32] = "PtPagesRecombined", + [33] = "PtHwpRequestValue", + [34] = "PtAutoSuspendEnableTime", + [35] = "PtAutoSuspendTriggerTime", + [36] = "PtAutoSuspendDisableTime", + [37] = "PtPlaceholder1", + [38] = "PtPlaceholder2", + [39] = "PtPlaceholder3", + [40] = "PtPlaceholder4", + [41] = "PtPlaceholder5", + [42] = "PtPlaceholder6", + [43] = "PtPlaceholder7", + [44] = "PtPlaceholder8", + [45] = "PtHypervisorStateTransferGeneration", + [46] = "PtNumberofActiveChildPartitions", +#elif IS_ENABLED(CONFIG_ARM64) + [27] = "PtHwpRequestValue", + [28] = "PtAutoSuspendEnableTime", + [29] = "PtAutoSuspendTriggerTime", + [30] = "PtAutoSuspendDisableTime", + [31] = "PtPlaceholder1", + [32] = "PtPlaceholder2", + [33] = "PtPlaceholder3", + [34] = "PtPlaceholder4", + [35] = "PtPlaceholder5", + [36] = "PtPlaceholder6", + [37] = "PtPlaceholder7", + [38] = "PtPlaceholder8", + [39] = "PtHypervisorStateTransferGeneration", + [40] = "PtNumberofActiveChildPartitions", +#endif +}; + +/* HV_THREAD_COUNTER */ +static char *hv_vp_counters[] = { + [1] = "VpTotalRunTime", + [2] = "VpHypervisorRunTime", + [3] = "VpRemoteNodeRunTime", + [4] = "VpNormalizedRunTime", + [5] = "VpIdealCpu", + + [7] = "VpHypercallsCount", + [8] = "VpHypercallsTime", +#if IS_ENABLED(CONFIG_X86_64) + [9] = "VpPageInvalidationsCount", + [10] = "VpPageInvalidationsTime", + [11] = "VpControlRegisterAccessesCount", + [12] = "VpControlRegisterAccessesTime", + [13] = "VpIoInstructionsCount", + [14] = "VpIoInstructionsTime", + [15] = "VpHltInstructionsCount", + [16] = "VpHltInstructionsTime", + [17] = "VpMwaitInstructionsCount", + [18] = "VpMwaitInstructionsTime", + [19] = "VpCpuidInstructionsCount", + [20] = "VpCpuidInstructionsTime", + [21] = "VpMsrAccessesCount", + [22] = "VpMsrAccessesTime", + [23] = "VpOtherInterceptsCount", + [24] = "VpOtherInterceptsTime", + [25] = "VpExternalInterruptsCount", + [26] = "VpExternalInterruptsTime", + [27] = "VpPendingInterruptsCount", + [28] = "VpPendingInterruptsTime", + [29] = "VpEmulatedInstructionsCount", + [30] = "VpEmulatedInstructionsTime", + [31] = "VpDebugRegisterAccessesCount", + [32] = "VpDebugRegisterAccessesTime", + [33] = "VpPageFaultInterceptsCount", + [34] = "VpPageFaultInterceptsTime", + [35] = "VpGuestPageTableMaps", + [36] = "VpLargePageTlbFills", + [37] = "VpSmallPageTlbFills", + [38] = "VpReflectedGuestPageFaults", + [39] = "VpApicMmioAccesses", + [40] = "VpIoInterceptMessages", + [41] = "VpMemoryInterceptMessages", + [42] = "VpApicEoiAccesses", + [43] = "VpOtherMessages", + [44] = "VpPageTableAllocations", + [45] = "VpLogicalProcessorMigrations", + [46] = "VpAddressSpaceEvictions", + [47] = "VpAddressSpaceSwitches", + [48] = "VpAddressDomainFlushes", + [49] = "VpAddressSpaceFlushes", + [50] = "VpGlobalGvaRangeFlushes", + [51] = "VpLocalGvaRangeFlushes", + [52] = "VpPageTableEvictions", + [53] = "VpPageTableReclamations", + [54] = "VpPageTableResets", + [55] = "VpPageTableValidations", + [56] = "VpApicTprAccesses", + [57] = "VpPageTableWriteIntercepts", + [58] = "VpSyntheticInterrupts", + [59] = "VpVirtualInterrupts", + [60] = "VpApicIpisSent", + [61] = "VpApicSelfIpisSent", + [62] = "VpGpaSpaceHypercalls", + [63] = "VpLogicalProcessorHypercalls", + [64] = "VpLongSpinWaitHypercalls", + [65] = "VpOtherHypercalls", + [66] = "VpSyntheticInterruptHypercalls", + [67] = "VpVirtualInterruptHypercalls", + [68] = "VpVirtualMmuHypercalls", + [69] = "VpVirtualProcessorHypercalls", + [70] = "VpHardwareInterrupts", + [71] = "VpNestedPageFaultInterceptsCount", + [72] = "VpNestedPageFaultInterceptsTime", + [73] = "VpPageScans", + [74] = "VpLogicalProcessorDispatches", + [75] = "VpWaitingForCpuTime", + [76] = "VpExtendedHypercalls", + [77] = "VpExtendedHypercallInterceptMessages", + [78] = "VpMbecNestedPageTableSwitches", + [79] = "VpOtherReflectedGuestExceptions", + [80] = "VpGlobalIoTlbFlushes", + [81] = "VpGlobalIoTlbFlushCost", + [82] = "VpLocalIoTlbFlushes", + [83] = "VpLocalIoTlbFlushCost", + [84] = "VpHypercallsForwardedCount", + [85] = "VpHypercallsForwardingTime", + [86] = "VpPageInvalidationsForwardedCount", + [87] = "VpPageInvalidationsForwardingTime", + [88] = "VpControlRegisterAccessesForwardedCount", + [89] = "VpControlRegisterAccessesForwardingTime", + [90] = "VpIoInstructionsForwardedCount", + [91] = "VpIoInstructionsForwardingTime", + [92] = "VpHltInstructionsForwardedCount", + [93] = "VpHltInstructionsForwardingTime", + [94] = "VpMwaitInstructionsForwardedCount", + [95] = "VpMwaitInstructionsForwardingTime", + [96] = "VpCpuidInstructionsForwardedCount", + [97] = "VpCpuidInstructionsForwardingTime", + [98] = "VpMsrAccessesForwardedCount", + [99] = "VpMsrAccessesForwardingTime", + [100] = "VpOtherInterceptsForwardedCount", + [101] = "VpOtherInterceptsForwardingTime", + [102] = "VpExternalInterruptsForwardedCount", + [103] = "VpExternalInterruptsForwardingTime", + [104] = "VpPendingInterruptsForwardedCount", + [105] = "VpPendingInterruptsForwardingTime", + [106] = "VpEmulatedInstructionsForwardedCount", + [107] = "VpEmulatedInstructionsForwardingTime", + [108] = "VpDebugRegisterAccessesForwardedCount", + [109] = "VpDebugRegisterAccessesForwardingTime", + [110] = "VpPageFaultInterceptsForwardedCount", + [111] = "VpPageFaultInterceptsForwardingTime", + [112] = "VpVmclearEmulationCount", + [113] = "VpVmclearEmulationTime", + [114] = "VpVmptrldEmulationCount", + [115] = "VpVmptrldEmulationTime", + [116] = "VpVmptrstEmulationCount", + [117] = "VpVmptrstEmulationTime", + [118] = "VpVmreadEmulationCount", + [119] = "VpVmreadEmulationTime", + [120] = "VpVmwriteEmulationCount", + [121] = "VpVmwriteEmulationTime", + [122] = "VpVmxoffEmulationCount", + [123] = "VpVmxoffEmulationTime", + [124] = "VpVmxonEmulationCount", + [125] = "VpVmxonEmulationTime", + [126] = "VpNestedVMEntriesCount", + [127] = "VpNestedVMEntriesTime", + [128] = "VpNestedSLATSoftPageFaultsCount", + [129] = "VpNestedSLATSoftPageFaultsTime", + [130] = "VpNestedSLATHardPageFaultsCount", + [131] = "VpNestedSLATHardPageFaultsTime", + [132] = "VpInvEptAllContextEmulationCount", + [133] = "VpInvEptAllContextEmulationTime", + [134] = "VpInvEptSingleContextEmulationCount", + [135] = "VpInvEptSingleContextEmulationTime", + [136] = "VpInvVpidAllContextEmulationCount", + [137] = "VpInvVpidAllContextEmulationTime", + [138] = "VpInvVpidSingleContextEmulationCount", + [139] = "VpInvVpidSingleContextEmulationTime", + [140] = "VpInvVpidSingleAddressEmulationCount", + [141] = "VpInvVpidSingleAddressEmulationTime", + [142] = "VpNestedTlbPageTableReclamations", + [143] = "VpNestedTlbPageTableEvictions", + [144] = "VpFlushGuestPhysicalAddressSpaceHypercalls", + [145] = "VpFlushGuestPhysicalAddressListHypercalls", + [146] = "VpPostedInterruptNotifications", + [147] = "VpPostedInterruptScans", + [148] = "VpTotalCoreRunTime", + [149] = "VpMaximumRunTime", + [150] = "VpHwpRequestContextSwitches", + [151] = "VpWaitingForCpuTimeBucket0", + [152] = "VpWaitingForCpuTimeBucket1", + [153] = "VpWaitingForCpuTimeBucket2", + [154] = "VpWaitingForCpuTimeBucket3", + [155] = "VpWaitingForCpuTimeBucket4", + [156] = "VpWaitingForCpuTimeBucket5", + [157] = "VpWaitingForCpuTimeBucket6", + [158] = "VpVmloadEmulationCount", + [159] = "VpVmloadEmulationTime", + [160] = "VpVmsaveEmulationCount", + [161] = "VpVmsaveEmulationTime", + [162] = "VpGifInstructionEmulationCount", + [163] = "VpGifInstructionEmulationTime", + [164] = "VpEmulatedErrataSvmInstructions", + [165] = "VpPlaceholder1", + [166] = "VpPlaceholder2", + [167] = "VpPlaceholder3", + [168] = "VpPlaceholder4", + [169] = "VpPlaceholder5", + [170] = "VpPlaceholder6", + [171] = "VpPlaceholder7", + [172] = "VpPlaceholder8", + [173] = "VpContentionTime", + [174] = "VpWakeUpTime", + [175] = "VpSchedulingPriority", + [176] = "VpRdpmcInstructionsCount", + [177] = "VpRdpmcInstructionsTime", + [178] = "VpPerfmonPmuMsrAccessesCount", + [179] = "VpPerfmonLbrMsrAccessesCount", + [180] = "VpPerfmonIptMsrAccessesCount", + [181] = "VpPerfmonInterruptCount", + [182] = "VpVtl1DispatchCount", + [183] = "VpVtl2DispatchCount", + [184] = "VpVtl2DispatchBucket0", + [185] = "VpVtl2DispatchBucket1", + [186] = "VpVtl2DispatchBucket2", + [187] = "VpVtl2DispatchBucket3", + [188] = "VpVtl2DispatchBucket4", + [189] = "VpVtl2DispatchBucket5", + [190] = "VpVtl2DispatchBucket6", + [191] = "VpVtl1RunTime", + [192] = "VpVtl2RunTime", + [193] = "VpIommuHypercalls", + [194] = "VpCpuGroupHypercalls", + [195] = "VpVsmHypercalls", + [196] = "VpEventLogHypercalls", + [197] = "VpDeviceDomainHypercalls", + [198] = "VpDepositHypercalls", + [199] = "VpSvmHypercalls", + [200] = "VpBusLockAcquisitionCount", + [201] = "VpLoadAvg", + [202] = "VpRootDispatchThreadBlocked", + [203] = "VpIdleCpuTime", + [204] = "VpWaitingForCpuTimeBucket7", + [205] = "VpWaitingForCpuTimeBucket8", + [206] = "VpWaitingForCpuTimeBucket9", + [207] = "VpWaitingForCpuTimeBucket10", + [208] = "VpWaitingForCpuTimeBucket11", + [209] = "VpWaitingForCpuTimeBucket12", + [210] = "VpHierarchicalSuspendTime", + [211] = "VpExpressSchedulingAttempts", + [212] = "VpExpressSchedulingCount", +#elif IS_ENABLED(CONFIG_ARM64) + [9] = "VpSysRegAccessesCount", + [10] = "VpSysRegAccessesTime", + [11] = "VpSmcInstructionsCount", + [12] = "VpSmcInstructionsTime", + [13] = "VpOtherInterceptsCount", + [14] = "VpOtherInterceptsTime", + [15] = "VpExternalInterruptsCount", + [16] = "VpExternalInterruptsTime", + [17] = "VpPendingInterruptsCount", + [18] = "VpPendingInterruptsTime", + [19] = "VpGuestPageTableMaps", + [20] = "VpLargePageTlbFills", + [21] = "VpSmallPageTlbFills", + [22] = "VpReflectedGuestPageFaults", + [23] = "VpMemoryInterceptMessages", + [24] = "VpOtherMessages", + [25] = "VpLogicalProcessorMigrations", + [26] = "VpAddressDomainFlushes", + [27] = "VpAddressSpaceFlushes", + [28] = "VpSyntheticInterrupts", + [29] = "VpVirtualInterrupts", + [30] = "VpApicSelfIpisSent", + [31] = "VpGpaSpaceHypercalls", + [32] = "VpLogicalProcessorHypercalls", + [33] = "VpLongSpinWaitHypercalls", + [34] = "VpOtherHypercalls", + [35] = "VpSyntheticInterruptHypercalls", + [36] = "VpVirtualInterruptHypercalls", + [37] = "VpVirtualMmuHypercalls", + [38] = "VpVirtualProcessorHypercalls", + [39] = "VpHardwareInterrupts", + [40] = "VpNestedPageFaultInterceptsCount", + [41] = "VpNestedPageFaultInterceptsTime", + [42] = "VpLogicalProcessorDispatches", + [43] = "VpWaitingForCpuTime", + [44] = "VpExtendedHypercalls", + [45] = "VpExtendedHypercallInterceptMessages", + [46] = "VpMbecNestedPageTableSwitches", + [47] = "VpOtherReflectedGuestExceptions", + [48] = "VpGlobalIoTlbFlushes", + [49] = "VpGlobalIoTlbFlushCost", + [50] = "VpLocalIoTlbFlushes", + [51] = "VpLocalIoTlbFlushCost", + [52] = "VpFlushGuestPhysicalAddressSpaceHypercalls", + [53] = "VpFlushGuestPhysicalAddressListHypercalls", + [54] = "VpPostedInterruptNotifications", + [55] = "VpPostedInterruptScans", + [56] = "VpTotalCoreRunTime", + [57] = "VpMaximumRunTime", + [58] = "VpWaitingForCpuTimeBucket0", + [59] = "VpWaitingForCpuTimeBucket1", + [60] = "VpWaitingForCpuTimeBucket2", + [61] = "VpWaitingForCpuTimeBucket3", + [62] = "VpWaitingForCpuTimeBucket4", + [63] = "VpWaitingForCpuTimeBucket5", + [64] = "VpWaitingForCpuTimeBucket6", + [65] = "VpHwpRequestContextSwitches", + [66] = "VpPlaceholder2", + [67] = "VpPlaceholder3", + [68] = "VpPlaceholder4", + [69] = "VpPlaceholder5", + [70] = "VpPlaceholder6", + [71] = "VpPlaceholder7", + [72] = "VpPlaceholder8", + [73] = "VpContentionTime", + [74] = "VpWakeUpTime", + [75] = "VpSchedulingPriority", + [76] = "VpVtl1DispatchCount", + [77] = "VpVtl2DispatchCount", + [78] = "VpVtl2DispatchBucket0", + [79] = "VpVtl2DispatchBucket1", + [80] = "VpVtl2DispatchBucket2", + [81] = "VpVtl2DispatchBucket3", + [82] = "VpVtl2DispatchBucket4", + [83] = "VpVtl2DispatchBucket5", + [84] = "VpVtl2DispatchBucket6", + [85] = "VpVtl1RunTime", + [86] = "VpVtl2RunTime", + [87] = "VpIommuHypercalls", + [88] = "VpCpuGroupHypercalls", + [89] = "VpVsmHypercalls", + [90] = "VpEventLogHypercalls", + [91] = "VpDeviceDomainHypercalls", + [92] = "VpDepositHypercalls", + [93] = "VpSvmHypercalls", + [94] = "VpLoadAvg", + [95] = "VpRootDispatchThreadBlocked", + [96] = "VpIdleCpuTime", + [97] = "VpWaitingForCpuTimeBucket7", + [98] = "VpWaitingForCpuTimeBucket8", + [99] = "VpWaitingForCpuTimeBucket9", + [100] = "VpWaitingForCpuTimeBucket10", + [101] = "VpWaitingForCpuTimeBucket11", + [102] = "VpWaitingForCpuTimeBucket12", + [103] = "VpHierarchicalSuspendTime", + [104] = "VpExpressSchedulingAttempts", + [105] = "VpExpressSchedulingCount", +#endif +}; diff --git a/drivers/hv/mshv_eventfd.c b/drivers/hv/mshv_eventfd.c new file mode 100644 index 000000000..90959f639 --- /dev/null +++ b/drivers/hv/mshv_eventfd.c @@ -0,0 +1,853 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * eventfd support for mshv + * + * Heavily inspired from KVM implementation of irqfd/ioeventfd. The basic + * framework code is taken from the kvm implementation. + * + * All credits to kvm developers. + */ + +#include <linux/syscalls.h> +#include <linux/wait.h> +#include <linux/poll.h> +#include <linux/file.h> +#include <linux/list.h> +#include <linux/workqueue.h> +#include <linux/eventfd.h> + +#if IS_ENABLED(CONFIG_X86_64) +#include <asm/apic.h> +#endif +#include <asm/mshyperv.h> + +#include "mshv_eventfd.h" +#include "mshv.h" +#include "mshv_root.h" + +static struct workqueue_struct *irqfd_cleanup_wq; + +void mshv_register_irq_ack_notifier(struct mshv_partition *partition, + struct mshv_irq_ack_notifier *mian) +{ + mutex_lock(&partition->pt_irq_lock); + hlist_add_head_rcu(&mian->link, &partition->irq_ack_notifier_list); + mutex_unlock(&partition->pt_irq_lock); +} + +void mshv_unregister_irq_ack_notifier(struct mshv_partition *partition, + struct mshv_irq_ack_notifier *mian) +{ + mutex_lock(&partition->pt_irq_lock); + hlist_del_init_rcu(&mian->link); + mutex_unlock(&partition->pt_irq_lock); + synchronize_rcu(); +} + +bool mshv_notify_acked_gsi(struct mshv_partition *partition, int gsi) +{ + struct mshv_irq_ack_notifier *mian; + bool acked = false; + + rcu_read_lock(); + hlist_for_each_entry_rcu(mian, &partition->irq_ack_notifier_list, + link) { + if (mian->irq_ack_gsi == gsi) { + mian->irq_acked(mian); + acked = true; + } + } + rcu_read_unlock(); + + return acked; +} + +#if IS_ENABLED(CONFIG_ARM64) +static inline bool hv_should_clear_interrupt(enum hv_interrupt_type type) +{ + return false; +} +#elif IS_ENABLED(CONFIG_X86_64) +static inline bool hv_should_clear_interrupt(enum hv_interrupt_type type) +{ + return type == HV_X64_INTERRUPT_TYPE_EXTINT; +} +#endif + +static void mshv_irqfd_resampler_ack(struct mshv_irq_ack_notifier *mian) +{ + struct mshv_irqfd_resampler *resampler; + struct mshv_partition *partition; + struct mshv_irqfd *irqfd; + int idx; + + resampler = container_of(mian, struct mshv_irqfd_resampler, + rsmplr_notifier); + partition = resampler->rsmplr_partn; + + idx = srcu_read_lock(&partition->pt_irq_srcu); + + hlist_for_each_entry_srcu(irqfd, &resampler->rsmplr_irqfd_list, + irqfd_resampler_hnode, + srcu_read_lock_held(&partition->pt_irq_srcu)) { + if (hv_should_clear_interrupt(irqfd->irqfd_lapic_irq.lapic_control.interrupt_type)) + hv_call_clear_virtual_interrupt(partition->pt_id); + + eventfd_signal(irqfd->irqfd_resamplefd); + } + + srcu_read_unlock(&partition->pt_irq_srcu, idx); +} + +#if IS_ENABLED(CONFIG_X86_64) +static bool +mshv_vp_irq_vector_injected(union hv_vp_register_page_interrupt_vectors iv, + u32 vector) +{ + int i; + + for (i = 0; i < iv.vector_count; i++) { + if (iv.vector[i] == vector) + return true; + } + + return false; +} + +static int mshv_vp_irq_try_set_vector(struct mshv_vp *vp, u32 vector) +{ + union hv_vp_register_page_interrupt_vectors iv, new_iv; + + iv = vp->vp_register_page->interrupt_vectors; + new_iv = iv; + + if (mshv_vp_irq_vector_injected(iv, vector)) + return 0; + + if (iv.vector_count >= HV_VP_REGISTER_PAGE_MAX_VECTOR_COUNT) + return -ENOSPC; + + new_iv.vector[new_iv.vector_count++] = vector; + + if (!try_cmpxchg(&vp->vp_register_page->interrupt_vectors.as_uint64, + &iv.as_uint64, new_iv.as_uint64)) + return -EAGAIN; + + return 0; +} + +static int mshv_vp_irq_set_vector(struct mshv_vp *vp, u32 vector) +{ + int ret; + + do { + ret = mshv_vp_irq_try_set_vector(vp, vector); + } while (ret == -EAGAIN && !need_resched()); + + return ret; +} + +/* + * Try to raise irq for guest via shared vector array. hyp does the actual + * inject of the interrupt. + */ +static int mshv_try_assert_irq_fast(struct mshv_irqfd *irqfd) +{ + struct mshv_partition *partition = irqfd->irqfd_partn; + struct mshv_lapic_irq *irq = &irqfd->irqfd_lapic_irq; + struct mshv_vp *vp; + + if (!(ms_hyperv.ext_features & + HV_VP_DISPATCH_INTERRUPT_INJECTION_AVAILABLE)) + return -EOPNOTSUPP; + + if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT) + return -EOPNOTSUPP; + +#if IS_ENABLED(CONFIG_X86) + if (irq->lapic_control.logical_dest_mode) + return -EOPNOTSUPP; +#endif + + vp = partition->pt_vp_array[irq->lapic_apic_id]; + + if (!vp->vp_register_page) + return -EOPNOTSUPP; + + if (mshv_vp_irq_set_vector(vp, irq->lapic_vector)) + return -EINVAL; + + if (vp->run.flags.root_sched_dispatched && + vp->vp_register_page->interrupt_vectors.as_uint64) + return -EBUSY; + + wake_up(&vp->run.vp_suspend_queue); + + return 0; +} +#else /* CONFIG_X86_64 */ +static int mshv_try_assert_irq_fast(struct mshv_irqfd *irqfd) +{ + return -EOPNOTSUPP; +} +#endif + +static void mshv_assert_irq_slow(struct mshv_irqfd *irqfd) +{ + struct mshv_partition *partition = irqfd->irqfd_partn; + struct mshv_lapic_irq *irq = &irqfd->irqfd_lapic_irq; + unsigned int seq; + int idx; + +#if IS_ENABLED(CONFIG_X86) + WARN_ON(irqfd->irqfd_resampler && + !irq->lapic_control.level_triggered); +#endif + + idx = srcu_read_lock(&partition->pt_irq_srcu); + if (irqfd->irqfd_girq_ent.guest_irq_num) { + if (!irqfd->irqfd_girq_ent.girq_entry_valid) { + srcu_read_unlock(&partition->pt_irq_srcu, idx); + return; + } + + do { + seq = read_seqcount_begin(&irqfd->irqfd_irqe_sc); + } while (read_seqcount_retry(&irqfd->irqfd_irqe_sc, seq)); + } + + hv_call_assert_virtual_interrupt(irqfd->irqfd_partn->pt_id, + irq->lapic_vector, irq->lapic_apic_id, + irq->lapic_control); + srcu_read_unlock(&partition->pt_irq_srcu, idx); +} + +static void mshv_irqfd_resampler_shutdown(struct mshv_irqfd *irqfd) +{ + struct mshv_irqfd_resampler *rp = irqfd->irqfd_resampler; + struct mshv_partition *pt = rp->rsmplr_partn; + + mutex_lock(&pt->irqfds_resampler_lock); + + hlist_del_rcu(&irqfd->irqfd_resampler_hnode); + synchronize_srcu(&pt->pt_irq_srcu); + + if (hlist_empty(&rp->rsmplr_irqfd_list)) { + hlist_del(&rp->rsmplr_hnode); + mshv_unregister_irq_ack_notifier(pt, &rp->rsmplr_notifier); + kfree(rp); + } + + mutex_unlock(&pt->irqfds_resampler_lock); +} + +/* + * Race-free decouple logic (ordering is critical) + */ +static void mshv_irqfd_shutdown(struct work_struct *work) +{ + struct mshv_irqfd *irqfd = + container_of(work, struct mshv_irqfd, irqfd_shutdown); + u64 cnt; + + /* + * Synchronize with the wait-queue and unhook ourselves to prevent + * further events. + */ + eventfd_ctx_remove_wait_queue(irqfd->irqfd_eventfd_ctx, &irqfd->irqfd_wait, &cnt); + + if (irqfd->irqfd_resampler) { + mshv_irqfd_resampler_shutdown(irqfd); + eventfd_ctx_put(irqfd->irqfd_resamplefd); + } + + /* + * It is now safe to release the object's resources + */ + eventfd_ctx_put(irqfd->irqfd_eventfd_ctx); + kfree(irqfd); +} + +/* assumes partition->pt_irqfds_lock is held */ +static bool mshv_irqfd_is_active(struct mshv_irqfd *irqfd) +{ + return !hlist_unhashed(&irqfd->irqfd_hnode); +} + +/* + * Mark the irqfd as inactive and schedule it for removal + * + * assumes partition->pt_irqfds_lock is held + */ +static void mshv_irqfd_deactivate(struct mshv_irqfd *irqfd) +{ + if (!mshv_irqfd_is_active(irqfd)) + return; + + hlist_del(&irqfd->irqfd_hnode); + + queue_work(irqfd_cleanup_wq, &irqfd->irqfd_shutdown); +} + +/* + * Called with wqh->lock held and interrupts disabled + */ +static int mshv_irqfd_wakeup(wait_queue_entry_t *wait, unsigned int mode, + int sync, void *key) +{ + struct mshv_irqfd *irqfd = container_of(wait, struct mshv_irqfd, + irqfd_wait); + __poll_t flags = key_to_poll(key); + int idx; + unsigned int seq; + struct mshv_partition *pt = irqfd->irqfd_partn; + int ret = 0; + + if (flags & EPOLLIN) { + u64 cnt; + + eventfd_ctx_do_read(irqfd->irqfd_eventfd_ctx, &cnt); + idx = srcu_read_lock(&pt->pt_irq_srcu); + do { + seq = read_seqcount_begin(&irqfd->irqfd_irqe_sc); + } while (read_seqcount_retry(&irqfd->irqfd_irqe_sc, seq)); + + /* An event has been signaled, raise an interrupt */ + ret = mshv_try_assert_irq_fast(irqfd); + if (ret) + mshv_assert_irq_slow(irqfd); + + srcu_read_unlock(&pt->pt_irq_srcu, idx); + + ret = 1; + } + + if (flags & EPOLLHUP) { + /* The eventfd is closing, detach from the partition */ + unsigned long flags; + + spin_lock_irqsave(&pt->pt_irqfds_lock, flags); + + /* + * We must check if someone deactivated the irqfd before + * we could acquire the pt_irqfds_lock since the item is + * deactivated from the mshv side before it is unhooked from + * the wait-queue. If it is already deactivated, we can + * simply return knowing the other side will cleanup for us. + * We cannot race against the irqfd going away since the + * other side is required to acquire wqh->lock, which we hold + */ + if (mshv_irqfd_is_active(irqfd)) + mshv_irqfd_deactivate(irqfd); + + spin_unlock_irqrestore(&pt->pt_irqfds_lock, flags); + } + + return ret; +} + +/* Must be called under pt_irqfds_lock */ +static void mshv_irqfd_update(struct mshv_partition *pt, + struct mshv_irqfd *irqfd) +{ + write_seqcount_begin(&irqfd->irqfd_irqe_sc); + irqfd->irqfd_girq_ent = mshv_ret_girq_entry(pt, + irqfd->irqfd_irqnum); + mshv_copy_girq_info(&irqfd->irqfd_girq_ent, &irqfd->irqfd_lapic_irq); + write_seqcount_end(&irqfd->irqfd_irqe_sc); +} + +void mshv_irqfd_routing_update(struct mshv_partition *pt) +{ + struct mshv_irqfd *irqfd; + + spin_lock_irq(&pt->pt_irqfds_lock); + hlist_for_each_entry(irqfd, &pt->pt_irqfds_list, irqfd_hnode) + mshv_irqfd_update(pt, irqfd); + spin_unlock_irq(&pt->pt_irqfds_lock); +} + +static void mshv_irqfd_queue_proc(struct file *file, wait_queue_head_t *wqh, + poll_table *polltbl) +{ + struct mshv_irqfd *irqfd = + container_of(polltbl, struct mshv_irqfd, irqfd_polltbl); + + /* + * TODO: Ensure there isn't already an exclusive, priority waiter, e.g. + * that the irqfd isn't already bound to another partition. Only the + * first exclusive waiter encountered will be notified, and + * add_wait_queue_priority() doesn't enforce exclusivity. + */ + irqfd->irqfd_wait.flags |= WQ_FLAG_EXCLUSIVE; + add_wait_queue_priority(wqh, &irqfd->irqfd_wait); +} + +static int mshv_irqfd_assign(struct mshv_partition *pt, + struct mshv_user_irqfd *args) +{ + struct eventfd_ctx *eventfd = NULL, *resamplefd = NULL; + struct mshv_irqfd *irqfd, *tmp; + __poll_t events; + int ret; + int idx; + + CLASS(fd, f)(args->fd); + + irqfd = kzalloc_obj(*irqfd); + if (!irqfd) + return -ENOMEM; + + irqfd->irqfd_partn = pt; + irqfd->irqfd_irqnum = args->gsi; + INIT_WORK(&irqfd->irqfd_shutdown, mshv_irqfd_shutdown); + seqcount_spinlock_init(&irqfd->irqfd_irqe_sc, &pt->pt_irqfds_lock); + + if (fd_empty(f)) { + ret = -EBADF; + goto out; + } + + eventfd = eventfd_ctx_fileget(fd_file(f)); + if (IS_ERR(eventfd)) { + ret = PTR_ERR(eventfd); + goto fail; + } + + irqfd->irqfd_eventfd_ctx = eventfd; + + if (args->flags & BIT(MSHV_IRQFD_BIT_RESAMPLE)) { + struct mshv_irqfd_resampler *rp; + + resamplefd = eventfd_ctx_fdget(args->resamplefd); + if (IS_ERR(resamplefd)) { + ret = PTR_ERR(resamplefd); + goto fail; + } + + irqfd->irqfd_resamplefd = resamplefd; + + mutex_lock(&pt->irqfds_resampler_lock); + + hlist_for_each_entry(rp, &pt->irqfds_resampler_list, + rsmplr_hnode) { + if (rp->rsmplr_notifier.irq_ack_gsi == + irqfd->irqfd_irqnum) { + irqfd->irqfd_resampler = rp; + break; + } + } + + if (!irqfd->irqfd_resampler) { + rp = kzalloc_obj(*rp, GFP_KERNEL_ACCOUNT); + if (!rp) { + ret = -ENOMEM; + mutex_unlock(&pt->irqfds_resampler_lock); + goto fail; + } + + rp->rsmplr_partn = pt; + INIT_HLIST_HEAD(&rp->rsmplr_irqfd_list); + rp->rsmplr_notifier.irq_ack_gsi = irqfd->irqfd_irqnum; + rp->rsmplr_notifier.irq_acked = + mshv_irqfd_resampler_ack; + + hlist_add_head(&rp->rsmplr_hnode, + &pt->irqfds_resampler_list); + mshv_register_irq_ack_notifier(pt, + &rp->rsmplr_notifier); + irqfd->irqfd_resampler = rp; + } + + hlist_add_head_rcu(&irqfd->irqfd_resampler_hnode, + &irqfd->irqfd_resampler->rsmplr_irqfd_list); + + mutex_unlock(&pt->irqfds_resampler_lock); + } + + /* + * Install our own custom wake-up handling so we are notified via + * a callback whenever someone signals the underlying eventfd + */ + init_waitqueue_func_entry(&irqfd->irqfd_wait, mshv_irqfd_wakeup); + init_poll_funcptr(&irqfd->irqfd_polltbl, mshv_irqfd_queue_proc); + + spin_lock_irq(&pt->pt_irqfds_lock); +#if IS_ENABLED(CONFIG_X86) + if (args->flags & BIT(MSHV_IRQFD_BIT_RESAMPLE) && + !irqfd->irqfd_lapic_irq.lapic_control.level_triggered) { + /* + * Resample Fd must be for level triggered interrupt + * Otherwise return with failure + */ + spin_unlock_irq(&pt->pt_irqfds_lock); + ret = -EINVAL; + goto fail; + } +#endif + ret = 0; + hlist_for_each_entry(tmp, &pt->pt_irqfds_list, irqfd_hnode) { + if (irqfd->irqfd_eventfd_ctx != tmp->irqfd_eventfd_ctx) + continue; + /* This fd is used for another irq already. */ + ret = -EBUSY; + spin_unlock_irq(&pt->pt_irqfds_lock); + goto fail; + } + + idx = srcu_read_lock(&pt->pt_irq_srcu); + mshv_irqfd_update(pt, irqfd); + hlist_add_head(&irqfd->irqfd_hnode, &pt->pt_irqfds_list); + spin_unlock_irq(&pt->pt_irqfds_lock); + + /* + * Check if there was an event already pending on the eventfd + * before we registered, and trigger it as if we didn't miss it. + */ + events = vfs_poll(fd_file(f), &irqfd->irqfd_polltbl); + + if (events & EPOLLIN) + mshv_assert_irq_slow(irqfd); + + srcu_read_unlock(&pt->pt_irq_srcu, idx); + return 0; + +fail: + if (irqfd->irqfd_resampler) + mshv_irqfd_resampler_shutdown(irqfd); + + if (resamplefd && !IS_ERR(resamplefd)) + eventfd_ctx_put(resamplefd); + + if (eventfd && !IS_ERR(eventfd)) + eventfd_ctx_put(eventfd); + +out: + kfree(irqfd); + return ret; +} + +/* + * shutdown any irqfd's that match fd+gsi + */ +static int mshv_irqfd_deassign(struct mshv_partition *pt, + struct mshv_user_irqfd *args) +{ + struct mshv_irqfd *irqfd; + struct hlist_node *n; + struct eventfd_ctx *eventfd; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + hlist_for_each_entry_safe(irqfd, n, &pt->pt_irqfds_list, + irqfd_hnode) { + if (irqfd->irqfd_eventfd_ctx == eventfd && + irqfd->irqfd_irqnum == args->gsi) + + mshv_irqfd_deactivate(irqfd); + } + + eventfd_ctx_put(eventfd); + + /* + * Block until we know all outstanding shutdown jobs have completed + * so that we guarantee there will not be any more interrupts on this + * gsi once this deassign function returns. + */ + flush_workqueue(irqfd_cleanup_wq); + + return 0; +} + +int mshv_set_unset_irqfd(struct mshv_partition *pt, + struct mshv_user_irqfd *args) +{ + if (args->flags & ~MSHV_IRQFD_FLAGS_MASK) + return -EINVAL; + + if (args->flags & BIT(MSHV_IRQFD_BIT_DEASSIGN)) + return mshv_irqfd_deassign(pt, args); + + return mshv_irqfd_assign(pt, args); +} + +/* + * This function is called as the mshv VM fd is being released. + * Shutdown all irqfds that still remain open + */ +static void mshv_irqfd_release(struct mshv_partition *pt) +{ + struct mshv_irqfd *irqfd; + struct hlist_node *n; + + spin_lock_irq(&pt->pt_irqfds_lock); + + hlist_for_each_entry_safe(irqfd, n, &pt->pt_irqfds_list, irqfd_hnode) + mshv_irqfd_deactivate(irqfd); + + spin_unlock_irq(&pt->pt_irqfds_lock); + + /* + * Block until we know all outstanding shutdown jobs have completed + * since we do not take a mshv_partition* reference. + */ + flush_workqueue(irqfd_cleanup_wq); +} + +int mshv_irqfd_wq_init(void) +{ + irqfd_cleanup_wq = alloc_workqueue("mshv-irqfd-cleanup", WQ_PERCPU, 0); + if (!irqfd_cleanup_wq) + return -ENOMEM; + + return 0; +} + +void mshv_irqfd_wq_cleanup(void) +{ + destroy_workqueue(irqfd_cleanup_wq); +} + +/* + * -------------------------------------------------------------------- + * ioeventfd: translate a MMIO memory write to an eventfd signal. + * + * userspace can register a MMIO address with an eventfd for receiving + * notification when the memory has been touched. + * -------------------------------------------------------------------- + */ + +static void ioeventfd_release(struct mshv_ioeventfd *p, u64 partition_id) +{ + if (p->iovntfd_doorbell_id > 0) + mshv_unregister_doorbell(partition_id, p->iovntfd_doorbell_id); + eventfd_ctx_put(p->iovntfd_eventfd); + kfree(p); +} + +/* MMIO writes trigger an event if the addr/val match */ +static void ioeventfd_mmio_write(int doorbell_id, void *data) +{ + struct mshv_partition *partition = (struct mshv_partition *)data; + struct mshv_ioeventfd *p; + + rcu_read_lock(); + hlist_for_each_entry_rcu(p, &partition->ioeventfds_list, iovntfd_hnode) + if (p->iovntfd_doorbell_id == doorbell_id) { + eventfd_signal(p->iovntfd_eventfd); + break; + } + + rcu_read_unlock(); +} + +static bool ioeventfd_check_collision(struct mshv_partition *pt, + struct mshv_ioeventfd *p) + __must_hold(&pt->mutex) +{ + struct mshv_ioeventfd *_p; + + hlist_for_each_entry(_p, &pt->ioeventfds_list, iovntfd_hnode) + if (_p->iovntfd_addr == p->iovntfd_addr && + _p->iovntfd_length == p->iovntfd_length && + (_p->iovntfd_wildcard || p->iovntfd_wildcard || + _p->iovntfd_datamatch == p->iovntfd_datamatch)) + return true; + + return false; +} + +static int mshv_assign_ioeventfd(struct mshv_partition *pt, + struct mshv_user_ioeventfd *args) + __must_hold(&pt->mutex) +{ + struct mshv_ioeventfd *p; + struct eventfd_ctx *eventfd; + u64 doorbell_flags = 0; + int ret; + + /* This mutex is currently protecting ioeventfd.items list */ + WARN_ON_ONCE(!mutex_is_locked(&pt->pt_mutex)); + + if (args->flags & BIT(MSHV_IOEVENTFD_BIT_PIO)) + return -EOPNOTSUPP; + + /* must be natural-word sized */ + switch (args->len) { + case 0: + doorbell_flags = HV_DOORBELL_FLAG_TRIGGER_SIZE_ANY; + break; + case 1: + doorbell_flags = HV_DOORBELL_FLAG_TRIGGER_SIZE_BYTE; + break; + case 2: + doorbell_flags = HV_DOORBELL_FLAG_TRIGGER_SIZE_WORD; + break; + case 4: + doorbell_flags = HV_DOORBELL_FLAG_TRIGGER_SIZE_DWORD; + break; + case 8: + doorbell_flags = HV_DOORBELL_FLAG_TRIGGER_SIZE_QWORD; + break; + default: + return -EINVAL; + } + + /* check for range overflow */ + if (args->addr + args->len < args->addr) + return -EINVAL; + + /* check for extra flags that we don't understand */ + if (args->flags & ~MSHV_IOEVENTFD_FLAGS_MASK) + return -EINVAL; + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + p = kzalloc_obj(*p); + if (!p) { + ret = -ENOMEM; + goto fail; + } + + p->iovntfd_addr = args->addr; + p->iovntfd_length = args->len; + p->iovntfd_eventfd = eventfd; + + /* The datamatch feature is optional, otherwise this is a wildcard */ + if (args->flags & BIT(MSHV_IOEVENTFD_BIT_DATAMATCH)) { + p->iovntfd_datamatch = args->datamatch; + } else { + p->iovntfd_wildcard = true; + doorbell_flags |= HV_DOORBELL_FLAG_TRIGGER_ANY_VALUE; + } + + if (ioeventfd_check_collision(pt, p)) { + ret = -EEXIST; + goto unlock_fail; + } + + ret = mshv_register_doorbell(pt->pt_id, ioeventfd_mmio_write, + (void *)pt, p->iovntfd_addr, + p->iovntfd_datamatch, doorbell_flags); + + trace_mshv_assign_ioeventfd(pt->pt_id, p->iovntfd_addr, + p->iovntfd_length, + p->iovntfd_datamatch, + p->iovntfd_wildcard, + p->iovntfd_eventfd, + ret); + + if (ret < 0) + goto unlock_fail; + + p->iovntfd_doorbell_id = ret; + + hlist_add_head_rcu(&p->iovntfd_hnode, &pt->ioeventfds_list); + + return 0; + +unlock_fail: + kfree(p); + +fail: + eventfd_ctx_put(eventfd); + + return ret; +} + +static int mshv_deassign_ioeventfd(struct mshv_partition *pt, + struct mshv_user_ioeventfd *args) + __must_hold(&pt->mutex) +{ + struct mshv_ioeventfd *p; + struct eventfd_ctx *eventfd; + struct hlist_node *n; + int ret = -ENOENT; + + /* This mutex is currently protecting ioeventfd.items list */ + WARN_ON_ONCE(!mutex_is_locked(&pt->pt_mutex)); + + eventfd = eventfd_ctx_fdget(args->fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + + hlist_for_each_entry_safe(p, n, &pt->ioeventfds_list, iovntfd_hnode) { + bool wildcard = !(args->flags & BIT(MSHV_IOEVENTFD_BIT_DATAMATCH)); + + if (p->iovntfd_eventfd != eventfd || + p->iovntfd_addr != args->addr || + p->iovntfd_length != args->len || + p->iovntfd_wildcard != wildcard) + continue; + + if (!p->iovntfd_wildcard && + p->iovntfd_datamatch != args->datamatch) + continue; + + trace_mshv_deassign_ioeventfd(pt->pt_id, p->iovntfd_addr, + p->iovntfd_length, + p->iovntfd_datamatch, + p->iovntfd_wildcard, + p->iovntfd_eventfd); + + hlist_del_rcu(&p->iovntfd_hnode); + synchronize_rcu(); + ioeventfd_release(p, pt->pt_id); + ret = 0; + break; + } + + eventfd_ctx_put(eventfd); + + return ret; +} + +int mshv_set_unset_ioeventfd(struct mshv_partition *pt, + struct mshv_user_ioeventfd *args) + __must_hold(&pt->mutex) +{ + if ((args->flags & ~MSHV_IOEVENTFD_FLAGS_MASK) || + mshv_field_nonzero(*args, rsvd)) + return -EINVAL; + + /* PIO not yet implemented */ + if (args->flags & BIT(MSHV_IOEVENTFD_BIT_PIO)) + return -EOPNOTSUPP; + + if (args->flags & BIT(MSHV_IOEVENTFD_BIT_DEASSIGN)) + return mshv_deassign_ioeventfd(pt, args); + + return mshv_assign_ioeventfd(pt, args); +} + +void mshv_eventfd_init(struct mshv_partition *pt) +{ + spin_lock_init(&pt->pt_irqfds_lock); + INIT_HLIST_HEAD(&pt->pt_irqfds_list); + + INIT_HLIST_HEAD(&pt->irqfds_resampler_list); + mutex_init(&pt->irqfds_resampler_lock); + + INIT_HLIST_HEAD(&pt->ioeventfds_list); +} + +void mshv_eventfd_release(struct mshv_partition *pt) +{ + struct hlist_head items; + struct hlist_node *n; + struct mshv_ioeventfd *p; + + hlist_move_list(&pt->ioeventfds_list, &items); + synchronize_rcu(); + + hlist_for_each_entry_safe(p, n, &items, iovntfd_hnode) { + hlist_del(&p->iovntfd_hnode); + ioeventfd_release(p, pt->pt_id); + } + + mshv_irqfd_release(pt); +} diff --git a/drivers/hv/mshv_eventfd.h b/drivers/hv/mshv_eventfd.h new file mode 100644 index 000000000..464c6b81a --- /dev/null +++ b/drivers/hv/mshv_eventfd.h @@ -0,0 +1,70 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * irqfd: Allows an fd to be used to inject an interrupt to the guest. + * ioeventfd: Allow an fd to be used to receive a signal from the guest. + * All credit goes to kvm developers. + */ + +#ifndef __LINUX_MSHV_EVENTFD_H +#define __LINUX_MSHV_EVENTFD_H + +#include <linux/poll.h> + +#include "mshv.h" +#include "mshv_root.h" + +/* struct to contain list of irqfds sharing an irq. Updates are protected by + * partition.irqfds.resampler_lock + */ +struct mshv_irqfd_resampler { + struct mshv_partition *rsmplr_partn; + struct hlist_head rsmplr_irqfd_list; + struct mshv_irq_ack_notifier rsmplr_notifier; + struct hlist_node rsmplr_hnode; +}; + +struct mshv_irqfd { + struct mshv_partition *irqfd_partn; + struct eventfd_ctx *irqfd_eventfd_ctx; + struct mshv_guest_irq_ent irqfd_girq_ent; + seqcount_spinlock_t irqfd_irqe_sc; + u32 irqfd_irqnum; + struct mshv_lapic_irq irqfd_lapic_irq; + struct hlist_node irqfd_hnode; + poll_table irqfd_polltbl; + wait_queue_entry_t irqfd_wait; + struct work_struct irqfd_shutdown; + struct mshv_irqfd_resampler *irqfd_resampler; + struct eventfd_ctx *irqfd_resamplefd; + struct hlist_node irqfd_resampler_hnode; +}; + +void mshv_eventfd_init(struct mshv_partition *partition); +void mshv_eventfd_release(struct mshv_partition *partition); + +void mshv_register_irq_ack_notifier(struct mshv_partition *partition, + struct mshv_irq_ack_notifier *mian); +void mshv_unregister_irq_ack_notifier(struct mshv_partition *partition, + struct mshv_irq_ack_notifier *mian); +bool mshv_notify_acked_gsi(struct mshv_partition *partition, int gsi); + +int mshv_set_unset_irqfd(struct mshv_partition *partition, + struct mshv_user_irqfd *args); + +int mshv_irqfd_wq_init(void); +void mshv_irqfd_wq_cleanup(void); + +struct mshv_ioeventfd { + struct hlist_node iovntfd_hnode; + u64 iovntfd_addr; + int iovntfd_length; + struct eventfd_ctx *iovntfd_eventfd; + u64 iovntfd_datamatch; + int iovntfd_doorbell_id; + bool iovntfd_wildcard; +}; + +int mshv_set_unset_ioeventfd(struct mshv_partition *pt, + struct mshv_user_ioeventfd *args); + +#endif /* __LINUX_MSHV_EVENTFD_H */ diff --git a/drivers/hv/mshv_irq.c b/drivers/hv/mshv_irq.c new file mode 100644 index 000000000..b3142c84d --- /dev/null +++ b/drivers/hv/mshv_irq.c @@ -0,0 +1,132 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023, Microsoft Corporation. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/slab.h> +#include <asm/mshyperv.h> + +#include "mshv_eventfd.h" +#include "mshv.h" +#include "mshv_root.h" + +/* called from the ioctl code, user wants to update the guest irq table */ +int mshv_update_routing_table(struct mshv_partition *partition, + const struct mshv_user_irq_entry *ue, + unsigned int numents) +{ + struct mshv_girq_routing_table *new = NULL, *old; + u32 i, nr_rt_entries = 0; + int r = 0; + + if (numents == 0) + goto swap_routes; + + for (i = 0; i < numents; i++) { + if (ue[i].gsi >= MSHV_MAX_GUEST_IRQS) + return -EINVAL; + + if (ue[i].address_hi) + return -EINVAL; + + nr_rt_entries = max(nr_rt_entries, ue[i].gsi); + } + nr_rt_entries += 1; + + new = kzalloc_flex(*new, mshv_girq_info_tbl, nr_rt_entries, + GFP_KERNEL_ACCOUNT); + if (!new) + return -ENOMEM; + + new->num_rt_entries = nr_rt_entries; + for (i = 0; i < numents; i++) { + struct mshv_guest_irq_ent *girq; + + girq = &new->mshv_girq_info_tbl[ue[i].gsi]; + + /* + * Allow only one to one mapping between GSI and MSI routing. + */ + if (girq->guest_irq_num != 0) { + r = -EINVAL; + goto out; + } + + girq->guest_irq_num = ue[i].gsi; + girq->girq_addr_lo = ue[i].address_lo; + girq->girq_addr_hi = ue[i].address_hi; + girq->girq_irq_data = ue[i].data; + girq->girq_entry_valid = true; + } + +swap_routes: + mutex_lock(&partition->pt_irq_lock); + old = rcu_dereference_protected(partition->pt_girq_tbl, 1); + rcu_assign_pointer(partition->pt_girq_tbl, new); + mshv_irqfd_routing_update(partition); + mutex_unlock(&partition->pt_irq_lock); + + synchronize_srcu_expedited(&partition->pt_irq_srcu); + + trace_mshv_update_routing_table(partition->pt_id, + old, new, numents); + + new = old; + +out: + kfree(new); + + return r; +} + +/* vm is going away, kfree the irq routing table */ +void mshv_free_routing_table(struct mshv_partition *partition) +{ + struct mshv_girq_routing_table *rt = + rcu_access_pointer(partition->pt_girq_tbl); + + kfree(rt); +} + +struct mshv_guest_irq_ent +mshv_ret_girq_entry(struct mshv_partition *partition, u32 irqnum) +{ + struct mshv_guest_irq_ent entry = { 0 }; + struct mshv_girq_routing_table *girq_tbl; + + girq_tbl = srcu_dereference_check(partition->pt_girq_tbl, + &partition->pt_irq_srcu, + lockdep_is_held(&partition->pt_irq_lock)); + if (!girq_tbl || irqnum >= girq_tbl->num_rt_entries) { + /* + * Premature register_irqfd, setting valid_entry = 0 + * would ignore this entry anyway + */ + entry.guest_irq_num = irqnum; + return entry; + } + + return girq_tbl->mshv_girq_info_tbl[irqnum]; +} + +void mshv_copy_girq_info(struct mshv_guest_irq_ent *ent, + struct mshv_lapic_irq *lirq) +{ + memset(lirq, 0, sizeof(*lirq)); + if (!ent || !ent->girq_entry_valid) + return; + + lirq->lapic_vector = ent->girq_irq_data & 0xFF; + lirq->lapic_apic_id = (ent->girq_addr_lo >> 12) & 0xFF; + lirq->lapic_control.interrupt_type = (ent->girq_irq_data & 0x700) >> 8; +#if IS_ENABLED(CONFIG_X86) + lirq->lapic_control.level_triggered = (ent->girq_irq_data >> 15) & 0x1; + lirq->lapic_control.logical_dest_mode = (ent->girq_addr_lo >> 2) & 0x1; +#elif IS_ENABLED(CONFIG_ARM64) + lirq->lapic_control.asserted = 1; +#endif +} diff --git a/drivers/hv/mshv_portid_table.c b/drivers/hv/mshv_portid_table.c new file mode 100644 index 000000000..c349af1f0 --- /dev/null +++ b/drivers/hv/mshv_portid_table.c @@ -0,0 +1,83 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <linux/types.h> +#include <linux/mm.h> +#include <linux/slab.h> +#include <linux/idr.h> +#include <asm/mshyperv.h> + +#include "mshv.h" +#include "mshv_root.h" + +/* + * Ports and connections are hypervisor struct used for inter-partition + * communication. Port represents the source and connection represents + * the destination. Partitions are responsible for managing the port and + * connection ids. + * + */ + +#define PORTID_MIN 1 +#define PORTID_MAX INT_MAX + +static DEFINE_IDR(port_table_idr); + +void +mshv_port_table_fini(void) +{ + struct port_table_info *port_info; + unsigned long i, tmp; + + idr_lock(&port_table_idr); + if (!idr_is_empty(&port_table_idr)) { + idr_for_each_entry_ul(&port_table_idr, port_info, tmp, i) { + port_info = idr_remove(&port_table_idr, i); + kfree_rcu(port_info, portbl_rcu); + } + } + idr_unlock(&port_table_idr); +} + +int +mshv_portid_alloc(struct port_table_info *info) +{ + int ret = 0; + + idr_lock(&port_table_idr); + ret = idr_alloc(&port_table_idr, info, PORTID_MIN, + PORTID_MAX, GFP_KERNEL); + idr_unlock(&port_table_idr); + + return ret; +} + +void +mshv_portid_free(int port_id) +{ + struct port_table_info *info; + + idr_lock(&port_table_idr); + info = idr_remove(&port_table_idr, port_id); + WARN_ON(!info); + idr_unlock(&port_table_idr); + + synchronize_rcu(); + kfree(info); +} + +int +mshv_portid_lookup(int port_id, struct port_table_info *info) +{ + struct port_table_info *_info; + int ret = -ENOENT; + + rcu_read_lock(); + _info = idr_find(&port_table_idr, port_id); + rcu_read_unlock(); + + if (_info) { + *info = *_info; + ret = 0; + } + + return ret; +} diff --git a/drivers/hv/mshv_regions.c b/drivers/hv/mshv_regions.c new file mode 100644 index 000000000..6d65e5b42 --- /dev/null +++ b/drivers/hv/mshv_regions.c @@ -0,0 +1,587 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2025, Microsoft Corporation. + * + * Memory region management for mshv_root module. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/hmm.h> +#include <linux/hyperv.h> +#include <linux/kref.h> +#include <linux/mm.h> +#include <linux/vmalloc.h> + +#include <asm/mshyperv.h> + +#include "mshv_root.h" + +#define MSHV_MAP_FAULT_IN_PAGES PTRS_PER_PMD + +/** + * mshv_chunk_stride - Compute stride for mapping guest memory + * @page : The page to check for huge page backing + * @gfn : Guest frame number for the mapping + * @page_count: Total number of pages in the mapping + * + * Determines the appropriate stride (in pages) for mapping guest memory. + * Uses huge page stride if the backing page is huge and the guest mapping + * is properly aligned; otherwise falls back to single page stride. + * + * Return: Stride in pages. + */ +static unsigned int mshv_chunk_stride(struct page *page, u64 gfn, + u64 page_count) +{ + unsigned int page_order = folio_order(page_folio(page)); + + /* + * Use single page stride by default. For huge page stride, the + * folio order must be at least PMD_ORDER, the page's PFN must be + * 2M-aligned (so that a 2M-aligned tail page of a larger folio is + * acceptable), and both gfn and page_count must be 2M-aligned. + */ + if (page_order < PMD_ORDER || + !IS_ALIGNED(page_to_pfn(page), PTRS_PER_PMD) || + !IS_ALIGNED(gfn, PTRS_PER_PMD) || + !IS_ALIGNED(page_count, PTRS_PER_PMD)) + return 1; + + /* Use 2M stride always i.e. process 1G folios as 2M chunks */ + return 1 << PMD_ORDER; +} + +/** + * mshv_region_process_chunk - Processes a contiguous chunk of memory pages + * in a region. + * @region : Pointer to the memory region structure. + * @flags : Flags to pass to the handler. + * @page_offset: Offset into the region's pages array to start processing. + * @page_count : Number of pages to process. + * @handler : Callback function to handle the chunk. + * + * This function scans the region's pages starting from @page_offset, + * checking for contiguous present pages of the same size (normal or huge). + * It invokes @handler for the chunk of contiguous pages found. Returns the + * number of pages handled, or a negative error code if the first page is + * not present or the handler fails. + * + * Note: The @handler callback must be able to handle both normal and huge + * pages. + * + * Return: Number of pages handled, or negative error code. + */ +static long mshv_region_process_chunk(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, u64 page_count, + int (*handler)(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, + u64 page_count, + bool huge_page)) +{ + u64 gfn = region->start_gfn + page_offset; + u64 count; + struct page *page; + unsigned int stride; + int ret; + + page = region->mreg_pages[page_offset]; + if (!page) + return -EINVAL; + + stride = mshv_chunk_stride(page, gfn, page_count); + + /* Start at stride since the first stride is validated */ + for (count = stride; count < page_count; count += stride) { + page = region->mreg_pages[page_offset + count]; + + /* Break if current page is not present */ + if (!page) + break; + + /* Break if stride size changes */ + if (stride != mshv_chunk_stride(page, gfn + count, + page_count - count)) + break; + } + + ret = handler(region, flags, page_offset, count, stride > 1); + if (ret) + return ret; + + return count; +} + +/** + * mshv_region_process_range - Processes a range of memory pages in a + * region. + * @region : Pointer to the memory region structure. + * @flags : Flags to pass to the handler. + * @page_offset: Offset into the region's pages array to start processing. + * @page_count : Number of pages to process. + * @handler : Callback function to handle each chunk of contiguous + * pages. + * + * Iterates over the specified range of pages in @region, skipping + * non-present pages. For each contiguous chunk of present pages, invokes + * @handler via mshv_region_process_chunk. + * + * Note: The @handler callback must be able to handle both normal and huge + * pages. + * + * Returns 0 on success, or a negative error code on failure. + */ +static int mshv_region_process_range(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, u64 page_count, + int (*handler)(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, + u64 page_count, + bool huge_page)) +{ + long ret; + + if (page_offset + page_count > region->nr_pages) + return -EINVAL; + + while (page_count) { + /* Skip non-present pages */ + if (!region->mreg_pages[page_offset]) { + page_offset++; + page_count--; + continue; + } + + ret = mshv_region_process_chunk(region, flags, + page_offset, + page_count, + handler); + if (ret < 0) + return ret; + + page_offset += ret; + page_count -= ret; + } + + return 0; +} + +struct mshv_mem_region *mshv_region_create(u64 guest_pfn, u64 nr_pages, + u64 uaddr, u32 flags) +{ + struct mshv_mem_region *region; + + region = vzalloc(sizeof(*region) + sizeof(struct page *) * nr_pages); + if (!region) + return ERR_PTR(-ENOMEM); + + region->nr_pages = nr_pages; + region->start_gfn = guest_pfn; + region->start_uaddr = uaddr; + region->hv_map_flags = HV_MAP_GPA_READABLE | HV_MAP_GPA_ADJUSTABLE; + if (flags & BIT(MSHV_SET_MEM_BIT_WRITABLE)) + region->hv_map_flags |= HV_MAP_GPA_WRITABLE; + if (flags & BIT(MSHV_SET_MEM_BIT_EXECUTABLE)) + region->hv_map_flags |= HV_MAP_GPA_EXECUTABLE; + + kref_init(®ion->mreg_refcount); + + return region; +} + +static int mshv_region_chunk_share(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, u64 page_count, + bool huge_page) +{ + if (huge_page) + flags |= HV_MODIFY_SPA_PAGE_HOST_ACCESS_LARGE_PAGE; + + return hv_call_modify_spa_host_access(region->partition->pt_id, + region->mreg_pages + page_offset, + page_count, + HV_MAP_GPA_READABLE | + HV_MAP_GPA_WRITABLE, + flags, true); +} + +int mshv_region_share(struct mshv_mem_region *region) +{ + u32 flags = HV_MODIFY_SPA_PAGE_HOST_ACCESS_MAKE_SHARED; + + return mshv_region_process_range(region, flags, + 0, region->nr_pages, + mshv_region_chunk_share); +} + +static int mshv_region_chunk_unshare(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, u64 page_count, + bool huge_page) +{ + if (huge_page) + flags |= HV_MODIFY_SPA_PAGE_HOST_ACCESS_LARGE_PAGE; + + return hv_call_modify_spa_host_access(region->partition->pt_id, + region->mreg_pages + page_offset, + page_count, 0, + flags, false); +} + +int mshv_region_unshare(struct mshv_mem_region *region) +{ + u32 flags = HV_MODIFY_SPA_PAGE_HOST_ACCESS_MAKE_EXCLUSIVE; + + return mshv_region_process_range(region, flags, + 0, region->nr_pages, + mshv_region_chunk_unshare); +} + +static int mshv_region_chunk_remap(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, u64 page_count, + bool huge_page) +{ + if (huge_page) + flags |= HV_MAP_GPA_LARGE_PAGE; + + return hv_call_map_gpa_pages(region->partition->pt_id, + region->start_gfn + page_offset, + page_count, flags, + region->mreg_pages + page_offset); +} + +static int mshv_region_remap_pages(struct mshv_mem_region *region, + u32 map_flags, + u64 page_offset, u64 page_count) +{ + return mshv_region_process_range(region, map_flags, + page_offset, page_count, + mshv_region_chunk_remap); +} + +int mshv_region_map(struct mshv_mem_region *region) +{ + u32 map_flags = region->hv_map_flags; + + return mshv_region_remap_pages(region, map_flags, + 0, region->nr_pages); +} + +static void mshv_region_invalidate_pages(struct mshv_mem_region *region, + u64 page_offset, u64 page_count) +{ + if (region->mreg_type == MSHV_REGION_TYPE_MEM_PINNED) + unpin_user_pages(region->mreg_pages + page_offset, page_count); + + memset(region->mreg_pages + page_offset, 0, + page_count * sizeof(struct page *)); +} + +void mshv_region_invalidate(struct mshv_mem_region *region) +{ + mshv_region_invalidate_pages(region, 0, region->nr_pages); +} + +int mshv_region_pin(struct mshv_mem_region *region) +{ + u64 done_count, nr_pages; + struct page **pages; + __u64 userspace_addr; + int ret; + + for (done_count = 0; done_count < region->nr_pages; done_count += ret) { + pages = region->mreg_pages + done_count; + userspace_addr = region->start_uaddr + + done_count * HV_HYP_PAGE_SIZE; + nr_pages = min(region->nr_pages - done_count, + MSHV_PIN_PAGES_BATCH_SIZE); + + /* + * Pinning assuming 4k pages works for large pages too. + * All page structs within the large page are returned. + * + * Pin requests are batched because pin_user_pages_fast + * with the FOLL_LONGTERM flag does a large temporary + * allocation of contiguous memory. + */ + ret = pin_user_pages_fast(userspace_addr, nr_pages, + FOLL_WRITE | FOLL_LONGTERM, + pages); + if (ret != nr_pages) + goto release_pages; + } + + return 0; + +release_pages: + if (ret > 0) + done_count += ret; + mshv_region_invalidate_pages(region, 0, done_count); + return ret < 0 ? ret : -ENOMEM; +} + +static int mshv_region_chunk_unmap(struct mshv_mem_region *region, + u32 flags, + u64 page_offset, u64 page_count, + bool huge_page) +{ + if (huge_page) + flags |= HV_UNMAP_GPA_LARGE_PAGE; + + return hv_call_unmap_gpa_pages(region->partition->pt_id, + region->start_gfn + page_offset, + page_count, flags); +} + +static int mshv_region_unmap(struct mshv_mem_region *region) +{ + return mshv_region_process_range(region, 0, + 0, region->nr_pages, + mshv_region_chunk_unmap); +} + +static void mshv_region_destroy(struct kref *ref) +{ + struct mshv_mem_region *region = + container_of(ref, struct mshv_mem_region, mreg_refcount); + struct mshv_partition *partition = region->partition; + int ret; + + if (region->mreg_type == MSHV_REGION_TYPE_MEM_MOVABLE) + mshv_region_movable_fini(region); + + if (mshv_partition_encrypted(partition)) { + ret = mshv_region_share(region); + if (ret) { + pt_err(partition, + "Failed to regain access to memory, unpinning user pages will fail and crash the host error: %d\n", + ret); + return; + } + } + + mshv_region_unmap(region); + + mshv_region_invalidate(region); + + vfree(region); +} + +void mshv_region_put(struct mshv_mem_region *region) +{ + kref_put(®ion->mreg_refcount, mshv_region_destroy); +} + +int mshv_region_get(struct mshv_mem_region *region) +{ + return kref_get_unless_zero(®ion->mreg_refcount); +} + +/** + * mshv_region_hmm_fault_and_lock - Handle HMM faults and lock the memory region + * @region: Pointer to the memory region structure + * @range: Pointer to the HMM range structure + * + * This function performs the following steps: + * 1. Reads the notifier sequence for the HMM range. + * 2. Acquires a read lock on the memory map. + * 3. Handles HMM faults for the specified range. + * 4. Releases the read lock on the memory map. + * 5. If successful, locks the memory region mutex. + * 6. Verifies if the notifier sequence has changed during the operation. + * If it has, releases the mutex and returns -EBUSY to match with + * hmm_range_fault() return code for repeating. + * + * Return: 0 on success, a negative error code otherwise. + */ +static int mshv_region_hmm_fault_and_lock(struct mshv_mem_region *region, + struct hmm_range *range) +{ + int ret; + + range->notifier_seq = mmu_interval_read_begin(range->notifier); + mmap_read_lock(region->mreg_mni.mm); + ret = hmm_range_fault(range); + mmap_read_unlock(region->mreg_mni.mm); + if (ret) + return ret; + + mutex_lock(®ion->mreg_mutex); + + if (mmu_interval_read_retry(range->notifier, range->notifier_seq)) { + mutex_unlock(®ion->mreg_mutex); + cond_resched(); + return -EBUSY; + } + + return 0; +} + +/** + * mshv_region_range_fault - Handle memory range faults for a given region. + * @region: Pointer to the memory region structure. + * @page_offset: Offset of the page within the region. + * @page_count: Number of pages to handle. + * + * This function resolves memory faults for a specified range of pages + * within a memory region. It uses HMM (Heterogeneous Memory Management) + * to fault in the required pages and updates the region's page array. + * + * Return: 0 on success, negative error code on failure. + */ +static int mshv_region_range_fault(struct mshv_mem_region *region, + u64 page_offset, u64 page_count) +{ + struct hmm_range range = { + .notifier = ®ion->mreg_mni, + .default_flags = HMM_PFN_REQ_FAULT | HMM_PFN_REQ_WRITE, + }; + unsigned long *pfns; + int ret; + u64 i; + + pfns = kmalloc_array(page_count, sizeof(*pfns), GFP_KERNEL); + if (!pfns) + return -ENOMEM; + + range.hmm_pfns = pfns; + range.start = region->start_uaddr + page_offset * HV_HYP_PAGE_SIZE; + range.end = range.start + page_count * HV_HYP_PAGE_SIZE; + + do { + ret = mshv_region_hmm_fault_and_lock(region, &range); + } while (ret == -EBUSY); + + if (ret) + goto out; + + for (i = 0; i < page_count; i++) + region->mreg_pages[page_offset + i] = hmm_pfn_to_page(pfns[i]); + + ret = mshv_region_remap_pages(region, region->hv_map_flags, + page_offset, page_count); + + mutex_unlock(®ion->mreg_mutex); +out: + kfree(pfns); + return ret; +} + +bool mshv_region_handle_gfn_fault(struct mshv_mem_region *region, u64 gfn) +{ + u64 page_offset, page_count; + int ret; + + /* Align the page offset to the nearest MSHV_MAP_FAULT_IN_PAGES. */ + page_offset = ALIGN_DOWN(gfn - region->start_gfn, + MSHV_MAP_FAULT_IN_PAGES); + + /* Map more pages than requested to reduce the number of faults. */ + page_count = min(region->nr_pages - page_offset, + MSHV_MAP_FAULT_IN_PAGES); + + ret = mshv_region_range_fault(region, page_offset, page_count); + + WARN_ONCE(ret, + "p%llu: GPA intercept failed: region %#llx-%#llx, gfn %#llx, page_offset %llu, page_count %llu\n", + region->partition->pt_id, region->start_uaddr, + region->start_uaddr + (region->nr_pages << HV_HYP_PAGE_SHIFT), + gfn, page_offset, page_count); + + return !ret; +} + +/** + * mshv_region_interval_invalidate - Invalidate a range of memory region + * @mni: Pointer to the mmu_interval_notifier structure + * @range: Pointer to the mmu_notifier_range structure + * @cur_seq: Current sequence number for the interval notifier + * + * This function invalidates a memory region by remapping its pages with + * no access permissions. It locks the region's mutex to ensure thread safety + * and updates the sequence number for the interval notifier. If the range + * is blockable, it uses a blocking lock; otherwise, it attempts a non-blocking + * lock and returns false if unsuccessful. + * + * NOTE: Failure to invalidate a region is a serious error, as the pages will + * be considered freed while they are still mapped by the hypervisor. + * Any attempt to access such pages will likely crash the system. + * + * Return: true if the region was successfully invalidated, false otherwise. + */ +static bool mshv_region_interval_invalidate(struct mmu_interval_notifier *mni, + const struct mmu_notifier_range *range, + unsigned long cur_seq) +{ + struct mshv_mem_region *region = container_of(mni, + struct mshv_mem_region, + mreg_mni); + u64 page_offset, page_count; + unsigned long mstart, mend; + int ret = -EPERM; + + mstart = max(range->start, region->start_uaddr); + mend = min(range->end, region->start_uaddr + + (region->nr_pages << HV_HYP_PAGE_SHIFT)); + + page_offset = HVPFN_DOWN(mstart - region->start_uaddr); + page_count = HVPFN_DOWN(mend - mstart); + + if (mmu_notifier_range_blockable(range)) + mutex_lock(®ion->mreg_mutex); + else if (!mutex_trylock(®ion->mreg_mutex)) + goto out_fail; + + mmu_interval_set_seq(mni, cur_seq); + + ret = mshv_region_remap_pages(region, HV_MAP_GPA_NO_ACCESS, + page_offset, page_count); + if (ret) + goto out_unlock; + + mshv_region_invalidate_pages(region, page_offset, page_count); + + mutex_unlock(®ion->mreg_mutex); + + return true; + +out_unlock: + mutex_unlock(®ion->mreg_mutex); +out_fail: + WARN_ONCE(ret, + "Failed to invalidate region %#llx-%#llx (range %#lx-%#lx, event: %u, pages %#llx-%#llx, mm: %#llx): %d\n", + region->start_uaddr, + region->start_uaddr + (region->nr_pages << HV_HYP_PAGE_SHIFT), + range->start, range->end, range->event, + page_offset, page_offset + page_count - 1, (u64)range->mm, ret); + return false; +} + +static const struct mmu_interval_notifier_ops mshv_region_mni_ops = { + .invalidate = mshv_region_interval_invalidate, +}; + +void mshv_region_movable_fini(struct mshv_mem_region *region) +{ + mmu_interval_notifier_remove(®ion->mreg_mni); +} + +bool mshv_region_movable_init(struct mshv_mem_region *region) +{ + int ret; + + ret = mmu_interval_notifier_insert(®ion->mreg_mni, current->mm, + region->start_uaddr, + region->nr_pages << HV_HYP_PAGE_SHIFT, + &mshv_region_mni_ops); + if (ret) + return false; + + mutex_init(®ion->mreg_mutex); + + return true; +} diff --git a/drivers/hv/mshv_root.h b/drivers/hv/mshv_root.h new file mode 100644 index 000000000..1f086dcb7 --- /dev/null +++ b/drivers/hv/mshv_root.h @@ -0,0 +1,381 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2023, Microsoft Corporation. + */ + +#ifndef _MSHV_ROOT_H_ +#define _MSHV_ROOT_H_ + +#include <linux/spinlock.h> +#include <linux/mutex.h> +#include <linux/semaphore.h> +#include <linux/sched.h> +#include <linux/srcu.h> +#include <linux/wait.h> +#include <linux/hashtable.h> +#include <linux/dev_printk.h> +#include <linux/build_bug.h> +#include <linux/mmu_notifier.h> +#include <uapi/linux/mshv.h> +#include "mshv_trace.h" + +/* + * Hypervisor must be between these version numbers (inclusive) + * to guarantee compatibility + */ +#define MSHV_HV_MIN_VERSION (27744) +#define MSHV_HV_MAX_VERSION (27751) + +static_assert(HV_HYP_PAGE_SIZE == MSHV_HV_PAGE_SIZE); + +#define MSHV_MAX_VPS 256 + +#define MSHV_PARTITIONS_HASH_BITS 9 + +#define MSHV_PIN_PAGES_BATCH_SIZE (0x10000000ULL / HV_HYP_PAGE_SIZE) + +struct mshv_vp { + u32 vp_index; + struct mshv_partition *vp_partition; + struct mutex vp_mutex; + struct hv_vp_register_page *vp_register_page; + struct hv_message *vp_intercept_msg_page; + void *vp_ghcb_page; + struct hv_stats_page *vp_stats_pages[2]; + struct { + atomic64_t vp_signaled_count; + struct { + u64 intercept_suspend: 1; + u64 root_sched_blocked: 1; /* root scheduler only */ + u64 root_sched_dispatched: 1; /* root scheduler only */ + u64 reserved: 61; + } flags; + unsigned int kicked_by_hv; + wait_queue_head_t vp_suspend_queue; + } run; +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *vp_stats_dentry; +#endif +}; + +#define vp_fmt(fmt) "p%lluvp%u: " fmt +#define vp_devprintk(level, v, fmt, ...) \ +do { \ + const struct mshv_vp *__vp = (v); \ + const struct mshv_partition *__pt = __vp->vp_partition; \ + dev_##level(__pt->pt_module_dev, vp_fmt(fmt), __pt->pt_id, \ + __vp->vp_index, ##__VA_ARGS__); \ +} while (0) +#define vp_emerg(v, fmt, ...) vp_devprintk(emerg, v, fmt, ##__VA_ARGS__) +#define vp_crit(v, fmt, ...) vp_devprintk(crit, v, fmt, ##__VA_ARGS__) +#define vp_alert(v, fmt, ...) vp_devprintk(alert, v, fmt, ##__VA_ARGS__) +#define vp_err(v, fmt, ...) vp_devprintk(err, v, fmt, ##__VA_ARGS__) +#define vp_warn(v, fmt, ...) vp_devprintk(warn, v, fmt, ##__VA_ARGS__) +#define vp_notice(v, fmt, ...) vp_devprintk(notice, v, fmt, ##__VA_ARGS__) +#define vp_info(v, fmt, ...) vp_devprintk(info, v, fmt, ##__VA_ARGS__) +#define vp_dbg(v, fmt, ...) vp_devprintk(dbg, v, fmt, ##__VA_ARGS__) + +enum mshv_region_type { + MSHV_REGION_TYPE_MEM_PINNED, + MSHV_REGION_TYPE_MEM_MOVABLE, + MSHV_REGION_TYPE_MMIO +}; + +struct mshv_mem_region { + struct hlist_node hnode; + struct kref mreg_refcount; + u64 nr_pages; + u64 start_gfn; + u64 start_uaddr; + u32 hv_map_flags; + struct mshv_partition *partition; + enum mshv_region_type mreg_type; + struct mmu_interval_notifier mreg_mni; + struct mutex mreg_mutex; /* protects region pages remapping */ + struct page *mreg_pages[]; +}; + +struct mshv_irq_ack_notifier { + struct hlist_node link; + unsigned int irq_ack_gsi; + void (*irq_acked)(struct mshv_irq_ack_notifier *mian); +}; + +struct mshv_partition { + struct device *pt_module_dev; + + struct hlist_node pt_hnode; + u64 pt_id; + refcount_t pt_ref_count; + struct mutex pt_mutex; + + spinlock_t pt_mem_regions_lock; + struct hlist_head pt_mem_regions; // not ordered + + u32 pt_vp_count; + struct mshv_vp *pt_vp_array[MSHV_MAX_VPS]; + + struct mutex pt_irq_lock; + struct srcu_struct pt_irq_srcu; + struct hlist_head irq_ack_notifier_list; + + struct hlist_head pt_devices; + + /* + * MSHV does not support more than one async hypercall in flight + * for a single partition. Thus, it is okay to define per partition + * async hypercall status. + */ + struct completion async_hypercall; + u64 async_hypercall_status; + + spinlock_t pt_irqfds_lock; + struct hlist_head pt_irqfds_list; + struct mutex irqfds_resampler_lock; + struct hlist_head irqfds_resampler_list; + + struct hlist_head ioeventfds_list; + + struct mshv_girq_routing_table __rcu *pt_girq_tbl; + u64 isolation_type; + bool import_completed; + bool pt_initialized; +#if IS_ENABLED(CONFIG_DEBUG_FS) + struct dentry *pt_stats_dentry; + struct dentry *pt_vp_dentry; +#endif +}; + +#define pt_fmt(fmt) "p%llu: " fmt +#define pt_devprintk(level, p, fmt, ...) \ +do { \ + const struct mshv_partition *__pt = (p); \ + dev_##level(__pt->pt_module_dev, pt_fmt(fmt), __pt->pt_id, \ + ##__VA_ARGS__); \ +} while (0) +#define pt_emerg(p, fmt, ...) pt_devprintk(emerg, p, fmt, ##__VA_ARGS__) +#define pt_crit(p, fmt, ...) pt_devprintk(crit, p, fmt, ##__VA_ARGS__) +#define pt_alert(p, fmt, ...) pt_devprintk(alert, p, fmt, ##__VA_ARGS__) +#define pt_err(p, fmt, ...) pt_devprintk(err, p, fmt, ##__VA_ARGS__) +#define pt_warn(p, fmt, ...) pt_devprintk(warn, p, fmt, ##__VA_ARGS__) +#define pt_notice(p, fmt, ...) pt_devprintk(notice, p, fmt, ##__VA_ARGS__) +#define pt_info(p, fmt, ...) pt_devprintk(info, p, fmt, ##__VA_ARGS__) +#define pt_dbg(p, fmt, ...) pt_devprintk(dbg, p, fmt, ##__VA_ARGS__) + +struct mshv_lapic_irq { + u32 lapic_vector; + u64 lapic_apic_id; + union hv_interrupt_control lapic_control; +}; + +#define MSHV_MAX_GUEST_IRQS 4096 + +/* representation of one guest irq entry, either msi or legacy */ +struct mshv_guest_irq_ent { + u32 girq_entry_valid; /* vfio looks at this */ + u32 guest_irq_num; /* a unique number for each irq */ + u32 girq_addr_lo; /* guest irq msi address info */ + u32 girq_addr_hi; + u32 girq_irq_data; /* idt vector in some cases */ +}; + +struct mshv_girq_routing_table { + u32 num_rt_entries; + struct mshv_guest_irq_ent mshv_girq_info_tbl[]; +}; + +struct hv_synic_pages { + struct hv_message_page *hyp_synic_message_page; + struct hv_synic_event_flags_page *synic_event_flags_page; + struct hv_synic_event_ring_page *synic_event_ring_page; +}; + +struct mshv_root { + spinlock_t pt_ht_lock; + DECLARE_HASHTABLE(pt_htable, MSHV_PARTITIONS_HASH_BITS); + struct hv_partition_property_vmm_capabilities vmm_caps; +}; + +/* + * Callback for doorbell events. + * NOTE: This is called in interrupt context. Callback + * should defer slow and sleeping logic to later. + */ +typedef void (*doorbell_cb_t) (int doorbell_id, void *); + +/* + * port table information + */ +struct port_table_info { + struct rcu_head portbl_rcu; + enum hv_port_type hv_port_type; + union { + struct { + u64 reserved[2]; + } hv_port_message; + struct { + u64 reserved[2]; + } hv_port_event; + struct { + u64 reserved[2]; + } hv_port_monitor; + struct { + doorbell_cb_t doorbell_cb; + void *data; + } hv_port_doorbell; + }; +}; + +int mshv_update_routing_table(struct mshv_partition *partition, + const struct mshv_user_irq_entry *entries, + unsigned int numents); +void mshv_free_routing_table(struct mshv_partition *partition); + +struct mshv_guest_irq_ent mshv_ret_girq_entry(struct mshv_partition *partition, + u32 irq_num); + +void mshv_copy_girq_info(struct mshv_guest_irq_ent *src_irq, + struct mshv_lapic_irq *dest_irq); + +void mshv_irqfd_routing_update(struct mshv_partition *partition); + +void mshv_port_table_fini(void); +int mshv_portid_alloc(struct port_table_info *info); +int mshv_portid_lookup(int port_id, struct port_table_info *info); +void mshv_portid_free(int port_id); + +int mshv_register_doorbell(u64 partition_id, doorbell_cb_t doorbell_cb, + void *data, u64 gpa, u64 val, u64 flags); +void mshv_unregister_doorbell(u64 partition_id, int doorbell_portid); + +void mshv_isr(void); +int mshv_synic_init(struct device *dev); +void mshv_synic_exit(void); + +static inline bool mshv_partition_encrypted(struct mshv_partition *partition) +{ + return partition->isolation_type == HV_PARTITION_ISOLATION_TYPE_SNP; +} + +struct mshv_partition *mshv_partition_get(struct mshv_partition *partition); +void mshv_partition_put(struct mshv_partition *partition); +struct mshv_partition *mshv_partition_find(u64 partition_id) __must_hold(RCU); + +static inline bool is_l1vh_parent(u64 partition_id) +{ + return hv_l1vh_partition() && (partition_id == HV_PARTITION_ID_SELF); +} + +int mshv_vp_stats_map(u64 partition_id, u32 vp_index, + struct hv_stats_page **stats_pages); +void mshv_vp_stats_unmap(u64 partition_id, u32 vp_index, + struct hv_stats_page **stats_pages); + +/* hypercalls */ + +int hv_call_withdraw_memory(u64 count, int node, u64 partition_id); +int hv_call_create_partition(u64 flags, + struct hv_partition_creation_properties creation_properties, + union hv_partition_isolation_properties isolation_properties, + u64 *partition_id); +int hv_call_initialize_partition(u64 partition_id); +int hv_call_finalize_partition(u64 partition_id); +int hv_call_delete_partition(u64 partition_id); +int hv_call_map_mmio_pages(u64 partition_id, u64 gfn, u64 mmio_spa, u64 numpgs); +int hv_call_map_gpa_pages(u64 partition_id, u64 gpa_target, u64 page_count, + u32 flags, struct page **pages); +int hv_call_unmap_gpa_pages(u64 partition_id, u64 gpa_target, u64 page_count, + u32 flags); +int hv_call_delete_vp(u64 partition_id, u32 vp_index); +int hv_call_assert_virtual_interrupt(u64 partition_id, u32 vector, + u64 dest_addr, + union hv_interrupt_control control); +int hv_call_clear_virtual_interrupt(u64 partition_id); +int hv_call_get_gpa_access_states(u64 partition_id, u32 count, u64 gpa_base_pfn, + union hv_gpa_page_access_state_flags state_flags, + int *written_total, + union hv_gpa_page_access_state *states); +int hv_call_get_vp_state(u32 vp_index, u64 partition_id, + struct hv_vp_state_data state_data, + /* Choose between pages and ret_output */ + u64 page_count, struct page **pages, + union hv_output_get_vp_state *ret_output); +int hv_call_set_vp_state(u32 vp_index, u64 partition_id, + /* Choose between pages and bytes */ + struct hv_vp_state_data state_data, u64 page_count, + struct page **pages, u32 num_bytes, u8 *bytes); +int hv_map_vp_state_page(u64 partition_id, u32 vp_index, u32 type, + union hv_input_vtl input_vtl, + struct page **state_page); +int hv_unmap_vp_state_page(u64 partition_id, u32 vp_index, u32 type, + struct page *state_page, + union hv_input_vtl input_vtl); +int hv_call_create_port(u64 port_partition_id, union hv_port_id port_id, + u64 connection_partition_id, struct hv_port_info *port_info, + u8 port_vtl, u8 min_connection_vtl, int node); +int hv_call_delete_port(u64 port_partition_id, union hv_port_id port_id); +int hv_call_connect_port(u64 port_partition_id, union hv_port_id port_id, + u64 connection_partition_id, + union hv_connection_id connection_id, + struct hv_connection_info *connection_info, + u8 connection_vtl, int node); +int hv_call_disconnect_port(u64 connection_partition_id, + union hv_connection_id connection_id); +int hv_call_notify_port_ring_empty(u32 sint_index); +int hv_map_stats_page(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity, + struct hv_stats_page **addr); +int hv_unmap_stats_page(enum hv_stats_object_type type, + struct hv_stats_page *page_addr, + const union hv_stats_object_identity *identity); +int hv_call_modify_spa_host_access(u64 partition_id, struct page **pages, + u64 page_struct_count, u32 host_access, + u32 flags, u8 acquire); +int hv_call_get_partition_property_ex(u64 partition_id, u64 property_code, u64 arg, + void *property_value, size_t property_value_sz); + +#if IS_ENABLED(CONFIG_DEBUG_FS) +int __init mshv_debugfs_init(void); +void mshv_debugfs_exit(void); + +int mshv_debugfs_partition_create(struct mshv_partition *partition); +void mshv_debugfs_partition_remove(struct mshv_partition *partition); +int mshv_debugfs_vp_create(struct mshv_vp *vp); +void mshv_debugfs_vp_remove(struct mshv_vp *vp); +#else +static inline int __init mshv_debugfs_init(void) +{ + return 0; +} +static inline void mshv_debugfs_exit(void) { } + +static inline int mshv_debugfs_partition_create(struct mshv_partition *partition) +{ + return 0; +} +static inline void mshv_debugfs_partition_remove(struct mshv_partition *partition) { } +static inline int mshv_debugfs_vp_create(struct mshv_vp *vp) +{ + return 0; +} +static inline void mshv_debugfs_vp_remove(struct mshv_vp *vp) { } +#endif + +extern struct mshv_root mshv_root; +extern enum hv_scheduler_type hv_scheduler_type; +extern u8 * __percpu *hv_synic_eventring_tail; + +struct mshv_mem_region *mshv_region_create(u64 guest_pfn, u64 nr_pages, + u64 uaddr, u32 flags); +int mshv_region_share(struct mshv_mem_region *region); +int mshv_region_unshare(struct mshv_mem_region *region); +int mshv_region_map(struct mshv_mem_region *region); +void mshv_region_invalidate(struct mshv_mem_region *region); +int mshv_region_pin(struct mshv_mem_region *region); +void mshv_region_put(struct mshv_mem_region *region); +int mshv_region_get(struct mshv_mem_region *region); +bool mshv_region_handle_gfn_fault(struct mshv_mem_region *region, u64 gfn); +void mshv_region_movable_fini(struct mshv_mem_region *region); +bool mshv_region_movable_init(struct mshv_mem_region *region); + +#endif /* _MSHV_ROOT_H_ */ diff --git a/drivers/hv/mshv_root_hv_call.c b/drivers/hv/mshv_root_hv_call.c new file mode 100644 index 000000000..cb55d4d4b --- /dev/null +++ b/drivers/hv/mshv_root_hv_call.c @@ -0,0 +1,1074 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023, Microsoft Corporation. + * + * Hypercall helper functions used by the mshv_root module. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/export.h> +#include <asm/mshyperv.h> + +#include "mshv_root.h" + +/* Determined empirically */ +#define HV_INIT_PARTITION_DEPOSIT_PAGES 208 +#define HV_MAP_GPA_DEPOSIT_PAGES 256 +#define HV_UMAP_GPA_PAGES 512 + +#define HV_PAGE_COUNT_2M_ALIGNED(pg_count) (!((pg_count) & (0x200 - 1))) + +#define HV_WITHDRAW_BATCH_SIZE (HV_HYP_PAGE_SIZE / sizeof(u64)) +#define HV_MAP_GPA_BATCH_SIZE \ + ((HV_HYP_PAGE_SIZE - sizeof(struct hv_input_map_gpa_pages)) \ + / sizeof(u64)) +#define HV_GET_VP_STATE_BATCH_SIZE \ + ((HV_HYP_PAGE_SIZE - sizeof(struct hv_input_get_vp_state)) \ + / sizeof(u64)) +#define HV_SET_VP_STATE_BATCH_SIZE \ + ((HV_HYP_PAGE_SIZE - sizeof(struct hv_input_set_vp_state)) \ + / sizeof(u64)) +#define HV_GET_GPA_ACCESS_STATES_BATCH_SIZE \ + ((HV_HYP_PAGE_SIZE - sizeof(union hv_gpa_page_access_state)) \ + / sizeof(union hv_gpa_page_access_state)) +#define HV_MODIFY_SPARSE_SPA_PAGE_HOST_ACCESS_MAX_PAGE_COUNT \ + ((HV_HYP_PAGE_SIZE - \ + sizeof(struct hv_input_modify_sparse_spa_page_host_access)) / \ + sizeof(u64)) + +int hv_call_withdraw_memory(u64 count, int node, u64 partition_id) +{ + struct hv_input_withdraw_memory *input_page; + struct hv_output_withdraw_memory *output_page; + struct page *page; + u16 completed; + u64 status, withdrawn = 0; + int i; + unsigned long flags; + + page = alloc_page(GFP_KERNEL); + if (!page) + return -ENOMEM; + output_page = page_address(page); + + while (withdrawn < count) { + local_irq_save(flags); + + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + + memset(input_page, 0, sizeof(*input_page)); + input_page->partition_id = partition_id; + status = hv_do_rep_hypercall(HVCALL_WITHDRAW_MEMORY, + min(count - withdrawn, HV_WITHDRAW_BATCH_SIZE), + 0, input_page, output_page); + + local_irq_restore(flags); + + completed = hv_repcomp(status); + + for (i = 0; i < completed; i++) + __free_page(pfn_to_page(output_page->gpa_page_list[i])); + + if (!hv_result_success(status)) { + if (hv_result(status) == HV_STATUS_NO_RESOURCES) + status = HV_STATUS_SUCCESS; + break; + } + + withdrawn += completed; + } + free_page((unsigned long)output_page); + + trace_mshv_hvcall_withdraw_memory(partition_id, withdrawn, status); + + return hv_result_to_errno(status); +} + +int hv_call_create_partition(u64 flags, + struct hv_partition_creation_properties creation_properties, + union hv_partition_isolation_properties isolation_properties, + u64 *partition_id) +{ + struct hv_input_create_partition *input; + struct hv_output_create_partition *output; + u64 status; + int ret; + unsigned long irq_flags; + + do { + local_irq_save(irq_flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + memset(input, 0, sizeof(*input)); + input->flags = flags; + input->compatibility_version = HV_COMPATIBILITY_21_H2; + + memcpy(&input->partition_creation_properties, &creation_properties, + sizeof(creation_properties)); + + memcpy(&input->isolation_properties, &isolation_properties, + sizeof(isolation_properties)); + + status = hv_do_hypercall(HVCALL_CREATE_PARTITION, + input, output); + + if (!hv_result_needs_memory(status)) { + if (hv_result_success(status)) + *partition_id = output->partition_id; + local_irq_restore(irq_flags); + ret = hv_result_to_errno(status); + break; + } + local_irq_restore(irq_flags); + ret = hv_deposit_memory(hv_current_partition_id, status); + } while (!ret); + + trace_mshv_hvcall_create_partition(flags, ret ? ret : *partition_id); + + return ret; +} + +int hv_call_initialize_partition(u64 partition_id) +{ + struct hv_input_initialize_partition input; + u64 status; + int ret; + + input.partition_id = partition_id; + + ret = hv_call_deposit_pages(NUMA_NO_NODE, partition_id, + HV_INIT_PARTITION_DEPOSIT_PAGES); + if (ret) + return ret; + + do { + status = hv_do_fast_hypercall8(HVCALL_INITIALIZE_PARTITION, + *(u64 *)&input); + + if (!hv_result_needs_memory(status)) { + ret = hv_result_to_errno(status); + break; + } + ret = hv_deposit_memory(partition_id, status); + } while (!ret); + + trace_mshv_hvcall_initialize_partition(partition_id, status); + + return ret; +} + +int hv_call_finalize_partition(u64 partition_id) +{ + struct hv_input_finalize_partition input; + u64 status; + + input.partition_id = partition_id; + status = hv_do_fast_hypercall8(HVCALL_FINALIZE_PARTITION, + *(u64 *)&input); + + trace_mshv_hvcall_finalize_partition(partition_id, status); + + return hv_result_to_errno(status); +} + +int hv_call_delete_partition(u64 partition_id) +{ + struct hv_input_delete_partition input; + u64 status; + + input.partition_id = partition_id; + status = hv_do_fast_hypercall8(HVCALL_DELETE_PARTITION, *(u64 *)&input); + + trace_mshv_hvcall_delete_partition(partition_id, status); + + return hv_result_to_errno(status); +} + +/* Ask the hypervisor to map guest ram pages or the guest mmio space */ +static int hv_do_map_gpa_hcall(u64 partition_id, u64 gfn, u64 page_struct_count, + u32 flags, struct page **pages, u64 mmio_spa) +{ + struct hv_input_map_gpa_pages *input_page; + u64 status, *pfnlist; + unsigned long irq_flags, large_shift = 0; + int ret = 0, done = 0; + u64 page_count = page_struct_count; + + if (page_count == 0 || (pages && mmio_spa)) + return -EINVAL; + + if (flags & HV_MAP_GPA_LARGE_PAGE) { + if (mmio_spa) + return -EINVAL; + + if (!HV_PAGE_COUNT_2M_ALIGNED(page_count)) + return -EINVAL; + + large_shift = HV_HYP_LARGE_PAGE_SHIFT - HV_HYP_PAGE_SHIFT; + page_count >>= large_shift; + } + + while (done < page_count) { + ulong i, completed, remain = page_count - done; + int rep_count = min(remain, HV_MAP_GPA_BATCH_SIZE); + + local_irq_save(irq_flags); + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + + input_page->target_partition_id = partition_id; + input_page->target_gpa_base = gfn + (done << large_shift); + input_page->map_flags = flags; + pfnlist = input_page->source_gpa_page_list; + + for (i = 0; i < rep_count; i++) + if (flags & HV_MAP_GPA_NO_ACCESS) { + pfnlist[i] = 0; + } else if (pages) { + u64 index = (done + i) << large_shift; + + if (index >= page_struct_count) { + ret = -EINVAL; + break; + } + pfnlist[i] = page_to_pfn(pages[index]); + } else { + pfnlist[i] = mmio_spa + done + i; + } + if (ret) + break; + + status = hv_do_rep_hypercall(HVCALL_MAP_GPA_PAGES, rep_count, 0, + input_page, NULL); + local_irq_restore(irq_flags); + + completed = hv_repcomp(status); + + if (hv_result_needs_memory(status)) { + ret = hv_call_deposit_pages(NUMA_NO_NODE, partition_id, + HV_MAP_GPA_DEPOSIT_PAGES); + if (ret) + break; + + } else if (!hv_result_success(status)) { + ret = hv_result_to_errno(status); + break; + } + + done += completed; + } + + if (ret && done) { + u32 unmap_flags = 0; + + if (flags & HV_MAP_GPA_LARGE_PAGE) + unmap_flags |= HV_UNMAP_GPA_LARGE_PAGE; + hv_call_unmap_gpa_pages(partition_id, gfn, done, unmap_flags); + } + + return ret; +} + +/* Ask the hypervisor to map guest ram pages */ +int hv_call_map_gpa_pages(u64 partition_id, u64 gpa_target, u64 page_count, + u32 flags, struct page **pages) +{ + return hv_do_map_gpa_hcall(partition_id, gpa_target, page_count, + flags, pages, 0); +} + +/* Ask the hypervisor to map guest mmio space */ +int hv_call_map_mmio_pages(u64 partition_id, u64 gfn, u64 mmio_spa, u64 numpgs) +{ + int i; + u32 flags = HV_MAP_GPA_READABLE | HV_MAP_GPA_WRITABLE | + HV_MAP_GPA_NOT_CACHED; + + for (i = 0; i < numpgs; i++) + if (page_is_ram(mmio_spa + i)) + return -EINVAL; + + return hv_do_map_gpa_hcall(partition_id, gfn, numpgs, flags, NULL, + mmio_spa); +} + +int hv_call_unmap_gpa_pages(u64 partition_id, u64 gfn, u64 page_count_4k, + u32 flags) +{ + struct hv_input_unmap_gpa_pages *input_page; + u64 status, page_count = page_count_4k; + unsigned long irq_flags, large_shift = 0; + int ret = 0, done = 0; + + if (page_count == 0) + return -EINVAL; + + if (flags & HV_UNMAP_GPA_LARGE_PAGE) { + if (!HV_PAGE_COUNT_2M_ALIGNED(page_count)) + return -EINVAL; + + large_shift = HV_HYP_LARGE_PAGE_SHIFT - HV_HYP_PAGE_SHIFT; + page_count >>= large_shift; + } + + while (done < page_count) { + ulong completed, remain = page_count - done; + int rep_count = min(remain, HV_UMAP_GPA_PAGES); + + local_irq_save(irq_flags); + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + + input_page->target_partition_id = partition_id; + input_page->target_gpa_base = gfn + (done << large_shift); + input_page->unmap_flags = flags; + status = hv_do_rep_hypercall(HVCALL_UNMAP_GPA_PAGES, rep_count, + 0, input_page, NULL); + local_irq_restore(irq_flags); + + completed = hv_repcomp(status); + if (!hv_result_success(status)) { + ret = hv_result_to_errno(status); + break; + } + + done += completed; + } + + return ret; +} + +int hv_call_get_gpa_access_states(u64 partition_id, u32 count, u64 gpa_base_pfn, + union hv_gpa_page_access_state_flags state_flags, + int *written_total, + union hv_gpa_page_access_state *states) +{ + struct hv_input_get_gpa_pages_access_state *input_page; + union hv_gpa_page_access_state *output_page; + int completed = 0; + unsigned long remaining = count; + int rep_count, i; + u64 status = 0; + unsigned long flags; + + *written_total = 0; + while (remaining) { + local_irq_save(flags); + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + output_page = *this_cpu_ptr(hyperv_pcpu_output_arg); + + input_page->partition_id = partition_id; + input_page->hv_gpa_page_number = gpa_base_pfn + *written_total; + input_page->flags = state_flags; + rep_count = min(remaining, HV_GET_GPA_ACCESS_STATES_BATCH_SIZE); + + status = hv_do_rep_hypercall(HVCALL_GET_GPA_PAGES_ACCESS_STATES, rep_count, + 0, input_page, output_page); + if (!hv_result_success(status)) { + local_irq_restore(flags); + break; + } + completed = hv_repcomp(status); + for (i = 0; i < completed; ++i) + states[i].as_uint8 = output_page[i].as_uint8; + + local_irq_restore(flags); + states += completed; + *written_total += completed; + remaining -= completed; + } + + return hv_result_to_errno(status); +} + +int hv_call_assert_virtual_interrupt(u64 partition_id, u32 vector, + u64 dest_addr, + union hv_interrupt_control control) +{ + struct hv_input_assert_virtual_interrupt *input; + unsigned long flags; + u64 status; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->partition_id = partition_id; + input->vector = vector; + /* + * NOTE: dest_addr only needs to be provided while asserting an + * interrupt on x86 platform + */ +#if IS_ENABLED(CONFIG_X86) + input->dest_addr = dest_addr; +#endif + input->control = control; + status = hv_do_hypercall(HVCALL_ASSERT_VIRTUAL_INTERRUPT, input, NULL); + local_irq_restore(flags); + + return hv_result_to_errno(status); +} + +int hv_call_delete_vp(u64 partition_id, u32 vp_index) +{ + union hv_input_delete_vp input = {}; + u64 status; + + input.partition_id = partition_id; + input.vp_index = vp_index; + + status = hv_do_fast_hypercall16(HVCALL_DELETE_VP, + input.as_uint64[0], input.as_uint64[1]); + + return hv_result_to_errno(status); +} +EXPORT_SYMBOL_GPL(hv_call_delete_vp); + +int hv_call_get_vp_state(u32 vp_index, u64 partition_id, + struct hv_vp_state_data state_data, + /* Choose between pages and ret_output */ + u64 page_count, struct page **pages, + union hv_output_get_vp_state *ret_output) +{ + struct hv_input_get_vp_state *input; + union hv_output_get_vp_state *output; + u64 status; + int i; + u64 control; + unsigned long flags; + int ret = 0; + + if (page_count > HV_GET_VP_STATE_BATCH_SIZE) + return -EINVAL; + + if (!page_count && !ret_output) + return -EINVAL; + + do { + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + memset(input, 0, sizeof(*input)); + memset(output, 0, sizeof(*output)); + + input->partition_id = partition_id; + input->vp_index = vp_index; + input->state_data = state_data; + for (i = 0; i < page_count; i++) + input->output_data_pfns[i] = page_to_pfn(pages[i]); + + control = (HVCALL_GET_VP_STATE) | + (page_count << HV_HYPERCALL_VARHEAD_OFFSET); + + status = hv_do_hypercall(control, input, output); + + if (!hv_result_needs_memory(status)) { + if (hv_result_success(status) && ret_output) + memcpy(ret_output, output, sizeof(*output)); + + local_irq_restore(flags); + ret = hv_result_to_errno(status); + break; + } + local_irq_restore(flags); + + ret = hv_deposit_memory(partition_id, status); + } while (!ret); + + return ret; +} + +int hv_call_set_vp_state(u32 vp_index, u64 partition_id, + /* Choose between pages and bytes */ + struct hv_vp_state_data state_data, u64 page_count, + struct page **pages, u32 num_bytes, u8 *bytes) +{ + struct hv_input_set_vp_state *input; + u64 status; + int i; + u64 control; + unsigned long flags; + int ret = 0; + u16 varhead_sz; + + if (page_count > HV_SET_VP_STATE_BATCH_SIZE) + return -EINVAL; + if (sizeof(*input) + num_bytes > HV_HYP_PAGE_SIZE) + return -EINVAL; + + if (num_bytes) + /* round up to 8 and divide by 8 */ + varhead_sz = (num_bytes + 7) >> 3; + else if (page_count) + varhead_sz = page_count; + else + return -EINVAL; + + do { + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + + input->partition_id = partition_id; + input->vp_index = vp_index; + input->state_data = state_data; + if (num_bytes) { + memcpy((u8 *)input->data, bytes, num_bytes); + } else { + for (i = 0; i < page_count; i++) + input->data[i].pfns = page_to_pfn(pages[i]); + } + + control = (HVCALL_SET_VP_STATE) | + (varhead_sz << HV_HYPERCALL_VARHEAD_OFFSET); + + status = hv_do_hypercall(control, input, NULL); + + if (!hv_result_needs_memory(status)) { + local_irq_restore(flags); + ret = hv_result_to_errno(status); + break; + } + local_irq_restore(flags); + + ret = hv_deposit_memory(partition_id, status); + } while (!ret); + + return ret; +} + +static int hv_call_map_vp_state_page(u64 partition_id, u32 vp_index, u32 type, + union hv_input_vtl input_vtl, + struct page **state_page) +{ + struct hv_input_map_vp_state_page *input; + struct hv_output_map_vp_state_page *output; + u64 status; + int ret; + unsigned long flags; + + do { + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + memset(input, 0, sizeof(*input)); + input->partition_id = partition_id; + input->vp_index = vp_index; + input->type = type; + input->input_vtl = input_vtl; + + if (*state_page) { + input->flags.map_location_provided = 1; + input->requested_map_location = + page_to_pfn(*state_page); + } + + status = hv_do_hypercall(HVCALL_MAP_VP_STATE_PAGE, input, + output); + + if (!hv_result_needs_memory(status)) { + if (hv_result_success(status)) + *state_page = pfn_to_page(output->map_location); + local_irq_restore(flags); + ret = hv_result_to_errno(status); + break; + } + + local_irq_restore(flags); + + ret = hv_deposit_memory(partition_id, status); + } while (!ret); + + trace_mshv_hvcall_map_vp_state_page(partition_id, vp_index, + type, status); + + return ret; +} + +static bool mshv_use_overlay_gpfn(void) +{ + return hv_l1vh_partition() && + mshv_root.vmm_caps.vmm_can_provide_overlay_gpfn; +} + +int hv_map_vp_state_page(u64 partition_id, u32 vp_index, u32 type, + union hv_input_vtl input_vtl, + struct page **state_page) +{ + int ret = 0; + struct page *allocated_page = NULL; + + if (mshv_use_overlay_gpfn()) { + allocated_page = alloc_page(GFP_KERNEL); + if (!allocated_page) + return -ENOMEM; + *state_page = allocated_page; + } else { + *state_page = NULL; + } + + ret = hv_call_map_vp_state_page(partition_id, vp_index, type, input_vtl, + state_page); + + if (ret && allocated_page) { + __free_page(allocated_page); + *state_page = NULL; + } + + return ret; +} + +static int hv_call_unmap_vp_state_page(u64 partition_id, u32 vp_index, u32 type, + union hv_input_vtl input_vtl) +{ + unsigned long flags; + u64 status; + struct hv_input_unmap_vp_state_page *input; + + local_irq_save(flags); + + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + + memset(input, 0, sizeof(*input)); + + input->partition_id = partition_id; + input->vp_index = vp_index; + input->type = type; + input->input_vtl = input_vtl; + + status = hv_do_hypercall(HVCALL_UNMAP_VP_STATE_PAGE, input, NULL); + + local_irq_restore(flags); + + return hv_result_to_errno(status); +} + +int hv_unmap_vp_state_page(u64 partition_id, u32 vp_index, u32 type, + struct page *state_page, union hv_input_vtl input_vtl) +{ + int ret = hv_call_unmap_vp_state_page(partition_id, vp_index, type, input_vtl); + + if (mshv_use_overlay_gpfn() && state_page) + __free_page(state_page); + + return ret; +} + +int hv_call_get_partition_property_ex(u64 partition_id, u64 property_code, + u64 arg, void *property_value, + size_t property_value_sz) +{ + u64 status; + unsigned long flags; + struct hv_input_get_partition_property_ex *input; + struct hv_output_get_partition_property_ex *output; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + memset(input, 0, sizeof(*input)); + input->partition_id = partition_id; + input->property_code = property_code; + input->arg = arg; + status = hv_do_hypercall(HVCALL_GET_PARTITION_PROPERTY_EX, input, output); + + if (!hv_result_success(status)) { + local_irq_restore(flags); + hv_status_debug(status, "\n"); + return hv_result_to_errno(status); + } + memcpy(property_value, &output->property_value, property_value_sz); + + local_irq_restore(flags); + + return 0; +} + +int +hv_call_clear_virtual_interrupt(u64 partition_id) +{ + int status; + + status = hv_do_fast_hypercall8(HVCALL_CLEAR_VIRTUAL_INTERRUPT, + partition_id); + + return hv_result_to_errno(status); +} + +int +hv_call_create_port(u64 port_partition_id, union hv_port_id port_id, + u64 connection_partition_id, + struct hv_port_info *port_info, + u8 port_vtl, u8 min_connection_vtl, int node) +{ + struct hv_input_create_port *input; + unsigned long flags; + int ret = 0; + int status; + + do { + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + + input->port_partition_id = port_partition_id; + input->port_id = port_id; + input->connection_partition_id = connection_partition_id; + input->port_info = *port_info; + input->port_vtl = port_vtl; + input->min_connection_vtl = min_connection_vtl; + input->proximity_domain_info = hv_numa_node_to_pxm_info(node); + status = hv_do_hypercall(HVCALL_CREATE_PORT, input, NULL); + local_irq_restore(flags); + if (hv_result_success(status)) + break; + + if (!hv_result_needs_memory(status)) { + ret = hv_result_to_errno(status); + break; + } + ret = hv_deposit_memory(port_partition_id, status); + } while (!ret); + + return ret; +} + +int +hv_call_delete_port(u64 port_partition_id, union hv_port_id port_id) +{ + union hv_input_delete_port input = { 0 }; + int status; + + input.port_partition_id = port_partition_id; + input.port_id = port_id; + status = hv_do_fast_hypercall16(HVCALL_DELETE_PORT, + input.as_uint64[0], + input.as_uint64[1]); + + return hv_result_to_errno(status); +} + +int +hv_call_connect_port(u64 port_partition_id, union hv_port_id port_id, + u64 connection_partition_id, + union hv_connection_id connection_id, + struct hv_connection_info *connection_info, + u8 connection_vtl, int node) +{ + struct hv_input_connect_port *input; + unsigned long flags; + int ret = 0, status; + + do { + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + memset(input, 0, sizeof(*input)); + input->port_partition_id = port_partition_id; + input->port_id = port_id; + input->connection_partition_id = connection_partition_id; + input->connection_id = connection_id; + input->connection_info = *connection_info; + input->connection_vtl = connection_vtl; + input->proximity_domain_info = hv_numa_node_to_pxm_info(node); + status = hv_do_hypercall(HVCALL_CONNECT_PORT, input, NULL); + + local_irq_restore(flags); + if (hv_result_success(status)) + break; + + if (!hv_result_needs_memory(status)) { + ret = hv_result_to_errno(status); + break; + } + ret = hv_deposit_memory(connection_partition_id, status); + } while (!ret); + + return ret; +} + +int +hv_call_disconnect_port(u64 connection_partition_id, + union hv_connection_id connection_id) +{ + union hv_input_disconnect_port input = { 0 }; + int status; + + input.connection_partition_id = connection_partition_id; + input.connection_id = connection_id; + input.is_doorbell = 1; + status = hv_do_fast_hypercall16(HVCALL_DISCONNECT_PORT, + input.as_uint64[0], + input.as_uint64[1]); + + return hv_result_to_errno(status); +} + +int +hv_call_notify_port_ring_empty(u32 sint_index) +{ + union hv_input_notify_port_ring_empty input = { 0 }; + int status; + + input.sint_index = sint_index; + status = hv_do_fast_hypercall8(HVCALL_NOTIFY_PORT_RING_EMPTY, + input.as_uint64); + + return hv_result_to_errno(status); +} + +/* + * Equivalent of hv_call_map_stats_page() for cases when the caller provides + * the map location. + * + * NOTE: This is a newer hypercall that always supports SELF and PARENT stats + * areas, unlike hv_call_map_stats_page(). + */ +static int hv_call_map_stats_page2(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity, + u64 map_location) +{ + unsigned long flags; + struct hv_input_map_stats_page2 *input; + u64 status; + int ret; + + if (!map_location || !mshv_use_overlay_gpfn()) + return -EINVAL; + + do { + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + + memset(input, 0, sizeof(*input)); + input->type = type; + input->identity = *identity; + input->map_location = map_location; + + status = hv_do_hypercall(HVCALL_MAP_STATS_PAGE2, input, NULL); + + local_irq_restore(flags); + + ret = hv_result_to_errno(status); + + if (!ret) + break; + + if (!hv_result_needs_memory(status)) { + hv_status_debug(status, "\n"); + break; + } + + ret = hv_deposit_memory(hv_current_partition_id, status); + } while (!ret); + + return ret; +} + +static int +hv_stats_get_area_type(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity) +{ + switch (type) { + case HV_STATS_OBJECT_HYPERVISOR: + return identity->hv.stats_area_type; + case HV_STATS_OBJECT_LOGICAL_PROCESSOR: + return identity->lp.stats_area_type; + case HV_STATS_OBJECT_PARTITION: + return identity->partition.stats_area_type; + case HV_STATS_OBJECT_VP: + return identity->vp.stats_area_type; + } + + return -EINVAL; +} + +/* + * Map a stats page, where the page location is provided by the hypervisor. + * + * NOTE: The concept of separate SELF and PARENT stats areas does not exist on + * older hypervisor versions. All the available stats information can be found + * on the SELF page. When attempting to map the PARENT area on a hypervisor + * that doesn't support it, return "success" but with a NULL address. The + * caller should check for this case and instead fallback to the SELF area + * alone. + */ +static int +hv_call_map_stats_page(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity, + struct hv_stats_page **addr) +{ + unsigned long flags; + struct hv_input_map_stats_page *input; + struct hv_output_map_stats_page *output; + u64 status, pfn; + int ret = 0; + + do { + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + memset(input, 0, sizeof(*input)); + input->type = type; + input->identity = *identity; + + status = hv_do_hypercall(HVCALL_MAP_STATS_PAGE, input, output); + pfn = output->map_location; + + local_irq_restore(flags); + + if (!hv_result_needs_memory(status)) { + if (hv_result_success(status)) + break; + + if (hv_stats_get_area_type(type, identity) == HV_STATS_AREA_PARENT && + hv_result(status) == HV_STATUS_INVALID_PARAMETER) { + *addr = NULL; + return 0; + } + + hv_status_debug(status, "\n"); + return hv_result_to_errno(status); + } + + ret = hv_deposit_memory(hv_current_partition_id, status); + if (ret) + return ret; + } while (!ret); + + *addr = page_address(pfn_to_page(pfn)); + + return ret; +} + +int hv_map_stats_page(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity, + struct hv_stats_page **addr) +{ + int ret; + struct page *allocated_page = NULL; + + if (!addr) + return -EINVAL; + + if (mshv_use_overlay_gpfn()) { + allocated_page = alloc_page(GFP_KERNEL); + if (!allocated_page) + return -ENOMEM; + + ret = hv_call_map_stats_page2(type, identity, + page_to_pfn(allocated_page)); + *addr = page_address(allocated_page); + } else { + ret = hv_call_map_stats_page(type, identity, addr); + } + + if (ret && allocated_page) { + __free_page(allocated_page); + *addr = NULL; + } + + return ret; +} + +static int hv_call_unmap_stats_page(enum hv_stats_object_type type, + const union hv_stats_object_identity *identity) +{ + unsigned long flags; + struct hv_input_unmap_stats_page *input; + u64 status; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + + memset(input, 0, sizeof(*input)); + input->type = type; + input->identity = *identity; + + status = hv_do_hypercall(HVCALL_UNMAP_STATS_PAGE, input, NULL); + local_irq_restore(flags); + + return hv_result_to_errno(status); +} + +int hv_unmap_stats_page(enum hv_stats_object_type type, + struct hv_stats_page *page_addr, + const union hv_stats_object_identity *identity) +{ + int ret; + + ret = hv_call_unmap_stats_page(type, identity); + + if (mshv_use_overlay_gpfn() && page_addr) + __free_page(virt_to_page(page_addr)); + + return ret; +} + +int hv_call_modify_spa_host_access(u64 partition_id, struct page **pages, + u64 page_struct_count, u32 host_access, + u32 flags, u8 acquire) +{ + struct hv_input_modify_sparse_spa_page_host_access *input_page; + u64 status; + int done = 0; + unsigned long irq_flags, large_shift = 0; + u64 page_count = page_struct_count; + u16 code = acquire ? HVCALL_ACQUIRE_SPARSE_SPA_PAGE_HOST_ACCESS : + HVCALL_RELEASE_SPARSE_SPA_PAGE_HOST_ACCESS; + + if (page_count == 0) + return -EINVAL; + + if (flags & HV_MODIFY_SPA_PAGE_HOST_ACCESS_LARGE_PAGE) { + if (!HV_PAGE_COUNT_2M_ALIGNED(page_count)) + return -EINVAL; + large_shift = HV_HYP_LARGE_PAGE_SHIFT - HV_HYP_PAGE_SHIFT; + page_count >>= large_shift; + } + + while (done < page_count) { + ulong i, completed, remain = page_count - done; + int rep_count = min(remain, + HV_MODIFY_SPARSE_SPA_PAGE_HOST_ACCESS_MAX_PAGE_COUNT); + + local_irq_save(irq_flags); + input_page = *this_cpu_ptr(hyperv_pcpu_input_arg); + + memset(input_page, 0, sizeof(*input_page)); + /* Only set the partition id if you are making the pages + * exclusive + */ + if (flags & HV_MODIFY_SPA_PAGE_HOST_ACCESS_MAKE_EXCLUSIVE) + input_page->partition_id = partition_id; + input_page->flags = flags; + input_page->host_access = host_access; + + for (i = 0; i < rep_count; i++) { + u64 index = (done + i) << large_shift; + + if (index >= page_struct_count) + return -EINVAL; + + input_page->spa_page_list[i] = + page_to_pfn(pages[index]); + } + + status = hv_do_rep_hypercall(code, rep_count, 0, input_page, + NULL); + local_irq_restore(irq_flags); + + completed = hv_repcomp(status); + + if (!hv_result_success(status)) + return hv_result_to_errno(status); + + done += completed; + } + + return 0; +} diff --git a/drivers/hv/mshv_root_main.c b/drivers/hv/mshv_root_main.c new file mode 100644 index 000000000..146726cc4 --- /dev/null +++ b/drivers/hv/mshv_root_main.c @@ -0,0 +1,2417 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2024, Microsoft Corporation. + * + * The main part of the mshv_root module, providing APIs to create + * and manage guest partitions. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/entry-virt.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/fs.h> +#include <linux/miscdevice.h> +#include <linux/slab.h> +#include <linux/file.h> +#include <linux/anon_inodes.h> +#include <linux/mm.h> +#include <linux/io.h> +#include <linux/cpuhotplug.h> +#include <linux/random.h> +#include <asm/mshyperv.h> +#include <linux/hyperv.h> +#include <linux/notifier.h> +#include <linux/reboot.h> +#include <linux/kexec.h> +#include <linux/page-flags.h> +#include <linux/crash_dump.h> +#include <linux/panic_notifier.h> +#include <linux/vmalloc.h> +#include <linux/rseq.h> + +#include "mshv_eventfd.h" +#include "mshv.h" +#include "mshv_root.h" + +MODULE_AUTHOR("Microsoft"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microsoft Hyper-V root partition VMM interface /dev/mshv"); + +/* HV_THREAD_COUNTER */ +#if defined(CONFIG_X86_64) +#define HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED 202 +#elif defined(CONFIG_ARM64) +#define HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED 95 +#endif + +struct mshv_root mshv_root; + +enum hv_scheduler_type hv_scheduler_type; + +/* Once we implement the fast extended hypercall ABI they can go away. */ +static void * __percpu *root_scheduler_input; +static void * __percpu *root_scheduler_output; + +static long mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); +static int mshv_dev_open(struct inode *inode, struct file *filp); +static int mshv_dev_release(struct inode *inode, struct file *filp); +static int mshv_vp_release(struct inode *inode, struct file *filp); +static long mshv_vp_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); +static int mshv_partition_release(struct inode *inode, struct file *filp); +static long mshv_partition_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg); +static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma); +static vm_fault_t mshv_vp_fault(struct vm_fault *vmf); +static int mshv_init_async_handler(struct mshv_partition *partition); +static void mshv_async_hvcall_handler(void *data, u64 *status); + +static const union hv_input_vtl input_vtl_zero; +static const union hv_input_vtl input_vtl_normal = { + .target_vtl = HV_NORMAL_VTL, + .use_target_vtl = 1, +}; + +static const struct vm_operations_struct mshv_vp_vm_ops = { + .fault = mshv_vp_fault, +}; + +static const struct file_operations mshv_vp_fops = { + .owner = THIS_MODULE, + .release = mshv_vp_release, + .unlocked_ioctl = mshv_vp_ioctl, + .llseek = noop_llseek, + .mmap = mshv_vp_mmap, +}; + +static const struct file_operations mshv_partition_fops = { + .owner = THIS_MODULE, + .release = mshv_partition_release, + .unlocked_ioctl = mshv_partition_ioctl, + .llseek = noop_llseek, +}; + +static const struct file_operations mshv_dev_fops = { + .owner = THIS_MODULE, + .open = mshv_dev_open, + .release = mshv_dev_release, + .unlocked_ioctl = mshv_dev_ioctl, + .llseek = noop_llseek, +}; + +static struct miscdevice mshv_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "mshv", + .fops = &mshv_dev_fops, + .mode = 0600, +}; + +/* + * Only allow hypercalls that have a u64 partition id as the first member of + * the input structure. + * These are sorted by value. + */ +static u16 mshv_passthru_hvcalls[] = { + HVCALL_GET_PARTITION_PROPERTY, + HVCALL_GET_PARTITION_PROPERTY_EX, + HVCALL_SET_PARTITION_PROPERTY, + HVCALL_INSTALL_INTERCEPT, + HVCALL_GET_VP_REGISTERS, + HVCALL_SET_VP_REGISTERS, + HVCALL_TRANSLATE_VIRTUAL_ADDRESS, + HVCALL_CLEAR_VIRTUAL_INTERRUPT, + HVCALL_REGISTER_INTERCEPT_RESULT, + HVCALL_ASSERT_VIRTUAL_INTERRUPT, + HVCALL_GET_GPA_PAGES_ACCESS_STATES, + HVCALL_SIGNAL_EVENT_DIRECT, + HVCALL_POST_MESSAGE_DIRECT, + HVCALL_GET_VP_CPUID_VALUES, +}; + +/* + * Only allow hypercalls that are safe to be called by the VMM with the host + * partition as target (i.e. HV_PARTITION_ID_SELF). Carefully audit that a + * hypercall cannot be misused by the VMM before adding it to this list. + */ +static u16 mshv_self_passthru_hvcalls[] = { + HVCALL_GET_PARTITION_PROPERTY, + HVCALL_GET_PARTITION_PROPERTY_EX, +}; + +static bool mshv_hvcall_is_async(u16 code) +{ + switch (code) { + case HVCALL_SET_PARTITION_PROPERTY: + return true; + default: + break; + } + return false; +} + +static bool mshv_passthru_hvcall_allowed(u16 code, u64 pt_id) +{ + int i; + int n = ARRAY_SIZE(mshv_passthru_hvcalls); + u16 *allowed_hvcalls = mshv_passthru_hvcalls; + + if (pt_id == HV_PARTITION_ID_SELF) { + n = ARRAY_SIZE(mshv_self_passthru_hvcalls); + allowed_hvcalls = mshv_self_passthru_hvcalls; + } + + for (i = 0; i < n; ++i) + if (allowed_hvcalls[i] == code) + return true; + + return false; +} + +static int mshv_ioctl_passthru_hvcall(struct mshv_partition *partition, + bool partition_locked, + void __user *user_args) +{ + u64 status; + int ret = 0; + bool is_async; + struct mshv_root_hvcall args; + struct page *page; + unsigned int pages_order; + void *input_pg = NULL; + void *output_pg = NULL; + u16 reps_completed; + u64 pt_id = partition ? partition->pt_id : HV_PARTITION_ID_SELF; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + if (args.status || !args.in_ptr || args.in_sz < sizeof(u64) || + mshv_field_nonzero(args, rsvd) || args.in_sz > HV_HYP_PAGE_SIZE) + return -EINVAL; + + if (args.out_ptr && (!args.out_sz || args.out_sz > HV_HYP_PAGE_SIZE)) + return -EINVAL; + + if (!mshv_passthru_hvcall_allowed(args.code, pt_id)) + return -EINVAL; + + is_async = mshv_hvcall_is_async(args.code); + if (is_async) { + /* async hypercalls can only be called from partition fd */ + if (!partition || !partition_locked) + return -EINVAL; + ret = mshv_init_async_handler(partition); + if (ret) + return ret; + } + + pages_order = args.out_ptr ? 1 : 0; + page = alloc_pages(GFP_KERNEL, pages_order); + if (!page) + return -ENOMEM; + input_pg = page_address(page); + + if (args.out_ptr) + output_pg = (char *)input_pg + PAGE_SIZE; + else + output_pg = NULL; + + if (copy_from_user(input_pg, (void __user *)args.in_ptr, + args.in_sz)) { + ret = -EFAULT; + goto free_pages_out; + } + + /* + * NOTE: This only works because all the allowed hypercalls' input + * structs begin with a u64 partition_id field. + */ + *(u64 *)input_pg = pt_id; + + reps_completed = 0; + do { + if (args.reps) { + status = hv_do_rep_hypercall_ex(args.code, args.reps, + 0, reps_completed, + input_pg, output_pg); + reps_completed = hv_repcomp(status); + } else { + status = hv_do_hypercall(args.code, input_pg, output_pg); + } + + if (hv_result(status) == HV_STATUS_CALL_PENDING) { + if (is_async) { + mshv_async_hvcall_handler(partition, &status); + } else { /* Paranoia check. This shouldn't happen! */ + ret = -EBADFD; + goto free_pages_out; + } + } + + if (hv_result_success(status)) + break; + + if (!hv_result_needs_memory(status)) + ret = hv_result_to_errno(status); + else + ret = hv_deposit_memory(pt_id, status); + } while (!ret); + + args.status = hv_result(status); + args.reps = reps_completed; + if (copy_to_user(user_args, &args, sizeof(args))) + ret = -EFAULT; + + if (!ret && output_pg && + copy_to_user((void __user *)args.out_ptr, output_pg, args.out_sz)) + ret = -EFAULT; + +free_pages_out: + free_pages((unsigned long)input_pg, pages_order); + + return ret; +} + +static inline bool is_ghcb_mapping_available(void) +{ +#if IS_ENABLED(CONFIG_X86_64) + return ms_hyperv.ext_features & HV_VP_GHCB_ROOT_MAPPING_AVAILABLE; +#else + return 0; +#endif +} + +static int mshv_get_vp_registers(u32 vp_index, u64 partition_id, u16 count, + struct hv_register_assoc *registers) +{ + return hv_call_get_vp_registers(vp_index, partition_id, + count, input_vtl_zero, registers); +} + +static int mshv_set_vp_registers(u32 vp_index, u64 partition_id, u16 count, + struct hv_register_assoc *registers) +{ + return hv_call_set_vp_registers(vp_index, partition_id, + count, input_vtl_zero, registers); +} + +/* + * Explicit guest vCPU suspend is asynchronous by nature (as it is requested by + * dom0 vCPU for guest vCPU) and thus it can race with "intercept" suspend, + * done by the hypervisor. + * "Intercept" suspend leads to asynchronous message delivery to dom0 which + * should be awaited to keep the VP loop consistent (i.e. no message pending + * upon VP resume). + * VP intercept suspend can't be done when the VP is explicitly suspended + * already, and thus can be only two possible race scenarios: + * 1. implicit suspend bit set -> explicit suspend bit set -> message sent + * 2. implicit suspend bit set -> message sent -> explicit suspend bit set + * Checking for implicit suspend bit set after explicit suspend request has + * succeeded in either case allows us to reliably identify, if there is a + * message to receive and deliver to VMM. + */ +static int +mshv_suspend_vp(const struct mshv_vp *vp, bool *message_in_flight) +{ + struct hv_register_assoc explicit_suspend = { + .name = HV_REGISTER_EXPLICIT_SUSPEND + }; + struct hv_register_assoc intercept_suspend = { + .name = HV_REGISTER_INTERCEPT_SUSPEND + }; + union hv_explicit_suspend_register *es = + &explicit_suspend.value.explicit_suspend; + union hv_intercept_suspend_register *is = + &intercept_suspend.value.intercept_suspend; + int ret; + + es->suspended = 1; + + ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id, + 1, &explicit_suspend); + if (ret) { + vp_err(vp, "Failed to explicitly suspend vCPU\n"); + return ret; + } + + ret = mshv_get_vp_registers(vp->vp_index, vp->vp_partition->pt_id, + 1, &intercept_suspend); + if (ret) { + vp_err(vp, "Failed to get intercept suspend state\n"); + return ret; + } + + *message_in_flight = is->suspended; + + return 0; +} + +/* + * This function is used when VPs are scheduled by the hypervisor's + * scheduler. + * + * Caller has to make sure the registers contain cleared + * HV_REGISTER_INTERCEPT_SUSPEND and HV_REGISTER_EXPLICIT_SUSPEND registers + * exactly in this order (the hypervisor clears them sequentially) to avoid + * potential invalid clearing a newly arrived HV_REGISTER_INTERCEPT_SUSPEND + * after VP is released from HV_REGISTER_EXPLICIT_SUSPEND in case of the + * opposite order. + */ +static long mshv_run_vp_with_hyp_scheduler(struct mshv_vp *vp) +{ + long ret; + struct hv_register_assoc suspend_regs[2] = { + { .name = HV_REGISTER_INTERCEPT_SUSPEND }, + { .name = HV_REGISTER_EXPLICIT_SUSPEND } + }; + size_t count = ARRAY_SIZE(suspend_regs); + + /* Resume VP execution */ + ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id, + count, suspend_regs); + if (ret) { + vp_err(vp, "Failed to resume vp execution. %lx\n", ret); + return ret; + } + + ret = wait_event_interruptible(vp->run.vp_suspend_queue, + vp->run.kicked_by_hv == 1); + if (ret) { + bool message_in_flight; + + /* + * Otherwise the waiting was interrupted by a signal: suspend + * the vCPU explicitly and copy message in flight (if any). + */ + ret = mshv_suspend_vp(vp, &message_in_flight); + if (ret) + return ret; + + /* Return if no message in flight */ + if (!message_in_flight) + return -EINTR; + + /* Wait for the message in flight. */ + wait_event(vp->run.vp_suspend_queue, vp->run.kicked_by_hv == 1); + } + + /* + * Reset the flag to make the wait_event call above work + * next time. + */ + vp->run.kicked_by_hv = 0; + + return 0; +} + +static int +mshv_vp_dispatch(struct mshv_vp *vp, u32 flags, + struct hv_output_dispatch_vp *res) +{ + struct hv_input_dispatch_vp *input; + struct hv_output_dispatch_vp *output; + u64 status; + + preempt_disable(); + input = *this_cpu_ptr(root_scheduler_input); + output = *this_cpu_ptr(root_scheduler_output); + + memset(input, 0, sizeof(*input)); + memset(output, 0, sizeof(*output)); + + input->partition_id = vp->vp_partition->pt_id; + input->vp_index = vp->vp_index; + input->time_slice = 0; /* Run forever until something happens */ + input->spec_ctrl = 0; /* TODO: set sensible flags */ + input->flags = flags; + + vp->run.flags.root_sched_dispatched = 1; + status = hv_do_hypercall(HVCALL_DISPATCH_VP, input, output); + vp->run.flags.root_sched_dispatched = 0; + + trace_mshv_hvcall_dispatch_vp(vp->vp_partition->pt_id, + vp->vp_index, flags, + output->dispatch_state, + output->dispatch_event, +#if defined(CONFIG_X86_64) + vp->vp_register_page->interrupt_vectors.as_uint64, +#else + 0, +#endif + status); + + *res = *output; + preempt_enable(); + + if (!hv_result_success(status)) + vp_err(vp, "%s: status %s\n", __func__, + hv_result_to_string(status)); + + return hv_result_to_errno(status); +} + +static int +mshv_vp_clear_explicit_suspend(struct mshv_vp *vp) +{ + struct hv_register_assoc explicit_suspend = { + .name = HV_REGISTER_EXPLICIT_SUSPEND, + .value.explicit_suspend.suspended = 0, + }; + int ret; + + ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id, + 1, &explicit_suspend); + + trace_mshv_vp_clear_explicit_suspend(vp->vp_partition->pt_id, + vp->vp_index, ret); + + if (ret) + vp_err(vp, "Failed to unsuspend\n"); + + return ret; +} + +#if IS_ENABLED(CONFIG_X86_64) +static u64 mshv_vp_interrupt_pending(struct mshv_vp *vp) +{ + if (!vp->vp_register_page) + return 0; + return vp->vp_register_page->interrupt_vectors.as_uint64; +} +#else +static u64 mshv_vp_interrupt_pending(struct mshv_vp *vp) +{ + return 0; +} +#endif + +static bool mshv_vp_dispatch_thread_blocked(struct mshv_vp *vp) +{ + struct hv_stats_page **stats = vp->vp_stats_pages; + u64 *self_vp_cntrs = stats[HV_STATS_AREA_SELF]->data; + u64 *parent_vp_cntrs = stats[HV_STATS_AREA_PARENT]->data; + + return parent_vp_cntrs[HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED] || + self_vp_cntrs[HV_VP_COUNTER_ROOT_DISPATCH_THREAD_BLOCKED]; +} + +static int +mshv_vp_wait_for_hv_kick(struct mshv_vp *vp) +{ + int ret; + + ret = wait_event_interruptible(vp->run.vp_suspend_queue, + (vp->run.kicked_by_hv == 1 && + !mshv_vp_dispatch_thread_blocked(vp)) || + mshv_vp_interrupt_pending(vp)); + if (ret) + return -EINTR; + + trace_mshv_vp_wait_for_hv_kick(vp->vp_partition->pt_id, + vp->vp_index, + vp->run.kicked_by_hv, + mshv_vp_dispatch_thread_blocked(vp), + mshv_vp_interrupt_pending(vp)); + + vp->run.flags.root_sched_blocked = 0; + vp->run.kicked_by_hv = 0; + + return 0; +} + +/* Must be called with interrupts enabled */ +static long mshv_run_vp_with_root_scheduler(struct mshv_vp *vp) +{ + long ret; + + if (vp->run.flags.root_sched_blocked) { + /* + * Dispatch state of this VP is blocked. Need to wait + * for the hypervisor to clear the blocked state before + * dispatching it. + */ + ret = mshv_vp_wait_for_hv_kick(vp); + if (ret) + return ret; + } + + do { + u32 flags = 0; + struct hv_output_dispatch_vp output; + + if (__xfer_to_guest_mode_work_pending()) { + ret = xfer_to_guest_mode_handle_work(); + + trace_mshv_xfer_to_guest_mode_work(vp->vp_partition->pt_id, + vp->vp_index, + read_thread_flags(), + ret); + + if (ret) + break; + } + + if (vp->run.flags.intercept_suspend) + flags |= HV_DISPATCH_VP_FLAG_CLEAR_INTERCEPT_SUSPEND; + + if (mshv_vp_interrupt_pending(vp)) + flags |= HV_DISPATCH_VP_FLAG_SCAN_INTERRUPT_INJECTION; + + ret = mshv_vp_dispatch(vp, flags, &output); + if (ret) + break; + + vp->run.flags.intercept_suspend = 0; + + if (output.dispatch_state == HV_VP_DISPATCH_STATE_BLOCKED) { + if (output.dispatch_event == + HV_VP_DISPATCH_EVENT_SUSPEND) { + /* + * TODO: remove the warning once VP canceling + * is supported + */ + WARN_ONCE(atomic64_read(&vp->run.vp_signaled_count), + "%s: vp#%d: unexpected explicit suspend\n", + __func__, vp->vp_index); + /* + * Need to clear explicit suspend before + * dispatching. + * Explicit suspend is either: + * - set right after the first VP dispatch or + * - set explicitly via hypercall + * Since the latter case is not yet supported, + * simply clear it here. + */ + ret = mshv_vp_clear_explicit_suspend(vp); + if (ret) + break; + + ret = mshv_vp_wait_for_hv_kick(vp); + if (ret) + break; + } else { + vp->run.flags.root_sched_blocked = 1; + ret = mshv_vp_wait_for_hv_kick(vp); + if (ret) + break; + } + } else { + /* HV_VP_DISPATCH_STATE_READY */ + if (output.dispatch_event == + HV_VP_DISPATCH_EVENT_INTERCEPT) + vp->run.flags.intercept_suspend = 1; + } + } while (!vp->run.flags.intercept_suspend); + + rseq_virt_userspace_exit(); + + return ret; +} + +static_assert(sizeof(struct hv_message) <= MSHV_RUN_VP_BUF_SZ, + "sizeof(struct hv_message) must not exceed MSHV_RUN_VP_BUF_SZ"); + +static struct mshv_mem_region * +mshv_partition_region_by_gfn(struct mshv_partition *partition, u64 gfn) +{ + struct mshv_mem_region *region; + + hlist_for_each_entry(region, &partition->pt_mem_regions, hnode) { + if (gfn >= region->start_gfn && + gfn < region->start_gfn + region->nr_pages) + return region; + } + + return NULL; +} + +static struct mshv_mem_region * +mshv_partition_region_by_gfn_get(struct mshv_partition *p, u64 gfn) +{ + struct mshv_mem_region *region; + + spin_lock(&p->pt_mem_regions_lock); + region = mshv_partition_region_by_gfn(p, gfn); + if (!region || !mshv_region_get(region)) { + spin_unlock(&p->pt_mem_regions_lock); + return NULL; + } + spin_unlock(&p->pt_mem_regions_lock); + + return region; +} + +/** + * mshv_handle_gpa_intercept - Handle GPA (Guest Physical Address) intercepts. + * @vp: Pointer to the virtual processor structure. + * + * This function processes GPA intercepts by identifying the memory region + * corresponding to the intercepted GPA, aligning the page offset, and + * mapping the required pages. It ensures that the region is valid and + * handles faults efficiently by mapping multiple pages at once. + * + * Return: true if the intercept was handled successfully, false otherwise. + */ +static bool mshv_handle_gpa_intercept(struct mshv_vp *vp) +{ + struct mshv_partition *p = vp->vp_partition; + struct mshv_mem_region *region; + bool ret = false; + u64 gfn; +#if defined(CONFIG_X86_64) + struct hv_x64_memory_intercept_message *msg = + (struct hv_x64_memory_intercept_message *) + vp->vp_intercept_msg_page->u.payload; +#elif defined(CONFIG_ARM64) + struct hv_arm64_memory_intercept_message *msg = + (struct hv_arm64_memory_intercept_message *) + vp->vp_intercept_msg_page->u.payload; +#endif + enum hv_intercept_access_type access_type = + msg->header.intercept_access_type; + + gfn = HVPFN_DOWN(msg->guest_physical_address); + + region = mshv_partition_region_by_gfn_get(p, gfn); + if (!region) + goto out; + + if (access_type == HV_INTERCEPT_ACCESS_WRITE && + !(region->hv_map_flags & HV_MAP_GPA_WRITABLE)) + goto put_region; + + if (access_type == HV_INTERCEPT_ACCESS_EXECUTE && + !(region->hv_map_flags & HV_MAP_GPA_EXECUTABLE)) + goto put_region; + + /* Only movable memory ranges are supported for GPA intercepts */ + if (region->mreg_type == MSHV_REGION_TYPE_MEM_MOVABLE) + ret = mshv_region_handle_gfn_fault(region, gfn); + +put_region: + mshv_region_put(region); +out: + trace_mshv_handle_gpa_intercept(p->pt_id, vp->vp_index, gfn, + access_type, ret); + return ret; +} + +static bool mshv_vp_handle_intercept(struct mshv_vp *vp) +{ + switch (vp->vp_intercept_msg_page->header.message_type) { + case HVMSG_GPA_INTERCEPT: + return mshv_handle_gpa_intercept(vp); + } + return false; +} + +static long mshv_vp_ioctl_run_vp(struct mshv_vp *vp, void __user *ret_msg) +{ + long rc; + + trace_mshv_run_vp_entry(vp->vp_partition->pt_id, vp->vp_index); + + do { + if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT) + rc = mshv_run_vp_with_root_scheduler(vp); + else + rc = mshv_run_vp_with_hyp_scheduler(vp); + } while (rc == 0 && mshv_vp_handle_intercept(vp)); + + trace_mshv_run_vp_exit(vp->vp_partition->pt_id, vp->vp_index, + vp->vp_intercept_msg_page->header.message_type, + rc); + + if (rc) + return rc; + + if (copy_to_user(ret_msg, vp->vp_intercept_msg_page, + sizeof(struct hv_message))) + rc = -EFAULT; + + return rc; +} + +static int +mshv_vp_ioctl_get_set_state_pfn(struct mshv_vp *vp, + struct hv_vp_state_data state_data, + unsigned long user_pfn, size_t page_count, + bool is_set) +{ + int completed, ret = 0; + unsigned long check; + struct page **pages; + + if (page_count > INT_MAX) + return -EINVAL; + /* + * Check the arithmetic for wraparound/overflow. + * The last page address in the buffer is: + * (user_pfn + (page_count - 1)) * PAGE_SIZE + */ + if (check_add_overflow(user_pfn, (page_count - 1), &check)) + return -EOVERFLOW; + if (check_mul_overflow(check, PAGE_SIZE, &check)) + return -EOVERFLOW; + + /* Pin user pages so hypervisor can copy directly to them */ + pages = kzalloc_objs(struct page *, page_count); + if (!pages) + return -ENOMEM; + + for (completed = 0; completed < page_count; completed += ret) { + unsigned long user_addr = (user_pfn + completed) * PAGE_SIZE; + int remaining = page_count - completed; + + ret = pin_user_pages_fast(user_addr, remaining, FOLL_WRITE, + &pages[completed]); + if (ret < 0) { + vp_err(vp, "%s: Failed to pin user pages error %i\n", + __func__, ret); + goto unpin_pages; + } + } + + if (is_set) + ret = hv_call_set_vp_state(vp->vp_index, + vp->vp_partition->pt_id, + state_data, page_count, pages, + 0, NULL); + else + ret = hv_call_get_vp_state(vp->vp_index, + vp->vp_partition->pt_id, + state_data, page_count, pages, + NULL); + +unpin_pages: + unpin_user_pages(pages, completed); + kfree(pages); + return ret; +} + +static long +mshv_vp_ioctl_get_set_state(struct mshv_vp *vp, + struct mshv_get_set_vp_state __user *user_args, + bool is_set) +{ + struct mshv_get_set_vp_state args; + long ret = 0; + union hv_output_get_vp_state vp_state; + u32 data_sz; + struct hv_vp_state_data state_data = {}; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + if (args.type >= MSHV_VP_STATE_COUNT || mshv_field_nonzero(args, rsvd) || + !args.buf_sz || !PAGE_ALIGNED(args.buf_sz) || + !PAGE_ALIGNED(args.buf_ptr)) + return -EINVAL; + + if (!access_ok((void __user *)args.buf_ptr, args.buf_sz)) + return -EFAULT; + + switch (args.type) { + case MSHV_VP_STATE_LAPIC: + state_data.type = HV_GET_SET_VP_STATE_LAPIC_STATE; + data_sz = HV_HYP_PAGE_SIZE; + break; + case MSHV_VP_STATE_XSAVE: + { + u64 data_sz_64; + + ret = hv_call_get_partition_property(vp->vp_partition->pt_id, + HV_PARTITION_PROPERTY_XSAVE_STATES, + &state_data.xsave.states.as_uint64); + if (ret) + return ret; + + ret = hv_call_get_partition_property(vp->vp_partition->pt_id, + HV_PARTITION_PROPERTY_MAX_XSAVE_DATA_SIZE, + &data_sz_64); + if (ret) + return ret; + + data_sz = (u32)data_sz_64; + state_data.xsave.flags = 0; + /* Always request legacy states */ + state_data.xsave.states.legacy_x87 = 1; + state_data.xsave.states.legacy_sse = 1; + state_data.type = HV_GET_SET_VP_STATE_XSAVE; + break; + } + case MSHV_VP_STATE_SIMP: + state_data.type = HV_GET_SET_VP_STATE_SIM_PAGE; + data_sz = HV_HYP_PAGE_SIZE; + break; + case MSHV_VP_STATE_SIEFP: + state_data.type = HV_GET_SET_VP_STATE_SIEF_PAGE; + data_sz = HV_HYP_PAGE_SIZE; + break; + case MSHV_VP_STATE_SYNTHETIC_TIMERS: + state_data.type = HV_GET_SET_VP_STATE_SYNTHETIC_TIMERS; + data_sz = sizeof(vp_state.synthetic_timers_state); + break; + default: + return -EINVAL; + } + + if (copy_to_user(&user_args->buf_sz, &data_sz, sizeof(user_args->buf_sz))) + return -EFAULT; + + if (data_sz > args.buf_sz) + return -EINVAL; + + /* If the data is transmitted via pfns, delegate to helper */ + if (state_data.type & HV_GET_SET_VP_STATE_TYPE_PFN) { + unsigned long user_pfn = PFN_DOWN(args.buf_ptr); + size_t page_count = PFN_DOWN(args.buf_sz); + + return mshv_vp_ioctl_get_set_state_pfn(vp, state_data, user_pfn, + page_count, is_set); + } + + /* Paranoia check - this shouldn't happen! */ + if (data_sz > sizeof(vp_state)) { + vp_err(vp, "Invalid vp state data size!\n"); + return -EINVAL; + } + + if (is_set) { + if (copy_from_user(&vp_state, (__user void *)args.buf_ptr, data_sz)) + return -EFAULT; + + return hv_call_set_vp_state(vp->vp_index, + vp->vp_partition->pt_id, + state_data, 0, NULL, + sizeof(vp_state), (u8 *)&vp_state); + } + + ret = hv_call_get_vp_state(vp->vp_index, vp->vp_partition->pt_id, + state_data, 0, NULL, &vp_state); + if (ret) + return ret; + + if (copy_to_user((void __user *)args.buf_ptr, &vp_state, data_sz)) + return -EFAULT; + + return 0; +} + +static long +mshv_vp_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct mshv_vp *vp = filp->private_data; + long r = -ENOTTY; + + if (mutex_lock_killable(&vp->vp_mutex)) + return -EINTR; + + switch (ioctl) { + case MSHV_RUN_VP: + r = mshv_vp_ioctl_run_vp(vp, (void __user *)arg); + break; + case MSHV_GET_VP_STATE: + r = mshv_vp_ioctl_get_set_state(vp, (void __user *)arg, false); + break; + case MSHV_SET_VP_STATE: + r = mshv_vp_ioctl_get_set_state(vp, (void __user *)arg, true); + break; + case MSHV_ROOT_HVCALL: + r = mshv_ioctl_passthru_hvcall(vp->vp_partition, false, + (void __user *)arg); + break; + default: + vp_warn(vp, "Invalid ioctl: %#x\n", ioctl); + break; + } + mutex_unlock(&vp->vp_mutex); + + return r; +} + +static vm_fault_t mshv_vp_fault(struct vm_fault *vmf) +{ + struct mshv_vp *vp = vmf->vma->vm_file->private_data; + + switch (vmf->vma->vm_pgoff) { + case MSHV_VP_MMAP_OFFSET_REGISTERS: + vmf->page = virt_to_page(vp->vp_register_page); + break; + case MSHV_VP_MMAP_OFFSET_INTERCEPT_MESSAGE: + vmf->page = virt_to_page(vp->vp_intercept_msg_page); + break; + case MSHV_VP_MMAP_OFFSET_GHCB: + vmf->page = virt_to_page(vp->vp_ghcb_page); + break; + default: + return VM_FAULT_SIGBUS; + } + + get_page(vmf->page); + + return 0; +} + +static int mshv_vp_mmap(struct file *file, struct vm_area_struct *vma) +{ + struct mshv_vp *vp = file->private_data; + + switch (vma->vm_pgoff) { + case MSHV_VP_MMAP_OFFSET_REGISTERS: + if (!vp->vp_register_page) + return -ENODEV; + break; + case MSHV_VP_MMAP_OFFSET_INTERCEPT_MESSAGE: + if (!vp->vp_intercept_msg_page) + return -ENODEV; + break; + case MSHV_VP_MMAP_OFFSET_GHCB: + if (!vp->vp_ghcb_page) + return -ENODEV; + break; + default: + return -EINVAL; + } + + vma->vm_ops = &mshv_vp_vm_ops; + return 0; +} + +static int +mshv_vp_release(struct inode *inode, struct file *filp) +{ + struct mshv_vp *vp = filp->private_data; + + trace_mshv_vp_release(vp->vp_partition->pt_id, vp->vp_index); + + /* Rest of VP cleanup happens in destroy_partition() */ + mshv_partition_put(vp->vp_partition); + return 0; +} + +void mshv_vp_stats_unmap(u64 partition_id, u32 vp_index, + struct hv_stats_page *stats_pages[]) +{ + union hv_stats_object_identity identity = { + .vp.partition_id = partition_id, + .vp.vp_index = vp_index, + }; + int err; + + identity.vp.stats_area_type = HV_STATS_AREA_SELF; + err = hv_unmap_stats_page(HV_STATS_OBJECT_VP, + stats_pages[HV_STATS_AREA_SELF], + &identity); + if (err) + pr_err("%s: failed to unmap partition %llu vp %u self stats, err: %d\n", + __func__, partition_id, vp_index, err); + + if (stats_pages[HV_STATS_AREA_PARENT] != stats_pages[HV_STATS_AREA_SELF]) { + identity.vp.stats_area_type = HV_STATS_AREA_PARENT; + err = hv_unmap_stats_page(HV_STATS_OBJECT_VP, + stats_pages[HV_STATS_AREA_PARENT], + &identity); + if (err) + pr_err("%s: failed to unmap partition %llu vp %u parent stats, err: %d\n", + __func__, partition_id, vp_index, err); + } +} + +int mshv_vp_stats_map(u64 partition_id, u32 vp_index, + struct hv_stats_page *stats_pages[]) +{ + union hv_stats_object_identity identity = { + .vp.partition_id = partition_id, + .vp.vp_index = vp_index, + }; + int err; + + identity.vp.stats_area_type = HV_STATS_AREA_SELF; + err = hv_map_stats_page(HV_STATS_OBJECT_VP, &identity, + &stats_pages[HV_STATS_AREA_SELF]); + if (err) { + pr_err("%s: failed to map partition %llu vp %u self stats, err: %d\n", + __func__, partition_id, vp_index, err); + return err; + } + + /* + * L1VH partition cannot access its vp stats in parent area. + */ + if (is_l1vh_parent(partition_id)) { + stats_pages[HV_STATS_AREA_PARENT] = stats_pages[HV_STATS_AREA_SELF]; + } else { + identity.vp.stats_area_type = HV_STATS_AREA_PARENT; + err = hv_map_stats_page(HV_STATS_OBJECT_VP, &identity, + &stats_pages[HV_STATS_AREA_PARENT]); + if (err) { + pr_err("%s: failed to map partition %llu vp %u parent stats, err: %d\n", + __func__, partition_id, vp_index, err); + goto unmap_self; + } + if (!stats_pages[HV_STATS_AREA_PARENT]) + stats_pages[HV_STATS_AREA_PARENT] = stats_pages[HV_STATS_AREA_SELF]; + } + + return 0; + +unmap_self: + identity.vp.stats_area_type = HV_STATS_AREA_SELF; + hv_unmap_stats_page(HV_STATS_OBJECT_VP, + stats_pages[HV_STATS_AREA_SELF], + &identity); + return err; +} + +static long +mshv_partition_ioctl_create_vp(struct mshv_partition *partition, + void __user *arg) +{ + struct mshv_create_vp args; + struct mshv_vp *vp; + struct page *intercept_msg_page, *register_page, *ghcb_page; + struct hv_stats_page *stats_pages[2]; + long ret; + + if (copy_from_user(&args, arg, sizeof(args))) + return -EFAULT; + + if (args.vp_index >= MSHV_MAX_VPS) + return -EINVAL; + + if (partition->pt_vp_array[args.vp_index]) + return -EEXIST; + + ret = hv_call_create_vp(NUMA_NO_NODE, partition->pt_id, args.vp_index, + 0 /* Only valid for root partition VPs */); + if (ret) + return ret; + + ret = hv_map_vp_state_page(partition->pt_id, args.vp_index, + HV_VP_STATE_PAGE_INTERCEPT_MESSAGE, + input_vtl_zero, &intercept_msg_page); + if (ret) + goto destroy_vp; + + if (!mshv_partition_encrypted(partition)) { + ret = hv_map_vp_state_page(partition->pt_id, args.vp_index, + HV_VP_STATE_PAGE_REGISTERS, + input_vtl_zero, ®ister_page); + if (ret) + goto unmap_intercept_message_page; + } + + if (mshv_partition_encrypted(partition) && + is_ghcb_mapping_available()) { + ret = hv_map_vp_state_page(partition->pt_id, args.vp_index, + HV_VP_STATE_PAGE_GHCB, + input_vtl_normal, &ghcb_page); + if (ret) + goto unmap_register_page; + } + + ret = mshv_vp_stats_map(partition->pt_id, args.vp_index, + stats_pages); + if (ret) + goto unmap_ghcb_page; + + vp = kzalloc_obj(*vp); + if (!vp) + goto unmap_stats_pages; + + vp->vp_partition = mshv_partition_get(partition); + if (!vp->vp_partition) { + ret = -EBADF; + goto free_vp; + } + + mutex_init(&vp->vp_mutex); + init_waitqueue_head(&vp->run.vp_suspend_queue); + atomic64_set(&vp->run.vp_signaled_count, 0); + + vp->vp_index = args.vp_index; + vp->vp_intercept_msg_page = page_to_virt(intercept_msg_page); + if (!mshv_partition_encrypted(partition)) + vp->vp_register_page = page_to_virt(register_page); + + if (mshv_partition_encrypted(partition) && is_ghcb_mapping_available()) + vp->vp_ghcb_page = page_to_virt(ghcb_page); + + memcpy(vp->vp_stats_pages, stats_pages, sizeof(stats_pages)); + + ret = mshv_debugfs_vp_create(vp); + if (ret) + goto put_partition; + + /* + * Keep anon_inode_getfd last: it installs fd in the file struct and + * thus makes the state accessible in user space. + */ + ret = anon_inode_getfd("mshv_vp", &mshv_vp_fops, vp, + O_RDWR | O_CLOEXEC); + if (ret < 0) + goto remove_debugfs_vp; + + /* already exclusive with the partition mutex for all ioctls */ + partition->pt_vp_count++; + partition->pt_vp_array[args.vp_index] = vp; + + goto out; + +remove_debugfs_vp: + mshv_debugfs_vp_remove(vp); +put_partition: + mshv_partition_put(partition); +free_vp: + kfree(vp); +unmap_stats_pages: + mshv_vp_stats_unmap(partition->pt_id, args.vp_index, stats_pages); +unmap_ghcb_page: + if (mshv_partition_encrypted(partition) && is_ghcb_mapping_available()) + hv_unmap_vp_state_page(partition->pt_id, args.vp_index, + HV_VP_STATE_PAGE_GHCB, ghcb_page, + input_vtl_normal); +unmap_register_page: + if (!mshv_partition_encrypted(partition)) + hv_unmap_vp_state_page(partition->pt_id, args.vp_index, + HV_VP_STATE_PAGE_REGISTERS, + register_page, input_vtl_zero); +unmap_intercept_message_page: + hv_unmap_vp_state_page(partition->pt_id, args.vp_index, + HV_VP_STATE_PAGE_INTERCEPT_MESSAGE, + intercept_msg_page, input_vtl_zero); +destroy_vp: + hv_call_delete_vp(partition->pt_id, args.vp_index); +out: + trace_mshv_create_vp(partition->pt_id, args.vp_index, ret); + return ret; +} + +static int mshv_init_async_handler(struct mshv_partition *partition) +{ + if (completion_done(&partition->async_hypercall)) { + pt_err(partition, + "Cannot issue async hypercall while another one in progress!\n"); + return -EPERM; + } + + reinit_completion(&partition->async_hypercall); + return 0; +} + +static void mshv_async_hvcall_handler(void *data, u64 *status) +{ + struct mshv_partition *partition = data; + + wait_for_completion(&partition->async_hypercall); + pt_dbg(partition, "Async hypercall completed!\n"); + + *status = partition->async_hypercall_status; +} + +/* + * NB: caller checks and makes sure mem->size is page aligned + * Returns: 0 with regionpp updated on success, or -errno + */ +static int mshv_partition_create_region(struct mshv_partition *partition, + struct mshv_user_mem_region *mem, + struct mshv_mem_region **regionpp, + bool is_mmio) +{ + struct mshv_mem_region *rg; + u64 nr_pages = HVPFN_DOWN(mem->size); + + /* Reject overlapping regions */ + spin_lock(&partition->pt_mem_regions_lock); + hlist_for_each_entry(rg, &partition->pt_mem_regions, hnode) { + if (mem->guest_pfn + nr_pages <= rg->start_gfn || + rg->start_gfn + rg->nr_pages <= mem->guest_pfn) + continue; + spin_unlock(&partition->pt_mem_regions_lock); + return -EEXIST; + } + spin_unlock(&partition->pt_mem_regions_lock); + + rg = mshv_region_create(mem->guest_pfn, nr_pages, + mem->userspace_addr, mem->flags); + if (IS_ERR(rg)) + return PTR_ERR(rg); + + if (is_mmio) + rg->mreg_type = MSHV_REGION_TYPE_MMIO; + else if (mshv_partition_encrypted(partition) || + !mshv_region_movable_init(rg)) + rg->mreg_type = MSHV_REGION_TYPE_MEM_PINNED; + else + rg->mreg_type = MSHV_REGION_TYPE_MEM_MOVABLE; + + rg->partition = partition; + + *regionpp = rg; + + return 0; +} + +/** + * mshv_prepare_pinned_region - Pin and map memory regions + * @region: Pointer to the memory region structure + * + * This function processes memory regions that are explicitly marked as pinned. + * Pinned regions are preallocated, mapped upfront, and do not rely on fault-based + * population. The function ensures the region is properly populated, handles + * encryption requirements for SNP partitions if applicable, maps the region, + * and performs necessary sharing or eviction operations based on the mapping + * result. + * + * Return: 0 on success, negative error code on failure. + */ +static int mshv_prepare_pinned_region(struct mshv_mem_region *region) +{ + struct mshv_partition *partition = region->partition; + int ret; + + ret = mshv_region_pin(region); + if (ret) { + pt_err(partition, "Failed to pin memory region: %d\n", + ret); + goto err_out; + } + + /* + * For an SNP partition it is a requirement that for every memory region + * that we are going to map for this partition we should make sure that + * host access to that region is released. This is ensured by doing an + * additional hypercall which will update the SLAT to release host + * access to guest memory regions. + */ + if (mshv_partition_encrypted(partition)) { + ret = mshv_region_unshare(region); + if (ret) { + pt_err(partition, + "Failed to unshare memory region (guest_pfn: %llu): %d\n", + region->start_gfn, ret); + goto invalidate_region; + } + } + + ret = mshv_region_map(region); + if (ret && mshv_partition_encrypted(partition)) { + int shrc; + + shrc = mshv_region_share(region); + if (!shrc) + goto invalidate_region; + + pt_err(partition, + "Failed to share memory region (guest_pfn: %llu): %d\n", + region->start_gfn, shrc); + /* + * Don't unpin if marking shared failed because pages are no + * longer mapped in the host, ie root, anymore. + */ + goto err_out; + } + + return 0; + +invalidate_region: + mshv_region_invalidate(region); +err_out: + return ret; +} + +/* + * This maps two things: guest RAM and for pci passthru mmio space. + * + * mmio: + * - vfio overloads vm_pgoff to store the mmio start pfn/spa. + * - Two things need to happen for mapping mmio range: + * 1. mapped in the uaddr so VMM can access it. + * 2. mapped in the hwpt (gfn <-> mmio phys addr) so guest can access it. + * + * This function takes care of the second. The first one is managed by vfio, + * and hence is taken care of via vfio_pci_mmap_fault(). + */ +static long +mshv_map_user_memory(struct mshv_partition *partition, + struct mshv_user_mem_region *mem) +{ + struct mshv_mem_region *region; + struct vm_area_struct *vma; + bool is_mmio; + ulong mmio_pfn; + long ret; + + if (mem->flags & BIT(MSHV_SET_MEM_BIT_UNMAP) || + !access_ok((const void __user *)mem->userspace_addr, mem->size)) + return -EINVAL; + + mmap_read_lock(current->mm); + vma = vma_lookup(current->mm, mem->userspace_addr); + is_mmio = vma ? !!(vma->vm_flags & (VM_IO | VM_PFNMAP)) : 0; + mmio_pfn = is_mmio ? vma->vm_pgoff : 0; + mmap_read_unlock(current->mm); + + if (!vma) + return -EINVAL; + + ret = mshv_partition_create_region(partition, mem, ®ion, + is_mmio); + if (ret) + return ret; + + switch (region->mreg_type) { + case MSHV_REGION_TYPE_MEM_PINNED: + ret = mshv_prepare_pinned_region(region); + break; + case MSHV_REGION_TYPE_MEM_MOVABLE: + /* + * For movable memory regions, remap with no access to let + * the hypervisor track dirty pages, enabling pre-copy live + * migration. + */ + ret = hv_call_map_gpa_pages(partition->pt_id, + region->start_gfn, + region->nr_pages, + HV_MAP_GPA_NO_ACCESS, NULL); + break; + case MSHV_REGION_TYPE_MMIO: + ret = hv_call_map_mmio_pages(partition->pt_id, + region->start_gfn, + mmio_pfn, + region->nr_pages); + break; + } + + trace_mshv_map_user_memory(partition->pt_id, region->start_uaddr, + region->start_gfn, region->nr_pages, + region->hv_map_flags, ret); + + if (ret) + goto errout; + + spin_lock(&partition->pt_mem_regions_lock); + hlist_add_head(®ion->hnode, &partition->pt_mem_regions); + spin_unlock(&partition->pt_mem_regions_lock); + + return 0; + +errout: + mshv_region_put(region); + return ret; +} + +/* Called for unmapping both the guest ram and the mmio space */ +static long +mshv_unmap_user_memory(struct mshv_partition *partition, + struct mshv_user_mem_region *mem) +{ + struct mshv_mem_region *region; + + if (!(mem->flags & BIT(MSHV_SET_MEM_BIT_UNMAP))) + return -EINVAL; + + spin_lock(&partition->pt_mem_regions_lock); + + region = mshv_partition_region_by_gfn(partition, mem->guest_pfn); + if (!region) { + spin_unlock(&partition->pt_mem_regions_lock); + return -ENOENT; + } + + /* Paranoia check */ + if (region->start_uaddr != mem->userspace_addr || + region->start_gfn != mem->guest_pfn || + region->nr_pages != HVPFN_DOWN(mem->size)) { + spin_unlock(&partition->pt_mem_regions_lock); + return -EINVAL; + } + + hlist_del(®ion->hnode); + + spin_unlock(&partition->pt_mem_regions_lock); + + mshv_region_put(region); + + return 0; +} + +static long +mshv_partition_ioctl_set_memory(struct mshv_partition *partition, + struct mshv_user_mem_region __user *user_mem) +{ + struct mshv_user_mem_region mem; + + if (copy_from_user(&mem, user_mem, sizeof(mem))) + return -EFAULT; + + if (!mem.size || + !PAGE_ALIGNED(mem.size) || + !PAGE_ALIGNED(mem.userspace_addr) || + (mem.flags & ~MSHV_SET_MEM_FLAGS_MASK) || + mshv_field_nonzero(mem, rsvd)) + return -EINVAL; + + if (mem.flags & BIT(MSHV_SET_MEM_BIT_UNMAP)) + return mshv_unmap_user_memory(partition, &mem); + + return mshv_map_user_memory(partition, &mem); +} + +static long +mshv_partition_ioctl_ioeventfd(struct mshv_partition *partition, + void __user *user_args) +{ + struct mshv_user_ioeventfd args; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + return mshv_set_unset_ioeventfd(partition, &args); +} + +static long +mshv_partition_ioctl_irqfd(struct mshv_partition *partition, + void __user *user_args) +{ + struct mshv_user_irqfd args; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + return mshv_set_unset_irqfd(partition, &args); +} + +static long +mshv_partition_ioctl_get_gpap_access_bitmap(struct mshv_partition *partition, + void __user *user_args) +{ + struct mshv_gpap_access_bitmap args; + union hv_gpa_page_access_state *states; + long ret, i; + union hv_gpa_page_access_state_flags hv_flags = {}; + u8 hv_type_mask; + ulong bitmap_buf_sz, states_buf_sz; + int written = 0; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + if (args.access_type >= MSHV_GPAP_ACCESS_TYPE_COUNT || + args.access_op >= MSHV_GPAP_ACCESS_OP_COUNT || + mshv_field_nonzero(args, rsvd) || !args.page_count || + !args.bitmap_ptr) + return -EINVAL; + + if (check_mul_overflow(args.page_count, sizeof(*states), &states_buf_sz)) + return -E2BIG; + + /* Num bytes needed to store bitmap; one bit per page rounded up */ + bitmap_buf_sz = DIV_ROUND_UP(args.page_count, 8); + + /* Sanity check */ + if (bitmap_buf_sz > states_buf_sz) + return -EBADFD; + + switch (args.access_type) { + case MSHV_GPAP_ACCESS_TYPE_ACCESSED: + hv_type_mask = 1; + if (args.access_op == MSHV_GPAP_ACCESS_OP_CLEAR) { + hv_flags.clear_accessed = 1; + /* not accessed implies not dirty */ + hv_flags.clear_dirty = 1; + } else { /* MSHV_GPAP_ACCESS_OP_SET */ + hv_flags.set_accessed = 1; + } + break; + case MSHV_GPAP_ACCESS_TYPE_DIRTY: + hv_type_mask = 2; + if (args.access_op == MSHV_GPAP_ACCESS_OP_CLEAR) { + hv_flags.clear_dirty = 1; + } else { /* MSHV_GPAP_ACCESS_OP_SET */ + hv_flags.set_dirty = 1; + /* dirty implies accessed */ + hv_flags.set_accessed = 1; + } + break; + } + + states = vzalloc(states_buf_sz); + if (!states) + return -ENOMEM; + + ret = hv_call_get_gpa_access_states(partition->pt_id, args.page_count, + args.gpap_base, hv_flags, &written, + states); + if (ret) + goto free_return; + + /* + * Overwrite states buffer with bitmap - the bits in hv_type_mask + * correspond to bitfields in hv_gpa_page_access_state + */ + for (i = 0; i < written; ++i) + __assign_bit(i, (ulong *)states, + states[i].as_uint8 & hv_type_mask); + + /* zero the unused bits in the last byte(s) of the returned bitmap */ + for (i = written; i < bitmap_buf_sz * 8; ++i) + __clear_bit(i, (ulong *)states); + + if (copy_to_user((void __user *)args.bitmap_ptr, states, bitmap_buf_sz)) + ret = -EFAULT; + +free_return: + vfree(states); + return ret; +} + +static long +mshv_partition_ioctl_set_msi_routing(struct mshv_partition *partition, + void __user *user_args) +{ + struct mshv_user_irq_entry *entries = NULL; + struct mshv_user_irq_table args; + long ret; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + if (args.nr > MSHV_MAX_GUEST_IRQS || + mshv_field_nonzero(args, rsvd)) + return -EINVAL; + + if (args.nr) { + struct mshv_user_irq_table __user *urouting = user_args; + + entries = vmemdup_user(urouting->entries, + array_size(sizeof(*entries), + args.nr)); + if (IS_ERR(entries)) + return PTR_ERR(entries); + } + ret = mshv_update_routing_table(partition, entries, args.nr); + kvfree(entries); + + return ret; +} + +static long +mshv_partition_ioctl_initialize(struct mshv_partition *partition) +{ + long ret; + + if (partition->pt_initialized) + return 0; + + ret = hv_call_initialize_partition(partition->pt_id); + if (ret) + goto withdraw_mem; + + ret = mshv_debugfs_partition_create(partition); + if (ret) + goto finalize_partition; + + partition->pt_initialized = true; + + return 0; + +finalize_partition: + hv_call_finalize_partition(partition->pt_id); +withdraw_mem: + hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id); + + return ret; +} + +static long +mshv_partition_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct mshv_partition *partition = filp->private_data; + long ret; + void __user *uarg = (void __user *)arg; + + if (mutex_lock_killable(&partition->pt_mutex)) + return -EINTR; + + switch (ioctl) { + case MSHV_INITIALIZE_PARTITION: + ret = mshv_partition_ioctl_initialize(partition); + break; + case MSHV_SET_GUEST_MEMORY: + ret = mshv_partition_ioctl_set_memory(partition, uarg); + break; + case MSHV_CREATE_VP: + ret = mshv_partition_ioctl_create_vp(partition, uarg); + break; + case MSHV_IRQFD: + ret = mshv_partition_ioctl_irqfd(partition, uarg); + break; + case MSHV_IOEVENTFD: + ret = mshv_partition_ioctl_ioeventfd(partition, uarg); + break; + case MSHV_SET_MSI_ROUTING: + ret = mshv_partition_ioctl_set_msi_routing(partition, uarg); + break; + case MSHV_GET_GPAP_ACCESS_BITMAP: + ret = mshv_partition_ioctl_get_gpap_access_bitmap(partition, + uarg); + break; + case MSHV_ROOT_HVCALL: + ret = mshv_ioctl_passthru_hvcall(partition, true, uarg); + break; + default: + ret = -ENOTTY; + } + + mutex_unlock(&partition->pt_mutex); + return ret; +} + +static int +disable_vp_dispatch(struct mshv_vp *vp) +{ + int ret; + struct hv_register_assoc dispatch_suspend = { + .name = HV_REGISTER_DISPATCH_SUSPEND, + .value.dispatch_suspend.suspended = 1, + }; + + ret = mshv_set_vp_registers(vp->vp_index, vp->vp_partition->pt_id, + 1, &dispatch_suspend); + if (ret) + vp_err(vp, "failed to suspend\n"); + + trace_mshv_disable_vp_dispatch(vp->vp_partition->pt_id, + vp->vp_index, ret); + + return ret; +} + +static int +get_vp_signaled_count(struct mshv_vp *vp, u64 *count) +{ + int ret; + struct hv_register_assoc root_signal_count = { + .name = HV_REGISTER_VP_ROOT_SIGNAL_COUNT, + }; + + ret = mshv_get_vp_registers(vp->vp_index, vp->vp_partition->pt_id, + 1, &root_signal_count); + + if (ret) { + vp_err(vp, "Failed to get root signal count"); + *count = 0; + return ret; + } + + *count = root_signal_count.value.reg64; + + return ret; +} + +static void +drain_vp_signals(struct mshv_vp *vp) +{ + u64 hv_signal_count; + u64 vp_signal_count; + + get_vp_signaled_count(vp, &hv_signal_count); + + vp_signal_count = atomic64_read(&vp->run.vp_signaled_count); + + /* + * There should be at most 1 outstanding notification, but be extra + * careful anyway. + */ + while (hv_signal_count != vp_signal_count) { + WARN_ON(hv_signal_count - vp_signal_count != 1); + + if (wait_event_interruptible(vp->run.vp_suspend_queue, + vp->run.kicked_by_hv == 1)) + break; + vp->run.kicked_by_hv = 0; + vp_signal_count = atomic64_read(&vp->run.vp_signaled_count); + } + + trace_mshv_drain_vp_signals(vp->vp_partition->pt_id, vp->vp_index); +} + +static void drain_all_vps(const struct mshv_partition *partition) +{ + int i; + struct mshv_vp *vp; + + /* + * VPs are reachable from ISR. It is safe to not take the partition + * lock because nobody else can enter this function and drop the + * partition from the list. + */ + for (i = 0; i < MSHV_MAX_VPS; i++) { + vp = partition->pt_vp_array[i]; + if (!vp) + continue; + /* + * Disable dispatching of the VP in the hypervisor. After this + * the hypervisor guarantees it won't generate any signals for + * the VP and the hypervisor's VP signal count won't change. + */ + disable_vp_dispatch(vp); + drain_vp_signals(vp); + } +} + +static void +remove_partition(struct mshv_partition *partition) +{ + spin_lock(&mshv_root.pt_ht_lock); + hlist_del_rcu(&partition->pt_hnode); + spin_unlock(&mshv_root.pt_ht_lock); + + synchronize_rcu(); +} + +/* + * Tear down a partition and remove it from the list. + * Partition's refcount must be 0 + */ +static void destroy_partition(struct mshv_partition *partition) +{ + struct mshv_vp *vp; + struct mshv_mem_region *region; + struct hlist_node *n; + int i; + + if (refcount_read(&partition->pt_ref_count)) { + pt_err(partition, + "Attempt to destroy partition but refcount > 0\n"); + return; + } + + trace_mshv_destroy_partition(partition->pt_id); + + if (partition->pt_initialized) { + /* + * We only need to drain signals for root scheduler. This should be + * done before removing the partition from the partition list. + */ + if (hv_scheduler_type == HV_SCHEDULER_TYPE_ROOT) + drain_all_vps(partition); + + /* Remove vps */ + for (i = 0; i < MSHV_MAX_VPS; ++i) { + vp = partition->pt_vp_array[i]; + if (!vp) + continue; + + mshv_debugfs_vp_remove(vp); + mshv_vp_stats_unmap(partition->pt_id, vp->vp_index, + vp->vp_stats_pages); + + if (vp->vp_register_page) { + (void)hv_unmap_vp_state_page(partition->pt_id, + vp->vp_index, + HV_VP_STATE_PAGE_REGISTERS, + virt_to_page(vp->vp_register_page), + input_vtl_zero); + vp->vp_register_page = NULL; + } + + (void)hv_unmap_vp_state_page(partition->pt_id, + vp->vp_index, + HV_VP_STATE_PAGE_INTERCEPT_MESSAGE, + virt_to_page(vp->vp_intercept_msg_page), + input_vtl_zero); + vp->vp_intercept_msg_page = NULL; + + if (vp->vp_ghcb_page) { + (void)hv_unmap_vp_state_page(partition->pt_id, + vp->vp_index, + HV_VP_STATE_PAGE_GHCB, + virt_to_page(vp->vp_ghcb_page), + input_vtl_normal); + vp->vp_ghcb_page = NULL; + } + + kfree(vp); + + partition->pt_vp_array[i] = NULL; + } + + mshv_debugfs_partition_remove(partition); + + /* Deallocates and unmaps everything including vcpus, GPA mappings etc */ + hv_call_finalize_partition(partition->pt_id); + + partition->pt_initialized = false; + } + + remove_partition(partition); + + hlist_for_each_entry_safe(region, n, &partition->pt_mem_regions, + hnode) { + hlist_del(®ion->hnode); + mshv_region_put(region); + } + + /* Withdraw and free all pages we deposited */ + hv_call_withdraw_memory(U64_MAX, NUMA_NO_NODE, partition->pt_id); + hv_call_delete_partition(partition->pt_id); + + mshv_free_routing_table(partition); + kfree(partition); +} + +struct +mshv_partition *mshv_partition_get(struct mshv_partition *partition) +{ + if (refcount_inc_not_zero(&partition->pt_ref_count)) + return partition; + return NULL; +} + +struct +mshv_partition *mshv_partition_find(u64 partition_id) + __must_hold(RCU) +{ + struct mshv_partition *p; + + hash_for_each_possible_rcu(mshv_root.pt_htable, p, pt_hnode, + partition_id) + if (p->pt_id == partition_id) + return p; + + return NULL; +} + +void +mshv_partition_put(struct mshv_partition *partition) +{ + if (refcount_dec_and_test(&partition->pt_ref_count)) + destroy_partition(partition); +} + +static int +mshv_partition_release(struct inode *inode, struct file *filp) +{ + struct mshv_partition *partition = filp->private_data; + + trace_mshv_partition_release(partition->pt_id); + + mshv_eventfd_release(partition); + + cleanup_srcu_struct(&partition->pt_irq_srcu); + + mshv_partition_put(partition); + + return 0; +} + +static int +add_partition(struct mshv_partition *partition) +{ + spin_lock(&mshv_root.pt_ht_lock); + + hash_add_rcu(mshv_root.pt_htable, &partition->pt_hnode, + partition->pt_id); + + spin_unlock(&mshv_root.pt_ht_lock); + + return 0; +} + +static_assert(MSHV_NUM_CPU_FEATURES_BANKS == + HV_PARTITION_PROCESSOR_FEATURES_BANKS); + +static long mshv_ioctl_process_pt_flags(void __user *user_arg, u64 *pt_flags, + struct hv_partition_creation_properties *cr_props, + union hv_partition_isolation_properties *isol_props) +{ + int i; + struct mshv_create_partition_v2 args; + union hv_partition_processor_features *disabled_procs; + union hv_partition_processor_xsave_features *disabled_xsave; + + /* First, copy v1 struct in case user is on previous versions */ + if (copy_from_user(&args, user_arg, + sizeof(struct mshv_create_partition))) + return -EFAULT; + + if ((args.pt_flags & ~MSHV_PT_FLAGS_MASK) || + args.pt_isolation >= MSHV_PT_ISOLATION_COUNT) + return -EINVAL; + + disabled_procs = &cr_props->disabled_processor_features; + disabled_xsave = &cr_props->disabled_processor_xsave_features; + + /* Check if user provided newer struct with feature fields */ + if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_CPU_AND_XSAVE_FEATURES)) { + if (copy_from_user(&args, user_arg, sizeof(args))) + return -EFAULT; + + /* Re-validate v1 fields after second copy_from_user() */ + if ((args.pt_flags & ~MSHV_PT_FLAGS_MASK) || + args.pt_isolation >= MSHV_PT_ISOLATION_COUNT) + return -EINVAL; + + if (args.pt_num_cpu_fbanks != MSHV_NUM_CPU_FEATURES_BANKS || + mshv_field_nonzero(args, pt_rsvd) || + mshv_field_nonzero(args, pt_rsvd1)) + return -EINVAL; + + /* + * Note this assumes MSHV_NUM_CPU_FEATURES_BANKS will never + * change and equals HV_PARTITION_PROCESSOR_FEATURES_BANKS + * (i.e. 2). + * + * Further banks (index >= 2) will be modifiable as 'early' + * properties via the set partition property hypercall. + */ + for (i = 0; i < HV_PARTITION_PROCESSOR_FEATURES_BANKS; i++) + disabled_procs->as_uint64[i] = args.pt_cpu_fbanks[i]; + +#if IS_ENABLED(CONFIG_X86_64) + disabled_xsave->as_uint64 = args.pt_disabled_xsave; +#else + /* + * In practice this field is ignored on arm64, but safer to + * zero it in case it is ever used. + */ + disabled_xsave->as_uint64 = 0; + + if (mshv_field_nonzero(args, pt_rsvd2)) + return -EINVAL; +#endif + } else { + /* + * v1 behavior: try to enable everything. The hypervisor will + * disable features that are not supported. The banks can be + * queried via the get partition property hypercall. + */ + for (i = 0; i < HV_PARTITION_PROCESSOR_FEATURES_BANKS; i++) + disabled_procs->as_uint64[i] = 0; + + disabled_xsave->as_uint64 = 0; + } + + /* Only support EXO partitions */ + *pt_flags = HV_PARTITION_CREATION_FLAG_EXO_PARTITION | + HV_PARTITION_CREATION_FLAG_INTERCEPT_MESSAGE_PAGE_ENABLED; + + if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_LAPIC)) + *pt_flags |= HV_PARTITION_CREATION_FLAG_LAPIC_ENABLED; + if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_X2APIC)) + *pt_flags |= HV_PARTITION_CREATION_FLAG_X2APIC_CAPABLE; + if (args.pt_flags & BIT_ULL(MSHV_PT_BIT_GPA_SUPER_PAGES)) + *pt_flags |= HV_PARTITION_CREATION_FLAG_GPA_SUPER_PAGES_ENABLED; + if (args.pt_flags & BIT(MSHV_PT_BIT_NESTED_VIRTUALIZATION)) + *pt_flags |= HV_PARTITION_CREATION_FLAG_NESTED_VIRTUALIZATION_CAPABLE; + if (args.pt_flags & BIT(MSHV_PT_BIT_SMT_ENABLED_GUEST)) + *pt_flags |= HV_PARTITION_CREATION_FLAG_SMT_ENABLED_GUEST; + + isol_props->as_uint64 = 0; + + switch (args.pt_isolation) { + case MSHV_PT_ISOLATION_NONE: + isol_props->isolation_type = HV_PARTITION_ISOLATION_TYPE_NONE; + break; + } + + return 0; +} + +static long +mshv_ioctl_create_partition(void __user *user_arg, struct device *module_dev) +{ + u64 creation_flags; + struct hv_partition_creation_properties creation_properties; + union hv_partition_isolation_properties isolation_properties; + struct mshv_partition *partition; + u64 pt_id = -1; + long ret; + + ret = mshv_ioctl_process_pt_flags(user_arg, &creation_flags, + &creation_properties, + &isolation_properties); + if (ret) + return ret; + + partition = kzalloc_obj(*partition); + if (!partition) + return -ENOMEM; + + partition->pt_module_dev = module_dev; + partition->isolation_type = isolation_properties.isolation_type; + + refcount_set(&partition->pt_ref_count, 1); + + mutex_init(&partition->pt_mutex); + + mutex_init(&partition->pt_irq_lock); + + init_completion(&partition->async_hypercall); + + INIT_HLIST_HEAD(&partition->irq_ack_notifier_list); + + INIT_HLIST_HEAD(&partition->pt_devices); + + spin_lock_init(&partition->pt_mem_regions_lock); + INIT_HLIST_HEAD(&partition->pt_mem_regions); + + mshv_eventfd_init(partition); + + ret = init_srcu_struct(&partition->pt_irq_srcu); + if (ret) + goto free_partition; + + ret = hv_call_create_partition(creation_flags, + creation_properties, + isolation_properties, + &pt_id); + if (ret) + goto cleanup_irq_srcu; + + partition->pt_id = pt_id; + + ret = add_partition(partition); + if (ret) + goto delete_partition; + + ret = mshv_init_async_handler(partition); + if (ret) + goto remove_partition; + + ret = FD_ADD(O_CLOEXEC, anon_inode_getfile("mshv_partition", + &mshv_partition_fops, + partition, O_RDWR)); + if (ret < 0) + goto remove_partition; + + goto out; + +remove_partition: + remove_partition(partition); +delete_partition: + hv_call_delete_partition(partition->pt_id); +cleanup_irq_srcu: + cleanup_srcu_struct(&partition->pt_irq_srcu); +free_partition: + kfree(partition); +out: + trace_mshv_create_partition(pt_id, ret); + return ret; +} + +static long mshv_dev_ioctl(struct file *filp, unsigned int ioctl, + unsigned long arg) +{ + struct miscdevice *misc = filp->private_data; + + switch (ioctl) { + case MSHV_CREATE_PARTITION: + return mshv_ioctl_create_partition((void __user *)arg, + misc->this_device); + case MSHV_ROOT_HVCALL: + return mshv_ioctl_passthru_hvcall(NULL, false, + (void __user *)arg); + } + + return -ENOTTY; +} + +static int +mshv_dev_open(struct inode *inode, struct file *filp) +{ + return 0; +} + +static int +mshv_dev_release(struct inode *inode, struct file *filp) +{ + return 0; +} + +static int mshv_root_sched_online; + +static const char *scheduler_type_to_string(enum hv_scheduler_type type) +{ + switch (type) { + case HV_SCHEDULER_TYPE_LP: + return "classic scheduler without SMT"; + case HV_SCHEDULER_TYPE_LP_SMT: + return "classic scheduler with SMT"; + case HV_SCHEDULER_TYPE_CORE_SMT: + return "core scheduler"; + case HV_SCHEDULER_TYPE_ROOT: + return "root scheduler"; + default: + return "unknown scheduler"; + }; +} + +static int __init l1vh_retrieve_scheduler_type(enum hv_scheduler_type *out) +{ + u64 integrated_sched_enabled; + int ret; + + *out = HV_SCHEDULER_TYPE_CORE_SMT; + + if (!mshv_root.vmm_caps.vmm_enable_integrated_scheduler) + return 0; + + ret = hv_call_get_partition_property_ex(HV_PARTITION_ID_SELF, + HV_PARTITION_PROPERTY_INTEGRATED_SCHEDULER_ENABLED, + 0, &integrated_sched_enabled, + sizeof(integrated_sched_enabled)); + if (ret) + return ret; + + if (integrated_sched_enabled) + *out = HV_SCHEDULER_TYPE_ROOT; + + return 0; +} + +/* TODO move this to hv_common.c when needed outside */ +static int __init hv_retrieve_scheduler_type(enum hv_scheduler_type *out) +{ + struct hv_input_get_system_property *input; + struct hv_output_get_system_property *output; + unsigned long flags; + u64 status; + + local_irq_save(flags); + input = *this_cpu_ptr(hyperv_pcpu_input_arg); + output = *this_cpu_ptr(hyperv_pcpu_output_arg); + + memset(input, 0, sizeof(*input)); + memset(output, 0, sizeof(*output)); + input->property_id = HV_SYSTEM_PROPERTY_SCHEDULER_TYPE; + + status = hv_do_hypercall(HVCALL_GET_SYSTEM_PROPERTY, input, output); + if (!hv_result_success(status)) { + local_irq_restore(flags); + pr_err("%s: %s\n", __func__, hv_result_to_string(status)); + return hv_result_to_errno(status); + } + + *out = output->scheduler_type; + local_irq_restore(flags); + + return 0; +} + +/* Retrieve and stash the supported scheduler type */ +static int __init mshv_retrieve_scheduler_type(struct device *dev) +{ + int ret; + + if (hv_l1vh_partition()) + ret = l1vh_retrieve_scheduler_type(&hv_scheduler_type); + else + ret = hv_retrieve_scheduler_type(&hv_scheduler_type); + if (ret) + return ret; + + dev_info(dev, "Hypervisor using %s\n", + scheduler_type_to_string(hv_scheduler_type)); + + switch (hv_scheduler_type) { + case HV_SCHEDULER_TYPE_CORE_SMT: + case HV_SCHEDULER_TYPE_LP_SMT: + case HV_SCHEDULER_TYPE_ROOT: + case HV_SCHEDULER_TYPE_LP: + /* Supported scheduler, nothing to do */ + break; + default: + dev_err(dev, "unsupported scheduler 0x%x, bailing.\n", + hv_scheduler_type); + return -EOPNOTSUPP; + } + + return 0; +} + +static int mshv_root_scheduler_init(unsigned int cpu) +{ + void **inputarg, **outputarg, *p; + + inputarg = (void **)this_cpu_ptr(root_scheduler_input); + outputarg = (void **)this_cpu_ptr(root_scheduler_output); + + /* Allocate two consecutive pages. One for input, one for output. */ + p = kmalloc_array(2, HV_HYP_PAGE_SIZE, GFP_KERNEL); + if (!p) + return -ENOMEM; + + *inputarg = p; + *outputarg = (char *)p + HV_HYP_PAGE_SIZE; + + return 0; +} + +static int mshv_root_scheduler_cleanup(unsigned int cpu) +{ + void *p, **inputarg, **outputarg; + + inputarg = (void **)this_cpu_ptr(root_scheduler_input); + outputarg = (void **)this_cpu_ptr(root_scheduler_output); + + p = *inputarg; + + *inputarg = NULL; + *outputarg = NULL; + + kfree(p); + + return 0; +} + +/* Must be called after retrieving the scheduler type */ +static int +root_scheduler_init(struct device *dev) +{ + int ret; + + if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT) + return 0; + + root_scheduler_input = alloc_percpu(void *); + root_scheduler_output = alloc_percpu(void *); + + if (!root_scheduler_input || !root_scheduler_output) { + dev_err(dev, "Failed to allocate root scheduler buffers\n"); + ret = -ENOMEM; + goto out; + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_root_sched", + mshv_root_scheduler_init, + mshv_root_scheduler_cleanup); + + if (ret < 0) { + dev_err(dev, "Failed to setup root scheduler state: %i\n", ret); + goto out; + } + + mshv_root_sched_online = ret; + + return 0; + +out: + free_percpu(root_scheduler_input); + free_percpu(root_scheduler_output); + return ret; +} + +static void +root_scheduler_deinit(void) +{ + if (hv_scheduler_type != HV_SCHEDULER_TYPE_ROOT) + return; + + cpuhp_remove_state(mshv_root_sched_online); + free_percpu(root_scheduler_input); + free_percpu(root_scheduler_output); +} + +static int __init mshv_init_vmm_caps(struct device *dev) +{ + int ret; + + ret = hv_call_get_partition_property_ex(HV_PARTITION_ID_SELF, + HV_PARTITION_PROPERTY_VMM_CAPABILITIES, + 0, &mshv_root.vmm_caps, + sizeof(mshv_root.vmm_caps)); + if (ret && hv_l1vh_partition()) { + dev_err(dev, "Failed to get VMM capabilities: %d\n", ret); + return ret; + } + + dev_dbg(dev, "vmm_caps = %#llx\n", mshv_root.vmm_caps.as_uint64[0]); + + return 0; +} + +static int __init mshv_parent_partition_init(void) +{ + int ret; + struct device *dev; + union hv_hypervisor_version_info version_info; + + if (!hv_parent_partition() || is_kdump_kernel()) + return -ENODEV; + + if (hv_get_hypervisor_version(&version_info)) + return -ENODEV; + + ret = misc_register(&mshv_dev); + if (ret) + return ret; + + dev = mshv_dev.this_device; + + if (version_info.build_number < MSHV_HV_MIN_VERSION || + version_info.build_number > MSHV_HV_MAX_VERSION) { + dev_err(dev, "Running on unvalidated Hyper-V version\n"); + dev_err(dev, "Versions: current: %u min: %u max: %u\n", + version_info.build_number, MSHV_HV_MIN_VERSION, + MSHV_HV_MAX_VERSION); + } + + ret = mshv_synic_init(dev); + if (ret) + goto device_deregister; + + ret = mshv_init_vmm_caps(dev); + if (ret) + goto synic_cleanup; + + ret = mshv_retrieve_scheduler_type(dev); + if (ret) + goto synic_cleanup; + + ret = root_scheduler_init(dev); + if (ret) + goto synic_cleanup; + + ret = mshv_debugfs_init(); + if (ret) + goto deinit_root_scheduler; + + ret = mshv_irqfd_wq_init(); + if (ret) + goto exit_debugfs; + + spin_lock_init(&mshv_root.pt_ht_lock); + hash_init(mshv_root.pt_htable); + + hv_setup_mshv_handler(mshv_isr); + + return 0; + +exit_debugfs: + mshv_debugfs_exit(); +deinit_root_scheduler: + root_scheduler_deinit(); +synic_cleanup: + mshv_synic_exit(); +device_deregister: + misc_deregister(&mshv_dev); + return ret; +} + +static void __exit mshv_parent_partition_exit(void) +{ + hv_setup_mshv_handler(NULL); + mshv_port_table_fini(); + mshv_debugfs_exit(); + misc_deregister(&mshv_dev); + mshv_irqfd_wq_cleanup(); + root_scheduler_deinit(); + mshv_synic_exit(); +} + +module_init(mshv_parent_partition_init); +module_exit(mshv_parent_partition_exit); diff --git a/drivers/hv/mshv_synic.c b/drivers/hv/mshv_synic.c new file mode 100644 index 000000000..7c168e5a7 --- /dev/null +++ b/drivers/hv/mshv_synic.c @@ -0,0 +1,873 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023, Microsoft Corporation. + * + * mshv_root module's main interrupt handler and associated functionality. + * + * Authors: Microsoft Linux virtualization team + */ + +#include <linux/kernel.h> +#include <linux/slab.h> +#include <linux/mm.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/cpuhotplug.h> +#include <linux/hyperv.h> +#include <linux/reboot.h> +#include <asm/mshyperv.h> +#include <linux/acpi.h> + +#include "mshv_eventfd.h" +#include "mshv.h" + +static int synic_cpuhp_online; +static struct hv_synic_pages __percpu *synic_pages; +static int mshv_sint_vector = -1; /* hwirq for the SynIC SINTs */ +static int mshv_sint_irq = -1; /* Linux IRQ for mshv_sint_vector */ + +static u32 synic_event_ring_get_queued_port(u32 sint_index) +{ + struct hv_synic_event_ring_page **event_ring_page; + volatile struct hv_synic_event_ring *ring; + struct hv_synic_pages *spages; + u8 **synic_eventring_tail; + u32 message; + u8 tail; + + spages = this_cpu_ptr(synic_pages); + event_ring_page = &spages->synic_event_ring_page; + synic_eventring_tail = (u8 **)this_cpu_ptr(hv_synic_eventring_tail); + + if (unlikely(!*synic_eventring_tail)) { + pr_debug("Missing synic event ring tail!\n"); + return 0; + } + tail = (*synic_eventring_tail)[sint_index]; + + if (unlikely(!*event_ring_page)) { + pr_debug("Missing synic event ring page!\n"); + return 0; + } + + ring = &(*event_ring_page)->sint_event_ring[sint_index]; + + /* + * Get the message. + */ + message = ring->data[tail]; + + if (!message) { + if (ring->ring_full) { + /* + * Ring is marked full, but we would have consumed all + * the messages. Notify the hypervisor that ring is now + * empty and check again. + */ + ring->ring_full = 0; + hv_call_notify_port_ring_empty(sint_index); + message = ring->data[tail]; + } + + if (!message) { + ring->signal_masked = 0; + /* + * Unmask the signal and sync with hypervisor + * before one last check for any message. + */ + mb(); + message = ring->data[tail]; + + /* + * Ok, lets bail out. + */ + if (!message) + return 0; + } + + ring->signal_masked = 1; + } + + /* + * Clear the message in the ring buffer. + */ + ring->data[tail] = 0; + + if (++tail == HV_SYNIC_EVENT_RING_MESSAGE_COUNT) + tail = 0; + + (*synic_eventring_tail)[sint_index] = tail; + + return message; +} + +static bool +mshv_doorbell_isr(struct hv_message *msg) +{ + struct hv_notification_message_payload *notification; + u32 port; + + if (msg->header.message_type != HVMSG_SYNIC_SINT_INTERCEPT) + return false; + + notification = (struct hv_notification_message_payload *)msg->u.payload; + if (notification->sint_index != HV_SYNIC_DOORBELL_SINT_INDEX) + return false; + + while ((port = synic_event_ring_get_queued_port(HV_SYNIC_DOORBELL_SINT_INDEX))) { + struct port_table_info ptinfo = { 0 }; + + if (mshv_portid_lookup(port, &ptinfo)) { + pr_debug("Failed to get port info from port_table!\n"); + continue; + } + + if (ptinfo.hv_port_type != HV_PORT_TYPE_DOORBELL) { + pr_debug("Not a doorbell port!, port: %d, port_type: %d\n", + port, ptinfo.hv_port_type); + continue; + } + + /* Invoke the callback */ + ptinfo.hv_port_doorbell.doorbell_cb(port, + ptinfo.hv_port_doorbell.data); + } + + return true; +} + +static bool mshv_async_call_completion_isr(struct hv_message *msg) +{ + bool handled = false; + struct hv_async_completion_message_payload *async_msg; + struct mshv_partition *partition; + u64 partition_id; + + if (msg->header.message_type != HVMSG_ASYNC_CALL_COMPLETION) + goto out; + + async_msg = + (struct hv_async_completion_message_payload *)msg->u.payload; + + partition_id = async_msg->partition_id; + + /* + * Hold this lock for the rest of the isr, because the partition could + * be released anytime. + * e.g. the MSHV_RUN_VP thread could wake on another cpu; it could + * release the partition unless we hold this! + */ + rcu_read_lock(); + + partition = mshv_partition_find(partition_id); + + if (unlikely(!partition)) { + pr_debug("failed to find partition %llu\n", partition_id); + goto unlock_out; + } + + partition->async_hypercall_status = async_msg->status; + complete(&partition->async_hypercall); + + handled = true; + +unlock_out: + rcu_read_unlock(); +out: + return handled; +} + +static void kick_vp(struct mshv_vp *vp) +{ + atomic64_inc(&vp->run.vp_signaled_count); + vp->run.kicked_by_hv = 1; + wake_up(&vp->run.vp_suspend_queue); +} + +static void +handle_bitset_message(const struct hv_vp_signal_bitset_scheduler_message *msg) +{ + int bank_idx, vps_signaled = 0, bank_mask_size; + struct mshv_partition *partition; + const struct hv_vpset *vpset; + const u64 *bank_contents; + u64 partition_id = msg->partition_id; + + if (msg->vp_bitset.bitset.format != HV_GENERIC_SET_SPARSE_4K) { + pr_debug("scheduler message format is not HV_GENERIC_SET_SPARSE_4K"); + return; + } + + if (msg->vp_count == 0) { + pr_debug("scheduler message with no VP specified"); + return; + } + + rcu_read_lock(); + + partition = mshv_partition_find(partition_id); + if (unlikely(!partition)) { + pr_debug("failed to find partition %llu\n", partition_id); + goto unlock_out; + } + + vpset = &msg->vp_bitset.bitset; + + bank_idx = -1; + bank_contents = vpset->bank_contents; + bank_mask_size = sizeof(vpset->valid_bank_mask) * BITS_PER_BYTE; + + while (true) { + int vp_bank_idx = -1; + int vp_bank_size = sizeof(*bank_contents) * BITS_PER_BYTE; + int vp_index; + + bank_idx = find_next_bit((unsigned long *)&vpset->valid_bank_mask, + bank_mask_size, bank_idx + 1); + if (bank_idx == bank_mask_size) + break; + + while (true) { + struct mshv_vp *vp; + + vp_bank_idx = find_next_bit((unsigned long *)bank_contents, + vp_bank_size, vp_bank_idx + 1); + if (vp_bank_idx == vp_bank_size) + break; + + vp_index = (bank_idx * vp_bank_size) + vp_bank_idx; + + /* This shouldn't happen, but just in case. */ + if (unlikely(vp_index >= MSHV_MAX_VPS)) { + pr_debug("VP index %u out of bounds\n", + vp_index); + goto unlock_out; + } + + vp = partition->pt_vp_array[vp_index]; + if (unlikely(!vp)) { + pr_debug("failed to find VP %u\n", vp_index); + goto unlock_out; + } + + kick_vp(vp); + vps_signaled++; + } + + bank_contents++; + } + +unlock_out: + rcu_read_unlock(); + + if (vps_signaled != msg->vp_count) + pr_debug("asked to signal %u VPs but only did %u\n", + msg->vp_count, vps_signaled); +} + +static void +handle_pair_message(const struct hv_vp_signal_pair_scheduler_message *msg) +{ + struct mshv_partition *partition = NULL; + struct mshv_vp *vp; + int idx; + + rcu_read_lock(); + + for (idx = 0; idx < msg->vp_count; idx++) { + u64 partition_id = msg->partition_ids[idx]; + u32 vp_index = msg->vp_indexes[idx]; + + if (idx == 0 || partition->pt_id != partition_id) { + partition = mshv_partition_find(partition_id); + if (unlikely(!partition)) { + pr_debug("failed to find partition %llu\n", + partition_id); + break; + } + } + + /* This shouldn't happen, but just in case. */ + if (unlikely(vp_index >= MSHV_MAX_VPS)) { + pr_debug("VP index %u out of bounds\n", vp_index); + break; + } + + vp = partition->pt_vp_array[vp_index]; + if (!vp) { + pr_debug("failed to find VP %u\n", vp_index); + break; + } + + kick_vp(vp); + } + + rcu_read_unlock(); +} + +static bool +mshv_scheduler_isr(struct hv_message *msg) +{ + if (msg->header.message_type != HVMSG_SCHEDULER_VP_SIGNAL_BITSET && + msg->header.message_type != HVMSG_SCHEDULER_VP_SIGNAL_PAIR) + return false; + + if (msg->header.message_type == HVMSG_SCHEDULER_VP_SIGNAL_BITSET) + handle_bitset_message((struct hv_vp_signal_bitset_scheduler_message *) + msg->u.payload); + else + handle_pair_message((struct hv_vp_signal_pair_scheduler_message *) + msg->u.payload); + + return true; +} + +static bool +mshv_intercept_isr(struct hv_message *msg) +{ + struct mshv_partition *partition; + bool handled = false; + struct mshv_vp *vp; + u64 partition_id; + u32 vp_index; + + partition_id = msg->header.sender; + + rcu_read_lock(); + + partition = mshv_partition_find(partition_id); + if (unlikely(!partition)) { + pr_debug("failed to find partition %llu\n", + partition_id); + goto unlock_out; + } + + if (msg->header.message_type == HVMSG_X64_APIC_EOI) { + /* + * Check if this gsi is registered in the + * ack_notifier list and invoke the callback + * if registered. + */ + + /* + * If there is a notifier, the ack callback is supposed + * to handle the VMEXIT. So we need not pass this message + * to vcpu thread. + */ + struct hv_x64_apic_eoi_message *eoi_msg = + (struct hv_x64_apic_eoi_message *)&msg->u.payload[0]; + + if (mshv_notify_acked_gsi(partition, eoi_msg->interrupt_vector)) { + handled = true; + goto unlock_out; + } + } + + /* + * We should get an opaque intercept message here for all intercept + * messages, since we're using the mapped VP intercept message page. + * + * The intercept message will have been placed in intercept message + * page at this point. + * + * Make sure the message type matches our expectation. + */ + if (msg->header.message_type != HVMSG_OPAQUE_INTERCEPT) { + pr_debug("wrong message type %d", msg->header.message_type); + goto unlock_out; + } + + /* + * Since we directly index the vp, and it has to exist for us to be here + * (because the vp is only deleted when the partition is), no additional + * locking is needed here + */ + vp_index = + ((struct hv_opaque_intercept_message *)msg->u.payload)->vp_index; + /* This shouldn't happen, but just in case. */ + if (unlikely(vp_index >= MSHV_MAX_VPS)) { + pr_debug("VP index %u out of bounds\n", vp_index); + goto unlock_out; + } + vp = partition->pt_vp_array[vp_index]; + if (unlikely(!vp)) { + pr_debug("failed to find VP %u\n", vp_index); + goto unlock_out; + } + + kick_vp(vp); + + handled = true; + +unlock_out: + rcu_read_unlock(); + + return handled; +} + +void mshv_isr(void) +{ + struct hv_synic_pages *spages = this_cpu_ptr(synic_pages); + struct hv_message_page **msg_page = &spages->hyp_synic_message_page; + struct hv_message *msg; + bool handled; + + if (unlikely(!(*msg_page))) { + pr_debug("Missing synic page!\n"); + return; + } + + msg = &((*msg_page)->sint_message[HV_SYNIC_INTERCEPTION_SINT_INDEX]); + + /* + * If the type isn't set, there isn't really a message; + * it may be some other hyperv interrupt + */ + if (msg->header.message_type == HVMSG_NONE) + return; + + handled = mshv_doorbell_isr(msg); + + if (!handled) + handled = mshv_scheduler_isr(msg); + + if (!handled) + handled = mshv_async_call_completion_isr(msg); + + if (!handled) + handled = mshv_intercept_isr(msg); + + if (handled) { + /* + * Acknowledge message with hypervisor if another message is + * pending. + */ + msg->header.message_type = HVMSG_NONE; + /* + * Ensure the write is complete so the hypervisor will deliver + * the next message if available. + */ + mb(); + if (msg->header.message_flags.msg_pending) + hv_set_non_nested_msr(HV_MSR_EOM, 0); + } else { + pr_warn_once("%s: unknown message type 0x%x\n", __func__, + msg->header.message_type); + } +} + +static int mshv_synic_cpu_init(unsigned int cpu) +{ + union hv_synic_simp simp; + union hv_synic_siefp siefp; + union hv_synic_sirbp sirbp; + union hv_synic_sint sint; + struct hv_synic_pages *spages = this_cpu_ptr(synic_pages); + struct hv_message_page **msg_page = &spages->hyp_synic_message_page; + struct hv_synic_event_flags_page **event_flags_page = + &spages->synic_event_flags_page; + struct hv_synic_event_ring_page **event_ring_page = + &spages->synic_event_ring_page; + /* + * VMBus owns SIMP/SIEFP/SCONTROL when it is active. + * See hv_hyp_synic_enable_regs() for that initialization. + */ + bool vmbus_active = hv_vmbus_exists(); + + /* + * Map the SYNIC message page. When VMBus is not active the + * hypervisor pre-provisions the SIMP GPA but may not set + * simp_enabled — enable it here. + */ + simp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIMP); + if (!vmbus_active) { + simp.simp_enabled = true; + hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64); + } + *msg_page = memremap(simp.base_simp_gpa << HV_HYP_PAGE_SHIFT, + HV_HYP_PAGE_SIZE, + MEMREMAP_WB); + + if (!(*msg_page)) + goto cleanup_simp; + + /* + * Map the event flags page. Same as SIMP: enable when + * VMBus is not active, already enabled by VMBus otherwise. + */ + siefp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIEFP); + if (!vmbus_active) { + siefp.siefp_enabled = true; + hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64); + } + *event_flags_page = memremap(siefp.base_siefp_gpa << HV_HYP_PAGE_SHIFT, + HV_HYP_PAGE_SIZE, MEMREMAP_WB); + + if (!(*event_flags_page)) + goto cleanup_siefp; + + /* Setup the Synic's event ring page */ + sirbp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIRBP); + + if (hv_root_partition()) { + *event_ring_page = memremap(sirbp.base_sirbp_gpa << HV_HYP_PAGE_SHIFT, + HV_HYP_PAGE_SIZE, MEMREMAP_WB); + + if (!(*event_ring_page)) + goto cleanup_siefp; + } else { + /* + * On L1VH the hypervisor does not provide a SIRBP page. + * Allocate one and program its GPA into the MSR. + */ + *event_ring_page = (struct hv_synic_event_ring_page *) + get_zeroed_page(GFP_KERNEL); + + if (!(*event_ring_page)) + goto cleanup_siefp; + + sirbp.base_sirbp_gpa = virt_to_phys(*event_ring_page) + >> HV_HYP_PAGE_SHIFT; + } + + sirbp.sirbp_enabled = true; + hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64); + + if (mshv_sint_irq != -1) + enable_percpu_irq(mshv_sint_irq, 0); + + /* Enable intercepts */ + sint.as_uint64 = 0; + sint.vector = mshv_sint_vector; + sint.masked = false; + sint.auto_eoi = hv_recommend_using_aeoi(); + hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX, + sint.as_uint64); + + /* Doorbell SINT */ + sint.as_uint64 = 0; + sint.vector = mshv_sint_vector; + sint.masked = false; + sint.as_intercept = 1; + sint.auto_eoi = hv_recommend_using_aeoi(); + hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_DOORBELL_SINT_INDEX, + sint.as_uint64); + + /* When VMBus is active it already enabled SCONTROL. */ + if (!vmbus_active) { + union hv_synic_scontrol sctrl; + + sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL); + sctrl.enable = 1; + hv_set_non_nested_msr(HV_MSR_SCONTROL, sctrl.as_uint64); + } + + return 0; + +cleanup_siefp: + if (*event_flags_page) + memunmap(*event_flags_page); + if (!vmbus_active) { + siefp.siefp_enabled = false; + hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64); + } +cleanup_simp: + if (*msg_page) + memunmap(*msg_page); + if (!vmbus_active) { + simp.simp_enabled = false; + hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64); + } + + return -EFAULT; +} + +static int mshv_synic_cpu_exit(unsigned int cpu) +{ + union hv_synic_sint sint; + union hv_synic_sirbp sirbp; + struct hv_synic_pages *spages = this_cpu_ptr(synic_pages); + struct hv_message_page **msg_page = &spages->hyp_synic_message_page; + struct hv_synic_event_flags_page **event_flags_page = + &spages->synic_event_flags_page; + struct hv_synic_event_ring_page **event_ring_page = + &spages->synic_event_ring_page; + /* VMBus owns SIMP/SIEFP/SCONTROL when it is active */ + bool vmbus_active = hv_vmbus_exists(); + + /* Disable the interrupt */ + sint.as_uint64 = hv_get_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX); + sint.masked = true; + hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX, + sint.as_uint64); + + /* Disable Doorbell SINT */ + sint.as_uint64 = hv_get_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_DOORBELL_SINT_INDEX); + sint.masked = true; + hv_set_non_nested_msr(HV_MSR_SINT0 + HV_SYNIC_DOORBELL_SINT_INDEX, + sint.as_uint64); + + if (mshv_sint_irq != -1) + disable_percpu_irq(mshv_sint_irq); + + /* Disable SYNIC event ring page owned by MSHV */ + sirbp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIRBP); + sirbp.sirbp_enabled = false; + + if (hv_root_partition()) { + hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64); + memunmap(*event_ring_page); + } else { + sirbp.base_sirbp_gpa = 0; + hv_set_non_nested_msr(HV_MSR_SIRBP, sirbp.as_uint64); + free_page((unsigned long)*event_ring_page); + } + + /* + * Release our mappings of the message and event flags pages. + * When VMBus is not active, we enabled SIMP/SIEFP — disable + * them. Otherwise VMBus owns the MSRs — leave them. + */ + memunmap(*event_flags_page); + if (!vmbus_active) { + union hv_synic_simp simp; + union hv_synic_siefp siefp; + + siefp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIEFP); + siefp.siefp_enabled = false; + hv_set_non_nested_msr(HV_MSR_SIEFP, siefp.as_uint64); + + simp.as_uint64 = hv_get_non_nested_msr(HV_MSR_SIMP); + simp.simp_enabled = false; + hv_set_non_nested_msr(HV_MSR_SIMP, simp.as_uint64); + } + memunmap(*msg_page); + + /* When VMBus is active it owns SCONTROL — leave it. */ + if (!vmbus_active) { + union hv_synic_scontrol sctrl; + + sctrl.as_uint64 = hv_get_non_nested_msr(HV_MSR_SCONTROL); + sctrl.enable = 0; + hv_set_non_nested_msr(HV_MSR_SCONTROL, sctrl.as_uint64); + } + + return 0; +} + +int +mshv_register_doorbell(u64 partition_id, doorbell_cb_t doorbell_cb, void *data, + u64 gpa, u64 val, u64 flags) +{ + struct hv_connection_info connection_info = { 0 }; + union hv_connection_id connection_id = { 0 }; + struct port_table_info *port_table_info; + struct hv_port_info port_info = { 0 }; + union hv_port_id port_id = { 0 }; + int ret; + + port_table_info = kmalloc_obj(*port_table_info); + if (!port_table_info) + return -ENOMEM; + + port_table_info->hv_port_type = HV_PORT_TYPE_DOORBELL; + port_table_info->hv_port_doorbell.doorbell_cb = doorbell_cb; + port_table_info->hv_port_doorbell.data = data; + ret = mshv_portid_alloc(port_table_info); + if (ret < 0) { + kfree(port_table_info); + return ret; + } + + port_id.u.id = ret; + port_info.port_type = HV_PORT_TYPE_DOORBELL; + port_info.doorbell_port_info.target_sint = HV_SYNIC_DOORBELL_SINT_INDEX; + port_info.doorbell_port_info.target_vp = HV_ANY_VP; + ret = hv_call_create_port(hv_current_partition_id, port_id, partition_id, + &port_info, + 0, 0, NUMA_NO_NODE); + + if (ret < 0) { + mshv_portid_free(port_id.u.id); + return ret; + } + + connection_id.u.id = port_id.u.id; + connection_info.port_type = HV_PORT_TYPE_DOORBELL; + connection_info.doorbell_connection_info.gpa = gpa; + connection_info.doorbell_connection_info.trigger_value = val; + connection_info.doorbell_connection_info.flags = flags; + + ret = hv_call_connect_port(hv_current_partition_id, port_id, partition_id, + connection_id, &connection_info, 0, NUMA_NO_NODE); + if (ret < 0) { + hv_call_delete_port(hv_current_partition_id, port_id); + mshv_portid_free(port_id.u.id); + return ret; + } + + // lets use the port_id as the doorbell_id + return port_id.u.id; +} + +void +mshv_unregister_doorbell(u64 partition_id, int doorbell_portid) +{ + union hv_port_id port_id = { 0 }; + union hv_connection_id connection_id = { 0 }; + + connection_id.u.id = doorbell_portid; + hv_call_disconnect_port(partition_id, connection_id); + + port_id.u.id = doorbell_portid; + hv_call_delete_port(hv_current_partition_id, port_id); + + mshv_portid_free(doorbell_portid); +} + +static int mshv_synic_reboot_notify(struct notifier_block *nb, + unsigned long code, void *unused) +{ + mshv_debugfs_exit(); + cpuhp_remove_state(synic_cpuhp_online); + return 0; +} + +static struct notifier_block mshv_synic_reboot_nb = { + .notifier_call = mshv_synic_reboot_notify, +}; + +#ifndef HYPERVISOR_CALLBACK_VECTOR +static DEFINE_PER_CPU(long, mshv_evt); + +static irqreturn_t mshv_percpu_isr(int irq, void *dev_id) +{ + mshv_isr(); + return IRQ_HANDLED; +} + +#ifdef CONFIG_ACPI +static int __init mshv_acpi_setup_sint_irq(void) +{ + return acpi_register_gsi(NULL, mshv_sint_vector, ACPI_EDGE_SENSITIVE, + ACPI_ACTIVE_HIGH); +} + +static void mshv_acpi_cleanup_sint_irq(void) +{ + acpi_unregister_gsi(mshv_sint_vector); +} +#else +static int __init mshv_acpi_setup_sint_irq(void) +{ + return -ENODEV; +} + +static void mshv_acpi_cleanup_sint_irq(void) +{ +} +#endif + +static int __init mshv_sint_vector_setup(void) +{ + int ret; + struct hv_register_assoc reg = { + .name = HV_ARM64_REGISTER_SINT_RESERVED_INTERRUPT_ID, + }; + union hv_input_vtl input_vtl = { 0 }; + + if (acpi_disabled) + return -ENODEV; + + ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, + 1, input_vtl, ®); + if (ret || !reg.value.reg64) + return -ENODEV; + + mshv_sint_vector = reg.value.reg64; + ret = mshv_acpi_setup_sint_irq(); + if (ret < 0) { + pr_err("Failed to setup IRQ for MSHV SINT vector %d: %d\n", + mshv_sint_vector, ret); + goto out_fail; + } + + mshv_sint_irq = ret; + + ret = request_percpu_irq(mshv_sint_irq, mshv_percpu_isr, "MSHV", + &mshv_evt); + if (ret) + goto out_unregister; + + return 0; + +out_unregister: + mshv_acpi_cleanup_sint_irq(); +out_fail: + return ret; +} + +static void mshv_sint_vector_cleanup(void) +{ + free_percpu_irq(mshv_sint_irq, &mshv_evt); + mshv_acpi_cleanup_sint_irq(); +} +#else /* !HYPERVISOR_CALLBACK_VECTOR */ +static int __init mshv_sint_vector_setup(void) +{ + mshv_sint_vector = HYPERVISOR_CALLBACK_VECTOR; + return 0; +} + +static void mshv_sint_vector_cleanup(void) +{ +} +#endif /* HYPERVISOR_CALLBACK_VECTOR */ + +int __init mshv_synic_init(struct device *dev) +{ + int ret = 0; + + ret = mshv_sint_vector_setup(); + if (ret) + return ret; + + synic_pages = alloc_percpu(struct hv_synic_pages); + if (!synic_pages) { + dev_err(dev, "Failed to allocate percpu synic page\n"); + ret = -ENOMEM; + goto sint_vector_cleanup; + } + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mshv_synic", + mshv_synic_cpu_init, + mshv_synic_cpu_exit); + if (ret < 0) { + dev_err(dev, "Failed to setup cpu hotplug state: %i\n", ret); + goto free_synic_pages; + } + + synic_cpuhp_online = ret; + + ret = register_reboot_notifier(&mshv_synic_reboot_nb); + if (ret) + goto remove_cpuhp_state; + + return 0; + +remove_cpuhp_state: + cpuhp_remove_state(synic_cpuhp_online); +free_synic_pages: + free_percpu(synic_pages); +sint_vector_cleanup: + mshv_sint_vector_cleanup(); + return ret; +} + +void mshv_synic_exit(void) +{ + unregister_reboot_notifier(&mshv_synic_reboot_nb); + cpuhp_remove_state(synic_cpuhp_online); + free_percpu(synic_pages); + mshv_sint_vector_cleanup(); +} diff --git a/drivers/hv/mshv_trace.c b/drivers/hv/mshv_trace.c new file mode 100644 index 000000000..0936b2f95 --- /dev/null +++ b/drivers/hv/mshv_trace.c @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2026, Microsoft Corporation. + * + * Tracepoint definitions for mshv driver. + */ + +#define CREATE_TRACE_POINTS +#include "mshv_trace.h" diff --git a/drivers/hv/mshv_trace.h b/drivers/hv/mshv_trace.h new file mode 100644 index 000000000..e7280c47e --- /dev/null +++ b/drivers/hv/mshv_trace.h @@ -0,0 +1,544 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2026, Microsoft Corporation. + * + * Tracepoint declarations for mshv driver. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM mshv + +#if !defined(__MSHV_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _MSHV_TRACE_H_ + +#include <linux/tracepoint.h> +#include <hyperv/hvhdk.h> + +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH ../../drivers/hv + +#undef TRACE_INCLUDE_FILE +#define TRACE_INCLUDE_FILE mshv_trace + +TRACE_EVENT(mshv_create_partition, + TP_PROTO(u64 partition_id, int vm_fd), + TP_ARGS(partition_id, vm_fd), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(int, vm_fd) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vm_fd = vm_fd; + ), + TP_printk("partition_id=%llu vm_fd=%d", + __entry->partition_id, + __entry->vm_fd + ) +); + +TRACE_EVENT(mshv_hvcall_create_partition, + TP_PROTO(u64 flags, s64 partition_id), + TP_ARGS(flags, partition_id), + TP_STRUCT__entry( + __field(u64, flags) + __field(s64, partition_id) + ), + TP_fast_assign( + __entry->flags = flags; + __entry->partition_id = partition_id; + ), + TP_printk("flags=%#llx partition_id=%lld", + __entry->flags, + __entry->partition_id + ) +); + +TRACE_EVENT(mshv_hvcall_initialize_partition, + TP_PROTO(u64 partition_id, u64 status), + TP_ARGS(partition_id, status), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, status) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->status = status; + ), + TP_printk("partition_id=%llu status=%#llx", + __entry->partition_id, + __entry->status + ) +); + +TRACE_EVENT(mshv_partition_release, + TP_PROTO(u64 partition_id), + TP_ARGS(partition_id), + TP_STRUCT__entry( + __field(u64, partition_id) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + ), + TP_printk("partition_id=%llu", + __entry->partition_id + ) +); + +TRACE_EVENT(mshv_destroy_partition, + TP_PROTO(u64 partition_id), + TP_ARGS(partition_id), + TP_STRUCT__entry( + __field(u64, partition_id) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + ), + TP_printk("partition_id=%llu", + __entry->partition_id + ) +); + +TRACE_EVENT(mshv_hvcall_finalize_partition, + TP_PROTO(u64 partition_id, u64 status), + TP_ARGS(partition_id, status), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, status) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->status = status; + ), + TP_printk("partition_id=%llu status=%#llx ", + __entry->partition_id, + __entry->status + ) +); + +TRACE_EVENT(mshv_hvcall_withdraw_memory, + TP_PROTO(u64 partition_id, u64 withdrawn, u64 status), + TP_ARGS(partition_id, withdrawn, status), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, withdrawn) + __field(u64, status) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->withdrawn = withdrawn; + __entry->status = status; + ), + TP_printk("partition_id=%llu withdrawn=%llu status=%#llx", + __entry->partition_id, + __entry->withdrawn, + __entry->status + ) +); + +TRACE_EVENT(mshv_hvcall_delete_partition, + TP_PROTO(u64 partition_id, u64 status), + TP_ARGS(partition_id, status), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, status) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->status = status; + ), + TP_printk("partition_id=%llu status=%#llx", + __entry->partition_id, + __entry->status + ) +); + +TRACE_EVENT(mshv_create_vp, + TP_PROTO(u64 partition_id, u32 vp_index, long vp_fd), + TP_ARGS(partition_id, vp_index, vp_fd), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(long, vp_fd) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->vp_fd = vp_fd; + ), + TP_printk("partition_id=%llu vp_index=%u vp_fd=%ld", + __entry->partition_id, + __entry->vp_index, + __entry->vp_fd + ) +); + +TRACE_EVENT(mshv_hvcall_map_vp_state_page, + TP_PROTO(u64 partition_id, u32 vp_index, u32 page_type, u64 status), + TP_ARGS(partition_id, vp_index, page_type, status), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(u32, page_type) + __field(u64, status) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->page_type = page_type; + __entry->status = status; + ), + TP_printk("partition_id=%llu vp_index=%u page_type=%u status=%#llx", + __entry->partition_id, + __entry->vp_index, + __entry->page_type, + __entry->status + ) +); + +TRACE_EVENT(mshv_drain_vp_signals, + TP_PROTO(u64 partition_id, u32 vp_index), + TP_ARGS(partition_id, vp_index), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + ), + TP_printk("partition_id=%llu vp_index=%u", + __entry->partition_id, + __entry->vp_index + ) +); + +TRACE_EVENT(mshv_disable_vp_dispatch, + TP_PROTO(u64 partition_id, u32 vp_index, int ret), + TP_ARGS(partition_id, vp_index, ret), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(int, ret) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->ret = ret; + ), + TP_printk("partition_id=%llu vp_index=%u ret=%d", + __entry->partition_id, + __entry->vp_index, + __entry->ret + ) +); + +TRACE_EVENT(mshv_vp_release, + TP_PROTO(u64 partition_id, u32 vp_index), + TP_ARGS(partition_id, vp_index), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + ), + TP_printk("partition_id=%llu vp_index=%u", + __entry->partition_id, + __entry->vp_index + ) +); + +TRACE_EVENT(mshv_run_vp_entry, + TP_PROTO(u64 partition_id, u32 vp_index), + TP_ARGS(partition_id, vp_index), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + ), + TP_printk("partition_id=%llu vp_index=%u", + __entry->partition_id, + __entry->vp_index + ) +); + +TRACE_EVENT(mshv_run_vp_exit, + TP_PROTO(u64 partition_id, u32 vp_index, u64 hv_message_type, long ret), + TP_ARGS(partition_id, vp_index, hv_message_type, ret), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(u64, hv_message_type) + __field(long, ret) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->hv_message_type = hv_message_type; + __entry->ret = ret; + ), + TP_printk("partition_id=%llu vp_index=%u hv_message_type=%#llx ret=%ld", + __entry->partition_id, + __entry->vp_index, + __entry->hv_message_type, + __entry->ret + ) +); + +TRACE_EVENT(mshv_vp_clear_explicit_suspend, + TP_PROTO(u64 partition_id, u32 vp_index, int ret), + TP_ARGS(partition_id, vp_index, ret), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(int, ret) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->ret = ret; + ), + TP_printk("partition_id=%llu vp_index=%u ret=%d", + __entry->partition_id, + __entry->vp_index, + __entry->ret + ) +); + +TRACE_EVENT(mshv_xfer_to_guest_mode_work, + TP_PROTO(u64 partition_id, u32 vp_index, unsigned long thread_info_flag, long ret), + TP_ARGS(partition_id, vp_index, thread_info_flag, ret), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(unsigned long, thread_info_flag) + __field(long, ret) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->thread_info_flag = thread_info_flag; + __entry->ret = ret; + ), + TP_printk("partition_id=%llu vp_index=%u thread_info_flag=%#lx ret=%ld", + __entry->partition_id, + __entry->vp_index, + __entry->thread_info_flag, + __entry->ret + ) +); + +TRACE_EVENT(mshv_hvcall_dispatch_vp, + TP_PROTO(u64 partition_id, u32 vp_index, u32 flags, + u32 dispatch_state, u32 dispatch_event, u64 irq_vectors, u64 status), + TP_ARGS(partition_id, vp_index, flags, dispatch_state, dispatch_event, irq_vectors, + status), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(u32, flags) + __field(u32, dispatch_state) + __field(u32, dispatch_event) + __field(u64, irq_vectors) + __field(u64, status) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->flags = flags; + __entry->dispatch_state = dispatch_state; + __entry->dispatch_event = dispatch_event; + __entry->irq_vectors = irq_vectors; + __entry->status = status; + ), + TP_printk("partition_id=%llu vp_index=%u flags=%#x dispatch_state=%#x dispatch_event=%#x irq_vectors=%#016llx status=%#llx", + __entry->partition_id, + __entry->vp_index, + __entry->flags, + __entry->dispatch_state, + __entry->dispatch_event, + __entry->irq_vectors, + __entry->status + ) +); + +TRACE_EVENT(mshv_update_routing_table, + TP_PROTO(u64 partition_id, void *old, void *new, u32 numents), + TP_ARGS(partition_id, old, new, numents), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(struct mshv_girq_routing_table *, old) + __field(struct mshv_girq_routing_table *, new) + __field(u32, numents) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->old = old; + __entry->new = new; + __entry->numents = numents; + ), + TP_printk("partition_id=%llu old=%p new=%p numents=%u", + __entry->partition_id, + __entry->old, + __entry->new, + __entry->numents + ) +); + +TRACE_EVENT(mshv_map_user_memory, + TP_PROTO(u64 partition_id, u64 start_uaddr, u64 start_gfn, u64 nr_pages, u32 map_flags, + long ret), + TP_ARGS(partition_id, start_uaddr, start_gfn, nr_pages, map_flags, ret), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, start_uaddr) + __field(u64, start_gfn) + __field(u64, nr_pages) + __field(u32, map_flags) + __field(long, ret) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->start_uaddr = start_uaddr; + __entry->start_gfn = start_gfn; + __entry->nr_pages = nr_pages; + __entry->map_flags = map_flags; + __entry->ret = ret; + ), + TP_printk("partition_id=%llu start_uaddr=%#llx start_gfn=%#llx nr_pages=%llu map_flags=%#x ret=%ld", + __entry->partition_id, + __entry->start_uaddr, + __entry->start_gfn, + __entry->nr_pages, + __entry->map_flags, + __entry->ret + ) +); + +TRACE_EVENT(mshv_assign_ioeventfd, + TP_PROTO(u64 partition_id, u64 addr, u64 length, u64 datamatch, bool wildcard, + void *eventfd, int ret), + TP_ARGS(partition_id, addr, length, datamatch, wildcard, eventfd, ret), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, addr) + __field(u64, length) + __field(u64, datamatch) + __field(bool, wildcard) + __field(struct eventfd_ctx *, eventfd) + __field(int, ret) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->addr = addr; + __entry->length = length; + __entry->datamatch = datamatch; + __entry->wildcard = wildcard; + __entry->eventfd = eventfd; + __entry->ret = ret; + ), + TP_printk("partition_id=%llu addr=%#016llx length=%#llx datamatch=%#llx wildcard=%d eventfd=%p ret=%d", + __entry->partition_id, + __entry->addr, + __entry->length, + __entry->datamatch, + __entry->wildcard, + __entry->eventfd, + __entry->ret + ) +); + +TRACE_EVENT(mshv_deassign_ioeventfd, + TP_PROTO(u64 partition_id, u64 addr, u64 length, u64 datamatch, bool wildcard, + void *eventfd), + TP_ARGS(partition_id, addr, length, datamatch, wildcard, eventfd), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u64, addr) + __field(u64, length) + __field(u64, datamatch) + __field(bool, wildcard) + __field(struct eventfd_ctx *, eventfd) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->addr = addr; + __entry->length = length; + __entry->datamatch = datamatch; + __entry->wildcard = wildcard; + __entry->eventfd = eventfd; + ), + TP_printk("partition_id=%llu addr=%#016llx length=%#llx datamatch=%#llx wildcard=%d eventfd=%p", + __entry->partition_id, + __entry->addr, + __entry->length, + __entry->datamatch, + __entry->wildcard, + __entry->eventfd + ) +); + +TRACE_EVENT(mshv_vp_wait_for_hv_kick, + TP_PROTO(u64 partition_id, u32 vp_index, bool kicked_by_hv, bool blocked, + bool irq_pending), + TP_ARGS(partition_id, vp_index, kicked_by_hv, blocked, irq_pending), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(bool, kicked_by_hv) + __field(bool, blocked) + __field(bool, irq_pending) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->kicked_by_hv = kicked_by_hv; + __entry->blocked = blocked; + __entry->irq_pending = irq_pending; + ), + TP_printk("partition_id=%llu vp_index=%u kicked_by_hv=%d blocked=%d irq_pending=%d", + __entry->partition_id, + __entry->vp_index, + __entry->kicked_by_hv, + __entry->blocked, + __entry->irq_pending + ) +); + +TRACE_EVENT(mshv_handle_gpa_intercept, + TP_PROTO(u64 partition_id, u32 vp_index, u64 gfn, u8 access_type, bool handled), + TP_ARGS(partition_id, vp_index, gfn, access_type, handled), + TP_STRUCT__entry( + __field(u64, partition_id) + __field(u32, vp_index) + __field(u64, gfn) + __field(u8, access_type) + __field(bool, handled) + ), + TP_fast_assign( + __entry->partition_id = partition_id; + __entry->vp_index = vp_index; + __entry->gfn = gfn; + __entry->access_type = access_type == HV_INTERCEPT_ACCESS_READ ? 'R' : + (access_type == HV_INTERCEPT_ACCESS_WRITE ? 'W' : + (access_type == HV_INTERCEPT_ACCESS_EXECUTE ? 'X' : '?')); + __entry->handled = handled; + ), + TP_printk("partition_id=%llu vp_index=%u gfn=0x%llx access_type=%c handled=%d", + __entry->partition_id, + __entry->vp_index, + __entry->gfn, + __entry->access_type, + __entry->handled + ) +); + +#endif /* _MSHV_TRACE_H_ */ + +/* This part must be outside protection */ +#include <trace/define_trace.h> diff --git a/drivers/hv/mshv_vtl.h b/drivers/hv/mshv_vtl.h new file mode 100644 index 000000000..a6eea52f7 --- /dev/null +++ b/drivers/hv/mshv_vtl.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +#ifndef _MSHV_VTL_H +#define _MSHV_VTL_H + +#include <linux/mshv.h> +#include <linux/types.h> + +struct mshv_vtl_run { + u32 cancel; + u32 vtl_ret_action_size; + u32 pad[2]; + char exit_message[MSHV_MAX_RUN_MSG_SIZE]; + union { + struct mshv_vtl_cpu_context cpu_context; + + /* + * Reserving room for the cpu context to grow and to maintain compatibility + * with user mode. + */ + char reserved[1024]; + }; + char vtl_ret_actions[MSHV_MAX_RUN_MSG_SIZE]; +}; + +#endif /* _MSHV_VTL_H */ diff --git a/drivers/hv/mshv_vtl_main.c b/drivers/hv/mshv_vtl_main.c new file mode 100644 index 000000000..0d3d41619 --- /dev/null +++ b/drivers/hv/mshv_vtl_main.c @@ -0,0 +1,1399 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2023, Microsoft Corporation. + * + * Author: + * Roman Kisel <romank@linux.microsoft.com> + * Saurabh Sengar <ssengar@linux.microsoft.com> + * Naman Jain <namjain@linux.microsoft.com> + */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/miscdevice.h> +#include <linux/anon_inodes.h> +#include <linux/cpuhotplug.h> +#include <linux/count_zeros.h> +#include <linux/entry-virt.h> +#include <linux/eventfd.h> +#include <linux/poll.h> +#include <linux/file.h> +#include <linux/vmalloc.h> +#include <asm/debugreg.h> +#include <asm/mshyperv.h> +#include <trace/events/ipi.h> +#include <uapi/asm/mtrr.h> +#include <uapi/linux/mshv.h> +#include <hyperv/hvhdk.h> + +#include "../../kernel/fpu/legacy.h" +#include "mshv.h" +#include "mshv_vtl.h" +#include "hyperv_vmbus.h" + +MODULE_AUTHOR("Microsoft"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microsoft Hyper-V VTL Driver"); + +#define MSHV_ENTRY_REASON_LOWER_VTL_CALL 0x1 +#define MSHV_ENTRY_REASON_INTERRUPT 0x2 +#define MSHV_ENTRY_REASON_INTERCEPT 0x3 + +#define MSHV_REAL_OFF_SHIFT 16 +#define MSHV_PG_OFF_CPU_MASK (BIT_ULL(MSHV_REAL_OFF_SHIFT) - 1) +#define MSHV_RUN_PAGE_OFFSET 0 +#define MSHV_REG_PAGE_OFFSET 1 +#define VTL2_VMBUS_SINT_INDEX 7 + +static struct device *mem_dev; + +static struct tasklet_struct msg_dpc; +static wait_queue_head_t fd_wait_queue; +static bool has_message; +static struct eventfd_ctx *flag_eventfds[HV_EVENT_FLAGS_COUNT]; +static DEFINE_MUTEX(flag_lock); +static bool __read_mostly mshv_has_reg_page; + +/* hvcall code is of type u16, allocate a bitmap of size (1 << 16) to accommodate it */ +#define MAX_BITMAP_SIZE ((U16_MAX + 1) / 8) + +struct mshv_vtl_hvcall_fd { + u8 allow_bitmap[MAX_BITMAP_SIZE]; + bool allow_map_initialized; + /* + * Used to protect hvcall setup in IOCTLs + */ + struct mutex init_mutex; + struct miscdevice *dev; +}; + +struct mshv_vtl_poll_file { + struct file *file; + wait_queue_entry_t wait; + wait_queue_head_t *wqh; + poll_table pt; + int cpu; +}; + +struct mshv_vtl { + struct device *module_dev; + u64 id; +}; + +struct mshv_vtl_per_cpu { + struct mshv_vtl_run *run; + struct page *reg_page; +}; + +/* SYNIC_OVERLAY_PAGE_MSR - internal, identical to hv_synic_simp */ +union hv_synic_overlay_page_msr { + u64 as_uint64; + struct { + u64 enabled: 1; + u64 reserved: 11; + u64 pfn: 52; + } __packed; +}; + +static struct mutex mshv_vtl_poll_file_lock; +static union hv_register_vsm_page_offsets mshv_vsm_page_offsets; +static union hv_register_vsm_capabilities mshv_vsm_capabilities; + +static DEFINE_PER_CPU(struct mshv_vtl_poll_file, mshv_vtl_poll_file); +static DEFINE_PER_CPU(unsigned long long, num_vtl0_transitions); +static DEFINE_PER_CPU(struct mshv_vtl_per_cpu, mshv_vtl_per_cpu); + +static const union hv_input_vtl input_vtl_zero; +static const union hv_input_vtl input_vtl_normal = { + .use_target_vtl = 1, +}; + +static const struct file_operations mshv_vtl_fops; + +static long +mshv_ioctl_create_vtl(void __user *user_arg, struct device *module_dev) +{ + struct mshv_vtl *vtl; + struct file *file; + int fd; + + vtl = kzalloc_obj(*vtl); + if (!vtl) + return -ENOMEM; + + fd = get_unused_fd_flags(O_CLOEXEC); + if (fd < 0) { + kfree(vtl); + return fd; + } + file = anon_inode_getfile("mshv_vtl", &mshv_vtl_fops, + vtl, O_RDWR); + if (IS_ERR(file)) { + kfree(vtl); + return PTR_ERR(file); + } + vtl->module_dev = module_dev; + fd_install(fd, file); + + return fd; +} + +static long +mshv_ioctl_check_extension(void __user *user_arg) +{ + u32 arg; + + if (copy_from_user(&arg, user_arg, sizeof(arg))) + return -EFAULT; + + switch (arg) { + case MSHV_CAP_CORE_API_STABLE: + return 0; + case MSHV_CAP_REGISTER_PAGE: + return mshv_has_reg_page; + case MSHV_CAP_VTL_RETURN_ACTION: + return mshv_vsm_capabilities.return_action_available; + case MSHV_CAP_DR6_SHARED: + return mshv_vsm_capabilities.dr6_shared; + } + + return -EOPNOTSUPP; +} + +static long +mshv_dev_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + struct miscdevice *misc = filp->private_data; + + switch (ioctl) { + case MSHV_CHECK_EXTENSION: + return mshv_ioctl_check_extension((void __user *)arg); + case MSHV_CREATE_VTL: + return mshv_ioctl_create_vtl((void __user *)arg, misc->this_device); + } + + return -ENOTTY; +} + +static const struct file_operations mshv_dev_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = mshv_dev_ioctl, + .llseek = noop_llseek, +}; + +static struct miscdevice mshv_dev = { + .minor = MISC_DYNAMIC_MINOR, + .name = "mshv", + .fops = &mshv_dev_fops, + .mode = 0600, +}; + +static struct mshv_vtl_run *mshv_vtl_this_run(void) +{ + return *this_cpu_ptr(&mshv_vtl_per_cpu.run); +} + +static struct mshv_vtl_run *mshv_vtl_cpu_run(int cpu) +{ + return *per_cpu_ptr(&mshv_vtl_per_cpu.run, cpu); +} + +static struct page *mshv_vtl_cpu_reg_page(int cpu) +{ + return *per_cpu_ptr(&mshv_vtl_per_cpu.reg_page, cpu); +} + +static void mshv_vtl_configure_reg_page(struct mshv_vtl_per_cpu *per_cpu) +{ + struct hv_register_assoc reg_assoc = {}; + union hv_synic_overlay_page_msr overlay = {}; + struct page *reg_page; + + reg_page = alloc_page(GFP_KERNEL | __GFP_ZERO | __GFP_RETRY_MAYFAIL); + if (!reg_page) { + WARN(1, "failed to allocate register page\n"); + return; + } + + overlay.enabled = 1; + overlay.pfn = page_to_hvpfn(reg_page); + reg_assoc.name = HV_X64_REGISTER_REG_PAGE; + reg_assoc.value.reg64 = overlay.as_uint64; + + if (hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, + 1, input_vtl_zero, ®_assoc)) { + WARN(1, "failed to setup register page\n"); + __free_page(reg_page); + return; + } + + per_cpu->reg_page = reg_page; + mshv_has_reg_page = true; +} + +static void mshv_vtl_synic_enable_regs(unsigned int cpu) +{ + union hv_synic_sint sint; + + sint.as_uint64 = 0; + sint.vector = HYPERVISOR_CALLBACK_VECTOR; + sint.masked = false; + sint.auto_eoi = hv_recommend_using_aeoi(); + + /* Enable intercepts */ + if (!mshv_vsm_capabilities.intercept_page_available) + hv_set_msr(HV_MSR_SINT0 + HV_SYNIC_INTERCEPTION_SINT_INDEX, + sint.as_uint64); + + /* VTL2 Host VSP SINT is (un)masked when the user mode requests that */ +} + +static int mshv_vtl_get_vsm_regs(void) +{ + struct hv_register_assoc registers[2]; + int ret, count = 2; + + registers[0].name = HV_REGISTER_VSM_CODE_PAGE_OFFSETS; + registers[1].name = HV_REGISTER_VSM_CAPABILITIES; + + ret = hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, + count, input_vtl_zero, registers); + if (ret) + return ret; + + mshv_vsm_page_offsets.as_uint64 = registers[0].value.reg64; + mshv_vsm_capabilities.as_uint64 = registers[1].value.reg64; + + return ret; +} + +static int mshv_vtl_configure_vsm_partition(struct device *dev) +{ + union hv_register_vsm_partition_config config; + struct hv_register_assoc reg_assoc; + + config.as_uint64 = 0; + config.default_vtl_protection_mask = HV_MAP_GPA_PERMISSIONS_MASK; + config.enable_vtl_protection = 1; + config.zero_memory_on_reset = 1; + config.intercept_vp_startup = 1; + config.intercept_cpuid_unimplemented = 1; + + if (mshv_vsm_capabilities.intercept_page_available) { + dev_dbg(dev, "using intercept page\n"); + config.intercept_page = 1; + } + + reg_assoc.name = HV_REGISTER_VSM_PARTITION_CONFIG; + reg_assoc.value.reg64 = config.as_uint64; + + return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, + 1, input_vtl_zero, ®_assoc); +} + +static void mshv_vtl_vmbus_isr(void) +{ + struct hv_per_cpu_context *per_cpu; + struct hv_message *msg; + u32 message_type; + union hv_synic_event_flags *event_flags; + struct eventfd_ctx *eventfd; + u16 i; + + per_cpu = this_cpu_ptr(hv_context.cpu_context); + if (smp_processor_id() == 0) { + msg = (struct hv_message *)per_cpu->hyp_synic_message_page + VTL2_VMBUS_SINT_INDEX; + message_type = READ_ONCE(msg->header.message_type); + if (message_type != HVMSG_NONE) + tasklet_schedule(&msg_dpc); + } + + event_flags = (union hv_synic_event_flags *)per_cpu->hyp_synic_event_page + + VTL2_VMBUS_SINT_INDEX; + for_each_set_bit(i, event_flags->flags, HV_EVENT_FLAGS_COUNT) { + if (!sync_test_and_clear_bit(i, event_flags->flags)) + continue; + rcu_read_lock(); + eventfd = READ_ONCE(flag_eventfds[i]); + if (eventfd) + eventfd_signal(eventfd); + rcu_read_unlock(); + } + + vmbus_isr(); +} + +static int mshv_vtl_alloc_context(unsigned int cpu) +{ + struct mshv_vtl_per_cpu *per_cpu = this_cpu_ptr(&mshv_vtl_per_cpu); + + per_cpu->run = (struct mshv_vtl_run *)__get_free_page(GFP_KERNEL | __GFP_ZERO); + if (!per_cpu->run) + return -ENOMEM; + + if (mshv_vsm_capabilities.intercept_page_available) + mshv_vtl_configure_reg_page(per_cpu); + + mshv_vtl_synic_enable_regs(cpu); + + return 0; +} + +static int mshv_vtl_cpuhp_online; + +static int hv_vtl_setup_synic(void) +{ + int ret; + + /* Use our isr to first filter out packets destined for userspace */ + hv_setup_vmbus_handler(mshv_vtl_vmbus_isr); + + ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "hyperv/vtl:online", + mshv_vtl_alloc_context, NULL); + if (ret < 0) { + hv_setup_vmbus_handler(vmbus_isr); + return ret; + } + + mshv_vtl_cpuhp_online = ret; + + return 0; +} + +static void hv_vtl_remove_synic(void) +{ + cpuhp_remove_state(mshv_vtl_cpuhp_online); + hv_setup_vmbus_handler(vmbus_isr); +} + +static int vtl_get_vp_register(struct hv_register_assoc *reg) +{ + return hv_call_get_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, + 1, input_vtl_normal, reg); +} + +static int vtl_set_vp_register(struct hv_register_assoc *reg) +{ + return hv_call_set_vp_registers(HV_VP_INDEX_SELF, HV_PARTITION_ID_SELF, + 1, input_vtl_normal, reg); +} + +static int mshv_vtl_ioctl_add_vtl0_mem(struct mshv_vtl *vtl, void __user *arg) +{ + struct mshv_vtl_ram_disposition vtl0_mem; + struct dev_pagemap *pgmap; + void *addr; + + if (copy_from_user(&vtl0_mem, arg, sizeof(vtl0_mem))) + return -EFAULT; + if (vtl0_mem.last_pfn <= vtl0_mem.start_pfn) { + dev_err(vtl->module_dev, "range start pfn (%llx) > end pfn (%llx)\n", + vtl0_mem.start_pfn, vtl0_mem.last_pfn); + return -EFAULT; + } + + pgmap = kzalloc_obj(*pgmap); + if (!pgmap) + return -ENOMEM; + + /* + * vtl0_mem.last_pfn is excluded in the pagemap range for VTL0 as per design. + * last_pfn is not reserved or wasted, and reflects 'start_pfn + size' of pagemap range. + */ + pgmap->ranges[0].start = PFN_PHYS(vtl0_mem.start_pfn); + pgmap->ranges[0].end = PFN_PHYS(vtl0_mem.last_pfn) - 1; + pgmap->nr_range = 1; + pgmap->type = MEMORY_DEVICE_GENERIC; + + /* + * Determine the highest page order that can be used for the given memory range. + * This works best when the range is aligned; i.e. both the start and the length. + * Clamp to MAX_FOLIO_ORDER to avoid a WARN in memremap_pages() when the range + * alignment exceeds the maximum supported folio order for this kernel config. + */ + pgmap->vmemmap_shift = min(count_trailing_zeros(vtl0_mem.start_pfn | vtl0_mem.last_pfn), + MAX_FOLIO_ORDER); + dev_dbg(vtl->module_dev, + "Add VTL0 memory: start: 0x%llx, end_pfn: 0x%llx, page order: %lu\n", + vtl0_mem.start_pfn, vtl0_mem.last_pfn, pgmap->vmemmap_shift); + + addr = devm_memremap_pages(mem_dev, pgmap); + if (IS_ERR(addr)) { + dev_err(vtl->module_dev, "devm_memremap_pages error: %ld\n", PTR_ERR(addr)); + kfree(pgmap); + return PTR_ERR(addr); + } + + /* Don't free pgmap, since it has to stick around until the memory + * is unmapped, which will never happen as there is no scenario + * where VTL0 can be released/shutdown without bringing down VTL2. + */ + return 0; +} + +static void mshv_vtl_cancel(int cpu) +{ + int here = get_cpu(); + + if (here != cpu) { + if (!xchg_relaxed(&mshv_vtl_cpu_run(cpu)->cancel, 1)) + smp_send_reschedule(cpu); + } else { + WRITE_ONCE(mshv_vtl_this_run()->cancel, 1); + } + put_cpu(); +} + +static int mshv_vtl_poll_file_wake(wait_queue_entry_t *wait, unsigned int mode, int sync, void *key) +{ + struct mshv_vtl_poll_file *poll_file = container_of(wait, struct mshv_vtl_poll_file, wait); + + mshv_vtl_cancel(poll_file->cpu); + + return 0; +} + +static void mshv_vtl_ptable_queue_proc(struct file *file, wait_queue_head_t *wqh, poll_table *pt) +{ + struct mshv_vtl_poll_file *poll_file = container_of(pt, struct mshv_vtl_poll_file, pt); + + WARN_ON(poll_file->wqh); + poll_file->wqh = wqh; + add_wait_queue(wqh, &poll_file->wait); +} + +static int mshv_vtl_ioctl_set_poll_file(struct mshv_vtl_set_poll_file __user *user_input) +{ + struct file *file, *old_file; + struct mshv_vtl_poll_file *poll_file; + struct mshv_vtl_set_poll_file input; + + if (copy_from_user(&input, user_input, sizeof(input))) + return -EFAULT; + + if (input.cpu >= num_possible_cpus() || !cpu_online(input.cpu)) + return -EINVAL; + /* + * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists. + * CPU is expected to remain online after above cpu_online() check. + */ + + file = NULL; + file = fget(input.fd); + if (!file) + return -EBADFD; + + poll_file = per_cpu_ptr(&mshv_vtl_poll_file, READ_ONCE(input.cpu)); + if (!poll_file) + return -EINVAL; + + mutex_lock(&mshv_vtl_poll_file_lock); + + if (poll_file->wqh) + remove_wait_queue(poll_file->wqh, &poll_file->wait); + poll_file->wqh = NULL; + + old_file = poll_file->file; + poll_file->file = file; + poll_file->cpu = input.cpu; + + if (file) { + init_waitqueue_func_entry(&poll_file->wait, mshv_vtl_poll_file_wake); + init_poll_funcptr(&poll_file->pt, mshv_vtl_ptable_queue_proc); + vfs_poll(file, &poll_file->pt); + } + + mutex_unlock(&mshv_vtl_poll_file_lock); + + if (old_file) + fput(old_file); + + return 0; +} + +/* Static table mapping register names to their corresponding actions */ +static const struct { + enum hv_register_name reg_name; + int debug_reg_num; /* -1 if not a debug register */ + u32 msr_addr; /* 0 if not an MSR */ +} reg_table[] = { + /* Debug registers */ + {HV_X64_REGISTER_DR0, 0, 0}, + {HV_X64_REGISTER_DR1, 1, 0}, + {HV_X64_REGISTER_DR2, 2, 0}, + {HV_X64_REGISTER_DR3, 3, 0}, + {HV_X64_REGISTER_DR6, 6, 0}, + /* MTRR MSRs */ + {HV_X64_REGISTER_MSR_MTRR_CAP, -1, MSR_MTRRcap}, + {HV_X64_REGISTER_MSR_MTRR_DEF_TYPE, -1, MSR_MTRRdefType}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE0, -1, MTRRphysBase_MSR(0)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE1, -1, MTRRphysBase_MSR(1)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE2, -1, MTRRphysBase_MSR(2)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE3, -1, MTRRphysBase_MSR(3)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE4, -1, MTRRphysBase_MSR(4)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE5, -1, MTRRphysBase_MSR(5)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE6, -1, MTRRphysBase_MSR(6)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE7, -1, MTRRphysBase_MSR(7)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE8, -1, MTRRphysBase_MSR(8)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASE9, -1, MTRRphysBase_MSR(9)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEA, -1, MTRRphysBase_MSR(0xa)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEB, -1, MTRRphysBase_MSR(0xb)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEC, -1, MTRRphysBase_MSR(0xc)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASED, -1, MTRRphysBase_MSR(0xd)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEE, -1, MTRRphysBase_MSR(0xe)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_BASEF, -1, MTRRphysBase_MSR(0xf)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK0, -1, MTRRphysMask_MSR(0)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK1, -1, MTRRphysMask_MSR(1)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK2, -1, MTRRphysMask_MSR(2)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK3, -1, MTRRphysMask_MSR(3)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK4, -1, MTRRphysMask_MSR(4)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK5, -1, MTRRphysMask_MSR(5)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK6, -1, MTRRphysMask_MSR(6)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK7, -1, MTRRphysMask_MSR(7)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK8, -1, MTRRphysMask_MSR(8)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASK9, -1, MTRRphysMask_MSR(9)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKA, -1, MTRRphysMask_MSR(0xa)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKB, -1, MTRRphysMask_MSR(0xb)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKC, -1, MTRRphysMask_MSR(0xc)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKD, -1, MTRRphysMask_MSR(0xd)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKE, -1, MTRRphysMask_MSR(0xe)}, + {HV_X64_REGISTER_MSR_MTRR_PHYS_MASKF, -1, MTRRphysMask_MSR(0xf)}, + {HV_X64_REGISTER_MSR_MTRR_FIX64K00000, -1, MSR_MTRRfix64K_00000}, + {HV_X64_REGISTER_MSR_MTRR_FIX16K80000, -1, MSR_MTRRfix16K_80000}, + {HV_X64_REGISTER_MSR_MTRR_FIX16KA0000, -1, MSR_MTRRfix16K_A0000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KC0000, -1, MSR_MTRRfix4K_C0000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KC8000, -1, MSR_MTRRfix4K_C8000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KD0000, -1, MSR_MTRRfix4K_D0000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KD8000, -1, MSR_MTRRfix4K_D8000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KE0000, -1, MSR_MTRRfix4K_E0000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KE8000, -1, MSR_MTRRfix4K_E8000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KF0000, -1, MSR_MTRRfix4K_F0000}, + {HV_X64_REGISTER_MSR_MTRR_FIX4KF8000, -1, MSR_MTRRfix4K_F8000}, +}; + +static int mshv_vtl_get_set_reg(struct hv_register_assoc *regs, bool set) +{ + u64 *reg64; + enum hv_register_name gpr_name; + int i; + + gpr_name = regs->name; + reg64 = ®s->value.reg64; + + /* Search for the register in the table */ + for (i = 0; i < ARRAY_SIZE(reg_table); i++) { + if (reg_table[i].reg_name != gpr_name) + continue; + if (reg_table[i].debug_reg_num != -1) { + /* Handle debug registers */ + if (gpr_name == HV_X64_REGISTER_DR6 && + !mshv_vsm_capabilities.dr6_shared) + goto hypercall; + if (set) + native_set_debugreg(reg_table[i].debug_reg_num, *reg64); + else + *reg64 = native_get_debugreg(reg_table[i].debug_reg_num); + } else { + /* Handle MSRs */ + if (set) + wrmsrq(reg_table[i].msr_addr, *reg64); + else + rdmsrq(reg_table[i].msr_addr, *reg64); + } + return 0; + } + +hypercall: + return 1; +} + +static void mshv_vtl_return(struct mshv_vtl_cpu_context *vtl0) +{ + struct hv_vp_assist_page *hvp; + + hvp = hv_vp_assist_page[smp_processor_id()]; + + /* + * Process signal event direct set in the run page, if any. + */ + if (mshv_vsm_capabilities.return_action_available) { + u32 offset = READ_ONCE(mshv_vtl_this_run()->vtl_ret_action_size); + + WRITE_ONCE(mshv_vtl_this_run()->vtl_ret_action_size, 0); + + /* + * Hypervisor will take care of clearing out the actions + * set in the assist page. + */ + memcpy(hvp->vtl_ret_actions, + mshv_vtl_this_run()->vtl_ret_actions, + min_t(u32, offset, sizeof(hvp->vtl_ret_actions))); + } + + mshv_vtl_return_call(vtl0); +} + +static bool mshv_vtl_process_intercept(void) +{ + struct hv_per_cpu_context *mshv_cpu; + void *synic_message_page; + struct hv_message *msg; + u32 message_type; + + mshv_cpu = this_cpu_ptr(hv_context.cpu_context); + synic_message_page = mshv_cpu->hyp_synic_message_page; + if (unlikely(!synic_message_page)) + return true; + + msg = (struct hv_message *)synic_message_page + HV_SYNIC_INTERCEPTION_SINT_INDEX; + message_type = READ_ONCE(msg->header.message_type); + if (message_type == HVMSG_NONE) + return true; + + memcpy(mshv_vtl_this_run()->exit_message, msg, sizeof(*msg)); + vmbus_signal_eom(msg, message_type); + + return false; +} + +static int mshv_vtl_ioctl_return_to_lower_vtl(void) +{ + preempt_disable(); + for (;;) { + unsigned long irq_flags; + struct hv_vp_assist_page *hvp; + int ret; + + if (__xfer_to_guest_mode_work_pending()) { + preempt_enable(); + ret = xfer_to_guest_mode_handle_work(); + if (ret) + return ret; + preempt_disable(); + } + + local_irq_save(irq_flags); + if (READ_ONCE(mshv_vtl_this_run()->cancel)) { + local_irq_restore(irq_flags); + preempt_enable(); + return -EINTR; + } + + mshv_vtl_return(&mshv_vtl_this_run()->cpu_context); + local_irq_restore(irq_flags); + + hvp = hv_vp_assist_page[smp_processor_id()]; + this_cpu_inc(num_vtl0_transitions); + switch (hvp->vtl_entry_reason) { + case MSHV_ENTRY_REASON_INTERRUPT: + if (!mshv_vsm_capabilities.intercept_page_available && + likely(!mshv_vtl_process_intercept())) + goto done; + break; + + case MSHV_ENTRY_REASON_INTERCEPT: + WARN_ON(!mshv_vsm_capabilities.intercept_page_available); + memcpy(mshv_vtl_this_run()->exit_message, hvp->intercept_message, + sizeof(hvp->intercept_message)); + goto done; + + default: + panic("unknown entry reason: %d", hvp->vtl_entry_reason); + } + } + +done: + preempt_enable(); + + return 0; +} + +static long +mshv_vtl_ioctl_get_regs(void __user *user_args) +{ + struct mshv_vp_registers args; + struct hv_register_assoc reg; + long ret; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + /* This IOCTL supports processing only one register at a time. */ + if (args.count != 1) + return -EINVAL; + + if (copy_from_user(®, (void __user *)args.regs_ptr, + sizeof(reg))) + return -EFAULT; + + ret = mshv_vtl_get_set_reg(®, false); + if (!ret) + goto copy_args; /* No need of hypercall */ + ret = vtl_get_vp_register(®); + if (ret) + return ret; + +copy_args: + if (copy_to_user((void __user *)args.regs_ptr, ®, sizeof(reg))) + ret = -EFAULT; + + return ret; +} + +static long +mshv_vtl_ioctl_set_regs(void __user *user_args) +{ + struct mshv_vp_registers args; + struct hv_register_assoc reg; + long ret; + + if (copy_from_user(&args, user_args, sizeof(args))) + return -EFAULT; + + /* This IOCTL supports processing only one register at a time. */ + if (args.count != 1) + return -EINVAL; + + if (copy_from_user(®, (void __user *)args.regs_ptr, sizeof(reg))) + return -EFAULT; + + ret = mshv_vtl_get_set_reg(®, true); + if (!ret) + return ret; /* No need of hypercall */ + ret = vtl_set_vp_register(®); + + return ret; +} + +static long +mshv_vtl_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg) +{ + long ret; + struct mshv_vtl *vtl = filp->private_data; + + switch (ioctl) { + case MSHV_SET_POLL_FILE: + ret = mshv_vtl_ioctl_set_poll_file((struct mshv_vtl_set_poll_file __user *)arg); + break; + case MSHV_GET_VP_REGISTERS: + ret = mshv_vtl_ioctl_get_regs((void __user *)arg); + break; + case MSHV_SET_VP_REGISTERS: + ret = mshv_vtl_ioctl_set_regs((void __user *)arg); + break; + case MSHV_RETURN_TO_LOWER_VTL: + ret = mshv_vtl_ioctl_return_to_lower_vtl(); + break; + case MSHV_ADD_VTL0_MEMORY: + ret = mshv_vtl_ioctl_add_vtl0_mem(vtl, (void __user *)arg); + break; + default: + dev_err(vtl->module_dev, "invalid vtl ioctl: %#x\n", ioctl); + ret = -ENOTTY; + } + + return ret; +} + +static vm_fault_t mshv_vtl_fault(struct vm_fault *vmf) +{ + struct page *page; + int cpu = vmf->pgoff & MSHV_PG_OFF_CPU_MASK; + int real_off = vmf->pgoff >> MSHV_REAL_OFF_SHIFT; + + if (!cpu_online(cpu)) + return VM_FAULT_SIGBUS; + /* + * CPU Hotplug is not supported in VTL2 in OpenHCL, where this kernel driver exists. + * CPU is expected to remain online after above cpu_online() check. + */ + + if (real_off == MSHV_RUN_PAGE_OFFSET) { + page = virt_to_page(mshv_vtl_cpu_run(cpu)); + } else if (real_off == MSHV_REG_PAGE_OFFSET) { + if (!mshv_has_reg_page) + return VM_FAULT_SIGBUS; + page = mshv_vtl_cpu_reg_page(cpu); + } else { + return VM_FAULT_NOPAGE; + } + + get_page(page); + vmf->page = page; + + return 0; +} + +static const struct vm_operations_struct mshv_vtl_vm_ops = { + .fault = mshv_vtl_fault, +}; + +static int mshv_vtl_mmap(struct file *filp, struct vm_area_struct *vma) +{ + vma->vm_ops = &mshv_vtl_vm_ops; + + return 0; +} + +static int mshv_vtl_release(struct inode *inode, struct file *filp) +{ + struct mshv_vtl *vtl = filp->private_data; + + kfree(vtl); + + return 0; +} + +static const struct file_operations mshv_vtl_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = mshv_vtl_ioctl, + .release = mshv_vtl_release, + .mmap = mshv_vtl_mmap, +}; + +static void mshv_vtl_synic_mask_vmbus_sint(void *info) +{ + union hv_synic_sint sint; + const u8 *mask = info; + + sint.as_uint64 = 0; + sint.vector = HYPERVISOR_CALLBACK_VECTOR; + sint.masked = (*mask != 0); + sint.auto_eoi = hv_recommend_using_aeoi(); + + hv_set_msr(HV_MSR_SINT0 + VTL2_VMBUS_SINT_INDEX, + sint.as_uint64); + + if (!sint.masked) + pr_debug("%s: Unmasking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id()); + else + pr_debug("%s: Masking VTL2 VMBUS SINT on VP %d\n", __func__, smp_processor_id()); +} + +static void mshv_vtl_read_remote(void *buffer) +{ + struct hv_per_cpu_context *mshv_cpu = this_cpu_ptr(hv_context.cpu_context); + struct hv_message *msg = (struct hv_message *)mshv_cpu->hyp_synic_message_page + + VTL2_VMBUS_SINT_INDEX; + u32 message_type = READ_ONCE(msg->header.message_type); + + WRITE_ONCE(has_message, false); + if (message_type == HVMSG_NONE) + return; + + memcpy(buffer, msg, sizeof(*msg)); + vmbus_signal_eom(msg, message_type); +} + +static bool vtl_synic_mask_vmbus_sint_masked = true; + +static ssize_t mshv_vtl_sint_read(struct file *filp, char __user *arg, size_t size, loff_t *offset) +{ + struct hv_message msg = {}; + int ret; + + if (size < sizeof(msg)) + return -EINVAL; + + for (;;) { + smp_call_function_single(VMBUS_CONNECT_CPU, mshv_vtl_read_remote, &msg, true); + if (msg.header.message_type != HVMSG_NONE) + break; + + if (READ_ONCE(vtl_synic_mask_vmbus_sint_masked)) + return 0; /* EOF */ + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + ret = wait_event_interruptible(fd_wait_queue, + READ_ONCE(has_message) || + READ_ONCE(vtl_synic_mask_vmbus_sint_masked)); + if (ret) + return ret; + } + + if (copy_to_user(arg, &msg, sizeof(msg))) + return -EFAULT; + + return sizeof(msg); +} + +static __poll_t mshv_vtl_sint_poll(struct file *filp, poll_table *wait) +{ + __poll_t mask = 0; + + poll_wait(filp, &fd_wait_queue, wait); + if (READ_ONCE(has_message) || READ_ONCE(vtl_synic_mask_vmbus_sint_masked)) + mask |= EPOLLIN | EPOLLRDNORM; + + return mask; +} + +static void mshv_vtl_sint_on_msg_dpc(unsigned long data) +{ + WRITE_ONCE(has_message, true); + wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN); +} + +static int mshv_vtl_sint_ioctl_post_msg(struct mshv_vtl_sint_post_msg __user *arg) +{ + struct mshv_vtl_sint_post_msg message; + u8 payload[HV_MESSAGE_PAYLOAD_BYTE_COUNT]; + + if (copy_from_user(&message, arg, sizeof(message))) + return -EFAULT; + if (message.payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) + return -EINVAL; + if (copy_from_user(payload, (void __user *)message.payload_ptr, + message.payload_size)) + return -EFAULT; + + return hv_post_message((union hv_connection_id)message.connection_id, + message.message_type, (void *)payload, + message.payload_size); +} + +static int mshv_vtl_sint_ioctl_signal_event(struct mshv_vtl_signal_event __user *arg) +{ + u64 input, status; + struct mshv_vtl_signal_event signal_event; + + if (copy_from_user(&signal_event, arg, sizeof(signal_event))) + return -EFAULT; + + input = signal_event.connection_id | ((u64)signal_event.flag << 32); + + status = hv_do_fast_hypercall8(HVCALL_SIGNAL_EVENT, input); + + return hv_result_to_errno(status); +} + +static int mshv_vtl_sint_ioctl_set_eventfd(struct mshv_vtl_set_eventfd __user *arg) +{ + struct mshv_vtl_set_eventfd set_eventfd; + struct eventfd_ctx *eventfd, *old_eventfd; + + if (copy_from_user(&set_eventfd, arg, sizeof(set_eventfd))) + return -EFAULT; + if (set_eventfd.flag >= HV_EVENT_FLAGS_COUNT) + return -EINVAL; + + eventfd = NULL; + if (set_eventfd.fd >= 0) { + eventfd = eventfd_ctx_fdget(set_eventfd.fd); + if (IS_ERR(eventfd)) + return PTR_ERR(eventfd); + } + + guard(mutex)(&flag_lock); + old_eventfd = READ_ONCE(flag_eventfds[set_eventfd.flag]); + WRITE_ONCE(flag_eventfds[set_eventfd.flag], eventfd); + + if (old_eventfd) { + synchronize_rcu(); + eventfd_ctx_put(old_eventfd); + } + + return 0; +} + +static int mshv_vtl_sint_ioctl_pause_msg_stream(struct mshv_sint_mask __user *arg) +{ + static DEFINE_MUTEX(vtl2_vmbus_sint_mask_mutex); + struct mshv_sint_mask mask; + + if (copy_from_user(&mask, arg, sizeof(mask))) + return -EFAULT; + guard(mutex)(&vtl2_vmbus_sint_mask_mutex); + on_each_cpu(mshv_vtl_synic_mask_vmbus_sint, &mask.mask, 1); + WRITE_ONCE(vtl_synic_mask_vmbus_sint_masked, mask.mask != 0); + if (mask.mask) + wake_up_interruptible_poll(&fd_wait_queue, EPOLLIN); + + return 0; +} + +static long mshv_vtl_sint_ioctl(struct file *f, unsigned int cmd, unsigned long arg) +{ + switch (cmd) { + case MSHV_SINT_POST_MESSAGE: + return mshv_vtl_sint_ioctl_post_msg((struct mshv_vtl_sint_post_msg __user *)arg); + case MSHV_SINT_SIGNAL_EVENT: + return mshv_vtl_sint_ioctl_signal_event((struct mshv_vtl_signal_event __user *)arg); + case MSHV_SINT_SET_EVENTFD: + return mshv_vtl_sint_ioctl_set_eventfd((struct mshv_vtl_set_eventfd __user *)arg); + case MSHV_SINT_PAUSE_MESSAGE_STREAM: + return mshv_vtl_sint_ioctl_pause_msg_stream((struct mshv_sint_mask __user *)arg); + default: + return -ENOIOCTLCMD; + } +} + +static const struct file_operations mshv_vtl_sint_ops = { + .owner = THIS_MODULE, + .read = mshv_vtl_sint_read, + .poll = mshv_vtl_sint_poll, + .unlocked_ioctl = mshv_vtl_sint_ioctl, +}; + +static struct miscdevice mshv_vtl_sint_dev = { + .name = "mshv_sint", + .fops = &mshv_vtl_sint_ops, + .mode = 0600, + .minor = MISC_DYNAMIC_MINOR, +}; + +static int mshv_vtl_hvcall_dev_open(struct inode *node, struct file *f) +{ + struct miscdevice *dev = f->private_data; + struct mshv_vtl_hvcall_fd *fd; + + if (!capable(CAP_SYS_ADMIN)) + return -EPERM; + + fd = vzalloc(sizeof(*fd)); + if (!fd) + return -ENOMEM; + fd->dev = dev; + f->private_data = fd; + mutex_init(&fd->init_mutex); + + return 0; +} + +static int mshv_vtl_hvcall_dev_release(struct inode *node, struct file *f) +{ + struct mshv_vtl_hvcall_fd *fd; + + fd = f->private_data; + if (fd) { + vfree(fd); + f->private_data = NULL; + } + + return 0; +} + +static int mshv_vtl_hvcall_do_setup(struct mshv_vtl_hvcall_fd *fd, + struct mshv_vtl_hvcall_setup __user *hvcall_setup_user) +{ + struct mshv_vtl_hvcall_setup hvcall_setup; + + guard(mutex)(&fd->init_mutex); + + if (fd->allow_map_initialized) { + dev_err(fd->dev->this_device, + "Hypercall allow map has already been set, pid %d\n", + current->pid); + return -EINVAL; + } + + if (copy_from_user(&hvcall_setup, hvcall_setup_user, + sizeof(struct mshv_vtl_hvcall_setup))) { + return -EFAULT; + } + if (hvcall_setup.bitmap_array_size > ARRAY_SIZE(fd->allow_bitmap)) + return -EINVAL; + + if (copy_from_user(&fd->allow_bitmap, + (void __user *)hvcall_setup.allow_bitmap_ptr, + hvcall_setup.bitmap_array_size)) { + return -EFAULT; + } + + dev_info(fd->dev->this_device, "Hypercall allow map has been set, pid %d\n", + current->pid); + fd->allow_map_initialized = true; + return 0; +} + +static bool mshv_vtl_hvcall_is_allowed(struct mshv_vtl_hvcall_fd *fd, u16 call_code) +{ + return test_bit(call_code, (unsigned long *)fd->allow_bitmap); +} + +static int mshv_vtl_hvcall_call(struct mshv_vtl_hvcall_fd *fd, + struct mshv_vtl_hvcall __user *hvcall_user) +{ + struct mshv_vtl_hvcall hvcall; + void *in, *out; + int ret; + + if (copy_from_user(&hvcall, hvcall_user, sizeof(struct mshv_vtl_hvcall))) + return -EFAULT; + if (hvcall.input_size > HV_HYP_PAGE_SIZE) + return -EINVAL; + if (hvcall.output_size > HV_HYP_PAGE_SIZE) + return -EINVAL; + + /* + * By default, all hypercalls are not allowed. + * The user mode code has to set up the allow bitmap once. + */ + + if (!mshv_vtl_hvcall_is_allowed(fd, hvcall.control & 0xFFFF)) { + dev_err(fd->dev->this_device, + "Hypercall with control data %#llx isn't allowed\n", + hvcall.control); + return -EPERM; + } + + /* + * This may create a problem for Confidential VM (CVM) usecase where we need to use + * Hyper-V driver allocated per-cpu input and output pages (hyperv_pcpu_input_arg and + * hyperv_pcpu_output_arg) for making a hypervisor call. + * + * TODO: Take care of this when CVM support is added. + */ + in = (void *)__get_free_page(GFP_KERNEL); + out = (void *)__get_free_page(GFP_KERNEL); + + if (copy_from_user(in, (void __user *)hvcall.input_ptr, hvcall.input_size)) { + ret = -EFAULT; + goto free_pages; + } + + hvcall.status = hv_do_hypercall(hvcall.control, in, out); + + if (copy_to_user((void __user *)hvcall.output_ptr, out, hvcall.output_size)) { + ret = -EFAULT; + goto free_pages; + } + ret = put_user(hvcall.status, &hvcall_user->status); +free_pages: + free_page((unsigned long)in); + free_page((unsigned long)out); + + return ret; +} + +static long mshv_vtl_hvcall_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg) +{ + struct mshv_vtl_hvcall_fd *fd = f->private_data; + + switch (cmd) { + case MSHV_HVCALL_SETUP: + return mshv_vtl_hvcall_do_setup(fd, (struct mshv_vtl_hvcall_setup __user *)arg); + case MSHV_HVCALL: + return mshv_vtl_hvcall_call(fd, (struct mshv_vtl_hvcall __user *)arg); + default: + break; + } + + return -ENOIOCTLCMD; +} + +static const struct file_operations mshv_vtl_hvcall_dev_file_ops = { + .owner = THIS_MODULE, + .open = mshv_vtl_hvcall_dev_open, + .release = mshv_vtl_hvcall_dev_release, + .unlocked_ioctl = mshv_vtl_hvcall_dev_ioctl, +}; + +static struct miscdevice mshv_vtl_hvcall_dev = { + .name = "mshv_hvcall", + .nodename = "mshv_hvcall", + .fops = &mshv_vtl_hvcall_dev_file_ops, + .mode = 0600, + .minor = MISC_DYNAMIC_MINOR, +}; + +static int mshv_vtl_low_open(struct inode *inodep, struct file *filp) +{ + pid_t pid = task_pid_vnr(current); + uid_t uid = current_uid().val; + int ret = 0; + + pr_debug("%s: Opening VTL low, task group %d, uid %d\n", __func__, pid, uid); + + if (capable(CAP_SYS_ADMIN)) { + filp->private_data = inodep; + } else { + pr_err("%s: VTL low open failed: CAP_SYS_ADMIN required. task group %d, uid %d", + __func__, pid, uid); + ret = -EPERM; + } + + return ret; +} + +static bool can_fault(struct vm_fault *vmf, unsigned long size, unsigned long *pfn) +{ + unsigned long mask = size - 1; + unsigned long start = vmf->address & ~mask; + unsigned long end = start + size; + bool is_valid; + + is_valid = (vmf->address & mask) == ((vmf->pgoff << PAGE_SHIFT) & mask) && + start >= vmf->vma->vm_start && + end <= vmf->vma->vm_end; + + if (is_valid) + *pfn = vmf->pgoff & ~(mask >> PAGE_SHIFT); + + return is_valid; +} + +static vm_fault_t mshv_vtl_low_huge_fault(struct vm_fault *vmf, unsigned int order) +{ + unsigned long pfn = vmf->pgoff; + vm_fault_t ret = VM_FAULT_FALLBACK; + + switch (order) { + case 0: + return vmf_insert_mixed(vmf->vma, vmf->address, pfn); + + case PMD_ORDER: + if (can_fault(vmf, PMD_SIZE, &pfn)) + ret = vmf_insert_pfn_pmd(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); + return ret; + + case PUD_ORDER: + if (can_fault(vmf, PUD_SIZE, &pfn)) + ret = vmf_insert_pfn_pud(vmf, pfn, vmf->flags & FAULT_FLAG_WRITE); + return ret; + + default: + return VM_FAULT_SIGBUS; + } +} + +static vm_fault_t mshv_vtl_low_fault(struct vm_fault *vmf) +{ + return mshv_vtl_low_huge_fault(vmf, 0); +} + +static const struct vm_operations_struct mshv_vtl_low_vm_ops = { + .fault = mshv_vtl_low_fault, + .huge_fault = mshv_vtl_low_huge_fault, +}; + +static int mshv_vtl_low_mmap(struct file *filp, struct vm_area_struct *vma) +{ + vma->vm_ops = &mshv_vtl_low_vm_ops; + vm_flags_set(vma, VM_HUGEPAGE | VM_MIXEDMAP); + + return 0; +} + +static const struct file_operations mshv_vtl_low_file_ops = { + .owner = THIS_MODULE, + .open = mshv_vtl_low_open, + .mmap = mshv_vtl_low_mmap, +}; + +static struct miscdevice mshv_vtl_low = { + .name = "mshv_vtl_low", + .nodename = "mshv_vtl_low", + .fops = &mshv_vtl_low_file_ops, + .mode = 0600, + .minor = MISC_DYNAMIC_MINOR, +}; + +static int __init mshv_vtl_init(void) +{ + int ret; + struct device *dev = mshv_dev.this_device; + + /* + * This creates /dev/mshv which provides functionality to create VTLs and partitions. + */ + ret = misc_register(&mshv_dev); + if (ret) { + dev_err(dev, "mshv device register failed: %d\n", ret); + goto free_dev; + } + + tasklet_init(&msg_dpc, mshv_vtl_sint_on_msg_dpc, 0); + init_waitqueue_head(&fd_wait_queue); + + if (mshv_vtl_get_vsm_regs()) { + dev_emerg(dev, "Unable to get VSM capabilities !!\n"); + ret = -ENODEV; + goto free_dev; + } + if (mshv_vtl_configure_vsm_partition(dev)) { + dev_emerg(dev, "VSM configuration failed !!\n"); + ret = -ENODEV; + goto free_dev; + } + + mshv_vtl_return_call_init(mshv_vsm_page_offsets.vtl_return_offset); + ret = hv_vtl_setup_synic(); + if (ret) + goto free_dev; + + /* + * mshv_sint device adds VMBus relay ioctl support. + * This provides a channel for VTL0 to communicate with VTL2. + */ + ret = misc_register(&mshv_vtl_sint_dev); + if (ret) + goto free_synic; + + /* + * mshv_hvcall device adds interface to enable userspace for direct hypercalls support. + */ + ret = misc_register(&mshv_vtl_hvcall_dev); + if (ret) + goto free_sint; + + /* + * mshv_vtl_low device is used to map VTL0 address space to a user-mode process in VTL2. + * It implements mmap() to allow a user-mode process in VTL2 to map to the address of VTL0. + */ + ret = misc_register(&mshv_vtl_low); + if (ret) + goto free_hvcall; + + /* + * "mshv vtl mem dev" device is later used to setup VTL0 memory. + */ + mem_dev = kzalloc_obj(*mem_dev); + if (!mem_dev) { + ret = -ENOMEM; + goto free_low; + } + + mutex_init(&mshv_vtl_poll_file_lock); + + device_initialize(mem_dev); + dev_set_name(mem_dev, "mshv vtl mem dev"); + ret = device_add(mem_dev); + if (ret) { + dev_err(dev, "mshv vtl mem dev add: %d\n", ret); + goto free_mem; + } + + return 0; + +free_mem: + kfree(mem_dev); +free_low: + misc_deregister(&mshv_vtl_low); +free_hvcall: + misc_deregister(&mshv_vtl_hvcall_dev); +free_sint: + misc_deregister(&mshv_vtl_sint_dev); +free_synic: + hv_vtl_remove_synic(); +free_dev: + misc_deregister(&mshv_dev); + + return ret; +} + +static void __exit mshv_vtl_exit(void) +{ + device_del(mem_dev); + kfree(mem_dev); + misc_deregister(&mshv_vtl_low); + misc_deregister(&mshv_vtl_hvcall_dev); + misc_deregister(&mshv_vtl_sint_dev); + hv_vtl_remove_synic(); + misc_deregister(&mshv_dev); +} + +module_init(mshv_vtl_init); +module_exit(mshv_vtl_exit); diff --git a/drivers/hv/ring_buffer.c b/drivers/hv/ring_buffer.c new file mode 100644 index 000000000..592a9601f --- /dev/null +++ b/drivers/hv/ring_buffer.c @@ -0,0 +1,654 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + * K. Y. Srinivasan <kys@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/kernel.h> +#include <linux/mm.h> +#include <linux/hyperv.h> +#include <linux/uio.h> +#include <linux/vmalloc.h> +#include <linux/slab.h> +#include <linux/prefetch.h> +#include <linux/io.h> +#include <linux/export.h> +#include <asm/mshyperv.h> + +#include "hyperv_vmbus.h" + +#define VMBUS_PKT_TRAILER 8 + +/* + * When we write to the ring buffer, check if the host needs to + * be signaled. Here is the details of this protocol: + * + * 1. The host guarantees that while it is draining the + * ring buffer, it will set the interrupt_mask to + * indicate it does not need to be interrupted when + * new data is placed. + * + * 2. The host guarantees that it will completely drain + * the ring buffer before exiting the read loop. Further, + * once the ring buffer is empty, it will clear the + * interrupt_mask and re-check to see if new data has + * arrived. + * + * KYS: Oct. 30, 2016: + * It looks like Windows hosts have logic to deal with DOS attacks that + * can be triggered if it receives interrupts when it is not expecting + * the interrupt. The host expects interrupts only when the ring + * transitions from empty to non-empty (or full to non full on the guest + * to host ring). + * So, base the signaling decision solely on the ring state until the + * host logic is fixed. + */ + +static void hv_signal_on_write(u32 old_write, struct vmbus_channel *channel) +{ + struct hv_ring_buffer_info *rbi = &channel->outbound; + + virt_mb(); + if (READ_ONCE(rbi->ring_buffer->interrupt_mask)) + return; + + /* check interrupt_mask before read_index */ + virt_rmb(); + /* + * This is the only case we need to signal when the + * ring transitions from being empty to non-empty. + */ + if (old_write == READ_ONCE(rbi->ring_buffer->read_index)) { + ++channel->intr_out_empty; + vmbus_setevent(channel); + } +} + +/* Get the next write location for the specified ring buffer. */ +static inline u32 +hv_get_next_write_location(struct hv_ring_buffer_info *ring_info) +{ + u32 next = ring_info->ring_buffer->write_index; + + return next; +} + +/* Set the next write location for the specified ring buffer. */ +static inline void +hv_set_next_write_location(struct hv_ring_buffer_info *ring_info, + u32 next_write_location) +{ + ring_info->ring_buffer->write_index = next_write_location; +} + +/* Get the size of the ring buffer. */ +static inline u32 +hv_get_ring_buffersize(const struct hv_ring_buffer_info *ring_info) +{ + return ring_info->ring_datasize; +} + +/* Get the read and write indices as u64 of the specified ring buffer. */ +static inline u64 +hv_get_ring_bufferindices(struct hv_ring_buffer_info *ring_info) +{ + return (u64)ring_info->ring_buffer->write_index << 32; +} + +/* + * Helper routine to copy from source to ring buffer. + * Assume there is enough room. Handles wrap-around in dest case only!! + */ +static u32 hv_copyto_ringbuffer( + struct hv_ring_buffer_info *ring_info, + u32 start_write_offset, + const void *src, + u32 srclen) +{ + void *ring_buffer = hv_get_ring_buffer(ring_info); + u32 ring_buffer_size = hv_get_ring_buffersize(ring_info); + + memcpy(ring_buffer + start_write_offset, src, srclen); + + start_write_offset += srclen; + if (start_write_offset >= ring_buffer_size) + start_write_offset -= ring_buffer_size; + + return start_write_offset; +} + +/* + * + * hv_get_ringbuffer_availbytes() + * + * Get number of bytes available to read and to write to + * for the specified ring buffer + */ +static void +hv_get_ringbuffer_availbytes(const struct hv_ring_buffer_info *rbi, + u32 *read, u32 *write) +{ + u32 read_loc, write_loc, dsize; + + /* Capture the read/write indices before they changed */ + read_loc = READ_ONCE(rbi->ring_buffer->read_index); + write_loc = READ_ONCE(rbi->ring_buffer->write_index); + dsize = rbi->ring_datasize; + + *write = write_loc >= read_loc ? dsize - (write_loc - read_loc) : + read_loc - write_loc; + *read = dsize - *write; +} + +/* Get various debug metrics for the specified ring buffer. */ +int hv_ringbuffer_get_debuginfo(struct hv_ring_buffer_info *ring_info, + struct hv_ring_buffer_debug_info *debug_info) +{ + u32 bytes_avail_towrite; + u32 bytes_avail_toread; + + mutex_lock(&ring_info->ring_buffer_mutex); + + if (!ring_info->ring_buffer) { + mutex_unlock(&ring_info->ring_buffer_mutex); + return -EINVAL; + } + + hv_get_ringbuffer_availbytes(ring_info, + &bytes_avail_toread, + &bytes_avail_towrite); + debug_info->bytes_avail_toread = bytes_avail_toread; + debug_info->bytes_avail_towrite = bytes_avail_towrite; + debug_info->current_read_index = ring_info->ring_buffer->read_index; + debug_info->current_write_index = ring_info->ring_buffer->write_index; + debug_info->current_interrupt_mask + = ring_info->ring_buffer->interrupt_mask; + mutex_unlock(&ring_info->ring_buffer_mutex); + + return 0; +} +EXPORT_SYMBOL_GPL(hv_ringbuffer_get_debuginfo); + +/* Initialize a channel's ring buffer info mutex locks */ +void hv_ringbuffer_pre_init(struct vmbus_channel *channel) +{ + mutex_init(&channel->inbound.ring_buffer_mutex); + mutex_init(&channel->outbound.ring_buffer_mutex); +} + +/* Initialize the ring buffer. */ +int hv_ringbuffer_init(struct hv_ring_buffer_info *ring_info, + struct page *pages, u32 page_cnt, u32 max_pkt_size, + bool confidential) +{ + struct page **pages_wraparound; + int i; + + BUILD_BUG_ON((sizeof(struct hv_ring_buffer) != PAGE_SIZE)); + + /* + * First page holds struct hv_ring_buffer, do wraparound mapping for + * the rest. + */ + pages_wraparound = kzalloc_objs(struct page *, page_cnt * 2 - 1); + if (!pages_wraparound) + return -ENOMEM; + + pages_wraparound[0] = pages; + for (i = 0; i < 2 * (page_cnt - 1); i++) + pages_wraparound[i + 1] = + &pages[i % (page_cnt - 1) + 1]; + + ring_info->ring_buffer = (struct hv_ring_buffer *) + vmap(pages_wraparound, page_cnt * 2 - 1, VM_MAP, + confidential ? PAGE_KERNEL : pgprot_decrypted(PAGE_KERNEL)); + + kfree(pages_wraparound); + if (!ring_info->ring_buffer) + return -ENOMEM; + + /* + * Ensure the header page is zero'ed since + * encryption status may have changed. + */ + memset(ring_info->ring_buffer, 0, HV_HYP_PAGE_SIZE); + + ring_info->ring_buffer->read_index = + ring_info->ring_buffer->write_index = 0; + + /* Set the feature bit for enabling flow control. */ + ring_info->ring_buffer->feature_bits.value = 1; + + ring_info->ring_size = page_cnt << PAGE_SHIFT; + ring_info->ring_size_div10_reciprocal = + reciprocal_value(ring_info->ring_size / 10); + ring_info->ring_datasize = ring_info->ring_size - + sizeof(struct hv_ring_buffer); + ring_info->priv_read_index = 0; + + /* Initialize buffer that holds copies of incoming packets */ + if (max_pkt_size) { + ring_info->pkt_buffer = kzalloc(max_pkt_size, GFP_KERNEL); + if (!ring_info->pkt_buffer) + return -ENOMEM; + ring_info->pkt_buffer_size = max_pkt_size; + } + + spin_lock_init(&ring_info->ring_lock); + + return 0; +} + +/* Cleanup the ring buffer. */ +void hv_ringbuffer_cleanup(struct hv_ring_buffer_info *ring_info) +{ + mutex_lock(&ring_info->ring_buffer_mutex); + vunmap(ring_info->ring_buffer); + ring_info->ring_buffer = NULL; + mutex_unlock(&ring_info->ring_buffer_mutex); + + kfree(ring_info->pkt_buffer); + ring_info->pkt_buffer = NULL; + ring_info->pkt_buffer_size = 0; +} + +/* + * Check if the ring buffer spinlock is available to take or not; used on + * atomic contexts, like panic path (see the Hyper-V framebuffer driver). + */ + +bool hv_ringbuffer_spinlock_busy(struct vmbus_channel *channel) +{ + struct hv_ring_buffer_info *rinfo = &channel->outbound; + + return spin_is_locked(&rinfo->ring_lock); +} +EXPORT_SYMBOL_GPL(hv_ringbuffer_spinlock_busy); + +/* Write to the ring buffer. */ +int hv_ringbuffer_write(struct vmbus_channel *channel, + const struct kvec *kv_list, u32 kv_count, + u64 requestid, u64 *trans_id) +{ + int i; + u32 bytes_avail_towrite; + u32 totalbytes_towrite = sizeof(u64); + u32 next_write_location; + u32 old_write; + u64 prev_indices; + unsigned long flags; + struct hv_ring_buffer_info *outring_info = &channel->outbound; + struct vmpacket_descriptor *desc = kv_list[0].iov_base; + u64 __trans_id, rqst_id = VMBUS_NO_RQSTOR; + + if (channel->rescind) + return -ENODEV; + + for (i = 0; i < kv_count; i++) + totalbytes_towrite += kv_list[i].iov_len; + + spin_lock_irqsave(&outring_info->ring_lock, flags); + + bytes_avail_towrite = hv_get_bytes_to_write(outring_info); + + /* + * If there is only room for the packet, assume it is full. + * Otherwise, the next time around, we think the ring buffer + * is empty since the read index == write index. + */ + if (bytes_avail_towrite <= totalbytes_towrite) { + ++channel->out_full_total; + + if (!channel->out_full_flag) { + ++channel->out_full_first; + channel->out_full_flag = true; + } + + spin_unlock_irqrestore(&outring_info->ring_lock, flags); + return -EAGAIN; + } + + channel->out_full_flag = false; + + /* Write to the ring buffer */ + next_write_location = hv_get_next_write_location(outring_info); + + old_write = next_write_location; + + for (i = 0; i < kv_count; i++) { + next_write_location = hv_copyto_ringbuffer(outring_info, + next_write_location, + kv_list[i].iov_base, + kv_list[i].iov_len); + } + + /* + * Allocate the request ID after the data has been copied into the + * ring buffer. Once this request ID is allocated, the completion + * path could find the data and free it. + */ + + if (desc->flags == VMBUS_DATA_PACKET_FLAG_COMPLETION_REQUESTED) { + if (channel->next_request_id_callback != NULL) { + rqst_id = channel->next_request_id_callback(channel, requestid); + if (rqst_id == VMBUS_RQST_ERROR) { + spin_unlock_irqrestore(&outring_info->ring_lock, flags); + return -EAGAIN; + } + } + } + desc = hv_get_ring_buffer(outring_info) + old_write; + __trans_id = (rqst_id == VMBUS_NO_RQSTOR) ? requestid : rqst_id; + /* + * Ensure the compiler doesn't generate code that reads the value of + * the transaction ID from the ring buffer, which is shared with the + * Hyper-V host and subject to being changed at any time. + */ + WRITE_ONCE(desc->trans_id, __trans_id); + if (trans_id) + *trans_id = __trans_id; + + /* Set previous packet start */ + prev_indices = hv_get_ring_bufferindices(outring_info); + + next_write_location = hv_copyto_ringbuffer(outring_info, + next_write_location, + &prev_indices, + sizeof(u64)); + + /* Issue a full memory barrier before updating the write index */ + virt_mb(); + + /* Now, update the write location */ + hv_set_next_write_location(outring_info, next_write_location); + + + spin_unlock_irqrestore(&outring_info->ring_lock, flags); + + hv_signal_on_write(old_write, channel); + + if (channel->rescind) { + if (rqst_id != VMBUS_NO_RQSTOR) { + /* Reclaim request ID to avoid leak of IDs */ + if (channel->request_addr_callback != NULL) + channel->request_addr_callback(channel, rqst_id); + } + return -ENODEV; + } + + return 0; +} + +int hv_ringbuffer_read(struct vmbus_channel *channel, + void *buffer, u32 buflen, u32 *buffer_actual_len, + u64 *requestid, bool raw) +{ + struct vmpacket_descriptor *desc; + u32 packetlen, offset; + + if (unlikely(buflen == 0)) + return -EINVAL; + + *buffer_actual_len = 0; + *requestid = 0; + + /* Make sure there is something to read */ + desc = hv_pkt_iter_first(channel); + if (desc == NULL) { + /* + * No error is set when there is even no header, drivers are + * supposed to analyze buffer_actual_len. + */ + return 0; + } + + offset = raw ? 0 : (desc->offset8 << 3); + packetlen = (desc->len8 << 3) - offset; + *buffer_actual_len = packetlen; + *requestid = desc->trans_id; + + if (unlikely(packetlen > buflen)) + return -ENOBUFS; + + /* since ring is double mapped, only one copy is necessary */ + memcpy(buffer, (const char *)desc + offset, packetlen); + + /* Advance ring index to next packet descriptor */ + __hv_pkt_iter_next(channel, desc); + + /* Notify host of update */ + hv_pkt_iter_close(channel); + + return 0; +} + +/* + * Determine number of bytes available in ring buffer after + * the current iterator (priv_read_index) location. + * + * This is similar to hv_get_bytes_to_read but with private + * read index instead. + */ +static u32 hv_pkt_iter_avail(const struct hv_ring_buffer_info *rbi) +{ + u32 priv_read_loc = rbi->priv_read_index; + u32 write_loc; + + /* + * The Hyper-V host writes the packet data, then uses + * store_release() to update the write_index. Use load_acquire() + * here to prevent loads of the packet data from being re-ordered + * before the read of the write_index and potentially getting + * stale data. + */ + write_loc = virt_load_acquire(&rbi->ring_buffer->write_index); + + if (write_loc >= priv_read_loc) + return write_loc - priv_read_loc; + else + return (rbi->ring_datasize - priv_read_loc) + write_loc; +} + +/* + * Get first vmbus packet from ring buffer after read_index + * + * If ring buffer is empty, returns NULL and no other action needed. + */ +struct vmpacket_descriptor *hv_pkt_iter_first(struct vmbus_channel *channel) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; + struct vmpacket_descriptor *desc, *desc_copy; + u32 bytes_avail, pkt_len, pkt_offset; + + hv_debug_delay_test(channel, MESSAGE_DELAY); + + bytes_avail = hv_pkt_iter_avail(rbi); + if (bytes_avail < sizeof(struct vmpacket_descriptor)) + return NULL; + bytes_avail = min(rbi->pkt_buffer_size, bytes_avail); + + desc = (struct vmpacket_descriptor *)(hv_get_ring_buffer(rbi) + rbi->priv_read_index); + + /* + * Ensure the compiler does not use references to incoming Hyper-V values (which + * could change at any moment) when reading local variables later in the code + */ + pkt_len = READ_ONCE(desc->len8) << 3; + pkt_offset = READ_ONCE(desc->offset8) << 3; + + /* + * If pkt_len is invalid, set it to the smaller of hv_pkt_iter_avail() and + * rbi->pkt_buffer_size + */ + if (pkt_len < sizeof(struct vmpacket_descriptor) || pkt_len > bytes_avail) + pkt_len = bytes_avail; + + /* + * If pkt_offset is invalid, arbitrarily set it to + * the size of vmpacket_descriptor + */ + if (pkt_offset < sizeof(struct vmpacket_descriptor) || pkt_offset > pkt_len) + pkt_offset = sizeof(struct vmpacket_descriptor); + + /* Copy the Hyper-V packet out of the ring buffer */ + desc_copy = (struct vmpacket_descriptor *)rbi->pkt_buffer; + memcpy(desc_copy, desc, pkt_len); + + /* + * Hyper-V could still change len8 and offset8 after the earlier read. + * Ensure that desc_copy has legal values for len8 and offset8 that + * are consistent with the copy we just made + */ + desc_copy->len8 = pkt_len >> 3; + desc_copy->offset8 = pkt_offset >> 3; + + return desc_copy; +} +EXPORT_SYMBOL_GPL(hv_pkt_iter_first); + +/* + * Get next vmbus packet from ring buffer. + * + * Advances the current location (priv_read_index) and checks for more + * data. If the end of the ring buffer is reached, then return NULL. + */ +struct vmpacket_descriptor * +__hv_pkt_iter_next(struct vmbus_channel *channel, + const struct vmpacket_descriptor *desc) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; + u32 packetlen = desc->len8 << 3; + u32 dsize = rbi->ring_datasize; + + hv_debug_delay_test(channel, MESSAGE_DELAY); + /* bump offset to next potential packet */ + rbi->priv_read_index += packetlen + VMBUS_PKT_TRAILER; + if (rbi->priv_read_index >= dsize) + rbi->priv_read_index -= dsize; + + /* more data? */ + return hv_pkt_iter_first(channel); +} +EXPORT_SYMBOL_GPL(__hv_pkt_iter_next); + +/* How many bytes were read in this iterator cycle */ +static u32 hv_pkt_iter_bytes_read(const struct hv_ring_buffer_info *rbi, + u32 start_read_index) +{ + if (rbi->priv_read_index >= start_read_index) + return rbi->priv_read_index - start_read_index; + else + return rbi->ring_datasize - start_read_index + + rbi->priv_read_index; +} + +/* + * Update host ring buffer after iterating over packets. If the host has + * stopped queuing new entries because it found the ring buffer full, and + * sufficient space is being freed up, signal the host. But be careful to + * only signal the host when necessary, both for performance reasons and + * because Hyper-V protects itself by throttling guests that signal + * inappropriately. + * + * Determining when to signal is tricky. There are three key data inputs + * that must be handled in this order to avoid race conditions: + * + * 1. Update the read_index + * 2. Read the pending_send_sz + * 3. Read the current write_index + * + * The interrupt_mask is not used to determine when to signal. The + * interrupt_mask is used only on the guest->host ring buffer when + * sending requests to the host. The host does not use it on the host-> + * guest ring buffer to indicate whether it should be signaled. + */ +void hv_pkt_iter_close(struct vmbus_channel *channel) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; + u32 curr_write_sz, pending_sz, bytes_read, start_read_index; + + /* + * Make sure all reads are done before we update the read index since + * the writer may start writing to the read area once the read index + * is updated. + */ + virt_rmb(); + start_read_index = rbi->ring_buffer->read_index; + rbi->ring_buffer->read_index = rbi->priv_read_index; + + /* + * Older versions of Hyper-V (before WS2102 and Win8) do not + * implement pending_send_sz and simply poll if the host->guest + * ring buffer is full. No signaling is needed or expected. + */ + if (!rbi->ring_buffer->feature_bits.feat_pending_send_sz) + return; + + /* + * Issue a full memory barrier before making the signaling decision. + * If reading pending_send_sz were to be reordered and happen + * before we commit the new read_index, a race could occur. If the + * host were to set the pending_send_sz after we have sampled + * pending_send_sz, and the ring buffer blocks before we commit the + * read index, we could miss sending the interrupt. Issue a full + * memory barrier to address this. + */ + virt_mb(); + + /* + * If the pending_send_sz is zero, then the ring buffer is not + * blocked and there is no need to signal. This is far by the + * most common case, so exit quickly for best performance. + */ + pending_sz = READ_ONCE(rbi->ring_buffer->pending_send_sz); + if (!pending_sz) + return; + + /* + * Ensure the read of write_index in hv_get_bytes_to_write() + * happens after the read of pending_send_sz. + */ + virt_rmb(); + curr_write_sz = hv_get_bytes_to_write(rbi); + bytes_read = hv_pkt_iter_bytes_read(rbi, start_read_index); + + /* + * We want to signal the host only if we're transitioning + * from a "not enough free space" state to a "enough free + * space" state. For example, it's possible that this function + * could run and free up enough space to signal the host, and then + * run again and free up additional space before the host has a + * chance to clear the pending_send_sz. The 2nd invocation would + * be a null transition from "enough free space" to "enough free + * space", which doesn't warrant a signal. + * + * Exactly filling the ring buffer is treated as "not enough + * space". The ring buffer always must have at least one byte + * empty so the empty and full conditions are distinguishable. + * hv_get_bytes_to_write() doesn't fully tell the truth in + * this regard. + * + * So first check if we were in the "enough free space" state + * before we began the iteration. If so, the host was not + * blocked, and there's no need to signal. + */ + if (curr_write_sz - bytes_read > pending_sz) + return; + + /* + * Similarly, if the new state is "not enough space", then + * there's no need to signal. + */ + if (curr_write_sz <= pending_sz) + return; + + ++channel->intr_in_full; + vmbus_setevent(channel); +} +EXPORT_SYMBOL_GPL(hv_pkt_iter_close); diff --git a/drivers/hv/vmbus_drv.c b/drivers/hv/vmbus_drv.c new file mode 100644 index 000000000..23206640c --- /dev/null +++ b/drivers/hv/vmbus_drv.c @@ -0,0 +1,3071 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2009, Microsoft Corporation. + * + * Authors: + * Haiyang Zhang <haiyangz@microsoft.com> + * Hank Janssen <hjanssen@microsoft.com> + * K. Y. Srinivasan <kys@microsoft.com> + */ +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/device.h> +#include <linux/platform_device.h> +#include <linux/interrupt.h> +#include <linux/sysctl.h> +#include <linux/slab.h> +#include <linux/acpi.h> +#include <linux/completion.h> +#include <linux/hyperv.h> +#include <linux/kernel_stat.h> +#include <linux/of_address.h> +#include <linux/clockchips.h> +#include <linux/cpu.h> +#include <linux/sched/isolation.h> +#include <linux/sched/task_stack.h> +#include <linux/smpboot.h> + +#include <linux/delay.h> +#include <linux/panic_notifier.h> +#include <linux/ptrace.h> +#include <linux/sysfb.h> +#include <linux/efi.h> +#include <linux/kernel.h> +#include <linux/syscore_ops.h> +#include <linux/dma-map-ops.h> +#include <linux/pci.h> +#include <linux/export.h> +#include <clocksource/hyperv_timer.h> +#include <asm/mshyperv.h> +#include "hyperv_vmbus.h" + +struct vmbus_dynid { + struct list_head node; + struct hv_vmbus_device_id id; +}; + +/* VMBus Root Device */ +static struct device *vmbus_root_device; + +static int hyperv_cpuhp_online; + +static DEFINE_PER_CPU(long, vmbus_evt); + +/* Values parsed from ACPI DSDT */ +int vmbus_irq; +int vmbus_interrupt; + +/* + * If the Confidential VMBus is used, the data on the "wire" is not + * visible to either the host or the hypervisor. + */ +static bool is_confidential; + +bool vmbus_is_confidential(void) +{ + return is_confidential; +} +EXPORT_SYMBOL_GPL(vmbus_is_confidential); + +static bool skip_vmbus_unload; + +/* + * Allow a VMBus framebuffer driver to specify that in the case of a panic, + * it will do the VMbus unload operation once it has flushed any dirty + * portions of the framebuffer to the Hyper-V host. + */ +void vmbus_set_skip_unload(bool skip) +{ + skip_vmbus_unload = skip; +} +EXPORT_SYMBOL_GPL(vmbus_set_skip_unload); + +/* + * The panic notifier below is responsible solely for unloading the + * vmbus connection, which is necessary in a panic event. + */ +static int hv_panic_vmbus_unload(struct notifier_block *nb, unsigned long val, + void *args) +{ + if (!skip_vmbus_unload) + vmbus_initiate_unload(true); + + return NOTIFY_DONE; +} +static struct notifier_block hyperv_panic_vmbus_unload_block = { + .notifier_call = hv_panic_vmbus_unload, + .priority = INT_MIN + 1, /* almost the latest one to execute */ +}; + +static const char *fb_mmio_name = "fb_range"; +static struct resource *fb_mmio; +static struct resource *hyperv_mmio; +static DEFINE_MUTEX(hyperv_mmio_lock); + +struct device *hv_get_vmbus_root_device(void) +{ + return vmbus_root_device; +} +EXPORT_SYMBOL_GPL(hv_get_vmbus_root_device); + +bool hv_vmbus_exists(void) +{ + return vmbus_root_device != NULL; +} +EXPORT_SYMBOL_GPL(hv_vmbus_exists); + +static u8 channel_monitor_group(const struct vmbus_channel *channel) +{ + return (u8)channel->offermsg.monitorid / 32; +} + +static u8 channel_monitor_offset(const struct vmbus_channel *channel) +{ + return (u8)channel->offermsg.monitorid % 32; +} + +static u32 channel_pending(const struct vmbus_channel *channel, + const struct hv_monitor_page *monitor_page) +{ + u8 monitor_group = channel_monitor_group(channel); + + return monitor_page->trigger_group[monitor_group].pending; +} + +static u32 channel_latency(const struct vmbus_channel *channel, + const struct hv_monitor_page *monitor_page) +{ + u8 monitor_group = channel_monitor_group(channel); + u8 monitor_offset = channel_monitor_offset(channel); + + return monitor_page->latency[monitor_group][monitor_offset]; +} + +static u32 channel_conn_id(struct vmbus_channel *channel, + struct hv_monitor_page *monitor_page) +{ + u8 monitor_group = channel_monitor_group(channel); + u8 monitor_offset = channel_monitor_offset(channel); + + return monitor_page->parameter[monitor_group][monitor_offset].connectionid.u.id; +} + +static ssize_t id_show(struct device *dev, struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", hv_dev->channel->offermsg.child_relid); +} +static DEVICE_ATTR_RO(id); + +static ssize_t state_show(struct device *dev, struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", hv_dev->channel->state); +} +static DEVICE_ATTR_RO(state); + +static ssize_t monitor_id_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", hv_dev->channel->offermsg.monitorid); +} +static DEVICE_ATTR_RO(monitor_id); + +static ssize_t class_id_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "{%pUl}\n", + &hv_dev->channel->offermsg.offer.if_type); +} +static DEVICE_ATTR_RO(class_id); + +static ssize_t device_id_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "{%pUl}\n", + &hv_dev->channel->offermsg.offer.if_instance); +} +static DEVICE_ATTR_RO(device_id); + +static ssize_t modalias_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + return sysfs_emit(buf, "vmbus:%*phN\n", UUID_SIZE, &hv_dev->dev_type); +} +static DEVICE_ATTR_RO(modalias); + +#ifdef CONFIG_NUMA +static ssize_t numa_node_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + + return sysfs_emit(buf, "%d\n", cpu_to_node(hv_dev->channel->target_cpu)); +} +static DEVICE_ATTR_RO(numa_node); +#endif + +static ssize_t server_monitor_pending_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", channel_pending(hv_dev->channel, + vmbus_connection.monitor_pages[0])); +} +static DEVICE_ATTR_RO(server_monitor_pending); + +static ssize_t client_monitor_pending_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", channel_pending(hv_dev->channel, + vmbus_connection.monitor_pages[1])); +} +static DEVICE_ATTR_RO(client_monitor_pending); + +static ssize_t server_monitor_latency_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", channel_latency(hv_dev->channel, + vmbus_connection.monitor_pages[0])); +} +static DEVICE_ATTR_RO(server_monitor_latency); + +static ssize_t client_monitor_latency_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", channel_latency(hv_dev->channel, + vmbus_connection.monitor_pages[1])); +} +static DEVICE_ATTR_RO(client_monitor_latency); + +static ssize_t server_monitor_conn_id_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", channel_conn_id(hv_dev->channel, + vmbus_connection.monitor_pages[0])); +} +static DEVICE_ATTR_RO(server_monitor_conn_id); + +static ssize_t client_monitor_conn_id_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + if (!hv_dev->channel) + return -ENODEV; + return sysfs_emit(buf, "%d\n", channel_conn_id(hv_dev->channel, + vmbus_connection.monitor_pages[1])); +} +static DEVICE_ATTR_RO(client_monitor_conn_id); + +static ssize_t out_intr_mask_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info outbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, + &outbound); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", outbound.current_interrupt_mask); +} +static DEVICE_ATTR_RO(out_intr_mask); + +static ssize_t out_read_index_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info outbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, + &outbound); + if (ret < 0) + return ret; + return sysfs_emit(buf, "%u\n", outbound.current_read_index); +} +static DEVICE_ATTR_RO(out_read_index); + +static ssize_t out_write_index_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info outbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, + &outbound); + if (ret < 0) + return ret; + return sysfs_emit(buf, "%u\n", outbound.current_write_index); +} +static DEVICE_ATTR_RO(out_write_index); + +static ssize_t out_read_bytes_avail_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info outbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, + &outbound); + if (ret < 0) + return ret; + return sysfs_emit(buf, "%d\n", outbound.bytes_avail_toread); +} +static DEVICE_ATTR_RO(out_read_bytes_avail); + +static ssize_t out_write_bytes_avail_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info outbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->outbound, + &outbound); + if (ret < 0) + return ret; + return sysfs_emit(buf, "%d\n", outbound.bytes_avail_towrite); +} +static DEVICE_ATTR_RO(out_write_bytes_avail); + +static ssize_t in_intr_mask_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info inbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", inbound.current_interrupt_mask); +} +static DEVICE_ATTR_RO(in_intr_mask); + +static ssize_t in_read_index_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info inbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", inbound.current_read_index); +} +static DEVICE_ATTR_RO(in_read_index); + +static ssize_t in_write_index_show(struct device *dev, + struct device_attribute *dev_attr, char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info inbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", inbound.current_write_index); +} +static DEVICE_ATTR_RO(in_write_index); + +static ssize_t in_read_bytes_avail_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info inbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", inbound.bytes_avail_toread); +} +static DEVICE_ATTR_RO(in_read_bytes_avail); + +static ssize_t in_write_bytes_avail_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct hv_ring_buffer_debug_info inbound; + int ret; + + if (!hv_dev->channel) + return -ENODEV; + + ret = hv_ringbuffer_get_debuginfo(&hv_dev->channel->inbound, &inbound); + if (ret < 0) + return ret; + + return sysfs_emit(buf, "%d\n", inbound.bytes_avail_towrite); +} +static DEVICE_ATTR_RO(in_write_bytes_avail); + +static ssize_t channel_vp_mapping_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + struct vmbus_channel *channel = hv_dev->channel, *cur_sc; + int n_written; + struct list_head *cur; + + if (!channel) + return -ENODEV; + + mutex_lock(&vmbus_connection.channel_mutex); + + n_written = sysfs_emit(buf, "%u:%u\n", + channel->offermsg.child_relid, + channel->target_cpu); + + list_for_each(cur, &channel->sc_list) { + + cur_sc = list_entry(cur, struct vmbus_channel, sc_list); + n_written += sysfs_emit_at(buf, n_written, "%u:%u\n", + cur_sc->offermsg.child_relid, + cur_sc->target_cpu); + } + + mutex_unlock(&vmbus_connection.channel_mutex); + + return n_written; +} +static DEVICE_ATTR_RO(channel_vp_mapping); + +static ssize_t vendor_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + return sysfs_emit(buf, "0x%x\n", hv_dev->vendor_id); +} +static DEVICE_ATTR_RO(vendor); + +static ssize_t device_show(struct device *dev, + struct device_attribute *dev_attr, + char *buf) +{ + struct hv_device *hv_dev = device_to_hv_device(dev); + + return sysfs_emit(buf, "0x%x\n", hv_dev->device_id); +} +static DEVICE_ATTR_RO(device); + +/* Set up per device attributes in /sys/bus/vmbus/devices/<bus device> */ +static struct attribute *vmbus_dev_attrs[] = { + &dev_attr_id.attr, + &dev_attr_state.attr, + &dev_attr_monitor_id.attr, + &dev_attr_class_id.attr, + &dev_attr_device_id.attr, + &dev_attr_modalias.attr, +#ifdef CONFIG_NUMA + &dev_attr_numa_node.attr, +#endif + &dev_attr_server_monitor_pending.attr, + &dev_attr_client_monitor_pending.attr, + &dev_attr_server_monitor_latency.attr, + &dev_attr_client_monitor_latency.attr, + &dev_attr_server_monitor_conn_id.attr, + &dev_attr_client_monitor_conn_id.attr, + &dev_attr_out_intr_mask.attr, + &dev_attr_out_read_index.attr, + &dev_attr_out_write_index.attr, + &dev_attr_out_read_bytes_avail.attr, + &dev_attr_out_write_bytes_avail.attr, + &dev_attr_in_intr_mask.attr, + &dev_attr_in_read_index.attr, + &dev_attr_in_write_index.attr, + &dev_attr_in_read_bytes_avail.attr, + &dev_attr_in_write_bytes_avail.attr, + &dev_attr_channel_vp_mapping.attr, + &dev_attr_vendor.attr, + &dev_attr_device.attr, + NULL, +}; + +/* + * Device-level attribute_group callback function. Returns the permission for + * each attribute, and returns 0 if an attribute is not visible. + */ +static umode_t vmbus_dev_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int idx) +{ + struct device *dev = kobj_to_dev(kobj); + const struct hv_device *hv_dev = device_to_hv_device(dev); + + /* Hide the monitor attributes if the monitor mechanism is not used. */ + if (!hv_dev->channel->offermsg.monitor_allocated && + (attr == &dev_attr_monitor_id.attr || + attr == &dev_attr_server_monitor_pending.attr || + attr == &dev_attr_client_monitor_pending.attr || + attr == &dev_attr_server_monitor_latency.attr || + attr == &dev_attr_client_monitor_latency.attr || + attr == &dev_attr_server_monitor_conn_id.attr || + attr == &dev_attr_client_monitor_conn_id.attr)) + return 0; + + return attr->mode; +} + +static const struct attribute_group vmbus_dev_group = { + .attrs = vmbus_dev_attrs, + .is_visible = vmbus_dev_attr_is_visible +}; +__ATTRIBUTE_GROUPS(vmbus_dev); + +/* Set up the attribute for /sys/bus/vmbus/hibernation */ +static ssize_t hibernation_show(const struct bus_type *bus, char *buf) +{ + return sprintf(buf, "%d\n", !!hv_is_hibernation_supported()); +} + +static BUS_ATTR_RO(hibernation); + +static struct attribute *vmbus_bus_attrs[] = { + &bus_attr_hibernation.attr, + NULL, +}; +static const struct attribute_group vmbus_bus_group = { + .attrs = vmbus_bus_attrs, +}; +__ATTRIBUTE_GROUPS(vmbus_bus); + +/* + * vmbus_uevent - add uevent for our device + * + * This routine is invoked when a device is added or removed on the vmbus to + * generate a uevent to udev in the userspace. The udev will then look at its + * rule and the uevent generated here to load the appropriate driver + * + * The alias string will be of the form vmbus:guid where guid is the string + * representation of the device guid (each byte of the guid will be + * represented with two hex characters. + */ +static int vmbus_uevent(const struct device *device, struct kobj_uevent_env *env) +{ + const struct hv_device *dev = device_to_hv_device(device); + const char *format = "MODALIAS=vmbus:%*phN"; + + return add_uevent_var(env, format, UUID_SIZE, &dev->dev_type); +} + +static const struct hv_vmbus_device_id * +hv_vmbus_dev_match(const struct hv_vmbus_device_id *id, const guid_t *guid) +{ + if (id == NULL) + return NULL; /* empty device table */ + + for (; !guid_is_null(&id->guid); id++) + if (guid_equal(&id->guid, guid)) + return id; + + return NULL; +} + +static const struct hv_vmbus_device_id * +hv_vmbus_dynid_match(struct hv_driver *drv, const guid_t *guid) +{ + const struct hv_vmbus_device_id *id = NULL; + struct vmbus_dynid *dynid; + + spin_lock(&drv->dynids.lock); + list_for_each_entry(dynid, &drv->dynids.list, node) { + if (guid_equal(&dynid->id.guid, guid)) { + id = &dynid->id; + break; + } + } + spin_unlock(&drv->dynids.lock); + + return id; +} + +static const struct hv_vmbus_device_id vmbus_device_null; + +/* + * Return a matching hv_vmbus_device_id pointer. + * If there is no match, return NULL. + */ +static const struct hv_vmbus_device_id *hv_vmbus_get_id(const struct hv_driver *drv, + struct hv_device *dev) +{ + const guid_t *guid = &dev->dev_type; + const struct hv_vmbus_device_id *id; + int ret; + + /* If a driver override is set, only bind to the matching driver */ + ret = device_match_driver_override(&dev->device, &drv->driver); + if (ret == 0) + return NULL; + + /* Look at the dynamic ids first, before the static ones */ + id = hv_vmbus_dynid_match((struct hv_driver *)drv, guid); + if (!id) + id = hv_vmbus_dev_match(drv->id_table, guid); + + /* + * If there's a matching driver override, this function should succeed, + * thus return a dummy device ID if no matching ID is found. + */ + if (!id && ret > 0) + id = &vmbus_device_null; + + return id; +} + +/* vmbus_add_dynid - add a new device ID to this driver and re-probe devices + * + * This function can race with vmbus_device_register(). This function is + * typically running on a user thread in response to writing to the "new_id" + * sysfs entry for a driver. vmbus_device_register() is running on a + * workqueue thread in response to the Hyper-V host offering a device to the + * guest. This function calls driver_attach(), which looks for an existing + * device matching the new id, and attaches the driver to which the new id + * has been assigned. vmbus_device_register() calls device_register(), which + * looks for a driver that matches the device being registered. If both + * operations are running simultaneously, the device driver probe function runs + * on whichever thread establishes the linkage between the driver and device. + * + * In most cases, it doesn't matter which thread runs the driver probe + * function. But if vmbus_device_register() does not find a matching driver, + * it proceeds to create the "channels" subdirectory and numbered per-channel + * subdirectory in sysfs. While that multi-step creation is in progress, this + * function could run the driver probe function. If the probe function checks + * for, or operates on, entries in the "channels" subdirectory, including by + * calling hv_create_ring_sysfs(), the operation may or may not succeed + * depending on the race. The race can't create a kernel failure in VMBus + * or device subsystem code, but probe functions in VMBus drivers doing such + * operations must be prepared for the failure case. + */ +static int vmbus_add_dynid(struct hv_driver *drv, guid_t *guid) +{ + struct vmbus_dynid *dynid; + + dynid = kzalloc_obj(*dynid); + if (!dynid) + return -ENOMEM; + + dynid->id.guid = *guid; + + spin_lock(&drv->dynids.lock); + list_add_tail(&dynid->node, &drv->dynids.list); + spin_unlock(&drv->dynids.lock); + + return driver_attach(&drv->driver); +} + +static void vmbus_free_dynids(struct hv_driver *drv) +{ + struct vmbus_dynid *dynid, *n; + + spin_lock(&drv->dynids.lock); + list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) { + list_del(&dynid->node); + kfree(dynid); + } + spin_unlock(&drv->dynids.lock); +} + +/* + * store_new_id - sysfs frontend to vmbus_add_dynid() + * + * Allow GUIDs to be added to an existing driver via sysfs. + */ +static ssize_t new_id_store(struct device_driver *driver, const char *buf, + size_t count) +{ + struct hv_driver *drv = drv_to_hv_drv(driver); + guid_t guid; + ssize_t retval; + + retval = guid_parse(buf, &guid); + if (retval) + return retval; + + if (hv_vmbus_dynid_match(drv, &guid)) + return -EEXIST; + + retval = vmbus_add_dynid(drv, &guid); + if (retval) + return retval; + return count; +} +static DRIVER_ATTR_WO(new_id); + +/* + * store_remove_id - remove a PCI device ID from this driver + * + * Removes a dynamic pci device ID to this driver. + */ +static ssize_t remove_id_store(struct device_driver *driver, const char *buf, + size_t count) +{ + struct hv_driver *drv = drv_to_hv_drv(driver); + struct vmbus_dynid *dynid, *n; + guid_t guid; + ssize_t retval; + + retval = guid_parse(buf, &guid); + if (retval) + return retval; + + retval = -ENODEV; + spin_lock(&drv->dynids.lock); + list_for_each_entry_safe(dynid, n, &drv->dynids.list, node) { + struct hv_vmbus_device_id *id = &dynid->id; + + if (guid_equal(&id->guid, &guid)) { + list_del(&dynid->node); + kfree(dynid); + retval = count; + break; + } + } + spin_unlock(&drv->dynids.lock); + + return retval; +} +static DRIVER_ATTR_WO(remove_id); + +static struct attribute *vmbus_drv_attrs[] = { + &driver_attr_new_id.attr, + &driver_attr_remove_id.attr, + NULL, +}; +ATTRIBUTE_GROUPS(vmbus_drv); + + +/* + * vmbus_match - Attempt to match the specified device to the specified driver + */ +static int vmbus_match(struct device *device, const struct device_driver *driver) +{ + const struct hv_driver *drv = drv_to_hv_drv(driver); + struct hv_device *hv_dev = device_to_hv_device(device); + + /* The hv_sock driver handles all hv_sock offers. */ + if (is_hvsock_channel(hv_dev->channel)) + return drv->hvsock; + + if (hv_vmbus_get_id(drv, hv_dev)) + return 1; + + return 0; +} + +/* + * vmbus_probe - Add the new vmbus's child device + */ +static int vmbus_probe(struct device *child_device) +{ + int ret = 0; + struct hv_driver *drv = + drv_to_hv_drv(child_device->driver); + struct hv_device *dev = device_to_hv_device(child_device); + const struct hv_vmbus_device_id *dev_id; + + dev_id = hv_vmbus_get_id(drv, dev); + if (drv->probe) { + ret = drv->probe(dev, dev_id); + if (ret != 0) + pr_err("probe failed for device %s (%d)\n", + dev_name(child_device), ret); + + } else { + pr_err("probe not set for driver %s\n", + dev_name(child_device)); + ret = -ENODEV; + } + return ret; +} + +/* + * vmbus_dma_configure -- Configure DMA coherence for VMbus device + */ +static int vmbus_dma_configure(struct device *child_device) +{ + /* + * On ARM64, propagate the DMA coherence setting from the top level + * VMbus ACPI device to the child VMbus device being added here. + * On x86/x64 coherence is assumed and these calls have no effect. + */ + hv_setup_dma_ops(child_device, + device_get_dma_attr(vmbus_root_device) == DEV_DMA_COHERENT); + return 0; +} + +/* + * vmbus_remove - Remove a vmbus device + */ +static void vmbus_remove(struct device *child_device) +{ + struct hv_driver *drv; + struct hv_device *dev = device_to_hv_device(child_device); + + if (child_device->driver) { + drv = drv_to_hv_drv(child_device->driver); + if (drv->remove) + drv->remove(dev); + } +} + +/* + * vmbus_shutdown - Shutdown a vmbus device + */ +static void vmbus_shutdown(struct device *child_device) +{ + struct hv_driver *drv; + struct hv_device *dev = device_to_hv_device(child_device); + + + /* The device may not be attached yet */ + if (!child_device->driver) + return; + + drv = drv_to_hv_drv(child_device->driver); + + if (drv->shutdown) + drv->shutdown(dev); +} + +#ifdef CONFIG_PM_SLEEP +/* + * vmbus_suspend - Suspend a vmbus device + */ +static int vmbus_suspend(struct device *child_device) +{ + struct hv_driver *drv; + struct hv_device *dev = device_to_hv_device(child_device); + + /* The device may not be attached yet */ + if (!child_device->driver) + return 0; + + drv = drv_to_hv_drv(child_device->driver); + if (!drv->suspend) + return -EOPNOTSUPP; + + return drv->suspend(dev); +} + +/* + * vmbus_resume - Resume a vmbus device + */ +static int vmbus_resume(struct device *child_device) +{ + struct hv_driver *drv; + struct hv_device *dev = device_to_hv_device(child_device); + + /* The device may not be attached yet */ + if (!child_device->driver) + return 0; + + drv = drv_to_hv_drv(child_device->driver); + if (!drv->resume) + return -EOPNOTSUPP; + + return drv->resume(dev); +} +#else +#define vmbus_suspend NULL +#define vmbus_resume NULL +#endif /* CONFIG_PM_SLEEP */ + +/* + * vmbus_device_release - Final callback release of the vmbus child device + */ +static void vmbus_device_release(struct device *device) +{ + struct hv_device *hv_dev = device_to_hv_device(device); + struct vmbus_channel *channel = hv_dev->channel; + + hv_debug_rm_dev_dir(hv_dev); + + mutex_lock(&vmbus_connection.channel_mutex); + hv_process_channel_removal(channel); + mutex_unlock(&vmbus_connection.channel_mutex); + kfree(hv_dev); +} + +/* + * Note: we must use the "noirq" ops: see the comment before vmbus_bus_pm. + * + * suspend_noirq/resume_noirq are set to NULL to support Suspend-to-Idle: we + * shouldn't suspend the vmbus devices upon Suspend-to-Idle, otherwise there + * is no way to wake up a Generation-2 VM. + * + * The other 4 ops are for hibernation. + */ + +static const struct dev_pm_ops vmbus_pm = { + .suspend_noirq = NULL, + .resume_noirq = NULL, + .freeze_noirq = vmbus_suspend, + .thaw_noirq = vmbus_resume, + .poweroff_noirq = vmbus_suspend, + .restore_noirq = vmbus_resume, +}; + +/* The one and only one */ +static const struct bus_type hv_bus = { + .name = "vmbus", + .driver_override = true, + .match = vmbus_match, + .shutdown = vmbus_shutdown, + .remove = vmbus_remove, + .probe = vmbus_probe, + .uevent = vmbus_uevent, + .dma_configure = vmbus_dma_configure, + .dev_groups = vmbus_dev_groups, + .drv_groups = vmbus_drv_groups, + .bus_groups = vmbus_bus_groups, + .pm = &vmbus_pm, +}; + +struct onmessage_work_context { + struct work_struct work; + struct { + struct hv_message_header header; + u8 payload[]; + } msg; +}; + +static void vmbus_onmessage_work(struct work_struct *work) +{ + struct onmessage_work_context *ctx; + + /* Do not process messages if we're in DISCONNECTED state */ + if (vmbus_connection.conn_state == DISCONNECTED) + return; + + ctx = container_of(work, struct onmessage_work_context, + work); + vmbus_onmessage((struct vmbus_channel_message_header *) + &ctx->msg.payload); + kfree(ctx); +} + +static void __vmbus_on_msg_dpc(void *message_page_addr) +{ + struct hv_message msg_copy, *msg; + struct vmbus_channel_message_header *hdr; + enum vmbus_channel_message_type msgtype; + const struct vmbus_channel_message_table_entry *entry; + struct onmessage_work_context *ctx; + __u8 payload_size; + u32 message_type; + + if (!message_page_addr) + return; + msg = (struct hv_message *)message_page_addr + VMBUS_MESSAGE_SINT; + + /* + * 'enum vmbus_channel_message_type' is supposed to always be 'u32' as + * it is being used in 'struct vmbus_channel_message_header' definition + * which is supposed to match hypervisor ABI. + */ + BUILD_BUG_ON(sizeof(enum vmbus_channel_message_type) != sizeof(u32)); + + /* + * Since the message is in memory shared with the host, an erroneous or + * malicious Hyper-V could modify the message while vmbus_on_msg_dpc() + * or individual message handlers are executing; to prevent this, copy + * the message into private memory. + */ + memcpy(&msg_copy, msg, sizeof(struct hv_message)); + + message_type = msg_copy.header.message_type; + if (message_type == HVMSG_NONE) + /* no msg */ + return; + + hdr = (struct vmbus_channel_message_header *)msg_copy.u.payload; + msgtype = hdr->msgtype; + + trace_vmbus_on_msg_dpc(hdr); + + if (msgtype >= CHANNELMSG_COUNT) { + WARN_ONCE(1, "unknown msgtype=%d\n", msgtype); + goto msg_handled; + } + + payload_size = msg_copy.header.payload_size; + if (payload_size > HV_MESSAGE_PAYLOAD_BYTE_COUNT) { + WARN_ONCE(1, "payload size is too large (%d)\n", payload_size); + goto msg_handled; + } + + entry = &channel_message_table[msgtype]; + + if (!entry->message_handler) + goto msg_handled; + + if (payload_size < entry->min_payload_len) { + WARN_ONCE(1, "message too short: msgtype=%d len=%d\n", msgtype, payload_size); + goto msg_handled; + } + + if (entry->handler_type == VMHT_BLOCKING) { + ctx = kmalloc_flex(*ctx, msg.payload, payload_size, GFP_ATOMIC); + if (ctx == NULL) + return; + + INIT_WORK(&ctx->work, vmbus_onmessage_work); + ctx->msg.header = msg_copy.header; + memcpy(&ctx->msg.payload, msg_copy.u.payload, payload_size); + + /* + * The host can generate a rescind message while we + * may still be handling the original offer. We deal with + * this condition by relying on the synchronization provided + * by offer_in_progress and by channel_mutex. See also the + * inline comments in vmbus_onoffer_rescind(). + */ + switch (msgtype) { + case CHANNELMSG_RESCIND_CHANNELOFFER: + /* + * If we are handling the rescind message; + * schedule the work on the global work queue. + * + * The OFFER message and the RESCIND message should + * not be handled by the same serialized work queue, + * because the OFFER handler may call vmbus_open(), + * which tries to open the channel by sending an + * OPEN_CHANNEL message to the host and waits for + * the host's response; however, if the host has + * rescinded the channel before it receives the + * OPEN_CHANNEL message, the host just silently + * ignores the OPEN_CHANNEL message; as a result, + * the guest's OFFER handler hangs for ever, if we + * handle the RESCIND message in the same serialized + * work queue: the RESCIND handler can not start to + * run before the OFFER handler finishes. + */ + if (vmbus_connection.ignore_any_offer_msg) + break; + queue_work(vmbus_connection.rescind_work_queue, &ctx->work); + break; + + case CHANNELMSG_OFFERCHANNEL: + /* + * The host sends the offer message of a given channel + * before sending the rescind message of the same + * channel. These messages are sent to the guest's + * connect CPU; the guest then starts processing them + * in the tasklet handler on this CPU: + * + * VMBUS_CONNECT_CPU + * + * [vmbus_on_msg_dpc()] + * atomic_inc() // CHANNELMSG_OFFERCHANNEL + * queue_work() + * ... + * [vmbus_on_msg_dpc()] + * schedule_work() // CHANNELMSG_RESCIND_CHANNELOFFER + * + * We rely on the memory-ordering properties of the + * queue_work() and schedule_work() primitives, which + * guarantee that the atomic increment will be visible + * to the CPUs which will execute the offer & rescind + * works by the time these works will start execution. + */ + if (vmbus_connection.ignore_any_offer_msg) + break; + atomic_inc(&vmbus_connection.offer_in_progress); + fallthrough; + + default: + queue_work(vmbus_connection.work_queue, &ctx->work); + } + } else + entry->message_handler(hdr); + +msg_handled: + vmbus_signal_eom(msg, message_type); +} + +void vmbus_on_msg_dpc(unsigned long data) +{ + struct hv_per_cpu_context *hv_cpu = (void *)data; + + __vmbus_on_msg_dpc(hv_cpu->hyp_synic_message_page); + __vmbus_on_msg_dpc(hv_cpu->para_synic_message_page); +} + +#ifdef CONFIG_PM_SLEEP +/* + * Fake RESCIND_CHANNEL messages to clean up hv_sock channels by force for + * hibernation, because hv_sock connections can not persist across hibernation. + */ +static void vmbus_force_channel_rescinded(struct vmbus_channel *channel) +{ + struct onmessage_work_context *ctx; + struct vmbus_channel_rescind_offer *rescind; + + WARN_ON(!is_hvsock_channel(channel)); + + /* + * Allocation size is small and the allocation should really not fail, + * otherwise the state of the hv_sock connections ends up in limbo. + */ + ctx = kzalloc(sizeof(*ctx) + sizeof(*rescind), + GFP_KERNEL | __GFP_NOFAIL); + + /* + * So far, these are not really used by Linux. Just set them to the + * reasonable values conforming to the definitions of the fields. + */ + ctx->msg.header.message_type = 1; + ctx->msg.header.payload_size = sizeof(*rescind); + + /* These values are actually used by Linux. */ + rescind = (struct vmbus_channel_rescind_offer *)ctx->msg.payload; + rescind->header.msgtype = CHANNELMSG_RESCIND_CHANNELOFFER; + rescind->child_relid = channel->offermsg.child_relid; + + INIT_WORK(&ctx->work, vmbus_onmessage_work); + + queue_work(vmbus_connection.work_queue, &ctx->work); +} +#endif /* CONFIG_PM_SLEEP */ + +/* + * Schedule all channels with events pending. + * The event page can be directly checked to get the id of + * the channel that has the interrupt pending. + */ +static void vmbus_chan_sched(void *event_page_addr) +{ + unsigned long *recv_int_page; + u32 maxbits, relid; + union hv_synic_event_flags *event; + + if (!event_page_addr) + return; + event = (union hv_synic_event_flags *)event_page_addr + VMBUS_MESSAGE_SINT; + + maxbits = READ_ONCE(vmbus_connection.relid_hiwater) + 1; + recv_int_page = event->flags; + + if (unlikely(!recv_int_page)) + return; + + for_each_set_bit(relid, recv_int_page, maxbits) { + void (*callback_fn)(void *context); + struct vmbus_channel *channel; + + if (!sync_test_and_clear_bit(relid, recv_int_page)) + continue; + + /* Special case - vmbus channel protocol msg */ + if (relid == 0) + continue; + + /* + * Pairs with the kfree_rcu() in vmbus_chan_release(). + * Guarantees that the channel data structure doesn't + * get freed while the channel pointer below is being + * dereferenced. + */ + rcu_read_lock(); + + /* Find channel based on relid */ + channel = relid2channel(relid); + if (channel == NULL) + goto sched_unlock_rcu; + + if (channel->rescind) + goto sched_unlock_rcu; + + /* + * Make sure that the ring buffer data structure doesn't get + * freed while we dereference the ring buffer pointer. Test + * for the channel's onchannel_callback being NULL within a + * sched_lock critical section. See also the inline comments + * in vmbus_reset_channel_cb(). + */ + spin_lock(&channel->sched_lock); + + callback_fn = channel->onchannel_callback; + if (unlikely(callback_fn == NULL)) + goto sched_unlock; + + trace_vmbus_chan_sched(channel); + + ++channel->interrupts; + + switch (channel->callback_mode) { + case HV_CALL_ISR: + (*callback_fn)(channel->channel_callback_context); + break; + + case HV_CALL_BATCHED: + hv_begin_read(&channel->inbound); + fallthrough; + case HV_CALL_DIRECT: + tasklet_schedule(&channel->callback_event); + } + +sched_unlock: + spin_unlock(&channel->sched_lock); +sched_unlock_rcu: + rcu_read_unlock(); + } +} + +static void vmbus_message_sched(struct hv_per_cpu_context *hv_cpu, void *message_page_addr) +{ + struct hv_message *msg; + + if (!message_page_addr) + return; + msg = (struct hv_message *)message_page_addr + VMBUS_MESSAGE_SINT; + + /* Check if there are actual msgs to be processed */ + if (msg->header.message_type != HVMSG_NONE) { + if (msg->header.message_type == HVMSG_TIMER_EXPIRED) { + hv_stimer0_isr(); + vmbus_signal_eom(msg, HVMSG_TIMER_EXPIRED); + } else { + tasklet_schedule(&hv_cpu->msg_dpc); + } + } +} + +static void __vmbus_isr(void) +{ + struct hv_per_cpu_context *hv_cpu + = this_cpu_ptr(hv_context.cpu_context); + + vmbus_chan_sched(hv_cpu->hyp_synic_event_page); + vmbus_chan_sched(hv_cpu->para_synic_event_page); + + vmbus_message_sched(hv_cpu, hv_cpu->hyp_synic_message_page); + vmbus_message_sched(hv_cpu, hv_cpu->para_synic_message_page); +} + +static DEFINE_PER_CPU(bool, vmbus_irq_pending); +static DEFINE_PER_CPU(struct task_struct *, vmbus_irqd); + +static void vmbus_irqd_wake(void) +{ + struct task_struct *tsk = __this_cpu_read(vmbus_irqd); + + __this_cpu_write(vmbus_irq_pending, true); + wake_up_process(tsk); +} + +static void vmbus_irqd_setup(unsigned int cpu) +{ + sched_set_fifo(current); +} + +static int vmbus_irqd_should_run(unsigned int cpu) +{ + return __this_cpu_read(vmbus_irq_pending); +} + +static void run_vmbus_irqd(unsigned int cpu) +{ + __this_cpu_write(vmbus_irq_pending, false); + __vmbus_isr(); +} + +static bool vmbus_irq_initialized; + +static struct smp_hotplug_thread vmbus_irq_threads = { + .store = &vmbus_irqd, + .setup = vmbus_irqd_setup, + .thread_should_run = vmbus_irqd_should_run, + .thread_fn = run_vmbus_irqd, + .thread_comm = "vmbus_irq/%u", +}; + +void vmbus_isr(void) +{ + if (IS_ENABLED(CONFIG_PREEMPT_RT)) { + vmbus_irqd_wake(); + } else { + lockdep_hardirq_threaded(); + __vmbus_isr(); + } +} +EXPORT_SYMBOL_FOR_MODULES(vmbus_isr, "mshv_vtl"); + +static irqreturn_t vmbus_percpu_isr(int irq, void *dev_id) +{ + vmbus_isr(); + return IRQ_HANDLED; +} + +static void vmbus_percpu_work(struct work_struct *work) +{ + unsigned int cpu = smp_processor_id(); + + hv_synic_init(cpu); +} + +static int vmbus_alloc_synic_and_connect(void) +{ + int ret, cpu; + struct work_struct __percpu *works; + + ret = hv_synic_alloc(); + if (ret < 0) + goto err_alloc; + + works = alloc_percpu(struct work_struct); + if (!works) { + ret = -ENOMEM; + goto err_alloc; + } + + /* + * Initialize the per-cpu interrupt state and stimer state. + * Then connect to the host. + */ + cpus_read_lock(); + for_each_online_cpu(cpu) { + struct work_struct *work = per_cpu_ptr(works, cpu); + + INIT_WORK(work, vmbus_percpu_work); + schedule_work_on(cpu, work); + } + + for_each_online_cpu(cpu) + flush_work(per_cpu_ptr(works, cpu)); + + /* Register the callbacks for possible CPU online/offline'ing */ + ret = cpuhp_setup_state_nocalls_cpuslocked(CPUHP_AP_ONLINE_DYN, "hyperv/vmbus:online", + hv_synic_init, hv_synic_cleanup); + cpus_read_unlock(); + free_percpu(works); + if (ret < 0) + goto err_alloc; + hyperv_cpuhp_online = ret; + + ret = vmbus_connect(); + if (ret) + goto err_connect; + return 0; + +err_connect: + cpuhp_remove_state(hyperv_cpuhp_online); + return -ENODEV; +err_alloc: + hv_synic_free(); + return -ENOMEM; +} + +/* + * vmbus_bus_init -Main vmbus driver initialization routine. + * + * Here, we + * - initialize the vmbus driver context + * - invoke the vmbus hv main init routine + * - retrieve the channel offers + */ +static int vmbus_bus_init(void) +{ + int ret; + + ret = hv_init(); + if (ret != 0) { + pr_err("Unable to initialize the hypervisor - 0x%x\n", ret); + return ret; + } + + ret = bus_register(&hv_bus); + if (ret) + return ret; + + /* + * VMbus interrupts are best modeled as per-cpu interrupts. If + * on an architecture with support for per-cpu IRQs (e.g. ARM64), + * allocate a per-cpu IRQ using standard Linux kernel functionality. + * If not on such an architecture (e.g., x86/x64), then rely on + * code in the arch-specific portion of the code tree to connect + * the VMbus interrupt handler. + */ + + if (IS_ENABLED(CONFIG_PREEMPT_RT) && !vmbus_irq_initialized) { + ret = smpboot_register_percpu_thread(&vmbus_irq_threads); + if (ret) + goto err_kthread; + vmbus_irq_initialized = true; + } + + if (vmbus_irq == -1) { + hv_setup_vmbus_handler(vmbus_isr); + } else { + ret = request_percpu_irq(vmbus_irq, vmbus_percpu_isr, + "Hyper-V VMbus", &vmbus_evt); + if (ret) { + pr_err("Can't request Hyper-V VMbus IRQ %d, Err %d", + vmbus_irq, ret); + goto err_setup; + } + } + + /* + * Cache the value as getting it involves a VM exit on x86(_64), and + * doing that on each VP while initializing SynIC's wastes time. + */ + is_confidential = ms_hyperv.confidential_vmbus_available; + if (is_confidential) + pr_info("Establishing connection to the confidential VMBus\n"); + hv_para_set_sint_proxy(!is_confidential); + ret = vmbus_alloc_synic_and_connect(); + if (ret) + goto err_connect; + + /* + * Always register the vmbus unload panic notifier because we + * need to shut the VMbus channel connection on panic. + */ + atomic_notifier_chain_register(&panic_notifier_list, + &hyperv_panic_vmbus_unload_block); + + vmbus_request_offers(); + + return 0; + +err_connect: + if (vmbus_irq == -1) + hv_remove_vmbus_handler(); + else + free_percpu_irq(vmbus_irq, &vmbus_evt); +err_setup: + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } +err_kthread: + bus_unregister(&hv_bus); + return ret; +} + +/** + * __vmbus_driver_register() - Register a vmbus's driver + * @hv_driver: Pointer to driver structure you want to register + * @owner: owner module of the drv + * @mod_name: module name string + * + * Registers the given driver with Linux through the 'driver_register()' call + * and sets up the hyper-v vmbus handling for this driver. + * It will return the state of the 'driver_register()' call. + * + */ +int __vmbus_driver_register(struct hv_driver *hv_driver, struct module *owner, const char *mod_name) +{ + int ret; + + if (!hv_vmbus_exists()) + return -ENODEV; + + pr_info("registering driver %s\n", hv_driver->name); + + hv_driver->driver.name = hv_driver->name; + hv_driver->driver.owner = owner; + hv_driver->driver.mod_name = mod_name; + hv_driver->driver.bus = &hv_bus; + + spin_lock_init(&hv_driver->dynids.lock); + INIT_LIST_HEAD(&hv_driver->dynids.list); + + ret = driver_register(&hv_driver->driver); + + return ret; +} +EXPORT_SYMBOL_GPL(__vmbus_driver_register); + +/** + * vmbus_driver_unregister() - Unregister a vmbus's driver + * @hv_driver: Pointer to driver structure you want to + * un-register + * + * Un-register the given driver that was previous registered with a call to + * vmbus_driver_register() + */ +void vmbus_driver_unregister(struct hv_driver *hv_driver) +{ + if (hv_vmbus_exists()) { + pr_info("unregistering driver %s\n", hv_driver->name); + driver_unregister(&hv_driver->driver); + vmbus_free_dynids(hv_driver); + } +} +EXPORT_SYMBOL_GPL(vmbus_driver_unregister); + + +/* + * Called when last reference to channel is gone. + */ +static void vmbus_chan_release(struct kobject *kobj) +{ + struct vmbus_channel *channel + = container_of(kobj, struct vmbus_channel, kobj); + + kfree_rcu(channel, rcu); +} + +struct vmbus_chan_attribute { + struct attribute attr; + ssize_t (*show)(struct vmbus_channel *chan, char *buf); + ssize_t (*store)(struct vmbus_channel *chan, + const char *buf, size_t count); +}; +#define VMBUS_CHAN_ATTR(_name, _mode, _show, _store) \ + struct vmbus_chan_attribute chan_attr_##_name \ + = __ATTR(_name, _mode, _show, _store) +#define VMBUS_CHAN_ATTR_RW(_name) \ + struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RW(_name) +#define VMBUS_CHAN_ATTR_RO(_name) \ + struct vmbus_chan_attribute chan_attr_##_name = __ATTR_RO(_name) +#define VMBUS_CHAN_ATTR_WO(_name) \ + struct vmbus_chan_attribute chan_attr_##_name = __ATTR_WO(_name) + +static ssize_t vmbus_chan_attr_show(struct kobject *kobj, + struct attribute *attr, char *buf) +{ + const struct vmbus_chan_attribute *attribute + = container_of(attr, struct vmbus_chan_attribute, attr); + struct vmbus_channel *chan + = container_of(kobj, struct vmbus_channel, kobj); + + if (!attribute->show) + return -EIO; + + return attribute->show(chan, buf); +} + +static ssize_t vmbus_chan_attr_store(struct kobject *kobj, + struct attribute *attr, const char *buf, + size_t count) +{ + const struct vmbus_chan_attribute *attribute + = container_of(attr, struct vmbus_chan_attribute, attr); + struct vmbus_channel *chan + = container_of(kobj, struct vmbus_channel, kobj); + + if (!attribute->store) + return -EIO; + + return attribute->store(chan, buf, count); +} + +static const struct sysfs_ops vmbus_chan_sysfs_ops = { + .show = vmbus_chan_attr_show, + .store = vmbus_chan_attr_store, +}; + +static ssize_t out_mask_show(struct vmbus_channel *channel, char *buf) +{ + struct hv_ring_buffer_info *rbi = &channel->outbound; + ssize_t ret; + + mutex_lock(&rbi->ring_buffer_mutex); + if (!rbi->ring_buffer) { + mutex_unlock(&rbi->ring_buffer_mutex); + return -EINVAL; + } + + ret = sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask); + mutex_unlock(&rbi->ring_buffer_mutex); + return ret; +} +static VMBUS_CHAN_ATTR_RO(out_mask); + +static ssize_t in_mask_show(struct vmbus_channel *channel, char *buf) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; + ssize_t ret; + + mutex_lock(&rbi->ring_buffer_mutex); + if (!rbi->ring_buffer) { + mutex_unlock(&rbi->ring_buffer_mutex); + return -EINVAL; + } + + ret = sprintf(buf, "%u\n", rbi->ring_buffer->interrupt_mask); + mutex_unlock(&rbi->ring_buffer_mutex); + return ret; +} +static VMBUS_CHAN_ATTR_RO(in_mask); + +static ssize_t read_avail_show(struct vmbus_channel *channel, char *buf) +{ + struct hv_ring_buffer_info *rbi = &channel->inbound; + ssize_t ret; + + mutex_lock(&rbi->ring_buffer_mutex); + if (!rbi->ring_buffer) { + mutex_unlock(&rbi->ring_buffer_mutex); + return -EINVAL; + } + + ret = sprintf(buf, "%u\n", hv_get_bytes_to_read(rbi)); + mutex_unlock(&rbi->ring_buffer_mutex); + return ret; +} +static VMBUS_CHAN_ATTR_RO(read_avail); + +static ssize_t write_avail_show(struct vmbus_channel *channel, char *buf) +{ + struct hv_ring_buffer_info *rbi = &channel->outbound; + ssize_t ret; + + mutex_lock(&rbi->ring_buffer_mutex); + if (!rbi->ring_buffer) { + mutex_unlock(&rbi->ring_buffer_mutex); + return -EINVAL; + } + + ret = sprintf(buf, "%u\n", hv_get_bytes_to_write(rbi)); + mutex_unlock(&rbi->ring_buffer_mutex); + return ret; +} +static VMBUS_CHAN_ATTR_RO(write_avail); + +static ssize_t target_cpu_show(struct vmbus_channel *channel, char *buf) +{ + return sprintf(buf, "%u\n", channel->target_cpu); +} + +int vmbus_channel_set_cpu(struct vmbus_channel *channel, u32 target_cpu) +{ + u32 origin_cpu; + int ret = 0; + + lockdep_assert_cpus_held(); + lockdep_assert_held(&vmbus_connection.channel_mutex); + + if (vmbus_proto_version < VERSION_WIN10_V4_1) + return -EIO; + + /* Validate target_cpu for the cpumask_test_cpu() operation below. */ + if (target_cpu >= nr_cpumask_bits) + return -EINVAL; + + if (!cpumask_test_cpu(target_cpu, housekeeping_cpumask(HK_TYPE_MANAGED_IRQ))) + return -EINVAL; + + if (!cpu_online(target_cpu)) + return -EINVAL; + + /* + * Synchronizes vmbus_channel_set_cpu() and channel closure: + * + * { Initially: state = CHANNEL_OPENED } + * + * CPU1 CPU2 + * + * [vmbus_channel_set_cpu()] [vmbus_disconnect_ring()] + * + * LOCK channel_mutex LOCK channel_mutex + * LOAD r1 = state LOAD r2 = state + * IF (r1 == CHANNEL_OPENED) IF (r2 == CHANNEL_OPENED) + * SEND MODIFYCHANNEL STORE state = CHANNEL_OPEN + * [...] SEND CLOSECHANNEL + * UNLOCK channel_mutex UNLOCK channel_mutex + * + * Forbids: r1 == r2 == CHANNEL_OPENED (i.e., CPU1's LOCK precedes + * CPU2's LOCK) && CPU2's SEND precedes CPU1's SEND + * + * Note. The host processes the channel messages "sequentially", in + * the order in which they are received on a per-partition basis. + */ + + /* + * Hyper-V will ignore MODIFYCHANNEL messages for "non-open" channels; + * avoid sending the message and fail here for such channels. + */ + if (channel->state != CHANNEL_OPENED_STATE) { + ret = -EIO; + goto end; + } + + origin_cpu = channel->target_cpu; + if (target_cpu == origin_cpu) + goto end; + + if (vmbus_send_modifychannel(channel, + hv_cpu_number_to_vp_number(target_cpu))) { + ret = -EIO; + goto end; + } + + /* + * For version before VERSION_WIN10_V5_3, the following warning holds: + * + * Warning. At this point, there is *no* guarantee that the host will + * have successfully processed the vmbus_send_modifychannel() request. + * See the header comment of vmbus_send_modifychannel() for more info. + * + * Lags in the processing of the above vmbus_send_modifychannel() can + * result in missed interrupts if the "old" target CPU is taken offline + * before Hyper-V starts sending interrupts to the "new" target CPU. + * But apart from this offlining scenario, the code tolerates such + * lags. It will function correctly even if a channel interrupt comes + * in on a CPU that is different from the channel target_cpu value. + */ + + channel->target_cpu = target_cpu; + + /* See init_vp_index(). */ + if (hv_is_perf_channel(channel)) + hv_update_allocated_cpus(origin_cpu, target_cpu); + + /* Currently set only for storvsc channels. */ + if (channel->change_target_cpu_callback) { + (*channel->change_target_cpu_callback)(channel, + origin_cpu, target_cpu); + } + +end: + return ret; +} + +static ssize_t target_cpu_store(struct vmbus_channel *channel, + const char *buf, size_t count) +{ + u32 target_cpu; + ssize_t ret; + + if (sscanf(buf, "%u", &target_cpu) != 1) + return -EIO; + + cpus_read_lock(); + mutex_lock(&vmbus_connection.channel_mutex); + ret = vmbus_channel_set_cpu(channel, target_cpu); + mutex_unlock(&vmbus_connection.channel_mutex); + cpus_read_unlock(); + + return ret ?: count; +} +static VMBUS_CHAN_ATTR(cpu, 0644, target_cpu_show, target_cpu_store); + +static ssize_t channel_pending_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%d\n", + channel_pending(channel, + vmbus_connection.monitor_pages[1])); +} +static VMBUS_CHAN_ATTR(pending, 0444, channel_pending_show, NULL); + +static ssize_t channel_latency_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%d\n", + channel_latency(channel, + vmbus_connection.monitor_pages[1])); +} +static VMBUS_CHAN_ATTR(latency, 0444, channel_latency_show, NULL); + +static ssize_t channel_interrupts_show(struct vmbus_channel *channel, char *buf) +{ + return sprintf(buf, "%llu\n", channel->interrupts); +} +static VMBUS_CHAN_ATTR(interrupts, 0444, channel_interrupts_show, NULL); + +static ssize_t channel_events_show(struct vmbus_channel *channel, char *buf) +{ + return sprintf(buf, "%llu\n", channel->sig_events); +} +static VMBUS_CHAN_ATTR(events, 0444, channel_events_show, NULL); + +static ssize_t channel_intr_in_full_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%llu\n", + (unsigned long long)channel->intr_in_full); +} +static VMBUS_CHAN_ATTR(intr_in_full, 0444, channel_intr_in_full_show, NULL); + +static ssize_t channel_intr_out_empty_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%llu\n", + (unsigned long long)channel->intr_out_empty); +} +static VMBUS_CHAN_ATTR(intr_out_empty, 0444, channel_intr_out_empty_show, NULL); + +static ssize_t channel_out_full_first_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%llu\n", + (unsigned long long)channel->out_full_first); +} +static VMBUS_CHAN_ATTR(out_full_first, 0444, channel_out_full_first_show, NULL); + +static ssize_t channel_out_full_total_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%llu\n", + (unsigned long long)channel->out_full_total); +} +static VMBUS_CHAN_ATTR(out_full_total, 0444, channel_out_full_total_show, NULL); + +static ssize_t subchannel_monitor_id_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%u\n", channel->offermsg.monitorid); +} +static VMBUS_CHAN_ATTR(monitor_id, 0444, subchannel_monitor_id_show, NULL); + +static ssize_t subchannel_id_show(struct vmbus_channel *channel, + char *buf) +{ + return sprintf(buf, "%u\n", + channel->offermsg.offer.sub_channel_index); +} +static VMBUS_CHAN_ATTR_RO(subchannel_id); + +static int hv_mmap_ring_buffer_wrapper(struct file *filp, struct kobject *kobj, + const struct bin_attribute *attr, + struct vm_area_struct *vma) +{ + struct vmbus_channel *channel = container_of(kobj, struct vmbus_channel, kobj); + struct vm_area_desc desc; + int err; + + /* + * hv_(create|remove)_ring_sysfs implementation ensures that + * mmap_prepare_ring_buffer is not NULL. + */ + compat_set_desc_from_vma(&desc, filp, vma); + err = channel->mmap_prepare_ring_buffer(channel, &desc); + if (err) + return err; + + return __compat_vma_mmap(&desc, vma); +} + +static struct bin_attribute chan_attr_ring_buffer = { + .attr = { + .name = "ring", + .mode = 0600, + }, + .mmap = hv_mmap_ring_buffer_wrapper, +}; +static struct attribute *vmbus_chan_attrs[] = { + &chan_attr_out_mask.attr, + &chan_attr_in_mask.attr, + &chan_attr_read_avail.attr, + &chan_attr_write_avail.attr, + &chan_attr_cpu.attr, + &chan_attr_pending.attr, + &chan_attr_latency.attr, + &chan_attr_interrupts.attr, + &chan_attr_events.attr, + &chan_attr_intr_in_full.attr, + &chan_attr_intr_out_empty.attr, + &chan_attr_out_full_first.attr, + &chan_attr_out_full_total.attr, + &chan_attr_monitor_id.attr, + &chan_attr_subchannel_id.attr, + NULL +}; + +static const struct bin_attribute *vmbus_chan_bin_attrs[] = { + &chan_attr_ring_buffer, + NULL +}; + +/* + * Channel-level attribute_group callback function. Returns the permission for + * each attribute, and returns 0 if an attribute is not visible. + */ +static umode_t vmbus_chan_attr_is_visible(struct kobject *kobj, + struct attribute *attr, int idx) +{ + const struct vmbus_channel *channel = + container_of(kobj, struct vmbus_channel, kobj); + + /* Hide the monitor attributes if the monitor mechanism is not used. */ + if (!channel->offermsg.monitor_allocated && + (attr == &chan_attr_pending.attr || + attr == &chan_attr_latency.attr || + attr == &chan_attr_monitor_id.attr)) + return 0; + + return attr->mode; +} + +static umode_t vmbus_chan_bin_attr_is_visible(struct kobject *kobj, + const struct bin_attribute *attr, int idx) +{ + const struct vmbus_channel *channel = + container_of(kobj, struct vmbus_channel, kobj); + + /* Hide ring attribute if channel's ring_sysfs_visible is set to false */ + if (attr == &chan_attr_ring_buffer && !channel->ring_sysfs_visible) + return 0; + + return attr->attr.mode; +} + +static size_t vmbus_chan_bin_size(struct kobject *kobj, + const struct bin_attribute *bin_attr, int a) +{ + const struct vmbus_channel *channel = + container_of(kobj, struct vmbus_channel, kobj); + + return channel->ringbuffer_pagecount << PAGE_SHIFT; +} + +static const struct attribute_group vmbus_chan_group = { + .attrs = vmbus_chan_attrs, + .bin_attrs = vmbus_chan_bin_attrs, + .is_visible = vmbus_chan_attr_is_visible, + .is_bin_visible = vmbus_chan_bin_attr_is_visible, + .bin_size = vmbus_chan_bin_size, +}; + +static const struct kobj_type vmbus_chan_ktype = { + .sysfs_ops = &vmbus_chan_sysfs_ops, + .release = vmbus_chan_release, +}; + +/** + * hv_create_ring_sysfs() - create "ring" sysfs entry corresponding to ring buffers for a channel. + * @channel: Pointer to vmbus_channel structure + * @hv_mmap_prepare_ring_buffer: function pointer for initializing the function to be called on mmap + * channel's "ring" sysfs node, which is for the ring buffer of that channel. + * Function pointer is of below type: + * int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel, + * struct vm_area_desc *desc)) + * This has a pointer to the channel and a pointer to vm_area_desc, + * used for mmap_prepare, as arguments. + * + * Sysfs node for ring buffer of a channel is created along with other fields, however its + * visibility is disabled by default. Sysfs creation needs to be controlled when the use-case + * is running. + * For example, HV_NIC device is used either by uio_hv_generic or hv_netvsc at any given point of + * time, and "ring" sysfs is needed only when uio_hv_generic is bound to that device. To avoid + * exposing the ring buffer by default, this function is responsible to enable visibility of + * ring for userspace to use. + * Note: Race conditions can happen with userspace and it is not encouraged to create new + * use-cases for this. This was added to maintain backward compatibility, while solving + * one of the race conditions in uio_hv_generic while creating sysfs. See comments with + * vmbus_add_dynid() and vmbus_device_register(). + * + * Returns 0 on success or error code on failure. + */ +int hv_create_ring_sysfs(struct vmbus_channel *channel, + int (*hv_mmap_prepare_ring_buffer)(struct vmbus_channel *channel, + struct vm_area_desc *desc)) +{ + struct kobject *kobj = &channel->kobj; + + channel->mmap_prepare_ring_buffer = hv_mmap_prepare_ring_buffer; + channel->ring_sysfs_visible = true; + + return sysfs_update_group(kobj, &vmbus_chan_group); +} +EXPORT_SYMBOL_GPL(hv_create_ring_sysfs); + +/** + * hv_remove_ring_sysfs() - remove ring sysfs entry corresponding to ring buffers for a channel. + * @channel: Pointer to vmbus_channel structure + * + * Hide "ring" sysfs for a channel by changing its is_visible attribute and updating sysfs group. + * + * Returns 0 on success or error code on failure. + */ +int hv_remove_ring_sysfs(struct vmbus_channel *channel) +{ + struct kobject *kobj = &channel->kobj; + int ret; + + channel->ring_sysfs_visible = false; + ret = sysfs_update_group(kobj, &vmbus_chan_group); + channel->mmap_prepare_ring_buffer = NULL; + return ret; +} +EXPORT_SYMBOL_GPL(hv_remove_ring_sysfs); + +/* + * vmbus_add_channel_kobj - setup a sub-directory under device/channels + */ +int vmbus_add_channel_kobj(struct hv_device *dev, struct vmbus_channel *channel) +{ + const struct device *device = &dev->device; + struct kobject *kobj = &channel->kobj; + u32 relid = channel->offermsg.child_relid; + int ret; + + kobj->kset = dev->channels_kset; + ret = kobject_init_and_add(kobj, &vmbus_chan_ktype, NULL, + "%u", relid); + if (ret) { + kobject_put(kobj); + return ret; + } + + ret = sysfs_create_group(kobj, &vmbus_chan_group); + + if (ret) { + /* + * The calling functions' error handling paths will cleanup the + * empty channel directory. + */ + kobject_put(kobj); + dev_err(device, "Unable to set up channel sysfs files\n"); + return ret; + } + + kobject_uevent(kobj, KOBJ_ADD); + + return 0; +} + +/* + * vmbus_remove_channel_attr_group - remove the channel's attribute group + */ +void vmbus_remove_channel_attr_group(struct vmbus_channel *channel) +{ + sysfs_remove_group(&channel->kobj, &vmbus_chan_group); +} + +/* + * vmbus_device_create - Creates and registers a new child device + * on the vmbus. + */ +struct hv_device *vmbus_device_create(const guid_t *type, + const guid_t *instance, + struct vmbus_channel *channel) +{ + struct hv_device *child_device_obj; + + child_device_obj = kzalloc_obj(struct hv_device); + if (!child_device_obj) { + pr_err("Unable to allocate device object for child device\n"); + return NULL; + } + + child_device_obj->channel = channel; + guid_copy(&child_device_obj->dev_type, type); + guid_copy(&child_device_obj->dev_instance, instance); + child_device_obj->vendor_id = PCI_VENDOR_ID_MICROSOFT; + + return child_device_obj; +} + +/* + * vmbus_device_register - Register the child device + */ +int vmbus_device_register(struct hv_device *child_device_obj) +{ + struct kobject *kobj = &child_device_obj->device.kobj; + int ret; + + dev_set_name(&child_device_obj->device, "%pUl", + &child_device_obj->channel->offermsg.offer.if_instance); + + child_device_obj->device.bus = &hv_bus; + child_device_obj->device.parent = vmbus_root_device; + child_device_obj->device.release = vmbus_device_release; + + child_device_obj->device.dma_parms = &child_device_obj->dma_parms; + child_device_obj->device.dma_mask = &child_device_obj->dma_mask; + dma_set_mask(&child_device_obj->device, DMA_BIT_MASK(64)); + + /* + * Register with the LDM. This will kick off the driver/device + * binding...which will eventually call vmbus_match() and vmbus_probe() + */ + ret = device_register(&child_device_obj->device); + if (ret) { + pr_err("Unable to register child device\n"); + put_device(&child_device_obj->device); + return ret; + } + + /* + * If device_register() found a driver to assign to the device, the + * driver's probe function has already run at this point. If that + * probe function accesses or operates on the "channels" subdirectory + * in sysfs, those operations will have failed because the "channels" + * subdirectory doesn't exist until the code below runs. Or if the + * probe function creates a /dev entry, a user space program could + * find and open the /dev entry, and then create a race by accessing + * the "channels" subdirectory while the creation steps are in progress + * here. The race can't result in a kernel failure, but the user space + * program may get an error in accessing "channels" or its + * subdirectories. See also comments with vmbus_add_dynid() about a + * related race condition. + */ + child_device_obj->channels_kset = kset_create_and_add("channels", + NULL, kobj); + if (!child_device_obj->channels_kset) { + ret = -ENOMEM; + goto err_dev_unregister; + } + + ret = vmbus_add_channel_kobj(child_device_obj, + child_device_obj->channel); + if (ret) { + pr_err("Unable to register primary channel\n"); + goto err_kset_unregister; + } + hv_debug_add_dev_dir(child_device_obj); + + return 0; + +err_kset_unregister: + kset_unregister(child_device_obj->channels_kset); + +err_dev_unregister: + device_unregister(&child_device_obj->device); + return ret; +} + +/* + * vmbus_device_unregister - Remove the specified child device + * from the vmbus. + */ +void vmbus_device_unregister(struct hv_device *device_obj) +{ + pr_debug("child device %s unregistered\n", + dev_name(&device_obj->device)); + + kset_unregister(device_obj->channels_kset); + + /* + * Kick off the process of unregistering the device. + * This will call vmbus_remove() and eventually vmbus_device_release() + */ + device_unregister(&device_obj->device); +} +EXPORT_SYMBOL_GPL(vmbus_device_unregister); + +#ifdef CONFIG_ACPI +/* + * VMBUS is an acpi enumerated device. Get the information we + * need from DSDT. + */ +static acpi_status vmbus_walk_resources(struct acpi_resource *res, void *ctx) +{ + resource_size_t start = 0; + resource_size_t end = 0; + struct resource *new_res; + struct resource **old_res = &hyperv_mmio; + struct resource **prev_res = NULL; + struct resource r; + + switch (res->type) { + + /* + * "Address" descriptors are for bus windows. Ignore + * "memory" descriptors, which are for registers on + * devices. + */ + case ACPI_RESOURCE_TYPE_ADDRESS32: + start = res->data.address32.address.minimum; + end = res->data.address32.address.maximum; + break; + + case ACPI_RESOURCE_TYPE_ADDRESS64: + start = res->data.address64.address.minimum; + end = res->data.address64.address.maximum; + break; + + /* + * The IRQ information is needed only on ARM64, which Hyper-V + * sets up in the extended format. IRQ information is present + * on x86/x64 in the non-extended format but it is not used by + * Linux. So don't bother checking for the non-extended format. + */ + case ACPI_RESOURCE_TYPE_EXTENDED_IRQ: + if (!acpi_dev_resource_interrupt(res, 0, &r)) { + pr_err("Unable to parse Hyper-V ACPI interrupt\n"); + return AE_ERROR; + } + /* ARM64 INTID for VMbus */ + vmbus_interrupt = res->data.extended_irq.interrupts[0]; + /* Linux IRQ number */ + vmbus_irq = r.start; + return AE_OK; + + default: + /* Unused resource type */ + return AE_OK; + + } + /* + * Ignore ranges that are below 1MB, as they're not + * necessary or useful here. + */ + if (end < 0x100000) + return AE_OK; + + new_res = kzalloc_obj(*new_res, GFP_ATOMIC); + if (!new_res) + return AE_NO_MEMORY; + + /* If this range overlaps the virtual TPM, truncate it. */ + if (end >= VTPM_BASE_ADDRESS && start < VTPM_BASE_ADDRESS) + end = VTPM_BASE_ADDRESS - 1; + + new_res->name = "hyperv mmio"; + new_res->flags = IORESOURCE_MEM; + new_res->start = start; + new_res->end = end; + + /* + * If two ranges are adjacent, merge them. + */ + do { + if (!*old_res) { + *old_res = new_res; + break; + } + + if (((*old_res)->end + 1) == new_res->start) { + (*old_res)->end = new_res->end; + kfree(new_res); + break; + } + + if ((*old_res)->start == new_res->end + 1) { + (*old_res)->start = new_res->start; + kfree(new_res); + break; + } + + if ((*old_res)->start > new_res->end) { + new_res->sibling = *old_res; + if (prev_res) + (*prev_res)->sibling = new_res; + *old_res = new_res; + break; + } + + prev_res = old_res; + old_res = &(*old_res)->sibling; + + } while (1); + + return AE_OK; +} +#endif + +static void vmbus_mmio_remove(void) +{ + struct resource *cur_res; + struct resource *next_res; + + if (hyperv_mmio) { + if (fb_mmio) { + __release_region(hyperv_mmio, fb_mmio->start, + resource_size(fb_mmio)); + fb_mmio = NULL; + } + + for (cur_res = hyperv_mmio; cur_res; cur_res = next_res) { + next_res = cur_res->sibling; + kfree(cur_res); + } + } +} + +static void __maybe_unused vmbus_reserve_fb(void) +{ + resource_size_t start = 0, size; + resource_size_t low_mmio_base; + struct pci_dev *pdev; + + if (efi_enabled(EFI_BOOT)) { + /* Gen2 VM: get FB base from EFI framebuffer */ + if (IS_ENABLED(CONFIG_SYSFB)) { + start = sysfb_primary_display.screen.lfb_base; + size = max_t(__u32, sysfb_primary_display.screen.lfb_size, 0x800000); + + low_mmio_base = hyperv_mmio->start; + if (!low_mmio_base || upper_32_bits(low_mmio_base) || + (start && start < low_mmio_base)) { + pr_warn("Unexpected low mmio base %pa\n", &low_mmio_base); + } else { + /* + * If the kdump/kexec or CVM kernel's lfb_base + * is 0, fall back to the low mmio base. + */ + if (!start) + start = low_mmio_base; + /* + * Reserve half of the space below 4GB for high + * resolutions, but cap the reservation to 128MB. + */ + size = min((SZ_4G - start) / 2, SZ_128M); + } + } + } else { + /* Gen1 VM: get FB base from PCI */ + pdev = pci_get_device(PCI_VENDOR_ID_MICROSOFT, + PCI_DEVICE_ID_HYPERV_VIDEO, NULL); + if (!pdev) + return; + + if (pdev->resource[0].flags & IORESOURCE_MEM) { + start = pci_resource_start(pdev, 0); + size = pci_resource_len(pdev, 0); + } + + /* + * Release the PCI device so hyperv_drm driver can grab it + * later. + */ + pci_dev_put(pdev); + } + + if (!start) { + pr_warn("Unexpected framebuffer mmio base of zero\n"); + return; + } + + /* + * Make a claim for the frame buffer in the resource tree under the + * first node, which will be the one below 4GB. The length seems to + * be underreported, particularly in a Generation 1 VM. So start out + * reserving a larger area and make it smaller until it succeeds. + */ + for (; !fb_mmio && (size >= 0x100000); size >>= 1) + fb_mmio = __request_region(hyperv_mmio, start, size, fb_mmio_name, 0); + + pr_info("hv_mmio=%pR,%pR fb=%pR\n", hyperv_mmio, hyperv_mmio->sibling, fb_mmio); +} + +/** + * vmbus_allocate_mmio() - Pick a memory-mapped I/O range. + * @new: If successful, supplied a pointer to the + * allocated MMIO space. + * @device_obj: Identifies the caller + * @min: Minimum guest physical address of the + * allocation + * @max: Maximum guest physical address + * @size: Size of the range to be allocated + * @align: Alignment of the range to be allocated + * @fb_overlap_ok: Whether this allocation can be allowed + * to overlap the video frame buffer. + * + * This function walks the resources granted to VMBus by the + * _CRS object in the ACPI namespace underneath the parent + * "bridge" whether that's a root PCI bus in the Generation 1 + * case or a Module Device in the Generation 2 case. It then + * attempts to allocate from the global MMIO pool in a way that + * matches the constraints supplied in these parameters and by + * that _CRS. + * + * Return: 0 on success, -errno on failure + */ +int vmbus_allocate_mmio(struct resource **new, struct hv_device *device_obj, + resource_size_t min, resource_size_t max, + resource_size_t size, resource_size_t align, + bool fb_overlap_ok) +{ + struct resource *iter, *shadow; + resource_size_t range_min, range_max, start, end; + const char *dev_n = dev_name(&device_obj->device); + int retval; + + retval = -ENXIO; + mutex_lock(&hyperv_mmio_lock); + + /* + * If overlaps with frame buffers are allowed, then first attempt to + * make the allocation from within the reserved region. Because it + * is already reserved, no shadow allocation is necessary. + */ + if (fb_overlap_ok && fb_mmio && !(min > fb_mmio->end) && + !(max < fb_mmio->start)) { + + range_min = fb_mmio->start; + range_max = fb_mmio->end; + start = (range_min + align - 1) & ~(align - 1); + for (; start + size - 1 <= range_max; start += align) { + *new = request_mem_region_exclusive(start, size, dev_n); + if (*new) { + retval = 0; + goto exit; + } + } + } + + for (iter = hyperv_mmio; iter; iter = iter->sibling) { + if ((iter->start >= max) || (iter->end <= min)) + continue; + + range_min = iter->start; + range_max = iter->end; + start = (range_min + align - 1) & ~(align - 1); + for (; start + size - 1 <= range_max; start += align) { + end = start + size - 1; + + /* Skip the whole fb_mmio region if not fb_overlap_ok */ + if (!fb_overlap_ok && fb_mmio && + (((start >= fb_mmio->start) && (start <= fb_mmio->end)) || + ((end >= fb_mmio->start) && (end <= fb_mmio->end)))) + continue; + + shadow = __request_region(iter, start, size, NULL, + IORESOURCE_BUSY); + if (!shadow) + continue; + + *new = request_mem_region_exclusive(start, size, dev_n); + if (*new) { + shadow->name = (char *)*new; + retval = 0; + goto exit; + } + + __release_region(iter, start, size); + } + } + +exit: + mutex_unlock(&hyperv_mmio_lock); + return retval; +} +EXPORT_SYMBOL_GPL(vmbus_allocate_mmio); + +/** + * vmbus_free_mmio() - Free a memory-mapped I/O range. + * @start: Base address of region to release. + * @size: Size of the range to be allocated + * + * This function releases anything requested by + * vmbus_mmio_allocate(). + */ +void vmbus_free_mmio(resource_size_t start, resource_size_t size) +{ + struct resource *iter; + + mutex_lock(&hyperv_mmio_lock); + + /* + * If all bytes of the MMIO range to be released are within the + * special case fb_mmio shadow region, skip releasing the shadow + * region since no corresponding __request_region() was done + * in vmbus_allocate_mmio(). + */ + if (fb_mmio && start >= fb_mmio->start && + (start + size - 1 <= fb_mmio->end)) + goto skip_shadow_release; + + for (iter = hyperv_mmio; iter; iter = iter->sibling) { + if ((iter->start >= start + size) || (iter->end <= start)) + continue; + + __release_region(iter, start, size); + } + +skip_shadow_release: + release_mem_region(start, size); + mutex_unlock(&hyperv_mmio_lock); + +} +EXPORT_SYMBOL_GPL(vmbus_free_mmio); + +#ifdef CONFIG_ACPI +static int vmbus_acpi_add(struct platform_device *pdev) +{ + acpi_status result; + int ret_val = -ENODEV; + struct acpi_device *ancestor; + struct acpi_device *device = ACPI_COMPANION(&pdev->dev); + + vmbus_root_device = &device->dev; + + /* + * Older versions of Hyper-V for ARM64 fail to include the _CCA + * method on the top level VMbus device in the DSDT. But devices + * are hardware coherent in all current Hyper-V use cases, so fix + * up the ACPI device to behave as if _CCA is present and indicates + * hardware coherence. + */ + ACPI_COMPANION_SET(&device->dev, device); + if (IS_ENABLED(CONFIG_ACPI_CCA_REQUIRED) && + device_get_dma_attr(&device->dev) == DEV_DMA_NOT_SUPPORTED) { + pr_info("No ACPI _CCA found; assuming coherent device I/O\n"); + device->flags.cca_seen = true; + device->flags.coherent_dma = true; + } + + result = acpi_walk_resources(device->handle, METHOD_NAME__CRS, + vmbus_walk_resources, NULL); + + if (ACPI_FAILURE(result)) + goto acpi_walk_err; + /* + * Some ancestor of the vmbus acpi device (Gen1 or Gen2 + * firmware) is the VMOD that has the mmio ranges. Get that. + */ + for (ancestor = acpi_dev_parent(device); + ancestor && ancestor->handle != ACPI_ROOT_OBJECT; + ancestor = acpi_dev_parent(ancestor)) { + result = acpi_walk_resources(ancestor->handle, METHOD_NAME__CRS, + vmbus_walk_resources, NULL); + + if (ACPI_FAILURE(result)) + continue; + if (hyperv_mmio) { + vmbus_reserve_fb(); + break; + } + } + ret_val = 0; + +acpi_walk_err: + if (ret_val) + vmbus_mmio_remove(); + return ret_val; +} +#else +static int vmbus_acpi_add(struct platform_device *pdev) +{ + return 0; +} +#endif +#ifndef HYPERVISOR_CALLBACK_VECTOR +static int vmbus_set_irq(struct platform_device *pdev) +{ + struct irq_data *data; + int irq; + irq_hw_number_t hwirq; + + irq = platform_get_irq(pdev, 0); + /* platform_get_irq() may not return 0. */ + if (irq < 0) + return irq; + + data = irq_get_irq_data(irq); + if (!data) { + pr_err("No interrupt data for VMBus virq %d\n", irq); + return -ENODEV; + } + hwirq = irqd_to_hwirq(data); + + vmbus_irq = irq; + vmbus_interrupt = hwirq; + pr_debug("VMBus virq %d, hwirq %d\n", vmbus_irq, vmbus_interrupt); + + return 0; +} +#endif + +static int vmbus_device_add(struct platform_device *pdev) +{ + struct resource **cur_res = &hyperv_mmio; + struct of_range range; + struct of_range_parser parser; + struct device_node *np = pdev->dev.of_node; + int ret; + + vmbus_root_device = &pdev->dev; + + ret = of_range_parser_init(&parser, np); + if (ret) + return ret; + +#ifndef HYPERVISOR_CALLBACK_VECTOR + ret = vmbus_set_irq(pdev); + if (ret) + return ret; +#endif + for_each_of_range(&parser, &range) { + struct resource *res; + + res = kzalloc_obj(*res); + if (!res) { + vmbus_mmio_remove(); + return -ENOMEM; + } + + res->name = "hyperv mmio"; + res->flags = range.flags; + res->start = range.cpu_addr; + res->end = range.cpu_addr + range.size; + + *cur_res = res; + cur_res = &res->sibling; + } + + return ret; +} + +static int vmbus_platform_driver_probe(struct platform_device *pdev) +{ + if (acpi_disabled) + return vmbus_device_add(pdev); + else + return vmbus_acpi_add(pdev); +} + +static void vmbus_platform_driver_remove(struct platform_device *pdev) +{ + vmbus_mmio_remove(); +} + +#ifdef CONFIG_PM_SLEEP +static int vmbus_bus_suspend(struct device *dev) +{ + struct hv_per_cpu_context *hv_cpu = per_cpu_ptr( + hv_context.cpu_context, VMBUS_CONNECT_CPU); + struct vmbus_channel *channel, *sc; + + tasklet_disable(&hv_cpu->msg_dpc); + vmbus_connection.ignore_any_offer_msg = true; + /* The tasklet_enable() takes care of providing a memory barrier */ + tasklet_enable(&hv_cpu->msg_dpc); + + /* Drain all the workqueues as we are in suspend */ + drain_workqueue(vmbus_connection.rescind_work_queue); + drain_workqueue(vmbus_connection.work_queue); + drain_workqueue(vmbus_connection.handle_primary_chan_wq); + drain_workqueue(vmbus_connection.handle_sub_chan_wq); + + mutex_lock(&vmbus_connection.channel_mutex); + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (!is_hvsock_channel(channel)) + continue; + + vmbus_force_channel_rescinded(channel); + } + mutex_unlock(&vmbus_connection.channel_mutex); + + /* + * Wait until all the sub-channels and hv_sock channels have been + * cleaned up. Sub-channels should be destroyed upon suspend, otherwise + * they would conflict with the new sub-channels that will be created + * in the resume path. hv_sock channels should also be destroyed, but + * a hv_sock channel of an established hv_sock connection can not be + * really destroyed since it may still be referenced by the userspace + * application, so we just force the hv_sock channel to be rescinded + * by vmbus_force_channel_rescinded(), and the userspace application + * will thoroughly destroy the channel after hibernation. + * + * Note: the counter nr_chan_close_on_suspend may never go above 0 if + * the VM has no sub-channel and hv_sock channel, e.g. a 1-vCPU VM. + */ + if (atomic_read(&vmbus_connection.nr_chan_close_on_suspend) > 0) + wait_for_completion(&vmbus_connection.ready_for_suspend_event); + + mutex_lock(&vmbus_connection.channel_mutex); + + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + /* + * Remove the channel from the array of channels and invalidate + * the channel's relid. Upon resume, vmbus_onoffer() will fix + * up the relid (and other fields, if necessary) and add the + * channel back to the array. + */ + vmbus_channel_unmap_relid(channel); + channel->offermsg.child_relid = INVALID_RELID; + + if (is_hvsock_channel(channel)) { + if (!channel->rescind) { + pr_err("hv_sock channel not rescinded!\n"); + WARN_ON_ONCE(1); + } + continue; + } + + list_for_each_entry(sc, &channel->sc_list, sc_list) { + pr_err("Sub-channel not deleted!\n"); + WARN_ON_ONCE(1); + } + } + + mutex_unlock(&vmbus_connection.channel_mutex); + + vmbus_initiate_unload(false); + + return 0; +} + +static int vmbus_bus_resume(struct device *dev) +{ + struct vmbus_channel *channel; + struct vmbus_channel_msginfo *msginfo; + size_t msgsize; + int ret; + + vmbus_connection.ignore_any_offer_msg = false; + + /* + * We only use the 'vmbus_proto_version', which was in use before + * hibernation, to re-negotiate with the host. + */ + if (!vmbus_proto_version) { + pr_err("Invalid proto version = 0x%x\n", vmbus_proto_version); + return -EINVAL; + } + + msgsize = sizeof(*msginfo) + + sizeof(struct vmbus_channel_initiate_contact); + + msginfo = kzalloc(msgsize, GFP_KERNEL); + + if (msginfo == NULL) + return -ENOMEM; + + ret = vmbus_negotiate_version(msginfo, vmbus_proto_version); + + kfree(msginfo); + + if (ret != 0) + return ret; + + vmbus_request_offers(); + + mutex_lock(&vmbus_connection.channel_mutex); + list_for_each_entry(channel, &vmbus_connection.chn_list, listentry) { + if (channel->offermsg.child_relid != INVALID_RELID) + continue; + + /* hvsock channels are not expected to be present. */ + if (is_hvsock_channel(channel)) + continue; + + pr_err("channel %pUl/%pUl not present after resume.\n", + &channel->offermsg.offer.if_type, + &channel->offermsg.offer.if_instance); + /* ToDo: Cleanup these channels here */ + } + mutex_unlock(&vmbus_connection.channel_mutex); + + /* Reset the event for the next suspend. */ + reinit_completion(&vmbus_connection.ready_for_suspend_event); + + return 0; +} +#else +#define vmbus_bus_suspend NULL +#define vmbus_bus_resume NULL +#endif /* CONFIG_PM_SLEEP */ + +static const __maybe_unused struct of_device_id vmbus_of_match[] = { + { + .compatible = "microsoft,vmbus", + }, + { + /* sentinel */ + }, +}; +MODULE_DEVICE_TABLE(of, vmbus_of_match); + +static const __maybe_unused struct acpi_device_id vmbus_acpi_device_ids[] = { + {"VMBUS", 0}, + {"VMBus", 0}, + {"", 0}, +}; +MODULE_DEVICE_TABLE(acpi, vmbus_acpi_device_ids); + +/* + * Note: we must use the "no_irq" ops, otherwise hibernation can not work with + * PCI device assignment, because "pci_dev_pm_ops" uses the "noirq" ops: in + * the resume path, the pci "noirq" restore op runs before "non-noirq" op (see + * resume_target_kernel() -> dpm_resume_start(), and hibernation_restore() -> + * dpm_resume_end()). This means vmbus_bus_resume() and the pci-hyperv's + * resume callback must also run via the "noirq" ops. + * + * Set suspend_noirq/resume_noirq to NULL for Suspend-to-Idle: see the comment + * earlier in this file before vmbus_pm. + */ + +static const struct dev_pm_ops vmbus_bus_pm = { + .suspend_noirq = NULL, + .resume_noirq = NULL, + .freeze_noirq = vmbus_bus_suspend, + .thaw_noirq = vmbus_bus_resume, + .poweroff_noirq = vmbus_bus_suspend, + .restore_noirq = vmbus_bus_resume +}; + +static struct platform_driver vmbus_platform_driver = { + .probe = vmbus_platform_driver_probe, + .remove = vmbus_platform_driver_remove, + .driver = { + .name = "vmbus", + .acpi_match_table = ACPI_PTR(vmbus_acpi_device_ids), + .of_match_table = of_match_ptr(vmbus_of_match), + .pm = &vmbus_bus_pm, + .probe_type = PROBE_FORCE_SYNCHRONOUS, + } +}; + +static void hv_kexec_handler(void) +{ + vmbus_initiate_unload(false); + /* Make sure conn_state is set as hv_synic_cleanup checks for it */ + mb(); + cpuhp_remove_state(hyperv_cpuhp_online); +}; + +static void hv_crash_handler(struct pt_regs *regs) +{ + int cpu; + + if (!skip_vmbus_unload) + vmbus_initiate_unload(true); + /* + * In crash handler we can't schedule synic cleanup for all CPUs, + * doing the cleanup for current CPU only. This should be sufficient + * for kdump. + */ + cpu = smp_processor_id(); + hv_stimer_cleanup(cpu); + hv_hyp_synic_disable_regs(cpu); +}; + +static int hv_synic_suspend(void *data) +{ + /* + * When we reach here, all the non-boot CPUs have been offlined. + * If we're in a legacy configuration where stimer Direct Mode is + * not enabled, the stimers on the non-boot CPUs have been unbound + * in hv_synic_cleanup() -> hv_stimer_legacy_cleanup() -> + * hv_stimer_cleanup() -> clockevents_unbind_device(). + * + * hv_synic_suspend() only runs on CPU0 with interrupts disabled. + * Here we do not call hv_stimer_legacy_cleanup() on CPU0 because: + * 1) it's unnecessary as interrupts remain disabled between + * syscore_suspend() and syscore_resume(): see create_image() and + * resume_target_kernel() + * 2) the stimer on CPU0 is automatically disabled later by + * syscore_suspend() -> timekeeping_suspend() -> tick_suspend() -> ... + * -> clockevents_shutdown() -> ... -> hv_ce_shutdown() + * 3) a warning would be triggered if we call + * clockevents_unbind_device(), which may sleep, in an + * interrupts-disabled context. + */ + + hv_hyp_synic_disable_regs(0); + + return 0; +} + +static void hv_synic_resume(void *data) +{ + hv_hyp_synic_enable_regs(0); + + /* + * Note: we don't need to call hv_stimer_init(0), because the timer + * on CPU0 is not unbound in hv_synic_suspend(), and the timer is + * automatically re-enabled in timekeeping_resume(). + */ +} + +/* The callbacks run only on CPU0, with irqs_disabled. */ +static const struct syscore_ops hv_synic_syscore_ops = { + .suspend = hv_synic_suspend, + .resume = hv_synic_resume, +}; + +static struct syscore hv_synic_syscore = { + .ops = &hv_synic_syscore_ops, +}; + +static int __init hv_acpi_init(void) +{ + int ret; + + if (!hv_is_hyperv_initialized()) + return -ENODEV; + + if (hv_root_partition() && !hv_nested) + return 0; + + /* + * Get ACPI resources first. + */ + ret = platform_driver_register(&vmbus_platform_driver); + if (ret) + return ret; + + if (!vmbus_root_device) { + ret = -ENODEV; + goto cleanup; + } + + /* + * If we're on an architecture with a hardcoded hypervisor + * vector (i.e. x86/x64), override the VMbus interrupt found + * in the ACPI tables. Ensure vmbus_irq is not set since the + * normal Linux IRQ mechanism is not used in this case. + */ +#ifdef HYPERVISOR_CALLBACK_VECTOR + vmbus_interrupt = HYPERVISOR_CALLBACK_VECTOR; + vmbus_irq = -1; +#endif + + hv_debug_init(); + + ret = vmbus_bus_init(); + if (ret) + goto cleanup; + + hv_setup_kexec_handler(hv_kexec_handler); + hv_setup_crash_handler(hv_crash_handler); + + register_syscore(&hv_synic_syscore); + + return 0; + +cleanup: + platform_driver_unregister(&vmbus_platform_driver); + vmbus_root_device = NULL; + return ret; +} + +static void __exit vmbus_exit(void) +{ + int cpu; + + unregister_syscore(&hv_synic_syscore); + + hv_remove_kexec_handler(); + hv_remove_crash_handler(); + vmbus_connection.conn_state = DISCONNECTED; + hv_stimer_global_cleanup(); + vmbus_disconnect(); + if (vmbus_irq == -1) + hv_remove_vmbus_handler(); + else + free_percpu_irq(vmbus_irq, &vmbus_evt); + if (IS_ENABLED(CONFIG_PREEMPT_RT) && vmbus_irq_initialized) { + smpboot_unregister_percpu_thread(&vmbus_irq_threads); + vmbus_irq_initialized = false; + } + for_each_online_cpu(cpu) { + struct hv_per_cpu_context *hv_cpu + = per_cpu_ptr(hv_context.cpu_context, cpu); + + tasklet_kill(&hv_cpu->msg_dpc); + } + hv_debug_rm_all_dir(); + + vmbus_free_channels(); + kfree(vmbus_connection.channels); + + /* + * The vmbus panic notifier is always registered, hence we should + * also unconditionally unregister it here as well. + */ + atomic_notifier_chain_unregister(&panic_notifier_list, + &hyperv_panic_vmbus_unload_block); + + bus_unregister(&hv_bus); + + cpuhp_remove_state(hyperv_cpuhp_online); + hv_synic_free(); + platform_driver_unregister(&vmbus_platform_driver); +} + + +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("Microsoft Hyper-V VMBus Driver"); + +subsys_initcall(hv_acpi_init); +module_exit(vmbus_exit); |