From 4352b8aee98005853aa63f57d6377282de17a33f Mon Sep 17 00:00:00 2001 From: Linus Torvalds Date: Sun, 23 Aug 2026 13:28:57 -0700 Subject: Merge tag 'i3c/for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux Pull I3C updates from Alexandre Belloni: "A new driver this cycle, for the AMD AXI I3C controller. There are also new features such as IBI wakeup support, SETAASA device discovery and ACPI support for the the DAA methods, meaning we can now communicate with the SPD devices on DDR5 modules. New driver: - AMD AXI I3C master controller Subsystem: - I3C Common Command Code (CCC) handling improvements, especially around GET CCCs - SETAASA device discovery support - ACPI support for all existing DAA methods like SETDASA, SETNEWDA as well as I2C devices on I3C bus - IBI-based wakeup support Drivers: - dw: SETAASA support - mipi-i3c-hci: advertise IBI wakeup capability, AMD_PT I3C controller support, PIO queue management support for HCI v1.2 - renesas: improve suspend to RAM support, add runtime PM support" * tag 'i3c/for-7.3' of git://git.kernel.org/pub/scm/linux/kernel/git/i3c/linux: (70 commits) i3c: dw: reduce do_daa time if there's no client i3c: mipi-i3c-hci: Add support for AMD_PT I3C controller i3c: mipi-i3c-hci: Add PIO queue management support for HCI v1.2 i3c: mipi-i3c-hci: Fix missing STAT_IBI_STATUS_THLD in PIO mode i3c: dw: rename "pclk" to "apb" to match dt-binding i3c: dw: make struct dw_i3c_cmd smaller i3c: dw: use COMMAND_PORT_TRANSFER_ARG instead of hardcoding i3c: renesas: Don't register devices when ENTDAA times out i3c: master: dw-i3c-master: fix OD timing for first broadcast i3c: mipi-i3c-hci: Advertise IBI wakeup capability i3c: mipi-i3c-hci: Factor out i3c_hci_sysdev() i3c: mipi-i3c-hci-pci: Propagate I3C wakeup requirements to PCI i3c: master: Reject IBI requests from non-IBI-capable devices i3c: master: Add helper to query bus wakeup requirements i3c: master: Report wakeup events for IBIs i3c: master: Support IBI-based wakeup capability i3c: master: Fix potential UAF in i3c_device_match() i3c: master: Fix potential UAF in i3c_device_uevent() i3c: Make dev->desc locking assumptions explicit i3c: master: Fix use-after-free of master->this ... --- drivers/net/ethernet/microsoft/Kconfig | 34 + drivers/net/ethernet/microsoft/Makefile | 5 + drivers/net/ethernet/microsoft/mana/Makefile | 6 + drivers/net/ethernet/microsoft/mana/gdma_main.c | 2767 +++++++++++++ drivers/net/ethernet/microsoft/mana/hw_channel.c | 937 +++++ drivers/net/ethernet/microsoft/mana/mana_bpf.c | 267 ++ drivers/net/ethernet/microsoft/mana/mana_en.c | 4332 ++++++++++++++++++++ drivers/net/ethernet/microsoft/mana/mana_ethtool.c | 893 ++++ drivers/net/ethernet/microsoft/mana/shm_channel.c | 287 ++ 9 files changed, 9528 insertions(+) create mode 100644 drivers/net/ethernet/microsoft/Kconfig create mode 100644 drivers/net/ethernet/microsoft/Makefile create mode 100644 drivers/net/ethernet/microsoft/mana/Makefile create mode 100644 drivers/net/ethernet/microsoft/mana/gdma_main.c create mode 100644 drivers/net/ethernet/microsoft/mana/hw_channel.c create mode 100644 drivers/net/ethernet/microsoft/mana/mana_bpf.c create mode 100644 drivers/net/ethernet/microsoft/mana/mana_en.c create mode 100644 drivers/net/ethernet/microsoft/mana/mana_ethtool.c create mode 100644 drivers/net/ethernet/microsoft/mana/shm_channel.c (limited to 'drivers/net/ethernet/microsoft') diff --git a/drivers/net/ethernet/microsoft/Kconfig b/drivers/net/ethernet/microsoft/Kconfig new file mode 100644 index 000000000..e9be18c92 --- /dev/null +++ b/drivers/net/ethernet/microsoft/Kconfig @@ -0,0 +1,34 @@ +# +# Microsoft Azure network device configuration +# + +config NET_VENDOR_MICROSOFT + bool "Microsoft Network Devices" + default y + help + If you have a network (Ethernet) device belonging to this class, say Y. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip the + question about Microsoft network devices. If you say Y, you will be + asked for your specific device in the following question. + +if NET_VENDOR_MICROSOFT + +config MICROSOFT_MANA + tristate "Microsoft Azure Network Adapter (MANA) support" + depends on PCI_MSI + depends on X86_64 || (ARM64 && !CPU_BIG_ENDIAN) + depends on PCI_HYPERV + select AUXILIARY_BUS + select DIMLIB + select PAGE_POOL + select NET_SHAPER + help + This driver supports Microsoft Azure Network Adapter (MANA). + So far, the driver is only supported on X86_64. + + To compile this driver as a module, choose M here. + The module will be called mana. + +endif #NET_VENDOR_MICROSOFT diff --git a/drivers/net/ethernet/microsoft/Makefile b/drivers/net/ethernet/microsoft/Makefile new file mode 100644 index 000000000..d2ddc2181 --- /dev/null +++ b/drivers/net/ethernet/microsoft/Makefile @@ -0,0 +1,5 @@ +# +# Makefile for the Microsoft Azure network device driver. +# + +obj-$(CONFIG_MICROSOFT_MANA) += mana/ diff --git a/drivers/net/ethernet/microsoft/mana/Makefile b/drivers/net/ethernet/microsoft/mana/Makefile new file mode 100644 index 000000000..e16a4221f --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +# +# Makefile for the Microsoft Azure Network Adapter driver + +obj-$(CONFIG_MICROSOFT_MANA) += mana.o +mana-objs := gdma_main.o shm_channel.o hw_channel.o mana_en.o mana_ethtool.o mana_bpf.o diff --git a/drivers/net/ethernet/microsoft/mana/gdma_main.c b/drivers/net/ethernet/microsoft/mana/gdma_main.c new file mode 100644 index 000000000..ed9af314e --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/gdma_main.c @@ -0,0 +1,2767 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +struct dentry *mana_debugfs_root; + +struct mana_dev_recovery { + struct list_head list; + struct pci_dev *pdev; + enum gdma_eqe_type type; +}; + +static struct mana_dev_recovery_work { + struct list_head dev_list; + struct delayed_work work; + + /* Lock for dev_list above */ + spinlock_t lock; +} mana_dev_recovery_work; + +static u32 mana_gd_r32(struct gdma_context *g, u64 offset) +{ + return readl(g->bar0_va + offset); +} + +static u64 mana_gd_r64(struct gdma_context *g, u64 offset) +{ + return readq(g->bar0_va + offset); +} + +static int mana_gd_init_pf_regs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + u64 remaining_barsize; + u64 sriov_base_off; + u64 sriov_shm_off; + + gc->db_page_size = mana_gd_r32(gc, GDMA_PF_REG_DB_PAGE_SIZE) & 0xFFFF; + + /* mana_gd_ring_doorbell() accesses offsets up to DOORBELL_OFFSET_EQ + * (0xFF8) + 8 bytes = 4KB within each doorbell page, so the page + * size must be at least SZ_4K. + */ + if (gc->db_page_size < SZ_4K) { + dev_err(gc->dev, + "Doorbell page size %llu too small (min %u)\n", + gc->db_page_size, SZ_4K); + return -EPROTO; + } + + gc->db_page_off = mana_gd_r64(gc, GDMA_PF_REG_DB_PAGE_OFF); + + /* Validate doorbell offset is within BAR0 */ + if (gc->db_page_off >= gc->bar0_size) { + dev_err(gc->dev, + "Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n", + gc->db_page_off, (u64)gc->bar0_size); + return -EPROTO; + } + + gc->db_page_base = gc->bar0_va + gc->db_page_off; + gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off; + + sriov_base_off = mana_gd_r64(gc, GDMA_SRIOV_REG_CFG_BASE_OFF); + if (sriov_base_off >= gc->bar0_size || + gc->bar0_size - sriov_base_off < + GDMA_PF_REG_SHM_OFF + sizeof(u64) || + !IS_ALIGNED(sriov_base_off, sizeof(u64))) { + dev_err(gc->dev, + "SRIOV base offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n", + sriov_base_off, (u64)gc->bar0_size); + return -EPROTO; + } + + remaining_barsize = gc->bar0_size - sriov_base_off; + sriov_shm_off = mana_gd_r64(gc, sriov_base_off + GDMA_PF_REG_SHM_OFF); + if (sriov_shm_off >= remaining_barsize || + remaining_barsize - sriov_shm_off < SMC_APERTURE_SIZE || + !IS_ALIGNED(sriov_shm_off, sizeof(u32))) { + dev_err(gc->dev, + "SRIOV SHM offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n", + sriov_shm_off, (u64)gc->bar0_size); + return -EPROTO; + } + + gc->shm_base = gc->bar0_va + sriov_base_off + sriov_shm_off; + + return 0; +} + +static int mana_gd_init_vf_regs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + u64 shm_off; + + gc->db_page_size = mana_gd_r32(gc, GDMA_REG_DB_PAGE_SIZE) & 0xFFFF; + + /* mana_gd_ring_doorbell() accesses offsets up to DOORBELL_OFFSET_EQ + * (0xFF8) + 8 bytes = 4KB within each doorbell page, so the page + * size must be at least SZ_4K. + */ + if (gc->db_page_size < SZ_4K) { + dev_err(gc->dev, + "Doorbell page size %llu too small (min %u)\n", + gc->db_page_size, SZ_4K); + return -EPROTO; + } + + gc->db_page_off = mana_gd_r64(gc, GDMA_REG_DB_PAGE_OFFSET); + + /* Validate doorbell offset is within BAR0 */ + if (gc->db_page_off >= gc->bar0_size) { + dev_err(gc->dev, + "Doorbell offset 0x%llx exceeds BAR0 size 0x%llx\n", + gc->db_page_off, (u64)gc->bar0_size); + return -EPROTO; + } + + gc->db_page_base = gc->bar0_va + gc->db_page_off; + gc->phys_db_page_base = gc->bar0_pa + gc->db_page_off; + + shm_off = mana_gd_r64(gc, GDMA_REG_SHM_OFFSET); + if (shm_off >= gc->bar0_size || + gc->bar0_size - shm_off < SMC_APERTURE_SIZE || + !IS_ALIGNED(shm_off, sizeof(u32))) { + dev_err(gc->dev, + "SHM offset 0x%llx out of range or unaligned (BAR0 size 0x%llx)\n", + shm_off, (u64)gc->bar0_size); + return -EPROTO; + } + + gc->shm_base = gc->bar0_va + shm_off; + + return 0; +} + +static int mana_gd_init_registers(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + if (gc->is_pf && !gc->is_pf2) + return mana_gd_init_pf_regs(pdev); + else + return mana_gd_init_vf_regs(pdev); +} + +/* Suppress logging when we set timeout to zero */ +bool mana_need_log(struct gdma_context *gc, int err) +{ + struct hw_channel_context *hwc; + + if (err != -ETIMEDOUT) + return true; + + if (!gc) + return true; + + hwc = gc->hwc.driver_data; + if (hwc && hwc->hwc_timeout == 0) + return false; + + return true; +} + +static int mana_gd_query_max_resources(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_query_max_resources_resp resp = {}; + struct gdma_general_req req = {}; + unsigned int max_num_queues; + u8 bm_hostmode; + u16 num_ports; + int err; + + /* Reset msi_sharing so it is recomputed from current hardware + * state. On resume, num_online_cpus() or num_msix_usable may + * have changed, making dedicated MSI-X feasible where it was + * not before. Only reset on platforms that support dynamic + * MSI-X allocation; on non-dyn platforms msi_sharing is + * unconditionally true (set in mana_gd_setup_hwc_irqs). + */ + if (pci_msix_can_alloc_dyn(to_pci_dev(gc->dev))) + gc->msi_sharing = false; + + mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_MAX_RESOURCES, + sizeof(req), sizeof(resp)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + dev_err(gc->dev, "Failed to query resource info: %d, 0x%x\n", + err, resp.hdr.status); + return err ? err : -EPROTO; + } + + if (!pci_msix_can_alloc_dyn(pdev)) { + if (gc->num_msix_usable > resp.max_msix) + gc->num_msix_usable = resp.max_msix; + } else { + /* If dynamic allocation is enabled we have already allocated + * hwc msi + * Also, we make sure in this case the following is always true + * (num_msix_usable - 1 HWC) <= num_online_cpus() + */ + gc->num_msix_usable = min(resp.max_msix, num_online_cpus() + 1); + } + + if (gc->num_msix_usable <= 1) + return -ENOSPC; + + gc->max_num_queues = num_online_cpus(); + if (gc->max_num_queues > MANA_MAX_NUM_QUEUES) + gc->max_num_queues = MANA_MAX_NUM_QUEUES; + + if (gc->max_num_queues > resp.max_eq) + gc->max_num_queues = resp.max_eq; + + if (gc->max_num_queues > resp.max_cq) + gc->max_num_queues = resp.max_cq; + + if (gc->max_num_queues > resp.max_sq) + gc->max_num_queues = resp.max_sq; + + if (gc->max_num_queues > resp.max_rq) + gc->max_num_queues = resp.max_rq; + + /* The Hardware Channel (HWC) used 1 MSI-X */ + if (gc->max_num_queues > gc->num_msix_usable - 1) + gc->max_num_queues = gc->num_msix_usable - 1; + + if (gc->max_num_queues == 0) + return -ENOSPC; + + debugfs_create_u32("num_msix_usable", 0400, gc->mana_pci_debugfs, + &gc->num_msix_usable); + debugfs_create_u32("max_num_queues", 0400, gc->mana_pci_debugfs, + &gc->max_num_queues); + + err = mana_gd_query_device_cfg(gc, MANA_MAJOR_VERSION, + MANA_MINOR_VERSION, + MANA_MICRO_VERSION, + &num_ports, &bm_hostmode); + if (err) + return err; + + if (!num_ports) { + dev_err(gc->dev, "Failed to detect any vPort\n"); + return -EINVAL; + } + + /* Cap to the same limit used by mana_probe() for port instantiation, + * so MSI-X and queue budgeting matches the actual port count. + */ + if (num_ports > MAX_PORTS_IN_MANA_DEV) + num_ports = MAX_PORTS_IN_MANA_DEV; + + /* + * Adjust the per-vPort max queue count to allow dedicated + * MSIx for each vPort. Prefer at least MANA_DEF_NUM_QUEUES, + * but the hardware max (gc->max_num_queues) takes precedence. + */ + max_num_queues = (gc->num_msix_usable - 1) / num_ports; + max_num_queues = rounddown_pow_of_two(max(max_num_queues, 1U)); + if (max_num_queues < MANA_DEF_NUM_QUEUES) + max_num_queues = MANA_DEF_NUM_QUEUES; + + /* + * Use dedicated MSIx for EQs whenever possible, use MSIx sharing for + * Ethernet EQs when (max_num_queues * num_ports > num_msix_usable - 1). + */ + max_num_queues = min(gc->max_num_queues, max_num_queues); + if (max_num_queues * num_ports > gc->num_msix_usable - 1) + gc->msi_sharing = true; + + /* If MSI is shared, use max allowed value */ + if (gc->msi_sharing) + gc->max_num_queues_vport = min(gc->num_msix_usable - 1, + gc->max_num_queues); + else + gc->max_num_queues_vport = max_num_queues; + + dev_info(gc->dev, "MSI sharing mode %u max queues %u\n", + gc->msi_sharing, gc->max_num_queues_vport); + + return 0; +} + +static int mana_gd_query_hwc_timeout(struct pci_dev *pdev, u32 *timeout_val) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_query_hwc_timeout_resp resp = {}; + struct gdma_query_hwc_timeout_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, GDMA_QUERY_HWC_TIMEOUT, + sizeof(req), sizeof(resp)); + req.timeout_ms = *timeout_val; + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) + return err ? err : -EPROTO; + + *timeout_val = resp.timeout_ms; + + return 0; +} + +static int mana_gd_detect_devices(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_list_devices_resp resp = {}; + struct gdma_general_req req = {}; + struct gdma_dev_id dev; + int found_dev = 0; + u16 dev_type; + int err; + u32 i; + + mana_gd_init_req_hdr(&req.hdr, GDMA_LIST_DEVICES, sizeof(req), + sizeof(resp)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + dev_err(gc->dev, "Failed to detect devices: %d, 0x%x\n", err, + resp.hdr.status); + return err ? err : -EPROTO; + } + + for (i = 0; i < GDMA_DEV_LIST_SIZE && + found_dev < resp.num_of_devs; i++) { + dev = resp.devs[i]; + dev_type = dev.type; + + /* Skip empty devices */ + if (dev.as_uint32 == 0) + continue; + + found_dev++; + + /* HWC is already detected in mana_hwc_create_channel(). */ + if (dev_type == GDMA_DEVICE_HWC) + continue; + + if (dev_type == GDMA_DEVICE_MANA) { + gc->mana.gdma_context = gc; + gc->mana.dev_id = dev; + } else if (dev_type == GDMA_DEVICE_MANA_IB) { + gc->mana_ib.dev_id = dev; + gc->mana_ib.gdma_context = gc; + } + } + + return gc->mana.dev_id.type == 0 ? -ENODEV : 0; +} + +int mana_gd_send_request(struct gdma_context *gc, u32 req_len, const void *req, + u32 resp_len, void *resp) +{ + struct hw_channel_context *hwc = gc->hwc.driver_data; + + return mana_hwc_send_request(hwc, req_len, req, resp_len, resp); +} +EXPORT_SYMBOL_NS(mana_gd_send_request, "NET_MANA"); + +int mana_gd_alloc_memory(struct gdma_context *gc, unsigned int length, + struct gdma_mem_info *gmi, bool allow_scatter) +{ + unsigned int npages, i; + dma_addr_t dma_handle; + bool can_fallback; + void *buf; + + if (length < MANA_PAGE_SIZE || !is_power_of_2(length)) + return -EINVAL; + + gmi->dev = gc->dev; + + /* An allocation that fits in one page does not benefit from + * fallback. + */ + can_fallback = allow_scatter && length > PAGE_SIZE; + + /* Warn only when there is no fallback to rescue the failure. */ + buf = dma_alloc_coherent(gmi->dev, length, &dma_handle, + GFP_KERNEL | + (can_fallback ? __GFP_NOWARN : 0)); + if (buf) { + gmi->dma_handle = dma_handle; + gmi->virt_addr = buf; + gmi->length = length; + gmi->nr_pages = 0; + return 0; + } + + if (!can_fallback) + return -ENOMEM; + + /* length is a power of 2 above PAGE_SIZE, so this divides exactly. */ + npages = length / PAGE_SIZE; + + gmi->pages_va = kvcalloc(npages, sizeof(*gmi->pages_va), GFP_KERNEL); + if (!gmi->pages_va) + return -ENOMEM; + + gmi->pages_dma = kvcalloc(npages, sizeof(*gmi->pages_dma), GFP_KERNEL); + if (!gmi->pages_dma) + goto free_va; + + for (i = 0; i < npages; i++) { + gmi->pages_va[i] = dma_alloc_coherent(gmi->dev, PAGE_SIZE, + &gmi->pages_dma[i], + GFP_KERNEL); + if (!gmi->pages_va[i]) + goto free_pages; + } + + dev_info_ratelimited(gmi->dev, + "contiguous %u-byte DMA alloc failed; using %u scattered pages\n", + length, npages); + + gmi->virt_addr = NULL; + gmi->dma_handle = 0; + gmi->length = length; + gmi->nr_pages = npages; + + return 0; + +free_pages: + while (i--) + dma_free_coherent(gmi->dev, PAGE_SIZE, gmi->pages_va[i], + gmi->pages_dma[i]); + kvfree(gmi->pages_dma); + gmi->pages_dma = NULL; +free_va: + kvfree(gmi->pages_va); + gmi->pages_va = NULL; + return -ENOMEM; +} + +void mana_gd_free_memory(struct gdma_mem_info *gmi) +{ + unsigned int i; + + if (gmi->nr_pages > 0) { + for (i = 0; i < gmi->nr_pages; i++) + dma_free_coherent(gmi->dev, PAGE_SIZE, gmi->pages_va[i], + gmi->pages_dma[i]); + kvfree(gmi->pages_va); + kvfree(gmi->pages_dma); + gmi->pages_va = NULL; + gmi->pages_dma = NULL; + gmi->nr_pages = 0; + return; + } + + dma_free_coherent(gmi->dev, gmi->length, gmi->virt_addr, + gmi->dma_handle); +} + +static int mana_gd_create_hw_eq(struct gdma_context *gc, + struct gdma_queue *queue) +{ + struct gdma_create_queue_resp resp = {}; + struct gdma_create_queue_req req = {}; + int err; + + if (queue->type != GDMA_EQ) + return -EINVAL; + + mana_gd_init_req_hdr(&req.hdr, GDMA_CREATE_QUEUE, + sizeof(req), sizeof(resp)); + + req.hdr.dev_id = queue->gdma_dev->dev_id; + req.type = queue->type; + req.pdid = queue->gdma_dev->pdid; + req.doolbell_id = queue->gdma_dev->doorbell; + req.gdma_region = queue->mem_info.dma_region_handle; + req.queue_size = queue->queue_size; + req.log2_throttle_limit = queue->eq.log2_throttle_limit; + req.eq_pci_msix_index = queue->eq.msix_index; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + dev_err(gc->dev, "Failed to create queue: %d, 0x%x\n", err, + resp.hdr.status); + return err ? err : -EPROTO; + } + + queue->id = resp.queue_index; + queue->eq.disable_needed = true; + queue->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; + return 0; +} + +static int mana_gd_disable_queue(struct gdma_queue *queue) +{ + struct gdma_context *gc = queue->gdma_dev->gdma_context; + struct gdma_disable_queue_req req = {}; + struct gdma_general_resp resp = {}; + int err; + + WARN_ON(queue->type != GDMA_EQ); + + mana_gd_init_req_hdr(&req.hdr, GDMA_DISABLE_QUEUE, + sizeof(req), sizeof(resp)); + + req.hdr.dev_id = queue->gdma_dev->dev_id; + req.type = queue->type; + req.queue_index = queue->id; + req.alloc_res_id_on_creation = 1; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + if (mana_need_log(gc, err)) + dev_err(gc->dev, "Failed to disable queue: %d, 0x%x\n", err, + resp.hdr.status); + return err ? err : -EPROTO; + } + + return 0; +} + +#define DOORBELL_OFFSET_SQ 0x0 +#define DOORBELL_OFFSET_RQ 0x400 +#define DOORBELL_OFFSET_CQ 0x800 +#define DOORBELL_OFFSET_EQ 0xFF8 +#define DOORBELL_OFFSET_DIM 0x820 + +static void mana_gd_ring_doorbell(struct gdma_context *gc, u32 db_index, + enum gdma_queue_type q_type, u32 qid, + u32 tail_ptr, u8 num_req) +{ + void __iomem *addr = gc->db_page_base + gc->db_page_size * db_index; + union gdma_doorbell_entry e = {}; + + switch (q_type) { + case GDMA_EQ: + e.eq.id = qid; + e.eq.tail_ptr = tail_ptr; + e.eq.arm = num_req; + + addr += DOORBELL_OFFSET_EQ; + break; + + case GDMA_CQ: + e.cq.id = qid; + e.cq.tail_ptr = tail_ptr; + e.cq.arm = num_req; + + addr += DOORBELL_OFFSET_CQ; + break; + + case GDMA_RQ: + e.rq.id = qid; + e.rq.tail_ptr = tail_ptr; + e.rq.wqe_cnt = num_req; + + addr += DOORBELL_OFFSET_RQ; + break; + + case GDMA_SQ: + e.sq.id = qid; + e.sq.tail_ptr = tail_ptr; + + addr += DOORBELL_OFFSET_SQ; + break; + + case GDMA_DIM: + e.dim.id = qid; + e.dim.mod_usec = FIELD_GET(MANA_INTR_MODR_USEC_MAX, tail_ptr); + e.dim.mod_usec_vld = !!(tail_ptr & MANA_INTR_MODR_USEC_VLD); + e.dim.mod_comps = FIELD_GET(MANA_INTR_MODR_COMP_MASK, tail_ptr); + e.dim.mod_comps_vld = num_req; + + addr += DOORBELL_OFFSET_DIM; + break; + + default: + WARN_ON(1); + return; + } + + /* Ensure all writes are done before ring doorbell */ + wmb(); + + writeq(e.as_uint64, addr); +} + +void mana_gd_wq_ring_doorbell(struct gdma_context *gc, struct gdma_queue *queue) +{ + /* Hardware Spec specifies that software client should set 0 for + * wqe_cnt for Receive Queues. This value is not used in Send Queues. + */ + mana_gd_ring_doorbell(gc, queue->gdma_dev->doorbell, queue->type, + queue->id, queue->head * GDMA_WQE_BU_SIZE, 0); +} +EXPORT_SYMBOL_NS(mana_gd_wq_ring_doorbell, "NET_MANA"); + +void mana_gd_ring_cq(struct gdma_queue *cq, u8 arm_bit) +{ + struct gdma_context *gc = cq->gdma_dev->gdma_context; + + u32 num_cqe = cq->queue_size / GDMA_CQE_SIZE; + + u32 head = cq->head % (num_cqe << GDMA_CQE_OWNER_BITS); + + mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, cq->type, cq->id, + head, arm_bit); +} +EXPORT_SYMBOL_NS(mana_gd_ring_cq, "NET_MANA"); + +void mana_gd_ring_dim(struct gdma_queue *cq, u32 mod_usec, bool mod_usec_vld, + u32 mod_comps, bool mod_comps_vld) +{ + struct gdma_context *gc = cq->gdma_dev->gdma_context; + u32 dim_val; + + /* Convert the DIM values to doorbell parameters */ + dim_val = FIELD_PREP(MANA_INTR_MODR_USEC_MAX, mod_usec) | + FIELD_PREP(MANA_INTR_MODR_COMP_MASK, mod_comps); + if (mod_usec_vld) + dim_val |= MANA_INTR_MODR_USEC_VLD; + + mana_gd_ring_doorbell(gc, cq->gdma_dev->doorbell, GDMA_DIM, cq->id, + dim_val, mod_comps_vld); +} +EXPORT_SYMBOL_NS(mana_gd_ring_dim, "NET_MANA"); + +#define MANA_SERVICE_PERIOD 10 + +static void mana_serv_rescan(struct pci_dev *pdev) +{ + struct pci_bus *parent; + + pci_lock_rescan_remove(); + + parent = pdev->bus; + if (!parent) { + dev_err(&pdev->dev, "MANA service: no parent bus\n"); + goto out; + } + + pci_stop_and_remove_bus_device(pdev); + pci_rescan_bus(parent); + +out: + pci_unlock_rescan_remove(); +} + +static void mana_serv_fpga(struct pci_dev *pdev) +{ + struct pci_bus *bus, *parent; + + pci_lock_rescan_remove(); + + bus = pdev->bus; + if (!bus) { + dev_err(&pdev->dev, "MANA service: no bus\n"); + goto out; + } + + parent = bus->parent; + if (!parent) { + dev_err(&pdev->dev, "MANA service: no parent bus\n"); + goto out; + } + + pci_stop_and_remove_bus_device(bus->self); + + msleep(MANA_SERVICE_PERIOD * 1000); + + pci_rescan_bus(parent); + +out: + pci_unlock_rescan_remove(); +} + +static void mana_serv_reset(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct hw_channel_context *hwc; + int ret; + + if (!gc) { + /* Perform PCI rescan on device if GC is not set up */ + dev_err(&pdev->dev, "MANA service: GC not setup, rescanning\n"); + mana_serv_rescan(pdev); + return; + } + + hwc = gc->hwc.driver_data; + if (!hwc) { + dev_err(&pdev->dev, "MANA service: no HWC\n"); + goto out; + } + + /* HWC is not responding in this case, so don't wait */ + hwc->hwc_timeout = 0; + + dev_info(&pdev->dev, "MANA reset cycle start\n"); + + mana_gd_suspend(pdev, PMSG_SUSPEND); + + msleep(MANA_SERVICE_PERIOD * 1000); + + ret = mana_gd_resume(pdev); + if (ret == -ETIMEDOUT || ret == -EPROTO) { + /* Perform PCI rescan on device if we failed on HWC */ + dev_err(&pdev->dev, "MANA service: resume failed, rescanning\n"); + mana_serv_rescan(pdev); + return; + } + + if (ret) + dev_info(&pdev->dev, "MANA reset cycle failed err %d\n", ret); + else + dev_info(&pdev->dev, "MANA reset cycle completed\n"); + +out: + clear_bit(GC_IN_SERVICE, &gc->flags); +} + +static void mana_do_service(enum gdma_eqe_type type, struct pci_dev *pdev) +{ + switch (type) { + case GDMA_EQE_HWC_FPGA_RECONFIG: + mana_serv_fpga(pdev); + break; + + case GDMA_EQE_HWC_RESET_REQUEST: + mana_serv_reset(pdev); + break; + + default: + dev_err(&pdev->dev, "MANA service: unknown type %d\n", type); + break; + } +} + +static void mana_recovery_delayed_func(struct work_struct *w) +{ + struct mana_dev_recovery_work *work; + struct mana_dev_recovery *dev; + unsigned long flags; + + work = container_of(w, struct mana_dev_recovery_work, work.work); + + spin_lock_irqsave(&work->lock, flags); + + while (!list_empty(&work->dev_list)) { + dev = list_first_entry(&work->dev_list, + struct mana_dev_recovery, list); + list_del(&dev->list); + spin_unlock_irqrestore(&work->lock, flags); + + mana_do_service(dev->type, dev->pdev); + pci_dev_put(dev->pdev); + kfree(dev); + + spin_lock_irqsave(&work->lock, flags); + } + + spin_unlock_irqrestore(&work->lock, flags); +} + +static void mana_serv_func(struct work_struct *w) +{ + struct mana_serv_work *mns_wk; + struct pci_dev *pdev; + + mns_wk = container_of(w, struct mana_serv_work, serv_work); + pdev = mns_wk->pdev; + + if (pdev) + mana_do_service(mns_wk->type, pdev); + + pci_dev_put(pdev); + kfree(mns_wk); + module_put(THIS_MODULE); +} + +int mana_schedule_serv_work(struct gdma_context *gc, enum gdma_eqe_type type) +{ + struct mana_serv_work *mns_wk; + + if (test_and_set_bit(GC_IN_SERVICE, &gc->flags)) { + dev_info(gc->dev, "Already in service\n"); + return -EBUSY; + } + + if (!try_module_get(THIS_MODULE)) { + dev_info(gc->dev, "Module is unloading\n"); + clear_bit(GC_IN_SERVICE, &gc->flags); + return -ENODEV; + } + + mns_wk = kzalloc(sizeof(*mns_wk), GFP_ATOMIC); + if (!mns_wk) { + module_put(THIS_MODULE); + clear_bit(GC_IN_SERVICE, &gc->flags); + return -ENOMEM; + } + + dev_info(gc->dev, "Start MANA service type:%d\n", type); + mns_wk->pdev = to_pci_dev(gc->dev); + mns_wk->type = type; + pci_dev_get(mns_wk->pdev); + INIT_WORK(&mns_wk->serv_work, mana_serv_func); + schedule_work(&mns_wk->serv_work); + return 0; +} + +/* Return the CPU address of byte @offset within a queue's ring buffer. */ +static void *mana_gd_ring_ptr(const struct gdma_queue *q, u32 offset) +{ + const struct gdma_mem_info *gmi = &q->mem_info; + + if (gmi->nr_pages > 0) + return (u8 *)gmi->pages_va[offset / PAGE_SIZE] + + (offset & (PAGE_SIZE - 1)); + + return q->queue_mem_ptr + offset; +} + +/* Number of bytes from @offset to the end of the CPU-contiguous region: the + * rest of the ring, or the rest of the current page when scattered. + */ +static u32 mana_gd_ring_contig_avail(const struct gdma_queue *q, u32 offset) +{ + if (q->mem_info.nr_pages > 0) + return PAGE_SIZE - (offset & (PAGE_SIZE - 1)); + + return q->queue_size - offset; +} + +/* Copy up to @count bytes from ring offset *@pos of @q into user buffer @buf, + * so a scattered ring reads back as if it were contiguous. Returns bytes + * copied, 0 at end of ring, or a negative errno. + */ +ssize_t mana_gd_read_ring(struct gdma_queue *q, char __user *buf, + size_t count, loff_t *pos) +{ + u32 size = q->queue_size; + loff_t off = *pos; + size_t copied = 0; + + if (off < 0) + return -EINVAL; + if (off >= size || !count) + return 0; + count = min_t(size_t, count, size - off); + + while (count) { + u32 offset = off; + u32 avail = mana_gd_ring_contig_avail(q, offset); + size_t chunk = min_t(size_t, count, avail); + size_t left = copy_to_user(buf, mana_gd_ring_ptr(q, offset), + chunk); + + chunk -= left; + buf += chunk; + off += chunk; + copied += chunk; + count -= chunk; + if (left) + break; + } + + if (!copied) + return -EFAULT; + + *pos = off; + return copied; +} + +static void mana_gd_process_eqe(struct gdma_queue *eq) +{ + u32 head = eq->head % (eq->queue_size / GDMA_EQE_SIZE); + struct gdma_context *gc = eq->gdma_dev->gdma_context; + union gdma_eqe_info eqe_info; + enum gdma_eqe_type type; + struct gdma_event event; + struct gdma_queue *cq; + struct gdma_eqe *eqe; + u32 cq_id; + + eqe = mana_gd_ring_ptr(eq, head * sizeof(*eqe)); + eqe_info.as_uint32 = eqe->eqe_info; + type = eqe_info.type; + + switch (type) { + case GDMA_EQE_COMPLETION: + cq_id = eqe->details[0] & 0xFFFFFF; + if (WARN_ON_ONCE(cq_id >= gc->max_num_cqs)) + break; + + cq = gc->cq_table[cq_id]; + if (WARN_ON_ONCE(!cq || cq->type != GDMA_CQ || cq->id != cq_id)) + break; + + if (cq->cq.callback) + cq->cq.callback(cq->cq.context, cq); + + break; + + case GDMA_EQE_TEST_EVENT: + gc->test_event_eq_id = eq->id; + complete(&gc->eq_test_event); + break; + + case GDMA_EQE_HWC_INIT_EQ_ID_DB: + case GDMA_EQE_HWC_INIT_DATA: + case GDMA_EQE_HWC_INIT_DONE: + case GDMA_EQE_HWC_SOC_SERVICE: + case GDMA_EQE_RNIC_QP_FATAL: + case GDMA_EQE_HWC_SOC_RECONFIG_DATA: + if (!eq->eq.callback) + break; + + event.type = type; + memcpy(&event.details, &eqe->details, GDMA_EVENT_DATA_SIZE); + eq->eq.callback(eq->eq.context, eq, &event); + break; + + case GDMA_EQE_HWC_FPGA_RECONFIG: + case GDMA_EQE_HWC_RESET_REQUEST: + dev_info(gc->dev, "Recv MANA service type:%d\n", type); + + if (!test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { + /* + * Device is in probe and we received a hardware reset + * event, the probe function will detect that the flag + * has changed and perform service procedure. + */ + dev_info(gc->dev, + "Service is to be processed in probe\n"); + break; + } + mana_schedule_serv_work(gc, type); + break; + + default: + break; + } +} + +static void mana_gd_process_eq_events(void *arg) +{ + u32 owner_bits, new_bits, old_bits; + union gdma_eqe_info eqe_info; + struct gdma_queue *eq = arg; + struct gdma_context *gc; + struct gdma_eqe *eqe; + u32 head, num_eqe; + int i; + + gc = eq->gdma_dev->gdma_context; + + num_eqe = eq->queue_size / GDMA_EQE_SIZE; + + /* Process up to 5 EQEs at a time, and update the HW head. */ + for (i = 0; i < 5; i++) { + eqe = mana_gd_ring_ptr(eq, (eq->head % num_eqe) * sizeof(*eqe)); + eqe_info.as_uint32 = eqe->eqe_info; + owner_bits = eqe_info.owner_bits; + + old_bits = (eq->head / num_eqe - 1) & GDMA_EQE_OWNER_MASK; + /* No more entries */ + if (owner_bits == old_bits) { + /* return here without ringing the doorbell */ + if (i == 0) + return; + break; + } + + new_bits = (eq->head / num_eqe) & GDMA_EQE_OWNER_MASK; + if (owner_bits != new_bits) { + dev_err(gc->dev, "EQ %d: overflow detected\n", eq->id); + break; + } + + /* Per GDMA spec, rmb is necessary after checking owner_bits, before + * reading eqe. + */ + rmb(); + + mana_gd_process_eqe(eq); + + eq->head++; + } + + head = eq->head % (num_eqe << GDMA_EQE_OWNER_BITS); + + mana_gd_ring_doorbell(gc, eq->gdma_dev->doorbell, eq->type, eq->id, + head, SET_ARM_BIT); +} + +static int mana_gd_register_irq(struct gdma_queue *queue, + const struct gdma_queue_spec *spec) +{ + struct gdma_dev *gd = queue->gdma_dev; + struct gdma_irq_context *gic; + struct gdma_context *gc; + unsigned int msi_index; + unsigned long flags; + struct device *dev; + int err = 0; + + gc = gd->gdma_context; + dev = gc->dev; + msi_index = spec->eq.msix_index; + + if (msi_index >= gc->num_msix_usable) { + err = -ENOSPC; + dev_err(dev, "Register IRQ err:%d, msi:%u nMSI:%u", + err, msi_index, gc->num_msix_usable); + + return err; + } + + queue->eq.msix_index = msi_index; + /* The caller acquired a GIC reference via mana_gd_get_gic(). + * That refcount prevents mana_gd_put_gic() from erasing this + * irq_contexts entry concurrently. + */ + gic = xa_load(&gc->irq_contexts, msi_index); + if (WARN_ON(!gic)) + return -EINVAL; + + spin_lock_irqsave(&gic->lock, flags); + list_add_rcu(&queue->entry, &gic->eq_list); + spin_unlock_irqrestore(&gic->lock, flags); + + return 0; +} + +static void mana_gd_deregister_irq(struct gdma_queue *queue) +{ + struct gdma_dev *gd = queue->gdma_dev; + struct gdma_irq_context *gic; + struct gdma_context *gc; + unsigned int msix_index; + unsigned long flags; + struct gdma_queue *eq; + + gc = gd->gdma_context; + + /* At most num_online_cpus() + 1 interrupts are used. */ + msix_index = queue->eq.msix_index; + if (WARN_ON(msix_index >= gc->num_msix_usable)) + return; + + /* The caller releases the GIC reference via mana_gd_put_gic() + * after this function returns. The refcount guarantees this + * irq_contexts entry is still valid. + */ + gic = xa_load(&gc->irq_contexts, msix_index); + if (WARN_ON(!gic)) + return; + + spin_lock_irqsave(&gic->lock, flags); + list_for_each_entry_rcu(eq, &gic->eq_list, entry) { + if (queue == eq) { + list_del_rcu(&eq->entry); + break; + } + } + spin_unlock_irqrestore(&gic->lock, flags); + + synchronize_rcu(); +} + +int mana_gd_test_eq(struct gdma_context *gc, struct gdma_queue *eq) +{ + struct gdma_generate_test_event_req req = {}; + struct gdma_general_resp resp = {}; + struct device *dev = gc->dev; + int err; + + mutex_lock(&gc->eq_test_event_mutex); + + init_completion(&gc->eq_test_event); + gc->test_event_eq_id = INVALID_QUEUE_ID; + + mana_gd_init_req_hdr(&req.hdr, GDMA_GENERATE_TEST_EQE, + sizeof(req), sizeof(resp)); + + req.hdr.dev_id = eq->gdma_dev->dev_id; + req.queue_index = eq->id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err) { + if (mana_need_log(gc, err)) + dev_err(dev, "test_eq failed: %d\n", err); + goto out; + } + + err = -EPROTO; + + if (resp.hdr.status) { + dev_err(dev, "test_eq failed: 0x%x\n", resp.hdr.status); + goto out; + } + + if (!wait_for_completion_timeout(&gc->eq_test_event, 30 * HZ)) { + dev_err(dev, "test_eq timed out on queue %d\n", eq->id); + goto out; + } + + if (eq->id != gc->test_event_eq_id) { + dev_err(dev, "test_eq got an event on wrong queue %d (%d)\n", + gc->test_event_eq_id, eq->id); + goto out; + } + + err = 0; +out: + mutex_unlock(&gc->eq_test_event_mutex); + return err; +} + +static void mana_gd_destroy_eq(struct gdma_context *gc, bool flush_evenets, + struct gdma_queue *queue) +{ + int err; + + if (flush_evenets) { + err = mana_gd_test_eq(gc, queue); + if (err && mana_need_log(gc, err)) + dev_warn(gc->dev, "Failed to flush EQ: %d\n", err); + } + + mana_gd_deregister_irq(queue); + + if (queue->eq.disable_needed) + mana_gd_disable_queue(queue); +} + +static int mana_gd_create_eq(struct gdma_dev *gd, + const struct gdma_queue_spec *spec, + bool create_hwq, struct gdma_queue *queue) +{ + struct gdma_context *gc = gd->gdma_context; + struct device *dev = gc->dev; + u32 log2_num_entries; + int err; + + queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; + queue->id = INVALID_QUEUE_ID; + + log2_num_entries = ilog2(queue->queue_size / GDMA_EQE_SIZE); + + if (spec->eq.log2_throttle_limit > log2_num_entries) { + dev_err(dev, "EQ throttling limit (%lu) > maximum EQE (%u)\n", + spec->eq.log2_throttle_limit, log2_num_entries); + return -EINVAL; + } + + err = mana_gd_register_irq(queue, spec); + if (err) { + dev_err(dev, "Failed to register irq: %d\n", err); + return err; + } + + queue->eq.callback = spec->eq.callback; + queue->eq.context = spec->eq.context; + queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); + queue->eq.log2_throttle_limit = spec->eq.log2_throttle_limit ?: 1; + + if (create_hwq) { + err = mana_gd_create_hw_eq(gc, queue); + if (err) + goto out; + + err = mana_gd_test_eq(gc, queue); + if (err) + goto out; + } + + return 0; +out: + dev_err(dev, "Failed to create EQ: %d\n", err); + mana_gd_destroy_eq(gc, false, queue); + queue->eq.msix_index = INVALID_PCI_MSIX_INDEX; + return err; +} + +static void mana_gd_create_cq(const struct gdma_queue_spec *spec, + struct gdma_queue *queue) +{ + u32 log2_num_entries = ilog2(spec->queue_size / GDMA_CQE_SIZE); + + queue->head |= INITIALIZED_OWNER_BIT(log2_num_entries); + queue->cq.parent = spec->cq.parent_eq; + queue->cq.context = spec->cq.context; + queue->cq.callback = spec->cq.callback; +} + +static void mana_gd_destroy_cq(struct gdma_context *gc, + struct gdma_queue *queue) +{ + u32 id = queue->id; + + if (id >= gc->max_num_cqs) + return; + + if (!gc->cq_table[id]) + return; + + gc->cq_table[id] = NULL; +} + +int mana_gd_create_hwc_queue(struct gdma_dev *gd, + const struct gdma_queue_spec *spec, + struct gdma_queue **queue_ptr) +{ + struct gdma_context *gc = gd->gdma_context; + struct gdma_mem_info *gmi; + struct gdma_queue *queue; + int err; + + queue = kzalloc_obj(*queue); + if (!queue) + return -ENOMEM; + + gmi = &queue->mem_info; + err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, false); + if (err) { + dev_err(gc->dev, "GDMA queue type: %d, size: %u, gdma memory allocation err: %d\n", + spec->type, spec->queue_size, err); + goto free_q; + } + + queue->head = 0; + queue->tail = 0; + queue->queue_mem_ptr = gmi->virt_addr; + queue->queue_size = spec->queue_size; + queue->monitor_avl_buf = spec->monitor_avl_buf; + queue->type = spec->type; + queue->gdma_dev = gd; + + if (spec->type == GDMA_EQ) + err = mana_gd_create_eq(gd, spec, false, queue); + else if (spec->type == GDMA_CQ) + mana_gd_create_cq(spec, queue); + + if (err) + goto out; + + *queue_ptr = queue; + return 0; +out: + dev_err(gc->dev, "Failed to create queue type %d of size %u, err: %d\n", + spec->type, spec->queue_size, err); + mana_gd_free_memory(gmi); +free_q: + kfree(queue); + return err; +} + +int mana_gd_destroy_dma_region(struct gdma_context *gc, u64 dma_region_handle) +{ + struct gdma_destroy_dma_region_req req = {}; + struct gdma_general_resp resp = {}; + int err; + + if (dma_region_handle == GDMA_INVALID_DMA_REGION) + return 0; + + mana_gd_init_req_hdr(&req.hdr, GDMA_DESTROY_DMA_REGION, sizeof(req), + sizeof(resp)); + req.dma_region_handle = dma_region_handle; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + if (mana_need_log(gc, err)) + dev_err(gc->dev, "Failed to destroy DMA region: %d, 0x%x\n", + err, resp.hdr.status); + return -EPROTO; + } + + return 0; +} +EXPORT_SYMBOL_NS(mana_gd_destroy_dma_region, "NET_MANA"); + +static int mana_gd_create_dma_region(struct gdma_dev *gd, + struct gdma_mem_info *gmi) +{ + unsigned int num_page = gmi->length / MANA_PAGE_SIZE; + struct gdma_create_dma_region_req *req = NULL; + struct gdma_create_dma_region_resp resp = {}; + struct gdma_context *gc = gd->gdma_context; + struct hw_channel_context *hwc; + u32 length = gmi->length; + size_t req_msg_size; + int err; + int i; + + if (length < MANA_PAGE_SIZE || !is_power_of_2(length)) + return -EINVAL; + + if (gmi->nr_pages == 0 && !MANA_PAGE_ALIGNED(gmi->virt_addr)) + return -EINVAL; + + hwc = gc->hwc.driver_data; + req_msg_size = struct_size(req, page_addr_list, num_page); + if (req_msg_size > hwc->max_req_msg_size) + return -EINVAL; + + req = kzalloc(req_msg_size, GFP_KERNEL); + if (!req) + return -ENOMEM; + + mana_gd_init_req_hdr(&req->hdr, GDMA_CREATE_DMA_REGION, + req_msg_size, sizeof(resp)); + req->length = length; + req->offset_in_page = 0; + req->gdma_page_type = GDMA_PAGE_TYPE_4K; + req->page_count = num_page; + req->page_addr_list_len = num_page; + + if (gmi->nr_pages > 0) { + unsigned int subpages = PAGE_SIZE / MANA_PAGE_SIZE; + unsigned int idx = 0; + unsigned int pg, sub; + + /* Each PAGE_SIZE chunk is physically contiguous and contains + * PAGE_SIZE / MANA_PAGE_SIZE consecutive device pages. + */ + for (pg = 0; pg < gmi->nr_pages; pg++) + for (sub = 0; sub < subpages; sub++) + req->page_addr_list[idx++] = + gmi->pages_dma[pg] + + sub * MANA_PAGE_SIZE; + } else { + for (i = 0; i < num_page; i++) + req->page_addr_list[i] = + gmi->dma_handle + i * MANA_PAGE_SIZE; + } + + err = mana_gd_send_request(gc, req_msg_size, req, sizeof(resp), &resp); + if (err) + goto out; + + if (resp.hdr.status || + resp.dma_region_handle == GDMA_INVALID_DMA_REGION) { + dev_err(gc->dev, "Failed to create DMA region: 0x%x\n", + resp.hdr.status); + err = -EPROTO; + goto out; + } + + gmi->dma_region_handle = resp.dma_region_handle; + dev_dbg(gc->dev, "Created DMA region handle 0x%llx\n", + gmi->dma_region_handle); +out: + if (err) + dev_dbg(gc->dev, + "Failed to create DMA region of length: %u, page_type: %d, status: 0x%x, err: %d\n", + length, req->gdma_page_type, resp.hdr.status, err); + kfree(req); + return err; +} + +int mana_gd_create_mana_eq(struct gdma_dev *gd, + const struct gdma_queue_spec *spec, + struct gdma_queue **queue_ptr) +{ + struct gdma_context *gc = gd->gdma_context; + struct gdma_mem_info *gmi; + struct gdma_queue *queue; + int err; + + if (spec->type != GDMA_EQ) + return -EINVAL; + + queue = kzalloc_obj(*queue); + if (!queue) + return -ENOMEM; + + gmi = &queue->mem_info; + err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, true); + if (err) { + dev_err(gc->dev, "GDMA queue type: %d, size: %u, gdma memory allocation err: %d\n", + spec->type, spec->queue_size, err); + goto free_q; + } + + err = mana_gd_create_dma_region(gd, gmi); + if (err) + goto out; + + queue->head = 0; + queue->tail = 0; + queue->queue_mem_ptr = gmi->virt_addr; + queue->queue_size = spec->queue_size; + queue->monitor_avl_buf = spec->monitor_avl_buf; + queue->type = spec->type; + queue->gdma_dev = gd; + + err = mana_gd_create_eq(gd, spec, true, queue); + if (err) + goto out; + + *queue_ptr = queue; + return 0; +out: + dev_err(gc->dev, "Failed to create queue type %d of size: %u, err: %d\n", + spec->type, spec->queue_size, err); + mana_gd_free_memory(gmi); +free_q: + kfree(queue); + return err; +} +EXPORT_SYMBOL_NS(mana_gd_create_mana_eq, "NET_MANA"); + +int mana_gd_create_mana_wq_cq(struct gdma_dev *gd, + const struct gdma_queue_spec *spec, + struct gdma_queue **queue_ptr) +{ + struct gdma_context *gc = gd->gdma_context; + struct gdma_mem_info *gmi; + struct gdma_queue *queue; + int err; + + if (spec->type != GDMA_CQ && spec->type != GDMA_SQ && + spec->type != GDMA_RQ) + return -EINVAL; + + queue = kzalloc_obj(*queue); + if (!queue) + return -ENOMEM; + + queue->id = INVALID_QUEUE_ID; + + gmi = &queue->mem_info; + err = mana_gd_alloc_memory(gc, spec->queue_size, gmi, true); + if (err) { + dev_err(gc->dev, "GDMA queue type: %d, size: %u, memory allocation err: %d\n", + spec->type, spec->queue_size, err); + goto free_q; + } + + err = mana_gd_create_dma_region(gd, gmi); + if (err) + goto out; + + queue->head = 0; + queue->tail = 0; + queue->queue_mem_ptr = gmi->virt_addr; + queue->queue_size = spec->queue_size; + queue->monitor_avl_buf = spec->monitor_avl_buf; + queue->type = spec->type; + queue->gdma_dev = gd; + + if (spec->type == GDMA_CQ) + mana_gd_create_cq(spec, queue); + + *queue_ptr = queue; + return 0; +out: + dev_err(gc->dev, "Failed to create queue type %d of size: %u, err: %d\n", + spec->type, spec->queue_size, err); + mana_gd_free_memory(gmi); +free_q: + kfree(queue); + return err; +} +EXPORT_SYMBOL_NS(mana_gd_create_mana_wq_cq, "NET_MANA"); + +void mana_gd_destroy_queue(struct gdma_context *gc, struct gdma_queue *queue) +{ + struct gdma_mem_info *gmi = &queue->mem_info; + + switch (queue->type) { + case GDMA_EQ: + mana_gd_destroy_eq(gc, queue->eq.disable_needed, queue); + break; + + case GDMA_CQ: + mana_gd_destroy_cq(gc, queue); + break; + + case GDMA_RQ: + break; + + case GDMA_SQ: + break; + + default: + dev_err(gc->dev, "Can't destroy unknown queue: type=%d\n", + queue->type); + return; + } + + mana_gd_destroy_dma_region(gc, gmi->dma_region_handle); + mana_gd_free_memory(gmi); + kfree(queue); +} +EXPORT_SYMBOL_NS(mana_gd_destroy_queue, "NET_MANA"); + +int mana_gd_verify_vf_version(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_verify_ver_resp resp = {}; + struct gdma_verify_ver_req req = {}; + struct hw_channel_context *hwc; + int err; + + hwc = gc->hwc.driver_data; + mana_gd_init_req_hdr(&req.hdr, GDMA_VERIFY_VF_DRIVER_VERSION, + sizeof(req), sizeof(resp)); + + req.protocol_ver_min = GDMA_PROTOCOL_FIRST; + req.protocol_ver_max = GDMA_PROTOCOL_LAST; + + req.gd_drv_cap_flags1 = GDMA_DRV_CAP_FLAGS1; + req.gd_drv_cap_flags2 = GDMA_DRV_CAP_FLAGS2; + req.gd_drv_cap_flags3 = GDMA_DRV_CAP_FLAGS3; + req.gd_drv_cap_flags4 = GDMA_DRV_CAP_FLAGS4; + + req.drv_ver = 0; /* Unused*/ + req.os_type = 0x10; /* Linux */ + req.os_ver_major = LINUX_VERSION_MAJOR; + req.os_ver_minor = LINUX_VERSION_PATCHLEVEL; + req.os_ver_build = LINUX_VERSION_SUBLEVEL; + strscpy(req.os_ver_str1, utsname()->sysname, sizeof(req.os_ver_str1)); + strscpy(req.os_ver_str2, utsname()->release, sizeof(req.os_ver_str2)); + strscpy(req.os_ver_str3, utsname()->version, sizeof(req.os_ver_str3)); + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + dev_err(gc->dev, "VfVerifyVersionOutput: %d, status=0x%x\n", + err, resp.hdr.status); + return err ? err : -EPROTO; + } + gc->pf_cap_flags1 = resp.pf_cap_flags1; + gc->gdma_protocol_ver = resp.gdma_protocol_ver; + + debugfs_create_x64("gdma_protocol_ver", 0400, gc->mana_pci_debugfs, + &gc->gdma_protocol_ver); + debugfs_create_x64("pf_cap_flags1", 0400, gc->mana_pci_debugfs, + &gc->pf_cap_flags1); + + if (resp.pf_cap_flags1 & GDMA_DRV_CAP_FLAG_1_HWC_TIMEOUT_RECONFIG) { + err = mana_gd_query_hwc_timeout(pdev, &hwc->hwc_timeout); + if (err) { + dev_err(gc->dev, "Failed to set the hwc timeout %d\n", err); + return err; + } + dev_dbg(gc->dev, "set the hwc timeout to %u\n", hwc->hwc_timeout); + } + return 0; +} + +int mana_gd_register_device(struct gdma_dev *gd) +{ + struct gdma_context *gc = gd->gdma_context; + struct gdma_register_device_resp resp = {}; + struct gdma_general_req req = {}; + int err; + + gd->pdid = INVALID_PDID; + gd->doorbell = INVALID_DOORBELL; + gd->gpa_mkey = INVALID_MEM_KEY; + + mana_gd_init_req_hdr(&req.hdr, GDMA_REGISTER_DEVICE, sizeof(req), + sizeof(resp)); + + req.hdr.dev_id = gd->dev_id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + dev_err(gc->dev, "gdma_register_device_resp failed: %d, 0x%x\n", + err, resp.hdr.status); + return err ? err : -EPROTO; + } + + /* Validate that doorbell page for db_id is within the BAR0 region. + * In mana_gd_ring_doorbell(), the address is calculated as: + * addr = db_page_base + db_page_size * db_id + * = (bar0_va + db_page_off) + (db_page_size * db_id) + * So we need: db_page_off + db_page_size * (db_id + 1) <= bar0_size + */ + if (gc->db_page_off + gc->db_page_size * ((u64)resp.db_id + 1) > gc->bar0_size) { + dev_err(gc->dev, "Doorbell ID %u out of range\n", resp.db_id); + return -EPROTO; + } + + gd->pdid = resp.pdid; + gd->gpa_mkey = resp.gpa_mkey; + gd->doorbell = resp.db_id; + + return 0; +} + +int mana_gd_deregister_device(struct gdma_dev *gd) +{ + struct gdma_context *gc = gd->gdma_context; + struct gdma_general_resp resp = {}; + struct gdma_general_req req = {}; + int err; + + if (gd->pdid == INVALID_PDID) + return -EINVAL; + + mana_gd_init_req_hdr(&req.hdr, GDMA_DEREGISTER_DEVICE, sizeof(req), + sizeof(resp)); + + req.hdr.dev_id = gd->dev_id; + + err = mana_gd_send_request(gc, sizeof(req), &req, sizeof(resp), &resp); + if (err || resp.hdr.status) { + if (mana_need_log(gc, err)) + dev_err(gc->dev, "Failed to deregister device: %d, 0x%x\n", + err, resp.hdr.status); + if (!err) + err = -EPROTO; + } + + gd->pdid = INVALID_PDID; + gd->doorbell = INVALID_DOORBELL; + gd->gpa_mkey = INVALID_MEM_KEY; + + return err; +} + +u32 mana_gd_wq_avail_space(struct gdma_queue *wq) +{ + u32 used_space = (wq->head - wq->tail) * GDMA_WQE_BU_SIZE; + u32 wq_size = wq->queue_size; + + WARN_ON_ONCE(used_space > wq_size); + + return wq_size - used_space; +} + +u8 *mana_gd_get_wqe_ptr(const struct gdma_queue *wq, u32 wqe_offset) +{ + u32 offset = (wqe_offset * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); + + WARN_ON_ONCE((offset + GDMA_WQE_BU_SIZE) > wq->queue_size); + + return mana_gd_ring_ptr(wq, offset); +} + +static u32 mana_gd_write_client_oob(const struct gdma_wqe_request *wqe_req, + enum gdma_queue_type q_type, + u32 client_oob_size, u32 sgl_data_size, + u8 *wqe_ptr) +{ + bool oob_in_sgl = !!(wqe_req->flags & GDMA_WR_OOB_IN_SGL); + bool pad_data = !!(wqe_req->flags & GDMA_WR_PAD_BY_SGE0); + struct gdma_wqe *header = (struct gdma_wqe *)wqe_ptr; + u8 *ptr; + + memset(header, 0, sizeof(struct gdma_wqe)); + header->num_sge = wqe_req->num_sge; + header->inline_oob_size_div4 = client_oob_size / sizeof(u32); + + if (oob_in_sgl) { + WARN_ON_ONCE(wqe_req->num_sge < 2); + + header->client_oob_in_sgl = 1; + + if (pad_data) + header->last_vbytes = wqe_req->sgl[0].size; + } + + if (q_type == GDMA_SQ) + header->client_data_unit = wqe_req->client_data_unit; + + /* The size of gdma_wqe + client_oob_size must be less than or equal + * to one Basic Unit (i.e. 32 bytes), so the pointer can't go beyond + * the queue memory buffer boundary. + */ + ptr = wqe_ptr + sizeof(header); + + if (wqe_req->inline_oob_data && wqe_req->inline_oob_size > 0) { + memcpy(ptr, wqe_req->inline_oob_data, wqe_req->inline_oob_size); + + if (client_oob_size > wqe_req->inline_oob_size) + memset(ptr + wqe_req->inline_oob_size, 0, + client_oob_size - wqe_req->inline_oob_size); + } + + return sizeof(header) + client_oob_size; +} + +static void mana_gd_write_sgl(struct gdma_queue *wq, u32 sgl_offset, + const struct gdma_wqe_request *wqe_req) +{ + u32 size_to_end = mana_gd_ring_contig_avail(wq, sgl_offset); + u32 sgl_size = sizeof(struct gdma_sge) * wqe_req->num_sge; + const u8 *address = (u8 *)wqe_req->sgl; + + if (size_to_end < sgl_size) { + memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, size_to_end); + + address += size_to_end; + sgl_size -= size_to_end; + sgl_offset += size_to_end; + if (sgl_offset == wq->queue_size) + sgl_offset = 0; + } + + memcpy(mana_gd_ring_ptr(wq, sgl_offset), address, sgl_size); +} + +int mana_gd_post_work_request(struct gdma_queue *wq, + const struct gdma_wqe_request *wqe_req, + struct gdma_posted_wqe_info *wqe_info) +{ + u32 client_oob_size = wqe_req->inline_oob_size; + u32 sgl_data_size; + u32 max_wqe_size; + u32 wqe_offset; + u32 sgl_offset; + u32 wqe_size; + u32 oob_len; + u8 *wqe_ptr; + u32 head; + + if (wqe_req->num_sge == 0) + return -EINVAL; + + if (wq->type == GDMA_RQ) { + if (client_oob_size != 0) + return -EINVAL; + + client_oob_size = INLINE_OOB_SMALL_SIZE; + + max_wqe_size = GDMA_MAX_RQE_SIZE; + } else { + if (client_oob_size != INLINE_OOB_SMALL_SIZE && + client_oob_size != INLINE_OOB_LARGE_SIZE) + return -EINVAL; + + max_wqe_size = GDMA_MAX_SQE_SIZE; + } + + sgl_data_size = sizeof(struct gdma_sge) * wqe_req->num_sge; + wqe_size = ALIGN(sizeof(struct gdma_wqe) + client_oob_size + + sgl_data_size, GDMA_WQE_BU_SIZE); + if (wqe_size > max_wqe_size) + return -EINVAL; + + if (wq->monitor_avl_buf && wqe_size > mana_gd_wq_avail_space(wq)) + return -ENOSPC; + + if (wqe_info) + wqe_info->wqe_size_in_bu = wqe_size / GDMA_WQE_BU_SIZE; + + head = wq->head; + wqe_offset = (head * GDMA_WQE_BU_SIZE) & (wq->queue_size - 1); + wqe_ptr = mana_gd_get_wqe_ptr(wq, head); + oob_len = mana_gd_write_client_oob(wqe_req, wq->type, client_oob_size, + sgl_data_size, wqe_ptr); + + sgl_offset = wqe_offset + oob_len; + if (sgl_offset >= wq->queue_size) + sgl_offset -= wq->queue_size; + + mana_gd_write_sgl(wq, sgl_offset, wqe_req); + + wq->head += wqe_size / GDMA_WQE_BU_SIZE; + + return 0; +} +EXPORT_SYMBOL_NS(mana_gd_post_work_request, "NET_MANA"); + +int mana_gd_post_and_ring(struct gdma_queue *queue, + const struct gdma_wqe_request *wqe_req, + struct gdma_posted_wqe_info *wqe_info) +{ + struct gdma_context *gc = queue->gdma_dev->gdma_context; + int err; + + err = mana_gd_post_work_request(queue, wqe_req, wqe_info); + if (err) { + dev_err(gc->dev, "Failed to post work req from queue type %d of size %u (err=%d)\n", + queue->type, queue->queue_size, err); + return err; + } + + mana_gd_wq_ring_doorbell(gc, queue); + + return 0; +} + +static int mana_gd_read_cqe(struct gdma_queue *cq, struct gdma_comp *comp) +{ + unsigned int num_cqe = cq->queue_size / sizeof(struct gdma_cqe); + u32 owner_bits, new_bits, old_bits; + struct gdma_cqe *cqe; + + cqe = mana_gd_ring_ptr(cq, (cq->head % num_cqe) * sizeof(*cqe)); + owner_bits = cqe->cqe_info.owner_bits; + + old_bits = (cq->head / num_cqe - 1) & GDMA_CQE_OWNER_MASK; + /* Return 0 if no more entries. */ + if (owner_bits == old_bits) + return 0; + + new_bits = (cq->head / num_cqe) & GDMA_CQE_OWNER_MASK; + /* Return -1 if overflow detected. */ + if (WARN_ON_ONCE(owner_bits != new_bits)) + return -1; + + /* Per GDMA spec, rmb is necessary after checking owner_bits, before + * reading completion info + */ + rmb(); + + comp->wq_num = cqe->cqe_info.wq_num; + comp->is_sq = cqe->cqe_info.is_sq; + memcpy(comp->cqe_data, cqe->cqe_data, GDMA_COMP_DATA_SIZE); + + return 1; +} + +int mana_gd_poll_cq(struct gdma_queue *cq, struct gdma_comp *comp, int num_cqe) +{ + int cqe_idx; + int ret; + + for (cqe_idx = 0; cqe_idx < num_cqe; cqe_idx++) { + ret = mana_gd_read_cqe(cq, &comp[cqe_idx]); + + if (ret < 0) { + cq->head -= cqe_idx; + return ret; + } + + if (ret == 0) + break; + + cq->head++; + } + + return cqe_idx; +} +EXPORT_SYMBOL_NS(mana_gd_poll_cq, "NET_MANA"); + +static irqreturn_t mana_gd_intr(int irq, void *arg) +{ + struct gdma_irq_context *gic = arg; + struct list_head *eq_list = &gic->eq_list; + struct gdma_queue *eq; + + rcu_read_lock(); + list_for_each_entry_rcu(eq, eq_list, entry) { + gic->handler(eq); + } + rcu_read_unlock(); + + return IRQ_HANDLED; +} + +void mana_gd_put_gic(struct gdma_context *gc, bool use_msi_bitmap, int msi) +{ + struct pci_dev *dev = to_pci_dev(gc->dev); + struct gdma_irq_context *gic; + struct msi_map irq_map; + int irq; + + mutex_lock(&gc->gic_mutex); + + gic = xa_load(&gc->irq_contexts, msi); + if (WARN_ON(!gic)) { + mutex_unlock(&gc->gic_mutex); + return; + } + + if (use_msi_bitmap) + gic->bitmap_refs--; + + if (use_msi_bitmap && gic->bitmap_refs == 0) + clear_bit(msi, gc->msi_bitmap); + + if (!refcount_dec_and_test(&gic->refcount)) + goto out; + + irq = gic->irq; + + irq_update_affinity_hint(irq, NULL); + free_irq(irq, gic); + + if (gic->dyn_msix) { + irq_map.virq = irq; + irq_map.index = msi; + pci_msix_free_irq(dev, irq_map); + } + + xa_erase(&gc->irq_contexts, msi); + kfree(gic); + +out: + mutex_unlock(&gc->gic_mutex); +} +EXPORT_SYMBOL_NS(mana_gd_put_gic, "NET_MANA"); + +/* + * Get a GIC (GDMA IRQ Context) on a MSI vector + * a MSI can be shared between different EQs, this function supports setting + * up separate MSIs using a bitmap, or directly using the MSI index + * + * @use_msi_bitmap: + * True if MSI is assigned by this function on available slots from bitmap. + * False if MSI is passed from *msi_requested + */ +struct gdma_irq_context *mana_gd_get_gic(struct gdma_context *gc, + bool use_msi_bitmap, + int *msi_requested) +{ + struct pci_dev *dev = to_pci_dev(gc->dev); + struct gdma_irq_context *gic; + struct msi_map irq_map = { }; + int irq; + int msi; + int err; + + mutex_lock(&gc->gic_mutex); + + if (use_msi_bitmap) { + msi = find_first_zero_bit(gc->msi_bitmap, gc->num_msix_usable); + if (msi >= gc->num_msix_usable) { + dev_err(gc->dev, "No free MSI vectors available\n"); + gic = ERR_PTR(-ENOSPC); + goto out; + } + *msi_requested = msi; + } else { + msi = *msi_requested; + } + + gic = xa_load(&gc->irq_contexts, msi); + if (gic) { + refcount_inc(&gic->refcount); + if (use_msi_bitmap) { + gic->bitmap_refs++; + set_bit(msi, gc->msi_bitmap); + } + goto out; + } + + irq = pci_irq_vector(dev, msi); + if (irq == -EINVAL) { + irq_map = pci_msix_alloc_irq_at(dev, msi, NULL); + if (!irq_map.virq) { + err = irq_map.index; + dev_err(gc->dev, + "Failed to alloc irq_map msi %d err %d\n", + msi, err); + gic = ERR_PTR(err); + goto out; + } + irq = irq_map.virq; + msi = irq_map.index; + *msi_requested = msi; + } + + gic = kzalloc(sizeof(*gic), GFP_KERNEL); + if (!gic) { + gic = ERR_PTR(-ENOMEM); + if (irq_map.virq) + pci_msix_free_irq(dev, irq_map); + goto out; + } + + gic->handler = mana_gd_process_eq_events; + gic->msi = msi; + gic->irq = irq; + INIT_LIST_HEAD(&gic->eq_list); + spin_lock_init(&gic->lock); + + if (!gic->msi) + snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_hwc@pci:%s", + pci_name(dev)); + else + snprintf(gic->name, MANA_IRQ_NAME_SZ, "mana_msi%d@pci:%s", + gic->msi, pci_name(dev)); + + err = request_irq(irq, mana_gd_intr, 0, gic->name, gic); + if (err) { + dev_err(gc->dev, "Failed to request irq %d %s\n", + irq, gic->name); + kfree(gic); + gic = ERR_PTR(err); + if (irq_map.virq) + pci_msix_free_irq(dev, irq_map); + goto out; + } + + gic->dyn_msix = !!irq_map.virq; + refcount_set(&gic->refcount, 1); + gic->bitmap_refs = use_msi_bitmap ? 1 : 0; + + err = xa_err(xa_store(&gc->irq_contexts, msi, gic, GFP_KERNEL)); + if (err) { + dev_err(gc->dev, "Failed to store irq context for msi %d: %d\n", + msi, err); + free_irq(irq, gic); + kfree(gic); + gic = ERR_PTR(err); + if (irq_map.virq) + pci_msix_free_irq(dev, irq_map); + goto out; + } + + if (use_msi_bitmap) + set_bit(msi, gc->msi_bitmap); + +out: + mutex_unlock(&gc->gic_mutex); + return gic; +} +EXPORT_SYMBOL_NS(mana_gd_get_gic, "NET_MANA"); + +int mana_gd_alloc_res_map(u32 res_avail, struct gdma_resource *r) +{ + r->map = bitmap_zalloc(res_avail, GFP_KERNEL); + if (!r->map) + return -ENOMEM; + + r->size = res_avail; + spin_lock_init(&r->lock); + + return 0; +} + +void mana_gd_free_res_map(struct gdma_resource *r) +{ + bitmap_free(r->map); + r->map = NULL; + r->size = 0; +} + +/* + * Spread on CPUs with the following heuristics: + * + * 1. No more than one IRQ per CPU, if possible; + * 2. NUMA locality is the second priority; + * 3. Sibling dislocality is the last priority. + * + * Let's consider this topology: + * + * Node 0 1 + * Core 0 1 2 3 + * CPU 0 1 2 3 4 5 6 7 + * + * The most performant IRQ distribution based on the above topology + * and heuristics may look like this: + * + * IRQ Nodes Cores CPUs + * 0 1 0 0-1 + * 1 1 1 2-3 + * 2 1 0 0-1 + * 3 1 1 2-3 + * 4 2 2 4-5 + * 5 2 3 6-7 + * 6 2 2 4-5 + * 7 2 3 6-7 + * + * The heuristics is implemented as follows. + * + * The outer for_each() loop resets the 'weight' to the actual number + * of CPUs in the hop. Then inner for_each() loop decrements it by the + * number of sibling groups (cores) while assigning first set of IRQs + * to each group. IRQs 0 and 1 above are distributed this way. + * + * Now, because NUMA locality is more important, we should walk the + * same set of siblings and assign 2nd set of IRQs (2 and 3), and it's + * implemented by the medium while() loop. We do like this unless the + * number of IRQs assigned on this hop will not become equal to number + * of CPUs in the hop (weight == 0). Then we switch to the next hop and + * do the same thing. + */ + +static int mana_irq_setup_numa_aware(unsigned int *irqs, unsigned int len, + int node, bool skip_first_cpu) +{ + const struct cpumask *next, *prev = cpu_none_mask; + cpumask_var_t cpus __free(free_cpumask_var); + int cpu, weight; + + if (!alloc_cpumask_var(&cpus, GFP_KERNEL)) + return -ENOMEM; + + rcu_read_lock(); + for_each_numa_hop_mask(next, node) { + weight = cpumask_weight_andnot(next, prev); + while (weight > 0) { + cpumask_andnot(cpus, next, prev); + for_each_cpu(cpu, cpus) { + cpumask_andnot(cpus, cpus, topology_sibling_cpumask(cpu)); + --weight; + + if (unlikely(skip_first_cpu)) { + skip_first_cpu = false; + continue; + } + + if (len-- == 0) + goto done; + + irq_set_affinity_and_hint(*irqs++, topology_sibling_cpumask(cpu)); + } + } + prev = next; + } +done: + rcu_read_unlock(); + return 0; +} + +/* must be called with cpus_read_lock() held */ +static void mana_irq_setup_linear(unsigned int *irqs, unsigned int len) +{ + int cpu; + + for_each_online_cpu(cpu) { + if (len == 0) + break; + + irq_set_affinity_and_hint(*irqs++, cpumask_of(cpu)); + len--; + } +} + +static int mana_gd_setup_dyn_irqs(struct pci_dev *pdev, int nvec) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_irq_context *gic; + int *irqs, err, i, msi; + + irqs = kmalloc_objs(int, nvec); + if (!irqs) + return -ENOMEM; + + /* + * In this function, num_msix_usable = HWC IRQ + Queue IRQ. + * nvec is only Queue IRQ (HWC already setup). + * While processing the next pci irq vector, we start with index 1, + * as IRQ vector at index 0 is already processed for HWC. + * However, the population of irqs array starts with index 0, to be + * further used in mana_irq_setup_numa_aware() + */ + for (i = 1; i <= nvec; i++) { + msi = i; + gic = mana_gd_get_gic(gc, false, &msi); + if (IS_ERR(gic)) { + err = PTR_ERR(gic); + goto free_irq; + } + + irqs[i - 1] = gic->irq; + } + + /* + * When calling mana_irq_setup_numa_aware() for dynamically added IRQs, + * if number of CPUs is more than or equal to allocated MSI-X, we need to + * skip the first CPU sibling group since they are already affinitized to + * HWC IRQ + */ + cpus_read_lock(); + if (gc->num_msix_usable <= num_online_cpus()) { + err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, + true); + if (err) { + cpus_read_unlock(); + goto free_irq; + } + } else { + /* + * When num_msix_usable are more than num_online_cpus, our + * queue IRQs should be equal to num of online vCPUs. + * We try to make sure queue IRQs spread across all vCPUs. + * In such a case NUMA or CPU core affinity does not matter. + * Note: in this case the total mana IRQ should always be + * num_online_cpus + 1. The first HWC IRQ is already handled + * in HWC setup calls + * However, if CPUs went offline since num_msix_usable was + * computed, queue IRQs will be more than num_online_cpus(). + * In such cases remaining extra IRQs will retain their default + * affinity. + */ + int first_unassigned = num_online_cpus(); + + if (nvec > first_unassigned) { + char buf[32]; + + if (first_unassigned == nvec - 1) + snprintf(buf, sizeof(buf), "%d", + first_unassigned); + else + snprintf(buf, sizeof(buf), "%d-%d", + first_unassigned, nvec - 1); + + dev_dbg(&pdev->dev, + "MANA IRQ indices #%s will retain the default CPU affinity\n", + buf); + } + + mana_irq_setup_linear(irqs, nvec); + } + + cpus_read_unlock(); + kfree(irqs); + return 0; + +free_irq: + for (i -= 1; i > 0; i--) + mana_gd_put_gic(gc, false, i); + kfree(irqs); + return err; +} + +static int mana_gd_setup_irqs(struct pci_dev *pdev, int nvec) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct gdma_irq_context *gic; + int *irqs, *start_irqs; + unsigned int cpu; + int err, i, msi; + + irqs = kmalloc_objs(int, nvec); + if (!irqs) + return -ENOMEM; + + start_irqs = irqs; + + for (i = 0; i < nvec; i++) { + msi = i; + gic = mana_gd_get_gic(gc, false, &msi); + if (IS_ERR(gic)) { + err = PTR_ERR(gic); + goto free_irq; + } + + irqs[i] = gic->irq; + } + + /* If number of IRQ is one extra than number of online CPUs, + * then we need to assign IRQ0 (hwc irq) and IRQ1 to + * same CPU. + * Else we will use different CPUs for IRQ0 and IRQ1. + * Also we are using cpumask_local_spread instead of + * cpumask_first for the node, because the node can be + * mem only. + */ + cpus_read_lock(); + if (nvec > num_online_cpus()) { + cpu = cpumask_local_spread(0, gc->numa_node); + irq_set_affinity_and_hint(irqs[0], cpumask_of(cpu)); + irqs++; + nvec -= 1; + } + + err = mana_irq_setup_numa_aware(irqs, nvec, gc->numa_node, false); + if (err) { + cpus_read_unlock(); + goto free_irq; + } + + cpus_read_unlock(); + kfree(start_irqs); + return 0; + +free_irq: + for (i -= 1; i >= 0; i--) + mana_gd_put_gic(gc, false, i); + + kfree(start_irqs); + return err; +} + +static int mana_gd_setup_hwc_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + unsigned int max_irqs, min_irqs; + int nvec, err; + + if (pci_msix_can_alloc_dyn(pdev)) { + max_irqs = 1; + min_irqs = 1; + } else { + /* Need 1 interrupt for HWC */ + max_irqs = min(num_online_cpus(), MANA_MAX_NUM_QUEUES) + 1; + min_irqs = 2; + gc->msi_sharing = true; + } + + nvec = pci_alloc_irq_vectors(pdev, min_irqs, max_irqs, PCI_IRQ_MSIX); + if (nvec < 0) + return nvec; + + err = mana_gd_setup_irqs(pdev, nvec); + if (err) { + pci_free_irq_vectors(pdev); + return err; + } + + gc->num_msix_usable = nvec; + gc->max_num_msix = nvec; + + return 0; +} + +static int mana_gd_setup_remaining_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + struct msi_map irq_map; + int max_irqs, i, err; + + if (!pci_msix_can_alloc_dyn(pdev)) + /* remain irqs are already allocated with HWC IRQ */ + return 0; + + /* allocate only remaining IRQs*/ + max_irqs = gc->num_msix_usable - 1; + + for (i = 1; i <= max_irqs; i++) { + irq_map = pci_msix_alloc_irq_at(pdev, i, NULL); + if (!irq_map.virq) { + err = irq_map.index; + /* caller will handle cleaning up all allocated + * irqs, after HWC is destroyed + */ + return err; + } + } + + err = mana_gd_setup_dyn_irqs(pdev, max_irqs); + if (err) + return err; + + gc->max_num_msix = gc->max_num_msix + max_irqs; + + return 0; +} + +static void mana_gd_remove_irqs(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + int i; + + if (gc->max_num_msix < 1) + return; + + for (i = 0; i < gc->max_num_msix; i++) { + if (!xa_load(&gc->irq_contexts, i)) + continue; + + mana_gd_put_gic(gc, false, i); + } + + WARN_ON(!xa_empty(&gc->irq_contexts)); + + pci_free_irq_vectors(pdev); + + bitmap_free(gc->msi_bitmap); + gc->msi_bitmap = NULL; + gc->max_num_msix = 0; + gc->num_msix_usable = 0; +} + +static int mana_gd_setup(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + int err; + + gc->mana_pci_debugfs = debugfs_create_dir(pci_name(pdev), + mana_debugfs_root); + + err = mana_gd_init_registers(pdev); + if (err) + goto remove_debugfs; + + mana_smc_init(&gc->shm_channel, gc->dev, gc->shm_base); + + gc->service_wq = alloc_ordered_workqueue("gdma_service_wq", 0); + if (!gc->service_wq) { + err = -ENOMEM; + goto remove_debugfs; + } + + err = mana_gd_setup_hwc_irqs(pdev); + if (err) { + dev_err(gc->dev, "Failed to setup IRQs for HWC creation: %d\n", + err); + goto free_workqueue; + } + + err = mana_hwc_create_channel(gc); + if (err) + goto remove_irq; + + err = mana_gd_verify_vf_version(pdev); + if (err) + goto destroy_hwc; + + err = mana_gd_detect_devices(pdev); + if (err) + goto destroy_hwc; + + err = mana_gd_query_max_resources(pdev); + if (err) + goto destroy_hwc; + + err = mana_gd_setup_remaining_irqs(pdev); + if (err) { + dev_err(gc->dev, "Failed to setup remaining IRQs: %d", err); + goto destroy_hwc; + } + + if (!gc->msi_sharing) { + gc->msi_bitmap = bitmap_zalloc(gc->num_msix_usable, GFP_KERNEL); + if (!gc->msi_bitmap) { + err = -ENOMEM; + goto destroy_hwc; + } + /* Set bit for HWC */ + set_bit(0, gc->msi_bitmap); + } + + dev_dbg(&pdev->dev, "mana gdma setup successful\n"); + return 0; + +destroy_hwc: + mana_hwc_destroy_channel(gc); +remove_irq: + mana_gd_remove_irqs(pdev); +free_workqueue: + destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; +remove_debugfs: + debugfs_remove_recursive(gc->mana_pci_debugfs); + gc->mana_pci_debugfs = NULL; + dev_err(&pdev->dev, "%s failed (error %d)\n", __func__, err); + return err; +} + +static void mana_gd_cleanup_device(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + mana_hwc_destroy_channel(gc); + + mana_gd_remove_irqs(pdev); + + if (gc->service_wq) { + destroy_workqueue(gc->service_wq); + gc->service_wq = NULL; + } + + debugfs_remove_recursive(gc->mana_pci_debugfs); + gc->mana_pci_debugfs = NULL; + + dev_dbg(&pdev->dev, "mana gdma cleanup successful\n"); +} + +static bool mana_is_pf(unsigned short dev_id) +{ + return dev_id == MANA_PF_DEVICE_ID || dev_id == MANA_PF2_DEVICE_ID; +} + +static int mana_gd_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + struct gdma_context *gc; + void __iomem *bar0_va; + int bar = 0; + int err; + + /* Each port has 2 CQs, each CQ has at most 1 EQE at a time */ + BUILD_BUG_ON(2 * MAX_PORTS_IN_MANA_DEV * GDMA_EQE_SIZE > EQ_SIZE); + + err = pci_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "Failed to enable pci device (err=%d)\n", err); + return -ENXIO; + } + + pci_set_master(pdev); + + err = pci_request_regions(pdev, "mana"); + if (err) + goto disable_dev; + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + if (err) { + dev_err(&pdev->dev, "DMA set mask failed: %d\n", err); + goto release_region; + } + dma_set_max_seg_size(&pdev->dev, UINT_MAX); + + err = -ENOMEM; + gc = vzalloc(sizeof(*gc)); + if (!gc) + goto release_region; + + mutex_init(&gc->eq_test_event_mutex); + mutex_init(&gc->gic_mutex); + pci_set_drvdata(pdev, gc); + gc->bar0_pa = pci_resource_start(pdev, 0); + gc->bar0_size = pci_resource_len(pdev, 0); + + bar0_va = pci_iomap(pdev, bar, 0); + if (!bar0_va) + goto free_gc; + + gc->numa_node = dev_to_node(&pdev->dev); + gc->is_pf = mana_is_pf(pdev->device); + gc->is_pf2 = (pdev->device == MANA_PF2_DEVICE_ID); + + gc->bar0_va = bar0_va; + gc->dev = &pdev->dev; + xa_init(&gc->irq_contexts); + + err = mana_gd_setup(pdev); + if (err) + goto unmap_bar; + + err = mana_probe(&gc->mana, false); + if (err) + goto cleanup_gd; + + err = mana_rdma_probe(&gc->mana_ib); + if (err) + goto cleanup_mana; + + /* + * If a hardware reset event has occurred over HWC during probe, + * rollback and perform hardware reset procedure. + */ + if (test_and_set_bit(GC_PROBE_SUCCEEDED, &gc->flags)) { + err = -EPROTO; + goto cleanup_mana_rdma; + } + + return 0; + +cleanup_mana_rdma: + mana_rdma_remove(&gc->mana_ib); +cleanup_mana: + mana_remove(&gc->mana, false); +cleanup_gd: + mana_gd_cleanup_device(pdev); +unmap_bar: + xa_destroy(&gc->irq_contexts); + pci_iounmap(pdev, bar0_va); +free_gc: + pci_set_drvdata(pdev, NULL); + vfree(gc); +release_region: + pci_release_regions(pdev); +disable_dev: + pci_disable_device(pdev); + dev_err(&pdev->dev, "gdma probe failed: err = %d\n", err); + + /* + * Hardware could be in recovery mode and the HWC returns TIMEDOUT or + * EPROTO from mana_gd_setup(), mana_probe() or mana_rdma_probe(), or + * we received a hardware reset event over HWC interrupt. In this case, + * perform the device recovery procedure after MANA_SERVICE_PERIOD + * seconds. + */ + if (err == -ETIMEDOUT || err == -EPROTO) { + struct mana_dev_recovery *dev; + unsigned long flags; + + dev_info(&pdev->dev, "Start MANA recovery mode\n"); + + dev = kzalloc_obj(*dev); + if (!dev) + return err; + + dev->pdev = pci_dev_get(pdev); + dev->type = GDMA_EQE_HWC_RESET_REQUEST; + + spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); + list_add_tail(&dev->list, &mana_dev_recovery_work.dev_list); + spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); + + schedule_delayed_work(&mana_dev_recovery_work.work, + secs_to_jiffies(MANA_SERVICE_PERIOD)); + } + + return err; +} + +static void mana_gd_remove(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + pci_disable_sriov(pdev); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, false); + + mana_gd_cleanup_device(pdev); + + xa_destroy(&gc->irq_contexts); + + pci_iounmap(pdev, gc->bar0_va); + + vfree(gc); + + pci_release_regions(pdev); + pci_disable_device(pdev); + + dev_dbg(&pdev->dev, "mana gdma remove successful\n"); +} + +/* The 'state' parameter is not used. */ +int mana_gd_suspend(struct pci_dev *pdev, pm_message_t state) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, true); + + mana_gd_cleanup_device(pdev); + + return 0; +} + +int mana_gd_resume(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + int err; + + err = mana_gd_setup(pdev); + if (err) + return err; + + err = mana_probe(&gc->mana, true); + if (err) + goto cleanup_gd; + + err = mana_rdma_probe(&gc->mana_ib); + if (err) + mana_rdma_remove(&gc->mana_ib); + + return err; + +cleanup_gd: + mana_gd_cleanup_device(pdev); + return err; +} + +/* Quiesce the device for kexec. This is also called upon reboot/shutdown. */ +static void mana_gd_shutdown(struct pci_dev *pdev) +{ + struct gdma_context *gc = pci_get_drvdata(pdev); + + dev_info(&pdev->dev, "Shutdown was called\n"); + + mana_rdma_remove(&gc->mana_ib); + mana_remove(&gc->mana, true); + + mana_gd_cleanup_device(pdev); + + pci_disable_device(pdev); +} + +static int mana_sriov_configure(struct pci_dev *pdev, int numvfs) +{ + int err = 0; + + dev_info(&pdev->dev, "Requested num VFs: %d\n", numvfs); + + if (numvfs > 0) { + err = pci_enable_sriov(pdev, numvfs); + } else { + if (pci_vfs_assigned(pdev)) { + dev_warn(&pdev->dev, + "Cannot disable SR-IOV while VFs are assigned\n"); + return -EPERM; + } + + pci_disable_sriov(pdev); + } + + return err ? err : numvfs; +} + +static const struct pci_device_id mana_id_table[] = { + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_PF2_DEVICE_ID) }, + { PCI_DEVICE(PCI_VENDOR_ID_MICROSOFT, MANA_VF_DEVICE_ID) }, + { } +}; + +static struct pci_driver mana_driver = { + .name = "mana", + .id_table = mana_id_table, + .probe = mana_gd_probe, + .remove = mana_gd_remove, + .suspend = mana_gd_suspend, + .resume = mana_gd_resume, + .shutdown = mana_gd_shutdown, + .sriov_configure = mana_sriov_configure, +}; + +static int __init mana_driver_init(void) +{ + int err; + + INIT_LIST_HEAD(&mana_dev_recovery_work.dev_list); + spin_lock_init(&mana_dev_recovery_work.lock); + INIT_DELAYED_WORK(&mana_dev_recovery_work.work, mana_recovery_delayed_func); + + mana_debugfs_root = debugfs_create_dir("mana", NULL); + + err = pci_register_driver(&mana_driver); + if (err) { + debugfs_remove(mana_debugfs_root); + mana_debugfs_root = NULL; + } + + return err; +} + +static void __exit mana_driver_exit(void) +{ + struct mana_dev_recovery *dev; + unsigned long flags; + + disable_delayed_work_sync(&mana_dev_recovery_work.work); + + spin_lock_irqsave(&mana_dev_recovery_work.lock, flags); + while (!list_empty(&mana_dev_recovery_work.dev_list)) { + dev = list_first_entry(&mana_dev_recovery_work.dev_list, + struct mana_dev_recovery, list); + list_del(&dev->list); + pci_dev_put(dev->pdev); + kfree(dev); + } + spin_unlock_irqrestore(&mana_dev_recovery_work.lock, flags); + + pci_unregister_driver(&mana_driver); + + debugfs_remove(mana_debugfs_root); + + mana_debugfs_root = NULL; +} + +module_init(mana_driver_init); +module_exit(mana_driver_exit); + +MODULE_DEVICE_TABLE(pci, mana_id_table); + +MODULE_LICENSE("Dual BSD/GPL"); +MODULE_DESCRIPTION("Microsoft Azure Network Adapter driver"); diff --git a/drivers/net/ethernet/microsoft/mana/hw_channel.c b/drivers/net/ethernet/microsoft/mana/hw_channel.c new file mode 100644 index 000000000..263e7c4e2 --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/hw_channel.c @@ -0,0 +1,937 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include +#include +#include +#include + +static int mana_hwc_get_msg_index(struct hw_channel_context *hwc, u16 *msg_id) +{ + struct gdma_resource *r = &hwc->inflight_msg_res; + unsigned long flags; + u32 index; + + down(&hwc->sema); + + spin_lock_irqsave(&r->lock, flags); + + index = find_first_zero_bit(hwc->inflight_msg_res.map, + hwc->inflight_msg_res.size); + + bitmap_set(hwc->inflight_msg_res.map, index, 1); + + spin_unlock_irqrestore(&r->lock, flags); + + *msg_id = index; + + return 0; +} + +static void mana_hwc_put_msg_index(struct hw_channel_context *hwc, u16 msg_id) +{ + struct gdma_resource *r = &hwc->inflight_msg_res; + unsigned long flags; + + spin_lock_irqsave(&r->lock, flags); + bitmap_clear(hwc->inflight_msg_res.map, msg_id, 1); + spin_unlock_irqrestore(&r->lock, flags); + + up(&hwc->sema); +} + +static int mana_hwc_verify_resp_msg(const struct hwc_caller_ctx *caller_ctx, + const struct gdma_resp_hdr *resp_msg, + u32 resp_len) +{ + if (resp_len < sizeof(*resp_msg)) + return -EPROTO; + + if (resp_len > caller_ctx->output_buflen) + return -EPROTO; + + return 0; +} + +static int mana_hwc_post_rx_wqe(const struct hwc_wq *hwc_rxq, + struct hwc_work_request *req) +{ + struct device *dev = hwc_rxq->hwc->dev; + struct gdma_sge *sge; + int err; + + sge = &req->sge; + sge->address = (u64)req->buf_sge_addr; + sge->mem_key = hwc_rxq->msg_buf->gpa_mkey; + sge->size = req->buf_len; + + memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); + req->wqe_req.sgl = sge; + req->wqe_req.num_sge = 1; + req->wqe_req.client_data_unit = 0; + + err = mana_gd_post_and_ring(hwc_rxq->gdma_wq, &req->wqe_req, NULL); + if (err) + dev_err(dev, "Failed to post WQE on HWC RQ: %d\n", err); + return err; +} + +static void mana_hwc_handle_resp(struct hw_channel_context *hwc, u32 resp_len, + struct hwc_work_request *rx_req, u16 msg_id) +{ + const struct gdma_resp_hdr *resp_msg = rx_req->buf_va; + struct hwc_caller_ctx *ctx; + int err; + + if (!test_bit(msg_id, hwc->inflight_msg_res.map)) { + dev_err(hwc->dev, "hwc_rx: invalid msg_id = %u\n", msg_id); + mana_hwc_post_rx_wqe(hwc->rxq, rx_req); + return; + } + + ctx = hwc->caller_ctx + msg_id; + err = mana_hwc_verify_resp_msg(ctx, resp_msg, resp_len); + if (err) + goto out; + + ctx->status_code = resp_msg->status; + + memcpy(ctx->output_buf, resp_msg, resp_len); +out: + ctx->error = err; + + /* Must post rx wqe before complete(), otherwise the next rx may + * hit no_wqe error. + */ + mana_hwc_post_rx_wqe(hwc->rxq, rx_req); + + complete(&ctx->comp_event); +} + +static void mana_hwc_init_event_handler(void *ctx, struct gdma_queue *q_self, + struct gdma_event *event) +{ + union hwc_init_soc_service_type service_data; + struct hw_channel_context *hwc = ctx; + struct gdma_dev *gd = hwc->gdma_dev; + union hwc_init_type_data type_data; + union hwc_init_eq_id_db eq_db; + struct mana_context *ac; + u32 type, val; + int ret; + + switch (event->type) { + case GDMA_EQE_HWC_INIT_EQ_ID_DB: + eq_db.as_uint32 = event->details[0]; + hwc->cq->gdma_eq->id = eq_db.eq_id; + gd->doorbell = eq_db.doorbell; + break; + + case GDMA_EQE_HWC_INIT_DATA: + type_data.as_uint32 = event->details[0]; + type = type_data.type; + val = type_data.value; + + switch (type) { + case HWC_INIT_DATA_CQID: + hwc->cq->gdma_cq->id = val; + break; + + case HWC_INIT_DATA_RQID: + hwc->rxq->gdma_wq->id = val; + break; + + case HWC_INIT_DATA_SQID: + hwc->txq->gdma_wq->id = val; + break; + + case HWC_INIT_DATA_QUEUE_DEPTH: + hwc->hwc_init_q_depth_max = (u16)val; + break; + + case HWC_INIT_DATA_MAX_REQUEST: + hwc->hwc_init_max_req_msg_size = val; + break; + + case HWC_INIT_DATA_MAX_RESPONSE: + hwc->hwc_init_max_resp_msg_size = val; + break; + + case HWC_INIT_DATA_MAX_NUM_CQS: + gd->gdma_context->max_num_cqs = val; + break; + + case HWC_INIT_DATA_PDID: + hwc->gdma_dev->pdid = val; + break; + + case HWC_INIT_DATA_GPA_MKEY: + hwc->rxq->msg_buf->gpa_mkey = val; + hwc->txq->msg_buf->gpa_mkey = val; + break; + + case HWC_INIT_DATA_PF_DEST_RQ_ID: + hwc->pf_dest_vrq_id = val; + break; + + case HWC_INIT_DATA_PF_DEST_CQ_ID: + hwc->pf_dest_vrcq_id = val; + break; + } + + break; + + case GDMA_EQE_HWC_INIT_DONE: + complete(&hwc->hwc_init_eqe_comp); + break; + + case GDMA_EQE_HWC_SOC_RECONFIG_DATA: + type_data.as_uint32 = event->details[0]; + type = type_data.type; + val = type_data.value; + + switch (type) { + case HWC_DATA_CFG_HWC_TIMEOUT: + hwc->hwc_timeout = val; + break; + + case HWC_DATA_HW_LINK_CONNECT: + case HWC_DATA_HW_LINK_DISCONNECT: + ac = gd->gdma_context->mana.driver_data; + if (!ac) + break; + + WRITE_ONCE(ac->link_event, type); + schedule_work(&ac->link_change_work); + + break; + + default: + dev_warn(hwc->dev, "Received unknown reconfig type %u\n", type); + break; + } + + break; + case GDMA_EQE_HWC_SOC_SERVICE: + service_data.as_uint32 = event->details[0]; + type = service_data.type; + + switch (type) { + case GDMA_SERVICE_TYPE_RDMA_SUSPEND: + case GDMA_SERVICE_TYPE_RDMA_RESUME: + ret = mana_rdma_service_event(gd->gdma_context, type); + if (ret) + dev_err(hwc->dev, "Failed to schedule adev service event: %d\n", + ret); + break; + default: + dev_warn(hwc->dev, "Received unknown SOC service type %u\n", type); + break; + } + + break; + default: + dev_warn(hwc->dev, "Received unknown gdma event %u\n", event->type); + /* Ignore unknown events, which should never happen. */ + break; + } +} + +static void mana_hwc_rx_event_handler(void *ctx, u32 gdma_rxq_id, + const struct hwc_rx_oob *rx_oob) +{ + struct hw_channel_context *hwc = ctx; + struct hwc_wq *hwc_rxq = hwc->rxq; + struct hwc_work_request *rx_req; + struct gdma_resp_hdr *resp; + struct gdma_wqe *dma_oob; + struct gdma_queue *rq; + struct gdma_sge *sge; + u64 rq_base_addr; + u64 rx_req_idx; + u16 msg_id; + u8 *wqe; + + if (WARN_ON_ONCE(hwc_rxq->gdma_wq->id != gdma_rxq_id)) + return; + + rq = hwc_rxq->gdma_wq; + wqe = mana_gd_get_wqe_ptr(rq, rx_oob->wqe_offset / GDMA_WQE_BU_SIZE); + dma_oob = (struct gdma_wqe *)wqe; + + sge = (struct gdma_sge *)(wqe + 8 + dma_oob->inline_oob_size_div4 * 4); + + /* Select the RX work request for virtual address and for reposting. */ + rq_base_addr = hwc_rxq->msg_buf->mem_info.dma_handle; + rx_req_idx = (sge->address - rq_base_addr) / hwc->max_req_msg_size; + + if (rx_req_idx >= hwc_rxq->msg_buf->num_reqs) { + dev_err(hwc->dev, "HWC RX: wrong rx_req_idx=%llu, num_reqs=%u\n", + rx_req_idx, hwc_rxq->msg_buf->num_reqs); + return; + } + + rx_req = &hwc_rxq->msg_buf->reqs[rx_req_idx]; + resp = (struct gdma_resp_hdr *)rx_req->buf_va; + + /* Read msg_id once from DMA buffer to prevent TOCTOU: + * DMA memory is shared/unencrypted in CVMs - host can + * modify it between reads. + */ + msg_id = READ_ONCE(resp->response.hwc_msg_id); + if (msg_id >= hwc->num_inflight_msg) { + dev_err(hwc->dev, "HWC RX: wrong msg_id=%u\n", msg_id); + return; + } + + mana_hwc_handle_resp(hwc, rx_oob->tx_oob_data_size, rx_req, msg_id); + + /* Can no longer use 'resp', because the buffer is posted to the HW + * in mana_hwc_handle_resp() above. + */ + resp = NULL; +} + +static void mana_hwc_tx_event_handler(void *ctx, u32 gdma_txq_id, + const struct hwc_rx_oob *rx_oob) +{ + struct hw_channel_context *hwc = ctx; + struct hwc_wq *hwc_txq = hwc->txq; + + WARN_ON_ONCE(!hwc_txq || hwc_txq->gdma_wq->id != gdma_txq_id); +} + +static int mana_hwc_create_gdma_wq(struct hw_channel_context *hwc, + enum gdma_queue_type type, u64 queue_size, + struct gdma_queue **queue) +{ + struct gdma_queue_spec spec = {}; + + if (type != GDMA_SQ && type != GDMA_RQ) + return -EINVAL; + + spec.type = type; + spec.monitor_avl_buf = false; + spec.queue_size = queue_size; + + return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); +} + +static int mana_hwc_create_gdma_cq(struct hw_channel_context *hwc, + u64 queue_size, + void *ctx, gdma_cq_callback *cb, + struct gdma_queue *parent_eq, + struct gdma_queue **queue) +{ + struct gdma_queue_spec spec = {}; + + spec.type = GDMA_CQ; + spec.monitor_avl_buf = false; + spec.queue_size = queue_size; + spec.cq.context = ctx; + spec.cq.callback = cb; + spec.cq.parent_eq = parent_eq; + + return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); +} + +static int mana_hwc_create_gdma_eq(struct hw_channel_context *hwc, + u64 queue_size, + void *ctx, gdma_eq_callback *cb, + struct gdma_queue **queue) +{ + struct gdma_queue_spec spec = {}; + + spec.type = GDMA_EQ; + spec.monitor_avl_buf = false; + spec.queue_size = queue_size; + spec.eq.context = ctx; + spec.eq.callback = cb; + spec.eq.log2_throttle_limit = DEFAULT_LOG2_THROTTLING_FOR_ERROR_EQ; + spec.eq.msix_index = 0; + + return mana_gd_create_hwc_queue(hwc->gdma_dev, &spec, queue); +} + +static void mana_hwc_comp_event(void *ctx, struct gdma_queue *q_self) +{ + struct hwc_rx_oob comp_data = {}; + struct gdma_comp *completions; + struct hwc_cq *hwc_cq = ctx; + int comp_read, i; + + WARN_ON_ONCE(hwc_cq->gdma_cq != q_self); + + completions = hwc_cq->comp_buf; + comp_read = mana_gd_poll_cq(q_self, completions, hwc_cq->queue_depth); + WARN_ON_ONCE(comp_read <= 0 || comp_read > hwc_cq->queue_depth); + + for (i = 0; i < comp_read; ++i) { + comp_data = *(struct hwc_rx_oob *)completions[i].cqe_data; + + if (completions[i].is_sq) + hwc_cq->tx_event_handler(hwc_cq->tx_event_ctx, + completions[i].wq_num, + &comp_data); + else + hwc_cq->rx_event_handler(hwc_cq->rx_event_ctx, + completions[i].wq_num, + &comp_data); + } + + mana_gd_ring_cq(q_self, SET_ARM_BIT); +} + +static void mana_hwc_destroy_cq(struct gdma_context *gc, struct hwc_cq *hwc_cq) +{ + kfree(hwc_cq->comp_buf); + + if (hwc_cq->gdma_cq) + mana_gd_destroy_queue(gc, hwc_cq->gdma_cq); + + if (hwc_cq->gdma_eq) + mana_gd_destroy_queue(gc, hwc_cq->gdma_eq); + + kfree(hwc_cq); +} + +static int mana_hwc_create_cq(struct hw_channel_context *hwc, u16 q_depth, + gdma_eq_callback *callback, void *ctx, + hwc_rx_event_handler_t *rx_ev_hdlr, + void *rx_ev_ctx, + hwc_tx_event_handler_t *tx_ev_hdlr, + void *tx_ev_ctx, struct hwc_cq **hwc_cq_ptr) +{ + struct gdma_queue *eq, *cq; + struct gdma_comp *comp_buf; + struct hwc_cq *hwc_cq; + u32 eq_size, cq_size; + int err; + + eq_size = roundup_pow_of_two(GDMA_EQE_SIZE * q_depth); + if (eq_size < MANA_MIN_QSIZE) + eq_size = MANA_MIN_QSIZE; + + cq_size = roundup_pow_of_two(GDMA_CQE_SIZE * q_depth); + if (cq_size < MANA_MIN_QSIZE) + cq_size = MANA_MIN_QSIZE; + + hwc_cq = kzalloc_obj(*hwc_cq); + if (!hwc_cq) + return -ENOMEM; + + err = mana_hwc_create_gdma_eq(hwc, eq_size, ctx, callback, &eq); + if (err) { + dev_err(hwc->dev, "Failed to create HWC EQ for RQ: %d\n", err); + goto out; + } + hwc_cq->gdma_eq = eq; + + err = mana_hwc_create_gdma_cq(hwc, cq_size, hwc_cq, mana_hwc_comp_event, + eq, &cq); + if (err) { + dev_err(hwc->dev, "Failed to create HWC CQ for RQ: %d\n", err); + goto out; + } + hwc_cq->gdma_cq = cq; + + comp_buf = kzalloc_objs(*comp_buf, q_depth); + if (!comp_buf) { + err = -ENOMEM; + goto out; + } + + hwc_cq->hwc = hwc; + hwc_cq->comp_buf = comp_buf; + hwc_cq->queue_depth = q_depth; + hwc_cq->rx_event_handler = rx_ev_hdlr; + hwc_cq->rx_event_ctx = rx_ev_ctx; + hwc_cq->tx_event_handler = tx_ev_hdlr; + hwc_cq->tx_event_ctx = tx_ev_ctx; + + *hwc_cq_ptr = hwc_cq; + return 0; +out: + mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc_cq); + return err; +} + +static int mana_hwc_alloc_dma_buf(struct hw_channel_context *hwc, u16 q_depth, + u32 max_msg_size, + struct hwc_dma_buf **dma_buf_ptr) +{ + struct gdma_context *gc = hwc->gdma_dev->gdma_context; + struct hwc_work_request *hwc_wr; + struct hwc_dma_buf *dma_buf; + struct gdma_mem_info *gmi; + void *virt_addr; + u32 buf_size; + u8 *base_pa; + int err; + u16 i; + + dma_buf = kzalloc_flex(*dma_buf, reqs, q_depth); + if (!dma_buf) + return -ENOMEM; + + dma_buf->num_reqs = q_depth; + + buf_size = MANA_PAGE_ALIGN(q_depth * max_msg_size); + + gmi = &dma_buf->mem_info; + err = mana_gd_alloc_memory(gc, buf_size, gmi, false); + if (err) { + dev_err(hwc->dev, "Failed to allocate DMA buffer size: %u, err %d\n", + buf_size, err); + goto out; + } + + virt_addr = dma_buf->mem_info.virt_addr; + base_pa = (u8 *)dma_buf->mem_info.dma_handle; + + for (i = 0; i < q_depth; i++) { + hwc_wr = &dma_buf->reqs[i]; + + hwc_wr->buf_va = virt_addr + i * max_msg_size; + hwc_wr->buf_sge_addr = base_pa + i * max_msg_size; + + hwc_wr->buf_len = max_msg_size; + } + + *dma_buf_ptr = dma_buf; + return 0; +out: + kfree(dma_buf); + return err; +} + +static void mana_hwc_dealloc_dma_buf(struct hw_channel_context *hwc, + struct hwc_dma_buf *dma_buf) +{ + if (!dma_buf) + return; + + mana_gd_free_memory(&dma_buf->mem_info); + + kfree(dma_buf); +} + +static void mana_hwc_destroy_wq(struct hw_channel_context *hwc, + struct hwc_wq *hwc_wq) +{ + mana_hwc_dealloc_dma_buf(hwc, hwc_wq->msg_buf); + + if (hwc_wq->gdma_wq) + mana_gd_destroy_queue(hwc->gdma_dev->gdma_context, + hwc_wq->gdma_wq); + + kfree(hwc_wq); +} + +static int mana_hwc_create_wq(struct hw_channel_context *hwc, + enum gdma_queue_type q_type, u16 q_depth, + u32 max_msg_size, struct hwc_cq *hwc_cq, + struct hwc_wq **hwc_wq_ptr) +{ + struct gdma_queue *queue; + struct hwc_wq *hwc_wq; + u32 queue_size; + int err; + + WARN_ON(q_type != GDMA_SQ && q_type != GDMA_RQ); + + if (q_type == GDMA_RQ) + queue_size = roundup_pow_of_two(GDMA_MAX_RQE_SIZE * q_depth); + else + queue_size = roundup_pow_of_two(GDMA_MAX_SQE_SIZE * q_depth); + + if (queue_size < MANA_MIN_QSIZE) + queue_size = MANA_MIN_QSIZE; + + hwc_wq = kzalloc_obj(*hwc_wq); + if (!hwc_wq) + return -ENOMEM; + + err = mana_hwc_create_gdma_wq(hwc, q_type, queue_size, &queue); + if (err) + goto out; + + hwc_wq->hwc = hwc; + hwc_wq->gdma_wq = queue; + hwc_wq->queue_depth = q_depth; + hwc_wq->hwc_cq = hwc_cq; + + err = mana_hwc_alloc_dma_buf(hwc, q_depth, max_msg_size, + &hwc_wq->msg_buf); + if (err) + goto out; + + *hwc_wq_ptr = hwc_wq; + return 0; +out: + if (err) + mana_hwc_destroy_wq(hwc, hwc_wq); + + dev_err(hwc->dev, "Failed to create HWC queue size= %u type= %d err= %d\n", + queue_size, q_type, err); + return err; +} + +static int mana_hwc_post_tx_wqe(const struct hwc_wq *hwc_txq, + struct hwc_work_request *req, + u32 dest_virt_rq_id, u32 dest_virt_rcq_id, + bool dest_pf) +{ + struct device *dev = hwc_txq->hwc->dev; + struct hwc_tx_oob *tx_oob; + struct gdma_sge *sge; + int err; + + if (req->msg_size == 0 || req->msg_size > req->buf_len) { + dev_err(dev, "wrong msg_size: %u, buf_len: %u\n", + req->msg_size, req->buf_len); + return -EINVAL; + } + + tx_oob = &req->tx_oob; + + tx_oob->vrq_id = dest_virt_rq_id; + tx_oob->dest_vfid = 0; + tx_oob->vrcq_id = dest_virt_rcq_id; + tx_oob->vscq_id = hwc_txq->hwc_cq->gdma_cq->id; + tx_oob->loopback = false; + tx_oob->lso_override = false; + tx_oob->dest_pf = dest_pf; + tx_oob->vsq_id = hwc_txq->gdma_wq->id; + + sge = &req->sge; + sge->address = (u64)req->buf_sge_addr; + sge->mem_key = hwc_txq->msg_buf->gpa_mkey; + sge->size = req->msg_size; + + memset(&req->wqe_req, 0, sizeof(struct gdma_wqe_request)); + req->wqe_req.sgl = sge; + req->wqe_req.num_sge = 1; + req->wqe_req.inline_oob_size = sizeof(struct hwc_tx_oob); + req->wqe_req.inline_oob_data = tx_oob; + req->wqe_req.client_data_unit = 0; + + err = mana_gd_post_and_ring(hwc_txq->gdma_wq, &req->wqe_req, NULL); + if (err) + dev_err(dev, "Failed to post WQE on HWC SQ: %d\n", err); + return err; +} + +static int mana_hwc_init_inflight_msg(struct hw_channel_context *hwc, + u16 num_msg) +{ + int err; + + sema_init(&hwc->sema, num_msg); + + err = mana_gd_alloc_res_map(num_msg, &hwc->inflight_msg_res); + if (err) + dev_err(hwc->dev, "Failed to init inflight_msg_res: %d\n", err); + return err; +} + +static int mana_hwc_test_channel(struct hw_channel_context *hwc, u16 q_depth, + u32 max_req_msg_size, u32 max_resp_msg_size) +{ + struct gdma_context *gc = hwc->gdma_dev->gdma_context; + struct hwc_wq *hwc_rxq = hwc->rxq; + struct hwc_work_request *req; + struct hwc_caller_ctx *ctx; + int err; + int i; + + /* Post all WQEs on the RQ */ + for (i = 0; i < q_depth; i++) { + req = &hwc_rxq->msg_buf->reqs[i]; + err = mana_hwc_post_rx_wqe(hwc_rxq, req); + if (err) + return err; + } + + ctx = kzalloc_objs(*ctx, q_depth); + if (!ctx) + return -ENOMEM; + + for (i = 0; i < q_depth; ++i) + init_completion(&ctx[i].comp_event); + + hwc->caller_ctx = ctx; + + return mana_gd_test_eq(gc, hwc->cq->gdma_eq); +} + +static int mana_hwc_establish_channel(struct gdma_context *gc, u16 *q_depth, + u32 *max_req_msg_size, + u32 *max_resp_msg_size) +{ + struct hw_channel_context *hwc = gc->hwc.driver_data; + struct gdma_queue *rq = hwc->rxq->gdma_wq; + struct gdma_queue *sq = hwc->txq->gdma_wq; + struct gdma_queue *eq = hwc->cq->gdma_eq; + struct gdma_queue *cq = hwc->cq->gdma_cq; + int err; + + init_completion(&hwc->hwc_init_eqe_comp); + + err = mana_smc_setup_hwc(&gc->shm_channel, false, + eq->mem_info.dma_handle, + cq->mem_info.dma_handle, + rq->mem_info.dma_handle, + sq->mem_info.dma_handle, + eq->eq.msix_index); + if (err) + return err; + + if (!wait_for_completion_timeout(&hwc->hwc_init_eqe_comp, 60 * HZ)) + return -ETIMEDOUT; + + *q_depth = hwc->hwc_init_q_depth_max; + *max_req_msg_size = hwc->hwc_init_max_req_msg_size; + *max_resp_msg_size = hwc->hwc_init_max_resp_msg_size; + + /* Both were set in mana_hwc_init_event_handler(). */ + if (WARN_ON(cq->id >= gc->max_num_cqs)) + return -EPROTO; + + gc->cq_table = vcalloc(gc->max_num_cqs, sizeof(struct gdma_queue *)); + if (!gc->cq_table) + return -ENOMEM; + + gc->cq_table[cq->id] = cq; + + return 0; +} + +static int mana_hwc_init_queues(struct hw_channel_context *hwc, u16 q_depth, + u32 max_req_msg_size, u32 max_resp_msg_size) +{ + int err; + + err = mana_hwc_init_inflight_msg(hwc, q_depth); + if (err) + return err; + + /* CQ is shared by SQ and RQ, so CQ's queue depth is the sum of SQ + * queue depth and RQ queue depth. + */ + err = mana_hwc_create_cq(hwc, q_depth * 2, + mana_hwc_init_event_handler, hwc, + mana_hwc_rx_event_handler, hwc, + mana_hwc_tx_event_handler, hwc, &hwc->cq); + if (err) { + dev_err(hwc->dev, "Failed to create HWC CQ: %d\n", err); + goto out; + } + + err = mana_hwc_create_wq(hwc, GDMA_RQ, q_depth, max_req_msg_size, + hwc->cq, &hwc->rxq); + if (err) { + dev_err(hwc->dev, "Failed to create HWC RQ: %d\n", err); + goto out; + } + + err = mana_hwc_create_wq(hwc, GDMA_SQ, q_depth, max_resp_msg_size, + hwc->cq, &hwc->txq); + if (err) { + dev_err(hwc->dev, "Failed to create HWC SQ: %d\n", err); + goto out; + } + + hwc->num_inflight_msg = q_depth; + hwc->max_req_msg_size = max_req_msg_size; + + return 0; +out: + /* mana_hwc_create_channel() will do the cleanup.*/ + return err; +} + +int mana_hwc_create_channel(struct gdma_context *gc) +{ + u32 max_req_msg_size, max_resp_msg_size; + struct gdma_dev *gd = &gc->hwc; + struct hw_channel_context *hwc; + u16 q_depth_max; + int err; + + hwc = kzalloc_obj(*hwc); + if (!hwc) + return -ENOMEM; + + gd->gdma_context = gc; + gd->driver_data = hwc; + hwc->gdma_dev = gd; + hwc->dev = gc->dev; + hwc->hwc_timeout = HW_CHANNEL_WAIT_RESOURCE_TIMEOUT_MS; + + /* HWC's instance number is always 0. */ + gd->dev_id.as_uint32 = 0; + gd->dev_id.type = GDMA_DEVICE_HWC; + + gd->pdid = INVALID_PDID; + gd->doorbell = INVALID_DOORBELL; + + /* mana_hwc_init_queues() only creates the required data structures, + * and doesn't touch the HWC device. + */ + err = mana_hwc_init_queues(hwc, HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, + HW_CHANNEL_MAX_REQUEST_SIZE, + HW_CHANNEL_MAX_RESPONSE_SIZE); + if (err) { + dev_err(hwc->dev, "Failed to initialize HWC: %d\n", err); + goto out; + } + + err = mana_hwc_establish_channel(gc, &q_depth_max, &max_req_msg_size, + &max_resp_msg_size); + if (err) { + dev_err(hwc->dev, "Failed to establish HWC: %d\n", err); + goto out; + } + + err = mana_hwc_test_channel(gc->hwc.driver_data, + HW_CHANNEL_VF_BOOTSTRAP_QUEUE_DEPTH, + max_req_msg_size, max_resp_msg_size); + if (err) { + dev_err(hwc->dev, "Failed to test HWC: %d\n", err); + goto out; + } + + return 0; +out: + mana_hwc_destroy_channel(gc); + return err; +} + +void mana_hwc_destroy_channel(struct gdma_context *gc) +{ + struct hw_channel_context *hwc = gc->hwc.driver_data; + + if (!hwc) + return; + + /* gc->max_num_cqs is set in mana_hwc_init_event_handler(). If it's + * non-zero, the HWC worked and we should tear down the HWC here. + */ + if (gc->max_num_cqs > 0) { + mana_smc_teardown_hwc(&gc->shm_channel, false); + gc->max_num_cqs = 0; + } + + if (hwc->txq) + mana_hwc_destroy_wq(hwc, hwc->txq); + + if (hwc->rxq) + mana_hwc_destroy_wq(hwc, hwc->rxq); + + if (hwc->cq) + mana_hwc_destroy_cq(hwc->gdma_dev->gdma_context, hwc->cq); + + kfree(hwc->caller_ctx); + hwc->caller_ctx = NULL; + + mana_gd_free_res_map(&hwc->inflight_msg_res); + + hwc->num_inflight_msg = 0; + + hwc->gdma_dev->doorbell = INVALID_DOORBELL; + hwc->gdma_dev->pdid = INVALID_PDID; + + hwc->hwc_timeout = 0; + + kfree(hwc); + gc->hwc.driver_data = NULL; + gc->hwc.gdma_context = NULL; + + vfree(gc->cq_table); + gc->cq_table = NULL; +} + +int mana_hwc_send_request(struct hw_channel_context *hwc, u32 req_len, + const void *req, u32 resp_len, void *resp) +{ + struct gdma_context *gc = hwc->gdma_dev->gdma_context; + struct hwc_work_request *tx_wr; + struct hwc_wq *txq = hwc->txq; + struct gdma_req_hdr *req_msg; + struct hwc_caller_ctx *ctx; + u32 dest_vrcq = 0; + u32 dest_vrq = 0; + u32 command; + u16 msg_id; + int err; + + mana_hwc_get_msg_index(hwc, &msg_id); + + tx_wr = &txq->msg_buf->reqs[msg_id]; + + if (req_len > tx_wr->buf_len) { + dev_err(hwc->dev, "HWC: req msg size: %d > %d\n", req_len, + tx_wr->buf_len); + err = -EINVAL; + goto out; + } + + ctx = hwc->caller_ctx + msg_id; + ctx->output_buf = resp; + ctx->output_buflen = resp_len; + + req_msg = (struct gdma_req_hdr *)tx_wr->buf_va; + if (req) + memcpy(req_msg, req, req_len); + + req_msg->req.hwc_msg_id = msg_id; + + tx_wr->msg_size = req_len; + command = req_msg->req.msg_type; + + if (gc->is_pf) { + dest_vrq = hwc->pf_dest_vrq_id; + dest_vrcq = hwc->pf_dest_vrcq_id; + } + + err = mana_hwc_post_tx_wqe(txq, tx_wr, dest_vrq, dest_vrcq, false); + if (err) { + dev_err(hwc->dev, "HWC: Failed to post send WQE: %d\n", err); + goto out; + } + + if (!wait_for_completion_timeout(&ctx->comp_event, + (msecs_to_jiffies(hwc->hwc_timeout)))) { + if (hwc->hwc_timeout != 0) + dev_err(hwc->dev, "Command 0x%x timed out: %u ms\n", + command, hwc->hwc_timeout); + + /* Reduce further waiting if HWC no response */ + if (hwc->hwc_timeout > 1) + hwc->hwc_timeout = 1; + + err = -ETIMEDOUT; + goto out; + } + + if (ctx->error) { + err = ctx->error; + goto out; + } + + if (ctx->status_code && ctx->status_code != GDMA_STATUS_MORE_ENTRIES) { + if (ctx->status_code == GDMA_STATUS_CMD_UNSUPPORTED) { + err = -EOPNOTSUPP; + goto out; + } + if (command != MANA_QUERY_PHY_STAT) + dev_err(hwc->dev, "Command 0x%x failed with status: 0x%x\n", + command, ctx->status_code); + err = -EPROTO; + goto out; + } +out: + mana_hwc_put_msg_index(hwc, msg_id); + return err; +} diff --git a/drivers/net/ethernet/microsoft/mana/mana_bpf.c b/drivers/net/ethernet/microsoft/mana/mana_bpf.c new file mode 100644 index 000000000..53308e139 --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/mana_bpf.c @@ -0,0 +1,267 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include +#include +#include +#include +#include +#include + +#include + +void mana_xdp_tx(struct sk_buff *skb, struct net_device *ndev) +{ + u16 txq_idx = skb_get_queue_mapping(skb); + struct netdev_queue *ndevtxq; + int rc; + + __skb_push(skb, ETH_HLEN); + + ndevtxq = netdev_get_tx_queue(ndev, txq_idx); + __netif_tx_lock(ndevtxq, smp_processor_id()); + + rc = mana_start_xmit(skb, ndev); + + __netif_tx_unlock(ndevtxq); + + if (dev_xmit_complete(rc)) + return; + + dev_kfree_skb_any(skb); + ndev->stats.tx_dropped++; +} + +static int mana_xdp_xmit_fm(struct net_device *ndev, struct xdp_frame *frame, + u16 q_idx) +{ + struct sk_buff *skb; + + skb = xdp_build_skb_from_frame(frame, ndev); + if (unlikely(!skb)) + return -ENOMEM; + + skb_set_queue_mapping(skb, q_idx); + + mana_xdp_tx(skb, ndev); + + return 0; +} + +int mana_xdp_xmit(struct net_device *ndev, int n, struct xdp_frame **frames, + u32 flags) +{ + struct mana_port_context *apc = netdev_priv(ndev); + struct mana_stats_tx *tx_stats; + int i, count = 0; + u16 q_idx; + + if (unlikely(!apc->port_is_up)) + return 0; + + q_idx = smp_processor_id() % ndev->real_num_tx_queues; + + for (i = 0; i < n; i++) { + if (mana_xdp_xmit_fm(ndev, frames[i], q_idx)) + break; + + count++; + } + + tx_stats = &apc->tx_qp[q_idx]->txq.stats; + + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->xdp_xmit += count; + u64_stats_update_end(&tx_stats->syncp); + + return count; +} + +u32 mana_run_xdp(struct net_device *ndev, struct mana_rxq *rxq, + struct xdp_buff *xdp, void *buf_va, uint pkt_len) +{ + struct mana_stats_rx *rx_stats; + struct bpf_prog *prog; + u32 act = XDP_PASS; + + rcu_read_lock(); + prog = rcu_dereference(rxq->bpf_prog); + + if (!prog) + goto out; + + xdp_init_buff(xdp, PAGE_SIZE, &rxq->xdp_rxq); + xdp_prepare_buff(xdp, buf_va, XDP_PACKET_HEADROOM, pkt_len, true); + + act = bpf_prog_run_xdp(prog, xdp); + + rx_stats = &rxq->stats; + + switch (act) { + case XDP_PASS: + case XDP_TX: + case XDP_DROP: + break; + + case XDP_REDIRECT: + rxq->xdp_rc = xdp_do_redirect(ndev, xdp, prog); + if (!rxq->xdp_rc) { + rxq->xdp_flush = true; + + u64_stats_update_begin(&rx_stats->syncp); + rx_stats->packets++; + rx_stats->bytes += pkt_len; + rx_stats->xdp_redirect++; + u64_stats_update_end(&rx_stats->syncp); + + break; + } + + fallthrough; + + case XDP_ABORTED: + trace_xdp_exception(ndev, prog, act); + break; + + default: + bpf_warn_invalid_xdp_action(ndev, prog, act); + } + +out: + rcu_read_unlock(); + + return act; +} + +struct bpf_prog *mana_xdp_get(struct mana_port_context *apc) +{ + ASSERT_RTNL(); + + return apc->bpf_prog; +} + +static struct bpf_prog *mana_chn_xdp_get(struct mana_port_context *apc) +{ + return rtnl_dereference(apc->rxqs[0]->bpf_prog); +} + +/* Set xdp program on channels */ +void mana_chn_setxdp(struct mana_port_context *apc, struct bpf_prog *prog) +{ + struct bpf_prog *old_prog = mana_chn_xdp_get(apc); + unsigned int num_queues = apc->num_queues; + int i; + + ASSERT_RTNL(); + + if (old_prog == prog) + return; + + if (prog) + bpf_prog_add(prog, num_queues); + + for (i = 0; i < num_queues; i++) + rcu_assign_pointer(apc->rxqs[i]->bpf_prog, prog); + + if (old_prog) + for (i = 0; i < num_queues; i++) + bpf_prog_put(old_prog); +} + +static int mana_xdp_set(struct net_device *ndev, struct bpf_prog *prog, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + struct bpf_prog *old_prog; + struct gdma_context *gc; + int err; + + gc = apc->ac->gdma_dev->gdma_context; + + old_prog = mana_xdp_get(apc); + + if (!old_prog && !prog) + return 0; + + if (prog && ndev->mtu > MANA_XDP_MTU_MAX) { + netdev_err(ndev, "XDP: mtu:%u too large, mtu_max:%lu\n", + ndev->mtu, MANA_XDP_MTU_MAX); + NL_SET_ERR_MSG_MOD(extack, "XDP: mtu too large"); + + return -EOPNOTSUPP; + } + + /* One refcnt of the prog is hold by the caller already, so + * don't increase refcnt for this one. + */ + apc->bpf_prog = prog; + + if (apc->port_is_up) { + /* Re-create rxq's after xdp prog was loaded or unloaded. + * Ex: re create rxq's to switch from full pages to smaller + * size page fragments when xdp prog is unloaded and + * vice-versa. + */ + + /* Pre-allocate buffers to prevent failure in mana_attach */ + err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "XDP: Insufficient memory for tx/rx re-config"); + return err; + } + + err = mana_detach(ndev, false); + if (err) { + netdev_err(ndev, + "mana_detach failed at xdp set: %d\n", err); + NL_SET_ERR_MSG_MOD(extack, + "XDP: Re-config failed at detach"); + goto err_dealloc_rxbuffs; + } + + err = mana_attach(ndev); + if (err) { + netdev_err(ndev, + "mana_attach failed at xdp set: %d\n", err); + NL_SET_ERR_MSG_MOD(extack, + "XDP: Re-config failed at attach"); + goto err_dealloc_rxbuffs; + } + + mana_chn_setxdp(apc, prog); + mana_pre_dealloc_rxbufs(apc); + } + + if (old_prog) + bpf_prog_put(old_prog); + + if (prog) + ndev->max_mtu = min_t(unsigned int, MANA_XDP_MTU_MAX, + gc->adapter_mtu - ETH_HLEN); + else + ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; + + return 0; + +err_dealloc_rxbuffs: + apc->bpf_prog = old_prog; + mana_pre_dealloc_rxbufs(apc); + return err; +} + +int mana_bpf(struct net_device *ndev, struct netdev_bpf *bpf) +{ + struct netlink_ext_ack *extack = bpf->extack; + int ret; + + switch (bpf->command) { + case XDP_SETUP_PROG: + return mana_xdp_set(ndev, bpf->prog, extack); + + default: + return -EOPNOTSUPP; + } + + return ret; +} diff --git a/drivers/net/ethernet/microsoft/mana/mana_en.c b/drivers/net/ethernet/microsoft/mana/mana_en.c new file mode 100644 index 000000000..7a1ac853e --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/mana_en.c @@ -0,0 +1,4332 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include +#include + +static DEFINE_IDA(mana_adev_ida); + +static int mana_adev_idx_alloc(void) +{ + return ida_alloc(&mana_adev_ida, GFP_KERNEL); +} + +static void mana_adev_idx_free(int idx) +{ + ida_free(&mana_adev_ida, idx); +} + +static ssize_t mana_dbg_q_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct gdma_queue *gdma_q = filp->private_data; + + if (gdma_q->mem_info.nr_pages) + return mana_gd_read_ring(gdma_q, buf, count, pos); + + return simple_read_from_buffer(buf, count, pos, gdma_q->queue_mem_ptr, + gdma_q->queue_size); +} + +static const struct file_operations mana_dbg_q_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = mana_dbg_q_read, +}; + +static bool mana_en_need_log(struct mana_port_context *apc, int err) +{ + if (apc && apc->ac && apc->ac->gdma_dev && + apc->ac->gdma_dev->gdma_context) + return mana_need_log(apc->ac->gdma_dev->gdma_context, err); + else + return true; +} + +static void mana_put_rx_page(struct mana_rxq *rxq, struct page *page, + bool from_pool) +{ + if (from_pool) + page_pool_put_full_page(rxq->page_pool, page, false); + else + put_page(page); +} + +/* Microsoft Azure Network Adapter (MANA) functions */ + +static int mana_open(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + int err; + err = mana_alloc_queues(ndev); + + if (err) { + netdev_err(ndev, "%s failed to allocate queues: %d\n", __func__, err); + return err; + } + + apc->port_is_up = true; + + /* Ensure port state updated before txq state */ + smp_wmb(); + + netif_tx_wake_all_queues(ndev); + netdev_dbg(ndev, "%s successful\n", __func__); + return 0; +} + +static int mana_close(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + if (!apc->port_is_up) + return 0; + + return mana_detach(ndev, true); +} + +static void mana_link_state_handle(struct work_struct *w) +{ + struct mana_context *ac; + struct net_device *ndev; + u32 link_event; + bool link_up; + int i; + + ac = container_of(w, struct mana_context, link_change_work); + + rtnl_lock(); + + link_event = READ_ONCE(ac->link_event); + + if (link_event == HWC_DATA_HW_LINK_CONNECT) + link_up = true; + else if (link_event == HWC_DATA_HW_LINK_DISCONNECT) + link_up = false; + else + goto out; + + /* Process all ports */ + for (i = 0; i < ac->num_ports; i++) { + ndev = ac->ports[i]; + if (!ndev) + continue; + + if (link_up) { + netif_carrier_on(ndev); + + __netdev_notify_peers(ndev); + } else { + netif_carrier_off(ndev); + } + } + +out: + rtnl_unlock(); +} + +static bool mana_can_tx(struct gdma_queue *wq) +{ + return mana_gd_wq_avail_space(wq) >= MAX_TX_WQE_SIZE; +} + +static unsigned int mana_checksum_info(struct sk_buff *skb) +{ + if (skb->protocol == htons(ETH_P_IP)) { + struct iphdr *ip = ip_hdr(skb); + + if (ip->protocol == IPPROTO_TCP) + return IPPROTO_TCP; + + if (ip->protocol == IPPROTO_UDP) + return IPPROTO_UDP; + } else if (skb->protocol == htons(ETH_P_IPV6)) { + struct ipv6hdr *ip6 = ipv6_hdr(skb); + + if (ip6->nexthdr == IPPROTO_TCP) + return IPPROTO_TCP; + + if (ip6->nexthdr == IPPROTO_UDP) + return IPPROTO_UDP; + } + + /* No csum offloading */ + return 0; +} + +static void mana_add_sge(struct mana_tx_package *tp, struct mana_skb_head *ash, + int sg_i, dma_addr_t da, int sge_len, u32 gpa_mkey) +{ + ash->dma_handle[sg_i] = da; + ash->size[sg_i] = sge_len; + + tp->wqe_req.sgl[sg_i].address = da; + tp->wqe_req.sgl[sg_i].mem_key = gpa_mkey; + tp->wqe_req.sgl[sg_i].size = sge_len; +} + +static int mana_map_skb(struct sk_buff *skb, struct mana_port_context *apc, + struct mana_tx_package *tp, int gso_hs) +{ + struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; + int hsg = 1; /* num of SGEs of linear part */ + struct gdma_dev *gd = apc->ac->gdma_dev; + int skb_hlen = skb_headlen(skb); + int sge0_len, sge1_len = 0; + struct gdma_context *gc; + struct device *dev; + skb_frag_t *frag; + dma_addr_t da; + int sg_i; + int i; + + gc = gd->gdma_context; + dev = gc->dev; + + if (gso_hs && gso_hs < skb_hlen) { + sge0_len = gso_hs; + sge1_len = skb_hlen - gso_hs; + } else { + sge0_len = skb_hlen; + } + + da = dma_map_single(dev, skb->data, sge0_len, DMA_TO_DEVICE); + if (dma_mapping_error(dev, da)) + return -ENOMEM; + + mana_add_sge(tp, ash, 0, da, sge0_len, gd->gpa_mkey); + + if (sge1_len) { + sg_i = 1; + da = dma_map_single(dev, skb->data + sge0_len, sge1_len, + DMA_TO_DEVICE); + if (dma_mapping_error(dev, da)) + goto frag_err; + + mana_add_sge(tp, ash, sg_i, da, sge1_len, gd->gpa_mkey); + hsg = 2; + } + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + sg_i = hsg + i; + + frag = &skb_shinfo(skb)->frags[i]; + da = skb_frag_dma_map(dev, frag, 0, skb_frag_size(frag), + DMA_TO_DEVICE); + if (dma_mapping_error(dev, da)) + goto frag_err; + + mana_add_sge(tp, ash, sg_i, da, skb_frag_size(frag), + gd->gpa_mkey); + } + + return 0; + +frag_err: + if (net_ratelimit()) + netdev_err(apc->ndev, "Failed to map skb of size %u to DMA\n", + skb->len); + for (i = sg_i - 1; i >= hsg; i--) + dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], + DMA_TO_DEVICE); + + for (i = hsg - 1; i >= 0; i--) + dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], + DMA_TO_DEVICE); + + return -ENOMEM; +} + +/* Handle the case when GSO SKB linear length is too large. + * MANA NIC requires GSO packets to put only the packet header to SGE0. + * So, we need 2 SGEs for the skb linear part which contains more than the + * header. + * Return a positive value for the number of SGEs, or a negative value + * for an error. + */ +static int mana_fix_skb_head(struct net_device *ndev, struct sk_buff *skb, + int gso_hs) +{ + int num_sge = 1 + skb_shinfo(skb)->nr_frags; + int skb_hlen = skb_headlen(skb); + + if (gso_hs < skb_hlen) { + num_sge++; + } else if (gso_hs > skb_hlen) { + if (net_ratelimit()) + netdev_err(ndev, + "TX nonlinear head: hs:%d, skb_hlen:%d\n", + gso_hs, skb_hlen); + + return -EINVAL; + } + + return num_sge; +} + +/* Get the GSO packet's header size */ +static int mana_get_gso_hs(struct sk_buff *skb) +{ + int gso_hs; + + if (skb->encapsulation) { + gso_hs = skb_inner_tcp_all_headers(skb); + } else { + if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { + gso_hs = skb_transport_offset(skb) + + sizeof(struct udphdr); + } else { + gso_hs = skb_tcp_all_headers(skb); + } + } + + return gso_hs; +} + +static void mana_per_port_queue_reset_work_handler(struct work_struct *work) +{ + struct mana_port_context *apc = container_of(work, + struct mana_port_context, + queue_reset_work); + struct net_device *ndev = apc->ndev; + int err; + + rtnl_lock(); + + /* Block RDMA from grabbing the vport during the detach/attach + * window, same as mana_set_channels(). + */ + mutex_lock(&apc->vport_mutex); + apc->channel_changing = true; + mutex_unlock(&apc->vport_mutex); + + /* Pre-allocate buffers to prevent failure in mana_attach later */ + err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); + if (err) { + netdev_err(ndev, "Insufficient memory for reset post tx stall detection\n"); + goto clear_flag; + } + + err = mana_detach(ndev, false); + if (err) { + netdev_err(ndev, "mana_detach failed: %d\n", err); + goto dealloc_pre_rxbufs; + } + + err = mana_attach(ndev); + if (err) + netdev_err(ndev, "mana_attach failed: %d\n", err); + +dealloc_pre_rxbufs: + mana_pre_dealloc_rxbufs(apc); +clear_flag: + mutex_lock(&apc->vport_mutex); + apc->channel_changing = false; + mutex_unlock(&apc->vport_mutex); + + rtnl_unlock(); +} + +netdev_tx_t mana_start_xmit(struct sk_buff *skb, struct net_device *ndev) +{ + enum mana_tx_pkt_format pkt_fmt = MANA_SHORT_PKT_FMT; + struct mana_port_context *apc = netdev_priv(ndev); + int gso_hs = 0; /* zero for non-GSO pkts */ + u16 txq_idx = skb_get_queue_mapping(skb); + struct gdma_dev *gd = apc->ac->gdma_dev; + bool ipv4 = false, ipv6 = false; + struct mana_tx_package pkg = {}; + struct netdev_queue *net_txq; + struct mana_stats_tx *tx_stats; + struct gdma_queue *gdma_sq; + int err, len, num_gso_seg; + unsigned int csum_type; + struct mana_txq *txq; + struct mana_cq *cq; + + if (unlikely(!apc->port_is_up)) + goto tx_drop; + + if (skb_cow_head(skb, MANA_HEADROOM)) + goto tx_drop_count; + + txq = &apc->tx_qp[txq_idx]->txq; + gdma_sq = txq->gdma_sq; + cq = &apc->tx_qp[txq_idx]->tx_cq; + tx_stats = &txq->stats; + + BUILD_BUG_ON(MAX_TX_WQE_SGL_ENTRIES != MANA_MAX_TX_WQE_SGL_ENTRIES); + if (MAX_SKB_FRAGS + 2 > MAX_TX_WQE_SGL_ENTRIES && + skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) { + /* GSO skb with Hardware SGE limit exceeded is not expected here + * as they are handled in mana_features_check() callback + */ + if (skb_linearize(skb)) { + netdev_warn_once(ndev, "Failed to linearize skb with nr_frags=%d and is_gso=%d\n", + skb_shinfo(skb)->nr_frags, + skb_is_gso(skb)); + goto tx_drop_count; + } + apc->eth_stats.tx_linear_pkt_cnt++; + } + + pkg.tx_oob.s_oob.vcq_num = cq->gdma_id; + pkg.tx_oob.s_oob.vsq_frame = txq->vsq_frame; + + if (txq->vp_offset > MANA_SHORT_VPORT_OFFSET_MAX) { + pkg.tx_oob.l_oob.long_vp_offset = txq->vp_offset; + pkt_fmt = MANA_LONG_PKT_FMT; + } else { + pkg.tx_oob.s_oob.short_vp_offset = txq->vp_offset; + } + + if (skb_vlan_tag_present(skb)) { + pkt_fmt = MANA_LONG_PKT_FMT; + pkg.tx_oob.l_oob.inject_vlan_pri_tag = 1; + pkg.tx_oob.l_oob.pcp = skb_vlan_tag_get_prio(skb); + pkg.tx_oob.l_oob.dei = skb_vlan_tag_get_cfi(skb); + pkg.tx_oob.l_oob.vlan_id = skb_vlan_tag_get_id(skb); + } + + pkg.tx_oob.s_oob.pkt_fmt = pkt_fmt; + + if (pkt_fmt == MANA_SHORT_PKT_FMT) { + pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_short_oob); + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->short_pkt_fmt++; + u64_stats_update_end(&tx_stats->syncp); + } else { + pkg.wqe_req.inline_oob_size = sizeof(struct mana_tx_oob); + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->long_pkt_fmt++; + u64_stats_update_end(&tx_stats->syncp); + } + + pkg.wqe_req.inline_oob_data = &pkg.tx_oob; + pkg.wqe_req.flags = 0; + pkg.wqe_req.client_data_unit = 0; + + pkg.wqe_req.num_sge = 1 + skb_shinfo(skb)->nr_frags; + + if (skb->protocol == htons(ETH_P_IP)) + ipv4 = true; + else if (skb->protocol == htons(ETH_P_IPV6)) + ipv6 = true; + + if (skb_is_gso(skb)) { + int num_sge; + + gso_hs = mana_get_gso_hs(skb); + + num_sge = mana_fix_skb_head(ndev, skb, gso_hs); + if (num_sge > 0) + pkg.wqe_req.num_sge = num_sge; + else + goto tx_drop_count; + + u64_stats_update_begin(&tx_stats->syncp); + if (skb->encapsulation) { + tx_stats->tso_inner_packets++; + tx_stats->tso_inner_bytes += skb->len - gso_hs; + } else { + tx_stats->tso_packets++; + tx_stats->tso_bytes += skb->len - gso_hs; + } + u64_stats_update_end(&tx_stats->syncp); + + pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; + pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; + + pkg.tx_oob.s_oob.comp_iphdr_csum = 1; + pkg.tx_oob.s_oob.comp_tcp_csum = 1; + pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); + + pkg.wqe_req.client_data_unit = skb_shinfo(skb)->gso_size; + pkg.wqe_req.flags = GDMA_WR_OOB_IN_SGL | GDMA_WR_PAD_BY_SGE0; + if (ipv4) { + ip_hdr(skb)->tot_len = 0; + ip_hdr(skb)->check = 0; + tcp_hdr(skb)->check = + ~csum_tcpudp_magic(ip_hdr(skb)->saddr, + ip_hdr(skb)->daddr, 0, + IPPROTO_TCP, 0); + } else { + ipv6_hdr(skb)->payload_len = 0; + tcp_hdr(skb)->check = + ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, + &ipv6_hdr(skb)->daddr, 0, + IPPROTO_TCP, 0); + } + } else if (skb->ip_summed == CHECKSUM_PARTIAL) { + csum_type = mana_checksum_info(skb); + + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->csum_partial++; + u64_stats_update_end(&tx_stats->syncp); + + if (csum_type == IPPROTO_TCP) { + pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; + pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; + + pkg.tx_oob.s_oob.comp_tcp_csum = 1; + pkg.tx_oob.s_oob.trans_off = skb_transport_offset(skb); + + } else if (csum_type == IPPROTO_UDP) { + pkg.tx_oob.s_oob.is_outer_ipv4 = ipv4; + pkg.tx_oob.s_oob.is_outer_ipv6 = ipv6; + + pkg.tx_oob.s_oob.comp_udp_csum = 1; + } else { + /* Can't do offload of this type of checksum */ + if (skb_checksum_help(skb)) + goto tx_drop_count; + } + } + + if (pkg.wqe_req.num_sge <= ARRAY_SIZE(pkg.sgl_array)) { + pkg.wqe_req.sgl = pkg.sgl_array; + } else { + pkg.sgl_ptr = kmalloc_objs(struct gdma_sge, pkg.wqe_req.num_sge, + GFP_ATOMIC); + if (!pkg.sgl_ptr) + goto tx_drop_count; + + pkg.wqe_req.sgl = pkg.sgl_ptr; + } + + if (mana_map_skb(skb, apc, &pkg, gso_hs)) { + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->mana_map_err++; + u64_stats_update_end(&tx_stats->syncp); + goto free_sgl_ptr; + } + + skb_queue_tail(&txq->pending_skbs, skb); + + len = skb->len; + num_gso_seg = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; + net_txq = netdev_get_tx_queue(ndev, txq_idx); + + err = mana_gd_post_work_request(gdma_sq, &pkg.wqe_req, + (struct gdma_posted_wqe_info *)skb->cb); + if (!mana_can_tx(gdma_sq)) { + netif_tx_stop_queue(net_txq); + apc->eth_stats.stop_queue++; + } + + if (err) { + (void)skb_dequeue_tail(&txq->pending_skbs); + mana_unmap_skb(skb, apc); + netdev_warn(ndev, "Failed to post TX OOB: %d\n", err); + goto free_sgl_ptr; + } + + err = NETDEV_TX_OK; + atomic_inc(&txq->pending_sends); + + mana_gd_wq_ring_doorbell(gd->gdma_context, gdma_sq); + + /* skb may be freed after mana_gd_post_work_request. Do not use it. */ + skb = NULL; + + /* Populated the packet and bytes counters based on post GSO packet + * calculations + */ + tx_stats = &txq->stats; + u64_stats_update_begin(&tx_stats->syncp); + tx_stats->packets += num_gso_seg; + tx_stats->bytes += len + ((num_gso_seg - 1) * gso_hs); + u64_stats_update_end(&tx_stats->syncp); + + if (netif_tx_queue_stopped(net_txq) && mana_can_tx(gdma_sq)) { + netif_tx_wake_queue(net_txq); + apc->eth_stats.wake_queue++; + } + + kfree(pkg.sgl_ptr); + return err; + +free_sgl_ptr: + kfree(pkg.sgl_ptr); +tx_drop_count: + ndev->stats.tx_dropped++; +tx_drop: + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +#if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) +static netdev_features_t mana_features_check(struct sk_buff *skb, + struct net_device *ndev, + netdev_features_t features) +{ + if (skb_shinfo(skb)->nr_frags + 2 > MAX_TX_WQE_SGL_ENTRIES) { + /* Exceeds HW SGE limit. + * GSO case: + * Disable GSO so the stack will software-segment the skb + * into smaller skbs that fit the SGE budget. + * Non-GSO case: + * The xmit path will attempt skb_linearize() as a fallback. + */ + features &= ~NETIF_F_GSO_MASK; + } + return features; +} +#endif + +static void mana_get_stats64(struct net_device *ndev, + struct rtnl_link_stats64 *st) +{ + struct mana_port_context *apc = netdev_priv(ndev); + unsigned int num_queues = apc->num_queues; + struct mana_stats_rx *rx_stats; + struct mana_stats_tx *tx_stats; + unsigned int start; + u64 packets, bytes; + int q; + + if (!apc->port_is_up) + return; + + netdev_stats_to_stats64(st, &ndev->stats); + + if (apc->ac->hwc_timeout_occurred) + netdev_warn_once(ndev, "HWC timeout occurred\n"); + + st->rx_missed_errors = apc->ac->hc_stats.hc_rx_discards_no_wqe; + + for (q = 0; q < num_queues; q++) { + rx_stats = &apc->rxqs[q]->stats; + + do { + start = u64_stats_fetch_begin(&rx_stats->syncp); + packets = rx_stats->packets; + bytes = rx_stats->bytes; + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); + + st->rx_packets += packets; + st->rx_bytes += bytes; + } + + for (q = 0; q < num_queues; q++) { + tx_stats = &apc->tx_qp[q]->txq.stats; + + do { + start = u64_stats_fetch_begin(&tx_stats->syncp); + packets = tx_stats->packets; + bytes = tx_stats->bytes; + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); + + st->tx_packets += packets; + st->tx_bytes += bytes; + } +} + +static int mana_get_tx_queue(struct net_device *ndev, struct sk_buff *skb, + int old_q) +{ + struct mana_port_context *apc = netdev_priv(ndev); + u32 hash = skb_get_hash(skb); + struct sock *sk = skb->sk; + int txq; + + txq = apc->indir_table[hash & (apc->indir_table_sz - 1)]; + + if (txq != old_q && sk && sk_fullsock(sk) && + rcu_access_pointer(sk->sk_dst_cache)) + sk_tx_queue_set(sk, txq); + + return txq; +} + +static u16 mana_select_queue(struct net_device *ndev, struct sk_buff *skb, + struct net_device *sb_dev) +{ + int txq; + + if (ndev->real_num_tx_queues == 1) + return 0; + + txq = sk_tx_queue_get(skb->sk); + + if (txq < 0 || skb->ooo_okay || txq >= ndev->real_num_tx_queues) { + if (skb_rx_queue_recorded(skb)) + txq = skb_get_rx_queue(skb); + else + txq = mana_get_tx_queue(ndev, skb, txq); + } + + return txq; +} + +/* Release pre-allocated RX buffers */ +void mana_pre_dealloc_rxbufs(struct mana_port_context *mpc) +{ + struct device *dev; + int i; + + dev = mpc->ac->gdma_dev->gdma_context->dev; + + if (!mpc->rxbufs_pre) + goto out1; + + if (!mpc->das_pre) + goto out2; + + while (mpc->rxbpre_total) { + i = --mpc->rxbpre_total; + dma_unmap_single(dev, mpc->das_pre[i], mpc->rxbpre_datasize, + DMA_FROM_DEVICE); + put_page(virt_to_head_page(mpc->rxbufs_pre[i])); + } + + kvfree(mpc->das_pre); + mpc->das_pre = NULL; + +out2: + kvfree(mpc->rxbufs_pre); + mpc->rxbufs_pre = NULL; + +out1: + mpc->rxbpre_datasize = 0; + mpc->rxbpre_alloc_size = 0; + mpc->rxbpre_headroom = 0; +} + +/* Get a buffer from the pre-allocated RX buffers */ +static void *mana_get_rxbuf_pre(struct mana_rxq *rxq, dma_addr_t *da) +{ + struct net_device *ndev = rxq->ndev; + struct mana_port_context *mpc; + void *va; + + mpc = netdev_priv(ndev); + + if (!mpc->rxbufs_pre || !mpc->das_pre || !mpc->rxbpre_total) { + netdev_err(ndev, "No RX pre-allocated bufs\n"); + return NULL; + } + + /* Check sizes to catch unexpected coding error */ + if (mpc->rxbpre_datasize != rxq->datasize) { + netdev_err(ndev, "rxbpre_datasize mismatch: %u: %u\n", + mpc->rxbpre_datasize, rxq->datasize); + return NULL; + } + + if (mpc->rxbpre_alloc_size != rxq->alloc_size) { + netdev_err(ndev, "rxbpre_alloc_size mismatch: %u: %u\n", + mpc->rxbpre_alloc_size, rxq->alloc_size); + return NULL; + } + + if (mpc->rxbpre_headroom != rxq->headroom) { + netdev_err(ndev, "rxbpre_headroom mismatch: %u: %u\n", + mpc->rxbpre_headroom, rxq->headroom); + return NULL; + } + + mpc->rxbpre_total--; + + *da = mpc->das_pre[mpc->rxbpre_total]; + va = mpc->rxbufs_pre[mpc->rxbpre_total]; + mpc->rxbufs_pre[mpc->rxbpre_total] = NULL; + + /* Deallocate the array after all buffers are gone */ + if (!mpc->rxbpre_total) + mana_pre_dealloc_rxbufs(mpc); + + return va; +} + +static bool +mana_use_single_rxbuf_per_page(struct mana_port_context *apc, u32 mtu) +{ + /* On some platforms with 4K PAGE_SIZE, page_pool fragment allocation + * in the RX refill path (~2kB buffer) can cause significant throughput + * regression under high connection counts. Allow user to force one RX + * buffer per page via ethtool private flag to bypass the fragment + * path. + */ + if (apc->priv_flags & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) + return true; + + /* For xdp and jumbo frames make sure only one packet fits per page. */ + if (mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || mana_xdp_get(apc)) + return true; + + return false; +} + +/* Get RX buffer's data size, alloc size, XDP headroom based on MTU */ +static void mana_get_rxbuf_cfg(struct mana_port_context *apc, + int mtu, u32 *datasize, u32 *alloc_size, + u32 *headroom, u32 *frag_count) +{ + u32 len, buf_size; + + /* Calculate datasize first (consistent across all cases) */ + *datasize = mtu + ETH_HLEN; + + if (mana_use_single_rxbuf_per_page(apc, mtu)) { + if (mana_xdp_get(apc)) { + *headroom = XDP_PACKET_HEADROOM; + *alloc_size = PAGE_SIZE; + } else { + *headroom = 0; /* no support for XDP */ + *alloc_size = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + + *headroom); + } + + *frag_count = 1; + + /* In the single-buffer path, napi_build_skb() must see the + * actual backing allocation size so skb->truesize reflects + * the full page (or higher-order page), not just the usable + * packet area. + */ + *alloc_size = PAGE_SIZE << get_order(*alloc_size); + return; + } + + /* Standard MTU case - optimize for multiple packets per page */ + *headroom = 0; + + /* Calculate base buffer size needed */ + len = SKB_DATA_ALIGN(mtu + MANA_RXBUF_PAD + *headroom); + buf_size = ALIGN(len, MANA_RX_FRAG_ALIGNMENT); + + /* Calculate how many packets can fit in a page */ + *frag_count = PAGE_SIZE / buf_size; + *alloc_size = buf_size; +} + +int mana_pre_alloc_rxbufs(struct mana_port_context *mpc, int new_mtu, int num_queues) +{ + struct device *dev; + struct page *page; + dma_addr_t da; + int num_rxb; + void *va; + int i; + + mana_get_rxbuf_cfg(mpc, new_mtu, &mpc->rxbpre_datasize, + &mpc->rxbpre_alloc_size, &mpc->rxbpre_headroom, + &mpc->rxbpre_frag_count); + + dev = mpc->ac->gdma_dev->gdma_context->dev; + + num_rxb = num_queues * mpc->rx_queue_size; + + WARN(mpc->rxbufs_pre, "mana rxbufs_pre exists\n"); + mpc->rxbufs_pre = kvmalloc_array(num_rxb, sizeof(void *), GFP_KERNEL); + if (!mpc->rxbufs_pre) + goto error; + + mpc->das_pre = kvmalloc_objs(dma_addr_t, num_rxb); + if (!mpc->das_pre) + goto error; + + mpc->rxbpre_total = 0; + + for (i = 0; i < num_rxb; i++) { + page = dev_alloc_pages(get_order(mpc->rxbpre_alloc_size)); + if (!page) + goto error; + + va = page_to_virt(page); + + da = dma_map_single(dev, va + mpc->rxbpre_headroom, + mpc->rxbpre_datasize, DMA_FROM_DEVICE); + if (dma_mapping_error(dev, da)) { + put_page(page); + goto error; + } + + mpc->rxbufs_pre[i] = va; + mpc->das_pre[i] = da; + mpc->rxbpre_total = i + 1; + } + + return 0; + +error: + netdev_err(mpc->ndev, "Failed to pre-allocate RX buffers for %d queues\n", num_queues); + mana_pre_dealloc_rxbufs(mpc); + return -ENOMEM; +} + +static int mana_change_mtu(struct net_device *ndev, int new_mtu) +{ + struct mana_port_context *mpc = netdev_priv(ndev); + unsigned int old_mtu = ndev->mtu; + int err; + + /* Pre-allocate buffers to prevent failure in mana_attach later */ + err = mana_pre_alloc_rxbufs(mpc, new_mtu, mpc->num_queues); + if (err) { + netdev_err(ndev, "Insufficient memory for new MTU\n"); + return err; + } + + err = mana_detach(ndev, false); + if (err) { + netdev_err(ndev, "mana_detach failed: %d\n", err); + goto out; + } + + WRITE_ONCE(ndev->mtu, new_mtu); + + err = mana_attach(ndev); + if (err) { + netdev_err(ndev, "mana_attach failed: %d\n", err); + WRITE_ONCE(ndev->mtu, old_mtu); + } + +out: + mana_pre_dealloc_rxbufs(mpc); + return err; +} + +static void mana_tx_timeout(struct net_device *netdev, unsigned int txqueue) +{ + struct mana_port_context *apc = netdev_priv(netdev); + struct mana_context *ac = apc->ac; + struct gdma_context *gc = ac->gdma_dev->gdma_context; + + /* Debug knob for bringup/qualification: when set, log the timeout and + * skip the reset so the failing state is preserved for telemetry. + * Disabled by default; production behaviour is unchanged. + */ + if (READ_ONCE(apc->tx_timeout_skip_reset)) { + netdev_warn(netdev, + "TX timeout on queue %u: reset skipped (tx_timeout_skip_reset enabled)\n", + txqueue); + return; + } + + /* Already in service, hence tx queue reset is not required.*/ + if (test_bit(GC_IN_SERVICE, &gc->flags)) + return; + + /* Note: If there are pending queue reset work for this port(apc), + * subsequent request queued up from here are ignored. This is because + * we are using the same work instance per port(apc). + */ + queue_work(ac->per_port_queue_reset_wq, &apc->queue_reset_work); +} + +static int mana_shaper_set(struct net_shaper_binding *binding, + const struct net_shaper *shaper, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(binding->netdev); + u32 old_speed, rate; + int err; + + if (shaper->handle.scope != NET_SHAPER_SCOPE_NETDEV) { + NL_SET_ERR_MSG_MOD(extack, "net shaper scope should be netdev"); + return -EINVAL; + } + + if (apc->handle.id && shaper->handle.id != apc->handle.id) { + NL_SET_ERR_MSG_MOD(extack, "Cannot create multiple shapers"); + return -EOPNOTSUPP; + } + + if (!shaper->bw_max || (shaper->bw_max % 100000000)) { + NL_SET_ERR_MSG_MOD(extack, "Please use multiples of 100Mbps for bandwidth"); + return -EINVAL; + } + + rate = div_u64(shaper->bw_max, 1000); /* Convert bps to Kbps */ + rate = div_u64(rate, 1000); /* Convert Kbps to Mbps */ + + /* Get current speed */ + err = mana_query_link_cfg(apc); + old_speed = (err) ? SPEED_UNKNOWN : apc->speed; + + if (!err) { + err = mana_set_bw_clamp(apc, rate, TRI_STATE_TRUE); + apc->speed = (err) ? old_speed : rate; + apc->handle = (err) ? apc->handle : shaper->handle; + } + + return err; +} + +static int mana_shaper_del(struct net_shaper_binding *binding, + const struct net_shaper_handle *handle, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(binding->netdev); + int err; + + err = mana_set_bw_clamp(apc, 0, TRI_STATE_FALSE); + + if (!err) { + /* Reset mana port context parameters */ + apc->handle.id = 0; + apc->handle.scope = NET_SHAPER_SCOPE_UNSPEC; + apc->speed = apc->max_speed; + } + + return err; +} + +static void mana_shaper_cap(struct net_shaper_binding *binding, + enum net_shaper_scope scope, + unsigned long *flags) +{ + *flags = BIT(NET_SHAPER_A_CAPS_SUPPORT_BW_MAX) | + BIT(NET_SHAPER_A_CAPS_SUPPORT_METRIC_BPS); +} + +static const struct net_shaper_ops mana_shaper_ops = { + .set = mana_shaper_set, + .delete = mana_shaper_del, + .capabilities = mana_shaper_cap, +}; + +static const struct net_device_ops mana_devops = { + .ndo_open = mana_open, + .ndo_stop = mana_close, + .ndo_select_queue = mana_select_queue, +#if (MAX_SKB_FRAGS + 2 > MANA_MAX_TX_WQE_SGL_ENTRIES) + .ndo_features_check = mana_features_check, +#endif + .ndo_start_xmit = mana_start_xmit, + .ndo_validate_addr = eth_validate_addr, + .ndo_get_stats64 = mana_get_stats64, + .ndo_bpf = mana_bpf, + .ndo_xdp_xmit = mana_xdp_xmit, + .ndo_change_mtu = mana_change_mtu, + .ndo_tx_timeout = mana_tx_timeout, + .net_shaper_ops = &mana_shaper_ops, +}; + +static void mana_cleanup_port_context(struct mana_port_context *apc) +{ + /* + * make sure subsequent cleanup attempts don't end up removing already + * cleaned dentry pointer + */ + debugfs_remove(apc->mana_port_debugfs); + apc->mana_port_debugfs = NULL; + kfree(apc->rxqs); + apc->rxqs = NULL; +} + +static void mana_cleanup_indir_table(struct mana_port_context *apc) +{ + apc->indir_table_sz = 0; + kfree(apc->indir_table); + kfree(apc->rxobj_table); +} + +static int mana_init_port_context(struct mana_port_context *apc) +{ + apc->rxqs = kzalloc_objs(struct mana_rxq *, apc->num_queues); + + return !apc->rxqs ? -ENOMEM : 0; +} + +static int gdma_mana_send_request(struct gdma_context *gc, void *in_buf, + u32 in_len, void *out_buf, u32 out_len) +{ + struct gdma_resp_hdr *resp = out_buf; + struct gdma_req_hdr *req = in_buf; + struct device *dev = gc->dev; + static atomic_t activity_id; + int err; + + req->dev_id = gc->mana.dev_id; + req->activity_id = atomic_inc_return(&activity_id); + + err = mana_gd_send_request(gc, in_len, in_buf, out_len, + out_buf); + if (err || resp->status) { + if (err == -EOPNOTSUPP) + return err; + + if (req->req.msg_type != MANA_QUERY_PHY_STAT && + mana_need_log(gc, err)) + dev_err(dev, "Command 0x%x failed with status: 0x%x, err: %d\n", + req->req.msg_type, resp->status, err); + return err ? err : -EPROTO; + } + + if (req->dev_id.as_uint32 != resp->dev_id.as_uint32 || + req->activity_id != resp->activity_id) { + dev_err(dev, "Unexpected mana message response: %x,%x,%x,%x\n", + req->dev_id.as_uint32, resp->dev_id.as_uint32, + req->activity_id, resp->activity_id); + return -EPROTO; + } + + return 0; +} + +static int mana_send_request(struct mana_context *ac, void *in_buf, + u32 in_len, void *out_buf, u32 out_len) +{ + struct gdma_context *gc = ac->gdma_dev->gdma_context; + + return gdma_mana_send_request(gc, in_buf, in_len, out_buf, out_len); +} + +static int mana_verify_resp_hdr(const struct gdma_resp_hdr *resp_hdr, + const enum mana_command_code expected_code, + const u32 min_size) +{ + if (resp_hdr->response.msg_type != expected_code) + return -EPROTO; + + if (resp_hdr->response.msg_version < GDMA_MESSAGE_V1) + return -EPROTO; + + if (resp_hdr->response.msg_size < min_size) + return -EPROTO; + + return 0; +} + +static int mana_pf_register_hw_vport(struct mana_port_context *apc) +{ + struct mana_register_hw_vport_resp resp = {}; + struct mana_register_hw_vport_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_HW_PORT, + sizeof(req), sizeof(resp)); + req.attached_gfid = 1; + req.is_pf_default_vport = 1; + req.allow_all_ether_types = 1; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + netdev_err(apc->ndev, "Failed to register hw vPort: %d\n", err); + return err; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_HW_PORT, + sizeof(resp)); + if (err || resp.hdr.status) { + netdev_err(apc->ndev, "Failed to register hw vPort: %d, 0x%x\n", + err, resp.hdr.status); + return err ? err : -EPROTO; + } + + apc->port_handle = resp.hw_vport_handle; + return 0; +} + +static void mana_pf_deregister_hw_vport(struct mana_port_context *apc) +{ + struct mana_deregister_hw_vport_resp resp = {}; + struct mana_deregister_hw_vport_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_HW_PORT, + sizeof(req), sizeof(resp)); + req.hw_vport_handle = apc->port_handle; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + if (mana_en_need_log(apc, err)) + netdev_err(apc->ndev, "Failed to unregister hw vPort: %d\n", + err); + + return; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_HW_PORT, + sizeof(resp)); + if (err || resp.hdr.status) + netdev_err(apc->ndev, + "Failed to deregister hw vPort: %d, 0x%x\n", + err, resp.hdr.status); +} + +static int mana_pf_register_filter(struct mana_port_context *apc) +{ + struct mana_register_filter_resp resp = {}; + struct mana_register_filter_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_REGISTER_FILTER, + sizeof(req), sizeof(resp)); + req.vport = apc->port_handle; + memcpy(req.mac_addr, apc->mac_addr, ETH_ALEN); + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + netdev_err(apc->ndev, "Failed to register filter: %d\n", err); + return err; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_REGISTER_FILTER, + sizeof(resp)); + if (err || resp.hdr.status) { + netdev_err(apc->ndev, "Failed to register filter: %d, 0x%x\n", + err, resp.hdr.status); + return err ? err : -EPROTO; + } + + apc->pf_filter_handle = resp.filter_handle; + return 0; +} + +static void mana_pf_deregister_filter(struct mana_port_context *apc) +{ + struct mana_deregister_filter_resp resp = {}; + struct mana_deregister_filter_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_DEREGISTER_FILTER, + sizeof(req), sizeof(resp)); + req.filter_handle = apc->pf_filter_handle; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + if (mana_en_need_log(apc, err)) + netdev_err(apc->ndev, "Failed to unregister filter: %d\n", + err); + + return; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_DEREGISTER_FILTER, + sizeof(resp)); + if (err || resp.hdr.status) + netdev_err(apc->ndev, + "Failed to deregister filter: %d, 0x%x\n", + err, resp.hdr.status); +} + +int mana_gd_query_device_cfg(struct gdma_context *gc, u32 proto_major_ver, + u32 proto_minor_ver, u32 proto_micro_ver, + u16 *max_num_vports, u8 *bm_hostmode) +{ + struct mana_query_device_cfg_resp resp = {}; + struct mana_query_device_cfg_req req = {}; + struct device *dev = gc->dev; + int err = 0; + + mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_DEV_CONFIG, + sizeof(req), sizeof(resp)); + + req.hdr.resp.msg_version = GDMA_MESSAGE_V3; + + req.proto_major_ver = proto_major_ver; + req.proto_minor_ver = proto_minor_ver; + req.proto_micro_ver = proto_micro_ver; + + err = gdma_mana_send_request(gc, &req, sizeof(req), + &resp, sizeof(resp)); + if (err) { + dev_err(dev, "Failed to query config: %d", err); + return err; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_DEV_CONFIG, + sizeof(resp)); + if (err || resp.hdr.status) { + dev_err(dev, "Invalid query result: %d, 0x%x\n", err, + resp.hdr.status); + if (!err) + err = -EPROTO; + return err; + } + + gc->cqe8_coalescing_sup = !!(resp.pf_cap_flags1 & + MANA_PF_FLAG_1_CQE_8_COALESCING_SUPPORTED); + + *max_num_vports = resp.max_num_vports; + + if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) { + if (resp.adapter_mtu == 0) { + /* + * Some older PF firmware versions report an + * adapter_mtu of 0. MANA hardware always supports the + * standard Ethernet MTU, so fall back to ETH_FRAME_LEN. + * Jumbo frames will not be available in this case. + */ + dev_info(dev, + "PF reported adapter_mtu of 0, falling back to %u (jumbo frames disabled)\n", + ETH_FRAME_LEN); + gc->adapter_mtu = ETH_FRAME_LEN; + } else if (resp.adapter_mtu < ETH_MIN_MTU + ETH_HLEN) { + dev_err(dev, "Adapter MTU too small: %u\n", + resp.adapter_mtu); + return -EPROTO; + } else { + gc->adapter_mtu = resp.adapter_mtu; + } + } else { + gc->adapter_mtu = ETH_FRAME_LEN; + } + + if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V3) + *bm_hostmode = resp.bm_hostmode; + else + *bm_hostmode = 0; + + return 0; +} + +static int mana_query_vport_cfg(struct mana_port_context *apc, u32 vport_index, + u32 *max_sq, u32 *max_rq, u32 *num_indir_entry) +{ + struct mana_query_vport_cfg_resp resp = {}; + struct mana_query_vport_cfg_req req = {}; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_VPORT_CONFIG, + sizeof(req), sizeof(resp)); + + req.vport_index = vport_index; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) + return err; + + err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_VPORT_CONFIG, + sizeof(resp)); + if (err) + return err; + + if (resp.hdr.status) + return -EPROTO; + + *max_sq = resp.max_num_sq; + *max_rq = resp.max_num_rq; + + if (*max_sq == 0 || *max_rq == 0) { + netdev_err(apc->ndev, "Invalid max queues from vPort config\n"); + return -EPROTO; + } + + if (resp.num_indirection_ent > 0 && + resp.num_indirection_ent <= MANA_INDIRECT_TABLE_MAX_SIZE && + is_power_of_2(resp.num_indirection_ent)) { + *num_indir_entry = resp.num_indirection_ent; + } else { + netdev_warn(apc->ndev, + "Setting indirection table size to default %d for vPort %d\n", + MANA_INDIRECT_TABLE_DEF_SIZE, apc->port_idx); + *num_indir_entry = MANA_INDIRECT_TABLE_DEF_SIZE; + } + + apc->port_handle = resp.vport; + ether_addr_copy(apc->mac_addr, resp.mac_addr); + + apc->vport_max_sq = *max_sq; + apc->vport_max_rq = *max_rq; + + return 0; +} + +void mana_uncfg_vport(struct mana_port_context *apc) +{ + mutex_lock(&apc->vport_mutex); + apc->vport_use_count--; + WARN_ON(apc->vport_use_count < 0); + mutex_unlock(&apc->vport_mutex); +} +EXPORT_SYMBOL_NS(mana_uncfg_vport, "NET_MANA"); + +int mana_cfg_vport(struct mana_port_context *apc, u32 protection_dom_id, + u32 doorbell_pg_id, bool check_channel_changing) +{ + struct mana_config_vport_resp resp = {}; + struct mana_config_vport_req req = {}; + int err; + + /* This function is used to program the Ethernet port in the hardware + * table. It can be called from the Ethernet driver or the RDMA driver. + * + * For Ethernet usage, the hardware supports only one active user on a + * physical port. The driver checks on the port usage before programming + * the hardware when creating the RAW QP (RDMA driver) or exposing the + * device to kernel NET layer (Ethernet driver). + * + * Because the RDMA driver doesn't know in advance which QP type the + * user will create, it exposes the device with all its ports. The user + * may not be able to create RAW QP on a port if this port is already + * in used by the Ethernet driver from the kernel. + * + * This physical port limitation only applies to the RAW QP. For RC QP, + * the hardware doesn't have this limitation. The user can create RC + * QPs on a physical port up to the hardware limits independent of the + * Ethernet usage on the same port. + */ + mutex_lock(&apc->vport_mutex); + if (apc->vport_use_count > 0 || + (check_channel_changing && apc->channel_changing)) { + mutex_unlock(&apc->vport_mutex); + return -EBUSY; + } + apc->vport_use_count++; + mutex_unlock(&apc->vport_mutex); + + mana_gd_init_req_hdr(&req.hdr, MANA_CONFIG_VPORT_TX, + sizeof(req), sizeof(resp)); + req.vport = apc->port_handle; + req.pdid = protection_dom_id; + req.doorbell_pageid = doorbell_pg_id; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + netdev_err(apc->ndev, "Failed to configure vPort: %d\n", err); + goto out; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_TX, + sizeof(resp)); + if (err || resp.hdr.status) { + netdev_err(apc->ndev, "Failed to configure vPort: %d, 0x%x\n", + err, resp.hdr.status); + if (!err) + err = -EPROTO; + + goto out; + } + + apc->tx_shortform_allowed = resp.short_form_allowed; + apc->tx_vp_offset = resp.tx_vport_offset; + + netdev_info(apc->ndev, "Enabled vPort %llu PD %u DB %u MAC %pM\n", + apc->port_handle, protection_dom_id, doorbell_pg_id, apc->mac_addr); +out: + if (err) + mana_uncfg_vport(apc); + + return err; +} +EXPORT_SYMBOL_NS(mana_cfg_vport, "NET_MANA"); + +static int mana_cfg_vport_steering(struct mana_port_context *apc, + enum TRI_STATE rx, + bool update_default_rxobj, bool update_key, + bool update_tab) +{ + struct mana_cfg_rx_steer_req_v2 *req; + struct mana_cfg_rx_steer_resp resp = {}; + struct net_device *ndev = apc->ndev; + u32 req_buf_size; + int err; + + req_buf_size = struct_size(req, indir_tab, apc->indir_table_sz); + req = kzalloc(req_buf_size, GFP_KERNEL); + if (!req) + return -ENOMEM; + + mana_gd_init_req_hdr(&req->hdr, MANA_CONFIG_VPORT_RX, req_buf_size, + sizeof(resp)); + + /* Request & response versions can be different. + * HW can handle newer msg versions, by skipping + * new fields. + */ + req->hdr.req.msg_version = GDMA_MESSAGE_V5; + req->hdr.resp.msg_version = GDMA_MESSAGE_V2; + + req->vport = apc->port_handle; + req->num_indir_entries = apc->indir_table_sz; + req->indir_tab_offset = offsetof(struct mana_cfg_rx_steer_req_v2, + indir_tab); + req->rx_enable = rx; + req->rss_enable = apc->rss_state; + req->update_default_rxobj = update_default_rxobj; + req->update_hashkey = update_key; + req->update_indir_tab = update_tab; + req->default_rxobj = apc->default_rxobj; + + /* Request-msg v5 requires this field */ + req->rss_hash_types = MANA_HASH_ENABLE_SUPPORTED; + + if (rx != TRI_STATE_FALSE) { + req->cqe_coalescing_enable = apc->cqe_coalescing_enable; + req->cqe8_coalescing_enable = apc->cqe8_coalescing_enable; + } + + if (update_key) + memcpy(&req->hashkey, apc->hashkey, MANA_HASH_KEY_SIZE); + + if (update_tab) + memcpy(req->indir_tab, apc->rxobj_table, + flex_array_size(req, indir_tab, req->num_indir_entries)); + + err = mana_send_request(apc->ac, req, req_buf_size, &resp, + sizeof(resp)); + if (err) { + if (mana_en_need_log(apc, err)) + netdev_err(ndev, "Failed to configure vPort RX: %d\n", err); + + goto out; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_CONFIG_VPORT_RX, + sizeof(resp)); + if (err) { + netdev_err(ndev, "vPort RX configuration failed: %d\n", err); + goto out; + } + + if (resp.hdr.status) { + netdev_err(ndev, "vPort RX configuration failed: 0x%x\n", + resp.hdr.status); + err = -EPROTO; + goto out; + } + + if (resp.hdr.response.msg_version >= GDMA_MESSAGE_V2) + apc->cqe_coalescing_timeout_ns = + resp.cqe_coalescing_timeout_ns; + + netdev_info(ndev, "Configured steering vPort %llu entries %u\n", + apc->port_handle, apc->indir_table_sz); + + apc->steer_rx = rx; + apc->steer_rss = apc->rss_state; + apc->steer_update_tab = update_tab; + apc->steer_cqe_coalescing = req->cqe_coalescing_enable; +out: + kfree(req); + return err; +} + +int mana_query_link_cfg(struct mana_port_context *apc) +{ + struct net_device *ndev = apc->ndev; + struct mana_query_link_config_resp resp = {}; + struct mana_query_link_config_req req = {}; + int err; + + netdev_assert_locked(ndev); + + err = apc->link_cfg_error; + if (err <= 0) + return err; + + mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_LINK_CONFIG, + sizeof(req), sizeof(resp)); + + req.vport = apc->port_handle; + req.hdr.resp.msg_version = GDMA_MESSAGE_V2; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + + if (err) { + if (err == -EOPNOTSUPP) { + netdev_info_once(ndev, "MANA_QUERY_LINK_CONFIG not supported\n"); + apc->link_cfg_error = err; + return err; + } + netdev_err(ndev, "Failed to query link config: %d\n", err); + return err; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_LINK_CONFIG, + sizeof(resp)); + + if (err || resp.hdr.status) { + netdev_err(ndev, "Failed to query link config: %d, 0x%x\n", err, + resp.hdr.status); + if (!err) + err = -EOPNOTSUPP; + return err; + } + + if (resp.qos_unconfigured) + return -EINVAL; + + apc->speed = resp.link_speed_mbps; + apc->max_speed = resp.qos_speed_mbps; + apc->link_cfg_error = 0; + return 0; +} + +int mana_set_bw_clamp(struct mana_port_context *apc, u32 speed, + int enable_clamping) +{ + struct mana_set_bw_clamp_resp resp = {}; + struct mana_set_bw_clamp_req req = {}; + struct net_device *ndev = apc->ndev; + int err; + + netdev_assert_locked(ndev); + + mana_gd_init_req_hdr(&req.hdr, MANA_SET_BW_CLAMP, + sizeof(req), sizeof(resp)); + req.vport = apc->port_handle; + req.link_speed_mbps = speed; + req.enable_clamping = enable_clamping; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + + if (err) { + if (err == -EOPNOTSUPP) { + netdev_info_once(ndev, "MANA_SET_BW_CLAMP not supported\n"); + return err; + } + netdev_err(ndev, "Failed to set bandwidth clamp for speed %u, err = %d", + speed, err); + return err; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_SET_BW_CLAMP, + sizeof(resp)); + + if (err || resp.hdr.status) { + netdev_err(ndev, "Failed to set bandwidth clamp: %d, 0x%x\n", err, + resp.hdr.status); + if (!err) + err = -EOPNOTSUPP; + return err; + } + + if (resp.qos_unconfigured) + netdev_info(ndev, "QoS is unconfigured\n"); + + /* Invalidate the cache; next query will re-fetch from firmware. */ + apc->link_cfg_error = 1; + return 0; +} + +int mana_create_wq_obj(struct mana_port_context *apc, + mana_handle_t vport, + u32 wq_type, struct mana_obj_spec *wq_spec, + struct mana_obj_spec *cq_spec, + mana_handle_t *wq_obj) +{ + struct mana_create_wqobj_resp resp = {}; + struct mana_create_wqobj_req req = {}; + struct net_device *ndev = apc->ndev; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_CREATE_WQ_OBJ, + sizeof(req), sizeof(resp)); + + /* Our driver uses different message versions for request and + * response in this case. + * Our firmware is forward compatible with newer message versions, so + * the old firmware still properly handles this message, just the new + * feature fields are ignored, and queue creation will be successful. + */ + req.hdr.req.msg_version = GDMA_MESSAGE_V3; + req.hdr.resp.msg_version = GDMA_MESSAGE_V2; + req.vport = vport; + req.wq_type = wq_type; + req.wq_gdma_region = wq_spec->gdma_region; + req.cq_gdma_region = cq_spec->gdma_region; + req.wq_size = wq_spec->queue_size; + req.cq_size = cq_spec->queue_size; + req.cq_moderation_ctx_id = cq_spec->modr_ctx_id; + req.cq_parent_qid = cq_spec->attached_eq; + req.req_cq_moderation = cq_spec->req_cq_moderation; + req.cq_moderation_comp = cq_spec->cq_moderation_comp; + req.cq_moderation_usec = cq_spec->cq_moderation_usec; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + netdev_err(ndev, "Failed to create WQ object: %d\n", err); + goto out; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_CREATE_WQ_OBJ, + sizeof(resp)); + if (err || resp.hdr.status) { + netdev_err(ndev, "Failed to create WQ object: %d, 0x%x\n", err, + resp.hdr.status); + if (!err) + err = -EPROTO; + goto out; + } + + if (resp.wq_obj == INVALID_MANA_HANDLE) { + netdev_err(ndev, "Got an invalid WQ object handle\n"); + err = -EPROTO; + goto out; + } + + *wq_obj = resp.wq_obj; + wq_spec->queue_index = resp.wq_id; + cq_spec->queue_index = resp.cq_id; + + return 0; +out: + return err; +} +EXPORT_SYMBOL_NS(mana_create_wq_obj, "NET_MANA"); + +void mana_destroy_wq_obj(struct mana_port_context *apc, u32 wq_type, + mana_handle_t wq_obj) +{ + struct mana_destroy_wqobj_resp resp = {}; + struct mana_destroy_wqobj_req req = {}; + struct net_device *ndev = apc->ndev; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_DESTROY_WQ_OBJ, + sizeof(req), sizeof(resp)); + req.wq_type = wq_type; + req.wq_obj_handle = wq_obj; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + if (mana_en_need_log(apc, err)) + netdev_err(ndev, "Failed to destroy WQ object: %d\n", err); + + return; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_DESTROY_WQ_OBJ, + sizeof(resp)); + if (err || resp.hdr.status) + netdev_err(ndev, "Failed to destroy WQ object: %d, 0x%x\n", err, + resp.hdr.status); +} +EXPORT_SYMBOL_NS(mana_destroy_wq_obj, "NET_MANA"); + +void mana_destroy_eq(struct mana_port_context *apc) +{ + struct mana_context *ac = apc->ac; + struct gdma_context *gc = ac->gdma_dev->gdma_context; + struct gdma_queue *eq; + unsigned int msi; + int i; + + if (!apc->eqs) + return; + + debugfs_remove_recursive(apc->mana_eqs_debugfs); + apc->mana_eqs_debugfs = NULL; + + for (i = 0; i < apc->num_queues; i++) { + eq = apc->eqs[i].eq; + if (!eq) + continue; + + msi = eq->eq.msix_index; + mana_gd_destroy_queue(gc, eq); + mana_gd_put_gic(gc, !gc->msi_sharing, msi); + } + + kfree(apc->eqs); + apc->eqs = NULL; +} +EXPORT_SYMBOL_NS(mana_destroy_eq, "NET_MANA"); + +static void mana_create_eq_debugfs(struct mana_port_context *apc, int i) +{ + struct mana_eq eq = apc->eqs[i]; + char eqnum[32]; + + sprintf(eqnum, "eq%d", i); + eq.mana_eq_debugfs = debugfs_create_dir(eqnum, apc->mana_eqs_debugfs); + debugfs_create_u32("head", 0400, eq.mana_eq_debugfs, &eq.eq->head); + debugfs_create_u32("tail", 0400, eq.mana_eq_debugfs, &eq.eq->tail); + debugfs_create_u32("irq", 0400, eq.mana_eq_debugfs, &eq.eq->eq.irq); + debugfs_create_file("eq_dump", 0400, eq.mana_eq_debugfs, eq.eq, &mana_dbg_q_fops); +} + +int mana_create_eq(struct mana_port_context *apc) +{ + struct gdma_dev *gd = apc->ac->gdma_dev; + struct gdma_context *gc = gd->gdma_context; + struct gdma_queue_spec spec = {}; + struct gdma_irq_context *gic; + int err; + int msi; + int i; + + if (WARN_ON(apc->eqs)) + return -EEXIST; + apc->eqs = kzalloc_objs(struct mana_eq, apc->num_queues); + if (!apc->eqs) + return -ENOMEM; + + spec.type = GDMA_EQ; + spec.monitor_avl_buf = false; + spec.queue_size = EQ_SIZE; + spec.eq.callback = NULL; + spec.eq.context = apc->eqs; + spec.eq.log2_throttle_limit = LOG2_EQ_THROTTLE; + + apc->mana_eqs_debugfs = + debugfs_create_dir("EQs", apc->mana_port_debugfs); + + for (i = 0; i < apc->num_queues; i++) { + msi = (i + 1) % gc->num_msix_usable; + + gic = mana_gd_get_gic(gc, !gc->msi_sharing, &msi); + if (IS_ERR(gic)) { + err = PTR_ERR(gic); + goto out; + } + spec.eq.msix_index = msi; + + err = mana_gd_create_mana_eq(gd, &spec, &apc->eqs[i].eq); + if (err) { + dev_err(gc->dev, "Failed to create EQ %d : %d\n", i, err); + mana_gd_put_gic(gc, !gc->msi_sharing, msi); + goto out; + } + apc->eqs[i].eq->eq.irq = gic->irq; + mana_create_eq_debugfs(apc, i); + } + + return 0; +out: + mana_destroy_eq(apc); + return err; +} +EXPORT_SYMBOL_NS(mana_create_eq, "NET_MANA"); + +static int mana_fence_rq(struct mana_port_context *apc, struct mana_rxq *rxq) +{ + struct mana_fence_rq_resp resp = {}; + struct mana_fence_rq_req req = {}; + int err; + + init_completion(&rxq->fence_event); + + mana_gd_init_req_hdr(&req.hdr, MANA_FENCE_RQ, + sizeof(req), sizeof(resp)); + req.wq_obj_handle = rxq->rxobj; + + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + netdev_err(apc->ndev, "Failed to fence RQ %u: %d\n", + rxq->rxq_idx, err); + return err; + } + + err = mana_verify_resp_hdr(&resp.hdr, MANA_FENCE_RQ, sizeof(resp)); + if (err || resp.hdr.status) { + netdev_err(apc->ndev, "Failed to fence RQ %u: %d, 0x%x\n", + rxq->rxq_idx, err, resp.hdr.status); + if (!err) + err = -EPROTO; + + return err; + } + + if (wait_for_completion_timeout(&rxq->fence_event, 10 * HZ) == 0) { + netdev_err(apc->ndev, "Failed to fence RQ %u: timed out\n", + rxq->rxq_idx); + return -ETIMEDOUT; + } + + return 0; +} + +static void mana_fence_rqs(struct mana_port_context *apc) +{ + unsigned int rxq_idx; + struct mana_rxq *rxq; + int err; + + if (!apc->rxqs) + return; + + for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { + rxq = apc->rxqs[rxq_idx]; + err = mana_fence_rq(apc, rxq); + + /* In case of any error, use sleep instead. */ + if (err) + msleep(100); + } +} + +static int mana_move_wq_tail(struct gdma_queue *wq, u32 num_units) +{ + u32 used_space_old; + u32 used_space_new; + + used_space_old = wq->head - wq->tail; + used_space_new = wq->head - (wq->tail + num_units); + + if (WARN_ON_ONCE(used_space_new > used_space_old)) + return -ERANGE; + + wq->tail += num_units; + return 0; +} + +void mana_unmap_skb(struct sk_buff *skb, struct mana_port_context *apc) +{ + struct mana_skb_head *ash = (struct mana_skb_head *)skb->head; + struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; + struct device *dev = gc->dev; + int hsg, i; + + /* Number of SGEs of linear part */ + hsg = (skb_is_gso(skb) && skb_headlen(skb) > ash->size[0]) ? 2 : 1; + + for (i = 0; i < hsg; i++) + dma_unmap_single(dev, ash->dma_handle[i], ash->size[i], + DMA_TO_DEVICE); + + for (i = hsg; i < skb_shinfo(skb)->nr_frags + hsg; i++) + dma_unmap_page(dev, ash->dma_handle[i], ash->size[i], + DMA_TO_DEVICE); +} + +static void mana_poll_tx_cq(struct mana_cq *cq) +{ + struct gdma_comp *completions = cq->gdma_comp_buf; + struct gdma_posted_wqe_info *wqe_info; + unsigned int pkt_transmitted = 0; + unsigned int wqe_unit_cnt = 0; + unsigned int tx_bytes = 0; + struct mana_txq *txq = cq->txq; + struct mana_port_context *apc; + struct netdev_queue *net_txq; + struct gdma_queue *gdma_wq; + unsigned int avail_space; + struct net_device *ndev; + struct sk_buff *skb; + bool txq_stopped; + int comp_read; + int i; + + ndev = txq->ndev; + apc = netdev_priv(ndev); + + /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the + * doorbell can be rung in time for the hardware's requirement + * of at least one doorbell ring every 8 wraparounds. + */ + comp_read = mana_gd_poll_cq(cq->gdma_cq, completions, + min((cq->gdma_cq->queue_size / + COMP_ENTRY_SIZE) * 4, + CQE_POLLING_BUFFER)); + + if (comp_read < 1) + return; + + for (i = 0; i < comp_read; i++) { + struct mana_tx_comp_oob *cqe_oob; + + if (WARN_ON_ONCE(!completions[i].is_sq)) + return; + + cqe_oob = (struct mana_tx_comp_oob *)completions[i].cqe_data; + if (WARN_ON_ONCE(cqe_oob->cqe_hdr.client_type != + MANA_CQE_COMPLETION)) + return; + + switch (cqe_oob->cqe_hdr.cqe_type) { + case CQE_TX_OKAY: + break; + + case CQE_TX_SA_DROP: + case CQE_TX_MTU_DROP: + case CQE_TX_INVALID_OOB: + case CQE_TX_INVALID_ETH_TYPE: + case CQE_TX_HDR_PROCESSING_ERROR: + case CQE_TX_VF_DISABLED: + case CQE_TX_VPORT_IDX_OUT_OF_RANGE: + case CQE_TX_VPORT_DISABLED: + case CQE_TX_VLAN_TAGGING_VIOLATION: + if (net_ratelimit()) + netdev_err(ndev, "TX: CQE error %d\n", + cqe_oob->cqe_hdr.cqe_type); + + apc->eth_stats.tx_cqe_err++; + break; + + default: + /* If the CQE type is unknown, log an error, + * and still free the SKB, update tail, etc. + */ + if (net_ratelimit()) + netdev_err(ndev, "TX: unknown CQE type %d\n", + cqe_oob->cqe_hdr.cqe_type); + + apc->eth_stats.tx_cqe_unknown_type++; + break; + } + + if (WARN_ON_ONCE(txq->gdma_txq_id != completions[i].wq_num)) + return; + + skb = skb_dequeue(&txq->pending_skbs); + if (WARN_ON_ONCE(!skb)) + return; + + wqe_info = (struct gdma_posted_wqe_info *)skb->cb; + wqe_unit_cnt += wqe_info->wqe_size_in_bu; + + mana_unmap_skb(skb, apc); + + tx_bytes += skb->len; + + napi_consume_skb(skb, cq->budget); + + pkt_transmitted++; + } + + if (WARN_ON_ONCE(wqe_unit_cnt == 0)) + return; + + mana_move_wq_tail(txq->gdma_sq, wqe_unit_cnt); + + gdma_wq = txq->gdma_sq; + avail_space = mana_gd_wq_avail_space(gdma_wq); + + /* Ensure tail updated before checking q stop */ + smp_mb(); + + net_txq = txq->net_txq; + txq_stopped = netif_tx_queue_stopped(net_txq); + + /* Ensure checking txq_stopped before apc->port_is_up. */ + smp_rmb(); + + if (txq_stopped && apc->port_is_up && avail_space >= MAX_TX_WQE_SIZE) { + netif_tx_wake_queue(net_txq); + apc->eth_stats.wake_queue++; + } + + if (atomic_sub_return(pkt_transmitted, &txq->pending_sends) < 0) + WARN_ON_ONCE(1); + + /* Feed DIM with the completion rate observed here, in NAPI context. */ + cq->tx_dim_pkts += pkt_transmitted; + cq->tx_dim_bytes += tx_bytes; + + cq->work_done = pkt_transmitted; +} + +static void mana_post_pkt_rxq(struct mana_rxq *rxq) +{ + struct mana_recv_buf_oob *recv_buf_oob; + u32 curr_index; + int err; + + curr_index = rxq->buf_index++; + if (rxq->buf_index == rxq->num_rx_buf) + rxq->buf_index = 0; + + recv_buf_oob = &rxq->rx_oobs[curr_index]; + + err = mana_gd_post_work_request(rxq->gdma_rq, &recv_buf_oob->wqe_req, + &recv_buf_oob->wqe_inf); + if (WARN_ON_ONCE(err)) + return; + + WARN_ON_ONCE(recv_buf_oob->wqe_inf.wqe_size_in_bu != 1); +} + +static struct sk_buff *mana_build_skb(struct mana_rxq *rxq, void *buf_va, + uint pkt_len, struct xdp_buff *xdp) +{ + struct sk_buff *skb = napi_build_skb(buf_va, rxq->alloc_size); + + if (!skb) + return NULL; + + if (xdp->data_hard_start) { + u32 metasize = xdp->data - xdp->data_meta; + + skb_reserve(skb, xdp->data - xdp->data_hard_start); + skb_put(skb, xdp->data_end - xdp->data); + if (metasize) + skb_metadata_set(skb, metasize); + return skb; + } + + skb_reserve(skb, rxq->headroom); + skb_put(skb, pkt_len); + + return skb; +} + +static void mana_rx_skb(void *buf_va, bool from_pool, + struct mana_rxcomp_oob *cqe, struct mana_rxq *rxq, + u32 pkt_len, u32 pkt_hash) +{ + struct mana_stats_rx *rx_stats = &rxq->stats; + struct net_device *ndev = rxq->ndev; + u16 rxq_idx = rxq->rxq_idx; + struct napi_struct *napi; + struct xdp_buff xdp = {}; + struct sk_buff *skb; + u32 act; + + rxq->rx_cq.work_done++; + napi = &rxq->rx_cq.napi; + + if (!buf_va) { + ++ndev->stats.rx_dropped; + return; + } + + act = mana_run_xdp(ndev, rxq, &xdp, buf_va, pkt_len); + + if (act == XDP_REDIRECT && !rxq->xdp_rc) + return; + + if (act != XDP_PASS && act != XDP_TX) + goto drop_xdp; + + skb = mana_build_skb(rxq, buf_va, pkt_len, &xdp); + + if (!skb) + goto drop; + + if (from_pool) + skb_mark_for_recycle(skb); + + skb->dev = napi->dev; + + skb->protocol = eth_type_trans(skb, ndev); + skb_checksum_none_assert(skb); + skb_record_rx_queue(skb, rxq_idx); + + if ((ndev->features & NETIF_F_RXCSUM) && cqe->rx_iphdr_csum_succeed) { + if (cqe->rx_tcp_csum_succeed || cqe->rx_udp_csum_succeed) + skb->ip_summed = CHECKSUM_UNNECESSARY; + } + + if (cqe->rx_hashtype != 0 && (ndev->features & NETIF_F_RXHASH)) { + if (cqe->rx_hashtype & MANA_HASH_L4) + skb_set_hash(skb, pkt_hash, PKT_HASH_TYPE_L4); + else + skb_set_hash(skb, pkt_hash, PKT_HASH_TYPE_L3); + } + + if (cqe->rx_vlantag_present) { + u16 vlan_tci = cqe->rx_vlan_id; + + __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tci); + } + + u64_stats_update_begin(&rx_stats->syncp); + rx_stats->packets++; + rx_stats->bytes += pkt_len; + + if (act == XDP_TX) + rx_stats->xdp_tx++; + u64_stats_update_end(&rx_stats->syncp); + + if (act == XDP_TX) { + skb_set_queue_mapping(skb, rxq_idx); + mana_xdp_tx(skb, ndev); + return; + } + + napi_gro_receive(napi, skb); + + return; + +drop_xdp: + u64_stats_update_begin(&rx_stats->syncp); + rx_stats->xdp_drop++; + u64_stats_update_end(&rx_stats->syncp); + +drop: + if (from_pool) { + if (rxq->frag_count == 1) + page_pool_recycle_direct(rxq->page_pool, + virt_to_head_page(buf_va)); + else + page_pool_free_va(rxq->page_pool, buf_va, true); + } else { + WARN_ON_ONCE(rxq->xdp_save_va); + /* Save for reuse */ + rxq->xdp_save_va = buf_va; + } + + ++ndev->stats.rx_dropped; + + return; +} + +static void *mana_get_rxfrag(struct mana_rxq *rxq, struct device *dev, + dma_addr_t *da, bool *from_pool, + struct page **pp_page, u32 *dma_sync_offset) +{ + struct page *page; + u32 offset; + void *va; + + *from_pool = false; + *pp_page = NULL; + *dma_sync_offset = 0; + + /* Don't use fragments for jumbo frames or XDP where it's 1 fragment + * per page. + */ + if (rxq->frag_count == 1) { + /* Reuse XDP dropped page if available */ + if (rxq->xdp_save_va) { + va = rxq->xdp_save_va; + page = virt_to_head_page(va); + rxq->xdp_save_va = NULL; + } else { + page = page_pool_dev_alloc_pages(rxq->page_pool); + if (!page) + return NULL; + + *from_pool = true; + va = page_to_virt(page); + } + + *da = dma_map_single(dev, va + rxq->headroom, rxq->datasize, + DMA_FROM_DEVICE); + if (dma_mapping_error(dev, *da)) { + mana_put_rx_page(rxq, page, *from_pool); + return NULL; + } + + return va; + } + + page = page_pool_dev_alloc_frag(rxq->page_pool, &offset, + rxq->alloc_size); + if (!page) + return NULL; + + va = page_to_virt(page) + offset; + *da = page_pool_get_dma_addr(page) + offset + rxq->headroom; + *from_pool = true; + *pp_page = page; + *dma_sync_offset = offset + rxq->headroom; + + return va; +} + +/* Allocate frag for rx buffer, and save the old buf */ +static void mana_refill_rx_oob(struct device *dev, struct mana_rxq *rxq, + struct mana_recv_buf_oob *rxoob, u32 pktlen, + void **old_buf, bool *old_fp) +{ + struct page *pp_page; + u32 dma_sync_offset; + bool from_pool; + dma_addr_t da; + void *va; + + va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, + &dma_sync_offset); + if (!va) + return; + if (!rxoob->from_pool || rxq->frag_count == 1) { + dma_unmap_single(dev, rxoob->sgl[0].address, rxq->datasize, + DMA_FROM_DEVICE); + } else { + /* The page pool maps the whole page and only syncs for device + * automatically (PP_FLAG_DMA_SYNC_DEV). Sync the received bytes + * for the CPU before they are read: this is required if DMA + * is incoherent or bounce buffers are used. + */ + page_pool_dma_sync_for_cpu(rxq->page_pool, rxoob->pp_page, + rxoob->dma_sync_offset, pktlen); + } + *old_buf = rxoob->buf_va; + *old_fp = rxoob->from_pool; + + rxoob->buf_va = va; + rxoob->sgl[0].address = da; + rxoob->from_pool = from_pool; + rxoob->pp_page = pp_page; + rxoob->dma_sync_offset = dma_sync_offset; +} + +static void mana_process_one_rx_pkt(struct device *dev, struct mana_rxq *rxq, + struct mana_rxcomp_oob *oob, + u32 pktlen, u32 pkt_hash) +{ + struct mana_recv_buf_oob *rxbuf_oob; + struct net_device *ndev = rxq->ndev; + void *old_buf = NULL; + bool old_fp; + + rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; + WARN_ON_ONCE(rxbuf_oob->wqe_inf.wqe_size_in_bu != 1); + + if (unlikely(pktlen > rxq->datasize)) { + /* Increase it even if mana_rx_skb() isn't called. */ + rxq->rx_cq.work_done++; + + ++ndev->stats.rx_dropped; + netdev_warn_once(ndev, + "Dropped oversized RX packet: len=%u, datasize=%u\n", + pktlen, rxq->datasize); + + /* Reuse the RX buffer since rxbuf_oob is unchanged. */ + } else { + mana_refill_rx_oob(dev, rxq, rxbuf_oob, pktlen, + &old_buf, &old_fp); + + /* Unsuccessful refill will have old_buf == NULL. + * In this case, mana_rx_skb() will drop the packet. + */ + mana_rx_skb(old_buf, old_fp, oob, rxq, pktlen, pkt_hash); + } + + mana_move_wq_tail(rxq->gdma_rq, rxbuf_oob->wqe_inf.wqe_size_in_bu); + + mana_post_pkt_rxq(rxq); +} + +static void mana_process_rx_cqe(struct mana_rxq *rxq, struct mana_cq *cq, + struct gdma_comp *cqe) +{ + struct mana_rxcomp_oob *oob = (struct mana_rxcomp_oob *)cqe->cqe_data; + struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; + struct net_device *ndev = rxq->ndev; + struct mana_recv_buf_oob *rxbuf_oob; + struct mana_port_context *apc; + struct device *dev = gc->dev; + bool coalesced_8 = false; + bool coalesced = false; + u32 pktlen; + int pkt_i; + int i; + + apc = netdev_priv(ndev); + + switch (oob->cqe_hdr.cqe_type) { + case CQE_RX_OKAY: + break; + + case CQE_RX_TRUNCATED: + ++ndev->stats.rx_dropped; + rxbuf_oob = &rxq->rx_oobs[rxq->buf_index]; + netdev_warn_once(ndev, "Dropped a truncated packet\n"); + + mana_move_wq_tail(rxq->gdma_rq, + rxbuf_oob->wqe_inf.wqe_size_in_bu); + mana_post_pkt_rxq(rxq); + return; + + case CQE_RX_COALESCED_4: + coalesced = true; + break; + + case CQE_RX_COALESCED_8: + coalesced = true; + coalesced_8 = true; + break; + + case CQE_RX_OBJECT_FENCE: + complete(&rxq->fence_event); + return; + + default: + netdev_err(ndev, "Unknown RX CQE type = %d\n", + oob->cqe_hdr.cqe_type); + apc->eth_stats.rx_cqe_unknown_type++; + return; + } + + pkt_i = 0; + for (i = 0; i < MANA_RXCOMP_OOB_NUM_PPI; i++) { + u32 pkt_hash; + + if (coalesced_8) { + /* 8-pkt mode: 2 packets per PPI entry */ + pktlen = oob->ppi[i].pkt_len0; + pkt_hash = oob->ppi[i].pkt_hash0; + } else { + pktlen = oob->ppi[i].pkt_len; + pkt_hash = oob->ppi[i].pkt_hash; + } + if (pktlen == 0) + break; + + mana_process_one_rx_pkt(dev, rxq, oob, pktlen, pkt_hash); + pkt_i++; + + if (!coalesced) + break; + + /* Process 2nd packet from the same PPI in 8-pkt mode */ + if (coalesced_8) { + pktlen = oob->ppi[i].pkt_len1; + pkt_hash = oob->ppi[i].pkt_hash1; + if (pktlen == 0) + break; + + mana_process_one_rx_pkt(dev, rxq, oob, pktlen, + pkt_hash); + pkt_i++; + } + } + + /* Collect coalesced CQE count based on packets processed. + * Coalesced CQEs have at least 2 packets, so index is pkt_i - 2. + */ + if (pkt_i > 1) { + u64_stats_update_begin(&rxq->stats.syncp); + rxq->stats.coalesced_cqe[pkt_i - 2]++; + u64_stats_update_end(&rxq->stats.syncp); + } else if (!pkt_i && !pktlen) { + u64_stats_update_begin(&rxq->stats.syncp); + rxq->stats.pkt_len0_err++; + u64_stats_update_end(&rxq->stats.syncp); + netdev_err_once(ndev, + "RX pkt len=0, rq=%u, cq=%u, rxobj=0x%llx\n", + rxq->gdma_id, cq->gdma_id, rxq->rxobj); + } +} + +static void mana_poll_rx_cq(struct mana_cq *cq) +{ + struct gdma_comp *comp = cq->gdma_comp_buf; + struct mana_rxq *rxq = cq->rxq; + int comp_read, i; + + /* Limit CQEs polled to 4 wraparounds of the CQ to ensure the + * doorbell can be rung in time for the hardware's requirement + * of at least one doorbell ring every 8 wraparounds. + */ + comp_read = mana_gd_poll_cq(cq->gdma_cq, comp, + min((cq->gdma_cq->queue_size / + COMP_ENTRY_SIZE) * 4, + CQE_POLLING_BUFFER)); + WARN_ON_ONCE(comp_read > CQE_POLLING_BUFFER); + + rxq->xdp_flush = false; + + for (i = 0; i < comp_read; i++) { + if (WARN_ON_ONCE(comp[i].is_sq)) + return; + + /* verify recv cqe references the right rxq */ + if (WARN_ON_ONCE(comp[i].wq_num != cq->rxq->gdma_id)) + return; + + mana_process_rx_cqe(rxq, cq, &comp[i]); + } + + if (comp_read > 0) { + struct gdma_context *gc = rxq->gdma_rq->gdma_dev->gdma_context; + + mana_gd_wq_ring_doorbell(gc, rxq->gdma_rq); + } + + if (rxq->xdp_flush) + xdp_do_flush(); +} + +static void mana_rx_dim_work(struct work_struct *work) +{ + struct dim *dim = container_of(work, struct dim, work); + struct dim_cq_moder cur_moder; + struct mana_cq *cq; + + cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); + cq = container_of(dim, struct mana_cq, dim); + + cur_moder.usec = min_t(u16, cur_moder.usec, MANA_INTR_MODR_USEC_MAX); + cur_moder.pkts = min_t(u16, cur_moder.pkts, MANA_INTR_MODR_COMP_MAX); + + mana_gd_ring_dim(cq->gdma_cq, cur_moder.usec, true, + cur_moder.pkts, true); + + dim->state = DIM_START_MEASURE; +} + +static void mana_tx_dim_work(struct work_struct *work) +{ + struct dim *dim = container_of(work, struct dim, work); + struct dim_cq_moder cur_moder; + struct mana_cq *cq; + + cur_moder = net_dim_get_tx_moderation(dim->mode, dim->profile_ix); + cq = container_of(dim, struct mana_cq, dim); + + cur_moder.usec = min_t(u16, cur_moder.usec, MANA_INTR_MODR_USEC_MAX); + cur_moder.pkts = min_t(u16, cur_moder.pkts, MANA_INTR_MODR_COMP_MAX); + + mana_gd_ring_dim(cq->gdma_cq, cur_moder.usec, true, + cur_moder.pkts, true); + + dim->state = DIM_START_MEASURE; +} + +/* The caller must update apc->rx/tx_dim_enabled before disabling and + * after enabling. And synchronize_net() before draining the DIM work, + * so that NAPI cannot observe a stale flag. + */ +void mana_dim_change(struct mana_cq *cq, bool enable) +{ + bool is_rx = cq->type == MANA_CQ_TYPE_RX; + struct mana_port_context *apc; + work_func_t work_func; + u32 usec, comp; + + if (is_rx) { + apc = netdev_priv(cq->rxq->ndev); + usec = apc->intr_modr_rx_usec; + comp = apc->intr_modr_rx_comp; + work_func = mana_rx_dim_work; + } else { + apc = netdev_priv(cq->txq->ndev); + usec = apc->intr_modr_tx_usec; + comp = apc->intr_modr_tx_comp; + work_func = mana_tx_dim_work; + } + + /* On enable, zero the DIM state so net_dim() starts measuring from + * scratch. + * On disable, drain any pending DIM work and restore the static + * moderation values. + */ + if (enable) { + memset(&cq->dim, 0, sizeof(cq->dim)); + cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + INIT_WORK(&cq->dim.work, work_func); + } else { + cancel_work_sync(&cq->dim.work); + mana_gd_ring_dim(cq->gdma_cq, usec, true, comp, true); + } +} + +static void mana_update_rx_dim(struct mana_cq *cq) +{ + struct mana_port_context *apc = netdev_priv(cq->rxq->ndev); + struct dim_sample dim_sample = {}; + struct mana_rxq *rxq = cq->rxq; + + /* Pairs with smp_store_release() in mana_set_coalesce(): observing the + * enable flag set guarantees the DIM (re)initialization is visible. + */ + if (!smp_load_acquire(&apc->rx_dim_enabled)) + return; + + dim_update_sample(READ_ONCE(cq->dim_event_ctr), rxq->stats.packets, + rxq->stats.bytes, &dim_sample); + net_dim(&cq->dim, &dim_sample); +} + +static void mana_update_tx_dim(struct mana_cq *cq) +{ + struct mana_port_context *apc = netdev_priv(cq->txq->ndev); + struct dim_sample dim_sample = {}; + + /* Pairs with smp_store_release() in mana_set_coalesce(): observing the + * enable flag set guarantees the DIM (re)initialization is visible. + */ + if (!smp_load_acquire(&apc->tx_dim_enabled)) + return; + + /* cq->tx_dim_pkts/bytes are accumulated in mana_poll_tx_cq(), in the + * same NAPI context as this read, so they track the hardware + * completion rate and need no u64_stats_sync protection. + */ + dim_update_sample(READ_ONCE(cq->dim_event_ctr), cq->tx_dim_pkts, + cq->tx_dim_bytes, &dim_sample); + net_dim(&cq->dim, &dim_sample); +} + +static int mana_cq_handler(void *context, struct gdma_queue *gdma_queue) +{ + struct mana_cq *cq = context; + int w; + + WARN_ON_ONCE(cq->gdma_cq != gdma_queue); + + if (cq->type == MANA_CQ_TYPE_RX) + mana_poll_rx_cq(cq); + else + mana_poll_tx_cq(cq); + + w = cq->work_done; + cq->work_done_since_doorbell += w; + + if (w < cq->budget) { + mana_gd_ring_cq(gdma_queue, SET_ARM_BIT); + cq->work_done_since_doorbell = 0; + + /* Update DIM before napi_complete_done() to prevent running + * net_dim() concurrently. + */ + if (cq->type == MANA_CQ_TYPE_RX) + mana_update_rx_dim(cq); + else + mana_update_tx_dim(cq); + + napi_complete_done(&cq->napi, w); + } else if (cq->work_done_since_doorbell >= + (cq->gdma_cq->queue_size / COMP_ENTRY_SIZE) * 4) { + /* MANA hardware requires at least one doorbell ring every 8 + * wraparounds of CQ even if there is no need to arm the CQ. + * This driver rings the doorbell as soon as it has processed + * 4 wraparounds. + */ + mana_gd_ring_cq(gdma_queue, 0); + cq->work_done_since_doorbell = 0; + } + + return w; +} + +static int mana_poll(struct napi_struct *napi, int budget) +{ + struct mana_cq *cq = container_of(napi, struct mana_cq, napi); + int w; + + cq->work_done = 0; + cq->budget = budget; + + w = mana_cq_handler(cq, cq->gdma_cq); + + return min(w, budget); +} + +static void mana_schedule_napi(void *context, struct gdma_queue *gdma_queue) +{ + struct mana_cq *cq = context; + + WRITE_ONCE(cq->dim_event_ctr, cq->dim_event_ctr + 1); + napi_schedule_irqoff(&cq->napi); +} + +static void mana_deinit_cq(struct mana_port_context *apc, struct mana_cq *cq) +{ + struct gdma_dev *gd = apc->ac->gdma_dev; + + if (!cq->gdma_cq) + return; + + mana_gd_destroy_queue(gd->gdma_context, cq->gdma_cq); +} + +static void mana_deinit_txq(struct mana_port_context *apc, struct mana_txq *txq) +{ + struct gdma_dev *gd = apc->ac->gdma_dev; + + if (!txq->gdma_sq) + return; + + mana_gd_destroy_queue(gd->gdma_context, txq->gdma_sq); +} + +static void mana_destroy_txq(struct mana_port_context *apc) +{ + struct napi_struct *napi; + int i; + + if (!apc->tx_qp) + return; + + for (i = 0; i < apc->num_queues; i++) { + if (!apc->tx_qp[i]) + continue; + + debugfs_remove_recursive(apc->tx_qp[i]->mana_tx_debugfs); + apc->tx_qp[i]->mana_tx_debugfs = NULL; + + napi = &apc->tx_qp[i]->tx_cq.napi; + if (apc->tx_qp[i]->txq.napi_initialized) { + napi_synchronize(napi); + napi_disable_locked(napi); + cancel_work_sync(&apc->tx_qp[i]->tx_cq.dim.work); + netif_napi_del_locked(napi); + apc->tx_qp[i]->txq.napi_initialized = false; + } + + if (apc->tx_qp[i]->tx_object != INVALID_MANA_HANDLE) + mana_destroy_wq_obj(apc, GDMA_SQ, apc->tx_qp[i]->tx_object); + + mana_deinit_cq(apc, &apc->tx_qp[i]->tx_cq); + + mana_deinit_txq(apc, &apc->tx_qp[i]->txq); + + kvfree(apc->tx_qp[i]); + } + + kfree(apc->tx_qp); + apc->tx_qp = NULL; +} + +static void mana_create_txq_debugfs(struct mana_port_context *apc, int idx) +{ + struct mana_tx_qp *tx_qp = apc->tx_qp[idx]; + char qnum[32]; + + sprintf(qnum, "TX-%d", idx); + tx_qp->mana_tx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); + debugfs_create_u32("sq_head", 0400, tx_qp->mana_tx_debugfs, + &tx_qp->txq.gdma_sq->head); + debugfs_create_u32("sq_tail", 0400, tx_qp->mana_tx_debugfs, + &tx_qp->txq.gdma_sq->tail); + debugfs_create_u32("sq_pend_skb_qlen", 0400, tx_qp->mana_tx_debugfs, + &tx_qp->txq.pending_skbs.qlen); + debugfs_create_u32("cq_head", 0400, tx_qp->mana_tx_debugfs, + &tx_qp->tx_cq.gdma_cq->head); + debugfs_create_u32("cq_tail", 0400, tx_qp->mana_tx_debugfs, + &tx_qp->tx_cq.gdma_cq->tail); + debugfs_create_u32("cq_budget", 0400, tx_qp->mana_tx_debugfs, + &tx_qp->tx_cq.budget); + debugfs_create_file("txq_dump", 0400, tx_qp->mana_tx_debugfs, + tx_qp->txq.gdma_sq, &mana_dbg_q_fops); + debugfs_create_file("cq_dump", 0400, tx_qp->mana_tx_debugfs, + tx_qp->tx_cq.gdma_cq, &mana_dbg_q_fops); +} + +static int mana_create_txq(struct mana_port_context *apc, + struct net_device *net) +{ + struct mana_context *ac = apc->ac; + struct gdma_dev *gd = ac->gdma_dev; + struct mana_obj_spec wq_spec; + struct mana_obj_spec cq_spec; + struct gdma_queue_spec spec; + struct gdma_context *gc; + struct mana_txq *txq; + struct mana_cq *cq; + u32 txq_size; + u32 cq_size; + int err; + int i; + + apc->tx_qp = kzalloc_objs(struct mana_tx_qp *, apc->num_queues); + if (!apc->tx_qp) + return -ENOMEM; + + /* The minimum size of the WQE is 32 bytes, hence + * apc->tx_queue_size represents the maximum number of WQEs + * the SQ can store. This value is then used to size other queues + * to prevent overflow. + * Also note that the txq_size is always going to be MANA_PAGE_ALIGNED, + * as min val of apc->tx_queue_size is 128 and that would make + * txq_size 128*32 = 4096 and the other higher values of apc->tx_queue_size + * are always power of two + */ + txq_size = apc->tx_queue_size * 32; + + cq_size = apc->tx_queue_size * COMP_ENTRY_SIZE; + + gc = gd->gdma_context; + + for (i = 0; i < apc->num_queues; i++) { + apc->tx_qp[i] = kvzalloc_obj(*apc->tx_qp[i]); + if (!apc->tx_qp[i]) { + err = -ENOMEM; + goto out; + } + + apc->tx_qp[i]->tx_object = INVALID_MANA_HANDLE; + + /* Create SQ */ + txq = &apc->tx_qp[i]->txq; + + u64_stats_init(&txq->stats.syncp); + txq->ndev = net; + txq->net_txq = netdev_get_tx_queue(net, i); + txq->vp_offset = apc->tx_vp_offset; + txq->napi_initialized = false; + skb_queue_head_init(&txq->pending_skbs); + + memset(&spec, 0, sizeof(spec)); + spec.type = GDMA_SQ; + spec.monitor_avl_buf = true; + spec.queue_size = txq_size; + err = mana_gd_create_mana_wq_cq(gd, &spec, &txq->gdma_sq); + if (err) + goto out; + + /* Create SQ's CQ */ + cq = &apc->tx_qp[i]->tx_cq; + cq->type = MANA_CQ_TYPE_TX; + + cq->txq = txq; + + memset(&spec, 0, sizeof(spec)); + spec.type = GDMA_CQ; + spec.monitor_avl_buf = false; + spec.queue_size = cq_size; + spec.cq.callback = mana_schedule_napi; + spec.cq.parent_eq = apc->eqs[i].eq; + spec.cq.context = cq; + err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); + if (err) + goto out; + + memset(&wq_spec, 0, sizeof(wq_spec)); + memset(&cq_spec, 0, sizeof(cq_spec)); + + wq_spec.gdma_region = txq->gdma_sq->mem_info.dma_region_handle; + wq_spec.queue_size = txq->gdma_sq->queue_size; + + cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; + cq_spec.queue_size = cq->gdma_cq->queue_size; + cq_spec.modr_ctx_id = 0; + cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; + + /* DIM setting can be changed at runtime */ + cq_spec.req_cq_moderation = true; + cq_spec.cq_moderation_usec = apc->intr_modr_tx_usec; + cq_spec.cq_moderation_comp = apc->intr_modr_tx_comp; + + err = mana_create_wq_obj(apc, apc->port_handle, GDMA_SQ, + &wq_spec, &cq_spec, + &apc->tx_qp[i]->tx_object); + + if (err) + goto out; + + txq->gdma_sq->id = wq_spec.queue_index; + cq->gdma_cq->id = cq_spec.queue_index; + + txq->gdma_sq->mem_info.dma_region_handle = + GDMA_INVALID_DMA_REGION; + cq->gdma_cq->mem_info.dma_region_handle = + GDMA_INVALID_DMA_REGION; + + txq->gdma_txq_id = txq->gdma_sq->id; + + cq->gdma_id = cq->gdma_cq->id; + + if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { + err = -EINVAL; + goto out; + } + + gc->cq_table[cq->gdma_id] = cq->gdma_cq; + + mana_create_txq_debugfs(apc, i); + + set_bit(NAPI_STATE_NO_BUSY_POLL, &cq->napi.state); + netif_napi_add_locked(net, &cq->napi, mana_poll); + + /* Initialize the DIM work before enabling NAPI, so that a poll + * cannot reach net_dim() with an uninitialized cq->dim.work. + */ + INIT_WORK(&cq->dim.work, mana_tx_dim_work); + cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + + napi_enable_locked(&cq->napi); + txq->napi_initialized = true; + + mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); + } + + return 0; +out: + netdev_err(net, "Failed to create %d TX queues, %d\n", + apc->num_queues, err); + mana_destroy_txq(apc); + return err; +} + +static void mana_destroy_rxq(struct mana_port_context *apc, + struct mana_rxq *rxq, bool napi_initialized) + +{ + struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; + struct mana_recv_buf_oob *rx_oob; + struct device *dev = gc->dev; + struct napi_struct *napi; + struct page *page; + int i; + + if (!rxq) + return; + + debugfs_remove_recursive(rxq->mana_rx_debugfs); + rxq->mana_rx_debugfs = NULL; + + napi = &rxq->rx_cq.napi; + + if (napi_initialized) { + napi_synchronize(napi); + + napi_disable_locked(napi); + cancel_work_sync(&rxq->rx_cq.dim.work); + netif_napi_del_locked(napi); + } + + if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) + xdp_rxq_info_unreg(&rxq->xdp_rxq); + + if (rxq->rxobj != INVALID_MANA_HANDLE) + mana_destroy_wq_obj(apc, GDMA_RQ, rxq->rxobj); + + mana_deinit_cq(apc, &rxq->rx_cq); + + if (rxq->xdp_save_va) + put_page(virt_to_head_page(rxq->xdp_save_va)); + + for (i = 0; i < rxq->num_rx_buf; i++) { + rx_oob = &rxq->rx_oobs[i]; + + if (!rx_oob->buf_va) + continue; + + page = virt_to_head_page(rx_oob->buf_va); + + if (rxq->frag_count == 1 || !rx_oob->from_pool) { + dma_unmap_single(dev, rx_oob->sgl[0].address, + rx_oob->sgl[0].size, DMA_FROM_DEVICE); + mana_put_rx_page(rxq, page, rx_oob->from_pool); + } else { + page_pool_free_va(rxq->page_pool, rx_oob->buf_va, true); + } + + rx_oob->buf_va = NULL; + } + + page_pool_destroy(rxq->page_pool); + + if (rxq->gdma_rq) + mana_gd_destroy_queue(gc, rxq->gdma_rq); + + kvfree(rxq); +} + +static int mana_fill_rx_oob(struct mana_recv_buf_oob *rx_oob, u32 mem_key, + struct mana_rxq *rxq, struct device *dev) +{ + struct mana_port_context *mpc = netdev_priv(rxq->ndev); + struct page *pp_page = NULL; + u32 dma_sync_offset = 0; + bool from_pool = false; + dma_addr_t da; + void *va; + + if (mpc->rxbufs_pre) + va = mana_get_rxbuf_pre(rxq, &da); + else + va = mana_get_rxfrag(rxq, dev, &da, &from_pool, &pp_page, + &dma_sync_offset); + + if (!va) + return -ENOMEM; + + rx_oob->buf_va = va; + rx_oob->from_pool = from_pool; + rx_oob->pp_page = pp_page; + rx_oob->dma_sync_offset = dma_sync_offset; + + rx_oob->sgl[0].address = da; + rx_oob->sgl[0].size = rxq->datasize; + rx_oob->sgl[0].mem_key = mem_key; + + return 0; +} + +#define MANA_WQE_HEADER_SIZE 16 +#define MANA_WQE_SGE_SIZE 16 + +static int mana_alloc_rx_wqe(struct mana_port_context *apc, + struct mana_rxq *rxq, u32 *rxq_size, u32 *cq_size) +{ + struct gdma_context *gc = apc->ac->gdma_dev->gdma_context; + struct mana_recv_buf_oob *rx_oob; + struct device *dev = gc->dev; + u32 buf_idx; + int ret; + + WARN_ON(rxq->datasize == 0); + + *rxq_size = 0; + *cq_size = 0; + + for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { + rx_oob = &rxq->rx_oobs[buf_idx]; + memset(rx_oob, 0, sizeof(*rx_oob)); + + rx_oob->num_sge = 1; + + ret = mana_fill_rx_oob(rx_oob, apc->ac->gdma_dev->gpa_mkey, rxq, + dev); + if (ret) + return ret; + + rx_oob->wqe_req.sgl = rx_oob->sgl; + rx_oob->wqe_req.num_sge = rx_oob->num_sge; + rx_oob->wqe_req.inline_oob_size = 0; + rx_oob->wqe_req.inline_oob_data = NULL; + rx_oob->wqe_req.flags = 0; + rx_oob->wqe_req.client_data_unit = 0; + + *rxq_size += ALIGN(MANA_WQE_HEADER_SIZE + + MANA_WQE_SGE_SIZE * rx_oob->num_sge, 32); + *cq_size += COMP_ENTRY_SIZE; + } + + return 0; +} + +static int mana_push_wqe(struct mana_rxq *rxq) +{ + struct mana_recv_buf_oob *rx_oob; + u32 buf_idx; + int err; + + for (buf_idx = 0; buf_idx < rxq->num_rx_buf; buf_idx++) { + rx_oob = &rxq->rx_oobs[buf_idx]; + + err = mana_gd_post_and_ring(rxq->gdma_rq, &rx_oob->wqe_req, + &rx_oob->wqe_inf); + if (err) + return -ENOSPC; + } + + return 0; +} + +static int mana_create_page_pool(struct mana_rxq *rxq, struct gdma_context *gc) +{ + struct mana_port_context *mpc = netdev_priv(rxq->ndev); + struct page_pool_params pprm = {}; + int ret; + + pprm.pool_size = mpc->rx_queue_size / rxq->frag_count + 1; + pprm.nid = gc->numa_node; + pprm.napi = &rxq->rx_cq.napi; + pprm.netdev = rxq->ndev; + pprm.order = get_order(rxq->alloc_size); + pprm.queue_idx = rxq->rxq_idx; + pprm.dev = gc->dev; + + /* Let the page pool do the dma map when page sharing with multiple + * fragments enabled for rx buffers. + */ + if (rxq->frag_count > 1) { + pprm.flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV; + pprm.max_len = PAGE_SIZE; + pprm.dma_dir = DMA_FROM_DEVICE; + } + + rxq->page_pool = page_pool_create(&pprm); + + if (IS_ERR(rxq->page_pool)) { + ret = PTR_ERR(rxq->page_pool); + rxq->page_pool = NULL; + return ret; + } + + return 0; +} + +static struct mana_rxq *mana_create_rxq(struct mana_port_context *apc, + u32 rxq_idx, struct mana_eq *eq, + struct net_device *ndev) +{ + struct gdma_dev *gd = apc->ac->gdma_dev; + struct mana_obj_spec wq_spec; + struct mana_obj_spec cq_spec; + struct gdma_queue_spec spec; + struct mana_cq *cq = NULL; + struct gdma_context *gc; + u32 cq_size, rq_size; + struct mana_rxq *rxq; + int err; + + gc = gd->gdma_context; + + rxq = kvzalloc_flex(*rxq, rx_oobs, apc->rx_queue_size); + if (!rxq) + return ERR_PTR(-ENOMEM); + + rxq->ndev = ndev; + rxq->num_rx_buf = apc->rx_queue_size; + rxq->rxq_idx = rxq_idx; + rxq->rxobj = INVALID_MANA_HANDLE; + + mana_get_rxbuf_cfg(apc, ndev->mtu, &rxq->datasize, &rxq->alloc_size, + &rxq->headroom, &rxq->frag_count); + /* Create page pool for RX queue */ + err = mana_create_page_pool(rxq, gc); + if (err) { + netdev_err(ndev, "Create page pool err:%d\n", err); + goto out; + } + + err = mana_alloc_rx_wqe(apc, rxq, &rq_size, &cq_size); + if (err) + goto out; + + rq_size = MANA_PAGE_ALIGN(rq_size); + cq_size = MANA_PAGE_ALIGN(cq_size); + + /* Create RQ */ + memset(&spec, 0, sizeof(spec)); + spec.type = GDMA_RQ; + spec.monitor_avl_buf = true; + spec.queue_size = rq_size; + err = mana_gd_create_mana_wq_cq(gd, &spec, &rxq->gdma_rq); + if (err) + goto out; + + /* Create RQ's CQ */ + cq = &rxq->rx_cq; + cq->type = MANA_CQ_TYPE_RX; + cq->rxq = rxq; + + memset(&spec, 0, sizeof(spec)); + spec.type = GDMA_CQ; + spec.monitor_avl_buf = false; + spec.queue_size = cq_size; + spec.cq.callback = mana_schedule_napi; + spec.cq.parent_eq = eq->eq; + spec.cq.context = cq; + err = mana_gd_create_mana_wq_cq(gd, &spec, &cq->gdma_cq); + if (err) + goto out; + + memset(&wq_spec, 0, sizeof(wq_spec)); + memset(&cq_spec, 0, sizeof(cq_spec)); + wq_spec.gdma_region = rxq->gdma_rq->mem_info.dma_region_handle; + wq_spec.queue_size = rxq->gdma_rq->queue_size; + + cq_spec.gdma_region = cq->gdma_cq->mem_info.dma_region_handle; + cq_spec.queue_size = cq->gdma_cq->queue_size; + cq_spec.modr_ctx_id = 0; + cq_spec.attached_eq = cq->gdma_cq->cq.parent->id; + + /* DIM setting can be changed at runtime */ + cq_spec.req_cq_moderation = true; + cq_spec.cq_moderation_usec = apc->intr_modr_rx_usec; + cq_spec.cq_moderation_comp = apc->intr_modr_rx_comp; + + err = mana_create_wq_obj(apc, apc->port_handle, GDMA_RQ, + &wq_spec, &cq_spec, &rxq->rxobj); + if (err) + goto out; + + rxq->gdma_rq->id = wq_spec.queue_index; + cq->gdma_cq->id = cq_spec.queue_index; + + rxq->gdma_rq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; + cq->gdma_cq->mem_info.dma_region_handle = GDMA_INVALID_DMA_REGION; + + rxq->gdma_id = rxq->gdma_rq->id; + cq->gdma_id = cq->gdma_cq->id; + + err = mana_push_wqe(rxq); + if (err) + goto out; + + if (WARN_ON(cq->gdma_id >= gc->max_num_cqs)) { + err = -EINVAL; + goto out; + } + + gc->cq_table[cq->gdma_id] = cq->gdma_cq; + + netif_napi_add_weight_locked(ndev, &cq->napi, mana_poll, 1); + + WARN_ON(xdp_rxq_info_reg(&rxq->xdp_rxq, ndev, rxq_idx, + cq->napi.napi_id)); + WARN_ON(xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, + rxq->page_pool)); + + /* Initialize the DIM work before enabling NAPI, so that a poll + * cannot reach net_dim() with an uninitialized cq->dim.work. + */ + INIT_WORK(&cq->dim.work, mana_rx_dim_work); + cq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; + + napi_enable_locked(&cq->napi); + + mana_gd_ring_cq(cq->gdma_cq, SET_ARM_BIT); +out: + if (!err) + return rxq; + + netdev_err(ndev, "Failed to create RXQ: err = %d\n", err); + + mana_destroy_rxq(apc, rxq, false); + + return ERR_PTR(err); +} + +static void mana_create_rxq_debugfs(struct mana_port_context *apc, int idx) +{ + struct mana_rxq *rxq; + char qnum[32]; + + rxq = apc->rxqs[idx]; + + sprintf(qnum, "RX-%d", idx); + rxq->mana_rx_debugfs = debugfs_create_dir(qnum, apc->mana_port_debugfs); + debugfs_create_u32("rq_head", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->head); + debugfs_create_u32("rq_tail", 0400, rxq->mana_rx_debugfs, &rxq->gdma_rq->tail); + debugfs_create_u32("rq_nbuf", 0400, rxq->mana_rx_debugfs, &rxq->num_rx_buf); + debugfs_create_u32("cq_head", 0400, rxq->mana_rx_debugfs, + &rxq->rx_cq.gdma_cq->head); + debugfs_create_u32("cq_tail", 0400, rxq->mana_rx_debugfs, + &rxq->rx_cq.gdma_cq->tail); + debugfs_create_u32("cq_budget", 0400, rxq->mana_rx_debugfs, &rxq->rx_cq.budget); + debugfs_create_file("rxq_dump", 0400, rxq->mana_rx_debugfs, rxq->gdma_rq, &mana_dbg_q_fops); + debugfs_create_file("cq_dump", 0400, rxq->mana_rx_debugfs, rxq->rx_cq.gdma_cq, + &mana_dbg_q_fops); +} + +static int mana_add_rx_queues(struct mana_port_context *apc, + struct net_device *ndev) +{ + struct mana_rxq *rxq; + int err = 0; + int i; + + for (i = 0; i < apc->num_queues; i++) { + rxq = mana_create_rxq(apc, i, &apc->eqs[i], ndev); + if (IS_ERR(rxq)) { + err = PTR_ERR(rxq); + netdev_err(ndev, "Failed to create rxq %d : %d\n", i, err); + goto out; + } + + u64_stats_init(&rxq->stats.syncp); + + apc->rxqs[i] = rxq; + + mana_create_rxq_debugfs(apc, i); + } + + apc->default_rxobj = apc->rxqs[0]->rxobj; +out: + return err; +} + +static void mana_destroy_rxqs(struct mana_port_context *apc) +{ + struct mana_rxq *rxq; + u32 rxq_idx; + + if (apc->rxqs) { + + for (rxq_idx = 0; rxq_idx < apc->num_queues; rxq_idx++) { + rxq = apc->rxqs[rxq_idx]; + if (!rxq) + continue; + + mana_destroy_rxq(apc, rxq, true); + apc->rxqs[rxq_idx] = NULL; + } + } +} + +static void mana_destroy_vport(struct mana_port_context *apc) +{ + struct gdma_dev *gd = apc->ac->gdma_dev; + + mana_uncfg_vport(apc); + + if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) + mana_pf_deregister_hw_vport(apc); +} + +static int mana_create_vport(struct mana_port_context *apc, + struct net_device *net) +{ + struct gdma_dev *gd = apc->ac->gdma_dev; + int err; + + apc->default_rxobj = INVALID_MANA_HANDLE; + + if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { + err = mana_pf_register_hw_vport(apc); + if (err) + return err; + } + + err = mana_cfg_vport(apc, gd->pdid, gd->doorbell, false); + if (err) { + if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) + mana_pf_deregister_hw_vport(apc); + return err; + } + + return 0; +} + +static int mana_rss_table_alloc(struct mana_port_context *apc) +{ + if (!apc->indir_table_sz) { + netdev_err(apc->ndev, + "Indirection table size not set for vPort %d\n", + apc->port_idx); + return -EINVAL; + } + + apc->indir_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); + if (!apc->indir_table) + return -ENOMEM; + + apc->rxobj_table = kzalloc_objs(mana_handle_t, apc->indir_table_sz); + if (!apc->rxobj_table) { + kfree(apc->indir_table); + return -ENOMEM; + } + + return 0; +} + +static void mana_rss_table_init(struct mana_port_context *apc) +{ + int i; + + for (i = 0; i < apc->indir_table_sz; i++) + apc->indir_table[i] = + ethtool_rxfh_indir_default(i, apc->num_queues); +} + +int mana_disable_vport_rx(struct mana_port_context *apc) +{ + return mana_cfg_vport_steering(apc, TRI_STATE_FALSE, false, false, + false); +} +EXPORT_SYMBOL_NS(mana_disable_vport_rx, "NET_MANA"); + +int mana_config_rss(struct mana_port_context *apc, enum TRI_STATE rx, + bool update_hash, bool update_tab) +{ + u32 queue_idx; + int err; + int i; + + if (update_tab) { + for (i = 0; i < apc->indir_table_sz; i++) { + queue_idx = apc->indir_table[i]; + apc->rxobj_table[i] = apc->rxqs[queue_idx]->rxobj; + } + } + + err = mana_cfg_vport_steering(apc, rx, true, update_hash, update_tab); + if (err) + return err; + + mana_fence_rqs(apc); + + return 0; +} + +int mana_query_gf_stats(struct mana_context *ac) +{ + struct gdma_context *gc = ac->gdma_dev->gdma_context; + struct mana_query_gf_stat_resp resp = {}; + struct mana_query_gf_stat_req req = {}; + struct device *dev = gc->dev; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_GF_STAT, + sizeof(req), sizeof(resp)); + req.hdr.resp.msg_version = GDMA_MESSAGE_V2; + req.req_stats = STATISTICS_FLAGS_RX_DISCARDS_NO_WQE | + STATISTICS_FLAGS_RX_ERRORS_VPORT_DISABLED | + STATISTICS_FLAGS_HC_RX_BYTES | + STATISTICS_FLAGS_HC_RX_UCAST_PACKETS | + STATISTICS_FLAGS_HC_RX_UCAST_BYTES | + STATISTICS_FLAGS_HC_RX_MCAST_PACKETS | + STATISTICS_FLAGS_HC_RX_MCAST_BYTES | + STATISTICS_FLAGS_HC_RX_BCAST_PACKETS | + STATISTICS_FLAGS_HC_RX_BCAST_BYTES | + STATISTICS_FLAGS_TX_ERRORS_GF_DISABLED | + STATISTICS_FLAGS_TX_ERRORS_VPORT_DISABLED | + STATISTICS_FLAGS_TX_ERRORS_INVAL_VPORT_OFFSET_PACKETS | + STATISTICS_FLAGS_TX_ERRORS_VLAN_ENFORCEMENT | + STATISTICS_FLAGS_TX_ERRORS_ETH_TYPE_ENFORCEMENT | + STATISTICS_FLAGS_TX_ERRORS_SA_ENFORCEMENT | + STATISTICS_FLAGS_TX_ERRORS_SQPDID_ENFORCEMENT | + STATISTICS_FLAGS_TX_ERRORS_CQPDID_ENFORCEMENT | + STATISTICS_FLAGS_TX_ERRORS_MTU_VIOLATION | + STATISTICS_FLAGS_TX_ERRORS_INVALID_OOB | + STATISTICS_FLAGS_HC_TX_BYTES | + STATISTICS_FLAGS_HC_TX_UCAST_PACKETS | + STATISTICS_FLAGS_HC_TX_UCAST_BYTES | + STATISTICS_FLAGS_HC_TX_MCAST_PACKETS | + STATISTICS_FLAGS_HC_TX_MCAST_BYTES | + STATISTICS_FLAGS_HC_TX_BCAST_PACKETS | + STATISTICS_FLAGS_HC_TX_BCAST_BYTES | + STATISTICS_FLAGS_TX_ERRORS_GDMA_ERROR; + + err = mana_send_request(ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) { + dev_err(dev, "Failed to query GF stats: %d\n", err); + return err; + } + err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_GF_STAT, + sizeof(resp)); + if (err || resp.hdr.status) { + dev_err(dev, "Failed to query GF stats: %d, 0x%x\n", err, + resp.hdr.status); + return err; + } + + ac->hc_stats.hc_rx_discards_no_wqe = resp.rx_discards_nowqe; + ac->hc_stats.hc_rx_err_vport_disabled = resp.rx_err_vport_disabled; + ac->hc_stats.hc_rx_bytes = resp.hc_rx_bytes; + ac->hc_stats.hc_rx_ucast_pkts = resp.hc_rx_ucast_pkts; + ac->hc_stats.hc_rx_ucast_bytes = resp.hc_rx_ucast_bytes; + ac->hc_stats.hc_rx_bcast_pkts = resp.hc_rx_bcast_pkts; + ac->hc_stats.hc_rx_bcast_bytes = resp.hc_rx_bcast_bytes; + ac->hc_stats.hc_rx_mcast_pkts = resp.hc_rx_mcast_pkts; + ac->hc_stats.hc_rx_mcast_bytes = resp.hc_rx_mcast_bytes; + ac->hc_stats.hc_tx_err_gf_disabled = resp.tx_err_gf_disabled; + ac->hc_stats.hc_tx_err_vport_disabled = resp.tx_err_vport_disabled; + ac->hc_stats.hc_tx_err_inval_vportoffset_pkt = + resp.tx_err_inval_vport_offset_pkt; + ac->hc_stats.hc_tx_err_vlan_enforcement = + resp.tx_err_vlan_enforcement; + ac->hc_stats.hc_tx_err_eth_type_enforcement = + resp.tx_err_ethtype_enforcement; + ac->hc_stats.hc_tx_err_sa_enforcement = resp.tx_err_SA_enforcement; + ac->hc_stats.hc_tx_err_sqpdid_enforcement = + resp.tx_err_SQPDID_enforcement; + ac->hc_stats.hc_tx_err_cqpdid_enforcement = + resp.tx_err_CQPDID_enforcement; + ac->hc_stats.hc_tx_err_mtu_violation = resp.tx_err_mtu_violation; + ac->hc_stats.hc_tx_err_inval_oob = resp.tx_err_inval_oob; + ac->hc_stats.hc_tx_bytes = resp.hc_tx_bytes; + ac->hc_stats.hc_tx_ucast_pkts = resp.hc_tx_ucast_pkts; + ac->hc_stats.hc_tx_ucast_bytes = resp.hc_tx_ucast_bytes; + ac->hc_stats.hc_tx_bcast_pkts = resp.hc_tx_bcast_pkts; + ac->hc_stats.hc_tx_bcast_bytes = resp.hc_tx_bcast_bytes; + ac->hc_stats.hc_tx_mcast_pkts = resp.hc_tx_mcast_pkts; + ac->hc_stats.hc_tx_mcast_bytes = resp.hc_tx_mcast_bytes; + ac->hc_stats.hc_tx_err_gdma = resp.tx_err_gdma; + + return 0; +} + +void mana_query_phy_stats(struct mana_port_context *apc) +{ + struct mana_query_phy_stat_resp resp = {}; + struct mana_query_phy_stat_req req = {}; + struct net_device *ndev = apc->ndev; + int err; + + mana_gd_init_req_hdr(&req.hdr, MANA_QUERY_PHY_STAT, + sizeof(req), sizeof(resp)); + err = mana_send_request(apc->ac, &req, sizeof(req), &resp, + sizeof(resp)); + if (err) + return; + + err = mana_verify_resp_hdr(&resp.hdr, MANA_QUERY_PHY_STAT, + sizeof(resp)); + if (err || resp.hdr.status) { + netdev_err(ndev, + "Failed to query PHY stats: %d, resp:0x%x\n", + err, resp.hdr.status); + return; + } + + /* Aggregate drop counters */ + apc->phy_stats.rx_pkt_drop_phy = resp.rx_pkt_drop_phy; + apc->phy_stats.tx_pkt_drop_phy = resp.tx_pkt_drop_phy; + + /* Per TC traffic Counters */ + apc->phy_stats.rx_pkt_tc0_phy = resp.rx_pkt_tc0_phy; + apc->phy_stats.tx_pkt_tc0_phy = resp.tx_pkt_tc0_phy; + apc->phy_stats.rx_pkt_tc1_phy = resp.rx_pkt_tc1_phy; + apc->phy_stats.tx_pkt_tc1_phy = resp.tx_pkt_tc1_phy; + apc->phy_stats.rx_pkt_tc2_phy = resp.rx_pkt_tc2_phy; + apc->phy_stats.tx_pkt_tc2_phy = resp.tx_pkt_tc2_phy; + apc->phy_stats.rx_pkt_tc3_phy = resp.rx_pkt_tc3_phy; + apc->phy_stats.tx_pkt_tc3_phy = resp.tx_pkt_tc3_phy; + apc->phy_stats.rx_pkt_tc4_phy = resp.rx_pkt_tc4_phy; + apc->phy_stats.tx_pkt_tc4_phy = resp.tx_pkt_tc4_phy; + apc->phy_stats.rx_pkt_tc5_phy = resp.rx_pkt_tc5_phy; + apc->phy_stats.tx_pkt_tc5_phy = resp.tx_pkt_tc5_phy; + apc->phy_stats.rx_pkt_tc6_phy = resp.rx_pkt_tc6_phy; + apc->phy_stats.tx_pkt_tc6_phy = resp.tx_pkt_tc6_phy; + apc->phy_stats.rx_pkt_tc7_phy = resp.rx_pkt_tc7_phy; + apc->phy_stats.tx_pkt_tc7_phy = resp.tx_pkt_tc7_phy; + + /* Per TC byte Counters */ + apc->phy_stats.rx_byte_tc0_phy = resp.rx_byte_tc0_phy; + apc->phy_stats.tx_byte_tc0_phy = resp.tx_byte_tc0_phy; + apc->phy_stats.rx_byte_tc1_phy = resp.rx_byte_tc1_phy; + apc->phy_stats.tx_byte_tc1_phy = resp.tx_byte_tc1_phy; + apc->phy_stats.rx_byte_tc2_phy = resp.rx_byte_tc2_phy; + apc->phy_stats.tx_byte_tc2_phy = resp.tx_byte_tc2_phy; + apc->phy_stats.rx_byte_tc3_phy = resp.rx_byte_tc3_phy; + apc->phy_stats.tx_byte_tc3_phy = resp.tx_byte_tc3_phy; + apc->phy_stats.rx_byte_tc4_phy = resp.rx_byte_tc4_phy; + apc->phy_stats.tx_byte_tc4_phy = resp.tx_byte_tc4_phy; + apc->phy_stats.rx_byte_tc5_phy = resp.rx_byte_tc5_phy; + apc->phy_stats.tx_byte_tc5_phy = resp.tx_byte_tc5_phy; + apc->phy_stats.rx_byte_tc6_phy = resp.rx_byte_tc6_phy; + apc->phy_stats.tx_byte_tc6_phy = resp.tx_byte_tc6_phy; + apc->phy_stats.rx_byte_tc7_phy = resp.rx_byte_tc7_phy; + apc->phy_stats.tx_byte_tc7_phy = resp.tx_byte_tc7_phy; + + /* Per TC pause Counters */ + apc->phy_stats.rx_pause_tc0_phy = resp.rx_pause_tc0_phy; + apc->phy_stats.tx_pause_tc0_phy = resp.tx_pause_tc0_phy; + apc->phy_stats.rx_pause_tc1_phy = resp.rx_pause_tc1_phy; + apc->phy_stats.tx_pause_tc1_phy = resp.tx_pause_tc1_phy; + apc->phy_stats.rx_pause_tc2_phy = resp.rx_pause_tc2_phy; + apc->phy_stats.tx_pause_tc2_phy = resp.tx_pause_tc2_phy; + apc->phy_stats.rx_pause_tc3_phy = resp.rx_pause_tc3_phy; + apc->phy_stats.tx_pause_tc3_phy = resp.tx_pause_tc3_phy; + apc->phy_stats.rx_pause_tc4_phy = resp.rx_pause_tc4_phy; + apc->phy_stats.tx_pause_tc4_phy = resp.tx_pause_tc4_phy; + apc->phy_stats.rx_pause_tc5_phy = resp.rx_pause_tc5_phy; + apc->phy_stats.tx_pause_tc5_phy = resp.tx_pause_tc5_phy; + apc->phy_stats.rx_pause_tc6_phy = resp.rx_pause_tc6_phy; + apc->phy_stats.tx_pause_tc6_phy = resp.tx_pause_tc6_phy; + apc->phy_stats.rx_pause_tc7_phy = resp.rx_pause_tc7_phy; + apc->phy_stats.tx_pause_tc7_phy = resp.tx_pause_tc7_phy; +} + +static int mana_init_port(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + struct gdma_dev *gd = apc->ac->gdma_dev; + u32 max_txq, max_rxq, max_queues; + int port_idx = apc->port_idx; + struct gdma_context *gc; + char vport[32]; + int err; + + err = mana_init_port_context(apc); + if (err) + return err; + + gc = gd->gdma_context; + + err = mana_query_vport_cfg(apc, port_idx, &max_txq, &max_rxq, + &apc->indir_table_sz); + if (err) { + netdev_err(ndev, "Failed to query info for vPort %d\n", + port_idx); + goto reset_apc; + } + + max_queues = min_t(u32, max_txq, max_rxq); + if (apc->max_queues > max_queues) + apc->max_queues = max_queues; + if (apc->max_queues > gc->max_num_queues_vport) + apc->max_queues = gc->max_num_queues_vport; + + if (apc->num_queues > apc->max_queues) + apc->num_queues = apc->max_queues; + + eth_hw_addr_set(ndev, apc->mac_addr); + sprintf(vport, "vport%d", port_idx); + apc->mana_port_debugfs = debugfs_create_dir(vport, gc->mana_pci_debugfs); + + debugfs_create_u64("port_handle", 0400, apc->mana_port_debugfs, + &apc->port_handle); + debugfs_create_u32("max_sq", 0400, apc->mana_port_debugfs, + &apc->vport_max_sq); + debugfs_create_u32("max_rq", 0400, apc->mana_port_debugfs, + &apc->vport_max_rq); + debugfs_create_u32("indir_table_sz", 0400, apc->mana_port_debugfs, + &apc->indir_table_sz); + debugfs_create_u32("steer_rx", 0400, apc->mana_port_debugfs, + &apc->steer_rx); + debugfs_create_u32("steer_rss", 0400, apc->mana_port_debugfs, + &apc->steer_rss); + debugfs_create_bool("steer_update_tab", 0400, apc->mana_port_debugfs, + &apc->steer_update_tab); + debugfs_create_u32("steer_cqe_coalescing", 0400, apc->mana_port_debugfs, + &apc->steer_cqe_coalescing); + debugfs_create_u32("current_speed", 0400, apc->mana_port_debugfs, + &apc->speed); + debugfs_create_bool("tx_timeout_skip_reset", 0600, + apc->mana_port_debugfs, + &apc->tx_timeout_skip_reset); + return 0; + +reset_apc: + mana_cleanup_port_context(apc); + return err; +} + +int mana_alloc_queues(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + struct gdma_dev *gd = apc->ac->gdma_dev; + int err; + + err = mana_create_vport(apc, ndev); + if (err) { + netdev_err(ndev, "Failed to create vPort %u : %d\n", + apc->port_idx, err); + return err; + } + + err = mana_create_eq(apc); + if (err) { + netdev_err(ndev, "Failed to create EQ on vPort %u: %d\n", + apc->port_idx, err); + goto destroy_vport; + } + + err = mana_create_txq(apc, ndev); + if (err) { + netdev_err(ndev, "Failed to create TXQ on vPort %u: %d\n", + apc->port_idx, err); + goto destroy_eq; + } + + err = netif_set_real_num_tx_queues(ndev, apc->num_queues); + if (err) { + netdev_err(ndev, + "netif_set_real_num_tx_queues () failed for ndev with num_queues %u : %d\n", + apc->num_queues, err); + goto destroy_txq; + } + + err = mana_add_rx_queues(apc, ndev); + if (err) + goto destroy_rxq; + + apc->rss_state = apc->num_queues > 1 ? TRI_STATE_TRUE : TRI_STATE_FALSE; + + err = netif_set_real_num_rx_queues(ndev, apc->num_queues); + if (err) { + netdev_err(ndev, + "netif_set_real_num_rx_queues () failed for ndev with num_queues %u : %d\n", + apc->num_queues, err); + goto destroy_rxq; + } + + mana_rss_table_init(apc); + + err = mana_config_rss(apc, TRI_STATE_TRUE, true, true); + if (err) { + netdev_err(ndev, "Failed to configure RSS table: %d\n", err); + goto destroy_rxq; + } + + if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) { + err = mana_pf_register_filter(apc); + if (err) + goto destroy_rxq; + } + + mana_chn_setxdp(apc, mana_xdp_get(apc)); + + return 0; + +destroy_rxq: + mana_destroy_rxqs(apc); +destroy_txq: + mana_destroy_txq(apc); +destroy_eq: + mana_destroy_eq(apc); +destroy_vport: + mana_destroy_vport(apc); + return err; +} + +int mana_attach(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + int err; + + ASSERT_RTNL(); + + err = mana_init_port(ndev); + if (err) + return err; + + if (apc->port_st_save) { + err = mana_alloc_queues(ndev); + if (err) { + mana_cleanup_port_context(apc); + return err; + } + } + + apc->port_is_up = apc->port_st_save; + + /* Ensure port state updated before txq state */ + smp_wmb(); + + netif_device_attach(ndev); + + return 0; +} + +static int mana_dealloc_queues(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + unsigned long timeout = jiffies + 120 * HZ; + struct gdma_dev *gd = apc->ac->gdma_dev; + struct mana_txq *txq; + struct sk_buff *skb; + int i, err; + u32 tsleep; + + if (apc->port_is_up) + return -EINVAL; + + if (apc->rxqs) + mana_chn_setxdp(apc, NULL); + + if (gd->gdma_context->is_pf && !apc->ac->bm_hostmode) + mana_pf_deregister_filter(apc); + + /* No packet can be transmitted now since apc->port_is_up is false. + * There is still a tiny chance that mana_poll_tx_cq() can re-enable + * a txq because it may not timely see apc->port_is_up being cleared + * to false, but it doesn't matter since mana_start_xmit() drops any + * new packets due to apc->port_is_up being false. + * + * Drain all the in-flight TX packets. + * A timeout of 120 seconds for all the queues is used. + * This will break the while loop when h/w is not responding. + * This value of 120 has been decided here considering max + * number of queues. + */ + + if (apc->tx_qp) { + for (i = 0; i < apc->num_queues; i++) { + txq = &apc->tx_qp[i]->txq; + tsleep = 1000; + while (atomic_read(&txq->pending_sends) > 0 && + time_before(jiffies, timeout)) { + usleep_range(tsleep, tsleep + 1000); + tsleep <<= 1; + } + if (atomic_read(&txq->pending_sends)) { + err = + pcie_flr(to_pci_dev(gd->gdma_context->dev)); + if (err) { + netdev_err(ndev, "flr failed %d with %d pkts pending in txq %u\n", + err, + atomic_read(&txq->pending_sends), + txq->gdma_txq_id); + } + break; + } + } + + for (i = 0; i < apc->num_queues; i++) { + txq = &apc->tx_qp[i]->txq; + while ((skb = skb_dequeue(&txq->pending_skbs))) { + mana_unmap_skb(skb, apc); + dev_kfree_skb_any(skb); + } + atomic_set(&txq->pending_sends, 0); + } + } + + /* We're 100% sure the queues can no longer be woken up, because + * we're sure now mana_poll_tx_cq() can't be running. + */ + + apc->rss_state = TRI_STATE_FALSE; + err = mana_disable_vport_rx(apc); + if (err && mana_en_need_log(apc, err)) + netdev_err(ndev, "Failed to disable vPort: %d\n", err); + + mana_fence_rqs(apc); + + /* Even in err case, still need to cleanup the vPort */ + mana_destroy_rxqs(apc); + mana_destroy_txq(apc); + mana_destroy_eq(apc); + mana_destroy_vport(apc); + + return 0; +} + +int mana_detach(struct net_device *ndev, bool from_close) +{ + struct mana_port_context *apc = netdev_priv(ndev); + int err; + + ASSERT_RTNL(); + + /* If already detached (indicates detach succeeded but attach failed + * previously). Now skip mana detach and just retry mana_attach. + */ + if (!from_close && !netif_device_present(ndev)) + return 0; + + apc->port_st_save = apc->port_is_up; + apc->port_is_up = false; + + /* Ensure port state updated before txq state */ + smp_wmb(); + + netif_tx_disable(ndev); + + if (apc->port_st_save) { + err = mana_dealloc_queues(ndev); + if (err) { + netdev_err(ndev, "%s failed to deallocate queues: %d\n", __func__, err); + return err; + } + } + + if (!from_close) { + netif_device_detach(ndev); + mana_cleanup_port_context(apc); + } + + return 0; +} + +static int mana_probe_port(struct mana_context *ac, int port_idx, + struct net_device **ndev_storage) +{ + struct gdma_context *gc = ac->gdma_dev->gdma_context; + struct mana_port_context *apc; + struct net_device *ndev; + int err; + + ndev = alloc_etherdev_mq(sizeof(struct mana_port_context), + gc->max_num_queues_vport); + if (!ndev) + return -ENOMEM; + + *ndev_storage = ndev; + + apc = netdev_priv(ndev); + apc->ac = ac; + apc->ndev = ndev; + apc->max_queues = gc->max_num_queues_vport; + /* Use MANA_DEF_NUM_QUEUES as default, still honoring the HW limit */ + apc->num_queues = min(gc->max_num_queues_vport, MANA_DEF_NUM_QUEUES); + apc->tx_queue_size = DEF_TX_BUFFERS_PER_QUEUE; + apc->rx_queue_size = DEF_RX_BUFFERS_PER_QUEUE; + apc->port_handle = INVALID_MANA_HANDLE; + apc->pf_filter_handle = INVALID_MANA_HANDLE; + apc->port_idx = port_idx; + apc->link_cfg_error = 1; + apc->cqe_coalescing_enable = 0; + apc->cqe8_coalescing_enable = 0; + + /* Initialize interrupt moderation settings if supported by HW */ + if (gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION) { + apc->intr_modr_rx_usec = MANA_INTR_MODR_USEC_DEF; + apc->intr_modr_rx_comp = MANA_INTR_MODR_COMP_DEF; + apc->intr_modr_tx_usec = MANA_INTR_MODR_USEC_DEF; + apc->intr_modr_tx_comp = MANA_INTR_MODR_COMP_DEF; + apc->rx_dim_enabled = MANA_ADAPTIVE_RX_DEF; + apc->tx_dim_enabled = MANA_ADAPTIVE_TX_DEF; + } + + mutex_init(&apc->vport_mutex); + apc->vport_use_count = 0; + + ndev->netdev_ops = &mana_devops; + ndev->ethtool_ops = &mana_ethtool_ops; + ndev->mtu = ETH_DATA_LEN; + ndev->max_mtu = gc->adapter_mtu - ETH_HLEN; + ndev->min_mtu = ETH_MIN_MTU; + ndev->needed_headroom = MANA_HEADROOM; + ndev->dev_port = port_idx; + /* Recommended timeout based on HW FPGA re-config scenario. */ + ndev->watchdog_timeo = 15 * HZ; + SET_NETDEV_DEV(ndev, gc->dev); + + netif_set_tso_max_size(ndev, GSO_MAX_SIZE); + + netif_carrier_off(ndev); + + netdev_rss_key_fill(apc->hashkey, MANA_HASH_KEY_SIZE); + + err = mana_init_port(ndev); + if (err) + goto free_net; + + err = mana_rss_table_alloc(apc); + if (err) + goto reset_apc; + + /* Initialize the per port queue reset work.*/ + INIT_WORK(&apc->queue_reset_work, + mana_per_port_queue_reset_work_handler); + + netdev_lockdep_set_classes(ndev); + + ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; + ndev->hw_features |= NETIF_F_RXCSUM; + ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6; + ndev->hw_features |= NETIF_F_RXHASH; + ndev->features = ndev->hw_features | NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_HW_VLAN_CTAG_RX; + ndev->vlan_features = ndev->features; + xdp_set_features_flag(ndev, NETDEV_XDP_ACT_BASIC | + NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_NDO_XMIT); + + err = register_netdev(ndev); + if (err) { + netdev_err(ndev, "Unable to register netdev.\n"); + goto free_indir; + } + + netif_carrier_on(ndev); + + return 0; + +free_indir: + mana_cleanup_indir_table(apc); +reset_apc: + mana_cleanup_port_context(apc); +free_net: + *ndev_storage = NULL; + netdev_err(ndev, "Failed to probe vPort %d: %d\n", port_idx, err); + free_netdev(ndev); + return err; +} + +static void adev_release(struct device *dev) +{ + struct mana_adev *madev = container_of(dev, struct mana_adev, adev.dev); + + kfree(madev); +} + +static void remove_adev(struct gdma_dev *gd) +{ + struct auxiliary_device *adev = gd->adev; + int id = adev->id; + + auxiliary_device_delete(adev); + auxiliary_device_uninit(adev); + + mana_adev_idx_free(id); + gd->adev = NULL; +} + +static int add_adev(struct gdma_dev *gd, const char *name) +{ + struct auxiliary_device *adev; + struct mana_adev *madev; + int ret; + int id; + + madev = kzalloc_obj(*madev); + if (!madev) + return -ENOMEM; + + adev = &madev->adev; + ret = mana_adev_idx_alloc(); + if (ret < 0) + goto idx_fail; + id = ret; + adev->id = id; + + adev->name = name; + adev->dev.parent = gd->gdma_context->dev; + adev->dev.release = adev_release; + madev->mdev = gd; + + ret = auxiliary_device_init(adev); + if (ret) + goto init_fail; + + /* madev is owned by the auxiliary device */ + madev = NULL; + ret = auxiliary_device_add(adev); + if (ret) + goto add_fail; + + gd->adev = adev; + dev_dbg(gd->gdma_context->dev, + "Auxiliary device added successfully\n"); + return 0; + +add_fail: + auxiliary_device_uninit(adev); + +init_fail: + mana_adev_idx_free(id); + +idx_fail: + kfree(madev); + + return ret; +} + +static void mana_rdma_service_handle(struct work_struct *work) +{ + struct mana_service_work *serv_work = + container_of(work, struct mana_service_work, work); + struct gdma_dev *gd = serv_work->gdma_dev; + struct device *dev = gd->gdma_context->dev; + int ret; + + if (READ_ONCE(gd->rdma_teardown)) + goto out; + + switch (serv_work->event) { + case GDMA_SERVICE_TYPE_RDMA_SUSPEND: + if (!gd->adev || gd->is_suspended) + break; + + remove_adev(gd); + gd->is_suspended = true; + break; + + case GDMA_SERVICE_TYPE_RDMA_RESUME: + if (!gd->is_suspended) + break; + + ret = add_adev(gd, "rdma"); + if (ret) + dev_err(dev, "Failed to add adev on resume: %d\n", ret); + else + gd->is_suspended = false; + break; + + default: + dev_warn(dev, "unknown adev service event %u\n", + serv_work->event); + break; + } + +out: + kfree(serv_work); +} + +int mana_rdma_service_event(struct gdma_context *gc, enum gdma_service_type event) +{ + struct gdma_dev *gd = &gc->mana_ib; + struct mana_service_work *serv_work; + + if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { + /* RDMA device is not detected on pci */ + return 0; + } + + serv_work = kzalloc_obj(*serv_work, GFP_ATOMIC); + if (!serv_work) + return -ENOMEM; + + serv_work->event = event; + serv_work->gdma_dev = gd; + + INIT_WORK(&serv_work->work, mana_rdma_service_handle); + queue_work(gc->service_wq, &serv_work->work); + + return 0; +} + +#define MANA_GF_STATS_PERIOD (2 * HZ) + +static void mana_gf_stats_work_handler(struct work_struct *work) +{ + struct mana_context *ac = + container_of(to_delayed_work(work), struct mana_context, gf_stats_work); + struct gdma_context *gc = ac->gdma_dev->gdma_context; + int err; + + err = mana_query_gf_stats(ac); + if (err == -ETIMEDOUT) { + /* HWC timeout detected - reset stats and stop rescheduling */ + ac->hwc_timeout_occurred = true; + memset(&ac->hc_stats, 0, sizeof(ac->hc_stats)); + dev_warn(gc->dev, + "Gf stats wk handler: gf stats query timed out.\n"); + /* As HWC timed out, indicating a faulty HW state and needs a + * reset. + */ + mana_schedule_serv_work(gc, GDMA_EQE_HWC_RESET_REQUEST); + return; + } + schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); +} + +int mana_probe(struct gdma_dev *gd, bool resuming) +{ + struct gdma_context *gc = gd->gdma_context; + struct mana_context *ac = gd->driver_data; + struct mana_port_context *apc = NULL; + struct device *dev = gc->dev; + u8 bm_hostmode = 0; + u16 num_ports = 0; + int err; + int i; + + dev_info(dev, + "Microsoft Azure Network Adapter protocol version: %d.%d.%d\n", + MANA_MAJOR_VERSION, MANA_MINOR_VERSION, MANA_MICRO_VERSION); + + err = mana_gd_register_device(gd); + if (err) + return err; + + if (!resuming) { + ac = kzalloc_obj(*ac); + if (!ac) + return -ENOMEM; + + ac->gdma_dev = gd; + gd->driver_data = ac; + + INIT_WORK(&ac->link_change_work, mana_link_state_handle); + } + + INIT_DELAYED_WORK(&ac->gf_stats_work, mana_gf_stats_work_handler); + + err = mana_gd_query_device_cfg(gc, MANA_MAJOR_VERSION, + MANA_MINOR_VERSION, + MANA_MICRO_VERSION, + &num_ports, &bm_hostmode); + if (err) + goto out; + + ac->bm_hostmode = bm_hostmode; + + debugfs_create_u16("adapter-MTU", 0400, + gc->mana_pci_debugfs, &gc->adapter_mtu); + + if (!resuming) { + ac->num_ports = num_ports; + } else { + if (ac->num_ports != num_ports) { + dev_err(dev, "The number of vPorts changed: %d->%d\n", + ac->num_ports, num_ports); + err = -EPROTO; + goto out; + } + + enable_work(&ac->link_change_work); + } + + if (ac->num_ports > MAX_PORTS_IN_MANA_DEV) + ac->num_ports = MAX_PORTS_IN_MANA_DEV; + + debugfs_create_u16("num_vports", 0400, gc->mana_pci_debugfs, + &ac->num_ports); + debugfs_create_u8("bm_hostmode", 0400, gc->mana_pci_debugfs, + &ac->bm_hostmode); + + ac->per_port_queue_reset_wq = + create_singlethread_workqueue("mana_per_port_queue_reset_wq"); + if (!ac->per_port_queue_reset_wq) { + dev_err(dev, "Failed to allocate per port queue reset workqueue\n"); + err = -ENOMEM; + goto out; + } + + if (!resuming) { + for (i = 0; i < ac->num_ports; i++) { + err = mana_probe_port(ac, i, &ac->ports[i]); + /* Log the port for which the probe failed, stop probing + * subsequent ports, and skip add_adev. + * mana_remove() will clean up already-probed ports. + */ + if (err) { + dev_err(dev, "Probe Failed for port %d\n", i); + break; + } + } + } else { + for (i = 0; i < ac->num_ports; i++) { + rtnl_lock(); + apc = netdev_priv(ac->ports[i]); + enable_work(&apc->queue_reset_work); + netdev_lock(ac->ports[i]); + apc->link_cfg_error = 1; + netdev_unlock(ac->ports[i]); + err = mana_attach(ac->ports[i]); + rtnl_unlock(); + /* Log the port for which the attach failed, stop + * attaching subsequent ports, and skip add_adev. + * mana_remove() will clean up already-attached ports. + */ + if (err) { + dev_err(dev, "Attach Failed for port %d\n", i); + break; + } + } + } + + if (!err) + err = add_adev(gd, "eth"); + + schedule_delayed_work(&ac->gf_stats_work, MANA_GF_STATS_PERIOD); + +out: + if (err) { + mana_remove(gd, false); + } else { + dev_dbg(dev, "gd=%p, id=%u, num_ports=%d, type=%u, instance=%u\n", + gd, gd->dev_id.as_uint32, ac->num_ports, + gd->dev_id.type, gd->dev_id.instance); + dev_dbg(dev, "%s succeeded\n", __func__); + } + + return err; +} + +void mana_remove(struct gdma_dev *gd, bool suspending) +{ + struct gdma_context *gc = gd->gdma_context; + struct mana_context *ac = gd->driver_data; + struct mana_port_context *apc; + struct device *dev; + struct net_device *ndev; + int err; + int i; + + if (!gc || !ac) + return; + + dev = gc->dev; + + disable_work_sync(&ac->link_change_work); + cancel_delayed_work_sync(&ac->gf_stats_work); + + /* adev currently doesn't support suspending, always remove it */ + if (gd->adev) + remove_adev(gd); + + for (i = 0; i < ac->num_ports; i++) { + ndev = ac->ports[i]; + if (!ndev) { + if (i == 0) + dev_err(dev, "No net device to remove\n"); + break; + } + + apc = netdev_priv(ndev); + disable_work_sync(&apc->queue_reset_work); + + /* All cleanup actions should stay after rtnl_lock(), otherwise + * other functions may access partially cleaned up data. + */ + rtnl_lock(); + + err = mana_detach(ndev, false); + if (err) + netdev_err(ndev, "Failed to detach vPort %d: %d\n", + i, err); + + if (suspending) { + /* No need to unregister the ndev. */ + rtnl_unlock(); + continue; + } + + unregister_netdevice(ndev); + mana_cleanup_indir_table(apc); + + rtnl_unlock(); + + free_netdev(ndev); + } + + if (ac->per_port_queue_reset_wq) { + destroy_workqueue(ac->per_port_queue_reset_wq); + ac->per_port_queue_reset_wq = NULL; + } + + mana_gd_deregister_device(gd); + + if (gc->mana_pci_debugfs) { + debugfs_lookup_and_remove("bm_hostmode", gc->mana_pci_debugfs); + debugfs_lookup_and_remove("num_vports", gc->mana_pci_debugfs); + } + + if (suspending) + return; + + gd->driver_data = NULL; + gd->gdma_context = NULL; + kfree(ac); + dev_dbg(dev, "%s succeeded\n", __func__); +} + +int mana_rdma_probe(struct gdma_dev *gd) +{ + int err = 0; + + if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { + /* RDMA device is not detected on pci */ + return err; + } + + err = mana_gd_register_device(gd); + if (err) + return err; + + err = add_adev(gd, "rdma"); + if (err) + mana_gd_deregister_device(gd); + + return err; +} + +void mana_rdma_remove(struct gdma_dev *gd) +{ + struct gdma_context *gc = gd->gdma_context; + + if (gd->dev_id.type != GDMA_DEVICE_MANA_IB) { + /* RDMA device is not detected on pci */ + return; + } + + WRITE_ONCE(gd->rdma_teardown, true); + + if (gc->service_wq) + flush_workqueue(gc->service_wq); + + if (gd->adev) + remove_adev(gd); + + mana_gd_deregister_device(gd); +} + +struct net_device *mana_get_primary_netdev(struct mana_context *ac, + u32 port_index, + netdevice_tracker *tracker) +{ + struct net_device *ndev; + + if (port_index >= ac->num_ports) + return NULL; + + rcu_read_lock(); + + /* If mana is used in netvsc, the upper netdevice should be returned. */ + ndev = netdev_master_upper_dev_get_rcu(ac->ports[port_index]); + + /* If there is no upper device, use the parent Ethernet device */ + if (!ndev) + ndev = ac->ports[port_index]; + + netdev_hold(ndev, tracker, GFP_ATOMIC); + rcu_read_unlock(); + + return ndev; +} +EXPORT_SYMBOL_NS(mana_get_primary_netdev, "NET_MANA"); diff --git a/drivers/net/ethernet/microsoft/mana/mana_ethtool.c b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c new file mode 100644 index 000000000..ece7ff9cc --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/mana_ethtool.c @@ -0,0 +1,893 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include +#include +#include + +#include + +struct mana_stats_desc { + char name[ETH_GSTRING_LEN]; + u16 offset; +}; + +static const struct mana_stats_desc mana_eth_stats[] = { + {"stop_queue", offsetof(struct mana_ethtool_stats, stop_queue)}, + {"wake_queue", offsetof(struct mana_ethtool_stats, wake_queue)}, + {"tx_cq_err", offsetof(struct mana_ethtool_stats, tx_cqe_err)}, + {"tx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, + tx_cqe_unknown_type)}, + {"tx_linear_pkt_cnt", offsetof(struct mana_ethtool_stats, + tx_linear_pkt_cnt)}, + {"rx_cqe_unknown_type", offsetof(struct mana_ethtool_stats, + rx_cqe_unknown_type)}, +}; + +static const struct mana_stats_desc mana_hc_stats[] = { + {"hc_rx_discards_no_wqe", offsetof(struct mana_ethtool_hc_stats, + hc_rx_discards_no_wqe)}, + {"hc_rx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats, + hc_rx_err_vport_disabled)}, + {"hc_rx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_rx_bytes)}, + {"hc_rx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats, + hc_rx_ucast_pkts)}, + {"hc_rx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats, + hc_rx_ucast_bytes)}, + {"hc_rx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats, + hc_rx_bcast_pkts)}, + {"hc_rx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats, + hc_rx_bcast_bytes)}, + {"hc_rx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats, + hc_rx_mcast_pkts)}, + {"hc_rx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats, + hc_rx_mcast_bytes)}, + {"hc_tx_err_gf_disabled", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_gf_disabled)}, + {"hc_tx_err_vport_disabled", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_vport_disabled)}, + {"hc_tx_err_inval_vportoffset_pkt", + offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_inval_vportoffset_pkt)}, + {"hc_tx_err_vlan_enforcement", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_vlan_enforcement)}, + {"hc_tx_err_eth_type_enforcement", + offsetof(struct mana_ethtool_hc_stats, hc_tx_err_eth_type_enforcement)}, + {"hc_tx_err_sa_enforcement", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_sa_enforcement)}, + {"hc_tx_err_sqpdid_enforcement", + offsetof(struct mana_ethtool_hc_stats, hc_tx_err_sqpdid_enforcement)}, + {"hc_tx_err_cqpdid_enforcement", + offsetof(struct mana_ethtool_hc_stats, hc_tx_err_cqpdid_enforcement)}, + {"hc_tx_err_mtu_violation", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_mtu_violation)}, + {"hc_tx_err_inval_oob", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_inval_oob)}, + {"hc_tx_err_gdma", offsetof(struct mana_ethtool_hc_stats, + hc_tx_err_gdma)}, + {"hc_tx_bytes", offsetof(struct mana_ethtool_hc_stats, hc_tx_bytes)}, + {"hc_tx_ucast_pkts", offsetof(struct mana_ethtool_hc_stats, + hc_tx_ucast_pkts)}, + {"hc_tx_ucast_bytes", offsetof(struct mana_ethtool_hc_stats, + hc_tx_ucast_bytes)}, + {"hc_tx_bcast_pkts", offsetof(struct mana_ethtool_hc_stats, + hc_tx_bcast_pkts)}, + {"hc_tx_bcast_bytes", offsetof(struct mana_ethtool_hc_stats, + hc_tx_bcast_bytes)}, + {"hc_tx_mcast_pkts", offsetof(struct mana_ethtool_hc_stats, + hc_tx_mcast_pkts)}, + {"hc_tx_mcast_bytes", offsetof(struct mana_ethtool_hc_stats, + hc_tx_mcast_bytes)}, +}; + +static const struct mana_stats_desc mana_phy_stats[] = { + { "hc_rx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_drop_phy) }, + { "hc_tx_pkt_drop_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_drop_phy) }, + { "hc_tc0_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc0_phy) }, + { "hc_tc0_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc0_phy) }, + { "hc_tc0_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc0_phy) }, + { "hc_tc0_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc0_phy) }, + { "hc_tc1_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc1_phy) }, + { "hc_tc1_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc1_phy) }, + { "hc_tc1_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc1_phy) }, + { "hc_tc1_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc1_phy) }, + { "hc_tc2_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc2_phy) }, + { "hc_tc2_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc2_phy) }, + { "hc_tc2_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc2_phy) }, + { "hc_tc2_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc2_phy) }, + { "hc_tc3_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc3_phy) }, + { "hc_tc3_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc3_phy) }, + { "hc_tc3_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc3_phy) }, + { "hc_tc3_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc3_phy) }, + { "hc_tc4_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc4_phy) }, + { "hc_tc4_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc4_phy) }, + { "hc_tc4_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc4_phy) }, + { "hc_tc4_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc4_phy) }, + { "hc_tc5_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc5_phy) }, + { "hc_tc5_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc5_phy) }, + { "hc_tc5_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc5_phy) }, + { "hc_tc5_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc5_phy) }, + { "hc_tc6_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc6_phy) }, + { "hc_tc6_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc6_phy) }, + { "hc_tc6_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc6_phy) }, + { "hc_tc6_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc6_phy) }, + { "hc_tc7_rx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, rx_pkt_tc7_phy) }, + { "hc_tc7_rx_byte_phy", offsetof(struct mana_ethtool_phy_stats, rx_byte_tc7_phy) }, + { "hc_tc7_tx_pkt_phy", offsetof(struct mana_ethtool_phy_stats, tx_pkt_tc7_phy) }, + { "hc_tc7_tx_byte_phy", offsetof(struct mana_ethtool_phy_stats, tx_byte_tc7_phy) }, + { "hc_tc0_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc0_phy) }, + { "hc_tc0_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc0_phy) }, + { "hc_tc1_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc1_phy) }, + { "hc_tc1_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc1_phy) }, + { "hc_tc2_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc2_phy) }, + { "hc_tc2_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc2_phy) }, + { "hc_tc3_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc3_phy) }, + { "hc_tc3_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc3_phy) }, + { "hc_tc4_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc4_phy) }, + { "hc_tc4_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc4_phy) }, + { "hc_tc5_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc5_phy) }, + { "hc_tc5_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc5_phy) }, + { "hc_tc6_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc6_phy) }, + { "hc_tc6_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc6_phy) }, + { "hc_tc7_rx_pause_phy", offsetof(struct mana_ethtool_phy_stats, rx_pause_tc7_phy) }, + { "hc_tc7_tx_pause_phy", offsetof(struct mana_ethtool_phy_stats, tx_pause_tc7_phy) }, +}; + +static const char mana_priv_flags[MANA_PRIV_FLAG_MAX][ETH_GSTRING_LEN] = { + [MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF] = "full-page-rx" +}; + +static int mana_get_sset_count(struct net_device *ndev, int stringset) +{ + struct mana_port_context *apc = netdev_priv(ndev); + unsigned int num_queues = apc->num_queues; + + switch (stringset) { + case ETH_SS_STATS: + return ARRAY_SIZE(mana_eth_stats) + + ARRAY_SIZE(mana_phy_stats) + + ARRAY_SIZE(mana_hc_stats) + + num_queues * (MANA_STATS_RX_COUNT + MANA_STATS_TX_COUNT); + + case ETH_SS_PRIV_FLAGS: + return MANA_PRIV_FLAG_MAX; + + default: + return -EINVAL; + } +} + +static void mana_get_strings_stats(struct mana_port_context *apc, u8 **data) +{ + unsigned int num_queues = apc->num_queues; + int i, j; + + for (i = 0; i < ARRAY_SIZE(mana_eth_stats); i++) + ethtool_puts(data, mana_eth_stats[i].name); + + for (i = 0; i < ARRAY_SIZE(mana_hc_stats); i++) + ethtool_puts(data, mana_hc_stats[i].name); + + for (i = 0; i < ARRAY_SIZE(mana_phy_stats); i++) + ethtool_puts(data, mana_phy_stats[i].name); + + for (i = 0; i < num_queues; i++) { + ethtool_sprintf(data, "rx_%d_packets", i); + ethtool_sprintf(data, "rx_%d_bytes", i); + ethtool_sprintf(data, "rx_%d_xdp_drop", i); + ethtool_sprintf(data, "rx_%d_xdp_tx", i); + ethtool_sprintf(data, "rx_%d_xdp_redirect", i); + ethtool_sprintf(data, "rx_%d_pkt_len0_err", i); + for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) + ethtool_sprintf(data, + "rx_%d_coalesced_cqe_%d", + i, + j + 2); + } + + for (i = 0; i < num_queues; i++) { + ethtool_sprintf(data, "tx_%d_packets", i); + ethtool_sprintf(data, "tx_%d_bytes", i); + ethtool_sprintf(data, "tx_%d_xdp_xmit", i); + ethtool_sprintf(data, "tx_%d_tso_packets", i); + ethtool_sprintf(data, "tx_%d_tso_bytes", i); + ethtool_sprintf(data, "tx_%d_tso_inner_packets", i); + ethtool_sprintf(data, "tx_%d_tso_inner_bytes", i); + ethtool_sprintf(data, "tx_%d_long_pkt_fmt", i); + ethtool_sprintf(data, "tx_%d_short_pkt_fmt", i); + ethtool_sprintf(data, "tx_%d_csum_partial", i); + ethtool_sprintf(data, "tx_%d_mana_map_err", i); + } +} + +static void mana_get_strings_priv_flags(u8 **data) +{ + int i; + + for (i = 0; i < MANA_PRIV_FLAG_MAX; i++) + ethtool_puts(data, mana_priv_flags[i]); +} + +static void mana_get_strings(struct net_device *ndev, u32 stringset, u8 *data) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + switch (stringset) { + case ETH_SS_STATS: + mana_get_strings_stats(apc, &data); + break; + case ETH_SS_PRIV_FLAGS: + mana_get_strings_priv_flags(&data); + break; + default: + break; + } +} + +static void mana_get_ethtool_stats(struct net_device *ndev, + struct ethtool_stats *e_stats, u64 *data) +{ + struct mana_port_context *apc = netdev_priv(ndev); + unsigned int num_queues = apc->num_queues; + void *eth_stats = &apc->eth_stats; + void *hc_stats = &apc->ac->hc_stats; + void *phy_stats = &apc->phy_stats; + struct mana_stats_rx *rx_stats; + struct mana_stats_tx *tx_stats; + unsigned int start; + u64 packets, bytes; + u64 xdp_redirect; + u64 xdp_xmit; + u64 xdp_drop; + u64 xdp_tx; + u64 pkt_len0_err; + u64 coalesced_cqe[MANA_CQE_COAL_PKTS_8 - 1]; + u64 tso_packets; + u64 tso_bytes; + u64 tso_inner_packets; + u64 tso_inner_bytes; + u64 long_pkt_fmt; + u64 short_pkt_fmt; + u64 csum_partial; + u64 mana_map_err; + int q, i = 0, j; + + if (!apc->port_is_up) + return; + + /* We call this mana function to get the phy stats from GDMA and includes + * aggregate tx/rx drop counters, Per-TC(Traffic Channel) tx/rx and pause + * counters. + */ + mana_query_phy_stats(apc); + + for (q = 0; q < ARRAY_SIZE(mana_eth_stats); q++) + data[i++] = *(u64 *)(eth_stats + mana_eth_stats[q].offset); + + for (q = 0; q < ARRAY_SIZE(mana_hc_stats); q++) + data[i++] = *(u64 *)(hc_stats + mana_hc_stats[q].offset); + + for (q = 0; q < ARRAY_SIZE(mana_phy_stats); q++) + data[i++] = *(u64 *)(phy_stats + mana_phy_stats[q].offset); + + for (q = 0; q < num_queues; q++) { + rx_stats = &apc->rxqs[q]->stats; + + do { + start = u64_stats_fetch_begin(&rx_stats->syncp); + packets = rx_stats->packets; + bytes = rx_stats->bytes; + xdp_drop = rx_stats->xdp_drop; + xdp_tx = rx_stats->xdp_tx; + xdp_redirect = rx_stats->xdp_redirect; + pkt_len0_err = rx_stats->pkt_len0_err; + for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) + coalesced_cqe[j] = rx_stats->coalesced_cqe[j]; + } while (u64_stats_fetch_retry(&rx_stats->syncp, start)); + + data[i++] = packets; + data[i++] = bytes; + data[i++] = xdp_drop; + data[i++] = xdp_tx; + data[i++] = xdp_redirect; + data[i++] = pkt_len0_err; + for (j = 0; j < MANA_CQE_COAL_PKTS_8 - 1; j++) + data[i++] = coalesced_cqe[j]; + } + + for (q = 0; q < num_queues; q++) { + tx_stats = &apc->tx_qp[q]->txq.stats; + + do { + start = u64_stats_fetch_begin(&tx_stats->syncp); + packets = tx_stats->packets; + bytes = tx_stats->bytes; + xdp_xmit = tx_stats->xdp_xmit; + tso_packets = tx_stats->tso_packets; + tso_bytes = tx_stats->tso_bytes; + tso_inner_packets = tx_stats->tso_inner_packets; + tso_inner_bytes = tx_stats->tso_inner_bytes; + long_pkt_fmt = tx_stats->long_pkt_fmt; + short_pkt_fmt = tx_stats->short_pkt_fmt; + csum_partial = tx_stats->csum_partial; + mana_map_err = tx_stats->mana_map_err; + } while (u64_stats_fetch_retry(&tx_stats->syncp, start)); + + data[i++] = packets; + data[i++] = bytes; + data[i++] = xdp_xmit; + data[i++] = tso_packets; + data[i++] = tso_bytes; + data[i++] = tso_inner_packets; + data[i++] = tso_inner_bytes; + data[i++] = long_pkt_fmt; + data[i++] = short_pkt_fmt; + data[i++] = csum_partial; + data[i++] = mana_map_err; + } +} + +static u32 mana_get_rx_ring_count(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + return apc->num_queues; +} + +static u32 mana_get_rxfh_key_size(struct net_device *ndev) +{ + return MANA_HASH_KEY_SIZE; +} + +static u32 mana_rss_indir_size(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + return apc->indir_table_sz; +} + +static int mana_get_rxfh(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh) +{ + struct mana_port_context *apc = netdev_priv(ndev); + int i; + + rxfh->hfunc = ETH_RSS_HASH_TOP; /* Toeplitz */ + + if (rxfh->indir) { + for (i = 0; i < apc->indir_table_sz; i++) + rxfh->indir[i] = apc->indir_table[i]; + } + + if (rxfh->key) + memcpy(rxfh->key, apc->hashkey, MANA_HASH_KEY_SIZE); + + return 0; +} + +static int mana_set_rxfh(struct net_device *ndev, + struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + bool update_hash = false, update_table = false; + u8 save_key[MANA_HASH_KEY_SIZE]; + u32 *save_table; + int i, err; + + if (!apc->port_is_up) + return -EOPNOTSUPP; + + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + save_table = kcalloc(apc->indir_table_sz, sizeof(u32), GFP_KERNEL); + if (!save_table) + return -ENOMEM; + + if (rxfh->indir) { + for (i = 0; i < apc->indir_table_sz; i++) + if (rxfh->indir[i] >= apc->num_queues) { + err = -EINVAL; + goto cleanup; + } + + update_table = true; + for (i = 0; i < apc->indir_table_sz; i++) { + save_table[i] = apc->indir_table[i]; + apc->indir_table[i] = rxfh->indir[i]; + } + } + + if (rxfh->key) { + update_hash = true; + memcpy(save_key, apc->hashkey, MANA_HASH_KEY_SIZE); + memcpy(apc->hashkey, rxfh->key, MANA_HASH_KEY_SIZE); + } + + err = mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); + + if (err) { /* recover to original values */ + if (update_table) { + for (i = 0; i < apc->indir_table_sz; i++) + apc->indir_table[i] = save_table[i]; + } + + if (update_hash) + memcpy(apc->hashkey, save_key, MANA_HASH_KEY_SIZE); + + mana_config_rss(apc, TRI_STATE_TRUE, update_hash, update_table); + } + +cleanup: + kfree(save_table); + + return err; +} + +static void mana_get_channels(struct net_device *ndev, + struct ethtool_channels *channel) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + channel->max_combined = apc->max_queues; + channel->combined_count = apc->num_queues; +} + +#define MANA_RX_CQE_NSEC_DEF 2048 +static int mana_get_coalesce(struct net_device *ndev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + kernel_coal->rx_cqe_frames = + apc->cqe8_coalescing_enable ? MANA_CQE_COAL_PKTS_8 : + apc->cqe_coalescing_enable ? MANA_RXCOMP_OOB_NUM_PPI : 1; + + kernel_coal->rx_cqe_nsecs = apc->cqe_coalescing_timeout_ns; + + /* Return the default timeout value for old FW not providing + * this value. + */ + if (apc->port_is_up && apc->cqe_coalescing_enable && + !kernel_coal->rx_cqe_nsecs) + kernel_coal->rx_cqe_nsecs = MANA_RX_CQE_NSEC_DEF; + + ec->rx_coalesce_usecs = apc->intr_modr_rx_usec; + ec->rx_max_coalesced_frames = apc->intr_modr_rx_comp; + + ec->tx_coalesce_usecs = apc->intr_modr_tx_usec; + ec->tx_max_coalesced_frames = apc->intr_modr_tx_comp; + + ec->use_adaptive_rx_coalesce = apc->rx_dim_enabled; + ec->use_adaptive_tx_coalesce = apc->tx_dim_enabled; + + return 0; +} + +static int mana_set_coalesce(struct net_device *ndev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + struct { + u16 intr_modr_rx_usec; + u16 intr_modr_rx_comp; + u16 intr_modr_tx_usec; + u16 intr_modr_tx_comp; + u8 cqe_coalescing_enable; + u8 cqe8_coalescing_enable; + bool rx_dim_enabled; + bool tx_dim_enabled; + } saved; + bool modr_changed = false; + bool dim_changed = false; + struct gdma_context *gc; + u32 max_cqe_frames; + int err; + + gc = apc->ac->gdma_dev->gdma_context; + max_cqe_frames = gc->cqe8_coalescing_sup ? MANA_CQE_COAL_PKTS_8 : + MANA_RXCOMP_OOB_NUM_PPI; + + /* Both static and dynamic interrupt moderation (DIM) rely on the + * same HW capability advertised by the PF. + */ + if ((ec->use_adaptive_rx_coalesce || ec->use_adaptive_tx_coalesce || + ec->rx_coalesce_usecs || ec->tx_coalesce_usecs || + ec->rx_max_coalesced_frames || ec->tx_max_coalesced_frames) && + !(gc->pf_cap_flags1 & GDMA_PF_CAP_FLAG_1_DYN_INTERRUPT_MODERATION)) { + NL_SET_ERR_MSG(extack, + "Interrupt Moderation is not supported by HW"); + return -EOPNOTSUPP; + } + + if (kernel_coal->rx_cqe_frames != 1 && + kernel_coal->rx_cqe_frames != MANA_RXCOMP_OOB_NUM_PPI && + kernel_coal->rx_cqe_frames != max_cqe_frames) { + NL_SET_ERR_MSG_FMT(extack, + "rx-frames must be 1 or %u%s, got %u", + MANA_RXCOMP_OOB_NUM_PPI, + gc->cqe8_coalescing_sup ? " or 8" : "", + kernel_coal->rx_cqe_frames); + return -EINVAL; + } + + if (ec->rx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX || + ec->tx_coalesce_usecs > MANA_INTR_MODR_USEC_MAX) { + NL_SET_ERR_MSG_FMT(extack, + "coalesce usecs must be <= %lu", + MANA_INTR_MODR_USEC_MAX); + return -EINVAL; + } + + if (ec->rx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX || + ec->tx_max_coalesced_frames > MANA_INTR_MODR_COMP_MAX) { + NL_SET_ERR_MSG_FMT(extack, + "coalesce frames must be <= %lu", + MANA_INTR_MODR_COMP_MAX); + return -EINVAL; + } + + if (ec->rx_coalesce_usecs != apc->intr_modr_rx_usec || + ec->rx_max_coalesced_frames != apc->intr_modr_rx_comp || + ec->tx_coalesce_usecs != apc->intr_modr_tx_usec || + ec->tx_max_coalesced_frames != apc->intr_modr_tx_comp) + modr_changed = true; + + saved.intr_modr_rx_usec = apc->intr_modr_rx_usec; + saved.intr_modr_rx_comp = apc->intr_modr_rx_comp; + saved.intr_modr_tx_usec = apc->intr_modr_tx_usec; + saved.intr_modr_tx_comp = apc->intr_modr_tx_comp; + + apc->intr_modr_rx_usec = ec->rx_coalesce_usecs; + apc->intr_modr_rx_comp = ec->rx_max_coalesced_frames; + apc->intr_modr_tx_usec = ec->tx_coalesce_usecs; + apc->intr_modr_tx_comp = ec->tx_max_coalesced_frames; + + if (!!ec->use_adaptive_rx_coalesce != apc->rx_dim_enabled || + !!ec->use_adaptive_tx_coalesce != apc->tx_dim_enabled) + dim_changed = true; + + saved.rx_dim_enabled = apc->rx_dim_enabled; + saved.tx_dim_enabled = apc->tx_dim_enabled; + + saved.cqe_coalescing_enable = apc->cqe_coalescing_enable; + saved.cqe8_coalescing_enable = apc->cqe8_coalescing_enable; + apc->cqe_coalescing_enable = + kernel_coal->rx_cqe_frames >= MANA_RXCOMP_OOB_NUM_PPI; + apc->cqe8_coalescing_enable = + kernel_coal->rx_cqe_frames == MANA_CQE_COAL_PKTS_8; + + if (!apc->port_is_up) { + WRITE_ONCE(apc->rx_dim_enabled, !!ec->use_adaptive_rx_coalesce); + WRITE_ONCE(apc->tx_dim_enabled, !!ec->use_adaptive_tx_coalesce); + return 0; + } + + if (apc->cqe_coalescing_enable != saved.cqe_coalescing_enable || + apc->cqe8_coalescing_enable != saved.cqe8_coalescing_enable) { + /* CQE coalescing setting is applied via RSS configuration. */ + err = mana_config_rss(apc, TRI_STATE_TRUE, false, false); + if (err) { + netdev_err(ndev, "Change CQE coalescing failed: %d\n", + err); + apc->cqe_coalescing_enable = + saved.cqe_coalescing_enable; + apc->cqe8_coalescing_enable = + saved.cqe8_coalescing_enable; + apc->intr_modr_rx_usec = saved.intr_modr_rx_usec; + apc->intr_modr_rx_comp = saved.intr_modr_rx_comp; + apc->intr_modr_tx_usec = saved.intr_modr_tx_usec; + apc->intr_modr_tx_comp = saved.intr_modr_tx_comp; + return err; + } + } + + if (modr_changed || dim_changed) { + bool new_rx_dim = !!ec->use_adaptive_rx_coalesce; + bool new_tx_dim = !!ec->use_adaptive_tx_coalesce; + bool disable_rx_dim = saved.rx_dim_enabled && !new_rx_dim; + bool disable_tx_dim = saved.tx_dim_enabled && !new_tx_dim; + bool enable_rx_dim = !saved.rx_dim_enabled && new_rx_dim; + bool enable_tx_dim = !saved.tx_dim_enabled && new_tx_dim; + int q; + + /* On disable: clear the per-port flag first and + * synchronize_net() so any in-flight NAPI poll observes + * the new value and will not schedule further DIM work; + * then drain pending work and restore the static + * moderation values. + */ + if (disable_rx_dim) + WRITE_ONCE(apc->rx_dim_enabled, false); + if (disable_tx_dim) + WRITE_ONCE(apc->tx_dim_enabled, false); + if (disable_rx_dim || disable_tx_dim) + synchronize_net(); + + for (q = 0; q < apc->num_queues; q++) { + struct mana_cq *rx_cq = &apc->rxqs[q]->rx_cq; + struct mana_cq *tx_cq = &apc->tx_qp[q]->tx_cq; + + if (disable_rx_dim) + mana_dim_change(rx_cq, false); + else if (enable_rx_dim) + mana_dim_change(rx_cq, true); + else if (!new_rx_dim && modr_changed) + mana_gd_ring_dim(rx_cq->gdma_cq, + apc->intr_modr_rx_usec, true, + apc->intr_modr_rx_comp, true); + + if (disable_tx_dim) + mana_dim_change(tx_cq, false); + else if (enable_tx_dim) + mana_dim_change(tx_cq, true); + else if (!new_tx_dim && modr_changed) + mana_gd_ring_dim(tx_cq->gdma_cq, + apc->intr_modr_tx_usec, true, + apc->intr_modr_tx_comp, true); + } + + /* Publish the enable flag with release semantics so a + * concurrent NAPI poll that observes it set also sees the DIM + * (re)init done by mana_dim_change() above. + */ + if (enable_rx_dim) + /* pairs with smp_load_acquire() in mana_update_rx_dim() */ + smp_store_release(&apc->rx_dim_enabled, true); + if (enable_tx_dim) + /* pairs with smp_load_acquire() in mana_update_tx_dim() */ + smp_store_release(&apc->tx_dim_enabled, true); + } + + return 0; +} + +/* mana_set_channels - change the number of queues on a port + * + * Returns -EBUSY if RDMA holds the vport with EQs sized to the + * current num_queues. + */ +static int mana_set_channels(struct net_device *ndev, + struct ethtool_channels *channels) +{ + struct mana_port_context *apc = netdev_priv(ndev); + unsigned int new_count = channels->combined_count; + unsigned int old_count = apc->num_queues; + int err; + + /* Set channel_changing to block RDMA from grabbing the vport + * during the detach/attach window. mana_cfg_vport() checks + * this flag under vport_mutex and returns -EBUSY if set. + */ + mutex_lock(&apc->vport_mutex); + if (!apc->port_is_up && apc->vport_use_count) { + mutex_unlock(&apc->vport_mutex); + return -EBUSY; + } + apc->channel_changing = true; + mutex_unlock(&apc->vport_mutex); + + err = mana_pre_alloc_rxbufs(apc, ndev->mtu, new_count); + if (err) { + netdev_err(ndev, "Insufficient memory for new allocations"); + goto clear_flag; + } + + err = mana_detach(ndev, false); + if (err) { + netdev_err(ndev, "mana_detach failed: %d\n", err); + goto out; + } + + apc->num_queues = new_count; + err = mana_attach(ndev); + if (err) { + apc->num_queues = old_count; + netdev_err(ndev, "mana_attach failed: %d\n", err); + } + +out: + mana_pre_dealloc_rxbufs(apc); +clear_flag: + mutex_lock(&apc->vport_mutex); + apc->channel_changing = false; + mutex_unlock(&apc->vport_mutex); + return err; +} + +static void mana_get_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + ring->rx_pending = apc->rx_queue_size; + ring->tx_pending = apc->tx_queue_size; + ring->rx_max_pending = MAX_RX_BUFFERS_PER_QUEUE; + ring->tx_max_pending = MAX_TX_BUFFERS_PER_QUEUE; +} + +static int mana_set_ringparam(struct net_device *ndev, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct mana_port_context *apc = netdev_priv(ndev); + u32 new_tx, new_rx; + u32 old_tx, old_rx; + int err; + + old_tx = apc->tx_queue_size; + old_rx = apc->rx_queue_size; + + if (ring->tx_pending < MIN_TX_BUFFERS_PER_QUEUE) { + NL_SET_ERR_MSG_FMT(extack, "tx:%d less than the min:%d", ring->tx_pending, + MIN_TX_BUFFERS_PER_QUEUE); + return -EINVAL; + } + + if (ring->rx_pending < MIN_RX_BUFFERS_PER_QUEUE) { + NL_SET_ERR_MSG_FMT(extack, "rx:%d less than the min:%d", ring->rx_pending, + MIN_RX_BUFFERS_PER_QUEUE); + return -EINVAL; + } + + new_rx = roundup_pow_of_two(ring->rx_pending); + new_tx = roundup_pow_of_two(ring->tx_pending); + netdev_info(ndev, "Using nearest power of 2 values for Txq:%d Rxq:%d\n", + new_tx, new_rx); + + /* pre-allocating new buffers to prevent failures in mana_attach() later */ + apc->rx_queue_size = new_rx; + err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); + apc->rx_queue_size = old_rx; + if (err) { + netdev_err(ndev, "Insufficient memory for new allocations\n"); + return err; + } + + err = mana_detach(ndev, false); + if (err) { + netdev_err(ndev, "mana_detach failed: %d\n", err); + goto out; + } + + apc->tx_queue_size = new_tx; + apc->rx_queue_size = new_rx; + + err = mana_attach(ndev); + if (err) { + netdev_err(ndev, "mana_attach failed: %d\n", err); + apc->tx_queue_size = old_tx; + apc->rx_queue_size = old_rx; + } +out: + mana_pre_dealloc_rxbufs(apc); + return err; +} + +static int mana_get_link_ksettings(struct net_device *ndev, + struct ethtool_link_ksettings *cmd) +{ + struct mana_port_context *apc = netdev_priv(ndev); + int err; + + err = mana_query_link_cfg(apc); + cmd->base.speed = (err) ? SPEED_UNKNOWN : apc->max_speed; + + cmd->base.duplex = DUPLEX_FULL; + cmd->base.port = PORT_OTHER; + + return 0; +} + +static u32 mana_get_priv_flags(struct net_device *ndev) +{ + struct mana_port_context *apc = netdev_priv(ndev); + + return apc->priv_flags; +} + +static int mana_set_priv_flags(struct net_device *ndev, u32 priv_flags) +{ + struct mana_port_context *apc = netdev_priv(ndev); + u32 changed = apc->priv_flags ^ priv_flags; + u32 old_priv_flags = apc->priv_flags; + int err = 0; + + if (!changed) + return 0; + + /* Reject unknown bits */ + if (priv_flags & ~GENMASK(MANA_PRIV_FLAG_MAX - 1, 0)) + return -EINVAL; + + apc->priv_flags = priv_flags; + + if (changed & BIT(MANA_PRIV_FLAG_USE_FULL_PAGE_RXBUF)) { + if (!apc->port_is_up) + return 0; + + /* If XDP is attached or MTU is jumbo, single-buffer-per-page + * is already forced regardless of this flag. Skip the + * expensive detach/attach cycle since nothing changes. + */ + if (ndev->mtu + MANA_RXBUF_PAD > PAGE_SIZE / 2 || + mana_xdp_get(apc)) + return 0; + + /* Block RDMA from grabbing the vport during detach/attach */ + mutex_lock(&apc->vport_mutex); + apc->channel_changing = true; + mutex_unlock(&apc->vport_mutex); + + err = mana_pre_alloc_rxbufs(apc, ndev->mtu, apc->num_queues); + if (err) { + netdev_err(ndev, + "Insufficient memory for new allocations\n"); + apc->priv_flags = old_priv_flags; + goto clear_flag; + } + + err = mana_detach(ndev, false); + if (err) { + netdev_err(ndev, "mana_detach failed: %d\n", err); + apc->priv_flags = old_priv_flags; + goto out; + } + + err = mana_attach(ndev); + if (err) { + netdev_err(ndev, "mana_attach failed: %d\n", err); + apc->priv_flags = old_priv_flags; + } + } + +out: + mana_pre_dealloc_rxbufs(apc); +clear_flag: + mutex_lock(&apc->vport_mutex); + apc->channel_changing = false; + mutex_unlock(&apc->vport_mutex); + + return err; +} + +const struct ethtool_ops mana_ethtool_ops = { + .supported_coalesce_params = ETHTOOL_COALESCE_RX_CQE_FRAMES | + ETHTOOL_COALESCE_RX_USECS | + ETHTOOL_COALESCE_RX_MAX_FRAMES | + ETHTOOL_COALESCE_TX_USECS | + ETHTOOL_COALESCE_TX_MAX_FRAMES | + ETHTOOL_COALESCE_USE_ADAPTIVE_RX | + ETHTOOL_COALESCE_USE_ADAPTIVE_TX, + .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | + ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | + ETHTOOL_OP_NEEDS_RTNL_SPFLAGS | + ETHTOOL_OP_NEEDS_RTNL_GLINK, + .get_ethtool_stats = mana_get_ethtool_stats, + .get_sset_count = mana_get_sset_count, + .get_strings = mana_get_strings, + .get_rx_ring_count = mana_get_rx_ring_count, + .get_rxfh_key_size = mana_get_rxfh_key_size, + .get_rxfh_indir_size = mana_rss_indir_size, + .get_rxfh = mana_get_rxfh, + .set_rxfh = mana_set_rxfh, + .get_channels = mana_get_channels, + .set_channels = mana_set_channels, + .get_coalesce = mana_get_coalesce, + .set_coalesce = mana_set_coalesce, + .get_ringparam = mana_get_ringparam, + .set_ringparam = mana_set_ringparam, + .get_link_ksettings = mana_get_link_ksettings, + .get_link = ethtool_op_get_link, + .get_priv_flags = mana_get_priv_flags, + .set_priv_flags = mana_set_priv_flags, +}; diff --git a/drivers/net/ethernet/microsoft/mana/shm_channel.c b/drivers/net/ethernet/microsoft/mana/shm_channel.c new file mode 100644 index 000000000..d21b5db06 --- /dev/null +++ b/drivers/net/ethernet/microsoft/mana/shm_channel.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause +/* Copyright (c) 2021, Microsoft Corporation. */ + +#include +#include +#include +#include + +#include +#include + +#define PAGE_FRAME_L48_WIDTH_BYTES 6 +#define PAGE_FRAME_L48_WIDTH_BITS (PAGE_FRAME_L48_WIDTH_BYTES * 8) +#define PAGE_FRAME_L48_MASK 0x0000FFFFFFFFFFFF +#define PAGE_FRAME_H4_WIDTH_BITS 4 +#define VECTOR_MASK 0xFFFF +#define SHMEM_VF_RESET_STATE ((u32)-1) + +#define SMC_MSG_TYPE_ESTABLISH_HWC 1 +#define SMC_MSG_TYPE_ESTABLISH_HWC_VERSION 0 + +#define SMC_MSG_TYPE_DESTROY_HWC 2 +#define SMC_MSG_TYPE_DESTROY_HWC_VERSION 0 + +#define SMC_MSG_DIRECTION_REQUEST 0 +#define SMC_MSG_DIRECTION_RESPONSE 1 + +/* Structures labeled with "HW DATA" are exchanged with the hardware. All of + * them are naturally aligned and hence don't need __packed. + */ + +/* Shared memory channel protocol header + * + * msg_type: set on request and response; response matches request. + * msg_version: newer PF writes back older response (matching request) + * older PF acts on latest version known and sets that version in result + * (less than request). + * direction: 0 for request, VF->PF; 1 for response, PF->VF. + * status: 0 on request, + * operation result on response (success = 0, failure = 1 or greater). + * reset_vf: If set on either establish or destroy request, indicates perform + * FLR before/after the operation. + * owner_is_pf: 1 indicates PF owned, 0 indicates VF owned. + */ +union smc_proto_hdr { + u32 as_uint32; + + struct { + u8 msg_type : 3; + u8 msg_version : 3; + u8 reserved_1 : 1; + u8 direction : 1; + + u8 status; + + u8 reserved_2; + + u8 reset_vf : 1; + u8 reserved_3 : 6; + u8 owner_is_pf : 1; + }; +}; /* HW DATA */ + +static int mana_smc_poll_register(void __iomem *base, bool reset) +{ + void __iomem *ptr = base + SMC_LAST_DWORD * SMC_BASIC_UNIT; + u32 last_dword; + int i; + + /* Poll the hardware for the ownership bit. This should be pretty fast, + * but let's do it in a loop just in case the hardware or the PF + * driver are temporarily busy. + */ + for (i = 0; i < 20 * 1000; i++) { + last_dword = readl(ptr); + + /* shmem reads as 0xFFFFFFFF in the reset case */ + if (reset && last_dword == SHMEM_VF_RESET_STATE) + return 0; + + /* If bit_31 is set, the PF currently owns the SMC. */ + if (!(last_dword & BIT(31))) + return 0; + + usleep_range(1000, 2000); + } + + return -ETIMEDOUT; +} + +static int mana_smc_read_response(struct shm_channel *sc, u32 msg_type, + u32 msg_version, bool reset_vf) +{ + void __iomem *base = sc->base; + union smc_proto_hdr hdr; + int err; + + /* Wait for PF to respond. */ + err = mana_smc_poll_register(base, reset_vf); + if (err) + return err; + + hdr.as_uint32 = readl(base + SMC_LAST_DWORD * SMC_BASIC_UNIT); + + if (reset_vf && hdr.as_uint32 == SHMEM_VF_RESET_STATE) + return 0; + + /* Validate protocol fields from the PF driver */ + if (hdr.msg_type != msg_type || hdr.msg_version > msg_version || + hdr.direction != SMC_MSG_DIRECTION_RESPONSE) { + dev_err(sc->dev, "Wrong SMC response 0x%x, type=%d, ver=%d\n", + hdr.as_uint32, msg_type, msg_version); + return -EPROTO; + } + + /* Validate the operation result */ + if (hdr.status != 0) { + dev_err(sc->dev, "SMC operation failed: 0x%x\n", hdr.status); + return -EPROTO; + } + + return 0; +} + +void mana_smc_init(struct shm_channel *sc, struct device *dev, + void __iomem *base) +{ + sc->dev = dev; + sc->base = base; +} + +int mana_smc_setup_hwc(struct shm_channel *sc, bool reset_vf, u64 eq_addr, + u64 cq_addr, u64 rq_addr, u64 sq_addr, + u32 eq_msix_index) +{ + union smc_proto_hdr *hdr; + u16 all_addr_h4bits = 0; + u16 frame_addr_seq = 0; + u64 frame_addr = 0; + u8 shm_buf[32]; + u64 *shmem; + u32 *dword; + u8 *ptr; + int err; + int i; + + /* Ensure VF already has possession of shared memory */ + err = mana_smc_poll_register(sc->base, false); + if (err) { + dev_err(sc->dev, "Timeout when setting up HWC: %d\n", err); + return err; + } + + if (!MANA_PAGE_ALIGNED(eq_addr) || !MANA_PAGE_ALIGNED(cq_addr) || + !MANA_PAGE_ALIGNED(rq_addr) || !MANA_PAGE_ALIGNED(sq_addr)) + return -EINVAL; + + if ((eq_msix_index & VECTOR_MASK) != eq_msix_index) + return -EINVAL; + + /* Scheme for packing four addresses and extra info into 256 bits. + * + * Addresses must be page frame aligned, so only frame address bits + * are transferred. + * + * 52-bit frame addresses are split into the lower 48 bits and upper + * 4 bits. Lower 48 bits of 4 address are written sequentially from + * the start of the 256-bit shared memory region followed by 16 bits + * containing the upper 4 bits of the 4 addresses in sequence. + * + * A 16 bit EQ vector number fills out the next-to-last 32-bit dword. + * + * The final 32-bit dword is used for protocol control information as + * defined in smc_proto_hdr. + */ + + memset(shm_buf, 0, sizeof(shm_buf)); + ptr = shm_buf; + + /* EQ addr: low 48 bits of frame address */ + shmem = (u64 *)ptr; + frame_addr = MANA_PFN(eq_addr); + *shmem = frame_addr & PAGE_FRAME_L48_MASK; + all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << + (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); + ptr += PAGE_FRAME_L48_WIDTH_BYTES; + + /* CQ addr: low 48 bits of frame address */ + shmem = (u64 *)ptr; + frame_addr = MANA_PFN(cq_addr); + *shmem = frame_addr & PAGE_FRAME_L48_MASK; + all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << + (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); + ptr += PAGE_FRAME_L48_WIDTH_BYTES; + + /* RQ addr: low 48 bits of frame address */ + shmem = (u64 *)ptr; + frame_addr = MANA_PFN(rq_addr); + *shmem = frame_addr & PAGE_FRAME_L48_MASK; + all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << + (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); + ptr += PAGE_FRAME_L48_WIDTH_BYTES; + + /* SQ addr: low 48 bits of frame address */ + shmem = (u64 *)ptr; + frame_addr = MANA_PFN(sq_addr); + *shmem = frame_addr & PAGE_FRAME_L48_MASK; + all_addr_h4bits |= (frame_addr >> PAGE_FRAME_L48_WIDTH_BITS) << + (frame_addr_seq++ * PAGE_FRAME_H4_WIDTH_BITS); + ptr += PAGE_FRAME_L48_WIDTH_BYTES; + + /* High 4 bits of the four frame addresses */ + *((u16 *)ptr) = all_addr_h4bits; + ptr += sizeof(u16); + + /* EQ MSIX vector number */ + *((u16 *)ptr) = (u16)eq_msix_index; + ptr += sizeof(u16); + + /* 32-bit protocol header in final dword */ + *((u32 *)ptr) = 0; + + hdr = (union smc_proto_hdr *)ptr; + hdr->msg_type = SMC_MSG_TYPE_ESTABLISH_HWC; + hdr->msg_version = SMC_MSG_TYPE_ESTABLISH_HWC_VERSION; + hdr->direction = SMC_MSG_DIRECTION_REQUEST; + hdr->reset_vf = reset_vf; + + /* Write 256-message buffer to shared memory (final 32-bit write + * triggers HW to set possession bit to PF). + */ + dword = (u32 *)shm_buf; + for (i = 0; i < SMC_APERTURE_DWORDS; i++) + writel(*dword++, sc->base + i * SMC_BASIC_UNIT); + + /* Read shmem response (polling for VF possession) and validate. + * For setup, waiting for response on shared memory is not strictly + * necessary, since wait occurs later for results to appear in EQE's. + */ + err = mana_smc_read_response(sc, SMC_MSG_TYPE_ESTABLISH_HWC, + SMC_MSG_TYPE_ESTABLISH_HWC_VERSION, + reset_vf); + if (err) { + dev_err(sc->dev, "Error when setting up HWC: %d\n", err); + return err; + } + + return 0; +} + +int mana_smc_teardown_hwc(struct shm_channel *sc, bool reset_vf) +{ + union smc_proto_hdr hdr = {}; + int err; + + /* Ensure already has possession of shared memory */ + err = mana_smc_poll_register(sc->base, false); + if (err) { + dev_err(sc->dev, "Timeout when tearing down HWC\n"); + return err; + } + + /* Set up protocol header for HWC destroy message */ + hdr.msg_type = SMC_MSG_TYPE_DESTROY_HWC; + hdr.msg_version = SMC_MSG_TYPE_DESTROY_HWC_VERSION; + hdr.direction = SMC_MSG_DIRECTION_REQUEST; + hdr.reset_vf = reset_vf; + + /* Write message in high 32 bits of 256-bit shared memory, causing HW + * to set possession bit to PF. + */ + writel(hdr.as_uint32, sc->base + SMC_LAST_DWORD * SMC_BASIC_UNIT); + + /* Read shmem response (polling for VF possession) and validate. + * For teardown, waiting for response is required to ensure hardware + * invalidates MST entries before software frees memory. + */ + err = mana_smc_read_response(sc, SMC_MSG_TYPE_DESTROY_HWC, + SMC_MSG_TYPE_DESTROY_HWC_VERSION, + reset_vf); + if (err) { + dev_err(sc->dev, "Error when tearing down HWC: %d\n", err); + return err; + } + + return 0; +} -- cgit v1.3.1