diff options
| author | Joel Fernandes <joelagnelf@nvidia.com> | 2026-07-18 13:15:36 -0400 |
|---|---|---|
| committer | Paul E. McKenney <paulmck@kernel.org> | 2026-07-23 10:39:57 -0700 |
| commit | 34ff636f322c6d5e4f0a52338fa63a7dff09a495 (patch) | |
| tree | 7629759c1ec7148b95bc8ab3080df20a5ecd2273 /drivers/net/ethernet/meta | |
| download | linux-stable-34ff636f322c6d5e4f0a52338fa63a7dff09a495.tar.gz linux-stable-34ff636f322c6d5e4f0a52338fa63a7dff09a495.zip | |
scftorture: Count single_rpc offline failures in statistics outputgrafted
scf_torture_stats_print() aggregates each invoker thread's counters
into a local scf_statistics structure before printing, but the
n_single_rpc_ofl field is missing from the aggregation loop. As a
result, the "single_rpc_ofl" value printed in the statistics line is
always zero, even when smp_call_function_single() invocations for the
RPC test have failed due to offline CPUs and been counted by the
invoker threads.
Add the missing accumulation so that the printed value reflects the
actual counts.
Signed-off-by: Joel Fernandes <joelagnelf@nvidia.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Diffstat (limited to 'drivers/net/ethernet/meta')
32 files changed, 18601 insertions, 0 deletions
diff --git a/drivers/net/ethernet/meta/Kconfig b/drivers/net/ethernet/meta/Kconfig new file mode 100644 index 000000000..ca5c7ac2a --- /dev/null +++ b/drivers/net/ethernet/meta/Kconfig @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: GPL-2.0-only +# +# Meta Platforms network device configuration +# + +config NET_VENDOR_META + bool "Meta Platforms devices" + default y + help + If you have a network (Ethernet) card designed by Meta, say Y. + That's Meta as in the parent company of Facebook. + + Note that the answer to this question doesn't directly affect the + kernel: saying N will just cause the configurator to skip all + the questions about Meta cards. If you say Y, you will be asked for + your specific card in the following questions. + +if NET_VENDOR_META + +config FBNIC + tristate "Meta Platforms Host Network Interface" + depends on 64BIT || COMPILE_TEST + depends on !S390 + depends on MAX_SKB_FRAGS < 22 + depends on PCI_MSI + depends on PTP_1588_CLOCK_OPTIONAL + select NET_DEVLINK + select PAGE_POOL + select PCS_XPCS + select PHYLINK + select PLDMFW + help + This driver supports Meta Platforms Host Network Interface. + + To compile this driver as a module, choose M here. The module + will be called fbnic. MSI-X interrupt support is required. + +endif # NET_VENDOR_META diff --git a/drivers/net/ethernet/meta/Makefile b/drivers/net/ethernet/meta/Makefile new file mode 100644 index 000000000..88804f3de --- /dev/null +++ b/drivers/net/ethernet/meta/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +# +# Makefile for the Meta Platforms network device drivers. +# + +obj-$(CONFIG_FBNIC) += fbnic/ diff --git a/drivers/net/ethernet/meta/fbnic/Makefile b/drivers/net/ethernet/meta/fbnic/Makefile new file mode 100644 index 000000000..72c41af65 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/Makefile @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: GPL-2.0 +# Copyright (c) Meta Platforms, Inc. and affiliates. + +# +# Makefile for the Meta(R) Host Network Interface +# + +obj-$(CONFIG_FBNIC) += fbnic.o + +fbnic-y := fbnic_csr.o \ + fbnic_debugfs.o \ + fbnic_devlink.o \ + fbnic_ethtool.o \ + fbnic_fw.o \ + fbnic_fw_log.o \ + fbnic_hw_stats.o \ + fbnic_hwmon.o \ + fbnic_irq.o \ + fbnic_mac.o \ + fbnic_netdev.o \ + fbnic_pci.o \ + fbnic_phylink.o \ + fbnic_rpc.o \ + fbnic_mdio.o \ + fbnic_time.o \ + fbnic_tlv.o \ + fbnic_txrx.o \ +# End of objects diff --git a/drivers/net/ethernet/meta/fbnic/fbnic.h b/drivers/net/ethernet/meta/fbnic/fbnic.h new file mode 100644 index 000000000..d0715695c --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic.h @@ -0,0 +1,263 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_H_ +#define _FBNIC_H_ + +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/types.h> +#include <linux/workqueue.h> + +#include "fbnic_csr.h" +#include "fbnic_fw.h" +#include "fbnic_fw_log.h" +#include "fbnic_hw_stats.h" +#include "fbnic_mac.h" +#include "fbnic_rpc.h" + +struct fbnic_napi_vector; + +#define FBNIC_MAX_NAPI_VECTORS 128u +#define FBNIC_MBX_CMPL_SLOTS 4 + +struct fbnic_dev { + struct device *dev; + struct net_device *netdev; + struct dentry *dbg_fbd; + struct device *hwmon; + struct devlink_health_reporter *fw_reporter; + struct devlink_health_reporter *otp_reporter; + + u32 __iomem *uc_addr0; + u32 __iomem *uc_addr4; + const struct fbnic_mac *mac; + unsigned int fw_msix_vector; + unsigned int mac_msix_vector; + unsigned short num_irqs; + + struct { + u8 users; + char name[IFNAMSIZ + 9]; + } napi_irq[FBNIC_MAX_NAPI_VECTORS]; + + struct delayed_work service_task; + + struct fbnic_fw_mbx mbx[FBNIC_IPC_MBX_INDICES]; + struct fbnic_fw_cap fw_cap; + struct fbnic_fw_completion *cmpl_data[FBNIC_MBX_CMPL_SLOTS]; + /* Lock protecting Tx Mailbox queue to prevent possible races */ + spinlock_t fw_tx_lock; + + unsigned long last_heartbeat_request; + unsigned long last_heartbeat_response; + u8 fw_heartbeat_enabled; + + u64 dsn; + u32 mps; + u32 readrq; + u8 relaxed_ord; + + /* Local copy of the devices TCAM */ + struct fbnic_act_tcam act_tcam[FBNIC_RPC_TCAM_ACT_NUM_ENTRIES]; + struct fbnic_mac_addr mac_addr[FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES]; + u8 mac_addr_boundary; + u8 tce_tcam_last; + + /* IP TCAM */ + struct fbnic_ip_addr ip_src[FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES]; + struct fbnic_ip_addr ip_dst[FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES]; + struct fbnic_ip_addr ipo_src[FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES]; + struct fbnic_ip_addr ipo_dst[FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES]; + + /* Number of TCQs/RCQs available on hardware */ + u16 max_num_queues; + + /* Lock protecting writes to @time_high, @time_offset of fbnic_netdev, + * and the HW time CSR machinery. + */ + spinlock_t time_lock; + /* Externally accessible PTP clock, may be NULL */ + struct ptp_clock *ptp; + struct ptp_clock_info ptp_info; + /* Last @time_high refresh time in jiffies (to catch stalls) */ + unsigned long last_read; + + /* PMD specific data */ + unsigned long end_of_pmd_training; + u8 pmd_state; + + /* Local copy of hardware statistics */ + struct fbnic_hw_stats hw_stats; + + /* Firmware time since boot in milliseconds */ + u64 firmware_time; + u64 prev_firmware_time; + + struct fbnic_fw_log fw_log; + + /* MDIO bus for PHYs */ + struct mii_bus *mdio_bus; + + /* In units of ms since API supports values in ms */ + u16 ps_timeout; +}; + +/* Reserve entry 0 in the MSI-X "others" array until we have filled all + * 32 of the possible interrupt slots. By doing this we can avoid any + * potential conflicts should we need to enable one of the debug interrupt + * causes later. + */ +enum { + FBNIC_FW_MSIX_ENTRY, + FBNIC_PCS_MSIX_ENTRY, + FBNIC_NON_NAPI_VECTORS +}; + +static inline bool fbnic_present(struct fbnic_dev *fbd) +{ + return !!READ_ONCE(fbd->uc_addr0); +} + +static inline void fbnic_wr32(struct fbnic_dev *fbd, u32 reg, u32 val) +{ + u32 __iomem *csr = READ_ONCE(fbd->uc_addr0); + + if (csr) + writel(val, csr + reg); +} + +u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg); + +static inline void fbnic_wrfl(struct fbnic_dev *fbd) +{ + fbnic_rd32(fbd, FBNIC_MASTER_SPARE_0); +} + +static inline void +fbnic_rmw32(struct fbnic_dev *fbd, u32 reg, u32 mask, u32 val) +{ + u32 v; + + v = fbnic_rd32(fbd, reg); + v &= ~mask; + v |= val; + fbnic_wr32(fbd, reg, v); +} + +#define wr32(_f, _r, _v) fbnic_wr32(_f, _r, _v) +#define rd32(_f, _r) fbnic_rd32(_f, _r) +#define wrfl(_f) fbnic_wrfl(_f) + +bool fbnic_fw_present(struct fbnic_dev *fbd); +u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg); +void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val); + +#define fw_rd32(_f, _r) fbnic_fw_rd32(_f, _r) +#define fw_wr32(_f, _r, _v) fbnic_fw_wr32(_f, _r, _v) +#define fw_wrfl(_f) fbnic_fw_rd32(_f, FBNIC_FW_ZERO_REG) + +static inline bool fbnic_bmc_present(struct fbnic_dev *fbd) +{ + return fbd->fw_cap.bmc_present; +} + +static inline bool fbnic_init_failure(struct fbnic_dev *fbd) +{ + return !fbd->netdev; +} + +extern char fbnic_driver_name[]; + +void fbnic_devlink_free(struct fbnic_dev *fbd); +struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev); +int fbnic_devlink_health_create(struct fbnic_dev *fbd); +void fbnic_devlink_health_destroy(struct fbnic_dev *fbd); +void fbnic_devlink_register(struct fbnic_dev *fbd); +void fbnic_devlink_unregister(struct fbnic_dev *fbd); +void __printf(2, 3) +fbnic_devlink_fw_report(struct fbnic_dev *fbd, const char *format, ...); +void fbnic_devlink_otp_check(struct fbnic_dev *fbd, const char *msg); + +int fbnic_fw_request_mbx(struct fbnic_dev *fbd); +void fbnic_fw_free_mbx(struct fbnic_dev *fbd); + +void fbnic_hwmon_register(struct fbnic_dev *fbd); +void fbnic_hwmon_unregister(struct fbnic_dev *fbd); + +int fbnic_mac_request_irq(struct fbnic_dev *fbd); +void fbnic_mac_free_irq(struct fbnic_dev *fbd); + +void fbnic_napi_name_irqs(struct fbnic_dev *fbd); +int fbnic_napi_request_irq(struct fbnic_dev *fbd, + struct fbnic_napi_vector *nv); +void fbnic_napi_free_irq(struct fbnic_dev *fbd, + struct fbnic_napi_vector *nv); +void fbnic_synchronize_irq(struct fbnic_dev *fbd, int nr); +int fbnic_request_irq(struct fbnic_dev *dev, int nr, irq_handler_t handler, + unsigned long flags, const char *name, void *data); +void fbnic_free_irq(struct fbnic_dev *dev, int nr, void *data); + +/** + * enum fbnic_msix_self_test_codes - return codes from self test routines + * + * These are the codes returned from the self test routines and + * stored in the test result array indexed by the specific + * test name. + * + * @FBNIC_TEST_MSIX_SUCCESS: no errors + * @FBNIC_TEST_MSIX_NOMEM: allocation failure + * @FBNIC_TEST_MSIX_IRQ_REQ_FAIL: IRQ request failure + * @FBNIC_TEST_MSIX_MASK: masking failed to prevent IRQ + * @FBNIC_TEST_MSIX_UNMASK: unmasking failure w/ sw status set + * @FBNIC_TEST_MSIX_IRQ_CLEAR: interrupt when clearing mask + * @FBNIC_TEST_MSIX_NO_INTERRUPT: no interrupt when not masked + * @FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK: status not cleared, or mask not set + * @FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST: Bits are set after test + */ +enum fbnic_msix_self_test_codes { + FBNIC_TEST_MSIX_SUCCESS = 0, + FBNIC_TEST_MSIX_NOMEM = 5, + FBNIC_TEST_MSIX_IRQ_REQ_FAIL = 10, + FBNIC_TEST_MSIX_MASK = 20, + FBNIC_TEST_MSIX_UNMASK = 30, + FBNIC_TEST_MSIX_IRQ_CLEAR = 40, + FBNIC_TEST_MSIX_NO_INTERRUPT = 50, + FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK = 60, + FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST = 70, +}; + +enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd); + +void fbnic_free_irqs(struct fbnic_dev *fbd); +int fbnic_alloc_irqs(struct fbnic_dev *fbd); + +void fbnic_get_fw_ver_commit_str(struct fbnic_dev *fbd, char *fw_version, + const size_t str_sz); + +void fbnic_dbg_fbd_init(struct fbnic_dev *fbd); +void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd); +void fbnic_dbg_init(void); +void fbnic_dbg_exit(void); + +void fbnic_rpc_reset_valid_entries(struct fbnic_dev *fbd); + +int fbnic_mdiobus_create(struct fbnic_dev *fbd); + +void fbnic_csr_get_regs(struct fbnic_dev *fbd, u32 *data, u32 *regs_version); +int fbnic_csr_regs_len(struct fbnic_dev *fbd); + +void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm); +void fbnic_config_rx_frames(struct fbnic_napi_vector *nv); + +enum fbnic_boards { + fbnic_board_asic +}; + +struct fbnic_info { + unsigned int max_num_queues; + unsigned int bar_mask; +}; + +#endif /* _FBNIC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.c b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c new file mode 100644 index 000000000..dc62d623e --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.c @@ -0,0 +1,277 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include "fbnic.h" + +#define FBNIC_BOUNDS(section) { \ + .start = FBNIC_CSR_START_##section, \ + .end = FBNIC_CSR_END_##section + 1, \ +} + +struct fbnic_csr_bounds { + u32 start; + u32 end; +}; + +static const struct fbnic_csr_bounds fbnic_csr_sects[] = { + FBNIC_BOUNDS(INTR), + FBNIC_BOUNDS(INTR_CQ), + FBNIC_BOUNDS(QM_TX), + FBNIC_BOUNDS(QM_RX), + FBNIC_BOUNDS(TCE), + FBNIC_BOUNDS(TCE_RAM), + FBNIC_BOUNDS(TMI), + FBNIC_BOUNDS(PTP), + FBNIC_BOUNDS(RXB), + FBNIC_BOUNDS(RPC), + FBNIC_BOUNDS(FAB), + FBNIC_BOUNDS(MASTER), + FBNIC_BOUNDS(PCS), + FBNIC_BOUNDS(RSFEC), + FBNIC_BOUNDS(MAC_MAC), + FBNIC_BOUNDS(SIG), + FBNIC_BOUNDS(PCIE_SS_COMPHY), + FBNIC_BOUNDS(PUL_USER), + FBNIC_BOUNDS(QUEUE), + FBNIC_BOUNDS(RPC_RAM), +}; + +#define FBNIC_RPC_TCAM_ACT_DW_PER_ENTRY 14 +#define FBNIC_RPC_TCAM_ACT_NUM_ENTRIES 64 + +#define FBNIC_RPC_TCAM_MACDA_DW_PER_ENTRY 4 +#define FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES 32 + +#define FBNIC_RPC_TCAM_OUTER_IPSRC_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_OUTER_IPSRC_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_OUTER_IPDST_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_OUTER_IPDST_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_IPSRC_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_IPSRC_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_IPDST_DW_PER_ENTRY 9 +#define FBNIC_RPC_TCAM_IPDST_NUM_ENTRIES 8 + +#define FBNIC_RPC_RSS_TBL_DW_PER_ENTRY 2 +#define FBNIC_RPC_RSS_TBL_NUM_ENTRIES 256 + +static void fbnic_csr_get_regs_rpc_ram(struct fbnic_dev *fbd, u32 **data_p) +{ + u32 start = FBNIC_CSR_START_RPC_RAM; + u32 end = FBNIC_CSR_END_RPC_RAM; + u32 *data = *data_p; + u32 i, j; + + *(data++) = start; + *(data++) = end; + + /* FBNIC_RPC_TCAM_ACT */ + for (i = 0; i < FBNIC_RPC_TCAM_ACT_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_ACT_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_ACT(i, j)); + } + + /* FBNIC_RPC_TCAM_MACDA */ + for (i = 0; i < FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_MACDA_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_MACDA(i, j)); + } + + /* FBNIC_RPC_TCAM_OUTER_IPSRC */ + for (i = 0; i < FBNIC_RPC_TCAM_OUTER_IPSRC_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_OUTER_IPSRC_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_OUTER_IPSRC(i, j)); + } + + /* FBNIC_RPC_TCAM_OUTER_IPDST */ + for (i = 0; i < FBNIC_RPC_TCAM_OUTER_IPDST_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_OUTER_IPDST_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_OUTER_IPDST(i, j)); + } + + /* FBNIC_RPC_TCAM_IPSRC */ + for (i = 0; i < FBNIC_RPC_TCAM_IPSRC_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_IPSRC_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_IPSRC(i, j)); + } + + /* FBNIC_RPC_TCAM_IPDST */ + for (i = 0; i < FBNIC_RPC_TCAM_IPDST_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_TCAM_IPDST_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_TCAM_IPDST(i, j)); + } + + /* FBNIC_RPC_RSS_TBL */ + for (i = 0; i < FBNIC_RPC_RSS_TBL_NUM_ENTRIES; i++) { + for (j = 0; j < FBNIC_RPC_RSS_TBL_DW_PER_ENTRY; j++) + *(data++) = rd32(fbd, FBNIC_RPC_RSS_TBL(i, j)); + } + + *data_p = data; +} + +void fbnic_csr_get_regs(struct fbnic_dev *fbd, u32 *data, u32 *regs_version) +{ + const struct fbnic_csr_bounds *bound; + u32 *start = data; + int i, j; + + *regs_version = 1u; + + /* Skip RPC_RAM section which cannot be dumped linearly */ + for (i = 0, bound = fbnic_csr_sects; + i < ARRAY_SIZE(fbnic_csr_sects) - 1; i++, ++bound) { + *(data++) = bound->start; + *(data++) = bound->end - 1; + for (j = bound->start; j < bound->end; j++) + *(data++) = rd32(fbd, j); + } + + /* Dump the RPC_RAM as special case registers */ + fbnic_csr_get_regs_rpc_ram(fbd, &data); + + WARN_ON(data - start != fbnic_csr_regs_len(fbd)); +} + +int fbnic_csr_regs_len(struct fbnic_dev *fbd) +{ + int i, len = 0; + + /* Dump includes start and end information of each section + * which results in an offset of 2 + */ + for (i = 0; i < ARRAY_SIZE(fbnic_csr_sects); i++) + len += fbnic_csr_sects[i].end - fbnic_csr_sects[i].start + 2; + + return len; +} + +/* CSR register test data + * + * The register test will be used to verify hardware is behaving as expected. + * + * The test itself will have us writing to registers that should have no + * side effects due to us resetting after the test has been completed. + * While the test is being run the interface should be offline. + */ +struct fbnic_csr_reg_test_data { + int reg; + u16 reg_offset; + u8 array_len; + u32 read; + u32 write; +}; + +#define FBNIC_QUEUE_REG_TEST(_name, _read, _write) { \ + .reg = FBNIC_QUEUE(0) + FBNIC_QUEUE_##_name, \ + .reg_offset = FBNIC_QUEUE_STRIDE, \ + .array_len = 64, \ + .read = _read, \ + .write = _write \ +} + +static const struct fbnic_csr_reg_test_data pattern_test[] = { + FBNIC_QUEUE_REG_TEST(TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET, + FBNIC_QUEUE_TWQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(TWQ0_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(TWQ0_SIZE, FBNIC_QUEUE_TWQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ0_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ0_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET, + FBNIC_QUEUE_TWQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(TWQ1_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_SIZE, FBNIC_QUEUE_TWQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TWQ1_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET, + FBNIC_QUEUE_TCQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(TCQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_SIZE, FBNIC_QUEUE_TCQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(TCQ_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET, + FBNIC_QUEUE_RCQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(RCQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_SIZE, FBNIC_QUEUE_RCQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(RCQ_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET, + FBNIC_QUEUE_BDQ_CTL_RESET), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_SIZE, FBNIC_QUEUE_BDQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_HPQ_BAH, ~0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_PTRS, 0, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_SIZE, FBNIC_QUEUE_BDQ_SIZE_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_BAL, FBNIC_QUEUE_BAL_MASK, ~0), + FBNIC_QUEUE_REG_TEST(BDQ_PPQ_BAH, ~0, ~0), +}; + +static enum fbnic_reg_self_test_codes +fbnic_csr_reg_pattern_test(struct fbnic_dev *fbd, int index, + const struct fbnic_csr_reg_test_data *test_data) +{ + static const u32 pattern[] = { ~0, 0x5A5A5A5A, 0xA5A5A5A5, 0}; + enum fbnic_reg_self_test_codes reg; + int i; + + reg = test_data->reg + test_data->reg_offset * index; + for (i = 0; i < ARRAY_SIZE(pattern); i++) { + u32 val = pattern[i] & test_data->write; + u32 result; + + wr32(fbd, reg, val); + result = rd32(fbd, reg); + val &= test_data->read; + + if (result == val) + continue; + + dev_err(fbd->dev, + "%s: reg 0x%06X failed, expected 0x%08X received 0x%08X\n", + __func__, reg, val, result); + + /* Note that FBNIC_INTR_STATUS(0) could be tested and fail + * and the result would not be reported since the register + * offset is 0. However as that register isn't included in + * the register test that isn't an issue. + */ + return reg; + } + + return FBNIC_REG_TEST_SUCCESS; +} + +/** + * fbnic_csr_regs_test() - Verify behavior of NIC registers + * @fbd: device to test + * + * This function is meant to test the bit values of various registers in + * the NIC device. Specifically this test will verify which bits are + * writable and which ones are not. It will write varying patterns of bits + * to the registers testing for sticky bits, or bits that are writable but + * should not be. + * + * Return: FBNIC_REG_TEST_SUCCESS on success, register number on failure + **/ +enum fbnic_reg_self_test_codes fbnic_csr_regs_test(struct fbnic_dev *fbd) +{ + const struct fbnic_csr_reg_test_data *test_data; + + for (test_data = pattern_test; + test_data < pattern_test + ARRAY_SIZE(pattern_test); test_data++) { + u32 i; + + for (i = 0; i < test_data->array_len; i++) { + enum fbnic_reg_self_test_codes reg = + fbnic_csr_reg_pattern_test(fbd, i, test_data); + + if (reg) + return reg; + } + } + + return FBNIC_REG_TEST_SUCCESS; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_csr.h b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h new file mode 100644 index 000000000..64b958df7 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_csr.h @@ -0,0 +1,1256 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_CSR_H_ +#define _FBNIC_CSR_H_ + +#include <linux/bitops.h> + +struct fbnic_dev; + +#define CSR_BIT(nr) (1u << (nr)) +#define CSR_GENMASK(h, l) GENMASK(h, l) + +#define DESC_BIT(nr) BIT_ULL(nr) +#define DESC_GENMASK(h, l) GENMASK_ULL(h, l) + +#define FW_VER_CODE(_major, _minor, _patch, _build) ( \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_MAJOR, _major) | \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_MINOR, _minor) | \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_PATCH, _patch) | \ + FIELD_PREP(FBNIC_FW_CAP_RESP_VERSION_BUILD, _build)) + +/* Defines the minimum firmware version required by the driver */ +#define MIN_FW_VER_CODE FW_VER_CODE(0, 10, 6, 0) + +/* Defines the minimum firmware version required for firmware logs */ +#define MIN_FW_VER_CODE_LOG FW_VER_CODE(0, 12, 9, 0) + +/* Driver can request that firmware sends all cached logs in bulk. This + * feature was enabled on older firmware however firmware has a bug + * which attempted to send 30 messages per mbx message which caused an + * overflow flooding the mailbox. This results in a kernel warning + * related to corrupt mailbox messages. + * + * If firmware is new enough only request sending historical logs when + * the log buffer is empty to prevent duplicate logs. + */ +#define MIN_FW_VER_CODE_HIST FW_VER_CODE(25, 5, 7, 0) + +#define PCI_DEVICE_ID_META_FBNIC_ASIC 0x0013 + +#define FBNIC_CLOCK_FREQ (600 * (1000 * 1000)) + +/* Transmit Work Descriptor Format */ +/* Length, Type, Offset Masks and Shifts */ +#define FBNIC_TWD_L2_HLEN_MASK DESC_GENMASK(5, 0) + +#define FBNIC_TWD_L3_TYPE_MASK DESC_GENMASK(7, 6) +enum { + FBNIC_TWD_L3_TYPE_OTHER = 0, + FBNIC_TWD_L3_TYPE_IPV4 = 1, + FBNIC_TWD_L3_TYPE_IPV6 = 2, + FBNIC_TWD_L3_TYPE_V6V6 = 3, +}; + +#define FBNIC_TWD_L3_OHLEN_MASK DESC_GENMASK(15, 8) +#define FBNIC_TWD_L3_IHLEN_MASK DESC_GENMASK(23, 16) + +enum { + FBNIC_TWD_L4_TYPE_OTHER = 0, + FBNIC_TWD_L4_TYPE_TCP = 1, + FBNIC_TWD_L4_TYPE_UDP = 2, +}; + +#define FBNIC_TWD_CSUM_OFFSET_MASK DESC_GENMASK(27, 24) +#define FBNIC_TWD_L4_HLEN_MASK DESC_GENMASK(31, 28) + +/* Flags and Type */ +#define FBNIC_TWD_L4_TYPE_MASK DESC_GENMASK(33, 32) +#define FBNIC_TWD_FLAG_REQ_TS DESC_BIT(34) +#define FBNIC_TWD_FLAG_REQ_LSO DESC_BIT(35) +#define FBNIC_TWD_FLAG_REQ_CSO DESC_BIT(36) +#define FBNIC_TWD_FLAG_REQ_COMPLETION DESC_BIT(37) +#define FBNIC_TWD_FLAG_DEST_MAC DESC_BIT(43) +#define FBNIC_TWD_FLAG_DEST_BMC DESC_BIT(44) +#define FBNIC_TWD_FLAG_DEST_FW DESC_BIT(45) +#define FBNIC_TWD_TYPE_MASK DESC_GENMASK(47, 46) +enum { + FBNIC_TWD_TYPE_META = 0, + FBNIC_TWD_TYPE_OPT_META = 1, + FBNIC_TWD_TYPE_AL = 2, + FBNIC_TWD_TYPE_LAST_AL = 3, +}; + +/* MSS and Completion Req */ +#define FBNIC_TWD_MSS_MASK DESC_GENMASK(61, 48) + +#define FBNIC_TWD_TS_MASK DESC_GENMASK(39, 0) +#define FBNIC_TWD_ADDR_MASK DESC_GENMASK(45, 0) +#define FBNIC_TWD_LEN_MASK DESC_GENMASK(63, 48) + +/* Tx Completion Descriptor Format */ +#define FBNIC_TCD_TYPE0_HEAD0_MASK DESC_GENMASK(15, 0) +#define FBNIC_TCD_TYPE0_HEAD1_MASK DESC_GENMASK(31, 16) + +#define FBNIC_TCD_TYPE1_TS_MASK DESC_GENMASK(39, 0) + +#define FBNIC_TCD_STATUS_MASK DESC_GENMASK(59, 48) +#define FBNIC_TCD_STATUS_TS_INVALID DESC_BIT(48) +#define FBNIC_TCD_STATUS_ILLEGAL_TS_REQ DESC_BIT(49) +#define FBNIC_TCD_TWQ1 DESC_BIT(60) +#define FBNIC_TCD_TYPE_MASK DESC_GENMASK(62, 61) +enum { + FBNIC_TCD_TYPE_0 = 0, + FBNIC_TCD_TYPE_1 = 1, +}; + +#define FBNIC_TCD_DONE DESC_BIT(63) + +/* Rx Buffer Descriptor Format + * + * The layout of this can vary depending on the page size of the system. + * + * If the page size is 4K then the layout will simply consist of ID for + * the 16 most significant bits, and the lower 46 are essentially the page + * address with the lowest 12 bits being reserved 0 due to the fact that + * a page will be aligned. + * + * If the page size is larger than 4K then the lower n bits of the ID and + * page address will be reserved for the fragment ID. This fragment will + * be 4K in size and will be used to index both the DMA address and the ID + * by the same amount. + */ +#define FBNIC_BD_DESC_ADDR_MASK DESC_GENMASK(45, 12) +#define FBNIC_BD_DESC_ID_MASK DESC_GENMASK(63, 48) +#define FBNIC_BD_FRAG_SIZE \ + (FBNIC_BD_DESC_ADDR_MASK & ~(FBNIC_BD_DESC_ADDR_MASK - 1)) +#define FBNIC_BD_FRAG_COUNT \ + (PAGE_SIZE / FBNIC_BD_FRAG_SIZE) +#define FBNIC_BD_FRAG_ADDR_MASK \ + (FBNIC_BD_DESC_ADDR_MASK & \ + ~(FBNIC_BD_DESC_ADDR_MASK * FBNIC_BD_FRAG_COUNT)) +#define FBNIC_BD_FRAG_ID_MASK \ + (FBNIC_BD_DESC_ID_MASK & \ + ~(FBNIC_BD_DESC_ID_MASK * FBNIC_BD_FRAG_COUNT)) +#define FBNIC_BD_PAGE_ADDR_MASK \ + (FBNIC_BD_DESC_ADDR_MASK & ~FBNIC_BD_FRAG_ADDR_MASK) +#define FBNIC_BD_PAGE_ID_MASK \ + (FBNIC_BD_DESC_ID_MASK & ~FBNIC_BD_FRAG_ID_MASK) + +/* Rx Completion Queue Descriptors */ +#define FBNIC_RCD_TYPE_MASK DESC_GENMASK(62, 61) +enum { + FBNIC_RCD_TYPE_HDR_AL = 0, + FBNIC_RCD_TYPE_PAY_AL = 1, + FBNIC_RCD_TYPE_OPT_META = 2, + FBNIC_RCD_TYPE_META = 3, +}; + +#define FBNIC_RCD_DONE DESC_BIT(63) + +/* Address/Length Completion Descriptors */ +#define FBNIC_RCD_AL_BUFF_ID_MASK DESC_GENMASK(15, 0) +#define FBNIC_RCD_AL_BUFF_FRAG_MASK (FBNIC_BD_FRAG_COUNT - 1) +#define FBNIC_RCD_AL_BUFF_PAGE_MASK \ + (FBNIC_RCD_AL_BUFF_ID_MASK & ~FBNIC_RCD_AL_BUFF_FRAG_MASK) +#define FBNIC_RCD_AL_BUFF_LEN_MASK DESC_GENMASK(28, 16) +#define FBNIC_RCD_AL_BUFF_OFF_MASK DESC_GENMASK(43, 32) +#define FBNIC_RCD_AL_PAGE_FIN DESC_BIT(60) + +/* Header AL specific values */ +#define FBNIC_RCD_HDR_AL_OVERFLOW DESC_BIT(53) +#define FBNIC_RCD_HDR_AL_DMA_HINT_MASK DESC_GENMASK(59, 54) +enum { + FBNIC_RCD_HDR_AL_DMA_HINT_NONE = 0, + FBNIC_RCD_HDR_AL_DMA_HINT_L2 = 1, + FBNIC_RCD_HDR_AL_DMA_HINT_L3 = 2, + FBNIC_RCD_HDR_AL_DMA_HINT_L4 = 4, +}; + +/* Optional Metadata Completion Descriptors */ +#define FBNIC_RCD_OPT_META_TS_MASK DESC_GENMASK(39, 0) +#define FBNIC_RCD_OPT_META_ACTION_MASK DESC_GENMASK(45, 40) +#define FBNIC_RCD_OPT_META_ACTION DESC_BIT(57) +#define FBNIC_RCD_OPT_META_TS DESC_BIT(58) +#define FBNIC_RCD_OPT_META_TYPE_MASK DESC_GENMASK(60, 59) + +/* Metadata Completion Descriptors */ +#define FBNIC_RCD_META_RSS_HASH_MASK DESC_GENMASK(31, 0) +#define FBNIC_RCD_META_L2_CSUM_MASK DESC_GENMASK(47, 32) +#define FBNIC_RCD_META_L3_TYPE_MASK DESC_GENMASK(49, 48) +enum { + FBNIC_RCD_META_L3_TYPE_OTHER = 0, + FBNIC_RCD_META_L3_TYPE_IPV4 = 1, + FBNIC_RCD_META_L3_TYPE_IPV6 = 2, + FBNIC_RCD_META_L3_TYPE_V6V6 = 3, +}; + +#define FBNIC_RCD_META_L4_TYPE_MASK DESC_GENMASK(51, 50) +enum { + FBNIC_RCD_META_L4_TYPE_OTHER = 0, + FBNIC_RCD_META_L4_TYPE_TCP = 1, + FBNIC_RCD_META_L4_TYPE_UDP = 2, +}; + +#define FBNIC_RCD_META_L4_CSUM_UNNECESSARY DESC_BIT(52) +#define FBNIC_RCD_META_ERR_MAC_EOP DESC_BIT(53) +#define FBNIC_RCD_META_ERR_TRUNCATED_FRAME DESC_BIT(54) +#define FBNIC_RCD_META_ERR_PARSER DESC_BIT(55) +#define FBNIC_RCD_META_UNCORRECTABLE_ERR_MASK \ + (FBNIC_RCD_META_ERR_MAC_EOP | FBNIC_RCD_META_ERR_TRUNCATED_FRAME) +#define FBNIC_RCD_META_ECN DESC_BIT(60) + +/* Register Definitions + * + * The registers are laid as indexes into an le32 array. As such the actual + * address is 4 times the index value. Below each register is defined as 3 + * fields, name, index, and Address. + * + * Name Index Address + *************************************************************************/ +/* Interrupt Registers */ +#define FBNIC_CSR_START_INTR 0x00000 /* CSR section delimiter */ +#define FBNIC_INTR_STATUS(n) (0x00000 + (n)) /* 0x00000 + 4*n */ +#define FBNIC_INTR_STATUS_CNT 8 +#define FBNIC_INTR_MASK(n) (0x00008 + (n)) /* 0x00020 + 4*n */ +#define FBNIC_INTR_MASK_CNT 8 +#define FBNIC_INTR_SET(n) (0x00010 + (n)) /* 0x00040 + 4*n */ +#define FBNIC_INTR_SET_CNT 8 +#define FBNIC_INTR_CLEAR(n) (0x00018 + (n)) /* 0x00060 + 4*n */ +#define FBNIC_INTR_CLEAR_CNT 8 +#define FBNIC_INTR_SW_STATUS(n) (0x00020 + (n)) /* 0x00080 + 4*n */ +#define FBNIC_INTR_SW_STATUS_CNT 8 +#define FBNIC_INTR_SW_AC_MODE(n) (0x00028 + (n)) /* 0x000a0 + 4*n */ +#define FBNIC_INTR_SW_AC_MODE_CNT 8 +#define FBNIC_INTR_MASK_SET(n) (0x00030 + (n)) /* 0x000c0 + 4*n */ +#define FBNIC_INTR_MASK_SET_CNT 8 +#define FBNIC_INTR_MASK_CLEAR(n) (0x00038 + (n)) /* 0x000e0 + 4*n */ +#define FBNIC_INTR_MASK_CLEAR_CNT 8 +#define FBNIC_MAX_MSIX_VECS 256U +#define FBNIC_INTR_MSIX_CTRL(n) (0x00040 + (n)) /* 0x00100 + 4*n */ +#define FBNIC_INTR_MSIX_CTRL_VECTOR_MASK CSR_GENMASK(7, 0) +#define FBNIC_INTR_MSIX_CTRL_ENABLE CSR_BIT(31) +enum { + FBNIC_INTR_MSIX_CTRL_RXB_IDX = 7, + FBNIC_INTR_MSIX_CTRL_PCS_IDX = 34, +}; + +#define FBNIC_CSR_END_INTR 0x0005f /* CSR section delimiter */ + +/* Interrupt MSIX Registers */ +#define FBNIC_CSR_START_INTR_CQ 0x00400 /* CSR section delimiter */ +#define FBNIC_INTR_CQ_REARM(n) \ + (0x00400 + 4 * (n)) /* 0x01000 + 16*n */ +#define FBNIC_INTR_CQ_REARM_CNT 256 +#define FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT CSR_GENMASK(13, 0) +#define FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT_UPD_EN CSR_BIT(14) +#define FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT CSR_GENMASK(28, 15) +#define FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT_UPD_EN CSR_BIT(29) +#define FBNIC_INTR_CQ_REARM_INTR_RELOAD CSR_BIT(30) +#define FBNIC_INTR_CQ_REARM_INTR_UNMASK CSR_BIT(31) + +#define FBNIC_INTR_RCQ_TIMEOUT(n) \ + (0x00401 + 4 * (n)) /* 0x01004 + 16*n */ +#define FBNIC_INTR_RCQ_TIMEOUT_CNT 256 +#define FBNIC_INTR_TCQ_TIMEOUT(n) \ + (0x00402 + 4 * (n)) /* 0x01008 + 16*n */ +#define FBNIC_INTR_TCQ_TIMEOUT_CNT 256 +#define FBNIC_CSR_END_INTR_CQ 0x007fe /* CSR section delimiter */ + +/* Global QM Tx registers */ +#define FBNIC_CSR_START_QM_TX 0x00800 /* CSR section delimiter */ +#define FBNIC_QM_TWQ_IDLE(n) (0x00800 + (n)) /* 0x02000 + 4*n */ +#define FBNIC_QM_TWQ_IDLE_CNT 8 +#define FBNIC_QM_TWQ_DEFAULT_META_L 0x00818 /* 0x02060 */ +#define FBNIC_QM_TWQ_DEFAULT_META_H 0x00819 /* 0x02064 */ + +#define FBNIC_QM_TQS_CTL0 0x0081b /* 0x0206c */ +#define FBNIC_QM_TQS_CTL0_LSO_TS_MASK CSR_BIT(0) +enum { + FBNIC_QM_TQS_CTL0_LSO_TS_FIRST = 0, + FBNIC_QM_TQS_CTL0_LSO_TS_LAST = 1, +}; + +#define FBNIC_QM_TQS_CTL0_PREFETCH_THRESH CSR_GENMASK(7, 1) +enum { + FBNIC_QM_TQS_CTL0_PREFETCH_THRESH_MIN = 16, +}; + +#define FBNIC_QM_TQS_CTL1 0x0081c /* 0x02070 */ +#define FBNIC_QM_TQS_CTL1_MC_MAX_CREDITS CSR_GENMASK(7, 0) +#define FBNIC_QM_TQS_CTL1_BULK_MAX_CREDITS CSR_GENMASK(15, 8) +#define FBNIC_QM_TQS_MTU_CTL0 0x0081d /* 0x02074 */ +#define FBNIC_QM_TQS_MTU_CTL1 0x0081e /* 0x02078 */ +#define FBNIC_QM_TQS_MTU_CTL1_BULK CSR_GENMASK(13, 0) +#define FBNIC_QM_TCQ_IDLE(n) (0x00821 + (n)) /* 0x02084 + 4*n */ +#define FBNIC_QM_TCQ_IDLE_CNT 4 +#define FBNIC_QM_TCQ_CTL0 0x0082d /* 0x020b4 */ +#define FBNIC_QM_TCQ_CTL0_COAL_WAIT CSR_GENMASK(15, 0) +#define FBNIC_QM_TCQ_CTL0_TICK_CYCLES CSR_GENMASK(26, 16) +#define FBNIC_QM_TQS_IDLE(n) (0x00830 + (n)) /* 0x020c0 + 4*n */ +#define FBNIC_QM_TQS_IDLE_CNT 8 +#define FBNIC_QM_TQS_EDT_TS_RANGE 0x00849 /* 0x2124 */ +#define FBNIC_QM_TDE_IDLE(n) (0x00853 + (n)) /* 0x0214c + 4*n */ +#define FBNIC_QM_TDE_IDLE_CNT 8 +#define FBNIC_QM_TNI_TDF_CTL 0x0086c /* 0x021b0 */ +#define FBNIC_QM_TNI_TDF_CTL_MRRS CSR_GENMASK(1, 0) +#define FBNIC_QM_TNI_TDF_CTL_CLS CSR_GENMASK(3, 2) +#define FBNIC_QM_TNI_TDF_CTL_MAX_OT CSR_GENMASK(11, 4) +#define FBNIC_QM_TNI_TDF_CTL_MAX_OB CSR_GENMASK(23, 12) +#define FBNIC_QM_TNI_TDE_CTL 0x0086d /* 0x021b4 */ +#define FBNIC_QM_TNI_TDE_CTL_MRRS CSR_GENMASK(1, 0) +#define FBNIC_QM_TNI_TDE_CTL_CLS CSR_GENMASK(3, 2) +#define FBNIC_QM_TNI_TDE_CTL_MAX_OT CSR_GENMASK(11, 4) +#define FBNIC_QM_TNI_TDE_CTL_MAX_OB CSR_GENMASK(24, 12) +#define FBNIC_QM_TNI_TDE_CTL_MRRS_1K CSR_BIT(25) +#define FBNIC_QM_TNI_TCM_CTL 0x0086e /* 0x021b8 */ +#define FBNIC_QM_TNI_TCM_CTL_MPS CSR_GENMASK(1, 0) +#define FBNIC_QM_TNI_TCM_CTL_CLS CSR_GENMASK(3, 2) +#define FBNIC_QM_TNI_TCM_CTL_MAX_OT CSR_GENMASK(11, 4) +#define FBNIC_QM_TNI_TCM_CTL_MAX_OB CSR_GENMASK(23, 12) +#define FBNIC_CSR_END_QM_TX 0x00873 /* CSR section delimiter */ + +/* Global QM Rx registers */ +#define FBNIC_CSR_START_QM_RX 0x00c00 /* CSR section delimiter */ +#define FBNIC_QM_RCQ_IDLE(n) (0x00c00 + (n)) /* 0x03000 + 4*n */ +#define FBNIC_QM_RCQ_IDLE_CNT 4 +#define FBNIC_QM_RCQ_CTL0 0x00c0c /* 0x03030 */ +#define FBNIC_QM_RCQ_CTL0_COAL_WAIT CSR_GENMASK(15, 0) +#define FBNIC_QM_RCQ_CTL0_TICK_CYCLES CSR_GENMASK(26, 16) +#define FBNIC_QM_HPQ_IDLE(n) (0x00c0f + (n)) /* 0x0303c + 4*n */ +#define FBNIC_QM_HPQ_IDLE_CNT 4 +#define FBNIC_QM_PPQ_IDLE(n) (0x00c13 + (n)) /* 0x0304c + 4*n */ +#define FBNIC_QM_PPQ_IDLE_CNT 4 +#define FBNIC_QM_RNI_RBP_CTL 0x00c2d /* 0x030b4 */ +#define FBNIC_QM_RNI_RBP_CTL_MRRS CSR_GENMASK(1, 0) +#define FBNIC_QM_RNI_RBP_CTL_CLS CSR_GENMASK(3, 2) +#define FBNIC_QM_RNI_RBP_CTL_MAX_OT CSR_GENMASK(11, 4) +#define FBNIC_QM_RNI_RBP_CTL_MAX_OB CSR_GENMASK(23, 12) +#define FBNIC_QM_RNI_RDE_CTL 0x00c2e /* 0x030b8 */ +#define FBNIC_QM_RNI_RDE_CTL_MPS CSR_GENMASK(1, 0) +#define FBNIC_QM_RNI_RDE_CTL_CLS CSR_GENMASK(3, 2) +#define FBNIC_QM_RNI_RDE_CTL_MAX_OT CSR_GENMASK(11, 4) +#define FBNIC_QM_RNI_RDE_CTL_MAX_OB CSR_GENMASK(23, 12) +#define FBNIC_QM_RNI_RCM_CTL 0x00c2f /* 0x030bc */ +#define FBNIC_QM_RNI_RCM_CTL_MPS CSR_GENMASK(1, 0) +#define FBNIC_QM_RNI_RCM_CTL_CLS CSR_GENMASK(3, 2) +#define FBNIC_QM_RNI_RCM_CTL_MAX_OT CSR_GENMASK(11, 4) +#define FBNIC_QM_RNI_RCM_CTL_MAX_OB CSR_GENMASK(23, 12) +#define FBNIC_CSR_END_QM_RX 0x00c34 /* CSR section delimiter */ + +/* TCE registers */ +#define FBNIC_CSR_START_TCE 0x04000 /* CSR section delimiter */ +#define FBNIC_TCE_REG_BASE 0x04000 /* 0x10000 */ + +#define FBNIC_TCE_LSO_CTRL 0x04000 /* 0x10000 */ +#define FBNIC_TCE_LSO_CTRL_TCPF_CLR_1ST CSR_GENMASK(8, 0) +#define FBNIC_TCE_LSO_CTRL_TCPF_CLR_MID CSR_GENMASK(17, 9) +#define FBNIC_TCE_LSO_CTRL_TCPF_CLR_END CSR_GENMASK(26, 18) +#define FBNIC_TCE_LSO_CTRL_IPID_MODE_INC CSR_BIT(27) + +#define FBNIC_TCE_CSO_CTRL 0x04001 /* 0x10004 */ +#define FBNIC_TCE_CSO_CTRL_TCP_ZERO_CSUM CSR_BIT(0) + +#define FBNIC_TCE_TXB_CTRL 0x04002 /* 0x10008 */ +#define FBNIC_TCE_TXB_CTRL_LOAD CSR_BIT(0) +#define FBNIC_TCE_TXB_CTRL_TCAM_ENABLE CSR_BIT(1) +#define FBNIC_TCE_TXB_CTRL_DISABLE CSR_BIT(2) + +#define FBNIC_TCE_TXB_ENQ_WRR_CTRL 0x04003 /* 0x1000c */ +#define FBNIC_TCE_TXB_ENQ_WRR_CTRL_WEIGHT0 CSR_GENMASK(7, 0) +#define FBNIC_TCE_TXB_ENQ_WRR_CTRL_WEIGHT1 CSR_GENMASK(15, 8) +#define FBNIC_TCE_TXB_ENQ_WRR_CTRL_WEIGHT2 CSR_GENMASK(23, 16) + +#define FBNIC_TCE_TXB_TEI_Q0_CTRL 0x04004 /* 0x10010 */ +#define FBNIC_TCE_TXB_TEI_Q1_CTRL 0x04005 /* 0x10014 */ +#define FBNIC_TCE_TXB_MC_Q_CTRL 0x04006 /* 0x10018 */ +#define FBNIC_TCE_TXB_RX_TEI_Q_CTRL 0x04007 /* 0x1001c */ +#define FBNIC_TCE_TXB_RX_BMC_Q_CTRL 0x04008 /* 0x10020 */ +#define FBNIC_TCE_TXB_Q_CTRL_START CSR_GENMASK(10, 0) +#define FBNIC_TCE_TXB_Q_CTRL_SIZE CSR_GENMASK(22, 11) + +#define FBNIC_TCE_TXB_TEI_DWRR_CTRL 0x04009 /* 0x10024 */ +#define FBNIC_TCE_TXB_TEI_DWRR_CTRL_QUANTUM0 CSR_GENMASK(7, 0) +#define FBNIC_TCE_TXB_TEI_DWRR_CTRL_QUANTUM1 CSR_GENMASK(15, 8) +#define FBNIC_TCE_TXB_NTWRK_DWRR_CTRL 0x0400a /* 0x10028 */ +#define FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_QUANTUM0 CSR_GENMASK(7, 0) +#define FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_QUANTUM1 CSR_GENMASK(15, 8) +#define FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_QUANTUM2 CSR_GENMASK(23, 16) + +#define FBNIC_TCE_TXB_CLDR_CFG 0x0400b /* 0x1002c */ +#define FBNIC_TCE_TXB_CLDR_CFG_NUM_SLOT CSR_GENMASK(5, 0) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG(n) (0x0400c + (n)) /* 0x10030 + 4*n */ +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_CNT 16 +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_0_0 CSR_GENMASK(1, 0) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_0_1 CSR_GENMASK(3, 2) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_0_2 CSR_GENMASK(5, 4) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_0_3 CSR_GENMASK(7, 6) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_1_0 CSR_GENMASK(9, 8) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_1_1 CSR_GENMASK(11, 10) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_1_2 CSR_GENMASK(13, 12) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_1_3 CSR_GENMASK(15, 14) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_2_0 CSR_GENMASK(17, 16) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_2_1 CSR_GENMASK(19, 18) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_2_2 CSR_GENMASK(21, 20) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_2_3 CSR_GENMASK(23, 22) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_3_0 CSR_GENMASK(25, 24) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_3_1 CSR_GENMASK(27, 26) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_3_2 CSR_GENMASK(29, 28) +#define FBNIC_TCE_TXB_CLDR_SLOT_CFG_DEST_ID_3_3 CSR_GENMASK(31, 30) + +#define FBNIC_TCE_BMC_MAX_PKTSZ 0x0403a /* 0x100e8 */ +#define FBNIC_TCE_BMC_MAX_PKTSZ_TX CSR_GENMASK(13, 0) +#define FBNIC_TCE_BMC_MAX_PKTSZ_RX CSR_GENMASK(27, 14) +#define FBNIC_TCE_MC_MAX_PKTSZ 0x0403b /* 0x100ec */ +#define FBNIC_TCE_MC_MAX_PKTSZ_TMI CSR_GENMASK(13, 0) + +#define FBNIC_TCE_SOP_PROT_CTRL 0x0403c /* 0x100f0 */ +#define FBNIC_TCE_SOP_PROT_CTRL_TBI CSR_GENMASK(7, 0) +#define FBNIC_TCE_SOP_PROT_CTRL_TTI_FRM CSR_GENMASK(14, 8) +#define FBNIC_TCE_SOP_PROT_CTRL_TTI_CM CSR_GENMASK(18, 15) + +#define FBNIC_TCE_DROP_CTRL 0x0403d /* 0x100f4 */ +#define FBNIC_TCE_DROP_CTRL_TTI_CM_DROP_EN CSR_BIT(0) +#define FBNIC_TCE_DROP_CTRL_TTI_FRM_DROP_EN CSR_BIT(1) +#define FBNIC_TCE_DROP_CTRL_TTI_TBI_DROP_EN CSR_BIT(2) + +#define FBNIC_TCE_TTI_CM_DROP_PKTS 0x0403e /* 0x100f8 */ +#define FBNIC_TCE_TTI_CM_DROP_BYTE_L 0x0403f /* 0x100fc */ +#define FBNIC_TCE_TTI_CM_DROP_BYTE_H 0x04040 /* 0x10100 */ +#define FBNIC_TCE_TTI_FRAME_DROP_PKTS 0x04041 /* 0x10104 */ +#define FBNIC_TCE_TTI_FRAME_DROP_BYTE_L 0x04042 /* 0x10108 */ +#define FBNIC_TCE_TTI_FRAME_DROP_BYTE_H 0x04043 /* 0x1010c */ +#define FBNIC_TCE_TBI_DROP_PKTS 0x04044 /* 0x10110 */ +#define FBNIC_TCE_TBI_DROP_BYTE_L 0x04045 /* 0x10114 */ + +#define FBNIC_TCE_TCAM_IDX2DEST_MAP 0x0404A /* 0x10128 */ +#define FBNIC_TCE_TCAM_IDX2DEST_MAP_DEST_ID_0 CSR_GENMASK(3, 0) +enum { + FBNIC_TCE_TCAM_DEST_MAC = 1, + FBNIC_TCE_TCAM_DEST_BMC = 2, + FBNIC_TCE_TCAM_DEST_FW = 4, +}; + +#define FBNIC_TCE_TXB_TX_BMC_Q_CTRL 0x0404B /* 0x1012c */ +#define FBNIC_TCE_TXB_BMC_DWRR_CTRL 0x0404C /* 0x10130 */ +#define FBNIC_TCE_TXB_BMC_DWRR_CTRL_QUANTUM0 CSR_GENMASK(7, 0) +#define FBNIC_TCE_TXB_BMC_DWRR_CTRL_QUANTUM1 CSR_GENMASK(15, 8) +#define FBNIC_TCE_TXB_TEI_DWRR_CTRL_EXT 0x0404D /* 0x10134 */ +#define FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_EXT \ + 0x0404E /* 0x10138 */ +#define FBNIC_TCE_TXB_BMC_DWRR_CTRL_EXT 0x0404F /* 0x1013c */ +#define FBNIC_CSR_END_TCE 0x04050 /* CSR section delimiter */ + +/* TCE RAM registers */ +#define FBNIC_CSR_START_TCE_RAM 0x04200 /* CSR section delimiter */ +#define FBNIC_TCE_RAM_TCAM(m, n) \ + (0x04200 + 0x8 * (n) + (m)) /* 0x10800 + 32*n + 4*m */ +#define FBNIC_TCE_RAM_TCAM_MASK CSR_GENMASK(15, 0) +#define FBNIC_TCE_RAM_TCAM_VALUE CSR_GENMASK(31, 16) +#define FBNIC_TCE_RAM_TCAM3(n) (0x04218 + (n)) /* 0x010860 + 4*n */ +#define FBNIC_TCE_RAM_TCAM3_DEST_MASK CSR_GENMASK(5, 3) +#define FBNIC_TCE_RAM_TCAM3_MCQ_MASK CSR_BIT(7) +#define FBNIC_TCE_RAM_TCAM3_VALIDATE CSR_BIT(31) +#define FBNIC_CSR_END_TCE_RAM 0x0421F /* CSR section delimiter */ + +/* TMI registers */ +#define FBNIC_CSR_START_TMI 0x04400 /* CSR section delimiter */ +#define FBNIC_TMI_SOP_PROT_CTRL 0x04400 /* 0x11000 */ +#define FBNIC_TMI_DROP_CTRL 0x04401 /* 0x11004 */ +#define FBNIC_TMI_DROP_CTRL_EN CSR_BIT(0) +#define FBNIC_TMI_DROP_PKTS 0x04402 /* 0x11008 */ +#define FBNIC_TMI_DROP_BYTE_L 0x04403 /* 0x1100c */ +#define FBNIC_TMI_ILLEGAL_PTP_REQS 0x04409 /* 0x11024 */ +#define FBNIC_TMI_GOOD_PTP_TS 0x0440a /* 0x11028 */ +#define FBNIC_TMI_BAD_PTP_TS 0x0440b /* 0x1102c */ +#define FBNIC_TMI_STAT_TX_PACKET_1519_2047_BYTES_L \ + 0x04433 /* 0x110cc */ +#define FBNIC_TMI_STAT_TX_PACKET_1519_2047_BYTES_H \ + 0x04434 /* 0x110d0 */ +#define FBNIC_TMI_STAT_TX_PACKET_2048_4095_BYTES_L \ + 0x04435 /* 0x110d4 */ +#define FBNIC_TMI_STAT_TX_PACKET_2048_4095_BYTES_H \ + 0x04436 /* 0x110d8 */ +#define FBNIC_TMI_STAT_TX_PACKET_4096_8191_BYTES_L \ + 0x04437 /* 0x110dc */ +#define FBNIC_TMI_STAT_TX_PACKET_4096_8191_BYTES_H \ + 0x04438 /* 0x110e0 */ +#define FBNIC_TMI_STAT_TX_PACKET_8192_9216_BYTES_L \ + 0x04439 /* 0x110e4 */ +#define FBNIC_TMI_STAT_TX_PACKET_8192_9216_BYTES_H \ + 0x0443a /* 0x110e8 */ +#define FBNIC_TMI_STAT_TX_PACKET_9217_MAX_BYTES_L \ + 0x0443b /* 0x110ec */ +#define FBNIC_TMI_STAT_TX_PACKET_9217_MAX_BYTES_H \ + 0x0443c /* 0x110f0 */ +#define FBNIC_CSR_END_TMI 0x0443f /* CSR section delimiter */ + +/* Precision Time Protocol Registers */ +#define FBNIC_CSR_START_PTP 0x04800 /* CSR section delimiter */ +#define FBNIC_PTP_REG_BASE 0x04800 /* 0x12000 */ + +#define FBNIC_PTP_CTRL 0x04800 /* 0x12000 */ +#define FBNIC_PTP_CTRL_EN CSR_BIT(0) +#define FBNIC_PTP_CTRL_MONO_EN CSR_BIT(4) +#define FBNIC_PTP_CTRL_TQS_OUT_EN CSR_BIT(8) +#define FBNIC_PTP_CTRL_MAC_OUT_IVAL CSR_GENMASK(16, 12) +#define FBNIC_PTP_CTRL_TICK_IVAL CSR_GENMASK(23, 20) + +#define FBNIC_PTP_ADJUST 0x04801 /* 0x12004 */ +#define FBNIC_PTP_ADJUST_INIT CSR_BIT(0) +#define FBNIC_PTP_ADJUST_SUB_NUDGE CSR_BIT(8) +#define FBNIC_PTP_ADJUST_ADD_NUDGE CSR_BIT(16) +#define FBNIC_PTP_ADJUST_ADDEND_SET CSR_BIT(24) + +#define FBNIC_PTP_INIT_HI 0x04802 /* 0x12008 */ +#define FBNIC_PTP_INIT_LO 0x04803 /* 0x1200c */ + +#define FBNIC_PTP_NUDGE_NS 0x04804 /* 0x12010 */ +#define FBNIC_PTP_NUDGE_SUBNS 0x04805 /* 0x12014 */ + +#define FBNIC_PTP_ADD_VAL_NS 0x04806 /* 0x12018 */ +#define FBNIC_PTP_ADD_VAL_NS_MASK CSR_GENMASK(15, 0) +#define FBNIC_PTP_ADD_VAL_SUBNS 0x04807 /* 0x1201c */ + +#define FBNIC_PTP_CTR_VAL_HI 0x04808 /* 0x12020 */ +#define FBNIC_PTP_CTR_VAL_LO 0x04809 /* 0x12024 */ + +#define FBNIC_PTP_MONO_PTP_CTR_HI 0x0480a /* 0x12028 */ +#define FBNIC_PTP_MONO_PTP_CTR_LO 0x0480b /* 0x1202c */ + +#define FBNIC_PTP_CDC_FIFO_STATUS 0x0480c /* 0x12030 */ +#define FBNIC_PTP_SPARE 0x0480d /* 0x12034 */ +#define FBNIC_CSR_END_PTP 0x0480d /* CSR section delimiter */ + +/* Rx Buffer Registers */ +#define FBNIC_CSR_START_RXB 0x08000 /* CSR section delimiter */ +enum { + FBNIC_RXB_FIFO_MC = 0, + /* Unused */ + /* Unused */ + FBNIC_RXB_FIFO_NET_TO_BMC = 3, + FBNIC_RXB_FIFO_HOST = 4, + /* Unused */ + FBNIC_RXB_FIFO_BMC_TO_HOST = 6, + /* Unused */ + FBNIC_RXB_FIFO_INDICES = 8 +}; + +enum { + FBNIC_RXB_INTF_NET = 0, + FBNIC_RXB_INTF_RBT = 1, + /* Unused */ + /* Unused */ + FBNIC_RXB_INTF_INDICES = 4 +}; + +#define FBNIC_RXB_CT_SIZE(n) (0x08000 + (n)) /* 0x20000 + 4*n */ +#define FBNIC_RXB_CT_SIZE_CNT 8 +#define FBNIC_RXB_CT_SIZE_HEADER CSR_GENMASK(5, 0) +#define FBNIC_RXB_CT_SIZE_PAYLOAD CSR_GENMASK(11, 6) +#define FBNIC_RXB_CT_SIZE_ENABLE CSR_BIT(12) +#define FBNIC_RXB_PAUSE_DROP_CTRL 0x08008 /* 0x20020 */ +#define FBNIC_RXB_PAUSE_DROP_CTRL_DROP_ENABLE CSR_GENMASK(7, 0) +#define FBNIC_RXB_PAUSE_DROP_CTRL_PAUSE_ENABLE CSR_GENMASK(15, 8) +#define FBNIC_RXB_PAUSE_DROP_CTRL_ECN_ENABLE CSR_GENMASK(23, 16) +#define FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE CSR_GENMASK(27, 24) +#define FBNIC_RXB_PAUSE_THLD(n) (0x08009 + (n)) /* 0x20024 + 4*n */ +#define FBNIC_RXB_PAUSE_THLD_CNT 8 +#define FBNIC_RXB_PAUSE_THLD_ON CSR_GENMASK(12, 0) +#define FBNIC_RXB_PAUSE_THLD_OFF CSR_GENMASK(25, 13) +#define FBNIC_RXB_DROP_THLD(n) (0x08011 + (n)) /* 0x20044 + 4*n */ +#define FBNIC_RXB_DROP_THLD_CNT 8 +#define FBNIC_RXB_DROP_THLD_ON CSR_GENMASK(12, 0) +#define FBNIC_RXB_DROP_THLD_OFF CSR_GENMASK(25, 13) +#define FBNIC_RXB_PAUSE_STORM(n) (0x08019 + (n)) /* 0x20064 + 4*n */ +#define FBNIC_RXB_PAUSE_STORM_CNT 4 +#define FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL CSR_BIT(20) +#define FBNIC_RXB_PAUSE_STORM_THLD_TIME CSR_GENMASK(19, 0) +#define FBNIC_RXB_PAUSE_STORM_UNIT_WR 0x0801d /* 0x20074 */ +#define FBNIC_RXB_ECN_THLD(n) (0x0801e + (n)) /* 0x20078 + 4*n */ +#define FBNIC_RXB_ECN_THLD_CNT 8 +#define FBNIC_RXB_ECN_THLD_ON CSR_GENMASK(12, 0) +#define FBNIC_RXB_ECN_THLD_OFF CSR_GENMASK(25, 13) +#define FBNIC_RXB_PBUF_CFG(n) (0x08027 + (n)) /* 0x2009c + 4*n */ +#define FBNIC_RXB_PBUF_CFG_CNT 8 +#define FBNIC_RXB_PBUF_BASE_ADDR CSR_GENMASK(12, 0) +#define FBNIC_RXB_PBUF_SIZE CSR_GENMASK(21, 13) +#define FBNIC_RXB_DWRR_RDE_WEIGHT0 0x0802f /* 0x200bc */ +#define FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM0 CSR_GENMASK(7, 0) +#define FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM1 CSR_GENMASK(15, 8) +#define FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM2 CSR_GENMASK(23, 16) +#define FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM3 CSR_GENMASK(31, 24) +#define FBNIC_RXB_DWRR_RDE_WEIGHT1 0x08030 /* 0x200c0 */ +#define FBNIC_RXB_DWRR_RDE_WEIGHT1_QUANTUM4 CSR_GENMASK(7, 0) +#define FBNIC_RXB_DWRR_BMC_WEIGHT 0x08031 /* 0x200c4 */ +#define FBNIC_RXB_CLDR_PRIO_CFG(n) (0x8034 + (n)) /* 0x200d0 + 4*n */ +#define FBNIC_RXB_CLDR_PRIO_CFG_CNT 16 +#define FBNIC_RXB_ENDIAN_FCS 0x08044 /* 0x20110 */ +enum { + /* Unused */ + /* Unused */ + FBNIC_RXB_DEQUEUE_BMC = 2, + FBNIC_RXB_DEQUEUE_HOST = 3, + FBNIC_RXB_DEQUEUE_INDICES = 4 +}; + +#define FBNIC_RXB_PBUF_CREDIT(n) (0x08047 + (n)) /* 0x2011C + 4*n */ +#define FBNIC_RXB_PBUF_CREDIT_CNT 8 +#define FBNIC_RXB_PBUF_CREDIT_MASK CSR_GENMASK(13, 0) +#define FBNIC_RXB_INTF_CREDIT 0x0804f /* 0x2013C */ +#define FBNIC_RXB_INTF_CREDIT_MASK0 CSR_GENMASK(3, 0) +#define FBNIC_RXB_INTF_CREDIT_MASK1 CSR_GENMASK(7, 4) +#define FBNIC_RXB_INTF_CREDIT_MASK2 CSR_GENMASK(11, 8) +#define FBNIC_RXB_INTF_CREDIT_MASK3 CSR_GENMASK(15, 12) + +#define FBNIC_RXB_ERR_INTR_STS 0x08050 /* 0x20140 */ +#define FBNIC_RXB_ERR_INTR_STS_PS CSR_GENMASK(15, 12) +#define FBNIC_RXB_ERR_INTR_MASK 0x08052 /* 0x20148 */ +#define FBNIC_RXB_PAUSE_EVENT_CNT(n) (0x08053 + (n)) /* 0x2014c + 4*n */ +#define FBNIC_RXB_DROP_FRMS_STS(n) (0x08057 + (n)) /* 0x2015c + 4*n */ +#define FBNIC_RXB_DROP_BYTES_STS_L(n) \ + (0x08080 + 2 * (n)) /* 0x20200 + 8*n */ +#define FBNIC_RXB_DROP_BYTES_STS_H(n) \ + (0x08081 + 2 * (n)) /* 0x20204 + 8*n */ +#define FBNIC_RXB_TRUN_FRMS_STS(n) (0x08091 + (n)) /* 0x20244 + 4*n */ +#define FBNIC_RXB_TRUN_BYTES_STS_L(n) \ + (0x080c0 + 2 * (n)) /* 0x20300 + 8*n */ +#define FBNIC_RXB_TRUN_BYTES_STS_H(n) \ + (0x080c1 + 2 * (n)) /* 0x20304 + 8*n */ +#define FBNIC_RXB_TRANS_PAUSE_STS(n) (0x080d1 + (n)) /* 0x20344 + 4*n */ +#define FBNIC_RXB_TRANS_DROP_STS(n) (0x080d9 + (n)) /* 0x20364 + 4*n */ +#define FBNIC_RXB_TRANS_ECN_STS(n) (0x080e1 + (n)) /* 0x20384 + 4*n */ +enum { + FBNIC_RXB_ENQUEUE_NET = 0, + FBNIC_RXB_ENQUEUE_BMC = 1, + /* Unused */ + /* Unused */ + FBNIC_RXB_ENQUEUE_INDICES = 4 +}; + +#define FBNIC_RXB_INTR_PS_COUNT(n) (0x080e9 + (n)) /* 0x203a4 + 4*n */ +#define FBNIC_RXB_DRBO_FRM_CNT_SRC(n) (0x080f9 + (n)) /* 0x203e4 + 4*n */ +#define FBNIC_RXB_DRBO_BYTE_CNT_SRC_L(n) \ + (0x080fd + (n)) /* 0x203f4 + 4*n */ +#define FBNIC_RXB_DRBO_BYTE_CNT_SRC_H(n) \ + (0x08101 + (n)) /* 0x20404 + 4*n */ +#define FBNIC_RXB_INTF_FRM_CNT_DST(n) (0x08105 + (n)) /* 0x20414 + 4*n */ +#define FBNIC_RXB_INTF_BYTE_CNT_DST_L(n) \ + (0x08109 + (n)) /* 0x20424 + 4*n */ +#define FBNIC_RXB_INTF_BYTE_CNT_DST_H(n) \ + (0x0810d + (n)) /* 0x20434 + 4*n */ +#define FBNIC_RXB_PBUF_FRM_CNT_DST(n) (0x08111 + (n)) /* 0x20444 + 4*n */ +#define FBNIC_RXB_PBUF_BYTE_CNT_DST_L(n) \ + (0x08115 + (n)) /* 0x20454 + 4*n */ +#define FBNIC_RXB_PBUF_BYTE_CNT_DST_H(n) \ + (0x08119 + (n)) /* 0x20464 + 4*n */ + +#define FBNIC_RXB_PBUF_FIFO_LEVEL(n) (0x0811d + (n)) /* 0x20474 + 4*n */ + +#define FBNIC_RXB_PAUSE_STORM_UNIT_RD 0x08125 /* 0x20494 */ +#define FBNIC_RXB_INTEGRITY_ERR(n) (0x0812f + (n)) /* 0x204bc + 4*n */ +#define FBNIC_RXB_MAC_ERR(n) (0x08133 + (n)) /* 0x204cc + 4*n */ +#define FBNIC_RXB_PARSER_ERR(n) (0x08137 + (n)) /* 0x204dc + 4*n */ +#define FBNIC_RXB_FRM_ERR(n) (0x0813b + (n)) /* 0x204ec + 4*n */ + +#define FBNIC_RXB_DWRR_RDE_WEIGHT0_EXT 0x08143 /* 0x2050c */ +#define FBNIC_RXB_DWRR_RDE_WEIGHT1_EXT 0x08144 /* 0x20510 */ +#define FBNIC_CSR_END_RXB 0x081b1 /* CSR section delimiter */ + +/* Rx Parser and Classifier Registers */ +#define FBNIC_CSR_START_RPC 0x08400 /* CSR section delimiter */ +#define FBNIC_RPC_RMI_CONFIG 0x08400 /* 0x21000 */ +#define FBNIC_RPC_RMI_CONFIG_OH_BYTES CSR_GENMASK(4, 0) +#define FBNIC_RPC_RMI_CONFIG_FCS_PRESENT CSR_BIT(8) +#define FBNIC_RPC_RMI_CONFIG_ENABLE CSR_BIT(12) +#define FBNIC_RPC_RMI_CONFIG_MTU CSR_GENMASK(31, 16) + +#define FBNIC_RPC_ACT_TBL0_DEFAULT 0x0840a /* 0x21028 */ +#define FBNIC_RPC_ACT_TBL0_DROP CSR_BIT(0) +#define FBNIC_RPC_ACT_TBL0_DEST_MASK CSR_GENMASK(3, 1) +enum { + FBNIC_RPC_ACT_TBL0_DEST_HOST = 1, + FBNIC_RPC_ACT_TBL0_DEST_BMC = 2, + FBNIC_RPC_ACT_TBL0_DEST_EI = 4, +}; + +#define FBNIC_RPC_ACT_TBL0_Q_SEL CSR_BIT(4) +#define FBNIC_RPC_ACT_TBL0_Q_ID CSR_GENMASK(15, 8) +#define FBNIC_RPC_ACT_TBL0_DMA_HINT CSR_GENMASK(24, 16) +#define FBNIC_RPC_ACT_TBL0_TS_ENA CSR_BIT(28) +#define FBNIC_RPC_ACT_TBL0_ACT_TBL_IDX CSR_BIT(29) +#define FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID CSR_BIT(30) + +#define FBNIC_RPC_ACT_TBL1_DEFAULT 0x0840b /* 0x2102c */ +#define FBNIC_RPC_ACT_TBL1_RSS_ENA_MASK CSR_GENMASK(15, 0) +enum { + FBNIC_RPC_ACT_TBL1_RSS_ENA_IP_SRC = 1, + FBNIC_RPC_ACT_TBL1_RSS_ENA_IP_DST = 2, + FBNIC_RPC_ACT_TBL1_RSS_ENA_L4_SRC = 4, + FBNIC_RPC_ACT_TBL1_RSS_ENA_L4_DST = 8, + FBNIC_RPC_ACT_TBL1_RSS_ENA_L2_DA = 16, + FBNIC_RPC_ACT_TBL1_RSS_ENA_L4_RSS_BYTE = 32, + FBNIC_RPC_ACT_TBL1_RSS_ENA_IV6_FL_LBL = 64, + FBNIC_RPC_ACT_TBL1_RSS_ENA_OV6_FL_LBL = 128, + FBNIC_RPC_ACT_TBL1_RSS_ENA_DSCP = 256, + FBNIC_RPC_ACT_TBL1_RSS_ENA_L3_PROT = 512, + FBNIC_RPC_ACT_TBL1_RSS_ENA_L4_PROT = 1024, +}; + +#define FBNIC_RPC_RSS_KEY(n) (0x0840c + (n)) /* 0x21030 + 4*n */ +#define FBNIC_RPC_RSS_KEY_BIT_LEN 425 +#define FBNIC_RPC_RSS_KEY_BYTE_LEN \ + DIV_ROUND_UP(FBNIC_RPC_RSS_KEY_BIT_LEN, 8) +#define FBNIC_RPC_RSS_KEY_DWORD_LEN \ + DIV_ROUND_UP(FBNIC_RPC_RSS_KEY_BIT_LEN, 32) +#define FBNIC_RPC_RSS_KEY_LAST_IDX \ + (FBNIC_RPC_RSS_KEY_DWORD_LEN - 1) +#define FBNIC_RPC_RSS_KEY_LAST_MASK \ + CSR_GENMASK(31, \ + FBNIC_RPC_RSS_KEY_DWORD_LEN * 32 - \ + FBNIC_RPC_RSS_KEY_BIT_LEN) + +#define FBNIC_RPC_CNTR_TCP_OPT_ERR 0x0849e /* 0x21278 */ +#define FBNIC_RPC_CNTR_UNKN_ETYPE 0x0849f /* 0x2127c */ +#define FBNIC_RPC_CNTR_IPV4_FRAG 0x084a0 /* 0x21280 */ +#define FBNIC_RPC_CNTR_IPV6_FRAG 0x084a1 /* 0x21284 */ +#define FBNIC_RPC_CNTR_IPV4_ESP 0x084a2 /* 0x21288 */ +#define FBNIC_RPC_CNTR_IPV6_ESP 0x084a3 /* 0x2128c */ +#define FBNIC_RPC_CNTR_UNKN_EXT_HDR 0x084a4 /* 0x21290 */ +#define FBNIC_RPC_CNTR_OUT_OF_HDR_ERR 0x084a5 /* 0x21294 */ +#define FBNIC_RPC_CNTR_OVR_SIZE_ERR 0x084a6 /* 0x21298 */ + +#define FBNIC_RPC_TCAM_MACDA_VALIDATE 0x0852d /* 0x214b4 */ +#define FBNIC_RPC_STAT_RX_PACKET_1519_2047_BYTES_L \ + 0x0855f /* 0x2157c */ +#define FBNIC_RPC_STAT_RX_PACKET_1519_2047_BYTES_H \ + 0x08560 /* 0x21580 */ +#define FBNIC_RPC_STAT_RX_PACKET_2048_4095_BYTES_L \ + 0x08561 /* 0x21584 */ +#define FBNIC_RPC_STAT_RX_PACKET_2048_4095_BYTES_H \ + 0x08562 /* 0x21588 */ +#define FBNIC_RPC_STAT_RX_PACKET_4096_8191_BYTES_L \ + 0x08563 /* 0x2158c */ +#define FBNIC_RPC_STAT_RX_PACKET_4096_8191_BYTES_H \ + 0x08564 /* 0x21590 */ +#define FBNIC_RPC_STAT_RX_PACKET_8192_9216_BYTES_L \ + 0x08565 /* 0x21594 */ +#define FBNIC_RPC_STAT_RX_PACKET_8192_9216_BYTES_H \ + 0x08566 /* 0x21598 */ +#define FBNIC_RPC_STAT_RX_PACKET_9217_MAX_BYTES_L \ + 0x08567 /* 0x2159c */ +#define FBNIC_RPC_STAT_RX_PACKET_9217_MAX_BYTES_H \ + 0x08568 /* 0x215a0 */ +#define FBNIC_CSR_END_RPC 0x0856b /* CSR section delimiter */ + +/* RPC RAM Registers */ + +#define FBNIC_CSR_START_RPC_RAM 0x08800 /* CSR section delimiter */ +#define FBNIC_RPC_ACT_TBL0(n) (0x08800 + (n)) /* 0x22000 + 4*n */ +#define FBNIC_RPC_ACT_TBL1(n) (0x08840 + (n)) /* 0x22100 + 4*n */ +#define FBNIC_RPC_ACT_TBL_NUM_ENTRIES 64 + +/* TCAM Tables */ +#define FBNIC_RPC_TCAM_VALIDATE CSR_BIT(31) + +/* 64 Action TCAM Entries, 12 registers + * 3 mixed, src port, dst port, 6 L4 words, and Validate + */ +#define FBNIC_RPC_TCAM_ACT(m, n) \ + (0x08880 + 0x40 * (n) + (m)) /* 0x22200 + 256*n + 4*m */ + +#define FBNIC_RPC_TCAM_ACT_VALUE CSR_GENMASK(15, 0) +#define FBNIC_RPC_TCAM_ACT_MASK CSR_GENMASK(31, 16) + +#define FBNIC_RPC_TCAM_MACDA(m, n) \ + (0x08b80 + 0x20 * (n) + (m)) /* 0x022e00 + 128*n + 4*m */ +#define FBNIC_RPC_TCAM_MACDA_VALUE CSR_GENMASK(15, 0) +#define FBNIC_RPC_TCAM_MACDA_MASK CSR_GENMASK(31, 16) + +#define FBNIC_RPC_TCAM_OUTER_IPSRC(m, n)\ + (0x08c00 + 0x08 * (n) + (m)) /* 0x023000 + 32*n + 4*m */ +#define FBNIC_RPC_TCAM_IP_ADDR_VALUE CSR_GENMASK(15, 0) +#define FBNIC_RPC_TCAM_IP_ADDR_MASK CSR_GENMASK(31, 16) + +#define FBNIC_RPC_TCAM_OUTER_IPDST(m, n)\ + (0x08c48 + 0x08 * (n) + (m)) /* 0x023120 + 32*n + 4*m */ +#define FBNIC_RPC_TCAM_IPSRC(m, n)\ + (0x08c90 + 0x08 * (n) + (m)) /* 0x023240 + 32*n + 4*m */ +#define FBNIC_RPC_TCAM_IPDST(m, n)\ + (0x08cd8 + 0x08 * (n) + (m)) /* 0x023360 + 32*n + 4*m */ + +#define FBNIC_RPC_RSS_TBL(n, m) \ + (0x08d20 + 0x100 * (n) + (m)) /* 0x023480 + 1024*n + 4*m */ +#define FBNIC_RPC_RSS_TBL_COUNT 2 +#define FBNIC_RPC_RSS_TBL_SIZE 256 +#define FBNIC_CSR_END_RPC_RAM 0x08f1f /* CSR section delimiter */ + +/* Fab Registers */ +#define FBNIC_CSR_START_FAB 0x0C000 /* CSR section delimiter */ +#define FBNIC_FAB_AXI4_AR_SPACER_2_CFG 0x0C005 /* 0x30014 */ +#define FBNIC_FAB_AXI4_AR_SPACER_MASK CSR_BIT(16) +#define FBNIC_FAB_AXI4_AR_SPACER_THREADSHOLD CSR_GENMASK(15, 0) +#define FBNIC_CSR_END_FAB 0x0C020 /* CSR section delimiter */ + +/* Master Registers */ +#define FBNIC_CSR_START_MASTER 0x0C400 /* CSR section delimiter */ +#define FBNIC_MASTER_SPARE_0 0x0C41B /* 0x3106c */ +#define FBNIC_CSR_END_MASTER 0x0C452 /* CSR section delimiter */ + +/* MAC PCS registers */ +#define FBNIC_CSR_START_PCS 0x10000 /* CSR section delimiter */ +#define FBNIC_PCS_PAGE(n) (0x10000 + 0x400 * (n)) /* 0x40000 + 1024*n */ +#define FBNIC_PCS(reg, n) ((reg) + FBNIC_PCS_PAGE(n)) +#define FBNIC_CSR_END_PCS 0x10668 /* CSR section delimiter */ + +#define FBNIC_CSR_START_RSFEC 0x10800 /* CSR section delimiter */ +#define FBNIC_RSFEC_CONTROL(n) (0x10800 + 8 * (n)) /* 0x42000 + 32*n */ + +/* We have 4 RSFEC engines present in our part, however we are only using 1. + * As such only CCW(0) and NCCW(0) will never be non-zero and the other + * registers can be ignored. + */ +#define FBNIC_RSFEC_CCW_LO(n) (0x10802 + 8 * (n)) /* 0x42008 + 32*n */ +#define FBNIC_RSFEC_CCW_HI(n) (0x10803 + 8 * (n)) /* 0x4200c + 32*n */ +#define FBNIC_RSFEC_NCCW_LO(n) (0x10804 + 8 * (n)) /* 0x42010 + 32*n */ +#define FBNIC_RSFEC_NCCW_HI(n) (0x10805 + 8 * (n)) /* 0x42014 + 32*n */ + +#define FBNIC_PCS_MAX_LANES 4 +#define FBNIC_PCS_SYMBLERR_LO(n) \ + (0x10880 + 2 * (n)) /* 0x42200 + 8*n */ +#define FBNIC_PCS_SYMBLERR_HI(n) \ + (0x10881 + 2 * (n)) /* 0x42204 + 8*n */ +#define FBNIC_CSR_END_RSFEC 0x108c8 /* CSR section delimiter */ + +/* MAC MAC registers (ASIC only) */ +#define FBNIC_CSR_START_MAC_MAC 0x11000 /* CSR section delimiter */ +#define FBNIC_MAC_COMMAND_CONFIG 0x11002 /* 0x44008 */ +#define FBNIC_MAC_COMMAND_CONFIG_RX_PAUSE_DIS CSR_BIT(29) +#define FBNIC_MAC_COMMAND_CONFIG_TX_PAUSE_DIS CSR_BIT(28) +#define FBNIC_MAC_COMMAND_CONFIG_FLT_HDL_DIS CSR_BIT(27) +#define FBNIC_MAC_COMMAND_CONFIG_TX_PAD_EN CSR_BIT(11) +#define FBNIC_MAC_COMMAND_CONFIG_LOOPBACK_EN CSR_BIT(10) +#define FBNIC_MAC_COMMAND_CONFIG_PROMISC_EN CSR_BIT(4) +#define FBNIC_MAC_COMMAND_CONFIG_RX_ENA CSR_BIT(1) +#define FBNIC_MAC_COMMAND_CONFIG_TX_ENA CSR_BIT(0) +#define FBNIC_MAC_CL01_PAUSE_QUANTA 0x11015 /* 0x44054 */ +#define FBNIC_MAC_CL01_QUANTA_THRESH 0x11019 /* 0x44064 */ +#define FBNIC_CSR_END_MAC_MAC 0x11028 /* CSR section delimiter */ + +/* Signals from MAC, AN, PCS, and LED CSR registers (ASIC only) */ +#define FBNIC_CSR_START_SIG 0x11800 /* CSR section delimiter */ +#define FBNIC_SIG_MAC_IN0 0x11800 /* 0x46000 */ +#define FBNIC_SIG_MAC_IN0_RESET_FF_TX_CLK CSR_BIT(14) +#define FBNIC_SIG_MAC_IN0_RESET_FF_RX_CLK CSR_BIT(13) +#define FBNIC_SIG_MAC_IN0_RESET_TX_CLK CSR_BIT(12) +#define FBNIC_SIG_MAC_IN0_RESET_RX_CLK CSR_BIT(11) +#define FBNIC_SIG_MAC_IN0_TX_CRC CSR_BIT(8) +#define FBNIC_SIG_MAC_IN0_CFG_MODE128 CSR_BIT(10) +#define FBNIC_SIG_PCS_OUT0 0x11808 /* 0x46020 */ +#define FBNIC_SIG_PCS_OUT0_LINK CSR_BIT(27) +#define FBNIC_SIG_PCS_OUT0_BLOCK_LOCK CSR_GENMASK(24, 5) +#define FBNIC_SIG_PCS_OUT0_AMPS_LOCK CSR_GENMASK(4, 1) +#define FBNIC_SIG_PCS_OUT1 0x11809 /* 0x46024 */ +#define FBNIC_SIG_PCS_OUT1_FCFEC_LOCK CSR_GENMASK(11, 8) +#define FBNIC_SIG_PCS_INTR_STS 0x11814 /* 0x46050 */ +#define FBNIC_SIG_PCS_INTR_LINK_DOWN CSR_BIT(1) +#define FBNIC_SIG_PCS_INTR_LINK_UP CSR_BIT(0) +#define FBNIC_SIG_PCS_INTR_MASK 0x11816 /* 0x46058 */ +#define FBNIC_CSR_END_SIG 0x1184e /* CSR section delimiter */ + +#define FBNIC_CSR_START_MAC_STAT 0x11a00 +#define FBNIC_MAC_STAT_RX_XOFF_STB_L 0x11a00 /* 0x46800 */ +#define FBNIC_MAC_STAT_RX_XOFF_STB_H 0x11a01 /* 0x46804 */ +#define FBNIC_MAC_STAT_TX_XOFF_STB_L 0x11a04 /* 0x46810 */ +#define FBNIC_MAC_STAT_TX_XOFF_STB_H 0x11a05 /* 0x46814 */ +#define FBNIC_MAC_STAT_RX_BYTE_COUNT_L 0x11a08 /* 0x46820 */ +#define FBNIC_MAC_STAT_RX_BYTE_COUNT_H 0x11a09 /* 0x46824 */ +#define FBNIC_MAC_STAT_RX_ALIGN_ERROR_L 0x11a0a /* 0x46828 */ +#define FBNIC_MAC_STAT_RX_ALIGN_ERROR_H 0x11a0b /* 0x4682c */ +#define FBNIC_MAC_STAT_RX_TOOLONG_L 0x11a0e /* 0x46838 */ +#define FBNIC_MAC_STAT_RX_TOOLONG_H 0x11a0f /* 0x4683c */ +#define FBNIC_MAC_STAT_RX_RECEIVED_OK_L 0x11a12 /* 0x46848 */ +#define FBNIC_MAC_STAT_RX_RECEIVED_OK_H 0x11a13 /* 0x4684c */ +#define FBNIC_MAC_STAT_RX_PACKET_BAD_FCS_L \ + 0x11a14 /* 0x46850 */ +#define FBNIC_MAC_STAT_RX_PACKET_BAD_FCS_H \ + 0x11a15 /* 0x46854 */ +#define FBNIC_MAC_STAT_RX_IFINERRORS_L 0x11a18 /* 0x46860 */ +#define FBNIC_MAC_STAT_RX_IFINERRORS_H 0x11a19 /* 0x46864 */ +#define FBNIC_MAC_STAT_RX_MULTICAST_L 0x11a1c /* 0x46870 */ +#define FBNIC_MAC_STAT_RX_MULTICAST_H 0x11a1d /* 0x46874 */ +#define FBNIC_MAC_STAT_RX_BROADCAST_L 0x11a1e /* 0x46878 */ +#define FBNIC_MAC_STAT_RX_BROADCAST_H 0x11a1f /* 0x4687c */ +#define FBNIC_MAC_STAT_RX_UNDERSIZE_L 0x11a24 /* 0x46890 */ +#define FBNIC_MAC_STAT_RX_UNDERSIZE_H 0x11a25 /* 0x46894 */ +#define FBNIC_MAC_STAT_RX_PACKET_64_BYTES_L \ + 0x11a26 /* 0x46898 */ +#define FBNIC_MAC_STAT_RX_PACKET_64_BYTES_H \ + 0x11a27 /* 0x4689c */ +#define FBNIC_MAC_STAT_RX_PACKET_65_127_BYTES_L \ + 0x11a28 /* 0x468a0 */ +#define FBNIC_MAC_STAT_RX_PACKET_65_127_BYTES_H \ + 0x11a29 /* 0x468a4 */ +#define FBNIC_MAC_STAT_RX_PACKET_128_255_BYTES_L \ + 0x11a2a /* 0x468a8 */ +#define FBNIC_MAC_STAT_RX_PACKET_128_255_BYTES_H \ + 0x11a2b /* 0x468ac */ +#define FBNIC_MAC_STAT_RX_PACKET_256_511_BYTES_L \ + 0x11a2c /* 0x468b0 */ +#define FBNIC_MAC_STAT_RX_PACKET_256_511_BYTES_H \ + 0x11a2d /* 0x468b4 */ +#define FBNIC_MAC_STAT_RX_PACKET_512_1023_BYTES_L \ + 0x11a2e /* 0x468b8 */ +#define FBNIC_MAC_STAT_RX_PACKET_512_1023_BYTES_H \ + 0x11a2f /* 0x468bc */ +#define FBNIC_MAC_STAT_RX_PACKET_1024_1518_BYTES_L \ + 0x11a30 /* 0x468c0 */ +#define FBNIC_MAC_STAT_RX_PACKET_1024_1518_BYTES_H \ + 0x11a31 /* 0x468c4 */ +#define FBNIC_MAC_STAT_RX_PACKET_1519_MAX_BYTES_L \ + 0x11a32 /* 0x468c8 */ +#define FBNIC_MAC_STAT_RX_PACKET_1519_MAX_BYTES_H \ + 0x11a33 /* 0x468cc */ +#define FBNIC_MAC_STAT_RX_OVERSIZE_L 0x11a34 /* 0x468d0 */ +#define FBNIC_MAC_STAT_RX_OVERSSIZE_H 0x11a35 /* 0x468d4 */ +#define FBNIC_MAC_STAT_RX_JABBER_L 0x11a36 /* 0x468d8 */ +#define FBNIC_MAC_STAT_RX_JABBER_H 0x11a37 /* 0x468dc */ +#define FBNIC_MAC_STAT_RX_FRAGMENT_L 0x11a38 /* 0x468e0 */ +#define FBNIC_MAC_STAT_RX_FRAGMENT_H 0x11a39 /* 0x468e4 */ +#define FBNIC_MAC_STAT_RX_CONTROL_FRAMES_L \ + 0x11a3c /* 0x468f0 */ +#define FBNIC_MAC_STAT_RX_CONTROL_FRAMES_H \ + 0x11a3d /* 0x468f4 */ +#define FBNIC_MAC_STAT_TX_BYTE_COUNT_L 0x11a3e /* 0x468f8 */ +#define FBNIC_MAC_STAT_TX_BYTE_COUNT_H 0x11a3f /* 0x468fc */ +#define FBNIC_MAC_STAT_TX_TRANSMITTED_OK_L \ + 0x11a42 /* 0x46908 */ +#define FBNIC_MAC_STAT_TX_TRANSMITTED_OK_H \ + 0x11a43 /* 0x4690c */ +#define FBNIC_MAC_STAT_TX_IFOUTERRORS_L 0x11a46 /* 0x46918 */ +#define FBNIC_MAC_STAT_TX_IFOUTERRORS_H 0x11a47 /* 0x4691c */ +#define FBNIC_MAC_STAT_TX_MULTICAST_L 0x11a4a /* 0x46928 */ +#define FBNIC_MAC_STAT_TX_MULTICAST_H 0x11a4b /* 0x4692c */ +#define FBNIC_MAC_STAT_TX_BROADCAST_L 0x11a4c /* 0x46930 */ +#define FBNIC_MAC_STAT_TX_BROADCAST_H 0x11a4d /* 0x46934 */ +#define FBNIC_MAC_STAT_TX_PACKET_64_BYTES_L \ + 0x11a4e /* 0x46938 */ +#define FBNIC_MAC_STAT_TX_PACKET_64_BYTES_H \ + 0x11a4f /* 0x4693c */ +#define FBNIC_MAC_STAT_TX_PACKET_65_127_BYTES_L \ + 0x11a50 /* 0x46940 */ +#define FBNIC_MAC_STAT_TX_PACKET_65_127_BYTES_H \ + 0x11a51 /* 0x46944 */ +#define FBNIC_MAC_STAT_TX_PACKET_128_255_BYTES_L \ + 0x11a52 /* 0x46948 */ +#define FBNIC_MAC_STAT_TX_PACKET_128_255_BYTES_H \ + 0x11a53 /* 0x4694c */ +#define FBNIC_MAC_STAT_TX_PACKET_256_511_BYTES_L \ + 0x11a54 /* 0x46950 */ +#define FBNIC_MAC_STAT_TX_PACKET_256_511_BYTES_H \ + 0x11a55 /* 0x46954 */ +#define FBNIC_MAC_STAT_TX_PACKET_512_1023_BYTES_L \ + 0x11a56 /* 0x46958 */ +#define FBNIC_MAC_STAT_TX_PACKET_512_1023_BYTES_H \ + 0x11a57 /* 0x4695c */ +#define FBNIC_MAC_STAT_TX_PACKET_1024_1518_BYTES_L \ + 0x11a58 /* 0x46960 */ +#define FBNIC_MAC_STAT_TX_PACKET_1024_1518_BYTES_H \ + 0x11a59 /* 0x46964 */ +#define FBNIC_MAC_STAT_TX_PACKET_1519_MAX_BYTES_L \ + 0x11a5a /* 0x46968 */ +#define FBNIC_MAC_STAT_TX_PACKET_1519_MAX_BYTES_H \ + 0x11a5b /* 0x4696c */ +#define FBNIC_MAC_STAT_TX_CONTROL_FRAMES_L \ + 0x11a5e /* 0x46978 */ +#define FBNIC_MAC_STAT_TX_CONTROL_FRAMES_H \ + 0x11a5f /* 0x4697c */ + +/* PCIE Comphy Registers */ +#define FBNIC_CSR_START_PCIE_SS_COMPHY 0x2442e /* CSR section delimiter */ +#define FBNIC_CSR_END_PCIE_SS_COMPHY 0x279d7 /* CSR section delimiter */ + +/* PUL User Registers */ +#define FBNIC_CSR_START_PUL_USER 0x31000 /* CSR section delimiter */ +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG 0x3103d /* 0xc40f4 */ +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH CSR_BIT(19) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME CSR_BIT(18) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR CSR_GENMASK(17, 15) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_RQM_ATTR CSR_GENMASK(14, 12) +#define FBNIC_PUL_OB_TLP_HDR_AW_CFG_TQM_ATTR CSR_GENMASK(11, 9) +enum { + FBNIC_TLP_ATTR_NS = 1, + FBNIC_TLP_ATTR_RO = 2, + FBNIC_TLP_ATTR_IDO = 4, +}; + +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG 0x3103e /* 0xc40f8 */ +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_FLUSH CSR_BIT(19) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_BME CSR_BIT(18) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_TDE_ATTR CSR_GENMASK(17, 15) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_RQM_ATTR CSR_GENMASK(14, 12) +#define FBNIC_PUL_OB_TLP_HDR_AR_CFG_TQM_ATTR CSR_GENMASK(11, 9) +#define FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0 \ + 0x3106e /* 0xc41b8 */ +#define FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0 \ + 0x31070 /* 0xc41c0 */ +#define FBNIC_PUL_USER_OB_RD_DWORD_CNT_63_32 \ + 0x31071 /* 0xc41c4 */ +#define FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0 \ + 0x31072 /* 0xc41c8 */ +#define FBNIC_PUL_USER_OB_WR_TLP_CNT_63_32 \ + 0x31073 /* 0xc41cc */ +#define FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0 \ + 0x31074 /* 0xc41d0 */ +#define FBNIC_PUL_USER_OB_WR_DWORD_CNT_63_32 \ + 0x31075 /* 0xc41d4 */ +#define FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0 \ + 0x31076 /* 0xc41d8 */ +#define FBNIC_PUL_USER_OB_CPL_TLP_CNT_63_32 \ + 0x31077 /* 0xc41dc */ +#define FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0 \ + 0x31078 /* 0xc41e0 */ +#define FBNIC_PUL_USER_OB_CPL_DWORD_CNT_63_32 \ + 0x31079 /* 0xc41e4 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0 \ + 0x3107a /* 0xc41e8 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_63_32 \ + 0x3107b /* 0xc41ec */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0 \ + 0x3107c /* 0xc41f0 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_63_32 \ + 0x3107d /* 0xc41f4 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0 \ + 0x3107e /* 0xc41f8 */ +#define FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_63_32 \ + 0x3107f /* 0xc41fc */ +#define FBNIC_CSR_END_PUL_USER 0x310ea /* CSR section delimiter */ + +/* Queue Registers + * + * The queue register offsets are specific for a given queue grouping. So to + * find the actual register offset it is necessary to combine FBNIC_QUEUE(n) + * with the register to get the actual register offset like so: + * FBNIC_QUEUE_TWQ0_CTL(n) == FBNIC_QUEUE(n) + FBNIC_QUEUE_TWQ0_CTL + */ +#define FBNIC_CSR_START_QUEUE 0x40000 /* CSR section delimiter */ +#define FBNIC_QUEUE_STRIDE 0x400 /* 0x1000 */ +#define FBNIC_QUEUE(n)\ + (0x40000 + FBNIC_QUEUE_STRIDE * (n)) /* 0x100000 + 4096*n */ + +#define FBNIC_QUEUE_TWQ0_CTL 0x000 /* 0x000 */ +#define FBNIC_QUEUE_TWQ1_CTL 0x001 /* 0x004 */ +#define FBNIC_QUEUE_TWQ_CTL_RESET CSR_BIT(0) +#define FBNIC_QUEUE_TWQ_CTL_ENABLE CSR_BIT(1) +#define FBNIC_QUEUE_TWQ0_TAIL 0x002 /* 0x008 */ +#define FBNIC_QUEUE_TWQ1_TAIL 0x003 /* 0x00c */ +#define FBNIC_QUEUE_TWQ0_PTRS 0x004 /* 0x010 */ +#define FBNIC_QUEUE_TWQ1_PTRS 0x005 /* 0x014 */ +#define FBNIC_QUEUE_TWQ_PTRS_HEAD_MASK CSR_GENMASK(31, 16) + +#define FBNIC_QUEUE_TWQ0_SIZE 0x00a /* 0x028 */ +#define FBNIC_QUEUE_TWQ1_SIZE 0x00b /* 0x02c */ +#define FBNIC_QUEUE_TWQ_SIZE_MASK CSR_GENMASK(3, 0) + +#define FBNIC_QUEUE_TWQ0_BAL 0x020 /* 0x080 */ +#define FBNIC_QUEUE_BAL_MASK CSR_GENMASK(31, 7) +#define FBNIC_QUEUE_TWQ0_BAH 0x021 /* 0x084 */ +#define FBNIC_QUEUE_TWQ1_BAL 0x022 /* 0x088 */ +#define FBNIC_QUEUE_TWQ1_BAH 0x023 /* 0x08c */ + +/* Tx Work Queue Statistics Registers */ +#define FBNIC_QUEUE_TWQ0_PKT_CNT 0x062 /* 0x188 */ +#define FBNIC_QUEUE_TWQ0_ERR_CNT 0x063 /* 0x18c */ +#define FBNIC_QUEUE_TWQ1_PKT_CNT 0x072 /* 0x1c8 */ +#define FBNIC_QUEUE_TWQ1_ERR_CNT 0x073 /* 0x1cc */ + +/* Tx Completion Queue Registers */ +#define FBNIC_QUEUE_TCQ_CTL 0x080 /* 0x200 */ +#define FBNIC_QUEUE_TCQ_CTL_RESET CSR_BIT(0) +#define FBNIC_QUEUE_TCQ_CTL_ENABLE CSR_BIT(1) + +#define FBNIC_QUEUE_TCQ_HEAD 0x081 /* 0x204 */ +#define FBNIC_QUEUE_TCQ_PTRS 0x082 /* 0x208 */ +#define FBNIC_QUEUE_TCQ_PTRS_TAIL_MASK CSR_GENMASK(31, 16) + +#define FBNIC_QUEUE_TCQ_SIZE 0x084 /* 0x210 */ +#define FBNIC_QUEUE_TCQ_SIZE_MASK CSR_GENMASK(3, 0) + +#define FBNIC_QUEUE_TCQ_BAL 0x0a0 /* 0x280 */ +#define FBNIC_QUEUE_TCQ_BAH 0x0a1 /* 0x284 */ + +/* Tx Interrupt Manager Registers */ +#define FBNIC_QUEUE_TIM_CTL 0x0c0 /* 0x300 */ +#define FBNIC_QUEUE_TIM_CTL_MSIX_MASK CSR_GENMASK(7, 0) + +#define FBNIC_QUEUE_TIM_THRESHOLD 0x0c1 /* 0x304 */ +#define FBNIC_QUEUE_TIM_THRESHOLD_TWD_MASK CSR_GENMASK(14, 0) + +#define FBNIC_QUEUE_TIM_CLEAR 0x0c2 /* 0x308 */ +#define FBNIC_QUEUE_TIM_CLEAR_MASK CSR_BIT(0) +#define FBNIC_QUEUE_TIM_SET 0x0c3 /* 0x30c */ +#define FBNIC_QUEUE_TIM_SET_MASK CSR_BIT(0) +#define FBNIC_QUEUE_TIM_MASK 0x0c4 /* 0x310 */ +#define FBNIC_QUEUE_TIM_MASK_MASK CSR_BIT(0) + +#define FBNIC_QUEUE_TIM_TIMER 0x0c5 /* 0x314 */ + +#define FBNIC_QUEUE_TIM_COUNTS 0x0c6 /* 0x318 */ +#define FBNIC_QUEUE_TIM_COUNTS_CNT1_MASK CSR_GENMASK(30, 16) +#define FBNIC_QUEUE_TIM_COUNTS_CNT0_MASK CSR_GENMASK(14, 0) + +/* Rx Completion Queue Registers */ +#define FBNIC_QUEUE_RCQ_CTL 0x200 /* 0x800 */ +#define FBNIC_QUEUE_RCQ_CTL_RESET CSR_BIT(0) +#define FBNIC_QUEUE_RCQ_CTL_ENABLE CSR_BIT(1) + +#define FBNIC_QUEUE_RCQ_HEAD 0x201 /* 0x804 */ +#define FBNIC_QUEUE_RCQ_PTRS 0x202 /* 0x808 */ +#define FBNIC_QUEUE_RCQ_PTRS_TAIL_MASK CSR_GENMASK(31, 16) +#define FBNIC_QUEUE_RCQ_PTRS_HEAD_MASK CSR_GENMASK(15, 0) + +#define FBNIC_QUEUE_RCQ_SIZE 0x204 /* 0x810 */ +#define FBNIC_QUEUE_RCQ_SIZE_MASK CSR_GENMASK(3, 0) + +#define FBNIC_QUEUE_RCQ_BAL 0x220 /* 0x880 */ +#define FBNIC_QUEUE_RCQ_BAH 0x221 /* 0x884 */ + +/* Rx Buffer Descriptor Queue Registers */ +#define FBNIC_QUEUE_BDQ_CTL 0x240 /* 0x900 */ +#define FBNIC_QUEUE_BDQ_CTL_RESET CSR_BIT(0) +#define FBNIC_QUEUE_BDQ_CTL_ENABLE CSR_BIT(1) +#define FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE CSR_BIT(30) + +#define FBNIC_QUEUE_BDQ_HPQ_TAIL 0x241 /* 0x904 */ +#define FBNIC_QUEUE_BDQ_PPQ_TAIL 0x242 /* 0x908 */ +#define FBNIC_QUEUE_BDQ_HPQ_PTRS 0x243 /* 0x90c */ +#define FBNIC_QUEUE_BDQ_PPQ_PTRS 0x244 /* 0x910 */ +#define FBNIC_QUEUE_BDQ_PTRS_HEAD_MASK CSR_GENMASK(31, 16) +#define FBNIC_QUEUE_BDQ_PTRS_TAIL_MASK CSR_GENMASK(15, 0) + +#define FBNIC_QUEUE_BDQ_HPQ_SIZE 0x247 /* 0x91c */ +#define FBNIC_QUEUE_BDQ_PPQ_SIZE 0x248 /* 0x920 */ +#define FBNIC_QUEUE_BDQ_SIZE_MASK CSR_GENMASK(3, 0) + +#define FBNIC_QUEUE_BDQ_HPQ_BAL 0x260 /* 0x980 */ +#define FBNIC_QUEUE_BDQ_HPQ_BAH 0x261 /* 0x984 */ +#define FBNIC_QUEUE_BDQ_PPQ_BAL 0x262 /* 0x988 */ +#define FBNIC_QUEUE_BDQ_PPQ_BAH 0x263 /* 0x98c */ + +/* Rx DMA Engine Configuration */ +#define FBNIC_QUEUE_RDE_CTL0 0x2a0 /* 0xa80 */ +#define FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT CSR_BIT(31) +#define FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK CSR_GENMASK(30, 29) +enum { + FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE = 0, + FBNIC_QUEUE_RDE_CTL0_DROP_WAIT = 1, + FBNIC_QUEUE_RDE_CTL0_DROP_NEVER = 2, +}; + +#define FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK CSR_GENMASK(28, 20) +#define FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK CSR_GENMASK(19, 11) + +#define FBNIC_QUEUE_RDE_CTL1 0x2a1 /* 0xa84 */ +#define FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK CSR_GENMASK(24, 12) +#define FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK CSR_GENMASK(11, 9) +#define FBNIC_QUEUE_RDE_CTL1_PAYLD_PG_CL_MASK CSR_GENMASK(8, 6) +#define FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK CSR_GENMASK(5, 2) +#define FBNIC_QUEUE_RDE_CTL1_PAYLD_PACK_MASK CSR_GENMASK(1, 0) +enum { + FBNIC_QUEUE_RDE_CTL1_PAYLD_PACK_NONE = 0, + FBNIC_QUEUE_RDE_CTL1_PAYLD_PACK_ALL = 1, + FBNIC_QUEUE_RDE_CTL1_PAYLD_PACK_RSS = 2, +}; + +/* Rx Per CQ Statistics Counters */ +#define FBNIC_QUEUE_RDE_PKT_CNT 0x2a2 /* 0xa88 */ +#define FBNIC_QUEUE_RDE_PKT_ERR_CNT 0x2a3 /* 0xa8c */ +#define FBNIC_QUEUE_RDE_CQ_DROP_CNT 0x2a4 /* 0xa90 */ +#define FBNIC_QUEUE_RDE_BDQ_DROP_CNT 0x2a5 /* 0xa94 */ + +/* Rx Interrupt Manager Registers */ +#define FBNIC_QUEUE_RIM_CTL 0x2c0 /* 0xb00 */ +#define FBNIC_QUEUE_RIM_CTL_MSIX_MASK CSR_GENMASK(7, 0) + +#define FBNIC_QUEUE_RIM_THRESHOLD 0x2c1 /* 0xb04 */ +#define FBNIC_QUEUE_RIM_THRESHOLD_RCD_MASK CSR_GENMASK(14, 0) + +#define FBNIC_QUEUE_RIM_CLEAR 0x2c2 /* 0xb08 */ +#define FBNIC_QUEUE_RIM_CLEAR_MASK CSR_BIT(0) +#define FBNIC_QUEUE_RIM_SET 0x2c3 /* 0xb0c */ +#define FBNIC_QUEUE_RIM_SET_MASK CSR_BIT(0) +#define FBNIC_QUEUE_RIM_MASK 0x2c4 /* 0xb10 */ +#define FBNIC_QUEUE_RIM_MASK_MASK CSR_BIT(0) + +#define FBNIC_QUEUE_RIM_COAL_STATUS 0x2c5 /* 0xb14 */ +#define FBNIC_QUEUE_RIM_RCD_COUNT_MASK CSR_GENMASK(30, 16) +#define FBNIC_QUEUE_RIM_TIMER_MASK CSR_GENMASK(13, 0) +#define FBNIC_MAX_QUEUES 128 +#define FBNIC_CSR_END_QUEUE (0x40000 + 0x400 * FBNIC_MAX_QUEUES - 1) + +/* BAR 4 CSRs */ + +/* The IPC mailbox consists of 32 mailboxes, with each mailbox consisting + * of 32 4 byte registers. We will use 2 registers per descriptor so the + * length of the mailbox is reduced to 16. + * + * Currently we use an offset of 0x6000 on BAR4 for the mailbox so we just + * have to do the math and determine the offset based on the mailbox + * direction and index inside that mailbox. + */ +#define FBNIC_IPC_MBX_DESC_LEN 16 +#define FBNIC_IPC_MBX(mbx_idx, desc_idx) \ + ((((mbx_idx) * FBNIC_IPC_MBX_DESC_LEN + (desc_idx)) * 2) + 0x6000) + +/* Use first register in mailbox to flush writes */ +#define FBNIC_FW_ZERO_REG FBNIC_IPC_MBX(0, 0) + +enum { + FBNIC_IPC_MBX_RX_IDX, + FBNIC_IPC_MBX_TX_IDX, + FBNIC_IPC_MBX_INDICES, +}; + +#define FBNIC_IPC_MBX_DESC_LEN_MASK DESC_GENMASK(63, 48) +#define FBNIC_IPC_MBX_DESC_EOM DESC_BIT(46) +#define FBNIC_IPC_MBX_DESC_ADDR_MASK DESC_GENMASK(45, 3) +#define FBNIC_IPC_MBX_DESC_FW_CMPL DESC_BIT(1) +#define FBNIC_IPC_MBX_DESC_HOST_CMPL DESC_BIT(0) + +/* OTP Registers + * These registers are accessible via bar4 offset and are written by CMRT + * on boot. For the write status, the register is broken up in half with OTP + * Write Data Status occupying the top 16 bits and the ECC status occupying the + * bottom 16 bits. + */ +#define FBNIC_NS_OTP_STATUS 0x0021d +#define FBNIC_NS_OTP_WRITE_STATUS 0x0021e + +#define FBNIC_NS_OTP_WRITE_DATA_STATUS_MASK CSR_GENMASK(31, 16) +#define FBNIC_NS_OTP_WRITE_ECC_STATUS_MASK CSR_GENMASK(15, 0) + +#define FBNIC_REGS_VERSION CSR_GENMASK(31, 16) +#define FBNIC_REGS_HW_TYPE CSR_GENMASK(15, 8) +enum{ + FBNIC_CSR_VERSION_V1_0_ASIC = 1, +}; + +/** + * enum fbnic_reg_self_test_codes - return codes from self test routines + * + * This is the code that is returned from the register self test + * routines. + * + * The test either returns success or the register number + * that failed during the test. + * + * @FBNIC_REG_TEST_SUCCESS: no errors + */ +enum fbnic_reg_self_test_codes { + FBNIC_REG_TEST_SUCCESS = 0, +}; + +enum fbnic_reg_self_test_codes fbnic_csr_regs_test(struct fbnic_dev *fbd); + +#endif /* _FBNIC_CSR_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c new file mode 100644 index 000000000..3c4563c8f --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_debugfs.c @@ -0,0 +1,678 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/debugfs.h> +#include <linux/pci.h> +#include <linux/rtnetlink.h> +#include <linux/seq_file.h> + +#include "fbnic.h" +#include "fbnic_txrx.h" + +static struct dentry *fbnic_dbg_root; + +/* Descriptor Seq Functions */ + +static void fbnic_dbg_desc_break(struct seq_file *s, int i) +{ + while (i--) + seq_putc(s, '-'); + + seq_putc(s, '\n'); +} + +static void fbnic_dbg_ring_show(struct seq_file *s) +{ + struct fbnic_ring *ring = s->private; + unsigned long doorbell_offset; + u32 head = 0, tail = 0; + u32 __iomem *csr_base; + + csr_base = fbnic_ring_csr_base(ring); + doorbell_offset = ring->doorbell - csr_base; + + seq_printf(s, "doorbell CSR: %#05lx q_idx: %d\n", + doorbell_offset, ring->q_idx); + seq_printf(s, "size_mask: %#06x size: %zu flags: 0x%02x\n", + ring->size_mask, ring->size, ring->flags); + seq_printf(s, "SW: head: %#06x tail: %#06x\n", + ring->head, ring->tail); + + switch (doorbell_offset) { + case FBNIC_QUEUE_TWQ0_TAIL: + tail = readl(csr_base + FBNIC_QUEUE_TWQ0_PTRS); + head = FIELD_GET(FBNIC_QUEUE_TWQ_PTRS_HEAD_MASK, tail); + break; + case FBNIC_QUEUE_TWQ1_TAIL: + tail = readl(csr_base + FBNIC_QUEUE_TWQ1_PTRS); + head = FIELD_GET(FBNIC_QUEUE_TWQ_PTRS_HEAD_MASK, tail); + break; + case FBNIC_QUEUE_TCQ_HEAD: + head = readl(csr_base + FBNIC_QUEUE_TCQ_PTRS); + tail = FIELD_GET(FBNIC_QUEUE_TCQ_PTRS_TAIL_MASK, head); + break; + case FBNIC_QUEUE_BDQ_HPQ_TAIL: + tail = readl(csr_base + FBNIC_QUEUE_BDQ_HPQ_PTRS); + head = FIELD_GET(FBNIC_QUEUE_BDQ_PTRS_HEAD_MASK, tail); + break; + case FBNIC_QUEUE_BDQ_PPQ_TAIL: + tail = readl(csr_base + FBNIC_QUEUE_BDQ_PPQ_PTRS); + head = FIELD_GET(FBNIC_QUEUE_BDQ_PTRS_HEAD_MASK, tail); + break; + case FBNIC_QUEUE_RCQ_HEAD: + head = readl(csr_base + FBNIC_QUEUE_RCQ_PTRS); + tail = FIELD_GET(FBNIC_QUEUE_RCQ_PTRS_TAIL_MASK, head); + break; + } + + tail &= FBNIC_QUEUE_BDQ_PTRS_TAIL_MASK; + head &= FBNIC_QUEUE_RCQ_PTRS_HEAD_MASK; + + seq_printf(s, "HW: head: %#06x tail: %#06x\n", head, tail); + + seq_puts(s, "\n"); +} + +static void fbnic_dbg_twd_desc_seq_show(struct seq_file *s, int i) +{ + struct fbnic_ring *ring = s->private; + u64 twd = le64_to_cpu(ring->desc[i]); + + switch (FIELD_GET(FBNIC_TWD_TYPE_MASK, twd)) { + case FBNIC_TWD_TYPE_META: + seq_printf(s, "%04x %#06llx %llx %llx %llx %llx %llx %#llx %#llx %llx %#04llx %#04llx %llx %#04llx\n", + i, FIELD_GET(FBNIC_TWD_LEN_MASK, twd), + FIELD_GET(FBNIC_TWD_TYPE_MASK, twd), + FIELD_GET(FBNIC_TWD_FLAG_REQ_COMPLETION, twd), + FIELD_GET(FBNIC_TWD_FLAG_REQ_CSO, twd), + FIELD_GET(FBNIC_TWD_FLAG_REQ_LSO, twd), + FIELD_GET(FBNIC_TWD_FLAG_REQ_TS, twd), + FIELD_GET(FBNIC_TWD_L4_HLEN_MASK, twd), + FIELD_GET(FBNIC_TWD_CSUM_OFFSET_MASK, twd), + FIELD_GET(FBNIC_TWD_L4_TYPE_MASK, twd), + FIELD_GET(FBNIC_TWD_L3_IHLEN_MASK, twd), + FIELD_GET(FBNIC_TWD_L3_OHLEN_MASK, twd), + FIELD_GET(FBNIC_TWD_L3_TYPE_MASK, twd), + FIELD_GET(FBNIC_TWD_L2_HLEN_MASK, twd)); + break; + default: + seq_printf(s, "%04x %#06llx %llx %#014llx\n", i, + FIELD_GET(FBNIC_TWD_LEN_MASK, twd), + FIELD_GET(FBNIC_TWD_TYPE_MASK, twd), + FIELD_GET(FBNIC_TWD_ADDR_MASK, twd)); + break; + } +} + +static int fbnic_dbg_twq_desc_seq_show(struct seq_file *s, void *v) +{ + struct fbnic_ring *ring = s->private; + char hdr[80]; + int i; + + /* Generate header on first entry */ + fbnic_dbg_ring_show(s); + snprintf(hdr, sizeof(hdr), "%4s %5s %s %s\n", + "DESC", "LEN/MSS", "T", "METADATA/TIMESTAMP/BUFFER_ADDR"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + /* Display descriptor */ + if (!ring->desc) { + seq_puts(s, "Descriptor ring not allocated.\n"); + return 0; + } + + for (i = 0; i <= ring->size_mask; i++) + fbnic_dbg_twd_desc_seq_show(s, i); + + return 0; +} + +static int fbnic_dbg_tcq_desc_seq_show(struct seq_file *s, void *v) +{ + struct fbnic_ring *ring = s->private; + char hdr[80]; + int i; + + /* Generate header on first entry */ + fbnic_dbg_ring_show(s); + snprintf(hdr, sizeof(hdr), "%4s %s %s %s %5s %-16s %-6s %-6s\n", + "DESC", "D", "T", "Q", "STATUS", "TIMESTAMP", "HEAD1", "HEAD0"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + /* Display descriptor */ + if (!ring->desc) { + seq_puts(s, "Descriptor ring not allocated.\n"); + return 0; + } + + for (i = 0; i <= ring->size_mask; i++) { + u64 tcd = le64_to_cpu(ring->desc[i]); + + switch (FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd)) { + case FBNIC_TCD_TYPE_0: + seq_printf(s, "%04x %llx %llx %llx %#05llx %-17s %#06llx %#06llx\n", + i, FIELD_GET(FBNIC_TCD_DONE, tcd), + FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd), + FIELD_GET(FBNIC_TCD_TWQ1, tcd), + FIELD_GET(FBNIC_TCD_STATUS_MASK, tcd), + "", + FIELD_GET(FBNIC_TCD_TYPE0_HEAD1_MASK, tcd), + FIELD_GET(FBNIC_TCD_TYPE0_HEAD0_MASK, tcd)); + break; + case FBNIC_TCD_TYPE_1: + seq_printf(s, "%04x %llx %llx %llx %#05llx %#012llx\n", + i, FIELD_GET(FBNIC_TCD_DONE, tcd), + FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd), + FIELD_GET(FBNIC_TCD_TWQ1, tcd), + FIELD_GET(FBNIC_TCD_STATUS_MASK, tcd), + FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd)); + break; + default: + break; + } + } + + return 0; +} + +static int fbnic_dbg_bdq_desc_seq_show(struct seq_file *s, void *v) +{ + struct fbnic_ring *ring = s->private; + char hdr[80]; + int i; + + /* Generate header on first entry */ + fbnic_dbg_ring_show(s); + snprintf(hdr, sizeof(hdr), "%4s %-4s %s\n", + "DESC", "ID", "BUFFER_ADDR"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + /* Display descriptor */ + if (!ring->desc) { + seq_puts(s, "Descriptor ring not allocated.\n"); + return 0; + } + + for (i = 0; i < (ring->size_mask + 1) * FBNIC_BD_FRAG_COUNT; i++) { + u64 bd = le64_to_cpu(ring->desc[i]); + + seq_printf(s, "%04x %#04llx %#014llx\n", i, + FIELD_GET(FBNIC_BD_DESC_ID_MASK, bd), + FIELD_GET(FBNIC_BD_DESC_ADDR_MASK, bd)); + } + + return 0; +} + +static void fbnic_dbg_rcd_desc_seq_show(struct seq_file *s, int i) +{ + struct fbnic_ring *ring = s->private; + u64 rcd = le64_to_cpu(ring->desc[i]); + + switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { + case FBNIC_RCD_TYPE_HDR_AL: + case FBNIC_RCD_TYPE_PAY_AL: + seq_printf(s, "%04x %llx %llx %llx %#06llx %#06llx %#06llx\n", + i, FIELD_GET(FBNIC_RCD_DONE, rcd), + FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd), + FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd), + FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd), + FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd), + FIELD_GET(FBNIC_RCD_AL_BUFF_ID_MASK, rcd)); + break; + case FBNIC_RCD_TYPE_OPT_META: + seq_printf(s, "%04x %llx %llx %llx %llx %llx %#06llx %#012llx\n", + i, FIELD_GET(FBNIC_RCD_DONE, rcd), + FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd), + FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd), + FIELD_GET(FBNIC_RCD_OPT_META_TS, rcd), + FIELD_GET(FBNIC_RCD_OPT_META_ACTION, rcd), + FIELD_GET(FBNIC_RCD_OPT_META_ACTION_MASK, rcd), + FIELD_GET(FBNIC_RCD_OPT_META_TS_MASK, rcd)); + break; + case FBNIC_RCD_TYPE_META: + seq_printf(s, "%04x %llx %llx %llx %llx %llx %llx %llx %llx %llx %#06llx %#010llx\n", + i, FIELD_GET(FBNIC_RCD_DONE, rcd), + FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd), + FIELD_GET(FBNIC_RCD_META_ECN, rcd), + FIELD_GET(FBNIC_RCD_META_L4_CSUM_UNNECESSARY, rcd), + FIELD_GET(FBNIC_RCD_META_ERR_MAC_EOP, rcd), + FIELD_GET(FBNIC_RCD_META_ERR_TRUNCATED_FRAME, rcd), + FIELD_GET(FBNIC_RCD_META_ERR_PARSER, rcd), + FIELD_GET(FBNIC_RCD_META_L4_TYPE_MASK, rcd), + FIELD_GET(FBNIC_RCD_META_L3_TYPE_MASK, rcd), + FIELD_GET(FBNIC_RCD_META_L2_CSUM_MASK, rcd), + FIELD_GET(FBNIC_RCD_META_RSS_HASH_MASK, rcd)); + break; + } +} + +static int fbnic_dbg_rcq_desc_seq_show(struct seq_file *s, void *v) +{ + struct fbnic_ring *ring = s->private; + char hdr[80]; + int i; + + /* Generate header on first entry */ + fbnic_dbg_ring_show(s); + snprintf(hdr, sizeof(hdr), + "%18s %s %s\n", "OFFSET/", "L", "L"); + seq_puts(s, hdr); + snprintf(hdr, sizeof(hdr), + "%4s %s %s %s %s %s %s %s %s %s %-8s %s\n", + "DESC", "D", "T", "F", "C", "M", "T", "P", "4", "3", "LEN/CSUM", "ID/TS/RSS"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + /* Display descriptor */ + if (!ring->desc) { + seq_puts(s, "Descriptor ring not allocated.\n"); + return 0; + } + + for (i = 0; i <= ring->size_mask; i++) + fbnic_dbg_rcd_desc_seq_show(s, i); + + return 0; +} + +static int fbnic_dbg_desc_open(struct inode *inode, struct file *file) +{ + struct fbnic_ring *ring = inode->i_private; + int (*show)(struct seq_file *s, void *v); + + switch (ring->doorbell - fbnic_ring_csr_base(ring)) { + case FBNIC_QUEUE_TWQ0_TAIL: + case FBNIC_QUEUE_TWQ1_TAIL: + show = fbnic_dbg_twq_desc_seq_show; + break; + case FBNIC_QUEUE_TCQ_HEAD: + show = fbnic_dbg_tcq_desc_seq_show; + break; + case FBNIC_QUEUE_BDQ_HPQ_TAIL: + case FBNIC_QUEUE_BDQ_PPQ_TAIL: + show = fbnic_dbg_bdq_desc_seq_show; + break; + case FBNIC_QUEUE_RCQ_HEAD: + show = fbnic_dbg_rcq_desc_seq_show; + break; + default: + return -EINVAL; + } + + return single_open(file, show, ring); +} + +static const struct file_operations fbnic_dbg_desc_fops = { + .owner = THIS_MODULE, + .open = fbnic_dbg_desc_open, + .read = seq_read, + .llseek = seq_lseek, + .release = single_release, +}; + +void fbnic_dbg_nv_init(struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + char name[16]; + int i, j; + + /* Generate a folder for each napi vector */ + snprintf(name, sizeof(name), "nv.%03d", nv->v_idx); + + nv->dbg_nv = debugfs_create_dir(name, fbd->dbg_fbd); + + /* Generate a file for each Tx ring in the napi vector */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + unsigned int hw_idx; + + hw_idx = fbnic_ring_csr_base(&qt->cmpl) - + &fbd->uc_addr0[FBNIC_QUEUE(0)]; + hw_idx /= FBNIC_QUEUE_STRIDE; + + snprintf(name, sizeof(name), "twq0.%03d", hw_idx); + debugfs_create_file(name, 0400, nv->dbg_nv, &qt->sub0, + &fbnic_dbg_desc_fops); + + snprintf(name, sizeof(name), "twq1.%03d", hw_idx); + debugfs_create_file(name, 0400, nv->dbg_nv, &qt->sub1, + &fbnic_dbg_desc_fops); + + snprintf(name, sizeof(name), "tcq.%03d", hw_idx); + debugfs_create_file(name, 0400, nv->dbg_nv, &qt->cmpl, + &fbnic_dbg_desc_fops); + } + + /* Generate a file for each Rx ring in the napi vector */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + unsigned int hw_idx; + + hw_idx = fbnic_ring_csr_base(&qt->cmpl) - + &fbd->uc_addr0[FBNIC_QUEUE(0)]; + hw_idx /= FBNIC_QUEUE_STRIDE; + + snprintf(name, sizeof(name), "hpq.%03d", hw_idx); + debugfs_create_file(name, 0400, nv->dbg_nv, &qt->sub0, + &fbnic_dbg_desc_fops); + + snprintf(name, sizeof(name), "ppq.%03d", hw_idx); + debugfs_create_file(name, 0400, nv->dbg_nv, &qt->sub1, + &fbnic_dbg_desc_fops); + + snprintf(name, sizeof(name), "rcq.%03d", hw_idx); + debugfs_create_file(name, 0400, nv->dbg_nv, &qt->cmpl, + &fbnic_dbg_desc_fops); + } +} + +void fbnic_dbg_nv_exit(struct fbnic_napi_vector *nv) +{ + debugfs_remove_recursive(nv->dbg_nv); + nv->dbg_nv = NULL; +} + +static int fbnic_dbg_mac_addr_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + char hdr[80]; + int i; + + /* Generate Header */ + snprintf(hdr, sizeof(hdr), "%3s %s %-17s %s\n", + "Idx", "S", "TCAM Bitmap", "Addr/Mask"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + for (i = 0; i < FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES; i++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + seq_printf(s, "%02d %d %64pb %pm\n", + i, mac_addr->state, mac_addr->act_tcam, + mac_addr->value.addr8); + seq_printf(s, " %pm\n", + mac_addr->mask.addr8); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_mac_addr); + +static int fbnic_dbg_tce_tcam_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + int i, tcam_idx = 0; + char hdr[80]; + + /* Generate Header */ + snprintf(hdr, sizeof(hdr), "%3s %s %-17s %s\n", + "Idx", "S", "TCAM Bitmap", "Addr/Mask"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + for (i = 0; i < ARRAY_SIZE(fbd->mac_addr); i++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + /* Verify BMC bit is set */ + if (!test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) + continue; + + if (tcam_idx == FBNIC_TCE_TCAM_NUM_ENTRIES) + break; + + seq_printf(s, "%02d %d %64pb %pm\n", + tcam_idx, mac_addr->state, mac_addr->act_tcam, + mac_addr->value.addr8); + seq_printf(s, " %pm\n", + mac_addr->mask.addr8); + tcam_idx++; + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_tce_tcam); + +static int fbnic_dbg_act_tcam_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + char hdr[80]; + int i; + + /* Generate Header */ + snprintf(hdr, sizeof(hdr), "%3s %s %-55s %-4s %s\n", + "Idx", "S", "Value/Mask", "RSS", "Dest"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + for (i = 0; i < FBNIC_RPC_TCAM_ACT_NUM_ENTRIES; i++) { + struct fbnic_act_tcam *act_tcam = &fbd->act_tcam[i]; + + seq_printf(s, "%02d %d %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %08x\n", + i, act_tcam->state, + act_tcam->value.tcam[10], act_tcam->value.tcam[9], + act_tcam->value.tcam[8], act_tcam->value.tcam[7], + act_tcam->value.tcam[6], act_tcam->value.tcam[5], + act_tcam->value.tcam[4], act_tcam->value.tcam[3], + act_tcam->value.tcam[2], act_tcam->value.tcam[1], + act_tcam->value.tcam[0], act_tcam->rss_en_mask, + act_tcam->dest); + seq_printf(s, " %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x %04x\n", + act_tcam->mask.tcam[10], act_tcam->mask.tcam[9], + act_tcam->mask.tcam[8], act_tcam->mask.tcam[7], + act_tcam->mask.tcam[6], act_tcam->mask.tcam[5], + act_tcam->mask.tcam[4], act_tcam->mask.tcam[3], + act_tcam->mask.tcam[2], act_tcam->mask.tcam[1], + act_tcam->mask.tcam[0]); + } + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_act_tcam); + +static int fbnic_dbg_ip_addr_show(struct seq_file *s, + struct fbnic_ip_addr *ip_addr) +{ + char hdr[80]; + int i; + + /* Generate Header */ + snprintf(hdr, sizeof(hdr), "%3s %s %-17s %s %s\n", + "Idx", "S", "TCAM Bitmap", "V", "Addr/Mask"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + for (i = 0; i < FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES; i++, ip_addr++) { + seq_printf(s, "%02d %d %64pb %d %pi6\n", + i, ip_addr->state, ip_addr->act_tcam, + ip_addr->version, &ip_addr->value); + seq_printf(s, " %pi6\n", + &ip_addr->mask); + } + + return 0; +} + +static int fbnic_dbg_ip_src_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + return fbnic_dbg_ip_addr_show(s, fbd->ip_src); +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_ip_src); + +static int fbnic_dbg_ip_dst_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + return fbnic_dbg_ip_addr_show(s, fbd->ip_dst); +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_ip_dst); + +static int fbnic_dbg_ipo_src_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + return fbnic_dbg_ip_addr_show(s, fbd->ipo_src); +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_ipo_src); + +static int fbnic_dbg_ipo_dst_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + return fbnic_dbg_ip_addr_show(s, fbd->ipo_dst); +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_ipo_dst); + +static void fbnic_dbg_fw_mbx_display(struct seq_file *s, + struct fbnic_dev *fbd, int mbx_idx) +{ + struct fbnic_fw_mbx *mbx = &fbd->mbx[mbx_idx]; + char hdr[80]; + int i; + + /* Generate header */ + seq_puts(s, mbx_idx == FBNIC_IPC_MBX_RX_IDX ? "Rx\n" : "Tx\n"); + + seq_printf(s, "Rdy: %d Head: %d Tail: %d\n", + mbx->ready, mbx->head, mbx->tail); + + snprintf(hdr, sizeof(hdr), "%3s %-4s %s %-12s %s %-3s %-16s\n", + "Idx", "Len", "E", "Addr", "F", "H", "Raw"); + seq_puts(s, hdr); + fbnic_dbg_desc_break(s, strnlen(hdr, sizeof(hdr))); + + for (i = 0; i < FBNIC_IPC_MBX_DESC_LEN; i++) { + u64 desc = __fbnic_mbx_rd_desc(fbd, mbx_idx, i); + + seq_printf(s, "%-3.2d %04lld %d %012llx %d %-3d %016llx\n", + i, FIELD_GET(FBNIC_IPC_MBX_DESC_LEN_MASK, desc), + !!(desc & FBNIC_IPC_MBX_DESC_EOM), + desc & FBNIC_IPC_MBX_DESC_ADDR_MASK, + !!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL), + !!(desc & FBNIC_IPC_MBX_DESC_HOST_CMPL), + desc); + } +} + +static int fbnic_dbg_fw_mbx_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + fbnic_dbg_fw_mbx_display(s, fbd, FBNIC_IPC_MBX_RX_IDX); + + /* Add blank line between Rx and Tx */ + seq_puts(s, "\n"); + + fbnic_dbg_fw_mbx_display(s, fbd, FBNIC_IPC_MBX_TX_IDX); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_fw_mbx); + +static int fbnic_dbg_fw_log_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + struct fbnic_fw_log_entry *entry; + unsigned long flags; + + if (!fbnic_fw_log_ready(fbd)) + return -ENXIO; + + spin_lock_irqsave(&fbd->fw_log.lock, flags); + + list_for_each_entry_reverse(entry, &fbd->fw_log.entries, list) { + seq_printf(s, FBNIC_FW_LOG_FMT, entry->index, + (entry->timestamp / (MSEC_PER_SEC * 60 * 60 * 24)), + (entry->timestamp / (MSEC_PER_SEC * 60 * 60)) % 24, + ((entry->timestamp / (MSEC_PER_SEC * 60) % 60)), + ((entry->timestamp / MSEC_PER_SEC) % 60), + (entry->timestamp % MSEC_PER_SEC), + entry->msg); + } + + spin_unlock_irqrestore(&fbd->fw_log.lock, flags); + + return 0; +} +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_fw_log); + +static int fbnic_dbg_pcie_stats_show(struct seq_file *s, void *v) +{ + struct fbnic_dev *fbd = s->private; + + rtnl_lock(); + fbnic_get_hw_stats(fbd); + + seq_printf(s, "ob_rd_tlp: %llu\n", fbd->hw_stats.pcie.ob_rd_tlp.value); + seq_printf(s, "ob_rd_dword: %llu\n", + fbd->hw_stats.pcie.ob_rd_dword.value); + seq_printf(s, "ob_wr_tlp: %llu\n", fbd->hw_stats.pcie.ob_wr_tlp.value); + seq_printf(s, "ob_wr_dword: %llu\n", + fbd->hw_stats.pcie.ob_wr_dword.value); + seq_printf(s, "ob_cpl_tlp: %llu\n", + fbd->hw_stats.pcie.ob_cpl_tlp.value); + seq_printf(s, "ob_cpl_dword: %llu\n", + fbd->hw_stats.pcie.ob_cpl_dword.value); + seq_printf(s, "ob_rd_no_tag: %llu\n", + fbd->hw_stats.pcie.ob_rd_no_tag.value); + seq_printf(s, "ob_rd_no_cpl_cred: %llu\n", + fbd->hw_stats.pcie.ob_rd_no_cpl_cred.value); + seq_printf(s, "ob_rd_no_np_cred: %llu\n", + fbd->hw_stats.pcie.ob_rd_no_np_cred.value); + rtnl_unlock(); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(fbnic_dbg_pcie_stats); + +void fbnic_dbg_fbd_init(struct fbnic_dev *fbd) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + const char *name = pci_name(pdev); + + fbd->dbg_fbd = debugfs_create_dir(name, fbnic_dbg_root); + debugfs_create_file("pcie_stats", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_pcie_stats_fops); + debugfs_create_file("mac_addr", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_mac_addr_fops); + debugfs_create_file("tce_tcam", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_tce_tcam_fops); + debugfs_create_file("act_tcam", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_act_tcam_fops); + debugfs_create_file("ip_src", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_ip_src_fops); + debugfs_create_file("ip_dst", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_ip_dst_fops); + debugfs_create_file("ipo_src", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_ipo_src_fops); + debugfs_create_file("ipo_dst", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_ipo_dst_fops); + debugfs_create_file("fw_mbx", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_fw_mbx_fops); + debugfs_create_file("fw_log", 0400, fbd->dbg_fbd, fbd, + &fbnic_dbg_fw_log_fops); +} + +void fbnic_dbg_fbd_exit(struct fbnic_dev *fbd) +{ + debugfs_remove_recursive(fbd->dbg_fbd); + fbd->dbg_fbd = NULL; +} + +void fbnic_dbg_init(void) +{ + fbnic_dbg_root = debugfs_create_dir(fbnic_driver_name, NULL); +} + +void fbnic_dbg_exit(void) +{ + debugfs_remove_recursive(fbnic_dbg_root); + fbnic_dbg_root = NULL; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c new file mode 100644 index 000000000..546e1c12d --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_devlink.c @@ -0,0 +1,669 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/unaligned.h> +#include <linux/pci.h> +#include <linux/pldmfw.h> +#include <linux/types.h> +#include <net/devlink.h> + +#include "fbnic.h" +#include "fbnic_fw.h" +#include "fbnic_tlv.h" + +#define FBNIC_SN_STR_LEN 24 + +static int fbnic_version_running_put(struct devlink_info_req *req, + struct fbnic_fw_ver *fw_ver, + char *ver_name) +{ + char running_ver[FBNIC_FW_VER_MAX_SIZE]; + int err; + + fbnic_mk_fw_ver_str(fw_ver->version, running_ver); + err = devlink_info_version_running_put(req, ver_name, running_ver); + if (err) + return err; + + if (strlen(fw_ver->commit) > 0) { + char commit_name[FBNIC_SN_STR_LEN]; + + snprintf(commit_name, FBNIC_SN_STR_LEN, "%s.commit", ver_name); + err = devlink_info_version_running_put(req, commit_name, + fw_ver->commit); + if (err) + return err; + } + + return 0; +} + +static int fbnic_version_stored_put(struct devlink_info_req *req, + struct fbnic_fw_ver *fw_ver, + char *ver_name) +{ + char stored_ver[FBNIC_FW_VER_MAX_SIZE]; + int err; + + fbnic_mk_fw_ver_str(fw_ver->version, stored_ver); + err = devlink_info_version_stored_put(req, ver_name, stored_ver); + if (err) + return err; + + if (strlen(fw_ver->commit) > 0) { + char commit_name[FBNIC_SN_STR_LEN]; + + snprintf(commit_name, FBNIC_SN_STR_LEN, "%s.commit", ver_name); + err = devlink_info_version_stored_put(req, commit_name, + fw_ver->commit); + if (err) + return err; + } + + return 0; +} + +static int fbnic_devlink_info_get(struct devlink *devlink, + struct devlink_info_req *req, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_priv(devlink); + int err; + + err = fbnic_version_running_put(req, &fbd->fw_cap.running.mgmt, + DEVLINK_INFO_VERSION_GENERIC_FW); + if (err) + return err; + + err = fbnic_version_running_put(req, &fbd->fw_cap.running.bootloader, + DEVLINK_INFO_VERSION_GENERIC_FW_BOOTLOADER); + if (err) + return err; + + err = fbnic_version_stored_put(req, &fbd->fw_cap.stored.mgmt, + DEVLINK_INFO_VERSION_GENERIC_FW); + if (err) + return err; + + err = fbnic_version_stored_put(req, &fbd->fw_cap.stored.bootloader, + DEVLINK_INFO_VERSION_GENERIC_FW_BOOTLOADER); + if (err) + return err; + + err = fbnic_version_stored_put(req, &fbd->fw_cap.stored.undi, + DEVLINK_INFO_VERSION_GENERIC_FW_UNDI); + if (err) + return err; + + if (fbd->dsn) { + unsigned char serial[FBNIC_SN_STR_LEN]; + u8 dsn[8]; + + put_unaligned_be64(fbd->dsn, dsn); + err = snprintf(serial, FBNIC_SN_STR_LEN, "%8phD", dsn); + if (err < 0) + return err; + + err = devlink_info_serial_number_put(req, serial); + if (err) + return err; + } + + return 0; +} + +static bool +fbnic_pldm_match_record(struct pldmfw *context, struct pldmfw_record *record) +{ + struct pldmfw_desc_tlv *desc; + u32 anti_rollback_ver = 0; + struct devlink *devlink; + struct fbnic_dev *fbd; + struct pci_dev *pdev; + + /* First, use the standard PCI matching function */ + if (!pldmfw_op_pci_match_record(context, record)) + return false; + + pdev = to_pci_dev(context->dev); + fbd = pci_get_drvdata(pdev); + devlink = priv_to_devlink(fbd); + + /* If PCI match is successful, check for vendor-specific descriptors */ + list_for_each_entry(desc, &record->descs, entry) { + if (desc->type != PLDM_DESC_ID_VENDOR_DEFINED) + continue; + + if (desc->size < 21 || desc->data[0] != 1 || + desc->data[1] != 15) + continue; + + if (memcmp(desc->data + 2, "AntiRollbackVer", 15) != 0) + continue; + + anti_rollback_ver = get_unaligned_le32(desc->data + 17); + break; + } + + /* Compare versions and return error if they do not match */ + if (anti_rollback_ver < fbd->fw_cap.anti_rollback_version) { + char buf[128]; + + snprintf(buf, sizeof(buf), + "New firmware anti-rollback version (0x%x) is older than device version (0x%x)!", + anti_rollback_ver, fbd->fw_cap.anti_rollback_version); + devlink_flash_update_status_notify(devlink, buf, + "Anti-Rollback", 0, 0); + + return false; + } + + return true; +} + +static int +fbnic_flash_start(struct fbnic_dev *fbd, struct pldmfw_component *component) +{ + struct fbnic_fw_completion *cmpl; + int err; + + cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_FW_START_UPGRADE_REQ); + if (!cmpl) + return -ENOMEM; + + err = fbnic_fw_xmit_fw_start_upgrade(fbd, cmpl, + component->identifier, + component->component_size); + if (err) + goto cmpl_free; + + /* Wait for firmware to ack firmware upgrade start */ + if (fbnic_mbx_wait_for_cmpl(cmpl)) + err = cmpl->result; + else + err = -ETIMEDOUT; + + fbnic_mbx_clear_cmpl(fbd, cmpl); +cmpl_free: + fbnic_fw_put_cmpl(cmpl); + + return err; +} + +static int +fbnic_flash_component(struct pldmfw *context, + struct pldmfw_component *component) +{ + const u8 *data = component->component_data; + const u32 size = component->component_size; + struct fbnic_fw_completion *cmpl; + const char *component_name; + struct devlink *devlink; + struct fbnic_dev *fbd; + struct pci_dev *pdev; + u32 offset = 0; + u32 length = 0; + char buf[32]; + int err; + + pdev = to_pci_dev(context->dev); + fbd = pci_get_drvdata(pdev); + devlink = priv_to_devlink(fbd); + + switch (component->identifier) { + case QSPI_SECTION_CMRT: + component_name = "boot1"; + break; + case QSPI_SECTION_CONTROL_FW: + component_name = "boot2"; + break; + case QSPI_SECTION_OPTION_ROM: + component_name = "option-rom"; + break; + default: + snprintf(buf, sizeof(buf), "Unknown component ID %u!", + component->identifier); + devlink_flash_update_status_notify(devlink, buf, NULL, 0, + size); + return -EINVAL; + } + + /* Once firmware receives the request to start upgrading it responds + * with two messages: + * 1. An ACK that it received the message and possible error code + * indicating that an upgrade is not currently possible. + * 2. A request for the first chunk of data + * + * Setup completions for write before issuing the start message so + * the driver can catch both messages. + */ + cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_REQ); + if (!cmpl) + return -ENOMEM; + + err = fbnic_mbx_set_cmpl(fbd, cmpl); + if (err) + goto cmpl_free; + + devlink_flash_update_timeout_notify(devlink, "Initializing", + component_name, 15); + err = fbnic_flash_start(fbd, component); + if (err) + goto err_no_msg; + + while (offset < size) { + if (!fbnic_mbx_wait_for_cmpl(cmpl)) { + err = -ETIMEDOUT; + break; + } + + err = cmpl->result; + if (err) + break; + + /* Verify firmware is requesting the next chunk in the seq. */ + if (cmpl->u.fw_update.offset != offset + length) { + err = -EFAULT; + break; + } + + offset = cmpl->u.fw_update.offset; + length = cmpl->u.fw_update.length; + + if (length > TLV_MAX_DATA || offset + length > size) { + err = -EFAULT; + break; + } + + devlink_flash_update_status_notify(devlink, "Flashing", + component_name, + offset, size); + + /* Mailbox will set length to 0 once it receives the finish + * message. + */ + if (!length) + continue; + + reinit_completion(&cmpl->done); + err = fbnic_fw_xmit_fw_write_chunk(fbd, data, offset, length, + 0); + if (err) + break; + } + + if (err) { + fbnic_fw_xmit_fw_write_chunk(fbd, NULL, 0, 0, err); +err_no_msg: + snprintf(buf, sizeof(buf), "Mailbox encountered error %d!", + err); + devlink_flash_update_status_notify(devlink, buf, + component_name, 0, 0); + } + + fbnic_mbx_clear_cmpl(fbd, cmpl); +cmpl_free: + fbnic_fw_put_cmpl(cmpl); + + return err; +} + +static const struct pldmfw_ops fbnic_pldmfw_ops = { + .match_record = fbnic_pldm_match_record, + .flash_component = fbnic_flash_component, +}; + +static int +fbnic_devlink_flash_update(struct devlink *devlink, + struct devlink_flash_update_params *params, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_priv(devlink); + const struct firmware *fw = params->fw; + struct device *dev = fbd->dev; + struct pldmfw context; + char *err_msg; + int err; + + context.ops = &fbnic_pldmfw_ops; + context.dev = dev; + + err = pldmfw_flash_image(&context, fw); + if (err) { + switch (err) { + case -EINVAL: + err_msg = "Invalid image"; + break; + case -EOPNOTSUPP: + err_msg = "Unsupported image"; + break; + case -ENOMEM: + err_msg = "Out of memory"; + break; + case -EFAULT: + err_msg = "Invalid header"; + break; + case -ENOENT: + err_msg = "No matching record"; + break; + case -ENODEV: + err_msg = "No matching device"; + break; + case -ETIMEDOUT: + err_msg = "Timed out waiting for reply"; + break; + default: + err_msg = "Unknown error"; + break; + } + + NL_SET_ERR_MSG_FMT_MOD(extack, + "Failed to flash PLDM Image: %s (error: %d)", + err_msg, err); + } + + return err; +} + +static const struct devlink_ops fbnic_devlink_ops = { + .info_get = fbnic_devlink_info_get, + .flash_update = fbnic_devlink_flash_update, +}; + +static int fbnic_fw_reporter_dump(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, void *priv_ctx, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_health_reporter_priv(reporter); + u32 offset, index, index_count, length, size; + struct fbnic_fw_completion *fw_cmpl; + u8 *dump_data, **data; + int err; + + fw_cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_RESP); + if (!fw_cmpl) + return -ENOMEM; + + err = fbnic_fw_xmit_coredump_info_msg(fbd, fw_cmpl, true); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to transmit core dump info msg"); + goto cmpl_free; + } + if (!fbnic_mbx_wait_for_cmpl(fw_cmpl)) { + NL_SET_ERR_MSG_MOD(extack, + "Timed out waiting on core dump info"); + err = -ETIMEDOUT; + goto cmpl_cleanup; + } + + size = fw_cmpl->u.coredump_info.size; + err = fw_cmpl->result; + + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); + fbnic_fw_put_cmpl(fw_cmpl); + + /* Handle error returned by firmware */ + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware core dump returned error"); + return err; + } + if (!size) { + NL_SET_ERR_MSG_MOD(extack, + "Firmware core dump returned size 0"); + return -EIO; + } + + /* Read the dump, we can only transfer TLV_MAX_DATA at a time */ + index_count = DIV_ROUND_UP(size, TLV_MAX_DATA); + + fw_cmpl = __fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_COREDUMP_READ_RESP, + sizeof(void *) * index_count + size); + if (!fw_cmpl) + return -ENOMEM; + + /* Populate pointer table w/ pointer offsets */ + dump_data = (void *)&fw_cmpl->u.coredump.data[index_count]; + data = fw_cmpl->u.coredump.data; + fw_cmpl->u.coredump.size = size; + fw_cmpl->u.coredump.stride = TLV_MAX_DATA; + + for (index = 0; index < index_count; index++) { + /* First iteration installs completion */ + struct fbnic_fw_completion *cmpl_arg = index ? NULL : fw_cmpl; + + offset = index * TLV_MAX_DATA; + length = min(size - offset, TLV_MAX_DATA); + + data[index] = dump_data + offset; + err = fbnic_fw_xmit_coredump_read_msg(fbd, cmpl_arg, + offset, length); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to transmit core dump msg"); + if (cmpl_arg) + goto cmpl_free; + else + goto cmpl_cleanup; + } + + if (fbnic_mbx_wait_for_cmpl(fw_cmpl)) { + reinit_completion(&fw_cmpl->done); + } else { + NL_SET_ERR_MSG_FMT_MOD(extack, + "Timed out waiting on core dump (%d/%d)", + index + 1, index_count); + err = -ETIMEDOUT; + goto cmpl_cleanup; + } + + /* If we didn't see the reply record as incomplete */ + if (fw_cmpl->u.coredump.data[index]) { + NL_SET_ERR_MSG_FMT_MOD(extack, + "No data for core dump chunk (%d/%d)", + index + 1, index_count); + err = -EIO; + goto cmpl_cleanup; + } + } + + devlink_fmsg_binary_pair_nest_start(fmsg, "FW coredump"); + + for (offset = 0; offset < size; offset += length) { + length = min_t(u32, size - offset, TLV_MAX_DATA); + + devlink_fmsg_binary_put(fmsg, dump_data + offset, length); + } + + devlink_fmsg_binary_pair_nest_end(fmsg); + +cmpl_cleanup: + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); +cmpl_free: + fbnic_fw_put_cmpl(fw_cmpl); + + return err; +} + +static int +fbnic_fw_reporter_diagnose(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_health_reporter_priv(reporter); + u32 sec, msec; + + /* Device is most likely down, we're not exchanging heartbeats */ + if (!fbd->prev_firmware_time) + return 0; + + sec = div_u64_rem(fbd->firmware_time, MSEC_PER_SEC, &msec); + + devlink_fmsg_pair_nest_start(fmsg, "last_heartbeat"); + devlink_fmsg_obj_nest_start(fmsg); + devlink_fmsg_pair_nest_start(fmsg, "fw_uptime"); + devlink_fmsg_obj_nest_start(fmsg); + devlink_fmsg_u32_pair_put(fmsg, "sec", sec); + devlink_fmsg_u32_pair_put(fmsg, "msec", msec); + devlink_fmsg_obj_nest_end(fmsg); + devlink_fmsg_pair_nest_end(fmsg); + devlink_fmsg_obj_nest_end(fmsg); + devlink_fmsg_pair_nest_end(fmsg); + + return 0; +} + +void __printf(2, 3) +fbnic_devlink_fw_report(struct fbnic_dev *fbd, const char *format, ...) +{ + char msg[FBNIC_FW_LOG_MAX_SIZE]; + va_list args; + + va_start(args, format); + vsnprintf(msg, FBNIC_FW_LOG_MAX_SIZE, format, args); + va_end(args); + + devlink_health_report(fbd->fw_reporter, msg, fbd); + if (fbnic_fw_log_ready(fbd)) + fbnic_fw_log_write(fbd, 0, fbd->firmware_time, msg); +} + +static const struct devlink_health_reporter_ops fbnic_fw_ops = { + .name = "fw", + .dump = fbnic_fw_reporter_dump, + .diagnose = fbnic_fw_reporter_diagnose, +}; + +static u32 fbnic_read_otp_status(struct fbnic_dev *fbd) +{ + return fbnic_fw_rd32(fbd, FBNIC_NS_OTP_STATUS); +} + +static int +fbnic_otp_reporter_dump(struct devlink_health_reporter *reporter, + struct devlink_fmsg *fmsg, void *priv_ctx, + struct netlink_ext_ack *extack) +{ + struct fbnic_dev *fbd = devlink_health_reporter_priv(reporter); + u32 otp_status, otp_write_status, m; + + otp_status = fbnic_read_otp_status(fbd); + otp_write_status = fbnic_fw_rd32(fbd, FBNIC_NS_OTP_WRITE_STATUS); + + /* Dump OTP status */ + devlink_fmsg_pair_nest_start(fmsg, "OTP"); + devlink_fmsg_obj_nest_start(fmsg); + + devlink_fmsg_u32_pair_put(fmsg, "Status", otp_status); + + /* Extract OTP Write Data status */ + m = FBNIC_NS_OTP_WRITE_DATA_STATUS_MASK; + devlink_fmsg_u32_pair_put(fmsg, "Data", + FIELD_GET(m, otp_write_status)); + + /* Extract OTP Write ECC status */ + m = FBNIC_NS_OTP_WRITE_ECC_STATUS_MASK; + devlink_fmsg_u32_pair_put(fmsg, "ECC", + FIELD_GET(m, otp_write_status)); + + devlink_fmsg_obj_nest_end(fmsg); + devlink_fmsg_pair_nest_end(fmsg); + + return 0; +} + +void fbnic_devlink_otp_check(struct fbnic_dev *fbd, const char *msg) +{ + /* Check if there is anything to report */ + if (!fbnic_read_otp_status(fbd)) + return; + + devlink_health_report(fbd->otp_reporter, msg, fbd); + if (fbnic_fw_log_ready(fbd)) + fbnic_fw_log_write(fbd, 0, fbd->firmware_time, msg); +} + +static const struct devlink_health_reporter_ops fbnic_otp_ops = { + .name = "otp", + .dump = fbnic_otp_reporter_dump, +}; + +int fbnic_devlink_health_create(struct fbnic_dev *fbd) +{ + fbd->fw_reporter = devlink_health_reporter_create(priv_to_devlink(fbd), + &fbnic_fw_ops, fbd); + if (IS_ERR(fbd->fw_reporter)) { + dev_warn(fbd->dev, + "Failed to create FW fault reporter: %pe\n", + fbd->fw_reporter); + return PTR_ERR(fbd->fw_reporter); + } + + fbd->otp_reporter = devlink_health_reporter_create(priv_to_devlink(fbd), + &fbnic_otp_ops, fbd); + if (IS_ERR(fbd->otp_reporter)) { + devlink_health_reporter_destroy(fbd->fw_reporter); + dev_warn(fbd->dev, + "Failed to create OTP fault reporter: %pe\n", + fbd->otp_reporter); + return PTR_ERR(fbd->otp_reporter); + } + + return 0; +} + +void fbnic_devlink_health_destroy(struct fbnic_dev *fbd) +{ + devlink_health_reporter_destroy(fbd->otp_reporter); + devlink_health_reporter_destroy(fbd->fw_reporter); +} + +void fbnic_devlink_free(struct fbnic_dev *fbd) +{ + struct devlink *devlink = priv_to_devlink(fbd); + + devlink_free(devlink); +} + +struct fbnic_dev *fbnic_devlink_alloc(struct pci_dev *pdev) +{ + void __iomem * const *iomap_table; + struct devlink *devlink; + struct fbnic_dev *fbd; + + devlink = devlink_alloc(&fbnic_devlink_ops, sizeof(struct fbnic_dev), + &pdev->dev); + if (!devlink) + return NULL; + + fbd = devlink_priv(devlink); + pci_set_drvdata(pdev, fbd); + fbd->dev = &pdev->dev; + + iomap_table = pcim_iomap_table(pdev); + fbd->uc_addr0 = iomap_table[0]; + fbd->uc_addr4 = iomap_table[4]; + + fbd->dsn = pci_get_dsn(pdev); + fbd->mps = pcie_get_mps(pdev); + fbd->readrq = pcie_get_readrq(pdev); + fbd->relaxed_ord = pcie_relaxed_ordering_enabled(pdev); + + fbd->mac_addr_boundary = FBNIC_RPC_TCAM_MACDA_DEFAULT_BOUNDARY; + + return fbd; +} + +void fbnic_devlink_register(struct fbnic_dev *fbd) +{ + struct devlink *devlink = priv_to_devlink(fbd); + + devlink_register(devlink); +} + +void fbnic_devlink_unregister(struct fbnic_dev *fbd) +{ + struct devlink *devlink = priv_to_devlink(fbd); + + devlink_unregister(devlink); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_drvinfo.h b/drivers/net/ethernet/meta/fbnic/fbnic_drvinfo.h new file mode 100644 index 000000000..809ba6729 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_drvinfo.h @@ -0,0 +1,5 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#define DRV_NAME "fbnic" +#define DRV_SUMMARY "Meta(R) Host Network Interface Driver" diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c new file mode 100644 index 000000000..0e47088ec --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_ethtool.c @@ -0,0 +1,2076 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/ethtool.h> +#include <linux/ethtool_netlink.h> +#include <linux/netdevice.h> +#include <linux/pci.h> +#include <net/ipv6.h> + +#include "fbnic.h" +#include "fbnic_netdev.h" +#include "fbnic_tlv.h" + +struct fbnic_stat { + u8 string[ETH_GSTRING_LEN]; + unsigned int size; + unsigned int offset; +}; + +#define FBNIC_STAT_FIELDS(type, name, stat) { \ + .string = name, \ + .size = sizeof_field(struct type, stat), \ + .offset = offsetof(struct type, stat), \ +} + +/* Hardware statistics not captured in rtnl_link_stats */ +#define FBNIC_HW_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_hw_stats, name, stat) + +static const struct fbnic_stat fbnic_gstrings_hw_stats[] = { + /* TTI */ + FBNIC_HW_STAT("tti_cm_drop_frames", tti.cm_drop.frames), + FBNIC_HW_STAT("tti_cm_drop_bytes", tti.cm_drop.bytes), + FBNIC_HW_STAT("tti_frame_drop_frames", tti.frame_drop.frames), + FBNIC_HW_STAT("tti_frame_drop_bytes", tti.frame_drop.bytes), + FBNIC_HW_STAT("tti_tbi_drop_frames", tti.tbi_drop.frames), + FBNIC_HW_STAT("tti_tbi_drop_bytes", tti.tbi_drop.bytes), + + /* TMI */ + FBNIC_HW_STAT("ptp_illegal_req", tmi.ptp_illegal_req), + FBNIC_HW_STAT("ptp_good_ts", tmi.ptp_good_ts), + FBNIC_HW_STAT("ptp_bad_ts", tmi.ptp_bad_ts), + + /* RPC */ + FBNIC_HW_STAT("rpc_unkn_etype", rpc.unkn_etype), + FBNIC_HW_STAT("rpc_unkn_ext_hdr", rpc.unkn_ext_hdr), + FBNIC_HW_STAT("rpc_ipv4_frag", rpc.ipv4_frag), + FBNIC_HW_STAT("rpc_ipv6_frag", rpc.ipv6_frag), + FBNIC_HW_STAT("rpc_ipv4_esp", rpc.ipv4_esp), + FBNIC_HW_STAT("rpc_ipv6_esp", rpc.ipv6_esp), + FBNIC_HW_STAT("rpc_tcp_opt_err", rpc.tcp_opt_err), + FBNIC_HW_STAT("rpc_out_of_hdr_err", rpc.out_of_hdr_err), +}; + +#define FBNIC_HW_FIXED_STATS_LEN ARRAY_SIZE(fbnic_gstrings_hw_stats) + +#define FBNIC_RXB_ENQUEUE_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_rxb_enqueue_stats, name, stat) + +static const struct fbnic_stat fbnic_gstrings_rxb_enqueue_stats[] = { + FBNIC_RXB_ENQUEUE_STAT("rxb_integrity_err%u", integrity_err), + FBNIC_RXB_ENQUEUE_STAT("rxb_mac_err%u", mac_err), + FBNIC_RXB_ENQUEUE_STAT("rxb_parser_err%u", parser_err), + FBNIC_RXB_ENQUEUE_STAT("rxb_frm_err%u", frm_err), + + FBNIC_RXB_ENQUEUE_STAT("rxb_drbo%u_frames", drbo.frames), + FBNIC_RXB_ENQUEUE_STAT("rxb_drbo%u_bytes", drbo.bytes), +}; + +#define FBNIC_HW_RXB_ENQUEUE_STATS_LEN \ + ARRAY_SIZE(fbnic_gstrings_rxb_enqueue_stats) + +#define FBNIC_RXB_FIFO_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_rxb_fifo_stats, name, stat) + +static const struct fbnic_stat fbnic_gstrings_rxb_fifo_stats[] = { + FBNIC_RXB_FIFO_STAT("rxb_fifo%u_drop", trans_drop), + FBNIC_RXB_FIFO_STAT("rxb_fifo%u_dropped_frames", drop.frames), + FBNIC_RXB_FIFO_STAT("rxb_fifo%u_ecn", trans_ecn), + FBNIC_RXB_FIFO_STAT("rxb_fifo%u_level", level), +}; + +#define FBNIC_HW_RXB_FIFO_STATS_LEN ARRAY_SIZE(fbnic_gstrings_rxb_fifo_stats) + +#define FBNIC_RXB_DEQUEUE_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_rxb_dequeue_stats, name, stat) + +static const struct fbnic_stat fbnic_gstrings_rxb_dequeue_stats[] = { + FBNIC_RXB_DEQUEUE_STAT("rxb_intf%u_frames", intf.frames), + FBNIC_RXB_DEQUEUE_STAT("rxb_intf%u_bytes", intf.bytes), + FBNIC_RXB_DEQUEUE_STAT("rxb_pbuf%u_frames", pbuf.frames), + FBNIC_RXB_DEQUEUE_STAT("rxb_pbuf%u_bytes", pbuf.bytes), +}; + +#define FBNIC_HW_RXB_DEQUEUE_STATS_LEN \ + ARRAY_SIZE(fbnic_gstrings_rxb_dequeue_stats) + +#define FBNIC_HW_Q_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_hw_q_stats, name, stat.value) + +static const struct fbnic_stat fbnic_gstrings_hw_q_stats[] = { + FBNIC_HW_Q_STAT("rde_%u_pkt_err", rde_pkt_err), + FBNIC_HW_Q_STAT("rde_%u_pkt_cq_drop", rde_pkt_cq_drop), + FBNIC_HW_Q_STAT("rde_%u_pkt_bdq_drop", rde_pkt_bdq_drop), +}; + +#define FBNIC_HW_Q_STATS_LEN ARRAY_SIZE(fbnic_gstrings_hw_q_stats) +#define FBNIC_HW_STATS_LEN \ + (FBNIC_HW_FIXED_STATS_LEN + \ + FBNIC_HW_RXB_ENQUEUE_STATS_LEN * FBNIC_RXB_ENQUEUE_INDICES + \ + FBNIC_HW_RXB_FIFO_STATS_LEN * FBNIC_RXB_FIFO_INDICES + \ + FBNIC_HW_RXB_DEQUEUE_STATS_LEN * FBNIC_RXB_DEQUEUE_INDICES + \ + FBNIC_HW_Q_STATS_LEN * FBNIC_MAX_QUEUES) + +#define FBNIC_QUEUE_STAT(name, stat) \ + FBNIC_STAT_FIELDS(fbnic_ring, name, stat) + +static const struct fbnic_stat fbnic_gstrings_xdp_stats[] = { + FBNIC_QUEUE_STAT("xdp_tx_queue_%u_packets", stats.packets), + FBNIC_QUEUE_STAT("xdp_tx_queue_%u_bytes", stats.bytes), + FBNIC_QUEUE_STAT("xdp_tx_queue_%u_dropped", stats.dropped), +}; + +#define FBNIC_XDP_STATS_LEN ARRAY_SIZE(fbnic_gstrings_xdp_stats) + +#define FBNIC_STATS_LEN \ + (FBNIC_HW_STATS_LEN + FBNIC_XDP_STATS_LEN * FBNIC_MAX_XDPQS) + +enum fbnic_self_test_results { + TEST_REG = 0, + TEST_MSIX, + TEST_MBX, +}; + +static const char fbnic_gstrings_self_test[][ETH_GSTRING_LEN] = { + [TEST_REG] = "Register test (offline)", + [TEST_MSIX] = "MSI-X Interrupt test (offline)", + [TEST_MBX] = "FW mailbox test (on/offline)", +}; + +#define FBNIC_TEST_LEN ARRAY_SIZE(fbnic_gstrings_self_test) + +static void +fbnic_get_drvinfo(struct net_device *netdev, struct ethtool_drvinfo *drvinfo) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + fbnic_get_fw_ver_commit_str(fbd, drvinfo->fw_version, + sizeof(drvinfo->fw_version)); +} + +static int fbnic_get_regs_len(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + return fbnic_csr_regs_len(fbn->fbd) * sizeof(u32); +} + +static void fbnic_get_regs(struct net_device *netdev, + struct ethtool_regs *regs, void *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + fbnic_csr_get_regs(fbn->fbd, data, ®s->version); +} + +static struct fbnic_net *fbnic_clone_create(struct fbnic_net *orig) +{ + struct fbnic_net *clone; + + clone = kmemdup(orig, sizeof(*orig), GFP_KERNEL); + if (!clone) + return NULL; + + memset(clone->tx, 0, sizeof(clone->tx)); + memset(clone->rx, 0, sizeof(clone->rx)); + memset(clone->napi, 0, sizeof(clone->napi)); + return clone; +} + +static void fbnic_clone_swap_cfg(struct fbnic_net *orig, + struct fbnic_net *clone) +{ + swap(clone->rcq_size, orig->rcq_size); + swap(clone->hpq_size, orig->hpq_size); + swap(clone->ppq_size, orig->ppq_size); + swap(clone->txq_size, orig->txq_size); + swap(clone->num_rx_queues, orig->num_rx_queues); + swap(clone->num_tx_queues, orig->num_tx_queues); + swap(clone->num_napi, orig->num_napi); + swap(clone->hds_thresh, orig->hds_thresh); +} + +static void fbnic_aggregate_vector_counters(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + int i, j; + + for (i = 0; i < nv->txt_count; i++) { + fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].sub0); + fbnic_aggregate_ring_xdp_counters(fbn, &nv->qt[i].sub1); + fbnic_aggregate_ring_tx_counters(fbn, &nv->qt[i].cmpl); + } + + for (j = 0; j < nv->rxt_count; j++, i++) { + fbnic_aggregate_ring_bdq_counters(fbn, &nv->qt[i].sub0); + fbnic_aggregate_ring_bdq_counters(fbn, &nv->qt[i].sub1); + fbnic_aggregate_ring_rx_counters(fbn, &nv->qt[i].cmpl); + } +} + +static void fbnic_clone_swap(struct fbnic_net *orig, + struct fbnic_net *clone) +{ + struct fbnic_dev *fbd = orig->fbd; + unsigned int i; + + for (i = 0; i < max(clone->num_napi, orig->num_napi); i++) + fbnic_synchronize_irq(fbd, FBNIC_NON_NAPI_VECTORS + i); + for (i = 0; i < orig->num_napi; i++) + fbnic_aggregate_vector_counters(orig, orig->napi[i]); + + fbnic_clone_swap_cfg(orig, clone); + + for (i = 0; i < ARRAY_SIZE(orig->napi); i++) + swap(clone->napi[i], orig->napi[i]); + for (i = 0; i < ARRAY_SIZE(orig->tx); i++) + swap(clone->tx[i], orig->tx[i]); + for (i = 0; i < ARRAY_SIZE(orig->rx); i++) + swap(clone->rx[i], orig->rx[i]); +} + +static void fbnic_clone_free(struct fbnic_net *clone) +{ + kfree(clone); +} + +static int fbnic_get_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + ec->tx_coalesce_usecs = fbn->tx_usecs; + ec->rx_coalesce_usecs = fbn->rx_usecs; + ec->rx_max_coalesced_frames = fbn->rx_max_frames; + + return 0; +} + +static int fbnic_set_coalesce(struct net_device *netdev, + struct ethtool_coalesce *ec, + struct kernel_ethtool_coalesce *kernel_coal, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + /* Verify against hardware limits */ + if (ec->rx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT)) { + NL_SET_ERR_MSG_MOD(extack, "rx_usecs is above device max"); + return -EINVAL; + } + if (ec->tx_coalesce_usecs > FIELD_MAX(FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT)) { + NL_SET_ERR_MSG_MOD(extack, "tx_usecs is above device max"); + return -EINVAL; + } + if (ec->rx_max_coalesced_frames > + FIELD_MAX(FBNIC_QUEUE_RIM_THRESHOLD_RCD_MASK) / + FBNIC_MIN_RXD_PER_FRAME) { + NL_SET_ERR_MSG_MOD(extack, "rx_frames is above device max"); + return -EINVAL; + } + + fbn->tx_usecs = ec->tx_coalesce_usecs; + fbn->rx_usecs = ec->rx_coalesce_usecs; + fbn->rx_max_frames = ec->rx_max_coalesced_frames; + + if (netif_running(netdev)) { + int i; + + for (i = 0; i < fbn->num_napi; i++) { + struct fbnic_napi_vector *nv = fbn->napi[i]; + + fbnic_config_txrx_usecs(nv, 0); + fbnic_config_rx_frames(nv); + } + } + + return 0; +} + +static void +fbnic_get_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + ring->rx_max_pending = FBNIC_QUEUE_SIZE_MAX; + ring->rx_mini_max_pending = FBNIC_QUEUE_SIZE_MAX; + ring->rx_jumbo_max_pending = FBNIC_QUEUE_SIZE_MAX; + ring->tx_max_pending = FBNIC_QUEUE_SIZE_MAX; + + ring->rx_pending = fbn->rcq_size; + ring->rx_mini_pending = fbn->hpq_size; + ring->rx_jumbo_pending = fbn->ppq_size; + ring->tx_pending = fbn->txq_size; + + kernel_ring->tcp_data_split = ETHTOOL_TCP_DATA_SPLIT_ENABLED; + kernel_ring->hds_thresh_max = FBNIC_HDS_THRESH_MAX; + kernel_ring->hds_thresh = fbn->hds_thresh; +} + +static void fbnic_set_rings(struct fbnic_net *fbn, + struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring) +{ + fbn->rcq_size = ring->rx_pending; + fbn->hpq_size = ring->rx_mini_pending; + fbn->ppq_size = ring->rx_jumbo_pending; + fbn->txq_size = ring->tx_pending; + fbn->hds_thresh = kernel_ring->hds_thresh; +} + +static int +fbnic_set_ringparam(struct net_device *netdev, struct ethtool_ringparam *ring, + struct kernel_ethtool_ringparam *kernel_ring, + struct netlink_ext_ack *extack) + +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_net *clone; + int err; + + ring->rx_pending = roundup_pow_of_two(ring->rx_pending); + ring->rx_mini_pending = roundup_pow_of_two(ring->rx_mini_pending); + ring->rx_jumbo_pending = roundup_pow_of_two(ring->rx_jumbo_pending); + ring->tx_pending = roundup_pow_of_two(ring->tx_pending); + + /* These are absolute minimums allowing the device and driver to operate + * but not necessarily guarantee reasonable performance. Settings below + * Rx queue size of 128 and BDQs smaller than 64 are likely suboptimal + * at best. + */ + if (ring->rx_pending < max(FBNIC_QUEUE_SIZE_MIN, FBNIC_RX_DESC_MIN) || + ring->rx_mini_pending < FBNIC_QUEUE_SIZE_MIN || + ring->rx_jumbo_pending < FBNIC_QUEUE_SIZE_MIN || + ring->tx_pending < max(FBNIC_QUEUE_SIZE_MIN, FBNIC_TX_DESC_MIN)) { + NL_SET_ERR_MSG_MOD(extack, "requested ring size too small"); + return -EINVAL; + } + + if (kernel_ring->tcp_data_split == ETHTOOL_TCP_DATA_SPLIT_DISABLED) { + NL_SET_ERR_MSG_MOD(extack, "Cannot disable TCP data split"); + return -EINVAL; + } + + /* If an XDP program is attached, we should check for potential frame + * splitting. If the new HDS threshold can cause splitting, we should + * only allow if the attached XDP program can handle frags. + */ + if (fbnic_check_split_frames(fbn->xdp_prog, netdev->mtu, + kernel_ring->hds_thresh)) { + NL_SET_ERR_MSG_MOD(extack, + "Use higher HDS threshold or multi-buf capable program"); + return -EINVAL; + } + + if (!netif_running(netdev)) { + fbnic_set_rings(fbn, ring, kernel_ring); + return 0; + } + + clone = fbnic_clone_create(fbn); + if (!clone) + return -ENOMEM; + + fbnic_set_rings(clone, ring, kernel_ring); + + err = fbnic_alloc_napi_vectors(clone); + if (err) + goto err_free_clone; + + err = fbnic_alloc_resources(clone); + if (err) + goto err_free_napis; + + fbnic_down_noidle(fbn); + err = fbnic_wait_all_queues_idle(fbn->fbd, true); + if (err) + goto err_start_stack; + + err = fbnic_set_netif_queues(clone); + if (err) + goto err_start_stack; + + /* Nothing can fail past this point */ + fbnic_flush(fbn); + + fbnic_clone_swap(fbn, clone); + + fbnic_up(fbn); + + fbnic_free_resources(clone); + fbnic_free_napi_vectors(clone); + fbnic_clone_free(clone); + + return 0; + +err_start_stack: + fbnic_flush(fbn); + fbnic_up(fbn); + fbnic_free_resources(clone); +err_free_napis: + fbnic_free_napi_vectors(clone); +err_free_clone: + fbnic_clone_free(clone); + return err; +} + +static void fbnic_get_rxb_enqueue_strings(u8 **data, unsigned int idx) +{ + const struct fbnic_stat *stat; + int i; + + stat = fbnic_gstrings_rxb_enqueue_stats; + for (i = 0; i < FBNIC_HW_RXB_ENQUEUE_STATS_LEN; i++, stat++) + ethtool_sprintf(data, stat->string, idx); +} + +static void fbnic_get_rxb_fifo_strings(u8 **data, unsigned int idx) +{ + const struct fbnic_stat *stat; + int i; + + stat = fbnic_gstrings_rxb_fifo_stats; + for (i = 0; i < FBNIC_HW_RXB_FIFO_STATS_LEN; i++, stat++) + ethtool_sprintf(data, stat->string, idx); +} + +static void fbnic_get_rxb_dequeue_strings(u8 **data, unsigned int idx) +{ + const struct fbnic_stat *stat; + int i; + + stat = fbnic_gstrings_rxb_dequeue_stats; + for (i = 0; i < FBNIC_HW_RXB_DEQUEUE_STATS_LEN; i++, stat++) + ethtool_sprintf(data, stat->string, idx); +} + +static void fbnic_get_xdp_queue_strings(u8 **data, unsigned int idx) +{ + const struct fbnic_stat *stat; + int i; + + stat = fbnic_gstrings_xdp_stats; + for (i = 0; i < FBNIC_XDP_STATS_LEN; i++, stat++) + ethtool_sprintf(data, stat->string, idx); +} + +static void fbnic_get_strings(struct net_device *dev, u32 sset, u8 *data) +{ + const struct fbnic_stat *stat; + int i, idx; + + switch (sset) { + case ETH_SS_STATS: + for (i = 0; i < FBNIC_HW_FIXED_STATS_LEN; i++) + ethtool_puts(&data, fbnic_gstrings_hw_stats[i].string); + + for (i = 0; i < FBNIC_RXB_ENQUEUE_INDICES; i++) + fbnic_get_rxb_enqueue_strings(&data, i); + + for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++) + fbnic_get_rxb_fifo_strings(&data, i); + + for (i = 0; i < FBNIC_RXB_DEQUEUE_INDICES; i++) + fbnic_get_rxb_dequeue_strings(&data, i); + + for (idx = 0; idx < FBNIC_MAX_QUEUES; idx++) { + stat = fbnic_gstrings_hw_q_stats; + + for (i = 0; i < FBNIC_HW_Q_STATS_LEN; i++, stat++) + ethtool_sprintf(&data, stat->string, idx); + } + + for (i = 0; i < FBNIC_MAX_XDPQS; i++) + fbnic_get_xdp_queue_strings(&data, i); + break; + case ETH_SS_TEST: + memcpy(data, fbnic_gstrings_self_test, + sizeof(fbnic_gstrings_self_test)); + break; + } +} + +static void fbnic_report_hw_stats(const struct fbnic_stat *stat, + const void *base, int len, u64 **data) +{ + while (len--) { + u8 *curr = (u8 *)base + stat->offset; + + **data = *(u64 *)curr; + + stat++; + (*data)++; + } +} + +static void fbnic_get_xdp_queue_stats(struct fbnic_ring *ring, u64 **data) +{ + const struct fbnic_stat *stat; + int i; + + if (!ring) { + *data += FBNIC_XDP_STATS_LEN; + return; + } + + stat = fbnic_gstrings_xdp_stats; + for (i = 0; i < FBNIC_XDP_STATS_LEN; i++, stat++, (*data)++) { + u8 *p = (u8 *)ring + stat->offset; + + **data = *(u64 *)p; + } +} + +static void fbnic_get_ethtool_stats(struct net_device *dev, + struct ethtool_stats *stats, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_dev *fbd = fbn->fbd; + int i; + + fbnic_get_hw_stats(fbn->fbd); + + spin_lock(&fbd->hw_stats.lock); + fbnic_report_hw_stats(fbnic_gstrings_hw_stats, &fbd->hw_stats, + FBNIC_HW_FIXED_STATS_LEN, &data); + + for (i = 0; i < FBNIC_RXB_ENQUEUE_INDICES; i++) { + const struct fbnic_rxb_enqueue_stats *enq; + + enq = &fbd->hw_stats.rxb.enq[i]; + fbnic_report_hw_stats(fbnic_gstrings_rxb_enqueue_stats, + enq, FBNIC_HW_RXB_ENQUEUE_STATS_LEN, + &data); + } + + for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++) { + const struct fbnic_rxb_fifo_stats *fifo; + + fifo = &fbd->hw_stats.rxb.fifo[i]; + fbnic_report_hw_stats(fbnic_gstrings_rxb_fifo_stats, + fifo, FBNIC_HW_RXB_FIFO_STATS_LEN, + &data); + } + + for (i = 0; i < FBNIC_RXB_DEQUEUE_INDICES; i++) { + const struct fbnic_rxb_dequeue_stats *deq; + + deq = &fbd->hw_stats.rxb.deq[i]; + fbnic_report_hw_stats(fbnic_gstrings_rxb_dequeue_stats, + deq, FBNIC_HW_RXB_DEQUEUE_STATS_LEN, + &data); + } + + for (i = 0; i < FBNIC_MAX_QUEUES; i++) { + const struct fbnic_hw_q_stats *hw_q = &fbd->hw_stats.hw_q[i]; + + fbnic_report_hw_stats(fbnic_gstrings_hw_q_stats, hw_q, + FBNIC_HW_Q_STATS_LEN, &data); + } + spin_unlock(&fbd->hw_stats.lock); + + for (i = 0; i < FBNIC_MAX_XDPQS; i++) + fbnic_get_xdp_queue_stats(fbn->tx[i + FBNIC_MAX_TXQS], &data); +} + +static int fbnic_get_sset_count(struct net_device *dev, int sset) +{ + switch (sset) { + case ETH_SS_STATS: + return FBNIC_STATS_LEN; + case ETH_SS_TEST: + return FBNIC_TEST_LEN; + default: + return -EOPNOTSUPP; + } +} + +static int fbnic_get_rss_hash_idx(u32 flow_type) +{ + switch (flow_type & ~(FLOW_EXT | FLOW_MAC_EXT | FLOW_RSS)) { + case TCP_V4_FLOW: + return FBNIC_TCP4_HASH_OPT; + case TCP_V6_FLOW: + return FBNIC_TCP6_HASH_OPT; + case UDP_V4_FLOW: + return FBNIC_UDP4_HASH_OPT; + case UDP_V6_FLOW: + return FBNIC_UDP6_HASH_OPT; + case AH_V4_FLOW: + case ESP_V4_FLOW: + case AH_ESP_V4_FLOW: + case SCTP_V4_FLOW: + case IPV4_FLOW: + case IPV4_USER_FLOW: + return FBNIC_IPV4_HASH_OPT; + case AH_V6_FLOW: + case ESP_V6_FLOW: + case AH_ESP_V6_FLOW: + case SCTP_V6_FLOW: + case IPV6_FLOW: + case IPV6_USER_FLOW: + return FBNIC_IPV6_HASH_OPT; + case ETHER_FLOW: + return FBNIC_ETHER_HASH_OPT; + } + + return -1; +} + +static int fbnic_get_cls_rule_all(struct fbnic_net *fbn, + struct ethtool_rxnfc *cmd, + u32 *rule_locs) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i, cnt = 0; + + /* Report maximum rule count */ + cmd->data = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; + + for (i = 0; i < FBNIC_RPC_ACT_TBL_NFC_ENTRIES; i++) { + int idx = i + FBNIC_RPC_ACT_TBL_NFC_OFFSET; + struct fbnic_act_tcam *act_tcam; + + act_tcam = &fbd->act_tcam[idx]; + if (act_tcam->state != FBNIC_TCAM_S_VALID) + continue; + + if (rule_locs) { + if (cnt == cmd->rule_cnt) + return -EMSGSIZE; + + rule_locs[cnt] = i; + } + + cnt++; + } + + return cnt; +} + +static int fbnic_get_cls_rule(struct fbnic_net *fbn, struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec *fsp; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_act_tcam *act_tcam; + int idx; + + fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + + if (fsp->location >= FBNIC_RPC_ACT_TBL_NFC_ENTRIES) + return -EINVAL; + + idx = fsp->location + FBNIC_RPC_ACT_TBL_NFC_OFFSET; + act_tcam = &fbd->act_tcam[idx]; + + if (act_tcam->state != FBNIC_TCAM_S_VALID) + return -EINVAL; + + /* Report maximum rule count */ + cmd->data = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; + + /* Set flow type field */ + if (!(act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_IP_VALID)) { + fsp->flow_type = ETHER_FLOW; + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, + act_tcam->mask.tcam[1])) { + struct fbnic_mac_addr *mac_addr; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, + act_tcam->value.tcam[1]); + mac_addr = &fbd->mac_addr[idx]; + + ether_addr_copy(fsp->h_u.ether_spec.h_dest, + mac_addr->value.addr8); + eth_broadcast_addr(fsp->m_u.ether_spec.h_dest); + } + } else if (act_tcam->value.tcam[1] & + FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID) { + fsp->flow_type = IPV6_USER_FLOW; + fsp->h_u.usr_ip6_spec.l4_proto = IPPROTO_IPV6; + fsp->m_u.usr_ip6_spec.l4_proto = 0xff; + + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX, + act_tcam->mask.tcam[0])) { + struct fbnic_ip_addr *ip_addr; + int i; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX, + act_tcam->value.tcam[0]); + ip_addr = &fbd->ipo_src[idx]; + + for (i = 0; i < 4; i++) { + fsp->h_u.usr_ip6_spec.ip6src[i] = + ip_addr->value.s6_addr32[i]; + fsp->m_u.usr_ip6_spec.ip6src[i] = + ~ip_addr->mask.s6_addr32[i]; + } + } + + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX, + act_tcam->mask.tcam[0])) { + struct fbnic_ip_addr *ip_addr; + int i; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX, + act_tcam->value.tcam[0]); + ip_addr = &fbd->ipo_dst[idx]; + + for (i = 0; i < 4; i++) { + fsp->h_u.usr_ip6_spec.ip6dst[i] = + ip_addr->value.s6_addr32[i]; + fsp->m_u.usr_ip6_spec.ip6dst[i] = + ~ip_addr->mask.s6_addr32[i]; + } + } + } else if ((act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_IP_IS_V6)) { + if (act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L4_VALID) { + if (act_tcam->value.tcam[1] & + FBNIC_RPC_TCAM_ACT1_L4_IS_UDP) + fsp->flow_type = UDP_V6_FLOW; + else + fsp->flow_type = TCP_V6_FLOW; + fsp->h_u.tcp_ip6_spec.psrc = + cpu_to_be16(act_tcam->value.tcam[3]); + fsp->m_u.tcp_ip6_spec.psrc = + cpu_to_be16(~act_tcam->mask.tcam[3]); + fsp->h_u.tcp_ip6_spec.pdst = + cpu_to_be16(act_tcam->value.tcam[4]); + fsp->m_u.tcp_ip6_spec.pdst = + cpu_to_be16(~act_tcam->mask.tcam[4]); + } else { + fsp->flow_type = IPV6_USER_FLOW; + } + + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, + act_tcam->mask.tcam[0])) { + struct fbnic_ip_addr *ip_addr; + int i; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, + act_tcam->value.tcam[0]); + ip_addr = &fbd->ip_src[idx]; + + for (i = 0; i < 4; i++) { + fsp->h_u.usr_ip6_spec.ip6src[i] = + ip_addr->value.s6_addr32[i]; + fsp->m_u.usr_ip6_spec.ip6src[i] = + ~ip_addr->mask.s6_addr32[i]; + } + } + + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, + act_tcam->mask.tcam[0])) { + struct fbnic_ip_addr *ip_addr; + int i; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, + act_tcam->value.tcam[0]); + ip_addr = &fbd->ip_dst[idx]; + + for (i = 0; i < 4; i++) { + fsp->h_u.usr_ip6_spec.ip6dst[i] = + ip_addr->value.s6_addr32[i]; + fsp->m_u.usr_ip6_spec.ip6dst[i] = + ~ip_addr->mask.s6_addr32[i]; + } + } + } else { + if (act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L4_VALID) { + if (act_tcam->value.tcam[1] & + FBNIC_RPC_TCAM_ACT1_L4_IS_UDP) + fsp->flow_type = UDP_V4_FLOW; + else + fsp->flow_type = TCP_V4_FLOW; + fsp->h_u.tcp_ip4_spec.psrc = + cpu_to_be16(act_tcam->value.tcam[3]); + fsp->m_u.tcp_ip4_spec.psrc = + cpu_to_be16(~act_tcam->mask.tcam[3]); + fsp->h_u.tcp_ip4_spec.pdst = + cpu_to_be16(act_tcam->value.tcam[4]); + fsp->m_u.tcp_ip4_spec.pdst = + cpu_to_be16(~act_tcam->mask.tcam[4]); + } else { + fsp->flow_type = IPV4_USER_FLOW; + fsp->h_u.usr_ip4_spec.ip_ver = ETH_RX_NFC_IP4; + } + + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, + act_tcam->mask.tcam[0])) { + struct fbnic_ip_addr *ip_addr; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, + act_tcam->value.tcam[0]); + ip_addr = &fbd->ip_src[idx]; + + fsp->h_u.usr_ip4_spec.ip4src = + ip_addr->value.s6_addr32[3]; + fsp->m_u.usr_ip4_spec.ip4src = + ~ip_addr->mask.s6_addr32[3]; + } + + if (!FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, + act_tcam->mask.tcam[0])) { + struct fbnic_ip_addr *ip_addr; + + idx = FIELD_GET(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, + act_tcam->value.tcam[0]); + ip_addr = &fbd->ip_dst[idx]; + + fsp->h_u.usr_ip4_spec.ip4dst = + ip_addr->value.s6_addr32[3]; + fsp->m_u.usr_ip4_spec.ip4dst = + ~ip_addr->mask.s6_addr32[3]; + } + } + + /* Record action */ + if (act_tcam->dest & FBNIC_RPC_ACT_TBL0_DROP) + fsp->ring_cookie = RX_CLS_FLOW_DISC; + else if (act_tcam->dest & FBNIC_RPC_ACT_TBL0_Q_SEL) + fsp->ring_cookie = FIELD_GET(FBNIC_RPC_ACT_TBL0_Q_ID, + act_tcam->dest); + else + fsp->flow_type |= FLOW_RSS; + + cmd->rss_context = FIELD_GET(FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID, + act_tcam->dest); + + return 0; +} + +static u32 fbnic_get_rx_ring_count(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + return fbn->num_rx_queues; +} + +static int fbnic_get_rxnfc(struct net_device *netdev, + struct ethtool_rxnfc *cmd, u32 *rule_locs) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int ret = -EOPNOTSUPP; + u32 special = 0; + + switch (cmd->cmd) { + case ETHTOOL_GRXCLSRULE: + ret = fbnic_get_cls_rule(fbn, cmd); + break; + case ETHTOOL_GRXCLSRLCNT: + rule_locs = NULL; + special = RX_CLS_LOC_SPECIAL; + fallthrough; + case ETHTOOL_GRXCLSRLALL: + ret = fbnic_get_cls_rule_all(fbn, cmd, rule_locs); + if (ret < 0) + break; + + cmd->data |= special; + cmd->rule_cnt = ret; + ret = 0; + break; + } + + return ret; +} + +static int fbnic_cls_rule_any_loc(struct fbnic_dev *fbd) +{ + int i; + + for (i = FBNIC_RPC_ACT_TBL_NFC_ENTRIES; i--;) { + int idx = i + FBNIC_RPC_ACT_TBL_NFC_OFFSET; + + if (fbd->act_tcam[idx].state != FBNIC_TCAM_S_VALID) + return i; + } + + return -ENOSPC; +} + +static int fbnic_set_cls_rule_ins(struct fbnic_net *fbn, + const struct ethtool_rxnfc *cmd) +{ + u16 flow_value = 0, flow_mask = 0xffff, ip_value = 0, ip_mask = 0xffff; + u16 sport = 0, sport_mask = ~0, dport = 0, dport_mask = ~0; + u16 misc = 0, misc_mask = ~0; + u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + FBNIC_RPC_ACT_TBL0_DEST_HOST); + struct fbnic_ip_addr *ip_src = NULL, *ip_dst = NULL; + struct fbnic_mac_addr *mac_addr = NULL; + struct ethtool_rx_flow_spec *fsp; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_act_tcam *act_tcam; + struct in6_addr *addr6, *mask6; + struct in_addr *addr4, *mask4; + int hash_idx, location; + u32 flow_type; + int idx, j; + + fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + + if (fsp->location != RX_CLS_LOC_ANY) + return -EINVAL; + location = fbnic_cls_rule_any_loc(fbd); + if (location < 0) + return location; + + if (fsp->ring_cookie == RX_CLS_FLOW_DISC) { + dest = FBNIC_RPC_ACT_TBL0_DROP; + } else if (fsp->flow_type & FLOW_RSS) { + if (cmd->rss_context == 1) + dest |= FBNIC_RPC_ACT_TBL0_RSS_CTXT_ID; + } else { + u32 ring_idx = ethtool_get_flow_spec_ring(fsp->ring_cookie); + + if (ring_idx >= fbn->num_rx_queues) + return -EINVAL; + + dest |= FBNIC_RPC_ACT_TBL0_Q_SEL | + FIELD_PREP(FBNIC_RPC_ACT_TBL0_Q_ID, ring_idx); + } + + idx = location + FBNIC_RPC_ACT_TBL_NFC_OFFSET; + act_tcam = &fbd->act_tcam[idx]; + + /* Do not allow overwriting for now. + * To support overwriting rules we will need to add logic to free + * any IP or MACDA TCAMs that may be associated with the old rule. + */ + if (act_tcam->state != FBNIC_TCAM_S_DISABLED) + return -EBUSY; + + flow_type = fsp->flow_type & ~(FLOW_EXT | FLOW_RSS); + hash_idx = fbnic_get_rss_hash_idx(flow_type); + + switch (flow_type) { + case UDP_V4_FLOW: +udp4_flow: + flow_value |= FBNIC_RPC_TCAM_ACT1_L4_IS_UDP; + fallthrough; + case TCP_V4_FLOW: +tcp4_flow: + flow_value |= FBNIC_RPC_TCAM_ACT1_L4_VALID; + flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_L4_IS_UDP | + FBNIC_RPC_TCAM_ACT1_L4_VALID); + + sport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.psrc); + sport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip4_spec.psrc); + dport = be16_to_cpu(fsp->h_u.tcp_ip4_spec.pdst); + dport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip4_spec.pdst); + goto ip4_flow; + case IP_USER_FLOW: + if (!fsp->m_u.usr_ip4_spec.proto) + goto ip4_flow; + if (fsp->m_u.usr_ip4_spec.proto != 0xff) + return -EINVAL; + if (fsp->h_u.usr_ip4_spec.proto == IPPROTO_UDP) + goto udp4_flow; + if (fsp->h_u.usr_ip4_spec.proto == IPPROTO_TCP) + goto tcp4_flow; + return -EINVAL; +ip4_flow: + addr4 = (struct in_addr *)&fsp->h_u.usr_ip4_spec.ip4src; + mask4 = (struct in_addr *)&fsp->m_u.usr_ip4_spec.ip4src; + if (mask4->s_addr) { + ip_src = __fbnic_ip4_sync(fbd, fbd->ip_src, + addr4, mask4); + if (!ip_src) + return -ENOSPC; + + set_bit(idx, ip_src->act_tcam); + ip_value |= FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | + FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, + ip_src - fbd->ip_src); + ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | + FBNIC_RPC_TCAM_ACT0_IPSRC_IDX); + } + + addr4 = (struct in_addr *)&fsp->h_u.usr_ip4_spec.ip4dst; + mask4 = (struct in_addr *)&fsp->m_u.usr_ip4_spec.ip4dst; + if (mask4->s_addr) { + ip_dst = __fbnic_ip4_sync(fbd, fbd->ip_dst, + addr4, mask4); + if (!ip_dst) { + if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD) + memset(ip_src, 0, sizeof(*ip_src)); + return -ENOSPC; + } + + set_bit(idx, ip_dst->act_tcam); + ip_value |= FBNIC_RPC_TCAM_ACT0_IPDST_VALID | + FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, + ip_dst - fbd->ip_dst); + ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPDST_VALID | + FBNIC_RPC_TCAM_ACT0_IPDST_IDX); + } + flow_value |= FBNIC_RPC_TCAM_ACT1_IP_VALID | + FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_IP_IS_V6 | + FBNIC_RPC_TCAM_ACT1_IP_VALID | + FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID); + break; + case UDP_V6_FLOW: +udp6_flow: + flow_value |= FBNIC_RPC_TCAM_ACT1_L4_IS_UDP; + fallthrough; + case TCP_V6_FLOW: +tcp6_flow: + flow_value |= FBNIC_RPC_TCAM_ACT1_L4_VALID; + flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_L4_IS_UDP | + FBNIC_RPC_TCAM_ACT1_L4_VALID); + + sport = be16_to_cpu(fsp->h_u.tcp_ip6_spec.psrc); + sport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip6_spec.psrc); + dport = be16_to_cpu(fsp->h_u.tcp_ip6_spec.pdst); + dport_mask = ~be16_to_cpu(fsp->m_u.tcp_ip6_spec.pdst); + goto ipv6_flow; + case IPV6_USER_FLOW: + if (!fsp->m_u.usr_ip6_spec.l4_proto) + goto ipv6_flow; + + if (fsp->m_u.usr_ip6_spec.l4_proto != 0xff) + return -EINVAL; + if (fsp->h_u.usr_ip6_spec.l4_proto == IPPROTO_UDP) + goto udp6_flow; + if (fsp->h_u.usr_ip6_spec.l4_proto == IPPROTO_TCP) + goto tcp6_flow; + if (fsp->h_u.usr_ip6_spec.l4_proto != IPPROTO_IPV6) + return -EINVAL; + + addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6src; + mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6src; + if (!ipv6_addr_any(mask6)) { + ip_src = __fbnic_ip6_sync(fbd, fbd->ipo_src, + addr6, mask6); + if (!ip_src) + return -ENOSPC; + + set_bit(idx, ip_src->act_tcam); + ip_value |= + FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID | + FIELD_PREP(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX, + ip_src - fbd->ipo_src); + ip_mask &= + ~(FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID | + FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX); + } + + addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6dst; + mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6dst; + if (!ipv6_addr_any(mask6)) { + ip_dst = __fbnic_ip6_sync(fbd, fbd->ipo_dst, + addr6, mask6); + if (!ip_dst) { + if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD) + memset(ip_src, 0, sizeof(*ip_src)); + return -ENOSPC; + } + + set_bit(idx, ip_dst->act_tcam); + ip_value |= + FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID | + FIELD_PREP(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX, + ip_dst - fbd->ipo_dst); + ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID | + FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX); + } + + flow_value |= FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID; + flow_mask &= FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID; +ipv6_flow: + addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6src; + mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6src; + if (!ip_src && !ipv6_addr_any(mask6)) { + ip_src = __fbnic_ip6_sync(fbd, fbd->ip_src, + addr6, mask6); + if (!ip_src) + return -ENOSPC; + + set_bit(idx, ip_src->act_tcam); + ip_value |= FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | + FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPSRC_IDX, + ip_src - fbd->ip_src); + ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | + FBNIC_RPC_TCAM_ACT0_IPSRC_IDX); + } + + addr6 = (struct in6_addr *)fsp->h_u.usr_ip6_spec.ip6dst; + mask6 = (struct in6_addr *)fsp->m_u.usr_ip6_spec.ip6dst; + if (!ip_dst && !ipv6_addr_any(mask6)) { + ip_dst = __fbnic_ip6_sync(fbd, fbd->ip_dst, + addr6, mask6); + if (!ip_dst) { + if (ip_src && ip_src->state == FBNIC_TCAM_S_ADD) + memset(ip_src, 0, sizeof(*ip_src)); + return -ENOSPC; + } + + set_bit(idx, ip_dst->act_tcam); + ip_value |= FBNIC_RPC_TCAM_ACT0_IPDST_VALID | + FIELD_PREP(FBNIC_RPC_TCAM_ACT0_IPDST_IDX, + ip_dst - fbd->ip_dst); + ip_mask &= ~(FBNIC_RPC_TCAM_ACT0_IPDST_VALID | + FBNIC_RPC_TCAM_ACT0_IPDST_IDX); + } + + flow_value |= FBNIC_RPC_TCAM_ACT1_IP_IS_V6 | + FBNIC_RPC_TCAM_ACT1_IP_VALID | + FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + flow_mask &= ~(FBNIC_RPC_TCAM_ACT1_IP_IS_V6 | + FBNIC_RPC_TCAM_ACT1_IP_VALID | + FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID); + break; + case ETHER_FLOW: + if (!is_zero_ether_addr(fsp->m_u.ether_spec.h_dest)) { + u8 *addr = fsp->h_u.ether_spec.h_dest; + u8 *mask = fsp->m_u.ether_spec.h_dest; + + /* Do not allow MAC addr of 0 */ + if (is_zero_ether_addr(addr)) + return -EINVAL; + + /* Only support full MAC address to avoid + * conflicts with other MAC addresses. + */ + if (!is_broadcast_ether_addr(mask)) + return -EINVAL; + + if (is_multicast_ether_addr(addr)) + mac_addr = __fbnic_mc_sync(fbd, addr); + else + mac_addr = __fbnic_uc_sync(fbd, addr); + + if (!mac_addr) + return -ENOSPC; + + set_bit(idx, mac_addr->act_tcam); + flow_value |= + FIELD_PREP(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, + mac_addr - fbd->mac_addr); + flow_mask &= ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX; + } + + flow_value |= FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + flow_mask &= ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + break; + default: + return -EINVAL; + } + + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); + + /* Write action table values */ + act_tcam->dest = dest; + act_tcam->rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, hash_idx); + + /* Write IP Match value/mask to action_tcam[0] */ + act_tcam->value.tcam[0] = ip_value; + act_tcam->mask.tcam[0] = ip_mask; + + /* Write flow type value/mask to action_tcam[1] */ + act_tcam->value.tcam[1] = flow_value; + act_tcam->mask.tcam[1] = flow_mask; + + /* Write error, DSCP, extra L4 matches to action_tcam[2] */ + act_tcam->value.tcam[2] = misc; + act_tcam->mask.tcam[2] = misc_mask; + + /* Write source/destination port values */ + act_tcam->value.tcam[3] = sport; + act_tcam->mask.tcam[3] = sport_mask; + act_tcam->value.tcam[4] = dport; + act_tcam->mask.tcam[4] = dport_mask; + + for (j = 5; j < FBNIC_RPC_TCAM_ACT_WORD_LEN; j++) + act_tcam->mask.tcam[j] = 0xffff; + + act_tcam->state = FBNIC_TCAM_S_UPDATE; + fsp->location = location; + + if (netif_running(fbn->netdev)) { + fbnic_write_rules(fbd); + if (ip_src || ip_dst) + fbnic_write_ip_addr(fbd); + if (mac_addr) + fbnic_write_macda(fbd); + } + + return 0; +} + +static void fbnic_clear_nfc_macda(struct fbnic_net *fbn, + unsigned int tcam_idx) +{ + struct fbnic_dev *fbd = fbn->fbd; + int idx; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) + __fbnic_xc_unsync(&fbd->mac_addr[idx], tcam_idx); + + /* Write updates to hardware */ + if (netif_running(fbn->netdev)) + fbnic_write_macda(fbd); +} + +static void fbnic_clear_nfc_ip_addr(struct fbnic_net *fbn, + unsigned int tcam_idx) +{ + struct fbnic_dev *fbd = fbn->fbd; + int idx; + + for (idx = ARRAY_SIZE(fbd->ip_src); idx--;) + __fbnic_ip_unsync(&fbd->ip_src[idx], tcam_idx); + for (idx = ARRAY_SIZE(fbd->ip_dst); idx--;) + __fbnic_ip_unsync(&fbd->ip_dst[idx], tcam_idx); + for (idx = ARRAY_SIZE(fbd->ipo_src); idx--;) + __fbnic_ip_unsync(&fbd->ipo_src[idx], tcam_idx); + for (idx = ARRAY_SIZE(fbd->ipo_dst); idx--;) + __fbnic_ip_unsync(&fbd->ipo_dst[idx], tcam_idx); + + /* Write updates to hardware */ + if (netif_running(fbn->netdev)) + fbnic_write_ip_addr(fbd); +} + +static int fbnic_set_cls_rule_del(struct fbnic_net *fbn, + const struct ethtool_rxnfc *cmd) +{ + struct ethtool_rx_flow_spec *fsp; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_act_tcam *act_tcam; + int idx; + + fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; + + if (fsp->location >= FBNIC_RPC_ACT_TBL_NFC_ENTRIES) + return -EINVAL; + + idx = fsp->location + FBNIC_RPC_ACT_TBL_NFC_OFFSET; + act_tcam = &fbd->act_tcam[idx]; + + if (act_tcam->state != FBNIC_TCAM_S_VALID) + return -EINVAL; + + act_tcam->state = FBNIC_TCAM_S_DELETE; + + if ((act_tcam->value.tcam[1] & FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID) && + (~act_tcam->mask.tcam[1] & FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX)) + fbnic_clear_nfc_macda(fbn, idx); + + if ((act_tcam->value.tcam[0] & + (FBNIC_RPC_TCAM_ACT0_IPSRC_VALID | + FBNIC_RPC_TCAM_ACT0_IPDST_VALID | + FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID | + FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID)) && + (~act_tcam->mask.tcam[0] & + (FBNIC_RPC_TCAM_ACT0_IPSRC_IDX | + FBNIC_RPC_TCAM_ACT0_IPDST_IDX | + FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX | + FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX))) + fbnic_clear_nfc_ip_addr(fbn, idx); + + if (netif_running(fbn->netdev)) + fbnic_write_rules(fbd); + + return 0; +} + +static int fbnic_set_rxnfc(struct net_device *netdev, struct ethtool_rxnfc *cmd) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int ret = -EOPNOTSUPP; + + switch (cmd->cmd) { + case ETHTOOL_SRXCLSRLINS: + ret = fbnic_set_cls_rule_ins(fbn, cmd); + break; + case ETHTOOL_SRXCLSRLDEL: + ret = fbnic_set_cls_rule_del(fbn, cmd); + break; + } + + return ret; +} + +static u32 fbnic_get_rxfh_key_size(struct net_device *netdev) +{ + return FBNIC_RPC_RSS_KEY_BYTE_LEN; +} + +static u32 fbnic_get_rxfh_indir_size(struct net_device *netdev) +{ + return FBNIC_RPC_RSS_TBL_SIZE; +} + +static int +fbnic_get_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + unsigned int i; + + rxfh->hfunc = ETH_RSS_HASH_TOP; + + if (rxfh->key) { + for (i = 0; i < FBNIC_RPC_RSS_KEY_BYTE_LEN; i++) { + u32 rss_key = fbn->rss_key[i / 4] << ((i % 4) * 8); + + rxfh->key[i] = rss_key >> 24; + } + } + + if (rxfh->indir) { + for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) + rxfh->indir[i] = fbn->indir_tbl[0][i]; + } + + return 0; +} + +static unsigned int +fbnic_set_indir(struct fbnic_net *fbn, unsigned int idx, const u32 *indir) +{ + unsigned int i, changes = 0; + + for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) { + if (fbn->indir_tbl[idx][i] == indir[i]) + continue; + + fbn->indir_tbl[idx][i] = indir[i]; + changes++; + } + + return changes; +} + +static int +fbnic_set_rxfh(struct net_device *netdev, struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + unsigned int i, changes = 0; + + if (rxfh->hfunc != ETH_RSS_HASH_NO_CHANGE && + rxfh->hfunc != ETH_RSS_HASH_TOP) + return -EINVAL; + + if (rxfh->key) { + u32 rss_key = 0; + + for (i = FBNIC_RPC_RSS_KEY_BYTE_LEN; i--;) { + rss_key >>= 8; + rss_key |= (u32)(rxfh->key[i]) << 24; + + if (i % 4) + continue; + + if (fbn->rss_key[i / 4] == rss_key) + continue; + + fbn->rss_key[i / 4] = rss_key; + changes++; + } + } + + if (rxfh->indir) + changes += fbnic_set_indir(fbn, 0, rxfh->indir); + + if (changes && netif_running(netdev)) + fbnic_rss_reinit_hw(fbn->fbd, fbn); + + return 0; +} + +static int +fbnic_get_rss_hash_opts(struct net_device *netdev, + struct ethtool_rxfh_fields *cmd) +{ + int hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); + struct fbnic_net *fbn = netdev_priv(netdev); + + if (hash_opt_idx < 0) + return -EINVAL; + + /* Report options from rss_en table in fbn */ + cmd->data = fbn->rss_flow_hash[hash_opt_idx]; + + return 0; +} + +#define FBNIC_L2_HASH_OPTIONS \ + (RXH_L2DA | RXH_DISCARD) +#define FBNIC_L3_HASH_OPTIONS \ + (FBNIC_L2_HASH_OPTIONS | RXH_IP_SRC | RXH_IP_DST | RXH_IP6_FL) +#define FBNIC_L4_HASH_OPTIONS \ + (FBNIC_L3_HASH_OPTIONS | RXH_L4_B_0_1 | RXH_L4_B_2_3) + +static int +fbnic_set_rss_hash_opts(struct net_device *netdev, + const struct ethtool_rxfh_fields *cmd, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int hash_opt_idx; + + /* Verify the type requested is correct */ + hash_opt_idx = fbnic_get_rss_hash_idx(cmd->flow_type); + if (hash_opt_idx < 0) + return -EINVAL; + + /* Verify the fields asked for can actually be assigned based on type */ + if (cmd->data & ~FBNIC_L4_HASH_OPTIONS || + (hash_opt_idx > FBNIC_L4_HASH_OPT && + cmd->data & ~FBNIC_L3_HASH_OPTIONS) || + (hash_opt_idx > FBNIC_IP_HASH_OPT && + cmd->data & ~FBNIC_L2_HASH_OPTIONS)) + return -EINVAL; + + fbn->rss_flow_hash[hash_opt_idx] = cmd->data; + + if (netif_running(fbn->netdev)) { + fbnic_rss_reinit(fbn->fbd, fbn); + fbnic_write_rules(fbn->fbd); + } + + return 0; +} + +static int +fbnic_modify_rxfh_context(struct net_device *netdev, + struct ethtool_rxfh_context *ctx, + const struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + const u32 *indir = rxfh->indir; + unsigned int changes; + + if (!indir) + indir = ethtool_rxfh_context_indir(ctx); + + changes = fbnic_set_indir(fbn, rxfh->rss_context, indir); + if (changes && netif_running(netdev)) + fbnic_rss_reinit_hw(fbn->fbd, fbn); + + return 0; +} + +static int +fbnic_create_rxfh_context(struct net_device *netdev, + struct ethtool_rxfh_context *ctx, + const struct ethtool_rxfh_param *rxfh, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + if (rxfh->hfunc && rxfh->hfunc != ETH_RSS_HASH_TOP) { + NL_SET_ERR_MSG_MOD(extack, "RSS hash function not supported"); + return -EOPNOTSUPP; + } + ctx->hfunc = ETH_RSS_HASH_TOP; + + if (!rxfh->indir) { + u32 *indir = ethtool_rxfh_context_indir(ctx); + unsigned int num_rx = fbn->num_rx_queues; + unsigned int i; + + for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) + indir[i] = ethtool_rxfh_indir_default(i, num_rx); + } + + return fbnic_modify_rxfh_context(netdev, ctx, rxfh, extack); +} + +static int +fbnic_remove_rxfh_context(struct net_device *netdev, + struct ethtool_rxfh_context *ctx, u32 rss_context, + struct netlink_ext_ack *extack) +{ + /* Nothing to do, contexts are allocated statically */ + return 0; +} + +static int fbnic_ethtool_regs_test(struct net_device *netdev, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + *data = fbnic_csr_regs_test(fbd); + + return !!*data; +} + +/** + * fbnic_ethtool_msix_test - Verify behavior of NIC interrupts + * @netdev: netdev device to test + * @data: Pointer to results storage + * + * This function is meant to test the global interrupt registers and the + * PCIe IP MSI-X functionality. It essentially goes through and tests + * test various combinations of the set, clear, and mask bits in order to + * verify the behavior is as we expect it to be from the driver. + * + * Return: non-zero on failure. + **/ +static int fbnic_ethtool_msix_test(struct net_device *netdev, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + *data = fbnic_msix_test(fbd); + + return !!*data; +} + +static int fbnic_ethtool_mbx_self_test(struct net_device *netdev, u64 *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + *data = fbnic_fw_mbx_self_test(fbd); + + return !!*data; +} + +static void fbnic_self_test(struct net_device *netdev, + struct ethtool_test *eth_test, u64 *data) +{ + bool if_running = netif_running(netdev); + + if (fbnic_ethtool_mbx_self_test(netdev, &data[TEST_MBX])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (!(eth_test->flags & ETH_TEST_FL_OFFLINE)) { + data[TEST_REG] = 0; + data[TEST_MSIX] = 0; + return; + } + + if (if_running) + netif_close(netdev); + + if (fbnic_ethtool_regs_test(netdev, &data[TEST_REG])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (fbnic_ethtool_msix_test(netdev, &data[TEST_MSIX])) + eth_test->flags |= ETH_TEST_FL_FAILED; + + if (if_running && netif_open(netdev, NULL)) { + netdev_err(netdev, + "Failed to re-initialize hardware following test\n"); + eth_test->flags |= ETH_TEST_FL_FAILED; + } +} + +static void fbnic_get_channels(struct net_device *netdev, + struct ethtool_channels *ch) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + ch->max_rx = fbd->max_num_queues; + ch->max_tx = fbd->max_num_queues; + ch->max_combined = min(ch->max_rx, ch->max_tx); + ch->max_other = FBNIC_NON_NAPI_VECTORS; + + if (fbn->num_rx_queues > fbn->num_napi || + fbn->num_tx_queues > fbn->num_napi) + ch->combined_count = min(fbn->num_rx_queues, + fbn->num_tx_queues); + else + ch->combined_count = + fbn->num_rx_queues + fbn->num_tx_queues - fbn->num_napi; + ch->rx_count = fbn->num_rx_queues - ch->combined_count; + ch->tx_count = fbn->num_tx_queues - ch->combined_count; + ch->other_count = FBNIC_NON_NAPI_VECTORS; +} + +static void fbnic_set_queues(struct fbnic_net *fbn, struct ethtool_channels *ch, + unsigned int max_napis) +{ + fbn->num_rx_queues = ch->rx_count + ch->combined_count; + fbn->num_tx_queues = ch->tx_count + ch->combined_count; + fbn->num_napi = min(ch->rx_count + ch->tx_count + ch->combined_count, + max_napis); +} + +static int fbnic_set_channels(struct net_device *netdev, + struct ethtool_channels *ch) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + unsigned int max_napis, standalone; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_net *clone; + int err; + + max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS; + standalone = ch->rx_count + ch->tx_count; + + /* Limits for standalone queues: + * - each queue has its own NAPI (num_napi >= rx + tx + combined) + * - combining queues (combined not 0, rx or tx must be 0) + */ + if ((ch->rx_count && ch->tx_count && ch->combined_count) || + (standalone && standalone + ch->combined_count > max_napis) || + ch->rx_count + ch->combined_count > fbd->max_num_queues || + ch->tx_count + ch->combined_count > fbd->max_num_queues || + ch->other_count != FBNIC_NON_NAPI_VECTORS) + return -EINVAL; + + if (!netif_running(netdev)) { + fbnic_set_queues(fbn, ch, max_napis); + fbnic_reset_indir_tbl(fbn); + return 0; + } + + clone = fbnic_clone_create(fbn); + if (!clone) + return -ENOMEM; + + fbnic_set_queues(clone, ch, max_napis); + + err = fbnic_alloc_napi_vectors(clone); + if (err) + goto err_free_clone; + + err = fbnic_alloc_resources(clone); + if (err) + goto err_free_napis; + + fbnic_down_noidle(fbn); + err = fbnic_wait_all_queues_idle(fbn->fbd, true); + if (err) + goto err_start_stack; + + err = fbnic_set_netif_queues(clone); + if (err) + goto err_start_stack; + + /* Nothing can fail past this point */ + fbnic_flush(fbn); + + fbnic_clone_swap(fbn, clone); + + /* Reset RSS indirection table */ + fbnic_reset_indir_tbl(fbn); + + fbnic_up(fbn); + + fbnic_free_resources(clone); + fbnic_free_napi_vectors(clone); + fbnic_clone_free(clone); + + return 0; + +err_start_stack: + fbnic_flush(fbn); + fbnic_up(fbn); + fbnic_free_resources(clone); +err_free_napis: + fbnic_free_napi_vectors(clone); +err_free_clone: + fbnic_clone_free(clone); + return err; +} + +static int +fbnic_get_ts_info(struct net_device *netdev, + struct kernel_ethtool_ts_info *tsinfo) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + tsinfo->phc_index = ptp_clock_index(fbn->fbd->ptp); + + tsinfo->so_timestamping = + SOF_TIMESTAMPING_TX_SOFTWARE | + SOF_TIMESTAMPING_TX_HARDWARE | + SOF_TIMESTAMPING_RX_HARDWARE | + SOF_TIMESTAMPING_RAW_HARDWARE; + + tsinfo->tx_types = + BIT(HWTSTAMP_TX_OFF) | + BIT(HWTSTAMP_TX_ON); + + tsinfo->rx_filters = + BIT(HWTSTAMP_FILTER_NONE) | + BIT(HWTSTAMP_FILTER_PTP_V1_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L4_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_L2_EVENT) | + BIT(HWTSTAMP_FILTER_PTP_V2_EVENT) | + BIT(HWTSTAMP_FILTER_ALL); + + return 0; +} + +static void fbnic_get_ts_stats(struct net_device *netdev, + struct ethtool_ts_stats *ts_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + u64 ts_packets, ts_lost; + struct fbnic_ring *ring; + unsigned int start; + int i; + + ts_stats->pkts = fbn->tx_stats.twq.ts_packets; + ts_stats->lost = fbn->tx_stats.twq.ts_lost; + for (i = 0; i < fbn->num_tx_queues; i++) { + ring = fbn->tx[i]; + do { + start = u64_stats_fetch_begin(&ring->stats.syncp); + ts_packets = ring->stats.twq.ts_packets; + ts_lost = ring->stats.twq.ts_lost; + } while (u64_stats_fetch_retry(&ring->stats.syncp, start)); + ts_stats->pkts += ts_packets; + ts_stats->lost += ts_lost; + } +} + +static int fbnic_get_tunable(struct net_device *netdev, + const struct ethtool_tunable *tun, + void *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err = 0; + + switch (tun->id) { + case ETHTOOL_PFC_PREVENTION_TOUT: + *(u16 *)data = fbn->fbd->ps_timeout; + break; + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int fbnic_set_tunable(struct net_device *netdev, + const struct ethtool_tunable *tun, + const void *data) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err; + + switch (tun->id) { + case ETHTOOL_PFC_PREVENTION_TOUT: { + u16 ps_timeout = *(u16 *)data; + + err = fbnic_mac_ps_protect_to_config(fbn->fbd, ps_timeout); + break; + } + default: + err = -EOPNOTSUPP; + break; + } + + return err; +} + +static int +fbnic_get_module_eeprom_by_page(struct net_device *netdev, + const struct ethtool_module_eeprom *page_data, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_fw_completion *fw_cmpl; + struct fbnic_dev *fbd = fbn->fbd; + int err; + + if (page_data->i2c_address != 0x50) { + NL_SET_ERR_MSG_MOD(extack, + "Invalid i2c address. Only 0x50 is supported"); + return -EINVAL; + } + + fw_cmpl = __fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_QSFP_READ_RESP, + page_data->length); + if (!fw_cmpl) + return -ENOMEM; + + /* Initialize completion and queue it for FW to process */ + fw_cmpl->u.qsfp.length = page_data->length; + fw_cmpl->u.qsfp.offset = page_data->offset; + fw_cmpl->u.qsfp.page = page_data->page; + fw_cmpl->u.qsfp.bank = page_data->bank; + + err = fbnic_fw_xmit_qsfp_read_msg(fbd, fw_cmpl, page_data->page, + page_data->bank, page_data->offset, + page_data->length); + if (err) { + NL_SET_ERR_MSG_MOD(extack, + "Failed to transmit EEPROM read request"); + goto exit_free; + } + + if (!fbnic_mbx_wait_for_cmpl(fw_cmpl)) { + err = -ETIMEDOUT; + NL_SET_ERR_MSG_MOD(extack, + "Timed out waiting for firmware response"); + goto exit_cleanup; + } + + if (fw_cmpl->result) { + err = fw_cmpl->result; + NL_SET_ERR_MSG_MOD(extack, "Failed to read EEPROM"); + goto exit_cleanup; + } + + memcpy(page_data->data, fw_cmpl->u.qsfp.data, page_data->length); + +exit_cleanup: + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); +exit_free: + fbnic_fw_put_cmpl(fw_cmpl); + + return err ? : page_data->length; +} + +static void fbnic_set_counter(u64 *stat, struct fbnic_stat_counter *counter) +{ + if (counter->reported) + *stat = counter->value; +} + +static void +fbnic_get_pause_stats(struct net_device *netdev, + struct ethtool_pause_stats *pause_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + u64 tx_ps_events; + + mac_stats = &fbd->hw_stats.mac; + + fbd->mac->get_pause_stats(fbd, false, &mac_stats->pause); + + pause_stats->tx_pause_frames = mac_stats->pause.tx_pause_frames.value; + pause_stats->rx_pause_frames = mac_stats->pause.rx_pause_frames.value; + tx_ps_events = mac_stats->pause.tx_pause_storm_events.value; + pause_stats->tx_pause_storm_events = tx_ps_events; +} + +static void +fbnic_get_fec_stats(struct net_device *netdev, + struct ethtool_fec_stats *fec_stats, + struct ethtool_fec_hist *hist) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_phy_stats *phy_stats; + struct fbnic_dev *fbd = fbn->fbd; + + fbnic_get_hw_stats32(fbd); + phy_stats = &fbd->hw_stats.phy; + + spin_lock(&fbd->hw_stats.lock); + fec_stats->corrected_blocks.total = + phy_stats->fec.corrected_blocks.value; + fec_stats->uncorrectable_blocks.total = + phy_stats->fec.uncorrectable_blocks.value; + spin_unlock(&fbd->hw_stats.lock); +} + +static void +fbnic_get_eth_phy_stats(struct net_device *netdev, + struct ethtool_eth_phy_stats *eth_phy_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_phy_stats *phy_stats; + struct fbnic_dev *fbd = fbn->fbd; + u64 total = 0; + int i; + + fbnic_get_hw_stats32(fbd); + phy_stats = &fbd->hw_stats.phy; + + spin_lock(&fbd->hw_stats.lock); + for (i = 0; i < FBNIC_PCS_MAX_LANES; i++) + total += phy_stats->pcs.SymbolErrorDuringCarrier.lanes[i].value; + + eth_phy_stats->SymbolErrorDuringCarrier = total; + spin_unlock(&fbd->hw_stats.lock); +} + +static void +fbnic_get_eth_mac_stats(struct net_device *netdev, + struct ethtool_eth_mac_stats *eth_mac_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + const struct fbnic_mac *mac; + + mac_stats = &fbd->hw_stats.mac; + mac = fbd->mac; + + mac->get_eth_mac_stats(fbd, false, &mac_stats->eth_mac); + + fbnic_set_counter(ð_mac_stats->FramesTransmittedOK, + &mac_stats->eth_mac.FramesTransmittedOK); + fbnic_set_counter(ð_mac_stats->FramesReceivedOK, + &mac_stats->eth_mac.FramesReceivedOK); + fbnic_set_counter(ð_mac_stats->FrameCheckSequenceErrors, + &mac_stats->eth_mac.FrameCheckSequenceErrors); + fbnic_set_counter(ð_mac_stats->AlignmentErrors, + &mac_stats->eth_mac.AlignmentErrors); + fbnic_set_counter(ð_mac_stats->OctetsTransmittedOK, + &mac_stats->eth_mac.OctetsTransmittedOK); + fbnic_set_counter(ð_mac_stats->FramesLostDueToIntMACXmitError, + &mac_stats->eth_mac.FramesLostDueToIntMACXmitError); + fbnic_set_counter(ð_mac_stats->OctetsReceivedOK, + &mac_stats->eth_mac.OctetsReceivedOK); + fbnic_set_counter(ð_mac_stats->FramesLostDueToIntMACRcvError, + &mac_stats->eth_mac.FramesLostDueToIntMACRcvError); + fbnic_set_counter(ð_mac_stats->MulticastFramesXmittedOK, + &mac_stats->eth_mac.MulticastFramesXmittedOK); + fbnic_set_counter(ð_mac_stats->BroadcastFramesXmittedOK, + &mac_stats->eth_mac.BroadcastFramesXmittedOK); + fbnic_set_counter(ð_mac_stats->MulticastFramesReceivedOK, + &mac_stats->eth_mac.MulticastFramesReceivedOK); + fbnic_set_counter(ð_mac_stats->BroadcastFramesReceivedOK, + &mac_stats->eth_mac.BroadcastFramesReceivedOK); + fbnic_set_counter(ð_mac_stats->FrameTooLongErrors, + &mac_stats->eth_mac.FrameTooLongErrors); +} + +static void +fbnic_get_eth_ctrl_stats(struct net_device *netdev, + struct ethtool_eth_ctrl_stats *eth_ctrl_stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + + mac_stats = &fbd->hw_stats.mac; + + fbd->mac->get_eth_ctrl_stats(fbd, false, &mac_stats->eth_ctrl); + + eth_ctrl_stats->MACControlFramesReceived = + mac_stats->eth_ctrl.MACControlFramesReceived.value; + eth_ctrl_stats->MACControlFramesTransmitted = + mac_stats->eth_ctrl.MACControlFramesTransmitted.value; +} + +static const struct ethtool_rmon_hist_range fbnic_rmon_ranges[] = { + { 0, 64 }, + { 65, 127 }, + { 128, 255 }, + { 256, 511 }, + { 512, 1023 }, + { 1024, 1518 }, + { 1519, 2047 }, + { 2048, 4095 }, + { 4096, 8191 }, + { 8192, 9216 }, + { 9217, FBNIC_MAX_JUMBO_FRAME_SIZE }, + {} +}; + +static void +fbnic_get_rmon_stats(struct net_device *netdev, + struct ethtool_rmon_stats *rmon_stats, + const struct ethtool_rmon_hist_range **ranges) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_stats *mac_stats; + struct fbnic_dev *fbd = fbn->fbd; + int i; + + mac_stats = &fbd->hw_stats.mac; + + fbd->mac->get_rmon_stats(fbd, false, &mac_stats->rmon); + + rmon_stats->undersize_pkts = + mac_stats->rmon.undersize_pkts.value; + rmon_stats->oversize_pkts = + mac_stats->rmon.oversize_pkts.value; + rmon_stats->fragments = + mac_stats->rmon.fragments.value; + rmon_stats->jabbers = + mac_stats->rmon.jabbers.value; + + for (i = 0; fbnic_rmon_ranges[i].high; i++) { + rmon_stats->hist[i] = mac_stats->rmon.hist[i].value; + rmon_stats->hist_tx[i] = mac_stats->rmon.hist_tx[i].value; + } + + *ranges = fbnic_rmon_ranges; +} + +static void fbnic_get_link_ext_stats(struct net_device *netdev, + struct ethtool_link_ext_stats *stats) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + stats->link_down_events = fbn->link_down_events; +} + +static const struct ethtool_ops fbnic_ethtool_ops = { + .cap_link_lanes_supported = true, + .supported_coalesce_params = ETHTOOL_COALESCE_USECS | + ETHTOOL_COALESCE_RX_MAX_FRAMES, + .supported_ring_params = ETHTOOL_RING_USE_TCP_DATA_SPLIT | + ETHTOOL_RING_USE_HDS_THRS, + .rxfh_max_num_contexts = FBNIC_RPC_RSS_TBL_COUNT, + .op_needs_rtnl = ETHTOOL_OP_NEEDS_RTNL_LINKSETTINGS | + ETHTOOL_OP_NEEDS_RTNL_GPAUSEPARAM | + ETHTOOL_OP_NEEDS_RTNL_SPAUSEPARAM | + ETHTOOL_OP_NEEDS_RTNL_SCHANNELS | + ETHTOOL_OP_NEEDS_RTNL_SRINGPARAM | + ETHTOOL_OP_NEEDS_RTNL_GLINK, + .get_drvinfo = fbnic_get_drvinfo, + .get_regs_len = fbnic_get_regs_len, + .get_regs = fbnic_get_regs, + .get_link = ethtool_op_get_link, + .get_link_ext_stats = fbnic_get_link_ext_stats, + .get_coalesce = fbnic_get_coalesce, + .set_coalesce = fbnic_set_coalesce, + .get_ringparam = fbnic_get_ringparam, + .set_ringparam = fbnic_set_ringparam, + .get_pause_stats = fbnic_get_pause_stats, + .get_pauseparam = fbnic_phylink_get_pauseparam, + .set_pauseparam = fbnic_phylink_set_pauseparam, + .self_test = fbnic_self_test, + .get_strings = fbnic_get_strings, + .get_ethtool_stats = fbnic_get_ethtool_stats, + .get_sset_count = fbnic_get_sset_count, + .get_rxnfc = fbnic_get_rxnfc, + .set_rxnfc = fbnic_set_rxnfc, + .get_rx_ring_count = fbnic_get_rx_ring_count, + .get_rxfh_key_size = fbnic_get_rxfh_key_size, + .get_rxfh_indir_size = fbnic_get_rxfh_indir_size, + .get_rxfh = fbnic_get_rxfh, + .set_rxfh = fbnic_set_rxfh, + .get_rxfh_fields = fbnic_get_rss_hash_opts, + .set_rxfh_fields = fbnic_set_rss_hash_opts, + .create_rxfh_context = fbnic_create_rxfh_context, + .modify_rxfh_context = fbnic_modify_rxfh_context, + .remove_rxfh_context = fbnic_remove_rxfh_context, + .get_channels = fbnic_get_channels, + .set_channels = fbnic_set_channels, + .get_ts_info = fbnic_get_ts_info, + .get_ts_stats = fbnic_get_ts_stats, + .get_tunable = fbnic_get_tunable, + .set_tunable = fbnic_set_tunable, + .get_link_ksettings = fbnic_phylink_ethtool_ksettings_get, + .get_fec_stats = fbnic_get_fec_stats, + .get_fecparam = fbnic_phylink_get_fecparam, + .get_module_eeprom_by_page = fbnic_get_module_eeprom_by_page, + .get_eth_phy_stats = fbnic_get_eth_phy_stats, + .get_eth_mac_stats = fbnic_get_eth_mac_stats, + .get_eth_ctrl_stats = fbnic_get_eth_ctrl_stats, + .get_rmon_stats = fbnic_get_rmon_stats, +}; + +void fbnic_set_ethtool_ops(struct net_device *dev) +{ + dev->ethtool_ops = &fbnic_ethtool_ops; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c new file mode 100644 index 000000000..283d25fae --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.c @@ -0,0 +1,2040 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/etherdevice.h> +#include <linux/delay.h> +#include <linux/dev_printk.h> +#include <linux/dma-mapping.h> +#include <linux/gfp.h> +#include <linux/types.h> + +#include "fbnic.h" +#include "fbnic_tlv.h" + +static void __fbnic_mbx_wr_desc(struct fbnic_dev *fbd, int mbx_idx, + int desc_idx, u64 desc) +{ + u32 desc_offset = FBNIC_IPC_MBX(mbx_idx, desc_idx); + + /* Write the upper 32b and then the lower 32b. Doing this the + * FW can then read lower, upper, lower to verify that the state + * of the descriptor wasn't changed mid-transaction. + */ + fw_wr32(fbd, desc_offset + 1, upper_32_bits(desc)); + fw_wrfl(fbd); + fw_wr32(fbd, desc_offset, lower_32_bits(desc)); +} + +static void __fbnic_mbx_invalidate_desc(struct fbnic_dev *fbd, int mbx_idx, + int desc_idx, u32 desc) +{ + u32 desc_offset = FBNIC_IPC_MBX(mbx_idx, desc_idx); + + /* For initialization we write the lower 32b of the descriptor first. + * This way we can set the state to mark it invalid before we clear the + * upper 32b. + */ + fw_wr32(fbd, desc_offset, desc); + fw_wrfl(fbd); + fw_wr32(fbd, desc_offset + 1, 0); +} + +u64 __fbnic_mbx_rd_desc(struct fbnic_dev *fbd, int mbx_idx, int desc_idx) +{ + u32 desc_offset = FBNIC_IPC_MBX(mbx_idx, desc_idx); + u64 desc; + + desc = fw_rd32(fbd, desc_offset); + desc |= (u64)fw_rd32(fbd, desc_offset + 1) << 32; + + return desc; +} + +static void fbnic_mbx_reset_desc_ring(struct fbnic_dev *fbd, int mbx_idx) +{ + int desc_idx; + + /* Disable DMA transactions from the device, + * and flush any transactions triggered during cleaning + */ + switch (mbx_idx) { + case FBNIC_IPC_MBX_RX_IDX: + wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG, + FBNIC_PUL_OB_TLP_HDR_AW_CFG_FLUSH); + break; + case FBNIC_IPC_MBX_TX_IDX: + wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG, + FBNIC_PUL_OB_TLP_HDR_AR_CFG_FLUSH); + break; + } + + wrfl(fbd); + + /* Initialize first descriptor to all 0s. Doing this gives us a + * solid stop for the firmware to hit when it is done looping + * through the ring. + */ + __fbnic_mbx_invalidate_desc(fbd, mbx_idx, 0, 0); + + /* We then fill the rest of the ring starting at the end and moving + * back toward descriptor 0 with skip descriptors that have no + * length nor address, and tell the firmware that they can skip + * them and just move past them to the one we initialized to 0. + */ + for (desc_idx = FBNIC_IPC_MBX_DESC_LEN; --desc_idx;) + __fbnic_mbx_invalidate_desc(fbd, mbx_idx, desc_idx, + FBNIC_IPC_MBX_DESC_FW_CMPL | + FBNIC_IPC_MBX_DESC_HOST_CMPL); +} + +void fbnic_mbx_init(struct fbnic_dev *fbd) +{ + int i; + + /* Initialize lock to protect Tx ring */ + spin_lock_init(&fbd->fw_tx_lock); + + /* Reset FW Capabilities */ + memset(&fbd->fw_cap, 0, sizeof(fbd->fw_cap)); + + /* Reinitialize mailbox memory */ + for (i = 0; i < FBNIC_IPC_MBX_INDICES; i++) + memset(&fbd->mbx[i], 0, sizeof(struct fbnic_fw_mbx)); + + /* Do not auto-clear the FW mailbox interrupt, let SW clear it */ + wr32(fbd, FBNIC_INTR_SW_AC_MODE(0), ~(1u << FBNIC_FW_MSIX_ENTRY)); + + /* Clear any stale causes in vector 0 as that is used for doorbell */ + wr32(fbd, FBNIC_INTR_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY); + + for (i = 0; i < FBNIC_IPC_MBX_INDICES; i++) + fbnic_mbx_reset_desc_ring(fbd, i); +} + +static int fbnic_mbx_map_msg(struct fbnic_dev *fbd, int mbx_idx, + struct fbnic_tlv_msg *msg, u16 length, u8 eom) +{ + struct fbnic_fw_mbx *mbx = &fbd->mbx[mbx_idx]; + u8 tail = mbx->tail; + dma_addr_t addr; + int direction; + + if (!mbx->ready || !fbnic_fw_present(fbd)) + return -ENODEV; + + direction = (mbx_idx == FBNIC_IPC_MBX_RX_IDX) ? DMA_FROM_DEVICE : + DMA_TO_DEVICE; + + if (mbx->head == ((tail + 1) % FBNIC_IPC_MBX_DESC_LEN)) + return -EBUSY; + + addr = dma_map_single(fbd->dev, msg, PAGE_SIZE, direction); + if (dma_mapping_error(fbd->dev, addr)) + return -ENOSPC; + + mbx->buf_info[tail].msg = msg; + mbx->buf_info[tail].addr = addr; + + mbx->tail = (tail + 1) % FBNIC_IPC_MBX_DESC_LEN; + + fw_wr32(fbd, FBNIC_IPC_MBX(mbx_idx, mbx->tail), 0); + + __fbnic_mbx_wr_desc(fbd, mbx_idx, tail, + FIELD_PREP(FBNIC_IPC_MBX_DESC_LEN_MASK, length) | + (addr & FBNIC_IPC_MBX_DESC_ADDR_MASK) | + (eom ? FBNIC_IPC_MBX_DESC_EOM : 0) | + FBNIC_IPC_MBX_DESC_HOST_CMPL); + + return 0; +} + +static void fbnic_mbx_unmap_and_free_msg(struct fbnic_dev *fbd, int mbx_idx, + int desc_idx) +{ + struct fbnic_fw_mbx *mbx = &fbd->mbx[mbx_idx]; + int direction; + + if (!mbx->buf_info[desc_idx].msg) + return; + + direction = (mbx_idx == FBNIC_IPC_MBX_RX_IDX) ? DMA_FROM_DEVICE : + DMA_TO_DEVICE; + dma_unmap_single(fbd->dev, mbx->buf_info[desc_idx].addr, + PAGE_SIZE, direction); + + free_page((unsigned long)mbx->buf_info[desc_idx].msg); + mbx->buf_info[desc_idx].msg = NULL; +} + +static void fbnic_mbx_clean_desc_ring(struct fbnic_dev *fbd, int mbx_idx) +{ + int i; + + fbnic_mbx_reset_desc_ring(fbd, mbx_idx); + + for (i = FBNIC_IPC_MBX_DESC_LEN; i--;) + fbnic_mbx_unmap_and_free_msg(fbd, mbx_idx, i); +} + +void fbnic_mbx_clean(struct fbnic_dev *fbd) +{ + int i; + + for (i = 0; i < FBNIC_IPC_MBX_INDICES; i++) + fbnic_mbx_clean_desc_ring(fbd, i); +} + +#define FBNIC_MBX_MAX_PAGE_SIZE FIELD_MAX(FBNIC_IPC_MBX_DESC_LEN_MASK) +#define FBNIC_RX_PAGE_SIZE min_t(int, PAGE_SIZE, FBNIC_MBX_MAX_PAGE_SIZE) + +static int fbnic_mbx_alloc_rx_msgs(struct fbnic_dev *fbd) +{ + struct fbnic_fw_mbx *rx_mbx = &fbd->mbx[FBNIC_IPC_MBX_RX_IDX]; + u8 tail = rx_mbx->tail, head = rx_mbx->head, count; + int err = 0; + + /* Do nothing if mailbox is not ready, or we already have pages on + * the ring that can be used by the firmware + */ + if (!rx_mbx->ready) + return -ENODEV; + + /* Fill all but 1 unused descriptors in the Rx queue. */ + count = (head - tail - 1) & (FBNIC_IPC_MBX_DESC_LEN - 1); + while (!err && count--) { + struct fbnic_tlv_msg *msg; + + msg = (struct fbnic_tlv_msg *)__get_free_page(GFP_KERNEL); + if (!msg) { + err = -ENOMEM; + break; + } + + err = fbnic_mbx_map_msg(fbd, FBNIC_IPC_MBX_RX_IDX, msg, + FBNIC_RX_PAGE_SIZE, 0); + if (err) + free_page((unsigned long)msg); + } + + return err; +} + +static int fbnic_mbx_map_tlv_msg(struct fbnic_dev *fbd, + struct fbnic_tlv_msg *msg) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&fbd->fw_tx_lock, flags); + + err = fbnic_mbx_map_msg(fbd, FBNIC_IPC_MBX_TX_IDX, msg, + le16_to_cpu(msg->hdr.len) * sizeof(u32), 1); + + spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); + + return err; +} + +static int fbnic_mbx_set_cmpl_slot(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data) +{ + struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + int free = -EXFULL; + int i; + + if (!tx_mbx->ready) + return -ENODEV; + + for (i = 0; i < FBNIC_MBX_CMPL_SLOTS; i++) { + if (!fbd->cmpl_data[i]) + free = i; + else if (fbd->cmpl_data[i]->msg_type == cmpl_data->msg_type) + return -EEXIST; + } + + if (free == -EXFULL) + return -EXFULL; + + fbd->cmpl_data[free] = cmpl_data; + + return 0; +} + +static void fbnic_mbx_clear_cmpl_slot(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data) +{ + int i; + + for (i = 0; i < FBNIC_MBX_CMPL_SLOTS; i++) { + if (fbd->cmpl_data[i] == cmpl_data) { + fbd->cmpl_data[i] = NULL; + break; + } + } +} + +static void fbnic_mbx_process_tx_msgs(struct fbnic_dev *fbd) +{ + struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + u8 head = tx_mbx->head; + u64 desc; + + while (head != tx_mbx->tail) { + desc = __fbnic_mbx_rd_desc(fbd, FBNIC_IPC_MBX_TX_IDX, head); + if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) + break; + + fbnic_mbx_unmap_and_free_msg(fbd, FBNIC_IPC_MBX_TX_IDX, head); + + head++; + head %= FBNIC_IPC_MBX_DESC_LEN; + } + + /* Record head for next interrupt */ + tx_mbx->head = head; +} + +int fbnic_mbx_set_cmpl(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&fbd->fw_tx_lock, flags); + err = fbnic_mbx_set_cmpl_slot(fbd, cmpl_data); + spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); + + return err; +} + +static int fbnic_mbx_map_req_w_cmpl(struct fbnic_dev *fbd, + struct fbnic_tlv_msg *msg, + struct fbnic_fw_completion *cmpl_data) +{ + unsigned long flags; + int err; + + spin_lock_irqsave(&fbd->fw_tx_lock, flags); + if (cmpl_data) { + err = fbnic_mbx_set_cmpl_slot(fbd, cmpl_data); + if (err) + goto unlock_mbx; + } + + err = fbnic_mbx_map_msg(fbd, FBNIC_IPC_MBX_TX_IDX, msg, + le16_to_cpu(msg->hdr.len) * sizeof(u32), 1); + + /* If we successfully reserved a completion and msg failed + * then clear completion data for next caller + */ + if (err && cmpl_data) + fbnic_mbx_clear_cmpl_slot(fbd, cmpl_data); + +unlock_mbx: + spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); + + return err; +} + +void fbnic_mbx_clear_cmpl(struct fbnic_dev *fbd, + struct fbnic_fw_completion *fw_cmpl) +{ + unsigned long flags; + + spin_lock_irqsave(&fbd->fw_tx_lock, flags); + fbnic_mbx_clear_cmpl_slot(fbd, fw_cmpl); + spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); +} + +static void fbnic_fw_release_cmpl_data(struct kref *kref) +{ + struct fbnic_fw_completion *cmpl_data; + + cmpl_data = container_of(kref, struct fbnic_fw_completion, + ref_count); + kfree(cmpl_data); +} + +static struct fbnic_fw_completion * +fbnic_fw_get_cmpl_by_type(struct fbnic_dev *fbd, u32 msg_type) +{ + struct fbnic_fw_completion *cmpl_data = NULL; + unsigned long flags; + int i; + + spin_lock_irqsave(&fbd->fw_tx_lock, flags); + for (i = 0; i < FBNIC_MBX_CMPL_SLOTS; i++) { + if (fbd->cmpl_data[i] && + fbd->cmpl_data[i]->msg_type == msg_type) { + cmpl_data = fbd->cmpl_data[i]; + kref_get(&cmpl_data->ref_count); + break; + } + } + + spin_unlock_irqrestore(&fbd->fw_tx_lock, flags); + + return cmpl_data; +} + +/** + * fbnic_fw_xmit_test_msg - Create and transmit a test message to FW mailbox + * @fbd: FBNIC device structure + * @cmpl: fw completion struct + * + * Return: zero on success, negative value on failure + * + * Generates a single page mailbox test message and places it in the Tx + * mailbox queue. Expectation is that the FW will validate that the nested + * value matches the external values, and then will echo them back to us. + * + * Also sets a completion slot for use in the completion wait calls when + * the cmpl arg is non-NULL. + */ +int fbnic_fw_xmit_test_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl) +{ + struct fbnic_tlv_msg *test_msg; + int err; + + test_msg = fbnic_tlv_test_create(fbd); + if (!test_msg) + return -ENOMEM; + + err = fbnic_mbx_map_req_w_cmpl(fbd, test_msg, cmpl); + if (err) + free_page((unsigned long)test_msg); + + return err; +} + +/** + * fbnic_fw_xmit_simple_msg - Transmit a simple single TLV message w/o data + * @fbd: FBNIC device structure + * @msg_type: ENUM value indicating message type to send + * + * Return: + * One the following values: + * -EOPNOTSUPP: Is not ASIC so mailbox is not supported + * -ENODEV: Device I/O error + * -ENOMEM: Failed to allocate message + * -EBUSY: No space in mailbox + * -ENOSPC: DMA mapping failed + * + * This function sends a single TLV header indicating the host wants to take + * some action. However there are no other side effects which means that any + * response will need to be caught via a completion if this action is + * expected to kick off a resultant action. + */ +static int fbnic_fw_xmit_simple_msg(struct fbnic_dev *fbd, u32 msg_type) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + msg = fbnic_tlv_msg_alloc(msg_type); + if (!msg) + return -ENOMEM; + + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + free_page((unsigned long)msg); + + return err; +} + +static int fbnic_mbx_init_desc_ring(struct fbnic_dev *fbd, int mbx_idx) +{ + u8 tlp_attr = fbd->relaxed_ord ? FBNIC_TLP_ATTR_RO : 0; + struct fbnic_fw_mbx *mbx = &fbd->mbx[mbx_idx]; + + mbx->ready = true; + + switch (mbx_idx) { + case FBNIC_IPC_MBX_RX_IDX: + /* Enable DMA writes from the device */ + wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AW_CFG, + FBNIC_PUL_OB_TLP_HDR_AW_CFG_BME | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AW_CFG_RDE_ATTR, + tlp_attr) | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AW_CFG_TQM_ATTR, + tlp_attr)); + + /* Make sure we have a page for the FW to write to */ + return fbnic_mbx_alloc_rx_msgs(fbd); + case FBNIC_IPC_MBX_TX_IDX: + /* Enable DMA reads from the device */ + wr32(fbd, FBNIC_PUL_OB_TLP_HDR_AR_CFG, + FBNIC_PUL_OB_TLP_HDR_AR_CFG_BME | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AR_CFG_TDE_ATTR, + tlp_attr) | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AR_CFG_RQM_ATTR, + tlp_attr) | + FIELD_PREP(FBNIC_PUL_OB_TLP_HDR_AR_CFG_TQM_ATTR, + tlp_attr)); + break; + } + + return 0; +} + +static bool fbnic_mbx_event(struct fbnic_dev *fbd) +{ + /* We only need to do this on the first interrupt following reset. + * this primes the mailbox so that we will have cleared all the + * skip descriptors. + */ + if (!(rd32(fbd, FBNIC_INTR_STATUS(0)) & (1u << FBNIC_FW_MSIX_ENTRY))) + return false; + + wr32(fbd, FBNIC_INTR_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY); + + return true; +} + +/** + * fbnic_fw_xmit_ownership_msg - Create and transmit a host ownership message + * to FW mailbox + * + * @fbd: FBNIC device structure + * @take_ownership: take/release the ownership + * + * Return: zero on success, negative value on failure + * + * Notifies the firmware that the driver either takes ownership of the NIC + * (when @take_ownership is true) or releases it. + */ +int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership) +{ + unsigned long req_time = jiffies; + struct fbnic_tlv_msg *msg; + int err = 0; + + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_OWNERSHIP_REQ); + if (!msg) + return -ENOMEM; + + if (take_ownership) { + err = fbnic_tlv_attr_put_flag(msg, FBNIC_FW_OWNERSHIP_FLAG); + if (err) + goto free_message; + } + + /* Initialize heartbeat, set last response to 1 second in the past + * so that we will trigger a timeout if the firmware doesn't respond + */ + fbd->last_heartbeat_response = req_time - HZ; + fbd->last_heartbeat_request = req_time; + + /* Set prev_firmware_time to 0 to avoid triggering firmware crash + * detection until we receive the second uptime in a heartbeat resp. + */ + fbd->prev_firmware_time = 0; + + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + + /* Set heartbeat detection based on if we are taking ownership */ + fbd->fw_heartbeat_enabled = take_ownership; + + return err; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_fw_cap_resp_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_VERSION), + FBNIC_TLV_ATTR_FLAG(FBNIC_FW_CAP_RESP_BMC_PRESENT), + FBNIC_TLV_ATTR_MAC_ADDR(FBNIC_FW_CAP_RESP_BMC_MAC_ADDR), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_CAP_RESP_BMC_MAC_ARRAY), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_STORED_VERSION), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_ACTIVE_FW_SLOT), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_CAP_RESP_VERSION_COMMIT_STR, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_BMC_ALL_MULTI), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_FW_LINK_SPEED), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_FW_LINK_FEC), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_CAP_RESP_STORED_COMMIT_STR, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_CMRT_VERSION), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_STORED_CMRT_VERSION), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_CAP_RESP_CMRT_COMMIT_STR, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_CAP_RESP_STORED_CMRT_COMMIT_STR, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_UEFI_VERSION), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_CAP_RESP_UEFI_COMMIT_STR, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_CAP_RESP_ANTI_ROLLBACK_VERSION), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_bmc_addrs(u8 bmc_mac_addr[][ETH_ALEN], + struct fbnic_tlv_msg *attr, int len) +{ + int attr_len = le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1; + struct fbnic_tlv_msg *mac_results[8]; + int err, i = 0; + + /* Make sure we have enough room to process all the MAC addresses */ + if (len > 8) + return -ENOSPC; + + /* Parse the array */ + err = fbnic_tlv_attr_parse_array(&attr[1], attr_len, mac_results, + fbnic_fw_cap_resp_index, + FBNIC_FW_CAP_RESP_BMC_MAC_ADDR, len); + if (err) + return err; + + /* Copy results into MAC addr array */ + for (i = 0; i < len && mac_results[i]; i++) + fbnic_tlv_attr_addr_copy(bmc_mac_addr[i], mac_results[i]); + + /* Zero remaining unused addresses */ + while (i < len) + eth_zero_addr(bmc_mac_addr[i++]); + + return 0; +} + +static int fbnic_fw_parse_cap_resp(void *opaque, struct fbnic_tlv_msg **results) +{ + u32 all_multi = 0, version = 0; + struct fbnic_dev *fbd = opaque; + bool bmc_present; + int err; + + version = fta_get_uint(results, FBNIC_FW_CAP_RESP_VERSION); + fbd->fw_cap.running.mgmt.version = version; + if (!fbd->fw_cap.running.mgmt.version) + return -EINVAL; + + if (fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE) { + char required_ver[FBNIC_FW_VER_MAX_SIZE]; + char running_ver[FBNIC_FW_VER_MAX_SIZE]; + + fbnic_mk_fw_ver_str(fbd->fw_cap.running.mgmt.version, + running_ver); + fbnic_mk_fw_ver_str(MIN_FW_VER_CODE, required_ver); + dev_err(fbd->dev, "Device firmware version(%s) is older than minimum required version(%s)\n", + running_ver, required_ver); + /* Disable TX mailbox to prevent card use until firmware is + * updated. + */ + fbd->mbx[FBNIC_IPC_MBX_TX_IDX].ready = false; + return -EINVAL; + } + + if (fta_get_str(results, FBNIC_FW_CAP_RESP_VERSION_COMMIT_STR, + fbd->fw_cap.running.mgmt.commit, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE) <= 0) + dev_warn(fbd->dev, "Firmware did not send mgmt commit!\n"); + + version = fta_get_uint(results, FBNIC_FW_CAP_RESP_STORED_VERSION); + fbd->fw_cap.stored.mgmt.version = version; + fta_get_str(results, FBNIC_FW_CAP_RESP_STORED_COMMIT_STR, + fbd->fw_cap.stored.mgmt.commit, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE); + + version = fta_get_uint(results, FBNIC_FW_CAP_RESP_CMRT_VERSION); + fbd->fw_cap.running.bootloader.version = version; + fta_get_str(results, FBNIC_FW_CAP_RESP_CMRT_COMMIT_STR, + fbd->fw_cap.running.bootloader.commit, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE); + + version = fta_get_uint(results, FBNIC_FW_CAP_RESP_STORED_CMRT_VERSION); + fbd->fw_cap.stored.bootloader.version = version; + fta_get_str(results, FBNIC_FW_CAP_RESP_STORED_CMRT_COMMIT_STR, + fbd->fw_cap.stored.bootloader.commit, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE); + + version = fta_get_uint(results, FBNIC_FW_CAP_RESP_UEFI_VERSION); + fbd->fw_cap.stored.undi.version = version; + fta_get_str(results, FBNIC_FW_CAP_RESP_UEFI_COMMIT_STR, + fbd->fw_cap.stored.undi.commit, + FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE); + + fbd->fw_cap.active_slot = + fta_get_uint(results, FBNIC_FW_CAP_RESP_ACTIVE_FW_SLOT); + fbd->fw_cap.link_speed = + fta_get_uint(results, FBNIC_FW_CAP_RESP_FW_LINK_SPEED); + fbd->fw_cap.link_fec = + fta_get_uint(results, FBNIC_FW_CAP_RESP_FW_LINK_FEC); + + bmc_present = !!results[FBNIC_FW_CAP_RESP_BMC_PRESENT]; + if (bmc_present) { + struct fbnic_tlv_msg *attr; + + attr = results[FBNIC_FW_CAP_RESP_BMC_MAC_ARRAY]; + if (!attr) + return -EINVAL; + + err = fbnic_fw_parse_bmc_addrs(fbd->fw_cap.bmc_mac_addr, + attr, 4); + if (err) + return err; + + all_multi = + fta_get_uint(results, FBNIC_FW_CAP_RESP_BMC_ALL_MULTI); + } else { + memset(fbd->fw_cap.bmc_mac_addr, 0, + sizeof(fbd->fw_cap.bmc_mac_addr)); + } + + fbd->fw_cap.bmc_present = bmc_present; + + if (results[FBNIC_FW_CAP_RESP_BMC_ALL_MULTI] || !bmc_present) + fbd->fw_cap.all_multi = all_multi; + + fbd->fw_cap.anti_rollback_version = + fta_get_uint(results, FBNIC_FW_CAP_RESP_ANTI_ROLLBACK_VERSION); + + /* Always assume we need a BMC reinit */ + fbd->fw_cap.need_bmc_tcam_reinit = true; + + return 0; +} + +static const struct fbnic_tlv_index fbnic_ownership_resp_index[] = { + FBNIC_TLV_ATTR_U64(FBNIC_FW_OWNERSHIP_TIME), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_ownership_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_dev *fbd = (struct fbnic_dev *)opaque; + + /* Count the ownership response as a heartbeat reply */ + fbd->last_heartbeat_response = jiffies; + + /* Capture firmware time for logging and firmware crash check */ + fbd->firmware_time = fta_get_uint(results, FBNIC_FW_OWNERSHIP_TIME); + + return 0; +} + +static const struct fbnic_tlv_index fbnic_heartbeat_resp_index[] = { + FBNIC_TLV_ATTR_U64(FBNIC_FW_HEARTBEAT_UPTIME), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_heartbeat_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_dev *fbd = (struct fbnic_dev *)opaque; + + fbd->last_heartbeat_response = jiffies; + + /* Capture firmware time for logging and firmware crash check */ + fbd->firmware_time = fta_get_uint(results, FBNIC_FW_HEARTBEAT_UPTIME); + + return 0; +} + +static int fbnic_fw_xmit_heartbeat_message(struct fbnic_dev *fbd) +{ + unsigned long req_time = jiffies; + struct fbnic_tlv_msg *msg; + int err = 0; + + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_HEARTBEAT_REQ); + if (!msg) + return -ENOMEM; + + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + + fbd->last_heartbeat_request = req_time; + fbd->prev_firmware_time = fbd->firmware_time; + + return err; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static bool fbnic_fw_heartbeat_current(struct fbnic_dev *fbd) +{ + unsigned long last_response = fbd->last_heartbeat_response; + unsigned long last_request = fbd->last_heartbeat_request; + + return !time_before(last_response, last_request); +} + +int fbnic_fw_init_heartbeat(struct fbnic_dev *fbd, bool poll) +{ + int err = -ETIMEDOUT; + int attempts = 50; + + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + while (attempts--) { + msleep(200); + if (poll) + fbnic_mbx_poll(fbd); + + if (!fbnic_fw_heartbeat_current(fbd)) + continue; + + /* Place new message on mailbox to elicit a response */ + err = fbnic_fw_xmit_heartbeat_message(fbd); + if (err) + dev_warn(fbd->dev, + "Failed to send heartbeat message: %d\n", + err); + break; + } + + return err; +} + +void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd) +{ + unsigned long last_request = fbd->last_heartbeat_request; + int err; + + /* Do not check heartbeat or send another request until current + * period has expired. Otherwise we might start spamming requests. + */ + if (time_is_after_jiffies(last_request + FW_HEARTBEAT_PERIOD)) + return; + + /* We already reported no mailbox. Wait for it to come back */ + if (!fbd->fw_heartbeat_enabled) + return; + + /* Was the last heartbeat response long time ago? */ + if (!fbnic_fw_heartbeat_current(fbd) || + fbd->firmware_time < fbd->prev_firmware_time) { + dev_warn(fbd->dev, + "Firmware did not respond to heartbeat message\n"); + fbd->fw_heartbeat_enabled = false; + } + + /* Place new message on mailbox to elicit a response */ + err = fbnic_fw_xmit_heartbeat_message(fbd); + if (err) + dev_warn(fbd->dev, "Failed to send heartbeat message\n"); +} + +/** + * fbnic_fw_xmit_coredump_info_msg - Create and transmit a coredump info message + * @fbd: FBNIC device structure + * @cmpl_data: Structure to store info in + * @force: Force coredump event if one hasn't already occurred + * + * Return: zero on success, negative errno on failure + * + * Asks the FW for info related to coredump. If a coredump doesn't exist it + * can optionally force one if force is true. + */ +int fbnic_fw_xmit_coredump_info_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + bool force) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_REQ); + if (!msg) + return -ENOMEM; + + if (force) { + err = fbnic_tlv_attr_put_flag(msg, FBNIC_FW_COREDUMP_REQ_INFO_CREATE); + if (err) + goto free_msg; + } + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_msg; + + return 0; + +free_msg: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_coredump_info_resp_index[] = { + FBNIC_TLV_ATTR_FLAG(FBNIC_FW_COREDUMP_INFO_AVAILABLE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_COREDUMP_INFO_SIZE), + FBNIC_TLV_ATTR_S32(FBNIC_FW_COREDUMP_INFO_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int +fbnic_fw_parse_coredump_info_resp(void *opaque, struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + u32 msg_type; + s32 err; + + /* Verify we have a completion pointer to provide with data */ + msg_type = FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_RESP; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + err = fta_get_sint(results, FBNIC_FW_COREDUMP_INFO_ERROR); + if (err) + goto msg_err; + + if (!results[FBNIC_FW_COREDUMP_INFO_AVAILABLE]) { + err = -ENOENT; + goto msg_err; + } + + cmpl_data->u.coredump_info.size = + fta_get_uint(results, FBNIC_FW_COREDUMP_INFO_SIZE); + +msg_err: + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + +/** + * fbnic_fw_xmit_coredump_read_msg - Create and transmit a coredump read request + * @fbd: FBNIC device structure + * @cmpl_data: Completion struct to store coredump + * @offset: Offset into coredump requested + * @length: Length of section of coredump to fetch + * + * Return: zero on success, negative errno on failure + * + * Asks the firmware to provide a section of the coredump back in a message. + * The response will have an offset and size matching the values provided. + */ +int fbnic_fw_xmit_coredump_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 offset, u32 length) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_COREDUMP_READ_REQ); + if (!msg) + return -ENOMEM; + + if (offset) { + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_COREDUMP_READ_OFFSET, + offset); + if (err) + goto free_message; + } + + if (length) { + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_COREDUMP_READ_LENGTH, + length); + if (err) + goto free_message; + } + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_coredump_resp_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_COREDUMP_READ_OFFSET), + FBNIC_TLV_ATTR_U32(FBNIC_FW_COREDUMP_READ_LENGTH), + FBNIC_TLV_ATTR_RAW_DATA(FBNIC_FW_COREDUMP_READ_DATA), + FBNIC_TLV_ATTR_S32(FBNIC_FW_COREDUMP_READ_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_coredump_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + u32 index, last_offset, last_length; + struct fbnic_dev *fbd = opaque; + struct fbnic_tlv_msg *data_hdr; + u32 length, offset; + u32 msg_type; + s32 err; + + /* Verify we have a completion pointer to provide with data */ + msg_type = FBNIC_TLV_MSG_ID_COREDUMP_READ_RESP; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + err = fta_get_sint(results, FBNIC_FW_COREDUMP_READ_ERROR); + if (err) + goto msg_err; + + data_hdr = results[FBNIC_FW_COREDUMP_READ_DATA]; + if (!data_hdr) { + err = -ENODATA; + goto msg_err; + } + + offset = fta_get_uint(results, FBNIC_FW_COREDUMP_READ_OFFSET); + length = fta_get_uint(results, FBNIC_FW_COREDUMP_READ_LENGTH); + + if (length > le16_to_cpu(data_hdr->hdr.len) - sizeof(u32)) { + dev_err(fbd->dev, "length greater than size of message\n"); + err = -EINVAL; + goto msg_err; + } + + /* Only the last offset can have a length != stride */ + last_length = + (cmpl_data->u.coredump.size % cmpl_data->u.coredump.stride) ? : + cmpl_data->u.coredump.stride; + last_offset = cmpl_data->u.coredump.size - last_length; + + /* Verify offset and length */ + if (offset % cmpl_data->u.coredump.stride || offset > last_offset) { + dev_err(fbd->dev, "offset %d out of range\n", offset); + err = -EINVAL; + } else if (length != ((offset == last_offset) ? + last_length : cmpl_data->u.coredump.stride)) { + dev_err(fbd->dev, "length %d out of range for offset %d\n", + length, offset); + err = -EINVAL; + } + if (err) + goto msg_err; + + /* If data pointer is NULL it is already filled, just skip the copy */ + index = offset / cmpl_data->u.coredump.stride; + if (!cmpl_data->u.coredump.data[index]) + goto msg_err; + + /* Copy data and mark index filled by setting pointer to NULL */ + memcpy(cmpl_data->u.coredump.data[index], + fbnic_tlv_attr_get_value_ptr(data_hdr), length); + cmpl_data->u.coredump.data[index] = NULL; + +msg_err: + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + +int fbnic_fw_xmit_fw_start_upgrade(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + unsigned int id, unsigned int len) +{ + struct fbnic_tlv_msg *msg; + int err; + + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + if (!len) + return -EINVAL; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_FW_START_UPGRADE_REQ); + if (!msg) + return -ENOMEM; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_START_UPGRADE_SECTION, id); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_START_UPGRADE_IMAGE_LENGTH, + len); + if (err) + goto free_message; + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_fw_start_upgrade_resp_index[] = { + FBNIC_TLV_ATTR_S32(FBNIC_FW_START_UPGRADE_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_fw_start_upgrade_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + u32 msg_type; + s32 err; + + /* Verify we have a completion pointer */ + msg_type = FBNIC_TLV_MSG_ID_FW_START_UPGRADE_REQ; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + /* Check for errors */ + err = fta_get_sint(results, FBNIC_FW_START_UPGRADE_ERROR); + + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return 0; +} + +int fbnic_fw_xmit_fw_write_chunk(struct fbnic_dev *fbd, + const u8 *data, u32 offset, u16 length, + int cancel_error) +{ + struct fbnic_tlv_msg *msg; + int err; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_RESP); + if (!msg) + return -ENOMEM; + + /* Report error to FW to cancel upgrade */ + if (cancel_error) { + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_WRITE_CHUNK_ERROR, + cancel_error); + if (err) + goto free_message; + } + + if (data) { + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_WRITE_CHUNK_OFFSET, + offset); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_WRITE_CHUNK_LENGTH, + length); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_value(msg, FBNIC_FW_WRITE_CHUNK_DATA, + data + offset, length); + if (err) + goto free_message; + } + + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_fw_write_chunk_req_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_WRITE_CHUNK_OFFSET), + FBNIC_TLV_ATTR_U32(FBNIC_FW_WRITE_CHUNK_LENGTH), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_fw_write_chunk_req(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + u32 msg_type; + u32 offset; + u32 length; + + /* Verify we have a completion pointer */ + msg_type = FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_REQ; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + /* Pull length/offset pair and mark it as complete */ + offset = fta_get_uint(results, FBNIC_FW_WRITE_CHUNK_OFFSET); + length = fta_get_uint(results, FBNIC_FW_WRITE_CHUNK_LENGTH); + cmpl_data->u.fw_update.offset = offset; + cmpl_data->u.fw_update.length = length; + + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return 0; +} + +static const struct fbnic_tlv_index fbnic_fw_finish_upgrade_req_index[] = { + FBNIC_TLV_ATTR_S32(FBNIC_FW_FINISH_UPGRADE_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_fw_finish_upgrade_req(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + u32 msg_type; + s32 err; + + /* Verify we have a completion pointer */ + msg_type = FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_REQ; + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, msg_type); + if (!cmpl_data) + return -ENOSPC; + + /* Check for errors */ + err = fta_get_sint(results, FBNIC_FW_FINISH_UPGRADE_ERROR); + + /* Close out update by incrementing offset by length which should + * match the total size of the component. Set length to 0 since no + * new chunks will be requested. + */ + cmpl_data->u.fw_update.offset += cmpl_data->u.fw_update.length; + cmpl_data->u.fw_update.length = 0; + + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return 0; +} + +/** + * fbnic_fw_xmit_qsfp_read_msg - Transmit a QSFP read request + * @fbd: FBNIC device structure + * @cmpl_data: Structure to store EEPROM response in + * @page: Refers to page number on page enabled QSFP modules + * @bank: Refers to a collection of pages + * @offset: Offset into QSFP EEPROM requested + * @length: Length of section of QSFP EEPROM to fetch + * + * Return: zero on success, negative value on failure + * + * Asks the firmware to provide a section of the QSFP EEPROM back in a + * message. The response will have an offset and size matching the values + * provided. + */ +int fbnic_fw_xmit_qsfp_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 page, u32 bank, u32 offset, u32 length) +{ + struct fbnic_tlv_msg *msg; + int err = 0; + + if (!length || length > TLV_MAX_DATA) + return -EINVAL; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_QSFP_READ_REQ); + if (!msg) + return -ENOMEM; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_BANK, bank); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_PAGE, page); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_OFFSET, offset); + if (err) + goto free_message; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_QSFP_LENGTH, length); + if (err) + goto free_message; + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_qsfp_read_resp_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_BANK), + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_PAGE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_OFFSET), + FBNIC_TLV_ATTR_U32(FBNIC_FW_QSFP_LENGTH), + FBNIC_TLV_ATTR_RAW_DATA(FBNIC_FW_QSFP_DATA), + FBNIC_TLV_ATTR_S32(FBNIC_FW_QSFP_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_qsfp_read_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + struct fbnic_tlv_msg *data_hdr; + u32 length, offset, page, bank; + u8 *data; + s32 err; + + /* Verify we have a completion pointer to provide with data */ + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, + FBNIC_TLV_MSG_ID_QSFP_READ_RESP); + if (!cmpl_data) + return -ENOSPC; + + bank = fta_get_uint(results, FBNIC_FW_QSFP_BANK); + if (bank != cmpl_data->u.qsfp.bank) { + dev_warn(fbd->dev, "bank not equal to bank requested: %d vs %d\n", + bank, cmpl_data->u.qsfp.bank); + err = -EINVAL; + goto msg_err; + } + + page = fta_get_uint(results, FBNIC_FW_QSFP_PAGE); + if (page != cmpl_data->u.qsfp.page) { + dev_warn(fbd->dev, "page not equal to page requested: %d vs %d\n", + page, cmpl_data->u.qsfp.page); + err = -EINVAL; + goto msg_err; + } + + offset = fta_get_uint(results, FBNIC_FW_QSFP_OFFSET); + length = fta_get_uint(results, FBNIC_FW_QSFP_LENGTH); + + if (length != cmpl_data->u.qsfp.length || + offset != cmpl_data->u.qsfp.offset) { + dev_warn(fbd->dev, + "offset/length not equal to size requested: %d/%d vs %d/%d\n", + offset, length, + cmpl_data->u.qsfp.offset, cmpl_data->u.qsfp.length); + err = -EINVAL; + goto msg_err; + } + + err = fta_get_sint(results, FBNIC_FW_QSFP_ERROR); + if (err) + goto msg_err; + + data_hdr = results[FBNIC_FW_QSFP_DATA]; + if (!data_hdr) { + err = -ENODATA; + goto msg_err; + } + + /* Copy data */ + data = fbnic_tlv_attr_get_value_ptr(data_hdr); + memcpy(cmpl_data->u.qsfp.data, data, length); +msg_err: + cmpl_data->result = err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + +/** + * fbnic_fw_xmit_tsene_read_msg - Create and transmit a sensor read request + * @fbd: FBNIC device structure + * @cmpl_data: Completion data structure to store sensor response + * + * Asks the firmware to provide an update with the latest sensor data. + * The response will contain temperature and voltage readings. + * + * Return: 0 on success, negative error value on failure + */ +int fbnic_fw_xmit_tsene_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data) +{ + struct fbnic_tlv_msg *msg; + int err; + + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_TSENE_READ_REQ); + if (!msg) + return -ENOMEM; + + err = fbnic_mbx_map_req_w_cmpl(fbd, msg, cmpl_data); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static const struct fbnic_tlv_index fbnic_tsene_read_resp_index[] = { + FBNIC_TLV_ATTR_S32(FBNIC_FW_TSENE_THERM), + FBNIC_TLV_ATTR_S32(FBNIC_FW_TSENE_VOLT), + FBNIC_TLV_ATTR_S32(FBNIC_FW_TSENE_ERROR), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_parse_tsene_read_resp(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl_data; + struct fbnic_dev *fbd = opaque; + s32 err_resp; + int err = 0; + + /* Verify we have a completion pointer to provide with data */ + cmpl_data = fbnic_fw_get_cmpl_by_type(fbd, + FBNIC_TLV_MSG_ID_TSENE_READ_RESP); + if (!cmpl_data) + return -ENOSPC; + + err_resp = fta_get_sint(results, FBNIC_FW_TSENE_ERROR); + if (err_resp) + goto msg_err; + + if (!results[FBNIC_FW_TSENE_THERM] || !results[FBNIC_FW_TSENE_VOLT]) { + err = -EINVAL; + goto msg_err; + } + + cmpl_data->u.tsene.millidegrees = + fta_get_sint(results, FBNIC_FW_TSENE_THERM); + cmpl_data->u.tsene.millivolts = + fta_get_sint(results, FBNIC_FW_TSENE_VOLT); + +msg_err: + cmpl_data->result = err_resp ? : err; + complete(&cmpl_data->done); + fbnic_fw_put_cmpl(cmpl_data); + + return err; +} + +static const struct fbnic_tlv_index fbnic_fw_log_req_index[] = { + FBNIC_TLV_ATTR_U32(FBNIC_FW_LOG_MSEC), + FBNIC_TLV_ATTR_U64(FBNIC_FW_LOG_INDEX), + FBNIC_TLV_ATTR_STRING(FBNIC_FW_LOG_MSG, FBNIC_FW_LOG_MAX_SIZE), + FBNIC_TLV_ATTR_U32(FBNIC_FW_LOG_LENGTH), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_LOG_MSEC_ARRAY), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_LOG_INDEX_ARRAY), + FBNIC_TLV_ATTR_ARRAY(FBNIC_FW_LOG_MSG_ARRAY), + FBNIC_TLV_ATTR_LAST +}; + +static int fbnic_fw_process_log_array(struct fbnic_tlv_msg **results, + u16 length, u16 arr_type_idx, + u16 attr_type_idx, + struct fbnic_tlv_msg **tlv_array_out) +{ + struct fbnic_tlv_msg *attr; + int attr_len; + int err; + + if (!results[attr_type_idx]) + return -EINVAL; + + tlv_array_out[0] = results[attr_type_idx]; + + if (!length) + return 0; + + if (!results[arr_type_idx]) + return -EINVAL; + + attr = results[arr_type_idx]; + attr_len = le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1; + err = fbnic_tlv_attr_parse_array(&attr[1], attr_len, &tlv_array_out[1], + fbnic_fw_log_req_index, + attr_type_idx, + length); + if (err) + return err; + + return 0; +} + +static int fbnic_fw_parse_logs(struct fbnic_dev *fbd, + struct fbnic_tlv_msg **msec_tlv, + struct fbnic_tlv_msg **index_tlv, + struct fbnic_tlv_msg **log_tlv, + int count) +{ + int i; + + for (i = 0; i < count; i++) { + char log[FBNIC_FW_LOG_MAX_SIZE]; + ssize_t len; + u64 index; + u32 msec; + int err; + + if (!msec_tlv[i] || !index_tlv[i] || !log_tlv[i]) { + dev_warn(fbd->dev, "Received log message with missing attributes!\n"); + return -EINVAL; + } + + index = fbnic_tlv_attr_get_signed(index_tlv[i], 0); + msec = fbnic_tlv_attr_get_signed(msec_tlv[i], 0); + len = fbnic_tlv_attr_get_string(log_tlv[i], log, + FBNIC_FW_LOG_MAX_SIZE); + if (len < 0) + return len; + + err = fbnic_fw_log_write(fbd, index, msec, log); + if (err) + return err; + } + + return 0; +} + +static int fbnic_fw_parse_log_req(void *opaque, + struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *index_tlv[FBNIC_FW_MAX_LOG_HISTORY]; + struct fbnic_tlv_msg *msec_tlv[FBNIC_FW_MAX_LOG_HISTORY]; + struct fbnic_tlv_msg *log_tlv[FBNIC_FW_MAX_LOG_HISTORY]; + struct fbnic_dev *fbd = opaque; + u16 length; + int err; + + length = fta_get_uint(results, FBNIC_FW_LOG_LENGTH); + if (length >= FBNIC_FW_MAX_LOG_HISTORY) + return -E2BIG; + + err = fbnic_fw_process_log_array(results, length, + FBNIC_FW_LOG_MSEC_ARRAY, + FBNIC_FW_LOG_MSEC, msec_tlv); + if (err) + return err; + + err = fbnic_fw_process_log_array(results, length, + FBNIC_FW_LOG_INDEX_ARRAY, + FBNIC_FW_LOG_INDEX, index_tlv); + if (err) + return err; + + err = fbnic_fw_process_log_array(results, length, + FBNIC_FW_LOG_MSG_ARRAY, + FBNIC_FW_LOG_MSG, log_tlv); + if (err) + return err; + + err = fbnic_fw_parse_logs(fbd, msec_tlv, index_tlv, log_tlv, + length + 1); + if (err) + return err; + + return 0; +} + +int fbnic_fw_xmit_send_logs(struct fbnic_dev *fbd, bool enable, + bool send_log_history) +{ + struct fbnic_tlv_msg *msg; + int err; + + if (fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE_LOG) { + dev_warn(fbd->dev, "Firmware version is too old to support firmware logs!\n"); + return -EOPNOTSUPP; + } + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_LOG_SEND_LOGS_REQ); + if (!msg) + return -ENOMEM; + + if (enable) { + err = fbnic_tlv_attr_put_flag(msg, FBNIC_SEND_LOGS); + if (err) + goto free_message; + + /* Report request for version 1 of logs */ + err = fbnic_tlv_attr_put_int(msg, FBNIC_SEND_LOGS_VERSION, + FBNIC_FW_LOG_VERSION); + if (err) + goto free_message; + + if (send_log_history) { + err = fbnic_tlv_attr_put_flag(msg, + FBNIC_SEND_LOGS_HISTORY); + if (err) + goto free_message; + } + } + + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + + return 0; + +free_message: + free_page((unsigned long)msg); + return err; +} + +static int +fbnic_fw_parser_test(void *opaque, struct fbnic_tlv_msg **results) +{ + struct fbnic_fw_completion *cmpl; + struct fbnic_dev *fbd = opaque; + int err; + + /* find cmpl */ + cmpl = fbnic_fw_get_cmpl_by_type(fbd, FBNIC_TLV_MSG_ID_TEST); + if (!cmpl) + return -ENOSPC; + + err = fbnic_tlv_parser_test(opaque, results); + + cmpl->result = err; + complete(&cmpl->done); + fbnic_fw_put_cmpl(cmpl); + + return err; +} + +static const struct fbnic_tlv_parser fbnic_fw_tlv_parser[] = { + FBNIC_TLV_PARSER(TEST, fbnic_tlv_test_index, fbnic_fw_parser_test), + FBNIC_TLV_PARSER(FW_CAP_RESP, fbnic_fw_cap_resp_index, + fbnic_fw_parse_cap_resp), + FBNIC_TLV_PARSER(OWNERSHIP_RESP, fbnic_ownership_resp_index, + fbnic_fw_parse_ownership_resp), + FBNIC_TLV_PARSER(HEARTBEAT_RESP, fbnic_heartbeat_resp_index, + fbnic_fw_parse_heartbeat_resp), + FBNIC_TLV_PARSER(COREDUMP_GET_INFO_RESP, + fbnic_coredump_info_resp_index, + fbnic_fw_parse_coredump_info_resp), + FBNIC_TLV_PARSER(COREDUMP_READ_RESP, fbnic_coredump_resp_index, + fbnic_fw_parse_coredump_resp), + FBNIC_TLV_PARSER(FW_START_UPGRADE_RESP, + fbnic_fw_start_upgrade_resp_index, + fbnic_fw_parse_fw_start_upgrade_resp), + FBNIC_TLV_PARSER(FW_WRITE_CHUNK_REQ, + fbnic_fw_write_chunk_req_index, + fbnic_fw_parse_fw_write_chunk_req), + FBNIC_TLV_PARSER(FW_FINISH_UPGRADE_REQ, + fbnic_fw_finish_upgrade_req_index, + fbnic_fw_parse_fw_finish_upgrade_req), + FBNIC_TLV_PARSER(QSFP_READ_RESP, + fbnic_qsfp_read_resp_index, + fbnic_fw_parse_qsfp_read_resp), + FBNIC_TLV_PARSER(TSENE_READ_RESP, + fbnic_tsene_read_resp_index, + fbnic_fw_parse_tsene_read_resp), + FBNIC_TLV_PARSER(LOG_MSG_REQ, + fbnic_fw_log_req_index, + fbnic_fw_parse_log_req), + FBNIC_TLV_MSG_ERROR +}; + +static void fbnic_mbx_process_rx_msgs(struct fbnic_dev *fbd) +{ + struct fbnic_fw_mbx *rx_mbx = &fbd->mbx[FBNIC_IPC_MBX_RX_IDX]; + u8 head = rx_mbx->head, tail = rx_mbx->tail; + u64 desc, length; + + while (head != rx_mbx->tail) { + struct fbnic_tlv_msg *msg; + int err; + + desc = __fbnic_mbx_rd_desc(fbd, FBNIC_IPC_MBX_RX_IDX, head); + if (!(desc & FBNIC_IPC_MBX_DESC_FW_CMPL)) + break; + + dma_sync_single_for_cpu(fbd->dev, rx_mbx->buf_info[head].addr, + FBNIC_RX_PAGE_SIZE, DMA_FROM_DEVICE); + + msg = rx_mbx->buf_info[head].msg; + + length = FIELD_GET(FBNIC_IPC_MBX_DESC_LEN_MASK, desc); + + /* Ignore NULL mailbox descriptors */ + if (!length) + goto next_page; + + /* Report descriptors with length greater than page size */ + if (length > PAGE_SIZE) { + dev_warn(fbd->dev, + "Invalid mailbox descriptor length: %lld\n", + length); + goto next_page; + } + + if (le16_to_cpu(msg->hdr.len) * sizeof(u32) > length) + dev_warn(fbd->dev, "Mailbox message length mismatch\n"); + + /* If parsing fails dump contents of message to dmesg */ + err = fbnic_tlv_msg_parse(fbd, msg, fbnic_fw_tlv_parser); + if (err) { + dev_warn(fbd->dev, "Unable to process message: %d\n", + err); + print_hex_dump(KERN_WARNING, "fbnic:", + DUMP_PREFIX_OFFSET, 16, 2, + msg, length, true); + } + + dev_dbg(fbd->dev, "Parsed msg type %d\n", msg->hdr.type); +next_page: + fw_wr32(fbd, FBNIC_IPC_MBX(FBNIC_IPC_MBX_RX_IDX, head), 0); + + rx_mbx->buf_info[tail] = rx_mbx->buf_info[head]; + rx_mbx->buf_info[head].msg = NULL; + rx_mbx->buf_info[head].addr = 0; + + __fbnic_mbx_wr_desc(fbd, FBNIC_IPC_MBX_RX_IDX, tail, + FIELD_PREP(FBNIC_IPC_MBX_DESC_LEN_MASK, + FBNIC_RX_PAGE_SIZE) | + (rx_mbx->buf_info[tail].addr & + FBNIC_IPC_MBX_DESC_ADDR_MASK) | + FBNIC_IPC_MBX_DESC_HOST_CMPL); + + head = (head + 1) & (FBNIC_IPC_MBX_DESC_LEN - 1); + tail = (tail + 1) & (FBNIC_IPC_MBX_DESC_LEN - 1); + } + + /* Record head for next interrupt */ + rx_mbx->head = head; + rx_mbx->tail = tail; +} + +void fbnic_mbx_poll(struct fbnic_dev *fbd) +{ + fbnic_mbx_event(fbd); + + fbnic_mbx_process_tx_msgs(fbd); + fbnic_mbx_process_rx_msgs(fbd); +} + +int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd) +{ + struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + unsigned long timeout = jiffies + 10 * HZ + 1; + int err, i; + + do { + if (!time_is_after_jiffies(timeout)) + return -ETIMEDOUT; + + /* Force the firmware to trigger an interrupt response to + * avoid the mailbox getting stuck closed if the interrupt + * is reset. + */ + fbnic_mbx_reset_desc_ring(fbd, FBNIC_IPC_MBX_TX_IDX); + + /* Immediate fail if BAR4 went away */ + if (!fbnic_fw_present(fbd)) + return -ENODEV; + + msleep(20); + } while (!fbnic_mbx_event(fbd)); + + /* FW has shown signs of life. Enable DMA and start Tx/Rx */ + for (i = 0; i < FBNIC_IPC_MBX_INDICES; i++) { + err = fbnic_mbx_init_desc_ring(fbd, i); + if (err) + goto clean_mbx; + } + + /* Request an update from the firmware. This should overwrite + * mgmt.version once we get the actual version from the firmware + * in the capabilities request message. + */ + err = fbnic_fw_xmit_simple_msg(fbd, FBNIC_TLV_MSG_ID_HOST_CAP_REQ); + if (err) + goto clean_mbx; + + /* Poll until we get a current management firmware version, use "1" + * to indicate we entered the polling state waiting for a response + */ + for (fbd->fw_cap.running.mgmt.version = 1; + fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE;) { + if (!tx_mbx->ready) + err = -ENODEV; + if (err) + goto clean_mbx; + + msleep(20); + fbnic_mbx_poll(fbd); + + /* set err, but wait till mgmt.version check to report it */ + if (!time_is_after_jiffies(timeout)) + err = -ETIMEDOUT; + } + + return 0; +clean_mbx: + /* Cleanup Rx buffers and disable mailbox */ + fbnic_mbx_clean(fbd); + return err; +} + +static void __fbnic_fw_evict_cmpl(struct fbnic_fw_completion *cmpl_data) +{ + cmpl_data->result = -EPIPE; + complete(&cmpl_data->done); +} + +static void fbnic_mbx_evict_all_cmpl(struct fbnic_dev *fbd) +{ + int i; + + for (i = 0; i < FBNIC_MBX_CMPL_SLOTS; i++) { + struct fbnic_fw_completion *cmpl_data = fbd->cmpl_data[i]; + + if (cmpl_data) + __fbnic_fw_evict_cmpl(cmpl_data); + } + + memset(fbd->cmpl_data, 0, sizeof(fbd->cmpl_data)); +} + +void fbnic_mbx_flush_tx(struct fbnic_dev *fbd) +{ + unsigned long timeout = jiffies + 10 * HZ + 1; + struct fbnic_fw_mbx *tx_mbx; + u8 tail; + + /* Record current Rx stats */ + tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + + spin_lock_irq(&fbd->fw_tx_lock); + + /* Clear ready to prevent any further attempts to transmit */ + tx_mbx->ready = false; + + /* Read tail to determine the last tail state for the ring */ + tail = tx_mbx->tail; + + /* Flush any completions as we are no longer processing Rx */ + fbnic_mbx_evict_all_cmpl(fbd); + + spin_unlock_irq(&fbd->fw_tx_lock); + + /* Give firmware time to process packet, + * we will wait up to 10 seconds which is 500 waits of 20ms. + */ + do { + u8 head = tx_mbx->head; + + /* Tx ring is empty once head == tail */ + if (head == tail) + break; + + msleep(20); + fbnic_mbx_process_tx_msgs(fbd); + } while (time_is_after_jiffies(timeout)); +} + +/** + * fbnic_fw_mbx_self_test() - verify firmware interface + * @fbd: device to test + * + * This function tests the interfaces to/from the firmware. + * + * Return: See enum fbnic_fw_self_test_codes + **/ +enum fbnic_fw_self_test_codes fbnic_fw_mbx_self_test(struct fbnic_dev *fbd) +{ + enum fbnic_fw_self_test_codes err; + struct fbnic_fw_completion *cmpl; + + /* Skip test if FW interface is not present */ + if (!fbnic_fw_present(fbd)) + return FBNIC_TEST_FW_NO_FIRMWARE; + + cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_TEST); + if (!cmpl) + return FBNIC_TEST_FW_NO_CMPL; + + /* Load a test message onto the FW mailbox interface + * and arm the completion. + */ + err = fbnic_fw_xmit_test_msg(fbd, cmpl); + if (err) { + err = FBNIC_TEST_FW_NO_XMIT; + goto exit_free; + } + + /* Verify we received a message back */ + if (!fbnic_mbx_wait_for_cmpl(cmpl)) { + err = FBNIC_TEST_FW_NO_MSG; + goto exit_cleanup; + } + + /* Verify there were no parsing errors */ + if (cmpl->result) + err = FBNIC_TEST_FW_PARSE; +exit_cleanup: + fbnic_mbx_clear_cmpl(fbd, cmpl); +exit_free: + fbnic_fw_put_cmpl(cmpl); + + return err; +} + +int fbnic_fw_xmit_rpc_macda_sync(struct fbnic_dev *fbd) +{ + struct fbnic_tlv_msg *mac_array; + int i, addr_count = 0, err; + struct fbnic_tlv_msg *msg; + u32 rx_flags = 0; + + /* Nothing to do if there is no FW to sync with */ + if (!fbd->mbx[FBNIC_IPC_MBX_TX_IDX].ready) + return 0; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_RPC_MAC_SYNC_REQ); + if (!msg) + return -ENOMEM; + + mac_array = fbnic_tlv_attr_nest_start(msg, + FBNIC_FW_RPC_MAC_SYNC_UC_ARRAY); + if (!mac_array) + goto free_message_nospc; + + /* Populate the unicast MAC addrs and capture PROMISC/ALLMULTI flags */ + for (addr_count = 0, i = FBNIC_RPC_TCAM_MACDA_PROMISC_IDX; + i >= fbd->mac_addr_boundary; i--) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (mac_addr->state != FBNIC_TCAM_S_VALID) + continue; + if (test_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI; + if (test_bit(FBNIC_MAC_ADDR_T_PROMISC, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_PROMISC; + if (!test_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam)) + continue; + if (addr_count == FW_RPC_MAC_SYNC_UC_ARRAY_SIZE) { + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_PROMISC; + continue; + } + + err = fbnic_tlv_attr_put_value(mac_array, + FBNIC_FW_RPC_MAC_SYNC_MAC_ADDR, + mac_addr->value.addr8, + ETH_ALEN); + if (err) + goto free_message; + addr_count++; + } + + /* Close array */ + fbnic_tlv_attr_nest_stop(msg); + + mac_array = fbnic_tlv_attr_nest_start(msg, + FBNIC_FW_RPC_MAC_SYNC_MC_ARRAY); + if (!mac_array) + goto free_message_nospc; + + /* Repeat for multicast addrs, record BROADCAST/ALLMULTI flags */ + for (addr_count = 0, i = FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX; + i < fbd->mac_addr_boundary; i++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (mac_addr->state != FBNIC_TCAM_S_VALID) + continue; + if (test_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_BROADCAST; + if (test_bit(FBNIC_MAC_ADDR_T_ALLMULTI, mac_addr->act_tcam)) + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI; + if (!test_bit(FBNIC_MAC_ADDR_T_MULTICAST, mac_addr->act_tcam)) + continue; + if (addr_count == FW_RPC_MAC_SYNC_MC_ARRAY_SIZE) { + rx_flags |= FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI; + continue; + } + + err = fbnic_tlv_attr_put_value(mac_array, + FBNIC_FW_RPC_MAC_SYNC_MAC_ADDR, + mac_addr->value.addr8, + ETH_ALEN); + if (err) + goto free_message; + addr_count++; + } + + /* Close array */ + fbnic_tlv_attr_nest_stop(msg); + + /* Report flags at end of list */ + err = fbnic_tlv_attr_put_int(msg, FBNIC_FW_RPC_MAC_SYNC_RX_FLAGS, + rx_flags); + if (err) + goto free_message; + + /* Send message off to FW notifying it of current RPC config */ + err = fbnic_mbx_map_tlv_msg(fbd, msg); + if (err) + goto free_message; + return 0; +free_message_nospc: + err = -ENOSPC; +free_message: + free_page((unsigned long)msg); + return err; +} + +void fbnic_get_fw_ver_commit_str(struct fbnic_dev *fbd, char *fw_version, + const size_t str_sz) +{ + struct fbnic_fw_ver *mgmt = &fbd->fw_cap.running.mgmt; + const char *delim = ""; + + if (mgmt->commit[0]) + delim = "_"; + + fbnic_mk_full_fw_ver_str(mgmt->version, delim, mgmt->commit, + fw_version, str_sz); +} + +struct fbnic_fw_completion *__fbnic_fw_alloc_cmpl(u32 msg_type, + size_t priv_size) +{ + struct fbnic_fw_completion *cmpl; + + cmpl = kzalloc(sizeof(*cmpl) + priv_size, GFP_KERNEL); + if (!cmpl) + return NULL; + + cmpl->msg_type = msg_type; + init_completion(&cmpl->done); + kref_init(&cmpl->ref_count); + + return cmpl; +} + +struct fbnic_fw_completion *fbnic_fw_alloc_cmpl(u32 msg_type) +{ + return __fbnic_fw_alloc_cmpl(msg_type, 0); +} + +void fbnic_fw_put_cmpl(struct fbnic_fw_completion *fw_cmpl) +{ + kref_put(&fw_cmpl->ref_count, fbnic_fw_release_cmpl_data); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h new file mode 100644 index 000000000..d84723e4c --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw.h @@ -0,0 +1,363 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_FW_H_ +#define _FBNIC_FW_H_ + +#include <linux/completion.h> +#include <linux/if_ether.h> +#include <linux/types.h> + +struct fbnic_dev; +struct fbnic_tlv_msg; + +struct fbnic_fw_mbx { + u8 ready, head, tail; + struct { + struct fbnic_tlv_msg *msg; + dma_addr_t addr; + } buf_info[FBNIC_IPC_MBX_DESC_LEN]; +}; + +// FW_VER_MAX_SIZE must match ETHTOOL_FWVERS_LEN +#define FBNIC_FW_VER_MAX_SIZE 32 +// Formatted version is in the format XX.YY.ZZ_RRR_COMMIT +#define FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE (FBNIC_FW_VER_MAX_SIZE - 13) +#define FBNIC_FW_LOG_VERSION 1 +#define FBNIC_FW_LOG_MAX_SIZE 256 +/* + * The max amount of logs which can fit in a single mailbox message. Firmware + * assumes each mailbox message is 4096B. The amount of messages supported is + * calculated as 4096 minus headers for message, arrays, and length minus the + * size of length divided by headers for each array plus the maximum LOG size, + * and the size of MSEC and INDEX. Put another way: + * + * MAX_LOG_HISTORY = ((4096 - TLV_HDR_SZ * 5 - LENGTH_SZ) + * / (FBNIC_FW_LOG_MAX_SIZE + TLV_HDR_SZ * 3 + MSEC_SZ + * + INDEX_SZ)) + */ +#define FBNIC_FW_MAX_LOG_HISTORY 14 +#define FBNIC_MBX_RX_TO_SEC 10 + +struct fbnic_fw_ver { + u32 version; + char commit[FBNIC_FW_CAP_RESP_COMMIT_MAX_SIZE]; +}; + +struct fbnic_fw_cap { + struct { + struct fbnic_fw_ver mgmt, bootloader; + } running; + struct { + struct fbnic_fw_ver mgmt, bootloader, undi; + } stored; + u8 active_slot; + u8 bmc_mac_addr[4][ETH_ALEN]; + u8 bmc_present : 1; + u8 need_bmc_tcam_reinit : 1; + u8 need_bmc_macda_sync : 1; + u8 all_multi : 1; + u8 link_speed; + u8 link_fec; + u32 anti_rollback_version; +}; + +struct fbnic_fw_completion { + u32 msg_type; + struct completion done; + struct kref ref_count; + int result; + union { + struct { + u32 size; + } coredump_info; + struct { + u32 size; + u16 stride; + u8 *data[]; + } coredump; + struct { + u32 offset; + u32 length; + } fw_update; + struct { + u16 length; + u8 offset; + u8 page; + u8 bank; + u8 data[] __aligned(sizeof(u32)) __counted_by(length); + } qsfp; + struct { + s32 millivolts; + s32 millidegrees; + } tsene; + } u; +}; + +u64 __fbnic_mbx_rd_desc(struct fbnic_dev *fbd, int mbx_idx, int desc_idx); +void fbnic_mbx_init(struct fbnic_dev *fbd); +void fbnic_mbx_clean(struct fbnic_dev *fbd); +int fbnic_mbx_set_cmpl(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data); +void fbnic_mbx_clear_cmpl(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data); +void fbnic_mbx_poll(struct fbnic_dev *fbd); +int fbnic_mbx_poll_tx_ready(struct fbnic_dev *fbd); +void fbnic_mbx_flush_tx(struct fbnic_dev *fbd); + +/** + * enum fbnic_fw_self_test_codes - return codes from self test routines + * + * These are the codes returned from the self test routines and + * stored in the test result array indexed by the specific + * test name. + * + * @FBNIC_TEST_FW_SUCCESS: test success + * @FBNIC_TEST_FW_NO_FIRMWARE: FW interface not present + * @FBNIC_TEST_FW_NO_CMPL: No completion available + * @FBNIC_TEST_FW_NO_XMIT: Could not xmit message + * @FBNIC_TEST_FW_NO_MSG: no message returned + * @FBNIC_TEST_FW_PARSE: returned message had parsing error + */ +enum fbnic_fw_self_test_codes { + FBNIC_TEST_FW_SUCCESS = 0, + FBNIC_TEST_FW_NO_FIRMWARE = 10, + FBNIC_TEST_FW_NO_CMPL = 20, + FBNIC_TEST_FW_NO_XMIT = 30, + FBNIC_TEST_FW_NO_MSG = 40, + FBNIC_TEST_FW_PARSE = 50, +}; + +enum fbnic_fw_self_test_codes fbnic_fw_mbx_self_test(struct fbnic_dev *fbd); +int fbnic_fw_xmit_test_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *c); +int fbnic_fw_xmit_ownership_msg(struct fbnic_dev *fbd, bool take_ownership); +int fbnic_fw_init_heartbeat(struct fbnic_dev *fbd, bool poll); +void fbnic_fw_check_heartbeat(struct fbnic_dev *fbd); +int fbnic_fw_xmit_coredump_info_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + bool force); +int fbnic_fw_xmit_coredump_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 offset, u32 length); +int fbnic_fw_xmit_fw_start_upgrade(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + unsigned int id, unsigned int len); +int fbnic_fw_xmit_fw_write_chunk(struct fbnic_dev *fbd, + const u8 *data, u32 offset, u16 length, + int cancel_error); +int fbnic_fw_xmit_qsfp_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data, + u32 page, u32 bank, u32 offset, u32 length); +int fbnic_fw_xmit_tsene_read_msg(struct fbnic_dev *fbd, + struct fbnic_fw_completion *cmpl_data); +int fbnic_fw_xmit_send_logs(struct fbnic_dev *fbd, bool enable, + bool send_log_history); +int fbnic_fw_xmit_rpc_macda_sync(struct fbnic_dev *fbd); +struct fbnic_fw_completion *__fbnic_fw_alloc_cmpl(u32 msg_type, + size_t priv_size); +struct fbnic_fw_completion *fbnic_fw_alloc_cmpl(u32 msg_type); +void fbnic_fw_put_cmpl(struct fbnic_fw_completion *cmpl_data); + +static inline unsigned long +fbnic_mbx_wait_for_cmpl(struct fbnic_fw_completion *cmpl) +{ + return wait_for_completion_timeout(&cmpl->done, + FBNIC_MBX_RX_TO_SEC * HZ); +} + +#define fbnic_mk_full_fw_ver_str(_rev_id, _delim, _commit, _str, _str_sz) \ +do { \ + const u32 __rev_id = _rev_id; \ + snprintf(_str, _str_sz, "%02lu.%02lu.%02lu-%03lu%s%s", \ + FIELD_GET(FBNIC_FW_CAP_RESP_VERSION_MAJOR, __rev_id), \ + FIELD_GET(FBNIC_FW_CAP_RESP_VERSION_MINOR, __rev_id), \ + FIELD_GET(FBNIC_FW_CAP_RESP_VERSION_PATCH, __rev_id), \ + FIELD_GET(FBNIC_FW_CAP_RESP_VERSION_BUILD, __rev_id), \ + _delim, _commit); \ +} while (0) + +#define fbnic_mk_fw_ver_str(_rev_id, _str) \ + fbnic_mk_full_fw_ver_str(_rev_id, "", "", _str, sizeof(_str)) + +enum { + QSPI_SECTION_CMRT = 0, + QSPI_SECTION_CONTROL_FW = 1, + QSPI_SECTION_UCODE = 2, + QSPI_SECTION_OPTION_ROM = 3, + QSPI_SECTION_USER = 4, + QSPI_SECTION_INVALID, +}; + +#define FW_HEARTBEAT_PERIOD (10 * HZ) + +enum { + FBNIC_TLV_MSG_ID_HOST_CAP_REQ = 0x10, + FBNIC_TLV_MSG_ID_FW_CAP_RESP = 0x11, + FBNIC_TLV_MSG_ID_OWNERSHIP_REQ = 0x12, + FBNIC_TLV_MSG_ID_OWNERSHIP_RESP = 0x13, + FBNIC_TLV_MSG_ID_HEARTBEAT_REQ = 0x14, + FBNIC_TLV_MSG_ID_HEARTBEAT_RESP = 0x15, + FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_REQ = 0x18, + FBNIC_TLV_MSG_ID_COREDUMP_GET_INFO_RESP = 0x19, + FBNIC_TLV_MSG_ID_COREDUMP_READ_REQ = 0x20, + FBNIC_TLV_MSG_ID_COREDUMP_READ_RESP = 0x21, + FBNIC_TLV_MSG_ID_FW_START_UPGRADE_REQ = 0x22, + FBNIC_TLV_MSG_ID_FW_START_UPGRADE_RESP = 0x23, + FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_REQ = 0x24, + FBNIC_TLV_MSG_ID_FW_WRITE_CHUNK_RESP = 0x25, + FBNIC_TLV_MSG_ID_FW_FINISH_UPGRADE_REQ = 0x28, + FBNIC_TLV_MSG_ID_FW_FINISH_UPGRADE_RESP = 0x29, + FBNIC_TLV_MSG_ID_QSFP_READ_REQ = 0x38, + FBNIC_TLV_MSG_ID_QSFP_READ_RESP = 0x39, + FBNIC_TLV_MSG_ID_TSENE_READ_REQ = 0x3C, + FBNIC_TLV_MSG_ID_TSENE_READ_RESP = 0x3D, + FBNIC_TLV_MSG_ID_LOG_SEND_LOGS_REQ = 0x43, + FBNIC_TLV_MSG_ID_LOG_MSG_REQ = 0x44, + FBNIC_TLV_MSG_ID_LOG_MSG_RESP = 0x45, + FBNIC_TLV_MSG_ID_RPC_MAC_SYNC_REQ = 0x46, +}; + +#define FBNIC_FW_CAP_RESP_VERSION_MAJOR CSR_GENMASK(31, 24) +#define FBNIC_FW_CAP_RESP_VERSION_MINOR CSR_GENMASK(23, 16) +#define FBNIC_FW_CAP_RESP_VERSION_PATCH CSR_GENMASK(15, 8) +#define FBNIC_FW_CAP_RESP_VERSION_BUILD CSR_GENMASK(7, 0) +enum { + FBNIC_FW_CAP_RESP_VERSION = 0x0, + FBNIC_FW_CAP_RESP_BMC_PRESENT = 0x1, + FBNIC_FW_CAP_RESP_BMC_MAC_ADDR = 0x2, + FBNIC_FW_CAP_RESP_BMC_MAC_ARRAY = 0x3, + FBNIC_FW_CAP_RESP_STORED_VERSION = 0x4, + FBNIC_FW_CAP_RESP_ACTIVE_FW_SLOT = 0x5, + FBNIC_FW_CAP_RESP_VERSION_COMMIT_STR = 0x6, + FBNIC_FW_CAP_RESP_BMC_ALL_MULTI = 0x8, + FBNIC_FW_CAP_RESP_FW_STATE = 0x9, + FBNIC_FW_CAP_RESP_FW_LINK_SPEED = 0xa, + FBNIC_FW_CAP_RESP_FW_LINK_FEC = 0xb, + FBNIC_FW_CAP_RESP_STORED_COMMIT_STR = 0xc, + FBNIC_FW_CAP_RESP_CMRT_VERSION = 0xd, + FBNIC_FW_CAP_RESP_STORED_CMRT_VERSION = 0xe, + FBNIC_FW_CAP_RESP_CMRT_COMMIT_STR = 0xf, + FBNIC_FW_CAP_RESP_STORED_CMRT_COMMIT_STR = 0x10, + FBNIC_FW_CAP_RESP_UEFI_VERSION = 0x11, + FBNIC_FW_CAP_RESP_UEFI_COMMIT_STR = 0x12, + FBNIC_FW_CAP_RESP_ANTI_ROLLBACK_VERSION = 0x15, + FBNIC_FW_CAP_RESP_MSG_MAX +}; + +enum { + FBNIC_FW_LINK_MODE_25CR = 1, + FBNIC_FW_LINK_MODE_50CR2 = 2, + FBNIC_FW_LINK_MODE_50CR = 3, + FBNIC_FW_LINK_MODE_100CR2 = 4, +}; + +enum { + FBNIC_FW_LINK_FEC_NONE = 1, + FBNIC_FW_LINK_FEC_RS = 2, + FBNIC_FW_LINK_FEC_BASER = 3, +}; + +enum { + FBNIC_FW_QSFP_BANK = 0x0, + FBNIC_FW_QSFP_PAGE = 0x1, + FBNIC_FW_QSFP_OFFSET = 0x2, + FBNIC_FW_QSFP_LENGTH = 0x3, + FBNIC_FW_QSFP_ERROR = 0x4, + FBNIC_FW_QSFP_DATA = 0x5, + FBNIC_FW_QSFP_MSG_MAX +}; + +enum { + FBNIC_FW_TSENE_THERM = 0x0, + FBNIC_FW_TSENE_VOLT = 0x1, + FBNIC_FW_TSENE_ERROR = 0x2, + FBNIC_FW_TSENE_MSG_MAX +}; + +enum { + FBNIC_FW_OWNERSHIP_FLAG = 0x0, + FBNIC_FW_OWNERSHIP_TIME = 0x1, + FBNIC_FW_OWNERSHIP_MSG_MAX +}; + +enum { + FBNIC_FW_HEARTBEAT_UPTIME = 0x0, + FBNIC_FW_HEARTBEAT_NUMBER_OF_MESSAGES = 0x1, + FBNIC_FW_HEARTBEAT_MSG_MAX +}; + +enum { + FBNIC_FW_COREDUMP_REQ_INFO_CREATE = 0x0, + FBNIC_FW_COREDUMP_REQ_INFO_MSG_MAX +}; + +enum { + FBNIC_FW_COREDUMP_INFO_AVAILABLE = 0x0, + FBNIC_FW_COREDUMP_INFO_SIZE = 0x1, + FBNIC_FW_COREDUMP_INFO_ERROR = 0x2, + FBNIC_FW_COREDUMP_INFO_MSG_MAX +}; + +enum { + FBNIC_FW_COREDUMP_READ_OFFSET = 0x0, + FBNIC_FW_COREDUMP_READ_LENGTH = 0x1, + FBNIC_FW_COREDUMP_READ_DATA = 0x2, + FBNIC_FW_COREDUMP_READ_ERROR = 0x3, + FBNIC_FW_COREDUMP_READ_MSG_MAX +}; + +enum { + FBNIC_FW_START_UPGRADE_ERROR = 0x0, + FBNIC_FW_START_UPGRADE_SECTION = 0x1, + FBNIC_FW_START_UPGRADE_IMAGE_LENGTH = 0x2, + FBNIC_FW_START_UPGRADE_MSG_MAX +}; + +enum { + FBNIC_FW_WRITE_CHUNK_OFFSET = 0x0, + FBNIC_FW_WRITE_CHUNK_LENGTH = 0x1, + FBNIC_FW_WRITE_CHUNK_DATA = 0x2, + FBNIC_FW_WRITE_CHUNK_ERROR = 0x3, + FBNIC_FW_WRITE_CHUNK_MSG_MAX +}; + +enum { + FBNIC_FW_FINISH_UPGRADE_ERROR = 0x0, + FBNIC_FW_FINISH_UPGRADE_MSG_MAX +}; + +enum { + FBNIC_SEND_LOGS = 0x0, + FBNIC_SEND_LOGS_VERSION = 0x1, + FBNIC_SEND_LOGS_HISTORY = 0x2, + FBNIC_SEND_LOGS_MSG_MAX +}; + +enum { + FBNIC_FW_LOG_MSEC = 0x0, + FBNIC_FW_LOG_INDEX = 0x1, + FBNIC_FW_LOG_MSG = 0x2, + FBNIC_FW_LOG_LENGTH = 0x3, + FBNIC_FW_LOG_MSEC_ARRAY = 0x4, + FBNIC_FW_LOG_INDEX_ARRAY = 0x5, + FBNIC_FW_LOG_MSG_ARRAY = 0x6, + FBNIC_FW_LOG_MSG_MAX +}; + +enum { + FBNIC_FW_RPC_MAC_SYNC_RX_FLAGS = 0x0, + FBNIC_FW_RPC_MAC_SYNC_UC_ARRAY = 0x1, + FBNIC_FW_RPC_MAC_SYNC_MC_ARRAY = 0x2, + FBNIC_FW_RPC_MAC_SYNC_MAC_ADDR = 0x3, + FBNIC_FW_RPC_MAC_SYNC_MSG_MAX +}; + +#define FW_RPC_MAC_SYNC_RX_FLAGS_PROMISC 1 +#define FW_RPC_MAC_SYNC_RX_FLAGS_ALLMULTI 2 +#define FW_RPC_MAC_SYNC_RX_FLAGS_BROADCAST 4 + +#define FW_RPC_MAC_SYNC_UC_ARRAY_SIZE 8 +#define FW_RPC_MAC_SYNC_MC_ARRAY_SIZE 8 + +#endif /* _FBNIC_FW_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c new file mode 100644 index 000000000..d8a9a7d7c --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.c @@ -0,0 +1,120 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/spinlock.h> +#include <linux/vmalloc.h> + +#include "fbnic.h" +#include "fbnic_fw.h" +#include "fbnic_fw_log.h" + +void fbnic_fw_log_enable(struct fbnic_dev *fbd, bool send_hist) +{ + int err; + + if (!fbnic_fw_log_ready(fbd)) + return; + + if (fbd->fw_cap.running.mgmt.version < MIN_FW_VER_CODE_HIST) + send_hist = false; + + err = fbnic_fw_xmit_send_logs(fbd, true, send_hist); + if (err && err != -EOPNOTSUPP) + dev_warn(fbd->dev, "Unable to enable firmware logs: %d\n", err); +} + +void fbnic_fw_log_disable(struct fbnic_dev *fbd) +{ + int err; + + err = fbnic_fw_xmit_send_logs(fbd, false, false); + if (err && err != -EOPNOTSUPP) + dev_warn(fbd->dev, "Unable to disable firmware logs: %d\n", + err); +} + +int fbnic_fw_log_init(struct fbnic_dev *fbd) +{ + struct fbnic_fw_log *log = &fbd->fw_log; + void *data; + + if (WARN_ON_ONCE(fbnic_fw_log_ready(fbd))) + return -EEXIST; + + data = vmalloc(FBNIC_FW_LOG_SIZE); + if (!data) + return -ENOMEM; + + spin_lock_init(&fbd->fw_log.lock); + INIT_LIST_HEAD(&log->entries); + log->size = FBNIC_FW_LOG_SIZE; + log->data_start = data; + log->data_end = data + FBNIC_FW_LOG_SIZE; + + return 0; +} + +void fbnic_fw_log_free(struct fbnic_dev *fbd) +{ + struct fbnic_fw_log *log = &fbd->fw_log; + + if (!fbnic_fw_log_ready(fbd)) + return; + + INIT_LIST_HEAD(&log->entries); + log->size = 0; + vfree(log->data_start); + log->data_start = NULL; + log->data_end = NULL; +} + +int fbnic_fw_log_write(struct fbnic_dev *fbd, u64 index, u32 timestamp, + const char *msg) +{ + struct fbnic_fw_log_entry *entry, *head, *tail, *next; + struct fbnic_fw_log *log = &fbd->fw_log; + size_t msg_len = strlen(msg) + 1; + unsigned long flags; + void *entry_end; + + if (!fbnic_fw_log_ready(fbd)) { + dev_err(fbd->dev, "Firmware sent log entry without being requested!\n"); + return -ENOSPC; + } + + spin_lock_irqsave(&log->lock, flags); + + if (list_empty(&log->entries)) { + entry = log->data_start; + } else { + head = list_first_entry(&log->entries, typeof(*head), list); + entry_end = head->msg + head->len + 1; + entry = PTR_ALIGN(entry_end, 8); + } + + entry_end = entry->msg + msg_len + 1; + + /* We've reached the end of the buffer, wrap around */ + if (entry_end > log->data_end) { + entry = log->data_start; + entry_end = entry->msg + msg_len + 1; + } + + /* Make room for entry by removing from tail. */ + list_for_each_entry_safe_reverse(tail, next, &log->entries, list) { + if (entry <= tail && entry_end > (void *)tail) + list_del(&tail->list); + else + break; + } + + entry->index = index; + entry->timestamp = timestamp; + entry->len = msg_len; + strscpy(entry->msg, msg, entry->len); + list_add(&entry->list, &log->entries); + + spin_unlock_irqrestore(&log->lock, flags); + + return 0; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.h b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.h new file mode 100644 index 000000000..50ec79003 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_fw_log.h @@ -0,0 +1,45 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_FW_LOG_H_ +#define _FBNIC_FW_LOG_H_ + +#include <linux/spinlock.h> +#include <linux/types.h> + +/* A 512K log buffer was chosen fairly arbitrarily */ +#define FBNIC_FW_LOG_SIZE (512 * 1024) /* bytes */ + +/* Firmware log output is prepended with log index followed by a timestamp. + * The timestamp is similar to Zephyr's format DD:HH:MM:SS.MMM + */ +#define FBNIC_FW_LOG_FMT "[%5lld] [%02ld:%02ld:%02ld:%02ld.%03ld] %s\n" + +struct fbnic_dev; + +struct fbnic_fw_log_entry { + struct list_head list; + u64 index; + u32 timestamp; + u16 len; + char msg[] __counted_by(len); +}; + +struct fbnic_fw_log { + void *data_start; + void *data_end; + size_t size; + struct list_head entries; + /* Spin lock for accessing or modifying entries */ + spinlock_t lock; +}; + +#define fbnic_fw_log_ready(_fbd) (!!(_fbd)->fw_log.data_start) + +void fbnic_fw_log_enable(struct fbnic_dev *fbd, bool send_hist); +void fbnic_fw_log_disable(struct fbnic_dev *fbd); +int fbnic_fw_log_init(struct fbnic_dev *fbd); +void fbnic_fw_log_free(struct fbnic_dev *fbd); +int fbnic_fw_log_write(struct fbnic_dev *fbd, u64 index, u32 timestamp, + const char *msg); +#endif /* _FBNIC_FW_LOG_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c new file mode 100644 index 000000000..8b9b2076b --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.c @@ -0,0 +1,601 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/rtnetlink.h> + +#include "fbnic.h" + +static void fbnic_hw_stat_rst32(struct fbnic_dev *fbd, u32 reg, + struct fbnic_stat_counter *stat) +{ + /* We do not touch the "value" field here. + * It gets zeroed out on fbd structure allocation. + * After that we want it to grow continuously + * through device resets and power state changes. + */ + stat->u.old_reg_value_32 = rd32(fbd, reg); +} + +static void fbnic_hw_stat_rd32(struct fbnic_dev *fbd, u32 reg, + struct fbnic_stat_counter *stat) +{ + u32 new_reg_value; + + new_reg_value = rd32(fbd, reg); + stat->value += new_reg_value - stat->u.old_reg_value_32; + stat->u.old_reg_value_32 = new_reg_value; +} + +u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset) +{ + u32 prev_upper, upper, lower, diff; + + prev_upper = rd32(fbd, reg + offset); + lower = rd32(fbd, reg); + upper = rd32(fbd, reg + offset); + + diff = upper - prev_upper; + if (!diff) + return ((u64)upper << 32) | lower; + + if (diff > 1) + dev_warn_once(fbd->dev, + "Stats inconsistent, upper 32b of %#010x updating too quickly\n", + reg * 4); + + /* Return only the upper bits as we cannot guarantee + * the accuracy of the lower bits. We will add them in + * when the counter slows down enough that we can get + * a snapshot with both upper values being the same + * between reads. + */ + return ((u64)upper << 32); +} + +static void fbnic_hw_stat_rst64(struct fbnic_dev *fbd, u32 reg, s32 offset, + struct fbnic_stat_counter *stat) +{ + /* Record initial counter values and compute deltas from there to ensure + * stats start at 0 after reboot/reset. This avoids exposing absolute + * hardware counter values to userspace. + */ + stat->u.old_reg_value_64 = fbnic_stat_rd64(fbd, reg, offset); +} + +static void fbnic_hw_stat_rd64(struct fbnic_dev *fbd, u32 reg, s32 offset, + struct fbnic_stat_counter *stat) +{ + u64 new_reg_value; + + new_reg_value = fbnic_stat_rd64(fbd, reg, offset); + stat->value += new_reg_value - stat->u.old_reg_value_64; + stat->u.old_reg_value_64 = new_reg_value; +} + +static void fbnic_reset_tmi_stats(struct fbnic_dev *fbd, + struct fbnic_tmi_stats *tmi) +{ + fbnic_hw_stat_rst32(fbd, FBNIC_TMI_DROP_PKTS, &tmi->drop.frames); + fbnic_hw_stat_rst64(fbd, FBNIC_TMI_DROP_BYTE_L, 1, &tmi->drop.bytes); + + fbnic_hw_stat_rst32(fbd, + FBNIC_TMI_ILLEGAL_PTP_REQS, + &tmi->ptp_illegal_req); + fbnic_hw_stat_rst32(fbd, FBNIC_TMI_GOOD_PTP_TS, &tmi->ptp_good_ts); + fbnic_hw_stat_rst32(fbd, FBNIC_TMI_BAD_PTP_TS, &tmi->ptp_bad_ts); +} + +static void fbnic_get_tmi_stats32(struct fbnic_dev *fbd, + struct fbnic_tmi_stats *tmi) +{ + fbnic_hw_stat_rd32(fbd, FBNIC_TMI_DROP_PKTS, &tmi->drop.frames); + + fbnic_hw_stat_rd32(fbd, + FBNIC_TMI_ILLEGAL_PTP_REQS, + &tmi->ptp_illegal_req); + fbnic_hw_stat_rd32(fbd, FBNIC_TMI_GOOD_PTP_TS, &tmi->ptp_good_ts); + fbnic_hw_stat_rd32(fbd, FBNIC_TMI_BAD_PTP_TS, &tmi->ptp_bad_ts); +} + +static void fbnic_get_tmi_stats(struct fbnic_dev *fbd, + struct fbnic_tmi_stats *tmi) +{ + fbnic_hw_stat_rd64(fbd, FBNIC_TMI_DROP_BYTE_L, 1, &tmi->drop.bytes); +} + +static void fbnic_reset_tti_stats(struct fbnic_dev *fbd, + struct fbnic_tti_stats *tti) +{ + fbnic_hw_stat_rst32(fbd, + FBNIC_TCE_TTI_CM_DROP_PKTS, + &tti->cm_drop.frames); + fbnic_hw_stat_rst64(fbd, + FBNIC_TCE_TTI_CM_DROP_BYTE_L, + 1, + &tti->cm_drop.bytes); + + fbnic_hw_stat_rst32(fbd, + FBNIC_TCE_TTI_FRAME_DROP_PKTS, + &tti->frame_drop.frames); + fbnic_hw_stat_rst64(fbd, + FBNIC_TCE_TTI_FRAME_DROP_BYTE_L, + 1, + &tti->frame_drop.bytes); + + fbnic_hw_stat_rst32(fbd, + FBNIC_TCE_TBI_DROP_PKTS, + &tti->tbi_drop.frames); + fbnic_hw_stat_rst64(fbd, + FBNIC_TCE_TBI_DROP_BYTE_L, + 1, + &tti->tbi_drop.bytes); +} + +static void fbnic_get_tti_stats32(struct fbnic_dev *fbd, + struct fbnic_tti_stats *tti) +{ + fbnic_hw_stat_rd32(fbd, + FBNIC_TCE_TTI_CM_DROP_PKTS, + &tti->cm_drop.frames); + + fbnic_hw_stat_rd32(fbd, + FBNIC_TCE_TTI_FRAME_DROP_PKTS, + &tti->frame_drop.frames); + + fbnic_hw_stat_rd32(fbd, + FBNIC_TCE_TBI_DROP_PKTS, + &tti->tbi_drop.frames); +} + +static void fbnic_get_tti_stats(struct fbnic_dev *fbd, + struct fbnic_tti_stats *tti) +{ + fbnic_hw_stat_rd64(fbd, + FBNIC_TCE_TTI_CM_DROP_BYTE_L, + 1, + &tti->cm_drop.bytes); + + fbnic_hw_stat_rd64(fbd, + FBNIC_TCE_TTI_FRAME_DROP_BYTE_L, + 1, + &tti->frame_drop.bytes); + + fbnic_hw_stat_rd64(fbd, + FBNIC_TCE_TBI_DROP_BYTE_L, + 1, + &tti->tbi_drop.bytes); +} + +static void fbnic_reset_rpc_stats(struct fbnic_dev *fbd, + struct fbnic_rpc_stats *rpc) +{ + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_UNKN_ETYPE, + &rpc->unkn_etype); + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_UNKN_EXT_HDR, + &rpc->unkn_ext_hdr); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV4_FRAG, &rpc->ipv4_frag); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV6_FRAG, &rpc->ipv6_frag); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV4_ESP, &rpc->ipv4_esp); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_IPV6_ESP, &rpc->ipv6_esp); + fbnic_hw_stat_rst32(fbd, FBNIC_RPC_CNTR_TCP_OPT_ERR, &rpc->tcp_opt_err); + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_OUT_OF_HDR_ERR, + &rpc->out_of_hdr_err); + fbnic_hw_stat_rst32(fbd, + FBNIC_RPC_CNTR_OVR_SIZE_ERR, + &rpc->ovr_size_err); +} + +static void fbnic_get_rpc_stats32(struct fbnic_dev *fbd, + struct fbnic_rpc_stats *rpc) +{ + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_UNKN_ETYPE, + &rpc->unkn_etype); + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_UNKN_EXT_HDR, + &rpc->unkn_ext_hdr); + + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV4_FRAG, &rpc->ipv4_frag); + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV6_FRAG, &rpc->ipv6_frag); + + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV4_ESP, &rpc->ipv4_esp); + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_IPV6_ESP, &rpc->ipv6_esp); + + fbnic_hw_stat_rd32(fbd, FBNIC_RPC_CNTR_TCP_OPT_ERR, &rpc->tcp_opt_err); + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_OUT_OF_HDR_ERR, + &rpc->out_of_hdr_err); + fbnic_hw_stat_rd32(fbd, + FBNIC_RPC_CNTR_OVR_SIZE_ERR, + &rpc->ovr_size_err); +} + +static void fbnic_reset_rxb_fifo_stats(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_fifo_stats *fifo) +{ + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_DROP_FRMS_STS(i), + &fifo->drop.frames); + fbnic_hw_stat_rst64(fbd, FBNIC_RXB_DROP_BYTES_STS_L(i), 1, + &fifo->drop.bytes); + + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_TRUN_FRMS_STS(i), + &fifo->trunc.frames); + fbnic_hw_stat_rst64(fbd, FBNIC_RXB_TRUN_BYTES_STS_L(i), 1, + &fifo->trunc.bytes); + + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_TRANS_DROP_STS(i), + &fifo->trans_drop); + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_TRANS_ECN_STS(i), + &fifo->trans_ecn); + + fifo->level.u.old_reg_value_32 = 0; +} + +static void fbnic_reset_rxb_enq_stats(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_enqueue_stats *enq) +{ + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_DRBO_FRM_CNT_SRC(i), + &enq->drbo.frames); + fbnic_hw_stat_rst64(fbd, FBNIC_RXB_DRBO_BYTE_CNT_SRC_L(i), 4, + &enq->drbo.bytes); + + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_INTEGRITY_ERR(i), + &enq->integrity_err); + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_MAC_ERR(i), + &enq->mac_err); + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_PARSER_ERR(i), + &enq->parser_err); + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_FRM_ERR(i), + &enq->frm_err); +} + +static void fbnic_reset_rxb_deq_stats(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_dequeue_stats *deq) +{ + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_INTF_FRM_CNT_DST(i), + &deq->intf.frames); + fbnic_hw_stat_rst64(fbd, FBNIC_RXB_INTF_BYTE_CNT_DST_L(i), 4, + &deq->intf.bytes); + + fbnic_hw_stat_rst32(fbd, FBNIC_RXB_PBUF_FRM_CNT_DST(i), + &deq->pbuf.frames); + fbnic_hw_stat_rst64(fbd, FBNIC_RXB_PBUF_BYTE_CNT_DST_L(i), 4, + &deq->pbuf.bytes); +} + +static void fbnic_reset_rxb_stats(struct fbnic_dev *fbd, + struct fbnic_rxb_stats *rxb) +{ + int i; + + for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++) + fbnic_reset_rxb_fifo_stats(fbd, i, &rxb->fifo[i]); + + for (i = 0; i < FBNIC_RXB_INTF_INDICES; i++) { + fbnic_reset_rxb_enq_stats(fbd, i, &rxb->enq[i]); + fbnic_reset_rxb_deq_stats(fbd, i, &rxb->deq[i]); + } +} + +static void fbnic_get_rxb_fifo_stats32(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_fifo_stats *fifo) +{ + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_DROP_FRMS_STS(i), + &fifo->drop.frames); + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_TRUN_FRMS_STS(i), + &fifo->trunc.frames); + + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_TRANS_DROP_STS(i), + &fifo->trans_drop); + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_TRANS_ECN_STS(i), + &fifo->trans_ecn); + + fifo->level.value = rd32(fbd, FBNIC_RXB_PBUF_FIFO_LEVEL(i)); +} + +static void fbnic_get_rxb_fifo_stats(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_fifo_stats *fifo) +{ + fbnic_hw_stat_rd64(fbd, FBNIC_RXB_DROP_BYTES_STS_L(i), 1, + &fifo->drop.bytes); + fbnic_hw_stat_rd64(fbd, FBNIC_RXB_TRUN_BYTES_STS_L(i), 1, + &fifo->trunc.bytes); + + fbnic_get_rxb_fifo_stats32(fbd, i, fifo); +} + +static void fbnic_get_rxb_enq_stats32(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_enqueue_stats *enq) +{ + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_DRBO_FRM_CNT_SRC(i), + &enq->drbo.frames); + + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_INTEGRITY_ERR(i), + &enq->integrity_err); + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_MAC_ERR(i), + &enq->mac_err); + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_PARSER_ERR(i), + &enq->parser_err); + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_FRM_ERR(i), + &enq->frm_err); +} + +static void fbnic_get_rxb_enq_stats(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_enqueue_stats *enq) +{ + fbnic_hw_stat_rd64(fbd, FBNIC_RXB_DRBO_BYTE_CNT_SRC_L(i), 4, + &enq->drbo.bytes); + + fbnic_get_rxb_enq_stats32(fbd, i, enq); +} + +static void fbnic_get_rxb_deq_stats32(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_dequeue_stats *deq) +{ + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_INTF_FRM_CNT_DST(i), + &deq->intf.frames); + fbnic_hw_stat_rd32(fbd, FBNIC_RXB_PBUF_FRM_CNT_DST(i), + &deq->pbuf.frames); +} + +static void fbnic_get_rxb_deq_stats(struct fbnic_dev *fbd, int i, + struct fbnic_rxb_dequeue_stats *deq) +{ + fbnic_hw_stat_rd64(fbd, FBNIC_RXB_INTF_BYTE_CNT_DST_L(i), 4, + &deq->intf.bytes); + fbnic_hw_stat_rd64(fbd, FBNIC_RXB_PBUF_BYTE_CNT_DST_L(i), 4, + &deq->pbuf.bytes); + + fbnic_get_rxb_deq_stats32(fbd, i, deq); +} + +static void fbnic_get_rxb_stats32(struct fbnic_dev *fbd, + struct fbnic_rxb_stats *rxb) +{ + int i; + + for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++) + fbnic_get_rxb_fifo_stats32(fbd, i, &rxb->fifo[i]); + + for (i = 0; i < FBNIC_RXB_INTF_INDICES; i++) { + fbnic_get_rxb_enq_stats32(fbd, i, &rxb->enq[i]); + fbnic_get_rxb_deq_stats32(fbd, i, &rxb->deq[i]); + } +} + +static void fbnic_get_rxb_stats(struct fbnic_dev *fbd, + struct fbnic_rxb_stats *rxb) +{ + int i; + + for (i = 0; i < FBNIC_RXB_FIFO_INDICES; i++) + fbnic_get_rxb_fifo_stats(fbd, i, &rxb->fifo[i]); + + for (i = 0; i < FBNIC_RXB_INTF_INDICES; i++) { + fbnic_get_rxb_enq_stats(fbd, i, &rxb->enq[i]); + fbnic_get_rxb_deq_stats(fbd, i, &rxb->deq[i]); + } +} + +static void fbnic_reset_hw_rxq_stats(struct fbnic_dev *fbd, + struct fbnic_hw_q_stats *hw_q) +{ + int i; + + for (i = 0; i < fbd->max_num_queues; i++, hw_q++) { + u32 base = FBNIC_QUEUE(i); + + fbnic_hw_stat_rst32(fbd, + base + FBNIC_QUEUE_RDE_PKT_ERR_CNT, + &hw_q->rde_pkt_err); + fbnic_hw_stat_rst32(fbd, + base + FBNIC_QUEUE_RDE_CQ_DROP_CNT, + &hw_q->rde_pkt_cq_drop); + fbnic_hw_stat_rst32(fbd, + base + FBNIC_QUEUE_RDE_BDQ_DROP_CNT, + &hw_q->rde_pkt_bdq_drop); + } +} + +static void fbnic_get_hw_rxq_stats32(struct fbnic_dev *fbd, + struct fbnic_hw_q_stats *hw_q) +{ + int i; + + for (i = 0; i < fbd->max_num_queues; i++, hw_q++) { + u32 base = FBNIC_QUEUE(i); + + fbnic_hw_stat_rd32(fbd, + base + FBNIC_QUEUE_RDE_PKT_ERR_CNT, + &hw_q->rde_pkt_err); + fbnic_hw_stat_rd32(fbd, + base + FBNIC_QUEUE_RDE_CQ_DROP_CNT, + &hw_q->rde_pkt_cq_drop); + fbnic_hw_stat_rd32(fbd, + base + FBNIC_QUEUE_RDE_BDQ_DROP_CNT, + &hw_q->rde_pkt_bdq_drop); + } +} + +void fbnic_get_hw_q_stats(struct fbnic_dev *fbd, + struct fbnic_hw_q_stats *hw_q) +{ + spin_lock(&fbd->hw_stats.lock); + fbnic_get_hw_rxq_stats32(fbd, hw_q); + spin_unlock(&fbd->hw_stats.lock); +} + +static void fbnic_reset_pcie_stats_asic(struct fbnic_dev *fbd, + struct fbnic_pcie_stats *pcie) +{ + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0, + 1, + &pcie->ob_rd_tlp); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0, + 1, + &pcie->ob_rd_dword); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0, + 1, + &pcie->ob_cpl_tlp); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0, + 1, + &pcie->ob_cpl_dword); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0, + 1, + &pcie->ob_wr_tlp); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0, + 1, + &pcie->ob_wr_dword); + + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0, + 1, + &pcie->ob_rd_no_tag); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0, + 1, + &pcie->ob_rd_no_cpl_cred); + fbnic_hw_stat_rst64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0, + 1, + &pcie->ob_rd_no_np_cred); +} + +static void fbnic_get_pcie_stats_asic64(struct fbnic_dev *fbd, + struct fbnic_pcie_stats *pcie) +{ + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_TLP_CNT_31_0, + 1, + &pcie->ob_rd_tlp); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DWORD_CNT_31_0, + 1, + &pcie->ob_rd_dword); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_WR_TLP_CNT_31_0, + 1, + &pcie->ob_wr_tlp); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_WR_DWORD_CNT_31_0, + 1, + &pcie->ob_wr_dword); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_CPL_TLP_CNT_31_0, + 1, + &pcie->ob_cpl_tlp); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_CPL_DWORD_CNT_31_0, + 1, + &pcie->ob_cpl_dword); + + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_TAG_31_0, + 1, + &pcie->ob_rd_no_tag); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_CPL_CRED_31_0, + 1, + &pcie->ob_rd_no_cpl_cred); + fbnic_hw_stat_rd64(fbd, + FBNIC_PUL_USER_OB_RD_DBG_CNT_NP_CRED_31_0, + 1, + &pcie->ob_rd_no_np_cred); +} + +static void fbnic_reset_phy_stats(struct fbnic_dev *fbd, + struct fbnic_phy_stats *phy_stats) +{ + const struct fbnic_mac *mac = fbd->mac; + + mac->get_fec_stats(fbd, true, &phy_stats->fec); + mac->get_pcs_stats(fbd, true, &phy_stats->pcs); +} + +static void fbnic_get_phy_stats32(struct fbnic_dev *fbd, + struct fbnic_phy_stats *phy_stats) +{ + const struct fbnic_mac *mac = fbd->mac; + + mac->get_fec_stats(fbd, false, &phy_stats->fec); + mac->get_pcs_stats(fbd, false, &phy_stats->pcs); +} + +static void fbnic_reset_hw_mac_stats(struct fbnic_dev *fbd, + struct fbnic_mac_stats *mac_stats) +{ + const struct fbnic_mac *mac = fbd->mac; + + mac->get_eth_mac_stats(fbd, true, &mac_stats->eth_mac); + mac->get_pause_stats(fbd, true, &mac_stats->pause); + mac->get_eth_ctrl_stats(fbd, true, &mac_stats->eth_ctrl); + mac->get_rmon_stats(fbd, true, &mac_stats->rmon); +} + +void fbnic_reset_hw_stats(struct fbnic_dev *fbd) +{ + spin_lock(&fbd->hw_stats.lock); + fbnic_reset_phy_stats(fbd, &fbd->hw_stats.phy); + fbnic_reset_tmi_stats(fbd, &fbd->hw_stats.tmi); + fbnic_reset_tti_stats(fbd, &fbd->hw_stats.tti); + fbnic_reset_rpc_stats(fbd, &fbd->hw_stats.rpc); + fbnic_reset_rxb_stats(fbd, &fbd->hw_stats.rxb); + fbnic_reset_hw_rxq_stats(fbd, fbd->hw_stats.hw_q); + fbnic_reset_pcie_stats_asic(fbd, &fbd->hw_stats.pcie); + spin_unlock(&fbd->hw_stats.lock); + + /* Once registered, the only other access to MAC stats is via the + * ethtool API which is protected by the rtnl_lock. The call to + * fbnic_reset_hw_stats() during PCI recovery is also protected + * by the rtnl_lock hence, we don't need the spinlock to access + * the MAC stats. + */ + if (fbd->netdev) + ASSERT_RTNL(); + fbnic_reset_hw_mac_stats(fbd, &fbd->hw_stats.mac); +} + +void fbnic_init_hw_stats(struct fbnic_dev *fbd) +{ + spin_lock_init(&fbd->hw_stats.lock); + + fbnic_reset_hw_stats(fbd); +} + +static void __fbnic_get_hw_stats32(struct fbnic_dev *fbd) +{ + fbnic_get_phy_stats32(fbd, &fbd->hw_stats.phy); + fbnic_get_tmi_stats32(fbd, &fbd->hw_stats.tmi); + fbnic_get_tti_stats32(fbd, &fbd->hw_stats.tti); + fbnic_get_rpc_stats32(fbd, &fbd->hw_stats.rpc); + fbnic_get_rxb_stats32(fbd, &fbd->hw_stats.rxb); + fbnic_get_hw_rxq_stats32(fbd, fbd->hw_stats.hw_q); +} + +void fbnic_get_hw_stats32(struct fbnic_dev *fbd) +{ + spin_lock(&fbd->hw_stats.lock); + __fbnic_get_hw_stats32(fbd); + spin_unlock(&fbd->hw_stats.lock); +} + +void fbnic_get_hw_stats(struct fbnic_dev *fbd) +{ + spin_lock(&fbd->hw_stats.lock); + __fbnic_get_hw_stats32(fbd); + + fbnic_get_tmi_stats(fbd, &fbd->hw_stats.tmi); + fbnic_get_tti_stats(fbd, &fbd->hw_stats.tti); + fbnic_get_rxb_stats(fbd, &fbd->hw_stats.rxb); + fbnic_get_pcie_stats_asic64(fbd, &fbd->hw_stats.pcie); + spin_unlock(&fbd->hw_stats.lock); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h new file mode 100644 index 000000000..caea4be46 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hw_stats.h @@ -0,0 +1,164 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_HW_STATS_H_ +#define _FBNIC_HW_STATS_H_ + +#include <linux/ethtool.h> +#include <linux/spinlock.h> + +#include "fbnic_csr.h" + +struct fbnic_stat_counter { + u64 value; + union { + u32 old_reg_value_32; + u64 old_reg_value_64; + } u; + bool reported; +}; + +struct fbnic_hw_stat { + struct fbnic_stat_counter frames; + struct fbnic_stat_counter bytes; +}; + +struct fbnic_fec_stats { + struct fbnic_stat_counter corrected_blocks, uncorrectable_blocks; +}; + +struct fbnic_pcs_stats { + struct { + struct fbnic_stat_counter lanes[FBNIC_PCS_MAX_LANES]; + } SymbolErrorDuringCarrier; +}; + +/* Note: not updated by fbnic_get_hw_stats() */ +struct fbnic_eth_ctrl_stats { + struct fbnic_stat_counter MACControlFramesTransmitted; + struct fbnic_stat_counter MACControlFramesReceived; +}; + +/* Note: not updated by fbnic_get_hw_stats() */ +struct fbnic_rmon_stats { + struct fbnic_stat_counter undersize_pkts; + struct fbnic_stat_counter oversize_pkts; + struct fbnic_stat_counter fragments; + struct fbnic_stat_counter jabbers; + + struct fbnic_stat_counter hist[ETHTOOL_RMON_HIST_MAX]; + struct fbnic_stat_counter hist_tx[ETHTOOL_RMON_HIST_MAX]; +}; + +/* Note: not updated by fbnic_get_hw_stats() */ +struct fbnic_pause_stats { + struct fbnic_stat_counter tx_pause_frames; + struct fbnic_stat_counter rx_pause_frames; + struct fbnic_stat_counter tx_pause_storm_events; +}; + +struct fbnic_eth_mac_stats { + struct fbnic_stat_counter FramesTransmittedOK; + struct fbnic_stat_counter FramesReceivedOK; + struct fbnic_stat_counter FrameCheckSequenceErrors; + struct fbnic_stat_counter AlignmentErrors; + struct fbnic_stat_counter OctetsTransmittedOK; + struct fbnic_stat_counter FramesLostDueToIntMACXmitError; + struct fbnic_stat_counter OctetsReceivedOK; + struct fbnic_stat_counter FramesLostDueToIntMACRcvError; + struct fbnic_stat_counter MulticastFramesXmittedOK; + struct fbnic_stat_counter BroadcastFramesXmittedOK; + struct fbnic_stat_counter MulticastFramesReceivedOK; + struct fbnic_stat_counter BroadcastFramesReceivedOK; + struct fbnic_stat_counter FrameTooLongErrors; +}; + +struct fbnic_phy_stats { + struct fbnic_fec_stats fec; + struct fbnic_pcs_stats pcs; +}; + +struct fbnic_mac_stats { + struct fbnic_eth_mac_stats eth_mac; + struct fbnic_pause_stats pause; + struct fbnic_eth_ctrl_stats eth_ctrl; + struct fbnic_rmon_stats rmon; +}; + +struct fbnic_tmi_stats { + struct fbnic_hw_stat drop; + struct fbnic_stat_counter ptp_illegal_req, ptp_good_ts, ptp_bad_ts; +}; + +struct fbnic_tti_stats { + struct fbnic_hw_stat cm_drop, frame_drop, tbi_drop; +}; + +struct fbnic_rpc_stats { + struct fbnic_stat_counter unkn_etype, unkn_ext_hdr; + struct fbnic_stat_counter ipv4_frag, ipv6_frag, ipv4_esp, ipv6_esp; + struct fbnic_stat_counter tcp_opt_err, out_of_hdr_err, ovr_size_err; +}; + +struct fbnic_rxb_enqueue_stats { + struct fbnic_hw_stat drbo; + struct fbnic_stat_counter integrity_err, mac_err; + struct fbnic_stat_counter parser_err, frm_err; +}; + +struct fbnic_rxb_fifo_stats { + struct fbnic_hw_stat drop, trunc; + struct fbnic_stat_counter trans_drop, trans_ecn; + struct fbnic_stat_counter level; +}; + +struct fbnic_rxb_dequeue_stats { + struct fbnic_hw_stat intf, pbuf; +}; + +struct fbnic_rxb_stats { + struct fbnic_rxb_enqueue_stats enq[FBNIC_RXB_ENQUEUE_INDICES]; + struct fbnic_rxb_fifo_stats fifo[FBNIC_RXB_FIFO_INDICES]; + struct fbnic_rxb_dequeue_stats deq[FBNIC_RXB_DEQUEUE_INDICES]; +}; + +struct fbnic_hw_q_stats { + struct fbnic_stat_counter rde_pkt_err; + struct fbnic_stat_counter rde_pkt_cq_drop; + struct fbnic_stat_counter rde_pkt_bdq_drop; +}; + +struct fbnic_pcie_stats { + struct fbnic_stat_counter ob_rd_tlp, ob_rd_dword; + struct fbnic_stat_counter ob_wr_tlp, ob_wr_dword; + struct fbnic_stat_counter ob_cpl_tlp, ob_cpl_dword; + + struct fbnic_stat_counter ob_rd_no_tag; + struct fbnic_stat_counter ob_rd_no_cpl_cred; + struct fbnic_stat_counter ob_rd_no_np_cred; +}; + +struct fbnic_hw_stats { + struct fbnic_phy_stats phy; + struct fbnic_mac_stats mac; + struct fbnic_tmi_stats tmi; + struct fbnic_tti_stats tti; + struct fbnic_rpc_stats rpc; + struct fbnic_rxb_stats rxb; + struct fbnic_hw_q_stats hw_q[FBNIC_MAX_QUEUES]; + struct fbnic_pcie_stats pcie; + + /* Lock protecting the access to hw stats */ + spinlock_t lock; +}; + +u64 fbnic_stat_rd64(struct fbnic_dev *fbd, u32 reg, u32 offset); + +void fbnic_reset_hw_stats(struct fbnic_dev *fbd); +void fbnic_init_hw_stats(struct fbnic_dev *fbd); +void fbnic_get_hw_q_stats(struct fbnic_dev *fbd, + struct fbnic_hw_q_stats *hw_q); +void fbnic_get_hw_stats32(struct fbnic_dev *fbd); +void fbnic_get_hw_stats(struct fbnic_dev *fbd); + +#endif /* _FBNIC_HW_STATS_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_hwmon.c b/drivers/net/ethernet/meta/fbnic/fbnic_hwmon.c new file mode 100644 index 000000000..def8598ac --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_hwmon.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/hwmon.h> + +#include "fbnic.h" +#include "fbnic_mac.h" + +static int fbnic_hwmon_sensor_id(enum hwmon_sensor_types type) +{ + if (type == hwmon_temp) + return FBNIC_SENSOR_TEMP; + if (type == hwmon_in) + return FBNIC_SENSOR_VOLTAGE; + + return -EOPNOTSUPP; +} + +static umode_t fbnic_hwmon_is_visible(const void *drvdata, + enum hwmon_sensor_types type, + u32 attr, int channel) +{ + if (type == hwmon_temp && attr == hwmon_temp_input) + return 0444; + if (type == hwmon_in && attr == hwmon_in_input) + return 0444; + + return 0; +} + +static int fbnic_hwmon_read(struct device *dev, enum hwmon_sensor_types type, + u32 attr, int channel, long *val) +{ + struct fbnic_dev *fbd = dev_get_drvdata(dev); + const struct fbnic_mac *mac = fbd->mac; + int id; + + id = fbnic_hwmon_sensor_id(type); + return id < 0 ? id : mac->get_sensor(fbd, id, val); +} + +static const struct hwmon_ops fbnic_hwmon_ops = { + .is_visible = fbnic_hwmon_is_visible, + .read = fbnic_hwmon_read, +}; + +static const struct hwmon_channel_info *fbnic_hwmon_info[] = { + HWMON_CHANNEL_INFO(temp, HWMON_T_INPUT), + HWMON_CHANNEL_INFO(in, HWMON_I_INPUT), + NULL +}; + +static const struct hwmon_chip_info fbnic_chip_info = { + .ops = &fbnic_hwmon_ops, + .info = fbnic_hwmon_info, +}; + +void fbnic_hwmon_register(struct fbnic_dev *fbd) +{ + if (!IS_REACHABLE(CONFIG_HWMON)) + return; + + fbd->hwmon = hwmon_device_register_with_info(fbd->dev, "fbnic", + fbd, &fbnic_chip_info, + NULL); + if (IS_ERR(fbd->hwmon)) { + dev_notice(fbd->dev, + "Failed to register hwmon device %pe\n", + fbd->hwmon); + fbd->hwmon = NULL; + } +} + +void fbnic_hwmon_unregister(struct fbnic_dev *fbd) +{ + if (!IS_REACHABLE(CONFIG_HWMON) || !fbd->hwmon) + return; + + hwmon_device_unregister(fbd->hwmon); + fbd->hwmon = NULL; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_irq.c b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c new file mode 100644 index 000000000..5e383d40a --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_irq.c @@ -0,0 +1,469 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/pci.h> +#include <linux/types.h> + +#include "fbnic.h" +#include "fbnic_netdev.h" +#include "fbnic_txrx.h" + +static irqreturn_t fbnic_fw_msix_intr(int __always_unused irq, void *data) +{ + struct fbnic_dev *fbd = (struct fbnic_dev *)data; + + fbnic_mbx_poll(fbd); + + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY); + + return IRQ_HANDLED; +} + +static int __fbnic_fw_enable_mbx(struct fbnic_dev *fbd, int vector) +{ + int err; + + /* Initialize mailbox and attempt to poll it into ready state */ + fbnic_mbx_init(fbd); + err = fbnic_mbx_poll_tx_ready(fbd); + if (err) { + dev_warn(fbd->dev, "FW mailbox did not enter ready state\n"); + return err; + } + + /* Enable interrupt and unmask the vector */ + enable_irq(vector); + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_FW_MSIX_ENTRY); + + return 0; +} + +/** + * fbnic_fw_request_mbx - Configure and initialize Firmware Mailbox + * @fbd: Pointer to device to initialize + * + * This function will allocate the IRQ and then reinitialize the mailbox + * starting communication between the host and firmware. + * + * Return: non-zero on failure. + **/ +int fbnic_fw_request_mbx(struct fbnic_dev *fbd) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + int vector, err; + + WARN_ON(fbd->fw_msix_vector); + + vector = pci_irq_vector(pdev, FBNIC_FW_MSIX_ENTRY); + if (vector < 0) + return vector; + + /* Request the IRQ for FW Mailbox vector. */ + err = request_threaded_irq(vector, NULL, &fbnic_fw_msix_intr, + IRQF_ONESHOT | IRQF_NO_AUTOEN, + dev_name(fbd->dev), fbd); + if (err) + return err; + + /* Initialize mailbox and attempt to poll it into ready state */ + err = __fbnic_fw_enable_mbx(fbd, vector); + if (err) + free_irq(vector, fbd); + + fbd->fw_msix_vector = vector; + + return err; +} + +/** + * fbnic_fw_disable_mbx - Temporarily place mailbox in standby state + * @fbd: Pointer to device + * + * Shutdown the mailbox by notifying the firmware to stop sending us logs, mask + * and synchronize the IRQ, and then clean up the rings. + **/ +static void fbnic_fw_disable_mbx(struct fbnic_dev *fbd) +{ + /* Disable interrupt and synchronize the IRQ */ + disable_irq(fbd->fw_msix_vector); + + /* Mask the vector */ + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(0), 1u << FBNIC_FW_MSIX_ENTRY); + + /* Make sure disabling logs message is sent, must be done here to + * avoid risk of completing without a running interrupt. + */ + fbnic_mbx_flush_tx(fbd); + fbnic_mbx_clean(fbd); +} + +/** + * fbnic_fw_free_mbx - Disable mailbox and place it in standby state + * @fbd: Pointer to device to disable + * + * This function will disable the mailbox interrupt, free any messages still + * in the mailbox and place it into a disabled state. The firmware is + * expected to see the update and assume that the host is in the reset state. + **/ +void fbnic_fw_free_mbx(struct fbnic_dev *fbd) +{ + /* Vector has already been freed */ + if (!fbd->fw_msix_vector) + return; + + fbnic_fw_disable_mbx(fbd); + + /* Free the vector */ + free_irq(fbd->fw_msix_vector, fbd); + fbd->fw_msix_vector = 0; +} + +static irqreturn_t fbnic_mac_msix_intr(int __always_unused irq, void *data) +{ + struct fbnic_dev *fbd = data; + struct fbnic_net *fbn; + + if (fbd->mac->get_link_event(fbd) == FBNIC_LINK_EVENT_NONE) { + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), + 1u << FBNIC_PCS_MSIX_ENTRY); + return IRQ_HANDLED; + } + + fbn = netdev_priv(fbd->netdev); + + /* Record link down events */ + if (!fbd->mac->get_link(fbd, fbn->aui, fbn->fec)) + phylink_pcs_change(fbn->pcs, false); + + return IRQ_HANDLED; +} + +/** + * fbnic_mac_request_irq - Configure the MAC to enable it to advertise link + * @fbd: Pointer to device to initialize + * + * This function provides basic bringup for the MAC/PHY IRQ. For now the IRQ + * will remain disabled until we start the MAC/PCS/PHY logic via phylink. + * + * Return: non-zero on failure. + **/ +int fbnic_mac_request_irq(struct fbnic_dev *fbd) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + int vector, err; + + WARN_ON(fbd->mac_msix_vector); + + vector = pci_irq_vector(pdev, FBNIC_PCS_MSIX_ENTRY); + if (vector < 0) + return vector; + + /* Request the IRQ for PCS link vector. + * Map PCS cause to it, and unmask it + */ + err = request_irq(vector, &fbnic_mac_msix_intr, 0, + fbd->netdev->name, fbd); + if (err) + return err; + + /* Map and enable interrupt, unmask vector after link is configured */ + fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX), + FBNIC_PCS_MSIX_ENTRY | FBNIC_INTR_MSIX_CTRL_ENABLE); + + fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_RXB_IDX), 0); + + fbd->mac_msix_vector = vector; + + return 0; +} + +/** + * fbnic_mac_free_irq - Teardown the MAC IRQ to prepare for stopping + * @fbd: Pointer to device that is stopping + * + * This function undoes the work done in fbnic_mac_request_irq and prepares + * the device to no longer receive traffic on the host interface. + **/ +void fbnic_mac_free_irq(struct fbnic_dev *fbd) +{ + /* Vector has already been freed */ + if (!fbd->mac_msix_vector) + return; + + /* Disable interrupt */ + fbnic_wr32(fbd, FBNIC_INTR_MSIX_CTRL(FBNIC_INTR_MSIX_CTRL_PCS_IDX), + FBNIC_PCS_MSIX_ENTRY); + fbnic_wrfl(fbd); + + /* Synchronize IRQ to prevent race that would unmask vector */ + synchronize_irq(fbd->mac_msix_vector); + + /* Mask the vector */ + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(0), 1u << FBNIC_PCS_MSIX_ENTRY); + + /* Free the vector */ + free_irq(fbd->mac_msix_vector, fbd); + fbd->mac_msix_vector = 0; +} + +void fbnic_synchronize_irq(struct fbnic_dev *fbd, int nr) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + int irq = pci_irq_vector(pdev, nr); + + if (irq < 0) + return; + + synchronize_irq(irq); +} + +int fbnic_request_irq(struct fbnic_dev *fbd, int nr, irq_handler_t handler, + unsigned long flags, const char *name, void *data) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + int irq = pci_irq_vector(pdev, nr); + + if (irq < 0) + return irq; + + return request_irq(irq, handler, flags, name, data); +} + +void fbnic_free_irq(struct fbnic_dev *fbd, int nr, void *data) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + int irq = pci_irq_vector(pdev, nr); + + if (irq < 0) + return; + + free_irq(irq, data); +} + +struct fbnic_msix_test_data { + struct fbnic_dev *fbd; + unsigned long test_msix_status[BITS_TO_LONGS(FBNIC_MAX_MSIX_VECS)]; + int irq_vector[FBNIC_MAX_MSIX_VECS]; +}; + +static irqreturn_t fbnic_irq_test(int irq, void *data) +{ + struct fbnic_msix_test_data *test_data = data; + struct fbnic_dev *fbd = test_data->fbd; + int i; + + for (i = fbd->num_irqs; i--;) { + if (test_data->irq_vector[i] == irq) { + set_bit(i, test_data->test_msix_status); + break; + } + } + + return IRQ_HANDLED; +} + +/** + * fbnic_msix_test - Verify behavior of NIC interrupts + * @fbd: device to test + * + * This function is meant to test the global interrupt registers and the + * PCIe IP MSI-X functionality. It essentially goes through and tests + * various combinations of the set, clear, and mask bits in order to + * verify the behavior is as we expect it to be from the driver. + * + * Return: See enum fbnic_msix_self_test_codes + **/ +enum fbnic_msix_self_test_codes fbnic_msix_test(struct fbnic_dev *fbd) +{ + enum fbnic_msix_self_test_codes result = FBNIC_TEST_MSIX_SUCCESS; + struct pci_dev *pdev = to_pci_dev(fbd->dev); + struct fbnic_msix_test_data *test_data; + u32 mask = 0; + int i; + + /* Allocate bitmap and IRQ vector table */ + test_data = kzalloc_obj(*test_data, GFP_KERNEL); + + /* memory allocation failure */ + if (!test_data) + return FBNIC_TEST_MSIX_NOMEM; + + /* Initialize test data */ + test_data->fbd = fbd; + + for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { + /* Add IRQ to vector table so it can be found */ + test_data->irq_vector[i] = pci_irq_vector(pdev, i); + + /* Enable the interrupt */ + if (!fbnic_request_irq(fbd, i, fbnic_irq_test, 0, + fbd->netdev->name, test_data)) + continue; + + while (i-- > FBNIC_NON_NAPI_VECTORS) + fbnic_free_irq(fbd, i, test_data); + kfree(test_data); + + /* IRQ request failure */ + return FBNIC_TEST_MSIX_IRQ_REQ_FAIL; + } + + /* Test each bit individually */ + for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { + mask = 1U << (i % 32); + + /* Start with mask set and interrupt cleared */ + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask); + fbnic_wrfl(fbd); + fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + + /* masking failure to prevent interrupt */ + result = FBNIC_TEST_MSIX_MASK; + + fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (test_bit(i, test_data->test_msix_status)) + break; + + /* unmasking failure w/ sw status set */ + result = FBNIC_TEST_MSIX_UNMASK; + + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (!test_bit(i, test_data->test_msix_status)) + break; + + /* interrupt when clearing mask */ + result = FBNIC_TEST_MSIX_IRQ_CLEAR; + + clear_bit(i, test_data->test_msix_status); + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (test_bit(i, test_data->test_msix_status)) + break; + + /* interrupt not triggering when not masked */ + result = FBNIC_TEST_MSIX_NO_INTERRUPT; + + fbnic_wr32(fbd, FBNIC_INTR_SET(i / 32), mask); + fbnic_wrfl(fbd); + usleep_range(10000, 11000); + + if (!test_bit(i, test_data->test_msix_status)) + break; + + /* status not cleared, or mask not set */ + result = FBNIC_TEST_MSIX_NO_CLEAR_OR_MASK; + if (mask & fbnic_rd32(fbd, FBNIC_INTR_STATUS(i / 32))) + break; + if (!(mask & fbnic_rd32(fbd, FBNIC_INTR_MASK(i / 32)))) + break; + + /* Result = 0 - Success */ + result = FBNIC_TEST_MSIX_SUCCESS; + + clear_bit(i, test_data->test_msix_status); + } + + if (i < fbd->num_irqs) { + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(i / 32), mask); + fbnic_wrfl(fbd); + fbnic_wr32(fbd, FBNIC_INTR_CLEAR(i / 32), mask); + fbnic_wrfl(fbd); + clear_bit(i, test_data->test_msix_status); + } + + for (i = FBNIC_NON_NAPI_VECTORS; i < fbd->num_irqs; i++) { + /* Test for bits set after testing */ + if (test_bit(i, test_data->test_msix_status)) + result = FBNIC_TEST_MSIX_BITS_SET_AFTER_TEST; + + /* Free IRQ */ + fbnic_free_irq(fbd, i, test_data); + } + + kfree(test_data); + + return result; +} + +void fbnic_napi_name_irqs(struct fbnic_dev *fbd) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(fbd->napi_irq); i++) + snprintf(fbd->napi_irq[i].name, + sizeof(fbd->napi_irq[i].name), + "%s-TxRx-%u", fbd->netdev->name, i); +} + +int fbnic_napi_request_irq(struct fbnic_dev *fbd, + struct fbnic_napi_vector *nv) +{ + struct fbnic_net *fbn = netdev_priv(fbd->netdev); + int i = fbnic_napi_idx(nv); + int err; + + if (!fbd->napi_irq[i].users) { + err = fbnic_request_irq(fbd, nv->v_idx, + fbnic_msix_clean_rings, 0, + fbd->napi_irq[i].name, + &fbn->napi[i]); + if (err) + return err; + } + + fbd->napi_irq[i].users++; + return 0; +} + +void fbnic_napi_free_irq(struct fbnic_dev *fbd, + struct fbnic_napi_vector *nv) +{ + struct fbnic_net *fbn = netdev_priv(fbd->netdev); + int i = fbnic_napi_idx(nv); + + if (--fbd->napi_irq[i].users) + return; + + fbnic_free_irq(fbd, nv->v_idx, &fbn->napi[i]); +} + +void fbnic_free_irqs(struct fbnic_dev *fbd) +{ + struct pci_dev *pdev = to_pci_dev(fbd->dev); + + fbd->num_irqs = 0; + + pci_free_irq_vectors(pdev); +} + +int fbnic_alloc_irqs(struct fbnic_dev *fbd) +{ + unsigned int wanted_irqs = FBNIC_NON_NAPI_VECTORS; + struct pci_dev *pdev = to_pci_dev(fbd->dev); + int num_irqs; + + wanted_irqs += min_t(unsigned int, num_online_cpus(), FBNIC_MAX_RXQS); + num_irqs = pci_alloc_irq_vectors(pdev, FBNIC_NON_NAPI_VECTORS + 1, + wanted_irqs, PCI_IRQ_MSIX); + if (num_irqs < 0) { + dev_err(fbd->dev, "Failed to allocate MSI-X entries\n"); + return num_irqs; + } + + if (num_irqs < wanted_irqs) + dev_warn(fbd->dev, "Allocated %d IRQs, expected %d\n", + num_irqs, wanted_irqs); + + fbd->num_irqs = num_irqs; + + return 0; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.c b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c new file mode 100644 index 000000000..53b7a938b --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.c @@ -0,0 +1,1031 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <net/tcp.h> + +#include "fbnic.h" +#include "fbnic_mac.h" +#include "fbnic_netdev.h" + +static void fbnic_init_readrq(struct fbnic_dev *fbd, unsigned int offset, + unsigned int cls, unsigned int readrq) +{ + u32 val = rd32(fbd, offset); + + /* The TDF_CTL masks are a superset of the RNI_RBP ones. So we can + * use them when setting either the TDE_CTF or RNI_RBP registers. + */ + val &= FBNIC_QM_TNI_TDF_CTL_MAX_OT | FBNIC_QM_TNI_TDF_CTL_MAX_OB; + + val |= FIELD_PREP(FBNIC_QM_TNI_TDF_CTL_MRRS, readrq) | + FIELD_PREP(FBNIC_QM_TNI_TDF_CTL_CLS, cls); + + wr32(fbd, offset, val); +} + +static void fbnic_init_mps(struct fbnic_dev *fbd, unsigned int offset, + unsigned int cls, unsigned int mps) +{ + u32 val = rd32(fbd, offset); + + /* Currently all MPS masks are identical so just use the first one */ + val &= ~(FBNIC_QM_TNI_TCM_CTL_MPS | FBNIC_QM_TNI_TCM_CTL_CLS); + + val |= FIELD_PREP(FBNIC_QM_TNI_TCM_CTL_MPS, mps) | + FIELD_PREP(FBNIC_QM_TNI_TCM_CTL_CLS, cls); + + wr32(fbd, offset, val); +} + +static void fbnic_mac_init_axi(struct fbnic_dev *fbd) +{ + bool override_1k = false; + int readrq, mps, cls; + + /* All of the values are based on being a power of 2 starting + * with 64 == 0. Therefore we can either divide by 64 in the + * case of constants, or just subtract 6 from the log2 of the value + * in order to get the value we will be programming into the + * registers. + */ + readrq = ilog2(fbd->readrq) - 6; + if (readrq > 3) + override_1k = true; + readrq = clamp(readrq, 0, 3); + + mps = ilog2(fbd->mps) - 6; + mps = clamp(mps, 0, 3); + + cls = ilog2(L1_CACHE_BYTES) - 6; + cls = clamp(cls, 0, 3); + + /* Configure Tx/Rx AXI Paths w/ Read Request and Max Payload sizes */ + fbnic_init_readrq(fbd, FBNIC_QM_TNI_TDF_CTL, cls, readrq); + fbnic_init_mps(fbd, FBNIC_QM_TNI_TCM_CTL, cls, mps); + + /* Configure QM TNI TDE: + * - Max outstanding AXI beats to 704(768 - 64) - guaranetees 8% of + * buffer capacity to descriptors. + * - Max outstanding transactions to 128 + */ + wr32(fbd, FBNIC_QM_TNI_TDE_CTL, + FIELD_PREP(FBNIC_QM_TNI_TDE_CTL_MRRS_1K, override_1k ? 1 : 0) | + FIELD_PREP(FBNIC_QM_TNI_TDE_CTL_MAX_OB, 704) | + FIELD_PREP(FBNIC_QM_TNI_TDE_CTL_MAX_OT, 128) | + FIELD_PREP(FBNIC_QM_TNI_TDE_CTL_MRRS, readrq) | + FIELD_PREP(FBNIC_QM_TNI_TDE_CTL_CLS, cls)); + + fbnic_init_readrq(fbd, FBNIC_QM_RNI_RBP_CTL, cls, readrq); + fbnic_init_mps(fbd, FBNIC_QM_RNI_RDE_CTL, cls, mps); + fbnic_init_mps(fbd, FBNIC_QM_RNI_RCM_CTL, cls, mps); +} + +static void fbnic_mac_init_qm(struct fbnic_dev *fbd) +{ + u64 default_meta = FIELD_PREP(FBNIC_TWD_L2_HLEN_MASK, ETH_HLEN) | + FBNIC_TWD_FLAG_REQ_COMPLETION; + u32 clock_freq; + + /* Configure default TWQ Metadata descriptor */ + wr32(fbd, FBNIC_QM_TWQ_DEFAULT_META_L, + lower_32_bits(default_meta)); + wr32(fbd, FBNIC_QM_TWQ_DEFAULT_META_H, + upper_32_bits(default_meta)); + + /* Configure TSO behavior */ + wr32(fbd, FBNIC_QM_TQS_CTL0, + FIELD_PREP(FBNIC_QM_TQS_CTL0_LSO_TS_MASK, + FBNIC_QM_TQS_CTL0_LSO_TS_LAST) | + FIELD_PREP(FBNIC_QM_TQS_CTL0_PREFETCH_THRESH, + FBNIC_QM_TQS_CTL0_PREFETCH_THRESH_MIN)); + + /* Limit EDT to INT_MAX as this is the limit of the EDT Qdisc */ + wr32(fbd, FBNIC_QM_TQS_EDT_TS_RANGE, INT_MAX); + + /* Configure MTU + * Due to known HW issue we cannot set the MTU to within 16 octets + * of a 64 octet aligned boundary. So we will set the TQS_MTU(s) to + * MTU + 1. + */ + wr32(fbd, FBNIC_QM_TQS_MTU_CTL0, FBNIC_MAX_JUMBO_FRAME_SIZE + 1); + wr32(fbd, FBNIC_QM_TQS_MTU_CTL1, + FIELD_PREP(FBNIC_QM_TQS_MTU_CTL1_BULK, + FBNIC_MAX_JUMBO_FRAME_SIZE + 1)); + + clock_freq = FBNIC_CLOCK_FREQ; + + /* Be aggressive on the timings. We will have the interrupt + * threshold timer tick once every 1 usec and coalesce writes for + * up to 80 usecs. + */ + wr32(fbd, FBNIC_QM_TCQ_CTL0, + FIELD_PREP(FBNIC_QM_TCQ_CTL0_TICK_CYCLES, + clock_freq / 1000000) | + FIELD_PREP(FBNIC_QM_TCQ_CTL0_COAL_WAIT, + clock_freq / 12500)); + + /* We will have the interrupt threshold timer tick once every + * 1 usec and coalesce writes for up to 2 usecs. + */ + wr32(fbd, FBNIC_QM_RCQ_CTL0, + FIELD_PREP(FBNIC_QM_RCQ_CTL0_TICK_CYCLES, + clock_freq / 1000000) | + FIELD_PREP(FBNIC_QM_RCQ_CTL0_COAL_WAIT, + clock_freq / 500000)); + + /* Configure spacer control to 64 beats. */ + wr32(fbd, FBNIC_FAB_AXI4_AR_SPACER_2_CFG, + FBNIC_FAB_AXI4_AR_SPACER_MASK | + FIELD_PREP(FBNIC_FAB_AXI4_AR_SPACER_THREADSHOLD, 2)); +} + +#define FBNIC_DROP_EN_MASK 0x7d +#define FBNIC_PAUSE_EN_MASK 0x14 +#define FBNIC_ECN_EN_MASK 0x10 +#define FBNIC_PS_EN_MASK 0x01 + +struct fbnic_fifo_config { + unsigned int addr; + unsigned int size; +}; + +/* Rx FIFO Configuration + * The table consists of 8 entries, of which only 4 are currently used + * The starting addr is in units of 64B and the size is in 2KB units + * Below is the human readable version of the table defined below: + * Function Addr Size + * ---------------------------------- + * Network to Host/BMC 384K 64K + * Unused + * Unused + * Network to BMC 448K 32K + * Network to Host 0 384K + * Unused + * BMC to Host 480K 32K + * Unused + */ +static const struct fbnic_fifo_config fifo_config[] = { + { .addr = 0x1800, .size = 0x20 }, /* Network to Host/BMC */ + { }, /* Unused */ + { }, /* Unused */ + { .addr = 0x1c00, .size = 0x10 }, /* Network to BMC */ + { .addr = 0x0000, .size = 0xc0 }, /* Network to Host */ + { }, /* Unused */ + { .addr = 0x1e00, .size = 0x10 }, /* BMC to Host */ + { } /* Unused */ +}; + +static void fbnic_mac_init_rxb(struct fbnic_dev *fbd) +{ + bool rx_enable; + int i; + + rx_enable = !!(rd32(fbd, FBNIC_RPC_RMI_CONFIG) & + FBNIC_RPC_RMI_CONFIG_ENABLE); + + for (i = 0; i < 8; i++) { + unsigned int size = fifo_config[i].size; + + /* If we are coming up on a system that already has the + * Rx data path enabled we don't need to reconfigure the + * FIFOs. Instead we can check to verify the values are + * large enough to meet our needs, and use the values to + * populate the flow control, ECN, and drop thresholds. + */ + if (rx_enable) { + size = FIELD_GET(FBNIC_RXB_PBUF_SIZE, + rd32(fbd, FBNIC_RXB_PBUF_CFG(i))); + if (size < fifo_config[i].size) + dev_warn(fbd->dev, + "fifo%d size of %d smaller than expected value of %d\n", + i, size << 11, + fifo_config[i].size << 11); + } else { + /* Program RXB Cuthrough */ + wr32(fbd, FBNIC_RXB_CT_SIZE(i), + FIELD_PREP(FBNIC_RXB_CT_SIZE_HEADER, 4) | + FIELD_PREP(FBNIC_RXB_CT_SIZE_PAYLOAD, 2)); + + /* The granularity for the packet buffer size is 2KB + * granularity while the packet buffer base address is + * only 64B granularity + */ + wr32(fbd, FBNIC_RXB_PBUF_CFG(i), + FIELD_PREP(FBNIC_RXB_PBUF_BASE_ADDR, + fifo_config[i].addr) | + FIELD_PREP(FBNIC_RXB_PBUF_SIZE, size)); + + /* The granularity for the credits is 64B. This is + * based on RXB_PBUF_SIZE * 32 + 4. + */ + wr32(fbd, FBNIC_RXB_PBUF_CREDIT(i), + FIELD_PREP(FBNIC_RXB_PBUF_CREDIT_MASK, + size ? size * 32 + 4 : 0)); + } + + if (!size) + continue; + + /* Pause is size of FIFO with 56KB skid to start/stop */ + wr32(fbd, FBNIC_RXB_PAUSE_THLD(i), + !(FBNIC_PAUSE_EN_MASK & (1u << i)) ? 0x1fff : + FIELD_PREP(FBNIC_RXB_PAUSE_THLD_ON, + size * 32 - 0x380) | + FIELD_PREP(FBNIC_RXB_PAUSE_THLD_OFF, 0x380)); + + /* Enable Drop when only one packet is left in the FIFO */ + wr32(fbd, FBNIC_RXB_DROP_THLD(i), + !(FBNIC_DROP_EN_MASK & (1u << i)) ? 0x1fff : + FIELD_PREP(FBNIC_RXB_DROP_THLD_ON, + size * 32 - + FBNIC_MAX_JUMBO_FRAME_SIZE / 64) | + FIELD_PREP(FBNIC_RXB_DROP_THLD_OFF, + size * 32 - + FBNIC_MAX_JUMBO_FRAME_SIZE / 64)); + + /* Enable ECN bit when 1/4 of RXB is filled with at least + * 1 room for one full jumbo frame before setting ECN + */ + wr32(fbd, FBNIC_RXB_ECN_THLD(i), + !(FBNIC_ECN_EN_MASK & (1u << i)) ? 0x1fff : + FIELD_PREP(FBNIC_RXB_ECN_THLD_ON, + max_t(unsigned int, + size * 32 / 4, + FBNIC_MAX_JUMBO_FRAME_SIZE / 64)) | + FIELD_PREP(FBNIC_RXB_ECN_THLD_OFF, + max_t(unsigned int, + size * 32 / 4, + FBNIC_MAX_JUMBO_FRAME_SIZE / 64))); + } + + /* For now only enable drop and ECN. We need to add driver/kernel + * interfaces for configuring pause. + */ + wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, + FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_DROP_ENABLE, + FBNIC_DROP_EN_MASK) | + FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_ECN_ENABLE, + FBNIC_ECN_EN_MASK)); + + /* Program INTF credits */ + wr32(fbd, FBNIC_RXB_INTF_CREDIT, + FBNIC_RXB_INTF_CREDIT_MASK0 | + FBNIC_RXB_INTF_CREDIT_MASK1 | + FBNIC_RXB_INTF_CREDIT_MASK2 | + FIELD_PREP(FBNIC_RXB_INTF_CREDIT_MASK3, 8)); + + /* Configure calendar slots. + * Rx: 0 - 62 RDE 1st, BMC 2nd + * 63 BMC 1st, RDE 2nd + */ + for (i = 0; i < 16; i++) { + u32 calendar_val = (i == 15) ? 0x1e1b1b1b : 0x1b1b1b1b; + + wr32(fbd, FBNIC_RXB_CLDR_PRIO_CFG(i), calendar_val); + } + + /* Split the credits for the DRR up as follows: + * Quantum0: 8000 Network to Host + * Quantum1: 0 Not used + * Quantum2: 80 BMC to Host + * Quantum3: 0 Not used + * Quantum4: 8000 Multicast to Host and BMC + */ + wr32(fbd, FBNIC_RXB_DWRR_RDE_WEIGHT0, + FIELD_PREP(FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM0, 0x40) | + FIELD_PREP(FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM2, 0x50)); + wr32(fbd, FBNIC_RXB_DWRR_RDE_WEIGHT0_EXT, + FIELD_PREP(FBNIC_RXB_DWRR_RDE_WEIGHT0_QUANTUM0, 0x1f)); + wr32(fbd, FBNIC_RXB_DWRR_RDE_WEIGHT1, + FIELD_PREP(FBNIC_RXB_DWRR_RDE_WEIGHT1_QUANTUM4, 0x40)); + wr32(fbd, FBNIC_RXB_DWRR_RDE_WEIGHT1_EXT, + FIELD_PREP(FBNIC_RXB_DWRR_RDE_WEIGHT1_QUANTUM4, 0x1f)); + + /* Program RXB FCS Endian register */ + wr32(fbd, FBNIC_RXB_ENDIAN_FCS, 0x0aaaaaa0); +} + +static void fbnic_mac_init_txb(struct fbnic_dev *fbd) +{ + int i; + + wr32(fbd, FBNIC_TCE_TXB_CTRL, 0); + + /* Configure Tx QM Credits */ + wr32(fbd, FBNIC_QM_TQS_CTL1, + FIELD_PREP(FBNIC_QM_TQS_CTL1_MC_MAX_CREDITS, 0x40) | + FIELD_PREP(FBNIC_QM_TQS_CTL1_BULK_MAX_CREDITS, 0x20)); + + /* Initialize internal Tx queues */ + wr32(fbd, FBNIC_TCE_TXB_TEI_Q0_CTRL, 0); + wr32(fbd, FBNIC_TCE_TXB_TEI_Q1_CTRL, 0); + wr32(fbd, FBNIC_TCE_TXB_MC_Q_CTRL, + FIELD_PREP(FBNIC_TCE_TXB_Q_CTRL_SIZE, 0x400) | + FIELD_PREP(FBNIC_TCE_TXB_Q_CTRL_START, 0x000)); + wr32(fbd, FBNIC_TCE_TXB_RX_TEI_Q_CTRL, 0); + wr32(fbd, FBNIC_TCE_TXB_TX_BMC_Q_CTRL, + FIELD_PREP(FBNIC_TCE_TXB_Q_CTRL_SIZE, 0x200) | + FIELD_PREP(FBNIC_TCE_TXB_Q_CTRL_START, 0x400)); + wr32(fbd, FBNIC_TCE_TXB_RX_BMC_Q_CTRL, + FIELD_PREP(FBNIC_TCE_TXB_Q_CTRL_SIZE, 0x200) | + FIELD_PREP(FBNIC_TCE_TXB_Q_CTRL_START, 0x600)); + + wr32(fbd, FBNIC_TCE_LSO_CTRL, + FBNIC_TCE_LSO_CTRL_IPID_MODE_INC | + FIELD_PREP(FBNIC_TCE_LSO_CTRL_TCPF_CLR_1ST, TCPHDR_PSH | + TCPHDR_FIN) | + FIELD_PREP(FBNIC_TCE_LSO_CTRL_TCPF_CLR_MID, TCPHDR_PSH | + TCPHDR_CWR | + TCPHDR_FIN) | + FIELD_PREP(FBNIC_TCE_LSO_CTRL_TCPF_CLR_END, TCPHDR_CWR)); + wr32(fbd, FBNIC_TCE_CSO_CTRL, 0); + + wr32(fbd, FBNIC_TCE_BMC_MAX_PKTSZ, + FIELD_PREP(FBNIC_TCE_BMC_MAX_PKTSZ_TX, + FBNIC_MAX_JUMBO_FRAME_SIZE) | + FIELD_PREP(FBNIC_TCE_BMC_MAX_PKTSZ_RX, + FBNIC_MAX_JUMBO_FRAME_SIZE)); + wr32(fbd, FBNIC_TCE_MC_MAX_PKTSZ, + FIELD_PREP(FBNIC_TCE_MC_MAX_PKTSZ_TMI, + FBNIC_MAX_JUMBO_FRAME_SIZE)); + + /* Configure calendar slots. + * Tx: 0 - 62 TMI 1st, BMC 2nd + * 63 BMC 1st, TMI 2nd + */ + for (i = 0; i < 16; i++) { + u32 calendar_val = (i == 15) ? 0x1e1b1b1b : 0x1b1b1b1b; + + wr32(fbd, FBNIC_TCE_TXB_CLDR_SLOT_CFG(i), calendar_val); + } + + /* Configure DWRR */ + wr32(fbd, FBNIC_TCE_TXB_ENQ_WRR_CTRL, + FIELD_PREP(FBNIC_TCE_TXB_ENQ_WRR_CTRL_WEIGHT0, 0x64) | + FIELD_PREP(FBNIC_TCE_TXB_ENQ_WRR_CTRL_WEIGHT2, 0x04)); + wr32(fbd, FBNIC_TCE_TXB_TEI_DWRR_CTRL, 0); + wr32(fbd, FBNIC_TCE_TXB_TEI_DWRR_CTRL_EXT, 0); + wr32(fbd, FBNIC_TCE_TXB_BMC_DWRR_CTRL, + FIELD_PREP(FBNIC_TCE_TXB_BMC_DWRR_CTRL_QUANTUM0, 0x50) | + FIELD_PREP(FBNIC_TCE_TXB_BMC_DWRR_CTRL_QUANTUM1, 0x82)); + wr32(fbd, FBNIC_TCE_TXB_BMC_DWRR_CTRL_EXT, 0); + wr32(fbd, FBNIC_TCE_TXB_NTWRK_DWRR_CTRL, + FIELD_PREP(FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_QUANTUM1, 0x50) | + FIELD_PREP(FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_QUANTUM2, 0x20)); + wr32(fbd, FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_EXT, + FIELD_PREP(FBNIC_TCE_TXB_NTWRK_DWRR_CTRL_QUANTUM2, 0x03)); + + /* Configure SOP protocol protection */ + wr32(fbd, FBNIC_TCE_SOP_PROT_CTRL, + FIELD_PREP(FBNIC_TCE_SOP_PROT_CTRL_TBI, 0x78) | + FIELD_PREP(FBNIC_TCE_SOP_PROT_CTRL_TTI_FRM, 0x40) | + FIELD_PREP(FBNIC_TCE_SOP_PROT_CTRL_TTI_CM, 0x0c)); + + /* Conservative configuration on MAC interface Start of Packet + * protection FIFO. This sets the minimum depth of the FIFO before + * we start sending packets to the MAC measured in 64B units and + * up to 160 entries deep. + * + * For the ASIC the clock is fast enough that we will likely fill + * the SOP FIFO before the MAC can drain it. So just use a minimum + * value of 8. + */ + wr32(fbd, FBNIC_TMI_SOP_PROT_CTRL, 8); + + wrfl(fbd); + wr32(fbd, FBNIC_TCE_TXB_CTRL, FBNIC_TCE_TXB_CTRL_TCAM_ENABLE | + FBNIC_TCE_TXB_CTRL_LOAD); +} + +static void fbnic_mac_init_regs(struct fbnic_dev *fbd) +{ + fbnic_mac_init_axi(fbd); + fbnic_mac_init_qm(fbd); + fbnic_mac_init_rxb(fbd); + fbnic_mac_init_txb(fbd); +} + +static void __fbnic_mac_stat_rd64(struct fbnic_dev *fbd, bool reset, u32 reg, + struct fbnic_stat_counter *stat) +{ + u64 new_reg_value; + + new_reg_value = fbnic_stat_rd64(fbd, reg, 1); + if (!reset) + stat->value += new_reg_value - stat->u.old_reg_value_64; + stat->u.old_reg_value_64 = new_reg_value; + stat->reported = true; +} + +static void fbnic_mac_stat_rd32(struct fbnic_dev *fbd, bool reset, u32 reg, + struct fbnic_stat_counter *stat) +{ + u32 new_reg_value; + + new_reg_value = rd32(fbd, reg); + if (!reset) + stat->value += new_reg_value - stat->u.old_reg_value_32; + stat->u.old_reg_value_32 = new_reg_value; + stat->reported = true; +} + +#define fbnic_mac_stat_rd64(fbd, reset, __stat, __CSR) \ + __fbnic_mac_stat_rd64(fbd, reset, FBNIC_##__CSR##_L, &(__stat)) + +bool fbnic_mac_check_tx_pause(struct fbnic_dev *fbd) +{ + u32 command_config; + + command_config = rd32(fbd, FBNIC_MAC_COMMAND_CONFIG); + return !(command_config & FBNIC_MAC_COMMAND_CONFIG_TX_PAUSE_DIS); +} + +static void fbnic_mac_tx_pause_config(struct fbnic_dev *fbd, bool tx_pause) +{ + u32 rxb_pause_ctrl; + + /* Enable generation of pause frames if enabled */ + rxb_pause_ctrl = rd32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL); + rxb_pause_ctrl &= ~FBNIC_RXB_PAUSE_DROP_CTRL_PAUSE_ENABLE; + if (tx_pause) + rxb_pause_ctrl |= + FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_PAUSE_ENABLE, + FBNIC_PAUSE_EN_MASK); + wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, rxb_pause_ctrl); +} + +static void +fbnic_mac_ps_protect_to_reset(struct fbnic_dev *fbd, u16 timeout_ms) +{ + wr32(fbd, FBNIC_RXB_PAUSE_STORM_UNIT_WR, FBNIC_RXB_PS_CLK_DIV); + + wr32(fbd, FBNIC_RXB_PAUSE_STORM(FBNIC_RXB_INTF_NET), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_RXB_PS_TO(timeout_ms)) | + FBNIC_RXB_PAUSE_STORM_FORCE_NORMAL); + wrfl(fbd); + wr32(fbd, FBNIC_RXB_PAUSE_STORM(FBNIC_RXB_INTF_NET), + FIELD_PREP(FBNIC_RXB_PAUSE_STORM_THLD_TIME, + FBNIC_MAC_RXB_PS_TO(timeout_ms))); +} + +static void +fbnic_mac_ps_protect_config(struct fbnic_dev *fbd, bool ps_protect) +{ + u16 timeout; + u32 reg; + + ps_protect = ps_protect && fbd->ps_timeout; + timeout = ps_protect ? fbd->ps_timeout : FBNIC_MAC_PS_TO_DEFAULT_MS; + + fbnic_mac_ps_protect_to_reset(fbd, timeout); + + reg = rd32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL); + reg &= ~FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE; + reg |= FIELD_PREP(FBNIC_RXB_PAUSE_DROP_CTRL_PS_ENABLE, ps_protect); + wr32(fbd, FBNIC_RXB_PAUSE_DROP_CTRL, reg); + + /* Clear any pending interrupt status first */ + wr32(fbd, FBNIC_RXB_ERR_INTR_STS, + FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK)); + + /* Unmask the Network to Host PS interrupt if tx_pause is on */ + reg = rd32(fbd, FBNIC_RXB_ERR_INTR_MASK); + reg |= FBNIC_RXB_ERR_INTR_STS_PS; + if (ps_protect) + reg &= ~FBNIC_RXB_ERR_INTR_STS_PS; + wr32(fbd, FBNIC_RXB_ERR_INTR_MASK, reg); +} + +static int fbnic_mac_get_link_event(struct fbnic_dev *fbd) +{ + u32 intr_mask = rd32(fbd, FBNIC_SIG_PCS_INTR_STS); + + if (intr_mask & FBNIC_SIG_PCS_INTR_LINK_DOWN) + return FBNIC_LINK_EVENT_DOWN; + + return (intr_mask & FBNIC_SIG_PCS_INTR_LINK_UP) ? + FBNIC_LINK_EVENT_UP : FBNIC_LINK_EVENT_NONE; +} + +static u32 __fbnic_mac_cmd_config_asic(struct fbnic_dev *fbd, + bool tx_pause, bool rx_pause) +{ + /* Enable MAC Promiscuous mode and Tx padding */ + u32 command_config = FBNIC_MAC_COMMAND_CONFIG_TX_PAD_EN | + FBNIC_MAC_COMMAND_CONFIG_PROMISC_EN; + struct fbnic_net *fbn = netdev_priv(fbd->netdev); + + /* Disable pause frames if not enabled */ + if (!tx_pause) + command_config |= FBNIC_MAC_COMMAND_CONFIG_TX_PAUSE_DIS; + if (!rx_pause) + command_config |= FBNIC_MAC_COMMAND_CONFIG_RX_PAUSE_DIS; + + /* Disable fault handling if no FEC is requested */ + if (fbn->fec == FBNIC_FEC_OFF) + command_config |= FBNIC_MAC_COMMAND_CONFIG_FLT_HDL_DIS; + + return command_config; +} + +static bool fbnic_mac_get_link_status(struct fbnic_dev *fbd, u8 aui, u8 fec) +{ + u32 pcs_status, lane_mask = ~0; + + pcs_status = rd32(fbd, FBNIC_SIG_PCS_OUT0); + if (!(pcs_status & FBNIC_SIG_PCS_OUT0_LINK)) + return false; + + /* Define the expected lane mask for the status bits we need to check */ + switch (aui) { + case FBNIC_AUI_100GAUI2: + lane_mask = 0xf; + break; + case FBNIC_AUI_50GAUI1: + lane_mask = 3; + break; + case FBNIC_AUI_LAUI2: + switch (fec) { + case FBNIC_FEC_OFF: + lane_mask = 0x63; + break; + case FBNIC_FEC_RS: + lane_mask = 5; + break; + case FBNIC_FEC_BASER: + lane_mask = 0xf; + break; + } + break; + case FBNIC_AUI_25GAUI: + lane_mask = 1; + break; + } + + /* Use an XOR to remove the bits we expect to see set */ + switch (fec) { + case FBNIC_FEC_OFF: + lane_mask ^= FIELD_GET(FBNIC_SIG_PCS_OUT0_BLOCK_LOCK, + pcs_status); + break; + case FBNIC_FEC_RS: + lane_mask ^= FIELD_GET(FBNIC_SIG_PCS_OUT0_AMPS_LOCK, + pcs_status); + break; + case FBNIC_FEC_BASER: + lane_mask ^= FIELD_GET(FBNIC_SIG_PCS_OUT1_FCFEC_LOCK, + rd32(fbd, FBNIC_SIG_PCS_OUT1)); + break; + } + + /* If all lanes cancelled then we have a lock on all lanes */ + return !lane_mask; +} + +static bool fbnic_pmd_update_state(struct fbnic_dev *fbd, bool signal_detect) +{ + /* Delay link up for 4 seconds to allow for link training. + * The state transitions for this are as follows: + * + * All states have the following two transitions in common: + * Loss of signal -> FBNIC_PMD_INITIALIZE + * The condition handled below (!signal) + * Reconfiguration -> FBNIC_PMD_INITIALIZE + * Occurs when mac_prepare starts a PHY reconfig + * FBNIC_PMD_TRAINING: + * signal still detected && 4s have passed -> Report link up + * When link is brought up in link_up -> FBNIC_PMD_SEND_DATA + * FBNIC_PMD_INITIALIZE: + * signal detected -> FBNIC_PMD_TRAINING + */ + if (!signal_detect) { + fbd->pmd_state = FBNIC_PMD_INITIALIZE; + return false; + } + + switch (fbd->pmd_state) { + case FBNIC_PMD_TRAINING: + return time_before(fbd->end_of_pmd_training, jiffies); + case FBNIC_PMD_LINK_READY: + case FBNIC_PMD_SEND_DATA: + return true; + } + + fbd->end_of_pmd_training = jiffies + 4 * HZ; + + /* Ensure end_of_training is visible before the state change */ + smp_wmb(); + + fbd->pmd_state = FBNIC_PMD_TRAINING; + + return false; +} + +static bool fbnic_mac_get_link(struct fbnic_dev *fbd, u8 aui, u8 fec) +{ + bool link; + + /* Flush status bits to clear possible stale data, + * bits should reset themselves back to 1 if link is truly up + */ + wr32(fbd, FBNIC_SIG_PCS_OUT0, FBNIC_SIG_PCS_OUT0_LINK | + FBNIC_SIG_PCS_OUT0_BLOCK_LOCK | + FBNIC_SIG_PCS_OUT0_AMPS_LOCK); + wr32(fbd, FBNIC_SIG_PCS_OUT1, FBNIC_SIG_PCS_OUT1_FCFEC_LOCK); + wrfl(fbd); + + /* Clear interrupt state due to recent changes. */ + wr32(fbd, FBNIC_SIG_PCS_INTR_STS, + FBNIC_SIG_PCS_INTR_LINK_DOWN | FBNIC_SIG_PCS_INTR_LINK_UP); + + link = fbnic_mac_get_link_status(fbd, aui, fec); + link = fbnic_pmd_update_state(fbd, link); + + /* Enable interrupt to only capture changes in link state */ + wr32(fbd, FBNIC_SIG_PCS_INTR_MASK, + ~FBNIC_SIG_PCS_INTR_LINK_DOWN & ~FBNIC_SIG_PCS_INTR_LINK_UP); + wr32(fbd, FBNIC_INTR_MASK_CLEAR(0), 1u << FBNIC_PCS_MSIX_ENTRY); + + return link; +} + +void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec) +{ + /* Retrieve default speed from FW */ + switch (fbd->fw_cap.link_speed) { + case FBNIC_FW_LINK_MODE_25CR: + *aui = FBNIC_AUI_25GAUI; + break; + case FBNIC_FW_LINK_MODE_50CR2: + *aui = FBNIC_AUI_LAUI2; + break; + case FBNIC_FW_LINK_MODE_50CR: + *aui = FBNIC_AUI_50GAUI1; + *fec = FBNIC_FEC_RS; + return; + case FBNIC_FW_LINK_MODE_100CR2: + *aui = FBNIC_AUI_100GAUI2; + *fec = FBNIC_FEC_RS; + return; + default: + *aui = FBNIC_AUI_UNKNOWN; + return; + } + + /* Update FEC first to reflect FW current mode */ + switch (fbd->fw_cap.link_fec) { + case FBNIC_FW_LINK_FEC_NONE: + *fec = FBNIC_FEC_OFF; + break; + case FBNIC_FW_LINK_FEC_RS: + default: + *fec = FBNIC_FEC_RS; + break; + case FBNIC_FW_LINK_FEC_BASER: + *fec = FBNIC_FEC_BASER; + break; + } +} + +static void fbnic_mac_prepare(struct fbnic_dev *fbd, u8 aui, u8 fec) +{ + /* Mask and clear the PCS interrupt, will be enabled by link handler */ + wr32(fbd, FBNIC_SIG_PCS_INTR_MASK, ~0); + wr32(fbd, FBNIC_SIG_PCS_INTR_STS, ~0); + + /* If we don't have link tear it all down and start over */ + if (!fbnic_mac_get_link_status(fbd, aui, fec)) + fbd->pmd_state = FBNIC_PMD_INITIALIZE; +} + +static void fbnic_mac_link_down_asic(struct fbnic_dev *fbd) +{ + u32 cmd_cfg, mac_ctrl; + + cmd_cfg = __fbnic_mac_cmd_config_asic(fbd, false, false); + mac_ctrl = rd32(fbd, FBNIC_SIG_MAC_IN0); + + mac_ctrl |= FBNIC_SIG_MAC_IN0_RESET_FF_TX_CLK | + FBNIC_SIG_MAC_IN0_RESET_TX_CLK | + FBNIC_SIG_MAC_IN0_RESET_FF_RX_CLK | + FBNIC_SIG_MAC_IN0_RESET_RX_CLK; + + wr32(fbd, FBNIC_SIG_MAC_IN0, mac_ctrl); + wr32(fbd, FBNIC_MAC_COMMAND_CONFIG, cmd_cfg); +} + +static void fbnic_mac_link_up_asic(struct fbnic_dev *fbd, + bool tx_pause, bool rx_pause) +{ + u32 cmd_cfg, mac_ctrl; + + fbnic_mac_tx_pause_config(fbd, tx_pause); + fbnic_mac_ps_protect_config(fbd, tx_pause); + + cmd_cfg = __fbnic_mac_cmd_config_asic(fbd, tx_pause, rx_pause); + mac_ctrl = rd32(fbd, FBNIC_SIG_MAC_IN0); + + mac_ctrl &= ~(FBNIC_SIG_MAC_IN0_RESET_FF_TX_CLK | + FBNIC_SIG_MAC_IN0_RESET_TX_CLK | + FBNIC_SIG_MAC_IN0_RESET_FF_RX_CLK | + FBNIC_SIG_MAC_IN0_RESET_RX_CLK); + cmd_cfg |= FBNIC_MAC_COMMAND_CONFIG_RX_ENA | + FBNIC_MAC_COMMAND_CONFIG_TX_ENA; + + wr32(fbd, FBNIC_SIG_MAC_IN0, mac_ctrl); + wr32(fbd, FBNIC_MAC_COMMAND_CONFIG, cmd_cfg); +} + +static void +fbnic_pcs_rsfec_stat_rd32(struct fbnic_dev *fbd, u32 reg, bool reset, + struct fbnic_stat_counter *stat) +{ + u32 pcs_rsfec_stat; + + /* The PCS/RFSEC registers are only 16b wide each. So what we will + * have after the 64b read is 0x0000xxxx0000xxxx. To make it usable + * as a full stat we will shift the upper bits into the lower set of + * 0s and then mask off the math at 32b. + * + * Read ordering must be lower reg followed by upper reg. + */ + pcs_rsfec_stat = rd32(fbd, reg) & 0xffff; + pcs_rsfec_stat |= rd32(fbd, reg + 1) << 16; + + /* RFSEC registers clear themselves upon being read so there is no + * need to store the old_reg_value. + */ + if (!reset) + stat->value += pcs_rsfec_stat; +} + +static void +fbnic_mac_get_fec_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_fec_stats *s) +{ + fbnic_pcs_rsfec_stat_rd32(fbd, FBNIC_RSFEC_CCW_LO(0), reset, + &s->corrected_blocks); + fbnic_pcs_rsfec_stat_rd32(fbd, FBNIC_RSFEC_NCCW_LO(0), reset, + &s->uncorrectable_blocks); +} + +static void +fbnic_mac_get_pcs_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_pcs_stats *s) +{ + int i; + + for (i = 0; i < FBNIC_PCS_MAX_LANES; i++) + fbnic_pcs_rsfec_stat_rd32(fbd, FBNIC_PCS_SYMBLERR_LO(i), reset, + &s->SymbolErrorDuringCarrier.lanes[i]); +} + +static void +fbnic_mac_get_eth_mac_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_eth_mac_stats *mac_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, mac_stats->OctetsReceivedOK, + MAC_STAT_RX_BYTE_COUNT); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->AlignmentErrors, + MAC_STAT_RX_ALIGN_ERROR); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->FrameTooLongErrors, + MAC_STAT_RX_TOOLONG); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->FramesReceivedOK, + MAC_STAT_RX_RECEIVED_OK); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->FrameCheckSequenceErrors, + MAC_STAT_RX_PACKET_BAD_FCS); + fbnic_mac_stat_rd64(fbd, reset, + mac_stats->FramesLostDueToIntMACRcvError, + MAC_STAT_RX_IFINERRORS); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->MulticastFramesReceivedOK, + MAC_STAT_RX_MULTICAST); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->BroadcastFramesReceivedOK, + MAC_STAT_RX_BROADCAST); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->OctetsTransmittedOK, + MAC_STAT_TX_BYTE_COUNT); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->FramesTransmittedOK, + MAC_STAT_TX_TRANSMITTED_OK); + fbnic_mac_stat_rd64(fbd, reset, + mac_stats->FramesLostDueToIntMACXmitError, + MAC_STAT_TX_IFOUTERRORS); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->MulticastFramesXmittedOK, + MAC_STAT_TX_MULTICAST); + fbnic_mac_stat_rd64(fbd, reset, mac_stats->BroadcastFramesXmittedOK, + MAC_STAT_TX_BROADCAST); +} + +static void +fbnic_mac_get_pause_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_pause_stats *pause_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, pause_stats->tx_pause_frames, + MAC_STAT_TX_XOFF_STB); + fbnic_mac_stat_rd64(fbd, reset, pause_stats->rx_pause_frames, + MAC_STAT_RX_XOFF_STB); + fbnic_mac_stat_rd32(fbd, reset, + FBNIC_RXB_INTR_PS_COUNT(FBNIC_RXB_INTF_NET), + &pause_stats->tx_pause_storm_events); +} + +static void +fbnic_mac_get_eth_ctrl_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_eth_ctrl_stats *ctrl_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, ctrl_stats->MACControlFramesReceived, + MAC_STAT_RX_CONTROL_FRAMES); + fbnic_mac_stat_rd64(fbd, reset, ctrl_stats->MACControlFramesTransmitted, + MAC_STAT_TX_CONTROL_FRAMES); +} + +static void +fbnic_mac_get_rmon_stats(struct fbnic_dev *fbd, bool reset, + struct fbnic_rmon_stats *rmon_stats) +{ + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->undersize_pkts, + MAC_STAT_RX_UNDERSIZE); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->oversize_pkts, + MAC_STAT_RX_OVERSIZE); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->fragments, + MAC_STAT_RX_FRAGMENT); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->jabbers, + MAC_STAT_RX_JABBER); + + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[0], + MAC_STAT_RX_PACKET_64_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[1], + MAC_STAT_RX_PACKET_65_127_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[2], + MAC_STAT_RX_PACKET_128_255_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[3], + MAC_STAT_RX_PACKET_256_511_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[4], + MAC_STAT_RX_PACKET_512_1023_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[5], + MAC_STAT_RX_PACKET_1024_1518_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[6], + RPC_STAT_RX_PACKET_1519_2047_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[7], + RPC_STAT_RX_PACKET_2048_4095_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[8], + RPC_STAT_RX_PACKET_4096_8191_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[9], + RPC_STAT_RX_PACKET_8192_9216_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist[10], + RPC_STAT_RX_PACKET_9217_MAX_BYTES); + + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[0], + MAC_STAT_TX_PACKET_64_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[1], + MAC_STAT_TX_PACKET_65_127_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[2], + MAC_STAT_TX_PACKET_128_255_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[3], + MAC_STAT_TX_PACKET_256_511_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[4], + MAC_STAT_TX_PACKET_512_1023_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[5], + MAC_STAT_TX_PACKET_1024_1518_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[6], + TMI_STAT_TX_PACKET_1519_2047_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[7], + TMI_STAT_TX_PACKET_2048_4095_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[8], + TMI_STAT_TX_PACKET_4096_8191_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[9], + TMI_STAT_TX_PACKET_8192_9216_BYTES); + fbnic_mac_stat_rd64(fbd, reset, rmon_stats->hist_tx[10], + TMI_STAT_TX_PACKET_9217_MAX_BYTES); +} + +static int fbnic_mac_get_sensor_asic(struct fbnic_dev *fbd, int id, + long *val) +{ + struct fbnic_fw_completion *fw_cmpl; + int err = 0; + s32 *sensor; + + fw_cmpl = fbnic_fw_alloc_cmpl(FBNIC_TLV_MSG_ID_TSENE_READ_RESP); + if (!fw_cmpl) + return -ENOMEM; + + switch (id) { + case FBNIC_SENSOR_TEMP: + sensor = &fw_cmpl->u.tsene.millidegrees; + break; + case FBNIC_SENSOR_VOLTAGE: + sensor = &fw_cmpl->u.tsene.millivolts; + break; + default: + err = -EINVAL; + goto exit_free; + } + + err = fbnic_fw_xmit_tsene_read_msg(fbd, fw_cmpl); + if (err) { + dev_err(fbd->dev, + "Failed to transmit TSENE read msg, err %d\n", + err); + goto exit_free; + } + + if (!wait_for_completion_timeout(&fw_cmpl->done, 10 * HZ)) { + dev_err(fbd->dev, "Timed out waiting for TSENE read\n"); + err = -ETIMEDOUT; + goto exit_cleanup; + } + + /* Handle error returned by firmware */ + if (fw_cmpl->result) { + err = fw_cmpl->result; + dev_err(fbd->dev, "%s: Firmware returned error %d\n", + __func__, err); + goto exit_cleanup; + } + + *val = *sensor; +exit_cleanup: + fbnic_mbx_clear_cmpl(fbd, fw_cmpl); +exit_free: + fbnic_fw_put_cmpl(fw_cmpl); + + return err; +} + +static const struct fbnic_mac fbnic_mac_asic = { + .init_regs = fbnic_mac_init_regs, + .get_link = fbnic_mac_get_link, + .get_link_event = fbnic_mac_get_link_event, + .prepare = fbnic_mac_prepare, + .get_fec_stats = fbnic_mac_get_fec_stats, + .get_pcs_stats = fbnic_mac_get_pcs_stats, + .get_eth_mac_stats = fbnic_mac_get_eth_mac_stats, + .get_pause_stats = fbnic_mac_get_pause_stats, + .get_eth_ctrl_stats = fbnic_mac_get_eth_ctrl_stats, + .get_rmon_stats = fbnic_mac_get_rmon_stats, + .link_down = fbnic_mac_link_down_asic, + .link_up = fbnic_mac_link_up_asic, + .get_sensor = fbnic_mac_get_sensor_asic, +}; + +/** + * fbnic_mac_init - Assign a MAC type and initialize the fbnic device + * @fbd: Device pointer to device to initialize + * + * Return: zero on success, negative on failure + * + * Initialize the MAC function pointers and initializes the MAC of + * the device. + **/ +int fbnic_mac_init(struct fbnic_dev *fbd) +{ + fbd->mac = &fbnic_mac_asic; + + fbd->mac->init_regs(fbd); + + return 0; +} + +int fbnic_mac_ps_protect_to_config(struct fbnic_dev *fbd, u16 timeout_ms) +{ + u16 old_timeout_ms = fbd->ps_timeout; + + if (timeout_ms == old_timeout_ms) + return 0; + + if (timeout_ms == PFC_STORM_PREVENTION_AUTO) + timeout_ms = FBNIC_MAC_PS_TO_DEFAULT_MS; + + if (timeout_ms > FBNIC_MAC_PS_TO_MAX_MS) + return -EINVAL; + + fbd->ps_timeout = timeout_ms; + + if (!fbnic_mac_check_tx_pause(fbd)) + return 0; + + if (timeout_ms == 0) + fbnic_mac_ps_protect_config(fbd, false); + else if (old_timeout_ms == 0) + fbnic_mac_ps_protect_config(fbd, true); + else + fbnic_mac_ps_protect_to_reset(fbd, fbd->ps_timeout); + + return 0; +} + +void fbnic_mac_ps_protect_handler(struct fbnic_dev *fbd) +{ + u32 rxb_err_sts = rd32(fbd, FBNIC_RXB_ERR_INTR_STS); + + /* Check if pause storm interrupt for network was triggered */ + if (rxb_err_sts & FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, + FBNIC_PS_EN_MASK)) { + /* Write 1 to clear the interrupt status first */ + wr32(fbd, FBNIC_RXB_ERR_INTR_STS, + FIELD_PREP(FBNIC_RXB_ERR_INTR_STS_PS, FBNIC_PS_EN_MASK)); + + fbnic_mac_ps_protect_to_reset(fbd, fbd->ps_timeout); + } +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mac.h b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h new file mode 100644 index 000000000..10f30e0e8 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mac.h @@ -0,0 +1,149 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_MAC_H_ +#define _FBNIC_MAC_H_ + +#include <linux/types.h> + +struct fbnic_dev; + +/* The RXB clock runs at 600 MHZ in the ASIC and the PAUSE_STORM_UNIT_WR + * is 10us granularity, so set the clock to 6000 (0x1770) + */ +#define FBNIC_RXB_PS_CLK_DIV 0x1770 + +/* Convert milliseconds to pause storm timeout units (10us granularity) */ +#define FBNIC_MAC_RXB_PS_TO(ms) ((ms) * 100) + +/* Convert pause storm timeout units (10us granularity) to milliseconds */ +#define FBNIC_MAC_RXB_PS_TO_MS(ps) ((ps) / 100) + +/* Set the default timer to 500ms, which should be longer than any + * reasonable period of continuous pausing. The service task, which runs + * once per second, periodically resets the pause storm trigger. + * + * As a result, on a functioning system, if pause continues, we enforce + * a duty cycle determined by the configured pause storm timeout (50% + * default). A crashed system will not have the service task and therefore + * pause will remain disabled until reboot recovery. + */ +#define FBNIC_MAC_PS_TO_DEFAULT_MS 500 +#define FBNIC_MAC_PS_TO_MAX_MS \ + FBNIC_MAC_RXB_PS_TO_MS(FIELD_MAX(FBNIC_RXB_PAUSE_STORM_THLD_TIME)) + +#define FBNIC_MAX_JUMBO_FRAME_SIZE 9742 + +/* States loosely based on section 136.8.11.7.5 of IEEE 802.3-2022 Ethernet + * Standard. These are needed to track the state of the PHY as it has a delay + * of several seconds from the time link comes up until it has completed + * training that we need to wait to report the link. + * + * Currently we treat training as a single block as this is managed by the + * firmware. + * + * We have FBNIC_PMD_SEND_DATA set to 0 as the expected default at driver load + * and we initialize the structure containing it to zero at allocation. + */ +enum { + FBNIC_PMD_SEND_DATA = 0x0, + FBNIC_PMD_INITIALIZE = 0x1, + FBNIC_PMD_TRAINING = 0x2, + FBNIC_PMD_LINK_READY = 0x3, +}; + +enum { + FBNIC_LINK_EVENT_NONE = 0, + FBNIC_LINK_EVENT_UP = 1, + FBNIC_LINK_EVENT_DOWN = 2, +}; + +/* Treat the FEC bits as a bitmask laid out as follows: + * Bit 0: RS Enabled + * Bit 1: BASER(Firecode) Enabled + * Bit 2: Retrieve FEC from FW + */ +enum { + FBNIC_FEC_OFF = 0, + FBNIC_FEC_RS = 1, + FBNIC_FEC_BASER = 2, +}; + +/* Treat the AUI modes as a modulation/lanes bitmask: + * Bit 0: Lane Count, 0 = R1, 1 = R2 + * Bit 1: Modulation, 0 = NRZ, 1 = PAM4 + * Bit 2: Unknown Modulation/Lane Configuration + */ +enum { + FBNIC_AUI_25GAUI = 0, /* 25.7812GBd 25.78125 * 1 */ + FBNIC_AUI_LAUI2 = 1, /* 51.5625GBd 25.78128 * 2 */ + FBNIC_AUI_50GAUI1 = 2, /* 53.125GBd 53.125 * 1 */ + FBNIC_AUI_100GAUI2 = 3, /* 106.25GBd 53.125 * 2 */ + FBNIC_AUI_UNKNOWN = 4, + __FBNIC_AUI_MAX__ +}; + +#define FBNIC_AUI_MODE_R2 (FBNIC_AUI_LAUI2) +#define FBNIC_AUI_MODE_PAM4 (FBNIC_AUI_50GAUI1) + +enum fbnic_sensor_id { + FBNIC_SENSOR_TEMP, /* Temp in millidegrees Centigrade */ + FBNIC_SENSOR_VOLTAGE, /* Voltage in millivolts */ +}; + +/* This structure defines the interface hooks for the MAC. The MAC hooks + * will be configured as a const struct provided with a set of function + * pointers. + * + * void (*init_regs)(struct fbnic_dev *fbd); + * Initialize MAC registers to enable Tx/Rx paths and FIFOs. + * + * int (*get_link_event)(struct fbnic_dev *fbd) + * Get the current link event status, reports true if link has + * changed to either FBNIC_LINK_EVENT_DOWN or FBNIC_LINK_EVENT_UP + * bool (*get_link)(struct fbnic_dev *fbd, u8 aui, u8 fec); + * Check link status + * + * void (*prepare)(struct fbnic_dev *fbd, u8 aui, u8 fec); + * Prepare PHY for init by fetching settings, disabling interrupts, + * and sending an updated PHY config to FW if needed. + * + * void (*link_down)(struct fbnic_dev *fbd); + * Configure MAC for link down event + * void (*link_up)(struct fbnic_dev *fbd, bool tx_pause, bool rx_pause); + * Configure MAC for link up event; + * + */ +struct fbnic_mac { + void (*init_regs)(struct fbnic_dev *fbd); + + int (*get_link_event)(struct fbnic_dev *fbd); + bool (*get_link)(struct fbnic_dev *fbd, u8 aui, u8 fec); + + void (*prepare)(struct fbnic_dev *fbd, u8 aui, u8 fec); + + void (*get_fec_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_fec_stats *fec_stats); + void (*get_pcs_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_pcs_stats *pcs_stats); + void (*get_eth_mac_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_eth_mac_stats *mac_stats); + void (*get_pause_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_pause_stats *pause_stats); + void (*get_eth_ctrl_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_eth_ctrl_stats *ctrl_stats); + void (*get_rmon_stats)(struct fbnic_dev *fbd, bool reset, + struct fbnic_rmon_stats *rmon_stats); + + void (*link_down)(struct fbnic_dev *fbd); + void (*link_up)(struct fbnic_dev *fbd, bool tx_pause, bool rx_pause); + + int (*get_sensor)(struct fbnic_dev *fbd, int id, long *val); +}; + +int fbnic_mac_init(struct fbnic_dev *fbd); +void fbnic_mac_get_fw_settings(struct fbnic_dev *fbd, u8 *aui, u8 *fec); +int fbnic_mac_ps_protect_to_config(struct fbnic_dev *fbd, u16 timeout); +void fbnic_mac_ps_protect_handler(struct fbnic_dev *fbd); +bool fbnic_mac_check_tx_pause(struct fbnic_dev *fbd); +#endif /* _FBNIC_MAC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c b/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c new file mode 100644 index 000000000..fe3a4ce88 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_mdio.c @@ -0,0 +1,282 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/mdio.h> +#include <linux/pcs/pcs-xpcs.h> + +#include "fbnic.h" +#include "fbnic_netdev.h" + +/* fbnic MDIO Interface Layout + * + * +-------------------+ + * | MAC | + * +-------------------+ + * | | | | <-- 25GMII, 50GMII, or CGMII + * +-------------------+ + * MMD 3 | PCS | + * +-------------------+ + * | FEC | + * +-------------------+ + * MMD 8 | Separated PMA | + * +-------------------+ + * | | <-- PMD Service Interface + * +-------------------+ + * MMD 1 | PMD | + * +-------------------+ + */ + +#define DW_VENDOR BIT(15) +#define FBNIC_PCS_VENDOR BIT(9) +#define FBNIC_PCS_ZERO_MASK (DW_VENDOR - FBNIC_PCS_VENDOR) + +static int +fbnic_mdio_ids(int id, int regnum) +{ + /* return correct IDs */ + switch (regnum) { + case MDIO_DEVID1: + return id >> 16; + case MDIO_DEVID2: + return id & 0xffff; + case MDIO_DEVS1: + return MDIO_DEVS_SEP_PMA1 | MDIO_DEVS_PMAPMD | MDIO_DEVS_PCS; + case MDIO_DEVS2: + return 0; + case MDIO_STAT2: + return MDIO_STAT2_DEVPRST_VAL; + } + + return 0; +} + +static int +fbnic_mdio_read_pmd(struct fbnic_dev *fbd, int addr, int regnum) +{ + u8 aui = FBNIC_AUI_UNKNOWN; + struct fbnic_net *fbn; + int ret = 0; + + /* We don't need a second PMD, just one can handle both lanes */ + if (addr) + return 0; + + if (fbd->netdev) { + fbn = netdev_priv(fbd->netdev); + if (fbn->aui < FBNIC_AUI_UNKNOWN) + aui = fbn->aui; + } + + switch (regnum) { + case MDIO_PMA_RXDET: + /* If training isn't complete default to 0 */ + if (fbd->pmd_state != FBNIC_PMD_SEND_DATA) + break; + /* Report either 1 or 2 lanes detected depending on config */ + ret = (MDIO_PMD_RXDET_GLOBAL | MDIO_PMD_RXDET_0) | + ((aui & FBNIC_AUI_MODE_R2) * + (MDIO_PMD_RXDET_1 / FBNIC_AUI_MODE_R2)); + break; + default: + ret = fbnic_mdio_ids(MP_FBNIC_XPCS_PMA_100G_ID, regnum); + break; + } + + dev_dbg(fbd->dev, + "SWMII PMD Rd: Addr: %d RegNum: %d Value: 0x%04x\n", + addr, regnum, ret); + + return ret; +} + +static int +fbnic_mdio_read_pcs(struct fbnic_dev *fbd, int addr, int regnum) +{ + int ret, offset = 0, overrides = 0; + + /* We will need access to both PCS instances to get config info */ + if (addr >= 2) + return 0; + + /* Report 0 for reserved registers */ + if (regnum & FBNIC_PCS_ZERO_MASK) + return 0; + + /* Intercept and return correct ID for PCS */ + switch (regnum) { + case MDIO_DEVID1 ... MDIO_DEVID2: + ret = fbnic_mdio_ids(DW_XPCS_ID, regnum); + break; + case MDIO_DEVS1: + /* DW IP returns MDIO_DEVS_SEP_PMA1, MDIO_DEVS_PMAPMD, + * and MDIO_DEVS_PCS as 0 + */ + overrides = fbnic_mdio_ids(DW_XPCS_ID, regnum); + fallthrough; + default: + /* Swap vendor page bit for FBNIC PCS vendor page bit */ + if (regnum & DW_VENDOR) + offset ^= DW_VENDOR | FBNIC_PCS_VENDOR; + + ret = fbnic_rd32(fbd, FBNIC_PCS_PAGE(addr) + (regnum ^ offset)); + ret |= overrides; + break; + } + + dev_dbg(fbd->dev, + "SWMII PCS Rd: Addr: %d RegNum: %d Value: 0x%04x\n", + addr, regnum, ret); + + return ret; +} + +static int +fbnic_mdio_read_pma(struct fbnic_dev *fbd, int addr, int regnum) +{ + int ret = 0; + + /* We will need access to both PMA instances to get config info */ + if (addr >= 2) + return 0; + + switch (regnum) { + case MDIO_PMA_RSFEC_CTRL ... MDIO_PMA_RSFEC_LANE_MAP: + ret = fbnic_rd32(fbd, FBNIC_RSFEC_CONTROL(addr) + + regnum - MDIO_PMA_RSFEC_CTRL); + break; + default: + ret = fbnic_mdio_ids(MP_FBNIC_XPCS_PMA_100G_ID, regnum); + break; + } + + dev_dbg(fbd->dev, + "SWMII PMA Rd: Addr: %d RegNum: %d Value: 0x%04x\n", + addr, regnum, ret); + + return ret; +} + +static int +fbnic_mdio_read_c45(struct mii_bus *bus, int addr, int devnum, int regnum) +{ + struct fbnic_dev *fbd = bus->priv; + + if (devnum == MDIO_MMD_PMAPMD) + return fbnic_mdio_read_pmd(fbd, addr, regnum); + + if (devnum == MDIO_MMD_PCS) + return fbnic_mdio_read_pcs(fbd, addr, regnum); + + if (devnum == MDIO_MMD_SEP_PMA1) + return fbnic_mdio_read_pma(fbd, addr, regnum); + + return 0; +} + +static void +fbnic_mdio_write_pmd(struct fbnic_dev *fbd, int addr, int regnum, u16 val) +{ + dev_dbg(fbd->dev, + "SWMII PMD Wr: Addr: %d RegNum: %d Value: 0x%04x\n", + addr, regnum, val); +} + +static void +fbnic_mdio_write_pcs(struct fbnic_dev *fbd, int addr, int regnum, u16 val) +{ + dev_dbg(fbd->dev, + "SWMII PCS Wr: Addr: %d RegNum: %d Value: 0x%04x\n", + addr, regnum, val); + + /* Allow access to both halves of PCS for 50R2 config */ + if (addr >= 2) + return; + + /* Skip write for reserved registers */ + if (regnum & FBNIC_PCS_ZERO_MASK) + return; + + /* Swap vendor page bit for FBNIC PCS vendor page bit */ + if (regnum & DW_VENDOR) + regnum ^= DW_VENDOR | FBNIC_PCS_VENDOR; + + fbnic_wr32(fbd, FBNIC_PCS_PAGE(addr) + regnum, val); +} + +static void +fbnic_mdio_write_pma(struct fbnic_dev *fbd, int addr, int regnum, u16 val) +{ + dev_dbg(fbd->dev, + "SWMII PMA Wr: Addr: %d RegNum: %d Value: 0x%04x\n", + addr, regnum, val); + + if (addr >= 2) + return; + + switch (regnum) { + case MDIO_PMA_RSFEC_CTRL ... MDIO_PMA_RSFEC_LANE_MAP: + fbnic_wr32(fbd, FBNIC_RSFEC_CONTROL(addr) + + regnum - MDIO_PMA_RSFEC_CTRL, val); + break; + default: + break; + } +} + +static int +fbnic_mdio_write_c45(struct mii_bus *bus, int addr, int devnum, + int regnum, u16 val) +{ + struct fbnic_dev *fbd = bus->priv; + + if (devnum == MDIO_MMD_PMAPMD) + fbnic_mdio_write_pmd(fbd, addr, regnum, val); + + if (devnum == MDIO_MMD_PCS) + fbnic_mdio_write_pcs(fbd, addr, regnum, val); + + if (devnum == MDIO_MMD_SEP_PMA1) + fbnic_mdio_write_pma(fbd, addr, regnum, val); + + return 0; +} + +/** + * fbnic_mdiobus_create - Create an MDIO bus to allow interfacing w/ PHYs + * @fbd: Pointer to FBNIC device structure to populate bus on + * + * Initialize an MDIO bus and place a pointer to it on the fbd struct. This bus + * will be used to interface with the PMA/PMD and PCS. + * + * Return: 0 on success, negative on failure + **/ +int fbnic_mdiobus_create(struct fbnic_dev *fbd) +{ + struct mii_bus *bus; + int err; + + bus = devm_mdiobus_alloc(fbd->dev); + if (!bus) + return -ENOMEM; + + bus->name = "fbnic_mii_bus"; + bus->read_c45 = &fbnic_mdio_read_c45; + bus->write_c45 = &fbnic_mdio_write_c45; + + /* Disable PHY auto probing. We will add PCS manually */ + bus->phy_mask = ~0; + + bus->parent = fbd->dev; + bus->priv = fbd; + snprintf(bus->id, MII_BUS_ID_SIZE, "%s-mii", dev_name(fbd->dev)); + + err = devm_mdiobus_register(fbd->dev, bus); + if (err) { + dev_err(fbd->dev, "Failed to create MDIO bus: %d\n", err); + return err; + } + + fbd->mdio_bus = bus; + + return 0; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c new file mode 100644 index 000000000..10bf99be3 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.c @@ -0,0 +1,891 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/etherdevice.h> +#include <linux/ipv6.h> +#include <linux/types.h> +#include <net/netdev_queues.h> + +#include "fbnic.h" +#include "fbnic_netdev.h" +#include "fbnic_txrx.h" + +int __fbnic_open(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + int err; + + err = fbnic_alloc_napi_vectors(fbn); + if (err) + return err; + + err = fbnic_alloc_resources(fbn); + if (err) + goto free_napi_vectors; + + err = fbnic_set_netif_queues(fbn); + if (err) + goto free_resources; + + /* Send ownership message and flush to verify FW has seen it */ + err = fbnic_fw_xmit_ownership_msg(fbd, true); + if (err) { + dev_warn(fbd->dev, + "Error %d sending host ownership message to the firmware\n", + err); + goto err_reset_queues; + } + + err = fbnic_time_start(fbn); + if (err) + goto release_ownership; + + err = fbnic_fw_init_heartbeat(fbd, false); + if (err) + goto time_stop; + + err = fbnic_mac_request_irq(fbd); + if (err) + goto time_stop; + + /* Pull the BMC config and initialize the RPC */ + fbnic_bmc_rpc_init(fbd); + fbnic_rss_reinit(fbd, fbn); + + phylink_resume(fbn->phylink); + + return 0; +time_stop: + fbnic_time_stop(fbn); +release_ownership: + fbnic_fw_xmit_ownership_msg(fbn->fbd, false); +err_reset_queues: + fbnic_reset_netif_queues(fbn); +free_resources: + fbnic_free_resources(fbn); +free_napi_vectors: + fbnic_free_napi_vectors(fbn); + return err; +} + +static int fbnic_open(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err; + + fbnic_napi_name_irqs(fbn->fbd); + + err = __fbnic_open(fbn); + if (!err) + fbnic_up(fbn); + + return err; +} + +static int fbnic_stop(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + fbnic_mac_free_irq(fbn->fbd); + phylink_suspend(fbn->phylink, fbnic_bmc_present(fbn->fbd)); + + fbnic_down(fbn); + + fbnic_time_stop(fbn); + fbnic_fw_xmit_ownership_msg(fbn->fbd, false); + + fbnic_reset_netif_queues(fbn); + fbnic_free_resources(fbn); + fbnic_free_napi_vectors(fbn); + + return 0; +} + +static int fbnic_uc_sync(struct net_device *netdev, const unsigned char *addr) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_addr *avail_addr; + + if (WARN_ON(!is_valid_ether_addr(addr))) + return -EADDRNOTAVAIL; + + avail_addr = __fbnic_uc_sync(fbn->fbd, addr); + if (!avail_addr) + return -ENOSPC; + + /* Add type flag indicating this address is in use by the host */ + set_bit(FBNIC_MAC_ADDR_T_UNICAST, avail_addr->act_tcam); + + return 0; +} + +static int fbnic_uc_unsync(struct net_device *netdev, const unsigned char *addr) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + int i, ret; + + /* Scan from middle of list to bottom, filling bottom up. + * Skip the first entry which is reserved for dev_addr and + * leave the last entry to use for promiscuous filtering. + */ + for (i = fbd->mac_addr_boundary, ret = -ENOENT; + i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX && ret; i++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (!ether_addr_equal(mac_addr->value.addr8, addr)) + continue; + + ret = __fbnic_uc_unsync(mac_addr); + } + + return ret; +} + +static int fbnic_mc_sync(struct net_device *netdev, const unsigned char *addr) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_mac_addr *avail_addr; + + if (WARN_ON(!is_multicast_ether_addr(addr))) + return -EADDRNOTAVAIL; + + avail_addr = __fbnic_mc_sync(fbn->fbd, addr); + if (!avail_addr) + return -ENOSPC; + + /* Add type flag indicating this address is in use by the host */ + set_bit(FBNIC_MAC_ADDR_T_MULTICAST, avail_addr->act_tcam); + + return 0; +} + +static int fbnic_mc_unsync(struct net_device *netdev, const unsigned char *addr) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + int i, ret; + + /* Scan from middle of list to top, filling top down. + * Skip over the address reserved for the BMC MAC and + * exclude index 0 as that belongs to the broadcast address + */ + for (i = fbd->mac_addr_boundary, ret = -ENOENT; + --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && ret;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (!ether_addr_equal(mac_addr->value.addr8, addr)) + continue; + + ret = __fbnic_mc_unsync(mac_addr); + } + + return ret; +} + +void __fbnic_set_rx_mode(struct fbnic_dev *fbd, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) +{ + bool uc_promisc = false, mc_promisc = false; + struct net_device *netdev = fbd->netdev; + struct fbnic_mac_addr *mac_addr; + int err; + + /* Populate host address from dev_addr */ + mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX]; + if (!ether_addr_equal(mac_addr->value.addr8, netdev->dev_addr) || + mac_addr->state != FBNIC_TCAM_S_VALID) { + ether_addr_copy(mac_addr->value.addr8, netdev->dev_addr); + mac_addr->state = FBNIC_TCAM_S_UPDATE; + set_bit(FBNIC_MAC_ADDR_T_UNICAST, mac_addr->act_tcam); + } + + /* Populate broadcast address if broadcast is enabled */ + mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX]; + if (netdev->flags & IFF_BROADCAST) { + if (!is_broadcast_ether_addr(mac_addr->value.addr8) || + mac_addr->state != FBNIC_TCAM_S_VALID) { + eth_broadcast_addr(mac_addr->value.addr8); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + set_bit(FBNIC_MAC_ADDR_T_BROADCAST, mac_addr->act_tcam); + } else if (mac_addr->state == FBNIC_TCAM_S_VALID) { + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BROADCAST); + } + + /* Synchronize unicast and multicast address lists */ + err = __hw_addr_sync_dev(uc, netdev, fbnic_uc_sync, fbnic_uc_unsync); + if (err == -ENOSPC) + uc_promisc = true; + err = __hw_addr_sync_dev(mc, netdev, fbnic_mc_sync, fbnic_mc_unsync); + if (err == -ENOSPC) + mc_promisc = true; + + uc_promisc |= !!(netdev->flags & IFF_PROMISC); + mc_promisc |= !!(netdev->flags & IFF_ALLMULTI) || uc_promisc; + + /* Update the promiscuous rules */ + fbnic_promisc_sync(fbd, uc_promisc, mc_promisc); + + /* Add rules for BMC all multicast if it is enabled */ + fbnic_bmc_rpc_all_multi_config(fbd, mc_promisc); + + /* Sift out any unshared BMC rules and place them in BMC only section */ + fbnic_sift_macda(fbd); + + /* Write updates to hardware */ + fbnic_write_rules(fbd); + fbnic_write_macda(fbd); + fbnic_write_tce_tcam(fbd); +} + +static int fbnic_set_rx_mode(struct net_device *netdev, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + /* No need to update the hardware if we are not running */ + if (netif_running(netdev)) + __fbnic_set_rx_mode(fbd, uc, mc); + + return 0; +} + +static int fbnic_set_mac(struct net_device *netdev, void *p) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct sockaddr *addr = p; + + if (!is_valid_ether_addr(addr->sa_data)) + return -EADDRNOTAVAIL; + + eth_hw_addr_set(netdev, addr->sa_data); + + if (netif_running(netdev)) { + netif_addr_lock_bh(netdev); + __fbnic_set_rx_mode(fbn->fbd, &netdev->uc, &netdev->mc); + netif_addr_unlock_bh(netdev); + } + + return 0; +} + +static int fbnic_change_mtu(struct net_device *dev, int new_mtu) +{ + struct fbnic_net *fbn = netdev_priv(dev); + + if (fbnic_check_split_frames(fbn->xdp_prog, new_mtu, fbn->hds_thresh)) { + dev_err(&dev->dev, + "MTU %d is larger than HDS threshold %d in XDP mode\n", + new_mtu, fbn->hds_thresh); + + return -EINVAL; + } + + WRITE_ONCE(dev->mtu, new_mtu); + + return 0; +} + +void fbnic_clear_rx_mode(struct fbnic_dev *fbd) +{ + struct net_device *netdev = fbd->netdev; + int idx; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + if (mac_addr->state != FBNIC_TCAM_S_VALID) + continue; + + bitmap_clear(mac_addr->act_tcam, + FBNIC_MAC_ADDR_T_HOST_START, + FBNIC_MAC_ADDR_T_HOST_LEN); + + if (bitmap_empty(mac_addr->act_tcam, + FBNIC_RPC_TCAM_ACT_NUM_ENTRIES)) + mac_addr->state = FBNIC_TCAM_S_DELETE; + } + + /* Write updates to hardware */ + fbnic_write_macda(fbd); + + netif_addr_lock_bh(netdev); + __dev_uc_unsync(netdev, NULL); + __dev_mc_unsync(netdev, NULL); + netif_addr_unlock_bh(netdev); +} + +static int fbnic_hwtstamp_get(struct net_device *netdev, + struct kernel_hwtstamp_config *config) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + *config = fbn->hwtstamp_config; + + return 0; +} + +static int fbnic_hwtstamp_set(struct net_device *netdev, + struct kernel_hwtstamp_config *config, + struct netlink_ext_ack *extack) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int old_rx_filter; + + if (config->source != HWTSTAMP_SOURCE_NETDEV) + return -EOPNOTSUPP; + + if (!kernel_hwtstamp_config_changed(config, &fbn->hwtstamp_config)) + return 0; + + /* Upscale the filters */ + switch (config->rx_filter) { + case HWTSTAMP_FILTER_NONE: + case HWTSTAMP_FILTER_ALL: + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_EVENT: + break; + case HWTSTAMP_FILTER_NTP_ALL: + config->rx_filter = HWTSTAMP_FILTER_ALL; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_L2_EVENT; + break; + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + break; + default: + return -ERANGE; + } + + /* Configure */ + old_rx_filter = fbn->hwtstamp_config.rx_filter; + memcpy(&fbn->hwtstamp_config, config, sizeof(*config)); + + if (old_rx_filter != config->rx_filter && netif_running(fbn->netdev)) { + fbnic_rss_reinit(fbn->fbd, fbn); + fbnic_write_rules(fbn->fbd); + } + + /* Save / report back filter configuration + * Note that our filter configuration is inexact. Instead of + * filtering for a specific UDP port or L2 Ethertype we are + * filtering in all UDP or all non-IP packets for timestamping. So + * if anything other than FILTER_ALL is requested we report + * FILTER_SOME indicating that we will be timestamping a few + * additional packets. + */ + if (config->rx_filter > HWTSTAMP_FILTER_ALL) + config->rx_filter = HWTSTAMP_FILTER_SOME; + + return 0; +} + +static void fbnic_get_stats64(struct net_device *dev, + struct rtnl_link_stats64 *stats64) +{ + u64 rx_bytes, rx_packets, rx_dropped = 0, rx_errors = 0; + u64 rx_over = 0, rx_missed = 0, rx_length = 0; + u64 tx_bytes, tx_packets, tx_dropped = 0; + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_queue_stats *stats; + + unsigned int start, i; + + fbnic_get_hw_stats(fbd); + + stats = &fbn->tx_stats; + + tx_bytes = stats->bytes; + tx_packets = stats->packets; + tx_dropped = stats->dropped; + + /* Record drops from Tx HW Datapath */ + spin_lock(&fbd->hw_stats.lock); + tx_dropped += fbd->hw_stats.tmi.drop.frames.value + + fbd->hw_stats.tti.cm_drop.frames.value + + fbd->hw_stats.tti.frame_drop.frames.value + + fbd->hw_stats.tti.tbi_drop.frames.value; + spin_unlock(&fbd->hw_stats.lock); + + stats64->tx_bytes = tx_bytes; + stats64->tx_packets = tx_packets; + stats64->tx_dropped = tx_dropped; + + for (i = 0; i < fbn->num_tx_queues; i++) { + struct fbnic_ring *txr = fbn->tx[i]; + + if (!txr) + continue; + + stats = &txr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + tx_bytes = stats->bytes; + tx_packets = stats->packets; + tx_dropped = stats->dropped; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats64->tx_bytes += tx_bytes; + stats64->tx_packets += tx_packets; + stats64->tx_dropped += tx_dropped; + } + + stats = &fbn->rx_stats; + + rx_bytes = stats->bytes; + rx_packets = stats->packets; + rx_dropped = stats->dropped; + + spin_lock(&fbd->hw_stats.lock); + /* Record drops for the host FIFOs. + * 4: network to Host, 6: BMC to Host + * Exclude the BMC and MC FIFOs as those stats may contain drops + * due to unrelated items such as TCAM misses. They are still + * accessible through the ethtool stats. + */ + i = FBNIC_RXB_FIFO_HOST; + rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value; + i = FBNIC_RXB_FIFO_BMC_TO_HOST; + rx_missed += fbd->hw_stats.rxb.fifo[i].drop.frames.value; + + for (i = 0; i < fbd->max_num_queues; i++) { + /* Report packets dropped due to CQ/BDQ being full/empty */ + rx_over += fbd->hw_stats.hw_q[i].rde_pkt_cq_drop.value; + rx_over += fbd->hw_stats.hw_q[i].rde_pkt_bdq_drop.value; + + /* Report packets with errors */ + rx_errors += fbd->hw_stats.hw_q[i].rde_pkt_err.value; + } + spin_unlock(&fbd->hw_stats.lock); + + stats64->rx_bytes = rx_bytes; + stats64->rx_packets = rx_packets; + stats64->rx_dropped = rx_dropped; + stats64->rx_over_errors = rx_over; + stats64->rx_errors = rx_errors; + stats64->rx_missed_errors = rx_missed; + + for (i = 0; i < fbn->num_rx_queues; i++) { + struct fbnic_ring *xdpr = fbn->tx[FBNIC_MAX_TXQS + i]; + struct fbnic_ring *rxr = fbn->rx[i]; + + if (!rxr) + continue; + + stats = &rxr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + rx_bytes = stats->bytes; + rx_packets = stats->packets; + rx_dropped = stats->dropped; + rx_length = stats->rx.length_errors; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats64->rx_bytes += rx_bytes; + stats64->rx_packets += rx_packets; + stats64->rx_dropped += rx_dropped; + stats64->rx_errors += rx_length; + stats64->rx_length_errors += rx_length; + + if (!xdpr) + continue; + + stats = &xdpr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + tx_bytes = stats->bytes; + tx_packets = stats->packets; + tx_dropped = stats->dropped; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats64->tx_bytes += tx_bytes; + stats64->tx_packets += tx_packets; + stats64->tx_dropped += tx_dropped; + } +} + +bool fbnic_check_split_frames(struct bpf_prog *prog, unsigned int mtu, + u32 hds_thresh) +{ + if (!prog) + return false; + + if (prog->aux->xdp_has_frags) + return false; + + return mtu + ETH_HLEN > hds_thresh; +} + +static int fbnic_bpf(struct net_device *netdev, struct netdev_bpf *bpf) +{ + struct bpf_prog *prog = bpf->prog, *prev_prog; + struct fbnic_net *fbn = netdev_priv(netdev); + + if (bpf->command != XDP_SETUP_PROG) + return -EINVAL; + + if (fbnic_check_split_frames(prog, netdev->mtu, + fbn->hds_thresh)) { + NL_SET_ERR_MSG_MOD(bpf->extack, + "MTU too high, or HDS threshold is too low for single buffer XDP"); + return -EOPNOTSUPP; + } + + prev_prog = xchg(&fbn->xdp_prog, prog); + if (prev_prog) + bpf_prog_put(prev_prog); + + return 0; +} + +static const struct net_device_ops fbnic_netdev_ops = { + .ndo_open = fbnic_open, + .ndo_stop = fbnic_stop, + .ndo_validate_addr = eth_validate_addr, + .ndo_start_xmit = fbnic_xmit_frame, + .ndo_features_check = fbnic_features_check, + .ndo_set_mac_address = fbnic_set_mac, + .ndo_change_mtu = fbnic_change_mtu, + .ndo_set_rx_mode_async = fbnic_set_rx_mode, + .ndo_get_stats64 = fbnic_get_stats64, + .ndo_bpf = fbnic_bpf, + .ndo_hwtstamp_get = fbnic_hwtstamp_get, + .ndo_hwtstamp_set = fbnic_hwtstamp_set, +}; + +static void fbnic_get_queue_stats_rx(struct net_device *dev, int idx, + struct netdev_queue_stats_rx *rx) +{ + u64 bytes, packets, alloc_fail, alloc_fail_bdq; + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_ring *rxr = fbn->rx[idx]; + struct fbnic_dev *fbd = fbn->fbd; + struct fbnic_queue_stats *stats; + u64 csum_complete, csum_none; + struct fbnic_q_triad *qt; + unsigned int start; + + if (!rxr) + return; + + /* fbn->rx points to completion queues */ + qt = container_of(rxr, struct fbnic_q_triad, cmpl); + + stats = &rxr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = stats->bytes; + packets = stats->packets; + alloc_fail = stats->rx.alloc_failed; + csum_complete = stats->rx.csum_complete; + csum_none = stats->rx.csum_none; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + stats = &qt->sub0.stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + alloc_fail_bdq = stats->bdq.alloc_failed; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + alloc_fail += alloc_fail_bdq; + + stats = &qt->sub1.stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + alloc_fail_bdq = stats->bdq.alloc_failed; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + alloc_fail += alloc_fail_bdq; + + rx->bytes = bytes; + rx->packets = packets; + rx->alloc_fail = alloc_fail; + rx->csum_complete = csum_complete; + rx->csum_none = csum_none; + + fbnic_get_hw_q_stats(fbd, fbd->hw_stats.hw_q); + + spin_lock(&fbd->hw_stats.lock); + rx->hw_drop_overruns = fbd->hw_stats.hw_q[idx].rde_pkt_cq_drop.value + + fbd->hw_stats.hw_q[idx].rde_pkt_bdq_drop.value; + rx->hw_drops = fbd->hw_stats.hw_q[idx].rde_pkt_err.value + + rx->hw_drop_overruns; + spin_unlock(&fbd->hw_stats.lock); +} + +static void fbnic_get_queue_stats_tx(struct net_device *dev, int idx, + struct netdev_queue_stats_tx *tx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_ring *txr = fbn->tx[idx]; + struct fbnic_queue_stats *stats; + u64 stop, wake, csum, lso; + struct fbnic_ring *xdpr; + unsigned int start; + u64 bytes, packets; + + if (!txr) + return; + + stats = &txr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = stats->bytes; + packets = stats->packets; + csum = stats->twq.csum_partial; + lso = stats->twq.lso; + stop = stats->twq.stop; + wake = stats->twq.wake; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + tx->bytes = bytes; + tx->packets = packets; + tx->needs_csum = csum + lso; + tx->hw_gso_wire_packets = lso; + tx->stop = stop; + tx->wake = wake; + + xdpr = fbn->tx[FBNIC_MAX_TXQS + idx]; + if (xdpr) { + stats = &xdpr->stats; + do { + start = u64_stats_fetch_begin(&stats->syncp); + bytes = stats->bytes; + packets = stats->packets; + } while (u64_stats_fetch_retry(&stats->syncp, start)); + + tx->bytes += bytes; + tx->packets += packets; + } +} + +static void fbnic_get_base_stats(struct net_device *dev, + struct netdev_queue_stats_rx *rx, + struct netdev_queue_stats_tx *tx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + + tx->bytes = fbn->tx_stats.bytes; + tx->packets = fbn->tx_stats.packets; + tx->needs_csum = fbn->tx_stats.twq.csum_partial + fbn->tx_stats.twq.lso; + tx->hw_gso_wire_packets = fbn->tx_stats.twq.lso; + tx->stop = fbn->tx_stats.twq.stop; + tx->wake = fbn->tx_stats.twq.wake; + + rx->bytes = fbn->rx_stats.bytes; + rx->packets = fbn->rx_stats.packets; + rx->alloc_fail = fbn->rx_stats.rx.alloc_failed + + fbn->bdq_stats.bdq.alloc_failed; + rx->csum_complete = fbn->rx_stats.rx.csum_complete; + rx->csum_none = fbn->rx_stats.rx.csum_none; +} + +static const struct netdev_stat_ops fbnic_stat_ops = { + .get_queue_stats_rx = fbnic_get_queue_stats_rx, + .get_queue_stats_tx = fbnic_get_queue_stats_tx, + .get_base_stats = fbnic_get_base_stats, +}; + +void fbnic_reset_queues(struct fbnic_net *fbn, + unsigned int tx, unsigned int rx) +{ + struct fbnic_dev *fbd = fbn->fbd; + unsigned int max_napis; + + max_napis = fbd->num_irqs - FBNIC_NON_NAPI_VECTORS; + + tx = min(tx, max_napis); + fbn->num_tx_queues = tx; + + rx = min(rx, max_napis); + fbn->num_rx_queues = rx; + + fbn->num_napi = max(tx, rx); +} + +/** + * fbnic_netdev_free - Free the netdev associate with fbnic + * @fbd: Driver specific structure to free netdev from + * + * Allocate and initialize the netdev and netdev private structure. Bind + * together the hardware, netdev, and pci data structures. + **/ +void fbnic_netdev_free(struct fbnic_dev *fbd) +{ + fbnic_phylink_destroy(fbd->netdev); + + free_netdev(fbd->netdev); + fbd->netdev = NULL; +} + +/** + * fbnic_netdev_alloc - Allocate a netdev and associate with fbnic + * @fbd: Driver specific structure to associate netdev with + * + * Allocate and initialize the netdev and netdev private structure. Bind + * together the hardware, netdev, and pci data structures. + * + * Return: Pointer to net_device on success, NULL on failure + **/ +struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd) +{ + struct net_device *netdev; + struct fbnic_net *fbn; + int default_queues; + + netdev = alloc_etherdev_mq(sizeof(*fbn), FBNIC_MAX_RXQS); + if (!netdev) + return NULL; + + SET_NETDEV_DEV(netdev, fbd->dev); + fbd->netdev = netdev; + + netdev->netdev_ops = &fbnic_netdev_ops; + netdev->stat_ops = &fbnic_stat_ops; + netdev->queue_mgmt_ops = &fbnic_queue_mgmt_ops; + netdev->netmem_tx = NETMEM_TX_DMA; + + fbnic_set_ethtool_ops(netdev); + + fbn = netdev_priv(netdev); + + fbn->netdev = netdev; + fbn->fbd = fbd; + + fbn->txq_size = FBNIC_TXQ_SIZE_DEFAULT; + fbn->hpq_size = FBNIC_HPQ_SIZE_DEFAULT; + fbn->ppq_size = FBNIC_PPQ_SIZE_DEFAULT; + fbn->rcq_size = FBNIC_RCQ_SIZE_DEFAULT; + + fbn->tx_usecs = FBNIC_TX_USECS_DEFAULT; + fbn->rx_usecs = FBNIC_RX_USECS_DEFAULT; + fbn->rx_max_frames = FBNIC_RX_FRAMES_DEFAULT; + + /* Initialize the hds_thresh */ + netdev->cfg->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; + fbn->hds_thresh = FBNIC_HDS_THRESH_DEFAULT; + + default_queues = netif_get_num_default_rss_queues(); + if (default_queues > fbd->max_num_queues) + default_queues = fbd->max_num_queues; + + fbnic_reset_queues(fbn, default_queues, default_queues); + + fbnic_reset_indir_tbl(fbn); + fbnic_rss_key_fill(fbn->rss_key); + fbnic_rss_init_en_mask(fbn); + + netdev->priv_flags |= IFF_UNICAST_FLT; + + netdev->gso_partial_features = + NETIF_F_GSO_GRE | + NETIF_F_GSO_GRE_CSUM | + NETIF_F_GSO_IPXIP4 | + NETIF_F_GSO_UDP_TUNNEL | + NETIF_F_GSO_UDP_TUNNEL_CSUM; + + netdev->features |= + netdev->gso_partial_features | + FBNIC_TUN_GSO_FEATURES | + NETIF_F_RXHASH | + NETIF_F_SG | + NETIF_F_HW_CSUM | + NETIF_F_RXCSUM | + NETIF_F_TSO | + NETIF_F_TSO_ECN | + NETIF_F_TSO6 | + NETIF_F_GSO_PARTIAL | + NETIF_F_GSO_UDP_L4; + + netdev->hw_features |= netdev->features; + netdev->vlan_features |= netdev->features; + netdev->hw_enc_features |= netdev->features; + netdev->features |= NETIF_F_NTUPLE; + + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_RX_SG; + + netdev->min_mtu = IPV6_MIN_MTU; + netdev->max_mtu = FBNIC_MAX_JUMBO_FRAME_SIZE - ETH_HLEN; + + /* TBD: This is workaround for BMC as phylink doesn't have support + * for leavling the link enabled if a BMC is present. + */ + netdev->ethtool->wol_enabled = true; + + netif_carrier_off(netdev); + + netif_tx_stop_all_queues(netdev); + + if (fbnic_phylink_create(netdev)) { + free_netdev(netdev); + fbd->netdev = NULL; + return NULL; + } + + return netdev; +} + +static int fbnic_dsn_to_mac_addr(u64 dsn, char *addr) +{ + addr[0] = (dsn >> 56) & 0xFF; + addr[1] = (dsn >> 48) & 0xFF; + addr[2] = (dsn >> 40) & 0xFF; + addr[3] = (dsn >> 16) & 0xFF; + addr[4] = (dsn >> 8) & 0xFF; + addr[5] = dsn & 0xFF; + + return is_valid_ether_addr(addr) ? 0 : -EINVAL; +} + +/** + * fbnic_netdev_register - Initialize general software structures + * @netdev: Netdev containing structure to initialize and register + * + * Initialize the MAC address for the netdev and register it. + * + * Return: 0 on success, negative on failure + **/ +int fbnic_netdev_register(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + u64 dsn = fbd->dsn; + u8 addr[ETH_ALEN]; + int err; + + err = fbnic_dsn_to_mac_addr(dsn, addr); + if (!err) { + ether_addr_copy(netdev->perm_addr, addr); + eth_hw_addr_set(netdev, addr); + } else { + /* A randomly assigned MAC address will cause provisioning + * issues so instead just fail to spawn the netdev and + * avoid any confusion. + */ + dev_err(fbd->dev, "MAC addr %pM invalid\n", addr); + return err; + } + + return register_netdev(netdev); +} + +void fbnic_netdev_unregister(struct net_device *netdev) +{ + unregister_netdev(netdev); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h new file mode 100644 index 000000000..eded20b0e --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_netdev.h @@ -0,0 +1,119 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_NETDEV_H_ +#define _FBNIC_NETDEV_H_ + +#include <linux/phylink.h> +#include <linux/types.h> + +#include "fbnic_csr.h" +#include "fbnic_rpc.h" +#include "fbnic_txrx.h" + +#define FBNIC_MAX_NAPI_VECTORS 128u +#define FBNIC_MIN_RXD_PER_FRAME 2 + +/* Natively supported tunnel GSO features (not thru GSO_PARTIAL) */ +#define FBNIC_TUN_GSO_FEATURES NETIF_F_GSO_IPXIP6 + +struct fbnic_net { + struct bpf_prog *xdp_prog; + + struct fbnic_ring *tx[FBNIC_MAX_TXQS + FBNIC_MAX_XDPQS]; + struct fbnic_ring *rx[FBNIC_MAX_RXQS]; + + struct fbnic_napi_vector *napi[FBNIC_MAX_NAPI_VECTORS]; + + struct net_device *netdev; + struct fbnic_dev *fbd; + + u32 txq_size; + u32 hpq_size; + u32 ppq_size; + u32 rcq_size; + + u32 hds_thresh; + + u16 rx_usecs; + u16 tx_usecs; + + u32 rx_max_frames; + + u16 num_napi; + + struct phylink *phylink; + struct phylink_config phylink_config; + struct phylink_pcs *pcs; + + u8 aui; + u8 fec; + + /* Cached top bits of the HW time counter for 40b -> 64b conversion */ + u32 time_high; + /* Protect readers of @time_offset, writers take @time_lock. */ + struct u64_stats_sync time_seq; + /* Offset in ns between free running NIC PHC and time set via PTP + * clock callbacks + */ + s64 time_offset; + + u16 num_tx_queues; + u16 num_rx_queues; + + u8 indir_tbl[FBNIC_RPC_RSS_TBL_COUNT][FBNIC_RPC_RSS_TBL_SIZE]; + u32 rss_key[FBNIC_RPC_RSS_KEY_DWORD_LEN]; + u32 rss_flow_hash[FBNIC_NUM_HASH_OPT]; + + /* Storage for stats after ring destruction */ + struct fbnic_queue_stats tx_stats; + struct fbnic_queue_stats rx_stats; + struct fbnic_queue_stats bdq_stats; + u64 link_down_events; + + /* Time stamping filter config */ + struct kernel_hwtstamp_config hwtstamp_config; + + bool tx_pause; +}; + +int __fbnic_open(struct fbnic_net *fbn); +void fbnic_up(struct fbnic_net *fbn); +void fbnic_down(struct fbnic_net *fbn); +void fbnic_down_noidle(struct fbnic_net *fbn); + +struct net_device *fbnic_netdev_alloc(struct fbnic_dev *fbd); +void fbnic_netdev_free(struct fbnic_dev *fbd); +int fbnic_netdev_register(struct net_device *netdev); +void fbnic_netdev_unregister(struct net_device *netdev); +void fbnic_reset_queues(struct fbnic_net *fbn, + unsigned int tx, unsigned int rx); + +void fbnic_set_ethtool_ops(struct net_device *dev); + +int fbnic_ptp_setup(struct fbnic_dev *fbd); +void fbnic_ptp_destroy(struct fbnic_dev *fbd); +void fbnic_time_init(struct fbnic_net *fbn); +int fbnic_time_start(struct fbnic_net *fbn); +void fbnic_time_stop(struct fbnic_net *fbn); + +void __fbnic_set_rx_mode(struct fbnic_dev *fbd, + struct netdev_hw_addr_list *uc, + struct netdev_hw_addr_list *mc); +void fbnic_clear_rx_mode(struct fbnic_dev *fbd); + +void fbnic_phylink_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause); +int fbnic_phylink_set_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause); +int fbnic_phylink_ethtool_ksettings_get(struct net_device *netdev, + struct ethtool_link_ksettings *cmd); +int fbnic_phylink_get_fecparam(struct net_device *netdev, + struct ethtool_fecparam *fecparam); +int fbnic_phylink_create(struct net_device *netdev); +void fbnic_phylink_destroy(struct net_device *netdev); +int fbnic_phylink_init(struct net_device *netdev); +void fbnic_phylink_pmd_training_complete_notify(struct net_device *netdev); +bool fbnic_check_split_frames(struct bpf_prog *prog, + unsigned int mtu, u32 hds_threshold); +#endif /* _FBNIC_NETDEV_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_pci.c b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c new file mode 100644 index 000000000..8b9bc9e8e --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_pci.c @@ -0,0 +1,673 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/init.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/rtnetlink.h> +#include <linux/types.h> +#include <net/devlink.h> + +#include "fbnic.h" +#include "fbnic_drvinfo.h" +#include "fbnic_hw_stats.h" +#include "fbnic_netdev.h" + +char fbnic_driver_name[] = DRV_NAME; + +MODULE_DESCRIPTION(DRV_SUMMARY); +MODULE_LICENSE("GPL"); + +static const struct fbnic_info fbnic_asic_info = { + .max_num_queues = FBNIC_MAX_QUEUES, + .bar_mask = BIT(0) | BIT(4) +}; + +static const struct fbnic_info *fbnic_info_tbl[] = { + [fbnic_board_asic] = &fbnic_asic_info, +}; + +static const struct pci_device_id fbnic_pci_tbl[] = { + { PCI_DEVICE_DATA(META, FBNIC_ASIC, fbnic_board_asic) }, + /* Required last entry */ + {0, } +}; +MODULE_DEVICE_TABLE(pci, fbnic_pci_tbl); + +u32 fbnic_rd32(struct fbnic_dev *fbd, u32 reg) +{ + u32 __iomem *csr = READ_ONCE(fbd->uc_addr0); + u32 value; + + if (!csr) + return ~0U; + + value = readl(csr + reg); + + /* If any bits are 0 value should be valid */ + if (~value) + return value; + + /* All 1's may be valid if ZEROs register still works */ + if (reg != FBNIC_MASTER_SPARE_0 && ~readl(csr + FBNIC_MASTER_SPARE_0)) + return value; + + /* Hardware is giving us all 1's reads, assume it is gone */ + WRITE_ONCE(fbd->uc_addr0, NULL); + WRITE_ONCE(fbd->uc_addr4, NULL); + + dev_err(fbd->dev, + "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", + reg, reg << 2); + + /* Notify stack that device has lost (PCIe) link */ + if (!fbnic_init_failure(fbd)) + netif_device_detach(fbd->netdev); + + return ~0U; +} + +bool fbnic_fw_present(struct fbnic_dev *fbd) +{ + return !!READ_ONCE(fbd->uc_addr4); +} + +void fbnic_fw_wr32(struct fbnic_dev *fbd, u32 reg, u32 val) +{ + u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); + + if (csr) + writel(val, csr + reg); +} + +u32 fbnic_fw_rd32(struct fbnic_dev *fbd, u32 reg) +{ + u32 __iomem *csr = READ_ONCE(fbd->uc_addr4); + u32 value; + + if (!csr) + return ~0U; + + value = readl(csr + reg); + + /* If any bits are 0 value should be valid */ + if (~value) + return value; + + /* All 1's may be valid if ZEROs register still works */ + if (reg != FBNIC_FW_ZERO_REG && ~readl(csr + FBNIC_FW_ZERO_REG)) + return value; + + /* Hardware is giving us all 1's reads, assume it is gone */ + WRITE_ONCE(fbd->uc_addr0, NULL); + WRITE_ONCE(fbd->uc_addr4, NULL); + + dev_err(fbd->dev, + "Failed read (idx 0x%x AKA addr 0x%x), disabled CSR access, awaiting reset\n", + reg, reg << 2); + + /* Notify stack that device has lost (PCIe) link */ + if (!fbnic_init_failure(fbd)) + netif_device_detach(fbd->netdev); + + return ~0U; +} + +static void fbnic_service_task_start(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + + schedule_delayed_work(&fbd->service_task, HZ); +} + +static void fbnic_service_task_stop(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + + cancel_delayed_work(&fbd->service_task); +} + +void fbnic_up(struct fbnic_net *fbn) +{ + fbnic_enable(fbn); + + fbnic_fill(fbn); + + fbnic_rss_reinit_hw(fbn->fbd, fbn); + + netif_addr_lock_bh(fbn->netdev); + __fbnic_set_rx_mode(fbn->fbd, &fbn->netdev->uc, &fbn->netdev->mc); + netif_addr_unlock_bh(fbn->netdev); + + /* Enable Tx/Rx processing */ + fbnic_napi_enable(fbn); + netif_tx_wake_all_queues(fbn->netdev); + + fbnic_service_task_start(fbn); + + fbnic_dbg_up(fbn); +} + +void fbnic_down_noidle(struct fbnic_net *fbn) +{ + fbnic_dbg_down(fbn); + + fbnic_service_task_stop(fbn); + + /* Disable Tx/Rx Processing */ + fbnic_napi_disable(fbn); + netif_tx_disable(fbn->netdev); + + fbnic_clear_rx_mode(fbn->fbd); + fbnic_clear_rules(fbn->fbd); + fbnic_rss_disable_hw(fbn->fbd); + fbnic_disable(fbn); +} + +void fbnic_down(struct fbnic_net *fbn) +{ + fbnic_down_noidle(fbn); + + fbnic_wait_all_queues_idle(fbn->fbd, false); + + fbnic_flush(fbn); +} + +static int fbnic_fw_config_after_crash(struct fbnic_dev *fbd) +{ + if (fbnic_fw_xmit_ownership_msg(fbd, true)) { + dev_err(fbd->dev, "NIC failed to take ownership\n"); + + return -1; + } + + fbnic_rpc_reset_valid_entries(fbd); + netif_addr_lock_bh(fbd->netdev); + __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc); + netif_addr_unlock_bh(fbd->netdev); + + return 0; +} + +static void fbnic_health_check(struct fbnic_dev *fbd) +{ + struct fbnic_fw_mbx *tx_mbx = &fbd->mbx[FBNIC_IPC_MBX_TX_IDX]; + + /* As long as the heart is beating the FW is healthy */ + if (fbd->fw_heartbeat_enabled) + return; + + /* If the Tx mailbox still has messages sitting in it then there likely + * isn't anything we can do. We will wait until the mailbox is empty to + * report the fault so we can collect the crashlog. + */ + if (tx_mbx->head != tx_mbx->tail) + return; + + fbnic_devlink_fw_report(fbd, "Firmware crash detected!"); + fbnic_devlink_otp_check(fbd, "error detected after firmware recovery"); + + if (fbnic_fw_config_after_crash(fbd)) + dev_err(fbd->dev, "Firmware recovery failed after crash\n"); +} + +static void fbnic_service_task(struct work_struct *work) +{ + struct fbnic_dev *fbd = container_of(to_delayed_work(work), + struct fbnic_dev, service_task); + struct net_device *netdev = fbd->netdev; + + if (netif_running(netdev)) + fbnic_phylink_pmd_training_complete_notify(netdev); + + rtnl_lock(); + + fbnic_get_hw_stats32(fbd); + + if (fbd->ps_timeout && fbnic_mac_check_tx_pause(fbd)) + fbnic_mac_ps_protect_handler(fbd); + + fbnic_fw_check_heartbeat(fbd); + + fbnic_health_check(fbd); + + fbnic_bmc_rpc_check(fbd); + + if (netif_carrier_ok(fbd->netdev)) { + netdev_lock(fbd->netdev); + fbnic_napi_depletion_check(fbd->netdev); + netdev_unlock(fbd->netdev); + } + + if (netif_running(netdev)) + schedule_delayed_work(&fbd->service_task, HZ); + + rtnl_unlock(); +} + +/** + * fbnic_probe - Device Initialization Routine + * @pdev: PCI device information struct + * @ent: entry in fbnic_pci_tbl + * + * Initializes a PCI device identified by a pci_dev structure. + * The OS initialization, configuring of the adapter private structure, + * and a hardware reset occur. + * + * Return: 0 on success, negative on failure + **/ +static int fbnic_probe(struct pci_dev *pdev, const struct pci_device_id *ent) +{ + const struct fbnic_info *info = fbnic_info_tbl[ent->driver_data]; + struct net_device *netdev; + struct fbnic_dev *fbd; + int err; + + if (pdev->error_state != pci_channel_io_normal) { + dev_err(&pdev->dev, + "PCI device still in an error state. Unable to load...\n"); + return -EIO; + } + + err = pcim_enable_device(pdev); + if (err) { + dev_err(&pdev->dev, "PCI enable device failed: %d\n", err); + return err; + } + + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(46)); + if (err) + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); + if (err) { + dev_err(&pdev->dev, "DMA configuration failed: %d\n", err); + return err; + } + + err = pcim_iomap_regions(pdev, info->bar_mask, fbnic_driver_name); + if (err) { + dev_err(&pdev->dev, + "pci_request_selected_regions failed: %d\n", err); + return err; + } + + fbd = fbnic_devlink_alloc(pdev); + if (!fbd) { + dev_err(&pdev->dev, "Devlink allocation failed\n"); + return -ENOMEM; + } + + err = fbnic_devlink_health_create(fbd); + if (err) + goto free_fbd; + + /* Populate driver with hardware-specific info and handlers */ + fbd->max_num_queues = info->max_num_queues; + + fbd->ps_timeout = FBNIC_MAC_PS_TO_DEFAULT_MS; + + pci_set_master(pdev); + pci_save_state(pdev); + + INIT_DELAYED_WORK(&fbd->service_task, fbnic_service_task); + + err = fbnic_alloc_irqs(fbd); + if (err) + goto err_destroy_health; + + err = fbnic_mac_init(fbd); + if (err) { + dev_err(&pdev->dev, "Failed to initialize MAC: %d\n", err); + goto free_irqs; + } + + err = fbnic_fw_log_init(fbd); + if (err) + dev_warn(fbd->dev, + "Unable to initialize firmware log buffer: %d\n", + err); + + err = fbnic_fw_request_mbx(fbd); + if (err) { + dev_err(&pdev->dev, + "Firmware mailbox initialization failure\n"); + goto free_fw_log; + } + + /* Send the request to enable the FW logging to host. Note if this + * fails we ignore the error and just display a message as it is + * possible the FW is just too old to support the logging and needs + * to be updated. + */ + fbnic_fw_log_enable(fbd, true); + + fbnic_devlink_register(fbd); + fbnic_devlink_otp_check(fbd, "error detected during probe"); + fbnic_dbg_fbd_init(fbd); + + /* Capture snapshot of hardware stats so netdev can calculate delta */ + fbnic_init_hw_stats(fbd); + + fbnic_hwmon_register(fbd); + + if (!fbd->dsn) { + dev_warn(&pdev->dev, "Reading serial number failed\n"); + goto init_failure_mode; + } + + if (fbnic_mdiobus_create(fbd)) + goto init_failure_mode; + + netdev = fbnic_netdev_alloc(fbd); + if (!netdev) { + dev_err(&pdev->dev, "Netdev allocation failed\n"); + goto init_failure_mode; + } + + err = fbnic_ptp_setup(fbd); + if (err) + goto ifm_free_netdev; + + err = fbnic_netdev_register(netdev); + if (err) { + dev_err(&pdev->dev, "Netdev registration failed: %d\n", err); + goto ifm_destroy_ptp; + } + + return 0; + +ifm_destroy_ptp: + fbnic_ptp_destroy(fbd); +ifm_free_netdev: + fbnic_netdev_free(fbd); +init_failure_mode: + dev_warn(&pdev->dev, "Probe error encountered, entering init failure mode. Normal networking functionality will not be available.\n"); + /* Always return 0 even on error so devlink is registered to allow + * firmware updates for fixes. + */ + return 0; +free_fw_log: + fbnic_fw_log_free(fbd); +free_irqs: + fbnic_free_irqs(fbd); +err_destroy_health: + fbnic_devlink_health_destroy(fbd); +free_fbd: + fbnic_devlink_free(fbd); + + return err; +} + +/** + * fbnic_remove - Device Removal Routine + * @pdev: PCI device information struct + * + * Called by the PCI subsystem to alert the driver that it should release + * a PCI device. This could be caused by a Hot-Plug event, or because the + * driver is going to be removed from memory. + **/ +static void fbnic_remove(struct pci_dev *pdev) +{ + struct fbnic_dev *fbd = pci_get_drvdata(pdev); + + if (!fbnic_init_failure(fbd)) { + struct net_device *netdev = fbd->netdev; + + fbnic_netdev_unregister(netdev); + cancel_delayed_work_sync(&fbd->service_task); + fbnic_ptp_destroy(fbd); + fbnic_netdev_free(fbd); + } + + fbnic_hwmon_unregister(fbd); + fbnic_dbg_fbd_exit(fbd); + fbnic_devlink_unregister(fbd); + fbnic_fw_log_disable(fbd); + fbnic_fw_free_mbx(fbd); + fbnic_fw_log_free(fbd); + fbnic_free_irqs(fbd); + + fbnic_devlink_health_destroy(fbd); + fbnic_devlink_free(fbd); +} + +static int fbnic_pm_suspend(struct device *dev) +{ + struct fbnic_dev *fbd = dev_get_drvdata(dev); + struct net_device *netdev = fbd->netdev; + + if (fbnic_init_failure(fbd)) + goto null_uc_addr; + + rtnl_lock(); + netdev_lock(netdev); + + netif_device_detach(netdev); + + if (netif_running(netdev)) + netdev->netdev_ops->ndo_stop(netdev); + + netdev_unlock(netdev); + rtnl_unlock(); + +null_uc_addr: + fbnic_fw_log_disable(fbd); + + devl_lock(priv_to_devlink(fbd)); + + fbnic_fw_free_mbx(fbd); + + devl_unlock(priv_to_devlink(fbd)); + + /* Free the IRQs so they aren't trying to occupy sleeping CPUs */ + fbnic_free_irqs(fbd); + + /* Hardware is about to go away, so switch off MMIO access internally */ + WRITE_ONCE(fbd->uc_addr0, NULL); + WRITE_ONCE(fbd->uc_addr4, NULL); + + return 0; +} + +static int __fbnic_pm_resume(struct device *dev) +{ + struct fbnic_dev *fbd = dev_get_drvdata(dev); + struct net_device *netdev = fbd->netdev; + void __iomem * const *iomap_table; + struct fbnic_net *fbn; + int err; + + /* Restore MMIO access */ + iomap_table = pcim_iomap_table(to_pci_dev(dev)); + fbd->uc_addr0 = iomap_table[0]; + fbd->uc_addr4 = iomap_table[4]; + + /* Rerequest the IRQs */ + err = fbnic_alloc_irqs(fbd); + if (err) + goto err_invalidate_uc_addr; + + fbd->mac->init_regs(fbd); + + devl_lock(priv_to_devlink(fbd)); + + /* Re-enable mailbox */ + err = fbnic_fw_request_mbx(fbd); + devl_unlock(priv_to_devlink(fbd)); + if (err) + goto err_free_irqs; + + /* Only send log history if log buffer is empty to prevent duplicate + * log entries. + */ + fbnic_fw_log_enable(fbd, list_empty(&fbd->fw_log.entries)); + + /* Since the FW should be up, check if it reported OTP errors */ + fbnic_devlink_otp_check(fbd, "error detected after PM resume"); + + /* No netdev means there isn't a network interface to bring up */ + if (fbnic_init_failure(fbd)) + return 0; + + fbn = netdev_priv(netdev); + + /* Reset the queues if needed */ + fbnic_reset_queues(fbn, fbn->num_tx_queues, fbn->num_rx_queues); + + rtnl_lock(); + netdev_lock(netdev); + + if (netif_running(netdev)) + err = __fbnic_open(fbn); + + netdev_unlock(netdev); + rtnl_unlock(); + if (err) + goto err_free_mbx; + + return 0; +err_free_mbx: + fbnic_fw_log_disable(fbd); + + devl_lock(priv_to_devlink(fbd)); + fbnic_fw_free_mbx(fbd); + devl_unlock(priv_to_devlink(fbd)); +err_free_irqs: + fbnic_free_irqs(fbd); +err_invalidate_uc_addr: + WRITE_ONCE(fbd->uc_addr0, NULL); + WRITE_ONCE(fbd->uc_addr4, NULL); + return err; +} + +static void __fbnic_pm_attach(struct device *dev) +{ + struct fbnic_dev *fbd = dev_get_drvdata(dev); + struct net_device *netdev = fbd->netdev; + struct fbnic_net *fbn; + + rtnl_lock(); + fbnic_reset_hw_stats(fbd); + rtnl_unlock(); + + if (fbnic_init_failure(fbd)) + return; + + fbn = netdev_priv(netdev); + + if (netif_running(netdev)) + fbnic_up(fbn); + + netif_device_attach(netdev); +} + +static int __maybe_unused fbnic_pm_resume(struct device *dev) +{ + int err; + + err = __fbnic_pm_resume(dev); + if (!err) + __fbnic_pm_attach(dev); + + return err; +} + +static const struct dev_pm_ops fbnic_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(fbnic_pm_suspend, fbnic_pm_resume) +}; + +static void fbnic_shutdown(struct pci_dev *pdev) +{ + fbnic_pm_suspend(&pdev->dev); +} + +static pci_ers_result_t fbnic_err_error_detected(struct pci_dev *pdev, + pci_channel_state_t state) +{ + /* Disconnect device if failure is not recoverable via reset */ + if (state == pci_channel_io_perm_failure) + return PCI_ERS_RESULT_DISCONNECT; + + fbnic_pm_suspend(&pdev->dev); + + /* Request a slot reset */ + return PCI_ERS_RESULT_NEED_RESET; +} + +static pci_ers_result_t fbnic_err_slot_reset(struct pci_dev *pdev) +{ + int err; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + + if (pci_enable_device_mem(pdev)) { + dev_err(&pdev->dev, + "Cannot re-enable PCI device after reset.\n"); + return PCI_ERS_RESULT_DISCONNECT; + } + + /* Restore device to previous state */ + err = __fbnic_pm_resume(&pdev->dev); + + return err ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED; +} + +static void fbnic_err_resume(struct pci_dev *pdev) +{ + __fbnic_pm_attach(&pdev->dev); +} + +static const struct pci_error_handlers fbnic_err_handler = { + .error_detected = fbnic_err_error_detected, + .slot_reset = fbnic_err_slot_reset, + .resume = fbnic_err_resume, +}; + +static struct pci_driver fbnic_driver = { + .name = fbnic_driver_name, + .id_table = fbnic_pci_tbl, + .probe = fbnic_probe, + .remove = fbnic_remove, + .driver.pm = &fbnic_pm_ops, + .shutdown = fbnic_shutdown, + .err_handler = &fbnic_err_handler, +}; + +/** + * fbnic_init_module - Driver Registration Routine + * + * The first routine called when the driver is loaded. All it does is + * register with the PCI subsystem. + * + * Return: 0 on success, negative on failure + **/ +static int __init fbnic_init_module(void) +{ + int err; + + fbnic_dbg_init(); + + err = pci_register_driver(&fbnic_driver); + if (err) { + fbnic_dbg_exit(); + goto out; + } + + pr_info(DRV_SUMMARY " (%s)", fbnic_driver.name); +out: + return err; +} +module_init(fbnic_init_module); + +/** + * fbnic_exit_module - Driver Exit Cleanup Routine + * + * Called just before the driver is removed from memory. + **/ +static void __exit fbnic_exit_module(void) +{ + pci_unregister_driver(&fbnic_driver); + + fbnic_dbg_exit(); +} +module_exit(fbnic_exit_module); diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c new file mode 100644 index 000000000..09c522511 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_phylink.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/pcs/pcs-xpcs.h> +#include <linux/phy.h> +#include <linux/phylink.h> + +#include "fbnic.h" +#include "fbnic_mac.h" +#include "fbnic_netdev.h" + +static phy_interface_t fbnic_phylink_select_interface(u8 aui) +{ + switch (aui) { + case FBNIC_AUI_100GAUI2: + return PHY_INTERFACE_MODE_100GBASEP; + case FBNIC_AUI_50GAUI1: + return PHY_INTERFACE_MODE_50GBASER; + case FBNIC_AUI_LAUI2: + return PHY_INTERFACE_MODE_LAUI; + case FBNIC_AUI_25GAUI: + return PHY_INTERFACE_MODE_25GBASER; + } + + return PHY_INTERFACE_MODE_NA; +} + +void fbnic_phylink_get_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + phylink_ethtool_get_pauseparam(fbn->phylink, pause); +} + +int fbnic_phylink_set_pauseparam(struct net_device *netdev, + struct ethtool_pauseparam *pause) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + return phylink_ethtool_set_pauseparam(fbn->phylink, pause); +} + +static void +fbnic_phylink_get_supported_fec_modes(unsigned long *supported) +{ + /* The NIC can support up to 8 possible combinations. + * Either 50G-CR, or 100G-CR2 + * This is with RS FEC mode only + * Either 25G-CR, or 50G-CR2 + * This is with No FEC, RS, or Base-R + */ + if (phylink_test(supported, 100000baseCR2_Full) || + phylink_test(supported, 50000baseCR_Full)) + phylink_set(supported, FEC_RS); + if (phylink_test(supported, 50000baseCR2_Full) || + phylink_test(supported, 25000baseCR_Full)) { + phylink_set(supported, FEC_BASER); + phylink_set(supported, FEC_NONE); + phylink_set(supported, FEC_RS); + } +} + +int fbnic_phylink_ethtool_ksettings_get(struct net_device *netdev, + struct ethtool_link_ksettings *cmd) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + int err; + + err = phylink_ethtool_ksettings_get(fbn->phylink, cmd); + if (!err) { + unsigned long *supp = cmd->link_modes.supported; + + cmd->base.port = PORT_DA; + cmd->lanes = (fbn->aui & FBNIC_AUI_MODE_R2) ? 2 : 1; + + fbnic_phylink_get_supported_fec_modes(supp); + } + + return err; +} + +int fbnic_phylink_get_fecparam(struct net_device *netdev, + struct ethtool_fecparam *fecparam) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + if (fbn->fec & FBNIC_FEC_RS) { + fecparam->active_fec = ETHTOOL_FEC_RS; + fecparam->fec = ETHTOOL_FEC_RS; + } else if (fbn->fec & FBNIC_FEC_BASER) { + fecparam->active_fec = ETHTOOL_FEC_BASER; + fecparam->fec = ETHTOOL_FEC_BASER; + } else { + fecparam->active_fec = ETHTOOL_FEC_OFF; + fecparam->fec = ETHTOOL_FEC_OFF; + } + + if (fbn->aui & FBNIC_AUI_MODE_PAM4) + fecparam->fec |= ETHTOOL_FEC_AUTO; + + return 0; +} + +static struct phylink_pcs * +fbnic_phylink_mac_select_pcs(struct phylink_config *config, + phy_interface_t interface) +{ + struct net_device *netdev = to_net_dev(config->dev); + struct fbnic_net *fbn = netdev_priv(netdev); + + return fbn->pcs; +} + +static int +fbnic_phylink_mac_prepare(struct phylink_config *config, unsigned int mode, + phy_interface_t iface) +{ + struct net_device *netdev = to_net_dev(config->dev); + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + fbd->mac->prepare(fbd, fbn->aui, fbn->fec); + + return 0; +} + +static void +fbnic_phylink_mac_config(struct phylink_config *config, unsigned int mode, + const struct phylink_link_state *state) +{ +} + +static int +fbnic_phylink_mac_finish(struct phylink_config *config, unsigned int mode, + phy_interface_t iface) +{ + struct net_device *netdev = to_net_dev(config->dev); + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + /* Retest the link state and restart interrupts */ + fbd->mac->get_link(fbd, fbn->aui, fbn->fec); + + return 0; +} + +static void +fbnic_phylink_mac_link_down(struct phylink_config *config, unsigned int mode, + phy_interface_t interface) +{ + struct net_device *netdev = to_net_dev(config->dev); + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + fbd->mac->link_down(fbd); + + fbn->link_down_events++; +} + +static void +fbnic_phylink_mac_link_up(struct phylink_config *config, + struct phy_device *phy, unsigned int mode, + phy_interface_t interface, int speed, int duplex, + bool tx_pause, bool rx_pause) +{ + struct net_device *netdev = to_net_dev(config->dev); + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + fbn->tx_pause = tx_pause; + fbnic_config_drop_mode(fbn, tx_pause); + + fbd->mac->link_up(fbd, tx_pause, rx_pause); +} + +static const struct phylink_mac_ops fbnic_phylink_mac_ops = { + .mac_select_pcs = fbnic_phylink_mac_select_pcs, + .mac_prepare = fbnic_phylink_mac_prepare, + .mac_config = fbnic_phylink_mac_config, + .mac_finish = fbnic_phylink_mac_finish, + .mac_link_down = fbnic_phylink_mac_link_down, + .mac_link_up = fbnic_phylink_mac_link_up, +}; + +/** + * fbnic_phylink_create - Phylink device creation + * @netdev: Network Device struct to attach phylink device + * + * Initialize and attach a phylink instance to the device. The phylink + * device will make use of the netdev struct to track carrier and will + * eventually be used to expose the current state of the MAC and PCS + * setup. + * + * Return: 0 on success, negative on failure + **/ +int fbnic_phylink_create(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + struct phylink_pcs *pcs; + struct phylink *phylink; + int err; + + pcs = xpcs_create_pcs_mdiodev(fbd->mdio_bus, 0); + if (IS_ERR(pcs)) { + err = PTR_ERR(pcs); + dev_err(fbd->dev, "Failed to create PCS device: %d\n", err); + return err; + } + + fbn->pcs = pcs; + + fbn->phylink_config.dev = &netdev->dev; + fbn->phylink_config.type = PHYLINK_NETDEV; + fbn->phylink_config.mac_capabilities = MAC_SYM_PAUSE | MAC_ASYM_PAUSE | + MAC_25000FD | MAC_50000FD | + MAC_100000FD; + fbn->phylink_config.default_an_inband = true; + + __set_bit(PHY_INTERFACE_MODE_100GBASEP, + fbn->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_50GBASER, + fbn->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_LAUI, + fbn->phylink_config.supported_interfaces); + __set_bit(PHY_INTERFACE_MODE_25GBASER, + fbn->phylink_config.supported_interfaces); + + fbnic_mac_get_fw_settings(fbd, &fbn->aui, &fbn->fec); + + phylink = phylink_create(&fbn->phylink_config, NULL, + fbnic_phylink_select_interface(fbn->aui), + &fbnic_phylink_mac_ops); + if (IS_ERR(phylink)) { + err = PTR_ERR(phylink); + dev_err(netdev->dev.parent, + "Failed to create Phylink interface, err: %d\n", err); + xpcs_destroy_pcs(pcs); + return err; + } + + fbn->phylink = phylink; + + return 0; +} + +/** + * fbnic_phylink_destroy - Teardown phylink related interfaces + * @netdev: Network Device struct containing phylink device + * + * Detach and free resources related to phylink interface. + **/ +void fbnic_phylink_destroy(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + + if (fbn->phylink) + phylink_destroy(fbn->phylink); + if (fbn->pcs) + xpcs_destroy_pcs(fbn->pcs); +} + +/** + * fbnic_phylink_pmd_training_complete_notify - PMD training complete notifier + * @netdev: Netdev struct phylink device attached to + * + * When the link first comes up the PMD will have a period of 2 to 3 seconds + * where the link will flutter due to link training. To avoid spamming the + * kernel log with messages about this we add a delay of 4 seconds from the + * time of the last PCS report of link so that we can guarantee we are unlikely + * to see any further link loss events due to link training. + **/ +void fbnic_phylink_pmd_training_complete_notify(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + struct fbnic_dev *fbd = fbn->fbd; + + if (fbd->pmd_state != FBNIC_PMD_TRAINING) + return; + + /* Prevent reading end_of_pmd_training until we verified state */ + smp_rmb(); + + if (!time_before(READ_ONCE(fbd->end_of_pmd_training), jiffies)) + return; + + /* At this point we have verified that the link has been up for + * the full training duration. As a first step we will try + * transitioning to link ready. + */ + if (cmpxchg(&fbd->pmd_state, FBNIC_PMD_TRAINING, + FBNIC_PMD_LINK_READY) != FBNIC_PMD_TRAINING) + return; + + /* Perform a follow-up check to verify that the link didn't flap + * just before our transition by rechecking the training timer. + */ + if (!time_before(READ_ONCE(fbd->end_of_pmd_training), jiffies)) + return; + + /* The training timeout has been completed. We are good to swap out + * link_ready for send_data assuming no other events have occurred + * that would have pulled us back into initialization or training. + */ + if (cmpxchg(&fbd->pmd_state, FBNIC_PMD_LINK_READY, + FBNIC_PMD_SEND_DATA) != FBNIC_PMD_LINK_READY) + return; + + phylink_pcs_change(fbn->pcs, false); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c new file mode 100644 index 000000000..bc0f38b6a --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.c @@ -0,0 +1,1247 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/etherdevice.h> +#include <linux/ethtool.h> +#include <net/ipv6.h> + +#include "fbnic.h" +#include "fbnic_fw.h" +#include "fbnic_netdev.h" +#include "fbnic_rpc.h" + +void fbnic_reset_indir_tbl(struct fbnic_net *fbn) +{ + unsigned int num_rx = fbn->num_rx_queues; + unsigned int i; + + if (netif_is_rxfh_configured(fbn->netdev)) + return; + + for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) + fbn->indir_tbl[0][i] = ethtool_rxfh_indir_default(i, num_rx); +} + +void fbnic_rss_key_fill(u32 *buffer) +{ + static u32 rss_key[FBNIC_RPC_RSS_KEY_DWORD_LEN]; + + net_get_random_once(rss_key, sizeof(rss_key)); + rss_key[FBNIC_RPC_RSS_KEY_LAST_IDX] &= FBNIC_RPC_RSS_KEY_LAST_MASK; + + memcpy(buffer, rss_key, sizeof(rss_key)); +} + +#define RX_HASH_OPT_L4 \ + (RXH_IP_SRC | RXH_IP_DST | RXH_L4_B_0_1 | RXH_L4_B_2_3) +#define RX_HASH_OPT_L3 \ + (RXH_IP_SRC | RXH_IP_DST) +#define RX_HASH_OPT_L2 RXH_L2DA + +void fbnic_rss_init_en_mask(struct fbnic_net *fbn) +{ + fbn->rss_flow_hash[FBNIC_TCP4_HASH_OPT] = RX_HASH_OPT_L4; + fbn->rss_flow_hash[FBNIC_TCP6_HASH_OPT] = RX_HASH_OPT_L4; + + fbn->rss_flow_hash[FBNIC_UDP4_HASH_OPT] = RX_HASH_OPT_L3; + fbn->rss_flow_hash[FBNIC_UDP6_HASH_OPT] = RX_HASH_OPT_L3; + fbn->rss_flow_hash[FBNIC_IPV4_HASH_OPT] = RX_HASH_OPT_L3; + fbn->rss_flow_hash[FBNIC_IPV6_HASH_OPT] = RX_HASH_OPT_L3; + + fbn->rss_flow_hash[FBNIC_ETHER_HASH_OPT] = RX_HASH_OPT_L2; +} + +void fbnic_rss_disable_hw(struct fbnic_dev *fbd) +{ + /* Disable RPC by clearing enable bit and configuration */ + if (!fbnic_bmc_present(fbd)) + wr32(fbd, FBNIC_RPC_RMI_CONFIG, + FIELD_PREP(FBNIC_RPC_RMI_CONFIG_OH_BYTES, 20)); +} + +#define FBNIC_FH_2_RSSEM_BIT(_fh, _rssem, _val) \ + FIELD_PREP(FBNIC_RPC_ACT_TBL1_RSS_ENA_##_rssem, \ + FIELD_GET(RXH_##_fh, _val)) +u16 fbnic_flow_hash_2_rss_en_mask(struct fbnic_net *fbn, int flow_type) +{ + u32 flow_hash = fbn->rss_flow_hash[flow_type]; + u32 rss_en_mask = 0; + + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(L2DA, L2_DA, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP_SRC, IP_SRC, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP_DST, IP_DST, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(L4_B_0_1, L4_SRC, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(L4_B_2_3, L4_DST, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP6_FL, OV6_FL_LBL, flow_hash); + rss_en_mask |= FBNIC_FH_2_RSSEM_BIT(IP6_FL, IV6_FL_LBL, flow_hash); + + return rss_en_mask; +} + +void fbnic_rss_reinit_hw(struct fbnic_dev *fbd, struct fbnic_net *fbn) +{ + unsigned int i; + + for (i = 0; i < FBNIC_RPC_RSS_TBL_SIZE; i++) { + wr32(fbd, FBNIC_RPC_RSS_TBL(0, i), fbn->indir_tbl[0][i]); + wr32(fbd, FBNIC_RPC_RSS_TBL(1, i), fbn->indir_tbl[1][i]); + } + + for (i = 0; i < FBNIC_RPC_RSS_KEY_DWORD_LEN; i++) + wr32(fbd, FBNIC_RPC_RSS_KEY(i), fbn->rss_key[i]); + + /* Default action for this to drop w/ no destination */ + wr32(fbd, FBNIC_RPC_ACT_TBL0_DEFAULT, FBNIC_RPC_ACT_TBL0_DROP); + wrfl(fbd); + + wr32(fbd, FBNIC_RPC_ACT_TBL1_DEFAULT, 0); + + /* If it isn't already enabled set the RMI Config value to enable RPC */ + wr32(fbd, FBNIC_RPC_RMI_CONFIG, + FIELD_PREP(FBNIC_RPC_RMI_CONFIG_MTU, FBNIC_MAX_JUMBO_FRAME_SIZE) | + FIELD_PREP(FBNIC_RPC_RMI_CONFIG_OH_BYTES, 20) | + FBNIC_RPC_RMI_CONFIG_ENABLE); +} + +void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, + bool enable_host) +{ + struct fbnic_act_tcam *act_tcam; + struct fbnic_mac_addr *mac_addr; + int j; + + /* We need to add the all multicast filter at the end of the + * multicast address list. This way if there are any that are + * shared between the host and the BMC they can be directed to + * both. Otherwise the remainder just get sent directly to the + * BMC. + */ + mac_addr = &fbd->mac_addr[fbd->mac_addr_boundary - 1]; + if (fbnic_bmc_present(fbd) && fbd->fw_cap.all_multi) { + if (mac_addr->state != FBNIC_TCAM_S_VALID) { + eth_zero_addr(mac_addr->value.addr8); + eth_broadcast_addr(mac_addr->mask.addr8); + mac_addr->value.addr8[0] ^= 1; + mac_addr->mask.addr8[0] ^= 1; + set_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + if (enable_host) + set_bit(FBNIC_MAC_ADDR_T_ALLMULTI, + mac_addr->act_tcam); + else + clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, + mac_addr->act_tcam); + } else { + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_BMC); + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_ALLMULTI); + } + + /* We have to add a special handler for multicast as the + * BMC may have an all-multi rule already in place. As such + * adding a rule ourselves won't do any good so we will have + * to modify the rules for the ALL MULTI below if the BMC + * already has the rule in place. + */ + act_tcam = &fbd->act_tcam[FBNIC_RPC_ACT_TBL_BMC_ALL_MULTI_OFFSET]; + + /* If we are not enabling the rule just delete it. We will fall + * back to the RSS rules that support the multicast addresses. + */ + if (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi || enable_host) { + if (act_tcam->state == FBNIC_TCAM_S_VALID) + act_tcam->state = FBNIC_TCAM_S_DELETE; + return; + } + + /* Rewrite TCAM rule 23 to handle BMC all-multi traffic */ + act_tcam->dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + FBNIC_RPC_ACT_TBL0_DEST_BMC); + act_tcam->mask.tcam[0] = 0xffff; + + /* MACDA 0 - 3 is reserved for the BMC MAC address */ + act_tcam->value.tcam[1] = + FIELD_PREP(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, + fbd->mac_addr_boundary - 1) | + FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + act_tcam->mask.tcam[1] = 0xffff & + ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX & + ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + + for (j = 2; j < FBNIC_RPC_TCAM_ACT_WORD_LEN; j++) + act_tcam->mask.tcam[j] = 0xffff; + + act_tcam->state = FBNIC_TCAM_S_UPDATE; +} + +void fbnic_bmc_rpc_init(struct fbnic_dev *fbd) +{ + int i = FBNIC_RPC_TCAM_MACDA_BMC_ADDR_IDX; + struct fbnic_act_tcam *act_tcam; + struct fbnic_mac_addr *mac_addr; + int j; + + /* Check if BMC is present */ + if (!fbnic_bmc_present(fbd)) + return; + + /* Fetch BMC MAC addresses from firmware capabilities */ + for (j = 0; j < 4; j++) { + u8 *bmc_mac = fbd->fw_cap.bmc_mac_addr[j]; + + /* Validate BMC MAC addresses */ + if (is_zero_ether_addr(bmc_mac)) + continue; + + if (is_multicast_ether_addr(bmc_mac)) + mac_addr = __fbnic_mc_sync(fbd, bmc_mac); + else + mac_addr = &fbd->mac_addr[i++]; + + if (!mac_addr) { + netdev_err(fbd->netdev, + "No slot for BMC MAC address[%d]\n", j); + continue; + } + + ether_addr_copy(mac_addr->value.addr8, bmc_mac); + eth_zero_addr(mac_addr->mask.addr8); + + set_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + + /* Validate Broadcast is also present, record it and tag it */ + mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX]; + eth_broadcast_addr(mac_addr->value.addr8); + set_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + + /* Rewrite TCAM rule 0 if it isn't present to relocate BMC rules */ + act_tcam = &fbd->act_tcam[FBNIC_RPC_ACT_TBL_BMC_OFFSET]; + act_tcam->dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + FBNIC_RPC_ACT_TBL0_DEST_BMC); + act_tcam->mask.tcam[0] = 0xffff; + + /* MACDA 0 - 3 is reserved for the BMC MAC address + * to account for that we have to mask out the lower 2 bits + * of the macda by performing an &= with 0x1c. + */ + act_tcam->value.tcam[1] = FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + act_tcam->mask.tcam[1] = 0xffff & + ~FIELD_PREP(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, 0x1c) & + ~FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + + for (j = 2; j < FBNIC_RPC_TCAM_ACT_WORD_LEN; j++) + act_tcam->mask.tcam[j] = 0xffff; + + act_tcam->state = FBNIC_TCAM_S_UPDATE; +} + +void fbnic_bmc_rpc_check(struct fbnic_dev *fbd) +{ + int err; + + if (fbd->fw_cap.need_bmc_tcam_reinit) { + fbnic_bmc_rpc_init(fbd); + netif_addr_lock_bh(fbd->netdev); + __fbnic_set_rx_mode(fbd, &fbd->netdev->uc, &fbd->netdev->mc); + netif_addr_unlock_bh(fbd->netdev); + fbd->fw_cap.need_bmc_tcam_reinit = false; + } + + if (fbd->fw_cap.need_bmc_macda_sync) { + err = fbnic_fw_xmit_rpc_macda_sync(fbd); + if (err) + dev_warn(fbd->dev, + "Writing MACDA table to FW failed, err: %d\n", err); + fbd->fw_cap.need_bmc_macda_sync = false; + } +} + +#define FBNIC_ACT1_INIT(_l4, _udp, _ip, _v6) \ + (((_l4) ? FBNIC_RPC_TCAM_ACT1_L4_VALID : 0) | \ + ((_udp) ? FBNIC_RPC_TCAM_ACT1_L4_IS_UDP : 0) | \ + ((_ip) ? FBNIC_RPC_TCAM_ACT1_IP_VALID : 0) | \ + ((_v6) ? FBNIC_RPC_TCAM_ACT1_IP_IS_V6 : 0)) + +#define FBNIC_TSTAMP_MASK(_all, _udp, _ether) \ + (((_all) ? ((1u << FBNIC_NUM_HASH_OPT) - 1) : 0) | \ + ((_udp) ? (1u << FBNIC_UDP6_HASH_OPT) | \ + (1u << FBNIC_UDP4_HASH_OPT) : 0) | \ + ((_ether) ? (1u << FBNIC_ETHER_HASH_OPT) : 0)) + +void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn) +{ + static const u32 act1_value[FBNIC_NUM_HASH_OPT] = { + FBNIC_ACT1_INIT(1, 1, 1, 1), /* UDP6 */ + FBNIC_ACT1_INIT(1, 1, 1, 0), /* UDP4 */ + FBNIC_ACT1_INIT(1, 0, 1, 1), /* TCP6 */ + FBNIC_ACT1_INIT(1, 0, 1, 0), /* TCP4 */ + FBNIC_ACT1_INIT(0, 0, 1, 1), /* IP6 */ + FBNIC_ACT1_INIT(0, 0, 1, 0), /* IP4 */ + 0 /* Ether */ + }; + u32 tstamp_mask = 0; + unsigned int i; + + /* To support scenarios where a BMC is present we must write the + * rules twice, once for the unicast cases, and once again for + * the broadcast/multicast cases as we have to support 2 destinations. + */ + BUILD_BUG_ON(FBNIC_RSS_EN_NUM_UNICAST * 2 != FBNIC_RSS_EN_NUM_ENTRIES); + BUILD_BUG_ON(ARRAY_SIZE(act1_value) != FBNIC_NUM_HASH_OPT); + + /* Set timestamp mask with 1b per flow type */ + if (fbn->hwtstamp_config.rx_filter != HWTSTAMP_FILTER_NONE) { + switch (fbn->hwtstamp_config.rx_filter) { + case HWTSTAMP_FILTER_ALL: + tstamp_mask = FBNIC_TSTAMP_MASK(1, 1, 1); + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + tstamp_mask = FBNIC_TSTAMP_MASK(0, 1, 1); + break; + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + tstamp_mask = FBNIC_TSTAMP_MASK(0, 1, 0); + break; + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + tstamp_mask = FBNIC_TSTAMP_MASK(0, 0, 1); + break; + default: + netdev_warn(fbn->netdev, "Unsupported hwtstamp_rx_filter\n"); + break; + } + } + + /* Program RSS hash enable mask for host in action TCAM/table. */ + for (i = fbnic_bmc_present(fbd) ? 0 : FBNIC_RSS_EN_NUM_UNICAST; + i < FBNIC_RSS_EN_NUM_ENTRIES; i++) { + unsigned int idx = i + FBNIC_RPC_ACT_TBL_RSS_OFFSET; + struct fbnic_act_tcam *act_tcam = &fbd->act_tcam[idx]; + u32 flow_hash, dest, rss_en_mask; + int flow_type, j; + u16 value = 0; + + flow_type = i % FBNIC_RSS_EN_NUM_UNICAST; + flow_hash = fbn->rss_flow_hash[flow_type]; + + /* Set DEST_HOST based on absence of RXH_DISCARD */ + dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + !(RXH_DISCARD & flow_hash) ? + FBNIC_RPC_ACT_TBL0_DEST_HOST : 0); + + if (i >= FBNIC_RSS_EN_NUM_UNICAST && fbnic_bmc_present(fbd)) + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + FBNIC_RPC_ACT_TBL0_DEST_BMC); + + if (!dest) + dest = FBNIC_RPC_ACT_TBL0_DROP; + else if (tstamp_mask & (1u << flow_type)) + dest |= FBNIC_RPC_ACT_TBL0_TS_ENA; + + dest |= FIELD_PREP(FBNIC_RPC_ACT_TBL0_DMA_HINT, + FBNIC_RCD_HDR_AL_DMA_HINT_L4); + + rss_en_mask = fbnic_flow_hash_2_rss_en_mask(fbn, flow_type); + + act_tcam->dest = dest; + act_tcam->rss_en_mask = rss_en_mask; + act_tcam->state = FBNIC_TCAM_S_UPDATE; + + act_tcam->mask.tcam[0] = 0xffff; + + /* We reserve the upper 8 MACDA TCAM entries for host + * unicast. So we set the value to 24, and the mask the + * lower bits so that the lower entries can be used as + * multicast or BMC addresses. + */ + if (i < FBNIC_RSS_EN_NUM_UNICAST) + value = FIELD_PREP(FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX, + fbd->mac_addr_boundary); + value |= FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID; + + flow_type = i % FBNIC_RSS_EN_NUM_UNICAST; + value |= act1_value[flow_type]; + + act_tcam->value.tcam[1] = value; + act_tcam->mask.tcam[1] = ~value; + + for (j = 2; j < FBNIC_RPC_TCAM_ACT_WORD_LEN; j++) + act_tcam->mask.tcam[j] = 0xffff; + + act_tcam->state = FBNIC_TCAM_S_UPDATE; + } +} + +struct fbnic_mac_addr *__fbnic_uc_sync(struct fbnic_dev *fbd, + const unsigned char *addr) +{ + struct fbnic_mac_addr *avail_addr = NULL; + unsigned int i; + + /* Scan from middle of list to bottom, filling bottom up. + * Skip the first entry which is reserved for dev_addr and + * leave the last entry to use for promiscuous filtering. + */ + for (i = fbd->mac_addr_boundary - 1; + i < FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX; i++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (mac_addr->state == FBNIC_TCAM_S_DISABLED) { + avail_addr = mac_addr; + } else if (ether_addr_equal(mac_addr->value.addr8, addr)) { + avail_addr = mac_addr; + break; + } + } + + if (avail_addr && avail_addr->state == FBNIC_TCAM_S_DISABLED) { + ether_addr_copy(avail_addr->value.addr8, addr); + eth_zero_addr(avail_addr->mask.addr8); + avail_addr->state = FBNIC_TCAM_S_ADD; + } + + return avail_addr; +} + +struct fbnic_mac_addr *__fbnic_mc_sync(struct fbnic_dev *fbd, + const unsigned char *addr) +{ + struct fbnic_mac_addr *avail_addr = NULL; + unsigned int i; + + /* Scan from middle of list to top, filling top down. + * Skip over the address reserved for the BMC MAC and + * exclude index 0 as that belongs to the broadcast address + */ + for (i = fbd->mac_addr_boundary; + --i > FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (mac_addr->state == FBNIC_TCAM_S_DISABLED) { + avail_addr = mac_addr; + } else if (ether_addr_equal(mac_addr->value.addr8, addr)) { + avail_addr = mac_addr; + break; + } + } + + /* Scan the BMC addresses to see if it may have already + * reserved the address. + */ + while (--i) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[i]; + + if (!is_zero_ether_addr(mac_addr->mask.addr8)) + continue; + + /* Only move on if we find a match */ + if (!ether_addr_equal(mac_addr->value.addr8, addr)) + continue; + + /* We need to pull this address to the shared area */ + if (avail_addr) { + memcpy(avail_addr, mac_addr, sizeof(*mac_addr)); + mac_addr->state = FBNIC_TCAM_S_DELETE; + avail_addr->state = FBNIC_TCAM_S_ADD; + } + + break; + } + + if (avail_addr && avail_addr->state == FBNIC_TCAM_S_DISABLED) { + ether_addr_copy(avail_addr->value.addr8, addr); + eth_zero_addr(avail_addr->mask.addr8); + avail_addr->state = FBNIC_TCAM_S_ADD; + } + + return avail_addr; +} + +int __fbnic_xc_unsync(struct fbnic_mac_addr *mac_addr, unsigned int tcam_idx) +{ + if (!test_and_clear_bit(tcam_idx, mac_addr->act_tcam)) + return -ENOENT; + + if (bitmap_empty(mac_addr->act_tcam, FBNIC_RPC_TCAM_ACT_NUM_ENTRIES)) + mac_addr->state = FBNIC_TCAM_S_DELETE; + + return 0; +} + +void fbnic_promisc_sync(struct fbnic_dev *fbd, + bool uc_promisc, bool mc_promisc) +{ + struct fbnic_mac_addr *mac_addr; + + /* Populate last TCAM entry with promiscuous entry and 0/1 bit mask */ + mac_addr = &fbd->mac_addr[FBNIC_RPC_TCAM_MACDA_PROMISC_IDX]; + if (uc_promisc) { + if (!is_zero_ether_addr(mac_addr->value.addr8) || + mac_addr->state != FBNIC_TCAM_S_VALID) { + eth_zero_addr(mac_addr->value.addr8); + eth_broadcast_addr(mac_addr->mask.addr8); + clear_bit(FBNIC_MAC_ADDR_T_ALLMULTI, + mac_addr->act_tcam); + set_bit(FBNIC_MAC_ADDR_T_PROMISC, + mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + } else if (mc_promisc && + (!fbnic_bmc_present(fbd) || !fbd->fw_cap.all_multi)) { + /* We have to add a special handler for multicast as the + * BMC may have an all-multi rule already in place. As such + * adding a rule ourselves won't do any good so we will have + * to modify the rules for the ALL MULTI below if the BMC + * already has the rule in place. + */ + if (!is_multicast_ether_addr(mac_addr->value.addr8) || + mac_addr->state != FBNIC_TCAM_S_VALID) { + eth_zero_addr(mac_addr->value.addr8); + eth_broadcast_addr(mac_addr->mask.addr8); + mac_addr->value.addr8[0] ^= 1; + mac_addr->mask.addr8[0] ^= 1; + set_bit(FBNIC_MAC_ADDR_T_ALLMULTI, + mac_addr->act_tcam); + clear_bit(FBNIC_MAC_ADDR_T_PROMISC, + mac_addr->act_tcam); + mac_addr->state = FBNIC_TCAM_S_ADD; + } + } else if (mac_addr->state == FBNIC_TCAM_S_VALID) { + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_ALLMULTI); + __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_PROMISC); + } +} + +void fbnic_sift_macda(struct fbnic_dev *fbd) +{ + int dest, src; + + /* Move BMC only addresses back into BMC region */ + for (dest = FBNIC_RPC_TCAM_MACDA_BMC_ADDR_IDX, + src = FBNIC_RPC_TCAM_MACDA_MULTICAST_IDX; + ++dest < FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX && + src < fbd->mac_addr_boundary;) { + struct fbnic_mac_addr *dest_addr = &fbd->mac_addr[dest]; + + if (dest_addr->state != FBNIC_TCAM_S_DISABLED) + continue; + + while (src < fbd->mac_addr_boundary) { + struct fbnic_mac_addr *src_addr = &fbd->mac_addr[src++]; + + /* Verify BMC bit is set */ + if (!test_bit(FBNIC_MAC_ADDR_T_BMC, src_addr->act_tcam)) + continue; + + /* Verify filter isn't already disabled */ + if (src_addr->state == FBNIC_TCAM_S_DISABLED || + src_addr->state == FBNIC_TCAM_S_DELETE) + continue; + + /* Verify only BMC bit is set */ + if (bitmap_weight(src_addr->act_tcam, + FBNIC_RPC_TCAM_ACT_NUM_ENTRIES) != 1) + continue; + + /* Verify we are not moving wildcard address */ + if (!is_zero_ether_addr(src_addr->mask.addr8)) + continue; + + memcpy(dest_addr, src_addr, sizeof(*src_addr)); + src_addr->state = FBNIC_TCAM_S_DELETE; + dest_addr->state = FBNIC_TCAM_S_ADD; + } + } +} + +static void fbnic_clear_macda_entry(struct fbnic_dev *fbd, unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_RPC_TCAM_MACDA_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_MACDA(idx, i), 0); +} + +static void fbnic_clear_macda(struct fbnic_dev *fbd) +{ + int idx; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + if (mac_addr->state == FBNIC_TCAM_S_DISABLED) + continue; + + if (test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) { + if (fbnic_bmc_present(fbd)) + continue; + dev_warn_once(fbd->dev, + "Found BMC MAC address w/ BMC not present\n"); + } + + fbnic_clear_macda_entry(fbd, idx); + + /* If rule was already destined for deletion just wipe it now */ + if (mac_addr->state == FBNIC_TCAM_S_DELETE) { + memset(mac_addr, 0, sizeof(*mac_addr)); + continue; + } + + /* Change state to update so that we will rewrite + * this tcam the next time fbnic_write_macda is called. + */ + mac_addr->state = FBNIC_TCAM_S_UPDATE; + } +} + +static void fbnic_clear_valid_macda(struct fbnic_dev *fbd) +{ + int idx; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + if (mac_addr->state == FBNIC_TCAM_S_VALID) { + fbnic_clear_macda_entry(fbd, idx); + + mac_addr->state = FBNIC_TCAM_S_UPDATE; + } + } +} + +static void fbnic_write_macda_entry(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_mac_addr *mac_addr) +{ + __be16 *mask, *value; + int i; + + mask = &mac_addr->mask.addr16[FBNIC_RPC_TCAM_MACDA_WORD_LEN - 1]; + value = &mac_addr->value.addr16[FBNIC_RPC_TCAM_MACDA_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_RPC_TCAM_MACDA_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_MACDA(idx, i), + FIELD_PREP(FBNIC_RPC_TCAM_MACDA_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_RPC_TCAM_MACDA_VALUE, ntohs(*value--))); + + wrfl(fbd); + + wr32(fbd, FBNIC_RPC_TCAM_MACDA(idx, i), FBNIC_RPC_TCAM_VALIDATE); +} + +void fbnic_write_macda(struct fbnic_dev *fbd) +{ + int idx, updates = 0; + + for (idx = ARRAY_SIZE(fbd->mac_addr); idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[idx]; + + /* Check if update flag is set else exit. */ + if (!(mac_addr->state & FBNIC_TCAM_S_UPDATE)) + continue; + + /* Record update count */ + updates++; + + /* Clear by writing 0s. */ + if (mac_addr->state == FBNIC_TCAM_S_DELETE) { + /* Invalidate entry and clear addr state info */ + fbnic_clear_macda_entry(fbd, idx); + memset(mac_addr, 0, sizeof(*mac_addr)); + + continue; + } + + fbnic_write_macda_entry(fbd, idx, mac_addr); + + mac_addr->state = FBNIC_TCAM_S_VALID; + } + + /* If reinitializing the BMC TCAM we are doing an initial update */ + if (fbd->fw_cap.need_bmc_tcam_reinit) + updates++; + + /* If needed notify firmware of changes to MACDA TCAM */ + if (updates != 0 && fbnic_bmc_present(fbd)) + fbd->fw_cap.need_bmc_macda_sync = true; +} + +static void fbnic_clear_act_tcam(struct fbnic_dev *fbd, unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_RPC_TCAM_ACT_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_ACT(idx, i), 0); +} + +static void fbnic_clear_tce_tcam_entry(struct fbnic_dev *fbd, unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_TCE_TCAM_WORD_LEN; i++) + wr32(fbd, FBNIC_TCE_RAM_TCAM(idx, i), 0); +} + +static void fbnic_write_tce_tcam_dest(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_mac_addr *mac_addr) +{ + u32 dest = FBNIC_TCE_TCAM_DEST_BMC; + u32 idx2dest_map; + + if (is_multicast_ether_addr(mac_addr->value.addr8)) + dest |= FBNIC_TCE_TCAM_DEST_MAC; + + idx2dest_map = rd32(fbd, FBNIC_TCE_TCAM_IDX2DEST_MAP); + idx2dest_map &= ~(FBNIC_TCE_TCAM_IDX2DEST_MAP_DEST_ID_0 << (4 * idx)); + idx2dest_map |= dest << (4 * idx); + + wr32(fbd, FBNIC_TCE_TCAM_IDX2DEST_MAP, idx2dest_map); +} + +static void fbnic_write_tce_tcam_entry(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_mac_addr *mac_addr) +{ + __be16 *mask, *value; + int i; + + mask = &mac_addr->mask.addr16[FBNIC_TCE_TCAM_WORD_LEN - 1]; + value = &mac_addr->value.addr16[FBNIC_TCE_TCAM_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_TCE_TCAM_WORD_LEN; i++) + wr32(fbd, FBNIC_TCE_RAM_TCAM(idx, i), + FIELD_PREP(FBNIC_TCE_RAM_TCAM_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_TCE_RAM_TCAM_VALUE, ntohs(*value--))); + + wrfl(fbd); + + wr32(fbd, FBNIC_TCE_RAM_TCAM3(idx), FBNIC_TCE_RAM_TCAM3_MCQ_MASK | + FBNIC_TCE_RAM_TCAM3_DEST_MASK | + FBNIC_TCE_RAM_TCAM3_VALIDATE); +} + +static void __fbnic_write_tce_tcam_rev(struct fbnic_dev *fbd) +{ + int tcam_idx = FBNIC_TCE_TCAM_NUM_ENTRIES; + int mac_idx; + + for (mac_idx = ARRAY_SIZE(fbd->mac_addr); mac_idx--;) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[mac_idx]; + + /* Verify BMC bit is set */ + if (!test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) + continue; + + if (!tcam_idx) { + dev_err(fbd->dev, "TCE TCAM overflow\n"); + return; + } + + tcam_idx--; + fbnic_write_tce_tcam_dest(fbd, tcam_idx, mac_addr); + fbnic_write_tce_tcam_entry(fbd, tcam_idx, mac_addr); + } + + while (tcam_idx) + fbnic_clear_tce_tcam_entry(fbd, --tcam_idx); + + fbd->tce_tcam_last = tcam_idx; +} + +static void __fbnic_write_tce_tcam(struct fbnic_dev *fbd) +{ + int tcam_idx = 0; + int mac_idx; + + for (mac_idx = 0; mac_idx < ARRAY_SIZE(fbd->mac_addr); mac_idx++) { + struct fbnic_mac_addr *mac_addr = &fbd->mac_addr[mac_idx]; + + /* Verify BMC bit is set */ + if (!test_bit(FBNIC_MAC_ADDR_T_BMC, mac_addr->act_tcam)) + continue; + + if (tcam_idx == FBNIC_TCE_TCAM_NUM_ENTRIES) { + dev_err(fbd->dev, "TCE TCAM overflow\n"); + return; + } + + fbnic_write_tce_tcam_dest(fbd, tcam_idx, mac_addr); + fbnic_write_tce_tcam_entry(fbd, tcam_idx, mac_addr); + tcam_idx++; + } + + while (tcam_idx < FBNIC_TCE_TCAM_NUM_ENTRIES) + fbnic_clear_tce_tcam_entry(fbd, tcam_idx++); + + fbd->tce_tcam_last = tcam_idx; +} + +void fbnic_write_tce_tcam(struct fbnic_dev *fbd) +{ + if (fbd->tce_tcam_last) + __fbnic_write_tce_tcam_rev(fbd); + else + __fbnic_write_tce_tcam(fbd); +} + +struct fbnic_ip_addr *__fbnic_ip4_sync(struct fbnic_dev *fbd, + struct fbnic_ip_addr *ip_addr, + const struct in_addr *addr, + const struct in_addr *mask) +{ + struct fbnic_ip_addr *avail_addr = NULL; + unsigned int i; + + /* Scan from top of list to bottom, filling bottom up. */ + for (i = 0; i < FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES; i++, ip_addr++) { + struct in6_addr *m = &ip_addr->mask; + + if (ip_addr->state == FBNIC_TCAM_S_DISABLED) { + avail_addr = ip_addr; + continue; + } + + if (ip_addr->version != 4) + continue; + + /* Drop avail_addr if mask is a subset of our current mask, + * This prevents us from inserting a longer prefix behind a + * shorter one. + * + * The mask is stored inverted value so as an example: + * m ffff ffff ffff ffff ffff ffff ffff 0000 0000 + * mask 0000 0000 0000 0000 0000 0000 0000 ffff ffff + * + * "m" and "mask" represent typical IPv4 mask stored in + * the TCAM and those provided by the stack. The code below + * should return a non-zero result if there is a 0 stored + * anywhere in "m" where "mask" has a 0. + */ + if (~m->s6_addr32[3] & ~mask->s_addr) { + avail_addr = NULL; + continue; + } + + /* Check to see if the mask actually contains fewer bits than + * our new mask "m". The XOR below should only result in 0 if + * "m" is masking a bit that we are looking for in our new + * "mask", we eliminated the 0^0 case with the check above. + * + * If it contains fewer bits we need to stop here, otherwise + * we might be adding an unreachable rule. + */ + if (~(m->s6_addr32[3] ^ mask->s_addr)) + break; + + if (ip_addr->value.s6_addr32[3] == addr->s_addr) { + avail_addr = ip_addr; + break; + } + } + + if (avail_addr && avail_addr->state == FBNIC_TCAM_S_DISABLED) { + ipv6_addr_set(&avail_addr->value, 0, 0, 0, addr->s_addr); + ipv6_addr_set(&avail_addr->mask, htonl(~0), htonl(~0), + htonl(~0), ~mask->s_addr); + avail_addr->version = 4; + + avail_addr->state = FBNIC_TCAM_S_ADD; + } + + return avail_addr; +} + +struct fbnic_ip_addr *__fbnic_ip6_sync(struct fbnic_dev *fbd, + struct fbnic_ip_addr *ip_addr, + const struct in6_addr *addr, + const struct in6_addr *mask) +{ + struct fbnic_ip_addr *avail_addr = NULL; + unsigned int i; + + ip_addr = &ip_addr[FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES - 1]; + + /* Scan from bottom of list to top, filling top down. */ + for (i = FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES; i--; ip_addr--) { + struct in6_addr *m = &ip_addr->mask; + + if (ip_addr->state == FBNIC_TCAM_S_DISABLED) { + avail_addr = ip_addr; + continue; + } + + if (ip_addr->version != 6) + continue; + + /* Drop avail_addr if mask is a superset of our current mask. + * This prevents us from inserting a longer prefix behind a + * shorter one. + * + * The mask is stored inverted value so as an example: + * m 0000 0000 0000 0000 0000 0000 0000 0000 0000 + * mask ffff ffff ffff ffff ffff ffff ffff ffff ffff + * + * "m" and "mask" represent typical IPv6 mask stored in + * the TCAM and those provided by the stack. The code below + * should return a non-zero result which will cause us + * to drop the avail_addr value that might be cached + * to prevent us from dropping a v6 address behind it. + */ + if ((m->s6_addr32[0] & mask->s6_addr32[0]) | + (m->s6_addr32[1] & mask->s6_addr32[1]) | + (m->s6_addr32[2] & mask->s6_addr32[2]) | + (m->s6_addr32[3] & mask->s6_addr32[3])) { + avail_addr = NULL; + continue; + } + + /* The previous test eliminated any overlap between the + * two values so now we need to check for gaps. + * + * If the mask is equal to our current mask then it should + * result with m ^ mask = ffff ffff, if however the value + * stored in m is bigger then we should see a 0 appear + * somewhere in the mask. + */ + if (~(m->s6_addr32[0] ^ mask->s6_addr32[0]) | + ~(m->s6_addr32[1] ^ mask->s6_addr32[1]) | + ~(m->s6_addr32[2] ^ mask->s6_addr32[2]) | + ~(m->s6_addr32[3] ^ mask->s6_addr32[3])) + break; + + if (ipv6_addr_cmp(&ip_addr->value, addr)) + continue; + + avail_addr = ip_addr; + break; + } + + if (avail_addr && avail_addr->state == FBNIC_TCAM_S_DISABLED) { + memcpy(&avail_addr->value, addr, sizeof(*addr)); + ipv6_addr_set(&avail_addr->mask, + ~mask->s6_addr32[0], ~mask->s6_addr32[1], + ~mask->s6_addr32[2], ~mask->s6_addr32[3]); + avail_addr->version = 6; + + avail_addr->state = FBNIC_TCAM_S_ADD; + } + + return avail_addr; +} + +int __fbnic_ip_unsync(struct fbnic_ip_addr *ip_addr, unsigned int tcam_idx) +{ + if (!test_and_clear_bit(tcam_idx, ip_addr->act_tcam)) + return -ENOENT; + + if (bitmap_empty(ip_addr->act_tcam, FBNIC_RPC_TCAM_ACT_NUM_ENTRIES)) + ip_addr->state = FBNIC_TCAM_S_DELETE; + + return 0; +} + +static void fbnic_clear_ip_src_entry(struct fbnic_dev *fbd, unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_IPSRC(idx, i), 0); +} + +static void fbnic_clear_ip_dst_entry(struct fbnic_dev *fbd, unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_IPDST(idx, i), 0); +} + +static void fbnic_clear_ip_outer_src_entry(struct fbnic_dev *fbd, + unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_OUTER_IPSRC(idx, i), 0); +} + +static void fbnic_clear_ip_outer_dst_entry(struct fbnic_dev *fbd, + unsigned int idx) +{ + int i; + + /* Invalidate entry and clear addr state info */ + for (i = 0; i <= FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_OUTER_IPDST(idx, i), 0); +} + +static void fbnic_write_ip_src_entry(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_ip_addr *ip_addr) +{ + __be16 *mask, *value; + int i; + + mask = &ip_addr->mask.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + value = &ip_addr->value.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_IPSRC(idx, i), + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_VALUE, ntohs(*value--))); + wrfl(fbd); + + /* Bit 129 is used to flag for v4/v6 */ + wr32(fbd, FBNIC_RPC_TCAM_IPSRC(idx, i), + (ip_addr->version == 6) | FBNIC_RPC_TCAM_VALIDATE); +} + +static void fbnic_write_ip_dst_entry(struct fbnic_dev *fbd, unsigned int idx, + struct fbnic_ip_addr *ip_addr) +{ + __be16 *mask, *value; + int i; + + mask = &ip_addr->mask.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + value = &ip_addr->value.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_IPDST(idx, i), + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_VALUE, ntohs(*value--))); + wrfl(fbd); + + /* Bit 129 is used to flag for v4/v6 */ + wr32(fbd, FBNIC_RPC_TCAM_IPDST(idx, i), + (ip_addr->version == 6) | FBNIC_RPC_TCAM_VALIDATE); +} + +static void fbnic_write_ip_outer_src_entry(struct fbnic_dev *fbd, + unsigned int idx, + struct fbnic_ip_addr *ip_addr) +{ + __be16 *mask, *value; + int i; + + mask = &ip_addr->mask.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + value = &ip_addr->value.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_OUTER_IPSRC(idx, i), + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_VALUE, ntohs(*value--))); + wrfl(fbd); + + wr32(fbd, FBNIC_RPC_TCAM_OUTER_IPSRC(idx, i), FBNIC_RPC_TCAM_VALIDATE); +} + +static void fbnic_write_ip_outer_dst_entry(struct fbnic_dev *fbd, + unsigned int idx, + struct fbnic_ip_addr *ip_addr) +{ + __be16 *mask, *value; + int i; + + mask = &ip_addr->mask.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + value = &ip_addr->value.s6_addr16[FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN - 1]; + + for (i = 0; i < FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_OUTER_IPDST(idx, i), + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_MASK, ntohs(*mask--)) | + FIELD_PREP(FBNIC_RPC_TCAM_IP_ADDR_VALUE, ntohs(*value--))); + wrfl(fbd); + + wr32(fbd, FBNIC_RPC_TCAM_OUTER_IPDST(idx, i), FBNIC_RPC_TCAM_VALIDATE); +} + +void fbnic_write_ip_addr(struct fbnic_dev *fbd) +{ + int idx; + + for (idx = ARRAY_SIZE(fbd->ip_src); idx--;) { + struct fbnic_ip_addr *ip_addr = &fbd->ip_src[idx]; + + /* Check if update flag is set else skip. */ + if (!(ip_addr->state & FBNIC_TCAM_S_UPDATE)) + continue; + + /* Clear by writing 0s. */ + if (ip_addr->state == FBNIC_TCAM_S_DELETE) { + /* Invalidate entry and clear addr state info */ + fbnic_clear_ip_src_entry(fbd, idx); + memset(ip_addr, 0, sizeof(*ip_addr)); + + continue; + } + + fbnic_write_ip_src_entry(fbd, idx, ip_addr); + + ip_addr->state = FBNIC_TCAM_S_VALID; + } + + /* Repeat process for other IP TCAMs */ + for (idx = ARRAY_SIZE(fbd->ip_dst); idx--;) { + struct fbnic_ip_addr *ip_addr = &fbd->ip_dst[idx]; + + if (!(ip_addr->state & FBNIC_TCAM_S_UPDATE)) + continue; + + if (ip_addr->state == FBNIC_TCAM_S_DELETE) { + fbnic_clear_ip_dst_entry(fbd, idx); + memset(ip_addr, 0, sizeof(*ip_addr)); + + continue; + } + + fbnic_write_ip_dst_entry(fbd, idx, ip_addr); + + ip_addr->state = FBNIC_TCAM_S_VALID; + } + + for (idx = ARRAY_SIZE(fbd->ipo_src); idx--;) { + struct fbnic_ip_addr *ip_addr = &fbd->ipo_src[idx]; + + if (!(ip_addr->state & FBNIC_TCAM_S_UPDATE)) + continue; + + if (ip_addr->state == FBNIC_TCAM_S_DELETE) { + fbnic_clear_ip_outer_src_entry(fbd, idx); + memset(ip_addr, 0, sizeof(*ip_addr)); + + continue; + } + + fbnic_write_ip_outer_src_entry(fbd, idx, ip_addr); + + ip_addr->state = FBNIC_TCAM_S_VALID; + } + + for (idx = ARRAY_SIZE(fbd->ipo_dst); idx--;) { + struct fbnic_ip_addr *ip_addr = &fbd->ipo_dst[idx]; + + if (!(ip_addr->state & FBNIC_TCAM_S_UPDATE)) + continue; + + if (ip_addr->state == FBNIC_TCAM_S_DELETE) { + fbnic_clear_ip_outer_dst_entry(fbd, idx); + memset(ip_addr, 0, sizeof(*ip_addr)); + + continue; + } + + fbnic_write_ip_outer_dst_entry(fbd, idx, ip_addr); + + ip_addr->state = FBNIC_TCAM_S_VALID; + } +} + +static void fbnic_clear_valid_act_tcam(struct fbnic_dev *fbd) +{ + int i = FBNIC_RPC_TCAM_ACT_NUM_ENTRIES - 1; + struct fbnic_act_tcam *act_tcam; + + /* Work from the bottom up deleting all other rules from hardware */ + do { + act_tcam = &fbd->act_tcam[i]; + + if (act_tcam->state != FBNIC_TCAM_S_VALID) + continue; + + fbnic_clear_act_tcam(fbd, i); + act_tcam->state = FBNIC_TCAM_S_UPDATE; + } while (i--); +} + +void fbnic_clear_rules(struct fbnic_dev *fbd) +{ + /* Clear MAC rules */ + fbnic_clear_macda(fbd); + + /* If BMC is present we need to preserve the last rule which + * will be used to route traffic to the BMC if it is received. + * + * At this point it should be the only MAC address in the MACDA + * so any unicast or multicast traffic received should be routed + * to it. So leave the last rule in place. + * + * It will be rewritten to add the host again when we bring + * the interface back up. + */ + if (fbnic_bmc_present(fbd)) { + u32 dest = FIELD_PREP(FBNIC_RPC_ACT_TBL0_DEST_MASK, + FBNIC_RPC_ACT_TBL0_DEST_BMC); + int i = FBNIC_RPC_TCAM_ACT_NUM_ENTRIES - 1; + struct fbnic_act_tcam *act_tcam; + + act_tcam = &fbd->act_tcam[i]; + + if (act_tcam->state == FBNIC_TCAM_S_VALID && + (act_tcam->dest & dest)) { + wr32(fbd, FBNIC_RPC_ACT_TBL0(i), dest); + wr32(fbd, FBNIC_RPC_ACT_TBL1(i), 0); + + act_tcam->state = FBNIC_TCAM_S_UPDATE; + } + } + + fbnic_clear_valid_act_tcam(fbd); +} + +static void fbnic_delete_act_tcam(struct fbnic_dev *fbd, unsigned int idx) +{ + fbnic_clear_act_tcam(fbd, idx); + memset(&fbd->act_tcam[idx], 0, sizeof(struct fbnic_act_tcam)); +} + +static void fbnic_update_act_tcam(struct fbnic_dev *fbd, unsigned int idx) +{ + struct fbnic_act_tcam *act_tcam = &fbd->act_tcam[idx]; + int i; + + /* Update entry by writing the destination and RSS mask */ + wr32(fbd, FBNIC_RPC_ACT_TBL0(idx), act_tcam->dest); + wr32(fbd, FBNIC_RPC_ACT_TBL1(idx), act_tcam->rss_en_mask); + + /* Write new TCAM rule to hardware */ + for (i = 0; i < FBNIC_RPC_TCAM_ACT_WORD_LEN; i++) + wr32(fbd, FBNIC_RPC_TCAM_ACT(idx, i), + FIELD_PREP(FBNIC_RPC_TCAM_ACT_MASK, + act_tcam->mask.tcam[i]) | + FIELD_PREP(FBNIC_RPC_TCAM_ACT_VALUE, + act_tcam->value.tcam[i])); + + wrfl(fbd); + + wr32(fbd, FBNIC_RPC_TCAM_ACT(idx, i), FBNIC_RPC_TCAM_VALIDATE); + act_tcam->state = FBNIC_TCAM_S_VALID; +} + +void fbnic_write_rules(struct fbnic_dev *fbd) +{ + int i; + + /* Flush any pending action table rules */ + for (i = 0; i < FBNIC_RPC_ACT_TBL_NUM_ENTRIES; i++) { + struct fbnic_act_tcam *act_tcam = &fbd->act_tcam[i]; + + /* Check if update flag is set else exit. */ + if (!(act_tcam->state & FBNIC_TCAM_S_UPDATE)) + continue; + + if (act_tcam->state == FBNIC_TCAM_S_DELETE) + fbnic_delete_act_tcam(fbd, i); + else + fbnic_update_act_tcam(fbd, i); + } +} + +void fbnic_rpc_reset_valid_entries(struct fbnic_dev *fbd) +{ + fbnic_clear_valid_act_tcam(fbd); + fbnic_clear_valid_macda(fbd); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h new file mode 100644 index 000000000..3d4925b2a --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_rpc.h @@ -0,0 +1,232 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_RPC_H_ +#define _FBNIC_RPC_H_ + +#include <uapi/linux/in6.h> +#include <linux/bitfield.h> + +struct in_addr; + +/* The TCAM state definitions follow an expected ordering. + * They start out disabled, then move through the following states: + * Disabled 0 -> Add 2 + * Add 2 -> Valid 1 + * + * Valid 1 -> Add/Update 2 + * Add 2 -> Valid 1 + * + * Valid 1 -> Delete 3 + * Delete 3 -> Disabled 0 + */ +enum { + FBNIC_TCAM_S_DISABLED = 0, + FBNIC_TCAM_S_VALID = 1, + FBNIC_TCAM_S_ADD = 2, + FBNIC_TCAM_S_UPDATE = FBNIC_TCAM_S_ADD, + FBNIC_TCAM_S_DELETE = 3, +}; + +/* 32 MAC Destination Address TCAM Entries + * 4 registers DA[1:0], DA[3:2], DA[5:4], Validate + */ +#define FBNIC_RPC_TCAM_MACDA_WORD_LEN 3 +#define FBNIC_RPC_TCAM_MACDA_NUM_ENTRIES 32 + +/* 8 IPSRC and IPDST TCAM Entries each + * 8 registers, Validate each + */ +#define FBNIC_RPC_TCAM_IP_ADDR_WORD_LEN 8 +#define FBNIC_RPC_TCAM_IP_ADDR_NUM_ENTRIES 8 + +#define FBNIC_RPC_TCAM_ACT_WORD_LEN 11 +#define FBNIC_RPC_TCAM_ACT_NUM_ENTRIES 64 + +#define FBNIC_TCE_TCAM_WORD_LEN 3 +#define FBNIC_TCE_TCAM_NUM_ENTRIES 8 + +struct fbnic_mac_addr { + union { + unsigned char addr8[ETH_ALEN]; + __be16 addr16[FBNIC_RPC_TCAM_MACDA_WORD_LEN]; + } mask, value; + unsigned char state; + DECLARE_BITMAP(act_tcam, FBNIC_RPC_TCAM_ACT_NUM_ENTRIES); +}; + +struct fbnic_ip_addr { + struct in6_addr mask, value; + unsigned char version; + unsigned char state; + DECLARE_BITMAP(act_tcam, FBNIC_RPC_TCAM_ACT_NUM_ENTRIES); +}; + +struct fbnic_act_tcam { + struct { + u16 tcam[FBNIC_RPC_TCAM_ACT_WORD_LEN]; + } mask, value; + unsigned char state; + u16 rss_en_mask; + u32 dest; +}; + +enum { + FBNIC_RSS_EN_HOST_UDP6, + FBNIC_RSS_EN_HOST_UDP4, + FBNIC_RSS_EN_HOST_TCP6, + FBNIC_RSS_EN_HOST_TCP4, + FBNIC_RSS_EN_HOST_IP6, + FBNIC_RSS_EN_HOST_IP4, + FBNIC_RSS_EN_HOST_ETHER, + FBNIC_RSS_EN_XCAST_UDP6, +#define FBNIC_RSS_EN_NUM_UNICAST FBNIC_RSS_EN_XCAST_UDP6 + FBNIC_RSS_EN_XCAST_UDP4, + FBNIC_RSS_EN_XCAST_TCP6, + FBNIC_RSS_EN_XCAST_TCP4, + FBNIC_RSS_EN_XCAST_IP6, + FBNIC_RSS_EN_XCAST_IP4, + FBNIC_RSS_EN_XCAST_ETHER, + FBNIC_RSS_EN_NUM_ENTRIES +}; + +/* Reserve the first 2 entries for the use by the BMC so that we can + * avoid allowing rules to get in the way of BMC unicast traffic. + */ +#define FBNIC_RPC_ACT_TBL_BMC_OFFSET 0 +#define FBNIC_RPC_ACT_TBL_BMC_ALL_MULTI_OFFSET 1 + +/* This should leave us with 48 total entries in the TCAM that can be used + * for NFC after also deducting the 14 needed for RSS table programming. + */ +#define FBNIC_RPC_ACT_TBL_NFC_OFFSET 2 + +/* We reserve the last 14 entries for RSS rules on the host. The BMC + * unicast rule will need to be populated above these and is expected to + * use MACDA TCAM entry 23 to store the BMC MAC address. + */ +#define FBNIC_RPC_ACT_TBL_RSS_OFFSET \ + (FBNIC_RPC_ACT_TBL_NUM_ENTRIES - FBNIC_RSS_EN_NUM_ENTRIES) + +#define FBNIC_RPC_ACT_TBL_NFC_ENTRIES \ + (FBNIC_RPC_ACT_TBL_RSS_OFFSET - FBNIC_RPC_ACT_TBL_NFC_OFFSET) + +/* Flags used to identify the owner for this MAC filter. Note that any + * flags set for Broadcast thru Promisc indicate that the rule belongs + * to the RSS filters for the host. + */ +enum { + FBNIC_MAC_ADDR_T_BMC = 0, + FBNIC_MAC_ADDR_T_BROADCAST = FBNIC_RPC_ACT_TBL_RSS_OFFSET, +#define FBNIC_MAC_ADDR_T_HOST_START FBNIC_MAC_ADDR_T_BROADCAST + FBNIC_MAC_ADDR_T_MULTICAST, + FBNIC_MAC_ADDR_T_UNICAST, + FBNIC_MAC_ADDR_T_ALLMULTI, /* BROADCAST ... MULTICAST*/ + FBNIC_MAC_ADDR_T_PROMISC, /* BROADCAST ... UNICAST */ + FBNIC_MAC_ADDR_T_HOST_LAST +}; + +#define FBNIC_MAC_ADDR_T_HOST_LEN \ + (FBNIC_MAC_ADDR_T_HOST_LAST - FBNIC_MAC_ADDR_T_HOST_START) + +#define FBNIC_RPC_TCAM_ACT0_IPSRC_IDX CSR_GENMASK(2, 0) +#define FBNIC_RPC_TCAM_ACT0_IPSRC_VALID CSR_BIT(3) +#define FBNIC_RPC_TCAM_ACT0_IPDST_IDX CSR_GENMASK(6, 4) +#define FBNIC_RPC_TCAM_ACT0_IPDST_VALID CSR_BIT(7) +#define FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_IDX CSR_GENMASK(10, 8) +#define FBNIC_RPC_TCAM_ACT0_OUTER_IPSRC_VALID CSR_BIT(11) +#define FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_IDX CSR_GENMASK(14, 12) +#define FBNIC_RPC_TCAM_ACT0_OUTER_IPDST_VALID CSR_BIT(15) + +#define FBNIC_RPC_TCAM_ACT1_L2_MACDA_IDX CSR_GENMASK(9, 5) +#define FBNIC_RPC_TCAM_ACT1_L2_MACDA_VALID CSR_BIT(10) +#define FBNIC_RPC_TCAM_ACT1_IP_IS_V6 CSR_BIT(11) +#define FBNIC_RPC_TCAM_ACT1_IP_VALID CSR_BIT(12) +#define FBNIC_RPC_TCAM_ACT1_OUTER_IP_VALID CSR_BIT(13) +#define FBNIC_RPC_TCAM_ACT1_L4_IS_UDP CSR_BIT(14) +#define FBNIC_RPC_TCAM_ACT1_L4_VALID CSR_BIT(15) + +/* TCAM 0 - 3 reserved for BMC MAC addresses */ +#define FBNIC_RPC_TCAM_MACDA_BMC_ADDR_IDX 0 +/* TCAM 4 reserved for broadcast MAC address */ +#define FBNIC_RPC_TCAM_MACDA_BROADCAST_IDX 4 +/* TCAMs 5 - 30 will be used for multicast and unicast addresses. The + * boundary between the two can be variable it is currently set to 24 + * on which the unicast addresses start. The general idea is that we will + * always go top-down with unicast, and bottom-up with multicast so that + * there should be free-space in the middle between the two. + * + * The entry at MADCA_DEFAULT_BOUNDARY is a special case as it can be used + * for the ALL MULTI address if the list is full, or the BMC has requested + * it. + */ +#define FBNIC_RPC_TCAM_MACDA_MULTICAST_IDX 5 +#define FBNIC_RPC_TCAM_MACDA_DEFAULT_BOUNDARY 24 +#define FBNIC_RPC_TCAM_MACDA_HOST_ADDR_IDX 30 +/* Reserved for use to record Multicast promisc, or Promiscuous */ +#define FBNIC_RPC_TCAM_MACDA_PROMISC_IDX 31 + +enum { + FBNIC_UDP6_HASH_OPT, + FBNIC_UDP4_HASH_OPT, + FBNIC_TCP6_HASH_OPT, + FBNIC_TCP4_HASH_OPT, +#define FBNIC_L4_HASH_OPT FBNIC_TCP4_HASH_OPT + FBNIC_IPV6_HASH_OPT, + FBNIC_IPV4_HASH_OPT, +#define FBNIC_IP_HASH_OPT FBNIC_IPV4_HASH_OPT + FBNIC_ETHER_HASH_OPT, + FBNIC_NUM_HASH_OPT, +}; + +struct fbnic_dev; +struct fbnic_net; + +void fbnic_bmc_rpc_init(struct fbnic_dev *fbd); +void fbnic_bmc_rpc_all_multi_config(struct fbnic_dev *fbd, bool enable_host); +void fbnic_bmc_rpc_check(struct fbnic_dev *fbd); + +void fbnic_reset_indir_tbl(struct fbnic_net *fbn); +void fbnic_rss_key_fill(u32 *buffer); +void fbnic_rss_init_en_mask(struct fbnic_net *fbn); +void fbnic_rss_disable_hw(struct fbnic_dev *fbd); +void fbnic_rss_reinit_hw(struct fbnic_dev *fbd, struct fbnic_net *fbn); +void fbnic_rss_reinit(struct fbnic_dev *fbd, struct fbnic_net *fbn); +u16 fbnic_flow_hash_2_rss_en_mask(struct fbnic_net *fbn, int flow_type); + +int __fbnic_xc_unsync(struct fbnic_mac_addr *mac_addr, unsigned int tcam_idx); +struct fbnic_mac_addr *__fbnic_uc_sync(struct fbnic_dev *fbd, + const unsigned char *addr); +struct fbnic_mac_addr *__fbnic_mc_sync(struct fbnic_dev *fbd, + const unsigned char *addr); +void fbnic_sift_macda(struct fbnic_dev *fbd); +void fbnic_write_macda(struct fbnic_dev *fbd); + +void fbnic_promisc_sync(struct fbnic_dev *fbd, + bool uc_promisc, bool mc_promisc); + +struct fbnic_ip_addr *__fbnic_ip4_sync(struct fbnic_dev *fbd, + struct fbnic_ip_addr *ip_addr, + const struct in_addr *addr, + const struct in_addr *mask); +struct fbnic_ip_addr *__fbnic_ip6_sync(struct fbnic_dev *fbd, + struct fbnic_ip_addr *ip_addr, + const struct in6_addr *addr, + const struct in6_addr *mask); +int __fbnic_ip_unsync(struct fbnic_ip_addr *ip_addr, unsigned int tcam_idx); +void fbnic_write_ip_addr(struct fbnic_dev *fbd); + +static inline int __fbnic_uc_unsync(struct fbnic_mac_addr *mac_addr) +{ + return __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_UNICAST); +} + +static inline int __fbnic_mc_unsync(struct fbnic_mac_addr *mac_addr) +{ + return __fbnic_xc_unsync(mac_addr, FBNIC_MAC_ADDR_T_MULTICAST); +} + +void fbnic_clear_rules(struct fbnic_dev *fbd); +void fbnic_write_rules(struct fbnic_dev *fbd); +void fbnic_write_tce_tcam(struct fbnic_dev *fbd); +#endif /* _FBNIC_RPC_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_time.c b/drivers/net/ethernet/meta/fbnic/fbnic_time.c new file mode 100644 index 000000000..db7748189 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_time.c @@ -0,0 +1,303 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/jiffies.h> +#include <linux/limits.h> +#include <linux/ptp_clock_kernel.h> +#include <linux/timer.h> + +#include "fbnic.h" +#include "fbnic_csr.h" +#include "fbnic_netdev.h" + +/* FBNIC timing & PTP implementation + * Datapath uses truncated 40b timestamps for scheduling and event reporting. + * We need to promote those to full 64b, hence we periodically cache the top + * 32bit of the HW time counter. Since this makes our time reporting non-atomic + * we leave the HW clock free running and adjust time offsets in SW as needed. + * Time offset is 64bit - we need a seq counter for 32bit machines. + * Time offset and the cache of top bits are independent so we don't need + * a coherent snapshot of both - READ_ONCE()/WRITE_ONCE() + writer side lock + * are enough. + */ + +/* Period of refresh of top bits of timestamp, give ourselves a 8x margin. + * This should translate to once a minute. + * The use of nsecs_to_jiffies() should be safe for a <=40b nsec value. + */ +#define FBNIC_TS_HIGH_REFRESH_JIF nsecs_to_jiffies((1ULL << 40) / 16) + +static struct fbnic_dev *fbnic_from_ptp_info(struct ptp_clock_info *ptp) +{ + return container_of(ptp, struct fbnic_dev, ptp_info); +} + +/* This function is "slow" because we could try guessing which high part + * is correct based on low instead of re-reading, and skip reading @hi + * twice altogether if @lo is far enough from 0. + */ +static u64 __fbnic_time_get_slow(struct fbnic_dev *fbd) +{ + u32 hi, lo; + + lockdep_assert_held(&fbd->time_lock); + + do { + hi = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI); + lo = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_LO); + } while (hi != fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI)); + + return (u64)hi << 32 | lo; +} + +static void __fbnic_time_set_addend(struct fbnic_dev *fbd, u64 addend) +{ + lockdep_assert_held(&fbd->time_lock); + + fbnic_wr32(fbd, FBNIC_PTP_ADD_VAL_NS, + FIELD_PREP(FBNIC_PTP_ADD_VAL_NS_MASK, addend >> 32)); + fbnic_wr32(fbd, FBNIC_PTP_ADD_VAL_SUBNS, (u32)addend); +} + +static void fbnic_ptp_fresh_check(struct fbnic_dev *fbd) +{ + if (time_is_after_jiffies(fbd->last_read + + FBNIC_TS_HIGH_REFRESH_JIF * 3 / 2)) + return; + + dev_warn(fbd->dev, "NIC timestamp refresh stall, delayed by %lu sec\n", + (jiffies - fbd->last_read - FBNIC_TS_HIGH_REFRESH_JIF) / HZ); +} + +static void fbnic_ptp_refresh_time(struct fbnic_dev *fbd, struct fbnic_net *fbn) +{ + unsigned long flags; + u32 hi; + + spin_lock_irqsave(&fbd->time_lock, flags); + hi = fbnic_rd32(fbn->fbd, FBNIC_PTP_CTR_VAL_HI); + if (!fbnic_present(fbd)) + goto out; /* Don't bother handling, reset is pending */ + /* Let's keep high cached value a bit lower to avoid race with + * incoming timestamps. The logic in fbnic_ts40_to_ns() will + * take care of overflow in this case. It will make cached time + * ~1 minute lower and incoming timestamp will always be later + * then cached time. + */ + WRITE_ONCE(fbn->time_high, hi - 16); + fbd->last_read = jiffies; + out: + spin_unlock_irqrestore(&fbd->time_lock, flags); +} + +static long fbnic_ptp_do_aux_work(struct ptp_clock_info *ptp) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + + fbn = netdev_priv(fbd->netdev); + + fbnic_ptp_fresh_check(fbd); + fbnic_ptp_refresh_time(fbd, fbn); + + return FBNIC_TS_HIGH_REFRESH_JIF; +} + +static int fbnic_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + u64 addend, dclk_period; + unsigned long flags; + + /* d_clock is 600 MHz; which in Q16.32 fixed point ns is: */ + dclk_period = (((u64)1000000000) << 32) / FBNIC_CLOCK_FREQ; + addend = adjust_by_scaled_ppm(dclk_period, scaled_ppm); + + spin_lock_irqsave(&fbd->time_lock, flags); + __fbnic_time_set_addend(fbd, addend); + fbnic_wr32(fbd, FBNIC_PTP_ADJUST, FBNIC_PTP_ADJUST_ADDEND_SET); + + /* Flush, make sure FBNIC_PTP_ADD_VAL_* is stable for at least 4 clks */ + fbnic_rd32(fbd, FBNIC_PTP_SPARE); + spin_unlock_irqrestore(&fbd->time_lock, flags); + + return fbnic_present(fbd) ? 0 : -EIO; +} + +static int fbnic_ptp_adjtime(struct ptp_clock_info *ptp, s64 delta) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + unsigned long flags; + + fbn = netdev_priv(fbd->netdev); + + spin_lock_irqsave(&fbd->time_lock, flags); + u64_stats_update_begin(&fbn->time_seq); + WRITE_ONCE(fbn->time_offset, READ_ONCE(fbn->time_offset) + delta); + u64_stats_update_end(&fbn->time_seq); + spin_unlock_irqrestore(&fbd->time_lock, flags); + + return 0; +} + +static int +fbnic_ptp_gettimex64(struct ptp_clock_info *ptp, struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + unsigned long flags; + u64 time_ns; + u32 hi, lo; + + fbn = netdev_priv(fbd->netdev); + + spin_lock_irqsave(&fbd->time_lock, flags); + + do { + hi = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI); + ptp_read_system_prets(sts); + lo = fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_LO); + ptp_read_system_postts(sts); + /* Similarly to comment above __fbnic_time_get_slow() + * - this can be optimized if needed. + */ + } while (hi != fbnic_rd32(fbd, FBNIC_PTP_CTR_VAL_HI)); + + time_ns = ((u64)hi << 32 | lo) + fbn->time_offset; + spin_unlock_irqrestore(&fbd->time_lock, flags); + + if (!fbnic_present(fbd)) + return -EIO; + + *ts = ns_to_timespec64(time_ns); + + return 0; +} + +static int +fbnic_ptp_settime64(struct ptp_clock_info *ptp, const struct timespec64 *ts) +{ + struct fbnic_dev *fbd = fbnic_from_ptp_info(ptp); + struct fbnic_net *fbn; + unsigned long flags; + u64 dev_ns, host_ns; + int ret; + + fbn = netdev_priv(fbd->netdev); + + host_ns = timespec64_to_ns(ts); + + spin_lock_irqsave(&fbd->time_lock, flags); + + dev_ns = __fbnic_time_get_slow(fbd); + + if (fbnic_present(fbd)) { + u64_stats_update_begin(&fbn->time_seq); + WRITE_ONCE(fbn->time_offset, host_ns - dev_ns); + u64_stats_update_end(&fbn->time_seq); + ret = 0; + } else { + ret = -EIO; + } + spin_unlock_irqrestore(&fbd->time_lock, flags); + + return ret; +} + +static const struct ptp_clock_info fbnic_ptp_info = { + .owner = THIS_MODULE, + /* 1,000,000,000 - 1 PPB to ensure increment is positive + * after max negative adjustment. + */ + .max_adj = 999999999, + .do_aux_work = fbnic_ptp_do_aux_work, + .adjfine = fbnic_ptp_adjfine, + .adjtime = fbnic_ptp_adjtime, + .gettimex64 = fbnic_ptp_gettimex64, + .settime64 = fbnic_ptp_settime64, +}; + +static void fbnic_ptp_reset(struct fbnic_dev *fbd) +{ + struct fbnic_net *fbn = netdev_priv(fbd->netdev); + u64 dclk_period; + + fbnic_wr32(fbd, FBNIC_PTP_CTRL, + FBNIC_PTP_CTRL_EN | + FIELD_PREP(FBNIC_PTP_CTRL_TICK_IVAL, 1)); + + /* d_clock is 600 MHz; which in Q16.32 fixed point ns is: */ + dclk_period = (((u64)1000000000) << 32) / FBNIC_CLOCK_FREQ; + + __fbnic_time_set_addend(fbd, dclk_period); + + fbnic_wr32(fbd, FBNIC_PTP_INIT_HI, 0); + fbnic_wr32(fbd, FBNIC_PTP_INIT_LO, 0); + + fbnic_wr32(fbd, FBNIC_PTP_ADJUST, FBNIC_PTP_ADJUST_INIT); + + fbnic_wr32(fbd, FBNIC_PTP_CTRL, + FBNIC_PTP_CTRL_EN | + FBNIC_PTP_CTRL_TQS_OUT_EN | + FIELD_PREP(FBNIC_PTP_CTRL_MAC_OUT_IVAL, 3) | + FIELD_PREP(FBNIC_PTP_CTRL_TICK_IVAL, 1)); + + fbnic_rd32(fbd, FBNIC_PTP_SPARE); + + fbn->time_offset = 0; + fbn->time_high = 0; +} + +void fbnic_time_init(struct fbnic_net *fbn) +{ + /* This is not really a statistic, but the locking primitive fits + * our usecase perfectly, we need an atomic 8 bytes READ_ONCE() / + * WRITE_ONCE() behavior. + */ + u64_stats_init(&fbn->time_seq); +} + +int fbnic_time_start(struct fbnic_net *fbn) +{ + fbnic_ptp_refresh_time(fbn->fbd, fbn); + /* Assume that fbnic_ptp_do_aux_work() will never be called if not + * scheduled here + */ + return ptp_schedule_worker(fbn->fbd->ptp, FBNIC_TS_HIGH_REFRESH_JIF); +} + +void fbnic_time_stop(struct fbnic_net *fbn) +{ + ptp_cancel_worker_sync(fbn->fbd->ptp); + fbnic_ptp_fresh_check(fbn->fbd); +} + +int fbnic_ptp_setup(struct fbnic_dev *fbd) +{ + struct device *dev = fbd->dev; + unsigned long flags; + + spin_lock_init(&fbd->time_lock); + + spin_lock_irqsave(&fbd->time_lock, flags); /* Appease lockdep */ + fbnic_ptp_reset(fbd); + spin_unlock_irqrestore(&fbd->time_lock, flags); + + memcpy(&fbd->ptp_info, &fbnic_ptp_info, sizeof(fbnic_ptp_info)); + + fbd->ptp = ptp_clock_register(&fbd->ptp_info, dev); + if (IS_ERR(fbd->ptp)) + dev_err(dev, "Failed to register PTP: %pe\n", fbd->ptp); + + return PTR_ERR_OR_ZERO(fbd->ptp); +} + +void fbnic_ptp_destroy(struct fbnic_dev *fbd) +{ + if (!fbd->ptp) + return; + ptp_clock_unregister(fbd->ptp); +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c new file mode 100644 index 000000000..c55d4f76a --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.c @@ -0,0 +1,836 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/gfp.h> +#include <linux/mm.h> +#include <linux/once.h> +#include <linux/random.h> +#include <linux/string.h> +#include <uapi/linux/if_ether.h> + +#include "fbnic_tlv.h" + +/** + * fbnic_tlv_msg_alloc - Allocate page and initialize FW message header + * @msg_id: Identifier for new message we are starting + * + * Return: pointer to start of message, or NULL on failure. + * + * Allocates a page and initializes message header at start of page. + * Initial message size is 1 DWORD which is just the header. + **/ +struct fbnic_tlv_msg *fbnic_tlv_msg_alloc(u16 msg_id) +{ + struct fbnic_tlv_hdr hdr = { 0 }; + struct fbnic_tlv_msg *msg; + + msg = (struct fbnic_tlv_msg *)__get_free_page(GFP_KERNEL); + if (!msg) + return NULL; + + /* Start with zero filled header and then back fill with data */ + hdr.type = msg_id; + hdr.is_msg = 1; + hdr.len = cpu_to_le16(1); + + /* Copy header into start of message */ + msg->hdr = hdr; + + return msg; +} + +/** + * fbnic_tlv_attr_put_flag - Add flag value to message + * @msg: Message header we are adding flag attribute to + * @attr_id: ID of flag attribute we are adding to message + * + * Return: -ENOSPC if there is no room for the attribute. Otherwise 0. + * + * Adds a 1 DWORD flag attribute to the message. The presence of this + * attribute can be used as a boolean value indicating true, otherwise the + * value is considered false. + **/ +int fbnic_tlv_attr_put_flag(struct fbnic_tlv_msg *msg, const u16 attr_id) +{ + int attr_max_len = PAGE_SIZE - offset_in_page(msg) - sizeof(*msg); + struct fbnic_tlv_hdr hdr = { 0 }; + struct fbnic_tlv_msg *attr; + + attr_max_len -= le16_to_cpu(msg->hdr.len) * sizeof(u32); + if (attr_max_len < sizeof(*attr)) + return -ENOSPC; + + /* Get header pointer and bump attr to start of data */ + attr = &msg[le16_to_cpu(msg->hdr.len)]; + + /* Record attribute type and size */ + hdr.type = attr_id; + hdr.len = cpu_to_le16(sizeof(hdr)); + + attr->hdr = hdr; + le16_add_cpu(&msg->hdr.len, + FBNIC_TLV_MSG_SIZE(le16_to_cpu(hdr.len))); + + return 0; +} + +/** + * fbnic_tlv_attr_put_value - Add data to message + * @msg: Message header we are adding flag attribute to + * @attr_id: ID of flag attribute we are adding to message + * @value: Pointer to data to be stored + * @len: Size of data to be stored. + * + * Return: -ENOSPC if there is no room for the attribute. Otherwise 0. + * + * Adds header and copies data pointed to by value into the message. The + * result is rounded up to the nearest DWORD for sizing so that the + * headers remain aligned. + * + * The assumption is that the value field is in a format where byte + * ordering can be guaranteed such as a byte array or a little endian + * format. + **/ +int fbnic_tlv_attr_put_value(struct fbnic_tlv_msg *msg, const u16 attr_id, + const void *value, const int len) +{ + int attr_max_len = PAGE_SIZE - offset_in_page(msg) - sizeof(*msg); + struct fbnic_tlv_hdr hdr = { 0 }; + struct fbnic_tlv_msg *attr; + + attr_max_len -= le16_to_cpu(msg->hdr.len) * sizeof(u32); + if (attr_max_len < sizeof(*attr) + len) + return -ENOSPC; + + /* Get header pointer and bump attr to start of data */ + attr = &msg[le16_to_cpu(msg->hdr.len)]; + + /* Record attribute type and size */ + hdr.type = attr_id; + hdr.len = cpu_to_le16(sizeof(hdr) + len); + + /* Zero pad end of region to be written if we aren't aligned */ + if (len % sizeof(hdr)) + attr->value[len / sizeof(hdr)] = 0; + + /* Copy data over */ + memcpy(attr->value, value, len); + + attr->hdr = hdr; + le16_add_cpu(&msg->hdr.len, + FBNIC_TLV_MSG_SIZE(le16_to_cpu(hdr.len))); + + return 0; +} + +/** + * __fbnic_tlv_attr_put_int - Add integer to message + * @msg: Message header we are adding flag attribute to + * @attr_id: ID of flag attribute we are adding to message + * @value: Data to be stored + * @len: Size of data to be stored, either 4 or 8 bytes. + * + * Return: -ENOSPC if there is no room for the attribute. Otherwise 0. + * + * Adds header and copies data pointed to by value into the message. Will + * format the data as little endian. + **/ +int __fbnic_tlv_attr_put_int(struct fbnic_tlv_msg *msg, const u16 attr_id, + s64 value, const int len) +{ + __le64 le64_value = cpu_to_le64(value); + + return fbnic_tlv_attr_put_value(msg, attr_id, &le64_value, len); +} + +/** + * fbnic_tlv_attr_put_mac_addr - Add mac_addr to message + * @msg: Message header we are adding flag attribute to + * @attr_id: ID of flag attribute we are adding to message + * @mac_addr: Byte pointer to MAC address to be stored + * + * Return: -ENOSPC if there is no room for the attribute. Otherwise 0. + * + * Adds header and copies data pointed to by mac_addr into the message. Will + * copy the address raw so it will be in big endian with start of MAC + * address at start of attribute. + **/ +int fbnic_tlv_attr_put_mac_addr(struct fbnic_tlv_msg *msg, const u16 attr_id, + const u8 *mac_addr) +{ + return fbnic_tlv_attr_put_value(msg, attr_id, mac_addr, ETH_ALEN); +} + +/** + * fbnic_tlv_attr_put_string - Add string to message + * @msg: Message header we are adding flag attribute to + * @attr_id: ID of flag attribute we are adding to message + * @string: Byte pointer to null terminated string to be stored + * + * Return: -ENOSPC if there is no room for the attribute. Otherwise 0. + * + * Adds header and copies data pointed to by string into the message. Will + * copy the address raw so it will be in byte order. + **/ +int fbnic_tlv_attr_put_string(struct fbnic_tlv_msg *msg, u16 attr_id, + const char *string) +{ + int attr_max_len = PAGE_SIZE - sizeof(*msg); + int str_len = 1; + + /* The max length will be message minus existing message and new + * attribute header. Since the message is measured in DWORDs we have + * to multiply the size by 4. + * + * The string length doesn't include the \0 so we have to add one to + * the final value, so start with that as our initial value. + * + * We will verify if the string will fit in fbnic_tlv_attr_put_value() + */ + attr_max_len -= le16_to_cpu(msg->hdr.len) * sizeof(u32); + str_len += strnlen(string, attr_max_len); + + return fbnic_tlv_attr_put_value(msg, attr_id, string, str_len); +} + +/** + * fbnic_tlv_attr_get_unsigned - Retrieve unsigned value from result + * @attr: Attribute to retrieve data from + * @def: The default value if attr is NULL + * + * Return: unsigned 64b value containing integer value + **/ +u64 fbnic_tlv_attr_get_unsigned(struct fbnic_tlv_msg *attr, u64 def) +{ + __le64 le64_value = 0; + + if (!attr) + return def; + + memcpy(&le64_value, &attr->value[0], + le16_to_cpu(attr->hdr.len) - sizeof(*attr)); + + return le64_to_cpu(le64_value); +} + +/** + * fbnic_tlv_attr_get_signed - Retrieve signed value from result + * @attr: Attribute to retrieve data from + * @def: The default value if attr is NULL + * + * Return: signed 64b value containing integer value + **/ +s64 fbnic_tlv_attr_get_signed(struct fbnic_tlv_msg *attr, s64 def) +{ + __le64 le64_value = 0; + int shift; + s64 value; + + if (!attr) + return def; + + shift = (8 + sizeof(*attr) - le16_to_cpu(attr->hdr.len)) * 8; + + /* Copy the value and adjust for byte ordering */ + memcpy(&le64_value, &attr->value[0], + le16_to_cpu(attr->hdr.len) - sizeof(*attr)); + value = le64_to_cpu(le64_value); + + /* Sign extend the return value by using a pair of shifts */ + return (value << shift) >> shift; +} + +/** + * fbnic_tlv_attr_get_string - Retrieve string value from result + * @attr: Attribute to retrieve data from + * @dst: Pointer to an allocated string to store the data + * @dstsize: The maximum size which can be in dst + * + * Return: the size of the string read from firmware or negative error. + **/ +ssize_t fbnic_tlv_attr_get_string(struct fbnic_tlv_msg *attr, char *dst, + size_t dstsize) +{ + size_t srclen, len; + ssize_t ret; + + if (!attr) + return -EINVAL; + + if (dstsize == 0) + return -E2BIG; + + srclen = le16_to_cpu(attr->hdr.len) - sizeof(*attr); + if (srclen > 0 && ((char *)attr->value)[srclen - 1] == '\0') + srclen--; + + if (srclen >= dstsize) { + len = dstsize - 1; + ret = -E2BIG; + } else { + len = srclen; + ret = len; + } + + memcpy(dst, &attr->value, len); + /* Zero pad end of dst. */ + memset(dst + len, 0, dstsize - len); + + return ret; +} + +/** + * fbnic_tlv_attr_nest_start - Add nested attribute header to message + * @msg: Message header we are adding flag attribute to + * @attr_id: ID of flag attribute we are adding to message + * + * Return: NULL if there is no room for the attribute. Otherwise a pointer + * to the new attribute header. + * + * New header length is stored initially in DWORDs. + **/ +struct fbnic_tlv_msg *fbnic_tlv_attr_nest_start(struct fbnic_tlv_msg *msg, + u16 attr_id) +{ + int attr_max_len = PAGE_SIZE - offset_in_page(msg) - sizeof(*msg); + struct fbnic_tlv_msg *attr = &msg[le16_to_cpu(msg->hdr.len)]; + struct fbnic_tlv_hdr hdr = { 0 }; + + /* Make sure we have space for at least the nest header plus one more */ + attr_max_len -= le16_to_cpu(msg->hdr.len) * sizeof(u32); + if (attr_max_len < sizeof(*attr) * 2) + return NULL; + + /* Record attribute type and size */ + hdr.type = attr_id; + + /* Add current message length to account for consumption within the + * page and leave it as a multiple of DWORDs, we will shift to + * bytes when we close it out. + */ + hdr.len = cpu_to_le16(1); + + attr->hdr = hdr; + + return attr; +} + +/** + * fbnic_tlv_attr_nest_stop - Close out nested attribute and add it to message + * @msg: Message header we are adding flag attribute to + * + * Closes out nested attribute, adds length to message, and then bumps + * length from DWORDs to bytes to match other attributes. + **/ +void fbnic_tlv_attr_nest_stop(struct fbnic_tlv_msg *msg) +{ + struct fbnic_tlv_msg *attr = &msg[le16_to_cpu(msg->hdr.len)]; + u16 len = le16_to_cpu(attr->hdr.len); + + /* Add attribute to message if there is more than just a header */ + if (len <= 1) + return; + + le16_add_cpu(&msg->hdr.len, len); + + /* Convert from DWORDs to bytes */ + attr->hdr.len = cpu_to_le16(len * sizeof(u32)); +} + +static int +fbnic_tlv_attr_validate(struct fbnic_tlv_msg *attr, + const struct fbnic_tlv_index *tlv_index) +{ + u16 len = le16_to_cpu(attr->hdr.len) - sizeof(*attr); + u16 attr_id = attr->hdr.type; + __le32 *value = &attr->value[0]; + + if (attr->hdr.is_msg) + return -EINVAL; + + if (attr_id >= FBNIC_TLV_RESULTS_MAX) + return -EINVAL; + + while (tlv_index->id != attr_id) { + if (tlv_index->id == FBNIC_TLV_ATTR_ID_UNKNOWN) { + if (attr->hdr.cannot_ignore) + return -ENOENT; + return le16_to_cpu(attr->hdr.len); + } + + tlv_index++; + } + + if (offset_in_page(attr) + len > PAGE_SIZE - sizeof(*attr)) + return -E2BIG; + + switch (tlv_index->type) { + case FBNIC_TLV_STRING: + if (!len || len > tlv_index->len) + return -EINVAL; + if (((char *)value)[len - 1]) + return -EINVAL; + break; + case FBNIC_TLV_FLAG: + if (len) + return -EINVAL; + break; + case FBNIC_TLV_UNSIGNED: + case FBNIC_TLV_SIGNED: + if (tlv_index->len > sizeof(__le64)) + return -EINVAL; + fallthrough; + case FBNIC_TLV_BINARY: + if (!len || len > tlv_index->len) + return -EINVAL; + break; + case FBNIC_TLV_NESTED: + case FBNIC_TLV_ARRAY: + if (len % 4) + return -EINVAL; + break; + default: + return -EINVAL; + } + + return 0; +} + +/** + * fbnic_tlv_attr_parse_array - Parse array of attributes into results array + * @attr: Start of attributes in the message + * @len: Length of attributes in the message + * @results: Array of pointers to store the results of parsing + * @tlv_index: List of TLV attributes to be parsed from message + * @tlv_attr_id: Specific ID that is repeated in array + * @array_len: Number of results to store in results array + * + * Return: zero on success, or negative value on error. + * + * Will take a list of attributes and a parser definition and will capture + * the results in the results array to have the data extracted later. + **/ +int fbnic_tlv_attr_parse_array(struct fbnic_tlv_msg *attr, int len, + struct fbnic_tlv_msg **results, + const struct fbnic_tlv_index *tlv_index, + u16 tlv_attr_id, size_t array_len) +{ + int i = 0; + + /* Initialize results table to NULL. */ + memset(results, 0, array_len * sizeof(results[0])); + + /* Nothing to parse if header was only thing there */ + if (!len) + return 0; + + /* Work through list of attributes, parsing them as necessary */ + while (len > 0) { + u16 attr_id = attr->hdr.type; + u16 attr_len; + int err; + + if (tlv_attr_id != attr_id) + return -EINVAL; + + /* Stop parsing on full error */ + err = fbnic_tlv_attr_validate(attr, tlv_index); + if (err < 0) + return err; + + if (i >= array_len) + return -ENOSPC; + + results[i++] = attr; + + attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len)); + len -= attr_len; + attr += attr_len; + } + + return len == 0 ? 0 : -EINVAL; +} + +/** + * fbnic_tlv_attr_parse - Parse attributes into a list of attribute results + * @attr: Start of attributes in the message + * @len: Length of attributes in the message + * @results: Array of pointers to store the results of parsing + * @tlv_index: List of TLV attributes to be parsed from message + * + * Return: zero on success, or negative value on error. + * + * Will take a list of attributes and a parser definition and will capture + * the results in the results array to have the data extracted later. + **/ +int fbnic_tlv_attr_parse(struct fbnic_tlv_msg *attr, int len, + struct fbnic_tlv_msg **results, + const struct fbnic_tlv_index *tlv_index) +{ + /* Initialize results table to NULL. */ + memset(results, 0, sizeof(results[0]) * FBNIC_TLV_RESULTS_MAX); + + /* Nothing to parse if header was only thing there */ + if (!len) + return 0; + + /* Work through list of attributes, parsing them as necessary */ + while (len > 0) { + int err = fbnic_tlv_attr_validate(attr, tlv_index); + u16 attr_id = attr->hdr.type; + u16 attr_len; + + /* Stop parsing on full error */ + if (err < 0) + return err; + + /* Ignore results for unsupported values */ + if (!err) { + /* Do not overwrite existing entries */ + if (results[attr_id]) + return -EADDRINUSE; + + results[attr_id] = attr; + } + + attr_len = FBNIC_TLV_MSG_SIZE(le16_to_cpu(attr->hdr.len)); + len -= attr_len; + attr += attr_len; + } + + return len == 0 ? 0 : -EINVAL; +} + +/** + * fbnic_tlv_msg_parse - Parse message and process via predetermined functions + * @opaque: Value passed to parser function to enable driver access + * @msg: Message to be parsed. + * @parser: TLV message parser definition. + * + * Return: zero on success, or negative value on error. + * + * Will take a message a number of message types via the attribute parsing + * definitions and function provided for the parser array. + **/ +int fbnic_tlv_msg_parse(void *opaque, struct fbnic_tlv_msg *msg, + const struct fbnic_tlv_parser *parser) +{ + struct fbnic_tlv_msg *results[FBNIC_TLV_RESULTS_MAX]; + u16 msg_id = msg->hdr.type; + int err; + + if (!msg->hdr.is_msg) + return -EINVAL; + + if (le16_to_cpu(msg->hdr.len) > PAGE_SIZE / sizeof(u32)) + return -E2BIG; + + while (parser->id != msg_id) { + if (parser->id == FBNIC_TLV_MSG_ID_UNKNOWN) + return -ENOENT; + parser++; + } + + err = fbnic_tlv_attr_parse(&msg[1], le16_to_cpu(msg->hdr.len) - 1, + results, parser->attr); + if (err) + return err; + + return parser->func(opaque, results); +} + +/** + * fbnic_tlv_parser_error - called if message doesn't match known type + * @opaque: (unused) + * @results: (unused) + * + * Return: -EBADMSG to indicate the message is an unsupported type + **/ +int fbnic_tlv_parser_error(void *opaque, struct fbnic_tlv_msg **results) +{ + return -EBADMSG; +} + +#define FBNIC_TLV_TEST_STRING_LEN 32 + +struct fbnic_tlv_test { + u64 test_u64; + s64 test_s64; + u32 test_u32; + s32 test_s32; + u16 test_u16; + s16 test_s16; + u8 test_mac[ETH_ALEN]; + u8 test_mac_array[4][ETH_ALEN]; + u8 test_true; + u8 test_false; + char test_string[FBNIC_TLV_TEST_STRING_LEN]; +}; + +static struct fbnic_tlv_test test_struct; + +const struct fbnic_tlv_index fbnic_tlv_test_index[] = { + FBNIC_TLV_ATTR_U64(FBNIC_TLV_TEST_MSG_U64), + FBNIC_TLV_ATTR_S64(FBNIC_TLV_TEST_MSG_S64), + FBNIC_TLV_ATTR_U32(FBNIC_TLV_TEST_MSG_U32), + FBNIC_TLV_ATTR_S32(FBNIC_TLV_TEST_MSG_S32), + FBNIC_TLV_ATTR_U32(FBNIC_TLV_TEST_MSG_U16), + FBNIC_TLV_ATTR_S32(FBNIC_TLV_TEST_MSG_S16), + FBNIC_TLV_ATTR_MAC_ADDR(FBNIC_TLV_TEST_MSG_MAC_ADDR), + FBNIC_TLV_ATTR_FLAG(FBNIC_TLV_TEST_MSG_FLAG_TRUE), + FBNIC_TLV_ATTR_FLAG(FBNIC_TLV_TEST_MSG_FLAG_FALSE), + FBNIC_TLV_ATTR_STRING(FBNIC_TLV_TEST_MSG_STRING, + FBNIC_TLV_TEST_STRING_LEN), + FBNIC_TLV_ATTR_ARRAY(FBNIC_TLV_TEST_MSG_ARRAY), + FBNIC_TLV_ATTR_NESTED(FBNIC_TLV_TEST_MSG_NESTED), + FBNIC_TLV_ATTR_LAST +}; + +static void fbnic_tlv_test_struct_init(void) +{ + int i = FBNIC_TLV_TEST_STRING_LEN - 1; + + /* Populate the struct with random data */ + get_random_once(&test_struct, + offsetof(struct fbnic_tlv_test, test_string) + i); + + /* Force true/false to their expected values */ + test_struct.test_false = false; + test_struct.test_true = true; + + /* Convert test_string to a true ASCII string */ + test_struct.test_string[i] = '\0'; + while (i--) { + /* Force characters into displayable range */ + if (test_struct.test_string[i] < 64 || + test_struct.test_string[i] >= 96) { + test_struct.test_string[i] %= 32; + test_struct.test_string[i] += 64; + } + } +} + +static int fbnic_tlv_test_attr_data(struct fbnic_tlv_msg *msg) +{ + struct fbnic_tlv_msg *array; + int err, i; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U64, + test_struct.test_u64); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S64, + test_struct.test_s64); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U32, + test_struct.test_u32); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S32, + test_struct.test_s32); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_U16, + test_struct.test_u16); + if (err) + return err; + + err = fbnic_tlv_attr_put_int(msg, FBNIC_TLV_TEST_MSG_S16, + test_struct.test_s16); + if (err) + return err; + + err = fbnic_tlv_attr_put_value(msg, FBNIC_TLV_TEST_MSG_MAC_ADDR, + test_struct.test_mac, ETH_ALEN); + if (err) + return err; + + /* Start MAC address array */ + array = fbnic_tlv_attr_nest_start(msg, FBNIC_TLV_TEST_MSG_ARRAY); + if (!array) + return -ENOSPC; + + for (i = 0; i < 4; i++) { + err = fbnic_tlv_attr_put_value(array, + FBNIC_TLV_TEST_MSG_MAC_ADDR, + test_struct.test_mac_array[i], + ETH_ALEN); + if (err) + return err; + } + + /* Close array */ + fbnic_tlv_attr_nest_stop(msg); + + err = fbnic_tlv_attr_put_flag(msg, FBNIC_TLV_TEST_MSG_FLAG_TRUE); + if (err) + return err; + + return fbnic_tlv_attr_put_string(msg, FBNIC_TLV_TEST_MSG_STRING, + test_struct.test_string); +} + +/** + * fbnic_tlv_test_create - Allocate a test message and fill it w/ data + * @fbd: FBNIC device structure + * + * Return: NULL on failure to allocate or pointer to new TLV test message. + **/ +struct fbnic_tlv_msg *fbnic_tlv_test_create(struct fbnic_dev *fbd) +{ + struct fbnic_tlv_msg *msg, *nest; + int err; + + msg = fbnic_tlv_msg_alloc(FBNIC_TLV_MSG_ID_TEST); + if (!msg) + return NULL; + + /* Randomize struct data */ + fbnic_tlv_test_struct_init(); + + /* Add first level of data to message */ + err = fbnic_tlv_test_attr_data(msg); + if (err) + goto free_message; + + /* Start second level nested */ + nest = fbnic_tlv_attr_nest_start(msg, FBNIC_TLV_TEST_MSG_NESTED); + if (!nest) + goto free_message; + + /* Add nested data */ + err = fbnic_tlv_test_attr_data(nest); + if (err) + goto free_message; + + /* Close nest and report full message */ + fbnic_tlv_attr_nest_stop(msg); + + return msg; +free_message: + free_page((unsigned long)msg); + return NULL; +} + +void fbnic_tlv_attr_addr_copy(u8 *dest, struct fbnic_tlv_msg *src) +{ + u8 *mac_addr; + + mac_addr = fbnic_tlv_attr_get_value_ptr(src); + memcpy(dest, mac_addr, ETH_ALEN); +} + +/** + * fbnic_tlv_parser_test_attr - Function loading test attributes into structure + * @str: Test structure to load + * @results: Pointer to results array + * + * Copies attributes into structure. Any attribute that doesn't exist in the + * results array is not populated. + **/ +static void fbnic_tlv_parser_test_attr(struct fbnic_tlv_test *str, + struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *array_results[4]; + struct fbnic_tlv_msg *attr; + char *string = NULL; + int i, err; + + str->test_u64 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U64); + str->test_u32 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U32); + str->test_u16 = fta_get_uint(results, FBNIC_TLV_TEST_MSG_U16); + + str->test_s64 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S64); + str->test_s32 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S32); + str->test_s16 = fta_get_sint(results, FBNIC_TLV_TEST_MSG_S16); + + attr = results[FBNIC_TLV_TEST_MSG_MAC_ADDR]; + if (attr) + fbnic_tlv_attr_addr_copy(str->test_mac, attr); + + attr = results[FBNIC_TLV_TEST_MSG_ARRAY]; + if (attr) { + int len = le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1; + + err = fbnic_tlv_attr_parse_array(&attr[1], len, + array_results, + fbnic_tlv_test_index, + FBNIC_TLV_TEST_MSG_MAC_ADDR, + 4); + if (!err) { + for (i = 0; i < 4 && array_results[i]; i++) + fbnic_tlv_attr_addr_copy(str->test_mac_array[i], + array_results[i]); + } + } + + str->test_true = !!results[FBNIC_TLV_TEST_MSG_FLAG_TRUE]; + str->test_false = !!results[FBNIC_TLV_TEST_MSG_FLAG_FALSE]; + + attr = results[FBNIC_TLV_TEST_MSG_STRING]; + if (attr) { + string = fbnic_tlv_attr_get_value_ptr(attr); + strscpy(str->test_string, string, FBNIC_TLV_TEST_STRING_LEN); + } +} + +static void fbnic_tlv_test_dump(struct fbnic_tlv_test *value, char *prefix) +{ + print_hex_dump(KERN_INFO, prefix, DUMP_PREFIX_OFFSET, 16, 1, + value, sizeof(*value), true); +} + +/** + * fbnic_tlv_parser_test - Function for parsing and testing test message + * @opaque: Unused value + * @results: Results of parser output + * + * Return: negative value on error, or 0 on success. + * + * Parses attributes to structures and compares the structure to the + * expected test value that should have been used to populate the message. + * + * Used to verify message generation and parser are working correctly. + **/ +int fbnic_tlv_parser_test(void *opaque, struct fbnic_tlv_msg **results) +{ + struct fbnic_tlv_msg *nest_results[FBNIC_TLV_RESULTS_MAX] = { 0 }; + struct fbnic_tlv_test result_struct; + struct fbnic_tlv_msg *attr; + int err; + + memset(&result_struct, 0, sizeof(result_struct)); + fbnic_tlv_parser_test_attr(&result_struct, results); + + if (memcmp(&test_struct, &result_struct, sizeof(test_struct))) { + fbnic_tlv_test_dump(&result_struct, "fbnic: found - "); + fbnic_tlv_test_dump(&test_struct, "fbnic: expected - "); + return -EINVAL; + } + + attr = results[FBNIC_TLV_TEST_MSG_NESTED]; + if (!attr) + return -EINVAL; + + err = fbnic_tlv_attr_parse(&attr[1], + le16_to_cpu(attr->hdr.len) / sizeof(u32) - 1, + nest_results, fbnic_tlv_test_index); + if (err) + return err; + + memset(&result_struct, 0, sizeof(result_struct)); + fbnic_tlv_parser_test_attr(&result_struct, nest_results); + + if (memcmp(&test_struct, &result_struct, sizeof(test_struct))) { + fbnic_tlv_test_dump(&result_struct, "fbnic: found - "); + fbnic_tlv_test_dump(&test_struct, "fbnic: expected - "); + return -EINVAL; + } + + return 0; +} diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h new file mode 100644 index 000000000..9c4e47593 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_tlv.h @@ -0,0 +1,185 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_TLV_H_ +#define _FBNIC_TLV_H_ + +#include <asm/byteorder.h> +#include <linux/bits.h> +#include <linux/const.h> +#include <linux/types.h> + +struct fbnic_dev; + +#define FBNIC_TLV_MSG_ALIGN(len) ALIGN(len, sizeof(u32)) +#define FBNIC_TLV_MSG_SIZE(len) \ + (FBNIC_TLV_MSG_ALIGN(len) / sizeof(u32)) + +/* TLV Header Format + * 3 2 1 + * 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 9 8 7 6 5 4 3 2 1 0 + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * | Length |M|I|RSV| Type / ID | + * +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + * + * The TLV header format described above will be used for transferring + * messages between the host and the firmware. To ensure byte ordering + * we have defined all fields as being little endian. + * Type/ID: Identifier for message and/or attribute + * RSV: Reserved field for future use, likely as additional flags + * I: cannot_ignore flag, identifies if unrecognized attribute can be ignored + * M: is_msg, indicates that this is the start of a new message + * Length: Total length of message in dwords including header + * or + * Total length of attribute in bytes including header + */ +struct fbnic_tlv_hdr { +#if defined(__LITTLE_ENDIAN_BITFIELD) + u16 type : 12; /* 0 .. 11 Type / ID */ + u16 rsvd : 2; /* 12 .. 13 Reserved for future use */ + u16 cannot_ignore : 1; /* 14 Attribute can be ignored */ + u16 is_msg : 1; /* 15 Header belongs to message */ +#elif defined(__BIG_ENDIAN_BITFIELD) + u16 is_msg : 1; /* 15 Header belongs to message */ + u16 cannot_ignore : 1; /* 14 Attribute can be ignored */ + u16 rsvd : 2; /* 13 .. 12 Reserved for future use */ + u16 type : 12; /* 11 .. 0 Type / ID */ +#else +#error "Missing defines from byteorder.h" +#endif + __le16 len; /* 16 .. 32 length including TLV header */ +}; + +#define FBNIC_TLV_RESULTS_MAX 32 + +struct fbnic_tlv_msg { + struct fbnic_tlv_hdr hdr; + __le32 value[]; +}; + +#define FBNIC_TLV_MSG_ID_UNKNOWN USHRT_MAX + +enum fbnic_tlv_type { + FBNIC_TLV_STRING, + FBNIC_TLV_FLAG, + FBNIC_TLV_UNSIGNED, + FBNIC_TLV_SIGNED, + FBNIC_TLV_BINARY, + FBNIC_TLV_NESTED, + FBNIC_TLV_ARRAY, + __FBNIC_TLV_MAX_TYPE +}; + +/* TLV Index + * Defines the relationship between the attribute IDs and their types. + * For each entry in the index there will be a size and type associated + * with it so that we can use this to parse the data and verify it matches + * the expected layout. + */ +struct fbnic_tlv_index { + u16 id; + u16 len; + enum fbnic_tlv_type type; +}; + +#define TLV_MAX_DATA ((PAGE_SIZE - 512) & 0xFFFF) +#define FBNIC_TLV_ATTR_ID_UNKNOWN USHRT_MAX +#define FBNIC_TLV_ATTR_STRING(id, len) { id, len, FBNIC_TLV_STRING } +#define FBNIC_TLV_ATTR_FLAG(id) { id, 0, FBNIC_TLV_FLAG } +#define FBNIC_TLV_ATTR_U32(id) { id, sizeof(u32), FBNIC_TLV_UNSIGNED } +#define FBNIC_TLV_ATTR_U64(id) { id, sizeof(u64), FBNIC_TLV_UNSIGNED } +#define FBNIC_TLV_ATTR_S32(id) { id, sizeof(s32), FBNIC_TLV_SIGNED } +#define FBNIC_TLV_ATTR_S64(id) { id, sizeof(s64), FBNIC_TLV_SIGNED } +#define FBNIC_TLV_ATTR_MAC_ADDR(id) { id, ETH_ALEN, FBNIC_TLV_BINARY } +#define FBNIC_TLV_ATTR_NESTED(id) { id, 0, FBNIC_TLV_NESTED } +#define FBNIC_TLV_ATTR_ARRAY(id) { id, 0, FBNIC_TLV_ARRAY } +#define FBNIC_TLV_ATTR_RAW_DATA(id) { id, TLV_MAX_DATA, FBNIC_TLV_BINARY } +#define FBNIC_TLV_ATTR_LAST { FBNIC_TLV_ATTR_ID_UNKNOWN, 0, 0 } + +struct fbnic_tlv_parser { + u16 id; + const struct fbnic_tlv_index *attr; + int (*func)(void *opaque, + struct fbnic_tlv_msg **results); +}; + +#define FBNIC_TLV_PARSER(id, attr, func) { FBNIC_TLV_MSG_ID_##id, attr, func } + +static inline void * +fbnic_tlv_attr_get_value_ptr(struct fbnic_tlv_msg *attr) +{ + return (void *)&attr->value[0]; +} + +static inline bool fbnic_tlv_attr_get_bool(struct fbnic_tlv_msg *attr) +{ + return !!attr; +} + +u64 fbnic_tlv_attr_get_unsigned(struct fbnic_tlv_msg *attr, u64 def); +s64 fbnic_tlv_attr_get_signed(struct fbnic_tlv_msg *attr, s64 def); +ssize_t fbnic_tlv_attr_get_string(struct fbnic_tlv_msg *attr, char *dst, + size_t dstsize); +struct fbnic_tlv_msg *fbnic_tlv_msg_alloc(u16 msg_id); +int fbnic_tlv_attr_put_flag(struct fbnic_tlv_msg *msg, const u16 attr_id); +int fbnic_tlv_attr_put_value(struct fbnic_tlv_msg *msg, const u16 attr_id, + const void *value, const int len); +int __fbnic_tlv_attr_put_int(struct fbnic_tlv_msg *msg, const u16 attr_id, + s64 value, const int len); +#define fbnic_tlv_attr_put_int(msg, attr_id, value) \ + __fbnic_tlv_attr_put_int(msg, attr_id, value, \ + FBNIC_TLV_MSG_ALIGN(sizeof(value))) +int fbnic_tlv_attr_put_mac_addr(struct fbnic_tlv_msg *msg, const u16 attr_id, + const u8 *mac_addr); +int fbnic_tlv_attr_put_string(struct fbnic_tlv_msg *msg, u16 attr_id, + const char *string); +struct fbnic_tlv_msg *fbnic_tlv_attr_nest_start(struct fbnic_tlv_msg *msg, + u16 attr_id); +void fbnic_tlv_attr_nest_stop(struct fbnic_tlv_msg *msg); +void fbnic_tlv_attr_addr_copy(u8 *dest, struct fbnic_tlv_msg *src); +int fbnic_tlv_attr_parse_array(struct fbnic_tlv_msg *attr, int len, + struct fbnic_tlv_msg **results, + const struct fbnic_tlv_index *tlv_index, + u16 tlv_attr_id, size_t array_len); +int fbnic_tlv_attr_parse(struct fbnic_tlv_msg *attr, int len, + struct fbnic_tlv_msg **results, + const struct fbnic_tlv_index *tlv_index); +int fbnic_tlv_msg_parse(void *opaque, struct fbnic_tlv_msg *msg, + const struct fbnic_tlv_parser *parser); +int fbnic_tlv_parser_error(void *opaque, struct fbnic_tlv_msg **results); + +#define fta_get_uint(_results, _id) \ + fbnic_tlv_attr_get_unsigned(_results[_id], 0) +#define fta_get_sint(_results, _id) \ + fbnic_tlv_attr_get_signed(_results[_id], 0) +#define fta_get_str(_results, _id, _dst, _dstsize) \ + fbnic_tlv_attr_get_string(_results[_id], _dst, _dstsize) + +#define FBNIC_TLV_MSG_ID_TEST 0 + +enum fbnic_tlv_test_attr_id { + FBNIC_TLV_TEST_MSG_U64, + FBNIC_TLV_TEST_MSG_S64, + FBNIC_TLV_TEST_MSG_U32, + FBNIC_TLV_TEST_MSG_S32, + FBNIC_TLV_TEST_MSG_U16, + FBNIC_TLV_TEST_MSG_S16, + FBNIC_TLV_TEST_MSG_MAC_ADDR, + FBNIC_TLV_TEST_MSG_FLAG_TRUE, + FBNIC_TLV_TEST_MSG_FLAG_FALSE, + FBNIC_TLV_TEST_MSG_STRING, + FBNIC_TLV_TEST_MSG_NESTED, + FBNIC_TLV_TEST_MSG_ARRAY, + FBNIC_TLV_TEST_MSG_MAX +}; + +extern const struct fbnic_tlv_index fbnic_tlv_test_index[]; +struct fbnic_tlv_msg *fbnic_tlv_test_create(struct fbnic_dev *fbd); +int fbnic_tlv_parser_test(void *opaque, struct fbnic_tlv_msg **results); + +#define FBNIC_TLV_MSG_TEST \ + FBNIC_TLV_PARSER(TEST, fbnic_tlv_test_index, \ + fbnic_tlv_parser_test) +#define FBNIC_TLV_MSG_ERROR \ + FBNIC_TLV_PARSER(UNKNOWN, NULL, fbnic_tlv_parser_error) +#endif /* _FBNIC_TLV_H_ */ diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c new file mode 100644 index 000000000..401f8b8ae --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.c @@ -0,0 +1,2956 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#include <linux/bitfield.h> +#include <linux/bpf.h> +#include <linux/bpf_trace.h> +#include <linux/iopoll.h> +#include <linux/pci.h> +#include <net/netdev_queues.h> +#include <net/page_pool/helpers.h> +#include <net/tcp.h> +#include <net/xdp.h> + +#include "fbnic.h" +#include "fbnic_csr.h" +#include "fbnic_netdev.h" +#include "fbnic_txrx.h" + +enum { + FBNIC_XDP_PASS = 0, + FBNIC_XDP_CONSUME, + FBNIC_XDP_TX, + FBNIC_XDP_LEN_ERR, +}; + +enum { + FBNIC_XMIT_CB_TS = 0x01, +}; + +struct fbnic_xmit_cb { + u32 bytecount; + u16 gso_segs; + u8 desc_count; + u8 flags; + int hw_head; +}; + +#define FBNIC_XMIT_CB(__skb) ((struct fbnic_xmit_cb *)((__skb)->cb)) + +#define FBNIC_XMIT_NOUNMAP ((void *)1) + +u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring) +{ + unsigned long csr_base = (unsigned long)ring->doorbell; + + csr_base &= ~(FBNIC_QUEUE_STRIDE * sizeof(u32) - 1); + + return (u32 __iomem *)csr_base; +} + +static u32 fbnic_ring_rd32(struct fbnic_ring *ring, unsigned int csr) +{ + u32 __iomem *csr_base = fbnic_ring_csr_base(ring); + + return readl(csr_base + csr); +} + +static void fbnic_ring_wr32(struct fbnic_ring *ring, unsigned int csr, u32 val) +{ + u32 __iomem *csr_base = fbnic_ring_csr_base(ring); + + writel(val, csr_base + csr); +} + +/** + * fbnic_ts40_to_ns() - convert descriptor timestamp to PHC time + * @fbn: netdev priv of the FB NIC + * @ts40: timestamp read from a descriptor + * + * Return: u64 value of PHC time in nanoseconds + * + * Convert truncated 40 bit device timestamp as read from a descriptor + * to the full PHC time in nanoseconds. + */ +static __maybe_unused u64 fbnic_ts40_to_ns(struct fbnic_net *fbn, u64 ts40) +{ + unsigned int s; + u64 time_ns; + s64 offset; + u8 ts_top; + u32 high; + + do { + s = u64_stats_fetch_begin(&fbn->time_seq); + offset = READ_ONCE(fbn->time_offset); + } while (u64_stats_fetch_retry(&fbn->time_seq, s)); + + high = READ_ONCE(fbn->time_high); + + /* Bits 63..40 from periodic clock reads, 39..0 from ts40 */ + time_ns = (u64)(high >> 8) << 40 | ts40; + + /* Compare bits 32-39 between periodic reads and ts40, + * see if HW clock may have wrapped since last read. We are sure + * that periodic reads are always at least ~1 minute behind, so + * this logic works perfectly fine. + */ + ts_top = ts40 >> 32; + if (ts_top < (u8)high && (u8)high - ts_top > U8_MAX / 2) + time_ns += 1ULL << 40; + + return time_ns + offset; +} + +static unsigned int fbnic_desc_unused(struct fbnic_ring *ring) +{ + return (ring->head - ring->tail - 1) & ring->size_mask; +} + +static unsigned int fbnic_desc_used(struct fbnic_ring *ring) +{ + return (ring->tail - ring->head) & ring->size_mask; +} + +static struct netdev_queue *txring_txq(const struct net_device *dev, + const struct fbnic_ring *ring) +{ + return netdev_get_tx_queue(dev, ring->q_idx); +} + +static int fbnic_maybe_stop_tx(const struct net_device *dev, + struct fbnic_ring *ring, + const unsigned int size) +{ + struct netdev_queue *txq = txring_txq(dev, ring); + int res; + + res = netif_txq_maybe_stop(txq, fbnic_desc_unused(ring), size, + FBNIC_TX_DESC_WAKEUP); + if (!res) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.twq.stop++; + u64_stats_update_end(&ring->stats.syncp); + } + + return !res; +} + +static bool fbnic_tx_sent_queue(struct sk_buff *skb, struct fbnic_ring *ring) +{ + struct netdev_queue *dev_queue = txring_txq(skb->dev, ring); + unsigned int bytecount = FBNIC_XMIT_CB(skb)->bytecount; + bool xmit_more = netdev_xmit_more(); + + /* TBD: Request completion more often if xmit_more becomes large */ + + return __netdev_tx_sent_queue(dev_queue, bytecount, xmit_more); +} + +static void fbnic_unmap_single_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + unsigned int len; + dma_addr_t dma; + + dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); + len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); + + dma_unmap_single(dev, dma, len, DMA_TO_DEVICE); +} + +static void fbnic_unmap_page_twd(struct device *dev, __le64 *twd) +{ + u64 raw_twd = le64_to_cpu(*twd); + unsigned int len; + dma_addr_t dma; + + dma = FIELD_GET(FBNIC_TWD_ADDR_MASK, raw_twd); + len = FIELD_GET(FBNIC_TWD_LEN_MASK, raw_twd); + + dma_unmap_page(dev, dma, len, DMA_TO_DEVICE); +} + +#define FBNIC_TWD_TYPE(_type) \ + cpu_to_le64(FIELD_PREP(FBNIC_TWD_TYPE_MASK, FBNIC_TWD_TYPE_##_type)) + +static bool fbnic_tx_tstamp(struct sk_buff *skb) +{ + struct fbnic_net *fbn; + + if (!unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + return false; + + fbn = netdev_priv(skb->dev); + if (fbn->hwtstamp_config.tx_type == HWTSTAMP_TX_OFF) + return false; + + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + FBNIC_XMIT_CB(skb)->flags |= FBNIC_XMIT_CB_TS; + FBNIC_XMIT_CB(skb)->hw_head = -1; + + return true; +} + +static bool +fbnic_tx_lso(struct fbnic_ring *ring, struct sk_buff *skb, + __le64 *meta, unsigned int *l2len, unsigned int *i3len) +{ + unsigned int l3_type, l4_type, l4len, hdrlen; + struct skb_shared_info *shinfo; + unsigned char *l4hdr; + __be16 payload_len; + + if (unlikely(skb_cow_head(skb, 0))) + return true; + + shinfo = skb_shinfo(skb); + + if (shinfo->gso_type & SKB_GSO_PARTIAL) { + l3_type = FBNIC_TWD_L3_TYPE_OTHER; + } else if (!skb->encapsulation) { + if (ip_hdr(skb)->version == 4) + l3_type = FBNIC_TWD_L3_TYPE_IPV4; + else + l3_type = FBNIC_TWD_L3_TYPE_IPV6; + } else { + unsigned int o3len; + + o3len = skb_inner_network_header(skb) - skb_network_header(skb); + *i3len -= o3len; + *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L3_OHLEN_MASK, + o3len / 2)); + l3_type = FBNIC_TWD_L3_TYPE_V6V6; + } + + l4hdr = skb_checksum_start(skb); + payload_len = cpu_to_be16(skb->len - (l4hdr - skb->data)); + + if (shinfo->gso_type & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6)) { + struct tcphdr *tcph = (struct tcphdr *)l4hdr; + + l4_type = FBNIC_TWD_L4_TYPE_TCP; + l4len = __tcp_hdrlen((struct tcphdr *)l4hdr); + csum_replace_by_diff(&tcph->check, (__force __wsum)payload_len); + } else { + struct udphdr *udph = (struct udphdr *)l4hdr; + + l4_type = FBNIC_TWD_L4_TYPE_UDP; + l4len = sizeof(struct udphdr); + csum_replace_by_diff(&udph->check, (__force __wsum)payload_len); + } + + hdrlen = (l4hdr - skb->data) + l4len; + *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L3_TYPE_MASK, l3_type) | + FIELD_PREP(FBNIC_TWD_L4_TYPE_MASK, l4_type) | + FIELD_PREP(FBNIC_TWD_L4_HLEN_MASK, l4len / 4) | + FIELD_PREP(FBNIC_TWD_MSS_MASK, shinfo->gso_size) | + FBNIC_TWD_FLAG_REQ_LSO); + + FBNIC_XMIT_CB(skb)->bytecount += (shinfo->gso_segs - 1) * hdrlen; + FBNIC_XMIT_CB(skb)->gso_segs = shinfo->gso_segs; + + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.twq.lso += shinfo->gso_segs; + u64_stats_update_end(&ring->stats.syncp); + + return false; +} + +static bool +fbnic_tx_offloads(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) +{ + unsigned int l2len, i3len; + + if (fbnic_tx_tstamp(skb)) + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_TS); + + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) + return false; + + l2len = skb_mac_header_len(skb); + i3len = skb_checksum_start(skb) - skb_network_header(skb); + + *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_CSUM_OFFSET_MASK, + skb->csum_offset / 2)); + + if (skb_is_gso(skb)) { + if (fbnic_tx_lso(ring, skb, meta, &l2len, &i3len)) + return true; + } else { + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_CSO); + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.twq.csum_partial++; + u64_stats_update_end(&ring->stats.syncp); + } + + *meta |= cpu_to_le64(FIELD_PREP(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) | + FIELD_PREP(FBNIC_TWD_L3_IHLEN_MASK, i3len / 2)); + return false; +} + +static void +fbnic_rx_csum(u64 rcd, struct sk_buff *skb, struct fbnic_ring *rcq, + u64 *csum_cmpl, u64 *csum_none) +{ + skb_checksum_none_assert(skb); + + if (unlikely(!(skb->dev->features & NETIF_F_RXCSUM))) { + (*csum_none)++; + return; + } + + if (FIELD_GET(FBNIC_RCD_META_L4_CSUM_UNNECESSARY, rcd)) { + skb->ip_summed = CHECKSUM_UNNECESSARY; + } else { + u16 csum = FIELD_GET(FBNIC_RCD_META_L2_CSUM_MASK, rcd); + + skb->ip_summed = CHECKSUM_COMPLETE; + skb->csum = (__force __wsum)csum; + (*csum_cmpl)++; + } +} + +static bool +fbnic_tx_map(struct fbnic_ring *ring, struct sk_buff *skb, __le64 *meta) +{ + struct device *dev = skb->dev->dev.parent; + unsigned int tail = ring->tail, first; + unsigned int size, data_len; + skb_frag_t *frag; + bool is_net_iov; + dma_addr_t dma; + __le64 *twd; + + ring->tx_buf[tail] = skb; + + tail++; + tail &= ring->size_mask; + first = tail; + + size = skb_headlen(skb); + data_len = skb->data_len; + + if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) + goto dma_error; + + is_net_iov = false; + dma = dma_map_single(dev, skb->data, size, DMA_TO_DEVICE); + + for (frag = &skb_shinfo(skb)->frags[0];; frag++) { + twd = &ring->desc[tail]; + + if (dma_mapping_error(dev, dma)) + goto dma_error; + + *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | + FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | + FIELD_PREP(FBNIC_TWD_TYPE_MASK, + FBNIC_TWD_TYPE_AL)); + if (is_net_iov) + ring->tx_buf[tail] = FBNIC_XMIT_NOUNMAP; + + tail++; + tail &= ring->size_mask; + + if (!data_len) + break; + + size = skb_frag_size(frag); + data_len -= size; + + if (size > FIELD_MAX(FBNIC_TWD_LEN_MASK)) + goto dma_error; + + is_net_iov = skb_frag_is_net_iov(frag); + dma = skb_frag_dma_map(dev, frag, 0, size, DMA_TO_DEVICE); + } + + *twd |= FBNIC_TWD_TYPE(LAST_AL); + + FBNIC_XMIT_CB(skb)->desc_count = ((twd - meta) + 1) & ring->size_mask; + + ring->tail = tail; + + /* Record SW timestamp */ + skb_tx_timestamp(skb); + + /* Verify there is room for another packet */ + fbnic_maybe_stop_tx(skb->dev, ring, FBNIC_MAX_SKB_DESC); + + if (fbnic_tx_sent_queue(skb, ring)) { + *meta |= cpu_to_le64(FBNIC_TWD_FLAG_REQ_COMPLETION); + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(tail, ring->doorbell); + } + + return false; +dma_error: + if (net_ratelimit()) + netdev_err(skb->dev, "TX DMA map failed\n"); + + while (tail != first) { + tail--; + tail &= ring->size_mask; + twd = &ring->desc[tail]; + if (tail == first) + fbnic_unmap_single_twd(dev, twd); + else if (ring->tx_buf[tail] == FBNIC_XMIT_NOUNMAP) + ring->tx_buf[tail] = NULL; + else + fbnic_unmap_page_twd(dev, twd); + } + + return true; +} + +#define FBNIC_MIN_FRAME_LEN 60 + +static netdev_tx_t +fbnic_xmit_frame_ring(struct sk_buff *skb, struct fbnic_ring *ring) +{ + __le64 *meta = &ring->desc[ring->tail]; + u16 desc_needed; + + if (skb_put_padto(skb, FBNIC_MIN_FRAME_LEN)) + goto err_count; + + /* Need: 1 descriptor per page, + * + 1 desc for skb_head, + * + 2 desc for metadata and timestamp metadata + * + 7 desc gap to keep tail from touching head + * otherwise try next time + */ + desc_needed = skb_shinfo(skb)->nr_frags + 10; + if (fbnic_maybe_stop_tx(skb->dev, ring, desc_needed)) + return NETDEV_TX_BUSY; + + *meta = cpu_to_le64(FBNIC_TWD_FLAG_DEST_MAC); + + /* Write all members within DWORD to condense this into 2 4B writes */ + FBNIC_XMIT_CB(skb)->bytecount = skb->len; + FBNIC_XMIT_CB(skb)->gso_segs = 1; + FBNIC_XMIT_CB(skb)->desc_count = 0; + FBNIC_XMIT_CB(skb)->flags = 0; + + if (fbnic_tx_offloads(ring, skb, meta)) + goto err_free; + + if (fbnic_tx_map(ring, skb, meta)) + goto err_free; + + return NETDEV_TX_OK; + +err_free: + dev_kfree_skb_any(skb); +err_count: + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.dropped++; + u64_stats_update_end(&ring->stats.syncp); + return NETDEV_TX_OK; +} + +netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev) +{ + struct fbnic_net *fbn = netdev_priv(dev); + unsigned int q_map = skb->queue_mapping; + + return fbnic_xmit_frame_ring(skb, fbn->tx[q_map]); +} + +static netdev_features_t +fbnic_features_check_encap_gso(struct sk_buff *skb, struct net_device *dev, + netdev_features_t features, unsigned int l3len) +{ + netdev_features_t skb_gso_features; + struct ipv6hdr *ip6_hdr; + unsigned char l4_hdr; + unsigned int start; + __be16 frag_off; + + /* Require MANGLEID for GSO_PARTIAL of IPv4. + * In theory we could support TSO with single, innermost v4 header + * by pretending everything before it is L2, but that needs to be + * parsed case by case.. so leaving it for when the need arises. + */ + if (!(features & NETIF_F_TSO_MANGLEID)) + features &= ~NETIF_F_TSO; + + skb_gso_features = skb_shinfo(skb)->gso_type; + skb_gso_features <<= NETIF_F_GSO_SHIFT; + + /* We'd only clear the native GSO features, so don't bother validating + * if the match can only be on those supported thru GSO_PARTIAL. + */ + if (!(skb_gso_features & FBNIC_TUN_GSO_FEATURES)) + return features; + + /* We can only do IPv6-in-IPv6, not v4-in-v6. It'd be nice + * to fall back to partial for this, or any failure below. + * This is just an optimization, UDPv4 will be caught later on. + */ + if (skb_gso_features & NETIF_F_TSO) + return features & ~FBNIC_TUN_GSO_FEATURES; + + /* Inner headers multiple of 2 */ + if ((skb_inner_network_header(skb) - skb_network_header(skb)) % 2) + return features & ~FBNIC_TUN_GSO_FEATURES; + + /* Encapsulated GSO packet, make 100% sure it's IPv6-in-IPv6. */ + ip6_hdr = ipv6_hdr(skb); + if (ip6_hdr->version != 6) + return features & ~FBNIC_TUN_GSO_FEATURES; + + l4_hdr = ip6_hdr->nexthdr; + start = (unsigned char *)ip6_hdr - skb->data + sizeof(struct ipv6hdr); + start = ipv6_skip_exthdr(skb, start, &l4_hdr, &frag_off); + if (frag_off || l4_hdr != IPPROTO_IPV6 || + skb->data + start != skb_inner_network_header(skb)) + return features & ~FBNIC_TUN_GSO_FEATURES; + + return features; +} + +netdev_features_t +fbnic_features_check(struct sk_buff *skb, struct net_device *dev, + netdev_features_t features) +{ + unsigned int l2len, l3len; + + if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) + return features; + + l2len = skb_mac_header_len(skb); + l3len = skb_checksum_start(skb) - skb_network_header(skb); + + /* Check header lengths are multiple of 2. + * In case of 6in6 we support longer headers (IHLEN + OHLEN) + * but keep things simple for now, 512B is plenty. + */ + if ((l2len | l3len | skb->csum_offset) % 2 || + !FIELD_FIT(FBNIC_TWD_L2_HLEN_MASK, l2len / 2) || + !FIELD_FIT(FBNIC_TWD_L3_IHLEN_MASK, l3len / 2) || + !FIELD_FIT(FBNIC_TWD_CSUM_OFFSET_MASK, skb->csum_offset / 2)) + return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); + + if (likely(!skb->encapsulation) || !skb_is_gso(skb)) + return features; + + return fbnic_features_check_encap_gso(skb, dev, features, l3len); +} + +static void fbnic_clean_twq0(struct fbnic_napi_vector *nv, int napi_budget, + struct fbnic_ring *ring, bool discard, + unsigned int hw_head) +{ + u64 total_bytes = 0, total_packets = 0, ts_lost = 0; + unsigned int head = ring->head; + struct netdev_queue *txq; + unsigned int clean_desc; + + clean_desc = (hw_head - head) & ring->size_mask; + + while (clean_desc) { + struct sk_buff *skb = ring->tx_buf[head]; + unsigned int desc_cnt; + + desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; + if (desc_cnt > clean_desc) + break; + + if (unlikely(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)) { + FBNIC_XMIT_CB(skb)->hw_head = hw_head; + if (likely(!discard)) + break; + ts_lost++; + } + + ring->tx_buf[head] = NULL; + + clean_desc -= desc_cnt; + + while (!(ring->desc[head] & FBNIC_TWD_TYPE(AL))) { + head++; + head &= ring->size_mask; + desc_cnt--; + } + + fbnic_unmap_single_twd(nv->dev, &ring->desc[head]); + head++; + head &= ring->size_mask; + desc_cnt--; + + while (desc_cnt--) { + if (ring->tx_buf[head] != FBNIC_XMIT_NOUNMAP) + fbnic_unmap_page_twd(nv->dev, + &ring->desc[head]); + else + ring->tx_buf[head] = NULL; + head++; + head &= ring->size_mask; + } + + total_bytes += FBNIC_XMIT_CB(skb)->bytecount; + total_packets += FBNIC_XMIT_CB(skb)->gso_segs; + + napi_consume_skb(skb, napi_budget); + } + + if (!total_bytes) + return; + + ring->head = head; + + txq = txring_txq(nv->napi.dev, ring); + + if (unlikely(discard)) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.dropped += total_packets; + ring->stats.twq.ts_lost += ts_lost; + u64_stats_update_end(&ring->stats.syncp); + + netdev_tx_completed_queue(txq, total_packets, total_bytes); + return; + } + + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.bytes += total_bytes; + ring->stats.packets += total_packets; + u64_stats_update_end(&ring->stats.syncp); + + if (!netif_txq_completed_wake(txq, total_packets, total_bytes, + fbnic_desc_unused(ring), + FBNIC_TX_DESC_WAKEUP)) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.twq.wake++; + u64_stats_update_end(&ring->stats.syncp); + } +} + +static void fbnic_clean_twq1(struct fbnic_napi_vector *nv, bool pp_allow_direct, + struct fbnic_ring *ring, bool discard, + unsigned int hw_head) +{ + u64 total_bytes = 0, total_packets = 0; + unsigned int head = ring->head; + + while (hw_head != head) { + struct page *page; + u64 twd; + + if (unlikely(!(ring->desc[head] & FBNIC_TWD_TYPE(AL)))) + goto next_desc; + + twd = le64_to_cpu(ring->desc[head]); + page = ring->tx_buf[head]; + + /* TYPE_AL is 2, TYPE_LAST_AL is 3. So this trick gives + * us one increment per packet, with no branches. + */ + total_packets += FIELD_GET(FBNIC_TWD_TYPE_MASK, twd) - + FBNIC_TWD_TYPE_AL; + total_bytes += FIELD_GET(FBNIC_TWD_LEN_MASK, twd); + + page_pool_put_page(pp_page_to_nmdesc(page)->pp, page, -1, + pp_allow_direct); +next_desc: + head++; + head &= ring->size_mask; + } + + if (!total_bytes) + return; + + ring->head = head; + + if (discard) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.dropped += total_packets; + u64_stats_update_end(&ring->stats.syncp); + return; + } + + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.bytes += total_bytes; + ring->stats.packets += total_packets; + u64_stats_update_end(&ring->stats.syncp); +} + +static void fbnic_clean_tsq(struct fbnic_napi_vector *nv, + struct fbnic_ring *ring, + u64 tcd, int *ts_head, int *head0) +{ + struct skb_shared_hwtstamps hwtstamp; + struct fbnic_net *fbn; + struct sk_buff *skb; + int head; + u64 ns; + + head = (*ts_head < 0) ? ring->head : *ts_head; + + do { + unsigned int desc_cnt; + + if (head == ring->tail) { + if (unlikely(net_ratelimit())) + netdev_err(nv->napi.dev, + "Tx timestamp without matching packet\n"); + return; + } + + skb = ring->tx_buf[head]; + desc_cnt = FBNIC_XMIT_CB(skb)->desc_count; + + head += desc_cnt; + head &= ring->size_mask; + } while (!(FBNIC_XMIT_CB(skb)->flags & FBNIC_XMIT_CB_TS)); + + fbn = netdev_priv(nv->napi.dev); + ns = fbnic_ts40_to_ns(fbn, FIELD_GET(FBNIC_TCD_TYPE1_TS_MASK, tcd)); + + memset(&hwtstamp, 0, sizeof(hwtstamp)); + hwtstamp.hwtstamp = ns_to_ktime(ns); + + *ts_head = head; + + FBNIC_XMIT_CB(skb)->flags &= ~FBNIC_XMIT_CB_TS; + if (*head0 < 0) { + head = FBNIC_XMIT_CB(skb)->hw_head; + if (head >= 0) + *head0 = head; + } + + skb_tstamp_tx(skb, &hwtstamp); + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.twq.ts_packets++; + u64_stats_update_end(&ring->stats.syncp); +} + +static void fbnic_page_pool_init(struct fbnic_ring *ring, unsigned int idx, + netmem_ref netmem) +{ + struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; + + page_pool_fragment_netmem(netmem, FBNIC_PAGECNT_BIAS_MAX); + rx_buf->pagecnt_bias = FBNIC_PAGECNT_BIAS_MAX; + rx_buf->netmem = netmem; +} + +static struct page * +fbnic_page_pool_get_head(struct fbnic_q_triad *qt, unsigned int idx) +{ + struct fbnic_rx_buf *rx_buf = &qt->sub0.rx_buf[idx]; + + rx_buf->pagecnt_bias--; + + /* sub0 is always fed system pages, from the NAPI-level page_pool */ + return netmem_to_page(rx_buf->netmem); +} + +static netmem_ref +fbnic_page_pool_get_data(struct fbnic_q_triad *qt, unsigned int idx) +{ + struct fbnic_rx_buf *rx_buf = &qt->sub1.rx_buf[idx]; + + rx_buf->pagecnt_bias--; + + return rx_buf->netmem; +} + +static void fbnic_page_pool_drain(struct fbnic_ring *ring, unsigned int idx, + int budget) +{ + struct fbnic_rx_buf *rx_buf = &ring->rx_buf[idx]; + netmem_ref netmem = rx_buf->netmem; + + if (!page_pool_unref_netmem(netmem, rx_buf->pagecnt_bias)) + page_pool_put_unrefed_netmem(ring->page_pool, netmem, -1, + !!budget); + + rx_buf->netmem = 0; +} + +static void fbnic_clean_twq(struct fbnic_napi_vector *nv, int napi_budget, + struct fbnic_q_triad *qt, s32 ts_head, s32 head0, + s32 head1) +{ + if (head0 >= 0) + fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, head0); + else if (ts_head >= 0) + fbnic_clean_twq0(nv, napi_budget, &qt->sub0, false, ts_head); + + if (head1 >= 0) { + qt->cmpl.deferred_head = -1; + if (napi_budget) + fbnic_clean_twq1(nv, true, &qt->sub1, false, head1); + else + qt->cmpl.deferred_head = head1; + } +} + +static void +fbnic_clean_tcq(struct fbnic_napi_vector *nv, struct fbnic_q_triad *qt, + int napi_budget) +{ + struct fbnic_ring *cmpl = &qt->cmpl; + s32 head1 = cmpl->deferred_head; + s32 head0 = -1, ts_head = -1; + __le64 *raw_tcd, done; + u32 head = cmpl->head; + + done = (head & (cmpl->size_mask + 1)) ? 0 : cpu_to_le64(FBNIC_TCD_DONE); + raw_tcd = &cmpl->desc[head & cmpl->size_mask]; + + /* Walk the completion queue collecting the heads reported by NIC */ + while ((*raw_tcd & cpu_to_le64(FBNIC_TCD_DONE)) == done) { + u64 tcd; + + dma_rmb(); + + tcd = le64_to_cpu(*raw_tcd); + + switch (FIELD_GET(FBNIC_TCD_TYPE_MASK, tcd)) { + case FBNIC_TCD_TYPE_0: + if (tcd & FBNIC_TCD_TWQ1) + head1 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD1_MASK, + tcd); + else + head0 = FIELD_GET(FBNIC_TCD_TYPE0_HEAD0_MASK, + tcd); + /* Currently all err status bits are related to + * timestamps and as those have yet to be added + * they are skipped for now. + */ + break; + case FBNIC_TCD_TYPE_1: + if (WARN_ON_ONCE(tcd & FBNIC_TCD_TWQ1)) + break; + + fbnic_clean_tsq(nv, &qt->sub0, tcd, &ts_head, &head0); + break; + default: + break; + } + + raw_tcd++; + head++; + if (!(head & cmpl->size_mask)) { + done ^= cpu_to_le64(FBNIC_TCD_DONE); + raw_tcd = &cmpl->desc[0]; + } + } + + /* Record the current head/tail of the queue */ + if (cmpl->head != head) { + cmpl->head = head; + writel(head & cmpl->size_mask, cmpl->doorbell); + } + + /* Unmap and free processed buffers */ + fbnic_clean_twq(nv, napi_budget, qt, ts_head, head0, head1); +} + +static void fbnic_clean_bdq(struct fbnic_ring *ring, unsigned int hw_head, + int napi_budget) +{ + unsigned int head = ring->head; + + if (head == hw_head) + return; + + do { + fbnic_page_pool_drain(ring, head, napi_budget); + + head++; + head &= ring->size_mask; + } while (head != hw_head); + + ring->head = head; +} + +static void fbnic_bd_prep(struct fbnic_ring *bdq, u16 id, netmem_ref netmem) +{ + __le64 *bdq_desc = &bdq->desc[id * FBNIC_BD_FRAG_COUNT]; + dma_addr_t dma = page_pool_get_dma_addr_netmem(netmem); + u64 bd, i = FBNIC_BD_FRAG_COUNT; + + bd = (FBNIC_BD_PAGE_ADDR_MASK & dma) | + FIELD_PREP(FBNIC_BD_PAGE_ID_MASK, id); + + /* In the case that a page size is larger than 4K we will map a + * single page to multiple fragments. The fragments will be + * FBNIC_BD_FRAG_COUNT in size and the lower n bits will be use + * to indicate the individual fragment IDs. + */ + do { + *bdq_desc = cpu_to_le64(bd); + bd += FIELD_PREP(FBNIC_BD_DESC_ADDR_MASK, 1) | + FIELD_PREP(FBNIC_BD_DESC_ID_MASK, 1); + bdq_desc++; + } while (--i); +} + +static void fbnic_fill_bdq(struct fbnic_ring *bdq) +{ + unsigned int count = fbnic_desc_unused(bdq); + unsigned int i = bdq->tail; + + if (!count) + return; + + do { + netmem_ref netmem; + + netmem = page_pool_dev_alloc_netmems(bdq->page_pool); + if (!netmem) { + u64_stats_update_begin(&bdq->stats.syncp); + bdq->stats.bdq.alloc_failed++; + u64_stats_update_end(&bdq->stats.syncp); + + break; + } + + fbnic_page_pool_init(bdq, i, netmem); + fbnic_bd_prep(bdq, i, netmem); + + i++; + i &= bdq->size_mask; + + count--; + } while (count); + + if (bdq->tail != i) { + bdq->tail = i; + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(i * FBNIC_BD_FRAG_COUNT, bdq->doorbell); + } +} + +static unsigned int fbnic_hdr_pg_start(unsigned int pg_off) +{ + /* The headroom of the first header may be larger than FBNIC_RX_HROOM + * due to alignment. So account for that by just making the page + * offset 0 if we are starting at the first header. + */ + if (ALIGN(FBNIC_RX_HROOM, 128) > FBNIC_RX_HROOM && + pg_off == ALIGN(FBNIC_RX_HROOM, 128)) + return 0; + + return pg_off - FBNIC_RX_HROOM; +} + +static unsigned int fbnic_hdr_pg_end(unsigned int pg_off, unsigned int len) +{ + /* Determine the end of the buffer by finding the start of the next + * and then subtracting the headroom from that frame. + */ + pg_off += len + FBNIC_RX_TROOM + FBNIC_RX_HROOM; + + return ALIGN(pg_off, 128) - FBNIC_RX_HROOM; +} + +static void fbnic_pkt_prepare(struct fbnic_napi_vector *nv, u64 rcd, + struct fbnic_pkt_buff *pkt, + struct fbnic_q_triad *qt) +{ + unsigned int hdr_pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + unsigned int hdr_pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); + struct page *page = fbnic_page_pool_get_head(qt, hdr_pg_idx); + unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); + unsigned int frame_sz, hdr_pg_start, hdr_pg_end, headroom; + unsigned char *hdr_start; + + /* data_hard_start should always be NULL when this is called */ + WARN_ON_ONCE(pkt->buff.data_hard_start); + + /* Short-cut the end calculation if we know page is fully consumed */ + hdr_pg_end = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? + FBNIC_BD_FRAG_SIZE : fbnic_hdr_pg_end(hdr_pg_off, len); + hdr_pg_start = fbnic_hdr_pg_start(hdr_pg_off); + + headroom = hdr_pg_off - hdr_pg_start + FBNIC_RX_PAD; + frame_sz = hdr_pg_end - hdr_pg_start; + xdp_init_buff(&pkt->buff, frame_sz, &qt->xdp_rxq); + hdr_pg_start += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * + FBNIC_BD_FRAG_SIZE; + + /* Sync DMA buffer */ + dma_sync_single_range_for_cpu(nv->dev, page_pool_get_dma_addr(page), + hdr_pg_start, frame_sz, + DMA_BIDIRECTIONAL); + + /* Build frame around buffer */ + hdr_start = page_address(page) + hdr_pg_start; + net_prefetch(pkt->buff.data); + xdp_prepare_buff(&pkt->buff, hdr_start, headroom, + len - FBNIC_RX_PAD, true); + + pkt->hwtstamp = 0; + pkt->add_frag_failed = false; +} + +static void fbnic_add_rx_frag(struct fbnic_napi_vector *nv, u64 rcd, + struct fbnic_pkt_buff *pkt, + struct fbnic_q_triad *qt) +{ + unsigned int pg_idx = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + unsigned int pg_off = FIELD_GET(FBNIC_RCD_AL_BUFF_OFF_MASK, rcd); + unsigned int len = FIELD_GET(FBNIC_RCD_AL_BUFF_LEN_MASK, rcd); + netmem_ref netmem = fbnic_page_pool_get_data(qt, pg_idx); + unsigned int truesize; + bool added; + + truesize = FIELD_GET(FBNIC_RCD_AL_PAGE_FIN, rcd) ? + FBNIC_BD_FRAG_SIZE - pg_off : ALIGN(len, 128); + + pg_off += (FBNIC_RCD_AL_BUFF_FRAG_MASK & rcd) * + FBNIC_BD_FRAG_SIZE; + + /* Sync DMA buffer */ + page_pool_dma_sync_netmem_for_cpu(qt->sub1.page_pool, netmem, + pg_off, truesize); + + added = xdp_buff_add_frag(&pkt->buff, netmem, pg_off, len, truesize); + if (unlikely(!added)) { + pkt->add_frag_failed = true; + netdev_err_once(nv->napi.dev, + "Failed to add fragment to xdp_buff\n"); + } +} + +static void fbnic_put_pkt_buff(struct fbnic_q_triad *qt, + struct fbnic_pkt_buff *pkt, int budget) +{ + struct page *page; + + if (!pkt->buff.data_hard_start) + return; + + if (xdp_buff_has_frags(&pkt->buff)) { + struct skb_shared_info *shinfo; + netmem_ref netmem; + int nr_frags; + + shinfo = xdp_get_shared_info_from_buff(&pkt->buff); + nr_frags = shinfo->nr_frags; + + while (nr_frags--) { + netmem = skb_frag_netmem(&shinfo->frags[nr_frags]); + page_pool_put_full_netmem(qt->sub1.page_pool, netmem, + !!budget); + } + } + + page = virt_to_page(pkt->buff.data_hard_start); + page_pool_put_full_page(qt->sub0.page_pool, page, !!budget); +} + +static struct sk_buff *fbnic_build_skb(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt) +{ + struct sk_buff *skb; + + skb = xdp_build_skb_from_buff(&pkt->buff); + if (!skb) + return NULL; + + /* Add timestamp if present */ + if (pkt->hwtstamp) + skb_hwtstamps(skb)->hwtstamp = pkt->hwtstamp; + + return skb; +} + +static long fbnic_pkt_tx(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt) +{ + struct fbnic_ring *ring = &nv->qt[0].sub1; + int size, offset, nsegs = 1, data_len = 0; + unsigned int tail = ring->tail; + struct skb_shared_info *shinfo; + skb_frag_t *frag = NULL; + struct page *page; + dma_addr_t dma; + __le64 *twd; + + if (unlikely(xdp_buff_has_frags(&pkt->buff))) { + shinfo = xdp_get_shared_info_from_buff(&pkt->buff); + nsegs += shinfo->nr_frags; + data_len = shinfo->xdp_frags_size; + frag = &shinfo->frags[0]; + } + + if (fbnic_desc_unused(ring) < nsegs) { + u64_stats_update_begin(&ring->stats.syncp); + ring->stats.dropped++; + u64_stats_update_end(&ring->stats.syncp); + return -FBNIC_XDP_CONSUME; + } + + page = virt_to_page(pkt->buff.data_hard_start); + offset = offset_in_page(pkt->buff.data); + dma = page_pool_get_dma_addr(page); + + size = pkt->buff.data_end - pkt->buff.data; + + while (nsegs--) { + dma_sync_single_range_for_device(nv->dev, dma, offset, size, + DMA_BIDIRECTIONAL); + dma += offset; + + ring->tx_buf[tail] = page; + + twd = &ring->desc[tail]; + *twd = cpu_to_le64(FIELD_PREP(FBNIC_TWD_ADDR_MASK, dma) | + FIELD_PREP(FBNIC_TWD_LEN_MASK, size) | + FIELD_PREP(FBNIC_TWD_TYPE_MASK, + FBNIC_TWD_TYPE_AL)); + + tail++; + tail &= ring->size_mask; + + if (!data_len) + break; + + offset = skb_frag_off(frag); + page = skb_frag_page(frag); + dma = page_pool_get_dma_addr(page); + + size = skb_frag_size(frag); + data_len -= size; + frag++; + } + + *twd |= FBNIC_TWD_TYPE(LAST_AL); + + ring->tail = tail; + + return -FBNIC_XDP_TX; +} + +static void fbnic_pkt_commit_tail(struct fbnic_napi_vector *nv, + unsigned int pkt_tail) +{ + struct fbnic_ring *ring = &nv->qt[0].sub1; + + /* Force DMA writes to flush before writing to tail */ + dma_wmb(); + + writel(pkt_tail, ring->doorbell); +} + +static struct sk_buff *fbnic_run_xdp(struct fbnic_napi_vector *nv, + struct fbnic_pkt_buff *pkt) +{ + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); + struct bpf_prog *xdp_prog; + int act; + + xdp_prog = READ_ONCE(fbn->xdp_prog); + if (!xdp_prog) + goto xdp_pass; + + /* Should never happen, config paths enforce HDS threshold > MTU */ + if (xdp_buff_has_frags(&pkt->buff) && !xdp_prog->aux->xdp_has_frags) + return ERR_PTR(-FBNIC_XDP_LEN_ERR); + + act = bpf_prog_run_xdp(xdp_prog, &pkt->buff); + switch (act) { + case XDP_PASS: +xdp_pass: + return fbnic_build_skb(nv, pkt); + case XDP_TX: + return ERR_PTR(fbnic_pkt_tx(nv, pkt)); + default: + bpf_warn_invalid_xdp_action(nv->napi.dev, xdp_prog, act); + fallthrough; + case XDP_ABORTED: + trace_xdp_exception(nv->napi.dev, xdp_prog, act); + fallthrough; + case XDP_DROP: + break; + } + + return ERR_PTR(-FBNIC_XDP_CONSUME); +} + +static enum pkt_hash_types fbnic_skb_hash_type(u64 rcd) +{ + return (FBNIC_RCD_META_L4_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L4 : + (FBNIC_RCD_META_L3_TYPE_MASK & rcd) ? PKT_HASH_TYPE_L3 : + PKT_HASH_TYPE_L2; +} + +static void fbnic_rx_tstamp(struct fbnic_napi_vector *nv, u64 rcd, + struct fbnic_pkt_buff *pkt) +{ + struct fbnic_net *fbn; + u64 ns, ts; + + if (!FIELD_GET(FBNIC_RCD_OPT_META_TS, rcd)) + return; + + fbn = netdev_priv(nv->napi.dev); + ts = FIELD_GET(FBNIC_RCD_OPT_META_TS_MASK, rcd); + ns = fbnic_ts40_to_ns(fbn, ts); + + /* Add timestamp to shared info */ + pkt->hwtstamp = ns_to_ktime(ns); +} + +static void fbnic_populate_skb_fields(struct fbnic_napi_vector *nv, + u64 rcd, struct sk_buff *skb, + struct fbnic_q_triad *qt, + u64 *csum_cmpl, u64 *csum_none) +{ + struct net_device *netdev = nv->napi.dev; + struct fbnic_ring *rcq = &qt->cmpl; + + fbnic_rx_csum(rcd, skb, rcq, csum_cmpl, csum_none); + + if (netdev->features & NETIF_F_RXHASH) + skb_set_hash(skb, + FIELD_GET(FBNIC_RCD_META_RSS_HASH_MASK, rcd), + fbnic_skb_hash_type(rcd)); + + skb_record_rx_queue(skb, rcq->q_idx); +} + +static bool fbnic_rcd_metadata_err(u64 rcd) +{ + return !!(FBNIC_RCD_META_UNCORRECTABLE_ERR_MASK & rcd); +} + +static int fbnic_clean_rcq(struct fbnic_napi_vector *nv, + struct fbnic_q_triad *qt, int budget) +{ + unsigned int packets = 0, bytes = 0, dropped = 0, alloc_failed = 0; + u64 csum_complete = 0, csum_none = 0, length_errors = 0; + s32 head0 = -1, head1 = -1, pkt_tail = -1; + struct fbnic_ring *rcq = &qt->cmpl; + struct fbnic_pkt_buff *pkt; + __le64 *raw_rcd, done; + u32 head = rcq->head; + + done = (head & (rcq->size_mask + 1)) ? cpu_to_le64(FBNIC_RCD_DONE) : 0; + raw_rcd = &rcq->desc[head & rcq->size_mask]; + pkt = rcq->pkt; + + /* Walk the completion queue collecting the heads reported by NIC */ + while (likely(packets < budget)) { + struct sk_buff *skb = ERR_PTR(-EINVAL); + u32 pkt_bytes; + u64 rcd; + + if ((*raw_rcd & cpu_to_le64(FBNIC_RCD_DONE)) == done) + break; + + dma_rmb(); + + rcd = le64_to_cpu(*raw_rcd); + + switch (FIELD_GET(FBNIC_RCD_TYPE_MASK, rcd)) { + case FBNIC_RCD_TYPE_HDR_AL: + head0 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + fbnic_pkt_prepare(nv, rcd, pkt, qt); + + break; + case FBNIC_RCD_TYPE_PAY_AL: + head1 = FIELD_GET(FBNIC_RCD_AL_BUFF_PAGE_MASK, rcd); + fbnic_add_rx_frag(nv, rcd, pkt, qt); + + break; + case FBNIC_RCD_TYPE_OPT_META: + /* Only type 0 is currently supported */ + if (FIELD_GET(FBNIC_RCD_OPT_META_TYPE_MASK, rcd)) + break; + + fbnic_rx_tstamp(nv, rcd, pkt); + + /* We currently ignore the action table index */ + break; + case FBNIC_RCD_TYPE_META: + if (likely(!fbnic_rcd_metadata_err(rcd) && + !pkt->add_frag_failed)) { + pkt_bytes = xdp_get_buff_len(&pkt->buff); + skb = fbnic_run_xdp(nv, pkt); + } + + /* Populate skb and invalidate XDP */ + if (!IS_ERR_OR_NULL(skb)) { + fbnic_populate_skb_fields(nv, rcd, skb, qt, + &csum_complete, + &csum_none); + napi_gro_receive(&nv->napi, skb); + } else if (skb == ERR_PTR(-FBNIC_XDP_TX)) { + pkt_tail = nv->qt[0].sub1.tail; + } else if (PTR_ERR(skb) == -FBNIC_XDP_CONSUME) { + fbnic_put_pkt_buff(qt, pkt, 1); + } else { + if (!skb) + alloc_failed++; + + if (skb == ERR_PTR(-FBNIC_XDP_LEN_ERR)) + length_errors++; + else + dropped++; + + fbnic_put_pkt_buff(qt, pkt, 1); + goto next_dont_count; + } + + packets++; + bytes += pkt_bytes; +next_dont_count: + pkt->buff.data_hard_start = NULL; + + break; + } + + raw_rcd++; + head++; + if (!(head & rcq->size_mask)) { + done ^= cpu_to_le64(FBNIC_RCD_DONE); + raw_rcd = &rcq->desc[0]; + } + } + + u64_stats_update_begin(&rcq->stats.syncp); + rcq->stats.packets += packets; + rcq->stats.bytes += bytes; + rcq->stats.dropped += dropped; + rcq->stats.rx.alloc_failed += alloc_failed; + rcq->stats.rx.csum_complete += csum_complete; + rcq->stats.rx.csum_none += csum_none; + rcq->stats.rx.length_errors += length_errors; + u64_stats_update_end(&rcq->stats.syncp); + + if (pkt_tail >= 0) + fbnic_pkt_commit_tail(nv, pkt_tail); + + /* Unmap and free processed buffers */ + if (head0 >= 0) + fbnic_clean_bdq(&qt->sub0, head0, budget); + fbnic_fill_bdq(&qt->sub0); + + if (head1 >= 0) + fbnic_clean_bdq(&qt->sub1, head1, budget); + fbnic_fill_bdq(&qt->sub1); + + /* Record the current head/tail of the queue */ + if (rcq->head != head) { + rcq->head = head; + writel(head & rcq->size_mask, rcq->doorbell); + } + + return packets; +} + +static void fbnic_nv_irq_disable(struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + u32 v_idx = nv->v_idx; + + fbnic_wr32(fbd, FBNIC_INTR_MASK_SET(v_idx / 32), 1 << (v_idx % 32)); +} + +static void fbnic_nv_irq_rearm(struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + u32 v_idx = nv->v_idx; + + fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(v_idx), + FBNIC_INTR_CQ_REARM_INTR_UNMASK); +} + +static int fbnic_poll(struct napi_struct *napi, int budget) +{ + struct fbnic_napi_vector *nv = container_of(napi, + struct fbnic_napi_vector, + napi); + int i, j, work_done = 0; + + for (i = 0; i < nv->txt_count; i++) + fbnic_clean_tcq(nv, &nv->qt[i], budget); + + for (j = 0; j < nv->rxt_count; j++, i++) + work_done += fbnic_clean_rcq(nv, &nv->qt[i], budget); + + if (work_done >= budget) + return budget; + + if (likely(napi_complete_done(napi, work_done))) + fbnic_nv_irq_rearm(nv); + + return work_done; +} + +irqreturn_t fbnic_msix_clean_rings(int __always_unused irq, void *data) +{ + struct fbnic_napi_vector *nv = *(void **)data; + + napi_schedule_irqoff(&nv->napi); + + return IRQ_HANDLED; +} + +void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + struct fbnic_queue_stats *stats = &rxr->stats; + + /* Capture stats from queues before dissasociating them */ + fbn->rx_stats.bytes += stats->bytes; + fbn->rx_stats.packets += stats->packets; + fbn->rx_stats.dropped += stats->dropped; + fbn->rx_stats.rx.alloc_failed += stats->rx.alloc_failed; + fbn->rx_stats.rx.csum_complete += stats->rx.csum_complete; + fbn->rx_stats.rx.csum_none += stats->rx.csum_none; + fbn->rx_stats.rx.length_errors += stats->rx.length_errors; + /* Remember to add new stats here */ + BUILD_BUG_ON(sizeof(fbn->rx_stats.rx) / 8 != 4); +} + +void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn, + struct fbnic_ring *bdq) +{ + struct fbnic_queue_stats *stats = &bdq->stats; + + /* Capture stats from queues before dissasociating them */ + fbn->bdq_stats.bdq.alloc_failed += stats->bdq.alloc_failed; + /* Remember to add new stats here */ + BUILD_BUG_ON(sizeof(fbn->rx_stats.bdq) / 8 != 1); +} + +void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + struct fbnic_queue_stats *stats = &txr->stats; + + /* Capture stats from queues before dissasociating them */ + fbn->tx_stats.bytes += stats->bytes; + fbn->tx_stats.packets += stats->packets; + fbn->tx_stats.dropped += stats->dropped; + fbn->tx_stats.twq.csum_partial += stats->twq.csum_partial; + fbn->tx_stats.twq.lso += stats->twq.lso; + fbn->tx_stats.twq.ts_lost += stats->twq.ts_lost; + fbn->tx_stats.twq.ts_packets += stats->twq.ts_packets; + fbn->tx_stats.twq.stop += stats->twq.stop; + fbn->tx_stats.twq.wake += stats->twq.wake; + /* Remember to add new stats here */ + BUILD_BUG_ON(sizeof(fbn->tx_stats.twq) / 8 != 6); +} + +void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn, + struct fbnic_ring *xdpr) +{ + struct fbnic_queue_stats *stats = &xdpr->stats; + + if (!(xdpr->flags & FBNIC_RING_F_STATS)) + return; + + /* Capture stats from queues before dissasociating them */ + fbn->tx_stats.dropped += stats->dropped; + fbn->tx_stats.bytes += stats->bytes; + fbn->tx_stats.packets += stats->packets; +} + +static void fbnic_remove_tx_ring(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + if (!(txr->flags & FBNIC_RING_F_STATS)) + return; + + fbnic_aggregate_ring_tx_counters(fbn, txr); + + /* Remove pointer to the Tx ring */ + WARN_ON(fbn->tx[txr->q_idx] && fbn->tx[txr->q_idx] != txr); + fbn->tx[txr->q_idx] = NULL; +} + +static void fbnic_remove_xdp_ring(struct fbnic_net *fbn, + struct fbnic_ring *xdpr) +{ + if (!(xdpr->flags & FBNIC_RING_F_STATS)) + return; + + fbnic_aggregate_ring_xdp_counters(fbn, xdpr); + + /* Remove pointer to the Tx ring */ + WARN_ON(fbn->tx[xdpr->q_idx] && fbn->tx[xdpr->q_idx] != xdpr); + fbn->tx[xdpr->q_idx] = NULL; +} + +static void fbnic_remove_rx_ring(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + if (!(rxr->flags & FBNIC_RING_F_STATS)) + return; + + fbnic_aggregate_ring_rx_counters(fbn, rxr); + + /* Remove pointer to the Rx ring */ + WARN_ON(fbn->rx[rxr->q_idx] && fbn->rx[rxr->q_idx] != rxr); + fbn->rx[rxr->q_idx] = NULL; +} + +static void fbnic_remove_bdq_ring(struct fbnic_net *fbn, + struct fbnic_ring *bdq) +{ + if (!(bdq->flags & FBNIC_RING_F_STATS)) + return; + + fbnic_aggregate_ring_bdq_counters(fbn, bdq); +} + +static void fbnic_free_qt_page_pools(struct fbnic_q_triad *qt) +{ + page_pool_destroy(qt->sub0.page_pool); + page_pool_destroy(qt->sub1.page_pool); +} + +static void fbnic_free_napi_vector(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = nv->fbd; + int i, j; + + for (i = 0; i < nv->txt_count; i++) { + fbnic_remove_tx_ring(fbn, &nv->qt[i].sub0); + fbnic_remove_xdp_ring(fbn, &nv->qt[i].sub1); + fbnic_remove_tx_ring(fbn, &nv->qt[i].cmpl); + } + + for (j = 0; j < nv->rxt_count; j++, i++) { + fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub0); + fbnic_remove_bdq_ring(fbn, &nv->qt[i].sub1); + fbnic_remove_rx_ring(fbn, &nv->qt[i].cmpl); + } + + fbnic_napi_free_irq(fbd, nv); + netif_napi_del_locked(&nv->napi); + fbn->napi[fbnic_napi_idx(nv)] = NULL; + kfree(nv); +} + +void fbnic_free_napi_vectors(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + if (fbn->napi[i]) + fbnic_free_napi_vector(fbn, fbn->napi[i]); +} + +static int +fbnic_alloc_qt_page_pools(struct fbnic_net *fbn, struct fbnic_q_triad *qt, + unsigned int rxq_idx) +{ + struct page_pool_params pp_params = { + .order = 0, + .flags = PP_FLAG_DMA_MAP | + PP_FLAG_DMA_SYNC_DEV, + .pool_size = fbn->hpq_size + fbn->ppq_size, + .nid = NUMA_NO_NODE, + .dev = fbn->netdev->dev.parent, + .dma_dir = DMA_BIDIRECTIONAL, + .offset = 0, + .max_len = PAGE_SIZE, + .netdev = fbn->netdev, + .queue_idx = rxq_idx, + }; + struct page_pool *pp; + + /* Page pool cannot exceed a size of 32768. This doesn't limit the + * pages on the ring but the number we can have cached waiting on + * the next use. + * + * TBD: Can this be reduced further? Would a multiple of + * NAPI_POLL_WEIGHT possibly make more sense? The question is how + * may pages do we need to hold in reserve to get the best return + * without hogging too much system memory. + */ + if (pp_params.pool_size > 32768) + pp_params.pool_size = 32768; + + pp = page_pool_create(&pp_params); + if (IS_ERR(pp)) + return PTR_ERR(pp); + + qt->sub0.page_pool = pp; + if (netif_rxq_has_unreadable_mp(fbn->netdev, rxq_idx)) { + pp_params.flags |= PP_FLAG_ALLOW_UNREADABLE_NETMEM; + pp_params.dma_dir = DMA_FROM_DEVICE; + + pp = page_pool_create(&pp_params); + if (IS_ERR(pp)) + goto err_destroy_sub0; + } else { + page_pool_get(pp); + } + qt->sub1.page_pool = pp; + + return 0; + +err_destroy_sub0: + page_pool_destroy(pp); + return PTR_ERR(pp); +} + +static void fbnic_ring_init(struct fbnic_ring *ring, u32 __iomem *doorbell, + int q_idx, u8 flags) +{ + u64_stats_init(&ring->stats.syncp); + ring->doorbell = doorbell; + ring->q_idx = q_idx; + ring->flags = flags; + ring->deferred_head = -1; +} + +static int fbnic_alloc_napi_vector(struct fbnic_dev *fbd, struct fbnic_net *fbn, + unsigned int v_count, unsigned int v_idx, + unsigned int txq_count, unsigned int txq_idx, + unsigned int rxq_count, unsigned int rxq_idx) +{ + int txt_count = txq_count, rxt_count = rxq_count; + u32 __iomem *uc_addr = fbd->uc_addr0; + int xdp_count = 0, qt_count, err; + struct fbnic_napi_vector *nv; + struct fbnic_q_triad *qt; + u32 __iomem *db; + + /* We need to reserve at least one Tx Queue Triad for an XDP ring */ + if (rxq_count) { + xdp_count = 1; + if (!txt_count) + txt_count = 1; + } + + qt_count = txt_count + rxq_count; + if (!qt_count) + return -EINVAL; + + /* If MMIO has already failed there are no rings to initialize */ + if (!uc_addr) + return -EIO; + + /* Allocate NAPI vector and queue triads */ + nv = kzalloc_flex(*nv, qt, qt_count); + if (!nv) + return -ENOMEM; + + /* Record queue triad counts */ + nv->txt_count = txt_count; + nv->rxt_count = rxt_count; + + /* Provide pointer back to fbnic and MSI-X vectors */ + nv->fbd = fbd; + nv->v_idx = v_idx; + + /* Tie napi to netdev */ + fbn->napi[fbnic_napi_idx(nv)] = nv; + netif_napi_add_config_locked(fbn->netdev, &nv->napi, fbnic_poll, + fbnic_napi_idx(nv)); + + /* Record IRQ to NAPI struct */ + netif_napi_set_irq_locked(&nv->napi, + pci_irq_vector(to_pci_dev(fbd->dev), + nv->v_idx)); + + /* Tie nv back to PCIe dev */ + nv->dev = fbd->dev; + + /* Request the IRQ for napi vector */ + err = fbnic_napi_request_irq(fbd, nv); + if (err) + goto napi_del; + + /* Initialize queue triads */ + qt = nv->qt; + + while (txt_count) { + u8 flags = FBNIC_RING_F_CTX | FBNIC_RING_F_STATS; + + /* Configure Tx queue */ + db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ0_TAIL]; + + /* Assign Tx queue to netdev if applicable */ + if (txq_count > 0) { + + fbnic_ring_init(&qt->sub0, db, txq_idx, flags); + fbn->tx[txq_idx] = &qt->sub0; + txq_count--; + } else { + fbnic_ring_init(&qt->sub0, db, 0, + FBNIC_RING_F_DISABLED); + } + + /* Configure XDP queue */ + db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TWQ1_TAIL]; + + /* Assign XDP queue to netdev if applicable + * + * The setup for this is in itself a bit different. + * 1. We only need one XDP Tx queue per NAPI vector. + * 2. We associate it to the first Rx queue index. + * 3. The hardware side is associated based on the Tx Queue. + * 4. The netdev queue is offset by FBNIC_MAX_TXQs. + */ + if (xdp_count > 0) { + unsigned int xdp_idx = FBNIC_MAX_TXQS + rxq_idx; + + fbnic_ring_init(&qt->sub1, db, xdp_idx, flags); + fbn->tx[xdp_idx] = &qt->sub1; + xdp_count--; + } else { + fbnic_ring_init(&qt->sub1, db, 0, + FBNIC_RING_F_DISABLED); + } + + /* Configure Tx completion queue */ + db = &uc_addr[FBNIC_QUEUE(txq_idx) + FBNIC_QUEUE_TCQ_HEAD]; + fbnic_ring_init(&qt->cmpl, db, 0, 0); + + /* Update Tx queue index */ + txt_count--; + txq_idx += v_count; + + /* Move to next queue triad */ + qt++; + } + + while (rxt_count) { + /* Configure header queue */ + db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_HPQ_TAIL]; + fbnic_ring_init(&qt->sub0, db, 0, + FBNIC_RING_F_CTX | FBNIC_RING_F_STATS); + + /* Configure payload queue */ + db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_BDQ_PPQ_TAIL]; + fbnic_ring_init(&qt->sub1, db, 0, + FBNIC_RING_F_CTX | FBNIC_RING_F_STATS); + + /* Configure Rx completion queue */ + db = &uc_addr[FBNIC_QUEUE(rxq_idx) + FBNIC_QUEUE_RCQ_HEAD]; + fbnic_ring_init(&qt->cmpl, db, rxq_idx, FBNIC_RING_F_STATS); + fbn->rx[rxq_idx] = &qt->cmpl; + + /* Update Rx queue index */ + rxt_count--; + rxq_idx += v_count; + + /* Move to next queue triad */ + qt++; + } + + return 0; + +napi_del: + netif_napi_del_locked(&nv->napi); + fbn->napi[fbnic_napi_idx(nv)] = NULL; + kfree(nv); + return err; +} + +int fbnic_alloc_napi_vectors(struct fbnic_net *fbn) +{ + unsigned int txq_idx = 0, rxq_idx = 0, v_idx = FBNIC_NON_NAPI_VECTORS; + unsigned int num_tx = fbn->num_tx_queues; + unsigned int num_rx = fbn->num_rx_queues; + unsigned int num_napi = fbn->num_napi; + struct fbnic_dev *fbd = fbn->fbd; + int err; + + /* Allocate 1 Tx queue per napi vector */ + if (num_napi < FBNIC_MAX_TXQS && num_napi == num_tx + num_rx) { + while (num_tx) { + err = fbnic_alloc_napi_vector(fbd, fbn, + num_napi, v_idx, + 1, txq_idx, 0, 0); + if (err) + goto free_vectors; + + /* Update counts and index */ + num_tx--; + txq_idx++; + + v_idx++; + } + } + + /* Allocate Tx/Rx queue pairs per vector, or allocate remaining Rx */ + while (num_rx | num_tx) { + int tqpv = DIV_ROUND_UP(num_tx, num_napi - txq_idx); + int rqpv = DIV_ROUND_UP(num_rx, num_napi - rxq_idx); + + err = fbnic_alloc_napi_vector(fbd, fbn, num_napi, v_idx, + tqpv, txq_idx, rqpv, rxq_idx); + if (err) + goto free_vectors; + + /* Update counts and index */ + num_tx -= tqpv; + txq_idx++; + + num_rx -= rqpv; + rxq_idx++; + + v_idx++; + } + + return 0; + +free_vectors: + fbnic_free_napi_vectors(fbn); + + return err; +} + +static void fbnic_free_ring_resources(struct device *dev, + struct fbnic_ring *ring) +{ + kvfree(ring->buffer); + ring->buffer = NULL; + + /* If size is not set there are no descriptors present */ + if (!ring->size) + return; + + dma_free_coherent(dev, ring->size, ring->desc, ring->dma); + ring->size_mask = 0; + ring->size = 0; +} + +static int fbnic_alloc_tx_ring_desc(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + struct device *dev = fbn->netdev->dev.parent; + size_t size; + + /* Round size up to nearest 4K */ + size = ALIGN(array_size(sizeof(*txr->desc), fbn->txq_size), 4096); + + txr->desc = dma_alloc_coherent(dev, size, &txr->dma, + GFP_KERNEL | __GFP_NOWARN); + if (!txr->desc) + return -ENOMEM; + + /* txq_size should be a power of 2, so mask is just that -1 */ + txr->size_mask = fbn->txq_size - 1; + txr->size = size; + + return 0; +} + +static int fbnic_alloc_tx_ring_buffer(struct fbnic_ring *txr) +{ + size_t size = array_size(sizeof(*txr->tx_buf), txr->size_mask + 1); + + txr->tx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); + + return txr->tx_buf ? 0 : -ENOMEM; +} + +static int fbnic_alloc_tx_ring_resources(struct fbnic_net *fbn, + struct fbnic_ring *txr) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + if (txr->flags & FBNIC_RING_F_DISABLED) + return 0; + + err = fbnic_alloc_tx_ring_desc(fbn, txr); + if (err) + return err; + + if (!(txr->flags & FBNIC_RING_F_CTX)) + return 0; + + err = fbnic_alloc_tx_ring_buffer(txr); + if (err) + goto free_desc; + + return 0; + +free_desc: + fbnic_free_ring_resources(dev, txr); + return err; +} + +static int fbnic_alloc_rx_ring_desc(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + struct device *dev = fbn->netdev->dev.parent; + size_t desc_size = sizeof(*rxr->desc); + u32 rxq_size; + size_t size; + + switch (rxr->doorbell - fbnic_ring_csr_base(rxr)) { + case FBNIC_QUEUE_BDQ_HPQ_TAIL: + rxq_size = fbn->hpq_size / FBNIC_BD_FRAG_COUNT; + desc_size *= FBNIC_BD_FRAG_COUNT; + break; + case FBNIC_QUEUE_BDQ_PPQ_TAIL: + rxq_size = fbn->ppq_size / FBNIC_BD_FRAG_COUNT; + desc_size *= FBNIC_BD_FRAG_COUNT; + break; + case FBNIC_QUEUE_RCQ_HEAD: + rxq_size = fbn->rcq_size; + break; + default: + return -EINVAL; + } + + /* Round size up to nearest 4K */ + size = ALIGN(array_size(desc_size, rxq_size), 4096); + + rxr->desc = dma_alloc_coherent(dev, size, &rxr->dma, + GFP_KERNEL | __GFP_NOWARN); + if (!rxr->desc) + return -ENOMEM; + + /* rxq_size should be a power of 2, so mask is just that -1 */ + rxr->size_mask = rxq_size - 1; + rxr->size = size; + + return 0; +} + +static int fbnic_alloc_rx_ring_buffer(struct fbnic_ring *rxr) +{ + size_t size = array_size(sizeof(*rxr->rx_buf), rxr->size_mask + 1); + + if (rxr->flags & FBNIC_RING_F_CTX) + size = sizeof(*rxr->rx_buf) * (rxr->size_mask + 1); + else + size = sizeof(*rxr->pkt); + + rxr->rx_buf = kvzalloc(size, GFP_KERNEL | __GFP_NOWARN); + + return rxr->rx_buf ? 0 : -ENOMEM; +} + +static int fbnic_alloc_rx_ring_resources(struct fbnic_net *fbn, + struct fbnic_ring *rxr) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + err = fbnic_alloc_rx_ring_desc(fbn, rxr); + if (err) + return err; + + err = fbnic_alloc_rx_ring_buffer(rxr); + if (err) + goto free_desc; + + return 0; + +free_desc: + fbnic_free_ring_resources(dev, rxr); + return err; +} + +static void fbnic_free_qt_resources(struct fbnic_net *fbn, + struct fbnic_q_triad *qt) +{ + struct device *dev = fbn->netdev->dev.parent; + + fbnic_free_ring_resources(dev, &qt->cmpl); + fbnic_free_ring_resources(dev, &qt->sub1); + fbnic_free_ring_resources(dev, &qt->sub0); + + if (xdp_rxq_info_is_reg(&qt->xdp_rxq)) { + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); + xdp_rxq_info_unreg(&qt->xdp_rxq); + fbnic_free_qt_page_pools(qt); + } +} + +static int fbnic_alloc_tx_qt_resources(struct fbnic_net *fbn, + struct fbnic_q_triad *qt) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub0); + if (err) + return err; + + err = fbnic_alloc_tx_ring_resources(fbn, &qt->sub1); + if (err) + goto free_sub0; + + err = fbnic_alloc_tx_ring_resources(fbn, &qt->cmpl); + if (err) + goto free_sub1; + + return 0; + +free_sub1: + fbnic_free_ring_resources(dev, &qt->sub1); +free_sub0: + fbnic_free_ring_resources(dev, &qt->sub0); + return err; +} + +static int fbnic_alloc_rx_qt_resources(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv, + struct fbnic_q_triad *qt) +{ + struct device *dev = fbn->netdev->dev.parent; + int err; + + err = fbnic_alloc_qt_page_pools(fbn, qt, qt->cmpl.q_idx); + if (err) + return err; + + err = xdp_rxq_info_reg(&qt->xdp_rxq, fbn->netdev, qt->sub0.q_idx, + nv->napi.napi_id); + if (err) + goto free_page_pools; + + err = xdp_rxq_info_reg_mem_model(&qt->xdp_rxq, MEM_TYPE_PAGE_POOL, + qt->sub0.page_pool); + if (err) + goto unreg_rxq; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub0); + if (err) + goto unreg_mm; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->sub1); + if (err) + goto free_sub0; + + err = fbnic_alloc_rx_ring_resources(fbn, &qt->cmpl); + if (err) + goto free_sub1; + + return 0; + +free_sub1: + fbnic_free_ring_resources(dev, &qt->sub1); +free_sub0: + fbnic_free_ring_resources(dev, &qt->sub0); +unreg_mm: + xdp_rxq_info_unreg_mem_model(&qt->xdp_rxq); +unreg_rxq: + xdp_rxq_info_unreg(&qt->xdp_rxq); +free_page_pools: + fbnic_free_qt_page_pools(qt); + return err; +} + +static void fbnic_free_nv_resources(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + int i; + + for (i = 0; i < nv->txt_count + nv->rxt_count; i++) + fbnic_free_qt_resources(fbn, &nv->qt[i]); +} + +static int fbnic_alloc_nv_resources(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + int i, j, err; + + /* Allocate Tx Resources */ + for (i = 0; i < nv->txt_count; i++) { + err = fbnic_alloc_tx_qt_resources(fbn, &nv->qt[i]); + if (err) + goto free_qt_resources; + } + + /* Allocate Rx Resources */ + for (j = 0; j < nv->rxt_count; j++, i++) { + err = fbnic_alloc_rx_qt_resources(fbn, nv, &nv->qt[i]); + if (err) + goto free_qt_resources; + } + + return 0; + +free_qt_resources: + while (i--) + fbnic_free_qt_resources(fbn, &nv->qt[i]); + return err; +} + +void fbnic_free_resources(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_free_nv_resources(fbn, fbn->napi[i]); +} + +int fbnic_alloc_resources(struct fbnic_net *fbn) +{ + int i, err = -ENODEV; + + for (i = 0; i < fbn->num_napi; i++) { + err = fbnic_alloc_nv_resources(fbn, fbn->napi[i]); + if (err) + goto free_resources; + } + + return 0; + +free_resources: + while (i--) + fbnic_free_nv_resources(fbn, fbn->napi[i]); + + return err; +} + +static void fbnic_set_netif_napi(struct fbnic_napi_vector *nv) +{ + int i, j; + + /* Associate Tx queue with NAPI */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, &nv->napi); + } + + /* Associate Rx queue with NAPI */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, + NETDEV_QUEUE_TYPE_RX, &nv->napi); + } +} + +static void fbnic_reset_netif_napi(struct fbnic_napi_vector *nv) +{ + int i, j; + + /* Disassociate Tx queue from NAPI */ + for (i = 0; i < nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + netif_queue_set_napi(nv->napi.dev, qt->sub0.q_idx, + NETDEV_QUEUE_TYPE_TX, NULL); + } + + /* Disassociate Rx queue from NAPI */ + for (j = 0; j < nv->rxt_count; j++, i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + netif_queue_set_napi(nv->napi.dev, qt->cmpl.q_idx, + NETDEV_QUEUE_TYPE_RX, NULL); + } +} + +int fbnic_set_netif_queues(struct fbnic_net *fbn) +{ + int i, err; + + err = netif_set_real_num_queues(fbn->netdev, fbn->num_tx_queues, + fbn->num_rx_queues); + if (err) + return err; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_set_netif_napi(fbn->napi[i]); + + return 0; +} + +void fbnic_reset_netif_queues(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_reset_netif_napi(fbn->napi[i]); +} + +static void fbnic_disable_twq0(struct fbnic_ring *txr) +{ + u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ0_CTL); + + twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; + + fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ0_CTL, twq_ctl); +} + +static void fbnic_disable_twq1(struct fbnic_ring *txr) +{ + u32 twq_ctl = fbnic_ring_rd32(txr, FBNIC_QUEUE_TWQ1_CTL); + + twq_ctl &= ~FBNIC_QUEUE_TWQ_CTL_ENABLE; + + fbnic_ring_wr32(txr, FBNIC_QUEUE_TWQ1_CTL, twq_ctl); +} + +static void fbnic_disable_tcq(struct fbnic_ring *txr) +{ + fbnic_ring_wr32(txr, FBNIC_QUEUE_TCQ_CTL, 0); + fbnic_ring_wr32(txr, FBNIC_QUEUE_TIM_MASK, FBNIC_QUEUE_TIM_MASK_MASK); +} + +static void fbnic_disable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) +{ + u32 bdq_ctl = fbnic_ring_rd32(hpq, FBNIC_QUEUE_BDQ_CTL); + + bdq_ctl &= ~FBNIC_QUEUE_BDQ_CTL_ENABLE; + + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); +} + +static void fbnic_disable_rcq(struct fbnic_ring *rxr) +{ + fbnic_ring_wr32(rxr, FBNIC_QUEUE_RCQ_CTL, 0); + fbnic_ring_wr32(rxr, FBNIC_QUEUE_RIM_MASK, FBNIC_QUEUE_RIM_MASK_MASK); +} + +void fbnic_napi_disable(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) { + napi_disable_locked(&fbn->napi[i]->napi); + + fbnic_nv_irq_disable(fbn->napi[i]); + } +} + +static void __fbnic_nv_disable(struct fbnic_napi_vector *nv) +{ + int i, t; + + /* Disable Tx queue triads */ + for (t = 0; t < nv->txt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + fbnic_disable_twq0(&qt->sub0); + fbnic_disable_twq1(&qt->sub1); + fbnic_disable_tcq(&qt->cmpl); + } + + /* Disable Rx queue triads */ + for (i = 0; i < nv->rxt_count; i++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + fbnic_disable_bdq(&qt->sub0, &qt->sub1); + fbnic_disable_rcq(&qt->cmpl); + } +} + +static void +fbnic_nv_disable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) +{ + __fbnic_nv_disable(nv); + fbnic_wrfl(fbn->fbd); +} + +void fbnic_dbg_down(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_dbg_nv_exit(fbn->napi[i]); +} + +void fbnic_dbg_up(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_dbg_nv_init(fbn->napi[i]); +} + +void fbnic_disable(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i; + + for (i = 0; i < fbn->num_napi; i++) + __fbnic_nv_disable(fbn->napi[i]); + + fbnic_wrfl(fbd); +} + +static void fbnic_tx_flush(struct fbnic_dev *fbd) +{ + netdev_warn(fbd->netdev, "triggering Tx flush\n"); + + fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, + FBNIC_TMI_DROP_CTRL_EN); +} + +static void fbnic_tx_flush_off(struct fbnic_dev *fbd) +{ + fbnic_rmw32(fbd, FBNIC_TMI_DROP_CTRL, FBNIC_TMI_DROP_CTRL_EN, 0); +} + +struct fbnic_idle_regs { + u32 reg_base; + u8 reg_cnt; +}; + +static bool fbnic_all_idle(struct fbnic_dev *fbd, + const struct fbnic_idle_regs *regs, + unsigned int nregs) +{ + unsigned int i, j; + + for (i = 0; i < nregs; i++) { + for (j = 0; j < regs[i].reg_cnt; j++) { + if (fbnic_rd32(fbd, regs[i].reg_base + j) != ~0U) + return false; + } + } + return true; +} + +static void fbnic_idle_dump(struct fbnic_dev *fbd, + const struct fbnic_idle_regs *regs, + unsigned int nregs, const char *dir, int err) +{ + unsigned int i, j; + + netdev_err(fbd->netdev, "error waiting for %s idle %d\n", dir, err); + for (i = 0; i < nregs; i++) + for (j = 0; j < regs[i].reg_cnt; j++) + netdev_err(fbd->netdev, "0x%04x: %08x\n", + regs[i].reg_base + j, + fbnic_rd32(fbd, regs[i].reg_base + j)); +} + +int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail) +{ + static const struct fbnic_idle_regs tx[] = { + { FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TWQ_IDLE_CNT, }, + { FBNIC_QM_TQS_IDLE(0), FBNIC_QM_TQS_IDLE_CNT, }, + { FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TDE_IDLE_CNT, }, + { FBNIC_QM_TCQ_IDLE(0), FBNIC_QM_TCQ_IDLE_CNT, }, + }, rx[] = { + { FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_HPQ_IDLE_CNT, }, + { FBNIC_QM_PPQ_IDLE(0), FBNIC_QM_PPQ_IDLE_CNT, }, + { FBNIC_QM_RCQ_IDLE(0), FBNIC_QM_RCQ_IDLE_CNT, }, + }; + bool idle; + int err; + + err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, + false, fbd, tx, ARRAY_SIZE(tx)); + if (err == -ETIMEDOUT) { + fbnic_tx_flush(fbd); + err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, + 2, 500000, false, + fbd, tx, ARRAY_SIZE(tx)); + fbnic_tx_flush_off(fbd); + } + if (err) { + fbnic_idle_dump(fbd, tx, ARRAY_SIZE(tx), "Tx", err); + if (may_fail) + return err; + } + + err = read_poll_timeout_atomic(fbnic_all_idle, idle, idle, 2, 500000, + false, fbd, rx, ARRAY_SIZE(rx)); + if (err) + fbnic_idle_dump(fbd, rx, ARRAY_SIZE(rx), "Rx", err); + return err; +} + +static int +fbnic_wait_queue_idle(struct fbnic_net *fbn, bool rx, unsigned int idx) +{ + static const unsigned int tx_regs[] = { + FBNIC_QM_TWQ_IDLE(0), FBNIC_QM_TQS_IDLE(0), + FBNIC_QM_TDE_IDLE(0), FBNIC_QM_TCQ_IDLE(0), + }, rx_regs[] = { + FBNIC_QM_HPQ_IDLE(0), FBNIC_QM_PPQ_IDLE(0), + FBNIC_QM_RCQ_IDLE(0), + }; + struct fbnic_dev *fbd = fbn->fbd; + unsigned int val, mask, off; + const unsigned int *regs; + unsigned int reg_cnt; + int i, err; + + regs = rx ? rx_regs : tx_regs; + reg_cnt = rx ? ARRAY_SIZE(rx_regs) : ARRAY_SIZE(tx_regs); + + off = idx / 32; + mask = BIT(idx % 32); + + for (i = 0; i < reg_cnt; i++) { + err = read_poll_timeout_atomic(fbnic_rd32, val, val & mask, + 2, 500000, false, + fbd, regs[i] + off); + if (err) { + netdev_err(fbd->netdev, + "wait for queue %s%d idle failed 0x%04x(%d): %08x (mask: %08x)\n", + rx ? "Rx" : "Tx", idx, regs[i] + off, i, + val, mask); + return err; + } + } + + return 0; +} + +static void fbnic_nv_flush(struct fbnic_napi_vector *nv) +{ + int j, t; + + /* Flush any processed Tx Queue Triads and drop the rest */ + for (t = 0; t < nv->txt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + struct netdev_queue *tx_queue; + + /* Clean the work queues of unprocessed work */ + fbnic_clean_twq0(nv, 0, &qt->sub0, true, qt->sub0.tail); + fbnic_clean_twq1(nv, false, &qt->sub1, true, + qt->sub1.tail); + + /* Reset completion queue descriptor ring */ + memset(qt->cmpl.desc, 0, qt->cmpl.size); + + /* Nothing else to do if Tx queue is disabled */ + if (qt->sub0.flags & FBNIC_RING_F_DISABLED) + continue; + + /* Reset BQL associated with Tx queue */ + tx_queue = netdev_get_tx_queue(nv->napi.dev, + qt->sub0.q_idx); + netdev_tx_reset_queue(tx_queue); + } + + /* Flush any processed Rx Queue Triads and drop the rest */ + for (j = 0; j < nv->rxt_count; j++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + /* Clean the work queues of unprocessed work */ + fbnic_clean_bdq(&qt->sub0, qt->sub0.tail, 0); + fbnic_clean_bdq(&qt->sub1, qt->sub1.tail, 0); + + /* Reset completion queue descriptor ring */ + memset(qt->cmpl.desc, 0, qt->cmpl.size); + + fbnic_put_pkt_buff(qt, qt->cmpl.pkt, 0); + memset(qt->cmpl.pkt, 0, sizeof(struct fbnic_pkt_buff)); + } +} + +void fbnic_flush(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_nv_flush(fbn->napi[i]); +} + +static void fbnic_nv_fill(struct fbnic_napi_vector *nv) +{ + int j, t; + + /* Configure NAPI mapping and populate pages + * in the BDQ rings to use for Rx + */ + for (j = 0, t = nv->txt_count; j < nv->rxt_count; j++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + /* Populate the header and payload BDQs */ + fbnic_fill_bdq(&qt->sub0); + fbnic_fill_bdq(&qt->sub1); + } +} + +void fbnic_fill(struct fbnic_net *fbn) +{ + int i; + + for (i = 0; i < fbn->num_napi; i++) + fbnic_nv_fill(fbn->napi[i]); +} + +static void fbnic_enable_twq0(struct fbnic_ring *twq) +{ + u32 log_size = fls(twq->size_mask); + + if (!twq->size_mask) + return; + + /* Reset head/tail */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); + twq->tail = 0; + twq->head = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAL, lower_32_bits(twq->dma)); + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_BAH, upper_32_bits(twq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_SIZE, log_size & 0xf); + + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ0_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); +} + +static void fbnic_enable_twq1(struct fbnic_ring *twq) +{ + u32 log_size = fls(twq->size_mask); + + if (!twq->size_mask) + return; + + /* Reset head/tail */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_RESET); + twq->tail = 0; + twq->head = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAL, lower_32_bits(twq->dma)); + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_BAH, upper_32_bits(twq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_SIZE, log_size & 0xf); + + fbnic_ring_wr32(twq, FBNIC_QUEUE_TWQ1_CTL, FBNIC_QUEUE_TWQ_CTL_ENABLE); +} + +static void fbnic_enable_tcq(struct fbnic_napi_vector *nv, + struct fbnic_ring *tcq) +{ + u32 log_size = fls(tcq->size_mask); + + if (!tcq->size_mask) + return; + + /* Reset head/tail */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_RESET); + tcq->tail = 0; + tcq->head = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAL, lower_32_bits(tcq->dma)); + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_BAH, upper_32_bits(tcq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_SIZE, log_size & 0xf); + + /* Store interrupt information for the completion queue */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_CTL, nv->v_idx); + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_THRESHOLD, tcq->size_mask / 2); + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TIM_MASK, 0); + + /* Enable queue */ + fbnic_ring_wr32(tcq, FBNIC_QUEUE_TCQ_CTL, FBNIC_QUEUE_TCQ_CTL_ENABLE); +} + +static void fbnic_enable_bdq(struct fbnic_ring *hpq, struct fbnic_ring *ppq) +{ + u32 bdq_ctl = FBNIC_QUEUE_BDQ_CTL_ENABLE; + u32 log_size; + + /* Reset head/tail */ + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, FBNIC_QUEUE_BDQ_CTL_RESET); + ppq->tail = 0; + ppq->head = 0; + hpq->tail = 0; + hpq->head = 0; + + log_size = fls(hpq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAL, lower_32_bits(hpq->dma)); + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_BAH, upper_32_bits(hpq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_HPQ_SIZE, log_size & 0xf); + + if (!ppq->size_mask) + goto write_ctl; + + log_size = fls(ppq->size_mask) + ilog2(FBNIC_BD_FRAG_COUNT); + + /* Add enabling of PPQ to BDQ control */ + bdq_ctl |= FBNIC_QUEUE_BDQ_CTL_PPQ_ENABLE; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAL, lower_32_bits(ppq->dma)); + fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_BAH, upper_32_bits(ppq->dma)); + fbnic_ring_wr32(ppq, FBNIC_QUEUE_BDQ_PPQ_SIZE, log_size & 0xf); + +write_ctl: + fbnic_ring_wr32(hpq, FBNIC_QUEUE_BDQ_CTL, bdq_ctl); +} + +static void fbnic_config_drop_mode_rcq(struct fbnic_napi_vector *nv, + struct fbnic_ring *rcq, bool tx_pause, + bool hdr_split) +{ + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); + u32 drop_mode, rcq_ctl; + + if (!tx_pause && fbn->num_rx_queues > 1) + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_IMMEDIATE; + else + drop_mode = FBNIC_QUEUE_RDE_CTL0_DROP_NEVER; + + /* Specify packet layout */ + rcq_ctl = FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_DROP_MODE_MASK, drop_mode) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_HROOM_MASK, FBNIC_RX_HROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_MIN_TROOM_MASK, FBNIC_RX_TROOM) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL0_EN_HDR_SPLIT, hdr_split); + + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL0, rcq_ctl); +} + +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool txp) +{ + bool hds; + int i, t; + + hds = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + + for (i = 0; i < fbn->num_napi; i++) { + struct fbnic_napi_vector *nv = fbn->napi[i]; + + for (t = 0; t < nv->rxt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[nv->txt_count + t]; + + fbnic_config_drop_mode_rcq(nv, &qt->cmpl, txp, hds); + } + } +} + +static void fbnic_config_rim_threshold(struct fbnic_ring *rcq, u16 nv_idx, u32 rx_desc) +{ + u32 threshold; + + /* Set the threhsold to half the ring size if rx_frames + * is not configured + */ + threshold = rx_desc ? : rcq->size_mask / 2; + + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_CTL, nv_idx); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_THRESHOLD, threshold); +} + +void fbnic_config_txrx_usecs(struct fbnic_napi_vector *nv, u32 arm) +{ + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); + struct fbnic_dev *fbd = nv->fbd; + u32 val = arm; + + val |= FIELD_PREP(FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT, fbn->rx_usecs) | + FBNIC_INTR_CQ_REARM_RCQ_TIMEOUT_UPD_EN; + val |= FIELD_PREP(FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT, fbn->tx_usecs) | + FBNIC_INTR_CQ_REARM_TCQ_TIMEOUT_UPD_EN; + + fbnic_wr32(fbd, FBNIC_INTR_CQ_REARM(nv->v_idx), val); +} + +void fbnic_config_rx_frames(struct fbnic_napi_vector *nv) +{ + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); + int i; + + for (i = nv->txt_count; i < nv->rxt_count + nv->txt_count; i++) { + struct fbnic_q_triad *qt = &nv->qt[i]; + + fbnic_config_rim_threshold(&qt->cmpl, nv->v_idx, + fbn->rx_max_frames * + FBNIC_MIN_RXD_PER_FRAME); + } +} + +static void fbnic_enable_rcq(struct fbnic_napi_vector *nv, + struct fbnic_ring *rcq) +{ + struct fbnic_net *fbn = netdev_priv(nv->napi.dev); + u32 log_size = fls(rcq->size_mask); + u32 rcq_ctl = 0; + bool hdr_split; + u32 hds_thresh; + + /* Force lower bound on MAX_HEADER_BYTES. Below this, all frames should + * be split at L4. It would also result in the frames being split at + * L2/L3 depending on the frame size. + */ + hdr_split = fbn->hds_thresh < FBNIC_HDR_BYTES_MIN; + fbnic_config_drop_mode_rcq(nv, rcq, fbn->tx_pause, hdr_split); + + hds_thresh = max(fbn->hds_thresh, FBNIC_HDR_BYTES_MIN); + rcq_ctl |= FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PADLEN_MASK, FBNIC_RX_PAD) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_MAX_HDR_MASK, hds_thresh) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_OFF_MASK, + FBNIC_RX_PAYLD_OFFSET) | + FIELD_PREP(FBNIC_QUEUE_RDE_CTL1_PAYLD_PG_CL_MASK, + FBNIC_RX_PAYLD_PG_CL); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RDE_CTL1, rcq_ctl); + + /* Reset head/tail */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_RESET); + rcq->head = 0; + rcq->tail = 0; + + /* Store descriptor ring address and size */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAL, lower_32_bits(rcq->dma)); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_BAH, upper_32_bits(rcq->dma)); + + /* Write lower 4 bits of log size as 64K ring size is 0 */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_SIZE, log_size & 0xf); + + /* Store interrupt information for the completion queue */ + fbnic_config_rim_threshold(rcq, nv->v_idx, fbn->rx_max_frames * + FBNIC_MIN_RXD_PER_FRAME); + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RIM_MASK, 0); + + /* Enable queue */ + fbnic_ring_wr32(rcq, FBNIC_QUEUE_RCQ_CTL, FBNIC_QUEUE_RCQ_CTL_ENABLE); +} + +static void __fbnic_nv_enable(struct fbnic_napi_vector *nv) +{ + int j, t; + + /* Setup Tx Queue Triads */ + for (t = 0; t < nv->txt_count; t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + fbnic_enable_twq0(&qt->sub0); + fbnic_enable_twq1(&qt->sub1); + fbnic_enable_tcq(nv, &qt->cmpl); + } + + /* Setup Rx Queue Triads */ + for (j = 0; j < nv->rxt_count; j++, t++) { + struct fbnic_q_triad *qt = &nv->qt[t]; + + page_pool_enable_direct_recycling(qt->sub0.page_pool, + &nv->napi); + page_pool_enable_direct_recycling(qt->sub1.page_pool, + &nv->napi); + + fbnic_enable_bdq(&qt->sub0, &qt->sub1); + fbnic_enable_rcq(nv, &qt->cmpl); + } +} + +static void fbnic_nv_enable(struct fbnic_net *fbn, struct fbnic_napi_vector *nv) +{ + __fbnic_nv_enable(nv); + fbnic_wrfl(fbn->fbd); +} + +void fbnic_enable(struct fbnic_net *fbn) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i; + + for (i = 0; i < fbn->num_napi; i++) + __fbnic_nv_enable(fbn->napi[i]); + + fbnic_wrfl(fbd); +} + +static void fbnic_nv_irq_enable(struct fbnic_napi_vector *nv) +{ + fbnic_config_txrx_usecs(nv, FBNIC_INTR_CQ_REARM_INTR_UNMASK); +} + +void fbnic_napi_enable(struct fbnic_net *fbn) +{ + u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; + struct fbnic_dev *fbd = fbn->fbd; + int i; + + for (i = 0; i < fbn->num_napi; i++) { + struct fbnic_napi_vector *nv = fbn->napi[i]; + + napi_enable_locked(&nv->napi); + + fbnic_nv_irq_enable(nv); + + /* Record bit used for NAPI IRQs so we can + * set the mask appropriately + */ + irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); + } + + /* Force the first interrupt on the device to guarantee + * that any packets that may have been enqueued during the + * bringup are processed. + */ + for (i = 0; i < ARRAY_SIZE(irqs); i++) { + if (!irqs[i]) + continue; + fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); + } + + fbnic_wrfl(fbd); +} + +void fbnic_napi_depletion_check(struct net_device *netdev) +{ + struct fbnic_net *fbn = netdev_priv(netdev); + u32 irqs[FBNIC_MAX_MSIX_VECS / 32] = {}; + struct fbnic_dev *fbd = fbn->fbd; + int i, j, t; + + for (i = 0; i < fbn->num_napi; i++) { + struct fbnic_napi_vector *nv = fbn->napi[i]; + + /* Find RQs which are completely out of pages */ + for (t = nv->txt_count, j = 0; j < nv->rxt_count; j++, t++) { + /* Assume 4 pages is always enough to fit a packet + * and therefore generate a completion and an IRQ. + */ + if (fbnic_desc_used(&nv->qt[t].sub0) < 4 || + fbnic_desc_used(&nv->qt[t].sub1) < 4) + irqs[nv->v_idx / 32] |= BIT(nv->v_idx % 32); + } + } + + for (i = 0; i < ARRAY_SIZE(irqs); i++) { + if (!irqs[i]) + continue; + fbnic_wr32(fbd, FBNIC_INTR_MASK_CLEAR(i), irqs[i]); + fbnic_wr32(fbd, FBNIC_INTR_SET(i), irqs[i]); + } + + fbnic_wrfl(fbd); +} + +static int fbnic_queue_mem_alloc(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + const struct fbnic_q_triad *real; + struct fbnic_q_triad *qt = qmem; + struct fbnic_napi_vector *nv; + + if (!netif_running(dev)) + return fbnic_alloc_qt_page_pools(fbn, qt, idx); + + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); + nv = fbn->napi[idx % fbn->num_napi]; + + fbnic_ring_init(&qt->sub0, real->sub0.doorbell, real->sub0.q_idx, + real->sub0.flags); + fbnic_ring_init(&qt->sub1, real->sub1.doorbell, real->sub1.q_idx, + real->sub1.flags); + fbnic_ring_init(&qt->cmpl, real->cmpl.doorbell, real->cmpl.q_idx, + real->cmpl.flags); + + return fbnic_alloc_rx_qt_resources(fbn, nv, qt); +} + +static void fbnic_queue_mem_free(struct net_device *dev, void *qmem) +{ + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_q_triad *qt = qmem; + + if (!netif_running(dev)) + fbnic_free_qt_page_pools(qt); + else + fbnic_free_qt_resources(fbn, qt); +} + +static void __fbnic_nv_restart(struct fbnic_net *fbn, + struct fbnic_napi_vector *nv) +{ + struct fbnic_dev *fbd = fbn->fbd; + int i; + + fbnic_nv_enable(fbn, nv); + fbnic_nv_fill(nv); + + napi_enable_locked(&nv->napi); + fbnic_nv_irq_enable(nv); + fbnic_wr32(fbd, FBNIC_INTR_SET(nv->v_idx / 32), BIT(nv->v_idx % 32)); + fbnic_wrfl(fbd); + + for (i = 0; i < nv->txt_count; i++) + netif_wake_subqueue(fbn->netdev, nv->qt[i].sub0.q_idx); + fbnic_dbg_nv_init(nv); +} + +static int fbnic_queue_start(struct net_device *dev, + struct netdev_queue_config *qcfg, + void *qmem, int idx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + struct fbnic_napi_vector *nv; + struct fbnic_q_triad *real; + + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); + nv = fbn->napi[idx % fbn->num_napi]; + + fbnic_aggregate_ring_bdq_counters(fbn, &real->sub0); + fbnic_aggregate_ring_bdq_counters(fbn, &real->sub1); + fbnic_aggregate_ring_rx_counters(fbn, &real->cmpl); + + memcpy(real, qmem, sizeof(*real)); + + __fbnic_nv_restart(fbn, nv); + + return 0; +} + +static int fbnic_queue_stop(struct net_device *dev, void *qmem, int idx) +{ + struct fbnic_net *fbn = netdev_priv(dev); + const struct fbnic_q_triad *real; + struct fbnic_napi_vector *nv; + int i, t; + int err; + + real = container_of(fbn->rx[idx], struct fbnic_q_triad, cmpl); + nv = fbn->napi[idx % fbn->num_napi]; + fbnic_dbg_nv_exit(nv); + + napi_disable_locked(&nv->napi); + fbnic_nv_irq_disable(nv); + + for (i = 0; i < nv->txt_count; i++) + netif_stop_subqueue(dev, nv->qt[i].sub0.q_idx); + fbnic_nv_disable(fbn, nv); + + for (t = 0; t < nv->txt_count + nv->rxt_count; t++) { + err = fbnic_wait_queue_idle(fbn, t >= nv->txt_count, + nv->qt[t].sub0.q_idx); + if (err) + goto err_restart; + } + + fbnic_synchronize_irq(fbn->fbd, nv->v_idx); + fbnic_nv_flush(nv); + + page_pool_disable_direct_recycling(real->sub0.page_pool); + page_pool_disable_direct_recycling(real->sub1.page_pool); + + memcpy(qmem, real, sizeof(*real)); + + return 0; + +err_restart: + __fbnic_nv_restart(fbn, nv); + return err; +} + +const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops = { + .ndo_queue_mem_size = sizeof(struct fbnic_q_triad), + .ndo_queue_mem_alloc = fbnic_queue_mem_alloc, + .ndo_queue_mem_free = fbnic_queue_mem_free, + .ndo_queue_start = fbnic_queue_start, + .ndo_queue_stop = fbnic_queue_stop, +}; diff --git a/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h new file mode 100644 index 000000000..e03c9d2c3 --- /dev/null +++ b/drivers/net/ethernet/meta/fbnic/fbnic_txrx.h @@ -0,0 +1,207 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) Meta Platforms, Inc. and affiliates. */ + +#ifndef _FBNIC_TXRX_H_ +#define _FBNIC_TXRX_H_ + +#include <linux/netdevice.h> +#include <linux/skbuff.h> +#include <linux/types.h> +#include <linux/u64_stats_sync.h> +#include <net/xdp.h> + +struct fbnic_net; + +/* Guarantee we have space needed for storing the buffer + * To store the buffer we need: + * 1 descriptor per page + * + 1 descriptor for skb head + * + 2 descriptors for metadata and optional metadata + * + 7 descriptors to keep tail out of the same cacheline as head + * If we cannot guarantee that then we should return TX_BUSY + */ +#define FBNIC_MAX_SKB_DESC (MAX_SKB_FRAGS + 10) +#define FBNIC_TX_DESC_WAKEUP (FBNIC_MAX_SKB_DESC * 2) +#define FBNIC_TX_DESC_MIN roundup_pow_of_two(FBNIC_TX_DESC_WAKEUP) + +/* To receive the worst case packet we need: + * 1 descriptor for primary metadata + * + 1 descriptor for optional metadata + * + 1 descriptor for headers + * + 4 descriptors for payload + */ +#define FBNIC_MAX_RX_PKT_DESC 7 +#define FBNIC_RX_DESC_MIN roundup_pow_of_two(FBNIC_MAX_RX_PKT_DESC * 2) + +#define FBNIC_MAX_TXQS 128u +#define FBNIC_MAX_RXQS 128u +#define FBNIC_MAX_XDPQS 128u + +/* These apply to TWQs, TCQ, RCQ */ +#define FBNIC_QUEUE_SIZE_MIN 64u +#define FBNIC_QUEUE_SIZE_MAX SZ_64K + +#define FBNIC_TXQ_SIZE_DEFAULT 1024 +#define FBNIC_HPQ_SIZE_DEFAULT 256 +#define FBNIC_PPQ_SIZE_DEFAULT 256 +#define FBNIC_RCQ_SIZE_DEFAULT 1024 +#define FBNIC_TX_USECS_DEFAULT 35 +#define FBNIC_RX_USECS_DEFAULT 30 +#define FBNIC_RX_FRAMES_DEFAULT 0 + +#define FBNIC_RX_TROOM \ + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +#define FBNIC_RX_HROOM_PAD 128 +#define FBNIC_RX_HROOM \ + (ALIGN(FBNIC_RX_TROOM + FBNIC_RX_HROOM_PAD, 128) - FBNIC_RX_TROOM) +#define FBNIC_RX_PAD 0 +#define FBNIC_RX_PAYLD_OFFSET 0 +#define FBNIC_RX_PAYLD_PG_CL 0 + +#define FBNIC_RING_F_DISABLED BIT(0) +#define FBNIC_RING_F_CTX BIT(1) +#define FBNIC_RING_F_STATS BIT(2) /* Ring's stats may be used */ + +#define FBNIC_HDS_THRESH_MAX \ + (4096 - FBNIC_RX_HROOM - FBNIC_RX_TROOM - FBNIC_RX_PAD) +#define FBNIC_HDS_THRESH_DEFAULT \ + (1536 - FBNIC_RX_PAD) +#define FBNIC_HDR_BYTES_MIN 256 + +struct fbnic_pkt_buff { + struct xdp_buff buff; + ktime_t hwtstamp; + bool add_frag_failed; +}; + +struct fbnic_queue_stats { + u64 packets; + u64 bytes; + union { + struct { + u64 csum_partial; + u64 lso; + u64 ts_packets; + u64 ts_lost; + u64 stop; + u64 wake; + } twq; + struct { + u64 alloc_failed; + u64 csum_complete; + u64 csum_none; + u64 length_errors; + } rx; + struct { + u64 alloc_failed; + } bdq; + }; + u64 dropped; + struct u64_stats_sync syncp; +}; + +#define FBNIC_PAGECNT_BIAS_MAX PAGE_SIZE + +struct fbnic_rx_buf { + netmem_ref netmem; + long pagecnt_bias; +}; + +struct fbnic_ring { + /* Pointer to buffer specific info */ + union { + struct fbnic_pkt_buff *pkt; /* RCQ */ + struct fbnic_rx_buf *rx_buf; /* BDQ */ + void **tx_buf; /* TWQ */ + void *buffer; /* Generic pointer */ + }; + + u32 __iomem *doorbell; /* Pointer to CSR space for ring */ + __le64 *desc; /* Descriptor ring memory */ + u16 size_mask; /* Size of ring in descriptors - 1 */ + u8 q_idx; /* Logical netdev ring index */ + u8 flags; /* Ring flags (FBNIC_RING_F_*) */ + + u32 head, tail; /* Head/Tail of ring */ + + union { + /* Rx BDQs only */ + struct page_pool *page_pool; + + /* Deferred_head is used to cache the head for TWQ1 if + * an attempt is made to clean TWQ1 with zero napi_budget. + * We do not use it for any other ring. + */ + s32 deferred_head; + }; + + struct fbnic_queue_stats stats; + + /* Slow path fields follow */ + dma_addr_t dma; /* Phys addr of descriptor memory */ + size_t size; /* Size of descriptor ring in memory */ +}; + +struct fbnic_q_triad { + struct fbnic_ring sub0, sub1, cmpl; + struct xdp_rxq_info xdp_rxq; +}; + +struct fbnic_napi_vector { + struct napi_struct napi; + struct device *dev; /* Device for DMA unmapping */ + struct fbnic_dev *fbd; + struct dentry *dbg_nv; + + u16 v_idx; + u8 txt_count; + u8 rxt_count; + + struct fbnic_q_triad qt[]; +}; + +extern const struct netdev_queue_mgmt_ops fbnic_queue_mgmt_ops; + +netdev_tx_t fbnic_xmit_frame(struct sk_buff *skb, struct net_device *dev); +netdev_features_t +fbnic_features_check(struct sk_buff *skb, struct net_device *dev, + netdev_features_t features); + +void fbnic_aggregate_ring_rx_counters(struct fbnic_net *fbn, + struct fbnic_ring *rxr); +void fbnic_aggregate_ring_bdq_counters(struct fbnic_net *fbn, + struct fbnic_ring *rxr); +void fbnic_aggregate_ring_tx_counters(struct fbnic_net *fbn, + struct fbnic_ring *txr); +void fbnic_aggregate_ring_xdp_counters(struct fbnic_net *fbn, + struct fbnic_ring *xdpr); + +int fbnic_alloc_napi_vectors(struct fbnic_net *fbn); +void fbnic_free_napi_vectors(struct fbnic_net *fbn); +int fbnic_alloc_resources(struct fbnic_net *fbn); +void fbnic_free_resources(struct fbnic_net *fbn); +int fbnic_set_netif_queues(struct fbnic_net *fbn); +void fbnic_reset_netif_queues(struct fbnic_net *fbn); +irqreturn_t fbnic_msix_clean_rings(int irq, void *data); +void fbnic_napi_enable(struct fbnic_net *fbn); +void fbnic_napi_disable(struct fbnic_net *fbn); +void fbnic_config_drop_mode(struct fbnic_net *fbn, bool tx_pause); +void fbnic_enable(struct fbnic_net *fbn); +void fbnic_disable(struct fbnic_net *fbn); +void fbnic_dbg_up(struct fbnic_net *fbn); +void fbnic_dbg_down(struct fbnic_net *fbn); +void fbnic_flush(struct fbnic_net *fbn); +void fbnic_fill(struct fbnic_net *fbn); + +u32 __iomem *fbnic_ring_csr_base(const struct fbnic_ring *ring); +void fbnic_napi_depletion_check(struct net_device *netdev); +int fbnic_wait_all_queues_idle(struct fbnic_dev *fbd, bool may_fail); + +static inline int fbnic_napi_idx(const struct fbnic_napi_vector *nv) +{ + return nv->v_idx - FBNIC_NON_NAPI_VECTORS; +} + +void fbnic_dbg_nv_init(struct fbnic_napi_vector *nv); +void fbnic_dbg_nv_exit(struct fbnic_napi_vector *nv); +#endif /* _FBNIC_TXRX_H_ */ |