diff options
Diffstat (limited to 'drivers/i3c/master')
28 files changed, 15692 insertions, 0 deletions
diff --git a/drivers/i3c/master/Kconfig b/drivers/i3c/master/Kconfig new file mode 100644 index 000000000..da96d2aaa --- /dev/null +++ b/drivers/i3c/master/Kconfig @@ -0,0 +1,103 @@ +# SPDX-License-Identifier: GPL-2.0-only +config ADI_I3C_MASTER + tristate "Analog Devices I3C master driver" + depends on HAS_IOMEM + help + Support for Analog Devices I3C Controller IP, an AXI-interfaced IP + core that supports I3C and I2C devices, multiple speed-grades and I3C + IBIs. + + This driver can also be built as a module. If so, the module will be + called adi-i3c-master. + +config CDNS_I3C_MASTER + tristate "Cadence I3C master driver" + depends on HAS_IOMEM + depends on !(ALPHA || PARISC) + help + Enable this driver if you want to support Cadence I3C master block. + +config DW_I3C_MASTER + tristate "Synospsys DesignWare I3C master driver" + depends on HAS_IOMEM + depends on !(ALPHA || PARISC) + # ALPHA and PARISC needs {read,write}sl() + help + Support for Synopsys DesignWare MIPI I3C Controller. + + For details please see + https://www.synopsys.com/dw/ipdir.php?ds=mipi_i3c + + This driver can also be built as a module. If so, the module + will be called dw-i3c-master. + +config AST2600_I3C_MASTER + tristate "ASPEED AST2600 I3C master driver" + depends on DW_I3C_MASTER + depends on ARCH_ASPEED || COMPILE_TEST + select MFD_SYSCON + help + Support for ASPEED AST2600 I3C Controller. + + This hardware is an instance of the DW I3C controller; this + driver adds platform- specific support for AST2600 hardware. + + This driver can also be built as a module. If so, the module + will be called ast2600-i3c-master. + +config SVC_I3C_MASTER + tristate "Silvaco I3C Dual-Role Master driver" + depends on HAS_IOMEM + depends on !(ALPHA || PARISC) + help + Support for Silvaco I3C Dual-Role Master Controller. + +config MIPI_I3C_HCI + tristate "MIPI I3C Host Controller Interface driver (EXPERIMENTAL)" + depends on HAS_IOMEM + help + Support for hardware following the MIPI Aliance's I3C Host Controller + Interface specification. + + For details please see: + https://www.mipi.org/specifications/i3c-hci + + This driver can also be built as a module. If so, the module will be + called mipi-i3c-hci. + +config MIPI_I3C_HCI_PCI + tristate "MIPI I3C Host Controller Interface PCI support" + depends on MIPI_I3C_HCI + depends on PCI + select MFD_CORE + help + Support for MIPI I3C Host Controller Interface compatible hardware + on the PCI bus. + + This driver can also be built as a module. If so, the module will be + called mipi-i3c-hci-pci. + +config RENESAS_I3C + tristate "Renesas I3C controller driver" + depends on HAS_IOMEM + depends on ARCH_RENESAS || COMPILE_TEST + help + Support the Renesas I3C controller as found in some RZ variants. + + This driver can also be built as a module. If so, the module will be + called renesas-i3c. + +config AMD_AXI_I3C_MASTER + tristate "AMD AXI I3C Master driver" + depends on HAS_IOMEM + help + Support for the AMD AXI I3C master controller, a soft IP used on + AMD (Xilinx) FPGAs and adaptive SoCs with ARM or MicroBlaze + processors. + + The controller currently supports Standard Data Rate (SDR) mode. + Features include Dynamic Address Assignment, private transfers, + and CCC transfers in both broadcast and direct modes. + + This driver can also be built as a module. If so, the module + will be called amd-i3c-master. diff --git a/drivers/i3c/master/Makefile b/drivers/i3c/master/Makefile new file mode 100644 index 000000000..8d82196dc --- /dev/null +++ b/drivers/i3c/master/Makefile @@ -0,0 +1,9 @@ +# SPDX-License-Identifier: GPL-2.0-only +obj-$(CONFIG_ADI_I3C_MASTER) += adi-i3c-master.o +obj-$(CONFIG_CDNS_I3C_MASTER) += i3c-master-cdns.o +obj-$(CONFIG_DW_I3C_MASTER) += dw-i3c-master.o +obj-$(CONFIG_AST2600_I3C_MASTER) += ast2600-i3c-master.o +obj-$(CONFIG_SVC_I3C_MASTER) += svc-i3c-master.o +obj-$(CONFIG_MIPI_I3C_HCI) += mipi-i3c-hci/ +obj-$(CONFIG_RENESAS_I3C) += renesas-i3c.o +obj-$(CONFIG_AMD_AXI_I3C_MASTER) += amd-i3c-master.o diff --git a/drivers/i3c/master/adi-i3c-master.c b/drivers/i3c/master/adi-i3c-master.c new file mode 100644 index 000000000..b35386260 --- /dev/null +++ b/drivers/i3c/master/adi-i3c-master.c @@ -0,0 +1,1019 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * I3C Controller driver + * Copyright 2025 Analog Devices Inc. + * Author: Jorge Marques <jorge.marques@analog.com> + */ + +#include <linux/bitops.h> +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/adi-axi-common.h> +#include <linux/i3c/master.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> + +#include "../internals.h" + +#define ADI_MAX_DEVS 16 +#define ADI_HAS_MDB_FROM_BCR(x) (FIELD_GET(BIT(2), (x))) + +#define REG_ENABLE 0x040 + +#define REG_PID_L 0x054 +#define REG_PID_H 0x058 +#define REG_DCR_BCR_DA 0x05c +#define REG_DCR_BCR_DA_GET_DA(x) FIELD_GET(GENMASK(22, 16), (x)) +#define REG_DCR_BCR_DA_GET_BCR(x) FIELD_GET(GENMASK(15, 8), (x)) +#define REG_DCR_BCR_DA_GET_DCR(x) FIELD_GET(GENMASK(7, 0), (x)) + +#define REG_IRQ_MASK 0x080 +#define REG_IRQ_PENDING 0x084 +#define REG_IRQ_PENDING_DAA BIT(7) +#define REG_IRQ_PENDING_IBI BIT(6) +#define REG_IRQ_PENDING_CMDR BIT(5) + +#define REG_CMD_FIFO 0x0d4 +#define REG_CMD_FIFO_0_IS_CCC BIT(22) +#define REG_CMD_FIFO_0_BCAST BIT(21) +#define REG_CMD_FIFO_0_SR BIT(20) +#define REG_CMD_FIFO_0_LEN(l) FIELD_PREP(GENMASK(19, 8), (l)) +#define REG_CMD_FIFO_0_DEV_ADDR(a) FIELD_PREP(GENMASK(7, 1), (a)) +#define REG_CMD_FIFO_0_RNW BIT(0) +#define REG_CMD_FIFO_1_CCC(id) FIELD_PREP(GENMASK(7, 0), (id)) + +#define REG_CMD_FIFO_ROOM 0x0c0 +#define REG_CMDR_FIFO 0x0d8 +#define REG_CMDR_FIFO_UDA_ERROR 8 +#define REG_CMDR_FIFO_NACK_RESP 6 +#define REG_CMDR_FIFO_CE2_ERROR 4 +#define REG_CMDR_FIFO_CE0_ERROR 1 +#define REG_CMDR_FIFO_NO_ERROR 0 +#define REG_CMDR_FIFO_ERROR(x) FIELD_GET(GENMASK(23, 20), (x)) +#define REG_CMDR_FIFO_XFER_BYTES(x) FIELD_GET(GENMASK(19, 8), (x)) + +#define REG_SDO_FIFO 0x0dc +#define REG_SDO_FIFO_ROOM 0x0c8 +#define REG_SDI_FIFO 0x0e0 +#define REG_IBI_FIFO 0x0e4 +#define REG_FIFO_STATUS 0x0e8 +#define REG_FIFO_STATUS_CMDR_EMPTY BIT(0) +#define REG_FIFO_STATUS_IBI_EMPTY BIT(1) + +#define REG_OPS 0x100 +#define REG_OPS_PP_SG_MASK GENMASK(6, 5) +#define REG_OPS_SET_SG(x) FIELD_PREP(REG_OPS_PP_SG_MASK, (x)) + +#define REG_IBI_CONFIG 0x140 +#define REG_IBI_CONFIG_ENABLE BIT(0) +#define REG_IBI_CONFIG_LISTEN BIT(1) + +#define REG_DEV_CHAR 0x180 +#define REG_DEV_CHAR_IS_I2C BIT(0) +#define REG_DEV_CHAR_IS_ATTACHED BIT(1) +#define REG_DEV_CHAR_BCR_IBI(x) FIELD_PREP(GENMASK(3, 2), (x)) +#define REG_DEV_CHAR_WEN BIT(8) +#define REG_DEV_CHAR_ADDR(x) FIELD_PREP(GENMASK(15, 9), (x)) + +enum speed_grade {PP_SG_UNSET, PP_SG_1MHZ, PP_SG_3MHZ, PP_SG_6MHZ, PP_SG_12MHZ}; + +struct adi_i3c_cmd { + u32 cmd0; + u32 cmd1; + u32 tx_len; + const void *tx_buf; + u32 rx_len; + void *rx_buf; + u32 error; +}; + +struct adi_i3c_xfer { + struct list_head node; + struct completion comp; + int ret; + unsigned int ncmds; + unsigned int ncmds_comp; + struct adi_i3c_cmd cmds[] __counted_by(ncmds); +}; + +struct adi_i3c_master { + struct i3c_master_controller base; + u32 free_rr_slots; + struct { + unsigned int num_slots; + struct i3c_dev_desc **slots; + spinlock_t lock; /* Protect IBI slot access */ + } ibi; + struct { + struct list_head list; + struct adi_i3c_xfer *cur; + spinlock_t lock; /* Protect transfer */ + } xferqueue; + void __iomem *regs; + struct clk *clk; + unsigned long i3c_scl_lim; + struct { + u8 addrs[ADI_MAX_DEVS]; + u8 index; + } daa; +}; + +static inline struct adi_i3c_master *to_adi_i3c_master(struct i3c_master_controller *master) +{ + return container_of(master, struct adi_i3c_master, base); +} + +static void adi_i3c_master_wr_to_tx_fifo(struct adi_i3c_master *master, + const u8 *buf, unsigned int nbytes) +{ + unsigned int n, m; + + n = readl(master->regs + REG_SDO_FIFO_ROOM); + m = min(n, nbytes); + i3c_writel_fifo(master->regs + REG_SDO_FIFO, buf, m); +} + +static void adi_i3c_master_rd_from_rx_fifo(struct adi_i3c_master *master, + u8 *buf, unsigned int nbytes) +{ + i3c_readl_fifo(master->regs + REG_SDI_FIFO, buf, nbytes); +} + +static bool adi_i3c_master_supports_ccc_cmd(struct i3c_master_controller *m, + const struct i3c_ccc_cmd *cmd) +{ + if (cmd->ndests > 1) + return false; + + switch (cmd->id) { + case I3C_CCC_ENEC(true): + case I3C_CCC_ENEC(false): + case I3C_CCC_DISEC(true): + case I3C_CCC_DISEC(false): + case I3C_CCC_RSTDAA(true): + case I3C_CCC_RSTDAA(false): + case I3C_CCC_ENTDAA: + case I3C_CCC_SETDASA: + case I3C_CCC_SETNEWDA: + case I3C_CCC_GETMWL: + case I3C_CCC_GETMRL: + case I3C_CCC_GETPID: + case I3C_CCC_GETBCR: + case I3C_CCC_GETDCR: + case I3C_CCC_GETSTATUS: + case I3C_CCC_GETHDRCAP: + return true; + default: + break; + } + + return false; +} + +static int adi_i3c_master_disable(struct adi_i3c_master *master) +{ + writel(0, master->regs + REG_IBI_CONFIG); + + return 0; +} + +static struct adi_i3c_xfer *adi_i3c_master_alloc_xfer(struct adi_i3c_master *master, + unsigned int ncmds) +{ + struct adi_i3c_xfer *xfer; + + xfer = kzalloc_flex(*xfer, cmds, ncmds); + if (!xfer) + return NULL; + + INIT_LIST_HEAD(&xfer->node); + xfer->ncmds = ncmds; + xfer->ret = -ETIMEDOUT; + + return xfer; +} + +static void adi_i3c_master_start_xfer_locked(struct adi_i3c_master *master) +{ + struct adi_i3c_xfer *xfer = master->xferqueue.cur; + unsigned int i, n, m; + + if (!xfer) + return; + + for (i = 0; i < xfer->ncmds; i++) { + struct adi_i3c_cmd *cmd = &xfer->cmds[i]; + + if (!(cmd->cmd0 & REG_CMD_FIFO_0_RNW)) + adi_i3c_master_wr_to_tx_fifo(master, cmd->tx_buf, cmd->tx_len); + } + + n = readl(master->regs + REG_CMD_FIFO_ROOM); + for (i = 0; i < xfer->ncmds; i++) { + struct adi_i3c_cmd *cmd = &xfer->cmds[i]; + + m = cmd->cmd0 & REG_CMD_FIFO_0_IS_CCC ? 2 : 1; + if (m > n) + break; + writel(cmd->cmd0, master->regs + REG_CMD_FIFO); + if (cmd->cmd0 & REG_CMD_FIFO_0_IS_CCC) + writel(cmd->cmd1, master->regs + REG_CMD_FIFO); + n -= m; + } +} + +static void adi_i3c_master_end_xfer_locked(struct adi_i3c_master *master, + u32 pending) +{ + struct adi_i3c_xfer *xfer = master->xferqueue.cur; + int i, ret = 0; + + if (!xfer) + return; + + while (!(readl(master->regs + REG_FIFO_STATUS) & REG_FIFO_STATUS_CMDR_EMPTY)) { + struct adi_i3c_cmd *cmd; + u32 cmdr, rx_len; + + cmdr = readl(master->regs + REG_CMDR_FIFO); + + cmd = &xfer->cmds[xfer->ncmds_comp++]; + if (cmd->cmd0 & REG_CMD_FIFO_0_RNW) { + rx_len = min_t(u32, REG_CMDR_FIFO_XFER_BYTES(cmdr), cmd->rx_len); + adi_i3c_master_rd_from_rx_fifo(master, cmd->rx_buf, rx_len); + cmd->rx_len = rx_len; + } + cmd->error = REG_CMDR_FIFO_ERROR(cmdr); + } + + for (i = 0; i < xfer->ncmds_comp; i++) { + switch (xfer->cmds[i].error) { + case REG_CMDR_FIFO_NO_ERROR: + break; + + case REG_CMDR_FIFO_CE0_ERROR: + case REG_CMDR_FIFO_CE2_ERROR: + case REG_CMDR_FIFO_NACK_RESP: + case REG_CMDR_FIFO_UDA_ERROR: + ret = -EIO; + break; + + default: + ret = -EINVAL; + break; + } + } + + xfer->ret = ret; + + if (xfer->ncmds_comp != xfer->ncmds) + return; + + complete(&xfer->comp); + + xfer = list_first_entry_or_null(&master->xferqueue.list, + struct adi_i3c_xfer, node); + if (xfer) + list_del_init(&xfer->node); + + master->xferqueue.cur = xfer; + adi_i3c_master_start_xfer_locked(master); +} + +static void adi_i3c_master_queue_xfer(struct adi_i3c_master *master, + struct adi_i3c_xfer *xfer) +{ + init_completion(&xfer->comp); + guard(spinlock_irqsave)(&master->xferqueue.lock); + if (master->xferqueue.cur) { + list_add_tail(&xfer->node, &master->xferqueue.list); + } else { + master->xferqueue.cur = xfer; + adi_i3c_master_start_xfer_locked(master); + } +} + +static void adi_i3c_master_unqueue_xfer(struct adi_i3c_master *master, + struct adi_i3c_xfer *xfer) +{ + guard(spinlock_irqsave)(&master->xferqueue.lock); + if (master->xferqueue.cur == xfer) + master->xferqueue.cur = NULL; + else + list_del_init(&xfer->node); + + writel(0x01, master->regs + REG_ENABLE); + writel(0x00, master->regs + REG_ENABLE); + writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK); +} + +static enum i3c_error_code adi_i3c_cmd_get_err(struct adi_i3c_cmd *cmd) +{ + switch (cmd->error) { + case REG_CMDR_FIFO_CE0_ERROR: + return I3C_ERROR_M0; + + case REG_CMDR_FIFO_CE2_ERROR: + case REG_CMDR_FIFO_NACK_RESP: + return I3C_ERROR_M2; + + default: + break; + } + + return I3C_ERROR_UNKNOWN; +} + +static int adi_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *cmd) +{ + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_cmd *ccmd; + + struct adi_i3c_xfer *xfer __free(kfree) = adi_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + ccmd = xfer->cmds; + ccmd->cmd1 = REG_CMD_FIFO_1_CCC(cmd->id); + ccmd->cmd0 = REG_CMD_FIFO_0_IS_CCC | + REG_CMD_FIFO_0_LEN(cmd->dests[0].payload.len); + + if (cmd->id & I3C_CCC_DIRECT) + ccmd->cmd0 |= REG_CMD_FIFO_0_DEV_ADDR(cmd->dests[0].addr); + + if (cmd->rnw) { + ccmd->cmd0 |= REG_CMD_FIFO_0_RNW; + ccmd->rx_buf = cmd->dests[0].payload.data; + ccmd->rx_len = cmd->dests[0].payload.len; + } else { + ccmd->tx_buf = cmd->dests[0].payload.data; + ccmd->tx_len = cmd->dests[0].payload.len; + } + + adi_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000))) + adi_i3c_master_unqueue_xfer(master, xfer); + + cmd->err = adi_i3c_cmd_get_err(&xfer->cmds[0]); + if (!xfer->ret && cmd->rnw) + cmd->dests[0].payload.actual_len = ccmd->rx_len; + + return xfer->ret; +} + +static int adi_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, + struct i3c_xfer *xfers, + int nxfers, enum i3c_xfer_mode mode) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + int i, ret; + + if (!nxfers) + return 0; + + struct adi_i3c_xfer *xfer __free(kfree) = adi_i3c_master_alloc_xfer(master, nxfers); + if (!xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + struct adi_i3c_cmd *ccmd = &xfer->cmds[i]; + + ccmd->cmd0 = REG_CMD_FIFO_0_DEV_ADDR(dev->info.dyn_addr); + + if (xfers[i].rnw) { + ccmd->cmd0 |= REG_CMD_FIFO_0_RNW; + ccmd->rx_buf = xfers[i].data.in; + ccmd->rx_len = xfers[i].len; + } else { + ccmd->tx_buf = xfers[i].data.out; + ccmd->tx_len = xfers[i].len; + } + + ccmd->cmd0 |= REG_CMD_FIFO_0_LEN(xfers[i].len); + + if (i < nxfers - 1) + ccmd->cmd0 |= REG_CMD_FIFO_0_SR; + + if (!i) + ccmd->cmd0 |= REG_CMD_FIFO_0_BCAST; + } + + adi_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, + msecs_to_jiffies(1000))) + adi_i3c_master_unqueue_xfer(master, xfer); + + ret = xfer->ret; + + for (i = 0; i < nxfers; i++) + xfers[i].err = adi_i3c_cmd_get_err(&xfer->cmds[i]); + + return ret; +} + +struct adi_i3c_i2c_dev_data { + struct i3c_generic_ibi_pool *ibi_pool; + u16 id; + s16 ibi; +}; + +static int adi_i3c_master_get_rr_slot(struct adi_i3c_master *master, + u8 dyn_addr) +{ + if (!master->free_rr_slots) + return -ENOSPC; + + return ffs(master->free_rr_slots) - 1; +} + +static int adi_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, u8 dyn_addr) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + u8 addr; + + addr = dev->info.dyn_addr ? dev->info.dyn_addr : dev->info.static_addr; + + writel(REG_DEV_CHAR_ADDR(dyn_addr), master->regs + REG_DEV_CHAR); + writel((readl(master->regs + REG_DEV_CHAR) & + ~REG_DEV_CHAR_IS_ATTACHED) | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + + writel(REG_DEV_CHAR_ADDR(addr), master->regs + REG_DEV_CHAR); + writel(readl(master->regs + REG_DEV_CHAR) | + REG_DEV_CHAR_IS_ATTACHED | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + + return 0; +} + +static int adi_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_i2c_dev_data *data; + int slot; + u8 addr; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + slot = adi_i3c_master_get_rr_slot(master, dev->info.dyn_addr); + if (slot < 0) { + kfree(data); + return slot; + } + + data->id = slot; + i3c_dev_set_master_data(dev, data); + master->free_rr_slots &= ~BIT(slot); + + addr = dev->info.dyn_addr ? dev->info.dyn_addr : dev->info.static_addr; + + writel(REG_DEV_CHAR_ADDR(addr), master->regs + REG_DEV_CHAR); + writel(readl(master->regs + REG_DEV_CHAR) | + REG_DEV_CHAR_IS_ATTACHED | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + + return 0; +} + +static void adi_i3c_master_sync_dev_char(struct i3c_master_controller *m) +{ + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct i3c_dev_desc *i3cdev; + u32 bcr_ibi; + u8 addr; + + i3c_bus_for_each_i3cdev(&m->bus, i3cdev) { + addr = i3cdev->info.dyn_addr ? + i3cdev->info.dyn_addr : i3cdev->info.static_addr; + writel(REG_DEV_CHAR_ADDR(addr), master->regs + REG_DEV_CHAR); + bcr_ibi = FIELD_GET(I3C_BCR_IBI_PAYLOAD | I3C_BCR_IBI_REQ_CAP, (i3cdev->info.bcr)); + writel(readl(master->regs + REG_DEV_CHAR) | + REG_DEV_CHAR_BCR_IBI(bcr_ibi) | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + } +} + +static void adi_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + u8 addr; + + addr = dev->info.dyn_addr ? dev->info.dyn_addr : dev->info.static_addr; + + writel(REG_DEV_CHAR_ADDR(addr), master->regs + REG_DEV_CHAR); + writel((readl(master->regs + REG_DEV_CHAR) & + ~REG_DEV_CHAR_IS_ATTACHED) | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + + i3c_dev_set_master_data(dev, NULL); + master->free_rr_slots |= BIT(data->id); + kfree(data); +} + +static int adi_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_i2c_dev_data *data; + int slot; + + slot = adi_i3c_master_get_rr_slot(master, 0); + if (slot < 0) + return slot; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + data->id = slot; + master->free_rr_slots &= ~BIT(slot); + i2c_dev_set_master_data(dev, data); + + writel(REG_DEV_CHAR_ADDR(dev->addr) | + REG_DEV_CHAR_IS_I2C | REG_DEV_CHAR_IS_ATTACHED | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + + return 0; +} + +static void adi_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + + writel(REG_DEV_CHAR_ADDR(dev->addr) | + REG_DEV_CHAR_IS_I2C | REG_DEV_CHAR_WEN, + master->regs + REG_DEV_CHAR); + + i2c_dev_set_master_data(dev, NULL); + master->free_rr_slots |= BIT(data->id); + kfree(data); +} + +static void adi_i3c_master_bus_cleanup(struct i3c_master_controller *m) +{ + struct adi_i3c_master *master = to_adi_i3c_master(m); + + adi_i3c_master_disable(master); +} + +static void adi_i3c_master_upd_i3c_scl_lim(struct adi_i3c_master *master) +{ + struct i3c_master_controller *m = &master->base; + struct i3c_bus *bus = i3c_master_get_bus(m); + u8 i3c_scl_lim = 0; + struct i3c_dev_desc *dev; + u8 pp_sg; + + i3c_bus_for_each_i3cdev(bus, dev) { + u8 max_fscl; + + max_fscl = max(I3C_CCC_MAX_SDR_FSCL(dev->info.max_read_ds), + I3C_CCC_MAX_SDR_FSCL(dev->info.max_write_ds)); + + switch (max_fscl) { + case I3C_SDR1_FSCL_8MHZ: + max_fscl = PP_SG_6MHZ; + break; + case I3C_SDR2_FSCL_6MHZ: + max_fscl = PP_SG_3MHZ; + break; + case I3C_SDR3_FSCL_4MHZ: + max_fscl = PP_SG_3MHZ; + break; + case I3C_SDR4_FSCL_2MHZ: + max_fscl = PP_SG_1MHZ; + break; + case I3C_SDR0_FSCL_MAX: + default: + max_fscl = PP_SG_12MHZ; + break; + } + + if (max_fscl && + (i3c_scl_lim > max_fscl || !i3c_scl_lim)) + i3c_scl_lim = max_fscl; + } + + if (!i3c_scl_lim) + return; + + master->i3c_scl_lim = i3c_scl_lim - 1; + + pp_sg = readl(master->regs + REG_OPS) & ~REG_OPS_PP_SG_MASK; + pp_sg |= REG_OPS_SET_SG(master->i3c_scl_lim); + + writel(pp_sg, master->regs + REG_OPS); +} + +static void adi_i3c_master_get_features(struct adi_i3c_master *master, + unsigned int slot, + struct i3c_device_info *info) +{ + u32 buf; + + /* Dynamic address and PID are for identification only */ + memset(info, 0, sizeof(*info)); + buf = readl(master->regs + REG_DCR_BCR_DA); + info->dyn_addr = REG_DCR_BCR_DA_GET_DA(buf); + info->dcr = REG_DCR_BCR_DA_GET_DCR(buf); + info->bcr = REG_DCR_BCR_DA_GET_BCR(buf); + info->pid = readl(master->regs + REG_PID_L); + info->pid |= (u64)readl(master->regs + REG_PID_H) << 32; +} + +static int adi_i3c_master_do_daa(struct i3c_master_controller *m) +{ + struct adi_i3c_master *master = to_adi_i3c_master(m); + int ret, addr = 0; + u32 irq_mask; + + for (u8 i = 0; i < ADI_MAX_DEVS; i++) { + addr = i3c_master_get_free_addr(m, addr); + if (addr < 0) + return addr; + master->daa.addrs[i] = addr; + } + + irq_mask = readl(master->regs + REG_IRQ_MASK); + writel(irq_mask | REG_IRQ_PENDING_DAA, + master->regs + REG_IRQ_MASK); + + master->daa.index = 0; + ret = i3c_master_entdaa_locked(&master->base); + + writel(irq_mask, master->regs + REG_IRQ_MASK); + + if (ret) + return ret; + + /* Add I3C devices discovered */ + for (u8 i = 0; i < master->daa.index; i++) + i3c_master_add_i3c_dev_locked(m, master->daa.addrs[i]); + /* Sync retrieved devs info with the IP */ + adi_i3c_master_sync_dev_char(m); + + i3c_master_defslvs_locked(&master->base); + + adi_i3c_master_upd_i3c_scl_lim(master); + + return 0; +} + +static int adi_i3c_master_bus_init(struct i3c_master_controller *m) +{ + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct i3c_device_info info = { }; + int ret; + + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + return ret; + + adi_i3c_master_get_features(master, 0, &info); + ret = i3c_master_set_info(&master->base, &info); + if (ret) + return ret; + + writel(REG_IBI_CONFIG_LISTEN, + master->regs + REG_IBI_CONFIG); + + return 0; +} + +static void adi_i3c_master_handle_ibi(struct adi_i3c_master *master, + u32 raw) +{ + struct adi_i3c_i2c_dev_data *data; + struct i3c_ibi_slot *slot; + struct i3c_dev_desc *dev; + u8 da, id, mdb, len; + u8 *buf; + + da = FIELD_GET(GENMASK(23, 17), raw); + mdb = FIELD_GET(GENMASK(15, 8), raw); + for (id = 0; id < master->ibi.num_slots; id++) { + if (master->ibi.slots[id] && + master->ibi.slots[id]->info.dyn_addr == da) + break; + } + + if (id == master->ibi.num_slots) + return; + + dev = master->ibi.slots[id]; + len = ADI_HAS_MDB_FROM_BCR(dev->info.bcr); + data = i3c_dev_get_master_data(dev); + + guard(spinlock)(&master->ibi.lock); + slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); + if (!slot) + return; + + slot->len = len; + buf = slot->data; + buf[0] = mdb; + i3c_master_queue_ibi(dev, slot); +} + +static void adi_i3c_master_demux_ibis(struct adi_i3c_master *master) +{ + while (!(readl(master->regs + REG_FIFO_STATUS) & REG_FIFO_STATUS_IBI_EMPTY)) { + u32 raw = readl(master->regs + REG_IBI_FIFO); + + adi_i3c_master_handle_ibi(master, raw); + } +} + +static void adi_i3c_master_handle_da_req(struct adi_i3c_master *master) +{ + u8 payload0[8]; + u32 addr; + + adi_i3c_master_rd_from_rx_fifo(master, payload0, 6); + addr = master->daa.addrs[master->daa.index++]; + addr = (addr << 1) | (parity8(addr) ? 0 : 1); + + writel(addr, master->regs + REG_SDO_FIFO); +} + +static irqreturn_t adi_i3c_master_irq(int irq, void *data) +{ + struct adi_i3c_master *master = data; + u32 pending; + + pending = readl(master->regs + REG_IRQ_PENDING); + writel(pending, master->regs + REG_IRQ_PENDING); + if (pending & REG_IRQ_PENDING_CMDR) { + scoped_guard(spinlock_irqsave, &master->xferqueue.lock) { + adi_i3c_master_end_xfer_locked(master, pending); + } + } + if (pending & REG_IRQ_PENDING_IBI) + adi_i3c_master_demux_ibis(master); + if (pending & REG_IRQ_PENDING_DAA) + adi_i3c_master_handle_da_req(master); + + return IRQ_HANDLED; +} + +static int adi_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *xfers, + int nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + int i; + + if (!nxfers) + return 0; + for (i = 0; i < nxfers; i++) { + if (xfers[i].flags & I2C_M_TEN) + return -EOPNOTSUPP; + } + + struct adi_i3c_xfer *xfer __free(kfree) = adi_i3c_master_alloc_xfer(master, nxfers); + if (!xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + struct adi_i3c_cmd *ccmd = &xfer->cmds[i]; + + ccmd->cmd0 = REG_CMD_FIFO_0_DEV_ADDR(xfers[i].addr); + + if (xfers[i].flags & I2C_M_RD) { + ccmd->cmd0 |= REG_CMD_FIFO_0_RNW; + ccmd->rx_buf = xfers[i].buf; + ccmd->rx_len = xfers[i].len; + } else { + ccmd->tx_buf = xfers[i].buf; + ccmd->tx_len = xfers[i].len; + } + + ccmd->cmd0 |= REG_CMD_FIFO_0_LEN(xfers[i].len); + } + + adi_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, + m->i2c.timeout)) + adi_i3c_master_unqueue_xfer(master, xfer); + + return xfer->ret; +} + +static int adi_i3c_master_disable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct i3c_dev_desc *i3cdev; + u32 enabled = 0; + int ret; + + ret = i3c_master_disec_locked(m, dev->info.dyn_addr, + I3C_CCC_EVENT_SIR); + + i3c_bus_for_each_i3cdev(&m->bus, i3cdev) { + if (dev != i3cdev && i3cdev->ibi) + enabled |= i3cdev->ibi->enabled; + } + if (!enabled) { + writel(REG_IBI_CONFIG_LISTEN, + master->regs + REG_IBI_CONFIG); + writel(readl(master->regs + REG_IRQ_MASK) & ~REG_IRQ_PENDING_IBI, + master->regs + REG_IRQ_MASK); + } + + return ret; +} + +static int adi_i3c_master_enable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + + writel(REG_IBI_CONFIG_LISTEN | REG_IBI_CONFIG_ENABLE, + master->regs + REG_IBI_CONFIG); + + writel(readl(master->regs + REG_IRQ_MASK) | REG_IRQ_PENDING_IBI, + master->regs + REG_IRQ_MASK); + + return i3c_master_enec_locked(m, dev->info.dyn_addr, + I3C_CCC_EVENT_SIR); +} + +static int adi_i3c_master_request_ibi(struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_i2c_dev_data *data; + unsigned int i; + + data = i3c_dev_get_master_data(dev); + data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(data->ibi_pool)) + return PTR_ERR(data->ibi_pool); + + scoped_guard(spinlock_irqsave, &master->ibi.lock) { + for (i = 0; i < master->ibi.num_slots; i++) { + if (!master->ibi.slots[i]) { + data->ibi = i; + master->ibi.slots[i] = dev; + break; + } + } + } + + if (i < master->ibi.num_slots) + return 0; + + i3c_generic_ibi_free_pool(data->ibi_pool); + data->ibi_pool = NULL; + + return -ENOSPC; +} + +static void adi_i3c_master_free_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct adi_i3c_master *master = to_adi_i3c_master(m); + struct adi_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + scoped_guard(spinlock_irqsave, &master->ibi.lock) { + master->ibi.slots[data->ibi] = NULL; + } + + i3c_generic_ibi_free_pool(data->ibi_pool); +} + +static void adi_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct adi_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); +} + +static const struct i3c_master_controller_ops adi_i3c_master_ops = { + .bus_init = adi_i3c_master_bus_init, + .bus_cleanup = adi_i3c_master_bus_cleanup, + .attach_i3c_dev = adi_i3c_master_attach_i3c_dev, + .reattach_i3c_dev = adi_i3c_master_reattach_i3c_dev, + .detach_i3c_dev = adi_i3c_master_detach_i3c_dev, + .attach_i2c_dev = adi_i3c_master_attach_i2c_dev, + .detach_i2c_dev = adi_i3c_master_detach_i2c_dev, + .do_daa = adi_i3c_master_do_daa, + .supports_ccc_cmd = adi_i3c_master_supports_ccc_cmd, + .send_ccc_cmd = adi_i3c_master_send_ccc_cmd, + .i3c_xfers = adi_i3c_master_i3c_xfers, + .i2c_xfers = adi_i3c_master_i2c_xfers, + .request_ibi = adi_i3c_master_request_ibi, + .enable_ibi = adi_i3c_master_enable_ibi, + .disable_ibi = adi_i3c_master_disable_ibi, + .free_ibi = adi_i3c_master_free_ibi, + .recycle_ibi_slot = adi_i3c_master_recycle_ibi_slot, +}; + +static const struct of_device_id adi_i3c_master_of_match[] = { + { .compatible = "adi,i3c-master-v1" }, + {} +}; +MODULE_DEVICE_TABLE(of, adi_i3c_master_of_match); + +static int adi_i3c_master_probe(struct platform_device *pdev) +{ + struct adi_i3c_master *master; + struct clk_bulk_data *clk; + unsigned int version; + int ret, irq; + + master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + master->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(master->regs)) + return PTR_ERR(master->regs); + + ret = devm_clk_bulk_get_all_enabled(&pdev->dev, &clk); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Failed to get clocks\n"); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + version = readl(master->regs + ADI_AXI_REG_VERSION); + if (ADI_AXI_PCORE_VER_MAJOR(version) != 1) + dev_err_probe(&pdev->dev, -ENODEV, "Unsupported peripheral version %u.%u.%u\n", + ADI_AXI_PCORE_VER_MAJOR(version), + ADI_AXI_PCORE_VER_MINOR(version), + ADI_AXI_PCORE_VER_PATCH(version)); + + writel(0x00, master->regs + REG_ENABLE); + writel(0x00, master->regs + REG_IRQ_MASK); + + platform_set_drvdata(pdev, master); + + master->free_rr_slots = GENMASK(ADI_MAX_DEVS, 1); + spin_lock_init(&master->ibi.lock); + master->ibi.num_slots = 15; + master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots, + sizeof(*master->ibi.slots), + GFP_KERNEL); + if (!master->ibi.slots) + return -ENOMEM; + + spin_lock_init(&master->xferqueue.lock); + INIT_LIST_HEAD(&master->xferqueue.list); + + ret = devm_request_irq(&pdev->dev, irq, adi_i3c_master_irq, 0, + dev_name(&pdev->dev), master); + if (ret) + return ret; + + writel(REG_IRQ_PENDING_CMDR, master->regs + REG_IRQ_MASK); + + return i3c_master_register(&master->base, &pdev->dev, + &adi_i3c_master_ops, false); +} + +static void adi_i3c_master_remove(struct platform_device *pdev) +{ + struct adi_i3c_master *master = platform_get_drvdata(pdev); + + writel(0xff, master->regs + REG_IRQ_PENDING); + writel(0x00, master->regs + REG_IRQ_MASK); + writel(0x01, master->regs + REG_ENABLE); + + i3c_master_unregister(&master->base); +} + +static struct platform_driver adi_i3c_master = { + .probe = adi_i3c_master_probe, + .remove = adi_i3c_master_remove, + .driver = { + .name = "adi-i3c-master", + .of_match_table = adi_i3c_master_of_match, + }, +}; +module_platform_driver(adi_i3c_master); + +MODULE_AUTHOR("Jorge Marques <jorge.marques@analog.com>"); +MODULE_DESCRIPTION("Analog Devices I3C master driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/i3c/master/amd-i3c-master.c b/drivers/i3c/master/amd-i3c-master.c new file mode 100644 index 000000000..e4da9c38e --- /dev/null +++ b/drivers/i3c/master/amd-i3c-master.c @@ -0,0 +1,1124 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * I3C master driver for the AMD I3C controller. + * + * Copyright (C) 2026, Advanced Micro Devices, Inc. + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/clk.h> +#include <linux/delay.h> +#include <linux/err.h> +#include <linux/i3c/master.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mutex.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/time.h> +#include <linux/unaligned.h> + +#define XI3C_VERSION_OFFSET 0x00 /* Version Register */ +#define XI3C_RESET_OFFSET 0x04 /* Soft Reset Register */ +#define XI3C_CR_OFFSET 0x08 /* Control Register */ +#define XI3C_ADDRESS_OFFSET 0x0C /* Target Address Register */ +#define XI3C_SR_OFFSET 0x10 /* Status Register */ +#define XI3C_CMD_FIFO_OFFSET 0x20 /* I3C Command FIFO Register */ +#define XI3C_WR_FIFO_OFFSET 0x24 /* I3C Write Data FIFO Register */ +#define XI3C_RD_FIFO_OFFSET 0x28 /* I3C Read Data FIFO Register */ +#define XI3C_RESP_STATUS_FIFO_OFFSET 0x2C /* I3C Response status FIFO Register */ +#define XI3C_FIFO_LVL_STATUS_OFFSET 0x30 /* CMD slots free | WR-FIFO free (words) */ +#define XI3C_FIFO_LVL_STATUS_1_OFFSET 0x34 /* RESP fill | RD-FIFO fill level (words) */ +#define XI3C_SCL_HIGH_TIME_OFFSET 0x38 /* I3C SCL HIGH Register */ +#define XI3C_SCL_LOW_TIME_OFFSET 0x3C /* I3C SCL LOW Register */ +#define XI3C_SDA_HOLD_TIME_OFFSET 0x40 /* I3C SDA HOLD Register */ +#define XI3C_TSU_START_OFFSET 0x48 /* I3C START SETUP Register */ +#define XI3C_THD_START_OFFSET 0x4C /* I3C START HOLD Register */ +#define XI3C_TSU_STOP_OFFSET 0x50 /* I3C STOP Setup Register */ +#define XI3C_OD_SCL_HIGH_TIME_OFFSET 0x54 /* I3C OD SCL HIGH Register */ +#define XI3C_OD_SCL_LOW_TIME_OFFSET 0x58 /* I3C OD SCL LOW Register */ +#define XI3C_PID0_OFFSET 0x6C /* LSB 4 bytes of the PID */ +#define XI3C_PID1_BCR_DCR 0x70 /* MSB 2 bytes of the PID, BCR and DCR */ + +#define XI3C_CR_EN_MASK BIT(0) /* Core Enable */ +#define XI3C_CR_RESUME_MASK BIT(2) /* Core Resume */ +#define XI3C_SR_RESP_NOT_EMPTY_MASK BIT(4) /* Resp Fifo not empty status mask */ +#define XI3C_RD_FIFO_NOT_EMPTY_MASK BIT(15) /* Read Fifo not empty status mask */ + +#define XI3C_BCR_MASK GENMASK(23, 16) +#define XI3C_DCR_MASK GENMASK(31, 24) +#define XI3C_PID_MASK GENMASK_ULL(63, 16) +#define XI3C_TIMING_MASK GENMASK(17, 0) +#define XI3C_REV_NUM_MASK GENMASK(15, 8) +#define XI3C_PID1_MASK GENMASK(15, 0) +#define XI3C_FIFO_LEVEL_MASK GENMASK(15, 0) +#define XI3C_RESP_CODE_MASK GENMASK(8, 5) + +/* Controller response codes; PG439 page 34, Table 46 */ +#define XI3C_RESP_CODE_SUCCESS 0 /* Transfer completed OK */ +#define XI3C_RESP_CODE_NO_TARGET 2 /* 7E NACK: no target on bus */ +#define XI3C_RESP_CODE_NACK 3 /* Target NACK / DAA end */ +#define XI3C_RESP_CODE_READ_EARLY_TERM 5 /* Target ended read early (T-bit), short read */ + +/* xi3c_get_response() private return: read ended early with valid data; resume needed */ +#define XI3C_XFER_SHORT_READ 1 + +#define XI3C_RESP_BYTES_MASK GENMASK(20, 9) /* NUM_BYTES processed */ +#define XI3C_ADDR_MASK GENMASK(6, 0) +#define XI3C_FIFOS_RST_MASK GENMASK(4, 1) + +/* Command FIFO word layout (bit ranges encoded in the GENMASK/BIT args) */ +#define XI3C_CMD_TYPE GENMASK(3, 0) /* command type */ +#define XI3C_CMD_TERMINATE BIT(4) /* terminate (last cmd of xfer) */ +#define XI3C_CMD_ADDR GENMASK(15, 8) /* target address << 1 | RnW */ +#define XI3C_CMD_LEN GENMASK(27, 16) /* payload length in bytes */ +#define XI3C_CMD_TID GENMASK(31, 28) /* transfer ID */ + +/* tLOW_OD open-drain SCL low; MIPI I3C v1.1.1 Table 74 min 200 ns, 500 ns chosen conservatively */ +#define XI3C_OD_TLOW_NS 500 +/* Open-drain SCL high (tHIGH) max; MIPI I3C v1.1.1 Table 74 (41 ns) */ +#define XI3C_OD_THIGH_NS 41 +/* + * tSU_STA/tHD_STA/tSU_STO min, 400 kHz/Fm; MIPI I3C v1.1.1 Table 73 + * (mixed bus with legacy I2C device) + */ +#define XI3C_I2C_TCASMIN_NS 600 +/* tSU_STA/tHD_STA/tSU_STO min, 1 MHz/Fm+; MIPI I3C v1.1.1 Table 73 (pure I3C bus) */ +#define XI3C_TCASMIN_NS 260 +/* Max payload per transfer: 12-bit CMD length field (XI3C_CMD_LEN); PG439 page 32, Table 42 */ +#define XI3C_MAXDATA_LENGTH 4095 +/* Max enumerated devices; PG439 page 27, AXI_I3C_IBI_TARGET_ADDR register detail */ +#define XI3C_MAX_DEVS 128 +/* DAA target response = 48-bit PID + BCR + DCR = 8 bytes; PG439 page 28 */ +#define XI3C_DAA_SLAVEINFO_READ_BYTECOUNT 8 + +/* + * Min SDA hold cycles, rev 0 IP. Revision-specific value, PG439 page 24, + * AXI_I3C_SDA_HOLD_TIME register detail + */ +#define XI3C_THOLD_MIN_REV0 5 +/* + * Min SDA hold cycles, rev >= 1 IP. Revision-specific value, PG439 page 24, + * AXI_I3C_SDA_HOLD_TIME register detail + */ +#define XI3C_THOLD_MIN_REV1 6 +/* + * SCL/SDA pre-bias to account for the HW pipeline. PG439 page 24, + * AXI_I3C_SDA_HOLD_TIME register detail + */ +#define XI3C_CYCLE_ADJUST 2 +/* Short settling delay so the FIFO reset assert/de-assert takes effect before the FIFOs are used */ +#define XI3C_FIFO_RESET_DELAY_US 10 +/* + * Poll/sleep slice for FIFO and response waits: small enough to stay + * responsive, avoids busy-waiting + */ +#define XI3C_POLL_INTERVAL_US 10 + +#define XI3C_I2C_MODE 0 +#define XI3C_I2C_TID 0 +#define XI3C_SDR_MODE 1 +#define XI3C_SDR_TID 1 + +#define XI3C_WORD_LEN 4 + +/* Software guard: 500 ms (us, for readl_poll_timeout) to bail out if no response word arrives */ +#define XI3C_RESP_TIMEOUT_US 500000 +/* Software guard: 1 s (ms, for msecs_to_jiffies) to bail out if a transfer never completes */ +#define XI3C_XFER_TIMEOUT_MS 1000 + +struct xi3c_cmd { + const void *tx_buf; + void *rx_buf; + u16 tx_len; + u16 rx_len; + u16 rx_actual; + u8 addr; + u8 type; + u8 tid; + bool rnw; + bool is_daa; + bool continued; + enum i3c_error_code err; +}; + +struct xi3c_xfer { + unsigned int ncmds; + unsigned int nissued; + struct xi3c_cmd cmds[] __counted_by(ncmds); +}; + +/** + * struct xi3c_master - I3C master controller state. + * @base: I3C master controller embedded by the framework. + * @dev: Pointer to the backing device structure. + * @membase: Memory base of the HW registers. + * @pclk: Input clock driving the controller. + * @lock: Serializes transfers and CCC submission. + * @daa: ENTDAA enumeration state. + * @daa.addrs: Dynamic addresses assigned in enumeration order. + * @daa.index: Number of responders enumerated so far. + */ +struct xi3c_master { + struct i3c_master_controller base; + struct device *dev; + void __iomem *membase; + struct clk *pclk; + struct mutex lock; /* serializes transfers and CCC submission */ + struct { + u8 addrs[XI3C_MAX_DEVS]; + u8 index; + } daa; +}; + +static inline struct xi3c_master * +to_xi3c_master(struct i3c_master_controller *master) +{ + return container_of(master, struct xi3c_master, base); +} + +static inline u8 xi3c_get_revision_number(struct xi3c_master *master) +{ + return FIELD_GET(XI3C_REV_NUM_MASK, + ioread32(master->membase + XI3C_VERSION_OFFSET)); +} + +static inline u16 xi3c_wr_fifo_level(struct xi3c_master *master) +{ + return ioread32(master->membase + XI3C_FIFO_LVL_STATUS_OFFSET) & + XI3C_FIFO_LEVEL_MASK; +} + +static inline u16 xi3c_rd_fifo_level(struct xi3c_master *master) +{ + return ioread32(master->membase + XI3C_FIFO_LVL_STATUS_1_OFFSET) & + XI3C_FIFO_LEVEL_MASK; +} + +static inline bool xi3c_is_resp_available(struct xi3c_master *master) +{ + return FIELD_GET(XI3C_SR_RESP_NOT_EMPTY_MASK, + ioread32(master->membase + XI3C_SR_OFFSET)); +} + +static int xi3c_get_response(struct xi3c_master *master, struct xi3c_cmd *cmd) +{ + u32 response_data; + u32 resp_reg; + u8 code; + int ret; + + ret = readl_poll_timeout(master->membase + XI3C_SR_OFFSET, + resp_reg, + resp_reg & XI3C_SR_RESP_NOT_EMPTY_MASK, + XI3C_POLL_INTERVAL_US, XI3C_RESP_TIMEOUT_US); + if (ret) { + dev_err(master->dev, "XI3C response timeout\n"); + return ret; + } + + response_data = ioread32(master->membase + XI3C_RESP_STATUS_FIFO_OFFSET); + code = FIELD_GET(XI3C_RESP_CODE_MASK, response_data); + + switch (code) { + case XI3C_RESP_CODE_SUCCESS: + cmd->err = I3C_ERROR_UNKNOWN; + cmd->rx_actual = FIELD_GET(XI3C_RESP_BYTES_MASK, response_data); + return 0; + case XI3C_RESP_CODE_READ_EARLY_TERM: + /* Short read: valid data, but controller parked in STOP and must be resumed */ + cmd->err = I3C_ERROR_UNKNOWN; + cmd->rx_actual = FIELD_GET(XI3C_RESP_BYTES_MASK, response_data); + return XI3C_XFER_SHORT_READ; + case XI3C_RESP_CODE_NO_TARGET: + case XI3C_RESP_CODE_NACK: + cmd->err = I3C_ERROR_M2; + return cmd->is_daa ? -ENODEV : -EIO; + default: + cmd->err = I3C_ERROR_M0; + dev_err(master->dev, "XI3C transfer error, response code %u\n", + code); + return -EIO; + } +} + +static inline void xi3c_writesl_be(void __iomem *addr, const void *buffer, + unsigned int count) +{ + const u32 *buf = buffer; + + while (count--) + iowrite32be(get_unaligned(buf++), addr); +} + +static inline void xi3c_readsl_be(const void __iomem *addr, void *buffer, + unsigned int count) +{ + u32 *buf = buffer; + + while (count--) + put_unaligned(ioread32be(addr), buf++); +} + +static inline void xi3c_writel_fifo(void __iomem *addr, const void *buf, + int nbytes) +{ + xi3c_writesl_be(addr, buf, nbytes / 4); + if (nbytes & 3) { + u32 tmp = 0; + + memcpy(&tmp, (const u8 *)buf + (nbytes & ~3), nbytes & 3); + xi3c_writesl_be(addr, &tmp, 1); + } +} + +static inline void xi3c_readl_fifo(const void __iomem *addr, void *buf, + int nbytes) +{ + xi3c_readsl_be(addr, buf, nbytes / 4); + if (nbytes & 3) { + u32 tmp; + + xi3c_readsl_be(addr, &tmp, 1); + memcpy((u8 *)buf + (nbytes & ~3), &tmp, nbytes & 3); + } +} + +static void xi3c_master_write_to_cmdfifo(struct xi3c_master *master, + struct xi3c_cmd *cmd, u16 len) +{ + u32 transfer_cmd; + u8 addr; + + addr = ((cmd->addr & XI3C_ADDR_MASK) << 1) | (u8)cmd->rnw; + + transfer_cmd = FIELD_PREP(XI3C_CMD_TYPE, cmd->type); + transfer_cmd |= FIELD_PREP(XI3C_CMD_TERMINATE, !cmd->continued); + transfer_cmd |= FIELD_PREP(XI3C_CMD_ADDR, addr); + transfer_cmd |= FIELD_PREP(XI3C_CMD_TID, cmd->tid); + + /* + * For dynamic addressing, an additional 1-byte length must be added + * to the command FIFO to account for the address present in the TX FIFO + */ + if (cmd->is_daa) { + xi3c_writel_fifo(master->membase + XI3C_WR_FIFO_OFFSET, + cmd->tx_buf, cmd->tx_len); + + len++; + } + + transfer_cmd |= FIELD_PREP(XI3C_CMD_LEN, len); + iowrite32(transfer_cmd, master->membase + XI3C_CMD_FIFO_OFFSET); +} + +static inline void xi3c_master_enable(struct xi3c_master *master) +{ + iowrite32(ioread32(master->membase + XI3C_CR_OFFSET) | XI3C_CR_EN_MASK, + master->membase + XI3C_CR_OFFSET); +} + +static inline void xi3c_master_disable(struct xi3c_master *master) +{ + iowrite32(ioread32(master->membase + XI3C_CR_OFFSET) & ~XI3C_CR_EN_MASK, + master->membase + XI3C_CR_OFFSET); +} + +static inline void xi3c_master_resume(struct xi3c_master *master) +{ + iowrite32(ioread32(master->membase + XI3C_CR_OFFSET) | + XI3C_CR_RESUME_MASK, master->membase + XI3C_CR_OFFSET); +} + +static void xi3c_master_reset_fifos(struct xi3c_master *master) +{ + u32 data; + + /* Assert FIFO reset. */ + data = ioread32(master->membase + XI3C_RESET_OFFSET); + data |= XI3C_FIFOS_RST_MASK; + iowrite32(data, master->membase + XI3C_RESET_OFFSET); + /* Read-back flushes the posted write before the settling delay below. */ + ioread32(master->membase + XI3C_RESET_OFFSET); + fsleep(XI3C_FIFO_RESET_DELAY_US); + + /* De-assert FIFO reset, then wait for the FIFOs to come back up. */ + data &= ~XI3C_FIFOS_RST_MASK; + iowrite32(data, master->membase + XI3C_RESET_OFFSET); + ioread32(master->membase + XI3C_RESET_OFFSET); + fsleep(XI3C_FIFO_RESET_DELAY_US); +} + +static inline void xi3c_master_init(struct xi3c_master *master) +{ + /* Reset fifos */ + xi3c_master_reset_fifos(master); + + /* Enable controller */ + xi3c_master_enable(master); +} + +static inline void xi3c_master_reinit(struct xi3c_master *master) +{ + /* Reset fifos */ + xi3c_master_reset_fifos(master); + + /* Resume controller */ + xi3c_master_resume(master); +} + +static struct xi3c_xfer *xi3c_master_alloc_xfer(unsigned int ncmds) +{ + struct xi3c_xfer *xfer; + + xfer = kzalloc_flex(*xfer, cmds, ncmds); + if (!xfer) + return NULL; + + xfer->ncmds = ncmds; + + return xfer; +} + +static void xi3c_master_rd_from_rx_fifo(struct xi3c_master *master, + struct xi3c_cmd *cmd) +{ + u16 rx_data_available; + u16 copy_len; + u16 len; + + rx_data_available = xi3c_rd_fifo_level(master); + len = rx_data_available * XI3C_WORD_LEN; + + if (!len) + return; + + copy_len = min(len, cmd->rx_len); + xi3c_readl_fifo(master->membase + XI3C_RD_FIFO_OFFSET, + (u8 *)cmd->rx_buf, copy_len); + + cmd->rx_buf = (u8 *)cmd->rx_buf + copy_len; + cmd->rx_len -= copy_len; +} + +static int xi3c_master_read(struct xi3c_master *master, struct xi3c_cmd *cmd) +{ + unsigned long timeout; + u32 status_reg; + int ret; + + if (cmd->rx_len > XI3C_MAXDATA_LENGTH) + return -EINVAL; + /* + * Zero-length probes (e.g. i2cdetect) legitimately pass a NULL + * buffer; only a non-zero length requires one. + */ + if (cmd->rx_len && !cmd->rx_buf) + return -EINVAL; + + /* Fill command fifo */ + xi3c_master_write_to_cmdfifo(master, cmd, cmd->rx_len); + + if (!cmd->rx_len) + return 0; + + ret = readl_poll_timeout(master->membase + XI3C_SR_OFFSET, + status_reg, + status_reg & (XI3C_RD_FIFO_NOT_EMPTY_MASK | + XI3C_SR_RESP_NOT_EMPTY_MASK), + XI3C_POLL_INTERVAL_US, XI3C_RESP_TIMEOUT_US); + if (ret) { + dev_err(master->dev, "XI3C read timeout\n"); + return ret; + } + + if (!(status_reg & XI3C_RD_FIFO_NOT_EMPTY_MASK)) + return 0; + + timeout = jiffies + msecs_to_jiffies(XI3C_XFER_TIMEOUT_MS); + + /* Read data from rx fifo */ + while (cmd->rx_len > 0 && !xi3c_is_resp_available(master)) { + if (time_after(jiffies, timeout)) { + dev_err(master->dev, "XI3C read timeout\n"); + return -EIO; + } + xi3c_master_rd_from_rx_fifo(master, cmd); + usleep_range(XI3C_POLL_INTERVAL_US, 2 * XI3C_POLL_INTERVAL_US); + } + + /* Read remaining data */ + xi3c_master_rd_from_rx_fifo(master, cmd); + + return 0; +} + +static void xi3c_master_wr_to_tx_fifo(struct xi3c_master *master, + struct xi3c_cmd *cmd) +{ + u16 wrfifo_space; + u16 len; + + wrfifo_space = xi3c_wr_fifo_level(master); + if (cmd->tx_len > wrfifo_space * XI3C_WORD_LEN) + len = wrfifo_space * XI3C_WORD_LEN; + else + len = cmd->tx_len; + + if (len) { + xi3c_writel_fifo(master->membase + XI3C_WR_FIFO_OFFSET, cmd->tx_buf, + len); + + cmd->tx_buf = (const u8 *)cmd->tx_buf + len; + cmd->tx_len -= len; + } +} + +static int xi3c_master_write(struct xi3c_master *master, struct xi3c_cmd *cmd) +{ + unsigned long timeout; + u16 cmd_len; + + if (cmd->tx_len > XI3C_MAXDATA_LENGTH) + return -EINVAL; + /* + * Zero-length probes (e.g. i2cdetect) legitimately pass a NULL + * buffer; only a non-zero length requires one. + */ + if (cmd->tx_len && !cmd->tx_buf) + return -EINVAL; + + cmd_len = cmd->tx_len; + + /* Fill Tx fifo */ + xi3c_master_wr_to_tx_fifo(master, cmd); + + /* Write to command fifo */ + xi3c_master_write_to_cmdfifo(master, cmd, cmd_len); + + timeout = jiffies + msecs_to_jiffies(XI3C_XFER_TIMEOUT_MS); + /* Fill if any remaining data to tx fifo */ + while (cmd->tx_len > 0 && !xi3c_is_resp_available(master)) { + if (time_after(jiffies, timeout)) { + dev_err(master->dev, "XI3C write timeout\n"); + return -EIO; + } + + xi3c_master_wr_to_tx_fifo(master, cmd); + usleep_range(XI3C_POLL_INTERVAL_US, 2 * XI3C_POLL_INTERVAL_US); + } + + return 0; +} + +static int xi3c_master_xfer(struct xi3c_master *master, struct xi3c_cmd *cmd) +{ + int ret; + + if (cmd->rnw) + ret = xi3c_master_read(master, cmd); + else + ret = xi3c_master_write(master, cmd); + + if (ret) + goto err_xfer_out; + + ret = xi3c_get_response(master, cmd); + if (ret < 0) + goto err_xfer_out; + + /* Short read leaves the controller parked in STOP; resume it for the next command */ + if (ret == XI3C_XFER_SHORT_READ) + xi3c_master_resume(master); + + return 0; + +err_xfer_out: + xi3c_master_reinit(master); + return ret; +} + +static int xi3c_master_common_xfer(struct xi3c_master *master, + struct xi3c_xfer *xfer) +{ + unsigned int i; + int ret; + + guard(mutex)(&master->lock); + + for (i = 0; i < xfer->ncmds; i++) { + ret = xi3c_master_xfer(master, &xfer->cmds[i]); + if (ret) { + /* Count commands sent on the bus; the rest never ran */ + xfer->nissued = i + 1; + return ret; + } + } + + xfer->nissued = xfer->ncmds; + + return 0; +} + +static int xi3c_master_do_daa(struct i3c_master_controller *m) +{ + u8 (*pid_bufs)[XI3C_DAA_SLAVEINFO_READ_BYTECOUNT]; + struct xi3c_master *master = to_xi3c_master(m); + struct xi3c_cmd *daa_cmd; + struct xi3c_xfer *xfer; + int addr, ret, i; + u8 last_addr = 0; + u8 *pid_buf; + u8 ccc_id; + + xfer = xi3c_master_alloc_xfer(1); + if (!xfer) + return -ENOMEM; + + pid_bufs = kcalloc(XI3C_MAX_DEVS, XI3C_DAA_SLAVEINFO_READ_BYTECOUNT, + GFP_KERNEL); + if (!pid_bufs) { + ret = -ENOMEM; + goto out; + } + + /* Fill ENTDAA CCC */ + ccc_id = I3C_CCC_ENTDAA; + daa_cmd = &xfer->cmds[0]; + daa_cmd->addr = I3C_BROADCAST_ADDR; + daa_cmd->rnw = false; + daa_cmd->tx_buf = &ccc_id; + daa_cmd->tx_len = 1; + daa_cmd->type = XI3C_SDR_MODE; + daa_cmd->tid = XI3C_SDR_TID; + daa_cmd->continued = true; + + ret = xi3c_master_common_xfer(master, xfer); + if (ret) { + /* NACK on ENTDAA broadcast means no devices to enumerate */ + if (daa_cmd->err == I3C_ERROR_M2) + ret = 0; + goto err_daa; + } + + master->daa.index = 0; + + while (true) { + struct xi3c_cmd *cmd = &xfer->cmds[0]; + u8 daa_byte; + + /* Out of device slots; stop and keep what was enumerated */ + if (master->daa.index >= XI3C_MAX_DEVS) { + dev_warn(master->dev, + "DAA: reached %d devices, stopping enumeration\n", + XI3C_MAX_DEVS); + xi3c_master_reinit(master); + break; + } + + addr = i3c_master_get_free_addr(m, last_addr + 1); + if (addr < 0) { + dev_warn(master->dev, + "DAA: no free dynamic address, stopping enumeration\n"); + xi3c_master_reinit(master); + break; + } + + pid_buf = pid_bufs[master->daa.index]; + + daa_byte = (addr << 1) | (parity8(addr) ^ 1); + + cmd->tx_buf = &daa_byte; + cmd->tx_len = 1; + cmd->addr = I3C_BROADCAST_ADDR; + cmd->rnw = true; + cmd->rx_buf = pid_buf; + cmd->rx_len = XI3C_DAA_SLAVEINFO_READ_BYTECOUNT; + cmd->is_daa = true; + cmd->type = XI3C_SDR_MODE; + cmd->tid = XI3C_SDR_TID; + cmd->continued = true; + + ret = xi3c_master_common_xfer(master, xfer); + + /* -ENODEV: no more responders, enumeration complete */ + if (ret == -ENODEV) { + ret = 0; + break; + } + if (ret) + goto err_daa; + + master->daa.addrs[master->daa.index] = addr; + last_addr = addr; + master->daa.index++; + } + + for (i = 0; i < master->daa.index; i++) { + u64 pid; + + /* Ignore per-device add errors so one failure doesn't abort the rest */ + i3c_master_add_i3c_dev_locked(m, master->daa.addrs[i]); + + pid = FIELD_GET(XI3C_PID_MASK, + get_unaligned_be64(pid_bufs[i])); + dev_dbg(master->dev, "Client %d: PID: 0x%llx\n", i, pid); + } + + ret = 0; + goto out; + +err_daa: + xi3c_master_reinit(master); +out: + kfree(pid_bufs); + kfree(xfer); + return ret; +} + +static bool +xi3c_master_supports_ccc_cmd(struct i3c_master_controller *master, + const struct i3c_ccc_cmd *cmd) +{ + if (cmd->ndests > 1) + return false; + + switch (cmd->id) { + case I3C_CCC_ENEC(true): + case I3C_CCC_ENEC(false): + case I3C_CCC_DISEC(true): + case I3C_CCC_DISEC(false): + case I3C_CCC_ENTAS(0, true): + case I3C_CCC_ENTAS(0, false): + case I3C_CCC_RSTDAA(true): + case I3C_CCC_RSTDAA(false): + case I3C_CCC_ENTDAA: + case I3C_CCC_SETMWL(true): + case I3C_CCC_SETMWL(false): + case I3C_CCC_SETMRL(true): + case I3C_CCC_SETMRL(false): + case I3C_CCC_SETDASA: + case I3C_CCC_SETNEWDA: + case I3C_CCC_GETMWL: + case I3C_CCC_GETMRL: + case I3C_CCC_GETPID: + case I3C_CCC_GETBCR: + case I3C_CCC_GETDCR: + case I3C_CCC_GETSTATUS: + case I3C_CCC_GETMXDS: + return true; + default: + return false; + } +} + +static int xi3c_master_send_bdcast_ccc_cmd(struct xi3c_master *master, + struct i3c_ccc_cmd *ccc) +{ + struct xi3c_xfer *xfer __free(kfree) = NULL; + u8 *buf __free(kfree) = NULL; + struct xi3c_cmd *cmd; + u16 xfer_len; + int ret; + + if (ccc->dests[0].payload.len >= XI3C_MAXDATA_LENGTH) + return -EINVAL; + + xfer_len = ccc->dests[0].payload.len + 1; + + xfer = xi3c_master_alloc_xfer(1); + if (!xfer) + return -ENOMEM; + + buf = kmalloc_objs(*buf, xfer_len); + if (!buf) + return -ENOMEM; + + buf[0] = ccc->id; + memcpy(&buf[1], ccc->dests[0].payload.data, ccc->dests[0].payload.len); + + cmd = &xfer->cmds[0]; + cmd->addr = ccc->dests[0].addr; + cmd->rnw = ccc->rnw; + cmd->tx_buf = buf; + cmd->tx_len = xfer_len; + cmd->type = XI3C_SDR_MODE; + cmd->tid = XI3C_SDR_TID; + cmd->continued = false; + + ret = xi3c_master_common_xfer(master, xfer); + ccc->err = cmd->err; + + return ret; +} + +static int xi3c_master_send_direct_ccc_cmd(struct xi3c_master *master, + struct i3c_ccc_cmd *ccc) +{ + struct xi3c_xfer *xfer __free(kfree) = NULL; + struct xi3c_cmd *cmd; + int ret; + + if (ccc->dests[0].payload.len > XI3C_MAXDATA_LENGTH) + return -EINVAL; + + xfer = xi3c_master_alloc_xfer(2); + if (!xfer) + return -ENOMEM; + + /* Broadcasted message */ + cmd = &xfer->cmds[0]; + cmd->addr = I3C_BROADCAST_ADDR; + cmd->rnw = false; + cmd->tx_buf = &ccc->id; + cmd->tx_len = 1; + cmd->type = XI3C_SDR_MODE; + cmd->tid = XI3C_SDR_TID; + cmd->continued = true; + + /* Directed message */ + cmd = &xfer->cmds[1]; + cmd->addr = ccc->dests[0].addr; + cmd->rnw = ccc->rnw; + if (cmd->rnw) { + cmd->rx_buf = ccc->dests[0].payload.data; + cmd->rx_len = ccc->dests[0].payload.len; + } else { + cmd->tx_buf = ccc->dests[0].payload.data; + cmd->tx_len = ccc->dests[0].payload.len; + } + cmd->type = XI3C_SDR_MODE; + cmd->tid = XI3C_SDR_TID; + cmd->continued = false; + + ret = xi3c_master_common_xfer(master, xfer); + + /* + * Report broadcast error if any, else the directed one, so either + * NACK reaches the caller + */ + ccc->err = xfer->cmds[0].err ? xfer->cmds[0].err : xfer->cmds[1].err; + + /* Report actual byte count so the core sees the right length on short reads */ + if (!ret && ccc->rnw) + ccc->dests[0].payload.len = min(xfer->cmds[1].rx_actual, + ccc->dests[0].payload.len); + + return ret; +} + +static int xi3c_master_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *cmd) +{ + struct xi3c_master *master = to_xi3c_master(m); + + if (cmd->id & I3C_CCC_DIRECT) + return xi3c_master_send_direct_ccc_cmd(master, cmd); + + return xi3c_master_send_bdcast_ccc_cmd(master, cmd); +} + +static int xi3c_master_i3c_xfers(struct i3c_dev_desc *dev, + struct i3c_xfer *xfers, + int nxfers, enum i3c_xfer_mode mode) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct xi3c_master *master = to_xi3c_master(m); + struct xi3c_xfer *xfer __free(kfree) = NULL; + int i, ret; + + if (!nxfers) + return 0; + + if (mode != I3C_SDR) + return -EOPNOTSUPP; + + for (i = 0; i < nxfers; i++) + if (xfers[i].len > XI3C_MAXDATA_LENGTH) + return -EINVAL; + + xfer = xi3c_master_alloc_xfer(nxfers); + if (!xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + struct xi3c_cmd *cmd = &xfer->cmds[i]; + + cmd->addr = dev->info.dyn_addr; + cmd->rnw = xfers[i].rnw; + + if (cmd->rnw) { + cmd->rx_buf = xfers[i].data.in; + cmd->rx_len = xfers[i].len; + } else { + cmd->tx_buf = xfers[i].data.out; + cmd->tx_len = xfers[i].len; + } + + cmd->type = XI3C_SDR_MODE; + cmd->tid = XI3C_SDR_TID; + cmd->continued = (i + 1) < nxfers; + } + + ret = xi3c_master_common_xfer(master, xfer); + + for (i = 0; i < xfer->nissued; i++) { + xfers[i].err = xfer->cmds[i].err; + if (xfers[i].rnw) + xfers[i].actual_len = min(xfer->cmds[i].rx_actual, + xfers[i].len); + } + + return ret; +} + +static int xi3c_master_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *xfers, + int nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct xi3c_master *master = to_xi3c_master(m); + struct xi3c_xfer *xfer __free(kfree) = NULL; + int i; + + if (!nxfers) + return 0; + + for (i = 0; i < nxfers; i++) + if (xfers[i].len > XI3C_MAXDATA_LENGTH) + return -EINVAL; + + xfer = xi3c_master_alloc_xfer(nxfers); + if (!xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + struct xi3c_cmd *cmd = &xfer->cmds[i]; + + cmd->addr = xfers[i].addr & XI3C_ADDR_MASK; + cmd->rnw = !!(xfers[i].flags & I2C_M_RD); + + if (cmd->rnw) { + cmd->rx_buf = xfers[i].buf; + cmd->rx_len = xfers[i].len; + } else { + cmd->tx_buf = xfers[i].buf; + cmd->tx_len = xfers[i].len; + } + + cmd->type = XI3C_I2C_MODE; + cmd->tid = XI3C_I2C_TID; + cmd->continued = (i + 1) < nxfers; + } + + return xi3c_master_common_xfer(master, xfer); +} + +static int xi3c_clk_cfg(struct xi3c_master *master, unsigned long sclhz, u8 mode) +{ + unsigned long core_rate, core_periodns; + u32 tcasmin, tsustart, tsustop, thdstart; + u32 thigh, tlow, thold; + u32 odthigh, odtlow; + + core_rate = clk_get_rate(master->pclk); + if (!core_rate) + return -EINVAL; + + if (!sclhz) + return -EINVAL; + + core_periodns = DIV_ROUND_UP(NSEC_PER_SEC, core_rate); + + thigh = DIV_ROUND_UP(core_rate, sclhz) >> 1; + tlow = thigh; + + /* Reject rates whose timing exceeds the 18-bit registers (would wrap) */ + if (thigh <= XI3C_CYCLE_ADJUST || + (thigh - XI3C_CYCLE_ADJUST) > XI3C_TIMING_MASK) + return -EINVAL; + + /* Hold time : 40% of tlow time */ + thold = (tlow * 4) / 10; + + if (xi3c_get_revision_number(master) == 0) + thold = max_t(u32, thold, XI3C_THOLD_MIN_REV0); + else + thold = max_t(u32, thold, XI3C_THOLD_MIN_REV1); + + iowrite32((thigh - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_SCL_HIGH_TIME_OFFSET); + iowrite32((tlow - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_SCL_LOW_TIME_OFFSET); + iowrite32((thold - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_SDA_HOLD_TIME_OFFSET); + + if (mode == XI3C_I2C_MODE) { + iowrite32((thigh - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_OD_SCL_HIGH_TIME_OFFSET); + iowrite32((tlow - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_OD_SCL_LOW_TIME_OFFSET); + + tcasmin = DIV_ROUND_UP(XI3C_I2C_TCASMIN_NS, core_periodns); + } else { + odtlow = DIV_ROUND_UP(XI3C_OD_TLOW_NS, core_periodns); + odthigh = DIV_ROUND_UP(XI3C_OD_THIGH_NS, core_periodns); + + odtlow = max(tlow, odtlow); + odthigh = min(thigh, odthigh); + + if (odthigh <= XI3C_CYCLE_ADJUST) + return -EINVAL; + + iowrite32((odthigh - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_OD_SCL_HIGH_TIME_OFFSET); + iowrite32((odtlow - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_OD_SCL_LOW_TIME_OFFSET); + + tcasmin = DIV_ROUND_UP(XI3C_TCASMIN_NS, core_periodns); + } + + thdstart = max(thigh, tcasmin); + tsustart = max(tlow, tcasmin); + tsustop = max(tlow, tcasmin); + + iowrite32((tsustart - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_TSU_START_OFFSET); + iowrite32((thdstart - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_THD_START_OFFSET); + iowrite32((tsustop - XI3C_CYCLE_ADJUST) & XI3C_TIMING_MASK, + master->membase + XI3C_TSU_STOP_OFFSET); + + return 0; +} + +static int xi3c_master_bus_init(struct i3c_master_controller *m) +{ + struct xi3c_master *master = to_xi3c_master(m); + struct i3c_bus *bus = i3c_master_get_bus(m); + struct i3c_device_info info = {}; + unsigned long sclhz; + u32 pid1_bcr_dcr; + u8 mode; + int ret; + + switch (bus->mode) { + case I3C_BUS_MODE_MIXED_FAST: + case I3C_BUS_MODE_MIXED_LIMITED: + case I3C_BUS_MODE_MIXED_SLOW: + mode = XI3C_I2C_MODE; + sclhz = bus->scl_rate.i2c; + break; + case I3C_BUS_MODE_PURE: + mode = XI3C_SDR_MODE; + sclhz = bus->scl_rate.i3c; + break; + default: + return -EINVAL; + } + + ret = xi3c_clk_cfg(master, sclhz, mode); + if (ret) + return ret; + + xi3c_master_init(master); + + /* Get an address for the master. */ + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + return ret; + + info.dyn_addr = ret; + + /* Write the dynamic address value to the address register. */ + iowrite32(info.dyn_addr, master->membase + XI3C_ADDRESS_OFFSET); + + /* Read PID, BCR and DCR values, and assign to i3c device info. */ + pid1_bcr_dcr = ioread32(master->membase + XI3C_PID1_BCR_DCR); + info.pid = ((u64)FIELD_GET(XI3C_PID1_MASK, pid1_bcr_dcr) << 32) | + ioread32(master->membase + XI3C_PID0_OFFSET); + info.bcr = FIELD_GET(XI3C_BCR_MASK, pid1_bcr_dcr); + info.dcr = FIELD_GET(XI3C_DCR_MASK, pid1_bcr_dcr); + + return i3c_master_set_info(&master->base, &info); +} + +static void xi3c_master_bus_cleanup(struct i3c_master_controller *m) +{ + struct xi3c_master *master = to_xi3c_master(m); + + xi3c_master_disable(master); +} + +static const struct i3c_master_controller_ops xi3c_master_ops = { + .bus_init = xi3c_master_bus_init, + .bus_cleanup = xi3c_master_bus_cleanup, + .do_daa = xi3c_master_do_daa, + .supports_ccc_cmd = xi3c_master_supports_ccc_cmd, + .send_ccc_cmd = xi3c_master_send_ccc_cmd, + .i3c_xfers = xi3c_master_i3c_xfers, + .i2c_xfers = xi3c_master_i2c_xfers, +}; + +static int xi3c_master_probe(struct platform_device *pdev) +{ + struct xi3c_master *master; + int ret; + + master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + master->dev = &pdev->dev; + + master->membase = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(master->membase)) + return dev_err_probe(master->dev, PTR_ERR(master->membase), + "Failed to map registers\n"); + + master->pclk = devm_clk_get_enabled(master->dev, NULL); + if (IS_ERR(master->pclk)) + return dev_err_probe(master->dev, PTR_ERR(master->pclk), + "Failed to get and enable clock\n"); + + ret = devm_mutex_init(master->dev, &master->lock); + if (ret) + return ret; + + platform_set_drvdata(pdev, master); + + return i3c_master_register(&master->base, master->dev, + &xi3c_master_ops, false); +} + +static void xi3c_master_remove(struct platform_device *pdev) +{ + struct xi3c_master *master = platform_get_drvdata(pdev); + + i3c_master_unregister(&master->base); +} + +static const struct of_device_id xi3c_master_of_ids[] = { + { .compatible = "xlnx,axi-i3c-1.0" }, + { }, +}; +MODULE_DEVICE_TABLE(of, xi3c_master_of_ids); + +static struct platform_driver xi3c_master_driver = { + .probe = xi3c_master_probe, + .remove = xi3c_master_remove, + .driver = { + .name = "axi-i3c-master", + .of_match_table = xi3c_master_of_ids, + }, +}; +module_platform_driver(xi3c_master_driver); + +MODULE_AUTHOR("Manikanta Guntupalli <manikanta.guntupalli@amd.com>"); +MODULE_DESCRIPTION("AMD AXI I3C master driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/i3c/master/ast2600-i3c-master.c b/drivers/i3c/master/ast2600-i3c-master.c new file mode 100644 index 000000000..e05e83812 --- /dev/null +++ b/drivers/i3c/master/ast2600-i3c-master.c @@ -0,0 +1,187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2023 Code Construct + * + * Author: Jeremy Kerr <jk@codeconstruct.com.au> + */ + +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> + +#include "dw-i3c-master.h" + +/* AST2600-specific global register set */ +#define AST2600_I3CG_REG0(idx) (((idx) * 4 * 4) + 0x10) +#define AST2600_I3CG_REG1(idx) (((idx) * 4 * 4) + 0x14) + +#define AST2600_I3CG_REG0_SDA_PULLUP_EN_MASK GENMASK(29, 28) +#define AST2600_I3CG_REG0_SDA_PULLUP_EN_2K (0x0 << 28) +#define AST2600_I3CG_REG0_SDA_PULLUP_EN_750 (0x2 << 28) +#define AST2600_I3CG_REG0_SDA_PULLUP_EN_545 (0x3 << 28) + +#define AST2600_I3CG_REG1_I2C_MODE BIT(0) +#define AST2600_I3CG_REG1_TEST_MODE BIT(1) +#define AST2600_I3CG_REG1_ACT_MODE_MASK GENMASK(3, 2) +#define AST2600_I3CG_REG1_ACT_MODE(x) (((x) << 2) & AST2600_I3CG_REG1_ACT_MODE_MASK) +#define AST2600_I3CG_REG1_PENDING_INT_MASK GENMASK(7, 4) +#define AST2600_I3CG_REG1_PENDING_INT(x) (((x) << 4) & AST2600_I3CG_REG1_PENDING_INT_MASK) +#define AST2600_I3CG_REG1_SA_MASK GENMASK(14, 8) +#define AST2600_I3CG_REG1_SA(x) (((x) << 8) & AST2600_I3CG_REG1_SA_MASK) +#define AST2600_I3CG_REG1_SA_EN BIT(15) +#define AST2600_I3CG_REG1_INST_ID_MASK GENMASK(19, 16) +#define AST2600_I3CG_REG1_INST_ID(x) (((x) << 16) & AST2600_I3CG_REG1_INST_ID_MASK) + +#define AST2600_DEFAULT_SDA_PULLUP_OHMS 2000 + +#define DEV_ADDR_TABLE_IBI_PEC BIT(11) + +struct ast2600_i3c { + struct dw_i3c_master dw; + struct regmap *global_regs; + unsigned int global_idx; + unsigned int sda_pullup; +}; + +static struct ast2600_i3c *to_ast2600_i3c(struct dw_i3c_master *dw) +{ + return container_of(dw, struct ast2600_i3c, dw); +} + +static int ast2600_i3c_pullup_to_reg(unsigned int ohms, u32 *regp) +{ + u32 reg; + + switch (ohms) { + case 2000: + reg = AST2600_I3CG_REG0_SDA_PULLUP_EN_2K; + break; + case 750: + reg = AST2600_I3CG_REG0_SDA_PULLUP_EN_750; + break; + case 545: + reg = AST2600_I3CG_REG0_SDA_PULLUP_EN_545; + break; + default: + return -EINVAL; + } + + if (regp) + *regp = reg; + + return 0; +} + +static int ast2600_i3c_init(struct dw_i3c_master *dw) +{ + struct ast2600_i3c *i3c = to_ast2600_i3c(dw); + u32 reg = 0; + int rc; + + /* reg0: set SDA pullup values */ + rc = ast2600_i3c_pullup_to_reg(i3c->sda_pullup, ®); + if (rc) + return rc; + + rc = regmap_write(i3c->global_regs, + AST2600_I3CG_REG0(i3c->global_idx), reg); + if (rc) + return rc; + + /* reg1: set up the instance id, but leave everything else disabled, + * as it's all for client mode + */ + reg = AST2600_I3CG_REG1_INST_ID(i3c->global_idx); + rc = regmap_write(i3c->global_regs, + AST2600_I3CG_REG1(i3c->global_idx), reg); + + return rc; +} + +static void ast2600_i3c_set_dat_ibi(struct dw_i3c_master *i3c, + struct i3c_dev_desc *dev, + bool enable, u32 *dat) +{ + /* + * The ast2600 i3c controller will lock up on receiving 4n+1-byte IBIs + * if the PEC is disabled. We have no way to restrict the length of + * IBIs sent to the controller, so we need to unconditionally enable + * PEC checking, which means we drop a byte of payload data + */ + if (enable && dev->info.bcr & I3C_BCR_IBI_PAYLOAD) { + dev_warn_once(&i3c->base.dev, + "Enabling PEC workaround. IBI payloads will be truncated\n"); + *dat |= DEV_ADDR_TABLE_IBI_PEC; + } +} + +static const struct dw_i3c_platform_ops ast2600_i3c_ops = { + .init = ast2600_i3c_init, + .set_dat_ibi = ast2600_i3c_set_dat_ibi, +}; + +static int ast2600_i3c_probe(struct platform_device *pdev) +{ + struct device_node *np = pdev->dev.of_node; + struct of_phandle_args gspec; + struct ast2600_i3c *i3c; + int rc; + + i3c = devm_kzalloc(&pdev->dev, sizeof(*i3c), GFP_KERNEL); + if (!i3c) + return -ENOMEM; + + rc = of_parse_phandle_with_fixed_args(np, "aspeed,global-regs", 1, 0, + &gspec); + if (rc) + return -ENODEV; + + i3c->global_regs = syscon_node_to_regmap(gspec.np); + of_node_put(gspec.np); + + if (IS_ERR(i3c->global_regs)) + return PTR_ERR(i3c->global_regs); + + i3c->global_idx = gspec.args[0]; + + rc = of_property_read_u32(np, "sda-pullup-ohms", &i3c->sda_pullup); + if (rc) + i3c->sda_pullup = AST2600_DEFAULT_SDA_PULLUP_OHMS; + + rc = ast2600_i3c_pullup_to_reg(i3c->sda_pullup, NULL); + if (rc) + dev_err(&pdev->dev, "invalid sda-pullup value %d\n", + i3c->sda_pullup); + + i3c->dw.platform_ops = &ast2600_i3c_ops; + return dw_i3c_common_probe(&i3c->dw, pdev); +} + +static void ast2600_i3c_remove(struct platform_device *pdev) +{ + struct dw_i3c_master *dw_i3c = platform_get_drvdata(pdev); + + dw_i3c_common_remove(dw_i3c); +} + +static const struct of_device_id ast2600_i3c_master_of_match[] = { + { .compatible = "aspeed,ast2600-i3c", }, + {}, +}; +MODULE_DEVICE_TABLE(of, ast2600_i3c_master_of_match); + +static struct platform_driver ast2600_i3c_driver = { + .probe = ast2600_i3c_probe, + .remove = ast2600_i3c_remove, + .driver = { + .name = "ast2600-i3c-master", + .of_match_table = ast2600_i3c_master_of_match, + }, +}; +module_platform_driver(ast2600_i3c_driver); + +MODULE_AUTHOR("Jeremy Kerr <jk@codeconstruct.com.au>"); +MODULE_DESCRIPTION("ASPEED AST2600 I3C driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/i3c/master/dw-i3c-master.c b/drivers/i3c/master/dw-i3c-master.c new file mode 100644 index 000000000..4563d8761 --- /dev/null +++ b/drivers/i3c/master/dw-i3c-master.c @@ -0,0 +1,1958 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2018 Synopsys, Inc. and/or its affiliates. + * + * Author: Vitor Soares <vitor.soares@synopsys.com> + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/interrupt.h> +#include <linux/ioport.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <linux/slab.h> + +#include "../internals.h" +#include "dw-i3c-master.h" + +#define DEVICE_CTRL 0x0 +#define DEV_CTRL_ENABLE BIT(31) +#define DEV_CTRL_RESUME BIT(30) +#define DEV_CTRL_HOT_JOIN_NACK BIT(8) +#define DEV_CTRL_I2C_SLAVE_PRESENT BIT(7) + +#define DEVICE_ADDR 0x4 +#define DEV_ADDR_DYNAMIC_ADDR_VALID BIT(31) +#define DEV_ADDR_DYNAMIC(x) (((x) << 16) & GENMASK(22, 16)) + +#define HW_CAPABILITY 0x8 +#define COMMAND_QUEUE_PORT 0xc +#define COMMAND_PORT_TOC BIT(30) +#define COMMAND_PORT_READ_TRANSFER BIT(28) +#define COMMAND_PORT_SDAP BIT(27) +#define COMMAND_PORT_ROC BIT(26) +#define COMMAND_PORT_SPEED(x) (((x) << 21) & GENMASK(23, 21)) +#define COMMAND_PORT_DEV_INDEX(x) (((x) << 16) & GENMASK(20, 16)) +#define COMMAND_PORT_CP BIT(15) +#define COMMAND_PORT_CMD(x) (((x) << 7) & GENMASK(14, 7)) +#define COMMAND_PORT_TID(x) (((x) << 3) & GENMASK(6, 3)) + +#define COMMAND_PORT_ARG_DATA_LEN(x) (((x) << 16) & GENMASK(31, 16)) +#define COMMAND_PORT_ARG_DATA_LEN_MAX 65536 +#define COMMAND_PORT_TRANSFER_ARG 0x01 + +#define COMMAND_PORT_SDA_DATA_BYTE_3(x) (((x) << 24) & GENMASK(31, 24)) +#define COMMAND_PORT_SDA_DATA_BYTE_2(x) (((x) << 16) & GENMASK(23, 16)) +#define COMMAND_PORT_SDA_DATA_BYTE_1(x) (((x) << 8) & GENMASK(15, 8)) +#define COMMAND_PORT_SDA_BYTE_STRB_3 BIT(5) +#define COMMAND_PORT_SDA_BYTE_STRB_2 BIT(4) +#define COMMAND_PORT_SDA_BYTE_STRB_1 BIT(3) +#define COMMAND_PORT_SHORT_DATA_ARG 0x02 + +#define COMMAND_PORT_DEV_COUNT(x) (((x) << 21) & GENMASK(25, 21)) +#define COMMAND_PORT_ADDR_ASSGN_CMD 0x03 + +#define RESPONSE_QUEUE_PORT 0x10 +#define RESPONSE_PORT_ERR_STATUS(x) (((x) & GENMASK(31, 28)) >> 28) +#define RESPONSE_NO_ERROR 0 +#define RESPONSE_ERROR_CRC 1 +#define RESPONSE_ERROR_PARITY 2 +#define RESPONSE_ERROR_FRAME 3 +#define RESPONSE_ERROR_IBA_NACK 4 +#define RESPONSE_ERROR_ADDRESS_NACK 5 +#define RESPONSE_ERROR_OVER_UNDER_FLOW 6 +#define RESPONSE_ERROR_TRANSF_ABORT 8 +#define RESPONSE_ERROR_I2C_W_NACK_ERR 9 +#define RESPONSE_PORT_TID(x) (((x) & GENMASK(27, 24)) >> 24) +#define RESPONSE_PORT_DATA_LEN(x) ((x) & GENMASK(15, 0)) + +#define RX_TX_DATA_PORT 0x14 +#define IBI_QUEUE_STATUS 0x18 +#define IBI_QUEUE_STATUS_IBI_ID(x) (((x) & GENMASK(15, 8)) >> 8) +#define IBI_QUEUE_STATUS_DATA_LEN(x) ((x) & GENMASK(7, 0)) +#define IBI_QUEUE_IBI_ADDR(x) (IBI_QUEUE_STATUS_IBI_ID(x) >> 1) +#define IBI_QUEUE_IBI_RNW(x) (IBI_QUEUE_STATUS_IBI_ID(x) & BIT(0)) +#define IBI_TYPE_MR(x) \ + ((IBI_QUEUE_IBI_ADDR(x) != I3C_HOT_JOIN_ADDR) && !IBI_QUEUE_IBI_RNW(x)) +#define IBI_TYPE_HJ(x) \ + ((IBI_QUEUE_IBI_ADDR(x) == I3C_HOT_JOIN_ADDR) && !IBI_QUEUE_IBI_RNW(x)) +#define IBI_TYPE_SIRQ(x) \ + ((IBI_QUEUE_IBI_ADDR(x) != I3C_HOT_JOIN_ADDR) && IBI_QUEUE_IBI_RNW(x)) + +#define QUEUE_THLD_CTRL 0x1c +#define QUEUE_THLD_CTRL_IBI_STAT_MASK GENMASK(31, 24) +#define QUEUE_THLD_CTRL_IBI_STAT(x) (((x) - 1) << 24) +#define QUEUE_THLD_CTRL_IBI_DATA_MASK GENMASK(20, 16) +#define QUEUE_THLD_CTRL_IBI_DATA(x) ((x) << 16) +#define QUEUE_THLD_CTRL_RESP_BUF_MASK GENMASK(15, 8) +#define QUEUE_THLD_CTRL_RESP_BUF(x) (((x) - 1) << 8) + +#define DATA_BUFFER_THLD_CTRL 0x20 +#define DATA_BUFFER_THLD_CTRL_RX_BUF GENMASK(11, 8) + +#define IBI_QUEUE_CTRL 0x24 +#define IBI_MR_REQ_REJECT 0x2C +#define IBI_SIR_REQ_REJECT 0x30 +#define IBI_REQ_REJECT_ALL GENMASK(31, 0) + +#define RESET_CTRL 0x34 +#define RESET_CTRL_IBI_QUEUE BIT(5) +#define RESET_CTRL_RX_FIFO BIT(4) +#define RESET_CTRL_TX_FIFO BIT(3) +#define RESET_CTRL_RESP_QUEUE BIT(2) +#define RESET_CTRL_CMD_QUEUE BIT(1) +#define RESET_CTRL_SOFT BIT(0) + +#define SLV_EVENT_CTRL 0x38 +#define INTR_STATUS 0x3c +#define INTR_STATUS_EN 0x40 +#define INTR_SIGNAL_EN 0x44 +#define INTR_FORCE 0x48 +#define INTR_BUSOWNER_UPDATE_STAT BIT(13) +#define INTR_IBI_UPDATED_STAT BIT(12) +#define INTR_READ_REQ_RECV_STAT BIT(11) +#define INTR_DEFSLV_STAT BIT(10) +#define INTR_TRANSFER_ERR_STAT BIT(9) +#define INTR_DYN_ADDR_ASSGN_STAT BIT(8) +#define INTR_CCC_UPDATED_STAT BIT(6) +#define INTR_TRANSFER_ABORT_STAT BIT(5) +#define INTR_RESP_READY_STAT BIT(4) +#define INTR_CMD_QUEUE_READY_STAT BIT(3) +#define INTR_IBI_THLD_STAT BIT(2) +#define INTR_RX_THLD_STAT BIT(1) +#define INTR_TX_THLD_STAT BIT(0) +#define INTR_ALL (INTR_BUSOWNER_UPDATE_STAT | \ + INTR_IBI_UPDATED_STAT | \ + INTR_READ_REQ_RECV_STAT | \ + INTR_DEFSLV_STAT | \ + INTR_TRANSFER_ERR_STAT | \ + INTR_DYN_ADDR_ASSGN_STAT | \ + INTR_CCC_UPDATED_STAT | \ + INTR_TRANSFER_ABORT_STAT | \ + INTR_RESP_READY_STAT | \ + INTR_CMD_QUEUE_READY_STAT | \ + INTR_IBI_THLD_STAT | \ + INTR_TX_THLD_STAT | \ + INTR_RX_THLD_STAT) + +#define INTR_MASTER_MASK (INTR_TRANSFER_ERR_STAT | \ + INTR_RESP_READY_STAT) + +#define QUEUE_STATUS_LEVEL 0x4c +#define QUEUE_STATUS_IBI_STATUS_CNT(x) (((x) & GENMASK(28, 24)) >> 24) +#define QUEUE_STATUS_IBI_BUF_BLR(x) (((x) & GENMASK(23, 16)) >> 16) +#define QUEUE_STATUS_LEVEL_RESP(x) (((x) & GENMASK(15, 8)) >> 8) +#define QUEUE_STATUS_LEVEL_CMD(x) ((x) & GENMASK(7, 0)) + +#define DATA_BUFFER_STATUS_LEVEL 0x50 +#define DATA_BUFFER_STATUS_LEVEL_TX(x) ((x) & GENMASK(7, 0)) + +#define PRESENT_STATE 0x54 +#define CCC_DEVICE_STATUS 0x58 +#define DEVICE_ADDR_TABLE_POINTER 0x5c +#define DEVICE_ADDR_TABLE_DEPTH(x) (((x) & GENMASK(31, 16)) >> 16) +#define DEVICE_ADDR_TABLE_ADDR(x) ((x) & GENMASK(7, 0)) + +#define DEV_CHAR_TABLE_POINTER 0x60 +#define VENDOR_SPECIFIC_REG_POINTER 0x6c +#define SLV_PID_VALUE 0x74 +#define SLV_CHAR_CTRL 0x78 +#define SLV_MAX_LEN 0x7c +#define MAX_READ_TURNAROUND 0x80 +#define MAX_DATA_SPEED 0x84 +#define SLV_DEBUG_STATUS 0x88 +#define SLV_INTR_REQ 0x8c +#define DEVICE_CTRL_EXTENDED 0xb0 +#define SCL_I3C_OD_TIMING 0xb4 +#define SCL_I3C_PP_TIMING 0xb8 +#define SCL_I3C_TIMING_HCNT(x) (((x) << 16) & GENMASK(23, 16)) +#define SCL_I3C_TIMING_LCNT(x) ((x) & GENMASK(7, 0)) +#define SCL_I3C_TIMING_CNT_MIN 5 + +#define SCL_I2C_FM_TIMING 0xbc +#define SCL_I2C_FM_TIMING_HCNT(x) (((x) << 16) & GENMASK(31, 16)) +#define SCL_I2C_FM_TIMING_LCNT(x) ((x) & GENMASK(15, 0)) + +#define SCL_I2C_FMP_TIMING 0xc0 +#define SCL_I2C_FMP_TIMING_HCNT(x) (((x) << 16) & GENMASK(23, 16)) +#define SCL_I2C_FMP_TIMING_LCNT(x) ((x) & GENMASK(15, 0)) + +#define SCL_EXT_LCNT_TIMING 0xc8 +#define SCL_EXT_LCNT_4(x) (((x) << 24) & GENMASK(31, 24)) +#define SCL_EXT_LCNT_3(x) (((x) << 16) & GENMASK(23, 16)) +#define SCL_EXT_LCNT_2(x) (((x) << 8) & GENMASK(15, 8)) +#define SCL_EXT_LCNT_1(x) ((x) & GENMASK(7, 0)) + +#define SCL_EXT_TERMN_LCNT_TIMING 0xcc +#define BUS_FREE_TIMING 0xd4 +#define BUS_I3C_MST_FREE(x) ((x) & GENMASK(15, 0)) + +#define BUS_IDLE_TIMING 0xd8 +#define I3C_VER_ID 0xe0 +#define I3C_VER_TYPE 0xe4 +#define EXTENDED_CAPABILITY 0xe8 +#define SLAVE_CONFIG 0xec + +#define DYN_ADDR_LO_MASK GENMASK(4, 0) +#define DYN_ADDR_HI_MASK GENMASK(6, 5) +#define IBI_SIR_BIT_MOD 32 /* 32-bit vector */ + +#define DW_I3C_DEV_NACK_RETRY_CNT_MAX 0x3 +#define DEV_ADDR_TABLE_DEV_NACK_RETRY_MASK GENMASK(30, 29) +#define DEV_ADDR_TABLE_DYNAMIC_MASK GENMASK(23, 16) +#define DEV_ADDR_TABLE_STATIC_MASK GENMASK(6, 0) +#define DEV_ADDR_TABLE_IBI_MDB BIT(12) +#define DEV_ADDR_TABLE_SIR_REJECT BIT(13) +#define DEV_ADDR_TABLE_DEV_NACK_RETRY_CNT(x) \ + FIELD_PREP(DEV_ADDR_TABLE_DEV_NACK_RETRY_MASK, (x)) +#define DEV_ADDR_TABLE_LEGACY_I2C_DEV BIT(31) +#define DEV_ADDR_TABLE_DYNAMIC_ADDR(x) FIELD_PREP(DEV_ADDR_TABLE_DYNAMIC_MASK, x) +#define DEV_ADDR_TABLE_STATIC_ADDR(x) FIELD_PREP(DEV_ADDR_TABLE_STATIC_MASK, x) +#define DEV_ADDR_TABLE_LOC(start, idx) ((start) + ((idx) << 2)) +#define DEV_ADDR_TABLE_GET_DYNAMIC_ADDR(x) FIELD_GET(DEV_ADDR_TABLE_DYNAMIC_MASK, x) + +#define I3C_BUS_SDR1_SCL_RATE 8000000 +#define I3C_BUS_SDR2_SCL_RATE 6000000 +#define I3C_BUS_SDR3_SCL_RATE 4000000 +#define I3C_BUS_SDR4_SCL_RATE 2000000 +#define I3C_BUS_I2C_FM_TLOW_MIN_NS 1300 +#define I3C_BUS_I2C_FMP_TLOW_MIN_NS 500 +#define I3C_BUS_THIGH_MAX_NS 41 + +#define XFER_TIMEOUT (msecs_to_jiffies(1000)) +#define RPM_AUTOSUSPEND_TIMEOUT 1000 /* ms */ + +/* Timing values to configure 12.5MHz frequency */ +#define AMD_I3C_OD_TIMING 0x4C007C +#define AMD_I3C_PP_TIMING 0x8001A + +/* List of quirks */ +#define AMD_I3C_OD_PP_TIMING BIT(1) +#define DW_I3C_DISABLE_RUNTIME_PM_QUIRK BIT(2) +#define DW_I3C_ACPI_SKIP_CLK_RST BIT(3) + +struct dw_i3c_cmd { + u32 cmd_lo; + u32 cmd_hi; + u16 tx_len; + u16 rx_len; + u8 error; + const void *tx_buf; + void *rx_buf; +}; + +struct dw_i3c_xfer { + struct list_head node; + struct completion comp; + int ret; + unsigned int ncmds; + struct dw_i3c_cmd cmds[] __counted_by(ncmds); +}; + +struct dw_i3c_i2c_dev_data { + u8 index; + struct i3c_generic_ibi_pool *ibi_pool; +}; + +struct dw_i3c_drvdata { + u32 flags; +}; + +static inline u32 get_ibi_sir_bit_index(u8 addr) +{ + u32 lo = FIELD_GET(DYN_ADDR_LO_MASK, addr); + u32 hi = FIELD_GET(DYN_ADDR_HI_MASK, addr); + + return (lo + hi) % IBI_SIR_BIT_MOD; +} + +static bool dw_i3c_master_supports_ccc_cmd(struct i3c_master_controller *m, + const struct i3c_ccc_cmd *cmd) +{ + if (cmd->ndests > 1) + return false; + + switch (cmd->id) { + case I3C_CCC_ENEC(true): + case I3C_CCC_ENEC(false): + case I3C_CCC_DISEC(true): + case I3C_CCC_DISEC(false): + case I3C_CCC_ENTAS(0, true): + case I3C_CCC_ENTAS(0, false): + case I3C_CCC_RSTDAA(true): + case I3C_CCC_RSTDAA(false): + case I3C_CCC_ENTDAA: + case I3C_CCC_SETMWL(true): + case I3C_CCC_SETMWL(false): + case I3C_CCC_SETMRL(true): + case I3C_CCC_SETMRL(false): + case I3C_CCC_ENTHDR(0): + case I3C_CCC_SETDASA: + case I3C_CCC_SETNEWDA: + case I3C_CCC_GETMWL: + case I3C_CCC_GETMRL: + case I3C_CCC_GETPID: + case I3C_CCC_GETBCR: + case I3C_CCC_GETDCR: + case I3C_CCC_GETSTATUS: + case I3C_CCC_GETMXDS: + case I3C_CCC_GETHDRCAP: + case I3C_CCC_SETAASA: + case I3C_CCC_VENDOR(0, true): /* SETHID */ + return true; + default: + return false; + } +} + +static inline struct dw_i3c_master * +to_dw_i3c_master(struct i3c_master_controller *master) +{ + return container_of(master, struct dw_i3c_master, base); +} + +static void dw_i3c_master_disable(struct dw_i3c_master *master) +{ + writel(readl(master->regs + DEVICE_CTRL) & ~DEV_CTRL_ENABLE, + master->regs + DEVICE_CTRL); +} + +static void dw_i3c_master_enable(struct dw_i3c_master *master) +{ + u32 dev_ctrl; + + dev_ctrl = readl(master->regs + DEVICE_CTRL); + /* For now don't support Hot-Join */ + dev_ctrl |= DEV_CTRL_HOT_JOIN_NACK; + if (master->i2c_slv_prsnt) + dev_ctrl |= DEV_CTRL_I2C_SLAVE_PRESENT; + writel(dev_ctrl | DEV_CTRL_ENABLE, + master->regs + DEVICE_CTRL); +} + +static int dw_i3c_master_get_addr_pos(struct dw_i3c_master *master, u8 addr) +{ + int pos; + + for (pos = 0; pos < master->maxdevs; pos++) { + if (addr == master->devs[pos].addr) + return pos; + } + + return -EINVAL; +} + +static int dw_i3c_master_get_free_pos(struct dw_i3c_master *master) +{ + if (!(master->free_pos & GENMASK(master->maxdevs - 1, 0))) + return -ENOSPC; + + return ffs(master->free_pos) - 1; +} + +static void dw_i3c_master_wr_tx_fifo(struct dw_i3c_master *master, + const u8 *bytes, int nbytes) +{ + i3c_writel_fifo(master->regs + RX_TX_DATA_PORT, bytes, nbytes); +} + +static void dw_i3c_master_read_rx_fifo(struct dw_i3c_master *master, + u8 *bytes, int nbytes) +{ + i3c_readl_fifo(master->regs + RX_TX_DATA_PORT, bytes, nbytes); +} + +static void dw_i3c_master_read_ibi_fifo(struct dw_i3c_master *master, + u8 *bytes, int nbytes) +{ + i3c_readl_fifo(master->regs + IBI_QUEUE_STATUS, bytes, nbytes); +} + +static struct dw_i3c_xfer * +dw_i3c_master_alloc_xfer(struct dw_i3c_master *master, unsigned int ncmds) +{ + struct dw_i3c_xfer *xfer; + + xfer = kzalloc_flex(*xfer, cmds, ncmds); + if (!xfer) + return NULL; + + INIT_LIST_HEAD(&xfer->node); + xfer->ncmds = ncmds; + xfer->ret = -ETIMEDOUT; + + return xfer; +} + +static void dw_i3c_master_start_xfer_locked(struct dw_i3c_master *master) +{ + struct dw_i3c_xfer *xfer = master->xferqueue.cur; + unsigned int i; + u32 thld_ctrl; + + if (!xfer) + return; + + for (i = 0; i < xfer->ncmds; i++) { + struct dw_i3c_cmd *cmd = &xfer->cmds[i]; + + dw_i3c_master_wr_tx_fifo(master, cmd->tx_buf, cmd->tx_len); + } + + thld_ctrl = readl(master->regs + QUEUE_THLD_CTRL); + thld_ctrl &= ~QUEUE_THLD_CTRL_RESP_BUF_MASK; + thld_ctrl |= QUEUE_THLD_CTRL_RESP_BUF(xfer->ncmds); + writel(thld_ctrl, master->regs + QUEUE_THLD_CTRL); + + for (i = 0; i < xfer->ncmds; i++) { + struct dw_i3c_cmd *cmd = &xfer->cmds[i]; + + writel(cmd->cmd_hi, master->regs + COMMAND_QUEUE_PORT); + writel(cmd->cmd_lo, master->regs + COMMAND_QUEUE_PORT); + } +} + +static void dw_i3c_master_enqueue_xfer(struct dw_i3c_master *master, + struct dw_i3c_xfer *xfer) +{ + init_completion(&xfer->comp); + guard(spinlock_irqsave)(&master->xferqueue.lock); + if (master->xferqueue.cur) { + list_add_tail(&xfer->node, &master->xferqueue.list); + } else { + master->xferqueue.cur = xfer; + dw_i3c_master_start_xfer_locked(master); + } +} + +static void dw_i3c_master_dequeue_xfer_locked(struct dw_i3c_master *master, + struct dw_i3c_xfer *xfer) +{ + if (master->xferqueue.cur == xfer) { + u32 status; + + master->xferqueue.cur = NULL; + + writel(RESET_CTRL_RX_FIFO | RESET_CTRL_TX_FIFO | + RESET_CTRL_RESP_QUEUE | RESET_CTRL_CMD_QUEUE, + master->regs + RESET_CTRL); + + readl_poll_timeout_atomic(master->regs + RESET_CTRL, status, + !status, 10, 1000000); + } else { + list_del_init(&xfer->node); + } +} + +static void dw_i3c_master_dequeue_xfer(struct dw_i3c_master *master, + struct dw_i3c_xfer *xfer) +{ + guard(spinlock_irqsave)(&master->xferqueue.lock); + dw_i3c_master_dequeue_xfer_locked(master, xfer); +} + +static void dw_i3c_master_end_xfer_locked(struct dw_i3c_master *master, u32 isr) +{ + struct dw_i3c_xfer *xfer = master->xferqueue.cur; + int i, ret = 0; + u32 nresp; + + if (!xfer) + return; + + nresp = readl(master->regs + QUEUE_STATUS_LEVEL); + nresp = QUEUE_STATUS_LEVEL_RESP(nresp); + + for (i = 0; i < nresp; i++) { + struct dw_i3c_cmd *cmd; + u32 resp; + + resp = readl(master->regs + RESPONSE_QUEUE_PORT); + + cmd = &xfer->cmds[RESPONSE_PORT_TID(resp)]; + cmd->rx_len = RESPONSE_PORT_DATA_LEN(resp); + cmd->error = RESPONSE_PORT_ERR_STATUS(resp); + if (cmd->rx_len && !cmd->error) + dw_i3c_master_read_rx_fifo(master, cmd->rx_buf, + cmd->rx_len); + } + + for (i = 0; i < nresp; i++) { + switch (xfer->cmds[i].error) { + case RESPONSE_NO_ERROR: + break; + case RESPONSE_ERROR_PARITY: + case RESPONSE_ERROR_IBA_NACK: + case RESPONSE_ERROR_ADDRESS_NACK: + case RESPONSE_ERROR_TRANSF_ABORT: + case RESPONSE_ERROR_CRC: + case RESPONSE_ERROR_FRAME: + ret = -EIO; + break; + case RESPONSE_ERROR_OVER_UNDER_FLOW: + ret = -ENOSPC; + break; + case RESPONSE_ERROR_I2C_W_NACK_ERR: + default: + ret = -EINVAL; + break; + } + } + + xfer->ret = ret; + complete(&xfer->comp); + + if (ret < 0) { + dw_i3c_master_dequeue_xfer_locked(master, xfer); + writel(readl(master->regs + DEVICE_CTRL) | DEV_CTRL_RESUME, + master->regs + DEVICE_CTRL); + } + + xfer = list_first_entry_or_null(&master->xferqueue.list, + struct dw_i3c_xfer, + node); + if (xfer) + list_del_init(&xfer->node); + + master->xferqueue.cur = xfer; + dw_i3c_master_start_xfer_locked(master); +} + +static void dw_i3c_master_set_intr_regs(struct dw_i3c_master *master) +{ + u32 thld_ctrl; + + thld_ctrl = readl(master->regs + QUEUE_THLD_CTRL); + thld_ctrl &= ~(QUEUE_THLD_CTRL_RESP_BUF_MASK | + QUEUE_THLD_CTRL_IBI_STAT_MASK | + QUEUE_THLD_CTRL_IBI_DATA_MASK); + thld_ctrl |= QUEUE_THLD_CTRL_IBI_STAT(1) | + QUEUE_THLD_CTRL_IBI_DATA(31); + writel(thld_ctrl, master->regs + QUEUE_THLD_CTRL); + + thld_ctrl = readl(master->regs + DATA_BUFFER_THLD_CTRL); + thld_ctrl &= ~DATA_BUFFER_THLD_CTRL_RX_BUF; + writel(thld_ctrl, master->regs + DATA_BUFFER_THLD_CTRL); + + writel(INTR_ALL, master->regs + INTR_STATUS); + writel(INTR_MASTER_MASK, master->regs + INTR_STATUS_EN); + writel(INTR_MASTER_MASK, master->regs + INTR_SIGNAL_EN); + + master->sir_rej_mask = IBI_REQ_REJECT_ALL; + writel(master->sir_rej_mask, master->regs + IBI_SIR_REQ_REJECT); + + writel(IBI_REQ_REJECT_ALL, master->regs + IBI_MR_REQ_REJECT); +} + +static unsigned long dw_i3c_master_get_core_rate(struct dw_i3c_master *master) +{ + unsigned int core_rate_prop; + + if (master->core_clk) + return clk_get_rate(master->core_clk); + + if (device_property_read_u32(master->dev, "clock-frequency", &core_rate_prop)) { + dev_err(master->dev, "missing clock-frequency property\n"); + return 0; + } + + return core_rate_prop; +} + +static void amd_configure_od_pp_quirk(struct dw_i3c_master *master) +{ + master->i3c_od_timing = AMD_I3C_OD_TIMING; + master->i3c_od_timing_normal = AMD_I3C_OD_TIMING; + master->i3c_pp_timing = AMD_I3C_PP_TIMING; +} + +static int dw_i3c_clk_cfg(struct dw_i3c_master *master) +{ + unsigned long core_rate, core_period; + u32 scl_timing; + u8 hcnt, lcnt; + + core_rate = dw_i3c_master_get_core_rate(master); + if (!core_rate) + return -EINVAL; + + core_period = DIV_ROUND_UP(1000000000, core_rate); + + hcnt = DIV_ROUND_UP(I3C_BUS_THIGH_MAX_NS, core_period) - 1; + if (hcnt < SCL_I3C_TIMING_CNT_MIN) + hcnt = SCL_I3C_TIMING_CNT_MIN; + + lcnt = DIV_ROUND_UP(core_rate, master->base.bus.scl_rate.i3c) - hcnt; + if (lcnt < SCL_I3C_TIMING_CNT_MIN) + lcnt = SCL_I3C_TIMING_CNT_MIN; + + scl_timing = SCL_I3C_TIMING_HCNT(hcnt) | SCL_I3C_TIMING_LCNT(lcnt); + writel(scl_timing, master->regs + SCL_I3C_PP_TIMING); + master->i3c_pp_timing = scl_timing; + + /* + * In pure i3c mode, MST_FREE represents tCAS. In shared mode, this + * will be set up by dw_i2c_clk_cfg as tLOW. + */ + if (master->base.bus.mode == I3C_BUS_MODE_PURE) { + writel(BUS_I3C_MST_FREE(lcnt), master->regs + BUS_FREE_TIMING); + master->bus_free_timing = BUS_I3C_MST_FREE(lcnt); + } + + lcnt = max_t(u8, + DIV_ROUND_UP(I3C_BUS_TLOW_OD_MIN_NS, core_period), lcnt); + scl_timing = SCL_I3C_TIMING_HCNT(hcnt) | SCL_I3C_TIMING_LCNT(lcnt); + writel(scl_timing, master->regs + SCL_I3C_OD_TIMING); + master->i3c_od_timing = scl_timing; + master->i3c_od_timing_normal = scl_timing; + + /* + * AMD legacy platforms need fixed OD/PP timings. Cache them as the + * normal OD baseline so set_speed(NORMAL) restores AMD values, and + * set_speed(SLOW) can stretch HCNT while keeping the AMD LCNT. + */ + if (master->quirks & AMD_I3C_OD_PP_TIMING) { + amd_configure_od_pp_quirk(master); + writel(master->i3c_pp_timing, master->regs + SCL_I3C_PP_TIMING); + writel(master->i3c_od_timing, master->regs + SCL_I3C_OD_TIMING); + } + + lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR1_SCL_RATE) - hcnt; + scl_timing = SCL_EXT_LCNT_1(lcnt); + lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR2_SCL_RATE) - hcnt; + scl_timing |= SCL_EXT_LCNT_2(lcnt); + lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR3_SCL_RATE) - hcnt; + scl_timing |= SCL_EXT_LCNT_3(lcnt); + lcnt = DIV_ROUND_UP(core_rate, I3C_BUS_SDR4_SCL_RATE) - hcnt; + scl_timing |= SCL_EXT_LCNT_4(lcnt); + writel(scl_timing, master->regs + SCL_EXT_LCNT_TIMING); + master->ext_lcnt_timing = scl_timing; + + return 0; +} + +static int dw_i2c_clk_cfg(struct dw_i3c_master *master) +{ + unsigned long core_rate, core_period; + u16 hcnt, lcnt; + u32 scl_timing; + + core_rate = dw_i3c_master_get_core_rate(master); + if (!core_rate) + return -EINVAL; + + core_period = DIV_ROUND_UP(1000000000, core_rate); + + lcnt = DIV_ROUND_UP(I3C_BUS_I2C_FMP_TLOW_MIN_NS, core_period); + hcnt = DIV_ROUND_UP(core_rate, I3C_BUS_I2C_FM_PLUS_SCL_MAX_RATE) - lcnt; + scl_timing = SCL_I2C_FMP_TIMING_HCNT(hcnt) | + SCL_I2C_FMP_TIMING_LCNT(lcnt); + writel(scl_timing, master->regs + SCL_I2C_FMP_TIMING); + master->i2c_fmp_timing = scl_timing; + + lcnt = DIV_ROUND_UP(I3C_BUS_I2C_FM_TLOW_MIN_NS, core_period); + hcnt = DIV_ROUND_UP(core_rate, I3C_BUS_I2C_FM_SCL_MAX_RATE) - lcnt; + scl_timing = SCL_I2C_FM_TIMING_HCNT(hcnt) | + SCL_I2C_FM_TIMING_LCNT(lcnt); + writel(scl_timing, master->regs + SCL_I2C_FM_TIMING); + master->i2c_fm_timing = scl_timing; + + writel(BUS_I3C_MST_FREE(lcnt), master->regs + BUS_FREE_TIMING); + master->bus_free_timing = BUS_I3C_MST_FREE(lcnt); + + writel(readl(master->regs + DEVICE_CTRL) | DEV_CTRL_I2C_SLAVE_PRESENT, + master->regs + DEVICE_CTRL); + master->i2c_slv_prsnt = true; + + return 0; +} + +static int dw_i3c_master_bus_init(struct i3c_master_controller *m) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + struct i3c_bus *bus = i3c_master_get_bus(m); + struct i3c_device_info info = { }; + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + ret = master->platform_ops->init(master); + if (ret) + goto rpm_out; + + switch (bus->mode) { + case I3C_BUS_MODE_MIXED_FAST: + case I3C_BUS_MODE_MIXED_LIMITED: + ret = dw_i2c_clk_cfg(master); + if (ret) + goto rpm_out; + fallthrough; + case I3C_BUS_MODE_PURE: + ret = dw_i3c_clk_cfg(master); + if (ret) + goto rpm_out; + break; + default: + ret = -EINVAL; + goto rpm_out; + } + + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + goto rpm_out; + + writel(DEV_ADDR_DYNAMIC_ADDR_VALID | DEV_ADDR_DYNAMIC(ret), + master->regs + DEVICE_ADDR); + master->dev_addr = ret; + memset(&info, 0, sizeof(info)); + info.dyn_addr = ret; + + ret = i3c_master_set_info(&master->base, &info); + if (ret) + goto rpm_out; + + dw_i3c_master_set_intr_regs(master); + dw_i3c_master_enable(master); + +rpm_out: + pm_runtime_put_autosuspend(master->dev); + return ret; +} + +static void dw_i3c_master_bus_cleanup(struct i3c_master_controller *m) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + + dw_i3c_master_disable(master); +} + +static enum i3c_error_code dw_i3c_ccc_map_err(u8 dw_err) +{ + switch (dw_err) { + case RESPONSE_ERROR_IBA_NACK: + return I3C_ERROR_M2; + case RESPONSE_ERROR_CRC: + case RESPONSE_ERROR_PARITY: + case RESPONSE_ERROR_FRAME: + case RESPONSE_ERROR_TRANSF_ABORT: + return I3C_ERROR_M0; + default: + return I3C_ERROR_UNKNOWN; + } +} + +static int dw_i3c_ccc_set(struct dw_i3c_master *master, + struct i3c_ccc_cmd *ccc) +{ + struct dw_i3c_cmd *cmd; + int ret, pos = 0; + + ccc->err = I3C_ERROR_UNKNOWN; + + if (ccc->id & I3C_CCC_DIRECT) { + pos = dw_i3c_master_get_addr_pos(master, ccc->dests[0].addr); + if (pos < 0) + return pos; + } + + struct dw_i3c_xfer *xfer __free(kfree) = dw_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + cmd = xfer->cmds; + cmd->tx_buf = ccc->dests[0].payload.data; + cmd->tx_len = ccc->dests[0].payload.len; + + cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(ccc->dests[0].payload.len) | + COMMAND_PORT_TRANSFER_ARG; + + cmd->cmd_lo = COMMAND_PORT_CP | + COMMAND_PORT_DEV_INDEX(pos) | + COMMAND_PORT_CMD(ccc->id) | + COMMAND_PORT_TOC | + COMMAND_PORT_ROC; + + dw_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) + dw_i3c_master_dequeue_xfer(master, xfer); + + ret = xfer->ret; + cmd = &xfer->cmds[0]; + ccc->err = dw_i3c_ccc_map_err(cmd->error); + + return ret; +} + +static int dw_i3c_ccc_get(struct dw_i3c_master *master, struct i3c_ccc_cmd *ccc) +{ + struct dw_i3c_cmd *cmd; + int ret, pos; + + ccc->err = I3C_ERROR_UNKNOWN; + + pos = dw_i3c_master_get_addr_pos(master, ccc->dests[0].addr); + if (pos < 0) + return pos; + + struct dw_i3c_xfer *xfer __free(kfree) = dw_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + cmd = xfer->cmds; + cmd->rx_buf = ccc->dests[0].payload.data; + cmd->rx_len = ccc->dests[0].payload.len; + + cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(ccc->dests[0].payload.len) | + COMMAND_PORT_TRANSFER_ARG; + + cmd->cmd_lo = COMMAND_PORT_READ_TRANSFER | + COMMAND_PORT_CP | + COMMAND_PORT_DEV_INDEX(pos) | + COMMAND_PORT_CMD(ccc->id) | + COMMAND_PORT_TOC | + COMMAND_PORT_ROC; + + dw_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) + dw_i3c_master_dequeue_xfer(master, xfer); + + ret = xfer->ret; + cmd = &xfer->cmds[0]; + ccc->err = dw_i3c_ccc_map_err(cmd->error); + if (!ret) + ccc->dests[0].payload.actual_len = cmd->rx_len; + + return ret; +} + +static int dw_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *ccc) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + int ret = 0; + + if (ccc->id == I3C_CCC_ENTDAA) + return -EINVAL; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + if (ccc->rnw) + ret = dw_i3c_ccc_get(master, ccc); + else + ret = dw_i3c_ccc_set(master, ccc); + + pm_runtime_put_autosuspend(master->dev); + return ret; +} + +static int dw_i3c_master_daa(struct i3c_master_controller *m) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + struct dw_i3c_cmd *cmd; + u32 olddevs, newdevs; + u8 last_addr = 0; + int ret, pos; + + struct dw_i3c_xfer *xfer __free(kfree) = dw_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + olddevs = ~(master->free_pos); + + /* Prepare DAT before launching DAA. */ + for (pos = 0; pos < master->maxdevs; pos++) { + if (olddevs & BIT(pos)) + continue; + + ret = i3c_master_get_free_addr(m, last_addr + 1); + if (ret < 0) { + ret = -ENOSPC; + goto rpm_out; + } + + master->devs[pos].addr = ret; + last_addr = ret; + + ret |= parity8(ret) ? 0 : BIT(7); + + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(ret), + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, pos)); + + ret = 0; + } + + pos = dw_i3c_master_get_free_pos(master); + if (pos < 0) { + ret = pos; + goto rpm_out; + } + cmd = &xfer->cmds[0]; + cmd->rx_len = master->maxdevs; + cmd->cmd_hi = COMMAND_PORT_TRANSFER_ARG; + cmd->cmd_lo = COMMAND_PORT_DEV_COUNT(master->maxdevs - pos) | + COMMAND_PORT_DEV_INDEX(pos) | + COMMAND_PORT_CMD(I3C_CCC_ENTDAA) | + COMMAND_PORT_ADDR_ASSGN_CMD | + COMMAND_PORT_TOC | + COMMAND_PORT_ROC; + + dw_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) + dw_i3c_master_dequeue_xfer(master, xfer); + + /* + * cmd->rx_len holds the number of addresses ENTDAA left unassigned. + * On an empty bus rx_len == maxdevs, so avoid GENMASK(-1, 0). + */ + if (cmd->rx_len >= master->maxdevs) + newdevs = 0; + else + newdevs = GENMASK(master->maxdevs - cmd->rx_len - 1, 0); + + newdevs &= ~olddevs; + + for (pos = 0; pos < master->maxdevs; pos++) { + if (newdevs & BIT(pos)) + i3c_master_add_i3c_dev_locked(m, master->devs[pos].addr); + } + +rpm_out: + pm_runtime_put_autosuspend(master->dev); + return ret; +} + +static int dw_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, + struct i3c_xfer *i3c_xfers, + int i3c_nxfers, enum i3c_xfer_mode mode) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + unsigned int nrxwords = 0, ntxwords = 0; + int i, ret = 0; + + if (!i3c_nxfers) + return 0; + + if (i3c_nxfers > master->caps.cmdfifodepth) + return -EOPNOTSUPP; + + for (i = 0; i < i3c_nxfers; i++) { + if (i3c_xfers[i].rnw) + nrxwords += DIV_ROUND_UP(i3c_xfers[i].len, 4); + else + ntxwords += DIV_ROUND_UP(i3c_xfers[i].len, 4); + } + + if (ntxwords > master->caps.datafifodepth || + nrxwords > master->caps.datafifodepth) + return -EOPNOTSUPP; + + struct dw_i3c_xfer *xfer __free(kfree) = dw_i3c_master_alloc_xfer(master, i3c_nxfers); + if (!xfer) + return -ENOMEM; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + for (i = 0; i < i3c_nxfers; i++) { + struct dw_i3c_cmd *cmd = &xfer->cmds[i]; + + cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(i3c_xfers[i].len) | + COMMAND_PORT_TRANSFER_ARG; + + if (i3c_xfers[i].rnw) { + cmd->rx_buf = i3c_xfers[i].data.in; + cmd->rx_len = i3c_xfers[i].len; + cmd->cmd_lo = COMMAND_PORT_READ_TRANSFER | + COMMAND_PORT_SPEED(dev->info.max_read_ds); + + } else { + cmd->tx_buf = i3c_xfers[i].data.out; + cmd->tx_len = i3c_xfers[i].len; + cmd->cmd_lo = + COMMAND_PORT_SPEED(dev->info.max_write_ds); + } + + cmd->cmd_lo |= COMMAND_PORT_TID(i) | + COMMAND_PORT_DEV_INDEX(data->index) | + COMMAND_PORT_ROC; + + if (i == (i3c_nxfers - 1)) + cmd->cmd_lo |= COMMAND_PORT_TOC; + } + + dw_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, XFER_TIMEOUT)) + dw_i3c_master_dequeue_xfer(master, xfer); + + for (i = 0; i < i3c_nxfers; i++) { + struct dw_i3c_cmd *cmd = &xfer->cmds[i]; + + if (i3c_xfers[i].rnw) + i3c_xfers[i].len = cmd->rx_len; + } + + ret = xfer->ret; + + pm_runtime_put_autosuspend(master->dev); + return ret; +} + +static int dw_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, + u8 old_dyn_addr) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + int pos; + + pos = dw_i3c_master_get_free_pos(master); + + if (data->index > pos && pos > 0) { + writel(0, + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); + + master->devs[data->index].addr = 0; + master->free_pos |= BIT(data->index); + + data->index = pos; + master->devs[pos].addr = dev->info.dyn_addr; + master->free_pos &= ~BIT(pos); + } + + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(dev->info.dyn_addr) | DEV_ADDR_TABLE_SIR_REJECT, + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); + + master->devs[data->index].addr = dev->info.dyn_addr; + + return 0; +} + +static int dw_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + struct dw_i3c_i2c_dev_data *data; + int pos; + + pos = dw_i3c_master_get_free_pos(master); + if (pos < 0) + return pos; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + data->index = pos; + master->devs[pos].addr = dev->info.dyn_addr ? : dev->info.static_addr; + master->free_pos &= ~BIT(pos); + i3c_dev_set_master_data(dev, data); + + writel(DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr) | DEV_ADDR_TABLE_SIR_REJECT, + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); + + return 0; +} + +static void dw_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + + writel(0, + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); + + i3c_dev_set_master_data(dev, NULL); + master->devs[data->index].addr = 0; + master->free_pos |= BIT(data->index); + kfree(data); +} + +static int dw_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *i2c_xfers, + int i2c_nxfers) +{ + struct dw_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + unsigned int nrxwords = 0, ntxwords = 0; + int i, ret = 0; + + if (!i2c_nxfers) + return 0; + + if (i2c_nxfers > master->caps.cmdfifodepth) + return -EOPNOTSUPP; + + for (i = 0; i < i2c_nxfers; i++) { + if (i2c_xfers[i].flags & I2C_M_RD) + nrxwords += DIV_ROUND_UP(i2c_xfers[i].len, 4); + else + ntxwords += DIV_ROUND_UP(i2c_xfers[i].len, 4); + } + + if (ntxwords > master->caps.datafifodepth || + nrxwords > master->caps.datafifodepth) + return -EOPNOTSUPP; + + struct dw_i3c_xfer *xfer __free(kfree) = dw_i3c_master_alloc_xfer(master, i2c_nxfers); + if (!xfer) + return -ENOMEM; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + for (i = 0; i < i2c_nxfers; i++) { + struct dw_i3c_cmd *cmd = &xfer->cmds[i]; + + cmd->cmd_hi = COMMAND_PORT_ARG_DATA_LEN(i2c_xfers[i].len) | + COMMAND_PORT_TRANSFER_ARG; + + cmd->cmd_lo = COMMAND_PORT_TID(i) | + COMMAND_PORT_DEV_INDEX(data->index) | + COMMAND_PORT_ROC; + + if (i2c_xfers[i].flags & I2C_M_RD) { + cmd->cmd_lo |= COMMAND_PORT_READ_TRANSFER; + cmd->rx_buf = i2c_xfers[i].buf; + cmd->rx_len = i2c_xfers[i].len; + } else { + cmd->tx_buf = i2c_xfers[i].buf; + cmd->tx_len = i2c_xfers[i].len; + } + + if (i == (i2c_nxfers - 1)) + cmd->cmd_lo |= COMMAND_PORT_TOC; + } + + dw_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, m->i2c.timeout)) + dw_i3c_master_dequeue_xfer(master, xfer); + + ret = xfer->ret; + + pm_runtime_put_autosuspend(master->dev); + return ret; +} + +static int dw_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + struct dw_i3c_i2c_dev_data *data; + int pos; + + pos = dw_i3c_master_get_free_pos(master); + if (pos < 0) + return pos; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + data->index = pos; + master->devs[pos].addr = dev->addr; + master->devs[pos].is_i2c_addr = true; + master->free_pos &= ~BIT(pos); + i2c_dev_set_master_data(dev, data); + + writel(DEV_ADDR_TABLE_LEGACY_I2C_DEV | + DEV_ADDR_TABLE_STATIC_ADDR(dev->addr), + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); + + return 0; +} + +static void dw_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct dw_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + + writel(0, + master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, data->index)); + + i2c_dev_set_master_data(dev, NULL); + master->devs[data->index].addr = 0; + master->free_pos |= BIT(data->index); + kfree(data); +} + +static int dw_i3c_master_request_ibi(struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + + data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(data->ibi_pool)) + return PTR_ERR(data->ibi_pool); + + guard(spinlock_irqsave)(&master->devs_lock); + master->devs[data->index].ibi_dev = dev; + + return 0; +} + +static void dw_i3c_master_free_ibi(struct i3c_dev_desc *dev) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + + scoped_guard(spinlock_irqsave, &master->devs_lock) { + master->devs[data->index].ibi_dev = NULL; + } + + i3c_generic_ibi_free_pool(data->ibi_pool); + data->ibi_pool = NULL; +} + +static void dw_i3c_master_enable_sir_signal(struct dw_i3c_master *master, bool enable) +{ + u32 reg; + + reg = readl(master->regs + INTR_STATUS_EN); + reg &= ~INTR_IBI_THLD_STAT; + if (enable) + reg |= INTR_IBI_THLD_STAT; + writel(reg, master->regs + INTR_STATUS_EN); + + reg = readl(master->regs + INTR_SIGNAL_EN); + reg &= ~INTR_IBI_THLD_STAT; + if (enable) + reg |= INTR_IBI_THLD_STAT; + writel(reg, master->regs + INTR_SIGNAL_EN); +} + +static void dw_i3c_master_set_sir_enabled(struct dw_i3c_master *master, + struct i3c_dev_desc *dev, + u8 idx, bool enable) +{ + u32 dat_entry, reg; + bool global; + u8 dynamic_addr; + + dat_entry = DEV_ADDR_TABLE_LOC(master->datstartaddr, idx); + + guard(spinlock_irqsave)(&master->devs_lock); + reg = readl(master->regs + dat_entry); + dynamic_addr = DEV_ADDR_TABLE_GET_DYNAMIC_ADDR(reg); + + if (!dynamic_addr) + dev_warn(master->dev, + "<%s> unassigned slave device, dynamic addr:%x\n", + __func__, dynamic_addr); + + if (enable) { + reg &= ~DEV_ADDR_TABLE_SIR_REJECT; + if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) + reg |= DEV_ADDR_TABLE_IBI_MDB; + } else { + reg |= DEV_ADDR_TABLE_SIR_REJECT; + } + master->platform_ops->set_dat_ibi(master, dev, enable, ®); + writel(reg, master->regs + dat_entry); + + if (enable) { + global = (master->sir_rej_mask == IBI_REQ_REJECT_ALL); + master->sir_rej_mask &= ~BIT(get_ibi_sir_bit_index(dynamic_addr)); + } else { + bool hj_rejected = !!(readl(master->regs + DEVICE_CTRL) & DEV_CTRL_HOT_JOIN_NACK); + + master->sir_rej_mask |= BIT(get_ibi_sir_bit_index(dynamic_addr)); + global = (master->sir_rej_mask == IBI_REQ_REJECT_ALL) && hj_rejected; + } + writel(master->sir_rej_mask, master->regs + IBI_SIR_REQ_REJECT); + + if (global) + dw_i3c_master_enable_sir_signal(master, enable); +} + +static int dw_i3c_master_enable_hotjoin(struct i3c_master_controller *m) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + dw_i3c_master_enable_sir_signal(master, true); + writel(readl(master->regs + DEVICE_CTRL) & ~DEV_CTRL_HOT_JOIN_NACK, + master->regs + DEVICE_CTRL); + + return 0; +} + +static int dw_i3c_master_disable_hotjoin(struct i3c_master_controller *m) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + + writel(readl(master->regs + DEVICE_CTRL) | DEV_CTRL_HOT_JOIN_NACK, + master->regs + DEVICE_CTRL); + + pm_runtime_put_autosuspend(master->dev); + return 0; +} + +static int dw_i3c_master_enable_ibi(struct i3c_dev_desc *dev) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + int rc; + + rc = pm_runtime_resume_and_get(master->dev); + if (rc < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, rc); + return rc; + } + + dw_i3c_master_set_sir_enabled(master, dev, data->index, true); + + rc = i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); + + if (rc) { + dw_i3c_master_set_sir_enabled(master, dev, data->index, false); + pm_runtime_put_autosuspend(master->dev); + } + + return rc; +} + +static int dw_i3c_master_disable_ibi(struct i3c_dev_desc *dev) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct dw_i3c_master *master = to_dw_i3c_master(m); + int rc; + + rc = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); + if (rc) + return rc; + + dw_i3c_master_set_sir_enabled(master, dev, data->index, false); + + pm_runtime_put_autosuspend(master->dev); + return 0; +} + +static void dw_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct dw_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); +} + +static void dw_i3c_master_drain_ibi_queue(struct dw_i3c_master *master, + int len) +{ + int i; + + for (i = 0; i < DIV_ROUND_UP(len, 4); i++) + readl(master->regs + IBI_QUEUE_STATUS); +} + +static void dw_i3c_master_handle_ibi_sir(struct dw_i3c_master *master, + u32 status) +{ + struct dw_i3c_i2c_dev_data *data; + struct i3c_ibi_slot *slot; + struct i3c_dev_desc *dev; + u8 addr, len; + int idx; + + addr = IBI_QUEUE_IBI_ADDR(status); + len = IBI_QUEUE_STATUS_DATA_LEN(status); + + /* + * We be tempted to check the error status in bit 30; however, due + * to the PEC errata workaround on some platform implementations (see + * ast2600_i3c_set_dat_ibi()), those will almost always have a PEC + * error on IBI payload data, as well as losing the last byte of + * payload. + * + * If we implement error status checking on that bit, we may need + * a new platform op to validate it. + */ + + guard(spinlock_irqsave)(&master->devs_lock); + idx = dw_i3c_master_get_addr_pos(master, addr); + if (idx < 0) { + dev_dbg_ratelimited(&master->base.dev, + "IBI from unknown addr 0x%x\n", addr); + goto err_drain; + } + + dev = master->devs[idx].ibi_dev; + if (!dev || !dev->ibi) { + dev_dbg_ratelimited(&master->base.dev, + "IBI from non-requested dev idx %d\n", idx); + goto err_drain; + } + + data = i3c_dev_get_master_data(dev); + slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); + if (!slot) { + dev_dbg_ratelimited(&master->base.dev, + "No IBI slots available\n"); + goto err_drain; + } + + if (dev->ibi->max_payload_len < len) { + dev_dbg_ratelimited(&master->base.dev, + "IBI payload len %d greater than max %d\n", + len, dev->ibi->max_payload_len); + goto err_drain; + } + + if (len) { + dw_i3c_master_read_ibi_fifo(master, slot->data, len); + slot->len = len; + } + i3c_master_queue_ibi(dev, slot); + + return; + +err_drain: + dw_i3c_master_drain_ibi_queue(master, len); +} + +/* "ibis": referring to In-Band Interrupts, and not + * https://en.wikipedia.org/wiki/Australian_white_ibis. The latter should + * not be handled. + */ +static void dw_i3c_master_irq_handle_ibis(struct dw_i3c_master *master) +{ + unsigned int i, len, n_ibis; + u32 reg; + + reg = readl(master->regs + QUEUE_STATUS_LEVEL); + if (master->has_ibi_data) + n_ibis = QUEUE_STATUS_IBI_STATUS_CNT(reg); + else + n_ibis = QUEUE_STATUS_IBI_BUF_BLR(reg); + + if (!n_ibis) + return; + + for (i = 0; i < n_ibis; i++) { + reg = readl(master->regs + IBI_QUEUE_STATUS); + + if (IBI_TYPE_SIRQ(reg)) { + dw_i3c_master_handle_ibi_sir(master, reg); + } else if (IBI_TYPE_HJ(reg)) { + i3c_master_queue_hotjoin(&master->base); + } else { + len = IBI_QUEUE_STATUS_DATA_LEN(reg); + dev_info(&master->base.dev, + "unsupported IBI type 0x%lx len %d\n", + IBI_QUEUE_STATUS_IBI_ID(reg), len); + dw_i3c_master_drain_ibi_queue(master, len); + } + } +} + +static irqreturn_t dw_i3c_master_irq_handler(int irq, void *dev_id) +{ + struct dw_i3c_master *master = dev_id; + u32 status; + + status = readl(master->regs + INTR_STATUS); + + if (!(status & readl(master->regs + INTR_STATUS_EN))) { + writel(INTR_ALL, master->regs + INTR_STATUS); + return IRQ_NONE; + } + + spin_lock(&master->xferqueue.lock); + dw_i3c_master_end_xfer_locked(master, status); + if (status & INTR_TRANSFER_ERR_STAT) + writel(INTR_TRANSFER_ERR_STAT, master->regs + INTR_STATUS); + spin_unlock(&master->xferqueue.lock); + + if (status & INTR_IBI_THLD_STAT) + dw_i3c_master_irq_handle_ibis(master); + + return IRQ_HANDLED; +} + +static int dw_i3c_master_set_speed(struct i3c_master_controller *m, + enum i3c_open_drain_speed speed) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + unsigned long core_rate; + u32 scl_timing, od_hcnt; + u8 lcnt; + + PM_RUNTIME_ACQUIRE_AUTOSUSPEND(master->dev, pm); + if (PM_RUNTIME_ACQUIRE_ERR(&pm)) + return -ENXIO; + + switch (speed) { + case I3C_OPEN_DRAIN_SLOW_SPEED: + core_rate = dw_i3c_master_get_core_rate(master); + if (!core_rate) + return -EINVAL; + + lcnt = SCL_I3C_TIMING_LCNT(master->i3c_od_timing_normal); + od_hcnt = DIV_ROUND_UP_ULL((u64)I3C_BUS_THIGH_INIT_OD_MIN_NS * + core_rate, NSEC_PER_SEC) - 1; + if (od_hcnt < SCL_I3C_TIMING_CNT_MIN) + od_hcnt = SCL_I3C_TIMING_CNT_MIN; + else if (od_hcnt > U8_MAX) + od_hcnt = U8_MAX; + scl_timing = SCL_I3C_TIMING_HCNT(od_hcnt) | + SCL_I3C_TIMING_LCNT(lcnt); + writel(scl_timing, master->regs + SCL_I3C_OD_TIMING); + master->i3c_od_timing = scl_timing; + break; + + case I3C_OPEN_DRAIN_NORMAL_SPEED: + writel(master->i3c_od_timing_normal, + master->regs + SCL_I3C_OD_TIMING); + master->i3c_od_timing = master->i3c_od_timing_normal; + break; + + default: + return -EINVAL; + } + + return 0; +} + +static int dw_i3c_master_set_dev_nack_retry(struct i3c_master_controller *m, + unsigned int dev_nack_retry_cnt) +{ + struct dw_i3c_master *master = to_dw_i3c_master(m); + u32 reg; + int i; + + if (dev_nack_retry_cnt > DW_I3C_DEV_NACK_RETRY_CNT_MAX) { + dev_err(&master->base.dev, + "Value %u exceeds maximum %d\n", + dev_nack_retry_cnt, DW_I3C_DEV_NACK_RETRY_CNT_MAX); + return -ERANGE; + } + + /* + * Update DAT entries for all currently attached devices. + * We directly iterate through the master's device array. + */ + for (i = 0; i < master->maxdevs; i++) { + /* Skip free/empty slots */ + if (master->free_pos & BIT(i)) + continue; + + reg = readl(master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, i)); + reg &= ~DEV_ADDR_TABLE_DEV_NACK_RETRY_MASK; + reg |= DEV_ADDR_TABLE_DEV_NACK_RETRY_CNT(dev_nack_retry_cnt); + writel(reg, master->regs + + DEV_ADDR_TABLE_LOC(master->datstartaddr, i)); + } + + return 0; +} + +static const struct i3c_master_controller_ops dw_mipi_i3c_ops = { + .bus_init = dw_i3c_master_bus_init, + .bus_cleanup = dw_i3c_master_bus_cleanup, + .attach_i3c_dev = dw_i3c_master_attach_i3c_dev, + .reattach_i3c_dev = dw_i3c_master_reattach_i3c_dev, + .detach_i3c_dev = dw_i3c_master_detach_i3c_dev, + .do_daa = dw_i3c_master_daa, + .supports_ccc_cmd = dw_i3c_master_supports_ccc_cmd, + .send_ccc_cmd = dw_i3c_master_send_ccc_cmd, + .i3c_xfers = dw_i3c_master_i3c_xfers, + .attach_i2c_dev = dw_i3c_master_attach_i2c_dev, + .detach_i2c_dev = dw_i3c_master_detach_i2c_dev, + .i2c_xfers = dw_i3c_master_i2c_xfers, + .request_ibi = dw_i3c_master_request_ibi, + .free_ibi = dw_i3c_master_free_ibi, + .enable_ibi = dw_i3c_master_enable_ibi, + .disable_ibi = dw_i3c_master_disable_ibi, + .recycle_ibi_slot = dw_i3c_master_recycle_ibi_slot, + .enable_hotjoin = dw_i3c_master_enable_hotjoin, + .disable_hotjoin = dw_i3c_master_disable_hotjoin, + .set_speed = dw_i3c_master_set_speed, + .set_dev_nack_retry = dw_i3c_master_set_dev_nack_retry, +}; + +/* default platform ops implementations */ +static int dw_i3c_platform_init_nop(struct dw_i3c_master *i3c) +{ + return 0; +} + +static void dw_i3c_platform_set_dat_ibi_nop(struct dw_i3c_master *i3c, + struct i3c_dev_desc *dev, + bool enable, u32 *dat) +{ +} + +static const struct dw_i3c_platform_ops dw_i3c_platform_ops_default = { + .init = dw_i3c_platform_init_nop, + .set_dat_ibi = dw_i3c_platform_set_dat_ibi_nop, +}; + +int dw_i3c_common_probe(struct dw_i3c_master *master, + struct platform_device *pdev) +{ + int ret, irq; + u32 thld_ctrl; + const struct dw_i3c_drvdata *drvdata; + unsigned long quirks = 0; + + if (!master->platform_ops) + master->platform_ops = &dw_i3c_platform_ops_default; + + master->dev = &pdev->dev; + + if (has_acpi_companion(&pdev->dev)) { + quirks = (unsigned long)device_get_match_data(&pdev->dev); + } else if (pdev->dev.of_node) { + drvdata = device_get_match_data(&pdev->dev); + if (drvdata) + quirks = drvdata->flags; + } + master->quirks = quirks; + + master->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(master->regs)) + return PTR_ERR(master->regs); + + master->core_clk = devm_clk_get_optional_enabled(&pdev->dev, NULL); + if (IS_ERR(master->core_clk)) + return PTR_ERR(master->core_clk); + + if (!master->core_clk && !(master->quirks & DW_I3C_ACPI_SKIP_CLK_RST)) { + dev_err(&pdev->dev, "missing core clock\n"); + return -EINVAL; + } + + master->pclk = devm_clk_get_optional_enabled(&pdev->dev, "apb"); + if (IS_ERR(master->pclk)) + return PTR_ERR(master->pclk); + + master->core_rst = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev, + NULL); + if (IS_ERR(master->core_rst)) + return PTR_ERR(master->core_rst); + + spin_lock_init(&master->xferqueue.lock); + INIT_LIST_HEAD(&master->xferqueue.list); + + spin_lock_init(&master->devs_lock); + + writel(INTR_ALL, master->regs + INTR_STATUS); + irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(&pdev->dev, irq, + dw_i3c_master_irq_handler, 0, + dev_name(&pdev->dev), master); + if (ret) + return ret; + + platform_set_drvdata(pdev, master); + + pm_runtime_set_autosuspend_delay(&pdev->dev, RPM_AUTOSUSPEND_TIMEOUT); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + /* Information regarding the FIFOs/QUEUEs depth */ + ret = readl(master->regs + QUEUE_STATUS_LEVEL); + master->caps.cmdfifodepth = QUEUE_STATUS_LEVEL_CMD(ret); + + ret = readl(master->regs + DATA_BUFFER_STATUS_LEVEL); + master->caps.datafifodepth = DATA_BUFFER_STATUS_LEVEL_TX(ret); + + ret = readl(master->regs + DEVICE_ADDR_TABLE_POINTER); + master->datstartaddr = ret; + master->maxdevs = ret >> 16; + master->free_pos = GENMASK(master->maxdevs - 1, 0); + + /* + * Detect IBI data capability (IC_HAS_IBI_DATA): write a non-zero value + * to IBI_DATA_THLD and read back. On controllers like Versalnet + * the field is hardwired to 0 and the write is ignored. Restore the + * original register value after detection. + */ + thld_ctrl = readl(master->regs + QUEUE_THLD_CTRL); + ret = thld_ctrl | QUEUE_THLD_CTRL_IBI_DATA(2); + writel(ret, master->regs + QUEUE_THLD_CTRL); + ret = readl(master->regs + QUEUE_THLD_CTRL); + if (ret & QUEUE_THLD_CTRL_IBI_DATA_MASK) + master->has_ibi_data = true; + writel(thld_ctrl, master->regs + QUEUE_THLD_CTRL); + + /* Keep controller enabled by preventing runtime suspend */ + if (master->quirks & DW_I3C_DISABLE_RUNTIME_PM_QUIRK) + pm_runtime_get_noresume(&pdev->dev); + + device_set_of_node_from_dev(&master->base.i2c.dev, &pdev->dev); + ret = i3c_master_register(&master->base, &pdev->dev, + &dw_mipi_i3c_ops, false); + if (ret) + goto err_disable_pm; + + return 0; + +err_disable_pm: + if (master->quirks & DW_I3C_DISABLE_RUNTIME_PM_QUIRK) + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + pm_runtime_dont_use_autosuspend(&pdev->dev); + + return ret; +} +EXPORT_SYMBOL_GPL(dw_i3c_common_probe); + +void dw_i3c_common_remove(struct dw_i3c_master *master) +{ + i3c_master_unregister(&master->base); + + /* Balance pm_runtime_get_noresume() from probe() */ + if (master->quirks & DW_I3C_DISABLE_RUNTIME_PM_QUIRK) + pm_runtime_put_noidle(master->dev); + + pm_runtime_disable(master->dev); + pm_runtime_set_suspended(master->dev); + pm_runtime_dont_use_autosuspend(master->dev); +} +EXPORT_SYMBOL_GPL(dw_i3c_common_remove); + +/* base platform implementation */ + +static int dw_i3c_probe(struct platform_device *pdev) +{ + struct dw_i3c_master *master; + + master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + return dw_i3c_common_probe(master, pdev); +} + +static void dw_i3c_remove(struct platform_device *pdev) +{ + struct dw_i3c_master *master = platform_get_drvdata(pdev); + + dw_i3c_common_remove(master); +} + +static void dw_i3c_master_restore_addrs(struct dw_i3c_master *master) +{ + u32 pos, reg_val; + + writel(DEV_ADDR_DYNAMIC_ADDR_VALID | DEV_ADDR_DYNAMIC(master->dev_addr), + master->regs + DEVICE_ADDR); + + for (pos = 0; pos < master->maxdevs; pos++) { + if (master->free_pos & BIT(pos)) + continue; + + reg_val = readl(master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, pos)); + + if (master->devs[pos].is_i2c_addr) { + reg_val &= ~DEV_ADDR_TABLE_STATIC_MASK; + reg_val |= DEV_ADDR_TABLE_LEGACY_I2C_DEV | + DEV_ADDR_TABLE_STATIC_ADDR(master->devs[pos].addr); + } else { + reg_val &= ~DEV_ADDR_TABLE_DYNAMIC_MASK; + reg_val |= DEV_ADDR_TABLE_DYNAMIC_ADDR(master->devs[pos].addr); + } + + writel(reg_val, master->regs + DEV_ADDR_TABLE_LOC(master->datstartaddr, pos)); + } +} + +static void dw_i3c_master_restore_timing_regs(struct dw_i3c_master *master) +{ + /* Preserve cached OD timing; it may be the SLOW setting from set_speed(). */ + writel(master->i3c_pp_timing, master->regs + SCL_I3C_PP_TIMING); + writel(master->bus_free_timing, master->regs + BUS_FREE_TIMING); + writel(master->i3c_od_timing, master->regs + SCL_I3C_OD_TIMING); + writel(master->ext_lcnt_timing, master->regs + SCL_EXT_LCNT_TIMING); + + if (master->i2c_slv_prsnt) { + writel(master->i2c_fmp_timing, master->regs + SCL_I2C_FMP_TIMING); + writel(master->i2c_fm_timing, master->regs + SCL_I2C_FM_TIMING); + } +} + +static int dw_i3c_master_enable_clks(struct dw_i3c_master *master) +{ + int ret = 0; + + ret = clk_prepare_enable(master->core_clk); + if (ret) + return ret; + + ret = clk_prepare_enable(master->pclk); + if (ret) { + clk_disable_unprepare(master->core_clk); + return ret; + } + + return 0; +} + +static inline void dw_i3c_master_disable_clks(struct dw_i3c_master *master) +{ + clk_disable_unprepare(master->pclk); + clk_disable_unprepare(master->core_clk); +} + +static int __maybe_unused dw_i3c_master_runtime_suspend(struct device *dev) +{ + struct dw_i3c_master *master = dev_get_drvdata(dev); + + dw_i3c_master_disable(master); + + reset_control_assert(master->core_rst); + dw_i3c_master_disable_clks(master); + pinctrl_pm_select_sleep_state(dev); + return 0; +} + +static int __maybe_unused dw_i3c_master_runtime_resume(struct device *dev) +{ + struct dw_i3c_master *master = dev_get_drvdata(dev); + + pinctrl_pm_select_default_state(dev); + dw_i3c_master_enable_clks(master); + reset_control_deassert(master->core_rst); + + dw_i3c_master_set_intr_regs(master); + dw_i3c_master_restore_timing_regs(master); + dw_i3c_master_restore_addrs(master); + + dw_i3c_master_enable(master); + return 0; +} + +static const struct dev_pm_ops dw_i3c_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(dw_i3c_master_runtime_suspend, dw_i3c_master_runtime_resume, NULL) +}; + +static void dw_i3c_shutdown(struct platform_device *pdev) +{ + struct dw_i3c_master *master = platform_get_drvdata(pdev); + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return; + } + + /* Disable interrupts */ + writel((u32)~INTR_ALL, master->regs + INTR_STATUS_EN); + writel((u32)~INTR_ALL, master->regs + INTR_SIGNAL_EN); + + pm_runtime_put_autosuspend(master->dev); +} + +static const struct dw_i3c_drvdata altr_agilex5_drvdata = { + .flags = DW_I3C_DISABLE_RUNTIME_PM_QUIRK, +}; + +static const struct of_device_id dw_i3c_master_of_match[] = { + { .compatible = "snps,dw-i3c-master-1.00a", }, + { .compatible = "altr,agilex5-dw-i3c-master", + .data = &altr_agilex5_drvdata, + }, + {}, +}; +MODULE_DEVICE_TABLE(of, dw_i3c_master_of_match); + +static const struct acpi_device_id dw_i3c_master_acpi_match[] = { + { "AMDI0015", AMD_I3C_OD_PP_TIMING }, + { "NVDA2018", DW_I3C_ACPI_SKIP_CLK_RST }, + { } +}; +MODULE_DEVICE_TABLE(acpi, dw_i3c_master_acpi_match); + +static struct platform_driver dw_i3c_driver = { + .probe = dw_i3c_probe, + .remove = dw_i3c_remove, + .shutdown = dw_i3c_shutdown, + .driver = { + .name = "dw-i3c-master", + .of_match_table = dw_i3c_master_of_match, + .acpi_match_table = dw_i3c_master_acpi_match, + .pm = &dw_i3c_pm_ops, + }, +}; +module_platform_driver(dw_i3c_driver); + +MODULE_AUTHOR("Vitor Soares <vitor.soares@synopsys.com>"); +MODULE_DESCRIPTION("DesignWare MIPI I3C driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/i3c/master/dw-i3c-master.h b/drivers/i3c/master/dw-i3c-master.h new file mode 100644 index 000000000..17ad817d1 --- /dev/null +++ b/drivers/i3c/master/dw-i3c-master.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Copyright (c) 2023 Code Construct + * + * Author: Jeremy Kerr <jk@codeconstruct.com.au> + */ + +#include <linux/clk.h> +#include <linux/i3c/master.h> +#include <linux/reset.h> +#include <linux/types.h> + +#define DW_I3C_MAX_DEVS 32 + +struct dw_i3c_master_caps { + u8 cmdfifodepth; + u8 datafifodepth; +}; + +struct dw_i3c_dat_entry { + u8 addr; + bool is_i2c_addr; + struct i3c_dev_desc *ibi_dev; +}; + +struct dw_i3c_master { + struct i3c_master_controller base; + struct device *dev; + u16 maxdevs; + u16 datstartaddr; + u32 free_pos; + struct { + struct list_head list; + struct dw_i3c_xfer *cur; + spinlock_t lock; + } xferqueue; + struct dw_i3c_master_caps caps; + void __iomem *regs; + struct reset_control *core_rst; + struct clk *core_clk; + struct clk *pclk; + char version[5]; + char type[5]; + u32 sir_rej_mask; + bool i2c_slv_prsnt; + u32 dev_addr; + u32 i3c_pp_timing; + u32 i3c_od_timing; + u32 i3c_od_timing_normal; + u32 ext_lcnt_timing; + u32 bus_free_timing; + u32 i2c_fm_timing; + u32 i2c_fmp_timing; + u32 quirks; + bool has_ibi_data; + /* + * Per-device hardware data, used to manage the device address table + * (DAT) + * + * Locking: the devs array may be referenced in IRQ context while + * processing an IBI. However, IBIs (for a specific device, which + * implies a specific DAT entry) can only happen while interrupts are + * requested for that device, which is serialised against other + * insertions/removals from the array by the global i3c infrastructure. + * So, devs_lock protects against concurrent updates to devs->ibi_dev + * between request_ibi/free_ibi and the IBI irq event. + */ + struct dw_i3c_dat_entry devs[DW_I3C_MAX_DEVS]; + spinlock_t devs_lock; + + /* platform-specific data */ + const struct dw_i3c_platform_ops *platform_ops; +}; + +struct dw_i3c_platform_ops { + /* + * Called on early bus init: the i3c has been set up, but before any + * transactions have taken place. Platform implementations may use to + * perform actual device enabling with the i3c core ready. + */ + int (*init)(struct dw_i3c_master *i3c); + + /* + * Initialise a DAT entry to enable/disable IBIs. Allows the platform + * to perform any device workarounds on the DAT entry before + * inserting into the hardware table. + * + * Called with the DAT lock held; must not sleep. + */ + void (*set_dat_ibi)(struct dw_i3c_master *i3c, + struct i3c_dev_desc *dev, bool enable, u32 *reg); +}; + +extern int dw_i3c_common_probe(struct dw_i3c_master *master, + struct platform_device *pdev); +extern void dw_i3c_common_remove(struct dw_i3c_master *master); + diff --git a/drivers/i3c/master/i3c-master-cdns.c b/drivers/i3c/master/i3c-master-cdns.c new file mode 100644 index 000000000..2d98c1ce9 --- /dev/null +++ b/drivers/i3c/master/i3c-master-cdns.c @@ -0,0 +1,1638 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (C) 2018 Cadence Design Systems Inc. + * + * Author: Boris Brezillon <boris.brezillon@bootlin.com> + */ + +#include <linux/bitops.h> +#include <linux/clk.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/ioport.h> +#include <linux/kernel.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <linux/workqueue.h> + +#include "../internals.h" + +#define DEV_ID 0x0 +#define DEV_ID_I3C_MASTER 0x5034 + +#define CONF_STATUS0 0x4 +#define CONF_STATUS0_CMDR_DEPTH(x) (4 << (((x) & GENMASK(31, 29)) >> 29)) +#define CONF_STATUS0_ECC_CHK BIT(28) +#define CONF_STATUS0_INTEG_CHK BIT(27) +#define CONF_STATUS0_CSR_DAP_CHK BIT(26) +#define CONF_STATUS0_TRANS_TOUT_CHK BIT(25) +#define CONF_STATUS0_PROT_FAULTS_CHK BIT(24) +#define CONF_STATUS0_GPO_NUM(x) (((x) & GENMASK(23, 16)) >> 16) +#define CONF_STATUS0_GPI_NUM(x) (((x) & GENMASK(15, 8)) >> 8) +#define CONF_STATUS0_IBIR_DEPTH(x) (4 << (((x) & GENMASK(7, 6)) >> 7)) +#define CONF_STATUS0_SUPPORTS_DDR BIT(5) +#define CONF_STATUS0_SEC_MASTER BIT(4) +#define CONF_STATUS0_DEVS_NUM(x) ((x) & GENMASK(3, 0)) + +#define CONF_STATUS1 0x8 +#define CONF_STATUS1_IBI_HW_RES(x) ((((x) & GENMASK(31, 28)) >> 28) + 1) +#define CONF_STATUS1_CMD_DEPTH(x) (4 << (((x) & GENMASK(27, 26)) >> 26)) +#define CONF_STATUS1_SLVDDR_RX_DEPTH(x) (8 << (((x) & GENMASK(25, 21)) >> 21)) +#define CONF_STATUS1_SLVDDR_TX_DEPTH(x) (8 << (((x) & GENMASK(20, 16)) >> 16)) +#define CONF_STATUS1_IBI_DEPTH(x) (2 << (((x) & GENMASK(12, 10)) >> 10)) +#define CONF_STATUS1_RX_DEPTH(x) (8 << (((x) & GENMASK(9, 5)) >> 5)) +#define CONF_STATUS1_TX_DEPTH(x) (8 << ((x) & GENMASK(4, 0))) + +#define REV_ID 0xc +#define REV_ID_VID(id) (((id) & GENMASK(31, 20)) >> 20) +#define REV_ID_PID(id) (((id) & GENMASK(19, 8)) >> 8) +#define REV_ID_REV_MAJOR(id) (((id) & GENMASK(7, 4)) >> 4) +#define REV_ID_REV_MINOR(id) ((id) & GENMASK(3, 0)) + +#define CTRL 0x10 +#define CTRL_DEV_EN BIT(31) +#define CTRL_HALT_EN BIT(30) +#define CTRL_MCS BIT(29) +#define CTRL_MCS_EN BIT(28) +#define CTRL_THD_DELAY(x) (((x) << 24) & GENMASK(25, 24)) +#define CTRL_HJ_DISEC BIT(8) +#define CTRL_MST_ACK BIT(7) +#define CTRL_HJ_ACK BIT(6) +#define CTRL_HJ_INIT BIT(5) +#define CTRL_MST_INIT BIT(4) +#define CTRL_AHDR_OPT BIT(3) +#define CTRL_PURE_BUS_MODE 0 +#define CTRL_MIXED_FAST_BUS_MODE 2 +#define CTRL_MIXED_SLOW_BUS_MODE 3 +#define CTRL_BUS_MODE_MASK GENMASK(1, 0) +#define THD_DELAY_MAX 3 + +#define PRESCL_CTRL0 0x14 +#define PRESCL_CTRL0_I2C(x) ((x) << 16) +#define PRESCL_CTRL0_I3C(x) (x) +#define PRESCL_CTRL0_I3C_MAX GENMASK(9, 0) +#define PRESCL_CTRL0_I2C_MAX GENMASK(15, 0) + +#define PRESCL_CTRL1 0x18 +#define PRESCL_CTRL1_PP_LOW_MASK GENMASK(15, 8) +#define PRESCL_CTRL1_PP_LOW(x) ((x) << 8) +#define PRESCL_CTRL1_OD_LOW_MASK GENMASK(7, 0) +#define PRESCL_CTRL1_OD_LOW(x) (x) + +#define MST_IER 0x20 +#define MST_IDR 0x24 +#define MST_IMR 0x28 +#define MST_ICR 0x2c +#define MST_ISR 0x30 +#define MST_INT_HALTED BIT(18) +#define MST_INT_MR_DONE BIT(17) +#define MST_INT_IMM_COMP BIT(16) +#define MST_INT_TX_THR BIT(15) +#define MST_INT_TX_OVF BIT(14) +#define MST_INT_IBID_THR BIT(12) +#define MST_INT_IBID_UNF BIT(11) +#define MST_INT_IBIR_THR BIT(10) +#define MST_INT_IBIR_UNF BIT(9) +#define MST_INT_IBIR_OVF BIT(8) +#define MST_INT_RX_THR BIT(7) +#define MST_INT_RX_UNF BIT(6) +#define MST_INT_CMDD_EMP BIT(5) +#define MST_INT_CMDD_THR BIT(4) +#define MST_INT_CMDD_OVF BIT(3) +#define MST_INT_CMDR_THR BIT(2) +#define MST_INT_CMDR_UNF BIT(1) +#define MST_INT_CMDR_OVF BIT(0) + +#define MST_STATUS0 0x34 +#define MST_STATUS0_IDLE BIT(18) +#define MST_STATUS0_HALTED BIT(17) +#define MST_STATUS0_MASTER_MODE BIT(16) +#define MST_STATUS0_TX_FULL BIT(13) +#define MST_STATUS0_IBID_FULL BIT(12) +#define MST_STATUS0_IBIR_FULL BIT(11) +#define MST_STATUS0_RX_FULL BIT(10) +#define MST_STATUS0_CMDD_FULL BIT(9) +#define MST_STATUS0_CMDR_FULL BIT(8) +#define MST_STATUS0_TX_EMP BIT(5) +#define MST_STATUS0_IBID_EMP BIT(4) +#define MST_STATUS0_IBIR_EMP BIT(3) +#define MST_STATUS0_RX_EMP BIT(2) +#define MST_STATUS0_CMDD_EMP BIT(1) +#define MST_STATUS0_CMDR_EMP BIT(0) + +#define CMDR 0x38 +#define CMDR_NO_ERROR 0 +#define CMDR_DDR_PREAMBLE_ERROR 1 +#define CMDR_DDR_PARITY_ERROR 2 +#define CMDR_DDR_RX_FIFO_OVF 3 +#define CMDR_DDR_TX_FIFO_UNF 4 +#define CMDR_M0_ERROR 5 +#define CMDR_M1_ERROR 6 +#define CMDR_M2_ERROR 7 +#define CMDR_MST_ABORT 8 +#define CMDR_NACK_RESP 9 +#define CMDR_INVALID_DA 10 +#define CMDR_DDR_DROPPED 11 +#define CMDR_ERROR(x) (((x) & GENMASK(27, 24)) >> 24) +#define CMDR_XFER_BYTES(x) (((x) & GENMASK(19, 8)) >> 8) +#define CMDR_CMDID_HJACK_DISEC 0xfe +#define CMDR_CMDID_HJACK_ENTDAA 0xff +#define CMDR_CMDID(x) ((x) & GENMASK(7, 0)) + +#define IBIR 0x3c +#define IBIR_ACKED BIT(12) +#define IBIR_SLVID(x) (((x) & GENMASK(11, 8)) >> 8) +#define IBIR_ERROR BIT(7) +#define IBIR_XFER_BYTES(x) (((x) & GENMASK(6, 2)) >> 2) +#define IBIR_TYPE_IBI 0 +#define IBIR_TYPE_HJ 1 +#define IBIR_TYPE_MR 2 +#define IBIR_TYPE(x) ((x) & GENMASK(1, 0)) + +#define SLV_IER 0x40 +#define SLV_IDR 0x44 +#define SLV_IMR 0x48 +#define SLV_ICR 0x4c +#define SLV_ISR 0x50 +#define SLV_INT_TM BIT(20) +#define SLV_INT_ERROR BIT(19) +#define SLV_INT_EVENT_UP BIT(18) +#define SLV_INT_HJ_DONE BIT(17) +#define SLV_INT_MR_DONE BIT(16) +#define SLV_INT_DA_UPD BIT(15) +#define SLV_INT_SDR_FAIL BIT(14) +#define SLV_INT_DDR_FAIL BIT(13) +#define SLV_INT_M_RD_ABORT BIT(12) +#define SLV_INT_DDR_RX_THR BIT(11) +#define SLV_INT_DDR_TX_THR BIT(10) +#define SLV_INT_SDR_RX_THR BIT(9) +#define SLV_INT_SDR_TX_THR BIT(8) +#define SLV_INT_DDR_RX_UNF BIT(7) +#define SLV_INT_DDR_TX_OVF BIT(6) +#define SLV_INT_SDR_RX_UNF BIT(5) +#define SLV_INT_SDR_TX_OVF BIT(4) +#define SLV_INT_DDR_RD_COMP BIT(3) +#define SLV_INT_DDR_WR_COMP BIT(2) +#define SLV_INT_SDR_RD_COMP BIT(1) +#define SLV_INT_SDR_WR_COMP BIT(0) + +#define SLV_STATUS0 0x54 +#define SLV_STATUS0_REG_ADDR(s) (((s) & GENMASK(23, 16)) >> 16) +#define SLV_STATUS0_XFRD_BYTES(s) ((s) & GENMASK(15, 0)) + +#define SLV_STATUS1 0x58 +#define SLV_STATUS1_AS(s) (((s) & GENMASK(21, 20)) >> 20) +#define SLV_STATUS1_VEN_TM BIT(19) +#define SLV_STATUS1_HJ_DIS BIT(18) +#define SLV_STATUS1_MR_DIS BIT(17) +#define SLV_STATUS1_PROT_ERR BIT(16) +#define SLV_STATUS1_DA(s) (((s) & GENMASK(15, 9)) >> 9) +#define SLV_STATUS1_HAS_DA BIT(8) +#define SLV_STATUS1_DDR_RX_FULL BIT(7) +#define SLV_STATUS1_DDR_TX_FULL BIT(6) +#define SLV_STATUS1_DDR_RX_EMPTY BIT(5) +#define SLV_STATUS1_DDR_TX_EMPTY BIT(4) +#define SLV_STATUS1_SDR_RX_FULL BIT(3) +#define SLV_STATUS1_SDR_TX_FULL BIT(2) +#define SLV_STATUS1_SDR_RX_EMPTY BIT(1) +#define SLV_STATUS1_SDR_TX_EMPTY BIT(0) + +#define CMD0_FIFO 0x60 +#define CMD0_FIFO_IS_DDR BIT(31) +#define CMD0_FIFO_IS_CCC BIT(30) +#define CMD0_FIFO_BCH BIT(29) +#define XMIT_BURST_STATIC_SUBADDR 0 +#define XMIT_SINGLE_INC_SUBADDR 1 +#define XMIT_SINGLE_STATIC_SUBADDR 2 +#define XMIT_BURST_WITHOUT_SUBADDR 3 +#define CMD0_FIFO_PRIV_XMIT_MODE(m) ((m) << 27) +#define CMD0_FIFO_SBCA BIT(26) +#define CMD0_FIFO_RSBC BIT(25) +#define CMD0_FIFO_IS_10B BIT(24) +#define CMD0_FIFO_PL_LEN(l) ((l) << 12) +#define CMD0_FIFO_PL_LEN_MAX 4095 +#define CMD0_FIFO_DEV_ADDR(a) ((a) << 1) +#define CMD0_FIFO_RNW BIT(0) + +#define CMD1_FIFO 0x64 +#define CMD1_FIFO_CMDID(id) ((id) << 24) +#define CMD1_FIFO_CSRADDR(a) (a) +#define CMD1_FIFO_CCC(id) (id) + +#define TX_FIFO 0x68 + +#define IMD_CMD0 0x70 +#define IMD_CMD0_PL_LEN(l) ((l) << 12) +#define IMD_CMD0_DEV_ADDR(a) ((a) << 1) +#define IMD_CMD0_RNW BIT(0) + +#define IMD_CMD1 0x74 +#define IMD_CMD1_CCC(id) (id) + +#define IMD_DATA 0x78 +#define RX_FIFO 0x80 +#define IBI_DATA_FIFO 0x84 +#define SLV_DDR_TX_FIFO 0x88 +#define SLV_DDR_RX_FIFO 0x8c + +#define CMD_IBI_THR_CTRL 0x90 +#define IBIR_THR(t) ((t) << 24) +#define CMDR_THR(t) ((t) << 16) +#define IBI_THR(t) ((t) << 8) +#define CMD_THR(t) (t) + +#define TX_RX_THR_CTRL 0x94 +#define RX_THR(t) ((t) << 16) +#define TX_THR(t) (t) + +#define SLV_DDR_TX_RX_THR_CTRL 0x98 +#define SLV_DDR_RX_THR(t) ((t) << 16) +#define SLV_DDR_TX_THR(t) (t) + +#define FLUSH_CTRL 0x9c +#define FLUSH_IBI_RESP BIT(23) +#define FLUSH_CMD_RESP BIT(22) +#define FLUSH_SLV_DDR_RX_FIFO BIT(22) +#define FLUSH_SLV_DDR_TX_FIFO BIT(21) +#define FLUSH_IMM_FIFO BIT(20) +#define FLUSH_IBI_FIFO BIT(19) +#define FLUSH_RX_FIFO BIT(18) +#define FLUSH_TX_FIFO BIT(17) +#define FLUSH_CMD_FIFO BIT(16) + +#define TTO_PRESCL_CTRL0 0xb0 +#define TTO_PRESCL_CTRL0_DIVB(x) ((x) << 16) +#define TTO_PRESCL_CTRL0_DIVA(x) (x) + +#define TTO_PRESCL_CTRL1 0xb4 +#define TTO_PRESCL_CTRL1_DIVB(x) ((x) << 16) +#define TTO_PRESCL_CTRL1_DIVA(x) (x) + +#define DEVS_CTRL 0xb8 +#define DEVS_CTRL_DEV_CLR_SHIFT 16 +#define DEVS_CTRL_DEV_CLR_ALL GENMASK(31, 16) +#define DEVS_CTRL_DEV_CLR(dev) BIT(16 + (dev)) +#define DEVS_CTRL_DEV_ACTIVE(dev) BIT(dev) +#define DEVS_CTRL_DEVS_ACTIVE_MASK GENMASK(15, 0) +#define MAX_DEVS 16 + +#define DEV_ID_RR0(d) (0xc0 + ((d) * 0x10)) +#define DEV_ID_RR0_LVR_EXT_ADDR BIT(11) +#define DEV_ID_RR0_HDR_CAP BIT(10) +#define DEV_ID_RR0_IS_I3C BIT(9) +#define DEV_ID_RR0_DEV_ADDR_MASK (GENMASK(6, 0) | GENMASK(15, 13)) +#define DEV_ID_RR0_SET_DEV_ADDR(a) (((a) & GENMASK(6, 0)) | \ + (((a) & GENMASK(9, 7)) << 6)) +#define DEV_ID_RR0_GET_DEV_ADDR(x) ((((x) >> 1) & GENMASK(6, 0)) | \ + (((x) >> 6) & GENMASK(9, 7))) + +#define DEV_ID_RR1(d) (0xc4 + ((d) * 0x10)) +#define DEV_ID_RR1_PID_MSB(pid) (pid) + +#define DEV_ID_RR2(d) (0xc8 + ((d) * 0x10)) +#define DEV_ID_RR2_PID_LSB(pid) ((pid) << 16) +#define DEV_ID_RR2_BCR(bcr) ((bcr) << 8) +#define DEV_ID_RR2_DCR(dcr) (dcr) +#define DEV_ID_RR2_LVR(lvr) (lvr) + +#define SIR_MAP(x) (0x180 + ((x) * 4)) +#define SIR_MAP_DEV_REG(d) SIR_MAP((d) / 2) +#define SIR_MAP_DEV_SHIFT(d, fs) ((fs) + (((d) % 2) ? 16 : 0)) +#define SIR_MAP_DEV_CONF_MASK(d) (GENMASK(15, 0) << (((d) % 2) ? 16 : 0)) +#define SIR_MAP_DEV_CONF(d, c) ((c) << (((d) % 2) ? 16 : 0)) +#define DEV_ROLE_SLAVE 0 +#define DEV_ROLE_MASTER 1 +#define SIR_MAP_DEV_ROLE(role) ((role) << 14) +#define SIR_MAP_DEV_SLOW BIT(13) +#define SIR_MAP_DEV_PL(l) ((l) << 8) +#define SIR_MAP_PL_MAX GENMASK(4, 0) +#define SIR_MAP_DEV_DA(a) ((a) << 1) +#define SIR_MAP_DEV_ACK BIT(0) + +#define GPIR_WORD(x) (0x200 + ((x) * 4)) +#define GPI_REG(val, id) \ + (((val) >> (((id) % 4) * 8)) & GENMASK(7, 0)) + +#define GPOR_WORD(x) (0x220 + ((x) * 4)) +#define GPO_REG(val, id) \ + (((val) >> (((id) % 4) * 8)) & GENMASK(7, 0)) + +#define ASF_INT_STATUS 0x300 +#define ASF_INT_RAW_STATUS 0x304 +#define ASF_INT_MASK 0x308 +#define ASF_INT_TEST 0x30c +#define ASF_INT_FATAL_SELECT 0x310 +#define ASF_INTEGRITY_ERR BIT(6) +#define ASF_PROTOCOL_ERR BIT(5) +#define ASF_TRANS_TIMEOUT_ERR BIT(4) +#define ASF_CSR_ERR BIT(3) +#define ASF_DAP_ERR BIT(2) +#define ASF_SRAM_UNCORR_ERR BIT(1) +#define ASF_SRAM_CORR_ERR BIT(0) + +#define ASF_SRAM_CORR_FAULT_STATUS 0x320 +#define ASF_SRAM_UNCORR_FAULT_STATUS 0x324 +#define ASF_SRAM_CORR_FAULT_INSTANCE(x) ((x) >> 24) +#define ASF_SRAM_CORR_FAULT_ADDR(x) ((x) & GENMASK(23, 0)) + +#define ASF_SRAM_FAULT_STATS 0x328 +#define ASF_SRAM_FAULT_UNCORR_STATS(x) ((x) >> 16) +#define ASF_SRAM_FAULT_CORR_STATS(x) ((x) & GENMASK(15, 0)) + +#define ASF_TRANS_TOUT_CTRL 0x330 +#define ASF_TRANS_TOUT_EN BIT(31) +#define ASF_TRANS_TOUT_VAL(x) (x) + +#define ASF_TRANS_TOUT_FAULT_MASK 0x334 +#define ASF_TRANS_TOUT_FAULT_STATUS 0x338 +#define ASF_TRANS_TOUT_FAULT_APB BIT(3) +#define ASF_TRANS_TOUT_FAULT_SCL_LOW BIT(2) +#define ASF_TRANS_TOUT_FAULT_SCL_HIGH BIT(1) +#define ASF_TRANS_TOUT_FAULT_FSCL_HIGH BIT(0) + +#define ASF_PROTO_FAULT_MASK 0x340 +#define ASF_PROTO_FAULT_STATUS 0x344 +#define ASF_PROTO_FAULT_SLVSDR_RD_ABORT BIT(31) +#define ASF_PROTO_FAULT_SLVDDR_FAIL BIT(30) +#define ASF_PROTO_FAULT_S(x) BIT(16 + (x)) +#define ASF_PROTO_FAULT_MSTSDR_RD_ABORT BIT(15) +#define ASF_PROTO_FAULT_MSTDDR_FAIL BIT(14) +#define ASF_PROTO_FAULT_M(x) BIT(x) + +struct cdns_i3c_master_caps { + u32 cmdfifodepth; + u32 cmdrfifodepth; + u32 txfifodepth; + u32 rxfifodepth; + u32 ibirfifodepth; +}; + +struct cdns_i3c_cmd { + u32 cmd0; + u32 cmd1; + u32 tx_len; + const void *tx_buf; + u32 rx_len; + void *rx_buf; + u32 error; +}; + +struct cdns_i3c_xfer { + struct list_head node; + struct completion comp; + int ret; + unsigned int ncmds; + struct cdns_i3c_cmd cmds[] __counted_by(ncmds); +}; + +struct cdns_i3c_data { + u8 thd_delay_ns; +}; + +struct cdns_i3c_master { + struct i3c_master_controller base; + u32 free_rr_slots; + unsigned int maxdevs; + struct { + unsigned int num_slots; + struct i3c_dev_desc **slots; + spinlock_t lock; + } ibi; + struct { + struct list_head list; + struct cdns_i3c_xfer *cur; + spinlock_t lock; + } xferqueue; + void __iomem *regs; + struct clk *sysclk; + struct cdns_i3c_master_caps caps; + unsigned long i3c_scl_lim; + const struct cdns_i3c_data *devdata; +}; + +static inline struct cdns_i3c_master * +to_cdns_i3c_master(struct i3c_master_controller *master) +{ + return container_of(master, struct cdns_i3c_master, base); +} + +static void cdns_i3c_master_wr_to_tx_fifo(struct cdns_i3c_master *master, + const u8 *bytes, int nbytes) +{ + i3c_writel_fifo(master->regs + TX_FIFO, bytes, nbytes); +} + +static void cdns_i3c_master_rd_from_rx_fifo(struct cdns_i3c_master *master, + u8 *bytes, int nbytes) +{ + i3c_readl_fifo(master->regs + RX_FIFO, bytes, nbytes); +} + +static bool cdns_i3c_master_supports_ccc_cmd(struct i3c_master_controller *m, + const struct i3c_ccc_cmd *cmd) +{ + if (cmd->ndests > 1) + return false; + + switch (cmd->id) { + case I3C_CCC_ENEC(true): + case I3C_CCC_ENEC(false): + case I3C_CCC_DISEC(true): + case I3C_CCC_DISEC(false): + case I3C_CCC_ENTAS(0, true): + case I3C_CCC_ENTAS(0, false): + case I3C_CCC_RSTDAA(true): + case I3C_CCC_RSTDAA(false): + case I3C_CCC_ENTDAA: + case I3C_CCC_SETMWL(true): + case I3C_CCC_SETMWL(false): + case I3C_CCC_SETMRL(true): + case I3C_CCC_SETMRL(false): + case I3C_CCC_DEFSLVS: + case I3C_CCC_ENTHDR(0): + case I3C_CCC_SETDASA: + case I3C_CCC_SETNEWDA: + case I3C_CCC_GETMWL: + case I3C_CCC_GETMRL: + case I3C_CCC_GETPID: + case I3C_CCC_GETBCR: + case I3C_CCC_GETDCR: + case I3C_CCC_GETSTATUS: + case I3C_CCC_GETACCMST: + case I3C_CCC_GETMXDS: + case I3C_CCC_GETHDRCAP: + return true; + default: + break; + } + + return false; +} + +static int cdns_i3c_master_disable(struct cdns_i3c_master *master) +{ + u32 status; + + writel(readl(master->regs + CTRL) & ~CTRL_DEV_EN, master->regs + CTRL); + + return readl_poll_timeout(master->regs + MST_STATUS0, status, + status & MST_STATUS0_IDLE, 10, 1000000); +} + +static void cdns_i3c_master_enable(struct cdns_i3c_master *master) +{ + writel(readl(master->regs + CTRL) | CTRL_DEV_EN, master->regs + CTRL); +} + +static struct cdns_i3c_xfer * +cdns_i3c_master_alloc_xfer(struct cdns_i3c_master *master, unsigned int ncmds) +{ + struct cdns_i3c_xfer *xfer; + + xfer = kzalloc_flex(*xfer, cmds, ncmds); + if (!xfer) + return NULL; + + INIT_LIST_HEAD(&xfer->node); + xfer->ncmds = ncmds; + xfer->ret = -ETIMEDOUT; + + return xfer; +} + +static void cdns_i3c_master_free_xfer(struct cdns_i3c_xfer *xfer) +{ + kfree(xfer); +} + +static void cdns_i3c_master_start_xfer_locked(struct cdns_i3c_master *master) +{ + struct cdns_i3c_xfer *xfer = master->xferqueue.cur; + unsigned int i; + + if (!xfer) + return; + + writel(MST_INT_CMDD_EMP, master->regs + MST_ICR); + for (i = 0; i < xfer->ncmds; i++) { + struct cdns_i3c_cmd *cmd = &xfer->cmds[i]; + + cdns_i3c_master_wr_to_tx_fifo(master, cmd->tx_buf, + cmd->tx_len); + } + + for (i = 0; i < xfer->ncmds; i++) { + struct cdns_i3c_cmd *cmd = &xfer->cmds[i]; + + writel(cmd->cmd1 | CMD1_FIFO_CMDID(i), + master->regs + CMD1_FIFO); + writel(cmd->cmd0, master->regs + CMD0_FIFO); + } + + writel(readl(master->regs + CTRL) | CTRL_MCS, + master->regs + CTRL); + writel(MST_INT_CMDD_EMP, master->regs + MST_IER); +} + +static void cdns_i3c_master_end_xfer_locked(struct cdns_i3c_master *master, + u32 isr) +{ + struct cdns_i3c_xfer *xfer = master->xferqueue.cur; + int i, ret = 0; + u32 status0; + + if (!xfer) + return; + + if (!(isr & MST_INT_CMDD_EMP)) + return; + + writel(MST_INT_CMDD_EMP, master->regs + MST_IDR); + + for (status0 = readl(master->regs + MST_STATUS0); + !(status0 & MST_STATUS0_CMDR_EMP); + status0 = readl(master->regs + MST_STATUS0)) { + struct cdns_i3c_cmd *cmd; + u32 cmdr, rx_len, id; + + cmdr = readl(master->regs + CMDR); + id = CMDR_CMDID(cmdr); + if (id == CMDR_CMDID_HJACK_DISEC || + id == CMDR_CMDID_HJACK_ENTDAA || + WARN_ON(id >= xfer->ncmds)) + continue; + + cmd = &xfer->cmds[CMDR_CMDID(cmdr)]; + rx_len = min_t(u32, CMDR_XFER_BYTES(cmdr), cmd->rx_len); + cdns_i3c_master_rd_from_rx_fifo(master, cmd->rx_buf, rx_len); + cmd->rx_len = rx_len; + cmd->error = CMDR_ERROR(cmdr); + } + + for (i = 0; i < xfer->ncmds; i++) { + switch (xfer->cmds[i].error) { + case CMDR_NO_ERROR: + break; + + case CMDR_DDR_PREAMBLE_ERROR: + case CMDR_DDR_PARITY_ERROR: + case CMDR_M0_ERROR: + case CMDR_M1_ERROR: + case CMDR_M2_ERROR: + case CMDR_MST_ABORT: + case CMDR_NACK_RESP: + case CMDR_DDR_DROPPED: + ret = -EIO; + break; + + case CMDR_DDR_RX_FIFO_OVF: + case CMDR_DDR_TX_FIFO_UNF: + ret = -ENOSPC; + break; + + case CMDR_INVALID_DA: + default: + ret = -EINVAL; + break; + } + } + + xfer->ret = ret; + complete(&xfer->comp); + + xfer = list_first_entry_or_null(&master->xferqueue.list, + struct cdns_i3c_xfer, node); + if (xfer) + list_del_init(&xfer->node); + + master->xferqueue.cur = xfer; + cdns_i3c_master_start_xfer_locked(master); +} + +static void cdns_i3c_master_queue_xfer(struct cdns_i3c_master *master, + struct cdns_i3c_xfer *xfer) +{ + unsigned long flags; + + init_completion(&xfer->comp); + spin_lock_irqsave(&master->xferqueue.lock, flags); + if (master->xferqueue.cur) { + list_add_tail(&xfer->node, &master->xferqueue.list); + } else { + master->xferqueue.cur = xfer; + cdns_i3c_master_start_xfer_locked(master); + } + spin_unlock_irqrestore(&master->xferqueue.lock, flags); +} + +static void cdns_i3c_master_unqueue_xfer(struct cdns_i3c_master *master, + struct cdns_i3c_xfer *xfer) +{ + unsigned long flags; + + spin_lock_irqsave(&master->xferqueue.lock, flags); + if (master->xferqueue.cur == xfer) { + u32 status; + + writel(readl(master->regs + CTRL) & ~CTRL_DEV_EN, + master->regs + CTRL); + readl_poll_timeout_atomic(master->regs + MST_STATUS0, status, + status & MST_STATUS0_IDLE, 10, + 1000000); + master->xferqueue.cur = NULL; + writel(FLUSH_RX_FIFO | FLUSH_TX_FIFO | FLUSH_CMD_FIFO | + FLUSH_CMD_RESP, + master->regs + FLUSH_CTRL); + writel(MST_INT_CMDD_EMP, master->regs + MST_IDR); + writel(readl(master->regs + CTRL) | CTRL_DEV_EN, + master->regs + CTRL); + } else { + list_del_init(&xfer->node); + } + spin_unlock_irqrestore(&master->xferqueue.lock, flags); +} + +static enum i3c_error_code cdns_i3c_cmd_get_err(struct cdns_i3c_cmd *cmd) +{ + switch (cmd->error) { + case CMDR_M0_ERROR: + return I3C_ERROR_M0; + + case CMDR_M1_ERROR: + return I3C_ERROR_M1; + + case CMDR_M2_ERROR: + case CMDR_NACK_RESP: + return I3C_ERROR_M2; + + default: + break; + } + + return I3C_ERROR_UNKNOWN; +} + +static int cdns_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *cmd) +{ + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_xfer *xfer; + struct cdns_i3c_cmd *ccmd; + int ret; + + xfer = cdns_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + ccmd = xfer->cmds; + ccmd->cmd1 = CMD1_FIFO_CCC(cmd->id); + ccmd->cmd0 = CMD0_FIFO_IS_CCC | + CMD0_FIFO_PL_LEN(cmd->dests[0].payload.len); + + if (cmd->id & I3C_CCC_DIRECT) + ccmd->cmd0 |= CMD0_FIFO_DEV_ADDR(cmd->dests[0].addr); + + if (cmd->rnw) { + ccmd->cmd0 |= CMD0_FIFO_RNW; + ccmd->rx_buf = cmd->dests[0].payload.data; + ccmd->rx_len = cmd->dests[0].payload.len; + } else { + ccmd->tx_buf = cmd->dests[0].payload.data; + ccmd->tx_len = cmd->dests[0].payload.len; + } + + cdns_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000))) + cdns_i3c_master_unqueue_xfer(master, xfer); + + ret = xfer->ret; + cmd->err = cdns_i3c_cmd_get_err(&xfer->cmds[0]); + if (!ret && cmd->rnw) + cmd->dests[0].payload.actual_len = ccmd->rx_len; + cdns_i3c_master_free_xfer(xfer); + + return ret; +} + +static int cdns_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, + struct i3c_xfer *xfers, + int nxfers, enum i3c_xfer_mode mode) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + int txslots = 0, rxslots = 0, i, ret; + struct cdns_i3c_xfer *cdns_xfer; + + for (i = 0; i < nxfers; i++) { + if (xfers[i].len > CMD0_FIFO_PL_LEN_MAX) + return -EOPNOTSUPP; + } + + if (!nxfers) + return 0; + + if (nxfers > master->caps.cmdfifodepth || + nxfers > master->caps.cmdrfifodepth) + return -EOPNOTSUPP; + + /* + * First make sure that all transactions (block of transfers separated + * by a STOP marker) fit in the FIFOs. + */ + for (i = 0; i < nxfers; i++) { + if (xfers[i].rnw) + rxslots += DIV_ROUND_UP(xfers[i].len, 4); + else + txslots += DIV_ROUND_UP(xfers[i].len, 4); + } + + if (rxslots > master->caps.rxfifodepth || + txslots > master->caps.txfifodepth) + return -EOPNOTSUPP; + + cdns_xfer = cdns_i3c_master_alloc_xfer(master, nxfers); + if (!cdns_xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + struct cdns_i3c_cmd *ccmd = &cdns_xfer->cmds[i]; + u32 pl_len = xfers[i].len; + + ccmd->cmd0 = CMD0_FIFO_DEV_ADDR(dev->info.dyn_addr) | + CMD0_FIFO_PRIV_XMIT_MODE(XMIT_BURST_WITHOUT_SUBADDR); + + if (xfers[i].rnw) { + ccmd->cmd0 |= CMD0_FIFO_RNW; + ccmd->rx_buf = xfers[i].data.in; + ccmd->rx_len = xfers[i].len; + pl_len++; + } else { + ccmd->tx_buf = xfers[i].data.out; + ccmd->tx_len = xfers[i].len; + } + + ccmd->cmd0 |= CMD0_FIFO_PL_LEN(pl_len); + + if (i < nxfers - 1) + ccmd->cmd0 |= CMD0_FIFO_RSBC; + + if (!i) + ccmd->cmd0 |= CMD0_FIFO_BCH; + } + + cdns_i3c_master_queue_xfer(master, cdns_xfer); + if (!wait_for_completion_timeout(&cdns_xfer->comp, + msecs_to_jiffies(1000))) + cdns_i3c_master_unqueue_xfer(master, cdns_xfer); + + ret = cdns_xfer->ret; + + for (i = 0; i < nxfers; i++) + xfers[i].err = cdns_i3c_cmd_get_err(&cdns_xfer->cmds[i]); + + cdns_i3c_master_free_xfer(cdns_xfer); + + return ret; +} + +static int cdns_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *xfers, int nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + unsigned int nrxwords = 0, ntxwords = 0; + struct cdns_i3c_xfer *xfer; + int i, ret = 0; + + if (nxfers > master->caps.cmdfifodepth) + return -EOPNOTSUPP; + + for (i = 0; i < nxfers; i++) { + if (xfers[i].len > CMD0_FIFO_PL_LEN_MAX) + return -EOPNOTSUPP; + + if (xfers[i].flags & I2C_M_RD) + nrxwords += DIV_ROUND_UP(xfers[i].len, 4); + else + ntxwords += DIV_ROUND_UP(xfers[i].len, 4); + } + + if (ntxwords > master->caps.txfifodepth || + nrxwords > master->caps.rxfifodepth) + return -EOPNOTSUPP; + + xfer = cdns_i3c_master_alloc_xfer(master, nxfers); + if (!xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + struct cdns_i3c_cmd *ccmd = &xfer->cmds[i]; + + ccmd->cmd0 = CMD0_FIFO_DEV_ADDR(xfers[i].addr) | + CMD0_FIFO_PL_LEN(xfers[i].len) | + CMD0_FIFO_PRIV_XMIT_MODE(XMIT_BURST_WITHOUT_SUBADDR); + + if (xfers[i].flags & I2C_M_TEN) + ccmd->cmd0 |= CMD0_FIFO_IS_10B; + + if (xfers[i].flags & I2C_M_RD) { + ccmd->cmd0 |= CMD0_FIFO_RNW; + ccmd->rx_buf = xfers[i].buf; + ccmd->rx_len = xfers[i].len; + } else { + ccmd->tx_buf = xfers[i].buf; + ccmd->tx_len = xfers[i].len; + } + } + + cdns_i3c_master_queue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, m->i2c.timeout)) + cdns_i3c_master_unqueue_xfer(master, xfer); + + ret = xfer->ret; + cdns_i3c_master_free_xfer(xfer); + + return ret; +} + +struct cdns_i3c_i2c_dev_data { + u16 id; + s16 ibi; + struct i3c_generic_ibi_pool *ibi_pool; +}; + +static u32 prepare_rr0_dev_address(u32 addr) +{ + u32 ret = (addr << 1) & 0xff; + + /* RR0[7:1] = addr[6:0] */ + ret |= (addr & GENMASK(6, 0)) << 1; + + /* RR0[15:13] = addr[9:7] */ + ret |= (addr & GENMASK(9, 7)) << 6; + + /* RR0[0] = ~XOR(addr[6:0]) */ + ret |= parity8(addr & 0x7f) ? 0 : BIT(0); + + return ret; +} + +static void cdns_i3c_master_upd_i3c_addr(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + u32 rr; + + rr = prepare_rr0_dev_address(dev->info.dyn_addr ? + dev->info.dyn_addr : + dev->info.static_addr); + writel(DEV_ID_RR0_IS_I3C | rr, master->regs + DEV_ID_RR0(data->id)); +} + +static int cdns_i3c_master_get_rr_slot(struct cdns_i3c_master *master, + u8 dyn_addr) +{ + unsigned long activedevs; + u32 rr; + int i; + + if (!dyn_addr) { + if (!master->free_rr_slots) + return -ENOSPC; + + return ffs(master->free_rr_slots) - 1; + } + + activedevs = readl(master->regs + DEVS_CTRL) & DEVS_CTRL_DEVS_ACTIVE_MASK; + activedevs &= ~BIT(0); + + for_each_set_bit(i, &activedevs, master->maxdevs + 1) { + rr = readl(master->regs + DEV_ID_RR0(i)); + if (!(rr & DEV_ID_RR0_IS_I3C) || + DEV_ID_RR0_GET_DEV_ADDR(rr) != dyn_addr) + continue; + + return i; + } + + return -EINVAL; +} + +static int cdns_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, + u8 old_dyn_addr) +{ + cdns_i3c_master_upd_i3c_addr(dev); + + return 0; +} + +static int cdns_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data; + int slot; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + slot = cdns_i3c_master_get_rr_slot(master, dev->info.dyn_addr); + if (slot < 0) { + kfree(data); + return slot; + } + + data->ibi = -1; + data->id = slot; + i3c_dev_set_master_data(dev, data); + master->free_rr_slots &= ~BIT(slot); + + if (!dev->info.dyn_addr) { + cdns_i3c_master_upd_i3c_addr(dev); + writel(readl(master->regs + DEVS_CTRL) | + DEVS_CTRL_DEV_ACTIVE(data->id), + master->regs + DEVS_CTRL); + } + + return 0; +} + +static void cdns_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + writel(readl(master->regs + DEVS_CTRL) | + DEVS_CTRL_DEV_CLR(data->id), + master->regs + DEVS_CTRL); + + i3c_dev_set_master_data(dev, NULL); + master->free_rr_slots |= BIT(data->id); + kfree(data); +} + +static int cdns_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data; + int slot; + + slot = cdns_i3c_master_get_rr_slot(master, 0); + if (slot < 0) + return slot; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + data->id = slot; + master->free_rr_slots &= ~BIT(slot); + i2c_dev_set_master_data(dev, data); + + writel(prepare_rr0_dev_address(dev->addr), + master->regs + DEV_ID_RR0(data->id)); + writel(dev->lvr, master->regs + DEV_ID_RR2(data->id)); + writel(readl(master->regs + DEVS_CTRL) | + DEVS_CTRL_DEV_ACTIVE(data->id), + master->regs + DEVS_CTRL); + + return 0; +} + +static void cdns_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + + writel(readl(master->regs + DEVS_CTRL) | + DEVS_CTRL_DEV_CLR(data->id), + master->regs + DEVS_CTRL); + master->free_rr_slots |= BIT(data->id); + + i2c_dev_set_master_data(dev, NULL); + kfree(data); +} + +static void cdns_i3c_master_bus_cleanup(struct i3c_master_controller *m) +{ + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + + cdns_i3c_master_disable(master); +} + +static void cdns_i3c_master_dev_rr_to_info(struct cdns_i3c_master *master, + unsigned int slot, + struct i3c_device_info *info) +{ + u32 rr; + + memset(info, 0, sizeof(*info)); + rr = readl(master->regs + DEV_ID_RR0(slot)); + info->dyn_addr = DEV_ID_RR0_GET_DEV_ADDR(rr); + rr = readl(master->regs + DEV_ID_RR2(slot)); + info->dcr = rr; + info->bcr = rr >> 8; + info->pid = rr >> 16; + info->pid |= (u64)readl(master->regs + DEV_ID_RR1(slot)) << 16; +} + +static void cdns_i3c_master_upd_i3c_scl_lim(struct cdns_i3c_master *master) +{ + struct i3c_master_controller *m = &master->base; + unsigned long i3c_lim_period, pres_step, ncycles; + struct i3c_bus *bus = i3c_master_get_bus(m); + unsigned long new_i3c_scl_lim = 0; + struct i3c_dev_desc *dev; + u32 prescl1, ctrl; + + i3c_bus_for_each_i3cdev(bus, dev) { + unsigned long max_fscl; + + max_fscl = max(I3C_CCC_MAX_SDR_FSCL(dev->info.max_read_ds), + I3C_CCC_MAX_SDR_FSCL(dev->info.max_write_ds)); + switch (max_fscl) { + case I3C_SDR1_FSCL_8MHZ: + max_fscl = 8000000; + break; + case I3C_SDR2_FSCL_6MHZ: + max_fscl = 6000000; + break; + case I3C_SDR3_FSCL_4MHZ: + max_fscl = 4000000; + break; + case I3C_SDR4_FSCL_2MHZ: + max_fscl = 2000000; + break; + case I3C_SDR0_FSCL_MAX: + default: + max_fscl = 0; + break; + } + + if (max_fscl && + (new_i3c_scl_lim > max_fscl || !new_i3c_scl_lim)) + new_i3c_scl_lim = max_fscl; + } + + /* Only update PRESCL_CTRL1 if the I3C SCL limitation has changed. */ + if (new_i3c_scl_lim == master->i3c_scl_lim) + return; + master->i3c_scl_lim = new_i3c_scl_lim; + if (!new_i3c_scl_lim) + return; + pres_step = 1000000000UL / (bus->scl_rate.i3c * 4); + + /* Configure PP_LOW to meet I3C slave limitations. */ + prescl1 = readl(master->regs + PRESCL_CTRL1) & + ~PRESCL_CTRL1_PP_LOW_MASK; + ctrl = readl(master->regs + CTRL); + + i3c_lim_period = DIV_ROUND_UP(1000000000, master->i3c_scl_lim); + ncycles = DIV_ROUND_UP(i3c_lim_period, pres_step); + if (ncycles < 4) + ncycles = 0; + else + ncycles -= 4; + + prescl1 |= PRESCL_CTRL1_PP_LOW(ncycles); + + /* Disable I3C master before updating PRESCL_CTRL1. */ + if (ctrl & CTRL_DEV_EN) + cdns_i3c_master_disable(master); + + writel(prescl1, master->regs + PRESCL_CTRL1); + + if (ctrl & CTRL_DEV_EN) + cdns_i3c_master_enable(master); +} + +static int cdns_i3c_master_do_daa(struct i3c_master_controller *m) +{ + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + unsigned long olddevs, newdevs; + int ret, slot; + u8 addrs[MAX_DEVS] = { }; + u8 last_addr = 0; + + olddevs = readl(master->regs + DEVS_CTRL) & DEVS_CTRL_DEVS_ACTIVE_MASK; + olddevs |= BIT(0); + + /* Prepare RR slots before launching DAA. */ + for_each_clear_bit(slot, &olddevs, master->maxdevs + 1) { + ret = i3c_master_get_free_addr(m, last_addr + 1); + if (ret < 0) + return -ENOSPC; + + last_addr = ret; + addrs[slot] = last_addr; + writel(prepare_rr0_dev_address(last_addr) | DEV_ID_RR0_IS_I3C, + master->regs + DEV_ID_RR0(slot)); + writel(0, master->regs + DEV_ID_RR1(slot)); + writel(0, master->regs + DEV_ID_RR2(slot)); + } + + ret = i3c_master_entdaa_locked(&master->base); + if (ret) + return ret; + + newdevs = readl(master->regs + DEVS_CTRL) & DEVS_CTRL_DEVS_ACTIVE_MASK; + newdevs &= ~olddevs; + + /* + * Clear all retaining registers filled during DAA. We already + * have the addressed assigned to them in the addrs array. + */ + for_each_set_bit(slot, &newdevs, master->maxdevs + 1) + i3c_master_add_i3c_dev_locked(m, addrs[slot]); + + /* + * Clear slots that ended up not being used. Can be caused by I3C + * device creation failure or when the I3C device was already known + * by the system but with a different address (in this case the device + * already has a slot and does not need a new one). + */ + writel(readl(master->regs + DEVS_CTRL) | + master->free_rr_slots << DEVS_CTRL_DEV_CLR_SHIFT, + master->regs + DEVS_CTRL); + + i3c_master_defslvs_locked(&master->base); + + cdns_i3c_master_upd_i3c_scl_lim(master); + + /* Unmask Hot-Join and Mastership request interrupts. */ + i3c_master_enec_locked(m, I3C_BROADCAST_ADDR, + I3C_CCC_EVENT_HJ | I3C_CCC_EVENT_MR); + + return 0; +} + +static u8 cdns_i3c_master_calculate_thd_delay(struct cdns_i3c_master *master) +{ + unsigned long sysclk_rate = clk_get_rate(master->sysclk); + u8 thd_delay = DIV_ROUND_UP(master->devdata->thd_delay_ns, + (NSEC_PER_SEC / sysclk_rate)); + + /* Every value greater than 3 is not valid. */ + if (thd_delay > THD_DELAY_MAX) + thd_delay = THD_DELAY_MAX; + + /* CTLR_THD_DEL value is encoded. */ + return (THD_DELAY_MAX - thd_delay); +} + +static int cdns_i3c_master_bus_init(struct i3c_master_controller *m) +{ + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + unsigned long pres_step, sysclk_rate, max_i2cfreq; + struct i3c_bus *bus = i3c_master_get_bus(m); + u32 ctrl, prescl0, prescl1, pres, low; + struct i3c_device_info info = { }; + int ret, ncycles; + + switch (bus->mode) { + case I3C_BUS_MODE_PURE: + ctrl = CTRL_PURE_BUS_MODE; + break; + + case I3C_BUS_MODE_MIXED_FAST: + ctrl = CTRL_MIXED_FAST_BUS_MODE; + break; + + case I3C_BUS_MODE_MIXED_SLOW: + ctrl = CTRL_MIXED_SLOW_BUS_MODE; + break; + + default: + return -EINVAL; + } + + sysclk_rate = clk_get_rate(master->sysclk); + if (!sysclk_rate) + return -EINVAL; + + pres = DIV_ROUND_UP(sysclk_rate, (bus->scl_rate.i3c * 4)) - 1; + if (pres > PRESCL_CTRL0_I3C_MAX) + return -ERANGE; + + bus->scl_rate.i3c = sysclk_rate / ((pres + 1) * 4); + + prescl0 = PRESCL_CTRL0_I3C(pres); + + low = ((I3C_BUS_TLOW_OD_MIN_NS * sysclk_rate) / (pres + 1)) - 2; + prescl1 = PRESCL_CTRL1_OD_LOW(low); + + max_i2cfreq = bus->scl_rate.i2c; + + pres = (sysclk_rate / (max_i2cfreq * 5)) - 1; + if (pres > PRESCL_CTRL0_I2C_MAX) + return -ERANGE; + + bus->scl_rate.i2c = sysclk_rate / ((pres + 1) * 5); + + prescl0 |= PRESCL_CTRL0_I2C(pres); + writel(prescl0, master->regs + PRESCL_CTRL0); + + /* Calculate OD and PP low. */ + pres_step = 1000000000 / (bus->scl_rate.i3c * 4); + ncycles = DIV_ROUND_UP(I3C_BUS_TLOW_OD_MIN_NS, pres_step) - 2; + if (ncycles < 0) + ncycles = 0; + prescl1 = PRESCL_CTRL1_OD_LOW(ncycles); + writel(prescl1, master->regs + PRESCL_CTRL1); + + /* Get an address for the master. */ + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + return ret; + + writel(prepare_rr0_dev_address(ret) | DEV_ID_RR0_IS_I3C, + master->regs + DEV_ID_RR0(0)); + + cdns_i3c_master_dev_rr_to_info(master, 0, &info); + if (info.bcr & I3C_BCR_HDR_CAP) + info.hdr_cap = I3C_CCC_HDR_MODE(I3C_HDR_DDR); + + ret = i3c_master_set_info(&master->base, &info); + if (ret) + return ret; + + /* + * Enable Hot-Join, and, when a Hot-Join request happens, disable all + * events coming from this device. + * + * We will issue ENTDAA afterwards from the threaded IRQ handler. + */ + ctrl |= CTRL_HJ_ACK | CTRL_HJ_DISEC | CTRL_HALT_EN | CTRL_MCS_EN; + + /* + * Configure data hold delay based on device-specific data. + * + * MIPI I3C Specification 1.0 defines non-zero minimal tHD_PP timing on + * master output. This setting allows to meet this timing on master's + * SoC outputs, regardless of PCB balancing. + */ + ctrl |= CTRL_THD_DELAY(cdns_i3c_master_calculate_thd_delay(master)); + writel(ctrl, master->regs + CTRL); + + cdns_i3c_master_enable(master); + + return 0; +} + +static void cdns_i3c_master_handle_ibi(struct cdns_i3c_master *master, + u32 ibir) +{ + struct cdns_i3c_i2c_dev_data *data; + bool data_consumed = false; + struct i3c_ibi_slot *slot; + u32 id = IBIR_SLVID(ibir); + struct i3c_dev_desc *dev; + size_t nbytes; + u8 *buf; + + /* + * FIXME: maybe we should report the FIFO OVF errors to the upper + * layer. + */ + if (id >= master->ibi.num_slots || (ibir & IBIR_ERROR)) + goto out; + + dev = master->ibi.slots[id]; + spin_lock(&master->ibi.lock); + + data = i3c_dev_get_master_data(dev); + slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); + if (!slot) + goto out_unlock; + + buf = slot->data; + + nbytes = IBIR_XFER_BYTES(ibir); + i3c_readl_fifo(master->regs + IBI_DATA_FIFO, buf, nbytes); + + slot->len = min_t(unsigned int, IBIR_XFER_BYTES(ibir), + dev->ibi->max_payload_len); + i3c_master_queue_ibi(dev, slot); + data_consumed = true; + +out_unlock: + spin_unlock(&master->ibi.lock); + +out: + /* Consume data from the FIFO if it's not been done already. */ + if (!data_consumed) { + int i; + + for (i = 0; i < IBIR_XFER_BYTES(ibir); i += 4) + readl(master->regs + IBI_DATA_FIFO); + } +} + +static void cnds_i3c_master_demux_ibis(struct cdns_i3c_master *master) +{ + u32 status0; + + writel(MST_INT_IBIR_THR, master->regs + MST_ICR); + + for (status0 = readl(master->regs + MST_STATUS0); + !(status0 & MST_STATUS0_IBIR_EMP); + status0 = readl(master->regs + MST_STATUS0)) { + u32 ibir = readl(master->regs + IBIR); + + switch (IBIR_TYPE(ibir)) { + case IBIR_TYPE_IBI: + cdns_i3c_master_handle_ibi(master, ibir); + break; + + case IBIR_TYPE_HJ: + WARN_ON(IBIR_XFER_BYTES(ibir) || (ibir & IBIR_ERROR)); + i3c_master_queue_hotjoin(&master->base); + break; + + case IBIR_TYPE_MR: + WARN_ON(IBIR_XFER_BYTES(ibir) || (ibir & IBIR_ERROR)); + break; + + default: + break; + } + } +} + +static irqreturn_t cdns_i3c_master_interrupt(int irq, void *data) +{ + struct cdns_i3c_master *master = data; + u32 status; + + status = readl(master->regs + MST_ISR); + if (!(status & readl(master->regs + MST_IMR))) + return IRQ_NONE; + + spin_lock(&master->xferqueue.lock); + cdns_i3c_master_end_xfer_locked(master, status); + spin_unlock(&master->xferqueue.lock); + + if (status & MST_INT_IBIR_THR) + cnds_i3c_master_demux_ibis(master); + + return IRQ_HANDLED; +} + +static int cdns_i3c_master_disable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + u32 sirmap; + int ret; + + ret = i3c_master_disec_locked(m, dev->info.dyn_addr, + I3C_CCC_EVENT_SIR); + if (ret) + return ret; + + spin_lock_irqsave(&master->ibi.lock, flags); + sirmap = readl(master->regs + SIR_MAP_DEV_REG(data->ibi)); + sirmap &= ~SIR_MAP_DEV_CONF_MASK(data->ibi); + sirmap |= SIR_MAP_DEV_CONF(data->ibi, + SIR_MAP_DEV_DA(I3C_BROADCAST_ADDR)); + writel(sirmap, master->regs + SIR_MAP_DEV_REG(data->ibi)); + spin_unlock_irqrestore(&master->ibi.lock, flags); + + return ret; +} + +static int cdns_i3c_master_enable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + u32 sircfg, sirmap; + int ret; + + spin_lock_irqsave(&master->ibi.lock, flags); + sirmap = readl(master->regs + SIR_MAP_DEV_REG(data->ibi)); + sirmap &= ~SIR_MAP_DEV_CONF_MASK(data->ibi); + sircfg = SIR_MAP_DEV_ROLE(dev->info.bcr >> 6) | + SIR_MAP_DEV_DA(dev->info.dyn_addr) | + SIR_MAP_DEV_PL(dev->info.max_ibi_len) | + SIR_MAP_DEV_ACK; + + if (dev->info.bcr & I3C_BCR_MAX_DATA_SPEED_LIM) + sircfg |= SIR_MAP_DEV_SLOW; + + sirmap |= SIR_MAP_DEV_CONF(data->ibi, sircfg); + writel(sirmap, master->regs + SIR_MAP_DEV_REG(data->ibi)); + spin_unlock_irqrestore(&master->ibi.lock, flags); + + ret = i3c_master_enec_locked(m, dev->info.dyn_addr, + I3C_CCC_EVENT_SIR); + if (ret) { + spin_lock_irqsave(&master->ibi.lock, flags); + sirmap = readl(master->regs + SIR_MAP_DEV_REG(data->ibi)); + sirmap &= ~SIR_MAP_DEV_CONF_MASK(data->ibi); + sirmap |= SIR_MAP_DEV_CONF(data->ibi, + SIR_MAP_DEV_DA(I3C_BROADCAST_ADDR)); + writel(sirmap, master->regs + SIR_MAP_DEV_REG(data->ibi)); + spin_unlock_irqrestore(&master->ibi.lock, flags); + } + + return ret; +} + +static int cdns_i3c_master_request_ibi(struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + unsigned int i; + + data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(data->ibi_pool)) + return PTR_ERR(data->ibi_pool); + + spin_lock_irqsave(&master->ibi.lock, flags); + for (i = 0; i < master->ibi.num_slots; i++) { + if (!master->ibi.slots[i]) { + data->ibi = i; + master->ibi.slots[i] = dev; + break; + } + } + spin_unlock_irqrestore(&master->ibi.lock, flags); + + if (i < master->ibi.num_slots) + return 0; + + i3c_generic_ibi_free_pool(data->ibi_pool); + data->ibi_pool = NULL; + + return -ENOSPC; +} + +static void cdns_i3c_master_free_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct cdns_i3c_master *master = to_cdns_i3c_master(m); + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + + spin_lock_irqsave(&master->ibi.lock, flags); + master->ibi.slots[data->ibi] = NULL; + data->ibi = -1; + spin_unlock_irqrestore(&master->ibi.lock, flags); + + i3c_generic_ibi_free_pool(data->ibi_pool); +} + +static void cdns_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct cdns_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); +} + +static const struct i3c_master_controller_ops cdns_i3c_master_ops = { + .bus_init = cdns_i3c_master_bus_init, + .bus_cleanup = cdns_i3c_master_bus_cleanup, + .do_daa = cdns_i3c_master_do_daa, + .attach_i3c_dev = cdns_i3c_master_attach_i3c_dev, + .reattach_i3c_dev = cdns_i3c_master_reattach_i3c_dev, + .detach_i3c_dev = cdns_i3c_master_detach_i3c_dev, + .attach_i2c_dev = cdns_i3c_master_attach_i2c_dev, + .detach_i2c_dev = cdns_i3c_master_detach_i2c_dev, + .supports_ccc_cmd = cdns_i3c_master_supports_ccc_cmd, + .send_ccc_cmd = cdns_i3c_master_send_ccc_cmd, + .i3c_xfers = cdns_i3c_master_i3c_xfers, + .i2c_xfers = cdns_i3c_master_i2c_xfers, + .enable_ibi = cdns_i3c_master_enable_ibi, + .disable_ibi = cdns_i3c_master_disable_ibi, + .request_ibi = cdns_i3c_master_request_ibi, + .free_ibi = cdns_i3c_master_free_ibi, + .recycle_ibi_slot = cdns_i3c_master_recycle_ibi_slot, +}; + +static struct cdns_i3c_data cdns_i3c_devdata = { + .thd_delay_ns = 10, +}; + +static const struct of_device_id cdns_i3c_master_of_ids[] = { + { .compatible = "cdns,i3c-master", .data = &cdns_i3c_devdata }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, cdns_i3c_master_of_ids); + +static int cdns_i3c_master_probe(struct platform_device *pdev) +{ + struct cdns_i3c_master *master; + struct clk *pclk; + int ret, irq; + u32 val; + + master = devm_kzalloc(&pdev->dev, sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + master->devdata = of_device_get_match_data(&pdev->dev); + if (!master->devdata) + return -EINVAL; + + master->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(master->regs)) + return PTR_ERR(master->regs); + + pclk = devm_clk_get_enabled(&pdev->dev, "pclk"); + if (IS_ERR(pclk)) + return PTR_ERR(pclk); + + master->sysclk = devm_clk_get_enabled(&pdev->dev, "sysclk"); + if (IS_ERR(master->sysclk)) + return PTR_ERR(master->sysclk); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + if (readl(master->regs + DEV_ID) != DEV_ID_I3C_MASTER) + return -EINVAL; + + spin_lock_init(&master->xferqueue.lock); + INIT_LIST_HEAD(&master->xferqueue.list); + + writel(0xffffffff, master->regs + MST_IDR); + writel(0xffffffff, master->regs + SLV_IDR); + ret = devm_request_irq(&pdev->dev, irq, cdns_i3c_master_interrupt, 0, + dev_name(&pdev->dev), master); + if (ret) + return ret; + + platform_set_drvdata(pdev, master); + + val = readl(master->regs + CONF_STATUS0); + + /* Device ID0 is reserved to describe this master. */ + master->maxdevs = CONF_STATUS0_DEVS_NUM(val); + master->free_rr_slots = GENMASK(master->maxdevs, 1); + master->caps.ibirfifodepth = CONF_STATUS0_IBIR_DEPTH(val); + master->caps.cmdrfifodepth = CONF_STATUS0_CMDR_DEPTH(val); + + val = readl(master->regs + CONF_STATUS1); + master->caps.cmdfifodepth = CONF_STATUS1_CMD_DEPTH(val); + master->caps.rxfifodepth = CONF_STATUS1_RX_DEPTH(val); + master->caps.txfifodepth = CONF_STATUS1_TX_DEPTH(val); + + spin_lock_init(&master->ibi.lock); + master->ibi.num_slots = CONF_STATUS1_IBI_HW_RES(val); + master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots, + sizeof(*master->ibi.slots), + GFP_KERNEL); + if (!master->ibi.slots) + return -ENOMEM; + + writel(IBIR_THR(1), master->regs + CMD_IBI_THR_CTRL); + writel(MST_INT_IBIR_THR, master->regs + MST_IER); + writel(DEVS_CTRL_DEV_CLR_ALL, master->regs + DEVS_CTRL); + + return i3c_master_register(&master->base, &pdev->dev, + &cdns_i3c_master_ops, false); +} + +static void cdns_i3c_master_remove(struct platform_device *pdev) +{ + struct cdns_i3c_master *master = platform_get_drvdata(pdev); + + i3c_master_unregister(&master->base); +} + +static struct platform_driver cdns_i3c_master = { + .probe = cdns_i3c_master_probe, + .remove = cdns_i3c_master_remove, + .driver = { + .name = "cdns-i3c-master", + .of_match_table = cdns_i3c_master_of_ids, + }, +}; +module_platform_driver(cdns_i3c_master); + +MODULE_AUTHOR("Boris Brezillon <boris.brezillon@bootlin.com>"); +MODULE_DESCRIPTION("Cadence I3C master driver"); +MODULE_LICENSE("GPL v2"); +MODULE_ALIAS("platform:cdns-i3c-master"); diff --git a/drivers/i3c/master/mipi-i3c-hci/Makefile b/drivers/i3c/master/mipi-i3c-hci/Makefile new file mode 100644 index 000000000..e3d3ef757 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/Makefile @@ -0,0 +1,8 @@ +# SPDX-License-Identifier: BSD-3-Clause + +obj-$(CONFIG_MIPI_I3C_HCI) += mipi-i3c-hci.o +mipi-i3c-hci-y := core.o ext_caps.o pio.o dma.o \ + cmd_v1.o cmd_v2.o \ + dat_v1.o dct_v1.o \ + hci_quirks.o +obj-$(CONFIG_MIPI_I3C_HCI_PCI) += mipi-i3c-hci-pci.o diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd.h b/drivers/i3c/master/mipi-i3c-hci/cmd.h new file mode 100644 index 000000000..7bada7b4b --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/cmd.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Common command/response related stuff + */ + +#ifndef CMD_H +#define CMD_H + +/* + * Those bits are common to all descriptor formats and + * may be manipulated by the core code. + */ +#define CMD_0_TOC W0_BIT_(31) +#define CMD_0_ROC W0_BIT_(30) +#define CMD_0_ATTR W0_MASK(2, 0) +#define CMD_0_TID W0_MASK(6, 3) + +/* + * Response Descriptor Structure + */ +#define RESP_STATUS(resp) FIELD_GET(GENMASK(31, 28), resp) +#define RESP_TID(resp) FIELD_GET(GENMASK(27, 24), resp) +#define RESP_DATA_LENGTH(resp) FIELD_GET(GENMASK(21, 0), resp) + +#define RESP_ERR_FIELD GENMASK(31, 28) + +enum hci_resp_err { + RESP_SUCCESS = 0x0, + RESP_ERR_CRC = 0x1, + RESP_ERR_PARITY = 0x2, + RESP_ERR_FRAME = 0x3, + RESP_ERR_ADDR_HEADER = 0x4, + RESP_ERR_BCAST_NACK_7E = 0x4, + RESP_ERR_NACK = 0x5, + RESP_ERR_OVL = 0x6, + RESP_ERR_I3C_SHORT_READ = 0x7, + RESP_ERR_HC_TERMINATED = 0x8, + RESP_ERR_I2C_WR_DATA_NACK = 0x9, + RESP_ERR_BUS_XFER_ABORTED = 0x9, + RESP_ERR_NOT_SUPPORTED = 0xa, + RESP_ERR_ABORTED_WITH_CRC = 0xb, + /* 0xc to 0xf are reserved for transfer specific errors */ +}; + +/* TID generation (4 bits wide in all cases) */ +#define hci_get_tid(bits) \ + (atomic_inc_return_relaxed(&hci->next_cmd_tid) % (1U << 4)) + +/* This abstracts operations with our command descriptor formats */ +struct hci_cmd_ops { + int (*prep_ccc)(struct i3c_hci *hci, struct hci_xfer *xfer, + u8 ccc_addr, u8 ccc_cmd, bool raw); + void (*prep_i3c_xfer)(struct i3c_hci *hci, struct i3c_dev_desc *dev, + struct hci_xfer *xfer); + void (*prep_i2c_xfer)(struct i3c_hci *hci, struct i2c_dev_desc *dev, + struct hci_xfer *xfer); + int (*perform_daa)(struct i3c_hci *hci); +}; + +/* Our various instances */ +extern const struct hci_cmd_ops mipi_i3c_hci_cmd_v1; +extern const struct hci_cmd_ops mipi_i3c_hci_cmd_v2; + +static inline void hci_cmd_set_resp_err(u32 *response, int resp_err) +{ + *response &= ~RESP_ERR_FIELD; + *response |= FIELD_PREP(RESP_ERR_FIELD, resp_err); +} + +#endif diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c new file mode 100644 index 000000000..3b9345718 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v1.c @@ -0,0 +1,375 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * I3C HCI v1.0/v1.1 Command Descriptor Handling + */ + +#include <linux/bitfield.h> +#include <linux/i3c/master.h> + +#include "hci.h" +#include "cmd.h" +#include "dat.h" +#include "dct.h" + +/* + * Address Assignment Command + */ + +#define CMD_0_ATTR_A FIELD_PREP(CMD_0_ATTR, 0x2) + +#define CMD_A0_TOC W0_BIT_(31) +#define CMD_A0_ROC W0_BIT_(30) +#define CMD_A0_DEV_COUNT(v) FIELD_PREP(W0_MASK(29, 26), v) +#define CMD_A0_DEV_INDEX(v) FIELD_PREP(W0_MASK(20, 16), v) +#define CMD_A0_CMD(v) FIELD_PREP(W0_MASK(14, 7), v) +#define CMD_A0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +/* + * Immediate Data Transfer Command + */ + +#define CMD_0_ATTR_I FIELD_PREP(CMD_0_ATTR, 0x1) + +#define CMD_I1_DATA_BYTE_4(v) FIELD_PREP(W1_MASK(63, 56), v) +#define CMD_I1_DATA_BYTE_3(v) FIELD_PREP(W1_MASK(55, 48), v) +#define CMD_I1_DATA_BYTE_2(v) FIELD_PREP(W1_MASK(47, 40), v) +#define CMD_I1_DATA_BYTE_1(v) FIELD_PREP(W1_MASK(39, 32), v) +#define CMD_I1_DEF_BYTE(v) FIELD_PREP(W1_MASK(39, 32), v) +#define CMD_I0_TOC W0_BIT_(31) +#define CMD_I0_ROC W0_BIT_(30) +#define CMD_I0_RNW W0_BIT_(29) +#define CMD_I0_MODE(v) FIELD_PREP(W0_MASK(28, 26), v) +#define CMD_I0_DTT(v) FIELD_PREP(W0_MASK(25, 23), v) +#define CMD_I0_DEV_INDEX(v) FIELD_PREP(W0_MASK(20, 16), v) +#define CMD_I0_CP W0_BIT_(15) +#define CMD_I0_CMD(v) FIELD_PREP(W0_MASK(14, 7), v) +#define CMD_I0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +/* + * Regular Data Transfer Command + */ + +#define CMD_0_ATTR_R FIELD_PREP(CMD_0_ATTR, 0x0) + +#define CMD_R1_DATA_LENGTH(v) FIELD_PREP(W1_MASK(63, 48), v) +#define CMD_R1_DEF_BYTE(v) FIELD_PREP(W1_MASK(39, 32), v) +#define CMD_R0_TOC W0_BIT_(31) +#define CMD_R0_ROC W0_BIT_(30) +#define CMD_R0_RNW W0_BIT_(29) +#define CMD_R0_MODE(v) FIELD_PREP(W0_MASK(28, 26), v) +#define CMD_R0_DBP W0_BIT_(25) +#define CMD_R0_DEV_INDEX(v) FIELD_PREP(W0_MASK(20, 16), v) +#define CMD_R0_CP W0_BIT_(15) +#define CMD_R0_CMD(v) FIELD_PREP(W0_MASK(14, 7), v) +#define CMD_R0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +/* + * Combo Transfer (Write + Write/Read) Command + */ + +#define CMD_0_ATTR_C FIELD_PREP(CMD_0_ATTR, 0x3) + +#define CMD_C1_DATA_LENGTH(v) FIELD_PREP(W1_MASK(63, 48), v) +#define CMD_C1_OFFSET(v) FIELD_PREP(W1_MASK(47, 32), v) +#define CMD_C0_TOC W0_BIT_(31) +#define CMD_C0_ROC W0_BIT_(30) +#define CMD_C0_RNW W0_BIT_(29) +#define CMD_C0_MODE(v) FIELD_PREP(W0_MASK(28, 26), v) +#define CMD_C0_16_BIT_SUBOFFSET W0_BIT_(25) +#define CMD_C0_FIRST_PHASE_MODE W0_BIT_(24) +#define CMD_C0_DATA_LENGTH_POSITION(v) FIELD_PREP(W0_MASK(23, 22), v) +#define CMD_C0_DEV_INDEX(v) FIELD_PREP(W0_MASK(20, 16), v) +#define CMD_C0_CP W0_BIT_(15) +#define CMD_C0_CMD(v) FIELD_PREP(W0_MASK(14, 7), v) +#define CMD_C0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +/* + * Internal Control Command + */ + +#define CMD_0_ATTR_M FIELD_PREP(CMD_0_ATTR, 0x7) + +#define CMD_M1_VENDOR_SPECIFIC W1_MASK(63, 32) +#define CMD_M0_MIPI_RESERVED W0_MASK(31, 12) +#define CMD_M0_MIPI_CMD W0_MASK(11, 8) +#define CMD_M0_VENDOR_INFO_PRESENT W0_BIT_( 7) +#define CMD_M0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +/* Data Transfer Speed and Mode */ +enum hci_cmd_mode { + MODE_I3C_SDR0 = 0x0, + MODE_I3C_SDR1 = 0x1, + MODE_I3C_SDR2 = 0x2, + MODE_I3C_SDR3 = 0x3, + MODE_I3C_SDR4 = 0x4, + MODE_I3C_HDR_TSx = 0x5, + MODE_I3C_HDR_DDR = 0x6, + MODE_I3C_HDR_BT = 0x7, + MODE_I3C_Fm_FmP = 0x8, + MODE_I2C_Fm = 0x0, + MODE_I2C_FmP = 0x1, + MODE_I2C_UD1 = 0x2, + MODE_I2C_UD2 = 0x3, + MODE_I2C_UD3 = 0x4, +}; + +static enum hci_cmd_mode get_i3c_mode(struct i3c_hci *hci) +{ + struct i3c_bus *bus = i3c_master_get_bus(&hci->master); + + if (bus->scl_rate.i3c > 8000000) + return MODE_I3C_SDR0; + if (bus->scl_rate.i3c > 6000000) + return MODE_I3C_SDR1; + if (bus->scl_rate.i3c > 4000000) + return MODE_I3C_SDR2; + if (bus->scl_rate.i3c > 2000000) + return MODE_I3C_SDR3; + return MODE_I3C_SDR4; +} + +static enum hci_cmd_mode get_i2c_mode(struct i3c_hci *hci) +{ + struct i3c_bus *bus = i3c_master_get_bus(&hci->master); + + if (bus->scl_rate.i2c >= 1000000) + return MODE_I2C_FmP; + return MODE_I2C_Fm; +} + +static void fill_data_bytes(struct hci_xfer *xfer, u8 *data, + unsigned int data_len) +{ + xfer->cmd_desc[1] = 0; + switch (data_len) { + case 4: + xfer->cmd_desc[1] |= CMD_I1_DATA_BYTE_4(data[3]); + fallthrough; + case 3: + xfer->cmd_desc[1] |= CMD_I1_DATA_BYTE_3(data[2]); + fallthrough; + case 2: + xfer->cmd_desc[1] |= CMD_I1_DATA_BYTE_2(data[1]); + fallthrough; + case 1: + xfer->cmd_desc[1] |= CMD_I1_DATA_BYTE_1(data[0]); + fallthrough; + case 0: + break; + } + /* we consumed all the data with the cmd descriptor */ + xfer->data = NULL; +} + +static int hci_cmd_v1_prep_ccc(struct i3c_hci *hci, + struct hci_xfer *xfer, + u8 ccc_addr, u8 ccc_cmd, bool raw) +{ + unsigned int dat_idx = 0; + enum hci_cmd_mode mode = get_i3c_mode(hci); + u8 *data = xfer->data; + unsigned int data_len = xfer->data_len; + bool rnw = xfer->rnw; + int ret; + + /* this should never happen */ + if (WARN_ON(raw)) + return -EINVAL; + + if (ccc_addr != I3C_BROADCAST_ADDR) { + ret = mipi_i3c_hci_dat_v1.get_index(hci, ccc_addr); + if (ret < 0) + return ret; + dat_idx = ret; + } + + xfer->cmd_tid = hci_get_tid(); + + if (!rnw && data_len <= 4) { + /* we use an Immediate Data Transfer Command */ + xfer->cmd_desc[0] = + CMD_0_ATTR_I | + CMD_I0_TID(xfer->cmd_tid) | + CMD_I0_CMD(ccc_cmd) | CMD_I0_CP | + CMD_I0_DEV_INDEX(dat_idx) | + CMD_I0_DTT(data_len) | + CMD_I0_MODE(mode); + fill_data_bytes(xfer, data, data_len); + } else { + /* we use a Regular Data Transfer Command */ + xfer->cmd_desc[0] = + CMD_0_ATTR_R | + CMD_R0_TID(xfer->cmd_tid) | + CMD_R0_CMD(ccc_cmd) | CMD_R0_CP | + CMD_R0_DEV_INDEX(dat_idx) | + CMD_R0_MODE(mode) | + (rnw ? CMD_R0_RNW : 0); + xfer->cmd_desc[1] = + CMD_R1_DATA_LENGTH(data_len); + } + + return 0; +} + +static void hci_cmd_v1_prep_i3c_xfer(struct i3c_hci *hci, + struct i3c_dev_desc *dev, + struct hci_xfer *xfer) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + unsigned int dat_idx = dev_data->dat_idx; + enum hci_cmd_mode mode = get_i3c_mode(hci); + u8 *data = xfer->data; + unsigned int data_len = xfer->data_len; + bool rnw = xfer->rnw; + + xfer->cmd_tid = hci_get_tid(); + + if (!rnw && data_len <= 4) { + /* we use an Immediate Data Transfer Command */ + xfer->cmd_desc[0] = + CMD_0_ATTR_I | + CMD_I0_TID(xfer->cmd_tid) | + CMD_I0_DEV_INDEX(dat_idx) | + CMD_I0_DTT(data_len) | + CMD_I0_MODE(mode); + fill_data_bytes(xfer, data, data_len); + } else { + /* we use a Regular Data Transfer Command */ + xfer->cmd_desc[0] = + CMD_0_ATTR_R | + CMD_R0_TID(xfer->cmd_tid) | + CMD_R0_DEV_INDEX(dat_idx) | + CMD_R0_MODE(mode) | + (rnw ? CMD_R0_RNW : 0); + xfer->cmd_desc[1] = + CMD_R1_DATA_LENGTH(data_len); + } +} + +static void hci_cmd_v1_prep_i2c_xfer(struct i3c_hci *hci, + struct i2c_dev_desc *dev, + struct hci_xfer *xfer) +{ + struct i3c_hci_dev_data *dev_data = i2c_dev_get_master_data(dev); + unsigned int dat_idx = dev_data->dat_idx; + enum hci_cmd_mode mode = get_i2c_mode(hci); + u8 *data = xfer->data; + unsigned int data_len = xfer->data_len; + bool rnw = xfer->rnw; + + xfer->cmd_tid = hci_get_tid(); + + if (!rnw && data_len <= 4) { + /* we use an Immediate Data Transfer Command */ + xfer->cmd_desc[0] = + CMD_0_ATTR_I | + CMD_I0_TID(xfer->cmd_tid) | + CMD_I0_DEV_INDEX(dat_idx) | + CMD_I0_DTT(data_len) | + CMD_I0_MODE(mode); + fill_data_bytes(xfer, data, data_len); + } else { + /* we use a Regular Data Transfer Command */ + xfer->cmd_desc[0] = + CMD_0_ATTR_R | + CMD_R0_TID(xfer->cmd_tid) | + CMD_R0_DEV_INDEX(dat_idx) | + CMD_R0_MODE(mode) | + (rnw ? CMD_R0_RNW : 0); + xfer->cmd_desc[1] = + CMD_R1_DATA_LENGTH(data_len); + } +} + +static int hci_cmd_v1_daa(struct i3c_hci *hci) +{ + struct hci_xfer *xfer; + int ret, dat_idx = -1; + u8 next_addr = 0; + u64 pid; + unsigned int dcr, bcr; + DECLARE_COMPLETION_ONSTACK(done); + + xfer = hci_alloc_xfer(1); + if (!xfer) + return -ENOMEM; + + /* + * Simple for now: we allocate a temporary DAT entry, do a single + * DAA, register the device which will allocate its own DAT entry + * via the core callback, then free the temporary DAT entry. + * Loop until there is no more devices to assign an address to. + * Yes, there is room for improvements. + */ + for (;;) { + ret = mipi_i3c_hci_dat_v1.alloc_entry(hci); + if (ret < 0) + break; + dat_idx = ret; + ret = i3c_master_get_free_addr(&hci->master, next_addr); + if (ret < 0) + break; + next_addr = ret; + + dev_dbg(&hci->master.dev, + "next_addr = 0x%02x, DAA using DAT %d", + next_addr, dat_idx); + mipi_i3c_hci_dat_v1.set_dynamic_addr(hci, dat_idx, next_addr); + mipi_i3c_hci_dct_index_reset(hci); + + xfer->cmd_tid = hci_get_tid(); + xfer->cmd_desc[0] = + CMD_0_ATTR_A | + CMD_A0_TID(xfer->cmd_tid) | + CMD_A0_CMD(I3C_CCC_ENTDAA) | + CMD_A0_DEV_INDEX(dat_idx) | + CMD_A0_DEV_COUNT(1) | + CMD_A0_ROC | CMD_A0_TOC; + xfer->cmd_desc[1] = 0; + xfer->completion = &done; + xfer->timeout = HZ; + ret = i3c_hci_process_xfer(hci, xfer, 1); + if (ret) + break; + if ((RESP_STATUS(xfer->response) == RESP_ERR_ADDR_HEADER || + RESP_STATUS(xfer->response) == RESP_ERR_NACK) && + RESP_DATA_LENGTH(xfer->response) == 1) { + ret = 0; /* no more devices to be assigned */ + break; + } + if (RESP_STATUS(xfer->response) != RESP_SUCCESS) { + ret = -EIO; + break; + } + + i3c_hci_dct_get_val(hci, 0, &pid, &dcr, &bcr); + dev_dbg(&hci->master.dev, + "assigned address %#x to device PID=0x%llx DCR=%#x BCR=%#x", + next_addr, pid, dcr, bcr); + + mipi_i3c_hci_dat_v1.free_entry(hci, dat_idx); + dat_idx = -1; + + /* + * TODO: Extend the subsystem layer to allow for registering + * new device and provide BCR/DCR/PID at the same time. + */ + i3c_master_add_i3c_dev_locked(&hci->master, next_addr); + } + + if (dat_idx >= 0) + mipi_i3c_hci_dat_v1.free_entry(hci, dat_idx); + hci_free_xfer(xfer, 1); + return ret; +} + +const struct hci_cmd_ops mipi_i3c_hci_cmd_v1 = { + .prep_ccc = hci_cmd_v1_prep_ccc, + .prep_i3c_xfer = hci_cmd_v1_prep_i3c_xfer, + .prep_i2c_xfer = hci_cmd_v1_prep_i2c_xfer, + .perform_daa = hci_cmd_v1_daa, +}; diff --git a/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c new file mode 100644 index 000000000..8d93748e8 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/cmd_v2.c @@ -0,0 +1,311 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * I3C HCI v2.0 Command Descriptor Handling + * + * Note: The I3C HCI v2.0 spec is still in flux. The code here will change. + */ + +#include <linux/bitfield.h> +#include <linux/i3c/master.h> + +#include "hci.h" +#include "cmd.h" +#include "xfer_mode_rate.h" + +/* + * Unified Data Transfer Command + */ + +#define CMD_0_ATTR_U FIELD_PREP(CMD_0_ATTR, 0x4) + +#define CMD_U3_HDR_TSP_ML_CTRL(v) FIELD_PREP(W3_MASK(107, 104), v) +#define CMD_U3_IDB4(v) FIELD_PREP(W3_MASK(103, 96), v) +#define CMD_U3_HDR_CMD(v) FIELD_PREP(W3_MASK(103, 96), v) +#define CMD_U2_IDB3(v) FIELD_PREP(W2_MASK( 95, 88), v) +#define CMD_U2_HDR_BT(v) FIELD_PREP(W2_MASK( 95, 88), v) +#define CMD_U2_IDB2(v) FIELD_PREP(W2_MASK( 87, 80), v) +#define CMD_U2_BT_CMD2(v) FIELD_PREP(W2_MASK( 87, 80), v) +#define CMD_U2_IDB1(v) FIELD_PREP(W2_MASK( 79, 72), v) +#define CMD_U2_BT_CMD1(v) FIELD_PREP(W2_MASK( 79, 72), v) +#define CMD_U2_IDB0(v) FIELD_PREP(W2_MASK( 71, 64), v) +#define CMD_U2_BT_CMD0(v) FIELD_PREP(W2_MASK( 71, 64), v) +#define CMD_U1_ERR_HANDLING(v) FIELD_PREP(W1_MASK( 63, 62), v) +#define CMD_U1_ADD_FUNC(v) FIELD_PREP(W1_MASK( 61, 56), v) +#define CMD_U1_COMBO_XFER W1_BIT_( 55) +#define CMD_U1_DATA_LENGTH(v) FIELD_PREP(W1_MASK( 53, 32), v) +#define CMD_U0_TOC W0_BIT_( 31) +#define CMD_U0_ROC W0_BIT_( 30) +#define CMD_U0_MAY_YIELD W0_BIT_( 29) +#define CMD_U0_NACK_RCNT(v) FIELD_PREP(W0_MASK( 28, 27), v) +#define CMD_U0_IDB_COUNT(v) FIELD_PREP(W0_MASK( 26, 24), v) +#define CMD_U0_MODE_INDEX(v) FIELD_PREP(W0_MASK( 22, 18), v) +#define CMD_U0_XFER_RATE(v) FIELD_PREP(W0_MASK( 17, 15), v) +#define CMD_U0_DEV_ADDRESS(v) FIELD_PREP(W0_MASK( 14, 8), v) +#define CMD_U0_RnW W0_BIT_( 7) +#define CMD_U0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +/* + * Address Assignment Command + */ + +#define CMD_0_ATTR_A FIELD_PREP(CMD_0_ATTR, 0x2) + +#define CMD_A1_DATA_LENGTH(v) FIELD_PREP(W1_MASK( 53, 32), v) +#define CMD_A0_TOC W0_BIT_( 31) +#define CMD_A0_ROC W0_BIT_( 30) +#define CMD_A0_XFER_RATE(v) FIELD_PREP(W0_MASK( 17, 15), v) +#define CMD_A0_ASSIGN_ADDRESS(v) FIELD_PREP(W0_MASK( 14, 8), v) +#define CMD_A0_TID(v) FIELD_PREP(W0_MASK( 6, 3), v) + +static unsigned int get_i3c_rate_idx(struct i3c_hci *hci) +{ + struct i3c_bus *bus = i3c_master_get_bus(&hci->master); + + if (bus->scl_rate.i3c >= 12000000) + return XFERRATE_I3C_SDR0; + if (bus->scl_rate.i3c > 8000000) + return XFERRATE_I3C_SDR1; + if (bus->scl_rate.i3c > 6000000) + return XFERRATE_I3C_SDR2; + if (bus->scl_rate.i3c > 4000000) + return XFERRATE_I3C_SDR3; + if (bus->scl_rate.i3c > 2000000) + return XFERRATE_I3C_SDR4; + return XFERRATE_I3C_SDR_FM_FMP; +} + +static unsigned int get_i2c_rate_idx(struct i3c_hci *hci) +{ + struct i3c_bus *bus = i3c_master_get_bus(&hci->master); + + if (bus->scl_rate.i2c >= 1000000) + return XFERRATE_I2C_FMP; + return XFERRATE_I2C_FM; +} + +static void hci_cmd_v2_prep_private_xfer(struct i3c_hci *hci, + struct hci_xfer *xfer, + u8 addr, unsigned int mode, + unsigned int rate) +{ + u8 *data = xfer->data; + unsigned int data_len = xfer->data_len; + bool rnw = xfer->rnw; + + xfer->cmd_tid = hci_get_tid(); + + if (!rnw && data_len <= 5) { + xfer->cmd_desc[0] = + CMD_0_ATTR_U | + CMD_U0_TID(xfer->cmd_tid) | + CMD_U0_DEV_ADDRESS(addr) | + CMD_U0_XFER_RATE(rate) | + CMD_U0_MODE_INDEX(mode) | + CMD_U0_IDB_COUNT(data_len); + xfer->cmd_desc[1] = + CMD_U1_DATA_LENGTH(0); + xfer->cmd_desc[2] = 0; + xfer->cmd_desc[3] = 0; + switch (data_len) { + case 5: + xfer->cmd_desc[3] |= CMD_U3_IDB4(data[4]); + fallthrough; + case 4: + xfer->cmd_desc[2] |= CMD_U2_IDB3(data[3]); + fallthrough; + case 3: + xfer->cmd_desc[2] |= CMD_U2_IDB2(data[2]); + fallthrough; + case 2: + xfer->cmd_desc[2] |= CMD_U2_IDB1(data[1]); + fallthrough; + case 1: + xfer->cmd_desc[2] |= CMD_U2_IDB0(data[0]); + fallthrough; + case 0: + break; + } + /* we consumed all the data with the cmd descriptor */ + xfer->data = NULL; + } else { + xfer->cmd_desc[0] = + CMD_0_ATTR_U | + CMD_U0_TID(xfer->cmd_tid) | + (rnw ? CMD_U0_RnW : 0) | + CMD_U0_DEV_ADDRESS(addr) | + CMD_U0_XFER_RATE(rate) | + CMD_U0_MODE_INDEX(mode); + xfer->cmd_desc[1] = + CMD_U1_DATA_LENGTH(data_len); + xfer->cmd_desc[2] = 0; + xfer->cmd_desc[3] = 0; + } +} + +static int hci_cmd_v2_prep_ccc(struct i3c_hci *hci, struct hci_xfer *xfer, + u8 ccc_addr, u8 ccc_cmd, bool raw) +{ + unsigned int mode = XFERMODE_IDX_I3C_SDR; + unsigned int rate = get_i3c_rate_idx(hci); + u8 *data = xfer->data; + unsigned int data_len = xfer->data_len; + bool rnw = xfer->rnw; + + if (raw && ccc_addr != I3C_BROADCAST_ADDR) { + hci_cmd_v2_prep_private_xfer(hci, xfer, ccc_addr, mode, rate); + return 0; + } + + xfer->cmd_tid = hci_get_tid(); + + if (!rnw && data_len <= 4) { + xfer->cmd_desc[0] = + CMD_0_ATTR_U | + CMD_U0_TID(xfer->cmd_tid) | + CMD_U0_DEV_ADDRESS(ccc_addr) | + CMD_U0_XFER_RATE(rate) | + CMD_U0_MODE_INDEX(mode) | + CMD_U0_IDB_COUNT(data_len + (!raw ? 0 : 1)); + xfer->cmd_desc[1] = + CMD_U1_DATA_LENGTH(0); + xfer->cmd_desc[2] = + CMD_U2_IDB0(ccc_cmd); + xfer->cmd_desc[3] = 0; + switch (data_len) { + case 4: + xfer->cmd_desc[3] |= CMD_U3_IDB4(data[3]); + fallthrough; + case 3: + xfer->cmd_desc[2] |= CMD_U2_IDB3(data[2]); + fallthrough; + case 2: + xfer->cmd_desc[2] |= CMD_U2_IDB2(data[1]); + fallthrough; + case 1: + xfer->cmd_desc[2] |= CMD_U2_IDB1(data[0]); + fallthrough; + case 0: + break; + } + /* we consumed all the data with the cmd descriptor */ + xfer->data = NULL; + } else { + xfer->cmd_desc[0] = + CMD_0_ATTR_U | + CMD_U0_TID(xfer->cmd_tid) | + (rnw ? CMD_U0_RnW : 0) | + CMD_U0_DEV_ADDRESS(ccc_addr) | + CMD_U0_XFER_RATE(rate) | + CMD_U0_MODE_INDEX(mode) | + CMD_U0_IDB_COUNT(!raw ? 0 : 1); + xfer->cmd_desc[1] = + CMD_U1_DATA_LENGTH(data_len); + xfer->cmd_desc[2] = + CMD_U2_IDB0(ccc_cmd); + xfer->cmd_desc[3] = 0; + } + + return 0; +} + +static void hci_cmd_v2_prep_i3c_xfer(struct i3c_hci *hci, + struct i3c_dev_desc *dev, + struct hci_xfer *xfer) +{ + unsigned int mode = XFERMODE_IDX_I3C_SDR; + unsigned int rate = get_i3c_rate_idx(hci); + u8 addr = dev->info.dyn_addr; + + hci_cmd_v2_prep_private_xfer(hci, xfer, addr, mode, rate); +} + +static void hci_cmd_v2_prep_i2c_xfer(struct i3c_hci *hci, + struct i2c_dev_desc *dev, + struct hci_xfer *xfer) +{ + unsigned int mode = XFERMODE_IDX_I2C; + unsigned int rate = get_i2c_rate_idx(hci); + u8 addr = dev->addr; + + hci_cmd_v2_prep_private_xfer(hci, xfer, addr, mode, rate); +} + +static int hci_cmd_v2_daa(struct i3c_hci *hci) +{ + struct hci_xfer *xfer; + int ret; + u8 next_addr = 0; + u32 device_id[2]; + u64 pid; + unsigned int dcr, bcr; + DECLARE_COMPLETION_ONSTACK(done); + + xfer = hci_alloc_xfer(2); + if (!xfer) + return -ENOMEM; + + xfer[0].data = &device_id; + xfer[0].data_len = 8; + xfer[0].rnw = true; + xfer[0].cmd_desc[1] = CMD_A1_DATA_LENGTH(8); + xfer[1].completion = &done; + xfer[1].timeout = HZ; + + for (;;) { + ret = i3c_master_get_free_addr(&hci->master, next_addr); + if (ret < 0) + break; + next_addr = ret; + dev_dbg(&hci->master.dev, "next_addr = 0x%02x", next_addr); + xfer[0].cmd_tid = hci_get_tid(); + xfer[0].cmd_desc[0] = + CMD_0_ATTR_A | + CMD_A0_TID(xfer[0].cmd_tid) | + CMD_A0_ROC; + xfer[1].cmd_tid = hci_get_tid(); + xfer[1].cmd_desc[0] = + CMD_0_ATTR_A | + CMD_A0_TID(xfer[1].cmd_tid) | + CMD_A0_ASSIGN_ADDRESS(next_addr) | + CMD_A0_ROC | + CMD_A0_TOC; + ret = i3c_hci_process_xfer(hci, xfer, 2); + if (ret) + break; + if (RESP_STATUS(xfer[0].response) != RESP_SUCCESS) { + ret = 0; /* no more devices to be assigned */ + break; + } + if (RESP_STATUS(xfer[1].response) != RESP_SUCCESS) { + ret = -EIO; + break; + } + + pid = FIELD_GET(W1_MASK(47, 32), device_id[1]); + pid = (pid << 32) | device_id[0]; + bcr = FIELD_GET(W1_MASK(55, 48), device_id[1]); + dcr = FIELD_GET(W1_MASK(63, 56), device_id[1]); + dev_dbg(&hci->master.dev, + "assigned address %#x to device PID=0x%llx DCR=%#x BCR=%#x", + next_addr, pid, dcr, bcr); + /* + * TODO: Extend the subsystem layer to allow for registering + * new device and provide BCR/DCR/PID at the same time. + */ + i3c_master_add_i3c_dev_locked(&hci->master, next_addr); + } + + hci_free_xfer(xfer, 2); + return ret; +} + +const struct hci_cmd_ops mipi_i3c_hci_cmd_v2 = { + .prep_ccc = hci_cmd_v2_prep_ccc, + .prep_i3c_xfer = hci_cmd_v2_prep_i3c_xfer, + .prep_i2c_xfer = hci_cmd_v2_prep_i2c_xfer, + .perform_daa = hci_cmd_v2_daa, +}; diff --git a/drivers/i3c/master/mipi-i3c-hci/core.c b/drivers/i3c/master/mipi-i3c-hci/core.c new file mode 100644 index 000000000..dadf049bd --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/core.c @@ -0,0 +1,1256 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Core driver code with main interface to the I3C subsystem. + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/platform_data/mipi-i3c-hci.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +#include "hci.h" +#include "ext_caps.h" +#include "cmd.h" +#include "dat.h" +#include "ibi.h" + +/* + * Host Controller Capabilities and Operation Registers + */ + +#define HCI_VERSION 0x00 /* HCI Version (in BCD) */ + +#define HC_CONTROL 0x04 +#define HC_CONTROL_BUS_ENABLE BIT(31) +#define HC_CONTROL_RESUME BIT(30) +#define HC_CONTROL_ABORT BIT(29) +#define HC_CONTROL_HALT_ON_CMD_TIMEOUT BIT(12) +#define HC_CONTROL_HOT_JOIN_CTRL BIT(8) /* Hot-Join ACK/NACK Control */ +#define HC_CONTROL_I2C_TARGET_PRESENT BIT(7) +#define HC_CONTROL_PIO_MODE BIT(6) /* DMA/PIO Mode Selector */ +#define HC_CONTROL_DATA_BIG_ENDIAN BIT(4) +#define HC_CONTROL_IBA_INCLUDE BIT(0) /* Include I3C Broadcast Address */ + +#define MASTER_DEVICE_ADDR 0x08 /* Master Device Address */ +#define MASTER_DYNAMIC_ADDR_VALID BIT(31) /* Dynamic Address is Valid */ +#define MASTER_DYNAMIC_ADDR(v) FIELD_PREP(GENMASK(22, 16), v) + +#define HC_CAPABILITIES 0x0c +#define HC_CAP_SG_DC_EN BIT(30) +#define HC_CAP_SG_IBI_EN BIT(29) +#define HC_CAP_SG_CR_EN BIT(28) +#define HC_CAP_MAX_DATA_LENGTH GENMASK(24, 22) +#define HC_CAP_CMD_SIZE GENMASK(21, 20) +#define HC_CAP_DIRECT_COMMANDS_EN BIT(18) +#define HC_CAP_MULTI_LANE_EN BIT(15) +#define HC_CAP_CMD_CCC_DEFBYTE BIT(10) +#define HC_CAP_HDR_BT_EN BIT(8) +#define HC_CAP_HDR_TS_EN BIT(7) +#define HC_CAP_HDR_DDR_EN BIT(6) +#define HC_CAP_NON_CURRENT_MASTER_CAP BIT(5) /* master handoff capable */ +#define HC_CAP_DATA_BYTE_CFG_EN BIT(4) /* endian selection possible */ +#define HC_CAP_AUTO_COMMAND BIT(3) +#define HC_CAP_COMBO_COMMAND BIT(2) + +#define RESET_CONTROL 0x10 +#define BUS_RESET BIT(31) +#define BUS_RESET_TYPE GENMASK(30, 29) +#define IBI_QUEUE_RST BIT(5) +#define RX_FIFO_RST BIT(4) +#define TX_FIFO_RST BIT(3) +#define RESP_QUEUE_RST BIT(2) +#define CMD_QUEUE_RST BIT(1) +#define SOFT_RST BIT(0) /* Core Reset */ + +#define PRESENT_STATE 0x14 +#define STATE_CURRENT_MASTER BIT(2) + +#define INTR_STATUS 0x20 +#define INTR_STATUS_ENABLE 0x24 +#define INTR_SIGNAL_ENABLE 0x28 +#define INTR_FORCE 0x2c +#define INTR_HC_CMD_SEQ_UFLOW_STAT BIT(12) /* Cmd Sequence Underflow */ +#define INTR_HC_SEQ_CANCEL BIT(11) /* HC Cancelled Transaction Sequence */ +#define INTR_HC_INTERNAL_ERR BIT(10) /* HC Internal Error */ + +#define DAT_SECTION 0x30 /* Device Address Table */ +#define DAT_ENTRY_SIZE GENMASK(31, 28) +#define DAT_TABLE_SIZE GENMASK(18, 12) +#define DAT_TABLE_OFFSET GENMASK(11, 0) + +#define DCT_SECTION 0x34 /* Device Characteristics Table */ +#define DCT_ENTRY_SIZE GENMASK(31, 28) +#define DCT_TABLE_INDEX GENMASK(23, 19) +#define DCT_TABLE_SIZE GENMASK(18, 12) +#define DCT_TABLE_OFFSET GENMASK(11, 0) + +#define RING_HEADERS_SECTION 0x38 +#define RING_HEADERS_OFFSET GENMASK(15, 0) + +#define PIO_SECTION 0x3c +#define PIO_REGS_OFFSET GENMASK(15, 0) /* PIO Offset */ + +#define EXT_CAPS_SECTION 0x40 +#define EXT_CAPS_OFFSET GENMASK(15, 0) + +#define IBI_NOTIFY_CTRL 0x58 /* IBI Notify Control */ +#define IBI_NOTIFY_SIR_REJECTED BIT(3) /* Rejected Target Interrupt Request */ +#define IBI_NOTIFY_MR_REJECTED BIT(1) /* Rejected Master Request Control */ +#define IBI_NOTIFY_HJ_REJECTED BIT(0) /* Rejected Hot-Join Control */ + +#define DEV_CTX_BASE_LO 0x60 +#define DEV_CTX_BASE_HI 0x64 + +static inline struct i3c_hci *to_i3c_hci(struct i3c_master_controller *m) +{ + return container_of(m, struct i3c_hci, master); +} + +/** + * i3c_hci_sysdev() - Get the device to use for DMA and system PM + * @dev: Device the HCI controller is bound to + * + * When an IOMMU is enabled, DMA API calls must use the device that IOMMU + * setup was done for. Under PCI enumeration that is the PCI device, not + * the "mipi-i3c-hci" platform device below it. The same device owns + * system PM and wakeup configuration. + * + * Return: @dev's parent if it is a PCI device, otherwise @dev. + */ +struct device *i3c_hci_sysdev(struct device *dev) +{ + return dev->parent && dev_is_pci(dev->parent) ? dev->parent : dev; +} + +static void i3c_hci_set_master_dyn_addr(struct i3c_hci *hci) +{ + reg_write(MASTER_DEVICE_ADDR, + MASTER_DYNAMIC_ADDR(hci->dyn_addr) | MASTER_DYNAMIC_ADDR_VALID); +} + +static int i3c_hci_bus_init(struct i3c_master_controller *m) +{ + struct i3c_hci *hci = to_i3c_hci(m); + struct device *dev = hci->master.dev.parent; + struct i3c_device_info info; + int ret; + + if (hci->cmd == &mipi_i3c_hci_cmd_v1) { + ret = mipi_i3c_hci_dat_v1.init(hci); + if (ret) + return ret; + } + + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + return ret; + hci->dyn_addr = ret; + i3c_hci_set_master_dyn_addr(hci); + memset(&info, 0, sizeof(info)); + info.dyn_addr = hci->dyn_addr; + ret = i3c_master_set_info(m, &info); + if (ret) + return ret; + + hci->ibi_devs = devm_kcalloc(dev, hci->DAT_entries, sizeof(*hci->ibi_devs), GFP_KERNEL); + if (!hci->ibi_devs) + return -ENOMEM; + + ret = hci->io->init(hci); + if (ret) + return ret; + + /* Set RESP_BUF_THLD to 0(n) to get 1(n+1) response */ + if (hci->quirks & HCI_QUIRK_RESP_BUF_THLD) + amd_set_resp_buf_thld(hci); + + scoped_guard(spinlock_irqsave, &hci->lock) + hci->irq_inactive = false; + + /* Enable bus with Hot-Join disabled */ + reg_set(HC_CONTROL, HC_CONTROL_BUS_ENABLE | HC_CONTROL_HOT_JOIN_CTRL); + dev_dbg(&hci->master.dev, "HC_CONTROL = %#x", reg_read(HC_CONTROL)); + + return 0; +} + +/* Bus disable should never fail, so be generous with the timeout */ +#define BUS_DISABLE_TIMEOUT_US (500 * USEC_PER_MSEC) + +static int i3c_hci_bus_disable(struct i3c_hci *hci) +{ + u32 regval; + int ret; + + reg_clear(HC_CONTROL, HC_CONTROL_BUS_ENABLE); + + /* Ensure controller is disabled */ + ret = readx_poll_timeout(reg_read, HC_CONTROL, regval, + !(regval & HC_CONTROL_BUS_ENABLE), 0, BUS_DISABLE_TIMEOUT_US); + if (ret) + dev_err(&hci->master.dev, "%s: Failed to disable bus\n", __func__); + + return ret; +} + +static int i3c_hci_software_reset(struct i3c_hci *hci) +{ + u32 regval; + int ret; + + /* + * SOFT_RST must be clear before we write to it. + * Then we must wait until it clears again. + */ + ret = readx_poll_timeout(reg_read, RESET_CONTROL, regval, + !(regval & SOFT_RST), 0, 10 * USEC_PER_MSEC); + if (ret) { + dev_err(&hci->master.dev, "%s: Software reset stuck\n", __func__); + return ret; + } + + reg_write(RESET_CONTROL, SOFT_RST); + + ret = readx_poll_timeout(reg_read, RESET_CONTROL, regval, + !(regval & SOFT_RST), 0, 10 * USEC_PER_MSEC); + if (ret) { + dev_err(&hci->master.dev, "%s: Software reset failed\n", __func__); + return ret; + } + + return 0; +} + +void i3c_hci_sync_irq_inactive(struct i3c_hci *hci) +{ + struct platform_device *pdev = to_platform_device(hci->master.dev.parent); + int irq = platform_get_irq(pdev, 0); + + reg_write(INTR_SIGNAL_ENABLE, 0x0); + synchronize_irq(irq); + scoped_guard(spinlock_irqsave, &hci->lock) + hci->irq_inactive = true; +} + +static void i3c_hci_bus_cleanup(struct i3c_master_controller *m) +{ + struct i3c_hci *hci = to_i3c_hci(m); + + if (i3c_hci_bus_disable(hci)) + i3c_hci_software_reset(hci); + hci->io->cleanup(hci); +} + +void mipi_i3c_hci_resume(struct i3c_hci *hci) +{ + u32 reg = reg_read(HC_CONTROL); + + reg |= HC_CONTROL_RESUME; + reg &= ~HC_CONTROL_ABORT; + reg_write(HC_CONTROL, reg); +} + +void mipi_i3c_hci_abort(struct i3c_hci *hci) +{ + u32 reg = reg_read(HC_CONTROL); + + reg &= ~HC_CONTROL_RESUME; /* Do not set resume */ + reg |= HC_CONTROL_ABORT; + reg_write(HC_CONTROL, reg); +} + +/* located here rather than pio.c because needed bits are in core reg space */ +void mipi_i3c_hci_pio_reset(struct i3c_hci *hci) +{ + reg_write(RESET_CONTROL, RX_FIFO_RST | TX_FIFO_RST | RESP_QUEUE_RST); +} + +#define ALL_QUEUES_RST (CMD_QUEUE_RST | RESP_QUEUE_RST | RX_FIFO_RST | TX_FIFO_RST | IBI_QUEUE_RST) + +void mipi_i3c_hci_pio_reset_all_queues(struct i3c_hci *hci) +{ + u32 regval; + + reg_write(RESET_CONTROL, ALL_QUEUES_RST); + if (readx_poll_timeout_atomic(reg_read, RESET_CONTROL, regval, + !(regval & ALL_QUEUES_RST), 0, 20)) + dev_err(&hci->master.dev, "%s: Reset queues failed\n", __func__); +} + +/* located here rather than dct.c because needed bits are in core reg space */ +void mipi_i3c_hci_dct_index_reset(struct i3c_hci *hci) +{ + reg_write(DCT_SECTION, FIELD_PREP(DCT_TABLE_INDEX, 0)); +} + +int i3c_hci_process_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) +{ + struct completion *done = xfer[n - 1].completion; + unsigned long timeout = xfer[n - 1].timeout; + unsigned long remaining_timeout = timeout; + long time_taken; + bool started; + int ret; + + xfer[0].started = false; + + ret = hci->io->queue_xfer(hci, xfer, n); + if (ret) + return ret; + + while (!wait_for_completion_timeout(done, remaining_timeout)) { + scoped_guard(spinlock_irqsave, &hci->lock) { + started = xfer[0].started; + time_taken = jiffies - xfer[0].start_jiffies; + } + /* Keep waiting if xfer has not started */ + if (!started) + continue; + /* Recalculate timeout based on actual start time */ + if (time_taken < timeout) { + remaining_timeout = timeout - time_taken; + continue; + } + if (hci->io->dequeue_xfer(hci, xfer, n)) { + dev_err(&hci->master.dev, "%s: timeout error\n", __func__); + return -ETIMEDOUT; + } + return 0; + } + + if (hci->io->handle_error) { + bool error = false; + + for (int i = 0; i < n && !error; i++) + error = RESP_STATUS(xfer[i].response); + if (error) + return hci->io->handle_error(hci, xfer, n); + } + + return 0; +} + +static int i3c_hci_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *ccc) +{ + struct i3c_hci *hci = to_i3c_hci(m); + struct hci_xfer *xfer; + bool raw = !!(hci->quirks & HCI_QUIRK_RAW_CCC); + bool prefixed = raw && !!(ccc->id & I3C_CCC_DIRECT); + unsigned int nxfers = ccc->ndests + prefixed; + DECLARE_COMPLETION_ONSTACK(done); + int i, last, ret = 0; + + dev_dbg(&hci->master.dev, "cmd=%#x rnw=%d ndests=%d data[0].len=%d", + ccc->id, ccc->rnw, ccc->ndests, ccc->dests[0].payload.len); + + xfer = hci_alloc_xfer(nxfers); + if (!xfer) + return -ENOMEM; + + if (prefixed) { + xfer->data = NULL; + xfer->data_len = 0; + xfer->rnw = false; + hci->cmd->prep_ccc(hci, xfer, I3C_BROADCAST_ADDR, + ccc->id, true); + xfer++; + } + + for (i = 0; i < nxfers - prefixed; i++) { + xfer[i].data = ccc->dests[i].payload.data; + xfer[i].data_len = ccc->dests[i].payload.len; + xfer[i].rnw = ccc->rnw; + ret = hci->cmd->prep_ccc(hci, &xfer[i], ccc->dests[i].addr, + ccc->id, raw); + if (ret) + goto out; + xfer[i].cmd_desc[0] |= CMD_0_ROC; + } + last = i - 1; + xfer[last].cmd_desc[0] |= CMD_0_TOC; + xfer[last].completion = &done; + xfer[last].timeout = HZ; + + if (prefixed) + xfer--; + + ret = i3c_hci_process_xfer(hci, xfer, nxfers); + if (ret) + goto out; + for (i = prefixed; i < nxfers; i++) { + if (ccc->rnw) + ccc->dests[i - prefixed].payload.actual_len = + RESP_DATA_LENGTH(xfer[i].response); + switch (RESP_STATUS(xfer[i].response)) { + case RESP_SUCCESS: + continue; + case RESP_ERR_ADDR_HEADER: + case RESP_ERR_NACK: + ccc->err = I3C_ERROR_M2; + fallthrough; + default: + ret = -EIO; + goto out; + } + } + + if (ccc->rnw) + dev_dbg(&hci->master.dev, "got: %*ph", + ccc->dests[0].payload.actual_len, + ccc->dests[0].payload.data); + +out: + hci_free_xfer(xfer, nxfers); + return ret; +} + +static int i3c_hci_enable_hotjoin(struct i3c_master_controller *m) +{ + struct i3c_hci *hci = to_i3c_hci(m); + int ret; + + reg_clear(HC_CONTROL, HC_CONTROL_HOT_JOIN_CTRL); + + /* + * Broadcast Hot_join enable, so that an I3C device that has previously + * had its Hot-Join request NACK'ed knows to try again. + */ + ret = i3c_master_enec_disec_locked(m, I3C_BROADCAST_ADDR, true, I3C_CCC_EVENT_HJ, true); + if (ret) { + reg_set(HC_CONTROL, HC_CONTROL_HOT_JOIN_CTRL); + dev_err(&hci->master.dev, "Hot-Join ENEC CCC failed\n"); + } + + return ret; +} + +static int i3c_hci_disable_hotjoin(struct i3c_master_controller *m) +{ + struct i3c_hci *hci = to_i3c_hci(m); + + reg_set(HC_CONTROL, HC_CONTROL_HOT_JOIN_CTRL); + return 0; +} + +static int i3c_hci_daa(struct i3c_master_controller *m) +{ + struct i3c_hci *hci = to_i3c_hci(m); + int ret; + + ret = hci->cmd->perform_daa(hci); + + if (!hci->hj_init_done) { + hci->hj_init_done = true; + /* + * Enable Hot-Join by default after initial DAA if it does not + * prevent runtime suspend. + */ + if (m->rpm_ibi_allowed && !ret) + m->hotjoin = !i3c_hci_enable_hotjoin(m); + } + + return ret; +} + +static int i3c_hci_i3c_xfers(struct i3c_dev_desc *dev, + struct i3c_xfer *i3c_xfers, int nxfers, + enum i3c_xfer_mode mode) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct hci_xfer *xfer; + DECLARE_COMPLETION_ONSTACK(done); + unsigned int size_limit; + int i, last, ret = 0; + + dev_dbg(&hci->master.dev, "nxfers = %d", nxfers); + + xfer = hci_alloc_xfer(nxfers); + if (!xfer) + return -ENOMEM; + + size_limit = 1U << (16 + FIELD_GET(HC_CAP_MAX_DATA_LENGTH, hci->caps)); + + for (i = 0; i < nxfers; i++) { + xfer[i].data_len = i3c_xfers[i].len; + ret = -EFBIG; + if (xfer[i].data_len >= size_limit) + goto out; + xfer[i].rnw = i3c_xfers[i].rnw; + if (i3c_xfers[i].rnw) { + xfer[i].data = i3c_xfers[i].data.in; + } else { + /* silence the const qualifier warning with a cast */ + xfer[i].data = (void *) i3c_xfers[i].data.out; + } + hci->cmd->prep_i3c_xfer(hci, dev, &xfer[i]); + xfer[i].cmd_desc[0] |= CMD_0_ROC; + } + last = i - 1; + xfer[last].cmd_desc[0] |= CMD_0_TOC; + xfer[last].completion = &done; + xfer[last].timeout = HZ; + + ret = i3c_hci_process_xfer(hci, xfer, nxfers); + if (ret) + goto out; + for (i = 0; i < nxfers; i++) { + if (i3c_xfers[i].rnw) + i3c_xfers[i].len = RESP_DATA_LENGTH(xfer[i].response); + if (RESP_STATUS(xfer[i].response) != RESP_SUCCESS) { + ret = -EIO; + goto out; + } + } + +out: + hci_free_xfer(xfer, nxfers); + return ret; +} + +static int i3c_hci_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *i2c_xfers, int nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct hci_xfer *xfer; + DECLARE_COMPLETION_ONSTACK(done); + int i, last, ret = 0; + + dev_dbg(&hci->master.dev, "nxfers = %d", nxfers); + + xfer = hci_alloc_xfer(nxfers); + if (!xfer) + return -ENOMEM; + + for (i = 0; i < nxfers; i++) { + xfer[i].data = i2c_xfers[i].buf; + xfer[i].data_len = i2c_xfers[i].len; + xfer[i].rnw = i2c_xfers[i].flags & I2C_M_RD; + hci->cmd->prep_i2c_xfer(hci, dev, &xfer[i]); + xfer[i].cmd_desc[0] |= CMD_0_ROC; + } + last = i - 1; + xfer[last].cmd_desc[0] |= CMD_0_TOC; + xfer[last].completion = &done; + xfer[last].timeout = m->i2c.timeout; + + ret = i3c_hci_process_xfer(hci, xfer, nxfers); + if (ret) + goto out; + for (i = 0; i < nxfers; i++) { + if (RESP_STATUS(xfer[i].response) != RESP_SUCCESS) { + ret = -EIO; + goto out; + } + } + +out: + hci_free_xfer(xfer, nxfers); + return ret; +} + +static int i3c_hci_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data; + int ret; + + dev_data = kzalloc_obj(*dev_data); + if (!dev_data) + return -ENOMEM; + if (hci->cmd == &mipi_i3c_hci_cmd_v1) { + ret = mipi_i3c_hci_dat_v1.alloc_entry(hci); + if (ret < 0) { + kfree(dev_data); + return ret; + } + mipi_i3c_hci_dat_v1.set_dynamic_addr(hci, ret, + dev->info.dyn_addr ?: dev->info.static_addr); + dev_data->dat_idx = ret; + } + i3c_dev_set_master_data(dev, dev_data); + return 0; +} + +static int i3c_hci_reattach_i3c_dev(struct i3c_dev_desc *dev, u8 old_dyn_addr) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + + if (hci->cmd == &mipi_i3c_hci_cmd_v1) + mipi_i3c_hci_dat_v1.set_dynamic_addr(hci, dev_data->dat_idx, + dev->info.dyn_addr); + return 0; +} + +static void i3c_hci_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + + i3c_dev_set_master_data(dev, NULL); + if (hci->cmd == &mipi_i3c_hci_cmd_v1) + mipi_i3c_hci_dat_v1.free_entry(hci, dev_data->dat_idx); + kfree(dev_data); +} + +static int i3c_hci_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data; + int ret; + + if (hci->cmd != &mipi_i3c_hci_cmd_v1) + return 0; + dev_data = kzalloc_obj(*dev_data); + if (!dev_data) + return -ENOMEM; + ret = mipi_i3c_hci_dat_v1.alloc_entry(hci); + if (ret < 0) { + kfree(dev_data); + return ret; + } + mipi_i3c_hci_dat_v1.set_static_addr(hci, ret, dev->addr); + mipi_i3c_hci_dat_v1.set_flags(hci, ret, DAT_0_I2C_DEVICE, 0); + dev_data->dat_idx = ret; + i2c_dev_set_master_data(dev, dev_data); + return 0; +} + +static void i3c_hci_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data = i2c_dev_get_master_data(dev); + + if (dev_data) { + i2c_dev_set_master_data(dev, NULL); + if (hci->cmd == &mipi_i3c_hci_cmd_v1) + mipi_i3c_hci_dat_v1.free_entry(hci, dev_data->dat_idx); + kfree(dev_data); + } +} + +static int i3c_hci_request_ibi(struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + unsigned int dat_idx = dev_data->dat_idx; + + if (req->max_payload_len != 0) + mipi_i3c_hci_dat_v1.set_flags(hci, dat_idx, DAT_0_IBI_PAYLOAD, 0); + else + mipi_i3c_hci_dat_v1.clear_flags(hci, dat_idx, DAT_0_IBI_PAYLOAD, 0); + return hci->io->request_ibi(hci, dev, req); +} + +static void __i3c_hci_disable_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + + mipi_i3c_hci_dat_v1.set_flags(hci, dev_data->dat_idx, DAT_0_SIR_REJECT, 0); + scoped_guard(spinlock_irqsave, &hci->lock) + hci->ibi_devs[dev_data->dat_idx] = NULL; +} + +static void i3c_hci_free_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + + /* Must ensure the IBI has been disabled */ + __i3c_hci_disable_ibi(hci, dev); + hci->io->free_ibi(hci, dev); +} + +struct i3c_dev_desc *i3c_hci_addr_to_dev(struct i3c_hci *hci, unsigned int addr) +{ + int dat_idx; + + lockdep_assert_held(&hci->lock); + + for (dat_idx = 0; dat_idx < hci->DAT_entries; dat_idx++) { + struct i3c_dev_desc *dev = hci->ibi_devs[dat_idx]; + + if (dev && dev->info.dyn_addr == addr) + return dev; + } + return NULL; +} + +static int i3c_hci_enable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + + mipi_i3c_hci_dat_v1.clear_flags(hci, dev_data->dat_idx, DAT_0_SIR_REJECT, 0); + scoped_guard(spinlock_irqsave, &hci->lock) + hci->ibi_devs[dev_data->dat_idx] = dev; + return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); +} + +static int i3c_hci_disable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + + __i3c_hci_disable_ibi(hci, dev); + /* + * The DAT entry is now set to NACK and DISEC this target's IBIs, so + * the IBI teardown can proceed even if DISEC below fails, so ignore + * errors. + */ + i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); + return 0; +} + +static void i3c_hci_recycle_ibi_slot(struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct i3c_hci *hci = to_i3c_hci(m); + + hci->io->recycle_ibi_slot(hci, dev, slot); +} + +static const struct i3c_master_controller_ops i3c_hci_ops = { + .bus_init = i3c_hci_bus_init, + .bus_cleanup = i3c_hci_bus_cleanup, + .do_daa = i3c_hci_daa, + .send_ccc_cmd = i3c_hci_send_ccc_cmd, + .i3c_xfers = i3c_hci_i3c_xfers, + .i2c_xfers = i3c_hci_i2c_xfers, + .attach_i3c_dev = i3c_hci_attach_i3c_dev, + .reattach_i3c_dev = i3c_hci_reattach_i3c_dev, + .detach_i3c_dev = i3c_hci_detach_i3c_dev, + .attach_i2c_dev = i3c_hci_attach_i2c_dev, + .detach_i2c_dev = i3c_hci_detach_i2c_dev, + .request_ibi = i3c_hci_request_ibi, + .free_ibi = i3c_hci_free_ibi, + .enable_ibi = i3c_hci_enable_ibi, + .disable_ibi = i3c_hci_disable_ibi, + .recycle_ibi_slot = i3c_hci_recycle_ibi_slot, + .enable_hotjoin = i3c_hci_enable_hotjoin, + .disable_hotjoin = i3c_hci_disable_hotjoin, +}; + +static irqreturn_t i3c_hci_irq_handler(int irq, void *dev_id) +{ + struct i3c_hci *hci = dev_id; + irqreturn_t result = IRQ_NONE; + u32 val; + + guard(spinlock)(&hci->lock); + + /* + * The IRQ can be shared, so the handler may be called when the IRQ is + * due to a different device. That could happen when runtime suspended, + * so exit immediately if IRQs are not expected for this device. + */ + if (hci->irq_inactive) + return IRQ_NONE; + + val = reg_read(INTR_STATUS); + reg_write(INTR_STATUS, val); + dev_dbg(&hci->master.dev, "INTR_STATUS %#x", val); + + if (val) + result = IRQ_HANDLED; + + if (val & INTR_HC_SEQ_CANCEL) { + dev_dbg(&hci->master.dev, + "Host Controller Cancelled Transaction Sequence\n"); + val &= ~INTR_HC_SEQ_CANCEL; + } + if (val & INTR_HC_INTERNAL_ERR) { + dev_err(&hci->master.dev, "Host Controller Internal Error\n"); + val &= ~INTR_HC_INTERNAL_ERR; + hci->recovery_needed = true; + } + + if (val) + dev_warn_once(&hci->master.dev, + "unexpected INTR_STATUS %#x\n", val); + + if (hci->io->irq_handler(hci)) + result = IRQ_HANDLED; + + return result; +} + +static inline bool is_version_1_1_or_newer(struct i3c_hci *hci) +{ + return hci->version_major > 1 || (hci->version_major == 1 && hci->version_minor > 0); +} + +static int i3c_hci_set_io_mode(struct i3c_hci *hci, bool dma) +{ + bool pio_mode; + + if (dma) + reg_clear(HC_CONTROL, HC_CONTROL_PIO_MODE); + else + reg_set(HC_CONTROL, HC_CONTROL_PIO_MODE); + + if (!is_version_1_1_or_newer(hci)) + return 0; + + pio_mode = reg_read(HC_CONTROL) & HC_CONTROL_PIO_MODE; + if ((dma && pio_mode) || (!dma && !pio_mode)) { + dev_err(&hci->master.dev, "%s mode is stuck\n", pio_mode ? "PIO" : "DMA"); + return -EIO; + } + + return 0; +} + +static int i3c_hci_reset_and_init(struct i3c_hci *hci) +{ + u32 regval; + int ret; + + ret = i3c_hci_software_reset(hci); + if (ret) + return -ENXIO; + + /* Disable all interrupts */ + reg_write(INTR_SIGNAL_ENABLE, 0x0); + /* + * Only allow bit 31:10 signal updates because + * Bit 0:9 are reserved in IP version >= 0.8 + * Bit 0:5 are defined in IP version < 0.8 but not handled by PIO code + */ + reg_write(INTR_STATUS_ENABLE, GENMASK(31, 10)); + + /* Make sure our data ordering fits the host's */ + regval = reg_read(HC_CONTROL); + if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN)) { + if (!(regval & HC_CONTROL_DATA_BIG_ENDIAN)) { + regval |= HC_CONTROL_DATA_BIG_ENDIAN; + reg_write(HC_CONTROL, regval); + regval = reg_read(HC_CONTROL); + if (!(regval & HC_CONTROL_DATA_BIG_ENDIAN)) { + dev_err(&hci->master.dev, "cannot set BE mode\n"); + return -EOPNOTSUPP; + } + } + } else { + if (regval & HC_CONTROL_DATA_BIG_ENDIAN) { + regval &= ~HC_CONTROL_DATA_BIG_ENDIAN; + reg_write(HC_CONTROL, regval); + regval = reg_read(HC_CONTROL); + if (regval & HC_CONTROL_DATA_BIG_ENDIAN) { + dev_err(&hci->master.dev, "cannot clear BE mode\n"); + return -EOPNOTSUPP; + } + } + } + + if (hci->io) { + ret = i3c_hci_set_io_mode(hci, hci->io == &mipi_i3c_hci_dma); + } else { + /* Try activating DMA operations first */ + if (hci->RHS_regs) { + ret = i3c_hci_set_io_mode(hci, true); + if (!ret) { + hci->io = &mipi_i3c_hci_dma; + dev_dbg(&hci->master.dev, "Using DMA\n"); + } + } + + /* If no DMA, try PIO */ + if (!hci->io && hci->PIO_regs) { + ret = i3c_hci_set_io_mode(hci, false); + if (!ret) { + hci->io = &mipi_i3c_hci_pio; + dev_dbg(&hci->master.dev, "Using PIO\n"); + } + } + + if (!hci->io) { + dev_err(&hci->master.dev, "neither DMA nor PIO can be used\n"); + ret = ret ?: -EINVAL; + } + } + if (ret) + return ret; + + /* Configure OD and PP timings for AMD platforms */ + if (hci->quirks & HCI_QUIRK_OD_PP_TIMING) + amd_set_od_pp_timing(hci); + + return 0; +} + +int i3c_hci_rpm_suspend(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + /* Fall back to software reset to disable the bus */ + if (i3c_hci_bus_disable(hci)) + i3c_hci_software_reset(hci); + + hci->io->suspend(hci); + + return 0; +} +EXPORT_SYMBOL_GPL(i3c_hci_rpm_suspend); + +static int i3c_hci_do_reset_and_restore(struct i3c_hci *hci) +{ + int ret; + + ret = i3c_hci_reset_and_init(hci); + if (ret) + return -EIO; + + i3c_hci_set_master_dyn_addr(hci); + + mipi_i3c_hci_dat_v1.restore(hci); + + hci->io->resume(hci); + + scoped_guard(spinlock_irqsave, &hci->lock) + hci->irq_inactive = false; + + /* Enable bus, restoring hot-join state */ + reg_set(HC_CONTROL, + HC_CONTROL_BUS_ENABLE | (hci->master.hotjoin ? 0 : HC_CONTROL_HOT_JOIN_CTRL)); + + return 0; +} + +int i3c_hci_reset_and_restore(struct i3c_hci *hci) +{ + i3c_hci_bus_disable(hci); + + hci->io->suspend(hci); + + return i3c_hci_do_reset_and_restore(hci); +} + +int i3c_hci_rpm_resume(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + return i3c_hci_do_reset_and_restore(hci); +} +EXPORT_SYMBOL_GPL(i3c_hci_rpm_resume); + +static int i3c_hci_runtime_suspend(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + if (hci->quirks & HCI_QUIRK_RPM_PARENT_MANAGED) + return 0; + + return i3c_hci_rpm_suspend(dev); +} + +static int i3c_hci_runtime_resume(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + if (hci->quirks & HCI_QUIRK_RPM_PARENT_MANAGED) + return 0; + + return i3c_hci_rpm_resume(dev); +} + +static int i3c_hci_suspend(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + if (!(hci->quirks & HCI_QUIRK_RPM_ALLOWED)) + return 0; + + return pm_runtime_force_suspend(dev); +} + +static int i3c_hci_resume_common(struct device *dev, bool rstdaa) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + int ret; + + if (!(hci->quirks & HCI_QUIRK_RPM_ALLOWED)) + return 0; + + ret = pm_runtime_force_resume(dev); + if (ret) + return ret; + + ret = i3c_master_do_daa_ext(&hci->master, rstdaa); + if (ret) + dev_err(dev, "Dynamic Address Assignment failed on resume, error %d\n", ret); + + /* + * I3C devices may have retained their dynamic address anyway. Do not + * fail the resume because of DAA error. + */ + return 0; +} + +static int i3c_hci_resume(struct device *dev) +{ + return i3c_hci_resume_common(dev, false); +} + +static int i3c_hci_restore(struct device *dev) +{ + return i3c_hci_resume_common(dev, true); +} + +static void i3c_hci_rpm_enable(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + pm_runtime_set_autosuspend_delay(dev, DEFAULT_AUTOSUSPEND_DELAY_MS); + pm_runtime_use_autosuspend(dev); + devm_pm_runtime_set_active_enabled(dev); + + hci->master.rpm_allowed = true; +} + +static int i3c_hci_init(struct i3c_hci *hci) +{ + bool size_in_dwords; + u32 regval, offset; + int ret; + + /* Validate HCI hardware version */ + regval = reg_read(HCI_VERSION); + hci->version_major = (regval >> 8) & 0xf; + hci->version_minor = (regval >> 4) & 0xf; + hci->revision = regval & 0xf; + dev_notice(&hci->master.dev, "MIPI I3C HCI v%u.%u r%02u\n", + hci->version_major, hci->version_minor, hci->revision); + /* known versions */ + switch (regval & ~0xf) { + case 0x100: /* version 1.0 */ + case 0x110: /* version 1.1 */ + case 0x120: /* version 1.2 */ + case 0x200: /* version 2.0 */ + break; + default: + dev_err(&hci->master.dev, "unsupported HCI version\n"); + return -EPROTONOSUPPORT; + } + + hci->caps = reg_read(HC_CAPABILITIES); + dev_dbg(&hci->master.dev, "caps = %#x", hci->caps); + + size_in_dwords = hci->version_major < 1 || + (hci->version_major == 1 && hci->version_minor < 1); + + regval = reg_read(DAT_SECTION); + offset = FIELD_GET(DAT_TABLE_OFFSET, regval); + hci->DAT_regs = offset ? hci->base_regs + offset : NULL; + hci->DAT_entries = FIELD_GET(DAT_TABLE_SIZE, regval); + hci->DAT_entry_size = FIELD_GET(DAT_ENTRY_SIZE, regval) ? 0 : 8; + if (size_in_dwords) + hci->DAT_entries = 4 * hci->DAT_entries / hci->DAT_entry_size; + dev_dbg(&hci->master.dev, "DAT: %u %u-bytes entries at offset %#x\n", + hci->DAT_entries, hci->DAT_entry_size, offset); + + regval = reg_read(DCT_SECTION); + offset = FIELD_GET(DCT_TABLE_OFFSET, regval); + hci->DCT_regs = offset ? hci->base_regs + offset : NULL; + hci->DCT_entries = FIELD_GET(DCT_TABLE_SIZE, regval); + hci->DCT_entry_size = FIELD_GET(DCT_ENTRY_SIZE, regval) ? 0 : 16; + if (size_in_dwords) + hci->DCT_entries = 4 * hci->DCT_entries / hci->DCT_entry_size; + dev_dbg(&hci->master.dev, "DCT: %u %u-bytes entries at offset %#x\n", + hci->DCT_entries, hci->DCT_entry_size, offset); + + regval = reg_read(RING_HEADERS_SECTION); + offset = FIELD_GET(RING_HEADERS_OFFSET, regval); + hci->RHS_regs = offset ? hci->base_regs + offset : NULL; + dev_dbg(&hci->master.dev, "Ring Headers at offset %#x\n", offset); + + regval = reg_read(PIO_SECTION); + offset = FIELD_GET(PIO_REGS_OFFSET, regval); + hci->PIO_regs = offset ? hci->base_regs + offset : NULL; + dev_dbg(&hci->master.dev, "PIO section at offset %#x\n", offset); + + regval = reg_read(EXT_CAPS_SECTION); + offset = FIELD_GET(EXT_CAPS_OFFSET, regval); + hci->EXTCAPS_regs = offset ? hci->base_regs + offset : NULL; + dev_dbg(&hci->master.dev, "Extended Caps at offset %#x\n", offset); + + ret = i3c_hci_parse_ext_caps(hci); + if (ret) + return ret; + + /* Select our command descriptor model */ + switch (FIELD_GET(HC_CAP_CMD_SIZE, hci->caps)) { + case 0: + hci->cmd = &mipi_i3c_hci_cmd_v1; + break; + case 1: + hci->cmd = &mipi_i3c_hci_cmd_v2; + break; + default: + dev_err(&hci->master.dev, "wrong CMD_SIZE capability value\n"); + return -EINVAL; + } + + /* Quirk for HCI_QUIRK_PIO_MODE on AMD platforms */ + if (hci->quirks & HCI_QUIRK_PIO_MODE) + hci->RHS_regs = NULL; + + return i3c_hci_reset_and_init(hci); +} + +static int i3c_hci_probe(struct platform_device *pdev) +{ + const struct mipi_i3c_hci_platform_data *pdata = pdev->dev.platform_data; + struct clk_bulk_data *clks; + struct i3c_hci *hci; + int irq, ret; + + hci = devm_kzalloc(&pdev->dev, sizeof(*hci), GFP_KERNEL); + if (!hci) + return -ENOMEM; + + spin_lock_init(&hci->lock); + mutex_init(&hci->control_mutex); + init_waitqueue_head(&hci->enqueue_wait_queue); + + /* + * Multi-bus instances share the same MMIO address range, but not + * necessarily in separate contiguous sub-ranges. To avoid overlapping + * mappings, provide base_regs from the parent mapping. + */ + if (pdata) + hci->base_regs = pdata->base_regs; + + if (!hci->base_regs) { + hci->base_regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(hci->base_regs)) + return PTR_ERR(hci->base_regs); + } + + platform_set_drvdata(pdev, hci); + /* temporary for dev_printk's, to be replaced in i3c_master_register */ + hci->master.dev.init_name = dev_name(&pdev->dev); + + hci->quirks = (unsigned long)device_get_match_data(&pdev->dev); + if (!hci->quirks && platform_get_device_id(pdev)) + hci->quirks = platform_get_device_id(pdev)->driver_data; + + ret = devm_clk_bulk_get_all_enabled(&pdev->dev, &clks); + if (ret < 0) + return dev_err_probe(&pdev->dev, ret, + "Failed to get clocks\n"); + + ret = i3c_hci_init(hci); + if (ret) + return ret; + + hci->irq_inactive = true; + + irq = platform_get_irq(pdev, 0); + ret = devm_request_irq(&pdev->dev, irq, i3c_hci_irq_handler, + IRQF_SHARED, NULL, hci); + if (ret) + return ret; + + if (hci->quirks & HCI_QUIRK_RPM_ALLOWED) + i3c_hci_rpm_enable(&pdev->dev); + + if (hci->quirks & HCI_QUIRK_RPM_IBI_ALLOWED) + hci->master.rpm_ibi_allowed = true; + + if (device_can_wakeup(i3c_hci_sysdev(&pdev->dev))) + hci->master.ibi_wakeup = true; + + return i3c_master_register(&hci->master, &pdev->dev, &i3c_hci_ops, false); +} + +static void i3c_hci_remove(struct platform_device *pdev) +{ + struct i3c_hci *hci = platform_get_drvdata(pdev); + + i3c_master_unregister(&hci->master); +} + +static const __maybe_unused struct of_device_id i3c_hci_of_match[] = { + { .compatible = "mipi-i3c-hci", }, + { .compatible = "microchip,sama7d65-i3c-hci", + .data = (void *)(ulong)(HCI_QUIRK_PIO_MODE | HCI_QUIRK_OD_PP_TIMING | + HCI_QUIRK_RESP_BUF_THLD) }, + {}, +}; +MODULE_DEVICE_TABLE(of, i3c_hci_of_match); + +static const struct acpi_device_id i3c_hci_acpi_match[] = { + { "AMDI5017", HCI_QUIRK_PIO_MODE | HCI_QUIRK_OD_PP_TIMING | HCI_QUIRK_RESP_BUF_THLD }, + {} +}; +MODULE_DEVICE_TABLE(acpi, i3c_hci_acpi_match); + +static const struct platform_device_id i3c_hci_driver_ids[] = { + { + .name = "intel-lpss-i3c", + .driver_data = HCI_QUIRK_RPM_ALLOWED | + HCI_QUIRK_RPM_IBI_ALLOWED | + HCI_QUIRK_RPM_PARENT_MANAGED | + HCI_QUIRK_DMA_ABORT_REQUIRES_PIO_RESET | + HCI_QUIRK_DMA_REQUIRES_HC_ABORT, + }, + { + .name = "amd-pt-i3c-hci", + .driver_data = HCI_QUIRK_RPM_ALLOWED, + }, + { /* sentinel */ } +}; +MODULE_DEVICE_TABLE(platform, i3c_hci_driver_ids); + +static const struct dev_pm_ops i3c_hci_pm_ops = { + .suspend = pm_sleep_ptr(i3c_hci_suspend), + .resume = pm_sleep_ptr(i3c_hci_resume), + .freeze = pm_sleep_ptr(i3c_hci_suspend), + .thaw = pm_sleep_ptr(i3c_hci_resume), + .poweroff = pm_sleep_ptr(i3c_hci_suspend), + .restore = pm_sleep_ptr(i3c_hci_restore), + RUNTIME_PM_OPS(i3c_hci_runtime_suspend, i3c_hci_runtime_resume, NULL) +}; + +static struct platform_driver i3c_hci_driver = { + .probe = i3c_hci_probe, + .remove = i3c_hci_remove, + .id_table = i3c_hci_driver_ids, + .driver = { + .name = "mipi-i3c-hci", + .of_match_table = of_match_ptr(i3c_hci_of_match), + .acpi_match_table = i3c_hci_acpi_match, + .pm = pm_ptr(&i3c_hci_pm_ops), + }, +}; +module_platform_driver(i3c_hci_driver); +MODULE_ALIAS("platform:mipi-i3c-hci"); + +MODULE_AUTHOR("Nicolas Pitre <npitre@baylibre.com>"); +MODULE_DESCRIPTION("MIPI I3C HCI driver"); +MODULE_LICENSE("Dual BSD/GPL"); diff --git a/drivers/i3c/master/mipi-i3c-hci/dat.h b/drivers/i3c/master/mipi-i3c-hci/dat.h new file mode 100644 index 000000000..6881f19da --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/dat.h @@ -0,0 +1,32 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Common DAT related stuff + */ + +#ifndef DAT_H +#define DAT_H + +/* Global DAT flags */ +#define DAT_0_I2C_DEVICE W0_BIT_(31) +#define DAT_0_SIR_REJECT W0_BIT_(13) +#define DAT_0_IBI_PAYLOAD W0_BIT_(12) + +struct hci_dat_ops { + int (*init)(struct i3c_hci *hci); + int (*alloc_entry)(struct i3c_hci *hci); + void (*free_entry)(struct i3c_hci *hci, unsigned int dat_idx); + void (*set_dynamic_addr)(struct i3c_hci *hci, unsigned int dat_idx, u8 addr); + void (*set_static_addr)(struct i3c_hci *hci, unsigned int dat_idx, u8 addr); + void (*set_flags)(struct i3c_hci *hci, unsigned int dat_idx, u32 w0, u32 w1); + void (*clear_flags)(struct i3c_hci *hci, unsigned int dat_idx, u32 w0, u32 w1); + int (*get_index)(struct i3c_hci *hci, u8 address); + void (*restore)(struct i3c_hci *hci); +}; + +extern const struct hci_dat_ops mipi_i3c_hci_dat_v1; + +#endif diff --git a/drivers/i3c/master/mipi-i3c-hci/dat_v1.c b/drivers/i3c/master/mipi-i3c-hci/dat_v1.c new file mode 100644 index 000000000..852966aa2 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/dat_v1.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + */ + +#include <linux/bitfield.h> +#include <linux/bitmap.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/io.h> + +#include "hci.h" +#include "dat.h" + +/* + * Device Address Table Structure + */ + +#define DAT_1_AUTOCMD_HDR_CODE W1_MASK(58, 51) +#define DAT_1_AUTOCMD_MODE W1_MASK(50, 48) +#define DAT_1_AUTOCMD_VALUE W1_MASK(47, 40) +#define DAT_1_AUTOCMD_MASK W1_MASK(39, 32) +/* DAT_0_I2C_DEVICE W0_BIT_(31) */ +#define DAT_0_DEV_NACK_RETRY_CNT W0_MASK(30, 29) +#define DAT_0_RING_ID W0_MASK(28, 26) +#define DAT_0_DYNADDR_PARITY W0_BIT_(23) +#define DAT_0_DYNAMIC_ADDRESS W0_MASK(22, 16) +#define DAT_0_TS W0_BIT_(15) +#define DAT_0_MR_REJECT W0_BIT_(14) +/* DAT_0_SIR_REJECT W0_BIT_(13) */ +/* DAT_0_IBI_PAYLOAD W0_BIT_(12) */ +#define DAT_0_STATIC_ADDRESS W0_MASK(6, 0) + +#define dat_w0_read(i) hci->DAT[i].w0 +#define dat_w1_read(i) hci->DAT[i].w1 +#define dat_w0_write(i, v) hci_dat_w0_write(hci, i, v) +#define dat_w1_write(i, v) hci_dat_w1_write(hci, i, v) + +static inline void hci_dat_w0_write(struct i3c_hci *hci, int i, u32 v) +{ + hci->DAT[i].w0 = v; + writel(v, hci->DAT_regs + i * 8); +} + +static inline void hci_dat_w1_write(struct i3c_hci *hci, int i, u32 v) +{ + hci->DAT[i].w1 = v; + writel(v, hci->DAT_regs + i * 8 + 4); +} + +static int hci_dat_v1_init(struct i3c_hci *hci) +{ + struct device *dev = hci->master.dev.parent; + unsigned int dat_idx; + + if (!hci->DAT_regs) { + dev_err(&hci->master.dev, + "only DAT in register space is supported at the moment\n"); + return -EOPNOTSUPP; + } + if (hci->DAT_entry_size != 8) { + dev_err(&hci->master.dev, + "only 8-bytes DAT entries are supported at the moment\n"); + return -EOPNOTSUPP; + } + + if (!hci->DAT) { + hci->DAT = devm_kcalloc(dev, hci->DAT_entries, hci->DAT_entry_size, GFP_KERNEL); + if (!hci->DAT) + return -ENOMEM; + } + + if (!hci->DAT_data) { + /* use a bitmap for faster free slot search */ + hci->DAT_data = devm_bitmap_zalloc(dev, hci->DAT_entries, GFP_KERNEL); + if (!hci->DAT_data) + return -ENOMEM; + + /* clear them */ + for (dat_idx = 0; dat_idx < hci->DAT_entries; dat_idx++) { + dat_w0_write(dat_idx, 0); + dat_w1_write(dat_idx, 0); + } + } + + return 0; +} + +static int hci_dat_v1_alloc_entry(struct i3c_hci *hci) +{ + unsigned int dat_idx; + int ret; + + if (!hci->DAT_data) { + ret = hci_dat_v1_init(hci); + if (ret) + return ret; + } + dat_idx = find_first_zero_bit(hci->DAT_data, hci->DAT_entries); + if (dat_idx >= hci->DAT_entries) + return -ENOENT; + __set_bit(dat_idx, hci->DAT_data); + + /* default flags */ + dat_w0_write(dat_idx, DAT_0_SIR_REJECT | DAT_0_MR_REJECT); + + return dat_idx; +} + +static void hci_dat_v1_free_entry(struct i3c_hci *hci, unsigned int dat_idx) +{ + dat_w0_write(dat_idx, 0); + dat_w1_write(dat_idx, 0); + if (hci->DAT_data) + __clear_bit(dat_idx, hci->DAT_data); +} + +static void hci_dat_v1_set_dynamic_addr(struct i3c_hci *hci, + unsigned int dat_idx, u8 address) +{ + u32 dat_w0; + + dat_w0 = dat_w0_read(dat_idx); + dat_w0 &= ~(DAT_0_DYNAMIC_ADDRESS | DAT_0_DYNADDR_PARITY); + dat_w0 |= FIELD_PREP(DAT_0_DYNAMIC_ADDRESS, address) | + (parity8(address) ? 0 : DAT_0_DYNADDR_PARITY); + dat_w0_write(dat_idx, dat_w0); +} + +static void hci_dat_v1_set_static_addr(struct i3c_hci *hci, + unsigned int dat_idx, u8 address) +{ + u32 dat_w0; + + dat_w0 = dat_w0_read(dat_idx); + dat_w0 &= ~DAT_0_STATIC_ADDRESS; + dat_w0 |= FIELD_PREP(DAT_0_STATIC_ADDRESS, address); + dat_w0_write(dat_idx, dat_w0); +} + +static void hci_dat_v1_set_flags(struct i3c_hci *hci, unsigned int dat_idx, + u32 w0_flags, u32 w1_flags) +{ + u32 dat_w0, dat_w1; + + dat_w0 = dat_w0_read(dat_idx); + dat_w1 = dat_w1_read(dat_idx); + dat_w0 |= w0_flags; + dat_w1 |= w1_flags; + dat_w0_write(dat_idx, dat_w0); + dat_w1_write(dat_idx, dat_w1); +} + +static void hci_dat_v1_clear_flags(struct i3c_hci *hci, unsigned int dat_idx, + u32 w0_flags, u32 w1_flags) +{ + u32 dat_w0, dat_w1; + + dat_w0 = dat_w0_read(dat_idx); + dat_w1 = dat_w1_read(dat_idx); + dat_w0 &= ~w0_flags; + dat_w1 &= ~w1_flags; + dat_w0_write(dat_idx, dat_w0); + dat_w1_write(dat_idx, dat_w1); +} + +static int hci_dat_v1_get_index(struct i3c_hci *hci, u8 dev_addr) +{ + unsigned int dat_idx; + u32 dat_w0; + + for_each_set_bit(dat_idx, hci->DAT_data, hci->DAT_entries) { + dat_w0 = dat_w0_read(dat_idx); + if (FIELD_GET(DAT_0_DYNAMIC_ADDRESS, dat_w0) == dev_addr) + return dat_idx; + } + + return -ENODEV; +} + +static void hci_dat_v1_restore(struct i3c_hci *hci) +{ + for (int i = 0; i < hci->DAT_entries; i++) { + writel(hci->DAT[i].w0, hci->DAT_regs + i * 8); + writel(hci->DAT[i].w1, hci->DAT_regs + i * 8 + 4); + } +} + +const struct hci_dat_ops mipi_i3c_hci_dat_v1 = { + .init = hci_dat_v1_init, + .alloc_entry = hci_dat_v1_alloc_entry, + .free_entry = hci_dat_v1_free_entry, + .set_dynamic_addr = hci_dat_v1_set_dynamic_addr, + .set_static_addr = hci_dat_v1_set_static_addr, + .set_flags = hci_dat_v1_set_flags, + .clear_flags = hci_dat_v1_clear_flags, + .get_index = hci_dat_v1_get_index, + .restore = hci_dat_v1_restore, +}; diff --git a/drivers/i3c/master/mipi-i3c-hci/dct.h b/drivers/i3c/master/mipi-i3c-hci/dct.h new file mode 100644 index 000000000..1028e0b40 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/dct.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Common DCT related stuff + */ + +#ifndef DCT_H +#define DCT_H + +void i3c_hci_dct_get_val(struct i3c_hci *hci, unsigned int dct_idx, + u64 *pid, unsigned int *dcr, unsigned int *bcr); + +#endif diff --git a/drivers/i3c/master/mipi-i3c-hci/dct_v1.c b/drivers/i3c/master/mipi-i3c-hci/dct_v1.c new file mode 100644 index 000000000..acfd4d60f --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/dct_v1.c @@ -0,0 +1,36 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + */ + +#include <linux/device.h> +#include <linux/bitfield.h> +#include <linux/i3c/master.h> +#include <linux/io.h> + +#include "hci.h" +#include "dct.h" + +/* + * Device Characteristic Table + */ + +void i3c_hci_dct_get_val(struct i3c_hci *hci, unsigned int dct_idx, + u64 *pid, unsigned int *dcr, unsigned int *bcr) +{ + void __iomem *reg = hci->DCT_regs + dct_idx * 4 * 4; + u32 dct_entry_data[4]; + unsigned int i; + + for (i = 0; i < 4; i++) { + dct_entry_data[i] = readl(reg); + reg += 4; + } + + *pid = ((u64)dct_entry_data[0]) << (47 - 32 + 1) | + FIELD_GET(W1_MASK(47, 32), dct_entry_data[1]); + *dcr = FIELD_GET(W2_MASK(71, 64), dct_entry_data[2]); + *bcr = FIELD_GET(W2_MASK(79, 72), dct_entry_data[2]); +} diff --git a/drivers/i3c/master/mipi-i3c-hci/dma.c b/drivers/i3c/master/mipi-i3c-hci/dma.c new file mode 100644 index 000000000..7c2b20474 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/dma.c @@ -0,0 +1,1079 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Note: The I3C HCI v2.0 spec is still in flux. The IBI support is based on + * v1.x of the spec and v2.0 will likely be split out. + */ + +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/device.h> +#include <linux/dma-mapping.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/io.h> + +#include "hci.h" +#include "cmd.h" +#include "ibi.h" + +/* + * Software Parameter Values (somewhat arb itrary for now). + * Some of them could be determined at run time eventually. + */ + +#define XFER_RINGS 1 /* max: 8 */ +#define XFER_RING_ENTRIES 255 /* max: 255 */ + +#define IBI_RINGS 1 /* max: 8 */ +#define IBI_STATUS_RING_ENTRIES 32 /* max: 255 */ +#define IBI_CHUNK_CACHELINES 1 /* max: 256 bytes equivalent */ +#define IBI_CHUNK_POOL_SIZE 128 /* max: 1023 */ + +/* + * Ring Header Preamble + */ + +#define rhs_reg_read(r) readl(hci->RHS_regs + (RHS_##r)) +#define rhs_reg_write(r, v) writel(v, hci->RHS_regs + (RHS_##r)) + +#define RHS_CONTROL 0x00 +#define PREAMBLE_SIZE GENMASK(31, 24) /* Preamble Section Size */ +#define HEADER_SIZE GENMASK(23, 16) /* Ring Header Size */ +#define MAX_HEADER_COUNT_CAP GENMASK(7, 4) /* HC Max Header Count */ +#define MAX_HEADER_COUNT GENMASK(3, 0) /* Driver Max Header Count */ + +#define RHS_RHn_OFFSET(n) (0x04 + (n)*4) + +/* + * Ring Header (Per-Ring Bundle) + */ + +#define rh_reg_read(r) readl(rh->regs + (RH_##r)) +#define rh_reg_write(r, v) writel(v, rh->regs + (RH_##r)) + +#define RH_CR_SETUP 0x00 /* Command/Response Ring */ +#define CR_XFER_STRUCT_SIZE GENMASK(31, 24) +#define CR_RESP_STRUCT_SIZE GENMASK(23, 16) +#define CR_RING_SIZE GENMASK(8, 0) + +#define RH_IBI_SETUP 0x04 +#define IBI_STATUS_STRUCT_SIZE GENMASK(31, 24) +#define IBI_STATUS_RING_SIZE GENMASK(23, 16) +#define IBI_DATA_CHUNK_SIZE GENMASK(12, 10) +#define IBI_DATA_CHUNK_COUNT GENMASK(9, 0) + +#define RH_CHUNK_CONTROL 0x08 + +#define RH_INTR_STATUS 0x10 +#define RH_INTR_STATUS_ENABLE 0x14 +#define RH_INTR_SIGNAL_ENABLE 0x18 +#define RH_INTR_FORCE 0x1c +#define INTR_IBI_READY BIT(12) +#define INTR_TRANSFER_COMPLETION BIT(11) +#define INTR_RING_OP BIT(10) +#define INTR_TRANSFER_ERR BIT(9) +#define INTR_IBI_RING_FULL BIT(6) +#define INTR_TRANSFER_ABORT BIT(5) + +#define RH_RING_STATUS 0x20 +#define RING_STATUS_LOCKED BIT(3) +#define RING_STATUS_ABORTED BIT(2) +#define RING_STATUS_RUNNING BIT(1) +#define RING_STATUS_ENABLED BIT(0) + +#define RH_RING_CONTROL 0x24 +#define RING_CTRL_ABORT BIT(2) +#define RING_CTRL_RUN_STOP BIT(1) +#define RING_CTRL_ENABLE BIT(0) + +#define RH_RING_OPERATION1 0x28 +#define RING_OP1_IBI_DEQ_PTR GENMASK(23, 16) +#define RING_OP1_CR_SW_DEQ_PTR GENMASK(15, 8) +#define RING_OP1_CR_ENQ_PTR GENMASK(7, 0) + +#define RH_RING_OPERATION2 0x2c +#define RING_OP2_IBI_ENQ_PTR GENMASK(23, 16) +#define RING_OP2_CR_DEQ_PTR GENMASK(7, 0) + +#define RH_CMD_RING_BASE_LO 0x30 +#define RH_CMD_RING_BASE_HI 0x34 +#define RH_RESP_RING_BASE_LO 0x38 +#define RH_RESP_RING_BASE_HI 0x3c +#define RH_IBI_STATUS_RING_BASE_LO 0x40 +#define RH_IBI_STATUS_RING_BASE_HI 0x44 +#define RH_IBI_DATA_RING_BASE_LO 0x48 +#define RH_IBI_DATA_RING_BASE_HI 0x4c + +#define RH_CMD_RING_SG 0x50 /* Ring Scatter Gather Support */ +#define RH_RESP_RING_SG 0x54 +#define RH_IBI_STATUS_RING_SG 0x58 +#define RH_IBI_DATA_RING_SG 0x5c +#define RING_SG_BLP BIT(31) /* Buffer Vs. List Pointer */ +#define RING_SG_LIST_SIZE GENMASK(15, 0) + +/* + * Data Buffer Descriptor (in memory) + */ + +#define DATA_BUF_BLP BIT(31) /* Buffer Vs. List Pointer */ +#define DATA_BUF_IOC BIT(30) /* Interrupt on Completion */ +#define DATA_BUF_BLOCK_SIZE GENMASK(15, 0) + +struct hci_rh_data { + void __iomem *regs; + void *xfer, *resp, *ibi_status, *ibi_data; + dma_addr_t xfer_dma, resp_dma, ibi_status_dma, ibi_data_dma; + unsigned int xfer_entries, ibi_status_entries, ibi_chunks_total; + unsigned int xfer_struct_sz, resp_struct_sz, ibi_status_sz, ibi_chunk_sz; + unsigned int xfer_alloc_sz, done_ptr, ibi_chunk_ptr, xfer_space; + struct hci_xfer **src_xfers; + struct completion op_done; +}; + +struct hci_rings_data { + struct device *sysdev; + unsigned int total; + struct hci_rh_data headers[] __counted_by(total); +}; + +struct hci_dma_dev_ibi_data { + struct i3c_generic_ibi_pool *pool; + unsigned int max_len; +}; + +static void hci_dma_cleanup(struct i3c_hci *hci) +{ + struct hci_rings_data *rings = hci->io_data; + struct hci_rh_data *rh; + unsigned int i; + + if (!rings) + return; + + for (i = 0; i < rings->total; i++) { + rh = &rings->headers[i]; + + rh_reg_write(INTR_SIGNAL_ENABLE, 0); + rh_reg_write(RING_CONTROL, 0); + } + + i3c_hci_sync_irq_inactive(hci); + + for (i = 0; i < rings->total; i++) { + rh = &rings->headers[i]; + + rh_reg_write(CR_SETUP, 0); + rh_reg_write(IBI_SETUP, 0); + } + + rhs_reg_write(CONTROL, 0); +} + +static void hci_dma_free(void *data) +{ + struct i3c_hci *hci = data; + struct hci_rings_data *rings = hci->io_data; + struct hci_rh_data *rh; + + if (!rings) + return; + + for (int i = 0; i < rings->total; i++) { + rh = &rings->headers[i]; + + if (rh->xfer) + dma_free_coherent(rings->sysdev, rh->xfer_alloc_sz, rh->xfer, rh->xfer_dma); + kfree(rh->src_xfers); + if (rh->ibi_status) + dma_free_coherent(rings->sysdev, + rh->ibi_status_sz * rh->ibi_status_entries, + rh->ibi_status, rh->ibi_status_dma); + if (rh->ibi_data_dma) + dma_unmap_single(rings->sysdev, rh->ibi_data_dma, + rh->ibi_chunk_sz * rh->ibi_chunks_total, + DMA_FROM_DEVICE); + kfree(rh->ibi_data); + } + + kfree(rings); + hci->io_data = NULL; +} + +static void hci_dma_init_rh(struct i3c_hci *hci, struct hci_rh_data *rh, int i) +{ + u32 regval; + + rh_reg_write(CMD_RING_BASE_LO, lower_32_bits(rh->xfer_dma)); + rh_reg_write(CMD_RING_BASE_HI, upper_32_bits(rh->xfer_dma)); + rh_reg_write(RESP_RING_BASE_LO, lower_32_bits(rh->resp_dma)); + rh_reg_write(RESP_RING_BASE_HI, upper_32_bits(rh->resp_dma)); + + regval = FIELD_PREP(CR_RING_SIZE, rh->xfer_entries); + rh_reg_write(CR_SETUP, regval); + + rh_reg_write(INTR_STATUS_ENABLE, 0xffffffff); + rh_reg_write(INTR_SIGNAL_ENABLE, INTR_IBI_READY | + INTR_TRANSFER_COMPLETION | + INTR_RING_OP | + INTR_TRANSFER_ERR | + INTR_IBI_RING_FULL | + INTR_TRANSFER_ABORT); + + if (i >= IBI_RINGS) + goto ring_ready; + + rh_reg_write(IBI_STATUS_RING_BASE_LO, lower_32_bits(rh->ibi_status_dma)); + rh_reg_write(IBI_STATUS_RING_BASE_HI, upper_32_bits(rh->ibi_status_dma)); + rh_reg_write(IBI_DATA_RING_BASE_LO, lower_32_bits(rh->ibi_data_dma)); + rh_reg_write(IBI_DATA_RING_BASE_HI, upper_32_bits(rh->ibi_data_dma)); + + regval = FIELD_PREP(IBI_STATUS_RING_SIZE, rh->ibi_status_entries) | + FIELD_PREP(IBI_DATA_CHUNK_SIZE, ilog2(rh->ibi_chunk_sz) - 2) | + FIELD_PREP(IBI_DATA_CHUNK_COUNT, rh->ibi_chunks_total); + rh_reg_write(IBI_SETUP, regval); + + regval = rh_reg_read(INTR_SIGNAL_ENABLE); + regval |= INTR_IBI_READY; + rh_reg_write(INTR_SIGNAL_ENABLE, regval); + +ring_ready: + /* + * The MIPI I3C HCI specification does not document reset values for + * RING_OPERATION1 fields and some controllers (e.g. Intel controllers) + * do not reset the values, so ensure the ring pointers are set to zero + * here. + */ + rh_reg_write(RING_OPERATION1, 0); + + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); + + /* + * Do not clear the entries of rh->src_xfers because the recovery uses + * them. In other cases they should be NULL anyway. + */ + rh->done_ptr = 0; + rh->ibi_chunk_ptr = 0; + rh->xfer_space = rh->xfer_entries; +} + +static void hci_dma_init_rings(struct i3c_hci *hci) +{ + struct hci_rings_data *rings = hci->io_data; + u32 regval; + + regval = FIELD_PREP(MAX_HEADER_COUNT, rings->total); + rhs_reg_write(CONTROL, regval); + + for (int i = 0; i < rings->total; i++) + hci_dma_init_rh(hci, &rings->headers[i], i); +} + +static void hci_dma_suspend(struct i3c_hci *hci) +{ + struct hci_rings_data *rings = hci->io_data; + int n = rings ? rings->total : 0; + + for (int i = 0; i < n; i++) { + struct hci_rh_data *rh = &rings->headers[i]; + + rh_reg_write(INTR_SIGNAL_ENABLE, 0); + rh_reg_write(RING_CONTROL, 0); + } + + i3c_hci_sync_irq_inactive(hci); +} + +static void hci_dma_resume(struct i3c_hci *hci) +{ + struct hci_rings_data *rings = hci->io_data; + + if (rings) + hci_dma_init_rings(hci); +} + +static int hci_dma_init(struct i3c_hci *hci) +{ + struct hci_rings_data *rings; + struct hci_rh_data *rh; + u32 regval; + unsigned int i, nr_rings, xfers_sz, resps_sz; + unsigned int ibi_status_ring_sz, ibi_data_ring_sz; + int ret; + + regval = rhs_reg_read(CONTROL); + nr_rings = FIELD_GET(MAX_HEADER_COUNT_CAP, regval); + dev_dbg(&hci->master.dev, "%d DMA rings available\n", nr_rings); + if (unlikely(nr_rings > 8)) { + dev_err(&hci->master.dev, "number of rings should be <= 8\n"); + nr_rings = 8; + } + if (nr_rings > XFER_RINGS) + nr_rings = XFER_RINGS; + rings = kzalloc_flex(*rings, headers, nr_rings); + if (!rings) + return -ENOMEM; + hci->io_data = rings; + rings->total = nr_rings; + rings->sysdev = i3c_hci_sysdev(hci->master.dev.parent); + + for (i = 0; i < rings->total; i++) { + u32 offset = rhs_reg_read(RHn_OFFSET(i)); + + dev_dbg(&hci->master.dev, "Ring %d at offset %#x\n", i, offset); + ret = -EINVAL; + if (!offset) + goto err_out; + rh = &rings->headers[i]; + rh->regs = hci->base_regs + offset; + init_completion(&rh->op_done); + + rh->xfer_entries = XFER_RING_ENTRIES; + + regval = rh_reg_read(CR_SETUP); + rh->xfer_struct_sz = FIELD_GET(CR_XFER_STRUCT_SIZE, regval); + rh->resp_struct_sz = FIELD_GET(CR_RESP_STRUCT_SIZE, regval); + dev_dbg(&hci->master.dev, + "xfer_struct_sz = %d, resp_struct_sz = %d", + rh->xfer_struct_sz, rh->resp_struct_sz); + xfers_sz = round_up(rh->xfer_struct_sz * rh->xfer_entries, 4); + resps_sz = rh->resp_struct_sz * rh->xfer_entries; + rh->xfer_alloc_sz = xfers_sz + resps_sz; + + rh->xfer = dma_alloc_coherent(rings->sysdev, rh->xfer_alloc_sz, + &rh->xfer_dma, GFP_KERNEL); + rh->src_xfers = + kzalloc_objs(*rh->src_xfers, rh->xfer_entries); + ret = -ENOMEM; + if (!rh->xfer || !rh->src_xfers) + goto err_out; + rh->resp = rh->xfer + xfers_sz; + rh->resp_dma = rh->xfer_dma + xfers_sz; + + /* IBIs */ + + if (i >= IBI_RINGS) + continue; + + regval = rh_reg_read(IBI_SETUP); + rh->ibi_status_sz = FIELD_GET(IBI_STATUS_STRUCT_SIZE, regval); + rh->ibi_status_entries = IBI_STATUS_RING_ENTRIES; + rh->ibi_chunks_total = IBI_CHUNK_POOL_SIZE; + + rh->ibi_chunk_sz = dma_get_cache_alignment(); + rh->ibi_chunk_sz *= IBI_CHUNK_CACHELINES; + /* + * Round IBI data chunk size to number of bytes supported by + * the HW. Chunk size can be 2^n number of DWORDs which is the + * same as 2^(n+2) bytes, where n is 0..6. + */ + rh->ibi_chunk_sz = umax(4, rh->ibi_chunk_sz); + rh->ibi_chunk_sz = roundup_pow_of_two(rh->ibi_chunk_sz); + if (rh->ibi_chunk_sz > 256) { + ret = -EINVAL; + goto err_out; + } + + ibi_status_ring_sz = rh->ibi_status_sz * rh->ibi_status_entries; + ibi_data_ring_sz = rh->ibi_chunk_sz * rh->ibi_chunks_total; + + rh->ibi_status = + dma_alloc_coherent(rings->sysdev, ibi_status_ring_sz, + &rh->ibi_status_dma, GFP_KERNEL); + rh->ibi_data = kmalloc(ibi_data_ring_sz, GFP_KERNEL); + ret = -ENOMEM; + if (!rh->ibi_status || !rh->ibi_data) + goto err_out; + rh->ibi_data_dma = + dma_map_single(rings->sysdev, rh->ibi_data, + ibi_data_ring_sz, DMA_FROM_DEVICE); + if (dma_mapping_error(rings->sysdev, rh->ibi_data_dma)) { + rh->ibi_data_dma = 0; + ret = -ENOMEM; + goto err_out; + } + } + + ret = devm_add_action(hci->master.dev.parent, hci_dma_free, hci); + if (ret) + goto err_out; + + hci_dma_init_rings(hci); + + return 0; + +err_out: + hci_dma_free(hci); + return ret; +} + +static void hci_dma_unmap_xfer(struct i3c_hci *hci, + struct hci_xfer *xfer_list, unsigned int n) +{ + struct hci_xfer *xfer; + unsigned int i; + + for (i = 0; i < n; i++) { + xfer = xfer_list + i; + if (!xfer->data) + continue; + i3c_master_dma_unmap_single(xfer->dma); + } +} + +static struct i3c_dma *hci_dma_map_xfer(struct device *dev, struct hci_xfer *xfer) +{ + enum dma_data_direction dir = xfer->rnw ? DMA_FROM_DEVICE : DMA_TO_DEVICE; + bool need_bounce = device_iommu_mapped(dev) && xfer->rnw && (xfer->data_len & 3); + + return i3c_master_dma_map_single(dev, xfer->data, xfer->data_len, need_bounce, dir); +} + +static int hci_dma_map_xfer_list(struct i3c_hci *hci, struct device *dev, + struct hci_xfer *xfer_list, int n) +{ + for (int i = 0; i < n; i++) { + struct hci_xfer *xfer = xfer_list + i; + + if (!xfer->data) + continue; + + xfer->dma = hci_dma_map_xfer(dev, xfer); + if (!xfer->dma) { + hci_dma_unmap_xfer(hci, xfer_list, i); + return -ENOMEM; + } + } + + return 0; +} + +static int hci_dma_queue_xfer(struct i3c_hci *hci, + struct hci_xfer *xfer_list, int n) +{ + struct hci_rings_data *rings = hci->io_data; + struct hci_rh_data *rh; + unsigned int i, ring, enqueue_ptr; + u32 op1_val; + int ret; + + ret = hci_dma_map_xfer_list(hci, rings->sysdev, xfer_list, n); + if (ret) + return ret; + + /* For now we only use ring 0 */ + ring = 0; + rh = &rings->headers[ring]; + + spin_lock_irq(&hci->lock); + + while (unlikely(hci->enqueue_blocked)) { + spin_unlock_irq(&hci->lock); + wait_event(hci->enqueue_wait_queue, !READ_ONCE(hci->enqueue_blocked)); + spin_lock_irq(&hci->lock); + } + + if (n > rh->xfer_space) { + spin_unlock_irq(&hci->lock); + hci_dma_unmap_xfer(hci, xfer_list, n); + return -EBUSY; + } + + op1_val = rh_reg_read(RING_OPERATION1); + enqueue_ptr = FIELD_GET(RING_OP1_CR_ENQ_PTR, op1_val); + for (i = 0; i < n; i++) { + struct hci_xfer *xfer = xfer_list + i; + u32 *ring_data = rh->xfer + rh->xfer_struct_sz * enqueue_ptr; + + xfer->final_xfer = xfer_list + n - 1; + xfer->xfer_list_pos = i; + + /* store cmd descriptor */ + *ring_data++ = xfer->cmd_desc[0]; + *ring_data++ = xfer->cmd_desc[1]; + if (hci->cmd == &mipi_i3c_hci_cmd_v2) { + *ring_data++ = xfer->cmd_desc[2]; + *ring_data++ = xfer->cmd_desc[3]; + } + + /* first word of Data Buffer Descriptor Structure */ + if (!xfer->data) + xfer->data_len = 0; + *ring_data++ = + FIELD_PREP(DATA_BUF_BLOCK_SIZE, xfer->data_len) | + ((i == n - 1) ? DATA_BUF_IOC : 0); + + /* 2nd and 3rd words of Data Buffer Descriptor Structure */ + if (xfer->data) { + *ring_data++ = lower_32_bits(xfer->dma->addr); + *ring_data++ = upper_32_bits(xfer->dma->addr); + } else { + *ring_data++ = 0; + *ring_data++ = 0; + } + + /* remember corresponding xfer struct */ + rh->src_xfers[enqueue_ptr] = xfer; + /* remember corresponding ring/entry for this xfer structure */ + xfer->ring_number = ring; + xfer->ring_entry = enqueue_ptr; + + enqueue_ptr = (enqueue_ptr + 1) % rh->xfer_entries; + } + + if (rh->xfer_space == rh->xfer_entries) + hci_start_xfer(xfer_list); + + rh->xfer_space -= n; + + op1_val &= ~RING_OP1_CR_ENQ_PTR; + op1_val |= FIELD_PREP(RING_OP1_CR_ENQ_PTR, enqueue_ptr); + rh_reg_write(RING_OPERATION1, op1_val); + spin_unlock_irq(&hci->lock); + + return 0; +} + +static void hci_dma_xfer_done(struct i3c_hci *hci, struct hci_rh_data *rh) +{ + u32 op1_val, op2_val, resp, *ring_resp; + unsigned int tid, done_ptr = rh->done_ptr; + unsigned int done_cnt = 0; + bool start_next = false; + struct hci_xfer *xfer; + + for (;;) { + op2_val = rh_reg_read(RING_OPERATION2); + if (done_ptr == FIELD_GET(RING_OP2_CR_DEQ_PTR, op2_val)) + break; + + ring_resp = rh->resp + rh->resp_struct_sz * done_ptr; + resp = *ring_resp; + tid = RESP_TID(resp); + dev_dbg(&hci->master.dev, "resp = 0x%08x", resp); + + xfer = rh->src_xfers[done_ptr]; + if (!xfer) { + dev_dbg(&hci->master.dev, "orphaned ring entry"); + } else { + hci_dma_unmap_xfer(hci, xfer, 1); + rh->src_xfers[done_ptr] = NULL; + xfer->ring_entry = -1; + if (tid != xfer->cmd_tid) { + dev_err(&hci->master.dev, + "response tid=%d when expecting %d\n", + tid, xfer->cmd_tid); + hci->recovery_needed = true; + if (!RESP_STATUS(resp)) + hci_cmd_set_resp_err(&resp, RESP_ERR_HC_TERMINATED); + } + xfer->response = resp; + if (xfer == xfer->final_xfer || RESP_STATUS(resp)) + complete(xfer->final_xfer->completion); + else + hci_start_xfer(xfer); + if (RESP_STATUS(resp)) { + hci->enqueue_blocked = true; + start_next = false; + } else { + start_next = true; + } + } + + done_ptr = (done_ptr + 1) % rh->xfer_entries; + rh->done_ptr = done_ptr; + done_cnt += 1; + } + + rh->xfer_space += done_cnt; + if (start_next && rh->xfer_space < rh->xfer_entries) { + xfer = rh->src_xfers[done_ptr]; + hci_start_xfer(xfer); + } + op1_val = rh_reg_read(RING_OPERATION1); + op1_val &= ~RING_OP1_CR_SW_DEQ_PTR; + op1_val |= FIELD_PREP(RING_OP1_CR_SW_DEQ_PTR, done_ptr); + rh_reg_write(RING_OPERATION1, op1_val); +} + +static void hci_dma_requires_hc_abort_quirk(struct i3c_hci *hci, struct hci_rh_data *rh) +{ + reinit_completion(&rh->op_done); + mipi_i3c_hci_abort(hci); + wait_for_completion_timeout(&rh->op_done, HZ); + rh_reg_write(RING_CONTROL, rh_reg_read(RING_CONTROL) | RING_CTRL_ABORT); +} + +static void hci_dma_abort(struct i3c_hci *hci, struct hci_rh_data *rh) +{ + if (hci->quirks & HCI_QUIRK_DMA_REQUIRES_HC_ABORT) { + hci_dma_requires_hc_abort_quirk(hci, rh); + return; + } + + reinit_completion(&rh->op_done); + rh_reg_write(RING_CONTROL, rh_reg_read(RING_CONTROL) | RING_CTRL_ABORT); + wait_for_completion_timeout(&rh->op_done, HZ); +} + +static void hci_dma_unblock_enqueue(struct i3c_hci *hci) +{ + if (hci->enqueue_blocked) { + hci->enqueue_blocked = false; + wake_up_all(&hci->enqueue_wait_queue); + } +} + +static void hci_dma_error_out_rh(struct i3c_hci *hci, struct hci_rh_data *rh) +{ + /* + * The entries of rh->src_xfers are not cleared by + * i3c_hci_reset_and_restore(), so can be used here. Do 2 passes so + * that the final_xfer of an xfer list is always processed last. + */ + for (int pass = 0; pass < 2; pass++) + for (int i = 0; i < rh->xfer_entries; i++) { + struct hci_xfer *xfer = rh->src_xfers[i]; + + if (!xfer || (!pass && xfer == xfer->final_xfer)) + continue; + hci_dma_unmap_xfer(hci, xfer, 1); + rh->src_xfers[i] = NULL; + xfer->ring_entry = -1; + hci_cmd_set_resp_err(&xfer->response, RESP_ERR_HC_TERMINATED); + if (xfer == xfer->final_xfer) + complete(xfer->final_xfer->completion); + } +} + +static void hci_dma_error_out_all(struct i3c_hci *hci) +{ + struct hci_rings_data *rings = hci->io_data; + + for (int i = 0; i < rings->total; i++) + hci_dma_error_out_rh(hci, &rings->headers[i]); +} + +static void hci_dma_recovery(struct i3c_hci *hci) +{ + int ret; + + dev_err(&hci->master.dev, "Attempting to recover from internal errors\n"); + + for (int i = 0; i < 3; i++) { + ret = i3c_hci_reset_and_restore(hci); + if (!ret) + break; + dev_err(&hci->master.dev, "Reset and restore failed, error %d\n", ret); + /* Just in case the controller is busy, give it some time */ + msleep(1000); + } + + spin_lock_irq(&hci->lock); + hci_dma_error_out_all(hci); + hci_dma_unblock_enqueue(hci); + hci->recovery_needed = false; + spin_unlock_irq(&hci->lock); + + dev_err(&hci->master.dev, "Recovery %s\n", ret ? "failed!" : "done"); +} + +static bool hci_dma_wait_for_noop(struct i3c_hci *hci, struct hci_xfer *xfer_list, int n, + int noop_pos) +{ + struct completion *done = xfer_list->final_xfer->completion; + bool timeout = !wait_for_completion_timeout(done, HZ); + u32 error = timeout; + + for (int i = noop_pos; i < n && !error; i++) + error = RESP_STATUS(xfer_list[i].response); + + if (!error) + return true; + + if (timeout) + dev_err(&hci->master.dev, "NoOp timeout error\n"); + else + dev_err(&hci->master.dev, "NoOp error %u\n", error); + + return false; +} + +static bool hci_dma_dequeue_xfer(struct i3c_hci *hci, + struct hci_xfer *xfer_list, int n) +{ + struct hci_rings_data *rings = hci->io_data; + struct hci_rh_data *rh = &rings->headers[xfer_list[0].ring_number]; + int noop_pos = -1; + unsigned int i; + bool did_unqueue = false; + u32 ring_status; + + guard(mutex)(&hci->control_mutex); + + spin_lock_irq(&hci->lock); +restart: + ring_status = rh_reg_read(RING_STATUS); + if (ring_status & RING_STATUS_RUNNING) { + /* + * The transfer may have already completed, especially + * if recovery has just run. Do nothing in that case. + */ + hci_dma_xfer_done(hci, rh); + if (xfer_list->final_xfer->ring_entry < 0 && + !hci->recovery_needed && !hci->enqueue_blocked && + ring_status == (RING_STATUS_ENABLED | RING_STATUS_RUNNING)) { + spin_unlock_irq(&hci->lock); + return false; + } + hci->enqueue_blocked = true; + spin_unlock_irq(&hci->lock); + /* stop the ring */ + hci_dma_abort(hci, rh); + spin_lock_irq(&hci->lock); + ring_status = rh_reg_read(RING_STATUS); + if (ring_status & RING_STATUS_RUNNING) { + dev_err(&hci->master.dev, "Unable to abort the DMA ring\n"); + hci->recovery_needed = true; + } + } + + if ((hci->quirks & HCI_QUIRK_DMA_ABORT_REQUIRES_PIO_RESET) && + (rh_reg_read(RING_STATUS) & RING_STATUS_ABORTED)) + mipi_i3c_hci_pio_reset_all_queues(hci); + + hci_dma_xfer_done(hci, rh); + + if (hci->recovery_needed) { + hci->enqueue_blocked = true; + spin_unlock_irq(&hci->lock); + hci_dma_recovery(hci); + return true; + } + + for (i = 0; i < n; i++) { + struct hci_xfer *xfer = xfer_list + i; + int idx = xfer->ring_entry; + + /* + * At the time the abort happened, the xfer might have + * completed already. If not then replace corresponding + * descriptor entries with a no-op. + */ + if (idx >= 0) { + u32 *ring_data = rh->xfer + rh->xfer_struct_sz * idx; + + /* store no-op cmd descriptor */ + *ring_data++ = FIELD_PREP(CMD_0_ATTR, 0x7) | FIELD_PREP(CMD_0_TID, xfer->cmd_tid); + *ring_data++ = 0; + if (hci->cmd == &mipi_i3c_hci_cmd_v2) { + *ring_data++ = 0; + *ring_data++ = 0; + } + + if (noop_pos < 0) { + reinit_completion(xfer->final_xfer->completion); + noop_pos = i; + } + + did_unqueue = true; + } + } + + /* + * A software ABORT may race with transfer completion and abort the next + * transfer list instead. Detect that case, and do not restart the ring. + * It will be handled by a subsequent dequeue. + */ + if (!did_unqueue) { + struct hci_xfer *xfer = rh->src_xfers[rh->done_ptr]; + + if (xfer && xfer->xfer_list_pos && xfer->final_xfer != xfer_list->final_xfer) { + spin_unlock_irq(&hci->lock); + return false; + } + } + + /* restart the ring */ + reinit_completion(&rh->op_done); + mipi_i3c_hci_resume(hci); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE); + rh_reg_write(RING_CONTROL, RING_CTRL_ENABLE | RING_CTRL_RUN_STOP); + + hci_dma_unblock_enqueue(hci); + + if (rh->xfer_space < rh->xfer_entries) + hci_start_xfer(rh->src_xfers[rh->done_ptr]); + + spin_unlock_irq(&hci->lock); + + wait_for_completion_timeout(&rh->op_done, HZ); + + if (did_unqueue && !hci_dma_wait_for_noop(hci, xfer_list, n, noop_pos)) { + spin_lock_irq(&hci->lock); + hci->recovery_needed = true; + goto restart; + } + + return did_unqueue; +} + +static int hci_dma_handle_error(struct i3c_hci *hci, struct hci_xfer *xfer_list, int n) +{ + return hci_dma_dequeue_xfer(hci, xfer_list, n) ? -EIO : 0; +} + +static int hci_dma_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + struct i3c_generic_ibi_pool *pool; + struct hci_dma_dev_ibi_data *dev_ibi; + + dev_ibi = kmalloc_obj(*dev_ibi); + if (!dev_ibi) + return -ENOMEM; + pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(pool)) { + kfree(dev_ibi); + return PTR_ERR(pool); + } + dev_ibi->pool = pool; + dev_ibi->max_len = req->max_payload_len; + dev_data->ibi_data = dev_ibi; + return 0; +} + +static void hci_dma_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; + + dev_data->ibi_data = NULL; + i3c_generic_ibi_free_pool(dev_ibi->pool); + kfree(dev_ibi); +} + +static void hci_dma_recycle_ibi_slot(struct i3c_hci *hci, + struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + struct hci_dma_dev_ibi_data *dev_ibi = dev_data->ibi_data; + + i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); +} + +static void hci_dma_process_ibi(struct i3c_hci *hci, struct hci_rh_data *rh) +{ + struct hci_rings_data *rings = hci->io_data; + struct i3c_dev_desc *dev; + struct i3c_hci_dev_data *dev_data; + struct hci_dma_dev_ibi_data *dev_ibi; + struct i3c_ibi_slot *slot; + u32 op1_val, op2_val, ibi_status_error; + unsigned int ptr, enq_ptr, deq_ptr; + unsigned int ibi_size, ibi_chunks, ibi_data_offset, first_part; + int ibi_addr, last_ptr; + void *ring_ibi_data; + dma_addr_t ring_ibi_data_dma; + + op1_val = rh_reg_read(RING_OPERATION1); + deq_ptr = FIELD_GET(RING_OP1_IBI_DEQ_PTR, op1_val); + + op2_val = rh_reg_read(RING_OPERATION2); + enq_ptr = FIELD_GET(RING_OP2_IBI_ENQ_PTR, op2_val); + + ibi_status_error = 0; + ibi_addr = -1; + ibi_chunks = 0; + ibi_size = 0; + last_ptr = -1; + + /* let's find all we can about this IBI */ + for (ptr = deq_ptr; ptr != enq_ptr; + ptr = (ptr + 1) % rh->ibi_status_entries) { + u32 ibi_status, *ring_ibi_status; + unsigned int chunks; + + ring_ibi_status = rh->ibi_status + rh->ibi_status_sz * ptr; + ibi_status = *ring_ibi_status; + dev_dbg(&hci->master.dev, "status = %#x", ibi_status); + + if (ibi_status_error) { + /* we no longer care */ + } else if (ibi_status & IBI_ERROR) { + ibi_status_error = ibi_status; + } else if (ibi_addr == -1) { + ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); + } else if (ibi_addr != FIELD_GET(IBI_TARGET_ADDR, ibi_status)) { + /* the address changed unexpectedly */ + ibi_status_error = ibi_status; + } + + chunks = FIELD_GET(IBI_CHUNKS, ibi_status); + ibi_chunks += chunks; + if (!(ibi_status & IBI_LAST_STATUS)) { + ibi_size += chunks * rh->ibi_chunk_sz; + } else { + if (chunks) { + ibi_size += (chunks - 1) * rh->ibi_chunk_sz; + ibi_size += FIELD_GET(IBI_DATA_LENGTH, ibi_status); + } + last_ptr = ptr; + break; + } + } + + /* validate what we've got */ + + if (last_ptr == -1) { + /* this IBI sequence is not yet complete */ + dev_dbg(&hci->master.dev, + "no LAST_STATUS available (e=%d d=%d)", + enq_ptr, deq_ptr); + return; + } + deq_ptr = last_ptr + 1; + deq_ptr %= rh->ibi_status_entries; + + if (ibi_status_error) { + dev_err(&hci->master.dev, "IBI error from %#x\n", ibi_addr); + goto done; + } + + /* determine who this is for */ + if (ibi_addr == I3C_HOT_JOIN_ADDR) { + i3c_master_queue_hotjoin(&hci->master); + goto done; + } + + dev = i3c_hci_addr_to_dev(hci, ibi_addr); + if (!dev) { + /* + * Either an IBI received just before IBI's were disabled, or + * the controller is broken. Assume the former. + */ + dev_dbg(&hci->master.dev, "IBI when not enabled at address %#x\n", ibi_addr); + goto done; + } + + dev_data = i3c_dev_get_master_data(dev); + dev_ibi = dev_data->ibi_data; + if (ibi_size > dev_ibi->max_len) { + dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", + ibi_size, dev_ibi->max_len); + goto done; + } + + /* + * This ring model is not suitable for zero-copy processing of IBIs. + * We have the data chunk ring wrap-around to deal with, meaning + * that the payload might span multiple chunks beginning at the + * end of the ring and wrap to the start of the ring. Furthermore + * there is no guarantee that those chunks will be released in order + * and in a timely manner by the upper driver. So let's just copy + * them to a discrete buffer. In practice they're supposed to be + * small anyway. + */ + slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); + if (!slot) { + dev_err(&hci->master.dev, "no free slot for IBI\n"); + goto done; + } + + /* copy first part of the payload */ + ibi_data_offset = rh->ibi_chunk_sz * rh->ibi_chunk_ptr; + ring_ibi_data = rh->ibi_data + ibi_data_offset; + ring_ibi_data_dma = rh->ibi_data_dma + ibi_data_offset; + first_part = (rh->ibi_chunks_total - rh->ibi_chunk_ptr) + * rh->ibi_chunk_sz; + if (first_part > ibi_size) + first_part = ibi_size; + dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma, + first_part, DMA_FROM_DEVICE); + memcpy(slot->data, ring_ibi_data, first_part); + + /* copy second part if any */ + if (ibi_size > first_part) { + /* we wrap back to the start and copy remaining data */ + ring_ibi_data = rh->ibi_data; + ring_ibi_data_dma = rh->ibi_data_dma; + dma_sync_single_for_cpu(rings->sysdev, ring_ibi_data_dma, + ibi_size - first_part, DMA_FROM_DEVICE); + memcpy(slot->data + first_part, ring_ibi_data, + ibi_size - first_part); + } + + /* submit it */ + slot->dev = dev; + slot->len = ibi_size; + i3c_master_queue_ibi(dev, slot); + +done: + op1_val = rh_reg_read(RING_OPERATION1); + op1_val &= ~RING_OP1_IBI_DEQ_PTR; + op1_val |= FIELD_PREP(RING_OP1_IBI_DEQ_PTR, deq_ptr); + rh_reg_write(RING_OPERATION1, op1_val); + + /* update the chunk pointer */ + rh->ibi_chunk_ptr += ibi_chunks; + rh->ibi_chunk_ptr %= rh->ibi_chunks_total; + + /* and tell the hardware about freed chunks */ + rh_reg_write(CHUNK_CONTROL, rh_reg_read(CHUNK_CONTROL) + ibi_chunks); +} + +static bool hci_dma_irq_handler(struct i3c_hci *hci) +{ + struct hci_rings_data *rings = hci->io_data; + unsigned int i; + bool handled = false; + + for (i = 0; i < rings->total; i++) { + struct hci_rh_data *rh; + u32 status; + + rh = &rings->headers[i]; + status = rh_reg_read(INTR_STATUS); + dev_dbg(&hci->master.dev, "Ring %d: RH_INTR_STATUS %#x", + i, status); + if (!status) + continue; + rh_reg_write(INTR_STATUS, status); + + if (status & INTR_IBI_READY) + hci_dma_process_ibi(hci, rh); + if (status & (INTR_TRANSFER_COMPLETION | INTR_TRANSFER_ERR)) + hci_dma_xfer_done(hci, rh); + if (status & INTR_RING_OP) + complete(&rh->op_done); + if (status & INTR_TRANSFER_ABORT) + dev_dbg(&hci->master.dev, "Ring %d: Transfer Aborted\n", i); + if (status & INTR_IBI_RING_FULL) + dev_err_ratelimited(&hci->master.dev, + "Ring %d: IBI Ring Full Condition\n", i); + + handled = true; + } + + return handled; +} + +const struct hci_io_ops mipi_i3c_hci_dma = { + .init = hci_dma_init, + .cleanup = hci_dma_cleanup, + .queue_xfer = hci_dma_queue_xfer, + .dequeue_xfer = hci_dma_dequeue_xfer, + .handle_error = hci_dma_handle_error, + .irq_handler = hci_dma_irq_handler, + .request_ibi = hci_dma_request_ibi, + .free_ibi = hci_dma_free_ibi, + .recycle_ibi_slot = hci_dma_recycle_ibi_slot, + .suspend = hci_dma_suspend, + .resume = hci_dma_resume, +}; diff --git a/drivers/i3c/master/mipi-i3c-hci/ext_caps.c b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c new file mode 100644 index 000000000..77840fd4a --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/ext_caps.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + */ + +#include <linux/bitfield.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/kernel.h> +#include <linux/io.h> + +#include "hci.h" +#include "ext_caps.h" +#include "xfer_mode_rate.h" + +/* Extended Capability Header */ +#define CAP_HEADER_LENGTH GENMASK(23, 8) +#define CAP_HEADER_ID GENMASK(7, 0) + +static int hci_extcap_hardware_id(struct i3c_hci *hci, void __iomem *base) +{ + hci->vendor_mipi_id = readl(base + 0x04); + hci->vendor_version_id = readl(base + 0x08); + hci->vendor_product_id = readl(base + 0x0c); + + dev_dbg(&hci->master.dev, "vendor MIPI ID: %#x\n", hci->vendor_mipi_id); + dev_dbg(&hci->master.dev, "vendor version ID: %#x\n", hci->vendor_version_id); + dev_dbg(&hci->master.dev, "vendor product ID: %#x\n", hci->vendor_product_id); + + /* ought to go in a table if this grows too much */ + switch (hci->vendor_mipi_id) { + case MIPI_VENDOR_NXP: + hci->quirks |= HCI_QUIRK_RAW_CCC; + dev_dbg(&hci->master.dev, "raw CCC quirks set"); + break; + } + + return 0; +} + +static int hci_extcap_master_config(struct i3c_hci *hci, void __iomem *base) +{ + u32 master_config = readl(base + 0x04); + unsigned int operation_mode = FIELD_GET(GENMASK(5, 4), master_config); + static const char * const functionality[] = { + "(unknown)", "master only", "target only", + "primary/secondary master" }; + dev_dbg(&hci->master.dev, "operation mode: %s\n", functionality[operation_mode]); + if (operation_mode & 0x1) + return 0; + dev_err(&hci->master.dev, "only master mode is currently supported\n"); + return -EOPNOTSUPP; +} + +static int hci_extcap_multi_bus(struct i3c_hci *hci, void __iomem *base) +{ + u32 bus_instance = readl(base + 0x04); + unsigned int count = FIELD_GET(GENMASK(3, 0), bus_instance); + + dev_dbg(&hci->master.dev, "%d bus instances\n", count); + return 0; +} + +static int hci_extcap_xfer_modes(struct i3c_hci *hci, void __iomem *base) +{ + u32 header = readl(base); + u32 entries = FIELD_GET(CAP_HEADER_LENGTH, header) - 1; + unsigned int index; + + dev_dbg(&hci->master.dev, "transfer mode table has %d entries\n", entries); + base += 4; /* skip header */ + for (index = 0; index < entries; index++) { + u32 mode_entry = readl(base); + + dev_dbg(&hci->master.dev, "mode %d: 0x%08x", + index, mode_entry); + /* TODO: will be needed when I3C core does more than SDR */ + base += 4; + } + + return 0; +} + +static int hci_extcap_xfer_rates(struct i3c_hci *hci, void __iomem *base) +{ + u32 header = readl(base); + u32 entries = FIELD_GET(CAP_HEADER_LENGTH, header) - 1; + u32 rate_entry; + unsigned int index, rate, rate_id, mode_id; + + base += 4; /* skip header */ + + dev_dbg(&hci->master.dev, "available data rates:\n"); + for (index = 0; index < entries; index++) { + rate_entry = readl(base); + dev_dbg(&hci->master.dev, "entry %d: 0x%08x", + index, rate_entry); + rate = FIELD_GET(XFERRATE_ACTUAL_RATE_KHZ, rate_entry); + rate_id = FIELD_GET(XFERRATE_RATE_ID, rate_entry); + mode_id = FIELD_GET(XFERRATE_MODE_ID, rate_entry); + dev_dbg(&hci->master.dev, "rate %d for %s = %d kHz\n", + rate_id, + mode_id == XFERRATE_MODE_I3C ? "I3C" : + mode_id == XFERRATE_MODE_I2C ? "I2C" : + "unknown mode", + rate); + base += 4; + } + + return 0; +} + +static int hci_extcap_auto_command(struct i3c_hci *hci, void __iomem *base) +{ + u32 autocmd_ext_caps = readl(base + 0x04); + unsigned int max_count = FIELD_GET(GENMASK(3, 0), autocmd_ext_caps); + u32 autocmd_ext_config = readl(base + 0x08); + unsigned int count = FIELD_GET(GENMASK(3, 0), autocmd_ext_config); + + dev_dbg(&hci->master.dev, "%d/%d active auto-command entries\n", + count, max_count); + /* remember auto-command register location for later use */ + hci->AUTOCMD_regs = base; + return 0; +} + +static int hci_extcap_debug(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "debug registers present\n"); + hci->DEBUG_regs = base; + return 0; +} + +static int hci_extcap_scheduled_cmd(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "scheduled commands available\n"); + /* hci->schedcmd_regs = base; */ + return 0; +} + +static int hci_extcap_non_curr_master(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "Non-Current Master support available\n"); + /* hci->NCM_regs = base; */ + return 0; +} + +static int hci_extcap_ccc_resp_conf(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "CCC Response Configuration available\n"); + return 0; +} + +static int hci_extcap_global_DAT(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "Global DAT available\n"); + return 0; +} + +static int hci_extcap_multilane(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "Master Multi-Lane support available\n"); + return 0; +} + +static int hci_extcap_ncm_multilane(struct i3c_hci *hci, void __iomem *base) +{ + dev_dbg(&hci->master.dev, "NCM Multi-Lane support available\n"); + return 0; +} + +struct hci_ext_caps { + u8 id; + u16 min_length; + int (*parser)(struct i3c_hci *hci, void __iomem *base); +}; + +#define EXT_CAP(_id, _highest_mandatory_reg_offset, _parser) \ + { .id = (_id), .parser = (_parser), \ + .min_length = (_highest_mandatory_reg_offset)/4 + 1 } + +static const struct hci_ext_caps ext_capabilities[] = { + EXT_CAP(0x01, 0x0c, hci_extcap_hardware_id), + EXT_CAP(0x02, 0x04, hci_extcap_master_config), + EXT_CAP(0x03, 0x04, hci_extcap_multi_bus), + EXT_CAP(0x04, 0x24, hci_extcap_xfer_modes), + EXT_CAP(0x05, 0x08, hci_extcap_auto_command), + EXT_CAP(0x08, 0x40, hci_extcap_xfer_rates), + EXT_CAP(0x0c, 0x10, hci_extcap_debug), + EXT_CAP(0x0d, 0x0c, hci_extcap_scheduled_cmd), + EXT_CAP(0x0e, 0x80, hci_extcap_non_curr_master), /* TODO confirm size */ + EXT_CAP(0x0f, 0x04, hci_extcap_ccc_resp_conf), + EXT_CAP(0x10, 0x08, hci_extcap_global_DAT), + EXT_CAP(0x9d, 0x04, hci_extcap_multilane), + EXT_CAP(0x9e, 0x04, hci_extcap_ncm_multilane), +}; + +static int hci_extcap_vendor_NXP(struct i3c_hci *hci, void __iomem *base) +{ + hci->vendor_data = (__force void *)base; + dev_dbg(&hci->master.dev, "Build Date Info = %#x\n", readl(base + 1 * 4)); + /* reset the FPGA */ + writel(0xdeadbeef, base + 1*4); + return 0; +} + +struct hci_ext_cap_vendor_specific { + u32 vendor; + u8 cap; + u16 min_length; + int (*parser)(struct i3c_hci *hci, void __iomem *base); +}; + +#define EXT_CAP_VENDOR(_vendor, _cap, _highest_mandatory_reg_offset) \ + { .vendor = (MIPI_VENDOR_##_vendor), .cap = (_cap), \ + .parser = (hci_extcap_vendor_##_vendor), \ + .min_length = (_highest_mandatory_reg_offset)/4 + 1 } + +static const struct hci_ext_cap_vendor_specific vendor_ext_caps[] = { + EXT_CAP_VENDOR(NXP, 0xc0, 0x20), +}; + +static int hci_extcap_vendor_specific(struct i3c_hci *hci, void __iomem *base, + u32 cap_id, u32 cap_length) +{ + const struct hci_ext_cap_vendor_specific *vendor_cap_entry; + int i; + + vendor_cap_entry = NULL; + for (i = 0; i < ARRAY_SIZE(vendor_ext_caps); i++) { + if (vendor_ext_caps[i].vendor == hci->vendor_mipi_id && + vendor_ext_caps[i].cap == cap_id) { + vendor_cap_entry = &vendor_ext_caps[i]; + break; + } + } + + if (!vendor_cap_entry) { + dev_dbg(&hci->master.dev, "unknown ext_cap 0x%02x for vendor 0x%02x\n", + cap_id, hci->vendor_mipi_id); + return 0; + } + if (cap_length < vendor_cap_entry->min_length) { + dev_err(&hci->master.dev, + "ext_cap 0x%02x has size %d (expecting >= %d)\n", + cap_id, cap_length, vendor_cap_entry->min_length); + return -EINVAL; + } + return vendor_cap_entry->parser(hci, base); +} + +int i3c_hci_parse_ext_caps(struct i3c_hci *hci) +{ + void __iomem *curr_cap = hci->EXTCAPS_regs; + void __iomem *end = curr_cap + 0x1000; /* some arbitrary limit */ + u32 cap_header, cap_id, cap_length; + const struct hci_ext_caps *cap_entry; + int i, err = 0; + + if (!curr_cap) + return 0; + + for (; !err && curr_cap < end; curr_cap += cap_length * 4) { + cap_header = readl(curr_cap); + cap_id = FIELD_GET(CAP_HEADER_ID, cap_header); + cap_length = FIELD_GET(CAP_HEADER_LENGTH, cap_header); + dev_dbg(&hci->master.dev, "id=0x%02x length=%d", + cap_id, cap_length); + if (!cap_id || !cap_length) + break; + if (curr_cap + cap_length * 4 >= end) { + dev_err(&hci->master.dev, + "ext_cap 0x%02x has size %d (too big)\n", + cap_id, cap_length); + err = -EINVAL; + break; + } + + if (cap_id >= 0xc0 && cap_id <= 0xcf) { + err = hci_extcap_vendor_specific(hci, curr_cap, + cap_id, cap_length); + continue; + } + + cap_entry = NULL; + for (i = 0; i < ARRAY_SIZE(ext_capabilities); i++) { + if (ext_capabilities[i].id == cap_id) { + cap_entry = &ext_capabilities[i]; + break; + } + } + if (!cap_entry) { + dev_dbg(&hci->master.dev, "unknown ext_cap 0x%02x\n", cap_id); + } else if (cap_length < cap_entry->min_length) { + dev_err(&hci->master.dev, + "ext_cap 0x%02x has size %d (expecting >= %d)\n", + cap_id, cap_length, cap_entry->min_length); + err = -EINVAL; + } else { + err = cap_entry->parser(hci, curr_cap); + } + } + return err; +} diff --git a/drivers/i3c/master/mipi-i3c-hci/ext_caps.h b/drivers/i3c/master/mipi-i3c-hci/ext_caps.h new file mode 100644 index 000000000..b15e62995 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/ext_caps.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Extended Capability Definitions + */ + +#ifndef EXTCAPS_H +#define EXTCAPS_H + +/* MIPI vendor IDs */ +#define MIPI_VENDOR_NXP 0x11b + +int i3c_hci_parse_ext_caps(struct i3c_hci *hci); + +#endif diff --git a/drivers/i3c/master/mipi-i3c-hci/hci.h b/drivers/i3c/master/mipi-i3c-hci/hci.h new file mode 100644 index 000000000..ee73f6e67 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/hci.h @@ -0,0 +1,201 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Common HCI stuff + */ + +#ifndef HCI_H +#define HCI_H + +#include <linux/io.h> +#include <linux/jiffies.h> + +/* 32-bit word aware bit and mask macros */ +#define W0_MASK(h, l) GENMASK((h) - 0, (l) - 0) +#define W1_MASK(h, l) GENMASK((h) - 32, (l) - 32) +#define W2_MASK(h, l) GENMASK((h) - 64, (l) - 64) +#define W3_MASK(h, l) GENMASK((h) - 96, (l) - 96) + +/* Same for single bit macros (trailing _ to align with W*_MASK width) */ +#define W0_BIT_(x) BIT((x) - 0) +#define W1_BIT_(x) BIT((x) - 32) +#define W2_BIT_(x) BIT((x) - 64) +#define W3_BIT_(x) BIT((x) - 96) + +#define reg_read(r) readl(hci->base_regs + (r)) +#define reg_write(r, v) writel(v, hci->base_regs + (r)) +#define reg_set(r, v) reg_write(r, reg_read(r) | (v)) +#define reg_clear(r, v) reg_write(r, reg_read(r) & ~(v)) + +/* helper macro for HCI version check */ +#define hci_version_at_least(hci, maj, min) \ + ((hci)->version_major > (maj) || \ + ((hci)->version_major == (maj) && (hci)->version_minor >= (min))) + +struct hci_cmd_ops; + +struct dat_words { + u32 w0; + u32 w1; +}; + +/* Our main structure */ +struct i3c_hci { + struct i3c_master_controller master; + void __iomem *base_regs; + void __iomem *DAT_regs; + void __iomem *DCT_regs; + void __iomem *RHS_regs; + void __iomem *PIO_regs; + void __iomem *EXTCAPS_regs; + void __iomem *AUTOCMD_regs; + void __iomem *DEBUG_regs; + const struct hci_io_ops *io; + void *io_data; + const struct hci_cmd_ops *cmd; + spinlock_t lock; + struct mutex control_mutex; + atomic_t next_cmd_tid; + bool irq_inactive; + bool enqueue_blocked; + bool recovery_needed; + bool hj_init_done; + wait_queue_head_t enqueue_wait_queue; + u32 caps; + unsigned int quirks; + unsigned int DAT_entries; + unsigned int DAT_entry_size; + void *DAT_data; + struct dat_words *DAT; + struct i3c_dev_desc **ibi_devs; + unsigned int DCT_entries; + unsigned int DCT_entry_size; + u8 version_major; + u8 version_minor; + u8 revision; + u8 dyn_addr; + u32 vendor_mipi_id; + u32 vendor_version_id; + u32 vendor_product_id; + void *vendor_data; +}; + +/* + * Structure to represent a master initiated transfer. + * The rnw, data and data_len fields must be initialized before calling any + * hci->cmd->*() method. The cmd method will initialize cmd_desc[] and + * possibly modify (clear) the data field. Then xfer->cmd_desc[0] can + * be augmented with CMD_0_ROC and/or CMD_0_TOC. + * The completion field needs to be initialized before queueing with + * hci->io->queue_xfer(), and requires CMD_0_ROC to be set. + */ +struct hci_xfer { + u32 cmd_desc[4]; + u32 response; + bool rnw; + bool started; + void *data; + unsigned int data_len; + unsigned int cmd_tid; + struct completion *completion; + unsigned long timeout; + unsigned long start_jiffies; + union { + struct { + /* PIO specific */ + struct hci_xfer *next_xfer; + struct hci_xfer *next_data; + struct hci_xfer *next_resp; + unsigned int data_left; + u32 data_word_before_partial; + }; + struct { + /* DMA specific */ + struct i3c_dma *dma; + struct hci_xfer *final_xfer; + int ring_number; + int ring_entry; + int xfer_list_pos; + }; + }; +}; + +static inline struct hci_xfer *hci_alloc_xfer(unsigned int n) +{ + return kzalloc_objs(struct hci_xfer, n); +} + +static inline void hci_free_xfer(struct hci_xfer *xfer, unsigned int n) +{ + kfree(xfer); +} + +static inline void hci_start_xfer(struct hci_xfer *xfer) +{ + if (!xfer->started) { + xfer->started = true; + xfer->start_jiffies = jiffies; + } +} + +/* This abstracts PIO vs DMA operations */ +struct hci_io_ops { + bool (*irq_handler)(struct i3c_hci *hci); + int (*queue_xfer)(struct i3c_hci *hci, struct hci_xfer *xfer, int n); + bool (*dequeue_xfer)(struct i3c_hci *hci, struct hci_xfer *xfer, int n); + int (*handle_error)(struct i3c_hci *hci, struct hci_xfer *xfer, int n); + int (*request_ibi)(struct i3c_hci *hci, struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req); + void (*free_ibi)(struct i3c_hci *hci, struct i3c_dev_desc *dev); + void (*recycle_ibi_slot)(struct i3c_hci *hci, struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot); + int (*init)(struct i3c_hci *hci); + void (*cleanup)(struct i3c_hci *hci); + void (*suspend)(struct i3c_hci *hci); + void (*resume)(struct i3c_hci *hci); +}; + +extern const struct hci_io_ops mipi_i3c_hci_pio; +extern const struct hci_io_ops mipi_i3c_hci_dma; + +/* Our per device master private data */ +struct i3c_hci_dev_data { + int dat_idx; + void *ibi_data; +}; + +/* list of quirks */ +#define HCI_QUIRK_RAW_CCC BIT(1) /* CCC framing must be explicit */ +#define HCI_QUIRK_PIO_MODE BIT(2) /* Set PIO mode for AMD platforms */ +#define HCI_QUIRK_OD_PP_TIMING BIT(3) /* Set OD and PP timings for AMD platforms */ +#define HCI_QUIRK_RESP_BUF_THLD BIT(4) /* Set resp buf thld to 0 for AMD platforms */ +#define HCI_QUIRK_RPM_ALLOWED BIT(5) /* Runtime PM allowed */ +#define HCI_QUIRK_RPM_IBI_ALLOWED BIT(6) /* IBI and Hot-Join allowed while runtime suspended */ +#define HCI_QUIRK_RPM_PARENT_MANAGED BIT(7) /* Runtime PM managed by parent device */ +#define HCI_QUIRK_DMA_ABORT_REQUIRES_PIO_RESET BIT(8) /* Do PIO queue SW resets after DMA abort */ +#define HCI_QUIRK_DMA_REQUIRES_HC_ABORT BIT(9) /* Use HC_CONTROL ABORT to abort DMA */ + +/* global functions */ +void mipi_i3c_hci_resume(struct i3c_hci *hci); +void mipi_i3c_hci_abort(struct i3c_hci *hci); +void mipi_i3c_hci_pio_reset(struct i3c_hci *hci); +void mipi_i3c_hci_pio_reset_all_queues(struct i3c_hci *hci); +void mipi_i3c_hci_dct_index_reset(struct i3c_hci *hci); +void amd_set_od_pp_timing(struct i3c_hci *hci); +void amd_set_resp_buf_thld(struct i3c_hci *hci); +void i3c_hci_sync_irq_inactive(struct i3c_hci *hci); +int i3c_hci_process_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n); + +struct device *i3c_hci_sysdev(struct device *dev); + +#define DEFAULT_AUTOSUSPEND_DELAY_MS 1000 + +int i3c_hci_rpm_suspend(struct device *dev); +int i3c_hci_rpm_resume(struct device *dev); + +int i3c_hci_reset_and_restore(struct i3c_hci *hci); + +#endif diff --git a/drivers/i3c/master/mipi-i3c-hci/hci_quirks.c b/drivers/i3c/master/mipi-i3c-hci/hci_quirks.c new file mode 100644 index 000000000..3b9c6e76c --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/hci_quirks.c @@ -0,0 +1,44 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * I3C HCI Quirks + * + * Copyright 2024 Advanced Micro Devices, Inc. + * + * Authors: Shyam Sundar S K <Shyam-sundar.S-k@amd.com> + * Guruvendra Punugupati <Guruvendra.Punugupati@amd.com> + */ + +#include <linux/i3c/master.h> +#include "hci.h" + +/* Timing registers */ +#define HCI_SCL_I3C_OD_TIMING 0x214 +#define HCI_SCL_I3C_PP_TIMING 0x218 +#define HCI_SDA_HOLD_SWITCH_DLY_TIMING 0x230 + +/* Timing values to configure 9MHz frequency */ +#define AMD_SCL_I3C_OD_TIMING 0x00cf00cf +#define AMD_SCL_I3C_PP_TIMING 0x00160016 + +#define QUEUE_THLD_CTRL 0xD0 + +void amd_set_od_pp_timing(struct i3c_hci *hci) +{ + u32 data; + + reg_write(HCI_SCL_I3C_OD_TIMING, AMD_SCL_I3C_OD_TIMING); + reg_write(HCI_SCL_I3C_PP_TIMING, AMD_SCL_I3C_PP_TIMING); + data = reg_read(HCI_SDA_HOLD_SWITCH_DLY_TIMING); + /* Configure maximum TX hold time */ + data |= W0_MASK(18, 16); + reg_write(HCI_SDA_HOLD_SWITCH_DLY_TIMING, data); +} + +void amd_set_resp_buf_thld(struct i3c_hci *hci) +{ + u32 data; + + data = reg_read(QUEUE_THLD_CTRL); + data = data & ~W0_MASK(15, 8); + reg_write(QUEUE_THLD_CTRL, data); +} diff --git a/drivers/i3c/master/mipi-i3c-hci/ibi.h b/drivers/i3c/master/mipi-i3c-hci/ibi.h new file mode 100644 index 000000000..073ca67b7 --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/ibi.h @@ -0,0 +1,31 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Common IBI related stuff + */ + +#ifndef IBI_H +#define IBI_H + +/* + * IBI Status Descriptor bits + */ +#define IBI_STS BIT(31) +#define IBI_ERROR BIT(30) +#define IBI_STATUS_TYPE BIT(29) +#define IBI_HW_CONTEXT GENMASK(28, 26) +#define IBI_TS BIT(25) +#define IBI_LAST_STATUS BIT(24) +#define IBI_CHUNKS GENMASK(23, 16) +#define IBI_ID GENMASK(15, 8) +#define IBI_TARGET_ADDR GENMASK(15, 9) +#define IBI_TARGET_RNW BIT(8) +#define IBI_DATA_LENGTH GENMASK(7, 0) + +/* handy helpers */ +struct i3c_dev_desc *i3c_hci_addr_to_dev(struct i3c_hci *hci, unsigned int addr); + +#endif diff --git a/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c b/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c new file mode 100644 index 000000000..ab595661d --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/mipi-i3c-hci-pci.c @@ -0,0 +1,524 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * PCI glue code for MIPI I3C HCI driver + * + * Copyright (C) 2024 Intel Corporation + * + * Author: Jarkko Nikula <jarkko.nikula@linux.intel.com> + */ +#include <linux/acpi.h> +#include <linux/bitfield.h> +#include <linux/debugfs.h> +#include <linux/i3c/master.h> +#include <linux/idr.h> +#include <linux/iopoll.h> +#include <linux/kernel.h> +#include <linux/mfd/core.h> +#include <linux/module.h> +#include <linux/pci.h> +#include <linux/platform_data/mipi-i3c-hci.h> +#include <linux/platform_device.h> +#include <linux/pm_qos.h> +#include <linux/pm_runtime.h> + +#include "hci.h" + +/* + * There can up to 15 instances, but implementations have at most 2 at this + * time. + */ +#define INST_MAX 2 + +struct mipi_i3c_hci_pci_instance { + struct device *dev; + bool operational; +}; + +struct mipi_i3c_hci_pci { + struct pci_dev *pci; + void __iomem *base; + const struct mipi_i3c_hci_pci_info *info; + struct mipi_i3c_hci_pci_instance instance[INST_MAX]; + void *private; +}; + +struct mipi_i3c_hci_pci_info { + int (*init)(struct mipi_i3c_hci_pci *hci); + void (*exit)(struct mipi_i3c_hci_pci *hci); + const char *name; + int id[INST_MAX]; + u32 instance_offset[INST_MAX]; + int instance_count; + bool control_instance_pm; +}; + +#define INTEL_PRIV_OFFSET 0x2b0 +#define INTEL_RESETS 0x04 +#define INTEL_RESETS_RESET BIT(0) +#define INTEL_RESETS_RESET_DONE BIT(1) +#define INTEL_RESETS_TIMEOUT_US (10 * USEC_PER_MSEC) + +#define INTEL_ACTIVELTR 0x0c +#define INTEL_IDLELTR 0x10 + +#define INTEL_LTR_REQ BIT(15) +#define INTEL_LTR_SCALE_MASK GENMASK(11, 10) +#define INTEL_LTR_SCALE_1US FIELD_PREP(INTEL_LTR_SCALE_MASK, 2) +#define INTEL_LTR_SCALE_32US FIELD_PREP(INTEL_LTR_SCALE_MASK, 3) +#define INTEL_LTR_VALUE_MASK GENMASK(9, 0) + +struct intel_host { + void __iomem *priv; + u32 active_ltr; + u32 idle_ltr; + struct dentry *debugfs_root; +}; + +static void intel_cache_ltr(struct intel_host *host) +{ + host->active_ltr = readl(host->priv + INTEL_ACTIVELTR); + host->idle_ltr = readl(host->priv + INTEL_IDLELTR); +} + +static void intel_ltr_set(struct device *dev, s32 val) +{ + struct mipi_i3c_hci_pci *hci = dev_get_drvdata(dev); + struct intel_host *host = hci->private; + u32 ltr; + + /* + * Program latency tolerance (LTR) accordingly what has been asked + * by the PM QoS layer or disable it in case we were passed + * negative value or PM_QOS_LATENCY_ANY. + */ + ltr = readl(host->priv + INTEL_ACTIVELTR); + + if (val == PM_QOS_LATENCY_ANY || val < 0) { + ltr &= ~INTEL_LTR_REQ; + } else { + ltr |= INTEL_LTR_REQ; + ltr &= ~INTEL_LTR_SCALE_MASK; + ltr &= ~INTEL_LTR_VALUE_MASK; + + if (val > INTEL_LTR_VALUE_MASK) { + val >>= 5; + if (val > INTEL_LTR_VALUE_MASK) + val = INTEL_LTR_VALUE_MASK; + ltr |= INTEL_LTR_SCALE_32US | val; + } else { + ltr |= INTEL_LTR_SCALE_1US | val; + } + } + + if (ltr == host->active_ltr) + return; + + writel(ltr, host->priv + INTEL_ACTIVELTR); + writel(ltr, host->priv + INTEL_IDLELTR); + + /* Cache the values into intel_host structure */ + intel_cache_ltr(host); +} + +static void intel_ltr_expose(struct device *dev) +{ + dev->power.set_latency_tolerance = intel_ltr_set; + dev_pm_qos_expose_latency_tolerance(dev); +} + +static void intel_ltr_hide(struct device *dev) +{ + dev_pm_qos_hide_latency_tolerance(dev); + dev->power.set_latency_tolerance = NULL; +} + +static void intel_add_debugfs(struct mipi_i3c_hci_pci *hci) +{ + struct dentry *dir = debugfs_create_dir(dev_name(&hci->pci->dev), NULL); + struct intel_host *host = hci->private; + + intel_cache_ltr(host); + + host->debugfs_root = dir; + debugfs_create_x32("active_ltr", 0444, dir, &host->active_ltr); + debugfs_create_x32("idle_ltr", 0444, dir, &host->idle_ltr); +} + +static void intel_remove_debugfs(struct mipi_i3c_hci_pci *hci) +{ + struct intel_host *host = hci->private; + + debugfs_remove_recursive(host->debugfs_root); +} + +static void intel_reset(void __iomem *priv) +{ + u32 reg; + + /* Assert reset, wait for completion and release reset */ + writel(0, priv + INTEL_RESETS); + readl_poll_timeout(priv + INTEL_RESETS, reg, + reg & INTEL_RESETS_RESET_DONE, 0, + INTEL_RESETS_TIMEOUT_US); + writel(INTEL_RESETS_RESET, priv + INTEL_RESETS); +} + +static int intel_i3c_init(struct mipi_i3c_hci_pci *hci) +{ + struct intel_host *host = devm_kzalloc(&hci->pci->dev, sizeof(*host), GFP_KERNEL); + void __iomem *priv = hci->base + INTEL_PRIV_OFFSET; + + if (!host) + return -ENOMEM; + + dma_set_mask_and_coherent(&hci->pci->dev, DMA_BIT_MASK(64)); + + hci->pci->d3cold_delay = 0; + hci->pci->d3hot_delay = 0; + + hci->private = host; + host->priv = priv; + + intel_reset(priv); + + intel_ltr_expose(&hci->pci->dev); + intel_add_debugfs(hci); + + return 0; +} + +static void intel_i3c_exit(struct mipi_i3c_hci_pci *hci) +{ + intel_remove_debugfs(hci); + intel_ltr_hide(&hci->pci->dev); +} + +static const struct mipi_i3c_hci_pci_info intel_mi_1_info = { + .init = intel_i3c_init, + .exit = intel_i3c_exit, + .name = "intel-lpss-i3c", + .id = {0, 1}, + .instance_offset = {0, 0x400}, + .instance_count = 2, + .control_instance_pm = true, +}; + +static const struct mipi_i3c_hci_pci_info intel_mi_2_info = { + .init = intel_i3c_init, + .exit = intel_i3c_exit, + .name = "intel-lpss-i3c", + .id = {2, 3}, + .instance_offset = {0, 0x400}, + .instance_count = 2, + .control_instance_pm = true, +}; + +static const struct mipi_i3c_hci_pci_info intel_si_2_info = { + .init = intel_i3c_init, + .exit = intel_i3c_exit, + .name = "intel-lpss-i3c", + .id = {2}, + .instance_offset = {0}, + .instance_count = 1, + .control_instance_pm = true, +}; + +static const struct mipi_i3c_hci_pci_info amd_pt_info = { + .name = "amd-pt-i3c-hci", + .id = {0}, + .instance_offset = {0}, + .instance_count = 1, +}; + +static int mipi_i3c_hci_pci_find_instance(struct mipi_i3c_hci_pci *hci, struct device *dev) +{ + for (int i = 0; i < INST_MAX; i++) { + if (!hci->instance[i].dev) + hci->instance[i].dev = dev; + if (hci->instance[i].dev == dev) + return i; + } + + return -1; +} + +#define HC_CONTROL 0x04 +#define HC_CONTROL_BUS_ENABLE BIT(31) + +static bool __mipi_i3c_hci_pci_is_operational(struct device *dev) +{ + const struct mipi_i3c_hci_platform_data *pdata = dev->platform_data; + u32 hc_control = readl(pdata->base_regs + HC_CONTROL); + + return hc_control & HC_CONTROL_BUS_ENABLE; +} + +static bool mipi_i3c_hci_pci_is_operational(struct device *dev, bool update) +{ + struct mipi_i3c_hci_pci *hci = dev_get_drvdata(dev->parent); + int pos = mipi_i3c_hci_pci_find_instance(hci, dev); + + if (pos < 0) { + dev_err(dev, "%s: I3C instance not found\n", __func__); + return false; + } + + if (update) + hci->instance[pos].operational = __mipi_i3c_hci_pci_is_operational(dev); + + return hci->instance[pos].operational; +} + +struct mipi_i3c_hci_pci_pm_data { + struct device *dev[INST_MAX]; + int dev_cnt; + bool can_wakeup; + bool may_wakeup; +}; + +static bool mipi_i3c_hci_pci_is_mfd(struct device *dev) +{ + return dev_is_platform(dev) && mfd_get_cell(to_platform_device(dev)); +} + +static bool mipi_i3c_hci_pci_any_wakeup_enabled(struct device *dev) +{ + struct i3c_hci *hci = dev_get_drvdata(dev); + + return i3c_master_has_wakeup_enabled_devs(&hci->master); +} + +static int mipi_i3c_hci_pci_suspend_instance(struct device *dev, void *data) +{ + struct mipi_i3c_hci_pci_pm_data *pm_data = data; + int ret; + + if (!mipi_i3c_hci_pci_is_mfd(dev) || + !mipi_i3c_hci_pci_is_operational(dev, true)) + return 0; + + ret = i3c_hci_rpm_suspend(dev); + if (ret) + return ret; + + pm_data->dev[pm_data->dev_cnt++] = dev; + + if (pm_data->can_wakeup && mipi_i3c_hci_pci_any_wakeup_enabled(dev)) + pm_data->may_wakeup = true; + + return 0; +} + +static int mipi_i3c_hci_pci_resume_instance(struct device *dev, void *data) +{ + struct mipi_i3c_hci_pci_pm_data *pm_data = data; + int ret; + + if (!mipi_i3c_hci_pci_is_mfd(dev) || + !mipi_i3c_hci_pci_is_operational(dev, false)) + return 0; + + ret = i3c_hci_rpm_resume(dev); + if (ret) + return ret; + + pm_data->dev[pm_data->dev_cnt++] = dev; + + return 0; +} + +static int mipi_i3c_hci_pci_suspend(struct device *dev) +{ + struct mipi_i3c_hci_pci *hci = dev_get_drvdata(dev); + struct mipi_i3c_hci_pci_pm_data pm_data = {}; + int ret; + + if (!hci->info->control_instance_pm) + return 0; + + pm_data.can_wakeup = device_can_wakeup(dev); + + ret = device_for_each_child_reverse(dev, &pm_data, mipi_i3c_hci_pci_suspend_instance); + if (ret) { + for (int i = 0; i < pm_data.dev_cnt; i++) + i3c_hci_rpm_resume(pm_data.dev[i]); + return ret; + } + + if (device_may_wakeup(dev) != pm_data.may_wakeup) + device_set_wakeup_enable(dev, pm_data.may_wakeup); + + return 0; +} + +static int mipi_i3c_hci_pci_resume(struct device *dev) +{ + struct mipi_i3c_hci_pci *hci = dev_get_drvdata(dev); + struct mipi_i3c_hci_pci_pm_data pm_data = {}; + int ret; + + if (!hci->info->control_instance_pm) + return 0; + + ret = device_for_each_child(dev, &pm_data, mipi_i3c_hci_pci_resume_instance); + if (ret) + for (int i = 0; i < pm_data.dev_cnt; i++) + i3c_hci_rpm_suspend(pm_data.dev[i]); + + return ret; +} + +static void mipi_i3c_hci_pci_rpm_allow(struct device *dev) +{ + pm_runtime_put(dev); + pm_runtime_allow(dev); +} + +static void mipi_i3c_hci_pci_rpm_forbid(struct device *dev) +{ + pm_runtime_forbid(dev); + pm_runtime_get_sync(dev); +} + +struct mipi_i3c_hci_pci_cell_data { + struct mipi_i3c_hci_platform_data pdata; + struct resource res; +}; + +static void mipi_i3c_hci_pci_setup_cell(struct mipi_i3c_hci_pci *hci, int idx, + struct mipi_i3c_hci_pci_cell_data *data, + struct mfd_cell *cell) +{ + data->pdata.base_regs = hci->base + hci->info->instance_offset[idx]; + + data->res = DEFINE_RES_IRQ(0); + + cell->name = hci->info->name; + cell->id = hci->info->id[idx]; + cell->platform_data = &data->pdata; + cell->pdata_size = sizeof(data->pdata); + cell->num_resources = 1; + cell->resources = &data->res; +} + +#define mipi_i3c_hci_pci_alloc(h, x) kzalloc_objs(*(x), (h)->info->instance_count) + +static int mipi_i3c_hci_pci_add_instances(struct mipi_i3c_hci_pci *hci) +{ + struct mipi_i3c_hci_pci_cell_data *data __free(kfree) = mipi_i3c_hci_pci_alloc(hci, data); + struct mfd_cell *cells __free(kfree) = mipi_i3c_hci_pci_alloc(hci, cells); + int irq = pci_irq_vector(hci->pci, 0); + int nr = hci->info->instance_count; + + if (!cells || !data) + return -ENOMEM; + + for (int i = 0; i < nr; i++) + mipi_i3c_hci_pci_setup_cell(hci, i, data + i, cells + i); + + return mfd_add_devices(&hci->pci->dev, 0, cells, nr, NULL, irq, NULL); +} + +static int mipi_i3c_hci_pci_probe(struct pci_dev *pci, + const struct pci_device_id *id) +{ + struct mipi_i3c_hci_pci *hci; + int ret; + + hci = devm_kzalloc(&pci->dev, sizeof(*hci), GFP_KERNEL); + if (!hci) + return -ENOMEM; + + hci->pci = pci; + + ret = pcim_enable_device(pci); + if (ret) + return ret; + + pci_set_master(pci); + + hci->base = pcim_iomap_region(pci, 0, pci_name(pci)); + if (IS_ERR(hci->base)) + return PTR_ERR(hci->base); + + ret = pci_alloc_irq_vectors(pci, 1, 1, PCI_IRQ_ALL_TYPES); + if (ret < 0) + return ret; + + hci->info = (const struct mipi_i3c_hci_pci_info *)id->driver_data; + + ret = hci->info->init ? hci->info->init(hci) : 0; + if (ret) + return ret; + + ret = mipi_i3c_hci_pci_add_instances(hci); + if (ret) + goto err_exit; + + pci_set_drvdata(pci, hci); + + mipi_i3c_hci_pci_rpm_allow(&pci->dev); + + return 0; + +err_exit: + if (hci->info->exit) + hci->info->exit(hci); + return ret; +} + +static void mipi_i3c_hci_pci_remove(struct pci_dev *pci) +{ + struct mipi_i3c_hci_pci *hci = pci_get_drvdata(pci); + + if (hci->info->exit) + hci->info->exit(hci); + + mipi_i3c_hci_pci_rpm_forbid(&pci->dev); + + mfd_remove_devices(&pci->dev); +} + +/* PM ops must exist for PCI to put a device to a low power state */ +static const struct dev_pm_ops mipi_i3c_hci_pci_pm_ops = { + RUNTIME_PM_OPS(mipi_i3c_hci_pci_suspend, mipi_i3c_hci_pci_resume, NULL) + SYSTEM_SLEEP_PM_OPS(mipi_i3c_hci_pci_suspend, mipi_i3c_hci_pci_resume) +}; + +static const struct pci_device_id mipi_i3c_hci_pci_devices[] = { + /* Wildcat Lake-U */ + { PCI_VDEVICE(INTEL, 0x4d7c), .driver_data = (kernel_ulong_t)&intel_mi_1_info }, + { PCI_VDEVICE(INTEL, 0x4d6f), .driver_data = (kernel_ulong_t)&intel_si_2_info }, + /* Panther Lake-H */ + { PCI_VDEVICE(INTEL, 0xe37c), .driver_data = (kernel_ulong_t)&intel_mi_1_info }, + { PCI_VDEVICE(INTEL, 0xe36f), .driver_data = (kernel_ulong_t)&intel_si_2_info }, + /* Panther Lake-P */ + { PCI_VDEVICE(INTEL, 0xe47c), .driver_data = (kernel_ulong_t)&intel_mi_1_info }, + { PCI_VDEVICE(INTEL, 0xe46f), .driver_data = (kernel_ulong_t)&intel_si_2_info }, + /* Nova Lake-S */ + { PCI_VDEVICE(INTEL, 0x6e2c), .driver_data = (kernel_ulong_t)&intel_mi_1_info }, + { PCI_VDEVICE(INTEL, 0x6e2d), .driver_data = (kernel_ulong_t)&intel_mi_2_info }, + /* Nova Lake-H */ + { PCI_VDEVICE(INTEL, 0xd37c), .driver_data = (kernel_ulong_t)&intel_mi_1_info }, + { PCI_VDEVICE(INTEL, 0xd36f), .driver_data = (kernel_ulong_t)&intel_mi_2_info }, + /* AMD_PT */ + { PCI_VDEVICE(AMD, 0x444c), .driver_data = (kernel_ulong_t)&amd_pt_info }, + { } +}; +MODULE_DEVICE_TABLE(pci, mipi_i3c_hci_pci_devices); + +static struct pci_driver mipi_i3c_hci_pci_driver = { + .name = "mipi_i3c_hci_pci", + .id_table = mipi_i3c_hci_pci_devices, + .probe = mipi_i3c_hci_pci_probe, + .remove = mipi_i3c_hci_pci_remove, + .driver = { + .pm = pm_ptr(&mipi_i3c_hci_pci_pm_ops) + }, +}; + +module_pci_driver(mipi_i3c_hci_pci_driver); + +MODULE_AUTHOR("Jarkko Nikula <jarkko.nikula@intel.com>"); +MODULE_LICENSE("GPL"); +MODULE_DESCRIPTION("MIPI I3C HCI driver on PCI bus"); diff --git a/drivers/i3c/master/mipi-i3c-hci/pio.c b/drivers/i3c/master/mipi-i3c-hci/pio.c new file mode 100644 index 000000000..439578a6e --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/pio.c @@ -0,0 +1,1141 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + */ + +#include <linux/bitfield.h> +#include <linux/device.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/io.h> + +#include "hci.h" +#include "cmd.h" +#include "ibi.h" + +/* + * PIO Access Area + */ + +#define pio_reg_read(r) readl(hci->PIO_regs + (PIO_##r)) +#define pio_reg_write(r, v) writel(v, hci->PIO_regs + (PIO_##r)) + +#define PIO_COMMAND_QUEUE_PORT 0x00 +#define PIO_RESPONSE_QUEUE_PORT 0x04 +#define PIO_XFER_DATA_PORT 0x08 +#define PIO_IBI_PORT 0x0c + +#define PIO_QUEUE_THLD_CTRL 0x10 +#define QUEUE_IBI_STATUS_THLD GENMASK(31, 24) +#define QUEUE_IBI_DATA_THLD GENMASK(23, 16) +#define QUEUE_RESP_BUF_THLD GENMASK(15, 8) +#define QUEUE_CMD_EMPTY_BUF_THLD GENMASK(7, 0) + +#define PIO_DATA_BUFFER_THLD_CTRL 0x14 +#define DATA_RX_START_THLD GENMASK(26, 24) +#define DATA_TX_START_THLD GENMASK(18, 16) +#define DATA_RX_BUF_THLD GENMASK(10, 8) +#define DATA_TX_BUF_THLD GENMASK(2, 0) + +#define PIO_QUEUE_SIZE 0x18 +#define TX_DATA_BUFFER_SIZE GENMASK(31, 24) +#define RX_DATA_BUFFER_SIZE GENMASK(23, 16) +#define IBI_STATUS_SIZE GENMASK(15, 8) +#define CR_QUEUE_SIZE GENMASK(7, 0) + +#define PIO_ALT_QUEUE_SIZE 0x1C +#define EXT_IBI_QUEUE_EN BIT(28) +#define ALT_RESP_QUEUE_EN BIT(24) +#define ALT_RESP_QUEUE_SIZE GENMASK(7, 0) + +#define PIO_INTR_STATUS 0x20 +#define PIO_INTR_STATUS_ENABLE 0x24 +#define PIO_INTR_SIGNAL_ENABLE 0x28 +#define PIO_INTR_FORCE 0x2c +#define STAT_TRANSFER_BLOCKED BIT(25) +#define STAT_PERR_RESP_UFLOW BIT(24) +#define STAT_PERR_CMD_OFLOW BIT(23) +#define STAT_PERR_IBI_UFLOW BIT(22) +#define STAT_PERR_RX_UFLOW BIT(21) +#define STAT_PERR_TX_OFLOW BIT(20) +#define STAT_ERR_RESP_QUEUE_FULL BIT(19) +#define STAT_WARN_RESP_QUEUE_FULL BIT(18) +#define STAT_ERR_IBI_QUEUE_FULL BIT(17) +#define STAT_WARN_IBI_QUEUE_FULL BIT(16) +#define STAT_ERR_RX_DATA_FULL BIT(15) +#define STAT_WARN_RX_DATA_FULL BIT(14) +#define STAT_ERR_TX_DATA_EMPTY BIT(13) +#define STAT_WARN_TX_DATA_EMPTY BIT(12) +#define STAT_TRANSFER_ERR BIT(9) +#define STAT_WARN_INS_STOP_MODE BIT(7) +#define STAT_TRANSFER_ABORT BIT(5) +#define STAT_RESP_READY BIT(4) +#define STAT_CMD_QUEUE_READY BIT(3) +#define STAT_IBI_STATUS_THLD BIT(2) +#define STAT_RX_THLD BIT(1) +#define STAT_TX_THLD BIT(0) + +#define PIO_CONTROL 0x30 +#define PIO_CONTROL_ABORT BIT(2) +#define PIO_CONTROL_RS BIT(1) +#define PIO_CONTROL_ENABLE BIT(0) + +#define PIO_QUEUE_CUR_STATUS 0x38 +#define CUR_IBI_Q_LEVEL GENMASK(28, 20) +#define CUR_RESP_Q_LEVEL GENMASK(18, 10) +#define CUR_CMD_Q_EMPTY_LEVEL GENMASK(8, 0) + +#define PIO_DATA_BUFFER_CUR_STATUS 0x3c +#define CUR_RX_BUF_LVL GENMASK(26, 16) +#define CUR_TX_BUF_LVL GENMASK(10, 0) + +/* + * Handy status bit combinations + */ + +#define STAT_LATENCY_WARNINGS (STAT_WARN_RESP_QUEUE_FULL | \ + STAT_WARN_IBI_QUEUE_FULL | \ + STAT_WARN_RX_DATA_FULL | \ + STAT_WARN_TX_DATA_EMPTY | \ + STAT_WARN_INS_STOP_MODE) + +#define STAT_LATENCY_ERRORS (STAT_ERR_RESP_QUEUE_FULL | \ + STAT_ERR_IBI_QUEUE_FULL | \ + STAT_ERR_RX_DATA_FULL | \ + STAT_ERR_TX_DATA_EMPTY) + +#define STAT_PROG_ERRORS (STAT_TRANSFER_BLOCKED | \ + STAT_PERR_RESP_UFLOW | \ + STAT_PERR_CMD_OFLOW | \ + STAT_PERR_IBI_UFLOW | \ + STAT_PERR_RX_UFLOW | \ + STAT_PERR_TX_OFLOW) + +#define STAT_ALL_ERRORS (STAT_TRANSFER_ABORT | \ + STAT_TRANSFER_ERR | \ + STAT_LATENCY_ERRORS | \ + STAT_PROG_ERRORS) + +struct hci_pio_dev_ibi_data { + struct i3c_generic_ibi_pool *pool; + unsigned int max_len; +}; + +struct hci_pio_ibi_data { + struct i3c_ibi_slot *slot; + void *data_ptr; + unsigned int addr; + unsigned int seg_len, seg_cnt; + unsigned int max_len; + bool last_seg; +}; + +struct hci_pio_data { + struct hci_xfer *curr_xfer, *xfer_queue; + struct hci_xfer *curr_rx, *rx_queue; + struct hci_xfer *curr_tx, *tx_queue; + struct hci_xfer *curr_resp, *resp_queue; + struct hci_pio_ibi_data ibi; + unsigned int rx_thresh_size, tx_thresh_size; + unsigned int max_ibi_thresh; + u32 reg_queue_thresh; + u32 enabled_irqs; +}; + +static void __hci_pio_init(struct i3c_hci *hci, u32 *size_val_ptr) +{ + u32 val, size_val, rx_thresh, tx_thresh, ibi_val; + struct hci_pio_data *pio = hci->io_data; + + size_val = pio_reg_read(QUEUE_SIZE); + if (size_val_ptr) + *size_val_ptr = size_val; + + /* + * Let's initialize data thresholds to half of the actual FIFO size. + * The start thresholds aren't used (set to 0) as the FIFO is always + * serviced before the corresponding command is queued. + */ + rx_thresh = FIELD_GET(RX_DATA_BUFFER_SIZE, size_val); + tx_thresh = FIELD_GET(TX_DATA_BUFFER_SIZE, size_val); + if (hci->version_major == 1) { + /* those are expressed as 2^[n+1), so just sub 1 if not 0 */ + if (rx_thresh) + rx_thresh -= 1; + if (tx_thresh) + tx_thresh -= 1; + pio->rx_thresh_size = 2 << rx_thresh; + pio->tx_thresh_size = 2 << tx_thresh; + } else { + /* size is 2^(n+1) and threshold is 2^n i.e. already halved */ + pio->rx_thresh_size = 1 << rx_thresh; + pio->tx_thresh_size = 1 << tx_thresh; + } + val = FIELD_PREP(DATA_RX_BUF_THLD, rx_thresh) | + FIELD_PREP(DATA_TX_BUF_THLD, tx_thresh); + pio_reg_write(DATA_BUFFER_THLD_CTRL, val); + + /* + * Let's raise an interrupt as soon as there is one free cmd slot + * or one available response or IBI. For IBI data let's use half the + * IBI queue size within allowed bounds. + */ + ibi_val = FIELD_GET(IBI_STATUS_SIZE, size_val); + /* Adjust actual IBI queue size based on v1.2 ALT_QUEUE_SIZE */ + if (hci_version_at_least(hci, 1, 2)) { + u32 alt_val = pio_reg_read(ALT_QUEUE_SIZE); + + if (alt_val & EXT_IBI_QUEUE_EN) + ibi_val *= 8; + } + + pio->max_ibi_thresh = clamp_val(ibi_val/2, 1, 63); + val = FIELD_PREP(QUEUE_IBI_STATUS_THLD, 1) | + FIELD_PREP(QUEUE_IBI_DATA_THLD, pio->max_ibi_thresh) | + FIELD_PREP(QUEUE_RESP_BUF_THLD, 1) | + FIELD_PREP(QUEUE_CMD_EMPTY_BUF_THLD, 1); + pio_reg_write(QUEUE_THLD_CTRL, val); + pio->reg_queue_thresh = val; + + /* Disable all IRQs but allow all status bits */ + pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); + pio_reg_write(INTR_STATUS_ENABLE, 0xffffffff); + + /* + * Always accept error interrupts and IBI threshold interrupt + * (will be activated on first xfer). + */ + pio->enabled_irqs = STAT_ALL_ERRORS | STAT_IBI_STATUS_THLD; + + /* MIPI I3C HCI v1.2 requires explicitly enabling and starting PIO queues */ + if (hci_version_at_least(hci, 1, 2)) { + u32 ctl_val = pio_reg_read(CONTROL); + + if (!(ctl_val & PIO_CONTROL_ENABLE)) { + ctl_val |= PIO_CONTROL_ENABLE; + pio_reg_write(CONTROL, ctl_val); + } + pio_reg_write(CONTROL, ctl_val | PIO_CONTROL_RS); + } +} + +static void hci_pio_suspend(struct i3c_hci *hci) +{ + pio_reg_write(INTR_SIGNAL_ENABLE, 0); + + i3c_hci_sync_irq_inactive(hci); +} + +static void hci_pio_resume(struct i3c_hci *hci) +{ + __hci_pio_init(hci, NULL); +} + +static int hci_pio_init(struct i3c_hci *hci) +{ + struct hci_pio_data *pio; + u32 size_val; + u32 cmd_sz, resp_sz, ibi_val; + + pio = devm_kzalloc(hci->master.dev.parent, sizeof(*pio), GFP_KERNEL); + if (!pio) + return -ENOMEM; + + hci->io_data = pio; + + __hci_pio_init(hci, &size_val); + + cmd_sz = FIELD_GET(CR_QUEUE_SIZE, size_val); + resp_sz = cmd_sz; + ibi_val = FIELD_GET(IBI_STATUS_SIZE, size_val); + + /* MIPI I3C HCI v1.2 supports alternate RESP/IBI queue size */ + if (hci_version_at_least(hci, 1, 2)) { + u32 alt_val = pio_reg_read(ALT_QUEUE_SIZE); + + if (alt_val & ALT_RESP_QUEUE_EN) + resp_sz = FIELD_GET(ALT_RESP_QUEUE_SIZE, alt_val); + if (alt_val & EXT_IBI_QUEUE_EN) + ibi_val *= 8; + } + + dev_dbg(&hci->master.dev, "CMD FIFO = %u, RESP FIFO = %u entries\n", + cmd_sz, resp_sz); + dev_dbg(&hci->master.dev, "IBI FIFO = %u bytes\n", + 4 * ibi_val); + dev_dbg(&hci->master.dev, "RX data FIFO = %d bytes\n", + 4 * (2 << FIELD_GET(RX_DATA_BUFFER_SIZE, size_val))); + dev_dbg(&hci->master.dev, "TX data FIFO = %d bytes\n", + 4 * (2 << FIELD_GET(TX_DATA_BUFFER_SIZE, size_val))); + + return 0; +} + +static void hci_pio_cleanup(struct i3c_hci *hci) +{ + struct hci_pio_data *pio = hci->io_data; + + pio_reg_write(INTR_SIGNAL_ENABLE, 0x0); + + i3c_hci_sync_irq_inactive(hci); + + if (pio) { + dev_dbg(&hci->master.dev, "status = %#x/%#x", + pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); + BUG_ON(pio->curr_xfer); + BUG_ON(pio->curr_rx); + BUG_ON(pio->curr_tx); + BUG_ON(pio->curr_resp); + /* MIPI I3C HCI v1.2 requires explicitly stopping and disabling PIO queues */ + if (hci_version_at_least(hci, 1, 2)) + pio_reg_write(CONTROL, 0x0); + } +} + +static void hci_pio_write_cmd(struct i3c_hci *hci, struct hci_xfer *xfer) +{ + dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", + 0, xfer->cmd_desc[0]); + dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", + 1, xfer->cmd_desc[1]); + pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[0]); + pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[1]); + if (hci->cmd == &mipi_i3c_hci_cmd_v2) { + dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", + 2, xfer->cmd_desc[2]); + dev_dbg(&hci->master.dev, "cmd_desc[%d] = 0x%08x", + 3, xfer->cmd_desc[3]); + pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[2]); + pio_reg_write(COMMAND_QUEUE_PORT, xfer->cmd_desc[3]); + } +} + +static bool hci_pio_do_rx(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_xfer *xfer = pio->curr_rx; + unsigned int nr_words; + u32 *p; + + p = xfer->data; + p += (xfer->data_len - xfer->data_left) / 4; + + while (xfer->data_left >= 4) { + /* bail out if FIFO hasn't reached the threshold value yet */ + if (!(pio_reg_read(INTR_STATUS) & STAT_RX_THLD)) + return false; + nr_words = min(xfer->data_left / 4, pio->rx_thresh_size); + /* extract data from FIFO */ + xfer->data_left -= nr_words * 4; + dev_dbg(&hci->master.dev, "now %d left %d", + nr_words * 4, xfer->data_left); + while (nr_words--) + *p++ = pio_reg_read(XFER_DATA_PORT); + } + + /* trailing data is retrieved upon response reception */ + return !xfer->data_left; +} + +static void hci_pio_do_trailing_rx(struct i3c_hci *hci, + struct hci_pio_data *pio, unsigned int count) +{ + struct hci_xfer *xfer = pio->curr_rx; + u32 *p; + + dev_dbg(&hci->master.dev, "%d remaining", count); + + p = xfer->data; + p += (xfer->data_len - xfer->data_left) / 4; + + if (count >= 4) { + unsigned int nr_words = count / 4; + /* extract data from FIFO */ + xfer->data_left -= nr_words * 4; + dev_dbg(&hci->master.dev, "now %d left %d", + nr_words * 4, xfer->data_left); + while (nr_words--) + *p++ = pio_reg_read(XFER_DATA_PORT); + } + + count &= 3; + if (count) { + /* + * There are trailing bytes in the last word. + * Fetch it and extract bytes in an endian independent way. + * Unlike the TX case, we must not write memory past the + * end of the destination buffer. + */ + u8 *p_byte = (u8 *)p; + u32 data = pio_reg_read(XFER_DATA_PORT); + + xfer->data_word_before_partial = data; + xfer->data_left -= count; + data = (__force u32) cpu_to_le32(data); + while (count--) { + *p_byte++ = data; + data >>= 8; + } + } +} + +static bool hci_pio_do_tx(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_xfer *xfer = pio->curr_tx; + unsigned int nr_words; + u32 *p; + + p = xfer->data; + p += (xfer->data_len - xfer->data_left) / 4; + + while (xfer->data_left >= 4) { + /* bail out if FIFO free space is below set threshold */ + if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD)) + return false; + /* we can fill up to that TX threshold */ + nr_words = min(xfer->data_left / 4, pio->tx_thresh_size); + /* push data into the FIFO */ + xfer->data_left -= nr_words * 4; + dev_dbg(&hci->master.dev, "now %d left %d", + nr_words * 4, xfer->data_left); + while (nr_words--) + pio_reg_write(XFER_DATA_PORT, *p++); + } + + if (xfer->data_left) { + /* + * There are trailing bytes to send. We can simply load + * them from memory as a word which will keep those bytes + * in their proper place even on a BE system. This will + * also get some bytes past the actual buffer but no one + * should care as they won't be sent out. + */ + if (!(pio_reg_read(INTR_STATUS) & STAT_TX_THLD)) + return false; + dev_dbg(&hci->master.dev, "trailing %d", xfer->data_left); + pio_reg_write(XFER_DATA_PORT, *p); + xfer->data_left = 0; + } + + return true; +} + +static bool hci_pio_process_rx(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + while (pio->curr_rx && hci_pio_do_rx(hci, pio)) + pio->curr_rx = pio->curr_rx->next_data; + return !pio->curr_rx; +} + +static bool hci_pio_process_tx(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + while (pio->curr_tx && hci_pio_do_tx(hci, pio)) + pio->curr_tx = pio->curr_tx->next_data; + return !pio->curr_tx; +} + +static void hci_pio_queue_data(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_xfer *xfer = pio->curr_xfer; + struct hci_xfer *prev_queue_tail; + + if (!xfer->data) { + xfer->data_len = xfer->data_left = 0; + return; + } + + if (xfer->rnw) { + prev_queue_tail = pio->rx_queue; + pio->rx_queue = xfer; + if (pio->curr_rx) { + prev_queue_tail->next_data = xfer; + } else { + pio->curr_rx = xfer; + if (!hci_pio_process_rx(hci, pio)) + pio->enabled_irqs |= STAT_RX_THLD; + } + } else { + prev_queue_tail = pio->tx_queue; + pio->tx_queue = xfer; + if (pio->curr_tx) { + prev_queue_tail->next_data = xfer; + } else { + pio->curr_tx = xfer; + if (!hci_pio_process_tx(hci, pio)) + pio->enabled_irqs |= STAT_TX_THLD; + } + } +} + +static void hci_pio_push_to_next_rx(struct i3c_hci *hci, struct hci_xfer *xfer, + unsigned int words_to_keep) +{ + u32 *from = xfer->data; + u32 from_last; + unsigned int received, count; + + received = (xfer->data_len - xfer->data_left) / 4; + if ((xfer->data_len - xfer->data_left) & 3) { + from_last = xfer->data_word_before_partial; + received += 1; + } else { + from_last = from[received]; + } + from += words_to_keep; + count = received - words_to_keep; + + while (count) { + unsigned int room, left, chunk, bytes_to_move; + u32 last_word; + + xfer = xfer->next_data; + if (!xfer) { + dev_err(&hci->master.dev, "pushing RX data to unexistent xfer\n"); + return; + } + + room = DIV_ROUND_UP(xfer->data_len, 4); + left = DIV_ROUND_UP(xfer->data_left, 4); + chunk = min(count, room); + if (chunk > left) { + hci_pio_push_to_next_rx(hci, xfer, chunk - left); + left = chunk; + xfer->data_left = left * 4; + } + + bytes_to_move = xfer->data_len - xfer->data_left; + if (bytes_to_move & 3) { + /* preserve word to become partial */ + u32 *p = xfer->data; + + xfer->data_word_before_partial = p[bytes_to_move / 4]; + } + memmove(xfer->data + chunk, xfer->data, bytes_to_move); + + /* treat last word specially because of partial word issues */ + chunk -= 1; + + memcpy(xfer->data, from, chunk * 4); + xfer->data_left -= chunk * 4; + from += chunk; + count -= chunk; + + last_word = (count == 1) ? from_last : *from++; + if (xfer->data_left < 4) { + /* + * Like in hci_pio_do_trailing_rx(), preserve original + * word to be stored partially then store bytes it + * in an endian independent way. + */ + u8 *p_byte = xfer->data; + + p_byte += chunk * 4; + xfer->data_word_before_partial = last_word; + last_word = (__force u32) cpu_to_le32(last_word); + while (xfer->data_left--) { + *p_byte++ = last_word; + last_word >>= 8; + } + } else { + u32 *p = xfer->data; + + p[chunk] = last_word; + xfer->data_left -= 4; + } + count--; + } +} + +static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio, + u32 status); + +static bool hci_pio_process_resp(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + while (pio->curr_resp && + (pio_reg_read(INTR_STATUS) & STAT_RESP_READY)) { + struct hci_xfer *xfer = pio->curr_resp; + u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT); + unsigned int tid = RESP_TID(resp); + + dev_dbg(&hci->master.dev, "resp = 0x%08x", resp); + if (tid != xfer->cmd_tid) { + dev_err(&hci->master.dev, + "response tid=%d when expecting %d\n", + tid, xfer->cmd_tid); + /* let's pretend it is a prog error... any of them */ + hci_pio_err(hci, pio, STAT_PROG_ERRORS); + return false; + } + xfer->response = resp; + + if (pio->curr_rx == xfer) { + /* + * Response availability implies RX completion. + * Retrieve trailing RX data if any. + * Note that short reads are possible. + */ + unsigned int received, expected, to_keep; + + received = xfer->data_len - xfer->data_left; + expected = RESP_DATA_LENGTH(xfer->response); + if (expected > received) { + hci_pio_do_trailing_rx(hci, pio, + expected - received); + } else if (received > expected) { + /* we consumed data meant for next xfer */ + to_keep = DIV_ROUND_UP(expected, 4); + hci_pio_push_to_next_rx(hci, xfer, to_keep); + } + + /* then process the RX list pointer */ + if (hci_pio_process_rx(hci, pio)) + pio->enabled_irqs &= ~STAT_RX_THLD; + } + + /* + * We're about to give back ownership of the xfer structure + * to the waiting instance. Make sure no reference to it + * still exists. + */ + if (pio->curr_rx == xfer) { + dev_dbg(&hci->master.dev, "short RX ?"); + pio->curr_rx = pio->curr_rx->next_data; + } else if (pio->curr_tx == xfer) { + dev_dbg(&hci->master.dev, "short TX ?"); + pio->curr_tx = pio->curr_tx->next_data; + } else if (xfer->data_left) { + dev_dbg(&hci->master.dev, + "PIO xfer count = %d after response", + xfer->data_left); + } + + pio->curr_resp = xfer->next_resp; + if (xfer->completion) + complete(xfer->completion); + } + return !pio->curr_resp; +} + +static void hci_pio_queue_resp(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_xfer *xfer = pio->curr_xfer; + struct hci_xfer *prev_queue_tail; + + if (!(xfer->cmd_desc[0] & CMD_0_ROC)) + return; + + prev_queue_tail = pio->resp_queue; + pio->resp_queue = xfer; + if (pio->curr_resp) { + prev_queue_tail->next_resp = xfer; + } else { + pio->curr_resp = xfer; + if (!hci_pio_process_resp(hci, pio)) + pio->enabled_irqs |= STAT_RESP_READY; + } +} + +static bool hci_pio_process_cmd(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + while (pio->curr_xfer && + (pio_reg_read(INTR_STATUS) & STAT_CMD_QUEUE_READY)) { + /* + * Always process the data FIFO before sending the command + * so needed TX data or RX space is available upfront. + */ + hci_pio_queue_data(hci, pio); + /* + * Then queue our response request. This will also process + * the response FIFO in case it got suddenly filled up + * with results from previous commands. + */ + hci_pio_queue_resp(hci, pio); + /* + * Finally send the command. + */ + hci_pio_write_cmd(hci, pio->curr_xfer); + hci_start_xfer(pio->curr_xfer); + /* + * And move on. + */ + pio->curr_xfer = pio->curr_xfer->next_xfer; + } + return !pio->curr_xfer; +} + +static int hci_pio_queue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) +{ + struct hci_pio_data *pio = hci->io_data; + struct hci_xfer *prev_queue_tail; + int i; + + dev_dbg(&hci->master.dev, "n = %d", n); + + /* link xfer instances together and initialize data count */ + for (i = 0; i < n; i++) { + xfer[i].next_xfer = (i + 1 < n) ? &xfer[i + 1] : NULL; + xfer[i].next_data = NULL; + xfer[i].next_resp = NULL; + xfer[i].data_left = xfer[i].data_len; + } + + spin_lock_irq(&hci->lock); + prev_queue_tail = pio->xfer_queue; + pio->xfer_queue = &xfer[n - 1]; + if (pio->curr_xfer) { + prev_queue_tail->next_xfer = xfer; + } else { + pio->curr_xfer = xfer; + if (!hci_pio_process_cmd(hci, pio)) + pio->enabled_irqs |= STAT_CMD_QUEUE_READY; + pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); + dev_dbg(&hci->master.dev, "status = %#x/%#x", + pio_reg_read(INTR_STATUS), + pio_reg_read(INTR_SIGNAL_ENABLE)); + } + spin_unlock_irq(&hci->lock); + return 0; +} + +static bool hci_pio_dequeue_xfer_common(struct i3c_hci *hci, + struct hci_pio_data *pio, + struct hci_xfer *xfer, int n) +{ + struct hci_xfer *p, **p_prev_next; + int i; + + /* + * To safely dequeue a transfer request, it must be either entirely + * processed, or not yet processed at all. If our request tail is + * reachable from either the data or resp list that means the command + * was submitted and not yet completed. + */ + for (p = pio->curr_resp; p; p = p->next_resp) + for (i = 0; i < n; i++) + if (p == &xfer[i]) + goto pio_screwed; + for (p = pio->curr_rx; p; p = p->next_data) + for (i = 0; i < n; i++) + if (p == &xfer[i]) + goto pio_screwed; + for (p = pio->curr_tx; p; p = p->next_data) + for (i = 0; i < n; i++) + if (p == &xfer[i]) + goto pio_screwed; + + /* + * The command was completed, or wasn't yet submitted. + * Unlink it from the que if the later. + */ + p_prev_next = &pio->curr_xfer; + for (p = pio->curr_xfer; p; p = p->next_xfer) { + if (p == &xfer[0]) { + *p_prev_next = xfer[n - 1].next_xfer; + break; + } + p_prev_next = &p->next_xfer; + } + + /* return true if we actually unqueued something */ + return !!p; + +pio_screwed: + /* + * Life is tough. We must invalidate the hardware state and + * discard everything that is still queued. + */ + for (p = pio->curr_resp; p; p = p->next_resp) { + p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED); + if (p->completion) + complete(p->completion); + } + for (p = pio->curr_xfer; p; p = p->next_xfer) { + p->response = FIELD_PREP(RESP_ERR_FIELD, RESP_ERR_HC_TERMINATED); + if (p->completion) + complete(p->completion); + } + pio->curr_xfer = pio->curr_rx = pio->curr_tx = pio->curr_resp = NULL; + + return true; +} + +static bool hci_pio_dequeue_xfer(struct i3c_hci *hci, struct hci_xfer *xfer, int n) +{ + struct hci_pio_data *pio = hci->io_data; + int ret; + + spin_lock_irq(&hci->lock); + dev_dbg(&hci->master.dev, "n=%d status=%#x/%#x", n, + pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); + dev_dbg(&hci->master.dev, "main_status = %#x/%#x", + readl(hci->base_regs + 0x20), readl(hci->base_regs + 0x28)); + + ret = hci_pio_dequeue_xfer_common(hci, pio, xfer, n); + spin_unlock_irq(&hci->lock); + return ret; +} + +static void hci_pio_err(struct i3c_hci *hci, struct hci_pio_data *pio, + u32 status) +{ + /* TODO: this ought to be more sophisticated eventually */ + + if (pio_reg_read(INTR_STATUS) & STAT_RESP_READY) { + /* this may happen when an error is signaled with ROC unset */ + u32 resp = pio_reg_read(RESPONSE_QUEUE_PORT); + + dev_err(&hci->master.dev, + "orphan response (%#x) on error\n", resp); + } + + /* dump states on programming errors */ + if (status & STAT_PROG_ERRORS) { + u32 queue = pio_reg_read(QUEUE_CUR_STATUS); + u32 data = pio_reg_read(DATA_BUFFER_CUR_STATUS); + + dev_err(&hci->master.dev, + "prog error %#lx (C/R/I = %ld/%ld/%ld, TX/RX = %ld/%ld)\n", + status & STAT_PROG_ERRORS, + FIELD_GET(CUR_CMD_Q_EMPTY_LEVEL, queue), + FIELD_GET(CUR_RESP_Q_LEVEL, queue), + FIELD_GET(CUR_IBI_Q_LEVEL, queue), + FIELD_GET(CUR_TX_BUF_LVL, data), + FIELD_GET(CUR_RX_BUF_LVL, data)); + } + + /* just bust out everything with pending responses for now */ + hci_pio_dequeue_xfer_common(hci, pio, pio->curr_resp, 1); + /* ... and half-way TX transfers if any */ + if (pio->curr_tx && pio->curr_tx->data_left != pio->curr_tx->data_len) + hci_pio_dequeue_xfer_common(hci, pio, pio->curr_tx, 1); + /* then reset the hardware */ + mipi_i3c_hci_pio_reset(hci); + + /* MIPI I3C HCI v1.2 requires explicitly restarting PIO queues after error/abort */ + if (hci_version_at_least(hci, 1, 2)) { + u32 ctl_val = pio_reg_read(CONTROL); + + if (!(ctl_val & PIO_CONTROL_ENABLE)) { + ctl_val |= PIO_CONTROL_ENABLE; + pio_reg_write(CONTROL, ctl_val); + } + pio_reg_write(CONTROL, ctl_val | PIO_CONTROL_RS); + } + + mipi_i3c_hci_resume(hci); + + dev_dbg(&hci->master.dev, "status=%#x/%#x", + pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); +} + +static void hci_pio_set_ibi_thresh(struct i3c_hci *hci, + struct hci_pio_data *pio, + unsigned int thresh_val) +{ + u32 regval = pio->reg_queue_thresh; + + regval &= ~QUEUE_IBI_STATUS_THLD; + regval |= FIELD_PREP(QUEUE_IBI_STATUS_THLD, thresh_val); + /* write the threshold reg only if it changes */ + if (regval != pio->reg_queue_thresh) { + pio_reg_write(QUEUE_THLD_CTRL, regval); + pio->reg_queue_thresh = regval; + dev_dbg(&hci->master.dev, "%d", thresh_val); + } +} + +static bool hci_pio_get_ibi_segment(struct i3c_hci *hci, + struct hci_pio_data *pio) +{ + struct hci_pio_ibi_data *ibi = &pio->ibi; + unsigned int nr_words, thresh_val; + u32 *p; + + p = ibi->data_ptr; + p += (ibi->seg_len - ibi->seg_cnt) / 4; + + while ((nr_words = ibi->seg_cnt/4)) { + /* determine our IBI queue threshold value */ + thresh_val = min(nr_words, pio->max_ibi_thresh); + hci_pio_set_ibi_thresh(hci, pio, thresh_val); + /* bail out if we don't have that amount of data ready */ + if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) + return false; + /* extract the data from the IBI port */ + nr_words = thresh_val; + ibi->seg_cnt -= nr_words * 4; + dev_dbg(&hci->master.dev, "now %d left %d", + nr_words * 4, ibi->seg_cnt); + while (nr_words--) + *p++ = pio_reg_read(IBI_PORT); + } + + if (ibi->seg_cnt) { + /* + * There are trailing bytes in the last word. + * Fetch it and extract bytes in an endian independent way. + * Unlike the TX case, we must not write past the end of + * the destination buffer. + */ + u32 data; + u8 *p_byte = (u8 *)p; + + hci_pio_set_ibi_thresh(hci, pio, 1); + if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) + return false; + dev_dbg(&hci->master.dev, "trailing %d", ibi->seg_cnt); + data = pio_reg_read(IBI_PORT); + data = (__force u32) cpu_to_le32(data); + while (ibi->seg_cnt--) { + *p_byte++ = data; + data >>= 8; + } + } + + return true; +} + +static bool hci_pio_prep_new_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_pio_ibi_data *ibi = &pio->ibi; + struct i3c_dev_desc *dev; + struct i3c_hci_dev_data *dev_data; + struct hci_pio_dev_ibi_data *dev_ibi; + u32 ibi_status; + + /* + * We have a new IBI. Try to set up its payload retrieval. + * When returning true, the IBI data has to be consumed whether + * or not we are set up to capture it. If we return true with + * ibi->slot == NULL that means the data payload has to be + * drained out of the IBI port and dropped. + */ + + ibi_status = pio_reg_read(IBI_PORT); + dev_dbg(&hci->master.dev, "status = %#x", ibi_status); + ibi->addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); + if (ibi_status & IBI_ERROR) { + dev_err(&hci->master.dev, "IBI error from %#x\n", ibi->addr); + return false; + } + + ibi->last_seg = ibi_status & IBI_LAST_STATUS; + ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status); + ibi->seg_cnt = ibi->seg_len; + + if (ibi->addr == I3C_HOT_JOIN_ADDR) { + i3c_master_queue_hotjoin(&hci->master); + return true; + } + + dev = i3c_hci_addr_to_dev(hci, ibi->addr); + if (!dev) { + /* + * Either an IBI received just before IBI's were disabled, or + * the controller is broken. Assume the former. + */ + dev_dbg(&hci->master.dev, "IBI when not enabled at address %#x\n", ibi->addr); + return true; + } + + dev_data = i3c_dev_get_master_data(dev); + dev_ibi = dev_data->ibi_data; + ibi->max_len = dev_ibi->max_len; + + if (ibi->seg_len > ibi->max_len) { + dev_err(&hci->master.dev, "IBI payload too big (%d > %d)\n", + ibi->seg_len, ibi->max_len); + return true; + } + + ibi->slot = i3c_generic_ibi_get_free_slot(dev_ibi->pool); + if (!ibi->slot) { + dev_err(&hci->master.dev, "no free slot for IBI\n"); + } else { + ibi->slot->len = 0; + ibi->data_ptr = ibi->slot->data; + } + return true; +} + +static void hci_pio_free_ibi_slot(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_pio_ibi_data *ibi = &pio->ibi; + struct hci_pio_dev_ibi_data *dev_ibi; + + if (ibi->slot) { + dev_ibi = ibi->slot->dev->common.master_priv; + i3c_generic_ibi_recycle_slot(dev_ibi->pool, ibi->slot); + ibi->slot = NULL; + } +} + +static bool hci_pio_process_ibi(struct i3c_hci *hci, struct hci_pio_data *pio) +{ + struct hci_pio_ibi_data *ibi = &pio->ibi; + + if (!ibi->slot && !ibi->seg_cnt && ibi->last_seg) + if (!hci_pio_prep_new_ibi(hci, pio)) + return false; + + for (;;) { + u32 ibi_status; + unsigned int ibi_addr; + + if (ibi->slot) { + if (!hci_pio_get_ibi_segment(hci, pio)) + return false; + ibi->slot->len += ibi->seg_len; + ibi->data_ptr += ibi->seg_len; + if (ibi->last_seg) { + /* was the last segment: submit it and leave */ + i3c_master_queue_ibi(ibi->slot->dev, ibi->slot); + ibi->slot = NULL; + hci_pio_set_ibi_thresh(hci, pio, 1); + return true; + } + } else if (ibi->seg_cnt) { + /* + * No slot but a non-zero count. This is the result + * of some error and the payload must be drained. + * This normally does not happen therefore no need + * to be extra optimized here. + */ + hci_pio_set_ibi_thresh(hci, pio, 1); + do { + if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) + return false; + pio_reg_read(IBI_PORT); + } while (--ibi->seg_cnt); + if (ibi->last_seg) + return true; + } + + /* try to move to the next segment right away */ + hci_pio_set_ibi_thresh(hci, pio, 1); + if (!(pio_reg_read(INTR_STATUS) & STAT_IBI_STATUS_THLD)) + return false; + ibi_status = pio_reg_read(IBI_PORT); + ibi_addr = FIELD_GET(IBI_TARGET_ADDR, ibi_status); + if (ibi->addr != ibi_addr) { + /* target address changed before last segment */ + dev_err(&hci->master.dev, + "unexp IBI address changed from %d to %d\n", + ibi->addr, ibi_addr); + hci_pio_free_ibi_slot(hci, pio); + } + ibi->last_seg = ibi_status & IBI_LAST_STATUS; + ibi->seg_len = FIELD_GET(IBI_DATA_LENGTH, ibi_status); + ibi->seg_cnt = ibi->seg_len; + if (ibi->slot && ibi->slot->len + ibi->seg_len > ibi->max_len) { + dev_err(&hci->master.dev, + "IBI payload too big (%d > %d)\n", + ibi->slot->len + ibi->seg_len, ibi->max_len); + hci_pio_free_ibi_slot(hci, pio); + } + } + + return false; +} + +static int hci_pio_request_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + struct i3c_generic_ibi_pool *pool; + struct hci_pio_dev_ibi_data *dev_ibi; + + dev_ibi = kmalloc_obj(*dev_ibi); + if (!dev_ibi) + return -ENOMEM; + pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(pool)) { + kfree(dev_ibi); + return PTR_ERR(pool); + } + dev_ibi->pool = pool; + dev_ibi->max_len = req->max_payload_len; + dev_data->ibi_data = dev_ibi; + return 0; +} + +static void hci_pio_free_ibi(struct i3c_hci *hci, struct i3c_dev_desc *dev) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data; + + dev_data->ibi_data = NULL; + i3c_generic_ibi_free_pool(dev_ibi->pool); + kfree(dev_ibi); +} + +static void hci_pio_recycle_ibi_slot(struct i3c_hci *hci, + struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct i3c_hci_dev_data *dev_data = i3c_dev_get_master_data(dev); + struct hci_pio_dev_ibi_data *dev_ibi = dev_data->ibi_data; + + i3c_generic_ibi_recycle_slot(dev_ibi->pool, slot); +} + +static bool hci_pio_irq_handler(struct i3c_hci *hci) +{ + struct hci_pio_data *pio = hci->io_data; + u32 status; + + status = pio_reg_read(INTR_STATUS); + dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", + status, pio->enabled_irqs); + status &= pio->enabled_irqs | STAT_LATENCY_WARNINGS; + if (!status) + return false; + + if (status & STAT_IBI_STATUS_THLD) + hci_pio_process_ibi(hci, pio); + + if (status & STAT_RX_THLD) + if (hci_pio_process_rx(hci, pio)) + pio->enabled_irqs &= ~STAT_RX_THLD; + if (status & STAT_TX_THLD) + if (hci_pio_process_tx(hci, pio)) + pio->enabled_irqs &= ~STAT_TX_THLD; + if (status & STAT_RESP_READY) + if (hci_pio_process_resp(hci, pio)) + pio->enabled_irqs &= ~STAT_RESP_READY; + + if (unlikely(status & STAT_LATENCY_WARNINGS)) { + pio_reg_write(INTR_STATUS, status & STAT_LATENCY_WARNINGS); + dev_warn_ratelimited(&hci->master.dev, + "encountered warning condition %#lx\n", + status & STAT_LATENCY_WARNINGS); + } + + if (unlikely(status & STAT_ALL_ERRORS)) { + pio_reg_write(INTR_STATUS, status & STAT_ALL_ERRORS); + hci_pio_err(hci, pio, status & STAT_ALL_ERRORS); + } + + if (status & STAT_CMD_QUEUE_READY) + if (hci_pio_process_cmd(hci, pio)) + pio->enabled_irqs &= ~STAT_CMD_QUEUE_READY; + + pio_reg_write(INTR_SIGNAL_ENABLE, pio->enabled_irqs); + dev_dbg(&hci->master.dev, "PIO_INTR_STATUS %#x/%#x", + pio_reg_read(INTR_STATUS), pio_reg_read(INTR_SIGNAL_ENABLE)); + return true; +} + +const struct hci_io_ops mipi_i3c_hci_pio = { + .init = hci_pio_init, + .cleanup = hci_pio_cleanup, + .queue_xfer = hci_pio_queue_xfer, + .dequeue_xfer = hci_pio_dequeue_xfer, + .irq_handler = hci_pio_irq_handler, + .request_ibi = hci_pio_request_ibi, + .free_ibi = hci_pio_free_ibi, + .recycle_ibi_slot = hci_pio_recycle_ibi_slot, + .suspend = hci_pio_suspend, + .resume = hci_pio_resume, +}; diff --git a/drivers/i3c/master/mipi-i3c-hci/xfer_mode_rate.h b/drivers/i3c/master/mipi-i3c-hci/xfer_mode_rate.h new file mode 100644 index 000000000..1e36b75af --- /dev/null +++ b/drivers/i3c/master/mipi-i3c-hci/xfer_mode_rate.h @@ -0,0 +1,79 @@ +/* SPDX-License-Identifier: BSD-3-Clause */ +/* + * Copyright (c) 2020, MIPI Alliance, Inc. + * + * Author: Nicolas Pitre <npitre@baylibre.com> + * + * Transfer Mode/Rate Table definitions as found in extended capability + * sections 0x04 and 0x08. + * This applies starting from I3C HCI v2.0. + */ + +#ifndef XFER_MODE_RATE_H +#define XFER_MODE_RATE_H + +/* + * Master Transfer Mode Table Fixed Indexes. + * + * Indexes 0x0 and 0x8 are mandatory. Availability for the rest must be + * obtained from the mode table in the extended capability area. + * Presence and definitions for indexes beyond these ones may vary. + */ +#define XFERMODE_IDX_I3C_SDR 0x00 /* I3C SDR Mode */ +#define XFERMODE_IDX_I3C_HDR_DDR 0x01 /* I3C HDR-DDR Mode */ +#define XFERMODE_IDX_I3C_HDR_T 0x02 /* I3C HDR-Ternary Mode */ +#define XFERMODE_IDX_I3C_HDR_BT 0x03 /* I3C HDR-BT Mode */ +#define XFERMODE_IDX_I2C 0x08 /* Legacy I2C Mode */ + +/* + * Transfer Mode Table Entry Bits Definitions + */ +#define XFERMODE_VALID_XFER_ADD_FUNC GENMASK(21, 16) +#define XFERMODE_ML_DATA_XFER_CODING GENMASK(15, 11) +#define XFERMODE_ML_ADDL_LANES GENMASK(10, 8) +#define XFERMODE_SUPPORTED BIT(7) +#define XFERMODE_MODE GENMASK(3, 0) + +/* + * Master Data Transfer Rate Selector Values. + * + * These are the values to be used in the command descriptor XFER_RATE field + * and found in the RATE_ID field below. + * The I3C_SDR0, I3C_SDR1, I3C_SDR2, I3C_SDR3, I3C_SDR4 and I2C_FM rates + * are required, everything else is optional and discoverable in the + * Data Transfer Rate Table. Indicated are typical rates. The actual + * rates may vary slightly and are also specified in the Data Transfer + * Rate Table. + */ +#define XFERRATE_I3C_SDR0 0x00 /* 12.5 MHz */ +#define XFERRATE_I3C_SDR1 0x01 /* 8 MHz */ +#define XFERRATE_I3C_SDR2 0x02 /* 6 MHz */ +#define XFERRATE_I3C_SDR3 0x03 /* 4 MHz */ +#define XFERRATE_I3C_SDR4 0x04 /* 2 MHz */ +#define XFERRATE_I3C_SDR_FM_FMP 0x05 /* 400 KHz / 1 MHz */ +#define XFERRATE_I3C_SDR_USER6 0x06 /* User Defined */ +#define XFERRATE_I3C_SDR_USER7 0x07 /* User Defined */ + +#define XFERRATE_I2C_FM 0x00 /* 400 KHz */ +#define XFERRATE_I2C_FMP 0x01 /* 1 MHz */ +#define XFERRATE_I2C_USER2 0x02 /* User Defined */ +#define XFERRATE_I2C_USER3 0x03 /* User Defined */ +#define XFERRATE_I2C_USER4 0x04 /* User Defined */ +#define XFERRATE_I2C_USER5 0x05 /* User Defined */ +#define XFERRATE_I2C_USER6 0x06 /* User Defined */ +#define XFERRATE_I2C_USER7 0x07 /* User Defined */ + +/* + * Master Data Transfer Rate Table Mode ID values. + */ +#define XFERRATE_MODE_I3C 0x00 +#define XFERRATE_MODE_I2C 0x08 + +/* + * Master Data Transfer Rate Table Entry Bits Definitions + */ +#define XFERRATE_MODE_ID GENMASK(31, 28) +#define XFERRATE_RATE_ID GENMASK(22, 20) +#define XFERRATE_ACTUAL_RATE_KHZ GENMASK(19, 0) + +#endif diff --git a/drivers/i3c/master/renesas-i3c.c b/drivers/i3c/master/renesas-i3c.c new file mode 100644 index 000000000..28c0927a0 --- /dev/null +++ b/drivers/i3c/master/renesas-i3c.c @@ -0,0 +1,1636 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Renesas I3C Controller driver + * Copyright (C) 2023-25 Renesas Electronics Corp. + * + * TODO: IBI support, HotJoin support, Target support + */ + +#include <linux/bitfield.h> +#include <linux/bitops.h> +#include <linux/cleanup.h> +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/err.h> +#include <linux/errno.h> +#include <linux/i2c.h> +#include <linux/i3c/master.h> +#include <linux/interrupt.h> +#include <linux/ioport.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> +#include <linux/reset.h> +#include <linux/slab.h> +#include "../internals.h" + +#define PRTS 0x00 +#define PRTS_PRTMD BIT(0) + +#define BCTL 0x14 +#define BCTL_INCBA BIT(0) +#define BCTL_HJACKCTL BIT(8) +#define BCTL_ABT BIT(29) +#define BCTL_BUSE BIT(31) + +#define MSDVAD 0x18 +#define MSDVAD_MDYAD(x) FIELD_PREP(GENMASK(21, 16), x) +#define MSDVAD_MDYADV BIT(31) + +#define RSTCTL 0x20 +#define RSTCTL_RI3CRST BIT(0) +#define RSTCTL_INTLRST BIT(16) + +#define INST 0x30 + +#define IBINCTL 0x58 +#define IBINCTL_NRHJCTL BIT(0) +#define IBINCTL_NRMRCTL BIT(1) +#define IBINCTL_NRSIRCTL BIT(3) + +#define SVCTL 0x64 + +#define REFCKCTL 0x70 +#define REFCKCTL_IREFCKS(x) FIELD_PREP(GENMASK(2, 0), x) + +#define STDBR 0x74 +#define STDBR_SBRLO(cond, x) FIELD_PREP(GENMASK(7, 0), (x) >> (cond)) +#define STDBR_SBRHO(cond, x) FIELD_PREP(GENMASK(15, 8), (x) >> (cond)) +#define STDBR_SBRLP(x) FIELD_PREP(GENMASK(21, 16), x) +#define STDBR_SBRHP(x) FIELD_PREP(GENMASK(29, 24), x) +#define STDBR_DSBRPO BIT(31) + +#define EXTBR 0x78 +#define EXTBR_EBRLO(x) FIELD_PREP(GENMASK(7, 0), x) +#define EXTBR_EBRHO(x) FIELD_PREP(GENMASK(15, 8), x) +#define EXTBR_EBRLP(x) FIELD_PREP(GENMASK(21, 16), x) +#define EXTBR_EBRHP(x) FIELD_PREP(GENMASK(29, 24), x) + +#define BFRECDT 0x7c +#define BFRECDT_FRECYC(x) FIELD_PREP(GENMASK(8, 0), x) + +#define BAVLCDT 0x80 +#define BAVLCDT_AVLCYC(x) FIELD_PREP(GENMASK(8, 0), x) + +#define BIDLCDT 0x84 +#define BIDLCDT_IDLCYC(x) FIELD_PREP(GENMASK(17, 0), x) + +#define ACKCTL 0xa0 +#define ACKCTL_ACKT BIT(1) +#define ACKCTL_ACKTWP BIT(2) + +#define SCSTRCTL 0xa4 +#define SCSTRCTL_ACKTWE BIT(0) +#define SCSTRCTL_RWE BIT(1) + +#define SCSTLCTL 0xb0 + +#define CNDCTL 0x140 +#define CNDCTL_STCND BIT(0) +#define CNDCTL_SRCND BIT(1) +#define CNDCTL_SPCND BIT(2) + +#define NCMDQP 0x150 /* Normal Command Queue */ +#define NCMDQP_CMD_ATTR(x) FIELD_PREP(GENMASK(2, 0), x) +#define NCMDQP_IMMED_XFER 0x01 +#define NCMDQP_ADDR_ASSGN 0x02 +#define NCMDQP_TID(x) FIELD_PREP(GENMASK(6, 3), x) +#define NCMDQP_CMD(x) FIELD_PREP(GENMASK(14, 7), x) +#define NCMDQP_CP BIT(15) +#define NCMDQP_DEV_INDEX(x) FIELD_PREP(GENMASK(20, 16), x) +#define NCMDQP_BYTE_CNT(x) FIELD_PREP(GENMASK(25, 23), x) +#define NCMDQP_DEV_COUNT(x) FIELD_PREP(GENMASK(29, 26), x) +#define NCMDQP_MODE(x) FIELD_PREP(GENMASK(28, 26), x) +#define NCMDQP_RNW(x) FIELD_PREP(GENMASK(29, 29), x) +#define NCMDQP_ROC BIT(30) +#define NCMDQP_TOC BIT(31) +#define NCMDQP_DATA_LENGTH(x) FIELD_PREP(GENMASK(31, 16), x) + +#define NRSPQP 0x154 /* Normal Respone Queue */ +#define NRSPQP_NO_ERROR 0 +#define NRSPQP_ERROR_CRC 1 +#define NRSPQP_ERROR_PARITY 2 +#define NRSPQP_ERROR_FRAME 3 +#define NRSPQP_ERROR_IBA_NACK 4 +#define NRSPQP_ERROR_ADDRESS_NACK 5 +#define NRSPQP_ERROR_OVER_UNDER_FLOW 6 +#define NRSPQP_ERROR_TRANSF_ABORT 8 +#define NRSPQP_ERROR_I2C_W_NACK_ERR 9 +#define NRSPQP_ERROR_UNSUPPORTED 10 +#define NRSPQP_DATA_LEN(x) FIELD_GET(GENMASK(15, 0), x) +#define NRSPQP_ERR_STATUS(x) FIELD_GET(GENMASK(31, 28), x) + +#define NTDTBP0 0x158 /* Normal Transfer Data Buffer */ +#define NTDTBP0_DEPTH 16 + +#define NQTHCTL 0x190 +#define NQTHCTL_CMDQTH(x) FIELD_PREP(GENMASK(1, 0), x) +#define NQTHCTL_IBIDSSZ(x) FIELD_PREP(GENMASK(23, 16), x) + +#define NTBTHCTL0 0x194 + +#define NRQTHCTL 0x1c0 + +#define BST 0x1d0 +#define BST_STCNDDF BIT(0) +#define BST_SPCNDDF BIT(1) +#define BST_NACKDF BIT(4) +#define BST_TENDF BIT(8) + +#define BSTE 0x1d4 +#define BSTE_STCNDDE BIT(0) +#define BSTE_SPCNDDE BIT(1) +#define BSTE_NACKDE BIT(4) +#define BSTE_TENDE BIT(8) +#define BSTE_ALE BIT(16) +#define BSTE_TODE BIT(20) +#define BSTE_WUCNDDE BIT(24) +#define BSTE_ALL_FLAG (BSTE_STCNDDE | BSTE_SPCNDDE |\ + BSTE_NACKDE | BSTE_TENDE |\ + BSTE_ALE | BSTE_TODE | BSTE_WUCNDDE) + +#define BIE 0x1d8 +#define BIE_STCNDDIE BIT(0) +#define BIE_SPCNDDIE BIT(1) +#define BIE_NACKDIE BIT(4) +#define BIE_TENDIE BIT(8) + +#define NTST 0x1e0 +#define NTST_TDBEF0 BIT(0) +#define NTST_RDBFF0 BIT(1) +#define NTST_CMDQEF BIT(3) +#define NTST_RSPQFF BIT(4) +#define NTST_TABTF BIT(5) +#define NTST_TEF BIT(9) + +#define NTSTE 0x1e4 +#define NTSTE_TDBEE0 BIT(0) +#define NTSTE_RDBFE0 BIT(1) +#define NTSTE_IBIQEFE BIT(2) +#define NTSTE_CMDQEE BIT(3) +#define NTSTE_RSPQFE BIT(4) +#define NTSTE_TABTE BIT(5) +#define NTSTE_TEE BIT(9) +#define NTSTE_RSQFE BIT(20) +#define NTSTE_ALL_FLAG (NTSTE_TDBEE0 | NTSTE_RDBFE0 |\ + NTSTE_IBIQEFE | NTSTE_CMDQEE |\ + NTSTE_RSPQFE | NTSTE_TABTE |\ + NTSTE_TEE | NTSTE_RSQFE) + +#define NTIE 0x1e8 +#define NTIE_TDBEIE0 BIT(0) +#define NTIE_RDBFIE0 BIT(1) +#define NTIE_IBIQEFIE BIT(2) +#define NTIE_RSPQFIE BIT(4) +#define NTIE_RSQFIE BIT(20) + +#define BCST 0x210 +#define BCST_BFREF BIT(0) + +#define DATBAS(x) (0x224 + 0x8 * (x)) +#define DATBAS_DVSTAD(x) FIELD_PREP(GENMASK(6, 0), x) +#define DATBAS_DVDYAD(x) FIELD_PREP(GENMASK(23, 16), x) + +#define NDBSTLV0 0x398 +#define NDBSTLV0_RDBLV(x) FIELD_GET(GENMASK(15, 8), x) + +#define RENESAS_I3C_MAX_DEVS 8 +#define I2C_INIT_MSG -1 + +enum i3c_internal_state { + I3C_INTERNAL_STATE_DISABLED, + I3C_INTERNAL_STATE_CONTROLLER_IDLE, + I3C_INTERNAL_STATE_CONTROLLER_ENTDAA, + I3C_INTERNAL_STATE_CONTROLLER_SETDASA, + I3C_INTERNAL_STATE_CONTROLLER_WRITE, + I3C_INTERNAL_STATE_CONTROLLER_READ, + I3C_INTERNAL_STATE_CONTROLLER_COMMAND_WRITE, + I3C_INTERNAL_STATE_CONTROLLER_COMMAND_READ, +}; + +enum renesas_i3c_event { + I3C_COMMAND_ADDRESS_ASSIGNMENT, + I3C_WRITE, + I3C_READ, + I3C_COMMAND_WRITE, + I3C_COMMAND_READ, +}; + +struct renesas_i3c_cmd { + const void *tx_buf; + void *rx_buf; + /* i2c xfer */ + u8 *i2c_buf; + const struct i2c_msg *msg; + int i2c_bytes_left; + int i2c_is_last; + u32 cmd0; + u32 len; + u32 tx_count; + u32 rx_count; + u32 err; + u8 rnw; +}; + +struct renesas_i3c_xfer { + struct list_head node; + struct completion comp; + int ret; + bool is_i2c_xfer; + unsigned int ncmds; + struct renesas_i3c_cmd cmds[] __counted_by(ncmds); +}; + +struct renesas_i3c_xferqueue { + struct list_head list; + struct renesas_i3c_xfer *cur; + /* Lock for accessing the xfer queue */ + spinlock_t lock; +}; + +struct renesas_i3c { + void __iomem *regs; + struct clk *tclk; + struct reset_control *presetn; + struct reset_control *tresetn; + struct device *dev; + struct renesas_i3c_xferqueue xferqueue; + struct i3c_master_controller base; + u8 addrs[RENESAS_I3C_MAX_DEVS]; + unsigned long rate; + enum i3c_internal_state internal_state; + u32 free_pos; + u32 dyn_addr; + u32 i2c_STDBR; + u32 i3c_STDBR; + u32 extbr; + u16 maxdevs; + u8 refclk_div; +}; + +struct renesas_i3c_i2c_dev_data { + u8 index; +}; + +struct renesas_i3c_irq_desc { + const char *name; + irq_handler_t isr; + const char *desc; +}; + +static inline void renesas_i3c_reg_update(void __iomem *reg, u32 mask, u32 val) +{ + u32 data = readl(reg); + + data &= ~mask; + data |= (val & mask); + writel(data, reg); +} + +static inline u32 renesas_readl(void __iomem *base, u32 reg) +{ + return readl(base + reg); +} + +static inline void renesas_writel(void __iomem *base, u32 reg, u32 val) +{ + writel(val, base + reg); +} + +static void renesas_set_bit(void __iomem *base, u32 reg, u32 val) +{ + renesas_i3c_reg_update(base + reg, val, val); +} + +static void renesas_clear_bit(void __iomem *base, u32 reg, u32 val) +{ + renesas_i3c_reg_update(base + reg, val, 0); +} + +static inline struct renesas_i3c *to_renesas_i3c(struct i3c_master_controller *m) +{ + return container_of(m, struct renesas_i3c, base); +} + +static inline u32 datbas_dvdyad_with_parity(u8 addr) +{ + return DATBAS_DVDYAD(addr | (parity8(addr) ? 0 : BIT(7))); +} + +static int renesas_i3c_get_free_pos(struct renesas_i3c *i3c) +{ + if (!(i3c->free_pos & GENMASK(i3c->maxdevs - 1, 0))) + return -ENOSPC; + + return ffs(i3c->free_pos) - 1; +} + +static int renesas_i3c_get_addr_pos(struct renesas_i3c *i3c, u8 addr) +{ + int pos; + + for (pos = 0; pos < i3c->maxdevs; pos++) { + if (addr == i3c->addrs[pos]) + return pos; + } + + return -EINVAL; +} + +static struct renesas_i3c_xfer *renesas_i3c_alloc_xfer(struct renesas_i3c *i3c, + unsigned int ncmds) +{ + struct renesas_i3c_xfer *xfer; + + xfer = kzalloc_flex(*xfer, cmds, ncmds); + if (!xfer) + return NULL; + + INIT_LIST_HEAD(&xfer->node); + xfer->ncmds = ncmds; + xfer->ret = -ETIMEDOUT; + + return xfer; +} + +static void renesas_i3c_start_xfer_locked(struct renesas_i3c *i3c) +{ + struct renesas_i3c_xfer *xfer = i3c->xferqueue.cur; + struct renesas_i3c_cmd *cmd; + u32 cmd1; + + if (!xfer) + return; + + cmd = xfer->cmds; + + switch (i3c->internal_state) { + case I3C_INTERNAL_STATE_CONTROLLER_ENTDAA: + case I3C_INTERNAL_STATE_CONTROLLER_SETDASA: + renesas_set_bit(i3c->regs, NTIE, NTIE_RSPQFIE); + renesas_writel(i3c->regs, NCMDQP, cmd->cmd0); + renesas_writel(i3c->regs, NCMDQP, 0); + break; + case I3C_INTERNAL_STATE_CONTROLLER_WRITE: + case I3C_INTERNAL_STATE_CONTROLLER_COMMAND_WRITE: + renesas_set_bit(i3c->regs, NTIE, NTIE_RSPQFIE); + if (cmd->len <= 4) { + cmd->cmd0 |= NCMDQP_CMD_ATTR(NCMDQP_IMMED_XFER); + cmd->cmd0 |= NCMDQP_BYTE_CNT(cmd->len); + cmd->tx_count = cmd->len; + cmd1 = cmd->len == 0 ? 0 : *(u32 *)cmd->tx_buf; + } else { + cmd1 = NCMDQP_DATA_LENGTH(cmd->len); + } + renesas_writel(i3c->regs, NCMDQP, cmd->cmd0); + renesas_writel(i3c->regs, NCMDQP, cmd1); + break; + case I3C_INTERNAL_STATE_CONTROLLER_READ: + case I3C_INTERNAL_STATE_CONTROLLER_COMMAND_READ: + renesas_set_bit(i3c->regs, NTIE, NTIE_RDBFIE0); + cmd1 = NCMDQP_DATA_LENGTH(cmd->len); + renesas_writel(i3c->regs, NCMDQP, cmd->cmd0); + renesas_writel(i3c->regs, NCMDQP, cmd1); + break; + default: + break; + } + + /* Clear the command queue empty flag */ + renesas_clear_bit(i3c->regs, NTST, NTST_CMDQEF); +} + +static void renesas_i3c_dequeue_xfer_locked(struct renesas_i3c *i3c, + struct renesas_i3c_xfer *xfer) +{ + if (i3c->xferqueue.cur == xfer) + i3c->xferqueue.cur = NULL; + else + list_del_init(&xfer->node); +} + +static void renesas_i3c_dequeue_xfer(struct renesas_i3c *i3c, struct renesas_i3c_xfer *xfer) +{ + scoped_guard(spinlock_irqsave, &i3c->xferqueue.lock) + renesas_i3c_dequeue_xfer_locked(i3c, xfer); +} + +static void renesas_i3c_enqueue_xfer(struct renesas_i3c *i3c, struct renesas_i3c_xfer *xfer) +{ + reinit_completion(&xfer->comp); + scoped_guard(spinlock_irqsave, &i3c->xferqueue.lock) { + if (i3c->xferqueue.cur) { + list_add_tail(&xfer->node, &i3c->xferqueue.list); + } else { + i3c->xferqueue.cur = xfer; + if (!xfer->is_i2c_xfer) + renesas_i3c_start_xfer_locked(i3c); + } + } +} + +static void renesas_i3c_irqs_mask_and_clear_locked(struct renesas_i3c *i3c) +{ + /* Disable all the interrupts. */ + renesas_writel(i3c->regs, BIE, 0); + renesas_writel(i3c->regs, NTIE, 0); + + /* Clear normal transfer status flags. */ + renesas_writel(i3c->regs, NTST, 0); + + /* Clear bus status flags. */ + renesas_writel(i3c->regs, BST, 0); + /* Read back registers to confirm writes have fully propagated. */ + renesas_readl(i3c->regs, BST); +} + +static void renesas_i3c_irqs_mask_and_clear(struct renesas_i3c *i3c) +{ + guard(spinlock_irqsave)(&i3c->xferqueue.lock); + + renesas_i3c_irqs_mask_and_clear_locked(i3c); +} + +static unsigned long renesas_i3c_wait_xfer(struct renesas_i3c *i3c, struct renesas_i3c_xfer *xfer) +{ + unsigned long time_left; + + renesas_i3c_enqueue_xfer(i3c, xfer); + + time_left = wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000)); + if (!time_left) + renesas_i3c_dequeue_xfer(i3c, xfer); + + return time_left; +} + +static void renesas_i3c_set_prts(struct renesas_i3c *i3c, u32 val) +{ + /* Required sequence according to tnrza0140ae */ + renesas_set_bit(i3c->regs, RSTCTL, RSTCTL_INTLRST); + renesas_writel(i3c->regs, PRTS, val); + renesas_clear_bit(i3c->regs, RSTCTL, RSTCTL_INTLRST); +} + +static void renesas_i3c_bus_enable(struct i3c_master_controller *m, bool i3c_mode) +{ + struct renesas_i3c *i3c = to_renesas_i3c(m); + + /* Setup either I3C or I2C protocol */ + if (i3c_mode) { + renesas_i3c_set_prts(i3c, 0); + /* Revisit: INCBA handling, especially after I2C transfers */ + renesas_set_bit(i3c->regs, BCTL, BCTL_HJACKCTL | BCTL_INCBA); + renesas_set_bit(i3c->regs, MSDVAD, MSDVAD_MDYADV); + renesas_writel(i3c->regs, STDBR, i3c->i3c_STDBR); + } else { + renesas_i3c_set_prts(i3c, PRTS_PRTMD); + renesas_writel(i3c->regs, STDBR, i3c->i2c_STDBR); + } + + /* Enable I3C bus */ + renesas_set_bit(i3c->regs, BCTL, BCTL_BUSE); +} + +static int renesas_i3c_reset(struct renesas_i3c *i3c) +{ + u32 val; + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + renesas_writel(i3c->regs, BCTL, 0); + renesas_set_bit(i3c->regs, RSTCTL, RSTCTL_RI3CRST); + + return read_poll_timeout(renesas_readl, val, !(val & RSTCTL_RI3CRST), + 0, 1000, false, i3c->regs, RSTCTL); +} + +static void renesas_i3c_hw_init(struct renesas_i3c *i3c) +{ + u32 val; + + /* Disable Slave Mode */ + renesas_writel(i3c->regs, SVCTL, 0); + + /* Initialize Queue/Buffer threshold */ + renesas_writel(i3c->regs, NQTHCTL, NQTHCTL_IBIDSSZ(6) | + NQTHCTL_CMDQTH(1)); + + /* The only supported configuration is two entries*/ + renesas_writel(i3c->regs, NTBTHCTL0, 0); + /* Interrupt when there is one entry in the queue */ + renesas_writel(i3c->regs, NRQTHCTL, 0); + + /* Enable all Bus/Transfer Status Flags */ + renesas_writel(i3c->regs, BSTE, BSTE_ALL_FLAG); + renesas_writel(i3c->regs, NTSTE, NTSTE_ALL_FLAG); + + /* Interrupt enable settings */ + renesas_writel(i3c->regs, BIE, BIE_NACKDIE | BIE_TENDIE); + renesas_writel(i3c->regs, NTIE, 0); + + /* Clear Status register */ + renesas_writel(i3c->regs, NTST, 0); + renesas_writel(i3c->regs, INST, 0); + renesas_writel(i3c->regs, BST, 0); + + /* Hot-Join Acknowlege setting. */ + renesas_set_bit(i3c->regs, BCTL, BCTL_HJACKCTL); + + renesas_writel(i3c->regs, IBINCTL, IBINCTL_NRHJCTL | IBINCTL_NRMRCTL | + IBINCTL_NRSIRCTL); + + renesas_writel(i3c->regs, SCSTLCTL, 0); + renesas_set_bit(i3c->regs, SCSTRCTL, SCSTRCTL_ACKTWE); + + /* Bus condition timing */ + val = DIV_ROUND_UP(I3C_BUS_TBUF_MIXED_FM_MIN_NS, + NSEC_PER_SEC / i3c->rate); + renesas_writel(i3c->regs, BFRECDT, BFRECDT_FRECYC(val)); + + val = DIV_ROUND_UP(I3C_BUS_TAVAL_MIN_NS, + NSEC_PER_SEC / i3c->rate); + renesas_writel(i3c->regs, BAVLCDT, BAVLCDT_AVLCYC(val)); + + val = DIV_ROUND_UP(I3C_BUS_TIDLE_MIN_NS, + NSEC_PER_SEC / i3c->rate); + renesas_writel(i3c->regs, BIDLCDT, BIDLCDT_IDLCYC(val)); +} + +static int renesas_i3c_bus_init(struct i3c_master_controller *m) +{ + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct i3c_bus *bus = i3c_master_get_bus(m); + struct i3c_device_info info = {}; + struct i2c_timings t; + u32 double_SBR; + int cks, pp_high_ticks, pp_low_ticks, i3c_total_ticks; + int od_high_ticks, od_low_ticks, i2c_total_ticks; + int ret; + + i3c->rate = clk_get_rate(i3c->tclk); + if (!i3c->rate) + return -EINVAL; + + i2c_total_ticks = DIV_ROUND_UP(i3c->rate, bus->scl_rate.i2c); + i3c_total_ticks = DIV_ROUND_UP(i3c->rate, bus->scl_rate.i3c); + + i2c_parse_fw_timings(&m->dev, &t, true); + + for (cks = 0; cks < 7; cks++) { + /* SCL low-period calculation in Open-drain mode */ + od_low_ticks = ((i2c_total_ticks * 6) / 10); + + /* SCL clock calculation in Push-Pull mode */ + if (bus->mode == I3C_BUS_MODE_PURE) + pp_high_ticks = ((i3c_total_ticks * 5) / 10); + else + pp_high_ticks = DIV_ROUND_UP(I3C_BUS_THIGH_MIXED_MAX_NS, + NSEC_PER_SEC / i3c->rate); + pp_low_ticks = i3c_total_ticks - pp_high_ticks; + + if ((od_low_ticks / 2) <= 0xFF && pp_low_ticks < 0x3F) + break; + + i2c_total_ticks /= 2; + i3c_total_ticks /= 2; + i3c->rate /= 2; + } + + /* SCL clock period calculation in Open-drain mode */ + if ((od_low_ticks / 2) > 0xFF || pp_low_ticks > 0x3F) { + dev_err(&m->dev, "invalid speed (i2c-scl = %lu Hz, i3c-scl = %lu Hz). Too slow.\n", + (unsigned long)bus->scl_rate.i2c, (unsigned long)bus->scl_rate.i3c); + return -EINVAL; + } + + /* SCL high-period calculation in Open-drain mode */ + od_high_ticks = i2c_total_ticks - od_low_ticks; + + /* Standard Bit Rate setting */ + double_SBR = od_low_ticks > 0xFF ? 1 : 0; + i3c->i3c_STDBR = (double_SBR ? STDBR_DSBRPO : 0) | + STDBR_SBRLO(double_SBR, od_low_ticks) | + STDBR_SBRHO(double_SBR, od_high_ticks) | + STDBR_SBRLP(pp_low_ticks) | + STDBR_SBRHP(pp_high_ticks); + + od_low_ticks -= t.scl_fall_ns / (NSEC_PER_SEC / i3c->rate) + 1; + od_high_ticks -= t.scl_rise_ns / (NSEC_PER_SEC / i3c->rate) + 1; + i3c->i2c_STDBR = (double_SBR ? STDBR_DSBRPO : 0) | + STDBR_SBRLO(double_SBR, od_low_ticks) | + STDBR_SBRHO(double_SBR, od_high_ticks) | + STDBR_SBRLP(pp_low_ticks) | + STDBR_SBRHP(pp_high_ticks); + + /* Extended Bit Rate setting */ + i3c->extbr = EXTBR_EBRLO(od_low_ticks) | EXTBR_EBRHO(od_high_ticks) | + EXTBR_EBRLP(pp_low_ticks) | EXTBR_EBRHP(pp_high_ticks); + + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + return ret; + + info.dyn_addr = ret; + i3c->dyn_addr = ret; + i3c->refclk_div = cks; + + ret = renesas_i3c_reset(i3c); + if (ret) + return ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + renesas_writel(i3c->regs, STDBR, i3c->i3c_STDBR); + renesas_writel(i3c->regs, EXTBR, i3c->extbr); + renesas_writel(i3c->regs, REFCKCTL, REFCKCTL_IREFCKS(cks)); + renesas_writel(i3c->regs, MSDVAD, MSDVAD_MDYAD(i3c->dyn_addr) | MSDVAD_MDYADV); + + /* I3C hw init*/ + renesas_i3c_hw_init(i3c); + + return i3c_master_set_info(&i3c->base, &info); +} + +static void renesas_i3c_bus_cleanup(struct i3c_master_controller *m) +{ + struct renesas_i3c *i3c = to_renesas_i3c(m); + + renesas_i3c_reset(i3c); +} + +static int renesas_i3c_daa(struct i3c_master_controller *m) +{ + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_cmd *cmd; + unsigned long time_left; + u32 olddevs, newdevs; + u8 last_addr = 0, pos; + int ret; + + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; + + init_completion(&xfer->comp); + cmd = xfer->cmds; + cmd->rx_count = i3c->maxdevs; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + /* Enable I3C bus. */ + renesas_i3c_bus_enable(m, true); + + olddevs = ~(i3c->free_pos); + i3c->internal_state = I3C_INTERNAL_STATE_CONTROLLER_ENTDAA; + + /* Setting DATBASn registers for target devices. */ + for (pos = 0; pos < i3c->maxdevs; pos++) { + if (olddevs & BIT(pos)) + continue; + + ret = i3c_master_get_free_addr(m, last_addr + 1); + if (ret < 0) + return -ENOSPC; + + i3c->addrs[pos] = ret; + last_addr = ret; + + renesas_writel(i3c->regs, DATBAS(pos), datbas_dvdyad_with_parity(ret)); + } + + ret = renesas_i3c_get_free_pos(i3c); + if (ret < 0) + return ret; + + /* + * Setup the command descriptor to start the ENTDAA command + * and starting at the selected device index. + */ + cmd->cmd0 = NCMDQP_CMD_ATTR(NCMDQP_ADDR_ASSGN) | NCMDQP_ROC | + NCMDQP_TID(I3C_COMMAND_ADDRESS_ASSIGNMENT) | + NCMDQP_CMD(I3C_CCC_ENTDAA) | NCMDQP_DEV_INDEX(ret) | + NCMDQP_DEV_COUNT(i3c->maxdevs - ret) | NCMDQP_TOC; + + time_left = renesas_i3c_wait_xfer(i3c, xfer); + if (!time_left) + renesas_i3c_irqs_mask_and_clear(i3c); + + if (cmd->rx_count >= i3c->maxdevs) + newdevs = 0; + else + newdevs = GENMASK(i3c->maxdevs - cmd->rx_count - 1, 0); + + newdevs &= ~olddevs; + + for (pos = 0; pos < i3c->maxdevs; pos++) { + if (newdevs & BIT(pos)) + i3c_master_add_i3c_dev_locked(m, i3c->addrs[pos]); + } + + return 0; +} + +static bool renesas_i3c_supports_ccc_cmd(struct i3c_master_controller *m, + const struct i3c_ccc_cmd *cmd) +{ + if (cmd->ndests > 1) + return false; + + switch (cmd->id) { + case I3C_CCC_ENEC(true): + case I3C_CCC_ENEC(false): + case I3C_CCC_DISEC(true): + case I3C_CCC_DISEC(false): + case I3C_CCC_ENTAS(0, true): + case I3C_CCC_ENTAS(1, true): + case I3C_CCC_ENTAS(2, true): + case I3C_CCC_ENTAS(3, true): + case I3C_CCC_ENTAS(0, false): + case I3C_CCC_ENTAS(1, false): + case I3C_CCC_ENTAS(2, false): + case I3C_CCC_ENTAS(3, false): + case I3C_CCC_RSTDAA(true): + case I3C_CCC_RSTDAA(false): + case I3C_CCC_ENTDAA: + case I3C_CCC_DEFSLVS: + case I3C_CCC_SETMWL(true): + case I3C_CCC_SETMWL(false): + case I3C_CCC_SETMRL(true): + case I3C_CCC_SETMRL(false): + case I3C_CCC_ENTTM: + case I3C_CCC_SETDASA: + case I3C_CCC_SETNEWDA: + case I3C_CCC_GETMWL: + case I3C_CCC_GETMRL: + case I3C_CCC_GETPID: + case I3C_CCC_GETBCR: + case I3C_CCC_GETDCR: + case I3C_CCC_GETSTATUS: + case I3C_CCC_GETACCMST: + case I3C_CCC_GETMXDS: + return true; + default: + return false; + } +} + +static int renesas_i3c_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *ccc) +{ + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_cmd *cmd; + unsigned long time_left; + int ret, pos = 0; + + if (ccc->id & I3C_CCC_DIRECT) { + pos = renesas_i3c_get_addr_pos(i3c, ccc->dests[0].addr); + if (pos < 0) + return pos; + } + + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; + + init_completion(&xfer->comp); + cmd = xfer->cmds; + cmd->rnw = ccc->rnw; + cmd->cmd0 = 0; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + renesas_i3c_bus_enable(m, true); + + /* Calculate the command descriptor. */ + switch (ccc->id) { + case I3C_CCC_SETDASA: + renesas_writel(i3c->regs, DATBAS(pos), + DATBAS_DVSTAD(ccc->dests[0].addr) | + DATBAS_DVDYAD(*(u8 *)ccc->dests[0].payload.data >> 1)); + cmd->cmd0 = NCMDQP_CMD_ATTR(NCMDQP_ADDR_ASSGN) | NCMDQP_ROC | + NCMDQP_TID(I3C_COMMAND_ADDRESS_ASSIGNMENT) | + NCMDQP_CMD(I3C_CCC_SETDASA) | NCMDQP_DEV_INDEX(pos) | + NCMDQP_DEV_COUNT(0) | NCMDQP_TOC; + i3c->internal_state = I3C_INTERNAL_STATE_CONTROLLER_SETDASA; + break; + default: + /* Calculate the command descriptor. */ + cmd->cmd0 = NCMDQP_TID(I3C_COMMAND_WRITE) | NCMDQP_MODE(0) | + NCMDQP_RNW(ccc->rnw) | NCMDQP_CMD(ccc->id) | + NCMDQP_ROC | NCMDQP_TOC | NCMDQP_CP | + NCMDQP_DEV_INDEX(pos); + + if (ccc->rnw) { + cmd->rx_buf = ccc->dests[0].payload.data; + cmd->len = ccc->dests[0].payload.len; + cmd->rx_count = 0; + i3c->internal_state = I3C_INTERNAL_STATE_CONTROLLER_COMMAND_READ; + } else { + cmd->tx_buf = ccc->dests[0].payload.data; + cmd->len = ccc->dests[0].payload.len; + cmd->tx_count = 0; + i3c->internal_state = I3C_INTERNAL_STATE_CONTROLLER_COMMAND_WRITE; + } + } + + time_left = renesas_i3c_wait_xfer(i3c, xfer); + if (!time_left) + renesas_i3c_irqs_mask_and_clear(i3c); + + ret = xfer->ret; + if (ret) + ccc->err = I3C_ERROR_M2; + else if (ccc->rnw) + ccc->dests[0].payload.actual_len = cmd->rx_count; + + return ret; +} + +static int renesas_i3c_i3c_xfers(struct i3c_dev_desc *dev, struct i3c_xfer *i3c_xfers, + int i3c_nxfers, enum i3c_xfer_mode mode) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long time_left; + bool xfer_failed = false; + int i, ret; + + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; + + init_completion(&xfer->comp); + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + /* Enable I3C bus. */ + renesas_i3c_bus_enable(m, true); + + for (i = 0; i < i3c_nxfers; i++) { + struct renesas_i3c_cmd *cmd = xfer->cmds; + + /* Calculate the Transfer Command Descriptor */ + cmd->rnw = i3c_xfers[i].rnw; + cmd->cmd0 = NCMDQP_DEV_INDEX(data->index) | NCMDQP_MODE(0) | + NCMDQP_RNW(cmd->rnw) | NCMDQP_ROC | NCMDQP_TOC; + + if (i3c_xfers[i].rnw) { + cmd->rx_count = 0; + cmd->cmd0 |= NCMDQP_TID(I3C_READ); + cmd->rx_buf = i3c_xfers[i].data.in; + cmd->len = i3c_xfers[i].len; + i3c->internal_state = I3C_INTERNAL_STATE_CONTROLLER_READ; + } else { + cmd->tx_count = 0; + cmd->cmd0 |= NCMDQP_TID(I3C_WRITE); + cmd->tx_buf = i3c_xfers[i].data.out; + cmd->len = i3c_xfers[i].len; + i3c->internal_state = I3C_INTERNAL_STATE_CONTROLLER_WRITE; + } + + if (!i3c_xfers[i].rnw && i3c_xfers[i].len > 4) { + i3c_writel_fifo(i3c->regs + NTDTBP0, cmd->tx_buf, cmd->len); + if (cmd->len > NTDTBP0_DEPTH * sizeof(u32)) + renesas_set_bit(i3c->regs, NTIE, NTIE_TDBEIE0); + } + + time_left = renesas_i3c_wait_xfer(i3c, xfer); + if (!time_left) + xfer_failed = true; + } + + if (xfer_failed) + renesas_i3c_irqs_mask_and_clear(i3c); + + return 0; +} + +static int renesas_i3c_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_i2c_dev_data *data; + int pos, ret; + + pos = renesas_i3c_get_free_pos(i3c); + if (pos < 0) + return pos; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + data->index = pos; + i3c->addrs[pos] = dev->info.dyn_addr ? : dev->info.static_addr; + i3c->free_pos &= ~BIT(pos); + + renesas_writel(i3c->regs, DATBAS(pos), DATBAS_DVSTAD(dev->info.static_addr) | + datbas_dvdyad_with_parity(i3c->addrs[pos])); + i3c_dev_set_master_data(dev, data); + + return 0; +} + +static int renesas_i3c_reattach_i3c_dev(struct i3c_dev_desc *dev, + u8 old_dyn_addr) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + int pos, ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + pos = renesas_i3c_get_free_pos(i3c); + + if (data->index > pos && pos >= 0) { + renesas_writel(i3c->regs, DATBAS(data->index), 0); + i3c->addrs[data->index] = 0; + i3c->free_pos |= BIT(data->index); + + data->index = pos; + i3c->free_pos &= ~BIT(data->index); + } + + i3c->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr : + dev->info.static_addr; + + renesas_writel(i3c->regs, DATBAS(data->index), + DATBAS_DVSTAD(dev->info.static_addr) | + datbas_dvdyad_with_parity(i3c->addrs[data->index])); + + return 0; +} + +static void renesas_i3c_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct renesas_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + int ret; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (!ret) + renesas_writel(i3c->regs, DATBAS(data->index), 0); + + i3c_dev_set_master_data(dev, NULL); + i3c->addrs[data->index] = 0; + i3c->free_pos |= BIT(data->index); + kfree(data); +} + +static int renesas_i3c_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *i2c_xfers, + int i2c_nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_cmd *cmd; + u8 start_bit = CNDCTL_STCND; + unsigned long time_left; + bool xfer_failed = false; + int i, ret; + + if (!i2c_nxfers) + return 0; + + struct renesas_i3c_xfer *xfer __free(kfree) = renesas_i3c_alloc_xfer(i3c, 1); + if (!xfer) + return -ENOMEM; + + init_completion(&xfer->comp); + xfer->is_i2c_xfer = true; + cmd = xfer->cmds; + + PM_RUNTIME_ACQUIRE_IF_ENABLED_AUTOSUSPEND(i3c->dev, pm); + ret = PM_RUNTIME_ACQUIRE_ERR(&pm); + if (ret) + return ret; + + renesas_i3c_bus_enable(m, false); + + if (!(renesas_readl(i3c->regs, BCST) & BCST_BFREF)) { + cmd->err = -EBUSY; + return cmd->err; + } + + renesas_writel(i3c->regs, BST, 0); + + renesas_i3c_enqueue_xfer(i3c, xfer); + + for (i = 0; i < i2c_nxfers; i++) { + cmd->i2c_bytes_left = I2C_INIT_MSG; + cmd->i2c_buf = i2c_xfers[i].buf; + cmd->msg = &i2c_xfers[i]; + cmd->i2c_is_last = (i == i2c_nxfers - 1); + + renesas_set_bit(i3c->regs, BIE, BIE_NACKDIE); + renesas_set_bit(i3c->regs, NTIE, NTIE_TDBEIE0); + renesas_set_bit(i3c->regs, BIE, BIE_STCNDDIE); + + /* Issue Start condition */ + renesas_set_bit(i3c->regs, CNDCTL, start_bit); + + renesas_set_bit(i3c->regs, NTSTE, NTSTE_TDBEE0); + + time_left = wait_for_completion_timeout(&xfer->comp, m->i2c.timeout); + if (!time_left) + xfer_failed = true; + + if (cmd->err) + break; + + start_bit = CNDCTL_SRCND; + } + + renesas_i3c_dequeue_xfer(i3c, xfer); + + if (xfer_failed) + renesas_i3c_irqs_mask_and_clear(i3c); + + return cmd->err; +} + +static int renesas_i3c_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + struct renesas_i3c_i2c_dev_data *data; + int pos; + + pos = renesas_i3c_get_free_pos(i3c); + if (pos < 0) + return pos; + + data = kzalloc_obj(*data); + if (!data) + return -ENOMEM; + + data->index = pos; + i3c->addrs[pos] = dev->addr; + i3c->free_pos &= ~BIT(pos); + i2c_dev_set_master_data(dev, data); + + return 0; +} + +static void renesas_i3c_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct renesas_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct renesas_i3c *i3c = to_renesas_i3c(m); + + i2c_dev_set_master_data(dev, NULL); + i3c->addrs[data->index] = 0; + i3c->free_pos |= BIT(data->index); + kfree(data); +} + +static irqreturn_t renesas_i3c_tx_isr(int irq, void *data) +{ + struct renesas_i3c *i3c = data; + struct renesas_i3c_xfer *xfer; + struct renesas_i3c_cmd *cmd; + u8 val; + + scoped_guard(spinlock, &i3c->xferqueue.lock) { + xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + + cmd = xfer->cmds; + + if (xfer->is_i2c_xfer) { + if (!cmd->i2c_bytes_left) + return IRQ_NONE; + + if (cmd->i2c_bytes_left != I2C_INIT_MSG) { + val = *cmd->i2c_buf; + cmd->i2c_buf++; + cmd->i2c_bytes_left--; + renesas_writel(i3c->regs, NTDTBP0, val); + } + + if (cmd->i2c_bytes_left == 0) { + renesas_clear_bit(i3c->regs, NTIE, NTIE_TDBEIE0); + renesas_set_bit(i3c->regs, BIE, BIE_TENDIE); + } + + /* Clear the Transmit Buffer Empty status flag. */ + renesas_clear_bit(i3c->regs, NTST, NTST_TDBEF0); + } else { + i3c_writel_fifo(i3c->regs + NTDTBP0, cmd->tx_buf, cmd->len); + } + } + + return IRQ_HANDLED; +} + +static irqreturn_t renesas_i3c_resp_isr(int irq, void *data) +{ + struct renesas_i3c *i3c = data; + struct renesas_i3c_xfer *xfer; + struct renesas_i3c_cmd *cmd; + u32 resp_descriptor = renesas_readl(i3c->regs, NRSPQP); + u32 bytes_remaining = 0; + u32 ntst, data_len; + int ret = 0; + + scoped_guard(spinlock, &i3c->xferqueue.lock) { + xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + + cmd = xfer->cmds; + + /* Clear the Respone Queue Full status flag*/ + renesas_clear_bit(i3c->regs, NTST, NTST_RSPQFF); + + data_len = NRSPQP_DATA_LEN(resp_descriptor); + + switch (i3c->internal_state) { + case I3C_INTERNAL_STATE_CONTROLLER_ENTDAA: + cmd->rx_count = data_len; + break; + case I3C_INTERNAL_STATE_CONTROLLER_WRITE: + case I3C_INTERNAL_STATE_CONTROLLER_COMMAND_WRITE: + /* Disable the transmit IRQ if it hasn't been disabled already. */ + renesas_clear_bit(i3c->regs, NTIE, NTIE_TDBEIE0); + break; + case I3C_INTERNAL_STATE_CONTROLLER_READ: + case I3C_INTERNAL_STATE_CONTROLLER_COMMAND_READ: + if (!cmd->err) { + u32 rx_count = min(cmd->rx_count, data_len); + + bytes_remaining = data_len - rx_count; + if (bytes_remaining) + i3c_readl_fifo(i3c->regs + NTDTBP0, + cmd->rx_buf + rx_count, + bytes_remaining); + cmd->rx_count = data_len; + } + renesas_clear_bit(i3c->regs, NTIE, NTIE_RDBFIE0); + break; + default: + break; + } + + switch (NRSPQP_ERR_STATUS(resp_descriptor)) { + case NRSPQP_NO_ERROR: + break; + case NRSPQP_ERROR_PARITY: + case NRSPQP_ERROR_IBA_NACK: + case NRSPQP_ERROR_TRANSF_ABORT: + case NRSPQP_ERROR_CRC: + case NRSPQP_ERROR_FRAME: + ret = -EIO; + break; + case NRSPQP_ERROR_OVER_UNDER_FLOW: + ret = -ENOSPC; + break; + case NRSPQP_ERROR_UNSUPPORTED: + ret = -EOPNOTSUPP; + break; + case NRSPQP_ERROR_I2C_W_NACK_ERR: + case NRSPQP_ERROR_ADDRESS_NACK: + default: + ret = -EINVAL; + break; + } + + /* + * If the transfer was aborted, then the abort flag must be cleared + * before notifying the application that a transfer has completed. + */ + ntst = renesas_readl(i3c->regs, NTST); + if (ntst & NTST_TABTF) + renesas_clear_bit(i3c->regs, BCTL, BCTL_ABT); + + /* Clear error status flags. */ + renesas_clear_bit(i3c->regs, NTST, NTST_TEF | NTST_TABTF); + + xfer->ret = ret; + complete(&xfer->comp); + + xfer = list_first_entry_or_null(&i3c->xferqueue.list, + struct renesas_i3c_xfer, node); + if (xfer) + list_del_init(&xfer->node); + + i3c->xferqueue.cur = xfer; + } + + return IRQ_HANDLED; +} + +static irqreturn_t renesas_i3c_tend_isr(int irq, void *data) +{ + struct renesas_i3c *i3c = data; + struct renesas_i3c_xfer *xfer; + struct renesas_i3c_cmd *cmd; + + scoped_guard(spinlock, &i3c->xferqueue.lock) { + xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + + cmd = xfer->cmds; + + if (xfer->is_i2c_xfer) { + if (renesas_readl(i3c->regs, BST) & BST_NACKDF) { + /* We got a NACKIE */ + renesas_readl(i3c->regs, NTDTBP0); /* dummy read */ + renesas_clear_bit(i3c->regs, BST, BST_NACKDF); + cmd->err = -ENXIO; + } else if (cmd->i2c_bytes_left) { + renesas_set_bit(i3c->regs, NTIE, NTIE_TDBEIE0); + return IRQ_NONE; + } + + if (cmd->i2c_is_last || cmd->err) { + renesas_clear_bit(i3c->regs, BIE, BIE_TENDIE); + renesas_set_bit(i3c->regs, BIE, BIE_SPCNDDIE); + renesas_set_bit(i3c->regs, CNDCTL, CNDCTL_SPCND); + } else { + /* Transfer is complete, but do not send STOP */ + renesas_clear_bit(i3c->regs, NTSTE, NTSTE_TDBEE0); + renesas_clear_bit(i3c->regs, BIE, BIE_TENDIE); + xfer->ret = 0; + complete(&xfer->comp); + } + } + + /* Clear the Transmit Buffer Empty status flag. */ + renesas_clear_bit(i3c->regs, BST, BST_TENDF); + } + + return IRQ_HANDLED; +} + +static irqreturn_t renesas_i3c_rx_isr(int irq, void *data) +{ + struct renesas_i3c *i3c = data; + struct renesas_i3c_xfer *xfer; + struct renesas_i3c_cmd *cmd; + int read_bytes; + + /* If resp_isr already read the data and updated 'xfer', we can just leave */ + if (!(renesas_readl(i3c->regs, NTIE) & NTIE_RDBFIE0)) + return IRQ_NONE; + + scoped_guard(spinlock, &i3c->xferqueue.lock) { + xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + + cmd = xfer->cmds; + + if (xfer->is_i2c_xfer) { + if (!cmd->i2c_bytes_left) + return IRQ_NONE; + + if (cmd->i2c_bytes_left == I2C_INIT_MSG) { + cmd->i2c_bytes_left = cmd->msg->len; + renesas_set_bit(i3c->regs, SCSTRCTL, SCSTRCTL_RWE); + renesas_readl(i3c->regs, NTDTBP0); /* dummy read */ + if (cmd->i2c_bytes_left == 1) + renesas_writel(i3c->regs, ACKCTL, ACKCTL_ACKT | ACKCTL_ACKTWP); + return IRQ_HANDLED; + } + + if (cmd->i2c_bytes_left == 1) { + /* STOP must come before we set ACKCTL! */ + if (cmd->i2c_is_last) { + renesas_set_bit(i3c->regs, BIE, BIE_SPCNDDIE); + renesas_clear_bit(i3c->regs, BST, BST_SPCNDDF); + renesas_set_bit(i3c->regs, CNDCTL, CNDCTL_SPCND); + } + renesas_writel(i3c->regs, ACKCTL, ACKCTL_ACKT | ACKCTL_ACKTWP); + } else { + renesas_writel(i3c->regs, ACKCTL, ACKCTL_ACKTWP); + } + + /* Reading acks the RIE interrupt */ + *cmd->i2c_buf = renesas_readl(i3c->regs, NTDTBP0); + cmd->i2c_buf++; + cmd->i2c_bytes_left--; + } else { + read_bytes = NDBSTLV0_RDBLV(renesas_readl(i3c->regs, NDBSTLV0)) * sizeof(u32); + i3c_readl_fifo(i3c->regs + NTDTBP0, cmd->rx_buf, read_bytes); + cmd->rx_count = read_bytes; + } + + /* Clear the Read Buffer Full status flag. */ + renesas_clear_bit(i3c->regs, NTST, NTST_RDBFF0); + } + + return IRQ_HANDLED; +} + +static irqreturn_t renesas_i3c_stop_isr(int irq, void *data) +{ + struct renesas_i3c *i3c = data; + struct renesas_i3c_xfer *xfer; + + scoped_guard(spinlock, &i3c->xferqueue.lock) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + renesas_clear_bit(i3c->regs, SCSTRCTL, SCSTRCTL_RWE); + + xfer = i3c->xferqueue.cur; + if (!xfer) + return IRQ_HANDLED; + + xfer->ret = 0; + complete(&xfer->comp); + } + + return IRQ_HANDLED; +} + +static irqreturn_t renesas_i3c_start_isr(int irq, void *data) +{ + struct renesas_i3c *i3c = data; + struct renesas_i3c_xfer *xfer; + struct renesas_i3c_cmd *cmd; + u8 val; + + scoped_guard(spinlock, &i3c->xferqueue.lock) { + xfer = i3c->xferqueue.cur; + if (!xfer) { + renesas_i3c_irqs_mask_and_clear_locked(i3c); + return IRQ_HANDLED; + } + + cmd = xfer->cmds; + + if (xfer->is_i2c_xfer) { + if (!cmd->i2c_bytes_left) + return IRQ_NONE; + + if (cmd->i2c_bytes_left == I2C_INIT_MSG) { + if (cmd->msg->flags & I2C_M_RD) { + /* On read, switch over to receive interrupt */ + renesas_clear_bit(i3c->regs, NTIE, NTIE_TDBEIE0); + renesas_set_bit(i3c->regs, NTIE, NTIE_RDBFIE0); + } else { + /* On write, initialize length */ + cmd->i2c_bytes_left = cmd->msg->len; + } + + val = i2c_8bit_addr_from_msg(cmd->msg); + renesas_writel(i3c->regs, NTDTBP0, val); + } + } + + renesas_clear_bit(i3c->regs, BIE, BIE_STCNDDIE); + renesas_clear_bit(i3c->regs, BST, BST_STCNDDF); + } + + return IRQ_HANDLED; +} + +static const struct i3c_master_controller_ops renesas_i3c_ops = { + .bus_init = renesas_i3c_bus_init, + .bus_cleanup = renesas_i3c_bus_cleanup, + .attach_i3c_dev = renesas_i3c_attach_i3c_dev, + .reattach_i3c_dev = renesas_i3c_reattach_i3c_dev, + .detach_i3c_dev = renesas_i3c_detach_i3c_dev, + .do_daa = renesas_i3c_daa, + .supports_ccc_cmd = renesas_i3c_supports_ccc_cmd, + .send_ccc_cmd = renesas_i3c_send_ccc_cmd, + .i3c_xfers = renesas_i3c_i3c_xfers, + .attach_i2c_dev = renesas_i3c_attach_i2c_dev, + .detach_i2c_dev = renesas_i3c_detach_i2c_dev, + .i2c_xfers = renesas_i3c_i2c_xfers, +}; + +static const struct renesas_i3c_irq_desc renesas_i3c_irqs[] = { + { .name = "resp", .isr = renesas_i3c_resp_isr, .desc = "i3c-resp" }, + { .name = "rx", .isr = renesas_i3c_rx_isr, .desc = "i3c-rx" }, + { .name = "tx", .isr = renesas_i3c_tx_isr, .desc = "i3c-tx" }, + { .name = "st", .isr = renesas_i3c_start_isr, .desc = "i3c-start" }, + { .name = "sp", .isr = renesas_i3c_stop_isr, .desc = "i3c-stop" }, + { .name = "tend", .isr = renesas_i3c_tend_isr, .desc = "i3c-tend" }, + { .name = "nack", .isr = renesas_i3c_tend_isr, .desc = "i3c-nack" }, +}; + +static int renesas_i3c_probe(struct platform_device *pdev) +{ + struct renesas_i3c *i3c; + int ret, i; + + i3c = devm_kzalloc(&pdev->dev, sizeof(*i3c), GFP_KERNEL); + if (!i3c) + return -ENOMEM; + + i3c->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(i3c->regs)) + return PTR_ERR(i3c->regs); + + i3c->tclk = devm_clk_get(&pdev->dev, "tclk"); + if (IS_ERR(i3c->tclk)) + return dev_err_probe(&pdev->dev, PTR_ERR(i3c->tclk), "Failed to get tclk"); + + i3c->dev = &pdev->dev; + pm_runtime_set_autosuspend_delay(&pdev->dev, 300); + pm_runtime_use_autosuspend(&pdev->dev); + ret = devm_pm_runtime_enable(&pdev->dev); + if (ret) + return ret; + + i3c->tresetn = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev, "tresetn"); + if (IS_ERR(i3c->tresetn)) + return dev_err_probe(&pdev->dev, PTR_ERR(i3c->tresetn), + "Error: missing tresetn ctrl\n"); + + i3c->presetn = devm_reset_control_get_optional_exclusive_deasserted(&pdev->dev, "presetn"); + if (IS_ERR(i3c->presetn)) + return dev_err_probe(&pdev->dev, PTR_ERR(i3c->presetn), + "Error: missing presetn ctrl\n"); + + spin_lock_init(&i3c->xferqueue.lock); + INIT_LIST_HEAD(&i3c->xferqueue.list); + + ret = renesas_i3c_reset(i3c); + if (ret) + return ret; + + for (i = 0; i < ARRAY_SIZE(renesas_i3c_irqs); i++) { + const char *irqname; + + ret = platform_get_irq_byname(pdev, renesas_i3c_irqs[i].name); + if (ret < 0) + return ret; + + irqname = devm_kasprintf(&pdev->dev, GFP_KERNEL, "%s:%s", dev_name(&pdev->dev), + renesas_i3c_irqs[i].desc); + if (!irqname) + return -ENOMEM; + + ret = devm_request_irq(&pdev->dev, ret, renesas_i3c_irqs[i].isr, + 0, irqname, i3c); + if (ret) + return ret; + } + + platform_set_drvdata(pdev, i3c); + + i3c->maxdevs = RENESAS_I3C_MAX_DEVS; + i3c->free_pos = GENMASK(i3c->maxdevs - 1, 0); + + return i3c_master_register(&i3c->base, &pdev->dev, &renesas_i3c_ops, false); +} + +static void renesas_i3c_remove(struct platform_device *pdev) +{ + struct renesas_i3c *i3c = platform_get_drvdata(pdev); + + i3c_master_unregister(&i3c->base); +} + +static int renesas_i3c_suspend(struct device *dev) +{ + struct renesas_i3c *i3c = dev_get_drvdata(dev); + struct reset_control_bulk_data resets[] = { + { .rstc = i3c->presetn }, + { .rstc = i3c->tresetn }, + }; + int ret; + + i2c_mark_adapter_suspended(&i3c->base.i2c); + + ret = reset_control_bulk_assert(ARRAY_SIZE(resets), resets); + if (ret) + goto err_mark_resumed; + + return 0; + +err_mark_resumed: + i2c_mark_adapter_resumed(&i3c->base.i2c); + + return ret; +} + +static int renesas_i3c_resume(struct device *dev) +{ + struct renesas_i3c *i3c = dev_get_drvdata(dev); + struct reset_control_bulk_data resets[] = { + { .rstc = i3c->presetn }, + { .rstc = i3c->tresetn }, + }; + int ret; + + ret = reset_control_bulk_deassert(ARRAY_SIZE(resets), resets); + if (ret) + return ret; + + ret = renesas_i3c_reset(i3c); + if (ret) + goto err_resets_asserted; + + ret = pm_runtime_resume_and_get(dev); + if (ret) + goto err_resets_asserted; + + /* Re-store I3C registers value. */ + renesas_writel(i3c->regs, STDBR, i3c->i3c_STDBR); + renesas_writel(i3c->regs, EXTBR, i3c->extbr); + renesas_writel(i3c->regs, REFCKCTL, + REFCKCTL_IREFCKS(i3c->refclk_div)); + renesas_writel(i3c->regs, MSDVAD, MSDVAD_MDYADV | + MSDVAD_MDYAD(i3c->dyn_addr)); + + /* I3C hw init. */ + renesas_i3c_hw_init(i3c); + + ret = i3c_master_do_daa_ext(&i3c->base, true); + if (ret) + dev_err(dev, "DAA failed on resume, ret=%d", ret); + + i2c_mark_adapter_resumed(&i3c->base.i2c); + + pm_runtime_put_autosuspend(dev); + + /* + * I3C devices may have retained their dynamic address anyway. Do not + * fail the resume because of DAA error. + */ + return 0; + +err_resets_asserted: + /* + * If this happens, there is no way to recover from this state without + * reloading the driver. We want to avoid keeping the reset line + * deasserted unnecessarily. The runtime paths will still work correctly + * even if the IP registers are accessed while reset is asserted (e.g. + * if a runtime path is triggered after a failed resume). Checked on + * RZ/G3S. + */ + reset_control_bulk_assert(ARRAY_SIZE(resets), resets); + return ret; +} + +static const struct dev_pm_ops renesas_i3c_pm_ops = { + SYSTEM_SLEEP_PM_OPS(renesas_i3c_suspend, renesas_i3c_resume) +}; + +static const struct of_device_id renesas_i3c_of_ids[] = { + { .compatible = "renesas,r9a08g045-i3c" }, + { .compatible = "renesas,r9a09g047-i3c" }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, renesas_i3c_of_ids); + +static struct platform_driver renesas_i3c = { + .probe = renesas_i3c_probe, + .remove = renesas_i3c_remove, + .driver = { + .name = "renesas-i3c", + .of_match_table = renesas_i3c_of_ids, + .pm = pm_sleep_ptr(&renesas_i3c_pm_ops), + }, +}; +module_platform_driver(renesas_i3c); + +MODULE_AUTHOR("Wolfram Sang <wsa+renesas@sang-engineering.com>"); +MODULE_AUTHOR("Renesas BSP teams"); +MODULE_DESCRIPTION("Renesas I3C controller driver"); +MODULE_LICENSE("GPL"); diff --git a/drivers/i3c/master/svc-i3c-master.c b/drivers/i3c/master/svc-i3c-master.c new file mode 100644 index 000000000..e700392f0 --- /dev/null +++ b/drivers/i3c/master/svc-i3c-master.c @@ -0,0 +1,2187 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Silvaco dual-role I3C master driver + * + * Copyright (C) 2020 Silvaco + * Author: Miquel RAYNAL <miquel.raynal@bootlin.com> + * Based on a work from: Conor Culhane <conor.culhane@silvaco.com> + */ + +#include <linux/bitfield.h> +#include <linux/clk.h> +#include <linux/completion.h> +#include <linux/errno.h> +#include <linux/i3c/master.h> +#include <linux/interrupt.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/pinctrl/consumer.h> +#include <linux/platform_device.h> +#include <linux/pm_runtime.h> + +/* Master Mode Registers */ +#define SVC_I3C_MCONFIG 0x000 +#define SVC_I3C_MCONFIG_MASTER_EN BIT(0) +#define SVC_I3C_MCONFIG_DISTO(x) FIELD_PREP(BIT(3), (x)) +#define SVC_I3C_MCONFIG_HKEEP(x) FIELD_PREP(GENMASK(5, 4), (x)) +#define SVC_I3C_MCONFIG_ODSTOP(x) FIELD_PREP(BIT(6), (x)) +#define SVC_I3C_MCONFIG_PPBAUD(x) FIELD_PREP(GENMASK(11, 8), (x)) +#define SVC_I3C_MCONFIG_PPLOW(x) FIELD_PREP(GENMASK(15, 12), (x)) +#define SVC_I3C_MCONFIG_ODBAUD(x) FIELD_PREP(GENMASK(23, 16), (x)) +#define SVC_I3C_MCONFIG_ODHPP(x) FIELD_PREP(BIT(24), (x)) +#define SVC_I3C_MCONFIG_SKEW(x) FIELD_PREP(GENMASK(27, 25), (x)) +#define SVC_I3C_MCONFIG_SKEW_MASK GENMASK(27, 25) +#define SVC_I3C_MCONFIG_I2CBAUD(x) FIELD_PREP(GENMASK(31, 28), (x)) + +#define SVC_I3C_MCTRL 0x084 +#define SVC_I3C_MCTRL_REQUEST_MASK GENMASK(2, 0) +#define SVC_I3C_MCTRL_REQUEST_NONE 0 +#define SVC_I3C_MCTRL_REQUEST_START_ADDR 1 +#define SVC_I3C_MCTRL_REQUEST_STOP 2 +#define SVC_I3C_MCTRL_REQUEST_FORCE_EXIT 6 +#define SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK 3 +#define SVC_I3C_MCTRL_REQUEST_PROC_DAA 4 +#define SVC_I3C_MCTRL_REQUEST_AUTO_IBI 7 +#define SVC_I3C_MCTRL_TYPE_I3C 0 +#define SVC_I3C_MCTRL_TYPE_I2C BIT(4) +#define SVC_I3C_MCTRL_TYPE_DDR BIT(5) +#define SVC_I3C_MCTRL_IBIRESP_AUTO 0 +#define SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE 0 +#define SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE BIT(7) +#define SVC_I3C_MCTRL_IBIRESP_NACK BIT(6) +#define SVC_I3C_MCTRL_IBIRESP_MANUAL GENMASK(7, 6) +#define SVC_I3C_MCTRL_DIR(x) FIELD_PREP(BIT(8), (x)) +#define SVC_I3C_MCTRL_DIR_WRITE 0 +#define SVC_I3C_MCTRL_DIR_READ 1 +#define SVC_I3C_MCTRL_ADDR(x) FIELD_PREP(GENMASK(15, 9), (x)) +#define SVC_I3C_MCTRL_RDTERM(x) FIELD_PREP(GENMASK(23, 16), (x)) + +#define SVC_I3C_MSTATUS 0x088 +#define SVC_I3C_MSTATUS_STATE(x) FIELD_GET(GENMASK(2, 0), (x)) +#define SVC_I3C_MSTATUS_STATE_DAA(x) (SVC_I3C_MSTATUS_STATE(x) == 5) +#define SVC_I3C_MSTATUS_STATE_SLVREQ(x) (SVC_I3C_MSTATUS_STATE(x) == 1) +#define SVC_I3C_MSTATUS_STATE_IDLE(x) (SVC_I3C_MSTATUS_STATE(x) == 0) +#define SVC_I3C_MSTATUS_BETWEEN(x) FIELD_GET(BIT(4), (x)) +#define SVC_I3C_MSTATUS_NACKED(x) FIELD_GET(BIT(5), (x)) +#define SVC_I3C_MSTATUS_IBITYPE(x) FIELD_GET(GENMASK(7, 6), (x)) +#define SVC_I3C_MSTATUS_IBITYPE_IBI 1 +#define SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST 2 +#define SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN 3 +#define SVC_I3C_MINT_SLVSTART BIT(8) +#define SVC_I3C_MINT_MCTRLDONE BIT(9) +#define SVC_I3C_MINT_COMPLETE BIT(10) +#define SVC_I3C_MINT_RXPEND BIT(11) +#define SVC_I3C_MINT_TXNOTFULL BIT(12) +#define SVC_I3C_MINT_IBIWON BIT(13) +#define SVC_I3C_MINT_ERRWARN BIT(15) +#define SVC_I3C_MSTATUS_SLVSTART(x) FIELD_GET(SVC_I3C_MINT_SLVSTART, (x)) +#define SVC_I3C_MSTATUS_MCTRLDONE(x) FIELD_GET(SVC_I3C_MINT_MCTRLDONE, (x)) +#define SVC_I3C_MSTATUS_COMPLETE(x) FIELD_GET(SVC_I3C_MINT_COMPLETE, (x)) +#define SVC_I3C_MSTATUS_RXPEND(x) FIELD_GET(SVC_I3C_MINT_RXPEND, (x)) +#define SVC_I3C_MSTATUS_TXNOTFULL(x) FIELD_GET(SVC_I3C_MINT_TXNOTFULL, (x)) +#define SVC_I3C_MSTATUS_IBIWON(x) FIELD_GET(SVC_I3C_MINT_IBIWON, (x)) +#define SVC_I3C_MSTATUS_ERRWARN(x) FIELD_GET(SVC_I3C_MINT_ERRWARN, (x)) +#define SVC_I3C_MSTATUS_IBIADDR(x) FIELD_GET(GENMASK(30, 24), (x)) + +#define SVC_I3C_IBIRULES 0x08C +#define SVC_I3C_IBIRULES_ADDR(slot, addr) FIELD_PREP(GENMASK(29, 0), \ + ((addr) & 0x3F) << ((slot) * 6)) +#define SVC_I3C_IBIRULES_ADDRS 5 +#define SVC_I3C_IBIRULES_MSB0 BIT(30) +#define SVC_I3C_IBIRULES_NOBYTE BIT(31) +#define SVC_I3C_IBIRULES_MANDBYTE 0 +#define SVC_I3C_MINTSET 0x090 +#define SVC_I3C_MINTCLR 0x094 +#define SVC_I3C_MINTMASKED 0x098 +#define SVC_I3C_MERRWARN 0x09C +#define SVC_I3C_MERRWARN_NACK BIT(2) +#define SVC_I3C_MERRWARN_CRC BIT(10) +#define SVC_I3C_MERRWARN_TIMEOUT BIT(20) +#define SVC_I3C_MDMACTRL 0x0A0 +#define SVC_I3C_MDATACTRL 0x0AC +#define SVC_I3C_MDATACTRL_FLUSHTB BIT(0) +#define SVC_I3C_MDATACTRL_FLUSHRB BIT(1) +#define SVC_I3C_MDATACTRL_UNLOCK_TRIG BIT(3) +#define SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL GENMASK(5, 4) +#define SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY 0 +#define SVC_I3C_MDATACTRL_RXCOUNT(x) FIELD_GET(GENMASK(28, 24), (x)) +#define SVC_I3C_MDATACTRL_TXCOUNT(x) FIELD_GET(GENMASK(20, 16), (x)) +#define SVC_I3C_MDATACTRL_TXFULL BIT(30) +#define SVC_I3C_MDATACTRL_RXEMPTY BIT(31) + +#define SVC_I3C_MWDATAB 0x0B0 +#define SVC_I3C_MWDATAB_END BIT(8) + +#define SVC_I3C_MWDATABE 0x0B4 +#define SVC_I3C_MWDATAH 0x0B8 +#define SVC_I3C_MWDATAHE 0x0BC +#define SVC_I3C_MRDATAB 0x0C0 +#define SVC_I3C_MRDATAH 0x0C8 +#define SVC_I3C_MWDATAB1 0x0CC +#define SVC_I3C_MWMSG_SDR 0x0D0 +#define SVC_I3C_MRMSG_SDR 0x0D4 +#define SVC_I3C_MWMSG_DDR 0x0D8 +#define SVC_I3C_MRMSG_DDR 0x0DC + +#define SVC_I3C_MDYNADDR 0x0E4 +#define SVC_MDYNADDR_VALID BIT(0) +#define SVC_MDYNADDR_ADDR(x) FIELD_PREP(GENMASK(7, 1), (x)) + +#define SVC_I3C_MAX_DEVS 32 +#define SVC_I3C_PM_TIMEOUT_MS 1000 + +/* This parameter depends on the implementation and may be tuned */ +#define SVC_I3C_FIFO_SIZE 16 +#define SVC_I3C_PPBAUD_MAX 15 +#define SVC_I3C_QUICK_I2C_CLK 4170000 + +#define SVC_I3C_EVENT_IBI GENMASK(7, 0) +#define SVC_I3C_EVENT_HOTJOIN BIT(31) + +/* + * SVC_I3C_QUIRK_FIFO_EMPTY: + * I3C HW stalls the write transfer if the transmit FIFO becomes empty, + * when new data is written to FIFO, I3C HW resumes the transfer but + * the first transmitted data bit may have the wrong value. + * Workaround: + * Fill the FIFO in advance to prevent FIFO from becoming empty. + */ +#define SVC_I3C_QUIRK_FIFO_EMPTY BIT(0) +/* + * SVC_I3C_QUIRK_FLASE_SLVSTART: + * I3C HW may generate an invalid SlvStart event when emitting a STOP. + * If it is a true SlvStart, the MSTATUS state is SLVREQ. + */ +#define SVC_I3C_QUIRK_FALSE_SLVSTART BIT(1) +/* + * SVC_I3C_QUIRK_DAA_CORRUPT: + * When MCONFIG.SKEW=0 and MCONFIG.ODHPP=0, the ENTDAA transaction gets + * corrupted and results in a no repeated-start condition at the end of + * address assignment. + * Workaround: + * Set MCONFIG.SKEW to 1 before initiating the DAA process. After the DAA + * process is completed, return MCONFIG.SKEW to its previous value. + */ +#define SVC_I3C_QUIRK_DAA_CORRUPT BIT(2) + +struct svc_i3c_cmd { + u8 addr; + union { + bool rnw; + u8 cmd; + u32 rnw_cmd; + }; + u8 *in; + const void *out; + unsigned int len; + unsigned int actual_len; + struct i3c_xfer *xfer; + bool continued; +}; + +struct svc_i3c_xfer { + struct list_head node; + struct completion comp; + int ret; + unsigned int type; + unsigned int ncmds; + struct svc_i3c_cmd cmds[] __counted_by(ncmds); +}; + +struct svc_i3c_regs_save { + u32 mconfig; + u32 mdynaddr; +}; + +struct svc_i3c_drvdata { + u32 quirks; +}; + +/** + * struct svc_i3c_master - Silvaco I3C Master structure + * @base: I3C master controller + * @dev: Corresponding device + * @regs: Memory mapping + * @saved_regs: Volatile values for PM operations + * @free_slots: Bit array of available slots + * @addrs: Array containing the dynamic addresses of each attached device + * @descs: Array of descriptors, one per attached device + * @irq: Main interrupt + * @num_clks: I3C clock number + * @fclk: Fast clock (bus) + * @clks: I3C clock array + * @xferqueue: Transfer queue structure + * @xferqueue.list: List member + * @xferqueue.cur: Current ongoing transfer + * @xferqueue.lock: Queue lock + * @ibi: IBI structure + * @ibi.num_slots: Number of slots available in @ibi.slots + * @ibi.slots: Available IBI slots + * @ibi.tbq_slot: To be queued IBI slot + * @ibi.lock: IBI lock + * @lock: Transfer lock, protect between IBI work thread and callbacks from master + * @drvdata: Driver data + * @enabled_events: Bit masks for enable events (IBI, HotJoin). + * @mctrl_config: Configuration value in SVC_I3C_MCTRL for setting speed back. + */ +struct svc_i3c_master { + struct i3c_master_controller base; + struct device *dev; + void __iomem *regs; + struct svc_i3c_regs_save saved_regs; + u32 free_slots; + u8 addrs[SVC_I3C_MAX_DEVS]; + struct i3c_dev_desc *descs[SVC_I3C_MAX_DEVS]; + int irq; + int num_clks; + struct clk *fclk; + struct clk_bulk_data *clks; + struct { + struct list_head list; + struct svc_i3c_xfer *cur; + /* Prevent races between transfers */ + spinlock_t lock; + } xferqueue; + struct { + unsigned int num_slots; + struct i3c_dev_desc **slots; + struct i3c_ibi_slot *tbq_slot; + /* Prevent races within IBI handlers */ + spinlock_t lock; + } ibi; + struct mutex lock; + const struct svc_i3c_drvdata *drvdata; + u32 enabled_events; + u32 mctrl_config; +}; + +/** + * struct svc_i3c_i2c_dev_data - Device specific data + * @index: Index in the master tables corresponding to this device + * @ibi: IBI slot index in the master structure + * @ibi_pool: IBI pool associated to this device + */ +struct svc_i3c_i2c_dev_data { + u8 index; + int ibi; + struct i3c_generic_ibi_pool *ibi_pool; +}; + +static inline bool svc_has_quirk(struct svc_i3c_master *master, u32 quirk) +{ + return (master->drvdata->quirks & quirk); +} + +static inline bool svc_has_daa_corrupt(struct svc_i3c_master *master) +{ + return ((master->drvdata->quirks & SVC_I3C_QUIRK_DAA_CORRUPT) && + !(master->mctrl_config & + (SVC_I3C_MCONFIG_SKEW_MASK | SVC_I3C_MCONFIG_ODHPP(1)))); +} + +static inline bool is_events_enabled(struct svc_i3c_master *master, u32 mask) +{ + return !!(master->enabled_events & mask); +} + +static bool svc_i3c_master_error(struct svc_i3c_master *master) +{ + u32 mstatus, merrwarn; + + mstatus = readl(master->regs + SVC_I3C_MSTATUS); + if (SVC_I3C_MSTATUS_ERRWARN(mstatus)) { + merrwarn = readl(master->regs + SVC_I3C_MERRWARN); + writel(merrwarn, master->regs + SVC_I3C_MERRWARN); + + /* Ignore timeout error */ + if (merrwarn & SVC_I3C_MERRWARN_TIMEOUT) { + dev_dbg(master->dev, "Warning condition: MSTATUS 0x%08x, MERRWARN 0x%08x\n", + mstatus, merrwarn); + return false; + } + + dev_err(master->dev, + "Error condition: MSTATUS 0x%08x, MERRWARN 0x%08x\n", + mstatus, merrwarn); + + return true; + } + + return false; +} + +static void svc_i3c_master_enable_interrupts(struct svc_i3c_master *master, u32 mask) +{ + writel(mask, master->regs + SVC_I3C_MINTSET); +} + +static void svc_i3c_master_disable_interrupts(struct svc_i3c_master *master) +{ + u32 mask = readl(master->regs + SVC_I3C_MINTSET); + + writel(mask, master->regs + SVC_I3C_MINTCLR); +} + +static void svc_i3c_master_clear_merrwarn(struct svc_i3c_master *master) +{ + /* Clear pending warnings */ + writel(readl(master->regs + SVC_I3C_MERRWARN), + master->regs + SVC_I3C_MERRWARN); +} + +static void svc_i3c_master_flush_fifo(struct svc_i3c_master *master) +{ + /* Flush FIFOs */ + writel(SVC_I3C_MDATACTRL_FLUSHTB | SVC_I3C_MDATACTRL_FLUSHRB, + master->regs + SVC_I3C_MDATACTRL); +} + +static void svc_i3c_master_reset_fifo_trigger(struct svc_i3c_master *master) +{ + u32 reg; + + /* Set RX and TX trigger levels, flush FIFOs */ + reg = SVC_I3C_MDATACTRL_FLUSHTB | + SVC_I3C_MDATACTRL_FLUSHRB | + SVC_I3C_MDATACTRL_UNLOCK_TRIG | + SVC_I3C_MDATACTRL_TXTRIG_FIFO_NOT_FULL | + SVC_I3C_MDATACTRL_RXTRIG_FIFO_NOT_EMPTY; + writel(reg, master->regs + SVC_I3C_MDATACTRL); +} + +static void svc_i3c_master_reset(struct svc_i3c_master *master) +{ + svc_i3c_master_clear_merrwarn(master); + svc_i3c_master_reset_fifo_trigger(master); + svc_i3c_master_disable_interrupts(master); +} + +static inline struct svc_i3c_master * +to_svc_i3c_master(struct i3c_master_controller *master) +{ + return container_of(master, struct svc_i3c_master, base); +} + +static struct i3c_dev_desc * +svc_i3c_master_dev_from_addr(struct svc_i3c_master *master, + unsigned int ibiaddr) +{ + int i; + + for (i = 0; i < SVC_I3C_MAX_DEVS; i++) + if (master->addrs[i] == ibiaddr) + break; + + if (i == SVC_I3C_MAX_DEVS) + return NULL; + + return master->descs[i]; +} + +static bool svc_cmd_is_read(u32 rnw_cmd, u32 type) +{ + return (type == SVC_I3C_MCTRL_TYPE_DDR) ? (rnw_cmd & 0x80) : rnw_cmd; +} + +static void svc_i3c_master_emit_force_exit(struct svc_i3c_master *master) +{ + u32 reg; + + writel(SVC_I3C_MCTRL_REQUEST_FORCE_EXIT, master->regs + SVC_I3C_MCTRL); + + /* + * Not need check error here because it is never happen at hardware. + * IP just wait for few fclk cycle to complete DDR exit pattern. Even + * though fclk stop, timeout happen here, the whole data actually + * already finish transfer. The next command will be timeout because + * wrong hardware state. + */ + readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg), 0, 1000); + + /* + * This delay is necessary after the emission of a stop, otherwise eg. + * repeating IBIs do not get detected. There is a note in the manual + * about it, stating that the stop condition might not be settled + * correctly if a start condition follows too rapidly. + */ + udelay(1); +} + +static void svc_i3c_master_emit_stop(struct svc_i3c_master *master) +{ + writel(SVC_I3C_MCTRL_REQUEST_STOP, master->regs + SVC_I3C_MCTRL); + + /* + * This delay is necessary after the emission of a stop, otherwise eg. + * repeating IBIs do not get detected. There is a note in the manual + * about it, stating that the stop condition might not be settled + * correctly if a start condition follows too rapidly. + */ + udelay(1); +} + +static int svc_i3c_master_handle_ibi(struct svc_i3c_master *master, + struct i3c_dev_desc *dev) +{ + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_ibi_slot *slot; + unsigned int count; + u32 mdatactrl; + int ret, val; + u8 *buf; + + /* + * Wait for transfer to complete before returning. Otherwise, the EmitStop + * request might be sent when the transfer is not complete. + */ + ret = readl_relaxed_poll_timeout(master->regs + SVC_I3C_MSTATUS, val, + SVC_I3C_MSTATUS_COMPLETE(val), 0, 1000); + if (ret) { + dev_err(master->dev, "Timeout when polling for COMPLETE\n"); + return ret; + } + + slot = i3c_generic_ibi_get_free_slot(data->ibi_pool); + if (!slot) { + dev_dbg(master->dev, "No free ibi slot, drop the data\n"); + writel(SVC_I3C_MDATACTRL_FLUSHRB, master->regs + SVC_I3C_MDATACTRL); + return -ENOSPC; + } + + slot->len = 0; + buf = slot->data; + + while (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS)) && + slot->len < dev->ibi->max_payload_len) { + mdatactrl = readl(master->regs + SVC_I3C_MDATACTRL); + count = SVC_I3C_MDATACTRL_RXCOUNT(mdatactrl); + count = min(count, dev->ibi->max_payload_len - slot->len); + readsb(master->regs + SVC_I3C_MRDATAB, buf, count); + slot->len += count; + buf += count; + } + + /* + * The device may have sent more than the requested payload. Drop the + * extra bytes so they do not leak into the next transfer. + */ + if (SVC_I3C_MSTATUS_RXPEND(readl(master->regs + SVC_I3C_MSTATUS))) + writel(SVC_I3C_MDATACTRL_FLUSHRB, master->regs + SVC_I3C_MDATACTRL); + + master->ibi.tbq_slot = slot; + + return 0; +} + +static int svc_i3c_master_ack_ibi(struct svc_i3c_master *master, + bool mandatory_byte) +{ + unsigned int ibi_ack_nack; + u32 reg; + + ibi_ack_nack = SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK; + if (mandatory_byte) + ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITH_BYTE; + else + ibi_ack_nack |= SVC_I3C_MCTRL_IBIRESP_ACK_WITHOUT_BYTE; + + writel(ibi_ack_nack, master->regs + SVC_I3C_MCTRL); + + return readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg), 1, 1000); + +} + +static int svc_i3c_master_nack_ibi(struct svc_i3c_master *master) +{ + int ret; + u32 reg; + + writel(SVC_I3C_MCTRL_REQUEST_IBI_ACKNACK | + SVC_I3C_MCTRL_IBIRESP_NACK, + master->regs + SVC_I3C_MCTRL); + + ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg), 1, 1000); + return ret; +} + +static int svc_i3c_master_handle_ibi_won(struct svc_i3c_master *master, u32 mstatus) +{ + u32 ibitype; + int ret = 0; + + ibitype = SVC_I3C_MSTATUS_IBITYPE(mstatus); + + writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); + + /* Hardware can't auto emit NACK for hot join and master request */ + switch (ibitype) { + case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN: + case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: + ret = svc_i3c_master_nack_ibi(master); + } + + return ret; +} + +static void svc_i3c_master_ibi_isr(struct svc_i3c_master *master) +{ + struct svc_i3c_i2c_dev_data *data; + struct i3c_dev_desc *dev = NULL; + unsigned int ibitype, ibiaddr; + u32 status, val; + int ret; + + /* + * According to I3C spec ver 1.1, 09-Jun-2021, section 5.1.2.5: + * + * The I3C Controller shall hold SCL low while the Bus is in ACK/NACK Phase of I3C/I2C + * transfer. But maximum stall time is 100us. The IRQs have to be disabled to prevent + * schedule during the whole I3C transaction, otherwise, the I3C bus timeout may happen if + * any irq or schedule happen during transaction. + */ + guard(spinlock)(&master->xferqueue.lock); + + /* + * IBIWON may be set before SVC_I3C_MCTRL_REQUEST_AUTO_IBI, causing + * readl_relaxed_poll_timeout() to return immediately. Consequently, + * ibitype will be 0 since it was last updated only after the 8th SCL + * cycle, leading to missed client IBI handlers. + * + * A typical scenario is when IBIWON occurs and bus arbitration is lost + * at svc_i3c_master_i3c_xfers(). + * + * Clear SVC_I3C_MINT_IBIWON before sending SVC_I3C_MCTRL_REQUEST_AUTO_IBI. + */ + writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); + + /* + * Write REQUEST_START_ADDR request to emit broadcast address for arbitration, + * instend of using AUTO_IBI. + * + * Using AutoIBI request may cause controller to remain in AutoIBI state when + * there is a glitch on SDA line (high->low->high). + * 1. SDA high->low, raising an interrupt to execute IBI isr. + * 2. SDA low->high. + * 3. IBI isr writes an AutoIBI request. + * 4. The controller will not start AutoIBI process because SDA is not low. + * 5. IBIWON polling times out. + * 6. Controller remains in AutoIBI state and doesn't accept EmitStop request. + */ + writel(SVC_I3C_MCTRL_REQUEST_START_ADDR | + SVC_I3C_MCTRL_TYPE_I3C | + SVC_I3C_MCTRL_IBIRESP_MANUAL | + SVC_I3C_MCTRL_DIR(SVC_I3C_MCTRL_DIR_WRITE) | + SVC_I3C_MCTRL_ADDR(I3C_BROADCAST_ADDR), + master->regs + SVC_I3C_MCTRL); + + /* Wait for IBIWON, should take approximately 100us */ + ret = readl_relaxed_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, val, + SVC_I3C_MSTATUS_IBIWON(val), 0, 100); + if (ret) { + dev_err(master->dev, "Timeout when polling for IBIWON\n"); + svc_i3c_master_emit_stop(master); + return; + } + + status = readl(master->regs + SVC_I3C_MSTATUS); + ibitype = SVC_I3C_MSTATUS_IBITYPE(status); + ibiaddr = SVC_I3C_MSTATUS_IBIADDR(status); + + /* Handle the critical responses to IBI's */ + switch (ibitype) { + case SVC_I3C_MSTATUS_IBITYPE_IBI: + dev = svc_i3c_master_dev_from_addr(master, ibiaddr); + if (!dev || !is_events_enabled(master, SVC_I3C_EVENT_IBI)) { + svc_i3c_master_nack_ibi(master); + } else { + if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) + svc_i3c_master_ack_ibi(master, true); + else + svc_i3c_master_ack_ibi(master, false); + svc_i3c_master_handle_ibi(master, dev); + } + break; + case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN: + if (is_events_enabled(master, SVC_I3C_EVENT_HOTJOIN)) + svc_i3c_master_ack_ibi(master, false); + else + svc_i3c_master_nack_ibi(master); + break; + case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: + svc_i3c_master_nack_ibi(master); + break; + default: + break; + } + + /* + * If an error happened, we probably got interrupted and the exchange + * timedout. In this case we just drop everything, emit a stop and wait + * for the slave to interrupt again. + */ + if (svc_i3c_master_error(master)) { + if (master->ibi.tbq_slot && dev) { + data = i3c_dev_get_master_data(dev); + i3c_generic_ibi_recycle_slot(data->ibi_pool, + master->ibi.tbq_slot); + master->ibi.tbq_slot = NULL; + } + + svc_i3c_master_emit_stop(master); + + return; + } + + /* Handle the non critical tasks */ + switch (ibitype) { + case SVC_I3C_MSTATUS_IBITYPE_IBI: + svc_i3c_master_emit_stop(master); + if (dev) { + i3c_master_queue_ibi(dev, master->ibi.tbq_slot); + master->ibi.tbq_slot = NULL; + } + break; + case SVC_I3C_MSTATUS_IBITYPE_HOT_JOIN: + svc_i3c_master_emit_stop(master); + if (is_events_enabled(master, SVC_I3C_EVENT_HOTJOIN)) + i3c_master_queue_hotjoin(&master->base); + break; + case SVC_I3C_MSTATUS_IBITYPE_MASTER_REQUEST: + svc_i3c_master_emit_stop(master); + + /* + * If a target gets stuck holding SDA low, the controller reports a MR. + * On NPCM845, emitting STOP may spuriously set SLVSTART, retriggering + * the interrupt and re-entering MR handling, leading to an IRQ storm. + * Clear SLVSTART after STOP to break the loop. + */ + if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART)) + writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS); + break; + default: + break; + } +} + +static irqreturn_t svc_i3c_master_irq_handler(int irq, void *dev_id) +{ + struct svc_i3c_master *master = (struct svc_i3c_master *)dev_id; + u32 active = readl(master->regs + SVC_I3C_MSTATUS); + + if (!SVC_I3C_MSTATUS_SLVSTART(active)) + return IRQ_NONE; + + /* Clear the interrupt status */ + writel(SVC_I3C_MINT_SLVSTART, master->regs + SVC_I3C_MSTATUS); + + if (svc_has_quirk(master, SVC_I3C_QUIRK_FALSE_SLVSTART)) { + /* + * Re-read MSTATUS to obtain the latest state and avoid + * missing an IBI that arrives after MSTATUS is latched + * but before SLVSTART is cleared. + */ + active = readl(master->regs + SVC_I3C_MSTATUS); + + /* Ignore the false event */ + if (!SVC_I3C_MSTATUS_STATE_SLVREQ(active)) + return IRQ_HANDLED; + } + + /* + * The SDA line remains low until the request is processed. + * Receive the request in the interrupt context to respond promptly + * and restore the bus to idle state. + */ + svc_i3c_master_ibi_isr(master); + + return IRQ_HANDLED; +} + +static int svc_i3c_master_set_speed(struct i3c_master_controller *m, + enum i3c_open_drain_speed speed) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct i3c_bus *bus = i3c_master_get_bus(&master->base); + u32 ppbaud, odbaud, odhpp, mconfig; + unsigned long fclk_rate; + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__); + return ret; + } + + switch (speed) { + case I3C_OPEN_DRAIN_SLOW_SPEED: + fclk_rate = clk_get_rate(master->fclk); + if (!fclk_rate) { + ret = -EINVAL; + goto rpm_out; + } + /* + * Set 50% duty-cycle I2C speed to I3C OPEN-DRAIN mode, so the first + * broadcast address is visible to all I2C/I3C devices on the I3C bus. + * I3C device working as a I2C device will turn off its 50ns Spike + * Filter to change to I3C mode. + */ + mconfig = master->mctrl_config; + ppbaud = FIELD_GET(GENMASK(11, 8), mconfig); + odhpp = 0; + odbaud = DIV_ROUND_UP(fclk_rate, bus->scl_rate.i2c * (2 + 2 * ppbaud)) - 1; + mconfig &= ~GENMASK(24, 16); + mconfig |= SVC_I3C_MCONFIG_ODBAUD(odbaud) | SVC_I3C_MCONFIG_ODHPP(odhpp); + writel(mconfig, master->regs + SVC_I3C_MCONFIG); + break; + case I3C_OPEN_DRAIN_NORMAL_SPEED: + writel(master->mctrl_config, master->regs + SVC_I3C_MCONFIG); + break; + } + +rpm_out: + pm_runtime_put_autosuspend(master->dev); + + return ret; +} + +static int svc_i3c_master_bus_init(struct i3c_master_controller *m) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct i3c_bus *bus = i3c_master_get_bus(m); + struct i3c_device_info info = {}; + unsigned long fclk_rate, fclk_period_ns; + unsigned long i2c_period_ns, i2c_scl_rate, i3c_scl_rate; + unsigned int high_period_ns, od_low_period_ns; + u32 ppbaud, pplow, odhpp, odbaud, odstop, i2cbaud, reg; + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, + "<%s> cannot resume i3c bus master, err: %d\n", + __func__, ret); + return ret; + } + + /* Timings derivation */ + fclk_rate = clk_get_rate(master->fclk); + if (!fclk_rate) { + ret = -EINVAL; + goto rpm_out; + } + + fclk_period_ns = DIV_ROUND_UP(1000000000, fclk_rate); + i2c_period_ns = DIV_ROUND_UP(1000000000, bus->scl_rate.i2c); + i2c_scl_rate = bus->scl_rate.i2c; + i3c_scl_rate = bus->scl_rate.i3c; + + /* + * Using I3C Push-Pull mode, target is 12.5MHz/80ns period. + * Simplest configuration is using a 50% duty-cycle of 40ns. + */ + ppbaud = DIV_ROUND_UP(fclk_rate / 2, i3c_scl_rate) - 1; + pplow = 0; + + /* + * Using I3C Open-Drain mode, target is 4.17MHz/240ns with a + * duty-cycle tuned so that high levels are filtered out by + * the 50ns filter (target being 40ns). + */ + odhpp = 1; + high_period_ns = (ppbaud + 1) * fclk_period_ns; + odbaud = DIV_ROUND_UP(fclk_rate, SVC_I3C_QUICK_I2C_CLK * (1 + ppbaud)) - 2; + od_low_period_ns = (odbaud + 1) * high_period_ns; + + switch (bus->mode) { + case I3C_BUS_MODE_PURE: + i2cbaud = 0; + odstop = 0; + break; + case I3C_BUS_MODE_MIXED_FAST: + /* + * Using I2C Fm+ mode, target is 1MHz/1000ns, the difference + * between the high and low period does not really matter. + */ + i2cbaud = DIV_ROUND_UP(i2c_period_ns, od_low_period_ns) - 2; + odstop = 1; + break; + case I3C_BUS_MODE_MIXED_LIMITED: + case I3C_BUS_MODE_MIXED_SLOW: + /* I3C PP + I3C OP + I2C OP both use i2c clk rate */ + if (ppbaud > SVC_I3C_PPBAUD_MAX) { + ppbaud = SVC_I3C_PPBAUD_MAX; + pplow = DIV_ROUND_UP(fclk_rate, i3c_scl_rate) - (2 + 2 * ppbaud); + } + + high_period_ns = (ppbaud + 1) * fclk_period_ns; + odhpp = 0; + odbaud = DIV_ROUND_UP(fclk_rate, i2c_scl_rate * (2 + 2 * ppbaud)) - 1; + + od_low_period_ns = (odbaud + 1) * high_period_ns; + i2cbaud = DIV_ROUND_UP(i2c_period_ns, od_low_period_ns) - 2; + odstop = 1; + break; + default: + goto rpm_out; + } + + reg = SVC_I3C_MCONFIG_MASTER_EN | + SVC_I3C_MCONFIG_DISTO(0) | + SVC_I3C_MCONFIG_HKEEP(0) | + SVC_I3C_MCONFIG_ODSTOP(odstop) | + SVC_I3C_MCONFIG_PPBAUD(ppbaud) | + SVC_I3C_MCONFIG_PPLOW(pplow) | + SVC_I3C_MCONFIG_ODBAUD(odbaud) | + SVC_I3C_MCONFIG_ODHPP(odhpp) | + SVC_I3C_MCONFIG_SKEW(0) | + SVC_I3C_MCONFIG_I2CBAUD(i2cbaud); + writel(reg, master->regs + SVC_I3C_MCONFIG); + + master->mctrl_config = reg; + /* Master core's registration */ + ret = i3c_master_get_free_addr(m, 0); + if (ret < 0) + goto rpm_out; + + info.dyn_addr = ret; + + info.hdr_cap = I3C_CCC_HDR_MODE(I3C_HDR_DDR); + + writel(SVC_MDYNADDR_VALID | SVC_MDYNADDR_ADDR(info.dyn_addr), + master->regs + SVC_I3C_MDYNADDR); + + ret = i3c_master_set_info(&master->base, &info); + if (ret) + goto rpm_out; + +rpm_out: + pm_runtime_put_autosuspend(master->dev); + + return ret; +} + +static void svc_i3c_master_bus_cleanup(struct i3c_master_controller *m) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__); + return; + } + + svc_i3c_master_disable_interrupts(master); + + /* Disable master */ + writel(0, master->regs + SVC_I3C_MCONFIG); + + pm_runtime_put_autosuspend(master->dev); +} + +static int svc_i3c_master_reserve_slot(struct svc_i3c_master *master) +{ + unsigned int slot; + + if (!(master->free_slots & GENMASK(SVC_I3C_MAX_DEVS - 1, 0))) + return -ENOSPC; + + slot = ffs(master->free_slots) - 1; + + master->free_slots &= ~BIT(slot); + + return slot; +} + +static void svc_i3c_master_release_slot(struct svc_i3c_master *master, + unsigned int slot) +{ + master->free_slots |= BIT(slot); +} + +static int svc_i3c_master_attach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data; + int slot; + + slot = svc_i3c_master_reserve_slot(master); + if (slot < 0) + return slot; + + data = kzalloc_obj(*data); + if (!data) { + svc_i3c_master_release_slot(master, slot); + return -ENOMEM; + } + + data->ibi = -1; + data->index = slot; + master->addrs[slot] = dev->info.dyn_addr ? dev->info.dyn_addr : + dev->info.static_addr; + master->descs[slot] = dev; + + i3c_dev_set_master_data(dev, data); + + return 0; +} + +static int svc_i3c_master_reattach_i3c_dev(struct i3c_dev_desc *dev, + u8 old_dyn_addr) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + master->addrs[data->index] = dev->info.dyn_addr ? dev->info.dyn_addr : + dev->info.static_addr; + + return 0; +} + +static void svc_i3c_master_detach_i3c_dev(struct i3c_dev_desc *dev) +{ + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + + master->addrs[data->index] = 0; + svc_i3c_master_release_slot(master, data->index); + + kfree(data); +} + +static int svc_i3c_master_attach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data; + int slot; + + slot = svc_i3c_master_reserve_slot(master); + if (slot < 0) + return slot; + + data = kzalloc_obj(*data); + if (!data) { + svc_i3c_master_release_slot(master, slot); + return -ENOMEM; + } + + data->index = slot; + master->addrs[slot] = dev->addr; + + i2c_dev_set_master_data(dev, data); + + return 0; +} + +static void svc_i3c_master_detach_i2c_dev(struct i2c_dev_desc *dev) +{ + struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + + svc_i3c_master_release_slot(master, data->index); + + kfree(data); +} + +static int svc_i3c_master_readb(struct svc_i3c_master *master, u8 *dst, + unsigned int len) +{ + int ret, i; + u32 reg; + + for (i = 0; i < len; i++) { + ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, + reg, + SVC_I3C_MSTATUS_RXPEND(reg), + 0, 1000); + if (ret) + return ret; + + dst[i] = readl(master->regs + SVC_I3C_MRDATAB); + } + + return 0; +} + +static int svc_i3c_master_do_daa_locked(struct svc_i3c_master *master, + u8 *addrs, unsigned int *count) +{ + u64 prov_id[SVC_I3C_MAX_DEVS] = {}, nacking_prov_id = 0; + unsigned int dev_nb = 0, last_addr = 0, dyn_addr = 0; + u32 reg; + int ret, i; + + svc_i3c_master_flush_fifo(master); + + while (true) { + /* clean SVC_I3C_MINT_IBIWON w1c bits */ + writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); + + /* SVC_I3C_MCTRL_REQUEST_PROC_DAA have two mode, ENTER DAA or PROCESS DAA. + * + * ENTER DAA: + * 1 will issue START, 7E, ENTDAA, and then emits 7E/R to process first target. + * 2 Stops just before the new Dynamic Address (DA) is to be emitted. + * + * PROCESS DAA: + * 1 The DA is written using MWDATAB or ADDR bits 6:0. + * 2 ProcessDAA is requested again to write the new address, and then starts the + * next (START, 7E, ENTDAA) unless marked to STOP; an MSTATUS indicating NACK + * means DA was not accepted (e.g. parity error). If PROCESSDAA is NACKed on the + * 7E/R, which means no more Slaves need a DA, then a COMPLETE will be signaled + * (along with DONE), and a STOP issued automatically. + */ + writel(SVC_I3C_MCTRL_REQUEST_PROC_DAA | + SVC_I3C_MCTRL_TYPE_I3C | + SVC_I3C_MCTRL_IBIRESP_NACK | + SVC_I3C_MCTRL_DIR(SVC_I3C_MCTRL_DIR_WRITE), + master->regs + SVC_I3C_MCTRL); + + /* + * Either one slave will send its ID, or the assignment process + * is done. + */ + ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, + reg, + SVC_I3C_MSTATUS_RXPEND(reg) | + SVC_I3C_MSTATUS_MCTRLDONE(reg), + 1, 1000); + if (ret) + break; + + if (SVC_I3C_MSTATUS_RXPEND(reg)) { + u8 data[6]; + + /* + * One slave sends its ID to request for address assignment, + * prefilling the dynamic address can reduce SCL clock stalls + * and also fix the SVC_I3C_QUIRK_FIFO_EMPTY quirk. + * + * Ideally, prefilling before the processDAA command is better. + * However, it requires an additional check to write the dyn_addr + * at the right time because the driver needs to write the processDAA + * command twice for one assignment. + * Prefilling here is safe and efficient because the FIFO starts + * filling within a few hundred nanoseconds, which is significantly + * faster compared to the 64 SCL clock cycles. + */ + ret = i3c_master_get_free_addr(&master->base, last_addr + 1); + if (ret < 0) + break; + + dyn_addr = ret; + writel(dyn_addr, master->regs + SVC_I3C_MWDATAB); + + /* + * We only care about the 48-bit provisioned ID yet to + * be sure a device does not nack an address twice. + * Otherwise, we would just need to flush the RX FIFO. + */ + ret = svc_i3c_master_readb(master, data, 6); + if (ret) + break; + + for (i = 0; i < 6; i++) + prov_id[dev_nb] |= (u64)(data[i]) << (8 * (5 - i)); + + /* We do not care about the BCR and DCR yet */ + ret = svc_i3c_master_readb(master, data, 2); + if (ret) + break; + } else if (SVC_I3C_MSTATUS_IBIWON(reg)) { + ret = svc_i3c_master_handle_ibi_won(master, reg); + if (ret) + break; + continue; + } else if (SVC_I3C_MSTATUS_MCTRLDONE(reg)) { + if (SVC_I3C_MSTATUS_STATE_IDLE(reg) && + SVC_I3C_MSTATUS_COMPLETE(reg)) { + /* + * All devices received and acked they dynamic + * address, this is the natural end of the DAA + * procedure. + * + * Hardware will auto emit STOP at this case. + */ + *count = dev_nb; + return 0; + + } else if (SVC_I3C_MSTATUS_NACKED(reg)) { + /* No I3C devices attached */ + if (dev_nb == 0) { + /* + * Hardware can't treat first NACK for ENTAA as normal + * COMPLETE. So need manual emit STOP. + */ + ret = 0; + *count = 0; + break; + } + + /* + * A slave device nacked the address, this is + * allowed only once, DAA will be stopped and + * then resumed. The same device is supposed to + * answer again immediately and shall ack the + * address this time. + */ + if (prov_id[dev_nb] == nacking_prov_id) { + ret = -EIO; + break; + } + + dev_nb--; + nacking_prov_id = prov_id[dev_nb]; + svc_i3c_master_emit_stop(master); + + continue; + } else { + break; + } + } + + /* Wait for the slave to be ready to receive its address */ + ret = readl_poll_timeout_atomic(master->regs + SVC_I3C_MSTATUS, + reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg) && + SVC_I3C_MSTATUS_STATE_DAA(reg) && + SVC_I3C_MSTATUS_BETWEEN(reg), + 0, 1000); + if (ret) + break; + + addrs[dev_nb] = dyn_addr; + dev_dbg(master->dev, "DAA: device %d assigned to 0x%02x\n", + dev_nb, addrs[dev_nb]); + last_addr = addrs[dev_nb++]; + } + + /* Need manual issue STOP except for Complete condition */ + svc_i3c_master_emit_stop(master); + svc_i3c_master_flush_fifo(master); + + return ret; +} + +static int svc_i3c_update_ibirules(struct svc_i3c_master *master) +{ + struct i3c_dev_desc *dev; + u32 reg_mbyte = 0, reg_nobyte = SVC_I3C_IBIRULES_NOBYTE; + unsigned int mbyte_addr_ok = 0, mbyte_addr_ko = 0, nobyte_addr_ok = 0, + nobyte_addr_ko = 0; + bool list_mbyte = false, list_nobyte = false; + + /* Create the IBIRULES register for both cases */ + i3c_bus_for_each_i3cdev(&master->base.bus, dev) { + if (!(dev->info.bcr & I3C_BCR_IBI_REQ_CAP)) + continue; + + if (dev->info.bcr & I3C_BCR_IBI_PAYLOAD) { + reg_mbyte |= SVC_I3C_IBIRULES_ADDR(mbyte_addr_ok, + dev->info.dyn_addr); + + /* IBI rules cannot be applied to devices with MSb=1 */ + if (dev->info.dyn_addr & BIT(7)) + mbyte_addr_ko++; + else + mbyte_addr_ok++; + } else { + reg_nobyte |= SVC_I3C_IBIRULES_ADDR(nobyte_addr_ok, + dev->info.dyn_addr); + + /* IBI rules cannot be applied to devices with MSb=1 */ + if (dev->info.dyn_addr & BIT(7)) + nobyte_addr_ko++; + else + nobyte_addr_ok++; + } + } + + /* Device list cannot be handled by hardware */ + if (!mbyte_addr_ko && mbyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS) + list_mbyte = true; + + if (!nobyte_addr_ko && nobyte_addr_ok <= SVC_I3C_IBIRULES_ADDRS) + list_nobyte = true; + + /* No list can be properly handled, return an error */ + if (!list_mbyte && !list_nobyte) + return -ERANGE; + + /* Pick the first list that can be handled by hardware, randomly */ + if (list_mbyte) + writel(reg_mbyte, master->regs + SVC_I3C_IBIRULES); + else + writel(reg_nobyte, master->regs + SVC_I3C_IBIRULES); + + return 0; +} + +static int svc_i3c_master_do_daa(struct i3c_master_controller *m) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + u8 addrs[SVC_I3C_MAX_DEVS]; + unsigned long flags; + unsigned int dev_nb; + int ret, i; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__); + return ret; + } + + spin_lock_irqsave(&master->xferqueue.lock, flags); + + if (svc_has_daa_corrupt(master)) + writel(master->mctrl_config | SVC_I3C_MCONFIG_SKEW(1), + master->regs + SVC_I3C_MCONFIG); + + ret = svc_i3c_master_do_daa_locked(master, addrs, &dev_nb); + + if (svc_has_daa_corrupt(master)) + writel(master->mctrl_config, master->regs + SVC_I3C_MCONFIG); + + spin_unlock_irqrestore(&master->xferqueue.lock, flags); + + svc_i3c_master_clear_merrwarn(master); + if (ret) + goto rpm_out; + + /* + * Register all devices who participated to the core + * + * If two devices (A and B) are detected in DAA and address 0xa is assigned to + * device A and 0xb to device B, a failure in i3c_master_add_i3c_dev_locked() + * for device A (addr: 0xa) could prevent device B (addr: 0xb) from being + * registered on the bus. The I3C stack might still consider 0xb a free + * address. If a subsequent Hotjoin occurs, 0xb might be assigned to Device A, + * causing both devices A and B to use the same address 0xb, violating the I3C + * specification. + * + * The return value for i3c_master_add_i3c_dev_locked() should not be checked + * because subsequent steps will scan the entire I3C bus, independent of + * whether i3c_master_add_i3c_dev_locked() returns success. + * + * If device A registration fails, there is still a chance to register device + * B. i3c_master_add_i3c_dev_locked() can reset DAA if a failure occurs while + * retrieving device information. + */ + for (i = 0; i < dev_nb; i++) + i3c_master_add_i3c_dev_locked(m, addrs[i]); + + /* Configure IBI auto-rules */ + ret = svc_i3c_update_ibirules(master); + if (ret) + dev_err(master->dev, "Cannot handle such a list of devices\n"); + +rpm_out: + pm_runtime_put_autosuspend(master->dev); + + return ret; +} + +static int svc_i3c_master_read(struct svc_i3c_master *master, + u8 *in, unsigned int len) +{ + int offset = 0, i; + u32 mdctrl, mstatus; + bool completed = false; + unsigned int count; + unsigned long start = jiffies; + + while (!completed) { + mstatus = readl(master->regs + SVC_I3C_MSTATUS); + if (SVC_I3C_MSTATUS_COMPLETE(mstatus) != 0) + completed = true; + + if (time_after(jiffies, start + msecs_to_jiffies(1000))) { + dev_dbg(master->dev, "I3C read timeout\n"); + return -ETIMEDOUT; + } + + mdctrl = readl(master->regs + SVC_I3C_MDATACTRL); + count = SVC_I3C_MDATACTRL_RXCOUNT(mdctrl); + if (offset + count > len) { + dev_err(master->dev, "I3C receive length too long!\n"); + return -EINVAL; + } + for (i = 0; i < count; i++) + in[offset + i] = readl(master->regs + SVC_I3C_MRDATAB); + + offset += count; + } + + return offset; +} + +static int svc_i3c_master_write(struct svc_i3c_master *master, + const u8 *out, unsigned int len) +{ + int offset = 0, ret; + u32 mdctrl; + + while (offset < len) { + ret = readl_poll_timeout(master->regs + SVC_I3C_MDATACTRL, + mdctrl, + !(mdctrl & SVC_I3C_MDATACTRL_TXFULL), + 0, 1000); + if (ret) + return ret; + + /* + * The last byte to be sent over the bus must either have the + * "end" bit set or be written in MWDATABE. + */ + if (likely(offset < (len - 1))) + writel(out[offset++], master->regs + SVC_I3C_MWDATAB); + else + writel(out[offset++], master->regs + SVC_I3C_MWDATABE); + } + + return 0; +} + +static int svc_i3c_master_xfer(struct svc_i3c_master *master, + u32 rnw_cmd, unsigned int xfer_type, u8 addr, + u8 *in, const u8 *out, unsigned int xfer_len, + unsigned int *actual_len, bool continued, bool repeat_start) +{ + bool rnw = svc_cmd_is_read(rnw_cmd, xfer_type); + int retry = repeat_start ? 1 : 2; + u32 reg; + int ret; + + /* clean SVC_I3C_MINT_IBIWON w1c bits */ + writel(SVC_I3C_MINT_IBIWON, master->regs + SVC_I3C_MSTATUS); + + if (xfer_type == SVC_I3C_MCTRL_TYPE_DDR) { + /* DDR command need prefill into FIFO */ + writel(rnw_cmd, master->regs + SVC_I3C_MWDATAB); + if (!rnw) { + /* write data also need prefill into FIFO */ + ret = svc_i3c_master_write(master, out, xfer_len); + if (ret) + goto emit_stop; + } + } + + while (retry--) { + writel(SVC_I3C_MCTRL_REQUEST_START_ADDR | + xfer_type | + SVC_I3C_MCTRL_IBIRESP_NACK | + SVC_I3C_MCTRL_DIR(rnw) | + SVC_I3C_MCTRL_ADDR(addr) | + SVC_I3C_MCTRL_RDTERM(*actual_len), + master->regs + SVC_I3C_MCTRL); + + /* + * The entire transaction can consist of multiple write transfers. + * Prefilling before EmitStartAddr causes the data to be emitted + * immediately, becoming part of the previous transfer. + * The only way to work around this hardware issue is to let the + * FIFO start filling as soon as possible after EmitStartAddr. + */ + if (svc_has_quirk(master, SVC_I3C_QUIRK_FIFO_EMPTY) && !rnw && xfer_len) { + u32 space, end, len; + + reg = readl(master->regs + SVC_I3C_MDATACTRL); + space = SVC_I3C_FIFO_SIZE - SVC_I3C_MDATACTRL_TXCOUNT(reg); + if (space) { + end = xfer_len > space ? 0 : SVC_I3C_MWDATAB_END; + len = min_t(u32, xfer_len, space); + writesb(master->regs + SVC_I3C_MWDATAB1, out, len - 1); + /* Mark END bit if this is the last byte */ + writel(out[len - 1] | end, master->regs + SVC_I3C_MWDATAB); + xfer_len -= len; + out += len; + } + } + + ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_MCTRLDONE(reg), 0, 1000); + if (ret) + goto emit_stop; + + /* + * According to I3C spec ver 1.1.1, 5.1.2.2.3 Consequence of Controller Starting a + * Frame with I3C Target Address. + * + * The I3C Controller normally should start a Frame, the Address may be arbitrated, + * and so the Controller shall monitor to see whether an In-Band Interrupt request, + * a Controller Role Request (i.e., Secondary Controller requests to become the + * Active Controller), or a Hot-Join Request has been made. + * + * If missed IBIWON check, the wrong data will be return. When IBIWON happen, issue + * repeat start. Address arbitrate only happen at START, never happen at REPEAT + * start. + */ + if (SVC_I3C_MSTATUS_IBIWON(reg)) { + ret = svc_i3c_master_handle_ibi_won(master, reg); + if (ret) + goto emit_stop; + continue; + } + + if (readl(master->regs + SVC_I3C_MERRWARN) & SVC_I3C_MERRWARN_NACK) { + /* + * According to I3C Spec 1.1.1, 11-Jun-2021, section: 5.1.2.2.3. + * If the Controller chooses to start an I3C Message with an I3C Dynamic + * Address, then special provisions shall be made because that same I3C + * Target may be initiating an IBI or a Controller Role Request. So, one of + * three things may happen: (skip 1, 2) + * + * 3. The Addresses match and the RnW bits also match, and so neither + * Controller nor Target will ACK since both are expecting the other side to + * provide ACK. As a result, each side might think it had "won" arbitration, + * but neither side would continue, as each would subsequently see that the + * other did not provide ACK. + * ... + * For either value of RnW: Due to the NACK, the Controller shall defer the + * Private Write or Private Read, and should typically transmit the Target + * Address again after a Repeated START (i.e., the next one or any one prior + * to a STOP in the Frame). Since the Address Header following a Repeated + * START is not arbitrated, the Controller will always win (see Section + * 5.1.2.2.4). + */ + if (retry && addr != 0x7e) { + writel(SVC_I3C_MERRWARN_NACK, master->regs + SVC_I3C_MERRWARN); + } else { + ret = -ENXIO; + *actual_len = 0; + goto emit_stop; + } + } else { + break; + } + } + + if (rnw) + ret = svc_i3c_master_read(master, in, xfer_len); + else if (xfer_type != SVC_I3C_MCTRL_TYPE_DDR) + ret = svc_i3c_master_write(master, out, xfer_len); + if (ret < 0) + goto emit_stop; + + if (rnw) + *actual_len = ret; + + ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_COMPLETE(reg), 0, 1000); + if (ret) + goto emit_stop; + + if (xfer_type == SVC_I3C_MCTRL_TYPE_DDR && + (readl(master->regs + SVC_I3C_MERRWARN) & SVC_I3C_MERRWARN_CRC)) { + ret = -ENXIO; + goto emit_stop; + } + + writel(SVC_I3C_MINT_COMPLETE, master->regs + SVC_I3C_MSTATUS); + + if (!continued) { + if (xfer_type != SVC_I3C_MCTRL_TYPE_DDR) + svc_i3c_master_emit_stop(master); + else + svc_i3c_master_emit_force_exit(master); + + /* Wait idle if stop is sent. */ + ret = readl_poll_timeout(master->regs + SVC_I3C_MSTATUS, reg, + SVC_I3C_MSTATUS_STATE_IDLE(reg), + 0, 1000); + if (ret) + goto cleanup; + } + + return 0; + +emit_stop: + if (xfer_type != SVC_I3C_MCTRL_TYPE_DDR) + svc_i3c_master_emit_stop(master); + else + svc_i3c_master_emit_force_exit(master); + +cleanup: + svc_i3c_master_clear_merrwarn(master); + svc_i3c_master_flush_fifo(master); + + return ret; +} + +static struct svc_i3c_xfer * +svc_i3c_master_alloc_xfer(struct svc_i3c_master *master, unsigned int ncmds) +{ + struct svc_i3c_xfer *xfer; + + xfer = kzalloc_flex(*xfer, cmds, ncmds); + if (!xfer) + return NULL; + + INIT_LIST_HEAD(&xfer->node); + xfer->ncmds = ncmds; + xfer->ret = -ETIMEDOUT; + + return xfer; +} + +static void svc_i3c_master_free_xfer(struct svc_i3c_xfer *xfer) +{ + kfree(xfer); +} + +static void svc_i3c_master_dequeue_xfer_locked(struct svc_i3c_master *master, + struct svc_i3c_xfer *xfer) +{ + if (master->xferqueue.cur == xfer) + master->xferqueue.cur = NULL; + else + list_del_init(&xfer->node); +} + +static void svc_i3c_master_dequeue_xfer(struct svc_i3c_master *master, + struct svc_i3c_xfer *xfer) +{ + unsigned long flags; + + spin_lock_irqsave(&master->xferqueue.lock, flags); + svc_i3c_master_dequeue_xfer_locked(master, xfer); + spin_unlock_irqrestore(&master->xferqueue.lock, flags); +} + +static int i3c_mode_to_svc_type(enum i3c_xfer_mode mode) +{ + return (mode == I3C_SDR) ? SVC_I3C_MCTRL_TYPE_I3C : SVC_I3C_MCTRL_TYPE_DDR; +} + +static void svc_i3c_master_start_xfer_locked(struct svc_i3c_master *master) +{ + struct svc_i3c_xfer *xfer = master->xferqueue.cur; + int ret, i; + + if (!xfer) + return; + + svc_i3c_master_clear_merrwarn(master); + svc_i3c_master_flush_fifo(master); + + for (i = 0; i < xfer->ncmds; i++) { + struct svc_i3c_cmd *cmd = &xfer->cmds[i]; + + ret = svc_i3c_master_xfer(master, cmd->rnw_cmd, xfer->type, + cmd->addr, cmd->in, cmd->out, + cmd->len, &cmd->actual_len, + cmd->continued, i > 0); + /* cmd->xfer is NULL if I2C or CCC transfer */ + if (cmd->xfer) + cmd->xfer->actual_len = cmd->actual_len; + + if (ret) + break; + } + + xfer->ret = ret; + complete(&xfer->comp); + + if (ret < 0) + svc_i3c_master_dequeue_xfer_locked(master, xfer); + + xfer = list_first_entry_or_null(&master->xferqueue.list, + struct svc_i3c_xfer, + node); + if (xfer) + list_del_init(&xfer->node); + + master->xferqueue.cur = xfer; + svc_i3c_master_start_xfer_locked(master); +} + +static void svc_i3c_master_enqueue_xfer(struct svc_i3c_master *master, + struct svc_i3c_xfer *xfer) +{ + unsigned long flags; + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__); + return; + } + + init_completion(&xfer->comp); + spin_lock_irqsave(&master->xferqueue.lock, flags); + if (master->xferqueue.cur) { + list_add_tail(&xfer->node, &master->xferqueue.list); + } else { + master->xferqueue.cur = xfer; + svc_i3c_master_start_xfer_locked(master); + } + spin_unlock_irqrestore(&master->xferqueue.lock, flags); + + pm_runtime_put_autosuspend(master->dev); +} + +static bool +svc_i3c_master_supports_ccc_cmd(struct i3c_master_controller *master, + const struct i3c_ccc_cmd *cmd) +{ + /* No software support for CCC commands targeting more than one slave */ + return (cmd->ndests == 1); +} + +static int svc_i3c_master_send_bdcast_ccc_cmd(struct svc_i3c_master *master, + struct i3c_ccc_cmd *ccc) +{ + unsigned int xfer_len = ccc->dests[0].payload.len + 1; + struct svc_i3c_xfer *xfer; + struct svc_i3c_cmd *cmd; + u8 *buf; + int ret; + + xfer = svc_i3c_master_alloc_xfer(master, 1); + if (!xfer) + return -ENOMEM; + + buf = kmalloc(xfer_len, GFP_KERNEL); + if (!buf) { + svc_i3c_master_free_xfer(xfer); + return -ENOMEM; + } + + buf[0] = ccc->id; + memcpy(&buf[1], ccc->dests[0].payload.data, ccc->dests[0].payload.len); + + xfer->type = SVC_I3C_MCTRL_TYPE_I3C; + + cmd = &xfer->cmds[0]; + cmd->addr = ccc->dests[0].addr; + cmd->rnw = ccc->rnw; + cmd->in = NULL; + cmd->out = buf; + cmd->len = xfer_len; + cmd->actual_len = 0; + cmd->continued = false; + + mutex_lock(&master->lock); + svc_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000))) + svc_i3c_master_dequeue_xfer(master, xfer); + mutex_unlock(&master->lock); + + ret = xfer->ret; + kfree(buf); + svc_i3c_master_free_xfer(xfer); + + return ret; +} + +static int svc_i3c_master_send_direct_ccc_cmd(struct svc_i3c_master *master, + struct i3c_ccc_cmd *ccc) +{ + unsigned int xfer_len = ccc->dests[0].payload.len; + unsigned int actual_len = ccc->rnw ? xfer_len : 0; + struct svc_i3c_xfer *xfer; + struct svc_i3c_cmd *cmd; + int ret; + + xfer = svc_i3c_master_alloc_xfer(master, 2); + if (!xfer) + return -ENOMEM; + + xfer->type = SVC_I3C_MCTRL_TYPE_I3C; + + /* Broadcasted message */ + cmd = &xfer->cmds[0]; + cmd->addr = I3C_BROADCAST_ADDR; + cmd->rnw = 0; + cmd->in = NULL; + cmd->out = &ccc->id; + cmd->len = 1; + cmd->actual_len = 0; + cmd->continued = true; + + /* Directed message */ + cmd = &xfer->cmds[1]; + cmd->addr = ccc->dests[0].addr; + cmd->rnw = ccc->rnw; + cmd->in = ccc->rnw ? ccc->dests[0].payload.data : NULL; + cmd->out = ccc->rnw ? NULL : ccc->dests[0].payload.data; + cmd->len = xfer_len; + cmd->actual_len = actual_len; + cmd->continued = false; + + mutex_lock(&master->lock); + svc_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000))) + svc_i3c_master_dequeue_xfer(master, xfer); + mutex_unlock(&master->lock); + + if (ccc->rnw) + ccc->dests[0].payload.actual_len = cmd->actual_len; + + ret = xfer->ret; + svc_i3c_master_free_xfer(xfer); + + return ret; +} + +static int svc_i3c_master_send_ccc_cmd(struct i3c_master_controller *m, + struct i3c_ccc_cmd *cmd) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + bool broadcast = cmd->id < 0x80; + int ret; + + if (broadcast) + ret = svc_i3c_master_send_bdcast_ccc_cmd(master, cmd); + else + ret = svc_i3c_master_send_direct_ccc_cmd(master, cmd); + + if (ret) + cmd->err = I3C_ERROR_M2; + + return ret; +} + +static int svc_i3c_master_i3c_xfers(struct i3c_dev_desc *dev, struct i3c_xfer *xfers, + int nxfers, enum i3c_xfer_mode mode) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + struct svc_i3c_xfer *xfer; + int ret, i; + + if (mode != I3C_SDR) { + /* + * Only support data size less than FIFO SIZE when using DDR + * mode. First entry is cmd in FIFO, so actual available FIFO + * for data is SVC_I3C_FIFO_SIZE - 2 since DDR only supports + * even length. + */ + for (i = 0; i < nxfers; i++) + if (xfers[i].len > SVC_I3C_FIFO_SIZE - 2) + return -EINVAL; + } + + xfer = svc_i3c_master_alloc_xfer(master, nxfers); + if (!xfer) + return -ENOMEM; + + xfer->type = i3c_mode_to_svc_type(mode); + + for (i = 0; i < nxfers; i++) { + u32 rnw_cmd = (mode == I3C_SDR) ? xfers[i].rnw : xfers[i].cmd; + bool rnw = svc_cmd_is_read(rnw_cmd, xfer->type); + struct svc_i3c_cmd *cmd = &xfer->cmds[i]; + + cmd->xfer = &xfers[i]; + cmd->addr = master->addrs[data->index]; + cmd->rnw_cmd = rnw_cmd; + cmd->in = rnw ? xfers[i].data.in : NULL; + cmd->out = rnw ? NULL : xfers[i].data.out; + cmd->len = xfers[i].len; + cmd->actual_len = rnw ? xfers[i].len : 0; + cmd->continued = (i + 1) < nxfers; + } + + mutex_lock(&master->lock); + svc_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, msecs_to_jiffies(1000))) + svc_i3c_master_dequeue_xfer(master, xfer); + mutex_unlock(&master->lock); + + ret = xfer->ret; + svc_i3c_master_free_xfer(xfer); + + return ret; +} + +static int svc_i3c_master_i2c_xfers(struct i2c_dev_desc *dev, + struct i2c_msg *xfers, + int nxfers) +{ + struct i3c_master_controller *m = i2c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data = i2c_dev_get_master_data(dev); + struct svc_i3c_xfer *xfer; + int ret, i; + + xfer = svc_i3c_master_alloc_xfer(master, nxfers); + if (!xfer) + return -ENOMEM; + + xfer->type = SVC_I3C_MCTRL_TYPE_I2C; + + for (i = 0; i < nxfers; i++) { + struct svc_i3c_cmd *cmd = &xfer->cmds[i]; + + cmd->addr = master->addrs[data->index]; + cmd->rnw = xfers[i].flags & I2C_M_RD; + cmd->in = cmd->rnw ? xfers[i].buf : NULL; + cmd->out = cmd->rnw ? NULL : xfers[i].buf; + cmd->len = xfers[i].len; + cmd->actual_len = cmd->rnw ? xfers[i].len : 0; + cmd->continued = (i + 1 < nxfers); + } + + mutex_lock(&master->lock); + svc_i3c_master_enqueue_xfer(master, xfer); + if (!wait_for_completion_timeout(&xfer->comp, m->i2c.timeout)) + svc_i3c_master_dequeue_xfer(master, xfer); + mutex_unlock(&master->lock); + + ret = xfer->ret; + svc_i3c_master_free_xfer(xfer); + + return ret; +} + +static int svc_i3c_master_request_ibi(struct i3c_dev_desc *dev, + const struct i3c_ibi_setup *req) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + unsigned int i; + + if (dev->ibi->max_payload_len > SVC_I3C_FIFO_SIZE) { + dev_err(master->dev, "IBI max payload %d should be < %d\n", + dev->ibi->max_payload_len, SVC_I3C_FIFO_SIZE); + return -ERANGE; + } + + data->ibi_pool = i3c_generic_ibi_alloc_pool(dev, req); + if (IS_ERR(data->ibi_pool)) + return PTR_ERR(data->ibi_pool); + + spin_lock_irqsave(&master->ibi.lock, flags); + for (i = 0; i < master->ibi.num_slots; i++) { + if (!master->ibi.slots[i]) { + data->ibi = i; + master->ibi.slots[i] = dev; + break; + } + } + spin_unlock_irqrestore(&master->ibi.lock, flags); + + if (i < master->ibi.num_slots) + return 0; + + i3c_generic_ibi_free_pool(data->ibi_pool); + data->ibi_pool = NULL; + + return -ENOSPC; +} + +static void svc_i3c_master_free_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + unsigned long flags; + + spin_lock_irqsave(&master->ibi.lock, flags); + master->ibi.slots[data->ibi] = NULL; + data->ibi = -1; + spin_unlock_irqrestore(&master->ibi.lock, flags); + + i3c_generic_ibi_free_pool(data->ibi_pool); +} + +static int svc_i3c_master_enable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__); + return ret; + } + + master->enabled_events++; + svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART); + + return i3c_master_enec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); +} + +static int svc_i3c_master_disable_ibi(struct i3c_dev_desc *dev) +{ + struct i3c_master_controller *m = i3c_dev_get_master(dev); + struct svc_i3c_master *master = to_svc_i3c_master(m); + int ret; + + master->enabled_events--; + if (!master->enabled_events) + svc_i3c_master_disable_interrupts(master); + + ret = i3c_master_disec_locked(m, dev->info.dyn_addr, I3C_CCC_EVENT_SIR); + + pm_runtime_put_autosuspend(master->dev); + + return ret; +} + +static int svc_i3c_master_enable_hotjoin(struct i3c_master_controller *m) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + int ret; + + ret = pm_runtime_resume_and_get(master->dev); + if (ret < 0) { + dev_err(master->dev, "<%s> Cannot get runtime PM.\n", __func__); + return ret; + } + + master->enabled_events |= SVC_I3C_EVENT_HOTJOIN; + + svc_i3c_master_enable_interrupts(master, SVC_I3C_MINT_SLVSTART); + + return 0; +} + +static int svc_i3c_master_disable_hotjoin(struct i3c_master_controller *m) +{ + struct svc_i3c_master *master = to_svc_i3c_master(m); + + master->enabled_events &= ~SVC_I3C_EVENT_HOTJOIN; + + if (!master->enabled_events) + svc_i3c_master_disable_interrupts(master); + + pm_runtime_put_autosuspend(master->dev); + + return 0; +} + +static void svc_i3c_master_recycle_ibi_slot(struct i3c_dev_desc *dev, + struct i3c_ibi_slot *slot) +{ + struct svc_i3c_i2c_dev_data *data = i3c_dev_get_master_data(dev); + + i3c_generic_ibi_recycle_slot(data->ibi_pool, slot); +} + +static const struct i3c_master_controller_ops svc_i3c_master_ops = { + .bus_init = svc_i3c_master_bus_init, + .bus_cleanup = svc_i3c_master_bus_cleanup, + .attach_i3c_dev = svc_i3c_master_attach_i3c_dev, + .detach_i3c_dev = svc_i3c_master_detach_i3c_dev, + .reattach_i3c_dev = svc_i3c_master_reattach_i3c_dev, + .attach_i2c_dev = svc_i3c_master_attach_i2c_dev, + .detach_i2c_dev = svc_i3c_master_detach_i2c_dev, + .do_daa = svc_i3c_master_do_daa, + .supports_ccc_cmd = svc_i3c_master_supports_ccc_cmd, + .send_ccc_cmd = svc_i3c_master_send_ccc_cmd, + .i3c_xfers = svc_i3c_master_i3c_xfers, + .i2c_xfers = svc_i3c_master_i2c_xfers, + .request_ibi = svc_i3c_master_request_ibi, + .free_ibi = svc_i3c_master_free_ibi, + .recycle_ibi_slot = svc_i3c_master_recycle_ibi_slot, + .enable_ibi = svc_i3c_master_enable_ibi, + .disable_ibi = svc_i3c_master_disable_ibi, + .enable_hotjoin = svc_i3c_master_enable_hotjoin, + .disable_hotjoin = svc_i3c_master_disable_hotjoin, + .set_speed = svc_i3c_master_set_speed, +}; + +static int svc_i3c_master_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct svc_i3c_master *master; + int ret, i; + + master = devm_kzalloc(dev, sizeof(*master), GFP_KERNEL); + if (!master) + return -ENOMEM; + + master->drvdata = of_device_get_match_data(dev); + if (!master->drvdata) + return -EINVAL; + + master->regs = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(master->regs)) + return PTR_ERR(master->regs); + + master->num_clks = devm_clk_bulk_get_all(dev, &master->clks); + if (master->num_clks < 0) + return dev_err_probe(dev, -EINVAL, "can't get I3C clocks\n"); + + for (i = 0; i < master->num_clks; i++) { + if (!strcmp(master->clks[i].id, "fast_clk")) + break; + } + + if (i == master->num_clks) + return dev_err_probe(dev, -EINVAL, + "can't get I3C peripheral clock\n"); + + master->fclk = master->clks[i].clk; + if (IS_ERR(master->fclk)) + return PTR_ERR(master->fclk); + + master->irq = platform_get_irq(pdev, 0); + if (master->irq < 0) + return master->irq; + + master->dev = dev; + ret = clk_bulk_prepare_enable(master->num_clks, master->clks); + if (ret) + return dev_err_probe(dev, ret, "can't enable I3C clocks\n"); + + mutex_init(&master->lock); + + ret = devm_request_irq(dev, master->irq, svc_i3c_master_irq_handler, + IRQF_NO_SUSPEND, "svc-i3c-irq", master); + if (ret) + goto err_disable_clks; + + master->free_slots = GENMASK(SVC_I3C_MAX_DEVS - 1, 0); + + spin_lock_init(&master->xferqueue.lock); + INIT_LIST_HEAD(&master->xferqueue.list); + + spin_lock_init(&master->ibi.lock); + master->ibi.num_slots = SVC_I3C_MAX_DEVS; + master->ibi.slots = devm_kcalloc(&pdev->dev, master->ibi.num_slots, + sizeof(*master->ibi.slots), + GFP_KERNEL); + if (!master->ibi.slots) { + ret = -ENOMEM; + goto err_disable_clks; + } + + platform_set_drvdata(pdev, master); + + pm_runtime_set_autosuspend_delay(&pdev->dev, SVC_I3C_PM_TIMEOUT_MS); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_get_noresume(&pdev->dev); + pm_runtime_set_active(&pdev->dev); + pm_runtime_enable(&pdev->dev); + + svc_i3c_master_reset(master); + + /* Register the master */ + ret = i3c_master_register(&master->base, &pdev->dev, + &svc_i3c_master_ops, false); + if (ret) + goto rpm_disable; + + pm_runtime_put_autosuspend(&pdev->dev); + + return 0; + +rpm_disable: + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_put_noidle(&pdev->dev); + pm_runtime_disable(&pdev->dev); + pm_runtime_set_suspended(&pdev->dev); + +err_disable_clks: + clk_bulk_disable_unprepare(master->num_clks, master->clks); + + return ret; +} + +static void svc_i3c_master_remove(struct platform_device *pdev) +{ + struct svc_i3c_master *master = platform_get_drvdata(pdev); + + i3c_master_unregister(&master->base); + + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_disable(&pdev->dev); +} + +static void svc_i3c_save_regs(struct svc_i3c_master *master) +{ + master->saved_regs.mconfig = readl(master->regs + SVC_I3C_MCONFIG); + master->saved_regs.mdynaddr = readl(master->regs + SVC_I3C_MDYNADDR); +} + +static void svc_i3c_restore_regs(struct svc_i3c_master *master) +{ + if (readl(master->regs + SVC_I3C_MDYNADDR) != + master->saved_regs.mdynaddr) { + writel(master->saved_regs.mconfig, + master->regs + SVC_I3C_MCONFIG); + writel(master->saved_regs.mdynaddr, + master->regs + SVC_I3C_MDYNADDR); + } +} + +static int __maybe_unused svc_i3c_runtime_suspend(struct device *dev) +{ + struct svc_i3c_master *master = dev_get_drvdata(dev); + + svc_i3c_save_regs(master); + clk_bulk_disable_unprepare(master->num_clks, master->clks); + pinctrl_pm_select_sleep_state(dev); + + return 0; +} + +static int __maybe_unused svc_i3c_runtime_resume(struct device *dev) +{ + struct svc_i3c_master *master = dev_get_drvdata(dev); + int ret; + + pinctrl_pm_select_default_state(dev); + ret = clk_bulk_prepare_enable(master->num_clks, master->clks); + if (ret) + return ret; + + svc_i3c_restore_regs(master); + + return 0; +} + +static const struct dev_pm_ops svc_i3c_pm_ops = { + SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(pm_runtime_force_suspend, + pm_runtime_force_resume) + SET_RUNTIME_PM_OPS(svc_i3c_runtime_suspend, + svc_i3c_runtime_resume, NULL) +}; + +static const struct svc_i3c_drvdata npcm845_drvdata = { + .quirks = SVC_I3C_QUIRK_FIFO_EMPTY | + SVC_I3C_QUIRK_FALSE_SLVSTART | + SVC_I3C_QUIRK_DAA_CORRUPT, +}; + +static const struct svc_i3c_drvdata svc_default_drvdata = {}; + +static const struct of_device_id svc_i3c_master_of_match_tbl[] = { + { .compatible = "nuvoton,npcm845-i3c", .data = &npcm845_drvdata }, + { .compatible = "silvaco,i3c-master-v1", .data = &svc_default_drvdata }, + { /* sentinel */ }, +}; +MODULE_DEVICE_TABLE(of, svc_i3c_master_of_match_tbl); + +static struct platform_driver svc_i3c_master = { + .probe = svc_i3c_master_probe, + .remove = svc_i3c_master_remove, + .driver = { + .name = "silvaco-i3c-master", + .of_match_table = svc_i3c_master_of_match_tbl, + .pm = &svc_i3c_pm_ops, + }, +}; +module_platform_driver(svc_i3c_master); + +MODULE_AUTHOR("Conor Culhane <conor.culhane@silvaco.com>"); +MODULE_AUTHOR("Miquel Raynal <miquel.raynal@bootlin.com>"); +MODULE_DESCRIPTION("Silvaco dual-role I3C master driver"); +MODULE_LICENSE("GPL v2"); |