Merge tag 'locking-urgent-2026-09-20' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tipgrafted

Pull futex fix from Ingo Molnar: - Also allocate a default private futex hash on vfork() as well, to avoid races with (private) futex waiters (Peter Zijlstra) * tag 'locking-urgent-2026-09-20' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/tip: futex: Also allocate private hash on vfork()
author: Linus Torvalds <torvalds@linux-foundation.org> 2026-09-20 08:15:23 -0700
committer: Linus Torvalds <torvalds@linux-foundation.org> 2026-09-20 08:15:23 -0700
commit: bdab18633a302c82b5e5d5dbc4e38b0baf9c0837 (patch)
tree: 0446500af9a76d4e8b2a72a0a41391ceede73a10 /drivers/scsi/leapraid
download: linux-stable-bdab18633a302c82b5e5d5dbc4e38b0baf9c0837.tar.gz
linux-stable-bdab18633a302c82b5e5d5dbc4e38b0baf9c0837.zip
8 files changed, 16879 insertions, 0 deletions
diff --git a/drivers/scsi/leapraid/Kconfig b/drivers/scsi/leapraid/Kconfig
new file mode 100644
index 000000000..f4ece78f8
--- /dev/null
+++ b/drivers/scsi/leapraid/Kconfig
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: GPL-2.0-or-later
+
+config SCSI_LEAPRAID
+	tristate "LeapIO RAID Adapter"
+	depends on PCI && SCSI
+	select SCSI_SAS_ATTRS
+	help
+	  This driver supports LeapIO PCIe-based storage
+	  and RAID controllers.
+
+	  <http://www.leap-io.com>
+
+	  To compile this driver as a module, choose M here: the
+	  resulting kernel module will be named leapraid.
diff --git a/drivers/scsi/leapraid/Makefile b/drivers/scsi/leapraid/Makefile
new file mode 100644
index 000000000..bdafc036c
--- /dev/null
+++ b/drivers/scsi/leapraid/Makefile
@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: GPL-2.0
+#
+# Makefile for the LEAPRAID drivers.
+#
+
+obj-$(CONFIG_SCSI_LEAPRAID) += leapraid.o
+leapraid-objs += leapraid_func.o \
+		leapraid_os.o \
+		leapraid_transport.o \
+		leapraid_app.o
diff --git a/drivers/scsi/leapraid/leapraid.h b/drivers/scsi/leapraid/leapraid.h
new file mode 100644
index 000000000..c2d28359b
--- /dev/null
+++ b/drivers/scsi/leapraid/leapraid.h
@@ -0,0 +1,2048 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026 LeapIO Tech Inc.
+ *
+ * LeapRAID storage and RAID controller driver.
+ */
+#ifndef LEAPRAID_H
+#define LEAPRAID_H
+
+/* Doorbell register definitions. */
+#define LEAPRAID_DB_RESET       0x00000000
+#define LEAPRAID_DB_READY       0x10000000
+#define LEAPRAID_DB_OPERATIONAL 0x20000000
+#define LEAPRAID_DB_FAULT       0x40000000
+
+#define LEAPRAID_DB_MASK        0xF0000000
+
+#define LEAPRAID_DB_OVER_TEMPERATURE    0x2810
+
+#define LEAPRAID_DB_USED                0x08000000
+#define LEAPRAID_DB_DATA_MASK           0x0000FFFF
+#define LEAPRAID_DB_FUNC_SHIFT          24
+#define LEAPRAID_DB_ADD_DWORDS_SHIFT    16
+
+/* Maximum number of retries waiting for doorbell to become ready. */
+#define LEAPRAID_DB_RETRY_COUNT_MAX    10
+/* Maximum number of retries waiting for doorbell to become operational. */
+#define LEAPRAID_DB_WAIT_OP_SHORT   10
+#define LEAPRAID_DB_WAIT_OP_LONG   200
+
+/* Maximum number of retries waiting for host to end recovery. */
+#define LEAPRAID_WAIT_SHOST_RECOVERY   400
+
+/* Diagnostic register definitions. */
+#define LEAPRAID_DIAG_WRITE_ENABLE      0x00000080
+#define LEAPRAID_DIAG_RESET             0x00000004
+
+/* Interrupt status register definitions. */
+#define LEAPRAID_HOST2ADAPTER_DB_STATUS 0x80000000
+#define LEAPRAID_ADAPTER2HOST_DB_STATUS 0x00000001
+
+/* The number of debug register. */
+#define LEAPRAID_DEBUGLOG_SZ_MAX    16
+
+/* Reply post host register definitions. */
+#define REP_POST_HOST_IDX_REG_CNT 16
+#define LEAPRAID_RPHI_MSIX_IDX_SHIFT    24
+
+/* Virtual PHY flags. */
+#define LEAPRAID_SAS_PHYINFO_VPHY       0x00001000
+
+/* Linux driver init firmware. */
+#define LEAPRAID_WHOINIT_LINUX_DRIVER    0x04
+
+/* Reply descriptor post queue array mode. */
+#define LEAPRAID_ADAPTER_INIT_MSGFLG_RDPQ_ARRAY_MODE    0x01
+
+/* Request description flags. */
+#define LEAPRAID_REQ_DESC_FLG_SCSI_IO   0x00
+#define LEAPRAID_REQ_DESC_FLG_HPR       0x06
+#define LEAPRAID_REQ_DESC_FLG_DFLT_TYPE 0x08
+
+/* Reply description flags. */
+#define LEAPRAID_RPY_DESC_FLG_TYPE_MASK                 0x0F
+#define LEAPRAID_RPY_DESC_FLG_SCSI_IO_SUCCESS           0x00
+#define LEAPRAID_RPY_DESC_FLG_ADDRESS_REPLY             0x01
+#define LEAPRAID_RPY_DESC_FLG_FP_SCSI_IO_SUCCESS        0x06
+#define LEAPRAID_RPY_DESC_FLG_UNUSED                    0x0F
+
+/* Request and reply messages share the same set of function codes.
+ *
+ * Note: SCSIIO (SCSI I/O) represents SCSI I/O operations and is used
+ * consistently throughout the driver for all SCSI command handling.
+ */
+#define LEAPRAID_FUNC_SCSIIO                    0x00
+#define LEAPRAID_FUNC_SCSI_TMF                  0x01
+#define LEAPRAID_FUNC_ADAPTER_INIT              0x02
+#define LEAPRAID_FUNC_GET_ADAPTER_FEATURES      0x03
+#define LEAPRAID_FUNC_CONFIG_OP                 0x04
+#define LEAPRAID_FUNC_SCAN_DEV                  0x06
+#define LEAPRAID_FUNC_EVENT_NOTIFY              0x07
+#define LEAPRAID_FUNC_FW_DOWNLOAD               0x09
+#define LEAPRAID_FUNC_FW_UPLOAD                 0x12
+#define LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH   0x16
+#define LEAPRAID_FUNC_SCSI_ENC_PROCESSOR        0x18
+#define LEAPRAID_FUNC_SMP_PASSTHROUGH           0x1A
+#define LEAPRAID_FUNC_SAS_IO_UNIT_CTRL          0x1B
+#define LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH   0x1C
+#define LEAPRAID_FUNC_ADAPTER_UNIT_RESET        0x40
+#define LEAPRAID_FUNC_HANDSHAKE                 0x42
+#define LEAPRAID_FUNC_LOGBUF_INIT               0x57
+
+/* Adapter status values. */
+#define LEAPRAID_ADAPTER_STATUS_MASK                    0x7FFF
+#define LEAPRAID_ADAPTER_STATUS_SUCCESS                 0x0000
+#define LEAPRAID_ADAPTER_STATUS_BUSY                    0x0002
+#define LEAPRAID_ADAPTER_STATUS_INTERNAL_ERROR          0x0004
+#define LEAPRAID_ADAPTER_STATUS_INSUFFICIENT_RESOURCES  0x0006
+#define LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_ACTION   0x0020
+#define LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_TYPE     0x0021
+#define LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE     0x0022
+#define LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_DATA     0x0023
+#define LEAPRAID_ADAPTER_STATUS_CONFIG_NO_DEFAULTS      0x0024
+#define LEAPRAID_ADAPTER_STATUS_CONFIG_CANT_COMMIT      0x0025
+#define LEAPRAID_ADAPTER_STATUS_SCSI_RECOVERED_ERROR    0x0040
+#define LEAPRAID_ADAPTER_STATUS_SCSI_DEVICE_NOT_THERE   0x0043
+#define LEAPRAID_ADAPTER_STATUS_SCSI_DATA_OVERRUN       0x0044
+#define LEAPRAID_ADAPTER_STATUS_SCSI_DATA_UNDERRUN      0x0045
+#define LEAPRAID_ADAPTER_STATUS_SCSI_IO_DATA_ERROR      0x0046
+#define LEAPRAID_ADAPTER_STATUS_SCSI_PROTOCOL_ERROR     0x0047
+#define LEAPRAID_ADAPTER_STATUS_SCSI_TASK_TERMINATED    0x0048
+#define LEAPRAID_ADAPTER_STATUS_SCSI_RESIDUAL_MISMATCH  0x0049
+#define LEAPRAID_ADAPTER_STATUS_SCSI_TASK_MGMT_FAILED   0x004A
+#define LEAPRAID_ADAPTER_STATUS_SCSI_ADAPTER_TERMINATED 0x004B
+#define LEAPRAID_ADAPTER_STATUS_SCSI_EXT_TERMINATED     0x004C
+
+/* SGE flags. */
+#define LEAPRAID_SGE_FLG_LAST_ONE       0x80
+#define LEAPRAID_SGE_FLG_EOB            0x40
+#define LEAPRAID_SGE_FLG_EOL            0x01
+#define LEAPRAID_SGE_FLG_SHIFT          24
+#define LEAPRAID_SGE_FLG_SIMPLE_ONE     0x10
+#define LEAPRAID_SGE_FLG_SYSTEM_ADDR    0x00
+#define LEAPRAID_SGE_FLG_H2C            0x04
+#define LEAPRAID_SGE_FLG_32             0x00
+#define LEAPRAID_SGE_FLG_64             0x02
+
+#define LEAPRAID_IEEE_SGE_FLG_EOL               0x40
+#define LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE        0x00
+#define LEAPRAID_IEEE_SGE_FLG_CHAIN_ONE         0x80
+#define LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR       0x00
+
+#define LEAPRAID_SGE_NO_DATA_ADDR -1
+#define LEAPRAID_SGE_OFFSET_SIZE  4
+
+/* The type of page. */
+#define LEAPRAID_CFG_PT_IO_UNIT         0x00
+#define LEAPRAID_CFG_PT_ADAPTER         0x01
+#define LEAPRAID_CFG_PT_BIOS            0x02
+#define LEAPRAID_CFG_PT_RAID_VOLUME     0x08
+#define LEAPRAID_CFG_PT_MANUFACTURING   0x09
+#define LEAPRAID_CFG_PT_RAID_PHYSDISK   0x0A
+#define LEAPRAID_CFG_PT_EXTENDED        0x0F
+
+/* The type of extended page. */
+#define LEAPRAID_CFG_EXTPT_SAS_IO_UNIT  0x10
+#define LEAPRAID_CFG_EXTPT_SAS_EXP      0x11
+#define LEAPRAID_CFG_EXTPT_SAS_DEV      0x12
+#define LEAPRAID_CFG_EXTPT_SAS_PHY      0x13
+#define LEAPRAID_CFG_EXTPT_ENC          0x15
+#define LEAPRAID_CFG_EXTPT_RAID_CONFIG  0x16
+
+/* Config page address. */
+#define LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP     0x00000000
+#define LEAPRAID_SAS_ENC_CFG_PGAD_HDL           0x10000000
+#define LEAPRAID_SAS_DEV_CFG_PGAD_HDL           0x20000000
+#define LEAPRAID_SAS_EXP_CFG_PGAD_HDL_PHY_NUM   0x10000000
+#define LEAPRAID_SAS_EXP_CFD_PGAD_HDL           0x20000000
+#define LEAPRAID_SAS_EXP_CFG_PGAD_PHYNUM_SHIFT  16
+#define LEAPRAID_RAID_VOL_CFG_PGAD_HDL          0x10000000
+#define LEAPRAID_SAS_PHY_CFG_PGAD_PHY_NUMBER    0x00000000
+#define LEAPRAID_PHYSDISK_CFG_PGAD_PHYSDISKNUM  0x10000000
+
+/* Config page operations. */
+#define LEAPRAID_CFG_ACT_PAGE_HEADER    0x00
+#define LEAPRAID_CFG_ACT_PAGE_READ_CUR  0x01
+#define LEAPRAID_CFG_ACT_PAGE_WRITE_CUR 0x02
+
+/* BIOS page number. */
+#define LEAPRAID_CFG_PAGE_NUM_BIOS2     0x2
+#define LEAPRAID_CFG_PAGE_NUM_BIOS3     0x3
+
+/* Manufacturing page number. */
+#define LEAPRAID_CFG_PAGE_NUM_MANU0     0x0
+
+/* SAS device page number. */
+#define LEAPRAID_CFG_PAGE_NUM_DEV0      0x0
+
+/* SAS device page 0 flags. */
+#define LEAPRAID_SAS_DEV_P0_FLG_ENC_LEVEL_VALID 0x0002
+#define LEAPRAID_SAS_DEV_P0_FLG_DEV_PRESENT     0x0001
+#define LEAPRAID_SAS_DEV_P0_CON_NAME_LEN        4
+
+/* SAS I/O unit page number. */
+#define LEAPRAID_CFG_PAGE_NUM_IOUNIT0   0x0
+#define LEAPRAID_CFG_PAGE_NUM_IOUNIT1   0x1
+
+/* SAS expander page number. */
+#define LEAPRAID_CFG_PAGE_NUM_EXP0      0x0
+#define LEAPRAID_CFG_PAGE_NUM_EXP1      0x1
+
+/* SAS enclosure page number. */
+#define LEAPRAID_CFG_PAGE_NUM_ENC0      0x0
+
+/* SAS PHY page number. */
+#define LEAPRAID_CFG_PAGE_NUM_PHY0      0x0
+
+/* RAID volume page number. */
+#define LEAPRAID_CFG_PAGE_NUM_VOL0      0x0
+#define LEAPRAID_CFG_PAGE_NUM_VOL1      0x1
+
+/* Physical disk page number. */
+#define LEAPRAID_CFG_PAGE_NUM_PD0       0x0
+
+#define LEAPRAID_CFG_UNIT_SIZE  4
+
+/* Raid volume type and state. */
+#define LEAPRAID_VOL_STATE_ONLINE       0x03
+#define LEAPRAID_VOL_STATE_DEGRADED     0x04
+#define LEAPRAID_VOL_STATE_OPTIMAL      0x05
+#define LEAPRAID_VOL_TYPE_RAID0         0x00
+#define LEAPRAID_VOL_TYPE_RAID1E        0x01
+#define LEAPRAID_VOL_TYPE_RAID1         0x02
+#define LEAPRAID_VOL_TYPE_RAID10        0x05
+#define LEAPRAID_VOL_TYPE_UNKNOWN       0xFF
+
+/* Raid volume element flags. */
+#define LEAPRAID_RAIDCFG_P0_EFLG_MASK_ELEMENT_TYPE      0x000F
+#define LEAPRAID_RAIDCFG_P0_EFLG_VOL_PHYS_DISK_ELEMENT  0x0001
+#define LEAPRAID_RAIDCFG_P0_EFLG_HOT_SPARE_ELEMENT      0x0002
+#define LEAPRAID_RAIDCFG_P0_EFLG_OCE_ELEMENT            0x0003
+
+/* SAS negotiated link rates. */
+#define LEAPRAID_SAS_NEG_LINK_RATE_MASK_PHYSICAL        0x0F
+#define LEAPRAID_SAS_NEG_LINK_RATE_UNKNOWN_LINK_RATE    0x00
+#define LEAPRAID_SAS_NEG_LINK_RATE_PHY_DISABLED         0x01
+#define LEAPRAID_SAS_NEG_LINK_RATE_NEGOTIATION_FAILED   0x02
+#define LEAPRAID_SAS_NEG_LINK_RATE_SATA_OOB_COMPLETE    0x03
+#define LEAPRAID_SAS_NEG_LINK_RATE_PORT_SELECTOR        0x04
+#define LEAPRAID_SAS_NEG_LINK_RATE_SMP_RESETTING        0x05
+#define LEAPRAID_SAS_NEG_LINK_RATE_SHIFT                4
+
+#define LEAPRAID_SAS_NEG_LINK_RATE_1_5  0x08
+#define LEAPRAID_SAS_NEG_LINK_RATE_3_0  0x09
+#define LEAPRAID_SAS_NEG_LINK_RATE_6_0  0x0A
+#define LEAPRAID_SAS_NEG_LINK_RATE_12_0 0x0B
+
+#define LEAPRAID_SAS_PRATE_MIN_RATE_MASK        0x0F
+#define LEAPRAID_SAS_HWRATE_MIN_RATE_MASK       0x0F
+
+/*
+ * Control flags for a SCSI I/O Request.
+ */
+#define LEAPRAID_SCSIIO_CTRL_CDB_32BYTE         4
+#define LEAPRAID_SCSIIO_CTRL_CDB_LEN_SHIFT      26
+#define LEAPRAID_SCSIIO_CTRL_NODATATRANSFER     0x00000000
+#define LEAPRAID_SCSIIO_CTRL_WRITE              0x01000000
+#define LEAPRAID_SCSIIO_CTRL_READ               0x02000000
+#define LEAPRAID_SCSIIO_CTRL_BIDIRECTIONAL      0x03000000
+#define LEAPRAID_SCSIIO_CTRL_SIMPLEQ            0x00000000
+#define LEAPRAID_SCSIIO_CTRL_ORDEREDQ           0x00000200
+#define LEAPRAID_SCSIIO_CTRL_CMDPRI             0x00000800
+
+/* SCSI state and status. */
+#define LEAPRAID_SCSI_STATUS_BUSY                       0x08
+#define LEAPRAID_SCSI_STATUS_RESERVATION_CONFLICT       0x18
+#define LEAPRAID_SCSI_STATUS_TASK_SET_FULL              0x28
+
+#define LEAPRAID_SCSI_STATE_RESPONSE_INFO_VALID 0x10
+#define LEAPRAID_SCSI_STATE_TERMINATED          0x08
+#define LEAPRAID_SCSI_STATE_NO_SCSI_STATUS      0x04
+#define LEAPRAID_SCSI_STATE_AUTOSENSE_FAILED    0x02
+#define LEAPRAID_SCSI_STATE_AUTOSENSE_VALID     0x01
+
+/* SCSI task management defines. */
+#define LEAPRAID_TM_TASKTYPE_ABORT_TASK         0x01
+#define LEAPRAID_TM_TASKTYPE_ABRT_TASK_SET      0x02
+#define LEAPRAID_TM_TASKTYPE_TARGET_RESET       0x03
+#define LEAPRAID_TM_TASKTYPE_LOGICAL_UNIT_RESET 0x05
+#define LEAPRAID_TM_TASKTYPE_CLEAR_TASK_SET     0x06
+#define LEAPRAID_TM_TASKTYPE_QUERY_TASK         0x07
+#define LEAPRAID_TM_TASKTYPE_CLEAR_ACA          0x08
+#define LEAPRAID_TM_TASKTYPE_QUERY_TASK_SET     0x09
+#define LEAPRAID_TM_TASKTYPE_QUERY_ASYNC_EVENT  0x0A
+
+#define LEAPRAID_TM_MSGFLAGS_LINK_RESET         0x00
+#define LEAPRAID_TM_RSP_INVALID_FRAME           0x02
+
+/* SCSI enclosure processor request defines. */
+#define LEAPRAID_SEP_REQ_ACT_WRITE_STATUS               0x00
+#define LEAPRAID_SEP_REQ_FLG_DEVHDL_ADDRESS             0x00
+#define LEAPRAID_SEP_REQ_FLG_ENCLOSURE_SLOT_ADDRESS     0x01
+#define LEAPRAID_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT     0x00000040
+
+/* The capabilities of the adapter. */
+#define LEAPRAID_ADAPTER_FEATURES_CAP_ATOMIC_REQ                0x00080000
+#define LEAPRAID_ADAPTER_FEATURES_CAP_INTEGRATED_RAID           0x00001000
+
+/* Event code definitions for the firmware. */
+#define LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE      0x000F
+#define LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST       0x001C
+#define LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE 0x001D
+#define LEAPRAID_EVT_IR_CHANGE          0x0020
+#define LEAPRAID_EVT_TURN_ON_PFA_LED    0xFFFC
+#define LEAPRAID_EVT_SCAN_DEV_DONE      0xFFFD
+#define LEAPRAID_EVT_REMOVE_DEAD_DEV    0xFFFF
+#define LEAPRAID_MAX_EVENT_NUM          128
+
+#define LEAPRAID_EVT_SAS_DEV_STAT_RC_INTERNAL_DEV_RESET         0x08
+#define LEAPRAID_EVT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET     0x0E
+
+/* RAID configuration change event. */
+#define LEAPRAID_EVT_IR_RC_VOLUME_ADD                   0x01
+#define LEAPRAID_EVT_IR_RC_VOLUME_DELETE                0x02
+#define LEAPRAID_EVT_IR_RC_PD_HIDDEN_TO_ADD             0x03
+#define LEAPRAID_EVT_IR_RC_PD_UNHIDDEN_TO_DELETE        0x04
+#define LEAPRAID_EVT_IR_RC_PD_CREATED_TO_HIDE           0x05
+#define LEAPRAID_EVT_IR_RC_PD_DELETED_TO_EXPOSE         0x06
+#define LEAPRAID_EVT_IR_RC_VOLUME_HIDE                  0x0A
+#define LEAPRAID_EVT_IR_RC_VOLUME_UNHIDE                0x0B
+
+/* SAS topology change event. */
+#define LEAPRAID_EVT_SAS_TOPO_ES_NO_EXPANDER    0x00
+#define LEAPRAID_EVT_SAS_TOPO_ES_ADDED          0x01
+#define LEAPRAID_EVT_SAS_TOPO_ES_NOT_RESPONDING 0x02
+#define LEAPRAID_EVT_SAS_TOPO_ES_RESPONDING     0x03
+
+#define LEAPRAID_EVT_SAS_TOPO_RC_MASK                   0x0F
+#define LEAPRAID_EVT_SAS_TOPO_RC_TARG_ADDED             0x01
+#define LEAPRAID_EVT_SAS_TOPO_RC_TARG_NOT_RESPONDING    0x02
+
+/* Enclosure device status change event. */
+#define LEAPRAID_EVT_SAS_ENCL_RC_ADDED          0x01
+#define LEAPRAID_EVT_SAS_ENCL_RC_NOT_RESPONDING 0x02
+
+/* Device type and identifiers. */
+#define LEAPRAID_DEVTYP_SEP             0x00004000
+#define LEAPRAID_DEVTYP_SSP_TGT         0x00000400
+#define LEAPRAID_DEVTYP_STP_TGT         0x00000200
+#define LEAPRAID_DEVTYP_SMP_TGT         0x00000100
+#define LEAPRAID_DEVTYP_SATA_DEV        0x00000080
+#define LEAPRAID_DEVTYP_SSP_INIT        0x00000040
+#define LEAPRAID_DEVTYP_STP_INIT        0x00000020
+#define LEAPRAID_DEVTYP_SMP_INIT        0x00000010
+#define LEAPRAID_DEVTYP_SATA_HOST       0x00000008
+
+#define LEAPRAID_DEVTYP_MASK_DEV_TYPE   0x00000007
+#define LEAPRAID_DEVTYP_NO_DEV          0x00000000
+#define LEAPRAID_DEVTYP_END_DEV         0x00000001
+#define LEAPRAID_DEVTYP_EDGE_EXPANDER   0x00000002
+#define LEAPRAID_DEVTYP_FANOUT_EXPANDER 0x00000003
+
+/* SAS control operation. */
+#define LEAPRAID_SAS_OP_PHY_LINK_RESET  0x06
+#define LEAPRAID_SAS_OP_PHY_HARD_RESET  0x07
+#define LEAPRAID_SAS_OP_SET_PARAMETER   0x0F
+
+/* Boot device defines */
+#define LEAPRAID_BOOTDEV_FORM_MASK      0x0F
+#define LEAPRAID_BOOTDEV_FORM_NONE      0x00
+#define LEAPRAID_BOOTDEV_FORM_SAS_WWID  0x05
+#define LEAPRAID_BOOTDEV_FORM_ENC_SLOT  0x06
+#define LEAPRAID_BOOTDEV_FORM_DEV_NAME  0x07
+
+/**
+ * struct leapraid_reg_base - Register layout of the LeapRAID controller
+ *
+ * @db: Doorbell register used to signal commands or status to firmware.
+ * @ws: Write sequence register for synchronizing doorbell operations.
+ * @host_diag: Diagnostic register used for status or debug reporting.
+ * @r1: Reserved.
+ * @host_int_status: Interrupt status register reporting active interrupts.
+ * @host_int_mask: Interrupt mask register enabling or disabling sources.
+ * @r2: Reserved.
+ * @rep_msg_host_idx: Reply message index for the next available reply slot.
+ * @r3: Reserved.
+ * @debug_log: DebugLog registers for firmware debug and diagnostic output.
+ * @r4: Reserved.
+ * @atomic_req_desc_post: Atomic register for single descriptor posting.
+ * @adapter_log_buf_pos: Adapter log buffer write position.
+ * @host_log_buf_pos: Host log buffer write position.
+ * @r5: Reserved.
+ * @rep_post_reg_idx: Array of reply post index registers, one per queue.
+ *                    The number of entries is defined by
+ *                    REP_POST_HOST_IDX_REG_CNT.
+ */
+struct leapraid_reg_base {
+	__le32 db;
+	__le32 ws;
+	__le32 host_diag;
+	__le32 r1[9];
+	__le32 host_int_status;
+	__le32 host_int_mask;
+	__le32 r2[4];
+	__le32 rep_msg_host_idx;
+	__le32 r3[13];
+	__le32 debug_log[LEAPRAID_DEBUGLOG_SZ_MAX];
+	__le32 r4[2];
+	__le32 atomic_req_desc_post;
+	__le32 adapter_log_buf_pos;
+	__le32 host_log_buf_pos;
+	__le32 r5[142];
+	struct leapraid_rep_post_reg_idx {
+		__le32 idx;
+		__le32 r1;
+		__le32 r2;
+		__le32 r3;
+	} rep_post_reg_idx[REP_POST_HOST_IDX_REG_CNT];
+} __packed;
+
+/**
+ * struct leapraid_atomic_req_desc - Atomic request descriptor
+ *
+ * @flg: Descriptor flag indicating the type of request (e.g. SCSI I/O).
+ * @msix_idx: MSI-X vector index used for interrupt routing.
+ * @taskid: Unique task identifier associated with this request.
+ */
+struct leapraid_atomic_req_desc {
+	u8 flg;
+	u8 msix_idx;
+	__le16 taskid;
+};
+
+/**
+ * union leapraid_rep_desc_union - Unified reply descriptor format
+ *
+ * @dflt_rep: Default reply descriptor containing basic completion info.
+ * @dflt_rep.rep_flg: Reply flag indicating reply type or status.
+ * @dflt_rep.msix_idx: MSI-X index for interrupt routing.
+ * @dflt_rep.taskid: Task identifier matching the submitted request.
+ * @r1: Reserved.
+ *
+ * @addr_rep: Address reply descriptor used when firmware returns a
+ *            memory address associated with the reply.
+ * @addr_rep.rep_flg: Reply flag indicating reply type or status.
+ * @addr_rep.msix_idx: MSI-X index for interrupt routing.
+ * @addr_rep.taskid: Task identifier matching the submitted request.
+ * @addr_rep.rep_frame_addr: Physical address of the reply frame.
+ *
+ * @words: Raw 64-bit representation of the reply descriptor.
+ * @u: Alternative access using 32-bit low/high words.
+ * @u.low: Lower 32 bits of the descriptor.
+ * @u.high: Upper 32 bits of the descriptor.
+ */
+union leapraid_rep_desc_union {
+	struct leapraid_rep_desc {
+		u8 rep_flg;
+		u8 msix_idx;
+		__le16 taskid;
+		u8 r1[4];
+	} dflt_rep;
+	struct leapraid_add_rep_desc {
+		u8 rep_flg;
+		u8 msix_idx;
+		__le16 taskid;
+		__le32 rep_frame_addr;
+	} addr_rep;
+	__le64 words;
+	struct {
+		u32 low;
+		u32 high;
+	} u;
+}  __packed __aligned(4);
+
+/**
+ * struct leapraid_req - Generic request header
+ *
+ * @func_dep1: Function-dependent parameter (low 16 bits).
+ * @r1: Reserved.
+ * @func: Function code identifying the command type.
+ * @r2: Reserved.
+ */
+struct leapraid_req {
+	__le16 func_dep1;
+	u8 r1;
+	u8 func;
+	u8 r2[8];
+};
+
+/**
+ * struct leapraid_rep - Generic reply header
+ *
+ * @r1: Reserved.
+ * @msg_len: Length of the reply message in bytes.
+ * @function: Function code corresponding to the request.
+ * @r2: Reserved.
+ * @adapter_status: Status code reported by the adapter.
+ * @r3: Reserved.
+ */
+struct leapraid_rep {
+	u8 r1[2];
+	u8 msg_len;
+	u8 function;
+	u8 r2[10];
+	__le16 adapter_status;
+	u8 r3[4];
+};
+
+/**
+ * struct leapraid_sge_simple32 - 32-bit simple scatter-gather entry
+ *
+ * @flg_and_len: Combined field for flags and segment length.
+ * @addr: 32-bit physical address of the data buffer.
+ */
+struct leapraid_sge_simple32 {
+	__le32 flg_and_len;
+	__le32 addr;
+};
+
+/**
+ * struct leapraid_sge_simple64 - 64-bit simple scatter-gather entry
+ *
+ * @flg_and_len: Combined field for flags and segment length.
+ * @addr: 64-bit physical address of the data buffer.
+ */
+struct leapraid_sge_simple64 {
+	__le32 flg_and_len;
+	__le64 addr;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_sge_simple_union - Unified 32/64-bit SGE representation
+ *
+ * @flg_and_len: Combined field for flags and segment length.
+ * @u.addr32: 32-bit address field.
+ * @u.addr64: 64-bit address field.
+ */
+struct leapraid_sge_simple_union {
+	__le32 flg_and_len;
+	union {
+		__le32 addr32;
+		__le64 addr64;
+	} __packed __aligned(4) u;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_sge_chain_union - Chained scatter-gather entry
+ *
+ * @len: Length of the chain descriptor.
+ * @next_chain_offset: Offset to the next SGE chain.
+ * @flg: Flags indicating chain or termination properties.
+ * @u.addr32: 32-bit physical address.
+ * @u.addr64: 64-bit physical address.
+ */
+struct leapraid_sge_chain_union {
+	__le16 len;
+	u8 next_chain_offset;
+	u8 flg;
+	union {
+		__le32 addr32;
+		__le64 addr64;
+	} __packed __aligned(4) u;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_ieee_sge_simple32 - IEEE 32-bit simple SGE format
+ *
+ * @addr: 32-bit physical address of the data buffer.
+ * @flg_and_len: Combined field for flags and data length.
+ */
+struct leapraid_ieee_sge_simple32 {
+	__le32 addr;
+	__le32 flg_and_len;
+};
+
+/**
+ * struct leapraid_ieee_sge_simple64 - IEEE 64-bit simple SGE format
+ *
+ * @addr: 64-bit physical address of the data buffer.
+ * @len: Length of the data segment.
+ * @r1: Reserved.
+ * @flg: Flags indicating transfer properties.
+ */
+struct leapraid_ieee_sge_simple64 {
+	__le64 addr;
+	__le32 len;
+	u8 r1[3];
+	u8 flg;
+} __packed __aligned(4);
+
+/**
+ * union leapraid_ieee_sge_simple_union - Unified IEEE SGE format
+ *
+ * @simple32: IEEE 32-bit simple SGE entry.
+ * @simple64: IEEE 64-bit simple SGE entry.
+ */
+union leapraid_ieee_sge_simple_union {
+	struct leapraid_ieee_sge_simple32 simple32;
+	struct leapraid_ieee_sge_simple64 simple64;
+};
+
+/**
+ * union leapraid_ieee_sge_chain_union - Unified IEEE SGE chain format
+ *
+ * @chain32: IEEE 32-bit chain SGE entry.
+ * @chain64: IEEE 64-bit chain SGE entry.
+ */
+union leapraid_ieee_sge_chain_union {
+	struct leapraid_ieee_sge_simple32 chain32;
+	struct leapraid_ieee_sge_simple64 chain64;
+};
+
+/**
+ * struct leapraid_chain64_ieee_sg - 64-bit IEEE chain SGE descriptor
+ *
+ * @addr: Physical address of the next chain segment.
+ * @len: Length of the current SGE.
+ * @r1: Reserved.
+ * @next_chain_offset: Offset to the next chain element.
+ * @flg: Flags that describe SGE attributes.
+ */
+struct leapraid_chain64_ieee_sg {
+	__le64 addr;
+	__le32 len;
+	u8 r1[2];
+	u8 next_chain_offset;
+	u8 flg;
+} __packed __aligned(4);
+
+/**
+ * union leapraid_ieee_sge_io_union - IEEE-style SGE union for I/O
+ *
+ * @ieee_simple: Simple IEEE SGE descriptor.
+ * @ieee_chain: IEEE chain SGE descriptor.
+ */
+union leapraid_ieee_sge_io_union {
+	struct leapraid_ieee_sge_simple64 ieee_simple;
+	struct leapraid_chain64_ieee_sg ieee_chain;
+};
+
+/**
+ * union leapraid_simple_sge_union - Union of simple SGE descriptors
+ *
+ * @leapraid_simple: LeapRAID simple SGE.
+ * @ieee_simple: IEEE-style simple SGE.
+ */
+union leapraid_simple_sge_union {
+	struct leapraid_sge_simple_union leapraid_simple;
+	union leapraid_ieee_sge_simple_union ieee_simple;
+};
+
+/**
+ * union leapraid_sge_io_union - Combined SGE union for all I/O types
+ *
+ * @leapraid_simple: LeapRAID simple SGE format.
+ * @leapraid_chain: LeapRAID chain SGE format.
+ * @ieee_simple: IEEE simple SGE format.
+ * @ieee_chain: IEEE chain SGE format.
+ */
+union leapraid_sge_io_union {
+	struct leapraid_sge_simple_union leapraid_simple;
+	struct leapraid_sge_chain_union leapraid_chain;
+	union leapraid_ieee_sge_simple_union ieee_simple;
+	union leapraid_ieee_sge_chain_union ieee_chain;
+};
+
+/**
+ * struct leapraid_cfg_pg_header - Standard configuration page header
+ *
+ * @r1: Reserved.
+ * @page_len: Length of the page in 4-byte units.
+ * @page_num: Page number.
+ * @page_type: Page type.
+ */
+struct leapraid_cfg_pg_header {
+	u8 r1;
+	u8 page_len;
+	u8 page_num;
+	u8 page_type;
+};
+
+/**
+ * struct leapraid_cfg_ext_pg_header - Extended configuration page header
+ *
+ * @r1: Reserved.
+ * @r2: Reserved.
+ * @page_num: Page number.
+ * @page_type: Page type.
+ * @ext_page_len: Extended page length.
+ * @ext_page_type: Extended page type.
+ * @r3: Reserved.
+ */
+struct leapraid_cfg_ext_pg_header {
+	u8 r1;
+	u8 r2;
+	u8 page_num;
+	u8 page_type;
+	__le16 ext_page_len;
+	u8 ext_page_type;
+	u8 r3;
+};
+
+/**
+ * struct leapraid_cfg_req - Configuration request message
+ *
+ * @action: Requested action type.
+ * @sgl_flag: SGL flag field.
+ * @chain_offset: Offset to next chain SGE.
+ * @func: Function code.
+ * @ext_page_len: Extended page length.
+ * @ext_page_type: Extended page type.
+ * @msg_flag: Message flags.
+ * @r1: Reserved.
+ * @header: Configuration page header.
+ * @page_addr: Address of the page buffer.
+ * @page_buf_sge: SGE describing the page buffer.
+ */
+struct leapraid_cfg_req {
+	u8 action;
+	u8 sgl_flag;
+	u8 chain_offset;
+	u8 func;
+	__le16 ext_page_len;
+	u8 ext_page_type;
+	u8 msg_flag;
+	u8 r1[12];
+	struct leapraid_cfg_pg_header header;
+	__le32 page_addr;
+	union leapraid_sge_io_union page_buf_sge;
+};
+
+/**
+ * struct leapraid_cfg_rep - Configuration reply message
+ *
+ * @action: Action type from the request.
+ * @r1: Reserved.
+ * @msg_len: Message length in bytes.
+ * @func: Function code.
+ * @ext_page_len: Extended page length.
+ * @ext_page_type: Extended page type.
+ * @msg_flag: Message flags.
+ * @r2: Reserved.
+ * @adapter_status: Adapter status code.
+ * @r3: Reserved.
+ * @header: Configuration page header.
+ */
+struct leapraid_cfg_rep {
+	u8 action;
+	u8 r1;
+	u8 msg_len;
+	u8 func;
+	__le16 ext_page_len;
+	u8 ext_page_type;
+	u8 msg_flag;
+	u8 r2[6];
+	__le16 adapter_status;
+	u8 r3[4];
+	struct leapraid_cfg_pg_header header;
+};
+
+/**
+ * struct leapraid_boot_dev_format_sas_wwid - Boot device identified by wwid
+ *
+ * @sas_addr: SAS address of the device.
+ * @lun: Logical unit number.
+ * @r1: Reserved.
+ */
+struct leapraid_boot_dev_format_sas_wwid {
+	__le64 sas_addr;
+	u8 lun[8];
+	u8 r1[8];
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_boot_dev_format_enc_slot - identified by enclosure
+ *
+ * @enc_lid: Enclosure logical ID.
+ * @r1: Reserved.
+ * @slot_num: Slot number in the enclosure.
+ * @r2: Reserved.
+ */
+struct leapraid_boot_dev_format_enc_slot {
+	__le64 enc_lid;
+	u8 r1[8];
+	__le16 slot_num;
+	u8 r2[6];
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_boot_dev_format_dev_name - Boot device by device name
+ *
+ * @dev_name: Device name identifier.
+ * @lun: Logical unit number.
+ * @r1: Reserved.
+ */
+struct leapraid_boot_dev_format_dev_name {
+	__le64 dev_name;
+	u8 lun[8];
+	u8 r1[8];
+} __packed __aligned(4);
+
+/**
+ * union leapraid_boot_dev_format - Boot device format union
+ *
+ * @sas_wwid: Format using SAS WWID and LUN.
+ * @enc_slot: Format using enclosure slot and ID.
+ * @dev_name: Format using device name and LUN.
+ */
+union leapraid_boot_dev_format {
+	struct leapraid_boot_dev_format_sas_wwid sas_wwid;
+	struct leapraid_boot_dev_format_enc_slot enc_slot;
+	struct leapraid_boot_dev_format_dev_name dev_name;
+};
+
+/**
+ * struct leapraid_manufacturing_p0 - Manufacturing configuration page 0
+ *
+ * @header: Configuration page header.
+ * @chip_name: Chip name as a 16-byte ASCII string.
+ * @chip_revision: Chip revision as an 8-byte ASCII string.
+ * @board_name: Board name as a 16-byte ASCII string.
+ * @board_assembly: Board assembly as a 16-byte ASCII string.
+ * @board_tracer_number: Board tracer number as a 16-byte ASCII string.
+ */
+struct leapraid_manufacturing_p0 {
+	struct leapraid_cfg_pg_header header;
+	u8 chip_name[16];
+	u8 chip_revision[8];
+	u8 board_name[16];
+	u8 board_assembly[16];
+	u8 board_tracer_number[16];
+};
+
+/**
+ * struct leapraid_bios_page2 - BIOS configuration page 2
+ *
+ * @header: Configuration page header.
+ * @r1: Reserved.
+ * @requested_boot_dev_form: Format type of the requested boot device.
+ * @r2: Reserved.
+ * @requested_boot_dev: Boot device requested by BIOS or user.
+ * @requested_alt_boot_dev_form: Format of the alternate boot device.
+ * @r3: Reserved.
+ * @requested_alt_boot_dev: Alternate boot device requested.
+ * @current_boot_dev_form: Format type of the active boot device.
+ * @r4: Reserved.
+ * @current_boot_dev: Currently active boot device in use.
+ */
+struct leapraid_bios_page2 {
+	struct leapraid_cfg_pg_header header;
+	u8 r1[24];
+	u8 requested_boot_dev_form;
+	u8 r2[3];
+	union leapraid_boot_dev_format requested_boot_dev;
+	u8 requested_alt_boot_dev_form;
+	u8 r3[3];
+	union leapraid_boot_dev_format requested_alt_boot_dev;
+	u8 current_boot_dev_form;
+	u8 r4[3];
+	union leapraid_boot_dev_format current_boot_dev;
+};
+
+/**
+ * struct leapraid_bios_page3 - BIOS configuration page 3
+ *
+ * @header: Configuration page header.
+ * @r1: Reserved.
+ * @bios_version: BIOS firmware version number.
+ * @r2: Reserved.
+ */
+struct leapraid_bios_page3 {
+	struct leapraid_cfg_pg_header header;
+	u8 r1[4];
+	__le32 bios_version;
+	u8 r2[84];
+};
+
+/**
+ * struct leapraid_raidvol0_phys_disk - Physical disk in RAID volume
+ *
+ * @r1: Reserved.
+ * @phys_disk_num: Physical disk number within the RAID volume.
+ * @r2: Reserved.
+ */
+struct leapraid_raidvol0_phys_disk {
+	u8 r1[2];
+	u8 phys_disk_num;
+	u8 r2;
+};
+
+/**
+ * struct leapraid_raidvol_p0 - RAID volume configuration page 0
+ *
+ * @header: Configuration page header.
+ * @dev_hdl: Device handle for the RAID volume.
+ * @volume_state: State of the RAID volume.
+ * @volume_type: RAID type.
+ * @r1: Reserved.
+ * @num_phys_disks: Number of physical disks in the volume.
+ * @r2: Reserved.
+ * @phys_disk: Array of physical disks in this volume.
+ */
+struct leapraid_raidvol_p0 {
+	struct leapraid_cfg_pg_header header;
+	__le16 dev_hdl;
+	u8 volume_state;
+	u8 volume_type;
+	u8 r1[28];
+	u8 num_phys_disks;
+	u8 r2[3];
+	struct leapraid_raidvol0_phys_disk phys_disk[];
+};
+
+/**
+ * struct leapraid_raidvol_p1 - RAID volume configuration page 1
+ *
+ * @header: Configuration page header.
+ * @dev_hdl: Device handle of the RAID volume.
+ * @r1: Reserved.
+ * @wwid: World-wide identifier for the volume.
+ * @r2: Reserved.
+ */
+struct leapraid_raidvol_p1 {
+	struct leapraid_cfg_pg_header header;
+	__le16 dev_hdl;
+	u8 r1[42];
+	__le64 wwid;
+	u8 r2[8];
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_raidpd_p0 - Physical disk configuration page 0
+ *
+ * @header: Configuration page header.
+ * @dev_hdl: Device handle of the physical disk.
+ * @r1: Reserved.
+ * @phys_disk_num: Physical disk number.
+ * @r2: Reserved.
+ */
+struct leapraid_raidpd_p0 {
+	struct leapraid_cfg_pg_header header;
+	__le16 dev_hdl;
+	u8 r1;
+	u8 phys_disk_num;
+	u8 r2[112];
+};
+
+/**
+ * struct leapraid_sas_io_unit0_phy_info - PHY info for SAS I/O unit
+ *
+ * @port: Port number the PHY belongs to.
+ * @port_flg: Flags describing port status.
+ * @phy_flg: Flags describing PHY status.
+ * @neg_link_rate: Negotiated link rate of the PHY.
+ * @controller_phy_dev_info: Controller PHY device info.
+ * @attached_dev_hdl: Handle of attached device.
+ * @controller_dev_hdl: Handle of the controller device.
+ * @r1: Reserved.
+ */
+struct leapraid_sas_io_unit0_phy_info {
+	u8 port;
+	u8 port_flg;
+	u8 phy_flg;
+	u8 neg_link_rate;
+	__le32 controller_phy_dev_info;
+	__le16 attached_dev_hdl;
+	__le16 controller_dev_hdl;
+	u8 r1[8];
+};
+
+/**
+ * struct leapraid_sas_io_unit_p0 - SAS I/O unit configuration page 0
+ *
+ * @header: Extended configuration page header.
+ * @r1: Reserved.
+ * @phy_num: Number of PHYs in this unit.
+ * @r2: Reserved.
+ * @phy_info: Array of PHY information.
+ */
+struct leapraid_sas_io_unit_p0 {
+	struct leapraid_cfg_ext_pg_header header;
+	u8 r1[4];
+	u8 phy_num;
+	u8 r2[3];
+	struct leapraid_sas_io_unit0_phy_info phy_info[];
+};
+
+/**
+ * struct leapraid_exp_p0 - SAS expander page 0
+ *
+ * @header: Extended page header.
+ * @physical_port: Physical port number.
+ * @r1: Reserved.
+ * @enc_hdl: Enclosure handle.
+ * @sas_address: SAS address of the expander.
+ * @r2: Reserved.
+ * @dev_hdl: Device handle of this expander.
+ * @parent_dev_hdl: Device handle of parent expander.
+ * @r3: Reserved.
+ * @phy_num: Number of PHYs.
+ * @r4: Reserved.
+ */
+struct leapraid_exp_p0 {
+	struct leapraid_cfg_ext_pg_header header;
+	u8 physical_port;
+	u8 r1;
+	__le16 enc_hdl;
+	__le64 sas_address;
+	u8 r2[4];
+	__le16 dev_hdl;
+	__le16 parent_dev_hdl;
+	u8 r3[4];
+	u8 phy_num;
+	u8 r4[27];
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_exp_p1 - SAS expander page 1
+ *
+ * @header: Extended page header.
+ * @r1: Reserved.
+ * @p_link_rate: PHY link rate.
+ * @hw_link_rate: Hardware supported link rate.
+ * @attached_dev_hdl: Attached device handle.
+ * @r2: Reserved.
+ * @neg_link_rate: Negotiated link rate.
+ * @r3: Reserved.
+ */
+struct leapraid_exp_p1 {
+	struct leapraid_cfg_ext_pg_header header;
+	u8 r1[8];
+	u8 p_link_rate;
+	u8 hw_link_rate;
+	__le16 attached_dev_hdl;
+	u8 r2[11];
+	u8 neg_link_rate;
+	u8 r3[12];
+};
+
+/**
+ * struct leapraid_sas_dev_p0 - SAS device page 0
+ *
+ * @header: Extended configuration page header.
+ * @slot: Slot number.
+ * @enc_hdl: Enclosure handle.
+ * @sas_address: SAS address.
+ * @parent_dev_hdl: Parent device handle.
+ * @phy_num: Number of PHYs.
+ * @r1: Reserved.
+ * @dev_hdl: Device handle.
+ * @r2: Reserved.
+ * @dev_info: Device information.
+ * @flg: Flags.
+ * @physical_port: Physical port number.
+ * @max_port_connections: Maximum port connections.
+ * @dev_name: Device name.
+ * @port_groups: Number of port groups.
+ * @r3: Reserved.
+ * @enc_level: Enclosure level.
+ * @connector_name: Connector identifier.
+ * @r4: Reserved.
+ */
+struct leapraid_sas_dev_p0 {
+	struct leapraid_cfg_ext_pg_header header;
+	__le16 slot;
+	__le16 enc_hdl;
+	__le64 sas_address;
+	__le16 parent_dev_hdl;
+	u8 phy_num;
+	u8 r1;
+	__le16 dev_hdl;
+	u8 r2[2];
+	__le32 dev_info;
+	__le16 flg;
+	u8 physical_port;
+	u8 max_port_connections;
+	__le64 dev_name;
+	u8 port_groups;
+	u8 r3[2];
+	u8 enc_level;
+	u8 connector_name[LEAPRAID_SAS_DEV_P0_CON_NAME_LEN];
+	u8 r4[4];
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_sas_phy_p0 - SAS PHY configuration page 0
+ *
+ * @header: Extended configuration page header.
+ * @r1: Reserved.
+ * @attached_dev_hdl: Handle of attached device.
+ * @r2: Reserved.
+ * @p_link_rate: PHY link rate.
+ * @hw_link_rate: Hardware supported link rate.
+ * @r3: Reserved.
+ * @phy_info: PHY information.
+ * @neg_link_rate: Negotiated link rate.
+ * @r4: Reserved.
+ */
+struct leapraid_sas_phy_p0 {
+	struct leapraid_cfg_ext_pg_header header;
+	u8 r1[4];
+	__le16 attached_dev_hdl;
+	u8 r2[6];
+	u8 p_link_rate;
+	u8 hw_link_rate;
+	u8 r3[2];
+	__le32 phy_info;
+	u8 neg_link_rate;
+	u8 r4[3];
+};
+
+/**
+ * struct leapraid_enc_p0 - SAS enclosure page 0
+ *
+ * @header: Extended configuration page header.
+ * @r1: Reserved.
+ * @enc_lid: Enclosure logical ID.
+ * @r2: Reserved.
+ * @enc_hdl: Enclosure handle.
+ * @r3: Reserved.
+ */
+struct leapraid_enc_p0 {
+	struct leapraid_cfg_ext_pg_header header;
+	u8 r1[4];
+	__le64 enc_lid;
+	u8 r2[2];
+	__le16 enc_hdl;
+	u8 r3[15];
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_raid_cfg_p0_element - RAID configuration element
+ *
+ * @element_flg: Element flags.
+ * @vol_dev_hdl: Volume device handle.
+ * @r1: Reserved.
+ * @phys_disk_dev_hdl: Physical disk device handle.
+ */
+struct leapraid_raid_cfg_p0_element {
+	__le16 element_flg;
+	__le16 vol_dev_hdl;
+	u8 r1[2];
+	__le16 phys_disk_dev_hdl;
+};
+
+/**
+ * struct leapraid_raid_cfg_p0 - RAID configuration page 0
+ *
+ * @header: Extended configuration page header.
+ * @r1: Reserved.
+ * @cfg_num: Configuration number.
+ * @r2: Reserved.
+ * @elements_num: Number of RAID elements.
+ * @r3: Reserved.
+ * @cfg_element: Array of RAID elements.
+ */
+struct leapraid_raid_cfg_p0 {
+	struct leapraid_cfg_ext_pg_header header;
+	u8 r1[3];
+	u8 cfg_num;
+	u8 r2[32];
+	u8 elements_num;
+	u8 r3[3];
+	struct leapraid_raid_cfg_p0_element cfg_element[];
+};
+
+/**
+ * union leapraid_mpi_scsi_io_cdb_union - SCSI I/O CDB or simple SGE
+ *
+ * @cdb32: 32-byte SCSI command descriptor block.
+ * @sge: Simple SGE format.
+ */
+union leapraid_mpi_scsi_io_cdb_union {
+	u8 cdb32[32];
+	struct leapraid_sge_simple_union sge;
+};
+
+/**
+ * struct leapraid_mpi_scsiio_req - MPI SCSI I/O request
+ *
+ * @dev_hdl: Device handle for the target.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @r1: Reserved.
+ * @msg_flg: Message flags.
+ * @r2: Reserved.
+ * @sense_buffer_low_add: Lower 32-bit address of sense buffer.
+ * @dma_flag: DMA flags.
+ * @r3: Reserved.
+ * @sense_buffer_len: Sense buffer length.
+ * @r4: Reserved.
+ * @sgl_offset0..3: SGL offsets.
+ * @skip_count: Bytes to skip before transfer.
+ * @data_len: Length of data transfer.
+ * @bi_dir_data_len: Bi-directional transfer length.
+ * @io_flg: I/O flags.
+ * @eedp_flag: End-to-end data protection flags.
+ * @eedp_block_size: End-to-end data protection block size.
+ * @r5: Reserved.
+ * @secondary_ref_tag: Secondary reference tag.
+ * @secondary_app_tag: Secondary application tag.
+ * @app_tag_trans_mask: Application tag mask.
+ * @lun: Logical Unit Number.
+ * @ctrl: Control flags.
+ * @cdb: SCSI Command Descriptor Block or simple SGE.
+ * @sgl: Scatter-gather list.
+ */
+struct leapraid_mpi_scsiio_req {
+	__le16 dev_hdl;
+	u8 chain_offset;
+	u8 func;
+	u8 r1[3];
+	u8 msg_flg;
+	u8 r2[4];
+	__le32 sense_buffer_low_add;
+	u8 dma_flag;
+	u8 r3;
+	u8 sense_buffer_len;
+	u8 r4;
+	u8 sgl_offset0;
+	u8 sgl_offset1;
+	u8 sgl_offset2;
+	u8 sgl_offset3;
+	__le32 skip_count;
+	__le32 data_len;
+	__le32 bi_dir_data_len;
+	__le16 io_flg;
+	__le16 eedp_flag;
+	__le16 eedp_block_size;
+	u8 r5[2];
+	__le32 secondary_ref_tag;
+	__le16 secondary_app_tag;
+	__le16 app_tag_trans_mask;
+	u8 lun[8];
+	__le32 ctrl;
+	union leapraid_mpi_scsi_io_cdb_union cdb;
+	union leapraid_sge_io_union sgl;
+};
+
+/**
+ * union leapraid_scsi_io_cdb_union - SCSI I/O CDB or IEEE simple SGE
+ *
+ * @cdb32: 32-byte SCSI CDB.
+ * @sge: IEEE simple 64-bit SGE.
+ */
+union leapraid_scsi_io_cdb_union {
+	u8 cdb32[32];
+	struct leapraid_ieee_sge_simple64 sge;
+};
+
+/**
+ * struct leapraid_scsiio_req - SCSI I/O request
+ *
+ * @dev_hdl: Device handle.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @r1: Reserved.
+ * @msg_flg: Message flags.
+ * @r2: Reserved.
+ * @sense_buffer_low_add: Lower 32-bit address of sense buffer.
+ * @dma_flag: DMA flag.
+ * @r3: Reserved.
+ * @sense_buffer_len: Sense buffer length.
+ * @r4: Reserved.
+ * @sgl_offset0-3: SGL offsets.
+ * @skip_count: Bytes to skip before transfer.
+ * @data_len: Length of data transfer.
+ * @bi_dir_data_len: Bi-directional transfer length.
+ * @io_flg: I/O flags.
+ * @eedp_flag: End-to-end data protection flags.
+ * @eedp_block_size: End-to-end data protection block size.
+ * @r5: Reserved.
+ * @secondary_ref_tag: Secondary reference tag.
+ * @secondary_app_tag: Secondary application tag.
+ * @app_tag_trans_mask: Application tag mask.
+ * @lun: Logical Unit Number.
+ * @ctrl: Control flags.
+ * @cdb: SCSI Command Descriptor Block or simple SGE.
+ * @sgl: Scatter-gather list.
+ */
+struct leapraid_scsiio_req {
+	__le16 dev_hdl;
+	u8 chain_offset;
+	u8 func;
+	u8 r1[3];
+	u8 msg_flg;
+	u8 r2[4];
+	__le32 sense_buffer_low_add;
+	u8 dma_flag;
+	u8 r3;
+	u8 sense_buffer_len;
+	u8 r4;
+	u8 sgl_offset0;
+	u8 sgl_offset1;
+	u8 sgl_offset2;
+	u8 sgl_offset3;
+	__le32 skip_count;
+	__le32 data_len;
+	__le32 bi_dir_data_len;
+	__le16 io_flg;
+	__le16 eedp_flag;
+	__le16 eedp_block_size;
+	u8 r5[2];
+	__le32 secondary_ref_tag;
+	__le16 secondary_app_tag;
+	__le16 app_tag_trans_mask;
+	u8 lun[8];
+	__le32 ctrl;
+	union leapraid_scsi_io_cdb_union cdb;
+	union leapraid_ieee_sge_io_union sgl;
+};
+
+/**
+ * struct leapraid_scsiio_rep - SCSI I/O response
+ *
+ * @dev_hdl: Device handle.
+ * @msg_len: Length of response message.
+ * @func: Function code.
+ * @r1: Reserved.
+ * @msg_flg: Message flags.
+ * @r2: Reserved.
+ * @scsi_status: SCSI status.
+ * @scsi_state: SCSI state.
+ * @adapter_status: Adapter status.
+ * @r3: Reserved.
+ * @transfer_count: Number of bytes transferred.
+ * @sense_count: Number of sense bytes.
+ * @resp_info: Additional response info.
+ * @task_tag: Task identifier.
+ * @scsi_status_qualifier: SCSI status qualifier.
+ * @bi_dir_trans_count: Bi-directional transfer count.
+ * @r4: Reserved.
+ */
+struct leapraid_scsiio_rep {
+	__le16 dev_hdl;
+	u8 msg_len;
+	u8 func;
+	u8 r1[3];
+	u8 msg_flg;
+	u8 r2[4];
+	u8 scsi_status;
+	u8 scsi_state;
+	__le16 adapter_status;
+	u8 r3[4];
+	__le32 transfer_count;
+	__le32 sense_count;
+	__le32 resp_info;
+	__le16 task_tag;
+	__le16 scsi_status_qualifier;
+	__le32 bi_dir_trans_count;
+	__le32 r4[3];
+};
+
+/**
+ * struct leapraid_scsi_tm_req - SCSI Task Management request
+ *
+ * @dev_hdl: Device handle.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @r1: Reserved.
+ * @task_type: Task management function type.
+ * @r2: Reserved.
+ * @msg_flg: Message flags.
+ * @r3: Reserved.
+ * @lun: Logical Unit Number.
+ * @r4: Reserved.
+ * @task_mid: Task identifier.
+ * @r5: Reserved.
+ */
+struct leapraid_scsi_tm_req {
+	__le16 dev_hdl;
+	u8 chain_offset;
+	u8 func;
+	u8 r1;
+	u8 task_type;
+	u8 r2;
+	u8 msg_flg;
+	u8 r3[4];
+	u8 lun[8];
+	u8 r4[28];
+	__le16 task_mid;
+	u8 r5[2];
+};
+
+/**
+ * struct leapraid_scsi_tm_rep - SCSI Task Management response
+ *
+ * @dev_hdl: Device handle.
+ * @msg_len: Length of response message.
+ * @func: Function code.
+ * @resp_code: Response code.
+ * @task_type: Task management type.
+ * @r1: Reserved.
+ * @msg_flag: Message flags.
+ * @r2: Reserved.
+ * @adapter_status: Adapter status.
+ * @r3: Reserved.
+ * @termination_count: Count of terminated tasks.
+ * @response_info: Additional response info.
+ */
+struct leapraid_scsi_tm_rep {
+	__le16 dev_hdl;
+	u8 msg_len;
+	u8 func;
+	u8 resp_code;
+	u8 task_type;
+	u8 r1;
+	u8 msg_flag;
+	u8 r2[6];
+	__le16 adapter_status;
+	u8 r3[4];
+	__le32 termination_count;
+	__le32 response_info;
+};
+
+/**
+ * struct leapraid_sep_req - SEP (SCSI Enclosure Processor) request
+ *
+ * @dev_hdl: Device handle.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @act: Action to perform.
+ * @flg: Flags.
+ * @r1: Reserved.
+ * @msg_flag: Message flags.
+ * @r2: Reserved.
+ * @slot_status: Slot status.
+ * @r3: Reserved.
+ * @slot: Slot number.
+ * @enc_hdl: Enclosure handle.
+ */
+struct leapraid_sep_req {
+	__le16 dev_hdl;
+	u8 chain_offset;
+	u8 func;
+	u8 act;
+	u8 flg;
+	u8 r1;
+	u8 msg_flag;
+	u8 r2[4];
+	__le32 slot_status;
+	u8 r3[12];
+	__le16 slot;
+	__le16 enc_hdl;
+};
+
+/**
+ * struct leapraid_sep_rep - SEP response
+ *
+ * @dev_hdl: Device handle.
+ * @msg_len: Message length.
+ * @func: Function code.
+ * @act: Action performed.
+ * @flg: Flags.
+ * @msg_flag: Message flags.
+ * @r1: Reserved.
+ * @adapter_status: Adapter status.
+ * @r2: Reserved.
+ * @slot_status: Slot status.
+ * @r3: Reserved.
+ * @slot: Slot number.
+ * @enc_hdl: Enclosure handle.
+ */
+struct leapraid_sep_rep {
+	__le16 dev_hdl;
+	u8 msg_len;
+	u8 func;
+	u8 act;
+	u8 flg;
+	u8 r1;
+	u8 msg_flag;
+	u8 r2[6];
+	__le16 adapter_status;
+	u8 r3[4];
+	__le32 slot_status;
+	u8 r4[4];
+	__le16 slot;
+	__le16 enc_hdl;
+};
+
+/**
+ * struct leapraid_adapter_init_req - Adapter initialization request
+ *
+ * @who_init: Initiator of the initialization.
+ * @r1: Reserved.
+ * @chain_offset: Chain offset.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @msg_flg: Message flags.
+ * @r3: Reserved.
+ * @msg_ver: Message version.
+ * @header_ver: Header version.
+ * @host_buf_addr: Host buffer address (non adapter-ref).
+ * @r4: Reserved.
+ * @host_buf_size: Host buffer size (non adapter-ref).
+ * @host_msix_vectors: Number of host MSI-X vectors.
+ * @r6: Reserved.
+ * @req_frame_size: Request frame size.
+ * @rep_desc_qd: Reply descriptor queue depth.
+ * @rep_msg_qd: Reply message queue depth.
+ * @sense_buffer_add_high: High 32-bit of sense buffer address.
+ * @rep_msg_dma_high: High 32-bit of reply message DMA address.
+ * @task_desc_base_addr: Base address of task descriptors.
+ * @rep_desc_q_arr_addr: Address of reply descriptor queue array.
+ * @rep_msg_addr_dma: Reply message DMA address.
+ * @time_stamp: Timestamp.
+ */
+struct leapraid_adapter_init_req {
+	u8 who_init;
+	u8 r1;
+	u8 chain_offset;
+	u8 func;
+	u8 r2[3];
+	u8 msg_flg;
+	__le32 driver_ver;
+	__le16 msg_ver;
+	__le16 header_ver;
+	__le32 host_buf_addr;
+	u8 r4[2];
+	u8 host_buf_size;
+	u8 host_msix_vectors;
+	u8 r6[2];
+	__le16 req_frame_size;
+	__le16 rep_desc_qd;
+	__le16 rep_msg_qd;
+	__le32 sense_buffer_add_high;
+	__le32 rep_msg_dma_high;
+	__le64 task_desc_base_addr;
+	__le64 rep_desc_q_arr_addr;
+	__le64 rep_msg_addr_dma;
+	__le64 time_stamp;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_rep_desc_q_arr - Reply descriptor queue array
+ *
+ * @rep_desc_base_addr: Base address of the reply descriptors.
+ * @r1: Reserved.
+ */
+struct leapraid_rep_desc_q_arr {
+	__le64 rep_desc_base_addr;
+	__le64 r1;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_adapter_init_rep - Adapter initialization reply
+ *
+ * @who_init: Initiator of the initialization.
+ * @r1: Reserved.
+ * @msg_len: Length of reply message.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @msg_flag: Message flags.
+ * @r3: Reserved.
+ * @adapter_status: Adapter status.
+ * @r4: Reserved.
+ */
+struct leapraid_adapter_init_rep {
+	u8 who_init;
+	u8 r1;
+	u8 msg_len;
+	u8 func;
+	u8 r2[3];
+	u8 msg_flag;
+	u8 r3[6];
+	__le16 adapter_status;
+	u8 r4[4];
+};
+
+/**
+ * struct leapraid_adapter_log_req - Adapter log request
+ *
+ * @action: Action code.
+ * @type: Log type.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * r1: Reserved.
+ * @msg_flag: Message flags.
+ * r2: Reserved.
+ * @mbox: Mailbox for command-specific parameters.
+ * @sge: Scatter-gather entry for data buffer.
+ */
+struct leapraid_adapter_log_req {
+	u8 action;
+	u8 type;
+	u8 chain_offset;
+	u8 func;
+	u8 r1[3];
+	u8 msg_flag;
+	u8 r2[4];
+	union {
+		u8	b[12];
+		__le16	s[6];
+		__le32	w[3];
+	} mbox;
+	struct leapraid_sge_simple64 sge;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_adapter_log_rep - Adapter log reply
+ *
+ * @action: Action code echoed.
+ * @type: Log type echoed.
+ * @msg_len: Length of message.
+ * @func: Function code.
+ * @r1: Reserved.
+ * @msg_flag: Message flags.
+ * @r2: Reserved.
+ * @adapter_status: Status returned by adapter.
+ */
+struct leapraid_adapter_log_rep {
+	u8 action;
+	u8 type;
+	u8 msg_len;
+	u8 func;
+	u8 r1[3];
+	u8 msg_flag;
+	u8 r2[6];
+	__le16 adapter_status;
+};
+
+/**
+ * struct leapraid_adapter_features_req - Request adapter features
+ *
+ * @r1: Reserved.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @msg_flag: Message flags.
+ * @r3: Reserved.
+ */
+struct leapraid_adapter_features_req {
+	u8 r1[2];
+	u8 chain_offset;
+	u8 func;
+	u8 r2[3];
+	u8 msg_flag;
+	u8 r3[4];
+};
+
+/**
+ * struct leapraid_adapter_features_rep - Adapter features reply
+ *
+ * @msg_ver: Message version.
+ * @msg_len: Length of reply message.
+ * @func: Function code.
+ * @header_ver: Header version.
+ * @r1: Reserved.
+ * @msg_flag: Message flags.
+ * @r2: Reserved.
+ * @adapter_status: Adapter status.
+ * @r3: Reserved.
+ * @who_init: Who initialized the adapter.
+ * @r4: Reserved.
+ * @max_msix_vectors: Max MSI-X vectors supported.
+ * @req_slot: Number of request slots.
+ * @r5: Reserved.
+ * @adapter_caps: Adapter capabilities.
+ * @fw_version: Firmware version.
+ * @sas_wide_max_qdepth: Max wide SAS queue depth.
+ * @sas_narrow_max_qdepth: Max narrow SAS queue depth.
+ * @r6: Reserved.
+ * @hp_slot: Number of high-priority slots.
+ * @r7: Reserved.
+ * @max_volumes: Maximum supported volumes.
+ * @max_dev_hdl: Maximum device handle.
+ * @r8: Reserved.
+ * @min_dev_hdl: Minimum device handle.
+ * @r9: Reserved.
+ */
+struct leapraid_adapter_features_rep {
+	__le16 msg_ver;
+	u8 msg_len;
+	u8 func;
+	u16 header_ver;
+	u8 r1;
+	u8 msg_flag;
+	u8 r2[6];
+	u16 adapter_status;
+	u8 r3[4];
+	u8 sata_max_qdepth;
+	u8 who_init;
+	u8 r4;
+	u8 max_msix_vectors;
+	__le16 req_slot;
+	__le16 product_id;
+	__le32 adapter_caps;
+	__le32 fw_version;
+	__le16 sas_wide_max_qdepth;
+	__le16 sas_narrow_max_qdepth;
+	u8 r6[10];
+	__le16 hp_slot;
+	u8 r7[3];
+	u8 max_volumes;
+	__le16 max_dev_hdl;
+	u8 r8[2];
+	__le16 min_dev_hdl;
+	u8 r9[6];
+};
+
+/**
+ * struct leapraid_scan_dev_req - Request to scan devices
+ *
+ * @r1: Reserved.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @msg_flag: Message flags.
+ * @r3: Reserved.
+ */
+struct leapraid_scan_dev_req {
+	u8 r1[2];
+	u8 chain_offset;
+	u8 func;
+	u8 r2[3];
+	u8 msg_flag;
+	u8 r3[4];
+};
+
+/**
+ * struct leapraid_scan_dev_rep - Scan devices reply
+ *
+ * @r1: Reserved.
+ * @msg_len: Length of message.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @msg_flag: Message flags.
+ * @r3: Reserved.
+ * @adapter_status: Adapter status.
+ * @r4: Reserved.
+ */
+struct leapraid_scan_dev_rep {
+	u8 r1[2];
+	u8 msg_len;
+	u8 func;
+	u8 r2[3];
+	u8 msg_flag;
+	u8 r3[6];
+	__le16 adapter_status;
+	u8 r4[4];
+};
+
+/**
+ * struct leapraid_evt_notify_req - Event notification request
+ *
+ * @r1: Reserved.
+ * @chain_offset: Offset for chained SGE.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @msg_flag: Message flags.
+ * @r3: Reserved.
+ * @evt_masks: Event masks to enable notifications.
+ * @r4: Reserved.
+ */
+struct leapraid_evt_notify_req {
+	u8 r1[2];
+	u8 chain_offset;
+	u8 func;
+	u8 r2[3];
+	u8 msg_flag;
+	u8 r3[12];
+	__le32 evt_masks[4];
+	u8 r4[8];
+};
+
+/**
+ * struct leapraid_evt_notify_rep - Event notification reply
+ *
+ * @evt_data_len: Length of event data.
+ * @msg_len: Length of message.
+ * @func: Function code.
+ * @r1: Reserved.
+ * @r2: Reserved.
+ * @msg_flag: Message flags.
+ * @r3: Reserved.
+ * @adapter_status: Adapter status.
+ * @r4: Reserved.
+ * @evt: Event code.
+ * @r5: Reserved.
+ * @evt_data: Event data array.
+ */
+struct leapraid_evt_notify_rep {
+	__le16 evt_data_len;
+	u8 msg_len;
+	u8 func;
+	u8 r1[2];
+	u8 r2;
+	u8 msg_flag;
+	u8 r3[6];
+	__le16 adapter_status;
+	u8 r4[4];
+	__le16 evt;
+	u8 r5[6];
+	__le32 evt_data[];
+};
+
+/**
+ * struct leapraid_evt_data_sas_dev_status_change - SAS device status change
+ *
+ * @task_tag: Task identifier.
+ * @reason_code: Reason for status change.
+ * @physical_port: Physical port number.
+ * @r1: Reserved.
+ * @dev_hdl: Device handle.
+ * @r2: Reserved.
+ * @sas_address: SAS address of device.
+ * @lun: Logical unit Number.
+ */
+struct leapraid_evt_data_sas_dev_status_change {
+	__le16 task_tag;
+	u8 reason_code;
+	u8 physical_port;
+	u8 r1[2];
+	__le16 dev_hdl;
+	u8 r2[4];
+	__le64 sas_address;
+	u8 lun[8];
+} __packed __aligned(4);
+/**
+ * struct leapraid_evt_data_ir_change - IR (Integrated RAID) change event data
+ *
+ * @r1: Reserved.
+ * @reason_code: Reason for IR change.
+ * @r2: Reserved.
+ * @vol_dev_hdl: Volume device handle.
+ * @phys_disk_dev_hdl: Physical disk device handle.
+ */
+struct leapraid_evt_data_ir_change {
+	u8 r1;
+	u8 reason_code;
+	u8 r2[2];
+	__le16 vol_dev_hdl;
+	__le16 phys_disk_dev_hdl;
+};
+
+/**
+ * struct leapraid_evt_data_sas_disc - SAS discovery event data
+ *
+ * @r1: Reserved.
+ * @reason_code: Reason for discovery event.
+ * @physical_port: Physical port number where event occurred.
+ * @r2: Reserved.
+ */
+struct leapraid_evt_data_sas_disc {
+	u8 r1;
+	u8 reason_code;
+	u8 physical_port;
+	u8 r2[5];
+};
+
+/**
+ * struct leapraid_evt_sas_topo_phy_entry - SAS topology PHY entry
+ *
+ * @attached_dev_hdl: Device handle attached to PHY.
+ * @link_rate: Current link rate.
+ * @phy_status: PHY status flags.
+ */
+struct leapraid_evt_sas_topo_phy_entry {
+	__le16 attached_dev_hdl;
+	u8 link_rate;
+	u8 phy_status;
+};
+
+/**
+ * struct leapraid_evt_data_sas_topo_change_list - SAS topology change list
+ *
+ * @encl_hdl: Enclosure handle.
+ * @exp_dev_hdl: Expander device handle.
+ * @num_phys: Number of PHYs in this entry.
+ * @r1: Reserved.
+ * @entry_num: Number of PHY elements.
+ * @start_phy_num: Start PHY number.
+ * @exp_status: Expander status.
+ * @physical_port: Physical port number.
+ * @phy: Array of SAS PHY entries.
+ */
+struct leapraid_evt_data_sas_topo_change_list {
+	__le16 encl_hdl;
+	__le16 exp_dev_hdl;
+	u8 num_phys;
+	u8 r1[3];
+	u8 entry_num;
+	u8 start_phy_num;
+	u8 exp_status;
+	u8 physical_port;
+	struct leapraid_evt_sas_topo_phy_entry phy[];
+};
+
+/**
+ * struct leapraid_evt_data_sas_enc_dev_status_change -
+ * SAS enclosure device status
+ *
+ * @enc_hdl: Enclosure handle.
+ * @reason_code: Reason code for status change.
+ * @physical_port: Physical port number.
+ * @encl_logical_id: Enclosure logical ID.
+ * @num_slots: Number of slots in enclosure.
+ * @start_slot: First affected slot.
+ * @phy_bits: Bitmap of affected PHYs.
+ */
+struct leapraid_evt_data_sas_enc_dev_status_change {
+	__le16 enc_hdl;
+	u8 reason_code;
+	u8 physical_port;
+	__le64 encl_logical_id;
+	__le16 num_slots;
+	__le16 start_slot;
+	__le32 phy_bits;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_io_unit_ctrl_req - I/O unit control request
+ *
+ * @op: Operation code.
+ * @r1: Reserved.
+ * @chain_offset: SGE chain offset.
+ * @func: Function code.
+ * @dev_hdl: Device handle.
+ * @adapter_para: Adapter parameter selector.
+ * @msg_flag: Message flags.
+ * @r2: Reserved.
+ * @phy_num: PHY number.
+ * @r3: Reserved.
+ * @adapter_para_value: Value for adapter parameter.
+ * @adapter_para_value2: Optional second parameter value.
+ * @r4: Reserved.
+ */
+struct leapraid_io_unit_ctrl_req {
+	u8 op;
+	u8 r1;
+	u8 chain_offset;
+	u8 func;
+	__le16 dev_hdl;
+	u8 adapter_para;
+	u8 msg_flag;
+	u8 r2[6];
+	u8 phy_num;
+	u8 r3[17];
+	__le32 adapter_para_value;
+	__le32 adapter_para_value2;
+	u8 r4[4];
+};
+
+/**
+ * struct leapraid_io_unit_ctrl_rep - I/O unit control reply
+ *
+ * @op: Operation code echoed.
+ * @r1: Reserved.
+ * @func: Function code.
+ * @dev_hdl: Device handle.
+ * @r2: Reserved.
+ */
+struct leapraid_io_unit_ctrl_rep {
+	u8 op;
+	u8 r1[2];
+	u8 func;
+	__le16 dev_hdl;
+	u8 r2[14];
+};
+
+/**
+ * struct leapraid_raid_act_req - RAID action request
+ *
+ * @act: RAID action code.
+ * @r1: Reserved.
+ * @func: Function code.
+ * @r2: Reserved.
+ * @phys_disk_num: Number of physical disks involved.
+ * @r3: Reserved.
+ * @action_data_sge: SGE describing action-specific data.
+ */
+struct leapraid_raid_act_req {
+	u8 act;
+	u8 r1[2];
+	u8 func;
+	u8 r2[2];
+	u8 phys_disk_num;
+	u8 r3[13];
+	struct leapraid_sge_simple_union action_data_sge;
+};
+
+/**
+ * struct leapraid_raid_act_rep - RAID action reply
+ *
+ * @act: RAID action code echoed.
+ * @r1: Reserved.
+ * @func: Function code.
+ * @vol_dev_hdl: Volume device handle.
+ * @r2: Reserved
+ * @adapter_status: Status returned by adapter.
+ * @r3: Reserved.
+ */
+struct leapraid_raid_act_rep {
+	u8 act;
+	u8 r1[2];
+	u8 func;
+	__le16 vol_dev_hdl;
+	u8 r2[8];
+	__le16 adapter_status;
+	u8 r3[76];
+};
+
+/**
+ * struct leapraid_smp_passthrough_req - SMP passthrough request
+ *
+ * @passthrough_flg: Passthrough flags.
+ * @physical_port: Target PHY port.
+ * @r1: Reserved.
+ * @func: Function code.
+ * @req_data_len: Request data length.
+ * @r2: Reserved.
+ * @sas_address: SAS address of target device.
+ * @r3: Reserved.
+ * @sgl: Scatter-gather list describing request buffer.
+ */
+struct leapraid_smp_passthrough_req {
+	u8 passthrough_flg;
+	u8 physical_port;
+	u8 r1;
+	u8 func;
+	__le16 req_data_len;
+	u8 r2[10];
+	__le64 sas_address;
+	u8 r3[8];
+	union leapraid_simple_sge_union sgl;
+} __packed __aligned(4);
+
+/**
+ * struct leapraid_smp_passthrough_rep - SMP passthrough reply
+ *
+ * @passthrough_flg: Passthrough flags echoed.
+ * @physical_port: Target PHY port.
+ * @r1: Reserved.
+ * @func: Function code.
+ * @resp_data_len: Length of response data.
+ * @r2: Reserved.
+ * @adapter_status: Adapter status.
+ * @r3: Reserved.
+ */
+struct leapraid_smp_passthrough_rep {
+	u8 passthrough_flg;
+	u8 physical_port;
+	u8 r1;
+	u8 func;
+	__le16 resp_data_len;
+	u8 r2[8];
+	__le16 adapter_status;
+	u8 r3[12];
+};
+
+/**
+ * struct leapraid_sas_io_unit_ctrl_req - SAS I/O unit control request
+ *
+ * @op: Operation code.
+ * @r1: Reserved.
+ * @func: Function code.
+ * @dev_hdl: Device handle.
+ * @r2: Reserved.
+ */
+struct leapraid_sas_io_unit_ctrl_req {
+	u8 op;
+	u8 r1[2];
+	u8 func;
+	__le16 dev_hdl;
+	u8 r2[38];
+};
+
+#endif /* LEAPRAID_H */
diff --git a/drivers/scsi/leapraid/leapraid_app.c b/drivers/scsi/leapraid/leapraid_app.c
new file mode 100644
index 000000000..84eef5094
--- /dev/null
+++ b/drivers/scsi/leapraid/leapraid_app.c
@@ -0,0 +1,809 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 LeapIO Tech Inc.
+ *
+ * LeapRAID storage and RAID controller driver.
+ */
+#include <linux/compat.h>
+#include <linux/module.h>
+#include <linux/miscdevice.h>
+
+#include "leapraid_func.h"
+
+/* IOCTL device file name. */
+#define LEAPRAID_DEV_NAME       "leapraid_ctl"
+
+/* IOCTL version. */
+#define LEAPRAID_IOCTL_VERSION  0x07
+
+/* IOCTL commands. */
+#define LEAPRAID_ADAPTER_INFO   17
+#define LEAPRAID_COMMAND        20
+#define LEAPRAID_EVENTQUERY     21
+#define LEAPRAID_EVENTREPORT    23
+
+/**
+ * struct leapraid_ioctl_header - IOCTL command header
+ *
+ * @adapter_id : Adapter identifier.
+ * @port_number: Port identifier.
+ * @max_data_size: Maximum data size for transfer.
+ */
+struct leapraid_ioctl_header {
+	u32 adapter_id;
+	u32 port_number;
+	u32 max_data_size;
+};
+
+/**
+ * struct leapraid_ioctl_diag_reset - Diagnostic reset request
+ *
+ * @hdr: Common IOCTL header.
+ */
+struct leapraid_ioctl_diag_reset {
+	struct leapraid_ioctl_header hdr;
+};
+
+/**
+ * struct leapraid_ioctl_pci_info - PCI device information
+ *
+ * @u: Union holding PCI bus/device/function information.
+ * @u.bits.dev: PCI device number.
+ * @u.bits.func: PCI function number.
+ * @u.bits.bus: PCI bus number.
+ * @u.word: Combined representation of PCI BDF.
+ * @seg_id: PCI segment identifier.
+ */
+struct leapraid_ioctl_pci_info {
+	union {
+		struct {
+			u32 dev:5;
+			u32 func:3;
+			u32 bus:24;
+		} bits;
+		u32 word;
+	} u;
+	u32 seg_id;
+};
+
+/**
+ * struct leapraid_ioctl_adapter_info - Adapter information for IOCTL
+ *
+ * @hdr: IOCTL header.
+ * @adapter_type: Adapter type identifier.
+ * @port_number: Port number.
+ * @pci_id: PCI device ID.
+ * @revision: Revision number.
+ * @sub_dev: Subsystem device ID.
+ * @sub_vendor: Subsystem vendor ID.
+ * @r0: Reserved.
+ * @fw_ver: Firmware version.
+ * @bios_ver: BIOS version.
+ * @driver_ver: Driver version.
+ * @r1: Reserved.
+ * @scsi_id: SCSI ID.
+ * @r2: Reserved.
+ * @pci_info: PCI information structure.
+ */
+struct leapraid_ioctl_adapter_info {
+	struct leapraid_ioctl_header hdr;
+	u32 adapter_type;
+	u32 port_number;
+	u32 pci_id;
+	u32 revision;
+	u32 sub_dev;
+	u32 sub_vendor;
+	u32 r0;
+	u32 fw_ver;
+	u32 bios_ver;
+	u8 driver_ver[32];
+	u8 r1;
+	u8 scsi_id;
+	u16 r2;
+	struct leapraid_ioctl_pci_info pci_info;
+};
+
+/**
+ * struct leapraid_ioctl_command - IOCTL command structure
+ *
+ * @hdr: IOCTL header.
+ * @timeout: Command timeout.
+ * @rep_msg_buf_ptr: User pointer to reply message buffer.
+ * @c2h_buf_ptr: User pointer to card-to-host data buffer.
+ * @h2c_buf_ptr: User pointer to host-to-card data buffer.
+ * @sense_data_ptr: User pointer to sense data buffer.
+ * @max_rep_bytes: Maximum reply bytes.
+ * @c2h_size: Card-to-host data size.
+ * @h2c_size: Host-to-card data size.
+ * @max_sense_bytes: Maximum sense data bytes.
+ * @data_sge_offset: Data SGE offset.
+ * @mf: Message frame data (flexible array).
+ */
+struct leapraid_ioctl_command {
+	struct leapraid_ioctl_header hdr;
+	u32 timeout;
+	void __user *rep_msg_buf_ptr;
+	void __user *c2h_buf_ptr;
+	void __user *h2c_buf_ptr;
+	void __user *sense_data_ptr;
+	u32 max_rep_bytes;
+	u32 c2h_size;
+	u32 h2c_size;
+	u32 max_sense_bytes;
+	u32 data_sge_offset;
+	u8 mf[];
+};
+
+static int leapraid_ctl_validate_sge_offset(struct leapraid_adapter *adapter,
+					    const struct leapraid_req *req,
+					    u32 offset_bytes,
+					    size_t h2c_size,
+					    size_t c2h_size)
+{
+	size_t sge_bytes;
+
+	switch (req->func) {
+	case LEAPRAID_FUNC_SCSIIO:
+	case LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH:
+	case LEAPRAID_FUNC_SMP_PASSTHROUGH:
+	case LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH:
+	case LEAPRAID_FUNC_FW_DOWNLOAD:
+	case LEAPRAID_FUNC_FW_UPLOAD:
+		sge_bytes = LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+		break;
+	default:
+		sge_bytes = adapter->adapter_attr.use_32_dma_mask ?
+			sizeof(struct leapraid_sge_simple32) :
+			sizeof(struct leapraid_sge_simple64);
+		break;
+	}
+
+	if (h2c_size && c2h_size)
+		sge_bytes *= 2;
+
+	if (offset_bytes > LEAPRAID_REQUEST_SIZE - sge_bytes) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid offset_bytes=%u for func=0x%x\n",
+			__func__, offset_bytes, req->func);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static struct leapraid_adapter *leapraid_ctl_lookup_adapter(int adapter_id,
+							    bool track_mmap)
+{
+	struct leapraid_adapter *adapter;
+	struct Scsi_Host *shost;
+
+	spin_lock(&leapraid_adapter_lock);
+	list_for_each_entry(adapter, &leapraid_adapter_list, list) {
+		if (adapter->adapter_attr.id == adapter_id) {
+			if (READ_ONCE(adapter->access_ctrl.host_removing))
+				break;
+			shost = adapter->shost;
+			if (!shost || !scsi_host_get(shost))
+				break;
+			if (track_mmap)
+				atomic_inc(&adapter->fw_log_desc.mmap_refcnt);
+			spin_unlock(&leapraid_adapter_lock);
+			return adapter;
+		}
+	}
+	spin_unlock(&leapraid_adapter_lock);
+
+	return NULL;
+}
+
+static void leapraid_ctl_put_adapter(struct leapraid_adapter *adapter)
+{
+	if (adapter && adapter->shost)
+		scsi_host_put(adapter->shost);
+}
+
+static void leapraid_ctl_scsiio_cmd(struct leapraid_adapter *adapter,
+				    void *ctl_sp_mpi_req,
+				    u16 taskid,
+				    dma_addr_t h2c_dma_addr, size_t h2c_size,
+				    dma_addr_t c2h_dma_addr, size_t c2h_size,
+				    u16 dev_hdl, void *psge)
+{
+	struct leapraid_mpi_scsiio_req *scsiio_request = ctl_sp_mpi_req;
+
+	scsiio_request->sense_buffer_len = SCSI_SENSE_BUFFERSIZE;
+	scsiio_request->sense_buffer_low_add =
+		leapraid_get_sense_buffer_dma(adapter, taskid);
+	memset(&adapter->driver_cmds.ctl_cmd.sense,
+	       0, SCSI_SENSE_BUFFERSIZE);
+	leapraid_build_ieee_sg(adapter, psge, h2c_dma_addr,
+			       h2c_size, c2h_dma_addr, c2h_size);
+	if (scsiio_request->func == LEAPRAID_FUNC_SCSIIO)
+		leapraid_fire_scsi_io(adapter, taskid, dev_hdl);
+	else
+		leapraid_fire_task(adapter, taskid);
+}
+
+static int leapraid_ctl_smp_passthrough_cmd(struct leapraid_adapter *adapter,
+					    void *ctl_sp_req,
+					    u16 taskid,
+					    dma_addr_t h2c_dma_addr,
+					    size_t h2c_size,
+					    dma_addr_t c2h_dma_addr,
+					    size_t c2h_size,
+					    void *psge, void *h2c)
+{
+	struct leapraid_smp_passthrough_req *smp_pt_req = ctl_sp_req;
+	u8 *data;
+
+	if (!adapter->adapter_attr.enable_mp)
+		smp_pt_req->physical_port = LEAPRAID_DISABLE_MP_PORT_ID;
+	if (smp_pt_req->passthrough_flg & LEAPRAID_SMP_PT_FLAG_SGL_PTR)
+		data = (u8 *)&smp_pt_req->sgl;
+	else
+		data = h2c;
+
+	if (!(smp_pt_req->passthrough_flg & LEAPRAID_SMP_PT_FLAG_SGL_PTR) &&
+	    h2c_size <= 10) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid request size=%zu\n",
+			__func__, h2c_size);
+		return -EINVAL;
+	}
+
+	if (data[1] == SMP_PHY_CONTROL &&
+	    (data[10] == SMP_PHY_CONTROL_LINK_RESET ||
+	     data[10] == SMP_PHY_CONTROL_HARD_RESET))
+		adapter->reset_desc.adapter_link_resetting = 1;
+
+	leapraid_build_ieee_sg(adapter, psge, h2c_dma_addr,
+			       h2c_size, c2h_dma_addr, c2h_size);
+	leapraid_fire_task(adapter, taskid);
+	return 0;
+}
+
+static void leapraid_ctl_sas_io_unit_ctrl_cmd(struct leapraid_adapter *adapter,
+					      void *ctl_sp_req,
+					      dma_addr_t h2c_dma_addr,
+					      size_t h2c_size,
+					      dma_addr_t c2h_dma_addr,
+					      size_t c2h_size,
+					      void *psge, u16 taskid)
+{
+	struct leapraid_sas_io_unit_ctrl_req *sas_io_unit_ctl_req = ctl_sp_req;
+
+	if (sas_io_unit_ctl_req->op == LEAPRAID_SAS_OP_PHY_HARD_RESET ||
+	    sas_io_unit_ctl_req->op == LEAPRAID_SAS_OP_PHY_LINK_RESET)
+		adapter->reset_desc.adapter_link_resetting = 1;
+	leapraid_build_mpi_sg(adapter, psge, h2c_dma_addr,
+			      h2c_size, c2h_dma_addr, c2h_size);
+	leapraid_fire_task(adapter, taskid);
+}
+
+static int leapraid_ctl_do_command(struct leapraid_adapter *adapter,
+				   struct leapraid_ioctl_command *karg,
+				   void __user *mf)
+{
+	struct leapraid_req *leap_mpi_req;
+	struct leapraid_req *ctl_sp_mpi_req;
+	u16 taskid;
+	void *h2c = NULL;
+	size_t h2c_size = 0;
+	dma_addr_t h2c_dma_addr = 0;
+	void *c2h = NULL;
+	size_t c2h_size = 0;
+	dma_addr_t c2h_dma_addr = 0;
+	void *psge;
+	unsigned long timeout;
+	u16 dev_hdl = LEAPRAID_INVALID_DEV_HANDLE;
+	bool issue_reset = false;
+	u32 data_sge_offset_bytes;
+	u32 sz;
+	int rc;
+
+	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
+					  __func__);
+	if (rc)
+		return rc;
+
+	leap_mpi_req = kzalloc(LEAPRAID_REQUEST_SIZE, GFP_KERNEL);
+	if (!leap_mpi_req)
+		return -ENOMEM;
+
+	if (karg->data_sge_offset > (UINT_MAX / LEAPRAID_SGE_OFFSET_SIZE)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid data_sge_offset=%u\n",
+			__func__, karg->data_sge_offset);
+		rc = -EINVAL;
+		goto out_cleanup;
+	}
+
+	data_sge_offset_bytes = karg->data_sge_offset *
+				LEAPRAID_SGE_OFFSET_SIZE;
+	if (data_sge_offset_bytes > LEAPRAID_REQUEST_SIZE) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid data_sge_offset=%u\n",
+			__func__, karg->data_sge_offset);
+		rc = -EINVAL;
+		goto out_cleanup;
+	}
+
+	if (copy_from_user(leap_mpi_req, mf, data_sge_offset_bytes)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to copy request message from user\n",
+			__func__);
+		rc = -EFAULT;
+		goto out_cleanup;
+	}
+
+	h2c_size = karg->h2c_size;
+	c2h_size = karg->c2h_size;
+	rc = leapraid_ctl_validate_sge_offset(adapter, leap_mpi_req,
+					      data_sge_offset_bytes,
+					      h2c_size, c2h_size);
+	if (rc)
+		goto out_cleanup;
+
+	taskid = adapter->driver_cmds.ctl_cmd.taskid;
+
+	adapter->driver_cmds.ctl_cmd.status = LEAPRAID_CMD_PENDING;
+	memset(&adapter->driver_cmds.ctl_cmd.reply, 0, LEAPRAID_REPLY_SIZE);
+	ctl_sp_mpi_req = leapraid_get_task_desc(adapter, taskid);
+	memset(ctl_sp_mpi_req, 0, LEAPRAID_REQUEST_SIZE);
+	memcpy(ctl_sp_mpi_req, leap_mpi_req, data_sge_offset_bytes);
+
+	if (ctl_sp_mpi_req->func == LEAPRAID_FUNC_SCSIIO ||
+	    ctl_sp_mpi_req->func == LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH ||
+	    ctl_sp_mpi_req->func == LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH) {
+		dev_hdl = le16_to_cpu(ctl_sp_mpi_req->func_dep1);
+		if (!dev_hdl ||
+		    dev_hdl > adapter->adapter_attr.features.max_dev_handle) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Invalid device handle\n", __func__);
+			rc = -EINVAL;
+			goto out_cleanup;
+		}
+	}
+
+	if (WARN_ON(ctl_sp_mpi_req->func == LEAPRAID_FUNC_SCSI_TMF)) {
+		rc = -EINVAL;
+		goto out_cleanup;
+	}
+
+	if (h2c_size) {
+		h2c = dma_alloc_coherent(&adapter->pdev->dev, h2c_size,
+					 &h2c_dma_addr, GFP_KERNEL);
+		if (!h2c) {
+			rc = -ENOMEM;
+			goto out_cleanup;
+		}
+		if (copy_from_user(h2c, karg->h2c_buf_ptr, h2c_size)) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Failed to copy h2c from user\n",
+				__func__);
+			rc = -EFAULT;
+			goto out_cleanup;
+		}
+	}
+	if (c2h_size) {
+		c2h = dma_alloc_coherent(&adapter->pdev->dev, c2h_size,
+					 &c2h_dma_addr, GFP_KERNEL);
+		if (!c2h) {
+			rc = -ENOMEM;
+			goto out_cleanup;
+		}
+	}
+
+	psge = (void *)ctl_sp_mpi_req + data_sge_offset_bytes;
+	init_completion(&adapter->driver_cmds.ctl_cmd.done);
+
+	switch (ctl_sp_mpi_req->func) {
+	case LEAPRAID_FUNC_SCSIIO:
+	case LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH:
+		leapraid_ctl_scsiio_cmd(adapter, ctl_sp_mpi_req, taskid,
+					h2c_dma_addr, h2c_size,
+					c2h_dma_addr, c2h_size,
+					dev_hdl, psge);
+		break;
+	case LEAPRAID_FUNC_SMP_PASSTHROUGH:
+		if (!h2c) {
+			rc = -EINVAL;
+			goto out_cleanup;
+		}
+		rc = leapraid_ctl_smp_passthrough_cmd(adapter,
+						      ctl_sp_mpi_req, taskid,
+						      h2c_dma_addr, h2c_size,
+						      c2h_dma_addr, c2h_size,
+						      psge, h2c);
+		if (rc)
+			goto out_cleanup;
+		break;
+	case LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH:
+		leapraid_build_ieee_sg(adapter, psge, h2c_dma_addr, h2c_size,
+				       c2h_dma_addr, c2h_size);
+		leapraid_fire_task(adapter, taskid);
+		break;
+	case LEAPRAID_FUNC_FW_DOWNLOAD:
+	case LEAPRAID_FUNC_FW_UPLOAD:
+		leapraid_build_ieee_sg(adapter, psge, h2c_dma_addr, h2c_size,
+				       c2h_dma_addr, c2h_size);
+		leapraid_fire_task(adapter, taskid);
+		break;
+	case LEAPRAID_FUNC_SAS_IO_UNIT_CTRL:
+		leapraid_ctl_sas_io_unit_ctrl_cmd(adapter, ctl_sp_mpi_req,
+						  h2c_dma_addr, h2c_size,
+						  c2h_dma_addr, c2h_size,
+						  psge, taskid);
+		break;
+	default:
+		leapraid_build_mpi_sg(adapter, psge, h2c_dma_addr,
+				      h2c_size, c2h_dma_addr, c2h_size);
+		leapraid_fire_task(adapter, taskid);
+		break;
+	}
+
+	timeout = karg->timeout;
+	if (timeout < LEAPRAID_CTL_CMD_TIMEOUT)
+		timeout = LEAPRAID_CTL_CMD_TIMEOUT;
+	if (!wait_for_completion_timeout(&adapter->driver_cmds.ctl_cmd.done,
+					 timeout * HZ)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: ctl_cmd timeout, status=0x%x\n",
+			__func__, adapter->driver_cmds.ctl_cmd.status);
+		leapraid_log_req_context(adapter, taskid, ctl_sp_mpi_req);
+	}
+
+	if ((leap_mpi_req->func == LEAPRAID_FUNC_SMP_PASSTHROUGH ||
+	     leap_mpi_req->func == LEAPRAID_FUNC_SAS_IO_UNIT_CTRL) &&
+	    adapter->reset_desc.adapter_link_resetting)
+		adapter->reset_desc.adapter_link_resetting = 0;
+
+	if (!(adapter->driver_cmds.ctl_cmd.status & LEAPRAID_CMD_DONE)) {
+		issue_reset =
+			leapraid_check_reset(
+				adapter->driver_cmds.ctl_cmd.status);
+		goto reset;
+	}
+
+	if (c2h_size && copy_to_user(karg->c2h_buf_ptr, c2h, c2h_size)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to copy c2h to user\n", __func__);
+		rc = -ENODATA;
+		goto out_cleanup;
+	}
+	if (karg->max_rep_bytes) {
+		sz = min_t(u32, karg->max_rep_bytes, LEAPRAID_REPLY_SIZE);
+		if (copy_to_user(karg->rep_msg_buf_ptr,
+				 &adapter->driver_cmds.ctl_cmd.reply,
+				 sz)) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Failed to copy reply to user\n",
+				__func__);
+			rc = -ENODATA;
+			goto out_cleanup;
+		}
+	}
+
+	if (karg->max_sense_bytes &&
+	    (leap_mpi_req->func == LEAPRAID_FUNC_SCSIIO ||
+	     leap_mpi_req->func == LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH)) {
+		if (!karg->sense_data_ptr)
+			goto out_cleanup;
+
+		sz = min_t(u32, karg->max_sense_bytes, SCSI_SENSE_BUFFERSIZE);
+		if (copy_to_user(karg->sense_data_ptr,
+				 &adapter->driver_cmds.ctl_cmd.sense,
+				 sz)) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Failed to copy sense data to user\n",
+				__func__);
+			rc = -ENODATA;
+			goto out_cleanup;
+		}
+	}
+reset:
+	if (issue_reset) {
+		rc = -ENODATA;
+		if (leap_mpi_req->func == LEAPRAID_FUNC_SCSIIO ||
+		    leap_mpi_req->func ==
+		    LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH ||
+		    leap_mpi_req->func ==
+		    LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH) {
+			dev_err(&adapter->pdev->dev,
+				"Fire tgt reset, hdl=0x%04x\n",
+				le16_to_cpu(leap_mpi_req->func_dep1));
+			leapraid_issue_locked_tm(
+				adapter,
+				le16_to_cpu(leap_mpi_req->func_dep1), 0, 0, 0,
+				LEAPRAID_TM_TASKTYPE_TARGET_RESET, taskid,
+				LEAPRAID_TM_MSGFLAGS_LINK_RESET);
+		} else {
+			dev_info(&adapter->pdev->dev,
+				 "%s:%d: call hard_reset\n",
+				 __func__, __LINE__);
+			leapraid_hard_reset_handler(adapter, FULL_RESET);
+		}
+	}
+out_cleanup:
+	if (c2h)
+		dma_free_coherent(&adapter->pdev->dev, c2h_size,
+				  c2h, c2h_dma_addr);
+	if (h2c)
+		dma_free_coherent(&adapter->pdev->dev, h2c_size,
+				  h2c, h2c_dma_addr);
+	kfree(leap_mpi_req);
+	adapter->driver_cmds.ctl_cmd.status = LEAPRAID_CMD_NOT_USED;
+	return rc;
+}
+
+static int leapraid_ctl_get_adapter_info(struct leapraid_adapter *adapter,
+					 void __user *arg)
+{
+	struct leapraid_ioctl_adapter_info *karg;
+	u8 revision;
+	int rc = 0;
+
+	karg = kzalloc_obj(*karg);
+	if (!karg)
+		return -ENOMEM;
+
+	pci_read_config_byte(adapter->pdev, PCI_CLASS_REVISION, &revision);
+	karg->revision = revision;
+	karg->pci_id = adapter->pdev->device;
+	karg->sub_dev = adapter->pdev->subsystem_device;
+	karg->sub_vendor = adapter->pdev->subsystem_vendor;
+	karg->pci_info.u.bits.bus = adapter->pdev->bus->number;
+	karg->pci_info.u.bits.dev = PCI_SLOT(adapter->pdev->devfn);
+	karg->pci_info.u.bits.func = PCI_FUNC(adapter->pdev->devfn);
+	karg->pci_info.seg_id = pci_domain_nr(adapter->pdev->bus);
+	karg->fw_ver = adapter->adapter_attr.features.fw_version;
+
+	snprintf(karg->driver_ver, sizeof(karg->driver_ver), "%s-%s",
+		 LEAPRAID_DRIVER_NAME, LEAPRAID_DRIVER_VERSION);
+
+	karg->adapter_type = LEAPRAID_IOCTL_VERSION;
+	karg->bios_ver = adapter->adapter_attr.bios_version;
+	if (copy_to_user(arg, karg,
+			 sizeof(struct leapraid_ioctl_adapter_info))) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to copy info to user\n", __func__);
+		rc = -EFAULT;
+		goto free_buf;
+	}
+
+free_buf:
+	kfree(karg);
+	return rc;
+}
+
+static int leapraid_ctl_ioctl_main(struct file *file, unsigned int cmd,
+				   void __user *arg)
+{
+	struct leapraid_ioctl_header ioctl_header;
+	struct leapraid_adapter *adapter = NULL;
+	struct leapraid_ioctl_command __user *uarg;
+	struct leapraid_ioctl_command karg;
+	int rc = -ENOIOCTLCMD;
+
+	if (copy_from_user(&ioctl_header, arg,
+			   sizeof(struct leapraid_ioctl_header))) {
+		pr_err("%s:%s: Failed to copy header from user\n",
+		       LEAPRAID_DRIVER_NAME, __func__);
+		return -EFAULT;
+	}
+
+	adapter = leapraid_ctl_lookup_adapter(ioctl_header.adapter_id, false);
+	if (!adapter)
+		return -EFAULT;
+
+	if (atomic_read(&adapter->overheat_desc.thermal_alert)) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed, thermal_alert=%d\n",
+			 __func__,
+			 atomic_read(&adapter->overheat_desc.thermal_alert));
+		rc = -EFAULT;
+		goto out_put;
+	}
+
+	mutex_lock(&adapter->access_ctrl.pci_access_lock);
+
+	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_LONG,
+					  __func__);
+	if (rc)
+		goto unlock;
+
+	if (!wait_event_timeout(adapter->access_ctrl.shost_recover_wq,
+				!adapter->access_ctrl.shost_recover_async,
+				LEAPRAID_WAIT_SHOST_RECOVERY * HZ)) {
+		dev_warn(&adapter->pdev->dev,
+			 "Timeout waiting for shost recovery async\n");
+		rc = -EAGAIN;
+		goto unlock;
+	}
+
+	if (adapter->access_ctrl.pcie_recovering ||
+	    adapter->scan_dev_desc.driver_loading ||
+	    adapter->access_ctrl.host_removing) {
+		rc = -EAGAIN;
+		goto unlock;
+	}
+
+	if (file->f_flags & O_NONBLOCK) {
+		if (!mutex_trylock(&adapter->driver_cmds.ctl_cmd.mutex)) {
+			rc = -EAGAIN;
+			goto unlock;
+		}
+	} else if (mutex_lock_interruptible(&adapter->driver_cmds
+					    .ctl_cmd.mutex)) {
+		rc = -ERESTARTSYS;
+		goto unlock;
+	}
+
+	switch (_IOC_NR(cmd)) {
+	case LEAPRAID_ADAPTER_INFO:
+		if (_IOC_SIZE(cmd) ==
+		    sizeof(struct leapraid_ioctl_adapter_info))
+			rc = leapraid_ctl_get_adapter_info(adapter, arg);
+		break;
+	case LEAPRAID_COMMAND:
+		if (copy_from_user(&karg, arg, sizeof(karg))) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Failed to copy data from user\n",
+				__func__);
+			rc = -EFAULT;
+			break;
+		}
+
+		if (karg.hdr.adapter_id != ioctl_header.adapter_id) {
+			rc = -EINVAL;
+			break;
+		}
+
+		if (_IOC_SIZE(cmd) == sizeof(struct leapraid_ioctl_command)) {
+			uarg = arg;
+			rc = leapraid_ctl_do_command(adapter, &karg,
+						     &uarg->mf);
+			if (rc)
+				dev_warn(&adapter->pdev->dev,
+					 "%s: IOCTL cmd failed rc=%d\n",
+					 __func__, rc);
+		}
+		break;
+	case LEAPRAID_EVENTQUERY:
+	case LEAPRAID_EVENTREPORT:
+		rc = 0;
+		break;
+	default:
+		dev_err(&adapter->pdev->dev,
+			"Unknown IOCTL opcode=0x%08x\n", cmd);
+		break;
+	}
+	mutex_unlock(&adapter->driver_cmds.ctl_cmd.mutex);
+
+unlock:
+	mutex_unlock(&adapter->access_ctrl.pci_access_lock);
+out_put:
+	leapraid_ctl_put_adapter(adapter);
+	return rc;
+}
+
+static long leapraid_ctl_ioctl(struct file *file, unsigned int cmd,
+			       unsigned long arg)
+{
+	return leapraid_ctl_ioctl_main(file, cmd, (void __user *)arg);
+}
+
+static void leapraid_fw_mmap_open(struct vm_area_struct *vma)
+{
+	struct leapraid_adapter *adapter = vma->vm_private_data;
+
+	if (!adapter)
+		return;
+
+	get_device(&adapter->shost->shost_gendev);
+	atomic_inc(&adapter->fw_log_desc.mmap_refcnt);
+}
+
+static void leapraid_fw_mmap_close(struct vm_area_struct *vma)
+{
+	struct leapraid_adapter *adapter = vma->vm_private_data;
+
+	if (!adapter)
+		return;
+
+	if (atomic_dec_and_test(&adapter->fw_log_desc.mmap_refcnt))
+		wake_up(&adapter->fw_log_desc.mmap_waitq);
+	leapraid_ctl_put_adapter(adapter);
+}
+
+static const struct vm_operations_struct leapraid_fw_mmap_vm_ops = {
+	.open = leapraid_fw_mmap_open,
+	.close = leapraid_fw_mmap_close,
+};
+
+static int leapraid_fw_mmap(struct file *filp, struct vm_area_struct *vma)
+{
+	struct leapraid_adapter *adapter = NULL;
+	/* Userspace passes the adapter ID via vma->vm_pgoff. */
+	u32 adapter_id = vma->vm_pgoff;
+	unsigned long length;
+	int rc = -EINVAL;
+
+	length = vma->vm_end - vma->vm_start;
+
+	adapter = leapraid_ctl_lookup_adapter(adapter_id, true);
+	if (!adapter) {
+		pr_err("%s: No adapter found!\n", __func__);
+		return -EINVAL;
+	}
+
+	if (READ_ONCE(adapter->access_ctrl.host_removing)) {
+		rc = -EAGAIN;
+		goto out_put;
+	}
+
+	if (length > (LEAPRAID_SYS_LOG_BUF_SIZE +
+		      LEAPRAID_SYS_LOG_BUF_RESERVE)) {
+		dev_err(&adapter->pdev->dev,
+			"Requested mapping size is too large!\n");
+		rc = -EINVAL;
+		goto out_put;
+	}
+
+	if (!adapter->fw_log_desc.fw_log_buffer) {
+		dev_err(&adapter->pdev->dev, "No log buffer!\n");
+		rc = -EINVAL;
+		goto out_put;
+	}
+
+	vma->vm_pgoff = 0;
+
+	rc = dma_mmap_coherent(&adapter->pdev->dev, vma,
+			       adapter->fw_log_desc.fw_log_buffer,
+			       adapter->fw_log_desc.fw_log_buffer_dma,
+			       length);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"Failed to map memory to user space!\n");
+		goto out_put;
+	}
+
+	vma->vm_private_data = adapter;
+	vma->vm_ops = &leapraid_fw_mmap_vm_ops;
+	leapraid_fw_mmap_open(vma);
+
+	rc = 0;
+out_put:
+	if (adapter &&
+	    atomic_dec_and_test(&adapter->fw_log_desc.mmap_refcnt))
+		wake_up(&adapter->fw_log_desc.mmap_waitq);
+	leapraid_ctl_put_adapter(adapter);
+	return rc;
+}
+
+static const struct file_operations leapraid_ctl_fops = {
+	.owner = THIS_MODULE,
+	.unlocked_ioctl = leapraid_ctl_ioctl,
+	.mmap = leapraid_fw_mmap,
+};
+
+static struct miscdevice leapraid_ctl_dev = {
+	.minor = MISC_DYNAMIC_MINOR,
+	.name = LEAPRAID_DEV_NAME,
+	.fops = &leapraid_ctl_fops,
+};
+
+int leapraid_ctl_init(void)
+{
+	if (misc_register(&leapraid_ctl_dev) < 0) {
+		pr_err("%s Can't register misc device\n",
+		       LEAPRAID_DRIVER_NAME);
+		return -ENODEV;
+	}
+	return 0;
+}
+
+void leapraid_ctl_exit(void)
+{
+	misc_deregister(&leapraid_ctl_dev);
+}
diff --git a/drivers/scsi/leapraid/leapraid_func.c b/drivers/scsi/leapraid/leapraid_func.c
new file mode 100644
index 000000000..66b43c1c7
--- /dev/null
+++ b/drivers/scsi/leapraid/leapraid_func.c
@@ -0,0 +1,8616 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 LeapIO Tech Inc.
+ *
+ * LeapRAID storage and RAID controller driver.
+ */
+#include <linux/module.h>
+
+#include "leapraid_func.h"
+
+static int poll_queues;
+module_param(poll_queues, int, 0444);
+MODULE_PARM_DESC(poll_queues,
+		 "specifies the number of queues for io_uring poll mode.");
+
+static int max_msix_vectors = LEAPRAID_INVALID_MSIX_VECTORS;
+module_param(max_msix_vectors, int, 0444);
+MODULE_PARM_DESC(max_msix_vectors, "max msix vectors");
+
+static void leapraid_remove_device(struct leapraid_adapter *adapter,
+				   struct leapraid_sas_dev *sas_dev);
+static void leapraid_set_led(struct leapraid_adapter *adapter,
+			     struct leapraid_sas_dev *sas_dev, bool on);
+static void leapraid_ublk_io_dev(struct leapraid_adapter *adapter,
+				 u64 sas_address,
+				 struct leapraid_card_port *port);
+static void leapraid_clear_cached_boot_dev(struct leapraid_adapter *adapter,
+					   void *dev, u32 chnl);
+static int leapraid_make_adapter_available(struct leapraid_adapter *adapter);
+static void leapraid_sync_irqs_for_cleanup(struct leapraid_adapter *adapter);
+static int leapraid_fw_log_init(struct leapraid_adapter *adapter);
+static bool leapraid_should_skip_poll_work(struct leapraid_adapter *adapter);
+static int leapraid_make_adapter_ready(struct leapraid_adapter *adapter,
+				       enum reset_type type);
+
+static noinline bool leapraid_shost_in_recovery(struct Scsi_Host *shost)
+{
+	enum scsi_host_state state;
+
+	state = scsi_get_host_state(shost);
+	return state == SHOST_RECOVERY ||
+	       state == SHOST_CANCEL_RECOVERY ||
+	       state == SHOST_DEL_RECOVERY ||
+	       shost->tmf_in_progress;
+}
+
+static inline bool leapraid_is_end_dev(u32 dev_type)
+{
+	return (dev_type & LEAPRAID_DEVTYP_END_DEV) &&
+		((dev_type & LEAPRAID_DEVTYP_SSP_TGT) ||
+		 (dev_type & LEAPRAID_DEVTYP_STP_TGT) ||
+		 (dev_type & LEAPRAID_DEVTYP_SATA_DEV));
+}
+
+bool leapraid_pci_removed(struct leapraid_adapter *adapter)
+{
+	struct pci_dev *pdev = adapter->pdev;
+	u32 vendor_id;
+
+	if (pci_bus_read_config_dword(pdev->bus,
+				      pdev->devfn,
+				      PCI_VENDOR_ID,
+				      &vendor_id))
+		return true;
+
+	return (vendor_id & LEAPRAID_PCI_VENDOR_ID_MASK) !=
+		LEAPRAID_VENDOR_ID;
+}
+
+static bool leapraid_pci_active(struct leapraid_adapter *adapter)
+{
+	return !(adapter->access_ctrl.pcie_recovering ||
+		leapraid_pci_removed(adapter));
+}
+
+void *leapraid_get_reply_vaddr(struct leapraid_adapter *adapter, u32 rep_paddr)
+{
+	if (!rep_paddr)
+		return NULL;
+
+	return adapter->mem_desc.rep_msg +
+		(rep_paddr - (u32)adapter->mem_desc.rep_msg_dma);
+}
+
+void *leapraid_get_task_desc(struct leapraid_adapter *adapter, u16 taskid)
+{
+	return adapter->mem_desc.task_desc +
+			taskid * LEAPRAID_REQUEST_SIZE;
+}
+
+void *leapraid_get_sense_buffer(struct leapraid_adapter *adapter, u16 taskid)
+{
+	return adapter->mem_desc.sense_data +
+			(taskid - 1) * SCSI_SENSE_BUFFERSIZE;
+}
+
+__le32 leapraid_get_sense_buffer_dma(struct leapraid_adapter *adapter,
+				     u16 taskid)
+{
+	return cpu_to_le32(adapter->mem_desc.sense_data_dma +
+			   ((taskid - 1) * SCSI_SENSE_BUFFERSIZE));
+}
+
+void leapraid_mask_int(struct leapraid_adapter *adapter)
+{
+	u32 reg;
+
+	adapter->mask_int = 1;
+	reg = leapraid_readl(&adapter->iomem_base->host_int_mask);
+	reg |= LEAPRAID_TO_SYS_DB_MASK + LEAPRAID_REPLY_INT_MASK +
+		LEAPRAID_RESET_IRQ_MASK;
+	writel(reg, &adapter->iomem_base->host_int_mask);
+	leapraid_readl(&adapter->iomem_base->host_int_mask);
+}
+
+void leapraid_unmask_int(struct leapraid_adapter *adapter)
+{
+	u32 reg;
+
+	reg = leapraid_readl(&adapter->iomem_base->host_int_mask);
+	reg &= ~LEAPRAID_REPLY_INT_MASK;
+	writel(reg, &adapter->iomem_base->host_int_mask);
+	adapter->mask_int = 0;
+}
+
+static void leapraid_overheat_suspend(struct leapraid_adapter *adapter)
+{
+	struct workqueue_struct *wq;
+	struct Scsi_Host *shost;
+	struct pci_dev *pdev;
+
+	if (!adapter)
+		return;
+
+	pdev = adapter->pdev;
+	shost = pci_get_drvdata(pdev);
+	if (!shost) {
+		dev_warn(&pdev->dev,
+			 "Overheat suspend failed, invalid host or adapter\n");
+		return;
+	}
+
+	wq = adapter->overheat_desc.fault_overheat_wq;
+	if (!wq)
+		return;
+
+	if (atomic_cmpxchg(&adapter->overheat_desc.thermal_alert, 0, 1))
+		return;
+
+	queue_work(wq, &adapter->overheat_desc.fault_overheat_work);
+}
+
+static void leapraid_stop_adapter_on_fault(struct leapraid_adapter *adapter,
+					   u32 db)
+{
+	u32 adapter_state = db & LEAPRAID_DB_MASK;
+	bool fault_1 = false;
+	bool fault_2 = false;
+
+	fault_1 = adapter_state == LEAPRAID_DB_MASK;
+	fault_2 = adapter_state == LEAPRAID_DB_FAULT &&
+		  (db & LEAPRAID_DB_DATA_MASK) == LEAPRAID_DB_OVER_TEMPERATURE;
+
+	if (!fault_1 && !fault_2)
+		return;
+
+	if (fault_1)
+		dev_err(&adapter->pdev->dev,
+			"%s: Doorbell status 0xFFFF!\n", __func__);
+	else
+		dev_err(&adapter->pdev->dev,
+			"%s: Adapter overheating detected!\n", __func__);
+
+	leapraid_overheat_suspend(adapter);
+}
+
+u32 leapraid_get_adapter_state(struct leapraid_adapter *adapter)
+{
+	u32 db;
+	u32 adapter_state;
+
+	db = leapraid_readl(&adapter->iomem_base->db);
+	adapter_state = db & LEAPRAID_DB_MASK;
+	leapraid_stop_adapter_on_fault(adapter, db);
+	return adapter_state;
+}
+
+static bool leapraid_wait_adapter_ready(struct leapraid_adapter *adapter)
+{
+	u32 cur_state;
+	u32 cnt;
+
+	for (cnt = LEAPRAID_ADAPTER_READY_MAX_RETRY; cnt > 0; cnt--) {
+		cur_state = leapraid_get_adapter_state(adapter);
+
+		if (cur_state == LEAPRAID_DB_READY)
+			return true;
+		if (cur_state == LEAPRAID_DB_FAULT)
+			break;
+		usleep_range(LEAPRAID_ADAPTER_READY_SLEEP_MIN_US,
+			     LEAPRAID_ADAPTER_READY_SLEEP_MAX_US);
+	}
+
+	return false;
+}
+
+static int leapraid_db_wait_int_host(struct leapraid_adapter *adapter)
+{
+	u32 cnt;
+
+	for (cnt = LEAPRAID_DB_WAIT_MAX_RETRY; cnt > 0; cnt--) {
+		if (leapraid_readl(&adapter->iomem_base->host_int_status) &
+		    LEAPRAID_ADAPTER2HOST_DB_STATUS)
+			return 0;
+
+		udelay(LEAPRAID_DB_WAIT_DELAY_US);
+	}
+
+	return -EFAULT;
+}
+
+static int leapraid_db_wait_ack_and_clear_int(struct leapraid_adapter *adapter)
+{
+	u32 adapter_state;
+	u32 int_status;
+	u32 cnt;
+
+	for (cnt = LEAPRAID_ADAPTER_READY_MAX_RETRY; cnt > 0; cnt--) {
+		int_status =
+			leapraid_readl(&adapter->iomem_base->host_int_status);
+
+		if (int_status == 0xFFFFFFFF)
+			return -EFAULT;
+
+		if (!(int_status & LEAPRAID_HOST2ADAPTER_DB_STATUS))
+			return 0;
+
+		if (int_status & LEAPRAID_ADAPTER2HOST_DB_STATUS) {
+			adapter_state = leapraid_get_adapter_state(adapter);
+			if (adapter_state == LEAPRAID_DB_FAULT)
+				return -EFAULT;
+		}
+
+		usleep_range(LEAPRAID_ADAPTER_READY_SLEEP_MIN_US,
+			     LEAPRAID_ADAPTER_READY_SLEEP_MAX_US);
+	}
+
+	return -EFAULT;
+}
+
+static int leapraid_handshake_func(struct leapraid_adapter *adapter,
+				   int req_bytes, u32 *req,
+				   int rep_bytes, u16 *rep)
+{
+	int failed, i;
+
+	if (leapraid_readl(&adapter->iomem_base->db) & LEAPRAID_DB_USED) {
+		dev_err(&adapter->pdev->dev, "doorbell used\n");
+		return -EFAULT;
+	}
+
+	if (leapraid_readl(&adapter->iomem_base->host_int_status) &
+	    LEAPRAID_ADAPTER2HOST_DB_STATUS)
+		writel(0, &adapter->iomem_base->host_int_status);
+
+	writel(((LEAPRAID_FUNC_HANDSHAKE << LEAPRAID_DB_FUNC_SHIFT) |
+		((req_bytes / LEAPRAID_DWORDS_BYTE_SIZE) <<
+		LEAPRAID_DB_ADD_DWORDS_SHIFT)),
+	       &adapter->iomem_base->db);
+
+	if (leapraid_db_wait_int_host(adapter)) {
+		dev_err(&adapter->pdev->dev, "%d: Wait db interrupt timeout\n",
+			__LINE__);
+		return -EFAULT;
+	}
+
+	writel(0, &adapter->iomem_base->host_int_status);
+
+	if (leapraid_db_wait_ack_and_clear_int(adapter)) {
+		dev_err(&adapter->pdev->dev, "%d: Wait ack failure\n",
+			__LINE__);
+		return -EFAULT;
+	}
+
+	for (i = 0, failed = 0;
+	     i < req_bytes / LEAPRAID_DWORDS_BYTE_SIZE && !failed;
+	     i++) {
+		writel((u32)req[i], &adapter->iomem_base->db);
+		if (leapraid_db_wait_ack_and_clear_int(adapter))
+			failed = 1;
+	}
+	if (failed) {
+		dev_err(&adapter->pdev->dev, "%d: Wait ack failure\n",
+			__LINE__);
+		return -EFAULT;
+	}
+
+	for (i = 0; i < rep_bytes / LEAPRAID_WORD_BYTE_SIZE; i++) {
+		if (leapraid_db_wait_int_host(adapter)) {
+			dev_err(&adapter->pdev->dev,
+				"%d: Wait db interrupt timeout\n", __LINE__);
+			return -EFAULT;
+		}
+		rep[i] = (u16)(leapraid_readl(&adapter->iomem_base->db)
+			       & LEAPRAID_DB_DATA_MASK);
+		writel(0, &adapter->iomem_base->host_int_status);
+	}
+
+	if (leapraid_db_wait_int_host(adapter)) {
+		dev_err(&adapter->pdev->dev, "%d: Wait db interrupt timeout\n",
+			__LINE__);
+		return -EFAULT;
+	}
+
+	writel(0, &adapter->iomem_base->host_int_status);
+
+	return 0;
+}
+
+int leapraid_check_adapter_is_op(struct leapraid_adapter *adapter, int wait,
+				 const char *caller)
+{
+	int wait_count;
+
+	for (wait_count = wait; wait_count > 0; wait_count--) {
+		if (READ_ONCE(adapter->access_ctrl.host_removing)) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Host is removing\n", caller);
+			return -EFAULT;
+		}
+
+		if (leapraid_pci_removed(adapter)) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: PCI device removed\n", __func__);
+			return -EFAULT;
+		}
+
+		if (leapraid_get_adapter_state(adapter) ==
+		    LEAPRAID_DB_OPERATIONAL)
+			return 0;
+
+		dev_dbg(&adapter->pdev->dev,
+			"%s: Wait for adapter to become op status(cnt=%d)\n",
+			caller, wait - wait_count);
+
+		ssleep(1);
+	}
+
+	dev_err(&adapter->pdev->dev,
+		"%s: Adapter failed to become op state, last state=%d\n",
+		caller, leapraid_get_adapter_state(adapter));
+
+	return -EFAULT;
+}
+
+struct leapraid_io_req_tracker *leapraid_get_io_tracker_from_taskid(
+		struct leapraid_adapter *adapter, u16 taskid)
+{
+	struct scsi_cmnd *scmd;
+
+	if (WARN_ON(!taskid))
+		return NULL;
+
+	if (WARN_ON(taskid > adapter->shost->can_queue))
+		return NULL;
+
+	scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
+	if (scmd)
+		return scsi_cmd_priv(scmd);
+
+	return NULL;
+}
+
+static u8 leapraid_get_cb_idx(struct leapraid_adapter *adapter, u16 taskid)
+{
+	struct leapraid_driver_cmd *sp_cmd;
+	u8 cb_idx = 0xFF;
+
+	if (WARN_ON(!taskid))
+		return cb_idx;
+
+	list_for_each_entry(sp_cmd, &adapter->driver_cmds.special_cmd_list,
+			    list)
+		if (taskid == sp_cmd->taskid ||
+		    taskid == sp_cmd->hp_taskid ||
+		    taskid == sp_cmd->inter_taskid)
+			return sp_cmd->cb_idx;
+
+	WARN_ON(cb_idx == 0xFF);
+	return cb_idx;
+}
+
+struct scsi_cmnd *leapraid_get_scmd_from_taskid(
+					struct leapraid_adapter *adapter,
+					u16 taskid)
+{
+	struct leapraid_scsiio_req *leap_mpi_req;
+	struct leapraid_io_req_tracker *st;
+	struct scsi_cmnd *scmd;
+	u32 uniq_tag;
+
+	if (taskid <= 0 || taskid > adapter->shost->can_queue)
+		return NULL;
+
+	uniq_tag = adapter->mem_desc.taskid_to_uniq_tag[taskid - 1] <<
+		 BLK_MQ_UNIQUE_TAG_BITS | (taskid - 1);
+	leap_mpi_req = leapraid_get_task_desc(adapter, taskid);
+	if (!leap_mpi_req->dev_hdl)
+		return NULL;
+
+	scmd = scsi_host_find_tag(adapter->shost, uniq_tag);
+	if (scmd) {
+		st = scsi_cmd_priv(scmd);
+		if (st && st->taskid == taskid)
+			return scmd;
+	}
+
+	return NULL;
+}
+
+u16 leapraid_alloc_scsiio_taskid(struct leapraid_adapter *adapter,
+				 struct scsi_cmnd *scmd)
+{
+	struct leapraid_io_req_tracker *request;
+	u16 taskid;
+	u32 tag = scsi_cmd_to_rq(scmd)->tag;
+	u32 unique_tag;
+
+	unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
+	tag = blk_mq_unique_tag_to_tag(unique_tag);
+	adapter->mem_desc.taskid_to_uniq_tag[tag] =
+		blk_mq_unique_tag_to_hwq(unique_tag);
+
+	request = scsi_cmd_priv(scmd);
+	taskid = tag + 1;
+	request->taskid = taskid;
+	request->scmd = scmd;
+	return taskid;
+}
+
+static void leapraid_check_pending_io(struct leapraid_adapter *adapter)
+{
+	if (adapter->access_ctrl.shost_recovering &&
+	    adapter->reset_desc.pending_io_cnt) {
+		if (adapter->reset_desc.pending_io_cnt == 1)
+			wake_up(&adapter->reset_desc.reset_wait_queue);
+		adapter->reset_desc.pending_io_cnt--;
+	}
+}
+
+static void leapraid_clear_io_tracker(
+		struct leapraid_adapter *adapter,
+		struct leapraid_io_req_tracker *io_tracker)
+{
+	if (!io_tracker)
+		return;
+
+	if (WARN_ON(io_tracker->taskid == 0))
+		return;
+
+	io_tracker->scmd = NULL;
+}
+
+static bool leapraid_is_fixed_taskid(struct leapraid_adapter *adapter,
+				     u16 taskid)
+{
+	struct leapraid_driver_cmds *driver_cmds = &adapter->driver_cmds;
+
+	return (taskid == driver_cmds->ctl_cmd.taskid ||
+		taskid == driver_cmds->tm_cmd.hp_taskid ||
+		taskid == driver_cmds->ctl_cmd.hp_taskid ||
+		taskid == driver_cmds->scan_dev_cmd.inter_taskid ||
+		taskid == driver_cmds->transport_cmd.inter_taskid ||
+		taskid == driver_cmds->cfg_op_cmd.inter_taskid ||
+		taskid == driver_cmds->enc_cmd.inter_taskid ||
+		taskid == driver_cmds->notify_event_cmd.inter_taskid);
+}
+
+void leapraid_free_taskid(struct leapraid_adapter *adapter, u16 taskid)
+{
+	struct leapraid_io_req_tracker *io_tracker;
+	void *task_desc;
+
+	if (leapraid_is_fixed_taskid(adapter, taskid))
+		return;
+
+	if (taskid <= adapter->shost->can_queue) {
+		io_tracker = leapraid_get_io_tracker_from_taskid(adapter,
+								 taskid);
+		if (!io_tracker) {
+			leapraid_check_pending_io(adapter);
+			return;
+		}
+
+		task_desc = leapraid_get_task_desc(adapter, taskid);
+		memset(task_desc, 0, LEAPRAID_REQUEST_SIZE);
+		leapraid_clear_io_tracker(adapter, io_tracker);
+		leapraid_check_pending_io(adapter);
+		adapter->mem_desc.taskid_to_uniq_tag[taskid - 1] = 0xFFFF;
+	}
+}
+
+static u8 leapraid_get_msix_idx(struct leapraid_adapter *adapter,
+				struct scsi_cmnd *scmd)
+{
+	if (scmd && adapter->shost->nr_hw_queues > 1) {
+		u32 tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
+
+		return blk_mq_unique_tag_to_hwq(tag);
+	}
+	return adapter->notification_desc.msix_cpu_map[raw_smp_processor_id()];
+}
+
+static u8 leapraid_get_and_set_msix_idx_from_taskid(
+		struct leapraid_adapter *adapter, u16 taskid)
+{
+	struct leapraid_io_req_tracker *io_tracker = NULL;
+
+	if (taskid <= adapter->shost->can_queue)
+		io_tracker = leapraid_get_io_tracker_from_taskid(adapter,
+								 taskid);
+
+	if (!io_tracker)
+		return leapraid_get_msix_idx(adapter, NULL);
+
+	io_tracker->msix_io = leapraid_get_msix_idx(adapter, io_tracker->scmd);
+
+	return io_tracker->msix_io;
+}
+
+void leapraid_fire_scsi_io(struct leapraid_adapter *adapter, u16 taskid,
+			   u16 handle)
+{
+	struct leapraid_atomic_req_desc desc;
+
+	desc.flg = LEAPRAID_REQ_DESC_FLG_SCSI_IO;
+	desc.msix_idx = leapraid_get_and_set_msix_idx_from_taskid(adapter,
+								  taskid);
+	desc.taskid = cpu_to_le16(taskid);
+	writel((__force u32)cpu_to_le32(*((u32 *)&desc)),
+	       &adapter->iomem_base->atomic_req_desc_post);
+}
+
+void leapraid_fire_hpr_task(struct leapraid_adapter *adapter, u16 taskid,
+			    u16 msix_task)
+{
+	struct leapraid_atomic_req_desc desc;
+
+	desc.flg = LEAPRAID_REQ_DESC_FLG_HPR;
+	desc.msix_idx = msix_task;
+	desc.taskid = cpu_to_le16(taskid);
+	writel((__force u32)cpu_to_le32(*((u32 *)&desc)),
+	       &adapter->iomem_base->atomic_req_desc_post);
+}
+
+void leapraid_fire_task(struct leapraid_adapter *adapter, u16 taskid)
+{
+	struct leapraid_atomic_req_desc desc;
+
+	desc.flg = LEAPRAID_REQ_DESC_FLG_DFLT_TYPE;
+	desc.msix_idx = leapraid_get_and_set_msix_idx_from_taskid(adapter,
+								  taskid);
+	desc.taskid = cpu_to_le16(taskid);
+	writel((__force u32)cpu_to_le32(*((u32 *)&desc)),
+	       &adapter->iomem_base->atomic_req_desc_post);
+}
+
+void leapraid_clean_active_scsi_cmds(struct leapraid_adapter *adapter)
+{
+	struct leapraid_io_req_tracker *io_tracker;
+	struct scsi_cmnd *scmd;
+	void *task_desc;
+	u16 taskid;
+
+	leapraid_sync_irqs_for_cleanup(adapter);
+
+	for (taskid = 1; taskid <= adapter->shost->can_queue; taskid++) {
+		scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
+		if (!scmd)
+			continue;
+
+		io_tracker = scsi_cmd_priv(scmd);
+		if (io_tracker && io_tracker->taskid == 0)
+			continue;
+
+		scsi_dma_unmap(scmd);
+		task_desc = leapraid_get_task_desc(adapter, taskid);
+		memset(task_desc, 0, LEAPRAID_REQUEST_SIZE);
+		leapraid_clear_io_tracker(adapter, io_tracker);
+		if (!leapraid_pci_active(adapter) ||
+		    adapter->reset_desc.adapter_reset_results != 0 ||
+		    atomic_read(&adapter->overheat_desc.thermal_alert) ||
+		    adapter->access_ctrl.host_removing)
+			scmd->result = DID_NO_CONNECT << 16;
+		else
+			scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+		scsi_done(scmd);
+	}
+}
+
+static void leapraid_clean_active_driver_cmd(struct leapraid_driver_cmd *cmd)
+{
+	if (cmd->status & LEAPRAID_CMD_PENDING) {
+		cmd->status |= LEAPRAID_CMD_RESET;
+		complete(&cmd->done);
+	}
+}
+
+static void leapraid_clean_active_driver_cmds(struct leapraid_adapter *adapter)
+{
+	struct leapraid_driver_cmds *driver_cmds;
+
+	driver_cmds = &adapter->driver_cmds;
+	leapraid_clean_active_driver_cmd(&driver_cmds->tm_cmd);
+	leapraid_clean_active_driver_cmd(&driver_cmds->transport_cmd);
+	leapraid_clean_active_driver_cmd(&driver_cmds->enc_cmd);
+	leapraid_clean_active_driver_cmd(&driver_cmds->notify_event_cmd);
+	leapraid_clean_active_driver_cmd(&driver_cmds->cfg_op_cmd);
+	leapraid_clean_active_driver_cmd(&driver_cmds->ctl_cmd);
+
+	if (driver_cmds->scan_dev_cmd.status & LEAPRAID_CMD_PENDING) {
+		adapter->scan_dev_desc.scan_dev_failed = 1;
+		driver_cmds->scan_dev_cmd.status |= LEAPRAID_CMD_RESET;
+		if (adapter->scan_dev_desc.driver_loading) {
+			adapter->scan_dev_desc.scan_start_failed =
+				LEAPRAID_ADAPTER_STATUS_INTERNAL_ERROR;
+			adapter->scan_dev_desc.scan_start = 0;
+		} else {
+			complete(&driver_cmds->scan_dev_cmd.done);
+		}
+	}
+}
+
+static void leapraid_clean_active_cmds(struct leapraid_adapter *adapter)
+{
+	leapraid_clean_active_driver_cmds(adapter);
+	leapraid_clean_active_fw_evt(adapter);
+	leapraid_clean_active_scsi_cmds(adapter);
+}
+
+static void leapraid_tgt_not_responding(struct leapraid_adapter *adapter,
+					u16 hdl)
+{
+	struct leapraid_starget_priv *starget_priv = NULL;
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags = 0;
+	u32 adapter_state;
+
+	if (adapter->access_ctrl.pcie_recovering)
+		return;
+
+	adapter_state = leapraid_get_adapter_state(adapter);
+	if (adapter_state != LEAPRAID_DB_OPERATIONAL)
+		return;
+
+	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle ||
+	    test_bit(hdl, adapter->dev_topo.pd_hdls))
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	if (sas_dev && sas_dev->starget && sas_dev->starget->hostdata) {
+		starget_priv = sas_dev->starget->hostdata;
+		starget_priv->deleted = 1;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	if (starget_priv)
+		starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;
+
+	if (sas_dev)
+		leapraid_sdev_put(sas_dev);
+}
+
+static void leapraid_tgt_rst_send(struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_starget_priv *starget_priv = NULL;
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_card_port *port;
+	u64 sas_address;
+	unsigned long flags;
+	u32 adapter_state;
+
+	if (adapter->access_ctrl.pcie_recovering)
+		return;
+
+	adapter_state = leapraid_get_adapter_state(adapter);
+	if (adapter_state != LEAPRAID_DB_OPERATIONAL)
+		return;
+
+	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle ||
+	    test_bit(hdl, adapter->dev_topo.pd_hdls))
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	if (sas_dev && sas_dev->starget && sas_dev->starget->hostdata) {
+		starget_priv = sas_dev->starget->hostdata;
+		starget_priv->deleted = 1;
+		sas_address = sas_dev->sas_addr;
+		port = sas_dev->card_port;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	if (starget_priv) {
+		leapraid_ublk_io_dev(adapter, sas_address, port);
+		starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;
+	}
+	if (sas_dev)
+		leapraid_sdev_put(sas_dev);
+}
+
+static inline void leapraid_single_mpi_sg_append(
+				struct leapraid_adapter *adapter,
+				void *sge,
+				u32 flag_and_len,
+				dma_addr_t dma_addr)
+{
+	if (adapter->adapter_attr.use_32_dma_mask) {
+		struct leapraid_sge_simple32 *sge32 = sge;
+
+		sge32->flg_and_len =
+			cpu_to_le32(flag_and_len |
+				    (LEAPRAID_SGE_FLG_32 |
+				     LEAPRAID_SGE_FLG_SYSTEM_ADDR) <<
+				    LEAPRAID_SGE_FLG_SHIFT);
+		sge32->addr = cpu_to_le32(dma_addr);
+	} else {
+		struct leapraid_sge_simple64 *sge64 = sge;
+
+		sge64->flg_and_len =
+			cpu_to_le32(flag_and_len |
+				    (LEAPRAID_SGE_FLG_64 |
+				     LEAPRAID_SGE_FLG_SYSTEM_ADDR) <<
+				    LEAPRAID_SGE_FLG_SHIFT);
+		sge64->addr = cpu_to_le64(dma_addr);
+	}
+}
+
+static inline void leapraid_single_ieee_sg_append(void *sge, u8 flag,
+						  u8 next_chain_offset,
+						  u32 len,
+						  dma_addr_t dma_addr)
+{
+	struct leapraid_chain64_ieee_sg *ieee_sg = sge;
+
+	ieee_sg->flg = flag;
+	ieee_sg->next_chain_offset = next_chain_offset;
+	ieee_sg->len = cpu_to_le32(len);
+	ieee_sg->addr = cpu_to_le64(dma_addr);
+}
+
+static void leapraid_build_nodata_mpi_sg(struct leapraid_adapter *adapter,
+					 void *sge)
+{
+	leapraid_single_mpi_sg_append(adapter,
+				      sge,
+				      (LEAPRAID_SGE_FLG_LAST_ONE |
+					LEAPRAID_SGE_FLG_EOB |
+					LEAPRAID_SGE_FLG_EOL |
+					LEAPRAID_SGE_FLG_SIMPLE_ONE) <<
+					LEAPRAID_SGE_FLG_SHIFT,
+					LEAPRAID_SGE_NO_DATA_ADDR);
+}
+
+void leapraid_build_mpi_sg(struct leapraid_adapter *adapter, void *sge,
+			   dma_addr_t h2c_dma_addr, size_t h2c_size,
+			   dma_addr_t c2h_dma_addr, size_t c2h_size)
+{
+	if (h2c_size && !c2h_size) {
+		leapraid_single_mpi_sg_append(adapter,
+					      sge,
+					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+						LEAPRAID_SGE_FLG_LAST_ONE |
+						LEAPRAID_SGE_FLG_EOB |
+						LEAPRAID_SGE_FLG_EOL |
+						LEAPRAID_SGE_FLG_H2C) <<
+					       LEAPRAID_SGE_FLG_SHIFT) |
+					       h2c_size,
+					      h2c_dma_addr);
+	} else if (!h2c_size && c2h_size) {
+		leapraid_single_mpi_sg_append(adapter,
+					      sge,
+					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+						LEAPRAID_SGE_FLG_LAST_ONE |
+						LEAPRAID_SGE_FLG_EOB |
+						LEAPRAID_SGE_FLG_EOL) <<
+						LEAPRAID_SGE_FLG_SHIFT) |
+						c2h_size,
+					      c2h_dma_addr);
+	} else if (h2c_size && c2h_size) {
+		leapraid_single_mpi_sg_append(adapter,
+					      sge,
+					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+						LEAPRAID_SGE_FLG_EOB |
+						LEAPRAID_SGE_FLG_H2C) <<
+					       LEAPRAID_SGE_FLG_SHIFT) |
+					      h2c_size,
+					      h2c_dma_addr);
+		if (adapter->adapter_attr.use_32_dma_mask)
+			sge += sizeof(struct leapraid_sge_simple32);
+		else
+			sge += sizeof(struct leapraid_sge_simple64);
+		leapraid_single_mpi_sg_append(adapter,
+					      sge,
+					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+						LEAPRAID_SGE_FLG_LAST_ONE |
+						LEAPRAID_SGE_FLG_EOB |
+						LEAPRAID_SGE_FLG_EOL) <<
+						LEAPRAID_SGE_FLG_SHIFT) |
+						c2h_size,
+					      c2h_dma_addr);
+	} else {
+		leapraid_build_nodata_mpi_sg(adapter, sge);
+	}
+}
+
+void leapraid_build_ieee_nodata_sg(struct leapraid_adapter *adapter, void *sge)
+{
+	leapraid_single_ieee_sg_append(sge,
+				       (LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
+					LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR |
+					LEAPRAID_IEEE_SGE_FLG_EOL),
+					0,
+					0,
+					LEAPRAID_SGE_NO_DATA_ADDR);
+}
+
+int leapraid_build_scmd_ieee_sg(struct leapraid_adapter *adapter,
+				struct scsi_cmnd *scmd, u16 taskid)
+{
+	struct leapraid_scsiio_req *scsiio_req;
+	struct leapraid_io_req_tracker *io_tracker;
+	struct scatterlist *scmd_sg_cur;
+	int sg_entries_left;
+	void *sg_entry_cur;
+	void *host_chain;
+	dma_addr_t host_chain_dma;
+	u8 host_chain_cursor;
+	u32 sg_entries_in_cur_seg;
+	u32 chain_offset_in_cur_seg;
+	u32 chain_len_in_cur_seg;
+
+	io_tracker = scsi_cmd_priv(scmd);
+	scsiio_req = leapraid_get_task_desc(adapter, taskid);
+	scmd_sg_cur = scsi_sglist(scmd);
+	sg_entries_left = scsi_dma_map(scmd);
+	if (sg_entries_left < 0)
+		return -ENOMEM;
+	sg_entry_cur = &scsiio_req->sgl;
+	if (sg_entries_left <= LEAPRAID_SGL_INLINE_THRESHOLD)
+		goto fill_last_seg;
+
+	scsiio_req->chain_offset = LEAPRAID_CHAIN_OFFSET_DWORDS;
+	leapraid_single_ieee_sg_append(sg_entry_cur,
+				       LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
+				       LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR,
+				       0, sg_dma_len(scmd_sg_cur),
+				       sg_dma_address(scmd_sg_cur));
+	scmd_sg_cur = sg_next(scmd_sg_cur);
+	sg_entry_cur += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+	sg_entries_left--;
+
+	host_chain_cursor = 0;
+	host_chain = io_tracker->chain +
+		host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
+	host_chain_dma = io_tracker->chain_dma +
+		host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
+	host_chain_cursor += 1;
+	for (;;) {
+		sg_entries_in_cur_seg =
+			(sg_entries_left <= LEAPRAID_MAX_SGES_IN_CHAIN) ?
+				sg_entries_left : LEAPRAID_MAX_SGES_IN_CHAIN;
+		chain_offset_in_cur_seg =
+			(sg_entries_left == (int)sg_entries_in_cur_seg) ?
+				0 : sg_entries_in_cur_seg;
+		chain_len_in_cur_seg = sg_entries_in_cur_seg *
+			LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+		if (chain_offset_in_cur_seg)
+			chain_len_in_cur_seg += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+
+		leapraid_single_ieee_sg_append(
+			sg_entry_cur,
+			LEAPRAID_IEEE_SGE_FLG_CHAIN_ONE |
+			LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR,
+			chain_offset_in_cur_seg, chain_len_in_cur_seg,
+			host_chain_dma);
+		sg_entry_cur = host_chain;
+		if (!chain_offset_in_cur_seg)
+			goto fill_last_seg;
+
+		while (sg_entries_in_cur_seg) {
+			leapraid_single_ieee_sg_append(
+				sg_entry_cur,
+				LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
+				LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR,
+				0, sg_dma_len(scmd_sg_cur),
+				sg_dma_address(scmd_sg_cur));
+			scmd_sg_cur = sg_next(scmd_sg_cur);
+			sg_entry_cur += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+			sg_entries_left--;
+			sg_entries_in_cur_seg--;
+		}
+		host_chain = io_tracker->chain +
+			host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
+		host_chain_dma = io_tracker->chain_dma +
+			host_chain_cursor * LEAPRAID_CHAIN_SEG_SIZE;
+		host_chain_cursor += 1;
+	}
+
+fill_last_seg:
+	while (sg_entries_left > 0) {
+		u32 flags = LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
+			LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR;
+		if (sg_entries_left == 1)
+			flags |= LEAPRAID_IEEE_SGE_FLG_EOL;
+		leapraid_single_ieee_sg_append(sg_entry_cur, flags,
+					       0, sg_dma_len(scmd_sg_cur),
+					       sg_dma_address(scmd_sg_cur));
+		scmd_sg_cur = sg_next(scmd_sg_cur);
+		sg_entry_cur += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+		sg_entries_left--;
+	}
+	return 0;
+}
+
+void leapraid_build_ieee_sg(struct leapraid_adapter *adapter, void *sge,
+			    dma_addr_t h2c_dma_addr, size_t h2c_size,
+			    dma_addr_t c2h_dma_addr, size_t c2h_size)
+{
+	u32 base_flag;
+	u32 flag;
+
+	base_flag = LEAPRAID_IEEE_SGE_FLG_SIMPLE_ONE |
+			LEAPRAID_IEEE_SGE_FLG_SYSTEM_ADDR;
+
+	if (h2c_size && !c2h_size) {
+		flag = base_flag | LEAPRAID_IEEE_SGE_FLG_EOL;
+
+		leapraid_single_ieee_sg_append(sge,
+					       flag,
+					       0,
+					       h2c_size,
+					       h2c_dma_addr);
+	} else if (!h2c_size && c2h_size) {
+		flag = base_flag | LEAPRAID_IEEE_SGE_FLG_EOL;
+
+		leapraid_single_ieee_sg_append(sge,
+					       flag,
+					       0,
+					       c2h_size,
+					       c2h_dma_addr);
+	} else if (h2c_size && c2h_size) {
+		leapraid_single_ieee_sg_append(sge,
+					       base_flag,
+					       0,
+					       h2c_size,
+					       h2c_dma_addr);
+		sge += LEAPRAID_IEEE_SGE64_ENTRY_SIZE;
+
+		flag = base_flag | LEAPRAID_IEEE_SGE_FLG_EOL;
+		leapraid_single_ieee_sg_append(sge,
+					       flag,
+					       0,
+					       c2h_size,
+					       c2h_dma_addr);
+	} else {
+		leapraid_build_ieee_nodata_sg(adapter, sge);
+	}
+}
+
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_from_tgt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_starget_priv *tgt_priv)
+{
+	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
+	if (tgt_priv->sas_dev)
+		leapraid_sdev_get(tgt_priv->sas_dev);
+
+	return tgt_priv->sas_dev;
+}
+
+struct leapraid_sas_dev *leapraid_get_sas_dev_from_tgt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_starget_priv *tgt_priv)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_from_tgt(adapter, tgt_priv);
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	return sas_dev;
+}
+
+static struct leapraid_card_port *leapraid_get_port_by_id(
+		struct leapraid_adapter *adapter,
+		u8 port_id, bool skip_dirty)
+{
+	struct leapraid_card_port *port;
+	struct leapraid_card_port *dirty_port = NULL;
+
+	if (!adapter->adapter_attr.enable_mp)
+		port_id = LEAPRAID_DISABLE_MP_PORT_ID;
+
+	list_for_each_entry(port, &adapter->dev_topo.card_port_list, list) {
+		if (port->port_id != port_id)
+			continue;
+
+		if (!(port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY))
+			return port;
+
+		if (skip_dirty && !dirty_port)
+			dirty_port = port;
+	}
+
+	if (dirty_port)
+		return dirty_port;
+
+	if (unlikely(!adapter->adapter_attr.enable_mp)) {
+		port = kzalloc_obj(*port, GFP_ATOMIC);
+		if (!port) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Failed to alloc port\n", __func__);
+			return NULL;
+		}
+
+		port->port_id = LEAPRAID_DISABLE_MP_PORT_ID;
+		list_add_tail(&port->list, &adapter->dev_topo.card_port_list);
+		return port;
+	}
+
+	return NULL;
+}
+
+struct leapraid_vphy *leapraid_get_vphy_by_phy(struct leapraid_card_port *port,
+					       u32 phy_seq_num)
+{
+	struct leapraid_vphy *vphy;
+
+	if (!port || !port->vphys_mask)
+		return NULL;
+
+	list_for_each_entry(vphy, &port->vphys_list, list) {
+		if (vphy->phy_mask & BIT(phy_seq_num))
+			return vphy;
+	}
+
+	return NULL;
+}
+
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_addr_and_rphy(
+		struct leapraid_adapter *adapter,
+		u64 sas_address,
+		struct sas_rphy *rphy)
+{
+	struct leapraid_sas_dev *sas_dev;
+
+	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list)
+		if (sas_dev->sas_addr == sas_address &&
+		    sas_dev->rphy == rphy) {
+			leapraid_sdev_get(sas_dev);
+			return sas_dev;
+		}
+
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_init_list,
+			    list)
+		if (sas_dev->sas_addr == sas_address &&
+		    sas_dev->rphy == rphy) {
+			leapraid_sdev_get(sas_dev);
+			return sas_dev;
+		}
+
+	return NULL;
+}
+
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_addr(
+		struct leapraid_adapter *adapter,
+		u64 sas_address, struct leapraid_card_port *port)
+{
+	struct leapraid_sas_dev *sas_dev;
+
+	if (!port) {
+		dev_warn(&adapter->pdev->dev, "%s: Invalid port\n", __func__);
+		return NULL;
+	}
+
+	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list)
+		if (sas_dev->sas_addr == sas_address &&
+		    sas_dev->card_port == port) {
+			leapraid_sdev_get(sas_dev);
+			return sas_dev;
+		}
+
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_init_list,
+			    list)
+		if (sas_dev->sas_addr == sas_address &&
+		    sas_dev->card_port == port) {
+			leapraid_sdev_get(sas_dev);
+			return sas_dev;
+		}
+
+	return NULL;
+}
+
+struct leapraid_sas_dev *leapraid_get_sas_dev_by_addr(
+		struct leapraid_adapter *adapter,
+		u64 sas_address, struct leapraid_card_port *port)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+
+	if (!port) {
+		dev_warn(&adapter->pdev->dev, "%s: Invalid port\n", __func__);
+		return NULL;
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(adapter, sas_address,
+							 port);
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	return sas_dev;
+}
+
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_hdl(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_sas_dev *sas_dev;
+
+	assert_spin_locked(&adapter->dev_topo.sas_dev_lock);
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list)
+		if (sas_dev->hdl == hdl) {
+			leapraid_sdev_get(sas_dev);
+			return sas_dev;
+		}
+
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_init_list,
+			    list)
+		if (sas_dev->hdl == hdl) {
+			leapraid_sdev_get(sas_dev);
+			return sas_dev;
+		}
+
+	return NULL;
+}
+
+struct leapraid_sas_dev *leapraid_get_sas_dev_by_hdl(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	return sas_dev;
+}
+
+void leapraid_sas_dev_remove(struct leapraid_adapter *adapter,
+			     struct leapraid_sas_dev *sas_dev)
+{
+	unsigned long flags;
+	bool del_from_list;
+
+	if (!sas_dev) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid SAS device\n", __func__);
+		return;
+	}
+
+	del_from_list = false;
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	if (!list_empty(&sas_dev->list)) {
+		list_del_init(&sas_dev->list);
+		del_from_list = true;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	if (del_from_list) {
+		leapraid_clear_cached_boot_dev(adapter, sas_dev, 0);
+		leapraid_sdev_put(sas_dev);
+	}
+}
+
+static void leapraid_sas_dev_remove_by_hdl(struct leapraid_adapter *adapter,
+					   u16 hdl)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+	bool del_from_list;
+
+	if (adapter->access_ctrl.shost_recovering)
+		return;
+
+	del_from_list = false;
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	if (sas_dev && (!list_empty(&sas_dev->list))) {
+		list_del_init(&sas_dev->list);
+		del_from_list = true;
+		leapraid_sdev_put(sas_dev);
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	if (del_from_list) {
+		leapraid_remove_device(adapter, sas_dev);
+		leapraid_sdev_put(sas_dev);
+	}
+}
+
+void leapraid_sas_dev_remove_by_sas_address(struct leapraid_adapter *adapter,
+					    u64 sas_address,
+					    struct leapraid_card_port *port)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+	bool del_from_list;
+
+	if (adapter->access_ctrl.shost_recovering)
+		return;
+
+	del_from_list = false;
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(adapter, sas_address,
+							 port);
+	if (sas_dev && (!list_empty(&sas_dev->list))) {
+		list_del_init(&sas_dev->list);
+		del_from_list = true;
+		leapraid_sdev_put(sas_dev);
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	if (del_from_list) {
+		leapraid_remove_device(adapter, sas_dev);
+		leapraid_sdev_put(sas_dev);
+	}
+}
+
+struct leapraid_raid_volume *leapraid_raid_volume_find_by_id(
+		struct leapraid_adapter *adapter, uint id, uint channel)
+{
+	struct leapraid_raid_volume *raid_volume;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	list_for_each_entry(raid_volume, &adapter->dev_topo.raid_volume_list,
+			    list) {
+		if (raid_volume->id == id && raid_volume->channel == channel) {
+			leapraid_raid_volume_get(raid_volume);
+			spin_unlock_irqrestore(
+				&adapter->dev_topo.raid_volume_lock, flags);
+			return raid_volume;
+		}
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	return NULL;
+}
+
+struct leapraid_raid_volume *leapraid_raid_volume_find_by_hdl(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_raid_volume *raid_volume;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	list_for_each_entry(raid_volume, &adapter->dev_topo.raid_volume_list,
+			    list) {
+		if (raid_volume->hdl == hdl) {
+			leapraid_raid_volume_get(raid_volume);
+			spin_unlock_irqrestore(
+				&adapter->dev_topo.raid_volume_lock, flags);
+			return raid_volume;
+		}
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	return NULL;
+}
+
+static struct leapraid_raid_volume *leapraid_raid_volume_find_by_wwid(
+		struct leapraid_adapter *adapter, u64 wwid)
+{
+	struct leapraid_raid_volume *raid_volume;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	list_for_each_entry(raid_volume, &adapter->dev_topo.raid_volume_list,
+			    list) {
+		if (raid_volume->wwid == wwid) {
+			leapraid_raid_volume_get(raid_volume);
+			spin_unlock_irqrestore(
+				&adapter->dev_topo.raid_volume_lock, flags);
+			return raid_volume;
+		}
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	return NULL;
+}
+
+static void leapraid_raid_volume_add(struct leapraid_adapter *adapter,
+				     struct leapraid_raid_volume *raid_volume)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	leapraid_raid_volume_get(raid_volume);
+	list_add_tail(&raid_volume->list, &adapter->dev_topo.raid_volume_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+}
+
+void leapraid_raid_volume_remove(struct leapraid_adapter *adapter,
+				 struct leapraid_raid_volume *raid_volume)
+{
+	unsigned long flags;
+	bool del_from_list = false;
+
+	if (!raid_volume) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid RAID volume\n", __func__);
+		return;
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	if (!list_empty(&raid_volume->list)) {
+		list_del_init(&raid_volume->list);
+		del_from_list = true;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	leapraid_clear_cached_boot_dev(adapter, raid_volume, RAID_CHANNEL);
+	if (del_from_list)
+		leapraid_raid_volume_put(raid_volume);
+}
+
+static struct leapraid_enc_node *leapraid_enc_find_by_hdl(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_enc_node *enc_dev;
+
+	list_for_each_entry(enc_dev, &adapter->dev_topo.enc_list, list)
+		if (le16_to_cpu(enc_dev->pg0.enc_hdl) == hdl)
+			return enc_dev;
+
+	return NULL;
+}
+
+struct leapraid_topo_node *leapraid_exp_find_by_sas_address(
+		struct leapraid_adapter *adapter,
+		u64 sas_address, struct leapraid_card_port *port)
+{
+	struct leapraid_topo_node *sas_exp;
+
+	if (!port) {
+		dev_warn(&adapter->pdev->dev, "%s: Invalid port\n", __func__);
+		return NULL;
+	}
+
+	list_for_each_entry(sas_exp, &adapter->dev_topo.exp_list, list)
+		if (sas_exp->sas_address == sas_address &&
+		    sas_exp->card_port == port)
+			return sas_exp;
+
+	dev_warn(&adapter->pdev->dev,
+		 "%s: No expander found for SAS addr=0x%016llx port=%p\n",
+		 __func__, (unsigned long long)sas_address, port);
+	return NULL;
+}
+
+bool leapraid_scmd_find_by_tgt(struct leapraid_adapter *adapter, uint id,
+			       uint channel)
+{
+	struct scsi_cmnd *scmd;
+	int taskid;
+
+	for (taskid = 1; taskid <= adapter->shost->can_queue; taskid++) {
+		scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
+		if (!scmd)
+			continue;
+
+		if (scmd->device->id == id && scmd->device->channel == channel)
+			return true;
+	}
+
+	return false;
+}
+
+bool leapraid_scmd_find_by_lun(struct leapraid_adapter *adapter, uint id,
+			       unsigned int lun, uint channel)
+{
+	struct scsi_cmnd *scmd;
+	int taskid;
+
+	for (taskid = 1; taskid <= adapter->shost->can_queue; taskid++) {
+		scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
+		if (!scmd)
+			continue;
+
+		if (scmd->device->id == id &&
+		    scmd->device->channel == channel &&
+		    scmd->device->lun == lun)
+			return true;
+	}
+
+	return false;
+}
+
+static struct leapraid_topo_node *leapraid_exp_find_by_hdl(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_topo_node *sas_exp;
+
+	list_for_each_entry(sas_exp, &adapter->dev_topo.exp_list, list)
+		if (sas_exp->hdl == hdl)
+			return sas_exp;
+
+	return NULL;
+}
+
+static enum leapraid_card_port_checking_flg leapraid_get_card_port_feature(
+		struct leapraid_card_port *old_card_port,
+		struct leapraid_card_port *card_port,
+		struct leapraid_card_port_feature *feature)
+{
+	feature->dirty_flg =
+		old_card_port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY;
+	feature->same_addr =
+		old_card_port->sas_address == card_port->sas_address;
+	feature->exact_phy =
+		old_card_port->phy_mask == card_port->phy_mask;
+	feature->phy_overlap =
+		old_card_port->phy_mask & card_port->phy_mask;
+	feature->same_port =
+		old_card_port->port_id == card_port->port_id;
+	feature->cur_chking_old_port = old_card_port;
+
+	if (!feature->dirty_flg || !feature->same_addr)
+		return CARD_PORT_SKIP_CHECKING;
+
+	return CARD_PORT_FURTHER_CHECKING_NEEDED;
+}
+
+static bool leapraid_process_card_port_feature(
+		struct leapraid_card_port_feature *feature)
+{
+	struct leapraid_card_port *old_card_port;
+
+	old_card_port = feature->cur_chking_old_port;
+	if (feature->exact_phy) {
+		feature->checking_state = SAME_PORT_WITH_NOTHING_CHANGED;
+		feature->expected_old_port = old_card_port;
+		return true;
+	}
+
+	if (feature->phy_overlap) {
+		if (feature->same_port) {
+			feature->checking_state =
+				SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS;
+			feature->expected_old_port = old_card_port;
+		} else if (feature->checking_state !=
+			   SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS) {
+			feature->checking_state =
+				SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS;
+			feature->expected_old_port = old_card_port;
+		}
+	} else if (feature->checking_state !=
+		   SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS &&
+		   feature->checking_state !=
+		   SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS) {
+		feature->checking_state = SAME_ADDR_ONLY;
+		feature->expected_old_port = old_card_port;
+		feature->same_addr_port_count++;
+	}
+
+	return false;
+}
+
+static int leapraid_check_card_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_card_port *card_port,
+		struct leapraid_card_port **expected_card_port,
+		int *count)
+{
+	struct leapraid_card_port *old_card_port;
+	struct leapraid_card_port_feature feature;
+
+	*expected_card_port = NULL;
+	memset(&feature, 0, sizeof(struct leapraid_card_port_feature));
+	feature.expected_old_port = NULL;
+	feature.same_addr_port_count = 0;
+	feature.checking_state = NEW_CARD_PORT;
+
+	list_for_each_entry(old_card_port, &adapter->dev_topo.card_port_list,
+			    list) {
+		if (leapraid_get_card_port_feature(old_card_port, card_port,
+						   &feature))
+			continue;
+
+		if (leapraid_process_card_port_feature(&feature))
+			break;
+	}
+
+	if (feature.checking_state == SAME_ADDR_ONLY)
+		*count = feature.same_addr_port_count;
+
+	*expected_card_port = feature.expected_old_port;
+	return feature.checking_state;
+}
+
+static void leapraid_del_phy_part_of_anther_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_card_port *card_port_table, int index,
+		u8 port_count, int offset)
+{
+	struct leapraid_topo_node *card_topo_node;
+	bool found = false;
+	int i;
+
+	card_topo_node = &adapter->dev_topo.card;
+	for (i = 0; i < port_count; i++) {
+		if (i == index)
+			continue;
+
+		if (card_port_table[i].phy_mask & BIT(offset)) {
+			leapraid_transport_detach_phy_to_port(
+				adapter,
+				card_topo_node,
+				&card_topo_node->card_phy[offset]);
+			found = true;
+			break;
+		}
+	}
+
+	if (!found)
+		card_port_table[index].phy_mask |= BIT(offset);
+}
+
+static void leapraid_add_or_del_phys_from_existing_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_card_port *card_port,
+		struct leapraid_card_port *card_port_table,
+		int index, u8 port_count)
+{
+	struct leapraid_topo_node *card_topo_node;
+	u32 phy_mask_diff;
+	u32 offset;
+
+	card_topo_node = &adapter->dev_topo.card;
+	phy_mask_diff = card_port->phy_mask ^
+		card_port_table[index].phy_mask;
+	for (offset = 0; offset < adapter->dev_topo.card.phys_num; offset++) {
+		if (!(phy_mask_diff & BIT(offset)))
+			continue;
+
+		if (!(card_port_table[index].phy_mask & BIT(offset))) {
+			leapraid_del_phy_part_of_anther_port(adapter,
+							     card_port_table,
+							     index, port_count,
+							     offset);
+			continue;
+		}
+
+		if (card_topo_node->card_phy[offset].phy_is_assigned)
+			leapraid_transport_detach_phy_to_port(
+				adapter,
+				card_topo_node,
+				&card_topo_node->card_phy[offset]);
+
+		leapraid_transport_attach_phy_to_port(
+			adapter,
+			card_topo_node,
+			&card_topo_node->card_phy[offset],
+			card_port->sas_address,
+			card_port);
+	}
+}
+
+struct leapraid_sas_dev *leapraid_get_next_sas_dev_from_init_list(
+		struct leapraid_adapter *adapter)
+{
+	struct leapraid_sas_dev *sas_dev = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	if (!list_empty(&adapter->dev_topo.sas_dev_init_list)) {
+		sas_dev = list_first_entry(&adapter->dev_topo.sas_dev_init_list,
+					   struct leapraid_sas_dev,
+					   list);
+		leapraid_sdev_get(sas_dev);
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	return sas_dev;
+}
+
+static bool leapraid_check_boot_dev_internal(u64 sas_address, u64 dev_name,
+					     u64 enc_lid, u16 slot,
+					     struct leapraid_boot_dev *boot_dev,
+					     u8 form)
+{
+	struct leapraid_boot_dev_format_sas_wwid *wwid;
+	struct leapraid_boot_dev_format_enc_slot *es;
+	struct leapraid_boot_dev_format_dev_name *dn;
+	void *pg_dev;
+
+	if (!boot_dev)
+		return false;
+
+	pg_dev = boot_dev->pg_dev;
+	switch (form & LEAPRAID_BOOTDEV_FORM_MASK) {
+	case LEAPRAID_BOOTDEV_FORM_SAS_WWID:
+		wwid = pg_dev;
+
+		if (!sas_address)
+			return false;
+
+		return sas_address == le64_to_cpu(wwid->sas_addr);
+	case LEAPRAID_BOOTDEV_FORM_ENC_SLOT:
+		es = pg_dev;
+
+		if (!enc_lid)
+			return false;
+
+		return (enc_lid == le64_to_cpu(es->enc_lid) &&
+			slot == le16_to_cpu(es->slot_num));
+	case LEAPRAID_BOOTDEV_FORM_DEV_NAME:
+		dn = pg_dev;
+
+		if (!dev_name)
+			return false;
+
+		return dev_name == le64_to_cpu(dn->dev_name);
+	case LEAPRAID_BOOTDEV_FORM_NONE:
+	default:
+		return false;
+	}
+}
+
+void leapraid_boot_dev_get(void *dev, u32 chnl)
+{
+	if (!dev)
+		return;
+
+	if (chnl == RAID_CHANNEL)
+		leapraid_raid_volume_get((struct leapraid_raid_volume *)dev);
+	else
+		leapraid_sdev_get((struct leapraid_sas_dev *)dev);
+}
+
+void leapraid_boot_dev_put(void *dev, u32 chnl)
+{
+	if (!dev)
+		return;
+
+	if (chnl == RAID_CHANNEL)
+		leapraid_raid_volume_put((struct leapraid_raid_volume *)dev);
+	else
+		leapraid_sdev_put((struct leapraid_sas_dev *)dev);
+}
+
+static void leapraid_try_set_boot_dev(struct leapraid_boot_dev *boot_dev,
+				      u64 sas_addr, u64 dev_name,
+				      u64 enc_lid, u16 slot,
+				      void *dev, u32 chnl)
+{
+	bool matched = false;
+
+	if (boot_dev->dev)
+		return;
+
+	matched = leapraid_check_boot_dev_internal(sas_addr, dev_name, enc_lid,
+						   slot, boot_dev,
+						   boot_dev->form);
+	if (matched) {
+		leapraid_boot_dev_get(dev, chnl);
+		boot_dev->dev = dev;
+		boot_dev->chnl = chnl;
+	}
+}
+
+static void leapraid_clear_boot_dev(struct leapraid_adapter *adapter,
+				    struct leapraid_boot_dev *boot_dev,
+				    void *dev, u32 chnl)
+{
+	void *cached_dev;
+	u32 cached_chnl;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->boot_devs.lock, flags);
+	if (boot_dev->dev != dev || boot_dev->chnl != chnl)
+		goto out_unlock;
+
+	cached_dev = boot_dev->dev;
+	cached_chnl = boot_dev->chnl;
+	boot_dev->dev = NULL;
+	boot_dev->chnl = 0;
+	spin_unlock_irqrestore(&adapter->boot_devs.lock, flags);
+	leapraid_boot_dev_put(cached_dev, cached_chnl);
+	return;
+
+out_unlock:
+	spin_unlock_irqrestore(&adapter->boot_devs.lock, flags);
+}
+
+static void leapraid_clear_cached_boot_dev(struct leapraid_adapter *adapter,
+					   void *dev, u32 chnl)
+{
+	leapraid_clear_boot_dev(adapter,
+				&adapter->boot_devs.requested_boot_dev,
+				dev, chnl);
+	leapraid_clear_boot_dev(adapter,
+				&adapter->boot_devs.requested_alt_boot_dev,
+				dev, chnl);
+	leapraid_clear_boot_dev(adapter,
+				&adapter->boot_devs.current_boot_dev,
+				dev, chnl);
+}
+
+static void leapraid_check_boot_dev(struct leapraid_adapter *adapter,
+				    void *dev, u32 chnl)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_sas_dev *sas_dev;
+	u64 sas_addr;
+	u64 dev_name = 0;
+	u64 enc_lid = 0;
+	u16 slot = 0;
+
+	if (!adapter->scan_dev_desc.driver_loading)
+		return;
+
+	switch (chnl) {
+	case RAID_CHANNEL:
+		raid_volume = dev;
+		sas_addr = raid_volume->wwid;
+		break;
+	default:
+		sas_dev = dev;
+		sas_addr = sas_dev->sas_addr;
+		dev_name = sas_dev->dev_name;
+		enc_lid = sas_dev->enc_lid;
+		slot = sas_dev->slot;
+		break;
+	}
+
+	leapraid_try_set_boot_dev(&adapter->boot_devs.requested_boot_dev,
+				  sas_addr, dev_name, enc_lid,
+				  slot, dev, chnl);
+	leapraid_try_set_boot_dev(&adapter->boot_devs.requested_alt_boot_dev,
+				  sas_addr, dev_name, enc_lid,
+				  slot, dev, chnl);
+	leapraid_try_set_boot_dev(&adapter->boot_devs.current_boot_dev,
+				  sas_addr, dev_name, enc_lid,
+				  slot, dev, chnl);
+}
+
+static const char *leapraid_func_name(u8 func)
+{
+	switch (func) {
+	case LEAPRAID_FUNC_SCSIIO:
+		return "SCSIIO";
+	case LEAPRAID_FUNC_SCSI_TMF:
+		return "SCSI_TMF";
+	case LEAPRAID_FUNC_ADAPTER_INIT:
+		return "ADAPTER_INIT";
+	case LEAPRAID_FUNC_GET_ADAPTER_FEATURES:
+		return "GET_ADAPTER_FEATURES";
+	case LEAPRAID_FUNC_CONFIG_OP:
+		return "CONFIG_OP";
+	case LEAPRAID_FUNC_SCAN_DEV:
+		return "SCAN_DEV";
+	case LEAPRAID_FUNC_EVENT_NOTIFY:
+		return "EVENT_NOTIFY";
+	case LEAPRAID_FUNC_FW_DOWNLOAD:
+		return "FW_DOWNLOAD";
+	case LEAPRAID_FUNC_FW_UPLOAD:
+		return "FW_UPLOAD";
+	case LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH:
+		return "SCSIIO_RAID_PASSTHROUGH";
+	case LEAPRAID_FUNC_SCSI_ENC_PROCESSOR:
+		return "SCSI_ENC_PROCESSOR";
+	case LEAPRAID_FUNC_SMP_PASSTHROUGH:
+		return "SMP_PASSTHROUGH";
+	case LEAPRAID_FUNC_SAS_IO_UNIT_CTRL:
+		return "SAS_IO_UNIT_CTRL";
+	case LEAPRAID_FUNC_SCSIIO_SATA_PASSTHROUGH:
+		return "SCSIIO_SATA_PASSTHROUGH";
+	case LEAPRAID_FUNC_ADAPTER_UNIT_RESET:
+		return "ADAPTER_UNIT_RESET";
+	case LEAPRAID_FUNC_HANDSHAKE:
+		return "HANDSHAKE";
+	case LEAPRAID_FUNC_LOGBUF_INIT:
+		return "LOGBUF_INIT";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *leapraid_cfg_action_name(u8 action)
+{
+	switch (action) {
+	case LEAPRAID_CFG_ACT_PAGE_HEADER:
+		return "PAGE_HEADER";
+	case LEAPRAID_CFG_ACT_PAGE_READ_CUR:
+		return "PAGE_READ_CUR";
+	case LEAPRAID_CFG_ACT_PAGE_WRITE_CUR:
+		return "PAGE_WRITE_CUR";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *leapraid_cfg_page_type_name(u8 page_type)
+{
+	switch (page_type) {
+	case LEAPRAID_CFG_PT_IO_UNIT:
+		return "IO_UNIT";
+	case LEAPRAID_CFG_PT_ADAPTER:
+		return "ADAPTER";
+	case LEAPRAID_CFG_PT_BIOS:
+		return "BIOS";
+	case LEAPRAID_CFG_PT_RAID_VOLUME:
+		return "RAID_VOLUME";
+	case LEAPRAID_CFG_PT_MANUFACTURING:
+		return "MANUFACTURING";
+	case LEAPRAID_CFG_PT_RAID_PHYSDISK:
+		return "RAID_PHYSDISK";
+	case LEAPRAID_CFG_PT_EXTENDED:
+		return "EXTENDED";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *leapraid_cfg_ext_page_type_name(u8 ext_page_type)
+{
+	switch (ext_page_type) {
+	case LEAPRAID_CFG_EXTPT_SAS_IO_UNIT:
+		return "SAS_IO_UNIT";
+	case LEAPRAID_CFG_EXTPT_SAS_EXP:
+		return "SAS_EXPANDER";
+	case LEAPRAID_CFG_EXTPT_SAS_DEV:
+		return "SAS_DEVICE";
+	case LEAPRAID_CFG_EXTPT_SAS_PHY:
+		return "SAS_PHY";
+	case LEAPRAID_CFG_EXTPT_ENC:
+		return "ENCLOSURE";
+	case LEAPRAID_CFG_EXTPT_RAID_CONFIG:
+		return "RAID_CONFIG";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *leapraid_sep_action_name(u8 action)
+{
+	switch (action) {
+	case LEAPRAID_SEP_REQ_ACT_WRITE_STATUS:
+		return "WRITE_STATUS";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *leapraid_sas_op_name(u8 op)
+{
+	switch (op) {
+	case LEAPRAID_SAS_OP_PHY_LINK_RESET:
+		return "PHY_LINK_RESET";
+	case LEAPRAID_SAS_OP_PHY_HARD_RESET:
+		return "PHY_HARD_RESET";
+	case LEAPRAID_SAS_OP_SET_PARAMETER:
+		return "SET_PARAMETER";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+static const char *leapraid_tm_type_name(u8 task_type)
+{
+	switch (task_type) {
+	case LEAPRAID_TM_TASKTYPE_ABORT_TASK:
+		return "ABORT_TASK";
+	case LEAPRAID_TM_TASKTYPE_ABRT_TASK_SET:
+		return "ABORT_TASK_SET";
+	case LEAPRAID_TM_TASKTYPE_TARGET_RESET:
+		return "TARGET_RESET";
+	case LEAPRAID_TM_TASKTYPE_LOGICAL_UNIT_RESET:
+		return "LOGICAL_UNIT_RESET";
+	case LEAPRAID_TM_TASKTYPE_CLEAR_TASK_SET:
+		return "CLEAR_TASK_SET";
+	case LEAPRAID_TM_TASKTYPE_QUERY_TASK:
+		return "QUERY_TASK";
+	case LEAPRAID_TM_TASKTYPE_CLEAR_ACA:
+		return "CLEAR_ACA";
+	case LEAPRAID_TM_TASKTYPE_QUERY_TASK_SET:
+		return "QUERY_TASK_SET";
+	case LEAPRAID_TM_TASKTYPE_QUERY_ASYNC_EVENT:
+		return "QUERY_ASYNC_EVENT";
+	default:
+		return "UNKNOWN";
+	}
+}
+
+void leapraid_log_req_context(struct leapraid_adapter *adapter, u16 smid,
+			      const void *req_data)
+{
+	const struct leapraid_req *req = req_data;
+
+	if (!adapter || !adapter->pdev || !req_data)
+		return;
+
+	switch (req->func) {
+	case LEAPRAID_FUNC_CONFIG_OP: {
+		const struct leapraid_cfg_req *cfg_req = req_data;
+
+		dev_err(&adapter->pdev->dev,
+			"cfg-req: smid=%u func=0x%02x(%s) action=0x%02x(%s)\n",
+			smid, req->func, leapraid_func_name(req->func),
+			cfg_req->action,
+			leapraid_cfg_action_name(cfg_req->action));
+		dev_err(&adapter->pdev->dev,
+			"cfg-req: page_type=0x%02x(%s) page_num=%u\n",
+			cfg_req->header.page_type,
+			leapraid_cfg_page_type_name(cfg_req->header.page_type),
+			cfg_req->header.page_num);
+		if (cfg_req->header.page_type == LEAPRAID_CFG_PT_EXTENDED)
+			dev_err(&adapter->pdev->dev,
+				"cfg-req: ext_page_type=0x%02x(%s)\n",
+				cfg_req->ext_page_type,
+				leapraid_cfg_ext_page_type_name(
+					cfg_req->ext_page_type));
+		dev_err(&adapter->pdev->dev, "cfg-req: page_addr=0x%08x\n",
+			le32_to_cpu(cfg_req->page_addr));
+		break;
+	}
+	case LEAPRAID_FUNC_SCSI_TMF: {
+		const struct leapraid_scsi_tm_req *tm_req = req_data;
+
+		dev_err(&adapter->pdev->dev,
+			"scsi_tm:: smid=%u func=0x%02x(%s) task=0x%02x(%s)\n",
+			smid, req->func, leapraid_func_name(req->func),
+			tm_req->task_type,
+			leapraid_tm_type_name(tm_req->task_type));
+		dev_err(&adapter->pdev->dev,
+			"scsi_tm:: dev_hdl=0x%04x task_mid=%u\n",
+			le16_to_cpu(tm_req->dev_hdl),
+			le16_to_cpu(tm_req->task_mid));
+		break;
+	}
+	case LEAPRAID_FUNC_SCSI_ENC_PROCESSOR: {
+		const struct leapraid_sep_req *sep_req = req_data;
+
+		dev_err(&adapter->pdev->dev,
+			"sep: smid=%u func=0x%02x(%s) action=0x%02x(%s)\n",
+			smid, req->func, leapraid_func_name(req->func),
+			sep_req->act,
+			leapraid_sep_action_name(sep_req->act));
+		dev_err(&adapter->pdev->dev,
+			"sep: dev_hdl=0x%04x slot=%u enc_hdl=0x%04x\n",
+			le16_to_cpu(sep_req->dev_hdl),
+			le16_to_cpu(sep_req->slot),
+			le16_to_cpu(sep_req->enc_hdl));
+		break;
+	}
+	case LEAPRAID_FUNC_SAS_IO_UNIT_CTRL: {
+		const struct leapraid_io_unit_ctrl_req *io_req = req_data;
+
+		dev_err(&adapter->pdev->dev,
+			"ctl_cmd: smid=%u func=0x%02x(%s) action=0x%02x(%s)\n",
+			smid, req->func, leapraid_func_name(req->func),
+			io_req->op, leapraid_sas_op_name(io_req->op));
+		dev_err(&adapter->pdev->dev,
+			"ctl_cmd: dev_hdl=0x%04x param=0x%02x\n",
+			le16_to_cpu(io_req->dev_hdl),
+			io_req->adapter_para);
+		break;
+	}
+	case LEAPRAID_FUNC_SMP_PASSTHROUGH: {
+		const struct leapraid_smp_passthrough_req *smp_req = req_data;
+
+		dev_err(&adapter->pdev->dev,
+			"smp_cmd: smid=%u func=0x%02x(%s) action=0x%02x\n",
+			smid, req->func, leapraid_func_name(req->func),
+			smp_req->passthrough_flg);
+		dev_err(&adapter->pdev->dev,
+			"smp_cmd: port=%u req_len=%u\n",
+			smp_req->physical_port,
+			le16_to_cpu(smp_req->req_data_len));
+		dev_err(&adapter->pdev->dev, "smp_cmd: sas_addr=0x%016llx\n",
+			(unsigned long long)le64_to_cpu(smp_req->sas_address));
+		break;
+	}
+	default:
+		dev_err(&adapter->pdev->dev,
+			"cmd: smid=%u func=0x%02x(%s)\n",
+			smid, req->func, leapraid_func_name(req->func));
+		break;
+	}
+}
+
+static void leapraid_build_and_fire_cfg_req(
+		struct leapraid_adapter *adapter,
+		struct leapraid_cfg_req *leap_mpi_cfgp_req,
+		struct leapraid_cfg_rep *leap_mpi_cfgp_rep)
+{
+	struct leapraid_cfg_req *local_leap_cfg_req;
+	u16 smid;
+
+	memset(leap_mpi_cfgp_rep, 0, sizeof(struct leapraid_cfg_rep));
+	memset(&adapter->driver_cmds.cfg_op_cmd.reply, 0,
+	       sizeof(struct leapraid_cfg_rep));
+	adapter->driver_cmds.cfg_op_cmd.status = LEAPRAID_CMD_PENDING;
+	smid = adapter->driver_cmds.cfg_op_cmd.inter_taskid;
+	local_leap_cfg_req = leapraid_get_task_desc(adapter, smid);
+	memcpy(local_leap_cfg_req, leap_mpi_cfgp_req,
+	       sizeof(struct leapraid_cfg_req));
+	init_completion(&adapter->driver_cmds.cfg_op_cmd.done);
+	leapraid_fire_task(adapter, smid);
+	wait_for_completion_timeout(&adapter->driver_cmds.cfg_op_cmd.done,
+				    LEAPRAID_CFG_OP_TIMEOUT * HZ);
+}
+
+static int leapraid_req_cfg_func(struct leapraid_adapter *adapter,
+				 struct leapraid_cfg_req *leap_mpi_cfgp_req,
+				 struct leapraid_cfg_rep *leap_mpi_cfgp_rep,
+				 void *target_cfg_pg, void *real_cfg_pg_addr,
+				 u16 target_real_cfg_pg_sz)
+{
+	u32 adapter_status = UINT_MAX;
+	bool issue_reset = false;
+	u16 smid;
+	u8 retry_cnt;
+	int rc;
+
+	retry_cnt = 0;
+	mutex_lock(&adapter->driver_cmds.cfg_op_cmd.mutex);
+	smid = adapter->driver_cmds.cfg_op_cmd.inter_taskid;
+retry:
+	if (retry_cnt) {
+		if (retry_cnt > LEAPRAID_CFG_REQ_RETRY_TIMES) {
+			rc = -EFAULT;
+			goto out_cleanup;
+		}
+		dev_warn(&adapter->pdev->dev,
+			 "cfg-req: Retry request, cnt=%u\n", retry_cnt);
+	}
+
+	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
+					  __func__);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"cfg-req: Adapter not operational\n");
+		goto out_cleanup;
+	}
+
+	leapraid_build_and_fire_cfg_req(adapter, leap_mpi_cfgp_req,
+					leap_mpi_cfgp_rep);
+	if (!(adapter->driver_cmds.cfg_op_cmd.status & LEAPRAID_CMD_DONE)) {
+		retry_cnt++;
+		if (adapter->driver_cmds.cfg_op_cmd.status &
+		    LEAPRAID_CMD_RESET) {
+			dev_warn(&adapter->pdev->dev,
+				 "cfg-req: CMD fail due to hard reset\n");
+			goto retry;
+		}
+
+		if (adapter->access_ctrl.shost_recovering ||
+		    adapter->access_ctrl.pcie_recovering) {
+			dev_err(&adapter->pdev->dev,
+				"cfg-req: pending in %s, status=0x%x\n",
+				adapter->access_ctrl.shost_recovering ?
+				"shost recovery" : "pcie recovery",
+				adapter->driver_cmds.cfg_op_cmd.status);
+			leapraid_log_req_context(adapter, smid,
+						 leap_mpi_cfgp_req);
+			issue_reset = false;
+			rc = -EFAULT;
+		} else {
+			dev_err(&adapter->pdev->dev,
+				"cfg-req: timeout, status=0x%x, reset\n",
+				adapter->driver_cmds.cfg_op_cmd.status);
+			leapraid_log_req_context(adapter, smid,
+						 leap_mpi_cfgp_req);
+			issue_reset = true;
+		}
+
+		goto out_cleanup;
+	}
+
+	if (adapter->driver_cmds.cfg_op_cmd.status &
+	    LEAPRAID_CMD_REPLY_VALID) {
+		memcpy(leap_mpi_cfgp_rep,
+		       &adapter->driver_cmds.cfg_op_cmd.reply,
+		       sizeof(struct leapraid_cfg_rep));
+		adapter_status = le16_to_cpu(
+			leap_mpi_cfgp_rep->adapter_status) &
+			LEAPRAID_ADAPTER_STATUS_MASK;
+		if (adapter_status == LEAPRAID_ADAPTER_STATUS_SUCCESS &&
+		    target_cfg_pg && real_cfg_pg_addr &&
+		    target_real_cfg_pg_sz &&
+		    leap_mpi_cfgp_req->action ==
+		    LEAPRAID_CFG_ACT_PAGE_READ_CUR)
+			memcpy(target_cfg_pg, real_cfg_pg_addr,
+			       target_real_cfg_pg_sz);
+		if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
+			if (adapter_status !=
+			    LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE)
+				dev_err(&adapter->pdev->dev,
+					"cfg-rep: adapter_status=0x%x\n",
+					adapter_status);
+			rc = -EFAULT;
+		}
+	} else {
+		dev_err(&adapter->pdev->dev, "cfg-rep: Reply invalid\n");
+		rc = -EFAULT;
+	}
+
+out_cleanup:
+	adapter->driver_cmds.cfg_op_cmd.status = LEAPRAID_CMD_NOT_USED;
+	mutex_unlock(&adapter->driver_cmds.cfg_op_cmd.mutex);
+	if (issue_reset) {
+		if (adapter->scan_dev_desc.first_scan_dev_fired) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s:%d cfg-req: Failure, issuing reset\n",
+				 __func__, __LINE__);
+			leapraid_hard_reset_handler(adapter, FULL_RESET);
+		} else {
+			dev_warn(&adapter->pdev->dev,
+				 "cfg-req: CMD fail in init, skip reset\n");
+		}
+		rc = -EFAULT;
+	}
+	return rc;
+}
+
+static int leapraid_request_cfg_pg_header(
+		struct leapraid_adapter *adapter,
+		struct leapraid_cfg_req *leap_mpi_cfgp_req,
+		struct leapraid_cfg_rep *leap_mpi_cfgp_rep)
+{
+	return leapraid_req_cfg_func(adapter, leap_mpi_cfgp_req,
+				     leap_mpi_cfgp_rep, NULL, NULL, 0);
+}
+
+static int leapraid_request_cfg_pg(struct leapraid_adapter *adapter,
+				   struct leapraid_cfg_req *leap_mpi_cfgp_req,
+				   struct leapraid_cfg_rep *leap_mpi_cfgp_rep,
+				   void *target_cfg_pg, void *real_cfg_pg_addr,
+				   u16 target_real_cfg_pg_sz)
+{
+	return leapraid_req_cfg_func(adapter, leap_mpi_cfgp_req,
+				     leap_mpi_cfgp_rep, target_cfg_pg,
+				     real_cfg_pg_addr, target_real_cfg_pg_sz);
+}
+
+int leapraid_op_config_page(struct leapraid_adapter *adapter,
+			    void *target_cfg_pg, union cfg_param_1 cfgp1,
+			    union cfg_param_2 cfgp2,
+			    enum config_page_action cfg_op)
+{
+	struct leapraid_cfg_req leap_mpi_cfgp_req;
+	struct leapraid_cfg_rep leap_mpi_cfgp_rep;
+	u16 real_cfg_pg_sz;
+	void *real_cfg_pg_addr;
+	dma_addr_t real_cfg_pg_dma = 0;
+	u32 __page_size;
+	int rc;
+
+	memset(&leap_mpi_cfgp_req, 0, sizeof(struct leapraid_cfg_req));
+	leap_mpi_cfgp_req.func = LEAPRAID_FUNC_CONFIG_OP;
+	leap_mpi_cfgp_req.action = LEAPRAID_CFG_ACT_PAGE_HEADER;
+
+	switch (cfg_op) {
+	case GET_BIOS_PG3:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_BIOS;
+		leap_mpi_cfgp_req.header.page_num =
+			LEAPRAID_CFG_PAGE_NUM_BIOS3;
+		__page_size = sizeof(struct leapraid_bios_page3);
+		break;
+	case GET_BIOS_PG2:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_BIOS;
+		leap_mpi_cfgp_req.header.page_num =
+			LEAPRAID_CFG_PAGE_NUM_BIOS2;
+		__page_size = sizeof(struct leapraid_bios_page2);
+		break;
+	case GET_MANUFACTURING_PG0:
+		leap_mpi_cfgp_req.header.page_type =
+			LEAPRAID_CFG_PT_MANUFACTURING;
+		leap_mpi_cfgp_req.header.page_num =
+			LEAPRAID_CFG_PAGE_NUM_MANU0;
+		__page_size = sizeof(struct leapraid_manufacturing_p0);
+		break;
+	case GET_SAS_DEVICE_PG0:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_DEV;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_DEV0;
+		__page_size = sizeof(struct leapraid_sas_dev_p0);
+		break;
+	case GET_SAS_IOUNIT_PG0:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type =
+			LEAPRAID_CFG_EXTPT_SAS_IO_UNIT;
+		leap_mpi_cfgp_req.header.page_num =
+			LEAPRAID_CFG_PAGE_NUM_IOUNIT0;
+		__page_size = cfgp1.size;
+		break;
+	case GET_SAS_IOUNIT_PG1:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type =
+			LEAPRAID_CFG_EXTPT_SAS_IO_UNIT;
+		leap_mpi_cfgp_req.header.page_num =
+			LEAPRAID_CFG_PAGE_NUM_IOUNIT1;
+		__page_size = cfgp1.size;
+		break;
+	case GET_SAS_EXPANDER_PG0:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_EXP;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_EXP0;
+		__page_size = sizeof(struct leapraid_exp_p0);
+		break;
+	case GET_SAS_EXPANDER_PG1:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_EXP;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_EXP1;
+		__page_size = sizeof(struct leapraid_exp_p1);
+		break;
+	case GET_SAS_ENCLOSURE_PG0:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_ENC;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_ENC0;
+		__page_size = sizeof(struct leapraid_enc_p0);
+		break;
+	case GET_PHY_PG0:
+		leap_mpi_cfgp_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+		leap_mpi_cfgp_req.ext_page_type = LEAPRAID_CFG_EXTPT_SAS_PHY;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_PHY0;
+		__page_size = sizeof(struct leapraid_sas_phy_p0);
+		break;
+	case GET_RAID_VOLUME_PG0:
+		leap_mpi_cfgp_req.header.page_type =
+			LEAPRAID_CFG_PT_RAID_VOLUME;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_VOL0;
+		__page_size = cfgp1.size;
+		break;
+	case GET_RAID_VOLUME_PG1:
+		leap_mpi_cfgp_req.header.page_type =
+			LEAPRAID_CFG_PT_RAID_VOLUME;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_VOL1;
+		__page_size = sizeof(struct leapraid_raidvol_p1);
+		break;
+	case GET_PHY_DISK_PG0:
+		leap_mpi_cfgp_req.header.page_type =
+			LEAPRAID_CFG_PT_RAID_PHYSDISK;
+		leap_mpi_cfgp_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_PD0;
+		__page_size = sizeof(struct leapraid_raidpd_p0);
+		break;
+	default:
+		dev_err(&adapter->pdev->dev,
+			"Unsupported config page action=%d!\n", cfg_op);
+		return -EINVAL;
+	}
+
+	leapraid_build_nodata_mpi_sg(adapter,
+				     &leap_mpi_cfgp_req.page_buf_sge);
+	rc = leapraid_request_cfg_pg_header(adapter,
+					    &leap_mpi_cfgp_req,
+					    &leap_mpi_cfgp_rep);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"cfg-req: Header failed rc=%dn", rc);
+		return rc;
+	}
+
+	if (cfg_op == GET_SAS_DEVICE_PG0 ||
+	    cfg_op == GET_SAS_EXPANDER_PG0 ||
+	    cfg_op == GET_SAS_ENCLOSURE_PG0 ||
+	    cfg_op == GET_RAID_VOLUME_PG1)
+		leap_mpi_cfgp_req.page_addr = cpu_to_le32(cfgp1.form |
+							  cfgp2.handle);
+	else if (cfg_op == GET_PHY_DISK_PG0)
+		leap_mpi_cfgp_req.page_addr = cpu_to_le32(cfgp1.form |
+							  cfgp2.form_specific);
+	else if (cfg_op == GET_RAID_VOLUME_PG0)
+		leap_mpi_cfgp_req.page_addr =
+			cpu_to_le32(cfgp2.handle |
+				    LEAPRAID_RAID_VOL_CFG_PGAD_HDL);
+	else if (cfg_op == GET_SAS_EXPANDER_PG1)
+		leap_mpi_cfgp_req.page_addr =
+			cpu_to_le32(cfgp2.handle |
+				    (cfgp1.phy_number <<
+				     LEAPRAID_SAS_EXP_CFG_PGAD_PHYNUM_SHIFT) |
+				    LEAPRAID_SAS_EXP_CFG_PGAD_HDL_PHY_NUM);
+	else if (cfg_op == GET_PHY_PG0)
+		leap_mpi_cfgp_req.page_addr = cpu_to_le32(cfgp1.phy_number |
+			LEAPRAID_SAS_PHY_CFG_PGAD_PHY_NUMBER);
+
+	leap_mpi_cfgp_req.action = LEAPRAID_CFG_ACT_PAGE_READ_CUR;
+
+	leap_mpi_cfgp_req.header.page_num = leap_mpi_cfgp_rep.header.page_num;
+	leap_mpi_cfgp_req.header.page_type =
+		leap_mpi_cfgp_rep.header.page_type;
+	leap_mpi_cfgp_req.header.page_len = leap_mpi_cfgp_rep.header.page_len;
+	leap_mpi_cfgp_req.ext_page_len = leap_mpi_cfgp_rep.ext_page_len;
+	leap_mpi_cfgp_req.ext_page_type = leap_mpi_cfgp_rep.ext_page_type;
+
+	real_cfg_pg_sz = (leap_mpi_cfgp_req.header.page_len) ?
+		leap_mpi_cfgp_req.header.page_len * sizeof(u32) :
+		le16_to_cpu(leap_mpi_cfgp_rep.ext_page_len) * sizeof(u32);
+	real_cfg_pg_addr = dma_alloc_coherent(&adapter->pdev->dev,
+					      real_cfg_pg_sz,
+					      &real_cfg_pg_dma,
+					      GFP_KERNEL);
+	if (!real_cfg_pg_addr)
+		return -ENOMEM;
+
+	if (leap_mpi_cfgp_req.action == LEAPRAID_CFG_ACT_PAGE_WRITE_CUR) {
+		leapraid_single_mpi_sg_append(adapter,
+					      &leap_mpi_cfgp_req.page_buf_sge,
+					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+						LEAPRAID_SGE_FLG_LAST_ONE |
+						LEAPRAID_SGE_FLG_EOB |
+						LEAPRAID_SGE_FLG_EOL |
+						LEAPRAID_SGE_FLG_H2C) <<
+						LEAPRAID_SGE_FLG_SHIFT) |
+					       real_cfg_pg_sz,
+					      real_cfg_pg_dma);
+		memcpy(real_cfg_pg_addr, target_cfg_pg,
+		       min_t(u16, real_cfg_pg_sz, __page_size));
+	} else {
+		memset(target_cfg_pg, 0, __page_size);
+		leapraid_single_mpi_sg_append(adapter,
+					      &leap_mpi_cfgp_req.page_buf_sge,
+					      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+						LEAPRAID_SGE_FLG_LAST_ONE |
+						LEAPRAID_SGE_FLG_EOB |
+						LEAPRAID_SGE_FLG_EOL) <<
+						LEAPRAID_SGE_FLG_SHIFT) |
+					       real_cfg_pg_sz,
+					      real_cfg_pg_dma);
+		memset(real_cfg_pg_addr, 0,
+		       min_t(u16, real_cfg_pg_sz, __page_size));
+	}
+
+	rc = leapraid_request_cfg_pg(adapter,
+				     &leap_mpi_cfgp_req,
+				     &leap_mpi_cfgp_rep,
+				     target_cfg_pg,
+				     real_cfg_pg_addr,
+				     min_t(u16, real_cfg_pg_sz, __page_size));
+	if (rc) {
+		u32 status;
+
+		status = le16_to_cpu(leap_mpi_cfgp_rep.adapter_status) &
+				     LEAPRAID_ADAPTER_STATUS_MASK;
+		if (status != LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE)
+			dev_err(&adapter->pdev->dev,
+				"cfg-req: rc=%d, pg_info: 0x%x, 0x%x, %d\n",
+				rc, leap_mpi_cfgp_req.header.page_type,
+				leap_mpi_cfgp_req.ext_page_type,
+				leap_mpi_cfgp_req.header.page_num);
+	}
+
+	if (real_cfg_pg_addr)
+		dma_free_coherent(&adapter->pdev->dev,
+				  real_cfg_pg_sz,
+				  real_cfg_pg_addr,
+				  real_cfg_pg_dma);
+
+	return rc;
+}
+
+static int leapraid_cfg_find_vol_in_page(
+		struct leapraid_raid_cfg_p0 *raid_cfg_p0,
+		u16 pd_hdl, u16 *vol_hdl)
+{
+	u16 elements = raid_cfg_p0->elements_num;
+	int i;
+
+	for (i = 0; i < elements; i++) {
+		struct leapraid_raid_cfg_p0_element *elem;
+		u16 type;
+
+		elem = &raid_cfg_p0->cfg_element[i];
+
+		type = le16_to_cpu(elem->element_flg) &
+		       LEAPRAID_RAIDCFG_P0_EFLG_MASK_ELEMENT_TYPE;
+
+		switch (type) {
+		case LEAPRAID_RAIDCFG_P0_EFLG_VOL_PHYS_DISK_ELEMENT:
+		case LEAPRAID_RAIDCFG_P0_EFLG_OCE_ELEMENT: {
+			u16 phys_hdl;
+
+			phys_hdl = le16_to_cpu(elem->phys_disk_dev_hdl);
+			if (phys_hdl == pd_hdl) {
+				*vol_hdl = le16_to_cpu(elem->vol_dev_hdl);
+				return 0;
+			}
+			break;
+		}
+		case LEAPRAID_RAIDCFG_P0_EFLG_HOT_SPARE_ELEMENT:
+			*vol_hdl = 0;
+			return 0;
+
+		default:
+			break;
+		}
+	}
+
+	return -ENOENT;
+}
+
+static int leapraid_cfg_get_volume_hdl_dispatch(
+		struct leapraid_adapter *adapter,
+		struct leapraid_cfg_req *cfg_req,
+		struct leapraid_cfg_rep *cfg_rep,
+		struct leapraid_raid_cfg_p0 *raid_cfg_p0,
+		void *real_cfg_pg_addr,
+		u16 real_cfg_pg_sz,
+		u16 raid_cfg_p0_sz,
+		u16 pd_hdl, u16 *vol_hdl)
+{
+	u16 adapter_status;
+	int config_num;
+	int rc;
+
+	config_num = 0xFF;
+	while (true) {
+		cfg_req->page_addr =
+			cpu_to_le32(config_num +
+				    LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP);
+		rc = leapraid_request_cfg_pg(
+			adapter, cfg_req, cfg_rep,
+			raid_cfg_p0, real_cfg_pg_addr,
+			min_t(u16, real_cfg_pg_sz, raid_cfg_p0_sz));
+		adapter_status = le16_to_cpu(cfg_rep->adapter_status) &
+			LEAPRAID_ADAPTER_STATUS_MASK;
+		if (rc) {
+			if (adapter_status ==
+			    LEAPRAID_ADAPTER_STATUS_CONFIG_INVALID_PAGE) {
+				*vol_hdl = 0;
+				return 0;
+			}
+			return rc;
+		}
+
+		if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS)
+			return LEAPRAID_OPERATION_FAILED;
+
+		rc = leapraid_cfg_find_vol_in_page(raid_cfg_p0,
+						   pd_hdl,
+						   vol_hdl);
+		if (rc != -ENOENT)
+			return rc;
+
+		config_num = raid_cfg_p0->cfg_num;
+	}
+	return 0;
+}
+
+int leapraid_cfg_get_volume_hdl(struct leapraid_adapter *adapter,
+				u16 pd_hdl, u16 *vol_hdl)
+{
+	struct leapraid_raid_cfg_p0 *raid_cfg_p0;
+	struct leapraid_cfg_req cfg_req;
+	struct leapraid_cfg_rep cfg_rep;
+	dma_addr_t real_cfg_pg_dma = 0;
+	void *real_cfg_pg_addr;
+	u16 real_cfg_pg_sz;
+	int rc, raid_cfg_p0_sz;
+
+	*vol_hdl = 0;
+	memset(&cfg_req, 0, sizeof(struct leapraid_cfg_req));
+	cfg_req.func = LEAPRAID_FUNC_CONFIG_OP;
+	cfg_req.action = LEAPRAID_CFG_ACT_PAGE_HEADER;
+	cfg_req.header.page_type = LEAPRAID_CFG_PT_EXTENDED;
+	cfg_req.ext_page_type = LEAPRAID_CFG_EXTPT_RAID_CONFIG;
+	cfg_req.header.page_num = LEAPRAID_CFG_PAGE_NUM_VOL0;
+
+	leapraid_build_nodata_mpi_sg(adapter, &cfg_req.page_buf_sge);
+	rc = leapraid_request_cfg_pg_header(adapter, &cfg_req, &cfg_rep);
+	if (rc)
+		return rc;
+
+	cfg_req.action = LEAPRAID_CFG_ACT_PAGE_READ_CUR;
+	raid_cfg_p0_sz = le16_to_cpu(cfg_rep.ext_page_len) *
+		LEAPRAID_CFG_UNIT_SIZE;
+	raid_cfg_p0 = kmalloc(raid_cfg_p0_sz, GFP_KERNEL);
+	if (!raid_cfg_p0)
+		return -ENOMEM;
+
+	real_cfg_pg_sz = (cfg_req.header.page_len) ?
+		cfg_req.header.page_len * LEAPRAID_CFG_UNIT_SIZE :
+		le16_to_cpu(cfg_rep.ext_page_len) * LEAPRAID_CFG_UNIT_SIZE;
+
+	real_cfg_pg_addr = dma_alloc_coherent(&adapter->pdev->dev,
+					      real_cfg_pg_sz, &real_cfg_pg_dma,
+					      GFP_KERNEL);
+	if (!real_cfg_pg_addr) {
+		rc = -ENOMEM;
+		goto out_free;
+	}
+
+	memset(raid_cfg_p0, 0, raid_cfg_p0_sz);
+	leapraid_single_mpi_sg_append(adapter,
+				      &cfg_req.page_buf_sge,
+				      ((LEAPRAID_SGE_FLG_SIMPLE_ONE |
+					LEAPRAID_SGE_FLG_LAST_ONE |
+					LEAPRAID_SGE_FLG_EOB |
+					LEAPRAID_SGE_FLG_EOL) <<
+				       LEAPRAID_SGE_FLG_SHIFT) |
+				      real_cfg_pg_sz,
+				      real_cfg_pg_dma);
+	memset(real_cfg_pg_addr, 0,
+	       min_t(u16, real_cfg_pg_sz, raid_cfg_p0_sz));
+
+	rc = leapraid_cfg_get_volume_hdl_dispatch(adapter,
+						  &cfg_req, &cfg_rep,
+						  raid_cfg_p0,
+						  real_cfg_pg_addr,
+						  real_cfg_pg_sz,
+						  raid_cfg_p0_sz,
+						  pd_hdl, vol_hdl);
+
+out_free:
+	if (real_cfg_pg_addr)
+		dma_free_coherent(&adapter->pdev->dev,
+				  real_cfg_pg_sz, real_cfg_pg_addr,
+				  real_cfg_pg_dma);
+	kfree(raid_cfg_p0);
+	return rc;
+}
+
+static int leapraid_get_adapter_phys(struct leapraid_adapter *adapter,
+				     u8 *nr_phys)
+{
+	struct leapraid_sas_io_unit_p0 sas_io_unit_page0;
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	int rc;
+
+	*nr_phys = 0;
+	cfgp1.size = sizeof(struct leapraid_sas_io_unit_p0);
+	rc = leapraid_op_config_page(adapter, &sas_io_unit_page0, cfgp1,
+				     cfgp2, GET_SAS_IOUNIT_PG0);
+	if (rc)
+		return rc;
+
+	*nr_phys = sas_io_unit_page0.phy_num;
+
+	return 0;
+}
+
+static int leapraid_cfg_get_number_pds(struct leapraid_adapter *adapter,
+				       u16 hdl, u8 *num_pds)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_raidvol_p0 raidvol_p0;
+	int rc;
+
+	*num_pds = 0;
+	cfgp1.size = sizeof(struct leapraid_raidvol_p0);
+	cfgp2.handle = hdl;
+	rc = leapraid_op_config_page(adapter, &raidvol_p0, cfgp1,
+				     cfgp2, GET_RAID_VOLUME_PG0);
+	if (!rc)
+		*num_pds = raidvol_p0.num_phys_disks;
+
+	return rc;
+}
+
+int leapraid_cfg_get_volume_wwid(struct leapraid_adapter *adapter,
+				 u16 vol_hdl, u64 *wwid)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_raidvol_p1 raidvol_p1;
+	int rc;
+
+	*wwid = 0;
+	cfgp1.form = LEAPRAID_RAID_VOL_CFG_PGAD_HDL;
+	cfgp2.handle = vol_hdl;
+	rc = leapraid_op_config_page(adapter, &raidvol_p1, cfgp1,
+				     cfgp2, GET_RAID_VOLUME_PG1);
+	if (!rc)
+		*wwid = le64_to_cpu(raidvol_p1.wwid);
+
+	return rc;
+}
+
+static int leapraid_get_sas_io_unit_page0(
+		struct leapraid_adapter *adapter,
+		struct leapraid_sas_io_unit_p0 *sas_io_unit_p0,
+		u16 sas_iou_pg0_sz)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+
+	cfgp1.size = sas_iou_pg0_sz;
+	return leapraid_op_config_page(adapter, sas_io_unit_p0, cfgp1,
+				       cfgp2, GET_SAS_IOUNIT_PG0);
+}
+
+static int leapraid_get_sas_address(struct leapraid_adapter *adapter,
+				    u16 hdl, u64 *sas_address)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_p0;
+
+	*sas_address = 0;
+	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+	cfgp2.handle = hdl;
+	if (leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1,
+				    cfgp2, GET_SAS_DEVICE_PG0))
+		return -ENXIO;
+
+	if (hdl <= adapter->dev_topo.card.phys_num &&
+	    (!(le32_to_cpu(sas_dev_p0.dev_info) & LEAPRAID_DEVTYP_SEP)))
+		*sas_address = adapter->dev_topo.card.sas_address;
+	else
+		*sas_address = le64_to_cpu(sas_dev_p0.sas_address);
+
+	return 0;
+}
+
+int leapraid_get_volume_cap(struct leapraid_adapter *adapter,
+			    struct leapraid_raid_volume *raid_volume)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_raidvol_p0 *raidvol_p0;
+	struct leapraid_sas_dev_p0 sas_dev_p0;
+	struct leapraid_raidpd_p0 raidpd_p0;
+	u8 num_pds;
+	u16 sz;
+	int rc = 0;
+
+	if (leapraid_cfg_get_number_pds(adapter, raid_volume->hdl,
+					&num_pds) || !num_pds)
+		return -EFAULT;
+
+	raid_volume->pd_num = num_pds;
+	sz = offsetof(struct leapraid_raidvol_p0, phys_disk) +
+		(num_pds * sizeof(struct leapraid_raidvol0_phys_disk));
+	raidvol_p0 = kzalloc(sz, GFP_KERNEL);
+	if (!raidvol_p0)
+		return -ENOMEM;
+
+	cfgp1.size = sz;
+	cfgp2.handle = raid_volume->hdl;
+	if (leapraid_op_config_page(adapter, raidvol_p0, cfgp1, cfgp2,
+				    GET_RAID_VOLUME_PG0)) {
+		rc = -EFAULT;
+		goto out_cleanup;
+	}
+
+	raid_volume->vol_type = raidvol_p0->volume_type;
+	cfgp1.form = LEAPRAID_PHYSDISK_CFG_PGAD_PHYSDISKNUM;
+	cfgp2.form_specific = raidvol_p0->phys_disk[0].phys_disk_num;
+	if (!(leapraid_op_config_page(adapter, &raidpd_p0, cfgp1, cfgp2,
+				      GET_PHY_DISK_PG0))) {
+		cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+		cfgp2.handle = le16_to_cpu(raidpd_p0.dev_hdl);
+		if (!(leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1,
+					      cfgp2, GET_SAS_DEVICE_PG0)))
+			raid_volume->dev_info =
+				le32_to_cpu(sas_dev_p0.dev_info);
+	}
+
+out_cleanup:
+	kfree(raidvol_p0);
+	return rc;
+}
+
+static bool leapraid_should_skip_poll_work(struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+	bool skip;
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	skip = adapter->access_ctrl.shost_recovering ||
+		adapter->access_ctrl.pcie_recovering ||
+		adapter->access_ctrl.host_removing;
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+
+	return skip;
+}
+
+static void leapraid_fw_log_work(struct work_struct *work)
+{
+	struct leapraid_adapter *adapter = container_of(work,
+		struct leapraid_adapter, fw_log_desc.fw_log_work.work);
+	struct leapraid_fw_log_info *infom;
+	struct leapraid_reg_base __iomem *iomem_base;
+	unsigned long flags;
+
+	if (leapraid_should_skip_poll_work(adapter))
+		goto scheduled_timer;
+
+	infom = (struct leapraid_fw_log_info *)
+		(adapter->fw_log_desc.fw_log_buffer +
+		 LEAPRAID_SYS_LOG_BUF_SIZE);
+
+	iomem_base = adapter->iomem_base;
+	if (adapter->fw_log_desc.fw_log_init_flag == 0) {
+		infom->user_position =
+			leapraid_readl(&iomem_base->host_log_buf_pos);
+		infom->adapter_position =
+			leapraid_readl(&iomem_base->adapter_log_buf_pos);
+		adapter->fw_log_desc.fw_log_init_flag++;
+	}
+
+	writel(infom->user_position, &iomem_base->host_log_buf_pos);
+	infom->adapter_position =
+		leapraid_readl(&iomem_base->adapter_log_buf_pos);
+
+scheduled_timer:
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	if (adapter->fw_log_desc.fw_log_wq)
+		queue_delayed_work(
+			adapter->fw_log_desc.fw_log_wq,
+			&adapter->fw_log_desc.fw_log_work,
+			msecs_to_jiffies(LEAPRAID_PCIE_LOG_POLLING_INTERVAL));
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+}
+
+void leapraid_fw_log_stop(struct leapraid_adapter *adapter)
+{
+	struct workqueue_struct *wq;
+	unsigned long flags;
+
+	if (!adapter->fw_log_desc.open_pcie_trace)
+		return;
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	wq = adapter->fw_log_desc.fw_log_wq;
+	adapter->fw_log_desc.fw_log_wq = NULL;
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+	if (wq) {
+		if (!cancel_delayed_work_sync(&adapter->fw_log_desc
+					      .fw_log_work))
+			flush_workqueue(wq);
+		destroy_workqueue(wq);
+	}
+}
+
+void leapraid_fw_log_start(struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+
+	if (!adapter->fw_log_desc.open_pcie_trace)
+		return;
+
+	if (adapter->fw_log_desc.fw_log_wq)
+		return;
+
+	INIT_DELAYED_WORK(&adapter->fw_log_desc.fw_log_work,
+			  leapraid_fw_log_work);
+	snprintf(adapter->fw_log_desc.fw_log_wq_name,
+		 sizeof(adapter->fw_log_desc.fw_log_wq_name),
+		 "poll_%s%u_fw_log",
+		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);
+	adapter->fw_log_desc.fw_log_wq =
+		create_singlethread_workqueue(
+			adapter->fw_log_desc.fw_log_wq_name);
+	if (!adapter->fw_log_desc.fw_log_wq)
+		return;
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	if (adapter->fw_log_desc.fw_log_wq)
+		queue_delayed_work(
+			adapter->fw_log_desc.fw_log_wq,
+			&adapter->fw_log_desc.fw_log_work,
+			msecs_to_jiffies(LEAPRAID_PCIE_LOG_POLLING_INTERVAL));
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+}
+
+static void leapraid_check_scheduled_fault_work(struct work_struct *work)
+{
+	struct leapraid_adapter *adapter;
+	unsigned long flags;
+	u32 adapter_state;
+	int rc;
+
+	adapter = container_of(work, struct leapraid_adapter,
+			       reset_desc.fault_reset_work.work);
+
+	if (leapraid_should_skip_poll_work(adapter))
+		goto scheduled_timer;
+
+	adapter_state = leapraid_get_adapter_state(adapter);
+	if (adapter_state != LEAPRAID_DB_OPERATIONAL) {
+		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset 0x%x\n",
+			 __func__, __LINE__, adapter_state);
+		rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
+		adapter_state = leapraid_get_adapter_state(adapter);
+		if (rc && adapter_state != LEAPRAID_DB_OPERATIONAL) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Hard reset failed, state=0x%x rc=%d\n",
+				__func__, adapter_state, rc);
+			return;
+		}
+	}
+
+scheduled_timer:
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	if (adapter->reset_desc.fault_reset_wq)
+		queue_delayed_work(
+			adapter->reset_desc.fault_reset_wq,
+			&adapter->reset_desc.fault_reset_work,
+			msecs_to_jiffies(LEAPRAID_FAULT_POLLING_INTERVAL));
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+}
+
+void leapraid_check_scheduled_fault_start(struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+
+	if (adapter->reset_desc.fault_reset_wq)
+		return;
+
+	INIT_DELAYED_WORK(&adapter->reset_desc.fault_reset_work,
+			  leapraid_check_scheduled_fault_work);
+	snprintf(adapter->reset_desc.fault_reset_wq_name,
+		 sizeof(adapter->reset_desc.fault_reset_wq_name),
+		 "poll_%s%u_status",
+		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);
+	adapter->reset_desc.fault_reset_wq =
+		create_singlethread_workqueue(
+			adapter->reset_desc.fault_reset_wq_name);
+	if (!adapter->reset_desc.fault_reset_wq) {
+		dev_err(&adapter->pdev->dev,
+			"Create single thread workqueue failed!\n");
+		return;
+	}
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	if (adapter->reset_desc.fault_reset_wq)
+		queue_delayed_work(
+			adapter->reset_desc.fault_reset_wq,
+			&adapter->reset_desc.fault_reset_work,
+			msecs_to_jiffies(LEAPRAID_FAULT_POLLING_INTERVAL));
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+}
+
+void leapraid_check_scheduled_fault_stop(struct leapraid_adapter *adapter)
+{
+	struct workqueue_struct *wq;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	wq = adapter->reset_desc.fault_reset_wq;
+	adapter->reset_desc.fault_reset_wq = NULL;
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+
+	if (!wq)
+		return;
+
+	if (!cancel_delayed_work_sync(&adapter->reset_desc.fault_reset_work))
+		flush_workqueue(wq);
+	destroy_workqueue(wq);
+}
+
+static void leapraid_overheat_work(struct work_struct *work)
+{
+	struct leapraid_overheat_desc *desc;
+	struct leapraid_adapter *adapter;
+	struct workqueue_struct *wq;
+	struct Scsi_Host *shost;
+	struct pci_dev *pdev;
+	unsigned long flags;
+
+	desc = container_of(work, struct leapraid_overheat_desc,
+			    fault_overheat_work);
+	adapter = container_of(desc, struct leapraid_adapter, overheat_desc);
+	pdev = adapter->pdev;
+	shost = pci_get_drvdata(pdev);
+
+	if (!shost) {
+		dev_err(&pdev->dev,
+			"Overheat processing failed: invalid host/adapter\n");
+		atomic_set(&adapter->overheat_desc.thermal_alert, 0);
+		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+		return;
+	}
+
+	if (adapter->access_ctrl.host_removing) {
+		atomic_set(&adapter->overheat_desc.thermal_alert, 0);
+		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+		return;
+	}
+
+	adapter->access_ctrl.host_removing = 1;
+	adapter->access_ctrl.shost_recover_async = 0;
+	adapter->scan_dev_desc.scan_start = 0;
+	adapter->scan_dev_desc.wait_scan_dev_done = 0;
+	adapter->scan_dev_desc.driver_loading = 0;
+	wake_up(&adapter->access_ctrl.shost_recover_wq);
+	wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+
+	leapraid_mask_int(adapter);
+
+	leapraid_check_scheduled_fault_stop(adapter);
+	leapraid_fw_log_stop(adapter);
+	leapraid_mq_polling_pause(adapter);
+
+	leapraid_clean_active_cmds(adapter);
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	wq = adapter->fw_evt_s.fw_evt_thread;
+	adapter->fw_evt_s.fw_evt_thread = NULL;
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (wq)
+		destroy_workqueue(wq);
+
+	sas_remove_host(shost);
+	leapraid_cleanup_lists(adapter);
+	atomic_set(&adapter->overheat_desc.thermal_alert, 0);
+	wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+	dev_err(&pdev->dev, "%s: Suspend adapter due to overheat\n", __func__);
+}
+
+static void leapraid_overheat_init(struct leapraid_adapter *adapter)
+{
+	if (adapter->overheat_desc.fault_overheat_wq)
+		return;
+
+	atomic_set(&adapter->overheat_desc.thermal_alert, 0);
+	snprintf(adapter->overheat_desc.fault_overheat_wq_name,
+		 sizeof(adapter->overheat_desc.fault_overheat_wq_name),
+		 "driver_%s%u_overheat",
+		 LEAPRAID_DRIVER_NAME,
+		 adapter->adapter_attr.id);
+	adapter->overheat_desc.fault_overheat_wq =
+		create_singlethread_workqueue(
+			adapter->overheat_desc.fault_overheat_wq_name);
+	if (!adapter->overheat_desc.fault_overheat_wq) {
+		dev_err(&adapter->pdev->dev,
+			"Failed to create overheat workqueue\n");
+		return;
+	}
+
+	INIT_WORK(&adapter->overheat_desc.fault_overheat_work,
+		  leapraid_overheat_work);
+}
+
+void leapraid_overheat_cleanup(struct leapraid_adapter *adapter)
+{
+	struct workqueue_struct *wq;
+
+	wq = xchg(&adapter->overheat_desc.fault_overheat_wq, NULL);
+	if (!wq)
+		return;
+
+	cancel_work_sync(&adapter->overheat_desc.fault_overheat_work);
+	destroy_workqueue(wq);
+}
+
+static void leapraid_fw_work(struct leapraid_adapter *adapter,
+			     struct leapraid_fw_evt_work *fw_evt);
+
+static void leapraid_fw_evt_free(struct kref *r)
+{
+	struct leapraid_fw_evt_work *fw_evt;
+
+	fw_evt = container_of(r, struct leapraid_fw_evt_work, refcnt);
+
+	kfree(fw_evt->evt_data);
+	kfree(fw_evt);
+}
+
+static void leapraid_fw_evt_get(struct leapraid_fw_evt_work *fw_evt)
+{
+	kref_get(&fw_evt->refcnt);
+}
+
+static void leapraid_fw_evt_put(struct leapraid_fw_evt_work *fw_work)
+{
+	kref_put(&fw_work->refcnt, leapraid_fw_evt_free);
+}
+
+static struct leapraid_fw_evt_work *leapraid_alloc_fw_evt_work(void)
+{
+	struct leapraid_fw_evt_work *fw_evt = kzalloc_obj(*fw_evt, GFP_ATOMIC);
+
+	if (fw_evt)
+		kref_init(&fw_evt->refcnt);
+
+	return fw_evt;
+}
+
+static void leapraid_run_fw_evt_work(struct work_struct *work)
+{
+	struct leapraid_fw_evt_work *fw_evt =
+		container_of(work, struct leapraid_fw_evt_work, work);
+
+	leapraid_fw_work(fw_evt->adapter, fw_evt);
+}
+
+static void leapraid_fw_evt_add(struct leapraid_adapter *adapter,
+				struct leapraid_fw_evt_work *fw_evt)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering ||
+	    !adapter->fw_evt_s.fw_evt_thread) {
+		spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+		return;
+	}
+
+	leapraid_fw_evt_get(fw_evt);
+	INIT_LIST_HEAD(&fw_evt->list);
+	list_add_tail(&fw_evt->list, &adapter->fw_evt_s.fw_evt_list);
+	INIT_WORK(&fw_evt->work, leapraid_run_fw_evt_work);
+	leapraid_fw_evt_get(fw_evt);
+	queue_work(adapter->fw_evt_s.fw_evt_thread, &fw_evt->work);
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+}
+
+static void leapraid_del_fw_evt_from_list(struct leapraid_adapter *adapter,
+					  struct leapraid_fw_evt_work *fw_evt)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (!list_empty(&fw_evt->list)) {
+		list_del_init(&fw_evt->list);
+		leapraid_fw_evt_put(fw_evt);
+	}
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+}
+
+static struct leapraid_fw_evt_work *leapraid_next_fw_evt(
+		struct leapraid_adapter *adapter)
+{
+	struct leapraid_fw_evt_work *fw_evt = NULL;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (!list_empty(&adapter->fw_evt_s.fw_evt_list)) {
+		fw_evt = list_first_entry(&adapter->fw_evt_s.fw_evt_list,
+					  struct leapraid_fw_evt_work, list);
+		list_del_init(&fw_evt->list);
+	}
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+	return fw_evt;
+}
+
+void leapraid_clean_active_fw_evt(struct leapraid_adapter *adapter)
+{
+	struct leapraid_fw_evt_work *fw_evt;
+	unsigned long flags;
+	bool in_fw_evt_context;
+	bool rc;
+
+	if ((list_empty(&adapter->fw_evt_s.fw_evt_list) &&
+	     !adapter->fw_evt_s.cur_evt) || !adapter->fw_evt_s.fw_evt_thread)
+		return;
+
+	adapter->fw_evt_s.fw_evt_cleanup = 1;
+	if (adapter->access_ctrl.shost_recovering &&
+	    adapter->fw_evt_s.cur_evt)
+		adapter->fw_evt_s.cur_evt->ignore = 1;
+
+	while ((fw_evt = leapraid_next_fw_evt(adapter))) {
+		rc = cancel_work_sync(&fw_evt->work);
+		if (rc)
+			leapraid_fw_evt_put(fw_evt);
+		leapraid_fw_evt_put(fw_evt);
+	}
+
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	fw_evt = adapter->fw_evt_s.cur_evt;
+	if (fw_evt) {
+		in_fw_evt_context = adapter->fw_evt_s.cur_evt_task == current;
+		leapraid_fw_evt_get(fw_evt);
+	}
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (fw_evt) {
+		if (!in_fw_evt_context)
+			cancel_work_sync(&fw_evt->work);
+		leapraid_fw_evt_put(fw_evt);
+	}
+
+	adapter->fw_evt_s.fw_evt_cleanup = 0;
+}
+
+static void leapraid_internal_dev_ublk(struct scsi_device *sdev,
+				       struct leapraid_sdev_priv *sdev_priv)
+{
+	int rc;
+
+	sdev_printk(KERN_WARNING, sdev,
+		    "hdl 0x%04x: Now internal unblkg dev\n",
+		    sdev_priv->starget_priv->hdl);
+	sdev_priv->block = 0;
+	rc = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
+	if (rc == -EINVAL) {
+		sdev_printk(KERN_WARNING, sdev,
+			    "hdl 0x%04x: unblkg failed, rc=%d\n",
+			    sdev_priv->starget_priv->hdl, rc);
+		sdev_priv->block = 1;
+		rc = scsi_internal_device_block_nowait(sdev);
+		if (rc)
+			sdev_printk(KERN_WARNING, sdev,
+				    "hdl 0x%04x: Earlier ublkg err, rc=%d\n",
+				    sdev_priv->starget_priv->hdl, rc);
+
+		sdev_priv->block = 0;
+		rc = scsi_internal_device_unblock_nowait(sdev, SDEV_RUNNING);
+		if (rc)
+			sdev_printk(KERN_WARNING, sdev,
+				    "hdl 0x%04x: ublkg failed again, rc=%d\n",
+				    sdev_priv->starget_priv->hdl, rc);
+	}
+}
+
+static void leapraid_ublk_io_dev(struct leapraid_adapter *adapter,
+				 u64 sas_addr,
+				 struct leapraid_card_port *card_port)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct scsi_device *sdev;
+
+	shost_for_each_device(sdev, adapter->shost) {
+		sdev_priv = sdev->hostdata;
+		if (!sdev_priv || !sdev_priv->starget_priv)
+			continue;
+
+		if (sdev_priv->starget_priv->sas_address != sas_addr)
+			continue;
+
+		if (sdev_priv->starget_priv->card_port != card_port)
+			continue;
+
+		if (sdev_priv->block)
+			leapraid_internal_dev_ublk(sdev, sdev_priv);
+
+		scsi_device_set_state(sdev, SDEV_OFFLINE);
+	}
+}
+
+static void leapraid_ublk_io_all_dev(struct leapraid_adapter *adapter)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct leapraid_starget_priv *stgt_priv;
+	struct scsi_device *sdev;
+
+	shost_for_each_device(sdev, adapter->shost) {
+		sdev_priv = sdev->hostdata;
+
+		if (!sdev_priv)
+			continue;
+
+		stgt_priv = sdev_priv->starget_priv;
+		if (!stgt_priv || stgt_priv->deleted)
+			continue;
+
+		if (!sdev_priv->block)
+			continue;
+
+		sdev_printk(KERN_WARNING, sdev, "hdl 0x%04x: blkg...\n",
+			    sdev_priv->starget_priv->hdl);
+		leapraid_internal_dev_ublk(sdev, sdev_priv);
+		continue;
+	}
+}
+
+static void leapraid_internal_dev_blk(
+		struct scsi_device *sdev,
+		struct leapraid_sdev_priv *sdev_priv)
+{
+	int rc;
+
+	sdev_printk(KERN_INFO, sdev, "Internal blkg hdl 0x%04x\n",
+		    sdev_priv->starget_priv->hdl);
+	sdev_priv->block = 1;
+	rc = scsi_internal_device_block_nowait(sdev);
+	if (rc == -EINVAL)
+		sdev_printk(KERN_WARNING, sdev,
+			    "hdl 0x%04x: blkg failed, rc=%d\n",
+			    rc, sdev_priv->starget_priv->hdl);
+}
+
+static void leapraid_imm_blkio_to_end_dev(struct leapraid_adapter *adapter,
+					  struct leapraid_sas_port *sas_port)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct leapraid_sas_dev *sas_dev;
+	struct scsi_device *sdev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(
+		adapter,
+		sas_port->remote_identify.sas_address,
+		sas_port->card_port);
+
+	if (sas_dev) {
+		shost_for_each_device(sdev, adapter->shost) {
+			sdev_priv = sdev->hostdata;
+			if (!sdev_priv)
+				continue;
+
+			if (sdev_priv->starget_priv->hdl != sas_dev->hdl)
+				continue;
+
+			if (sdev_priv->block)
+				continue;
+
+			if (sas_dev && sas_dev->pend_sas_rphy_add)
+				continue;
+
+			if (sdev_priv->sep) {
+				sdev_printk(KERN_INFO, sdev,
+					    "skip dev blk: sep hdl 0x%04x\n",
+					    sdev_priv->starget_priv->hdl);
+				continue;
+			}
+
+			leapraid_internal_dev_blk(sdev, sdev_priv);
+		}
+
+		leapraid_sdev_put(sas_dev);
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+}
+
+static void leapraid_imm_blkio_set_end_dev_blk_hdls(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node_exp)
+{
+	struct leapraid_sas_port *sas_port;
+
+	list_for_each_entry(sas_port,
+			    &topo_node_exp->sas_port_list, port_list) {
+		if (sas_port->remote_identify.device_type == SAS_END_DEVICE)
+			leapraid_imm_blkio_to_end_dev(adapter, sas_port);
+	}
+}
+
+static void leapraid_imm_blkio_to_kids_attchd_to_ex(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node_exp);
+
+static void leapraid_imm_blkio_to_sib_exp(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node_exp)
+{
+	struct leapraid_topo_node *topo_node_exp_sib;
+	struct leapraid_sas_port *sas_port;
+
+	list_for_each_entry(sas_port,
+			    &topo_node_exp->sas_port_list, port_list) {
+		if (sas_port->remote_identify.device_type ==
+			 SAS_EDGE_EXPANDER_DEVICE ||
+		    sas_port->remote_identify.device_type ==
+			 SAS_FANOUT_EXPANDER_DEVICE) {
+			topo_node_exp_sib =
+				leapraid_exp_find_by_sas_address(
+					adapter,
+					sas_port->remote_identify.sas_address,
+					sas_port->card_port);
+			leapraid_imm_blkio_to_kids_attchd_to_ex(
+				adapter,
+				topo_node_exp_sib);
+		}
+	}
+}
+
+static void leapraid_imm_blkio_to_kids_attchd_to_ex(
+				struct leapraid_adapter *adapter,
+				struct leapraid_topo_node *topo_node_exp)
+{
+	if (!topo_node_exp)
+		return;
+
+	leapraid_imm_blkio_set_end_dev_blk_hdls(adapter, topo_node_exp);
+
+	leapraid_imm_blkio_to_sib_exp(adapter, topo_node_exp);
+}
+
+static void leapraid_report_sdev_directly(struct leapraid_adapter *adapter,
+					  struct leapraid_sas_dev *sas_dev)
+{
+	struct leapraid_sas_port *sas_port;
+
+	sas_port = leapraid_transport_port_add(adapter,
+					       sas_dev->hdl,
+					       sas_dev->parent_sas_addr,
+					       sas_dev->card_port);
+	if (!sas_port) {
+		leapraid_sas_dev_remove(adapter, sas_dev);
+		return;
+	}
+
+	if (!sas_dev->starget) {
+		if (!adapter->scan_dev_desc.driver_loading) {
+			leapraid_transport_port_remove(
+				adapter,
+				sas_dev->sas_addr,
+				sas_dev->parent_sas_addr,
+				sas_dev->card_port);
+			leapraid_sas_dev_remove(adapter, sas_dev);
+		}
+		return;
+	}
+}
+
+static struct leapraid_sas_dev *leapraid_init_sas_dev(
+		struct leapraid_adapter *adapter,
+		struct leapraid_sas_dev_p0 *sas_dev_pg0,
+		struct leapraid_card_port *card_port, u16 hdl,
+		u64 parent_sas_addr, u64 sas_addr, u32 dev_info)
+{
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_enc_node *enc_dev;
+	unsigned long flags;
+
+	sas_dev = kzalloc_obj(*sas_dev);
+	if (!sas_dev)
+		return NULL;
+
+	kref_init(&sas_dev->refcnt);
+	sas_dev->hdl = hdl;
+	sas_dev->dev_info = dev_info;
+	sas_dev->sas_addr = sas_addr;
+	sas_dev->card_port = card_port;
+	sas_dev->parent_sas_addr = parent_sas_addr;
+	sas_dev->phy = sas_dev_pg0->phy_num;
+	sas_dev->enc_hdl = le16_to_cpu(sas_dev_pg0->enc_hdl);
+	sas_dev->dev_name = le64_to_cpu(sas_dev_pg0->dev_name);
+	sas_dev->port_connection = sas_dev_pg0->max_port_connections;
+	sas_dev->slot = sas_dev->enc_hdl ? le16_to_cpu(sas_dev_pg0->slot) : 0;
+	if (le16_to_cpu(sas_dev_pg0->flg) &
+	    LEAPRAID_SAS_DEV_P0_FLG_ENC_LEVEL_VALID) {
+		sas_dev->enc_level = sas_dev_pg0->enc_level;
+		memcpy(sas_dev->connector_name,
+		       sas_dev_pg0->connector_name,
+		       LEAPRAID_SAS_DEV_P0_CON_NAME_LEN);
+		sas_dev->connector_name[LEAPRAID_SAS_DEV_P0_CON_NAME_LEN] =
+			'\0';
+	} else {
+		sas_dev->enc_level = 0;
+		sas_dev->connector_name[0] = '\0';
+	}
+	if (sas_dev->enc_hdl) {
+		spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
+		enc_dev = leapraid_enc_find_by_hdl(adapter, sas_dev->enc_hdl);
+		if (enc_dev)
+			sas_dev->enc_lid = le64_to_cpu(enc_dev->pg0.enc_lid);
+		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+	}
+	dev_info(&adapter->pdev->dev,
+		 "add dev: hdl=0x%x, SAS addr=0x%016llx, port connect=0x%x\n",
+		 hdl, sas_dev->sas_addr, sas_dev->port_connection);
+
+	return sas_dev;
+}
+
+static void leapraid_add_dev(struct leapraid_adapter *adapter, u16 hdl)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_pg0;
+	struct leapraid_card_port *card_port;
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+	u64 parent_sas_addr;
+	u32 dev_info;
+	u64 sas_addr;
+	u8 port_id;
+
+	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+	cfgp2.handle = hdl;
+	if (leapraid_op_config_page(adapter, &sas_dev_pg0,
+				    cfgp1, cfgp2, GET_SAS_DEVICE_PG0))
+		return;
+
+	dev_info = le32_to_cpu(sas_dev_pg0.dev_info);
+	if (!(leapraid_is_end_dev(dev_info)))
+		return;
+
+	sas_addr = le64_to_cpu(sas_dev_pg0.sas_address);
+	if (!(le16_to_cpu(sas_dev_pg0.flg) &
+	      LEAPRAID_SAS_DEV_P0_FLG_DEV_PRESENT))
+		return;
+
+	port_id = sas_dev_pg0.physical_port;
+	card_port = leapraid_get_port_by_id(adapter, port_id, false);
+	if (!card_port)
+		return;
+
+	sas_dev = leapraid_get_sas_dev_by_addr(adapter, sas_addr, card_port);
+	if (sas_dev) {
+		leapraid_sdev_put(sas_dev);
+		return;
+	}
+
+	if (leapraid_get_sas_address(adapter,
+				     le16_to_cpu(sas_dev_pg0.parent_dev_hdl),
+				     &parent_sas_addr))
+		return;
+
+	sas_dev = leapraid_init_sas_dev(adapter, &sas_dev_pg0, card_port,
+					hdl, parent_sas_addr, sas_addr,
+					dev_info);
+	if (!sas_dev)
+		return;
+
+	if (adapter->scan_dev_desc.wait_scan_dev_done) {
+		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+		leapraid_sdev_get(sas_dev);
+		list_add_tail(&sas_dev->list,
+			      &adapter->dev_topo.sas_dev_init_list);
+		leapraid_check_boot_dev(adapter, sas_dev, 0);
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	} else {
+		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+		leapraid_sdev_get(sas_dev);
+		list_add_tail(&sas_dev->list, &adapter->dev_topo.sas_dev_list);
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		leapraid_report_sdev_directly(adapter, sas_dev);
+	}
+	leapraid_sdev_put(sas_dev);
+}
+
+static void leapraid_remove_device(struct leapraid_adapter *adapter,
+				   struct leapraid_sas_dev *sas_dev)
+{
+	struct leapraid_starget_priv *starget_priv;
+
+	leapraid_clear_cached_boot_dev(adapter, sas_dev, 0);
+	if (sas_dev->led_on) {
+		leapraid_set_led(adapter, sas_dev, false);
+		sas_dev->led_on = 0;
+	}
+
+	if (sas_dev->starget && sas_dev->starget->hostdata) {
+		starget_priv = sas_dev->starget->hostdata;
+		starget_priv->deleted = 1;
+		leapraid_ublk_io_dev(adapter,
+				     sas_dev->sas_addr, sas_dev->card_port);
+		starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;
+	}
+
+	leapraid_transport_port_remove(adapter,
+				       sas_dev->sas_addr,
+				       sas_dev->parent_sas_addr,
+				       sas_dev->card_port);
+
+	dev_info(&adapter->pdev->dev,
+		 "remove dev: hdl=0x%04x, SAS addr=0x%016llx\n",
+		 sas_dev->hdl, (unsigned long long)sas_dev->sas_addr);
+}
+
+static struct leapraid_vphy *leapraid_alloc_vphy(
+		struct leapraid_adapter *adapter,
+		u8 port_id, u8 phy_num)
+{
+	struct leapraid_card_port *port;
+	struct leapraid_vphy *vphy;
+
+	port = leapraid_get_port_by_id(adapter, port_id, false);
+	if (!port)
+		return NULL;
+
+	vphy = leapraid_get_vphy_by_phy(port, phy_num);
+	if (vphy)
+		return vphy;
+	vphy = kzalloc_obj(*vphy);
+	if (!vphy)
+		return NULL;
+
+	if (!port->vphys_mask)
+		INIT_LIST_HEAD(&port->vphys_list);
+
+	port->vphys_mask |= BIT(phy_num);
+	vphy->phy_mask |= BIT(phy_num);
+	list_add_tail(&vphy->list, &port->vphys_list);
+	return vphy;
+}
+
+static int leapraid_add_port_to_card_port_list(
+					struct leapraid_adapter *adapter,
+					u8 port_id, bool refresh)
+{
+	struct leapraid_card_port *card_port;
+
+	card_port = leapraid_get_port_by_id(adapter, port_id, false);
+	if (card_port)
+		return 0;
+	card_port = kzalloc_obj(*card_port);
+	if (!card_port)
+		return -ENOMEM;
+
+	card_port->port_id = port_id;
+	dev_dbg(&adapter->pdev->dev,
+		"port: %d is added to card_port list\n",
+		card_port->port_id);
+
+	if (refresh && adapter->access_ctrl.shost_recovering)
+		card_port->flg = LEAPRAID_CARD_PORT_FLG_NEW;
+	list_add_tail(&card_port->list, &adapter->dev_topo.card_port_list);
+	return 0;
+}
+
+static int leapraid_add_card_phy(struct leapraid_adapter *adapter,
+				 struct leapraid_sas_io_unit_p0 *iou,
+				 int i)
+{
+	struct leapraid_sas_phy_p0 phy_pg0;
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_topo_node *card = &adapter->dev_topo.card;
+	u8 port_id;
+
+	cfgp1.phy_number = i;
+
+	if (leapraid_op_config_page(adapter, &phy_pg0,
+				    cfgp1, cfgp2, GET_PHY_PG0))
+		return -EINVAL;
+
+	port_id = iou->phy_info[i].port;
+
+	if (leapraid_add_port_to_card_port_list(adapter, port_id, false))
+		return -EINVAL;
+
+	if ((le32_to_cpu(phy_pg0.phy_info) &
+	     LEAPRAID_SAS_PHYINFO_VPHY) &&
+	    ((phy_pg0.neg_link_rate >>
+	      LEAPRAID_SAS_NEG_LINK_RATE_SHIFT) >=
+	     LEAPRAID_SAS_NEG_LINK_RATE_1_5)) {
+		if (!leapraid_alloc_vphy(adapter, port_id, i))
+			return -ENOMEM;
+
+		card->card_phy[i].vphy = 1;
+	}
+
+	card->card_phy[i].hdl = card->hdl;
+	card->card_phy[i].phy_id = i;
+	card->card_phy[i].card_port =
+		leapraid_get_port_by_id(adapter, port_id, false);
+
+	leapraid_transport_add_card_phy(adapter,
+					&card->card_phy[i],
+					&phy_pg0,
+					card->parent_dev);
+
+	return 0;
+}
+
+static int leapraid_refresh_card_phy(struct leapraid_adapter *adapter,
+				     struct leapraid_sas_io_unit_p0 *iou,
+				     int i)
+{
+	struct leapraid_topo_node *card = &adapter->dev_topo.card;
+	struct leapraid_sas_phy_p0 phy_pg0;
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	u16 attached_hdl;
+	u32 dev_info;
+	u8 link_rate;
+	u8 port_id;
+
+	link_rate = iou->phy_info[i].neg_link_rate >>
+		LEAPRAID_SAS_NEG_LINK_RATE_SHIFT;
+	port_id = iou->phy_info[i].port;
+	if (leapraid_add_port_to_card_port_list(adapter, port_id, true))
+		return -EINVAL;
+
+	dev_info = le32_to_cpu(iou->phy_info[i].controller_phy_dev_info);
+	if (dev_info & LEAPRAID_DEVTYP_SEP &&
+	    link_rate >= LEAPRAID_SAS_NEG_LINK_RATE_1_5) {
+		cfgp1.phy_number = i;
+
+		if (leapraid_op_config_page(adapter, &phy_pg0,
+					    cfgp1, cfgp2, GET_PHY_PG0))
+			return 0;
+
+		if (le32_to_cpu(phy_pg0.phy_info) &
+		    LEAPRAID_SAS_PHYINFO_VPHY) {
+			if (!leapraid_alloc_vphy(adapter, port_id, i))
+				return -ENOMEM;
+			card->card_phy[i].vphy = 1;
+		}
+	}
+
+	card->card_phy[i].hdl = card->hdl;
+
+	attached_hdl = le16_to_cpu(iou->phy_info[i].attached_dev_hdl);
+	if (attached_hdl && link_rate < LEAPRAID_SAS_NEG_LINK_RATE_1_5)
+		link_rate = LEAPRAID_SAS_NEG_LINK_RATE_1_5;
+
+	card->card_phy[i].card_port =
+		leapraid_get_port_by_id(adapter, port_id, false);
+
+	if (!card->card_phy[i].phy) {
+		cfgp1.phy_number = i;
+		if (leapraid_op_config_page(adapter, &phy_pg0,
+					    cfgp1, cfgp2, GET_PHY_PG0))
+			return 0;
+
+		card->card_phy[i].phy_id = i;
+
+		leapraid_transport_add_card_phy(adapter,
+						&card->card_phy[i],
+						&phy_pg0,
+						card->parent_dev);
+		return 0;
+	}
+
+	leapraid_transport_update_links(adapter,
+					card->sas_address,
+					attached_hdl,
+					i,
+					link_rate,
+					card->card_phy[i].card_port);
+
+	return 0;
+}
+
+static void leapraid_sas_host_add(struct leapraid_adapter *adapter,
+				  bool refresh)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_pg0;
+	struct leapraid_enc_p0 enc_pg0;
+	struct leapraid_sas_io_unit_p0 *sas_iou_pg0;
+	u16 sas_iou_pg0_sz;
+	u8 phys_num;
+	int i;
+	int rc;
+
+	if (!refresh) {
+		if (leapraid_get_adapter_phys(adapter, &phys_num) || !phys_num)
+			return;
+
+		adapter->dev_topo.card.card_phy =
+			kzalloc_objs(struct leapraid_card_phy, phys_num);
+		if (!adapter->dev_topo.card.card_phy)
+			return;
+
+		adapter->dev_topo.card.phys_num = phys_num;
+	}
+
+	sas_iou_pg0_sz =
+		offsetof(struct leapraid_sas_io_unit_p0, phy_info) +
+			 (adapter->dev_topo.card.phys_num *
+			  sizeof(struct leapraid_sas_io_unit0_phy_info));
+	sas_iou_pg0 = kzalloc(sas_iou_pg0_sz, GFP_KERNEL);
+	if (!sas_iou_pg0)
+		return;
+
+	if (leapraid_get_sas_io_unit_page0(adapter,
+					   sas_iou_pg0,
+					   sas_iou_pg0_sz))
+		goto out_free;
+
+	adapter->dev_topo.card.parent_dev = &adapter->shost->shost_gendev;
+	adapter->dev_topo.card.hdl =
+		le16_to_cpu(sas_iou_pg0->phy_info[0].controller_dev_hdl);
+	for (i = 0; i < adapter->dev_topo.card.phys_num; i++) {
+		if (!refresh) /* add */
+			rc = leapraid_add_card_phy(adapter, sas_iou_pg0, i);
+		else /* refresh */
+			rc = leapraid_refresh_card_phy(adapter,
+						       sas_iou_pg0,
+						       i);
+		if (rc)
+			goto out_free;
+	}
+
+	if (!refresh) {
+		cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+		cfgp2.handle = adapter->dev_topo.card.hdl;
+		if (leapraid_op_config_page(adapter, &sas_dev_pg0, cfgp1,
+					    cfgp2, GET_SAS_DEVICE_PG0))
+			goto out_free;
+
+		adapter->dev_topo.card.enc_hdl =
+			le16_to_cpu(sas_dev_pg0.enc_hdl);
+		adapter->dev_topo.card.sas_address =
+			le64_to_cpu(sas_dev_pg0.sas_address);
+		dev_info(&adapter->pdev->dev,
+			 "add host: hdl=0x%04x, SAS addr=0x%016llx, phy=%d\n",
+			 adapter->dev_topo.card.hdl,
+			 (unsigned long long)adapter->dev_topo.card.sas_address,
+			 adapter->dev_topo.card.phys_num);
+
+		if (adapter->dev_topo.card.enc_hdl) {
+			cfgp1.form = LEAPRAID_SAS_ENC_CFG_PGAD_HDL;
+			cfgp2.handle = adapter->dev_topo.card.enc_hdl;
+			if (!(leapraid_op_config_page(adapter, &enc_pg0,
+						      cfgp1, cfgp2,
+						      GET_SAS_ENCLOSURE_PG0)))
+				adapter->dev_topo.card.enc_lid =
+					le64_to_cpu(enc_pg0.enc_lid);
+		}
+	}
+out_free:
+	kfree(sas_iou_pg0);
+}
+
+static int leapraid_internal_exp_add(struct leapraid_adapter *adapter,
+				     struct leapraid_exp_p0 *exp_pg0,
+				     union cfg_param_1 *cfgp1,
+				     union cfg_param_2 *cfgp2,
+				     u16 hdl)
+{
+	struct leapraid_topo_node *topo_node_exp;
+	struct leapraid_sas_port *sas_port = NULL;
+	struct leapraid_enc_node *enc_dev;
+	struct leapraid_exp_p1 exp_pg1;
+	int ret;
+	int rc;
+	unsigned long flags;
+	u8 port_id;
+	u16 parent_handle;
+	u64 sas_addr_parent;
+	int i;
+
+	port_id = exp_pg0->physical_port;
+	parent_handle = le16_to_cpu(exp_pg0->parent_dev_hdl);
+
+	rc = leapraid_get_sas_address(adapter,
+				      parent_handle, &sas_addr_parent);
+	if (rc)
+		return rc;
+	topo_node_exp = kzalloc_obj(*topo_node_exp);
+	if (!topo_node_exp)
+		return -ENOMEM;
+
+	topo_node_exp->hdl = hdl;
+	topo_node_exp->phys_num = exp_pg0->phy_num;
+	topo_node_exp->sas_address_parent = sas_addr_parent;
+	topo_node_exp->sas_address = le64_to_cpu(exp_pg0->sas_address);
+	topo_node_exp->card_port =
+		leapraid_get_port_by_id(adapter, port_id, false);
+	if (!topo_node_exp->card_port) {
+		rc = -EPERM;
+		goto out_fail;
+	}
+
+	dev_info(&adapter->pdev->dev,
+		 "add exp: saddr=0x%016llx, hdl=0x%04x, phdl=0x%04x, phy=%d\n",
+		 (unsigned long long)topo_node_exp->sas_address,
+		 hdl, parent_handle,
+		 topo_node_exp->phys_num);
+	if (!topo_node_exp->phys_num) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid PHY num from exp_pg0\n", __func__);
+		rc = -EPERM;
+		goto out_fail;
+	}
+
+	topo_node_exp->card_phy =
+		kzalloc_objs(struct leapraid_card_phy, topo_node_exp->phys_num);
+	if (!topo_node_exp->card_phy) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to alloc expander phy array, count=%u\n",
+			__func__, topo_node_exp->phys_num);
+		rc = -EPERM;
+		goto out_fail;
+	}
+
+	INIT_LIST_HEAD(&topo_node_exp->sas_port_list);
+	sas_port = leapraid_transport_port_add(adapter, hdl, sas_addr_parent,
+					       topo_node_exp->card_port);
+	if (!sas_port) {
+		rc = -EPERM;
+		goto out_fail;
+	}
+
+	topo_node_exp->parent_dev = &sas_port->rphy->dev;
+	topo_node_exp->rphy = sas_port->rphy;
+	for (i = 0; i < topo_node_exp->phys_num; i++) {
+		cfgp1->phy_number = i;
+		cfgp2->handle = hdl;
+		if (leapraid_op_config_page(adapter, &exp_pg1, *cfgp1, *cfgp2,
+					    GET_SAS_EXPANDER_PG1)) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Failed to get exp_pg1, phy=%d\n",
+				__func__, i);
+			rc = -EPERM;
+			goto out_fail;
+		}
+
+		topo_node_exp->card_phy[i].hdl = hdl;
+		topo_node_exp->card_phy[i].phy_id = i;
+		topo_node_exp->card_phy[i].card_port =
+			leapraid_get_port_by_id(adapter, port_id, false);
+		ret = leapraid_transport_add_exp_phy(
+				adapter,
+				&topo_node_exp->card_phy[i],
+				&exp_pg1,
+				topo_node_exp->parent_dev);
+		if (ret) {
+			rc = -EPERM;
+			goto out_fail;
+		}
+	}
+
+	if (topo_node_exp->enc_hdl) {
+		spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
+		enc_dev = leapraid_enc_find_by_hdl(adapter,
+						   topo_node_exp->enc_hdl);
+		if (enc_dev)
+			topo_node_exp->enc_lid =
+				le64_to_cpu(enc_dev->pg0.enc_lid);
+		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	list_add_tail(&topo_node_exp->list, &adapter->dev_topo.exp_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+	return 0;
+
+out_fail:
+	if (sas_port)
+		leapraid_transport_port_remove(adapter,
+					       topo_node_exp->sas_address,
+					       sas_addr_parent,
+					       topo_node_exp->card_port);
+	kfree(topo_node_exp->card_phy);
+	kfree(topo_node_exp);
+	return rc;
+}
+
+static int leapraid_exp_add(struct leapraid_adapter *adapter, u16 hdl)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_topo_node *topo_node_exp;
+	struct leapraid_exp_p0 exp_pg0;
+	u16 parent_handle;
+	u64 sas_addr, sas_addr_parent;
+	unsigned long flags;
+	u8 port_id;
+	int rc;
+
+	if (!hdl) {
+		dev_warn(&adapter->pdev->dev, "%s: Invalid hdl\n", __func__);
+		return -EPERM;
+	}
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.pcie_recovering) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed, shost_recovering=%d pcie_recovering=%d\n",
+			 __func__, adapter->access_ctrl.shost_recovering,
+			adapter->access_ctrl.pcie_recovering);
+		return -EPERM;
+	}
+
+	cfgp1.form = LEAPRAID_SAS_EXP_CFD_PGAD_HDL;
+	cfgp2.handle = hdl;
+	if (leapraid_op_config_page(adapter, &exp_pg0, cfgp1, cfgp2,
+				    GET_SAS_EXPANDER_PG0))
+		return -EPERM;
+
+	parent_handle = le16_to_cpu(exp_pg0.parent_dev_hdl);
+	if (leapraid_get_sas_address(adapter, parent_handle, &sas_addr_parent))
+		return -EPERM;
+
+	port_id = exp_pg0.physical_port;
+	if (sas_addr_parent != adapter->dev_topo.card.sas_address) {
+		spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+		topo_node_exp =
+			leapraid_exp_find_by_sas_address(
+				adapter,
+				sas_addr_parent,
+				leapraid_get_port_by_id(adapter,
+							port_id,
+							false));
+		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
+				       flags);
+		if (!topo_node_exp) {
+			rc = leapraid_exp_add(adapter, parent_handle);
+			if (rc != 0)
+				return rc;
+		}
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	sas_addr = le64_to_cpu(exp_pg0.sas_address);
+	topo_node_exp =
+		leapraid_exp_find_by_sas_address(
+			adapter,
+			sas_addr,
+			leapraid_get_port_by_id(adapter, port_id, false));
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	if (topo_node_exp)
+		return 0;
+
+	return leapraid_internal_exp_add(adapter, &exp_pg0, &cfgp1,
+					 &cfgp2, hdl);
+}
+
+static void leapraid_exp_node_rm(struct leapraid_adapter *adapter,
+				 struct leapraid_topo_node *topo_node_exp)
+{
+	struct leapraid_sas_port *sas_port, *sas_port_next;
+	unsigned long flags;
+	int port_id;
+
+	list_for_each_entry_safe(sas_port, sas_port_next,
+				 &topo_node_exp->sas_port_list,
+				 port_list) {
+		if (adapter->access_ctrl.shost_recovering)
+			return;
+
+		switch (sas_port->remote_identify.device_type) {
+		case SAS_END_DEVICE:
+			leapraid_sas_dev_remove_by_sas_address(
+				adapter,
+				sas_port->remote_identify.sas_address,
+				sas_port->card_port);
+			break;
+		case SAS_EDGE_EXPANDER_DEVICE:
+		case SAS_FANOUT_EXPANDER_DEVICE:
+			leapraid_exp_rm(
+				adapter,
+				sas_port->remote_identify.sas_address,
+				sas_port->card_port);
+			break;
+		default:
+			break;
+		}
+	}
+
+	port_id = topo_node_exp->card_port->port_id;
+	leapraid_transport_port_remove(adapter, topo_node_exp->sas_address,
+				       topo_node_exp->sas_address_parent,
+				       topo_node_exp->card_port);
+	dev_info(&adapter->pdev->dev,
+		 "removing exp: port=%d, SAS addr=0x%016llx, hdl=0x%04x\n",
+		 port_id, (unsigned long long)topo_node_exp->sas_address,
+		 topo_node_exp->hdl);
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	list_del(&topo_node_exp->list);
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+	kfree(topo_node_exp->card_phy);
+	kfree(topo_node_exp);
+}
+
+void leapraid_exp_rm(struct leapraid_adapter *adapter, u64 sas_addr,
+		     struct leapraid_card_port *port)
+{
+	struct leapraid_topo_node *topo_node_exp;
+	unsigned long flags;
+
+	if (adapter->access_ctrl.shost_recovering)
+		return;
+
+	if (!port)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	topo_node_exp = leapraid_exp_find_by_sas_address(adapter,
+							 sas_addr,
+							 port);
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	if (topo_node_exp)
+		leapraid_exp_node_rm(adapter, topo_node_exp);
+}
+
+static void leapraid_internal_sas_topo_chg_evt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_card_port *card_port,
+		struct leapraid_topo_node *topo_node_exp,
+		struct leapraid_fw_evt_work *fw_evt,
+		u64 sas_addr, u8 max_phys)
+{
+	struct leapraid_evt_data_sas_topo_change_list *evt_data;
+	u8 phy_number;
+	u8 link_rate;
+	u16 reason_code;
+	u16 hdl;
+	int i;
+
+	evt_data = fw_evt->evt_data;
+	for (i = 0; i < evt_data->entry_num; i++) {
+		if (fw_evt->ignore)
+			return;
+
+		if (adapter->access_ctrl.host_removing ||
+		    adapter->access_ctrl.pcie_recovering)
+			return;
+
+		phy_number = evt_data->start_phy_num + i;
+		if (phy_number >= max_phys)
+			continue;
+
+		reason_code = evt_data->phy[i].phy_status &
+			LEAPRAID_EVT_SAS_TOPO_RC_MASK;
+
+		hdl = le16_to_cpu(evt_data->phy[i].attached_dev_hdl);
+		if (!hdl ||
+		    hdl > adapter->adapter_attr.features.max_dev_handle) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Invalid device handle\n", __func__);
+			continue;
+		}
+		link_rate = evt_data->phy[i].link_rate >>
+			LEAPRAID_SAS_NEG_LINK_RATE_SHIFT;
+		switch (reason_code) {
+		case LEAPRAID_EVT_SAS_TOPO_RC_TARG_ADDED:
+			if (adapter->access_ctrl.shost_recovering)
+				break;
+			leapraid_transport_update_links(adapter, sas_addr,
+							hdl, phy_number,
+							link_rate, card_port);
+			if (link_rate < LEAPRAID_SAS_NEG_LINK_RATE_1_5)
+				break;
+			leapraid_add_dev(adapter, hdl);
+			break;
+		case LEAPRAID_EVT_SAS_TOPO_RC_TARG_NOT_RESPONDING:
+			leapraid_sas_dev_remove_by_hdl(adapter, hdl);
+			break;
+		}
+	}
+
+	if (evt_data->exp_status == LEAPRAID_EVT_SAS_TOPO_ES_NOT_RESPONDING &&
+	    topo_node_exp)
+		leapraid_exp_rm(adapter, sas_addr, card_port);
+}
+
+static void leapraid_sas_topo_chg_evt(struct leapraid_adapter *adapter,
+				      struct leapraid_fw_evt_work *fw_evt)
+{
+	struct leapraid_topo_node *topo_node_exp;
+	struct leapraid_card_port *card_port;
+	struct leapraid_evt_data_sas_topo_change_list *evt_data;
+	u16 phdl;
+	u8 max_phys;
+	u64 sas_addr;
+	unsigned long flags;
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering)
+		return;
+
+	evt_data = fw_evt->evt_data;
+	leapraid_sas_host_add(adapter, adapter->dev_topo.card.phys_num > 0);
+
+	if (fw_evt->ignore)
+		return;
+
+	phdl = le16_to_cpu(evt_data->exp_dev_hdl);
+	card_port = leapraid_get_port_by_id(adapter,
+					    evt_data->physical_port,
+					    false);
+	if (evt_data->exp_status == LEAPRAID_EVT_SAS_TOPO_ES_ADDED &&
+	    leapraid_exp_add(adapter, phdl) != 0)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	topo_node_exp = leapraid_exp_find_by_hdl(adapter, phdl);
+	if (topo_node_exp) {
+		sas_addr = topo_node_exp->sas_address;
+		max_phys = topo_node_exp->phys_num;
+		card_port = topo_node_exp->card_port;
+	} else if (phdl < adapter->dev_topo.card.phys_num) {
+		sas_addr = adapter->dev_topo.card.sas_address;
+		max_phys = adapter->dev_topo.card.phys_num;
+	} else {
+		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
+				       flags);
+		return;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	leapraid_internal_sas_topo_chg_evt(adapter, card_port,
+					   topo_node_exp, fw_evt,
+					   sas_addr, max_phys);
+}
+
+static void leapraid_reprobe_lun(struct scsi_device *sdev, void *no_uld_attach)
+{
+	sdev->no_uld_attach = no_uld_attach ? 1 : 0;
+	sdev_printk(KERN_INFO, sdev,
+		    "%s RAID component to upper layer\n",
+		    sdev->no_uld_attach ? "hide" : "expose");
+	WARN_ON(scsi_device_reprobe(sdev));
+}
+
+static void leapraid_sas_pd_add(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_p0;
+	struct leapraid_sas_dev *sas_dev;
+	u64 sas_address;
+	u16 parent_hdl;
+	u16 hdl;
+
+	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
+	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid device handle\n", __func__);
+		return;
+	}
+
+	set_bit(hdl, adapter->dev_topo.pd_hdls);
+	sas_dev = leapraid_get_sas_dev_by_hdl(adapter, hdl);
+	if (sas_dev) {
+		leapraid_sdev_put(sas_dev);
+		dev_warn(&adapter->pdev->dev,
+			 "Dev handle 0x%x already exists\n", hdl);
+		return;
+	}
+
+	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+	cfgp2.handle = hdl;
+	if (leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1, cfgp2,
+				    GET_SAS_DEVICE_PG0)) {
+		dev_warn(&adapter->pdev->dev, "Failed to read dev page0\n");
+		return;
+	}
+
+	parent_hdl = le16_to_cpu(sas_dev_p0.parent_dev_hdl);
+	if (!leapraid_get_sas_address(adapter, parent_hdl, &sas_address))
+		leapraid_transport_update_links(adapter, sas_address, hdl,
+						sas_dev_p0.phy_num,
+						LEAPRAID_SAS_NEG_LINK_RATE_1_5,
+			 leapraid_get_port_by_id(adapter,
+						 sas_dev_p0.physical_port,
+						 false));
+	leapraid_add_dev(adapter, hdl);
+}
+
+static void leapraid_sas_pd_delete(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	u16 hdl;
+
+	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
+	leapraid_sas_dev_remove_by_hdl(adapter, hdl);
+}
+
+static void leapraid_sas_pd_hide(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct scsi_target *starget = NULL;
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+	u64 volume_wwid = 0;
+	u16 volume_hdl;
+	u16 hdl;
+
+	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
+	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid device handle\n", __func__);
+		return;
+	}
+	leapraid_cfg_get_volume_hdl(adapter, hdl, &volume_hdl);
+	if (volume_hdl)
+		leapraid_cfg_get_volume_wwid(adapter,
+					     volume_hdl,
+					     &volume_wwid);
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	if (!sas_dev) {
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		return;
+	}
+
+	set_bit(hdl, adapter->dev_topo.pd_hdls);
+	if (sas_dev->starget && sas_dev->starget->hostdata) {
+		starget = sas_dev->starget;
+		starget_priv = starget->hostdata;
+		starget_priv->flg |= LEAPRAID_TGT_FLG_RAID_MEMBER;
+		sas_dev->volume_hdl = volume_hdl;
+		sas_dev->volume_wwid = volume_wwid;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	if (starget) {
+		starget_for_each_device(starget,
+					(void *)LEAPRAID_NO_ULD_ATTACH_FLAG,
+					leapraid_reprobe_lun);
+	}
+
+	leapraid_sdev_put(sas_dev);
+}
+
+static void leapraid_sas_pd_expose(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct scsi_target *starget = NULL;
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+	u16 hdl;
+
+	hdl = le16_to_cpu(evt_data->phys_disk_dev_hdl);
+	if (!hdl || hdl > adapter->adapter_attr.features.max_dev_handle) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid device handle\n", __func__);
+		return;
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	if (!sas_dev) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s:%d: sas_dev not found, hdl=0x%x\n",
+			 __func__, __LINE__, hdl);
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		return;
+	}
+
+	sas_dev->volume_hdl = 0;
+	sas_dev->volume_wwid = 0;
+	clear_bit(hdl, adapter->dev_topo.pd_hdls);
+	if (sas_dev->starget && sas_dev->starget->hostdata) {
+		starget = sas_dev->starget;
+		starget_priv = starget->hostdata;
+		starget_priv->flg &= ~LEAPRAID_TGT_FLG_RAID_MEMBER;
+		sas_dev->led_on = 0;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	if (starget) {
+		starget_for_each_device(starget,
+					NULL,
+					leapraid_reprobe_lun);
+	}
+
+	leapraid_sdev_put(sas_dev);
+}
+
+static void leapraid_sas_vol_visibility(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct scsi_device *sdev;
+	bool reprobe_flg = false;
+	bool sdev_held = false;
+	unsigned long flags;
+	u16 hdl;
+	u8 rc;
+
+	hdl = le16_to_cpu(evt_data->vol_dev_hdl);
+	rc = evt_data->reason_code;
+	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, hdl);
+	if (!raid_volume) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s:%d: Volume handle 0x%x not found\n",
+			 __func__, __LINE__, hdl);
+		return;
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	sdev = raid_volume->sdev;
+	if (!sdev) {
+		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
+				       flags);
+		leapraid_raid_volume_put(raid_volume);
+		dev_warn(&adapter->pdev->dev,
+			 "%s:%d: Volume handle 0x%x has no sdev\n",
+			 __func__, __LINE__, hdl);
+		return;
+	}
+
+	if (sdev->no_uld_attach &&
+	    rc == LEAPRAID_EVT_IR_RC_VOLUME_UNHIDE) {
+		sdev->no_uld_attach = 0;
+		reprobe_flg = true;
+	} else if (!sdev->no_uld_attach &&
+		   rc == LEAPRAID_EVT_IR_RC_VOLUME_HIDE) {
+		sdev->no_uld_attach = 1;
+		reprobe_flg = true;
+	}
+
+	if (reprobe_flg && !scsi_device_get(sdev))
+		sdev_held = true;
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+	leapraid_raid_volume_put(raid_volume);
+
+	if (!reprobe_flg) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s:rc(0x%x): Request matches, skipping\n",
+			 __func__, rc);
+		return;
+	}
+
+	if (!sdev_held) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed to hold sdev for reprobe, hdl=0x%x\n",
+			 __func__, hdl);
+		return;
+	}
+
+	if (sdev->no_uld_attach)
+		sdev_printk(KERN_INFO, sdev, "hide vol\n");
+	else
+		sdev_printk(KERN_INFO, sdev, "unhide vol\n");
+
+	WARN_ON(scsi_device_reprobe(sdev));
+	scsi_device_put(sdev);
+}
+
+static void leapraid_sas_volume_add(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	struct leapraid_raid_volume *raid_volume;
+	unsigned long flags;
+	u64 wwid;
+	u16 hdl;
+
+	hdl = le16_to_cpu(evt_data->vol_dev_hdl);
+
+	if (leapraid_cfg_get_volume_wwid(adapter, hdl, &wwid)) {
+		dev_warn(&adapter->pdev->dev, "Failed to read volume page1\n");
+		return;
+	}
+
+	if (!wwid) {
+		dev_warn(&adapter->pdev->dev, "Invalid WWID(handle=0x%x)\n",
+			 hdl);
+		return;
+	}
+
+	raid_volume = leapraid_raid_volume_find_by_wwid(adapter, wwid);
+	if (raid_volume) {
+		dev_warn(&adapter->pdev->dev,
+			 "Volume handle 0x%x already exists\n", hdl);
+		leapraid_raid_volume_put(raid_volume);
+		return;
+	}
+
+	raid_volume = kzalloc_obj(*raid_volume);
+	if (!raid_volume)
+		return;
+
+	INIT_LIST_HEAD(&raid_volume->list);
+	kref_init(&raid_volume->refcnt);
+	raid_volume->id = adapter->dev_topo.sas_id++;
+	raid_volume->channel = RAID_CHANNEL;
+	raid_volume->hdl = hdl;
+	raid_volume->wwid = wwid;
+	leapraid_raid_volume_add(adapter, raid_volume);
+	if (!adapter->scan_dev_desc.wait_scan_dev_done) {
+		if (scsi_add_device(adapter->shost, RAID_CHANNEL,
+				    raid_volume->id, 0))
+			leapraid_raid_volume_remove(adapter, raid_volume);
+		dev_info(&adapter->pdev->dev,
+			 "add RAID volume: hdl=0x%x, wwid=0x%llx\n", hdl, wwid);
+	} else {
+		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+		leapraid_check_boot_dev(adapter, raid_volume, RAID_CHANNEL);
+		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
+				       flags);
+	}
+	leapraid_raid_volume_put(raid_volume);
+}
+
+static void leapraid_sas_volume_delete_by_ptr(
+		struct leapraid_adapter *adapter,
+		struct leapraid_raid_volume *raid_volume)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct scsi_target *starget = NULL;
+	unsigned long flags;
+	bool in_list;
+
+	if (!raid_volume)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	in_list = !list_empty(&raid_volume->list);
+	if (in_list && raid_volume->starget) {
+		starget = raid_volume->starget;
+		starget_priv = starget->hostdata;
+		if (starget_priv)
+			starget_priv->deleted = 1;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	if (!in_list)
+		return;
+
+	dev_info(&adapter->pdev->dev,
+		 "delete RAID volume: hdl=0x%x, wwid=0x%llx\n",
+		 raid_volume->hdl, raid_volume->wwid);
+	leapraid_raid_volume_remove(adapter, raid_volume);
+
+	if (starget)
+		scsi_remove_target(&starget->dev);
+}
+
+static void leapraid_sas_volume_delete(struct leapraid_adapter *adapter,
+				       u16 hdl)
+{
+	struct leapraid_raid_volume *raid_volume;
+
+	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, hdl);
+	if (!raid_volume) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s:%d: Volume handle 0x%x not found\n",
+			 __func__, __LINE__, hdl);
+		return;
+	}
+
+	leapraid_sas_volume_delete_by_ptr(adapter, raid_volume);
+	leapraid_raid_volume_put(raid_volume);
+}
+
+static void leapraid_sas_ir_chg_evt(struct leapraid_adapter *adapter,
+				    struct leapraid_fw_evt_work *fw_evt)
+{
+	struct leapraid_evt_data_ir_change *evt_data;
+
+	evt_data = fw_evt->evt_data;
+
+	switch (evt_data->reason_code) {
+	case LEAPRAID_EVT_IR_RC_VOLUME_ADD:
+		leapraid_sas_volume_add(adapter, evt_data);
+		break;
+	case LEAPRAID_EVT_IR_RC_VOLUME_DELETE:
+		leapraid_sas_volume_delete(adapter,
+					   le16_to_cpu(evt_data->vol_dev_hdl));
+		break;
+	case LEAPRAID_EVT_IR_RC_PD_HIDDEN_TO_ADD:
+		leapraid_sas_pd_add(adapter, evt_data);
+		break;
+	case LEAPRAID_EVT_IR_RC_PD_UNHIDDEN_TO_DELETE:
+		leapraid_sas_pd_delete(adapter, evt_data);
+		break;
+	case LEAPRAID_EVT_IR_RC_PD_CREATED_TO_HIDE:
+		leapraid_sas_pd_hide(adapter, evt_data);
+		break;
+	case LEAPRAID_EVT_IR_RC_PD_DELETED_TO_EXPOSE:
+		leapraid_sas_pd_expose(adapter, evt_data);
+		break;
+	case LEAPRAID_EVT_IR_RC_VOLUME_HIDE:
+	case LEAPRAID_EVT_IR_RC_VOLUME_UNHIDE:
+		leapraid_sas_vol_visibility(adapter, evt_data);
+		break;
+	default:
+		break;
+	}
+}
+
+static void leapraid_sas_enc_dev_stat_add_node(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_enc_node *enc_node;
+	struct leapraid_enc_node *enc_exist;
+	unsigned long flags;
+	int rc;
+
+	enc_node = kzalloc_obj(*enc_node);
+	if (!enc_node)
+		return;
+
+	cfgp1.form = LEAPRAID_SAS_ENC_CFG_PGAD_HDL;
+	cfgp2.handle = hdl;
+	rc = leapraid_op_config_page(adapter, &enc_node->pg0, cfgp1, cfgp2,
+				     GET_SAS_ENCLOSURE_PG0);
+	if (rc) {
+		kfree(enc_node);
+		return;
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
+	enc_exist = leapraid_enc_find_by_hdl(adapter, hdl);
+	if (enc_exist) {
+		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+		kfree(enc_node);
+		return;
+	}
+	list_add_tail(&enc_node->list, &adapter->dev_topo.enc_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+}
+
+static void leapraid_sas_enc_dev_stat_del_node(
+		struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_enc_node *enc_node;
+	unsigned long flags;
+
+	if (!hdl)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
+	enc_node = leapraid_enc_find_by_hdl(adapter, hdl);
+	if (enc_node)
+		list_del(&enc_node->list);
+	spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+
+	kfree(enc_node);
+}
+
+static void leapraid_sas_enc_dev_stat_chg_evt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_fw_evt_work *fw_evt)
+{
+	struct leapraid_evt_data_sas_enc_dev_status_change *evt_data;
+	u16 enc_hdl;
+
+	if (adapter->access_ctrl.shost_recovering)
+		return;
+
+	evt_data = fw_evt->evt_data;
+	enc_hdl = le16_to_cpu(evt_data->enc_hdl);
+	switch (evt_data->reason_code) {
+	case LEAPRAID_EVT_SAS_ENCL_RC_ADDED:
+		if (enc_hdl)
+			leapraid_sas_enc_dev_stat_add_node(adapter, enc_hdl);
+		break;
+	case LEAPRAID_EVT_SAS_ENCL_RC_NOT_RESPONDING:
+		leapraid_sas_enc_dev_stat_del_node(adapter, enc_hdl);
+		break;
+	default:
+		break;
+	}
+}
+
+static void leapraid_remove_unresp_sas_end_dev(
+		struct leapraid_adapter *adapter)
+{
+	struct leapraid_sas_dev *sas_dev, *sas_dev_next;
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	list_for_each_entry_safe(sas_dev, sas_dev_next,
+				 &adapter->dev_topo.sas_dev_init_list, list) {
+		if (sas_dev->rphy || sas_dev->pend_sas_rphy_add)
+			continue;
+
+		list_del_init(&sas_dev->list);
+		leapraid_clear_cached_boot_dev(adapter, sas_dev, 0);
+		leapraid_sdev_put(sas_dev);
+	}
+	list_for_each_entry_safe(sas_dev, sas_dev_next,
+				 &adapter->dev_topo.sas_dev_list, list) {
+		if (!sas_dev->resp)
+			list_move_tail(&sas_dev->list, &head);
+		else
+			sas_dev->resp = 0;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	list_for_each_entry_safe(sas_dev, sas_dev_next, &head, list) {
+		leapraid_remove_device(adapter, sas_dev);
+		list_del_init(&sas_dev->list);
+		leapraid_sdev_put(sas_dev);
+	}
+
+	dev_warn(&adapter->pdev->dev,
+		 "Unresponsive SAS end devices removed\n");
+}
+
+static void leapraid_remove_unresp_raid_volumes(
+		struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+	struct leapraid_raid_volume *raid_volume, *raid_volume_next;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	list_for_each_entry_safe(raid_volume, raid_volume_next,
+				 &adapter->dev_topo.raid_volume_list, list) {
+		if (!raid_volume->resp)
+			list_move_tail(&raid_volume->list, &head);
+		else
+			raid_volume->resp = 0;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	list_for_each_entry_safe(raid_volume, raid_volume_next, &head, list) {
+		leapraid_sas_volume_delete_by_ptr(adapter, raid_volume);
+	}
+
+	dev_warn(&adapter->pdev->dev,
+		 "Unresponsive RAID volumes removed\n");
+}
+
+static void leapraid_remove_unresp_sas_exp(struct leapraid_adapter *adapter)
+{
+	struct leapraid_topo_node *topo_node_exp, *topo_node_exp_next;
+	unsigned long flags;
+	LIST_HEAD(head);
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	list_for_each_entry_safe(topo_node_exp, topo_node_exp_next,
+				 &adapter->dev_topo.exp_list, list) {
+		if (!topo_node_exp->resp)
+			list_move_tail(&topo_node_exp->list, &head);
+		else
+			topo_node_exp->resp = 0;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	list_for_each_entry_safe(topo_node_exp, topo_node_exp_next,
+				 &head, list)
+		leapraid_exp_node_rm(adapter, topo_node_exp);
+
+	dev_warn(&adapter->pdev->dev,
+		 "Unresponsive SAS expanders removed\n");
+}
+
+static void leapraid_remove_unresp_dev(struct leapraid_adapter *adapter)
+{
+	leapraid_remove_unresp_sas_end_dev(adapter);
+	if (adapter->adapter_attr.raid_support)
+		leapraid_remove_unresp_raid_volumes(adapter);
+	leapraid_remove_unresp_sas_exp(adapter);
+	leapraid_ublk_io_all_dev(adapter);
+}
+
+static void leapraid_del_dirty_vphy(struct leapraid_adapter *adapter)
+{
+	struct leapraid_card_port *card_port, *card_port_next;
+	struct leapraid_vphy *vphy, *vphy_next;
+
+	list_for_each_entry_safe(card_port, card_port_next,
+				 &adapter->dev_topo.card_port_list, list) {
+		if (!card_port->vphys_mask)
+			continue;
+
+		list_for_each_entry_safe(vphy, vphy_next,
+					 &card_port->vphys_list, list) {
+			if (!(vphy->flg & LEAPRAID_VPHY_FLG_DIRTY))
+				continue;
+
+			card_port->vphys_mask &= ~vphy->phy_mask;
+			list_del(&vphy->list);
+			kfree(vphy);
+		}
+
+		if (!card_port->vphys_mask && !card_port->sas_address)
+			card_port->flg |= LEAPRAID_CARD_PORT_FLG_DIRTY;
+	}
+}
+
+static void leapraid_del_dirty_card_port(struct leapraid_adapter *adapter)
+{
+	struct leapraid_card_port *card_port, *card_port_next;
+
+	list_for_each_entry_safe(card_port, card_port_next,
+				 &adapter->dev_topo.card_port_list, list) {
+		if (!(card_port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY) ||
+		    card_port->flg & LEAPRAID_CARD_PORT_FLG_NEW)
+			continue;
+
+		list_del(&card_port->list);
+		kfree(card_port);
+	}
+}
+
+static void leapraid_update_dev_qdepth(struct leapraid_adapter *adapter)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_adapter_attr *attr;
+	struct scsi_device *sdev;
+	u16 qdepth;
+
+	attr = &adapter->adapter_attr;
+	shost_for_each_device(sdev, adapter->shost) {
+		sdev_priv = sdev->hostdata;
+		if (!sdev_priv || !sdev_priv->starget_priv)
+			continue;
+		sas_dev = sdev_priv->starget_priv->sas_dev;
+		if (sas_dev && sas_dev->dev_info & LEAPRAID_DEVTYP_SSP_TGT)
+			qdepth = (sas_dev->port_connection > 1) ?
+				attr->wideport_max_queue_depth :
+				attr->narrowport_max_queue_depth;
+		else if (sas_dev && sas_dev->dev_info &
+			 LEAPRAID_DEVTYP_SATA_DEV)
+			qdepth = attr->sata_max_queue_depth;
+		else
+			continue;
+
+		leapraid_change_queue_depth(sdev, qdepth);
+	}
+}
+
+static void leapraid_update_exp_links(struct leapraid_adapter *adapter,
+				      struct leapraid_topo_node *topo_node_exp,
+				      u16 hdl)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_exp_p1 exp_p1;
+	int i;
+
+	cfgp2.handle = hdl;
+	for (i = 0; i < topo_node_exp->phys_num; i++) {
+		cfgp1.phy_number = i;
+		if (leapraid_op_config_page(adapter, &exp_p1, cfgp1, cfgp2,
+					    GET_SAS_EXPANDER_PG1))
+			return;
+
+		leapraid_transport_update_links(
+			adapter,
+			topo_node_exp->sas_address,
+			le16_to_cpu(exp_p1.attached_dev_hdl),
+			i,
+			exp_p1.neg_link_rate >>
+				LEAPRAID_SAS_NEG_LINK_RATE_SHIFT,
+			topo_node_exp->card_port);
+	}
+}
+
+static void leapraid_scan_exp_after_reset(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_topo_node *topo_node_exp;
+	struct leapraid_exp_p0 exp_p0;
+	unsigned long flags;
+	u16 hdl;
+	u8 port_id;
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (hdl = 0xFFFF, cfgp2.handle = hdl;
+	     !leapraid_op_config_page(adapter, &exp_p0, cfgp1, cfgp2,
+				      GET_SAS_EXPANDER_PG0);
+	     cfgp2.handle = hdl) {
+		hdl = le16_to_cpu(exp_p0.dev_hdl);
+		port_id = exp_p0.physical_port;
+		spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+		topo_node_exp =
+			leapraid_exp_find_by_sas_address(
+				adapter,
+				le64_to_cpu(exp_p0.sas_address),
+				leapraid_get_port_by_id(adapter,
+							port_id,
+							false));
+		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
+				       flags);
+
+		if (topo_node_exp) {
+			leapraid_update_exp_links(adapter, topo_node_exp, hdl);
+		} else {
+			leapraid_exp_add(adapter, hdl);
+
+			dev_info(&adapter->pdev->dev,
+				 "add exp: hdl=0x%04x, SAS addr=0x%016llx\n",
+				 hdl,
+				 (unsigned long long)le64_to_cpu(
+					exp_p0.sas_address));
+		}
+	}
+}
+
+static void leapraid_scan_phy_disks_after_reset(
+		struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	union cfg_param_1 cfgp1_extra = {0};
+	union cfg_param_2 cfgp2_extra = {0};
+	struct leapraid_sas_dev_p0 sas_dev_p0;
+	struct leapraid_raidpd_p0 raidpd_p0;
+	struct leapraid_sas_dev *sas_dev;
+	u8 phys_disk_num, port_id;
+	u16 hdl, parent_hdl;
+	u64 sas_addr;
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (phys_disk_num = 0xFF, cfgp2.form_specific = phys_disk_num;
+	     !leapraid_op_config_page(adapter, &raidpd_p0,
+				      cfgp1, cfgp2, GET_PHY_DISK_PG0);
+	     cfgp2.form_specific = phys_disk_num) {
+		phys_disk_num = raidpd_p0.phys_disk_num;
+		hdl = le16_to_cpu(raidpd_p0.dev_hdl);
+		sas_dev = leapraid_get_sas_dev_by_hdl(adapter, hdl);
+		if (sas_dev) {
+			leapraid_sdev_put(sas_dev);
+			continue;
+		}
+
+		cfgp1_extra.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+		cfgp2_extra.handle = hdl;
+		if (leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1_extra,
+					    cfgp2_extra, GET_SAS_DEVICE_PG0) !=
+					    0)
+			continue;
+
+		parent_hdl = le16_to_cpu(sas_dev_p0.parent_dev_hdl);
+		if (!leapraid_get_sas_address(adapter,
+					      parent_hdl,
+					      &sas_addr)) {
+			port_id = sas_dev_p0.physical_port;
+			leapraid_transport_update_links(
+				adapter, sas_addr, hdl,
+				sas_dev_p0.phy_num,
+				LEAPRAID_SAS_NEG_LINK_RATE_1_5,
+				leapraid_get_port_by_id(
+					adapter, port_id, false));
+			if (!hdl || hdl >
+			    adapter->adapter_attr.features.max_dev_handle) {
+				dev_warn(&adapter->pdev->dev,
+					 "%s: Invalid device handle\n",
+					 __func__);
+			} else {
+				set_bit(hdl, adapter->dev_topo.pd_hdls);
+				leapraid_add_dev(adapter, hdl);
+
+				dev_info(&adapter->pdev->dev,
+					 "add pd hdl=0x%04x saddr=0x%016llx\n",
+					 hdl,
+					 (unsigned long long)le64_to_cpu(
+						sas_dev_p0.sas_address));
+			}
+		}
+	}
+}
+
+static void leapraid_scan_vol_after_reset(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	union cfg_param_1 cfgp1_extra = {0};
+	union cfg_param_2 cfgp2_extra = {0};
+	struct leapraid_evt_data_ir_change evt_data;
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_raidvol_p1 *vol_p1;
+	struct leapraid_raidvol_p0 *vol_p0;
+	u16 hdl;
+
+	vol_p0 = kzalloc_obj(*vol_p0);
+	if (!vol_p0)
+		return;
+
+	vol_p1 = kzalloc_obj(*vol_p1);
+	if (!vol_p1) {
+		kfree(vol_p0);
+		return;
+	}
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (hdl = 0xFFFF, cfgp2.handle = hdl;
+	     !leapraid_op_config_page(adapter, vol_p1, cfgp1,
+				      cfgp2, GET_RAID_VOLUME_PG1);
+	     cfgp2.handle = hdl) {
+		hdl = le16_to_cpu(vol_p1->dev_hdl);
+		raid_volume = leapraid_raid_volume_find_by_wwid(
+				adapter,
+				le64_to_cpu(vol_p1->wwid));
+		if (raid_volume) {
+			leapraid_raid_volume_put(raid_volume);
+			continue;
+		}
+
+		cfgp1_extra.size = sizeof(struct leapraid_raidvol_p0);
+		cfgp2_extra.handle = hdl;
+		if (leapraid_op_config_page(adapter, vol_p0, cfgp1_extra,
+					    cfgp2_extra, GET_RAID_VOLUME_PG0))
+			continue;
+
+		if (vol_p0->volume_state == LEAPRAID_VOL_STATE_OPTIMAL ||
+		    vol_p0->volume_state == LEAPRAID_VOL_STATE_ONLINE ||
+		    vol_p0->volume_state == LEAPRAID_VOL_STATE_DEGRADED) {
+			memset(&evt_data, 0,
+			       sizeof(struct leapraid_evt_data_ir_change));
+			evt_data.reason_code = LEAPRAID_EVT_IR_RC_VOLUME_ADD;
+			evt_data.vol_dev_hdl = vol_p1->dev_hdl;
+			leapraid_sas_volume_add(adapter, &evt_data);
+			dev_info(&adapter->pdev->dev,
+				 "add volume: hdl=0x%04x\n",
+				 vol_p1->dev_hdl);
+		}
+	}
+
+	kfree(vol_p0);
+	kfree(vol_p1);
+}
+
+static void leapraid_scan_sas_dev_after_reset(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_p0;
+	struct leapraid_sas_dev *sas_dev;
+	u16 hdl, parent_hdl;
+	u64 sas_address;
+	u8 port_id;
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (hdl = 0xFFFF, cfgp2.handle = hdl;
+	     !leapraid_op_config_page(adapter, &sas_dev_p0, cfgp1, cfgp2,
+				      GET_SAS_DEVICE_PG0);
+	     cfgp2.handle = hdl) {
+		hdl = le16_to_cpu(sas_dev_p0.dev_hdl);
+		if (!hdl ||
+		    hdl > adapter->adapter_attr.features.max_dev_handle) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Invalid device handle\n", __func__);
+			continue;
+		}
+
+		if (!(leapraid_is_end_dev(le32_to_cpu(sas_dev_p0.dev_info))))
+			continue;
+
+		port_id = sas_dev_p0.physical_port;
+		sas_dev = leapraid_get_sas_dev_by_addr(
+				adapter,
+				le64_to_cpu(sas_dev_p0.sas_address),
+				leapraid_get_port_by_id(
+					adapter,
+					port_id,
+					false));
+		if (sas_dev) {
+			leapraid_sdev_put(sas_dev);
+			continue;
+		}
+
+		parent_hdl = le16_to_cpu(sas_dev_p0.parent_dev_hdl);
+		if (!leapraid_get_sas_address(adapter, parent_hdl,
+					      &sas_address)) {
+			leapraid_transport_update_links(
+				adapter,
+				sas_address,
+				hdl,
+				sas_dev_p0.phy_num,
+				LEAPRAID_SAS_NEG_LINK_RATE_1_5,
+				leapraid_get_port_by_id(adapter,
+							port_id,
+							false));
+			leapraid_add_dev(adapter, hdl);
+			dev_info(&adapter->pdev->dev,
+				 "Add SAS dev: hdl=0x%04x, saddr=0x%016llx\n",
+				 hdl,
+				 (unsigned long long)le64_to_cpu(
+					sas_dev_p0.sas_address));
+		}
+	}
+}
+
+static void leapraid_scan_all_dev_after_reset(struct leapraid_adapter *adapter)
+{
+	leapraid_sas_host_add(adapter, adapter->dev_topo.card.phys_num > 0);
+	leapraid_scan_exp_after_reset(adapter);
+	if (adapter->adapter_attr.raid_support) {
+		leapraid_scan_phy_disks_after_reset(adapter);
+		leapraid_scan_vol_after_reset(adapter);
+	}
+	leapraid_scan_sas_dev_after_reset(adapter);
+}
+
+static void leapraid_hardreset_async_logic(struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+
+	leapraid_remove_unresp_dev(adapter);
+	leapraid_del_dirty_vphy(adapter);
+	leapraid_del_dirty_card_port(adapter);
+	leapraid_update_dev_qdepth(adapter);
+	leapraid_scan_all_dev_after_reset(adapter);
+
+	if (adapter->scan_dev_desc.driver_loading)
+		leapraid_scan_dev_done(adapter);
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	adapter->access_ctrl.shost_recover_async = 0;
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+	wake_up(&adapter->access_ctrl.shost_recover_wq);
+}
+
+static int leapraid_send_enc_cmd(struct leapraid_adapter *adapter,
+				 struct leapraid_sep_rep *sep_rep,
+				 struct leapraid_sep_req *sep_req)
+{
+	void *req;
+	bool reset_flg = false;
+	int rc;
+	u16 smid;
+
+	mutex_lock(&adapter->driver_cmds.enc_cmd.mutex);
+	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
+					  __func__);
+	if (rc)
+		goto unlock;
+
+	adapter->driver_cmds.enc_cmd.status = LEAPRAID_CMD_PENDING;
+	smid = adapter->driver_cmds.enc_cmd.inter_taskid;
+	req = leapraid_get_task_desc(adapter, smid);
+	memset(req, 0, LEAPRAID_REQUEST_SIZE);
+	memcpy(req, sep_req, sizeof(struct leapraid_sep_req));
+	init_completion(&adapter->driver_cmds.enc_cmd.done);
+	leapraid_fire_task(adapter, smid);
+	wait_for_completion_timeout(&adapter->driver_cmds.enc_cmd.done,
+				    LEAPRAID_ENC_CMD_TIMEOUT * HZ);
+	if (!(adapter->driver_cmds.enc_cmd.status & LEAPRAID_CMD_DONE)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: SEP command timeout, status=0x%x\n",
+			__func__, adapter->driver_cmds.enc_cmd.status);
+		leapraid_log_req_context(adapter, smid, sep_req);
+		reset_flg =
+			leapraid_check_reset(
+				adapter->driver_cmds.enc_cmd.status);
+		rc = -EFAULT;
+		goto do_hard_reset;
+	}
+
+	if (adapter->driver_cmds.enc_cmd.status & LEAPRAID_CMD_REPLY_VALID)
+		memcpy(sep_rep, &adapter->driver_cmds.enc_cmd.reply,
+		       sizeof(struct leapraid_sep_rep));
+do_hard_reset:
+	if (reset_flg) {
+		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
+			 __func__, __LINE__);
+		leapraid_hard_reset_handler(adapter, FULL_RESET);
+	}
+
+	adapter->driver_cmds.enc_cmd.status = LEAPRAID_CMD_NOT_USED;
+unlock:
+	mutex_unlock(&adapter->driver_cmds.enc_cmd.mutex);
+	return rc;
+}
+
+static void leapraid_set_led(struct leapraid_adapter *adapter,
+			     struct leapraid_sas_dev *sas_dev, bool on)
+{
+	struct leapraid_sep_rep sep_rep;
+	struct leapraid_sep_req sep_req;
+
+	if (!sas_dev)
+		return;
+
+	memset(&sep_req, 0, sizeof(struct leapraid_sep_req));
+	memset(&sep_rep, 0, sizeof(struct leapraid_sep_rep));
+	sep_req.func = LEAPRAID_FUNC_SCSI_ENC_PROCESSOR;
+	sep_req.act = LEAPRAID_SEP_REQ_ACT_WRITE_STATUS;
+	if (on) {
+		u32 status = LEAPRAID_SEP_REQ_SLOTSTATUS_PREDICTED_FAULT;
+
+		sep_req.slot_status = cpu_to_le32(status);
+		sep_req.dev_hdl = cpu_to_le16(sas_dev->hdl);
+		sep_req.flg = LEAPRAID_SEP_REQ_FLG_DEVHDL_ADDRESS;
+		if (leapraid_send_enc_cmd(adapter, &sep_rep, &sep_req)) {
+			leapraid_sdev_put(sas_dev);
+			return;
+		}
+
+		sas_dev->led_on = 1;
+		leapraid_sdev_put(sas_dev);
+	} else {
+		sep_req.slot_status = 0;
+		sep_req.slot = cpu_to_le16(sas_dev->slot);
+		sep_req.dev_hdl = 0;
+		sep_req.enc_hdl = cpu_to_le16(sas_dev->enc_hdl);
+		sep_req.flg = LEAPRAID_SEP_REQ_FLG_ENCLOSURE_SLOT_ADDRESS;
+		leapraid_send_enc_cmd(adapter, &sep_rep, &sep_req);
+	}
+}
+
+static int leapraid_wait_adapter_recovery(struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+
+	while (leapraid_shost_in_recovery(adapter->shost) ||
+	       READ_ONCE(adapter->access_ctrl.shost_recovering)) {
+		if (READ_ONCE(adapter->access_ctrl.host_removing) ||
+		    READ_ONCE(adapter->fw_evt_s.fw_evt_cleanup)) {
+			spin_lock_irqsave(
+				&adapter->reset_desc.adapter_reset_lock,
+				flags);
+			adapter->access_ctrl.shost_recover_async = 0;
+			spin_unlock_irqrestore(
+				&adapter->reset_desc.adapter_reset_lock,
+				flags);
+
+			wake_up(&adapter->access_ctrl.shost_recover_wq);
+
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Failed, shost %d, host %d\n",
+				 __func__,
+				 adapter->access_ctrl.shost_recovering,
+				 adapter->access_ctrl.host_removing);
+
+			return -EFAULT;
+		}
+
+		wait_event_timeout(
+			adapter->access_ctrl.recovery_waitq,
+			(!leapraid_shost_in_recovery(adapter->shost) &&
+			 !READ_ONCE(adapter->access_ctrl.shost_recovering)),
+			msecs_to_jiffies(1000));
+	}
+
+	return 0;
+}
+
+static void leapraid_fw_work(struct leapraid_adapter *adapter,
+			     struct leapraid_fw_evt_work *fw_evt)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	adapter->fw_evt_s.cur_evt = fw_evt;
+	adapter->fw_evt_s.cur_evt_task = current;
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+	leapraid_del_fw_evt_from_list(adapter, fw_evt);
+	if (adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering) {
+		spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+		leapraid_fw_evt_put(fw_evt);
+		adapter->fw_evt_s.cur_evt = NULL;
+		adapter->fw_evt_s.cur_evt_task = NULL;
+		spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+		return;
+	}
+	switch (fw_evt->evt_type) {
+	case LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST:
+		leapraid_sas_topo_chg_evt(adapter, fw_evt);
+		break;
+	case LEAPRAID_EVT_IR_CHANGE:
+		leapraid_sas_ir_chg_evt(adapter, fw_evt);
+		break;
+	case LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE:
+		leapraid_sas_enc_dev_stat_chg_evt(adapter, fw_evt);
+		break;
+	case LEAPRAID_EVT_REMOVE_DEAD_DEV:
+		if (leapraid_wait_adapter_recovery(adapter))
+			goto out_cleanup;
+
+		leapraid_hardreset_async_logic(adapter);
+		break;
+	case LEAPRAID_EVT_TURN_ON_PFA_LED:
+		sas_dev = leapraid_get_sas_dev_by_hdl(adapter,
+						      fw_evt->dev_handle);
+		leapraid_set_led(adapter, sas_dev, true);
+		break;
+	case LEAPRAID_EVT_SCAN_DEV_DONE:
+		adapter->scan_dev_desc.scan_start = 0;
+		break;
+	default:
+		break;
+	}
+out_cleanup:
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	leapraid_fw_evt_put(fw_evt);
+	adapter->fw_evt_s.cur_evt = NULL;
+	adapter->fw_evt_s.cur_evt_task = NULL;
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+}
+
+static void leapraid_sas_dev_stat_chg_evt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_sas_dev_status_change *event_data)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_sas_dev *sas_dev;
+	u64 sas_address;
+	unsigned long flags;
+
+	switch (event_data->reason_code) {
+	case LEAPRAID_EVT_SAS_DEV_STAT_RC_INTERNAL_DEV_RESET:
+	case LEAPRAID_EVT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
+		break;
+	default:
+		return;
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+
+	sas_address = le64_to_cpu(event_data->sas_address);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(
+			adapter,
+			sas_address,
+			leapraid_get_port_by_id(adapter,
+						event_data->physical_port,
+						false));
+	if (!sas_dev || !sas_dev->starget)
+		goto out_unlock;
+
+	starget_priv = sas_dev->starget->hostdata;
+	if (starget_priv) {
+		switch (event_data->reason_code) {
+		case LEAPRAID_EVT_SAS_DEV_STAT_RC_INTERNAL_DEV_RESET:
+			starget_priv->tm_busy = 1;
+			break;
+		case LEAPRAID_EVT_SAS_DEV_STAT_RC_CMP_INTERNAL_DEV_RESET:
+			starget_priv->tm_busy = 0;
+			break;
+		}
+	}
+
+out_unlock:
+	if (sas_dev)
+		leapraid_sdev_put(sas_dev);
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+}
+
+static void leapraid_set_volume_delete_flag(struct leapraid_adapter *adapter,
+					    u16 handle)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_starget_priv *sas_target_priv_data;
+	unsigned long flags;
+
+	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, handle);
+	if (raid_volume) {
+		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+		if (raid_volume->starget &&
+		    raid_volume->starget->hostdata) {
+			sas_target_priv_data = raid_volume->starget->hostdata;
+			sas_target_priv_data->deleted = 1;
+		}
+		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
+				       flags);
+		leapraid_raid_volume_put(raid_volume);
+	}
+}
+
+static void leapraid_check_ir_change_evt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_ir_change *evt_data)
+{
+	u16 phys_disk_dev_hdl;
+
+	switch (evt_data->reason_code) {
+	case LEAPRAID_EVT_IR_RC_VOLUME_DELETE:
+		leapraid_set_volume_delete_flag(
+			adapter,
+			le16_to_cpu(evt_data->vol_dev_hdl));
+		break;
+	case LEAPRAID_EVT_IR_RC_PD_UNHIDDEN_TO_DELETE:
+		phys_disk_dev_hdl =
+			le16_to_cpu(evt_data->phys_disk_dev_hdl);
+		if (!phys_disk_dev_hdl ||
+		    phys_disk_dev_hdl >
+		    adapter->adapter_attr.features.max_dev_handle) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Invalid device handle\n", __func__);
+		} else {
+			clear_bit(phys_disk_dev_hdl,
+				  adapter->dev_topo.pd_hdls);
+			leapraid_tgt_rst_send(adapter, phys_disk_dev_hdl);
+		}
+		break;
+	}
+}
+
+static void leapraid_topo_del_evts_process_exp_status(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_sas_topo_change_list *evt_data)
+{
+	struct leapraid_fw_evt_work *fw_evt = NULL;
+	struct leapraid_evt_data_sas_topo_change_list *loc_evt_data;
+	unsigned long flags;
+	u16 exp_hdl;
+
+	exp_hdl = le16_to_cpu(evt_data->exp_dev_hdl);
+
+	switch (evt_data->exp_status) {
+	case LEAPRAID_EVT_SAS_TOPO_ES_NOT_RESPONDING:
+		spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+		list_for_each_entry(fw_evt,
+				    &adapter->fw_evt_s.fw_evt_list, list) {
+			if (fw_evt->evt_type !=
+			    LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST ||
+			    fw_evt->ignore)
+				continue;
+
+			loc_evt_data = fw_evt->evt_data;
+			if ((loc_evt_data->exp_status ==
+			     LEAPRAID_EVT_SAS_TOPO_ES_ADDED ||
+			     loc_evt_data->exp_status ==
+			     LEAPRAID_EVT_SAS_TOPO_ES_RESPONDING) &&
+			    le16_to_cpu(loc_evt_data->exp_dev_hdl) == exp_hdl)
+				fw_evt->ignore = 1;
+		}
+		spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+		break;
+	default:
+		break;
+	}
+}
+
+static void leapraid_check_topo_del_evts(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_data_sas_topo_change_list *evt_data)
+{
+	int reason_code;
+	u16 hdl;
+	int i;
+
+	for (i = 0; i < evt_data->entry_num; i++) {
+		hdl = le16_to_cpu(evt_data->phy[i].attached_dev_hdl);
+		if (!hdl)
+			continue;
+
+		reason_code = evt_data->phy[i].phy_status &
+				LEAPRAID_EVT_SAS_TOPO_RC_MASK;
+		if (reason_code ==
+		    LEAPRAID_EVT_SAS_TOPO_RC_TARG_NOT_RESPONDING)
+			leapraid_tgt_not_responding(adapter, hdl);
+	}
+	leapraid_topo_del_evts_process_exp_status(adapter, evt_data);
+}
+
+static bool leapraid_async_evt_validate(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_notify_rep *event_notify_rep)
+{
+	size_t msg_len;
+	size_t evt_sz;
+	size_t evt_avail;
+	u16 evt;
+
+	msg_len = event_notify_rep->msg_len * sizeof(u32);
+	if (msg_len > LEAPRAID_REPLY_SIZE ||
+	    msg_len < offsetof(struct leapraid_evt_notify_rep, evt_data)) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid async event msg_len=%zu\n",
+			 __func__, msg_len);
+		return false;
+	}
+
+	evt_sz = le16_to_cpu(event_notify_rep->evt_data_len) * sizeof(u32);
+	evt_avail = msg_len -
+		    offsetof(struct leapraid_evt_notify_rep, evt_data);
+	if (evt_sz > evt_avail) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid async event evt_data_len=%zu\n",
+			 __func__, evt_sz);
+		return false;
+	}
+
+	evt = le16_to_cpu(event_notify_rep->evt);
+	switch (evt) {
+	case LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE:
+		if (evt_sz <
+		    sizeof(struct leapraid_evt_data_sas_dev_status_change))
+			goto invalid_evt_sz;
+		break;
+	case LEAPRAID_EVT_IR_CHANGE:
+		if (evt_sz < sizeof(struct leapraid_evt_data_ir_change))
+			goto invalid_evt_sz;
+		break;
+	case LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST:
+	{
+		struct leapraid_evt_data_sas_topo_change_list *evt_data =
+			(void *)event_notify_rep->evt_data;
+		size_t hdr_sz;
+
+		hdr_sz =
+			offsetof(struct leapraid_evt_data_sas_topo_change_list,
+				 phy);
+		if (evt_sz < hdr_sz)
+			goto invalid_evt_sz;
+
+		if (evt_data->entry_num >
+		    (evt_sz - hdr_sz) / sizeof(evt_data->phy[0]))
+			goto invalid_evt_sz;
+		break;
+	}
+	case LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE:
+		if (evt_sz <
+		    sizeof(struct leapraid_evt_data_sas_enc_dev_status_change))
+			goto invalid_evt_sz;
+		break;
+	default:
+		break;
+	}
+
+	return true;
+
+invalid_evt_sz:
+	dev_warn(&adapter->pdev->dev,
+		 "%s: Invalid async event size=%zu for evt=0x%x\n",
+		 __func__, evt_sz, evt);
+	return false;
+}
+
+static bool leapraid_async_process_evt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_notify_rep *event_notify_rep)
+{
+	u16 evt = le16_to_cpu(event_notify_rep->evt);
+	bool exit_flag = false;
+
+	if (adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering)
+		return true;
+
+	switch (evt) {
+	case LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE:
+		leapraid_sas_dev_stat_chg_evt(
+			adapter,
+			(struct leapraid_evt_data_sas_dev_status_change
+				*)event_notify_rep->evt_data);
+		break;
+	case LEAPRAID_EVT_IR_CHANGE:
+		leapraid_check_ir_change_evt(
+			adapter,
+			(struct leapraid_evt_data_ir_change
+				*)event_notify_rep->evt_data);
+		break;
+	case LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST:
+		leapraid_check_topo_del_evts(
+			adapter,
+			(struct leapraid_evt_data_sas_topo_change_list
+				*)event_notify_rep->evt_data);
+		if (adapter->access_ctrl.shost_recovering) {
+			exit_flag = true;
+			return exit_flag;
+		}
+		break;
+	case LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE:
+		break;
+	default:
+		exit_flag = true;
+		return exit_flag;
+	}
+
+	return exit_flag;
+}
+
+static void leapraid_async_evt_cb_enqueue(
+		struct leapraid_adapter *adapter,
+		struct leapraid_evt_notify_rep *evt_notify_rep)
+{
+	struct leapraid_fw_evt_work *fw_evt;
+	u16 evt_sz;
+
+	fw_evt = leapraid_alloc_fw_evt_work();
+	if (!fw_evt)
+		return;
+
+	evt_sz = le16_to_cpu(evt_notify_rep->evt_data_len) * sizeof(u32);
+	fw_evt->evt_data = kmemdup(evt_notify_rep->evt_data,
+				   evt_sz, GFP_ATOMIC);
+	if (!fw_evt->evt_data) {
+		leapraid_fw_evt_put(fw_evt);
+		return;
+	}
+	fw_evt->adapter = adapter;
+	fw_evt->evt_type = le16_to_cpu(evt_notify_rep->evt);
+	leapraid_fw_evt_add(adapter, fw_evt);
+	leapraid_fw_evt_put(fw_evt);
+}
+
+static void leapraid_async_evt_cb(struct leapraid_adapter *adapter,
+				  u8 msix_index, u32 rep_paddr)
+{
+	struct leapraid_evt_notify_rep *evt_notify_rep;
+
+	if (adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering)
+		return;
+
+	evt_notify_rep = leapraid_get_reply_vaddr(adapter, rep_paddr);
+	if (unlikely(!evt_notify_rep))
+		return;
+
+	if (!leapraid_async_evt_validate(adapter, evt_notify_rep))
+		return;
+
+	if (leapraid_async_process_evt(adapter, evt_notify_rep))
+		return;
+
+	leapraid_async_evt_cb_enqueue(adapter, evt_notify_rep);
+}
+
+static void leapraid_handle_async_event(struct leapraid_adapter *adapter,
+					u8 msix_index, u32 reply)
+{
+	struct leapraid_evt_notify_rep *leap_mpi_rep =
+		leapraid_get_reply_vaddr(adapter, reply);
+
+	if (!leap_mpi_rep)
+		return;
+
+	if (leap_mpi_rep->func != LEAPRAID_FUNC_EVENT_NOTIFY)
+		return;
+
+	leapraid_async_evt_cb(adapter, msix_index, reply);
+}
+
+void leapraid_async_turn_on_led(struct leapraid_adapter *adapter, u16 handle)
+{
+	struct leapraid_fw_evt_work *fw_event;
+
+	fw_event = leapraid_alloc_fw_evt_work();
+	if (!fw_event)
+		return;
+
+	fw_event->dev_handle = handle;
+	fw_event->adapter = adapter;
+	fw_event->evt_type = LEAPRAID_EVT_TURN_ON_PFA_LED;
+	leapraid_fw_evt_add(adapter, fw_event);
+	leapraid_fw_evt_put(fw_event);
+}
+
+static void leapraid_hardreset_barrier(struct leapraid_adapter *adapter)
+{
+	struct leapraid_fw_evt_work *fw_event;
+
+	fw_event = leapraid_alloc_fw_evt_work();
+	if (!fw_event)
+		return;
+
+	fw_event->adapter = adapter;
+	fw_event->evt_type = LEAPRAID_EVT_REMOVE_DEAD_DEV;
+	leapraid_fw_evt_add(adapter, fw_event);
+	leapraid_fw_evt_put(fw_event);
+}
+
+static void leapraid_scan_dev_complete(struct leapraid_adapter *adapter)
+{
+	struct leapraid_fw_evt_work *fw_evt;
+
+	fw_evt = leapraid_alloc_fw_evt_work();
+	if (!fw_evt)
+		return;
+
+	fw_evt->evt_type = LEAPRAID_EVT_SCAN_DEV_DONE;
+	fw_evt->adapter = adapter;
+	leapraid_fw_evt_add(adapter, fw_evt);
+	leapraid_fw_evt_put(fw_evt);
+}
+
+static void leapraid_handle_scan_cb(struct leapraid_adapter *adapter,
+				    struct leapraid_driver_cmd *cmd,
+				    struct leapraid_rep *rep)
+{
+	u16 status;
+
+	cmd->status &= ~LEAPRAID_CMD_PENDING;
+
+	status = le16_to_cpu(rep->adapter_status) &
+		 LEAPRAID_ADAPTER_STATUS_MASK;
+	if (status != LEAPRAID_ADAPTER_STATUS_SUCCESS)
+		adapter->scan_dev_desc.scan_dev_failed = 1;
+
+	if (!cmd->async_scan_dev) {
+		complete(&cmd->done);
+		return;
+	}
+
+	if (status == LEAPRAID_ADAPTER_STATUS_SUCCESS)
+		leapraid_scan_dev_complete(adapter);
+	else
+		adapter->scan_dev_desc.scan_start_failed = status;
+}
+
+static void leapraid_handle_ctl_cb(struct leapraid_adapter *adapter,
+				   struct leapraid_rep *rep,
+				   u16 taskid)
+{
+	struct leapraid_scsiio_rep *scsiio_reply;
+
+	if (rep->function != LEAPRAID_FUNC_SCSIIO &&
+	    rep->function != LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH)
+		return;
+
+	scsiio_reply = (struct leapraid_scsiio_rep *)rep;
+	if (!(scsiio_reply->scsi_state &
+	      LEAPRAID_SCSI_STATE_AUTOSENSE_VALID))
+		return;
+
+	memcpy(&adapter->driver_cmds.ctl_cmd.sense,
+	       leapraid_get_sense_buffer(adapter, taskid),
+	       min_t(u32, SCSI_SENSE_BUFFERSIZE,
+		     le32_to_cpu(scsiio_reply->sense_count)));
+}
+
+static bool leapraid_driver_cmds_done(struct leapraid_adapter *adapter,
+				      u16 taskid, u8 msix_index,
+				      u32 rep_paddr, u8 cb_idx)
+{
+	struct leapraid_rep *leap_mpi_rep =
+		leapraid_get_reply_vaddr(adapter, rep_paddr);
+	struct leapraid_driver_cmd *sp_cmd, *_sp_cmd = NULL;
+	u8 reply_len;
+
+	list_for_each_entry(sp_cmd, &adapter->driver_cmds.special_cmd_list,
+			    list)
+		if (cb_idx == sp_cmd->cb_idx) {
+			_sp_cmd = sp_cmd;
+			break;
+		}
+
+	if (WARN_ON(!_sp_cmd))
+		return true;
+	if (WARN_ON(_sp_cmd->status == LEAPRAID_CMD_NOT_USED))
+		return true;
+	if (WARN_ON(taskid != _sp_cmd->hp_taskid &&
+		    taskid != _sp_cmd->taskid &&
+		    taskid != _sp_cmd->inter_taskid))
+		return true;
+	_sp_cmd->status |= LEAPRAID_CMD_DONE;
+	if (leap_mpi_rep) {
+		reply_len = leap_mpi_rep->msg_len * sizeof(u32);
+		if (reply_len > LEAPRAID_REPLY_SIZE)
+			reply_len = LEAPRAID_REPLY_SIZE;
+		memcpy(&_sp_cmd->reply, leap_mpi_rep, reply_len);
+		_sp_cmd->status |= LEAPRAID_CMD_REPLY_VALID;
+
+		if (_sp_cmd->cb_idx == LEAPRAID_SCAN_DEV_CB_IDX) {
+			leapraid_handle_scan_cb(adapter, _sp_cmd,
+						leap_mpi_rep);
+			return true;
+		}
+
+		if (_sp_cmd->cb_idx == LEAPRAID_CTL_CB_IDX)
+			leapraid_handle_ctl_cb(adapter, leap_mpi_rep, taskid);
+	}
+
+	_sp_cmd->status &= ~LEAPRAID_CMD_PENDING;
+	complete(&_sp_cmd->done);
+
+	return true;
+}
+
+static void leapraid_complete_task(struct leapraid_adapter *adapter,
+				   u16 taskid, u8 msix_idx, u32 rep)
+{
+	bool scsiio_task = taskid <= adapter->shost->can_queue;
+
+	if (scsiio_task) {
+		if (leapraid_scsiio_done(adapter, taskid, msix_idx, rep))
+			leapraid_free_taskid(adapter, taskid);
+		return;
+	}
+
+	if (leapraid_driver_cmds_done(adapter, taskid, msix_idx, rep,
+				      leapraid_get_cb_idx(adapter, taskid)))
+		leapraid_free_taskid(adapter, taskid);
+}
+
+static void leapraid_request_descript_handler(
+		struct leapraid_adapter *adapter,
+		union leapraid_rep_desc_union *rpf,
+		u8 req_desc_type, u8 msix_idx)
+{
+	u32 rep;
+	u16 taskid;
+
+	rep = 0;
+	taskid = le16_to_cpu(rpf->dflt_rep.taskid);
+	switch (req_desc_type) {
+	case LEAPRAID_RPY_DESC_FLG_FP_SCSI_IO_SUCCESS:
+	case LEAPRAID_RPY_DESC_FLG_SCSI_IO_SUCCESS:
+		leapraid_complete_task(adapter, taskid, msix_idx, 0);
+		break;
+	case LEAPRAID_RPY_DESC_FLG_ADDRESS_REPLY:
+		rep = le32_to_cpu(rpf->addr_rep.rep_frame_addr);
+		if (rep > (u32)adapter->mem_desc.rep_msg_dma +
+			   adapter->adapter_attr.rep_msg_qd *
+			   LEAPRAID_REPLY_SIZE ||
+		    rep < (u32)adapter->mem_desc.rep_msg_dma)
+			rep = 0;
+
+		if (taskid)
+			leapraid_complete_task(adapter, taskid, msix_idx, rep);
+		else
+			leapraid_handle_async_event(adapter, msix_idx, rep);
+
+		if (!rep)
+			return;
+
+		adapter->rep_msg_host_idx =
+			(adapter->rep_msg_host_idx ==
+				(adapter->adapter_attr.rep_msg_qd - 1)) ?
+					0 : adapter->rep_msg_host_idx + 1;
+		adapter->mem_desc.rep_msg_addr[adapter->rep_msg_host_idx] =
+			cpu_to_le32(rep);
+		wmb(); /* Make sure that all write ops are in order */
+		writel(adapter->rep_msg_host_idx,
+		       &adapter->iomem_base->rep_msg_host_idx);
+
+		break;
+	default:
+		break;
+	}
+}
+
+int leapraid_rep_queue_handler(struct leapraid_rq *rq)
+{
+	struct leapraid_adapter *adapter = rq->adapter;
+	union leapraid_rep_desc_union *rep_desc;
+	u8 req_desc_type;
+	u64 finish_cmds;
+	u8 msix_idx;
+
+	msix_idx = rq->msix_idx;
+	finish_cmds = 0;
+	if (!atomic_add_unless(&rq->busy, LEAPRAID_BUSY_LIMIT,
+			       LEAPRAID_BUSY_LIMIT))
+		return finish_cmds;
+
+	rep_desc = &rq->rep_desc[rq->rep_post_host_idx];
+	req_desc_type = rep_desc->dflt_rep.rep_flg &
+				LEAPRAID_RPY_DESC_FLG_TYPE_MASK;
+	if (req_desc_type == LEAPRAID_RPY_DESC_FLG_UNUSED) {
+		atomic_dec(&rq->busy);
+		return finish_cmds;
+	}
+
+	for (;;) {
+		if (rep_desc->u.low == UINT_MAX ||
+		    rep_desc->u.high == UINT_MAX)
+			break;
+
+		leapraid_request_descript_handler(adapter, rep_desc,
+						  req_desc_type, msix_idx);
+		dev_dbg(&adapter->pdev->dev,
+			"LEAPRAID_SCSIIO: Handled Desc taskid %d, msix %d\n",
+			rep_desc->dflt_rep.taskid, msix_idx);
+		rep_desc->words = cpu_to_le64(ULLONG_MAX);
+		rq->rep_post_host_idx =
+			(rq->rep_post_host_idx ==
+				(adapter->adapter_attr.rep_desc_qd -
+					LEAPRAID_BUSY_LIMIT)) ?
+					0 : rq->rep_post_host_idx + 1;
+		req_desc_type =
+			rq->rep_desc[rq->rep_post_host_idx].dflt_rep.rep_flg &
+					LEAPRAID_RPY_DESC_FLG_TYPE_MASK;
+		finish_cmds++;
+		if (req_desc_type == LEAPRAID_RPY_DESC_FLG_UNUSED)
+			break;
+		rep_desc = rq->rep_desc + rq->rep_post_host_idx;
+	}
+
+	if (!finish_cmds) {
+		atomic_dec(&rq->busy);
+		return finish_cmds;
+	}
+
+	wmb(); /* Make sure that all write ops are in order */
+	writel(rq->rep_post_host_idx | ((msix_idx & LEAPRAID_MSIX_GROUP_MASK) <<
+					LEAPRAID_RPHI_MSIX_IDX_SHIFT),
+	       &adapter->iomem_base->rep_post_reg_idx[
+		       msix_idx / LEAPRAID_MSIX_GROUP_SIZE].idx);
+	atomic_dec(&rq->busy);
+	return finish_cmds;
+}
+
+static irqreturn_t leapraid_irq_handler(int irq, void *bus_id)
+{
+	struct leapraid_rq *rq = bus_id;
+	struct leapraid_adapter *adapter = rq->adapter;
+
+	dev_dbg(&adapter->pdev->dev,
+		"LEAPRAID_SCSIIO: Receive a interrupt, irq %d msix %d\n",
+		irq, rq->msix_idx);
+
+	if (adapter->mask_int)
+		return IRQ_NONE;
+
+	return (leapraid_rep_queue_handler(rq) > 0 ?
+		 IRQ_HANDLED : IRQ_NONE);
+}
+
+void leapraid_sync_irqs(struct leapraid_adapter *adapter, bool poll)
+{
+	struct leapraid_int_rq *int_rq;
+	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
+	unsigned int i;
+
+	if (!adapter->notification_desc.msix_enable)
+		return;
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering)
+		return;
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
+		int_rq = &adapter->notification_desc.int_rqs[i];
+		if (adapter->access_ctrl.shost_recovering ||
+		    adapter->access_ctrl.host_removing ||
+		    adapter->access_ctrl.pcie_recovering)
+			return;
+
+		if (int_rq->rq.msix_idx == 0)
+			continue;
+
+		synchronize_irq(pci_irq_vector(adapter->pdev,
+					       int_rq->rq.msix_idx));
+		if (poll)
+			leapraid_rep_queue_handler(&int_rq->rq);
+	}
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
+		blk_mq_poll_rq =
+			&adapter->notification_desc.blk_mq_poll_rqs[i];
+		if (adapter->access_ctrl.shost_recovering ||
+		    adapter->access_ctrl.host_removing ||
+		    adapter->access_ctrl.pcie_recovering)
+			return;
+
+		if (blk_mq_poll_rq->rq.msix_idx == 0)
+			continue;
+
+		leapraid_rep_queue_handler(&blk_mq_poll_rq->rq);
+	}
+}
+
+static void leapraid_sync_irqs_for_cleanup(struct leapraid_adapter *adapter)
+{
+	struct leapraid_notification_desc *desc = &adapter->notification_desc;
+	struct leapraid_int_rq *int_rq;
+	unsigned int i;
+
+	if (!adapter->mask_int && leapraid_pci_active(adapter))
+		leapraid_mask_int(adapter);
+
+	for (i = 0; i < desc->int_rqs_allocated; i++) {
+		int_rq = &desc->int_rqs[i];
+		synchronize_irq(pci_irq_vector(adapter->pdev,
+					       int_rq->rq.msix_idx));
+	}
+}
+
+void leapraid_mq_polling_pause(struct leapraid_adapter *adapter)
+{
+	struct leapraid_notification_desc *desc;
+	int iopoll_q_count;
+	int qid;
+
+	desc = &adapter->notification_desc;
+	iopoll_q_count = adapter->adapter_attr.rq_cnt - desc->iopoll_qdex;
+
+	for (qid = 0; qid < iopoll_q_count; qid++)
+		atomic_set(&desc->blk_mq_poll_rqs[qid].pause, 1);
+
+	for (qid = 0; qid < iopoll_q_count; qid++) {
+		while (atomic_read(&desc->blk_mq_poll_rqs[qid].busy)) {
+			cpu_relax();
+			udelay(LEAPRAID_IO_POLL_DELAY_US);
+		}
+	}
+}
+
+void leapraid_mq_polling_resume(struct leapraid_adapter *adapter)
+{
+	struct leapraid_notification_desc *desc;
+	int iopoll_q_count;
+	int qid;
+
+	desc = &adapter->notification_desc;
+	iopoll_q_count = adapter->adapter_attr.rq_cnt - desc->iopoll_qdex;
+
+	for (qid = 0; qid < iopoll_q_count; qid++)
+		atomic_set(&desc->blk_mq_poll_rqs[qid].pause, 0);
+}
+
+static int leapraid_unlock_host_diag(struct leapraid_adapter *adapter,
+				     u32 *host_diag)
+{
+	const u32 unlock_seq[] = {
+		LEAPRAID_WRSEQ_FLUSH_KEY_VALUE,
+		LEAPRAID_WRSEQ_1ST_KEY_VALUE,
+		LEAPRAID_WRSEQ_2ND_KEY_VALUE,
+		LEAPRAID_WRSEQ_3RD_KEY_VALUE,
+		LEAPRAID_WRSEQ_4TH_KEY_VALUE,
+		LEAPRAID_WRSEQ_5TH_KEY_VALUE,
+		LEAPRAID_WRSEQ_6TH_KEY_VALUE
+	};
+
+	const int max_retries = LEAPRAID_UNLOCK_RETRY_LIMIT;
+	int retry = 0;
+	unsigned int i;
+
+	*host_diag = 0;
+	while (retry++ <= max_retries) {
+		for (i = 0; i < ARRAY_SIZE(unlock_seq); i++)
+			writel(unlock_seq[i], &adapter->iomem_base->ws);
+
+		msleep(LEAPRAID_UNLOCK_SLEEP_MS);
+
+		*host_diag = leapraid_readl(&adapter->iomem_base->host_diag);
+		if (*host_diag & LEAPRAID_DIAG_WRITE_ENABLE)
+			return 0;
+	}
+
+	dev_err(&adapter->pdev->dev, "Try host reset timeout!\n");
+	return -EFAULT;
+}
+
+static int leapraid_host_diag_reset(struct leapraid_adapter *adapter)
+{
+	u32 host_diag;
+
+	pci_cfg_access_lock(adapter->pdev);
+
+	mutex_lock(&adapter->reset_desc.host_diag_mutex);
+	if (leapraid_unlock_host_diag(adapter, &host_diag))
+		goto out_cleanup;
+
+	writel(host_diag | LEAPRAID_DIAG_RESET,
+	       &adapter->iomem_base->host_diag);
+
+	msleep(LEAPRAID_MSLEEP_EXTRA_LONG_MS);
+	msleep(LEAPRAID_MSLEEP_NORMAL_MS);
+
+	host_diag = leapraid_readl(&adapter->iomem_base->host_diag);
+	if (host_diag == LEAPRAID_INVALID_HOST_DIAG_VAL ||
+	    host_diag & LEAPRAID_DIAG_RESET)
+		goto out_cleanup;
+
+	writel(0x0, &adapter->iomem_base->ws);
+	mutex_unlock(&adapter->reset_desc.host_diag_mutex);
+	if (!leapraid_wait_adapter_ready(adapter))
+		goto out_failed;
+
+	pci_cfg_access_unlock(adapter->pdev);
+	return 0;
+
+out_cleanup:
+	mutex_unlock(&adapter->reset_desc.host_diag_mutex);
+out_failed:
+	pci_cfg_access_unlock(adapter->pdev);
+	dev_err(&adapter->pdev->dev, "Host diag failed\n");
+	return -EFAULT;
+}
+
+static int leapraid_find_matching_port(
+		struct leapraid_card_port *card_port_table,
+		u8 count, u8 port_id, u64 sas_addr)
+{
+	int i;
+
+	for (i = 0; i < count; i++)
+		if (card_port_table[i].port_id == port_id &&
+		    card_port_table[i].sas_address == sas_addr)
+			return i;
+
+	return LEAPRAID_INVALID_INDEX;
+}
+
+static u8 leapraid_fill_card_port_table(
+		struct leapraid_adapter *adapter,
+		struct leapraid_sas_io_unit_p0 *sas_iounit_p0,
+		struct leapraid_card_port *new_card_port_table)
+{
+	u8 port_entry_num = 0, port_id;
+	u16 attached_hdl;
+	u64 attached_sas_addr;
+	int i, idx;
+
+	for (i = 0; i < adapter->dev_topo.card.phys_num; i++) {
+		if (sas_iounit_p0->phy_info[i].neg_link_rate >>
+		    LEAPRAID_SAS_NEG_LINK_RATE_SHIFT <
+		    LEAPRAID_SAS_NEG_LINK_RATE_1_5)
+			continue;
+
+		attached_hdl = le16_to_cpu(sas_iounit_p0->phy_info[i]
+					   .attached_dev_hdl);
+		if (leapraid_get_sas_address(adapter,
+					     attached_hdl,
+					     &attached_sas_addr) != 0)
+			continue;
+
+		port_id = sas_iounit_p0->phy_info[i].port;
+
+		idx = leapraid_find_matching_port(new_card_port_table,
+						  port_entry_num,
+						  port_id,
+						  attached_sas_addr);
+		if (idx >= 0) {
+			new_card_port_table[idx].phy_mask |= BIT(i);
+		} else {
+			new_card_port_table[port_entry_num].port_id = port_id;
+			new_card_port_table[port_entry_num].phy_mask = BIT(i);
+			new_card_port_table[port_entry_num].sas_address =
+				attached_sas_addr;
+			port_entry_num++;
+		}
+	}
+
+	return port_entry_num;
+}
+
+static u8 leapraid_set_new_card_port_table_after_reset(
+		struct leapraid_adapter *adapter,
+		struct leapraid_card_port *new_card_port_table)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_io_unit_p0 *sas_iounit_p0;
+	u8 port_entry_num = 0;
+	u16 sz;
+
+	sz = offsetof(struct leapraid_sas_io_unit_p0, phy_info) +
+		(adapter->dev_topo.card.phys_num *
+		 sizeof(struct leapraid_sas_io_unit0_phy_info));
+	sas_iounit_p0 = kzalloc(sz, GFP_KERNEL);
+	if (!sas_iounit_p0)
+		return port_entry_num;
+
+	cfgp1.size = sz;
+	if (leapraid_op_config_page(adapter, sas_iounit_p0, cfgp1, cfgp2,
+				    GET_SAS_IOUNIT_PG0) != 0)
+		goto out_free;
+
+	port_entry_num = leapraid_fill_card_port_table(adapter,
+						       sas_iounit_p0,
+						       new_card_port_table);
+out_free:
+	kfree(sas_iounit_p0);
+	return port_entry_num;
+}
+
+static void leapraid_update_existing_port(struct leapraid_adapter *adapter,
+					  struct leapraid_card_port *new_table,
+					  int entry_idx, int port_entry_num)
+{
+	struct leapraid_card_port *matched_card_port;
+	int matched_code;
+	int count, lcount = 0;
+	u64 sas_addr;
+	int i;
+
+	matched_code = leapraid_check_card_port(adapter,
+						&new_table[entry_idx],
+						&matched_card_port,
+						&count);
+
+	if (!matched_card_port)
+		return;
+
+	if (matched_code == SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS ||
+	    matched_code == SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS) {
+		leapraid_add_or_del_phys_from_existing_port(adapter,
+							    matched_card_port,
+							    new_table,
+							    entry_idx,
+							    port_entry_num);
+	} else if (matched_code == SAME_ADDR_ONLY) {
+		sas_addr = new_table[entry_idx].sas_address;
+		for (i = 0; i < port_entry_num; i++)
+			if (new_table[i].sas_address == sas_addr)
+				lcount++;
+
+		if (count > 1 || lcount > 1)
+			return;
+
+		leapraid_add_or_del_phys_from_existing_port(adapter,
+							    matched_card_port,
+							    new_table,
+							    entry_idx,
+							    port_entry_num);
+	}
+
+	if (matched_card_port->port_id != new_table[entry_idx].port_id)
+		matched_card_port->port_id = new_table[entry_idx].port_id;
+
+	matched_card_port->flg &= ~LEAPRAID_CARD_PORT_FLG_DIRTY;
+	matched_card_port->phy_mask = new_table[entry_idx].phy_mask;
+}
+
+static void leapraid_update_card_port_after_reset(
+		struct leapraid_adapter *adapter)
+{
+	struct leapraid_card_port *new_card_port_table;
+	struct leapraid_card_port *matched_card_port;
+	u8 port_entry_num;
+	u8 nr_phys;
+	int i;
+
+	if (leapraid_get_adapter_phys(adapter, &nr_phys) || !nr_phys)
+		return;
+
+	if (!adapter->dev_topo.card.card_phy) {
+		adapter->dev_topo.card.card_phy =
+			kzalloc_objs(struct leapraid_card_phy, nr_phys);
+		if (!adapter->dev_topo.card.card_phy)
+			return;
+	}
+
+	adapter->dev_topo.card.phys_num = nr_phys;
+
+	new_card_port_table = kzalloc_objs(struct leapraid_card_port,
+					   adapter->dev_topo.card.phys_num);
+	if (!new_card_port_table)
+		return;
+
+	port_entry_num =
+		leapraid_set_new_card_port_table_after_reset(
+			adapter,
+			new_card_port_table);
+	if (!port_entry_num)
+		goto out_free_port_table;
+
+	list_for_each_entry(matched_card_port,
+			    &adapter->dev_topo.card_port_list, list)
+		matched_card_port->flg |= LEAPRAID_CARD_PORT_FLG_DIRTY;
+
+	matched_card_port = NULL;
+	for (i = 0; i < port_entry_num; i++)
+		leapraid_update_existing_port(adapter,
+					      new_card_port_table,
+					      i, port_entry_num);
+out_free_port_table:
+	kfree(new_card_port_table);
+}
+
+static bool leapraid_is_valid_vphy(
+		struct leapraid_adapter *adapter,
+		struct leapraid_sas_io_unit_p0 *sas_io_unit_p0,
+		int phy_index)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_phy_p0 phy_p0;
+	u32 dev_info;
+
+	if (sas_io_unit_p0->phy_info[phy_index].neg_link_rate >>
+	    LEAPRAID_SAS_NEG_LINK_RATE_SHIFT <
+	    LEAPRAID_SAS_NEG_LINK_RATE_1_5)
+		return false;
+
+	dev_info = le32_to_cpu(sas_io_unit_p0->phy_info[phy_index]
+			       .controller_phy_dev_info);
+	if (!(dev_info & LEAPRAID_DEVTYP_SEP))
+		return false;
+
+	cfgp1.phy_number = phy_index;
+	if (leapraid_op_config_page(adapter, &phy_p0, cfgp1, cfgp2,
+				    GET_PHY_PG0))
+		return false;
+
+	if (!(le32_to_cpu(phy_p0.phy_info) & LEAPRAID_SAS_PHYINFO_VPHY))
+		return false;
+
+	return true;
+}
+
+static void leapraid_update_vphy_binding(struct leapraid_adapter *adapter,
+					 struct leapraid_card_port *card_port,
+					 struct leapraid_vphy *vphy,
+					 int phy_index, u8 may_new_port_id,
+					 u64 attached_sas_addr)
+{
+	struct leapraid_card_port *may_new_card_port;
+	struct leapraid_sas_dev *sas_dev;
+
+	may_new_card_port = leapraid_get_port_by_id(adapter,
+						    may_new_port_id,
+						    true);
+	if (!may_new_card_port) {
+		may_new_card_port = kzalloc_obj(*may_new_card_port);
+		if (!may_new_card_port)
+			return;
+		may_new_card_port->port_id = may_new_port_id;
+		dev_err(&adapter->pdev->dev,
+			"%s: New card port %p added, port=%d\n",
+			__func__, may_new_card_port, may_new_port_id);
+		list_add_tail(&may_new_card_port->list,
+			      &adapter->dev_topo.card_port_list);
+	}
+
+	if (card_port != may_new_card_port) {
+		if (!may_new_card_port->vphys_mask)
+			INIT_LIST_HEAD(&may_new_card_port->vphys_list);
+		may_new_card_port->vphys_mask |= BIT(phy_index);
+		card_port->vphys_mask &= ~BIT(phy_index);
+		list_move(&vphy->list, &may_new_card_port->vphys_list);
+
+		sas_dev = leapraid_get_sas_dev_by_addr(adapter,
+						       attached_sas_addr,
+						       card_port);
+		if (sas_dev) {
+			sas_dev->card_port = may_new_card_port;
+			leapraid_sdev_put(sas_dev);
+		}
+	}
+
+	if (may_new_card_port->flg & LEAPRAID_CARD_PORT_FLG_DIRTY) {
+		may_new_card_port->sas_address = 0;
+		may_new_card_port->phy_mask = 0;
+		may_new_card_port->flg &= ~LEAPRAID_CARD_PORT_FLG_DIRTY;
+	}
+	vphy->flg &= ~LEAPRAID_VPHY_FLG_DIRTY;
+}
+
+static void leapraid_update_vphys_after_reset(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_io_unit_p0 *sas_iounit_p0;
+	struct leapraid_card_port *card_port, *card_port_next;
+	struct leapraid_vphy *vphy, *vphy_next;
+	u64 attached_sas_addr;
+	u16 sz;
+	u16 attached_hdl;
+	bool found = false;
+	u8 port_id;
+	int i;
+
+	list_for_each_entry_safe(card_port, card_port_next,
+				 &adapter->dev_topo.card_port_list, list) {
+		if (!card_port->vphys_mask)
+			continue;
+
+		list_for_each_entry_safe(vphy, vphy_next,
+					 &card_port->vphys_list, list)
+			vphy->flg |= LEAPRAID_VPHY_FLG_DIRTY;
+	}
+
+	sz = offsetof(struct leapraid_sas_io_unit_p0, phy_info) +
+		(adapter->dev_topo.card.phys_num *
+		 sizeof(struct leapraid_sas_io_unit0_phy_info));
+	sas_iounit_p0 = kzalloc(sz, GFP_KERNEL);
+	if (!sas_iounit_p0)
+		return;
+
+	cfgp1.size = sz;
+	if (leapraid_op_config_page(adapter, sas_iounit_p0, cfgp1, cfgp2,
+				    GET_SAS_IOUNIT_PG0) != 0)
+		goto out_free;
+
+	for (i = 0; i < adapter->dev_topo.card.phys_num; i++) {
+		if (!leapraid_is_valid_vphy(adapter, sas_iounit_p0, i))
+			continue;
+
+		attached_hdl =
+			le16_to_cpu(sas_iounit_p0->phy_info[i]
+					.attached_dev_hdl);
+		if (leapraid_get_sas_address(adapter, attached_hdl,
+					     &attached_sas_addr) != 0)
+			continue;
+
+		found = false;
+		card_port = NULL;
+		card_port_next = NULL;
+		list_for_each_entry_safe(card_port, card_port_next,
+					 &adapter->dev_topo.card_port_list,
+					 list) {
+			if (!card_port->vphys_mask)
+				continue;
+
+			list_for_each_entry_safe(vphy, vphy_next,
+						 &card_port->vphys_list,
+						 list) {
+				if (!(vphy->flg & LEAPRAID_VPHY_FLG_DIRTY))
+					continue;
+
+				if (vphy->sas_address != attached_sas_addr)
+					continue;
+
+				if (!(vphy->phy_mask & BIT(i)))
+					vphy->phy_mask = BIT(i);
+
+				port_id = sas_iounit_p0->phy_info[i].port;
+
+				leapraid_update_vphy_binding(adapter,
+							     card_port,
+							     vphy,
+							     i,
+							     port_id,
+							     attached_sas_addr);
+
+				found = true;
+				break;
+			}
+			if (found)
+				break;
+		}
+	}
+out_free:
+	kfree(sas_iounit_p0);
+}
+
+static void leapraid_mark_all_dev_deleted(struct leapraid_adapter *adapter)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct scsi_device *sdev;
+
+	shost_for_each_device(sdev, adapter->shost) {
+		sdev_priv = sdev->hostdata;
+		if (sdev_priv && sdev_priv->starget_priv)
+			sdev_priv->starget_priv->deleted = 1;
+	}
+}
+
+static void leapraid_free_enc_list(struct leapraid_adapter *adapter)
+{
+	struct leapraid_enc_node *enc_dev, *enc_dev_next;
+	LIST_HEAD(free_list);
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
+	list_splice_init(&adapter->dev_topo.enc_list, &free_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+
+	list_for_each_entry_safe(enc_dev, enc_dev_next, &free_list, list) {
+		list_del(&enc_dev->list);
+		kfree(enc_dev);
+	}
+}
+
+static void leapraid_rebuild_enc_list_after_reset(
+		struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_enc_node *enc_node;
+	u16 enc_hdl;
+	unsigned long flags;
+	int rc;
+
+	leapraid_free_enc_list(adapter);
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (enc_hdl = 0xFFFF; ; enc_hdl = le16_to_cpu(enc_node->pg0.enc_hdl)) {
+		enc_node = kzalloc_obj(*enc_node);
+		if (!enc_node)
+			return;
+
+		cfgp2.handle = enc_hdl;
+		rc = leapraid_op_config_page(adapter, &enc_node->pg0, cfgp1,
+					     cfgp2, GET_SAS_ENCLOSURE_PG0);
+		if (rc) {
+			kfree(enc_node);
+			return;
+		}
+
+		spin_lock_irqsave(&adapter->dev_topo.enc_lock, flags);
+		list_add_tail(&enc_node->list, &adapter->dev_topo.enc_list);
+		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, flags);
+	}
+}
+
+static void leapraid_mark_resp_sas_dev(struct leapraid_adapter *adapter,
+				       struct leapraid_sas_dev_p0 *sas_dev_p0)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_enc_node *enc_node = NULL;
+	struct leapraid_card_port *card_port;
+	struct leapraid_sas_dev *sas_dev;
+	struct scsi_target *starget;
+	unsigned long flags;
+	unsigned long enc_flags = 0;
+	u16 enc_hdl;
+	u64 enc_lid = 0;
+
+	card_port = leapraid_get_port_by_id(adapter, sas_dev_p0->physical_port,
+					    false);
+	enc_hdl = le16_to_cpu(sas_dev_p0->enc_hdl);
+	if (enc_hdl) {
+		spin_lock_irqsave(&adapter->dev_topo.enc_lock, enc_flags);
+		enc_node = leapraid_enc_find_by_hdl(adapter, enc_hdl);
+		if (enc_node)
+			enc_lid = le64_to_cpu(enc_node->pg0.enc_lid);
+		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, enc_flags);
+		if (!enc_node)
+			dev_info(&adapter->pdev->dev,
+				 "enc hdl 0x%04x has no matched enc dev\n",
+				 enc_hdl);
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	list_for_each_entry(sas_dev, &adapter->dev_topo.sas_dev_list, list) {
+		if (sas_dev->sas_addr != le64_to_cpu(sas_dev_p0->sas_address) ||
+		    sas_dev->slot != le16_to_cpu(sas_dev_p0->slot) ||
+		    sas_dev->card_port != card_port)
+			continue;
+
+		sas_dev->resp = 1;
+		starget = sas_dev->starget;
+		if (starget && starget->hostdata) {
+			starget_priv = starget->hostdata;
+			starget_priv->tm_busy = 0;
+			starget_priv->deleted = 0;
+		} else {
+			starget_priv = NULL;
+		}
+
+		if (starget) {
+			starget_printk(
+				KERN_INFO, starget,
+				"dev: hdl=0x%04x saddr=0x%016llx port_id=%d\n",
+				sas_dev->hdl,
+				(unsigned long long)sas_dev->sas_addr,
+				sas_dev->card_port->port_id);
+			if (sas_dev->enc_hdl != 0)
+				starget_printk(
+					KERN_INFO,
+					starget,
+					"enc info: enc_lid=0x%016llx slot=%d\n",
+					(unsigned long long)sas_dev->enc_lid,
+					sas_dev->slot);
+		}
+
+		if (le16_to_cpu(sas_dev_p0->flg) &
+		    LEAPRAID_SAS_DEV_P0_FLG_ENC_LEVEL_VALID) {
+			sas_dev->enc_level = sas_dev_p0->enc_level;
+			memcpy(sas_dev->connector_name,
+			       sas_dev_p0->connector_name,
+			       LEAPRAID_SAS_DEV_P0_CON_NAME_LEN);
+			sas_dev->connector_name[
+				LEAPRAID_SAS_DEV_P0_CON_NAME_LEN] = '\0';
+		} else {
+			sas_dev->enc_level = 0;
+			sas_dev->connector_name[0] = '\0';
+		}
+
+		sas_dev->enc_hdl = enc_hdl;
+		sas_dev->enc_lid = enc_lid;
+
+		if (sas_dev->hdl == le16_to_cpu(sas_dev_p0->dev_hdl))
+			goto unlock;
+
+		dev_info(&adapter->pdev->dev,
+			 "hdl changed: 0x%04x -> 0x%04x\n",
+			 sas_dev->hdl, sas_dev_p0->dev_hdl);
+		sas_dev->hdl = le16_to_cpu(sas_dev_p0->dev_hdl);
+		if (starget_priv)
+			starget_priv->hdl = le16_to_cpu(sas_dev_p0->dev_hdl);
+
+		goto unlock;
+	}
+unlock:
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+}
+
+static void leapraid_search_resp_sas_dev(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_p0;
+	u32 device_info;
+
+	if (list_empty(&adapter->dev_topo.sas_dev_list))
+		return;
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (cfgp2.handle = 0xFFFF;
+	     !leapraid_op_config_page(adapter, &sas_dev_p0,
+				      cfgp1, cfgp2, GET_SAS_DEVICE_PG0);
+	     cfgp2.handle = le16_to_cpu(sas_dev_p0.dev_hdl)) {
+		device_info = le32_to_cpu(sas_dev_p0.dev_info);
+		if (!(leapraid_is_end_dev(device_info)))
+			continue;
+
+		leapraid_mark_resp_sas_dev(adapter, &sas_dev_p0);
+	}
+}
+
+static void leapraid_mark_resp_raid_volume(struct leapraid_adapter *adapter,
+					   u64 wwid, u16 hdl)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_starget_priv *starget_priv = NULL;
+	struct scsi_target *starget;
+	unsigned long flags;
+	u64 volume_wwid;
+	u16 old_hdl;
+
+	raid_volume = leapraid_raid_volume_find_by_wwid(adapter, wwid);
+	if (!raid_volume)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	if (!raid_volume->starget) {
+		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
+				       flags);
+		leapraid_raid_volume_put(raid_volume);
+		return;
+	}
+
+	starget = raid_volume->starget;
+	if (starget->hostdata) {
+		starget_priv = starget->hostdata;
+		starget_priv->deleted = 0;
+	}
+
+	raid_volume->resp = 1;
+	volume_wwid = raid_volume->wwid;
+	old_hdl = raid_volume->hdl;
+	if (old_hdl != hdl) {
+		raid_volume->hdl = hdl;
+		if (starget_priv)
+			starget_priv->hdl = hdl;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	starget_printk(KERN_INFO, starget,
+		       "raid volume: hdl=0x%04x, wwid=0x%016llx\n",
+		       hdl, (unsigned long long)volume_wwid);
+	if (old_hdl != hdl)
+		dev_info(&adapter->pdev->dev,
+			 "hdl changed: 0x%04x -> 0x%04x\n",
+			 old_hdl, hdl);
+
+	leapraid_raid_volume_put(raid_volume);
+}
+
+static void leapraid_search_resp_raid_volume(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_1 cfgp1_extra = {0};
+	union cfg_param_2 cfgp2 = {0};
+	union cfg_param_2 cfgp2_extra = {0};
+	struct leapraid_raidvol_p1 raidvol_p1;
+	struct leapraid_raidvol_p0 raidvol_p0;
+	struct leapraid_raidpd_p0 raidpd_p0;
+	unsigned long flags;
+	u16 hdl;
+	u8 phys_disk_num;
+	bool is_empty;
+
+	if (!adapter->adapter_attr.raid_support)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	is_empty = list_empty(&adapter->dev_topo.raid_volume_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+	if (is_empty)
+		return;
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (hdl = 0xFFFF, cfgp2.handle = hdl;
+	     !leapraid_op_config_page(adapter, &raidvol_p1, cfgp1, cfgp2,
+				      GET_RAID_VOLUME_PG1);
+	     cfgp2.handle = hdl) {
+		hdl = le16_to_cpu(raidvol_p1.dev_hdl);
+		cfgp1_extra.size = sizeof(struct leapraid_raidvol_p0);
+		cfgp2_extra.handle = hdl;
+		if (leapraid_op_config_page(adapter, &raidvol_p0, cfgp1_extra,
+					    cfgp2_extra, GET_RAID_VOLUME_PG0))
+			continue;
+
+		if (raidvol_p0.volume_state == LEAPRAID_VOL_STATE_OPTIMAL ||
+		    raidvol_p0.volume_state == LEAPRAID_VOL_STATE_ONLINE ||
+		    raidvol_p0.volume_state == LEAPRAID_VOL_STATE_DEGRADED)
+			leapraid_mark_resp_raid_volume(
+				adapter,
+				le64_to_cpu(raidvol_p1.wwid),
+				hdl);
+	}
+
+	memset(adapter->dev_topo.pd_hdls, 0, adapter->dev_topo.pd_hdls_sz);
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (phys_disk_num = 0xFF, cfgp2.form_specific = phys_disk_num;
+	     !leapraid_op_config_page(adapter, &raidpd_p0, cfgp1, cfgp2,
+				      GET_PHY_DISK_PG0);
+	     cfgp2.form_specific = phys_disk_num) {
+		phys_disk_num = raidpd_p0.phys_disk_num;
+		hdl = le16_to_cpu(raidpd_p0.dev_hdl);
+		if (!hdl ||
+		    hdl > adapter->adapter_attr.features.max_dev_handle)
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Invalid device handle\n", __func__);
+		else
+			set_bit(hdl, adapter->dev_topo.pd_hdls);
+	}
+}
+
+static void leapraid_mark_resp_exp(struct leapraid_adapter *adapter,
+				   struct leapraid_exp_p0 *exp_pg0)
+{
+	struct leapraid_enc_node *enc_node = NULL;
+	struct leapraid_topo_node *topo_node_exp;
+	u16 enc_hdl = le16_to_cpu(exp_pg0->enc_hdl);
+	u64 sas_address = le64_to_cpu(exp_pg0->sas_address);
+	u16 hdl = le16_to_cpu(exp_pg0->dev_hdl);
+	u8 port_id = exp_pg0->physical_port;
+	struct leapraid_card_port *card_port = leapraid_get_port_by_id(adapter,
+								       port_id,
+								       false);
+	unsigned long flags;
+	unsigned long enc_flags = 0;
+	u64 enc_lid = 0;
+	int i;
+
+	if (enc_hdl) {
+		spin_lock_irqsave(&adapter->dev_topo.enc_lock, enc_flags);
+		enc_node = leapraid_enc_find_by_hdl(adapter, enc_hdl);
+		if (enc_node)
+			enc_lid = le64_to_cpu(enc_node->pg0.enc_lid);
+		spin_unlock_irqrestore(&adapter->dev_topo.enc_lock, enc_flags);
+	}
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	list_for_each_entry(topo_node_exp, &adapter->dev_topo.exp_list, list) {
+		if (topo_node_exp->sas_address != sas_address ||
+		    topo_node_exp->card_port != card_port)
+			continue;
+
+		topo_node_exp->resp = 1;
+		topo_node_exp->enc_hdl = enc_hdl;
+		topo_node_exp->enc_lid = enc_lid;
+		if (topo_node_exp->hdl == hdl)
+			goto unlock;
+
+		dev_info(&adapter->pdev->dev,
+			 "hdl changed: 0x%04x -> 0x%04x\n",
+			 topo_node_exp->hdl, hdl);
+		topo_node_exp->hdl = hdl;
+		for (i = 0; i < topo_node_exp->phys_num; i++)
+			topo_node_exp->card_phy[i].hdl = hdl;
+		goto unlock;
+	}
+unlock:
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+}
+
+static void leapraid_search_resp_exp(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_exp_p0 exp_p0;
+	u64 sas_address;
+	u16 hdl;
+	u8 port;
+
+	if (list_empty(&adapter->dev_topo.exp_list))
+		return;
+
+	cfgp1.form = LEAPRAID_SAS_CFG_PGAD_GET_NEXT_LOOP;
+	for (hdl = 0xFFFF, cfgp2.handle = hdl;
+	     !leapraid_op_config_page(adapter, &exp_p0, cfgp1, cfgp2,
+				      GET_SAS_EXPANDER_PG0);
+	     cfgp2.handle = hdl) {
+		hdl = le16_to_cpu(exp_p0.dev_hdl);
+		sas_address = le64_to_cpu(exp_p0.sas_address);
+		port = exp_p0.physical_port;
+
+		dev_dbg(&adapter->pdev->dev,
+			"exp detected: hdl=0x%04x, sas=0x%016llx, port=%u",
+			hdl, (unsigned long long)sas_address,
+			adapter->adapter_attr.enable_mp ? port :
+				LEAPRAID_DISABLE_MP_PORT_ID);
+		leapraid_mark_resp_exp(adapter, &exp_p0);
+	}
+}
+
+void leapraid_wait_cmds_done(struct leapraid_adapter *adapter)
+{
+	struct leapraid_io_req_tracker *io_req_tracker;
+	unsigned long flags;
+	u16 i;
+
+	adapter->reset_desc.pending_io_cnt = 0;
+	if (!leapraid_pci_active(adapter)) {
+		dev_err(&adapter->pdev->dev,
+			"%s %s: PCI error, device reset or unplugged!\n",
+			adapter->adapter_attr.name, __func__);
+		return;
+	}
+
+	if (leapraid_get_adapter_state(adapter) != LEAPRAID_DB_OPERATIONAL)
+		return;
+
+	spin_lock_irqsave(&adapter->dynamic_task_desc.task_lock, flags);
+	for (i = 1; i <= adapter->shost->can_queue; i++) {
+		io_req_tracker = leapraid_get_io_tracker_from_taskid(adapter,
+								     i);
+		if (io_req_tracker && io_req_tracker->taskid != 0 &&
+		    io_req_tracker->scmd)
+			adapter->reset_desc.pending_io_cnt++;
+	}
+	spin_unlock_irqrestore(&adapter->dynamic_task_desc.task_lock, flags);
+
+	if (!adapter->reset_desc.pending_io_cnt)
+		return;
+
+	wait_event_timeout(adapter->reset_desc.reset_wait_queue,
+			   adapter->reset_desc.pending_io_cnt == 0,
+			   LEAPRAID_IO_CMD_TIMEOUT * HZ);
+}
+
+int leapraid_hard_reset_handler(struct leapraid_adapter *adapter,
+				enum reset_type type)
+{
+	unsigned long flags;
+	bool wake = false;
+	int rc;
+
+	mutex_lock(&adapter->reset_desc.adapter_reset_mutex);
+
+	if (adapter->access_ctrl.shost_recover_async) {
+		rc = adapter->reset_desc.adapter_reset_results;
+		dev_info(&adapter->pdev->dev,
+			 "Skip nested hard reset, async evt running, rc=%d\n",
+			 rc);
+		mutex_unlock(&adapter->reset_desc.adapter_reset_mutex);
+		return rc;
+	}
+
+	if (!leapraid_pci_active(adapter)) {
+		if (leapraid_pci_removed(adapter)) {
+			dev_info(&adapter->pdev->dev,
+				 "pci_dev removed, pause poll, clean cmds\n");
+			leapraid_mq_polling_pause(adapter);
+			leapraid_clean_active_scsi_cmds(adapter);
+			leapraid_mq_polling_resume(adapter);
+		}
+		dev_err(&adapter->pdev->dev, "PCIe unavailable!\n");
+		mutex_unlock(&adapter->reset_desc.adapter_reset_mutex);
+		return -ENXIO;
+	}
+
+	dev_info(&adapter->pdev->dev, "Starting hard reset\n");
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	adapter->access_ctrl.shost_recovering = 1;
+	adapter->access_ctrl.shost_recover_async = 1;
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+
+	leapraid_wait_cmds_done(adapter);
+	leapraid_mask_int(adapter);
+	leapraid_mq_polling_pause(adapter);
+	rc = leapraid_make_adapter_ready(adapter, type);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"Failed to make adapter ready, rc=%d\n", rc);
+		goto out_cleanup;
+	}
+
+	rc = leapraid_fw_log_init(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Firmware log init failed\n");
+		goto out_cleanup;
+	}
+
+	leapraid_clean_active_cmds(adapter);
+	if (adapter->scan_dev_desc.driver_loading &&
+	    adapter->scan_dev_desc.scan_dev_failed) {
+		dev_err(&adapter->pdev->dev,
+			"Previous device scan failed or driver loading\n");
+		adapter->access_ctrl.host_removing = 1;
+		rc = -EFAULT;
+		goto out_cleanup;
+	}
+
+	rc = leapraid_make_adapter_available(adapter);
+	if (!rc) {
+		dev_info(&adapter->pdev->dev,
+			 "Adapter is now available, rebuilding topology\n");
+		if (adapter->adapter_attr.enable_mp) {
+			leapraid_update_card_port_after_reset(adapter);
+			leapraid_update_vphys_after_reset(adapter);
+		}
+		leapraid_mark_all_dev_deleted(adapter);
+		leapraid_rebuild_enc_list_after_reset(adapter);
+		leapraid_search_resp_sas_dev(adapter);
+		leapraid_search_resp_raid_volume(adapter);
+		leapraid_search_resp_exp(adapter);
+		leapraid_hardreset_barrier(adapter);
+	}
+out_cleanup:
+	if (rc)
+		dev_err(&adapter->pdev->dev, "Hard reset failed\n");
+
+	spin_lock_irqsave(&adapter->reset_desc.adapter_reset_lock, flags);
+	adapter->reset_desc.adapter_reset_results = rc;
+	adapter->access_ctrl.shost_recovering = 0;
+	wake_up(&adapter->access_ctrl.recovery_waitq);
+	if (rc) {
+		adapter->access_ctrl.shost_recover_async = 0;
+		wake = true;
+	}
+	spin_unlock_irqrestore(&adapter->reset_desc.adapter_reset_lock, flags);
+	if (wake)
+		wake_up(&adapter->access_ctrl.shost_recover_wq);
+
+	adapter->reset_desc.reset_cnt++;
+	mutex_unlock(&adapter->reset_desc.adapter_reset_mutex);
+
+	if (rc)
+		leapraid_clean_active_scsi_cmds(adapter);
+	leapraid_mq_polling_resume(adapter);
+
+	return rc;
+}
+
+static int leapraid_get_adapter_features(struct leapraid_adapter *adapter)
+{
+	struct leapraid_adapter_features_req leap_mpi_req;
+	struct leapraid_adapter_features_rep leap_mpi_rep;
+	u8 fw_major, fw_minor, fw_build, fw_release;
+	u32 req_sz;
+	u32 rep_sz;
+	u32 db;
+	int r;
+
+	db = leapraid_readl(&adapter->iomem_base->db);
+	if (db & LEAPRAID_DB_USED ||
+	    (db & LEAPRAID_DB_MASK) == LEAPRAID_DB_FAULT) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Doorbell used or fault\n", __func__);
+		return -EFAULT;
+	}
+
+	if ((db & LEAPRAID_DB_MASK) != LEAPRAID_DB_READY &&
+	    (db & LEAPRAID_DB_MASK) != LEAPRAID_DB_OPERATIONAL &&
+	    !leapraid_wait_adapter_ready(adapter)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: adapter not ready\n", __func__);
+		return -EFAULT;
+	}
+
+	req_sz = sizeof(struct leapraid_adapter_features_req);
+	rep_sz = sizeof(struct leapraid_adapter_features_rep);
+	memset(&leap_mpi_req, 0, req_sz);
+	memset(&leap_mpi_rep, 0, rep_sz);
+	leap_mpi_req.func = LEAPRAID_FUNC_GET_ADAPTER_FEATURES;
+	r = leapraid_handshake_func(adapter,
+				    req_sz,
+				    (u32 *)&leap_mpi_req,
+				    rep_sz,
+				    (u16 *)&leap_mpi_rep);
+	if (r) {
+		dev_err(&adapter->pdev->dev,
+			"%s %s: Handshake failed, r=%d\n",
+			adapter->adapter_attr.name, __func__, r);
+		return r;
+	}
+
+	memset(&adapter->adapter_attr.features, 0,
+	       sizeof(struct leapraid_adapter_features));
+	adapter->adapter_attr.features.req_slot =
+		le16_to_cpu(leap_mpi_rep.req_slot);
+	adapter->adapter_attr.features.hp_slot =
+		le16_to_cpu(leap_mpi_rep.hp_slot);
+	adapter->adapter_attr.features.adapter_caps =
+		le32_to_cpu(leap_mpi_rep.adapter_caps);
+	adapter->adapter_attr.features.max_msix_vectors =
+		leap_mpi_rep.max_msix_vectors;
+	adapter->adapter_attr.features.max_volumes =
+		leap_mpi_rep.max_volumes;
+	if (!adapter->adapter_attr.features.max_volumes)
+		adapter->adapter_attr.features.max_volumes =
+			LEAPRAID_MAX_VOLUMES_DEFAULT;
+	adapter->adapter_attr.features.max_dev_handle =
+		le16_to_cpu(leap_mpi_rep.max_dev_hdl);
+	if (!adapter->adapter_attr.features.max_dev_handle)
+		adapter->adapter_attr.features.max_dev_handle =
+			LEAPRAID_MAX_DEV_HANDLE_DEFAULT;
+	adapter->adapter_attr.features.min_dev_handle =
+		le16_to_cpu(leap_mpi_rep.min_dev_hdl);
+	if (adapter->adapter_attr.features.adapter_caps &
+	    LEAPRAID_ADAPTER_FEATURES_CAP_INTEGRATED_RAID)
+		adapter->adapter_attr.raid_support = 1;
+	adapter->adapter_attr.wideport_max_queue_depth =
+		le16_to_cpu(leap_mpi_rep.sas_wide_max_qdepth) ?
+		le16_to_cpu(leap_mpi_rep.sas_wide_max_qdepth) :
+		LEAPRAID_SAS_QUEUE_DEPTH;
+	adapter->adapter_attr.narrowport_max_queue_depth =
+		le16_to_cpu(leap_mpi_rep.sas_narrow_max_qdepth) ?
+		le16_to_cpu(leap_mpi_rep.sas_narrow_max_qdepth) :
+		LEAPRAID_SAS_QUEUE_DEPTH;
+	adapter->adapter_attr.sata_max_queue_depth =
+		leap_mpi_rep.sata_max_qdepth ?
+		leap_mpi_rep.sata_max_qdepth :
+		LEAPRAID_SATA_QUEUE_DEPTH;
+	adapter->adapter_attr.raid_volume_max_queue_depth =
+		LEAPRAID_RAID_QUEUE_DEPTH;
+	dev_info(&adapter->pdev->dev,
+		 "max: wp qd=%d, np qd=%d, SATA qd=%d, raid qd=%d\n",
+		 adapter->adapter_attr.wideport_max_queue_depth,
+		 adapter->adapter_attr.narrowport_max_queue_depth,
+		 adapter->adapter_attr.sata_max_queue_depth,
+		 adapter->adapter_attr.raid_volume_max_queue_depth);
+	if (WARN_ON(!(adapter->adapter_attr.features.adapter_caps &
+		      LEAPRAID_ADAPTER_FEATURES_CAP_ATOMIC_REQ)))
+		return -EFAULT;
+	adapter->adapter_attr.features.fw_version =
+		le32_to_cpu(leap_mpi_rep.fw_version);
+
+	fw_major = (adapter->adapter_attr.features.fw_version >>
+		    LEAPRAID_VER_MAJOR_SHIFT) & LEAPRAID_VER_MASK;
+	fw_minor = (adapter->adapter_attr.features.fw_version >>
+		    LEAPRAID_VER_MINOR_SHIFT) & LEAPRAID_VER_MASK;
+	fw_build = (adapter->adapter_attr.features.fw_version >>
+		    LEAPRAID_VER_BUILD_SHIFT)  & LEAPRAID_VER_MASK;
+	fw_release =
+		adapter->adapter_attr.features.fw_version & LEAPRAID_VER_MASK;
+
+	dev_info(&adapter->pdev->dev,
+		 "Firmware version: %u.%u.%u.%u (0x%08x)\n",
+		 fw_major, fw_minor, fw_build, fw_release,
+		 adapter->adapter_attr.features.fw_version);
+
+	adapter->adapter_attr.features.msg_ver =
+		le16_to_cpu(leap_mpi_rep.msg_ver);
+	adapter->adapter_attr.features.product_id =
+		le16_to_cpu(leap_mpi_rep.product_id);
+	dev_info(&adapter->pdev->dev,
+		 "message version: 0x%x, product id 0x%x\n",
+		 adapter->adapter_attr.features.msg_ver,
+		 adapter->adapter_attr.features.product_id);
+
+	if (adapter->adapter_attr.features.msg_ver < 0x1000) {
+		dev_err(&adapter->pdev->dev, "Device not supported\n");
+		return -EFAULT;
+	}
+	adapter->shost->max_id = LEAPRAID_INVALID_INITIAL_VALUE;
+
+	return 0;
+}
+
+static inline void leapraid_disable_pcie(struct leapraid_adapter *adapter)
+{
+	mutex_lock(&adapter->access_ctrl.pci_access_lock);
+	if (adapter->iomem_base) {
+		iounmap(adapter->iomem_base);
+		adapter->iomem_base = NULL;
+	}
+	if (pci_is_enabled(adapter->pdev)) {
+		pci_release_regions(adapter->pdev);
+		pci_disable_device(adapter->pdev);
+	}
+	mutex_unlock(&adapter->access_ctrl.pci_access_lock);
+}
+
+static int leapraid_enable_pcie(struct leapraid_adapter *adapter)
+{
+	u64 dma_mask;
+	int rc;
+
+	rc = pci_enable_device(adapter->pdev);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Failed to enable PCI device\n");
+		return rc;
+	}
+
+	rc = pci_request_regions(adapter->pdev, LEAPRAID_DRIVER_NAME);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"Failed to obtain PCI resources\n");
+		return rc;
+	}
+
+	if (sizeof(dma_addr_t) > 4) {
+		dma_mask = DMA_BIT_MASK(DMA_64_BITS);
+		adapter->adapter_attr.use_32_dma_mask = 0;
+	} else {
+		dma_mask = DMA_BIT_MASK(DMA_32_BITS);
+		adapter->adapter_attr.use_32_dma_mask = 1;
+	}
+
+	rc = dma_set_mask_and_coherent(&adapter->pdev->dev, dma_mask);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"Failed to set %lld DMA mask\n", dma_mask);
+		return rc;
+	}
+	adapter->iomem_base = ioremap(pci_resource_start(adapter->pdev, 0),
+				      sizeof(struct leapraid_reg_base));
+	if (!adapter->iomem_base) {
+		dev_err(&adapter->pdev->dev,
+			"Failed to map memory for controller registers\n");
+		return -ENOMEM;
+	}
+
+	pci_set_master(adapter->pdev);
+
+	return 0;
+}
+
+static void leapraid_cpus_on_irq(struct leapraid_adapter *adapter)
+{
+	struct leapraid_int_rq *int_rq;
+	unsigned int i, base_group, this_group;
+	unsigned int cpu, nr_cpus, total_msix, index;
+
+	total_msix = adapter->notification_desc.iopoll_qdex;
+	nr_cpus = num_online_cpus();
+
+	if (!nr_cpus || !total_msix)
+		return;
+	base_group = nr_cpus / total_msix;
+
+	cpu = cpumask_first(cpu_online_mask);
+	for (index = 0; index < adapter->notification_desc.iopoll_qdex;
+	     index++) {
+		int_rq = &adapter->notification_desc.int_rqs[index];
+
+		if (cpu >= adapter->notification_desc.msix_cpu_map_sz)
+			break;
+
+		this_group = base_group +
+				(index < (nr_cpus % total_msix) ? 1 : 0);
+
+		for (i = 0 ; i < this_group ; i++) {
+			if (cpu >= adapter->notification_desc.msix_cpu_map_sz)
+				break;
+
+			adapter->notification_desc.msix_cpu_map[cpu] =
+				int_rq->rq.msix_idx;
+			cpu = cpumask_next(cpu, cpu_online_mask);
+		}
+	}
+}
+
+static void leapraid_map_msix_to_cpu(struct leapraid_adapter *adapter)
+{
+	struct leapraid_int_rq *int_rq;
+	const cpumask_t *affinity_mask;
+	int cpu;
+	u32 i;
+
+	if (!adapter->adapter_attr.rq_cnt)
+		return;
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
+		int_rq = &adapter->notification_desc.int_rqs[i];
+		affinity_mask = pci_irq_get_affinity(adapter->pdev,
+						     int_rq->rq.msix_idx);
+		if (!affinity_mask) {
+			dev_warn(&adapter->pdev->dev,
+				 "%s: IRQ affinity NULL, msix_idx=%d\n",
+				 __func__, int_rq->rq.msix_idx);
+			goto out_apply_irq_affinity;
+		}
+
+		for_each_cpu_and(cpu, affinity_mask, cpu_online_mask) {
+			if (cpu >= adapter->notification_desc.msix_cpu_map_sz)
+				continue;
+
+			adapter->notification_desc.msix_cpu_map[cpu] =
+				int_rq->rq.msix_idx;
+		}
+	}
+	return;
+out_apply_irq_affinity:
+	leapraid_cpus_on_irq(adapter);
+}
+
+static int leapraid_alloc_msix_cpu_map(struct leapraid_adapter *adapter)
+{
+	adapter->notification_desc.msix_cpu_map_sz = nr_cpu_ids;
+	adapter->notification_desc.msix_cpu_map =
+		kzalloc(adapter->notification_desc.msix_cpu_map_sz,
+			GFP_KERNEL);
+	if (!adapter->notification_desc.msix_cpu_map)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void leapraid_configure_reply_queue_affinity(
+		struct leapraid_adapter *adapter)
+{
+	if (!adapter || !adapter->notification_desc.msix_enable)
+		return;
+
+	leapraid_map_msix_to_cpu(adapter);
+}
+
+static void leapraid_free_irq(struct leapraid_adapter *adapter)
+{
+	struct leapraid_int_rq *int_rq;
+	unsigned int i;
+	int irq;
+
+	for (i = 0; adapter->notification_desc.int_rqs &&
+	     i < adapter->notification_desc.int_rqs_allocated; i++) {
+		int_rq = &adapter->notification_desc.int_rqs[i];
+		if (!int_rq)
+			continue;
+
+		irq = pci_irq_vector(adapter->pdev, int_rq->rq.msix_idx);
+		irq_set_affinity_hint(irq, NULL);
+		free_irq(irq, &int_rq->rq);
+	}
+	adapter->notification_desc.int_rqs_allocated = 0;
+
+	if (adapter->notification_desc.irq_vectors_allocated) {
+		pci_free_irq_vectors(adapter->pdev);
+		adapter->notification_desc.irq_vectors_allocated = 0;
+	}
+
+	adapter->notification_desc.msix_enable = 0;
+
+	kfree(adapter->notification_desc.blk_mq_poll_rqs);
+	adapter->notification_desc.blk_mq_poll_rqs = NULL;
+
+	kfree(adapter->notification_desc.int_rqs);
+	adapter->notification_desc.int_rqs = NULL;
+
+	kfree(adapter->notification_desc.msix_cpu_map);
+	adapter->notification_desc.msix_cpu_map = NULL;
+	adapter->notification_desc.msix_cpu_map_sz = 0;
+	adapter->notification_desc.iopoll_qdex = 0;
+	adapter->notification_desc.iopoll_qcnt = 0;
+}
+
+static int leapraid_setup_irqs(struct leapraid_adapter *adapter)
+{
+	int irq_mode = adapter->notification_desc.irq_mode;
+	unsigned int i;
+	int rc = 0;
+
+	if (irq_mode == LEAPRAID_INTERRUPT_MODE_MSIX) {
+		rc = pci_alloc_irq_vectors_affinity(
+			adapter->pdev,
+			adapter->notification_desc.iopoll_qdex,
+			adapter->notification_desc.iopoll_qdex,
+			PCI_IRQ_MSIX | PCI_IRQ_AFFINITY, NULL);
+		if (rc < 0) {
+			dev_err(&adapter->pdev->dev,
+				"%d MSI/MSIX vectors allocated failed!\n",
+				adapter->notification_desc.iopoll_qdex);
+			return rc;
+		}
+
+		adapter->notification_desc.irq_vectors_allocated = 1;
+	}
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
+		adapter->notification_desc.int_rqs[i].rq.adapter = adapter;
+		adapter->notification_desc.int_rqs[i].rq.msix_idx = i;
+		atomic_set(&adapter->notification_desc.int_rqs[i].rq.busy, 0);
+		if (irq_mode == LEAPRAID_INTERRUPT_MODE_MSIX)
+			snprintf(adapter->notification_desc.int_rqs[i].rq.name,
+				 LEAPRAID_NAME_LENGTH, "%s%u-MSIx%u",
+				 LEAPRAID_DRIVER_NAME,
+				 adapter->adapter_attr.id, i);
+		else if (irq_mode == LEAPRAID_INTERRUPT_MODE_MSI)
+			snprintf(adapter->notification_desc.int_rqs[i].rq.name,
+				 LEAPRAID_NAME_LENGTH, "%s%u-MSI%u",
+				 LEAPRAID_DRIVER_NAME,
+				 adapter->adapter_attr.id, i);
+
+		rc = request_irq(pci_irq_vector(adapter->pdev, i),
+				 leapraid_irq_handler,
+				 IRQF_SHARED,
+				 adapter->notification_desc.int_rqs[i].rq.name,
+				 &adapter->notification_desc.int_rqs[i].rq);
+		if (rc) {
+			dev_err(&adapter->pdev->dev,
+				"MSI/MSIx: request_irq %s failed!\n",
+				adapter->notification_desc.int_rqs[i].rq.name);
+			return rc;
+		}
+		adapter->notification_desc.int_rqs_allocated++;
+	}
+
+	return 0;
+}
+
+static int leapraid_setup_legacy_int(struct leapraid_adapter *adapter)
+{
+	int rc;
+
+	adapter->notification_desc.int_rqs[0].rq.adapter = adapter;
+	adapter->notification_desc.int_rqs[0].rq.msix_idx = 0;
+	atomic_set(&adapter->notification_desc.int_rqs[0].rq.busy, 0);
+	snprintf(adapter->notification_desc.int_rqs[0].rq.name,
+		 LEAPRAID_NAME_LENGTH, "%s%d-LegacyInt",
+		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);
+
+	rc = pci_alloc_irq_vectors_affinity(
+		adapter->pdev,
+		adapter->notification_desc.iopoll_qdex,
+		adapter->notification_desc.iopoll_qdex,
+		PCI_IRQ_INTX | PCI_IRQ_AFFINITY,
+		NULL);
+	if (rc < 0) {
+		dev_err(&adapter->pdev->dev,
+			"Legacy irq allocated failed!\n");
+		return rc;
+	}
+
+	adapter->notification_desc.irq_vectors_allocated = 1;
+	adapter->notification_desc.irq_mode = LEAPRAID_INTERRUPT_MODE_LEGACY;
+	rc = request_irq(pci_irq_vector(adapter->pdev, 0),
+			 leapraid_irq_handler,
+			 IRQF_SHARED,
+			 adapter->notification_desc.int_rqs[0].rq.name,
+			 &adapter->notification_desc.int_rqs[0].rq);
+	if (rc) {
+		irq_set_affinity_hint(pci_irq_vector(adapter->pdev, 0), NULL);
+		pci_free_irq_vectors(adapter->pdev);
+		adapter->notification_desc.irq_vectors_allocated = 0;
+		dev_err(&adapter->pdev->dev,
+			"Legacy Int: request_irq %s failed!\n",
+			adapter->notification_desc.int_rqs[0].rq.name);
+		return -EBUSY;
+	}
+	adapter->notification_desc.int_rqs_allocated = 1;
+	return rc;
+}
+
+static int leapraid_set_legacy_int(struct leapraid_adapter *adapter)
+{
+	int rc;
+
+	rc = leapraid_alloc_msix_cpu_map(adapter);
+	if (rc)
+		return rc;
+
+	adapter->adapter_attr.rq_cnt = 1;
+	adapter->notification_desc.iopoll_qdex =
+		adapter->adapter_attr.rq_cnt;
+	adapter->notification_desc.iopoll_qcnt = 0;
+	dev_info(&adapter->pdev->dev,
+		 "Legacy Intr: req queue cnt=%d intr=%d/poll=%d rep queues!\n",
+		 adapter->adapter_attr.rq_cnt,
+		 adapter->notification_desc.iopoll_qdex,
+		 adapter->notification_desc.iopoll_qcnt);
+	adapter->notification_desc.int_rqs =
+		kzalloc_objs(struct leapraid_int_rq,
+			     adapter->notification_desc.iopoll_qdex);
+	if (!adapter->notification_desc.int_rqs)
+		return -ENOMEM;
+
+	return leapraid_setup_legacy_int(adapter);
+}
+
+static int leapraid_set_msix(struct leapraid_adapter *adapter)
+{
+	int iopoll_qcnt = 0;
+	unsigned int i;
+	int rc, msix_cnt;
+
+	msix_cnt = pci_msix_vec_count(adapter->pdev);
+	if (msix_cnt <= 0 ||
+	    adapter->adapter_attr.features.max_msix_vectors == 0) {
+		dev_info(&adapter->pdev->dev, "MSIX unsupported!\n");
+		return -EOPNOTSUPP;
+	}
+
+	msix_cnt = min_t(int, msix_cnt,
+			 adapter->adapter_attr.features.max_msix_vectors);
+
+	if (reset_devices)
+		adapter->adapter_attr.rq_cnt = 1;
+	else
+		adapter->adapter_attr.rq_cnt = min_t(int,
+						     num_online_cpus(),
+						     msix_cnt);
+
+	if (max_msix_vectors > 0)
+		adapter->adapter_attr.rq_cnt = min_t(
+			int, max_msix_vectors, adapter->adapter_attr.rq_cnt);
+
+	if (adapter->adapter_attr.rq_cnt <= 1)
+		adapter->shost->host_tagset = 0;
+	if (adapter->shost->host_tagset) {
+		iopoll_qcnt = poll_queues;
+		if (iopoll_qcnt >= adapter->adapter_attr.rq_cnt)
+			iopoll_qcnt = 0;
+	}
+	if (iopoll_qcnt) {
+		adapter->notification_desc.blk_mq_poll_rqs =
+			kzalloc_objs(struct leapraid_blk_mq_poll_rq,
+				     iopoll_qcnt);
+		if (!adapter->notification_desc.blk_mq_poll_rqs)
+			return -ENOMEM;
+		adapter->adapter_attr.rq_cnt =
+			min(adapter->adapter_attr.rq_cnt + iopoll_qcnt,
+			    msix_cnt);
+	}
+
+	adapter->notification_desc.iopoll_qdex =
+		adapter->adapter_attr.rq_cnt - iopoll_qcnt;
+
+	adapter->notification_desc.iopoll_qcnt = iopoll_qcnt;
+	dev_info(&adapter->pdev->dev,
+		 "MSIx: req queue cnt=%d, intr=%d/poll=%d rep queues!\n",
+		 adapter->adapter_attr.rq_cnt,
+		 adapter->notification_desc.iopoll_qdex,
+		 adapter->notification_desc.iopoll_qcnt);
+
+	adapter->notification_desc.int_rqs =
+		kzalloc_objs(struct leapraid_int_rq,
+			     adapter->notification_desc.iopoll_qdex);
+	if (!adapter->notification_desc.int_rqs)
+		return -ENOMEM;
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
+		adapter->notification_desc.blk_mq_poll_rqs[i].rq.adapter =
+			adapter;
+		adapter->notification_desc.blk_mq_poll_rqs[i].rq.msix_idx =
+			i + adapter->notification_desc.iopoll_qdex;
+		atomic_set(
+			&adapter->notification_desc.blk_mq_poll_rqs[i].rq.busy,
+			0);
+		snprintf(adapter->notification_desc.blk_mq_poll_rqs[i].rq.name,
+			 LEAPRAID_NAME_LENGTH,
+			 "%s%u-MQ-Poll%u", LEAPRAID_DRIVER_NAME,
+			 adapter->adapter_attr.id, i);
+		atomic_set(&adapter->notification_desc.blk_mq_poll_rqs[i].busy,
+			   0);
+		atomic_set(&adapter->notification_desc.blk_mq_poll_rqs[i].pause,
+			   0);
+	}
+
+	rc = leapraid_alloc_msix_cpu_map(adapter);
+	if (rc)
+		return rc;
+
+	adapter->notification_desc.irq_mode = LEAPRAID_INTERRUPT_MODE_MSIX;
+	adapter->notification_desc.msix_enable = 1;
+	rc = leapraid_setup_irqs(adapter);
+	if (rc) {
+		leapraid_free_irq(adapter);
+		adapter->notification_desc.msix_enable = 0;
+		return rc;
+	}
+
+	return 0;
+}
+
+static int leapraid_set_msi(struct leapraid_adapter *adapter)
+{
+	int iopoll_qcnt = 0;
+	unsigned int i;
+	int rc, msi_cnt;
+
+	msi_cnt = pci_msi_vec_count(adapter->pdev);
+	if (msi_cnt <= 0 ||
+	    adapter->adapter_attr.features.max_msix_vectors == 0) {
+		dev_info(&adapter->pdev->dev, "MSI unsupported!\n");
+		return -EOPNOTSUPP;
+	}
+
+	msi_cnt = min_t(int, msi_cnt,
+			adapter->adapter_attr.features.max_msix_vectors);
+
+	if (reset_devices)
+		adapter->adapter_attr.rq_cnt = 1;
+	else
+		adapter->adapter_attr.rq_cnt = min_t(int,
+						     num_online_cpus(),
+						     msi_cnt);
+
+	if (max_msix_vectors > 0)
+		adapter->adapter_attr.rq_cnt = min_t(
+			int, max_msix_vectors, adapter->adapter_attr.rq_cnt);
+
+	if (adapter->adapter_attr.rq_cnt <= 1)
+		adapter->shost->host_tagset = 0;
+	if (adapter->shost->host_tagset) {
+		iopoll_qcnt = poll_queues;
+		if (iopoll_qcnt >= adapter->adapter_attr.rq_cnt)
+			iopoll_qcnt = 0;
+	}
+
+	if (iopoll_qcnt) {
+		adapter->notification_desc.blk_mq_poll_rqs =
+			kzalloc_objs(struct leapraid_blk_mq_poll_rq,
+				     iopoll_qcnt);
+		if (!adapter->notification_desc.blk_mq_poll_rqs)
+			return -ENOMEM;
+
+		adapter->adapter_attr.rq_cnt =
+			 min(adapter->adapter_attr.rq_cnt + iopoll_qcnt,
+			     msi_cnt);
+	}
+
+	adapter->notification_desc.iopoll_qdex =
+		adapter->adapter_attr.rq_cnt - iopoll_qcnt;
+	rc = pci_alloc_irq_vectors_affinity(
+		adapter->pdev,
+		1,
+		adapter->notification_desc.iopoll_qdex,
+		PCI_IRQ_MSI | PCI_IRQ_AFFINITY, NULL);
+	if (rc < 0) {
+		dev_err(&adapter->pdev->dev,
+			"%d MSI vectors allocated failed!\n",
+			adapter->notification_desc.iopoll_qdex);
+		leapraid_free_irq(adapter);
+		return rc;
+	}
+	adapter->notification_desc.irq_vectors_allocated = 1;
+	if (rc != adapter->notification_desc.iopoll_qdex) {
+		adapter->notification_desc.iopoll_qdex = rc;
+		adapter->adapter_attr.rq_cnt =
+			adapter->notification_desc.iopoll_qdex + iopoll_qcnt;
+	}
+	adapter->notification_desc.iopoll_qcnt = iopoll_qcnt;
+	dev_info(&adapter->pdev->dev,
+		 "MSI: req queue cnt=%d, intr=%d/poll=%d rep queues!\n",
+		 adapter->adapter_attr.rq_cnt,
+		 adapter->notification_desc.iopoll_qdex,
+		 adapter->notification_desc.iopoll_qcnt);
+
+	adapter->notification_desc.int_rqs =
+		kzalloc_objs(struct leapraid_int_rq,
+			     adapter->notification_desc.iopoll_qdex);
+	if (!adapter->notification_desc.int_rqs)
+		return -ENOMEM;
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
+		adapter->notification_desc.blk_mq_poll_rqs[i].rq.adapter =
+			adapter;
+		adapter->notification_desc.blk_mq_poll_rqs[i].rq.msix_idx =
+			i + adapter->notification_desc.iopoll_qdex;
+		atomic_set(
+			&adapter->notification_desc.blk_mq_poll_rqs[i].rq.busy,
+			0);
+		snprintf(adapter->notification_desc.blk_mq_poll_rqs[i].rq.name,
+			 LEAPRAID_NAME_LENGTH,
+			 "%s%u-MQ-Poll%u", LEAPRAID_DRIVER_NAME,
+			 adapter->adapter_attr.id, i);
+		atomic_set(
+			&adapter->notification_desc.blk_mq_poll_rqs[i].busy,
+			0);
+		atomic_set(
+			&adapter->notification_desc.blk_mq_poll_rqs[i].pause,
+			0);
+	}
+
+	rc = leapraid_alloc_msix_cpu_map(adapter);
+	if (rc)
+		return rc;
+
+	adapter->notification_desc.irq_mode = LEAPRAID_INTERRUPT_MODE_MSI;
+	adapter->notification_desc.msix_enable = 1;
+	rc = leapraid_setup_irqs(adapter);
+	if (rc) {
+		leapraid_free_irq(adapter);
+		adapter->notification_desc.msix_enable = 0;
+		return rc;
+	}
+
+	return 0;
+}
+
+static int leapraid_set_notification_auto(struct leapraid_adapter *adapter)
+{
+	int rc;
+
+	rc = leapraid_set_msix(adapter);
+	if (!rc)
+		return 0;
+
+	leapraid_free_irq(adapter);
+	dev_info(&adapter->pdev->dev,
+		 "MSI-X setup failed (%d), trying MSI\n", rc);
+
+	rc = leapraid_set_msi(adapter);
+	if (!rc)
+		return 0;
+
+	leapraid_free_irq(adapter);
+	dev_info(&adapter->pdev->dev,
+		 "MSI setup failed (%d), back to legacy INTx\n", rc);
+
+	rc = leapraid_set_legacy_int(adapter);
+	if (rc)
+		dev_err(&adapter->pdev->dev,
+			"%s: Enable legacy irq failed!\n", __func__);
+
+	return rc;
+}
+
+int leapraid_set_pcie_and_notification(struct leapraid_adapter *adapter)
+{
+	int rc;
+
+	rc = leapraid_enable_pcie(adapter);
+	if (rc)
+		goto out_fail;
+
+	leapraid_mask_int(adapter);
+
+	rc = leapraid_make_adapter_ready(adapter, PART_RESET);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Make adapter ready failure\n");
+		goto out_fail;
+	}
+
+	rc = leapraid_get_adapter_features(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Get adapter feature failure\n");
+		goto out_fail;
+	}
+
+	rc = leapraid_set_notification_auto(adapter);
+	if (rc)
+		goto out_fail;
+
+	pci_save_state(adapter->pdev);
+
+	return 0;
+
+out_fail:
+	leapraid_free_irq(adapter);
+	leapraid_disable_pcie(adapter);
+	return rc;
+}
+
+void leapraid_disable_controller(struct leapraid_adapter *adapter)
+{
+	if (!adapter->iomem_base)
+		return;
+
+	leapraid_mask_int(adapter);
+
+	adapter->access_ctrl.shost_recovering = 1;
+	leapraid_make_adapter_ready(adapter, PART_RESET);
+	adapter->access_ctrl.shost_recovering = 0;
+	wake_up(&adapter->access_ctrl.recovery_waitq);
+
+	leapraid_free_irq(adapter);
+	leapraid_disable_pcie(adapter);
+}
+
+static int leapraid_adapter_unit_reset(struct leapraid_adapter *adapter)
+{
+	int rc = 0;
+
+	writel(LEAPRAID_FUNC_ADAPTER_UNIT_RESET << LEAPRAID_DB_FUNC_SHIFT,
+	       &adapter->iomem_base->db);
+	if (leapraid_db_wait_ack_and_clear_int(adapter))
+		rc = -EFAULT;
+
+	if (!leapraid_wait_adapter_ready(adapter)) {
+		dev_err(&adapter->pdev->dev, "unit reset failed\n");
+		return -EFAULT;
+	}
+
+	return rc;
+}
+
+static int leapraid_make_adapter_ready(struct leapraid_adapter *adapter,
+				       enum reset_type type)
+{
+	u32 db;
+	int count;
+
+	if (!leapraid_pci_active(adapter))
+		return 0;
+
+	count = 0;
+	db = leapraid_readl(&adapter->iomem_base->db);
+	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_RESET) {
+		while ((db & LEAPRAID_DB_MASK) != LEAPRAID_DB_READY) {
+			if (count++ == LEAPRAID_DB_RETRY_COUNT_MAX) {
+				dev_err(&adapter->pdev->dev,
+					"Wait adapter ready timeout\n");
+				return -EFAULT;
+			}
+			ssleep(1);
+			db = leapraid_readl(&adapter->iomem_base->db);
+			dev_info(&adapter->pdev->dev,
+				 "Wait adapter ready, count=%d, db=0x%x\n",
+				 count, db);
+		}
+	}
+	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_READY)
+		return 0;
+
+	if (db & LEAPRAID_DB_USED)
+		goto full_reset;
+
+	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_FAULT)
+		goto full_reset;
+
+	if (type == FULL_RESET)
+		goto full_reset;
+
+	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_OPERATIONAL &&
+	    !leapraid_adapter_unit_reset(adapter))
+		return 0;
+
+full_reset:
+	return leapraid_host_diag_reset(adapter);
+}
+
+static void leapraid_fw_log_exit(struct leapraid_adapter *adapter)
+{
+	if (!adapter->fw_log_desc.open_pcie_trace)
+		return;
+
+	if (adapter->fw_log_desc.fw_log_buffer) {
+		wait_event(adapter->fw_log_desc.mmap_waitq,
+			   !atomic_read(&adapter->fw_log_desc.mmap_refcnt));
+		dma_free_coherent(&adapter->pdev->dev,
+				  (LEAPRAID_SYS_LOG_BUF_SIZE +
+				   LEAPRAID_SYS_LOG_BUF_RESERVE),
+				  adapter->fw_log_desc.fw_log_buffer,
+				  adapter->fw_log_desc.fw_log_buffer_dma);
+		adapter->fw_log_desc.fw_log_buffer = NULL;
+	}
+}
+
+static int leapraid_fw_log_init(struct leapraid_adapter *adapter)
+{
+	struct leapraid_adapter_log_req adapter_log_req;
+	struct leapraid_adapter_log_rep adapter_log_rep;
+	u16 adapter_status;
+	u64 buf_addr;
+	u32 rc;
+
+	if (!adapter->fw_log_desc.open_pcie_trace)
+		return 0;
+
+	if (!adapter->fw_log_desc.fw_log_buffer) {
+		adapter->fw_log_desc.fw_log_buffer =
+			dma_alloc_coherent(
+				&adapter->pdev->dev,
+				(LEAPRAID_SYS_LOG_BUF_SIZE +
+				 LEAPRAID_SYS_LOG_BUF_RESERVE),
+				&adapter->fw_log_desc.fw_log_buffer_dma,
+				GFP_KERNEL);
+		if (!adapter->fw_log_desc.fw_log_buffer)
+			return -ENOMEM;
+	}
+
+	memset(&adapter_log_req, 0, sizeof(struct leapraid_adapter_log_req));
+	adapter_log_req.func = LEAPRAID_FUNC_LOGBUF_INIT;
+	buf_addr = adapter->fw_log_desc.fw_log_buffer_dma;
+
+	adapter_log_req.mbox.w[0] =
+		cpu_to_le32((u32)(buf_addr & 0xFFFFFFFF));
+	adapter_log_req.mbox.w[1] =
+		cpu_to_le32((u32)((buf_addr >> 32) & 0xFFFFFFFF));
+	adapter_log_req.mbox.w[2] =
+		cpu_to_le32(LEAPRAID_SYS_LOG_BUF_SIZE);
+	rc = leapraid_handshake_func(adapter,
+				     sizeof(struct leapraid_adapter_log_req),
+				     (u32 *)&adapter_log_req,
+				     sizeof(struct leapraid_adapter_log_rep),
+				     (u16 *)&adapter_log_rep);
+	if (rc != 0) {
+		dev_err(&adapter->pdev->dev, "%s: Handshake failed, rc=%d\n",
+			__func__, rc);
+		return rc;
+	}
+
+	adapter_status = le16_to_cpu(adapter_log_rep.adapter_status) &
+			LEAPRAID_ADAPTER_STATUS_MASK;
+	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
+		dev_err(&adapter->pdev->dev, "%s: failed!\n", __func__);
+		rc = -EIO;
+	}
+
+	return rc;
+}
+
+static void leapraid_free_host_memory(struct leapraid_adapter *adapter)
+{
+	unsigned int i;
+
+	if (adapter->mem_desc.task_desc) {
+		dma_free_coherent(&adapter->pdev->dev,
+				  adapter->adapter_attr.task_desc_dma_size,
+				  adapter->mem_desc.task_desc,
+				  adapter->mem_desc.task_desc_dma);
+		adapter->mem_desc.task_desc = NULL;
+	}
+
+	if (adapter->mem_desc.sense_data) {
+		dma_free_coherent(
+			&adapter->pdev->dev,
+			adapter->adapter_attr.io_qd * SCSI_SENSE_BUFFERSIZE,
+			adapter->mem_desc.sense_data,
+			adapter->mem_desc.sense_data_dma);
+		adapter->mem_desc.sense_data = NULL;
+	}
+
+	if (adapter->mem_desc.rep_msg) {
+		dma_free_coherent(
+			&adapter->pdev->dev,
+			adapter->adapter_attr.rep_msg_qd * LEAPRAID_REPLY_SIZE,
+			adapter->mem_desc.rep_msg,
+			adapter->mem_desc.rep_msg_dma);
+		adapter->mem_desc.rep_msg = NULL;
+	}
+
+	if (adapter->mem_desc.rep_msg_addr) {
+		dma_free_coherent(&adapter->pdev->dev,
+				  adapter->adapter_attr.rep_msg_qd *
+				  LEAPRAID_REP_MSG_ADDR_SIZE,
+				  adapter->mem_desc.rep_msg_addr,
+				  adapter->mem_desc.rep_msg_addr_dma);
+		adapter->mem_desc.rep_msg_addr = NULL;
+	}
+
+	if (adapter->mem_desc.rep_desc_seg_maint) {
+		for (i = 0; i < adapter->adapter_attr.rep_desc_q_seg_cnt;
+		     i++) {
+			if (adapter->mem_desc.rep_desc_seg_maint[i]
+			    .rep_desc_seg) {
+				dma_free_coherent(
+					&adapter->pdev->dev,
+					(adapter->adapter_attr.rep_desc_qd *
+					 LEAPRAID_REP_DESC_ENTRY_SIZE) *
+					LEAPRAID_REP_DESC_CHUNK_SIZE,
+					adapter->mem_desc.rep_desc_seg_maint[i]
+						.rep_desc_seg,
+					adapter->mem_desc.rep_desc_seg_maint[i]
+						.rep_desc_seg_dma);
+				adapter->mem_desc.rep_desc_seg_maint[i]
+					.rep_desc_seg = NULL;
+			}
+		}
+
+		if (adapter->mem_desc.rep_desc_q_arr) {
+			dma_free_coherent(
+				&adapter->pdev->dev,
+				adapter->adapter_attr.rq_cnt *
+				LEAPRAID_REP_RQ_CNT_SIZE,
+				adapter->mem_desc.rep_desc_q_arr,
+				adapter->mem_desc.rep_desc_q_arr_dma);
+			adapter->mem_desc.rep_desc_q_arr = NULL;
+		}
+
+		for (i = 0; i < adapter->adapter_attr.rep_desc_q_seg_cnt; i++) {
+			struct leapraid_mem_desc *mem_desc =
+				 &adapter->mem_desc;
+			kfree(adapter->mem_desc.rep_desc_seg_maint[i]
+				.rep_desc_maint);
+			mem_desc->rep_desc_seg_maint[i].rep_desc_maint = NULL;
+		}
+		kfree(adapter->mem_desc.rep_desc_seg_maint);
+		adapter->mem_desc.rep_desc_seg_maint = NULL;
+	}
+
+	kfree(adapter->mem_desc.taskid_to_uniq_tag);
+	adapter->mem_desc.taskid_to_uniq_tag = NULL;
+
+	dma_pool_destroy(adapter->mem_desc.sg_chain_pool);
+	adapter->mem_desc.sg_chain_pool = NULL;
+}
+
+static inline bool leapraid_is_in_same_4g_seg(dma_addr_t start, u32 size)
+{
+	return upper_32_bits(start) == upper_32_bits(start + size - 1);
+}
+
+int leapraid_internal_init_cmd_priv(struct leapraid_adapter *adapter,
+				    struct leapraid_io_req_tracker *io_tracker)
+{
+	io_tracker->chain =
+		dma_pool_alloc(adapter->mem_desc.sg_chain_pool,
+			       GFP_KERNEL,
+			       &io_tracker->chain_dma);
+
+	if (!io_tracker->chain)
+		return -ENOMEM;
+
+	return 0;
+}
+
+void leapraid_internal_exit_cmd_priv(struct leapraid_adapter *adapter,
+				     struct leapraid_io_req_tracker *io_tracker)
+{
+	if (io_tracker && io_tracker->chain)
+		dma_pool_free(adapter->mem_desc.sg_chain_pool,
+			      io_tracker->chain,
+			      io_tracker->chain_dma);
+}
+
+static int leapraid_request_host_memory(struct leapraid_adapter *adapter)
+{
+	struct leapraid_adapter_features *facts =
+			&adapter->adapter_attr.features;
+	u16 rep_desc_q_cnt_allocated;
+	unsigned int i, j;
+	int rc;
+
+	/* Scatter-gather table size. */
+	adapter->shost->sg_tablesize = LEAPRAID_SG_DEPTH;
+	if (reset_devices)
+		adapter->shost->sg_tablesize =
+				LEAPRAID_KDUMP_MIN_PHYS_SEGMENTS;
+	/* High-priority commands queue depth. */
+	adapter->dynamic_task_desc.hp_cmd_qd = LEAPRAID_FIXED_HP_CMDS;
+	/* Internal commands queue depth. */
+	adapter->dynamic_task_desc.inter_cmd_qd = LEAPRAID_FIXED_INTER_CMDS;
+	/* Adapter commands total queue depth. */
+	if (reset_devices)
+		adapter->adapter_attr.adapter_total_qd =
+			LEAPRAID_DEFAULT_CMD_QD_OFFSET +
+			adapter->dynamic_task_desc.inter_cmd_qd +
+			adapter->dynamic_task_desc.hp_cmd_qd;
+	else
+		adapter->adapter_attr.adapter_total_qd = facts->req_slot +
+				adapter->dynamic_task_desc.hp_cmd_qd;
+	/* Reply message queue depth. */
+	adapter->adapter_attr.rep_msg_qd =
+		adapter->adapter_attr.adapter_total_qd +
+		LEAPRAID_DEFAULT_CMD_QD_OFFSET;
+	/* Reply descriptor queue depth. */
+	adapter->adapter_attr.rep_desc_qd =
+		round_up(adapter->adapter_attr.adapter_total_qd +
+			 adapter->adapter_attr.rep_msg_qd +
+			 LEAPRAID_TASKID_OFFSET_CTRL_CMD,
+			 LEAPRAID_REPLY_QD_ALIGNMENT);
+	/* SCSI command I/O depth. */
+	adapter->adapter_attr.io_qd =
+		adapter->adapter_attr.adapter_total_qd -
+		adapter->dynamic_task_desc.hp_cmd_qd -
+		adapter->dynamic_task_desc.inter_cmd_qd;
+	/* SCSI host can queue. */
+	adapter->shost->can_queue = adapter->adapter_attr.io_qd -
+		LEAPRAID_TASKID_OFFSET_CTRL_CMD;
+	adapter->driver_cmds.ctl_cmd.taskid = adapter->shost->can_queue +
+		LEAPRAID_TASKID_OFFSET_CTRL_CMD;
+
+	/* Allocate task descriptor. */
+try_again:
+	adapter->adapter_attr.task_desc_dma_size =
+		(adapter->adapter_attr.adapter_total_qd +
+			LEAPRAID_TASKID_OFFSET_CTRL_CMD) *
+		LEAPRAID_REQUEST_SIZE;
+	adapter->mem_desc.task_desc =
+		dma_alloc_coherent(&adapter->pdev->dev,
+				   adapter->adapter_attr.task_desc_dma_size,
+				   &adapter->mem_desc.task_desc_dma,
+				   GFP_KERNEL);
+	if (!adapter->mem_desc.task_desc)
+		return -ENOMEM;
+
+	/* Allocate chain message pool. */
+	adapter->mem_desc.sg_chain_pool_size =
+		LEAPRAID_DEFAULT_CHAINS_PER_IO * LEAPRAID_CHAIN_SEG_SIZE;
+	adapter->mem_desc.sg_chain_pool =
+		dma_pool_create("leapraid chain pool",
+				&adapter->pdev->dev,
+				adapter->mem_desc.sg_chain_pool_size,
+				LEAPRAID_DMA_ALIGN, 0);
+	if (!adapter->mem_desc.sg_chain_pool)
+		return -ENOMEM;
+
+	/* Allocate I/O tracker to SCSI I/O. */
+	adapter->mem_desc.taskid_to_uniq_tag =
+		kcalloc(adapter->shost->can_queue, sizeof(u16), GFP_KERNEL);
+	if (!adapter->mem_desc.taskid_to_uniq_tag)
+		return -ENOMEM;
+
+	adapter->dynamic_task_desc.hp_taskid =
+		adapter->adapter_attr.io_qd +
+		LEAPRAID_HP_TASKID_OFFSET_CTL_CMD;
+	/* Allocate static high-priority task ID. */
+	adapter->driver_cmds.ctl_cmd.hp_taskid =
+		adapter->dynamic_task_desc.hp_taskid;
+	adapter->driver_cmds.tm_cmd.hp_taskid =
+		adapter->dynamic_task_desc.hp_taskid +
+		LEAPRAID_HP_TASKID_OFFSET_TM_CMD;
+
+	adapter->dynamic_task_desc.inter_taskid =
+		adapter->dynamic_task_desc.hp_taskid +
+		adapter->dynamic_task_desc.hp_cmd_qd;
+	adapter->driver_cmds.scan_dev_cmd.inter_taskid =
+		adapter->dynamic_task_desc.inter_taskid;
+	adapter->driver_cmds.cfg_op_cmd.inter_taskid =
+		adapter->dynamic_task_desc.inter_taskid +
+		LEAPRAID_TASKID_OFFSET_CFG_OP_CMD;
+	adapter->driver_cmds.transport_cmd.inter_taskid =
+		adapter->dynamic_task_desc.inter_taskid +
+		LEAPRAID_TASKID_OFFSET_TRANSPORT_CMD;
+	adapter->driver_cmds.enc_cmd.inter_taskid =
+		adapter->dynamic_task_desc.inter_taskid +
+		LEAPRAID_TASKID_OFFSET_ENC_CMD;
+	adapter->driver_cmds.notify_event_cmd.inter_taskid =
+		adapter->dynamic_task_desc.inter_taskid +
+		LEAPRAID_TASKID_OFFSET_NOTIFY_EVENT_CMD;
+	dev_info(&adapter->pdev->dev, "queue depth:\n");
+	dev_info(&adapter->pdev->dev, "	host->can_queue: %d\n",
+		 adapter->shost->can_queue);
+	dev_info(&adapter->pdev->dev, "	io_qd: %d\n",
+		 adapter->adapter_attr.io_qd);
+	dev_info(&adapter->pdev->dev, "	hpr_cmd_qd: %d\n",
+		 adapter->dynamic_task_desc.hp_cmd_qd);
+	dev_info(&adapter->pdev->dev, "	inter_cmd_qd: %d\n",
+		 adapter->dynamic_task_desc.inter_cmd_qd);
+	dev_info(&adapter->pdev->dev, "	adapter_total_qd: %d\n",
+		 adapter->adapter_attr.adapter_total_qd);
+
+	dev_info(&adapter->pdev->dev, "taskid range:\n");
+	dev_info(&adapter->pdev->dev,
+		 "	adapter->dynamic_task_desc.hp_taskid: %d\n",
+		 adapter->dynamic_task_desc.hp_taskid);
+	dev_info(&adapter->pdev->dev,
+		 "	adapter->dynamic_task_desc.inter_taskid: %d\n",
+		 adapter->dynamic_task_desc.inter_taskid);
+
+	/*
+	 * Allocate sense data DMA buffer.
+	 * Constraint: Must reside within the same 4GB segment
+	 * (driver-maintained).
+	 */
+	adapter->mem_desc.sense_data =
+		dma_alloc_coherent(
+			&adapter->pdev->dev,
+			adapter->adapter_attr.io_qd * SCSI_SENSE_BUFFERSIZE,
+			&adapter->mem_desc.sense_data_dma,
+			GFP_KERNEL);
+	if (!adapter->mem_desc.sense_data)
+		return -ENOMEM;
+
+	if (!leapraid_is_in_same_4g_seg(adapter->mem_desc.sense_data_dma,
+					adapter->adapter_attr.io_qd *
+					SCSI_SENSE_BUFFERSIZE)) {
+		dev_warn(&adapter->pdev->dev,
+			 "Try 32bit DMA: Sense not in same 4G\n");
+		rc = -EAGAIN;
+		goto out_fail;
+	}
+
+	/*
+	 * Allocate reply frame buffer.
+	 * Constraint: Must reside in the same 4GB segment as the sense DMA.
+	 */
+	adapter->mem_desc.rep_msg =
+		dma_alloc_coherent(&adapter->pdev->dev,
+				   adapter->adapter_attr.rep_msg_qd *
+				   LEAPRAID_REPLY_SIZE,
+				   &adapter->mem_desc.rep_msg_dma,
+				   GFP_KERNEL);
+	if (!adapter->mem_desc.rep_msg) {
+		rc = -ENOMEM;
+		goto out_fail;
+	}
+
+	if (!leapraid_is_in_same_4g_seg(adapter->mem_desc.rep_msg_dma,
+					adapter->adapter_attr.rep_msg_qd *
+					LEAPRAID_REPLY_SIZE)) {
+		dev_warn(&adapter->pdev->dev,
+			 "Use 32 bit DMA due to rep msg is not in same 4g!\n");
+		rc = -EAGAIN;
+		goto out_fail;
+	}
+
+	/* Address of reply frame. */
+	adapter->mem_desc.rep_msg_addr =
+		dma_alloc_coherent(&adapter->pdev->dev,
+				   adapter->adapter_attr.rep_msg_qd *
+				   LEAPRAID_REP_MSG_ADDR_SIZE,
+				   &adapter->mem_desc.rep_msg_addr_dma,
+				   GFP_KERNEL);
+	if (!adapter->mem_desc.rep_msg_addr)
+		return -ENOMEM;
+	adapter->adapter_attr.rep_desc_q_seg_cnt =
+		DIV_ROUND_UP(adapter->adapter_attr.rq_cnt,
+			     LEAPRAID_REP_DESC_CHUNK_SIZE);
+	adapter->mem_desc.rep_desc_seg_maint =
+		kzalloc_objs(struct leapraid_rep_desc_seg_maint,
+			     adapter->adapter_attr.rep_desc_q_seg_cnt);
+	if (!adapter->mem_desc.rep_desc_seg_maint)
+		return -ENOMEM;
+
+	rep_desc_q_cnt_allocated = 0;
+	for (i = 0; i < adapter->adapter_attr.rep_desc_q_seg_cnt; i++) {
+		adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_maint =
+			kzalloc_objs(struct leapraid_rep_desc_maint,
+				     LEAPRAID_REP_DESC_CHUNK_SIZE);
+		if (!adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_maint)
+			return -ENOMEM;
+
+		adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_seg =
+			dma_alloc_coherent(
+				&adapter->pdev->dev,
+				(adapter->adapter_attr.rep_desc_qd *
+				LEAPRAID_REP_DESC_ENTRY_SIZE) *
+				LEAPRAID_REP_DESC_CHUNK_SIZE,
+				&adapter->mem_desc.rep_desc_seg_maint[i]
+					.rep_desc_seg_dma,
+				GFP_KERNEL);
+		if (!adapter->mem_desc.rep_desc_seg_maint[i].rep_desc_seg)
+			return -ENOMEM;
+
+		for (j = 0; j < LEAPRAID_REP_DESC_CHUNK_SIZE; j++) {
+			if (rep_desc_q_cnt_allocated >=
+			    adapter->adapter_attr.rq_cnt)
+				break;
+			adapter->mem_desc
+				.rep_desc_seg_maint[i]
+				.rep_desc_maint[j]
+				.rep_desc =
+					(void *)((u8 *)(
+					adapter->mem_desc
+						.rep_desc_seg_maint[i]
+						.rep_desc_seg) +
+					j *
+					(adapter->adapter_attr.rep_desc_qd *
+						LEAPRAID_REP_DESC_ENTRY_SIZE));
+			adapter->mem_desc
+				.rep_desc_seg_maint[i]
+				.rep_desc_maint[j]
+				.rep_desc_dma =
+					adapter->mem_desc
+						.rep_desc_seg_maint[i]
+						.rep_desc_seg_dma +
+					j *
+					(adapter->adapter_attr.rep_desc_qd *
+						LEAPRAID_REP_DESC_ENTRY_SIZE);
+			rep_desc_q_cnt_allocated++;
+		}
+	}
+
+	if (!reset_devices) {
+		adapter->mem_desc.rep_desc_q_arr =
+			dma_alloc_coherent(
+				&adapter->pdev->dev,
+				adapter->adapter_attr.rq_cnt *
+				LEAPRAID_REP_RQ_CNT_SIZE,
+				&adapter->mem_desc.rep_desc_q_arr_dma,
+				GFP_KERNEL);
+		if (!adapter->mem_desc.rep_desc_q_arr)
+			return -ENOMEM;
+	}
+
+	return 0;
+out_fail:
+	if (rc == -EAGAIN) {
+		leapraid_free_host_memory(adapter);
+		adapter->adapter_attr.use_32_dma_mask = 1;
+		rc = dma_set_mask_and_coherent(&adapter->pdev->dev,
+					       DMA_BIT_MASK(DMA_32_BITS));
+		if (rc) {
+			dev_err(&adapter->pdev->dev,
+				"Failed to set 32 DMA mask\n");
+			return rc;
+		}
+		goto try_again;
+	}
+	return rc;
+}
+
+static int leapraid_alloc_dev_topo_bitmaps(struct leapraid_adapter *adapter)
+{
+	u16 pd_hdls_sz;
+
+	pd_hdls_sz =
+		BITS_TO_LONGS(
+			adapter->adapter_attr.features.max_dev_handle + 1) *
+		sizeof(unsigned long);
+
+	adapter->dev_topo.pd_hdls_sz = pd_hdls_sz;
+	adapter->dev_topo.pd_hdls =
+		kzalloc(adapter->dev_topo.pd_hdls_sz, GFP_KERNEL);
+	if (!adapter->dev_topo.pd_hdls)
+		return -ENOMEM;
+
+	adapter->dev_topo.blocking_hdls =
+		kzalloc(adapter->dev_topo.pd_hdls_sz, GFP_KERNEL);
+	if (!adapter->dev_topo.blocking_hdls)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static void leapraid_free_dev_topo_bitmaps(struct leapraid_adapter *adapter)
+{
+	kfree(adapter->dev_topo.pd_hdls);
+	kfree(adapter->dev_topo.blocking_hdls);
+}
+
+static int leapraid_init_driver_cmds(struct leapraid_adapter *adapter)
+{
+	INIT_LIST_HEAD(&adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.scan_dev_cmd.cb_idx = LEAPRAID_SCAN_DEV_CB_IDX;
+	list_add_tail(&adapter->driver_cmds.scan_dev_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.cfg_op_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.cfg_op_cmd.cb_idx = LEAPRAID_CONFIG_CB_IDX;
+	mutex_init(&adapter->driver_cmds.cfg_op_cmd.mutex);
+	list_add_tail(&adapter->driver_cmds.cfg_op_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.transport_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.transport_cmd.cb_idx = LEAPRAID_TRANSPORT_CB_IDX;
+	mutex_init(&adapter->driver_cmds.transport_cmd.mutex);
+	list_add_tail(&adapter->driver_cmds.transport_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.enc_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.enc_cmd.cb_idx = LEAPRAID_ENC_CB_IDX;
+	mutex_init(&adapter->driver_cmds.enc_cmd.mutex);
+	list_add_tail(&adapter->driver_cmds.enc_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.notify_event_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.notify_event_cmd.cb_idx =
+		LEAPRAID_NOTIFY_EVENT_CB_IDX;
+	mutex_init(&adapter->driver_cmds.notify_event_cmd.mutex);
+	list_add_tail(&adapter->driver_cmds.notify_event_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.ctl_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.ctl_cmd.cb_idx = LEAPRAID_CTL_CB_IDX;
+	mutex_init(&adapter->driver_cmds.ctl_cmd.mutex);
+	list_add_tail(&adapter->driver_cmds.ctl_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	adapter->driver_cmds.tm_cmd.status = LEAPRAID_CMD_NOT_USED;
+	adapter->driver_cmds.tm_cmd.cb_idx = LEAPRAID_TM_CB_IDX;
+	mutex_init(&adapter->driver_cmds.tm_cmd.mutex);
+	list_add_tail(&adapter->driver_cmds.tm_cmd.list,
+		      &adapter->driver_cmds.special_cmd_list);
+
+	return 0;
+}
+
+static void leapraid_unmask_evts(struct leapraid_adapter *adapter, u16 evt)
+{
+	if (evt >= LEAPRAID_MAX_EVENT_NUM)
+		return;
+
+	clear_bit(evt, (unsigned long *)adapter->fw_evt_s.leapraid_evt_masks);
+}
+
+static void leapraid_init_event_mask(struct leapraid_adapter *adapter)
+{
+	int i;
+
+	for (i = 0; i < LEAPRAID_EVT_MASK_COUNT; i++)
+		adapter->fw_evt_s.leapraid_evt_masks[i] =
+			LEAPRAID_INVALID_INITIAL_VALUE;
+
+	leapraid_unmask_evts(adapter, LEAPRAID_EVT_SAS_TOPO_CHANGE_LIST);
+	leapraid_unmask_evts(adapter, LEAPRAID_EVT_SAS_ENCL_DEV_STATUS_CHANGE);
+	leapraid_unmask_evts(adapter, LEAPRAID_EVT_SAS_DEV_STATUS_CHANGE);
+	leapraid_unmask_evts(adapter, LEAPRAID_EVT_IR_CHANGE);
+}
+
+static void leapraid_prepare_adapter_init_req(
+		struct leapraid_adapter *adapter,
+		struct leapraid_adapter_init_req *init_req)
+{
+	ktime_t cur_time;
+	int i, chunk;
+	u32 reply_post_free_ary_sz;
+
+	memset(init_req, 0, sizeof(struct leapraid_adapter_init_req));
+	init_req->func = LEAPRAID_FUNC_ADAPTER_INIT;
+	init_req->who_init = LEAPRAID_WHOINIT_LINUX_DRIVER;
+	init_req->msg_ver = cpu_to_le16(LEAPRAID_MSG_VERSION);
+	init_req->header_ver = cpu_to_le16(LEAPRAID_HEADER_VERSION);
+	init_req->driver_ver =
+		cpu_to_le32((LEAPRAID_MAJOR_VERSION <<
+				LEAPRAID_VER_MAJOR_SHIFT) |
+			(LEAPRAID_MINOR_VERSION << LEAPRAID_VER_MINOR_SHIFT) |
+			(LEAPRAID_BUILD_VERSION << LEAPRAID_VER_BUILD_SHIFT) |
+			LEAPRAID_RELEASE_VERSION);
+	if (adapter->notification_desc.msix_enable)
+		init_req->host_msix_vectors = adapter->adapter_attr.rq_cnt;
+
+	init_req->req_frame_size =
+		cpu_to_le16(LEAPRAID_REQUEST_SIZE / LEAPRAID_DWORDS_BYTE_SIZE);
+	init_req->rep_desc_qd =
+		cpu_to_le16(adapter->adapter_attr.rep_desc_qd);
+	init_req->rep_msg_qd =
+		cpu_to_le16(adapter->adapter_attr.rep_msg_qd);
+	init_req->sense_buffer_add_high =
+		cpu_to_le32((u64)adapter->mem_desc.sense_data_dma >> 32);
+	init_req->rep_msg_dma_high =
+		cpu_to_le32((u64)adapter->mem_desc.rep_msg_dma >> 32);
+	init_req->task_desc_base_addr =
+		cpu_to_le64((u64)adapter->mem_desc.task_desc_dma);
+	init_req->rep_msg_addr_dma =
+		cpu_to_le64((u64)adapter->mem_desc.rep_msg_addr_dma);
+	if (!reset_devices) {
+		reply_post_free_ary_sz =
+			adapter->adapter_attr.rq_cnt * LEAPRAID_REP_RQ_CNT_SIZE;
+		memset(adapter->mem_desc.rep_desc_q_arr, 0,
+		       reply_post_free_ary_sz);
+
+		chunk = LEAPRAID_REP_DESC_CHUNK_SIZE;
+		for (i = 0; i < adapter->adapter_attr.rq_cnt; i++)
+			adapter->mem_desc.rep_desc_q_arr[i].rep_desc_base_addr =
+				cpu_to_le64 ((u64)adapter->mem_desc
+					     .rep_desc_seg_maint[i / chunk]
+					     .rep_desc_maint[i % chunk]
+					     .rep_desc_dma);
+
+		init_req->msg_flg =
+			LEAPRAID_ADAPTER_INIT_MSGFLG_RDPQ_ARRAY_MODE;
+		init_req->rep_desc_q_arr_addr =
+			cpu_to_le64((u64)adapter->mem_desc.rep_desc_q_arr_dma);
+	} else {
+		init_req->rep_desc_q_arr_addr =
+			cpu_to_le64((u64)adapter->mem_desc
+					.rep_desc_seg_maint[0]
+					.rep_desc_maint[0]
+					.rep_desc_dma);
+	}
+	cur_time = ktime_get_real();
+	init_req->time_stamp = cpu_to_le64(ktime_to_ms(cur_time));
+}
+
+static int leapraid_send_adapter_init(struct leapraid_adapter *adapter)
+{
+	struct leapraid_adapter_init_req init_req;
+	struct leapraid_adapter_init_rep init_rep;
+	u16 adapter_status;
+	int rc;
+
+	leapraid_prepare_adapter_init_req(adapter, &init_req);
+
+	rc = leapraid_handshake_func(adapter,
+				     sizeof(struct leapraid_adapter_init_req),
+				     (u32 *)&init_req,
+				     sizeof(struct leapraid_adapter_init_rep),
+				     (u16 *)&init_rep);
+	if (rc != 0) {
+		dev_err(&adapter->pdev->dev, "%s: Handshake failed, rc=%d\n",
+			__func__, rc);
+		return rc;
+	}
+
+	adapter_status =
+		le16_to_cpu(init_rep.adapter_status) &
+		LEAPRAID_ADAPTER_STATUS_MASK;
+	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
+		dev_err(&adapter->pdev->dev, "%s: failed\n", __func__);
+		rc = -EIO;
+	}
+
+	return rc;
+}
+
+static int leapraid_cfg_pages(struct leapraid_adapter *adapter)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	union {
+		struct leapraid_manufacturing_p0 manufacturing_page0;
+		struct leapraid_bios_page3 bios_page3;
+		struct leapraid_bios_page2 bios_page2;
+	} cfg_page;
+	int rc;
+
+	rc = leapraid_op_config_page(adapter, &cfg_page.bios_page3, cfgp1,
+				     cfgp2, GET_BIOS_PG3);
+	if (rc)
+		return rc;
+
+	adapter->adapter_attr.bios_version =
+		le32_to_cpu(cfg_page.bios_page3.bios_version);
+
+	rc = leapraid_op_config_page(adapter, &cfg_page.bios_page2, cfgp1,
+				     cfgp2, GET_BIOS_PG2);
+	if (rc)
+		return rc;
+
+	adapter->boot_devs.requested_boot_dev.form =
+		cfg_page.bios_page2.requested_boot_dev_form;
+	memcpy(adapter->boot_devs.requested_boot_dev.pg_dev,
+	       &cfg_page.bios_page2.requested_boot_dev,
+	       LEAPRAID_BOOT_DEV_SIZE);
+	adapter->boot_devs.requested_alt_boot_dev.form =
+		cfg_page.bios_page2.requested_alt_boot_dev_form;
+	memcpy(adapter->boot_devs.requested_alt_boot_dev.pg_dev,
+	       &cfg_page.bios_page2.requested_alt_boot_dev,
+	       LEAPRAID_BOOT_DEV_SIZE);
+	adapter->boot_devs.current_boot_dev.form =
+		cfg_page.bios_page2.current_boot_dev_form;
+	memcpy(adapter->boot_devs.current_boot_dev.pg_dev,
+	       &cfg_page.bios_page2.current_boot_dev,
+	       LEAPRAID_BOOT_DEV_SIZE);
+
+	rc = leapraid_op_config_page(adapter, &cfg_page.manufacturing_page0,
+				     cfgp1, cfgp2, GET_MANUFACTURING_PG0);
+	if (rc)
+		return rc;
+
+	snprintf(adapter->adapter_attr.board_name,
+		 sizeof(adapter->adapter_attr.board_name),
+		 "%.*s",
+		 (int)sizeof(cfg_page.manufacturing_page0.board_name),
+		 cfg_page.manufacturing_page0.board_name);
+	return rc;
+}
+
+static int leapraid_evt_notify(struct leapraid_adapter *adapter)
+{
+	struct leapraid_evt_notify_req *evt_notify_req;
+	struct leapraid_evt_notify_rep *evt_notify_rep;
+	u16 adapter_status;
+	int rc = 0;
+	int i;
+
+	mutex_lock(&adapter->driver_cmds.notify_event_cmd.mutex);
+	adapter->driver_cmds.notify_event_cmd.status = LEAPRAID_CMD_PENDING;
+	evt_notify_req =
+		leapraid_get_task_desc(
+			adapter,
+			adapter->driver_cmds.notify_event_cmd.inter_taskid);
+	memset(evt_notify_req, 0, sizeof(struct leapraid_evt_notify_req));
+	evt_notify_req->func = LEAPRAID_FUNC_EVENT_NOTIFY;
+	for (i = 0; i < LEAPRAID_EVT_MASK_COUNT; i++)
+		evt_notify_req->evt_masks[i] =
+			cpu_to_le32(adapter->fw_evt_s.leapraid_evt_masks[i]);
+	init_completion(&adapter->driver_cmds.notify_event_cmd.done);
+	leapraid_fire_task(adapter,
+			   adapter->driver_cmds.notify_event_cmd.inter_taskid);
+	wait_for_completion_timeout(
+		&adapter->driver_cmds.notify_event_cmd.done,
+		LEAPRAID_NOTIFY_EVENT_CMD_TIMEOUT * HZ);
+
+	if (!(adapter->driver_cmds.notify_event_cmd.status &
+	      LEAPRAID_CMD_DONE)) {
+		rc = -EFAULT;
+		goto out_cleanup;
+	}
+
+	if (!(adapter->driver_cmds.notify_event_cmd.status &
+	      LEAPRAID_CMD_REPLY_VALID)) {
+		rc = -EFAULT;
+		goto out_cleanup;
+	}
+
+	evt_notify_rep = (void *)&adapter->driver_cmds.notify_event_cmd.reply;
+	adapter_status = le16_to_cpu(evt_notify_rep->adapter_status) &
+			 LEAPRAID_ADAPTER_STATUS_MASK;
+	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS)
+		rc = -EFAULT;
+
+out_cleanup:
+	adapter->driver_cmds.notify_event_cmd.status = LEAPRAID_CMD_NOT_USED;
+	mutex_unlock(&adapter->driver_cmds.notify_event_cmd.mutex);
+
+	return rc;
+}
+
+int leapraid_scan_dev(struct leapraid_adapter *adapter, bool async_scan_dev)
+{
+	struct leapraid_scan_dev_req *scan_dev_req;
+	struct leapraid_scan_dev_rep *scan_dev_rep;
+	u16 adapter_status;
+	int rc = 0;
+
+	dev_info(&adapter->pdev->dev,
+		 "Send device scan, async_scan_dev=%d!\n", async_scan_dev);
+
+	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_PENDING;
+	adapter->driver_cmds.scan_dev_cmd.async_scan_dev = async_scan_dev;
+	scan_dev_req =
+		leapraid_get_task_desc(
+			adapter,
+			adapter->driver_cmds.scan_dev_cmd.inter_taskid);
+	memset(scan_dev_req, 0, sizeof(struct leapraid_scan_dev_req));
+	scan_dev_req->func = LEAPRAID_FUNC_SCAN_DEV;
+
+	if (async_scan_dev) {
+		adapter->scan_dev_desc.first_scan_dev_fired = 1;
+		leapraid_fire_task(
+			adapter,
+			adapter->driver_cmds.scan_dev_cmd.inter_taskid);
+		return 0;
+	}
+
+	init_completion(&adapter->driver_cmds.scan_dev_cmd.done);
+	leapraid_fire_task(adapter,
+			   adapter->driver_cmds.scan_dev_cmd.inter_taskid);
+	wait_for_completion_timeout(&adapter->driver_cmds.scan_dev_cmd.done,
+				    LEAPRAID_SCAN_DEV_CMD_TIMEOUT * HZ);
+	if (!(adapter->driver_cmds.scan_dev_cmd.status & LEAPRAID_CMD_DONE)) {
+		dev_err(&adapter->pdev->dev, "Device scan timeout!\n");
+		if (adapter->driver_cmds.scan_dev_cmd.status &
+		    LEAPRAID_CMD_RESET)
+			rc = -EFAULT;
+		else
+			rc = -ETIME;
+		goto out_cleanup;
+	}
+
+	scan_dev_rep = (void *)&adapter->driver_cmds.scan_dev_cmd.reply;
+	adapter_status =
+		le16_to_cpu(scan_dev_rep->adapter_status) &
+		LEAPRAID_ADAPTER_STATUS_MASK;
+	if (adapter_status != LEAPRAID_ADAPTER_STATUS_SUCCESS) {
+		dev_err(&adapter->pdev->dev, "Device scan failure!\n");
+		rc = -EFAULT;
+		goto out_cleanup;
+	}
+
+out_cleanup:
+	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_NOT_USED;
+	return rc;
+}
+
+static void leapraid_init_task_tracker(struct leapraid_adapter *adapter)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dynamic_task_desc.task_lock, flags);
+
+	spin_unlock_irqrestore(&adapter->dynamic_task_desc.task_lock, flags);
+}
+
+static void leapraid_init_rep_msg_addr(struct leapraid_adapter *adapter)
+{
+	u32 reply_address;
+	unsigned int i;
+
+	for (i = 0, reply_address = (u32)adapter->mem_desc.rep_msg_dma;
+	     i < adapter->adapter_attr.rep_msg_qd;
+	     i++, reply_address += LEAPRAID_REPLY_SIZE)
+		adapter->mem_desc.rep_msg_addr[i] = cpu_to_le32(reply_address);
+}
+
+static void init_rep_desc(
+		struct leapraid_rq *rq,
+		int index,
+		union leapraid_rep_desc_union *reply_post_free_contig)
+{
+	struct leapraid_adapter *adapter = rq->adapter;
+	unsigned int i;
+
+	if (!reset_devices)
+		rq->rep_desc =
+			adapter->mem_desc
+				.rep_desc_seg_maint[index /
+					LEAPRAID_REP_DESC_CHUNK_SIZE]
+				.rep_desc_maint[index %
+					LEAPRAID_REP_DESC_CHUNK_SIZE]
+				.rep_desc;
+	else
+		rq->rep_desc = reply_post_free_contig;
+
+	rq->rep_post_host_idx = 0;
+	for (i = 0; i < adapter->adapter_attr.rep_desc_qd; i++)
+		rq->rep_desc[i].words = cpu_to_le64(ULLONG_MAX);
+}
+
+static void leapraid_init_rep_desc(struct leapraid_adapter *adapter)
+{
+	union leapraid_rep_desc_union *reply_post_free_contig;
+	struct leapraid_int_rq *int_rq;
+	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
+	unsigned int i;
+	int index;
+
+	index = 0;
+	reply_post_free_contig = adapter->mem_desc
+					.rep_desc_seg_maint[0]
+					.rep_desc_maint[0]
+					.rep_desc;
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
+		int_rq = &adapter->notification_desc.int_rqs[i];
+		init_rep_desc(&int_rq->rq, index, reply_post_free_contig);
+		if (!reset_devices)
+			index++;
+		else
+			reply_post_free_contig +=
+				adapter->adapter_attr.rep_desc_qd;
+	}
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
+		blk_mq_poll_rq = &adapter->notification_desc.blk_mq_poll_rqs[i];
+		init_rep_desc(&blk_mq_poll_rq->rq,
+			      index, reply_post_free_contig);
+		if (!reset_devices)
+			index++;
+		else
+			reply_post_free_contig +=
+				adapter->adapter_attr.rep_desc_qd;
+	}
+}
+
+static void leapraid_init_bar_idx_regs(struct leapraid_adapter *adapter)
+{
+	struct leapraid_int_rq *int_rq;
+	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
+	unsigned int i, j;
+
+	adapter->rep_msg_host_idx = adapter->adapter_attr.rep_msg_qd - 1;
+	writel(adapter->rep_msg_host_idx,
+	       &adapter->iomem_base->rep_msg_host_idx);
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qdex; i++) {
+		int_rq = &adapter->notification_desc.int_rqs[i];
+		for (j = 0; j < REP_POST_HOST_IDX_REG_CNT; j++)
+			writel((int_rq->rq.msix_idx & 7) <<
+				LEAPRAID_RPHI_MSIX_IDX_SHIFT,
+				&adapter->iomem_base->rep_post_reg_idx[j].idx);
+	}
+
+	for (i = 0; i < adapter->notification_desc.iopoll_qcnt; i++) {
+		blk_mq_poll_rq =
+			&adapter->notification_desc.blk_mq_poll_rqs[i];
+		for (j = 0; j < REP_POST_HOST_IDX_REG_CNT; j++)
+			writel((blk_mq_poll_rq->rq.msix_idx & 7) <<
+				LEAPRAID_RPHI_MSIX_IDX_SHIFT,
+				&adapter->iomem_base->rep_post_reg_idx[j].idx);
+	}
+}
+
+static int leapraid_make_adapter_available(struct leapraid_adapter *adapter)
+{
+	int rc;
+
+	leapraid_init_task_tracker(adapter);
+	leapraid_init_rep_msg_addr(adapter);
+
+	if (adapter->scan_dev_desc.driver_loading)
+		leapraid_configure_reply_queue_affinity(adapter);
+
+	leapraid_init_rep_desc(adapter);
+	rc = leapraid_send_adapter_init(adapter);
+	if (rc)
+		return rc;
+
+	leapraid_init_bar_idx_regs(adapter);
+	leapraid_unmask_int(adapter);
+	rc = leapraid_cfg_pages(adapter);
+	if (rc)
+		return rc;
+
+	rc = leapraid_evt_notify(adapter);
+	if (rc)
+		return rc;
+
+	if (!adapter->access_ctrl.shost_recovering) {
+		adapter->scan_dev_desc.wait_scan_dev_done = 1;
+		return 0;
+	}
+
+	return leapraid_scan_dev(adapter, false);
+}
+
+int leapraid_ctrl_init(struct leapraid_adapter *adapter)
+{
+	u32 cap;
+	int rc;
+
+	rc = leapraid_init_driver_cmds(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Init driver cmds failure\n");
+		goto out_free;
+	}
+
+	rc = leapraid_set_pcie_and_notification(adapter);
+	if (rc)
+		goto out_free;
+
+	pci_set_drvdata(adapter->pdev, adapter->shost);
+
+	pcie_capability_read_dword(adapter->pdev, PCI_EXP_DEVCAP, &cap);
+
+	if (cap & PCI_EXP_DEVCAP_EXT_TAG)
+		pcie_capability_set_word(adapter->pdev, PCI_EXP_DEVCTL,
+					 PCI_EXP_DEVCTL_EXT_TAG);
+
+	rc = leapraid_fw_log_init(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "FW log init failure\n");
+		goto out_free;
+	}
+
+	rc = leapraid_request_host_memory(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Request host memory failure\n");
+		goto out_free;
+	}
+
+	init_waitqueue_head(&adapter->reset_desc.reset_wait_queue);
+	init_waitqueue_head(&adapter->access_ctrl.shost_recover_wq);
+
+	rc = leapraid_alloc_dev_topo_bitmaps(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev, "Alloc topo bitmaps failure\n");
+		goto out_free;
+	}
+
+	leapraid_init_event_mask(adapter);
+
+	rc = leapraid_make_adapter_available(adapter);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"Make adapter available failure\n");
+		goto out_free;
+	}
+
+	leapraid_overheat_init(adapter);
+	if (!adapter->overheat_desc.fault_overheat_wq) {
+		rc = -ENOMEM;
+		goto out_free;
+	}
+
+	return 0;
+
+out_free:
+	adapter->access_ctrl.host_removing = 1;
+	leapraid_fw_log_exit(adapter);
+	leapraid_disable_controller(adapter);
+	leapraid_free_host_memory(adapter);
+	leapraid_free_dev_topo_bitmaps(adapter);
+	pci_set_drvdata(adapter->pdev, NULL);
+	return rc;
+}
+
+void leapraid_remove_ctrl(struct leapraid_adapter *adapter)
+{
+	leapraid_overheat_cleanup(adapter);
+	leapraid_check_scheduled_fault_stop(adapter);
+	leapraid_fw_log_stop(adapter);
+	leapraid_fw_log_exit(adapter);
+	leapraid_disable_controller(adapter);
+	leapraid_free_host_memory(adapter);
+	leapraid_free_dev_topo_bitmaps(adapter);
+	leapraid_free_enc_list(adapter);
+	pci_set_drvdata(adapter->pdev, NULL);
+}
diff --git a/drivers/scsi/leapraid/leapraid_func.h b/drivers/scsi/leapraid/leapraid_func.h
new file mode 100644
index 000000000..4c0b9ca72
--- /dev/null
+++ b/drivers/scsi/leapraid/leapraid_func.h
@@ -0,0 +1,1486 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/*
+ * Copyright (C) 2026 LeapIO Tech Inc.
+ *
+ * LeapRAID storage and RAID controller driver.
+ */
+#ifndef LEAPRAID_FUNC_H_INCLUDED
+#define LEAPRAID_FUNC_H_INCLUDED
+
+#include <linux/pci.h>
+#include <linux/aer.h>
+#include <linux/poll.h>
+#include <linux/atomic.h>
+#include <linux/errno.h>
+#include <linux/ktime.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <scsi/scsi.h>
+#include <scsi/scsi_eh.h>
+#include <scsi/scsicam.h>
+#include <scsi/scsi_tcq.h>
+#include <scsi/scsi_dbg.h>
+#include <scsi/scsi_host.h>
+#include <scsi/scsi_cmnd.h>
+#include <scsi/scsi_device.h>
+#include <scsi/scsi_transport_sas.h>
+
+#include "leapraid.h"
+
+/* Request and reply buffer size. */
+#define LEAPRAID_REQUEST_SIZE                   128
+#define LEAPRAID_REPLY_SIZE                     128
+#define LEAPRAID_CHAIN_SEG_SIZE                 128
+#define LEAPRAID_MAX_SGES_IN_CHAIN              7
+#define LEAPRAID_DEFAULT_CHAINS_PER_IO          19
+#define LEAPRAID_DEFAULT_DIX_CHAINS_PER_IO      \
+	(2 * LEAPRAID_DEFAULT_CHAINS_PER_IO) /* TODO DIX */
+#define LEAPRAID_IEEE_SGE64_ENTRY_SIZE          16
+#define LEAPRAID_REP_DESC_CHUNK_SIZE            16
+#define LEAPRAID_REP_DESC_ENTRY_SIZE            8
+#define LEAPRAID_REP_MSG_ADDR_SIZE              4
+#define LEAPRAID_REP_RQ_CNT_SIZE                16
+
+#define LEAPRAID_SYS_LOG_BUF_SIZE       0x200000
+#define LEAPRAID_SYS_LOG_BUF_RESERVE    0x1000
+
+/* Driver version and name. */
+#define LEAPRAID_DRIVER_NAME            "LeapRAID"
+#define LEAPRAID_NAME_LENGTH            48
+#define LEAPRAID_BOARD_NAME_LENGTH      17
+#define LEAPRAID_AUTHOR                 "LeapIO Inc."
+#define LEAPRAID_DESCRIPTION            "LeapRAID Driver"
+#define LEAPRAID_DRIVER_VERSION         "2.00.01.09"
+#define LEAPRAID_MAJOR_VERSION          2
+#define LEAPRAID_MINOR_VERSION          0
+#define LEAPRAID_BUILD_VERSION          1
+#define LEAPRAID_RELEASE_VERSION        9
+#define LEAPRAID_MSG_VERSION            0x1021
+#define LEAPRAID_HEADER_VERSION         0x0000
+
+/* Device ID. */
+#define LEAPRAID_VENDOR_ID      0xD405
+#define LEAPRAID_DEVID_HBA      0x8200
+#define LEAPRAID_SUBVENDOR_ID   0xD405
+#define LEAPRAID_SUBDEVID_HBA   0x8200
+
+#define LEAPRAID_PCI_VENDOR_ID_MASK     0xFFFF
+
+ /* RAID virtual channel ID. */
+#define RAID_CHANNEL    1
+
+/* Scatter-gather (SG) segment limits. */
+#define LEAPRAID_MAX_PHYS_SEGMENTS      SG_CHUNK_SIZE
+
+#define LEAPRAID_KDUMP_MIN_PHYS_SEGMENTS        32
+#define LEAPRAID_SG_DEPTH                       LEAPRAID_MAX_PHYS_SEGMENTS
+
+/* Firmware and config page operations. */
+#define LEAPRAID_CFG_REQ_RETRY_TIMES    2
+
+/* Hardware access helpers. */
+#define leapraid_readl(addr) readl(addr)
+#define leapraid_check_reset(status) \
+	(!((status) & LEAPRAID_CMD_RESET))
+
+/* Polling intervals. */
+#define LEAPRAID_PCIE_LOG_POLLING_INTERVAL      1
+#define LEAPRAID_FAULT_POLLING_INTERVAL         1000
+
+/* Init mask. */
+#define LEAPRAID_RESET_IRQ_MASK 0x40000000
+#define LEAPRAID_REPLY_INT_MASK 0x00000008
+#define LEAPRAID_TO_SYS_DB_MASK 0x00000001
+
+/* Queue depth. */
+#define LEAPRAID_SATA_QUEUE_DEPTH       32
+#define LEAPRAID_SAS_QUEUE_DEPTH        64
+#define LEAPRAID_RAID_QUEUE_DEPTH       64
+
+/* Target probe flag. */
+#define LEAPRAID_NO_ULD_ATTACH_FLAG 1
+
+/* SCSI device and queue limits. */
+#define LEAPRAID_MAX_SECTORS            2048
+#define LEAPRAID_MAX_CDB_LEN            32
+#define LEAPRAID_MAX_LUNS               16384
+#define LEAPRAID_CAN_QUEUE_MIN          1
+#define LEAPRAID_THIS_ID_NONE           -1
+#define LEAPRAID_CMD_PER_LUN            128
+#define LEAPRAID_MAX_SEGMENT_SIZE       0xffffffff
+
+/* SCSI sense and ASC/ASCQ and disk geometry configuration. */
+#define DESC_FORMAT_THRESHOLD                   0x72
+#define SENSE_KEY_MASK                          0x0F
+#define SCSI_SENSE_RESPONSE_CODE_MASK           0x7F
+#define ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED       0x5D
+#define LEAPRAID_LARGE_DISK_THRESHOLD           0x200000UL
+#define LEAPRAID_LARGE_DISK_HEADS               255
+#define LEAPRAID_LARGE_DISK_SECTORS             63
+#define LEAPRAID_SMALL_DISK_HEADS               64
+#define LEAPRAID_SMALL_DISK_SECTORS             32
+
+/* SMP (Serial Management Protocol). */
+#define LEAPRAID_SMP_PT_FLAG_SGL_PTR            0x80
+#define LEAPRAID_SMP_FRAME_HEADER_SIZE          4
+#define LEAPRAID_SCSI_HOST_SHIFT                16
+#define LEAPRAID_SCSI_DRIVER_SHIFT              24
+
+/* SCSI ASC/ASCQ definitions. */
+#define LEAPRAID_SCSI_ASCQ_DEFAULT              0x00
+#define LEAPRAID_SCSI_ASC_POWER_ON_RESET        0x29
+#define LEAPRAID_SCSI_ASC_INVALID_CMD_CODE      0x20
+#define LEAPRAID_SCSI_ASCQ_POWER_ON_RESET       0x07
+
+/* VPD Page 0x89 (ATA Information). */
+#define LEAPRAID_VPD_PAGE_ATA_INFO      0x89
+#define LEAPRAID_VPD_PG89_MAX_LEN       255
+#define LEAPRAID_VPD_PG89_MIN_LEN       214
+
+/* Byte index for NCQ support flag in VPD page 0x89. */
+#define LEAPRAID_VPD_PG89_NCQ_BYTE_IDX  213
+#define LEAPRAID_VPD_PG89_NCQ_BIT_SHIFT 4
+#define LEAPRAID_VPD_PG89_NCQ_BIT_MASK  0x1
+
+/* Readiness polling: max retries, sleep µs between. */
+#define LEAPRAID_ADAPTER_READY_MAX_RETRY        15000
+#define LEAPRAID_ADAPTER_READY_SLEEP_MIN_US     1000
+#define LEAPRAID_ADAPTER_READY_SLEEP_MAX_US     1100
+
+/* Doorbell wait parameters. */
+#define LEAPRAID_DB_WAIT_MAX_RETRY      20000
+#define LEAPRAID_DB_WAIT_DELAY_US       500
+
+/* Basic data size definitions. */
+#define LEAPRAID_DWORDS_BYTE_SIZE       4
+#define LEAPRAID_WORD_BYTE_SIZE         2
+
+/* SGL threshold and chain offset. */
+#define LEAPRAID_SGL_INLINE_THRESHOLD   2
+#define LEAPRAID_CHAIN_OFFSET_DWORDS    7
+
+/* MSI-X group size and mask. */
+#define LEAPRAID_MSIX_GROUP_SIZE        8
+#define LEAPRAID_MSIX_GROUP_MASK        7
+
+/* Basic constants and limits.   */
+#define LEAPRAID_BUSY_LIMIT             1
+#define LEAPRAID_INVALID_HOST_DIAG_VAL  0xFFFFFFFF
+
+/* Retry/Sleep configuration. */
+#define LEAPRAID_WRSEQ_FLUSH_KEY_VALUE  0x0
+#define LEAPRAID_WRSEQ_1ST_KEY_VALUE    0xF
+#define LEAPRAID_WRSEQ_2ND_KEY_VALUE    0x4
+#define LEAPRAID_WRSEQ_3RD_KEY_VALUE    0xB
+#define LEAPRAID_WRSEQ_4TH_KEY_VALUE    0x2
+#define LEAPRAID_WRSEQ_5TH_KEY_VALUE    0x7
+#define LEAPRAID_WRSEQ_6TH_KEY_VALUE    0xD
+#define LEAPRAID_UNLOCK_RETRY_LIMIT     20
+#define LEAPRAID_UNLOCK_SLEEP_MS        100
+#define LEAPRAID_MSLEEP_NORMAL_MS       100
+#define LEAPRAID_MSLEEP_EXTRA_LONG_MS   500
+#define LEAPRAID_IO_POLL_DELAY_US       500
+
+/* Device/Volume configuration. */
+#define LEAPRAID_MAX_VOLUMES_DEFAULT    32
+#define LEAPRAID_MAX_DEV_HANDLE_DEFAULT 2048
+#define LEAPRAID_INVALID_DEV_HANDLE     0xFFFF
+
+/* Commands queue depth. */
+#define LEAPRAID_DEFAULT_CMD_QD_OFFSET  64
+#define LEAPRAID_REPLY_QD_ALIGNMENT     16
+/* Task ID offset. */
+#define LEAPRAID_TASKID_OFFSET_CTRL_CMD         1
+#define LEAPRAID_TASKID_OFFSET_CFG_OP_CMD               1
+#define LEAPRAID_TASKID_OFFSET_TRANSPORT_CMD            2
+#define LEAPRAID_TASKID_OFFSET_ENC_CMD                  3
+#define LEAPRAID_TASKID_OFFSET_NOTIFY_EVENT_CMD         4
+
+/* Task ID offset for high-priority. */
+#define LEAPRAID_HP_TASKID_OFFSET_CTL_CMD       0
+#define LEAPRAID_HP_TASKID_OFFSET_TM_CMD        1
+
+/* Event/Boot configuration. */
+#define LEAPRAID_EVT_MASK_COUNT 4
+#define LEAPRAID_BOOT_DEV_SIZE  24
+
+/* Commands timeout. */
+#define LEAPRAID_UNIFIED_TIMEOUT               30
+#define LEAPRAID_CFG_OP_TIMEOUT                LEAPRAID_UNIFIED_TIMEOUT
+#define LEAPRAID_CTL_CMD_TIMEOUT               LEAPRAID_UNIFIED_TIMEOUT
+#define LEAPRAID_SCAN_DEV_CMD_TIMEOUT          300
+#define LEAPRAID_ENC_CMD_TIMEOUT               LEAPRAID_UNIFIED_TIMEOUT
+#define LEAPRAID_IO_CMD_TIMEOUT                LEAPRAID_UNIFIED_TIMEOUT
+#define LEAPRAID_NOTIFY_EVENT_CMD_TIMEOUT      LEAPRAID_UNIFIED_TIMEOUT
+#define LEAPRAID_TM_CMD_TIMEOUT                LEAPRAID_UNIFIED_TIMEOUT
+#define LEAPRAID_TRANSPORT_CMD_TIMEOUT         LEAPRAID_UNIFIED_TIMEOUT
+
+/* Host DMA cap. */
+#define DMA_32_BITS    32
+#define DMA_64_BITS    64
+#define LEAPRAID_DMA_ALIGN    16
+
+/* Values used to represent an invalid. */
+#define LEAPRAID_INVALID_INDEX         -1
+#define LEAPRAID_INVALID_MSIX_VECTORS  -1
+#define LEAPRAID_INVALID_INITIAL_VALUE -1
+#define LEAPRAID_OPERATION_FAILED      -1
+
+/* Version shift and mask. */
+#define LEAPRAID_VER_MAJOR_SHIFT 24
+#define LEAPRAID_VER_MINOR_SHIFT 16
+#define LEAPRAID_VER_BUILD_SHIFT 8
+#define LEAPRAID_VER_MASK        0xFF
+
+/* Interrupt type. */
+#define LEAPRAID_INTERRUPT_MODE_MSIX 0
+#define LEAPRAID_INTERRUPT_MODE_MSI  1
+#define LEAPRAID_INTERRUPT_MODE_LEGACY 2
+
+/* SMP link reset. */
+#define SMP_PHY_CONTROL_LINK_RESET    0x01
+#define SMP_PHY_CONTROL_HARD_RESET    0x02
+#define SMP_PHY_CONTROL_DISABLE       0x03
+
+/**
+ * struct leapraid_adapter_features - Adapter features/capabilities.
+ *
+ * @req_slot: Number of request slots supported by the adapter.
+ * @hp_slot: Number of high-priority slots supported by the adapter.
+ * @adapter_caps: Adapter capabilities.
+ * @fw_version: Firmware version of the adapter.
+ * @max_dev_handle: Maximum device handle supported by the adapter.
+ * @min_dev_handle: Minimum device handle supported by the adapter.
+ */
+struct leapraid_adapter_features {
+	u16 req_slot;
+	u16 hp_slot;
+	u32 adapter_caps;
+	u32 fw_version;
+	u8 max_msix_vectors;
+	u8 max_volumes;
+	u16 max_dev_handle;
+	u16 min_dev_handle;
+	u16 msg_ver;
+	u16 product_id;
+};
+
+/**
+ * struct leapraid_adapter_attr - Adapter attributes and capabilities
+ *
+ * @id: Adapter identifier.
+ * @raid_support: Indicates if RAID is supported.
+ * @bios_version: Version of the adapter BIOS.
+ * @enable_mp: Indicates if multipath (MP) support is enabled.
+ * @wideport_max_queue_depth: Maximum queue depth for wide ports.
+ * @narrowport_max_queue_depth: Maximum queue depth for narrow ports.
+ * @sata_max_queue_depth: Maximum queue depth for SATA.
+ * @raid_volume_max_queue_depth: Maximum queue depth for RAID volumes.
+ * @features: Detailed features of the adapter.
+ * @adapter_total_qd: Total queue depth available on the adapter.
+ * @io_qd: Queue depth allocated for I/O operations.
+ * @rep_msg_qd: Queue depth for reply messages.
+ * @rep_desc_qd: Queue depth for reply descriptors.
+ * @rep_desc_q_seg_cnt: Number of segments in a reply descriptor queue.
+ * @rq_cnt: Number of request queues.
+ * @task_desc_dma_size: Size of task descriptor DMA memory.
+ * @use_32_dma_mask: Indicates if 32-bit DMA mask is used.
+ * @name: Adapter name string.
+ * @board_name: Board name retrieved from manufacturing page 0.
+ */
+struct leapraid_adapter_attr {
+	u8 id;
+	u8 raid_support;
+	u32 bios_version;
+	u8 enable_mp;
+	u32 wideport_max_queue_depth;
+	u32 narrowport_max_queue_depth;
+	u32 sata_max_queue_depth;
+	u32 raid_volume_max_queue_depth;
+	struct leapraid_adapter_features features;
+	u32 adapter_total_qd;
+	u32 io_qd;
+	u32 rep_msg_qd;
+	u32 rep_desc_qd;
+	u32 rep_desc_q_seg_cnt;
+	u16 rq_cnt;
+	u32 task_desc_dma_size;
+	u8 use_32_dma_mask;
+	char name[LEAPRAID_NAME_LENGTH];
+	char board_name[LEAPRAID_BOARD_NAME_LENGTH];
+};
+
+/**
+ * struct leapraid_io_req_tracker - Track a SCSI I/O request for the adapter
+ *
+ * @taskid: Unique task ID for this I/O request.
+ * @scmd: Pointer to the associated SCSI command.
+ * @chain_list: List of chain frames associated with this request.
+ * @msix_io: MSI-X vector assigned to this I/O request.
+ * @chain: Pointer to the chain memory for this request.
+ * @chain_dma: DMA address of the chain memory.
+ */
+struct leapraid_io_req_tracker {
+	u16 taskid;
+	struct scsi_cmnd *scmd;
+	struct list_head chain_list;
+	u16 msix_io;
+	void *chain;
+	dma_addr_t chain_dma;
+};
+
+/**
+ * struct leapraid_task_tracker - Tracks a task in the adapter
+ *
+ * @taskid: Unique task ID for this tracker.
+ * @cb_idx: Callback index associated with this task.
+ * @tracker_list: Linked list node to chain this tracker in lists.
+ */
+struct leapraid_task_tracker {
+	u16 taskid;
+	u8 cb_idx;
+	struct list_head tracker_list;
+};
+
+/**
+ * struct leapraid_rep_desc_maint - Maintains reply descriptor memory
+ *
+ * @rep_desc: Pointer to the reply descriptor.
+ * @rep_desc_dma: DMA address of the reply descriptor.
+ */
+struct leapraid_rep_desc_maint {
+	union leapraid_rep_desc_union *rep_desc;
+	dma_addr_t rep_desc_dma;
+};
+
+/**
+ * struct leapraid_rep_desc_seg_maint -
+ * Maintains reply descriptor segment memory
+ *
+ * @rep_desc_seg: Pointer to the reply descriptor segment.
+ * @rep_desc_seg_dma: DMA address of the reply descriptor segment.
+ * @rep_desc_maint: Pointer to the main reply descriptor structure.
+ */
+struct leapraid_rep_desc_seg_maint {
+	void *rep_desc_seg;
+	dma_addr_t rep_desc_seg_dma;
+	struct leapraid_rep_desc_maint *rep_desc_maint;
+};
+
+/**
+ * struct leapraid_mem_desc - Memory descriptor for LeapRAID adapter
+ *
+ * @task_desc: Pointer to task descriptor.
+ * @task_desc_dma: DMA address of task descriptor.
+ * @sg_chain_pool: DMA pool for SGL chain allocations.
+ * @sg_chain_pool_size: Size of the sg_chain_pool.
+ * @taskid_to_uniq_tag: Mapping from task ID to unique tag.
+ * @sense_data: Buffer for SCSI sense data.
+ * @sense_data_dma: DMA address of sense_data buffer.
+ * @rep_msg: Buffer for reply message.
+ * @rep_msg_dma: DMA address of reply message buffer.
+ * @rep_msg_addr: Pointer to reply message address.
+ * @rep_msg_addr_dma: DMA address of reply message address.
+ * @rep_desc_seg_maint: Pointer to reply descriptor segment.
+ * @rep_desc_q_arr: Pointer to reply descriptor queue array.
+ * @rep_desc_q_arr_dma: DMA address of reply descriptor queue array.
+ */
+struct leapraid_mem_desc {
+	void *task_desc;
+	dma_addr_t task_desc_dma;
+	struct dma_pool *sg_chain_pool;
+	u16 sg_chain_pool_size;
+	u16 *taskid_to_uniq_tag;
+	u8 *sense_data;
+	dma_addr_t sense_data_dma;
+	u8 *rep_msg;
+	dma_addr_t rep_msg_dma;
+	__le32 *rep_msg_addr;
+	dma_addr_t rep_msg_addr_dma;
+	struct leapraid_rep_desc_seg_maint *rep_desc_seg_maint;
+	struct leapraid_rep_desc_q_arr *rep_desc_q_arr;
+	dma_addr_t rep_desc_q_arr_dma;
+};
+
+/* internal cmd description */
+#define LEAPRAID_FIXED_INTER_CMDS       5
+#define LEAPRAID_FIXED_HP_CMDS          2
+
+#define LEAPRAID_CMD_NOT_USED           0x8000
+#define LEAPRAID_CMD_DONE               0x0001
+#define LEAPRAID_CMD_PENDING            0x0002
+#define LEAPRAID_CMD_REPLY_VALID        0x0004
+#define LEAPRAID_CMD_RESET              0x0008
+
+/**
+ * enum LEAPRAID_CB_INDEX - Callback index for LeapRAID driver
+ *
+ * @LEAPRAID_SCAN_DEV_CB_IDX: Scan device callback index.
+ * @LEAPRAID_CONFIG_CB_IDX: Configuration callback index.
+ * @LEAPRAID_TRANSPORT_CB_IDX: Transport callback index.
+ * @LEAPRAID_SAS_CTRL_CB_IDX: SAS controller callback index.
+ * @LEAPRAID_ENC_CB_IDX: Encryption callback index.
+ * @LEAPRAID_NOTIFY_EVENT_CB_IDX: Notify event callback index.
+ * @LEAPRAID_CTL_CB_IDX: Control callback index.
+ * @LEAPRAID_TM_CB_IDX: Task management callback index.
+ */
+enum LEAPRAID_CB_INDEX {
+	LEAPRAID_SCAN_DEV_CB_IDX	= 0x1,
+	LEAPRAID_CONFIG_CB_IDX		= 0x2,
+	LEAPRAID_TRANSPORT_CB_IDX	= 0x3,
+	LEAPRAID_SAS_CTRL_CB_IDX	= 0x5,
+	LEAPRAID_ENC_CB_IDX		= 0x6,
+	LEAPRAID_NOTIFY_EVENT_CB_IDX	= 0x7,
+	LEAPRAID_CTL_CB_IDX		= 0x8,
+	LEAPRAID_TM_CB_IDX		= 0x9,
+	LEAPRAID_NUM_CB_IDXS
+};
+
+/**
+ * struct leapraid_default_reply - Default reply frame buffer
+ * @pad: Raw reply data buffer returned by firmware.
+ *
+ * This structure represents a generic reply frame used by the firmware
+ * when no specific reply format is required. The buffer size is defined
+ * by LEAPRAID_REPLY_SIZE and stores the raw reply data.
+ */
+struct leapraid_default_reply {
+	u8 pad[LEAPRAID_REPLY_SIZE];
+};
+
+/**
+ * struct leapraid_sense_buffer - SCSI sense data buffer
+ * @pad: Buffer used to store sense data returned by a SCSI command.
+ *
+ * This buffer holds the sense data provided by a device.The size is
+ * defined by SCSI_SENSE_BUFFERSIZE.
+ */
+struct leapraid_sense_buffer {
+	u8 pad[SCSI_SENSE_BUFFERSIZE];
+};
+
+/**
+ * struct leapraid_driver_cmd - Driver command tracking structure
+ *
+ * @reply: Default reply structure returned by the adapter.
+ * @done: Completion object used to signal command completion.
+ * @status: Status code returned by the firmware.
+ * @taskid: Unique task identifier for this command.
+ * @hp_taskid: Task identifier for high-priority commands.
+ * @inter_taskid: Task identifier for internal commands.
+ * @cb_idx: Callback index used to identify completion context.
+ * @async_scan_dev: True if this command is for asynchronous device scan.
+ * @sense: Sense buffer holding error information from device.
+ * @mutex: Mutex to protect access to this command structure.
+ * @list: List node for linking driver commands into lists.
+ */
+struct leapraid_driver_cmd {
+	struct leapraid_default_reply reply;
+	struct completion done;
+	u16 status;
+	u16 taskid;
+	u16 hp_taskid;
+	u16 inter_taskid;
+	u8 cb_idx;
+	u8 async_scan_dev;
+	struct leapraid_sense_buffer sense;
+	struct mutex mutex; /* Mutex for driver cmd */
+	struct list_head list;
+};
+
+/**
+ * struct leapraid_driver_cmds - Collection of driver command objects
+ *
+ * @special_cmd_list: List head for tracking special driver commands.
+ * @scan_dev_cmd: Command used for asynchronous device scan operations.
+ * @cfg_op_cmd: Command for configuration operations.
+ * @transport_cmd: Command for transport-level operations.
+ * @enc_cmd: Command for enclosure management operations.
+ * @notify_event_cmd: Command for asynchronous event notification handling.
+ * @ctl_cmd: Command for generic control or maintenance operations.
+ * @tm_cmd: Task management command.
+ */
+struct leapraid_driver_cmds {
+	struct list_head special_cmd_list;
+	struct leapraid_driver_cmd scan_dev_cmd;
+	struct leapraid_driver_cmd cfg_op_cmd;
+	struct leapraid_driver_cmd transport_cmd;
+	struct leapraid_driver_cmd enc_cmd;
+	struct leapraid_driver_cmd notify_event_cmd;
+	struct leapraid_driver_cmd ctl_cmd;
+	struct leapraid_driver_cmd tm_cmd;
+};
+
+/**
+ * struct leapraid_dynamic_task_desc - Dynamic task descriptor
+ *
+ * @task_lock: Spinlock to protect concurrent access.
+ * @hp_taskid: Current high-priority task ID.
+ * @hp_cmd_qd: Fixed command queue depth for high-priority tasks.
+ * @inter_taskid: Current internal task ID.
+ * @inter_cmd_qd: Fixed command queue depth for internal tasks.
+ */
+struct leapraid_dynamic_task_desc {
+	spinlock_t task_lock; /* protects dynamic task */
+	u16 hp_taskid;
+	u16 hp_cmd_qd;
+	u16 inter_taskid;
+	u16 inter_cmd_qd;
+};
+
+/**
+ * struct leapraid_fw_evt_work - Firmware event work structure
+ *
+ * @list: Linked list node for queuing the work.
+ * @adapter: Pointer to the associated LeapRAID adapter.
+ * @work: Work structure used by the kernel workqueue.
+ * @refcnt: Reference counter for managing the lifetime of this work.
+ * @evt_data: Pointer to firmware event data.
+ * @dev_handle: Device handle associated with the event.
+ * @evt_type: Type of firmware event.
+ * @ignore: Flag indicating whether the event should be ignored.
+ */
+struct leapraid_fw_evt_work {
+	struct list_head list;
+	struct leapraid_adapter *adapter;
+	struct work_struct work;
+	struct kref refcnt;
+	void *evt_data;
+	u16 dev_handle;
+	u16 evt_type;
+	u8 ignore;
+};
+
+/**
+ * struct leapraid_fw_evt_struct - Firmware event handling structure
+ *
+ * @fw_evt_name: Name of the firmware event.
+ * @fw_evt_thread: Workqueue used for processing firmware events.
+ * @fw_evt_lock: Spinlock protecting access to the firmware event list.
+ * @fw_evt_list: Linked list of pending firmware events.
+ * @cur_evt: Pointer to the currently processing firmware event.
+ * @cur_evt_task: Task currently executing @cur_evt work.
+ * @fw_evt_cleanup: Flag indicating whether cleanup of events is in progress.
+ * @leapraid_evt_masks: Array of event masks for filtering firmware events.
+ */
+struct leapraid_fw_evt_struct {
+	u32 leapraid_evt_masks[4];
+	char fw_evt_name[48];
+	struct workqueue_struct *fw_evt_thread;
+	spinlock_t fw_evt_lock; /* protects firmware event */
+	struct list_head fw_evt_list;
+	struct leapraid_fw_evt_work *cur_evt;
+	struct task_struct *cur_evt_task;
+	u8 fw_evt_cleanup;
+};
+
+/**
+ * struct leapraid_rq - Represents a LeapRAID request queue
+ *
+ * @adapter: Pointer to the associated LeapRAID adapter.
+ * @msix_idx: MSI-X vector index used by this queue.
+ * @rep_post_host_idx: Index of the last processed reply descriptor.
+ * @rep_desc: Pointer to the reply descriptor associated with this queue.
+ * @name: Name of the request queue.
+ * @busy: Atomic counter indicating if the queue is busy.
+ */
+struct leapraid_rq {
+	struct leapraid_adapter *adapter;
+	u8 msix_idx;
+	u32 rep_post_host_idx;
+	union leapraid_rep_desc_union *rep_desc;
+	char name[LEAPRAID_NAME_LENGTH];
+	atomic_t busy;
+};
+
+/**
+ * struct leapraid_int_rq - Internal request queue for a CPU
+ *
+ * @affinity_hint: CPU affinity mask for the queue.
+ * @rq: Underlying LeapRAID request queue structure.
+ */
+struct leapraid_int_rq {
+	cpumask_var_t affinity_hint;
+	struct leapraid_rq rq;
+};
+
+/**
+ * struct leapraid_blk_mq_poll_rq - Polling request for LeapRAID blk-mq
+ *
+ * @busy: Atomic flag indicating request is being processed.
+ * @pause: Atomic flag to temporarily suspend polling.
+ * @rq: The underlying LeapRAID request structure.
+ */
+struct leapraid_blk_mq_poll_rq {
+	atomic_t busy;
+	atomic_t pause;
+	struct leapraid_rq rq;
+};
+
+/**
+ * struct leapraid_notification_desc - Notification descriptor for LeapRAID
+ *
+ * @iopoll_qdex: Index of the I/O polling queue.
+ * @iopoll_qcnt: Count of I/O polling queues.
+ * @msix_enable: Flag indicating MSI-X is enabled.
+ * @irq_vectors_allocated: Flag indicating PCI IRQ vectors are allocated.
+ * @irq_mode: Actual interrupt mode used for allocated PCI IRQ vectors.
+ * @msix_cpu_map: CPU-ID indexed MSI-X queue map.
+ * @msix_cpu_map_sz: Number of CPU-ID slots allocated in @msix_cpu_map.
+ * @int_rqs: Array of interrupt request queues.
+ * @int_rqs_allocated: Count of allocated interrupt request queues.
+ * @blk_mq_poll_rqs: Array of blk-mq polling requests.
+ */
+struct leapraid_notification_desc {
+	u32 iopoll_qdex;
+	u32 iopoll_qcnt;
+	u8 msix_enable;
+	u8 irq_vectors_allocated;
+	u8 irq_mode;
+	u8 *msix_cpu_map;
+	u32 msix_cpu_map_sz;
+	struct leapraid_int_rq *int_rqs;
+	u32 int_rqs_allocated;
+	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rqs;
+};
+
+/**
+ * struct leapraid_overheat_desc - Overheat descriptor for LeapRAID
+ *
+ * @fault_overheat_wq: Workqueue for adapter thermal overheat operations.
+ * @fault_overheat_work: Work structure for thermal overheat.
+ * @thermal_alert: Flag indicating if adapter thermal alert is active.
+ * @fault_overheat_wq_name: Name of the fault overheat workqueue.
+ */
+struct leapraid_overheat_desc {
+	struct workqueue_struct *fault_overheat_wq;
+	struct work_struct fault_overheat_work;
+	atomic_t thermal_alert;
+	char fault_overheat_wq_name[48];
+};
+
+/**
+ * struct leapraid_reset_desc - Reset descriptor for LeapRAID
+ *
+ * @fault_reset_wq: Workqueue for fault reset operations.
+ * @fault_reset_work: Delayed work structure for fault reset.
+ * @fault_reset_wq_name: Name of the fault reset workqueue.
+ * @host_diag_mutex: Mutex for host diagnostic operations.
+ * @adapter_reset_lock: Spinlock for adapter reset operations.
+ * @adapter_reset_mutex: Mutex for adapter reset operations.
+ * @adapter_link_resetting: Flag indicating if adapter link is resetting.
+ * @adapter_reset_results: Results of the adapter reset operation.
+ * @pending_io_cnt: Count of pending I/O operations.
+ * @reset_wait_queue: Wait queue for reset operations.
+ * @reset_cnt: Counter for reset operations.
+ */
+struct leapraid_reset_desc {
+	struct workqueue_struct *fault_reset_wq;
+	struct delayed_work fault_reset_work;
+	char fault_reset_wq_name[48];
+	struct mutex host_diag_mutex; /* Mutex for operations. */
+	spinlock_t adapter_reset_lock; /* Protects adapter reset state. */
+	struct mutex adapter_reset_mutex; /* Serializes adapter reset. */
+	u8 adapter_link_resetting;
+	int adapter_reset_results;
+	int pending_io_cnt;
+	wait_queue_head_t reset_wait_queue;
+	u32 reset_cnt;
+};
+
+/**
+ * struct leapraid_scan_dev_desc - Scan device descriptor for LeapRAID
+ *
+ * @wait_scan_dev_done: Flag indicating if scan device operation is done.
+ * @driver_loading: Flag indicating if driver is loading.
+ * @first_scan_dev_fired: Flag indicating if first scan device operation fired.
+ * @scan_dev_failed: Flag indicating if scan device operation failed.
+ * @scan_start: Flag indicating if scan operation started.
+ * @scan_start_failed: Count of failed scan start operations.
+ */
+struct leapraid_scan_dev_desc {
+	u8 wait_scan_dev_done;
+	u8 driver_loading;
+	wait_queue_head_t wait_driver_loading;
+	u8 first_scan_dev_fired;
+	u8 scan_dev_failed;
+	u8 scan_start;
+	u16 scan_start_failed;
+};
+
+/**
+ * struct leapraid_access_ctrl - Access control structure for LeapRAID
+ *
+ * @pci_access_lock: Mutex for PCI access control.
+ * @shost_recovering: Flag indicating if host is recovering.
+ * @shost_recover_async: Flag indicating if host is recovering
+ *                       and the 0xFFFF event.
+ * @shost_recover_wq: Wait queue for threads waiting on shost_recover_async.
+ * @host_removing: Flag indicating if host is being removed.
+ * @pcie_recovering: Flag indicating if PCIe is recovering.
+ */
+struct leapraid_access_ctrl {
+	struct mutex pci_access_lock; /* serializes PCI register access. */
+	u8 shost_recovering;
+	wait_queue_head_t recovery_waitq;
+	u8 shost_recover_async;
+	wait_queue_head_t shost_recover_wq;
+	u8 host_removing;
+	u8 pcie_recovering;
+};
+
+/**
+ * struct leapraid_fw_log_desc - Firmware log descriptor for LeapRAID
+ *
+ * @fw_log_buffer: Buffer for firmware log data.
+ * @fw_log_buffer_dma: DMA address of the firmware log buffer.
+ * @fw_log_wq_name: Name of the firmware log workqueue.
+ * @fw_log_wq: Workqueue for firmware log operations.
+ * @fw_log_work: Delayed work structure for firmware log.
+ * @open_pcie_trace: Flag indicating if PCIe tracing is open.
+ * @fw_log_init_flag: Flag indicating if firmware log is initialized.
+ * @mmap_refcnt: Number of active user VMAs on fw_log_buffer.
+ * @mmap_waitq: Waitqueue for fw_log_buffer VMA teardown.
+ */
+struct leapraid_fw_log_desc {
+	u8 *fw_log_buffer;
+	dma_addr_t fw_log_buffer_dma;
+	char fw_log_wq_name[48];
+	struct workqueue_struct *fw_log_wq;
+	struct delayed_work fw_log_work;
+	int open_pcie_trace;
+	int fw_log_init_flag;
+	atomic_t mmap_refcnt;
+	wait_queue_head_t mmap_waitq;
+};
+
+#define LEAPRAID_CARD_PORT_FLG_DIRTY	0x01
+#define LEAPRAID_CARD_PORT_FLG_NEW	0x02
+#define LEAPRAID_DISABLE_MP_PORT_ID	0xFF
+/**
+ * struct leapraid_card_port - Card port structure for LeapRAID
+ *
+ * @list: List head for card port.
+ * @vphys_list: List head for virtual PHY list.
+ * @port_id: Port ID.
+ * @sas_address: SAS address.
+ * @phy_mask: Mask of PHY.
+ * @vphys_mask: Mask of virtual PHY.
+ * @flg: Flags for the port.
+ */
+struct leapraid_card_port {
+	struct list_head list;
+	struct list_head vphys_list;
+	u8 port_id;
+	u64 sas_address;
+	u32 phy_mask;
+	u32 vphys_mask;
+	u8 flg;
+};
+
+/**
+ * struct leapraid_card_phy - Card PHY structure for LeapRAID
+ *
+ * @port_siblings: List head for port siblings.
+ * @card_port: Pointer to the card port.
+ * @identify: SAS identify structure.
+ * @remote_identify: Remote SAS identify structure.
+ * @phy: SAS PHY structure.
+ * @phy_id: PHY ID.
+ * @hdl: Handle for the port.
+ * @attached_hdl: Handle for the attached port.
+ * @phy_is_assigned: Flag indicating if PHY is assigned.
+ * @vphy: Flag indicating if virtual PHY.
+ */
+struct leapraid_card_phy {
+	struct list_head port_siblings;
+	struct leapraid_card_port *card_port;
+	struct sas_identify identify;
+	struct sas_identify remote_identify;
+	struct sas_phy *phy;
+	u8 phy_id;
+	u16 hdl;
+	u16 attached_hdl;
+	u8 phy_is_assigned;
+	u8 vphy;
+};
+
+/**
+ * struct leapraid_topo_node - SAS topology node for LeapRAID
+ *
+ * @list: List head for linking nodes.
+ * @sas_port_list: List of SAS ports.
+ * @card_port: Associated card port.
+ * @card_phy: Associated card PHY.
+ * @rphy: SAS remote PHY device.
+ * @parent_dev: Parent device pointer.
+ * @sas_address: SAS address of this node.
+ * @sas_address_parent: Parent node's SAS address.
+ * @phys_num: Number of physical links.
+ * @hdl: Handle identifier.
+ * @enc_hdl: Enclosure handle.
+ * @enc_lid: Enclosure logical identifier.
+ * @resp: Response status flag.
+ */
+struct leapraid_topo_node {
+	struct list_head list;
+	struct list_head sas_port_list;
+	struct leapraid_card_port *card_port;
+	struct leapraid_card_phy *card_phy;
+	struct sas_rphy *rphy;
+	struct device *parent_dev;
+	u64 sas_address;
+	u64 sas_address_parent;
+	u8 phys_num;
+	u16 hdl;
+	u16 enc_hdl;
+	u64 enc_lid;
+	u8 resp;
+};
+
+/**
+ * struct leapraid_dev_topo - LeapRAID device topology management structure
+ *
+ * @topo_node_lock: Spinlock for protecting topology node operations.
+ * @sas_dev_lock: Spinlock for SAS device list access.
+ * @raid_volume_lock: Spinlock for RAID volume list access.
+ * @enc_lock: Spinlock for enclosure device list access.
+ * @sas_id: SAS domain identifier.
+ * @card: Main card topology node.
+ * @exp_list: List of expander devices.
+ * @enc_list: List of enclosure devices.
+ * @sas_dev_list: List of SAS devices.
+ * @sas_dev_init_list: List of SAS devices being initialized.
+ * @raid_volume_list: List of RAID volumes.
+ * @card_port_list: List of card ports.
+ * @pd_hdls: Array of physical disk handles.
+ * @blocking_hdls: Array of blocking handles.
+ */
+struct leapraid_dev_topo {
+	spinlock_t topo_node_lock; /* Protects topology node. */
+	spinlock_t sas_dev_lock; /* Protects SAS device list access. */
+	spinlock_t raid_volume_lock; /* Protects RAID volume list access. */
+	spinlock_t enc_lock; /* Protects enclosure device list access. */
+	int sas_id;
+	struct leapraid_topo_node card;
+	struct list_head exp_list;
+	struct list_head enc_list;
+	struct list_head sas_dev_list;
+	struct list_head sas_dev_init_list;
+	struct list_head raid_volume_list;
+	struct list_head card_port_list;
+	u16 pd_hdls_sz;
+	unsigned long *pd_hdls;
+	unsigned long *blocking_hdls;
+};
+
+/**
+ * struct leapraid_boot_dev - Boot device structure for LeapRAID
+ *
+ * @dev: Device pointer.
+ * @chnl: Channel number.
+ * @form: Form factor.
+ * @pg_dev: Config page device content.
+ */
+struct leapraid_boot_dev {
+	void *dev;
+	u8 chnl;
+	u8 form;
+	u8 pg_dev[24];
+};
+
+/**
+ * struct leapraid_boot_devs - Boot device management structure
+ *
+ * @lock: Spinlock protecting boot device cache access.
+ * @requested_boot_dev: Requested primary boot device.
+ * @requested_alt_boot_dev: Requested alternate boot device.
+ * @current_boot_dev: Currently active boot device.
+ */
+struct leapraid_boot_devs {
+	spinlock_t lock; /* protects boot dev */
+	struct leapraid_boot_dev requested_boot_dev;
+	struct leapraid_boot_dev requested_alt_boot_dev;
+	struct leapraid_boot_dev current_boot_dev;
+};
+
+/**
+ * struct leapraid_adapter - Main LeapRAID adapter structure
+ *
+ * @list: List head for adapter management.
+ * @shost: SCSI host structure.
+ * @pdev: PCI device structure.
+ * @iomem_base: I/O memory mapped base address.
+ * @rep_msg_host_idx: Host index for reply messages.
+ * @mask_int: Interrupt masking flag.
+ * @adapter_attr: Adapter attributes.
+ * @mem_desc: Memory descriptor.
+ * @driver_cmds: Driver commands.
+ * @dynamic_task_desc: Dynamic task descriptor.
+ * @fw_evt_s: Firmware event structure.
+ * @notification_desc: Notification descriptor.
+ * @reset_desc: Reset descriptor.
+ * @scan_dev_desc: Device scan descriptor.
+ * @access_ctrl: Access control.
+ * @fw_log_desc: Firmware log descriptor.
+ * @dev_topo: Device topology.
+ * @boot_devs: Boot devices.
+ * @overheat_desc: Overheat processing descriptor.
+ */
+struct leapraid_adapter {
+	struct list_head list;
+	struct Scsi_Host *shost;
+	struct pci_dev *pdev;
+	struct leapraid_reg_base __iomem *iomem_base;
+	u32 rep_msg_host_idx;
+	u8 mask_int;
+
+	struct leapraid_adapter_attr adapter_attr;
+	struct leapraid_mem_desc mem_desc;
+	struct leapraid_driver_cmds driver_cmds;
+	struct leapraid_dynamic_task_desc dynamic_task_desc;
+	struct leapraid_fw_evt_struct fw_evt_s;
+	struct leapraid_notification_desc notification_desc;
+	struct leapraid_reset_desc reset_desc;
+	struct leapraid_scan_dev_desc scan_dev_desc;
+	struct leapraid_access_ctrl access_ctrl;
+	struct leapraid_fw_log_desc fw_log_desc;
+	struct leapraid_dev_topo dev_topo;
+	struct leapraid_boot_devs boot_devs;
+	struct leapraid_overheat_desc overheat_desc;
+};
+
+union cfg_param_1 {
+	u32 form;
+	u32 size;
+	u32 phy_number;
+};
+
+union cfg_param_2 {
+	u32 handle;
+	u32 form_specific;
+};
+
+enum config_page_action {
+	GET_BIOS_PG2,
+	GET_BIOS_PG3,
+	GET_MANUFACTURING_PG0,
+	GET_SAS_DEVICE_PG0,
+	GET_SAS_IOUNIT_PG0,
+	GET_SAS_IOUNIT_PG1,
+	GET_SAS_EXPANDER_PG0,
+	GET_SAS_EXPANDER_PG1,
+	GET_SAS_ENCLOSURE_PG0,
+	GET_PHY_PG0,
+	GET_RAID_VOLUME_PG0,
+	GET_RAID_VOLUME_PG1,
+	GET_PHY_DISK_PG0,
+};
+
+/**
+ * struct leapraid_enc_node - Enclosure node structure
+ *
+ * @list: List head for enclosure management.
+ * @pg0: Enclosure page 0 data.
+ */
+struct leapraid_enc_node {
+	struct list_head list;
+	struct leapraid_enc_p0 pg0;
+};
+
+/**
+ * struct leapraid_raid_volume - RAID volume structure
+ *
+ * @list: List head for volume management.
+ * @refcnt: Reference count.
+ * @starget: SCSI target structure.
+ * @sdev: SCSI device structure.
+ * @id: Volume ID.
+ * @channel: SCSI channel.
+ * @wwid: World wide Identifier.
+ * @hdl: Volume handle.
+ * @vol_type: Volume type.
+ * @pd_num: Number of physical disks.
+ * @resp: Response status.
+ * @dev_info: Device information.
+ */
+struct leapraid_raid_volume {
+	struct list_head list;
+	struct kref refcnt;
+	struct scsi_target *starget;
+	struct scsi_device *sdev;
+	unsigned int id;
+	unsigned int channel;
+	u64 wwid;
+	u16 hdl;
+	u8 vol_type;
+	u8 pd_num;
+	u8 resp;
+	u32 dev_info;
+};
+
+static inline void leapraid_raid_volume_free(struct kref *ref)
+{
+	kfree(container_of(ref, struct leapraid_raid_volume, refcnt));
+}
+
+#define leapraid_raid_volume_get(volume) \
+	kref_get(&(volume)->refcnt)
+#define leapraid_raid_volume_put(volume) \
+	kref_put(&(volume)->refcnt, leapraid_raid_volume_free)
+
+#define LEAPRAID_TGT_FLG_RAID_MEMBER	0x01
+#define LEAPRAID_TGT_FLG_VOLUME		0x02
+#define LEAPRAID_NO_ULD_ATTACH		1
+/**
+ * struct leapraid_starget_priv - SCSI target private data
+ *
+ * @starget: SCSI target structure.
+ * @sas_address: SAS address.
+ * @hdl: Device handle.
+ * @num_luns: Number of LUNs.
+ * @flg: Flags.
+ * @deleted: Deletion flag.
+ * @tm_busy: Task management busy flag.
+ * @card_port: Associated card port.
+ * @sas_dev: SAS device structure.
+ */
+struct leapraid_starget_priv {
+	struct scsi_target *starget;
+	u64 sas_address;
+	u16 hdl;
+	int num_luns;
+	u32 flg;
+	u8 deleted;
+	u8 tm_busy;
+	struct leapraid_card_port *card_port;
+	struct leapraid_sas_dev *sas_dev;
+};
+
+#define LEAPRAID_DEVICE_FLG_INIT	0x01
+/**
+ * struct leapraid_sdev_priv - SCSI device private data
+ *
+ * @starget_priv: Associated target private data.
+ * @lun: Logical Unit Number.
+ * @flg: Flags.
+ * @ncq_prio_enable: Enables NCQ command priority for RT I/O.
+ * @block: Block flag.
+ * @deleted: Deletion flag.
+ * @sep: SEP flag.
+ */
+struct leapraid_sdev_priv {
+	struct leapraid_starget_priv *starget_priv;
+	unsigned int lun;
+	u32 flg;
+	u8 ncq_prio_enable;
+	u8 block;
+	u8 deleted;
+	u8 sep;
+};
+
+/**
+ * struct leapraid_sas_dev - SAS device structure
+ *
+ * @list: List head for device management.
+ * @starget: SCSI target structure.
+ * @card_port: Associated card port.
+ * @rphy: SAS remote PHY.
+ * @refcnt: Reference count.
+ * @id: Device ID.
+ * @channel: SCSI channel.
+ * @slot: Slot number.
+ * @phy: PHY identifier.
+ * @resp: Response status.
+ * @led_on: LED state.
+ * @sas_addr: SAS address.
+ * @dev_name: Device name.
+ * @hdl: Device handle.
+ * @parent_sas_addr: Parent SAS address.
+ * @enc_hdl: Enclosure handle.
+ * @enc_lid: Enclosure logical ID.
+ * @volume_hdl: Volume handle.
+ * @volume_wwid: Volume WWID.
+ * @dev_info: Device information.
+ * @pend_sas_rphy_add: Pending SAS remote PHY addition flag.
+ * @enc_level: Enclosure level.
+ * @port_connection: Port connection.
+ * @connector_name: Connector name.
+ */
+struct leapraid_sas_dev {
+	struct list_head list;
+	struct scsi_target *starget;
+	struct leapraid_card_port *card_port;
+	struct sas_rphy *rphy;
+	struct kref refcnt;
+	unsigned int id;
+	unsigned int channel;
+	u16 slot;
+	u8 phy;
+	u8 resp;
+	u8 led_on;
+	u64 sas_addr;
+	u64 dev_name;
+	u16 hdl;
+	u64 parent_sas_addr;
+	u16 enc_hdl;
+	u64 enc_lid;
+	u16 volume_hdl;
+	u64 volume_wwid;
+	u32 dev_info;
+	u8 pend_sas_rphy_add;
+	u8 enc_level;
+	u8 port_connection;
+	u8 connector_name[LEAPRAID_SAS_DEV_P0_CON_NAME_LEN + 1];
+};
+
+static inline void leapraid_sdev_free(struct kref *ref)
+{
+	kfree(container_of(ref, struct leapraid_sas_dev, refcnt));
+}
+
+#define leapraid_sdev_get(sdev)	kref_get(&(sdev)->refcnt)
+#define leapraid_sdev_put(sdev)	kref_put(&(sdev)->refcnt, leapraid_sdev_free)
+
+void leapraid_boot_dev_get(void *dev, u32 chnl);
+void leapraid_boot_dev_put(void *dev, u32 chnl);
+
+/**
+ * struct leapraid_sas_port - SAS port structure
+ *
+ * @port_list: List head for port management.
+ * @phy_list: List of PHYs in this port.
+ * @port: SAS port structure.
+ * @card_port: Associated card port.
+ * @remote_identify: Remote device identification.
+ * @rphy: SAS remote PHY.
+ * @phys_num: Number of PHYs in this port.
+ */
+struct leapraid_sas_port {
+	struct list_head port_list;
+	struct list_head phy_list;
+	struct sas_port *port;
+	struct leapraid_card_port *card_port;
+	struct sas_identify remote_identify;
+	struct sas_rphy *rphy;
+	u8 phys_num;
+};
+
+#define LEAPRAID_VPHY_FLG_DIRTY	0x01
+/**
+ * struct leapraid_vphy - Virtual PHY structure
+ *
+ * @list: List head for PHY management.
+ * @sas_address: SAS address.
+ * @phy_mask: PHY mask.
+ * @flg: Flags.
+ */
+struct leapraid_vphy {
+	struct list_head list;
+	u64 sas_address;
+	u32 phy_mask;
+	u8 flg;
+};
+
+/**
+ * struct leapraid_tgt_rst_list - Target reset tracking entry
+ * @list: List node used to link reset entries.
+ * @handle: Device handle of the target being reset.
+ * @state: Current state of the target reset operation.
+ *
+ * Each entry represents a target device that is undergoing or has
+ * undergone a target reset operation.
+ */
+struct leapraid_tgt_rst_list {
+	struct list_head list;
+	u16 handle;
+	u16 state;
+};
+
+/**
+ * struct sense_info - Parsed SCSI sense information
+ * @sense_key: Sense key extracted from sense data.
+ * @asc: Additional Sense Code (ASC).
+ * @ascq: Additional Sense Code Qualifier (ASCQ).
+ *
+ * This structure stores key fields parsed from SCSI sense data to
+ * simplify error handling and reporting.
+ */
+struct sense_info {
+	u8 sense_key;
+	u8 asc;
+	u8 ascq;
+};
+
+/**
+ * struct leapraid_fw_log_info - Firmware log position information
+ * @user_position: Current log read position used by the driver.
+ * @adapter_position: Current log write position maintained by firmware.
+ *
+ * This structure tracks firmware log buffer positions used for retrieving
+ * log entries from the adapter.
+ */
+struct leapraid_fw_log_info {
+	u32 user_position;
+	u32 adapter_position;
+};
+
+/**
+ * enum reset_type - Reset type enumeration
+ *
+ * @FULL_RESET: Full hardware reset.
+ * @PART_RESET: Partial reset.
+ */
+enum reset_type {
+	FULL_RESET,
+	PART_RESET,
+};
+
+enum leapraid_card_port_checking_flg {
+	CARD_PORT_FURTHER_CHECKING_NEEDED = 0,
+	CARD_PORT_SKIP_CHECKING,
+};
+
+enum leapraid_port_checking_state {
+	NEW_CARD_PORT = 0,
+	SAME_PORT_WITH_NOTHING_CHANGED,
+	SAME_PORT_WITH_PARTIALLY_CHANGED_PHYS,
+	SAME_ADDR_WITH_PARTIALLY_CHANGED_PHYS,
+	SAME_ADDR_ONLY,
+};
+
+/**
+ * struct leapraid_card_port_feature - Card port feature
+ *
+ * @dirty_flg: Dirty flag indicator.
+ * @same_addr: Same address flag.
+ * @exact_phy: Exact PHY match flag.
+ * @phy_overlap: PHY overlap bitmap.
+ * @same_port: Same port flag.
+ * @cur_chking_old_port: Current checking old port.
+ * @expected_old_port: Expected old port.
+ * @same_addr_port_count: Same address port count.
+ * @checking_state: Port checking state.
+ */
+struct leapraid_card_port_feature {
+	u8 dirty_flg;
+	u8 same_addr;
+	u8 exact_phy;
+	u32 phy_overlap;
+	u8 same_port;
+	struct leapraid_card_port *cur_chking_old_port;
+	struct leapraid_card_port *expected_old_port;
+	int same_addr_port_count;
+	enum leapraid_port_checking_state checking_state;
+};
+
+#define SMP_REPORT_MANUFACTURER_INFORMATION_FRAME_TYPE	0x40
+#define SMP_REPORT_MANUFACTURER_INFORMATION_FUNC	0x01
+
+/**
+ * ref: SAS-2(INCITS 457-2010) 10.4.3.5
+ */
+struct leapraid_rep_manu_request {
+	u8 smp_frame_type;
+	u8 function;
+	u8 allocated_response_length;
+	u8 request_length;
+};
+
+/**
+ * ref: SAS-2(INCITS 457-2010) 10.4.3.5
+ */
+struct leapraid_rep_manu_reply {
+	u8 smp_frame_type;
+	u8 function;
+	u8 function_result;
+	u8 response_length;
+	u16 expander_change_count;
+	u8 r1[2];
+	u8 sas_format;
+	u8 r2[3];
+	u8 vendor_identification[SAS_EXPANDER_VENDOR_ID_LEN];
+	u8 product_identification[SAS_EXPANDER_PRODUCT_ID_LEN];
+	u8 product_revision_level[SAS_EXPANDER_PRODUCT_REV_LEN];
+	u8 comp_vendor_identification[SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN];
+	u16 component_id;
+	u8 component_revision_level;
+	u8 r3;
+	u8 vendor_specific[8];
+};
+
+extern struct list_head leapraid_adapter_list;
+extern spinlock_t leapraid_adapter_lock;
+extern char driver_name[LEAPRAID_NAME_LENGTH];
+
+int leapraid_ctrl_init(struct leapraid_adapter *adapter);
+void leapraid_remove_ctrl(struct leapraid_adapter *adapter);
+void leapraid_check_scheduled_fault_start(struct leapraid_adapter *adapter);
+void leapraid_check_scheduled_fault_stop(struct leapraid_adapter *adapter);
+void leapraid_fw_log_start(struct leapraid_adapter *adapter);
+void leapraid_fw_log_stop(struct leapraid_adapter *adapter);
+int leapraid_set_pcie_and_notification(struct leapraid_adapter *adapter);
+void leapraid_disable_controller(struct leapraid_adapter *adapter);
+int leapraid_hard_reset_handler(struct leapraid_adapter *adapter,
+				enum reset_type type);
+void leapraid_mask_int(struct leapraid_adapter *adapter);
+void leapraid_unmask_int(struct leapraid_adapter *adapter);
+u32 leapraid_get_adapter_state(struct leapraid_adapter *adapter);
+bool leapraid_pci_removed(struct leapraid_adapter *adapter);
+int leapraid_check_adapter_is_op(struct leapraid_adapter *adapter, int wait,
+				 const char *caller);
+void *leapraid_get_task_desc(struct leapraid_adapter *adapter, u16 taskid);
+void *leapraid_get_sense_buffer(struct leapraid_adapter *adapter, u16 taskid);
+__le32 leapraid_get_sense_buffer_dma(struct leapraid_adapter *adapter,
+				     u16 taskid);
+void *leapraid_get_reply_vaddr(struct leapraid_adapter *adapter,
+			       u32 phys_addr);
+u16 leapraid_alloc_scsiio_taskid(struct leapraid_adapter *adapter,
+				 struct scsi_cmnd *scmd);
+void leapraid_free_taskid(struct leapraid_adapter *adapter, u16 taskid);
+struct leapraid_io_req_tracker *leapraid_get_io_tracker_from_taskid(
+		struct leapraid_adapter *adapter, u16 taskid);
+struct scsi_cmnd *leapraid_get_scmd_from_taskid(
+		struct leapraid_adapter *adapter, u16 taskid);
+int leapraid_scan_dev(struct leapraid_adapter *adapter, bool async_scan_dev);
+void leapraid_scan_dev_done(struct leapraid_adapter *adapter);
+void leapraid_cleanup_lists(struct leapraid_adapter *adapter);
+void leapraid_wait_cmds_done(struct leapraid_adapter *adapter);
+void leapraid_clean_active_scsi_cmds(struct leapraid_adapter *adapter);
+void leapraid_sync_irqs(struct leapraid_adapter *adapter, bool poll);
+int leapraid_rep_queue_handler(struct leapraid_rq *rq);
+int leapraid_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num);
+void leapraid_mq_polling_pause(struct leapraid_adapter *adapter);
+void leapraid_mq_polling_resume(struct leapraid_adapter *adapter);
+void leapraid_set_tm_flg(struct leapraid_adapter *adapter, u16 handle);
+void leapraid_clear_tm_flg(struct leapraid_adapter *adapter, u16 handle);
+void leapraid_async_turn_on_led(struct leapraid_adapter *adapter, u16 handle);
+int leapraid_issue_locked_tm(struct leapraid_adapter *adapter, u16 handle,
+			     uint channel, uint id, uint lun, u8 type,
+			     u16 taskid_task, u8 tr_method);
+int leapraid_issue_tm(struct leapraid_adapter *adapter, u16 handle,
+		      uint channel, uint id, uint lun, u8 type,
+		      u16 taskid_task, u8 tr_method);
+bool leapraid_scsiio_done(struct leapraid_adapter *adapter, u16 taskid,
+			  u8 msix_index, u32 rep);
+int leapraid_get_volume_cap(struct leapraid_adapter *adapter,
+			    struct leapraid_raid_volume *raid_volume);
+int leapraid_internal_init_cmd_priv(
+		struct leapraid_adapter *adapter,
+		struct leapraid_io_req_tracker *io_tracker);
+void leapraid_internal_exit_cmd_priv(
+		struct leapraid_adapter *adapter,
+		struct leapraid_io_req_tracker *io_tracker);
+void leapraid_clean_active_fw_evt(struct leapraid_adapter *adapter);
+bool leapraid_scmd_find_by_lun(struct leapraid_adapter *adapter,
+			       uint id, unsigned int lun, uint channel);
+bool leapraid_scmd_find_by_tgt(struct leapraid_adapter *adapter,
+			       uint id, uint channel);
+struct leapraid_vphy *leapraid_get_vphy_by_phy(struct leapraid_card_port *port,
+					       u32 phy);
+struct leapraid_raid_volume *leapraid_raid_volume_find_by_id(
+		struct leapraid_adapter *adapter, uint id, uint channel);
+struct leapraid_raid_volume *leapraid_raid_volume_find_by_hdl(
+		struct leapraid_adapter *adapter, u16 handle);
+struct leapraid_topo_node *leapraid_exp_find_by_sas_address(
+		struct leapraid_adapter *adapter, u64 sas_address,
+		struct leapraid_card_port *port);
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_addr_and_rphy(
+		struct leapraid_adapter *adapter,
+		u64 sas_address, struct sas_rphy *rphy);
+struct leapraid_sas_dev *leapraid_get_sas_dev_by_addr(
+		struct leapraid_adapter *adapter, u64 sas_address,
+		struct leapraid_card_port *port);
+struct leapraid_sas_dev *leapraid_get_sas_dev_by_hdl(
+		struct leapraid_adapter *adapter, u16 handle);
+struct leapraid_sas_dev *leapraid_get_sas_dev_from_tgt(
+		struct leapraid_adapter *adapter,
+		struct leapraid_starget_priv *tgt_priv);
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_from_tgt(
+		struct leapraid_adapter *adapter,
+		 struct leapraid_starget_priv *tgt_priv);
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_hdl(
+		struct leapraid_adapter *adapter, u16 handle);
+struct leapraid_sas_dev *leapraid_hold_lock_get_sas_dev_by_addr(
+		struct leapraid_adapter *adapter, u64 sas_address,
+		struct leapraid_card_port *port);
+struct leapraid_sas_dev *leapraid_get_next_sas_dev_from_init_list(
+		struct leapraid_adapter *adapter);
+void leapraid_sas_dev_remove_by_sas_address(
+		struct leapraid_adapter *adapter,
+		u64 sas_address, struct leapraid_card_port *port);
+void leapraid_sas_dev_remove(struct leapraid_adapter *adapter,
+			     struct leapraid_sas_dev *sas_dev);
+void leapraid_raid_volume_remove(struct leapraid_adapter *adapter,
+				 struct leapraid_raid_volume *raid_volume);
+void leapraid_exp_rm(struct leapraid_adapter *adapter,
+		     u64 sas_address, struct leapraid_card_port *port);
+void leapraid_build_mpi_sg(struct leapraid_adapter *adapter,
+			   void *sge, dma_addr_t h2c_dma_addr, size_t h2c_size,
+			   dma_addr_t c2h_dma_addr, size_t c2h_size);
+void leapraid_build_ieee_nodata_sg(struct leapraid_adapter *adapter,
+				   void *sge);
+void leapraid_build_ieee_sg(struct leapraid_adapter *adapter,
+			    void *psge, dma_addr_t h2c_dma_addr,
+			    size_t h2c_size, dma_addr_t c2h_dma_addr,
+			    size_t c2h_size);
+int leapraid_build_scmd_ieee_sg(struct leapraid_adapter *adapter,
+				struct scsi_cmnd *scmd, u16 taskid);
+void leapraid_fire_scsi_io(struct leapraid_adapter *adapter,
+			   u16 taskid, u16 handle);
+void leapraid_fire_hpr_task(struct leapraid_adapter *adapter, u16 taskid,
+			    u16 msix_task);
+void leapraid_fire_task(struct leapraid_adapter *adapter, u16 taskid);
+int leapraid_cfg_get_volume_hdl(struct leapraid_adapter *adapter,
+				u16 pd_handle, u16 *volume_handle);
+int leapraid_cfg_get_volume_wwid(struct leapraid_adapter *adapter,
+				 u16 volume_handle, u64 *wwid);
+int leapraid_op_config_page(struct leapraid_adapter *adapter,
+			    void *cfgp, union cfg_param_1 cfgp1,
+			    union cfg_param_2 cfgp2,
+			    enum config_page_action cfg_op);
+void leapraid_log_req_context(struct leapraid_adapter *adapter, u16 smid,
+			      const void *req_data);
+
+int leapraid_change_queue_depth(struct scsi_device *sdev, int qdepth);
+
+int leapraid_ctl_release(struct inode *inode, struct file *filep);
+int leapraid_ctl_init(void);
+void leapraid_ctl_exit(void);
+
+extern struct sas_function_template leapraid_transport_functions;
+extern struct scsi_transport_template *leapraid_transport_template;
+struct leapraid_sas_port *leapraid_transport_port_add(
+		struct leapraid_adapter *adapter, u16 handle, u64 sas_address,
+		struct leapraid_card_port *card_port);
+void leapraid_transport_port_remove(struct leapraid_adapter *adapter,
+				    u64 sas_address, u64 sas_address_parent,
+				    struct leapraid_card_port *card_port);
+void leapraid_transport_add_card_phy(struct leapraid_adapter *adapter,
+				     struct leapraid_card_phy *card_phy,
+				     struct leapraid_sas_phy_p0 *phy_pg0,
+				     struct device *parent_dev);
+int leapraid_transport_add_exp_phy(struct leapraid_adapter *adapter,
+				   struct leapraid_card_phy *card_phy,
+				   struct leapraid_exp_p1 *exp_pg1,
+				   struct device *parent_dev);
+void leapraid_transport_update_links(struct leapraid_adapter *adapter,
+				     u64 sas_address, u16 handle,
+				     u8 phy_number, u8 link_rate,
+				     struct leapraid_card_port *card_port);
+void leapraid_transport_detach_phy_to_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node,
+		struct leapraid_card_phy *card_phy);
+void leapraid_transport_attach_phy_to_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *sas_node,
+		struct leapraid_card_phy *card_phy,
+		u64 sas_address,
+		struct leapraid_card_port *card_port);
+enum scsi_qc_status leapraid_queuecommand(struct Scsi_Host *shost,
+					  struct scsi_cmnd *scmd);
+void leapraid_overheat_cleanup(struct leapraid_adapter *adapter);
+void leapraid_smart_fault_detect(struct leapraid_adapter *adapter, u16 hdl);
+
+#endif /* LEAPRAID_FUNC_H_INCLUDED */
diff --git a/drivers/scsi/leapraid/leapraid_os.c b/drivers/scsi/leapraid/leapraid_os.c
new file mode 100644
index 000000000..ee3242779
--- /dev/null
+++ b/drivers/scsi/leapraid/leapraid_os.c
@@ -0,0 +1,2493 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 LeapIO Tech Inc.
+ *
+ * LeapRAID storage and RAID controller driver.
+ */
+#include <linux/module.h>
+
+#include "leapraid_func.h"
+#include "leapraid.h"
+
+LIST_HEAD(leapraid_adapter_list);
+DEFINE_SPINLOCK(leapraid_adapter_lock);
+
+MODULE_AUTHOR(LEAPRAID_AUTHOR);
+MODULE_DESCRIPTION(LEAPRAID_DESCRIPTION);
+MODULE_LICENSE("GPL");
+MODULE_VERSION(LEAPRAID_DRIVER_VERSION);
+
+static atomic_t leapraid_ids = ATOMIC_INIT(0);
+
+static int open_pcie_trace = 1;
+module_param(open_pcie_trace, int, 0644);
+MODULE_PARM_DESC(open_pcie_trace, "Default=1(open)/0(close)");
+
+static int enable_mp = 1;
+module_param(enable_mp, int, 0444);
+MODULE_PARM_DESC(enable_mp,
+		 "Enable multipath on target device. default=1(enable)");
+
+static inline void leapraid_get_sense_data(char *sense,
+					   struct sense_info *data)
+{
+	bool desc_format = (sense[0] & SCSI_SENSE_RESPONSE_CODE_MASK) >=
+			    DESC_FORMAT_THRESHOLD;
+
+	if (desc_format) {
+		data->sense_key = sense[1] & SENSE_KEY_MASK;
+		data->asc = sense[2];
+		data->ascq = sense[3];
+	} else {
+		data->sense_key = sense[2] & SENSE_KEY_MASK;
+		data->asc = sense[12];
+		data->ascq = sense[13];
+	}
+}
+
+static struct leapraid_adapter *pdev_to_adapter(struct pci_dev *pdev)
+{
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+
+	if (!shost)
+		return NULL;
+
+	return shost_priv(shost);
+}
+
+void leapraid_set_tm_flg(struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct scsi_device *sdev;
+	bool skip = false;
+
+	/* Iterate over all devices. */
+	shost_for_each_device(sdev, adapter->shost) {
+		if (skip)
+			continue;
+
+		sdev_priv = sdev->hostdata;
+		if (!sdev_priv)
+			continue;
+
+		if (sdev_priv->starget_priv->hdl == hdl) {
+			sdev_priv->starget_priv->tm_busy = 1;
+			skip = true;
+		}
+	}
+}
+
+void leapraid_clear_tm_flg(struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_sdev_priv *sdev_priv;
+	struct scsi_device *sdev;
+	bool skip = false;
+
+	/* Iterate over all devices. */
+	shost_for_each_device(sdev, adapter->shost) {
+		if (skip)
+			continue;
+
+		sdev_priv = sdev->hostdata;
+		if (!sdev_priv)
+			continue;
+
+		if (sdev_priv->starget_priv->hdl == hdl) {
+			sdev_priv->starget_priv->tm_busy = 0;
+			skip = true;
+		}
+	}
+}
+
+static int leapraid_tm_cmd_map_status(struct leapraid_adapter *adapter,
+				      uint channel,
+				      uint id,
+				      uint lun,
+				      u8 type,
+				      u16 taskid_task)
+{
+	int rc = FAILED;
+
+	if (taskid_task <= adapter->shost->can_queue) {
+		switch (type) {
+		case LEAPRAID_TM_TASKTYPE_ABRT_TASK_SET:
+		case LEAPRAID_TM_TASKTYPE_LOGICAL_UNIT_RESET:
+			if (!leapraid_scmd_find_by_lun(adapter, id, lun,
+						       channel))
+				rc = SUCCESS;
+			break;
+		case LEAPRAID_TM_TASKTYPE_TARGET_RESET:
+			if (!leapraid_scmd_find_by_tgt(adapter, id, channel))
+				rc = SUCCESS;
+			break;
+		default:
+			rc = SUCCESS;
+		}
+	}
+
+	if (taskid_task == adapter->driver_cmds.ctl_cmd.hp_taskid &&
+	    (adapter->driver_cmds.ctl_cmd.status &
+	     LEAPRAID_CMD_DONE ||
+	     adapter->driver_cmds.ctl_cmd.status &
+	     LEAPRAID_CMD_NOT_USED))
+		rc = SUCCESS;
+
+	return rc;
+}
+
+static int leapraid_tm_post_processing(struct leapraid_adapter *adapter,
+				       u16 hdl, uint channel, uint id,
+				       uint lun, u8 type, u16 taskid_task)
+{
+	int rc;
+
+	rc = leapraid_tm_cmd_map_status(adapter, channel, id, lun,
+					type, taskid_task);
+	if (rc == SUCCESS)
+		return rc;
+
+	leapraid_mask_int(adapter);
+	leapraid_sync_irqs(adapter, true);
+	leapraid_unmask_int(adapter);
+
+	return leapraid_tm_cmd_map_status(adapter, channel, id, lun, type,
+					  taskid_task);
+}
+
+static void leapraid_build_tm_req(struct leapraid_scsi_tm_req *scsi_tm_req,
+				  u16 hdl, uint lun, u8 type, u8 tr_method,
+				  u16 target_taskid)
+{
+	memset(scsi_tm_req, 0, sizeof(*scsi_tm_req));
+	scsi_tm_req->func = LEAPRAID_FUNC_SCSI_TMF;
+	scsi_tm_req->dev_hdl = cpu_to_le16(hdl);
+	scsi_tm_req->task_type = type;
+	scsi_tm_req->msg_flg = tr_method;
+	if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK ||
+	    type == LEAPRAID_TM_TASKTYPE_QUERY_TASK)
+		scsi_tm_req->task_mid = cpu_to_le16(target_taskid);
+	int_to_scsilun(lun, (struct scsi_lun *)scsi_tm_req->lun);
+}
+
+int leapraid_issue_tm(struct leapraid_adapter *adapter, u16 hdl, uint channel,
+		      uint id, uint lun, u8 type,
+		      u16 target_taskid, u8 tr_method)
+{
+	struct leapraid_scsi_tm_req *scsi_tm_req;
+	struct leapraid_scsiio_req *scsiio_req;
+	struct leapraid_io_req_tracker *io_req_tracker = NULL;
+	u16 msix_task;
+	u16 taskid;
+	bool issue_reset = false;
+	u32 db;
+	int rc;
+
+	lockdep_assert_held(&adapter->driver_cmds.tm_cmd.mutex);
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.host_removing ||
+	    adapter->access_ctrl.pcie_recovering) {
+		dev_info(&adapter->pdev->dev,
+			 "%s %s: Host is recovering, skip tm command!\n",
+			 __func__, adapter->adapter_attr.name);
+		return FAILED;
+	}
+
+	db = leapraid_readl(&adapter->iomem_base->db);
+	if (db & LEAPRAID_DB_USED) {
+		dev_info(&adapter->pdev->dev,
+			 "%s Unexpected db status, issuing hard reset!\n",
+			 adapter->adapter_attr.name);
+		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
+			 __func__, __LINE__);
+		rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
+		return !rc ? SUCCESS : FAILED;
+	}
+
+	if ((db & LEAPRAID_DB_MASK) == LEAPRAID_DB_FAULT) {
+		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
+			 __func__, __LINE__);
+		rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
+		return !rc ? SUCCESS : FAILED;
+	}
+
+	if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK)
+		io_req_tracker =
+			leapraid_get_io_tracker_from_taskid(adapter,
+							    target_taskid);
+
+	adapter->driver_cmds.tm_cmd.status = LEAPRAID_CMD_PENDING;
+	scsi_tm_req =
+		leapraid_get_task_desc(adapter,
+				       adapter->driver_cmds.tm_cmd.hp_taskid);
+	leapraid_build_tm_req(scsi_tm_req, hdl, lun, type, tr_method,
+			      target_taskid);
+	memset(&adapter->driver_cmds.tm_cmd.reply, 0,
+	       sizeof(struct leapraid_scsi_tm_rep));
+	leapraid_set_tm_flg(adapter, hdl);
+	init_completion(&adapter->driver_cmds.tm_cmd.done);
+	if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK &&
+	    io_req_tracker &&
+	    io_req_tracker->msix_io < adapter->adapter_attr.rq_cnt)
+		msix_task = io_req_tracker->msix_io;
+	else
+		msix_task = 0;
+	taskid = adapter->driver_cmds.tm_cmd.hp_taskid;
+	leapraid_fire_hpr_task(adapter, taskid, msix_task);
+	wait_for_completion_timeout(&adapter->driver_cmds.tm_cmd.done,
+				    LEAPRAID_TM_CMD_TIMEOUT * HZ);
+	if (!(adapter->driver_cmds.tm_cmd.status & LEAPRAID_CMD_DONE)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: TM cmd timeout, status=0x%x\n",
+			__func__, adapter->driver_cmds.tm_cmd.status);
+		leapraid_log_req_context(adapter, taskid, scsi_tm_req);
+		issue_reset =
+			leapraid_check_reset(
+				adapter->driver_cmds.tm_cmd.status);
+		if (issue_reset) {
+			dev_info(&adapter->pdev->dev,
+				 "%s:%d: call hard_reset\n",
+				 __func__, __LINE__);
+			rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
+			rc = !rc ? SUCCESS : FAILED;
+			goto out_cleanup;
+		}
+	}
+
+	leapraid_sync_irqs(adapter, false);
+
+	switch (type) {
+	case LEAPRAID_TM_TASKTYPE_TARGET_RESET:
+	case LEAPRAID_TM_TASKTYPE_ABRT_TASK_SET:
+	case LEAPRAID_TM_TASKTYPE_LOGICAL_UNIT_RESET:
+		rc = leapraid_tm_post_processing(adapter, hdl, channel, id,
+						 lun, type, target_taskid);
+		break;
+	case LEAPRAID_TM_TASKTYPE_ABORT_TASK:
+		rc = SUCCESS;
+		scsiio_req = leapraid_get_task_desc(adapter, target_taskid);
+		if (le16_to_cpu(scsiio_req->dev_hdl) != hdl)
+			break;
+		dev_err(&adapter->pdev->dev, "%s: Abort failed, hdl=0x%04x\n",
+			adapter->adapter_attr.name, hdl);
+		rc = FAILED;
+		break;
+	case LEAPRAID_TM_TASKTYPE_QUERY_TASK:
+		rc = SUCCESS;
+		break;
+	default:
+		rc = FAILED;
+		break;
+	}
+
+out_cleanup:
+	leapraid_clear_tm_flg(adapter, hdl);
+	adapter->driver_cmds.tm_cmd.status = LEAPRAID_CMD_NOT_USED;
+	return rc;
+}
+
+int leapraid_issue_locked_tm(struct leapraid_adapter *adapter, u16 hdl,
+			     uint channel, uint id, uint lun, u8 type,
+			     u16 target_taskid, u8 tr_method)
+{
+	int rc;
+
+	mutex_lock(&adapter->driver_cmds.tm_cmd.mutex);
+	rc = leapraid_issue_tm(adapter, hdl, channel, id, lun, type,
+			       target_taskid, tr_method);
+	mutex_unlock(&adapter->driver_cmds.tm_cmd.mutex);
+
+	return rc;
+}
+
+void leapraid_smart_fault_detect(struct leapraid_adapter *adapter, u16 hdl)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_sas_dev *sas_dev;
+	struct scsi_target *starget;
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_hdl(adapter, hdl);
+	if (!sas_dev) {
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		return;
+	}
+
+	starget = sas_dev->starget;
+	starget_priv = starget ? starget->hostdata : NULL;
+	if (!starget_priv) {
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		goto release_sdev;
+	}
+
+	if (starget_priv->flg & LEAPRAID_TGT_FLG_RAID_MEMBER ||
+	    starget_priv->flg & LEAPRAID_TGT_FLG_VOLUME) {
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		goto release_sdev;
+	}
+
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	leapraid_async_turn_on_led(adapter, hdl);
+release_sdev:
+	if (sas_dev)
+		leapraid_sdev_put(sas_dev);
+}
+
+static void leapraid_process_sense_data(struct leapraid_adapter *adapter,
+					struct leapraid_scsiio_rep *scsiio_rep,
+					struct scsi_cmnd *scmd, u16 taskid)
+{
+	struct sense_info data;
+	const void *sense_data;
+	u32 sz;
+
+	if (!(scsiio_rep->scsi_state & LEAPRAID_SCSI_STATE_AUTOSENSE_VALID))
+		return;
+
+	sense_data = leapraid_get_sense_buffer(adapter, taskid);
+	sz = min_t(u32, SCSI_SENSE_BUFFERSIZE,
+		   le32_to_cpu(scsiio_rep->sense_count));
+
+	memcpy(scmd->sense_buffer, sense_data, sz);
+	leapraid_get_sense_data(scmd->sense_buffer, &data);
+	if (data.asc == ASC_FAILURE_PREDICTION_THRESHOLD_EXCEEDED)
+		leapraid_smart_fault_detect(adapter,
+					    le16_to_cpu(scsiio_rep->dev_hdl));
+}
+
+static void leapraid_handle_data_underrun(
+		struct leapraid_scsiio_rep *scsiio_rep,
+		struct scsi_cmnd *scmd, u32 xfer_cnt)
+{
+	u8 scsi_status = scsiio_rep->scsi_status;
+	u8 scsi_state = scsiio_rep->scsi_state;
+
+	scmd->result = (DID_OK << LEAPRAID_SCSI_HOST_SHIFT) | scsi_status;
+
+	if (scsi_state & LEAPRAID_SCSI_STATE_AUTOSENSE_VALID)
+		return;
+
+	if (xfer_cnt < scmd->underflow) {
+		if (scsi_status == SAM_STAT_BUSY)
+			scmd->result = SAM_STAT_BUSY;
+		else
+			scmd->result = DID_SOFT_ERROR <<
+				LEAPRAID_SCSI_HOST_SHIFT;
+	} else if (scsi_state & (LEAPRAID_SCSI_STATE_AUTOSENSE_FAILED |
+				 LEAPRAID_SCSI_STATE_NO_SCSI_STATUS)) {
+		scmd->result = DID_SOFT_ERROR << LEAPRAID_SCSI_HOST_SHIFT;
+	} else if (scsi_state & LEAPRAID_SCSI_STATE_TERMINATED) {
+		scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+	} else if (!xfer_cnt && scmd->cmnd[0] == REPORT_LUNS) {
+		scsiio_rep->scsi_state = LEAPRAID_SCSI_STATE_AUTOSENSE_VALID;
+		scsiio_rep->scsi_status = SAM_STAT_CHECK_CONDITION;
+		scsi_build_sense(scmd, 0, ILLEGAL_REQUEST,
+				 LEAPRAID_SCSI_ASC_INVALID_CMD_CODE,
+				 LEAPRAID_SCSI_ASCQ_DEFAULT);
+	}
+}
+
+static void leapraid_handle_success_status(
+		struct leapraid_scsiio_rep *scsiio_rep,
+		struct scsi_cmnd *scmd,
+		u32 response_code)
+{
+	u8 scsi_status = scsiio_rep->scsi_status;
+	u8 scsi_state = scsiio_rep->scsi_state;
+
+	scmd->result = (DID_OK << LEAPRAID_SCSI_HOST_SHIFT) | scsi_status;
+
+	if (response_code == LEAPRAID_TM_RSP_INVALID_FRAME ||
+	    (scsi_state & (LEAPRAID_SCSI_STATE_AUTOSENSE_FAILED |
+			   LEAPRAID_SCSI_STATE_NO_SCSI_STATUS)))
+		scmd->result = DID_SOFT_ERROR << LEAPRAID_SCSI_HOST_SHIFT;
+	else if (scsi_state & LEAPRAID_SCSI_STATE_TERMINATED)
+		scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+}
+
+static void leapraid_scsiio_done_dispatch(
+		struct leapraid_adapter *adapter,
+		struct leapraid_scsiio_rep *scsiio_rep,
+		struct leapraid_sdev_priv *sdev_priv,
+		struct scsi_cmnd *scmd,
+		u16 taskid, u32 response_code)
+{
+	u8 scsi_status = scsiio_rep->scsi_status;
+	u8 scsi_state = scsiio_rep->scsi_state;
+	u16 adapter_status;
+	u32 xfer_cnt;
+	u32 sz;
+
+	adapter_status = le16_to_cpu(scsiio_rep->adapter_status) &
+				     LEAPRAID_ADAPTER_STATUS_MASK;
+
+	xfer_cnt = le32_to_cpu(scsiio_rep->transfer_count);
+	scsi_set_resid(scmd, scsi_bufflen(scmd) - xfer_cnt);
+
+	if (adapter_status == LEAPRAID_ADAPTER_STATUS_SCSI_DATA_UNDERRUN &&
+	    xfer_cnt == 0 &&
+	    (scsi_status == LEAPRAID_SCSI_STATUS_BUSY ||
+	    scsi_status == LEAPRAID_SCSI_STATUS_RESERVATION_CONFLICT ||
+	    scsi_status == LEAPRAID_SCSI_STATUS_TASK_SET_FULL))
+		adapter_status = LEAPRAID_ADAPTER_STATUS_SUCCESS;
+
+	switch (adapter_status) {
+	case LEAPRAID_ADAPTER_STATUS_SCSI_DEVICE_NOT_THERE:
+		scmd->result = DID_NO_CONNECT << LEAPRAID_SCSI_HOST_SHIFT;
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_BUSY:
+	case LEAPRAID_ADAPTER_STATUS_INSUFFICIENT_RESOURCES:
+		scmd->result = SAM_STAT_BUSY;
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_SCSI_RESIDUAL_MISMATCH:
+		if (xfer_cnt == 0 || scmd->underflow > xfer_cnt)
+			scmd->result = DID_SOFT_ERROR <<
+				LEAPRAID_SCSI_HOST_SHIFT;
+		else
+			scmd->result = (DID_OK << LEAPRAID_SCSI_HOST_SHIFT) |
+				scsi_status;
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_SCSI_ADAPTER_TERMINATED:
+		if (sdev_priv->block) {
+			scmd->result = DID_TRANSPORT_DISRUPTED <<
+				LEAPRAID_SCSI_HOST_SHIFT;
+			return;
+		}
+
+		if (scmd->device->channel == RAID_CHANNEL &&
+		    scsi_state == (LEAPRAID_SCSI_STATE_TERMINATED |
+				    LEAPRAID_SCSI_STATE_NO_SCSI_STATUS)) {
+			scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+			break;
+		}
+
+		scmd->result = DID_SOFT_ERROR << LEAPRAID_SCSI_HOST_SHIFT;
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_SCSI_TASK_TERMINATED:
+	case LEAPRAID_ADAPTER_STATUS_SCSI_EXT_TERMINATED:
+		scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_SCSI_DATA_UNDERRUN:
+		leapraid_handle_data_underrun(scsiio_rep, scmd, xfer_cnt);
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_SCSI_DATA_OVERRUN:
+		scsi_set_resid(scmd, 0);
+		leapraid_handle_success_status(scsiio_rep, scmd,
+					       response_code);
+		break;
+	case LEAPRAID_ADAPTER_STATUS_SCSI_RECOVERED_ERROR:
+	case LEAPRAID_ADAPTER_STATUS_SUCCESS:
+		leapraid_handle_success_status(scsiio_rep, scmd,
+					       response_code);
+		break;
+
+	case LEAPRAID_ADAPTER_STATUS_SCSI_PROTOCOL_ERROR:
+	case LEAPRAID_ADAPTER_STATUS_INTERNAL_ERROR:
+	case LEAPRAID_ADAPTER_STATUS_SCSI_IO_DATA_ERROR:
+	case LEAPRAID_ADAPTER_STATUS_SCSI_TASK_MGMT_FAILED:
+	default:
+		scmd->result = DID_SOFT_ERROR << LEAPRAID_SCSI_HOST_SHIFT;
+		break;
+	}
+
+	if (!scmd->result)
+		return;
+
+	scsi_print_command(scmd);
+	dev_warn(&adapter->pdev->dev,
+		 "SCSI I/O: hdl=0x%x, status: 0x%x, 0x%x, 0x%x\n",
+		 le16_to_cpu(scsiio_rep->dev_hdl), adapter_status,
+		 scsi_status, scsi_state);
+
+	if (scsi_state & LEAPRAID_SCSI_STATE_AUTOSENSE_VALID) {
+		struct scsi_sense_hdr sshdr;
+
+		sz = min_t(u32, SCSI_SENSE_BUFFERSIZE,
+			   le32_to_cpu(scsiio_rep->sense_count));
+		if (scsi_normalize_sense(scmd->sense_buffer, sz,
+					 &sshdr))
+			dev_warn(&adapter->pdev->dev,
+				 "Sense: key=0x%x asc=0x%x ascq=0x%x\n",
+				 sshdr.sense_key, sshdr.asc,
+				 sshdr.ascq);
+		else
+			dev_warn(&adapter->pdev->dev,
+				 "Sense: Invalid sense data\n");
+	}
+}
+
+bool leapraid_scsiio_done(struct leapraid_adapter *adapter, u16 taskid,
+			  u8 msix_index, u32 rep)
+{
+	struct leapraid_scsiio_rep *scsiio_rep;
+	struct leapraid_sdev_priv *sdev_priv;
+	struct scsi_cmnd *scmd;
+	u32 response_code = 0;
+
+	scmd = leapraid_get_scmd_from_taskid(adapter, taskid);
+	if (!scmd)
+		return true;
+
+	scsiio_rep = leapraid_get_reply_vaddr(adapter, rep);
+	if (!scsiio_rep) {
+		scmd->result = DID_OK << LEAPRAID_SCSI_HOST_SHIFT;
+		goto out_scsiio_done;
+	}
+
+	sdev_priv = scmd->device->hostdata;
+	if (!sdev_priv ||
+	    !sdev_priv->starget_priv ||
+	    sdev_priv->starget_priv->deleted) {
+		scmd->result = DID_NO_CONNECT << LEAPRAID_SCSI_HOST_SHIFT;
+		goto out_scsiio_done;
+	}
+
+	if (scsiio_rep->scsi_state & LEAPRAID_SCSI_STATE_RESPONSE_INFO_VALID)
+		response_code = le32_to_cpu(scsiio_rep->resp_info) & 0xFF;
+
+	leapraid_process_sense_data(adapter, scsiio_rep, scmd, taskid);
+	leapraid_scsiio_done_dispatch(adapter, scsiio_rep, sdev_priv, scmd,
+				      taskid, response_code);
+
+out_scsiio_done:
+	scsi_dma_unmap(scmd);
+	leapraid_free_taskid(adapter, taskid);
+	scsi_done(scmd);
+	return false;
+}
+
+static void leapraid_probe_raid(struct leapraid_adapter *adapter)
+{
+	struct leapraid_raid_volume *raid_volume, *next_raid_volume;
+	unsigned long flags;
+	LIST_HEAD(head);
+	int rc;
+
+	spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+	list_splice_init(&adapter->dev_topo.raid_volume_list, &head);
+	spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock, flags);
+
+	list_for_each_entry_safe(raid_volume, next_raid_volume, &head, list) {
+		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+		list_move_tail(&raid_volume->list,
+			       &adapter->dev_topo.raid_volume_list);
+		if (raid_volume->starget) {
+			spin_unlock_irqrestore(
+				&adapter->dev_topo.raid_volume_lock, flags);
+			continue;
+		}
+
+		leapraid_raid_volume_get(raid_volume);
+		spin_unlock_irqrestore(
+			&adapter->dev_topo.raid_volume_lock, flags);
+
+		rc = scsi_add_device(adapter->shost, RAID_CHANNEL,
+				     raid_volume->id, 0);
+		if (rc)
+			leapraid_raid_volume_remove(adapter, raid_volume);
+
+		leapraid_raid_volume_put(raid_volume);
+	}
+}
+
+static void leapraid_sas_dev_make_active(struct leapraid_adapter *adapter,
+					 struct leapraid_sas_dev *sas_dev)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	if (!list_empty(&sas_dev->list)) {
+		list_del_init(&sas_dev->list);
+		leapraid_sdev_put(sas_dev);
+	}
+
+	leapraid_sdev_get(sas_dev);
+	list_add_tail(&sas_dev->list, &adapter->dev_topo.sas_dev_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+}
+
+static void leapraid_probe_sas(struct leapraid_adapter *adapter)
+{
+	struct leapraid_sas_dev *sas_dev;
+	bool added;
+
+	for (;;) {
+		sas_dev = leapraid_get_next_sas_dev_from_init_list(adapter);
+		if (!sas_dev)
+			break;
+		added = leapraid_transport_port_add(adapter,
+						    sas_dev->hdl,
+						    sas_dev->parent_sas_addr,
+						    sas_dev->card_port);
+		if (!added)
+			goto remove_dev;
+
+		if (!sas_dev->starget &&
+		    !adapter->scan_dev_desc.driver_loading) {
+			leapraid_transport_port_remove(adapter,
+						       sas_dev->sas_addr,
+						       sas_dev->parent_sas_addr,
+						       sas_dev->card_port);
+			goto remove_dev;
+		}
+
+		leapraid_sas_dev_make_active(adapter, sas_dev);
+		leapraid_sdev_put(sas_dev);
+		continue;
+
+remove_dev:
+		leapraid_sas_dev_remove(adapter, sas_dev);
+		leapraid_sdev_put(sas_dev);
+	}
+}
+
+static bool leapraid_get_boot_dev(struct leapraid_adapter *adapter,
+				  struct leapraid_boot_dev *boot_dev,
+				  void **pdev, u32 *pchnl)
+{
+	unsigned long flags;
+	void *dev;
+	u32 chnl;
+
+	spin_lock_irqsave(&adapter->boot_devs.lock, flags);
+	if (!boot_dev->dev) {
+		spin_unlock_irqrestore(&adapter->boot_devs.lock, flags);
+		return false;
+	}
+
+	dev = boot_dev->dev;
+	chnl = boot_dev->chnl;
+	leapraid_boot_dev_get(dev, chnl);
+	spin_unlock_irqrestore(&adapter->boot_devs.lock, flags);
+
+	*pdev = dev;
+	*pchnl = chnl;
+	return true;
+}
+
+static void leapraid_probe_boot_dev(struct leapraid_adapter *adapter)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_sas_port *sport;
+	unsigned long flags;
+	void *dev = NULL;
+	u32 chnl;
+
+	if (leapraid_get_boot_dev(adapter,
+				  &adapter->boot_devs.requested_boot_dev,
+				  &dev, &chnl))
+		goto boot_dev_found;
+
+	if (leapraid_get_boot_dev(adapter,
+				  &adapter->boot_devs.requested_alt_boot_dev,
+				  &dev, &chnl))
+		goto boot_dev_found;
+
+	if (leapraid_get_boot_dev(adapter,
+				  &adapter->boot_devs.current_boot_dev,
+				  &dev, &chnl))
+		goto boot_dev_found;
+
+	return;
+
+boot_dev_found:
+	switch (chnl) {
+	case RAID_CHANNEL:
+		raid_volume = dev;
+
+		if (raid_volume->starget)
+			break;
+
+		/* TODO eedp */
+
+		if (scsi_add_device(adapter->shost, RAID_CHANNEL,
+				    raid_volume->id, 0))
+			leapraid_raid_volume_remove(adapter, raid_volume);
+		break;
+	default:
+		sas_dev = dev;
+
+		if (sas_dev->starget)
+			break;
+
+		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+		list_move_tail(&sas_dev->list,
+			       &adapter->dev_topo.sas_dev_list);
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+		if (!sas_dev->card_port)
+			break;
+
+		sport = leapraid_transport_port_add(adapter, sas_dev->hdl,
+						    sas_dev->parent_sas_addr,
+						    sas_dev->card_port);
+		if (!sport)
+			leapraid_sas_dev_remove(adapter, sas_dev);
+		break;
+	}
+
+	leapraid_boot_dev_put(dev, chnl);
+}
+
+static void leapraid_probe_devices(struct leapraid_adapter *adapter)
+{
+	leapraid_probe_boot_dev(adapter);
+
+	if (adapter->adapter_attr.raid_support) {
+		leapraid_probe_raid(adapter);
+		leapraid_probe_sas(adapter);
+	} else {
+		leapraid_probe_sas(adapter);
+	}
+}
+
+void leapraid_scan_dev_done(struct leapraid_adapter *adapter)
+{
+	if (adapter->scan_dev_desc.wait_scan_dev_done) {
+		adapter->scan_dev_desc.wait_scan_dev_done = 0;
+		leapraid_probe_devices(adapter);
+	}
+
+	adapter->scan_dev_desc.scan_start = 0;
+
+	leapraid_check_scheduled_fault_start(adapter);
+	leapraid_fw_log_start(adapter);
+	adapter->scan_dev_desc.driver_loading = 0;
+	wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+}
+
+static const struct pci_device_id leapraid_pci_table[] = {
+	{ PCI_DEVICE_SUB(LEAPRAID_VENDOR_ID, LEAPRAID_DEVID_HBA,
+			 LEAPRAID_SUBVENDOR_ID,
+			 LEAPRAID_SUBDEVID_HBA) },
+	{ 0, }
+};
+
+static inline bool leapraid_is_scmd_permitted(struct leapraid_adapter *adapter,
+					      struct scsi_cmnd *scmd)
+{
+	u8 opcode;
+
+	if (adapter->access_ctrl.pcie_recovering ||
+	    atomic_read(&adapter->overheat_desc.thermal_alert))
+		return false;
+
+	if (adapter->access_ctrl.host_removing) {
+		if (leapraid_pci_removed(adapter))
+			return false;
+
+		opcode = scmd->cmnd[0];
+		return opcode == SYNCHRONIZE_CACHE || opcode == START_STOP;
+	}
+	return true;
+}
+
+static bool leapraid_should_queuecommand(struct leapraid_adapter *adapter,
+					 struct leapraid_sdev_priv *sdev_priv,
+					 struct scsi_cmnd *scmd,
+					 enum scsi_qc_status *rc)
+{
+	struct leapraid_starget_priv *starget_priv;
+
+	if (!sdev_priv || !sdev_priv->starget_priv)
+		goto no_connect;
+
+	if (!leapraid_is_scmd_permitted(adapter, scmd))
+		goto no_connect;
+
+	starget_priv = sdev_priv->starget_priv;
+	if (starget_priv->hdl == LEAPRAID_INVALID_DEV_HANDLE)
+		goto no_connect;
+
+	if (sdev_priv->block &&
+	    scsi_get_host_state(scmd->device->host) == SHOST_RECOVERY &&
+	    scmd->cmnd[0] == TEST_UNIT_READY) {
+		scsi_build_sense(scmd, 0, UNIT_ATTENTION,
+				 LEAPRAID_SCSI_ASC_POWER_ON_RESET,
+				 LEAPRAID_SCSI_ASCQ_POWER_ON_RESET);
+		goto scsiio_done;
+	}
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->reset_desc.adapter_link_resetting) {
+		*rc = SCSI_MLQUEUE_HOST_BUSY;
+		return false;
+	}
+
+	if (starget_priv->deleted || sdev_priv->deleted)
+		goto no_connect;
+
+	if (starget_priv->tm_busy || sdev_priv->block) {
+		*rc = SCSI_MLQUEUE_DEVICE_BUSY;
+		return false;
+	}
+
+	return true;
+
+no_connect:
+	scmd->result = DID_NO_CONNECT << LEAPRAID_SCSI_HOST_SHIFT;
+scsiio_done:
+	scsi_done(scmd);
+
+	return false;
+}
+
+static u32 build_scsiio_req_control(struct scsi_cmnd *scmd,
+				    struct leapraid_sdev_priv *sdev_priv)
+{
+	u32 control;
+
+	switch (scmd->sc_data_direction) {
+	case DMA_FROM_DEVICE:
+		control = LEAPRAID_SCSIIO_CTRL_READ;
+		break;
+	case DMA_TO_DEVICE:
+		control = LEAPRAID_SCSIIO_CTRL_WRITE;
+		break;
+	default:
+		control = LEAPRAID_SCSIIO_CTRL_NODATATRANSFER;
+		break;
+	}
+
+	control |= LEAPRAID_SCSIIO_CTRL_SIMPLEQ;
+
+	if (sdev_priv->ncq_prio_enable &&
+	    (IOPRIO_PRIO_CLASS(req_get_ioprio(scsi_cmd_to_rq(scmd))) ==
+	     IOPRIO_CLASS_RT))
+		control |= LEAPRAID_SCSIIO_CTRL_CMDPRI;
+	if (scmd->cmd_len == 32)
+		control |= LEAPRAID_SCSIIO_CTRL_CDB_32BYTE <<
+				LEAPRAID_SCSIIO_CTRL_CDB_LEN_SHIFT;
+
+	return control;
+}
+
+enum scsi_qc_status leapraid_queuecommand(struct Scsi_Host *shost,
+					  struct scsi_cmnd *scmd)
+{
+	struct leapraid_adapter *adapter = shost_priv(scmd->device->host);
+	struct leapraid_sdev_priv *sdev_priv = scmd->device->hostdata;
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_scsiio_req *scsiio_req;
+	u32 control;
+	u16 taskid;
+	u16 hdl;
+	enum scsi_qc_status rc = 0;
+
+	if (!leapraid_should_queuecommand(adapter, sdev_priv, scmd, &rc))
+		return rc;
+
+	starget_priv = sdev_priv->starget_priv;
+	hdl = starget_priv->hdl;
+	control = build_scsiio_req_control(scmd, sdev_priv);
+
+	taskid = leapraid_alloc_scsiio_taskid(adapter, scmd);
+	scsiio_req = leapraid_get_task_desc(adapter, taskid);
+
+	scsiio_req->func = LEAPRAID_FUNC_SCSIIO;
+	if (sdev_priv->starget_priv->flg & LEAPRAID_TGT_FLG_RAID_MEMBER)
+		scsiio_req->func = LEAPRAID_FUNC_SCSIIO_RAID_PASSTHROUGH;
+	else
+		scsiio_req->func = LEAPRAID_FUNC_SCSIIO;
+
+	scsiio_req->dev_hdl = cpu_to_le16(hdl);
+	scsiio_req->data_len = cpu_to_le32(scsi_bufflen(scmd));
+	scsiio_req->ctrl = cpu_to_le32(control);
+	scsiio_req->io_flg = cpu_to_le16(scmd->cmd_len);
+	scsiio_req->msg_flg = 0;
+	scsiio_req->sense_buffer_len = SCSI_SENSE_BUFFERSIZE;
+	scsiio_req->sense_buffer_low_add =
+		leapraid_get_sense_buffer_dma(adapter, taskid);
+	scsiio_req->sgl_offset0 =
+		offsetof(struct leapraid_scsiio_req, sgl) /
+		LEAPRAID_DWORDS_BYTE_SIZE;
+	int_to_scsilun(sdev_priv->lun, (struct scsi_lun *)scsiio_req->lun);
+	memcpy(scsiio_req->cdb.cdb32, scmd->cmnd, scmd->cmd_len);
+	if (scsiio_req->data_len) {
+		if (leapraid_build_scmd_ieee_sg(adapter, scmd, taskid)) {
+			leapraid_free_taskid(adapter, taskid);
+			return SCSI_MLQUEUE_HOST_BUSY;
+		}
+	} else {
+		leapraid_build_ieee_nodata_sg(adapter, &scsiio_req->sgl);
+	}
+
+	if (likely(scsiio_req->func == LEAPRAID_FUNC_SCSIIO))
+		leapraid_fire_scsi_io(adapter, taskid,
+				      le16_to_cpu(scsiio_req->dev_hdl));
+	else
+		leapraid_fire_task(adapter, taskid);
+
+	dev_dbg(&adapter->pdev->dev,
+		"LEAPRAID_SCSIIO: Send Descriptor taskid %d, req type 0x%x\n",
+		taskid, scsiio_req->func);
+	return rc;
+}
+
+static int leapraid_init_cmd_priv(struct Scsi_Host *shost,
+				  struct scsi_cmnd *scmd)
+{
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	struct leapraid_io_req_tracker *io_tracker;
+
+	io_tracker = scsi_cmd_priv(scmd);
+	leapraid_internal_init_cmd_priv(adapter, io_tracker);
+
+	return 0;
+}
+
+static int leapraid_exit_cmd_priv(struct Scsi_Host *shost,
+				  struct scsi_cmnd *scmd)
+{
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	struct leapraid_io_req_tracker *io_tracker;
+
+	io_tracker = scsi_cmd_priv(scmd);
+	leapraid_internal_exit_cmd_priv(adapter, io_tracker);
+
+	return 0;
+}
+
+static int leapraid_error_handler(struct scsi_cmnd *scmd,
+				  const char *str, u8 type)
+{
+	struct leapraid_adapter *adapter = shost_priv(scmd->device->host);
+	struct scsi_target *starget = scmd->device->sdev_target;
+	struct leapraid_starget_priv *starget_priv = starget->hostdata;
+	struct leapraid_io_req_tracker *io_req_tracker = NULL;
+	struct leapraid_sdev_priv *sdev_priv;
+	struct leapraid_sas_dev *sas_dev = NULL;
+	u16 hdl;
+	int rc;
+
+	dev_info(&adapter->pdev->dev,
+		 "EH enter: type=%s, scmd=0x%p, req tag=%d\n", str, scmd,
+		 scsi_cmd_to_rq(scmd)->tag);
+	scsi_print_command(scmd);
+
+	if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK) {
+		io_req_tracker = scsi_cmd_priv(scmd);
+		dev_info(&adapter->pdev->dev,
+			 "EH ABORT: scmd=0x%p, pend=%ums, tout=%ums, tag=%d\n",
+			 scmd,
+			 jiffies_to_msecs(jiffies - scmd->jiffies_at_alloc),
+			 (scsi_cmd_to_rq(scmd)->timeout / HZ) * 1000,
+			 scsi_cmd_to_rq(scmd)->tag);
+	}
+
+	if (leapraid_pci_removed(adapter) ||
+	    adapter->access_ctrl.host_removing) {
+		dev_err(&adapter->pdev->dev,
+			"EH %s failed: %s scmd=0x%p\n", str,
+			(adapter->access_ctrl.host_removing ?
+			"shost removing!" : "pci_dev removed!"), scmd);
+		if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK &&
+		    io_req_tracker && io_req_tracker->taskid)
+			leapraid_free_taskid(adapter, io_req_tracker->taskid);
+		scmd->result = DID_NO_CONNECT << LEAPRAID_SCSI_HOST_SHIFT;
+#ifdef FAST_IO_FAIL
+		rc = FAST_IO_FAIL;
+#else
+		rc = FAILED;
+#endif
+		goto out_eh_done;
+	}
+
+	sdev_priv = scmd->device->hostdata;
+	if (!sdev_priv || !sdev_priv->starget_priv) {
+		dev_warn(&adapter->pdev->dev,
+			 "EH %s: SAS dev or starget gone, scmd=0x%p\n",
+			 str, scmd);
+		scmd->result = DID_NO_CONNECT << LEAPRAID_SCSI_HOST_SHIFT;
+		scsi_done(scmd);
+		rc = SUCCESS;
+		goto out_eh_done;
+	}
+
+	if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK) {
+		if (!io_req_tracker) {
+			dev_warn(&adapter->pdev->dev,
+				 "EH ABORT: No I/O tracker, scmd 0x%p\n", scmd);
+			scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+			rc = SUCCESS;
+			goto out_eh_done;
+		}
+
+		if (sdev_priv->starget_priv->flg &
+			LEAPRAID_TGT_FLG_RAID_MEMBER ||
+		    sdev_priv->starget_priv->flg & LEAPRAID_TGT_FLG_VOLUME) {
+			dev_err(&adapter->pdev->dev,
+				"EH ABORT: Skip RAID/VOLUME, scmd=0x%p\n",
+				scmd);
+			scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+			rc = FAILED;
+			goto out_eh_done;
+		}
+
+		hdl = sdev_priv->starget_priv->hdl;
+	} else {
+		hdl = 0;
+		if (sdev_priv->starget_priv->flg &
+		    LEAPRAID_TGT_FLG_RAID_MEMBER) {
+			sas_dev = leapraid_get_sas_dev_from_tgt(adapter,
+								starget_priv);
+			if (sas_dev)
+				hdl = sas_dev->volume_hdl;
+		} else {
+			hdl = sdev_priv->starget_priv->hdl;
+		}
+
+		if (!hdl) {
+			dev_err(&adapter->pdev->dev,
+				"EH %s failed: Target handle 0, scmd=0x%p\n",
+				str, scmd);
+			scmd->result = DID_RESET << LEAPRAID_SCSI_HOST_SHIFT;
+			rc = FAILED;
+			goto out_eh_done;
+		}
+	}
+
+	dev_info(&adapter->pdev->dev,
+		 "EH issue TM: type=%s, scmd=0x%p, hdl=0x%x\n",
+		 str, scmd, hdl);
+
+	rc = leapraid_issue_locked_tm(
+			adapter,
+			hdl,
+			scmd->device->channel,
+			scmd->device->id,
+			(type == LEAPRAID_TM_TASKTYPE_TARGET_RESET ?
+				0 : scmd->device->lun),
+			type,
+			(type == LEAPRAID_TM_TASKTYPE_ABORT_TASK ?
+				io_req_tracker->taskid : 0),
+			LEAPRAID_TM_MSGFLAGS_LINK_RESET);
+
+out_eh_done:
+	if (type == LEAPRAID_TM_TASKTYPE_ABORT_TASK) {
+		if (rc != SUCCESS)
+			dev_err(&adapter->pdev->dev,
+				"EH ABORT result: failed, scmd=0x%p\n",
+				scmd);
+	} else {
+		if (rc != SUCCESS)
+			dev_err(&adapter->pdev->dev,
+				"EH %s result: failed, scmd=0x%p\n",
+				str, scmd);
+		if (sas_dev)
+			leapraid_sdev_put(sas_dev);
+	}
+	return rc;
+}
+
+static int leapraid_eh_abort_handler(struct scsi_cmnd *scmd)
+{
+	return leapraid_error_handler(scmd, "ABORT TASK",
+				      LEAPRAID_TM_TASKTYPE_ABORT_TASK);
+}
+
+static int leapraid_eh_device_reset_handler(struct scsi_cmnd *scmd)
+{
+	return leapraid_error_handler(scmd, "UNIT RESET",
+				      LEAPRAID_TM_TASKTYPE_LOGICAL_UNIT_RESET);
+}
+
+static int leapraid_eh_target_reset_handler(struct scsi_cmnd *scmd)
+{
+	return leapraid_error_handler(scmd, "TARGET RESET",
+				      LEAPRAID_TM_TASKTYPE_TARGET_RESET);
+}
+
+static int leapraid_eh_host_reset_handler(struct scsi_cmnd *scmd)
+{
+	struct leapraid_adapter *adapter = shost_priv(scmd->device->host);
+	int rc;
+
+	dev_info(&adapter->pdev->dev,
+		 "EH HOST RESET enter: scmd=%p, req tag=%d\n",
+		 scmd,
+		 scsi_cmd_to_rq(scmd)->tag);
+	scsi_print_command(scmd);
+
+	if (adapter->scan_dev_desc.driver_loading ||
+	    adapter->access_ctrl.host_removing) {
+		dev_err(&adapter->pdev->dev,
+			"EH HOST RESET failed: %s scmd=0x%p\n",
+			(adapter->access_ctrl.host_removing ?
+			"shost removing!" : "driver loading!"), scmd);
+		rc = FAILED;
+		goto out_host_reset_done;
+	}
+
+	dev_info(&adapter->pdev->dev, "%s:%d: Issuing hard reset\n",
+		 __func__, __LINE__);
+	if (leapraid_hard_reset_handler(adapter, FULL_RESET) < 0)
+		rc = FAILED;
+	else
+		rc = SUCCESS;
+
+out_host_reset_done:
+	if (rc != SUCCESS)
+		dev_err(&adapter->pdev->dev,
+			"EH HOST RESET result: failed, scmd=0x%p\n",
+			scmd);
+	return rc;
+}
+
+static int leapraid_sdev_init(struct scsi_device *sdev)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_starget_priv *stgt_priv;
+	struct leapraid_sdev_priv *sdev_priv;
+	struct leapraid_adapter *adapter;
+	struct leapraid_sas_dev *sas_dev;
+	struct scsi_target *tgt;
+	struct Scsi_Host *shost;
+	unsigned long flags;
+
+	sdev_priv = kzalloc_obj(*sdev_priv);
+	if (!sdev_priv)
+		return -ENOMEM;
+
+	sdev_priv->lun = sdev->lun;
+	sdev_priv->flg = LEAPRAID_DEVICE_FLG_INIT;
+	tgt = scsi_target(sdev);
+	stgt_priv = tgt->hostdata;
+	stgt_priv->num_luns++;
+	sdev_priv->starget_priv = stgt_priv;
+	sdev->hostdata = sdev_priv;
+	if (stgt_priv->flg & LEAPRAID_TGT_FLG_RAID_MEMBER)
+		sdev->no_uld_attach = LEAPRAID_NO_ULD_ATTACH;
+
+	shost = dev_to_shost(&tgt->dev);
+	adapter = shost_priv(shost);
+	if (tgt->channel == RAID_CHANNEL) {
+		raid_volume = leapraid_raid_volume_find_by_id(adapter,
+							      tgt->id,
+							      tgt->channel);
+		if (raid_volume) {
+			spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock,
+					  flags);
+			raid_volume->sdev = sdev;
+			spin_unlock_irqrestore(
+				&adapter->dev_topo.raid_volume_lock, flags);
+			leapraid_raid_volume_put(raid_volume);
+		}
+	}
+
+	if (!(stgt_priv->flg & LEAPRAID_TGT_FLG_VOLUME)) {
+		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+		sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(
+				adapter,
+				stgt_priv->sas_address,
+				stgt_priv->card_port);
+		if (sas_dev && !sas_dev->starget) {
+			sdev_printk(KERN_INFO, sdev,
+				    "%s: Assign starget to sas_dev\n",
+				    __func__);
+			sas_dev->starget = tgt;
+		}
+
+		if (sas_dev)
+			leapraid_sdev_put(sas_dev);
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	}
+	return 0;
+}
+
+static bool leapraid_slave_cfg_volume(struct scsi_device *sdev,
+				      struct queue_limits *lim)
+{
+	struct Scsi_Host *shost = sdev->host;
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_sdev_priv *sdev_priv;
+	int qd;
+	u16 hdl;
+
+	sdev_priv = sdev->hostdata;
+	starget_priv = sdev_priv->starget_priv;
+	hdl = starget_priv->hdl;
+
+	raid_volume = leapraid_raid_volume_find_by_hdl(adapter, hdl);
+	if (!raid_volume) {
+		sdev_printk(KERN_WARNING, sdev,
+			    "%s: RAID volume not found, hdl=0x%x\n",
+			    __func__, hdl);
+		return 1;
+	}
+
+	if (leapraid_get_volume_cap(adapter, raid_volume)) {
+		sdev_printk(KERN_ERR, sdev,
+			    "%s: Failed to get volume cap, hdl=0x%x\n",
+			    __func__, hdl);
+		leapraid_raid_volume_put(raid_volume);
+		return 1;
+	}
+
+	qd = (raid_volume->dev_info & LEAPRAID_DEVTYP_SSP_TGT) ?
+		adapter->adapter_attr.narrowport_max_queue_depth :
+		adapter->adapter_attr.sata_max_queue_depth;
+	if (raid_volume->vol_type != LEAPRAID_VOL_TYPE_RAID0)
+		qd = adapter->adapter_attr.raid_volume_max_queue_depth;
+
+	sdev_printk(KERN_INFO, sdev,
+		    "RAID volume: hdl=0x%04x, wwid=0x%016llx\n",
+		    raid_volume->hdl, (unsigned long long)raid_volume->wwid);
+
+	if (shost->max_sectors > LEAPRAID_MAX_SECTORS)
+		lim->max_hw_sectors = LEAPRAID_MAX_SECTORS;
+
+	leapraid_change_queue_depth(sdev, qd);
+	leapraid_raid_volume_put(raid_volume);
+	return 0;
+}
+
+static bool leapraid_slave_configure_extra(struct scsi_device *sdev,
+					   struct leapraid_sas_dev **psas_dev,
+					   u16 vol_hdl, u64 volume_wwid,
+					   bool *is_target_ssp, int *qd)
+{
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_sdev_priv *sdev_priv;
+	struct Scsi_Host *shost = sdev->host;
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	unsigned long flags;
+
+	sdev_priv = sdev->hostdata;
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	*is_target_ssp = false;
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr(
+			adapter,
+			sdev_priv->starget_priv->sas_address,
+			sdev_priv->starget_priv->card_port);
+	if (!sas_dev) {
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+		sdev_printk(KERN_WARNING, sdev,
+			    "%s: SAS dev not found, sas=0x%llx\n",
+			    __func__, sdev_priv->starget_priv->sas_address);
+		return 1;
+	}
+
+	*psas_dev = sas_dev;
+	sas_dev->volume_hdl = vol_hdl;
+	sas_dev->volume_wwid = volume_wwid;
+	if (sas_dev->dev_info & LEAPRAID_DEVTYP_SSP_TGT) {
+		*qd = (sas_dev->port_connection > 1) ?
+			adapter->adapter_attr.wideport_max_queue_depth :
+			adapter->adapter_attr.narrowport_max_queue_depth;
+		*is_target_ssp = true;
+		if (sas_dev->dev_info & LEAPRAID_DEVTYP_SEP)
+			sdev_priv->sep = 1;
+	} else {
+		*qd = adapter->adapter_attr.sata_max_queue_depth;
+	}
+
+	sdev_printk(KERN_INFO, sdev,
+		    "device name=0x%016llx, SAS addr=0x%016llx\n",
+		    (unsigned long long)sas_dev->dev_name,
+		    (unsigned long long)sas_dev->sas_addr);
+	leapraid_sdev_put(sas_dev);
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	return 0;
+}
+
+static int leapraid_sdev_configure(struct scsi_device *sdev,
+				   struct queue_limits *lim)
+{
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_sdev_priv *sdev_priv;
+	struct Scsi_Host *shost = sdev->host;
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_adapter *adapter;
+	u16 hdl, vol_hdl = 0;
+	bool is_target_ssp = false;
+	u64 volume_wwid = 0;
+	int qd = 1;
+
+	adapter = shost_priv(shost);
+	sdev_priv = sdev->hostdata;
+	sdev_priv->flg &= ~LEAPRAID_DEVICE_FLG_INIT;
+	starget_priv = sdev_priv->starget_priv;
+	hdl = starget_priv->hdl;
+	if (starget_priv->flg & LEAPRAID_TGT_FLG_VOLUME)
+		return leapraid_slave_cfg_volume(sdev, lim);
+
+	if (starget_priv->flg & LEAPRAID_TGT_FLG_RAID_MEMBER) {
+		if (leapraid_cfg_get_volume_hdl(adapter, hdl, &vol_hdl)) {
+			sdev_printk(KERN_WARNING, sdev,
+				    "%s: Get volume hdl failed, hdl=0x%x\n",
+				    __func__, hdl);
+			return 1;
+		}
+
+		if (vol_hdl && leapraid_cfg_get_volume_wwid(adapter, vol_hdl,
+							    &volume_wwid)) {
+			sdev_printk(KERN_WARNING, sdev,
+				    "%s: Get wwid failed, volume_hdl=0x%x\n",
+				    __func__, vol_hdl);
+			return 1;
+		}
+	}
+
+	if (leapraid_slave_configure_extra(sdev, &sas_dev, vol_hdl,
+					   volume_wwid, &is_target_ssp, &qd)) {
+		sdev_printk(KERN_WARNING, sdev,
+			    "%s: slave_configure_extra failed\n", __func__);
+		return 1;
+	}
+
+	leapraid_change_queue_depth(sdev, qd);
+	if (is_target_ssp)
+		sas_read_port_mode_page(sdev);
+
+	return 0;
+}
+
+static void leapraid_sdev_destroy(struct scsi_device *sdev)
+{
+	struct leapraid_adapter *adapter;
+	struct Scsi_Host *shost;
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_starget_priv *starget_priv;
+	struct scsi_target *stgt;
+	unsigned long flags;
+
+	if (!sdev->hostdata)
+		return;
+
+	stgt = scsi_target(sdev);
+	starget_priv = stgt->hostdata;
+	starget_priv->num_luns--;
+	shost = dev_to_shost(&stgt->dev);
+	adapter = shost_priv(shost);
+	if (!(starget_priv->flg & LEAPRAID_TGT_FLG_VOLUME)) {
+		spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+		sas_dev = leapraid_hold_lock_get_sas_dev_from_tgt(adapter,
+								  starget_priv);
+		if (sas_dev && !starget_priv->num_luns)
+			sas_dev->starget = NULL;
+		if (sas_dev)
+			leapraid_sdev_put(sas_dev);
+		spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+	}
+
+	kfree(sdev->hostdata);
+	sdev->hostdata = NULL;
+}
+
+static int leapraid_target_alloc_raid(struct scsi_target *tgt)
+{
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_raid_volume *raid_volume;
+	struct Scsi_Host *shost = dev_to_shost(&tgt->dev);
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	unsigned long flags;
+
+	starget_priv = tgt->hostdata;
+	raid_volume = leapraid_raid_volume_find_by_id(adapter, tgt->id,
+						      tgt->channel);
+	if (raid_volume) {
+		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+		starget_priv->hdl = raid_volume->hdl;
+		starget_priv->sas_address = raid_volume->wwid;
+		starget_priv->flg |= LEAPRAID_TGT_FLG_VOLUME;
+		raid_volume->starget = tgt;
+		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
+				       flags);
+		leapraid_raid_volume_put(raid_volume);
+	}
+	return 0;
+}
+
+static int leapraid_target_alloc_sas(struct scsi_target *tgt)
+{
+	struct sas_rphy *rphy;
+	struct Scsi_Host *shost;
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_adapter *adapter;
+	struct leapraid_starget_priv *starget_priv;
+	unsigned long flags;
+
+	shost = dev_to_shost(&tgt->dev);
+	adapter = shost_priv(shost);
+	starget_priv = tgt->hostdata;
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	rphy = dev_to_rphy(tgt->dev.parent);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr_and_rphy(
+			adapter,
+			rphy->identify.sas_address,
+			rphy);
+	if (sas_dev) {
+		starget_priv->sas_dev = sas_dev;
+		starget_priv->card_port = sas_dev->card_port;
+		starget_priv->sas_address = sas_dev->sas_addr;
+		starget_priv->hdl = sas_dev->hdl;
+		sas_dev->channel = tgt->channel;
+		sas_dev->id = tgt->id;
+		sas_dev->starget = tgt;
+		if (sas_dev->hdl &&
+		    sas_dev->hdl <=
+		    adapter->adapter_attr.features.max_dev_handle &&
+		    test_bit(sas_dev->hdl, adapter->dev_topo.pd_hdls))
+			starget_priv->flg |= LEAPRAID_TGT_FLG_RAID_MEMBER;
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	return 0;
+}
+
+static int leapraid_target_alloc(struct scsi_target *tgt)
+{
+	struct leapraid_starget_priv *starget_priv;
+
+	starget_priv = kzalloc_obj(*starget_priv);
+	if (!starget_priv)
+		return -ENOMEM;
+
+	tgt->hostdata = starget_priv;
+	starget_priv->starget = tgt;
+	starget_priv->hdl = LEAPRAID_INVALID_DEV_HANDLE;
+	if (tgt->channel == RAID_CHANNEL)
+		return leapraid_target_alloc_raid(tgt);
+
+	return leapraid_target_alloc_sas(tgt);
+}
+
+static void leapraid_target_destroy_raid(struct scsi_target *tgt)
+{
+	struct leapraid_raid_volume *raid_volume;
+	struct Scsi_Host *shost = dev_to_shost(&tgt->dev);
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	unsigned long flags;
+
+	raid_volume = leapraid_raid_volume_find_by_id(adapter, tgt->id,
+						      tgt->channel);
+	if (raid_volume) {
+		spin_lock_irqsave(&adapter->dev_topo.raid_volume_lock, flags);
+		raid_volume->starget = NULL;
+		raid_volume->sdev = NULL;
+		spin_unlock_irqrestore(&adapter->dev_topo.raid_volume_lock,
+				       flags);
+		leapraid_raid_volume_put(raid_volume);
+	}
+}
+
+static void leapraid_target_destroy_sas(struct scsi_target *tgt)
+{
+	struct leapraid_adapter *adapter;
+	struct leapraid_sas_dev *sas_dev;
+	struct leapraid_starget_priv *starget_priv;
+	struct Scsi_Host *shost;
+	unsigned long flags;
+
+	shost = dev_to_shost(&tgt->dev);
+	adapter = shost_priv(shost);
+	starget_priv = tgt->hostdata;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_from_tgt(adapter,
+							  starget_priv);
+	if (sas_dev &&
+	    sas_dev->starget == tgt &&
+	    sas_dev->id == tgt->id &&
+	    sas_dev->channel == tgt->channel)
+		sas_dev->starget = NULL;
+
+	if (sas_dev) {
+		starget_priv->sas_dev = NULL;
+		leapraid_sdev_put(sas_dev);
+		leapraid_sdev_put(sas_dev);
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+}
+
+static void leapraid_target_destroy(struct scsi_target *tgt)
+{
+	struct leapraid_starget_priv *starget_priv;
+
+	starget_priv = tgt->hostdata;
+	if (!starget_priv)
+		return;
+
+	if (tgt->channel == RAID_CHANNEL) {
+		leapraid_target_destroy_raid(tgt);
+		goto out_free;
+	}
+
+	leapraid_target_destroy_sas(tgt);
+
+out_free:
+	kfree(starget_priv);
+	tgt->hostdata = NULL;
+}
+
+static bool leapraid_scan_check_status(struct leapraid_adapter *adapter,
+				       bool *need_hard_reset)
+{
+	u32 adapter_state;
+
+	if (adapter->scan_dev_desc.scan_start) {
+		adapter_state = leapraid_get_adapter_state(adapter);
+		if (adapter_state == LEAPRAID_DB_FAULT) {
+			*need_hard_reset = true;
+			return true;
+		}
+		return false;
+	}
+
+	if (adapter->driver_cmds.scan_dev_cmd.status & LEAPRAID_CMD_RESET) {
+		dev_err(&adapter->pdev->dev,
+			"Device scan: Aborted due to reset\n");
+		adapter->driver_cmds.scan_dev_cmd.status =
+			LEAPRAID_CMD_NOT_USED;
+		adapter->scan_dev_desc.driver_loading = 0;
+		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+		return true;
+	}
+
+	if (adapter->scan_dev_desc.scan_start_failed) {
+		dev_err(&adapter->pdev->dev,
+			"Device scan: Failed with adapter_status=0x%08x\n",
+			adapter->scan_dev_desc.scan_start_failed);
+		adapter->scan_dev_desc.driver_loading = 0;
+		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+		adapter->scan_dev_desc.wait_scan_dev_done = 0;
+		adapter->access_ctrl.host_removing = 1;
+		return true;
+	}
+
+	adapter->driver_cmds.scan_dev_cmd.status = LEAPRAID_CMD_NOT_USED;
+	leapraid_scan_dev_done(adapter);
+	return true;
+}
+
+static int leapraid_scan_finished(struct Scsi_Host *shost, unsigned long time)
+{
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	bool need_hard_reset = false;
+
+	if (time >= (LEAPRAID_SCAN_DEV_CMD_TIMEOUT * HZ)) {
+		adapter->driver_cmds.scan_dev_cmd.status =
+			LEAPRAID_CMD_NOT_USED;
+		dev_err(&adapter->pdev->dev,
+			"Device scan: Failed with timeout 300s\n");
+		adapter->scan_dev_desc.driver_loading = 0;
+		wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+		return 1;
+	}
+
+	if (!leapraid_scan_check_status(adapter, &need_hard_reset))
+		return 0;
+
+	if (need_hard_reset) {
+		adapter->driver_cmds.scan_dev_cmd.status =
+			LEAPRAID_CMD_NOT_USED;
+		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
+			 __func__, __LINE__);
+		if (leapraid_hard_reset_handler(adapter, PART_RESET)) {
+			adapter->scan_dev_desc.driver_loading = 0;
+			wake_up(&adapter->scan_dev_desc.wait_driver_loading);
+		}
+	}
+
+	return 1;
+}
+
+static void leapraid_scan_start(struct Scsi_Host *shost)
+{
+	struct leapraid_adapter *adapter = shost_priv(shost);
+
+	adapter->scan_dev_desc.scan_start = 1;
+	leapraid_scan_dev(adapter, true);
+}
+
+static u32 leapraid_get_raid_qd(struct leapraid_adapter *adapter,
+				const struct leapraid_raid_volume *raid_volume,
+				bool default_qd)
+{
+	const struct leapraid_adapter_attr *attr = &adapter->adapter_attr;
+
+	if (raid_volume->vol_type != LEAPRAID_VOL_TYPE_RAID0)
+		return default_qd ? LEAPRAID_RAID_QUEUE_DEPTH :
+			attr->raid_volume_max_queue_depth;
+
+	if (raid_volume->dev_info & LEAPRAID_DEVTYP_SSP_TGT)
+		return default_qd ? LEAPRAID_SAS_QUEUE_DEPTH :
+			attr->narrowport_max_queue_depth;
+
+	return default_qd ? LEAPRAID_SATA_QUEUE_DEPTH :
+		attr->sata_max_queue_depth;
+}
+
+static int leapraid_calc_max_queue_depth(struct scsi_device *sdev, int qdepth)
+{
+	struct Scsi_Host *shost;
+	struct leapraid_adapter *adapter;
+	struct leapraid_starget_priv *starget_priv;
+	struct leapraid_sdev_priv *sdev_priv;
+	struct leapraid_raid_volume *raid_volume;
+	struct leapraid_sas_dev *sas_dev;
+	int max_depth;
+	u32 default_qdepth = 0;
+	u32 fw_qdepth = 0;
+
+	shost = sdev->host;
+	adapter = shost_priv(shost);
+	max_depth = shost->can_queue;
+
+	sdev_priv = sdev->hostdata;
+	if (!sdev_priv)
+		goto out_tag_check;
+
+	starget_priv = sdev_priv->starget_priv;
+	if (!starget_priv)
+		goto out_tag_check;
+
+	if (starget_priv->flg & LEAPRAID_TGT_FLG_VOLUME) {
+		raid_volume = leapraid_raid_volume_find_by_hdl(
+					adapter, starget_priv->hdl);
+		if (raid_volume) {
+			default_qdepth = leapraid_get_raid_qd(
+					adapter, raid_volume, true);
+			fw_qdepth = leapraid_get_raid_qd(
+					adapter, raid_volume, false);
+			leapraid_raid_volume_put(raid_volume);
+		}
+		goto out_limit_check;
+	}
+
+	sas_dev = leapraid_get_sas_dev_from_tgt(adapter, starget_priv);
+	if (sas_dev) {
+		if (sas_dev->dev_info & LEAPRAID_DEVTYP_SSP_TGT) {
+			default_qdepth = LEAPRAID_SAS_QUEUE_DEPTH;
+			fw_qdepth = (sas_dev->port_connection > 1) ?
+			      adapter->adapter_attr.wideport_max_queue_depth :
+			      adapter->adapter_attr.narrowport_max_queue_depth;
+		}
+		if (sas_dev->dev_info & LEAPRAID_DEVTYP_SATA_DEV) {
+			default_qdepth = LEAPRAID_SATA_QUEUE_DEPTH;
+			fw_qdepth = adapter->adapter_attr.sata_max_queue_depth;
+		}
+		leapraid_sdev_put(sas_dev);
+	}
+
+out_limit_check:
+	if (fw_qdepth > shost->can_queue && default_qdepth)
+		fw_qdepth = default_qdepth;
+
+	if (fw_qdepth)
+		max_depth = min_t(int, max_depth, fw_qdepth);
+
+out_tag_check:
+	if (!sdev->tagged_supported)
+		max_depth = 1;
+
+	if (qdepth > max_depth)
+		qdepth = max_depth;
+
+	return qdepth;
+}
+
+int leapraid_change_queue_depth(struct scsi_device *sdev, int qdepth)
+{
+	qdepth = leapraid_calc_max_queue_depth(sdev, qdepth);
+	scsi_change_queue_depth(sdev, qdepth);
+	return sdev->queue_depth;
+}
+
+static void leapraid_map_queues(struct Scsi_Host *shost)
+{
+	struct leapraid_adapter *adapter;
+	struct blk_mq_queue_map *queue_map;
+	int msix_queue_count;
+	int poll_queue_count;
+	int queue_offset;
+	int map_index;
+
+	adapter = (struct leapraid_adapter *)shost->hostdata;
+	if (shost->nr_hw_queues == 1)
+		return;
+
+	msix_queue_count = adapter->notification_desc.iopoll_qdex;
+	poll_queue_count = adapter->adapter_attr.rq_cnt - msix_queue_count;
+
+	queue_offset = 0;
+	for (map_index = 0; map_index < shost->nr_maps; map_index++) {
+		queue_map = &shost->tag_set.map[map_index];
+		queue_map->nr_queues = 0;
+
+		switch (map_index) {
+		case HCTX_TYPE_DEFAULT:
+			queue_map->nr_queues = msix_queue_count;
+			queue_map->queue_offset = queue_offset;
+			WARN_ON_ONCE(!queue_map->nr_queues);
+			blk_mq_map_hw_queues(queue_map,
+					     &adapter->pdev->dev, 0);
+			break;
+		case HCTX_TYPE_POLL:
+			queue_map->nr_queues = poll_queue_count;
+			queue_map->queue_offset = queue_offset;
+			blk_mq_map_queues(queue_map);
+			break;
+		default:
+			queue_map->queue_offset = queue_offset;
+			blk_mq_map_hw_queues(queue_map,
+					     &adapter->pdev->dev, 0);
+			break;
+		}
+		queue_offset += queue_map->nr_queues;
+	}
+}
+
+int leapraid_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
+{
+	struct leapraid_adapter *adapter =
+		(struct leapraid_adapter *)shost->hostdata;
+	struct leapraid_blk_mq_poll_rq *blk_mq_poll_rq;
+	int num_entries;
+	int qid = queue_num - adapter->notification_desc.iopoll_qdex;
+
+	blk_mq_poll_rq = &adapter->notification_desc.blk_mq_poll_rqs[qid];
+	if (atomic_read(&blk_mq_poll_rq->pause) ||
+	    !atomic_add_unless(&blk_mq_poll_rq->busy, 1, 1))
+		return 0;
+
+	num_entries = leapraid_rep_queue_handler(&blk_mq_poll_rq->rq);
+	atomic_dec(&blk_mq_poll_rq->busy);
+	return num_entries;
+}
+
+static int leapraid_bios_param(struct scsi_device *sdev,
+			       struct gendisk *disk, sector_t capacity,
+			       int geom[])
+{
+	int heads;
+	int sectors;
+	sector_t cylinders;
+
+	if (scsi_partsize(disk, capacity, geom))
+		return 0;
+
+	if ((ulong)capacity >= LEAPRAID_LARGE_DISK_THRESHOLD) {
+		heads = LEAPRAID_LARGE_DISK_HEADS;
+		sectors = LEAPRAID_LARGE_DISK_SECTORS;
+	} else {
+		heads = LEAPRAID_SMALL_DISK_HEADS;
+		sectors = LEAPRAID_SMALL_DISK_SECTORS;
+	}
+
+	cylinders = capacity;
+	sector_div(cylinders, heads * sectors);
+
+	geom[0] = heads;
+	geom[1] = sectors;
+	geom[2] = cylinders;
+	return 0;
+}
+
+static ssize_t fw_queue_depth_show(struct device *cdev,
+				   struct device_attribute *attr,
+				   char *buf)
+{
+	struct Scsi_Host *shost = class_to_shost(cdev);
+	struct leapraid_adapter *adapter = shost_priv(shost);
+
+	return sysfs_emit(buf, "%02d\n",
+			  adapter->adapter_attr.features.req_slot);
+}
+
+static ssize_t host_sas_address_show(struct device *cdev,
+				     struct device_attribute *attr, char *buf)
+{
+	struct Scsi_Host *shost = class_to_shost(cdev);
+	struct leapraid_adapter *adapter = shost_priv(shost);
+
+	return sysfs_emit(buf, "0x%016llx\n",
+		(unsigned long long)adapter->dev_topo.card.sas_address);
+}
+
+static ssize_t board_name_show(struct device *cdev,
+			       struct device_attribute *attr, char *buf)
+{
+	struct Scsi_Host *shost = class_to_shost(cdev);
+	struct leapraid_adapter *adapter = shost_priv(shost);
+
+	return sysfs_emit(buf, "%s\n", adapter->adapter_attr.board_name);
+}
+
+static DEVICE_ATTR_RO(fw_queue_depth);
+static DEVICE_ATTR_RO(host_sas_address);
+static DEVICE_ATTR_RO(board_name);
+
+static struct attribute *leapraid_shost_attrs[] = {
+	&dev_attr_fw_queue_depth.attr,
+	&dev_attr_host_sas_address.attr,
+	&dev_attr_board_name.attr,
+	NULL,
+};
+
+ATTRIBUTE_GROUPS(leapraid_shost);
+
+static ssize_t sas_device_handle_show(struct device *dev,
+				      struct device_attribute *attr, char *buf)
+{
+	struct scsi_device *sdev = to_scsi_device(dev);
+	struct leapraid_sdev_priv *sas_device_priv_data = sdev->hostdata;
+
+	if (!sas_device_priv_data || !sas_device_priv_data->starget_priv) {
+		dev_err(&sdev->sdev_gendev,
+			"%s: Invalid sdev_priv or starget_priv\n", __func__);
+		return -EINVAL;
+	}
+
+	return sysfs_emit(buf, "0x%04x\n",
+			  sas_device_priv_data->starget_priv->hdl);
+}
+
+static ssize_t sas_ncq_prio_supported_show(struct device *dev,
+					   struct device_attribute *attr,
+					   char *buf)
+{
+	struct scsi_device *sdev = to_scsi_device(dev);
+
+	return sysfs_emit(buf, "%d\n", sas_ata_ncq_prio_supported(sdev));
+}
+
+static ssize_t sas_ncq_prio_enable_show(struct device *dev,
+					struct device_attribute *attr,
+					char *buf)
+{
+	struct scsi_device *sdev = to_scsi_device(dev);
+	struct leapraid_sdev_priv *sas_device_priv_data = sdev->hostdata;
+
+	if (!sas_device_priv_data) {
+		dev_err(&sdev->sdev_gendev,
+			"%s: Invalid sdev_priv\n", __func__);
+		return -EINVAL;
+	}
+
+	return sysfs_emit(buf, "%d\n", sas_device_priv_data->ncq_prio_enable);
+}
+
+static ssize_t sas_ncq_prio_enable_store(struct device *dev,
+					 struct device_attribute *attr,
+					 const char *buf, size_t count)
+{
+	struct scsi_device *sdev = to_scsi_device(dev);
+	struct leapraid_sdev_priv *sas_device_priv_data = sdev->hostdata;
+	bool enable;
+
+	if (!sas_device_priv_data) {
+		dev_err(&sdev->sdev_gendev,
+			"%s: Invalid sdev_priv\n", __func__);
+		return -EINVAL;
+	}
+
+	if (kstrtobool(buf, &enable))
+		return -EINVAL;
+
+	if (!sas_ata_ncq_prio_supported(sdev))
+		return -EINVAL;
+
+	sas_device_priv_data->ncq_prio_enable = enable;
+	return count;
+}
+
+static DEVICE_ATTR_RO(sas_device_handle);
+static DEVICE_ATTR_RO(sas_ncq_prio_supported);
+static DEVICE_ATTR_RW(sas_ncq_prio_enable);
+
+static bool leapraid_sdev_is_sata(struct scsi_device *sdev)
+{
+	struct scsi_target *starget = sdev->sdev_target;
+	struct leapraid_starget_priv *starget_priv = starget->hostdata;
+	struct leapraid_sas_dev *sas_dev;
+
+	if (!starget_priv)
+		return false;
+
+	sas_dev = starget_priv->sas_dev;
+	return sas_dev && (sas_dev->dev_info & LEAPRAID_DEVTYP_SATA_DEV);
+}
+
+static struct attribute *leapraid_sdev_attrs[] = {
+	&dev_attr_sas_device_handle.attr,
+	&dev_attr_sas_ncq_prio_supported.attr,
+	&dev_attr_sas_ncq_prio_enable.attr,
+	NULL,
+};
+
+static umode_t leapraid_sdev_attr_is_visible(struct kobject *kobj,
+					     struct attribute *attr, int i)
+{
+	struct device *dev = kobj_to_dev(kobj);
+	struct scsi_device *sdev = to_scsi_device(dev);
+
+	if (attr == &dev_attr_sas_ncq_prio_supported.attr ||
+	    attr == &dev_attr_sas_ncq_prio_enable.attr)
+		if (!leapraid_sdev_is_sata(sdev))
+			return 0;
+
+	return attr->mode;
+}
+
+static const struct attribute_group leapraid_sdev_attr_group = {
+	.attrs = leapraid_sdev_attrs,
+	.is_visible = leapraid_sdev_attr_is_visible,
+};
+
+static const struct attribute_group *leapraid_sdev_groups[] = {
+	&leapraid_sdev_attr_group,
+	NULL,
+};
+
+static struct scsi_host_template leapraid_driver_template = {
+	.module = THIS_MODULE,
+	.name = "LEAPIO RAID Host",
+	.proc_name = LEAPRAID_DRIVER_NAME,
+	.queuecommand = leapraid_queuecommand,
+	.cmd_size = sizeof(struct leapraid_io_req_tracker),
+	.init_cmd_priv = leapraid_init_cmd_priv,
+	.exit_cmd_priv = leapraid_exit_cmd_priv,
+	.eh_abort_handler = leapraid_eh_abort_handler,
+	.eh_device_reset_handler = leapraid_eh_device_reset_handler,
+	.eh_target_reset_handler = leapraid_eh_target_reset_handler,
+	.eh_host_reset_handler = leapraid_eh_host_reset_handler,
+	.sdev_init = leapraid_sdev_init,
+	.sdev_destroy = leapraid_sdev_destroy,
+	.sdev_configure = leapraid_sdev_configure,
+	.target_alloc = leapraid_target_alloc,
+	.target_destroy = leapraid_target_destroy,
+	.scan_finished = leapraid_scan_finished,
+	.scan_start = leapraid_scan_start,
+	.change_queue_depth = leapraid_change_queue_depth,
+	.map_queues = leapraid_map_queues,
+	.mq_poll = leapraid_blk_mq_poll,
+	.bios_param = leapraid_bios_param,
+	.can_queue = LEAPRAID_CAN_QUEUE_MIN,
+	.this_id = LEAPRAID_THIS_ID_NONE,
+	.sg_tablesize = LEAPRAID_SG_DEPTH,
+	.max_sectors = LEAPRAID_MAX_SECTORS,
+	.max_segment_size = LEAPRAID_MAX_SEGMENT_SIZE,
+	.cmd_per_lun = LEAPRAID_CMD_PER_LUN,
+	.shost_groups = leapraid_shost_groups,
+	.sdev_groups = leapraid_sdev_groups,
+	.track_queue_depth = 1,
+};
+
+static void leapraid_lock_init(struct leapraid_adapter *adapter)
+{
+	mutex_init(&adapter->reset_desc.adapter_reset_mutex);
+	mutex_init(&adapter->reset_desc.host_diag_mutex);
+	mutex_init(&adapter->access_ctrl.pci_access_lock);
+
+	spin_lock_init(&adapter->reset_desc.adapter_reset_lock);
+	spin_lock_init(&adapter->dynamic_task_desc.task_lock);
+	spin_lock_init(&adapter->dev_topo.sas_dev_lock);
+	spin_lock_init(&adapter->dev_topo.topo_node_lock);
+	spin_lock_init(&adapter->fw_evt_s.fw_evt_lock);
+	spin_lock_init(&adapter->dev_topo.raid_volume_lock);
+	spin_lock_init(&adapter->dev_topo.enc_lock);
+	spin_lock_init(&adapter->boot_devs.lock);
+}
+
+static void leapraid_list_init(struct leapraid_adapter *adapter)
+{
+	INIT_LIST_HEAD(&adapter->dev_topo.sas_dev_list);
+	INIT_LIST_HEAD(&adapter->dev_topo.card_port_list);
+	INIT_LIST_HEAD(&adapter->dev_topo.sas_dev_init_list);
+	INIT_LIST_HEAD(&adapter->dev_topo.exp_list);
+	INIT_LIST_HEAD(&adapter->dev_topo.enc_list);
+	INIT_LIST_HEAD(&adapter->fw_evt_s.fw_evt_list);
+	INIT_LIST_HEAD(&adapter->dev_topo.raid_volume_list);
+	INIT_LIST_HEAD(&adapter->dev_topo.card.sas_port_list);
+}
+
+static int leapraid_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct leapraid_adapter *adapter;
+	struct Scsi_Host *shost;
+	int iopoll_q_count;
+	int rc;
+
+	shost = scsi_host_alloc(&leapraid_driver_template,
+				sizeof(struct leapraid_adapter));
+	if (!shost) {
+		dev_err(&pdev->dev,
+			"%s: SCSI host alloc failed\n", __func__);
+		return -ENODEV;
+	}
+
+	adapter = shost_priv(shost);
+	memset(adapter, 0, sizeof(struct leapraid_adapter));
+
+	init_waitqueue_head(&adapter->access_ctrl.recovery_waitq);
+	adapter->adapter_attr.id = atomic_inc_return(&leapraid_ids) - 1;
+
+	adapter->adapter_attr.enable_mp = enable_mp;
+
+	adapter = shost_priv(shost);
+	INIT_LIST_HEAD(&adapter->list);
+
+	adapter->shost = shost;
+	adapter->pdev = pdev;
+	adapter->fw_log_desc.open_pcie_trace = open_pcie_trace;
+	atomic_set(&adapter->fw_log_desc.mmap_refcnt, 0);
+	init_waitqueue_head(&adapter->fw_log_desc.mmap_waitq);
+	leapraid_lock_init(adapter);
+	leapraid_list_init(adapter);
+	snprintf(adapter->adapter_attr.name, LEAPRAID_NAME_LENGTH, "%s%d",
+		 LEAPRAID_DRIVER_NAME, adapter->adapter_attr.id);
+
+	shost->max_cmd_len = LEAPRAID_MAX_CDB_LEN;
+	shost->max_lun = LEAPRAID_MAX_LUNS;
+	shost->transportt = leapraid_transport_template;
+	shost->unique_id = adapter->adapter_attr.id;
+
+	snprintf(adapter->fw_evt_s.fw_evt_name,
+		 sizeof(adapter->fw_evt_s.fw_evt_name),
+		 "fw_event_%s%d", LEAPRAID_DRIVER_NAME,
+		 adapter->adapter_attr.id);
+	adapter->fw_evt_s.fw_evt_thread =
+		alloc_ordered_workqueue(adapter->fw_evt_s.fw_evt_name, 0);
+	if (!adapter->fw_evt_s.fw_evt_thread) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to create fw event workqueue\n", __func__);
+		rc = -ENODEV;
+		goto evt_wq_fail;
+	}
+
+	shost->host_tagset = 1;
+	init_waitqueue_head(&adapter->scan_dev_desc.wait_driver_loading);
+	adapter->scan_dev_desc.driver_loading = 1;
+	if (leapraid_ctrl_init(adapter)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Adapter init failed\n", __func__);
+		rc = -ENODEV;
+		goto ctrl_init_fail;
+	}
+
+	shost->nr_hw_queues = 1;
+	if (shost->host_tagset) {
+		shost->nr_hw_queues = adapter->adapter_attr.rq_cnt;
+		iopoll_q_count = adapter->adapter_attr.rq_cnt -
+				 adapter->notification_desc.iopoll_qdex;
+		shost->nr_maps = iopoll_q_count ? 3 : 1;
+		dev_info(&adapter->pdev->dev,
+			 "Max scsi I/O cmds %d shared with nr_hw_queues=%d\n",
+			 shost->can_queue, shost->nr_hw_queues);
+	}
+
+	rc = scsi_add_host(shost, &pdev->dev);
+	if (rc) {
+		dev_err(&pdev->dev,
+			"%s: SCSI host add failed\n", __func__);
+		goto scsi_add_shost_fail;
+	}
+
+	spin_lock(&leapraid_adapter_lock);
+	list_add_tail(&adapter->list, &leapraid_adapter_list);
+	spin_unlock(&leapraid_adapter_lock);
+
+	scsi_scan_host(shost);
+	return 0;
+
+scsi_add_shost_fail:
+	leapraid_remove_ctrl(adapter);
+ctrl_init_fail:
+	leapraid_overheat_cleanup(adapter);
+	destroy_workqueue(adapter->fw_evt_s.fw_evt_thread);
+evt_wq_fail:
+	scsi_host_put(shost);
+	return rc;
+}
+
+void leapraid_cleanup_lists(struct leapraid_adapter *adapter)
+{
+	struct leapraid_raid_volume *raid_volume, *next_raid_volume;
+	struct leapraid_starget_priv *starget_priv_data;
+	struct leapraid_sas_port *leapraid_port, *next_port;
+	struct leapraid_card_port *port, *port_next;
+	struct leapraid_vphy *vphy, *vphy_next;
+
+	list_for_each_entry_safe(raid_volume, next_raid_volume,
+				 &adapter->dev_topo.raid_volume_list, list) {
+		if (raid_volume->starget) {
+			starget_priv_data = raid_volume->starget->hostdata;
+			if (starget_priv_data)
+				starget_priv_data->deleted = 1;
+			scsi_remove_target(&raid_volume->starget->dev);
+		}
+		dev_info(&adapter->pdev->dev,
+			 "removing hdl=0x%04x, wwid=0x%016llx\n",
+			 raid_volume->hdl,
+			 (unsigned long long)raid_volume->wwid);
+		leapraid_raid_volume_remove(adapter, raid_volume);
+	}
+
+	list_for_each_entry_safe(leapraid_port, next_port,
+				 &adapter->dev_topo.card.sas_port_list,
+				 port_list) {
+		if (leapraid_port->remote_identify.device_type ==
+		    SAS_END_DEVICE)
+			leapraid_sas_dev_remove_by_sas_address(
+				adapter,
+				leapraid_port->remote_identify.sas_address,
+				leapraid_port->card_port);
+		else if (leapraid_port->remote_identify.device_type ==
+				SAS_EDGE_EXPANDER_DEVICE ||
+			 leapraid_port->remote_identify.device_type ==
+				SAS_FANOUT_EXPANDER_DEVICE)
+			leapraid_exp_rm(
+				adapter,
+				leapraid_port->remote_identify.sas_address,
+				leapraid_port->card_port);
+	}
+
+	list_for_each_entry_safe(port, port_next,
+				 &adapter->dev_topo.card_port_list, list) {
+		if (port->vphys_mask)
+			list_for_each_entry_safe(vphy, vphy_next,
+						 &port->vphys_list, list) {
+				list_del(&vphy->list);
+				kfree(vphy);
+			}
+		list_del(&port->list);
+		kfree(port);
+	}
+
+	if (adapter->dev_topo.card.phys_num) {
+		kfree(adapter->dev_topo.card.card_phy);
+		adapter->dev_topo.card.card_phy = NULL;
+		adapter->dev_topo.card.phys_num = 0;
+	}
+}
+
+static void leapraid_remove(struct pci_dev *pdev)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	struct workqueue_struct *wq;
+	unsigned long flags;
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev, "Unable to remove!\n");
+		return;
+	}
+
+	wait_event(adapter->scan_dev_desc.wait_driver_loading,
+		   !adapter->scan_dev_desc.driver_loading);
+	wait_event(adapter->scan_dev_desc.wait_driver_loading,
+		   !atomic_read(&adapter->overheat_desc.thermal_alert));
+
+	WRITE_ONCE(adapter->access_ctrl.host_removing, 1);
+	spin_lock(&leapraid_adapter_lock);
+	list_del(&adapter->list);
+	spin_unlock(&leapraid_adapter_lock);
+
+	leapraid_wait_cmds_done(adapter);
+
+	if (leapraid_pci_removed(adapter)) {
+		leapraid_mq_polling_pause(adapter);
+		leapraid_clean_active_scsi_cmds(adapter);
+	}
+	leapraid_clean_active_fw_evt(adapter);
+
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	wq = adapter->fw_evt_s.fw_evt_thread;
+	adapter->fw_evt_s.fw_evt_thread = NULL;
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (wq)
+		destroy_workqueue(wq);
+
+	sas_remove_host(shost);
+	leapraid_cleanup_lists(adapter);
+	leapraid_remove_ctrl(adapter);
+	scsi_host_put(shost);
+}
+
+static void leapraid_shutdown(struct pci_dev *pdev)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	struct workqueue_struct *wq;
+	unsigned long flags;
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev, "Unable to shutdown!\n");
+		return;
+	}
+
+	adapter->access_ctrl.host_removing = 1;
+	leapraid_wait_cmds_done(adapter);
+	leapraid_clean_active_fw_evt(adapter);
+	leapraid_overheat_cleanup(adapter);
+	leapraid_fw_log_stop(adapter);
+	spin_lock_irqsave(&adapter->fw_evt_s.fw_evt_lock, flags);
+	wq = adapter->fw_evt_s.fw_evt_thread;
+	adapter->fw_evt_s.fw_evt_thread = NULL;
+	spin_unlock_irqrestore(&adapter->fw_evt_s.fw_evt_lock, flags);
+	if (wq)
+		destroy_workqueue(wq);
+
+	leapraid_disable_controller(adapter);
+}
+
+static pci_ers_result_t leapraid_pci_error_detected(struct pci_dev *pdev,
+						    pci_channel_state_t state)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev, "Failed to error detected for device\n");
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	dev_err(&pdev->dev, "%s: PCI error detected, state=%d\n",
+		adapter->adapter_attr.name, state);
+
+	switch (state) {
+	case pci_channel_io_normal:
+		return PCI_ERS_RESULT_CAN_RECOVER;
+	case pci_channel_io_frozen:
+		adapter->access_ctrl.pcie_recovering = 1;
+		scsi_block_requests(adapter->shost);
+		leapraid_overheat_cleanup(adapter);
+		leapraid_check_scheduled_fault_stop(adapter);
+		leapraid_fw_log_stop(adapter);
+		leapraid_disable_controller(adapter);
+		return PCI_ERS_RESULT_NEED_RESET;
+	case pci_channel_io_perm_failure:
+		adapter->access_ctrl.pcie_recovering = 1;
+		leapraid_overheat_cleanup(adapter);
+		leapraid_check_scheduled_fault_stop(adapter);
+		leapraid_fw_log_stop(adapter);
+		leapraid_mq_polling_pause(adapter);
+		leapraid_clean_active_scsi_cmds(adapter);
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	return PCI_ERS_RESULT_NEED_RESET;
+}
+
+static pci_ers_result_t leapraid_pci_mmio_enabled(struct pci_dev *pdev)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev,
+			"Failed to enable mmio for device\n");
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	dev_info(&pdev->dev, "%s: PCI error mmio enabled\n",
+		 adapter->adapter_attr.name);
+
+	return PCI_ERS_RESULT_RECOVERED;
+}
+
+static pci_ers_result_t leapraid_pci_slot_reset(struct pci_dev *pdev)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	int rc;
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev,
+			"Failed to slot reset for device\n");
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	dev_err(&pdev->dev, "%s PCI error slot reset\n",
+		adapter->adapter_attr.name);
+
+	adapter->pdev = pdev;
+	pci_restore_state(pdev);
+	if (leapraid_set_pcie_and_notification(adapter)) {
+		dev_err(&pdev->dev,
+			"%s: Failed to set PCIe state and notification\n",
+			__func__);
+		return PCI_ERS_RESULT_DISCONNECT;
+	}
+
+	adapter->access_ctrl.pcie_recovering = 0;
+	dev_info(&pdev->dev, "%s: Hard reset triggered by PCI slot reset\n",
+		 adapter->adapter_attr.name);
+	dev_info(&adapter->pdev->dev, "%s: %d: call hard_reset\n",
+		 __func__, __LINE__);
+	rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
+	if (rc)
+		dev_err(&pdev->dev, "%s hard reset: failed\n",
+			adapter->adapter_attr.name);
+
+	return rc == 0 ? PCI_ERS_RESULT_RECOVERED :
+		 PCI_ERS_RESULT_DISCONNECT;
+}
+
+static void leapraid_pci_resume(struct pci_dev *pdev)
+{
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev, "Failed to resume\n");
+		return;
+	}
+
+	dev_err(&pdev->dev, "PCI error resume!\n");
+
+	pci_aer_clear_nonfatal_status(pdev);
+	leapraid_check_scheduled_fault_start(adapter);
+	leapraid_fw_log_start(adapter);
+	scsi_unblock_requests(adapter->shost);
+}
+
+MODULE_DEVICE_TABLE(pci, leapraid_pci_table);
+static struct pci_error_handlers leapraid_err_handler = {
+	.error_detected = leapraid_pci_error_detected,
+	.mmio_enabled = leapraid_pci_mmio_enabled,
+	.slot_reset = leapraid_pci_slot_reset,
+	.resume = leapraid_pci_resume,
+};
+
+#ifdef CONFIG_PM
+static int leapraid_suspend(struct pci_dev *pdev, pm_message_t state)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	pci_power_t device_state;
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev,
+			"Suspend failed, invalid host or adapter\n");
+		return -ENXIO;
+	}
+
+	leapraid_overheat_cleanup(adapter);
+	leapraid_check_scheduled_fault_stop(adapter);
+	leapraid_fw_log_stop(adapter);
+	scsi_block_requests(shost);
+	device_state = pci_choose_state(pdev, state);
+
+	dev_info(&pdev->dev, "Entering PCI power state D%d, (slot=%s)\n",
+		 device_state, pci_name(pdev));
+
+	pci_save_state(pdev);
+	leapraid_disable_controller(adapter);
+	pci_set_power_state(pdev, device_state);
+	return 0;
+}
+
+static int leapraid_resume(struct pci_dev *pdev)
+{
+	struct leapraid_adapter *adapter = pdev_to_adapter(pdev);
+	struct Scsi_Host *shost = pci_get_drvdata(pdev);
+	pci_power_t device_state = pdev->current_state;
+	int rc;
+
+	if (!shost || !adapter) {
+		dev_err(&pdev->dev,
+			"Resume failed, invalid host or adapter\n");
+		return -ENXIO;
+	}
+
+	dev_info(&pdev->dev,
+		 "Resuming device %s, previous state D%d\n",
+		 pci_name(pdev), device_state);
+
+	pci_set_power_state(pdev, PCI_D0);
+	pci_enable_wake(pdev, PCI_D0, 0);
+	pci_restore_state(pdev);
+	adapter->pdev = pdev;
+	rc = leapraid_set_pcie_and_notification(adapter);
+	if (rc) {
+		dev_err(&pdev->dev,
+			"%s: Failed to set PCIe state and notification\n",
+			__func__);
+		return rc;
+	}
+
+	dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
+		 __func__, __LINE__);
+	rc = leapraid_hard_reset_handler(adapter, PART_RESET);
+	if (rc) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Hard reset failed during resume, rc=%d\n",
+			__func__, rc);
+		return rc;
+	}
+	scsi_unblock_requests(shost);
+	leapraid_check_scheduled_fault_start(adapter);
+	leapraid_fw_log_start(adapter);
+	return 0;
+}
+#endif /* CONFIG_PM */
+
+static struct pci_driver leapraid_driver = {
+	.name = LEAPRAID_DRIVER_NAME,
+	.id_table = leapraid_pci_table,
+	.probe = leapraid_probe,
+	.remove = leapraid_remove,
+	.shutdown = leapraid_shutdown,
+	.err_handler = &leapraid_err_handler,
+#ifdef CONFIG_PM
+	.suspend = leapraid_suspend,
+	.resume = leapraid_resume,
+#endif /* CONFIG_PM */
+};
+
+static int __init leapraid_init(void)
+{
+	int error;
+
+	pr_info("%s initializing\n", LEAPRAID_DRIVER_NAME);
+
+	leapraid_transport_template =
+		sas_attach_transport(&leapraid_transport_functions);
+	if (!leapraid_transport_template) {
+		pr_err("%s: Failed to attach SAS transport\n",
+		       LEAPRAID_DRIVER_NAME);
+		return -ENODEV;
+	}
+
+	error = pci_register_driver(&leapraid_driver);
+	if (error) {
+		pr_err("%s: PCI driver registration failed: %d\n",
+		       LEAPRAID_DRIVER_NAME, error);
+		sas_release_transport(leapraid_transport_template);
+		return error;
+	}
+
+	error = leapraid_ctl_init();
+	if (error) {
+		pci_unregister_driver(&leapraid_driver);
+		sas_release_transport(leapraid_transport_template);
+		return error;
+	}
+
+	return 0;
+}
+
+static void __exit leapraid_exit(void)
+{
+	pr_info("%s exiting\n", LEAPRAID_DRIVER_NAME);
+
+	leapraid_ctl_exit();
+	pci_unregister_driver(&leapraid_driver);
+	sas_release_transport(leapraid_transport_template);
+}
+
+module_init(leapraid_init);
+module_exit(leapraid_exit);
diff --git a/drivers/scsi/leapraid/leapraid_transport.c b/drivers/scsi/leapraid/leapraid_transport.c
new file mode 100644
index 000000000..77ffbd6dd
--- /dev/null
+++ b/drivers/scsi/leapraid/leapraid_transport.c
@@ -0,0 +1,1403 @@
+// SPDX-License-Identifier: GPL-2.0
+/*
+ * Copyright (C) 2026 LeapIO Tech Inc.
+ *
+ * LeapRAID storage and RAID controller driver.
+ */
+#include <scsi/scsi_host.h>
+
+#include "leapraid_func.h"
+
+static struct leapraid_topo_node *leapraid_transport_topo_node_by_sas_addr(
+		struct leapraid_adapter *adapter,
+		u64 sas_addr,
+		struct leapraid_card_port *card_port)
+{
+	if (adapter->dev_topo.card.sas_address == sas_addr)
+		return &adapter->dev_topo.card;
+
+	return leapraid_exp_find_by_sas_address(adapter, sas_addr, card_port);
+}
+
+static u8 leapraid_get_port_id_by_expander(struct leapraid_adapter *adapter,
+					   struct sas_rphy *rphy)
+{
+	struct leapraid_topo_node *topo_node_exp;
+	unsigned long flags;
+	u8 port_id = 0xFF;
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	list_for_each_entry(topo_node_exp, &adapter->dev_topo.exp_list, list) {
+		if (topo_node_exp->rphy == rphy) {
+			port_id = topo_node_exp->card_port->port_id;
+			break;
+		}
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	return port_id;
+}
+
+static u8 leapraid_get_port_id_by_end_dev(struct leapraid_adapter *adapter,
+					  struct sas_rphy *rphy)
+{
+	struct leapraid_sas_dev *sas_dev;
+	unsigned long flags;
+	u8 port_id = 0xFF;
+
+	spin_lock_irqsave(&adapter->dev_topo.sas_dev_lock, flags);
+	sas_dev = leapraid_hold_lock_get_sas_dev_by_addr_and_rphy(
+			adapter,
+			rphy->identify.sas_address,
+			rphy);
+	if (sas_dev) {
+		port_id = sas_dev->card_port->port_id;
+		leapraid_sdev_put(sas_dev);
+	}
+	spin_unlock_irqrestore(&adapter->dev_topo.sas_dev_lock, flags);
+
+	return port_id;
+}
+
+static u8 leapraid_transport_get_port_id_by_rphy(
+		struct leapraid_adapter *adapter,
+		struct sas_rphy *rphy)
+{
+	if (!rphy)
+		return 0xFF;
+
+	switch (rphy->identify.device_type) {
+	case SAS_EDGE_EXPANDER_DEVICE:
+	case SAS_FANOUT_EXPANDER_DEVICE:
+		return leapraid_get_port_id_by_expander(adapter, rphy);
+	case SAS_END_DEVICE:
+		return leapraid_get_port_id_by_end_dev(adapter, rphy);
+	default:
+		return 0xFF;
+	}
+}
+
+static enum sas_linkrate leapraid_transport_convert_phy_link_rate(u8 link_rate)
+{
+	unsigned int i;
+
+	#define SAS_RATE_12G SAS_LINK_RATE_12_0_GBPS
+
+	const struct linkrate_map {
+		u8 in;
+		enum sas_linkrate out;
+	} linkrate_table[] = {
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_1_5,
+			SAS_LINK_RATE_1_5_GBPS
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_3_0,
+			SAS_LINK_RATE_3_0_GBPS
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_6_0,
+			SAS_LINK_RATE_6_0_GBPS
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_12_0,
+			SAS_RATE_12G
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_PHY_DISABLED,
+			SAS_PHY_DISABLED
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_NEGOTIATION_FAILED,
+			SAS_LINK_RATE_FAILED
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_PORT_SELECTOR,
+			SAS_SATA_PORT_SELECTOR
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_SMP_RESETTING,
+			SAS_LINK_RATE_UNKNOWN
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_SATA_OOB_COMPLETE,
+			SAS_LINK_RATE_UNKNOWN
+		},
+		{
+			LEAPRAID_SAS_NEG_LINK_RATE_UNKNOWN_LINK_RATE,
+			SAS_LINK_RATE_UNKNOWN
+		},
+	};
+
+	for (i = 0; i < ARRAY_SIZE(linkrate_table); i++)
+		if (linkrate_table[i].in == link_rate)
+			return linkrate_table[i].out;
+
+	return SAS_LINK_RATE_UNKNOWN;
+}
+
+static void leapraid_set_identify_protocol_flags(u32 dev_info,
+						 struct sas_identify *identify)
+{
+	unsigned int i;
+
+	const struct protocol_mapping {
+		u32 mask;
+		u32 *target;
+		u32 protocol;
+	} mappings[] = {
+		{
+			LEAPRAID_DEVTYP_SSP_INIT,
+			&identify->initiator_port_protocols,
+			SAS_PROTOCOL_SSP
+		},
+		{
+			LEAPRAID_DEVTYP_STP_INIT,
+			&identify->initiator_port_protocols,
+			SAS_PROTOCOL_STP
+		},
+		{
+			LEAPRAID_DEVTYP_SMP_INIT,
+			&identify->initiator_port_protocols,
+			SAS_PROTOCOL_SMP
+		},
+		{
+			LEAPRAID_DEVTYP_SATA_HOST,
+			&identify->initiator_port_protocols,
+			SAS_PROTOCOL_SATA
+		},
+		{
+			LEAPRAID_DEVTYP_SSP_TGT,
+			&identify->target_port_protocols,
+			SAS_PROTOCOL_SSP
+		},
+		{
+			LEAPRAID_DEVTYP_STP_TGT,
+			&identify->target_port_protocols,
+			SAS_PROTOCOL_STP
+		},
+		{
+			LEAPRAID_DEVTYP_SMP_TGT,
+			&identify->target_port_protocols,
+			SAS_PROTOCOL_SMP
+		},
+		{
+			LEAPRAID_DEVTYP_SATA_DEV,
+			&identify->target_port_protocols,
+			SAS_PROTOCOL_SATA
+		},
+	};
+
+	for (i = 0; i < ARRAY_SIZE(mappings); i++)
+		if (dev_info & mappings[i].mask && mappings[i].target)
+			*mappings[i].target |= mappings[i].protocol;
+}
+
+static int leapraid_transport_set_identify(struct leapraid_adapter *adapter,
+					   u16 hdl,
+					   struct sas_identify *identify)
+{
+	union cfg_param_1 cfgp1 = {0};
+	union cfg_param_2 cfgp2 = {0};
+	struct leapraid_sas_dev_p0 sas_dev_pg0;
+	u32 dev_info;
+
+	if ((adapter->access_ctrl.shost_recovering &&
+	     !adapter->scan_dev_desc.driver_loading) ||
+	    adapter->access_ctrl.pcie_recovering) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed, shost_recovering=%d pcie_recovering=%d\n",
+			 __func__,
+			 adapter->access_ctrl.shost_recovering,
+			 adapter->access_ctrl.pcie_recovering);
+		return -EFAULT;
+	}
+
+	cfgp1.form = LEAPRAID_SAS_DEV_CFG_PGAD_HDL;
+	cfgp2.handle = hdl;
+	if (leapraid_op_config_page(adapter, &sas_dev_pg0, cfgp1,
+				    cfgp2, GET_SAS_DEVICE_PG0))
+		return -ENXIO;
+
+	memset(identify, 0, sizeof(struct sas_identify));
+	dev_info = le32_to_cpu(sas_dev_pg0.dev_info);
+	identify->sas_address = le64_to_cpu(sas_dev_pg0.sas_address);
+	identify->phy_identifier = sas_dev_pg0.phy_num;
+
+	switch (dev_info & LEAPRAID_DEVTYP_MASK_DEV_TYPE) {
+	case LEAPRAID_DEVTYP_NO_DEV:
+		identify->device_type = SAS_PHY_UNUSED;
+		break;
+	case LEAPRAID_DEVTYP_END_DEV:
+		identify->device_type = SAS_END_DEVICE;
+		break;
+	case LEAPRAID_DEVTYP_EDGE_EXPANDER:
+		identify->device_type = SAS_EDGE_EXPANDER_DEVICE;
+		break;
+	case LEAPRAID_DEVTYP_FANOUT_EXPANDER:
+		identify->device_type = SAS_FANOUT_EXPANDER_DEVICE;
+		break;
+	}
+
+	leapraid_set_identify_protocol_flags(dev_info, identify);
+
+	return 0;
+}
+
+static void leapraid_transport_exp_set_edev(struct leapraid_adapter *adapter,
+					    void *data_out,
+					    struct sas_expander_device *edev)
+{
+	struct leapraid_smp_passthrough_rep *smp_passthrough_rep;
+	struct leapraid_rep_manu_reply *rep_manu_reply;
+	u8 *component_id;
+
+	smp_passthrough_rep =
+		(void *)&adapter->driver_cmds.transport_cmd.reply;
+	if (le16_to_cpu(smp_passthrough_rep->resp_data_len) !=
+	    sizeof(struct leapraid_rep_manu_reply))
+		return;
+
+	rep_manu_reply = data_out + sizeof(struct leapraid_rep_manu_request);
+
+	memcpy(edev->vendor_id, rep_manu_reply->vendor_identification,
+	       SAS_EXPANDER_VENDOR_ID_LEN);
+	edev->vendor_id[SAS_EXPANDER_VENDOR_ID_LEN] = '\0';
+
+	memcpy(edev->product_id, rep_manu_reply->product_identification,
+	       SAS_EXPANDER_PRODUCT_ID_LEN);
+	edev->product_id[SAS_EXPANDER_PRODUCT_ID_LEN] = '\0';
+
+	memcpy(edev->product_rev, rep_manu_reply->product_revision_level,
+	       SAS_EXPANDER_PRODUCT_REV_LEN);
+	edev->product_rev[SAS_EXPANDER_PRODUCT_REV_LEN] = '\0';
+
+	edev->level = rep_manu_reply->sas_format & 1;
+	if (edev->level) {
+		memcpy(edev->component_vendor_id,
+		       rep_manu_reply->comp_vendor_identification,
+		       SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN);
+		edev->component_vendor_id[
+			SAS_EXPANDER_COMPONENT_VENDOR_ID_LEN] = '\0';
+
+		component_id = (u8 *)&rep_manu_reply->component_id;
+		edev->component_id = component_id[0] << 8 | component_id[1];
+		edev->component_revision_id =
+			rep_manu_reply->component_revision_level;
+	}
+}
+
+static int leapraid_transport_exp_report_manu(struct leapraid_adapter *adapter,
+					      u64 sas_address,
+					      struct sas_expander_device *edev,
+					      u8 port_id)
+{
+	struct leapraid_smp_passthrough_req *smp_passthrough_req;
+	struct leapraid_rep_manu_request *rep_manu_request;
+	dma_addr_t h2c_dma_addr;
+	dma_addr_t c2h_dma_addr;
+	bool issue_reset = false;
+	void *data_out = NULL;
+	u16 inter_taskid;
+	size_t c2h_size;
+	size_t h2c_size;
+	void *psge;
+	int rc;
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.pcie_recovering) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed, shost_recovering=%d pcie_recovering=%d\n",
+			 __func__,
+			 adapter->access_ctrl.shost_recovering,
+			 adapter->access_ctrl.pcie_recovering);
+		return -EFAULT;
+	}
+
+	mutex_lock(&adapter->driver_cmds.transport_cmd.mutex);
+	adapter->driver_cmds.transport_cmd.status = LEAPRAID_CMD_PENDING;
+	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
+					  __func__);
+	if (rc)
+		goto out_cleanup;
+
+	h2c_size = sizeof(struct leapraid_rep_manu_request);
+	c2h_size = sizeof(struct leapraid_rep_manu_reply);
+	data_out = dma_alloc_coherent(&adapter->pdev->dev,
+				      h2c_size + c2h_size,
+				      &h2c_dma_addr,
+				      GFP_ATOMIC);
+	if (!data_out) {
+		rc = -ENOMEM;
+		goto out_cleanup;
+	}
+
+	rep_manu_request = data_out;
+	rep_manu_request->smp_frame_type =
+		SMP_REPORT_MANUFACTURER_INFORMATION_FRAME_TYPE;
+	rep_manu_request->function = SMP_REPORT_MANUFACTURER_INFORMATION_FUNC;
+	rep_manu_request->allocated_response_length = 0;
+	rep_manu_request->request_length = 0;
+
+	inter_taskid = adapter->driver_cmds.transport_cmd.inter_taskid;
+	smp_passthrough_req = leapraid_get_task_desc(adapter, inter_taskid);
+	memset(smp_passthrough_req, 0,
+	       sizeof(struct leapraid_smp_passthrough_req));
+	smp_passthrough_req->func = LEAPRAID_FUNC_SMP_PASSTHROUGH;
+	smp_passthrough_req->physical_port = port_id;
+	smp_passthrough_req->sas_address = cpu_to_le64(sas_address);
+	smp_passthrough_req->req_data_len = cpu_to_le16(h2c_size);
+	psge = &smp_passthrough_req->sgl;
+	c2h_dma_addr = h2c_dma_addr + sizeof(struct leapraid_rep_manu_request);
+	leapraid_build_ieee_sg(adapter, psge, h2c_dma_addr, h2c_size,
+			       c2h_dma_addr, c2h_size);
+
+	init_completion(&adapter->driver_cmds.transport_cmd.done);
+	leapraid_fire_task(adapter, inter_taskid);
+	wait_for_completion_timeout(&adapter->driver_cmds.transport_cmd.done,
+				    LEAPRAID_TRANSPORT_CMD_TIMEOUT * HZ);
+	if (!(adapter->driver_cmds.transport_cmd.status & LEAPRAID_CMD_DONE)) {
+		rc = -ETIMEDOUT;
+		dev_err(&adapter->pdev->dev,
+			"%s: SMP passthrough timeout, st=0x%x\n",
+			__func__, adapter->driver_cmds.transport_cmd.status);
+		leapraid_log_req_context(adapter, inter_taskid,
+					 smp_passthrough_req);
+		if (!(adapter->driver_cmds.transport_cmd.status &
+		      LEAPRAID_CMD_RESET))
+			issue_reset = true;
+
+		goto hard_reset;
+	}
+
+	if (adapter->driver_cmds.transport_cmd.status &
+	    LEAPRAID_CMD_REPLY_VALID)
+		leapraid_transport_exp_set_edev(adapter, data_out, edev);
+
+hard_reset:
+	if (issue_reset) {
+		dev_info(&adapter->pdev->dev, "%s:%d: call hard_reset\n",
+			 __func__, __LINE__);
+		rc = leapraid_hard_reset_handler(adapter, FULL_RESET);
+	}
+out_cleanup:
+	adapter->driver_cmds.transport_cmd.status = LEAPRAID_CMD_NOT_USED;
+	if (data_out)
+		dma_free_coherent(&adapter->pdev->dev, h2c_size + c2h_size,
+				  data_out, h2c_dma_addr);
+
+	mutex_unlock(&adapter->driver_cmds.transport_cmd.mutex);
+	return rc;
+}
+
+static void leapraid_transport_del_port(struct leapraid_adapter *adapter,
+					struct leapraid_sas_port *sas_port)
+{
+	dev_info(&sas_port->port->dev,
+		 "Remove port: SAS addr=0x%016llx\n",
+		 (unsigned long long)sas_port->remote_identify.sas_address);
+	switch (sas_port->remote_identify.device_type) {
+	case SAS_END_DEVICE:
+		leapraid_sas_dev_remove_by_sas_address(
+			adapter,
+			sas_port->remote_identify.sas_address,
+			sas_port->card_port);
+		break;
+	case SAS_EDGE_EXPANDER_DEVICE:
+	case SAS_FANOUT_EXPANDER_DEVICE:
+		leapraid_exp_rm(adapter, sas_port->remote_identify.sas_address,
+				sas_port->card_port);
+		break;
+	default:
+		break;
+	}
+}
+
+static void leapraid_transport_del_phy(struct leapraid_adapter *adapter,
+				       struct leapraid_sas_port *sas_port,
+				       struct leapraid_card_phy *card_phy)
+{
+	dev_info(&card_phy->phy->dev,
+		 "Remove PHY: SAS addr=0x%016llx, phy=%d\n",
+		 (unsigned long long)sas_port->remote_identify.sas_address,
+		 card_phy->phy_id);
+	list_del(&card_phy->port_siblings);
+	sas_port->phys_num--;
+	sas_port_delete_phy(sas_port->port, card_phy->phy);
+	card_phy->phy_is_assigned = 0;
+}
+
+static void leapraid_transport_add_phy(struct leapraid_adapter *adapter,
+				       struct leapraid_sas_port *sas_port,
+				       struct leapraid_card_phy *card_phy)
+{
+	dev_info(&card_phy->phy->dev,
+		 "Add PHY: SAS addr=0x%016llx, phy=%d\n",
+		 (unsigned long long)sas_port->remote_identify.sas_address,
+		 card_phy->phy_id);
+	list_add_tail(&card_phy->port_siblings, &sas_port->phy_list);
+	sas_port->phys_num++;
+	sas_port_add_phy(sas_port->port, card_phy->phy);
+	card_phy->phy_is_assigned = 1;
+}
+
+void leapraid_transport_attach_phy_to_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node,
+		struct leapraid_card_phy *card_phy,
+		u64 sas_address,
+		struct leapraid_card_port *card_port)
+{
+	struct leapraid_sas_port *sas_port;
+	struct leapraid_card_phy *card_phy_srch;
+
+	if (card_phy->phy_is_assigned)
+		return;
+
+	if (!card_port)
+		return;
+
+	list_for_each_entry(sas_port, &topo_node->sas_port_list, port_list) {
+		if (sas_port->remote_identify.sas_address != sas_address)
+			continue;
+
+		if (sas_port->card_port != card_port)
+			continue;
+
+		list_for_each_entry(card_phy_srch, &sas_port->phy_list,
+				    port_siblings) {
+			if (card_phy_srch == card_phy)
+				return;
+		}
+		leapraid_transport_add_phy(adapter, sas_port, card_phy);
+		return;
+	}
+}
+
+void leapraid_transport_detach_phy_to_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node,
+		struct leapraid_card_phy *target_card_phy)
+{
+	struct leapraid_sas_port *sas_port, *sas_port_next;
+	struct leapraid_card_phy *cur_card_phy;
+
+	if (!target_card_phy->phy_is_assigned)
+		return;
+
+	list_for_each_entry_safe(sas_port, sas_port_next,
+				 &topo_node->sas_port_list, port_list) {
+		list_for_each_entry(cur_card_phy, &sas_port->phy_list,
+				    port_siblings) {
+			if (cur_card_phy != target_card_phy)
+				continue;
+
+			if (sas_port->phys_num == 1 &&
+			    !adapter->access_ctrl.shost_recovering)
+				leapraid_transport_del_port(adapter, sas_port);
+			else
+				leapraid_transport_del_phy(adapter, sas_port,
+							   target_card_phy);
+			return;
+		}
+	}
+}
+
+static void leapraid_detach_phy_from_old_port(
+		struct leapraid_adapter *adapter,
+		struct leapraid_topo_node *topo_node,
+		u64 sas_address,
+		struct leapraid_card_port *card_port)
+{
+	int i;
+
+	for (i = 0; i < topo_node->phys_num; i++) {
+		if (topo_node->card_phy[i].remote_identify.sas_address !=
+			sas_address ||
+		    topo_node->card_phy[i].card_port != card_port)
+			continue;
+		if (topo_node->card_phy[i].phy_is_assigned)
+			leapraid_transport_detach_phy_to_port(
+				adapter,
+				topo_node,
+				&topo_node->card_phy[i]);
+	}
+}
+
+static struct leapraid_sas_port *leapraid_prepare_sas_port(
+		struct leapraid_adapter *adapter,
+		u16 handle, u64 sas_address,
+		struct leapraid_card_port *card_port,
+		struct leapraid_topo_node **out_topo_node)
+{
+	struct leapraid_topo_node *topo_node;
+	struct leapraid_sas_port *sas_port;
+	unsigned long flags;
+
+	sas_port = kzalloc_obj(*sas_port);
+	if (!sas_port) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed alloc sas_port\n", __func__);
+		return NULL;
+	}
+
+	INIT_LIST_HEAD(&sas_port->port_list);
+	INIT_LIST_HEAD(&sas_port->phy_list);
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	topo_node = leapraid_transport_topo_node_by_sas_addr(adapter,
+							     sas_address,
+							     card_port);
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	if (!topo_node) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to locate parent for SAS 0x%016llx!\n",
+			__func__, sas_address);
+		goto out_cleanup;
+	}
+
+	if (leapraid_transport_set_identify(adapter, handle,
+					    &sas_port->remote_identify))
+		goto out_cleanup;
+
+	if (sas_port->remote_identify.device_type == SAS_PHY_UNUSED) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Failed, device type is SAS_PHY_UNUSED\n",
+			 __func__);
+		goto out_cleanup;
+	}
+
+	sas_port->card_port = card_port;
+	*out_topo_node = topo_node;
+
+	return sas_port;
+
+out_cleanup:
+	kfree(sas_port);
+	return NULL;
+}
+
+static int leapraid_bind_phys_and_vphy(struct leapraid_adapter *adapter,
+				       struct leapraid_sas_port *sas_port,
+				       struct leapraid_topo_node *topo_node,
+				       struct leapraid_card_port *card_port,
+				       struct leapraid_vphy **out_vphy)
+{
+	struct leapraid_vphy *vphy = NULL;
+	int i;
+
+	for (i = 0; i < topo_node->phys_num; i++) {
+		if (topo_node->card_phy[i].remote_identify.sas_address !=
+			sas_port->remote_identify.sas_address ||
+		    topo_node->card_phy[i].card_port != card_port)
+			continue;
+
+		list_add_tail(&topo_node->card_phy[i].port_siblings,
+			      &sas_port->phy_list);
+		sas_port->phys_num++;
+
+		if (topo_node->hdl <= adapter->dev_topo.card.phys_num) {
+			if (!topo_node->card_phy[i].vphy) {
+				card_port->phy_mask |= BIT(i);
+				continue;
+			}
+
+			vphy = leapraid_get_vphy_by_phy(card_port, i);
+			if (!vphy)
+				return LEAPRAID_OPERATION_FAILED;
+		}
+	}
+
+	*out_vphy = vphy;
+	return sas_port->phys_num ? 0 : LEAPRAID_OPERATION_FAILED;
+}
+
+static struct sas_rphy *leapraid_create_and_register_rphy(
+		struct leapraid_adapter *adapter,
+		struct leapraid_sas_port *sas_port,
+		struct leapraid_topo_node *topo_node,
+		struct leapraid_card_port *card_port,
+		struct leapraid_vphy *vphy)
+{
+	struct leapraid_sas_dev *sas_dev = NULL;
+	struct leapraid_card_phy *card_phy;
+	struct sas_port *port;
+	struct sas_rphy *rphy;
+
+	if (sas_port->remote_identify.device_type == SAS_END_DEVICE) {
+		sas_dev = leapraid_get_sas_dev_by_addr(
+				adapter,
+				sas_port->remote_identify.sas_address,
+				card_port);
+		if (!sas_dev)
+			return NULL;
+		sas_dev->pend_sas_rphy_add = 1;
+	}
+
+	if (!topo_node->parent_dev) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: topo_node parent device is NULL\n",  __func__);
+		goto cleanup_sas_dev;
+	}
+
+	port = sas_port_alloc_num(topo_node->parent_dev);
+	if (!port) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to allocate SAS port\n", __func__);
+		goto cleanup_sas_dev;
+	}
+
+	if (sas_port_add(port)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to add SAS port\n", __func__);
+		goto out_delete_port;
+	}
+
+	list_for_each_entry(card_phy, &sas_port->phy_list, port_siblings) {
+		sas_port_add_phy(port, card_phy->phy);
+		card_phy->phy_is_assigned = 1;
+		card_phy->card_port = card_port;
+	}
+
+	if (sas_dev) {
+		rphy = sas_end_device_alloc(port);
+		sas_dev->rphy = rphy;
+
+		if (topo_node->hdl <= adapter->dev_topo.card.phys_num) {
+			if (!vphy)
+				card_port->sas_address = sas_dev->sas_addr;
+			else
+				vphy->sas_address = sas_dev->sas_addr;
+		}
+	} else {
+		rphy = sas_expander_alloc(
+				port,
+				sas_port->remote_identify.device_type);
+		if (topo_node->hdl <= adapter->dev_topo.card.phys_num)
+			card_port->sas_address =
+				sas_port->remote_identify.sas_address;
+	}
+
+	if (!rphy) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to allocate RPHY\n", __func__);
+		goto cleanup_card_phy;
+	}
+
+	rphy->identify = sas_port->remote_identify;
+
+	if (sas_rphy_add(rphy)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to add RPHY\n", __func__);
+		if (sas_dev)
+			sas_dev->rphy = NULL;
+
+		sas_rphy_free(rphy);
+		goto cleanup_card_phy;
+	}
+	sas_port->port = port;
+
+	if (sas_dev) {
+		sas_dev->pend_sas_rphy_add = 0;
+		leapraid_sdev_put(sas_dev);
+	}
+
+	return rphy;
+
+cleanup_card_phy:
+	if (topo_node->hdl <= adapter->dev_topo.card.phys_num) {
+		if (!vphy)
+			card_port->sas_address = 0;
+		else
+			vphy->sas_address = 0;
+	}
+
+	list_for_each_entry(card_phy, &sas_port->phy_list, port_siblings) {
+		card_phy->phy_is_assigned = 0;
+		card_phy->card_port = NULL;
+	}
+
+out_delete_port:
+	sas_port_delete(port);
+
+cleanup_sas_dev:
+	if (sas_dev) {
+		sas_dev->pend_sas_rphy_add = 0;
+		leapraid_sdev_put(sas_dev);
+	}
+	return NULL;
+}
+
+struct leapraid_sas_port *leapraid_transport_port_add(
+		struct leapraid_adapter *adapter,
+		u16 hdl, u64 sas_address,
+		struct leapraid_card_port *card_port)
+{
+	struct leapraid_card_phy *card_phy, *card_phy_next;
+	struct leapraid_topo_node *topo_node = NULL;
+	struct leapraid_sas_port *sas_port;
+	struct leapraid_vphy *vphy = NULL;
+	struct sas_rphy *rphy;
+	unsigned long flags;
+
+	if (!card_port) {
+		dev_warn(&adapter->pdev->dev,
+			 "%s: Invalid card_port\n", __func__);
+		return NULL;
+	}
+
+	sas_port = leapraid_prepare_sas_port(adapter, hdl, sas_address,
+					     card_port, &topo_node);
+	if (!sas_port)
+		return NULL;
+
+	leapraid_detach_phy_from_old_port(
+		adapter,
+		topo_node,
+		sas_port->remote_identify.sas_address,
+		card_port);
+
+	if (leapraid_bind_phys_and_vphy(adapter, sas_port, topo_node,
+					card_port, &vphy)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to bind phy to vphy\n", __func__);
+		goto out_fail;
+	}
+
+	rphy = leapraid_create_and_register_rphy(adapter, sas_port, topo_node,
+						 card_port, vphy);
+	if (!rphy) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed to create rphy\n", __func__);
+		goto out_fail;
+	}
+
+	dev_info(&rphy->dev,
+		 "%s: Added dev: hdl=0x%04x, SAS addr=0x%016llx\n",
+		 __func__, hdl,
+		 (unsigned long long)sas_port->remote_identify.sas_address);
+
+	sas_port->rphy = rphy;
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	list_add_tail(&sas_port->port_list, &topo_node->sas_port_list);
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	if (sas_port->remote_identify.device_type ==
+	    SAS_EDGE_EXPANDER_DEVICE ||
+	    sas_port->remote_identify.device_type ==
+	    SAS_FANOUT_EXPANDER_DEVICE)
+		leapraid_transport_exp_report_manu(
+			adapter,
+			sas_port->remote_identify.sas_address,
+			rphy_to_expander_device(rphy),
+			card_port->port_id);
+
+	return sas_port;
+
+out_fail:
+	list_for_each_entry_safe(card_phy, card_phy_next,
+				 &sas_port->phy_list, port_siblings) {
+		if (topo_node->hdl <= adapter->dev_topo.card.phys_num &&
+		    !card_phy->vphy)
+			card_port->phy_mask &= ~BIT(card_phy->phy_id);
+
+		list_del_init(&card_phy->port_siblings);
+	}
+
+	kfree(sas_port);
+	return NULL;
+}
+
+static struct leapraid_sas_port *leapraid_find_and_remove_sas_port(
+		struct leapraid_topo_node *topo_node,
+		u64 sas_address,
+		struct leapraid_card_port *remove_card_port,
+		bool *found)
+{
+	struct leapraid_sas_port *sas_port, *sas_port_next;
+
+	list_for_each_entry_safe(sas_port, sas_port_next,
+				 &topo_node->sas_port_list, port_list) {
+		if (sas_port->remote_identify.sas_address != sas_address)
+			continue;
+
+		if (sas_port->card_port != remove_card_port)
+			continue;
+
+		*found = true;
+		list_del(&sas_port->port_list);
+		return sas_port;
+	}
+	return NULL;
+}
+
+static void leapraid_cleanup_card_port_and_vphys(
+		struct leapraid_adapter *adapter,
+		u64 sas_address,
+		struct leapraid_card_port *remove_card_port)
+{
+	struct leapraid_card_port *card_port, *card_port_next;
+	struct leapraid_vphy *vphy, *vphy_next;
+
+	if (remove_card_port->vphys_mask) {
+		list_for_each_entry_safe(vphy, vphy_next,
+					 &remove_card_port->vphys_list, list) {
+			if (vphy->sas_address != sas_address)
+				continue;
+
+			dev_dbg(&adapter->pdev->dev,
+				"%s: Remove vphy=%p from port=%p port_id=%d\n",
+				__func__, vphy, remove_card_port,
+				remove_card_port->port_id);
+
+			remove_card_port->vphys_mask &= ~vphy->phy_mask;
+			list_del(&vphy->list);
+			kfree(vphy);
+		}
+
+		if (!remove_card_port->vphys_mask &&
+		    !remove_card_port->sas_address) {
+			dev_dbg(&adapter->pdev->dev,
+				"%s: Remove empty hba_port: %p, port_id=%d\n",
+				__func__,
+				remove_card_port,
+				remove_card_port->port_id);
+			list_del(&remove_card_port->list);
+			kfree(remove_card_port);
+			remove_card_port = NULL;
+		}
+	}
+
+	list_for_each_entry_safe(card_port, card_port_next,
+				 &adapter->dev_topo.card_port_list, list) {
+		if (card_port != remove_card_port)
+			continue;
+
+		if (card_port->sas_address != sas_address)
+			continue;
+
+		if (!remove_card_port->vphys_mask) {
+			dev_dbg(&adapter->pdev->dev,
+				"%s: Remove hba_port: %p, port_id=%d\n",
+				__func__, card_port, card_port->port_id);
+			list_del(&card_port->list);
+			kfree(card_port);
+		} else {
+			dev_dbg(&adapter->pdev->dev,
+				"Clear sas_address of port=%p, port_id=%d\n",
+				card_port, card_port->port_id);
+			remove_card_port->sas_address = 0;
+		}
+		break;
+	}
+}
+
+static void leapraid_clear_topo_node_phys(struct leapraid_topo_node *topo_node,
+					  u64 sas_address)
+{
+	int i;
+
+	for (i = 0; i < topo_node->phys_num; i++)
+		if (topo_node->card_phy[i].remote_identify.sas_address ==
+		    sas_address) {
+			memset(&topo_node->card_phy[i].remote_identify, 0,
+			       sizeof(struct sas_identify));
+			topo_node->card_phy[i].vphy = 0;
+		}
+}
+
+void leapraid_transport_port_remove(
+		struct leapraid_adapter *adapter,
+		u64 sas_address,
+		u64 sas_address_parent,
+		struct leapraid_card_port *remove_card_port)
+{
+	struct leapraid_card_phy *card_phy, *card_phy_next;
+	struct leapraid_sas_port *sas_port;
+	struct leapraid_topo_node *topo_node;
+	unsigned long flags;
+	bool found = false;
+
+	if (!remove_card_port)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+
+	topo_node = leapraid_transport_topo_node_by_sas_addr(adapter,
+							     sas_address_parent,
+							     remove_card_port);
+	if (!topo_node) {
+		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
+				       flags);
+		return;
+	}
+
+	sas_port = leapraid_find_and_remove_sas_port(topo_node, sas_address,
+						     remove_card_port, &found);
+
+	if (!found) {
+		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
+				       flags);
+		return;
+	}
+
+	if (topo_node->hdl <= adapter->dev_topo.card.phys_num &&
+	    adapter->adapter_attr.enable_mp)
+		leapraid_cleanup_card_port_and_vphys(adapter, sas_address,
+						     remove_card_port);
+
+	leapraid_clear_topo_node_phys(topo_node, sas_address);
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	list_for_each_entry_safe(card_phy, card_phy_next,
+				 &sas_port->phy_list, port_siblings) {
+		card_phy->phy_is_assigned = 0;
+		if (!adapter->access_ctrl.host_removing)
+			sas_port_delete_phy(sas_port->port, card_phy->phy);
+
+		list_del(&card_phy->port_siblings);
+	}
+
+	if (!adapter->access_ctrl.host_removing)
+		sas_port_delete(sas_port->port);
+
+	dev_dbg(&adapter->pdev->dev,
+		"%s: Removed sas_port for SAS addr=0x%016llx\n",
+		__func__, (unsigned long long)sas_address);
+
+	kfree(sas_port);
+}
+
+static void leapraid_init_sas_or_exp_phy(struct leapraid_adapter *adapter,
+					 struct leapraid_card_phy *card_phy,
+					 struct sas_phy *phy,
+					 struct leapraid_sas_phy_p0 *phy_pg0,
+					 struct leapraid_exp_p1 *exp_pg1)
+{
+	if (exp_pg1 && phy_pg0)
+		return;
+
+	if (!exp_pg1 && !phy_pg0)
+		return;
+
+	phy->identify = card_phy->identify;
+	phy->identify.phy_identifier = card_phy->phy_id;
+	phy->negotiated_linkrate = phy_pg0 ?
+		leapraid_transport_convert_phy_link_rate(
+			phy_pg0->neg_link_rate &
+			LEAPRAID_SAS_NEG_LINK_RATE_MASK_PHYSICAL) :
+		 leapraid_transport_convert_phy_link_rate(
+			exp_pg1->neg_link_rate &
+			LEAPRAID_SAS_NEG_LINK_RATE_MASK_PHYSICAL);
+	phy->minimum_linkrate_hw = phy_pg0 ?
+		 leapraid_transport_convert_phy_link_rate(
+			phy_pg0->hw_link_rate &
+			LEAPRAID_SAS_HWRATE_MIN_RATE_MASK) :
+		 leapraid_transport_convert_phy_link_rate(
+			exp_pg1->hw_link_rate &
+			LEAPRAID_SAS_HWRATE_MIN_RATE_MASK);
+	phy->maximum_linkrate_hw = phy_pg0 ?
+		 leapraid_transport_convert_phy_link_rate(
+			phy_pg0->hw_link_rate >>
+			 LEAPRAID_SAS_NEG_LINK_RATE_SHIFT) :
+		 leapraid_transport_convert_phy_link_rate(
+			exp_pg1->hw_link_rate >>
+			 LEAPRAID_SAS_NEG_LINK_RATE_SHIFT);
+	phy->minimum_linkrate = phy_pg0 ?
+		 leapraid_transport_convert_phy_link_rate(
+			phy_pg0->p_link_rate &
+			LEAPRAID_SAS_PRATE_MIN_RATE_MASK) :
+		 leapraid_transport_convert_phy_link_rate(
+			exp_pg1->p_link_rate &
+			LEAPRAID_SAS_PRATE_MIN_RATE_MASK);
+	phy->maximum_linkrate = phy_pg0 ?
+		 leapraid_transport_convert_phy_link_rate(
+			phy_pg0->p_link_rate >>
+			 LEAPRAID_SAS_NEG_LINK_RATE_SHIFT) :
+		 leapraid_transport_convert_phy_link_rate(
+			exp_pg1->p_link_rate >>
+			 LEAPRAID_SAS_NEG_LINK_RATE_SHIFT);
+	phy->hostdata = card_phy->card_port;
+}
+
+void leapraid_transport_add_card_phy(struct leapraid_adapter *adapter,
+				     struct leapraid_card_phy *card_phy,
+				     struct leapraid_sas_phy_p0 *phy_pg0,
+				     struct device *parent_dev)
+{
+	struct sas_phy *phy;
+	int ret;
+
+	INIT_LIST_HEAD(&card_phy->port_siblings);
+	phy = sas_phy_alloc(parent_dev, card_phy->phy_id);
+	if (!phy) {
+		dev_err(&adapter->pdev->dev,
+			"%s: sas_phy_alloc failed!\n", __func__);
+		return;
+	}
+
+	if (leapraid_transport_set_identify(adapter, card_phy->hdl,
+					    &card_phy->identify)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Set PHY handle identify failed!\n", __func__);
+		sas_phy_free(phy);
+		return;
+	}
+
+	card_phy->attached_hdl = le16_to_cpu(phy_pg0->attached_dev_hdl);
+	if (card_phy->attached_hdl) {
+		ret = leapraid_transport_set_identify(
+				adapter,
+				card_phy->attached_hdl,
+				&card_phy->remote_identify);
+		if (ret) {
+			dev_err(&adapter->pdev->dev,
+				"%s: Set PHY attached hdl identify failed!\n",
+				__func__);
+			sas_phy_free(phy);
+			return;
+		}
+	}
+
+	leapraid_init_sas_or_exp_phy(adapter, card_phy, phy, phy_pg0, NULL);
+
+	if (sas_phy_add(phy)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: SAS PHY add failed!\n", __func__);
+		sas_phy_free(phy);
+		return;
+	}
+
+	card_phy->phy = phy;
+}
+
+int leapraid_transport_add_exp_phy(struct leapraid_adapter *adapter,
+				   struct leapraid_card_phy *card_phy,
+				   struct leapraid_exp_p1 *exp_pg1,
+				   struct device *parent_dev)
+{
+	struct sas_phy *phy;
+	int ret;
+
+	INIT_LIST_HEAD(&card_phy->port_siblings);
+	phy = sas_phy_alloc(parent_dev, card_phy->phy_id);
+	if (!phy) {
+		dev_err(&adapter->pdev->dev,
+			"%s: sas_phy_alloc failed!\n", __func__);
+		return -EFAULT;
+	}
+
+	if (leapraid_transport_set_identify(adapter, card_phy->hdl,
+					    &card_phy->identify)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Set PHY hdl identify failed!\n", __func__);
+		sas_phy_free(phy);
+		return -EFAULT;
+	}
+
+	card_phy->attached_hdl = le16_to_cpu(exp_pg1->attached_dev_hdl);
+	if (card_phy->attached_hdl) {
+		ret = leapraid_transport_set_identify(
+				adapter,
+				card_phy->attached_hdl,
+				&card_phy->remote_identify);
+		if (ret)
+			dev_warn(&adapter->pdev->dev,
+				 "%s: Set PHY attached hdl identify failed!\n",
+				 __func__);
+	}
+
+	leapraid_init_sas_or_exp_phy(adapter, card_phy, phy, NULL, exp_pg1);
+
+	if (sas_phy_add(phy)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: SAS PHY add failed!\n", __func__);
+		sas_phy_free(phy);
+		return -EFAULT;
+	}
+
+	card_phy->phy = phy;
+	return 0;
+}
+
+void leapraid_transport_update_links(
+		struct leapraid_adapter *adapter,
+		u64 sas_address, u16 hdl, u8 phy_index,
+		u8 link_rate, struct leapraid_card_port *target_card_port)
+{
+	struct leapraid_topo_node *topo_node;
+	struct leapraid_card_phy *card_phy;
+	struct leapraid_card_port *card_port;
+	unsigned long flags;
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.pcie_recovering)
+		return;
+
+	spin_lock_irqsave(&adapter->dev_topo.topo_node_lock, flags);
+	topo_node = leapraid_transport_topo_node_by_sas_addr(adapter,
+							     sas_address,
+							     target_card_port);
+	if (!topo_node) {
+		spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock,
+				       flags);
+		return;
+	}
+
+	card_phy = &topo_node->card_phy[phy_index];
+	card_phy->attached_hdl = hdl;
+	spin_unlock_irqrestore(&adapter->dev_topo.topo_node_lock, flags);
+
+	if (hdl && link_rate >= LEAPRAID_SAS_NEG_LINK_RATE_1_5) {
+		leapraid_transport_set_identify(adapter, hdl,
+						&card_phy->remote_identify);
+		if (topo_node->hdl <= adapter->dev_topo.card.phys_num &&
+		    adapter->adapter_attr.enable_mp) {
+			list_for_each_entry(card_port,
+					    &adapter->dev_topo.card_port_list,
+					    list) {
+				if (card_port->sas_address == sas_address &&
+				    card_port == target_card_port)
+					card_port->phy_mask |=
+						BIT(card_phy->phy_id);
+			}
+		}
+		leapraid_transport_attach_phy_to_port(
+			adapter,
+			topo_node,
+			card_phy,
+			card_phy->remote_identify.sas_address,
+			target_card_port);
+	} else {
+		memset(&card_phy->remote_identify, 0,
+		       sizeof(struct sas_identify));
+	}
+
+	if (card_phy->phy)
+		card_phy->phy->negotiated_linkrate =
+			leapraid_transport_convert_phy_link_rate(link_rate);
+}
+
+static int leapraid_dma_map_buffer(struct device *dev, struct bsg_buffer *buf,
+				   dma_addr_t *dma_addr,
+				   size_t *dma_len, void **p)
+{
+	if (buf->sg_cnt > 1) {
+		*p = dma_alloc_coherent(dev, buf->payload_len, dma_addr,
+					GFP_KERNEL);
+		if (!*p)
+			return -ENOMEM;
+
+		*dma_len = buf->payload_len;
+	} else {
+		if (!dma_map_sg(dev, buf->sg_list, 1, DMA_BIDIRECTIONAL)) {
+			dev_err(dev,
+				"%s: Failed to map sg buffer for dma\n",
+				__func__);
+			return -ENOMEM;
+		}
+
+		*dma_addr = sg_dma_address(buf->sg_list);
+		*dma_len = sg_dma_len(buf->sg_list);
+		*p = NULL;
+	}
+	return 0;
+}
+
+static void leapraid_dma_unmap_buffer(struct device *dev,
+				      struct bsg_buffer *buf,
+				      dma_addr_t dma_addr,
+				      void *p)
+{
+	if (p)
+		dma_free_coherent(dev, buf->payload_len, p, dma_addr);
+	else
+		dma_unmap_sg(dev, buf->sg_list, 1, DMA_BIDIRECTIONAL);
+}
+
+static void leapraid_build_smp_task(struct leapraid_adapter *adapter,
+				    struct sas_rphy *rphy,
+				    dma_addr_t h2c_dma_addr, size_t h2c_size,
+				    dma_addr_t c2h_dma_addr, size_t c2h_size)
+{
+	struct leapraid_smp_passthrough_req *smp_passthrough_req;
+	u16 inter_taskid;
+	void *psge;
+
+	inter_taskid = adapter->driver_cmds.transport_cmd.inter_taskid;
+	smp_passthrough_req = leapraid_get_task_desc(adapter, inter_taskid);
+	memset(smp_passthrough_req, 0, sizeof(*smp_passthrough_req));
+
+	smp_passthrough_req->func = LEAPRAID_FUNC_SMP_PASSTHROUGH;
+	smp_passthrough_req->physical_port =
+		leapraid_transport_get_port_id_by_rphy(adapter, rphy);
+	smp_passthrough_req->sas_address = (rphy) ?
+		cpu_to_le64(rphy->identify.sas_address) :
+		cpu_to_le64(adapter->dev_topo.card.sas_address);
+	smp_passthrough_req->req_data_len =
+		cpu_to_le16(h2c_size - LEAPRAID_SMP_FRAME_HEADER_SIZE);
+	psge = &smp_passthrough_req->sgl;
+	leapraid_build_ieee_sg(adapter, psge, h2c_dma_addr,
+			       h2c_size - LEAPRAID_SMP_FRAME_HEADER_SIZE,
+			       c2h_dma_addr,
+			       c2h_size - LEAPRAID_SMP_FRAME_HEADER_SIZE);
+}
+
+static int leapraid_send_smp_req(struct leapraid_adapter *adapter)
+{
+	const struct leapraid_smp_passthrough_req *smp_passthrough_req;
+	u16 inter_taskid;
+
+	dev_dbg(&adapter->pdev->dev,
+		"%s: Sending smp request\n", __func__);
+	inter_taskid = adapter->driver_cmds.transport_cmd.inter_taskid;
+	smp_passthrough_req = leapraid_get_task_desc(adapter, inter_taskid);
+	init_completion(&adapter->driver_cmds.transport_cmd.done);
+	leapraid_fire_task(adapter, inter_taskid);
+	wait_for_completion_timeout(&adapter->driver_cmds.transport_cmd.done,
+				    LEAPRAID_TRANSPORT_CMD_TIMEOUT * HZ);
+	if (!(adapter->driver_cmds.transport_cmd.status & LEAPRAID_CMD_DONE)) {
+		dev_err(&adapter->pdev->dev, "%s: timeout, st=0x%x\n",
+			__func__, adapter->driver_cmds.transport_cmd.status);
+		leapraid_log_req_context(adapter, inter_taskid,
+					 smp_passthrough_req);
+		if (!(adapter->driver_cmds.transport_cmd.status &
+		      LEAPRAID_CMD_RESET)) {
+			dev_dbg(&adapter->pdev->dev,
+				"%s:%d: call hard_reset\n",
+				__func__, __LINE__);
+			leapraid_hard_reset_handler(adapter, FULL_RESET);
+			return -ETIMEDOUT;
+		}
+	}
+
+	dev_dbg(&adapter->pdev->dev, "%s: SMP request complete\n", __func__);
+	if (!(adapter->driver_cmds.transport_cmd.status &
+	      LEAPRAID_CMD_REPLY_VALID)) {
+		dev_err(&adapter->pdev->dev,
+			"%s: SMP request no reply\n", __func__);
+		return -ENXIO;
+	}
+
+	return 0;
+}
+
+static void leapraid_handle_smp_rep(struct leapraid_adapter *adapter,
+				    struct bsg_job *job, void *addr_in,
+				    unsigned int *reslen)
+{
+	struct leapraid_smp_passthrough_rep *smp_passthrough_rep;
+
+	smp_passthrough_rep =
+		(void *)&adapter->driver_cmds.transport_cmd.reply;
+
+	dev_dbg(&adapter->pdev->dev, "%s: Response data len=%d\n",
+		__func__, le16_to_cpu(smp_passthrough_rep->resp_data_len));
+
+	memcpy(job->reply, smp_passthrough_rep, sizeof(*smp_passthrough_rep));
+	job->reply_len = sizeof(*smp_passthrough_rep);
+	*reslen = le16_to_cpu(smp_passthrough_rep->resp_data_len);
+
+	if (addr_in)
+		sg_copy_from_buffer(job->reply_payload.sg_list,
+				    job->reply_payload.sg_cnt, addr_in,
+				    job->reply_payload.payload_len);
+}
+
+static void leapraid_transport_smp_handler(struct bsg_job *job,
+					   struct Scsi_Host *shost,
+					   struct sas_rphy *rphy)
+{
+	struct leapraid_adapter *adapter = shost_priv(shost);
+	dma_addr_t c2h_dma_addr;
+	dma_addr_t h2c_dma_addr;
+	void *addr_in = NULL;
+	void *addr_out = NULL;
+	size_t c2h_size;
+	size_t h2c_size;
+	int rc;
+	unsigned int reslen = 0;
+
+	if (adapter->access_ctrl.shost_recovering ||
+	    adapter->access_ctrl.pcie_recovering) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Failed, shost_recovering=%d pcie_recovering=%d\n",
+			__func__,
+			adapter->access_ctrl.shost_recovering,
+			adapter->access_ctrl.pcie_recovering);
+		rc = -EFAULT;
+		goto exit_bsg_job;
+	}
+
+	rc = mutex_lock_interruptible(&adapter->driver_cmds
+				      .transport_cmd.mutex);
+	if (rc)
+		goto exit_bsg_job;
+
+	adapter->driver_cmds.transport_cmd.status = LEAPRAID_CMD_PENDING;
+	rc = leapraid_dma_map_buffer(&adapter->pdev->dev,
+				     &job->request_payload,
+				     &h2c_dma_addr, &h2c_size, &addr_out);
+	if (rc)
+		goto release_lock;
+
+	if (h2c_size < LEAPRAID_SMP_FRAME_HEADER_SIZE) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid SMP request payload size=%zu\n",
+			__func__, h2c_size);
+		rc = -EINVAL;
+		goto free_req_buf;
+	}
+
+	if (addr_out)
+		sg_copy_to_buffer(job->request_payload.sg_list,
+				  job->request_payload.sg_cnt, addr_out,
+				  job->request_payload.payload_len);
+
+	rc = leapraid_dma_map_buffer(&adapter->pdev->dev, &job->reply_payload,
+				     &c2h_dma_addr, &c2h_size, &addr_in);
+	if (rc)
+		goto free_req_buf;
+
+	if (c2h_size < LEAPRAID_SMP_FRAME_HEADER_SIZE) {
+		dev_err(&adapter->pdev->dev,
+			"%s: Invalid SMP reply payload size=%zu\n",
+			__func__, c2h_size);
+		rc = -EINVAL;
+		goto free_rep_buf;
+	}
+
+	rc = leapraid_check_adapter_is_op(adapter, LEAPRAID_DB_WAIT_OP_SHORT,
+					  __func__);
+	if (rc)
+		goto free_rep_buf;
+
+	leapraid_build_smp_task(adapter, rphy, h2c_dma_addr,
+				h2c_size, c2h_dma_addr, c2h_size);
+
+	rc = leapraid_send_smp_req(adapter);
+	if (rc)
+		goto free_rep_buf;
+
+	leapraid_handle_smp_rep(adapter, job, addr_in, &reslen);
+
+free_rep_buf:
+	leapraid_dma_unmap_buffer(&adapter->pdev->dev, &job->reply_payload,
+				  c2h_dma_addr, addr_in);
+free_req_buf:
+	leapraid_dma_unmap_buffer(&adapter->pdev->dev, &job->request_payload,
+				  h2c_dma_addr, addr_out);
+release_lock:
+	adapter->driver_cmds.transport_cmd.status = LEAPRAID_CMD_NOT_USED;
+	mutex_unlock(&adapter->driver_cmds.transport_cmd.mutex);
+exit_bsg_job:
+	bsg_job_done(job, rc, reslen);
+}
+
+struct sas_function_template leapraid_transport_functions = {
+	.smp_handler = leapraid_transport_smp_handler,
+};
+
+struct scsi_transport_template *leapraid_transport_template;
author	Linus Torvalds <torvalds@linux-foundation.org>	2026-09-20 08:15:23 -0700
committer	Linus Torvalds <torvalds@linux-foundation.org>	2026-09-20 08:15:23 -0700
commit	bdab18633a302c82b5e5d5dbc4e38b0baf9c0837 (patch)
tree	0446500af9a76d4e8b2a72a0a41391ceede73a10 /drivers/scsi/leapraid
download	linux-stable-bdab18633a302c82b5e5d5dbc4e38b0baf9c0837.tar.gz linux-stable-bdab18633a302c82b5e5d5dbc4e38b0baf9c0837.zip