path: root/drivers/scsi/ibmvscsi/ibmvfc-nvme.c

/* SPDX-License-Identifier: GPL-2.0-or-later */

/*
 * ibmvfc-nvme.c -- IBM Power Virtual Fibre Channel NVMeoF HBA driver
 *
 * Written By: Tyrel Datwyler <tyreld@linux.ibm.com>, IBM Corporation
 *
 * Copyright (C) IBM Corporation, 2026
 */

#include <linux/dmapool.h>
#include <scsi/scsi_transport_fc.h>

#include "ibmvfc-nvme.h"

static unsigned int default_timeout = IBMVFC_DEFAULT_TIMEOUT;

static void ibmvfc_nvme_localport_delete(struct nvme_fc_local_port *lport)
{
	struct ibmvfc_host *vhost = lport->private;

	vhost->nvme_local_port = NULL;
	complete(&vhost->nvme_delete_done);
}

static void ibmvfc_nvme_remoteport_delete(struct nvme_fc_remote_port *rport)
{
	struct ibmvfc_target *tgt = rport->private;

	tgt->nvme_remote_port = NULL;
	complete(&tgt->nvme_delete_done);
}

static int ibmvfc_nvme_create_queue(struct nvme_fc_local_port *lport, unsigned int qidx,
				    u16 qsize, void **handle)
{
	struct ibmvfc_host *vhost = lport->private;
	struct ibmvfc_nvme_qhandle *qhandle;

	if (!vhost->nvme_scrqs.active_queues)
		return -ENODEV;

	qhandle = kzalloc_obj(struct ibmvfc_nvme_qhandle);
	if (!qhandle)
		return -ENOMEM;

	qhandle->cpu_id = raw_smp_processor_id();
	qhandle->qidx = qidx;

	/* Admin and first IO queue are both mapped to index 0 */
	if (qidx)
		qhandle->index = (qidx - 1) % vhost->nvme_scrqs.active_queues;
	else
		qhandle->index = qidx;

	qhandle->queue = &vhost->nvme_scrqs.scrqs[qhandle->index];

	*handle = qhandle;
	return 0;
}

static void ibmvfc_nvme_delete_queue(struct nvme_fc_local_port *lport, unsigned int qidx,
				     void *handle)
{
	kfree(handle);
}

static void ibmvfc_ls_req_done(struct ibmvfc_event *evt)
{
	struct ibmvfc_target *tgt = evt->tgt;
	struct ibmvfc_passthru_mad *mad = &evt->xfer_iu->passthru;
	struct fcnvme_ls_rqst_w0 *ls_rqst;
	struct fcnvme_ls_cr_assoc_acc *ls_resp;
	u32 status = be16_to_cpu(mad->common.status);
	int rc = 0;

	ls_rqst = (struct fcnvme_ls_rqst_w0 *)evt->ls_req->rqstaddr;
	ls_resp = (struct fcnvme_ls_cr_assoc_acc *)evt->ls_req->rspaddr;

	switch (status) {
	case IBMVFC_MAD_SUCCESS:
		tgt_dbg(tgt, "ls_req succeeded\n");
		if ((ls_rqst->ls_cmd == FCNVME_LS_CREATE_ASSOCIATION) &&
		    (ls_resp->hdr.w0.ls_cmd == FCNVME_LS_ACC)) {
			tgt->assoc_id = be64_to_cpu(ls_resp->associd.association_id);
			tgt_dbg(tgt, "assoc_id 0x%llx\n", tgt->assoc_id);
		}
		break;
	case IBMVFC_MAD_DRIVER_FAILED:
		break;
	case IBMVFC_MAD_FAILED:
	default:
		tgt_info(tgt, "ls_req failed: %s (%x:%x) rc=0x%02X\n",
			 ibmvfc_get_cmd_error(be16_to_cpu(mad->iu.status), be16_to_cpu(mad->iu.error)),
			 be16_to_cpu(mad->iu.status), be16_to_cpu(mad->iu.error), status);
		break;
	}

	if (status)
		rc = -EIO;

	evt->ls_req->done(evt->ls_req, rc);

	kref_put(&tgt->kref, ibmvfc_release_tgt);
	ibmvfc_free_event(evt);
}

static void ibmvfc_init_ls_req(struct ibmvfc_event *evt, struct nvmefc_ls_req *ls_req)
{
	struct ibmvfc_passthru_mad *mad = &evt->iu.passthru;

	memset(mad, 0, sizeof(*mad));
	mad->common.version = cpu_to_be32(2);
	mad->common.opcode = cpu_to_be32(IBMVFC_NVMF_PASSTHRU);
	mad->common.length = cpu_to_be16(sizeof(*mad) - sizeof(mad->fc_iu) - sizeof(mad->iu));
	mad->cmd_ioba.va = cpu_to_be64((u64)be64_to_cpu(evt->crq.ioba) +
				       offsetof(struct ibmvfc_passthru_mad, iu));
	mad->cmd_ioba.len = cpu_to_be32(sizeof(mad->iu));
	mad->iu.cmd_len = cpu_to_be32(ls_req->rqstlen);
	mad->iu.rsp_len = cpu_to_be32(ls_req->rsplen);
	mad->iu.cmd.va = cpu_to_be64(ls_req->rqstdma);
	mad->iu.cmd.len = cpu_to_be32(ls_req->rqstlen);
	mad->iu.rsp.va = cpu_to_be64(ls_req->rspdma);
	mad->iu.rsp.len = cpu_to_be32(ls_req->rsplen);
}

static int ibmvfc_nvme_ls_req(struct nvme_fc_local_port *lport,
			      struct nvme_fc_remote_port *rport,
			      struct nvmefc_ls_req *ls_req)
{
	struct ibmvfc_host *vhost = lport->private;
	struct ibmvfc_target *tgt = rport->private;
	struct ibmvfc_passthru_mad *mad;
	struct ibmvfc_event *evt;

	kref_get(&tgt->kref);
	evt = ibmvfc_get_event(&vhost->crq);
	if (!evt) {
		kref_put(&tgt->kref, ibmvfc_release_tgt);
		return -EBUSY;
	}

	ibmvfc_init_event(evt, ibmvfc_ls_req_done, IBMVFC_MAD_FORMAT);
	evt->tgt = tgt;
	evt->ls_req = ls_req;
	ls_req->private = evt;

	ibmvfc_init_ls_req(evt, ls_req);
	mad = &evt->iu.passthru;
	mad->iu.flags = cpu_to_be32(IBMVFC_FC4_LS_DSC_CTRL);
	mad->iu.scsi_id = cpu_to_be64(tgt->scsi_id);
	mad->iu.cancel_key = cpu_to_be32((u64)evt);
	mad->iu.target_wwpn = cpu_to_be64(tgt->wwpn);

	ibmvfc_dbg(vhost, "nvme_ls_req\n");
	if (ibmvfc_send_event(evt, vhost, IBMVFC_FC4_LS_PLUS_CANCEL_TIMEOUT)) {
		kref_put(&tgt->kref, ibmvfc_release_tgt);
		return -ENXIO;
	}

	return 0;
}

static void ibmvfc_sync_nvme_completion(struct ibmvfc_event *evt)
{
	/* copy the response back */
	if (evt->sync_iu)
		*evt->sync_iu = *evt->xfer_iu;

	complete(&evt->comp);
}

static void ibmvfc_init_ls_abort(struct ibmvfc_event *evt, struct nvmefc_ls_req *ls_abort)
{
	struct ibmvfc_tmf *tmf;
	struct ibmvfc_event *abt_evt = ls_abort->private;
	struct ibmvfc_target *tgt = abt_evt->tgt;
	struct ibmvfc_host *vhost = evt->vhost;

	tmf = &evt->iu.tmf;
	memset(tmf, 0, sizeof(*tmf));
	tmf->common.version = cpu_to_be32(2);
	tmf->target_wwpn = cpu_to_be64(tgt->wwpn);
	tmf->common.opcode = cpu_to_be32(IBMVFC_NVMF_TMF_MAD);
	tmf->common.length = cpu_to_be16(sizeof(*tmf));
	if (vhost->state != IBMVFC_ACTIVE)
		if (!ibmvfc_check_caps(vhost, IBMVFC_CAN_SUPPRESS_ABTS))
			tmf->flags = cpu_to_be32(IBMVFC_TMF_SUPPRESS_ABTS);
	tmf->cancel_key = cpu_to_be32((u64)abt_evt);
	tmf->my_cancel_key = cpu_to_be32((u64)evt);
	tmf->assoc_id = cpu_to_be64(tgt->assoc_id);

	init_completion(&evt->comp);
}

static void ibmvfc_nvme_ls_abort(struct nvme_fc_local_port *lport,
				struct nvme_fc_remote_port *rport,
				struct nvmefc_ls_req *ls_abort)
{
	struct ibmvfc_host *vhost = lport->private;
	struct ibmvfc_target *tgt = rport->private;
	struct ibmvfc_event *evt;
	union ibmvfc_iu rsp;
	unsigned long flags;
	u16 status;

	evt = ibmvfc_get_event(&vhost->crq);
	if (!vhost->logged_in || !evt)
		return;

	spin_lock_irqsave(vhost->host->host_lock, flags);
	kref_get(&tgt->kref);
	ibmvfc_init_event(evt, ibmvfc_sync_nvme_completion, IBMVFC_MAD_FORMAT);
	ibmvfc_init_ls_abort(evt, ls_abort);
	evt->sync_iu = &rsp;

	if (ibmvfc_send_event(evt, vhost, default_timeout))
		goto out;

	spin_unlock_irqrestore(vhost->host->host_lock, flags);

	wait_for_completion(&evt->comp);
	status = be16_to_cpu(rsp.mad_common.status);
	spin_lock_irqsave(vhost->host->host_lock, flags);
	ibmvfc_free_event(evt);
out:
	spin_unlock_irqrestore(vhost->host->host_lock, flags);
	ibmvfc_dbg(vhost, "ls_abort: cancel failed with rc=%x\n", status);
	kref_put(&tgt->kref, ibmvfc_release_tgt);
}

static void ibmvfc_nvme_done(struct ibmvfc_event *evt)
{
	struct ibmvfc_cmd *vfc_cmd = &evt->xfer_iu->cmd;
	struct nvmefc_fcp_req *fcp_req = evt->fcp_req;
	struct nvme_fc_ersp_iu *ersp = (struct nvme_fc_ersp_iu *)fcp_req->rspaddr;
	struct nvme_completion *cqe = &ersp->cqe;
	struct nvme_command *sqe = &((struct nvme_fc_cmd_iu *)fcp_req->cmdaddr)->sqe;

	ibmvfc_dbg(evt->vhost, "fc_done: (%x:%x)\n", be16_to_cpu(vfc_cmd->status),
		   be16_to_cpu(vfc_cmd->error));
	ibmvfc_dbg(evt->vhost, "fc_done: cmdlen: %d, rsplen %d, payload_len %d\n",
		   fcp_req->cmdlen, fcp_req->rsplen, fcp_req->payload_length);

	fcp_req->status = 0;
	if (!vfc_cmd->status) {
		fcp_req->rcv_rsplen = NVME_FC_SIZEOF_ZEROS_RSP;
		fcp_req->transferred_length = fcp_req->payload_length;
	} else if (be16_to_cpu(vfc_cmd->status) & IBMVFC_FC_NVME_STATUS) {
		fcp_req->rcv_rsplen = sizeof(struct nvme_fc_ersp_iu);
		fcp_req->transferred_length = be32_to_cpu(ersp->xfrd_len);
		if (be16_to_cpu(vfc_cmd->error) & IBMVFC_NVMS_VALID_NODMA_CQE)
			cqe->command_id = sqe->common.command_id;
	} else {
		fcp_req->rcv_rsplen = 0;
		fcp_req->transferred_length = 0;
		fcp_req->status = NVME_SC_INTERNAL;
	}

	fcp_req->done(fcp_req);
	ibmvfc_free_event(evt);
}

static struct ibmvfc_cmd *ibmvfc_nvme_init_vfc_cmd(struct ibmvfc_event *evt,
						   struct nvme_fc_remote_port *rport,
						   struct nvmefc_fcp_req *fcp_req)
{
	struct ibmvfc_target *tgt = rport->private;
	struct ibmvfc_cmd *vfc_cmd = &evt->iu.cmd;

	memset(vfc_cmd, 0, sizeof(*vfc_cmd));

	vfc_cmd->resp.va = cpu_to_be64(fcp_req->rspdma);
	vfc_cmd->resp.len = cpu_to_be32(fcp_req->rsplen);
	vfc_cmd->frame_type = cpu_to_be32(IBMVFC_NVME_FCP_TYPE);
	vfc_cmd->flags |= cpu_to_be16(IBMVFC_NVMEOF_PROTOCOL);
	vfc_cmd->payload_len = cpu_to_be32(fcp_req->cmdlen);
	vfc_cmd->resp_len = cpu_to_be32(fcp_req->rsplen);
	vfc_cmd->cancel_key = cpu_to_be32((u64)evt);
	vfc_cmd->target_wwpn = cpu_to_be64(rport->port_name);
	vfc_cmd->tgt_scsi_id = cpu_to_be64(rport->port_id);
	vfc_cmd->assoc_id = cpu_to_be64(tgt->assoc_id);

	memcpy(&vfc_cmd->v3nvme.iu, fcp_req->cmdaddr, fcp_req->cmdlen);

	return vfc_cmd;
}

static void ibmvfc_nvme_map_sg_list(struct nvmefc_fcp_req *fcp_req,
				    struct srp_direct_buf *md)
{
	int i;
	struct scatterlist *sg;

	for_each_sg(fcp_req->first_sgl, sg, fcp_req->sg_cnt, i) {
		md[i].va = cpu_to_be64(sg_dma_address(sg));
		md[i].len = cpu_to_be32(sg_dma_len(sg));
		md[i].key = 0;
	}
}

static int ibmvfc_nvme_map_sg_data(struct nvmefc_fcp_req *fcp_req,
				    struct ibmvfc_event *evt,
				    struct ibmvfc_cmd *vfc_cmd)
{
	struct srp_direct_buf *data = &vfc_cmd->ioba;
	struct ibmvfc_host *vhost = evt->vhost;

	if (!fcp_req->sg_cnt) {
		vfc_cmd->flags |= cpu_to_be16(IBMVFC_NO_MEM_DESC);
		return 0;
	}

	if (fcp_req->io_dir == NVMEFC_FCP_WRITE)
		vfc_cmd->flags |= cpu_to_be16(IBMVFC_WRITE);
	else
		vfc_cmd->flags |= cpu_to_be16(IBMVFC_READ);

	if (fcp_req->sg_cnt == 1) {
		ibmvfc_nvme_map_sg_list(fcp_req, data);
		return 0;
	}

	vfc_cmd->flags |= cpu_to_be16(IBMVFC_SCATTERLIST);

	if (!evt->ext_list) {
		evt->ext_list = dma_pool_alloc(vhost->sg_pool, GFP_ATOMIC,
					       &evt->ext_list_token);

		if (!evt->ext_list)
			return -ENOMEM;
	}

	ibmvfc_nvme_map_sg_list(fcp_req, evt->ext_list);

	data->va = cpu_to_be64(evt->ext_list_token);
	data->len = cpu_to_be32(fcp_req->sg_cnt * sizeof(struct srp_direct_buf));
	data->key = 0;
	return 0;
}

static int ibmvfc_nvme_fcp_io(struct nvme_fc_local_port *lport,
			      struct nvme_fc_remote_port *rport,
			      void *hw_queue_handle,
			      struct nvmefc_fcp_req *fcp_req)
{
	struct ibmvfc_host *vhost = lport->private;
	struct ibmvfc_nvme_qhandle *qhandle = hw_queue_handle;
	struct ibmvfc_cmd *vfc_cmd;
	struct ibmvfc_event *evt;
	int rc;

	ibmvfc_dbg(vhost, "nvme_fcp_io\n");
	evt = ibmvfc_get_event(qhandle->queue);
	if (!evt)
		return -EBUSY;

	evt->hwq = qhandle->index;
	ibmvfc_dbg(vhost, "vfc-nvme-mq-%d\n", evt->hwq);

	ibmvfc_init_event(evt, ibmvfc_nvme_done, IBMVFC_CMD_FORMAT);
	evt->fcp_req = fcp_req;
	fcp_req->private = evt;

	vfc_cmd = ibmvfc_nvme_init_vfc_cmd(evt, rport, fcp_req);

	vfc_cmd->correlation = cpu_to_be64((u64)evt);

	if (likely(!(rc = ibmvfc_nvme_map_sg_data(fcp_req, evt, vfc_cmd))))
		return ibmvfc_send_event(evt, vhost, 0);

	ibmvfc_free_event(evt);

	return rc;
}

static void ibmvfc_init_fcp_abort(struct ibmvfc_event *evt,
				  struct nvmefc_fcp_req *abort_req)
{
	struct ibmvfc_tmf *tmf;
	struct ibmvfc_event *abt_evt = abort_req->private;
	struct ibmvfc_target *tgt = abt_evt->tgt;

	tmf = &evt->iu.tmf;
	memset(tmf, 0, sizeof(*tmf));
	tmf->common.version = cpu_to_be32(2);
	tmf->common.opcode = cpu_to_be32(IBMVFC_NVMF_TMF_MAD);
	tmf->common.length = cpu_to_be16(sizeof(*tmf));
	tmf->flags = cpu_to_be32(IBMVFC_TMF_ABORT_TASK | IBMVFC_TMF_NVMF_ASSOC);
	tmf->cancel_key = cpu_to_be32((u64)abt_evt);
	tmf->my_cancel_key = cpu_to_be32((u64)evt);
	tmf->target_wwpn = cpu_to_be64(tgt->wwpn);
	tmf->assoc_id = cpu_to_be64(tgt->assoc_id);
	tmf->task_tag = cpu_to_be64((u64)abt_evt);

	init_completion(&evt->comp);
}

static void ibmvfc_nvme_fcp_abort(struct nvme_fc_local_port *lport,
				  struct nvme_fc_remote_port *rport,
				  void *hw_queue_handle,
				  struct nvmefc_fcp_req *abort_req)
{
	struct ibmvfc_host *vhost = lport->private;
	struct ibmvfc_target *tgt = rport->private;
	struct ibmvfc_event *evt, *abt_evt = abort_req->private;
	struct ibmvfc_queue *queue;
	union ibmvfc_iu rsp;
	unsigned long flags;
	u16 status = 0;

	if (!abt_evt)
		return;

	queue = abt_evt->queue;
	if (!vhost->logged_in || !queue)
		return;

	evt = ibmvfc_get_event(queue);
	if (!evt)
		return;

	spin_lock_irqsave(queue->q_lock, flags);
	kref_get(&tgt->kref);
	ibmvfc_init_event(evt, ibmvfc_sync_nvme_completion, IBMVFC_MAD_FORMAT);
	ibmvfc_init_fcp_abort(evt, abort_req);
	evt->sync_iu = &rsp;

	if (ibmvfc_send_event(evt, vhost, default_timeout))
		goto out;

	spin_unlock_irqrestore(queue->q_lock, flags);

	wait_for_completion(&evt->comp);
	status = be16_to_cpu(rsp.mad_common.status);

	spin_lock_irqsave(queue->q_lock, flags);
	ibmvfc_free_event(evt);
out:
	spin_unlock_irqrestore(queue->q_lock, flags);

	if (status)
		ibmvfc_dbg(vhost, "fcp_abort: cancel failed with rc=%x\n", status);

	kref_put(&tgt->kref, ibmvfc_release_tgt);
}

static struct nvme_fc_port_template ibmvfc_nvme_fc_transport = {
	.localport_delete	= ibmvfc_nvme_localport_delete,
	.remoteport_delete	= ibmvfc_nvme_remoteport_delete,
	.create_queue		= ibmvfc_nvme_create_queue,
	.delete_queue		= ibmvfc_nvme_delete_queue,
	.ls_req			= ibmvfc_nvme_ls_req,
	.ls_abort		= ibmvfc_nvme_ls_abort,
	.fcp_io			= ibmvfc_nvme_fcp_io,
	.fcp_abort		= ibmvfc_nvme_fcp_abort,
	.map_queues		= NULL,
	.max_hw_queues		= IBMVFC_NVME_HW_QUEUES,
	.max_sgl_segments	= 1024,
	.max_dif_sgl_segments	= 64,
	.dma_boundary		= 0xFFFFFFFF,
	.local_priv_sz		= sizeof(struct ibmvfc_host *),
	.remote_priv_sz		= sizeof(struct ibmvfc_target *),
	.lsrqst_priv_sz		= sizeof(struct ibmvfc_event *),
	.fcprqst_priv_sz	= sizeof(struct ibmvfc_event *),
};

int ibmvfc_nvme_register_remoteport(struct ibmvfc_target *tgt)
{
	struct ibmvfc_host *vhost = tgt->vhost;
	struct nvme_fc_port_info pinfo;
	int rc;

	if (!IS_ENABLED(CONFIG_NVME_FC))
		return 0;

	if (!vhost->nvme_local_port) {
		dev_err(vhost->dev, "Attempt to register NVMe fc remoteport without valid localport\n");
		return -EINVAL;
	}

	memset(&pinfo, 0, sizeof(struct nvme_fc_port_info));
	pinfo.node_name = tgt->ids.node_name;
	pinfo.port_name = tgt->ids.port_name;
	pinfo.port_id = tgt->ids.port_id;
	pinfo.port_role = FC_PORT_ROLE_NVME_TARGET;

	rc = nvme_fc_register_remoteport(vhost->nvme_local_port, &pinfo,
					 &tgt->nvme_remote_port);

	if (!rc) {
		ibmvfc_log(vhost, 2, "register_remoteport: traddr=nn-0x%llx:pn-0x%llx PortID:%x\n",
			   pinfo.node_name, pinfo.port_name, pinfo.port_id);
		tgt->nvme_remote_port->private = tgt;
	}

	return rc;
}

void ibmvfc_nvme_unregister_remoteport(struct ibmvfc_target *tgt)
{
	struct ibmvfc_host *vhost = tgt->vhost;
	struct nvme_fc_remote_port *rport = tgt->nvme_remote_port;
	int rc;

	if (!IS_ENABLED(CONFIG_NVME_FC))
		return;

	if (!tgt->nvme_remote_port)
		return;

	ibmvfc_log(vhost, 2, "unregister_remoteport: traddr=nn-0x%llx:pn-0x%llx PortID:%x\n",
		   rport->node_name, rport->port_name, rport->port_id);
	init_completion(&tgt->nvme_delete_done);
	rc = nvme_fc_unregister_remoteport(tgt->nvme_remote_port);

	if (!rc) {
		wait_for_completion(&tgt->nvme_delete_done);
	}
}

int ibmvfc_nvme_register(struct ibmvfc_host *vhost)
{
	struct nvme_fc_port_info pinfo;
	int rc;

	if (!IS_ENABLED(CONFIG_NVME_FC))
		return 0;

	pinfo.node_name = fc_host_node_name(vhost->host);
	pinfo.port_name = fc_host_port_name(vhost->host);
	pinfo.port_id = fc_host_port_id(vhost->host);
	pinfo.port_role = FC_PORT_ROLE_NVME_INITIATOR;
	pinfo.dev_loss_tmo = 0;

	rc = nvme_fc_register_localport(&pinfo, &ibmvfc_nvme_fc_transport,
					get_device(vhost->dev),
					&vhost->nvme_local_port);

	if (!rc) {
		ibmvfc_log(vhost, 2, "register_localport: host-traddr=nn-0x%llx:pn-0x%llx on portID:%x\n",
			   pinfo.node_name, pinfo.port_name, pinfo.port_id);
		vhost->nvme_local_port->private = vhost;
	} else {
		dev_err(vhost->dev, "Failed to register NVMe fc localport (%d)\n", rc);
		put_device(vhost->dev);
	}

	return rc;
}

void ibmvfc_nvme_unregister(struct ibmvfc_host *vhost)
{
	int rc;

	if (!IS_ENABLED(CONFIG_NVME_FC))
		return;

	if (vhost->nvme_local_port) {
		ibmvfc_log(vhost, 2, "unregister_localport: host-traddr=nn-0x%llx:pn-0x%llx on portID:%x\n",
			   vhost->nvme_local_port->node_name,
			   vhost->nvme_local_port->port_name,
			   vhost->nvme_local_port->port_id);
		init_completion(&vhost->nvme_delete_done);
		rc = nvme_fc_unregister_localport(vhost->nvme_local_port);
		if (!rc) {
			wait_for_completion(&vhost->nvme_delete_done);
		} else
			dev_err(vhost->dev, "Failed to unregister NVMe fc localport (%d)\n", rc);

		put_device(vhost->dev);
	}
}