clients/iser/iser_cq.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/types.h>
#include <sys/ddi.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <sys/sunddi.h>
#include <sys/sdt.h>
#include <sys/ib/ibtl/ibti.h>
#include <sys/ib/ibtl/ibtl_types.h>

#include <sys/ib/clients/iser/iser.h>

/*
 * iser_cq.c
 *    Routines for completion queue handlers for iSER.
 */
static void iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg);
int iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg);
static int iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl,
    iser_chan_t *iser_chan);
static int iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl,
    iser_chan_t *iser_chan);

void
iser_ib_sendcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
{
	iser_chan_t	*iser_chan;
	ibt_status_t	status;

	iser_chan = (iser_chan_t *)arg;

	/*
	 * Poll for work request completion while successful. If the
	 * queue empties or otherwise becomes invalid, stop polling.
	 */
	do {
		status = iser_ib_poll_send_completions(cq_hdl, iser_chan);
	} while (status == IBT_SUCCESS);

	if (status == IBT_CQ_EMPTY) {
		/* We've emptied the CQ, rearm it before we're done here */
		status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
		if (status != IBT_SUCCESS) {
			/* Unexpected error */
			ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: "
			    "ibt_enable_cq_notify error (%d)", status);
			return;
		}

		/* Now, check for more completions after the rearm */
		do {
			status = iser_ib_poll_send_completions(
			    cq_hdl, iser_chan);
		} while (status == IBT_SUCCESS);
	}
}

static int
iser_ib_poll_send_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
{
	ibt_wc_t	wc[ISER_IB_SCQ_POLL_MAX];
	ibt_wrid_t	wrid;
	idm_buf_t	*idb = NULL;
	idm_task_t	*idt = NULL;
	iser_wr_t	*wr = NULL;
	int		i;
	uint_t		npoll = 0;
	ibt_status_t	status;
	iser_conn_t	*iser_conn;
	idm_status_t	idm_status;
	iser_mr_t	*mr;

	iser_conn = iser_chan->ic_conn;

	/* Poll ISER_IB_SCQ_POLL_MAX completions from the CQ */
	status = ibt_poll_cq(cq_hdl, wc, ISER_IB_SCQ_POLL_MAX, &npoll);

	if (status != IBT_SUCCESS) {
		if (status != IBT_CQ_EMPTY) {
			/* Unexpected error */
			ISER_LOG(CE_NOTE, "iser_ib_sendcq_handler: ibt_poll_cq "
			    "unexpected error (%d)", status);
		}
		/* CQ is empty. Either way, move along... */
		return (status);
	}

	/*
	 * Handle each of the completions we've polled
	 */
	for (i = 0; i < npoll; i++) {

		DTRACE_PROBE3(iser__send__cqe, iser_chan_t *, iser_chan,
		    ibt_wc_t *, &wc[i], ibt_wc_status_t, wc[i].wc_status);

		/* Grab the wrid of the completion */
		wrid = wc[i].wc_id;

		/* Decrement this channel's SQ posted count */
		mutex_enter(&iser_chan->ic_sq_post_lock);
		iser_chan->ic_sq_post_count--;
		mutex_exit(&iser_chan->ic_sq_post_lock);

		/* Pull in the wr handle */
		wr = (iser_wr_t *)(uintptr_t)wrid;
		ASSERT(wr != NULL);

		/* Set an idm_status for return to IDM */
		idm_status = (wc[i].wc_status == IBT_WC_SUCCESS) ?
		    IDM_STATUS_SUCCESS : IDM_STATUS_FAIL;

		/*
		 * A non-success status here indicates the QP went
		 * into an error state while this WR was being
		 * processed. This can also happen when the
		 * channel is closed on the remote end. Clean up
		 * the resources, then push CE_TRANSPORT_FAIL
		 * into IDM.
		 */
		if (wc[i].wc_status != IBT_WC_SUCCESS) {
			/*
			 * Free the resources attached to this
			 * completion.
			 */
			if (wr->iw_msg != NULL) {
				/* Free iser_msg handle */
				iser_msg_free(wr->iw_msg);
			}

			if (wr->iw_pdu != NULL) {
				/* Complete the PDU */
				idm_pdu_complete(wr->iw_pdu, idm_status);
			}

			if (wr->iw_buf != NULL) {
				/* Invoke buffer callback */
				idb = wr->iw_buf;
				mr = ((iser_buf_t *)
				    idb->idb_buf_private)->iser_mr;
#ifdef DEBUG
				bcopy(&wc[i],
				    &((iser_buf_t *)idb->idb_buf_private)->
				    buf_wc, sizeof (ibt_wc_t));
#endif
				idt = idb->idb_task_binding;
				mutex_enter(&idt->idt_mutex);
				if (wr->iw_type == ISER_WR_RDMAW) {
					DTRACE_ISCSI_8(xfer__done,
					    idm_conn_t *, idt->idt_ic,
					    uintptr_t, idb->idb_buf,
					    uint32_t, idb->idb_bufoffset,
					    uint64_t, mr->is_mrva, uint32_t, 0,
					    uint32_t, mr->is_mrrkey,
					    uint32_t, idb->idb_xfer_len,
					    int, XFER_BUF_TX_TO_INI);
					idm_buf_tx_to_ini_done(idt, idb,
					    IDM_STATUS_FAIL);
				} else { /* ISER_WR_RDMAR */
					DTRACE_ISCSI_8(xfer__done,
					    idm_conn_t *, idt->idt_ic,
					    uintptr_t, idb->idb_buf,
					    uint32_t, idb->idb_bufoffset,
					    uint64_t, mr->is_mrva, uint32_t, 0,
					    uint32_t, mr->is_mrrkey,
					    uint32_t, idb->idb_xfer_len,
					    int, XFER_BUF_RX_FROM_INI);
					idm_buf_rx_from_ini_done(idt, idb,
					    IDM_STATUS_FAIL);
				}
			}

			/* Free the iser wr handle */
			iser_wr_free(wr);

			/*
			 * Tell IDM that the channel has gone down,
			 * unless he already knows.
			 */
			mutex_enter(&iser_conn->ic_lock);
			switch (iser_conn->ic_stage) {
			case ISER_CONN_STAGE_IC_DISCONNECTED:
			case ISER_CONN_STAGE_IC_FREED:
			case ISER_CONN_STAGE_CLOSING:
			case ISER_CONN_STAGE_CLOSED:
				break;

			default:
				idm_conn_event(iser_conn->ic_idmc,
				    CE_TRANSPORT_FAIL, idm_status);
				iser_conn->ic_stage = ISER_CONN_STAGE_CLOSING;
			}
			mutex_exit(&iser_conn->ic_lock);

			/* Move onto the next completion */
			continue;
		}

		/*
		 * For a success status, just invoke the PDU or
		 * buffer completion. We use our WR handle's
		 * "iw_type" here so that we can properly process
		 * because the CQE's opcode is invalid if the status
		 * is failed.
		 */
		switch (wr->iw_type) {
		case ISER_WR_SEND:
			/* Free the msg handle */
			ASSERT(wr->iw_msg != NULL);
			iser_msg_free(wr->iw_msg);

			if (wr->iw_pdu == NULL) {
				/* This is a hello exchange message */
				mutex_enter(&iser_conn->ic_lock);
				if (iser_conn->ic_stage ==
				    ISER_CONN_STAGE_HELLOREPLY_SENT) {
					/*
					 * We're on the target side,
					 * and have just successfully
					 * sent the HelloReply msg.
					 */
					iser_conn->ic_stage =
					    ISER_CONN_STAGE_LOGGED_IN;
				}
				mutex_exit(&iser_conn->ic_lock);
			} else {
				/* This is a normal control message */
				idm_pdu_complete(wr->iw_pdu, idm_status);
			}

			/* Free the wr handle */
			iser_wr_free(wr);

			break;

		case ISER_WR_RDMAW:
		case ISER_WR_RDMAR:
			/*
			 * Invoke the appropriate callback;
			 * the buffer will be freed there.
			 */
			idb = wr->iw_buf;
			mr = ((iser_buf_t *)idb->idb_buf_private)->iser_mr;
#ifdef DEBUG
			bcopy(&wc[i],
			    &((iser_buf_t *)idb->idb_buf_private)->buf_wc,
			    sizeof (ibt_wc_t));
#endif
			idt = idb->idb_task_binding;

			mutex_enter(&idt->idt_mutex);
			if (wr->iw_type == ISER_WR_RDMAW) {
				DTRACE_ISCSI_8(xfer__done,
				    idm_conn_t *, idt->idt_ic,
				    uintptr_t, idb->idb_buf,
				    uint32_t, idb->idb_bufoffset,
				    uint64_t, mr->is_mrva, uint32_t, 0,
				    uint32_t, mr->is_mrrkey,
				    uint32_t, idb->idb_xfer_len,
				    int, XFER_BUF_TX_TO_INI);
				idm_buf_tx_to_ini_done(idt, idb, idm_status);
			} else {
				DTRACE_ISCSI_8(xfer__done,
				    idm_conn_t *, idt->idt_ic,
				    uintptr_t, idb->idb_buf,
				    uint32_t, idb->idb_bufoffset,
				    uint64_t, mr->is_mrva, uint32_t, 0,
				    uint32_t, mr->is_mrrkey,
				    uint32_t, idb->idb_xfer_len,
				    int, XFER_BUF_RX_FROM_INI);
				idm_buf_rx_from_ini_done(idt, idb, idm_status);
			}

			/* Free the wr handle */
			iser_wr_free(wr);

			break;

		default:
			ASSERT(0);
			break;
		}
	}

	return (status);
}

void
iser_ib_recvcq_handler(ibt_cq_hdl_t cq_hdl, void *arg)
{
	iser_chan_t	*iser_chan;
	ibt_status_t	status;

	iser_chan = (iser_chan_t *)arg;

	/*
	 * Poll for work request completion while successful. If the
	 * queue empties or otherwise becomes invalid, stop polling.
	 */
	do {
		status = iser_ib_poll_recv_completions(cq_hdl, iser_chan);
	} while (status == IBT_SUCCESS);

	if (status == IBT_CQ_EMPTY) {
		/* We've emptied the CQ, rearm it before we're done here */
		status = ibt_enable_cq_notify(cq_hdl, IBT_NEXT_COMPLETION);
		if (status != IBT_SUCCESS) {
			/* Unexpected error */
			ISER_LOG(CE_NOTE, "iser_ib_recvcq_handler: "
			    "ibt_enable_cq_notify error (%d)", status);
			return;
		}

		/* Now, check for more completions after the rearm */
		do {
			status = iser_ib_poll_recv_completions(
			    cq_hdl, iser_chan);
		} while (status == IBT_SUCCESS);
	}
}

static int
iser_ib_poll_recv_completions(ibt_cq_hdl_t cq_hdl, iser_chan_t *iser_chan)
{
	ibt_wc_t	wc;
	iser_msg_t	*msg;
	iser_qp_t	*iser_qp;
	int		status;

	iser_qp = &(iser_chan->ic_qp);

	bzero(&wc, sizeof (ibt_wc_t));
	status = ibt_poll_cq(cq_hdl, &wc, 1, NULL);
	if (status == IBT_CQ_EMPTY) {
		/* CQ is empty, return */
		return (status);
	}

	if (status != IBT_SUCCESS) {
		/* Unexpected error */
		ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
		    "ibt_poll_cq error (%d)", status);
		mutex_enter(&iser_qp->qp_lock);
		iser_qp->rq_level--;
		mutex_exit(&iser_qp->qp_lock);
		/* Free the msg handle (if we got it back) */
		if ((msg = (iser_msg_t *)(uintptr_t)wc.wc_id) != NULL) {
			iser_msg_free(msg);
		}
		return (status);
	}

	/* Retrieve the iSER msg handle */
	msg = (iser_msg_t *)(uintptr_t)wc.wc_id;
	ASSERT(msg != NULL);

	/*
	 * Decrement the posted level in the RQ, then check
	 * to see if we need to fill the RQ back up (or if
	 * we are already on the taskq).
	 */
	mutex_enter(&iser_chan->ic_conn->ic_lock);
	mutex_enter(&iser_qp->qp_lock);
	iser_qp->rq_level--;

	if ((iser_qp->rq_taskqpending == B_FALSE) &&
	    (iser_qp->rq_level <= iser_qp->rq_lwm) &&
	    (iser_chan->ic_conn->ic_stage >= ISER_CONN_STAGE_IC_CONNECTED) &&
	    (iser_chan->ic_conn->ic_stage <= ISER_CONN_STAGE_LOGGED_IN)) {
		/* Set the pending flag and fire off a post_recv */
		iser_qp->rq_taskqpending = B_TRUE;
		mutex_exit(&iser_qp->qp_lock);

		status = iser_ib_post_recv_async(iser_chan->ic_chanhdl);

		if (status != DDI_SUCCESS) {
			ISER_LOG(CE_NOTE, "iser_ib_poll_recv_completions: "
			    "task dispatch failed");
			/* Failure to launch, unset the pending flag */
			mutex_enter(&iser_qp->qp_lock);
			iser_qp->rq_taskqpending = B_FALSE;
			mutex_exit(&iser_qp->qp_lock);
		}
	} else {
		mutex_exit(&iser_qp->qp_lock);
	}

	DTRACE_PROBE3(iser__recv__cqe, iser_chan_t *, iser_chan,
	    ibt_wc_t *, &wc, ibt_wc_status_t, wc.wc_status);
	if (wc.wc_status != IBT_WC_SUCCESS) {
		/*
		 * Tell IDM that the channel has gone down,
		 * unless he already knows.
		 */
		switch (iser_chan->ic_conn->ic_stage) {
		case ISER_CONN_STAGE_IC_DISCONNECTED:
		case ISER_CONN_STAGE_IC_FREED:
		case ISER_CONN_STAGE_CLOSING:
		case ISER_CONN_STAGE_CLOSED:
			break;

		default:
			idm_conn_event(iser_chan->ic_conn->ic_idmc,
			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
			iser_chan->ic_conn->ic_stage =
			    ISER_CONN_STAGE_CLOSING;
		}
		mutex_exit(&iser_chan->ic_conn->ic_lock);

		iser_msg_free(msg);
		return (DDI_SUCCESS);
	} else {
		mutex_exit(&iser_chan->ic_conn->ic_lock);

		/*
		 * We have an iSER message in, let's handle it.
		 * We will free the iser_msg_t later in this path,
		 * depending upon the action required.
		 */
		iser_msg_handle(iser_chan, msg);
		return (DDI_SUCCESS);
	}
}

static void
iser_msg_handle(iser_chan_t *chan, iser_msg_t *msg)
{
	int		opcode;
	iser_ctrl_hdr_t	*hdr = NULL;
	iser_conn_t	*iser_conn = chan->ic_conn;
	int		status;

	hdr = (iser_ctrl_hdr_t *)(uintptr_t)msg->msg_ds.ds_va;
	ASSERT(hdr != NULL);

	opcode = hdr->opcode;
	if (opcode == ISER_OPCODE_CTRL_TYPE_PDU) {
		/*
		 * Handle an iSCSI Control PDU iSER message.
		 * Note we'll free the msg handle in the PDU callback.
		 */
		status = iser_iscsihdr_handle(chan, msg);
		if (status != DDI_SUCCESS) {
			/*
			 * We are unable to handle this message, and
			 * have no way to recover from this.  Fail the
			 * transport.
			 */
			ISER_LOG(CE_NOTE, "iser_msg_handle: failed "
			    "iser_iscsihdr_handle");
			iser_msg_free(msg);
			idm_conn_event(iser_conn->ic_idmc,
			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
		}
	} else if (opcode == ISER_OPCODE_HELLO_MSG) { /* at the target */
		/*
		 * We are currently not supporting Hello Exchange,
		 * since OFED iSER does not. May be revisited.
		 */
		ASSERT(opcode != ISER_OPCODE_HELLO_MSG);

		if (iser_conn->ic_type != ISER_CONN_TYPE_TGT) {
			idm_conn_event(iser_conn->ic_idmc,
			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
		}

		iser_hello_hdr_t *hello_hdr = (iser_hello_hdr_t *)hdr;

		ISER_LOG(CE_NOTE, "received Hello message: opcode[%d], "
		    "maxver[%d], minver[%d], iser_ird[%d], msg (0x%p)",
		    hello_hdr->opcode, hello_hdr->maxver, hello_hdr->minver,
		    ntohs(hello_hdr->iser_ird), (void *)msg);

		mutex_enter(&iser_conn->ic_lock);

		if (iser_conn->ic_stage != ISER_CONN_STAGE_HELLO_WAIT) {
			/* target is not expected to receive a Hello */
			idm_conn_event(iser_conn->ic_idmc,
			    CE_TRANSPORT_FAIL, IDM_STATUS_FAIL);
		}

		iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_SENT;
		mutex_exit(&iser_conn->ic_lock);

		/* Prepare and send a HelloReply message */
		status = iser_xfer_helloreply_msg(chan);
		if (status != ISER_STATUS_SUCCESS) {

			mutex_enter(&iser_conn->ic_lock);
			iser_conn->ic_stage =
			    ISER_CONN_STAGE_HELLOREPLY_SENT_FAIL;
			mutex_exit(&iser_conn->ic_lock);

			idm_conn_event(iser_conn->ic_idmc,
			    CE_TRANSPORT_FAIL, status);
		}

		/* Free this msg handle */
		iser_msg_free(msg);

	} else if (opcode == ISER_OPCODE_HELLOREPLY_MSG) { /* at initiator */

		/*
		 * We are currently not supporting Hello Exchange,
		 * since OFED iSER does not. May be revisited.
		 */
		ASSERT(opcode != ISER_OPCODE_HELLOREPLY_MSG);

		if (iser_conn->ic_type != ISER_CONN_TYPE_INI) {
			idm_conn_event(iser_conn->ic_idmc,
			    CE_TRANSPORT_FAIL, status);
		}

		iser_helloreply_hdr_t *hello_hdr = (iser_helloreply_hdr_t *)hdr;

		ISER_LOG(CE_NOTE, "received Hello Reply message: opcode[%d], "
		    "maxver[%d], curver[%d], iser_ord[%d], msg (0x%p)",
		    hello_hdr->opcode, hello_hdr->maxver, hello_hdr->curver,
		    ntohs(hello_hdr->iser_ord), (void *)msg);

		/* Free this msg handle */
		iser_msg_free(msg);

		/*
		 * Signal the receipt of HelloReply to the waiting thread
		 * so that the initiator can proceed to the Full Feature
		 * Phase.
		 */
		mutex_enter(&iser_conn->ic_lock);
		iser_conn->ic_stage = ISER_CONN_STAGE_HELLOREPLY_RCV;
		cv_signal(&iser_conn->ic_stage_cv);
		mutex_exit(&iser_conn->ic_lock);
	} else {
		/* Protocol error: free the msg handle and fail the session */
		ISER_LOG(CE_NOTE, "iser_msg_handle: unsupported opcode (0x%x): "
		    "terminating session on IDM handle (0x%p)", opcode,
		    (void *) iser_conn->ic_idmc);

		iser_msg_free(msg);
		idm_conn_event(iser_conn->ic_idmc, CE_TRANSPORT_FAIL,
		    IDM_STATUS_FAIL);
	}
}

#define	IDM_PDU_OPCODE(PDU) \
	((PDU)->isp_hdr->opcode & ISCSI_OPCODE_MASK)

/* network to host translation for 24b integers */
static uint32_t
n2h24(uchar_t *ptr)
{
	return ((ptr[0] << 16) | (ptr[1] << 8) | ptr[2]);
}

/* ARGSUSED */
static void
iser_rx_pdu_cb(idm_pdu_t *pdu, idm_status_t status)
{
	/* Free the iser msg handle and the PDU handle */
	iser_msg_free((iser_msg_t *)pdu->isp_transport_private);
	idm_pdu_free(pdu);
}

int
iser_iscsihdr_handle(iser_chan_t *chan, iser_msg_t *msg)
{
	idm_pdu_t	*pdu;
	uint8_t		*iser_hdrp;
	uint8_t		*iscsi_hdrp;
	iscsi_hdr_t	*bhs;

	pdu = idm_pdu_alloc_nosleep(sizeof (iscsi_hdr_t), 0);
	pdu->isp_ic = chan->ic_conn->ic_idmc;
	ASSERT(pdu->isp_ic != NULL);

	/* Set the iser_msg handle into the transport-private field */
	pdu->isp_transport_private = (void *)msg;

	/* Set up a pointer in the pdu handle to the iSER header */
	iser_hdrp = (uint8_t *)(uintptr_t)msg->msg_ds.ds_va;
	if (iser_hdrp == NULL) {
		ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iser_hdrp is NULL");
		idm_pdu_free(pdu);
		return (ISER_STATUS_FAIL);
	}
	pdu->isp_transport_hdr = (void *)iser_hdrp;
	pdu->isp_transport_hdrlen = ISER_HEADER_LENGTH;

	/*
	 * Set up a pointer to the iSCSI header, which is directly
	 * after the iSER header in the message.
	 */
	iscsi_hdrp = ((uint8_t *)(uintptr_t)msg->msg_ds.ds_va) +
	    ISER_HEADER_LENGTH;
	if (iscsi_hdrp == NULL) {
		ISER_LOG(CE_NOTE, "iser_iscsihdr_handle: iscsi_hdrp is NULL");
		idm_pdu_free(pdu);
		return (ISER_STATUS_FAIL);
	}
	pdu->isp_hdr = (iscsi_hdr_t *)(uintptr_t)iscsi_hdrp;

	/* Fill in the BHS */
	bhs = pdu->isp_hdr;
	pdu->isp_hdrlen	= sizeof (iscsi_hdr_t) +
	    (bhs->hlength * sizeof (uint32_t));
	pdu->isp_datalen = n2h24(bhs->dlength);
	pdu->isp_callback = iser_rx_pdu_cb;

	/*
	 * If datalen > 0, then non-scsi data may be present. Allocate
	 * space in the PDU handle and set a pointer to the data.
	 */
	if (pdu->isp_datalen) {
		pdu->isp_data = ((uint8_t *)(uintptr_t)pdu->isp_hdr) +
		    pdu->isp_hdrlen;
	}

	/* Process RX PDU */
	idm_pdu_rx(pdu->isp_ic, pdu);

	return (DDI_SUCCESS);
}