targets/av1394/av1394_isoch_xmit.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * av1394 isochronous transmit module
 */
#include <sys/1394/targets/av1394/av1394_impl.h>

static int	av1394_it_start_common(av1394_ic_t *);

/* configuration routines */
static void	av1394_it_cleanup(av1394_ic_t *, int);
static int	av1394_it_bld_ixl(av1394_ic_t *);
static void	av1394_it_destroy_ixl(av1394_ic_t *);
static int	av1394_it_ixl_bld_data(av1394_ic_t *);
static void	av1394_it_ixl_destroy_data(av1394_ic_t *);
static av1394_it_ixl_buf_t *av1394_it_ixl_bld_buf(av1394_ic_t *, int, int,
		off_t, int, int);
static void	av1394_it_ixl_complete_buf(av1394_it_ixl_buf_t *,
		av1394_it_ixl_empty_cip_t *);
static void	av1394_it_ixl_complete_buf2(av1394_it_ixl_buf_t *,
		av1394_it_ixl_buf_t *);
static av1394_it_ixl_empty_cip_t *av1394_it_ixl_bld_empty_cip(av1394_ic_t *,
		int);
static void	av1394_it_ixl_complete_empty_cip(av1394_it_ixl_empty_cip_t *,
		av1394_it_ixl_buf_t *);
static void	av1394_it_ixl_bld_begin(av1394_ic_t *);
static void	av1394_it_ixl_begin_update_pkts(av1394_ic_t *,
		av1394_it_ixl_buf_t *);
static int	av1394_it_alloc_isoch_dma(av1394_ic_t *);
static void	av1394_it_free_isoch_dma(av1394_ic_t *);
static void	av1394_it_dma_sync_frames(av1394_ic_t *, int, int);

/* callbacks */
static void	av1394_it_ixl_begin_cb(opaque_t, struct ixl1394_callback *);
static void	av1394_it_ixl_buf_cb(opaque_t, struct ixl1394_callback *);
static void	av1394_it_ixl_eof_cb(av1394_it_ixl_buf_t *bp);
static int	av1394_it_underrun_resume(av1394_ic_t *);
static void	av1394_it_dma_stopped_cb(t1394_isoch_dma_handle_t,
		opaque_t, id1394_isoch_dma_stopped_t);

/* data transfer routines */
static int	av1394_it_add_frames(av1394_ic_t *, int, int);
static int	av1394_it_wait_frames(av1394_ic_t *, int *, int *, int *);

static void	av1394_it_update_frame_syt(av1394_ic_t *, int, int, uint16_t);
static uint16_t	av1394_it_ts_cyc2syt(uint16_t);
static uint16_t	av1394_it_ts_syt_inc(uint16_t, uint16_t);

static void	av1394_it_kcopyin(av1394_ic_t *, void *, size_t);
static int	av1394_it_copyin(av1394_ic_t *, struct uio *, int *, int);
static boolean_t av1394_it_is_dv_frame_start(caddr_t);
static void	av1394_it_reset_frame_syt(av1394_ic_t *, int);

/* tunables */
int av1394_it_hiwat_sub = 2;
int av1394_it_lowat = 3;
int av1394_it_start_thre = 3;	/* xmit start threshold */
int av1394_it_syt_off = 3;	/* SYT offset in cycles */
int av1394_it_dump_ixl = 0;

#define	AV1394_TNF_ENTER(func)	\
	TNF_PROBE_0_DEBUG(func##_enter, AV1394_TNF_ISOCH_STACK, "");

#define	AV1394_TNF_EXIT(func)	\
	TNF_PROBE_0_DEBUG(func##_exit, AV1394_TNF_ISOCH_STACK, "");

int
av1394_it_init(av1394_ic_t *icp, int *error)
{
	av1394_it_t	*itp = &icp->ic_it;
	av1394_isoch_pool_t *pool = &itp->it_data_pool;
	int		nframes;

	AV1394_TNF_ENTER(av1394_it_init);

	nframes = av1394_ic_alloc_pool(pool, icp->ic_framesz, icp->ic_nframes,
	    AV1394_IT_NFRAMES_MIN);
	if (nframes == 0) {
		*error = IEC61883_ERR_NOMEM;
		AV1394_TNF_EXIT(av1394_it_init);
		return (EINVAL);
	}
	mutex_enter(&icp->ic_mutex);
	icp->ic_nframes = nframes;
	itp->it_hiwat = nframes - av1394_it_hiwat_sub;
	itp->it_lowat = av1394_it_lowat;
	itp->it_start_thre = av1394_it_start_thre;
	itp->it_nempty = icp->ic_nframes;
	itp->it_last_full = icp->ic_nframes - 1;

	if (av1394_ic_dma_setup(icp, pool) != DDI_SUCCESS) {
		mutex_exit(&icp->ic_mutex);
		*error = IEC61883_ERR_NOMEM;
		av1394_it_cleanup(icp, 1);
		AV1394_TNF_EXIT(av1394_it_init);
		return (EINVAL);
	}

	if (av1394_it_bld_ixl(icp) != DDI_SUCCESS) {
		mutex_exit(&icp->ic_mutex);
		*error = IEC61883_ERR_NOMEM;
		av1394_it_cleanup(icp, 2);
		AV1394_TNF_EXIT(av1394_it_init);
		return (EINVAL);
	}
	mutex_exit(&icp->ic_mutex);

	if (av1394_it_alloc_isoch_dma(icp) != DDI_SUCCESS) {
		*error = IEC61883_ERR_NOMEM;
		av1394_it_cleanup(icp, 3);
		AV1394_TNF_EXIT(av1394_it_init);
		return (EINVAL);
	}

	AV1394_TNF_EXIT(av1394_it_init);
	return (0);
}

void
av1394_it_fini(av1394_ic_t *icp)
{
	AV1394_TNF_ENTER(av1394_it_fini);

	av1394_it_cleanup(icp, AV1394_CLEANUP_LEVEL_MAX);

	AV1394_TNF_ENTER(av1394_it_fini);
}

int
av1394_it_start(av1394_ic_t *icp)
{
	av1394_it_t	*itp = &icp->ic_it;
	int		ret = 0;

	AV1394_TNF_ENTER(av1394_it_start);

	mutex_enter(&icp->ic_mutex);
	ASSERT(icp->ic_state == AV1394_IC_IDLE);

	/* should be enough full frames to be able to start */
	if (itp->it_nfull >= itp->it_start_thre) {
		ret = av1394_it_start_common(icp);
	}
	mutex_exit(&icp->ic_mutex);

	AV1394_TNF_EXIT(av1394_it_start);
	return (ret);
}

static int
av1394_it_start_common(av1394_ic_t *icp)
{
	av1394_inst_t	*avp = icp->ic_avp;
	id1394_isoch_dma_ctrlinfo_t idma_ctrlinfo = { 0 };
	int		result;
	int		err;
	int		ret = 0;

	ASSERT(icp->ic_state == AV1394_IC_IDLE);

	err = t1394_start_isoch_dma(avp->av_t1394_hdl, icp->ic_isoch_hdl,
	    &idma_ctrlinfo, 0, &result);
	if (err == DDI_SUCCESS) {
		icp->ic_state = AV1394_IC_DMA;
	} else {
		TNF_PROBE_1(av1394_it_start_common_error,
		    AV1394_TNF_ISOCH_ERROR, "", tnf_int, result, result);
		ret = EIO;
	}

	return (ret);
}


int
av1394_it_stop(av1394_ic_t *icp)
{
	av1394_inst_t	*avp = icp->ic_avp;
	av1394_it_t	*itp = &icp->ic_it;

	AV1394_TNF_ENTER(av1394_it_stop);

	mutex_enter(&icp->ic_mutex);
	if (icp->ic_state != AV1394_IC_IDLE) {
		mutex_exit(&icp->ic_mutex);
		t1394_stop_isoch_dma(avp->av_t1394_hdl, icp->ic_isoch_hdl, 0);
		mutex_enter(&icp->ic_mutex);

		icp->ic_state = AV1394_IC_IDLE;
		itp->it_nempty = icp->ic_nframes;
		itp->it_first_empty = 0;
		itp->it_last_full = icp->ic_nframes - 1;
		itp->it_nfull = 0;
	}
	mutex_exit(&icp->ic_mutex);

	AV1394_TNF_EXIT(av1394_it_stop);
	return (0);
}

int
av1394_it_xmit(av1394_ic_t *icp, iec61883_xmit_t *xmit)
{
	av1394_it_t	*itp = &icp->ic_it;
	int		ret = 0;
	int		idx, cnt;

	idx = xmit->tx_xfer.xf_full_idx;
	cnt = xmit->tx_xfer.xf_full_cnt;

	mutex_enter(&icp->ic_mutex);
	/* check arguments */
	if ((idx < 0) || (cnt < 0) || (cnt > itp->it_nempty)) {
		mutex_exit(&icp->ic_mutex);
		TNF_PROBE_2(av1394_it_xmit_error_args, AV1394_TNF_ISOCH_ERROR,
		    "", tnf_int, idx, idx, tnf_int, cnt, cnt);
		return (EINVAL);
	}

	/* add full frames to the pool */
	if (cnt > 0) {
		if ((ret = av1394_it_add_frames(icp, idx, cnt)) != 0) {
			mutex_exit(&icp->ic_mutex);
			return (ret);
		}
	}

	if ((icp->ic_state == AV1394_IC_IDLE) &&
	    (itp->it_nfull >= itp->it_start_thre)) {
		if ((ret = av1394_it_start_common(icp)) != 0) {
			mutex_exit(&icp->ic_mutex);
			return (ret);
		}
	}

	/* wait for new empty frames */
	ret = av1394_it_wait_frames(icp, &xmit->tx_xfer.xf_empty_idx,
	    &xmit->tx_xfer.xf_empty_cnt, &xmit->tx_miss_cnt);
	mutex_exit(&icp->ic_mutex);

	return (ret);
}

int
av1394_it_write(av1394_ic_t *icp, struct uio *uiop)
{
	av1394_inst_t	*avp = icp->ic_avp;
	av1394_it_t	*itp = &icp->ic_it;
	av1394_isoch_autoxmit_t *axp = &avp->av_i.i_autoxmit;
	int		dv;
	int		ret = 0;
	int		full_cnt;
	int		miss_cnt;

	AV1394_TNF_ENTER(av1394_it_write);

	mutex_enter(&icp->ic_mutex);
	dv = (axp->ax_fmt & AV1394_ISOCH_AUTOXMIT_DV);

	while (uiop->uio_resid > 0) {
		/* must have at least one empty frame */
		if (itp->it_write_cnt == 0) {
			ret = av1394_it_wait_frames(icp, &itp->it_write_idx,
			    &itp->it_write_cnt, &miss_cnt);
			if (ret != 0) {
				break;
			}
		}

		/* copyin as much data as we can */
		if (axp->ax_copy_ciph) {
			ASSERT(itp->it_write_off == 0);
			av1394_it_kcopyin(icp, axp->ax_ciph, AV1394_CIPSZ);
			axp->ax_copy_ciph = B_FALSE;
		}
		if ((ret = av1394_it_copyin(icp, uiop, &full_cnt, dv)) != 0) {
			break;
		}

		/* add full frames to the pool */
		if (full_cnt > 0) {
			ret = av1394_it_add_frames(icp,
			    itp->it_write_idx, full_cnt);
			if (ret != 0) {
				break;
			}
			itp->it_write_idx += full_cnt;
			itp->it_write_idx %= icp->ic_nframes;
		}

		/* start xfer if not already */
		if ((icp->ic_state == AV1394_IC_IDLE) &&
		    (itp->it_nfull >= itp->it_start_thre)) {
			if ((ret = av1394_it_start_common(icp)) != 0) {
				mutex_exit(&icp->ic_mutex);
				AV1394_TNF_EXIT(av1394_it_write);
				return (ret);
			}
		}
	}
	mutex_exit(&icp->ic_mutex);

	AV1394_TNF_EXIT(av1394_it_write);
	return (ret);
}

/*
 *
 * --- configuration routines
 *
 */
static void
av1394_it_cleanup(av1394_ic_t *icp, int level)
{
	av1394_isoch_pool_t *pool = &icp->ic_it.it_data_pool;

	ASSERT((level > 0) && (level <= AV1394_CLEANUP_LEVEL_MAX));

	switch (level) {
	default:
		av1394_it_free_isoch_dma(icp);
		/* FALLTHRU */
	case 3:
		av1394_it_destroy_ixl(icp);
		/* FALLTHRU */
	case 2:
		av1394_ic_dma_cleanup(icp, pool);
		/* FALLTHRU */
	case 1:
		av1394_ic_free_pool(pool);
		/* FALLTHRU */
	}
}

/*
 * av1394_it_bld_ixl()
 *    Build an IXL chain out of several blocks.
 */
static int
av1394_it_bld_ixl(av1394_ic_t *icp)
{
	av1394_it_t	*itp = &icp->ic_it;
	int		ret;

	AV1394_TNF_ENTER(av1394_it_bld_ixl);

	/* data block */
	if ((ret = av1394_it_ixl_bld_data(icp)) != DDI_SUCCESS) {
		AV1394_TNF_EXIT(av1394_it_bld_ixl);
		return (ret);
	}

	/* begin block */
	if (icp->ic_param.cp_ts_mode != IEC61883_TS_NONE) {
		av1394_it_ixl_bld_begin(icp);

		itp->it_ixlp = (ixl1394_command_t *)&itp->it_ixl_begin;
	} else {
		itp->it_ixlp = (ixl1394_command_t *)
		    &((av1394_it_ixl_buf_t *)itp->it_ixl_data)->tb_label;
	}

	if (av1394_it_dump_ixl) {
		av1394_ic_ixl_dump(itp->it_ixlp);
	}

	AV1394_TNF_EXIT(av1394_it_bld_ixl);
	return (ret);
}

static void
av1394_it_destroy_ixl(av1394_ic_t *icp)
{
	av1394_it_t	*itp = &icp->ic_it;

	av1394_it_ixl_destroy_data(icp);
	itp->it_ixlp = NULL;
}

/*
 * build data transmit part of the IXL chain
 */
static int
av1394_it_ixl_bld_data(av1394_ic_t *icp)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_isoch_pool_t	*pool = &itp->it_data_pool;
	int			total = 0;	/* # of pkts in the chain */
	int			nfull = 0;	/* # of full CIPs in a series */
	int			framenum = -1;	/* frame number */
	int			bufsz_max;	/* max buffer size in pkts */
	int			segnum = 0;	/* current segment number */
	int			segsz;		/* segment size in pkts */
	off_t			segoff = 0;	/* segment offset in pkts */
	av1394_it_ixl_empty_cip_t *ep = NULL;	/* last empty CIP */
	av1394_it_ixl_buf_t	*bp = NULL;	/* last data buffer */
	av1394_it_ixl_buf_t	*prevbp = NULL;
	int			a, n, d;	/* N/D algorithm variables */
	int			type, ptype;	/* current and prev CIP type */
	int			tb_flags;

	itp->it_frame_info = kmem_zalloc(icp->ic_nframes *
	    sizeof (av1394_it_frame_info_t), KM_SLEEP);

	bufsz_max = AV1394_IXL_BUFSZ_MAX / icp->ic_pktsz;
	n = icp->ic_param.cp_n;
	d = icp->ic_param.cp_d;
	/*
	 * following assert guarantees no more than one empty CIP in a row,
	 * i.e. empty CIPs account for <=50% of all packets.
	 * this should be ensured by ioctl argument validation.
	 */
	ASSERT((n == 0) || (d / n > 1));
	/*
	 * build the chain. it is hard to precalculate amount of memory
	 * needed for the entire chain, so we simply allocate as we go.
	 */
	ptype = AV1394_CIP_EMPTY;
	segsz = pool->ip_seg[0].is_size / icp->ic_pktsz;
	a = n;
	while (total < icp->ic_nframes * icp->ic_npkts) {
		/* insert empty CIPs using N/D algorithm */
		a += n;
		if (a > d) {
			a -= d;
			type = AV1394_CIP_EMPTY;
		} else {
			type = AV1394_CIP_FULL;
			nfull++;
		}

		/*
		 * merge series of full packets into single SEND_BUF commands.
		 * a series can be terminated by:
		 *  - an empty CIP;
		 *  - series buffer size reached maximum;
		 *  - end of isoch segment;
		 *  - end of frame (which is always at the end of segment);
		 */
		if (((type == AV1394_CIP_EMPTY) || (segoff + nfull == segsz) ||
		    (nfull == bufsz_max)) && (nfull > 0)) {

			/* build buffer block */
			prevbp = bp;
			tb_flags = 0;
			if (type == AV1394_CIP_EMPTY) {
				tb_flags |= AV1394_IT_IXL_BUF_NEXT_EMPTY;
			}
			if (total % icp->ic_npkts == 0) {
				tb_flags |= AV1394_IT_IXL_BUF_SOF;
				framenum++;
			}
			if ((total + nfull) % icp->ic_npkts == 0) {
				tb_flags |= AV1394_IT_IXL_BUF_EOF;
			}
			bp = av1394_it_ixl_bld_buf(icp, nfull, segnum, segoff,
			    tb_flags, framenum);

			if (itp->it_ixl_data == NULL) {
				itp->it_ixl_data = &bp->tb_common;
			}

			/* complete previous empty CIP or a buffer */
			if (ep) {
				av1394_it_ixl_complete_empty_cip(ep, bp);
				ep = NULL;
			} else if (prevbp) {
				av1394_it_ixl_complete_buf2(prevbp, bp);
			}

			/* if current segment is used up, pick next one */
			segoff += nfull;
			if (segoff == segsz) {
				if (++segnum < pool->ip_nsegs) {
					segsz = pool->ip_seg[segnum].is_size /
					    icp->ic_pktsz;
				}
				segoff = 0;
			}

			total += nfull;
			nfull = 0;
		}
		/* insert an empty packet if needed */
		if (type == AV1394_CIP_EMPTY) {
			ep = av1394_it_ixl_bld_empty_cip(icp, framenum);
			av1394_it_ixl_complete_buf(bp, ep);
		}
		ptype = type;
	}
	ASSERT(nfull == 0);

	/* last packet must be an empty CIP, except when n == 0 */
	if (n != 0) {
		if (ptype != AV1394_CIP_EMPTY) {
			ep = av1394_it_ixl_bld_empty_cip(icp, framenum);
			av1394_it_ixl_complete_buf(bp, ep);
		}
		av1394_it_ixl_complete_empty_cip(ep,
		    (av1394_it_ixl_buf_t *)itp->it_ixl_data);
		ep->te_jump.next_ixlp = NULL;
		ep->te_common.tc_next = NULL;
	} else {
		bp->tb_jump.label = (ixl1394_command_t *)
		    &(((av1394_it_ixl_buf_t *)itp->it_ixl_data)->tb_label);
	}

	return (DDI_SUCCESS);
}

static void
av1394_it_ixl_destroy_data(av1394_ic_t *icp)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_it_ixl_common_t	*cmd, *cmd_next;

	for (cmd = itp->it_ixl_data; cmd != NULL; cmd = cmd_next) {
		cmd_next = cmd->tc_next;
		kmem_free(cmd, cmd->tc_size);
	}
	itp->it_ixl_data = NULL;

	kmem_free(itp->it_frame_info,
	    icp->ic_nframes * sizeof (av1394_it_frame_info_t));
}

static av1394_it_ixl_buf_t *
av1394_it_ixl_bld_buf(av1394_ic_t *icp, int cnt, int segnum, off_t off,
		int flags, int framenum)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_isoch_seg_t	*isp = &itp->it_data_pool.ip_seg[segnum];
	av1394_it_ixl_buf_t	*bp;
	int			pktsz = icp->ic_pktsz;

	bp = kmem_zalloc(sizeof (av1394_it_ixl_buf_t), KM_SLEEP);
	bp->tb_common.tc_size = sizeof (av1394_it_ixl_buf_t);
	/* tc_next later */
	bp->tb_flags = flags;
	bp->tb_framenum = framenum;
	bp->tb_icp = icp;

	bp->tb_label.ixl_opcode = IXL1394_OP_LABEL;
	bp->tb_label.next_ixlp = (ixl1394_command_t *)&bp->tb_buf;

	bp->tb_buf.ixl_opcode = IXL1394_OP_SEND_BUF;
	bp->tb_buf.pkt_size = pktsz;
	bp->tb_buf.size = cnt * pktsz;
	bp->tb_buf.ixl_buf._dmac_ll =
	    isp->is_dma_cookie[0].dmac_laddress + off * pktsz;
	bp->tb_buf.mem_bufp = isp->is_kaddr + off * pktsz;

	if (flags & AV1394_IT_IXL_BUF_EOF) {
		bp->tb_buf.next_ixlp = (ixl1394_command_t *)&bp->tb_store_ts;

		bp->tb_store_ts.ixl_opcode = IXL1394_OP_STORE_TIMESTAMP;
		bp->tb_store_ts.next_ixlp = (ixl1394_command_t *)&bp->tb_cb;

		bp->tb_cb.ixl_opcode = IXL1394_OP_CALLBACK;
		bp->tb_cb.callback = av1394_it_ixl_buf_cb;
		bp->tb_cb.callback_arg = bp;
		bp->tb_cb.next_ixlp = (ixl1394_command_t *)&bp->tb_jump;

		bp->tb_jump.ixl_opcode = IXL1394_OP_JUMP_U;
	} else {
		bp->tb_buf.next_ixlp = (ixl1394_command_t *)&bp->tb_jump;

		bp->tb_jump.ixl_opcode = IXL1394_OP_JUMP;
	}
	/*
	 * jump label and next_ixlp later.
	 * unset fields will be set in av1394_it_ixl_complete_buf()
	 *
	 * save additional frame info
	 */
	if (flags & AV1394_IT_IXL_BUF_SOF) {
		itp->it_frame_info[framenum].fi_first_buf = bp;
		itp->it_frame_info[framenum].fi_ts_off = bp->tb_buf.mem_bufp +
		    AV1394_TS_MODE_GET_OFF(icp->ic_param.cp_ts_mode);
	} else if (flags & AV1394_IT_IXL_BUF_EOF) {
		itp->it_frame_info[framenum].fi_last_buf = bp;
	}
	itp->it_frame_info[framenum].fi_ncycs += cnt;

	return (bp);
}

static void
av1394_it_ixl_complete_buf(av1394_it_ixl_buf_t *bp,
	av1394_it_ixl_empty_cip_t *ep)
{
	bp->tb_common.tc_next = &ep->te_common;
	bp->tb_jump.label = bp->tb_jump.next_ixlp =
	    (ixl1394_command_t *)&ep->te_label;
}

static void
av1394_it_ixl_complete_buf2(av1394_it_ixl_buf_t *bp,
	av1394_it_ixl_buf_t *nextbp)
{
	bp->tb_common.tc_next = &nextbp->tb_common;
	bp->tb_jump.label = bp->tb_jump.next_ixlp =
	    (ixl1394_command_t *)&nextbp->tb_label;
}

static av1394_it_ixl_empty_cip_t *
av1394_it_ixl_bld_empty_cip(av1394_ic_t *icp, int framenum)
{
	av1394_it_t	*itp = &icp->ic_it;
	av1394_it_ixl_empty_cip_t *ep;

	ep = kmem_zalloc(sizeof (av1394_it_ixl_empty_cip_t), KM_SLEEP);
	ep->te_common.tc_size = sizeof (av1394_it_ixl_empty_cip_t);
	/* tc_next later */

	ep->te_label.ixl_opcode = IXL1394_OP_LABEL;
	ep->te_label.next_ixlp = (ixl1394_command_t *)&ep->te_pkt;

	ep->te_pkt.ixl_opcode = IXL1394_OP_SEND_PKT_ST;
	ep->te_pkt.size = AV1394_CIPSZ;
	/* ixl_buf and mem_bufp later */
	ep->te_pkt.next_ixlp = (ixl1394_command_t *)&ep->te_jump;

	ep->te_jump.ixl_opcode = IXL1394_OP_JUMP;
	/*
	 * label and next_ixlp later.
	 * unset fields will be set in av1394_it_ixl_complete_empty_cip()
	 */

	itp->it_frame_info[framenum].fi_ncycs++;

	return (ep);
}

/*
 * empty CIP packet contains CIP header of the next packet,
 * so we just point to the same address as the next packet's header
 */
static void
av1394_it_ixl_complete_empty_cip(av1394_it_ixl_empty_cip_t *ep,
	av1394_it_ixl_buf_t *bp)
{
	ep->te_common.tc_next = &bp->tb_common;

	ep->te_pkt.ixl_buf._dmac_ll = bp->tb_buf.ixl_buf._dmac_ll;
	ep->te_pkt.mem_bufp = bp->tb_buf.mem_bufp;

	ep->te_jump.label = ep->te_jump.next_ixlp =
	    (ixl1394_command_t *)&bp->tb_label;
}

static void
av1394_it_ixl_bld_begin(av1394_ic_t *icp)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_it_ixl_buf_t	*bp = (av1394_it_ixl_buf_t *)itp->it_ixl_data;
	av1394_it_ixl_begin_t	*bep = &itp->it_ixl_begin;
	int			i;

	bep->be_label.ixl_opcode = IXL1394_OP_LABEL;
	bep->be_label.next_ixlp = (ixl1394_command_t *)&bep->be_empty_pre;

	bep->be_empty_pre.ixl_opcode = IXL1394_OP_SEND_PKT_ST;
	bep->be_empty_pre.size = AV1394_CIPSZ;
	bep->be_empty_pre.ixl_buf._dmac_ll = bp->tb_buf.ixl_buf._dmac_ll;
	bep->be_empty_pre.mem_bufp = bp->tb_buf.mem_bufp;
	bep->be_empty_pre.next_ixlp = (ixl1394_command_t *)&bep->be_store_ts;

	bep->be_store_ts.ixl_opcode = IXL1394_OP_STORE_TIMESTAMP;
	bep->be_store_ts.next_ixlp = (ixl1394_command_t *)&bep->be_cb;

	bep->be_cb.ixl_opcode = IXL1394_OP_CALLBACK;
	bep->be_cb.callback = av1394_it_ixl_begin_cb;
	bep->be_cb.callback_arg = &bep->be_store_ts.timestamp;
	bep->be_cb.next_ixlp = (ixl1394_command_t *)&bep->be_empty_post[0];

	for (i = 0; i < AV1394_IT_IXL_BEGIN_NPOST; i++) {
		bep->be_empty_post[i].ixl_opcode = IXL1394_OP_SEND_PKT_ST;
		bep->be_empty_post[i].size = AV1394_CIPSZ;
		bep->be_empty_post[i].ixl_buf._dmac_ll =
		    bp->tb_buf.ixl_buf._dmac_ll;
		bep->be_empty_post[i].mem_bufp = bp->tb_buf.mem_bufp;
		bep->be_empty_post[i].next_ixlp =
		    (ixl1394_command_t *)&bep->be_empty_post[i + 1];
	}
	bep->be_empty_post[AV1394_IT_IXL_BEGIN_NPOST - 1].next_ixlp =
	    (ixl1394_command_t *)&bep->be_jump;

	bep->be_jump.ixl_opcode = IXL1394_OP_JUMP_U;
	bep->be_jump.label = (ixl1394_command_t *)&bp->tb_label;
	bep->be_jump.next_ixlp = (ixl1394_command_t *)&bp->tb_label;
}

static void
av1394_it_ixl_begin_update_pkts(av1394_ic_t *icp, av1394_it_ixl_buf_t *bp)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_it_ixl_begin_t	*bep = &itp->it_ixl_begin;
	int			i;

	for (i = 0; i < AV1394_IT_IXL_BEGIN_NPOST; i++) {
		bep->be_empty_post[i].ixl_buf._dmac_ll =
		    bp->tb_buf.ixl_buf._dmac_ll;
		bep->be_empty_post[i].mem_bufp = bp->tb_buf.mem_bufp;
	}
}

static int
av1394_it_alloc_isoch_dma(av1394_ic_t *icp)
{
	av1394_inst_t		*avp = icp->ic_avp;
	av1394_it_t		*itp = &icp->ic_it;
	id1394_isoch_dmainfo_t	di;
	int			result;
	int			ret;

	AV1394_TNF_ENTER(av1394_it_alloc_isoch_dma);

	di.ixlp = itp->it_ixlp;
	di.channel_num = icp->ic_num;
	di.idma_options = ID1394_TALK;
	di.it_speed = icp->ic_param.cp_bus_speed;
	/*
	 * XXX this should really be IXL1394_SKIP_TO_NEXT,
	 * but it can't be used yet due to the Framework bug
	 */
	di.it_default_skip = IXL1394_SKIP_TO_SELF;
	di.default_tag = 1;
	di.default_sync = 0;
	di.global_callback_arg = icp;
	di.isoch_dma_stopped = av1394_it_dma_stopped_cb;
	di.idma_evt_arg = icp;

	if ((ret = t1394_alloc_isoch_dma(avp->av_t1394_hdl, &di, 0,
	    &icp->ic_isoch_hdl, &result)) != DDI_SUCCESS) {
		TNF_PROBE_1(av1394_it_alloc_isoch_dma_error,
		    AV1394_TNF_ISOCH_ERROR, "", tnf_int, result, result);
	}

	AV1394_TNF_EXIT(av1394_it_alloc_isoch_dma);
	return (ret);
}

static void
av1394_it_free_isoch_dma(av1394_ic_t *icp)
{
	av1394_inst_t		*avp = icp->ic_avp;

	AV1394_TNF_ENTER(av1394_it_free_isoch_rsrc);

	t1394_free_isoch_dma(avp->av_t1394_hdl, 0, &icp->ic_isoch_hdl);

	AV1394_TNF_EXIT(av1394_it_free_isoch_rsrc);
}

static void
av1394_it_dma_sync_frames(av1394_ic_t *icp, int idx, int cnt)
{
	av1394_ic_dma_sync_frames(icp, idx, cnt,
	    &icp->ic_it.it_data_pool, DDI_DMA_SYNC_FORDEV);
}

/*
 *
 * --- callbacks
 *
 */
static void
av1394_it_ixl_begin_cb(opaque_t arg, struct ixl1394_callback *cb)
{
	av1394_ic_t	*icp = arg;
	av1394_it_t	*itp = &icp->ic_it;
	uint16_t	*cycp = cb->callback_arg; /* cycle timestamp pointer */
	uint16_t	syt;
	int		first;

	AV1394_TNF_ENTER(av1394_it_ixl_begin_cb);

	mutex_enter(&icp->ic_mutex);
	/* save initial timestamp value */
	itp->it_ts_init.ts_syt = av1394_it_ts_cyc2syt(*cycp);

	/*
	 * update frame timestamps if needed
	 */
	if ((itp->it_nfull <= 0) ||
	    (AV1394_TS_MODE_GET_SIZE(icp->ic_param.cp_ts_mode) == 0)) {
		mutex_exit(&icp->ic_mutex);
		AV1394_TNF_EXIT(av1394_it_ixl_begin_cb);
		return;
	}
	ASSERT(itp->it_nfull <= icp->ic_nframes);

	syt = av1394_it_ts_syt_inc(itp->it_ts_init.ts_syt,
	    AV1394_IT_IXL_BEGIN_NPOST + av1394_it_syt_off);
	first = (itp->it_last_full + icp->ic_nframes - itp->it_nfull + 1) %
	    icp->ic_nframes;
	av1394_it_update_frame_syt(icp, first, itp->it_nfull, syt);
	mutex_exit(&icp->ic_mutex);

	AV1394_TNF_EXIT(av1394_it_ixl_begin_cb);
}

/*ARGSUSED*/
static void
av1394_it_ixl_buf_cb(opaque_t arg, struct ixl1394_callback *cb)
{
	av1394_it_ixl_buf_t	*bp = cb->callback_arg;

	AV1394_TNF_ENTER(av1394_it_ixl_buf_cb);
	if (bp->tb_flags & AV1394_IT_IXL_BUF_EOF) {
		av1394_it_ixl_eof_cb(bp);
	}
	AV1394_TNF_EXIT(av1394_it_ixl_buf_cb);
}

static void
av1394_it_ixl_eof_cb(av1394_it_ixl_buf_t *bp)
{
	av1394_ic_t	*icp = bp->tb_icp;
	av1394_isoch_t	*ip = &icp->ic_avp->av_i;
	av1394_it_t	*itp = &icp->ic_it;

	AV1394_TNF_ENTER(av1394_it_ixl_eof_cb);

	mutex_enter(&ip->i_mutex);
	mutex_enter(&icp->ic_mutex);
	if (itp->it_nempty < icp->ic_nframes) {
		itp->it_nempty++;
		itp->it_nfull--;
		cv_signal(&icp->ic_xfer_cv);
	}

	if ((itp->it_nempty >= itp->it_hiwat) &&
	    (icp->ic_state == AV1394_IC_DMA)) {
		av1394_ic_trigger_softintr(icp, icp->ic_num,
		    AV1394_PREQ_IT_UNDERRUN);
	}
	mutex_exit(&icp->ic_mutex);
	mutex_exit(&ip->i_mutex);

	AV1394_TNF_EXIT(av1394_it_ixl_eof_cb);
}

void
av1394_it_underrun(av1394_ic_t *icp)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_inst_t		*avp = icp->ic_avp;
	int			idx;
	ixl1394_jump_t		*old_jmp;
	ixl1394_jump_t		new_jmp;
	id1394_isoch_dma_updateinfo_t update_info;
	int			err;
	int			result;

	AV1394_TNF_ENTER(av1394_it_underrun);

	/*
	 * update the last full frame's jump to NULL
	 */
	idx = (itp->it_first_empty + icp->ic_nframes - 1) % icp->ic_nframes;

	old_jmp = &itp->it_frame_info[idx].fi_last_buf->tb_jump;
	itp->it_saved_label = old_jmp->label;

	new_jmp.ixl_opcode = IXL1394_OP_JUMP_U;
	new_jmp.label = NULL;
	new_jmp.next_ixlp = NULL;

	update_info.orig_ixlp = (ixl1394_command_t *)old_jmp;
	update_info.temp_ixlp = (ixl1394_command_t *)&new_jmp;
	update_info.ixl_count = 1;

	mutex_exit(&icp->ic_mutex);
	err = t1394_update_isoch_dma(avp->av_t1394_hdl,
	    icp->ic_isoch_hdl, &update_info, 0, &result);
	mutex_enter(&icp->ic_mutex);

	if (err == DDI_SUCCESS) {
		itp->it_underrun_idx = idx;
		icp->ic_state = AV1394_IC_SUSPENDED;
		cv_signal(&icp->ic_xfer_cv);
	} else {
		TNF_PROBE_2(av1394_it_underrun_error_update,
		    AV1394_TNF_ISOCH_ERROR, "", tnf_int, err, err,
		    tnf_int, result, result);
	}

	AV1394_TNF_EXIT(av1394_it_underrun);
}

/*
 * resume from the underrun condition
 */
static int
av1394_it_underrun_resume(av1394_ic_t *icp)
{
	av1394_it_t		*itp = &icp->ic_it;
	av1394_inst_t		*avp = icp->ic_avp;
	av1394_it_ixl_buf_t	*bp;
	int			idx;
	ixl1394_jump_t		*old_jmp;
	ixl1394_jump_t		new_jmp;
	id1394_isoch_dma_updateinfo_t update_info;
	int			err;
	int			result;

	AV1394_TNF_ENTER(av1394_it_underrun_resume);

	/*
	 * resuming the transfer it a lot like starting the transfer:
	 * first the IXL begin block needs to be executed, then the rest
	 * of the IXL chain. The following dynamic updates are needed:
	 *
	 *  1. update the begin block to jump to the first empty frame;
	 *  2. restore the original jump label which we previously
	 *    changed to jump to the underrun block;
	 *
	 * update #1
	 *   start by updating the begin block with a new buffer address
	 */
	idx = (itp->it_underrun_idx + 1) % icp->ic_nframes;
	bp = itp->it_frame_info[idx].fi_first_buf;
	av1394_it_ixl_begin_update_pkts(icp, bp);

	old_jmp = &itp->it_ixl_begin.be_jump;

	new_jmp.ixl_opcode = IXL1394_OP_JUMP_U;
	new_jmp.label = (ixl1394_command_t *)&bp->tb_label;
	new_jmp.next_ixlp = NULL;

	update_info.orig_ixlp = (ixl1394_command_t *)old_jmp;
	update_info.temp_ixlp = (ixl1394_command_t *)&new_jmp;
	update_info.ixl_count = 1;

	mutex_exit(&icp->ic_mutex);
	err = t1394_update_isoch_dma(avp->av_t1394_hdl,
	    icp->ic_isoch_hdl, &update_info, 0, &result);
	mutex_enter(&icp->ic_mutex);

	if (err != DDI_SUCCESS) {
		TNF_PROBE_2(av1394_it_underrun_resume_error_update1,
		    AV1394_TNF_ISOCH_ERROR, "", tnf_int, err, err,
		    tnf_int, result, result);
		AV1394_TNF_EXIT(av1394_it_underrun_resume);
		return (EIO);
	}

	/*
	 * update #2
	 */
	bp = itp->it_frame_info[itp->it_underrun_idx].fi_last_buf;
	old_jmp = &bp->tb_jump;

	new_jmp.ixl_opcode = IXL1394_OP_JUMP_U;
	new_jmp.label = itp->it_saved_label;
	new_jmp.next_ixlp = NULL;

	update_info.orig_ixlp = (ixl1394_command_t *)old_jmp;
	update_info.temp_ixlp = (ixl1394_command_t *)&new_jmp;
	update_info.ixl_count = 1;

	mutex_exit(&icp->ic_mutex);
	err = t1394_update_isoch_dma(avp->av_t1394_hdl,
	    icp->ic_isoch_hdl, &update_info, 0, &result);
	mutex_enter(&icp->ic_mutex);

	if (err != DDI_SUCCESS) {
		TNF_PROBE_2(av1394_it_underrun_resume_error_update2,
		    AV1394_TNF_ISOCH_ERROR, "", tnf_int, err, err,
		    tnf_int, result, result);
		AV1394_TNF_EXIT(av1394_it_underrun_resume);
		return (EIO);
	}

	icp->ic_state = AV1394_IC_DMA;

	AV1394_TNF_EXIT(av1394_it_underrun_resume);
	return (0);
}

/*ARGSUSED*/
static void
av1394_it_dma_stopped_cb(t1394_isoch_dma_handle_t t1394_idma_hdl,
	opaque_t idma_evt_arg, id1394_isoch_dma_stopped_t status)
{
	av1394_ic_t	*icp = idma_evt_arg;

	AV1394_TNF_ENTER(av1394_it_dma_stopped_cb);

	mutex_enter(&icp->ic_mutex);
	icp->ic_state = AV1394_IC_IDLE;
	mutex_exit(&icp->ic_mutex);

	AV1394_TNF_EXIT(av1394_it_dma_stopped_cb);
}


/*
 *
 * --- data transfer routines
 *
 * av1394_it_add_frames()
 *    Add full frames to the pool.
 */
static int
av1394_it_add_frames(av1394_ic_t *icp, int idx, int cnt)
{
	av1394_it_t	*itp = &icp->ic_it;
	av1394_it_frame_info_t *fip;
	int		prev_full = itp->it_last_full;
	uint16_t	syt;
	int		ret = 0;

	/* can only add to tail */
	if (idx != ((itp->it_last_full + 1) % icp->ic_nframes)) {
		TNF_PROBE_1(av1394_it_add_frames_error,
		    AV1394_TNF_ISOCH_ERROR, "", tnf_int, idx, idx);
		return (EINVAL);
	}

	/* turn empty frames into full ones */
	itp->it_nempty -= cnt;
	itp->it_first_empty = (itp->it_first_empty + cnt) % icp->ic_nframes;
	itp->it_nfull += cnt;
	itp->it_last_full = (itp->it_last_full + cnt) % icp->ic_nframes;
	ASSERT((itp->it_nempty >= 0) && (itp->it_nfull <= icp->ic_nframes));

	/*
	 * update frame timestamps if needed
	 */
	if (AV1394_TS_MODE_GET_SIZE(icp->ic_param.cp_ts_mode) > 0) {
		ASSERT(prev_full >= 0);
		fip = &itp->it_frame_info[prev_full];
		syt = *(uint16_t *)fip->fi_ts_off;
		syt = av1394_it_ts_syt_inc(syt, fip->fi_ncycs);
		av1394_it_update_frame_syt(icp, idx, cnt, syt);
	}

	av1394_it_dma_sync_frames(icp, idx, cnt);

	/* if suspended due to overrun, check if we can resume */
	if ((icp->ic_state == AV1394_IC_SUSPENDED) &&
	    (itp->it_nempty >= itp->it_lowat)) {
		ret = av1394_it_underrun_resume(icp);
	}

	return (ret);
}

/*
 * wait for empty frames
 */
static int
av1394_it_wait_frames(av1394_ic_t *icp, int *idx, int *cnt, int *nlost)
{
	av1394_it_t	*itp = &icp->ic_it;
	int		ret = 0;

	while ((itp->it_nempty == 0) && (icp->ic_state == AV1394_IC_DMA)) {
		if (cv_wait_sig(&icp->ic_xfer_cv, &icp->ic_mutex) <= 0) {
			ret = EINTR;
			break;
		}
	}

	if (itp->it_nempty > 0) {
		*idx = itp->it_first_empty;
		*cnt = itp->it_nempty;
		*nlost = 0;
		ret = 0;
	}
	return (ret);
}

/*
 * update frame timestamps for a range of frames
 */
static void
av1394_it_update_frame_syt(av1394_ic_t *icp, int first, int cnt, uint16_t syt)
{
	av1394_it_t	*itp = &icp->ic_it;
	int		i;
	int		j = first;	/* frame number */

	for (i = cnt; i > 0; i--) {
		*(uint16_t *)itp->it_frame_info[j].fi_ts_off = syt;
		syt = av1394_it_ts_syt_inc(syt, itp->it_frame_info[j].fi_ncycs);
		j = (j + 1) % icp->ic_nframes;
	}
}

/*
 * convert cycle timestamp into SYT timestamp:
 *
 * Cycle timer:          cycleSeconds         cycleCount     cycleOffset
 *                   31_30_29_28_27_26_25  24___15_14_13_12  11________0
 * Cycle timestamp:              |------------------------|
 * SYT timestamp:                               |----------------------|
 */
static uint16_t
av1394_it_ts_cyc2syt(uint16_t cyc)
{
	return (((cyc & 0xF) << 12) + 0x800);
}

/*
 * increment SYT by a number of cycles
 */
static uint16_t
av1394_it_ts_syt_inc(uint16_t syt, uint16_t ncycs)
{
	return (syt + (ncycs << 12));
}

/*
 * copyin from the kernel buffer
 */
static void
av1394_it_kcopyin(av1394_ic_t *icp, void *buf, size_t len)
{
	av1394_it_t	*itp = &icp->ic_it;
	av1394_isoch_seg_t *seg = itp->it_data_pool.ip_seg;

	ASSERT(itp->it_write_off + len < icp->ic_framesz);

	bcopy(buf, seg[itp->it_write_idx].is_kaddr + itp->it_write_off, len);
	itp->it_write_off += len;
}

/*
 * copyin from the user buffer
 */
static int
av1394_it_copyin(av1394_ic_t *icp, struct uio *uiop, int *full_cnt, int dv)
{
	av1394_it_t	*itp = &icp->ic_it;
	av1394_isoch_seg_t *seg = itp->it_data_pool.ip_seg;
	int		idx = itp->it_write_idx;
	int		framesz = icp->ic_framesz;
	size_t		len, frame_resid, start_resid;
	caddr_t		kaddr, kaddr_end;
	int		ret = 0;

	*full_cnt = 0;

	while ((uiop->uio_resid > 0) && (itp->it_write_cnt > 0)) {
		kaddr = seg[idx].is_kaddr + itp->it_write_off;
		frame_resid = framesz - itp->it_write_off;
		len = min(uiop->uio_resid, frame_resid);

		mutex_exit(&icp->ic_mutex);
		ret = uiomove(kaddr, len, UIO_WRITE, uiop);
		mutex_enter(&icp->ic_mutex);
		if (ret != 0) {
			break;
		}

		itp->it_write_off += len;
		if ((itp->it_write_off == framesz) && dv) {
			/*
			 * for DV formats, make sure we got a frame start.
			 * this is to ensure correct timestamping
			 */
			kaddr = seg[idx].is_kaddr;
			kaddr_end = kaddr + framesz;
			while (!av1394_it_is_dv_frame_start(kaddr)) {
				kaddr += icp->ic_pktsz;
				if (kaddr == kaddr_end) {
					break;
				}
			}
			start_resid = kaddr_end - kaddr;
			if (start_resid != framesz) {
				bcopy(kaddr, seg[idx].is_kaddr, start_resid);
				itp->it_write_off = start_resid;
			}
		}
		if (itp->it_write_off == framesz) {
			/* for DV formats, reset frame's SYT fields */
			if (dv) {
				av1394_it_reset_frame_syt(icp, idx);
			}
			itp->it_write_off = 0;
			itp->it_write_cnt--;
			idx = (idx + 1) % icp->ic_nframes;
			(*full_cnt)++;
		}
	}

	return (ret);
}

/*
 * check if a packet starts a new DV frame
 */
static boolean_t
av1394_it_is_dv_frame_start(caddr_t kaddr)
{
	uint8_t *p = (uint8_t *)kaddr + 8;
	/*
	 * in the DIF block ID data, which immediately follows CIP header,
	 * SCT, Dseq and DBN fields should be zero (Ref: IEC 61834-2, Fig. 66)
	 */
	return (((p[0] & 0xE0) == 0) && ((p[1] & 0xF0) == 0) && (p[2] == 0));
}

/*
 * reset all frame's SYT fields
 */
static void
av1394_it_reset_frame_syt(av1394_ic_t *icp, int idx)
{
	caddr_t		kaddr = icp->ic_it.it_data_pool.ip_seg[idx].is_kaddr;
	caddr_t		kaddr_end = kaddr + icp->ic_framesz;

	kaddr += 6;
	while (kaddr < kaddr_end) {
		*(uint16_t *)kaddr = 0xFFFF;
		kaddr += icp->ic_pktsz;
	}
}