xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/pmcs/pmcs_scsa.c (revision b46556d07aa5469d61a8b3e3067bb7c3c26f0a7c)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * SCSI (SCSA) midlayer interface for PMC drier.
26  */
27 
28 #include <sys/scsi/adapters/pmcs/pmcs.h>
29 
30 extern scsi_lun_t scsi_lun64_to_lun(scsi_lun64_t lun64);
31 
32 static int pmcs_scsa_tran_tgt_init(dev_info_t *, dev_info_t *,
33     scsi_hba_tran_t *, struct scsi_device *);
34 static void pmcs_scsa_tran_tgt_free(dev_info_t *, dev_info_t *,
35     scsi_hba_tran_t *, struct scsi_device *);
36 static int pmcs_scsa_start(struct scsi_address *, struct scsi_pkt *);
37 static int pmcs_scsa_abort(struct scsi_address *, struct scsi_pkt *);
38 static int pmcs_scsa_reset(struct scsi_address *, int);
39 static int pmcs_scsi_reset_notify(struct scsi_address *, int,
40     void (*)(caddr_t), caddr_t);
41 static int pmcs_scsa_getcap(struct scsi_address *, char *, int);
42 static int pmcs_scsa_setcap(struct scsi_address *, char *, int, int);
43 static int pmcs_scsa_setup_pkt(struct scsi_pkt *, int (*)(caddr_t), caddr_t);
44 static void pmcs_scsa_teardown_pkt(struct scsi_pkt *);
45 
46 static int pmcs_smp_init(dev_info_t *, dev_info_t *, smp_hba_tran_t *,
47     smp_device_t *);
48 static void pmcs_smp_free(dev_info_t *, dev_info_t *, smp_hba_tran_t *,
49     smp_device_t *);
50 static int pmcs_smp_start(struct smp_pkt *);
51 
52 static int pmcs_scsi_quiesce(dev_info_t *);
53 static int pmcs_scsi_unquiesce(dev_info_t *);
54 
55 static int pmcs_cap(struct scsi_address *, char *, int, int, int);
56 static pmcs_xscsi_t *
57     pmcs_addr2xp(struct scsi_address *, uint64_t *, pmcs_cmd_t *);
58 static int pmcs_SAS_run(pmcs_cmd_t *, pmcwork_t *);
59 static void pmcs_SAS_done(pmcs_hw_t *, pmcwork_t *, uint32_t *);
60 
61 static int pmcs_SATA_run(pmcs_cmd_t *, pmcwork_t *);
62 static void pmcs_SATA_done(pmcs_hw_t *, pmcwork_t *, uint32_t *);
63 static uint8_t pmcs_SATA_rwparm(uint8_t *, uint32_t *, uint64_t *, uint64_t);
64 
65 static void pmcs_ioerror(pmcs_hw_t *, pmcs_dtype_t pmcs_dtype,
66     pmcwork_t *, uint32_t *, uint32_t);
67 
68 
69 int
pmcs_scsa_init(pmcs_hw_t * pwp,const ddi_dma_attr_t * ap)70 pmcs_scsa_init(pmcs_hw_t *pwp, const ddi_dma_attr_t *ap)
71 {
72 	scsi_hba_tran_t *tran;
73 	ddi_dma_attr_t pmcs_scsa_dattr;
74 	int flags;
75 
76 	(void) memcpy(&pmcs_scsa_dattr, ap, sizeof (ddi_dma_attr_t));
77 	pmcs_scsa_dattr.dma_attr_sgllen =
78 	    ((PMCS_SGL_NCHUNKS - 1) * (PMCS_MAX_CHUNKS - 1)) + PMCS_SGL_NCHUNKS;
79 	pmcs_scsa_dattr.dma_attr_flags = DDI_DMA_RELAXED_ORDERING;
80 	pmcs_scsa_dattr.dma_attr_flags |= DDI_DMA_FLAGERR;
81 
82 	/*
83 	 * Allocate a transport structure
84 	 */
85 	tran = scsi_hba_tran_alloc(pwp->dip, SCSI_HBA_CANSLEEP);
86 	if (tran == NULL) {
87 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
88 		    "scsi_hba_tran_alloc failed");
89 		return (DDI_FAILURE);
90 	}
91 
92 	tran->tran_hba_private		= pwp;
93 	tran->tran_tgt_init		= pmcs_scsa_tran_tgt_init;
94 	tran->tran_tgt_free		= pmcs_scsa_tran_tgt_free;
95 	tran->tran_start		= pmcs_scsa_start;
96 	tran->tran_abort		= pmcs_scsa_abort;
97 	tran->tran_reset		= pmcs_scsa_reset;
98 	tran->tran_reset_notify		= pmcs_scsi_reset_notify;
99 	tran->tran_getcap		= pmcs_scsa_getcap;
100 	tran->tran_setcap		= pmcs_scsa_setcap;
101 	tran->tran_setup_pkt		= pmcs_scsa_setup_pkt;
102 	tran->tran_teardown_pkt		= pmcs_scsa_teardown_pkt;
103 	tran->tran_quiesce		= pmcs_scsi_quiesce;
104 	tran->tran_unquiesce		= pmcs_scsi_unquiesce;
105 	tran->tran_interconnect_type	= INTERCONNECT_SAS;
106 	tran->tran_hba_len		= sizeof (pmcs_cmd_t);
107 
108 	/*
109 	 * Attach this instance of the hba
110 	 */
111 
112 	flags = SCSI_HBA_TRAN_SCB | SCSI_HBA_TRAN_CDB | SCSI_HBA_ADDR_COMPLEX |
113 	    SCSI_HBA_TRAN_PHCI | SCSI_HBA_HBA;
114 
115 	if (scsi_hba_attach_setup(pwp->dip, &pmcs_scsa_dattr, tran, flags)) {
116 		scsi_hba_tran_free(tran);
117 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
118 		    "scsi_hba_attach failed");
119 		return (DDI_FAILURE);
120 	}
121 	pwp->tran = tran;
122 
123 	/*
124 	 * Attach the SMP part of this hba
125 	 */
126 	pwp->smp_tran = smp_hba_tran_alloc(pwp->dip);
127 	ASSERT(pwp->smp_tran != NULL);
128 	pwp->smp_tran->smp_tran_hba_private = pwp;
129 	pwp->smp_tran->smp_tran_init = pmcs_smp_init;
130 	pwp->smp_tran->smp_tran_free = pmcs_smp_free;
131 	pwp->smp_tran->smp_tran_start = pmcs_smp_start;
132 
133 	if (smp_hba_attach_setup(pwp->dip, pwp->smp_tran) != DDI_SUCCESS) {
134 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
135 		    "smp_hba_attach failed");
136 		smp_hba_tran_free(pwp->smp_tran);
137 		pwp->smp_tran = NULL;
138 		scsi_hba_tran_free(tran);
139 		return (DDI_FAILURE);
140 	}
141 
142 	return (DDI_SUCCESS);
143 }
144 
145 /*
146  * SCSA entry points
147  */
148 
149 static int
pmcs_scsa_tran_tgt_init(dev_info_t * hba_dip,dev_info_t * tgt_dip,scsi_hba_tran_t * tran,struct scsi_device * sd)150 pmcs_scsa_tran_tgt_init(dev_info_t *hba_dip, dev_info_t *tgt_dip,
151     scsi_hba_tran_t *tran, struct scsi_device *sd)
152 {
153 	pmcs_hw_t	*pwp = NULL;
154 	int		rval;
155 	char		*variant_prop = "sata";
156 	char		*tgt_port = NULL, *ua = NULL;
157 	pmcs_xscsi_t	*tgt = NULL;
158 	pmcs_iport_t	*iport;
159 	pmcs_lun_t	*lun = NULL;
160 	pmcs_phy_t	*phyp = NULL;
161 	uint64_t	lun_num;
162 	boolean_t	got_scratch = B_FALSE;
163 
164 	/*
165 	 * First, make sure we're an iport and get the pointer to the HBA
166 	 * node's softstate
167 	 */
168 	if (scsi_hba_iport_unit_address(hba_dip) == NULL) {
169 		pmcs_prt(TRAN2PMC(tran), PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
170 		    "%s: We don't enumerate devices on the HBA node", __func__);
171 		goto tgt_init_fail;
172 	}
173 
174 	pwp = ITRAN2PMC(tran);
175 	iport = ITRAN2IPORT(tran);
176 
177 	/*
178 	 * Get the unit-address
179 	 */
180 	ua = scsi_device_unit_address(sd);
181 	if (ua == NULL) {
182 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
183 		    "%s: Couldn't get UA", __func__);
184 		pwp = NULL;
185 		goto tgt_init_fail;
186 	}
187 	pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
188 	    "got ua '%s'", ua);
189 
190 	/*
191 	 * Get the target address
192 	 */
193 	rval = scsi_device_prop_lookup_string(sd, SCSI_DEVICE_PROP_PATH,
194 	    SCSI_ADDR_PROP_TARGET_PORT, &tgt_port);
195 	if (rval != DDI_PROP_SUCCESS) {
196 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
197 		    "Couldn't get target UA");
198 		pwp = NULL;
199 		goto tgt_init_fail;
200 	}
201 	pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
202 	    "got tgt_port '%s'", tgt_port);
203 
204 	/*
205 	 * Validate that this tran_tgt_init is for an active iport.
206 	 */
207 	if (iport->ua_state == UA_INACTIVE) {
208 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
209 		    "%s: Got tran_tgt_init on inactive iport for '%s'",
210 		    __func__, tgt_port);
211 		pwp = NULL;
212 		goto tgt_init_fail;
213 	}
214 
215 	/*
216 	 * Since we're going to wait for scratch, be sure to acquire it while
217 	 * we're not holding any other locks
218 	 */
219 	(void) pmcs_acquire_scratch(pwp, B_TRUE);
220 	got_scratch = B_TRUE;
221 
222 	mutex_enter(&pwp->lock);
223 
224 	/*
225 	 * See if there's already a target softstate.  If not, allocate one.
226 	 */
227 	tgt = pmcs_get_target(iport, tgt_port, B_TRUE);
228 
229 	if (tgt == NULL) {
230 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "%s: "
231 		    "No tgt for tgt_port (%s)", __func__, tgt_port);
232 		goto tgt_init_fail;
233 	}
234 
235 	phyp = tgt->phy;
236 	if (!IS_ROOT_PHY(phyp)) {
237 		pmcs_inc_phy_ref_count(phyp);
238 	}
239 	ASSERT(mutex_owned(&phyp->phy_lock));
240 
241 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, tgt, "@%s tgt = 0x%p, dip = 0x%p",
242 	    ua, (void *)tgt, (void *)tgt_dip);
243 
244 	/* Now get the lun */
245 	lun_num = scsi_device_prop_get_int64(sd, SCSI_DEVICE_PROP_PATH,
246 	    SCSI_ADDR_PROP_LUN64, SCSI_LUN64_ILLEGAL);
247 	if (lun_num == SCSI_LUN64_ILLEGAL) {
248 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
249 		    "No LUN for tgt %p", (void *)tgt);
250 		goto tgt_init_fail;
251 	}
252 
253 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt, "%s: @%s tgt 0x%p phy "
254 	    "0x%p (%s)", __func__, ua, (void *)tgt, (void *)phyp, phyp->path);
255 
256 	mutex_enter(&tgt->statlock);
257 	tgt->dtype = phyp->dtype;
258 	if (tgt->dtype != SAS && tgt->dtype != SATA) {
259 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
260 		    "PHY 0x%p went away?", (void *)phyp);
261 		goto tgt_init_fail;
262 	}
263 
264 	/* We don't support SATA devices at LUN > 0. */
265 	if ((tgt->dtype == SATA) && (lun_num > 0)) {
266 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
267 		    "%s: No support for SATA devices at LUN > 0 "
268 		    "(target = 0x%p)", __func__, (void *)tgt);
269 		goto tgt_init_fail;
270 	}
271 
272 	/*
273 	 * Allocate LU soft state. We use ddi_soft_state_bystr_zalloc instead
274 	 * of kmem_alloc because ddi_soft_state_bystr_zalloc allows us to
275 	 * verify that the framework never tries to initialize two scsi_device
276 	 * structures with the same unit-address at the same time.
277 	 */
278 	if (ddi_soft_state_bystr_zalloc(tgt->lun_sstate, ua) != DDI_SUCCESS) {
279 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, tgt,
280 		    "Couldn't allocate LU soft state");
281 		goto tgt_init_fail;
282 	}
283 
284 	lun = ddi_soft_state_bystr_get(tgt->lun_sstate, ua);
285 	if (lun == NULL) {
286 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, tgt,
287 		    "Couldn't get LU soft state");
288 		goto tgt_init_fail;
289 	}
290 	scsi_device_hba_private_set(sd, lun);
291 	lun->lun_num = lun_num;
292 
293 	/* convert the scsi_lun64_t value to SCSI standard form */
294 	lun->scsi_lun = scsi_lun64_to_lun(lun_num);
295 
296 	ASSERT(strlen(ua) < (PMCS_MAX_UA_SIZE - 1));
297 	bcopy(ua, lun->unit_address, strnlen(ua, PMCS_MAX_UA_SIZE - 1));
298 
299 	lun->target = tgt;
300 
301 	/*
302 	 * If this is the first tran_tgt_init, add this target to our list
303 	 */
304 	if (tgt->target_num == PMCS_INVALID_TARGET_NUM) {
305 		int target;
306 		for (target = 0; target < pwp->max_dev; target++) {
307 			if (pwp->targets[target] != NULL) {
308 				continue;
309 			}
310 
311 			pwp->targets[target] = tgt;
312 			tgt->target_num = (uint16_t)target;
313 			break;
314 		}
315 
316 		if (target == pwp->max_dev) {
317 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
318 			    "Target list full.");
319 			goto tgt_init_fail;
320 		}
321 	}
322 
323 	tgt->dip = sd->sd_dev;
324 	lun->sd = sd;
325 	list_insert_tail(&tgt->lun_list, lun);
326 
327 	if (!pmcs_assign_device(pwp, tgt)) {
328 		pmcs_release_scratch(pwp);
329 		pwp->targets[tgt->target_num] = NULL;
330 		tgt->target_num = PMCS_INVALID_TARGET_NUM;
331 		tgt->phy = NULL;
332 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
333 		    "%s: pmcs_assign_device failed for target 0x%p",
334 		    __func__, (void *)tgt);
335 		goto tgt_init_fail;
336 	}
337 
338 	pmcs_release_scratch(pwp);
339 	tgt->ref_count++;
340 
341 	(void) scsi_device_prop_update_int(sd, SCSI_DEVICE_PROP_PATH,
342 	    SCSI_ADDR_PROP_TARGET, (uint32_t)(tgt->target_num));
343 
344 	/* SM-HBA */
345 	if (tgt->dtype == SATA) {
346 		/* TCR in PSARC/1997/281 opinion */
347 		(void) scsi_device_prop_update_string(sd,
348 		    SCSI_DEVICE_PROP_PATH, "variant", variant_prop);
349 	}
350 
351 	tgt->phy_addressable = PMCS_PHY_ADDRESSABLE(phyp);
352 
353 	if (tgt->phy_addressable) {
354 		(void) scsi_device_prop_update_int(sd, SCSI_DEVICE_PROP_PATH,
355 		    SCSI_ADDR_PROP_SATA_PHY, phyp->phynum);
356 	}
357 
358 	/* SM-HBA */
359 	(void) pmcs_smhba_set_scsi_device_props(pwp, phyp, sd);
360 	/*
361 	 * Make sure attached port and target port pm props are updated
362 	 * By passing in 0s, we're not actually updating any values, but
363 	 * the properties should now get updated on the node.
364 	 */
365 
366 	mutex_exit(&tgt->statlock);
367 	pmcs_update_phy_pm_props(phyp, 0, 0, B_TRUE);
368 	pmcs_unlock_phy(phyp);
369 	mutex_exit(&pwp->lock);
370 	scsi_device_prop_free(sd, SCSI_DEVICE_PROP_PATH, tgt_port);
371 	return (DDI_SUCCESS);
372 
373 tgt_init_fail:
374 	scsi_device_hba_private_set(sd, NULL);
375 	if (got_scratch) {
376 		pmcs_release_scratch(pwp);
377 	}
378 	if (lun) {
379 		list_remove(&tgt->lun_list, lun);
380 		ddi_soft_state_bystr_free(tgt->lun_sstate, ua);
381 	}
382 	if (phyp) {
383 		mutex_exit(&tgt->statlock);
384 		pmcs_unlock_phy(phyp);
385 		/*
386 		 * phyp's ref count was incremented in pmcs_new_tport.
387 		 * We're failing configuration, we now need to decrement it.
388 		 */
389 		if (!IS_ROOT_PHY(phyp)) {
390 			pmcs_dec_phy_ref_count(phyp);
391 		}
392 		phyp->target = NULL;
393 	}
394 	if (tgt && tgt->ref_count == 0) {
395 		ddi_soft_state_bystr_free(iport->tgt_sstate, tgt_port);
396 	}
397 	if (pwp) {
398 		mutex_exit(&pwp->lock);
399 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
400 		    "%s: failed for @%s tgt 0x%p phy 0x%p", __func__, ua,
401 		    (void *)tgt, (void *)phyp);
402 	}
403 	if (tgt_port) {
404 		scsi_device_prop_free(sd, SCSI_DEVICE_PROP_PATH, tgt_port);
405 	}
406 	return (DDI_FAILURE);
407 }
408 
409 static void
pmcs_scsa_tran_tgt_free(dev_info_t * hba_dip,dev_info_t * tgt_dip,scsi_hba_tran_t * tran,struct scsi_device * sd)410 pmcs_scsa_tran_tgt_free(dev_info_t *hba_dip, dev_info_t *tgt_dip,
411     scsi_hba_tran_t *tran, struct scsi_device *sd)
412 {
413 	_NOTE(ARGUNUSED(hba_dip, tgt_dip));
414 	pmcs_hw_t	*pwp;
415 	pmcs_lun_t	*lun;
416 	pmcs_xscsi_t	*target;
417 	char		*unit_address;
418 	pmcs_phy_t	*phyp;
419 
420 	if (scsi_hba_iport_unit_address(hba_dip) == NULL) {
421 		pwp = TRAN2PMC(tran);
422 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
423 		    "%s: We don't enumerate devices on the HBA node", __func__);
424 		return;
425 	}
426 
427 	lun = (pmcs_lun_t *)scsi_device_hba_private_get(sd);
428 
429 	ASSERT((lun != NULL) && (lun->target != NULL));
430 	ASSERT(lun->target->ref_count > 0);
431 
432 	target = lun->target;
433 	unit_address = lun->unit_address;
434 	list_remove(&target->lun_list, lun);
435 
436 	pwp = ITRAN2PMC(tran);
437 	mutex_enter(&pwp->lock);
438 	phyp = target->phy;
439 	if (phyp) {
440 		mutex_enter(&phyp->phy_lock);
441 	}
442 	mutex_enter(&target->statlock);
443 
444 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, target,
445 	    "%s: for @%s tgt 0x%p phy 0x%p", __func__, unit_address,
446 	    (void *)target, (void *)phyp);
447 	ddi_soft_state_bystr_free(lun->target->lun_sstate, unit_address);
448 
449 	if (target->recover_wait) {
450 		mutex_exit(&target->statlock);
451 		if (phyp) {
452 			mutex_exit(&phyp->phy_lock);
453 		}
454 		mutex_exit(&pwp->lock);
455 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, target, "%s: "
456 		    "Target 0x%p in device state recovery, fail tran_tgt_free",
457 		    __func__, (void *)target);
458 		return;
459 	}
460 
461 	/*
462 	 * If this target still has a PHY pointer and that PHY's target pointer
463 	 * has been cleared, then that PHY has been reaped. In that case, there
464 	 * would be no need to decrement the reference count
465 	 */
466 	if (phyp && !IS_ROOT_PHY(phyp) && phyp->target) {
467 		pmcs_dec_phy_ref_count(phyp);
468 	}
469 
470 	if (--target->ref_count == 0) {
471 		/*
472 		 * Remove this target from our list.  The target soft
473 		 * state will remain, and the device will remain registered
474 		 * with the hardware unless/until we're told the device
475 		 * physically went away.
476 		 */
477 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, target,
478 		    "%s: Free target 0x%p (vtgt %d)", __func__, (void *)target,
479 		    target->target_num);
480 		pwp->targets[target->target_num] = NULL;
481 		target->target_num = PMCS_INVALID_TARGET_NUM;
482 		/* If the PHY has a pointer to this target, clear it */
483 		if (phyp && (phyp->target == target)) {
484 			phyp->target = NULL;
485 		}
486 		target->phy = NULL;
487 		if (phyp) {
488 			mutex_exit(&phyp->phy_lock);
489 		}
490 		pmcs_destroy_target(target);
491 	} else {
492 		mutex_exit(&target->statlock);
493 		if (phyp) {
494 			mutex_exit(&phyp->phy_lock);
495 		}
496 	}
497 
498 	mutex_exit(&pwp->lock);
499 }
500 
501 static int
pmcs_scsa_start(struct scsi_address * ap,struct scsi_pkt * pkt)502 pmcs_scsa_start(struct scsi_address *ap, struct scsi_pkt *pkt)
503 {
504 	pmcs_cmd_t *sp = PKT2CMD(pkt);
505 	pmcs_hw_t *pwp = ADDR2PMC(ap);
506 	pmcs_xscsi_t *xp;
507 	boolean_t blocked;
508 	uint32_t hba_state;
509 
510 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
511 	    "%s: pkt %p sd %p cdb0=0x%02x dl=%lu", __func__, (void *)pkt,
512 	    (void *)scsi_address_device(&pkt->pkt_address),
513 	    pkt->pkt_cdbp[0] & 0xff, pkt->pkt_dma_len);
514 
515 	if (pkt->pkt_flags & FLAG_NOINTR) {
516 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
517 		    "%s: nointr pkt", __func__);
518 		return (TRAN_BADPKT);
519 	}
520 
521 	sp->cmd_tag = 0;
522 	pkt->pkt_state = pkt->pkt_statistics = 0;
523 	pkt->pkt_reason = CMD_INCOMPLETE;
524 
525 	mutex_enter(&pwp->lock);
526 	hba_state = pwp->state;
527 	blocked = pwp->blocked;
528 	mutex_exit(&pwp->lock);
529 
530 	if (hba_state != STATE_RUNNING) {
531 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
532 		    "%s: hba dead", __func__);
533 		return (TRAN_FATAL_ERROR);
534 	}
535 
536 	xp = pmcs_addr2xp(ap, NULL, sp);
537 	if (xp == NULL) {
538 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
539 		    "%s: dropping due to null target", __func__);
540 		goto dead_target;
541 	}
542 	ASSERT(mutex_owned(&xp->statlock));
543 
544 	/*
545 	 * First, check to see if the device is gone.
546 	 */
547 	if (xp->dev_gone) {
548 		xp->actv_pkts++;
549 		mutex_exit(&xp->statlock);
550 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, xp,
551 		    "%s: dropping due to dead target 0x%p",
552 		    __func__, (void *)xp);
553 		goto dead_target;
554 	}
555 
556 	/*
557 	 * If we're blocked (quiesced) just return.
558 	 */
559 	if (blocked) {
560 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
561 		    "%s: hba blocked", __func__);
562 		xp->actv_pkts++;
563 		mutex_exit(&xp->statlock);
564 		mutex_enter(&xp->wqlock);
565 		STAILQ_INSERT_TAIL(&xp->wq, sp, cmd_next);
566 		mutex_exit(&xp->wqlock);
567 		return (TRAN_ACCEPT);
568 	}
569 
570 	/*
571 	 * If we're draining or resetting, queue and return.
572 	 */
573 	if (xp->draining || xp->resetting || xp->recover_wait) {
574 		xp->actv_pkts++;
575 		mutex_exit(&xp->statlock);
576 		mutex_enter(&xp->wqlock);
577 		STAILQ_INSERT_TAIL(&xp->wq, sp, cmd_next);
578 		mutex_exit(&xp->wqlock);
579 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, xp,
580 		    "%s: draining/resetting/recovering (cnt %u)",
581 		    __func__, xp->actv_cnt);
582 		/*
583 		 * By the time we get here, draining or
584 		 * resetting may have come and gone, not
585 		 * yet noticing that we had put something
586 		 * on the wait queue, so schedule a worker
587 		 * to look at this later.
588 		 */
589 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
590 		return (TRAN_ACCEPT);
591 	}
592 
593 	xp->actv_pkts++;
594 	mutex_exit(&xp->statlock);
595 
596 	/*
597 	 * Queue this command to the tail of the wait queue.
598 	 * This keeps us getting commands out of order.
599 	 */
600 	mutex_enter(&xp->wqlock);
601 	STAILQ_INSERT_TAIL(&xp->wq, sp, cmd_next);
602 	mutex_exit(&xp->wqlock);
603 
604 	/*
605 	 * Now run the queue for this device.
606 	 */
607 	(void) pmcs_scsa_wq_run_one(pwp, xp);
608 
609 	return (TRAN_ACCEPT);
610 
611 dead_target:
612 	pkt->pkt_state = STATE_GOT_BUS;
613 	pkt->pkt_reason = CMD_DEV_GONE;
614 	mutex_enter(&pwp->cq_lock);
615 	STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
616 	PMCS_CQ_RUN_LOCKED(pwp);
617 	mutex_exit(&pwp->cq_lock);
618 	return (TRAN_ACCEPT);
619 }
620 
621 /* Return code 1 = Success */
622 static int
pmcs_scsa_abort(struct scsi_address * ap,struct scsi_pkt * pkt)623 pmcs_scsa_abort(struct scsi_address *ap, struct scsi_pkt *pkt)
624 {
625 	pmcs_hw_t *pwp = ADDR2PMC(ap);
626 	pmcs_cmd_t *sp = NULL;
627 	pmcs_xscsi_t *xp = NULL;
628 	pmcs_phy_t *pptr = NULL;
629 	pmcs_lun_t *pmcs_lun = (pmcs_lun_t *)
630 	    scsi_device_hba_private_get(scsi_address_device(ap));
631 	uint32_t tag;
632 	uint64_t lun;
633 	pmcwork_t *pwrk;
634 
635 	mutex_enter(&pwp->lock);
636 	if (pwp->state != STATE_RUNNING) {
637 		mutex_exit(&pwp->lock);
638 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
639 		    "%s: hba dead", __func__);
640 		return (0);
641 	}
642 	mutex_exit(&pwp->lock);
643 
644 	if (pkt == NULL) {
645 		if (pmcs_lun == NULL) {
646 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: "
647 			    "No pmcs_lun_t struct to do ABORT_ALL", __func__);
648 			return (0);
649 		}
650 		xp = pmcs_lun->target;
651 		if (xp != NULL) {
652 			pptr = xp->phy;
653 		}
654 		if (pptr == NULL) {
655 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp, "%s: pkt is "
656 			    "NULL. No tgt/phy to do ABORT_ALL", __func__);
657 			return (0);
658 		}
659 		pmcs_lock_phy(pptr);
660 		if (pmcs_abort(pwp, pptr, 0, 1, 0)) {
661 			pptr->abort_pending = 1;
662 			SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
663 		}
664 		pmcs_unlock_phy(pptr);
665 		return (1);
666 	}
667 
668 	sp = PKT2CMD(pkt);
669 	xp = sp->cmd_target;
670 
671 	if (sp->cmd_lun) {
672 		lun = sp->cmd_lun->lun_num;
673 	} else {
674 		lun = 0;
675 	}
676 	if (xp == NULL) {
677 		return (0);
678 	}
679 
680 	/*
681 	 * See if we have a real work structure associated with this cmd.
682 	 */
683 	pwrk = pmcs_tag2wp(pwp, sp->cmd_tag, B_FALSE);
684 	if (pwrk && pwrk->arg == sp) {
685 		tag = pwrk->htag;
686 		pptr = pwrk->phy;
687 		pwrk->timer = 0;	/* we don't time this here */
688 		ASSERT(pwrk->state == PMCS_WORK_STATE_ONCHIP);
689 		mutex_exit(&pwrk->lock);
690 		pmcs_lock_phy(pptr);
691 		if (pptr->dtype == SAS) {
692 			if (pmcs_ssp_tmf(pwp, pptr, SAS_ABORT_TASK, tag, lun,
693 			    NULL)) {
694 				pptr->abort_pending = 1;
695 				pmcs_unlock_phy(pptr);
696 				SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
697 				return (0);
698 			}
699 		} else {
700 			/*
701 			 * XXX: Was the command that was active an
702 			 * NCQ I/O command?
703 			 */
704 			pptr->need_rl_ext = 1;
705 			if (pmcs_sata_abort_ncq(pwp, pptr)) {
706 				pptr->abort_pending = 1;
707 				pmcs_unlock_phy(pptr);
708 				SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
709 				return (0);
710 			}
711 		}
712 		pptr->abort_pending = 1;
713 		pmcs_unlock_phy(pptr);
714 		SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
715 		return (1);
716 	}
717 	if (pwrk) {
718 		mutex_exit(&pwrk->lock);
719 	}
720 	/*
721 	 * Okay, those weren't the droids we were looking for.
722 	 * See if the command is on any of the wait queues.
723 	 */
724 	mutex_enter(&xp->wqlock);
725 	sp = NULL;
726 	STAILQ_FOREACH(sp, &xp->wq, cmd_next) {
727 		if (sp == PKT2CMD(pkt)) {
728 			STAILQ_REMOVE(&xp->wq, sp, pmcs_cmd, cmd_next);
729 			break;
730 		}
731 	}
732 	mutex_exit(&xp->wqlock);
733 	if (sp) {
734 		pkt->pkt_reason = CMD_ABORTED;
735 		pkt->pkt_statistics |= STAT_ABORTED;
736 		mutex_enter(&pwp->cq_lock);
737 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
738 		PMCS_CQ_RUN_LOCKED(pwp);
739 		mutex_exit(&pwp->cq_lock);
740 		return (1);
741 	}
742 	return (0);
743 }
744 
745 /*
746  * SCSA reset functions
747  */
748 static int
pmcs_scsa_reset(struct scsi_address * ap,int level)749 pmcs_scsa_reset(struct scsi_address *ap, int level)
750 {
751 	pmcs_hw_t *pwp = ADDR2PMC(ap);
752 	pmcs_phy_t *pptr;
753 	pmcs_xscsi_t *xp;
754 	uint64_t lun = (uint64_t)-1, *lp = NULL;
755 	int rval;
756 
757 	mutex_enter(&pwp->lock);
758 	if (pwp->state != STATE_RUNNING) {
759 		mutex_exit(&pwp->lock);
760 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
761 		    "%s: hba dead", __func__);
762 		return (0);
763 	}
764 	mutex_exit(&pwp->lock);
765 
766 	switch (level)  {
767 	case RESET_ALL:
768 		rval = 0;
769 		break;
770 	case RESET_LUN:
771 		/*
772 		 * Point lp at lun so that pmcs_addr2xp
773 		 * will fill out the 64 bit lun number.
774 		 */
775 		lp = &lun;
776 		/* FALLTHROUGH */
777 	case RESET_TARGET:
778 		xp = pmcs_addr2xp(ap, lp, NULL);
779 		if (xp == NULL) {
780 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
781 			    "%s: no xp found for this scsi address", __func__);
782 			return (0);
783 		}
784 
785 		if (xp->dev_gone) {
786 			mutex_exit(&xp->statlock);
787 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
788 			    "%s: Target 0x%p has gone away", __func__,
789 			    (void *)xp);
790 			return (0);
791 		}
792 
793 		/*
794 		 * If we're already performing this action, or if device
795 		 * state recovery is already running, just return failure.
796 		 */
797 		if (xp->resetting || xp->recover_wait) {
798 			mutex_exit(&xp->statlock);
799 			return (0);
800 		}
801 		xp->reset_wait = 0;
802 		xp->reset_success = 0;
803 		xp->resetting = 1;
804 		pptr = xp->phy;
805 		mutex_exit(&xp->statlock);
806 
807 		if (pmcs_reset_dev(pwp, pptr, lun)) {
808 			rval = 0;
809 		} else {
810 			rval = 1;
811 		}
812 
813 		mutex_enter(&xp->statlock);
814 		if (rval == 1) {
815 			xp->reset_success = 1;
816 		}
817 		if (xp->reset_wait) {
818 			xp->reset_wait = 0;
819 			cv_signal(&xp->reset_cv);
820 		}
821 		xp->resetting = 0;
822 		mutex_exit(&xp->statlock);
823 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
824 		break;
825 	default:
826 		rval = 0;
827 		break;
828 	}
829 
830 	return (rval);
831 }
832 
833 static int
pmcs_scsi_reset_notify(struct scsi_address * ap,int flag,void (* callback)(caddr_t),caddr_t arg)834 pmcs_scsi_reset_notify(struct scsi_address *ap, int flag,
835     void (*callback)(caddr_t), caddr_t arg)
836 {
837 	pmcs_hw_t *pwp = ADDR2PMC(ap);
838 	return (scsi_hba_reset_notify_setup(ap, flag, callback, arg,
839 	    &pwp->lock, &pwp->reset_notify_listf));
840 }
841 
842 
843 static int
pmcs_cap(struct scsi_address * ap,char * cap,int val,int tonly,int set)844 pmcs_cap(struct scsi_address *ap, char *cap, int val, int tonly, int set)
845 {
846 	_NOTE(ARGUNUSED(val, tonly));
847 	int cidx, rval = 0;
848 	pmcs_xscsi_t *xp;
849 
850 	cidx = scsi_hba_lookup_capstr(cap);
851 	if (cidx == -1) {
852 		return (-1);
853 	}
854 
855 	xp = pmcs_addr2xp(ap, NULL, NULL);
856 	if (xp == NULL) {
857 		return (-1);
858 	}
859 
860 	switch (cidx) {
861 	case SCSI_CAP_DMA_MAX:
862 	case SCSI_CAP_INITIATOR_ID:
863 		if (set == 0) {
864 			rval = INT_MAX;	/* argh */
865 		}
866 		break;
867 	case SCSI_CAP_DISCONNECT:
868 	case SCSI_CAP_SYNCHRONOUS:
869 	case SCSI_CAP_WIDE_XFER:
870 	case SCSI_CAP_PARITY:
871 	case SCSI_CAP_ARQ:
872 	case SCSI_CAP_UNTAGGED_QING:
873 		if (set == 0) {
874 			rval = 1;
875 		}
876 		break;
877 
878 	case SCSI_CAP_TAGGED_QING:
879 		rval = 1;
880 		break;
881 
882 	case SCSI_CAP_MSG_OUT:
883 	case SCSI_CAP_RESET_NOTIFICATION:
884 	case SCSI_CAP_QFULL_RETRIES:
885 	case SCSI_CAP_QFULL_RETRY_INTERVAL:
886 		break;
887 	case SCSI_CAP_SCSI_VERSION:
888 		if (set == 0) {
889 			rval = SCSI_VERSION_3;
890 		}
891 		break;
892 	case SCSI_CAP_INTERCONNECT_TYPE:
893 		if (set) {
894 			break;
895 		}
896 		if (xp->phy_addressable) {
897 			rval = INTERCONNECT_SATA;
898 		} else {
899 			rval = INTERCONNECT_SAS;
900 		}
901 		break;
902 	case SCSI_CAP_CDB_LEN:
903 		if (set == 0) {
904 			rval = 16;
905 		}
906 		break;
907 	case SCSI_CAP_LUN_RESET:
908 		if (set) {
909 			break;
910 		}
911 		if (xp->dtype == SATA) {
912 			rval = 0;
913 		} else {
914 			rval = 1;
915 		}
916 		break;
917 	default:
918 		rval = -1;
919 		break;
920 	}
921 	mutex_exit(&xp->statlock);
922 	pmcs_prt(ADDR2PMC(ap), PMCS_PRT_DEBUG3, NULL, NULL,
923 	    "%s: cap %s val %d set %d rval %d",
924 	    __func__, cap, val, set, rval);
925 	return (rval);
926 }
927 
928 /*
929  * Returns with statlock held if the xp is found.
930  * Fills in pmcs_cmd_t with values if pmcs_cmd_t pointer non-NULL.
931  */
932 static pmcs_xscsi_t *
pmcs_addr2xp(struct scsi_address * ap,uint64_t * lp,pmcs_cmd_t * sp)933 pmcs_addr2xp(struct scsi_address *ap, uint64_t *lp, pmcs_cmd_t *sp)
934 {
935 	pmcs_xscsi_t *xp;
936 	pmcs_lun_t *lun = (pmcs_lun_t *)
937 	    scsi_device_hba_private_get(scsi_address_device(ap));
938 
939 	if ((lun == NULL) || (lun->target == NULL)) {
940 		return (NULL);
941 	}
942 	xp = lun->target;
943 	mutex_enter(&xp->statlock);
944 
945 	if (xp->dev_gone || (xp->phy == NULL)) {
946 		/*
947 		 * This may be a retried packet, so it's possible cmd_target
948 		 * and cmd_lun may still be populated.  Clear them.
949 		 */
950 		if (sp != NULL) {
951 			sp->cmd_target = NULL;
952 			sp->cmd_lun = NULL;
953 		}
954 		mutex_exit(&xp->statlock);
955 		return (NULL);
956 	}
957 
958 	if (sp != NULL) {
959 		sp->cmd_target = xp;
960 		sp->cmd_lun = lun;
961 	}
962 	if (lp) {
963 		*lp = lun->lun_num;
964 	}
965 	return (xp);
966 }
967 
968 static int
pmcs_scsa_getcap(struct scsi_address * ap,char * cap,int whom)969 pmcs_scsa_getcap(struct scsi_address *ap, char *cap, int whom)
970 {
971 	int r;
972 	if (cap == NULL) {
973 		return (-1);
974 	}
975 	r = pmcs_cap(ap, cap, 0, whom, 0);
976 	return (r);
977 }
978 
979 static int
pmcs_scsa_setcap(struct scsi_address * ap,char * cap,int value,int whom)980 pmcs_scsa_setcap(struct scsi_address *ap, char *cap, int value, int whom)
981 {
982 	int r;
983 	if (cap == NULL) {
984 		return (-1);
985 	}
986 	r = pmcs_cap(ap, cap, value, whom, 1);
987 	return (r);
988 }
989 
990 static int
pmcs_scsa_setup_pkt(struct scsi_pkt * pkt,int (* callback)(caddr_t),caddr_t cbarg)991 pmcs_scsa_setup_pkt(struct scsi_pkt *pkt, int (*callback)(caddr_t),
992     caddr_t cbarg)
993 {
994 	_NOTE(ARGUNUSED(callback, cbarg));
995 	pmcs_cmd_t *sp = pkt->pkt_ha_private;
996 
997 	bzero(sp, sizeof (pmcs_cmd_t));
998 	sp->cmd_pkt = pkt;
999 	return (0);
1000 }
1001 
1002 static void
pmcs_scsa_teardown_pkt(struct scsi_pkt * pkt)1003 pmcs_scsa_teardown_pkt(struct scsi_pkt *pkt)
1004 {
1005 	pmcs_cmd_t *sp = pkt->pkt_ha_private;
1006 	sp->cmd_target = NULL;
1007 	sp->cmd_lun = NULL;
1008 }
1009 
1010 static int
pmcs_smp_start(struct smp_pkt * smp_pkt)1011 pmcs_smp_start(struct smp_pkt *smp_pkt)
1012 {
1013 	struct pmcwork *pwrk;
1014 	pmcs_iport_t *iport;
1015 	const uint_t rdoff = SAS_SMP_MAX_PAYLOAD;
1016 	uint32_t msg[PMCS_MSG_SIZE], *ptr, htag, status;
1017 	uint64_t wwn;
1018 	pmcs_hw_t *pwp;
1019 	pmcs_phy_t *pptr;
1020 	pmcs_xscsi_t *xp;
1021 	uint_t reqsz, rspsz, will_retry;
1022 	int result;
1023 
1024 	pwp = smp_pkt->smp_pkt_address->smp_a_hba_tran->smp_tran_hba_private;
1025 	bcopy(smp_pkt->smp_pkt_address->smp_a_wwn, &wwn, SAS_WWN_BYTE_SIZE);
1026 
1027 	pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
1028 	    "%s: starting for wwn 0x%" PRIx64, __func__, wwn);
1029 
1030 	will_retry = smp_pkt->smp_pkt_will_retry;
1031 
1032 	(void) pmcs_acquire_scratch(pwp, B_TRUE);
1033 	reqsz = smp_pkt->smp_pkt_reqsize;
1034 	if (reqsz > SAS_SMP_MAX_PAYLOAD) {
1035 		reqsz = SAS_SMP_MAX_PAYLOAD;
1036 	}
1037 	(void) memcpy(pwp->scratch, smp_pkt->smp_pkt_req, reqsz);
1038 
1039 	rspsz = smp_pkt->smp_pkt_rspsize;
1040 	if (rspsz > SAS_SMP_MAX_PAYLOAD) {
1041 		rspsz = SAS_SMP_MAX_PAYLOAD;
1042 	}
1043 
1044 	/*
1045 	 * The request size from the SMP driver always includes 4 bytes
1046 	 * for the CRC. The PMCS chip, however, doesn't want to see those
1047 	 * counts as part of the transfer size.
1048 	 */
1049 	reqsz -= 4;
1050 
1051 	pptr = pmcs_find_phy_by_wwn(pwp, wwn);
1052 	/* PHY is now locked */
1053 	if (pptr == NULL || pptr->dtype != EXPANDER) {
1054 		if (pptr) {
1055 			pmcs_unlock_phy(pptr);
1056 		}
1057 		pmcs_release_scratch(pwp);
1058 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1059 		    "%s: could not find phy", __func__);
1060 		smp_pkt->smp_pkt_reason = ENXIO;
1061 		return (DDI_FAILURE);
1062 	}
1063 
1064 	if ((pptr->iport == NULL) || !pptr->valid_device_id) {
1065 		pmcs_unlock_phy(pptr);
1066 		pmcs_release_scratch(pwp);
1067 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, pptr->target,
1068 		    "%s: Can't reach PHY %s", __func__, pptr->path);
1069 		smp_pkt->smp_pkt_reason = ENXIO;
1070 		return (DDI_FAILURE);
1071 	}
1072 
1073 	pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_WAIT, pptr);
1074 	if (pwrk == NULL) {
1075 		pmcs_unlock_phy(pptr);
1076 		pmcs_release_scratch(pwp);
1077 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1078 		    "%s: could not get work structure", __func__);
1079 		smp_pkt->smp_pkt_reason = will_retry ? EAGAIN : EBUSY;
1080 		return (DDI_FAILURE);
1081 	}
1082 
1083 	pwrk->arg = msg;
1084 	pwrk->dtype = EXPANDER;
1085 	mutex_enter(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1086 	ptr = GET_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1087 	if (ptr == NULL) {
1088 		pmcs_pwork(pwp, pwrk);
1089 		mutex_exit(&pwp->iqp_lock[PMCS_IQ_OTHER]);
1090 		pmcs_unlock_phy(pptr);
1091 		pmcs_release_scratch(pwp);
1092 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1093 		    "%s: could not get IQ entry", __func__);
1094 		smp_pkt->smp_pkt_reason = will_retry ? EAGAIN :EBUSY;
1095 		return (DDI_FAILURE);
1096 	}
1097 	msg[0] = LE_32(PMCS_HIPRI(pwp, PMCS_OQ_GENERAL, PMCIN_SMP_REQUEST));
1098 	msg[1] = LE_32(pwrk->htag);
1099 	msg[2] = LE_32(pptr->device_id);
1100 	msg[3] = LE_32(SMP_INDIRECT_RESPONSE | SMP_INDIRECT_REQUEST);
1101 	msg[8] = LE_32(DWORD0(pwp->scratch_dma));
1102 	msg[9] = LE_32(DWORD1(pwp->scratch_dma));
1103 	msg[10] = LE_32(reqsz);
1104 	msg[11] = 0;
1105 	msg[12] = LE_32(DWORD0(pwp->scratch_dma+rdoff));
1106 	msg[13] = LE_32(DWORD1(pwp->scratch_dma+rdoff));
1107 	msg[14] = LE_32(rspsz);
1108 	msg[15] = 0;
1109 
1110 	COPY_MESSAGE(ptr, msg, PMCS_MSG_SIZE);
1111 
1112 	pmcs_hold_iport(pptr->iport);
1113 	iport = pptr->iport;
1114 	pmcs_smp_acquire(iport);
1115 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1116 	htag = pwrk->htag;
1117 	INC_IQ_ENTRY(pwp, PMCS_IQ_OTHER);
1118 	pmcs_unlock_phy(pptr);
1119 	WAIT_FOR(pwrk, smp_pkt->smp_pkt_timeout * 1000, result);
1120 	pmcs_pwork(pwp, pwrk);
1121 	pmcs_smp_release(iport);
1122 	pmcs_rele_iport(iport);
1123 	pmcs_lock_phy(pptr);
1124 	if (result) {
1125 		pmcs_timed_out(pwp, htag, __func__);
1126 		if (pmcs_abort(pwp, pptr, htag, 0, 0)) {
1127 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1128 			    "%s: Unable to issue SMP ABORT for htag 0x%08x",
1129 			    __func__, htag);
1130 		} else {
1131 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1132 			    "%s: Issuing SMP ABORT for htag 0x%08x",
1133 			    __func__, htag);
1134 		}
1135 		pmcs_unlock_phy(pptr);
1136 		pmcs_release_scratch(pwp);
1137 		smp_pkt->smp_pkt_reason = ETIMEDOUT;
1138 		return (DDI_FAILURE);
1139 	}
1140 	status = LE_32(msg[2]);
1141 	if (status == PMCOUT_STATUS_OVERFLOW) {
1142 		status = PMCOUT_STATUS_OK;
1143 		smp_pkt->smp_pkt_reason = EOVERFLOW;
1144 	}
1145 	if (status != PMCOUT_STATUS_OK) {
1146 		const char *emsg = pmcs_status_str(status);
1147 		if (emsg == NULL) {
1148 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1149 			    "SMP operation failed (0x%x)", status);
1150 		} else {
1151 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, pptr->target,
1152 			    "SMP operation failed (%s)", emsg);
1153 		}
1154 
1155 		if ((status == PMCOUT_STATUS_ERROR_HW_TIMEOUT) ||
1156 		    (status == PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT)) {
1157 			smp_pkt->smp_pkt_reason =
1158 			    will_retry ? EAGAIN : ETIMEDOUT;
1159 			result = DDI_FAILURE;
1160 		} else if (status ==
1161 		    PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS) {
1162 			xp = pptr->target;
1163 			if (xp == NULL) {
1164 				smp_pkt->smp_pkt_reason = EIO;
1165 				result = DDI_FAILURE;
1166 				goto out;
1167 			}
1168 			if (xp->dev_state !=
1169 			    PMCS_DEVICE_STATE_NON_OPERATIONAL) {
1170 				xp->dev_state =
1171 				    PMCS_DEVICE_STATE_NON_OPERATIONAL;
1172 				pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, xp->phy,
1173 				    xp, "%s: Got _IT_NEXUS_LOSS SMP status. "
1174 				    "Tgt(0x%p) dev_state set to "
1175 				    "_NON_OPERATIONAL", __func__,
1176 				    (void *)xp);
1177 			}
1178 			/* ABORT any pending commands related to this device */
1179 			if (pmcs_abort(pwp, pptr, pptr->device_id, 1, 1) != 0) {
1180 				pptr->abort_pending = 1;
1181 				smp_pkt->smp_pkt_reason = EIO;
1182 				result = DDI_FAILURE;
1183 			}
1184 		} else {
1185 			smp_pkt->smp_pkt_reason = will_retry ? EAGAIN : EIO;
1186 			result = DDI_FAILURE;
1187 		}
1188 	} else {
1189 		(void) memcpy(smp_pkt->smp_pkt_rsp,
1190 		    &((uint8_t *)pwp->scratch)[rdoff], rspsz);
1191 		if (smp_pkt->smp_pkt_reason == EOVERFLOW) {
1192 			result = DDI_FAILURE;
1193 		} else {
1194 			result = DDI_SUCCESS;
1195 		}
1196 	}
1197 out:
1198 	pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, pptr->target,
1199 	    "%s: done for wwn 0x%" PRIx64, __func__, wwn);
1200 
1201 	pmcs_unlock_phy(pptr);
1202 	pmcs_release_scratch(pwp);
1203 	return (result);
1204 }
1205 
1206 static int
pmcs_smp_init(dev_info_t * self,dev_info_t * child,smp_hba_tran_t * tran,smp_device_t * smp_sd)1207 pmcs_smp_init(dev_info_t *self, dev_info_t *child,
1208     smp_hba_tran_t *tran, smp_device_t *smp_sd)
1209 {
1210 	_NOTE(ARGUNUSED(tran, smp_sd));
1211 	pmcs_iport_t *iport;
1212 	pmcs_hw_t *pwp;
1213 	pmcs_xscsi_t *tgt;
1214 	pmcs_phy_t *phy, *pphy;
1215 	uint64_t wwn;
1216 	char *addr, *tgt_port;
1217 	int ua_form = 1;
1218 
1219 	iport = ddi_get_soft_state(pmcs_iport_softstate,
1220 	    ddi_get_instance(self));
1221 	ASSERT(iport);
1222 	if (iport == NULL)
1223 		return (DDI_FAILURE);
1224 	pwp = iport->pwp;
1225 	ASSERT(pwp);
1226 	if (pwp == NULL)
1227 		return (DDI_FAILURE);
1228 
1229 	/* Get "target-port" prop from devinfo node */
1230 	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, child,
1231 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
1232 	    SCSI_ADDR_PROP_TARGET_PORT, &tgt_port) != DDI_SUCCESS) {
1233 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed to "
1234 		    "lookup prop ("SCSI_ADDR_PROP_TARGET_PORT")", __func__);
1235 		/* Dont fail _smp_init() because we couldnt get/set a prop */
1236 		return (DDI_SUCCESS);
1237 	}
1238 
1239 	/*
1240 	 * Validate that this tran_tgt_init is for an active iport.
1241 	 */
1242 	if (iport->ua_state == UA_INACTIVE) {
1243 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1244 		    "%s: Init on inactive iport for '%s'", __func__, tgt_port);
1245 		ddi_prop_free(tgt_port);
1246 		return (DDI_FAILURE);
1247 	}
1248 
1249 	mutex_enter(&pwp->lock);
1250 
1251 	/* Retrieve softstate using unit-address */
1252 	tgt = pmcs_get_target(iport, tgt_port, B_TRUE);
1253 	if (tgt == NULL) {
1254 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1255 		    "%s: tgt softstate not found", __func__);
1256 		ddi_prop_free(tgt_port);
1257 		mutex_exit(&pwp->lock);
1258 		return (DDI_FAILURE);
1259 	}
1260 
1261 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt, "%s: %s (%s)",
1262 	    __func__, ddi_get_name(child), tgt_port);
1263 
1264 	mutex_enter(&tgt->statlock);
1265 	phy = tgt->phy;
1266 	ASSERT(mutex_owned(&phy->phy_lock));
1267 
1268 	if (IS_ROOT_PHY(phy)) {
1269 		/* Expander attached to HBA - don't ref_count it */
1270 		wwn = pwp->sas_wwns[0];
1271 	} else {
1272 		pmcs_inc_phy_ref_count(phy);
1273 
1274 		/*
1275 		 * Parent (in topology) is also an expander
1276 		 * Now that we've increased the ref count on phy, it's OK
1277 		 * to drop the lock so we can acquire the parent's lock.
1278 		 */
1279 		pphy = phy->parent;
1280 		mutex_exit(&tgt->statlock);
1281 		pmcs_unlock_phy(phy);
1282 		pmcs_lock_phy(pphy);
1283 		wwn = pmcs_barray2wwn(pphy->sas_address);
1284 		pmcs_unlock_phy(pphy);
1285 		pmcs_lock_phy(phy);
1286 		mutex_enter(&tgt->statlock);
1287 	}
1288 
1289 	/*
1290 	 * If this is the 1st smp_init, add this to our list.
1291 	 */
1292 	if (tgt->target_num == PMCS_INVALID_TARGET_NUM) {
1293 		int target;
1294 		for (target = 0; target < pwp->max_dev; target++) {
1295 			if (pwp->targets[target] != NULL) {
1296 				continue;
1297 			}
1298 
1299 			pwp->targets[target] = tgt;
1300 			tgt->target_num = (uint16_t)target;
1301 			tgt->assigned = 1;
1302 			tgt->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
1303 			break;
1304 		}
1305 
1306 		if (target == pwp->max_dev) {
1307 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
1308 			    "Target list full.");
1309 			goto smp_init_fail;
1310 		}
1311 	}
1312 
1313 	if (!pmcs_assign_device(pwp, tgt)) {
1314 		pwp->targets[tgt->target_num] = NULL;
1315 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt,
1316 		    "%s: pmcs_assign_device failed for target 0x%p",
1317 		    __func__, (void *)tgt);
1318 		goto smp_init_fail;
1319 	}
1320 
1321 	/*
1322 	 * Update the attached port and target port pm properties
1323 	 */
1324 	tgt->smpd = smp_sd;
1325 
1326 	pmcs_unlock_phy(phy);
1327 	mutex_exit(&pwp->lock);
1328 
1329 	tgt->ref_count++;
1330 	tgt->dtype = phy->dtype;
1331 	mutex_exit(&tgt->statlock);
1332 
1333 	pmcs_update_phy_pm_props(phy, 0, 0, B_TRUE);
1334 
1335 	addr = scsi_wwn_to_wwnstr(wwn, ua_form, NULL);
1336 	if (smp_device_prop_update_string(smp_sd, SCSI_ADDR_PROP_ATTACHED_PORT,
1337 	    addr) != DDI_SUCCESS) {
1338 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed to set "
1339 		    "prop ("SCSI_ADDR_PROP_ATTACHED_PORT")", __func__);
1340 	}
1341 	(void) scsi_free_wwnstr(addr);
1342 	ddi_prop_free(tgt_port);
1343 	return (DDI_SUCCESS);
1344 
1345 smp_init_fail:
1346 	tgt->phy = NULL;
1347 	tgt->target_num = PMCS_INVALID_TARGET_NUM;
1348 	phy->target = NULL;
1349 	if (!IS_ROOT_PHY(phy)) {
1350 		pmcs_dec_phy_ref_count(phy);
1351 	}
1352 	mutex_exit(&tgt->statlock);
1353 	pmcs_unlock_phy(phy);
1354 	mutex_exit(&pwp->lock);
1355 	ddi_soft_state_bystr_free(iport->tgt_sstate, tgt->unit_address);
1356 	ddi_prop_free(tgt_port);
1357 	return (DDI_FAILURE);
1358 }
1359 
1360 static void
pmcs_smp_free(dev_info_t * self,dev_info_t * child,smp_hba_tran_t * tran,smp_device_t * smp)1361 pmcs_smp_free(dev_info_t *self, dev_info_t *child,
1362     smp_hba_tran_t *tran, smp_device_t *smp)
1363 {
1364 	_NOTE(ARGUNUSED(tran, smp));
1365 	pmcs_iport_t *iport;
1366 	pmcs_hw_t *pwp;
1367 	pmcs_xscsi_t *tgt;
1368 	pmcs_phy_t *phyp;
1369 	char *tgt_port;
1370 
1371 	iport = ddi_get_soft_state(pmcs_iport_softstate,
1372 	    ddi_get_instance(self));
1373 	ASSERT(iport);
1374 	if (iport == NULL)
1375 		return;
1376 
1377 	pwp = iport->pwp;
1378 	if (pwp == NULL)
1379 		return;
1380 	ASSERT(pwp);
1381 
1382 	/* Get "target-port" prop from devinfo node */
1383 	if (ddi_prop_lookup_string(DDI_DEV_T_ANY, child,
1384 	    DDI_PROP_DONTPASS | DDI_PROP_NOTPROM,
1385 	    SCSI_ADDR_PROP_TARGET_PORT, &tgt_port) != DDI_SUCCESS) {
1386 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: Failed to "
1387 		    "lookup prop ("SCSI_ADDR_PROP_TARGET_PORT")", __func__);
1388 		return;
1389 	}
1390 
1391 	/* Retrieve softstate using unit-address */
1392 	mutex_enter(&pwp->lock);
1393 	tgt = ddi_soft_state_bystr_get(iport->tgt_sstate, tgt_port);
1394 	pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt, "%s: %s (%s)", __func__,
1395 	    ddi_get_name(child), tgt_port);
1396 	ddi_prop_free(tgt_port);
1397 
1398 	if (tgt == NULL) {
1399 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1400 		    "%s: tgt softstate not found", __func__);
1401 		mutex_exit(&pwp->lock);
1402 		return;
1403 	}
1404 
1405 	phyp = tgt->phy;
1406 	if (phyp) {
1407 		mutex_enter(&phyp->phy_lock);
1408 		if (!IS_ROOT_PHY(phyp)) {
1409 			pmcs_dec_phy_ref_count(phyp);
1410 		}
1411 	}
1412 	mutex_enter(&tgt->statlock);
1413 
1414 	if (--tgt->ref_count == 0) {
1415 		/*
1416 		 * Remove this target from our list. The softstate
1417 		 * will remain, and the device will remain registered
1418 		 * with the hardware unless/until we're told that the
1419 		 * device physically went away.
1420 		 */
1421 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, tgt,
1422 		    "Removing target 0x%p (vtgt %d) from target list",
1423 		    (void *)tgt, tgt->target_num);
1424 		pwp->targets[tgt->target_num] = NULL;
1425 		tgt->target_num = PMCS_INVALID_TARGET_NUM;
1426 		/* If the PHY has a pointer to this target, clear it */
1427 		if (phyp && (phyp->target == tgt)) {
1428 			phyp->target = NULL;
1429 		}
1430 		tgt->phy = NULL;
1431 		pmcs_destroy_target(tgt);
1432 	} else {
1433 		mutex_exit(&tgt->statlock);
1434 	}
1435 
1436 	if (phyp) {
1437 		mutex_exit(&phyp->phy_lock);
1438 	}
1439 	mutex_exit(&pwp->lock);
1440 }
1441 
1442 static int
pmcs_scsi_quiesce(dev_info_t * dip)1443 pmcs_scsi_quiesce(dev_info_t *dip)
1444 {
1445 	pmcs_hw_t *pwp;
1446 	int totactive = -1;
1447 	pmcs_xscsi_t *xp;
1448 	uint16_t target;
1449 
1450 	if (ddi_get_soft_state(pmcs_iport_softstate, ddi_get_instance(dip)))
1451 		return (0);		/* iport */
1452 
1453 	pwp  = ddi_get_soft_state(pmcs_softc_state, ddi_get_instance(dip));
1454 	if (pwp == NULL) {
1455 		return (-1);
1456 	}
1457 	mutex_enter(&pwp->lock);
1458 	if (pwp->state != STATE_RUNNING) {
1459 		mutex_exit(&pwp->lock);
1460 		return (-1);
1461 	}
1462 
1463 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s called", __func__);
1464 	pwp->quiesced = pwp->blocked = 1;
1465 	while (totactive) {
1466 		totactive = 0;
1467 		for (target = 0; target < pwp->max_dev; target++) {
1468 			xp = pwp->targets[target];
1469 			if (xp == NULL) {
1470 				continue;
1471 			}
1472 			mutex_enter(&xp->statlock);
1473 			if (xp->actv_cnt) {
1474 				totactive += xp->actv_cnt;
1475 				xp->draining = 1;
1476 			}
1477 			mutex_exit(&xp->statlock);
1478 		}
1479 		if (totactive) {
1480 			cv_wait(&pwp->drain_cv, &pwp->lock);
1481 		}
1482 		/*
1483 		 * The pwp->blocked may have been reset. e.g a SCSI bus reset
1484 		 */
1485 		pwp->blocked = 1;
1486 	}
1487 
1488 	for (target = 0; target < pwp->max_dev; target++) {
1489 		xp = pwp->targets[target];
1490 		if (xp == NULL) {
1491 			continue;
1492 		}
1493 		mutex_enter(&xp->statlock);
1494 		xp->draining = 0;
1495 		mutex_exit(&xp->statlock);
1496 	}
1497 
1498 	mutex_exit(&pwp->lock);
1499 	if (totactive == 0) {
1500 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1501 		    "%s drain complete", __func__);
1502 	}
1503 	return (0);
1504 }
1505 
1506 static int
pmcs_scsi_unquiesce(dev_info_t * dip)1507 pmcs_scsi_unquiesce(dev_info_t *dip)
1508 {
1509 	pmcs_hw_t *pwp;
1510 
1511 	if (ddi_get_soft_state(pmcs_iport_softstate, ddi_get_instance(dip)))
1512 		return (0);		/* iport */
1513 
1514 	pwp  = ddi_get_soft_state(pmcs_softc_state, ddi_get_instance(dip));
1515 	if (pwp == NULL) {
1516 		return (-1);
1517 	}
1518 	mutex_enter(&pwp->lock);
1519 	if (pwp->state != STATE_RUNNING) {
1520 		mutex_exit(&pwp->lock);
1521 		return (-1);
1522 	}
1523 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s called", __func__);
1524 	pwp->blocked = pwp->quiesced = 0;
1525 	mutex_exit(&pwp->lock);
1526 
1527 	/*
1528 	 * Run all pending commands.
1529 	 */
1530 	pmcs_scsa_wq_run(pwp);
1531 
1532 	/*
1533 	 * Complete all completed commands.
1534 	 * This also unlocks us.
1535 	 */
1536 	PMCS_CQ_RUN(pwp);
1537 	return (0);
1538 }
1539 
1540 /*
1541  * Start commands for a particular device
1542  * If the actual start of a command fails, return B_FALSE.  Any other result
1543  * is a B_TRUE return.
1544  */
1545 boolean_t
pmcs_scsa_wq_run_one(pmcs_hw_t * pwp,pmcs_xscsi_t * xp)1546 pmcs_scsa_wq_run_one(pmcs_hw_t *pwp, pmcs_xscsi_t *xp)
1547 {
1548 	pmcs_cmd_t *sp;
1549 	pmcs_phy_t *phyp;
1550 	pmcwork_t *pwrk;
1551 	boolean_t run_one, blocked;
1552 	int rval;
1553 
1554 	/*
1555 	 * First, check to see if we're blocked or resource limited
1556 	 */
1557 	mutex_enter(&pwp->lock);
1558 	blocked = pwp->blocked;
1559 	/*
1560 	 * If resource_limited is set, we're resource constrained and
1561 	 * we will run only one work request for this target.
1562 	 */
1563 	run_one = pwp->resource_limited;
1564 	mutex_exit(&pwp->lock);
1565 
1566 	if (blocked) {
1567 		/* Queues will get restarted when we get unblocked */
1568 		return (B_TRUE);
1569 	}
1570 
1571 	/*
1572 	 * Might as well verify the queue is not empty before moving on
1573 	 */
1574 	mutex_enter(&xp->wqlock);
1575 	if (STAILQ_EMPTY(&xp->wq)) {
1576 		mutex_exit(&xp->wqlock);
1577 		return (B_TRUE);
1578 	}
1579 	mutex_exit(&xp->wqlock);
1580 
1581 	/*
1582 	 * If we're draining or resetting, just reschedule work queue and bail.
1583 	 */
1584 	mutex_enter(&xp->statlock);
1585 	if (xp->draining || xp->resetting || xp->special_running ||
1586 	    xp->special_needed) {
1587 		mutex_exit(&xp->statlock);
1588 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1589 		return (B_TRUE);
1590 	}
1591 
1592 	/*
1593 	 * Next, check to see if the target is gone.
1594 	 */
1595 	if (xp->dev_gone) {
1596 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1597 		    "%s: Flushing wait queue for dead tgt 0x%p", __func__,
1598 		    (void *)xp);
1599 		pmcs_flush_target_queues(pwp, xp, PMCS_TGT_WAIT_QUEUE);
1600 		mutex_exit(&xp->statlock);
1601 		return (B_TRUE);
1602 	}
1603 
1604 	/*
1605 	 * Increment the PHY's ref_count now so we know it won't go away
1606 	 * after we drop the target lock.  Drop it before returning.  If the
1607 	 * PHY dies, the commands we attempt to send will fail, but at least
1608 	 * we know we have a real PHY pointer.
1609 	 */
1610 	phyp = xp->phy;
1611 	pmcs_inc_phy_ref_count(phyp);
1612 	mutex_exit(&xp->statlock);
1613 
1614 	mutex_enter(&xp->wqlock);
1615 	while ((sp = STAILQ_FIRST(&xp->wq)) != NULL) {
1616 		pwrk = pmcs_gwork(pwp, PMCS_TAG_TYPE_CBACK, phyp);
1617 		if (pwrk == NULL) {
1618 			mutex_exit(&xp->wqlock);
1619 			mutex_enter(&pwp->lock);
1620 			if (pwp->resource_limited == 0) {
1621 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1622 				    "%s: out of work structures", __func__);
1623 			}
1624 			pwp->resource_limited = 1;
1625 			SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1626 			mutex_exit(&pwp->lock);
1627 			return (B_FALSE);
1628 		}
1629 		STAILQ_REMOVE_HEAD(&xp->wq, cmd_next);
1630 		mutex_exit(&xp->wqlock);
1631 
1632 		pwrk->xp = xp;
1633 		pwrk->arg = sp;
1634 		pwrk->timer = 0;
1635 		sp->cmd_tag = pwrk->htag;
1636 
1637 		pwrk->dtype = xp->dtype;
1638 
1639 		if (xp->dtype == SAS) {
1640 			pwrk->ptr = (void *) pmcs_SAS_done;
1641 			if ((rval = pmcs_SAS_run(sp, pwrk)) != 0) {
1642 				if (rval != PMCS_WQ_RUN_FAIL_RES_CMP) {
1643 					sp->cmd_tag = 0;
1644 				}
1645 				pmcs_dec_phy_ref_count(phyp);
1646 				pmcs_pwork(pwp, pwrk);
1647 				SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1648 				if (rval == PMCS_WQ_RUN_FAIL_RES) {
1649 					return (B_FALSE);
1650 				} else {
1651 					return (B_TRUE);
1652 				}
1653 			}
1654 		} else {
1655 			ASSERT(xp->dtype == SATA);
1656 			pwrk->ptr = (void *) pmcs_SATA_done;
1657 			if ((rval = pmcs_SATA_run(sp, pwrk)) != 0) {
1658 				sp->cmd_tag = 0;
1659 				pmcs_dec_phy_ref_count(phyp);
1660 				pmcs_pwork(pwp, pwrk);
1661 				SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1662 				if (rval == PMCS_WQ_RUN_FAIL_RES) {
1663 					return (B_FALSE);
1664 				} else {
1665 					return (B_TRUE);
1666 				}
1667 			}
1668 		}
1669 
1670 		if (run_one) {
1671 			goto wq_out;
1672 		}
1673 		mutex_enter(&xp->wqlock);
1674 	}
1675 
1676 	mutex_exit(&xp->wqlock);
1677 
1678 wq_out:
1679 	pmcs_dec_phy_ref_count(phyp);
1680 	return (B_TRUE);
1681 }
1682 
1683 /*
1684  * Start commands for all devices.
1685  */
1686 void
pmcs_scsa_wq_run(pmcs_hw_t * pwp)1687 pmcs_scsa_wq_run(pmcs_hw_t *pwp)
1688 {
1689 	pmcs_xscsi_t *xp;
1690 	uint16_t target_start, target;
1691 	boolean_t	rval = B_TRUE;
1692 
1693 	mutex_enter(&pwp->lock);
1694 	target_start = pwp->last_wq_dev;
1695 	target = target_start;
1696 
1697 	do {
1698 		xp = pwp->targets[target];
1699 		if ((xp == NULL) || (STAILQ_EMPTY(&xp->wq))) {
1700 			if (++target == pwp->max_dev) {
1701 				target = 0;
1702 			}
1703 			continue;
1704 		}
1705 
1706 		mutex_exit(&pwp->lock);
1707 		rval = pmcs_scsa_wq_run_one(pwp, xp);
1708 		mutex_enter(&pwp->lock);
1709 
1710 		if (rval == B_FALSE) {
1711 			break;
1712 		}
1713 
1714 		if (++target == pwp->max_dev) {
1715 			target = 0;
1716 		}
1717 	} while (target != target_start);
1718 
1719 	if (rval) {
1720 		/*
1721 		 * If we were resource limited, but apparently are not now,
1722 		 * reschedule the work queues anyway.
1723 		 */
1724 		if (pwp->resource_limited) {
1725 			SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1726 		}
1727 		pwp->resource_limited = 0; /* Not resource-constrained */
1728 	} else {
1729 		/*
1730 		 * Give everybody a chance, and reschedule to run the queues
1731 		 * again as long as we're limited.
1732 		 */
1733 		pwp->resource_limited = 1;
1734 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
1735 	}
1736 
1737 	pwp->last_wq_dev = target;
1738 	mutex_exit(&pwp->lock);
1739 }
1740 
1741 /*
1742  * Pull the completion queue, drop the lock and complete all elements.
1743  */
1744 
1745 void
pmcs_scsa_cq_run(void * arg)1746 pmcs_scsa_cq_run(void *arg)
1747 {
1748 	pmcs_cq_thr_info_t *cqti = (pmcs_cq_thr_info_t *)arg;
1749 	pmcs_hw_t *pwp = cqti->cq_pwp;
1750 	pmcs_cmd_t *sp, *nxt;
1751 	struct scsi_pkt *pkt;
1752 	pmcs_xscsi_t *tgt;
1753 	pmcs_iocomp_cb_t *ioccb, *ioccb_next;
1754 	pmcs_cb_t callback;
1755 
1756 	DTRACE_PROBE1(pmcs__scsa__cq__run__start, pmcs_cq_thr_info_t *, cqti);
1757 
1758 	mutex_enter(&pwp->cq_lock);
1759 
1760 	while (!pwp->cq_info.cq_stop) {
1761 		/*
1762 		 * First, check the I/O completion callback queue.
1763 		 */
1764 		ioccb = pwp->iocomp_cb_head;
1765 		pwp->iocomp_cb_head = NULL;
1766 		pwp->iocomp_cb_tail = NULL;
1767 		mutex_exit(&pwp->cq_lock);
1768 
1769 		while (ioccb) {
1770 			/*
1771 			 * Grab the lock on the work structure. The callback
1772 			 * routine is responsible for clearing it.
1773 			 */
1774 			mutex_enter(&ioccb->pwrk->lock);
1775 			ioccb_next = ioccb->next;
1776 			callback = (pmcs_cb_t)ioccb->pwrk->ptr;
1777 			(*callback)(pwp, ioccb->pwrk,
1778 			    (uint32_t *)((void *)ioccb->iomb));
1779 			kmem_cache_free(pwp->iocomp_cb_cache, ioccb);
1780 			ioccb = ioccb_next;
1781 		}
1782 
1783 		/*
1784 		 * Next, run the completion queue
1785 		 */
1786 		mutex_enter(&pwp->cq_lock);
1787 		sp = STAILQ_FIRST(&pwp->cq);
1788 		STAILQ_INIT(&pwp->cq);
1789 		mutex_exit(&pwp->cq_lock);
1790 
1791 		DTRACE_PROBE1(pmcs__scsa__cq__run__start__loop,
1792 		    pmcs_cq_thr_info_t *, cqti);
1793 
1794 		if (sp && pmcs_check_acc_dma_handle(pwp)) {
1795 			ddi_fm_service_impact(pwp->dip, DDI_SERVICE_UNAFFECTED);
1796 		}
1797 
1798 		while (sp) {
1799 			nxt = STAILQ_NEXT(sp, cmd_next);
1800 			pkt = CMD2PKT(sp);
1801 			tgt = sp->cmd_target;
1802 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, tgt,
1803 			    "%s: calling completion on %p for tgt %p", __func__,
1804 			    (void *)sp, (void *)tgt);
1805 			if (tgt) {
1806 				mutex_enter(&tgt->statlock);
1807 				ASSERT(tgt->actv_pkts != 0);
1808 				tgt->actv_pkts--;
1809 				mutex_exit(&tgt->statlock);
1810 			}
1811 			scsi_hba_pkt_comp(pkt);
1812 			sp = nxt;
1813 		}
1814 
1815 		DTRACE_PROBE1(pmcs__scsa__cq__run__end__loop,
1816 		    pmcs_cq_thr_info_t *, cqti);
1817 
1818 		/*
1819 		 * Check if there are more completions to do.  If so, and we've
1820 		 * not been told to stop, skip the wait and cycle through again.
1821 		 */
1822 		mutex_enter(&pwp->cq_lock);
1823 		if ((pwp->iocomp_cb_head == NULL) && STAILQ_EMPTY(&pwp->cq) &&
1824 		    !pwp->cq_info.cq_stop) {
1825 			mutex_exit(&pwp->cq_lock);
1826 			mutex_enter(&cqti->cq_thr_lock);
1827 			cv_wait(&cqti->cq_cv, &cqti->cq_thr_lock);
1828 			mutex_exit(&cqti->cq_thr_lock);
1829 			mutex_enter(&pwp->cq_lock);
1830 		}
1831 	}
1832 
1833 	mutex_exit(&pwp->cq_lock);
1834 	DTRACE_PROBE1(pmcs__scsa__cq__run__stop, pmcs_cq_thr_info_t *, cqti);
1835 	thread_exit();
1836 }
1837 
1838 /*
1839  * Run a SAS command.  Called with pwrk->lock held, returns unlocked.
1840  */
1841 static int
pmcs_SAS_run(pmcs_cmd_t * sp,pmcwork_t * pwrk)1842 pmcs_SAS_run(pmcs_cmd_t *sp, pmcwork_t *pwrk)
1843 {
1844 	pmcs_hw_t *pwp = CMD2PMC(sp);
1845 	struct scsi_pkt *pkt = CMD2PKT(sp);
1846 	pmcs_xscsi_t *xp = pwrk->xp;
1847 	uint32_t iq, lhtag, *ptr;
1848 	sas_ssp_cmd_iu_t sc;
1849 	int sp_pkt_time = 0;
1850 
1851 	ASSERT(xp != NULL);
1852 	mutex_enter(&xp->statlock);
1853 	if (!xp->assigned) {
1854 		mutex_exit(&xp->statlock);
1855 		return (PMCS_WQ_RUN_FAIL_OTHER);
1856 	}
1857 	if ((xp->actv_cnt >= xp->qdepth) || xp->recover_wait) {
1858 		mutex_exit(&xp->statlock);
1859 		mutex_enter(&xp->wqlock);
1860 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
1861 		mutex_exit(&xp->wqlock);
1862 		return (PMCS_WQ_RUN_FAIL_OTHER);
1863 	}
1864 	GET_IO_IQ_ENTRY(pwp, ptr, pwrk->phy->device_id, iq);
1865 	if (ptr == NULL) {
1866 		mutex_exit(&xp->statlock);
1867 		/*
1868 		 * This is a temporary failure not likely to unblocked by
1869 		 * commands completing as the test for scheduling the
1870 		 * restart of work is a per-device test.
1871 		 */
1872 		mutex_enter(&xp->wqlock);
1873 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
1874 		mutex_exit(&xp->wqlock);
1875 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1876 		    "%s: Failed to get IO IQ entry for tgt %d",
1877 		    __func__, xp->target_num);
1878 		return (PMCS_WQ_RUN_FAIL_RES);
1879 
1880 	}
1881 
1882 	ptr[0] =
1883 	    LE_32(PMCS_IOMB_IN_SAS(PMCS_OQ_IODONE, PMCIN_SSP_INI_IO_START));
1884 	ptr[1] = LE_32(pwrk->htag);
1885 	ptr[2] = LE_32(pwrk->phy->device_id);
1886 	ptr[3] = LE_32(pkt->pkt_dma_len);
1887 	if (ptr[3]) {
1888 		ASSERT(pkt->pkt_numcookies);
1889 		if (pkt->pkt_dma_flags & DDI_DMA_READ) {
1890 			ptr[4] = LE_32(PMCIN_DATADIR_2_INI);
1891 		} else {
1892 			ptr[4] = LE_32(PMCIN_DATADIR_2_DEV);
1893 		}
1894 		if (pmcs_dma_load(pwp, sp, ptr)) {
1895 			mutex_exit(&pwp->iqp_lock[iq]);
1896 			mutex_exit(&xp->statlock);
1897 			mutex_enter(&xp->wqlock);
1898 			if (STAILQ_EMPTY(&xp->wq)) {
1899 				STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
1900 				mutex_exit(&xp->wqlock);
1901 				return (PMCS_WQ_RUN_FAIL_RES);
1902 			} else {
1903 				mutex_exit(&xp->wqlock);
1904 				CMD2PKT(sp)->pkt_scbp[0] = STATUS_QFULL;
1905 				CMD2PKT(sp)->pkt_reason = CMD_CMPLT;
1906 				CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS |
1907 				    STATE_GOT_TARGET | STATE_SENT_CMD |
1908 				    STATE_GOT_STATUS;
1909 				sp->cmd_tag = 0;
1910 				mutex_enter(&pwp->cq_lock);
1911 				STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
1912 				PMCS_CQ_RUN_LOCKED(pwp);
1913 				mutex_exit(&pwp->cq_lock);
1914 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
1915 				    "%s: Failed to dma_load for tgt %d (QF)",
1916 				    __func__, xp->target_num);
1917 				return (PMCS_WQ_RUN_FAIL_RES_CMP);
1918 			}
1919 		}
1920 	} else {
1921 		ptr[4] = LE_32(PMCIN_DATADIR_NONE);
1922 		CLEAN_MESSAGE(ptr, 12);
1923 	}
1924 	xp->actv_cnt++;
1925 	if (xp->actv_cnt > xp->maxdepth) {
1926 		xp->maxdepth = xp->actv_cnt;
1927 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pwrk->phy, xp, "%s: max depth "
1928 		    "now %u", pwrk->phy->path, xp->maxdepth);
1929 	}
1930 	mutex_exit(&xp->statlock);
1931 
1932 
1933 #ifdef	DEBUG
1934 	/*
1935 	 * Generate a PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED
1936 	 * event when this goes out on the wire.
1937 	 */
1938 	ptr[4] |= PMCIN_MESSAGE_REPORT;
1939 #endif
1940 	/*
1941 	 * Fill in the SSP IU
1942 	 */
1943 
1944 	bzero(&sc, sizeof (sas_ssp_cmd_iu_t));
1945 	bcopy((uint8_t *)&sp->cmd_lun->scsi_lun, sc.lun, sizeof (scsi_lun_t));
1946 
1947 	switch (pkt->pkt_flags & FLAG_TAGMASK) {
1948 	case FLAG_HTAG:
1949 		sc.task_attribute = SAS_CMD_TASK_ATTR_HEAD;
1950 		break;
1951 	case FLAG_OTAG:
1952 		sc.task_attribute = SAS_CMD_TASK_ATTR_ORDERED;
1953 		break;
1954 	case FLAG_STAG:
1955 	default:
1956 		sc.task_attribute = SAS_CMD_TASK_ATTR_SIMPLE;
1957 		break;
1958 	}
1959 	(void) memcpy(sc.cdb, pkt->pkt_cdbp,
1960 	    min(SCSA_CDBLEN(sp), sizeof (sc.cdb)));
1961 	(void) memcpy(&ptr[5], &sc, sizeof (sas_ssp_cmd_iu_t));
1962 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
1963 	lhtag = pwrk->htag;
1964 	mutex_exit(&pwrk->lock);
1965 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
1966 	    "%s: giving pkt %p (tag %x) to the hardware", __func__,
1967 	    (void *)pkt, pwrk->htag);
1968 #ifdef DEBUG
1969 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "SAS INI Message", ptr);
1970 #endif
1971 	mutex_enter(&xp->aqlock);
1972 	STAILQ_INSERT_TAIL(&xp->aq, sp, cmd_next);
1973 	mutex_exit(&xp->aqlock);
1974 	sp_pkt_time = CMD2PKT(sp)->pkt_time;
1975 	INC_IQ_ENTRY(pwp, iq);
1976 	mutex_enter(&pwrk->lock);
1977 	if (lhtag == pwrk->htag) {
1978 		pwrk->timer = US2WT(sp_pkt_time * 1000000);
1979 		if (pwrk->timer == 0) {
1980 			pwrk->timer = US2WT(1000000);
1981 		}
1982 	}
1983 	mutex_exit(&pwrk->lock);
1984 
1985 	/*
1986 	 * If we just submitted the last command queued from device state
1987 	 * recovery, clear the wq_recovery_tail pointer.
1988 	 */
1989 	mutex_enter(&xp->wqlock);
1990 	if (xp->wq_recovery_tail == sp) {
1991 		xp->wq_recovery_tail = NULL;
1992 	}
1993 	mutex_exit(&xp->wqlock);
1994 
1995 	return (PMCS_WQ_RUN_SUCCESS);
1996 }
1997 
1998 /*
1999  * Complete a SAS command
2000  *
2001  * Called with pwrk lock held.
2002  * The free of pwrk releases the lock.
2003  */
2004 
2005 static void
pmcs_SAS_done(pmcs_hw_t * pwp,pmcwork_t * pwrk,uint32_t * msg)2006 pmcs_SAS_done(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *msg)
2007 {
2008 	pmcs_cmd_t *sp = pwrk->arg;
2009 	pmcs_phy_t *pptr = pwrk->phy;
2010 	pmcs_xscsi_t *xp = pwrk->xp;
2011 	struct scsi_pkt *pkt = CMD2PKT(sp);
2012 	int dead;
2013 	uint32_t sts;
2014 	boolean_t aborted = B_FALSE;
2015 	boolean_t do_ds_recovery = B_FALSE;
2016 
2017 	ASSERT(xp != NULL);
2018 	ASSERT(sp != NULL);
2019 	ASSERT(pptr != NULL);
2020 
2021 	DTRACE_PROBE4(pmcs__io__done, uint64_t, pkt->pkt_dma_len, int,
2022 	    (pkt->pkt_dma_flags & DDI_DMA_READ) != 0, hrtime_t, pwrk->start,
2023 	    hrtime_t, gethrtime());
2024 
2025 	dead = pwrk->dead;
2026 
2027 	if (msg) {
2028 		sts = LE_32(msg[2]);
2029 	} else {
2030 		sts = 0;
2031 	}
2032 
2033 	if (dead != 0) {
2034 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp, "%s: dead cmd tag "
2035 		    "0x%x for %s", __func__, pwrk->htag, pptr->path);
2036 		goto out;
2037 	}
2038 
2039 	if (sts == PMCOUT_STATUS_ABORTED) {
2040 		aborted = B_TRUE;
2041 	}
2042 
2043 	if (pwrk->state == PMCS_WORK_STATE_TIMED_OUT) {
2044 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2045 		    "%s: cmd 0x%p (tag 0x%x) timed out for %s",
2046 		    __func__, (void *)sp, pwrk->htag, pptr->path);
2047 		CMD2PKT(sp)->pkt_scbp[0] = STATUS_GOOD;
2048 		CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
2049 		    STATE_SENT_CMD;
2050 		CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2051 		goto out;
2052 	}
2053 
2054 	/*
2055 	 * If the status isn't okay but not underflow,
2056 	 * step to the side and parse the (possible) error.
2057 	 */
2058 #ifdef DEBUG
2059 	if (msg) {
2060 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "Outbound Message", msg);
2061 	}
2062 #endif
2063 	if (!msg) {
2064 		goto out;
2065 	}
2066 
2067 	switch (sts) {
2068 	case PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS:
2069 	case PMCOUT_STATUS_IO_DS_NON_OPERATIONAL:
2070 	case PMCOUT_STATUS_IO_DS_IN_RECOVERY:
2071 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2072 		    "%s: PHY %s requires DS recovery (status=%d)",
2073 		    __func__, pptr->path, sts);
2074 		do_ds_recovery = B_TRUE;
2075 		break;
2076 	case PMCOUT_STATUS_UNDERFLOW:
2077 		(void) pmcs_set_resid(pkt, pkt->pkt_dma_len, LE_32(msg[3]));
2078 		pmcs_prt(pwp, PMCS_PRT_DEBUG_UNDERFLOW, NULL, NULL,
2079 		    "%s: underflow %u for cdb 0x%x",
2080 		    __func__, LE_32(msg[3]), pkt->pkt_cdbp[0] & 0xff);
2081 		sts = PMCOUT_STATUS_OK;
2082 		msg[3] = 0;
2083 		break;
2084 	case PMCOUT_STATUS_OK:
2085 		pkt->pkt_resid = 0;
2086 		break;
2087 	}
2088 
2089 	if (sts != PMCOUT_STATUS_OK) {
2090 		pmcs_ioerror(pwp, SAS, pwrk, msg, sts);
2091 	} else {
2092 		if (msg[3]) {
2093 			uint8_t local[PMCS_QENTRY_SIZE << 1], *xd;
2094 			sas_ssp_rsp_iu_t *rptr = (void *)local;
2095 			const int lim =
2096 			    (PMCS_QENTRY_SIZE << 1) - SAS_RSP_HDR_SIZE;
2097 			static const uint8_t ssp_rsp_evec[] = {
2098 				0x58, 0x61, 0x56, 0x72, 0x00
2099 			};
2100 
2101 			/*
2102 			 * Transform the the first part of the response
2103 			 * to host canonical form. This gives us enough
2104 			 * information to figure out what to do with the
2105 			 * rest (which remains unchanged in the incoming
2106 			 * message which can be up to two queue entries
2107 			 * in length).
2108 			 */
2109 			pmcs_endian_transform(pwp, local, &msg[5],
2110 			    ssp_rsp_evec);
2111 			xd = (uint8_t *)(&msg[5]);
2112 			xd += SAS_RSP_HDR_SIZE;
2113 
2114 			if (rptr->datapres == SAS_RSP_DATAPRES_RESPONSE_DATA) {
2115 				if (rptr->response_data_length != 4) {
2116 					pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
2117 					    "Bad SAS RESPONSE DATA LENGTH",
2118 					    msg);
2119 					pkt->pkt_reason = CMD_TRAN_ERR;
2120 					goto out;
2121 				}
2122 				(void) memcpy(&sts, xd, sizeof (uint32_t));
2123 				sts = BE_32(sts);
2124 				/*
2125 				 * The only response code we should legally get
2126 				 * here is an INVALID FRAME response code.
2127 				 */
2128 				if (sts == SAS_RSP_INVALID_FRAME) {
2129 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2130 					    "%s: pkt %p tgt %u path %s "
2131 					    "completed: INVALID FRAME response",
2132 					    __func__, (void *)pkt,
2133 					    xp->target_num, pptr->path);
2134 				} else {
2135 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2136 					    "%s: pkt %p tgt %u path %s "
2137 					    "completed: illegal response 0x%x",
2138 					    __func__, (void *)pkt,
2139 					    xp->target_num, pptr->path, sts);
2140 				}
2141 				pkt->pkt_reason = CMD_TRAN_ERR;
2142 				goto out;
2143 			}
2144 			if (rptr->datapres == SAS_RSP_DATAPRES_SENSE_DATA) {
2145 				uint32_t slen;
2146 				slen = rptr->sense_data_length;
2147 				if (slen > lim) {
2148 					slen = lim;
2149 				}
2150 				pmcs_latch_status(pwp, sp, rptr->status, xd,
2151 				    slen, pptr->path);
2152 			} else if (rptr->datapres == SAS_RSP_DATAPRES_NO_DATA) {
2153 				pmcout_ssp_comp_t *sspcp;
2154 				sspcp = (pmcout_ssp_comp_t *)msg;
2155 				uint32_t *residp;
2156 				/*
2157 				 * This is the case for a plain SCSI status.
2158 				 * Note: If RESC_V is set and we're here, there
2159 				 * is a residual.  We need to find it and update
2160 				 * the packet accordingly.
2161 				 */
2162 				pmcs_latch_status(pwp, sp, rptr->status, NULL,
2163 				    0, pptr->path);
2164 
2165 				if (sspcp->resc_v) {
2166 					/*
2167 					 * Point residual to the SSP_RESP_IU
2168 					 */
2169 					residp = (uint32_t *)(sspcp + 1);
2170 					/*
2171 					 * param contains the number of bytes
2172 					 * between where the SSP_RESP_IU may
2173 					 * or may not be and the residual.
2174 					 * Increment residp by the appropriate
2175 					 * number of words: (param+resc_pad)/4).
2176 					 */
2177 					residp += (LE_32(sspcp->param) +
2178 					    sspcp->resc_pad) /
2179 					    sizeof (uint32_t);
2180 					pmcs_prt(pwp, PMCS_PRT_DEBUG_UNDERFLOW,
2181 					    pptr, xp, "%s: tgt 0x%p "
2182 					    "residual %d for pkt 0x%p",
2183 					    __func__, (void *) xp, *residp,
2184 					    (void *) pkt);
2185 					ASSERT(LE_32(*residp) <=
2186 					    pkt->pkt_dma_len);
2187 					(void) pmcs_set_resid(pkt,
2188 					    pkt->pkt_dma_len, LE_32(*residp));
2189 				}
2190 			} else {
2191 				pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
2192 				    "illegal SAS response", msg);
2193 				pkt->pkt_reason = CMD_TRAN_ERR;
2194 				goto out;
2195 			}
2196 		} else {
2197 			pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0,
2198 			    pptr->path);
2199 		}
2200 		if (pkt->pkt_dma_len) {
2201 			pkt->pkt_state |= STATE_XFERRED_DATA;
2202 		}
2203 	}
2204 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2205 	    "%s: pkt %p tgt %u done reason=%x state=%x resid=%ld status=%x",
2206 	    __func__, (void *)pkt, xp->target_num, pkt->pkt_reason,
2207 	    pkt->pkt_state, pkt->pkt_resid, pkt->pkt_scbp[0]);
2208 
2209 	if (pwrk->state == PMCS_WORK_STATE_ABORTED) {
2210 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2211 		    "%s: scsi_pkt 0x%p aborted for PHY %s; work = 0x%p",
2212 		    __func__, (void *)pkt, pptr->path, (void *)pwrk);
2213 		aborted = B_TRUE;
2214 	}
2215 
2216 out:
2217 	pmcs_dma_unload(pwp, sp);
2218 	mutex_enter(&xp->statlock);
2219 
2220 	/*
2221 	 * If the device no longer has a PHY pointer, clear the PHY pointer
2222 	 * from the work structure before we free it.  Otherwise, pmcs_pwork
2223 	 * may decrement the ref_count on a PHY that's been freed.
2224 	 */
2225 	if (xp->phy == NULL) {
2226 		pwrk->phy = NULL;
2227 	}
2228 
2229 	/*
2230 	 * We may arrive here due to a command timing out, which in turn
2231 	 * could be addressed in a different context.  So, free the work
2232 	 * back, but only after confirming it's not already been freed
2233 	 * elsewhere.
2234 	 */
2235 	if (pwrk->htag != PMCS_TAG_FREE) {
2236 		pmcs_pwork(pwp, pwrk);
2237 	}
2238 
2239 	/*
2240 	 * If the device is gone, we only put this command on the completion
2241 	 * queue if the work structure is not marked dead.  If it's marked
2242 	 * dead, it will already have been put there.
2243 	 */
2244 	if (xp->dev_gone) {
2245 		mutex_exit(&xp->statlock);
2246 		if (!dead) {
2247 			mutex_enter(&xp->aqlock);
2248 			STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2249 			mutex_exit(&xp->aqlock);
2250 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, xp,
2251 			    "%s: Removing cmd 0x%p (htag 0x%x) from aq",
2252 			    __func__, (void *)sp, sp->cmd_tag);
2253 			mutex_enter(&pwp->cq_lock);
2254 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2255 			PMCS_CQ_RUN_LOCKED(pwp);
2256 			mutex_exit(&pwp->cq_lock);
2257 			pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2258 			    "%s: Completing command for dead target 0x%p",
2259 			    __func__, (void *)xp);
2260 		}
2261 		return;
2262 	}
2263 
2264 	ASSERT(xp->actv_cnt > 0);
2265 	if (--(xp->actv_cnt) == 0) {
2266 		if (xp->draining) {
2267 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, xp,
2268 			    "%s: waking up drain waiters", __func__);
2269 			cv_signal(&pwp->drain_cv);
2270 		}
2271 	}
2272 	mutex_exit(&xp->statlock);
2273 
2274 	/*
2275 	 * If the status is other than OK, determine if it's something that
2276 	 * is worth re-attempting enumeration.  If so, mark the PHY.
2277 	 */
2278 	if (sts != PMCOUT_STATUS_OK) {
2279 		pmcs_status_disposition(pptr, sts);
2280 	}
2281 
2282 	if (dead == 0) {
2283 #ifdef	DEBUG
2284 		pmcs_cmd_t *wp;
2285 		mutex_enter(&xp->aqlock);
2286 		STAILQ_FOREACH(wp, &xp->aq, cmd_next) {
2287 			if (wp == sp) {
2288 				break;
2289 			}
2290 		}
2291 		ASSERT(wp != NULL);
2292 #else
2293 		mutex_enter(&xp->aqlock);
2294 #endif
2295 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, xp,
2296 		    "%s: Removing cmd 0x%p (htag 0x%x) from aq", __func__,
2297 		    (void *)sp, sp->cmd_tag);
2298 		STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2299 		if (aborted) {
2300 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2301 			    "%s: Aborted cmd for tgt 0x%p, signaling waiters",
2302 			    __func__, (void *)xp);
2303 			cv_signal(&xp->abort_cv);
2304 		}
2305 		mutex_exit(&xp->aqlock);
2306 	}
2307 
2308 	/*
2309 	 * If do_ds_recovery is set, we need to initiate device state
2310 	 * recovery.  In this case, we put this I/O back on the head of
2311 	 * the wait queue to run again after recovery is complete
2312 	 */
2313 	if (do_ds_recovery) {
2314 		mutex_enter(&xp->statlock);
2315 		pmcs_start_dev_state_recovery(xp, pptr);
2316 		mutex_exit(&xp->statlock);
2317 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, xp, "%s: Putting cmd 0x%p "
2318 		    "back on wq during recovery for tgt 0x%p", __func__,
2319 		    (void *)sp, (void *)xp);
2320 		mutex_enter(&xp->wqlock);
2321 		if (xp->wq_recovery_tail == NULL) {
2322 			STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2323 		} else {
2324 			/*
2325 			 * If there are other I/Os waiting at the head due to
2326 			 * device state recovery, add this one in the right spot
2327 			 * to maintain proper order.
2328 			 */
2329 			STAILQ_INSERT_AFTER(&xp->wq, xp->wq_recovery_tail, sp,
2330 			    cmd_next);
2331 		}
2332 		xp->wq_recovery_tail = sp;
2333 		mutex_exit(&xp->wqlock);
2334 	} else {
2335 		/*
2336 		 * If we're not initiating device state recovery and this
2337 		 * command was not "dead", put it on the completion queue
2338 		 */
2339 		if (!dead) {
2340 			mutex_enter(&pwp->cq_lock);
2341 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2342 			PMCS_CQ_RUN_LOCKED(pwp);
2343 			mutex_exit(&pwp->cq_lock);
2344 		}
2345 	}
2346 }
2347 
2348 /*
2349  * Run a SATA command (normal reads and writes),
2350  * or block and schedule a SATL interpretation
2351  * of the command.
2352  *
2353  * Called with pwrk lock held, returns unlocked.
2354  */
2355 
2356 static int
pmcs_SATA_run(pmcs_cmd_t * sp,pmcwork_t * pwrk)2357 pmcs_SATA_run(pmcs_cmd_t *sp, pmcwork_t *pwrk)
2358 {
2359 	pmcs_hw_t *pwp = CMD2PMC(sp);
2360 	struct scsi_pkt *pkt = CMD2PKT(sp);
2361 	pmcs_xscsi_t *xp;
2362 	uint8_t cdb_base, asc, tag;
2363 	uint32_t *ptr, lhtag, iq, nblk, i, mtype;
2364 	fis_t fis;
2365 	size_t amt;
2366 	uint64_t lba;
2367 	int sp_pkt_time = 0;
2368 
2369 	xp = pwrk->xp;
2370 	ASSERT(xp != NULL);
2371 
2372 	/*
2373 	 * First, see if this is just a plain read/write command.
2374 	 * If not, we have to queue it up for processing, block
2375 	 * any additional commands from coming in, and wake up
2376 	 * the thread that will process this command.
2377 	 */
2378 	cdb_base = pkt->pkt_cdbp[0] & 0x1f;
2379 	if (cdb_base != SCMD_READ && cdb_base != SCMD_WRITE) {
2380 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
2381 		    "%s: special SATA cmd %p", __func__, (void *)sp);
2382 
2383 		ASSERT(xp->phy != NULL);
2384 		pmcs_pwork(pwp, pwrk);
2385 		pmcs_lock_phy(xp->phy);
2386 		mutex_enter(&xp->statlock);
2387 		xp->special_needed = 1; /* Set the special_needed flag */
2388 		STAILQ_INSERT_TAIL(&xp->sq, sp, cmd_next);
2389 		if (pmcs_run_sata_special(pwp, xp)) {
2390 			SCHEDULE_WORK(pwp, PMCS_WORK_SATA_RUN);
2391 		}
2392 		mutex_exit(&xp->statlock);
2393 		pmcs_unlock_phy(xp->phy);
2394 
2395 		return (PMCS_WQ_RUN_SUCCESS);
2396 	}
2397 
2398 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL, "%s: regular cmd", __func__);
2399 
2400 	mutex_enter(&xp->statlock);
2401 	if (!xp->assigned) {
2402 		mutex_exit(&xp->statlock);
2403 		return (PMCS_WQ_RUN_FAIL_OTHER);
2404 	}
2405 	if (xp->special_running || xp->special_needed || xp->recover_wait) {
2406 		mutex_exit(&xp->statlock);
2407 		mutex_enter(&xp->wqlock);
2408 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2409 		mutex_exit(&xp->wqlock);
2410 		/*
2411 		 * By the time we get here the special
2412 		 * commands running or waiting to be run
2413 		 * may have come and gone, so kick our
2414 		 * worker to run the waiting queues
2415 		 * just in case.
2416 		 */
2417 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
2418 		return (PMCS_WQ_RUN_FAIL_OTHER);
2419 	}
2420 	lba = xp->capacity;
2421 	mutex_exit(&xp->statlock);
2422 
2423 	/*
2424 	 * Extract data length and lba parameters out of the command. The
2425 	 * function pmcs_SATA_rwparm returns a non-zero ASC value if the CDB
2426 	 * values are considered illegal.
2427 	 */
2428 	asc = pmcs_SATA_rwparm(pkt->pkt_cdbp, &nblk, &lba, lba);
2429 	if (asc) {
2430 		uint8_t sns[18];
2431 		bzero(sns, sizeof (sns));
2432 		sns[0] = 0xf0;
2433 		sns[2] = 0x5;
2434 		sns[12] = asc;
2435 		pmcs_latch_status(pwp, sp, STATUS_CHECK, sns, sizeof (sns),
2436 		    pwrk->phy->path);
2437 		pmcs_pwork(pwp, pwrk);
2438 		mutex_enter(&pwp->cq_lock);
2439 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2440 		PMCS_CQ_RUN_LOCKED(pwp);
2441 		mutex_exit(&pwp->cq_lock);
2442 		return (PMCS_WQ_RUN_SUCCESS);
2443 	}
2444 
2445 	/*
2446 	 * If the command decodes as not moving any data, complete it here.
2447 	 */
2448 	amt = nblk;
2449 	amt <<= 9;
2450 	amt = pmcs_set_resid(pkt, amt, nblk << 9);
2451 	if (amt == 0) {
2452 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0,
2453 		    pwrk->phy->path);
2454 		pmcs_pwork(pwp, pwrk);
2455 		mutex_enter(&pwp->cq_lock);
2456 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2457 		PMCS_CQ_RUN_LOCKED(pwp);
2458 		mutex_exit(&pwp->cq_lock);
2459 		return (PMCS_WQ_RUN_SUCCESS);
2460 	}
2461 
2462 	/*
2463 	 * Get an inbound queue entry for this I/O
2464 	 */
2465 	GET_IO_IQ_ENTRY(pwp, ptr, xp->phy->device_id, iq);
2466 	if (ptr == NULL) {
2467 		/*
2468 		 * This is a temporary failure not likely to unblocked by
2469 		 * commands completing as the test for scheduling the
2470 		 * restart of work is a per-device test.
2471 		 */
2472 		mutex_enter(&xp->wqlock);
2473 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2474 		mutex_exit(&xp->wqlock);
2475 		pmcs_dma_unload(pwp, sp);
2476 		SCHEDULE_WORK(pwp, PMCS_WORK_RUN_QUEUES);
2477 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
2478 		    "%s: Failed to get IO IQ entry for tgt %d",
2479 		    __func__, xp->target_num);
2480 		return (PMCS_WQ_RUN_FAIL_RES);
2481 	}
2482 
2483 	/*
2484 	 * Get a tag.  At this point, hold statlock until the tagmap is
2485 	 * updated (just prior to sending the cmd to the hardware).
2486 	 */
2487 	mutex_enter(&xp->statlock);
2488 	for (tag = 0; tag < xp->qdepth; tag++) {
2489 		if ((xp->tagmap & (1 << tag)) == 0) {
2490 			break;
2491 		}
2492 	}
2493 
2494 	if (tag == xp->qdepth) {
2495 		mutex_exit(&xp->statlock);
2496 		mutex_exit(&pwp->iqp_lock[iq]);
2497 		mutex_enter(&xp->wqlock);
2498 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2499 		mutex_exit(&xp->wqlock);
2500 		return (PMCS_WQ_RUN_FAIL_OTHER);
2501 	}
2502 
2503 	sp->cmd_satltag = (uint8_t)tag;
2504 
2505 	/*
2506 	 * Set up the command
2507 	 */
2508 	bzero(fis, sizeof (fis));
2509 	ptr[0] =
2510 	    LE_32(PMCS_IOMB_IN_SAS(PMCS_OQ_IODONE, PMCIN_SATA_HOST_IO_START));
2511 	ptr[1] = LE_32(pwrk->htag);
2512 	ptr[2] = LE_32(pwrk->phy->device_id);
2513 	ptr[3] = LE_32(amt);
2514 
2515 	if (xp->ncq) {
2516 		mtype = SATA_PROTOCOL_FPDMA | (tag << 16);
2517 		fis[0] = ((nblk & 0xff) << 24) | (C_BIT << 8) | FIS_REG_H2DEV;
2518 		if (cdb_base == SCMD_READ) {
2519 			fis[0] |= (READ_FPDMA_QUEUED << 16);
2520 		} else {
2521 			fis[0] |= (WRITE_FPDMA_QUEUED << 16);
2522 		}
2523 		fis[1] = (FEATURE_LBA << 24) | (lba & 0xffffff);
2524 		fis[2] = ((nblk & 0xff00) << 16) | ((lba >> 24) & 0xffffff);
2525 		fis[3] = tag << 3;
2526 	} else {
2527 		int op;
2528 		fis[0] = (C_BIT << 8) | FIS_REG_H2DEV;
2529 		if (xp->pio) {
2530 			mtype = SATA_PROTOCOL_PIO;
2531 			if (cdb_base == SCMD_READ) {
2532 				op = READ_SECTORS_EXT;
2533 			} else {
2534 				op = WRITE_SECTORS_EXT;
2535 			}
2536 		} else {
2537 			mtype = SATA_PROTOCOL_DMA;
2538 			if (cdb_base == SCMD_READ) {
2539 				op = READ_DMA_EXT;
2540 			} else {
2541 				op = WRITE_DMA_EXT;
2542 			}
2543 		}
2544 		fis[0] |= (op << 16);
2545 		fis[1] = (FEATURE_LBA << 24) | (lba & 0xffffff);
2546 		fis[2] = (lba >> 24) & 0xffffff;
2547 		fis[3] = nblk;
2548 	}
2549 
2550 	if (cdb_base == SCMD_READ) {
2551 		ptr[4] = LE_32(mtype | PMCIN_DATADIR_2_INI);
2552 	} else {
2553 		ptr[4] = LE_32(mtype | PMCIN_DATADIR_2_DEV);
2554 	}
2555 #ifdef	DEBUG
2556 	/*
2557 	 * Generate a PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED
2558 	 * event when this goes out on the wire.
2559 	 */
2560 	ptr[4] |= PMCIN_MESSAGE_REPORT;
2561 #endif
2562 	for (i = 0; i < (sizeof (fis_t))/(sizeof (uint32_t)); i++) {
2563 		ptr[i+5] = LE_32(fis[i]);
2564 	}
2565 	if (pmcs_dma_load(pwp, sp, ptr)) {
2566 		mutex_exit(&xp->statlock);
2567 		mutex_exit(&pwp->iqp_lock[iq]);
2568 		mutex_enter(&xp->wqlock);
2569 		STAILQ_INSERT_HEAD(&xp->wq, sp, cmd_next);
2570 		mutex_exit(&xp->wqlock);
2571 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, xp,
2572 		    "%s: Failed to dma_load for tgt %d",
2573 		    __func__, xp->target_num);
2574 		return (PMCS_WQ_RUN_FAIL_RES);
2575 
2576 	}
2577 
2578 	pwrk->state = PMCS_WORK_STATE_ONCHIP;
2579 	lhtag = pwrk->htag;
2580 	mutex_exit(&pwrk->lock);
2581 	xp->tagmap |= (1 << tag);
2582 	xp->actv_cnt++;
2583 	if (xp->actv_cnt > xp->maxdepth) {
2584 		xp->maxdepth = xp->actv_cnt;
2585 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pwrk->phy, xp,
2586 		    "%s: max depth now %u", pwrk->phy->path, xp->maxdepth);
2587 	}
2588 	mutex_exit(&xp->statlock);
2589 	mutex_enter(&xp->aqlock);
2590 	STAILQ_INSERT_TAIL(&xp->aq, sp, cmd_next);
2591 	mutex_exit(&xp->aqlock);
2592 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, NULL, NULL,
2593 	    "%s: giving pkt %p to hardware", __func__, (void *)pkt);
2594 #ifdef DEBUG
2595 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "SATA INI Message", ptr);
2596 #endif
2597 	sp_pkt_time = CMD2PKT(sp)->pkt_time;
2598 	INC_IQ_ENTRY(pwp, iq);
2599 	mutex_enter(&pwrk->lock);
2600 	if (lhtag == pwrk->htag) {
2601 		pwrk->timer = US2WT(sp_pkt_time * 1000000);
2602 		if (pwrk->timer == 0) {
2603 			pwrk->timer = US2WT(1000000);
2604 		}
2605 	}
2606 	mutex_exit(&pwrk->lock);
2607 
2608 	return (PMCS_WQ_RUN_SUCCESS);
2609 }
2610 
2611 /*
2612  * Complete a SATA command.  Called with pwrk lock held.
2613  */
2614 void
pmcs_SATA_done(pmcs_hw_t * pwp,pmcwork_t * pwrk,uint32_t * msg)2615 pmcs_SATA_done(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *msg)
2616 {
2617 	pmcs_cmd_t *sp = pwrk->arg;
2618 	struct scsi_pkt *pkt = CMD2PKT(sp);
2619 	pmcs_phy_t *pptr = pwrk->phy;
2620 	int dead;
2621 	uint32_t sts;
2622 	pmcs_xscsi_t *xp;
2623 	boolean_t aborted = B_FALSE;
2624 
2625 	xp = pwrk->xp;
2626 	ASSERT(xp != NULL);
2627 
2628 	DTRACE_PROBE4(pmcs__io__done, uint64_t, pkt->pkt_dma_len, int,
2629 	    (pkt->pkt_dma_flags & DDI_DMA_READ) != 0, hrtime_t, pwrk->start,
2630 	    hrtime_t, gethrtime());
2631 
2632 	dead = pwrk->dead;
2633 
2634 	if (msg) {
2635 		sts = LE_32(msg[2]);
2636 	} else {
2637 		sts = 0;
2638 	}
2639 
2640 	if (dead != 0) {
2641 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp, "%s: dead cmd tag "
2642 		    "0x%x for %s", __func__, pwrk->htag, pptr->path);
2643 		goto out;
2644 	}
2645 	if ((pwrk->state == PMCS_WORK_STATE_TIMED_OUT) &&
2646 	    (sts != PMCOUT_STATUS_ABORTED)) {
2647 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2648 		    "%s: cmd 0x%p (tag 0x%x) timed out for %s",
2649 		    __func__, (void *)sp, pwrk->htag, pptr->path);
2650 		CMD2PKT(sp)->pkt_scbp[0] = STATUS_GOOD;
2651 		/* pkt_reason already set to CMD_TIMEOUT */
2652 		ASSERT(CMD2PKT(sp)->pkt_reason == CMD_TIMEOUT);
2653 		CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
2654 		    STATE_SENT_CMD;
2655 		CMD2PKT(sp)->pkt_statistics |= STAT_TIMEOUT;
2656 		goto out;
2657 	}
2658 
2659 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp, "%s: pkt %p tgt %u done",
2660 	    __func__, (void *)pkt, xp->target_num);
2661 
2662 	/*
2663 	 * If the status isn't okay but not underflow,
2664 	 * step to the side and parse the (possible) error.
2665 	 */
2666 #ifdef DEBUG
2667 	if (msg) {
2668 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG3, "Outbound Message", msg);
2669 	}
2670 #endif
2671 	if (!msg) {
2672 		goto out;
2673 	}
2674 
2675 	/*
2676 	 * If the status isn't okay or we got a FIS response of some kind,
2677 	 * step to the side and parse the (possible) error.
2678 	 */
2679 	if ((sts != PMCOUT_STATUS_OK) || (LE_32(msg[3]) != 0)) {
2680 		if (sts == PMCOUT_STATUS_IO_DS_NON_OPERATIONAL) {
2681 			mutex_exit(&pwrk->lock);
2682 			pmcs_lock_phy(pptr);
2683 			mutex_enter(&xp->statlock);
2684 			if ((xp->resetting == 0) && (xp->reset_success != 0) &&
2685 			    (xp->reset_wait == 0)) {
2686 				mutex_exit(&xp->statlock);
2687 				if (pmcs_reset_phy(pwp, pptr,
2688 				    PMCS_PHYOP_LINK_RESET) != 0) {
2689 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2690 					    "%s: PHY (%s) Local Control/Link "
2691 					    "Reset FAILED as part of error "
2692 					    "recovery", __func__, pptr->path);
2693 				}
2694 				mutex_enter(&xp->statlock);
2695 			}
2696 			mutex_exit(&xp->statlock);
2697 			pmcs_unlock_phy(pptr);
2698 			mutex_enter(&pwrk->lock);
2699 		}
2700 		pmcs_ioerror(pwp, SATA, pwrk, msg, sts);
2701 	} else {
2702 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0,
2703 		    pwrk->phy->path);
2704 		pkt->pkt_state |= STATE_XFERRED_DATA;
2705 		pkt->pkt_resid = 0;
2706 	}
2707 
2708 	pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2709 	    "%s: pkt %p tgt %u done reason=%x state=%x resid=%ld status=%x",
2710 	    __func__, (void *)pkt, xp->target_num, pkt->pkt_reason,
2711 	    pkt->pkt_state, pkt->pkt_resid, pkt->pkt_scbp[0]);
2712 
2713 	if (pwrk->state == PMCS_WORK_STATE_ABORTED) {
2714 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2715 		    "%s: scsi_pkt 0x%p aborted for PHY %s; work = 0x%p",
2716 		    __func__, (void *)pkt, pptr->path, (void *)pwrk);
2717 		aborted = B_TRUE;
2718 	}
2719 
2720 out:
2721 	pmcs_dma_unload(pwp, sp);
2722 	mutex_enter(&xp->statlock);
2723 	xp->tagmap &= ~(1 << sp->cmd_satltag);
2724 
2725 	/*
2726 	 * If the device no longer has a PHY pointer, clear the PHY pointer
2727 	 * from the work structure before we free it.  Otherwise, pmcs_pwork
2728 	 * may decrement the ref_count on a PHY that's been freed.
2729 	 */
2730 	if (xp->phy == NULL) {
2731 		pwrk->phy = NULL;
2732 	}
2733 
2734 	/*
2735 	 * We may arrive here due to a command timing out, which in turn
2736 	 * could be addressed in a different context.  So, free the work
2737 	 * back, but only after confirming it's not already been freed
2738 	 * elsewhere.
2739 	 */
2740 	if (pwrk->htag != PMCS_TAG_FREE) {
2741 		pmcs_pwork(pwp, pwrk);
2742 	}
2743 
2744 	if (xp->dev_gone) {
2745 		mutex_exit(&xp->statlock);
2746 		if (!dead) {
2747 			mutex_enter(&xp->aqlock);
2748 			STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2749 			mutex_exit(&xp->aqlock);
2750 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, pptr, xp,
2751 			    "%s: Removing cmd 0x%p (htag 0x%x) from aq",
2752 			    __func__, (void *)sp, sp->cmd_tag);
2753 			mutex_enter(&pwp->cq_lock);
2754 			STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2755 			PMCS_CQ_RUN_LOCKED(pwp);
2756 			mutex_exit(&pwp->cq_lock);
2757 			pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, xp,
2758 			    "%s: Completing command for dead target 0x%p",
2759 			    __func__, (void *)xp);
2760 		}
2761 		return;
2762 	}
2763 
2764 	ASSERT(xp->actv_cnt > 0);
2765 	if (--(xp->actv_cnt) == 0) {
2766 		if (xp->draining) {
2767 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, pptr, xp,
2768 			    "%s: waking up drain waiters", __func__);
2769 			cv_signal(&pwp->drain_cv);
2770 		} else if (xp->special_needed) {
2771 			SCHEDULE_WORK(pwp, PMCS_WORK_SATA_RUN);
2772 		}
2773 	}
2774 	mutex_exit(&xp->statlock);
2775 
2776 	/*
2777 	 * If the status is other than OK, determine if it's something that
2778 	 * is worth re-attempting enumeration.  If so, mark the PHY.
2779 	 */
2780 	if (sts != PMCOUT_STATUS_OK) {
2781 		pmcs_status_disposition(pptr, sts);
2782 	}
2783 
2784 	if (dead == 0) {
2785 #ifdef	DEBUG
2786 		pmcs_cmd_t *wp;
2787 		mutex_enter(&xp->aqlock);
2788 		STAILQ_FOREACH(wp, &xp->aq, cmd_next) {
2789 			if (wp == sp) {
2790 				break;
2791 			}
2792 		}
2793 		ASSERT(wp != NULL);
2794 #else
2795 		mutex_enter(&xp->aqlock);
2796 #endif
2797 		STAILQ_REMOVE(&xp->aq, sp, pmcs_cmd, cmd_next);
2798 		if (aborted) {
2799 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, xp,
2800 			    "%s: Aborted cmd for tgt 0x%p, signaling waiters",
2801 			    __func__, (void *)xp);
2802 			cv_signal(&xp->abort_cv);
2803 		}
2804 		mutex_exit(&xp->aqlock);
2805 		mutex_enter(&pwp->cq_lock);
2806 		STAILQ_INSERT_TAIL(&pwp->cq, sp, cmd_next);
2807 		PMCS_CQ_RUN_LOCKED(pwp);
2808 		mutex_exit(&pwp->cq_lock);
2809 	}
2810 }
2811 
2812 static uint8_t
pmcs_SATA_rwparm(uint8_t * cdb,uint32_t * xfr,uint64_t * lba,uint64_t lbamax)2813 pmcs_SATA_rwparm(uint8_t *cdb, uint32_t *xfr, uint64_t *lba, uint64_t lbamax)
2814 {
2815 	uint8_t asc = 0;
2816 	switch (cdb[0]) {
2817 	case SCMD_READ_G5:
2818 	case SCMD_WRITE_G5:
2819 		*xfr =
2820 		    (((uint32_t)cdb[10]) <<  24) |
2821 		    (((uint32_t)cdb[11]) <<  16) |
2822 		    (((uint32_t)cdb[12]) <<   8) |
2823 		    ((uint32_t)cdb[13]);
2824 		*lba =
2825 		    (((uint64_t)cdb[2]) << 56) |
2826 		    (((uint64_t)cdb[3]) << 48) |
2827 		    (((uint64_t)cdb[4]) << 40) |
2828 		    (((uint64_t)cdb[5]) << 32) |
2829 		    (((uint64_t)cdb[6]) << 24) |
2830 		    (((uint64_t)cdb[7]) << 16) |
2831 		    (((uint64_t)cdb[8]) <<  8) |
2832 		    ((uint64_t)cdb[9]);
2833 		/* Check for illegal bits */
2834 		if (cdb[15]) {
2835 			asc = 0x24;	/* invalid field in cdb */
2836 		}
2837 		break;
2838 	case SCMD_READ_G4:
2839 	case SCMD_WRITE_G4:
2840 		*xfr =
2841 		    (((uint32_t)cdb[6]) <<  16) |
2842 		    (((uint32_t)cdb[7]) <<   8) |
2843 		    ((uint32_t)cdb[8]);
2844 		*lba =
2845 		    (((uint32_t)cdb[2]) << 24) |
2846 		    (((uint32_t)cdb[3]) << 16) |
2847 		    (((uint32_t)cdb[4]) <<  8) |
2848 		    ((uint32_t)cdb[5]);
2849 		/* Check for illegal bits */
2850 		if (cdb[11]) {
2851 			asc = 0x24;	/* invalid field in cdb */
2852 		}
2853 		break;
2854 	case SCMD_READ_G1:
2855 	case SCMD_WRITE_G1:
2856 		*xfr = (((uint32_t)cdb[7]) <<  8) | ((uint32_t)cdb[8]);
2857 		*lba =
2858 		    (((uint32_t)cdb[2]) << 24) |
2859 		    (((uint32_t)cdb[3]) << 16) |
2860 		    (((uint32_t)cdb[4]) <<  8) |
2861 		    ((uint32_t)cdb[5]);
2862 		/* Check for illegal bits */
2863 		if (cdb[9]) {
2864 			asc = 0x24;	/* invalid field in cdb */
2865 		}
2866 		break;
2867 	case SCMD_READ:
2868 	case SCMD_WRITE:
2869 		*xfr = cdb[4];
2870 		if (*xfr == 0) {
2871 			*xfr = 256;
2872 		}
2873 		*lba =
2874 		    (((uint32_t)cdb[1] & 0x1f) << 16) |
2875 		    (((uint32_t)cdb[2]) << 8) |
2876 		    ((uint32_t)cdb[3]);
2877 		/* Check for illegal bits */
2878 		if (cdb[5]) {
2879 			asc = 0x24;	/* invalid field in cdb */
2880 		}
2881 		break;
2882 	}
2883 
2884 	if (asc == 0) {
2885 		if ((*lba + *xfr) > lbamax) {
2886 			asc = 0x21;	/* logical block out of range */
2887 		}
2888 	}
2889 	return (asc);
2890 }
2891 
2892 /*
2893  * Called with pwrk lock held.
2894  */
2895 static void
pmcs_ioerror(pmcs_hw_t * pwp,pmcs_dtype_t t,pmcwork_t * pwrk,uint32_t * w,uint32_t status)2896 pmcs_ioerror(pmcs_hw_t *pwp, pmcs_dtype_t t, pmcwork_t *pwrk, uint32_t *w,
2897     uint32_t status)
2898 {
2899 	static uint8_t por[] = {
2900 	    0xf0, 0x0, 0x6, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x28
2901 	};
2902 	static uint8_t parity[] = {
2903 	    0xf0, 0x0, 0xb, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0x47, 5
2904 	};
2905 	const char *msg;
2906 	char buf[20];
2907 	pmcs_cmd_t *sp = pwrk->arg;
2908 	pmcs_phy_t *phyp = pwrk->phy;
2909 	struct scsi_pkt *pkt = CMD2PKT(sp);
2910 	uint32_t resid;
2911 
2912 	ASSERT(w != NULL);
2913 	resid = LE_32(w[3]);
2914 
2915 	msg = pmcs_status_str(status);
2916 	if (msg == NULL) {
2917 		(void) snprintf(buf, sizeof (buf), "Error 0x%x", status);
2918 		msg = buf;
2919 	}
2920 
2921 	if (status != PMCOUT_STATUS_OK) {
2922 		pmcs_prt(pwp, PMCS_PRT_DEBUG1, phyp, NULL,
2923 		    "%s: device %s tag 0x%x status %s @ %llu", __func__,
2924 		    phyp->path, pwrk->htag, msg,
2925 		    (unsigned long long)gethrtime());
2926 	}
2927 
2928 	pkt->pkt_reason = CMD_CMPLT;		/* default reason */
2929 
2930 	switch (status) {
2931 	case PMCOUT_STATUS_OK:
2932 		if (t == SATA) {
2933 			int i;
2934 			fis_t fis;
2935 			for (i = 0; i < sizeof (fis) / sizeof (fis[0]); i++) {
2936 				fis[i] = LE_32(w[4+i]);
2937 			}
2938 			if ((fis[0] & 0xff) != FIS_REG_D2H) {
2939 				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
2940 				    "unexpected fis code 0x%x", fis[0] & 0xff);
2941 			} else {
2942 				pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
2943 				    "FIS ERROR");
2944 				pmcs_fis_dump(pwp, fis);
2945 			}
2946 			pkt->pkt_reason = CMD_TRAN_ERR;
2947 			break;
2948 		}
2949 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0, phyp->path);
2950 		break;
2951 
2952 	case PMCOUT_STATUS_ABORTED:
2953 		/*
2954 		 * Command successfully aborted.
2955 		 */
2956 		if (phyp->dead) {
2957 			pkt->pkt_reason = CMD_DEV_GONE;
2958 			pkt->pkt_state = STATE_GOT_BUS;
2959 		} else if (pwrk->ssp_event != 0) {
2960 			pkt->pkt_reason = CMD_TRAN_ERR;
2961 			pkt->pkt_state = STATE_GOT_BUS;
2962 		} else if (pwrk->state == PMCS_WORK_STATE_TIMED_OUT) {
2963 			pkt->pkt_reason = CMD_TIMEOUT;
2964 			pkt->pkt_statistics |= STAT_TIMEOUT;
2965 			pkt->pkt_state = STATE_GOT_BUS | STATE_GOT_TARGET |
2966 			    STATE_SENT_CMD;
2967 		} else {
2968 			pkt->pkt_reason = CMD_ABORTED;
2969 			pkt->pkt_statistics |= STAT_ABORTED;
2970 			pkt->pkt_state = STATE_GOT_BUS | STATE_GOT_TARGET |
2971 			    STATE_SENT_CMD;
2972 		}
2973 
2974 		/*
2975 		 * PMCS_WORK_STATE_TIMED_OUT doesn't need to be preserved past
2976 		 * this point, so go ahead and mark it as aborted.
2977 		 */
2978 		pwrk->state = PMCS_WORK_STATE_ABORTED;
2979 		break;
2980 
2981 	case PMCOUT_STATUS_UNDERFLOW:
2982 		/*
2983 		 * This will only get called for SATA
2984 		 */
2985 		pkt->pkt_resid = resid;
2986 		if (pkt->pkt_dma_len < pkt->pkt_resid) {
2987 			(void) pmcs_set_resid(pkt, pkt->pkt_dma_len, resid);
2988 		}
2989 		pmcs_latch_status(pwp, sp, STATUS_GOOD, NULL, 0, phyp->path);
2990 		break;
2991 
2992 	case PMCOUT_STATUS_NO_DEVICE:
2993 	case PMCOUT_STATUS_XFER_ERROR_SATA_LINK_TIMEOUT:
2994 		pkt->pkt_reason = CMD_DEV_GONE;
2995 		break;
2996 
2997 	case PMCOUT_STATUS_OPEN_CNX_ERROR_WRONG_DESTINATION:
2998 		/*
2999 		 * Need to do rediscovery. We probably have
3000 		 * the wrong device (disk swap), so kill
3001 		 * this one.
3002 		 */
3003 	case PMCOUT_STATUS_OPEN_CNX_PROTOCOL_NOT_SUPPORTED:
3004 	case PMCOUT_STATUS_OPEN_CNX_ERROR_ZONE_VIOLATION:
3005 	case PMCOUT_STATUS_OPEN_CNX_ERROR_CONNECTION_RATE_NOT_SUPPORTED:
3006 	case PMCOUT_STATUS_OPEN_CNX_ERROR_UNKNOWN_ERROR:
3007 		/*
3008 		 * Need to do rediscovery.
3009 		 */
3010 		if (!phyp->dead) {
3011 			mutex_exit(&pwrk->lock);
3012 			pmcs_lock_phy(pwrk->phy);
3013 			pmcs_kill_changed(pwp, pwrk->phy, 0);
3014 			pmcs_unlock_phy(pwrk->phy);
3015 			mutex_enter(&pwrk->lock);
3016 			pkt->pkt_reason = CMD_INCOMPLETE;
3017 			pkt->pkt_state = STATE_GOT_BUS;
3018 		} else {
3019 			pkt->pkt_reason = CMD_DEV_GONE;
3020 		}
3021 		break;
3022 
3023 	case PMCOUT_STATUS_OPEN_CNX_ERROR_BREAK:
3024 	case PMCOUT_STATUS_OPEN_CNX_ERROR_IT_NEXUS_LOSS:
3025 	case PMCOUT_STATUS_OPENCNX_ERROR_BAD_DESTINATION:
3026 	case PMCOUT_STATUS_IO_XFER_ERROR_NAK_RECEIVED:
3027 		/* cmd is pending on the target */
3028 	case PMCOUT_STATUS_XFER_ERROR_OFFSET_MISMATCH:
3029 	case PMCOUT_STATUS_XFER_ERROR_REJECTED_NCQ_MODE:
3030 		/* transitory - commands sent while in NCQ failure mode */
3031 	case PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE:
3032 		/* NCQ failure */
3033 	case PMCOUT_STATUS_IO_PORT_IN_RESET:
3034 	case PMCOUT_STATUS_XFER_ERR_BREAK:
3035 	case PMCOUT_STATUS_XFER_ERR_PHY_NOT_READY:
3036 		pkt->pkt_reason = CMD_INCOMPLETE;
3037 		pkt->pkt_state = STATE_GOT_BUS;
3038 		break;
3039 
3040 	case PMCOUT_STATUS_IO_XFER_OPEN_RETRY_TIMEOUT:
3041 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, phyp->target,
3042 		    "STATUS_BUSY for htag 0x%08x", sp->cmd_tag);
3043 		pmcs_latch_status(pwp, sp, STATUS_BUSY, NULL, 0, phyp->path);
3044 		break;
3045 
3046 	case PMCOUT_STATUS_OPEN_CNX_ERROR_STP_RESOURCES_BUSY:
3047 		/* synthesize a RESERVATION CONFLICT */
3048 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, phyp->target,
3049 		    "%s: Potential affiliation active on 0x%" PRIx64, __func__,
3050 		    pmcs_barray2wwn(phyp->sas_address));
3051 		pmcs_latch_status(pwp, sp, STATUS_RESERVATION_CONFLICT, NULL,
3052 		    0, phyp->path);
3053 		break;
3054 
3055 	case PMCOUT_STATUS_XFER_ERROR_ABORTED_DUE_TO_SRST:
3056 		/* synthesize a power-on/reset */
3057 		pmcs_latch_status(pwp, sp, STATUS_CHECK, por, sizeof (por),
3058 		    phyp->path);
3059 		break;
3060 
3061 	case PMCOUT_STATUS_XFER_ERROR_UNEXPECTED_PHASE:
3062 	case PMCOUT_STATUS_XFER_ERROR_RDY_OVERRUN:
3063 	case PMCOUT_STATUS_XFER_ERROR_RDY_NOT_EXPECTED:
3064 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_ACK_NAK_TIMEOUT:
3065 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_BREAK_BEFORE_ACK_NACK:
3066 	case PMCOUT_STATUS_XFER_ERROR_CMD_ISSUE_PHY_DOWN_BEFORE_ACK_NAK:
3067 		/* synthesize a PARITY ERROR */
3068 		pmcs_latch_status(pwp, sp, STATUS_CHECK, parity,
3069 		    sizeof (parity), phyp->path);
3070 		break;
3071 
3072 	case PMCOUT_STATUS_IO_XFER_ERROR_DMA:
3073 	case PMCOUT_STATUS_IO_NOT_VALID:
3074 	case PMCOUT_STATUS_PROG_ERROR:
3075 	case PMCOUT_STATUS_XFER_ERROR_PEER_ABORTED:
3076 	case PMCOUT_STATUS_XFER_ERROR_SATA: /* non-NCQ failure */
3077 	default:
3078 		pkt->pkt_reason = CMD_TRAN_ERR;
3079 		break;
3080 	}
3081 }
3082 
3083 /*
3084  * Latch up SCSI status
3085  */
3086 
3087 void
pmcs_latch_status(pmcs_hw_t * pwp,pmcs_cmd_t * sp,uint8_t status,uint8_t * snsp,size_t snslen,char * path)3088 pmcs_latch_status(pmcs_hw_t *pwp, pmcs_cmd_t *sp, uint8_t status,
3089     uint8_t *snsp, size_t snslen, char *path)
3090 {
3091 	static const char c1[] =
3092 	    "%s: Status Byte 0x%02x for CDB0=0x%02x (%02x %02x %02x) "
3093 	    "HTAG 0x%x @ %llu";
3094 	static const char c2[] =
3095 	    "%s: Status Byte 0x%02x for CDB0=0x%02x HTAG 0x%x @ %llu";
3096 
3097 	CMD2PKT(sp)->pkt_state |= STATE_GOT_BUS | STATE_GOT_TARGET |
3098 	    STATE_SENT_CMD | STATE_GOT_STATUS;
3099 	CMD2PKT(sp)->pkt_scbp[0] = status;
3100 
3101 	if (status == STATUS_CHECK && snsp &&
3102 	    (size_t)SCSA_STSLEN(sp) >= sizeof (struct scsi_arq_status)) {
3103 		struct scsi_arq_status *aqp =
3104 		    (void *) CMD2PKT(sp)->pkt_scbp;
3105 		size_t amt = sizeof (struct scsi_extended_sense);
3106 		uint8_t key = scsi_sense_key(snsp);
3107 		uint8_t asc = scsi_sense_asc(snsp);
3108 		uint8_t ascq = scsi_sense_ascq(snsp);
3109 		if (amt > snslen) {
3110 			amt = snslen;
3111 		}
3112 		pmcs_prt(pwp, PMCS_PRT_DEBUG_SCSI_STATUS, NULL, NULL, c1, path,
3113 		    status, CMD2PKT(sp)->pkt_cdbp[0] & 0xff, key, asc, ascq,
3114 		    sp->cmd_tag, (unsigned long long)gethrtime());
3115 		CMD2PKT(sp)->pkt_state |= STATE_ARQ_DONE;
3116 		(*(uint8_t *)&aqp->sts_rqpkt_status) = STATUS_GOOD;
3117 		aqp->sts_rqpkt_statistics = 0;
3118 		aqp->sts_rqpkt_reason = CMD_CMPLT;
3119 		aqp->sts_rqpkt_state = STATE_GOT_BUS |
3120 		    STATE_GOT_TARGET | STATE_SENT_CMD |
3121 		    STATE_XFERRED_DATA | STATE_GOT_STATUS;
3122 		(void) memcpy(&aqp->sts_sensedata, snsp, amt);
3123 		if (aqp->sts_sensedata.es_class != CLASS_EXTENDED_SENSE) {
3124 			aqp->sts_rqpkt_reason = CMD_TRAN_ERR;
3125 			aqp->sts_rqpkt_state = 0;
3126 			aqp->sts_rqpkt_resid =
3127 			    sizeof (struct scsi_extended_sense);
3128 		} else {
3129 			aqp->sts_rqpkt_resid =
3130 			    sizeof (struct scsi_extended_sense) - amt;
3131 		}
3132 	} else if (status) {
3133 		pmcs_prt(pwp, PMCS_PRT_DEBUG_SCSI_STATUS, NULL, NULL, c2,
3134 		    path, status, CMD2PKT(sp)->pkt_cdbp[0] & 0xff,
3135 		    sp->cmd_tag, (unsigned long long)gethrtime());
3136 	}
3137 
3138 	CMD2PKT(sp)->pkt_reason = CMD_CMPLT;
3139 }
3140 
3141 /*
3142  * Calculate and set packet residual and return the amount
3143  * left over after applying various filters.
3144  */
3145 size_t
pmcs_set_resid(struct scsi_pkt * pkt,size_t amt,uint32_t cdbamt)3146 pmcs_set_resid(struct scsi_pkt *pkt, size_t amt, uint32_t cdbamt)
3147 {
3148 	pkt->pkt_resid = cdbamt;
3149 	if (amt > pkt->pkt_resid) {
3150 		amt = pkt->pkt_resid;
3151 	}
3152 	if (amt > pkt->pkt_dma_len) {
3153 		amt = pkt->pkt_dma_len;
3154 	}
3155 	return (amt);
3156 }
3157 
3158 /*
3159  * Return the existing target softstate (unlocked) if there is one.  If so,
3160  * the PHY is locked and that lock must be freed by the caller after the
3161  * target/PHY linkage is established.  If there isn't one, and alloc_tgt is
3162  * TRUE, then allocate one.
3163  */
3164 pmcs_xscsi_t *
pmcs_get_target(pmcs_iport_t * iport,char * tgt_port,boolean_t alloc_tgt)3165 pmcs_get_target(pmcs_iport_t *iport, char *tgt_port, boolean_t alloc_tgt)
3166 {
3167 	pmcs_hw_t *pwp = iport->pwp;
3168 	pmcs_phy_t *phyp;
3169 	pmcs_xscsi_t *tgt;
3170 	uint64_t wwn;
3171 	char unit_address[PMCS_MAX_UA_SIZE];
3172 	int ua_form = 1;
3173 
3174 	/*
3175 	 * Find the PHY for this target
3176 	 */
3177 	phyp = pmcs_find_phy_by_sas_address(pwp, iport, NULL, tgt_port);
3178 	if (phyp == NULL) {
3179 		pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
3180 		    "%s: No PHY for target @ %s", __func__, tgt_port);
3181 		return (NULL);
3182 	}
3183 
3184 	tgt = ddi_soft_state_bystr_get(iport->tgt_sstate, tgt_port);
3185 
3186 	if (tgt) {
3187 		mutex_enter(&tgt->statlock);
3188 		/*
3189 		 * There's already a target.  Check its PHY pointer to see
3190 		 * if we need to clear the old linkages
3191 		 */
3192 		if (tgt->phy && (tgt->phy != phyp)) {
3193 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
3194 			    "%s: Target PHY updated from %p to %p", __func__,
3195 			    (void *)tgt->phy, (void *)phyp);
3196 			if (!IS_ROOT_PHY(tgt->phy)) {
3197 				pmcs_dec_phy_ref_count(tgt->phy);
3198 				pmcs_inc_phy_ref_count(phyp);
3199 			}
3200 			tgt->phy->target = NULL;
3201 		}
3202 
3203 		/*
3204 		 * If this target has no PHY pointer and alloc_tgt is FALSE,
3205 		 * that implies we expect the target to already exist.  This
3206 		 * implies that there has already been a tran_tgt_init on at
3207 		 * least one LU.
3208 		 */
3209 		if ((tgt->phy == NULL) && !alloc_tgt) {
3210 			pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, tgt,
3211 			    "%s: Establish linkage from new PHY to old target @"
3212 			    "%s", __func__, tgt->unit_address);
3213 			for (int idx = 0; idx < tgt->ref_count; idx++) {
3214 				pmcs_inc_phy_ref_count(phyp);
3215 			}
3216 		}
3217 
3218 		/*
3219 		 * Set this target pointer back up, since it's been
3220 		 * through pmcs_clear_xp().
3221 		 */
3222 		tgt->dev_gone = 0;
3223 		tgt->assigned = 1;
3224 		tgt->dtype = phyp->dtype;
3225 		tgt->dev_state = PMCS_DEVICE_STATE_OPERATIONAL;
3226 		tgt->phy = phyp;
3227 		phyp->target = tgt;
3228 
3229 		mutex_exit(&tgt->statlock);
3230 		return (tgt);
3231 	}
3232 
3233 	/*
3234 	 * Make sure the PHY we found is on the correct iport
3235 	 */
3236 	if (phyp->iport != iport) {
3237 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL,
3238 		    "%s: No target at %s on this iport", __func__, tgt_port);
3239 		pmcs_unlock_phy(phyp);
3240 		return (NULL);
3241 	}
3242 
3243 	/*
3244 	 * If this was just a lookup (i.e. alloc_tgt is false), return now.
3245 	 */
3246 	if (alloc_tgt == B_FALSE) {
3247 		pmcs_unlock_phy(phyp);
3248 		return (NULL);
3249 	}
3250 
3251 	/*
3252 	 * Allocate the new softstate
3253 	 */
3254 	wwn = pmcs_barray2wwn(phyp->sas_address);
3255 	(void) scsi_wwn_to_wwnstr(wwn, ua_form, unit_address);
3256 
3257 	if (ddi_soft_state_bystr_zalloc(iport->tgt_sstate, unit_address) !=
3258 	    DDI_SUCCESS) {
3259 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
3260 		    "%s: Couldn't alloc softstate for device at %s",
3261 		    __func__, unit_address);
3262 		pmcs_unlock_phy(phyp);
3263 		return (NULL);
3264 	}
3265 
3266 	tgt = ddi_soft_state_bystr_get(iport->tgt_sstate, unit_address);
3267 	ASSERT(tgt != NULL);
3268 	STAILQ_INIT(&tgt->wq);
3269 	STAILQ_INIT(&tgt->aq);
3270 	STAILQ_INIT(&tgt->sq);
3271 	mutex_init(&tgt->statlock, NULL, MUTEX_DRIVER,
3272 	    DDI_INTR_PRI(pwp->intr_pri));
3273 	mutex_init(&tgt->wqlock, NULL, MUTEX_DRIVER,
3274 	    DDI_INTR_PRI(pwp->intr_pri));
3275 	mutex_init(&tgt->aqlock, NULL, MUTEX_DRIVER,
3276 	    DDI_INTR_PRI(pwp->intr_pri));
3277 	cv_init(&tgt->reset_cv, NULL, CV_DRIVER, NULL);
3278 	cv_init(&tgt->abort_cv, NULL, CV_DRIVER, NULL);
3279 	list_create(&tgt->lun_list, sizeof (pmcs_lun_t),
3280 	    offsetof(pmcs_lun_t, lun_list_next));
3281 	tgt->qdepth = 1;
3282 	tgt->target_num = PMCS_INVALID_TARGET_NUM;
3283 	bcopy(unit_address, tgt->unit_address, PMCS_MAX_UA_SIZE);
3284 	tgt->pwp = pwp;
3285 	tgt->ua = strdup(iport->ua);
3286 	tgt->phy = phyp;
3287 	ASSERT((phyp->target == NULL) || (phyp->target == tgt));
3288 	if (phyp->target == NULL) {
3289 		phyp->target = tgt;
3290 	}
3291 
3292 	/*
3293 	 * Don't allocate LUN softstate for SMP targets
3294 	 */
3295 	if (phyp->dtype == EXPANDER) {
3296 		return (tgt);
3297 	}
3298 
3299 	if (ddi_soft_state_bystr_init(&tgt->lun_sstate,
3300 	    sizeof (pmcs_lun_t), PMCS_LUN_SSTATE_SZ) != 0) {
3301 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, phyp, tgt,
3302 		    "%s: LUN soft_state_bystr_init failed", __func__);
3303 		ddi_soft_state_bystr_free(iport->tgt_sstate, tgt_port);
3304 		pmcs_unlock_phy(phyp);
3305 		return (NULL);
3306 	}
3307 
3308 	return (tgt);
3309 }
3310