xref: /illumos-gate/usr/src/uts/common/io/scsi/adapters/pmcs/pmcs_intr.c (revision 6a634c9dca3093f3922e4b7ab826d7bdf17bf78e)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 /*
26  * This file contains functions that are called via interrupts.
27  */
28 
29 #include <sys/scsi/adapters/pmcs/pmcs.h>
30 
31 #ifdef	DEBUG
32 #define	VALID_IOMB_CHECK(p, w, m, b, c)					\
33 	if (!(w & PMCS_IOMB_VALID)) {					\
34 		char l[64];						\
35 		(void) snprintf(l, sizeof (l),				\
36 		    "%s: INVALID IOMB (oq_ci=%u oq_pi=%u)", __func__, b, c); \
37 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG, l, m);		\
38 		STEP_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, b, 1);		\
39 		continue;						\
40 	}
41 #define	WRONG_OBID_CHECK(pwp, w, q)	\
42 	if (((w & PMCS_IOMB_OBID_MASK) >> PMCS_IOMB_OBID_SHIFT) != q) {	\
43 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,		\
44 		    "%s: COMPLETION WITH WRONG OBID (0x%x)", __func__,	\
45 		    (w & PMCS_IOMB_OBID_MASK) >> PMCS_IOMB_OBID_SHIFT);	\
46 	}
47 #else
48 #define	VALID_IOMB_CHECK(a, b, c, d, e)
49 #define	WRONG_OBID_CHECK(a, b, c)
50 #endif
51 
52 #define	OQLIM_CHECK(p, l)				\
53 	if (++l == (p)->ioq_depth) {			\
54 		pmcs_prt(p, PMCS_PRT_DEBUG, NULL, NULL,	\
55 		    "%s: possible ob queue overflow",	\
56 		    __func__);				\
57 		break;					\
58 	}
59 
60 #define	COPY_OUTBOUND(p, w, l, n, a, x, q, c)				\
61 	n = ((w & PMCS_IOMB_BC_MASK) >> PMCS_IOMB_BC_SHIFT);		\
62 	a = PMCS_QENTRY_SIZE;						\
63 	(void) memcpy(l, x, PMCS_QENTRY_SIZE);				\
64 	if (n > 1) {							\
65 		a <<= 1;						\
66 		(void) memcpy(&l[PMCS_QENTRY_SIZE],			\
67 		    GET_OQ_ENTRY(p, q, c, 1), PMCS_QENTRY_SIZE);	\
68 	}								\
69 	pmcs_prt(p, PMCS_PRT_DEBUG3, NULL, NULL,			\
70 	    "%s: ptr %p ci %d w0 %x nbuf %d",				\
71 	    __func__, (void *)x, ci, w0, n)
72 
73 #define	EVT_PRT(hwp, msg, phy)	\
74 	pmcs_prt(hwp, PMCS_PRT_DEBUG, NULL, NULL, "Phy 0x%x: %s", phy, # msg)
75 
76 
77 /*
78  * Map the link rate reported in the event to the SAS link rate value
79  */
80 static uint8_t
pmcs_link_rate(uint32_t event_link_rate)81 pmcs_link_rate(uint32_t event_link_rate)
82 {
83 	uint8_t sas_link_rate = 0;
84 
85 	switch (event_link_rate) {
86 	case 1:
87 		sas_link_rate = SAS_LINK_RATE_1_5GBIT;
88 		break;
89 	case 2:
90 		sas_link_rate = SAS_LINK_RATE_3GBIT;
91 		break;
92 	case 4:
93 		sas_link_rate = SAS_LINK_RATE_6GBIT;
94 		break;
95 	}
96 
97 	return (sas_link_rate);
98 }
99 
100 /*
101  * Called with pwrk lock
102  */
103 static void
pmcs_complete_work(pmcs_hw_t * pwp,pmcwork_t * pwrk,uint32_t * iomb,size_t amt)104 pmcs_complete_work(pmcs_hw_t *pwp, pmcwork_t *pwrk, uint32_t *iomb, size_t amt)
105 {
106 #ifdef	DEBUG
107 	pwp->ltime[pwp->lti] = gethrtime();
108 	pwp->ltags[pwp->lti++] = pwrk->htag;
109 #endif
110 	pwrk->htag |= PMCS_TAG_DONE;
111 
112 	/*
113 	 * If the command has timed out, leave it in that state.
114 	 */
115 	if (pwrk->state != PMCS_WORK_STATE_TIMED_OUT) {
116 		pwrk->state = PMCS_WORK_STATE_INTR;
117 		pwrk->onwire = 0;
118 	}
119 
120 	pmcs_complete_work_impl(pwp, pwrk, iomb, amt);
121 }
122 
123 static void
pmcs_work_not_found(pmcs_hw_t * pwp,uint32_t htag,uint32_t * iomb)124 pmcs_work_not_found(pmcs_hw_t *pwp, uint32_t htag, uint32_t *iomb)
125 {
126 #ifdef	DEBUG
127 	int i;
128 	hrtime_t now;
129 	char buf[64];
130 
131 	(void) snprintf(buf, sizeof (buf),
132 	    "unable to find work structure for tag 0x%x", htag);
133 
134 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG1, buf, iomb);
135 	if (htag == 0) {
136 		return;
137 	}
138 	now = gethrtime();
139 	for (i = 0; i < 256; i++) {
140 		mutex_enter(&pwp->dbglock);
141 		if (pwp->ltags[i] == htag) {
142 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
143 			    "same tag already completed (%llu us ago)",
144 			    (unsigned long long) (now - pwp->ltime[i]));
145 		}
146 		if (pwp->ftags[i] == htag) {
147 			pmcs_prt(pwp, PMCS_PRT_DEBUG1, NULL, NULL,
148 			    "same tag started (line %d) (%llu ns ago)",
149 			    pwp->ftag_lines[i], (unsigned long long)
150 			    (now - pwp->ftime[i]));
151 		}
152 		mutex_exit(&pwp->dbglock);
153 	}
154 #else
155 	char buf[64];
156 	(void) snprintf(buf, sizeof (buf),
157 	    "unable to find work structure for tag 0x%x", htag);
158 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG1, buf, iomb);
159 #endif
160 }
161 
162 
163 static void
pmcs_process_io_completion(pmcs_hw_t * pwp,pmcs_iocomp_cb_t * ioccb,size_t amt)164 pmcs_process_io_completion(pmcs_hw_t *pwp, pmcs_iocomp_cb_t *ioccb, size_t amt)
165 {
166 	pmcwork_t *pwrk;
167 	uint32_t tag_type;
168 	uint32_t htag = LE_32(((uint32_t *)((void *)ioccb->iomb))[1]);
169 
170 	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
171 	if (pwrk == NULL) {
172 		pmcs_work_not_found(pwp, htag, (void *)&ioccb->iomb);
173 		kmem_cache_free(pwp->iocomp_cb_cache, ioccb);
174 		return;
175 	}
176 
177 	pwrk->htag |= PMCS_TAG_DONE;
178 
179 	/*
180 	 * If the command has timed out, leave it in that state.
181 	 */
182 	if (pwrk->state != PMCS_WORK_STATE_TIMED_OUT) {
183 		pwrk->state = PMCS_WORK_STATE_INTR;
184 		pwrk->onwire = 0;
185 	}
186 
187 	/*
188 	 * Some SATA and SAS commands are run in "WAIT" mode.
189 	 * We can tell this from the tag type. In this case,
190 	 * we just do a wakeup (not a callback).
191 	 */
192 	tag_type = PMCS_TAG_TYPE(pwrk->htag);
193 	if (tag_type == PMCS_TAG_TYPE_WAIT) {
194 		ASSERT(PMCS_TAG_TYPE(pwrk->htag) == PMCS_TAG_TYPE_WAIT);
195 		if (pwrk->arg && amt) {
196 			(void) memcpy(pwrk->arg, ioccb->iomb, amt);
197 		}
198 		cv_signal(&pwrk->sleep_cv);
199 		mutex_exit(&pwrk->lock);
200 		kmem_cache_free(pwp->iocomp_cb_cache, ioccb);
201 		return;
202 	}
203 	ASSERT(tag_type == PMCS_TAG_TYPE_CBACK);
204 
205 #ifdef	DEBUG
206 	pwp->ltime[pwp->lti] = gethrtime();
207 	pwp->ltags[pwp->lti++] = pwrk->htag;
208 #endif
209 
210 	ioccb->pwrk = pwrk;
211 
212 	/*
213 	 * Only update state to IOCOMPQ if we were in the INTR state.
214 	 * Any other state (e.g. TIMED_OUT, ABORTED) needs to remain.
215 	 */
216 	if (pwrk->state == PMCS_WORK_STATE_INTR) {
217 		pwrk->state = PMCS_WORK_STATE_IOCOMPQ;
218 	}
219 
220 	mutex_enter(&pwp->cq_lock);
221 	if (pwp->iocomp_cb_tail) {
222 		pwp->iocomp_cb_tail->next = ioccb;
223 		pwp->iocomp_cb_tail = ioccb;
224 	} else {
225 		pwp->iocomp_cb_head = ioccb;
226 		pwp->iocomp_cb_tail = ioccb;
227 	}
228 	ioccb->next = NULL;
229 	mutex_exit(&pwp->cq_lock);
230 
231 	mutex_exit(&pwrk->lock);
232 	/* Completion queue will be run at end of pmcs_iodone_intr */
233 }
234 
235 
236 static void
pmcs_process_completion(pmcs_hw_t * pwp,void * iomb,size_t amt)237 pmcs_process_completion(pmcs_hw_t *pwp, void *iomb, size_t amt)
238 {
239 	pmcwork_t *pwrk;
240 	uint32_t htag = LE_32(((uint32_t *)iomb)[1]);
241 
242 	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
243 	if (pwrk == NULL) {
244 		pmcs_work_not_found(pwp, htag, iomb);
245 		return;
246 	}
247 
248 	pmcs_complete_work(pwp, pwrk, iomb, amt);
249 	/*
250 	 * The pwrk lock is now released
251 	 */
252 }
253 
254 static void
pmcs_kill_port(pmcs_hw_t * pwp,int portid)255 pmcs_kill_port(pmcs_hw_t *pwp, int portid)
256 {
257 	pmcs_phy_t *pptr = pwp->ports[portid];
258 
259 	if (pptr == NULL) {
260 		return;
261 	}
262 
263 	/*
264 	 * Clear any subsidiary phys
265 	 */
266 	mutex_enter(&pwp->lock);
267 
268 	for (pptr = pwp->root_phys; pptr; pptr = pptr->sibling) {
269 		pmcs_lock_phy(pptr);
270 		if (pptr->link_rate && pptr->portid == portid &&
271 		    pptr->subsidiary) {
272 			pmcs_clear_phy(pwp, pptr);
273 		}
274 		pmcs_unlock_phy(pptr);
275 	}
276 
277 	pptr = pwp->ports[portid];
278 	pwp->ports[portid] = NULL;
279 	mutex_exit(&pwp->lock);
280 
281 	pmcs_lock_phy(pptr);
282 	pmcs_kill_changed(pwp, pptr, 0);
283 	pmcs_unlock_phy(pptr);
284 
285 	RESTART_DISCOVERY(pwp);
286 	pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL, "PortID 0x%x Cleared", portid);
287 }
288 
289 void
pmcs_process_sas_hw_event(pmcs_hw_t * pwp,void * iomb,size_t amt)290 pmcs_process_sas_hw_event(pmcs_hw_t *pwp, void *iomb, size_t amt)
291 {
292 	uint32_t w1 = LE_32(((uint32_t *)iomb)[1]);
293 	uint32_t w3 = LE_32(((uint32_t *)iomb)[3]);
294 	char buf[32];
295 	uint8_t phynum = IOP_EVENT_PHYNUM(w1);
296 	uint8_t portid = IOP_EVENT_PORTID(w1);
297 	pmcs_iport_t *iport;
298 	pmcs_phy_t *pptr, *subphy, *tphyp;
299 	int need_ack = 0;
300 	int primary;
301 
302 	switch (IOP_EVENT_EVENT(w1)) {
303 	case IOP_EVENT_PHY_STOP_STATUS:
304 		if (IOP_EVENT_STATUS(w1)) {
305 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
306 			    "PORT %d failed to stop (0x%x)",
307 			    phynum, IOP_EVENT_STATUS(w1));
308 		} else {
309 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
310 			    "PHY 0x%x Stopped", phynum);
311 			mutex_enter(&pwp->lock);
312 			pptr = pwp->root_phys + phynum;
313 			pmcs_lock_phy(pptr);
314 			mutex_exit(&pwp->lock);
315 			if (pptr->configured) {
316 				pmcs_kill_changed(pwp, pptr, 0);
317 			} else {
318 				pmcs_set_changed(pwp, pptr, B_TRUE, 0);
319 			}
320 			pmcs_unlock_phy(pptr);
321 			RESTART_DISCOVERY(pwp);
322 		}
323 		/* Reposition htag to the 'expected' position. */
324 		((uint32_t *)iomb)[1] = ((uint32_t *)iomb)[2];
325 		pmcs_process_completion(pwp, iomb, amt);
326 		break;
327 	case IOP_EVENT_SAS_PHY_UP:
328 	{
329 		static const uint8_t sas_identify_af_endian_xfvec[] = {
330 			0x5c, 0x5a, 0x56, 0x00
331 		};
332 		pmcs_phy_t *rp;
333 		sas_identify_af_t af;
334 		uint64_t phy_id, wwn;
335 
336 		/*
337 		 * If we're not at running state, don't do anything
338 		 */
339 		mutex_enter(&pwp->lock);
340 		if (pwp->state != STATE_RUNNING) {
341 			mutex_exit(&pwp->lock);
342 			break;
343 		}
344 		pptr = pwp->root_phys + phynum;
345 		pmcs_lock_phy(pptr);
346 
347 		/*
348 		 * No need to lock the primary root PHY.  It can never go
349 		 * away, and we're only concerned with the port width and
350 		 * the portid, both of which only ever change in this function.
351 		 */
352 		rp = pwp->ports[portid];
353 
354 		mutex_exit(&pwp->lock);
355 
356 		pmcs_endian_transform(pwp, &af, &((uint32_t *)iomb)[4],
357 		    sas_identify_af_endian_xfvec);
358 
359 		/* Copy the remote address into our phy handle */
360 		(void) memcpy(pptr->sas_address, af.sas_address, 8);
361 		wwn = pmcs_barray2wwn(pptr->sas_address);
362 		phy_id = (uint64_t)af.phy_identifier;
363 
364 		/*
365 		 * Check to see if there is a PortID already active.
366 		 */
367 		if (rp) {
368 			if (rp->portid != portid) {
369 				pmcs_unlock_phy(pptr);
370 				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
371 				    "PortID 0x%x: PHY 0x%x SAS LINK UP IS FOR "
372 				    "A DIFFERENT PORTID 0x%x", rp->portid,
373 				    phynum, portid);
374 				break;
375 			}
376 
377 			/*
378 			 * If the dtype isn't NOTHING, then this is actually
379 			 * the primary PHY for this port.  It probably went
380 			 * down and came back up, so be sure not to mark it
381 			 * as a subsidiary.
382 			 */
383 			if (pptr->dtype == NOTHING) {
384 				pptr->subsidiary = 1;
385 			}
386 			pptr->link_rate =
387 			    pmcs_link_rate(IOP_EVENT_LINK_RATE(w1));
388 			pptr->portid = portid;
389 			pptr->dead = 0;
390 			pmcs_unlock_phy(pptr);
391 
392 			rp->width = IOP_EVENT_NPIP(w3);
393 
394 			/* Add this PHY to the phymap */
395 			if (sas_phymap_phy_add(pwp->hss_phymap, phynum,
396 			    pwp->sas_wwns[0], wwn) != DDI_SUCCESS) {
397 				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
398 				    "Unable to add phy %u for 0x%" PRIx64 ".0x%"
399 				    PRIx64, phynum, pwp->sas_wwns[rp->phynum],
400 				    wwn);
401 			}
402 
403 			/*
404 			 * Get our iport, if attached, and set it up.  Update
405 			 * the PHY's phymask props while we're locked.
406 			 */
407 			pmcs_lock_phy(pptr);
408 			pmcs_update_phy_pm_props(pptr, (1ULL << phynum),
409 			    (1ULL << phy_id), B_TRUE);
410 			pmcs_unlock_phy(pptr);
411 			iport = pmcs_get_iport_by_wwn(pwp, wwn);
412 			if (iport) {
413 				primary = !pptr->subsidiary;
414 
415 				mutex_enter(&iport->lock);
416 				if (primary) {
417 					iport->pptr = pptr;
418 				}
419 				if (iport->ua_state == UA_ACTIVE) {
420 					pmcs_add_phy_to_iport(iport, pptr);
421 					pptr->iport = iport;
422 				}
423 				mutex_exit(&iport->lock);
424 				pmcs_rele_iport(iport);
425 			}
426 
427 			pmcs_update_phy_pm_props(rp, (1ULL << phynum),
428 			    (1ULL << phy_id), B_TRUE);
429 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
430 			    "PortID 0x%x: PHY 0x%x SAS LINK UP WIDENS PORT "
431 			    "TO %d PHYS", portid, phynum, rp->width);
432 
433 			break;
434 		}
435 
436 		/*
437 		 * Check to see if anything is here already
438 		 */
439 		if (pptr->dtype != NOTHING && pptr->configured) {
440 			pmcs_unlock_phy(pptr);
441 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
442 			    "PortID 0x%x: SAS PHY 0x%x UP HITS EXISTING "
443 			    "CONFIGURED TREE", portid, phynum);
444 			break;
445 		}
446 
447 		if (af.address_frame_type != SAS_AF_IDENTIFY) {
448 			pmcs_unlock_phy(pptr);
449 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
450 			    "SAS link up on phy 0x%x, "
451 			    "but unexpected frame type 0x%x found", phynum,
452 			    af.address_frame_type);
453 			break;
454 		}
455 		pptr->width = IOP_EVENT_NPIP(w3);
456 		pptr->portid = portid;
457 		pptr->dead = 0;
458 		pptr->link_rate = pmcs_link_rate(IOP_EVENT_LINK_RATE(w1));
459 
460 		/*
461 		 * Check to see whether this is an expander or an endpoint
462 		 */
463 		switch (af.device_type) {
464 		case SAS_IF_DTYPE_ENDPOINT:
465 			pptr->pend_dtype = SAS;
466 			pptr->dtype = SAS;
467 			break;
468 		case SAS_IF_DTYPE_EDGE:
469 		case SAS_IF_DTYPE_FANOUT:
470 			pptr->pend_dtype = EXPANDER;
471 			pptr->dtype = EXPANDER;
472 			break;
473 		default:
474 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
475 			    "unknown device type 0x%x", af.device_type);
476 			pptr->pend_dtype = NOTHING;
477 			pptr->dtype = NOTHING;
478 			break;
479 		}
480 
481 		/*
482 		 * If this is a direct-attached SAS drive, do the spinup
483 		 * release now.
484 		 */
485 		if (pptr->dtype == SAS) {
486 			pptr->spinup_hold = 1;
487 			pmcs_spinup_release(pwp, pptr);
488 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, pptr, NULL,
489 			    "Release spinup hold on PHY 0x%x", phynum);
490 		}
491 
492 		pmcs_set_changed(pwp, pptr, B_TRUE, 0);
493 		if (pptr->width > 1) {
494 			pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
495 			    "PortID 0x%x: PHY 0x%x SAS"
496 			    " LINK UP @ %s Gb with %d phys/s", portid, phynum,
497 			    pmcs_get_rate(pptr->link_rate), pptr->width);
498 		} else {
499 			pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
500 			    "PortID 0x%x: PHY 0x%x SAS"
501 			    " LINK UP @ %s Gb/s", portid, phynum,
502 			    pmcs_get_rate(pptr->link_rate));
503 		}
504 		pmcs_unlock_phy(pptr);
505 
506 		/* Add this PHY to the phymap */
507 		if (sas_phymap_phy_add(pwp->hss_phymap, phynum,
508 		    pwp->sas_wwns[0], wwn) != DDI_SUCCESS) {
509 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
510 			    "Unable to add phy %u for 0x%" PRIx64 ".0x%"
511 			    PRIx64, phynum, pwp->sas_wwns[pptr->phynum], wwn);
512 		}
513 
514 		/* Get a pointer to our iport and set it up if attached */
515 		iport = pmcs_get_iport_by_wwn(pwp, wwn);
516 		if (iport) {
517 			primary = !pptr->subsidiary;
518 
519 			mutex_enter(&iport->lock);
520 			if (primary) {
521 				iport->pptr = pptr;
522 			}
523 			if (iport->ua_state == UA_ACTIVE) {
524 				pmcs_add_phy_to_iport(iport, pptr);
525 				pptr->iport = iport;
526 			}
527 			mutex_exit(&iport->lock);
528 			pmcs_rele_iport(iport);
529 		}
530 
531 		pmcs_lock_phy(pptr);
532 		pmcs_update_phy_pm_props(pptr, (1ULL << phynum),
533 		    (1ULL << phy_id), B_TRUE);
534 		pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT,
535 		    SAS_PHY_ONLINE, pptr);
536 		pmcs_unlock_phy(pptr);
537 
538 		mutex_enter(&pwp->lock);
539 		pwp->ports[portid] = pptr;
540 		mutex_exit(&pwp->lock);
541 		RESTART_DISCOVERY(pwp);
542 
543 		break;
544 	}
545 	case IOP_EVENT_SATA_PHY_UP: {
546 		uint64_t wwn;
547 		/*
548 		 * If we're not at running state, don't do anything
549 		 */
550 		mutex_enter(&pwp->lock);
551 		if (pwp->state != STATE_RUNNING) {
552 			mutex_exit(&pwp->lock);
553 			break;
554 		}
555 
556 		/*
557 		 * Check to see if anything is here already
558 		 */
559 		pmcs_lock_phy(pwp->root_phys + phynum);
560 		pptr = pwp->root_phys + phynum;
561 		mutex_exit(&pwp->lock);
562 
563 		if (pptr->dtype != NOTHING && pptr->configured) {
564 			pmcs_unlock_phy(pptr);
565 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
566 			    "PortID 0x%x: SATA PHY 0x%x"
567 			    " UP HITS EXISTING CONFIGURED TREE",
568 			    portid, phynum);
569 			break;
570 		}
571 
572 		pptr->width = 1;
573 		pptr->dead = 0;
574 
575 		/*
576 		 * Install the PHY number in the least significant byte
577 		 * with a NAA=3 (locally assigned address) in the most
578 		 * significant nubble.
579 		 *
580 		 * Later, we'll either use that or dig a
581 		 * WWN out of words 108..111.
582 		 */
583 		pptr->sas_address[0] = 0x30;
584 		pptr->sas_address[1] = 0;
585 		pptr->sas_address[2] = 0;
586 		pptr->sas_address[3] = 0;
587 		pptr->sas_address[4] = 0;
588 		pptr->sas_address[5] = 0;
589 		pptr->sas_address[6] = 0;
590 		pptr->sas_address[7] = phynum;
591 		pptr->portid = portid;
592 		pptr->link_rate = pmcs_link_rate(IOP_EVENT_LINK_RATE(w1));
593 		pptr->dtype = SATA;
594 		pmcs_set_changed(pwp, pptr, B_TRUE, 0);
595 		pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
596 		    "PortID 0x%x: PHY 0x%x SATA LINK UP @ %s Gb/s",
597 		    pptr->portid, phynum, pmcs_get_rate(pptr->link_rate));
598 		wwn = pmcs_barray2wwn(pptr->sas_address);
599 		pmcs_unlock_phy(pptr);
600 
601 		/* Add this PHY to the phymap */
602 		if (sas_phymap_phy_add(pwp->hss_phymap, phynum,
603 		    pwp->sas_wwns[0], wwn) != DDI_SUCCESS) {
604 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
605 			    "Unable to add phy %u for 0x%" PRIx64 ".0x%"
606 			    PRIx64, phynum, pwp->sas_wwns[pptr->phynum],
607 			    wwn);
608 		}
609 
610 		/* Get our iport, if attached, and set it up */
611 		iport = pmcs_get_iport_by_wwn(pwp, wwn);
612 		if (iport) {
613 			mutex_enter(&iport->lock);
614 			iport->pptr = pptr;
615 			if (iport->ua_state == UA_ACTIVE) {
616 				pmcs_add_phy_to_iport(iport, pptr);
617 				pptr->iport = iport;
618 				ASSERT(iport->nphy == 1);
619 				iport->nphy = 1;
620 			}
621 			mutex_exit(&iport->lock);
622 			pmcs_rele_iport(iport);
623 		}
624 
625 		pmcs_lock_phy(pptr);
626 		pmcs_update_phy_pm_props(pptr, (1ULL << phynum), 1ULL, B_TRUE);
627 		pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT,
628 		    SAS_PHY_ONLINE, pptr);
629 		pmcs_unlock_phy(pptr);
630 
631 		mutex_enter(&pwp->lock);
632 		pwp->ports[pptr->portid] = pptr;
633 		mutex_exit(&pwp->lock);
634 		RESTART_DISCOVERY(pwp);
635 		break;
636 	}
637 
638 	case IOP_EVENT_SATA_SPINUP_HOLD:
639 		tphyp = (pmcs_phy_t *)(pwp->root_phys + phynum);
640 		/*
641 		 * No need to lock the entire tree for this
642 		 */
643 		mutex_enter(&tphyp->phy_lock);
644 		tphyp->spinup_hold = 1;
645 		pmcs_spinup_release(pwp, tphyp);
646 		mutex_exit(&tphyp->phy_lock);
647 		break;
648 	case IOP_EVENT_PHY_DOWN: {
649 		uint64_t wwn;
650 
651 		/*
652 		 * If we're not at running state, don't do anything
653 		 */
654 		mutex_enter(&pwp->lock);
655 		if (pwp->state != STATE_RUNNING) {
656 			mutex_exit(&pwp->lock);
657 			break;
658 		}
659 		pptr = pwp->ports[portid];
660 
661 		subphy = pwp->root_phys + phynum;
662 		/*
663 		 * subphy is a pointer to the PHY corresponding to the incoming
664 		 * event. pptr points to the primary PHY for the corresponding
665 		 * port.  So, subphy and pptr may or may not be the same PHY,
666 		 * but that doesn't change what we need to do with each.
667 		 */
668 		ASSERT(subphy);
669 		mutex_exit(&pwp->lock);
670 
671 		if (pptr == NULL) {
672 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
673 			    "PortID 0x%x: PHY 0x%x LINK DOWN- no portid ptr",
674 			    portid, phynum);
675 			break;
676 		}
677 		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_NIL) {
678 			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
679 			    "PortID 0x%x: PHY 0x%x NOT VALID YET",
680 			    portid, phynum);
681 			need_ack = 1;
682 			break;
683 		}
684 		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_IN_RESET) {
685 			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
686 			    "PortID 0x%x: PHY 0x%x IN RESET",
687 			    portid, phynum);
688 			/* Entire port is down due to a host-initiated reset */
689 			mutex_enter(&pptr->phy_lock);
690 			/* Clear the phymask props in pptr */
691 			pmcs_update_phy_pm_props(pptr, pptr->att_port_pm_tmp,
692 			    pptr->tgt_port_pm_tmp, B_FALSE);
693 			iport = pptr->iport;
694 			mutex_exit(&pptr->phy_lock);
695 			if (iport) {
696 				mutex_enter(&iport->lock);
697 				pmcs_iport_teardown_phys(iport);
698 				mutex_exit(&iport->lock);
699 			}
700 
701 			/* Clear down all PHYs in the port */
702 			for (pptr = pwp->root_phys; pptr;
703 			    pptr = pptr->sibling) {
704 				pmcs_lock_phy(pptr);
705 				if (pptr->portid == portid) {
706 					pptr->pend_dtype = NOTHING;
707 					pptr->dtype = NOTHING;
708 					pptr->portid =
709 					    PMCS_IPORT_INVALID_PORT_ID;
710 					if (pptr->valid_device_id) {
711 						pptr->deregister_wait = 1;
712 					}
713 				}
714 				pmcs_unlock_phy(pptr);
715 				SCHEDULE_WORK(pwp, PMCS_WORK_DEREGISTER_DEV);
716 				(void) ddi_taskq_dispatch(pwp->tq, pmcs_worker,
717 				    pwp, DDI_NOSLEEP);
718 			}
719 
720 			break;
721 		}
722 
723 		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_LOSTCOMM) {
724 			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
725 			    "PortID 0x%x: PHY 0x%x TEMPORARILY DOWN",
726 			    portid, phynum);
727 			need_ack = 1;
728 			break;
729 		}
730 
731 		if (IOP_EVENT_PORT_STATE(w3) == IOP_EVENT_PS_VALID) {
732 
733 			/*
734 			 * This is not the last phy in the port, so if this
735 			 * is the primary PHY, promote another PHY to primary.
736 			 */
737 			if (pptr == subphy) {
738 				primary = !subphy->subsidiary;
739 				ASSERT(primary);
740 
741 				tphyp = pptr;
742 				pptr = pmcs_promote_next_phy(tphyp);
743 
744 				if (pptr) {
745 					/* Update primary pptr in ports */
746 					pwp->ports[portid] = pptr;
747 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr,
748 					    NULL, "PortID 0x%x: PHY 0x%x "
749 					    "promoted to primary", portid,
750 					    pptr->phynum);
751 				} else {
752 					pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr,
753 					    NULL, "PortID 0x%x: PHY 0x%x: "
754 					    "unable to promote phy", portid,
755 					    phynum);
756 				}
757 			}
758 
759 			/*
760 			 * Drop port width on the primary phy handle
761 			 * No need to lock the entire tree for this
762 			 */
763 			mutex_enter(&pptr->phy_lock);
764 			pmcs_update_phy_pm_props(pptr, subphy->att_port_pm_tmp,
765 			    subphy->tgt_port_pm_tmp, B_FALSE);
766 			pptr->width = IOP_EVENT_NPIP(w3);
767 			mutex_exit(&pptr->phy_lock);
768 
769 			/* Clear the iport reference and portid on the subphy */
770 			mutex_enter(&subphy->phy_lock);
771 			iport = subphy->iport;
772 			subphy->iport = NULL;
773 			subphy->portid = PMCS_PHY_INVALID_PORT_ID;
774 			subphy->pend_dtype = NOTHING;
775 			subphy->dtype = NOTHING;
776 			mutex_exit(&subphy->phy_lock);
777 
778 			/*
779 			 * If the iport was set on this phy, decrement its
780 			 * nphy count and remove this phy from the phys list.
781 			 */
782 			if (iport) {
783 				mutex_enter(&iport->lock);
784 				if (iport->ua_state == UA_ACTIVE) {
785 					pmcs_remove_phy_from_iport(iport,
786 					    subphy);
787 				}
788 				mutex_exit(&iport->lock);
789 			}
790 
791 			pmcs_lock_phy(subphy);
792 			wwn = pmcs_barray2wwn(pptr->sas_address);
793 			if (subphy->subsidiary)
794 				pmcs_clear_phy(pwp, subphy);
795 			pmcs_unlock_phy(subphy);
796 
797 			/* Remove this PHY from the phymap */
798 			if (sas_phymap_phy_rem(pwp->hss_phymap, phynum) !=
799 			    DDI_SUCCESS) {
800 				pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
801 				    "Unable to remove phy %u for 0x%" PRIx64
802 				    ".0x%" PRIx64, phynum,
803 				    pwp->sas_wwns[pptr->phynum], wwn);
804 			}
805 
806 			pmcs_prt(pwp, PMCS_PRT_INFO, pptr, NULL,
807 			    "PortID 0x%x: PHY 0x%x LINK DOWN NARROWS PORT "
808 			    "TO %d PHYS", portid, phynum, pptr->width);
809 			break;
810 		}
811 		if (IOP_EVENT_PORT_STATE(w3) != IOP_EVENT_PS_INVALID) {
812 			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
813 			    "PortID 0x%x: PHY 0x%x LINK DOWN NOT HANDLED "
814 			    "(state 0x%x)", portid, phynum,
815 			    IOP_EVENT_PORT_STATE(w3));
816 			need_ack = 1;
817 			break;
818 		}
819 		/* Remove this PHY from the phymap */
820 		if (sas_phymap_phy_rem(pwp->hss_phymap, phynum) !=
821 		    DDI_SUCCESS) {
822 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
823 			    "Unable to remove phy %u for 0x%" PRIx64
824 			    ".0x%" PRIx64, phynum,
825 			    pwp->sas_wwns[pptr->phynum], wwn);
826 		}
827 
828 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
829 		    "PortID 0x%x: PHY 0x%x LINK DOWN (port invalid)",
830 		    portid, phynum);
831 
832 		/*
833 		 * Last PHY on the port.
834 		 * Assumption: pptr and subphy are both "valid".  In fact,
835 		 * they should be one and the same.
836 		 *
837 		 * Drop port width on the primary phy handle
838 		 * Report the event and clear its PHY pm props while we've
839 		 * got the lock
840 		 */
841 		ASSERT(pptr == subphy);
842 		mutex_enter(&pptr->phy_lock);
843 		pptr->width = 0;
844 		pmcs_update_phy_pm_props(pptr, pptr->att_port_pm_tmp,
845 		    pptr->tgt_port_pm_tmp, B_FALSE);
846 		pmcs_smhba_log_sysevent(pwp, ESC_SAS_PHY_EVENT,
847 		    SAS_PHY_OFFLINE, pptr);
848 		mutex_exit(&pptr->phy_lock);
849 
850 		/* Clear the iport reference and portid on the subphy */
851 		pmcs_lock_phy(subphy);
852 		iport = subphy->iport;
853 		subphy->deregister_wait = 1;
854 		subphy->iport = NULL;
855 		subphy->portid = PMCS_PHY_INVALID_PORT_ID;
856 		subphy->pend_dtype = NOTHING;
857 		subphy->dtype = NOTHING;
858 		pmcs_unlock_phy(subphy);
859 		SCHEDULE_WORK(pwp, PMCS_WORK_DEREGISTER_DEV);
860 		(void) ddi_taskq_dispatch(pwp->tq, pmcs_worker,
861 		    pwp, DDI_NOSLEEP);
862 
863 		/*
864 		 * If the iport was set on this phy, decrement its
865 		 * nphy count and remove this phy from the phys list.
866 		 * Also, clear the iport's pptr as this port is now
867 		 * down.
868 		 */
869 		if (iport) {
870 			mutex_enter(&iport->lock);
871 			if (iport->ua_state == UA_ACTIVE) {
872 				pmcs_remove_phy_from_iport(iport, subphy);
873 				iport->pptr = NULL;
874 				iport->ua_state = UA_PEND_DEACTIVATE;
875 			}
876 			mutex_exit(&iport->lock);
877 		}
878 
879 		pmcs_lock_phy(subphy);
880 		if (subphy->subsidiary)
881 			pmcs_clear_phy(pwp, subphy);
882 		pmcs_unlock_phy(subphy);
883 
884 		/*
885 		 * Since we're now really dead, it's time to clean up.
886 		 */
887 		pmcs_kill_port(pwp, portid);
888 		need_ack = 1;
889 
890 		break;
891 	}
892 	case IOP_EVENT_BROADCAST_CHANGE:
893 		pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
894 		    "PortID 0x%x: PHY 0x%x Broadcast Change", portid, phynum);
895 		need_ack = 1;
896 		mutex_enter(&pwp->lock);
897 		pptr = pwp->ports[portid];
898 		if (pptr) {
899 			pmcs_lock_phy(pptr);
900 			if (pptr->phynum == phynum) {
901 				pmcs_set_changed(pwp, pptr, B_TRUE, 0);
902 			}
903 			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
904 			    SAS_PORT_BROADCAST_CHANGE, pptr);
905 			pmcs_unlock_phy(pptr);
906 		}
907 		mutex_exit(&pwp->lock);
908 		RESTART_DISCOVERY(pwp);
909 		break;
910 	case IOP_EVENT_BROADCAST_SES:
911 		EVT_PRT(pwp, IOP_EVENT_BROADCAST_SES, phynum);
912 		mutex_enter(&pwp->lock);
913 		pptr = pwp->ports[portid];
914 		mutex_exit(&pwp->lock);
915 		if (pptr) {
916 			pmcs_lock_phy(pptr);
917 			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
918 			    SAS_PORT_BROADCAST_SES, pptr);
919 			pmcs_unlock_phy(pptr);
920 		}
921 		break;
922 	case IOP_EVENT_PHY_ERR_INBOUND_CRC:
923 	{
924 		char buf[32];
925 		(void) snprintf(buf, sizeof (buf), "Inbound PHY CRC error");
926 		need_ack = 1;
927 		break;
928 	}
929 	case IOP_EVENT_HARD_RESET_RECEIVED:
930 		EVT_PRT(pwp, IOP_EVENT_HARD_RESET_RECEIVED, phynum);
931 		break;
932 	case IOP_EVENT_EVENT_ID_FRAME_TIMO:
933 		EVT_PRT(pwp, IOP_EVENT_EVENT_ID_FRAME_TIMO, phynum);
934 		break;
935 	case IOP_EVENT_BROADCAST_EXP:
936 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
937 		    "PortID 0x%x: PHY 0x%x Broadcast Exp Change",
938 		    portid, phynum);
939 		/*
940 		 * Comparing Section 6.8.1.4 of SMHBA (rev 7) spec and Section
941 		 * 7.2.3 of SAS2 (Rev 15) spec,
942 		 * _BROADCAST_EXPANDER event corresponds to _D01_4 primitive
943 		 */
944 		mutex_enter(&pwp->lock);
945 		pptr = pwp->ports[portid];
946 		mutex_exit(&pwp->lock);
947 		if (pptr) {
948 			pmcs_lock_phy(pptr);
949 			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
950 			    SAS_PORT_BROADCAST_D01_4, pptr);
951 			pmcs_unlock_phy(pptr);
952 		}
953 		break;
954 	case IOP_EVENT_PHY_START_STATUS:
955 		switch (IOP_EVENT_STATUS(w1)) {
956 		case IOP_PHY_START_OK:
957 			pmcs_prt(pwp, PMCS_PRT_DEBUG_CONFIG, NULL, NULL,
958 			    "PHY 0x%x Started", phynum);
959 			break;
960 		case IOP_PHY_START_ALREADY:
961 			pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
962 			    "PHY 0x%x Started (Already)", phynum);
963 			break;
964 		case IOP_PHY_START_INVALID:
965 			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
966 			    "PHY 0x%x failed to start (invalid phy)", phynum);
967 			break;
968 		case IOP_PHY_START_ERROR:
969 			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
970 			    "PHY 0x%x Start Error", phynum);
971 			break;
972 		default:
973 			pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
974 			    "PHY 0x%x failed to start (0x%x)", phynum,
975 			    IOP_EVENT_STATUS(w1));
976 			break;
977 		}
978 		/* Reposition htag to the 'expected' position. */
979 		((uint32_t *)iomb)[1] = ((uint32_t *)iomb)[2];
980 		pmcs_process_completion(pwp, iomb, amt);
981 		break;
982 	case IOP_EVENT_PHY_ERR_INVALID_DWORD:
983 		need_ack = 1;
984 		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_INVALID_DWORD, phynum);
985 		break;
986 	case IOP_EVENT_PHY_ERR_DISPARITY_ERROR:
987 		need_ack = 1;
988 		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_DISPARITY_ERROR, phynum);
989 		break;
990 	case IOP_EVENT_PHY_ERR_CODE_VIOLATION:
991 		need_ack = 1;
992 		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_CODE_VIOLATION, phynum);
993 		break;
994 	case IOP_EVENT_PHY_ERR_LOSS_OF_DWORD_SYN:
995 		need_ack = 1;
996 		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_LOSS_OF_DWORD_SYN, phynum);
997 		break;
998 	case IOP_EVENT_PHY_ERR_PHY_RESET_FAILD:
999 		need_ack = 1;
1000 		EVT_PRT(pwp, IOP_EVENT_PHY_ERR_PHY_RESET_FAILD, phynum);
1001 		break;
1002 	case IOP_EVENT_PORT_RECOVERY_TIMER_TMO:
1003 		EVT_PRT(pwp, IOP_EVENT_PORT_RECOVERY_TIMER_TMO, phynum);
1004 		break;
1005 	case IOP_EVENT_PORT_RECOVER:
1006 		EVT_PRT(pwp, IOP_EVENT_PORT_RECOVER, phynum);
1007 		break;
1008 	case IOP_EVENT_PORT_INVALID:
1009 		mutex_enter(&pwp->lock);
1010 		if (pwp->state != STATE_RUNNING) {
1011 			mutex_exit(&pwp->lock);
1012 			break;
1013 		}
1014 		mutex_exit(&pwp->lock);
1015 		pmcs_kill_port(pwp, portid);
1016 		pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1017 		    "PortID 0x%x: PORT Now Invalid", portid);
1018 		break;
1019 	case IOP_EVENT_PORT_RESET_TIMER_TMO:
1020 		EVT_PRT(pwp, IOP_EVENT_PORT_RESET_TIMER_TMO, phynum);
1021 		break;
1022 	case IOP_EVENT_PORT_RESET_COMPLETE:
1023 		EVT_PRT(pwp, IOP_EVENT_PORT_RESET_COMPLETE, phynum);
1024 		break;
1025 	case IOP_EVENT_BROADCAST_ASYNC_EVENT:
1026 		EVT_PRT(pwp, IOP_EVENT_BROADCAST_ASYNC_EVENT, phynum);
1027 		/*
1028 		 * Comparing Section 6.8.1.4 of SMHBA (rev 7) spec and Section
1029 		 * 7.2.3 of SAS2 (Rev 15) spec,
1030 		 * _BROADCAST_ASYNC event corresponds to _D04_7 primitive
1031 		 */
1032 		mutex_enter(&pwp->lock);
1033 		pptr = pwp->ports[portid];
1034 		mutex_exit(&pwp->lock);
1035 		if (pptr) {
1036 			pmcs_lock_phy(pptr);
1037 			pmcs_smhba_log_sysevent(pwp, ESC_SAS_HBA_PORT_BROADCAST,
1038 			    SAS_PORT_BROADCAST_D04_7, pptr);
1039 			pmcs_unlock_phy(pptr);
1040 		}
1041 		break;
1042 	default:
1043 		(void) snprintf(buf, sizeof (buf),
1044 		    "unknown SAS H/W Event PHY 0x%x", phynum);
1045 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG, buf, iomb);
1046 		break;
1047 	}
1048 	if (need_ack) {
1049 		mutex_enter(&pwp->lock);
1050 		/*
1051 		 * Don't lock the entire tree for this.  Just grab the mutex
1052 		 * on the root PHY.
1053 		 */
1054 		tphyp = pwp->root_phys + phynum;
1055 		mutex_enter(&tphyp->phy_lock);
1056 		tphyp->hw_event_ack = w1;
1057 		mutex_exit(&tphyp->phy_lock);
1058 		mutex_exit(&pwp->lock);
1059 		pmcs_ack_events(pwp);
1060 	}
1061 }
1062 
1063 static void
pmcs_process_echo_completion(pmcs_hw_t * pwp,void * iomb,size_t amt)1064 pmcs_process_echo_completion(pmcs_hw_t *pwp, void *iomb, size_t amt)
1065 {
1066 	echo_test_t fred;
1067 	pmcwork_t *pwrk;
1068 	uint32_t *msg = iomb, htag = LE_32(msg[1]);
1069 	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
1070 	if (pwrk) {
1071 		(void) memcpy(&fred, &((uint32_t *)iomb)[2], sizeof (fred));
1072 		fred.ptr[0]++;
1073 		msg[2] = LE_32(PMCOUT_STATUS_OK);
1074 		pmcs_complete_work(pwp, pwrk, msg, amt);
1075 	} else {
1076 		pmcs_print_entry(pwp, PMCS_PRT_DEBUG,
1077 		    "ECHO completion with no work structure", iomb);
1078 	}
1079 }
1080 
1081 static void
pmcs_process_ssp_event(pmcs_hw_t * pwp,void * iomb,size_t amt)1082 pmcs_process_ssp_event(pmcs_hw_t *pwp, void *iomb, size_t amt)
1083 {
1084 	_NOTE(ARGUNUSED(amt));
1085 	uint32_t status, htag, *w;
1086 	pmcwork_t *pwrk;
1087 	pmcs_phy_t *phyp = NULL;
1088 	char *path;
1089 
1090 	w = iomb;
1091 	htag = LE_32(w[1]);
1092 	status = LE_32(w[2]);
1093 
1094 
1095 	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
1096 	if (pwrk == NULL) {
1097 		path = "????";
1098 	} else {
1099 		phyp = pwrk->phy;
1100 		path = pwrk->phy->path;
1101 	}
1102 
1103 	if (status != PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) {
1104 		char buf[20];
1105 		const char *emsg = pmcs_status_str(status);
1106 
1107 		if (emsg == NULL) {
1108 			(void) snprintf(buf, sizeof (buf), "Status 0x%x",
1109 			    status);
1110 			emsg = buf;
1111 		}
1112 		pmcs_prt(pwp, PMCS_PRT_DEBUG, phyp, NULL, "%s: Bad SAS Status "
1113 		    "(tag 0x%x) %s on %s", __func__, htag, emsg, path);
1114 		if (pwrk != NULL) {
1115 			/*
1116 			 * There may be pending command on a target device.
1117 			 * Or, it may be a double fault.
1118 			 */
1119 			pmcs_start_ssp_event_recovery(pwp, pwrk, iomb, amt);
1120 		}
1121 	} else {
1122 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, phyp, NULL,
1123 		    "%s: tag %x put onto the wire for %s",
1124 		    __func__, htag, path);
1125 		if (pwrk) {
1126 			pwrk->onwire = 1;
1127 			mutex_exit(&pwrk->lock);
1128 		}
1129 	}
1130 }
1131 
1132 static void
pmcs_process_sata_event(pmcs_hw_t * pwp,void * iomb,size_t amt)1133 pmcs_process_sata_event(pmcs_hw_t *pwp, void *iomb, size_t amt)
1134 {
1135 	_NOTE(ARGUNUSED(amt));
1136 	pmcwork_t *pwrk = NULL;
1137 	pmcs_phy_t *pptr = NULL;
1138 	uint32_t status, htag, *w;
1139 	char *path = NULL;
1140 
1141 	w = iomb;
1142 	htag = LE_32(w[1]);
1143 	status = LE_32(w[2]);
1144 
1145 	/*
1146 	 * If the status is PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE,
1147 	 * we have to issue a READ LOG EXT ATA (page 0x10) command
1148 	 * to the device. In this case, htag is not valid.
1149 	 *
1150 	 * If the status is PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED, we're
1151 	 * just noting that an I/O got put onto the wire.
1152 	 *
1153 	 * Othewise, other errors are indicative that things need to
1154 	 * be aborted.
1155 	 */
1156 	if (htag) {
1157 		pwrk = pmcs_tag2wp(pwp, htag, B_TRUE);
1158 	}
1159 
1160 	if (pwrk) {
1161 		pptr = pwrk->phy;
1162 		path = pptr->path;
1163 	} else {
1164 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL, "%s: "
1165 		    "cannot find work structure for SATA completion", __func__);
1166 		return;
1167 	}
1168 
1169 	if (status != PMCOUT_STATUS_XFER_CMD_FRAME_ISSUED) {
1170 		char buf[20];
1171 		const char *emsg = pmcs_status_str(status);
1172 
1173 		if (emsg == NULL) {
1174 			(void) snprintf(buf, sizeof (buf), "Status 0x%x",
1175 			    status);
1176 			emsg = buf;
1177 		}
1178 		if (status == PMCOUT_STATUS_XFER_ERROR_ABORTED_NCQ_MODE) {
1179 			pptr->need_rl_ext = 1;
1180 			htag = 0;
1181 		} else {
1182 			pptr->abort_pending = 1;
1183 		}
1184 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1185 		    "%s: Bad SATA Status (tag 0x%x) %s on %s",
1186 		    __func__, htag, emsg, path);
1187 		SCHEDULE_WORK(pwp, PMCS_WORK_ABORT_HANDLE);
1188 		/*
1189 		 * Unlike SSP devices, we let the abort we
1190 		 * schedule above force the completion of
1191 		 * problem commands.
1192 		 */
1193 		mutex_exit(&pwrk->lock);
1194 	} else {
1195 		pmcs_prt(pwp, PMCS_PRT_DEBUG2, pptr, NULL,
1196 		    "%s: tag %x put onto the wire for %s",
1197 		    __func__, htag, path);
1198 		pwrk->onwire = 1;
1199 		mutex_exit(&pwrk->lock);
1200 	}
1201 
1202 	mutex_exit(&pptr->phy_lock);
1203 }
1204 
1205 static void
pmcs_process_abort_completion(pmcs_hw_t * pwp,void * iomb,size_t amt)1206 pmcs_process_abort_completion(pmcs_hw_t *pwp, void *iomb, size_t amt)
1207 {
1208 	pmcs_phy_t *pptr;
1209 	struct pmcwork *pwrk;
1210 	uint32_t htag = LE_32(((uint32_t *)iomb)[1]);
1211 	uint32_t status = LE_32(((uint32_t *)iomb)[2]);
1212 	uint32_t scope = LE_32(((uint32_t *)iomb)[3]) & 0x1;
1213 	char *path;
1214 
1215 	pwrk = pmcs_tag2wp(pwp, htag, B_TRUE);
1216 	if (pwrk == NULL) {
1217 		pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1218 		    "%s: cannot find work structure for ABORT", __func__);
1219 		return;
1220 	}
1221 
1222 	pptr = pwrk->phy;
1223 	if (pptr) {
1224 		pptr->abort_pending = 0;
1225 		pptr->abort_sent = 0;
1226 
1227 		/*
1228 		 * Don't do this if the status was ABORT_IN_PROGRESS and
1229 		 * the scope bit was set
1230 		 */
1231 		if ((status != PMCOUT_STATUS_IO_ABORT_IN_PROGRESS) || !scope) {
1232 			pptr->abort_all_start = 0;
1233 			cv_signal(&pptr->abort_all_cv);
1234 		}
1235 		path = pptr->path;
1236 		mutex_exit(&pptr->phy_lock);
1237 	} else {
1238 		path = "(no phy)";
1239 	}
1240 
1241 	switch (status) {
1242 	case PMCOUT_STATUS_OK:
1243 		if (scope) {
1244 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1245 			    "%s: abort all succeeded for %s. (htag=0x%x)",
1246 			    __func__, path, htag);
1247 		} else {
1248 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1249 			    "%s: abort tag 0x%x succeeded for %s. (htag=0x%x)",
1250 			    __func__, pwrk->abt_htag, path, htag);
1251 		}
1252 		break;
1253 
1254 	case PMCOUT_STATUS_IO_NOT_VALID:
1255 		if (scope) {
1256 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1257 			    "%s: ABORT %s failed (DEV NOT VALID) for %s. "
1258 			    "(htag=0x%x)", __func__, scope ? "all" : "tag",
1259 			    path, htag);
1260 		} else {
1261 			pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL,
1262 			    "%s: ABORT %s failed (I/O NOT VALID) for %s. "
1263 			    "(htag=0x%x)", __func__, scope ? "all" : "tag",
1264 			    path, htag);
1265 		}
1266 		break;
1267 
1268 	case PMCOUT_STATUS_IO_ABORT_IN_PROGRESS:
1269 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: ABORT %s failed "
1270 		    "for %s, htag 0x%x (ABORT IN PROGRESS)", __func__,
1271 		    scope ? "all" : "tag", path, htag);
1272 		break;
1273 
1274 	default:
1275 		pmcs_prt(pwp, PMCS_PRT_DEBUG, pptr, NULL, "%s: Unknown status "
1276 		    "%d for ABORT %s, htag 0x%x, PHY %s", __func__, status,
1277 		    scope ? "all" : "tag", htag, path);
1278 		break;
1279 	}
1280 
1281 	pmcs_complete_work(pwp, pwrk, iomb, amt);
1282 }
1283 
1284 static void
pmcs_process_general_event(pmcs_hw_t * pwp,uint32_t * iomb)1285 pmcs_process_general_event(pmcs_hw_t *pwp, uint32_t *iomb)
1286 {
1287 	uint32_t htag;
1288 	char local[60];
1289 	struct pmcwork *pwrk;
1290 	int i;
1291 
1292 	if (LE_32(iomb[1]) == INBOUND_IOMB_V_BIT_NOT_SET) {
1293 		(void) snprintf(local, sizeof (local),
1294 		    "VALID bit not set on INBOUND IOMB");
1295 	} else if (LE_32(iomb[1]) ==
1296 	    INBOUND_IOMB_OPC_NOT_SUPPORTED) {
1297 		(void) snprintf(local, sizeof (local),
1298 		    "opcode not set on inbound IOMB");
1299 	} else {
1300 		(void) snprintf(local, sizeof (local),
1301 		    "unknown GENERAL EVENT status (0x%x)",
1302 		    LE_32(iomb[1]));
1303 	}
1304 	/* Pull up bad IOMB into usual position */
1305 	for (i = 0; i < PMCS_MSG_SIZE - 2; i++) {
1306 		iomb[i] = iomb[i+2];
1307 	}
1308 	/* overwrite status with an error */
1309 	iomb[2] = LE_32(PMCOUT_STATUS_PROG_ERROR);
1310 	iomb[PMCS_MSG_SIZE - 2] = 0;
1311 	iomb[PMCS_MSG_SIZE - 1] = 0;
1312 	htag = LE_32(iomb[1]);
1313 	pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, iomb);
1314 	pwrk = pmcs_tag2wp(pwp, htag, B_FALSE);
1315 	if (pwrk) {
1316 		pmcs_complete_work(pwp, pwrk, iomb, PMCS_QENTRY_SIZE);
1317 	}
1318 }
1319 
1320 void
pmcs_general_intr(pmcs_hw_t * pwp)1321 pmcs_general_intr(pmcs_hw_t *pwp)
1322 {
1323 	char local[PMCS_QENTRY_SIZE << 1];
1324 	uint32_t w0, pi, ci;
1325 	uint32_t *ptr, nbuf, lim = 0;
1326 	size_t amt;
1327 
1328 	ci = pmcs_rd_oqci(pwp, PMCS_OQ_GENERAL);
1329 	pi = pmcs_rd_oqpi(pwp, PMCS_OQ_GENERAL);
1330 
1331 	while (ci != pi) {
1332 		OQLIM_CHECK(pwp, lim);
1333 		ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, 0);
1334 		w0 = LE_32(ptr[0]);
1335 		VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi);
1336 		WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_GENERAL);
1337 		COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr,
1338 		    PMCS_OQ_GENERAL, ci);
1339 
1340 		switch (w0 & PMCS_IOMB_OPCODE_MASK) {
1341 		case PMCOUT_SSP_COMPLETION:
1342 			/*
1343 			 * We only get SSP completion here for Task Management
1344 			 * completions.
1345 			 */
1346 		case PMCOUT_SMP_COMPLETION:
1347 		case PMCOUT_LOCAL_PHY_CONTROL:
1348 		case PMCOUT_DEVICE_REGISTRATION:
1349 		case PMCOUT_DEREGISTER_DEVICE_HANDLE:
1350 		case PMCOUT_GET_NVMD_DATA:
1351 		case PMCOUT_SET_NVMD_DATA:
1352 		case PMCOUT_GET_DEVICE_STATE:
1353 		case PMCOUT_SET_DEVICE_STATE:
1354 			pmcs_process_completion(pwp, local, amt);
1355 			break;
1356 		case PMCOUT_SSP_ABORT:
1357 		case PMCOUT_SATA_ABORT:
1358 		case PMCOUT_SMP_ABORT:
1359 			pmcs_process_abort_completion(pwp, local, amt);
1360 			break;
1361 		case PMCOUT_SSP_EVENT:
1362 			pmcs_process_ssp_event(pwp, local, amt);
1363 			break;
1364 		case PMCOUT_ECHO:
1365 			pmcs_process_echo_completion(pwp, local, amt);
1366 			break;
1367 		case PMCOUT_SAS_HW_EVENT_ACK_ACK:
1368 			if (LE_32(ptr[2]) != SAS_HW_EVENT_ACK_OK) {
1369 				pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1370 				    "SAS H/W EVENT ACK/ACK Status=0x%b",
1371 				    LE_32(ptr[2]), "\020\4InvParm\3"
1372 				    "InvPort\2InvPhy\1InvSEA");
1373 			}
1374 			pmcs_process_completion(pwp, local, amt);
1375 			break;
1376 		case PMCOUT_SKIP_ENTRIES:
1377 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
1378 			    "%s: skip %d entries", __func__, nbuf);
1379 			break;
1380 		default:
1381 			(void) snprintf(local, sizeof (local),
1382 			    "%s: unhandled message", __func__);
1383 			pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, ptr);
1384 			break;
1385 		}
1386 		STEP_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, nbuf);
1387 	}
1388 	if (lim) {
1389 		SYNC_OQ_ENTRY(pwp, PMCS_OQ_GENERAL, ci, pi);
1390 	}
1391 }
1392 
1393 /*
1394  * pmcs_check_intr_coal
1395  *
1396  * This function makes a determination on the dynamic value of the
1397  * interrupt coalescing timer register.  We only use this for I/O
1398  * completions.
1399  *
1400  * The basic algorithm is as follows:
1401  *
1402  * PMCS_MAX_IO_COMPS_PER_INTR: The maximum number of I/O completions per
1403  * I/O completion interrupt.  We won't increase the interrupt coalescing
1404  * timer if we're already processing this many completions per interrupt
1405  * beyond the threshold.
1406  *
1407  * Values in io_intr_coal structure:
1408  *
1409  * intr_latency: The average number of nsecs between interrupts during
1410  * the echo test.  Used to help determine whether to increase the coalescing
1411  * timer.
1412  *
1413  * intr_threshold: Calculated number of interrupts beyond which we may
1414  * increase the timer.  This value is calculated based on the calculated
1415  * interrupt latency during the ECHO test and the current value of the
1416  * coalescing timer.
1417  *
1418  * nsecs_between_intrs: Total number of nsecs between all the interrupts
1419  * in the current timeslice.
1420  *
1421  * last_io_comp: Time of the last I/O interrupt.
1422  *
1423  * num_io_completions: Number of I/O completions during the slice
1424  *
1425  * num_intrs: Number of I/O completion interrupts during the slice
1426  *
1427  * max_io_completions: Number of times we hit >= PMCS_MAX_IO_COMPS_PER_INTR
1428  * during interrupt processing.
1429  *
1430  * PMCS_MAX_IO_COMPS_LOWAT_SHIFT/HIWAT_SHIFT
1431  * Low and high marks used to determine whether we processed enough interrupts
1432  * that contained the maximum number of I/O completions to warrant increasing
1433  * the timer
1434  *
1435  * intr_coal_timer: The current value of the register (in usecs)
1436  *
1437  * timer_on: B_TRUE means we are using the timer
1438  *
1439  * The timer is increased if we processed more than intr_threshold interrupts
1440  * during the quantum and the number of interrupts containing the maximum
1441  * number of I/O completions is between PMCS_MAX_IO_COMPS_LOWAT_SHIFT and
1442  * _HIWAT_SHIFT
1443  *
1444  * If the average time between completions is greater than twice
1445  * the current timer value, the timer value is decreased.
1446  *
1447  * If we did not take any interrupts during a quantum, we turn the timer off.
1448  */
1449 void
pmcs_check_intr_coal(void * arg)1450 pmcs_check_intr_coal(void *arg)
1451 {
1452 	pmcs_hw_t	*pwp = (pmcs_hw_t *)arg;
1453 	uint32_t	avg_nsecs;
1454 	clock_t		lbolt, ret;
1455 	pmcs_io_intr_coal_t *ici;
1456 
1457 	ici = &pwp->io_intr_coal;
1458 	mutex_enter(&pwp->ict_lock);
1459 	while (ici->stop_thread == B_FALSE) {
1460 		/*
1461 		 * Wait for next time quantum... collect stats
1462 		 */
1463 		lbolt = ddi_get_lbolt();
1464 		while (ici->stop_thread == B_FALSE) {
1465 			ret = cv_timedwait(&pwp->ict_cv, &pwp->ict_lock,
1466 			    lbolt + ici->quantum);
1467 			if (ret == -1) {
1468 				break;
1469 			}
1470 		}
1471 
1472 		if (ici->stop_thread == B_TRUE) {
1473 			continue;
1474 		}
1475 
1476 		DTRACE_PROBE1(pmcs__check__intr__coal, pmcs_io_intr_coal_t *,
1477 		    &pwp->io_intr_coal);
1478 
1479 		/*
1480 		 * Determine whether to adjust timer
1481 		 */
1482 		if (ici->num_intrs == 0) {
1483 			/*
1484 			 * If timer is off, nothing more to do.
1485 			 */
1486 			if (!pwp->io_intr_coal.timer_on) {
1487 				continue;
1488 			}
1489 
1490 			/*
1491 			 * No interrupts.  Turn off the timer.
1492 			 */
1493 			pmcs_wr_topunit(pwp, PMCS_INT_COALESCING_CONTROL, 0);
1494 
1495 			if (pwp->odb_auto_clear & (1 << PMCS_MSIX_IODONE)) {
1496 				pmcs_wr_topunit(pwp, PMCS_OBDB_AUTO_CLR,
1497 				    pwp->odb_auto_clear);
1498 			}
1499 
1500 			ici->timer_on = B_FALSE;
1501 			ici->max_io_completions = 0;
1502 			ici->num_intrs = 0;
1503 			ici->int_cleared = B_FALSE;
1504 			ici->num_io_completions = 0;
1505 			DTRACE_PROBE1(pmcs__intr__coalesce__timer__off,
1506 			    pmcs_io_intr_coal_t *, ici);
1507 			continue;
1508 		}
1509 
1510 		avg_nsecs = ici->nsecs_between_intrs / ici->num_intrs;
1511 
1512 		if ((ici->num_intrs > ici->intr_threshold) &&
1513 		    (ici->max_io_completions > (ici->num_intrs >>
1514 		    PMCS_MAX_IO_COMPS_LOWAT_SHIFT)) &&
1515 		    (ici->max_io_completions < (ici->num_intrs >>
1516 		    PMCS_MAX_IO_COMPS_HIWAT_SHIFT))) {
1517 			pmcs_set_intr_coal_timer(pwp, INCREASE_TIMER);
1518 		} else if (avg_nsecs >
1519 		    (ici->intr_coal_timer * 1000 * 2)) {
1520 			pmcs_set_intr_coal_timer(pwp, DECREASE_TIMER);
1521 		}
1522 
1523 		/*
1524 		 * Reset values for new sampling period.
1525 		 */
1526 		ici->max_io_completions = 0;
1527 		ici->nsecs_between_intrs = 0;
1528 		ici->num_intrs = 0;
1529 		ici->num_io_completions = 0;
1530 
1531 		/*
1532 		 * If a firmware event log file is configured, check to see
1533 		 * if it needs to be written to the file.  We do this here
1534 		 * because writing to a file from a callout thread (i.e.
1535 		 * from the watchdog timer) can cause livelocks.
1536 		 */
1537 		if (pwp->fwlog_file) {
1538 			mutex_exit(&pwp->ict_lock);
1539 			pmcs_gather_fwlog(pwp);
1540 			mutex_enter(&pwp->ict_lock);
1541 		}
1542 	}
1543 
1544 	mutex_exit(&pwp->ict_lock);
1545 	thread_exit();
1546 }
1547 
1548 void
pmcs_iodone_intr(pmcs_hw_t * pwp)1549 pmcs_iodone_intr(pmcs_hw_t *pwp)
1550 {
1551 	char local[PMCS_QENTRY_SIZE << 1];
1552 	pmcs_iocomp_cb_t *ioccb;
1553 	uint32_t w0, ci, pi, nbuf, lim = 0, niodone = 0, iomb_opcode;
1554 	size_t amt;
1555 	uint32_t *ptr;
1556 	hrtime_t curtime = gethrtime();
1557 
1558 	ci = pmcs_rd_oqci(pwp, PMCS_OQ_IODONE);
1559 	pi = pmcs_rd_oqpi(pwp, PMCS_OQ_IODONE);
1560 
1561 	while (ci != pi) {
1562 		OQLIM_CHECK(pwp, lim);
1563 		ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, 0);
1564 		w0 = LE_32(ptr[0]);
1565 		VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi);
1566 		WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_IODONE);
1567 		iomb_opcode = (w0 & PMCS_IOMB_OPCODE_MASK);
1568 
1569 		if ((iomb_opcode == PMCOUT_SSP_COMPLETION) ||
1570 		    (iomb_opcode == PMCOUT_SATA_COMPLETION)) {
1571 			ioccb =
1572 			    kmem_cache_alloc(pwp->iocomp_cb_cache, KM_NOSLEEP);
1573 			if (ioccb == NULL) {
1574 				pmcs_prt(pwp, PMCS_PRT_WARN, NULL, NULL,
1575 				    "%s: kmem_cache_alloc failed", __func__);
1576 				break;
1577 			}
1578 
1579 			COPY_OUTBOUND(pwp, w0, ioccb->iomb, nbuf, amt, ptr,
1580 			    PMCS_OQ_IODONE, ci);
1581 
1582 			niodone++;
1583 			pmcs_process_io_completion(pwp, ioccb, amt);
1584 		} else {
1585 			COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr,
1586 			    PMCS_OQ_IODONE, ci);
1587 
1588 			switch (iomb_opcode) {
1589 			case PMCOUT_ECHO:
1590 				pmcs_process_echo_completion(pwp, local, amt);
1591 				break;
1592 			case PMCOUT_SATA_EVENT:
1593 				pmcs_process_sata_event(pwp, local, amt);
1594 				break;
1595 			case PMCOUT_SSP_EVENT:
1596 				pmcs_process_ssp_event(pwp, local, amt);
1597 				break;
1598 			case PMCOUT_SKIP_ENTRIES:
1599 				pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
1600 				    "%s: skip %d entries", __func__, nbuf);
1601 				break;
1602 			default:
1603 				(void) snprintf(local, sizeof (local),
1604 				    "%s: unhandled message", __func__);
1605 				pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local,
1606 				    ptr);
1607 				break;
1608 			}
1609 		}
1610 
1611 		STEP_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, nbuf);
1612 	}
1613 
1614 	if (lim != 0) {
1615 		SYNC_OQ_ENTRY(pwp, PMCS_OQ_IODONE, ci, pi);
1616 	}
1617 
1618 	/*
1619 	 * Update the interrupt coalescing timer check stats and run
1620 	 * completions for queued up commands.
1621 	 */
1622 
1623 	if (niodone > 0) {
1624 		/*
1625 		 * If we can't get the lock, then completions are either
1626 		 * already running or will be scheduled to do so shortly.
1627 		 */
1628 		if (mutex_tryenter(&pwp->cq_lock) != 0) {
1629 			PMCS_CQ_RUN_LOCKED(pwp);
1630 			mutex_exit(&pwp->cq_lock);
1631 		}
1632 
1633 		mutex_enter(&pwp->ict_lock);
1634 		pwp->io_intr_coal.nsecs_between_intrs +=
1635 		    curtime - pwp->io_intr_coal.last_io_comp;
1636 		pwp->io_intr_coal.num_intrs++;
1637 		pwp->io_intr_coal.num_io_completions += niodone;
1638 		if (niodone >= PMCS_MAX_IO_COMPS_PER_INTR) {
1639 			pwp->io_intr_coal.max_io_completions++;
1640 		}
1641 		pwp->io_intr_coal.last_io_comp = gethrtime();
1642 		mutex_exit(&pwp->ict_lock);
1643 	}
1644 }
1645 
1646 void
pmcs_event_intr(pmcs_hw_t * pwp)1647 pmcs_event_intr(pmcs_hw_t *pwp)
1648 {
1649 	char local[PMCS_QENTRY_SIZE << 1];
1650 	uint32_t w0, ci, pi, nbuf, lim =  0;
1651 	size_t amt;
1652 	uint32_t *ptr;
1653 
1654 	ci = pmcs_rd_oqci(pwp, PMCS_OQ_EVENTS);
1655 	pi = pmcs_rd_oqpi(pwp, PMCS_OQ_EVENTS);
1656 
1657 	while (ci != pi) {
1658 		OQLIM_CHECK(pwp, lim);
1659 		ptr = GET_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, 0);
1660 		w0 = LE_32(ptr[0]);
1661 		VALID_IOMB_CHECK(pwp, w0, ptr, ci, pi);
1662 		WRONG_OBID_CHECK(pwp, w0, PMCS_OQ_EVENTS);
1663 		COPY_OUTBOUND(pwp, w0, local, nbuf, amt, ptr,
1664 		    PMCS_OQ_EVENTS, ci);
1665 
1666 		switch (w0 & PMCS_IOMB_OPCODE_MASK) {
1667 		case PMCOUT_ECHO:
1668 			pmcs_process_echo_completion(pwp, local, amt);
1669 			break;
1670 		case PMCOUT_SATA_EVENT:
1671 			pmcs_process_sata_event(pwp, local, amt);
1672 			break;
1673 		case PMCOUT_SSP_EVENT:
1674 			pmcs_process_ssp_event(pwp, local, amt);
1675 			break;
1676 		case PMCOUT_GENERAL_EVENT:
1677 			pmcs_process_general_event(pwp, ptr);
1678 			break;
1679 		case PMCOUT_DEVICE_HANDLE_REMOVED:
1680 		{
1681 			uint32_t port = IOP_EVENT_PORTID(LE_32(ptr[1]));
1682 			uint32_t did = LE_32(ptr[2]);
1683 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1684 			    "PortID 0x%x device_id 0x%x removed", port, did);
1685 			break;
1686 		}
1687 		case PMCOUT_SAS_HW_EVENT:
1688 			if (nbuf > 1) {
1689 				pmcs_prt(pwp, PMCS_PRT_INFO, NULL, NULL,
1690 				    "multiple SAS HW_EVENT (%d) responses "
1691 				    "in EVENT OQ", nbuf);
1692 			}
1693 			pmcs_process_sas_hw_event(pwp, local, PMCS_QENTRY_SIZE);
1694 			break;
1695 		case PMCOUT_FW_FLASH_UPDATE:
1696 		case PMCOUT_GET_TIME_STAMP:
1697 		case PMCOUT_GET_DEVICE_STATE:
1698 		case PMCOUT_SET_DEVICE_STATE:
1699 		case PMCOUT_SAS_DIAG_EXECUTE:
1700 			pmcs_process_completion(pwp, local, amt);
1701 			break;
1702 		case PMCOUT_SKIP_ENTRIES:
1703 			pmcs_prt(pwp, PMCS_PRT_DEBUG3, NULL, NULL,
1704 			    "%s: skip %d entries", __func__, nbuf);
1705 			break;
1706 		default:
1707 			(void) snprintf(local, sizeof (local),
1708 			    "%s: unhandled message", __func__);
1709 			pmcs_print_entry(pwp, PMCS_PRT_DEBUG, local, ptr);
1710 			break;
1711 		}
1712 		STEP_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, nbuf);
1713 	}
1714 	if (lim) {
1715 		SYNC_OQ_ENTRY(pwp, PMCS_OQ_EVENTS, ci, pi);
1716 	}
1717 }
1718 
1719 void
pmcs_timed_out(pmcs_hw_t * pwp,uint32_t htag,const char * func)1720 pmcs_timed_out(pmcs_hw_t *pwp, uint32_t htag, const char *func)
1721 {
1722 #ifdef	DEBUG
1723 	hrtime_t now = gethrtime();
1724 	int i;
1725 
1726 	for (i = 0; i < 256; i++) {
1727 		if (pwp->ftags[i] == htag) {
1728 			pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1729 			    "Inbound msg (tag 0x%8x) timed out - "
1730 			    "was started %llu ns ago in %s:%d",
1731 			    htag, (unsigned long long) (now - pwp->ftime[i]),
1732 			    func, pwp->ftag_lines[i]);
1733 			return;
1734 		}
1735 	}
1736 #endif
1737 	pmcs_prt(pwp, PMCS_PRT_DEBUG, NULL, NULL,
1738 	    "Inbound Message (tag 0x%08x) timed out- was started in %s",
1739 	    htag, func);
1740 }
1741