xref: /illumos-gate/usr/src/uts/common/io/usb/hcd/ehci/ehci_polled.c (revision 0a1044f1d1346ec799488913a15abfae3e3c848a)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 /*
27  * EHCI Host Controller Driver (EHCI)
28  *
29  * The EHCI driver is a software driver which interfaces to the Universal
30  * Serial Bus layer (USBA) and the Host Controller (HC). The interface to
31  * the Host Controller is defined by the EHCI Host Controller Interface.
32  *
33  * This module contains the specific EHCI code used in POLLED mode. This
34  * code is in a separate file since it will never become part of the EHCI
35  * driver.
36  */
37 
38 #include <sys/usb/usba/usbai_version.h>
39 #include <sys/usb/hcd/ehci/ehcid.h>
40 #include <sys/usb/hcd/ehci/ehci_xfer.h>
41 #include <sys/usb/hcd/ehci/ehci_intr.h>
42 #include <sys/usb/hcd/ehci/ehci_util.h>
43 #include <sys/usb/hcd/ehci/ehci_polled.h>
44 
45 #ifndef __sparc
46 extern void invalidate_cache();
47 #endif
48 
49 /*
50  * Internal Function Prototypes
51  */
52 
53 /* Polled initialization routines */
54 static int	ehci_polled_init(
55 				usba_pipe_handle_data_t	*ph,
56 				ehci_state_t		*ehcip,
57 				usb_console_info_impl_t	*console_input_info);
58 
59 /* Polled deinitialization routines */
60 static int	ehci_polled_fini(ehci_polled_t		*ehci_polledp);
61 
62 /* Polled save state routines */
63 static void	ehci_polled_save_state(ehci_polled_t	*ehci_polledp);
64 
65 /* Polled restore state routines */
66 static void	ehci_polled_restore_state(ehci_polled_t	*ehci_polledp);
67 static void	ehci_polled_stop_processing(
68 				ehci_polled_t		*ehci_polledp);
69 static void	ehci_polled_start_processing(
70 				ehci_polled_t		*ehci_polledp);
71 
72 /* Polled read routines */
73 static int	ehci_polled_process_active_intr_qtd_list(
74 				ehci_polled_t		*ehci_polledp);
75 static int	ehci_polled_handle_normal_qtd(
76 				ehci_polled_t		*ehci_polledp,
77 				ehci_qtd_t		*qtd);
78 static void	ehci_polled_insert_intr_qtd(
79 				ehci_polled_t		*ehci_polledp,
80 				ehci_qtd_t		*qtd);
81 static void	ehci_polled_insert_bulk_qtd(
82 				ehci_polled_t		*ehci_polledp);
83 static void	ehci_polled_fill_in_qtd(
84 				ehci_state_t		*ehcip,
85 				ehci_qtd_t		*qtd,
86 				uint_t			qtd_ctrl,
87 				size_t			qtd_dma_offs,
88 				size_t			qtd_length,
89 				ehci_trans_wrapper_t	*tw);
90 static void	ehci_polled_insert_qtd_on_tw(
91 				ehci_state_t		*ehcip,
92 				ehci_trans_wrapper_t	*tw,
93 				ehci_qtd_t		*qtd);
94 static ehci_qtd_t *ehci_polled_create_done_qtd_list(
95 				ehci_polled_t		*ehci_polledp);
96 static void	ehci_polled_insert_qtd_into_active_intr_qtd_list(
97 				ehci_polled_t		*ehci_polledp,
98 				ehci_qtd_t		*curr_qtd);
99 static void	ehci_polled_remove_qtd_from_active_intr_qtd_list(
100 				ehci_polled_t		*ehci_polledp,
101 				ehci_qtd_t		*curr_qtd);
102 static void	ehci_polled_traverse_qtds(
103 				ehci_polled_t		*ehci_polledp,
104 				usba_pipe_handle_data_t	*ph);
105 static void	ehci_polled_finish_interrupt(
106 				ehci_state_t		*ehcip,
107 				uint_t			intr);
108 static int	ehci_polled_create_tw(
109 				ehci_polled_t		*ehci_polledp,
110 				usba_pipe_handle_data_t	*ph,
111 				usb_flags_t		usb_flags);
112 static void	ehci_polled_insert_async_qh(
113 				ehci_state_t		*ehcip,
114 				ehci_pipe_private_t	*pp);
115 static void	ehci_polled_remove_async_qh(
116 				ehci_state_t		*ehcip,
117 				ehci_pipe_private_t	*pp);
118 
119 /*
120  * POLLED entry points
121  *
122  * These functions are entry points into the POLLED code.
123  */
124 
125 /*
126  * ehci_hcdi_polled_input_init:
127  *
128  * This is the initialization routine for handling the USB input device
129  * in POLLED mode.  This routine is not called from POLLED mode, so
130  * it is OK to acquire mutexes.
131  */
132 int
ehci_hcdi_polled_input_init(usba_pipe_handle_data_t * ph,uchar_t ** polled_buf,usb_console_info_impl_t * console_input_info)133 ehci_hcdi_polled_input_init(
134 	usba_pipe_handle_data_t	*ph,
135 	uchar_t			**polled_buf,
136 	usb_console_info_impl_t	*console_input_info)
137 {
138 	ehci_polled_t		*ehci_polledp;
139 	ehci_state_t		*ehcip;
140 	int			ret;
141 
142 	ehcip = ehci_obtain_state(ph->p_usba_device->usb_root_hub_dip);
143 
144 	/*
145 	 * Grab the ehci_int_mutex so that things don't change on us
146 	 * if an interrupt comes in.
147 	 */
148 	mutex_enter(&ehcip->ehci_int_mutex);
149 
150 	ret = ehci_polled_init(ph, ehcip, console_input_info);
151 
152 	if (ret != USB_SUCCESS) {
153 
154 		/* Allow interrupts to continue */
155 		mutex_exit(&ehcip->ehci_int_mutex);
156 		return (ret);
157 	}
158 
159 	ehci_polledp = (ehci_polled_t *)console_input_info->uci_private;
160 
161 	/*
162 	 * Mark the structure so that if we are using it, we don't free
163 	 * the structures if one of them is unplugged.
164 	 */
165 	ehci_polledp->ehci_polled_flags |= POLLED_INPUT_MODE;
166 
167 	/* increase the counter for keyboard connected */
168 	ehcip->ehci_polled_kbd_count ++;
169 
170 	/*
171 	 * This is the buffer we will copy characters into. It will be
172 	 * copied into at this layer, so we need to keep track of it.
173 	 */
174 	ehci_polledp->ehci_polled_buf =
175 	    (uchar_t *)kmem_zalloc(POLLED_RAW_BUF_SIZE, KM_SLEEP);
176 
177 	*polled_buf = ehci_polledp->ehci_polled_buf;
178 
179 	/*
180 	 * This is a software workaround to fix schizo hardware bug.
181 	 * Existence of "no-prom-cdma-sync"  property means consistent
182 	 * dma sync should not be done while in prom or polled mode.
183 	 */
184 	if (ddi_prop_exists(DDI_DEV_T_ANY, ehcip->ehci_dip,
185 	    DDI_PROP_NOTPROM, "no-prom-cdma-sync")) {
186 		ehci_polledp->ehci_polled_no_sync_flag = B_TRUE;
187 	}
188 
189 	/* Allow interrupts to continue */
190 	mutex_exit(&ehcip->ehci_int_mutex);
191 
192 	return (USB_SUCCESS);
193 }
194 
195 
196 /*
197  * ehci_hcdi_polled_input_fini:
198  */
199 int
ehci_hcdi_polled_input_fini(usb_console_info_impl_t * info)200 ehci_hcdi_polled_input_fini(usb_console_info_impl_t *info)
201 {
202 	ehci_polled_t		*ehci_polledp;
203 	ehci_state_t		*ehcip;
204 	int			ret;
205 
206 	ehci_polledp = (ehci_polled_t *)info->uci_private;
207 
208 	ehcip = ehci_polledp->ehci_polled_ehcip;
209 
210 	mutex_enter(&ehcip->ehci_int_mutex);
211 
212 	/*
213 	 * Reset the POLLED_INPUT_MODE flag so that we can tell if
214 	 * this structure is in use in the ehci_polled_fini routine.
215 	 */
216 	ehci_polledp->ehci_polled_flags &= ~POLLED_INPUT_MODE;
217 
218 	/* decrease the counter for keyboard disconnected */
219 	ehcip->ehci_polled_kbd_count --;
220 
221 	/* Free the buffer that we copied data into */
222 	kmem_free(ehci_polledp->ehci_polled_buf, POLLED_RAW_BUF_SIZE);
223 
224 	ret = ehci_polled_fini(ehci_polledp);
225 
226 	mutex_exit(&ehcip->ehci_int_mutex);
227 
228 	return (ret);
229 }
230 
231 
232 /*
233  * ehci_hcdi_polled_input_enter:
234  *
235  * This is where we enter into POLLED mode.  This routine sets up
236  * everything so that calls to	ehci_hcdi_polled_read will return
237  * characters.
238  */
239 int
ehci_hcdi_polled_input_enter(usb_console_info_impl_t * info)240 ehci_hcdi_polled_input_enter(usb_console_info_impl_t *info)
241 {
242 	ehci_polled_t		*ehci_polledp;
243 	ehci_state_t		*ehcip;
244 	usba_pipe_handle_data_t	*ph;
245 	ehci_pipe_private_t	*pp;
246 	int			pipe_attr;
247 
248 #ifndef lint
249 	_NOTE(NO_COMPETING_THREADS_NOW);
250 #endif
251 	ehci_polledp = (ehci_polled_t *)info->uci_private;
252 	ehcip = ehci_polledp->ehci_polled_ehcip;
253 	ph = ehci_polledp->ehci_polled_input_pipe_handle;
254 	pp = (ehci_pipe_private_t *)ph->p_hcd_private;
255 
256 	pipe_attr = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK;
257 #ifndef lint
258 	_NOTE(COMPETING_THREADS_NOW);
259 #endif
260 
261 	ehci_polledp->ehci_polled_entry++;
262 	/*
263 	 * If the controller is already switched over, just return
264 	 */
265 	if (ehci_polledp->ehci_polled_entry > 1) {
266 
267 		return (USB_SUCCESS);
268 	}
269 
270 	switch (pipe_attr) {
271 	case USB_EP_ATTR_INTR:
272 		ehci_polled_save_state(ehci_polledp);
273 		break;
274 	case USB_EP_ATTR_BULK:
275 #ifndef lint
276 	_NOTE(NO_COMPETING_THREADS_NOW);
277 #endif
278 		Set_OpReg(ehci_command, (Get_OpReg(ehci_command) &
279 		    ~(EHCI_CMD_PERIODIC_SCHED_ENABLE |
280 		    EHCI_CMD_ASYNC_SCHED_ENABLE)));
281 		/* Wait for few milliseconds */
282 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
283 
284 		ehci_polled_insert_async_qh(ehcip, pp);
285 
286 		Set_OpReg(ehci_command,
287 		    (Get_OpReg(ehci_command) | EHCI_CMD_ASYNC_SCHED_ENABLE));
288 #ifndef lint
289 	_NOTE(COMPETING_THREADS_NOW);
290 #endif
291 		/* Wait for few milliseconds */
292 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
293 		break;
294 	default:
295 		return (USB_FAILURE);
296 	}
297 
298 	ehci_polledp->ehci_polled_flags |= POLLED_INPUT_MODE_INUSE;
299 
300 	return (USB_SUCCESS);
301 }
302 
303 
304 /*
305  * ehci_hcdi_polled_input_exit:
306  *
307  * This is where we exit POLLED mode. This routine restores
308  * everything that is needed to continue operation.
309  */
310 int
ehci_hcdi_polled_input_exit(usb_console_info_impl_t * info)311 ehci_hcdi_polled_input_exit(usb_console_info_impl_t *info)
312 {
313 	ehci_polled_t		*ehci_polledp;
314 	ehci_state_t		*ehcip;
315 	ehci_pipe_private_t	*pp;
316 	int			pipe_attr;
317 
318 #ifndef lint
319 	_NOTE(NO_COMPETING_THREADS_NOW);
320 #endif
321 	ehci_polledp = (ehci_polled_t *)info->uci_private;
322 	ehcip = ehci_polledp->ehci_polled_ehcip;
323 	pp = (ehci_pipe_private_t *)ehci_polledp->
324 	    ehci_polled_input_pipe_handle->p_hcd_private;
325 
326 	pipe_attr = ehci_polledp->ehci_polled_input_pipe_handle->
327 	    p_ep.bmAttributes & USB_EP_ATTR_MASK;
328 #ifndef lint
329 	_NOTE(COMPETING_THREADS_NOW);
330 #endif
331 
332 	ehci_polledp->ehci_polled_entry--;
333 
334 	/*
335 	 * If there are still outstanding "enters", just return
336 	 */
337 	if (ehci_polledp->ehci_polled_entry > 0) {
338 
339 		return (USB_SUCCESS);
340 	}
341 
342 	ehci_polledp->ehci_polled_flags &= ~POLLED_INPUT_MODE_INUSE;
343 
344 	switch (pipe_attr & USB_EP_ATTR_MASK) {
345 	case USB_EP_ATTR_INTR:
346 		ehci_polled_restore_state(ehci_polledp);
347 		break;
348 	case USB_EP_ATTR_BULK:
349 #ifndef lint
350 	_NOTE(NO_COMPETING_THREADS_NOW);
351 #endif
352 		Set_OpReg(ehci_command, (Get_OpReg(ehci_command) &
353 		    ~(EHCI_CMD_PERIODIC_SCHED_ENABLE |
354 		    EHCI_CMD_ASYNC_SCHED_ENABLE)));
355 		/* Wait for few milliseconds */
356 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
357 
358 		ehci_polled_remove_async_qh(ehcip, pp);
359 
360 		Set_OpReg(ehci_command,
361 		    (Get_OpReg(ehci_command) | EHCI_CMD_ASYNC_SCHED_ENABLE |
362 		    EHCI_CMD_ASYNC_SCHED_ENABLE));
363 #ifndef lint
364 	_NOTE(COMPETING_THREADS_NOW);
365 #endif
366 		/* Wait for few milliseconds */
367 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
368 		break;
369 	default:
370 		return (USB_FAILURE);
371 
372 	}
373 
374 	return (USB_SUCCESS);
375 }
376 
377 
378 /*
379  * ehci_hcdi_polled_read:
380  *
381  * Get a key character
382  */
383 int
ehci_hcdi_polled_read(usb_console_info_impl_t * info,uint_t * num_characters)384 ehci_hcdi_polled_read(
385 	usb_console_info_impl_t	*info,
386 	uint_t			*num_characters)
387 {
388 	ehci_state_t		*ehcip;
389 	ehci_polled_t		*ehci_polledp;
390 	uint_t			intr;
391 	int			pipe_attr;
392 
393 	ehci_polledp = (ehci_polled_t *)info->uci_private;
394 
395 	ehcip = ehci_polledp->ehci_polled_ehcip;
396 
397 #ifndef lint
398 	_NOTE(NO_COMPETING_THREADS_NOW);
399 #endif
400 
401 	*num_characters = 0;
402 
403 	pipe_attr = ehci_polledp->ehci_polled_input_pipe_handle->
404 	    p_ep.bmAttributes & USB_EP_ATTR_MASK;
405 
406 	if (pipe_attr == USB_EP_ATTR_BULK) {
407 		ehci_polled_insert_bulk_qtd(ehci_polledp);
408 	}
409 
410 	intr = ((Get_OpReg(ehci_status) & Get_OpReg(ehci_interrupt)) &
411 	    (EHCI_INTR_FRAME_LIST_ROLLOVER |
412 	    EHCI_INTR_USB | EHCI_INTR_USB_ERROR));
413 
414 	/*
415 	 * Check whether any frame list rollover interrupt is pending
416 	 * and if it is pending, process this interrupt.
417 	 */
418 	if (intr & EHCI_INTR_FRAME_LIST_ROLLOVER) {
419 		/* Check any frame list rollover interrupt is pending */
420 		ehci_handle_frame_list_rollover(ehcip);
421 		ehci_polled_finish_interrupt(ehcip,
422 		    EHCI_INTR_FRAME_LIST_ROLLOVER);
423 	}
424 
425 	/* Process any QTD's on the active interrupt qtd list */
426 	*num_characters =
427 	    ehci_polled_process_active_intr_qtd_list(ehci_polledp);
428 
429 #ifndef lint
430 	_NOTE(COMPETING_THREADS_NOW);
431 #endif
432 
433 	return (USB_SUCCESS);
434 }
435 
436 
437 /*
438  * ehci_hcdi_polled_output_init:
439  *
440  * This is the initialization routine for handling the USB serial output
441  * in POLLED mode.  This routine is not called from POLLED mode, so
442  * it is OK to acquire mutexes.
443  */
444 int
ehci_hcdi_polled_output_init(usba_pipe_handle_data_t * ph,usb_console_info_impl_t * console_output_info)445 ehci_hcdi_polled_output_init(
446 	usba_pipe_handle_data_t	*ph,
447 	usb_console_info_impl_t	*console_output_info)
448 {
449 	ehci_polled_t		*ehci_polledp;
450 	ehci_state_t		*ehcip;
451 	ehci_pipe_private_t	*pp;
452 	int			ret;
453 
454 	ehcip = ehci_obtain_state(ph->p_usba_device->usb_root_hub_dip);
455 
456 	/*
457 	 * Grab the ehci_int_mutex so that things don't change on us
458 	 * if an interrupt comes in.
459 	 */
460 	mutex_enter(&ehcip->ehci_int_mutex);
461 
462 	ret = ehci_polled_init(ph, ehcip, console_output_info);
463 
464 	if (ret != USB_SUCCESS) {
465 
466 		/* Allow interrupts to continue */
467 		mutex_exit(&ehcip->ehci_int_mutex);
468 
469 		return (ret);
470 	}
471 
472 	ehci_polledp = (ehci_polled_t *)console_output_info->uci_private;
473 	/*
474 	 * Mark the structure so that if we are using it, we don't free
475 	 * the structures if one of them is unplugged.
476 	 */
477 	ehci_polledp->ehci_polled_flags |= POLLED_OUTPUT_MODE;
478 
479 	/*
480 	 * Insert the Endpoint Descriptor to appropriate endpoint list.
481 	 */
482 	pp = (ehci_pipe_private_t *)ehci_polledp->
483 	    ehci_polled_input_pipe_handle->p_hcd_private;
484 	ehci_polled_insert_async_qh(ehcip, pp);
485 
486 	/*
487 	 * This is a software workaround to fix schizo hardware bug.
488 	 * Existence of "no-prom-cdma-sync"  property means consistent
489 	 * dma sync should not be done while in prom or polled mode.
490 	 */
491 	if (ddi_prop_exists(DDI_DEV_T_ANY, ehcip->ehci_dip,
492 	    DDI_PROP_NOTPROM, "no-prom-cdma-sync")) {
493 		ehci_polledp->ehci_polled_no_sync_flag = B_TRUE;
494 	}
495 
496 	/* Allow interrupts to continue */
497 	mutex_exit(&ehcip->ehci_int_mutex);
498 
499 	return (USB_SUCCESS);
500 }
501 
502 
503 /*
504  * ehci_hcdi_polled_output_fini:
505  */
506 int
ehci_hcdi_polled_output_fini(usb_console_info_impl_t * info)507 ehci_hcdi_polled_output_fini(usb_console_info_impl_t *info)
508 {
509 	ehci_polled_t		*ehci_polledp;
510 	ehci_state_t		*ehcip;
511 	ehci_pipe_private_t	*pp;
512 	int			ret;
513 
514 	ehci_polledp = (ehci_polled_t *)info->uci_private;
515 
516 	ehcip = ehci_polledp->ehci_polled_ehcip;
517 
518 	mutex_enter(&ehcip->ehci_int_mutex);
519 
520 	/* Remove the Endpoint Descriptor. */
521 	pp = (ehci_pipe_private_t *)ehci_polledp->
522 	    ehci_polled_input_pipe_handle->p_hcd_private;
523 	ehci_polled_remove_async_qh(ehcip, pp);
524 
525 	/*
526 	 * Reset the POLLED_INPUT_MODE flag so that we can tell if
527 	 * this structure is in use in the ehci_polled_fini routine.
528 	 */
529 	ehci_polledp->ehci_polled_flags &= ~POLLED_OUTPUT_MODE;
530 
531 	ret = ehci_polled_fini(ehci_polledp);
532 
533 	info->uci_private = NULL;
534 
535 	mutex_exit(&ehcip->ehci_int_mutex);
536 
537 	return (ret);
538 }
539 
540 
541 /*
542  * ehci_hcdi_polled_output_enter:
543  *
544  * everything is done in input enter
545  */
546 /*ARGSUSED*/
547 int
ehci_hcdi_polled_output_enter(usb_console_info_impl_t * info)548 ehci_hcdi_polled_output_enter(usb_console_info_impl_t *info)
549 {
550 	return (USB_SUCCESS);
551 }
552 
553 
554 /*
555  * ehci_hcdi_polled_output_exit:
556  *
557  * everything is done in input exit
558  */
559 /*ARGSUSED*/
560 int
ehci_hcdi_polled_output_exit(usb_console_info_impl_t * info)561 ehci_hcdi_polled_output_exit(usb_console_info_impl_t *info)
562 {
563 	return (USB_SUCCESS);
564 }
565 
566 
567 /*
568  * ehci_hcdi_polled_write:
569  *	Put a key character.
570  */
571 int
ehci_hcdi_polled_write(usb_console_info_impl_t * info,uchar_t * buf,uint_t num_characters,uint_t * num_characters_written)572 ehci_hcdi_polled_write(usb_console_info_impl_t *info, uchar_t *buf,
573     uint_t num_characters, uint_t *num_characters_written)
574 {
575 	ehci_state_t		*ehcip;
576 	ehci_polled_t		*ehci_polledp;
577 	ehci_trans_wrapper_t	*tw;
578 	ehci_pipe_private_t	*pp;
579 	usba_pipe_handle_data_t	*ph;
580 	int			intr;
581 
582 #ifndef lint
583 	_NOTE(NO_COMPETING_THREADS_NOW);
584 #endif
585 	ehci_polledp = (ehci_polled_t *)info->uci_private;
586 	ehcip = ehci_polledp->ehci_polled_ehcip;
587 	ph = ehci_polledp->ehci_polled_input_pipe_handle;
588 	pp = (ehci_pipe_private_t *)ph->p_hcd_private;
589 
590 	/* Disable all list processing */
591 	Set_OpReg(ehci_command, Get_OpReg(ehci_command) &
592 	    ~(EHCI_CMD_ASYNC_SCHED_ENABLE |
593 	    EHCI_CMD_PERIODIC_SCHED_ENABLE));
594 
595 	/* Wait for few milliseconds */
596 	drv_usecwait(EHCI_POLLED_TIMEWAIT);
597 
598 	tw = pp->pp_tw_head;
599 	ASSERT(tw != NULL);
600 
601 	/* copy transmit buffer */
602 	if (num_characters > POLLED_RAW_BUF_SIZE) {
603 		cmn_err(CE_NOTE, "polled write size %d bigger than %d",
604 		    num_characters, POLLED_RAW_BUF_SIZE);
605 		num_characters = POLLED_RAW_BUF_SIZE;
606 	}
607 	tw->tw_length = num_characters;
608 	ddi_rep_put8(tw->tw_accesshandle,
609 	    buf, (uint8_t *)tw->tw_buf,
610 	    tw->tw_length, DDI_DEV_AUTOINCR);
611 	Sync_IO_Buffer_for_device(tw->tw_dmahandle, tw->tw_length);
612 
613 	ehci_polled_insert_bulk_qtd(ehci_polledp);
614 
615 	/* Enable async list processing */
616 	Set_OpReg(ehci_command, (Get_OpReg(ehci_command) |
617 	    EHCI_CMD_ASYNC_SCHED_ENABLE));
618 
619 	/* Wait for few milliseconds */
620 	drv_usecwait(EHCI_POLLED_TIMEWAIT);
621 
622 	while (!((Get_OpReg(ehci_status)) & (EHCI_INTR_USB
623 	    |EHCI_INTR_FRAME_LIST_ROLLOVER | EHCI_INTR_USB_ERROR))) {
624 #ifndef __sparc
625 		invalidate_cache();
626 #else
627 		;
628 #endif
629 	}
630 
631 	intr = (Get_OpReg(ehci_status)) &
632 	    (EHCI_INTR_FRAME_LIST_ROLLOVER |
633 	    EHCI_INTR_USB | EHCI_INTR_USB_ERROR);
634 
635 	/*
636 	 * Check whether any frame list rollover interrupt is pending
637 	 * and if it is pending, process this interrupt.
638 	 */
639 	if (intr & EHCI_INTR_FRAME_LIST_ROLLOVER) {
640 
641 		ehci_handle_frame_list_rollover(ehcip);
642 		ehci_polled_finish_interrupt(ehcip,
643 		    EHCI_INTR_FRAME_LIST_ROLLOVER);
644 	}
645 
646 	/* Check for any USB transaction completion notification */
647 	if (intr & (EHCI_INTR_USB | EHCI_INTR_USB_ERROR)) {
648 
649 		(void) ehci_polled_process_active_intr_qtd_list(ehci_polledp);
650 
651 		/* Acknowledge the USB and USB error interrupt */
652 		ehci_polled_finish_interrupt(ehcip,
653 		    intr & (EHCI_INTR_USB | EHCI_INTR_USB_ERROR));
654 
655 	}
656 
657 	*num_characters_written = num_characters;
658 
659 #ifndef lint
660 	_NOTE(COMPETING_THREADS_NOW);
661 #endif
662 
663 	return (USB_SUCCESS);
664 }
665 
666 
667 /*
668  * Internal Functions
669  */
670 
671 /*
672  * Polled initialization routines
673  */
674 
675 
676 /*
677  * ehci_polled_init:
678  *
679  * Initialize generic information Uthat is needed to provide USB/POLLED
680  * support.
681  */
682 static int
ehci_polled_init(usba_pipe_handle_data_t * ph,ehci_state_t * ehcip,usb_console_info_impl_t * console_info)683 ehci_polled_init(
684 	usba_pipe_handle_data_t	*ph,
685 	ehci_state_t		*ehcip,
686 	usb_console_info_impl_t	*console_info)
687 {
688 	ehci_polled_t		*ehci_polledp;
689 	ehci_pipe_private_t	*pp;
690 	ehci_qtd_t		*qtd;
691 	int			pipe_attr;
692 
693 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
694 
695 	/*
696 	 * We have already initialized this structure. If the structure
697 	 * has already been initialized, then we don't need to redo it.
698 	 */
699 	if (console_info->uci_private) {
700 
701 		return (USB_SUCCESS);
702 	}
703 
704 	/* Allocate and intitialize a state structure */
705 	ehci_polledp = (ehci_polled_t *)
706 	    kmem_zalloc(sizeof (ehci_polled_t), KM_SLEEP);
707 
708 	console_info->uci_private = (usb_console_info_private_t)ehci_polledp;
709 
710 	/*
711 	 * Store away the ehcip so that we can get to it when we are in
712 	 * POLLED mode. We don't want to have to call ehci_obtain_state
713 	 * every time we want to access this structure.
714 	 */
715 	ehci_polledp->ehci_polled_ehcip = ehcip;
716 	/*
717 	 * Save usb device and endpoint number information from the usb
718 	 * pipe handle.
719 	 */
720 	mutex_enter(&ph->p_mutex);
721 	ehci_polledp->ehci_polled_usb_dev = ph->p_usba_device;
722 	ehci_polledp->ehci_polled_ep_addr = ph->p_ep.bEndpointAddress;
723 	mutex_exit(&ph->p_mutex);
724 
725 	/*
726 	 * Allocate memory to make duplicate of original usb pipe handle.
727 	 */
728 	ehci_polledp->ehci_polled_input_pipe_handle =
729 	    kmem_zalloc(sizeof (usba_pipe_handle_data_t), KM_SLEEP);
730 
731 	/*
732 	 * Copy the USB handle into the new pipe handle. Also
733 	 * create new lock for the new pipe handle.
734 	 */
735 	bcopy((void *)ph,
736 	    (void *)ehci_polledp->ehci_polled_input_pipe_handle,
737 	    sizeof (usba_pipe_handle_data_t));
738 
739 	/*
740 	 * uint64_t typecast to make sure amd64 can compile
741 	 */
742 	mutex_init(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex,
743 	    NULL, MUTEX_DRIVER, DDI_INTR_PRI(ehcip->ehci_intr_pri));
744 
745 	/*
746 	 * Create a new ehci pipe private structure
747 	 */
748 	pp = (ehci_pipe_private_t *)
749 	    kmem_zalloc(sizeof (ehci_pipe_private_t), KM_SLEEP);
750 
751 	/*
752 	 * Store the pointer in the pipe handle. This structure was also
753 	 * just allocated.
754 	 */
755 	mutex_enter(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex);
756 
757 	ehci_polledp->ehci_polled_input_pipe_handle->
758 	    p_hcd_private = (usb_opaque_t)pp;
759 
760 	mutex_exit(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex);
761 
762 	/*
763 	 * Store a pointer to the pipe handle. This structure was  just
764 	 * allocated and it is not in use yet.	The locking is there to
765 	 * satisfy warlock.
766 	 */
767 	mutex_enter(&ph->p_mutex);
768 
769 	bcopy(&ph->p_policy, &pp->pp_policy, sizeof (usb_pipe_policy_t));
770 
771 	mutex_exit(&ph->p_mutex);
772 
773 	pp->pp_pipe_handle = ehci_polledp->ehci_polled_input_pipe_handle;
774 
775 	/*
776 	 * Allocate a dummy for the interrupt table. This dummy will be
777 	 * put into the action when we	switch interrupt  tables during
778 	 * ehci_hcdi_polled_enter. Dummy is placed on the unused lattice
779 	 * entries. When the QH is allocated we will replace dummy QH by
780 	 * valid interrupt QH in one or more locations in the interrupt
781 	 * lattice depending on the requested polling interval. Also we
782 	 * will hang a dummy QTD to the QH & dummy QTD is used to indicate
783 	 * the end of the QTD chain.
784 	 */
785 	ehci_polledp->ehci_polled_dummy_qh =
786 	    ehci_alloc_qh(ehcip, NULL, EHCI_POLLED_MODE_FLAG);
787 
788 	if (ehci_polledp->ehci_polled_dummy_qh == NULL) {
789 
790 		return (USB_NO_RESOURCES);
791 	}
792 
793 	/*
794 	 * Allocate the endpoint. This QH will be inserted in
795 	 * to the lattice chain for the device. This endpoint
796 	 * will have the QTDs hanging off of it for the processing.
797 	 */
798 	ehci_polledp->ehci_polled_qh = ehci_alloc_qh(
799 	    ehcip, ph, EHCI_POLLED_MODE_FLAG);
800 
801 	if (ehci_polledp->ehci_polled_qh == NULL) {
802 
803 		return (USB_NO_RESOURCES);
804 	}
805 
806 	/* Set the state of pipe as idle */
807 	pp->pp_state = EHCI_PIPE_STATE_IDLE;
808 
809 	/* Set polled mode flag */
810 	pp->pp_flag = EHCI_POLLED_MODE_FLAG;
811 
812 	/* Insert the endpoint onto the pipe handle */
813 	pp->pp_qh = ehci_polledp->ehci_polled_qh;
814 
815 	pipe_attr = ph->p_ep.bmAttributes & USB_EP_ATTR_MASK;
816 
817 	switch (pipe_attr) {
818 	case USB_EP_ATTR_INTR:
819 		/*
820 		 * Set soft interrupt handler flag in the normal mode usb
821 		 * pipe handle.
822 		 */
823 		mutex_enter(&ph->p_mutex);
824 		ph->p_spec_flag |= USBA_PH_FLAG_USE_SOFT_INTR;
825 		mutex_exit(&ph->p_mutex);
826 
827 		/*
828 		 * Insert a Interrupt polling request onto the endpoint.
829 		 *
830 		 * There will now be two QTDs on the QH, one is the dummy QTD
831 		 * that was allocated above in the  ehci_alloc_qh and this
832 		 * new one.
833 		 */
834 		if ((ehci_start_periodic_pipe_polling(ehcip,
835 		    ehci_polledp->ehci_polled_input_pipe_handle,
836 		    NULL, USB_FLAGS_SLEEP)) != USB_SUCCESS) {
837 
838 			return (USB_NO_RESOURCES);
839 		}
840 		/* Get the given new interrupt qtd */
841 		qtd = (ehci_qtd_t *)(ehci_qtd_iommu_to_cpu(ehcip,
842 		    (Get_QH(pp->pp_qh->qh_next_qtd) & EHCI_QH_NEXT_QTD_PTR)));
843 
844 		/* Insert this qtd into active interrupt QTD list */
845 		ehci_polled_insert_qtd_into_active_intr_qtd_list(ehci_polledp,
846 		    qtd);
847 		break;
848 	case USB_EP_ATTR_BULK:
849 		if ((ehci_polled_create_tw(ehci_polledp,
850 		    ehci_polledp->ehci_polled_input_pipe_handle,
851 		    USB_FLAGS_SLEEP)) != USB_SUCCESS) {
852 
853 			return (USB_NO_RESOURCES);
854 		}
855 		break;
856 	default:
857 		return (USB_FAILURE);
858 	}
859 
860 	return (USB_SUCCESS);
861 }
862 
863 
864 /*
865  * Polled deinitialization routines
866  */
867 
868 
869 /*
870  * ehci_polled_fini:
871  */
872 static int
ehci_polled_fini(ehci_polled_t * ehci_polledp)873 ehci_polled_fini(ehci_polled_t	*ehci_polledp)
874 {
875 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
876 	ehci_pipe_private_t	*pp;
877 
878 	ASSERT(mutex_owned(&ehcip->ehci_int_mutex));
879 
880 	/* If the structure is already in use, then don't free it */
881 	if (ehci_polledp->ehci_polled_flags & POLLED_INPUT_MODE) {
882 
883 		return (USB_SUCCESS);
884 	}
885 
886 	pp = (ehci_pipe_private_t *)
887 	    ehci_polledp->ehci_polled_input_pipe_handle->p_hcd_private;
888 
889 	/* Deallocate all the pre-allocated interrupt requests */
890 	ehci_handle_outstanding_requests(ehcip, pp);
891 
892 	/*
893 	 * Traverse the list of QTD's on this endpoint and these QTD's
894 	 * have outstanding transfer requests. Since list processing
895 	 * is stopped, these QTDs can be deallocated.
896 	 */
897 	ehci_polled_traverse_qtds(ehci_polledp, pp->pp_pipe_handle);
898 
899 	/* Free DMA resources */
900 	ehci_free_dma_resources(ehcip, pp->pp_pipe_handle);
901 
902 	/*
903 	 * Deallocate the endpoint descriptors that we allocated
904 	 * with ehci_alloc_qh.
905 	 */
906 	if (ehci_polledp->ehci_polled_dummy_qh) {
907 		ehci_deallocate_qh(ehcip, ehci_polledp->ehci_polled_dummy_qh);
908 	}
909 
910 	if (ehci_polledp->ehci_polled_qh) {
911 		ehci_deallocate_qh(ehcip, ehci_polledp->ehci_polled_qh);
912 	}
913 
914 	mutex_destroy(&ehci_polledp->ehci_polled_input_pipe_handle->p_mutex);
915 
916 	/*
917 	 * Destroy everything about the pipe that we allocated in
918 	 * ehci_polled_duplicate_pipe_handle
919 	 */
920 	kmem_free(pp, sizeof (ehci_pipe_private_t));
921 
922 	kmem_free(ehci_polledp->ehci_polled_input_pipe_handle,
923 	    sizeof (usba_pipe_handle_data_t));
924 
925 	/*
926 	 * We use this field to determine if a QTD is for input or not,
927 	 * so NULL the pointer so we don't check deallocated data.
928 	 */
929 	ehci_polledp->ehci_polled_input_pipe_handle = NULL;
930 
931 	/*
932 	 * Finally, free off the structure that we use to keep track
933 	 * of all this.
934 	 */
935 	kmem_free(ehci_polledp, sizeof (ehci_polled_t));
936 
937 	return (USB_SUCCESS);
938 }
939 
940 
941 /*
942  * Polled save state routines
943  */
944 
945 
946 /*
947  * ehci_polled_save_state:
948  */
949 static void
ehci_polled_save_state(ehci_polled_t * ehci_polledp)950 ehci_polled_save_state(ehci_polled_t	*ehci_polledp)
951 {
952 	int				i;
953 	ehci_state_t			*ehcip;
954 	uint_t				polled_toggle;
955 	uint_t				real_toggle;
956 	ehci_pipe_private_t		*pp = NULL; /* Normal mode Pipe */
957 	ehci_pipe_private_t		*polled_pp; /* Polled mode Pipe */
958 	usba_pipe_handle_data_t		*ph;
959 	uint8_t				ep_addr;
960 	ehci_regs_t			*ehci_polled_regsp;
961 	ehci_qh_t			*qh;
962 
963 #ifndef lint
964 	_NOTE(NO_COMPETING_THREADS_NOW);
965 #endif
966 
967 	/*
968 	 * If either of these two flags are set, then we have already
969 	 * saved off the state information and setup the controller.
970 	 */
971 	if (ehci_polledp->ehci_polled_flags & POLLED_INPUT_MODE_INUSE) {
972 #ifndef lint
973 		_NOTE(COMPETING_THREADS_NOW);
974 #endif
975 		return;
976 	}
977 
978 	ehcip = ehci_polledp->ehci_polled_ehcip;
979 
980 	/*
981 	 * Check if the number of keyboard reach the max number we can
982 	 * support in polled mode
983 	 */
984 	if (++ ehcip->ehci_polled_enter_count > MAX_NUM_FOR_KEYBOARD) {
985 #ifndef lint
986 		_NOTE(COMPETING_THREADS_NOW);
987 #endif
988 		return;
989 	}
990 	ehci_polled_regsp = &ehcip->ehci_polled_save_regs;
991 
992 	/* Get the endpoint addr. */
993 	ep_addr = ehci_polledp->ehci_polled_ep_addr;
994 
995 	/* Get the normal mode usb pipe handle */
996 	ph = usba_hcdi_get_ph_data(ehci_polledp->ehci_polled_usb_dev, ep_addr);
997 
998 	/*
999 	 * The first enter keyboard entry should save info of the normal mode,
1000 	 * disable all list processing and interrupt, initialize the
1001 	 * frame list table with dummy QHs.
1002 	 */
1003 	if (ehcip->ehci_polled_enter_count == 1) {
1004 		/*
1005 		 * Save the current normal mode ehci registers	and later this
1006 		 * saved register copy is used to replace some of required ehci
1007 		 * registers before switching from polled mode to normal mode.
1008 		 */
1009 
1010 		bzero((void *)ehci_polled_regsp, sizeof (ehci_regs_t));
1011 
1012 		/* Save current ehci registers */
1013 		ehci_polled_regsp->ehci_command = Get_OpReg(ehci_command);
1014 		ehci_polled_regsp->ehci_interrupt = Get_OpReg(ehci_interrupt);
1015 		ehci_polled_regsp->ehci_ctrl_segment =
1016 		    Get_OpReg(ehci_ctrl_segment);
1017 		ehci_polled_regsp->
1018 		    ehci_async_list_addr = Get_OpReg(ehci_async_list_addr);
1019 		ehci_polled_regsp->ehci_config_flag =
1020 		    Get_OpReg(ehci_config_flag);
1021 		ehci_polled_regsp->ehci_periodic_list_base =
1022 		    Get_OpReg(ehci_periodic_list_base);
1023 
1024 		/* Disable all list processing and interrupts */
1025 		Set_OpReg(ehci_command, Get_OpReg(ehci_command) &
1026 		    ~(EHCI_CMD_ASYNC_SCHED_ENABLE |
1027 		    EHCI_CMD_PERIODIC_SCHED_ENABLE));
1028 
1029 		/* Wait for few milliseconds */
1030 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
1031 
1032 		/* Save any unprocessed normal mode ehci interrupts */
1033 		ehcip->ehci_missed_intr_sts = EHCI_INTR_USB;
1034 
1035 		/*
1036 		 * Save the current interrupt lattice and  replace this lattice
1037 		 * with an lattice used in POLLED mode. We will restore lattice
1038 		 * back when we exit from the POLLED mode.
1039 		 */
1040 		for (i = 0; i < EHCI_NUM_PERIODIC_FRAME_LISTS; i++) {
1041 			ehcip->ehci_polled_frame_list_table[i] =
1042 			    (ehci_qh_t *)(uintptr_t)Get_PFLT(ehcip->
1043 			    ehci_periodic_frame_list_tablep->
1044 			    ehci_periodic_frame_list_table[i]);
1045 		}
1046 
1047 		/*
1048 		 * Fill in the lattice with dummy QHs. These QHs are used so the
1049 		 * controller can tell that it is at the end of the QH list.
1050 		 */
1051 		for (i = 0; i < EHCI_NUM_PERIODIC_FRAME_LISTS; i++) {
1052 			Set_PFLT(ehcip->ehci_periodic_frame_list_tablep->
1053 			    ehci_periodic_frame_list_table[i],
1054 			    ehci_qh_cpu_to_iommu(ehcip,
1055 			    ehci_polledp->ehci_polled_dummy_qh) |
1056 			    (EHCI_QH_LINK_REF_QH | EHCI_QH_LINK_PTR_VALID));
1057 		}
1058 
1059 	}
1060 
1061 	/* Get the polled mode ehci pipe private structure */
1062 	polled_pp = (ehci_pipe_private_t *)
1063 	    ehci_polledp->ehci_polled_input_pipe_handle->p_hcd_private;
1064 
1065 	/*
1066 	 * Before replacing the lattice, adjust the data togggle on the
1067 	 * on the ehci's interrupt ed
1068 	 */
1069 	polled_toggle = (Get_QH(polled_pp->pp_qh->qh_status) &
1070 	    EHCI_QH_STS_DATA_TOGGLE) ? DATA1:DATA0;
1071 
1072 	/*
1073 	 * If normal mode interrupt pipe endpoint is active, get the data
1074 	 * toggle from the this interrupt endpoint through the corresponding
1075 	 * interrupt pipe handle. Else get the data toggle information from
1076 	 * the usb device structure and this information is saved during the
1077 	 * normal mode interrupt pipe close. Use this data toggle information
1078 	 * to fix the data toggle of polled mode interrupt endpoint.
1079 	 */
1080 	if (ph) {
1081 		/* Get the normal mode ehci pipe private structure */
1082 		pp = (ehci_pipe_private_t *)ph->p_hcd_private;
1083 
1084 		real_toggle = (Get_QH(pp->pp_qh->qh_status) &
1085 		    EHCI_QH_STS_DATA_TOGGLE) ? DATA1:DATA0;
1086 	} else {
1087 		real_toggle = usba_hcdi_get_data_toggle(
1088 		    ehci_polledp->ehci_polled_usb_dev, ep_addr);
1089 	}
1090 
1091 	if (polled_toggle != real_toggle) {
1092 		if (real_toggle == DATA0) {
1093 			Set_QH(polled_pp->pp_qh->qh_status,
1094 			    Get_QH(polled_pp->pp_qh->qh_status) &
1095 			    ~EHCI_QH_STS_DATA_TOGGLE);
1096 		} else {
1097 			Set_QH(polled_pp->pp_qh->qh_status,
1098 			    Get_QH(polled_pp->pp_qh->qh_status) |
1099 			    EHCI_QH_STS_DATA_TOGGLE);
1100 		}
1101 	}
1102 
1103 	/*
1104 	 * Check whether Halt bit is set in the QH and if so  clear the
1105 	 * halt bit.
1106 	 */
1107 	if (polled_pp->pp_qh->qh_status & EHCI_QH_STS_HALTED) {
1108 
1109 		/* Clear the halt bit */
1110 		Set_QH(polled_pp->pp_qh->qh_status,
1111 		    (Get_QH(polled_pp->pp_qh->qh_status) &
1112 		    ~EHCI_QH_STS_HALTED));
1113 	}
1114 
1115 	/*
1116 	 * Initialize the qh overlay area
1117 	 */
1118 	qh = ehci_polledp->ehci_polled_qh;
1119 	for (i = 0; i < 5; i++) {
1120 		Set_QH(qh->qh_buf[i], 0);
1121 		Set_QH(qh->qh_buf_high[i], 0);
1122 	}
1123 	Set_QH(qh->qh_next_qtd, ehci_qtd_cpu_to_iommu(ehcip,
1124 	    ehci_polledp->ehci_polled_active_intr_qtd_list));
1125 
1126 	/*
1127 	 * Now, add the endpoint to the lattice that we will  hang  our
1128 	 * QTD's off of.  We need to poll this device at  every 8 ms and
1129 	 * hence add this QH needs 4 entries in interrupt lattice.
1130 	 */
1131 	for (i = ehcip->ehci_polled_enter_count - 1;
1132 	    i < EHCI_NUM_PERIODIC_FRAME_LISTS;
1133 	    i = i + LS_MIN_POLL_INTERVAL) {
1134 		Set_PFLT(ehcip->ehci_periodic_frame_list_tablep->
1135 		    ehci_periodic_frame_list_table[i],
1136 		    ehci_qh_cpu_to_iommu(ehcip,
1137 		    ehci_polledp->ehci_polled_qh) | EHCI_QH_LINK_REF_QH);
1138 	}
1139 	/* The first enter keyboard entry enable interrupts and periodic list */
1140 	if (ehcip->ehci_polled_enter_count == 1) {
1141 		/* Enable USB and Frame list rollover interrupts */
1142 		Set_OpReg(ehci_interrupt, (EHCI_INTR_USB |
1143 		    EHCI_INTR_USB_ERROR | EHCI_INTR_FRAME_LIST_ROLLOVER));
1144 
1145 		/* Enable the periodic list */
1146 		Set_OpReg(ehci_command,
1147 		    (Get_OpReg(ehci_command) | EHCI_CMD_PERIODIC_SCHED_ENABLE));
1148 
1149 		/* Wait for few milliseconds */
1150 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
1151 	}
1152 #ifndef lint
1153 	_NOTE(COMPETING_THREADS_NOW);
1154 #endif
1155 }
1156 
1157 
1158 /*
1159  * Polled restore state routines
1160  */
1161 
1162 
1163 /*
1164  * ehci_polled_restore_state:
1165  */
1166 static void
ehci_polled_restore_state(ehci_polled_t * ehci_polledp)1167 ehci_polled_restore_state(ehci_polled_t	*ehci_polledp)
1168 {
1169 	ehci_state_t			*ehcip;
1170 	int				i;
1171 	uint_t				polled_toggle;
1172 	uint_t				real_toggle;
1173 	ehci_pipe_private_t		*pp = NULL; /* Normal mode Pipe */
1174 	ehci_pipe_private_t		*polled_pp; /* Polled mode Pipe */
1175 	usba_pipe_handle_data_t		*ph;
1176 	uint8_t				ep_addr;
1177 
1178 #ifndef lint
1179 	_NOTE(NO_COMPETING_THREADS_NOW);
1180 #endif
1181 
1182 	/*
1183 	 * If this flag is set, then we are still using this structure,
1184 	 * so don't restore any controller state information yet.
1185 	 */
1186 	if (ehci_polledp->ehci_polled_flags & POLLED_INPUT_MODE_INUSE) {
1187 
1188 #ifndef lint
1189 		_NOTE(COMPETING_THREADS_NOW);
1190 #endif
1191 
1192 		return;
1193 	}
1194 
1195 	ehcip = ehci_polledp->ehci_polled_ehcip;
1196 	ehcip->ehci_polled_enter_count --;
1197 
1198 	/* Get the endpoint addr */
1199 	ep_addr = ehci_polledp->ehci_polled_ep_addr;
1200 
1201 	/* Get the normal mode usb pipe handle */
1202 	ph = usba_hcdi_get_ph_data(ehci_polledp->ehci_polled_usb_dev, ep_addr);
1203 
1204 	/* Disable list processing and other things */
1205 	ehci_polled_stop_processing(ehci_polledp);
1206 
1207 	/* Get the polled mode ehci pipe private structure */
1208 	polled_pp = (ehci_pipe_private_t *)
1209 	    ehci_polledp->ehci_polled_input_pipe_handle->p_hcd_private;
1210 
1211 	/*
1212 	 * Before replacing the lattice, adjust the data togggle
1213 	 * on the on the ehci's interrupt ed
1214 	 */
1215 	polled_toggle = (Get_QH(polled_pp->pp_qh->qh_status) &
1216 	    EHCI_QH_STS_DATA_TOGGLE) ? DATA1:DATA0;
1217 
1218 	/*
1219 	 * If normal mode interrupt pipe endpoint is active, fix the
1220 	 * data toggle for this interrupt endpoint by getting the data
1221 	 * toggle information from the polled interrupt endpoint. Else
1222 	 * save the data toggle information in usb device structure.
1223 	 */
1224 	if (ph) {
1225 		/* Get the normal mode ehci pipe private structure */
1226 		pp = (ehci_pipe_private_t *)ph->p_hcd_private;
1227 
1228 		real_toggle = (Get_QH(pp->pp_qh->qh_status) &
1229 		    EHCI_QH_STS_DATA_TOGGLE) ? DATA1:DATA0;
1230 
1231 		if (polled_toggle != real_toggle) {
1232 			if (polled_toggle == DATA0) {
1233 				Set_QH(pp->pp_qh->qh_status,
1234 				    Get_QH(pp->pp_qh->qh_status) &
1235 				    ~EHCI_QH_STS_DATA_TOGGLE);
1236 			} else {
1237 				Set_QH(pp->pp_qh->qh_status,
1238 				    Get_QH(pp->pp_qh->qh_status) |
1239 				    EHCI_QH_STS_DATA_TOGGLE);
1240 			}
1241 		}
1242 	} else {
1243 		usba_hcdi_set_data_toggle(ehci_polledp->ehci_polled_usb_dev,
1244 		    ep_addr, polled_toggle);
1245 	}
1246 
1247 	/*
1248 	 * Only the last leave keyboard entry restore the save frame
1249 	 * list table and start processing.
1250 	 */
1251 	if (ehcip->ehci_polled_enter_count == 0) {
1252 
1253 		/* Replace the lattice */
1254 		for (i = 0; i < EHCI_NUM_PERIODIC_FRAME_LISTS; i++) {
1255 			Set_PFLT(ehcip->ehci_periodic_frame_list_tablep->
1256 			    ehci_periodic_frame_list_table[i],
1257 			    ehcip->ehci_polled_frame_list_table[i]);
1258 		}
1259 		ehci_polled_start_processing(ehci_polledp);
1260 	}
1261 
1262 #ifndef lint
1263 	_NOTE(COMPETING_THREADS_NOW);
1264 #endif
1265 }
1266 
1267 
1268 /*
1269  * ehci_polled_stop_processing:
1270  */
1271 static void
ehci_polled_stop_processing(ehci_polled_t * ehci_polledp)1272 ehci_polled_stop_processing(ehci_polled_t	*ehci_polledp)
1273 {
1274 	ehci_state_t		*ehcip;
1275 	ehci_qh_t		*qh = ehci_polledp->ehci_polled_qh;
1276 
1277 	ehcip = ehci_polledp->ehci_polled_ehcip;
1278 
1279 	/* First inactive this QH */
1280 	Set_QH(qh->qh_ctrl,
1281 	    Get_QH(qh->qh_ctrl) | EHCI_QH_CTRL_ED_INACTIVATE);
1282 
1283 	/* Only first leave keyboard entry turn off periodic list processing */
1284 	if (Get_OpReg(ehci_command) & EHCI_CMD_PERIODIC_SCHED_ENABLE) {
1285 		Set_OpReg(ehci_command, (Get_OpReg(ehci_command) &
1286 		    ~EHCI_CMD_PERIODIC_SCHED_ENABLE));
1287 
1288 		/* Wait for few milliseconds */
1289 		drv_usecwait(EHCI_POLLED_TIMEWAIT);
1290 	}
1291 	/*
1292 	 * Now clear all required fields of QH
1293 	 * including inactive bit.
1294 	 */
1295 	Set_QH(qh->qh_ctrl,
1296 	    Get_QH(qh->qh_ctrl) & ~(EHCI_QH_CTRL_ED_INACTIVATE));
1297 	Set_QH(qh->qh_status,
1298 	    Get_QH(qh->qh_status) & ~(EHCI_QH_STS_XACT_STATUS));
1299 	Set_QH(qh->qh_curr_qtd, 0);
1300 	Set_QH(qh->qh_alt_next_qtd, EHCI_QH_ALT_NEXT_QTD_PTR_VALID);
1301 
1302 	/*
1303 	 * Now look up at the QTD's that are in the active qtd list &
1304 	 * re-insert them back into the QH's QTD list.
1305 	 */
1306 	(void) ehci_polled_process_active_intr_qtd_list(ehci_polledp);
1307 }
1308 
1309 
1310 /*
1311  * ehci_polled_start_processing:
1312  */
1313 static void
ehci_polled_start_processing(ehci_polled_t * ehci_polledp)1314 ehci_polled_start_processing(ehci_polled_t	*ehci_polledp)
1315 {
1316 	ehci_state_t		*ehcip;
1317 	uint32_t		mask;
1318 	ehci_regs_t		*ehci_polled_regsp;
1319 
1320 	ehcip = ehci_polledp->ehci_polled_ehcip;
1321 	ehci_polled_regsp = &ehcip->ehci_polled_save_regs;
1322 
1323 	mask = ((uint32_t)ehci_polled_regsp->ehci_interrupt &
1324 	    (EHCI_INTR_HOST_SYSTEM_ERROR | EHCI_INTR_FRAME_LIST_ROLLOVER |
1325 	    EHCI_INTR_USB_ERROR | EHCI_INTR_USB | EHCI_INTR_ASYNC_ADVANCE));
1326 
1327 	/* Enable all required EHCI interrupts */
1328 	Set_OpReg(ehci_interrupt, mask);
1329 
1330 	mask = ((uint32_t)ehci_polled_regsp->ehci_command &
1331 	    (EHCI_CMD_ASYNC_SCHED_ENABLE | EHCI_CMD_PERIODIC_SCHED_ENABLE));
1332 
1333 	/* Enable all reuired list processing */
1334 	Set_OpReg(ehci_command, (Get_OpReg(ehci_command) | mask));
1335 
1336 	/* Wait for few milliseconds */
1337 	drv_usecwait(EHCI_POLLED_TIMEWAIT);
1338 }
1339 
1340 
1341 /*
1342  * Polled read routines
1343  */
1344 
1345 
1346 /*
1347  * ehci_polled_process_active_intr_qtd_list:
1348  *
1349  * This routine takes the QTD's off of the input done head and processes
1350  * them.  It returns the number of characters that have been copied for
1351  * input.
1352  */
1353 static int
ehci_polled_process_active_intr_qtd_list(ehci_polled_t * ehci_polledp)1354 ehci_polled_process_active_intr_qtd_list(ehci_polled_t	*ehci_polledp)
1355 {
1356 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1357 	ehci_qtd_t		*qtd, *next_qtd;
1358 	uint_t			num_characters = 0;
1359 	uint_t			ctrl;
1360 	ehci_trans_wrapper_t	*tw;
1361 	ehci_pipe_private_t	*pp;
1362 	usb_cr_t		error;
1363 	int			pipe_attr, pipe_dir;
1364 
1365 	/* Sync QH and QTD pool */
1366 	if (ehci_polledp->ehci_polled_no_sync_flag == B_FALSE) {
1367 		Sync_QH_QTD_Pool(ehcip);
1368 	}
1369 
1370 	/* Create done qtd list */
1371 	qtd = ehci_polled_create_done_qtd_list(ehci_polledp);
1372 
1373 	pipe_attr = ehci_polledp->ehci_polled_input_pipe_handle->
1374 	    p_ep.bmAttributes & USB_EP_ATTR_MASK;
1375 	pipe_dir = ehci_polledp->ehci_polled_input_pipe_handle->
1376 	    p_ep.bEndpointAddress & USB_EP_DIR_MASK;
1377 	/*
1378 	 * Traverse the list of transfer descriptors.  We can't destroy
1379 	 * the qtd_next pointers of these QTDs because we are using it
1380 	 * to traverse the done list.  Therefore, we can not put these
1381 	 * QTD's back on the QH until we are done processing all of them.
1382 	 */
1383 	while (qtd) {
1384 		/* Get next active QTD from the active QTD list */
1385 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1386 		    Get_QTD(qtd->qtd_active_qtd_next));
1387 
1388 		/* Obtain the transfer wrapper from the QTD */
1389 		tw = (ehci_trans_wrapper_t *)EHCI_LOOKUP_ID(
1390 		    (uint32_t)Get_QTD(qtd->qtd_trans_wrapper));
1391 
1392 		/* Get ehci pipe from transfer wrapper */
1393 		pp = tw->tw_pipe_private;
1394 
1395 		/* Look at the status */
1396 		ctrl = (uint_t)Get_QTD(qtd->qtd_ctrl) &
1397 		    (uint32_t)EHCI_QTD_CTRL_XACT_STATUS;
1398 
1399 		error = ehci_check_for_error(ehcip, pp, tw, qtd, ctrl);
1400 
1401 		/*
1402 		 * Check to see if there is an error. If there is error
1403 		 * clear the halt condition in the Endpoint  Descriptor
1404 		 * (QH) associated with this Transfer  Descriptor (QTD).
1405 		 */
1406 		if (error != USB_CR_OK) {
1407 			/* Clear the halt bit */
1408 			Set_QH(pp->pp_qh->qh_status,
1409 			    Get_QH(pp->pp_qh->qh_status) &
1410 			    ~(EHCI_QH_STS_XACT_STATUS));
1411 		} else if (pipe_dir == USB_EP_DIR_IN) {
1412 
1413 			num_characters +=
1414 			    ehci_polled_handle_normal_qtd(ehci_polledp,
1415 			    qtd);
1416 		}
1417 
1418 		/* Insert this qtd back into QH's qtd list */
1419 		switch (pipe_attr) {
1420 		case USB_EP_ATTR_INTR:
1421 			ehci_polled_insert_intr_qtd(ehci_polledp, qtd);
1422 			break;
1423 		case USB_EP_ATTR_BULK:
1424 			if (tw->tw_qtd_free_list != NULL) {
1425 				uint32_t	td_addr;
1426 				td_addr = ehci_qtd_cpu_to_iommu(ehcip,
1427 				    tw->tw_qtd_free_list);
1428 				Set_QTD(qtd->qtd_tw_next_qtd, td_addr);
1429 				Set_QTD(qtd->qtd_state, EHCI_QTD_DUMMY);
1430 				tw->tw_qtd_free_list = qtd;
1431 			} else {
1432 				tw->tw_qtd_free_list = qtd;
1433 				Set_QTD(qtd->qtd_tw_next_qtd, 0);
1434 				Set_QTD(qtd->qtd_state, EHCI_QTD_DUMMY);
1435 			}
1436 			break;
1437 		}
1438 		qtd = next_qtd;
1439 	}
1440 
1441 	return (num_characters);
1442 }
1443 
1444 
1445 /*
1446  * ehci_polled_handle_normal_qtd:
1447  */
1448 static int
ehci_polled_handle_normal_qtd(ehci_polled_t * ehci_polledp,ehci_qtd_t * qtd)1449 ehci_polled_handle_normal_qtd(
1450 	ehci_polled_t		*ehci_polledp,
1451 	ehci_qtd_t		*qtd)
1452 {
1453 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1454 	uchar_t			*buf;
1455 	ehci_trans_wrapper_t	*tw;
1456 	size_t			length;
1457 	uint32_t		residue;
1458 
1459 	/* Obtain the transfer wrapper from the QTD */
1460 	tw = (ehci_trans_wrapper_t *)EHCI_LOOKUP_ID((uint32_t)
1461 	    Get_QTD(qtd->qtd_trans_wrapper));
1462 
1463 	ASSERT(tw != NULL);
1464 
1465 	buf = (uchar_t *)tw->tw_buf;
1466 
1467 	length = tw->tw_length;
1468 
1469 	/*
1470 	 * If "Total bytes of xfer" in control field of qtd is not equal to 0,
1471 	 * then we received less data from the usb device than requested by us.
1472 	 * In that case, get the actual received data size.
1473 	 */
1474 	residue = ((Get_QTD(qtd->qtd_ctrl) &
1475 	    EHCI_QTD_CTRL_BYTES_TO_XFER) >> EHCI_QTD_CTRL_BYTES_TO_XFER_SHIFT);
1476 
1477 	if (residue) {
1478 
1479 		length = Get_QTD(qtd->qtd_xfer_offs) +
1480 		    Get_QTD(qtd->qtd_xfer_len) - residue;
1481 	}
1482 
1483 	/* Sync IO buffer */
1484 	if (ehci_polledp->ehci_polled_no_sync_flag == B_FALSE) {
1485 		Sync_IO_Buffer(tw->tw_dmahandle, length);
1486 	}
1487 
1488 	/* Copy the data into the message */
1489 	bcopy(buf, ehci_polledp->ehci_polled_buf, length);
1490 
1491 	return ((int)length);
1492 }
1493 
1494 
1495 /*
1496  * ehci_polled_insert_intr_qtd:
1497  *
1498  * Insert a Transfer Descriptor (QTD) on an Endpoint Descriptor (QH).
1499  */
1500 static void
ehci_polled_insert_intr_qtd(ehci_polled_t * ehci_polledp,ehci_qtd_t * qtd)1501 ehci_polled_insert_intr_qtd(
1502 	ehci_polled_t		*ehci_polledp,
1503 	ehci_qtd_t		*qtd)
1504 {
1505 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1506 	ehci_qtd_t		*curr_dummy_qtd, *next_dummy_qtd;
1507 	ehci_qtd_t		*new_dummy_qtd;
1508 	uint_t			qtd_control;
1509 	ehci_pipe_private_t	*pp;
1510 	ehci_qh_t		*qh;
1511 	ehci_trans_wrapper_t	*tw;
1512 
1513 	/* Obtain the transfer wrapper from the QTD */
1514 	tw = (ehci_trans_wrapper_t *)EHCI_LOOKUP_ID(
1515 	    (uint32_t)Get_QTD(qtd->qtd_trans_wrapper));
1516 
1517 	pp = tw->tw_pipe_private;
1518 
1519 	/* Obtain the endpoint and interrupt request */
1520 	qh = pp->pp_qh;
1521 
1522 	/*
1523 	 * Take this QTD off the transfer wrapper's list since
1524 	 * the pipe is FIFO, this must be the first QTD on the
1525 	 * list.
1526 	 */
1527 	ASSERT((ehci_qtd_t *)tw->tw_qtd_head == qtd);
1528 
1529 	tw->tw_qtd_head = (ehci_qtd_t *)
1530 	    ehci_qtd_iommu_to_cpu(ehcip, Get_QTD(qtd->qtd_tw_next_qtd));
1531 
1532 	/*
1533 	 * If the head becomes NULL, then there are no more
1534 	 * active QTD's for this transfer wrapper. Also	set
1535 	 * the tail to NULL.
1536 	 */
1537 	if (tw->tw_qtd_head == NULL) {
1538 		tw->tw_qtd_tail = NULL;
1539 	}
1540 
1541 	/* Convert current valid QTD as new dummy QTD */
1542 	bzero((char *)qtd, sizeof (ehci_qtd_t));
1543 	Set_QTD(qtd->qtd_state, EHCI_QTD_DUMMY);
1544 
1545 	/* Rename qtd as new_dummy_qtd */
1546 	new_dummy_qtd = qtd;
1547 
1548 	/* Get the current and next dummy QTDs */
1549 	curr_dummy_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1550 	    Get_QH(qh->qh_dummy_qtd));
1551 	next_dummy_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1552 	    Get_QTD(curr_dummy_qtd->qtd_next_qtd));
1553 
1554 	/* Update QH's dummy qtd field */
1555 	Set_QH(qh->qh_dummy_qtd, ehci_qtd_cpu_to_iommu(ehcip, next_dummy_qtd));
1556 
1557 	/* Update next dummy's next qtd pointer */
1558 	Set_QTD(next_dummy_qtd->qtd_next_qtd,
1559 	    ehci_qtd_cpu_to_iommu(ehcip, new_dummy_qtd));
1560 
1561 	qtd_control = (tw->tw_direction | EHCI_QTD_CTRL_INTR_ON_COMPLETE);
1562 
1563 	/*
1564 	 * Fill in the current dummy qtd and
1565 	 * add the new dummy to the end.
1566 	 */
1567 	ehci_polled_fill_in_qtd(ehcip, curr_dummy_qtd, qtd_control,
1568 	    0, tw->tw_length, tw);
1569 
1570 	/* Insert this qtd onto the tw */
1571 	ehci_polled_insert_qtd_on_tw(ehcip, tw, curr_dummy_qtd);
1572 
1573 	/* Insert this qtd into active interrupt QTD list */
1574 	ehci_polled_insert_qtd_into_active_intr_qtd_list(
1575 	    ehci_polledp, curr_dummy_qtd);
1576 }
1577 
1578 
1579 static void
ehci_polled_insert_bulk_qtd(ehci_polled_t * ehci_polledp)1580 ehci_polled_insert_bulk_qtd(
1581 	ehci_polled_t	*ehci_polledp)
1582 {
1583 	ehci_state_t		*ehcip;
1584 	ehci_pipe_private_t	*pp;
1585 	ehci_trans_wrapper_t	*tw;
1586 	ehci_qh_t		*qh;
1587 	ehci_qtd_t		*new_dummy_qtd;
1588 	ehci_qtd_t		*curr_dummy_qtd, *next_dummy_qtd;
1589 	uint_t			qtd_control;
1590 
1591 	ehcip = ehci_polledp->ehci_polled_ehcip;
1592 	pp = (ehci_pipe_private_t *)ehci_polledp->
1593 	    ehci_polled_input_pipe_handle->p_hcd_private;
1594 	tw = pp->pp_tw_head;
1595 	qh = ehci_polledp->ehci_polled_qh;
1596 	new_dummy_qtd = tw->tw_qtd_free_list;
1597 
1598 	if (new_dummy_qtd == NULL) {
1599 		return;
1600 	}
1601 
1602 	tw->tw_qtd_free_list = ehci_qtd_iommu_to_cpu(ehcip,
1603 	    Get_QTD(new_dummy_qtd->qtd_tw_next_qtd));
1604 	Set_QTD(new_dummy_qtd->qtd_tw_next_qtd, 0);
1605 
1606 	/* Get the current and next dummy QTDs */
1607 	curr_dummy_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1608 	    Get_QH(qh->qh_dummy_qtd));
1609 	next_dummy_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1610 	    Get_QTD(curr_dummy_qtd->qtd_next_qtd));
1611 
1612 	/* Update QH's dummy qtd field */
1613 	Set_QH(qh->qh_dummy_qtd, ehci_qtd_cpu_to_iommu(ehcip, next_dummy_qtd));
1614 
1615 	/* Update next dummy's next qtd pointer */
1616 	Set_QTD(next_dummy_qtd->qtd_next_qtd,
1617 	    ehci_qtd_cpu_to_iommu(ehcip, new_dummy_qtd));
1618 
1619 	qtd_control = (tw->tw_direction | EHCI_QTD_CTRL_INTR_ON_COMPLETE);
1620 
1621 	/*
1622 	 * Fill in the current dummy qtd and
1623 	 * add the new dummy to the end.
1624 	 */
1625 	ehci_polled_fill_in_qtd(ehcip, curr_dummy_qtd, qtd_control,
1626 	    0, tw->tw_length, tw);
1627 
1628 	/* Insert this qtd into active interrupt QTD list */
1629 	ehci_polled_insert_qtd_into_active_intr_qtd_list(
1630 	    ehci_polledp, curr_dummy_qtd);
1631 }
1632 
1633 
1634 /*
1635  * ehci_polled_fill_in_qtd:
1636  *
1637  * Fill in the fields of a Transfer Descriptor (QTD).
1638  * The "Buffer Pointer" fields of a QTD are retrieved from the TW
1639  * it is associated with.
1640  *
1641  * Unlike the it's ehci_fill_in_qtd counterpart, we do not
1642  * set the alternative ptr in polled mode.  There is not need
1643  * for it in polled mode, because it doesn't need to cleanup
1644  * short xfer conditions.
1645  *
1646  * Note:
1647  * qtd_dma_offs - the starting offset into the TW buffer, where the QTD
1648  *		  should transfer from. It should be 4K aligned. And when
1649  *		  a TW has more than one QTDs, the QTDs must be filled in
1650  *		  increasing order.
1651  * qtd_length - the total bytes to transfer.
1652  */
1653 static void
ehci_polled_fill_in_qtd(ehci_state_t * ehcip,ehci_qtd_t * qtd,uint_t qtd_ctrl,size_t qtd_dma_offs,size_t qtd_length,ehci_trans_wrapper_t * tw)1654 ehci_polled_fill_in_qtd(
1655 	ehci_state_t		*ehcip,
1656 	ehci_qtd_t		*qtd,
1657 	uint_t			qtd_ctrl,
1658 	size_t			qtd_dma_offs,
1659 	size_t			qtd_length,
1660 	ehci_trans_wrapper_t	*tw)
1661 {
1662 	uint32_t		buf_addr;
1663 	size_t			buf_len = qtd_length;
1664 	uint32_t		ctrl = qtd_ctrl;
1665 	uint_t			i = 0;
1666 	int			rem_len;
1667 
1668 	/* Assert that the qtd to be filled in is a dummy */
1669 	ASSERT(Get_QTD(qtd->qtd_state) == EHCI_QTD_DUMMY);
1670 
1671 	/* Change QTD's state Active */
1672 	Set_QTD(qtd->qtd_state, EHCI_QTD_ACTIVE);
1673 
1674 	/* Set the total length data tarnsfer */
1675 	ctrl |= (((qtd_length << EHCI_QTD_CTRL_BYTES_TO_XFER_SHIFT)
1676 	    & EHCI_QTD_CTRL_BYTES_TO_XFER) | EHCI_QTD_CTRL_MAX_ERR_COUNTS);
1677 
1678 	/*
1679 	 * QTDs must be filled in increasing DMA offset order.
1680 	 * tw_dma_offs is initialized to be 0 at TW creation and
1681 	 * is only increased in this function.
1682 	 */
1683 	ASSERT(buf_len == 0 || qtd_dma_offs >= tw->tw_dma_offs);
1684 
1685 	/*
1686 	 * Save the starting dma buffer offset used and
1687 	 * length of data that will be transfered in
1688 	 * the current QTD.
1689 	 */
1690 	Set_QTD(qtd->qtd_xfer_offs, qtd_dma_offs);
1691 	Set_QTD(qtd->qtd_xfer_len, buf_len);
1692 
1693 	while (buf_len) {
1694 		/*
1695 		 * Advance to the next DMA cookie until finding the cookie
1696 		 * that qtd_dma_offs falls in.
1697 		 * It is very likely this loop will never repeat more than
1698 		 * once. It is here just to accommodate the case qtd_dma_offs
1699 		 * is increased by multiple cookies during two consecutive
1700 		 * calls into this function. In that case, the interim DMA
1701 		 * buffer is allowed to be skipped.
1702 		 */
1703 		while ((tw->tw_dma_offs + tw->tw_cookie.dmac_size) <=
1704 		    qtd_dma_offs) {
1705 			/*
1706 			 * tw_dma_offs always points to the starting offset
1707 			 * of a cookie
1708 			 */
1709 			tw->tw_dma_offs += tw->tw_cookie.dmac_size;
1710 			ddi_dma_nextcookie(tw->tw_dmahandle, &tw->tw_cookie);
1711 			tw->tw_cookie_idx++;
1712 			ASSERT(tw->tw_cookie_idx < tw->tw_ncookies);
1713 		}
1714 
1715 		/*
1716 		 * Counting the remained buffer length to be filled in
1717 		 * the QTD for current DMA cookie
1718 		 */
1719 		rem_len = (tw->tw_dma_offs + tw->tw_cookie.dmac_size) -
1720 		    qtd_dma_offs;
1721 
1722 		/* Update the beginning of the buffer */
1723 		buf_addr = (qtd_dma_offs - tw->tw_dma_offs) +
1724 		    tw->tw_cookie.dmac_address;
1725 		ASSERT((buf_addr % EHCI_4K_ALIGN) == 0);
1726 		Set_QTD(qtd->qtd_buf[i], buf_addr);
1727 
1728 		if (buf_len <= EHCI_MAX_QTD_BUF_SIZE) {
1729 			ASSERT(buf_len <= rem_len);
1730 			break;
1731 		} else {
1732 			ASSERT(rem_len >= EHCI_MAX_QTD_BUF_SIZE);
1733 			buf_len -= EHCI_MAX_QTD_BUF_SIZE;
1734 			qtd_dma_offs += EHCI_MAX_QTD_BUF_SIZE;
1735 		}
1736 
1737 		i++;
1738 	}
1739 
1740 	/*
1741 	 * For control, bulk and interrupt QTD, now
1742 	 * enable current QTD by setting active bit.
1743 	 */
1744 	Set_QTD(qtd->qtd_ctrl, (ctrl | EHCI_QTD_CTRL_ACTIVE_XACT));
1745 
1746 	Set_QTD(qtd->qtd_trans_wrapper, (uint32_t)tw->tw_id);
1747 }
1748 
1749 
1750 /*
1751  * ehci_polled_insert_qtd_on_tw:
1752  *
1753  * The transfer wrapper keeps a list of all Transfer Descriptors (QTD) that
1754  * are allocated for this transfer. Insert a QTD  onto this list. The  list
1755  * of QTD's does not include the dummy QTD that is at the end of the list of
1756  * QTD's for the endpoint.
1757  */
1758 static void
ehci_polled_insert_qtd_on_tw(ehci_state_t * ehcip,ehci_trans_wrapper_t * tw,ehci_qtd_t * qtd)1759 ehci_polled_insert_qtd_on_tw(
1760 	ehci_state_t		*ehcip,
1761 	ehci_trans_wrapper_t	*tw,
1762 	ehci_qtd_t		*qtd)
1763 {
1764 	/*
1765 	 * Set the next pointer to NULL because
1766 	 * this is the last QTD on list.
1767 	 */
1768 	Set_QTD(qtd->qtd_tw_next_qtd, 0);
1769 
1770 	if (tw->tw_qtd_head == NULL) {
1771 		ASSERT(tw->tw_qtd_tail == NULL);
1772 		tw->tw_qtd_head = qtd;
1773 		tw->tw_qtd_tail = qtd;
1774 	} else {
1775 		ehci_qtd_t *dummy = (ehci_qtd_t *)tw->tw_qtd_tail;
1776 
1777 		ASSERT(dummy != NULL);
1778 		ASSERT(dummy != qtd);
1779 		ASSERT(Get_QTD(qtd->qtd_state) != EHCI_QTD_DUMMY);
1780 
1781 		/* Add the qtd to the end of the list */
1782 		Set_QTD(dummy->qtd_tw_next_qtd,
1783 		    ehci_qtd_cpu_to_iommu(ehcip, qtd));
1784 
1785 		tw->tw_qtd_tail = qtd;
1786 
1787 		ASSERT(Get_QTD(qtd->qtd_tw_next_qtd) == 0);
1788 	}
1789 }
1790 
1791 
1792 /*
1793  * ehci_polled_create_done_qtd_list:
1794  *
1795  * Create done qtd list from active qtd list.
1796  */
1797 static ehci_qtd_t *
ehci_polled_create_done_qtd_list(ehci_polled_t * ehci_polledp)1798 ehci_polled_create_done_qtd_list(
1799 	ehci_polled_t		*ehci_polledp)
1800 {
1801 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1802 	ehci_qtd_t		*curr_qtd = NULL, *next_qtd = NULL;
1803 	ehci_qtd_t		*done_qtd_list = NULL, *last_done_qtd = NULL;
1804 
1805 	USB_DPRINTF_L4(PRINT_MASK_INTR, ehcip->ehci_log_hdl,
1806 	    "ehci_polled_create_done_qtd_list:");
1807 
1808 	curr_qtd = ehci_polledp->ehci_polled_active_intr_qtd_list;
1809 
1810 	while (curr_qtd) {
1811 
1812 		/* Get next qtd from the active qtd list */
1813 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1814 		    Get_QTD(curr_qtd->qtd_active_qtd_next));
1815 
1816 		/* Check this QTD has been processed by Host Controller */
1817 		if (!(Get_QTD(curr_qtd->qtd_ctrl) &
1818 		    EHCI_QTD_CTRL_ACTIVE_XACT)) {
1819 
1820 			/* Remove this QTD from active QTD list */
1821 			ehci_polled_remove_qtd_from_active_intr_qtd_list(
1822 			    ehci_polledp, curr_qtd);
1823 
1824 			Set_QTD(curr_qtd->qtd_active_qtd_next, 0);
1825 
1826 			if (done_qtd_list) {
1827 				Set_QTD(last_done_qtd->qtd_active_qtd_next,
1828 				    ehci_qtd_cpu_to_iommu(ehcip, curr_qtd));
1829 
1830 				last_done_qtd = curr_qtd;
1831 			} else {
1832 				done_qtd_list = curr_qtd;
1833 				last_done_qtd = curr_qtd;
1834 			}
1835 		}
1836 
1837 		curr_qtd = next_qtd;
1838 	}
1839 
1840 	return (done_qtd_list);
1841 }
1842 
1843 
1844 /*
1845  * ehci_polled_insert_qtd_into_active_intr_qtd_list:
1846  *
1847  * Insert current QTD into active interrupt QTD list.
1848  */
1849 static void
ehci_polled_insert_qtd_into_active_intr_qtd_list(ehci_polled_t * ehci_polledp,ehci_qtd_t * qtd)1850 ehci_polled_insert_qtd_into_active_intr_qtd_list(
1851 	ehci_polled_t		*ehci_polledp,
1852 	ehci_qtd_t		*qtd)
1853 {
1854 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1855 	ehci_qtd_t		*curr_qtd, *next_qtd;
1856 
1857 	curr_qtd = ehci_polledp->ehci_polled_active_intr_qtd_list;
1858 
1859 	/* Insert this qtd into active intr qtd list */
1860 	if (curr_qtd) {
1861 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1862 		    Get_QTD(curr_qtd->qtd_active_qtd_next));
1863 
1864 		while (next_qtd) {
1865 			curr_qtd = next_qtd;
1866 			next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1867 			    Get_QTD(curr_qtd->qtd_active_qtd_next));
1868 		}
1869 
1870 		Set_QTD(qtd->qtd_active_qtd_prev,
1871 		    ehci_qtd_cpu_to_iommu(ehcip, curr_qtd));
1872 
1873 		Set_QTD(curr_qtd->qtd_active_qtd_next,
1874 		    ehci_qtd_cpu_to_iommu(ehcip, qtd));
1875 	} else {
1876 		ehci_polledp->ehci_polled_active_intr_qtd_list = qtd;
1877 		Set_QTD(qtd->qtd_active_qtd_next, 0);
1878 		Set_QTD(qtd->qtd_active_qtd_prev, 0);
1879 	}
1880 }
1881 
1882 
1883 /*
1884  * ehci_polled_remove_qtd_from_active_intr_qtd_list:
1885  *
1886  * Remove current QTD from the active QTD list.
1887  */
1888 void
ehci_polled_remove_qtd_from_active_intr_qtd_list(ehci_polled_t * ehci_polledp,ehci_qtd_t * qtd)1889 ehci_polled_remove_qtd_from_active_intr_qtd_list(
1890 	ehci_polled_t		*ehci_polledp,
1891 	ehci_qtd_t		*qtd)
1892 {
1893 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1894 	ehci_qtd_t		*curr_qtd, *prev_qtd, *next_qtd;
1895 
1896 	ASSERT(qtd != NULL);
1897 
1898 	curr_qtd = ehci_polledp->ehci_polled_active_intr_qtd_list;
1899 
1900 	while ((curr_qtd) && (curr_qtd != qtd)) {
1901 		curr_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1902 		    Get_QTD(curr_qtd->qtd_active_qtd_next));
1903 	}
1904 
1905 	if ((curr_qtd) && (curr_qtd == qtd)) {
1906 		prev_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1907 		    Get_QTD(curr_qtd->qtd_active_qtd_prev));
1908 		next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1909 		    Get_QTD(curr_qtd->qtd_active_qtd_next));
1910 
1911 		if (prev_qtd) {
1912 			Set_QTD(prev_qtd->qtd_active_qtd_next,
1913 			    Get_QTD(curr_qtd->qtd_active_qtd_next));
1914 		} else {
1915 			ehci_polledp->
1916 			    ehci_polled_active_intr_qtd_list = next_qtd;
1917 		}
1918 
1919 		if (next_qtd) {
1920 			Set_QTD(next_qtd->qtd_active_qtd_prev,
1921 			    Get_QTD(curr_qtd->qtd_active_qtd_prev));
1922 		}
1923 	}
1924 }
1925 
1926 
1927 /*
1928  * ehci_polled_traverse_qtds:
1929  *
1930  * Traverse the list of QTDs for given pipe using transfer wrapper.  Since
1931  * the endpoint is marked as Halted, the Host Controller (HC) is no longer
1932  * accessing these QTDs. Remove all the QTDs that are attached to endpoint.
1933  */
1934 static void
ehci_polled_traverse_qtds(ehci_polled_t * ehci_polledp,usba_pipe_handle_data_t * ph)1935 ehci_polled_traverse_qtds(
1936 	ehci_polled_t		*ehci_polledp,
1937 	usba_pipe_handle_data_t	*ph)
1938 {
1939 	ehci_state_t		*ehcip = ehci_polledp->ehci_polled_ehcip;
1940 	ehci_pipe_private_t	*pp = (ehci_pipe_private_t *)ph->p_hcd_private;
1941 	ehci_trans_wrapper_t	*next_tw;
1942 	ehci_qtd_t		*qtd;
1943 	ehci_qtd_t		*next_qtd;
1944 
1945 	/* Process the transfer wrappers for this pipe */
1946 	next_tw = pp->pp_tw_head;
1947 
1948 	while (next_tw) {
1949 		qtd = (ehci_qtd_t *)next_tw->tw_qtd_head;
1950 
1951 		/* Walk through each QTD for this transfer wrapper */
1952 		while (qtd) {
1953 			/* Remove this QTD from active QTD list */
1954 			ehci_polled_remove_qtd_from_active_intr_qtd_list(
1955 			    ehci_polledp, qtd);
1956 
1957 			next_qtd = ehci_qtd_iommu_to_cpu(ehcip,
1958 			    Get_QTD(qtd->qtd_tw_next_qtd));
1959 
1960 			/* Deallocate this QTD */
1961 			ehci_deallocate_qtd(ehcip, qtd);
1962 
1963 			qtd = next_qtd;
1964 		}
1965 
1966 		next_tw = next_tw->tw_next;
1967 	}
1968 
1969 	/* Clear current qtd pointer */
1970 	Set_QH(pp->pp_qh->qh_curr_qtd, (uint32_t)0x00000000);
1971 
1972 	/* Update the next qtd pointer in the QH */
1973 	Set_QH(pp->pp_qh->qh_next_qtd, Get_QH(pp->pp_qh->qh_dummy_qtd));
1974 }
1975 
1976 
1977 /*
1978  * ehci_polled_finish_interrupt:
1979  */
1980 static void
ehci_polled_finish_interrupt(ehci_state_t * ehcip,uint_t intr)1981 ehci_polled_finish_interrupt(
1982 	ehci_state_t	*ehcip,
1983 	uint_t		intr)
1984 {
1985 	/* Acknowledge the interrupt */
1986 	Set_OpReg(ehci_status, intr);
1987 
1988 	/*
1989 	 * Read interrupt status register to make sure that any PIO
1990 	 * store to clear the ISR has made it on the PCI bus before
1991 	 * returning from its interrupt handler.
1992 	 */
1993 	(void) Get_OpReg(ehci_status);
1994 }
1995 
1996 
1997 static int
ehci_polled_create_tw(ehci_polled_t * ehci_polledp,usba_pipe_handle_data_t * ph,usb_flags_t usb_flags)1998 ehci_polled_create_tw(
1999 	ehci_polled_t	*ehci_polledp,
2000 	usba_pipe_handle_data_t	*ph,
2001 	usb_flags_t	usb_flags)
2002 {
2003 	uint_t			ccount;
2004 	size_t			real_length;
2005 	ehci_trans_wrapper_t	*tw;
2006 	ddi_device_acc_attr_t	dev_attr;
2007 	int			result, pipe_dir, qtd_count;
2008 	ehci_state_t		*ehcip;
2009 	ehci_pipe_private_t	*pp;
2010 	ddi_dma_attr_t		dma_attr;
2011 
2012 	ehcip = ehci_polledp->ehci_polled_ehcip;
2013 	pp = (ehci_pipe_private_t *)ph->p_hcd_private;
2014 
2015 	/* Get the required qtd counts */
2016 	qtd_count = (POLLED_RAW_BUF_SIZE - 1) / EHCI_MAX_QTD_XFER_SIZE + 1;
2017 
2018 	if ((tw = kmem_zalloc(sizeof (ehci_trans_wrapper_t),
2019 	    KM_NOSLEEP)) == NULL) {
2020 		return (USB_FAILURE);
2021 	}
2022 
2023 	/* allow sg lists for transfer wrapper dma memory */
2024 	bcopy(&ehcip->ehci_dma_attr, &dma_attr, sizeof (ddi_dma_attr_t));
2025 	dma_attr.dma_attr_sgllen = EHCI_DMA_ATTR_TW_SGLLEN;
2026 	dma_attr.dma_attr_align = EHCI_DMA_ATTR_ALIGNMENT;
2027 
2028 	/* Allocate the DMA handle */
2029 	if ((result = ddi_dma_alloc_handle(ehcip->ehci_dip,
2030 	    &dma_attr, DDI_DMA_DONTWAIT, 0, &tw->tw_dmahandle)) !=
2031 	    DDI_SUCCESS) {
2032 		kmem_free(tw, sizeof (ehci_trans_wrapper_t));
2033 
2034 		return (USB_FAILURE);
2035 	}
2036 
2037 	dev_attr.devacc_attr_version		= DDI_DEVICE_ATTR_V0;
2038 	dev_attr.devacc_attr_endian_flags	= DDI_STRUCTURE_LE_ACC;
2039 	dev_attr.devacc_attr_dataorder		= DDI_STRICTORDER_ACC;
2040 
2041 	/* Allocate the memory */
2042 	if ((result = ddi_dma_mem_alloc(tw->tw_dmahandle, POLLED_RAW_BUF_SIZE,
2043 	    &dev_attr, DDI_DMA_CONSISTENT, DDI_DMA_DONTWAIT, NULL,
2044 	    &tw->tw_buf, &real_length, &tw->tw_accesshandle)) !=
2045 	    DDI_SUCCESS) {
2046 		ddi_dma_free_handle(&tw->tw_dmahandle);
2047 		kmem_free(tw, sizeof (ehci_trans_wrapper_t));
2048 
2049 		return (USB_FAILURE);
2050 	}
2051 
2052 	/* Bind the handle */
2053 	if ((result = ddi_dma_addr_bind_handle(tw->tw_dmahandle, NULL,
2054 	    tw->tw_buf, real_length, DDI_DMA_RDWR|DDI_DMA_CONSISTENT,
2055 	    DDI_DMA_DONTWAIT, NULL, &tw->tw_cookie, &ccount)) !=
2056 	    DDI_DMA_MAPPED) {
2057 		ddi_dma_mem_free(&tw->tw_accesshandle);
2058 		ddi_dma_free_handle(&tw->tw_dmahandle);
2059 		kmem_free(tw, sizeof (ehci_trans_wrapper_t));
2060 
2061 		return (USB_FAILURE);
2062 	}
2063 
2064 	/* The cookie count should be 1 */
2065 	if (ccount != 1) {
2066 		result = ddi_dma_unbind_handle(tw->tw_dmahandle);
2067 		ASSERT(result == DDI_SUCCESS);
2068 
2069 		ddi_dma_mem_free(&tw->tw_accesshandle);
2070 		ddi_dma_free_handle(&tw->tw_dmahandle);
2071 		kmem_free(tw, sizeof (ehci_trans_wrapper_t));
2072 
2073 		return (USB_FAILURE);
2074 	}
2075 
2076 	if (ehci_allocate_tds_for_tw(ehcip, pp, tw, qtd_count) == USB_SUCCESS) {
2077 		tw->tw_num_qtds = qtd_count;
2078 	} else {
2079 		ehci_deallocate_tw(ehcip, pp, tw);
2080 		return (USB_FAILURE);
2081 	}
2082 	tw->tw_cookie_idx = 0;
2083 	tw->tw_dma_offs = 0;
2084 
2085 	/*
2086 	 * Only allow one wrapper to be added at a time. Insert the
2087 	 * new transaction wrapper into the list for this pipe.
2088 	 */
2089 	if (pp->pp_tw_head == NULL) {
2090 		pp->pp_tw_head = tw;
2091 		pp->pp_tw_tail = tw;
2092 	} else {
2093 		pp->pp_tw_tail->tw_next = tw;
2094 		pp->pp_tw_tail = tw;
2095 	}
2096 
2097 	/* Store the transfer length */
2098 	tw->tw_length = POLLED_RAW_BUF_SIZE;
2099 
2100 	/* Store a back pointer to the pipe private structure */
2101 	tw->tw_pipe_private = pp;
2102 
2103 	/* Store the transfer type - synchronous or asynchronous */
2104 	tw->tw_flags = usb_flags;
2105 
2106 	/* Get and Store 32bit ID */
2107 	tw->tw_id = EHCI_GET_ID((void *)tw);
2108 
2109 	pipe_dir = ph->p_ep.bEndpointAddress & USB_EP_DIR_MASK;
2110 	tw->tw_direction = (pipe_dir == USB_EP_DIR_OUT)?
2111 	    EHCI_QTD_CTRL_OUT_PID : EHCI_QTD_CTRL_IN_PID;
2112 
2113 	USB_DPRINTF_L4(PRINT_MASK_ALLOC, ehcip->ehci_log_hdl,
2114 	    "ehci_create_transfer_wrapper: tw = 0x%p, ncookies = %u",
2115 	    (void *)tw, tw->tw_ncookies);
2116 
2117 	return (USB_SUCCESS);
2118 }
2119 
2120 
2121 /*
2122  * ehci_polled_insert_async_qh:
2123  *
2124  * Insert a bulk endpoint into the Host Controller's (HC)
2125  * Asynchronous schedule endpoint list.
2126  */
2127 static void
ehci_polled_insert_async_qh(ehci_state_t * ehcip,ehci_pipe_private_t * pp)2128 ehci_polled_insert_async_qh(
2129 	ehci_state_t		*ehcip,
2130 	ehci_pipe_private_t	*pp)
2131 {
2132 	ehci_qh_t		*qh = pp->pp_qh;
2133 	ehci_qh_t		*async_head_qh;
2134 	ehci_qh_t		*next_qh;
2135 	uintptr_t		qh_addr;
2136 
2137 	/* Make sure this QH is not already in the list */
2138 	ASSERT((Get_QH(qh->qh_prev) & EHCI_QH_LINK_PTR) == 0);
2139 
2140 	qh_addr = ehci_qh_cpu_to_iommu(ehcip, qh);
2141 
2142 	/* Obtain a ptr to the head of the Async schedule list */
2143 	async_head_qh = ehcip->ehci_head_of_async_sched_list;
2144 
2145 	if (async_head_qh == NULL) {
2146 		/* Set this QH to be the "head" of the circular list */
2147 		Set_QH(qh->qh_ctrl,
2148 		    (Get_QH(qh->qh_ctrl) | EHCI_QH_CTRL_RECLAIM_HEAD));
2149 
2150 		/* Set new QH's link and previous pointer to itself */
2151 		Set_QH(qh->qh_link_ptr, qh_addr | EHCI_QH_LINK_REF_QH);
2152 		Set_QH(qh->qh_prev, qh_addr);
2153 
2154 		ehcip->ehci_head_of_async_sched_list = qh;
2155 
2156 		/* Set the head ptr to the new endpoint */
2157 		Set_OpReg(ehci_async_list_addr, qh_addr);
2158 
2159 		/*
2160 		 * For some reason this register might get nulled out by
2161 		 * the Uli M1575 South Bridge. To workaround the hardware
2162 		 * problem, check the value after write and retry if the
2163 		 * last write fails.
2164 		 *
2165 		 * If the ASYNCLISTADDR remains "stuck" after
2166 		 * EHCI_MAX_RETRY retries, then the M1575 is broken
2167 		 * and is stuck in an inconsistent state and is about
2168 		 * to crash the machine with a trn_oor panic when it
2169 		 * does a DMA read from 0x0.  It is better to panic
2170 		 * now rather than wait for the trn_oor crash; this
2171 		 * way Customer Service will have a clean signature
2172 		 * that indicts the M1575 chip rather than a
2173 		 * mysterious and hard-to-diagnose trn_oor panic.
2174 		 */
2175 		if ((ehcip->ehci_vendor_id == PCI_VENDOR_ULi_M1575) &&
2176 		    (ehcip->ehci_device_id == PCI_DEVICE_ULi_M1575) &&
2177 		    (qh_addr != Get_OpReg(ehci_async_list_addr))) {
2178 			int retry = 0;
2179 
2180 			Set_OpRegRetry(ehci_async_list_addr, qh_addr, retry);
2181 			if (retry >= EHCI_MAX_RETRY)
2182 				cmn_err(CE_PANIC, "ehci_insert_async_qh:"
2183 				    " ASYNCLISTADDR write failed.");
2184 
2185 			USB_DPRINTF_L2(PRINT_MASK_ATTA, ehcip->ehci_log_hdl,
2186 			    "ehci_insert_async_qh: ASYNCLISTADDR "
2187 			    "write failed, retry=%d", retry);
2188 		}
2189 	} else {
2190 		ASSERT(Get_QH(async_head_qh->qh_ctrl) &
2191 		    EHCI_QH_CTRL_RECLAIM_HEAD);
2192 
2193 		/* Ensure this QH's "H" bit is not set */
2194 		Set_QH(qh->qh_ctrl,
2195 		    (Get_QH(qh->qh_ctrl) & ~EHCI_QH_CTRL_RECLAIM_HEAD));
2196 
2197 		next_qh = ehci_qh_iommu_to_cpu(ehcip,
2198 		    Get_QH(async_head_qh->qh_link_ptr) & EHCI_QH_LINK_PTR);
2199 
2200 		/* Set new QH's link and previous pointers */
2201 		Set_QH(qh->qh_link_ptr,
2202 		    Get_QH(async_head_qh->qh_link_ptr) | EHCI_QH_LINK_REF_QH);
2203 		Set_QH(qh->qh_prev, ehci_qh_cpu_to_iommu(ehcip, async_head_qh));
2204 
2205 		/* Set next QH's prev pointer */
2206 		Set_QH(next_qh->qh_prev, ehci_qh_cpu_to_iommu(ehcip, qh));
2207 
2208 		/* Set QH Head's link pointer points to new QH */
2209 		Set_QH(async_head_qh->qh_link_ptr,
2210 		    qh_addr | EHCI_QH_LINK_REF_QH);
2211 	}
2212 }
2213 
2214 
2215 /*
2216  * ehci_remove_async_qh:
2217  *
2218  * Remove a control/bulk endpoint into the Host Controller's (HC)
2219  * Asynchronous schedule endpoint list.
2220  */
2221 static void
ehci_polled_remove_async_qh(ehci_state_t * ehcip,ehci_pipe_private_t * pp)2222 ehci_polled_remove_async_qh(
2223 	ehci_state_t		*ehcip,
2224 	ehci_pipe_private_t	*pp)
2225 {
2226 	ehci_qh_t		*qh = pp->pp_qh; /* qh to be removed */
2227 	ehci_qh_t		*prev_qh, *next_qh;
2228 
2229 	prev_qh = ehci_qh_iommu_to_cpu(ehcip,
2230 	    Get_QH(qh->qh_prev) & EHCI_QH_LINK_PTR);
2231 	next_qh = ehci_qh_iommu_to_cpu(ehcip,
2232 	    Get_QH(qh->qh_link_ptr) & EHCI_QH_LINK_PTR);
2233 
2234 	/* Make sure this QH is in the list */
2235 	ASSERT(prev_qh != NULL);
2236 
2237 	/*
2238 	 * If next QH and current QH are the same, then this is the last
2239 	 * QH on the Asynchronous Schedule list.
2240 	 */
2241 	if (qh == next_qh) {
2242 		ASSERT(Get_QH(qh->qh_ctrl) & EHCI_QH_CTRL_RECLAIM_HEAD);
2243 		/*
2244 		 * Null our pointer to the async sched list, but do not
2245 		 * touch the host controller's list_addr.
2246 		 */
2247 		ehcip->ehci_head_of_async_sched_list = NULL;
2248 	} else {
2249 		/* If this QH is the HEAD then find another one to replace it */
2250 		if (ehcip->ehci_head_of_async_sched_list == qh) {
2251 
2252 			ASSERT(Get_QH(qh->qh_ctrl) & EHCI_QH_CTRL_RECLAIM_HEAD);
2253 			ehcip->ehci_head_of_async_sched_list = next_qh;
2254 			Set_QH(next_qh->qh_ctrl,
2255 			    Get_QH(next_qh->qh_ctrl) |
2256 			    EHCI_QH_CTRL_RECLAIM_HEAD);
2257 		}
2258 		Set_QH(prev_qh->qh_link_ptr, Get_QH(qh->qh_link_ptr));
2259 		Set_QH(next_qh->qh_prev, Get_QH(qh->qh_prev));
2260 	}
2261 
2262 	/* qh_prev to indicate it is no longer in the circular list */
2263 	Set_QH(qh->qh_prev, 0);
2264 
2265 }
2266