xref: /illumos-gate/usr/src/uts/i86pc/io/immu_intrmap.c (revision 7fd1b424e267a2cdece3dce940e5722f9b785fed)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
24  */
25 
26 /*
27  * Copyright (c) 2009, Intel Corporation.
28  * All rights reserved.
29  */
30 
31 
32 #include <sys/apic.h>
33 #include <vm/hat_i86.h>
34 #include <sys/sysmacros.h>
35 #include <sys/smp_impldefs.h>
36 #include <sys/immu.h>
37 
38 
39 typedef struct intrmap_private {
40 	immu_t		*ir_immu;
41 	immu_inv_wait_t	ir_inv_wait;
42 	uint16_t	ir_idx;
43 	uint32_t	ir_sid_svt_sq;
44 } intrmap_private_t;
45 
46 #define	INTRMAP_PRIVATE(intrmap) ((intrmap_private_t *)intrmap)
47 
48 /* interrupt remapping table entry */
49 typedef struct intrmap_rte {
50 	uint64_t	lo;
51 	uint64_t	hi;
52 } intrmap_rte_t;
53 
54 #define	IRTE_HIGH(sid_svt_sq) (sid_svt_sq)
55 #define	IRTE_LOW(dst, vector, dlm, tm, rh, dm, fpd, p)	\
56 	    (((uint64_t)(dst) << 32) |  \
57 	    ((uint64_t)(vector) << 16) | \
58 	    ((uint64_t)(dlm) << 5) | \
59 	    ((uint64_t)(tm) << 4) | \
60 	    ((uint64_t)(rh) << 3) | \
61 	    ((uint64_t)(dm) << 2) | \
62 	    ((uint64_t)(fpd) << 1) | \
63 	    (p))
64 
65 typedef enum {
66 	SVT_NO_VERIFY = 0,	/* no verification */
67 	SVT_ALL_VERIFY,		/* using sid and sq to verify */
68 	SVT_BUS_VERIFY,		/* verify #startbus and #endbus */
69 	SVT_RSVD
70 } intrmap_svt_t;
71 
72 typedef enum {
73 	SQ_VERIFY_ALL = 0,	/* verify all 16 bits */
74 	SQ_VERIFY_IGR_1,	/* ignore bit 3 */
75 	SQ_VERIFY_IGR_2,	/* ignore bit 2-3 */
76 	SQ_VERIFY_IGR_3		/* ignore bit 1-3 */
77 } intrmap_sq_t;
78 
79 /*
80  * S field of the Interrupt Remapping Table Address Register
81  * the size of the interrupt remapping table is 1 << (immu_intrmap_irta_s + 1)
82  */
83 static uint_t intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
84 
85 /*
86  * If true, arrange to suppress broadcast EOI by setting edge-triggered mode
87  * even for level-triggered interrupts in the interrupt-remapping engine.
88  * If false, broadcast EOI can still be suppressed if the CPU supports the
89  * APIC_SVR_SUPPRESS_BROADCAST_EOI bit.  In both cases, the IOAPIC is still
90  * programmed with the correct trigger mode, and pcplusmp must send an EOI
91  * to the IOAPIC by writing to the IOAPIC's EOI register to make up for the
92  * missing broadcast EOI.
93  */
94 static int intrmap_suppress_brdcst_eoi = 0;
95 
96 /*
97  * whether verify the source id of interrupt request
98  */
99 static int intrmap_enable_sid_verify = 0;
100 
101 /* fault types for DVMA remapping */
102 static char *immu_dvma_faults[] = {
103 	"Reserved",
104 	"The present field in root-entry is Clear",
105 	"The present field in context-entry is Clear",
106 	"Hardware detected invalid programming of a context-entry",
107 	"The DMA request attempted to access an address beyond max support",
108 	"The Write field in a page-table entry is Clear when DMA write",
109 	"The Read field in a page-table entry is Clear when DMA read",
110 	"Access the next level page table resulted in error",
111 	"Access the root-entry table resulted in error",
112 	"Access the context-entry table resulted in error",
113 	"Reserved field not initialized to zero in a present root-entry",
114 	"Reserved field not initialized to zero in a present context-entry",
115 	"Reserved field not initialized to zero in a present page-table entry",
116 	"DMA blocked due to the Translation Type field in context-entry",
117 	"Incorrect fault event reason number",
118 };
119 #define	DVMA_MAX_FAULTS (sizeof (immu_dvma_faults)/(sizeof (char *))) - 1
120 
121 /* fault types for interrupt remapping */
122 static char *immu_intrmap_faults[] = {
123 	"reserved field set in IRTE",
124 	"interrupt_index exceed the intr-remap table size",
125 	"present field in IRTE is clear",
126 	"hardware access intr-remap table address resulted in error",
127 	"reserved field set in IRTE, include various conditional",
128 	"hardware blocked an interrupt request in Compatibility format",
129 	"remappable interrupt request blocked due to verification failure"
130 };
131 #define	INTRMAP_MAX_FAULTS \
132 	(sizeof (immu_intrmap_faults) / (sizeof (char *))) - 1
133 
134 /* Function prototypes */
135 static int immu_intrmap_init(int apic_mode);
136 static void immu_intrmap_switchon(int suppress_brdcst_eoi);
137 static void immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
138     uint16_t type, int count, uchar_t ioapic_index);
139 static void immu_intrmap_map(void *intrmap_private, void *intrmap_data,
140     uint16_t type, int count);
141 static void immu_intrmap_free(void **intrmap_privatep);
142 static void immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt);
143 static void immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs);
144 
145 static struct apic_intrmap_ops intrmap_ops = {
146 	immu_intrmap_init,
147 	immu_intrmap_switchon,
148 	immu_intrmap_alloc,
149 	immu_intrmap_map,
150 	immu_intrmap_free,
151 	immu_intrmap_rdt,
152 	immu_intrmap_msi,
153 };
154 
155 /* apic mode, APIC/X2APIC */
156 static int intrmap_apic_mode = LOCAL_APIC;
157 
158 
159 /*
160  * helper functions
161  */
162 static uint_t
bitset_find_free(bitset_t * b,uint_t post)163 bitset_find_free(bitset_t *b, uint_t post)
164 {
165 	uint_t	i;
166 	uint_t	cap = bitset_capacity(b);
167 
168 	if (post == cap)
169 		post = 0;
170 
171 	ASSERT(post < cap);
172 
173 	for (i = post; i < cap; i++) {
174 		if (!bitset_in_set(b, i))
175 			return (i);
176 	}
177 
178 	for (i = 0; i < post; i++) {
179 		if (!bitset_in_set(b, i))
180 			return (i);
181 	}
182 
183 	return (INTRMAP_IDX_FULL);	/* no free index */
184 }
185 
186 /*
187  * helper function to find 'count' contigous free
188  * interrupt remapping table entries
189  */
190 static uint_t
bitset_find_multi_free(bitset_t * b,uint_t post,uint_t count)191 bitset_find_multi_free(bitset_t *b, uint_t post, uint_t count)
192 {
193 	uint_t  i, j;
194 	uint_t	cap = bitset_capacity(b);
195 
196 	if (post == INTRMAP_IDX_FULL) {
197 		return (INTRMAP_IDX_FULL);
198 	}
199 
200 	if (count > cap)
201 		return (INTRMAP_IDX_FULL);
202 
203 	ASSERT(post < cap);
204 
205 	for (i = post; (i + count) <= cap; i++) {
206 		for (j = 0; j < count; j++) {
207 			if (bitset_in_set(b, (i + j))) {
208 				i = i + j;
209 				break;
210 			}
211 			if (j == count - 1)
212 				return (i);
213 		}
214 	}
215 
216 	for (i = 0; (i < post) && ((i + count) <= cap); i++) {
217 		for (j = 0; j < count; j++) {
218 			if (bitset_in_set(b, (i + j))) {
219 				i = i + j;
220 				break;
221 			}
222 			if (j == count - 1)
223 				return (i);
224 		}
225 	}
226 
227 	return (INTRMAP_IDX_FULL);		/* no free index */
228 }
229 
230 /* alloc one interrupt remapping table entry */
231 static int
alloc_tbl_entry(intrmap_t * intrmap)232 alloc_tbl_entry(intrmap_t *intrmap)
233 {
234 	uint32_t idx;
235 
236 	for (;;) {
237 		mutex_enter(&intrmap->intrmap_lock);
238 		idx = intrmap->intrmap_free;
239 		if (idx != INTRMAP_IDX_FULL) {
240 			bitset_add(&intrmap->intrmap_map, idx);
241 			intrmap->intrmap_free =
242 			    bitset_find_free(&intrmap->intrmap_map, idx + 1);
243 			mutex_exit(&intrmap->intrmap_lock);
244 			break;
245 		}
246 
247 		/* no free intr entry, use compatible format intr */
248 		mutex_exit(&intrmap->intrmap_lock);
249 
250 		if (intrmap_apic_mode != LOCAL_X2APIC) {
251 			break;
252 		}
253 
254 		/*
255 		 * x2apic mode not allowed compatible
256 		 * interrupt
257 		 */
258 		delay(IMMU_ALLOC_RESOURCE_DELAY);
259 	}
260 
261 	return (idx);
262 }
263 
264 /* alloc 'cnt' contigous interrupt remapping table entries */
265 static int
alloc_tbl_multi_entries(intrmap_t * intrmap,uint_t cnt)266 alloc_tbl_multi_entries(intrmap_t *intrmap, uint_t cnt)
267 {
268 	uint_t idx, pos, i;
269 
270 	for (; ; ) {
271 		mutex_enter(&intrmap->intrmap_lock);
272 		pos = intrmap->intrmap_free;
273 		idx = bitset_find_multi_free(&intrmap->intrmap_map, pos, cnt);
274 
275 		if (idx != INTRMAP_IDX_FULL) {
276 			if (idx <= pos && pos < (idx + cnt)) {
277 				intrmap->intrmap_free = bitset_find_free(
278 				    &intrmap->intrmap_map, idx + cnt);
279 			}
280 			for (i = 0; i < cnt; i++) {
281 				bitset_add(&intrmap->intrmap_map, idx + i);
282 			}
283 			mutex_exit(&intrmap->intrmap_lock);
284 			break;
285 		}
286 
287 		mutex_exit(&intrmap->intrmap_lock);
288 
289 		if (intrmap_apic_mode != LOCAL_X2APIC) {
290 			break;
291 		}
292 
293 		/* x2apic mode not allowed comapitible interrupt */
294 		delay(IMMU_ALLOC_RESOURCE_DELAY);
295 	}
296 
297 	return (idx);
298 }
299 
300 /* init interrupt remapping table */
301 static int
init_unit(immu_t * immu)302 init_unit(immu_t *immu)
303 {
304 	intrmap_t *intrmap;
305 	size_t size;
306 
307 	ddi_dma_attr_t intrmap_dma_attr = {
308 		DMA_ATTR_V0,
309 		0U,
310 		0xffffffffffffffffULL,
311 		0xffffffffU,
312 		MMU_PAGESIZE,	/* page aligned */
313 		0x1,
314 		0x1,
315 		0xffffffffU,
316 		0xffffffffffffffffULL,
317 		1,
318 		4,
319 		0
320 	};
321 
322 	ddi_device_acc_attr_t intrmap_acc_attr = {
323 		DDI_DEVICE_ATTR_V0,
324 		DDI_NEVERSWAP_ACC,
325 		DDI_STRICTORDER_ACC
326 	};
327 
328 	/*
329 	 * Using interrupt remapping implies using the queue
330 	 * invalidation interface. According to Intel,
331 	 * hardware that supports interrupt remapping should
332 	 * also support QI.
333 	 */
334 	ASSERT(IMMU_ECAP_GET_QI(immu->immu_regs_excap));
335 
336 	if (intrmap_apic_mode == LOCAL_X2APIC) {
337 		if (!IMMU_ECAP_GET_EIM(immu->immu_regs_excap)) {
338 			return (DDI_FAILURE);
339 		}
340 	}
341 
342 	if (intrmap_irta_s > INTRMAP_MAX_IRTA_SIZE) {
343 		intrmap_irta_s = INTRMAP_MAX_IRTA_SIZE;
344 	}
345 
346 	intrmap =  kmem_zalloc(sizeof (intrmap_t), KM_SLEEP);
347 
348 	if (ddi_dma_alloc_handle(immu->immu_dip,
349 	    &intrmap_dma_attr,
350 	    DDI_DMA_SLEEP,
351 	    NULL,
352 	    &(intrmap->intrmap_dma_hdl)) != DDI_SUCCESS) {
353 		kmem_free(intrmap, sizeof (intrmap_t));
354 		return (DDI_FAILURE);
355 	}
356 
357 	intrmap->intrmap_size = 1 << (intrmap_irta_s + 1);
358 	size = intrmap->intrmap_size * INTRMAP_RTE_SIZE;
359 	if (ddi_dma_mem_alloc(intrmap->intrmap_dma_hdl,
360 	    size,
361 	    &intrmap_acc_attr,
362 	    DDI_DMA_CONSISTENT | IOMEM_DATA_UNCACHED,
363 	    DDI_DMA_SLEEP,
364 	    NULL,
365 	    &(intrmap->intrmap_vaddr),
366 	    &size,
367 	    &(intrmap->intrmap_acc_hdl)) != DDI_SUCCESS) {
368 		ddi_dma_free_handle(&(intrmap->intrmap_dma_hdl));
369 		kmem_free(intrmap, sizeof (intrmap_t));
370 		return (DDI_FAILURE);
371 	}
372 
373 	ASSERT(!((uintptr_t)intrmap->intrmap_vaddr & MMU_PAGEOFFSET));
374 	bzero(intrmap->intrmap_vaddr, size);
375 	intrmap->intrmap_paddr = pfn_to_pa(
376 	    hat_getpfnum(kas.a_hat, intrmap->intrmap_vaddr));
377 
378 	mutex_init(&(intrmap->intrmap_lock), NULL, MUTEX_DRIVER, NULL);
379 	bitset_init(&intrmap->intrmap_map);
380 	bitset_resize(&intrmap->intrmap_map, intrmap->intrmap_size);
381 	intrmap->intrmap_free = 0;
382 
383 	immu->immu_intrmap = intrmap;
384 
385 	return (DDI_SUCCESS);
386 }
387 
388 static immu_t *
get_immu(dev_info_t * dip,uint16_t type,uchar_t ioapic_index)389 get_immu(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
390 {
391 	immu_t	*immu = NULL;
392 
393 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
394 		immu = immu_dmar_ioapic_immu(ioapic_index);
395 	} else {
396 		if (dip != NULL)
397 			immu = immu_dmar_get_immu(dip);
398 	}
399 
400 	return (immu);
401 }
402 
403 static int
get_top_pcibridge(dev_info_t * dip,void * arg)404 get_top_pcibridge(dev_info_t *dip, void *arg)
405 {
406 	dev_info_t **topdipp = arg;
407 	immu_devi_t *immu_devi;
408 
409 	mutex_enter(&(DEVI(dip)->devi_lock));
410 	immu_devi = DEVI(dip)->devi_iommu;
411 	mutex_exit(&(DEVI(dip)->devi_lock));
412 
413 	if (immu_devi == NULL || immu_devi->imd_pcib_type == IMMU_PCIB_BAD ||
414 	    immu_devi->imd_pcib_type == IMMU_PCIB_ENDPOINT) {
415 		return (DDI_WALK_CONTINUE);
416 	}
417 
418 	*topdipp = dip;
419 
420 	return (DDI_WALK_CONTINUE);
421 }
422 
423 static dev_info_t *
intrmap_top_pcibridge(dev_info_t * rdip)424 intrmap_top_pcibridge(dev_info_t *rdip)
425 {
426 	dev_info_t *top_pcibridge = NULL;
427 
428 	if (immu_walk_ancestor(rdip, NULL, get_top_pcibridge,
429 	    &top_pcibridge, NULL, 0) != DDI_SUCCESS) {
430 		return (NULL);
431 	}
432 
433 	return (top_pcibridge);
434 }
435 
436 /* function to get interrupt request source id */
437 static uint32_t
get_sid(dev_info_t * dip,uint16_t type,uchar_t ioapic_index)438 get_sid(dev_info_t *dip, uint16_t type, uchar_t ioapic_index)
439 {
440 	dev_info_t	*pdip;
441 	immu_devi_t	*immu_devi;
442 	uint16_t	sid;
443 	uchar_t		svt, sq;
444 
445 	if (!intrmap_enable_sid_verify) {
446 		return (0);
447 	}
448 
449 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
450 		/* for interrupt through I/O APIC */
451 		sid = immu_dmar_ioapic_sid(ioapic_index);
452 		svt = SVT_ALL_VERIFY;
453 		sq = SQ_VERIFY_ALL;
454 	} else {
455 		/* MSI/MSI-X interrupt */
456 		ASSERT(dip);
457 		pdip = intrmap_top_pcibridge(dip);
458 		ASSERT(pdip);
459 		immu_devi = DEVI(pdip)->devi_iommu;
460 		ASSERT(immu_devi);
461 		if (immu_devi->imd_pcib_type == IMMU_PCIB_PCIE_PCI) {
462 			/* device behind pcie to pci bridge */
463 			sid = (immu_devi->imd_bus << 8) | immu_devi->imd_sec;
464 			svt = SVT_BUS_VERIFY;
465 			sq = SQ_VERIFY_ALL;
466 		} else {
467 			/* pcie device or device behind pci to pci bridge */
468 			sid = (immu_devi->imd_bus << 8) |
469 			    immu_devi->imd_devfunc;
470 			svt = SVT_ALL_VERIFY;
471 			sq = SQ_VERIFY_ALL;
472 		}
473 	}
474 
475 	return (sid | (svt << 18) | (sq << 16));
476 }
477 
478 static void
intrmap_enable(immu_t * immu)479 intrmap_enable(immu_t *immu)
480 {
481 	intrmap_t *intrmap;
482 	uint64_t irta_reg;
483 
484 	intrmap = immu->immu_intrmap;
485 
486 	irta_reg = intrmap->intrmap_paddr | intrmap_irta_s;
487 	if (intrmap_apic_mode == LOCAL_X2APIC) {
488 		irta_reg |= (0x1 << 11);
489 	}
490 
491 	immu_regs_intrmap_enable(immu, irta_reg);
492 }
493 
494 /* ####################################################################### */
495 
496 /*
497  * immu_intr_handler()
498  *	the fault event handler for a single immu unit
499  */
500 uint_t
immu_intr_handler(caddr_t arg,caddr_t arg1 __unused)501 immu_intr_handler(caddr_t arg, caddr_t arg1 __unused)
502 {
503 	immu_t *immu = (immu_t *)arg;
504 	uint32_t status;
505 	int index, fault_reg_offset;
506 	int max_fault_index;
507 	boolean_t found_fault;
508 	dev_info_t *idip;
509 
510 	mutex_enter(&(immu->immu_intr_lock));
511 	mutex_enter(&(immu->immu_regs_lock));
512 
513 	/* read the fault status */
514 	status = immu_regs_get32(immu, IMMU_REG_FAULT_STS);
515 
516 	idip = immu->immu_dip;
517 	ASSERT(idip);
518 
519 	/* check if we have a pending fault for this immu unit */
520 	if ((status & IMMU_FAULT_STS_PPF) == 0) {
521 		mutex_exit(&(immu->immu_regs_lock));
522 		mutex_exit(&(immu->immu_intr_lock));
523 		return (DDI_INTR_UNCLAIMED);
524 	}
525 
526 	/*
527 	 * handle all primary pending faults
528 	 */
529 	index = IMMU_FAULT_GET_INDEX(status);
530 	max_fault_index =  IMMU_CAP_GET_NFR(immu->immu_regs_cap) - 1;
531 	fault_reg_offset = IMMU_CAP_GET_FRO(immu->immu_regs_cap);
532 
533 	found_fault = B_FALSE;
534 	_NOTE(CONSTCOND)
535 	while (1) {
536 		uint64_t val;
537 		uint8_t fault_reason;
538 		uint8_t fault_type;
539 		uint16_t sid;
540 		uint64_t pg_addr;
541 		uint64_t idx;
542 
543 		/* read the higher 64bits */
544 		val = immu_regs_get64(immu, fault_reg_offset + index * 16 + 8);
545 
546 		/* check if this fault register has pending fault */
547 		if (!IMMU_FRR_GET_F(val)) {
548 			break;
549 		}
550 
551 		found_fault = B_TRUE;
552 
553 		/* get the fault reason, fault type and sid */
554 		fault_reason = IMMU_FRR_GET_FR(val);
555 		fault_type = IMMU_FRR_GET_FT(val);
556 		sid = IMMU_FRR_GET_SID(val);
557 
558 		/* read the first 64bits */
559 		val = immu_regs_get64(immu, fault_reg_offset + index * 16);
560 		pg_addr = val & IMMU_PAGEMASK;
561 		idx = val >> 48;
562 
563 		/* clear the fault */
564 		immu_regs_put32(immu, fault_reg_offset + index * 16 + 12,
565 		    (((uint32_t)1) << 31));
566 
567 		/* report the fault info */
568 		if (fault_reason < 0x20) {
569 			/* immu-remapping fault */
570 			ddi_err(DER_WARN, idip,
571 			    "generated a fault event when translating DMA %s\n"
572 			    "\t on address 0x%" PRIx64 " for PCI(%d, %d, %d), "
573 			    "the reason is:\n\t %s",
574 			    fault_type ? "read" : "write", pg_addr,
575 			    (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
576 			    immu_dvma_faults[MIN(fault_reason,
577 			    DVMA_MAX_FAULTS)]);
578 			immu_print_fault_info(sid, pg_addr);
579 		} else if (fault_reason < 0x27) {
580 			/* intr-remapping fault */
581 			ddi_err(DER_WARN, idip,
582 			    "generated a fault event when translating "
583 			    "interrupt request\n"
584 			    "\t on index 0x%" PRIx64 " for PCI(%d, %d, %d), "
585 			    "the reason is:\n\t %s",
586 			    idx,
587 			    (sid >> 8) & 0xff, (sid >> 3) & 0x1f, sid & 0x7,
588 			    immu_intrmap_faults[MIN((fault_reason - 0x20),
589 			    INTRMAP_MAX_FAULTS)]);
590 		} else {
591 			ddi_err(DER_WARN, idip, "Unknown fault reason: 0x%x",
592 			    fault_reason);
593 		}
594 
595 		index++;
596 		if (index > max_fault_index)
597 			index = 0;
598 	}
599 
600 	/* Clear the fault */
601 	if (!found_fault) {
602 		ddi_err(DER_MODE, idip,
603 		    "Fault register set but no fault present");
604 	}
605 	immu_regs_put32(immu, IMMU_REG_FAULT_STS, 1);
606 	mutex_exit(&(immu->immu_regs_lock));
607 	mutex_exit(&(immu->immu_intr_lock));
608 	return (DDI_INTR_CLAIMED);
609 }
610 /* ######################################################################### */
611 
612 /*
613  * Interrupt remap entry points
614  */
615 
616 /* initialize interrupt remapping */
617 static int
immu_intrmap_init(int apic_mode)618 immu_intrmap_init(int apic_mode)
619 {
620 	immu_t *immu;
621 	int error = DDI_FAILURE;
622 
623 	if (immu_intrmap_enable == B_FALSE) {
624 		return (DDI_SUCCESS);
625 	}
626 
627 	intrmap_apic_mode = apic_mode;
628 
629 	immu = list_head(&immu_list);
630 	for (; immu; immu = list_next(&immu_list, immu)) {
631 		if ((immu->immu_intrmap_running == B_TRUE) &&
632 		    IMMU_ECAP_GET_IR(immu->immu_regs_excap)) {
633 			if (init_unit(immu) == DDI_SUCCESS) {
634 				error = DDI_SUCCESS;
635 			}
636 		}
637 	}
638 
639 	/*
640 	 * if all IOMMU units disable intr remapping,
641 	 * return FAILURE
642 	 */
643 	return (error);
644 }
645 
646 
647 
648 /* enable interrupt remapping */
649 static void
immu_intrmap_switchon(int suppress_brdcst_eoi)650 immu_intrmap_switchon(int suppress_brdcst_eoi)
651 {
652 	immu_t *immu;
653 
654 
655 	intrmap_suppress_brdcst_eoi = suppress_brdcst_eoi;
656 
657 	immu = list_head(&immu_list);
658 	for (; immu; immu = list_next(&immu_list, immu)) {
659 		if (immu->immu_intrmap_setup == B_TRUE) {
660 			intrmap_enable(immu);
661 		}
662 	}
663 }
664 
665 /* alloc remapping entry for the interrupt */
666 static void
immu_intrmap_alloc(void ** intrmap_private_tbl,dev_info_t * dip,uint16_t type,int count,uchar_t ioapic_index)667 immu_intrmap_alloc(void **intrmap_private_tbl, dev_info_t *dip,
668     uint16_t type, int count, uchar_t ioapic_index)
669 {
670 	immu_t	*immu;
671 	intrmap_t *intrmap;
672 	immu_inv_wait_t *iwp;
673 	uint32_t		idx, i;
674 	uint32_t		sid_svt_sq;
675 	intrmap_private_t	*intrmap_private;
676 
677 	if (intrmap_private_tbl[0] == INTRMAP_DISABLE ||
678 	    intrmap_private_tbl[0] != NULL) {
679 		return;
680 	}
681 
682 	intrmap_private_tbl[0] =
683 	    kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
684 	intrmap_private = INTRMAP_PRIVATE(intrmap_private_tbl[0]);
685 
686 	immu = get_immu(dip, type, ioapic_index);
687 	if ((immu != NULL) && (immu->immu_intrmap_running == B_TRUE)) {
688 		intrmap_private->ir_immu = immu;
689 	} else {
690 		goto intrmap_disable;
691 	}
692 
693 	intrmap = immu->immu_intrmap;
694 
695 	if (count == 1) {
696 		idx = alloc_tbl_entry(intrmap);
697 	} else {
698 		idx = alloc_tbl_multi_entries(intrmap, count);
699 	}
700 
701 	if (idx == INTRMAP_IDX_FULL) {
702 		goto intrmap_disable;
703 	}
704 
705 	intrmap_private->ir_idx = idx;
706 
707 	sid_svt_sq = intrmap_private->ir_sid_svt_sq =
708 	    get_sid(dip, type, ioapic_index);
709 	iwp = &intrmap_private->ir_inv_wait;
710 	immu_init_inv_wait(iwp, "intrmaplocal", B_TRUE);
711 
712 	if (count == 1) {
713 		if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
714 			immu_qinv_intr_one_cache(immu, idx, iwp);
715 		} else {
716 			immu_regs_wbf_flush(immu);
717 		}
718 		return;
719 	}
720 
721 	for (i = 1; i < count; i++) {
722 		intrmap_private_tbl[i] =
723 		    kmem_zalloc(sizeof (intrmap_private_t), KM_SLEEP);
724 
725 		INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_immu = immu;
726 		INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_sid_svt_sq =
727 		    sid_svt_sq;
728 		INTRMAP_PRIVATE(intrmap_private_tbl[i])->ir_idx = idx + i;
729 	}
730 
731 	if (IMMU_CAP_GET_CM(immu->immu_regs_cap)) {
732 		immu_qinv_intr_caches(immu, idx, count, iwp);
733 	} else {
734 		immu_regs_wbf_flush(immu);
735 	}
736 
737 	return;
738 
739 intrmap_disable:
740 	kmem_free(intrmap_private_tbl[0], sizeof (intrmap_private_t));
741 	intrmap_private_tbl[0] = INTRMAP_DISABLE;
742 }
743 
744 
745 /* remapping the interrupt */
746 static void
immu_intrmap_map(void * intrmap_private,void * intrmap_data,uint16_t type,int count)747 immu_intrmap_map(void *intrmap_private, void *intrmap_data, uint16_t type,
748     int count)
749 {
750 	immu_t	*immu;
751 	immu_inv_wait_t	*iwp;
752 	intrmap_t	*intrmap;
753 	ioapic_rdt_t	*irdt = (ioapic_rdt_t *)intrmap_data;
754 	msi_regs_t	*mregs = (msi_regs_t *)intrmap_data;
755 	intrmap_rte_t	irte;
756 	uint_t		idx, i;
757 	uint32_t	dst, sid_svt_sq;
758 	uchar_t		vector, dlm, tm, rh, dm;
759 
760 	if (intrmap_private == INTRMAP_DISABLE)
761 		return;
762 
763 	idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
764 	immu = INTRMAP_PRIVATE(intrmap_private)->ir_immu;
765 	iwp = &INTRMAP_PRIVATE(intrmap_private)->ir_inv_wait;
766 	intrmap = immu->immu_intrmap;
767 	sid_svt_sq = INTRMAP_PRIVATE(intrmap_private)->ir_sid_svt_sq;
768 
769 	if (!DDI_INTR_IS_MSI_OR_MSIX(type)) {
770 		dm = RDT_DM(irdt->ir_lo);
771 		rh = 0;
772 		tm = RDT_TM(irdt->ir_lo);
773 		dlm = RDT_DLM(irdt->ir_lo);
774 		dst = irdt->ir_hi;
775 
776 		/*
777 		 * Mark the IRTE's TM as Edge to suppress broadcast EOI.
778 		 */
779 		if (intrmap_suppress_brdcst_eoi) {
780 			tm = TRIGGER_MODE_EDGE;
781 		}
782 
783 		vector = RDT_VECTOR(irdt->ir_lo);
784 	} else {
785 		dm = MSI_ADDR_DM_PHYSICAL;
786 		rh = MSI_ADDR_RH_FIXED;
787 		tm = TRIGGER_MODE_EDGE;
788 		dlm = 0;
789 		dst = mregs->mr_addr;
790 
791 		vector = mregs->mr_data & 0xff;
792 	}
793 
794 	if (intrmap_apic_mode == LOCAL_APIC)
795 		dst = (dst & 0xFF) << 8;
796 
797 	if (count == 1) {
798 		irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
799 		irte.hi = IRTE_HIGH(sid_svt_sq);
800 
801 		/* set interrupt remapping table entry */
802 		bcopy(&irte, intrmap->intrmap_vaddr +
803 		    idx * INTRMAP_RTE_SIZE,
804 		    INTRMAP_RTE_SIZE);
805 
806 		immu_qinv_intr_one_cache(immu, idx, iwp);
807 
808 	} else {
809 		for (i = 0; i < count; i++) {
810 			irte.lo = IRTE_LOW(dst, vector, dlm, tm, rh, dm, 0, 1);
811 			irte.hi = IRTE_HIGH(sid_svt_sq);
812 
813 			/* set interrupt remapping table entry */
814 			bcopy(&irte, intrmap->intrmap_vaddr +
815 			    idx * INTRMAP_RTE_SIZE,
816 			    INTRMAP_RTE_SIZE);
817 			vector++;
818 			idx++;
819 		}
820 
821 		immu_qinv_intr_caches(immu, idx, count, iwp);
822 	}
823 }
824 
825 /* free the remapping entry */
826 static void
immu_intrmap_free(void ** intrmap_privatep)827 immu_intrmap_free(void **intrmap_privatep)
828 {
829 	immu_t *immu;
830 	immu_inv_wait_t *iwp;
831 	intrmap_t *intrmap;
832 	uint32_t idx;
833 
834 	if (*intrmap_privatep == INTRMAP_DISABLE || *intrmap_privatep == NULL) {
835 		*intrmap_privatep = NULL;
836 		return;
837 	}
838 
839 	immu = INTRMAP_PRIVATE(*intrmap_privatep)->ir_immu;
840 	iwp = &INTRMAP_PRIVATE(*intrmap_privatep)->ir_inv_wait;
841 	intrmap = immu->immu_intrmap;
842 	idx = INTRMAP_PRIVATE(*intrmap_privatep)->ir_idx;
843 
844 	bzero(intrmap->intrmap_vaddr + idx * INTRMAP_RTE_SIZE,
845 	    INTRMAP_RTE_SIZE);
846 
847 	immu_qinv_intr_one_cache(immu, idx, iwp);
848 
849 	mutex_enter(&intrmap->intrmap_lock);
850 	bitset_del(&intrmap->intrmap_map, idx);
851 	if (intrmap->intrmap_free == INTRMAP_IDX_FULL) {
852 		intrmap->intrmap_free = idx;
853 	}
854 	mutex_exit(&intrmap->intrmap_lock);
855 
856 	kmem_free(*intrmap_privatep, sizeof (intrmap_private_t));
857 	*intrmap_privatep = NULL;
858 }
859 
860 /* record the ioapic rdt entry */
861 static void
immu_intrmap_rdt(void * intrmap_private,ioapic_rdt_t * irdt)862 immu_intrmap_rdt(void *intrmap_private, ioapic_rdt_t *irdt)
863 {
864 	uint32_t rdt_entry, tm, pol, idx, vector;
865 
866 	rdt_entry = irdt->ir_lo;
867 
868 	if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
869 		idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
870 		tm = RDT_TM(rdt_entry);
871 		pol = RDT_POL(rdt_entry);
872 		vector = RDT_VECTOR(rdt_entry);
873 		irdt->ir_lo = (tm << INTRMAP_IOAPIC_TM_SHIFT) |
874 		    (pol << INTRMAP_IOAPIC_POL_SHIFT) |
875 		    ((idx >> 15) << INTRMAP_IOAPIC_IDX15_SHIFT) |
876 		    vector;
877 		irdt->ir_hi = (idx << INTRMAP_IOAPIC_IDX_SHIFT) |
878 		    (1 << INTRMAP_IOAPIC_FORMAT_SHIFT);
879 	} else {
880 		irdt->ir_hi <<= APIC_ID_BIT_OFFSET;
881 	}
882 }
883 
884 /* record the msi interrupt structure */
885 /*ARGSUSED*/
886 static void
immu_intrmap_msi(void * intrmap_private,msi_regs_t * mregs)887 immu_intrmap_msi(void *intrmap_private, msi_regs_t *mregs)
888 {
889 	uint_t	idx;
890 
891 	if (intrmap_private != INTRMAP_DISABLE && intrmap_private != NULL) {
892 		idx = INTRMAP_PRIVATE(intrmap_private)->ir_idx;
893 
894 		mregs->mr_data = 0;
895 		mregs->mr_addr = MSI_ADDR_HDR |
896 		    ((idx & 0x7fff) << INTRMAP_MSI_IDX_SHIFT) |
897 		    (1 << INTRMAP_MSI_FORMAT_SHIFT) |
898 		    (1 << INTRMAP_MSI_SHV_SHIFT) |
899 		    ((idx >> 15) << INTRMAP_MSI_IDX15_SHIFT);
900 	} else {
901 		mregs->mr_addr = MSI_ADDR_HDR |
902 		    (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
903 		    (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT) |
904 		    (mregs->mr_addr << MSI_ADDR_DEST_SHIFT);
905 		mregs->mr_data = (MSI_DATA_TM_EDGE << MSI_DATA_TM_SHIFT) |
906 		    mregs->mr_data;
907 	}
908 }
909 
910 /* ######################################################################### */
911 /*
912  * Functions exported by immu_intr.c
913  */
914 void
immu_intrmap_setup(list_t * listp)915 immu_intrmap_setup(list_t *listp)
916 {
917 	immu_t *immu;
918 
919 	/*
920 	 * Check if ACPI DMAR tables say that
921 	 * interrupt remapping is supported
922 	 */
923 	if (immu_dmar_intrmap_supported() == B_FALSE) {
924 		return;
925 	}
926 
927 	/*
928 	 * Check if interrupt remapping is disabled.
929 	 */
930 	if (immu_intrmap_enable == B_FALSE) {
931 		return;
932 	}
933 
934 	psm_vt_ops = &intrmap_ops;
935 
936 	immu = list_head(listp);
937 	for (; immu; immu = list_next(listp, immu)) {
938 		mutex_init(&(immu->immu_intrmap_lock), NULL,
939 		    MUTEX_DEFAULT, NULL);
940 		mutex_enter(&(immu->immu_intrmap_lock));
941 		immu_init_inv_wait(&immu->immu_intrmap_inv_wait,
942 		    "intrmapglobal", B_TRUE);
943 		immu->immu_intrmap_setup = B_TRUE;
944 		mutex_exit(&(immu->immu_intrmap_lock));
945 	}
946 }
947 
948 void
immu_intrmap_startup(immu_t * immu)949 immu_intrmap_startup(immu_t *immu)
950 {
951 	/* do nothing */
952 	mutex_enter(&(immu->immu_intrmap_lock));
953 	if (immu->immu_intrmap_setup == B_TRUE) {
954 		immu->immu_intrmap_running = B_TRUE;
955 	}
956 	mutex_exit(&(immu->immu_intrmap_lock));
957 }
958 
959 /*
960  * Register a Intel IOMMU unit (i.e. DMAR unit's)
961  * interrupt handler
962  */
963 void
immu_intr_register(immu_t * immu)964 immu_intr_register(immu_t *immu)
965 {
966 	int irq, vect;
967 	char intr_handler_name[IMMU_MAXNAMELEN];
968 	uint32_t msi_data;
969 	uint32_t uaddr;
970 	uint32_t msi_addr;
971 	uint32_t localapic_id = 0;
972 
973 	if (psm_get_localapicid)
974 		localapic_id = psm_get_localapicid(0);
975 
976 	msi_addr = (MSI_ADDR_HDR |
977 	    ((localapic_id & 0xFF) << MSI_ADDR_DEST_SHIFT) |
978 	    (MSI_ADDR_RH_FIXED << MSI_ADDR_RH_SHIFT) |
979 	    (MSI_ADDR_DM_PHYSICAL << MSI_ADDR_DM_SHIFT));
980 
981 	if (intrmap_apic_mode == LOCAL_X2APIC) {
982 		uaddr = localapic_id & 0xFFFFFF00;
983 	} else {
984 		uaddr = 0;
985 	}
986 
987 	/* Dont need to hold immu_intr_lock since we are in boot */
988 	irq = vect = psm_get_ipivect(IMMU_INTR_IPL, -1);
989 	if (psm_xlate_vector_by_irq != NULL)
990 		vect = psm_xlate_vector_by_irq(irq);
991 
992 	msi_data = ((MSI_DATA_DELIVERY_FIXED <<
993 	    MSI_DATA_DELIVERY_SHIFT) | vect);
994 
995 	(void) snprintf(intr_handler_name, sizeof (intr_handler_name),
996 	    "%s-intr-handler", immu->immu_name);
997 
998 	(void) add_avintr((void *)NULL, IMMU_INTR_IPL,
999 	    immu_intr_handler, intr_handler_name, irq,
1000 	    (caddr_t)immu, NULL, NULL, NULL);
1001 
1002 	immu_regs_intr_enable(immu, msi_addr, msi_data, uaddr);
1003 
1004 	(void) immu_intr_handler((caddr_t)immu, NULL);
1005 }
1006