1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 * $FreeBSD$ 29 */ 30 /* 31 * This file and its contents are supplied under the terms of the 32 * Common Development and Distribution License ("CDDL"), version 1.0. 33 * You may only use this file in accordance with the terms of version 34 * 1.0 of the CDDL. 35 * 36 * A full copy of the text of the CDDL should have accompanied this 37 * source. A copy of the CDDL is also available via the Internet at 38 * http://www.illumos.org/license/CDDL. 39 * 40 * Copyright 2022 Oxide Computer Company 41 */ 42 43 #include <sys/cdefs.h> 44 __FBSDID("$FreeBSD$"); 45 46 #include <sys/param.h> 47 #include <sys/bus.h> 48 #include <sys/eventhandler.h> 49 #include <sys/sysctl.h> 50 #include <sys/systm.h> 51 52 #include <dev/pci/pcivar.h> 53 #include <dev/pci/pcireg.h> 54 55 #include <machine/cpu.h> 56 #include <machine/md_var.h> 57 58 #include <sys/ddi.h> 59 #include <sys/sunddi.h> 60 #include <sys/pci.h> 61 62 #include "vmm_util.h" 63 #include "iommu.h" 64 65 66 static kmutex_t iommu_lock; 67 68 static uint_t iommu_refcnt; 69 ddi_modhandle_t iommu_modhdl; 70 static const struct iommu_ops *ops; 71 static void *host_domain; 72 73 static int 74 iommu_find_device(dev_info_t *dip, void *arg) 75 { 76 boolean_t add = (boolean_t)(uintptr_t)arg; 77 78 if (pcie_is_pci_device(dip)) { 79 if (add) 80 iommu_add_device(host_domain, pci_get_rid(dip)); 81 else 82 iommu_remove_device(host_domain, pci_get_rid(dip)); 83 } 84 85 return (DDI_WALK_CONTINUE); 86 } 87 88 static vm_paddr_t 89 vmm_mem_maxaddr(void) 90 { 91 return (ptoa(physmax + 1)); 92 } 93 94 static int 95 iommu_init(void) 96 { 97 const char *mod_name; 98 int error = 0; 99 100 ASSERT(MUTEX_HELD(&iommu_lock)); 101 102 if (vmm_is_intel()) { 103 mod_name = "misc/vmm_vtd"; 104 } else if (vmm_is_svm()) { 105 /* Use the expected name for if/when this is ported */ 106 mod_name = "misc/vmm_amdvi"; 107 } else { 108 return (ENXIO); 109 } 110 111 /* Load the backend driver */ 112 iommu_modhdl = ddi_modopen(mod_name, KRTLD_MODE_FIRST, &error); 113 if (iommu_modhdl == NULL) { 114 return (error); 115 } 116 117 /* Locate the iommu_ops struct */ 118 ops = ddi_modsym(iommu_modhdl, IOMMU_OPS_SYM_NAME, &error); 119 if (ops == NULL) { 120 goto bail; 121 } 122 123 /* Initialize the backend */ 124 error = ops->init(); 125 if (error != 0) { 126 goto bail; 127 } 128 129 /* Create a domain for the devices owned by the host */ 130 const vm_paddr_t maxaddr = vmm_mem_maxaddr(); 131 host_domain = ops->create_domain(maxaddr); 132 if (host_domain == NULL) { 133 goto bail; 134 } 135 136 /* ... and populate it with 1:1 mappings for all of physical mem */ 137 iommu_create_mapping(host_domain, 0, 0, maxaddr); 138 139 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_TRUE); 140 ops->enable(); 141 142 return (0); 143 144 bail: 145 if (ops != NULL) { 146 ops->cleanup(); 147 ops = NULL; 148 } 149 if (iommu_modhdl != NULL) { 150 (void) ddi_modclose(iommu_modhdl); 151 iommu_modhdl = NULL; 152 } 153 return (error); 154 } 155 156 static void 157 iommu_cleanup(void) 158 { 159 ASSERT(MUTEX_HELD(&iommu_lock)); 160 ASSERT3P(ops, !=, NULL); 161 ASSERT0(iommu_refcnt); 162 163 ops->disable(); 164 ddi_walk_devs(ddi_root_node(), iommu_find_device, (void *)B_FALSE); 165 166 ops->destroy_domain(host_domain); 167 host_domain = NULL; 168 169 ops->cleanup(); 170 ops = NULL; 171 172 (void) ddi_modclose(iommu_modhdl); 173 iommu_modhdl = NULL; 174 } 175 176 static bool 177 iommu_ref(void) 178 { 179 mutex_enter(&iommu_lock); 180 if (ops == NULL) { 181 int err = iommu_init(); 182 183 if (err != 0) { 184 VERIFY3P(ops, ==, NULL); 185 mutex_exit(&iommu_lock); 186 return (false); 187 } 188 VERIFY3P(ops, !=, NULL); 189 } 190 iommu_refcnt++; 191 VERIFY3U(iommu_refcnt, <, UINT_MAX); 192 mutex_exit(&iommu_lock); 193 194 return (true); 195 } 196 197 static void 198 iommu_unref(void) 199 { 200 mutex_enter(&iommu_lock); 201 VERIFY3U(iommu_refcnt, >, 0); 202 iommu_refcnt--; 203 if (iommu_refcnt == 0) { 204 iommu_cleanup(); 205 VERIFY3P(ops, ==, NULL); 206 } 207 mutex_exit(&iommu_lock); 208 } 209 210 void * 211 iommu_create_domain(vm_paddr_t maxaddr) 212 { 213 if (iommu_ref()) { 214 return (ops->create_domain(maxaddr)); 215 } else { 216 return (NULL); 217 } 218 } 219 220 void 221 iommu_destroy_domain(void *domain) 222 { 223 ASSERT3P(domain, !=, NULL); 224 225 ops->destroy_domain(domain); 226 iommu_unref(); 227 } 228 229 void 230 iommu_create_mapping(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 231 { 232 uint64_t remaining = len; 233 234 ASSERT3P(domain, !=, NULL); 235 236 while (remaining > 0) { 237 uint64_t mapped; 238 239 mapped = ops->create_mapping(domain, gpa, hpa, remaining); 240 gpa += mapped; 241 hpa += mapped; 242 remaining -= mapped; 243 } 244 } 245 246 void 247 iommu_remove_mapping(void *domain, vm_paddr_t gpa, size_t len) 248 { 249 uint64_t remaining = len; 250 251 ASSERT3P(domain, !=, NULL); 252 253 while (remaining > 0) { 254 uint64_t unmapped; 255 256 unmapped = ops->remove_mapping(domain, gpa, remaining); 257 gpa += unmapped; 258 remaining -= unmapped; 259 } 260 } 261 262 void * 263 iommu_host_domain(void) 264 { 265 return (host_domain); 266 } 267 268 void 269 iommu_add_device(void *domain, uint16_t rid) 270 { 271 ASSERT3P(domain, !=, NULL); 272 273 ops->add_device(domain, rid); 274 } 275 276 void 277 iommu_remove_device(void *domain, uint16_t rid) 278 { 279 ASSERT3P(domain, !=, NULL); 280 281 ops->remove_device(domain, rid); 282 } 283 284 void 285 iommu_invalidate_tlb(void *domain) 286 { 287 ASSERT3P(domain, !=, NULL); 288 289 ops->invalidate_tlb(domain); 290 } 291