1 /* 2 * This file and its contents are supplied under the terms of the 3 * Common Development and Distribution License ("CDDL"), version 1.0. 4 * You may only use this file in accordance with the terms of version 5 * 1.0 of the CDDL. 6 * 7 * A full copy of the text of the CDDL should have accompanied this 8 * source. A copy of the CDDL is also available via the Internet at 9 * http://www.illumos.org/license/CDDL. 10 */ 11 12 /* 13 * Copyright 2019, Joyent, Inc. 14 * Copyright 2021 Oxide Computer Company 15 */ 16 17 /* 18 * Nexus Driver for AMD Zen family systems. The purpose of this driver is to 19 * provide access to the following resources in a single, centralized fashion: 20 * 21 * - The per-chip Data Fabric 22 * - The North Bridge 23 * - The System Management Network (SMN) 24 * 25 * This is a nexus driver as once we have attached to all the requisite 26 * components, we will enumerate child devices which consume this functionality. 27 * 28 * ------------------------ 29 * Mapping Devices Together 30 * ------------------------ 31 * 32 * The operating system needs to expose things like temperature sensors and DRAM 33 * configuration registers in terms that are meaningful to the system such as 34 * logical CPUs, cores, etc. This driver attaches to the PCI IDs that represent 35 * the northbridge and data fabric; however, there are multiple PCI devices (one 36 * per die) that exist. This driver does manage to map all of these three things 37 * together; however, it requires some acrobatics. Unfortunately, there's no 38 * direct way to map a northbridge to its corresponding die. However, we can map 39 * a CPU die to a data fabric PCI device and a data fabric PCI device to a 40 * corresponding northbridge PCI device. 41 * 42 * In current Zen based products, there is a direct mapping between processor 43 * nodes and a data fabric PCI device. All of the devices are on PCI Bus 0 and 44 * start from Device 0x18. Device 0x18 maps to processor node 0, 0x19 to 45 * processor node 1, etc. This means that to map a logical CPU to a data fabric 46 * device, we take its processor node id, add it to 0x18 and find the PCI device 47 * that is on bus 0, device 0x18. As each data fabric device is attached based 48 * on its PCI ID, we add it to the global list, amd_nbdf_dfs that is in the 49 * amd_f17nbdf_t structure. 50 * 51 * The northbridge PCI device has a defined device and function, but the PCI bus 52 * that it's on can vary. Each die has its own series of PCI buses that are 53 * assigned to it and the northbridge PCI device is on the first of die-specific 54 * PCI bus for each die. This also means that the northbridge will not show up 55 * on PCI bus 0, which is the PCI bus that all of the data fabric devices are 56 * on. While conventionally the northbridge with the lowest PCI bus value 57 * would correspond to processor node zero, hardware does not guarantee that at 58 * all. Because we don't want to be at the mercy of firmware, we don't rely on 59 * this ordering, even though we have yet to find a system that deviates from 60 * this scheme. 61 * 62 * One of the registers in the data fabric device's function 0 63 * (AMDZEN_DF_F0_CFG_ADDR_CTL) happens to have the first PCI bus that is 64 * associated with the processor node. This means that we can map a data fabric 65 * device to a northbridge by finding the northbridge whose PCI bus matches the 66 * value in the corresponding data fabric's AMDZEN_DF_F0_CFG_ADDR_CTL. 67 * 68 * We can map a northbridge to a data fabric device and a data fabric device to 69 * a die. Because these are generally 1:1 mappings, there is a transitive 70 * relationship and therefore we know which northbridge is associated with which 71 * processor die. This is summarized in the following image: 72 * 73 * +-------+ +-----------------------------------+ +--------------+ 74 * | Die 0 |--->| Data Fabric PCI BDF 0/18/0 |------->| Northbridge | 75 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 10 | | PCI 10/0/0 | 76 * ... +-----------------------------------+ +--------------+ 77 * +-------+ +------------------------------------+ +--------------+ 78 * | Die n |---->| Data Fabric PCI BDF 0/18+n/0 |------->| Northbridge | 79 * +-------+ | AMDZEN_DF_F0_CFG_ADDR_CTL: bus 133 | | PCI 133/0/0 | 80 * +------------------------------------+ +--------------+ 81 * 82 * Note, the PCI buses used by the northbridges here are arbitrary. They do not 83 * reflect the actual values by hardware; however, the bus/device/function (BDF) 84 * of the data fabric accurately models hardware. All of the BDF values are in 85 * hex. 86 * 87 * Starting with the Rome generation of processors (Family 17h Model 30-3Fh), 88 * AMD has multiple northbridges that exist on a given die. All of these 89 * northbridges share the same data fabric and system management network port. 90 * From our perspective this means that some of the northbridge devices will be 91 * redundant and that we will no longer have a 1:1 mapping between the 92 * northbridge and the data fabric devices. Every data fabric will have a 93 * northbridge, but not every northbridge will have a data fabric device mapped. 94 * Because we're always trying to map from a die to a northbridge and not the 95 * reverse, the fact that there are extra northbridge devices hanging around 96 * that we don't know about shouldn't be a problem. 97 * 98 * ------------------------------- 99 * Attach and Detach Complications 100 * ------------------------------- 101 * 102 * Because we need to map different PCI devices together, this means that we 103 * have multiple dev_info_t structures that we need to manage. Each of these is 104 * independently attached and detached. While this is easily managed for attach, 105 * it is not for detach. Each of these devices is a 'stub'. 106 * 107 * Once a device has been detached it will only come back if we have an active 108 * minor node that will be accessed. This means that if they are detached, 109 * nothing would ever cause them to be reattached. The system also doesn't 110 * provide us a way or any guarantees around making sure that we're attached to 111 * all such devices before we detach. As a result, unfortunately, it's easier to 112 * basically have detach always fail. 113 * 114 * --------------- 115 * Exposed Devices 116 * --------------- 117 * 118 * Rather than try and have all of the different functions that could be 119 * provided by one driver, we instead have created a nexus driver that will 120 * itself try and load children. Children are all pseudo-device drivers that 121 * provide different pieces of functionality that use this. 122 */ 123 124 #include <sys/modctl.h> 125 #include <sys/conf.h> 126 #include <sys/devops.h> 127 #include <sys/ddi.h> 128 #include <sys/sunddi.h> 129 #include <sys/pci.h> 130 #include <sys/sysmacros.h> 131 #include <sys/sunndi.h> 132 #include <sys/x86_archext.h> 133 #include <sys/cpuvar.h> 134 135 #include "amdzen.h" 136 137 amdzen_t *amdzen_data; 138 139 /* 140 * Array of northbridge IDs that we care about. 141 */ 142 static const uint16_t amdzen_nb_ids[] = { 143 /* Family 17h Ryzen, Epyc Models 00h-0fh (Zen uarch) */ 144 0x1450, 145 /* Family 17h Raven Ridge, Kestrel, Dali Models 10h-2fh (Zen uarch) */ 146 0x15d0, 147 /* Family 17h/19h Rome, Milan, Matisse, Vermeer Zen 2/Zen 3 uarch */ 148 0x1480, 149 /* Family 17h Renoir Models 60-6fh (Zen 2 uarch) */ 150 0x1630 151 }; 152 153 typedef struct { 154 char *acd_name; 155 amdzen_child_t acd_addr; 156 } amdzen_child_data_t; 157 158 static const amdzen_child_data_t amdzen_children[] = { 159 { "smntemp", AMDZEN_C_SMNTEMP }, 160 { "usmn", AMDZEN_C_USMN }, 161 { "zen_udf", AMDZEN_C_ZEN_UDF } 162 }; 163 164 static uint32_t 165 amdzen_stub_get32(amdzen_stub_t *stub, off_t reg) 166 { 167 return (pci_config_get32(stub->azns_cfgspace, reg)); 168 } 169 170 static uint64_t 171 amdzen_stub_get64(amdzen_stub_t *stub, off_t reg) 172 { 173 return (pci_config_get64(stub->azns_cfgspace, reg)); 174 } 175 176 static void 177 amdzen_stub_put32(amdzen_stub_t *stub, off_t reg, uint32_t val) 178 { 179 pci_config_put32(stub->azns_cfgspace, reg, val); 180 } 181 182 /* 183 * Perform a targeted 32-bit indirect read to a specific instance and function. 184 */ 185 static uint32_t 186 amdzen_df_read32(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, uint8_t func, 187 uint16_t reg) 188 { 189 uint32_t val; 190 191 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 192 val = AMDZEN_DF_F4_FICAA_TARG_INST | AMDZEN_DF_F4_FICAA_SET_REG(reg) | 193 AMDZEN_DF_F4_FICAA_SET_FUNC(func) | 194 AMDZEN_DF_F4_FICAA_SET_INST(inst); 195 amdzen_stub_put32(df->adf_funcs[4], AMDZEN_DF_F4_FICAA, val); 196 return (amdzen_stub_get32(df->adf_funcs[4], AMDZEN_DF_F4_FICAD_LO)); 197 } 198 199 /* 200 * Perform a targeted 64-bit indirect read to a specific instance and function. 201 */ 202 static uint64_t 203 amdzen_df_read64(amdzen_t *azn, amdzen_df_t *df, uint8_t inst, uint8_t func, 204 uint16_t reg) 205 { 206 uint32_t val; 207 208 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 209 val = AMDZEN_DF_F4_FICAA_TARG_INST | AMDZEN_DF_F4_FICAA_SET_REG(reg) | 210 AMDZEN_DF_F4_FICAA_SET_FUNC(func) | 211 AMDZEN_DF_F4_FICAA_SET_INST(inst) | AMDZEN_DF_F4_FICAA_SET_64B; 212 amdzen_stub_put32(df->adf_funcs[4], AMDZEN_DF_F4_FICAA, val); 213 return (amdzen_stub_get64(df->adf_funcs[4], AMDZEN_DF_F4_FICAD_LO)); 214 } 215 216 static uint32_t 217 amdzen_smn_read32(amdzen_t *azn, amdzen_df_t *df, uint32_t reg) 218 { 219 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 220 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, reg); 221 return (amdzen_stub_get32(df->adf_nb, AMDZEN_NB_SMN_DATA)); 222 } 223 224 static void 225 amdzen_smn_write32(amdzen_t *azn, amdzen_df_t *df, uint32_t reg, uint32_t val) 226 { 227 VERIFY(MUTEX_HELD(&azn->azn_mutex)); 228 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_ADDR, reg); 229 amdzen_stub_put32(df->adf_nb, AMDZEN_NB_SMN_DATA, val); 230 } 231 232 static amdzen_df_t * 233 amdzen_df_find(amdzen_t *azn, uint_t dfno) 234 { 235 uint_t i; 236 237 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 238 if (dfno >= azn->azn_ndfs) { 239 return (NULL); 240 } 241 242 for (i = 0; i < azn->azn_ndfs; i++) { 243 amdzen_df_t *df = &azn->azn_dfs[i]; 244 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) { 245 continue; 246 } 247 248 if (dfno == 0) { 249 return (df); 250 } 251 dfno--; 252 } 253 254 return (NULL); 255 } 256 257 /* 258 * Client functions that are used by nexus children. 259 */ 260 int 261 amdzen_c_smn_read32(uint_t dfno, uint32_t reg, uint32_t *valp) 262 { 263 amdzen_df_t *df; 264 amdzen_t *azn = amdzen_data; 265 266 mutex_enter(&azn->azn_mutex); 267 df = amdzen_df_find(azn, dfno); 268 if (df == NULL) { 269 mutex_exit(&azn->azn_mutex); 270 return (ENOENT); 271 } 272 273 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 274 mutex_exit(&azn->azn_mutex); 275 return (ENXIO); 276 } 277 278 *valp = amdzen_smn_read32(azn, df, reg); 279 mutex_exit(&azn->azn_mutex); 280 return (0); 281 } 282 283 int 284 amdzen_c_smn_write32(uint_t dfno, uint32_t reg, uint32_t val) 285 { 286 amdzen_df_t *df; 287 amdzen_t *azn = amdzen_data; 288 289 mutex_enter(&azn->azn_mutex); 290 df = amdzen_df_find(azn, dfno); 291 if (df == NULL) { 292 mutex_exit(&azn->azn_mutex); 293 return (ENOENT); 294 } 295 296 if ((df->adf_flags & AMDZEN_DF_F_FOUND_NB) == 0) { 297 mutex_exit(&azn->azn_mutex); 298 return (ENXIO); 299 } 300 301 amdzen_smn_write32(azn, df, reg, val); 302 mutex_exit(&azn->azn_mutex); 303 return (0); 304 } 305 306 307 uint_t 308 amdzen_c_df_count(void) 309 { 310 uint_t ret; 311 amdzen_t *azn = amdzen_data; 312 313 mutex_enter(&azn->azn_mutex); 314 ret = azn->azn_ndfs; 315 mutex_exit(&azn->azn_mutex); 316 return (ret); 317 } 318 319 int 320 amdzen_c_df_read32(uint_t dfno, uint8_t inst, uint8_t func, 321 uint16_t reg, uint32_t *valp) 322 { 323 amdzen_df_t *df; 324 amdzen_t *azn = amdzen_data; 325 326 mutex_enter(&azn->azn_mutex); 327 df = amdzen_df_find(azn, dfno); 328 if (df == NULL) { 329 mutex_exit(&azn->azn_mutex); 330 return (ENOENT); 331 } 332 333 *valp = amdzen_df_read32(azn, df, inst, func, reg); 334 mutex_exit(&azn->azn_mutex); 335 336 return (0); 337 } 338 339 int 340 amdzen_c_df_read64(uint_t dfno, uint8_t inst, uint8_t func, 341 uint16_t reg, uint64_t *valp) 342 { 343 amdzen_df_t *df; 344 amdzen_t *azn = amdzen_data; 345 346 mutex_enter(&azn->azn_mutex); 347 df = amdzen_df_find(azn, dfno); 348 if (df == NULL) { 349 mutex_exit(&azn->azn_mutex); 350 return (ENOENT); 351 } 352 353 *valp = amdzen_df_read64(azn, df, inst, func, reg); 354 mutex_exit(&azn->azn_mutex); 355 356 return (0); 357 } 358 359 static boolean_t 360 amdzen_create_child(amdzen_t *azn, const amdzen_child_data_t *acd) 361 { 362 int ret; 363 dev_info_t *child; 364 365 if (ndi_devi_alloc(azn->azn_dip, acd->acd_name, 366 (pnode_t)DEVI_SID_NODEID, &child) != NDI_SUCCESS) { 367 dev_err(azn->azn_dip, CE_WARN, "!failed to allocate child " 368 "dip for %s", acd->acd_name); 369 return (B_FALSE); 370 } 371 372 ddi_set_parent_data(child, (void *)acd); 373 if ((ret = ndi_devi_online(child, 0)) != NDI_SUCCESS) { 374 dev_err(azn->azn_dip, CE_WARN, "!failed to online child " 375 "dip %s: %d", acd->acd_name, ret); 376 return (B_FALSE); 377 } 378 379 return (B_TRUE); 380 } 381 382 static boolean_t 383 amdzen_map_dfs(amdzen_t *azn) 384 { 385 amdzen_stub_t *stub; 386 387 ASSERT(MUTEX_HELD(&azn->azn_mutex)); 388 389 for (stub = list_head(&azn->azn_df_stubs); stub != NULL; 390 stub = list_next(&azn->azn_df_stubs, stub)) { 391 amdzen_df_t *df; 392 uint_t dfno; 393 394 dfno = stub->azns_dev - AMDZEN_DF_FIRST_DEVICE; 395 if (dfno > AMDZEN_MAX_DFS) { 396 dev_err(stub->azns_dip, CE_WARN, "encountered df " 397 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 398 stub->azns_bus, stub->azns_dev, stub->azns_func); 399 goto err; 400 } 401 402 df = &azn->azn_dfs[dfno]; 403 404 if (stub->azns_func >= AMDZEN_MAX_DF_FUNCS) { 405 dev_err(stub->azns_dip, CE_WARN, "encountered df " 406 "device with illegal DF PCI b/d/f: 0x%x/%x/%x", 407 stub->azns_bus, stub->azns_dev, stub->azns_func); 408 goto err; 409 } 410 411 if (df->adf_funcs[stub->azns_func] != NULL) { 412 dev_err(stub->azns_dip, CE_WARN, "encountered " 413 "duplicate df device with DF PCI b/d/f: 0x%x/%x/%x", 414 stub->azns_bus, stub->azns_dev, stub->azns_func); 415 goto err; 416 } 417 df->adf_funcs[stub->azns_func] = stub; 418 } 419 420 return (B_TRUE); 421 422 err: 423 azn->azn_flags |= AMDZEN_F_DEVICE_ERROR; 424 return (B_FALSE); 425 } 426 427 static boolean_t 428 amdzen_check_dfs(amdzen_t *azn) 429 { 430 uint_t i; 431 boolean_t ret = B_TRUE; 432 433 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 434 amdzen_df_t *df = &azn->azn_dfs[i]; 435 uint_t count = 0; 436 437 /* 438 * We require all platforms to have DFs functions 0-6. Not all 439 * platforms have DF function 7. 440 */ 441 for (uint_t func = 0; func < AMDZEN_MAX_DF_FUNCS - 1; func++) { 442 if (df->adf_funcs[func] != NULL) { 443 count++; 444 } 445 } 446 447 if (count == 0) 448 continue; 449 450 if (count != 7) { 451 ret = B_FALSE; 452 dev_err(azn->azn_dip, CE_WARN, "df %u devices " 453 "incomplete", i); 454 } else { 455 df->adf_flags |= AMDZEN_DF_F_VALID; 456 azn->azn_ndfs++; 457 } 458 } 459 460 return (ret); 461 } 462 463 static const uint8_t amdzen_df_rome_ids[0x2b] = { 464 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 16, 17, 18, 19, 20, 21, 22, 23, 465 24, 25, 26, 27, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 466 44, 45, 46, 47, 48 467 }; 468 469 /* 470 * Check the first df entry to see if it belongs to Rome or Milan. If so, then 471 * it uses the disjoint ID space. 472 */ 473 static boolean_t 474 amdzen_is_rome_style(uint_t id) 475 { 476 return (id == 0x1490 || id == 0x1650); 477 } 478 479 /* 480 * Initialize our knowledge about a given series of nodes on the data fabric. 481 */ 482 static void 483 amdzen_setup_df(amdzen_t *azn, amdzen_df_t *df) 484 { 485 uint_t i; 486 uint32_t val; 487 488 val = amdzen_stub_get32(df->adf_funcs[0], AMDZEN_DF_F0_CFG_ADDR_CTL); 489 df->adf_nb_busno = AMDZEN_DF_F0_CFG_ADDR_CTL_BUS_NUM(val); 490 val = amdzen_stub_get32(df->adf_funcs[0], AMDZEN_DF_F0_FBICNT); 491 df->adf_nents = AMDZEN_DF_F0_FBICNT_COUNT(val); 492 if (df->adf_nents == 0) 493 return; 494 df->adf_ents = kmem_zalloc(sizeof (amdzen_df_ent_t) * df->adf_nents, 495 KM_SLEEP); 496 497 for (i = 0; i < df->adf_nents; i++) { 498 amdzen_df_ent_t *dfe = &df->adf_ents[i]; 499 uint8_t inst = i; 500 501 /* 502 * Unfortunately, Rome uses a discontinuous instance ID pattern 503 * while everything else we can find uses a contiguous instance 504 * ID pattern. This means that for Rome, we need to adjust the 505 * indexes that we iterate over, though the total number of 506 * entries is right. 507 */ 508 if (amdzen_is_rome_style(df->adf_funcs[0]->azns_did)) { 509 if (inst > ARRAY_SIZE(amdzen_df_rome_ids)) { 510 dev_err(azn->azn_dip, CE_WARN, "Rome family " 511 "processor reported more ids than the PPR, " 512 "resetting %u to instance zero", inst); 513 inst = 0; 514 } else { 515 inst = amdzen_df_rome_ids[inst]; 516 } 517 } 518 519 dfe->adfe_drvid = inst; 520 dfe->adfe_info0 = amdzen_df_read32(azn, df, inst, 0, 521 AMDZEN_DF_F0_FBIINFO0); 522 dfe->adfe_info1 = amdzen_df_read32(azn, df, inst, 0, 523 AMDZEN_DF_F0_FBIINFO1); 524 dfe->adfe_info2 = amdzen_df_read32(azn, df, inst, 0, 525 AMDZEN_DF_F0_FBIINFO2); 526 dfe->adfe_info3 = amdzen_df_read32(azn, df, inst, 0, 527 AMDZEN_DF_F0_FBIINFO3); 528 dfe->adfe_syscfg = amdzen_df_read32(azn, df, inst, 1, 529 AMDZEN_DF_F1_SYSCFG); 530 dfe->adfe_mask0 = amdzen_df_read32(azn, df, inst, 1, 531 AMDZEN_DF_F1_FIDMASK0); 532 dfe->adfe_mask1 = amdzen_df_read32(azn, df, inst, 1, 533 AMDZEN_DF_F1_FIDMASK1); 534 535 dfe->adfe_type = AMDZEN_DF_F0_FBIINFO0_TYPE(dfe->adfe_info0); 536 dfe->adfe_sdp_width = 537 AMDZEN_DF_F0_FBIINFO0_SDP_WIDTH(dfe->adfe_info0); 538 if (AMDZEN_DF_F0_FBIINFO0_ENABLED(dfe->adfe_info0)) { 539 dfe->adfe_flags |= AMDZEN_DFE_F_ENABLED; 540 } 541 dfe->adfe_fti_width = 542 AMDZEN_DF_F0_FBIINFO0_FTI_WIDTH(dfe->adfe_info0); 543 dfe->adfe_sdp_count = 544 AMDZEN_DF_F0_FBIINFO0_SDP_PCOUNT(dfe->adfe_info0); 545 dfe->adfe_fti_count = 546 AMDZEN_DF_F0_FBIINFO0_FTI_PCOUNT(dfe->adfe_info0); 547 if (AMDZEN_DF_F0_FBIINFO0_HAS_MCA(dfe->adfe_info0)) { 548 dfe->adfe_flags |= AMDZEN_DFE_F_MCA; 549 } 550 dfe->adfe_subtype = 551 AMDZEN_DF_F0_FBIINFO0_SUBTYPE(dfe->adfe_info0); 552 553 dfe->adfe_inst_id = 554 AMDZEN_DF_F0_FBIINFO3_INSTID(dfe->adfe_info3); 555 dfe->adfe_fabric_id = 556 AMDZEN_DF_F0_FBIINFO3_FABID(dfe->adfe_info3); 557 } 558 559 df->adf_syscfg = amdzen_stub_get32(df->adf_funcs[1], 560 AMDZEN_DF_F1_SYSCFG); 561 df->adf_nodeid = AMDZEN_DF_F1_SYSCFG_NODEID(df->adf_syscfg); 562 df->adf_mask0 = amdzen_stub_get32(df->adf_funcs[1], 563 AMDZEN_DF_F1_FIDMASK0); 564 df->adf_mask1 = amdzen_stub_get32(df->adf_funcs[1], 565 AMDZEN_DF_F1_FIDMASK1); 566 } 567 568 static void 569 amdzen_find_nb(amdzen_t *azn, amdzen_df_t *df) 570 { 571 amdzen_stub_t *stub; 572 573 for (stub = list_head(&azn->azn_nb_stubs); stub != NULL; 574 stub = list_next(&azn->azn_nb_stubs, stub)) { 575 if (stub->azns_bus == df->adf_nb_busno) { 576 df->adf_flags |= AMDZEN_DF_F_FOUND_NB; 577 df->adf_nb = stub; 578 return; 579 } 580 } 581 } 582 583 static void 584 amdzen_nexus_init(void *arg) 585 { 586 uint_t i; 587 amdzen_t *azn = arg; 588 589 /* 590 * First go through all of the stubs and assign the DF entries. 591 */ 592 mutex_enter(&azn->azn_mutex); 593 if (!amdzen_map_dfs(azn) || !amdzen_check_dfs(azn)) { 594 azn->azn_flags |= AMDZEN_F_MAP_ERROR; 595 goto done; 596 } 597 598 for (i = 0; i < AMDZEN_MAX_DFS; i++) { 599 amdzen_df_t *df = &azn->azn_dfs[i]; 600 601 if ((df->adf_flags & AMDZEN_DF_F_VALID) == 0) 602 continue; 603 amdzen_setup_df(azn, df); 604 amdzen_find_nb(azn, df); 605 } 606 607 /* 608 * Not all children may be installed. As such, we do not treat the 609 * failure of a child as fatal to the driver. 610 */ 611 mutex_exit(&azn->azn_mutex); 612 for (i = 0; i < ARRAY_SIZE(amdzen_children); i++) { 613 (void) amdzen_create_child(azn, &amdzen_children[i]); 614 } 615 mutex_enter(&azn->azn_mutex); 616 617 done: 618 azn->azn_flags &= ~AMDZEN_F_ATTACH_DISPATCHED; 619 azn->azn_flags |= AMDZEN_F_ATTACH_COMPLETE; 620 azn->azn_taskqid = TASKQID_INVALID; 621 cv_broadcast(&azn->azn_cv); 622 mutex_exit(&azn->azn_mutex); 623 } 624 625 static int 626 amdzen_stub_scan_cb(dev_info_t *dip, void *arg) 627 { 628 amdzen_t *azn = arg; 629 uint16_t vid, did; 630 int *regs; 631 uint_t nregs, i; 632 boolean_t match = B_FALSE; 633 634 if (dip == ddi_root_node()) { 635 return (DDI_WALK_CONTINUE); 636 } 637 638 /* 639 * If a node in question is not a pci node, then we have no interest in 640 * it as all the stubs that we care about are related to pci devices. 641 */ 642 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 643 return (DDI_WALK_PRUNECHILD); 644 } 645 646 /* 647 * If we can't get a device or vendor ID and prove that this is an AMD 648 * part, then we don't care about it. 649 */ 650 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 651 "vendor-id", PCI_EINVAL16); 652 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 653 "device-id", PCI_EINVAL16); 654 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 655 return (DDI_WALK_CONTINUE); 656 } 657 658 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 659 return (DDI_WALK_CONTINUE); 660 } 661 662 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 663 if (amdzen_nb_ids[i] == did) { 664 match = B_TRUE; 665 } 666 } 667 668 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 669 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 670 return (DDI_WALK_CONTINUE); 671 } 672 673 if (nregs == 0) { 674 ddi_prop_free(regs); 675 return (DDI_WALK_CONTINUE); 676 } 677 678 if (PCI_REG_BUS_G(regs[0]) == AMDZEN_DF_BUSNO && 679 PCI_REG_DEV_G(regs[0]) >= AMDZEN_DF_FIRST_DEVICE) { 680 match = B_TRUE; 681 } 682 683 ddi_prop_free(regs); 684 if (match) { 685 mutex_enter(&azn->azn_mutex); 686 azn->azn_nscanned++; 687 mutex_exit(&azn->azn_mutex); 688 } 689 690 return (DDI_WALK_CONTINUE); 691 } 692 693 static void 694 amdzen_stub_scan(void *arg) 695 { 696 amdzen_t *azn = arg; 697 698 mutex_enter(&azn->azn_mutex); 699 azn->azn_nscanned = 0; 700 mutex_exit(&azn->azn_mutex); 701 702 ddi_walk_devs(ddi_root_node(), amdzen_stub_scan_cb, azn); 703 704 mutex_enter(&azn->azn_mutex); 705 azn->azn_flags &= ~AMDZEN_F_SCAN_DISPATCHED; 706 azn->azn_flags |= AMDZEN_F_SCAN_COMPLETE; 707 708 if (azn->azn_nscanned == 0) { 709 azn->azn_flags |= AMDZEN_F_UNSUPPORTED; 710 azn->azn_taskqid = TASKQID_INVALID; 711 cv_broadcast(&azn->azn_cv); 712 } else if (azn->azn_npresent == azn->azn_nscanned) { 713 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 714 azn->azn_taskqid = taskq_dispatch(system_taskq, 715 amdzen_nexus_init, azn, TQ_SLEEP); 716 } 717 mutex_exit(&azn->azn_mutex); 718 } 719 720 /* 721 * Unfortunately we can't really let the stubs detach as we may need them to be 722 * available for client operations. We may be able to improve this if we know 723 * that the actual nexus is going away. However, as long as it's active, we need 724 * all the stubs. 725 */ 726 int 727 amdzen_detach_stub(dev_info_t *dip, ddi_detach_cmd_t cmd) 728 { 729 if (cmd == DDI_SUSPEND) { 730 return (DDI_SUCCESS); 731 } 732 733 return (DDI_FAILURE); 734 } 735 736 int 737 amdzen_attach_stub(dev_info_t *dip, ddi_attach_cmd_t cmd) 738 { 739 int *regs, reg; 740 uint_t nregs, i; 741 uint16_t vid, did; 742 amdzen_stub_t *stub; 743 amdzen_t *azn = amdzen_data; 744 boolean_t valid = B_FALSE; 745 boolean_t nb = B_FALSE; 746 747 if (cmd == DDI_RESUME) { 748 return (DDI_SUCCESS); 749 } else if (cmd != DDI_ATTACH) { 750 return (DDI_FAILURE); 751 } 752 753 /* 754 * Make sure that the stub that we've been asked to attach is a pci type 755 * device. If not, then there is no reason for us to proceed. 756 */ 757 if (strncmp("pci", ddi_get_name(dip), 3) != 0) { 758 dev_err(dip, CE_WARN, "asked to attach a bad AMD Zen nexus " 759 "stub: %s", ddi_get_name(dip)); 760 return (DDI_FAILURE); 761 } 762 vid = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 763 "vendor-id", PCI_EINVAL16); 764 did = ddi_prop_get_int(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 765 "device-id", PCI_EINVAL16); 766 if (vid == PCI_EINVAL16 || did == PCI_EINVAL16) { 767 dev_err(dip, CE_WARN, "failed to get PCI ID properties"); 768 return (DDI_FAILURE); 769 } 770 771 if (vid != AMDZEN_PCI_VID_AMD && vid != AMDZEN_PCI_VID_HYGON) { 772 dev_err(dip, CE_WARN, "expected vendor ID (0x%x), found 0x%x", 773 cpuid_getvendor(CPU) == X86_VENDOR_HYGON ? 774 AMDZEN_PCI_VID_HYGON : AMDZEN_PCI_VID_AMD, vid); 775 return (DDI_FAILURE); 776 } 777 778 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, DDI_PROP_DONTPASS, 779 "reg", ®s, &nregs) != DDI_PROP_SUCCESS) { 780 dev_err(dip, CE_WARN, "failed to get 'reg' property"); 781 return (DDI_FAILURE); 782 } 783 784 if (nregs == 0) { 785 ddi_prop_free(regs); 786 dev_err(dip, CE_WARN, "missing 'reg' property values"); 787 return (DDI_FAILURE); 788 } 789 reg = *regs; 790 ddi_prop_free(regs); 791 792 for (i = 0; i < ARRAY_SIZE(amdzen_nb_ids); i++) { 793 if (amdzen_nb_ids[i] == did) { 794 valid = B_TRUE; 795 nb = B_TRUE; 796 } 797 } 798 799 if (!valid && PCI_REG_BUS_G(reg) == AMDZEN_DF_BUSNO && 800 PCI_REG_DEV_G(reg) >= AMDZEN_DF_FIRST_DEVICE) { 801 valid = B_TRUE; 802 nb = B_FALSE; 803 } 804 805 if (!valid) { 806 dev_err(dip, CE_WARN, "device %s didn't match the nexus list", 807 ddi_get_name(dip)); 808 return (DDI_FAILURE); 809 } 810 811 stub = kmem_alloc(sizeof (amdzen_stub_t), KM_SLEEP); 812 if (pci_config_setup(dip, &stub->azns_cfgspace) != DDI_SUCCESS) { 813 dev_err(dip, CE_WARN, "failed to set up config space"); 814 kmem_free(stub, sizeof (amdzen_stub_t)); 815 return (DDI_FAILURE); 816 } 817 818 stub->azns_dip = dip; 819 stub->azns_vid = vid; 820 stub->azns_did = did; 821 stub->azns_bus = PCI_REG_BUS_G(reg); 822 stub->azns_dev = PCI_REG_DEV_G(reg); 823 stub->azns_func = PCI_REG_FUNC_G(reg); 824 ddi_set_driver_private(dip, stub); 825 826 mutex_enter(&azn->azn_mutex); 827 azn->azn_npresent++; 828 if (nb) { 829 list_insert_tail(&azn->azn_nb_stubs, stub); 830 } else { 831 list_insert_tail(&azn->azn_df_stubs, stub); 832 } 833 834 if ((azn->azn_flags & AMDZEN_F_TASKQ_MASK) == AMDZEN_F_SCAN_COMPLETE && 835 azn->azn_nscanned == azn->azn_npresent) { 836 azn->azn_flags |= AMDZEN_F_ATTACH_DISPATCHED; 837 azn->azn_taskqid = taskq_dispatch(system_taskq, 838 amdzen_nexus_init, azn, TQ_SLEEP); 839 } 840 mutex_exit(&azn->azn_mutex); 841 842 return (DDI_SUCCESS); 843 } 844 845 static int 846 amdzen_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 847 void *arg, void *result) 848 { 849 char buf[32]; 850 dev_info_t *child; 851 const amdzen_child_data_t *acd; 852 853 switch (ctlop) { 854 case DDI_CTLOPS_REPORTDEV: 855 if (rdip == NULL) { 856 return (DDI_FAILURE); 857 } 858 cmn_err(CE_CONT, "amdzen nexus: %s@%s, %s%d\n", 859 ddi_node_name(rdip), ddi_get_name_addr(rdip), 860 ddi_driver_name(rdip), ddi_get_instance(rdip)); 861 break; 862 case DDI_CTLOPS_INITCHILD: 863 child = arg; 864 if (child == NULL) { 865 dev_err(dip, CE_WARN, "!no child passed for " 866 "DDI_CTLOPS_INITCHILD"); 867 } 868 869 acd = ddi_get_parent_data(child); 870 if (acd == NULL) { 871 dev_err(dip, CE_WARN, "!missing child parent data"); 872 return (DDI_FAILURE); 873 } 874 875 if (snprintf(buf, sizeof (buf), "%d", acd->acd_addr) >= 876 sizeof (buf)) { 877 dev_err(dip, CE_WARN, "!failed to construct device " 878 "addr due to overflow"); 879 return (DDI_FAILURE); 880 } 881 882 ddi_set_name_addr(child, buf); 883 break; 884 case DDI_CTLOPS_UNINITCHILD: 885 child = arg; 886 if (child == NULL) { 887 dev_err(dip, CE_WARN, "!no child passed for " 888 "DDI_CTLOPS_UNINITCHILD"); 889 } 890 891 ddi_set_name_addr(child, NULL); 892 break; 893 default: 894 return (ddi_ctlops(dip, rdip, ctlop, arg, result)); 895 } 896 return (DDI_SUCCESS); 897 } 898 899 static int 900 amdzen_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 901 { 902 amdzen_t *azn = amdzen_data; 903 904 if (cmd == DDI_RESUME) { 905 return (DDI_SUCCESS); 906 } else if (cmd != DDI_ATTACH) { 907 return (DDI_FAILURE); 908 } 909 910 mutex_enter(&azn->azn_mutex); 911 if (azn->azn_dip != NULL) { 912 dev_err(dip, CE_WARN, "driver is already attached!"); 913 mutex_exit(&azn->azn_mutex); 914 return (DDI_FAILURE); 915 } 916 917 azn->azn_dip = dip; 918 azn->azn_taskqid = taskq_dispatch(system_taskq, amdzen_stub_scan, 919 azn, TQ_SLEEP); 920 azn->azn_flags |= AMDZEN_F_SCAN_DISPATCHED; 921 mutex_exit(&azn->azn_mutex); 922 923 return (DDI_SUCCESS); 924 } 925 926 static int 927 amdzen_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 928 { 929 amdzen_t *azn = amdzen_data; 930 931 if (cmd == DDI_SUSPEND) { 932 return (DDI_SUCCESS); 933 } else if (cmd != DDI_DETACH) { 934 return (DDI_FAILURE); 935 } 936 937 mutex_enter(&azn->azn_mutex); 938 while (azn->azn_taskqid != TASKQID_INVALID) { 939 cv_wait(&azn->azn_cv, &azn->azn_mutex); 940 } 941 942 /* 943 * If we've attached any stub drivers, e.g. this platform is important 944 * for us, then we fail detach. 945 */ 946 if (!list_is_empty(&azn->azn_df_stubs) || 947 !list_is_empty(&azn->azn_nb_stubs)) { 948 mutex_exit(&azn->azn_mutex); 949 return (DDI_FAILURE); 950 } 951 952 azn->azn_dip = NULL; 953 mutex_exit(&azn->azn_mutex); 954 955 return (DDI_SUCCESS); 956 } 957 958 static void 959 amdzen_free(void) 960 { 961 if (amdzen_data == NULL) { 962 return; 963 } 964 965 VERIFY(list_is_empty(&amdzen_data->azn_df_stubs)); 966 list_destroy(&amdzen_data->azn_df_stubs); 967 VERIFY(list_is_empty(&amdzen_data->azn_nb_stubs)); 968 list_destroy(&amdzen_data->azn_nb_stubs); 969 cv_destroy(&amdzen_data->azn_cv); 970 mutex_destroy(&amdzen_data->azn_mutex); 971 kmem_free(amdzen_data, sizeof (amdzen_t)); 972 amdzen_data = NULL; 973 } 974 975 static void 976 amdzen_alloc(void) 977 { 978 amdzen_data = kmem_zalloc(sizeof (amdzen_t), KM_SLEEP); 979 mutex_init(&amdzen_data->azn_mutex, NULL, MUTEX_DRIVER, NULL); 980 list_create(&amdzen_data->azn_df_stubs, sizeof (amdzen_stub_t), 981 offsetof(amdzen_stub_t, azns_link)); 982 list_create(&amdzen_data->azn_nb_stubs, sizeof (amdzen_stub_t), 983 offsetof(amdzen_stub_t, azns_link)); 984 cv_init(&amdzen_data->azn_cv, NULL, CV_DRIVER, NULL); 985 } 986 987 struct bus_ops amdzen_bus_ops = { 988 .busops_rev = BUSO_REV, 989 .bus_map = nullbusmap, 990 .bus_dma_map = ddi_no_dma_map, 991 .bus_dma_allochdl = ddi_no_dma_allochdl, 992 .bus_dma_freehdl = ddi_no_dma_freehdl, 993 .bus_dma_bindhdl = ddi_no_dma_bindhdl, 994 .bus_dma_unbindhdl = ddi_no_dma_unbindhdl, 995 .bus_dma_flush = ddi_no_dma_flush, 996 .bus_dma_win = ddi_no_dma_win, 997 .bus_dma_ctl = ddi_no_dma_mctl, 998 .bus_prop_op = ddi_bus_prop_op, 999 .bus_ctl = amdzen_bus_ctl 1000 }; 1001 1002 static struct dev_ops amdzen_dev_ops = { 1003 .devo_rev = DEVO_REV, 1004 .devo_refcnt = 0, 1005 .devo_getinfo = nodev, 1006 .devo_identify = nulldev, 1007 .devo_probe = nulldev, 1008 .devo_attach = amdzen_attach, 1009 .devo_detach = amdzen_detach, 1010 .devo_reset = nodev, 1011 .devo_quiesce = ddi_quiesce_not_needed, 1012 .devo_bus_ops = &amdzen_bus_ops 1013 }; 1014 1015 static struct modldrv amdzen_modldrv = { 1016 .drv_modops = &mod_driverops, 1017 .drv_linkinfo = "AMD Zen Nexus Driver", 1018 .drv_dev_ops = &amdzen_dev_ops 1019 }; 1020 1021 static struct modlinkage amdzen_modlinkage = { 1022 .ml_rev = MODREV_1, 1023 .ml_linkage = { &amdzen_modldrv, NULL } 1024 }; 1025 1026 int 1027 _init(void) 1028 { 1029 int ret; 1030 1031 if (cpuid_getvendor(CPU) != X86_VENDOR_AMD && 1032 cpuid_getvendor(CPU) != X86_VENDOR_HYGON) { 1033 return (ENOTSUP); 1034 } 1035 1036 if ((ret = mod_install(&amdzen_modlinkage)) == 0) { 1037 amdzen_alloc(); 1038 } 1039 1040 return (ret); 1041 } 1042 1043 int 1044 _info(struct modinfo *modinfop) 1045 { 1046 return (mod_info(&amdzen_modlinkage, modinfop)); 1047 } 1048 1049 int 1050 _fini(void) 1051 { 1052 int ret; 1053 1054 if ((ret = mod_remove(&amdzen_modlinkage)) == 0) { 1055 amdzen_free(); 1056 } 1057 1058 return (ret); 1059 } 1060