1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor.c 29 * Tavor (InfiniBand) HCA Driver attach/detach Routines 30 * 31 * Implements all the routines necessary for the attach, setup, 32 * initialization (and subsequent possible teardown and detach) of the 33 * Tavor InfiniBand HCA driver. 34 */ 35 36 #include <sys/types.h> 37 #include <sys/file.h> 38 #include <sys/open.h> 39 #include <sys/conf.h> 40 #include <sys/ddi.h> 41 #include <sys/sunddi.h> 42 #include <sys/modctl.h> 43 #include <sys/stat.h> 44 #include <sys/pci.h> 45 #include <sys/pci_cap.h> 46 #include <sys/bitmap.h> 47 #include <sys/policy.h> 48 49 #include <sys/ib/adapters/tavor/tavor.h> 50 #include <sys/pci.h> 51 52 /* Tavor HCA State Pointer */ 53 void *tavor_statep; 54 55 /* 56 * The Tavor "userland resource database" is common to instances of the 57 * Tavor HCA driver. This structure "tavor_userland_rsrc_db" contains all 58 * the necessary information to maintain it. 59 */ 60 tavor_umap_db_t tavor_userland_rsrc_db; 61 62 static int tavor_attach(dev_info_t *, ddi_attach_cmd_t); 63 static int tavor_detach(dev_info_t *, ddi_detach_cmd_t); 64 static int tavor_open(dev_t *, int, int, cred_t *); 65 static int tavor_close(dev_t, int, int, cred_t *); 66 static int tavor_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 67 static int tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance); 68 static void tavor_drv_fini(tavor_state_t *state); 69 static void tavor_drv_fini2(tavor_state_t *state); 70 static int tavor_isr_init(tavor_state_t *state); 71 static void tavor_isr_fini(tavor_state_t *state); 72 static int tavor_hw_init(tavor_state_t *state); 73 static void tavor_hw_fini(tavor_state_t *state, 74 tavor_drv_cleanup_level_t cleanup); 75 static int tavor_soft_state_init(tavor_state_t *state); 76 static void tavor_soft_state_fini(tavor_state_t *state); 77 static int tavor_hca_port_init(tavor_state_t *state); 78 static int tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init); 79 static void tavor_hca_config_setup(tavor_state_t *state, 80 tavor_hw_initqueryhca_t *inithca); 81 static int tavor_internal_uarpgs_init(tavor_state_t *state); 82 static void tavor_internal_uarpgs_fini(tavor_state_t *state); 83 static int tavor_special_qp_contexts_reserve(tavor_state_t *state); 84 static void tavor_special_qp_contexts_unreserve(tavor_state_t *state); 85 static int tavor_sw_reset(tavor_state_t *state); 86 static int tavor_mcg_init(tavor_state_t *state); 87 static void tavor_mcg_fini(tavor_state_t *state); 88 static int tavor_fw_version_check(tavor_state_t *state); 89 static void tavor_device_info_report(tavor_state_t *state); 90 static void tavor_pci_capability_list(tavor_state_t *state, 91 ddi_acc_handle_t hdl); 92 static void tavor_pci_capability_vpd(tavor_state_t *state, 93 ddi_acc_handle_t hdl, uint_t offset); 94 static int tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, 95 uint32_t addr, uint32_t *data); 96 static void tavor_pci_capability_pcix(tavor_state_t *state, 97 ddi_acc_handle_t hdl, uint_t offset); 98 static int tavor_intr_or_msi_init(tavor_state_t *state); 99 static int tavor_add_intrs(tavor_state_t *state, int intr_type); 100 static int tavor_intr_or_msi_fini(tavor_state_t *state); 101 102 /* X86 fastreboot support */ 103 static int tavor_intr_disable(tavor_state_t *); 104 static int tavor_quiesce(dev_info_t *); 105 106 /* Character/Block Operations */ 107 static struct cb_ops tavor_cb_ops = { 108 tavor_open, /* open */ 109 tavor_close, /* close */ 110 nodev, /* strategy (block) */ 111 nodev, /* print (block) */ 112 nodev, /* dump (block) */ 113 nodev, /* read */ 114 nodev, /* write */ 115 tavor_ioctl, /* ioctl */ 116 tavor_devmap, /* devmap */ 117 NULL, /* mmap */ 118 nodev, /* segmap */ 119 nochpoll, /* chpoll */ 120 ddi_prop_op, /* prop_op */ 121 NULL, /* streams */ 122 D_NEW | D_MP | 123 D_64BIT | D_HOTPLUG | 124 D_DEVMAP, /* flags */ 125 CB_REV /* rev */ 126 }; 127 128 /* Driver Operations */ 129 static struct dev_ops tavor_ops = { 130 DEVO_REV, /* struct rev */ 131 0, /* refcnt */ 132 tavor_getinfo, /* getinfo */ 133 nulldev, /* identify */ 134 nulldev, /* probe */ 135 tavor_attach, /* attach */ 136 tavor_detach, /* detach */ 137 nodev, /* reset */ 138 &tavor_cb_ops, /* cb_ops */ 139 NULL, /* bus_ops */ 140 nodev, /* power */ 141 tavor_quiesce, /* devo_quiesce */ 142 }; 143 144 /* Module Driver Info */ 145 static struct modldrv tavor_modldrv = { 146 &mod_driverops, 147 "Tavor InfiniBand HCA Driver", 148 &tavor_ops 149 }; 150 151 /* Module Linkage */ 152 static struct modlinkage tavor_modlinkage = { 153 MODREV_1, 154 &tavor_modldrv, 155 NULL 156 }; 157 158 /* 159 * This extern refers to the ibc_operations_t function vector that is defined 160 * in the tavor_ci.c file. 161 */ 162 extern ibc_operations_t tavor_ibc_ops; 163 164 #ifndef NPROBE 165 extern int tnf_mod_load(void); 166 extern int tnf_mod_unload(struct modlinkage *mlp); 167 #endif 168 169 170 /* 171 * _init() 172 */ 173 int 174 _init() 175 { 176 int status; 177 178 #ifndef NPROBE 179 (void) tnf_mod_load(); 180 #endif 181 TAVOR_TNF_ENTER(tavor_init); 182 183 status = ddi_soft_state_init(&tavor_statep, sizeof (tavor_state_t), 184 (size_t)TAVOR_INITIAL_STATES); 185 if (status != 0) { 186 TNF_PROBE_0(tavor_init_ssi_fail, TAVOR_TNF_ERROR, ""); 187 TAVOR_TNF_EXIT(tavor_init); 188 #ifndef NPROBE 189 (void) tnf_mod_unload(&tavor_modlinkage); 190 #endif 191 return (status); 192 } 193 194 status = ibc_init(&tavor_modlinkage); 195 if (status != 0) { 196 TNF_PROBE_0(tavor_init_ibc_init_fail, TAVOR_TNF_ERROR, ""); 197 ddi_soft_state_fini(&tavor_statep); 198 TAVOR_TNF_EXIT(tavor_init); 199 #ifndef NPROBE 200 (void) tnf_mod_unload(&tavor_modlinkage); 201 #endif 202 return (status); 203 } 204 status = mod_install(&tavor_modlinkage); 205 if (status != 0) { 206 TNF_PROBE_0(tavor_init_modi_fail, TAVOR_TNF_ERROR, ""); 207 ibc_fini(&tavor_modlinkage); 208 ddi_soft_state_fini(&tavor_statep); 209 TAVOR_TNF_EXIT(tavor_init); 210 #ifndef NPROBE 211 (void) tnf_mod_unload(&tavor_modlinkage); 212 #endif 213 return (status); 214 } 215 216 /* Initialize the Tavor "userland resources database" */ 217 tavor_umap_db_init(); 218 219 TAVOR_TNF_EXIT(tavor_init); 220 return (status); 221 } 222 223 224 /* 225 * _info() 226 */ 227 int 228 _info(struct modinfo *modinfop) 229 { 230 int status; 231 232 TAVOR_TNF_ENTER(tavor_info); 233 status = mod_info(&tavor_modlinkage, modinfop); 234 TAVOR_TNF_EXIT(tavor_info); 235 return (status); 236 } 237 238 239 /* 240 * _fini() 241 */ 242 int 243 _fini() 244 { 245 int status; 246 247 TAVOR_TNF_ENTER(tavor_fini); 248 249 status = mod_remove(&tavor_modlinkage); 250 if (status != 0) { 251 TNF_PROBE_0(tavor_fini_modr_fail, TAVOR_TNF_ERROR, ""); 252 TAVOR_TNF_EXIT(tavor_fini); 253 return (status); 254 } 255 256 /* Destroy the Tavor "userland resources database" */ 257 tavor_umap_db_fini(); 258 259 ibc_fini(&tavor_modlinkage); 260 ddi_soft_state_fini(&tavor_statep); 261 #ifndef NPROBE 262 (void) tnf_mod_unload(&tavor_modlinkage); 263 #endif 264 TAVOR_TNF_EXIT(tavor_fini); 265 return (status); 266 } 267 268 269 /* 270 * tavor_getinfo() 271 */ 272 /* ARGSUSED */ 273 static int 274 tavor_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 275 { 276 dev_t dev; 277 tavor_state_t *state; 278 minor_t instance; 279 280 TAVOR_TNF_ENTER(tavor_getinfo); 281 282 switch (cmd) { 283 case DDI_INFO_DEVT2DEVINFO: 284 dev = (dev_t)arg; 285 instance = TAVOR_DEV_INSTANCE(dev); 286 state = ddi_get_soft_state(tavor_statep, instance); 287 if (state == NULL) { 288 TNF_PROBE_0(tavor_getinfo_gss_fail, 289 TAVOR_TNF_ERROR, ""); 290 TAVOR_TNF_EXIT(tavor_getinfo); 291 return (DDI_FAILURE); 292 } 293 *result = (void *)state->ts_dip; 294 return (DDI_SUCCESS); 295 296 case DDI_INFO_DEVT2INSTANCE: 297 dev = (dev_t)arg; 298 instance = TAVOR_DEV_INSTANCE(dev); 299 *result = (void *)(uintptr_t)instance; 300 return (DDI_SUCCESS); 301 302 default: 303 TNF_PROBE_0(tavor_getinfo_default_fail, TAVOR_TNF_ERROR, ""); 304 break; 305 } 306 307 TAVOR_TNF_EXIT(tavor_getinfo); 308 return (DDI_FAILURE); 309 } 310 311 312 /* 313 * tavor_open() 314 */ 315 /* ARGSUSED */ 316 static int 317 tavor_open(dev_t *devp, int flag, int otyp, cred_t *credp) 318 { 319 tavor_state_t *state; 320 tavor_rsrc_t *rsrcp; 321 tavor_umap_db_entry_t *umapdb, *umapdb2; 322 minor_t instance; 323 uint64_t key, value; 324 uint_t tr_indx; 325 dev_t dev; 326 int status; 327 328 TAVOR_TNF_ENTER(tavor_open); 329 330 instance = TAVOR_DEV_INSTANCE(*devp); 331 state = ddi_get_soft_state(tavor_statep, instance); 332 if (state == NULL) { 333 TNF_PROBE_0(tavor_open_gss_fail, TAVOR_TNF_ERROR, ""); 334 TAVOR_TNF_EXIT(tavor_open); 335 return (ENXIO); 336 } 337 338 /* 339 * Only allow driver to be opened for character access, and verify 340 * whether exclusive access is allowed. 341 */ 342 if ((otyp != OTYP_CHR) || ((flag & FEXCL) && 343 secpolicy_excl_open(credp) != 0)) { 344 TNF_PROBE_0(tavor_open_invflags_fail, TAVOR_TNF_ERROR, ""); 345 TAVOR_TNF_EXIT(tavor_open); 346 return (EINVAL); 347 } 348 349 /* 350 * Search for the current process PID in the "userland resources 351 * database". If it is not found, then attempt to allocate a UAR 352 * page and add the ("key", "value") pair to the database. 353 * Note: As a last step we always return a devp appropriate for 354 * the open. Either we return a new minor number (based on the 355 * instance and the UAR page index) or we return the current minor 356 * number for the given client process. 357 * 358 * We also add an entry to the database to allow for lookup from 359 * "dev_t" to the current process PID. This is necessary because, 360 * under certain circumstance, the process PID that calls the Tavor 361 * close() entry point may not be the same as the one who called 362 * open(). Specifically, this can happen if a child process calls 363 * the Tavor's open() entry point, gets a UAR page, maps it out (using 364 * mmap()), and then exits without calling munmap(). Because mmap() 365 * adds a reference to the file descriptor, at the exit of the child 366 * process the file descriptor is "inherited" by the parent (and will 367 * be close()'d by the parent's PID only when it exits). 368 * 369 * Note: We use the tavor_umap_db_find_nolock() and 370 * tavor_umap_db_add_nolock() database access routines below (with 371 * an explicit mutex_enter of the database lock - "tdl_umapdb_lock") 372 * to ensure that the multiple accesses (in this case searching for, 373 * and then adding _two_ database entries) can be done atomically. 374 */ 375 key = ddi_get_pid(); 376 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); 377 status = tavor_umap_db_find_nolock(instance, key, 378 MLNX_UMAP_UARPG_RSRC, &value, 0, NULL); 379 if (status != DDI_SUCCESS) { 380 /* 381 * If we are in 'maintenance mode', we cannot alloc a UAR page. 382 * But we still need some rsrcp value, and a mostly unique 383 * tr_indx value. So we set rsrcp to NULL for maintenance 384 * mode, and use a rolling count for tr_indx. The field 385 * 'ts_open_tr_indx' is used only in this maintenance mode 386 * condition. 387 * 388 * Otherwise, if we are in operational mode then we allocate 389 * the UAR page as normal, and use the rsrcp value and tr_indx 390 * value from that allocation. 391 */ 392 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 393 rsrcp = NULL; 394 tr_indx = state->ts_open_tr_indx++; 395 } else { 396 /* Allocate a new UAR page for this process */ 397 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, 398 TAVOR_NOSLEEP, &rsrcp); 399 if (status != DDI_SUCCESS) { 400 mutex_exit( 401 &tavor_userland_rsrc_db.tdl_umapdb_lock); 402 TNF_PROBE_0(tavor_open_rsrcalloc_uarpg_fail, 403 TAVOR_TNF_ERROR, ""); 404 TAVOR_TNF_EXIT(tavor_open); 405 return (EAGAIN); 406 } 407 408 tr_indx = rsrcp->tr_indx; 409 } 410 411 /* 412 * Allocate an entry to track the UAR page resource in the 413 * "userland resources database". 414 */ 415 umapdb = tavor_umap_db_alloc(instance, key, 416 MLNX_UMAP_UARPG_RSRC, (uint64_t)(uintptr_t)rsrcp); 417 if (umapdb == NULL) { 418 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 419 /* If in "maintenance mode", don't free the rsrc */ 420 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 421 tavor_rsrc_free(state, &rsrcp); 422 } 423 TNF_PROBE_0(tavor_open_umap_db_alloc_fail, 424 TAVOR_TNF_ERROR, ""); 425 TAVOR_TNF_EXIT(tavor_open); 426 return (EAGAIN); 427 } 428 429 /* 430 * Create a new device number. Minor number is a function of 431 * the UAR page index (15 bits) and the device instance number 432 * (3 bits). 433 */ 434 dev = makedevice(getmajor(*devp), (tr_indx << 435 TAVOR_MINORNUM_SHIFT) | instance); 436 437 /* 438 * Allocate another entry in the "userland resources database" 439 * to track the association of the device number (above) to 440 * the current process ID (in "key"). 441 */ 442 umapdb2 = tavor_umap_db_alloc(instance, dev, 443 MLNX_UMAP_PID_RSRC, (uint64_t)key); 444 if (umapdb2 == NULL) { 445 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 446 tavor_umap_db_free(umapdb); 447 /* If in "maintenance mode", don't free the rsrc */ 448 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 449 tavor_rsrc_free(state, &rsrcp); 450 } 451 TNF_PROBE_0(tavor_open_umap_db_alloc_fail, 452 TAVOR_TNF_ERROR, ""); 453 TAVOR_TNF_EXIT(tavor_open); 454 return (EAGAIN); 455 } 456 457 /* Add the entries to the database */ 458 tavor_umap_db_add_nolock(umapdb); 459 tavor_umap_db_add_nolock(umapdb2); 460 461 } else { 462 /* 463 * Return the same device number as on the original open() 464 * call. This was calculated as a function of the UAR page 465 * index (top 16 bits) and the device instance number 466 */ 467 rsrcp = (tavor_rsrc_t *)(uintptr_t)value; 468 dev = makedevice(getmajor(*devp), (rsrcp->tr_indx << 469 TAVOR_MINORNUM_SHIFT) | instance); 470 } 471 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 472 473 *devp = dev; 474 475 TAVOR_TNF_EXIT(tavor_open); 476 return (0); 477 } 478 479 480 /* 481 * tavor_close() 482 */ 483 /* ARGSUSED */ 484 static int 485 tavor_close(dev_t dev, int flag, int otyp, cred_t *credp) 486 { 487 tavor_state_t *state; 488 tavor_rsrc_t *rsrcp; 489 tavor_umap_db_entry_t *umapdb; 490 tavor_umap_db_priv_t *priv; 491 minor_t instance; 492 uint64_t key, value; 493 int status; 494 495 TAVOR_TNF_ENTER(tavor_close); 496 497 instance = TAVOR_DEV_INSTANCE(dev); 498 state = ddi_get_soft_state(tavor_statep, instance); 499 if (state == NULL) { 500 TNF_PROBE_0(tavor_close_gss_fail, TAVOR_TNF_ERROR, ""); 501 TAVOR_TNF_EXIT(tavor_close); 502 return (ENXIO); 503 } 504 505 /* 506 * Search for "dev_t" in the "userland resources database". As 507 * explained above in tavor_open(), we can't depend on using the 508 * current process ID here to do the lookup because the process 509 * that ultimately closes may not be the same one who opened 510 * (because of inheritance). 511 * So we lookup the "dev_t" (which points to the PID of the process 512 * that opened), and we remove the entry from the database (and free 513 * it up). Then we do another query based on the PID value. And when 514 * we find that database entry, we free it up too and then free the 515 * Tavor UAR page resource. 516 * 517 * Note: We use the tavor_umap_db_find_nolock() database access 518 * routine below (with an explicit mutex_enter of the database lock) 519 * to ensure that the multiple accesses (which attempt to remove the 520 * two database entries) can be done atomically. 521 * 522 * This works the same in both maintenance mode and HCA mode, except 523 * for the call to tavor_rsrc_free(). In the case of maintenance mode, 524 * this call is not needed, as it was not allocated in tavor_open() 525 * above. 526 */ 527 key = dev; 528 mutex_enter(&tavor_userland_rsrc_db.tdl_umapdb_lock); 529 status = tavor_umap_db_find_nolock(instance, key, MLNX_UMAP_PID_RSRC, 530 &value, TAVOR_UMAP_DB_REMOVE, &umapdb); 531 if (status == DDI_SUCCESS) { 532 /* 533 * If the "tdb_priv" field is non-NULL, it indicates that 534 * some "on close" handling is still necessary. Call 535 * tavor_umap_db_handle_onclose_cb() to do the handling (i.e. 536 * to invoke all the registered callbacks). Then free up 537 * the resources associated with "tdb_priv" and continue 538 * closing. 539 */ 540 priv = (tavor_umap_db_priv_t *)umapdb->tdbe_common.tdb_priv; 541 if (priv != NULL) { 542 tavor_umap_db_handle_onclose_cb(priv); 543 kmem_free(priv, sizeof (tavor_umap_db_priv_t)); 544 umapdb->tdbe_common.tdb_priv = (void *)NULL; 545 } 546 547 tavor_umap_db_free(umapdb); 548 549 /* 550 * Now do another lookup using PID as the key (copy it from 551 * "value"). When this lookup is complete, the "value" field 552 * will contain the tavor_rsrc_t pointer for the UAR page 553 * resource. 554 */ 555 key = value; 556 status = tavor_umap_db_find_nolock(instance, key, 557 MLNX_UMAP_UARPG_RSRC, &value, TAVOR_UMAP_DB_REMOVE, 558 &umapdb); 559 if (status == DDI_SUCCESS) { 560 tavor_umap_db_free(umapdb); 561 /* If in "maintenance mode", don't free the rsrc */ 562 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 563 rsrcp = (tavor_rsrc_t *)(uintptr_t)value; 564 tavor_rsrc_free(state, &rsrcp); 565 } 566 } 567 } 568 mutex_exit(&tavor_userland_rsrc_db.tdl_umapdb_lock); 569 570 TAVOR_TNF_EXIT(tavor_close); 571 return (0); 572 } 573 574 575 /* 576 * tavor_attach() 577 * Context: Only called from attach() path context 578 */ 579 static int 580 tavor_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 581 { 582 tavor_state_t *state; 583 ibc_clnt_hdl_t tmp_ibtfpriv; 584 ibc_status_t ibc_status; 585 int instance; 586 int status; 587 588 TAVOR_TNF_ENTER(tavor_attach); 589 590 #ifdef __lock_lint 591 (void) tavor_quiesce(dip); 592 #endif 593 594 switch (cmd) { 595 case DDI_ATTACH: 596 instance = ddi_get_instance(dip); 597 status = ddi_soft_state_zalloc(tavor_statep, instance); 598 if (status != DDI_SUCCESS) { 599 TNF_PROBE_0(tavor_attach_ssz_fail, TAVOR_TNF_ERROR, ""); 600 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " 601 "attach_ssz_fail", instance); 602 goto fail_attach_nomsg; 603 604 } 605 state = ddi_get_soft_state(tavor_statep, instance); 606 if (state == NULL) { 607 ddi_soft_state_free(tavor_statep, instance); 608 TNF_PROBE_0(tavor_attach_gss_fail, TAVOR_TNF_ERROR, ""); 609 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: " 610 "attach_gss_fail", instance); 611 goto fail_attach_nomsg; 612 } 613 614 /* clear the attach error buffer */ 615 TAVOR_ATTACH_MSG_INIT(state->ts_attach_buf); 616 617 /* 618 * Initialize Tavor driver and hardware. 619 * 620 * Note: If this initialization fails we may still wish to 621 * create a device node and remain operational so that Tavor 622 * firmware can be updated/flashed (i.e. "maintenance mode"). 623 * If this is the case, then "ts_operational_mode" will be 624 * equal to TAVOR_MAINTENANCE_MODE. We will not attempt to 625 * attach to the IBTF or register with the IBMF (i.e. no 626 * InfiniBand interfaces will be enabled). 627 */ 628 status = tavor_drv_init(state, dip, instance); 629 if ((status != DDI_SUCCESS) && 630 (TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { 631 TNF_PROBE_0(tavor_attach_drvinit_fail, 632 TAVOR_TNF_ERROR, ""); 633 goto fail_attach; 634 } 635 636 /* Create the minor node for device */ 637 status = ddi_create_minor_node(dip, "devctl", S_IFCHR, instance, 638 DDI_PSEUDO, 0); 639 if (status != DDI_SUCCESS) { 640 tavor_drv_fini(state); 641 TAVOR_ATTACH_MSG(state->ts_attach_buf, 642 "attach_create_mn_fail"); 643 TNF_PROBE_0(tavor_attach_create_mn_fail, 644 TAVOR_TNF_ERROR, ""); 645 goto fail_attach; 646 } 647 648 /* 649 * If we are in "maintenance mode", then we don't want to 650 * register with the IBTF. All InfiniBand interfaces are 651 * uninitialized, and the device is only capable of handling 652 * requests to update/flash firmware (or test/debug requests). 653 */ 654 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 655 656 /* Attach to InfiniBand Transport Framework (IBTF) */ 657 ibc_status = ibc_attach(&tmp_ibtfpriv, 658 &state->ts_ibtfinfo); 659 if (ibc_status != IBC_SUCCESS) { 660 ddi_remove_minor_node(dip, "devctl"); 661 tavor_drv_fini(state); 662 TNF_PROBE_0(tavor_attach_ibcattach_fail, 663 TAVOR_TNF_ERROR, ""); 664 TAVOR_ATTACH_MSG(state->ts_attach_buf, 665 "attach_ibcattach_fail"); 666 goto fail_attach; 667 } 668 669 /* 670 * Now that we've successfully attached to the IBTF, 671 * we enable all appropriate asynch and CQ events to 672 * be forwarded to the IBTF. 673 */ 674 TAVOR_ENABLE_IBTF_CALLB(state, tmp_ibtfpriv); 675 676 ibc_post_attach(state->ts_ibtfpriv); 677 678 /* Register agents with IB Mgmt Framework (IBMF) */ 679 status = tavor_agent_handlers_init(state); 680 if (status != DDI_SUCCESS) { 681 (void) ibc_pre_detach(tmp_ibtfpriv, DDI_DETACH); 682 TAVOR_QUIESCE_IBTF_CALLB(state); 683 if (state->ts_in_evcallb != 0) { 684 TAVOR_WARNING(state, "unable to " 685 "quiesce Tavor IBTF callbacks"); 686 } 687 ibc_detach(tmp_ibtfpriv); 688 ddi_remove_minor_node(dip, "devctl"); 689 tavor_drv_fini(state); 690 TNF_PROBE_0(tavor_attach_agentinit_fail, 691 TAVOR_TNF_ERROR, ""); 692 TAVOR_ATTACH_MSG(state->ts_attach_buf, 693 "attach_agentinit_fail"); 694 goto fail_attach; 695 } 696 } 697 698 /* Report that driver was loaded */ 699 ddi_report_dev(dip); 700 701 /* Send device information to log file */ 702 tavor_device_info_report(state); 703 704 /* Report attach in maintenance mode, if appropriate */ 705 if (!(TAVOR_IS_OPERATIONAL(state->ts_operational_mode))) { 706 cmn_err(CE_NOTE, "tavor%d: driver attached " 707 "(for maintenance mode only)", state->ts_instance); 708 } 709 710 TAVOR_TNF_EXIT(tavor_attach); 711 return (DDI_SUCCESS); 712 713 case DDI_RESUME: 714 /* Add code here for DDI_RESUME XXX */ 715 TAVOR_TNF_EXIT(tavor_attach); 716 return (DDI_FAILURE); 717 718 default: 719 TNF_PROBE_0(tavor_attach_default_fail, TAVOR_TNF_ERROR, ""); 720 break; 721 } 722 723 fail_attach: 724 cmn_err(CE_NOTE, "tavor%d: driver failed to attach: %s", instance, 725 state->ts_attach_buf); 726 tavor_drv_fini2(state); 727 ddi_soft_state_free(tavor_statep, instance); 728 fail_attach_nomsg: 729 TAVOR_TNF_EXIT(tavor_attach); 730 return (DDI_FAILURE); 731 } 732 733 734 /* 735 * tavor_detach() 736 * Context: Only called from detach() path context 737 */ 738 static int 739 tavor_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 740 { 741 tavor_state_t *state; 742 ibc_clnt_hdl_t tmp_ibtfpriv; 743 ibc_status_t ibc_status; 744 int instance, status; 745 746 TAVOR_TNF_ENTER(tavor_detach); 747 748 instance = ddi_get_instance(dip); 749 state = ddi_get_soft_state(tavor_statep, instance); 750 if (state == NULL) { 751 TNF_PROBE_0(tavor_detach_gss_fail, TAVOR_TNF_ERROR, ""); 752 TAVOR_TNF_EXIT(tavor_detach); 753 return (DDI_FAILURE); 754 } 755 756 switch (cmd) { 757 case DDI_DETACH: 758 /* 759 * If we are in "maintenance mode", then we do not want to 760 * do teardown for any of the InfiniBand interfaces. 761 * Specifically, this means not detaching from IBTF (we never 762 * attached to begin with) and not deregistering from IBMF. 763 */ 764 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 765 /* Unregister agents from IB Mgmt Framework (IBMF) */ 766 status = tavor_agent_handlers_fini(state); 767 if (status != DDI_SUCCESS) { 768 TNF_PROBE_0(tavor_detach_agentfini_fail, 769 TAVOR_TNF_ERROR, ""); 770 TAVOR_TNF_EXIT(tavor_detach); 771 return (DDI_FAILURE); 772 } 773 774 /* 775 * Attempt the "pre-detach" from InfiniBand Transport 776 * Framework (IBTF). At this point the IBTF is still 777 * capable of handling incoming asynch and completion 778 * events. This "pre-detach" is primarily a mechanism 779 * to notify the appropriate IBTF clients that the 780 * HCA is being removed/offlined. 781 */ 782 ibc_status = ibc_pre_detach(state->ts_ibtfpriv, cmd); 783 if (ibc_status != IBC_SUCCESS) { 784 status = tavor_agent_handlers_init(state); 785 if (status != DDI_SUCCESS) { 786 TAVOR_WARNING(state, "failed to " 787 "restart Tavor agents"); 788 } 789 TNF_PROBE_0(tavor_detach_ibcpredetach_fail, 790 TAVOR_TNF_ERROR, ""); 791 TAVOR_TNF_EXIT(tavor_detach); 792 return (DDI_FAILURE); 793 } 794 795 /* 796 * Before we can fully detach from the IBTF we need to 797 * ensure that we have handled all outstanding event 798 * callbacks. This is accomplished by quiescing the 799 * event callback mechanism. Note: if we are unable 800 * to successfully quiesce the callbacks, then this is 801 * an indication that something has probably gone 802 * seriously wrong. We print out a warning, but 803 * continue. 804 */ 805 tmp_ibtfpriv = state->ts_ibtfpriv; 806 TAVOR_QUIESCE_IBTF_CALLB(state); 807 if (state->ts_in_evcallb != 0) { 808 TAVOR_WARNING(state, "unable to quiesce Tavor " 809 "IBTF callbacks"); 810 } 811 812 /* Complete the detach from the IBTF */ 813 ibc_detach(tmp_ibtfpriv); 814 } 815 816 /* Remove the minor node for device */ 817 ddi_remove_minor_node(dip, "devctl"); 818 819 /* 820 * Only call tavor_drv_fini() if we are in Tavor HCA mode. 821 * (Because if we are in "maintenance mode", then we never 822 * successfully finished init.) Only report successful 823 * detach for normal HCA mode. 824 */ 825 if (TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 826 /* Cleanup driver resources and shutdown hardware */ 827 tavor_drv_fini(state); 828 cmn_err(CE_CONT, "Tavor driver successfully " 829 "detached\n"); 830 } 831 832 tavor_drv_fini2(state); 833 ddi_soft_state_free(tavor_statep, instance); 834 835 TAVOR_TNF_EXIT(tavor_detach); 836 return (DDI_SUCCESS); 837 838 case DDI_SUSPEND: 839 /* Add code here for DDI_SUSPEND XXX */ 840 TAVOR_TNF_EXIT(tavor_detach); 841 return (DDI_FAILURE); 842 843 default: 844 TNF_PROBE_0(tavor_detach_default_fail, TAVOR_TNF_ERROR, ""); 845 break; 846 } 847 848 TAVOR_TNF_EXIT(tavor_detach); 849 return (DDI_FAILURE); 850 } 851 852 853 /* 854 * tavor_drv_init() 855 * Context: Only called from attach() path context 856 */ 857 static int 858 tavor_drv_init(tavor_state_t *state, dev_info_t *dip, int instance) 859 { 860 int status; 861 862 TAVOR_TNF_ENTER(tavor_drv_init); 863 864 /* Save away devinfo and instance */ 865 state->ts_dip = dip; 866 state->ts_instance = instance; 867 868 /* 869 * Check and set the operational mode of the device. If the driver is 870 * bound to the Tavor device in "maintenance mode", then this generally 871 * means that either the device has been specifically jumpered to 872 * start in this mode or the firmware boot process has failed to 873 * successfully load either the primary or the secondary firmware 874 * image. 875 */ 876 if (TAVOR_IS_HCA_MODE(state->ts_dip)) { 877 state->ts_operational_mode = TAVOR_HCA_MODE; 878 879 } else if (TAVOR_IS_COMPAT_MODE(state->ts_dip)) { 880 state->ts_operational_mode = TAVOR_COMPAT_MODE; 881 882 } else if (TAVOR_IS_MAINTENANCE_MODE(state->ts_dip)) { 883 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; 884 return (DDI_FAILURE); 885 886 } else { 887 state->ts_operational_mode = 0; /* invalid operational mode */ 888 TAVOR_WARNING(state, "unexpected device type detected"); 889 TNF_PROBE_0(tavor_hw_init_unexpected_dev_fail, 890 TAVOR_TNF_ERROR, ""); 891 TAVOR_TNF_EXIT(tavor_hw_init); 892 return (DDI_FAILURE); 893 } 894 895 /* 896 * Initialize the Tavor hardware. 897 * Note: If this routine returns an error, it is often an reasonably 898 * good indication that something Tavor firmware-related has caused 899 * the failure. In order to give the user an opportunity (if desired) 900 * to update or reflash the Tavor firmware image, we set 901 * "ts_operational_mode" flag (described above) to indicate that we 902 * wish to enter maintenance mode. 903 */ 904 status = tavor_hw_init(state); 905 if (status != DDI_SUCCESS) { 906 state->ts_operational_mode = TAVOR_MAINTENANCE_MODE; 907 cmn_err(CE_NOTE, "tavor%d: error during attach: %s", instance, 908 state->ts_attach_buf); 909 TNF_PROBE_0(tavor_drv_init_hwinit_fail, TAVOR_TNF_ERROR, ""); 910 TAVOR_TNF_EXIT(tavor_drv_init); 911 return (DDI_FAILURE); 912 } 913 914 /* Setup Tavor interrupt handler */ 915 status = tavor_isr_init(state); 916 if (status != DDI_SUCCESS) { 917 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 918 TNF_PROBE_0(tavor_drv_init_isrinit_fail, TAVOR_TNF_ERROR, ""); 919 TAVOR_TNF_EXIT(tavor_drv_init); 920 return (DDI_FAILURE); 921 } 922 923 /* Initialize Tavor softstate */ 924 status = tavor_soft_state_init(state); 925 if (status != DDI_SUCCESS) { 926 tavor_isr_fini(state); 927 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 928 TNF_PROBE_0(tavor_drv_init_ssiinit_fail, TAVOR_TNF_ERROR, ""); 929 TAVOR_TNF_EXIT(tavor_drv_init); 930 return (DDI_FAILURE); 931 } 932 933 TAVOR_TNF_EXIT(tavor_drv_init); 934 return (DDI_SUCCESS); 935 } 936 937 938 /* 939 * tavor_drv_fini() 940 * Context: Only called from attach() and/or detach() path contexts 941 */ 942 static void 943 tavor_drv_fini(tavor_state_t *state) 944 { 945 TAVOR_TNF_ENTER(tavor_drv_fini); 946 947 /* Cleanup Tavor softstate */ 948 tavor_soft_state_fini(state); 949 950 /* Teardown Tavor interrupts */ 951 tavor_isr_fini(state); 952 953 /* Cleanup Tavor resources and shutdown hardware */ 954 tavor_hw_fini(state, TAVOR_DRV_CLEANUP_ALL); 955 956 TAVOR_TNF_EXIT(tavor_drv_fini); 957 } 958 959 /* 960 * tavor_drv_fini2() 961 * Context: Only called from attach() and/or detach() path contexts 962 */ 963 static void 964 tavor_drv_fini2(tavor_state_t *state) 965 { 966 TAVOR_TNF_ENTER(tavor_drv_fini2); 967 968 /* TAVOR_DRV_CLEANUP_LEVEL1 */ 969 if (state->ts_reg_cmdhdl) { 970 ddi_regs_map_free(&state->ts_reg_cmdhdl); 971 state->ts_reg_cmdhdl = NULL; 972 } 973 974 /* TAVOR_DRV_CLEANUP_LEVEL0 */ 975 if (state->ts_pci_cfghdl) { 976 pci_config_teardown(&state->ts_pci_cfghdl); 977 state->ts_pci_cfghdl = NULL; 978 } 979 980 TAVOR_TNF_EXIT(tavor_drv_fini2); 981 } 982 983 /* 984 * tavor_isr_init() 985 * Context: Only called from attach() path context 986 */ 987 static int 988 tavor_isr_init(tavor_state_t *state) 989 { 990 int status; 991 992 TAVOR_TNF_ENTER(tavor_isr_init); 993 994 /* 995 * Add a handler for the interrupt or MSI 996 */ 997 status = ddi_intr_add_handler(state->ts_intrmsi_hdl, tavor_isr, 998 (caddr_t)state, NULL); 999 if (status != DDI_SUCCESS) { 1000 TNF_PROBE_0(tavor_isr_init_addhndlr_fail, TAVOR_TNF_ERROR, ""); 1001 TAVOR_TNF_EXIT(tavor_isr_init); 1002 return (DDI_FAILURE); 1003 } 1004 1005 /* 1006 * Enable the software interrupt. Note: Even though we are only 1007 * using one (1) interrupt/MSI, depending on the value returned in 1008 * the capability flag, we have to call either ddi_intr_block_enable() 1009 * or ddi_intr_enable(). 1010 */ 1011 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { 1012 status = ddi_intr_block_enable(&state->ts_intrmsi_hdl, 1); 1013 if (status != DDI_SUCCESS) { 1014 TNF_PROBE_0(tavor_isr_init_blockenable_fail, 1015 TAVOR_TNF_ERROR, ""); 1016 TAVOR_TNF_EXIT(tavor_isr_init); 1017 return (DDI_FAILURE); 1018 } 1019 } else { 1020 status = ddi_intr_enable(state->ts_intrmsi_hdl); 1021 if (status != DDI_SUCCESS) { 1022 TNF_PROBE_0(tavor_isr_init_intrenable_fail, 1023 TAVOR_TNF_ERROR, ""); 1024 TAVOR_TNF_EXIT(tavor_isr_init); 1025 return (DDI_FAILURE); 1026 } 1027 } 1028 1029 /* 1030 * Now that the ISR has been setup, arm all the EQs for event 1031 * generation. 1032 */ 1033 tavor_eq_arm_all(state); 1034 1035 TAVOR_TNF_EXIT(tavor_isr_init); 1036 return (DDI_SUCCESS); 1037 } 1038 1039 1040 /* 1041 * tavor_isr_fini() 1042 * Context: Only called from attach() and/or detach() path contexts 1043 */ 1044 static void 1045 tavor_isr_fini(tavor_state_t *state) 1046 { 1047 TAVOR_TNF_ENTER(tavor_isr_fini); 1048 1049 /* Disable the software interrupt */ 1050 if (state->ts_intrmsi_cap & DDI_INTR_FLAG_BLOCK) { 1051 (void) ddi_intr_block_disable(&state->ts_intrmsi_hdl, 1); 1052 } else { 1053 (void) ddi_intr_disable(state->ts_intrmsi_hdl); 1054 } 1055 1056 /* 1057 * Remove the software handler for the interrupt or MSI 1058 */ 1059 (void) ddi_intr_remove_handler(state->ts_intrmsi_hdl); 1060 1061 TAVOR_TNF_EXIT(tavor_isr_fini); 1062 } 1063 1064 1065 /* 1066 * tavor_fix_error_buf() 1067 * Context: Only called from attach(). 1068 * 1069 * The error_buf_addr returned from QUERY_FW is a PCI address. 1070 * We need to convert it to an offset from the base address, 1071 * which is stored in the assigned-addresses property. 1072 */ 1073 static int 1074 tavor_fix_error_buf(tavor_state_t *state) 1075 { 1076 int assigned_addr_len; 1077 pci_regspec_t *assigned_addr; 1078 1079 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, state->ts_dip, 1080 DDI_PROP_DONTPASS, "assigned-addresses", (int **)&assigned_addr, 1081 (uint_t *)&assigned_addr_len) != DDI_PROP_SUCCESS) 1082 return (DDI_FAILURE); 1083 1084 state->ts_fw.error_buf_addr -= assigned_addr[0].pci_phys_low + 1085 ((uint64_t)(assigned_addr[0].pci_phys_mid) << 32); 1086 ddi_prop_free(assigned_addr); 1087 return (DDI_SUCCESS); 1088 } 1089 1090 /* 1091 * tavor_hw_init() 1092 * Context: Only called from attach() path context 1093 */ 1094 static int 1095 tavor_hw_init(tavor_state_t *state) 1096 { 1097 tavor_drv_cleanup_level_t cleanup; 1098 sm_nodeinfo_t nodeinfo; 1099 uint64_t errorcode; 1100 off_t ddr_size; 1101 int status; 1102 int retries; 1103 1104 TAVOR_TNF_ENTER(tavor_hw_init); 1105 1106 /* This is where driver initialization begins */ 1107 cleanup = TAVOR_DRV_CLEANUP_LEVEL0; 1108 1109 /* Setup device access attributes */ 1110 state->ts_reg_accattr.devacc_attr_version = DDI_DEVICE_ATTR_V0; 1111 state->ts_reg_accattr.devacc_attr_endian_flags = DDI_STRUCTURE_BE_ACC; 1112 state->ts_reg_accattr.devacc_attr_dataorder = DDI_STRICTORDER_ACC; 1113 1114 /* Setup for PCI config read/write of HCA device */ 1115 status = pci_config_setup(state->ts_dip, &state->ts_pci_cfghdl); 1116 if (status != DDI_SUCCESS) { 1117 tavor_hw_fini(state, cleanup); 1118 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1119 "hw_init_PCI_config_space_regmap_fail"); 1120 /* This case is not the degraded one */ 1121 return (DDI_FAILURE); 1122 } 1123 1124 /* Map in Tavor registers (CMD, UAR, DDR) and setup offsets */ 1125 status = ddi_regs_map_setup(state->ts_dip, TAVOR_CMD_BAR, 1126 &state->ts_reg_cmd_baseaddr, 0, 0, &state->ts_reg_accattr, 1127 &state->ts_reg_cmdhdl); 1128 if (status != DDI_SUCCESS) { 1129 tavor_hw_fini(state, cleanup); 1130 TNF_PROBE_0(tavor_hw_init_CMD_ddirms_fail, TAVOR_TNF_ERROR, ""); 1131 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1132 "hw_init_CMD_ddirms_fail"); 1133 TAVOR_TNF_EXIT(tavor_hw_init); 1134 return (DDI_FAILURE); 1135 } 1136 cleanup = TAVOR_DRV_CLEANUP_LEVEL1; 1137 1138 status = ddi_regs_map_setup(state->ts_dip, TAVOR_UAR_BAR, 1139 &state->ts_reg_uar_baseaddr, 0, 0, &state->ts_reg_accattr, 1140 &state->ts_reg_uarhdl); 1141 if (status != DDI_SUCCESS) { 1142 tavor_hw_fini(state, cleanup); 1143 TNF_PROBE_0(tavor_hw_init_UAR_ddirms_fail, TAVOR_TNF_ERROR, ""); 1144 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1145 "hw_init_UAR_ddirms_fail"); 1146 TAVOR_TNF_EXIT(tavor_hw_init); 1147 return (DDI_FAILURE); 1148 } 1149 cleanup = TAVOR_DRV_CLEANUP_LEVEL2; 1150 1151 status = ddi_dev_regsize(state->ts_dip, TAVOR_DDR_BAR, &ddr_size); 1152 if (status != DDI_SUCCESS) { 1153 cmn_err(CE_CONT, "Tavor: ddi_dev_regsize() failed " 1154 "(check HCA-attached DIMM memory?)\n"); 1155 tavor_hw_fini(state, cleanup); 1156 TNF_PROBE_0(tavor_hw_init_DDR_ddi_regsize_fail, 1157 TAVOR_TNF_ERROR, ""); 1158 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1159 "hw_init_DDR_ddi_regsize_fail"); 1160 TAVOR_TNF_EXIT(tavor_hw_init); 1161 return (DDI_FAILURE); 1162 } 1163 1164 #if !defined(_ELF64) && !defined(__sparc) 1165 /* 1166 * For 32 bit x86/x64 kernels, where there is limited kernel virtual 1167 * memory available, define a minimal memory footprint. This is 1168 * specified in order to not take up too much resources, thus starving 1169 * out others. Only specified if the HCA DIMM is equal to or greater 1170 * than 256MB. 1171 * 1172 * Note: x86/x64 install and safemode boot are both 32bit. 1173 */ 1174 ddr_size = TAVOR_DDR_SIZE_MIN; 1175 #endif /* !(_ELF64) && !(__sparc) */ 1176 1177 state->ts_cfg_profile_setting = ddr_size; 1178 1179 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, 1180 &state->ts_reg_ddr_baseaddr, 0, ddr_size, &state->ts_reg_accattr, 1181 &state->ts_reg_ddrhdl); 1182 1183 /* 1184 * On 32-bit platform testing (primarily x86), it was seen that the 1185 * ddi_regs_map_setup() call would fail because there wasn't enough 1186 * kernel virtual address space available to map in the entire 256MB 1187 * DDR. So we add this check in here, so that if the 256 (or other 1188 * larger value of DDR) map in fails, that we fallback to try the lower 1189 * size of 128MB. 1190 * 1191 * Note: If we only have 128MB of DDR in the system in the first place, 1192 * we don't try another ddi_regs_map_setup(), and just skip over this 1193 * check and return failures. 1194 */ 1195 if (status == DDI_ME_NORESOURCES && ddr_size > TAVOR_DDR_SIZE_128) { 1196 /* Try falling back to 128MB DDR mapping */ 1197 status = ddi_regs_map_setup(state->ts_dip, TAVOR_DDR_BAR, 1198 &state->ts_reg_ddr_baseaddr, 0, TAVOR_DDR_SIZE_128, 1199 &state->ts_reg_accattr, &state->ts_reg_ddrhdl); 1200 1201 /* 1202 * 128MB DDR mapping worked. 1203 * Set the updated config profile setting here. 1204 */ 1205 if (status == DDI_SUCCESS) { 1206 TNF_PROBE_0(tavor_hw_init_DDR_128mb_fallback_success, 1207 TAVOR_TNF_TRACE, ""); 1208 state->ts_cfg_profile_setting = TAVOR_DDR_SIZE_128; 1209 } 1210 } 1211 1212 if (status != DDI_SUCCESS) { 1213 if (status == DDI_ME_RNUMBER_RANGE) { 1214 cmn_err(CE_CONT, "Tavor: ddi_regs_map_setup() failed " 1215 "(check HCA-attached DIMM memory?)\n"); 1216 } 1217 tavor_hw_fini(state, cleanup); 1218 TNF_PROBE_0(tavor_hw_init_DDR_ddirms_fail, TAVOR_TNF_ERROR, ""); 1219 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1220 "hw_init_DDR_ddirms_fail"); 1221 TAVOR_TNF_EXIT(tavor_hw_init); 1222 return (DDI_FAILURE); 1223 } 1224 cleanup = TAVOR_DRV_CLEANUP_LEVEL3; 1225 1226 /* Setup Tavor Host Command Register (HCR) */ 1227 state->ts_cmd_regs.hcr = (tavor_hw_hcr_t *) 1228 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_HCR_OFFSET); 1229 1230 /* Setup Tavor Event Cause Register (ecr and clr_ecr) */ 1231 state->ts_cmd_regs.ecr = (uint64_t *) 1232 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_ECR_OFFSET); 1233 state->ts_cmd_regs.clr_ecr = (uint64_t *) 1234 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_ECR_OFFSET); 1235 1236 /* Setup Tavor Software Reset register (sw_reset) */ 1237 state->ts_cmd_regs.sw_reset = (uint32_t *) 1238 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_SW_RESET_OFFSET); 1239 1240 /* Setup Tavor Clear Interrupt register (clr_int) */ 1241 state->ts_cmd_regs.clr_int = (uint64_t *) 1242 ((uintptr_t)state->ts_reg_cmd_baseaddr + TAVOR_CMD_CLR_INT_OFFSET); 1243 1244 /* Initialize the Phase1 Tavor configuration profile */ 1245 status = tavor_cfg_profile_init_phase1(state); 1246 if (status != DDI_SUCCESS) { 1247 tavor_hw_fini(state, cleanup); 1248 TNF_PROBE_0(tavor_hw_init_cfginit_fail, TAVOR_TNF_ERROR, ""); 1249 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit_fail"); 1250 TAVOR_TNF_EXIT(tavor_hw_init); 1251 return (DDI_FAILURE); 1252 } 1253 cleanup = TAVOR_DRV_CLEANUP_LEVEL4; 1254 1255 /* Do a software reset of the Tavor HW to ensure proper state */ 1256 status = tavor_sw_reset(state); 1257 if (status != TAVOR_CMD_SUCCESS) { 1258 tavor_hw_fini(state, cleanup); 1259 TNF_PROBE_0(tavor_hw_init_sw_reset_fail, TAVOR_TNF_ERROR, ""); 1260 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_sw_reset_fail"); 1261 TAVOR_TNF_EXIT(tavor_hw_init); 1262 return (DDI_FAILURE); 1263 } 1264 1265 /* Post the SYS_EN command to start the hardware */ 1266 status = tavor_sys_en_cmd_post(state, TAVOR_CMD_SYS_EN_NORMAL, 1267 &errorcode, TAVOR_CMD_NOSLEEP_SPIN); 1268 if (status != TAVOR_CMD_SUCCESS) { 1269 if ((status == TAVOR_CMD_BAD_NVMEM) || 1270 (status == TAVOR_CMD_DDR_MEM_ERR)) { 1271 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " 1272 "0x%" PRIx64 " (invalid firmware image?)\n", 1273 status, errorcode); 1274 } else { 1275 cmn_err(CE_CONT, "Tavor: SYS_EN command failed: 0x%x " 1276 "0x%" PRIx64 "\n", status, errorcode); 1277 } 1278 tavor_hw_fini(state, cleanup); 1279 TNF_PROBE_0(tavor_hw_init_sys_en_cmd_fail, 1280 TAVOR_TNF_ERROR, ""); 1281 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1282 "hw_init_sys_en_cmd_fail"); 1283 TAVOR_TNF_EXIT(tavor_hw_init); 1284 return (DDI_FAILURE); 1285 } 1286 cleanup = TAVOR_DRV_CLEANUP_LEVEL5; 1287 1288 /* First phase of init for Tavor configuration/resources */ 1289 status = tavor_rsrc_init_phase1(state); 1290 if (status != DDI_SUCCESS) { 1291 tavor_hw_fini(state, cleanup); 1292 TNF_PROBE_0(tavor_hw_init_rsrcinit1_fail, TAVOR_TNF_ERROR, ""); 1293 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1294 "hw_init_rsrcinit1_fail"); 1295 TAVOR_TNF_EXIT(tavor_hw_init); 1296 return (DDI_FAILURE); 1297 } 1298 cleanup = TAVOR_DRV_CLEANUP_LEVEL6; 1299 1300 /* Query the DDR properties (e.g. total DDR size) */ 1301 status = tavor_cmn_query_cmd_post(state, QUERY_DDR, 0, 1302 &state->ts_ddr, sizeof (tavor_hw_queryddr_t), 1303 TAVOR_CMD_NOSLEEP_SPIN); 1304 if (status != TAVOR_CMD_SUCCESS) { 1305 cmn_err(CE_CONT, "Tavor: QUERY_DDR command failed: %08x\n", 1306 status); 1307 tavor_hw_fini(state, cleanup); 1308 TNF_PROBE_0(tavor_hw_init_query_ddr_cmd_fail, 1309 TAVOR_TNF_ERROR, ""); 1310 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1311 "hw_init_query_ddr_cmd_fail"); 1312 TAVOR_TNF_EXIT(tavor_hw_init); 1313 return (DDI_FAILURE); 1314 } 1315 1316 /* Figure out how big the firmware image (in DDR) is */ 1317 status = tavor_cmn_query_cmd_post(state, QUERY_FW, 0, &state->ts_fw, 1318 sizeof (tavor_hw_queryfw_t), TAVOR_CMD_NOSLEEP_SPIN); 1319 if (status != TAVOR_CMD_SUCCESS) { 1320 cmn_err(CE_CONT, "Tavor: QUERY_FW command failed: %08x\n", 1321 status); 1322 tavor_hw_fini(state, cleanup); 1323 TNF_PROBE_0(tavor_hw_init_query_fw_cmd_fail, 1324 TAVOR_TNF_ERROR, ""); 1325 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1326 "hw_init_query_fw_cmd_fail"); 1327 TAVOR_TNF_EXIT(tavor_hw_init); 1328 return (DDI_FAILURE); 1329 } 1330 1331 if (tavor_fix_error_buf(state) != DDI_SUCCESS) { 1332 tavor_hw_fini(state, cleanup); 1333 TNF_PROBE_0(tavor_hw_init_fixerrorbuf_fail, 1334 TAVOR_TNF_ERROR, ""); 1335 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1336 "hw_init_fixerrorbuf_fail"); 1337 TAVOR_TNF_EXIT(tavor_hw_init); 1338 return (DDI_FAILURE); 1339 } 1340 1341 /* Validate that the FW version is appropriate */ 1342 status = tavor_fw_version_check(state); 1343 if (status != DDI_SUCCESS) { 1344 if (state->ts_operational_mode == TAVOR_HCA_MODE) { 1345 cmn_err(CE_CONT, "Unsupported Tavor FW version: " 1346 "expected: %04d.%04d.%04d, " 1347 "actual: %04d.%04d.%04d\n", 1348 TAVOR_FW_VER_MAJOR, 1349 TAVOR_FW_VER_MINOR, 1350 TAVOR_FW_VER_SUBMINOR, 1351 state->ts_fw.fw_rev_major, 1352 state->ts_fw.fw_rev_minor, 1353 state->ts_fw.fw_rev_subminor); 1354 } else if (state->ts_operational_mode == TAVOR_COMPAT_MODE) { 1355 cmn_err(CE_CONT, "Unsupported Tavor Compat FW version: " 1356 "expected: %04d.%04d.%04d, " 1357 "actual: %04d.%04d.%04d\n", 1358 TAVOR_COMPAT_FW_VER_MAJOR, 1359 TAVOR_COMPAT_FW_VER_MINOR, 1360 TAVOR_COMPAT_FW_VER_SUBMINOR, 1361 state->ts_fw.fw_rev_major, 1362 state->ts_fw.fw_rev_minor, 1363 state->ts_fw.fw_rev_subminor); 1364 } else { 1365 cmn_err(CE_CONT, "Unsupported FW version: " 1366 "%04d.%04d.%04d\n", 1367 state->ts_fw.fw_rev_major, 1368 state->ts_fw.fw_rev_minor, 1369 state->ts_fw.fw_rev_subminor); 1370 } 1371 tavor_hw_fini(state, cleanup); 1372 TNF_PROBE_0(tavor_hw_init_checkfwver_fail, 1373 TAVOR_TNF_ERROR, ""); 1374 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1375 "hw_init_checkfwver_fail"); 1376 TAVOR_TNF_EXIT(tavor_hw_init); 1377 return (DDI_FAILURE); 1378 } 1379 1380 drv_usecwait(10); 1381 retries = 1000; /* retry up to 1 second before giving up */ 1382 retry: 1383 /* Call MOD_STAT_CFG to setup SRQ support (or disable) */ 1384 status = tavor_mod_stat_cfg_cmd_post(state); 1385 if (status != DDI_SUCCESS) { 1386 if (retries > 0) { 1387 drv_usecwait(1000); 1388 retries--; 1389 goto retry; 1390 } 1391 cmn_err(CE_CONT, "Tavor: MOD_STAT_CFG command failed: %08x\n", 1392 status); 1393 tavor_hw_fini(state, cleanup); 1394 TNF_PROBE_0(tavor_hw_init_mod_stat_cfg_cmd_fail, 1395 TAVOR_TNF_ERROR, ""); 1396 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1397 "hw_init_mod_stat_cfg_cmd_fail"); 1398 TAVOR_TNF_EXIT(tavor_hw_init); 1399 return (DDI_FAILURE); 1400 } 1401 1402 /* Figure out Tavor device limits */ 1403 status = tavor_cmn_query_cmd_post(state, QUERY_DEV_LIM, 0, 1404 &state->ts_devlim, sizeof (tavor_hw_querydevlim_t), 1405 TAVOR_CMD_NOSLEEP_SPIN); 1406 if (status != TAVOR_CMD_SUCCESS) { 1407 cmn_err(CE_CONT, "Tavor: QUERY_DEV_LIM command failed: %08x\n", 1408 status); 1409 tavor_hw_fini(state, cleanup); 1410 TNF_PROBE_0(tavor_hw_init_query_devlim_cmd_fail, 1411 TAVOR_TNF_ERROR, ""); 1412 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1413 "hw_init_query_devlim_cmd_fail"); 1414 TAVOR_TNF_EXIT(tavor_hw_init); 1415 return (DDI_FAILURE); 1416 } 1417 1418 /* Initialize the Phase2 Tavor configuration profile */ 1419 status = tavor_cfg_profile_init_phase2(state); 1420 if (status != DDI_SUCCESS) { 1421 tavor_hw_fini(state, cleanup); 1422 TNF_PROBE_0(tavor_hw_init_cfginit2_fail, TAVOR_TNF_ERROR, ""); 1423 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_cfginit2_fail"); 1424 TAVOR_TNF_EXIT(tavor_hw_init); 1425 return (DDI_FAILURE); 1426 } 1427 1428 /* Second phase of init for Tavor configuration/resources */ 1429 status = tavor_rsrc_init_phase2(state); 1430 if (status != DDI_SUCCESS) { 1431 tavor_hw_fini(state, cleanup); 1432 TNF_PROBE_0(tavor_hw_init_rsrcinit2_fail, TAVOR_TNF_ERROR, ""); 1433 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1434 "hw_init_rsrcinit2_fail"); 1435 TAVOR_TNF_EXIT(tavor_hw_init); 1436 return (DDI_FAILURE); 1437 } 1438 cleanup = TAVOR_DRV_CLEANUP_LEVEL7; 1439 1440 /* Miscellaneous query information */ 1441 status = tavor_cmn_query_cmd_post(state, QUERY_ADAPTER, 0, 1442 &state->ts_adapter, sizeof (tavor_hw_queryadapter_t), 1443 TAVOR_CMD_NOSLEEP_SPIN); 1444 if (status != TAVOR_CMD_SUCCESS) { 1445 cmn_err(CE_CONT, "Tavor: QUERY_ADAPTER command failed: %08x\n", 1446 status); 1447 tavor_hw_fini(state, cleanup); 1448 TNF_PROBE_0(tavor_hw_init_query_adapter_cmd_fail, 1449 TAVOR_TNF_ERROR, ""); 1450 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1451 "hw_init_query_adapter_cmd_fail"); 1452 TAVOR_TNF_EXIT(tavor_hw_init); 1453 return (DDI_FAILURE); 1454 } 1455 1456 /* Prepare configuration for Tavor INIT_HCA command */ 1457 tavor_hca_config_setup(state, &state->ts_hcaparams); 1458 1459 /* Post command to init Tavor HCA */ 1460 status = tavor_init_hca_cmd_post(state, &state->ts_hcaparams, 1461 TAVOR_CMD_NOSLEEP_SPIN); 1462 if (status != TAVOR_CMD_SUCCESS) { 1463 cmn_err(CE_CONT, "Tavor: INIT_HCA command failed: %08x\n", 1464 status); 1465 tavor_hw_fini(state, cleanup); 1466 TNF_PROBE_0(tavor_hw_init_init_hca_cmd_fail, 1467 TAVOR_TNF_ERROR, ""); 1468 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1469 "hw_init_init_hca_cmd_fail"); 1470 TAVOR_TNF_EXIT(tavor_hw_init); 1471 return (DDI_FAILURE); 1472 } 1473 cleanup = TAVOR_DRV_CLEANUP_LEVEL8; 1474 1475 /* Allocate protection domain (PD) for Tavor internal use */ 1476 status = tavor_pd_alloc(state, &state->ts_pdhdl_internal, TAVOR_SLEEP); 1477 if (status != DDI_SUCCESS) { 1478 tavor_hw_fini(state, cleanup); 1479 TNF_PROBE_0(tavor_hw_init_internal_pd_alloc_fail, 1480 TAVOR_TNF_ERROR, ""); 1481 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1482 "hw_init_internal_pd_alloc_fail"); 1483 TAVOR_TNF_EXIT(tavor_hw_init); 1484 return (DDI_FAILURE); 1485 } 1486 cleanup = TAVOR_DRV_CLEANUP_LEVEL9; 1487 1488 /* Setup Tavor internal UAR pages (0 and 1) */ 1489 status = tavor_internal_uarpgs_init(state); 1490 if (status != DDI_SUCCESS) { 1491 tavor_hw_fini(state, cleanup); 1492 TNF_PROBE_0(tavor_hw_init_internal_uarpgs_alloc_fail, 1493 TAVOR_TNF_ERROR, ""); 1494 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1495 "hw_init_internal_uarpgs_alloc_fail"); 1496 TAVOR_TNF_EXIT(tavor_hw_init); 1497 return (DDI_FAILURE); 1498 } 1499 cleanup = TAVOR_DRV_CLEANUP_LEVEL10; 1500 1501 /* Query and initialize the Tavor interrupt/MSI information */ 1502 status = tavor_intr_or_msi_init(state); 1503 if (status != DDI_SUCCESS) { 1504 tavor_hw_fini(state, cleanup); 1505 TNF_PROBE_0(tavor_intr_or_msi_init_fail, 1506 TAVOR_TNF_ERROR, ""); 1507 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1508 "intr_or_msi_init_fail"); 1509 TAVOR_TNF_EXIT(tavor_hw_init); 1510 return (DDI_FAILURE); 1511 } 1512 cleanup = TAVOR_DRV_CLEANUP_LEVEL11; 1513 1514 /* Setup all of the Tavor EQs */ 1515 status = tavor_eq_init_all(state); 1516 if (status != DDI_SUCCESS) { 1517 tavor_hw_fini(state, cleanup); 1518 TNF_PROBE_0(tavor_hw_init_eqinitall_fail, TAVOR_TNF_ERROR, ""); 1519 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1520 "hw_init_eqinitall_fail"); 1521 TAVOR_TNF_EXIT(tavor_hw_init); 1522 return (DDI_FAILURE); 1523 } 1524 cleanup = TAVOR_DRV_CLEANUP_LEVEL12; 1525 1526 /* Set aside contexts for QP0 and QP1 */ 1527 status = tavor_special_qp_contexts_reserve(state); 1528 if (status != DDI_SUCCESS) { 1529 tavor_hw_fini(state, cleanup); 1530 TNF_PROBE_0(tavor_hw_init_reserve_special_qp_fail, 1531 TAVOR_TNF_ERROR, ""); 1532 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1533 "hw_init_reserve_special_qp_fail"); 1534 TAVOR_TNF_EXIT(tavor_hw_init); 1535 return (DDI_FAILURE); 1536 } 1537 cleanup = TAVOR_DRV_CLEANUP_LEVEL13; 1538 1539 /* Initialize for multicast group handling */ 1540 status = tavor_mcg_init(state); 1541 if (status != DDI_SUCCESS) { 1542 tavor_hw_fini(state, cleanup); 1543 TNF_PROBE_0(tavor_hw_init_mcg_init_fail, TAVOR_TNF_ERROR, ""); 1544 TAVOR_ATTACH_MSG(state->ts_attach_buf, "hw_init_mcg_init_fail"); 1545 TAVOR_TNF_EXIT(tavor_hw_init); 1546 return (DDI_FAILURE); 1547 } 1548 cleanup = TAVOR_DRV_CLEANUP_LEVEL14; 1549 1550 /* Initialize the Tavor IB port(s) */ 1551 status = tavor_hca_port_init(state); 1552 if (status != DDI_SUCCESS) { 1553 tavor_hw_fini(state, cleanup); 1554 TNF_PROBE_0(tavor_hw_init_hca_port_init_fail, 1555 TAVOR_TNF_ERROR, ""); 1556 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1557 "hw_init_hca_port_init_fail"); 1558 TAVOR_TNF_EXIT(tavor_hw_init); 1559 return (DDI_FAILURE); 1560 } 1561 cleanup = TAVOR_DRV_CLEANUP_ALL; 1562 1563 /* Determine NodeGUID and SystemImageGUID */ 1564 status = tavor_getnodeinfo_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, 1565 &nodeinfo); 1566 if (status != TAVOR_CMD_SUCCESS) { 1567 cmn_err(CE_CONT, "Tavor: GetNodeInfo command failed: %08x\n", 1568 status); 1569 tavor_hw_fini(state, cleanup); 1570 TNF_PROBE_0(tavor_hw_init_getnodeinfo_cmd_fail, 1571 TAVOR_TNF_ERROR, ""); 1572 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1573 "hw_init_getnodeinfo_cmd_fail"); 1574 TAVOR_TNF_EXIT(tavor_hw_init); 1575 return (DDI_FAILURE); 1576 } 1577 1578 /* 1579 * If the NodeGUID value was set in OBP properties, then we use that 1580 * value. But we still print a message if the value we queried from 1581 * firmware does not match this value. 1582 * 1583 * Otherwise if OBP value is not set then we use the value from 1584 * firmware unconditionally. 1585 */ 1586 if (state->ts_cfg_profile->cp_nodeguid) { 1587 state->ts_nodeguid = state->ts_cfg_profile->cp_nodeguid; 1588 } else { 1589 state->ts_nodeguid = nodeinfo.NodeGUID; 1590 } 1591 1592 if (state->ts_nodeguid != nodeinfo.NodeGUID) { 1593 cmn_err(CE_NOTE, "!NodeGUID value queried from firmware " 1594 "does not match value set by device property"); 1595 } 1596 1597 /* 1598 * If the SystemImageGUID value was set in OBP properties, then we use 1599 * that value. But we still print a message if the value we queried 1600 * from firmware does not match this value. 1601 * 1602 * Otherwise if OBP value is not set then we use the value from 1603 * firmware unconditionally. 1604 */ 1605 if (state->ts_cfg_profile->cp_sysimgguid) { 1606 state->ts_sysimgguid = state->ts_cfg_profile->cp_sysimgguid; 1607 } else { 1608 state->ts_sysimgguid = nodeinfo.SystemImageGUID; 1609 } 1610 1611 if (state->ts_sysimgguid != nodeinfo.SystemImageGUID) { 1612 cmn_err(CE_NOTE, "!SystemImageGUID value queried from firmware " 1613 "does not match value set by device property"); 1614 } 1615 1616 /* Get NodeDescription */ 1617 status = tavor_getnodedesc_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN, 1618 (sm_nodedesc_t *)&state->ts_nodedesc); 1619 if (status != TAVOR_CMD_SUCCESS) { 1620 cmn_err(CE_CONT, "Tavor: GetNodeDesc command failed: %08x\n", 1621 status); 1622 tavor_hw_fini(state, cleanup); 1623 TNF_PROBE_0(tavor_hw_init_getnodedesc_cmd_fail, 1624 TAVOR_TNF_ERROR, ""); 1625 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1626 "hw_init_getnodedesc_cmd_fail"); 1627 TAVOR_TNF_EXIT(tavor_hw_init); 1628 return (DDI_FAILURE); 1629 } 1630 1631 TAVOR_TNF_EXIT(tavor_hw_init); 1632 return (DDI_SUCCESS); 1633 } 1634 1635 1636 /* 1637 * tavor_hw_fini() 1638 * Context: Only called from attach() and/or detach() path contexts 1639 */ 1640 static void 1641 tavor_hw_fini(tavor_state_t *state, tavor_drv_cleanup_level_t cleanup) 1642 { 1643 uint_t num_ports; 1644 int status; 1645 1646 TAVOR_TNF_ENTER(tavor_hw_fini); 1647 1648 switch (cleanup) { 1649 /* 1650 * If we add more driver initialization steps that should be cleaned 1651 * up here, we need to ensure that TAVOR_DRV_CLEANUP_ALL is still the 1652 * first entry (i.e. corresponds to the last init step). 1653 */ 1654 case TAVOR_DRV_CLEANUP_ALL: 1655 /* Shutdown the Tavor IB port(s) */ 1656 num_ports = state->ts_cfg_profile->cp_num_ports; 1657 (void) tavor_hca_ports_shutdown(state, num_ports); 1658 /* FALLTHROUGH */ 1659 1660 case TAVOR_DRV_CLEANUP_LEVEL14: 1661 /* Teardown resources used for multicast group handling */ 1662 tavor_mcg_fini(state); 1663 /* FALLTHROUGH */ 1664 1665 case TAVOR_DRV_CLEANUP_LEVEL13: 1666 /* Unreserve the special QP contexts */ 1667 tavor_special_qp_contexts_unreserve(state); 1668 /* FALLTHROUGH */ 1669 1670 case TAVOR_DRV_CLEANUP_LEVEL12: 1671 /* 1672 * Attempt to teardown all event queues (EQ). If we fail 1673 * here then print a warning message and return. Something 1674 * (either in HW or SW) has gone seriously wrong. 1675 */ 1676 status = tavor_eq_fini_all(state); 1677 if (status != DDI_SUCCESS) { 1678 TAVOR_WARNING(state, "failed to teardown EQs"); 1679 TNF_PROBE_0(tavor_hw_fini_eqfiniall_fail, 1680 TAVOR_TNF_ERROR, ""); 1681 TAVOR_TNF_EXIT(tavor_hw_fini); 1682 return; 1683 } 1684 /* FALLTHROUGH */ 1685 1686 case TAVOR_DRV_CLEANUP_LEVEL11: 1687 status = tavor_intr_or_msi_fini(state); 1688 if (status != DDI_SUCCESS) { 1689 TAVOR_WARNING(state, "failed to free intr/MSI"); 1690 TNF_PROBE_0(tavor_hw_fini_intrmsifini_fail, 1691 TAVOR_TNF_ERROR, ""); 1692 TAVOR_TNF_EXIT(tavor_hw_fini); 1693 return; 1694 } 1695 /* FALLTHROUGH */ 1696 1697 case TAVOR_DRV_CLEANUP_LEVEL10: 1698 /* Free the resources for the Tavor internal UAR pages */ 1699 tavor_internal_uarpgs_fini(state); 1700 /* FALLTHROUGH */ 1701 1702 case TAVOR_DRV_CLEANUP_LEVEL9: 1703 /* 1704 * Free the PD that was used internally by Tavor software. If 1705 * we fail here then print a warning and return. Something 1706 * (probably software-related, but perhaps HW) has gone wrong. 1707 */ 1708 status = tavor_pd_free(state, &state->ts_pdhdl_internal); 1709 if (status != DDI_SUCCESS) { 1710 TAVOR_WARNING(state, "failed to free internal PD"); 1711 TNF_PROBE_0(tavor_hw_fini_internal_pd_free_fail, 1712 TAVOR_TNF_ERROR, ""); 1713 TAVOR_TNF_EXIT(tavor_hw_fini); 1714 return; 1715 } 1716 /* FALLTHROUGH */ 1717 1718 case TAVOR_DRV_CLEANUP_LEVEL8: 1719 /* 1720 * Post the CLOSE_HCA command to Tavor firmware. If we fail 1721 * here then print a warning and return. Something (either in 1722 * HW or SW) has gone seriously wrong. 1723 */ 1724 status = tavor_close_hca_cmd_post(state, 1725 TAVOR_CMD_NOSLEEP_SPIN); 1726 if (status != TAVOR_CMD_SUCCESS) { 1727 TAVOR_WARNING(state, "failed to shutdown HCA"); 1728 TNF_PROBE_0(tavor_hw_fini_closehcacmd_fail, 1729 TAVOR_TNF_ERROR, ""); 1730 TAVOR_TNF_EXIT(tavor_hw_fini); 1731 return; 1732 } 1733 /* FALLTHROUGH */ 1734 1735 case TAVOR_DRV_CLEANUP_LEVEL7: 1736 /* Cleanup all the phase2 resources first */ 1737 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_ALL); 1738 /* FALLTHROUGH */ 1739 1740 case TAVOR_DRV_CLEANUP_LEVEL6: 1741 /* Then cleanup the phase1 resources */ 1742 tavor_rsrc_fini(state, TAVOR_RSRC_CLEANUP_PHASE1_COMPLETE); 1743 /* FALLTHROUGH */ 1744 1745 case TAVOR_DRV_CLEANUP_LEVEL5: 1746 /* 1747 * Post the SYS_DIS command to Tavor firmware to shut 1748 * everything down again. If we fail here then print a 1749 * warning and return. Something (probably in HW, but maybe 1750 * in SW) has gone seriously wrong. 1751 */ 1752 status = tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN); 1753 if (status != TAVOR_CMD_SUCCESS) { 1754 TAVOR_WARNING(state, "failed to shutdown hardware"); 1755 TNF_PROBE_0(tavor_hw_fini_sys_dis_fail, 1756 TAVOR_TNF_ERROR, ""); 1757 TAVOR_TNF_EXIT(tavor_hw_fini); 1758 return; 1759 } 1760 /* FALLTHROUGH */ 1761 1762 case TAVOR_DRV_CLEANUP_LEVEL4: 1763 /* Teardown any resources allocated for the config profile */ 1764 tavor_cfg_profile_fini(state); 1765 /* FALLTHROUGH */ 1766 1767 case TAVOR_DRV_CLEANUP_LEVEL3: 1768 ddi_regs_map_free(&state->ts_reg_ddrhdl); 1769 /* FALLTHROUGH */ 1770 1771 case TAVOR_DRV_CLEANUP_LEVEL2: 1772 ddi_regs_map_free(&state->ts_reg_uarhdl); 1773 /* FALLTHROUGH */ 1774 1775 case TAVOR_DRV_CLEANUP_LEVEL1: 1776 case TAVOR_DRV_CLEANUP_LEVEL0: 1777 /* 1778 * LEVEL1 and LEVEL0 resources are freed in 1779 * tavor_drv_fini2(). 1780 */ 1781 break; 1782 1783 default: 1784 TAVOR_WARNING(state, "unexpected driver cleanup level"); 1785 TNF_PROBE_0(tavor_hw_fini_default_fail, TAVOR_TNF_ERROR, ""); 1786 TAVOR_TNF_EXIT(tavor_hw_fini); 1787 return; 1788 } 1789 1790 TAVOR_TNF_EXIT(tavor_hw_fini); 1791 } 1792 1793 1794 /* 1795 * tavor_soft_state_init() 1796 * Context: Only called from attach() path context 1797 */ 1798 static int 1799 tavor_soft_state_init(tavor_state_t *state) 1800 { 1801 ibt_hca_attr_t *hca_attr; 1802 uint64_t maxval, val; 1803 ibt_hca_flags_t caps = IBT_HCA_NO_FLAGS; 1804 int status; 1805 1806 TAVOR_TNF_ENTER(tavor_soft_state_init); 1807 1808 /* 1809 * The ibc_hca_info_t struct is passed to the IBTF. This is the 1810 * routine where we initialize it. Many of the init values come from 1811 * either configuration variables or successful queries of the Tavor 1812 * hardware abilities 1813 */ 1814 state->ts_ibtfinfo.hca_ci_vers = IBCI_V3; 1815 state->ts_ibtfinfo.hca_dip = state->ts_dip; 1816 state->ts_ibtfinfo.hca_handle = (ibc_hca_hdl_t)state; 1817 state->ts_ibtfinfo.hca_ops = &tavor_ibc_ops; 1818 1819 hca_attr = kmem_zalloc(sizeof (ibt_hca_attr_t), KM_SLEEP); 1820 state->ts_ibtfinfo.hca_attr = hca_attr; 1821 1822 hca_attr->hca_fw_major_version = state->ts_fw.fw_rev_major; 1823 hca_attr->hca_fw_minor_version = state->ts_fw.fw_rev_minor; 1824 hca_attr->hca_fw_micro_version = state->ts_fw.fw_rev_subminor; 1825 1826 /* 1827 * Determine HCA capabilities: 1828 * No default support for IBT_HCA_RD, IBT_HCA_RAW_MULTICAST, 1829 * IBT_HCA_ATOMICS_GLOBAL, IBT_HCA_RESIZE_CHAN, IBT_HCA_INIT_TYPE, 1830 * or IBT_HCA_SHUTDOWN_PORT 1831 * But IBT_HCA_AH_PORT_CHECK, IBT_HCA_SQD_RTS_PORT, IBT_HCA_SI_GUID, 1832 * IBT_HCA_RNR_NAK, and IBT_HCA_CURRENT_QP_STATE are always 1833 * supported 1834 * All other features are conditionally supported, depending on the 1835 * status return by the Tavor HCA (in QUERY_DEV_LIM) 1836 */ 1837 if (state->ts_devlim.ud_multi) { 1838 caps |= IBT_HCA_UD_MULTICAST; 1839 } 1840 if (state->ts_devlim.atomic) { 1841 caps |= IBT_HCA_ATOMICS_HCA; 1842 } 1843 if (state->ts_devlim.apm) { 1844 caps |= IBT_HCA_AUTO_PATH_MIG; 1845 } 1846 if (state->ts_devlim.pkey_v) { 1847 caps |= IBT_HCA_PKEY_CNTR; 1848 } 1849 if (state->ts_devlim.qkey_v) { 1850 caps |= IBT_HCA_QKEY_CNTR; 1851 } 1852 if (state->ts_cfg_profile->cp_srq_enable) { 1853 caps |= IBT_HCA_SRQ | IBT_HCA_RESIZE_SRQ; 1854 } 1855 if (state->ts_cfg_profile->cp_fmr_enable) { 1856 caps |= IBT_HCA_FMR; 1857 } 1858 caps |= (IBT_HCA_AH_PORT_CHECK | IBT_HCA_SQD_SQD_PORT | 1859 IBT_HCA_SI_GUID | IBT_HCA_RNR_NAK | IBT_HCA_CURRENT_QP_STATE | 1860 IBT_HCA_PORT_UP | IBT_HCA_SQD_STATE); 1861 hca_attr->hca_flags = caps; 1862 1863 /* Determine VendorID, DeviceID, and revision ID */ 1864 hca_attr->hca_vendor_id = state->ts_adapter.vendor_id; 1865 hca_attr->hca_device_id = state->ts_adapter.device_id; 1866 hca_attr->hca_version_id = state->ts_adapter.rev_id; 1867 1868 /* 1869 * Determine number of available QPs and max QP size. Number of 1870 * available QPs is determined by subtracting the number of 1871 * "reserved QPs" (i.e. reserved for firmware use) from the 1872 * total number configured. 1873 */ 1874 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); 1875 hca_attr->hca_max_qp = val - ((uint64_t)1 << 1876 state->ts_devlim.log_rsvd_qp); 1877 maxval = ((uint64_t)1 << state->ts_devlim.log_max_qp_sz); 1878 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_qp_sz); 1879 if (val > maxval) { 1880 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1881 TNF_PROBE_2(tavor_soft_state_init_maxqpsz_toobig_fail, 1882 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max QP size " 1883 "exceeds device maximum", tnf_uint, maxsz, maxval); 1884 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1885 "soft_state_init_maxqpsz_toobig_fail"); 1886 TAVOR_TNF_EXIT(tavor_soft_state_init); 1887 return (DDI_FAILURE); 1888 } 1889 hca_attr->hca_max_qp_sz = val; 1890 1891 /* Determine max scatter-gather size in WQEs */ 1892 maxval = state->ts_devlim.max_sg; 1893 val = state->ts_cfg_profile->cp_wqe_max_sgl; 1894 if (val > maxval) { 1895 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1896 TNF_PROBE_2(tavor_soft_state_init_toomanysgl_fail, 1897 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of sgl " 1898 "exceeds device maximum", tnf_uint, maxsgl, maxval); 1899 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1900 "soft_state_init_toomanysgl_fail"); 1901 TAVOR_TNF_EXIT(tavor_soft_state_init); 1902 return (DDI_FAILURE); 1903 } 1904 /* If the rounded value for max SGL is too large, cap it */ 1905 if (state->ts_cfg_profile->cp_wqe_real_max_sgl > maxval) { 1906 state->ts_cfg_profile->cp_wqe_real_max_sgl = maxval; 1907 val = maxval; 1908 } else { 1909 val = state->ts_cfg_profile->cp_wqe_real_max_sgl; 1910 } 1911 1912 hca_attr->hca_max_sgl = val; 1913 hca_attr->hca_max_rd_sgl = 0; /* zero because RD is unsupported */ 1914 1915 /* 1916 * Determine number of available CQs and max CQ size. Number of 1917 * available CQs is determined by subtracting the number of 1918 * "reserved CQs" (i.e. reserved for firmware use) from the 1919 * total number configured. 1920 */ 1921 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_cq); 1922 hca_attr->hca_max_cq = val - ((uint64_t)1 << 1923 state->ts_devlim.log_rsvd_cq); 1924 maxval = ((uint64_t)1 << state->ts_devlim.log_max_cq_sz); 1925 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_cq_sz) - 1; 1926 if (val > maxval) { 1927 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1928 TNF_PROBE_2(tavor_soft_state_init_maxcqsz_toobig_fail, 1929 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max CQ size " 1930 "exceeds device maximum", tnf_uint, maxsz, maxval); 1931 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1932 "soft_state_init_maxcqsz_toobig_fail"); 1933 TAVOR_TNF_EXIT(tavor_soft_state_init); 1934 return (DDI_FAILURE); 1935 } 1936 hca_attr->hca_max_cq_sz = val; 1937 1938 /* 1939 * Determine number of available SRQs and max SRQ size. Number of 1940 * available SRQs is determined by subtracting the number of 1941 * "reserved SRQs" (i.e. reserved for firmware use) from the 1942 * total number configured. 1943 */ 1944 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_srq); 1945 hca_attr->hca_max_srqs = val - ((uint64_t)1 << 1946 state->ts_devlim.log_rsvd_srq); 1947 maxval = ((uint64_t)1 << state->ts_devlim.log_max_srq_sz); 1948 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_srq_sz); 1949 1950 if (val > maxval) { 1951 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1952 TNF_PROBE_2(tavor_soft_state_init_maxsrqsz_toobig_fail, 1953 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max SRQ size " 1954 "exceeds device maximum", tnf_uint, maxsz, maxval); 1955 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1956 "soft_state_init_maxsrqsz_toobig_fail"); 1957 TAVOR_TNF_EXIT(tavor_soft_state_init); 1958 return (DDI_FAILURE); 1959 } 1960 hca_attr->hca_max_srqs_sz = val; 1961 1962 val = state->ts_cfg_profile->cp_srq_max_sgl; 1963 maxval = state->ts_devlim.max_sg; 1964 if (val > maxval) { 1965 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 1966 TNF_PROBE_2(tavor_soft_state_init_toomanysrqsgl_fail, 1967 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of srq " 1968 "sgl exceeds device maximum", tnf_uint, maxsgl, maxval); 1969 TAVOR_ATTACH_MSG(state->ts_attach_buf, 1970 "soft_state_init_toomanysrqsgl_fail"); 1971 TAVOR_TNF_EXIT(tavor_soft_state_init); 1972 return (DDI_FAILURE); 1973 } 1974 hca_attr->hca_max_srq_sgl = val; 1975 1976 /* 1977 * Determine supported HCA page sizes 1978 * XXX 1979 * For now we simply return the system pagesize as the only supported 1980 * pagesize 1981 */ 1982 hca_attr->hca_page_sz = ((PAGESIZE == (1 << 13)) ? IBT_PAGE_8K : 1983 IBT_PAGE_4K); 1984 1985 /* 1986 * Determine number of available MemReg, MemWin, and their max size. 1987 * Number of available MRs and MWs is determined by subtracting 1988 * the number of "reserved MPTs" (i.e. reserved for firmware use) 1989 * from the total number configured for each. 1990 */ 1991 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mpt); 1992 hca_attr->hca_max_memr = val - ((uint64_t)1 << 1993 state->ts_devlim.log_rsvd_mpt); 1994 hca_attr->hca_max_mem_win = val - ((uint64_t)1 << 1995 state->ts_devlim.log_rsvd_mpt); 1996 maxval = state->ts_devlim.log_max_mrw_sz; 1997 val = state->ts_cfg_profile->cp_log_max_mrw_sz; 1998 if (val > maxval) { 1999 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2000 TNF_PROBE_2(tavor_soft_state_init_maxmrwsz_toobig_fail, 2001 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max mrw size " 2002 "exceeds device maximum", tnf_uint, maxsz, maxval); 2003 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2004 "soft_state_init_maxmrwsz_toobig_fail"); 2005 TAVOR_TNF_EXIT(tavor_soft_state_init); 2006 return (DDI_FAILURE); 2007 } 2008 hca_attr->hca_max_memr_len = ((uint64_t)1 << val); 2009 2010 /* Determine RDMA/Atomic properties */ 2011 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_rdb); 2012 hca_attr->hca_max_rsc = val; 2013 val = state->ts_cfg_profile->cp_hca_max_rdma_in_qp; 2014 hca_attr->hca_max_rdma_in_qp = val; 2015 val = state->ts_cfg_profile->cp_hca_max_rdma_out_qp; 2016 hca_attr->hca_max_rdma_out_qp = val; 2017 hca_attr->hca_max_rdma_in_ee = 0; 2018 hca_attr->hca_max_rdma_out_ee = 0; 2019 2020 /* 2021 * Determine maximum number of raw IPv6 and Ether QPs. Set to 0 2022 * because neither type of raw QP is supported 2023 */ 2024 hca_attr->hca_max_ipv6_qp = 0; 2025 hca_attr->hca_max_ether_qp = 0; 2026 2027 /* Determine max number of MCGs and max QP-per-MCG */ 2028 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_qp); 2029 hca_attr->hca_max_mcg_qps = val; 2030 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_mcg); 2031 hca_attr->hca_max_mcg = val; 2032 val = state->ts_cfg_profile->cp_num_qp_per_mcg; 2033 hca_attr->hca_max_qp_per_mcg = val; 2034 2035 /* Determine max number partitions (i.e. PKeys) */ 2036 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); 2037 val = ((uint64_t)state->ts_cfg_profile->cp_num_ports << 2038 state->ts_cfg_profile->cp_log_max_pkeytbl); 2039 2040 if (val > maxval) { 2041 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2042 TNF_PROBE_2(tavor_soft_state_init_toomanypkey_fail, 2043 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PKeys " 2044 "exceeds device maximum", tnf_uint, maxpkey, maxval); 2045 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2046 "soft_state_init_toomanypkey_fail"); 2047 TAVOR_TNF_EXIT(tavor_soft_state_init); 2048 return (DDI_FAILURE); 2049 } 2050 hca_attr->hca_max_partitions = val; 2051 2052 /* Determine number of ports */ 2053 maxval = state->ts_devlim.num_ports; 2054 val = state->ts_cfg_profile->cp_num_ports; 2055 if ((val > maxval) || (val == 0)) { 2056 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2057 TNF_PROBE_2(tavor_soft_state_init_toomanyports_fail, 2058 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of ports " 2059 "exceeds device maximum", tnf_uint, maxports, maxval); 2060 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2061 "soft_state_init_toomanyports_fail"); 2062 TAVOR_TNF_EXIT(tavor_soft_state_init); 2063 return (DDI_FAILURE); 2064 } 2065 hca_attr->hca_nports = val; 2066 2067 /* Copy NodeGUID and SystemImageGUID from softstate */ 2068 hca_attr->hca_node_guid = state->ts_nodeguid; 2069 hca_attr->hca_si_guid = state->ts_sysimgguid; 2070 2071 /* 2072 * Determine local ACK delay. Use the value suggested by the Tavor 2073 * hardware (from the QUERY_DEV_LIM command) 2074 */ 2075 hca_attr->hca_local_ack_delay = state->ts_devlim.ca_ack_delay; 2076 2077 /* Determine max SGID table and PKey table sizes */ 2078 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_gidtbl); 2079 hca_attr->hca_max_port_sgid_tbl_sz = val; 2080 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_max_pkeytbl); 2081 hca_attr->hca_max_port_pkey_tbl_sz = val; 2082 2083 /* Determine max number of PDs */ 2084 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pd); 2085 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_pd); 2086 if (val > maxval) { 2087 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2088 TNF_PROBE_2(tavor_soft_state_init_toomanypd_fail, 2089 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of PD " 2090 "exceeds device maximum", tnf_uint, maxpd, maxval); 2091 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2092 "soft_state_init_toomanypd_fail"); 2093 TAVOR_TNF_EXIT(tavor_soft_state_init); 2094 return (DDI_FAILURE); 2095 } 2096 hca_attr->hca_max_pd = val; 2097 2098 /* Determine max number of Address Handles */ 2099 maxval = ((uint64_t)1 << state->ts_devlim.log_max_av); 2100 val = ((uint64_t)1 << state->ts_cfg_profile->cp_log_num_ah); 2101 if (val > maxval) { 2102 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2103 TNF_PROBE_2(tavor_soft_state_init_toomanyah_fail, 2104 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "number of AH " 2105 "exceeds device maximum", tnf_uint, maxah, maxval); 2106 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2107 "soft_state_init_toomanyah_fail"); 2108 TAVOR_TNF_EXIT(tavor_soft_state_init); 2109 return (DDI_FAILURE); 2110 } 2111 hca_attr->hca_max_ah = val; 2112 2113 /* No RDDs or EECs (since Reliable Datagram is not supported) */ 2114 hca_attr->hca_max_rdd = 0; 2115 hca_attr->hca_max_eec = 0; 2116 2117 /* Initialize lock for reserved UAR page access */ 2118 mutex_init(&state->ts_uar_lock, NULL, MUTEX_DRIVER, 2119 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2120 2121 /* Initialize the flash fields */ 2122 state->ts_fw_flashstarted = 0; 2123 mutex_init(&state->ts_fw_flashlock, NULL, MUTEX_DRIVER, 2124 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2125 2126 /* Initialize the lock for the info ioctl */ 2127 mutex_init(&state->ts_info_lock, NULL, MUTEX_DRIVER, 2128 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2129 2130 /* Initialize the AVL tree for QP number support */ 2131 tavor_qpn_avl_init(state); 2132 2133 /* Initialize the kstat info structure */ 2134 status = tavor_kstat_init(state); 2135 if (status != DDI_SUCCESS) { 2136 tavor_qpn_avl_fini(state); 2137 mutex_destroy(&state->ts_info_lock); 2138 mutex_destroy(&state->ts_fw_flashlock); 2139 mutex_destroy(&state->ts_uar_lock); 2140 kmem_free(hca_attr, sizeof (ibt_hca_attr_t)); 2141 TNF_PROBE_0(tavor_soft_state_init_kstatinit_fail, 2142 TAVOR_TNF_ERROR, ""); 2143 TAVOR_ATTACH_MSG(state->ts_attach_buf, 2144 "soft_state_init_kstatinit_fail"); 2145 TAVOR_TNF_EXIT(tavor_soft_state_init); 2146 return (DDI_FAILURE); 2147 } 2148 2149 TAVOR_TNF_EXIT(tavor_soft_state_init); 2150 return (DDI_SUCCESS); 2151 } 2152 2153 2154 /* 2155 * tavor_soft_state_fini() 2156 * Context: Called only from detach() path context 2157 */ 2158 static void 2159 tavor_soft_state_fini(tavor_state_t *state) 2160 { 2161 TAVOR_TNF_ENTER(tavor_soft_state_fini); 2162 2163 /* Teardown the kstat info */ 2164 tavor_kstat_fini(state); 2165 2166 /* Teardown the AVL tree for QP number support */ 2167 tavor_qpn_avl_fini(state); 2168 2169 /* Free up info ioctl mutex */ 2170 mutex_destroy(&state->ts_info_lock); 2171 2172 /* Free up flash mutex */ 2173 mutex_destroy(&state->ts_fw_flashlock); 2174 2175 /* Free up the UAR page access mutex */ 2176 mutex_destroy(&state->ts_uar_lock); 2177 2178 /* Free up the hca_attr struct */ 2179 kmem_free(state->ts_ibtfinfo.hca_attr, sizeof (ibt_hca_attr_t)); 2180 2181 TAVOR_TNF_EXIT(tavor_soft_state_fini); 2182 } 2183 2184 2185 /* 2186 * tavor_hca_config_setup() 2187 * Context: Only called from attach() path context 2188 */ 2189 static void 2190 tavor_hca_config_setup(tavor_state_t *state, 2191 tavor_hw_initqueryhca_t *inithca) 2192 { 2193 tavor_rsrc_pool_info_t *rsrc_pool; 2194 uint64_t ddr_baseaddr, ddr_base_map_addr; 2195 uint64_t offset, addr; 2196 uint_t mcg_size; 2197 2198 TAVOR_TNF_ENTER(tavor_hca_config_setup); 2199 2200 /* Set "host endianness". Default is big endian */ 2201 #ifdef _LITTLE_ENDIAN 2202 inithca->big_endian = 0; 2203 #else 2204 inithca->big_endian = 1; 2205 #endif 2206 /* No Address Vector Protection, but Port Checking on by default */ 2207 inithca->udav_chk = TAVOR_UDAV_PROTECT_DISABLED; 2208 inithca->udav_port_chk = TAVOR_UDAV_PORTCHK_ENABLED; 2209 2210 ddr_baseaddr = (uint64_t)(uintptr_t)state->ts_reg_ddr_baseaddr; 2211 ddr_base_map_addr = (uint64_t)state->ts_ddr.ddr_baseaddr; 2212 2213 /* Setup QPC table */ 2214 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_QPC]; 2215 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2216 addr = ddr_base_map_addr + offset; 2217 inithca->context.qpc_baseaddr_h = (addr >> 32); 2218 inithca->context.qpc_baseaddr_l = (addr & 0xFFFFFFFF) >> 7; 2219 inithca->context.log_num_qp = state->ts_cfg_profile->cp_log_num_qp; 2220 2221 /* Setup EEC table (initialize to zero - RD unsupported) */ 2222 inithca->context.eec_baseaddr_h = 0; 2223 inithca->context.eec_baseaddr_l = 0; 2224 inithca->context.log_num_ee = 0; 2225 2226 /* Setup CQC table */ 2227 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_CQC]; 2228 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2229 addr = ddr_base_map_addr + offset; 2230 inithca->context.cqc_baseaddr_h = (addr >> 32); 2231 inithca->context.cqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2232 inithca->context.log_num_cq = state->ts_cfg_profile->cp_log_num_cq; 2233 2234 /* Setup SRQC table */ 2235 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_SRQC]; 2236 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2237 addr = ddr_base_map_addr + offset; 2238 inithca->context.srqc_baseaddr_h = (addr >> 32); 2239 inithca->context.srqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2240 inithca->context.log_num_srq = 2241 state->ts_cfg_profile->cp_log_num_srq; 2242 2243 /* Setup EQPC table */ 2244 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQPC]; 2245 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2246 addr = ddr_base_map_addr + offset; 2247 inithca->context.eqpc_baseaddr = addr; 2248 2249 /* Setup EEEC table (initialize to zero - RD unsupported) */ 2250 inithca->context.eeec_baseaddr = 0; 2251 2252 /* Setup EQC table */ 2253 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_EQC]; 2254 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2255 addr = ddr_base_map_addr + offset; 2256 inithca->context.eqc_baseaddr_h = (addr >> 32); 2257 inithca->context.eqc_baseaddr_l = (addr & 0xFFFFFFFF) >> 6; 2258 inithca->context.log_num_eq = TAVOR_NUM_EQ_SHIFT; 2259 2260 /* Setup RDB table */ 2261 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_RDB]; 2262 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2263 addr = ddr_base_map_addr + offset; 2264 inithca->context.rdb_baseaddr_h = (addr >> 32); 2265 inithca->context.rdb_baseaddr_l = 0; 2266 2267 /* Setup Multicast */ 2268 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MCG]; 2269 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2270 addr = ddr_base_map_addr + offset; 2271 inithca->multi.mc_baseaddr = addr; 2272 mcg_size = TAVOR_MCGMEM_SZ(state); 2273 inithca->multi.log_mc_tbl_ent = highbit(mcg_size) - 1; 2274 inithca->multi.mc_tbl_hash_sz = 2275 (1 << state->ts_cfg_profile->cp_log_num_mcg_hash); 2276 inithca->multi.mc_hash_fn = TAVOR_MCG_DEFAULT_HASH_FN; 2277 inithca->multi.log_mc_tbl_sz = state->ts_cfg_profile->cp_log_num_mcg; 2278 2279 2280 /* Setup TPT */ 2281 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MPT]; 2282 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2283 addr = ddr_base_map_addr + offset; 2284 inithca->tpt.mpt_baseaddr = addr; 2285 inithca->tpt.mttseg_sz = TAVOR_MTTSEG_SIZE_SHIFT; 2286 inithca->tpt.log_mpt_sz = state->ts_cfg_profile->cp_log_num_mpt; 2287 inithca->tpt.mtt_version = TAVOR_MTT_PG_WALK_VER; 2288 2289 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_MTT]; 2290 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2291 addr = ddr_base_map_addr + offset; 2292 inithca->tpt.mtt_baseaddr = addr; 2293 2294 /* Setup UAR */ 2295 rsrc_pool = &state->ts_rsrc_hdl[TAVOR_UAR_SCR]; 2296 offset = (uint64_t)(uintptr_t)rsrc_pool->rsrc_start - ddr_baseaddr; 2297 addr = ddr_base_map_addr + offset; 2298 inithca->uar.uarscr_baseaddr = addr; 2299 2300 inithca->uar.uar_pg_sz = PAGESHIFT - 0xC; 2301 2302 TAVOR_TNF_EXIT(tavor_hca_config_setup); 2303 } 2304 2305 2306 /* 2307 * tavor_hca_port_init() 2308 * Context: Only called from attach() path context 2309 */ 2310 static int 2311 tavor_hca_port_init(tavor_state_t *state) 2312 { 2313 tavor_hw_initib_t *portinits, *initib; 2314 tavor_cfg_profile_t *cfgprof; 2315 uint_t num_ports; 2316 int i, status; 2317 uint64_t maxval, val; 2318 uint64_t sysimgguid, nodeguid, portguid; 2319 2320 TAVOR_TNF_ENTER(tavor_hca_port_init); 2321 2322 cfgprof = state->ts_cfg_profile; 2323 2324 /* Get number of HCA ports */ 2325 num_ports = cfgprof->cp_num_ports; 2326 2327 /* Allocate space for Tavor port init struct(s) */ 2328 portinits = (tavor_hw_initib_t *)kmem_zalloc(num_ports * 2329 sizeof (tavor_hw_initib_t), KM_SLEEP); 2330 2331 /* Post command to initialize Tavor HCA port */ 2332 for (i = 0; i < num_ports; i++) { 2333 initib = &portinits[i]; 2334 2335 /* 2336 * Determine whether we need to override the firmware's 2337 * default SystemImageGUID setting. 2338 */ 2339 sysimgguid = cfgprof->cp_sysimgguid; 2340 if (sysimgguid != 0) { 2341 initib->set_sysimg_guid = 1; 2342 initib->sysimg_guid = sysimgguid; 2343 } 2344 2345 /* 2346 * Determine whether we need to override the firmware's 2347 * default NodeGUID setting. 2348 */ 2349 nodeguid = cfgprof->cp_nodeguid; 2350 if (nodeguid != 0) { 2351 initib->set_node_guid = 1; 2352 initib->node_guid = nodeguid; 2353 } 2354 2355 /* 2356 * Determine whether we need to override the firmware's 2357 * default PortGUID setting. 2358 */ 2359 portguid = cfgprof->cp_portguid[i]; 2360 if (portguid != 0) { 2361 initib->set_port_guid0 = 1; 2362 initib->guid0 = portguid; 2363 } 2364 2365 /* Validate max MTU size */ 2366 maxval = state->ts_devlim.max_mtu; 2367 val = cfgprof->cp_max_mtu; 2368 if (val > maxval) { 2369 TNF_PROBE_2(tavor_hca_port_init_maxmtu_fail, 2370 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2371 "MTU size exceeds device maximum", tnf_uint, 2372 maxmtu, maxval); 2373 TAVOR_TNF_EXIT(tavor_hca_port_init); 2374 goto init_ports_fail; 2375 } 2376 initib->mtu_cap = val; 2377 2378 /* Validate the max port width */ 2379 maxval = state->ts_devlim.max_port_width; 2380 val = cfgprof->cp_max_port_width; 2381 if (val > maxval) { 2382 TNF_PROBE_2(tavor_hca_port_init_maxportwidth_fail, 2383 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2384 "port width exceeds device maximum", tnf_uint, 2385 maxportwidth, maxval); 2386 TAVOR_TNF_EXIT(tavor_hca_port_init); 2387 goto init_ports_fail; 2388 } 2389 initib->port_width_cap = val; 2390 2391 /* Validate max VL cap size */ 2392 maxval = state->ts_devlim.max_vl; 2393 val = cfgprof->cp_max_vlcap; 2394 if (val > maxval) { 2395 TNF_PROBE_2(tavor_hca_port_init_maxvlcap_fail, 2396 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2397 "VLcap size exceeds device maximum", tnf_uint, 2398 maxvlcap, maxval); 2399 TAVOR_TNF_EXIT(tavor_hca_port_init); 2400 goto init_ports_fail; 2401 } 2402 initib->vl_cap = val; 2403 2404 /* Validate max GID table size */ 2405 maxval = ((uint64_t)1 << state->ts_devlim.log_max_gid); 2406 val = ((uint64_t)1 << cfgprof->cp_log_max_gidtbl); 2407 if (val > maxval) { 2408 TNF_PROBE_2(tavor_hca_port_init_gidtable_fail, 2409 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2410 "GID table size exceeds device maximum", tnf_uint, 2411 maxgidtbl, maxval); 2412 TAVOR_TNF_EXIT(tavor_hca_port_init); 2413 goto init_ports_fail; 2414 } 2415 initib->max_gid = val; 2416 2417 /* Validate max PKey table size */ 2418 maxval = ((uint64_t)1 << state->ts_devlim.log_max_pkey); 2419 val = ((uint64_t)1 << cfgprof->cp_log_max_pkeytbl); 2420 if (val > maxval) { 2421 TNF_PROBE_2(tavor_hca_port_init_pkeytable_fail, 2422 TAVOR_TNF_ERROR, "", tnf_string, errmsg, "max " 2423 "PKey table size exceeds device maximum", tnf_uint, 2424 maxpkeytbl, maxval); 2425 TAVOR_TNF_EXIT(tavor_hca_port_init); 2426 goto init_ports_fail; 2427 } 2428 initib->max_pkey = val; 2429 2430 /* 2431 * Post the INIT_IB command to Tavor firmware. When this 2432 * command completes, the corresponding Tavor port will be 2433 * physically "Up" and initialized. 2434 */ 2435 status = tavor_init_ib_cmd_post(state, initib, i + 1, 2436 TAVOR_CMD_NOSLEEP_SPIN); 2437 if (status != TAVOR_CMD_SUCCESS) { 2438 cmn_err(CE_CONT, "Tavor: INIT_IB (port %02d) command " 2439 "failed: %08x\n", i + 1, status); 2440 TNF_PROBE_2(tavor_hca_port_init_init_ib_cmd_fail, 2441 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, 2442 tnf_uint, port, i + 1); 2443 TAVOR_TNF_EXIT(tavor_hca_port_init); 2444 goto init_ports_fail; 2445 } 2446 } 2447 2448 /* Free up the memory for Tavor port init struct(s), return success */ 2449 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); 2450 TAVOR_TNF_EXIT(tavor_hca_port_init); 2451 return (DDI_SUCCESS); 2452 2453 init_ports_fail: 2454 /* 2455 * Free up the memory for Tavor port init struct(s), shutdown any 2456 * successfully initialized ports, and return failure 2457 */ 2458 kmem_free(portinits, num_ports * sizeof (tavor_hw_initib_t)); 2459 (void) tavor_hca_ports_shutdown(state, i); 2460 2461 TAVOR_TNF_EXIT(tavor_hca_port_init); 2462 return (DDI_FAILURE); 2463 } 2464 2465 2466 /* 2467 * tavor_hca_ports_shutdown() 2468 * Context: Only called from attach() and/or detach() path contexts 2469 */ 2470 static int 2471 tavor_hca_ports_shutdown(tavor_state_t *state, uint_t num_init) 2472 { 2473 int i, status; 2474 2475 TAVOR_TNF_ENTER(tavor_hca_ports_shutdown); 2476 2477 /* 2478 * Post commands to shutdown all init'd Tavor HCA ports. Note: if 2479 * any of these commands fail for any reason, it would be entirely 2480 * unexpected and probably indicative a serious problem (HW or SW). 2481 * Although we do return void from this function, this type of failure 2482 * should not go unreported. That is why we have the warning message 2483 * and the detailed TNF information. 2484 */ 2485 for (i = 0; i < num_init; i++) { 2486 status = tavor_close_ib_cmd_post(state, i + 1, 2487 TAVOR_CMD_NOSLEEP_SPIN); 2488 if (status != TAVOR_CMD_SUCCESS) { 2489 TAVOR_WARNING(state, "failed to shutdown HCA port"); 2490 TNF_PROBE_2(tavor_hca_ports_shutdown_close_ib_cmd_fail, 2491 TAVOR_TNF_ERROR, "", tnf_uint, cmd_status, status, 2492 tnf_uint, port, i + 1); 2493 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); 2494 return (status); 2495 } 2496 } 2497 2498 TAVOR_TNF_EXIT(tavor_hca_ports_shutdown); 2499 2500 return (TAVOR_CMD_SUCCESS); 2501 } 2502 2503 2504 /* 2505 * tavor_internal_uarpgs_init 2506 * Context: Only called from attach() path context 2507 */ 2508 static int 2509 tavor_internal_uarpgs_init(tavor_state_t *state) 2510 { 2511 int status; 2512 2513 TAVOR_TNF_ENTER(tavor_internal_uarpgs_init); 2514 2515 /* 2516 * Save away reserved Tavor UAR page #0. This UAR page is not to 2517 * be used by software. 2518 */ 2519 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, 2520 &state->ts_uarpg0_rsrc_rsrvd); 2521 if (status != DDI_SUCCESS) { 2522 TNF_PROBE_0(tavor_uarpg0_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); 2523 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2524 return (DDI_FAILURE); 2525 } 2526 2527 /* 2528 * Save away Tavor UAR page #1 (for internal use). This UAR page is 2529 * the privileged UAR page through which all kernel generated 2530 * doorbells will be rung. 2531 */ 2532 status = tavor_rsrc_alloc(state, TAVOR_UARPG, 1, TAVOR_SLEEP, 2533 &state->ts_uarpg1_rsrc); 2534 if (status != DDI_SUCCESS) { 2535 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); 2536 TNF_PROBE_0(tavor_uarpg1_rsrcalloc_fail, TAVOR_TNF_ERROR, ""); 2537 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2538 return (DDI_FAILURE); 2539 } 2540 2541 /* Setup pointer to UAR page #1 doorbells */ 2542 state->ts_uar = (tavor_hw_uar_t *)state->ts_uarpg1_rsrc->tr_addr; 2543 2544 TAVOR_TNF_EXIT(tavor_internal_uarpgs_init); 2545 return (DDI_SUCCESS); 2546 } 2547 2548 2549 /* 2550 * tavor_internal_uarpgs_fini 2551 * Context: Only called from attach() and/or detach() path contexts 2552 */ 2553 static void 2554 tavor_internal_uarpgs_fini(tavor_state_t *state) 2555 { 2556 TAVOR_TNF_ENTER(tavor_internal_uarpgs_fini); 2557 2558 /* Free up Tavor UAR page #1 (kernel driver doorbells) */ 2559 tavor_rsrc_free(state, &state->ts_uarpg1_rsrc); 2560 2561 /* Free up Tavor UAR page #0 (reserved) */ 2562 tavor_rsrc_free(state, &state->ts_uarpg0_rsrc_rsrvd); 2563 2564 TAVOR_TNF_EXIT(tavor_internal_uarpgs_fini); 2565 } 2566 2567 2568 /* 2569 * tavor_special_qp_contexts_reserve() 2570 * Context: Only called from attach() path context 2571 */ 2572 static int 2573 tavor_special_qp_contexts_reserve(tavor_state_t *state) 2574 { 2575 tavor_rsrc_t *qp0_rsrc, *qp1_rsrc; 2576 int status; 2577 2578 TAVOR_TNF_ENTER(tavor_special_qp_contexts_reserve); 2579 2580 /* Initialize the lock used for special QP rsrc management */ 2581 mutex_init(&state->ts_spec_qplock, NULL, MUTEX_DRIVER, 2582 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2583 2584 /* 2585 * Reserve contexts for QP0. These QP contexts will be setup to 2586 * act as aliases for the real QP0. Note: We are required to grab 2587 * two QPs (one per port) even if we are operating in single-port 2588 * mode. 2589 */ 2590 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp0_rsrc); 2591 if (status != DDI_SUCCESS) { 2592 mutex_destroy(&state->ts_spec_qplock); 2593 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp0_fail, 2594 TAVOR_TNF_ERROR, ""); 2595 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2596 return (DDI_FAILURE); 2597 } 2598 state->ts_spec_qp0 = qp0_rsrc; 2599 2600 /* 2601 * Reserve contexts for QP1. These QP contexts will be setup to 2602 * act as aliases for the real QP1. Note: We are required to grab 2603 * two QPs (one per port) even if we are operating in single-port 2604 * mode. 2605 */ 2606 status = tavor_rsrc_alloc(state, TAVOR_QPC, 2, TAVOR_SLEEP, &qp1_rsrc); 2607 if (status != DDI_SUCCESS) { 2608 tavor_rsrc_free(state, &qp0_rsrc); 2609 mutex_destroy(&state->ts_spec_qplock); 2610 TNF_PROBE_0(tavor_special_qp_contexts_reserve_qp1_fail, 2611 TAVOR_TNF_ERROR, ""); 2612 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2613 return (DDI_FAILURE); 2614 } 2615 state->ts_spec_qp1 = qp1_rsrc; 2616 2617 TAVOR_TNF_EXIT(tavor_special_qp_contexts_reserve); 2618 return (DDI_SUCCESS); 2619 } 2620 2621 2622 /* 2623 * tavor_special_qp_contexts_unreserve() 2624 * Context: Only called from attach() and/or detach() path contexts 2625 */ 2626 static void 2627 tavor_special_qp_contexts_unreserve(tavor_state_t *state) 2628 { 2629 TAVOR_TNF_ENTER(tavor_special_qp_contexts_unreserve); 2630 2631 /* Unreserve contexts for QP1 */ 2632 tavor_rsrc_free(state, &state->ts_spec_qp1); 2633 2634 /* Unreserve contexts for QP0 */ 2635 tavor_rsrc_free(state, &state->ts_spec_qp0); 2636 2637 /* Destroy the lock used for special QP rsrc management */ 2638 mutex_destroy(&state->ts_spec_qplock); 2639 2640 TAVOR_TNF_EXIT(tavor_special_qp_contexts_unreserve); 2641 } 2642 2643 2644 /* 2645 * tavor_sw_reset() 2646 * Context: Currently called only from attach() path context 2647 */ 2648 static int 2649 tavor_sw_reset(tavor_state_t *state) 2650 { 2651 dev_info_t *dip, *pdip; 2652 ddi_acc_handle_t hdl = state->ts_pci_cfghdl, phdl; 2653 uint32_t reset_delay; 2654 int status, i; 2655 2656 TAVOR_TNF_ENTER(tavor_sw_reset); 2657 2658 /* 2659 * If the configured software reset delay is set to zero, then we 2660 * will not attempt a software reset of the Tavor device. 2661 */ 2662 reset_delay = state->ts_cfg_profile->cp_sw_reset_delay; 2663 if (reset_delay == 0) { 2664 TAVOR_TNF_EXIT(tavor_sw_reset); 2665 return (DDI_SUCCESS); 2666 } 2667 2668 /* 2669 * Get dip for HCA device _and_ parent device as well. Parent access 2670 * is necessary here because software reset of the Tavor hardware 2671 * will reinitialize both the config registers of the PCI bridge 2672 * (parent, if it exists) and the IB HCA (self) 2673 */ 2674 dip = state->ts_dip; 2675 pdip = ddi_get_parent(dip); 2676 2677 /* Query the PCI capabilities of the HCA device */ 2678 tavor_pci_capability_list(state, hdl); 2679 2680 /* 2681 * Read all PCI config info (reg0...reg63). Note: According to the 2682 * Tavor software reset application note, we should not read or 2683 * restore the values in reg22 and reg23. 2684 */ 2685 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2686 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2687 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2688 state->ts_cfg_data[i] = pci_config_get32(hdl, i << 2); 2689 } 2690 } 2691 2692 if (TAVOR_PARENT_IS_BRIDGE(pdip)) { 2693 /* 2694 * Setup for PCI config read/write of bridge device 2695 */ 2696 status = pci_config_setup(pdip, &phdl); 2697 if (status != DDI_SUCCESS) { 2698 TNF_PROBE_0(tavor_sw_reset_pcicfg_p_fail, 2699 TAVOR_TNF_ERROR, ""); 2700 TAVOR_TNF_EXIT(tavor_sw_reset); 2701 return (DDI_FAILURE); 2702 } 2703 2704 /* 2705 * Read all PCI config info (reg0...reg63). Note: According to 2706 * the Tavor software reset application note, we should not 2707 * read or restore the values in reg22 and reg23. 2708 */ 2709 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2710 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2711 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2712 state->ts_cfg_pdata[i] = 2713 pci_config_get32(phdl, i << 2); 2714 } 2715 } 2716 } 2717 2718 /* 2719 * Perform the software reset (by writing 1 at offset 0xF0010) 2720 */ 2721 ddi_put32(state->ts_reg_cmdhdl, state->ts_cmd_regs.sw_reset, 2722 TAVOR_SW_RESET_START); 2723 2724 drv_usecwait(reset_delay); 2725 2726 if (TAVOR_PARENT_IS_BRIDGE(pdip)) { 2727 /* 2728 * Bridge exists, so wait for the bridge to become ready. 2729 * 2730 * The above delay is necessary to avoid system panic from 2731 * Master Abort. If the device is accessed before this delay, 2732 * device will not respond to config cycles and they will be 2733 * terminate with a Master Abort which will panic the system. 2734 * Below is the loop we use to poll status from the device to 2735 * determine if it is OK to proceed. 2736 */ 2737 i = 0; 2738 while (pci_config_get32(phdl, 0) == TAVOR_SW_RESET_NOTDONE) { 2739 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); 2740 } 2741 2742 /* 2743 * Write all the PCI config registers back into each device 2744 * (except for reg22 and reg23 - see above) 2745 */ 2746 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2747 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2748 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2749 pci_config_put32(phdl, i << 2, 2750 state->ts_cfg_pdata[i]); 2751 } 2752 } 2753 2754 /* 2755 * Tear down the config setup (for bridge device) 2756 */ 2757 pci_config_teardown(&phdl); 2758 2759 /* No Bridge Device */ 2760 } else { 2761 /* 2762 * Bridge does not exist, so instead wait for the device itself 2763 * to become ready. 2764 * 2765 * The above delay is necessary to avoid system panic from 2766 * Master Abort. If the device is accessed before this delay, 2767 * device will not respond to config cycles and they will be 2768 * terminate with a Master Abort which will panic the system. 2769 * Below is the loop we use to poll status from the device to 2770 * determine if it is OK to proceed. 2771 */ 2772 i = 0; 2773 while (pci_config_get32(hdl, 0) == TAVOR_SW_RESET_NOTDONE) { 2774 drv_usecwait(TAVOR_SW_RESET_POLL_DELAY); 2775 } 2776 } 2777 2778 for (i = 0; i < TAVOR_SW_RESET_NUMREGS; i++) { 2779 if ((i != TAVOR_SW_RESET_REG22_RSVD) && 2780 (i != TAVOR_SW_RESET_REG23_RSVD)) { 2781 pci_config_put32(hdl, i << 2, state->ts_cfg_data[i]); 2782 } 2783 } 2784 2785 TAVOR_TNF_EXIT(tavor_sw_reset); 2786 return (DDI_SUCCESS); 2787 } 2788 2789 2790 /* 2791 * tavor_mcg_init() 2792 * Context: Only called from attach() path context 2793 */ 2794 static int 2795 tavor_mcg_init(tavor_state_t *state) 2796 { 2797 uint_t mcg_tmp_sz; 2798 2799 TAVOR_TNF_ENTER(tavor_mcg_init); 2800 2801 /* 2802 * Allocate space for the MCG temporary copy buffer. This is 2803 * used by the Attach/Detach Multicast Group code 2804 */ 2805 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); 2806 state->ts_mcgtmp = kmem_zalloc(mcg_tmp_sz, KM_SLEEP); 2807 2808 /* 2809 * Initialize the multicast group mutex. This ensures atomic 2810 * access to add, modify, and remove entries in the multicast 2811 * group hash lists. 2812 */ 2813 mutex_init(&state->ts_mcglock, NULL, MUTEX_DRIVER, 2814 DDI_INTR_PRI(state->ts_intrmsi_pri)); 2815 2816 TAVOR_TNF_EXIT(tavor_mcg_init); 2817 return (DDI_SUCCESS); 2818 } 2819 2820 2821 /* 2822 * tavor_mcg_fini() 2823 * Context: Only called from attach() and/or detach() path contexts 2824 */ 2825 static void 2826 tavor_mcg_fini(tavor_state_t *state) 2827 { 2828 uint_t mcg_tmp_sz; 2829 2830 TAVOR_TNF_ENTER(tavor_mcg_fini); 2831 2832 /* Free up the space used for the MCG temporary copy buffer */ 2833 mcg_tmp_sz = TAVOR_MCGMEM_SZ(state); 2834 kmem_free(state->ts_mcgtmp, mcg_tmp_sz); 2835 2836 /* Destroy the multicast group mutex */ 2837 mutex_destroy(&state->ts_mcglock); 2838 2839 TAVOR_TNF_EXIT(tavor_mcg_fini); 2840 } 2841 2842 2843 /* 2844 * tavor_fw_version_check() 2845 * Context: Only called from attach() path context 2846 */ 2847 static int 2848 tavor_fw_version_check(tavor_state_t *state) 2849 { 2850 uint_t tavor_fw_ver_major; 2851 uint_t tavor_fw_ver_minor; 2852 uint_t tavor_fw_ver_subminor; 2853 2854 /* 2855 * Depending on which version of driver we have attached, the firmware 2856 * version checks will be different. We set up the comparison values 2857 * for both HCA Mode (Tavor hardware) or COMPAT Mode (Arbel hardware 2858 * running in tavor mode). 2859 */ 2860 switch (state->ts_operational_mode) { 2861 case TAVOR_HCA_MODE: 2862 tavor_fw_ver_major = TAVOR_FW_VER_MAJOR; 2863 tavor_fw_ver_minor = TAVOR_FW_VER_MINOR; 2864 tavor_fw_ver_subminor = TAVOR_FW_VER_SUBMINOR; 2865 break; 2866 2867 case TAVOR_COMPAT_MODE: 2868 tavor_fw_ver_major = TAVOR_COMPAT_FW_VER_MAJOR; 2869 tavor_fw_ver_minor = TAVOR_COMPAT_FW_VER_MINOR; 2870 tavor_fw_ver_subminor = TAVOR_COMPAT_FW_VER_SUBMINOR; 2871 break; 2872 2873 default: 2874 return (DDI_FAILURE); 2875 } 2876 2877 /* 2878 * If FW revision major number is less than acceptable, 2879 * return failure, else if greater return success. If 2880 * the major numbers are equal than check the minor number 2881 */ 2882 if (state->ts_fw.fw_rev_major < tavor_fw_ver_major) { 2883 return (DDI_FAILURE); 2884 } else if (state->ts_fw.fw_rev_major > tavor_fw_ver_major) { 2885 return (DDI_SUCCESS); 2886 } 2887 /* 2888 * Do the same check as above, except for minor revision numbers 2889 * If the minor numbers are equal than check the subminor number 2890 */ 2891 if (state->ts_fw.fw_rev_minor < tavor_fw_ver_minor) { 2892 return (DDI_FAILURE); 2893 } else if (state->ts_fw.fw_rev_minor > tavor_fw_ver_minor) { 2894 return (DDI_SUCCESS); 2895 } 2896 2897 /* 2898 * Once again we do the same check as above, except for the subminor 2899 * revision number. If the subminor numbers are equal here, then 2900 * these are the same firmware version, return success 2901 */ 2902 if (state->ts_fw.fw_rev_subminor < tavor_fw_ver_subminor) { 2903 return (DDI_FAILURE); 2904 } else if (state->ts_fw.fw_rev_subminor > tavor_fw_ver_subminor) { 2905 return (DDI_SUCCESS); 2906 } 2907 2908 return (DDI_SUCCESS); 2909 } 2910 2911 2912 /* 2913 * tavor_device_info_report() 2914 * Context: Only called from attach() path context 2915 */ 2916 static void 2917 tavor_device_info_report(tavor_state_t *state) 2918 { 2919 cmn_err(CE_CONT, "?tavor%d: FW ver: %04d.%04d.%04d, " 2920 "HW rev: %02x\n", state->ts_instance, state->ts_fw.fw_rev_major, 2921 state->ts_fw.fw_rev_minor, state->ts_fw.fw_rev_subminor, 2922 state->ts_adapter.rev_id); 2923 cmn_err(CE_CONT, "?tavor%d: %64s (0x%016" PRIx64 ")\n", 2924 state->ts_instance, state->ts_nodedesc, state->ts_nodeguid); 2925 } 2926 2927 2928 /* 2929 * tavor_pci_capability_list() 2930 * Context: Only called from attach() path context 2931 */ 2932 static void 2933 tavor_pci_capability_list(tavor_state_t *state, ddi_acc_handle_t hdl) 2934 { 2935 uint_t offset, data; 2936 2937 TAVOR_TNF_ENTER(tavor_pci_capability_list); 2938 2939 /* 2940 * Check for the "PCI Capabilities" bit in the "Status Register". 2941 * Bit 4 in this register indicates the presence of a "PCI 2942 * Capabilities" list. 2943 */ 2944 data = pci_config_get16(hdl, 0x6); 2945 if ((data & 0x10) == 0) { 2946 TNF_PROBE_0(tavor_pci_capab_list_fail, TAVOR_TNF_ERROR, ""); 2947 TAVOR_TNF_EXIT(tavor_pci_capability_list); 2948 return; 2949 } 2950 2951 /* 2952 * Starting from offset 0x34 in PCI config space, find the 2953 * head of "PCI capabilities" list, and walk the list. If 2954 * capabilities of a known type are encountered (e.g. 2955 * "PCI-X Capability"), then call the appropriate handler 2956 * function. 2957 */ 2958 offset = pci_config_get8(hdl, 0x34); 2959 while (offset != 0x0) { 2960 data = pci_config_get8(hdl, offset); 2961 2962 /* 2963 * Check for known capability types. Tavor has the 2964 * following: 2965 * o VPD Capability (0x03) 2966 * o PCI-X Capability (0x07) 2967 * o MSI Capability (0x05) 2968 * o MSIX Capability (0x11) 2969 */ 2970 switch (data) { 2971 case 0x03: 2972 tavor_pci_capability_vpd(state, hdl, offset); 2973 break; 2974 case 0x07: 2975 tavor_pci_capability_pcix(state, hdl, offset); 2976 break; 2977 case 0x05: 2978 break; 2979 default: 2980 break; 2981 } 2982 2983 /* Get offset of next entry in list */ 2984 offset = pci_config_get8(hdl, offset + 1); 2985 } 2986 2987 TAVOR_TNF_EXIT(tavor_pci_capability_list); 2988 } 2989 2990 /* 2991 * tavor_pci_read_vpd() 2992 * Context: Only called from attach() path context 2993 * utility routine for tavor_pci_capability_vpd() 2994 */ 2995 static int 2996 tavor_pci_read_vpd(ddi_acc_handle_t hdl, uint_t offset, uint32_t addr, 2997 uint32_t *data) 2998 { 2999 int retry = 4; /* retry counter for EEPROM poll */ 3000 uint32_t val; 3001 int vpd_addr = offset + 2; 3002 int vpd_data = offset + 4; 3003 3004 TAVOR_TNF_ENTER(tavor_pci_read_vpd); 3005 3006 /* 3007 * In order to read a 32-bit value from VPD, we are to write down 3008 * the address (offset in the VPD itself) to the address register. 3009 * To signal the read, we also clear bit 31. We then poll on bit 31 3010 * and when it is set, we can then read our 4 bytes from the data 3011 * register. 3012 */ 3013 (void) pci_config_put32(hdl, offset, addr << 16); 3014 do { 3015 drv_usecwait(1000); 3016 val = pci_config_get16(hdl, vpd_addr); 3017 if ((val >> 15) & 0x01) { 3018 *data = pci_config_get32(hdl, vpd_data); 3019 TAVOR_TNF_EXIT(tavor_pci_read_vpd); 3020 return (DDI_SUCCESS); 3021 } 3022 } while (--retry); 3023 3024 TNF_PROBE_0(tavor_pci_read_vpd_fail, TAVOR_TNF_ERROR, ""); 3025 TAVOR_TNF_EXIT(tavor_pci_read_vpd); 3026 return (DDI_FAILURE); 3027 } 3028 3029 3030 /* 3031 * tavor_pci_capability_vpd() 3032 * Context: Only called from attach() path context 3033 */ 3034 static void 3035 tavor_pci_capability_vpd(tavor_state_t *state, ddi_acc_handle_t hdl, 3036 uint_t offset) 3037 { 3038 uint8_t name_length; 3039 uint8_t pn_length; 3040 int i, err = 0; 3041 int vpd_str_id = 0; 3042 int vpd_ro_desc; 3043 int vpd_ro_pn_desc; 3044 #ifndef _LITTLE_ENDIAN 3045 uint32_t data32; 3046 #endif /* _LITTLE_ENDIAN */ 3047 union { 3048 uint32_t vpd_int[TAVOR_VPD_HDR_DWSIZE]; 3049 uchar_t vpd_char[TAVOR_VPD_HDR_BSIZE]; 3050 } vpd; 3051 3052 TAVOR_TNF_ENTER(tavor_pci_capability_vpd); 3053 3054 /* 3055 * Read Vital Product Data (VPD) from PCI-X capability. 3056 */ 3057 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { 3058 err = tavor_pci_read_vpd(hdl, offset, i << 2, &vpd.vpd_int[i]); 3059 if (err != DDI_SUCCESS) { 3060 cmn_err(CE_NOTE, "!VPD read failed\n"); 3061 goto out; 3062 } 3063 } 3064 3065 #ifndef _LITTLE_ENDIAN 3066 /* 3067 * Need to swap bytes for big endian. 3068 */ 3069 for (i = 0; i < TAVOR_VPD_HDR_DWSIZE; i++) { 3070 data32 = vpd.vpd_int[i]; 3071 vpd.vpd_char[(i << 2) + 3] = 3072 (uchar_t)((data32 & 0xFF000000) >> 24); 3073 vpd.vpd_char[(i << 2) + 2] = 3074 (uchar_t)((data32 & 0x00FF0000) >> 16); 3075 vpd.vpd_char[(i << 2) + 1] = 3076 (uchar_t)((data32 & 0x0000FF00) >> 8); 3077 vpd.vpd_char[i << 2] = (uchar_t)(data32 & 0x000000FF); 3078 } 3079 #endif /* _LITTLE_ENDIAN */ 3080 3081 /* Check for VPD String ID Tag */ 3082 if (vpd.vpd_char[vpd_str_id] == 0x82) { 3083 /* get the product name */ 3084 name_length = (uint8_t)vpd.vpd_char[vpd_str_id + 1]; 3085 if (name_length > sizeof (state->ts_hca_name)) { 3086 cmn_err(CE_NOTE, "!VPD name too large (0x%x)\n", 3087 name_length); 3088 goto out; 3089 } 3090 (void) memcpy(state->ts_hca_name, &vpd.vpd_char[vpd_str_id + 3], 3091 name_length); 3092 state->ts_hca_name[name_length] = 0; 3093 3094 /* get the part number */ 3095 vpd_ro_desc = name_length + 3; /* read-only tag location */ 3096 vpd_ro_pn_desc = vpd_ro_desc + 3; /* P/N keyword location */ 3097 /* 3098 * Verify read-only tag and Part Number keyword. 3099 */ 3100 if (vpd.vpd_char[vpd_ro_desc] != 0x90 || 3101 (vpd.vpd_char[vpd_ro_pn_desc] != 'P' && 3102 vpd.vpd_char[vpd_ro_pn_desc + 1] != 'N')) { 3103 cmn_err(CE_NOTE, "!VPD Part Number not found\n"); 3104 goto out; 3105 } 3106 3107 pn_length = (uint8_t)vpd.vpd_char[vpd_ro_pn_desc + 2]; 3108 if (pn_length > sizeof (state->ts_hca_pn)) { 3109 cmn_err(CE_NOTE, "!VPD part number too large (0x%x)\n", 3110 name_length); 3111 goto out; 3112 } 3113 (void) memcpy(state->ts_hca_pn, 3114 &vpd.vpd_char[vpd_ro_pn_desc + 3], 3115 pn_length); 3116 state->ts_hca_pn[pn_length] = 0; 3117 state->ts_hca_pn_len = pn_length; 3118 } else { 3119 /* Wrong VPD String ID Tag */ 3120 cmn_err(CE_NOTE, "!VPD String ID Tag not found, tag: %02x\n", 3121 vpd.vpd_char[0]); 3122 goto out; 3123 } 3124 TAVOR_TNF_EXIT(tavor_pci_capability_vpd); 3125 return; 3126 out: 3127 state->ts_hca_pn_len = 0; 3128 TNF_PROBE_0(tavor_pci_capability_vpd_fail, TAVOR_TNF_ERROR, ""); 3129 TAVOR_TNF_EXIT(tavor_pci_capability_vpd); 3130 } 3131 3132 /* 3133 * tavor_pci_capability_pcix() 3134 * Context: Only called from attach() path context 3135 */ 3136 static void 3137 tavor_pci_capability_pcix(tavor_state_t *state, ddi_acc_handle_t hdl, 3138 uint_t offset) 3139 { 3140 uint_t command, status; 3141 int max_out_splt_trans, max_mem_rd_byte_cnt; 3142 int designed_max_out_splt_trans, designed_max_mem_rd_byte_cnt; 3143 3144 TAVOR_TNF_ENTER(tavor_pci_capability_pcix); 3145 3146 /* 3147 * Query the current values for the PCI-X Command Register and 3148 * the PCI-X Status Register. 3149 */ 3150 command = pci_config_get16(hdl, offset + 2); 3151 status = pci_config_get32(hdl, offset + 4); 3152 3153 /* 3154 * Check for config property specifying "maximum outstanding 3155 * split transactions". If the property is defined and valid 3156 * (i.e. no larger than the so-called "designed maximum"), 3157 * then use the specified value to update the PCI-X Command Register. 3158 * Otherwise, extract the value from the Tavor config profile. 3159 */ 3160 designed_max_out_splt_trans = ((status >> 23) & 7); 3161 max_out_splt_trans = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, 3162 DDI_PROP_DONTPASS, "pcix-max-outstanding-split-trans", -1); 3163 if ((max_out_splt_trans != -1) && 3164 ((max_out_splt_trans < 0) || 3165 (max_out_splt_trans > designed_max_out_splt_trans))) { 3166 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-outstanding-" 3167 "split-trans\" (%d) invalid or exceeds device maximum" 3168 " (%d), using default value (%d)\n", state->ts_instance, 3169 max_out_splt_trans, designed_max_out_splt_trans, 3170 state->ts_cfg_profile->cp_max_out_splt_trans); 3171 max_out_splt_trans = 3172 state->ts_cfg_profile->cp_max_out_splt_trans; 3173 } else if (max_out_splt_trans == -1) { 3174 max_out_splt_trans = 3175 state->ts_cfg_profile->cp_max_out_splt_trans; 3176 } 3177 3178 /* 3179 * The config profile setting for max_out_splt_trans is determined 3180 * based on arch. Check tavor_cfg.c for more information. A value of 3181 * '-1' in the patchable variable means "do not change". A value of 3182 * '0' means 1 outstanding splt trans and other values as defined by 3183 * PCI. So we do one more check here, that if 'max_out_splt_trans' is 3184 * -1 (ie: < 0) we do not set the PCI command and leave it at the 3185 * default. 3186 */ 3187 if (max_out_splt_trans >= 0) { 3188 command = ((command & 0xFF8F) | max_out_splt_trans << 4); 3189 } 3190 3191 /* 3192 * Check for config property specifying "maximum memory read 3193 * byte count. If the property is defined and valid 3194 * (i.e. no larger than the so-called "designed maximum"), 3195 * then use the specified value to update the PCI-X Command Register. 3196 * Otherwise, extract the value from the Tavor config profile. 3197 */ 3198 designed_max_mem_rd_byte_cnt = ((status >> 21) & 3); 3199 max_mem_rd_byte_cnt = ddi_prop_get_int(DDI_DEV_T_ANY, state->ts_dip, 3200 DDI_PROP_DONTPASS, "pcix-max-read-byte-count", -1); 3201 if ((max_mem_rd_byte_cnt != -1) && 3202 ((max_mem_rd_byte_cnt < 0) || 3203 (max_mem_rd_byte_cnt > designed_max_mem_rd_byte_cnt))) { 3204 cmn_err(CE_NOTE, "!tavor%d: property \"pcix-max-read-byte-" 3205 "count\" (%d) invalid or exceeds device maximum" 3206 " (%d), using default value (%d)\n", state->ts_instance, 3207 max_mem_rd_byte_cnt, designed_max_mem_rd_byte_cnt, 3208 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt); 3209 max_mem_rd_byte_cnt = 3210 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; 3211 } else if (max_mem_rd_byte_cnt == -1) { 3212 max_mem_rd_byte_cnt = 3213 state->ts_cfg_profile->cp_max_mem_rd_byte_cnt; 3214 } 3215 3216 /* 3217 * The config profile setting for max_mem_rd_byte_cnt is determined 3218 * based on arch. Check tavor_cfg.c for more information. A value of 3219 * '-1' in the patchable variable means "do not change". A value of 3220 * '0' means minimum (512B) read, and other values as defined by 3221 * PCI. So we do one more check here, that if 'max_mem_rd_byte_cnt' is 3222 * -1 (ie: < 0) we do not set the PCI command and leave it at the 3223 * default. 3224 */ 3225 if (max_mem_rd_byte_cnt >= 0) { 3226 command = ((command & 0xFFF3) | max_mem_rd_byte_cnt << 2); 3227 } 3228 3229 /* 3230 * Update the PCI-X Command Register with the newly configured 3231 * values. 3232 */ 3233 pci_config_put16(hdl, offset + 2, command); 3234 3235 TAVOR_TNF_EXIT(tavor_pci_capability_pcix); 3236 } 3237 3238 3239 /* 3240 * tavor_intr_or_msi_init() 3241 * Context: Only called from attach() path context 3242 */ 3243 static int 3244 tavor_intr_or_msi_init(tavor_state_t *state) 3245 { 3246 int status; 3247 3248 TAVOR_TNF_ENTER(tavor_intr_or_msi_init); 3249 3250 /* Query for the list of supported interrupt event types */ 3251 status = ddi_intr_get_supported_types(state->ts_dip, 3252 &state->ts_intr_types_avail); 3253 if (status != DDI_SUCCESS) { 3254 TNF_PROBE_0(tavor_intr_or_msi_init_gettypes_fail, 3255 TAVOR_TNF_ERROR, ""); 3256 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3257 return (DDI_FAILURE); 3258 } 3259 3260 /* 3261 * If Tavor/Arbel supports MSI in this system (and, if it 3262 * hasn't been overridden by a configuration variable), then 3263 * the default behavior is to use a single MSI. Otherwise, 3264 * fallback to using legacy interrupts. Also, if MSI allocatis chosen, 3265 * but fails for whatever reasons, then fallback to using legacy 3266 * interrupts. 3267 */ 3268 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && 3269 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { 3270 status = tavor_add_intrs(state, DDI_INTR_TYPE_MSI); 3271 if (status == DDI_SUCCESS) { 3272 state->ts_intr_type_chosen = DDI_INTR_TYPE_MSI; 3273 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3274 return (DDI_SUCCESS); 3275 } 3276 } 3277 3278 /* 3279 * MSI interrupt allocation failed, or was not available. Fallback to 3280 * legacy interrupt support. 3281 */ 3282 if (state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED) { 3283 status = tavor_add_intrs(state, DDI_INTR_TYPE_FIXED); 3284 if (status == DDI_SUCCESS) { 3285 state->ts_intr_type_chosen = DDI_INTR_TYPE_FIXED; 3286 TAVOR_TNF_EXIT(tavor_intr_or_msi_init); 3287 return (DDI_SUCCESS); 3288 } 3289 } 3290 3291 /* 3292 * Neither MSI or legacy interrupts were successful. return failure. 3293 */ 3294 TAVOR_TNF_EXIT(tavor_intr_or_msi_setup); 3295 return (DDI_FAILURE); 3296 } 3297 3298 /* 3299 * tavor_add_intrs() 3300 * Context: Only called from attach() patch context 3301 */ 3302 static int 3303 tavor_add_intrs(tavor_state_t *state, int intr_type) 3304 { 3305 int status; 3306 3307 TAVOR_TNF_ENTER(tavor_add_intrs); 3308 3309 /* Get number of interrupts/MSI supported */ 3310 status = ddi_intr_get_nintrs(state->ts_dip, intr_type, 3311 &state->ts_intrmsi_count); 3312 if (status != DDI_SUCCESS) { 3313 TNF_PROBE_0(tavor_add_intrs_getnintrs_fail, 3314 TAVOR_TNF_ERROR, ""); 3315 TAVOR_TNF_EXIT(tavor_add_intrs); 3316 return (DDI_FAILURE); 3317 } 3318 3319 /* Get number of available interrupts/MSI */ 3320 status = ddi_intr_get_navail(state->ts_dip, intr_type, 3321 &state->ts_intrmsi_avail); 3322 if (status != DDI_SUCCESS) { 3323 TNF_PROBE_0(tavor_add_intrs_getnavail_fail, 3324 TAVOR_TNF_ERROR, ""); 3325 TAVOR_TNF_EXIT(tavor_add_intrs); 3326 return (DDI_FAILURE); 3327 } 3328 3329 /* Ensure that we have at least one (1) usable MSI or interrupt */ 3330 if ((state->ts_intrmsi_avail < 1) || (state->ts_intrmsi_count < 1)) { 3331 TNF_PROBE_0(tavor_add_intrs_notenoughts_intrmsi_fail, 3332 TAVOR_TNF_ERROR, ""); 3333 TAVOR_TNF_EXIT(tavor_add_intrs); 3334 return (DDI_FAILURE); 3335 } 3336 3337 /* Attempt to allocate a single interrupt/MSI handle */ 3338 status = ddi_intr_alloc(state->ts_dip, &state->ts_intrmsi_hdl, 3339 intr_type, 0, 1, &state->ts_intrmsi_allocd, 3340 DDI_INTR_ALLOC_STRICT); 3341 if (status != DDI_SUCCESS) { 3342 TNF_PROBE_0(tavor_add_intrs_intralloc_fail, 3343 TAVOR_TNF_ERROR, ""); 3344 TAVOR_TNF_EXIT(tavor_add_intrs); 3345 return (DDI_FAILURE); 3346 } 3347 3348 /* Ensure that we have allocated at least one (1) MSI or interrupt */ 3349 if (state->ts_intrmsi_allocd < 1) { 3350 TNF_PROBE_0(tavor_add_intrs_noallocts_intrmsi_fail, 3351 TAVOR_TNF_ERROR, ""); 3352 TAVOR_TNF_EXIT(tavor_add_intrs); 3353 return (DDI_FAILURE); 3354 } 3355 3356 /* 3357 * Extract the priority for the allocated interrupt/MSI. This 3358 * will be used later when initializing certain mutexes. 3359 */ 3360 status = ddi_intr_get_pri(state->ts_intrmsi_hdl, 3361 &state->ts_intrmsi_pri); 3362 if (status != DDI_SUCCESS) { 3363 /* Free the allocated interrupt/MSI handle */ 3364 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3365 3366 TNF_PROBE_0(tavor_add_intrs_getpri_fail, 3367 TAVOR_TNF_ERROR, ""); 3368 TAVOR_TNF_EXIT(tavor_add_intrs); 3369 return (DDI_FAILURE); 3370 } 3371 3372 /* Make sure the interrupt/MSI priority is below 'high level' */ 3373 if (state->ts_intrmsi_pri >= ddi_intr_get_hilevel_pri()) { 3374 /* Free the allocated interrupt/MSI handle */ 3375 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3376 3377 TNF_PROBE_0(tavor_add_intrs_hilevelpri_fail, 3378 TAVOR_TNF_ERROR, ""); 3379 TAVOR_TNF_EXIT(tavor_add_intrs); 3380 return (DDI_FAILURE); 3381 } 3382 3383 /* Get add'l capability information regarding interrupt/MSI */ 3384 status = ddi_intr_get_cap(state->ts_intrmsi_hdl, 3385 &state->ts_intrmsi_cap); 3386 if (status != DDI_SUCCESS) { 3387 /* Free the allocated interrupt/MSI handle */ 3388 (void) ddi_intr_free(state->ts_intrmsi_hdl); 3389 3390 TNF_PROBE_0(tavor_add_intrs_getcap_fail, 3391 TAVOR_TNF_ERROR, ""); 3392 TAVOR_TNF_EXIT(tavor_add_intrs); 3393 return (DDI_FAILURE); 3394 } 3395 3396 TAVOR_TNF_EXIT(tavor_add_intrs); 3397 return (DDI_SUCCESS); 3398 } 3399 3400 3401 /* 3402 * tavor_intr_or_msi_fini() 3403 * Context: Only called from attach() and/or detach() path contexts 3404 */ 3405 static int 3406 tavor_intr_or_msi_fini(tavor_state_t *state) 3407 { 3408 int status; 3409 3410 TAVOR_TNF_ENTER(tavor_intr_or_msi_fini); 3411 3412 /* Free the allocated interrupt/MSI handle */ 3413 status = ddi_intr_free(state->ts_intrmsi_hdl); 3414 if (status != DDI_SUCCESS) { 3415 TNF_PROBE_0(tavor_intr_or_msi_fini_freehdl_fail, 3416 TAVOR_TNF_ERROR, ""); 3417 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); 3418 return (DDI_FAILURE); 3419 } 3420 3421 TAVOR_TNF_EXIT(tavor_intr_or_msi_fini); 3422 return (DDI_SUCCESS); 3423 } 3424 3425 3426 /* Disable Tavor interrupts */ 3427 static int 3428 tavor_intr_disable(tavor_state_t *state) 3429 { 3430 ushort_t msi_ctrl = 0, caps_ctrl = 0; 3431 ddi_acc_handle_t pci_cfg_hdl = state->ts_pci_cfghdl; 3432 ASSERT(pci_cfg_hdl != NULL); 3433 ASSERT(state->ts_intr_types_avail & 3434 (DDI_INTR_TYPE_FIXED | DDI_INTR_TYPE_MSI)); 3435 3436 /* 3437 * Check if MSI interrupts are used. If so, disable MSI interupts. 3438 * If not, since Tavor doesn't support MSI-X interrupts, assuming the 3439 * legacy interrupt is used instead, disable the legacy interrupt. 3440 */ 3441 if ((state->ts_cfg_profile->cp_use_msi_if_avail != 0) && 3442 (state->ts_intr_types_avail & DDI_INTR_TYPE_MSI)) { 3443 3444 if ((PCI_CAP_LOCATE(pci_cfg_hdl, PCI_CAP_ID_MSI, 3445 &caps_ctrl) == DDI_SUCCESS)) { 3446 if ((msi_ctrl = PCI_CAP_GET16(pci_cfg_hdl, NULL, 3447 caps_ctrl, PCI_MSI_CTRL)) == PCI_CAP_EINVAL16) 3448 return (DDI_FAILURE); 3449 } 3450 ASSERT(msi_ctrl != 0); 3451 3452 if (!(msi_ctrl & PCI_MSI_ENABLE_BIT)) 3453 return (DDI_SUCCESS); 3454 3455 if (msi_ctrl & PCI_MSI_PVM_MASK) { 3456 int offset = (msi_ctrl & PCI_MSI_64BIT_MASK) ? 3457 PCI_MSI_64BIT_MASKBITS : PCI_MSI_32BIT_MASK; 3458 3459 /* Clear all inums in MSI */ 3460 PCI_CAP_PUT32(pci_cfg_hdl, NULL, caps_ctrl, 3461 offset, 0x0); 3462 } 3463 3464 /* Disable MSI interrupts */ 3465 msi_ctrl &= ~PCI_MSI_ENABLE_BIT; 3466 PCI_CAP_PUT16(pci_cfg_hdl, NULL, caps_ctrl, PCI_MSI_CTRL, 3467 msi_ctrl); 3468 3469 } else { 3470 uint16_t cmdreg = pci_config_get16(pci_cfg_hdl, PCI_CONF_COMM); 3471 ASSERT(state->ts_intr_types_avail & DDI_INTR_TYPE_FIXED); 3472 3473 /* Disable the legacy interrupts */ 3474 cmdreg |= PCI_COMM_INTX_DISABLE; 3475 pci_config_put16(pci_cfg_hdl, PCI_CONF_COMM, cmdreg); 3476 } 3477 3478 return (DDI_SUCCESS); 3479 } 3480 3481 /* Tavor quiesce(9F) entry */ 3482 static int 3483 tavor_quiesce(dev_info_t *dip) 3484 { 3485 tavor_state_t *state = ddi_get_soft_state(tavor_statep, 3486 DEVI(dip)->devi_instance); 3487 ASSERT(state != NULL); 3488 3489 /* start fastreboot */ 3490 state->ts_quiescing = B_TRUE; 3491 3492 /* If it's in maintenance mode, do nothing but return with SUCCESS */ 3493 if (!TAVOR_IS_OPERATIONAL(state->ts_operational_mode)) { 3494 return (DDI_SUCCESS); 3495 } 3496 3497 /* Shutdown HCA ports */ 3498 if (tavor_hca_ports_shutdown(state, 3499 state->ts_cfg_profile->cp_num_ports) != TAVOR_CMD_SUCCESS) { 3500 state->ts_quiescing = B_FALSE; 3501 return (DDI_FAILURE); 3502 } 3503 3504 /* Close HCA */ 3505 if (tavor_close_hca_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != 3506 TAVOR_CMD_SUCCESS) { 3507 state->ts_quiescing = B_FALSE; 3508 return (DDI_FAILURE); 3509 } 3510 3511 /* Shutdown FW */ 3512 if (tavor_sys_dis_cmd_post(state, TAVOR_CMD_NOSLEEP_SPIN) != 3513 TAVOR_CMD_SUCCESS) { 3514 state->ts_quiescing = B_FALSE; 3515 return (DDI_FAILURE); 3516 } 3517 3518 /* Disable interrupts */ 3519 if (tavor_intr_disable(state) != DDI_SUCCESS) { 3520 state->ts_quiescing = B_FALSE; 3521 return (DDI_FAILURE); 3522 } 3523 3524 /* SW-reset */ 3525 if (tavor_sw_reset(state) != DDI_SUCCESS) { 3526 state->ts_quiescing = B_FALSE; 3527 return (DDI_FAILURE); 3528 } 3529 3530 return (DDI_SUCCESS); 3531 } 3532