1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Copyright (c) 2017, Joyent, Inc. 29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved. 31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved. 32 * Copyright 2016 OmniTI Computer Consulting, Inc. All rights reserved. 33 */ 34 35 #include "ixgbe_sw.h" 36 37 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 38 39 /* 40 * Local function protoypes 41 */ 42 static int ixgbe_register_mac(ixgbe_t *); 43 static int ixgbe_identify_hardware(ixgbe_t *); 44 static int ixgbe_regs_map(ixgbe_t *); 45 static void ixgbe_init_properties(ixgbe_t *); 46 static int ixgbe_init_driver_settings(ixgbe_t *); 47 static void ixgbe_init_locks(ixgbe_t *); 48 static void ixgbe_destroy_locks(ixgbe_t *); 49 static int ixgbe_init(ixgbe_t *); 50 static int ixgbe_chip_start(ixgbe_t *); 51 static void ixgbe_chip_stop(ixgbe_t *); 52 static int ixgbe_reset(ixgbe_t *); 53 static void ixgbe_tx_clean(ixgbe_t *); 54 static boolean_t ixgbe_tx_drain(ixgbe_t *); 55 static boolean_t ixgbe_rx_drain(ixgbe_t *); 56 static int ixgbe_alloc_rings(ixgbe_t *); 57 static void ixgbe_free_rings(ixgbe_t *); 58 static int ixgbe_alloc_rx_data(ixgbe_t *); 59 static void ixgbe_free_rx_data(ixgbe_t *); 60 static void ixgbe_setup_rings(ixgbe_t *); 61 static void ixgbe_setup_rx(ixgbe_t *); 62 static void ixgbe_setup_tx(ixgbe_t *); 63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 65 static void ixgbe_setup_rss(ixgbe_t *); 66 static void ixgbe_setup_vmdq(ixgbe_t *); 67 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 68 static void ixgbe_setup_rss_table(ixgbe_t *); 69 static void ixgbe_init_unicst(ixgbe_t *); 70 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 71 static void ixgbe_setup_multicst(ixgbe_t *); 72 static void ixgbe_get_hw_state(ixgbe_t *); 73 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 74 static void ixgbe_get_conf(ixgbe_t *); 75 static void ixgbe_init_params(ixgbe_t *); 76 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 77 static void ixgbe_driver_link_check(ixgbe_t *); 78 static void ixgbe_sfp_check(void *); 79 static void ixgbe_overtemp_check(void *); 80 static void ixgbe_phy_check(void *); 81 static void ixgbe_link_timer(void *); 82 static void ixgbe_local_timer(void *); 83 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 84 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 85 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 86 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 87 static boolean_t is_valid_mac_addr(uint8_t *); 88 static boolean_t ixgbe_stall_check(ixgbe_t *); 89 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 90 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 91 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 92 static int ixgbe_alloc_intrs(ixgbe_t *); 93 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 94 static int ixgbe_add_intr_handlers(ixgbe_t *); 95 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 96 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 97 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 98 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 99 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 100 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 101 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 102 static void ixgbe_setup_adapter_vector(ixgbe_t *); 103 static void ixgbe_rem_intr_handlers(ixgbe_t *); 104 static void ixgbe_rem_intrs(ixgbe_t *); 105 static int ixgbe_enable_intrs(ixgbe_t *); 106 static int ixgbe_disable_intrs(ixgbe_t *); 107 static uint_t ixgbe_intr_legacy(void *, void *); 108 static uint_t ixgbe_intr_msi(void *, void *); 109 static uint_t ixgbe_intr_msix(void *, void *); 110 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 111 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 112 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 113 static void ixgbe_get_driver_control(struct ixgbe_hw *); 114 static int ixgbe_addmac(void *, const uint8_t *); 115 static int ixgbe_remmac(void *, const uint8_t *); 116 static void ixgbe_release_driver_control(struct ixgbe_hw *); 117 118 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 119 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 120 static int ixgbe_resume(dev_info_t *); 121 static int ixgbe_suspend(dev_info_t *); 122 static int ixgbe_quiesce(dev_info_t *); 123 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 124 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 125 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 126 static int ixgbe_intr_cb_register(ixgbe_t *); 127 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 128 129 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 130 const void *impl_data); 131 static void ixgbe_fm_init(ixgbe_t *); 132 static void ixgbe_fm_fini(ixgbe_t *); 133 134 char *ixgbe_priv_props[] = { 135 "_tx_copy_thresh", 136 "_tx_recycle_thresh", 137 "_tx_overload_thresh", 138 "_tx_resched_thresh", 139 "_rx_copy_thresh", 140 "_rx_limit_per_intr", 141 "_intr_throttling", 142 "_adv_pause_cap", 143 "_adv_asym_pause_cap", 144 NULL 145 }; 146 147 #define IXGBE_MAX_PRIV_PROPS \ 148 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 149 150 static struct cb_ops ixgbe_cb_ops = { 151 nulldev, /* cb_open */ 152 nulldev, /* cb_close */ 153 nodev, /* cb_strategy */ 154 nodev, /* cb_print */ 155 nodev, /* cb_dump */ 156 nodev, /* cb_read */ 157 nodev, /* cb_write */ 158 nodev, /* cb_ioctl */ 159 nodev, /* cb_devmap */ 160 nodev, /* cb_mmap */ 161 nodev, /* cb_segmap */ 162 nochpoll, /* cb_chpoll */ 163 ddi_prop_op, /* cb_prop_op */ 164 NULL, /* cb_stream */ 165 D_MP | D_HOTPLUG, /* cb_flag */ 166 CB_REV, /* cb_rev */ 167 nodev, /* cb_aread */ 168 nodev /* cb_awrite */ 169 }; 170 171 static struct dev_ops ixgbe_dev_ops = { 172 DEVO_REV, /* devo_rev */ 173 0, /* devo_refcnt */ 174 NULL, /* devo_getinfo */ 175 nulldev, /* devo_identify */ 176 nulldev, /* devo_probe */ 177 ixgbe_attach, /* devo_attach */ 178 ixgbe_detach, /* devo_detach */ 179 nodev, /* devo_reset */ 180 &ixgbe_cb_ops, /* devo_cb_ops */ 181 NULL, /* devo_bus_ops */ 182 ddi_power, /* devo_power */ 183 ixgbe_quiesce, /* devo_quiesce */ 184 }; 185 186 static struct modldrv ixgbe_modldrv = { 187 &mod_driverops, /* Type of module. This one is a driver */ 188 ixgbe_ident, /* Discription string */ 189 &ixgbe_dev_ops /* driver ops */ 190 }; 191 192 static struct modlinkage ixgbe_modlinkage = { 193 MODREV_1, &ixgbe_modldrv, NULL 194 }; 195 196 /* 197 * Access attributes for register mapping 198 */ 199 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 200 DDI_DEVICE_ATTR_V1, 201 DDI_STRUCTURE_LE_ACC, 202 DDI_STRICTORDER_ACC, 203 DDI_FLAGERR_ACC 204 }; 205 206 /* 207 * Loopback property 208 */ 209 static lb_property_t lb_normal = { 210 normal, "normal", IXGBE_LB_NONE 211 }; 212 213 static lb_property_t lb_mac = { 214 internal, "MAC", IXGBE_LB_INTERNAL_MAC 215 }; 216 217 static lb_property_t lb_external = { 218 external, "External", IXGBE_LB_EXTERNAL 219 }; 220 221 #define IXGBE_M_CALLBACK_FLAGS \ 222 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 223 224 static mac_callbacks_t ixgbe_m_callbacks = { 225 IXGBE_M_CALLBACK_FLAGS, 226 ixgbe_m_stat, 227 ixgbe_m_start, 228 ixgbe_m_stop, 229 ixgbe_m_promisc, 230 ixgbe_m_multicst, 231 NULL, 232 NULL, 233 NULL, 234 ixgbe_m_ioctl, 235 ixgbe_m_getcapab, 236 NULL, 237 NULL, 238 ixgbe_m_setprop, 239 ixgbe_m_getprop, 240 ixgbe_m_propinfo 241 }; 242 243 /* 244 * Initialize capabilities of each supported adapter type 245 */ 246 static adapter_info_t ixgbe_82598eb_cap = { 247 64, /* maximum number of rx queues */ 248 1, /* minimum number of rx queues */ 249 64, /* default number of rx queues */ 250 16, /* maximum number of rx groups */ 251 1, /* minimum number of rx groups */ 252 1, /* default number of rx groups */ 253 32, /* maximum number of tx queues */ 254 1, /* minimum number of tx queues */ 255 8, /* default number of tx queues */ 256 16366, /* maximum MTU size */ 257 0xFFFF, /* maximum interrupt throttle rate */ 258 0, /* minimum interrupt throttle rate */ 259 200, /* default interrupt throttle rate */ 260 18, /* maximum total msix vectors */ 261 16, /* maximum number of ring vectors */ 262 2, /* maximum number of other vectors */ 263 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 264 0, /* "other" interrupt types enable mask */ 265 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 266 | IXGBE_FLAG_RSS_CAPABLE 267 | IXGBE_FLAG_VMDQ_CAPABLE) 268 }; 269 270 static adapter_info_t ixgbe_82599eb_cap = { 271 128, /* maximum number of rx queues */ 272 1, /* minimum number of rx queues */ 273 128, /* default number of rx queues */ 274 64, /* maximum number of rx groups */ 275 1, /* minimum number of rx groups */ 276 1, /* default number of rx groups */ 277 128, /* maximum number of tx queues */ 278 1, /* minimum number of tx queues */ 279 8, /* default number of tx queues */ 280 15500, /* maximum MTU size */ 281 0xFF8, /* maximum interrupt throttle rate */ 282 0, /* minimum interrupt throttle rate */ 283 200, /* default interrupt throttle rate */ 284 64, /* maximum total msix vectors */ 285 16, /* maximum number of ring vectors */ 286 2, /* maximum number of other vectors */ 287 (IXGBE_EICR_LSC 288 | IXGBE_EICR_GPI_SDP1 289 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 290 291 (IXGBE_SDP1_GPIEN 292 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 293 294 (IXGBE_FLAG_DCA_CAPABLE 295 | IXGBE_FLAG_RSS_CAPABLE 296 | IXGBE_FLAG_VMDQ_CAPABLE 297 | IXGBE_FLAG_RSC_CAPABLE 298 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */ 299 }; 300 301 static adapter_info_t ixgbe_X540_cap = { 302 128, /* maximum number of rx queues */ 303 1, /* minimum number of rx queues */ 304 128, /* default number of rx queues */ 305 64, /* maximum number of rx groups */ 306 1, /* minimum number of rx groups */ 307 1, /* default number of rx groups */ 308 128, /* maximum number of tx queues */ 309 1, /* minimum number of tx queues */ 310 8, /* default number of tx queues */ 311 15500, /* maximum MTU size */ 312 0xFF8, /* maximum interrupt throttle rate */ 313 0, /* minimum interrupt throttle rate */ 314 200, /* default interrupt throttle rate */ 315 64, /* maximum total msix vectors */ 316 16, /* maximum number of ring vectors */ 317 2, /* maximum number of other vectors */ 318 (IXGBE_EICR_LSC 319 | IXGBE_EICR_GPI_SDP1_X540 320 | IXGBE_EICR_GPI_SDP2_X540), /* "other" interrupt types handled */ 321 322 (IXGBE_SDP1_GPIEN_X540 323 | IXGBE_SDP2_GPIEN_X540), /* "other" interrupt types enable mask */ 324 325 (IXGBE_FLAG_DCA_CAPABLE 326 | IXGBE_FLAG_RSS_CAPABLE 327 | IXGBE_FLAG_VMDQ_CAPABLE 328 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 329 }; 330 331 static adapter_info_t ixgbe_X550_cap = { 332 128, /* maximum number of rx queues */ 333 1, /* minimum number of rx queues */ 334 128, /* default number of rx queues */ 335 64, /* maximum number of rx groups */ 336 1, /* minimum number of rx groups */ 337 1, /* default number of rx groups */ 338 128, /* maximum number of tx queues */ 339 1, /* minimum number of tx queues */ 340 8, /* default number of tx queues */ 341 15500, /* maximum MTU size */ 342 0xFF8, /* maximum interrupt throttle rate */ 343 0, /* minimum interrupt throttle rate */ 344 0x200, /* default interrupt throttle rate */ 345 64, /* maximum total msix vectors */ 346 16, /* maximum number of ring vectors */ 347 2, /* maximum number of other vectors */ 348 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 349 0, /* "other" interrupt types enable mask */ 350 (IXGBE_FLAG_RSS_CAPABLE 351 | IXGBE_FLAG_VMDQ_CAPABLE 352 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 353 }; 354 355 /* 356 * Module Initialization Functions. 357 */ 358 359 int 360 _init(void) 361 { 362 int status; 363 364 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 365 366 status = mod_install(&ixgbe_modlinkage); 367 368 if (status != DDI_SUCCESS) { 369 mac_fini_ops(&ixgbe_dev_ops); 370 } 371 372 return (status); 373 } 374 375 int 376 _fini(void) 377 { 378 int status; 379 380 status = mod_remove(&ixgbe_modlinkage); 381 382 if (status == DDI_SUCCESS) { 383 mac_fini_ops(&ixgbe_dev_ops); 384 } 385 386 return (status); 387 } 388 389 int 390 _info(struct modinfo *modinfop) 391 { 392 int status; 393 394 status = mod_info(&ixgbe_modlinkage, modinfop); 395 396 return (status); 397 } 398 399 /* 400 * ixgbe_attach - Driver attach. 401 * 402 * This function is the device specific initialization entry 403 * point. This entry point is required and must be written. 404 * The DDI_ATTACH command must be provided in the attach entry 405 * point. When attach() is called with cmd set to DDI_ATTACH, 406 * all normal kernel services (such as kmem_alloc(9F)) are 407 * available for use by the driver. 408 * 409 * The attach() function will be called once for each instance 410 * of the device on the system with cmd set to DDI_ATTACH. 411 * Until attach() succeeds, the only driver entry points which 412 * may be called are open(9E) and getinfo(9E). 413 */ 414 static int 415 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 416 { 417 ixgbe_t *ixgbe; 418 struct ixgbe_osdep *osdep; 419 struct ixgbe_hw *hw; 420 int instance; 421 char taskqname[32]; 422 423 /* 424 * Check the command and perform corresponding operations 425 */ 426 switch (cmd) { 427 default: 428 return (DDI_FAILURE); 429 430 case DDI_RESUME: 431 return (ixgbe_resume(devinfo)); 432 433 case DDI_ATTACH: 434 break; 435 } 436 437 /* Get the device instance */ 438 instance = ddi_get_instance(devinfo); 439 440 /* Allocate memory for the instance data structure */ 441 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 442 443 ixgbe->dip = devinfo; 444 ixgbe->instance = instance; 445 446 hw = &ixgbe->hw; 447 osdep = &ixgbe->osdep; 448 hw->back = osdep; 449 osdep->ixgbe = ixgbe; 450 451 /* Attach the instance pointer to the dev_info data structure */ 452 ddi_set_driver_private(devinfo, ixgbe); 453 454 /* 455 * Initialize for FMA support 456 */ 457 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 458 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 459 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 460 ixgbe_fm_init(ixgbe); 461 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 462 463 /* 464 * Map PCI config space registers 465 */ 466 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 467 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 468 goto attach_fail; 469 } 470 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 471 472 /* 473 * Identify the chipset family 474 */ 475 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 476 ixgbe_error(ixgbe, "Failed to identify hardware"); 477 goto attach_fail; 478 } 479 480 /* 481 * Map device registers 482 */ 483 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 484 ixgbe_error(ixgbe, "Failed to map device registers"); 485 goto attach_fail; 486 } 487 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 488 489 /* 490 * Initialize driver parameters 491 */ 492 ixgbe_init_properties(ixgbe); 493 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 494 495 /* 496 * Register interrupt callback 497 */ 498 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 499 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 500 goto attach_fail; 501 } 502 503 /* 504 * Allocate interrupts 505 */ 506 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 507 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 508 goto attach_fail; 509 } 510 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 511 512 /* 513 * Allocate rx/tx rings based on the ring numbers. 514 * The actual numbers of rx/tx rings are decided by the number of 515 * allocated interrupt vectors, so we should allocate the rings after 516 * interrupts are allocated. 517 */ 518 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 519 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 520 goto attach_fail; 521 } 522 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 523 524 /* 525 * Map rings to interrupt vectors 526 */ 527 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 528 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 529 goto attach_fail; 530 } 531 532 /* 533 * Add interrupt handlers 534 */ 535 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 536 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 537 goto attach_fail; 538 } 539 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 540 541 /* 542 * Create a taskq for sfp-change 543 */ 544 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance); 545 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 546 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 547 ixgbe_error(ixgbe, "sfp_taskq create failed"); 548 goto attach_fail; 549 } 550 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 551 552 /* 553 * Create a taskq for over-temp 554 */ 555 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance); 556 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname, 557 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 558 ixgbe_error(ixgbe, "overtemp_taskq create failed"); 559 goto attach_fail; 560 } 561 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ; 562 563 /* 564 * Create a taskq for processing external PHY interrupts 565 */ 566 (void) sprintf(taskqname, "ixgbe%d_phy_taskq", instance); 567 if ((ixgbe->phy_taskq = ddi_taskq_create(devinfo, taskqname, 568 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 569 ixgbe_error(ixgbe, "phy_taskq create failed"); 570 goto attach_fail; 571 } 572 ixgbe->attach_progress |= ATTACH_PROGRESS_PHY_TASKQ; 573 574 /* 575 * Initialize driver parameters 576 */ 577 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 578 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 579 goto attach_fail; 580 } 581 582 /* 583 * Initialize mutexes for this device. 584 * Do this before enabling the interrupt handler and 585 * register the softint to avoid the condition where 586 * interrupt handler can try using uninitialized mutex. 587 */ 588 ixgbe_init_locks(ixgbe); 589 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 590 591 /* 592 * Initialize chipset hardware 593 */ 594 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 595 ixgbe_error(ixgbe, "Failed to initialize adapter"); 596 goto attach_fail; 597 } 598 ixgbe->link_check_complete = B_FALSE; 599 ixgbe->link_check_hrtime = gethrtime() + 600 (IXGBE_LINK_UP_TIME * 100000000ULL); 601 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 602 603 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 604 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 605 goto attach_fail; 606 } 607 608 /* 609 * Initialize adapter capabilities 610 */ 611 ixgbe_init_params(ixgbe); 612 613 /* 614 * Initialize statistics 615 */ 616 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 617 ixgbe_error(ixgbe, "Failed to initialize statistics"); 618 goto attach_fail; 619 } 620 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 621 622 /* 623 * Register the driver to the MAC 624 */ 625 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 626 ixgbe_error(ixgbe, "Failed to register MAC"); 627 goto attach_fail; 628 } 629 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 630 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 631 632 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 633 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 634 if (ixgbe->periodic_id == 0) { 635 ixgbe_error(ixgbe, "Failed to add the link check timer"); 636 goto attach_fail; 637 } 638 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 639 640 /* 641 * Now that mutex locks are initialized, and the chip is also 642 * initialized, enable interrupts. 643 */ 644 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 645 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 646 goto attach_fail; 647 } 648 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 649 650 ixgbe_log(ixgbe, "%s", ixgbe_ident); 651 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 652 653 return (DDI_SUCCESS); 654 655 attach_fail: 656 ixgbe_unconfigure(devinfo, ixgbe); 657 return (DDI_FAILURE); 658 } 659 660 /* 661 * ixgbe_detach - Driver detach. 662 * 663 * The detach() function is the complement of the attach routine. 664 * If cmd is set to DDI_DETACH, detach() is used to remove the 665 * state associated with a given instance of a device node 666 * prior to the removal of that instance from the system. 667 * 668 * The detach() function will be called once for each instance 669 * of the device for which there has been a successful attach() 670 * once there are no longer any opens on the device. 671 * 672 * Interrupts routine are disabled, All memory allocated by this 673 * driver are freed. 674 */ 675 static int 676 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 677 { 678 ixgbe_t *ixgbe; 679 680 /* 681 * Check detach command 682 */ 683 switch (cmd) { 684 default: 685 return (DDI_FAILURE); 686 687 case DDI_SUSPEND: 688 return (ixgbe_suspend(devinfo)); 689 690 case DDI_DETACH: 691 break; 692 } 693 694 /* 695 * Get the pointer to the driver private data structure 696 */ 697 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 698 if (ixgbe == NULL) 699 return (DDI_FAILURE); 700 701 /* 702 * If the device is still running, it needs to be stopped first. 703 * This check is necessary because under some specific circumstances, 704 * the detach routine can be called without stopping the interface 705 * first. 706 */ 707 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 708 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 709 mutex_enter(&ixgbe->gen_lock); 710 ixgbe_stop(ixgbe, B_TRUE); 711 mutex_exit(&ixgbe->gen_lock); 712 /* Disable and stop the watchdog timer */ 713 ixgbe_disable_watchdog_timer(ixgbe); 714 } 715 716 /* 717 * Check if there are still rx buffers held by the upper layer. 718 * If so, fail the detach. 719 */ 720 if (!ixgbe_rx_drain(ixgbe)) 721 return (DDI_FAILURE); 722 723 /* 724 * Do the remaining unconfigure routines 725 */ 726 ixgbe_unconfigure(devinfo, ixgbe); 727 728 return (DDI_SUCCESS); 729 } 730 731 /* 732 * quiesce(9E) entry point. 733 * 734 * This function is called when the system is single-threaded at high 735 * PIL with preemption disabled. Therefore, this function must not be 736 * blocked. 737 * 738 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 739 * DDI_FAILURE indicates an error condition and should almost never happen. 740 */ 741 static int 742 ixgbe_quiesce(dev_info_t *devinfo) 743 { 744 ixgbe_t *ixgbe; 745 struct ixgbe_hw *hw; 746 747 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 748 749 if (ixgbe == NULL) 750 return (DDI_FAILURE); 751 752 hw = &ixgbe->hw; 753 754 /* 755 * Disable the adapter interrupts 756 */ 757 ixgbe_disable_adapter_interrupts(ixgbe); 758 759 /* 760 * Tell firmware driver is no longer in control 761 */ 762 ixgbe_release_driver_control(hw); 763 764 /* 765 * Reset the chipset 766 */ 767 (void) ixgbe_reset_hw(hw); 768 769 /* 770 * Reset PHY 771 */ 772 (void) ixgbe_reset_phy(hw); 773 774 return (DDI_SUCCESS); 775 } 776 777 static void 778 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 779 { 780 /* 781 * Disable interrupt 782 */ 783 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 784 (void) ixgbe_disable_intrs(ixgbe); 785 } 786 787 /* 788 * remove the link check timer 789 */ 790 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 791 if (ixgbe->periodic_id != NULL) { 792 ddi_periodic_delete(ixgbe->periodic_id); 793 ixgbe->periodic_id = NULL; 794 } 795 } 796 797 /* 798 * Unregister MAC 799 */ 800 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 801 (void) mac_unregister(ixgbe->mac_hdl); 802 } 803 804 /* 805 * Free statistics 806 */ 807 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 808 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 809 } 810 811 /* 812 * Remove interrupt handlers 813 */ 814 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 815 ixgbe_rem_intr_handlers(ixgbe); 816 } 817 818 /* 819 * Remove taskq for sfp-status-change 820 */ 821 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 822 ddi_taskq_destroy(ixgbe->sfp_taskq); 823 } 824 825 /* 826 * Remove taskq for over-temp 827 */ 828 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) { 829 ddi_taskq_destroy(ixgbe->overtemp_taskq); 830 } 831 832 /* 833 * Remove taskq for external PHYs 834 */ 835 if (ixgbe->attach_progress & ATTACH_PROGRESS_PHY_TASKQ) { 836 ddi_taskq_destroy(ixgbe->phy_taskq); 837 } 838 839 /* 840 * Remove interrupts 841 */ 842 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 843 ixgbe_rem_intrs(ixgbe); 844 } 845 846 /* 847 * Unregister interrupt callback handler 848 */ 849 if (ixgbe->cb_hdl != NULL) { 850 (void) ddi_cb_unregister(ixgbe->cb_hdl); 851 } 852 853 /* 854 * Remove driver properties 855 */ 856 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 857 (void) ddi_prop_remove_all(devinfo); 858 } 859 860 /* 861 * Stop the chipset 862 */ 863 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 864 mutex_enter(&ixgbe->gen_lock); 865 ixgbe_chip_stop(ixgbe); 866 mutex_exit(&ixgbe->gen_lock); 867 } 868 869 /* 870 * Free register handle 871 */ 872 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 873 if (ixgbe->osdep.reg_handle != NULL) 874 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 875 } 876 877 /* 878 * Free PCI config handle 879 */ 880 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 881 if (ixgbe->osdep.cfg_handle != NULL) 882 pci_config_teardown(&ixgbe->osdep.cfg_handle); 883 } 884 885 /* 886 * Free locks 887 */ 888 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 889 ixgbe_destroy_locks(ixgbe); 890 } 891 892 /* 893 * Free the rx/tx rings 894 */ 895 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 896 ixgbe_free_rings(ixgbe); 897 } 898 899 /* 900 * Unregister FMA capabilities 901 */ 902 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 903 ixgbe_fm_fini(ixgbe); 904 } 905 906 /* 907 * Free the driver data structure 908 */ 909 kmem_free(ixgbe, sizeof (ixgbe_t)); 910 911 ddi_set_driver_private(devinfo, NULL); 912 } 913 914 /* 915 * ixgbe_register_mac - Register the driver and its function pointers with 916 * the GLD interface. 917 */ 918 static int 919 ixgbe_register_mac(ixgbe_t *ixgbe) 920 { 921 struct ixgbe_hw *hw = &ixgbe->hw; 922 mac_register_t *mac; 923 int status; 924 925 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 926 return (IXGBE_FAILURE); 927 928 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 929 mac->m_driver = ixgbe; 930 mac->m_dip = ixgbe->dip; 931 mac->m_src_addr = hw->mac.addr; 932 mac->m_callbacks = &ixgbe_m_callbacks; 933 mac->m_min_sdu = 0; 934 mac->m_max_sdu = ixgbe->default_mtu; 935 mac->m_margin = VLAN_TAGSZ; 936 mac->m_priv_props = ixgbe_priv_props; 937 mac->m_v12n = MAC_VIRT_LEVEL1; 938 939 status = mac_register(mac, &ixgbe->mac_hdl); 940 941 mac_free(mac); 942 943 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 944 } 945 946 /* 947 * ixgbe_identify_hardware - Identify the type of the chipset. 948 */ 949 static int 950 ixgbe_identify_hardware(ixgbe_t *ixgbe) 951 { 952 struct ixgbe_hw *hw = &ixgbe->hw; 953 struct ixgbe_osdep *osdep = &ixgbe->osdep; 954 955 /* 956 * Get the device id 957 */ 958 hw->vendor_id = 959 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 960 hw->device_id = 961 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 962 hw->revision_id = 963 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 964 hw->subsystem_device_id = 965 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 966 hw->subsystem_vendor_id = 967 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 968 969 /* 970 * Set the mac type of the adapter based on the device id 971 */ 972 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 973 return (IXGBE_FAILURE); 974 } 975 976 /* 977 * Install adapter capabilities 978 */ 979 switch (hw->mac.type) { 980 case ixgbe_mac_82598EB: 981 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n"); 982 ixgbe->capab = &ixgbe_82598eb_cap; 983 984 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 985 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 986 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 987 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN; 988 } 989 break; 990 991 case ixgbe_mac_82599EB: 992 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n"); 993 ixgbe->capab = &ixgbe_82599eb_cap; 994 995 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) { 996 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE; 997 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0; 998 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN; 999 } 1000 break; 1001 1002 case ixgbe_mac_X540: 1003 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n"); 1004 ixgbe->capab = &ixgbe_X540_cap; 1005 /* 1006 * For now, X540 is all set in its capab structure. 1007 * As other X540 variants show up, things can change here. 1008 */ 1009 break; 1010 1011 case ixgbe_mac_X550: 1012 case ixgbe_mac_X550EM_x: 1013 case ixgbe_mac_X550EM_a: 1014 IXGBE_DEBUGLOG_0(ixgbe, "identify X550 adapter\n"); 1015 ixgbe->capab = &ixgbe_X550_cap; 1016 1017 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 1018 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 1019 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 1020 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP || 1021 hw->device_id == IXGBE_DEV_ID_X550EM_A_QSFP_N) { 1022 ixgbe->capab->flags |= IXGBE_FLAG_SFP_PLUG_CAPABLE; 1023 } 1024 1025 /* 1026 * Link detection on X552 SFP+ and X552/X557-AT 1027 */ 1028 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_SFP || 1029 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP || 1030 hw->device_id == IXGBE_DEV_ID_X550EM_A_SFP_N || 1031 hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T) { 1032 ixgbe->capab->other_intr |= 1033 IXGBE_EIMS_GPI_SDP0_BY_MAC(hw); 1034 } 1035 if (hw->phy.type == ixgbe_phy_x550em_ext_t) { 1036 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN_X540; 1037 } 1038 break; 1039 1040 default: 1041 IXGBE_DEBUGLOG_1(ixgbe, 1042 "adapter not supported in ixgbe_identify_hardware(): %d\n", 1043 hw->mac.type); 1044 return (IXGBE_FAILURE); 1045 } 1046 1047 return (IXGBE_SUCCESS); 1048 } 1049 1050 /* 1051 * ixgbe_regs_map - Map the device registers. 1052 * 1053 */ 1054 static int 1055 ixgbe_regs_map(ixgbe_t *ixgbe) 1056 { 1057 dev_info_t *devinfo = ixgbe->dip; 1058 struct ixgbe_hw *hw = &ixgbe->hw; 1059 struct ixgbe_osdep *osdep = &ixgbe->osdep; 1060 off_t mem_size; 1061 1062 /* 1063 * First get the size of device registers to be mapped. 1064 */ 1065 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 1066 != DDI_SUCCESS) { 1067 return (IXGBE_FAILURE); 1068 } 1069 1070 /* 1071 * Call ddi_regs_map_setup() to map registers 1072 */ 1073 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 1074 (caddr_t *)&hw->hw_addr, 0, 1075 mem_size, &ixgbe_regs_acc_attr, 1076 &osdep->reg_handle)) != DDI_SUCCESS) { 1077 return (IXGBE_FAILURE); 1078 } 1079 1080 return (IXGBE_SUCCESS); 1081 } 1082 1083 /* 1084 * ixgbe_init_properties - Initialize driver properties. 1085 */ 1086 static void 1087 ixgbe_init_properties(ixgbe_t *ixgbe) 1088 { 1089 /* 1090 * Get conf file properties, including link settings 1091 * jumbo frames, ring number, descriptor number, etc. 1092 */ 1093 ixgbe_get_conf(ixgbe); 1094 } 1095 1096 /* 1097 * ixgbe_init_driver_settings - Initialize driver settings. 1098 * 1099 * The settings include hardware function pointers, bus information, 1100 * rx/tx rings settings, link state, and any other parameters that 1101 * need to be setup during driver initialization. 1102 */ 1103 static int 1104 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 1105 { 1106 struct ixgbe_hw *hw = &ixgbe->hw; 1107 dev_info_t *devinfo = ixgbe->dip; 1108 ixgbe_rx_ring_t *rx_ring; 1109 ixgbe_rx_group_t *rx_group; 1110 ixgbe_tx_ring_t *tx_ring; 1111 uint32_t rx_size; 1112 uint32_t tx_size; 1113 uint32_t ring_per_group; 1114 int i; 1115 1116 /* 1117 * Initialize chipset specific hardware function pointers 1118 */ 1119 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 1120 return (IXGBE_FAILURE); 1121 } 1122 1123 /* 1124 * Get the system page size 1125 */ 1126 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 1127 1128 /* 1129 * Set rx buffer size 1130 * 1131 * The IP header alignment room is counted in the calculation. 1132 * The rx buffer size is in unit of 1K that is required by the 1133 * chipset hardware. 1134 */ 1135 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 1136 ixgbe->rx_buf_size = ((rx_size >> 10) + 1137 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1138 1139 /* 1140 * Set tx buffer size 1141 */ 1142 tx_size = ixgbe->max_frame_size; 1143 ixgbe->tx_buf_size = ((tx_size >> 10) + 1144 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1145 1146 /* 1147 * Initialize rx/tx rings/groups parameters 1148 */ 1149 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 1150 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1151 rx_ring = &ixgbe->rx_rings[i]; 1152 rx_ring->index = i; 1153 rx_ring->ixgbe = ixgbe; 1154 rx_ring->group_index = i / ring_per_group; 1155 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 1156 } 1157 1158 for (i = 0; i < ixgbe->num_rx_groups; i++) { 1159 rx_group = &ixgbe->rx_groups[i]; 1160 rx_group->index = i; 1161 rx_group->ixgbe = ixgbe; 1162 } 1163 1164 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1165 tx_ring = &ixgbe->tx_rings[i]; 1166 tx_ring->index = i; 1167 tx_ring->ixgbe = ixgbe; 1168 if (ixgbe->tx_head_wb_enable) 1169 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 1170 else 1171 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 1172 1173 tx_ring->ring_size = ixgbe->tx_ring_size; 1174 tx_ring->free_list_size = ixgbe->tx_ring_size + 1175 (ixgbe->tx_ring_size >> 1); 1176 } 1177 1178 /* 1179 * Initialize values of interrupt throttling rate 1180 */ 1181 for (i = 1; i < MAX_INTR_VECTOR; i++) 1182 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 1183 1184 /* 1185 * The initial link state should be "unknown" 1186 */ 1187 ixgbe->link_state = LINK_STATE_UNKNOWN; 1188 1189 return (IXGBE_SUCCESS); 1190 } 1191 1192 /* 1193 * ixgbe_init_locks - Initialize locks. 1194 */ 1195 static void 1196 ixgbe_init_locks(ixgbe_t *ixgbe) 1197 { 1198 ixgbe_rx_ring_t *rx_ring; 1199 ixgbe_tx_ring_t *tx_ring; 1200 int i; 1201 1202 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1203 rx_ring = &ixgbe->rx_rings[i]; 1204 mutex_init(&rx_ring->rx_lock, NULL, 1205 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1206 } 1207 1208 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1209 tx_ring = &ixgbe->tx_rings[i]; 1210 mutex_init(&tx_ring->tx_lock, NULL, 1211 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1212 mutex_init(&tx_ring->recycle_lock, NULL, 1213 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1214 mutex_init(&tx_ring->tcb_head_lock, NULL, 1215 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1216 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1217 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1218 } 1219 1220 mutex_init(&ixgbe->gen_lock, NULL, 1221 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1222 1223 mutex_init(&ixgbe->watchdog_lock, NULL, 1224 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1225 } 1226 1227 /* 1228 * ixgbe_destroy_locks - Destroy locks. 1229 */ 1230 static void 1231 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1232 { 1233 ixgbe_rx_ring_t *rx_ring; 1234 ixgbe_tx_ring_t *tx_ring; 1235 int i; 1236 1237 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1238 rx_ring = &ixgbe->rx_rings[i]; 1239 mutex_destroy(&rx_ring->rx_lock); 1240 } 1241 1242 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1243 tx_ring = &ixgbe->tx_rings[i]; 1244 mutex_destroy(&tx_ring->tx_lock); 1245 mutex_destroy(&tx_ring->recycle_lock); 1246 mutex_destroy(&tx_ring->tcb_head_lock); 1247 mutex_destroy(&tx_ring->tcb_tail_lock); 1248 } 1249 1250 mutex_destroy(&ixgbe->gen_lock); 1251 mutex_destroy(&ixgbe->watchdog_lock); 1252 } 1253 1254 /* 1255 * We need to try and determine which LED index in hardware corresponds to the 1256 * link/activity LED. This is the one that'll be overwritten when we perform 1257 * GLDv3 LED activity. 1258 */ 1259 static void 1260 ixgbe_led_init(ixgbe_t *ixgbe) 1261 { 1262 uint32_t reg, i; 1263 struct ixgbe_hw *hw = &ixgbe->hw; 1264 1265 reg = IXGBE_READ_REG(hw, IXGBE_LEDCTL); 1266 for (i = 0; i < 4; i++) { 1267 if (((reg >> IXGBE_LED_MODE_SHIFT(i)) & 1268 IXGBE_LED_MODE_MASK_BASE) == IXGBE_LED_LINK_ACTIVE) { 1269 ixgbe->ixgbe_led_index = i; 1270 return; 1271 } 1272 } 1273 1274 /* 1275 * If we couldn't determine this, we use the default for various MACs 1276 * based on information Intel has inserted into other drivers over the 1277 * years. 1278 */ 1279 switch (hw->mac.type) { 1280 case ixgbe_mac_X550EM_a: 1281 ixgbe->ixgbe_led_index = 0; 1282 break; 1283 case ixgbe_mac_X550EM_x: 1284 ixgbe->ixgbe_led_index = 1; 1285 break; 1286 default: 1287 ixgbe->ixgbe_led_index = 2; 1288 break; 1289 } 1290 } 1291 1292 static int 1293 ixgbe_resume(dev_info_t *devinfo) 1294 { 1295 ixgbe_t *ixgbe; 1296 int i; 1297 1298 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1299 if (ixgbe == NULL) 1300 return (DDI_FAILURE); 1301 1302 mutex_enter(&ixgbe->gen_lock); 1303 1304 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1305 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1306 mutex_exit(&ixgbe->gen_lock); 1307 return (DDI_FAILURE); 1308 } 1309 1310 /* 1311 * Enable and start the watchdog timer 1312 */ 1313 ixgbe_enable_watchdog_timer(ixgbe); 1314 } 1315 1316 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1317 1318 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1319 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1320 mac_tx_ring_update(ixgbe->mac_hdl, 1321 ixgbe->tx_rings[i].ring_handle); 1322 } 1323 } 1324 1325 mutex_exit(&ixgbe->gen_lock); 1326 1327 return (DDI_SUCCESS); 1328 } 1329 1330 static int 1331 ixgbe_suspend(dev_info_t *devinfo) 1332 { 1333 ixgbe_t *ixgbe; 1334 1335 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1336 if (ixgbe == NULL) 1337 return (DDI_FAILURE); 1338 1339 mutex_enter(&ixgbe->gen_lock); 1340 1341 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1342 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1343 mutex_exit(&ixgbe->gen_lock); 1344 return (DDI_SUCCESS); 1345 } 1346 ixgbe_stop(ixgbe, B_FALSE); 1347 1348 mutex_exit(&ixgbe->gen_lock); 1349 1350 /* 1351 * Disable and stop the watchdog timer 1352 */ 1353 ixgbe_disable_watchdog_timer(ixgbe); 1354 1355 return (DDI_SUCCESS); 1356 } 1357 1358 /* 1359 * ixgbe_init - Initialize the device. 1360 */ 1361 static int 1362 ixgbe_init(ixgbe_t *ixgbe) 1363 { 1364 struct ixgbe_hw *hw = &ixgbe->hw; 1365 u8 pbanum[IXGBE_PBANUM_LENGTH]; 1366 int rv; 1367 1368 mutex_enter(&ixgbe->gen_lock); 1369 1370 /* 1371 * Configure/Initialize hardware 1372 */ 1373 rv = ixgbe_init_hw(hw); 1374 if (rv != IXGBE_SUCCESS) { 1375 switch (rv) { 1376 1377 /* 1378 * The first three errors are not prohibitive to us progressing 1379 * further, and are maily advisory in nature. In the case of a 1380 * SFP module not being present or not deemed supported by the 1381 * common code, we adivse the operator of this fact but carry on 1382 * instead of failing hard, as SFPs can be inserted or replaced 1383 * while the driver is running. In the case of a unknown error, 1384 * we fail-hard, logging the reason and emitting a FMA event. 1385 */ 1386 case IXGBE_ERR_EEPROM_VERSION: 1387 ixgbe_error(ixgbe, 1388 "This Intel 10Gb Ethernet device is pre-release and" 1389 " contains outdated firmware. Please contact your" 1390 " hardware vendor for a replacement."); 1391 break; 1392 case IXGBE_ERR_SFP_NOT_PRESENT: 1393 ixgbe_error(ixgbe, 1394 "No SFP+ module detected on this interface. Please " 1395 "install a supported SFP+ module for this " 1396 "interface to become operational."); 1397 break; 1398 case IXGBE_ERR_SFP_NOT_SUPPORTED: 1399 ixgbe_error(ixgbe, 1400 "Unsupported SFP+ module detected. Please replace " 1401 "it with a supported SFP+ module per Intel " 1402 "documentation, or bypass this check with " 1403 "allow_unsupported_sfp=1 in ixgbe.conf."); 1404 break; 1405 default: 1406 ixgbe_error(ixgbe, 1407 "Failed to initialize hardware. ixgbe_init_hw " 1408 "returned %d", rv); 1409 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1410 goto init_fail; 1411 } 1412 } 1413 1414 /* 1415 * Need to init eeprom before validating the checksum. 1416 */ 1417 if (ixgbe_init_eeprom_params(hw) < 0) { 1418 ixgbe_error(ixgbe, 1419 "Unable to intitialize the eeprom interface."); 1420 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1421 goto init_fail; 1422 } 1423 1424 /* 1425 * NVM validation 1426 */ 1427 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1428 /* 1429 * Some PCI-E parts fail the first check due to 1430 * the link being in sleep state. Call it again, 1431 * if it fails a second time it's a real issue. 1432 */ 1433 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1434 ixgbe_error(ixgbe, 1435 "Invalid NVM checksum. Please contact " 1436 "the vendor to update the NVM."); 1437 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1438 goto init_fail; 1439 } 1440 } 1441 1442 /* 1443 * Setup default flow control thresholds - enable/disable 1444 * & flow control type is controlled by ixgbe.conf 1445 */ 1446 hw->fc.high_water[0] = DEFAULT_FCRTH; 1447 hw->fc.low_water[0] = DEFAULT_FCRTL; 1448 hw->fc.pause_time = DEFAULT_FCPAUSE; 1449 hw->fc.send_xon = B_TRUE; 1450 1451 /* 1452 * Initialize flow control 1453 */ 1454 (void) ixgbe_start_hw(hw); 1455 1456 /* 1457 * Initialize link settings 1458 */ 1459 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1460 1461 /* 1462 * Initialize the chipset hardware 1463 */ 1464 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1465 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1466 goto init_fail; 1467 } 1468 1469 /* 1470 * Read identifying information and place in devinfo. 1471 */ 1472 pbanum[0] = '\0'; 1473 (void) ixgbe_read_pba_string(hw, pbanum, sizeof (pbanum)); 1474 if (*pbanum != '\0') { 1475 (void) ddi_prop_update_string(DDI_DEV_T_NONE, ixgbe->dip, 1476 "printed-board-assembly", (char *)pbanum); 1477 } 1478 1479 /* 1480 * Determine LED index. 1481 */ 1482 ixgbe_led_init(ixgbe); 1483 1484 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1485 goto init_fail; 1486 } 1487 1488 mutex_exit(&ixgbe->gen_lock); 1489 return (IXGBE_SUCCESS); 1490 1491 init_fail: 1492 /* 1493 * Reset PHY 1494 */ 1495 (void) ixgbe_reset_phy(hw); 1496 1497 mutex_exit(&ixgbe->gen_lock); 1498 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1499 return (IXGBE_FAILURE); 1500 } 1501 1502 /* 1503 * ixgbe_chip_start - Initialize and start the chipset hardware. 1504 */ 1505 static int 1506 ixgbe_chip_start(ixgbe_t *ixgbe) 1507 { 1508 struct ixgbe_hw *hw = &ixgbe->hw; 1509 int i; 1510 1511 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1512 1513 /* 1514 * Get the mac address 1515 * This function should handle SPARC case correctly. 1516 */ 1517 if (!ixgbe_find_mac_address(ixgbe)) { 1518 ixgbe_error(ixgbe, "Failed to get the mac address"); 1519 return (IXGBE_FAILURE); 1520 } 1521 1522 /* 1523 * Validate the mac address 1524 */ 1525 (void) ixgbe_init_rx_addrs(hw); 1526 if (!is_valid_mac_addr(hw->mac.addr)) { 1527 ixgbe_error(ixgbe, "Invalid mac address"); 1528 return (IXGBE_FAILURE); 1529 } 1530 1531 /* 1532 * Re-enable relaxed ordering for performance. It is disabled 1533 * by default in the hardware init. 1534 */ 1535 if (ixgbe->relax_order_enable == B_TRUE) 1536 ixgbe_enable_relaxed_ordering(hw); 1537 1538 /* 1539 * Setup adapter interrupt vectors 1540 */ 1541 ixgbe_setup_adapter_vector(ixgbe); 1542 1543 /* 1544 * Initialize unicast addresses. 1545 */ 1546 ixgbe_init_unicst(ixgbe); 1547 1548 /* 1549 * Setup and initialize the mctable structures. 1550 */ 1551 ixgbe_setup_multicst(ixgbe); 1552 1553 /* 1554 * Set interrupt throttling rate 1555 */ 1556 for (i = 0; i < ixgbe->intr_cnt; i++) { 1557 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1558 } 1559 1560 /* 1561 * Disable Wake-on-LAN 1562 */ 1563 IXGBE_WRITE_REG(hw, IXGBE_WUC, 0); 1564 1565 /* 1566 * Some adapters offer Energy Efficient Ethernet (EEE) support. 1567 * Due to issues with EEE in e1000g/igb, we disable this by default 1568 * as a precautionary measure. 1569 * 1570 * Currently, this is present on a number of the X550 family parts. 1571 */ 1572 (void) ixgbe_setup_eee(hw, B_FALSE); 1573 1574 /* 1575 * Turn on any present SFP Tx laser 1576 */ 1577 ixgbe_enable_tx_laser(hw); 1578 1579 /* 1580 * Power on the PHY 1581 */ 1582 (void) ixgbe_set_phy_power(hw, B_TRUE); 1583 1584 /* 1585 * Save the state of the PHY 1586 */ 1587 ixgbe_get_hw_state(ixgbe); 1588 1589 /* 1590 * Make sure driver has control 1591 */ 1592 ixgbe_get_driver_control(hw); 1593 1594 return (IXGBE_SUCCESS); 1595 } 1596 1597 /* 1598 * ixgbe_chip_stop - Stop the chipset hardware 1599 */ 1600 static void 1601 ixgbe_chip_stop(ixgbe_t *ixgbe) 1602 { 1603 struct ixgbe_hw *hw = &ixgbe->hw; 1604 int rv; 1605 1606 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1607 1608 /* 1609 * Stop interupt generation and disable Tx unit 1610 */ 1611 hw->adapter_stopped = B_FALSE; 1612 (void) ixgbe_stop_adapter(hw); 1613 1614 /* 1615 * Reset the chipset 1616 */ 1617 (void) ixgbe_reset_hw(hw); 1618 1619 /* 1620 * Reset PHY 1621 */ 1622 (void) ixgbe_reset_phy(hw); 1623 1624 /* 1625 * Enter LPLU (Low Power, Link Up) mode, if available. Avoid resetting 1626 * the PHY while doing so. Else, just power down the PHY. 1627 */ 1628 if (hw->phy.ops.enter_lplu != NULL) { 1629 hw->phy.reset_disable = B_TRUE; 1630 rv = hw->phy.ops.enter_lplu(hw); 1631 if (rv != IXGBE_SUCCESS) 1632 ixgbe_error(ixgbe, "Error while entering LPLU: %d", rv); 1633 hw->phy.reset_disable = B_FALSE; 1634 } else { 1635 (void) ixgbe_set_phy_power(hw, B_FALSE); 1636 } 1637 1638 /* 1639 * Turn off any present SFP Tx laser 1640 * Expected for health and safety reasons 1641 */ 1642 ixgbe_disable_tx_laser(hw); 1643 1644 /* 1645 * Tell firmware driver is no longer in control 1646 */ 1647 ixgbe_release_driver_control(hw); 1648 1649 } 1650 1651 /* 1652 * ixgbe_reset - Reset the chipset and re-start the driver. 1653 * 1654 * It involves stopping and re-starting the chipset, 1655 * and re-configuring the rx/tx rings. 1656 */ 1657 static int 1658 ixgbe_reset(ixgbe_t *ixgbe) 1659 { 1660 int i; 1661 1662 /* 1663 * Disable and stop the watchdog timer 1664 */ 1665 ixgbe_disable_watchdog_timer(ixgbe); 1666 1667 mutex_enter(&ixgbe->gen_lock); 1668 1669 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1670 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1671 1672 ixgbe_stop(ixgbe, B_FALSE); 1673 1674 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1675 mutex_exit(&ixgbe->gen_lock); 1676 return (IXGBE_FAILURE); 1677 } 1678 1679 /* 1680 * After resetting, need to recheck the link status. 1681 */ 1682 ixgbe->link_check_complete = B_FALSE; 1683 ixgbe->link_check_hrtime = gethrtime() + 1684 (IXGBE_LINK_UP_TIME * 100000000ULL); 1685 1686 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1687 1688 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1689 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1690 mac_tx_ring_update(ixgbe->mac_hdl, 1691 ixgbe->tx_rings[i].ring_handle); 1692 } 1693 } 1694 1695 mutex_exit(&ixgbe->gen_lock); 1696 1697 /* 1698 * Enable and start the watchdog timer 1699 */ 1700 ixgbe_enable_watchdog_timer(ixgbe); 1701 1702 return (IXGBE_SUCCESS); 1703 } 1704 1705 /* 1706 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1707 */ 1708 static void 1709 ixgbe_tx_clean(ixgbe_t *ixgbe) 1710 { 1711 ixgbe_tx_ring_t *tx_ring; 1712 tx_control_block_t *tcb; 1713 link_list_t pending_list; 1714 uint32_t desc_num; 1715 int i, j; 1716 1717 LINK_LIST_INIT(&pending_list); 1718 1719 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1720 tx_ring = &ixgbe->tx_rings[i]; 1721 1722 mutex_enter(&tx_ring->recycle_lock); 1723 1724 /* 1725 * Clean the pending tx data - the pending packets in the 1726 * work_list that have no chances to be transmitted again. 1727 * 1728 * We must ensure the chipset is stopped or the link is down 1729 * before cleaning the transmit packets. 1730 */ 1731 desc_num = 0; 1732 for (j = 0; j < tx_ring->ring_size; j++) { 1733 tcb = tx_ring->work_list[j]; 1734 if (tcb != NULL) { 1735 desc_num += tcb->desc_num; 1736 1737 tx_ring->work_list[j] = NULL; 1738 1739 ixgbe_free_tcb(tcb); 1740 1741 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1742 } 1743 } 1744 1745 if (desc_num > 0) { 1746 atomic_add_32(&tx_ring->tbd_free, desc_num); 1747 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1748 1749 /* 1750 * Reset the head and tail pointers of the tbd ring; 1751 * Reset the writeback head if it's enable. 1752 */ 1753 tx_ring->tbd_head = 0; 1754 tx_ring->tbd_tail = 0; 1755 if (ixgbe->tx_head_wb_enable) 1756 *tx_ring->tbd_head_wb = 0; 1757 1758 IXGBE_WRITE_REG(&ixgbe->hw, 1759 IXGBE_TDH(tx_ring->index), 0); 1760 IXGBE_WRITE_REG(&ixgbe->hw, 1761 IXGBE_TDT(tx_ring->index), 0); 1762 } 1763 1764 mutex_exit(&tx_ring->recycle_lock); 1765 1766 /* 1767 * Add the tx control blocks in the pending list to 1768 * the free list. 1769 */ 1770 ixgbe_put_free_list(tx_ring, &pending_list); 1771 } 1772 } 1773 1774 /* 1775 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1776 * transmitted. 1777 */ 1778 static boolean_t 1779 ixgbe_tx_drain(ixgbe_t *ixgbe) 1780 { 1781 ixgbe_tx_ring_t *tx_ring; 1782 boolean_t done; 1783 int i, j; 1784 1785 /* 1786 * Wait for a specific time to allow pending tx packets 1787 * to be transmitted. 1788 * 1789 * Check the counter tbd_free to see if transmission is done. 1790 * No lock protection is needed here. 1791 * 1792 * Return B_TRUE if all pending packets have been transmitted; 1793 * Otherwise return B_FALSE; 1794 */ 1795 for (i = 0; i < TX_DRAIN_TIME; i++) { 1796 1797 done = B_TRUE; 1798 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1799 tx_ring = &ixgbe->tx_rings[j]; 1800 done = done && 1801 (tx_ring->tbd_free == tx_ring->ring_size); 1802 } 1803 1804 if (done) 1805 break; 1806 1807 msec_delay(1); 1808 } 1809 1810 return (done); 1811 } 1812 1813 /* 1814 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1815 */ 1816 static boolean_t 1817 ixgbe_rx_drain(ixgbe_t *ixgbe) 1818 { 1819 boolean_t done = B_TRUE; 1820 int i; 1821 1822 /* 1823 * Polling the rx free list to check if those rx buffers held by 1824 * the upper layer are released. 1825 * 1826 * Check the counter rcb_free to see if all pending buffers are 1827 * released. No lock protection is needed here. 1828 * 1829 * Return B_TRUE if all pending buffers have been released; 1830 * Otherwise return B_FALSE; 1831 */ 1832 for (i = 0; i < RX_DRAIN_TIME; i++) { 1833 done = (ixgbe->rcb_pending == 0); 1834 1835 if (done) 1836 break; 1837 1838 msec_delay(1); 1839 } 1840 1841 return (done); 1842 } 1843 1844 /* 1845 * ixgbe_start - Start the driver/chipset. 1846 */ 1847 int 1848 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1849 { 1850 struct ixgbe_hw *hw = &ixgbe->hw; 1851 int i; 1852 1853 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1854 1855 if (alloc_buffer) { 1856 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1857 ixgbe_error(ixgbe, 1858 "Failed to allocate software receive rings"); 1859 return (IXGBE_FAILURE); 1860 } 1861 1862 /* Allocate buffers for all the rx/tx rings */ 1863 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1864 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1865 return (IXGBE_FAILURE); 1866 } 1867 1868 ixgbe->tx_ring_init = B_TRUE; 1869 } else { 1870 ixgbe->tx_ring_init = B_FALSE; 1871 } 1872 1873 for (i = 0; i < ixgbe->num_rx_rings; i++) 1874 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1875 for (i = 0; i < ixgbe->num_tx_rings; i++) 1876 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1877 1878 /* 1879 * Start the chipset hardware 1880 */ 1881 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1882 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1883 goto start_failure; 1884 } 1885 1886 /* 1887 * Configure link now for X550 1888 * 1889 * X550 possesses a LPLU (Low-Power Link Up) mode which keeps the 1890 * resting state of the adapter at a 1Gb FDX speed. Prior to the X550, 1891 * the resting state of the link would be the maximum speed that 1892 * autonegotiation will allow (usually 10Gb, infrastructure allowing) 1893 * so we never bothered with explicitly setting the link to 10Gb as it 1894 * would already be at that state on driver attach. With X550, we must 1895 * trigger a re-negotiation of the link in order to switch from a LPLU 1896 * 1Gb link to 10Gb (cable and link partner permitting.) 1897 */ 1898 if (hw->mac.type == ixgbe_mac_X550 || 1899 hw->mac.type == ixgbe_mac_X550EM_a || 1900 hw->mac.type == ixgbe_mac_X550EM_x) { 1901 (void) ixgbe_driver_setup_link(ixgbe, B_TRUE); 1902 ixgbe_get_hw_state(ixgbe); 1903 } 1904 1905 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1906 goto start_failure; 1907 } 1908 1909 /* 1910 * Setup the rx/tx rings 1911 */ 1912 ixgbe_setup_rings(ixgbe); 1913 1914 /* 1915 * ixgbe_start() will be called when resetting, however if reset 1916 * happens, we need to clear the ERROR, STALL and OVERTEMP flags 1917 * before enabling the interrupts. 1918 */ 1919 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR 1920 | IXGBE_STALL| IXGBE_OVERTEMP)); 1921 1922 /* 1923 * Enable adapter interrupts 1924 * The interrupts must be enabled after the driver state is START 1925 */ 1926 ixgbe_enable_adapter_interrupts(ixgbe); 1927 1928 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1929 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1930 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1931 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1932 1933 return (IXGBE_SUCCESS); 1934 1935 start_failure: 1936 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1937 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1938 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1939 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1940 1941 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1942 1943 return (IXGBE_FAILURE); 1944 } 1945 1946 /* 1947 * ixgbe_stop - Stop the driver/chipset. 1948 */ 1949 void 1950 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1951 { 1952 int i; 1953 1954 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1955 1956 /* 1957 * Disable the adapter interrupts 1958 */ 1959 ixgbe_disable_adapter_interrupts(ixgbe); 1960 1961 /* 1962 * Drain the pending tx packets 1963 */ 1964 (void) ixgbe_tx_drain(ixgbe); 1965 1966 for (i = 0; i < ixgbe->num_rx_rings; i++) 1967 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1968 for (i = 0; i < ixgbe->num_tx_rings; i++) 1969 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1970 1971 /* 1972 * Stop the chipset hardware 1973 */ 1974 ixgbe_chip_stop(ixgbe); 1975 1976 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1977 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1978 } 1979 1980 /* 1981 * Clean the pending tx data/resources 1982 */ 1983 ixgbe_tx_clean(ixgbe); 1984 1985 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1986 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1987 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1988 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1989 1990 if (ixgbe->link_state == LINK_STATE_UP) { 1991 ixgbe->link_state = LINK_STATE_UNKNOWN; 1992 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1993 } 1994 1995 if (free_buffer) { 1996 /* 1997 * Release the DMA/memory resources of rx/tx rings 1998 */ 1999 ixgbe_free_dma(ixgbe); 2000 ixgbe_free_rx_data(ixgbe); 2001 } 2002 } 2003 2004 /* 2005 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 2006 */ 2007 /* ARGSUSED */ 2008 static int 2009 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 2010 void *arg1, void *arg2) 2011 { 2012 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 2013 2014 switch (cbaction) { 2015 /* IRM callback */ 2016 int count; 2017 case DDI_CB_INTR_ADD: 2018 case DDI_CB_INTR_REMOVE: 2019 count = (int)(uintptr_t)cbarg; 2020 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 2021 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 2022 int, ixgbe->intr_cnt); 2023 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 2024 DDI_SUCCESS) { 2025 ixgbe_error(ixgbe, 2026 "IRM CB: Failed to adjust interrupts"); 2027 goto cb_fail; 2028 } 2029 break; 2030 default: 2031 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 2032 cbaction); 2033 return (DDI_ENOTSUP); 2034 } 2035 return (DDI_SUCCESS); 2036 cb_fail: 2037 return (DDI_FAILURE); 2038 } 2039 2040 /* 2041 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 2042 */ 2043 static int 2044 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 2045 { 2046 int i, rc, actual; 2047 2048 if (count == 0) 2049 return (DDI_SUCCESS); 2050 2051 if ((cbaction == DDI_CB_INTR_ADD && 2052 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 2053 (cbaction == DDI_CB_INTR_REMOVE && 2054 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 2055 return (DDI_FAILURE); 2056 2057 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 2058 return (DDI_FAILURE); 2059 } 2060 2061 for (i = 0; i < ixgbe->num_rx_rings; i++) 2062 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 2063 for (i = 0; i < ixgbe->num_tx_rings; i++) 2064 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 2065 2066 mutex_enter(&ixgbe->gen_lock); 2067 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 2068 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 2069 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 2070 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 2071 2072 ixgbe_stop(ixgbe, B_FALSE); 2073 /* 2074 * Disable interrupts 2075 */ 2076 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 2077 rc = ixgbe_disable_intrs(ixgbe); 2078 ASSERT(rc == IXGBE_SUCCESS); 2079 } 2080 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 2081 2082 /* 2083 * Remove interrupt handlers 2084 */ 2085 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 2086 ixgbe_rem_intr_handlers(ixgbe); 2087 } 2088 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 2089 2090 /* 2091 * Clear vect_map 2092 */ 2093 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 2094 switch (cbaction) { 2095 case DDI_CB_INTR_ADD: 2096 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 2097 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 2098 DDI_INTR_ALLOC_NORMAL); 2099 if (rc != DDI_SUCCESS || actual != count) { 2100 ixgbe_log(ixgbe, "Adjust interrupts failed." 2101 "return: %d, irm cb size: %d, actual: %d", 2102 rc, count, actual); 2103 goto intr_adjust_fail; 2104 } 2105 ixgbe->intr_cnt += count; 2106 break; 2107 2108 case DDI_CB_INTR_REMOVE: 2109 for (i = ixgbe->intr_cnt - count; 2110 i < ixgbe->intr_cnt; i ++) { 2111 rc = ddi_intr_free(ixgbe->htable[i]); 2112 ixgbe->htable[i] = NULL; 2113 if (rc != DDI_SUCCESS) { 2114 ixgbe_log(ixgbe, "Adjust interrupts failed." 2115 "return: %d, irm cb size: %d, actual: %d", 2116 rc, count, actual); 2117 goto intr_adjust_fail; 2118 } 2119 } 2120 ixgbe->intr_cnt -= count; 2121 break; 2122 } 2123 2124 /* 2125 * Get priority for first vector, assume remaining are all the same 2126 */ 2127 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 2128 if (rc != DDI_SUCCESS) { 2129 ixgbe_log(ixgbe, 2130 "Get interrupt priority failed: %d", rc); 2131 goto intr_adjust_fail; 2132 } 2133 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 2134 if (rc != DDI_SUCCESS) { 2135 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 2136 goto intr_adjust_fail; 2137 } 2138 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 2139 2140 /* 2141 * Map rings to interrupt vectors 2142 */ 2143 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 2144 ixgbe_error(ixgbe, 2145 "IRM CB: Failed to map interrupts to vectors"); 2146 goto intr_adjust_fail; 2147 } 2148 2149 /* 2150 * Add interrupt handlers 2151 */ 2152 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 2153 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 2154 goto intr_adjust_fail; 2155 } 2156 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 2157 2158 /* 2159 * Now that mutex locks are initialized, and the chip is also 2160 * initialized, enable interrupts. 2161 */ 2162 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 2163 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 2164 goto intr_adjust_fail; 2165 } 2166 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 2167 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 2168 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 2169 goto intr_adjust_fail; 2170 } 2171 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 2172 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 2173 ixgbe->ixgbe_state |= IXGBE_STARTED; 2174 mutex_exit(&ixgbe->gen_lock); 2175 2176 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2177 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 2178 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 2179 } 2180 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2181 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 2182 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 2183 } 2184 2185 /* Wakeup all Tx rings */ 2186 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2187 mac_tx_ring_update(ixgbe->mac_hdl, 2188 ixgbe->tx_rings[i].ring_handle); 2189 } 2190 2191 IXGBE_DEBUGLOG_3(ixgbe, 2192 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 2193 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 2194 return (DDI_SUCCESS); 2195 2196 intr_adjust_fail: 2197 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 2198 mutex_exit(&ixgbe->gen_lock); 2199 return (DDI_FAILURE); 2200 } 2201 2202 /* 2203 * ixgbe_intr_cb_register - Register interrupt callback function. 2204 */ 2205 static int 2206 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 2207 { 2208 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 2209 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 2210 return (IXGBE_FAILURE); 2211 } 2212 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 2213 return (IXGBE_SUCCESS); 2214 } 2215 2216 /* 2217 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 2218 */ 2219 static int 2220 ixgbe_alloc_rings(ixgbe_t *ixgbe) 2221 { 2222 /* 2223 * Allocate memory space for rx rings 2224 */ 2225 ixgbe->rx_rings = kmem_zalloc( 2226 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 2227 KM_NOSLEEP); 2228 2229 if (ixgbe->rx_rings == NULL) { 2230 return (IXGBE_FAILURE); 2231 } 2232 2233 /* 2234 * Allocate memory space for tx rings 2235 */ 2236 ixgbe->tx_rings = kmem_zalloc( 2237 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 2238 KM_NOSLEEP); 2239 2240 if (ixgbe->tx_rings == NULL) { 2241 kmem_free(ixgbe->rx_rings, 2242 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2243 ixgbe->rx_rings = NULL; 2244 return (IXGBE_FAILURE); 2245 } 2246 2247 /* 2248 * Allocate memory space for rx ring groups 2249 */ 2250 ixgbe->rx_groups = kmem_zalloc( 2251 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 2252 KM_NOSLEEP); 2253 2254 if (ixgbe->rx_groups == NULL) { 2255 kmem_free(ixgbe->rx_rings, 2256 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2257 kmem_free(ixgbe->tx_rings, 2258 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2259 ixgbe->rx_rings = NULL; 2260 ixgbe->tx_rings = NULL; 2261 return (IXGBE_FAILURE); 2262 } 2263 2264 return (IXGBE_SUCCESS); 2265 } 2266 2267 /* 2268 * ixgbe_free_rings - Free the memory space of rx/tx rings. 2269 */ 2270 static void 2271 ixgbe_free_rings(ixgbe_t *ixgbe) 2272 { 2273 if (ixgbe->rx_rings != NULL) { 2274 kmem_free(ixgbe->rx_rings, 2275 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2276 ixgbe->rx_rings = NULL; 2277 } 2278 2279 if (ixgbe->tx_rings != NULL) { 2280 kmem_free(ixgbe->tx_rings, 2281 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2282 ixgbe->tx_rings = NULL; 2283 } 2284 2285 if (ixgbe->rx_groups != NULL) { 2286 kmem_free(ixgbe->rx_groups, 2287 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 2288 ixgbe->rx_groups = NULL; 2289 } 2290 } 2291 2292 static int 2293 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 2294 { 2295 ixgbe_rx_ring_t *rx_ring; 2296 int i; 2297 2298 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2299 rx_ring = &ixgbe->rx_rings[i]; 2300 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 2301 goto alloc_rx_rings_failure; 2302 } 2303 return (IXGBE_SUCCESS); 2304 2305 alloc_rx_rings_failure: 2306 ixgbe_free_rx_data(ixgbe); 2307 return (IXGBE_FAILURE); 2308 } 2309 2310 static void 2311 ixgbe_free_rx_data(ixgbe_t *ixgbe) 2312 { 2313 ixgbe_rx_ring_t *rx_ring; 2314 ixgbe_rx_data_t *rx_data; 2315 int i; 2316 2317 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2318 rx_ring = &ixgbe->rx_rings[i]; 2319 2320 mutex_enter(&ixgbe->rx_pending_lock); 2321 rx_data = rx_ring->rx_data; 2322 2323 if (rx_data != NULL) { 2324 rx_data->flag |= IXGBE_RX_STOPPED; 2325 2326 if (rx_data->rcb_pending == 0) { 2327 ixgbe_free_rx_ring_data(rx_data); 2328 rx_ring->rx_data = NULL; 2329 } 2330 } 2331 2332 mutex_exit(&ixgbe->rx_pending_lock); 2333 } 2334 } 2335 2336 /* 2337 * ixgbe_setup_rings - Setup rx/tx rings. 2338 */ 2339 static void 2340 ixgbe_setup_rings(ixgbe_t *ixgbe) 2341 { 2342 /* 2343 * Setup the rx/tx rings, including the following: 2344 * 2345 * 1. Setup the descriptor ring and the control block buffers; 2346 * 2. Initialize necessary registers for receive/transmit; 2347 * 3. Initialize software pointers/parameters for receive/transmit; 2348 */ 2349 ixgbe_setup_rx(ixgbe); 2350 2351 ixgbe_setup_tx(ixgbe); 2352 } 2353 2354 static void 2355 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 2356 { 2357 ixgbe_t *ixgbe = rx_ring->ixgbe; 2358 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2359 struct ixgbe_hw *hw = &ixgbe->hw; 2360 rx_control_block_t *rcb; 2361 union ixgbe_adv_rx_desc *rbd; 2362 uint32_t size; 2363 uint32_t buf_low; 2364 uint32_t buf_high; 2365 uint32_t reg_val; 2366 int i; 2367 2368 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2369 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2370 2371 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2372 rcb = rx_data->work_list[i]; 2373 rbd = &rx_data->rbd_ring[i]; 2374 2375 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2376 rbd->read.hdr_addr = 0; 2377 } 2378 2379 /* 2380 * Initialize the length register 2381 */ 2382 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2383 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2384 2385 /* 2386 * Initialize the base address registers 2387 */ 2388 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2389 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2390 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2391 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2392 2393 /* 2394 * Setup head & tail pointers 2395 */ 2396 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2397 rx_data->ring_size - 1); 2398 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2399 2400 rx_data->rbd_next = 0; 2401 rx_data->lro_first = 0; 2402 2403 /* 2404 * Setup the Receive Descriptor Control Register (RXDCTL) 2405 * PTHRESH=32 descriptors (half the internal cache) 2406 * HTHRESH=0 descriptors (to minimize latency on fetch) 2407 * WTHRESH defaults to 1 (writeback each descriptor) 2408 */ 2409 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2410 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2411 2412 /* Not a valid value for 82599, X540 or X550 */ 2413 if (hw->mac.type == ixgbe_mac_82598EB) { 2414 reg_val |= 0x0020; /* pthresh */ 2415 } 2416 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2417 2418 if (hw->mac.type == ixgbe_mac_82599EB || 2419 hw->mac.type == ixgbe_mac_X540 || 2420 hw->mac.type == ixgbe_mac_X550 || 2421 hw->mac.type == ixgbe_mac_X550EM_x || 2422 hw->mac.type == ixgbe_mac_X550EM_a) { 2423 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2424 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2425 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2426 } 2427 2428 /* 2429 * Setup the Split and Replication Receive Control Register. 2430 * Set the rx buffer size and the advanced descriptor type. 2431 */ 2432 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2433 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2434 reg_val |= IXGBE_SRRCTL_DROP_EN; 2435 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2436 } 2437 2438 static void 2439 ixgbe_setup_rx(ixgbe_t *ixgbe) 2440 { 2441 ixgbe_rx_ring_t *rx_ring; 2442 struct ixgbe_hw *hw = &ixgbe->hw; 2443 uint32_t reg_val; 2444 uint32_t ring_mapping; 2445 uint32_t i, index; 2446 uint32_t psrtype_rss_bit; 2447 2448 /* 2449 * Ensure that Rx is disabled while setting up 2450 * the Rx unit and Rx descriptor ring(s) 2451 */ 2452 ixgbe_disable_rx(hw); 2453 2454 /* PSRTYPE must be configured for 82599 */ 2455 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2456 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2457 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2458 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2459 reg_val |= IXGBE_PSRTYPE_L2HDR; 2460 reg_val |= 0x80000000; 2461 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2462 } else { 2463 if (ixgbe->num_rx_groups > 32) { 2464 psrtype_rss_bit = 0x20000000; 2465 } else { 2466 psrtype_rss_bit = 0x40000000; 2467 } 2468 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2469 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2470 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2471 reg_val |= IXGBE_PSRTYPE_L2HDR; 2472 reg_val |= psrtype_rss_bit; 2473 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2474 } 2475 } 2476 2477 /* 2478 * Set filter control in FCTRL to determine types of packets are passed 2479 * up to the driver. 2480 * - Pass broadcast packets. 2481 * - Do not pass flow control pause frames (82598-specific) 2482 */ 2483 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2484 reg_val |= IXGBE_FCTRL_BAM; /* Broadcast Accept Mode */ 2485 if (hw->mac.type == ixgbe_mac_82598EB) { 2486 reg_val |= IXGBE_FCTRL_DPF; /* Discard Pause Frames */ 2487 } 2488 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2489 2490 /* 2491 * Hardware checksum settings 2492 */ 2493 if (ixgbe->rx_hcksum_enable) { 2494 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2495 reg_val |= IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2496 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2497 } 2498 2499 /* 2500 * Setup VMDq and RSS for multiple receive queues 2501 */ 2502 switch (ixgbe->classify_mode) { 2503 case IXGBE_CLASSIFY_RSS: 2504 /* 2505 * One group, only RSS is needed when more than 2506 * one ring enabled. 2507 */ 2508 ixgbe_setup_rss(ixgbe); 2509 break; 2510 2511 case IXGBE_CLASSIFY_VMDQ: 2512 /* 2513 * Multiple groups, each group has one ring, 2514 * only VMDq is needed. 2515 */ 2516 ixgbe_setup_vmdq(ixgbe); 2517 break; 2518 2519 case IXGBE_CLASSIFY_VMDQ_RSS: 2520 /* 2521 * Multiple groups and multiple rings, both 2522 * VMDq and RSS are needed. 2523 */ 2524 ixgbe_setup_vmdq_rss(ixgbe); 2525 break; 2526 2527 default: 2528 break; 2529 } 2530 2531 /* 2532 * Enable the receive unit. This must be done after filter 2533 * control is set in FCTRL. On 82598, we disable the descriptor monitor. 2534 * 82598 is the only adapter which defines this RXCTRL option. 2535 */ 2536 reg_val = IXGBE_READ_REG(hw, IXGBE_RXCTRL); 2537 if (hw->mac.type == ixgbe_mac_82598EB) 2538 reg_val |= IXGBE_RXCTRL_DMBYPS; /* descriptor monitor bypass */ 2539 reg_val |= IXGBE_RXCTRL_RXEN; 2540 (void) ixgbe_enable_rx_dma(hw, reg_val); 2541 2542 /* 2543 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2544 */ 2545 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2546 rx_ring = &ixgbe->rx_rings[i]; 2547 ixgbe_setup_rx_ring(rx_ring); 2548 } 2549 2550 /* 2551 * Setup the per-ring statistics mapping. 2552 */ 2553 ring_mapping = 0; 2554 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2555 index = ixgbe->rx_rings[i].hw_index; 2556 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2)); 2557 ring_mapping |= (i & 0xF) << (8 * (index & 0x3)); 2558 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping); 2559 } 2560 2561 /* 2562 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2563 * by four bytes if the packet has a VLAN field, so includes MTU, 2564 * ethernet header and frame check sequence. 2565 * Register is MAXFRS in 82599. 2566 */ 2567 reg_val = IXGBE_READ_REG(hw, IXGBE_MHADD); 2568 reg_val &= ~IXGBE_MHADD_MFS_MASK; 2569 reg_val |= (ixgbe->default_mtu + sizeof (struct ether_header) 2570 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2571 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2572 2573 /* 2574 * Setup Jumbo Frame enable bit 2575 */ 2576 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2577 if (ixgbe->default_mtu > ETHERMTU) 2578 reg_val |= IXGBE_HLREG0_JUMBOEN; 2579 else 2580 reg_val &= ~IXGBE_HLREG0_JUMBOEN; 2581 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2582 2583 /* 2584 * Setup RSC for multiple receive queues. 2585 */ 2586 if (ixgbe->lro_enable) { 2587 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2588 /* 2589 * Make sure rx_buf_size * MAXDESC not greater 2590 * than 65535. 2591 * Intel recommends 4 for MAXDESC field value. 2592 */ 2593 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i)); 2594 reg_val |= IXGBE_RSCCTL_RSCEN; 2595 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k) 2596 reg_val |= IXGBE_RSCCTL_MAXDESC_1; 2597 else 2598 reg_val |= IXGBE_RSCCTL_MAXDESC_4; 2599 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val); 2600 } 2601 2602 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU); 2603 reg_val |= IXGBE_RSCDBU_RSCACKDIS; 2604 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val); 2605 2606 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2607 reg_val |= IXGBE_RDRXCTL_RSCACKC; 2608 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX; 2609 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2610 2611 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2612 } 2613 } 2614 2615 static void 2616 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2617 { 2618 ixgbe_t *ixgbe = tx_ring->ixgbe; 2619 struct ixgbe_hw *hw = &ixgbe->hw; 2620 uint32_t size; 2621 uint32_t buf_low; 2622 uint32_t buf_high; 2623 uint32_t reg_val; 2624 2625 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2626 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2627 2628 /* 2629 * Initialize the length register 2630 */ 2631 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2632 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2633 2634 /* 2635 * Initialize the base address registers 2636 */ 2637 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2638 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2639 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2640 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2641 2642 /* 2643 * Setup head & tail pointers 2644 */ 2645 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2646 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2647 2648 /* 2649 * Setup head write-back 2650 */ 2651 if (ixgbe->tx_head_wb_enable) { 2652 /* 2653 * The memory of the head write-back is allocated using 2654 * the extra tbd beyond the tail of the tbd ring. 2655 */ 2656 tx_ring->tbd_head_wb = (uint32_t *) 2657 ((uintptr_t)tx_ring->tbd_area.address + size); 2658 *tx_ring->tbd_head_wb = 0; 2659 2660 buf_low = (uint32_t) 2661 (tx_ring->tbd_area.dma_address + size); 2662 buf_high = (uint32_t) 2663 ((tx_ring->tbd_area.dma_address + size) >> 32); 2664 2665 /* Set the head write-back enable bit */ 2666 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2667 2668 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2669 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2670 2671 /* 2672 * Turn off relaxed ordering for head write back or it will 2673 * cause problems with the tx recycling 2674 */ 2675 2676 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ? 2677 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) : 2678 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index)); 2679 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2680 if (hw->mac.type == ixgbe_mac_82598EB) { 2681 IXGBE_WRITE_REG(hw, 2682 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2683 } else { 2684 IXGBE_WRITE_REG(hw, 2685 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val); 2686 } 2687 } else { 2688 tx_ring->tbd_head_wb = NULL; 2689 } 2690 2691 tx_ring->tbd_head = 0; 2692 tx_ring->tbd_tail = 0; 2693 tx_ring->tbd_free = tx_ring->ring_size; 2694 2695 if (ixgbe->tx_ring_init == B_TRUE) { 2696 tx_ring->tcb_head = 0; 2697 tx_ring->tcb_tail = 0; 2698 tx_ring->tcb_free = tx_ring->free_list_size; 2699 } 2700 2701 /* 2702 * Initialize the s/w context structure 2703 */ 2704 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2705 } 2706 2707 static void 2708 ixgbe_setup_tx(ixgbe_t *ixgbe) 2709 { 2710 struct ixgbe_hw *hw = &ixgbe->hw; 2711 ixgbe_tx_ring_t *tx_ring; 2712 uint32_t reg_val; 2713 uint32_t ring_mapping; 2714 int i; 2715 2716 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2717 tx_ring = &ixgbe->tx_rings[i]; 2718 ixgbe_setup_tx_ring(tx_ring); 2719 } 2720 2721 /* 2722 * Setup the per-ring statistics mapping. 2723 */ 2724 ring_mapping = 0; 2725 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2726 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 2727 if ((i & 0x3) == 0x3) { 2728 switch (hw->mac.type) { 2729 case ixgbe_mac_82598EB: 2730 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 2731 ring_mapping); 2732 break; 2733 2734 case ixgbe_mac_82599EB: 2735 case ixgbe_mac_X540: 2736 case ixgbe_mac_X550: 2737 case ixgbe_mac_X550EM_x: 2738 case ixgbe_mac_X550EM_a: 2739 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 2740 ring_mapping); 2741 break; 2742 2743 default: 2744 break; 2745 } 2746 2747 ring_mapping = 0; 2748 } 2749 } 2750 if (i & 0x3) { 2751 switch (hw->mac.type) { 2752 case ixgbe_mac_82598EB: 2753 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2754 break; 2755 2756 case ixgbe_mac_82599EB: 2757 case ixgbe_mac_X540: 2758 case ixgbe_mac_X550: 2759 case ixgbe_mac_X550EM_x: 2760 case ixgbe_mac_X550EM_a: 2761 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 2762 break; 2763 2764 default: 2765 break; 2766 } 2767 } 2768 2769 /* 2770 * Enable CRC appending and TX padding (for short tx frames) 2771 */ 2772 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2773 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2774 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2775 2776 /* 2777 * enable DMA for 82599, X540 and X550 parts 2778 */ 2779 if (hw->mac.type == ixgbe_mac_82599EB || 2780 hw->mac.type == ixgbe_mac_X540 || 2781 hw->mac.type == ixgbe_mac_X550 || 2782 hw->mac.type == ixgbe_mac_X550EM_x || 2783 hw->mac.type == ixgbe_mac_X550EM_a) { 2784 /* DMATXCTL.TE must be set after all Tx config is complete */ 2785 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2786 reg_val |= IXGBE_DMATXCTL_TE; 2787 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2788 2789 /* Disable arbiter to set MTQC */ 2790 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2791 reg_val |= IXGBE_RTTDCS_ARBDIS; 2792 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2793 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2794 reg_val &= ~IXGBE_RTTDCS_ARBDIS; 2795 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2796 } 2797 2798 /* 2799 * Enabling tx queues .. 2800 * For 82599 must be done after DMATXCTL.TE is set 2801 */ 2802 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2803 tx_ring = &ixgbe->tx_rings[i]; 2804 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2805 reg_val |= IXGBE_TXDCTL_ENABLE; 2806 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2807 } 2808 } 2809 2810 /* 2811 * ixgbe_setup_rss - Setup receive-side scaling feature. 2812 */ 2813 static void 2814 ixgbe_setup_rss(ixgbe_t *ixgbe) 2815 { 2816 struct ixgbe_hw *hw = &ixgbe->hw; 2817 uint32_t mrqc; 2818 2819 /* 2820 * Initialize RETA/ERETA table 2821 */ 2822 ixgbe_setup_rss_table(ixgbe); 2823 2824 /* 2825 * Enable RSS & perform hash on these packet types 2826 */ 2827 mrqc = IXGBE_MRQC_RSSEN | 2828 IXGBE_MRQC_RSS_FIELD_IPV4 | 2829 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2830 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2831 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2832 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2833 IXGBE_MRQC_RSS_FIELD_IPV6 | 2834 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2835 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2836 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2837 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2838 } 2839 2840 /* 2841 * ixgbe_setup_vmdq - Setup MAC classification feature 2842 */ 2843 static void 2844 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2845 { 2846 struct ixgbe_hw *hw = &ixgbe->hw; 2847 uint32_t vmdctl, i, vtctl; 2848 2849 /* 2850 * Setup the VMDq Control register, enable VMDq based on 2851 * packet destination MAC address: 2852 */ 2853 switch (hw->mac.type) { 2854 case ixgbe_mac_82598EB: 2855 /* 2856 * VMDq Enable = 1; 2857 * VMDq Filter = 0; MAC filtering 2858 * Default VMDq output index = 0; 2859 */ 2860 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2861 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2862 break; 2863 2864 case ixgbe_mac_82599EB: 2865 case ixgbe_mac_X540: 2866 case ixgbe_mac_X550: 2867 case ixgbe_mac_X550EM_x: 2868 case ixgbe_mac_X550EM_a: 2869 /* 2870 * Enable VMDq-only. 2871 */ 2872 vmdctl = IXGBE_MRQC_VMDQEN; 2873 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2874 2875 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2876 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2877 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2878 } 2879 2880 /* 2881 * Enable Virtualization and Replication. 2882 */ 2883 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2884 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2885 2886 /* 2887 * Enable receiving packets to all VFs 2888 */ 2889 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2890 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2891 break; 2892 2893 default: 2894 break; 2895 } 2896 } 2897 2898 /* 2899 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2900 */ 2901 static void 2902 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2903 { 2904 struct ixgbe_hw *hw = &ixgbe->hw; 2905 uint32_t i, mrqc; 2906 uint32_t vtctl, vmdctl; 2907 2908 /* 2909 * Initialize RETA/ERETA table 2910 */ 2911 ixgbe_setup_rss_table(ixgbe); 2912 2913 /* 2914 * Enable and setup RSS and VMDq 2915 */ 2916 switch (hw->mac.type) { 2917 case ixgbe_mac_82598EB: 2918 /* 2919 * Enable RSS & Setup RSS Hash functions 2920 */ 2921 mrqc = IXGBE_MRQC_RSSEN | 2922 IXGBE_MRQC_RSS_FIELD_IPV4 | 2923 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2924 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2925 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2926 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2927 IXGBE_MRQC_RSS_FIELD_IPV6 | 2928 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2929 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2930 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2931 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2932 2933 /* 2934 * Enable and Setup VMDq 2935 * VMDq Filter = 0; MAC filtering 2936 * Default VMDq output index = 0; 2937 */ 2938 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2939 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2940 break; 2941 2942 case ixgbe_mac_82599EB: 2943 case ixgbe_mac_X540: 2944 case ixgbe_mac_X550: 2945 case ixgbe_mac_X550EM_x: 2946 case ixgbe_mac_X550EM_a: 2947 /* 2948 * Enable RSS & Setup RSS Hash functions 2949 */ 2950 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2951 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2952 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2953 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2954 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2955 IXGBE_MRQC_RSS_FIELD_IPV6 | 2956 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2957 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2958 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2959 2960 /* 2961 * Enable VMDq+RSS. 2962 */ 2963 if (ixgbe->num_rx_groups > 32) { 2964 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2965 } else { 2966 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2967 } 2968 2969 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2970 2971 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2972 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2973 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2974 } 2975 break; 2976 2977 default: 2978 break; 2979 2980 } 2981 2982 if (hw->mac.type == ixgbe_mac_82599EB || 2983 hw->mac.type == ixgbe_mac_X540 || 2984 hw->mac.type == ixgbe_mac_X550 || 2985 hw->mac.type == ixgbe_mac_X550EM_x || 2986 hw->mac.type == ixgbe_mac_X550EM_a) { 2987 /* 2988 * Enable Virtualization and Replication. 2989 */ 2990 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2991 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2992 2993 /* 2994 * Enable receiving packets to all VFs 2995 */ 2996 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2997 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2998 } 2999 } 3000 3001 /* 3002 * ixgbe_setup_rss_table - Setup RSS table 3003 */ 3004 static void 3005 ixgbe_setup_rss_table(ixgbe_t *ixgbe) 3006 { 3007 struct ixgbe_hw *hw = &ixgbe->hw; 3008 uint32_t i, j; 3009 uint32_t random; 3010 uint32_t reta; 3011 uint32_t ring_per_group; 3012 uint32_t ring; 3013 uint32_t table_size; 3014 uint32_t index_mult; 3015 uint32_t rxcsum; 3016 3017 /* 3018 * Set multiplier for RETA setup and table size based on MAC type. 3019 * RETA table sizes vary by model: 3020 * 3021 * 82598, 82599, X540: 128 table entries. 3022 * X550: 512 table entries. 3023 */ 3024 index_mult = 0x1; 3025 table_size = 128; 3026 switch (ixgbe->hw.mac.type) { 3027 case ixgbe_mac_82598EB: 3028 index_mult = 0x11; 3029 break; 3030 case ixgbe_mac_X550: 3031 case ixgbe_mac_X550EM_x: 3032 case ixgbe_mac_X550EM_a: 3033 table_size = 512; 3034 break; 3035 default: 3036 break; 3037 } 3038 3039 /* 3040 * Fill out RSS redirection table. The configuation of the indices is 3041 * hardware-dependent. 3042 * 3043 * 82598: 8 bits wide containing two 4 bit RSS indices 3044 * 82599, X540: 8 bits wide containing one 4 bit RSS index 3045 * X550: 8 bits wide containing one 6 bit RSS index 3046 */ 3047 reta = 0; 3048 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3049 3050 for (i = 0, j = 0; i < table_size; i++, j++) { 3051 if (j == ring_per_group) j = 0; 3052 3053 /* 3054 * The low 8 bits are for hash value (n+0); 3055 * The next 8 bits are for hash value (n+1), etc. 3056 */ 3057 ring = (j * index_mult); 3058 reta = reta >> 8; 3059 reta = reta | (((uint32_t)ring) << 24); 3060 3061 if ((i & 3) == 3) { 3062 /* 3063 * The first 128 table entries are programmed into the 3064 * RETA register, with any beyond that (eg; on X550) 3065 * into ERETA. 3066 */ 3067 if (i < 128) 3068 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 3069 else 3070 IXGBE_WRITE_REG(hw, IXGBE_ERETA((i >> 2) - 32), 3071 reta); 3072 reta = 0; 3073 } 3074 } 3075 3076 /* 3077 * Fill out hash function seeds with a random constant 3078 */ 3079 for (i = 0; i < 10; i++) { 3080 (void) random_get_pseudo_bytes((uint8_t *)&random, 3081 sizeof (uint32_t)); 3082 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 3083 } 3084 3085 /* 3086 * Disable Packet Checksum to enable RSS for multiple receive queues. 3087 * It is an adapter hardware limitation that Packet Checksum is 3088 * mutually exclusive with RSS. 3089 */ 3090 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 3091 rxcsum |= IXGBE_RXCSUM_PCSD; 3092 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 3093 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 3094 } 3095 3096 /* 3097 * ixgbe_init_unicst - Initialize the unicast addresses. 3098 */ 3099 static void 3100 ixgbe_init_unicst(ixgbe_t *ixgbe) 3101 { 3102 struct ixgbe_hw *hw = &ixgbe->hw; 3103 uint8_t *mac_addr; 3104 int slot; 3105 /* 3106 * Here we should consider two situations: 3107 * 3108 * 1. Chipset is initialized at the first time, 3109 * Clear all the multiple unicast addresses. 3110 * 3111 * 2. Chipset is reset 3112 * Recover the multiple unicast addresses from the 3113 * software data structure to the RAR registers. 3114 */ 3115 if (!ixgbe->unicst_init) { 3116 /* 3117 * Initialize the multiple unicast addresses 3118 */ 3119 ixgbe->unicst_total = hw->mac.num_rar_entries; 3120 ixgbe->unicst_avail = ixgbe->unicst_total; 3121 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 3122 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 3123 bzero(mac_addr, ETHERADDRL); 3124 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0); 3125 ixgbe->unicst_addr[slot].mac.set = 0; 3126 } 3127 ixgbe->unicst_init = B_TRUE; 3128 } else { 3129 /* Re-configure the RAR registers */ 3130 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 3131 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 3132 if (ixgbe->unicst_addr[slot].mac.set == 1) { 3133 (void) ixgbe_set_rar(hw, slot, mac_addr, 3134 ixgbe->unicst_addr[slot].mac.group_index, 3135 IXGBE_RAH_AV); 3136 } else { 3137 bzero(mac_addr, ETHERADDRL); 3138 (void) ixgbe_set_rar(hw, slot, mac_addr, 0, 0); 3139 } 3140 } 3141 } 3142 } 3143 3144 /* 3145 * ixgbe_unicst_find - Find the slot for the specified unicast address 3146 */ 3147 int 3148 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 3149 { 3150 int slot; 3151 3152 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3153 3154 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 3155 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 3156 mac_addr, ETHERADDRL) == 0) 3157 return (slot); 3158 } 3159 3160 return (-1); 3161 } 3162 3163 /* 3164 * ixgbe_multicst_add - Add a multicst address. 3165 */ 3166 int 3167 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 3168 { 3169 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3170 3171 if ((multiaddr[0] & 01) == 0) { 3172 return (EINVAL); 3173 } 3174 3175 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 3176 return (ENOENT); 3177 } 3178 3179 bcopy(multiaddr, 3180 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 3181 ixgbe->mcast_count++; 3182 3183 /* 3184 * Update the multicast table in the hardware 3185 */ 3186 ixgbe_setup_multicst(ixgbe); 3187 3188 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3189 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3190 return (EIO); 3191 } 3192 3193 return (0); 3194 } 3195 3196 /* 3197 * ixgbe_multicst_remove - Remove a multicst address. 3198 */ 3199 int 3200 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 3201 { 3202 int i; 3203 3204 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3205 3206 for (i = 0; i < ixgbe->mcast_count; i++) { 3207 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 3208 ETHERADDRL) == 0) { 3209 for (i++; i < ixgbe->mcast_count; i++) { 3210 ixgbe->mcast_table[i - 1] = 3211 ixgbe->mcast_table[i]; 3212 } 3213 ixgbe->mcast_count--; 3214 break; 3215 } 3216 } 3217 3218 /* 3219 * Update the multicast table in the hardware 3220 */ 3221 ixgbe_setup_multicst(ixgbe); 3222 3223 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3224 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3225 return (EIO); 3226 } 3227 3228 return (0); 3229 } 3230 3231 /* 3232 * ixgbe_setup_multicast - Setup multicast data structures. 3233 * 3234 * This routine initializes all of the multicast related structures 3235 * and save them in the hardware registers. 3236 */ 3237 static void 3238 ixgbe_setup_multicst(ixgbe_t *ixgbe) 3239 { 3240 uint8_t *mc_addr_list; 3241 uint32_t mc_addr_count; 3242 struct ixgbe_hw *hw = &ixgbe->hw; 3243 3244 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3245 3246 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 3247 3248 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 3249 mc_addr_count = ixgbe->mcast_count; 3250 3251 /* 3252 * Update the multicast addresses to the MTA registers 3253 */ 3254 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 3255 ixgbe_mc_table_itr, TRUE); 3256 } 3257 3258 /* 3259 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 3260 * 3261 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 3262 * Different chipsets may have different allowed configuration of vmdq and rss. 3263 */ 3264 static void 3265 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 3266 { 3267 struct ixgbe_hw *hw = &ixgbe->hw; 3268 uint32_t ring_per_group; 3269 3270 switch (hw->mac.type) { 3271 case ixgbe_mac_82598EB: 3272 /* 3273 * 82598 supports the following combination: 3274 * vmdq no. x rss no. 3275 * [5..16] x 1 3276 * [1..4] x [1..16] 3277 * However 8 rss queue per pool (vmdq) is sufficient for 3278 * most cases. 3279 */ 3280 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3281 if (ixgbe->num_rx_groups > 4) { 3282 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 3283 } else { 3284 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3285 min(8, ring_per_group); 3286 } 3287 3288 break; 3289 3290 case ixgbe_mac_82599EB: 3291 case ixgbe_mac_X540: 3292 case ixgbe_mac_X550: 3293 case ixgbe_mac_X550EM_x: 3294 case ixgbe_mac_X550EM_a: 3295 /* 3296 * 82599 supports the following combination: 3297 * vmdq no. x rss no. 3298 * [33..64] x [1..2] 3299 * [2..32] x [1..4] 3300 * 1 x [1..16] 3301 * However 8 rss queue per pool (vmdq) is sufficient for 3302 * most cases. 3303 * 3304 * For now, treat X540 and X550 like the 82599. 3305 */ 3306 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3307 if (ixgbe->num_rx_groups == 1) { 3308 ixgbe->num_rx_rings = min(8, ring_per_group); 3309 } else if (ixgbe->num_rx_groups <= 32) { 3310 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3311 min(4, ring_per_group); 3312 } else if (ixgbe->num_rx_groups <= 64) { 3313 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3314 min(2, ring_per_group); 3315 } 3316 break; 3317 3318 default: 3319 break; 3320 } 3321 3322 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3323 3324 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 3325 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3326 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 3327 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 3328 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 3329 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 3330 } else { 3331 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 3332 } 3333 3334 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d", 3335 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 3336 } 3337 3338 /* 3339 * ixgbe_get_conf - Get driver configurations set in driver.conf. 3340 * 3341 * This routine gets user-configured values out of the configuration 3342 * file ixgbe.conf. 3343 * 3344 * For each configurable value, there is a minimum, a maximum, and a 3345 * default. 3346 * If user does not configure a value, use the default. 3347 * If user configures below the minimum, use the minumum. 3348 * If user configures above the maximum, use the maxumum. 3349 */ 3350 static void 3351 ixgbe_get_conf(ixgbe_t *ixgbe) 3352 { 3353 struct ixgbe_hw *hw = &ixgbe->hw; 3354 uint32_t flow_control; 3355 3356 /* 3357 * ixgbe driver supports the following user configurations: 3358 * 3359 * Jumbo frame configuration: 3360 * default_mtu 3361 * 3362 * Ethernet flow control configuration: 3363 * flow_control 3364 * 3365 * Multiple rings configurations: 3366 * tx_queue_number 3367 * tx_ring_size 3368 * rx_queue_number 3369 * rx_ring_size 3370 * 3371 * Call ixgbe_get_prop() to get the value for a specific 3372 * configuration parameter. 3373 */ 3374 3375 /* 3376 * Jumbo frame configuration - max_frame_size controls host buffer 3377 * allocation, so includes MTU, ethernet header, vlan tag and 3378 * frame check sequence. 3379 */ 3380 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 3381 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 3382 3383 ixgbe->max_frame_size = ixgbe->default_mtu + 3384 sizeof (struct ether_vlan_header) + ETHERFCSL; 3385 3386 /* 3387 * Ethernet flow control configuration 3388 */ 3389 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 3390 ixgbe_fc_none, 3, ixgbe_fc_none); 3391 if (flow_control == 3) 3392 flow_control = ixgbe_fc_default; 3393 3394 /* 3395 * fc.requested mode is what the user requests. After autoneg, 3396 * fc.current_mode will be the flow_control mode that was negotiated. 3397 */ 3398 hw->fc.requested_mode = flow_control; 3399 3400 /* 3401 * Multiple rings configurations 3402 */ 3403 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 3404 ixgbe->capab->min_tx_que_num, 3405 ixgbe->capab->max_tx_que_num, 3406 ixgbe->capab->def_tx_que_num); 3407 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 3408 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 3409 3410 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 3411 ixgbe->capab->min_rx_que_num, 3412 ixgbe->capab->max_rx_que_num, 3413 ixgbe->capab->def_rx_que_num); 3414 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 3415 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 3416 3417 /* 3418 * Multiple groups configuration 3419 */ 3420 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 3421 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 3422 ixgbe->capab->def_rx_grp_num); 3423 3424 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 3425 0, 1, DEFAULT_MR_ENABLE); 3426 3427 if (ixgbe->mr_enable == B_FALSE) { 3428 ixgbe->num_tx_rings = 1; 3429 ixgbe->num_rx_rings = 1; 3430 ixgbe->num_rx_groups = 1; 3431 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3432 } else { 3433 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3434 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 3435 /* 3436 * The combination of num_rx_rings and num_rx_groups 3437 * may be not supported by h/w. We need to adjust 3438 * them to appropriate values. 3439 */ 3440 ixgbe_setup_vmdq_rss_conf(ixgbe); 3441 } 3442 3443 /* 3444 * Tunable used to force an interrupt type. The only use is 3445 * for testing of the lesser interrupt types. 3446 * 0 = don't force interrupt type 3447 * 1 = force interrupt type MSI-X 3448 * 2 = force interrupt type MSI 3449 * 3 = force interrupt type Legacy 3450 */ 3451 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 3452 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 3453 3454 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 3455 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 3456 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 3457 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 3458 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 3459 0, 1, DEFAULT_LSO_ENABLE); 3460 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 3461 0, 1, DEFAULT_LRO_ENABLE); 3462 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 3463 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 3464 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe, 3465 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE); 3466 3467 /* Head Write Back not recommended for 82599, X540 and X550 */ 3468 if (hw->mac.type == ixgbe_mac_82599EB || 3469 hw->mac.type == ixgbe_mac_X540 || 3470 hw->mac.type == ixgbe_mac_X550 || 3471 hw->mac.type == ixgbe_mac_X550EM_x || 3472 hw->mac.type == ixgbe_mac_X550EM_a) { 3473 ixgbe->tx_head_wb_enable = B_FALSE; 3474 } 3475 3476 /* 3477 * ixgbe LSO needs the tx h/w checksum support. 3478 * LSO will be disabled if tx h/w checksum is not 3479 * enabled. 3480 */ 3481 if (ixgbe->tx_hcksum_enable == B_FALSE) { 3482 ixgbe->lso_enable = B_FALSE; 3483 } 3484 3485 /* 3486 * ixgbe LRO needs the rx h/w checksum support. 3487 * LRO will be disabled if rx h/w checksum is not 3488 * enabled. 3489 */ 3490 if (ixgbe->rx_hcksum_enable == B_FALSE) { 3491 ixgbe->lro_enable = B_FALSE; 3492 } 3493 3494 /* 3495 * ixgbe LRO only supported by 82599, X540 and X550 3496 */ 3497 if (hw->mac.type == ixgbe_mac_82598EB) { 3498 ixgbe->lro_enable = B_FALSE; 3499 } 3500 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 3501 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 3502 DEFAULT_TX_COPY_THRESHOLD); 3503 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 3504 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 3505 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 3506 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 3507 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 3508 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 3509 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3510 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3511 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3512 3513 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3514 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3515 DEFAULT_RX_COPY_THRESHOLD); 3516 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3517 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3518 DEFAULT_RX_LIMIT_PER_INTR); 3519 3520 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3521 ixgbe->capab->min_intr_throttle, 3522 ixgbe->capab->max_intr_throttle, 3523 ixgbe->capab->def_intr_throttle); 3524 /* 3525 * 82599, X540 and X550 require the interrupt throttling rate is 3526 * a multiple of 8. This is enforced by the register definiton. 3527 */ 3528 if (hw->mac.type == ixgbe_mac_82599EB || 3529 hw->mac.type == ixgbe_mac_X540 || 3530 hw->mac.type == ixgbe_mac_X550 || 3531 hw->mac.type == ixgbe_mac_X550EM_x || 3532 hw->mac.type == ixgbe_mac_X550EM_a) 3533 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3534 3535 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe, 3536 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP); 3537 } 3538 3539 static void 3540 ixgbe_init_params(ixgbe_t *ixgbe) 3541 { 3542 struct ixgbe_hw *hw = &ixgbe->hw; 3543 ixgbe_link_speed speeds_supported = 0; 3544 boolean_t negotiate; 3545 3546 /* 3547 * Get a list of speeds the adapter supports. If the hw struct hasn't 3548 * been populated with this information yet, retrieve it from the 3549 * adapter and save it to our own variable. 3550 * 3551 * On certain adapters, such as ones which use SFPs, the contents of 3552 * hw->phy.speeds_supported (and hw->phy.autoneg_advertised) are not 3553 * updated, so we must rely on calling ixgbe_get_link_capabilities() 3554 * in order to ascertain the speeds which we are capable of supporting, 3555 * and in the case of SFP-equipped adapters, which speed we are 3556 * advertising. If ixgbe_get_link_capabilities() fails for some reason, 3557 * we'll go with a default list of speeds as a last resort. 3558 */ 3559 speeds_supported = hw->phy.speeds_supported; 3560 3561 if (speeds_supported == 0) { 3562 if (ixgbe_get_link_capabilities(hw, &speeds_supported, 3563 &negotiate) != IXGBE_SUCCESS) { 3564 if (hw->mac.type == ixgbe_mac_82598EB) { 3565 speeds_supported = 3566 IXGBE_LINK_SPEED_82598_AUTONEG; 3567 } else { 3568 speeds_supported = 3569 IXGBE_LINK_SPEED_82599_AUTONEG; 3570 } 3571 } 3572 } 3573 ixgbe->speeds_supported = speeds_supported; 3574 3575 /* 3576 * By default, all supported speeds are enabled and advertised. 3577 */ 3578 if (speeds_supported & IXGBE_LINK_SPEED_10GB_FULL) { 3579 ixgbe->param_en_10000fdx_cap = 1; 3580 ixgbe->param_adv_10000fdx_cap = 1; 3581 } else { 3582 ixgbe->param_en_10000fdx_cap = 0; 3583 ixgbe->param_adv_10000fdx_cap = 0; 3584 } 3585 3586 if (speeds_supported & IXGBE_LINK_SPEED_5GB_FULL) { 3587 ixgbe->param_en_5000fdx_cap = 1; 3588 ixgbe->param_adv_5000fdx_cap = 1; 3589 } else { 3590 ixgbe->param_en_5000fdx_cap = 0; 3591 ixgbe->param_adv_5000fdx_cap = 0; 3592 } 3593 3594 if (speeds_supported & IXGBE_LINK_SPEED_2_5GB_FULL) { 3595 ixgbe->param_en_2500fdx_cap = 1; 3596 ixgbe->param_adv_2500fdx_cap = 1; 3597 } else { 3598 ixgbe->param_en_2500fdx_cap = 0; 3599 ixgbe->param_adv_2500fdx_cap = 0; 3600 } 3601 3602 if (speeds_supported & IXGBE_LINK_SPEED_1GB_FULL) { 3603 ixgbe->param_en_1000fdx_cap = 1; 3604 ixgbe->param_adv_1000fdx_cap = 1; 3605 } else { 3606 ixgbe->param_en_1000fdx_cap = 0; 3607 ixgbe->param_adv_1000fdx_cap = 0; 3608 } 3609 3610 if (speeds_supported & IXGBE_LINK_SPEED_100_FULL) { 3611 ixgbe->param_en_100fdx_cap = 1; 3612 ixgbe->param_adv_100fdx_cap = 1; 3613 } else { 3614 ixgbe->param_en_100fdx_cap = 0; 3615 ixgbe->param_adv_100fdx_cap = 0; 3616 } 3617 3618 ixgbe->param_pause_cap = 1; 3619 ixgbe->param_asym_pause_cap = 1; 3620 ixgbe->param_rem_fault = 0; 3621 3622 ixgbe->param_adv_autoneg_cap = 1; 3623 ixgbe->param_adv_pause_cap = 1; 3624 ixgbe->param_adv_asym_pause_cap = 1; 3625 ixgbe->param_adv_rem_fault = 0; 3626 3627 ixgbe->param_lp_10000fdx_cap = 0; 3628 ixgbe->param_lp_5000fdx_cap = 0; 3629 ixgbe->param_lp_2500fdx_cap = 0; 3630 ixgbe->param_lp_1000fdx_cap = 0; 3631 ixgbe->param_lp_100fdx_cap = 0; 3632 ixgbe->param_lp_autoneg_cap = 0; 3633 ixgbe->param_lp_pause_cap = 0; 3634 ixgbe->param_lp_asym_pause_cap = 0; 3635 ixgbe->param_lp_rem_fault = 0; 3636 } 3637 3638 /* 3639 * ixgbe_get_prop - Get a property value out of the configuration file 3640 * ixgbe.conf. 3641 * 3642 * Caller provides the name of the property, a default value, a minimum 3643 * value, and a maximum value. 3644 * 3645 * Return configured value of the property, with default, minimum and 3646 * maximum properly applied. 3647 */ 3648 static int 3649 ixgbe_get_prop(ixgbe_t *ixgbe, 3650 char *propname, /* name of the property */ 3651 int minval, /* minimum acceptable value */ 3652 int maxval, /* maximim acceptable value */ 3653 int defval) /* default value */ 3654 { 3655 int value; 3656 3657 /* 3658 * Call ddi_prop_get_int() to read the conf settings 3659 */ 3660 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3661 DDI_PROP_DONTPASS, propname, defval); 3662 if (value > maxval) 3663 value = maxval; 3664 3665 if (value < minval) 3666 value = minval; 3667 3668 return (value); 3669 } 3670 3671 /* 3672 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3673 */ 3674 int 3675 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3676 { 3677 struct ixgbe_hw *hw = &ixgbe->hw; 3678 ixgbe_link_speed advertised = 0; 3679 3680 /* 3681 * Assemble a list of enabled speeds to auto-negotiate with. 3682 */ 3683 if (ixgbe->param_en_10000fdx_cap == 1) 3684 advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3685 3686 if (ixgbe->param_en_5000fdx_cap == 1) 3687 advertised |= IXGBE_LINK_SPEED_5GB_FULL; 3688 3689 if (ixgbe->param_en_2500fdx_cap == 1) 3690 advertised |= IXGBE_LINK_SPEED_2_5GB_FULL; 3691 3692 if (ixgbe->param_en_1000fdx_cap == 1) 3693 advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3694 3695 if (ixgbe->param_en_100fdx_cap == 1) 3696 advertised |= IXGBE_LINK_SPEED_100_FULL; 3697 3698 /* 3699 * As a last resort, autoneg with a default list of speeds. 3700 */ 3701 if (ixgbe->param_adv_autoneg_cap == 1 && advertised == 0) { 3702 ixgbe_notice(ixgbe, "Invalid link settings. Setting link " 3703 "to autonegotiate with full capabilities."); 3704 3705 if (hw->mac.type == ixgbe_mac_82598EB) 3706 advertised = IXGBE_LINK_SPEED_82598_AUTONEG; 3707 else 3708 advertised = IXGBE_LINK_SPEED_82599_AUTONEG; 3709 } 3710 3711 if (setup_hw) { 3712 if (ixgbe_setup_link(&ixgbe->hw, advertised, 3713 ixgbe->param_adv_autoneg_cap) != IXGBE_SUCCESS) { 3714 ixgbe_notice(ixgbe, "Setup link failed on this " 3715 "device."); 3716 return (IXGBE_FAILURE); 3717 } 3718 } 3719 3720 return (IXGBE_SUCCESS); 3721 } 3722 3723 /* 3724 * ixgbe_driver_link_check - Link status processing. 3725 * 3726 * This function can be called in both kernel context and interrupt context 3727 */ 3728 static void 3729 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3730 { 3731 struct ixgbe_hw *hw = &ixgbe->hw; 3732 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3733 boolean_t link_up = B_FALSE; 3734 boolean_t link_changed = B_FALSE; 3735 3736 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3737 3738 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE); 3739 if (link_up) { 3740 ixgbe->link_check_complete = B_TRUE; 3741 3742 /* Link is up, enable flow control settings */ 3743 (void) ixgbe_fc_enable(hw); 3744 3745 /* 3746 * The Link is up, check whether it was marked as down earlier 3747 */ 3748 if (ixgbe->link_state != LINK_STATE_UP) { 3749 switch (speed) { 3750 case IXGBE_LINK_SPEED_10GB_FULL: 3751 ixgbe->link_speed = SPEED_10GB; 3752 break; 3753 case IXGBE_LINK_SPEED_5GB_FULL: 3754 ixgbe->link_speed = SPEED_5GB; 3755 break; 3756 case IXGBE_LINK_SPEED_2_5GB_FULL: 3757 ixgbe->link_speed = SPEED_2_5GB; 3758 break; 3759 case IXGBE_LINK_SPEED_1GB_FULL: 3760 ixgbe->link_speed = SPEED_1GB; 3761 break; 3762 case IXGBE_LINK_SPEED_100_FULL: 3763 ixgbe->link_speed = SPEED_100; 3764 } 3765 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3766 ixgbe->link_state = LINK_STATE_UP; 3767 link_changed = B_TRUE; 3768 } 3769 } else { 3770 if (ixgbe->link_check_complete == B_TRUE || 3771 (ixgbe->link_check_complete == B_FALSE && 3772 gethrtime() >= ixgbe->link_check_hrtime)) { 3773 /* 3774 * The link is really down 3775 */ 3776 ixgbe->link_check_complete = B_TRUE; 3777 3778 if (ixgbe->link_state != LINK_STATE_DOWN) { 3779 ixgbe->link_speed = 0; 3780 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3781 ixgbe->link_state = LINK_STATE_DOWN; 3782 link_changed = B_TRUE; 3783 } 3784 } 3785 } 3786 3787 /* 3788 * If we are in an interrupt context, need to re-enable the 3789 * interrupt, which was automasked 3790 */ 3791 if (servicing_interrupt() != 0) { 3792 ixgbe->eims |= IXGBE_EICR_LSC; 3793 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3794 } 3795 3796 if (link_changed) { 3797 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3798 } 3799 } 3800 3801 /* 3802 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3803 */ 3804 static void 3805 ixgbe_sfp_check(void *arg) 3806 { 3807 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3808 uint32_t eicr = ixgbe->eicr; 3809 struct ixgbe_hw *hw = &ixgbe->hw; 3810 3811 mutex_enter(&ixgbe->gen_lock); 3812 (void) hw->phy.ops.identify_sfp(hw); 3813 if (eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) { 3814 /* clear the interrupt */ 3815 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1_BY_MAC(hw)); 3816 3817 /* if link up, do multispeed fiber setup */ 3818 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3819 B_TRUE); 3820 ixgbe_driver_link_check(ixgbe); 3821 ixgbe_get_hw_state(ixgbe); 3822 } else if (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)) { 3823 /* clear the interrupt */ 3824 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2_BY_MAC(hw)); 3825 3826 /* if link up, do sfp module setup */ 3827 (void) hw->mac.ops.setup_sfp(hw); 3828 3829 /* do multispeed fiber setup */ 3830 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3831 B_TRUE); 3832 ixgbe_driver_link_check(ixgbe); 3833 ixgbe_get_hw_state(ixgbe); 3834 } 3835 mutex_exit(&ixgbe->gen_lock); 3836 3837 /* 3838 * We need to fully re-check the link later. 3839 */ 3840 ixgbe->link_check_complete = B_FALSE; 3841 ixgbe->link_check_hrtime = gethrtime() + 3842 (IXGBE_LINK_UP_TIME * 100000000ULL); 3843 } 3844 3845 /* 3846 * ixgbe_overtemp_check - overtemp module processing done in taskq 3847 * 3848 * This routine will only be called on adapters with temperature sensor. 3849 * The indication of over-temperature can be either SDP0 interrupt or the link 3850 * status change interrupt. 3851 */ 3852 static void 3853 ixgbe_overtemp_check(void *arg) 3854 { 3855 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3856 struct ixgbe_hw *hw = &ixgbe->hw; 3857 uint32_t eicr = ixgbe->eicr; 3858 ixgbe_link_speed speed; 3859 boolean_t link_up; 3860 3861 mutex_enter(&ixgbe->gen_lock); 3862 3863 /* make sure we know current state of link */ 3864 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE); 3865 3866 /* check over-temp condition */ 3867 if (((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) && (!link_up)) || 3868 (eicr & IXGBE_EICR_LSC)) { 3869 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) { 3870 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3871 3872 /* 3873 * Disable the adapter interrupts 3874 */ 3875 ixgbe_disable_adapter_interrupts(ixgbe); 3876 3877 /* 3878 * Disable Rx/Tx units 3879 */ 3880 (void) ixgbe_stop_adapter(hw); 3881 3882 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3883 ixgbe_error(ixgbe, 3884 "Problem: Network adapter has been stopped " 3885 "because it has overheated"); 3886 ixgbe_error(ixgbe, 3887 "Action: Restart the computer. " 3888 "If the problem persists, power off the system " 3889 "and replace the adapter"); 3890 } 3891 } 3892 3893 /* write to clear the interrupt */ 3894 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3895 3896 mutex_exit(&ixgbe->gen_lock); 3897 } 3898 3899 /* 3900 * ixgbe_phy_check - taskq to process interrupts from an external PHY 3901 * 3902 * This routine will only be called on adapters with external PHYs 3903 * (such as X550) that may be trying to raise our attention to some event. 3904 * Currently, this is limited to claiming PHY overtemperature and link status 3905 * change (LSC) events, however this may expand to include other things in 3906 * future adapters. 3907 */ 3908 static void 3909 ixgbe_phy_check(void *arg) 3910 { 3911 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3912 struct ixgbe_hw *hw = &ixgbe->hw; 3913 int rv; 3914 3915 mutex_enter(&ixgbe->gen_lock); 3916 3917 /* 3918 * X550 baseT PHY overtemp and LSC events are handled here. 3919 * 3920 * If an overtemp event occurs, it will be reflected in the 3921 * return value of phy.ops.handle_lasi() and the common code will 3922 * automatically power off the baseT PHY. This is our cue to trigger 3923 * an FMA event. 3924 * 3925 * If a link status change event occurs, phy.ops.handle_lasi() will 3926 * automatically initiate a link setup between the integrated KR PHY 3927 * and the external X557 PHY to ensure that the link speed between 3928 * them matches the link speed of the baseT link. 3929 */ 3930 rv = ixgbe_handle_lasi(hw); 3931 3932 if (rv == IXGBE_ERR_OVERTEMP) { 3933 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3934 3935 /* 3936 * Disable the adapter interrupts 3937 */ 3938 ixgbe_disable_adapter_interrupts(ixgbe); 3939 3940 /* 3941 * Disable Rx/Tx units 3942 */ 3943 (void) ixgbe_stop_adapter(hw); 3944 3945 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3946 ixgbe_error(ixgbe, 3947 "Problem: Network adapter has been stopped due to a " 3948 "overtemperature event being detected."); 3949 ixgbe_error(ixgbe, 3950 "Action: Shut down or restart the computer. If the issue " 3951 "persists, please take action in accordance with the " 3952 "recommendations from your system vendor."); 3953 } 3954 3955 mutex_exit(&ixgbe->gen_lock); 3956 } 3957 3958 /* 3959 * ixgbe_link_timer - timer for link status detection 3960 */ 3961 static void 3962 ixgbe_link_timer(void *arg) 3963 { 3964 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3965 3966 mutex_enter(&ixgbe->gen_lock); 3967 ixgbe_driver_link_check(ixgbe); 3968 mutex_exit(&ixgbe->gen_lock); 3969 } 3970 3971 /* 3972 * ixgbe_local_timer - Driver watchdog function. 3973 * 3974 * This function will handle the transmit stall check and other routines. 3975 */ 3976 static void 3977 ixgbe_local_timer(void *arg) 3978 { 3979 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3980 3981 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP) 3982 goto out; 3983 3984 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3985 ixgbe->reset_count++; 3986 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3987 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3988 goto out; 3989 } 3990 3991 if (ixgbe_stall_check(ixgbe)) { 3992 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3993 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3994 3995 ixgbe->reset_count++; 3996 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3997 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3998 } 3999 4000 out: 4001 ixgbe_restart_watchdog_timer(ixgbe); 4002 } 4003 4004 /* 4005 * ixgbe_stall_check - Check for transmit stall. 4006 * 4007 * This function checks if the adapter is stalled (in transmit). 4008 * 4009 * It is called each time the watchdog timeout is invoked. 4010 * If the transmit descriptor reclaim continuously fails, 4011 * the watchdog value will increment by 1. If the watchdog 4012 * value exceeds the threshold, the ixgbe is assumed to 4013 * have stalled and need to be reset. 4014 */ 4015 static boolean_t 4016 ixgbe_stall_check(ixgbe_t *ixgbe) 4017 { 4018 ixgbe_tx_ring_t *tx_ring; 4019 boolean_t result; 4020 int i; 4021 4022 if (ixgbe->link_state != LINK_STATE_UP) 4023 return (B_FALSE); 4024 4025 /* 4026 * If any tx ring is stalled, we'll reset the chipset 4027 */ 4028 result = B_FALSE; 4029 for (i = 0; i < ixgbe->num_tx_rings; i++) { 4030 tx_ring = &ixgbe->tx_rings[i]; 4031 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 4032 tx_ring->tx_recycle(tx_ring); 4033 } 4034 4035 if (tx_ring->recycle_fail > 0) 4036 tx_ring->stall_watchdog++; 4037 else 4038 tx_ring->stall_watchdog = 0; 4039 4040 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 4041 result = B_TRUE; 4042 break; 4043 } 4044 } 4045 4046 if (result) { 4047 tx_ring->stall_watchdog = 0; 4048 tx_ring->recycle_fail = 0; 4049 } 4050 4051 return (result); 4052 } 4053 4054 4055 /* 4056 * is_valid_mac_addr - Check if the mac address is valid. 4057 */ 4058 static boolean_t 4059 is_valid_mac_addr(uint8_t *mac_addr) 4060 { 4061 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 4062 const uint8_t addr_test2[6] = 4063 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 4064 4065 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 4066 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 4067 return (B_FALSE); 4068 4069 return (B_TRUE); 4070 } 4071 4072 static boolean_t 4073 ixgbe_find_mac_address(ixgbe_t *ixgbe) 4074 { 4075 #ifdef __sparc 4076 struct ixgbe_hw *hw = &ixgbe->hw; 4077 uchar_t *bytes; 4078 struct ether_addr sysaddr; 4079 uint_t nelts; 4080 int err; 4081 boolean_t found = B_FALSE; 4082 4083 /* 4084 * The "vendor's factory-set address" may already have 4085 * been extracted from the chip, but if the property 4086 * "local-mac-address" is set we use that instead. 4087 * 4088 * We check whether it looks like an array of 6 4089 * bytes (which it should, if OBP set it). If we can't 4090 * make sense of it this way, we'll ignore it. 4091 */ 4092 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 4093 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 4094 if (err == DDI_PROP_SUCCESS) { 4095 if (nelts == ETHERADDRL) { 4096 while (nelts--) 4097 hw->mac.addr[nelts] = bytes[nelts]; 4098 found = B_TRUE; 4099 } 4100 ddi_prop_free(bytes); 4101 } 4102 4103 /* 4104 * Look up the OBP property "local-mac-address?". If the user has set 4105 * 'local-mac-address? = false', use "the system address" instead. 4106 */ 4107 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 4108 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 4109 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 4110 if (localetheraddr(NULL, &sysaddr) != 0) { 4111 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 4112 found = B_TRUE; 4113 } 4114 } 4115 ddi_prop_free(bytes); 4116 } 4117 4118 /* 4119 * Finally(!), if there's a valid "mac-address" property (created 4120 * if we netbooted from this interface), we must use this instead 4121 * of any of the above to ensure that the NFS/install server doesn't 4122 * get confused by the address changing as illumos takes over! 4123 */ 4124 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 4125 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 4126 if (err == DDI_PROP_SUCCESS) { 4127 if (nelts == ETHERADDRL) { 4128 while (nelts--) 4129 hw->mac.addr[nelts] = bytes[nelts]; 4130 found = B_TRUE; 4131 } 4132 ddi_prop_free(bytes); 4133 } 4134 4135 if (found) { 4136 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 4137 return (B_TRUE); 4138 } 4139 #else 4140 _NOTE(ARGUNUSED(ixgbe)); 4141 #endif 4142 4143 return (B_TRUE); 4144 } 4145 4146 #pragma inline(ixgbe_arm_watchdog_timer) 4147 static void 4148 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 4149 { 4150 /* 4151 * Fire a watchdog timer 4152 */ 4153 ixgbe->watchdog_tid = 4154 timeout(ixgbe_local_timer, 4155 (void *)ixgbe, 1 * drv_usectohz(1000000)); 4156 4157 } 4158 4159 /* 4160 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 4161 */ 4162 void 4163 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 4164 { 4165 mutex_enter(&ixgbe->watchdog_lock); 4166 4167 if (!ixgbe->watchdog_enable) { 4168 ixgbe->watchdog_enable = B_TRUE; 4169 ixgbe->watchdog_start = B_TRUE; 4170 ixgbe_arm_watchdog_timer(ixgbe); 4171 } 4172 4173 mutex_exit(&ixgbe->watchdog_lock); 4174 } 4175 4176 /* 4177 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 4178 */ 4179 void 4180 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 4181 { 4182 timeout_id_t tid; 4183 4184 mutex_enter(&ixgbe->watchdog_lock); 4185 4186 ixgbe->watchdog_enable = B_FALSE; 4187 ixgbe->watchdog_start = B_FALSE; 4188 tid = ixgbe->watchdog_tid; 4189 ixgbe->watchdog_tid = 0; 4190 4191 mutex_exit(&ixgbe->watchdog_lock); 4192 4193 if (tid != 0) 4194 (void) untimeout(tid); 4195 } 4196 4197 /* 4198 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 4199 */ 4200 void 4201 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 4202 { 4203 mutex_enter(&ixgbe->watchdog_lock); 4204 4205 if (ixgbe->watchdog_enable) { 4206 if (!ixgbe->watchdog_start) { 4207 ixgbe->watchdog_start = B_TRUE; 4208 ixgbe_arm_watchdog_timer(ixgbe); 4209 } 4210 } 4211 4212 mutex_exit(&ixgbe->watchdog_lock); 4213 } 4214 4215 /* 4216 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 4217 */ 4218 static void 4219 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 4220 { 4221 mutex_enter(&ixgbe->watchdog_lock); 4222 4223 if (ixgbe->watchdog_start) 4224 ixgbe_arm_watchdog_timer(ixgbe); 4225 4226 mutex_exit(&ixgbe->watchdog_lock); 4227 } 4228 4229 /* 4230 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 4231 */ 4232 void 4233 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 4234 { 4235 timeout_id_t tid; 4236 4237 mutex_enter(&ixgbe->watchdog_lock); 4238 4239 ixgbe->watchdog_start = B_FALSE; 4240 tid = ixgbe->watchdog_tid; 4241 ixgbe->watchdog_tid = 0; 4242 4243 mutex_exit(&ixgbe->watchdog_lock); 4244 4245 if (tid != 0) 4246 (void) untimeout(tid); 4247 } 4248 4249 /* 4250 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 4251 */ 4252 static void 4253 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 4254 { 4255 struct ixgbe_hw *hw = &ixgbe->hw; 4256 4257 /* 4258 * mask all interrupts off 4259 */ 4260 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 4261 4262 /* 4263 * for MSI-X, also disable autoclear 4264 */ 4265 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 4266 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 4267 } 4268 4269 IXGBE_WRITE_FLUSH(hw); 4270 } 4271 4272 /* 4273 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 4274 */ 4275 static void 4276 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 4277 { 4278 struct ixgbe_hw *hw = &ixgbe->hw; 4279 uint32_t eiac, eiam; 4280 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 4281 4282 /* interrupt types to enable */ 4283 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 4284 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 4285 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 4286 4287 /* enable automask on "other" causes that this adapter can generate */ 4288 eiam = ixgbe->capab->other_intr; 4289 4290 /* 4291 * msi-x mode 4292 */ 4293 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 4294 /* enable autoclear but not on bits 29:20 */ 4295 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 4296 4297 /* general purpose interrupt enable */ 4298 gpie |= (IXGBE_GPIE_MSIX_MODE 4299 | IXGBE_GPIE_PBA_SUPPORT 4300 | IXGBE_GPIE_OCD 4301 | IXGBE_GPIE_EIAME); 4302 /* 4303 * non-msi-x mode 4304 */ 4305 } else { 4306 4307 /* disable autoclear, leave gpie at default */ 4308 eiac = 0; 4309 4310 /* 4311 * General purpose interrupt enable. 4312 * For 82599, X540 and X550, extended interrupt 4313 * automask enable only in MSI or MSI-X mode 4314 */ 4315 if ((hw->mac.type == ixgbe_mac_82598EB) || 4316 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 4317 gpie |= IXGBE_GPIE_EIAME; 4318 } 4319 } 4320 4321 /* Enable specific "other" interrupt types */ 4322 switch (hw->mac.type) { 4323 case ixgbe_mac_82598EB: 4324 gpie |= ixgbe->capab->other_gpie; 4325 break; 4326 4327 case ixgbe_mac_82599EB: 4328 case ixgbe_mac_X540: 4329 case ixgbe_mac_X550: 4330 case ixgbe_mac_X550EM_x: 4331 case ixgbe_mac_X550EM_a: 4332 gpie |= ixgbe->capab->other_gpie; 4333 4334 /* Enable RSC Delay 8us when LRO enabled */ 4335 if (ixgbe->lro_enable) { 4336 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 4337 } 4338 break; 4339 4340 default: 4341 break; 4342 } 4343 4344 /* write to interrupt control registers */ 4345 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4346 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 4347 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 4348 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 4349 IXGBE_WRITE_FLUSH(hw); 4350 } 4351 4352 /* 4353 * ixgbe_loopback_ioctl - Loopback support. 4354 */ 4355 enum ioc_reply 4356 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 4357 { 4358 lb_info_sz_t *lbsp; 4359 lb_property_t *lbpp; 4360 uint32_t *lbmp; 4361 uint32_t size; 4362 uint32_t value; 4363 4364 if (mp->b_cont == NULL) 4365 return (IOC_INVAL); 4366 4367 switch (iocp->ioc_cmd) { 4368 default: 4369 return (IOC_INVAL); 4370 4371 case LB_GET_INFO_SIZE: 4372 size = sizeof (lb_info_sz_t); 4373 if (iocp->ioc_count != size) 4374 return (IOC_INVAL); 4375 4376 value = sizeof (lb_normal); 4377 value += sizeof (lb_mac); 4378 value += sizeof (lb_external); 4379 4380 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 4381 *lbsp = value; 4382 break; 4383 4384 case LB_GET_INFO: 4385 value = sizeof (lb_normal); 4386 value += sizeof (lb_mac); 4387 value += sizeof (lb_external); 4388 4389 size = value; 4390 if (iocp->ioc_count != size) 4391 return (IOC_INVAL); 4392 4393 value = 0; 4394 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 4395 4396 lbpp[value++] = lb_normal; 4397 lbpp[value++] = lb_mac; 4398 lbpp[value++] = lb_external; 4399 break; 4400 4401 case LB_GET_MODE: 4402 size = sizeof (uint32_t); 4403 if (iocp->ioc_count != size) 4404 return (IOC_INVAL); 4405 4406 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 4407 *lbmp = ixgbe->loopback_mode; 4408 break; 4409 4410 case LB_SET_MODE: 4411 size = 0; 4412 if (iocp->ioc_count != sizeof (uint32_t)) 4413 return (IOC_INVAL); 4414 4415 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 4416 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 4417 return (IOC_INVAL); 4418 break; 4419 } 4420 4421 iocp->ioc_count = size; 4422 iocp->ioc_error = 0; 4423 4424 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4425 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4426 return (IOC_INVAL); 4427 } 4428 4429 return (IOC_REPLY); 4430 } 4431 4432 /* 4433 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 4434 */ 4435 static boolean_t 4436 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 4437 { 4438 if (mode == ixgbe->loopback_mode) 4439 return (B_TRUE); 4440 4441 ixgbe->loopback_mode = mode; 4442 4443 if (mode == IXGBE_LB_NONE) { 4444 /* 4445 * Reset the chip 4446 */ 4447 (void) ixgbe_reset(ixgbe); 4448 return (B_TRUE); 4449 } 4450 4451 mutex_enter(&ixgbe->gen_lock); 4452 4453 switch (mode) { 4454 default: 4455 mutex_exit(&ixgbe->gen_lock); 4456 return (B_FALSE); 4457 4458 case IXGBE_LB_EXTERNAL: 4459 break; 4460 4461 case IXGBE_LB_INTERNAL_MAC: 4462 ixgbe_set_internal_mac_loopback(ixgbe); 4463 break; 4464 } 4465 4466 mutex_exit(&ixgbe->gen_lock); 4467 4468 return (B_TRUE); 4469 } 4470 4471 /* 4472 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 4473 */ 4474 static void 4475 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 4476 { 4477 struct ixgbe_hw *hw; 4478 uint32_t reg; 4479 uint8_t atlas; 4480 4481 hw = &ixgbe->hw; 4482 4483 /* 4484 * Setup MAC loopback 4485 */ 4486 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 4487 reg |= IXGBE_HLREG0_LPBK; 4488 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 4489 4490 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4491 reg &= ~IXGBE_AUTOC_LMS_MASK; 4492 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4493 4494 /* 4495 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 4496 */ 4497 switch (hw->mac.type) { 4498 case ixgbe_mac_82598EB: 4499 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4500 &atlas); 4501 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 4502 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4503 atlas); 4504 4505 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4506 &atlas); 4507 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 4508 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4509 atlas); 4510 4511 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4512 &atlas); 4513 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 4514 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4515 atlas); 4516 4517 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4518 &atlas); 4519 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 4520 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4521 atlas); 4522 break; 4523 4524 case ixgbe_mac_82599EB: 4525 case ixgbe_mac_X540: 4526 case ixgbe_mac_X550: 4527 case ixgbe_mac_X550EM_x: 4528 case ixgbe_mac_X550EM_a: 4529 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4530 reg |= (IXGBE_AUTOC_FLU | 4531 IXGBE_AUTOC_10G_KX4); 4532 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4533 4534 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL, 4535 B_FALSE); 4536 break; 4537 4538 default: 4539 break; 4540 } 4541 } 4542 4543 #pragma inline(ixgbe_intr_rx_work) 4544 /* 4545 * ixgbe_intr_rx_work - RX processing of ISR. 4546 */ 4547 static void 4548 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 4549 { 4550 mblk_t *mp; 4551 4552 mutex_enter(&rx_ring->rx_lock); 4553 4554 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4555 mutex_exit(&rx_ring->rx_lock); 4556 4557 if (mp != NULL) 4558 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4559 rx_ring->ring_gen_num); 4560 } 4561 4562 #pragma inline(ixgbe_intr_tx_work) 4563 /* 4564 * ixgbe_intr_tx_work - TX processing of ISR. 4565 */ 4566 static void 4567 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 4568 { 4569 ixgbe_t *ixgbe = tx_ring->ixgbe; 4570 4571 /* 4572 * Recycle the tx descriptors 4573 */ 4574 tx_ring->tx_recycle(tx_ring); 4575 4576 /* 4577 * Schedule the re-transmit 4578 */ 4579 if (tx_ring->reschedule && 4580 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 4581 tx_ring->reschedule = B_FALSE; 4582 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 4583 tx_ring->ring_handle); 4584 tx_ring->stat_reschedule++; 4585 } 4586 } 4587 4588 #pragma inline(ixgbe_intr_other_work) 4589 /* 4590 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 4591 */ 4592 static void 4593 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 4594 { 4595 struct ixgbe_hw *hw = &ixgbe->hw; 4596 4597 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4598 4599 /* 4600 * handle link status change 4601 */ 4602 if (eicr & IXGBE_EICR_LSC) { 4603 ixgbe_driver_link_check(ixgbe); 4604 ixgbe_get_hw_state(ixgbe); 4605 } 4606 4607 /* 4608 * check for fan failure on adapters with fans 4609 */ 4610 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 4611 (eicr & IXGBE_EICR_GPI_SDP1)) { 4612 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 4613 4614 /* 4615 * Disable the adapter interrupts 4616 */ 4617 ixgbe_disable_adapter_interrupts(ixgbe); 4618 4619 /* 4620 * Disable Rx/Tx units 4621 */ 4622 (void) ixgbe_stop_adapter(&ixgbe->hw); 4623 4624 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 4625 ixgbe_error(ixgbe, 4626 "Problem: Network adapter has been stopped " 4627 "because the fan has stopped.\n"); 4628 ixgbe_error(ixgbe, 4629 "Action: Replace the adapter.\n"); 4630 4631 /* re-enable the interrupt, which was automasked */ 4632 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 4633 } 4634 4635 /* 4636 * Do SFP check for adapters with hot-plug capability 4637 */ 4638 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) && 4639 ((eicr & IXGBE_EICR_GPI_SDP1_BY_MAC(hw)) || 4640 (eicr & IXGBE_EICR_GPI_SDP2_BY_MAC(hw)))) { 4641 ixgbe->eicr = eicr; 4642 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 4643 ixgbe_sfp_check, (void *)ixgbe, 4644 DDI_NOSLEEP)) != DDI_SUCCESS) { 4645 ixgbe_log(ixgbe, "No memory available to dispatch " 4646 "taskq for SFP check"); 4647 } 4648 } 4649 4650 /* 4651 * Do over-temperature check for adapters with temp sensor 4652 */ 4653 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) && 4654 ((eicr & IXGBE_EICR_GPI_SDP0_BY_MAC(hw)) || 4655 (eicr & IXGBE_EICR_LSC))) { 4656 ixgbe->eicr = eicr; 4657 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq, 4658 ixgbe_overtemp_check, (void *)ixgbe, 4659 DDI_NOSLEEP)) != DDI_SUCCESS) { 4660 ixgbe_log(ixgbe, "No memory available to dispatch " 4661 "taskq for overtemp check"); 4662 } 4663 } 4664 4665 /* 4666 * Process an external PHY interrupt 4667 */ 4668 if (hw->device_id == IXGBE_DEV_ID_X550EM_X_10G_T && 4669 (eicr & IXGBE_EICR_GPI_SDP0_X540)) { 4670 ixgbe->eicr = eicr; 4671 if ((ddi_taskq_dispatch(ixgbe->phy_taskq, 4672 ixgbe_phy_check, (void *)ixgbe, 4673 DDI_NOSLEEP)) != DDI_SUCCESS) { 4674 ixgbe_log(ixgbe, "No memory available to dispatch " 4675 "taskq for PHY check"); 4676 } 4677 } 4678 } 4679 4680 /* 4681 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 4682 */ 4683 static uint_t 4684 ixgbe_intr_legacy(void *arg1, void *arg2) 4685 { 4686 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4687 struct ixgbe_hw *hw = &ixgbe->hw; 4688 ixgbe_tx_ring_t *tx_ring; 4689 ixgbe_rx_ring_t *rx_ring; 4690 uint32_t eicr; 4691 mblk_t *mp; 4692 boolean_t tx_reschedule; 4693 uint_t result; 4694 4695 _NOTE(ARGUNUSED(arg2)); 4696 4697 mutex_enter(&ixgbe->gen_lock); 4698 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4699 mutex_exit(&ixgbe->gen_lock); 4700 return (DDI_INTR_UNCLAIMED); 4701 } 4702 4703 mp = NULL; 4704 tx_reschedule = B_FALSE; 4705 4706 /* 4707 * Any bit set in eicr: claim this interrupt 4708 */ 4709 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4710 4711 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4712 mutex_exit(&ixgbe->gen_lock); 4713 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4714 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4715 return (DDI_INTR_CLAIMED); 4716 } 4717 4718 if (eicr) { 4719 /* 4720 * For legacy interrupt, we have only one interrupt, 4721 * so we have only one rx ring and one tx ring enabled. 4722 */ 4723 ASSERT(ixgbe->num_rx_rings == 1); 4724 ASSERT(ixgbe->num_tx_rings == 1); 4725 4726 /* 4727 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 4728 */ 4729 if (eicr & 0x1) { 4730 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 4731 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4732 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4733 /* 4734 * Clean the rx descriptors 4735 */ 4736 rx_ring = &ixgbe->rx_rings[0]; 4737 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4738 } 4739 4740 /* 4741 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 4742 */ 4743 if (eicr & 0x2) { 4744 /* 4745 * Recycle the tx descriptors 4746 */ 4747 tx_ring = &ixgbe->tx_rings[0]; 4748 tx_ring->tx_recycle(tx_ring); 4749 4750 /* 4751 * Schedule the re-transmit 4752 */ 4753 tx_reschedule = (tx_ring->reschedule && 4754 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 4755 } 4756 4757 /* any interrupt type other than tx/rx */ 4758 if (eicr & ixgbe->capab->other_intr) { 4759 switch (hw->mac.type) { 4760 case ixgbe_mac_82598EB: 4761 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4762 break; 4763 4764 case ixgbe_mac_82599EB: 4765 case ixgbe_mac_X540: 4766 case ixgbe_mac_X550: 4767 case ixgbe_mac_X550EM_x: 4768 case ixgbe_mac_X550EM_a: 4769 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4770 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4771 break; 4772 4773 default: 4774 break; 4775 } 4776 ixgbe_intr_other_work(ixgbe, eicr); 4777 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4778 } 4779 4780 mutex_exit(&ixgbe->gen_lock); 4781 4782 result = DDI_INTR_CLAIMED; 4783 } else { 4784 mutex_exit(&ixgbe->gen_lock); 4785 4786 /* 4787 * No interrupt cause bits set: don't claim this interrupt. 4788 */ 4789 result = DDI_INTR_UNCLAIMED; 4790 } 4791 4792 /* re-enable the interrupts which were automasked */ 4793 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4794 4795 /* 4796 * Do the following work outside of the gen_lock 4797 */ 4798 if (mp != NULL) { 4799 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4800 rx_ring->ring_gen_num); 4801 } 4802 4803 if (tx_reschedule) { 4804 tx_ring->reschedule = B_FALSE; 4805 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 4806 tx_ring->stat_reschedule++; 4807 } 4808 4809 return (result); 4810 } 4811 4812 /* 4813 * ixgbe_intr_msi - Interrupt handler for MSI. 4814 */ 4815 static uint_t 4816 ixgbe_intr_msi(void *arg1, void *arg2) 4817 { 4818 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4819 struct ixgbe_hw *hw = &ixgbe->hw; 4820 uint32_t eicr; 4821 4822 _NOTE(ARGUNUSED(arg2)); 4823 4824 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4825 4826 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4827 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4828 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4829 return (DDI_INTR_CLAIMED); 4830 } 4831 4832 /* 4833 * For MSI interrupt, we have only one vector, 4834 * so we have only one rx ring and one tx ring enabled. 4835 */ 4836 ASSERT(ixgbe->num_rx_rings == 1); 4837 ASSERT(ixgbe->num_tx_rings == 1); 4838 4839 /* 4840 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4841 */ 4842 if (eicr & 0x1) { 4843 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4844 } 4845 4846 /* 4847 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4848 */ 4849 if (eicr & 0x2) { 4850 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4851 } 4852 4853 /* any interrupt type other than tx/rx */ 4854 if (eicr & ixgbe->capab->other_intr) { 4855 mutex_enter(&ixgbe->gen_lock); 4856 switch (hw->mac.type) { 4857 case ixgbe_mac_82598EB: 4858 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4859 break; 4860 4861 case ixgbe_mac_82599EB: 4862 case ixgbe_mac_X540: 4863 case ixgbe_mac_X550: 4864 case ixgbe_mac_X550EM_x: 4865 case ixgbe_mac_X550EM_a: 4866 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4867 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4868 break; 4869 4870 default: 4871 break; 4872 } 4873 ixgbe_intr_other_work(ixgbe, eicr); 4874 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4875 mutex_exit(&ixgbe->gen_lock); 4876 } 4877 4878 /* re-enable the interrupts which were automasked */ 4879 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4880 4881 return (DDI_INTR_CLAIMED); 4882 } 4883 4884 /* 4885 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4886 */ 4887 static uint_t 4888 ixgbe_intr_msix(void *arg1, void *arg2) 4889 { 4890 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4891 ixgbe_t *ixgbe = vect->ixgbe; 4892 struct ixgbe_hw *hw = &ixgbe->hw; 4893 uint32_t eicr; 4894 int r_idx = 0; 4895 4896 _NOTE(ARGUNUSED(arg2)); 4897 4898 /* 4899 * Clean each rx ring that has its bit set in the map 4900 */ 4901 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4902 while (r_idx >= 0) { 4903 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4904 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4905 (ixgbe->num_rx_rings - 1)); 4906 } 4907 4908 /* 4909 * Clean each tx ring that has its bit set in the map 4910 */ 4911 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4912 while (r_idx >= 0) { 4913 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4914 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4915 (ixgbe->num_tx_rings - 1)); 4916 } 4917 4918 4919 /* 4920 * Clean other interrupt (link change) that has its bit set in the map 4921 */ 4922 if (BT_TEST(vect->other_map, 0) == 1) { 4923 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4924 4925 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4926 DDI_FM_OK) { 4927 ddi_fm_service_impact(ixgbe->dip, 4928 DDI_SERVICE_DEGRADED); 4929 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4930 return (DDI_INTR_CLAIMED); 4931 } 4932 4933 /* 4934 * Check "other" cause bits: any interrupt type other than tx/rx 4935 */ 4936 if (eicr & ixgbe->capab->other_intr) { 4937 mutex_enter(&ixgbe->gen_lock); 4938 switch (hw->mac.type) { 4939 case ixgbe_mac_82598EB: 4940 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4941 ixgbe_intr_other_work(ixgbe, eicr); 4942 break; 4943 4944 case ixgbe_mac_82599EB: 4945 case ixgbe_mac_X540: 4946 case ixgbe_mac_X550: 4947 case ixgbe_mac_X550EM_x: 4948 case ixgbe_mac_X550EM_a: 4949 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4950 ixgbe_intr_other_work(ixgbe, eicr); 4951 break; 4952 4953 default: 4954 break; 4955 } 4956 mutex_exit(&ixgbe->gen_lock); 4957 } 4958 4959 /* re-enable the interrupts which were automasked */ 4960 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4961 } 4962 4963 return (DDI_INTR_CLAIMED); 4964 } 4965 4966 /* 4967 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4968 * 4969 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4970 * if not successful, try Legacy. 4971 * ixgbe->intr_force can be used to force sequence to start with 4972 * any of the 3 types. 4973 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4974 */ 4975 static int 4976 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4977 { 4978 dev_info_t *devinfo; 4979 int intr_types; 4980 int rc; 4981 4982 devinfo = ixgbe->dip; 4983 4984 /* 4985 * Get supported interrupt types 4986 */ 4987 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4988 4989 if (rc != DDI_SUCCESS) { 4990 ixgbe_log(ixgbe, 4991 "Get supported interrupt types failed: %d", rc); 4992 return (IXGBE_FAILURE); 4993 } 4994 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4995 4996 ixgbe->intr_type = 0; 4997 4998 /* 4999 * Install MSI-X interrupts 5000 */ 5001 if ((intr_types & DDI_INTR_TYPE_MSIX) && 5002 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 5003 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 5004 if (rc == IXGBE_SUCCESS) 5005 return (IXGBE_SUCCESS); 5006 5007 ixgbe_log(ixgbe, 5008 "Allocate MSI-X failed, trying MSI interrupts..."); 5009 } 5010 5011 /* 5012 * MSI-X not used, force rings and groups to 1 5013 */ 5014 ixgbe->num_rx_rings = 1; 5015 ixgbe->num_rx_groups = 1; 5016 ixgbe->num_tx_rings = 1; 5017 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 5018 ixgbe_log(ixgbe, 5019 "MSI-X not used, force rings and groups number to 1"); 5020 5021 /* 5022 * Install MSI interrupts 5023 */ 5024 if ((intr_types & DDI_INTR_TYPE_MSI) && 5025 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 5026 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 5027 if (rc == IXGBE_SUCCESS) 5028 return (IXGBE_SUCCESS); 5029 5030 ixgbe_log(ixgbe, 5031 "Allocate MSI failed, trying Legacy interrupts..."); 5032 } 5033 5034 /* 5035 * Install legacy interrupts 5036 */ 5037 if (intr_types & DDI_INTR_TYPE_FIXED) { 5038 /* 5039 * Disallow legacy interrupts for X550. X550 has a silicon 5040 * bug which prevents Shared Legacy interrupts from working. 5041 * For details, please reference: 5042 * 5043 * Intel Ethernet Controller X550 Specification Update rev. 2.1 5044 * May 2016, erratum 22: PCIe Interrupt Status Bit 5045 */ 5046 if (ixgbe->hw.mac.type == ixgbe_mac_X550 || 5047 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x || 5048 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a || 5049 ixgbe->hw.mac.type == ixgbe_mac_X550_vf || 5050 ixgbe->hw.mac.type == ixgbe_mac_X550EM_x_vf || 5051 ixgbe->hw.mac.type == ixgbe_mac_X550EM_a_vf) { 5052 ixgbe_log(ixgbe, 5053 "Legacy interrupts are not supported on this " 5054 "adapter. Please use MSI or MSI-X instead."); 5055 return (IXGBE_FAILURE); 5056 } 5057 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 5058 if (rc == IXGBE_SUCCESS) 5059 return (IXGBE_SUCCESS); 5060 5061 ixgbe_log(ixgbe, 5062 "Allocate Legacy interrupts failed"); 5063 } 5064 5065 /* 5066 * If none of the 3 types succeeded, return failure 5067 */ 5068 return (IXGBE_FAILURE); 5069 } 5070 5071 /* 5072 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 5073 * 5074 * For legacy and MSI, only 1 handle is needed. For MSI-X, 5075 * if fewer than 2 handles are available, return failure. 5076 * Upon success, this maps the vectors to rx and tx rings for 5077 * interrupts. 5078 */ 5079 static int 5080 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 5081 { 5082 dev_info_t *devinfo; 5083 int request, count, actual; 5084 int minimum; 5085 int rc; 5086 uint32_t ring_per_group; 5087 5088 devinfo = ixgbe->dip; 5089 5090 switch (intr_type) { 5091 case DDI_INTR_TYPE_FIXED: 5092 request = 1; /* Request 1 legacy interrupt handle */ 5093 minimum = 1; 5094 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 5095 break; 5096 5097 case DDI_INTR_TYPE_MSI: 5098 request = 1; /* Request 1 MSI interrupt handle */ 5099 minimum = 1; 5100 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 5101 break; 5102 5103 case DDI_INTR_TYPE_MSIX: 5104 /* 5105 * Best number of vectors for the adapter is 5106 * (# rx rings + # tx rings), however we will 5107 * limit the request number. 5108 */ 5109 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 5110 if (request > ixgbe->capab->max_ring_vect) 5111 request = ixgbe->capab->max_ring_vect; 5112 minimum = 1; 5113 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 5114 break; 5115 5116 default: 5117 ixgbe_log(ixgbe, 5118 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 5119 intr_type); 5120 return (IXGBE_FAILURE); 5121 } 5122 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 5123 request, minimum); 5124 5125 /* 5126 * Get number of supported interrupts 5127 */ 5128 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 5129 if ((rc != DDI_SUCCESS) || (count < minimum)) { 5130 ixgbe_log(ixgbe, 5131 "Get interrupt number failed. Return: %d, count: %d", 5132 rc, count); 5133 return (IXGBE_FAILURE); 5134 } 5135 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 5136 5137 actual = 0; 5138 ixgbe->intr_cnt = 0; 5139 ixgbe->intr_cnt_max = 0; 5140 ixgbe->intr_cnt_min = 0; 5141 5142 /* 5143 * Allocate an array of interrupt handles 5144 */ 5145 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 5146 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 5147 5148 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 5149 request, &actual, DDI_INTR_ALLOC_NORMAL); 5150 if (rc != DDI_SUCCESS) { 5151 ixgbe_log(ixgbe, "Allocate interrupts failed. " 5152 "return: %d, request: %d, actual: %d", 5153 rc, request, actual); 5154 goto alloc_handle_fail; 5155 } 5156 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 5157 5158 /* 5159 * upper/lower limit of interrupts 5160 */ 5161 ixgbe->intr_cnt = actual; 5162 ixgbe->intr_cnt_max = request; 5163 ixgbe->intr_cnt_min = minimum; 5164 5165 /* 5166 * rss number per group should not exceed the rx interrupt number, 5167 * else need to adjust rx ring number. 5168 */ 5169 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 5170 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 5171 if (actual < ring_per_group) { 5172 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual; 5173 ixgbe_setup_vmdq_rss_conf(ixgbe); 5174 } 5175 5176 /* 5177 * Now we know the actual number of vectors. Here we map the vector 5178 * to other, rx rings and tx ring. 5179 */ 5180 if (actual < minimum) { 5181 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 5182 actual); 5183 goto alloc_handle_fail; 5184 } 5185 5186 /* 5187 * Get priority for first vector, assume remaining are all the same 5188 */ 5189 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 5190 if (rc != DDI_SUCCESS) { 5191 ixgbe_log(ixgbe, 5192 "Get interrupt priority failed: %d", rc); 5193 goto alloc_handle_fail; 5194 } 5195 5196 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 5197 if (rc != DDI_SUCCESS) { 5198 ixgbe_log(ixgbe, 5199 "Get interrupt cap failed: %d", rc); 5200 goto alloc_handle_fail; 5201 } 5202 5203 ixgbe->intr_type = intr_type; 5204 5205 return (IXGBE_SUCCESS); 5206 5207 alloc_handle_fail: 5208 ixgbe_rem_intrs(ixgbe); 5209 5210 return (IXGBE_FAILURE); 5211 } 5212 5213 /* 5214 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 5215 * 5216 * Before adding the interrupt handlers, the interrupt vectors have 5217 * been allocated, and the rx/tx rings have also been allocated. 5218 */ 5219 static int 5220 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 5221 { 5222 int vector = 0; 5223 int rc; 5224 5225 switch (ixgbe->intr_type) { 5226 case DDI_INTR_TYPE_MSIX: 5227 /* 5228 * Add interrupt handler for all vectors 5229 */ 5230 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 5231 /* 5232 * install pointer to vect_map[vector] 5233 */ 5234 rc = ddi_intr_add_handler(ixgbe->htable[vector], 5235 (ddi_intr_handler_t *)ixgbe_intr_msix, 5236 (void *)&ixgbe->vect_map[vector], NULL); 5237 5238 if (rc != DDI_SUCCESS) { 5239 ixgbe_log(ixgbe, 5240 "Add interrupt handler failed. " 5241 "return: %d, vector: %d", rc, vector); 5242 for (vector--; vector >= 0; vector--) { 5243 (void) ddi_intr_remove_handler( 5244 ixgbe->htable[vector]); 5245 } 5246 return (IXGBE_FAILURE); 5247 } 5248 } 5249 5250 break; 5251 5252 case DDI_INTR_TYPE_MSI: 5253 /* 5254 * Add interrupt handlers for the only vector 5255 */ 5256 rc = ddi_intr_add_handler(ixgbe->htable[vector], 5257 (ddi_intr_handler_t *)ixgbe_intr_msi, 5258 (void *)ixgbe, NULL); 5259 5260 if (rc != DDI_SUCCESS) { 5261 ixgbe_log(ixgbe, 5262 "Add MSI interrupt handler failed: %d", rc); 5263 return (IXGBE_FAILURE); 5264 } 5265 5266 break; 5267 5268 case DDI_INTR_TYPE_FIXED: 5269 /* 5270 * Add interrupt handlers for the only vector 5271 */ 5272 rc = ddi_intr_add_handler(ixgbe->htable[vector], 5273 (ddi_intr_handler_t *)ixgbe_intr_legacy, 5274 (void *)ixgbe, NULL); 5275 5276 if (rc != DDI_SUCCESS) { 5277 ixgbe_log(ixgbe, 5278 "Add legacy interrupt handler failed: %d", rc); 5279 return (IXGBE_FAILURE); 5280 } 5281 5282 break; 5283 5284 default: 5285 return (IXGBE_FAILURE); 5286 } 5287 5288 return (IXGBE_SUCCESS); 5289 } 5290 5291 #pragma inline(ixgbe_map_rxring_to_vector) 5292 /* 5293 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 5294 */ 5295 static void 5296 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 5297 { 5298 /* 5299 * Set bit in map 5300 */ 5301 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5302 5303 /* 5304 * Count bits set 5305 */ 5306 ixgbe->vect_map[v_idx].rxr_cnt++; 5307 5308 /* 5309 * Remember bit position 5310 */ 5311 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 5312 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 5313 } 5314 5315 #pragma inline(ixgbe_map_txring_to_vector) 5316 /* 5317 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 5318 */ 5319 static void 5320 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 5321 { 5322 /* 5323 * Set bit in map 5324 */ 5325 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 5326 5327 /* 5328 * Count bits set 5329 */ 5330 ixgbe->vect_map[v_idx].txr_cnt++; 5331 5332 /* 5333 * Remember bit position 5334 */ 5335 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 5336 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 5337 } 5338 5339 /* 5340 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 5341 * allocation register (IVAR). 5342 * cause: 5343 * -1 : other cause 5344 * 0 : rx 5345 * 1 : tx 5346 */ 5347 static void 5348 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 5349 int8_t cause) 5350 { 5351 struct ixgbe_hw *hw = &ixgbe->hw; 5352 u32 ivar, index; 5353 5354 switch (hw->mac.type) { 5355 case ixgbe_mac_82598EB: 5356 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5357 if (cause == -1) { 5358 cause = 0; 5359 } 5360 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 5361 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5362 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 5363 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 5364 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 5365 break; 5366 5367 case ixgbe_mac_82599EB: 5368 case ixgbe_mac_X540: 5369 case ixgbe_mac_X550: 5370 case ixgbe_mac_X550EM_x: 5371 case ixgbe_mac_X550EM_a: 5372 if (cause == -1) { 5373 /* other causes */ 5374 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5375 index = (intr_alloc_entry & 1) * 8; 5376 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5377 ivar &= ~(0xFF << index); 5378 ivar |= (msix_vector << index); 5379 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5380 } else { 5381 /* tx or rx causes */ 5382 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 5383 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 5384 ivar = IXGBE_READ_REG(hw, 5385 IXGBE_IVAR(intr_alloc_entry >> 1)); 5386 ivar &= ~(0xFF << index); 5387 ivar |= (msix_vector << index); 5388 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 5389 ivar); 5390 } 5391 break; 5392 5393 default: 5394 break; 5395 } 5396 } 5397 5398 /* 5399 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 5400 * given interrupt vector allocation register (IVAR). 5401 * cause: 5402 * -1 : other cause 5403 * 0 : rx 5404 * 1 : tx 5405 */ 5406 static void 5407 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 5408 { 5409 struct ixgbe_hw *hw = &ixgbe->hw; 5410 u32 ivar, index; 5411 5412 switch (hw->mac.type) { 5413 case ixgbe_mac_82598EB: 5414 if (cause == -1) { 5415 cause = 0; 5416 } 5417 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 5418 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5419 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 5420 (intr_alloc_entry & 0x3))); 5421 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 5422 break; 5423 5424 case ixgbe_mac_82599EB: 5425 case ixgbe_mac_X540: 5426 case ixgbe_mac_X550: 5427 case ixgbe_mac_X550EM_x: 5428 case ixgbe_mac_X550EM_a: 5429 if (cause == -1) { 5430 /* other causes */ 5431 index = (intr_alloc_entry & 1) * 8; 5432 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5433 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 5434 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5435 } else { 5436 /* tx or rx causes */ 5437 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 5438 ivar = IXGBE_READ_REG(hw, 5439 IXGBE_IVAR(intr_alloc_entry >> 1)); 5440 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 5441 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 5442 ivar); 5443 } 5444 break; 5445 5446 default: 5447 break; 5448 } 5449 } 5450 5451 /* 5452 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 5453 * given interrupt vector allocation register (IVAR). 5454 * cause: 5455 * -1 : other cause 5456 * 0 : rx 5457 * 1 : tx 5458 */ 5459 static void 5460 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 5461 { 5462 struct ixgbe_hw *hw = &ixgbe->hw; 5463 u32 ivar, index; 5464 5465 switch (hw->mac.type) { 5466 case ixgbe_mac_82598EB: 5467 if (cause == -1) { 5468 cause = 0; 5469 } 5470 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 5471 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 5472 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 5473 (intr_alloc_entry & 0x3))); 5474 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 5475 break; 5476 5477 case ixgbe_mac_82599EB: 5478 case ixgbe_mac_X540: 5479 case ixgbe_mac_X550: 5480 case ixgbe_mac_X550EM_x: 5481 case ixgbe_mac_X550EM_a: 5482 if (cause == -1) { 5483 /* other causes */ 5484 index = (intr_alloc_entry & 1) * 8; 5485 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 5486 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 5487 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 5488 } else { 5489 /* tx or rx causes */ 5490 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 5491 ivar = IXGBE_READ_REG(hw, 5492 IXGBE_IVAR(intr_alloc_entry >> 1)); 5493 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 5494 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 5495 ivar); 5496 } 5497 break; 5498 5499 default: 5500 break; 5501 } 5502 } 5503 5504 /* 5505 * Convert the rx ring index driver maintained to the rx ring index 5506 * in h/w. 5507 */ 5508 static uint32_t 5509 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 5510 { 5511 5512 struct ixgbe_hw *hw = &ixgbe->hw; 5513 uint32_t rx_ring_per_group, hw_rx_index; 5514 5515 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 5516 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 5517 return (sw_rx_index); 5518 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 5519 switch (hw->mac.type) { 5520 case ixgbe_mac_82598EB: 5521 return (sw_rx_index); 5522 5523 case ixgbe_mac_82599EB: 5524 case ixgbe_mac_X540: 5525 case ixgbe_mac_X550: 5526 case ixgbe_mac_X550EM_x: 5527 case ixgbe_mac_X550EM_a: 5528 return (sw_rx_index * 2); 5529 5530 default: 5531 break; 5532 } 5533 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 5534 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 5535 5536 switch (hw->mac.type) { 5537 case ixgbe_mac_82598EB: 5538 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 5539 16 + (sw_rx_index % rx_ring_per_group); 5540 return (hw_rx_index); 5541 5542 case ixgbe_mac_82599EB: 5543 case ixgbe_mac_X540: 5544 case ixgbe_mac_X550: 5545 case ixgbe_mac_X550EM_x: 5546 case ixgbe_mac_X550EM_a: 5547 if (ixgbe->num_rx_groups > 32) { 5548 hw_rx_index = (sw_rx_index / 5549 rx_ring_per_group) * 2 + 5550 (sw_rx_index % rx_ring_per_group); 5551 } else { 5552 hw_rx_index = (sw_rx_index / 5553 rx_ring_per_group) * 4 + 5554 (sw_rx_index % rx_ring_per_group); 5555 } 5556 return (hw_rx_index); 5557 5558 default: 5559 break; 5560 } 5561 } 5562 5563 /* 5564 * Should never reach. Just to make compiler happy. 5565 */ 5566 return (sw_rx_index); 5567 } 5568 5569 /* 5570 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 5571 * 5572 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 5573 * to vector[0 - (intr_cnt -1)]. 5574 */ 5575 static int 5576 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 5577 { 5578 int i, vector = 0; 5579 5580 /* initialize vector map */ 5581 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 5582 for (i = 0; i < ixgbe->intr_cnt; i++) { 5583 ixgbe->vect_map[i].ixgbe = ixgbe; 5584 } 5585 5586 /* 5587 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 5588 * tx rings[0] on RTxQ[1]. 5589 */ 5590 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5591 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 5592 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 5593 return (IXGBE_SUCCESS); 5594 } 5595 5596 /* 5597 * Interrupts/vectors mapping for MSI-X 5598 */ 5599 5600 /* 5601 * Map other interrupt to vector 0, 5602 * Set bit in map and count the bits set. 5603 */ 5604 BT_SET(ixgbe->vect_map[vector].other_map, 0); 5605 ixgbe->vect_map[vector].other_cnt++; 5606 5607 /* 5608 * Map rx ring interrupts to vectors 5609 */ 5610 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5611 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 5612 vector = (vector +1) % ixgbe->intr_cnt; 5613 } 5614 5615 /* 5616 * Map tx ring interrupts to vectors 5617 */ 5618 for (i = 0; i < ixgbe->num_tx_rings; i++) { 5619 ixgbe_map_txring_to_vector(ixgbe, i, vector); 5620 vector = (vector +1) % ixgbe->intr_cnt; 5621 } 5622 5623 return (IXGBE_SUCCESS); 5624 } 5625 5626 /* 5627 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 5628 * 5629 * This relies on ring/vector mapping already set up in the 5630 * vect_map[] structures 5631 */ 5632 static void 5633 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 5634 { 5635 struct ixgbe_hw *hw = &ixgbe->hw; 5636 ixgbe_intr_vector_t *vect; /* vector bitmap */ 5637 int r_idx; /* ring index */ 5638 int v_idx; /* vector index */ 5639 uint32_t hw_index; 5640 5641 /* 5642 * Clear any previous entries 5643 */ 5644 switch (hw->mac.type) { 5645 case ixgbe_mac_82598EB: 5646 for (v_idx = 0; v_idx < 25; v_idx++) 5647 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5648 break; 5649 5650 case ixgbe_mac_82599EB: 5651 case ixgbe_mac_X540: 5652 case ixgbe_mac_X550: 5653 case ixgbe_mac_X550EM_x: 5654 case ixgbe_mac_X550EM_a: 5655 for (v_idx = 0; v_idx < 64; v_idx++) 5656 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5657 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 5658 break; 5659 5660 default: 5661 break; 5662 } 5663 5664 /* 5665 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 5666 * tx rings[0] will use RTxQ[1]. 5667 */ 5668 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5669 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 5670 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 5671 return; 5672 } 5673 5674 /* 5675 * For MSI-X interrupt, "Other" is always on vector[0]. 5676 */ 5677 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 5678 5679 /* 5680 * For each interrupt vector, populate the IVAR table 5681 */ 5682 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 5683 vect = &ixgbe->vect_map[v_idx]; 5684 5685 /* 5686 * For each rx ring bit set 5687 */ 5688 r_idx = bt_getlowbit(vect->rx_map, 0, 5689 (ixgbe->num_rx_rings - 1)); 5690 5691 while (r_idx >= 0) { 5692 hw_index = ixgbe->rx_rings[r_idx].hw_index; 5693 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 5694 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 5695 (ixgbe->num_rx_rings - 1)); 5696 } 5697 5698 /* 5699 * For each tx ring bit set 5700 */ 5701 r_idx = bt_getlowbit(vect->tx_map, 0, 5702 (ixgbe->num_tx_rings - 1)); 5703 5704 while (r_idx >= 0) { 5705 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 5706 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 5707 (ixgbe->num_tx_rings - 1)); 5708 } 5709 } 5710 } 5711 5712 /* 5713 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 5714 */ 5715 static void 5716 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 5717 { 5718 int i; 5719 int rc; 5720 5721 for (i = 0; i < ixgbe->intr_cnt; i++) { 5722 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 5723 if (rc != DDI_SUCCESS) { 5724 IXGBE_DEBUGLOG_1(ixgbe, 5725 "Remove intr handler failed: %d", rc); 5726 } 5727 } 5728 } 5729 5730 /* 5731 * ixgbe_rem_intrs - Remove the allocated interrupts. 5732 */ 5733 static void 5734 ixgbe_rem_intrs(ixgbe_t *ixgbe) 5735 { 5736 int i; 5737 int rc; 5738 5739 for (i = 0; i < ixgbe->intr_cnt; i++) { 5740 rc = ddi_intr_free(ixgbe->htable[i]); 5741 if (rc != DDI_SUCCESS) { 5742 IXGBE_DEBUGLOG_1(ixgbe, 5743 "Free intr failed: %d", rc); 5744 } 5745 } 5746 5747 kmem_free(ixgbe->htable, ixgbe->intr_size); 5748 ixgbe->htable = NULL; 5749 } 5750 5751 /* 5752 * ixgbe_enable_intrs - Enable all the ddi interrupts. 5753 */ 5754 static int 5755 ixgbe_enable_intrs(ixgbe_t *ixgbe) 5756 { 5757 int i; 5758 int rc; 5759 5760 /* 5761 * Enable interrupts 5762 */ 5763 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5764 /* 5765 * Call ddi_intr_block_enable() for MSI 5766 */ 5767 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 5768 if (rc != DDI_SUCCESS) { 5769 ixgbe_log(ixgbe, 5770 "Enable block intr failed: %d", rc); 5771 return (IXGBE_FAILURE); 5772 } 5773 } else { 5774 /* 5775 * Call ddi_intr_enable() for Legacy/MSI non block enable 5776 */ 5777 for (i = 0; i < ixgbe->intr_cnt; i++) { 5778 rc = ddi_intr_enable(ixgbe->htable[i]); 5779 if (rc != DDI_SUCCESS) { 5780 ixgbe_log(ixgbe, 5781 "Enable intr failed: %d", rc); 5782 return (IXGBE_FAILURE); 5783 } 5784 } 5785 } 5786 5787 return (IXGBE_SUCCESS); 5788 } 5789 5790 /* 5791 * ixgbe_disable_intrs - Disable all the interrupts. 5792 */ 5793 static int 5794 ixgbe_disable_intrs(ixgbe_t *ixgbe) 5795 { 5796 int i; 5797 int rc; 5798 5799 /* 5800 * Disable all interrupts 5801 */ 5802 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5803 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 5804 if (rc != DDI_SUCCESS) { 5805 ixgbe_log(ixgbe, 5806 "Disable block intr failed: %d", rc); 5807 return (IXGBE_FAILURE); 5808 } 5809 } else { 5810 for (i = 0; i < ixgbe->intr_cnt; i++) { 5811 rc = ddi_intr_disable(ixgbe->htable[i]); 5812 if (rc != DDI_SUCCESS) { 5813 ixgbe_log(ixgbe, 5814 "Disable intr failed: %d", rc); 5815 return (IXGBE_FAILURE); 5816 } 5817 } 5818 } 5819 5820 return (IXGBE_SUCCESS); 5821 } 5822 5823 /* 5824 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 5825 */ 5826 static void 5827 ixgbe_get_hw_state(ixgbe_t *ixgbe) 5828 { 5829 struct ixgbe_hw *hw = &ixgbe->hw; 5830 ixgbe_link_speed speed = 0; 5831 boolean_t link_up = B_FALSE; 5832 uint32_t pcs1g_anlp = 0; 5833 5834 ASSERT(mutex_owned(&ixgbe->gen_lock)); 5835 ixgbe->param_lp_1000fdx_cap = 0; 5836 ixgbe->param_lp_100fdx_cap = 0; 5837 5838 /* check for link, don't wait */ 5839 (void) ixgbe_check_link(hw, &speed, &link_up, B_FALSE); 5840 5841 /* 5842 * Update the observed Link Partner's capabilities. Not all adapters 5843 * can provide full information on the LP's capable speeds, so we 5844 * provide what we can. 5845 */ 5846 if (link_up) { 5847 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 5848 5849 ixgbe->param_lp_1000fdx_cap = 5850 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5851 ixgbe->param_lp_100fdx_cap = 5852 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5853 } 5854 5855 /* 5856 * Update GLD's notion of the adapter's currently advertised speeds. 5857 * Since the common code doesn't always record the current autonegotiate 5858 * settings in the phy struct for all parts (specifically, adapters with 5859 * SFPs) we first test to see if it is 0, and if so, we fall back to 5860 * using the adapter's speed capabilities which we saved during instance 5861 * init in ixgbe_init_params(). 5862 * 5863 * Adapters with SFPs will always be shown as advertising all of their 5864 * supported speeds, and adapters with baseT PHYs (where the phy struct 5865 * is maintained by the common code) will always have a factual view of 5866 * their currently-advertised speeds. In the case of SFPs, this is 5867 * acceptable as we default to advertising all speeds that the adapter 5868 * claims to support, and those properties are immutable; unlike on 5869 * baseT (copper) PHYs, where speeds can be enabled or disabled at will. 5870 */ 5871 speed = hw->phy.autoneg_advertised; 5872 if (speed == 0) 5873 speed = ixgbe->speeds_supported; 5874 5875 ixgbe->param_adv_10000fdx_cap = 5876 (speed & IXGBE_LINK_SPEED_10GB_FULL) ? 1 : 0; 5877 ixgbe->param_adv_5000fdx_cap = 5878 (speed & IXGBE_LINK_SPEED_5GB_FULL) ? 1 : 0; 5879 ixgbe->param_adv_2500fdx_cap = 5880 (speed & IXGBE_LINK_SPEED_2_5GB_FULL) ? 1 : 0; 5881 ixgbe->param_adv_1000fdx_cap = 5882 (speed & IXGBE_LINK_SPEED_1GB_FULL) ? 1 : 0; 5883 ixgbe->param_adv_100fdx_cap = 5884 (speed & IXGBE_LINK_SPEED_100_FULL) ? 1 : 0; 5885 } 5886 5887 /* 5888 * ixgbe_get_driver_control - Notify that driver is in control of device. 5889 */ 5890 static void 5891 ixgbe_get_driver_control(struct ixgbe_hw *hw) 5892 { 5893 uint32_t ctrl_ext; 5894 5895 /* 5896 * Notify firmware that driver is in control of device 5897 */ 5898 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5899 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 5900 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5901 } 5902 5903 /* 5904 * ixgbe_release_driver_control - Notify that driver is no longer in control 5905 * of device. 5906 */ 5907 static void 5908 ixgbe_release_driver_control(struct ixgbe_hw *hw) 5909 { 5910 uint32_t ctrl_ext; 5911 5912 /* 5913 * Notify firmware that driver is no longer in control of device 5914 */ 5915 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5916 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5917 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5918 } 5919 5920 /* 5921 * ixgbe_atomic_reserve - Atomic decrease operation. 5922 */ 5923 int 5924 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5925 { 5926 uint32_t oldval; 5927 uint32_t newval; 5928 5929 /* 5930 * ATOMICALLY 5931 */ 5932 do { 5933 oldval = *count_p; 5934 if (oldval < n) 5935 return (-1); 5936 newval = oldval - n; 5937 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5938 5939 return (newval); 5940 } 5941 5942 /* 5943 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5944 */ 5945 static uint8_t * 5946 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5947 { 5948 uint8_t *addr = *upd_ptr; 5949 uint8_t *new_ptr; 5950 5951 _NOTE(ARGUNUSED(hw)); 5952 _NOTE(ARGUNUSED(vmdq)); 5953 5954 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5955 *upd_ptr = new_ptr; 5956 return (addr); 5957 } 5958 5959 /* 5960 * FMA support 5961 */ 5962 int 5963 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5964 { 5965 ddi_fm_error_t de; 5966 5967 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5968 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5969 return (de.fme_status); 5970 } 5971 5972 int 5973 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5974 { 5975 ddi_fm_error_t de; 5976 5977 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5978 return (de.fme_status); 5979 } 5980 5981 /* 5982 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5983 */ 5984 static int 5985 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5986 { 5987 _NOTE(ARGUNUSED(impl_data)); 5988 /* 5989 * as the driver can always deal with an error in any dma or 5990 * access handle, we can just return the fme_status value. 5991 */ 5992 pci_ereport_post(dip, err, NULL); 5993 return (err->fme_status); 5994 } 5995 5996 static void 5997 ixgbe_fm_init(ixgbe_t *ixgbe) 5998 { 5999 ddi_iblock_cookie_t iblk; 6000 int fma_dma_flag; 6001 6002 /* 6003 * Only register with IO Fault Services if we have some capability 6004 */ 6005 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 6006 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 6007 } else { 6008 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 6009 } 6010 6011 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 6012 fma_dma_flag = 1; 6013 } else { 6014 fma_dma_flag = 0; 6015 } 6016 6017 ixgbe_set_fma_flags(fma_dma_flag); 6018 6019 if (ixgbe->fm_capabilities) { 6020 6021 /* 6022 * Register capabilities with IO Fault Services 6023 */ 6024 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 6025 6026 /* 6027 * Initialize pci ereport capabilities if ereport capable 6028 */ 6029 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 6030 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6031 pci_ereport_setup(ixgbe->dip); 6032 6033 /* 6034 * Register error callback if error callback capable 6035 */ 6036 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6037 ddi_fm_handler_register(ixgbe->dip, 6038 ixgbe_fm_error_cb, (void*) ixgbe); 6039 } 6040 } 6041 6042 static void 6043 ixgbe_fm_fini(ixgbe_t *ixgbe) 6044 { 6045 /* 6046 * Only unregister FMA capabilities if they are registered 6047 */ 6048 if (ixgbe->fm_capabilities) { 6049 6050 /* 6051 * Release any resources allocated by pci_ereport_setup() 6052 */ 6053 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 6054 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6055 pci_ereport_teardown(ixgbe->dip); 6056 6057 /* 6058 * Un-register error callback if error callback capable 6059 */ 6060 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 6061 ddi_fm_handler_unregister(ixgbe->dip); 6062 6063 /* 6064 * Unregister from IO Fault Service 6065 */ 6066 ddi_fm_fini(ixgbe->dip); 6067 } 6068 } 6069 6070 void 6071 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 6072 { 6073 uint64_t ena; 6074 char buf[FM_MAX_CLASS]; 6075 6076 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 6077 ena = fm_ena_generate(0, FM_ENA_FMT1); 6078 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 6079 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 6080 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 6081 } 6082 } 6083 6084 static int 6085 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 6086 { 6087 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 6088 6089 mutex_enter(&rx_ring->rx_lock); 6090 rx_ring->ring_gen_num = mr_gen_num; 6091 mutex_exit(&rx_ring->rx_lock); 6092 return (0); 6093 } 6094 6095 /* 6096 * Get the global ring index by a ring index within a group. 6097 */ 6098 static int 6099 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 6100 { 6101 ixgbe_rx_ring_t *rx_ring; 6102 int i; 6103 6104 for (i = 0; i < ixgbe->num_rx_rings; i++) { 6105 rx_ring = &ixgbe->rx_rings[i]; 6106 if (rx_ring->group_index == gindex) 6107 rindex--; 6108 if (rindex < 0) 6109 return (i); 6110 } 6111 6112 return (-1); 6113 } 6114 6115 /* 6116 * Callback funtion for MAC layer to register all rings. 6117 */ 6118 /* ARGSUSED */ 6119 void 6120 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 6121 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 6122 { 6123 ixgbe_t *ixgbe = (ixgbe_t *)arg; 6124 mac_intr_t *mintr = &infop->mri_intr; 6125 6126 switch (rtype) { 6127 case MAC_RING_TYPE_RX: { 6128 /* 6129 * 'index' is the ring index within the group. 6130 * Need to get the global ring index by searching in groups. 6131 */ 6132 int global_ring_index = ixgbe_get_rx_ring_index( 6133 ixgbe, group_index, ring_index); 6134 6135 ASSERT(global_ring_index >= 0); 6136 6137 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 6138 rx_ring->ring_handle = rh; 6139 6140 infop->mri_driver = (mac_ring_driver_t)rx_ring; 6141 infop->mri_start = ixgbe_ring_start; 6142 infop->mri_stop = NULL; 6143 infop->mri_poll = ixgbe_ring_rx_poll; 6144 infop->mri_stat = ixgbe_rx_ring_stat; 6145 6146 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 6147 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 6148 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 6149 if (ixgbe->intr_type & 6150 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 6151 mintr->mi_ddi_handle = 6152 ixgbe->htable[rx_ring->intr_vector]; 6153 } 6154 6155 break; 6156 } 6157 case MAC_RING_TYPE_TX: { 6158 ASSERT(group_index == -1); 6159 ASSERT(ring_index < ixgbe->num_tx_rings); 6160 6161 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 6162 tx_ring->ring_handle = rh; 6163 6164 infop->mri_driver = (mac_ring_driver_t)tx_ring; 6165 infop->mri_start = NULL; 6166 infop->mri_stop = NULL; 6167 infop->mri_tx = ixgbe_ring_tx; 6168 infop->mri_stat = ixgbe_tx_ring_stat; 6169 if (ixgbe->intr_type & 6170 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 6171 mintr->mi_ddi_handle = 6172 ixgbe->htable[tx_ring->intr_vector]; 6173 } 6174 break; 6175 } 6176 default: 6177 break; 6178 } 6179 } 6180 6181 /* 6182 * Callback funtion for MAC layer to register all groups. 6183 */ 6184 void 6185 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 6186 mac_group_info_t *infop, mac_group_handle_t gh) 6187 { 6188 ixgbe_t *ixgbe = (ixgbe_t *)arg; 6189 6190 switch (rtype) { 6191 case MAC_RING_TYPE_RX: { 6192 ixgbe_rx_group_t *rx_group; 6193 6194 rx_group = &ixgbe->rx_groups[index]; 6195 rx_group->group_handle = gh; 6196 6197 infop->mgi_driver = (mac_group_driver_t)rx_group; 6198 infop->mgi_start = NULL; 6199 infop->mgi_stop = NULL; 6200 infop->mgi_addmac = ixgbe_addmac; 6201 infop->mgi_remmac = ixgbe_remmac; 6202 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 6203 6204 break; 6205 } 6206 case MAC_RING_TYPE_TX: 6207 break; 6208 default: 6209 break; 6210 } 6211 } 6212 6213 /* 6214 * Enable interrupt on the specificed rx ring. 6215 */ 6216 int 6217 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 6218 { 6219 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 6220 ixgbe_t *ixgbe = rx_ring->ixgbe; 6221 int r_idx = rx_ring->index; 6222 int hw_r_idx = rx_ring->hw_index; 6223 int v_idx = rx_ring->intr_vector; 6224 6225 mutex_enter(&ixgbe->gen_lock); 6226 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 6227 mutex_exit(&ixgbe->gen_lock); 6228 /* 6229 * Simply return 0. 6230 * Interrupts are being adjusted. ixgbe_intr_adjust() 6231 * will eventually re-enable the interrupt when it's 6232 * done with the adjustment. 6233 */ 6234 return (0); 6235 } 6236 6237 /* 6238 * To enable interrupt by setting the VAL bit of given interrupt 6239 * vector allocation register (IVAR). 6240 */ 6241 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 6242 6243 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 6244 6245 /* 6246 * Trigger a Rx interrupt on this ring 6247 */ 6248 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 6249 IXGBE_WRITE_FLUSH(&ixgbe->hw); 6250 6251 mutex_exit(&ixgbe->gen_lock); 6252 6253 return (0); 6254 } 6255 6256 /* 6257 * Disable interrupt on the specificed rx ring. 6258 */ 6259 int 6260 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 6261 { 6262 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 6263 ixgbe_t *ixgbe = rx_ring->ixgbe; 6264 int r_idx = rx_ring->index; 6265 int hw_r_idx = rx_ring->hw_index; 6266 int v_idx = rx_ring->intr_vector; 6267 6268 mutex_enter(&ixgbe->gen_lock); 6269 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 6270 mutex_exit(&ixgbe->gen_lock); 6271 /* 6272 * Simply return 0. 6273 * In the rare case where an interrupt is being 6274 * disabled while interrupts are being adjusted, 6275 * we don't fail the operation. No interrupts will 6276 * be generated while they are adjusted, and 6277 * ixgbe_intr_adjust() will cause the interrupts 6278 * to be re-enabled once it completes. Note that 6279 * in this case, packets may be delivered to the 6280 * stack via interrupts before xgbe_rx_ring_intr_enable() 6281 * is called again. This is acceptable since interrupt 6282 * adjustment is infrequent, and the stack will be 6283 * able to handle these packets. 6284 */ 6285 return (0); 6286 } 6287 6288 /* 6289 * To disable interrupt by clearing the VAL bit of given interrupt 6290 * vector allocation register (IVAR). 6291 */ 6292 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 6293 6294 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 6295 6296 mutex_exit(&ixgbe->gen_lock); 6297 6298 return (0); 6299 } 6300 6301 /* 6302 * Add a mac address. 6303 */ 6304 static int 6305 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 6306 { 6307 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 6308 ixgbe_t *ixgbe = rx_group->ixgbe; 6309 struct ixgbe_hw *hw = &ixgbe->hw; 6310 int slot, i; 6311 6312 mutex_enter(&ixgbe->gen_lock); 6313 6314 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 6315 mutex_exit(&ixgbe->gen_lock); 6316 return (ECANCELED); 6317 } 6318 6319 if (ixgbe->unicst_avail == 0) { 6320 /* no slots available */ 6321 mutex_exit(&ixgbe->gen_lock); 6322 return (ENOSPC); 6323 } 6324 6325 /* 6326 * The first ixgbe->num_rx_groups slots are reserved for each respective 6327 * group. The rest slots are shared by all groups. While adding a 6328 * MAC address, reserved slots are firstly checked then the shared 6329 * slots are searched. 6330 */ 6331 slot = -1; 6332 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 6333 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 6334 if (ixgbe->unicst_addr[i].mac.set == 0) { 6335 slot = i; 6336 break; 6337 } 6338 } 6339 } else { 6340 slot = rx_group->index; 6341 } 6342 6343 if (slot == -1) { 6344 /* no slots available */ 6345 mutex_exit(&ixgbe->gen_lock); 6346 return (ENOSPC); 6347 } 6348 6349 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 6350 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 6351 rx_group->index, IXGBE_RAH_AV); 6352 ixgbe->unicst_addr[slot].mac.set = 1; 6353 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 6354 ixgbe->unicst_avail--; 6355 6356 mutex_exit(&ixgbe->gen_lock); 6357 6358 return (0); 6359 } 6360 6361 /* 6362 * Remove a mac address. 6363 */ 6364 static int 6365 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 6366 { 6367 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 6368 ixgbe_t *ixgbe = rx_group->ixgbe; 6369 struct ixgbe_hw *hw = &ixgbe->hw; 6370 int slot; 6371 6372 mutex_enter(&ixgbe->gen_lock); 6373 6374 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 6375 mutex_exit(&ixgbe->gen_lock); 6376 return (ECANCELED); 6377 } 6378 6379 slot = ixgbe_unicst_find(ixgbe, mac_addr); 6380 if (slot == -1) { 6381 mutex_exit(&ixgbe->gen_lock); 6382 return (EINVAL); 6383 } 6384 6385 if (ixgbe->unicst_addr[slot].mac.set == 0) { 6386 mutex_exit(&ixgbe->gen_lock); 6387 return (EINVAL); 6388 } 6389 6390 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 6391 (void) ixgbe_clear_rar(hw, slot); 6392 ixgbe->unicst_addr[slot].mac.set = 0; 6393 ixgbe->unicst_avail++; 6394 6395 mutex_exit(&ixgbe->gen_lock); 6396 6397 return (0); 6398 } 6399