1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 98 static int ahci_initialize_controller(ahci_ctl_t *); 99 static void ahci_uninitialize_controller(ahci_ctl_t *); 100 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 101 static int ahci_config_space_init(ahci_ctl_t *); 102 103 static void ahci_drain_ports_taskq(ahci_ctl_t *); 104 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 105 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 106 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, uint8_t); 107 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 108 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 109 uint8_t, sata_pkt_t *); 110 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 111 uint8_t, sata_pkt_t *); 112 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, int); 113 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 114 static void ahci_copy_ncq_err_page(sata_cmd_t *, 115 struct sata_ncq_error_recovery_page *); 116 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 117 118 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 119 static int ahci_hba_reset(ahci_ctl_t *); 120 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, uint8_t); 121 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 122 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 124 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 125 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 126 uint8_t); 127 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 128 uint8_t, int, int *); 129 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 130 uint32_t, uint32_t, uint32_t, uint32_t); 131 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 132 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 133 uint8_t, sata_pkt_t *); 134 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 135 uint8_t, uint32_t); 136 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 137 uint8_t, uint32_t); 138 static void ahci_events_handler(void *); 139 static void ahci_watchdog_handler(ahci_ctl_t *); 140 141 static uint_t ahci_intr(caddr_t, caddr_t); 142 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 143 static int ahci_add_intrs(ahci_ctl_t *, int); 144 static void ahci_rem_intrs(ahci_ctl_t *); 145 static void ahci_enable_all_intrs(ahci_ctl_t *); 146 static void ahci_disable_all_intrs(ahci_ctl_t *); 147 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 148 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 149 150 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 151 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 152 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 153 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 154 ahci_port_t *, uint8_t); 155 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, uint32_t); 158 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 159 uint8_t, uint32_t); 160 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 161 162 static int ahci_get_num_implemented_ports(uint32_t); 163 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 164 uint32_t); 165 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 166 #if AHCI_DEBUG 167 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 168 #endif 169 170 171 /* 172 * DMA attributes for the data buffer 173 * 174 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 175 * does not support 64-bit addressing 176 */ 177 static ddi_dma_attr_t buffer_dma_attr = { 178 DMA_ATTR_V0, /* dma_attr_version */ 179 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 180 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 181 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 182 0x2ull, /* dma_attr_align: word aligned */ 183 1, /* dma_attr_burstsizes */ 184 1, /* dma_attr_minxfer */ 185 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 186 0xffffffffull, /* dma_attr_seg */ 187 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 188 512, /* dma_attr_granular */ 189 0, /* dma_attr_flags */ 190 }; 191 192 /* 193 * DMA attributes for the rcvd FIS 194 * 195 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 196 * does not support 64-bit addressing 197 */ 198 static ddi_dma_attr_t rcvd_fis_dma_attr = { 199 DMA_ATTR_V0, /* dma_attr_version */ 200 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 201 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 202 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 203 0x100ull, /* dma_attr_align: 256-byte aligned */ 204 1, /* dma_attr_burstsizes */ 205 1, /* dma_attr_minxfer */ 206 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 207 0xffffffffull, /* dma_attr_seg */ 208 1, /* dma_attr_sgllen */ 209 1, /* dma_attr_granular */ 210 0, /* dma_attr_flags */ 211 }; 212 213 /* 214 * DMA attributes for the command list 215 * 216 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 217 * does not support 64-bit addressing 218 */ 219 static ddi_dma_attr_t cmd_list_dma_attr = { 220 DMA_ATTR_V0, /* dma_attr_version */ 221 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 222 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 223 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 224 0x400ull, /* dma_attr_align: 1K-byte aligned */ 225 1, /* dma_attr_burstsizes */ 226 1, /* dma_attr_minxfer */ 227 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 228 0xffffffffull, /* dma_attr_seg */ 229 1, /* dma_attr_sgllen */ 230 1, /* dma_attr_granular */ 231 0, /* dma_attr_flags */ 232 }; 233 234 /* 235 * DMA attributes for cmd tables 236 * 237 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 238 * does not support 64-bit addressing 239 */ 240 static ddi_dma_attr_t cmd_table_dma_attr = { 241 DMA_ATTR_V0, /* dma_attr_version */ 242 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 243 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 244 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 245 0x80ull, /* dma_attr_align: 128-byte aligned */ 246 1, /* dma_attr_burstsizes */ 247 1, /* dma_attr_minxfer */ 248 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 249 0xffffffffull, /* dma_attr_seg */ 250 1, /* dma_attr_sgllen */ 251 1, /* dma_attr_granular */ 252 0, /* dma_attr_flags */ 253 }; 254 255 256 /* Device access attributes */ 257 static ddi_device_acc_attr_t accattr = { 258 DDI_DEVICE_ATTR_V0, 259 DDI_STRUCTURE_LE_ACC, 260 DDI_STRICTORDER_ACC 261 }; 262 263 264 static struct dev_ops ahcictl_dev_ops = { 265 DEVO_REV, /* devo_rev */ 266 0, /* refcnt */ 267 ahci_getinfo, /* info */ 268 nulldev, /* identify */ 269 nulldev, /* probe */ 270 ahci_attach, /* attach */ 271 ahci_detach, /* detach */ 272 nodev, /* no reset */ 273 (struct cb_ops *)0, /* driver operations */ 274 NULL, /* bus operations */ 275 NULL, /* power */ 276 ahci_quiesce, /* quiesce */ 277 }; 278 279 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 280 SATA_TRAN_HOTPLUG_OPS_REV_1, 281 ahci_tran_hotplug_port_activate, 282 ahci_tran_hotplug_port_deactivate 283 }; 284 285 extern struct mod_ops mod_driverops; 286 287 static struct modldrv modldrv = { 288 &mod_driverops, /* driverops */ 289 ahci_ident, /* short description */ 290 &ahcictl_dev_ops, /* driver ops */ 291 }; 292 293 static struct modlinkage modlinkage = { 294 MODREV_1, 295 &modldrv, 296 NULL 297 }; 298 299 /* The following variables are watchdog handler related */ 300 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 301 static int ahci_watchdog_tick; 302 303 /* 304 * This static variable indicates the size of command table, 305 * and it's changeable with prdt number, which ahci_dma_prdt_number 306 * indicates. 307 */ 308 static size_t ahci_cmd_table_size; 309 310 /* 311 * ahci_dma_prdt_number, ahci_msi_enabled and ahci_64bit_dma_addressed 312 * are global variables, and can be changed via /etc/system file. 313 */ 314 315 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 316 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 317 318 /* AHCI MSI is tunable */ 319 boolean_t ahci_msi_enabled = B_TRUE; 320 321 /* 64-bit dma addressing is tunable */ 322 boolean_t ahci_64bit_dma_addressed = B_TRUE; 323 324 #if AHCI_DEBUG 325 uint32_t ahci_debug_flags = 0; 326 327 /* The following is needed for ahci_log() */ 328 static kmutex_t ahci_log_mutex; 329 static char ahci_log_buf[512]; 330 #endif 331 332 /* Opaque state pointer initialized by ddi_soft_state_init() */ 333 static void *ahci_statep = NULL; 334 335 /* 336 * ahci module initialization. 337 */ 338 int 339 _init(void) 340 { 341 int ret; 342 343 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 344 if (ret != 0) { 345 goto err_out; 346 } 347 348 #if AHCI_DEBUG 349 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 350 #endif 351 352 if ((ret = sata_hba_init(&modlinkage)) != 0) { 353 #if AHCI_DEBUG 354 mutex_destroy(&ahci_log_mutex); 355 #endif 356 ddi_soft_state_fini(&ahci_statep); 357 goto err_out; 358 } 359 360 ret = mod_install(&modlinkage); 361 if (ret != 0) { 362 sata_hba_fini(&modlinkage); 363 #if AHCI_DEBUG 364 mutex_destroy(&ahci_log_mutex); 365 #endif 366 ddi_soft_state_fini(&ahci_statep); 367 goto err_out; 368 } 369 370 /* watchdog tick */ 371 ahci_watchdog_tick = drv_usectohz( 372 (clock_t)ahci_watchdog_timeout * 1000000); 373 return (ret); 374 375 err_out: 376 cmn_err(CE_WARN, "!ahci: Module init failed"); 377 return (ret); 378 } 379 380 /* 381 * ahci module uninitialize. 382 */ 383 int 384 _fini(void) 385 { 386 int ret; 387 388 ret = mod_remove(&modlinkage); 389 if (ret != 0) { 390 return (ret); 391 } 392 393 /* Remove the resources allocated in _init(). */ 394 sata_hba_fini(&modlinkage); 395 #if AHCI_DEBUG 396 mutex_destroy(&ahci_log_mutex); 397 #endif 398 ddi_soft_state_fini(&ahci_statep); 399 400 return (ret); 401 } 402 403 /* 404 * _info entry point 405 */ 406 int 407 _info(struct modinfo *modinfop) 408 { 409 return (mod_info(&modlinkage, modinfop)); 410 } 411 412 /* 413 * The attach entry point for dev_ops. 414 */ 415 static int 416 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 417 { 418 ahci_ctl_t *ahci_ctlp; 419 int instance = ddi_get_instance(dip); 420 int status; 421 int attach_state; 422 uint32_t cap_status, ahci_version; 423 int intr_types; 424 int i; 425 pci_regspec_t *regs; 426 int regs_length; 427 int rnumber; 428 #if AHCI_DEBUG 429 int speed; 430 #endif 431 432 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, NULL, "ahci_attach enter"); 433 434 switch (cmd) { 435 case DDI_ATTACH: 436 break; 437 438 case DDI_RESUME: 439 440 /* 441 * During DDI_RESUME, the hardware state of the device 442 * (power may have been removed from the device) must be 443 * restored, allow pending requests to continue, and 444 * service new requests. 445 */ 446 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 447 mutex_enter(&ahci_ctlp->ahcictl_mutex); 448 449 /* Restart watch thread */ 450 if (ahci_ctlp->ahcictl_timeout_id == 0) 451 ahci_ctlp->ahcictl_timeout_id = timeout( 452 (void (*)(void *))ahci_watchdog_handler, 453 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 454 455 mutex_exit(&ahci_ctlp->ahcictl_mutex); 456 457 /* 458 * Re-initialize the controller and enable the interrupts and 459 * restart all the ports. 460 * 461 * Note that so far we don't support hot-plug during 462 * suspend/resume. 463 */ 464 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 465 AHCIDBG0(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 466 "Failed to initialize the controller " 467 "during DDI_RESUME"); 468 return (DDI_FAILURE); 469 } 470 471 mutex_enter(&ahci_ctlp->ahcictl_mutex); 472 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 473 mutex_exit(&ahci_ctlp->ahcictl_mutex); 474 475 return (DDI_SUCCESS); 476 477 default: 478 return (DDI_FAILURE); 479 } 480 481 attach_state = AHCI_ATTACH_STATE_NONE; 482 483 /* Allocate soft state */ 484 status = ddi_soft_state_zalloc(ahci_statep, instance); 485 if (status != DDI_SUCCESS) { 486 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 487 instance); 488 goto err_out; 489 } 490 491 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 492 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 493 ahci_ctlp->ahcictl_dip = dip; 494 495 /* Initialize the cport/port mapping */ 496 for (i = 0; i < AHCI_MAX_PORTS; i++) { 497 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 498 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 499 } 500 501 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 502 503 /* 504 * Now map the AHCI base address; which includes global 505 * registers and port control registers 506 * 507 * According to the spec, the AHCI Base Address is BAR5, 508 * but BAR0-BAR4 are optional, so we need to check which 509 * rnumber is used for BAR5. 510 */ 511 512 /* 513 * search through DDI "reg" property for the AHCI register set 514 */ 515 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 516 DDI_PROP_DONTPASS, "reg", (int **)®s, 517 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 518 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 519 instance); 520 goto err_out; 521 } 522 523 /* AHCI Base Address is located at 0x24 offset */ 524 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 525 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 526 == AHCI_PCI_RNUM) 527 break; 528 } 529 530 ddi_prop_free(regs); 531 532 if (rnumber == regs_length) { 533 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 534 instance); 535 goto err_out; 536 } 537 538 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 539 540 status = ddi_regs_map_setup(dip, 541 rnumber, 542 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 543 0, 544 0, 545 &accattr, 546 &ahci_ctlp->ahcictl_ahci_acc_handle); 547 if (status != DDI_SUCCESS) { 548 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 549 instance); 550 goto err_out; 551 } 552 553 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 554 555 /* Get the AHCI version information */ 556 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 557 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 558 559 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 560 (ahci_version & 0xffff0000) >> 16, 561 ((ahci_version & 0x0000ff00) >> 4 | 562 (ahci_version & 0x000000ff))); 563 564 /* We don't support controllers whose versions are lower than 1.0 */ 565 if (!(ahci_version & 0xffff0000)) { 566 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 567 "than version 1.0", instance); 568 goto err_out; 569 } 570 571 /* Get the HBA capabilities information */ 572 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 573 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 574 575 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 576 cap_status); 577 578 #if AHCI_DEBUG 579 /* Get the interface speed supported by the HBA */ 580 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 581 if (speed == 0x01) { 582 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 583 "hba interface speed support: Gen 1 (1.5Gbps)"); 584 } else if (speed == 0x10) { 585 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 586 "hba interface speed support: Gen 2 (3 Gbps)"); 587 } else if (speed == 0x11) { 588 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 589 "hba interface speed support: Gen 3 (6 Gbps)"); 590 } 591 #endif 592 593 /* Get the number of command slots supported by the HBA */ 594 ahci_ctlp->ahcictl_num_cmd_slots = 595 ((cap_status & AHCI_HBA_CAP_NCS) >> 596 AHCI_HBA_CAP_NCS_SHIFT) + 1; 597 598 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 599 ahci_ctlp->ahcictl_num_cmd_slots); 600 601 /* Get the bit map which indicates ports implemented by the HBA */ 602 ahci_ctlp->ahcictl_ports_implemented = 603 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 604 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 605 606 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 607 ahci_ctlp->ahcictl_ports_implemented); 608 609 /* 610 * According to the AHCI spec, CAP.NP should indicate the maximum 611 * number of ports supported by the HBA silicon, but we found 612 * this value of ICH8 chipset only indicates the number of ports 613 * implemented (exposed) by it. Therefore, the driver should calculate 614 * the potential maximum value by checking PI register, and use 615 * the maximum of this value and CAP.NP. 616 */ 617 ahci_ctlp->ahcictl_num_ports = max( 618 (cap_status & AHCI_HBA_CAP_NP) + 1, 619 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 620 621 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 622 ahci_ctlp->ahcictl_num_ports); 623 624 /* Get the number of implemented ports by the HBA */ 625 ahci_ctlp->ahcictl_num_implemented_ports = 626 ahci_get_num_implemented_ports( 627 ahci_ctlp->ahcictl_ports_implemented); 628 629 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 630 "hba number of implemented ports: %d", 631 ahci_ctlp->ahcictl_num_implemented_ports); 632 633 /* Check whether HBA supports 64bit DMA addressing */ 634 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 635 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 636 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 637 "hba does not support 64-bit addressing"); 638 } 639 640 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 641 != DDI_SUCCESS) { 642 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 643 instance); 644 goto err_out; 645 } 646 647 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 648 649 /* 650 * Check the pci configuration space, and set caps. We also 651 * handle the hardware defect in this function. 652 * 653 * For example, force ATI SB600/SB700/SB750 to use 32-bit dma 654 * addressing since it doesn't support 64-bit dma though their 655 * registers declare they support. 656 */ 657 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 658 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 659 instance); 660 goto err_out; 661 } 662 663 /* 664 * Disable the whole controller interrupts before adding 665 * interrupt handlers(s). 666 */ 667 ahci_disable_all_intrs(ahci_ctlp); 668 669 /* Get supported interrupt types */ 670 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 671 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 672 instance); 673 goto err_out; 674 } 675 676 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 677 "ddi_intr_get_supported_types() returned: 0x%x", 678 intr_types); 679 680 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 681 /* 682 * Try MSI first, but fall back to FIXED if failed 683 */ 684 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 685 DDI_SUCCESS) { 686 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 687 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 688 "Using MSI interrupt type"); 689 goto intr_done; 690 } 691 692 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 693 "MSI registration failed, " 694 "trying FIXED interrupts"); 695 } 696 697 if (intr_types & DDI_INTR_TYPE_FIXED) { 698 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 699 DDI_SUCCESS) { 700 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 701 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 702 "Using FIXED interrupt type"); 703 goto intr_done; 704 } 705 706 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 707 "FIXED interrupt registration failed"); 708 } 709 710 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 711 712 goto err_out; 713 714 intr_done: 715 716 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 717 718 /* Initialize the controller mutex */ 719 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 720 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 721 722 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 723 724 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 725 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 726 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 727 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 728 } 729 730 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 731 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 732 sizeof (ahci_prdt_item_t)); 733 734 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 735 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 736 " ahci_cmd_table_size is 0x%x", 737 ahci_dma_prdt_number, ahci_cmd_table_size); 738 739 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 740 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 741 ahci_dma_prdt_number; 742 743 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 744 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 745 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 746 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 747 748 if ((ahci_64bit_dma_addressed == B_FALSE) || 749 (ahci_ctlp->ahcictl_cap & AHCI_CAP_32BIT_DMA)) { 750 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 751 0xffffffffull; 752 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 753 0xffffffffull; 754 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 755 0xffffffffull; 756 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 757 0xffffffffull; 758 } 759 760 /* Allocate the ports structure */ 761 status = ahci_alloc_ports_state(ahci_ctlp); 762 if (status != AHCI_SUCCESS) { 763 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 764 instance); 765 goto err_out; 766 } 767 768 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 769 770 /* 771 * Initialize the controller and ports. 772 */ 773 status = ahci_initialize_controller(ahci_ctlp); 774 if (status != AHCI_SUCCESS) { 775 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 776 instance); 777 goto err_out; 778 } 779 780 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 781 782 /* Start one thread to check packet timeouts */ 783 ahci_ctlp->ahcictl_timeout_id = timeout( 784 (void (*)(void *))ahci_watchdog_handler, 785 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 786 787 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 788 789 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 790 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 791 instance); 792 goto err_out; 793 } 794 795 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 796 797 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!"); 798 799 return (DDI_SUCCESS); 800 801 err_out: 802 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 803 mutex_enter(&ahci_ctlp->ahcictl_mutex); 804 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 805 ahci_ctlp->ahcictl_timeout_id = 0; 806 mutex_exit(&ahci_ctlp->ahcictl_mutex); 807 } 808 809 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 810 ahci_uninitialize_controller(ahci_ctlp); 811 } 812 813 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 814 ahci_dealloc_ports_state(ahci_ctlp); 815 } 816 817 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 818 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 819 } 820 821 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 822 ahci_rem_intrs(ahci_ctlp); 823 } 824 825 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 826 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 827 } 828 829 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 830 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 831 } 832 833 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 834 ddi_soft_state_free(ahci_statep, instance); 835 } 836 837 return (DDI_FAILURE); 838 } 839 840 /* 841 * The detach entry point for dev_ops. 842 */ 843 static int 844 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 845 { 846 ahci_ctl_t *ahci_ctlp; 847 int instance; 848 int ret; 849 850 instance = ddi_get_instance(dip); 851 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 852 853 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter"); 854 855 switch (cmd) { 856 case DDI_DETACH: 857 858 /* disable the interrupts for an uninterrupted detach */ 859 mutex_enter(&ahci_ctlp->ahcictl_mutex); 860 ahci_disable_all_intrs(ahci_ctlp); 861 mutex_exit(&ahci_ctlp->ahcictl_mutex); 862 863 /* unregister from the sata framework. */ 864 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 865 if (ret != AHCI_SUCCESS) { 866 mutex_enter(&ahci_ctlp->ahcictl_mutex); 867 ahci_enable_all_intrs(ahci_ctlp); 868 mutex_exit(&ahci_ctlp->ahcictl_mutex); 869 return (DDI_FAILURE); 870 } 871 872 mutex_enter(&ahci_ctlp->ahcictl_mutex); 873 874 /* stop the watchdog handler */ 875 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 876 ahci_ctlp->ahcictl_timeout_id = 0; 877 878 mutex_exit(&ahci_ctlp->ahcictl_mutex); 879 880 /* uninitialize the controller */ 881 ahci_uninitialize_controller(ahci_ctlp); 882 883 /* remove the interrupts */ 884 ahci_rem_intrs(ahci_ctlp); 885 886 /* deallocate the ports structures */ 887 ahci_dealloc_ports_state(ahci_ctlp); 888 889 /* destroy mutex */ 890 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 891 892 /* teardown the pci config */ 893 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 894 895 /* remove the reg maps. */ 896 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 897 898 /* free the soft state. */ 899 ddi_soft_state_free(ahci_statep, instance); 900 901 return (DDI_SUCCESS); 902 903 case DDI_SUSPEND: 904 905 /* 906 * The steps associated with suspension must include putting 907 * the underlying device into a quiescent state so that it 908 * will not generate interrupts or modify or access memory. 909 */ 910 mutex_enter(&ahci_ctlp->ahcictl_mutex); 911 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 912 mutex_exit(&ahci_ctlp->ahcictl_mutex); 913 return (DDI_SUCCESS); 914 } 915 916 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 917 918 /* stop the watchdog handler */ 919 if (ahci_ctlp->ahcictl_timeout_id) { 920 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 921 ahci_ctlp->ahcictl_timeout_id = 0; 922 } 923 924 mutex_exit(&ahci_ctlp->ahcictl_mutex); 925 926 /* 927 * drain the taskq 928 */ 929 ahci_drain_ports_taskq(ahci_ctlp); 930 931 /* 932 * Disable the interrupts and stop all the ports. 933 */ 934 ahci_uninitialize_controller(ahci_ctlp); 935 936 return (DDI_SUCCESS); 937 938 default: 939 return (DDI_FAILURE); 940 } 941 } 942 943 /* 944 * The info entry point for dev_ops. 945 * 946 */ 947 static int 948 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 949 void *arg, void **result) 950 { 951 #ifndef __lock_lint 952 _NOTE(ARGUNUSED(dip)) 953 #endif /* __lock_lint */ 954 955 ahci_ctl_t *ahci_ctlp; 956 int instance; 957 dev_t dev; 958 959 dev = (dev_t)arg; 960 instance = getminor(dev); 961 962 switch (infocmd) { 963 case DDI_INFO_DEVT2DEVINFO: 964 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 965 if (ahci_ctlp != NULL) { 966 *result = ahci_ctlp->ahcictl_dip; 967 return (DDI_SUCCESS); 968 } else { 969 *result = NULL; 970 return (DDI_FAILURE); 971 } 972 case DDI_INFO_DEVT2INSTANCE: 973 *(int *)result = instance; 974 break; 975 default: 976 break; 977 } 978 979 return (DDI_SUCCESS); 980 } 981 982 /* 983 * Registers the ahci with sata framework. 984 */ 985 static int 986 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 987 { 988 struct sata_hba_tran *sata_hba_tran; 989 990 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 991 "ahci_register_sata_hba_tran enter"); 992 993 mutex_enter(&ahci_ctlp->ahcictl_mutex); 994 995 /* Allocate memory for the sata_hba_tran */ 996 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 997 998 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV_2; 999 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1000 sata_hba_tran->sata_tran_hba_dma_attr = 1001 &ahci_ctlp->ahcictl_buffer_dma_attr; 1002 1003 /* Report the number of implemented ports */ 1004 sata_hba_tran->sata_tran_hba_num_cports = 1005 ahci_ctlp->ahcictl_num_implemented_ports; 1006 1007 /* Support ATAPI device */ 1008 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1009 1010 /* Get the data transfer capability for PIO command by the HBA */ 1011 if (cap_status & AHCI_HBA_CAP_PMD) { 1012 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1013 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1014 "DRQ block data transfer for PIO command protocol"); 1015 } 1016 1017 /* 1018 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1019 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1020 * QUEUED (EXT), and SERVICE commands). Queued operations are 1021 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1022 * QUEUED commands when the HBA and device support native command 1023 * queuing(NCQ). 1024 * 1025 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1026 * CAP register of the HBA indicates NCQ is supported. 1027 * 1028 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1029 * set because the previous register content of PxCI can be re-written 1030 * in the register write. 1031 */ 1032 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1033 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1034 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1035 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1036 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1037 "Command Queuing"); 1038 } 1039 1040 /* Report the number of command slots */ 1041 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1042 1043 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1044 sata_hba_tran->sata_tran_start = ahci_tran_start; 1045 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1046 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1047 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1048 #ifdef __lock_lint 1049 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1050 #endif 1051 /* 1052 * When SATA framework adds support for pwrmgt the 1053 * pwrmgt_ops needs to be updated 1054 */ 1055 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1056 sata_hba_tran->sata_tran_ioctl = NULL; 1057 1058 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1059 1060 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1061 1062 /* Attach it to SATA framework */ 1063 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1064 != DDI_SUCCESS) { 1065 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1066 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1067 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1068 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1069 return (AHCI_FAILURE); 1070 } 1071 1072 return (AHCI_SUCCESS); 1073 } 1074 1075 /* 1076 * Unregisters the ahci with sata framework. 1077 */ 1078 static int 1079 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1080 { 1081 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 1082 "ahci_unregister_sata_hba_tran enter"); 1083 1084 /* Detach from the SATA framework. */ 1085 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1086 DDI_SUCCESS) { 1087 return (AHCI_FAILURE); 1088 } 1089 1090 /* Deallocate sata_hba_tran. */ 1091 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1092 sizeof (sata_hba_tran_t)); 1093 1094 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1095 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1096 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1097 1098 return (AHCI_SUCCESS); 1099 } 1100 1101 /* 1102 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1103 * port status registers and an attached device type via sata_device 1104 * structure. 1105 * 1106 * We return the cached information from a previous hardware probe. The 1107 * actual hardware probing itself was done either from within 1108 * ahci_initialize_controller() during the driver attach or from a phy 1109 * ready change interrupt handler. 1110 */ 1111 static int 1112 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1113 { 1114 ahci_ctl_t *ahci_ctlp; 1115 ahci_port_t *ahci_portp; 1116 uint8_t cport = sd->satadev_addr.cport; 1117 #if AHCI_DEBUG 1118 uint8_t pmport = sd->satadev_addr.pmport; 1119 uint8_t qual = sd->satadev_addr.qual; 1120 #endif 1121 uint8_t device_type; 1122 uint32_t port_state; 1123 uint8_t port; 1124 1125 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1126 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1127 1128 AHCIDBG3(AHCIDBG_ENTRY, ahci_ctlp, 1129 "ahci_tran_probe_port enter: cport: %d, " 1130 "pmport: %d, qual: %d", cport, pmport, qual); 1131 1132 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1133 1134 mutex_enter(&ahci_portp->ahciport_mutex); 1135 1136 port_state = ahci_portp->ahciport_port_state; 1137 switch (port_state) { 1138 1139 case SATA_PSTATE_FAILED: 1140 sd->satadev_state = SATA_PSTATE_FAILED; 1141 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1142 "ahci_tran_probe_port: port %d PORT FAILED", port); 1143 goto out; 1144 1145 case SATA_PSTATE_SHUTDOWN: 1146 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1147 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1148 "ahci_tran_probe_port: port %d PORT SHUTDOWN", port); 1149 goto out; 1150 1151 case SATA_PSTATE_PWROFF: 1152 sd->satadev_state = SATA_PSTATE_PWROFF; 1153 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1154 "ahci_tran_probe_port: port %d PORT PWROFF", port); 1155 goto out; 1156 1157 case SATA_PSTATE_PWRON: 1158 sd->satadev_state = SATA_PSTATE_PWRON; 1159 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1160 "ahci_tran_probe_port: port %d PORT PWRON", port); 1161 break; 1162 1163 default: 1164 sd->satadev_state = port_state; 1165 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 1166 "ahci_tran_probe_port: port %d PORT NORMAL %x", 1167 port, port_state); 1168 break; 1169 } 1170 1171 device_type = ahci_portp->ahciport_device_type; 1172 1173 switch (device_type) { 1174 1175 case SATA_DTYPE_ATADISK: 1176 sd->satadev_type = SATA_DTYPE_ATADISK; 1177 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1178 "ahci_tran_probe_port: port %d DISK found", port); 1179 break; 1180 1181 case SATA_DTYPE_ATAPI: 1182 /* 1183 * HBA driver only knows it's an ATAPI device, and don't know 1184 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1185 * type need to be determined by checking IDENTIFY PACKET 1186 * DEVICE data 1187 */ 1188 sd->satadev_type = SATA_DTYPE_ATAPI; 1189 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1190 "ahci_tran_probe_port: port %d ATAPI found", port); 1191 break; 1192 1193 case SATA_DTYPE_PMULT: 1194 sd->satadev_type = SATA_DTYPE_PMULT; 1195 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1196 "ahci_tran_probe_port: port %d Port Multiplier found", 1197 port); 1198 break; 1199 1200 case SATA_DTYPE_UNKNOWN: 1201 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1202 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1203 "ahci_tran_probe_port: port %d Unknown device found", port); 1204 break; 1205 1206 default: 1207 /* we don't support any other device types */ 1208 sd->satadev_type = SATA_DTYPE_NONE; 1209 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1210 "ahci_tran_probe_port: port %d No device found", port); 1211 break; 1212 } 1213 1214 out: 1215 ahci_update_sata_registers(ahci_ctlp, port, sd); 1216 mutex_exit(&ahci_portp->ahciport_mutex); 1217 1218 return (SATA_SUCCESS); 1219 } 1220 1221 /* 1222 * There are four operation modes in sata framework: 1223 * SATA_OPMODE_INTERRUPTS 1224 * SATA_OPMODE_POLLING 1225 * SATA_OPMODE_ASYNCH 1226 * SATA_OPMODE_SYNCH 1227 * 1228 * Their combined meanings as following: 1229 * 1230 * SATA_OPMODE_SYNCH 1231 * The command has to be completed before sata_tran_start functions returns. 1232 * Either interrupts or polling could be used - it's up to the driver. 1233 * Mode used currently for internal, sata-module initiated operations. 1234 * 1235 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1236 * It is the same as the one above. 1237 * 1238 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1239 * The command has to be completed before sata_tran_start function returns. 1240 * No interrupt used, polling only. This should be the mode used for scsi 1241 * packets with FLAG_NOINTR. 1242 * 1243 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1244 * The command may be queued (callback function specified). Interrupts could 1245 * be used. It's normal operation mode. 1246 */ 1247 /* 1248 * Called by sata framework to transport a sata packet down stream. 1249 */ 1250 static int 1251 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1252 { 1253 ahci_ctl_t *ahci_ctlp; 1254 ahci_port_t *ahci_portp; 1255 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1256 uint8_t port; 1257 1258 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1259 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1260 1261 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, 1262 "ahci_tran_start enter: cport %d satapkt 0x%p", 1263 cport, (void *)spkt); 1264 1265 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1266 1267 mutex_enter(&ahci_portp->ahciport_mutex); 1268 1269 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1270 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1271 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1272 /* 1273 * In case the targer driver would send the packet before 1274 * sata framework can have the opportunity to process those 1275 * event reports. 1276 */ 1277 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1278 spkt->satapkt_device.satadev_state = 1279 ahci_portp->ahciport_port_state; 1280 ahci_update_sata_registers(ahci_ctlp, port, 1281 &spkt->satapkt_device); 1282 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1283 "ahci_tran_start returning PORT_ERROR while " 1284 "port in FAILED/SHUTDOWN/PWROFF state: " 1285 "port: %d", port); 1286 mutex_exit(&ahci_portp->ahciport_mutex); 1287 return (SATA_TRAN_PORT_ERROR); 1288 } 1289 1290 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1291 /* 1292 * ahci_intr_phyrdy_change() may have rendered it to 1293 * SATA_DTYPE_NONE. 1294 */ 1295 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1296 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1297 spkt->satapkt_device.satadev_state = 1298 ahci_portp->ahciport_port_state; 1299 ahci_update_sata_registers(ahci_ctlp, port, 1300 &spkt->satapkt_device); 1301 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1302 "ahci_tran_start returning PORT_ERROR while " 1303 "no device attached: port: %d", port); 1304 mutex_exit(&ahci_portp->ahciport_mutex); 1305 return (SATA_TRAN_PORT_ERROR); 1306 } 1307 1308 /* 1309 * SATA HBA driver should remember that a device was reset and it 1310 * is supposed to reject any packets which do not specify either 1311 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1312 * 1313 * This is to prevent a race condition when a device was arbitrarily 1314 * reset by the HBA driver (and lost it's setting) and a target 1315 * driver sending some commands to a device before the sata framework 1316 * has a chance to restore the device setting (such as cache enable/ 1317 * disable or other resettable stuff). 1318 */ 1319 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1320 ahci_portp->ahciport_reset_in_progress = 0; 1321 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1322 "ahci_tran_start clearing the " 1323 "reset_in_progress for port: %d", port); 1324 } 1325 1326 if (ahci_portp->ahciport_reset_in_progress && 1327 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1328 ! ddi_in_panic()) { 1329 spkt->satapkt_reason = SATA_PKT_BUSY; 1330 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1331 "ahci_tran_start returning BUSY while " 1332 "reset in progress: port: %d", port); 1333 mutex_exit(&ahci_portp->ahciport_mutex); 1334 return (SATA_TRAN_BUSY); 1335 } 1336 1337 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1338 spkt->satapkt_reason = SATA_PKT_BUSY; 1339 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1340 "ahci_tran_start returning BUSY while " 1341 "mopping in progress: port: %d", port); 1342 mutex_exit(&ahci_portp->ahciport_mutex); 1343 return (SATA_TRAN_BUSY); 1344 } 1345 1346 if (spkt->satapkt_op_mode & 1347 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1348 /* 1349 * If a SYNC command to be executed in interrupt context, 1350 * bounce it back to sata module. 1351 */ 1352 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1353 servicing_interrupt()) { 1354 spkt->satapkt_reason = SATA_PKT_BUSY; 1355 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1356 "ahci_tran_start returning BUSY while " 1357 "sending SYNC mode under interrupt context: " 1358 "port : %d", port); 1359 mutex_exit(&ahci_portp->ahciport_mutex); 1360 return (SATA_TRAN_BUSY); 1361 } 1362 1363 /* We need to do the sync start now */ 1364 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, port, 1365 spkt) == AHCI_FAILURE) { 1366 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1367 "return QUEUE_FULL: port %d", port); 1368 mutex_exit(&ahci_portp->ahciport_mutex); 1369 return (SATA_TRAN_QUEUE_FULL); 1370 } 1371 } else { 1372 /* Async start, using interrupt */ 1373 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, port, spkt) 1374 == AHCI_FAILURE) { 1375 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1376 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1377 "returning QUEUE_FULL: port %d", port); 1378 mutex_exit(&ahci_portp->ahciport_mutex); 1379 return (SATA_TRAN_QUEUE_FULL); 1380 } 1381 } 1382 1383 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1384 "sata tran accepted: port %d", port); 1385 1386 mutex_exit(&ahci_portp->ahciport_mutex); 1387 return (SATA_TRAN_ACCEPTED); 1388 } 1389 1390 /* 1391 * SATA_OPMODE_SYNCH flag is set 1392 * 1393 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1394 * without interrupt, otherwise we can still use the interrupt. 1395 * 1396 * WARNING!!! ahciport_mutex should be acquired before the function 1397 * is called. 1398 */ 1399 static int 1400 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1401 uint8_t port, sata_pkt_t *spkt) 1402 { 1403 int pkt_timeout_ticks; 1404 uint32_t timeout_tags; 1405 int rval; 1406 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1407 1408 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1409 "port %d spkt 0x%p", port, spkt); 1410 1411 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1412 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1413 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1414 port, spkt)) == AHCI_FAILURE) { 1415 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1416 return (rval); 1417 } 1418 1419 pkt_timeout_ticks = 1420 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1421 1422 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1423 mutex_exit(&ahci_portp->ahciport_mutex); 1424 1425 /* Simulate the interrupt */ 1426 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1427 1428 drv_usecwait(AHCI_1MS_USECS); 1429 1430 mutex_enter(&ahci_portp->ahciport_mutex); 1431 pkt_timeout_ticks -= AHCI_1MS_TICKS; 1432 if (pkt_timeout_ticks < 0) { 1433 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1434 "port %d satapkt 0x%p timed out\n", 1435 instance, port, (void *)spkt); 1436 timeout_tags = (0x1 << rval); 1437 mutex_exit(&ahci_portp->ahciport_mutex); 1438 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1439 port, timeout_tags); 1440 mutex_enter(&ahci_portp->ahciport_mutex); 1441 } 1442 } 1443 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1444 return (AHCI_SUCCESS); 1445 1446 } else { 1447 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1448 port, spkt)) == AHCI_FAILURE) 1449 return (rval); 1450 1451 #if AHCI_DEBUG 1452 /* 1453 * Note that the driver always uses the slot 0 to deliver 1454 * REQUEST SENSE or READ LOG EXT command 1455 */ 1456 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1457 ASSERT(rval == 0); 1458 #endif 1459 1460 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1461 cv_wait(&ahci_portp->ahciport_cv, 1462 &ahci_portp->ahciport_mutex); 1463 1464 return (AHCI_SUCCESS); 1465 } 1466 } 1467 1468 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1469 if (satapkt) { \ 1470 satapkt->satapkt_reason = reason; \ 1471 /* \ 1472 * We set the satapkt_reason in both sync and \ 1473 * non-sync cases. \ 1474 */ \ 1475 } \ 1476 if (satapkt && \ 1477 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1478 satapkt->satapkt_comp) { \ 1479 mutex_exit(&ahci_portp->ahciport_mutex); \ 1480 (*satapkt->satapkt_comp)(satapkt); \ 1481 mutex_enter(&ahci_portp->ahciport_mutex); \ 1482 } else { \ 1483 if (satapkt && \ 1484 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1485 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1486 cv_broadcast(&ahci_portp->ahciport_cv); \ 1487 } 1488 1489 /* 1490 * Searches for and claims a free command slot. 1491 * 1492 * Returns: 1493 * 1494 * AHCI_FAILURE if failed 1495 * 1. if no empty slot left 1496 * 2. non-queued command requested while queued command(s) is outstanding 1497 * 3. queued command requested whild non-queued command(s) is outstanding 1498 * 4. HBA doesn't support multiple-use of command list while already a 1499 * non-queued command is oustanding 1500 * 1501 * claimed slot number if succeeded 1502 * 1503 * NOTE: it will always return slot 0 during error recovery process for 1504 * REQUEST SENSE or READ LOG EXT command to simplify the algorithm. 1505 * 1506 * WARNING!!! ahciport_mutex should be acquired before the function 1507 * is called. 1508 */ 1509 static int 1510 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, 1511 ahci_port_t *ahci_portp, int command_type) 1512 { 1513 uint32_t free_slots; 1514 int slot; 1515 1516 AHCIDBG2(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1517 "ahciport_pending_tags = 0x%x " 1518 "ahciport_pending_ncq_tags = 0x%x", 1519 ahci_portp->ahciport_pending_tags, 1520 ahci_portp->ahciport_pending_ncq_tags); 1521 1522 /* 1523 * According to the AHCI spec, system software is responsible to 1524 * ensure that queued and non-queued commands are not mixed in 1525 * the command list. 1526 */ 1527 if (command_type == AHCI_NON_NCQ_CMD) { 1528 /* Non-NCQ command request */ 1529 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1530 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1531 "ahci_claim_free_slot: there is still pending " 1532 "queued command(s) in the command list, " 1533 "so no available slot for the non-queued " 1534 "command"); 1535 return (AHCI_FAILURE); 1536 } 1537 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1538 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1539 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 1540 "ahci_claim_free_slot: HBA cannot support multiple-" 1541 "use of the command list for non-queued commands"); 1542 return (AHCI_FAILURE); 1543 } 1544 free_slots = (~ahci_portp->ahciport_pending_tags) & 1545 AHCI_SLOT_MASK(ahci_ctlp); 1546 } else if (command_type == AHCI_NCQ_CMD) { 1547 /* NCQ command request */ 1548 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1549 AHCIDBG0(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1550 "ahci_claim_free_slot: there is still pending " 1551 "non-queued command(s) in the command list, " 1552 "so no available slot for the queued command"); 1553 return (AHCI_FAILURE); 1554 } 1555 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1556 AHCI_NCQ_SLOT_MASK(ahci_portp); 1557 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1558 /* Error retrieval command request */ 1559 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 1560 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1561 "SENSE or READ LOG EXT command"); 1562 slot = 0; 1563 goto out; 1564 } 1565 1566 slot = ddi_ffs(free_slots) - 1; 1567 if (slot == -1) { 1568 AHCIDBG0(AHCIDBG_VERBOSE, ahci_ctlp, 1569 "ahci_claim_free_slot: no empty slots"); 1570 return (AHCI_FAILURE); 1571 } 1572 1573 /* 1574 * According to the AHCI spec, to allow a simple mechanism for the 1575 * HBA to map command list slots to queue entries, software must 1576 * match the tag number it uses to the slot it is placing the command 1577 * in. For example, if a queued command is placed in slot 5, the tag 1578 * for that command must be 5. 1579 */ 1580 if (command_type == AHCI_NCQ_CMD) { 1581 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1582 } 1583 1584 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1585 1586 out: 1587 AHCIDBG1(AHCIDBG_VERBOSE, ahci_ctlp, 1588 "ahci_claim_free_slot: found slot: 0x%x", slot); 1589 1590 return (slot); 1591 } 1592 1593 /* 1594 * Builds the Command Table for the sata packet and delivers it to controller. 1595 * 1596 * Returns: 1597 * slot number if we can obtain a slot successfully 1598 * otherwise, return AHCI_FAILURE 1599 * 1600 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1601 */ 1602 static int 1603 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1604 uint8_t port, sata_pkt_t *spkt) 1605 { 1606 int cmd_slot; 1607 sata_cmd_t *scmd; 1608 ahci_fis_h2d_register_t *h2d_register_fisp; 1609 ahci_cmd_table_t *cmd_table; 1610 ahci_cmd_header_t *cmd_header; 1611 int ncookies; 1612 int i; 1613 int command_type = AHCI_NON_NCQ_CMD; 1614 int ncq_qdepth; 1615 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1616 #if AHCI_DEBUG 1617 uint32_t *ptr; 1618 uint8_t *ptr2; 1619 #endif 1620 1621 spkt->satapkt_reason = SATA_PKT_BUSY; 1622 1623 scmd = &spkt->satapkt_cmd; 1624 1625 /* Check if the command is a NCQ command */ 1626 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1627 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1628 command_type = AHCI_NCQ_CMD; 1629 1630 /* 1631 * When NCQ is support, system software must determine the 1632 * maximum tag allowed by the device and the HBA, and it 1633 * must use a value not beyond of the lower bound of the two. 1634 * 1635 * Sata module is going to calculate the qdepth and send 1636 * down to HBA driver via sata_cmd. 1637 */ 1638 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 1639 1640 /* 1641 * At the moment, the driver doesn't support the dynamic 1642 * setting of the maximum ncq depth, and the value can be 1643 * set either during the attach or after hot-plug insertion. 1644 */ 1645 if (ahci_portp->ahciport_max_ncq_tags == 0) { 1646 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 1647 AHCIDBG2(AHCIDBG_NCQ, ahci_ctlp, 1648 "ahci_deliver_satapkt: port %d the max tags for " 1649 "NCQ command is %d", port, ncq_qdepth); 1650 } else { 1651 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 1652 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 1653 " port %d the max tag for NCQ command is " 1654 "requested to change from %d to %d, at the" 1655 " moment the driver doesn't support the " 1656 "dynamic change so it's going to " 1657 "still use the previous tag value", 1658 instance, port, 1659 ahci_portp->ahciport_max_ncq_tags, 1660 ncq_qdepth); 1661 } 1662 } 1663 } 1664 1665 /* Check if the command is an error retrieval command */ 1666 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1667 command_type = AHCI_ERR_RETRI_CMD; 1668 1669 /* Check if there is an empty command slot */ 1670 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, command_type); 1671 if (cmd_slot == AHCI_FAILURE) { 1672 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, "no free command slot"); 1673 return (AHCI_FAILURE); 1674 } 1675 1676 AHCIDBG4(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 1677 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 1678 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 1679 cmd_slot, port, (void *)spkt); 1680 1681 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 1682 bzero((void *)cmd_table, ahci_cmd_table_size); 1683 1684 /* For data transfer operations, it is the H2D Register FIS */ 1685 h2d_register_fisp = 1686 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 1687 1688 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 1689 if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) || 1690 (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) { 1691 SET_FIS_PMP(h2d_register_fisp, 1692 spkt->satapkt_device.satadev_addr.pmport); 1693 } 1694 1695 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 1696 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 1697 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 1698 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 1699 1700 switch (scmd->satacmd_addr_type) { 1701 1702 case 0: 1703 /* 1704 * satacmd_addr_type will be 0 for the commands below: 1705 * ATAPI command 1706 * SATAC_IDLE_IM 1707 * SATAC_STANDBY_IM 1708 * SATAC_DOWNLOAD_MICROCODE 1709 * SATAC_FLUSH_CACHE 1710 * SATAC_SET_FEATURES 1711 * SATAC_SMART 1712 * SATAC_ID_PACKET_DEVICE 1713 * SATAC_ID_DEVICE 1714 */ 1715 /* FALLTHRU */ 1716 1717 case ATA_ADDR_LBA: 1718 /* FALLTHRU */ 1719 1720 case ATA_ADDR_LBA28: 1721 /* LBA[7:0] */ 1722 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1723 1724 /* LBA[15:8] */ 1725 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1726 1727 /* LBA[23:16] */ 1728 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1729 1730 /* LBA [27:24] (also called dev_head) */ 1731 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 1732 1733 break; 1734 1735 case ATA_ADDR_LBA48: 1736 /* LBA[7:0] */ 1737 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 1738 1739 /* LBA[15:8] */ 1740 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 1741 1742 /* LBA[23:16] */ 1743 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 1744 1745 /* LBA [31:24] */ 1746 SET_FIS_SECTOR_EXP(h2d_register_fisp, 1747 scmd->satacmd_lba_low_msb); 1748 1749 /* LBA [39:32] */ 1750 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 1751 scmd->satacmd_lba_mid_msb); 1752 1753 /* LBA [47:40] */ 1754 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 1755 scmd->satacmd_lba_high_msb); 1756 1757 /* Set dev_head */ 1758 SET_FIS_DEV_HEAD(h2d_register_fisp, 1759 scmd->satacmd_device_reg); 1760 1761 /* Set the extended sector count and features */ 1762 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 1763 scmd->satacmd_sec_count_msb); 1764 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1765 scmd->satacmd_features_reg_ext); 1766 break; 1767 } 1768 1769 /* 1770 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 1771 * filled into features field, and sector count 8:15 is filled into 1772 * features (exp) field. TAG is filled into sector field. 1773 */ 1774 if (command_type == AHCI_NCQ_CMD) { 1775 SET_FIS_FEATURES(h2d_register_fisp, 1776 scmd->satacmd_sec_count_lsb); 1777 SET_FIS_FEATURES_EXP(h2d_register_fisp, 1778 scmd->satacmd_sec_count_msb); 1779 1780 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 1781 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 1782 } 1783 1784 ncookies = scmd->satacmd_num_dma_cookies; 1785 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1786 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 1787 ncookies, ahci_dma_prdt_number); 1788 1789 ASSERT(ncookies <= ahci_dma_prdt_number); 1790 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 1791 1792 /* *** now fill the scatter gather list ******* */ 1793 for (i = 0; i < ncookies; i++) { 1794 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 1795 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 1796 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 1797 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 1798 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 1799 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 1800 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 1801 scmd->satacmd_dma_cookie_list[i].dmac_size; 1802 } 1803 1804 AHCIDBG2(AHCIDBG_PRDT, ahci_ctlp, 1805 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 1806 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 1807 1808 /* The ACMD field is filled in for ATAPI command */ 1809 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 1810 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 1811 SATA_ATAPI_MAX_CDB_LEN); 1812 } 1813 1814 /* Set Command Header in Command List */ 1815 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 1816 BZERO_DESCR_INFO(cmd_header); 1817 BZERO_PRD_BYTE_COUNT(cmd_header); 1818 1819 /* Set the number of entries in the PRD table */ 1820 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 1821 1822 /* Set the length of the command in the CFIS area */ 1823 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 1824 1825 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 1826 "sata_data_direction = 0x%x", 1827 scmd->satacmd_flags.sata_data_direction); 1828 1829 /* Set A bit if it is an ATAPI command */ 1830 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 1831 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 1832 1833 /* Set W bit if data is going to the device */ 1834 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 1835 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 1836 1837 /* 1838 * Set the prefetchable bit - this bit is only valid if the PRDTL 1839 * field is non-zero or the ATAPI 'A' bit is set in the command 1840 * header. This bit cannot be set when using native command 1841 * queuing commands or when using FIS-based switching with a Port 1842 * multiplier. 1843 */ 1844 if (command_type != AHCI_NCQ_CMD) 1845 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 1846 1847 /* Now remember the sata packet in ahciport_slot_pkts[]. */ 1848 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1849 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 1850 1851 /* 1852 * We are overloading satapkt_hba_driver_private with 1853 * watched_cycle count. 1854 */ 1855 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 1856 1857 #if AHCI_DEBUG 1858 if (ahci_debug_flags & AHCIDBG_ATACMD && 1859 scmd->satacmd_cmd_reg != SATAC_PACKET || 1860 ahci_debug_flags & AHCIDBG_ATAPICMD && 1861 scmd->satacmd_cmd_reg == SATAC_PACKET) { 1862 1863 /* Dump the command header and table */ 1864 ahci_log(ahci_ctlp, CE_WARN, "\n"); 1865 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 1866 "0x%p cmd_reg 0x%x port %d", spkt, 1867 scmd->satacmd_cmd_reg, port); 1868 ptr = (uint32_t *)cmd_header; 1869 ahci_log(ahci_ctlp, CE_WARN, 1870 " Command Header:%8x %8x %8x %8x", 1871 ptr[0], ptr[1], ptr[2], ptr[3]); 1872 1873 /* Dump the H2D register FIS */ 1874 ptr = (uint32_t *)h2d_register_fisp; 1875 ahci_log(ahci_ctlp, CE_WARN, 1876 " Command FIS: %8x %8x %8x %8x", 1877 ptr[0], ptr[1], ptr[2], ptr[3]); 1878 1879 /* Dump the ACMD register FIS */ 1880 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 1881 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 1882 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 1883 ahci_log(ahci_ctlp, CE_WARN, 1884 " ATAPI command: %2x %2x %2x %2x " 1885 "%2x %2x %2x %2x", 1886 ptr2[8 * i], ptr2[8 * i + 1], 1887 ptr2[8 * i + 2], ptr2[8 * i + 3], 1888 ptr2[8 * i + 4], ptr2[8 * i + 5], 1889 ptr2[8 * i + 6], ptr2[8 * i + 7]); 1890 1891 /* Dump the PRDT */ 1892 for (i = 0; i < ncookies; i++) { 1893 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 1894 ahci_log(ahci_ctlp, CE_WARN, 1895 " Cookie %d: %8x %8x %8x %8x", 1896 i, ptr[0], ptr[1], ptr[2], ptr[3]); 1897 } 1898 } 1899 #endif 1900 1901 (void) ddi_dma_sync( 1902 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 1903 0, 1904 ahci_cmd_table_size, 1905 DDI_DMA_SYNC_FORDEV); 1906 1907 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 1908 cmd_slot * sizeof (ahci_cmd_header_t), 1909 sizeof (ahci_cmd_header_t), 1910 DDI_DMA_SYNC_FORDEV); 1911 1912 /* Set the corresponding bit in the PxSACT.DS for queued command */ 1913 if (command_type == AHCI_NCQ_CMD) { 1914 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1915 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 1916 (0x1 << cmd_slot)); 1917 } 1918 1919 /* Indicate to the HBA that a command is active. */ 1920 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 1921 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 1922 (0x1 << cmd_slot)); 1923 1924 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 1925 "exit: port %d", port); 1926 1927 return (cmd_slot); 1928 } 1929 1930 /* 1931 * Called by the sata framework to abort the previously sent packet(s). 1932 * 1933 * Reset device to abort commands. 1934 */ 1935 static int 1936 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 1937 { 1938 ahci_ctl_t *ahci_ctlp; 1939 ahci_port_t *ahci_portp; 1940 uint32_t slot_status = 0; 1941 uint32_t aborted_tags = 0; 1942 uint32_t finished_tags = 0; 1943 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1944 uint8_t port; 1945 int tmp_slot; 1946 int instance = ddi_get_instance(dip); 1947 1948 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1949 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1950 1951 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 1952 "ahci_tran_abort enter: port %d", port); 1953 1954 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1955 mutex_enter(&ahci_portp->ahciport_mutex); 1956 1957 /* 1958 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 1959 * commands are being mopped, therefore there is nothing else to do 1960 */ 1961 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1962 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 1963 "ahci_tran_abort: port %d is in " 1964 "mopping process, so just return directly ", port); 1965 mutex_exit(&ahci_portp->ahciport_mutex); 1966 return (SATA_SUCCESS); 1967 } 1968 1969 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 1970 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 1971 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 1972 /* 1973 * In case the targer driver would send the request before 1974 * sata framework can have the opportunity to process those 1975 * event reports. 1976 */ 1977 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1978 spkt->satapkt_device.satadev_state = 1979 ahci_portp->ahciport_port_state; 1980 ahci_update_sata_registers(ahci_ctlp, port, 1981 &spkt->satapkt_device); 1982 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 1983 "ahci_tran_abort returning SATA_FAILURE while " 1984 "port in FAILED/SHUTDOWN/PWROFF state: " 1985 "port: %d", port); 1986 mutex_exit(&ahci_portp->ahciport_mutex); 1987 return (SATA_FAILURE); 1988 } 1989 1990 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 1991 /* 1992 * ahci_intr_phyrdy_change() may have rendered it to 1993 * AHCI_PORT_TYPE_NODEV. 1994 */ 1995 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1996 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1997 spkt->satapkt_device.satadev_state = 1998 ahci_portp->ahciport_port_state; 1999 ahci_update_sata_registers(ahci_ctlp, port, 2000 &spkt->satapkt_device); 2001 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2002 "ahci_tran_abort returning SATA_FAILURE while " 2003 "no device attached: port: %d", port); 2004 mutex_exit(&ahci_portp->ahciport_mutex); 2005 return (SATA_FAILURE); 2006 } 2007 2008 if (flag == SATA_ABORT_ALL_PACKETS) { 2009 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2010 aborted_tags = ahci_portp->ahciport_pending_tags; 2011 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2012 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2013 2014 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2015 instance, port); 2016 } else { 2017 aborted_tags = 0xffffffff; 2018 /* 2019 * Aborting one specific packet, first search the 2020 * ahciport_slot_pkts[] list for matching spkt. 2021 */ 2022 for (tmp_slot = 0; 2023 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2024 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2025 aborted_tags = (0x1 << tmp_slot); 2026 break; 2027 } 2028 } 2029 2030 if (aborted_tags == 0xffffffff) { 2031 /* request packet is not on the pending list */ 2032 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 2033 "Cannot find the aborting pkt 0x%p on the " 2034 "pending list", (void *)spkt); 2035 ahci_update_sata_registers(ahci_ctlp, port, 2036 &spkt->satapkt_device); 2037 mutex_exit(&ahci_portp->ahciport_mutex); 2038 return (SATA_FAILURE); 2039 } 2040 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2041 instance, port, (void *)spkt); 2042 } 2043 2044 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2045 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2046 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2047 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2048 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2049 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2050 2051 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2052 ahci_portp->ahciport_mop_in_progress++; 2053 2054 /* 2055 * To abort the packet(s), first we are trying to clear PxCMD.ST 2056 * to stop the port, and if the port can be stopped 2057 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2058 * then we just send back the aborted packet(s) with ABORTED flag 2059 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2060 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2061 * perform a COMRESET. 2062 */ 2063 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2064 ahci_portp, port, NULL, NULL); 2065 2066 /* 2067 * Compute which have finished and which need to be retried. 2068 * 2069 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2070 * minus the slot_status. The aborted_tags has to be deducted by 2071 * finished_tags since we can't possibly abort a tag which had finished 2072 * already. 2073 */ 2074 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2075 finished_tags = ahci_portp->ahciport_pending_tags & 2076 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2077 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2078 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2079 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2080 2081 aborted_tags &= ~finished_tags; 2082 2083 ahci_mop_commands(ahci_ctlp, 2084 ahci_portp, 2085 slot_status, 2086 0, /* failed tags */ 2087 0, /* timeout tags */ 2088 aborted_tags, 2089 0); /* reset tags */ 2090 2091 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2092 mutex_exit(&ahci_portp->ahciport_mutex); 2093 2094 return (SATA_SUCCESS); 2095 } 2096 2097 /* 2098 * Used to do device reset and reject all the pending packets on a device 2099 * during the reset operation. 2100 * 2101 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2102 */ 2103 static int 2104 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2105 ahci_port_t *ahci_portp, uint8_t port) 2106 { 2107 uint32_t slot_status = 0; 2108 uint32_t reset_tags = 0; 2109 uint32_t finished_tags = 0; 2110 sata_device_t sdevice; 2111 int ret; 2112 2113 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2114 "ahci_reset_device_reject_pkts on port: %d", port); 2115 2116 /* 2117 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2118 * commands are being mopped, therefore there is nothing else to do 2119 */ 2120 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2121 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2122 "ahci_reset_device_reject_pkts: port %d is in " 2123 "mopping process, so return directly ", port); 2124 return (SATA_SUCCESS); 2125 } 2126 2127 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2128 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2129 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2130 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2131 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2132 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2133 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2134 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2135 } 2136 2137 if (ahci_software_reset(ahci_ctlp, ahci_portp, port) 2138 != AHCI_SUCCESS) { 2139 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2140 "Try to do a port reset after software " 2141 "reset failed", port); 2142 ret = ahci_port_reset(ahci_ctlp, ahci_portp, port); 2143 if (ret != AHCI_SUCCESS) { 2144 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2145 "ahci_reset_device_reject_pkts: port %d " 2146 "failed", port); 2147 return (SATA_FAILURE); 2148 } 2149 } 2150 /* Set the reset in progress flag */ 2151 ahci_portp->ahciport_reset_in_progress = 1; 2152 2153 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2154 ahci_portp->ahciport_mop_in_progress++; 2155 2156 /* Indicate to the framework that a reset has happened */ 2157 bzero((void *)&sdevice, sizeof (sata_device_t)); 2158 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2159 sdevice.satadev_addr.pmport = 0; 2160 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2161 2162 sdevice.satadev_state = SATA_DSTATE_RESET | 2163 SATA_DSTATE_PWR_ACTIVE; 2164 mutex_exit(&ahci_portp->ahciport_mutex); 2165 sata_hba_event_notify( 2166 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2167 &sdevice, 2168 SATA_EVNT_DEVICE_RESET); 2169 mutex_enter(&ahci_portp->ahciport_mutex); 2170 2171 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 2172 "port %d sending event up: SATA_EVNT_RESET", port); 2173 2174 /* Next try to mop the pending commands */ 2175 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2176 finished_tags = ahci_portp->ahciport_pending_tags & 2177 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2178 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2179 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2180 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2181 2182 reset_tags &= ~finished_tags; 2183 2184 ahci_mop_commands(ahci_ctlp, 2185 ahci_portp, 2186 slot_status, 2187 0, /* failed tags */ 2188 0, /* timeout tags */ 2189 0, /* aborted tags */ 2190 reset_tags); /* reset tags */ 2191 2192 return (SATA_SUCCESS); 2193 } 2194 2195 /* 2196 * Used to do port reset and reject all the pending packets on a port during 2197 * the reset operation. 2198 * 2199 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2200 */ 2201 static int 2202 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2203 ahci_port_t *ahci_portp, uint8_t port) 2204 { 2205 uint32_t slot_status = 0; 2206 uint32_t reset_tags = 0; 2207 uint32_t finished_tags = 0; 2208 2209 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2210 "ahci_reset_port_reject_pkts on port: %d", port); 2211 2212 /* 2213 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2214 * commands are being mopped, therefore there is nothing else to do 2215 */ 2216 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2217 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2218 "ahci_reset_port_reject_pkts: port %d is in " 2219 "mopping process, so return directly ", port); 2220 return (SATA_SUCCESS); 2221 } 2222 2223 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2224 ahci_portp->ahciport_mop_in_progress++; 2225 2226 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2227 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2228 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2229 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2230 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2231 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2232 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2233 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2234 } 2235 2236 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2237 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2238 NULL) != AHCI_SUCCESS) 2239 return (SATA_FAILURE); 2240 2241 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2242 finished_tags = ahci_portp->ahciport_pending_tags & 2243 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2244 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2245 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2246 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2247 2248 reset_tags &= ~finished_tags; 2249 2250 ahci_mop_commands(ahci_ctlp, 2251 ahci_portp, 2252 slot_status, 2253 0, /* failed tags */ 2254 0, /* timeout tags */ 2255 0, /* aborted tags */ 2256 reset_tags); /* reset tags */ 2257 2258 return (SATA_SUCCESS); 2259 } 2260 2261 /* 2262 * Used to do hba reset and reject all the pending packets on all ports 2263 * during the reset operation. 2264 */ 2265 static int 2266 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2267 { 2268 ahci_port_t *ahci_portp; 2269 uint32_t slot_status[AHCI_MAX_PORTS]; 2270 uint32_t reset_tags[AHCI_MAX_PORTS]; 2271 uint32_t finished_tags[AHCI_MAX_PORTS]; 2272 uint8_t port; 2273 int ret = SATA_SUCCESS; 2274 2275 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, 2276 "ahci_reset_hba_reject_pkts enter"); 2277 2278 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2279 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2280 continue; 2281 } 2282 2283 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2284 2285 mutex_enter(&ahci_portp->ahciport_mutex); 2286 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2287 slot_status[port] = ddi_get32( 2288 ahci_ctlp->ahcictl_ahci_acc_handle, 2289 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2290 reset_tags[port] = slot_status[port] & 2291 AHCI_SLOT_MASK(ahci_ctlp); 2292 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2293 slot_status[port] = ddi_get32( 2294 ahci_ctlp->ahcictl_ahci_acc_handle, 2295 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2296 reset_tags[port] = slot_status[port] & 2297 AHCI_NCQ_SLOT_MASK(ahci_portp); 2298 } 2299 mutex_exit(&ahci_portp->ahciport_mutex); 2300 } 2301 2302 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2303 ret = SATA_FAILURE; 2304 goto out; 2305 } 2306 2307 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2308 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2309 continue; 2310 } 2311 2312 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2313 2314 mutex_enter(&ahci_portp->ahciport_mutex); 2315 /* 2316 * To prevent recursive enter to ahci_mop_commands, we need 2317 * check AHCI_PORT_FLAG_MOPPING flag. 2318 */ 2319 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2320 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2321 "ahci_reset_hba_reject_pkts: port %d is in " 2322 "mopping process, so return directly ", port); 2323 mutex_exit(&ahci_portp->ahciport_mutex); 2324 continue; 2325 } 2326 2327 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2328 ahci_portp->ahciport_mop_in_progress++; 2329 2330 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2331 finished_tags[port] = 2332 ahci_portp->ahciport_pending_tags & 2333 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2334 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2335 finished_tags[port] = 2336 ahci_portp->ahciport_pending_ncq_tags & 2337 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2338 2339 reset_tags[port] &= ~finished_tags[port]; 2340 2341 ahci_mop_commands(ahci_ctlp, 2342 ahci_portp, 2343 slot_status[port], 2344 0, /* failed tags */ 2345 0, /* timeout tags */ 2346 0, /* aborted tags */ 2347 reset_tags[port]); /* reset tags */ 2348 mutex_exit(&ahci_portp->ahciport_mutex); 2349 } 2350 out: 2351 return (ret); 2352 } 2353 2354 /* 2355 * Called by sata framework to reset a port(s) or device. 2356 */ 2357 static int 2358 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2359 { 2360 ahci_ctl_t *ahci_ctlp; 2361 ahci_port_t *ahci_portp; 2362 uint8_t cport = sd->satadev_addr.cport; 2363 uint8_t port; 2364 int ret = SATA_SUCCESS; 2365 int instance = ddi_get_instance(dip); 2366 2367 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2368 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2369 2370 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2371 "ahci_tran_reset_port enter: cport: %d", cport); 2372 2373 switch (sd->satadev_addr.qual) { 2374 case SATA_ADDR_CPORT: 2375 /* Port reset */ 2376 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2377 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2378 "reset port", instance, port); 2379 2380 mutex_enter(&ahci_portp->ahciport_mutex); 2381 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, port); 2382 mutex_exit(&ahci_portp->ahciport_mutex); 2383 2384 break; 2385 2386 case SATA_ADDR_DCPORT: 2387 /* Device reset */ 2388 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2389 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2390 "reset device", instance, port); 2391 2392 mutex_enter(&ahci_portp->ahciport_mutex); 2393 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2394 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2395 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2396 /* 2397 * In case the targer driver would send the request 2398 * before sata framework can have the opportunity to 2399 * process those event reports. 2400 */ 2401 sd->satadev_state = ahci_portp->ahciport_port_state; 2402 ahci_update_sata_registers(ahci_ctlp, port, sd); 2403 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2404 "ahci_tran_reset_dport returning SATA_FAILURE " 2405 "while port in FAILED/SHUTDOWN/PWROFF state: " 2406 "port: %d", port); 2407 mutex_exit(&ahci_portp->ahciport_mutex); 2408 ret = SATA_FAILURE; 2409 break; 2410 } 2411 2412 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2413 /* 2414 * ahci_intr_phyrdy_change() may have rendered it to 2415 * AHCI_PORT_TYPE_NODEV. 2416 */ 2417 sd->satadev_type = SATA_DTYPE_NONE; 2418 sd->satadev_state = ahci_portp->ahciport_port_state; 2419 ahci_update_sata_registers(ahci_ctlp, port, sd); 2420 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2421 "ahci_tran_reset_dport returning SATA_FAILURE " 2422 "while no device attached: port: %d", port); 2423 mutex_exit(&ahci_portp->ahciport_mutex); 2424 ret = SATA_FAILURE; 2425 break; 2426 } 2427 2428 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2429 ahci_portp, port); 2430 mutex_exit(&ahci_portp->ahciport_mutex); 2431 break; 2432 2433 case SATA_ADDR_CNTRL: 2434 /* Reset the whole controller */ 2435 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport port %d " 2436 "reset the whole hba", instance, port); 2437 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2438 break; 2439 2440 case SATA_ADDR_PMPORT: 2441 case SATA_ADDR_DPMPORT: 2442 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 2443 "ahci_tran_reset_dport: port multiplier will be " 2444 "supported later"); 2445 /* FALLTHRU */ 2446 default: 2447 ret = SATA_FAILURE; 2448 } 2449 2450 return (ret); 2451 } 2452 2453 /* 2454 * Called by sata framework to activate a port as part of hotplug. 2455 * (cfgadm -c connect satax/y) 2456 * Note: Not port-mult aware. 2457 */ 2458 static int 2459 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 2460 { 2461 ahci_ctl_t *ahci_ctlp; 2462 ahci_port_t *ahci_portp; 2463 uint8_t cport = satadev->satadev_addr.cport; 2464 uint8_t port; 2465 int instance = ddi_get_instance(dip); 2466 2467 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2468 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2469 2470 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2471 "ahci_tran_hotplug_port_activate cport %d enter", cport); 2472 2473 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2474 2475 mutex_enter(&ahci_portp->ahciport_mutex); 2476 ahci_enable_port_intrs(ahci_ctlp, port); 2477 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", instance, port); 2478 2479 /* 2480 * Reset the port so that the PHY communication would be re-established. 2481 * But this reset is an internal operation and the sata module doesn't 2482 * need to know about it. Moreover, the port with a device attached will 2483 * be started too. 2484 */ 2485 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2486 ahci_portp, port, 2487 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2488 NULL); 2489 2490 /* 2491 * Need to check the link status and device status of the port 2492 * and consider raising power if the port was in D3 state 2493 */ 2494 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 2495 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 2496 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 2497 2498 satadev->satadev_state = ahci_portp->ahciport_port_state; 2499 2500 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2501 2502 mutex_exit(&ahci_portp->ahciport_mutex); 2503 return (SATA_SUCCESS); 2504 } 2505 2506 /* 2507 * Called by sata framework to deactivate a port as part of hotplug. 2508 * (cfgadm -c disconnect satax/y) 2509 * Note: Not port-mult aware. 2510 */ 2511 static int 2512 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 2513 { 2514 ahci_ctl_t *ahci_ctlp; 2515 ahci_port_t *ahci_portp; 2516 uint8_t cport = satadev->satadev_addr.cport; 2517 uint8_t port; 2518 uint32_t port_scontrol; 2519 int instance = ddi_get_instance(dip); 2520 2521 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2522 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2523 2524 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 2525 "ahci_tran_hotplug_port_deactivate cport %d enter", cport); 2526 2527 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2528 2529 mutex_enter(&ahci_portp->ahciport_mutex); 2530 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 2531 instance, port); 2532 2533 /* Disable the interrupts on the port */ 2534 ahci_disable_port_intrs(ahci_ctlp, port); 2535 2536 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2537 goto phy_offline; 2538 } 2539 2540 /* First to abort all the pending commands */ 2541 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 2542 2543 /* Then stop the port */ 2544 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2545 ahci_portp, port); 2546 2547 /* Next put the PHY offline */ 2548 2549 phy_offline: 2550 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2551 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 2552 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 2553 2554 /* Update ahciport_port_state */ 2555 ahci_portp->ahciport_port_state = SATA_PSTATE_SHUTDOWN; 2556 satadev->satadev_state = ahci_portp->ahciport_port_state; 2557 2558 ahci_update_sata_registers(ahci_ctlp, port, satadev); 2559 2560 mutex_exit(&ahci_portp->ahciport_mutex); 2561 return (SATA_SUCCESS); 2562 } 2563 2564 /* 2565 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 2566 * when a device is unplugged or a port is deactivated. 2567 * 2568 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2569 */ 2570 static void 2571 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 2572 ahci_port_t *ahci_portp, uint8_t port) 2573 { 2574 uint32_t slot_status = 0; 2575 uint32_t abort_tags = 0; 2576 2577 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 2578 "ahci_reject_all_abort_pkts on port: %d", port); 2579 2580 /* 2581 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 2582 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 2583 * to get the error data, if yes when the device is removed, the 2584 * command needs to be aborted too. 2585 */ 2586 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2587 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 2588 slot_status = 0x1; 2589 abort_tags = 0x1; 2590 goto out; 2591 } else { 2592 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2593 "ahci_reject_all_abort_pkts return directly " 2594 "port %d no needs to reject any outstanding " 2595 "commands", port); 2596 return; 2597 } 2598 } 2599 2600 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2601 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2602 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2603 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2604 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2605 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2606 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2607 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2608 } 2609 2610 out: 2611 /* No need to do mop when there is no outstanding commands */ 2612 if (slot_status != 0) { 2613 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2614 ahci_portp->ahciport_mop_in_progress++; 2615 2616 ahci_mop_commands(ahci_ctlp, 2617 ahci_portp, 2618 slot_status, 2619 0, /* failed tags */ 2620 0, /* timeout tags */ 2621 abort_tags, /* aborting tags */ 2622 0); /* reset tags */ 2623 } 2624 } 2625 2626 #if defined(__lock_lint) 2627 static int 2628 ahci_selftest(dev_info_t *dip, sata_device_t *device) 2629 { 2630 return (SATA_SUCCESS); 2631 } 2632 #endif 2633 2634 /* 2635 * Allocate the ports structure, only called by ahci_attach 2636 */ 2637 static int 2638 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 2639 { 2640 int port, cport = 0; 2641 2642 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2643 "ahci_alloc_ports_state enter"); 2644 2645 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2646 2647 /* Allocate structures only for the implemented ports */ 2648 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2649 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2650 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2651 "hba port %d not implemented", port); 2652 continue; 2653 } 2654 2655 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 2656 ahci_ctlp->ahcictl_port_to_cport[port] = 2657 (uint8_t)cport++; 2658 2659 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 2660 goto err_out; 2661 } 2662 } 2663 2664 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2665 return (AHCI_SUCCESS); 2666 2667 err_out: 2668 for (port--; port >= 0; port--) { 2669 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2670 ahci_dealloc_port_state(ahci_ctlp, port); 2671 } 2672 } 2673 2674 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2675 return (AHCI_FAILURE); 2676 } 2677 2678 /* 2679 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 2680 */ 2681 static void 2682 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 2683 { 2684 int port; 2685 2686 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2687 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2688 /* if this port is implemented by the HBA */ 2689 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 2690 ahci_dealloc_port_state(ahci_ctlp, port); 2691 } 2692 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2693 } 2694 2695 /* 2696 * Drain the taskq. 2697 */ 2698 static void 2699 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 2700 { 2701 ahci_port_t *ahci_portp; 2702 int port; 2703 2704 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2705 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2706 continue; 2707 } 2708 2709 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2710 2711 mutex_enter(&ahci_portp->ahciport_mutex); 2712 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 2713 mutex_exit(&ahci_portp->ahciport_mutex); 2714 } 2715 } 2716 2717 /* 2718 * Initialize the controller and all ports. And then try to start the ports 2719 * if there are devices attached. 2720 * 2721 * This routine can be called from three seperate cases: DDI_ATTACH, 2722 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 2723 * other two cases; device signature probing are attempted only during 2724 * DDI_ATTACH case. 2725 * 2726 * WARNING!!! Disable the whole controller's interrupts before calling and 2727 * the interrupts will be enabled upon successfully return. 2728 */ 2729 static int 2730 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 2731 { 2732 ahci_port_t *ahci_portp; 2733 uint32_t ghc_control; 2734 int port; 2735 2736 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2737 "ahci_initialize_controller enter"); 2738 2739 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2740 2741 /* 2742 * Indicate that system software is AHCI aware by setting 2743 * GHC.AE to 1 2744 */ 2745 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2746 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 2747 2748 ghc_control |= AHCI_HBA_GHC_AE; 2749 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2750 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 2751 ghc_control); 2752 2753 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2754 2755 /* Initialize the implemented ports and structures */ 2756 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2757 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2758 continue; 2759 } 2760 2761 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2762 mutex_enter(&ahci_portp->ahciport_mutex); 2763 2764 /* 2765 * Ensure that the controller is not in the running state 2766 * by checking every implemented port's PxCMD register 2767 */ 2768 if (ahci_initialize_port(ahci_ctlp, ahci_portp, port) 2769 != AHCI_SUCCESS) { 2770 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 2771 "ahci_initialize_controller: failed to " 2772 "initialize port %d", port); 2773 /* 2774 * Set the port state to SATA_PSTATE_FAILED if 2775 * failed to initialize it. 2776 */ 2777 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 2778 } 2779 2780 mutex_exit(&ahci_portp->ahciport_mutex); 2781 } 2782 2783 /* Enable the whole controller interrupts */ 2784 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2785 ahci_enable_all_intrs(ahci_ctlp); 2786 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2787 2788 return (AHCI_SUCCESS); 2789 } 2790 2791 /* 2792 * Reverse of ahci_initialize_controller() 2793 * 2794 * We only need to stop the ports and disable the interrupt. 2795 */ 2796 static void 2797 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 2798 { 2799 ahci_port_t *ahci_portp; 2800 int port; 2801 2802 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2803 "ahci_uninitialize_controller enter"); 2804 2805 /* disable all the interrupts. */ 2806 mutex_enter(&ahci_ctlp->ahcictl_mutex); 2807 ahci_disable_all_intrs(ahci_ctlp); 2808 mutex_exit(&ahci_ctlp->ahcictl_mutex); 2809 2810 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2811 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2812 continue; 2813 } 2814 2815 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2816 2817 /* Stop the port by clearing PxCMD.ST */ 2818 mutex_enter(&ahci_portp->ahciport_mutex); 2819 2820 /* 2821 * Here we must disable the port interrupt because 2822 * ahci_disable_all_intrs only clear GHC.IE, and IS 2823 * register will be still set if PxIE is enabled. 2824 * When ahci shares one IRQ with other drivers, the 2825 * intr handler may claim the intr mistakenly. 2826 */ 2827 ahci_disable_port_intrs(ahci_ctlp, port); 2828 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 2829 ahci_portp, port); 2830 mutex_exit(&ahci_portp->ahciport_mutex); 2831 } 2832 } 2833 2834 /* 2835 * The routine is to initialize the port. First put the port in NOTRunning 2836 * state, then enable port interrupt and clear Serror register. And under 2837 * AHCI_ATTACH case, find device signature and then try to start the port. 2838 * 2839 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2840 */ 2841 static int 2842 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, 2843 ahci_port_t *ahci_portp, uint8_t port) 2844 { 2845 uint32_t port_cmd_status; 2846 2847 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2848 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 2849 2850 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 2851 "ahci_initialize_port: port %d " 2852 "port_cmd_status = 0x%x", port, port_cmd_status); 2853 /* 2854 * Check whether the port is in NotRunning state, if not, 2855 * put the port in NotRunning state 2856 */ 2857 if (!(port_cmd_status & 2858 (AHCI_CMD_STATUS_ST | 2859 AHCI_CMD_STATUS_CR | 2860 AHCI_CMD_STATUS_FRE | 2861 AHCI_CMD_STATUS_FR))) { 2862 2863 goto next; 2864 } 2865 2866 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 2867 port, AHCI_RESET_NO_EVENTS_UP|AHCI_PORT_INIT, NULL) != AHCI_SUCCESS) 2868 return (AHCI_FAILURE); 2869 2870 next: 2871 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2872 "port %d is in NotRunning state now", port); 2873 2874 /* 2875 * At the time being, only probe ports/devices and get the types of 2876 * attached devices during DDI_ATTACH. In fact, the device can be 2877 * changed during power state changes, but at the time being, we 2878 * don't support the situation. 2879 */ 2880 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 2881 /* Try to get the device signature */ 2882 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 2883 } else { 2884 2885 /* 2886 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 2887 * and PxFBU registers in case these registers were cleared 2888 * during the suspend. 2889 */ 2890 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2891 "ahci_initialize_port: port %d " 2892 "reset the port during resume", port); 2893 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 2894 2895 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 2896 "ahci_initialize_port: port %d " 2897 "set PxCLB, PxCLBU, PxFB and PxFBU " 2898 "during resume", port); 2899 2900 /* Config Port Received FIS Base Address */ 2901 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2902 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 2903 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 2904 2905 /* Config Port Command List Base Address */ 2906 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 2907 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 2908 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 2909 } 2910 2911 /* Disable the interface power management */ 2912 ahci_disable_interface_pm(ahci_ctlp, port); 2913 2914 /* Return directly if no device connected */ 2915 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2916 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2917 "No device connected to port %d", port); 2918 goto out; 2919 } 2920 2921 /* Try to start the port */ 2922 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 2923 != AHCI_SUCCESS) { 2924 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2925 "failed to start port %d", port); 2926 return (AHCI_FAILURE); 2927 } 2928 out: 2929 /* Enable port interrupts */ 2930 ahci_enable_port_intrs(ahci_ctlp, port); 2931 2932 return (AHCI_SUCCESS); 2933 } 2934 2935 /* 2936 * Check the hardware defects and the power management capability. 2937 */ 2938 static int 2939 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 2940 { 2941 ushort_t venid, devid; 2942 ushort_t caps_ptr, cap_count, cap; 2943 #if AHCI_DEBUG 2944 ushort_t pmcap, pmcsr; 2945 ushort_t msimc; 2946 #endif 2947 uint8_t revision; 2948 2949 venid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2950 PCI_CONF_VENID); 2951 2952 devid = pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2953 PCI_CONF_DEVID); 2954 2955 /* 2956 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 2957 * controllers with 0x00 revision id work on 4-byte aligned buffer, 2958 * which is a bug and was fixed after 0x00 revision id controllers. 2959 * 2960 * Moreover, VT8251 cannot use multiple command slots in the command 2961 * list for non-queued commands because the previous register content 2962 * of PxCI can be re-written in the register write, so a flag will be 2963 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 2964 */ 2965 if (venid == VIA_VENID) { 2966 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 2967 PCI_CONF_REVID); 2968 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 2969 "revision id = 0x%x", revision); 2970 if (revision == 0x00) { 2971 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 2972 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2973 "change ddi_attr_align to 0x4"); 2974 } 2975 2976 ahci_ctlp->ahcictl_cap = AHCI_CAP_NO_MCMDLIST_NONQUEUE; 2977 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2978 "VT8251 cannot use multiple command lists for " 2979 "non-queued commands"); 2980 } 2981 2982 /* 2983 * ATI SB600 (1002,4380) and SB700/750 (1002,4391) AHCI chipsets don't 2984 * support 64-bit DMA addressing though they declare the support, 2985 * so we need to set AHCI_CAP_32BIT_DMA flag to force 32-bit DMA. 2986 */ 2987 if (venid == 0x1002 && (devid == 0x4380 || devid == 0x4391)) { 2988 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 2989 "ATI SB600/700/750 cannot do 64-bit DMA though CAP " 2990 "indicates support, so force it to use 32-bit DMA"); 2991 ahci_ctlp->ahcictl_cap |= AHCI_CAP_32BIT_DMA; 2992 } 2993 2994 /* 2995 * Check if capabilities list is supported and if so, 2996 * get initial capabilities pointer and clear bits 0,1. 2997 */ 2998 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 2999 PCI_CONF_STAT) & PCI_STAT_CAP) { 3000 caps_ptr = P2ALIGN(pci_config_get8( 3001 ahci_ctlp->ahcictl_pci_conf_handle, 3002 PCI_CONF_CAP_PTR), 4); 3003 } else { 3004 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3005 } 3006 3007 /* 3008 * Walk capabilities if supported. 3009 */ 3010 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3011 3012 /* 3013 * Check that we haven't exceeded the maximum number of 3014 * capabilities and that the pointer is in a valid range. 3015 */ 3016 if (++cap_count > PCI_CAP_MAX_PTR) { 3017 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3018 "too many device capabilities"); 3019 return (AHCI_FAILURE); 3020 } 3021 if (caps_ptr < PCI_CAP_PTR_OFF) { 3022 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3023 "capabilities pointer 0x%x out of range", 3024 caps_ptr); 3025 return (AHCI_FAILURE); 3026 } 3027 3028 /* 3029 * Get next capability and check that it is valid. 3030 * For now, we only support power management. 3031 */ 3032 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3033 caps_ptr); 3034 switch (cap) { 3035 case PCI_CAP_ID_PM: 3036 3037 /* power management supported */ 3038 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3039 3040 /* Save PMCSR offset */ 3041 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3042 3043 #if AHCI_DEBUG 3044 pmcap = pci_config_get16( 3045 ahci_ctlp->ahcictl_pci_conf_handle, 3046 caps_ptr + PCI_PMCAP); 3047 pmcsr = pci_config_get16( 3048 ahci_ctlp->ahcictl_pci_conf_handle, 3049 ahci_ctlp->ahcictl_pmcsr_offset); 3050 AHCIDBG2(AHCIDBG_PM, ahci_ctlp, 3051 "Power Management capability found PCI_PMCAP " 3052 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3053 if ((pmcap & 0x3) == 0x3) 3054 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3055 "PCI Power Management Interface " 3056 "spec 1.2 compliant"); 3057 #endif 3058 break; 3059 3060 case PCI_CAP_ID_MSI: 3061 #if AHCI_DEBUG 3062 msimc = pci_config_get16( 3063 ahci_ctlp->ahcictl_pci_conf_handle, 3064 caps_ptr + PCI_MSI_CTRL); 3065 AHCIDBG1(AHCIDBG_MSI, ahci_ctlp, 3066 "Message Signaled Interrupt capability found " 3067 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 3068 #endif 3069 AHCIDBG0(AHCIDBG_MSI, ahci_ctlp, 3070 "MSI capability found"); 3071 break; 3072 3073 case PCI_CAP_ID_PCIX: 3074 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3075 "PCI-X capability found"); 3076 break; 3077 3078 case PCI_CAP_ID_PCI_E: 3079 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3080 "PCI Express capability found"); 3081 break; 3082 3083 case PCI_CAP_ID_MSI_X: 3084 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3085 "MSI-X capability found"); 3086 break; 3087 3088 case PCI_CAP_ID_SATA: 3089 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3090 "SATA capability found"); 3091 break; 3092 3093 case PCI_CAP_ID_VS: 3094 AHCIDBG0(AHCIDBG_PM, ahci_ctlp, 3095 "Vendor Specific capability found"); 3096 break; 3097 3098 default: 3099 AHCIDBG1(AHCIDBG_PM, ahci_ctlp, 3100 "unrecognized capability 0x%x", cap); 3101 break; 3102 } 3103 3104 /* 3105 * Get next capabilities pointer and clear bits 0,1. 3106 */ 3107 caps_ptr = P2ALIGN(pci_config_get8( 3108 ahci_ctlp->ahcictl_pci_conf_handle, 3109 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3110 } 3111 3112 return (AHCI_SUCCESS); 3113 } 3114 3115 /* 3116 * AHCI device reset ...; a single device on one of the ports is reset, 3117 * but the HBA and physical communication remain intact. This is the 3118 * least intrusive. 3119 * 3120 * When issuing a software reset sequence, there should not be other 3121 * commands in the command list, so we will first clear and then re-set 3122 * PxCMD.ST to clear PxCI. And before issuing the software reset, 3123 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 3124 * cleared unless command list override (PxCMD.CLO) is supported. 3125 * 3126 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 3127 * set before the function is called. 3128 */ 3129 static int 3130 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3131 uint8_t port) 3132 { 3133 ahci_fis_h2d_register_t *h2d_register_fisp; 3134 ahci_cmd_table_t *cmd_table; 3135 ahci_cmd_header_t *cmd_header; 3136 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 3137 int slot, loop_count; 3138 3139 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 3140 "Port %d device resetting", port); 3141 3142 /* First clear PxCMD.ST */ 3143 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3144 ahci_portp, port); 3145 3146 /* Then re-set PxCMD.ST */ 3147 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 3148 3149 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 3150 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3151 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3152 3153 if (port_task_file & AHCI_TFD_STS_BSY || 3154 port_task_file & AHCI_TFD_STS_DRQ) { 3155 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3156 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3157 if (!(port_cmd_status & AHCI_CMD_STATUS_CLO)) { 3158 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 3159 "PxTFD.STS.BSY or PxTFD.STS.DRQ is still set, " 3160 "but PxCMD.CLO isn't supported, so a port " 3161 "reset is needed."); 3162 return (AHCI_FAILURE); 3163 } 3164 } 3165 3166 slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, AHCI_NON_NCQ_CMD); 3167 if (slot == AHCI_FAILURE) { 3168 AHCIDBG0(AHCIDBG_INFO, ahci_ctlp, 3169 "ahci_software_reset: no free slot"); 3170 return (AHCI_FAILURE); 3171 } 3172 3173 /* Now send the first H2D Register FIS with SRST set to 1 */ 3174 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3175 bzero((void *)cmd_table, ahci_cmd_table_size); 3176 3177 h2d_register_fisp = 3178 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3179 3180 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3181 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3182 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 3183 3184 /* Set Command Header in Command List */ 3185 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3186 BZERO_DESCR_INFO(cmd_header); 3187 BZERO_PRD_BYTE_COUNT(cmd_header); 3188 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3189 3190 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 3191 SET_RESET(cmd_header, 1); 3192 SET_WRITE(cmd_header, 1); 3193 3194 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3195 0, 3196 ahci_cmd_table_size, 3197 DDI_DMA_SYNC_FORDEV); 3198 3199 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3200 slot * sizeof (ahci_cmd_header_t), 3201 sizeof (ahci_cmd_header_t), 3202 DDI_DMA_SYNC_FORDEV); 3203 3204 /* Indicate to the HBA that a command is active. */ 3205 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3206 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3207 (0x1 << slot)); 3208 3209 loop_count = 0; 3210 3211 /* Loop till the first command is finished */ 3212 do { 3213 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3214 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3215 3216 /* We are effectively timing out after 1 sec. */ 3217 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3218 break; 3219 } 3220 /* Wait for 10 millisec */ 3221 delay(AHCI_10MS_TICKS); 3222 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3223 3224 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3225 "ahci_software_reset: 1st loop count: %d, " 3226 "port_cmd_issue = 0x%x, slot = 0x%x", 3227 loop_count, port_cmd_issue, slot); 3228 3229 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3230 3231 /* Now send the second H2D Register FIS with SRST cleard to zero */ 3232 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 3233 bzero((void *)cmd_table, ahci_cmd_table_size); 3234 3235 h2d_register_fisp = 3236 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 3237 3238 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 3239 SET_FIS_PMP(h2d_register_fisp, AHCI_PORTMULT_CONTROL_PORT); 3240 3241 /* Set Command Header in Command List */ 3242 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 3243 BZERO_DESCR_INFO(cmd_header); 3244 BZERO_PRD_BYTE_COUNT(cmd_header); 3245 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 3246 3247 SET_WRITE(cmd_header, 1); 3248 3249 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 3250 0, 3251 ahci_cmd_table_size, 3252 DDI_DMA_SYNC_FORDEV); 3253 3254 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 3255 slot * sizeof (ahci_cmd_header_t), 3256 sizeof (ahci_cmd_header_t), 3257 DDI_DMA_SYNC_FORDEV); 3258 3259 /* Indicate to the HBA that a command is active. */ 3260 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3261 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 3262 (0x1 << slot)); 3263 3264 loop_count = 0; 3265 3266 /* Loop till the second command is finished */ 3267 do { 3268 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3269 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3270 3271 /* We are effectively timing out after 1 sec. */ 3272 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 3273 break; 3274 } 3275 /* Wait for 10 millisec */ 3276 delay(AHCI_10MS_TICKS); 3277 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 3278 3279 AHCIDBG3(AHCIDBG_POLL_LOOP, ahci_ctlp, 3280 "ahci_software_reset: 2nd loop count: %d, " 3281 "port_cmd_issue = 0x%x, slot = 0x%x", 3282 loop_count, port_cmd_issue, slot); 3283 3284 CLEAR_BIT(ahci_portp->ahciport_pending_tags, slot); 3285 3286 return (AHCI_SUCCESS); 3287 } 3288 3289 /* 3290 * AHCI port reset ...; the physical communication between the HBA and device 3291 * on a port are disabled. This is more intrusive. 3292 * 3293 * When an HBA or port reset occurs, Phy communication is going to 3294 * be re-established with the device through a COMRESET followed by the 3295 * normal out-of-band communication sequence defined in Serial ATA. AT 3296 * the end of reset, the device, if working properly, will send a D2H 3297 * Register FIS, which contains the device signature. When the HBA receives 3298 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 3299 * the PxSIG register with the signature. 3300 * 3301 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 3302 * to individually spin-up attached devices. Please refer to chapter 10.9 of 3303 * AHCI 1.0 spec. 3304 */ 3305 /* 3306 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 3307 * cleared before the function is called. 3308 */ 3309 static int 3310 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3311 { 3312 uint32_t cap_status, port_cmd_status; 3313 uint32_t port_scontrol, port_sstatus; 3314 uint32_t port_intr_status, port_task_file; 3315 #if AHCI_DEBUG 3316 uint32_t port_signature; 3317 #endif 3318 int loop_count; 3319 int rval = AHCI_SUCCESS; 3320 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 3321 3322 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3323 "Port %d port resetting...", port); 3324 ahci_portp->ahciport_port_state = 0; 3325 3326 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3327 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 3328 3329 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3330 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3331 3332 if (cap_status & AHCI_HBA_CAP_SSS) { 3333 /* 3334 * HBA support staggered spin-up, if the port has 3335 * not spin up yet, then force it to do spin-up 3336 */ 3337 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 3338 if (!(ahci_portp->ahciport_flags 3339 & AHCI_PORT_FLAG_SPINUP)) { 3340 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3341 "Port %d PxCMD.SUD is zero, force " 3342 "it to do spin-up", port); 3343 ahci_portp->ahciport_flags |= 3344 AHCI_PORT_FLAG_SPINUP; 3345 } 3346 } 3347 } else { 3348 /* 3349 * HBA doesn't support stagger spin-up, force it 3350 * to do normal COMRESET 3351 */ 3352 if (ahci_portp->ahciport_flags & 3353 AHCI_PORT_FLAG_SPINUP) { 3354 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3355 "HBA does not support staggered spin-up " 3356 "force it to do normal COMRESET"); 3357 ahci_portp->ahciport_flags &= 3358 ~AHCI_PORT_FLAG_SPINUP; 3359 } 3360 } 3361 3362 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 3363 /* Do normal COMRESET */ 3364 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3365 "ahci_port_reset: do normal COMRESET", port); 3366 3367 /* 3368 * According to the spec, SUD bit should be set here, 3369 * but JMicron JMB363 doesn't follow it, so remove 3370 * the assertion, and just print a debug message. 3371 */ 3372 #if AHCI_DEBUG 3373 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 3374 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3375 "port %d SUD bit not set", port) 3376 #endif 3377 3378 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3379 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3380 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 3381 3382 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3383 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3384 port_scontrol); 3385 3386 /* Enable PxCMD.FRE to read device */ 3387 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3388 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3389 port_cmd_status|AHCI_CMD_STATUS_FRE); 3390 3391 /* 3392 * Give time for COMRESET to percolate, according to the AHCI 3393 * spec, software shall wait at least 1 millisecond before 3394 * clearing PxSCTL.DET 3395 */ 3396 delay(AHCI_1MS_TICKS*2); 3397 3398 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 3399 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3400 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3401 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3402 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3403 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3404 port_scontrol); 3405 } else { 3406 /* Do staggered spin-up */ 3407 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3408 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3409 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 3410 3411 /* PxSCTL.DET must be 0 */ 3412 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3413 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 3414 port_scontrol); 3415 3416 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 3417 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3418 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3419 port_cmd_status); 3420 3421 /* 0 -> 1 edge */ 3422 delay(AHCI_1MS_TICKS*2); 3423 3424 /* Set PxCMD.SUD to 1 */ 3425 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3426 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3427 port_cmd_status |= AHCI_CMD_STATUS_SUD; 3428 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3429 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3430 port_cmd_status); 3431 3432 /* Enable PxCMD.FRE to read device */ 3433 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3434 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3435 port_cmd_status|AHCI_CMD_STATUS_FRE); 3436 } 3437 3438 /* 3439 * The port enters P:StartComm state, and HBA tells link layer to 3440 * start communication, which involves sending COMRESET to device. 3441 * And the HBA resets PxTFD.STS to 7Fh. 3442 * 3443 * When a COMINIT is received from the device, then the port enters 3444 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 3445 * PxSSTS.DET to 1h to indicate a device is detected but communication 3446 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 3447 * a COMINIT has been received. 3448 */ 3449 /* 3450 * The DET field is valid only if IPM field indicates 3451 * that the interface is in active state. 3452 */ 3453 loop_count = 0; 3454 do { 3455 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3456 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3457 3458 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 3459 /* 3460 * If the interface is not active, the DET field 3461 * is considered not accurate. So we want to 3462 * continue looping. 3463 */ 3464 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 3465 } 3466 3467 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 3468 /* 3469 * We are effectively timing out after 0.1 sec. 3470 */ 3471 break; 3472 } 3473 3474 /* Wait for 10 millisec */ 3475 delay(AHCI_10MS_TICKS); 3476 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 3477 3478 AHCIDBG3(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3479 "ahci_port_reset: 1st loop count: %d, " 3480 "port_sstatus = 0x%x port %d", 3481 loop_count, port_sstatus, port); 3482 3483 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 3484 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 3485 /* 3486 * Either the port is not active or there 3487 * is no device present. 3488 */ 3489 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 3490 goto out; 3491 } 3492 3493 /* Now we can make sure there is a device connected to the port */ 3494 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3495 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 3496 3497 /* a COMINIT signal is supposed to be received */ 3498 if (!(port_intr_status & AHCI_INTR_STATUS_PCS)) { 3499 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 3500 "COMINIT signal from the device not received", 3501 instance, port); 3502 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3503 rval = AHCI_FAILURE; 3504 goto out; 3505 } 3506 3507 /* 3508 * According to the spec, when PxSCTL.DET is set to 0h, upon 3509 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 3510 * shall be set to '1' by the HBA. 3511 * 3512 * However, we found JMicron JMB363 doesn't follow this, so 3513 * remove this check, and just print a debug message. 3514 */ 3515 #if AHCI_DEBUG 3516 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3517 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3518 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 3519 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 3520 "port %d BSY bit is not set after COMINIT signal " 3521 "is received", port); 3522 } 3523 #endif 3524 3525 /* 3526 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 3527 * the D2H FIS received by HBA. 3528 */ 3529 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3530 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3531 SERROR_EXCHANGED_ERR); 3532 3533 /* 3534 * Next check whether COMRESET is completed successfully 3535 */ 3536 loop_count = 0; 3537 do { 3538 port_task_file = 3539 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3540 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3541 3542 /* 3543 * The Error bit '1' means COMRESET is finished successfully 3544 * The device hardware has been initialized and the power-up 3545 * diagnostics successfully completed. 3546 */ 3547 if (((port_task_file & AHCI_TFD_ERR_MASK) 3548 >> AHCI_TFD_ERR_SHIFT) == 0x1) { 3549 #if AHCI_DEBUG 3550 port_signature = ddi_get32( 3551 ahci_ctlp->ahcictl_ahci_acc_handle, 3552 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3553 AHCIDBG2(AHCIDBG_INFO, ahci_ctlp, 3554 "COMRESET success, D2H register FIS " 3555 "post to received FIS structure " 3556 "port %d signature = 0x%x", 3557 port, port_signature); 3558 #endif 3559 goto out_check; 3560 } 3561 3562 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 3563 /* 3564 * We are effectively timing out after 11 sec. 3565 */ 3566 break; 3567 } 3568 3569 /* Wait for 10 millisec */ 3570 delay(AHCI_10MS_TICKS); 3571 } while (((port_task_file & AHCI_TFD_ERR_MASK) 3572 >> AHCI_TFD_ERR_SHIFT) != 0x1); 3573 3574 AHCIDBG3(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: 2nd loop " 3575 "count: %d, port_task_file = 0x%x port %d", 3576 loop_count, port_task_file, port); 3577 3578 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d the device hardware " 3579 "has been initialized and the power-up diagnostics failed", 3580 instance, port); 3581 3582 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3583 rval = AHCI_FAILURE; 3584 3585 out: 3586 /* Clear port serror register for the port */ 3587 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3588 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 3589 AHCI_SERROR_CLEAR_ALL); 3590 3591 return (rval); 3592 3593 out_check: 3594 /* 3595 * Check device status, if keep busy or COMRESET error 3596 * do device reset to patch some SATA disks' issue 3597 * 3598 * For VT8251, sometimes need to do the device reset 3599 */ 3600 if ((port_task_file & AHCI_TFD_STS_BSY) || 3601 (port_task_file & AHCI_TFD_STS_DRQ)) { 3602 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "port %d keep BSY/DRQ set " 3603 "need to do device reset", port); 3604 3605 (void) ahci_software_reset(ahci_ctlp, ahci_portp, port); 3606 3607 port_task_file = 3608 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3609 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3610 3611 if (port_task_file & AHCI_TFD_STS_BSY || 3612 port_task_file & AHCI_TFD_STS_DRQ) { 3613 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset: port %d " 3614 "BSY/DRQ still set after device reset " 3615 "port_task_file = 0x%x", instance, 3616 port, port_task_file); 3617 ahci_portp->ahciport_port_state |= SATA_PSTATE_FAILED; 3618 rval = AHCI_FAILURE; 3619 } 3620 } 3621 3622 goto out; 3623 } 3624 3625 /* 3626 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 3627 * This is the most intrusive. 3628 * 3629 * When an HBA reset occurs, Phy communication will be re-established with 3630 * the device through a COMRESET followed by the normal out-of-band 3631 * communication sequence defined in Serial ATA. AT the end of reset, the 3632 * device, if working properly, will send a D2H Register FIS, which contains 3633 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 3634 * and PxTFD.ERR register fields, and updates the PxSIG register with the 3635 * signature. 3636 * 3637 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 3638 */ 3639 static int 3640 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 3641 { 3642 ahci_port_t *ahci_portp; 3643 uint32_t ghc_control; 3644 uint8_t port; 3645 int loop_count; 3646 int rval = AHCI_SUCCESS; 3647 3648 AHCIDBG0(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting"); 3649 3650 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3651 3652 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3653 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3654 3655 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 3656 ghc_control |= AHCI_HBA_GHC_HR; 3657 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3658 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3659 3660 /* 3661 * Wait until HBA Reset complete or timeout 3662 */ 3663 loop_count = 0; 3664 do { 3665 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3666 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3667 3668 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 3669 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, 3670 "ahci hba reset is timing out, " 3671 "ghc_control = 0x%x", ghc_control); 3672 /* We are effectively timing out after 1 sec. */ 3673 break; 3674 } 3675 3676 /* Wait for 10 millisec */ 3677 delay(AHCI_10MS_TICKS); 3678 } while (ghc_control & AHCI_HBA_GHC_HR); 3679 3680 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 3681 "ahci_hba_reset: 1st loop count: %d, " 3682 "ghc_control = 0x%x", loop_count, ghc_control); 3683 3684 if (ghc_control & AHCI_HBA_GHC_HR) { 3685 /* The hba is not reset for some reasons */ 3686 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 3687 "hba reset failed: HBA in a hung or locked state"); 3688 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3689 return (AHCI_FAILURE); 3690 } 3691 3692 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3693 /* Only check implemented ports */ 3694 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3695 continue; 3696 } 3697 3698 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3699 mutex_enter(&ahci_portp->ahciport_mutex); 3700 3701 if (ahci_port_reset(ahci_ctlp, ahci_portp, port) 3702 != AHCI_SUCCESS) { 3703 rval = AHCI_FAILURE; 3704 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 3705 "ahci_hba_reset: port %d failed", port); 3706 } 3707 3708 mutex_exit(&ahci_portp->ahciport_mutex); 3709 } 3710 3711 /* 3712 * Indicate that system software is AHCI aware by setting 3713 * GHC.AE to 1 3714 */ 3715 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3716 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3717 3718 ghc_control |= AHCI_HBA_GHC_AE; 3719 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3720 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 3721 3722 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3723 3724 return (rval); 3725 } 3726 3727 /* 3728 * This routine is only called from AHCI_ATTACH or phyrdy change 3729 * case. It first calls port reset to initialize port, probe port and probe 3730 * device, then try to read PxSIG register to find the type of device 3731 * attached to the port. 3732 * 3733 * WARNING!!! ahciport_mutex should be acquired before the function 3734 * is called. And the port interrupt is disabled. 3735 */ 3736 static void 3737 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, 3738 ahci_port_t *ahci_portp, uint8_t port) 3739 { 3740 uint32_t signature; 3741 3742 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3743 "ahci_find_dev_signature enter: port %d", port); 3744 3745 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3746 3747 /* Call port reset to check link status and get device signature */ 3748 (void) ahci_port_reset(ahci_ctlp, ahci_portp, port); 3749 3750 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3751 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3752 "ahci_find_dev_signature: No device is found " 3753 "at port %d", port); 3754 return; 3755 } 3756 3757 /* Check the port state */ 3758 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3759 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 3760 "ahci_find_dev_signature: port %d state 0x%x", 3761 port, ahci_portp->ahciport_port_state); 3762 return; 3763 } 3764 3765 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3766 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 3767 3768 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 3769 "ahci_find_dev_signature: port %d signature = 0x%x", 3770 port, signature); 3771 3772 switch (signature) { 3773 3774 case AHCI_SIGNATURE_DISK: 3775 ahci_portp->ahciport_device_type = SATA_DTYPE_ATADISK; 3776 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3777 "Disk is found at port: %d", port); 3778 break; 3779 3780 case AHCI_SIGNATURE_ATAPI: 3781 ahci_portp->ahciport_device_type = SATA_DTYPE_ATAPI; 3782 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3783 "ATAPI device is found at port: %d", port); 3784 break; 3785 3786 case AHCI_SIGNATURE_PORT_MULTIPLIER: 3787 ahci_portp->ahciport_device_type = SATA_DTYPE_PMULT; 3788 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3789 "Port Multiplier is found at port: %d", port); 3790 break; 3791 3792 default: 3793 ahci_portp->ahciport_device_type = SATA_DTYPE_UNKNOWN; 3794 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, 3795 "Unknown device is found at port: %d", port); 3796 } 3797 } 3798 3799 /* 3800 * According to the spec, to reliably detect hot plug removals, software 3801 * must disable interface power management. Software should perform the 3802 * following initialization on a port after a device is attached: 3803 * Set PxSCTL.IPM to 3h to disable interface state transitions 3804 * Set PxCMD.ALPE to '0' to disable aggressive power management 3805 * Disable device initiated interface power management by SET FEATURE 3806 * 3807 * We can ignore the last item because by default the feature is disabled 3808 */ 3809 static void 3810 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 3811 { 3812 uint32_t port_scontrol, port_cmd_status; 3813 3814 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3815 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3816 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 3817 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3818 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3819 3820 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3821 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3822 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 3823 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3824 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 3825 } 3826 3827 /* 3828 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 3829 * to 1, then set it firstly. 3830 * 3831 * Each port contains two major DMA engines. One DMA engine walks through 3832 * the command list, and is controlled by PxCMD.ST. The second DMA engine 3833 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 3834 * 3835 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 3836 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 3837 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 3838 * 3839 * Software shall not set PxCMD.ST to '1' unless a functional device is 3840 * present on the port(as determined by PxTFD.STS.BSY = '0', 3841 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 3842 * 3843 * WARNING!!! ahciport_mutex should be acquired before the function 3844 * is called. 3845 */ 3846 static int 3847 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 3848 { 3849 uint32_t port_cmd_status; 3850 3851 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 3852 3853 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 3854 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3855 "the state for port %d is 0x%x", 3856 port, ahci_portp->ahciport_port_state); 3857 return (AHCI_FAILURE); 3858 } 3859 3860 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3861 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 3862 "no device is attached at port %d", port); 3863 return (AHCI_FAILURE); 3864 } 3865 3866 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 3867 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3868 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3869 3870 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 3871 port_cmd_status |= AHCI_CMD_STATUS_FRE; 3872 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3873 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3874 port_cmd_status); 3875 } 3876 3877 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3878 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3879 3880 port_cmd_status |= AHCI_CMD_STATUS_ST; 3881 3882 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3883 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3884 port_cmd_status); 3885 3886 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 3887 3888 return (AHCI_SUCCESS); 3889 } 3890 3891 /* 3892 * Allocate the ahci_port_t including Received FIS and Command List. 3893 * The argument - port is the physical port number, and not logical 3894 * port number seen by the SATA framework. 3895 * 3896 * WARNING!!! ahcictl_mutex should be acquired before the function 3897 * is called. 3898 */ 3899 static int 3900 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3901 { 3902 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 3903 ahci_port_t *ahci_portp; 3904 char taskq_name[64] = "event_handle_taskq"; 3905 3906 ahci_portp = 3907 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 3908 3909 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 3910 ahci_portp->ahciport_port_num = port; 3911 3912 /* Intialize the port condition variable */ 3913 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 3914 3915 /* Initialize the port mutex */ 3916 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 3917 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 3918 3919 mutex_enter(&ahci_portp->ahciport_mutex); 3920 3921 /* 3922 * Allocate memory for received FIS structure and 3923 * command list for this port 3924 */ 3925 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3926 goto err_case1; 3927 } 3928 3929 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 3930 goto err_case2; 3931 } 3932 3933 (void) snprintf(taskq_name + strlen(taskq_name), 3934 sizeof (taskq_name) - strlen(taskq_name), 3935 "_port%d", port); 3936 3937 /* Create the taskq for the port */ 3938 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 3939 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 3940 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 3941 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 3942 goto err_case3; 3943 } 3944 3945 /* Allocate the argument for the taskq */ 3946 ahci_portp->ahciport_event_args = 3947 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 3948 3949 if (ahci_portp->ahciport_event_args == NULL) 3950 goto err_case4; 3951 3952 mutex_exit(&ahci_portp->ahciport_mutex); 3953 3954 return (AHCI_SUCCESS); 3955 3956 err_case4: 3957 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 3958 3959 err_case3: 3960 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3961 3962 err_case2: 3963 ahci_dealloc_rcvd_fis(ahci_portp); 3964 3965 err_case1: 3966 mutex_exit(&ahci_portp->ahciport_mutex); 3967 mutex_destroy(&ahci_portp->ahciport_mutex); 3968 cv_destroy(&ahci_portp->ahciport_cv); 3969 3970 kmem_free(ahci_portp, sizeof (ahci_port_t)); 3971 3972 return (AHCI_FAILURE); 3973 } 3974 3975 /* 3976 * Reverse of ahci_dealloc_port_state(). 3977 * 3978 * WARNING!!! ahcictl_mutex should be acquired before the function 3979 * is called. 3980 */ 3981 static void 3982 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 3983 { 3984 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3985 3986 ASSERT(ahci_portp != NULL); 3987 3988 mutex_enter(&ahci_portp->ahciport_mutex); 3989 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 3990 ahci_portp->ahciport_event_args = NULL; 3991 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 3992 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 3993 ahci_dealloc_rcvd_fis(ahci_portp); 3994 mutex_exit(&ahci_portp->ahciport_mutex); 3995 3996 mutex_destroy(&ahci_portp->ahciport_mutex); 3997 cv_destroy(&ahci_portp->ahciport_cv); 3998 3999 kmem_free(ahci_portp, sizeof (ahci_port_t)); 4000 4001 ahci_ctlp->ahcictl_ports[port] = NULL; 4002 } 4003 4004 /* 4005 * Allocates memory for the Received FIS Structure 4006 * 4007 * WARNING!!! ahciport_mutex should be acquired before the function 4008 * is called. 4009 */ 4010 static int 4011 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4012 uint8_t port) 4013 { 4014 size_t rcvd_fis_size; 4015 size_t ret_len; 4016 uint_t cookie_count; 4017 4018 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 4019 4020 /* allocate rcvd FIS dma handle. */ 4021 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4022 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 4023 DDI_DMA_SLEEP, 4024 NULL, 4025 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 4026 DDI_SUCCESS) { 4027 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4028 "rcvd FIS dma handle alloc failed"); 4029 4030 return (AHCI_FAILURE); 4031 } 4032 4033 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 4034 rcvd_fis_size, 4035 &accattr, 4036 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4037 DDI_DMA_SLEEP, 4038 NULL, 4039 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 4040 &ret_len, 4041 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 4042 4043 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4044 "rcvd FIS dma mem alloc fail"); 4045 /* error.. free the dma handle. */ 4046 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4047 return (AHCI_FAILURE); 4048 } 4049 4050 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 4051 NULL, 4052 (caddr_t)ahci_portp->ahciport_rcvd_fis, 4053 rcvd_fis_size, 4054 DDI_DMA_CONSISTENT, 4055 DDI_DMA_SLEEP, 4056 NULL, 4057 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 4058 &cookie_count) != DDI_DMA_MAPPED) { 4059 4060 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4061 "rcvd FIS dma handle bind fail"); 4062 /* error.. free the dma handle & free the memory. */ 4063 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4064 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4065 return (AHCI_FAILURE); 4066 } 4067 4068 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 4069 4070 /* Config Port Received FIS Base Address */ 4071 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4072 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 4073 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4074 4075 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4076 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 4077 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4078 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 4079 4080 return (AHCI_SUCCESS); 4081 } 4082 4083 /* 4084 * Deallocates the Received FIS Structure 4085 * 4086 * WARNING!!! ahciport_mutex should be acquired before the function 4087 * is called. 4088 */ 4089 static void 4090 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 4091 { 4092 /* Unbind the cmd list dma handle first. */ 4093 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 4094 4095 /* Then free the underlying memory. */ 4096 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 4097 4098 /* Now free the handle itself. */ 4099 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 4100 } 4101 4102 /* 4103 * Allocates memory for the Command List, which contains up to 32 entries. 4104 * Each entry contains a command header, which is a 32-byte structure that 4105 * includes the pointer to the command table. 4106 * 4107 * WARNING!!! ahciport_mutex should be acquired before the function 4108 * is called. 4109 */ 4110 static int 4111 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4112 uint8_t port) 4113 { 4114 size_t cmd_list_size; 4115 size_t ret_len; 4116 uint_t cookie_count; 4117 4118 cmd_list_size = 4119 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 4120 4121 /* allocate cmd list dma handle. */ 4122 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4123 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 4124 DDI_DMA_SLEEP, 4125 NULL, 4126 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 4127 4128 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4129 "cmd list dma handle alloc failed"); 4130 return (AHCI_FAILURE); 4131 } 4132 4133 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 4134 cmd_list_size, 4135 &accattr, 4136 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4137 DDI_DMA_SLEEP, 4138 NULL, 4139 (caddr_t *)&ahci_portp->ahciport_cmd_list, 4140 &ret_len, 4141 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 4142 4143 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4144 "cmd list dma mem alloc fail"); 4145 /* error.. free the dma handle. */ 4146 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4147 return (AHCI_FAILURE); 4148 } 4149 4150 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 4151 NULL, 4152 (caddr_t)ahci_portp->ahciport_cmd_list, 4153 cmd_list_size, 4154 DDI_DMA_CONSISTENT, 4155 DDI_DMA_SLEEP, 4156 NULL, 4157 &ahci_portp->ahciport_cmd_list_dma_cookie, 4158 &cookie_count) != DDI_DMA_MAPPED) { 4159 4160 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4161 "cmd list dma handle bind fail"); 4162 /* error.. free the dma handle & free the memory. */ 4163 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4164 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4165 return (AHCI_FAILURE); 4166 } 4167 4168 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 4169 4170 /* Config Port Command List Base Address */ 4171 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 4172 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 4173 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4174 4175 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 4176 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 4177 4178 AHCIDBG1(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 4179 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 4180 4181 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 4182 goto err_out; 4183 } 4184 4185 return (AHCI_SUCCESS); 4186 4187 err_out: 4188 /* Unbind the cmd list dma handle first. */ 4189 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4190 4191 /* Then free the underlying memory. */ 4192 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4193 4194 /* Now free the handle itself. */ 4195 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4196 4197 return (AHCI_FAILURE); 4198 } 4199 4200 /* 4201 * Deallocates the Command List 4202 * 4203 * WARNING!!! ahciport_mutex should be acquired before the function 4204 * is called. 4205 */ 4206 static void 4207 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4208 { 4209 /* First dealloc command table */ 4210 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 4211 4212 /* Unbind the cmd list dma handle first. */ 4213 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 4214 4215 /* Then free the underlying memory. */ 4216 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 4217 4218 /* Now free the handle itself. */ 4219 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 4220 } 4221 4222 /* 4223 * Allocates memory for all Command Tables, which contains Command FIS, 4224 * ATAPI Command and Physical Region Descriptor Table. 4225 * 4226 * WARNING!!! ahciport_mutex should be acquired before the function 4227 * is called. 4228 */ 4229 static int 4230 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4231 { 4232 size_t ret_len; 4233 ddi_dma_cookie_t cmd_table_dma_cookie; 4234 uint_t cookie_count; 4235 int slot; 4236 4237 AHCIDBG1(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4238 "ahci_alloc_cmd_tables: port %d enter", 4239 ahci_portp->ahciport_port_num); 4240 4241 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4242 /* Allocate cmd table dma handle. */ 4243 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 4244 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 4245 DDI_DMA_SLEEP, 4246 NULL, 4247 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 4248 DDI_SUCCESS) { 4249 4250 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4251 "cmd table dma handle alloc failed"); 4252 4253 goto err_out; 4254 } 4255 4256 if (ddi_dma_mem_alloc( 4257 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4258 ahci_cmd_table_size, 4259 &accattr, 4260 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 4261 DDI_DMA_SLEEP, 4262 NULL, 4263 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 4264 &ret_len, 4265 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 4266 NULL) { 4267 4268 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4269 "cmd table dma mem alloc fail"); 4270 4271 /* error.. free the dma handle. */ 4272 ddi_dma_free_handle( 4273 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4274 goto err_out; 4275 } 4276 4277 if (ddi_dma_addr_bind_handle( 4278 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4279 NULL, 4280 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 4281 ahci_cmd_table_size, 4282 DDI_DMA_CONSISTENT, 4283 DDI_DMA_SLEEP, 4284 NULL, 4285 &cmd_table_dma_cookie, 4286 &cookie_count) != DDI_DMA_MAPPED) { 4287 4288 AHCIDBG0(AHCIDBG_INIT, ahci_ctlp, 4289 "cmd table dma handle bind fail"); 4290 /* error.. free the dma handle & free the memory. */ 4291 ddi_dma_mem_free( 4292 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4293 ddi_dma_free_handle( 4294 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4295 goto err_out; 4296 } 4297 4298 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 4299 ahci_cmd_table_size); 4300 4301 /* Config Port Command Table Base Address */ 4302 SET_COMMAND_TABLE_BASE_ADDR( 4303 (&ahci_portp->ahciport_cmd_list[slot]), 4304 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 4305 4306 #ifndef __lock_lint 4307 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 4308 (&ahci_portp->ahciport_cmd_list[slot]), 4309 cmd_table_dma_cookie.dmac_laddress >> 32); 4310 #endif 4311 } 4312 4313 return (AHCI_SUCCESS); 4314 err_out: 4315 4316 for (slot--; slot >= 0; slot--) { 4317 /* Unbind the cmd table dma handle first */ 4318 (void) ddi_dma_unbind_handle( 4319 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4320 4321 /* Then free the underlying memory */ 4322 ddi_dma_mem_free( 4323 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4324 4325 /* Now free the handle itself */ 4326 ddi_dma_free_handle( 4327 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4328 } 4329 4330 return (AHCI_FAILURE); 4331 } 4332 4333 /* 4334 * Deallocates memory for all Command Tables. 4335 * 4336 * WARNING!!! ahciport_mutex should be acquired before the function 4337 * is called. 4338 */ 4339 static void 4340 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 4341 { 4342 int slot; 4343 4344 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 4345 "ahci_dealloc_cmd_tables: %d enter", 4346 ahci_portp->ahciport_port_num); 4347 4348 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 4349 /* Unbind the cmd table dma handle first. */ 4350 (void) ddi_dma_unbind_handle( 4351 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4352 4353 /* Then free the underlying memory. */ 4354 ddi_dma_mem_free( 4355 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 4356 4357 /* Now free the handle itself. */ 4358 ddi_dma_free_handle( 4359 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 4360 } 4361 } 4362 4363 /* 4364 * WARNING!!! ahciport_mutex should be acquired before the function 4365 * is called. 4366 */ 4367 static void 4368 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 4369 sata_device_t *sd) 4370 { 4371 sd->satadev_scr.sstatus = 4372 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4373 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 4374 sd->satadev_scr.serror = 4375 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4376 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 4377 sd->satadev_scr.scontrol = 4378 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4379 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 4380 sd->satadev_scr.sactive = 4381 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4382 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 4383 } 4384 4385 /* 4386 * For poll mode, ahci_port_intr will be called to emulate the interrupt 4387 */ 4388 static void 4389 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 4390 { 4391 uint32_t port_intr_status; 4392 uint32_t port_intr_enable; 4393 4394 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4395 "ahci_port_intr enter: port %d", port); 4396 4397 mutex_enter(&ahci_portp->ahciport_mutex); 4398 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 4399 /* For SATA_OPMODE_POLLING commands */ 4400 port_intr_enable = 4401 (AHCI_INTR_STATUS_DHRS | 4402 AHCI_INTR_STATUS_PSS | 4403 AHCI_INTR_STATUS_SDBS | 4404 AHCI_INTR_STATUS_UFS | 4405 AHCI_INTR_STATUS_PCS | 4406 AHCI_INTR_STATUS_PRCS | 4407 AHCI_INTR_STATUS_OFS | 4408 AHCI_INTR_STATUS_INFS | 4409 AHCI_INTR_STATUS_IFS | 4410 AHCI_INTR_STATUS_HBDS | 4411 AHCI_INTR_STATUS_HBFS | 4412 AHCI_INTR_STATUS_TFES); 4413 mutex_exit(&ahci_portp->ahciport_mutex); 4414 goto next; 4415 } 4416 mutex_exit(&ahci_portp->ahciport_mutex); 4417 4418 /* 4419 * port_intr_enable indicates that the corresponding interrrupt 4420 * reporting is enabled. 4421 */ 4422 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4423 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4424 next: 4425 /* 4426 * port_intr_stats indicates that the corresponding interrupt 4427 * condition is active. 4428 */ 4429 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4430 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 4431 4432 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4433 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 4434 "port_intr_enable = 0x%x", 4435 port, port_intr_status, port_intr_enable); 4436 4437 port_intr_status &= port_intr_enable; 4438 4439 /* First clear the port interrupts status */ 4440 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4441 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 4442 port_intr_status); 4443 4444 /* Check the completed non-queued commands */ 4445 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 4446 AHCI_INTR_STATUS_PSS)) { 4447 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 4448 ahci_portp, port); 4449 } 4450 4451 /* Check the completed queued commands */ 4452 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 4453 (void) ahci_intr_set_device_bits(ahci_ctlp, 4454 ahci_portp, port); 4455 } 4456 4457 /* Check the port connect change status interrupt bit */ 4458 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 4459 (void) ahci_intr_port_connect_change(ahci_ctlp, 4460 ahci_portp, port); 4461 } 4462 4463 /* Check the device mechanical presence status interrupt bit */ 4464 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 4465 (void) ahci_intr_device_mechanical_presence_status( 4466 ahci_ctlp, ahci_portp, port); 4467 } 4468 4469 /* Check the PhyRdy change status interrupt bit */ 4470 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 4471 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 4472 port); 4473 } 4474 4475 /* 4476 * Check the non-fatal error interrupt bits, there are three 4477 * kinds of non-fatal errors at the time being: 4478 * 4479 * PxIS.UFS - Unknown FIS Error 4480 * PxIS.OFS - Overflow Error 4481 * PxIS.INFS - Interface Non-Fatal Error 4482 * 4483 * For these non-fatal errors, the HBA can continue to operate, 4484 * so the driver just log the error messages. 4485 */ 4486 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 4487 AHCI_INTR_STATUS_OFS | 4488 AHCI_INTR_STATUS_INFS)) { 4489 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 4490 port, port_intr_status); 4491 } 4492 4493 /* 4494 * Check the fatal error interrupt bits, there are four kinds 4495 * of fatal errors for AHCI controllers: 4496 * 4497 * PxIS.HBFS - Host Bus Fatal Error 4498 * PxIS.HBDS - Host Bus Data Error 4499 * PxIS.IFS - Interface Fatal Error 4500 * PxIS.TFES - Task File Error 4501 * 4502 * The fatal error means the HBA can not recover from it by 4503 * itself, and it will try to abort the transfer, and the software 4504 * must intervene to restart the port. 4505 */ 4506 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 4507 AHCI_INTR_STATUS_HBDS | 4508 AHCI_INTR_STATUS_HBFS | 4509 AHCI_INTR_STATUS_TFES)) 4510 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 4511 port, port_intr_status); 4512 4513 /* Check the cold port detect interrupt bit */ 4514 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 4515 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 4516 } 4517 4518 /* Second clear the corresponding bit in IS.IPS */ 4519 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4520 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 4521 } 4522 4523 /* 4524 * Interrupt service handler 4525 */ 4526 static uint_t 4527 ahci_intr(caddr_t arg1, caddr_t arg2) 4528 { 4529 #ifndef __lock_lint 4530 _NOTE(ARGUNUSED(arg2)) 4531 #endif 4532 /* LINTED */ 4533 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 4534 ahci_port_t *ahci_portp; 4535 int32_t global_intr_status; 4536 uint8_t port; 4537 4538 /* 4539 * global_intr_status indicates that the corresponding port has 4540 * an interrupt pending. 4541 */ 4542 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4543 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 4544 4545 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 4546 /* The interrupt is not ours */ 4547 return (DDI_INTR_UNCLAIMED); 4548 } 4549 4550 /* Loop for all the ports */ 4551 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 4552 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 4553 continue; 4554 } 4555 if (!((0x1 << port) & global_intr_status)) { 4556 continue; 4557 } 4558 4559 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 4560 4561 /* Call ahci_port_intr */ 4562 ahci_port_intr(ahci_ctlp, ahci_portp, port); 4563 } 4564 4565 return (DDI_INTR_CLAIMED); 4566 } 4567 4568 /* 4569 * For non-queued commands, when the corresponding bit in the PxCI register 4570 * is cleared, it means the command is completed successfully. And according 4571 * to the HBA state machine, there are three conditions which possibly will 4572 * try to clear the PxCI register bit. 4573 * 1. Receive one D2H Register FIS which is with 'I' bit set 4574 * 2. Update PIO Setup FIS 4575 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 4576 * 4577 * Process completed non-queued commands when the interrupt status bit - 4578 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 4579 * 4580 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 4581 * with the 'I' bit set. And the following commands will send thus 4582 * FIS with 'I' bit set upon the successful completion: 4583 * 1. Non-data commands 4584 * 2. DMA data-in command 4585 * 3. DMA data-out command 4586 * 4. PIO data-out command 4587 * 5. PACKET non-data commands 4588 * 6. PACKET PIO data-in command 4589 * 7. PACKET PIO data-out command 4590 * 8. PACKET DMA data-in command 4591 * 9. PACKET DMA data-out command 4592 * 4593 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 4594 * with the 'I' bit set. And the following commands will send this 4595 * FIS upon the successful completion: 4596 * 1. PIO data-in command 4597 */ 4598 static int 4599 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 4600 ahci_port_t *ahci_portp, uint8_t port) 4601 { 4602 uint32_t port_cmd_issue = 0; 4603 uint32_t finished_tags; 4604 int finished_slot; 4605 sata_pkt_t *satapkt; 4606 ahci_fis_d2h_register_t *rcvd_fisp; 4607 #if AHCI_DEBUG 4608 ahci_cmd_header_t *cmd_header; 4609 uint32_t cmd_dmacount; 4610 #endif 4611 4612 mutex_enter(&ahci_portp->ahciport_mutex); 4613 4614 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4615 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4616 /* 4617 * Spurious interrupt. Nothing to be done. 4618 */ 4619 mutex_exit(&ahci_portp->ahciport_mutex); 4620 return (AHCI_SUCCESS); 4621 } 4622 4623 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4624 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4625 4626 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 4627 /* Slot 0 is always used during error recovery */ 4628 finished_tags = 0x1 & ~port_cmd_issue; 4629 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 4630 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 4631 "retrieval is finished, and finished_tags = 0x%x", 4632 port, finished_tags); 4633 } else { 4634 finished_tags = ahci_portp->ahciport_pending_tags & 4635 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4636 } 4637 4638 AHCIDBG3(AHCIDBG_INTR, ahci_ctlp, 4639 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 4640 "port_cmd_issue = 0x%x finished_tags = 0x%x", 4641 ahci_portp->ahciport_pending_tags, port_cmd_issue, 4642 finished_tags); 4643 4644 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 4645 (finished_tags == 0x1)) { 4646 satapkt = ahci_portp->ahciport_err_retri_pkt; 4647 ASSERT(satapkt != NULL); 4648 4649 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4650 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4651 "with SATA_PKT_COMPLETED", (void *)satapkt); 4652 4653 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4654 goto out; 4655 } 4656 4657 while (finished_tags) { 4658 finished_slot = ddi_ffs(finished_tags) - 1; 4659 if (finished_slot == -1) { 4660 goto out; 4661 } 4662 4663 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4664 ASSERT(satapkt != NULL); 4665 #if AHCI_DEBUG 4666 /* 4667 * For non-native queued commands, the PRD byte count field 4668 * shall contain an accurate count of the number of bytes 4669 * transferred for the command before the PxCI bit is cleared 4670 * to '0' for the command. 4671 * 4672 * The purpose of this field is to let software know how many 4673 * bytes transferred for a given operation in order to 4674 * determine if underflow occurred. When issuing native command 4675 * queuing commands, this field should not be used and is not 4676 * required to be valid since in this case underflow is always 4677 * illegal. 4678 * 4679 * For data reads, the HBA will update its PRD byte count with 4680 * the total number of bytes received from the last FIS, and 4681 * may be able to continue normally. For data writes, the 4682 * device will detect an error, and HBA most likely will get 4683 * a fatal error. 4684 * 4685 * Therefore, here just put code to debug part. And please 4686 * refer to the comment above ahci_intr_fatal_error for the 4687 * definition of underflow error. 4688 */ 4689 cmd_dmacount = 4690 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 4691 if (cmd_dmacount) { 4692 cmd_header = 4693 &ahci_portp->ahciport_cmd_list[finished_slot]; 4694 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 4695 "ahci_intr_cmd_cmplt: port %d, " 4696 "PRD Byte Count = 0x%x, " 4697 "ahciport_prd_bytecounts = 0x%x", port, 4698 cmd_header->ahcich_prd_byte_count, 4699 cmd_dmacount); 4700 4701 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 4702 AHCIDBG1(AHCIDBG_UNDERFLOW, ahci_ctlp, 4703 "ahci_intr_cmd_cmplt: port %d, " 4704 "an underflow occurred", port); 4705 } 4706 } 4707 #endif 4708 4709 /* 4710 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 4711 * feature, sata_special_regs flag will be set, and the 4712 * driver should copy the status and the other corresponding 4713 * register values in the D2H Register FIS received (It's 4714 * working on Non-data protocol) from the device back to 4715 * the sata_cmd. 4716 * 4717 * For every AHCI port, there is only one Received FIS 4718 * structure, which contains the FISes received from the 4719 * device, So we're trying to copy the content of D2H 4720 * Register FIS in the Received FIS structure back to 4721 * the sata_cmd. 4722 */ 4723 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 4724 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 4725 ahcirf_d2h_register_fis); 4726 satapkt->satapkt_cmd.satacmd_status_reg = 4727 GET_RFIS_STATUS(rcvd_fisp); 4728 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 4729 } 4730 4731 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 4732 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 4733 "with SATA_PKT_COMPLETED", (void *)satapkt); 4734 4735 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 4736 CLEAR_BIT(finished_tags, finished_slot); 4737 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4738 4739 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4740 } 4741 out: 4742 AHCIDBG1(AHCIDBG_PKTCOMP, ahci_ctlp, 4743 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 4744 ahci_portp->ahciport_pending_tags); 4745 4746 mutex_exit(&ahci_portp->ahciport_mutex); 4747 4748 return (AHCI_SUCCESS); 4749 } 4750 4751 /* 4752 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 4753 * with the 'I' bit set and has been copied into system memory. It will 4754 * be sent under the following situations: 4755 * 4756 * 1. NCQ command is completed 4757 * 2. Asynchronous notification 4758 * 4759 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 4760 * via the Set Device Bits FIS. When such event is generated, the software 4761 * needs to read PxSACT register and compares the current value to the 4762 * list of commands previously issue by software. ahciport_pending_ncq_tags 4763 * keeps the tags of previously issued commands. 4764 * 4765 * Asynchronous Notification is a feature in SATA II, which allows an 4766 * ATAPI device to send a signal to the host when media is inserted or 4767 * removed and avoids polling the device for media changes. The signal 4768 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 4769 * set to '1'. At the moment, it's not supported yet. 4770 */ 4771 static int 4772 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 4773 ahci_port_t *ahci_portp, uint8_t port) 4774 { 4775 uint32_t port_sactive; 4776 uint32_t port_cmd_issue; 4777 uint32_t issued_tags; 4778 int issued_slot; 4779 uint32_t finished_tags; 4780 int finished_slot; 4781 sata_pkt_t *satapkt; 4782 4783 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4784 "ahci_intr_set_device_bits enter: port %d", port); 4785 4786 mutex_enter(&ahci_portp->ahciport_mutex); 4787 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4788 mutex_exit(&ahci_portp->ahciport_mutex); 4789 return (AHCI_SUCCESS); 4790 } 4791 4792 /* 4793 * First the handler got which commands are finished by checking 4794 * PxSACT register 4795 */ 4796 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4797 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4798 4799 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4800 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 4801 4802 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4803 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 4804 "port_sactive = 0x%x", port, 4805 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 4806 4807 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4808 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 4809 4810 /* 4811 * For NCQ commands, the software can determine which command has 4812 * already been transmitted to the device by checking PxCI register. 4813 */ 4814 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4815 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4816 4817 issued_tags = ahci_portp->ahciport_pending_tags & 4818 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 4819 4820 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4821 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 4822 "port_cmd_issue = 0x%x", port, 4823 ahci_portp->ahciport_pending_tags, port_cmd_issue); 4824 4825 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4826 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 4827 4828 /* 4829 * Clear ahciport_pending_tags bit when the corresponding command 4830 * is already sent down to the device. 4831 */ 4832 while (issued_tags) { 4833 issued_slot = ddi_ffs(issued_tags) - 1; 4834 if (issued_slot == -1) { 4835 goto next; 4836 } 4837 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 4838 CLEAR_BIT(issued_tags, issued_slot); 4839 } 4840 4841 next: 4842 while (finished_tags) { 4843 finished_slot = ddi_ffs(finished_tags) - 1; 4844 if (finished_slot == -1) { 4845 goto out; 4846 } 4847 4848 /* The command is certainly transmitted to the device */ 4849 ASSERT(!(ahci_portp->ahciport_pending_tags & 4850 (0x1 << finished_slot))); 4851 4852 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 4853 ASSERT(satapkt != NULL); 4854 4855 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 4856 "ahci_intr_set_device_bits: sending up pkt 0x%p " 4857 "with SATA_PKT_COMPLETED", (void *)satapkt); 4858 4859 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 4860 CLEAR_BIT(finished_tags, finished_slot); 4861 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 4862 4863 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 4864 } 4865 out: 4866 AHCIDBG3(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 4867 "ahci_intr_set_device_bits: port %d " 4868 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 4869 port, ahci_portp->ahciport_pending_ncq_tags, 4870 ahci_portp->ahciport_pending_tags); 4871 4872 mutex_exit(&ahci_portp->ahciport_mutex); 4873 4874 return (AHCI_SUCCESS); 4875 } 4876 4877 /* 4878 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 4879 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 4880 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 4881 * indicates a COMINIT signal was received. 4882 * 4883 * Hot plug insertion is detected by reception of a COMINIT signal from the 4884 * device. On reception of unsolicited COMINIT, the HBA shall generate a 4885 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 4886 * begin the normal communication negotiation sequence as outlined in the 4887 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 4888 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 4889 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 4890 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 4891 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 4892 * to handle hot plug insertion. In this interrupt handler, just do nothing 4893 * but print some log message and clear the bit. 4894 */ 4895 static int 4896 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 4897 ahci_port_t *ahci_portp, uint8_t port) 4898 { 4899 #if AHCI_DEBUG 4900 uint32_t port_serror; 4901 #endif 4902 4903 mutex_enter(&ahci_portp->ahciport_mutex); 4904 4905 #if AHCI_DEBUG 4906 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4907 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 4908 4909 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4910 "ahci_intr_port_connect_change: port %d, " 4911 "port_serror = 0x%x", port, port_serror); 4912 #endif 4913 4914 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 4915 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4916 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 4917 SERROR_EXCHANGED_ERR); 4918 4919 mutex_exit(&ahci_portp->ahciport_mutex); 4920 4921 return (AHCI_SUCCESS); 4922 } 4923 4924 /* 4925 * Hot Plug Operation for platforms that support Mechanical Presence 4926 * Switches. 4927 * 4928 * When set, it indicates that a mechanical presence switch attached to this 4929 * port has been opened or closed, which may lead to a change in the connection 4930 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 4931 * are set to '1'. 4932 * 4933 * At the moment, this interrupt is not needed and disabled and we just log 4934 * the debug message. 4935 */ 4936 static int 4937 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 4938 ahci_port_t *ahci_portp, uint8_t port) 4939 { 4940 uint32_t cap_status, port_cmd_status; 4941 4942 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 4943 "ahci_intr_device_mechanical_presence_status enter, " 4944 "port %d", port); 4945 4946 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4947 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 4948 4949 mutex_enter(&ahci_portp->ahciport_mutex); 4950 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4951 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4952 4953 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 4954 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 4955 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4956 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 4957 "the interrupt: cap_status = 0x%x, " 4958 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 4959 mutex_exit(&ahci_portp->ahciport_mutex); 4960 4961 return (AHCI_SUCCESS); 4962 } 4963 4964 #if AHCI_DEBUG 4965 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 4966 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4967 "The mechanical presence switch is open: " 4968 "port %d, port_cmd_status = 0x%x", 4969 port, port_cmd_status); 4970 } else { 4971 AHCIDBG2(AHCIDBG_INTR, ahci_ctlp, 4972 "The mechanical presence switch is close: " 4973 "port %d, port_cmd_status = 0x%x", 4974 port, port_cmd_status); 4975 } 4976 #endif 4977 4978 mutex_exit(&ahci_portp->ahciport_mutex); 4979 4980 return (AHCI_SUCCESS); 4981 } 4982 4983 /* 4984 * Native Hot Plug Support. 4985 * 4986 * When set, it indicates that the internal PHYRDY signal changed state. 4987 * This bit reflects the state of PxSERR.DIAG.N. 4988 * 4989 * There are three kinds of conditions to generate this interrupt event: 4990 * 1. a device is inserted 4991 * 2. a device is disconnected 4992 * 3. when the link enters/exits a Partial or Slumber interface power 4993 * management state 4994 * 4995 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 4996 * bit may be set due to the link entering the Partial or Slumber power 4997 * management state, rather than due to a hot plug insertion or removal 4998 * event. So far, the interface power management is disabled, so the 4999 * driver can reliably get removal detection notification via the 5000 * PxSERR.DIAG.N bit. 5001 */ 5002 static int 5003 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 5004 ahci_port_t *ahci_portp, uint8_t port) 5005 { 5006 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 5007 sata_device_t sdevice; 5008 int dev_exists_now = 0; 5009 int dev_existed_previously = 0; 5010 5011 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 5012 "ahci_intr_phyrdy_change enter, port %d", port); 5013 5014 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 5015 mutex_enter(&ahci_portp->ahciport_mutex); 5016 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5017 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5018 SERROR_PHY_RDY_CHG); 5019 mutex_exit(&ahci_portp->ahciport_mutex); 5020 5021 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5022 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 5023 (ahci_portp == NULL)) { 5024 /* The whole controller setup is not yet done. */ 5025 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5026 return (AHCI_SUCCESS); 5027 } 5028 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5029 5030 mutex_enter(&ahci_portp->ahciport_mutex); 5031 5032 /* SStatus tells the presence of device. */ 5033 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5034 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 5035 5036 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 5037 dev_exists_now = 1; 5038 } 5039 5040 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 5041 dev_existed_previously = 1; 5042 } 5043 5044 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 5045 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 5046 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5047 "ahci_intr_phyrdy_change: port %d " 5048 "AHCI_PORT_FLAG_NODEV is cleared", port); 5049 if (dev_exists_now == 0) 5050 dev_existed_previously = 1; 5051 } 5052 5053 bzero((void *)&sdevice, sizeof (sata_device_t)); 5054 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5055 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5056 sdevice.satadev_addr.pmport = 0; 5057 sdevice.satadev_state = SATA_PSTATE_PWRON; 5058 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 5059 5060 if (dev_exists_now) { 5061 if (dev_existed_previously) { 5062 /* Things are fine now. The loss was temporary. */ 5063 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5064 "ahci_intr_phyrdy_change port %d " 5065 "device link lost/established", port); 5066 5067 mutex_exit(&ahci_portp->ahciport_mutex); 5068 sata_hba_event_notify( 5069 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5070 &sdevice, 5071 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 5072 mutex_enter(&ahci_portp->ahciport_mutex); 5073 5074 } else { 5075 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5076 "ahci_intr_phyrdy_change: port %d " 5077 "device link established", port); 5078 5079 /* A new device has been detected. */ 5080 ahci_find_dev_signature(ahci_ctlp, ahci_portp, port); 5081 5082 /* Try to start the port */ 5083 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 5084 != AHCI_SUCCESS) { 5085 sdevice.satadev_state |= SATA_PSTATE_FAILED; 5086 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5087 "ahci_intr_phyrdy_change: port %d failed " 5088 "at start port", port); 5089 } 5090 5091 /* Clear the max queue depth for inserted device */ 5092 ahci_portp->ahciport_max_ncq_tags = 0; 5093 5094 mutex_exit(&ahci_portp->ahciport_mutex); 5095 sata_hba_event_notify( 5096 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5097 &sdevice, 5098 SATA_EVNT_LINK_ESTABLISHED); 5099 mutex_enter(&ahci_portp->ahciport_mutex); 5100 5101 } 5102 } else { /* No device exists now */ 5103 5104 if (dev_existed_previously) { 5105 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 5106 "ahci_intr_phyrdy_change: port %d " 5107 "device link lost", port); 5108 5109 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 5110 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 5111 ahci_portp, port); 5112 5113 /* An existing device is lost. */ 5114 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 5115 5116 mutex_exit(&ahci_portp->ahciport_mutex); 5117 sata_hba_event_notify( 5118 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5119 &sdevice, 5120 SATA_EVNT_LINK_LOST); 5121 mutex_enter(&ahci_portp->ahciport_mutex); 5122 } 5123 } 5124 5125 mutex_exit(&ahci_portp->ahciport_mutex); 5126 5127 return (AHCI_SUCCESS); 5128 } 5129 5130 /* 5131 * PxIS.UFS - Unknown FIS Error 5132 * 5133 * This interrupt event means an unknown FIS was received and has been 5134 * copied into system memory. An unknown FIS is not considered an illegal 5135 * FIS, unless the length received is more than 64 bytes. If an unknown 5136 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 5137 * normal operation. If the unknown FIS is more than 64 bytes, then it 5138 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 5139 * a fatal error. 5140 * 5141 * PxIS.OFS - Overflow Error 5142 * 5143 * Command list overflow is defined as software building a command table 5144 * that has fewer total bytes than the transaction given to the device. 5145 * On device writes, the HBA will run out of data, and on reads, there 5146 * will be no room to put the data. 5147 * 5148 * For an overflow on data read, either PIO or DMA, the HBA will set 5149 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 5150 * non-fatal error when the HBA can continues. Sometimes, it will cause 5151 * a fatal error and need the software to do something. 5152 * 5153 * For an overflow on data write, setting PxIS.OFS is optional for both 5154 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 5155 * software to clean up from this serious error. 5156 * 5157 * PxIS.INFS - Interface Non-Fatal Error 5158 * 5159 * This interrupt event indicates that the HBA encountered an error on 5160 * the Serial ATA interface but was able to continue operation. The kind 5161 * of error usually occurred during a non-Data FIS, and under this condition 5162 * the FIS will be re-transmitted by HBA automatically. 5163 * 5164 * When the FMA is implemented, there should be a stat structure to 5165 * record how many every kind of error happens. 5166 */ 5167 static int 5168 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5169 uint8_t port, uint32_t intr_status) 5170 { 5171 uint32_t port_serror; 5172 #if AHCI_DEBUG 5173 uint32_t port_cmd_status; 5174 uint32_t port_cmd_issue; 5175 uint32_t port_sactive; 5176 int current_slot; 5177 uint32_t current_tags; 5178 sata_pkt_t *satapkt; 5179 ahci_cmd_header_t *cmd_header; 5180 uint32_t cmd_dmacount; 5181 #endif 5182 5183 mutex_enter(&ahci_portp->ahciport_mutex); 5184 5185 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5186 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5187 5188 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 5189 "ahci_intr_non_fatal_error: port %d, " 5190 "port_serror = 0x%x", port, port_serror); 5191 5192 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 5193 5194 if (intr_status & AHCI_INTR_STATUS_UFS) { 5195 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 5196 "ahci port %d has unknown FIS error", port); 5197 5198 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 5199 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5200 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5201 SERROR_FIS_TYPE); 5202 } 5203 5204 #if AHCI_DEBUG 5205 if (intr_status & AHCI_INTR_STATUS_OFS) { 5206 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5207 "ahci port %d has overflow error", port); 5208 } 5209 5210 if (intr_status & AHCI_INTR_STATUS_INFS) { 5211 AHCIDBG1(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5212 "ahci port %d has interface non fatal error", port); 5213 } 5214 5215 /* 5216 * Record the error occurred command's slot. 5217 */ 5218 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 5219 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5220 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5221 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5222 5223 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5224 AHCI_CMD_STATUS_CCS_SHIFT; 5225 5226 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 5227 satapkt = ahci_portp->ahciport_err_retri_pkt; 5228 ASSERT(satapkt != NULL); 5229 ASSERT(current_slot == 0); 5230 } else { 5231 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5232 } 5233 5234 if (satapkt != NULL) { 5235 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5236 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 5237 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 5238 satapkt->satapkt_cmd.satacmd_cmd_reg); 5239 5240 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5241 "ahci_intr_non_fatal_error: port %d, " 5242 "satapkt 0x%p is being processed when error occurs", 5243 port, (void *)satapkt); 5244 5245 /* 5246 * PRD Byte Count field of command header is not 5247 * required to reflect the total number of bytes 5248 * transferred when an overflow occurs, so here 5249 * just log the value. 5250 */ 5251 cmd_dmacount = 5252 ahci_portp->ahciport_prd_bytecounts[current_slot]; 5253 if (cmd_dmacount) { 5254 cmd_header = &ahci_portp-> 5255 ahciport_cmd_list[current_slot]; 5256 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5257 "ahci_intr_non_fatal_error: port %d, " 5258 "PRD Byte Count = 0x%x, " 5259 "ahciport_prd_bytecounts = 0x%x", port, 5260 cmd_header->ahcich_prd_byte_count, 5261 cmd_dmacount); 5262 } 5263 } 5264 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5265 /* 5266 * For queued command, list those command which have already 5267 * been transmitted to the device and still not completed. 5268 */ 5269 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5270 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 5271 5272 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5273 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 5274 5275 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 5276 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 5277 "port_sactive = 0x%x port_cmd_issue = 0x%x", 5278 ahci_portp->ahciport_pending_ncq_tags, 5279 port_sactive, port_cmd_issue); 5280 5281 current_tags = ahci_portp->ahciport_pending_ncq_tags & 5282 port_sactive & ~port_cmd_issue & 5283 AHCI_NCQ_SLOT_MASK(ahci_portp); 5284 5285 while (current_tags) { 5286 current_slot = ddi_ffs(current_tags) - 1; 5287 if (current_slot == -1) { 5288 goto out; 5289 } 5290 5291 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 5292 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 5293 ahci_ctlp, "ahci_intr_non_fatal_error: " 5294 "port %d, satapkt 0x%p is outstanding when " 5295 "error occurs", port, (void *)satapkt); 5296 } 5297 } 5298 out: 5299 #endif 5300 mutex_exit(&ahci_portp->ahciport_mutex); 5301 5302 return (AHCI_SUCCESS); 5303 } 5304 5305 /* 5306 * According to the AHCI spec, the error types include system memory 5307 * errors, interface errors, port multiplier errors, device errors, 5308 * command list overflow, command list underflow, native command 5309 * queuing tag errors and pio data transfer errors. 5310 * 5311 * System memory errors such as target abort, master abort, and parity 5312 * may cause the host to stop, and they are serious errors and needed 5313 * to be recovered with software intervention. When system software 5314 * has given a pointer to the HBA that doesn't exist in physical memory, 5315 * a master/target abort error occurs, and PxIS.HBFS will be set. A 5316 * data error such as CRC or parity occurs, the HBA aborts the transfer 5317 * (if necessary) and PxIS.HBDS will be set. 5318 * 5319 * Interface errors are errors that occur due to electrical issues on 5320 * the interface, or protocol miscommunication between the device and 5321 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 5322 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 5323 * causes PxIS.IFS/PxIS.INFS to be set are 5324 * 1. in PxSERR.ERR, P bit is set to '1' 5325 * 2. in PxSERR.DIAG, C or H bit is set to '1' 5326 * 3. PhyRdy drop unexpectly, N bit is set to '1' 5327 * If the error occurred during a non-data FIS, the FIS must be 5328 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 5329 * the error occurred during a data FIS, the transfer will stop, so 5330 * the error is fatal and PxIS.IFS is set. 5331 * 5332 * When a FIS arrives that updates the taskfile, the HBA checks to see 5333 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 5334 * stops processing any more commands. 5335 * 5336 * Command list overflow is defined as software building a command table 5337 * that has fewer total bytes than the transaction given to the device. 5338 * On device writes, the HBA will run out of data, and on reads, there 5339 * will be no room to put the data. For an overflow on data read, either 5340 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 5341 * For an overflow on data write, setting PxIS.OFS is optional for both 5342 * DMA and PIO, and a COMRESET is required by software to clean up from 5343 * this serious error. 5344 * 5345 * Command list underflow is defined as software building a command 5346 * table that has more total bytes than the transaction given to the 5347 * device. For data writes, both PIO and DMA, the device will detect 5348 * an error and end the transfer. And these errors are most likely going 5349 * to be fatal errors that will cause the port to be restarted. For 5350 * data reads, the HBA updates its PRD byte count, and may be 5351 * able to continue normally, but is not required to. And The HBA is 5352 * not required to detect underflow conditions for native command 5353 * queuing command. 5354 * 5355 * The HBA does not actively check incoming DMA Setup FISes to ensure 5356 * that the PxSACT register bit for that slot is set. Existing error 5357 * mechanisms, such as host bus failure, or bad protocol, are used to 5358 * recover from this case. 5359 * 5360 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 5361 * DATA FIS must be an integral number of Dwords. If the HBA receives 5362 * a request which is not an integral number of Dwords, the HBA 5363 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 5364 * software restarts the port. And the HBA ensures that the size 5365 * of the DATA FIS received during a PIO command matches the size in 5366 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 5367 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 5368 * stop running until software restarts the port. 5369 */ 5370 /* 5371 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 5372 * 5373 * PxIS.IFS indicates that the hba encountered an error on the serial ata 5374 * interface which caused the transfer to stop. 5375 * 5376 * PxIS.HBDS indicates that the hba encountered a data error 5377 * (uncorrectable ecc/parity) when reading from or writing to system memory. 5378 * 5379 * PxIS.HBFS indicates that the hba encountered a host bus error that it 5380 * cannot recover from, such as a bad software pointer. 5381 * 5382 * PxIS.TFES is set whenever the status register is updated by the device 5383 * and the error bit (bit 0) is set. 5384 */ 5385 static int 5386 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 5387 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 5388 { 5389 uint32_t port_cmd_status; 5390 uint32_t port_serror; 5391 uint32_t task_file_status; 5392 int failed_slot; 5393 sata_pkt_t *spkt = NULL; 5394 uint8_t err_byte; 5395 ahci_event_arg_t *args; 5396 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5397 5398 mutex_enter(&ahci_portp->ahciport_mutex); 5399 5400 /* 5401 * ahci_intr_phyrdy_change() may have rendered it to 5402 * SATA_DTYPE_NONE. 5403 */ 5404 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5405 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 5406 "ahci_intr_fatal_error: port %d no device attached, " 5407 "and just return without doing anything", port); 5408 goto out0; 5409 } 5410 5411 if (intr_status & AHCI_INTR_STATUS_TFES) { 5412 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5413 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5414 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5415 "ahci_intr_fatal_error: port %d " 5416 "task_file_status = 0x%x", port, task_file_status); 5417 } 5418 5419 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 5420 /* 5421 * Read PxCMD.CCS to determine the slot that the HBA 5422 * was processing when the error occurred. 5423 */ 5424 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5425 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5426 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 5427 AHCI_CMD_STATUS_CCS_SHIFT; 5428 5429 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 5430 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 5431 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 5432 "fatal error occurred for port %d", spkt, port); 5433 5434 if (intr_status & AHCI_INTR_STATUS_TFES) { 5435 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 5436 >> AHCI_TFD_ERR_SHIFT; 5437 5438 /* 5439 * Won't emit the error message if it is an IDENTIFY 5440 * DEVICE command sent to an ATAPI device. 5441 */ 5442 if ((spkt != NULL) && 5443 (spkt->satapkt_cmd.satacmd_cmd_reg == 5444 SATAC_ID_DEVICE) && 5445 (err_byte == SATA_ERROR_ABORT)) 5446 goto out1; 5447 5448 /* 5449 * Won't emit the error message if it is an ATAPI PACKET 5450 * command 5451 */ 5452 if ((spkt != NULL) && 5453 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 5454 goto out1; 5455 } 5456 } 5457 5458 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 5459 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5460 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5461 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 5462 out1: 5463 /* Prepare the argument for the taskq */ 5464 args = ahci_portp->ahciport_event_args; 5465 args->ahciea_ctlp = (void *)ahci_ctlp; 5466 args->ahciea_portp = (void *)ahci_portp; 5467 args->ahciea_event = intr_status; 5468 5469 /* Start the taskq to handle error recovery */ 5470 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 5471 ahci_events_handler, 5472 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 5473 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 5474 "failed", instance); 5475 } 5476 out0: 5477 mutex_exit(&ahci_portp->ahciport_mutex); 5478 5479 return (AHCI_SUCCESS); 5480 } 5481 5482 /* 5483 * Hot Plug Operation for platforms that support Cold Presence Detect. 5484 * 5485 * When set, a device status has changed as detected by the cold presence 5486 * detect logic. This bit can either be set due to a non-connected port 5487 * receiving a device, or a connected port having its device removed. 5488 * This bit is only valid if the port supports cold presence detect as 5489 * indicated by PxCMD.CPD set to '1'. 5490 * 5491 * At the moment, this interrupt is not needed and disabled and we just 5492 * log the debug message. 5493 */ 5494 static int 5495 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 5496 ahci_port_t *ahci_portp, uint8_t port) 5497 { 5498 uint32_t port_cmd_status; 5499 sata_device_t sdevice; 5500 5501 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5502 "ahci_intr_cold_port_detect enter, port %d", port); 5503 5504 mutex_enter(&ahci_portp->ahciport_mutex); 5505 5506 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5507 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5508 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 5509 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5510 "port %d does not support cold presence detect, so " 5511 "we just ignore this interrupt", port); 5512 mutex_exit(&ahci_portp->ahciport_mutex); 5513 return (AHCI_SUCCESS); 5514 } 5515 5516 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5517 "port %d device status has changed", port); 5518 5519 bzero((void *)&sdevice, sizeof (sata_device_t)); 5520 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 5521 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 5522 sdevice.satadev_addr.pmport = 0; 5523 sdevice.satadev_state = SATA_PSTATE_PWRON; 5524 5525 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 5526 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5527 "port %d: a device is hot plugged", port); 5528 mutex_exit(&ahci_portp->ahciport_mutex); 5529 sata_hba_event_notify( 5530 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5531 &sdevice, 5532 SATA_EVNT_DEVICE_ATTACHED); 5533 mutex_enter(&ahci_portp->ahciport_mutex); 5534 5535 } else { 5536 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5537 "port %d: a device is hot unplugged", port); 5538 mutex_exit(&ahci_portp->ahciport_mutex); 5539 sata_hba_event_notify( 5540 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 5541 &sdevice, 5542 SATA_EVNT_DEVICE_DETACHED); 5543 mutex_enter(&ahci_portp->ahciport_mutex); 5544 } 5545 5546 mutex_exit(&ahci_portp->ahciport_mutex); 5547 5548 return (AHCI_SUCCESS); 5549 } 5550 5551 /* 5552 * Enable the interrupts for a particular port. 5553 * 5554 * WARNING!!! ahciport_mutex should be acquired before the function 5555 * is called. 5556 */ 5557 static void 5558 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5559 { 5560 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5561 "ahci_enable_port_intrs enter, port %d", port); 5562 5563 /* 5564 * Clear port interrupt status before enabling interrupt 5565 */ 5566 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5567 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5568 AHCI_PORT_INTR_MASK); 5569 5570 /* 5571 * Clear the pending bit from IS.IPS 5572 */ 5573 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5574 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 5575 5576 /* 5577 * Enable the following interrupts: 5578 * Device to Host Register FIS Interrupt (DHRS) 5579 * PIO Setup FIS Interrupt (PSS) 5580 * Set Device Bits Interrupt (SDBS) 5581 * Unknown FIS Interrupt (UFS) 5582 * Port Connect Change Status (PCS) 5583 * PhyRdy Change Status (PRCS) 5584 * Overflow Status (OFS) 5585 * Interface Non-fatal Error Status (INFS) 5586 * Interface Fatal Error Status (IFS) 5587 * Host Bus Data Error Status (HBDS) 5588 * Host Bus Fatal Error Status (HBFS) 5589 * Task File Error Status (TFES) 5590 */ 5591 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5592 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 5593 (AHCI_INTR_STATUS_DHRS | 5594 AHCI_INTR_STATUS_PSS | 5595 AHCI_INTR_STATUS_SDBS | 5596 AHCI_INTR_STATUS_UFS | 5597 AHCI_INTR_STATUS_DPS | 5598 AHCI_INTR_STATUS_PCS | 5599 AHCI_INTR_STATUS_PRCS | 5600 AHCI_INTR_STATUS_OFS | 5601 AHCI_INTR_STATUS_INFS | 5602 AHCI_INTR_STATUS_IFS | 5603 AHCI_INTR_STATUS_HBDS | 5604 AHCI_INTR_STATUS_HBFS | 5605 AHCI_INTR_STATUS_TFES)); 5606 } 5607 5608 /* 5609 * Enable interrupts for all the ports. 5610 * 5611 * WARNING!!! ahcictl_mutex should be acquired before the function 5612 * is called. 5613 */ 5614 static void 5615 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 5616 { 5617 uint32_t ghc_control; 5618 5619 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter"); 5620 5621 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5622 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5623 5624 ghc_control |= AHCI_HBA_GHC_IE; 5625 5626 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5627 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5628 } 5629 5630 /* 5631 * Disable interrupts for a particular port. 5632 * 5633 * WARNING!!! ahciport_mutex should be acquired before the function 5634 * is called. 5635 */ 5636 static void 5637 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 5638 { 5639 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5640 "ahci_disable_port_intrs enter, port %d", port); 5641 5642 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5643 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 5644 } 5645 5646 /* 5647 * Disable interrupts for the whole HBA. 5648 * 5649 * The global bit is cleared, then all interrupt sources from all 5650 * ports are disabled. 5651 * 5652 * WARNING!!! ahcictl_mutex should be acquired before the function 5653 * is called. 5654 */ 5655 static void 5656 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 5657 { 5658 uint32_t ghc_control; 5659 5660 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter"); 5661 5662 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5663 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5664 5665 ghc_control &= ~ AHCI_HBA_GHC_IE; 5666 5667 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5668 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5669 } 5670 5671 /* 5672 * Handle FIXED or MSI interrupts. 5673 */ 5674 /* 5675 * According to AHCI spec, the HBA may support several interrupt modes: 5676 * * pin based interrupts (FIXED) 5677 * * single MSI message interrupts 5678 * * multiple MSI based message interrupts 5679 * 5680 * For pin based interrupts, the software interrupt handler need to check IS 5681 * register to find out which port has pending interrupts. And then check 5682 * PxIS register to find out which interrupt events happened on that port. 5683 * 5684 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 5685 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 5686 * in that software interrupt handler need to check IS register to determine 5687 * which port triggered the interrupts since it uses a single message for all 5688 * port interrupts. 5689 * 5690 * HBA may optionally support multiple MSI message for better performance. In 5691 * this mode, each port may have its own interrupt message, and thus generation 5692 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 5693 * represents a power-of-2 wrapper on the number of implemented ports, and 5694 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 5695 * maximum number of assigned interrupts. When the number of MSI messages 5696 * allocated is less than the number requested, then hardware may have two 5697 * implementation behaviors: 5698 * * assign each ports its own interrupt and then force all additional 5699 * ports to share the last interrupt message, and this condition is 5700 * indicated by clearing GHC.MRSM to '0' 5701 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 5702 * When multiple-message MSI is enabled, hardware will still set IS register 5703 * as single message case. And this IS register may be used by software when 5704 * fewer than the requested number of messages is granted in order to determine 5705 * which port had the interrupt. 5706 * 5707 * Note: The current ahci driver only supports the first two interrupt modes: 5708 * pin based interrupts and single MSI message interrupts, and the reason 5709 * is indicated in below code. 5710 */ 5711 static int 5712 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 5713 { 5714 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5715 int count, avail, actual; 5716 int i, rc; 5717 5718 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 5719 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 5720 5721 /* get number of interrupts. */ 5722 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 5723 if ((rc != DDI_SUCCESS) || (count == 0)) { 5724 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5725 "ddi_intr_get_nintrs() failed, " 5726 "rc %d count %d\n", rc, count); 5727 return (DDI_FAILURE); 5728 } 5729 5730 /* get number of available interrupts. */ 5731 rc = ddi_intr_get_navail(dip, intr_type, &avail); 5732 if ((rc != DDI_SUCCESS) || (avail == 0)) { 5733 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5734 "ddi_intr_get_navail() failed, " 5735 "rc %d avail %d\n", rc, avail); 5736 return (DDI_FAILURE); 5737 } 5738 5739 #if AHCI_DEBUG 5740 if (avail < count) { 5741 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5742 "ddi_intr_get_nintrs returned %d, navail() returned %d", 5743 count, avail); 5744 } 5745 #endif 5746 5747 /* 5748 * Note: So far Solaris restricts the maximum number of messages for 5749 * x86 to 2, that is avail is 2, so here we set the count with 1 to 5750 * force the driver to use single MSI message interrupt. In future if 5751 * Solaris remove the restriction, then we need to delete the below 5752 * code and try to use multiple interrupt routine to gain better 5753 * performance. 5754 */ 5755 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 5756 AHCIDBG1(AHCIDBG_INTR, ahci_ctlp, 5757 "force to use one interrupt routine though the " 5758 "HBA supports %d interrupt", count); 5759 count = 1; 5760 } 5761 5762 /* Allocate an array of interrupt handles. */ 5763 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 5764 ahci_ctlp->ahcictl_intr_htable = 5765 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 5766 5767 /* call ddi_intr_alloc(). */ 5768 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 5769 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 5770 5771 if ((rc != DDI_SUCCESS) || (actual == 0)) { 5772 AHCIDBG4(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5773 "ddi_intr_alloc() failed, rc %d count %d actual %d " 5774 "avail %d\n", rc, count, actual, avail); 5775 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5776 ahci_ctlp->ahcictl_intr_size); 5777 return (DDI_FAILURE); 5778 } 5779 5780 /* use interrupt count returned */ 5781 #if AHCI_DEBUG 5782 if (actual < count) { 5783 AHCIDBG2(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5784 "Requested: %d, Received: %d", count, actual); 5785 } 5786 #endif 5787 5788 ahci_ctlp->ahcictl_intr_cnt = actual; 5789 5790 /* 5791 * Get priority for first, assume remaining are all the same. 5792 */ 5793 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 5794 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 5795 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5796 "ddi_intr_get_pri() failed"); 5797 5798 /* Free already allocated intr. */ 5799 for (i = 0; i < actual; i++) { 5800 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5801 } 5802 5803 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5804 ahci_ctlp->ahcictl_intr_size); 5805 return (DDI_FAILURE); 5806 } 5807 5808 /* Test for high level interrupt. */ 5809 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 5810 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5811 "ahci_add_intrs: Hi level intr not supported"); 5812 5813 /* Free already allocated intr. */ 5814 for (i = 0; i < actual; i++) { 5815 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 5816 } 5817 5818 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5819 sizeof (ddi_intr_handle_t)); 5820 5821 return (DDI_FAILURE); 5822 } 5823 5824 /* Call ddi_intr_add_handler(). */ 5825 for (i = 0; i < actual; i++) { 5826 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 5827 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 5828 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5829 "ddi_intr_add_handler() failed"); 5830 5831 /* Free already allocated intr. */ 5832 for (i = 0; i < actual; i++) { 5833 (void) ddi_intr_free( 5834 ahci_ctlp->ahcictl_intr_htable[i]); 5835 } 5836 5837 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5838 ahci_ctlp->ahcictl_intr_size); 5839 return (DDI_FAILURE); 5840 } 5841 } 5842 5843 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 5844 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 5845 AHCIDBG0(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 5846 "ddi_intr_get_cap() failed"); 5847 5848 /* Free already allocated intr. */ 5849 for (i = 0; i < actual; i++) { 5850 (void) ddi_intr_free( 5851 ahci_ctlp->ahcictl_intr_htable[i]); 5852 } 5853 5854 kmem_free(ahci_ctlp->ahcictl_intr_htable, 5855 ahci_ctlp->ahcictl_intr_size); 5856 return (DDI_FAILURE); 5857 } 5858 5859 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 5860 /* Call ddi_intr_block_enable() for MSI. */ 5861 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 5862 ahci_ctlp->ahcictl_intr_cnt); 5863 } else { 5864 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 5865 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 5866 (void) ddi_intr_enable( 5867 ahci_ctlp->ahcictl_intr_htable[i]); 5868 } 5869 } 5870 5871 return (DDI_SUCCESS); 5872 } 5873 5874 /* 5875 * Removes the registered interrupts irrespective of whether they 5876 * were legacy or MSI. 5877 * 5878 * WARNING!!! The controller interrupts must be disabled before calling 5879 * this routine. 5880 */ 5881 static void 5882 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 5883 { 5884 int x; 5885 5886 AHCIDBG0(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered"); 5887 5888 /* Disable all interrupts. */ 5889 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 5890 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 5891 /* Call ddi_intr_block_disable(). */ 5892 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 5893 ahci_ctlp->ahcictl_intr_cnt); 5894 } else { 5895 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5896 (void) ddi_intr_disable( 5897 ahci_ctlp->ahcictl_intr_htable[x]); 5898 } 5899 } 5900 5901 /* Call ddi_intr_remove_handler(). */ 5902 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 5903 (void) ddi_intr_remove_handler( 5904 ahci_ctlp->ahcictl_intr_htable[x]); 5905 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 5906 } 5907 5908 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 5909 } 5910 5911 /* 5912 * This routine tries to put port into P:NotRunning state by clearing 5913 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 5914 * and PxCMD.CR to '0'. 5915 * 5916 * WARNING!!! ahciport_mutex should be acquired before the function 5917 * is called. 5918 */ 5919 static int 5920 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 5921 ahci_port_t *ahci_portp, uint8_t port) 5922 { 5923 uint32_t port_cmd_status; 5924 int loop_count; 5925 5926 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 5927 "ahci_put_port_into_notrunning_state enter: port %d", port); 5928 5929 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5930 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5931 5932 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 5933 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5934 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5935 5936 /* Wait until PxCMD.CR is cleared */ 5937 loop_count = 0; 5938 do { 5939 port_cmd_status = 5940 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5941 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5942 5943 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 5944 AHCIDBG2(AHCIDBG_INIT, ahci_ctlp, 5945 "clearing port %d CMD.CR timeout, " 5946 "port_cmd_status = 0x%x", port, 5947 port_cmd_status); 5948 /* 5949 * We are effectively timing out after 0.5 sec. 5950 * This value is specified in AHCI spec. 5951 */ 5952 break; 5953 } 5954 5955 /* Wait for 10 millisec */ 5956 drv_usecwait(10000); 5957 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 5958 5959 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 5960 5961 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 5962 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5963 "ahci_put_port_into_notrunning_state: failed to clear " 5964 "PxCMD.CR to '0' after loop count: %d, and " 5965 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 5966 return (AHCI_FAILURE); 5967 } else { 5968 AHCIDBG2(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5969 "ahci_put_port_into_notrunning_state: succeeded to clear " 5970 "PxCMD.CR to '0' after loop count: %d, and " 5971 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 5972 return (AHCI_SUCCESS); 5973 } 5974 } 5975 5976 /* 5977 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 5978 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 5979 * stable state, then set PxCMD.ST to '1' to start the port directly. 5980 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 5981 * COMRESET to the device to put it in an idle state. 5982 * 5983 * The fifth argument returns whether the port reset is involved during 5984 * the process. 5985 * 5986 * The routine will be called under six scenarios: 5987 * 1. Initialize the port 5988 * 2. To abort the packet(s) 5989 * 3. To reset the port 5990 * 4. To activate the port 5991 * 5. Fatal error recovery 5992 * 6. To abort the timeout packet(s) 5993 * 5994 * WARNING!!! ahciport_mutex should be acquired before the function 5995 * is called. And ahciport_mutex will be released before the reset 5996 * event is reported to sata module by calling sata_hba_event_notify, 5997 * and then be acquired again later. 5998 * 5999 * NOTES!!! During this procedure, PxSERR register will be cleared, and 6000 * according to the spec, the clearance of three bits will also clear 6001 * three interrupt status bits. 6002 * 1. PxSERR.DIAG.F will clear PxIS.UFS 6003 * 2. PxSERR.DIAG.X will clear PxIS.PCS 6004 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 6005 * 6006 * Among these three interrupt events, the driver needs to take care of 6007 * PxIS.PRCS, which is the hot plug event. When the driver found out 6008 * a device was unplugged, it will call the interrupt handler. 6009 */ 6010 static int 6011 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 6012 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 6013 { 6014 uint32_t port_sstatus; 6015 uint32_t task_file_status; 6016 sata_device_t sdevice; 6017 int rval; 6018 int dev_exists_begin = 0; 6019 int dev_exists_end = 0; 6020 6021 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6022 "ahci_restart_port_wait_till_ready: port %d enter", port); 6023 6024 if ((flag != AHCI_PORT_INIT) && 6025 (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)) 6026 dev_exists_begin = 1; 6027 6028 /* First clear PxCMD.ST */ 6029 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 6030 port); 6031 if (rval != AHCI_SUCCESS) 6032 /* 6033 * If PxCMD.CR does not clear within a reasonable time, it 6034 * may assume the interface is in a hung condition and may 6035 * continue with issuing the port reset. 6036 */ 6037 goto reset; 6038 6039 /* Then clear PxSERR */ 6040 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6041 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6042 AHCI_SERROR_CLEAR_ALL); 6043 6044 /* Then get PxTFD */ 6045 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6046 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 6047 6048 /* 6049 * Check whether the device is in a stable status, if yes, 6050 * then start the port directly. However for ahci_tran_dport_reset, 6051 * we may have to perform a port reset. 6052 */ 6053 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 6054 !(flag & AHCI_PORT_RESET)) 6055 goto out; 6056 6057 reset: 6058 /* 6059 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 6060 * a COMRESET to the device 6061 */ 6062 rval = ahci_port_reset(ahci_ctlp, ahci_portp, port); 6063 6064 if (reset_flag != NULL) 6065 *reset_flag = 1; 6066 6067 /* Indicate to the framework that a reset has happened. */ 6068 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 6069 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 6070 /* Set the reset in progress flag */ 6071 ahci_portp->ahciport_reset_in_progress = 1; 6072 6073 bzero((void *)&sdevice, sizeof (sata_device_t)); 6074 sdevice.satadev_addr.cport = 6075 ahci_ctlp->ahcictl_port_to_cport[port]; 6076 sdevice.satadev_addr.pmport = 0; 6077 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6078 6079 sdevice.satadev_state = SATA_DSTATE_RESET | 6080 SATA_DSTATE_PWR_ACTIVE; 6081 if (ahci_ctlp->ahcictl_sata_hba_tran) { 6082 mutex_exit(&ahci_portp->ahciport_mutex); 6083 sata_hba_event_notify( 6084 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6085 &sdevice, 6086 SATA_EVNT_DEVICE_RESET); 6087 mutex_enter(&ahci_portp->ahciport_mutex); 6088 } 6089 6090 AHCIDBG1(AHCIDBG_EVENT, ahci_ctlp, 6091 "port %d sending event up: SATA_EVNT_RESET", port); 6092 } else { 6093 ahci_portp->ahciport_reset_in_progress = 0; 6094 } 6095 6096 out: 6097 /* Start the port if not in initialization phase */ 6098 if (flag & AHCI_PORT_INIT) 6099 return (rval); 6100 6101 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 6102 6103 /* SStatus tells the presence of device. */ 6104 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6105 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 6106 6107 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 6108 dev_exists_end = 1; 6109 ASSERT(ahci_portp->ahciport_device_type != SATA_DTYPE_NONE); 6110 } 6111 6112 /* Check whether a hot plug event happened */ 6113 if (dev_exists_begin == 1 && dev_exists_end == 0) { 6114 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6115 "ahci_restart_port_wait_till_ready: port %d " 6116 "device is removed", port); 6117 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 6118 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6119 "ahci_restart_port_wait_till_ready: port %d " 6120 "AHCI_PORT_FLAG_NODEV flag is set", port); 6121 mutex_exit(&ahci_portp->ahciport_mutex); 6122 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 6123 mutex_enter(&ahci_portp->ahciport_mutex); 6124 } 6125 6126 return (rval); 6127 } 6128 6129 /* 6130 * This routine may be called under four scenarios: 6131 * a) do the recovery from fatal error 6132 * b) or we need to timeout some commands 6133 * c) or we need to abort some commands 6134 * d) or we need reset device/port/controller 6135 * 6136 * In all these scenarios, we need to send any pending unfinished 6137 * commands up to sata framework. 6138 * 6139 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6140 */ 6141 static void 6142 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 6143 ahci_port_t *ahci_portp, 6144 uint32_t slot_status, 6145 uint32_t failed_tags, 6146 uint32_t timeout_tags, 6147 uint32_t aborted_tags, 6148 uint32_t reset_tags) 6149 { 6150 uint32_t finished_tags = 0; 6151 uint32_t unfinished_tags = 0; 6152 int tmp_slot; 6153 sata_pkt_t *satapkt; 6154 int ncq_cmd_in_progress = 0; 6155 int err_retri_cmd_in_progress = 0; 6156 6157 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6158 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 6159 ahci_portp->ahciport_port_num, slot_status); 6160 6161 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 6162 "ahci_mop_commands: failed_tags: 0x%x, " 6163 "timeout_tags: 0x%x aborted_tags: 0x%x, " 6164 "reset_tags: 0x%x", failed_tags, 6165 timeout_tags, aborted_tags, reset_tags); 6166 6167 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6168 finished_tags = ahci_portp->ahciport_pending_tags & 6169 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6170 6171 unfinished_tags = slot_status & 6172 AHCI_SLOT_MASK(ahci_ctlp) & 6173 ~failed_tags & 6174 ~aborted_tags & 6175 ~reset_tags & 6176 ~timeout_tags; 6177 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6178 ncq_cmd_in_progress = 1; 6179 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6180 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6181 6182 unfinished_tags = slot_status & 6183 AHCI_NCQ_SLOT_MASK(ahci_portp) & 6184 ~failed_tags & 6185 ~aborted_tags & 6186 ~reset_tags & 6187 ~timeout_tags; 6188 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6189 6190 /* 6191 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 6192 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 6193 * complete successfully due to one of the following three 6194 * conditions: 6195 * 6196 * 1. Fatal error - failed_tags includes its slot 6197 * 2. Timed out - timeout_tags includes its slot 6198 * 3. Aborted when hot unplug - aborted_tags includes its 6199 * slot 6200 * 6201 * Please note that the command is always sent down in Slot 0 6202 */ 6203 err_retri_cmd_in_progress = 1; 6204 AHCIDBG2(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 6205 "ahci_mop_commands is called for port %d while " 6206 "REQUEST SENSE or READ LOG EXT for error retrieval " 6207 "is being executed slot_status = 0x%x", 6208 ahci_portp->ahciport_port_num, slot_status); 6209 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 6210 ASSERT(slot_status == 0x1); 6211 } 6212 6213 /* Send up finished packets with SATA_PKT_COMPLETED */ 6214 while (finished_tags) { 6215 tmp_slot = ddi_ffs(finished_tags) - 1; 6216 if (tmp_slot == -1) { 6217 break; 6218 } 6219 6220 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6221 ASSERT(satapkt != NULL); 6222 6223 AHCIDBG1(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 6224 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 6225 (void *)satapkt); 6226 6227 /* 6228 * Cannot fetch the return register content since the port 6229 * was restarted, so the corresponding tag will be set to 6230 * aborted tags. 6231 */ 6232 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6233 CLEAR_BIT(finished_tags, tmp_slot); 6234 aborted_tags |= tmp_slot; 6235 continue; 6236 } 6237 6238 if (ncq_cmd_in_progress) 6239 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6240 tmp_slot); 6241 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6242 CLEAR_BIT(finished_tags, tmp_slot); 6243 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6244 6245 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6246 } 6247 6248 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 6249 while (failed_tags) { 6250 if (err_retri_cmd_in_progress) { 6251 satapkt = ahci_portp->ahciport_err_retri_pkt; 6252 ASSERT(satapkt != NULL); 6253 ASSERT(failed_tags == 0x1); 6254 6255 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6256 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6257 (void *)satapkt); 6258 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6259 break; 6260 } 6261 6262 tmp_slot = ddi_ffs(failed_tags) - 1; 6263 if (tmp_slot == -1) { 6264 break; 6265 } 6266 6267 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6268 ASSERT(satapkt != NULL); 6269 6270 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6271 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 6272 (void *)satapkt); 6273 6274 if (ncq_cmd_in_progress) 6275 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6276 tmp_slot); 6277 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6278 CLEAR_BIT(failed_tags, tmp_slot); 6279 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6280 6281 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 6282 } 6283 6284 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 6285 while (timeout_tags) { 6286 if (err_retri_cmd_in_progress) { 6287 satapkt = ahci_portp->ahciport_err_retri_pkt; 6288 ASSERT(satapkt != NULL); 6289 ASSERT(timeout_tags == 0x1); 6290 6291 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6292 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6293 (void *)satapkt); 6294 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6295 break; 6296 } 6297 6298 tmp_slot = ddi_ffs(timeout_tags) - 1; 6299 if (tmp_slot == -1) { 6300 break; 6301 } 6302 6303 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6304 ASSERT(satapkt != NULL); 6305 6306 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6307 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 6308 (void *)satapkt); 6309 6310 if (ncq_cmd_in_progress) 6311 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6312 tmp_slot); 6313 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6314 CLEAR_BIT(timeout_tags, tmp_slot); 6315 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6316 6317 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 6318 } 6319 6320 /* Send up aborted packets with SATA_PKT_ABORTED */ 6321 while (aborted_tags) { 6322 if (err_retri_cmd_in_progress) { 6323 satapkt = ahci_portp->ahciport_err_retri_pkt; 6324 ASSERT(satapkt != NULL); 6325 ASSERT(aborted_tags == 0x1); 6326 6327 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6328 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6329 (void *)satapkt); 6330 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6331 break; 6332 } 6333 6334 tmp_slot = ddi_ffs(aborted_tags) - 1; 6335 if (tmp_slot == -1) { 6336 break; 6337 } 6338 6339 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6340 ASSERT(satapkt != NULL); 6341 6342 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6343 "sending up pkt 0x%p with SATA_PKT_ABORTED", 6344 (void *)satapkt); 6345 6346 if (ncq_cmd_in_progress) 6347 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6348 tmp_slot); 6349 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6350 CLEAR_BIT(aborted_tags, tmp_slot); 6351 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6352 6353 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 6354 } 6355 6356 /* Send up reset packets with SATA_PKT_RESET. */ 6357 while (reset_tags) { 6358 tmp_slot = ddi_ffs(reset_tags) - 1; 6359 if (tmp_slot == -1) { 6360 break; 6361 } 6362 6363 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6364 ASSERT(satapkt != NULL); 6365 6366 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6367 "sending up pkt 0x%p with SATA_PKT_RESET", 6368 (void *)satapkt); 6369 6370 if (ncq_cmd_in_progress) 6371 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6372 tmp_slot); 6373 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6374 CLEAR_BIT(reset_tags, tmp_slot); 6375 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6376 6377 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6378 } 6379 6380 /* Send up unfinished packets with SATA_PKT_RESET */ 6381 while (unfinished_tags) { 6382 tmp_slot = ddi_ffs(unfinished_tags) - 1; 6383 if (tmp_slot == -1) { 6384 break; 6385 } 6386 6387 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 6388 ASSERT(satapkt != NULL); 6389 6390 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 6391 "sending up pkt 0x%p with SATA_PKT_RESET", 6392 (void *)satapkt); 6393 6394 if (ncq_cmd_in_progress) 6395 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 6396 tmp_slot); 6397 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 6398 CLEAR_BIT(unfinished_tags, tmp_slot); 6399 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 6400 6401 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 6402 } 6403 6404 ahci_portp->ahciport_mop_in_progress--; 6405 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 6406 6407 if (ahci_portp->ahciport_mop_in_progress == 0) 6408 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 6409 } 6410 6411 /* 6412 * This routine is going to first request a READ LOG EXT sata pkt from sata 6413 * module, and then deliver it to the HBA to get the ncq failure context. 6414 * The return value is the exactly failed tags. 6415 */ 6416 static uint32_t 6417 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6418 uint8_t port) 6419 { 6420 sata_device_t sdevice; 6421 sata_pkt_t *rdlog_spkt, *spkt; 6422 ddi_dma_handle_t buf_dma_handle; 6423 int loop_count; 6424 int rval; 6425 int failed_slot; 6426 uint32_t failed_tags = 0; 6427 struct sata_ncq_error_recovery_page *ncq_err_page; 6428 6429 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 6430 "ahci_get_rdlogext_data enter: port %d", port); 6431 6432 /* Prepare the sdevice data */ 6433 bzero((void *)&sdevice, sizeof (sata_device_t)); 6434 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6435 6436 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6437 sdevice.satadev_addr.pmport = 0; 6438 6439 /* 6440 * Call the sata hba interface to get a rdlog spkt 6441 */ 6442 loop_count = 0; 6443 loop: 6444 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6445 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 6446 if (rdlog_spkt == NULL) { 6447 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6448 /* Sleep for a while */ 6449 delay(AHCI_10MS_TICKS); 6450 goto loop; 6451 } 6452 /* Timed out after 1s */ 6453 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6454 "failed to get rdlog spkt for port %d", port); 6455 return (failed_tags); 6456 } 6457 6458 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6459 6460 /* 6461 * This flag is used to handle the specific error recovery when the 6462 * READ LOG EXT command gets a failure (fatal error or time-out). 6463 */ 6464 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 6465 6466 /* 6467 * This start is not supposed to fail because after port is restarted, 6468 * the whole command list is empty. 6469 */ 6470 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 6471 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rdlog_spkt); 6472 ahci_portp->ahciport_err_retri_pkt = NULL; 6473 6474 /* Remove the flag after READ LOG EXT command is completed */ 6475 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 6476 6477 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6478 /* Update the request log data */ 6479 buf_dma_handle = *(ddi_dma_handle_t *) 6480 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6481 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 6482 DDI_DMA_SYNC_FORKERNEL); 6483 if (rval == DDI_SUCCESS) { 6484 ncq_err_page = 6485 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 6486 satapkt_cmd.satacmd_bp->b_un.b_addr; 6487 6488 /* Get the failed tag */ 6489 failed_slot = ncq_err_page->ncq_tag; 6490 AHCIDBG2(AHCIDBG_ERRS, ahci_ctlp, 6491 "ahci_get_rdlogext_data: port %d " 6492 "failed slot %d", port, failed_slot); 6493 if (failed_slot & NQ) { 6494 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6495 "the failed slot is not a valid tag"); 6496 goto out; 6497 } 6498 6499 failed_slot &= NCQ_TAG_MASK; 6500 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6501 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6502 "ahci_get_rdlogext_data: failed spkt 0x%p", 6503 (void *)spkt); 6504 if (spkt == NULL) { 6505 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, 6506 "the failed slot spkt is NULL") 6507 goto out; 6508 } 6509 6510 failed_tags = 0x1 << failed_slot; 6511 6512 /* Fill out the error context */ 6513 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 6514 ncq_err_page); 6515 ahci_update_sata_registers(ahci_ctlp, port, 6516 &spkt->satapkt_device); 6517 } 6518 } 6519 out: 6520 sata_free_error_retrieval_pkt(rdlog_spkt); 6521 6522 return (failed_tags); 6523 } 6524 6525 /* 6526 * This routine is going to first request a REQUEST SENSE sata pkt from sata 6527 * module, and then deliver it to the HBA to get the sense data and copy 6528 * the sense data back to the orignal failed sata pkt, and free the REQUEST 6529 * SENSE sata pkt later. 6530 */ 6531 static void 6532 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6533 uint8_t port, sata_pkt_t *spkt) 6534 { 6535 sata_device_t sdevice; 6536 sata_pkt_t *rs_spkt; 6537 sata_cmd_t *sata_cmd; 6538 ddi_dma_handle_t buf_dma_handle; 6539 int loop_count; 6540 #if AHCI_DEBUG 6541 struct scsi_extended_sense *rqsense; 6542 #endif 6543 6544 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6545 "ahci_get_rqsense_data enter: port %d", port); 6546 6547 /* Prepare the sdevice data */ 6548 bzero((void *)&sdevice, sizeof (sata_device_t)); 6549 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 6550 6551 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 6552 sdevice.satadev_addr.pmport = 0; 6553 6554 sata_cmd = &spkt->satapkt_cmd; 6555 6556 /* 6557 * Call the sata hba interface to get a rs spkt 6558 */ 6559 loop_count = 0; 6560 loop: 6561 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 6562 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 6563 if (rs_spkt == NULL) { 6564 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 6565 /* Sleep for a while */ 6566 delay(AHCI_10MS_TICKS); 6567 goto loop; 6568 6569 } 6570 /* Timed out after 1s */ 6571 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6572 "failed to get rs spkt for port %d", port); 6573 return; 6574 } 6575 6576 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 6577 6578 /* 6579 * This flag is used to handle the specific error recovery when the 6580 * REQUEST SENSE command gets a faiure (fatal error or time-out). 6581 */ 6582 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 6583 6584 /* 6585 * This start is not supposed to fail because after port is restarted, 6586 * the whole command list is empty. 6587 */ 6588 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 6589 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, port, rs_spkt); 6590 ahci_portp->ahciport_err_retri_pkt = NULL; 6591 6592 /* Remove the flag after REQUEST SENSE command is completed */ 6593 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 6594 6595 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 6596 /* Update the request sense data */ 6597 buf_dma_handle = *(ddi_dma_handle_t *) 6598 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 6599 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 6600 DDI_DMA_SYNC_FORKERNEL); 6601 /* Copy the request sense data */ 6602 bcopy(rs_spkt-> 6603 satapkt_cmd.satacmd_bp->b_un.b_addr, 6604 &sata_cmd->satacmd_rqsense, 6605 SATA_ATAPI_MIN_RQSENSE_LEN); 6606 #if AHCI_DEBUG 6607 rqsense = (struct scsi_extended_sense *) 6608 sata_cmd->satacmd_rqsense; 6609 6610 /* Dump the sense data */ 6611 AHCIDBG0(AHCIDBG_SENSEDATA, ahci_ctlp, "\n"); 6612 AHCIDBG2(AHCIDBG_SENSEDATA, ahci_ctlp, 6613 "Sense data for satapkt %p ATAPI cmd 0x%x", 6614 spkt, sata_cmd->satacmd_acdb[0]); 6615 AHCIDBG5(AHCIDBG_SENSEDATA, ahci_ctlp, 6616 " es_code 0x%x es_class 0x%x " 6617 "es_key 0x%x es_add_code 0x%x " 6618 "es_qual_code 0x%x", 6619 rqsense->es_code, rqsense->es_class, 6620 rqsense->es_key, rqsense->es_add_code, 6621 rqsense->es_qual_code); 6622 #endif 6623 } 6624 6625 sata_free_error_retrieval_pkt(rs_spkt); 6626 } 6627 6628 /* 6629 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 6630 * the port must be restarted. When the HBA detects thus error, it may try 6631 * to abort a transfer. And if the transfer was aborted, the device is 6632 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 6633 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 6634 * that the device is in a stable status and transfers may be restarted without 6635 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 6636 * then the software will send the COMRESET to do the port reset. 6637 * 6638 * Software should perform the appropriate error recovery actions based on 6639 * whether non-queued commands were being issued or natived command queuing 6640 * commands were being issued. 6641 * 6642 * And software will complete the command that had the error with error mark 6643 * to higher level software. 6644 * 6645 * Fatal errors include the following: 6646 * PxIS.IFS - Interface Fatal Error Status 6647 * PxIS.HBDS - Host Bus Data Error Status 6648 * PxIS.HBFS - Host Bus Fatal Error Status 6649 * PxIS.TFES - Task File Error Status 6650 * 6651 * WARNING!!! ahciport_mutex should be acquired before the function is called. 6652 */ 6653 static void 6654 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 6655 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 6656 { 6657 uint32_t port_cmd_status; 6658 uint32_t slot_status = 0; 6659 uint32_t failed_tags = 0; 6660 int failed_slot; 6661 int reset_flag = 0; 6662 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 6663 sata_cmd_t *sata_cmd = NULL; 6664 sata_pkt_t *spkt = NULL; 6665 #if AHCI_DEBUG 6666 ahci_cmd_header_t *cmd_header; 6667 #endif 6668 6669 AHCIDBG1(AHCIDBG_ENTRY, ahci_ctlp, 6670 "ahci_fatal_error_recovery_handler enter: port %d", port); 6671 6672 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6673 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6674 6675 /* Read PxCI to see which commands are still outstanding */ 6676 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6677 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6678 6679 /* 6680 * Read PxCMD.CCS to determine the slot that the HBA 6681 * was processing when the error occurred. 6682 */ 6683 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6684 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6685 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 6686 AHCI_CMD_STATUS_CCS_SHIFT; 6687 6688 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6689 spkt = ahci_portp->ahciport_err_retri_pkt; 6690 ASSERT(spkt != NULL); 6691 } else { 6692 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 6693 if (spkt == NULL) { 6694 /* May happen when interface errors occur? */ 6695 goto next; 6696 } 6697 } 6698 6699 #if AHCI_DEBUG 6700 /* 6701 * Debugging purpose... 6702 */ 6703 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 6704 cmd_header = 6705 &ahci_portp->ahciport_cmd_list[failed_slot]; 6706 AHCIDBG3(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6707 "ahci_fatal_error_recovery_handler: port %d, " 6708 "PRD Byte Count = 0x%x, " 6709 "ahciport_prd_bytecounts = 0x%x", port, 6710 cmd_header->ahcich_prd_byte_count, 6711 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 6712 } 6713 #endif 6714 6715 sata_cmd = &spkt->satapkt_cmd; 6716 6717 /* Fill out the status and error registers for PxIS.TFES */ 6718 if (intr_status & AHCI_INTR_STATUS_TFES) { 6719 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6720 ahcirf_d2h_register_fis); 6721 6722 /* Copy the error context back to the sata_cmd */ 6723 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 6724 } 6725 6726 /* The failed command must be one of the outstanding commands */ 6727 failed_tags = 0x1 << failed_slot; 6728 ASSERT(failed_tags & slot_status); 6729 6730 /* Update the sata registers, especially PxSERR register */ 6731 ahci_update_sata_registers(ahci_ctlp, port, 6732 &spkt->satapkt_device); 6733 6734 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6735 /* Read PxSACT to see which commands are still outstanding */ 6736 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6737 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6738 } 6739 next: 6740 6741 #if AHCI_DEBUG 6742 /* 6743 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6744 * set, it means a fatal error happened after REQUEST SENSE command 6745 * or READ LOG EXT command is delivered to the HBA during the error 6746 * recovery process. At this time, the only outstanding command is 6747 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6748 */ 6749 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6750 AHCIDBG1(AHCIDBG_ERRS, ahci_ctlp, 6751 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 6752 "command or READ LOG EXT command for error data retrieval " 6753 "failed", port); 6754 ASSERT(slot_status == 0x1); 6755 ASSERT(failed_slot == 0); 6756 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 6757 SCMD_REQUEST_SENSE || 6758 spkt->satapkt_cmd.satacmd_cmd_reg == 6759 SATAC_READ_LOG_EXT); 6760 } 6761 #endif 6762 6763 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6764 ahci_portp->ahciport_mop_in_progress++; 6765 6766 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6767 port, NULL, &reset_flag); 6768 6769 /* 6770 * Won't retrieve error information: 6771 * 1. Port reset was involved to recover 6772 * 2. Device is gone 6773 * 3. IDENTIFY DEVICE command sent to ATAPI device 6774 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 6775 */ 6776 if (reset_flag || 6777 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 6778 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 6779 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 6780 goto out; 6781 6782 /* 6783 * Deliver READ LOG EXT to gather information about the error when 6784 * a COMRESET has not been performed as part of the error recovery 6785 * during NCQ command processing. 6786 */ 6787 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6788 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 6789 ahci_portp, port); 6790 goto out; 6791 } 6792 6793 /* 6794 * Deliver REQUEST SENSE for ATAPI command to gather information about 6795 * the error when a COMRESET has not been performed as part of the 6796 * error recovery. 6797 */ 6798 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 6799 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 6800 out: 6801 AHCIDBG5(AHCIDBG_ERRS, ahci_ctlp, 6802 "ahci_fatal_error_recovery_handler: port %d fatal error " 6803 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 6804 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 6805 port, slot_status, ahci_portp->ahciport_pending_tags, 6806 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 6807 6808 ahci_mop_commands(ahci_ctlp, 6809 ahci_portp, 6810 slot_status, 6811 failed_tags, /* failed tags */ 6812 0, /* timeout tags */ 6813 0, /* aborted tags */ 6814 0); /* reset tags */ 6815 } 6816 6817 /* 6818 * Handle events - fatal error recovery 6819 */ 6820 static void 6821 ahci_events_handler(void *args) 6822 { 6823 ahci_event_arg_t *ahci_event_arg; 6824 ahci_ctl_t *ahci_ctlp; 6825 ahci_port_t *ahci_portp; 6826 uint32_t event; 6827 uint8_t port; 6828 6829 ahci_event_arg = (ahci_event_arg_t *)args; 6830 6831 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 6832 ahci_portp = ahci_event_arg->ahciea_portp; 6833 event = ahci_event_arg->ahciea_event; 6834 port = ahci_portp->ahciport_port_num; 6835 6836 AHCIDBG2(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 6837 "ahci_events_handler enter: port %d intr_status = 0x%x", 6838 port, event); 6839 6840 mutex_enter(&ahci_portp->ahciport_mutex); 6841 6842 /* 6843 * ahci_intr_phyrdy_change() may have rendered it to 6844 * SATA_DTYPE_NONE. 6845 */ 6846 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 6847 AHCIDBG1(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6848 "ahci_events_handler: port %d no device attached, " 6849 "and just return without doing anything", port); 6850 goto out; 6851 } 6852 6853 if (event & (AHCI_INTR_STATUS_IFS | 6854 AHCI_INTR_STATUS_HBDS | 6855 AHCI_INTR_STATUS_HBFS | 6856 AHCI_INTR_STATUS_TFES)) 6857 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 6858 port, event); 6859 6860 out: 6861 mutex_exit(&ahci_portp->ahciport_mutex); 6862 } 6863 6864 /* 6865 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 6866 * detect there are some commands which are timed out. 6867 */ 6868 static void 6869 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6870 uint8_t port, uint32_t tmp_timeout_tags) 6871 { 6872 uint32_t slot_status = 0; 6873 uint32_t finished_tags = 0; 6874 uint32_t timeout_tags = 0; 6875 6876 AHCIDBG1(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 6877 "ahci_timeout_pkts enter: port %d", port); 6878 6879 mutex_enter(&ahci_portp->ahciport_mutex); 6880 6881 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 6882 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6883 /* Read PxCI to see which commands are still outstanding */ 6884 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6885 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6886 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6887 /* Read PxSACT to see which commands are still outstanding */ 6888 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6889 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6890 } 6891 6892 #if AHCI_DEBUG 6893 /* 6894 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 6895 * set, it means a fatal error happened after REQUEST SENSE command 6896 * or READ LOG EXT command is delivered to the HBA during the error 6897 * recovery process. At this time, the only outstanding command is 6898 * supposed to be REQUEST SENSE command or READ LOG EXT command. 6899 */ 6900 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6901 AHCIDBG4(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 6902 "ahci_timeout_pkts called while REQUEST SENSE " 6903 "command or READ LOG EXT command for error recovery " 6904 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 6905 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 6906 tmp_timeout_tags, slot_status, 6907 ahci_portp->ahciport_pending_tags, 6908 ahci_portp->ahciport_pending_ncq_tags); 6909 ASSERT(slot_status == 0x1); 6910 } 6911 #endif 6912 6913 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 6914 ahci_portp->ahciport_mop_in_progress++; 6915 6916 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 6917 port, NULL, NULL); 6918 6919 /* 6920 * Re-identify timeout tags because some previously checked commands 6921 * could already complete. 6922 */ 6923 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6924 finished_tags = ahci_portp->ahciport_pending_tags & 6925 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 6926 timeout_tags = tmp_timeout_tags & ~finished_tags; 6927 6928 AHCIDBG5(AHCIDBG_TIMEOUT, ahci_ctlp, 6929 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6930 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 6931 "pending_tags = 0x%x ", 6932 port, finished_tags, timeout_tags, 6933 slot_status, ahci_portp->ahciport_pending_tags); 6934 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6935 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6936 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 6937 timeout_tags = tmp_timeout_tags & ~finished_tags; 6938 6939 AHCIDBG5(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 6940 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 6941 "timeout_tags = 0x%x, port_sactive = 0x%x, " 6942 "pending_ncq_tags = 0x%x ", 6943 port, finished_tags, timeout_tags, 6944 slot_status, ahci_portp->ahciport_pending_ncq_tags); 6945 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6946 timeout_tags = tmp_timeout_tags; 6947 } 6948 6949 ahci_mop_commands(ahci_ctlp, 6950 ahci_portp, 6951 slot_status, 6952 0, /* failed tags */ 6953 timeout_tags, /* timeout tags */ 6954 0, /* aborted tags */ 6955 0); /* reset tags */ 6956 6957 mutex_exit(&ahci_portp->ahciport_mutex); 6958 } 6959 6960 /* 6961 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 6962 * for long time. 6963 */ 6964 static void 6965 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 6966 { 6967 ahci_port_t *ahci_portp; 6968 sata_pkt_t *spkt; 6969 uint32_t pending_tags; 6970 uint32_t timeout_tags; 6971 uint32_t port_cmd_status; 6972 uint32_t port_sactive; 6973 uint8_t port; 6974 int tmp_slot; 6975 int current_slot; 6976 uint32_t current_tags; 6977 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 6978 /* max number of cycles this packet should survive */ 6979 int max_life_cycles; 6980 6981 /* how many cycles this packet survived so far */ 6982 int watched_cycles; 6983 6984 mutex_enter(&ahci_ctlp->ahcictl_mutex); 6985 6986 AHCIDBG0(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 6987 "ahci_watchdog_handler entered"); 6988 6989 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 6990 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 6991 continue; 6992 } 6993 6994 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6995 6996 mutex_enter(&ahci_portp->ahciport_mutex); 6997 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 6998 mutex_exit(&ahci_portp->ahciport_mutex); 6999 continue; 7000 } 7001 7002 /* Skip the check for those ports in error recovery */ 7003 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 7004 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 7005 mutex_exit(&ahci_portp->ahciport_mutex); 7006 continue; 7007 } 7008 7009 pending_tags = 0; 7010 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7011 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7012 7013 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7014 current_slot = 0; 7015 pending_tags = 0x1; 7016 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7017 current_slot = 7018 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7019 AHCI_CMD_STATUS_CCS_SHIFT; 7020 pending_tags = ahci_portp->ahciport_pending_tags; 7021 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7022 port_sactive = ddi_get32( 7023 ahci_ctlp->ahcictl_ahci_acc_handle, 7024 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7025 current_tags = port_sactive & 7026 ~port_cmd_status & 7027 AHCI_NCQ_SLOT_MASK(ahci_portp); 7028 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 7029 } 7030 7031 timeout_tags = 0; 7032 while (pending_tags) { 7033 tmp_slot = ddi_ffs(pending_tags) - 1; 7034 if (tmp_slot == -1) { 7035 break; 7036 } 7037 7038 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 7039 spkt = ahci_portp->ahciport_err_retri_pkt; 7040 else 7041 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 7042 7043 if ((spkt != NULL) && spkt->satapkt_time && 7044 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 7045 /* 7046 * We are overloading satapkt_hba_driver_private 7047 * with watched_cycle count. 7048 * 7049 * If a packet has survived for more than it's 7050 * max life cycles, it is a candidate for time 7051 * out. 7052 */ 7053 watched_cycles = (int)(intptr_t) 7054 spkt->satapkt_hba_driver_private; 7055 watched_cycles++; 7056 max_life_cycles = (spkt->satapkt_time + 7057 ahci_watchdog_timeout - 1) / 7058 ahci_watchdog_timeout; 7059 7060 spkt->satapkt_hba_driver_private = 7061 (void *)(intptr_t)watched_cycles; 7062 7063 if (watched_cycles <= max_life_cycles) 7064 goto next; 7065 7066 AHCIDBG1(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 7067 ahci_ctlp, "the current slot is %d", 7068 current_slot); 7069 /* 7070 * We need to check whether the HBA has 7071 * begun to execute the command, if not, 7072 * then re-set the timer of the command. 7073 */ 7074 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 7075 (tmp_slot != current_slot) || 7076 NCQ_CMD_IN_PROGRESS(ahci_portp) && 7077 ((0x1 << tmp_slot) & current_tags)) { 7078 spkt->satapkt_hba_driver_private = 7079 (void *)(intptr_t)0; 7080 } else { 7081 timeout_tags |= (0x1 << tmp_slot); 7082 cmn_err(CE_WARN, "!ahci%d: watchdog " 7083 "port %d satapkt 0x%p timed out\n", 7084 instance, port, (void *)spkt); 7085 } 7086 } 7087 next: 7088 CLEAR_BIT(pending_tags, tmp_slot); 7089 } 7090 7091 if (timeout_tags) { 7092 mutex_exit(&ahci_portp->ahciport_mutex); 7093 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7094 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 7095 port, timeout_tags); 7096 mutex_enter(&ahci_ctlp->ahcictl_mutex); 7097 mutex_enter(&ahci_portp->ahciport_mutex); 7098 } 7099 7100 mutex_exit(&ahci_portp->ahciport_mutex); 7101 } 7102 7103 /* Re-install the watchdog timeout handler */ 7104 if (ahci_ctlp->ahcictl_timeout_id != 0) { 7105 ahci_ctlp->ahcictl_timeout_id = 7106 timeout((void (*)(void *))ahci_watchdog_handler, 7107 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 7108 } 7109 7110 mutex_exit(&ahci_ctlp->ahcictl_mutex); 7111 } 7112 7113 /* 7114 * Fill the error context into sata_cmd for non-queued command error. 7115 */ 7116 static void 7117 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7118 { 7119 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 7120 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7121 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7122 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7123 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7124 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7125 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7126 7127 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 7128 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7129 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7130 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7131 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7132 } 7133 } 7134 7135 /* 7136 * Fill the ncq error page into sata_cmd for queued command error. 7137 */ 7138 static void 7139 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 7140 struct sata_ncq_error_recovery_page *ncq_err_page) 7141 { 7142 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 7143 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 7144 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 7145 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 7146 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 7147 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 7148 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 7149 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 7150 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 7151 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 7152 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 7153 } 7154 7155 /* 7156 * Put the respective register value to sata_cmd_t for satacmd_flags. 7157 */ 7158 static void 7159 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 7160 { 7161 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 7162 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 7163 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 7164 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 7165 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 7166 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 7167 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 7168 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 7169 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 7170 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 7171 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 7172 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 7173 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 7174 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 7175 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 7176 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 7177 if (scmd->satacmd_flags.sata_copy_out_device_reg) 7178 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 7179 if (scmd->satacmd_flags.sata_copy_out_error_reg) 7180 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 7181 } 7182 7183 static void 7184 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7185 uint32_t intr_status) 7186 { 7187 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7188 7189 if (intr_status & AHCI_INTR_STATUS_IFS) 7190 cmn_err(CE_WARN, "ahci%d: ahci port %d has interface fatal " 7191 "error", instance, port); 7192 7193 if (intr_status & AHCI_INTR_STATUS_HBDS) 7194 cmn_err(CE_WARN, "ahci%d: ahci port %d has bus data error", 7195 instance, port); 7196 7197 if (intr_status & AHCI_INTR_STATUS_HBFS) 7198 cmn_err(CE_WARN, "ahci%d: ahci port %d has bus fatal error", 7199 instance, port); 7200 7201 if (intr_status & AHCI_INTR_STATUS_TFES) 7202 cmn_err(CE_WARN, "ahci%d: ahci port %d has task file error", 7203 instance, port); 7204 7205 cmn_err(CE_WARN, "ahci%d: ahci port %d is trying to do error " 7206 "recovery", instance, port); 7207 } 7208 7209 /* 7210 * Dump the serror message to the log. 7211 */ 7212 static void 7213 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 7214 uint32_t port_serror, int debug_only) 7215 { 7216 static char err_buf[512]; 7217 static char err_msg_header[16]; 7218 char *err_msg = err_buf; 7219 7220 *err_buf = '\0'; 7221 *err_msg_header = '\0'; 7222 7223 if (port_serror & SERROR_DATA_ERR_FIXED) { 7224 err_msg = strcat(err_msg, 7225 "\tRecovered Data Integrity Error (I)\n"); 7226 } 7227 7228 if (port_serror & SERROR_COMM_ERR_FIXED) { 7229 err_msg = strcat(err_msg, 7230 "\tRecovered Communication Error (M)\n"); 7231 } 7232 7233 if (port_serror & SERROR_DATA_ERR) { 7234 err_msg = strcat(err_msg, 7235 "\tTransient Data Integrity Error (T)\n"); 7236 } 7237 7238 if (port_serror & SERROR_PERSISTENT_ERR) { 7239 err_msg = strcat(err_msg, 7240 "\tPersistent Communication or Data Integrity Error (C)\n"); 7241 } 7242 7243 if (port_serror & SERROR_PROTOCOL_ERR) { 7244 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 7245 } 7246 7247 if (port_serror & SERROR_INT_ERR) { 7248 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 7249 } 7250 7251 if (port_serror & SERROR_PHY_RDY_CHG) { 7252 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 7253 } 7254 7255 if (port_serror & SERROR_PHY_INT_ERR) { 7256 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 7257 } 7258 7259 if (port_serror & SERROR_COMM_WAKE) { 7260 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 7261 } 7262 7263 if (port_serror & SERROR_10B_TO_8B_ERR) { 7264 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 7265 } 7266 7267 if (port_serror & SERROR_DISPARITY_ERR) { 7268 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 7269 } 7270 7271 if (port_serror & SERROR_CRC_ERR) { 7272 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 7273 } 7274 7275 if (port_serror & SERROR_HANDSHAKE_ERR) { 7276 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 7277 } 7278 7279 if (port_serror & SERROR_LINK_SEQ_ERR) { 7280 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 7281 } 7282 7283 if (port_serror & SERROR_TRANS_ERR) { 7284 err_msg = strcat(err_msg, 7285 "\tTransport state transition error (T)\n"); 7286 } 7287 7288 if (port_serror & SERROR_FIS_TYPE) { 7289 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 7290 } 7291 7292 if (port_serror & SERROR_EXCHANGED_ERR) { 7293 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 7294 } 7295 7296 if (err_msg == NULL) 7297 return; 7298 7299 if (debug_only) { 7300 (void) sprintf(err_msg_header, "port %d", port); 7301 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg_header); 7302 AHCIDBG0(AHCIDBG_ERRS, ahci_ctlp, err_msg); 7303 } else if (ahci_ctlp) { 7304 cmn_err(CE_WARN, "ahci%d: %s %s", 7305 ddi_get_instance(ahci_ctlp->ahcictl_dip), 7306 err_msg_header, err_msg); 7307 } else { 7308 cmn_err(CE_WARN, "ahci: %s %s", err_msg_header, err_msg); 7309 } 7310 } 7311 7312 /* 7313 * This routine is to calculate the total number of ports implemented 7314 * by the HBA. 7315 */ 7316 static int 7317 ahci_get_num_implemented_ports(uint32_t ports_implemented) 7318 { 7319 uint8_t i; 7320 int num = 0; 7321 7322 for (i = 0; i < AHCI_MAX_PORTS; i++) { 7323 if (((uint32_t)0x1 << i) & ports_implemented) 7324 num++; 7325 } 7326 7327 return (num); 7328 } 7329 7330 #if AHCI_DEBUG 7331 static void 7332 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 7333 { 7334 static char name[16]; 7335 va_list ap; 7336 7337 mutex_enter(&ahci_log_mutex); 7338 7339 va_start(ap, fmt); 7340 if (ahci_ctlp) { 7341 (void) sprintf(name, "ahci%d: ", 7342 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 7343 } else { 7344 (void) sprintf(name, "ahci: "); 7345 } 7346 7347 (void) vsprintf(ahci_log_buf, fmt, ap); 7348 va_end(ap); 7349 7350 cmn_err(level, "%s%s", name, ahci_log_buf); 7351 7352 mutex_exit(&ahci_log_mutex); 7353 } 7354 #endif 7355 7356 /* 7357 * quiesce(9E) entry point. 7358 * 7359 * This function is called when the system is single-threaded at high 7360 * PIL with preemption disabled. Therefore, this function must not be 7361 * blocked. 7362 * 7363 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 7364 * DDI_FAILURE indicates an error condition and should almost never happen. 7365 */ 7366 static int 7367 ahci_quiesce(dev_info_t *dip) 7368 { 7369 ahci_ctl_t *ahci_ctlp; 7370 ahci_port_t *ahci_portp; 7371 int instance, port; 7372 7373 instance = ddi_get_instance(dip); 7374 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 7375 7376 if (ahci_ctlp == NULL) 7377 return (DDI_FAILURE); 7378 7379 #if AHCI_DEBUG 7380 ahci_debug_flags = 0; 7381 #endif 7382 7383 /* disable all the interrupts. */ 7384 ahci_disable_all_intrs(ahci_ctlp); 7385 7386 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 7387 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 7388 continue; 7389 } 7390 7391 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 7392 7393 /* 7394 * Stop the port by clearing PxCMD.ST 7395 * 7396 * Here we must disable the port interrupt because 7397 * ahci_disable_all_intrs only clear GHC.IE, and IS 7398 * register will be still set if PxIE is enabled. 7399 * When ahci shares one IRQ with other drivers, the 7400 * intr handler may claim the intr mistakenly. 7401 */ 7402 ahci_disable_port_intrs(ahci_ctlp, port); 7403 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7404 ahci_portp, port); 7405 } 7406 7407 return (DDI_SUCCESS); 7408 } 7409