1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * AHCI (Advanced Host Controller Interface) SATA HBA Driver 29 * 30 * Power Management Support 31 * ------------------------ 32 * 33 * At the moment, the ahci driver only implements suspend/resume to 34 * support Suspend to RAM on X86 feature. Device power management isn't 35 * implemented, link power management is disabled, and hot plug isn't 36 * allowed during the period from suspend to resume. 37 * 38 * For s/r support, the ahci driver only need to implement DDI_SUSPEND 39 * and DDI_RESUME entries, and don't need to take care of new requests 40 * sent down after suspend because the target driver (sd) has already 41 * handled these conditions, and blocked these requests. For the detailed 42 * information, please check with sdopen, sdclose and sdioctl routines. 43 * 44 */ 45 46 #include <sys/note.h> 47 #include <sys/scsi/scsi.h> 48 #include <sys/pci.h> 49 #include <sys/disp.h> 50 #include <sys/sata/sata_hba.h> 51 #include <sys/sata/adapters/ahci/ahcireg.h> 52 #include <sys/sata/adapters/ahci/ahcivar.h> 53 54 /* 55 * This is the string displayed by modinfo, etc. 56 * Make sure you keep the version ID up to date! 57 */ 58 static char ahci_ident[] = "ahci driver"; 59 60 /* 61 * Function prototypes for driver entry points 62 */ 63 static int ahci_attach(dev_info_t *, ddi_attach_cmd_t); 64 static int ahci_detach(dev_info_t *, ddi_detach_cmd_t); 65 static int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 66 static int ahci_quiesce(dev_info_t *); 67 68 /* 69 * Function prototypes for SATA Framework interfaces 70 */ 71 static int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t); 72 static int ahci_unregister_sata_hba_tran(ahci_ctl_t *); 73 74 static int ahci_tran_probe_port(dev_info_t *, sata_device_t *); 75 static int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt); 76 static int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int); 77 static int ahci_tran_reset_dport(dev_info_t *, sata_device_t *); 78 static int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *); 79 static int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *); 80 #if defined(__lock_lint) 81 static int ahci_selftest(dev_info_t *, sata_device_t *); 82 #endif 83 84 /* 85 * Local function prototypes 86 */ 87 static int ahci_alloc_ports_state(ahci_ctl_t *); 88 static void ahci_dealloc_ports_state(ahci_ctl_t *); 89 static int ahci_alloc_port_state(ahci_ctl_t *, uint8_t); 90 static void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t); 91 static int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *, uint8_t); 92 static void ahci_dealloc_rcvd_fis(ahci_port_t *); 93 static int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *, uint8_t); 94 static void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *); 95 static int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 96 static void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *); 97 static void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *); 98 static void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *); 99 100 static int ahci_initialize_controller(ahci_ctl_t *); 101 static void ahci_uninitialize_controller(ahci_ctl_t *); 102 static int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 103 static int ahci_config_space_init(ahci_ctl_t *); 104 105 static void ahci_drain_ports_taskq(ahci_ctl_t *); 106 static int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *, 107 uint8_t); 108 static int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *); 109 static int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t); 110 static int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *, 111 sata_device_t *); 112 static int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *, 113 sata_pmult_gscr_t *); 114 static int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 115 sata_device_t *); 116 static int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 117 static int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 118 static int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *, 119 sata_device_t *); 120 121 static void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t); 122 static int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t); 123 static void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *, 124 ahci_addr_t *); 125 static void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *); 126 static int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *, 127 ahci_addr_t *, sata_pkt_t *); 128 static int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *, 129 ahci_addr_t *, sata_pkt_t *); 130 static int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *, 131 ahci_addr_t *, int); 132 static void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *); 133 static void ahci_copy_ncq_err_page(sata_cmd_t *, 134 struct sata_ncq_error_recovery_page *); 135 static void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *); 136 137 static int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 138 static int ahci_hba_reset(ahci_ctl_t *); 139 static int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 140 static int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 141 static void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t); 142 static int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *, 143 ahci_addr_t *); 144 static int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *, 145 ahci_addr_t *); 146 static int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *, 147 ahci_addr_t *); 148 static int ahci_reset_hba_reject_pkts(ahci_ctl_t *); 149 static int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *, 150 uint8_t); 151 static int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *, 152 uint8_t, int, int *); 153 static void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t, 154 uint32_t, uint32_t, uint32_t, uint32_t); 155 static uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t); 156 static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *, 157 uint8_t, sata_pkt_t *); 158 static void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 159 ahci_addr_t *, uint32_t); 160 static void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *, 161 uint8_t, uint32_t); 162 static void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *, 163 uint8_t, uint32_t); 164 static void ahci_events_handler(void *); 165 static void ahci_watchdog_handler(ahci_ctl_t *); 166 167 static uint_t ahci_intr(caddr_t, caddr_t); 168 static void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t); 169 static int ahci_add_intrs(ahci_ctl_t *, int); 170 static void ahci_rem_intrs(ahci_ctl_t *); 171 static void ahci_enable_all_intrs(ahci_ctl_t *); 172 static void ahci_disable_all_intrs(ahci_ctl_t *); 173 static void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t); 174 static void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t); 175 176 static int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t); 177 static int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t); 178 static int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *); 179 static int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t); 180 static int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 181 static int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *, 182 ahci_port_t *, uint8_t); 183 static int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t); 184 static int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *, 185 uint8_t, uint32_t); 186 static int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *, 187 uint8_t, uint32_t); 188 static int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t); 189 190 static void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *); 191 static int ahci_get_num_implemented_ports(uint32_t); 192 static void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t port, 193 uint32_t); 194 static void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int); 195 #if AHCI_DEBUG 196 static void ahci_log(ahci_ctl_t *, uint_t, char *, ...); 197 #endif 198 199 200 /* 201 * DMA attributes for the data buffer 202 * 203 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 204 * does not support 64-bit addressing 205 */ 206 static ddi_dma_attr_t buffer_dma_attr = { 207 DMA_ATTR_V0, /* dma_attr_version */ 208 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 209 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 210 0x3fffffull, /* dma_attr_count_max i.e. for one cookie */ 211 0x2ull, /* dma_attr_align: word aligned */ 212 1, /* dma_attr_burstsizes */ 213 1, /* dma_attr_minxfer */ 214 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 215 0xffffffffull, /* dma_attr_seg */ 216 AHCI_PRDT_NUMBER, /* dma_attr_sgllen */ 217 512, /* dma_attr_granular */ 218 0, /* dma_attr_flags */ 219 }; 220 221 /* 222 * DMA attributes for the rcvd FIS 223 * 224 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 225 * does not support 64-bit addressing 226 */ 227 static ddi_dma_attr_t rcvd_fis_dma_attr = { 228 DMA_ATTR_V0, /* dma_attr_version */ 229 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 230 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 231 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 232 0x100ull, /* dma_attr_align: 256-byte aligned */ 233 1, /* dma_attr_burstsizes */ 234 1, /* dma_attr_minxfer */ 235 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 236 0xffffffffull, /* dma_attr_seg */ 237 1, /* dma_attr_sgllen */ 238 1, /* dma_attr_granular */ 239 0, /* dma_attr_flags */ 240 }; 241 242 /* 243 * DMA attributes for the command list 244 * 245 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 246 * does not support 64-bit addressing 247 */ 248 static ddi_dma_attr_t cmd_list_dma_attr = { 249 DMA_ATTR_V0, /* dma_attr_version */ 250 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 251 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 252 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 253 0x400ull, /* dma_attr_align: 1K-byte aligned */ 254 1, /* dma_attr_burstsizes */ 255 1, /* dma_attr_minxfer */ 256 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 257 0xffffffffull, /* dma_attr_seg */ 258 1, /* dma_attr_sgllen */ 259 1, /* dma_attr_granular */ 260 0, /* dma_attr_flags */ 261 }; 262 263 /* 264 * DMA attributes for cmd tables 265 * 266 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA 267 * does not support 64-bit addressing 268 */ 269 static ddi_dma_attr_t cmd_table_dma_attr = { 270 DMA_ATTR_V0, /* dma_attr_version */ 271 0x0ull, /* dma_attr_addr_lo: lowest bus address */ 272 0xffffffffffffffffull, /* dma_attr_addr_hi: highest bus address */ 273 0xffffffffull, /* dma_attr_count_max i.e. for one cookie */ 274 0x80ull, /* dma_attr_align: 128-byte aligned */ 275 1, /* dma_attr_burstsizes */ 276 1, /* dma_attr_minxfer */ 277 0xffffffffull, /* dma_attr_maxxfer i.e. includes all cookies */ 278 0xffffffffull, /* dma_attr_seg */ 279 1, /* dma_attr_sgllen */ 280 1, /* dma_attr_granular */ 281 0, /* dma_attr_flags */ 282 }; 283 284 285 /* Device access attributes */ 286 static ddi_device_acc_attr_t accattr = { 287 DDI_DEVICE_ATTR_V0, 288 DDI_STRUCTURE_LE_ACC, 289 DDI_STRICTORDER_ACC 290 }; 291 292 293 static struct dev_ops ahcictl_dev_ops = { 294 DEVO_REV, /* devo_rev */ 295 0, /* refcnt */ 296 ahci_getinfo, /* info */ 297 nulldev, /* identify */ 298 nulldev, /* probe */ 299 ahci_attach, /* attach */ 300 ahci_detach, /* detach */ 301 nodev, /* no reset */ 302 (struct cb_ops *)0, /* driver operations */ 303 NULL, /* bus operations */ 304 NULL, /* power */ 305 ahci_quiesce, /* quiesce */ 306 }; 307 308 static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = { 309 SATA_TRAN_HOTPLUG_OPS_REV_1, 310 ahci_tran_hotplug_port_activate, 311 ahci_tran_hotplug_port_deactivate 312 }; 313 314 extern struct mod_ops mod_driverops; 315 316 static struct modldrv modldrv = { 317 &mod_driverops, /* driverops */ 318 ahci_ident, /* short description */ 319 &ahcictl_dev_ops, /* driver ops */ 320 }; 321 322 static struct modlinkage modlinkage = { 323 MODREV_1, 324 &modldrv, 325 NULL 326 }; 327 328 /* The following variables are watchdog handler related */ 329 static int ahci_watchdog_timeout = 5; /* 5 seconds */ 330 static int ahci_watchdog_tick; 331 332 /* 333 * This static variable indicates the size of command table, 334 * and it's changeable with prdt number, which ahci_dma_prdt_number 335 * indicates. 336 */ 337 static size_t ahci_cmd_table_size; 338 339 /* 340 * The below global variables are tunable via /etc/system 341 * 342 * ahci_dma_prdt_number 343 * ahci_msi_enabled 344 * ahci_buf_64bit_dma 345 * ahci_commu_64bit_dma 346 */ 347 348 /* The number of Physical Region Descriptor Table(PRDT) in Command Table */ 349 int ahci_dma_prdt_number = AHCI_PRDT_NUMBER; 350 351 /* AHCI MSI is tunable */ 352 boolean_t ahci_msi_enabled = B_TRUE; 353 354 /* 355 * 64-bit dma addressing for data buffer is tunable 356 * 357 * The variable controls only the below value: 358 * DBAU (upper 32-bits physical address of data block) 359 */ 360 boolean_t ahci_buf_64bit_dma = B_TRUE; 361 362 /* 363 * 64-bit dma addressing for communication system descriptors is tunable 364 * 365 * The variable controls the below three values: 366 * 367 * PxCLBU (upper 32-bits for the command list base physical address) 368 * PxFBU (upper 32-bits for the received FIS base physical address) 369 * CTBAU (upper 32-bits of command table base) 370 */ 371 boolean_t ahci_commu_64bit_dma = B_TRUE; 372 373 /* 374 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600 375 * chipset. If the users want to have a try with 64-bit dma, please change 376 * the below variable value to enable it. 377 */ 378 boolean_t sb600_buf_64bit_dma_disable = B_TRUE; 379 380 /* 381 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI 382 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma, 383 * please change the below value to enable it. 384 */ 385 boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE; 386 387 388 /* 389 * End of global tunable variable definition 390 */ 391 392 #if AHCI_DEBUG 393 uint32_t ahci_debug_flags = 0; 394 #else 395 uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT); 396 #endif 397 398 399 #if AHCI_DEBUG 400 /* The following is needed for ahci_log() */ 401 static kmutex_t ahci_log_mutex; 402 static char ahci_log_buf[512]; 403 #endif 404 405 /* Opaque state pointer initialized by ddi_soft_state_init() */ 406 static void *ahci_statep = NULL; 407 408 /* 409 * ahci module initialization. 410 */ 411 int 412 _init(void) 413 { 414 int ret; 415 416 ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0); 417 if (ret != 0) { 418 goto err_out; 419 } 420 421 #if AHCI_DEBUG 422 mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL); 423 #endif 424 425 if ((ret = sata_hba_init(&modlinkage)) != 0) { 426 #if AHCI_DEBUG 427 mutex_destroy(&ahci_log_mutex); 428 #endif 429 ddi_soft_state_fini(&ahci_statep); 430 goto err_out; 431 } 432 433 ret = mod_install(&modlinkage); 434 if (ret != 0) { 435 sata_hba_fini(&modlinkage); 436 #if AHCI_DEBUG 437 mutex_destroy(&ahci_log_mutex); 438 #endif 439 ddi_soft_state_fini(&ahci_statep); 440 goto err_out; 441 } 442 443 /* watchdog tick */ 444 ahci_watchdog_tick = drv_usectohz( 445 (clock_t)ahci_watchdog_timeout * 1000000); 446 return (ret); 447 448 err_out: 449 cmn_err(CE_WARN, "!ahci: Module init failed"); 450 return (ret); 451 } 452 453 /* 454 * ahci module uninitialize. 455 */ 456 int 457 _fini(void) 458 { 459 int ret; 460 461 ret = mod_remove(&modlinkage); 462 if (ret != 0) { 463 return (ret); 464 } 465 466 /* Remove the resources allocated in _init(). */ 467 sata_hba_fini(&modlinkage); 468 #if AHCI_DEBUG 469 mutex_destroy(&ahci_log_mutex); 470 #endif 471 ddi_soft_state_fini(&ahci_statep); 472 473 return (ret); 474 } 475 476 /* 477 * _info entry point 478 */ 479 int 480 _info(struct modinfo *modinfop) 481 { 482 return (mod_info(&modlinkage, modinfop)); 483 } 484 485 /* 486 * The attach entry point for dev_ops. 487 */ 488 static int 489 ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 490 { 491 ahci_ctl_t *ahci_ctlp = NULL; 492 int instance = ddi_get_instance(dip); 493 int status; 494 int attach_state; 495 uint32_t cap_status, ahci_version; 496 int intr_types; 497 int i; 498 pci_regspec_t *regs; 499 int regs_length; 500 int rnumber; 501 #if AHCI_DEBUG 502 int speed; 503 #endif 504 505 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter", 506 NULL); 507 508 switch (cmd) { 509 case DDI_ATTACH: 510 break; 511 512 case DDI_RESUME: 513 514 /* 515 * During DDI_RESUME, the hardware state of the device 516 * (power may have been removed from the device) must be 517 * restored, allow pending requests to continue, and 518 * service new requests. 519 */ 520 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 521 mutex_enter(&ahci_ctlp->ahcictl_mutex); 522 523 /* Restart watch thread */ 524 if (ahci_ctlp->ahcictl_timeout_id == 0) 525 ahci_ctlp->ahcictl_timeout_id = timeout( 526 (void (*)(void *))ahci_watchdog_handler, 527 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 528 529 mutex_exit(&ahci_ctlp->ahcictl_mutex); 530 531 /* 532 * Re-initialize the controller and enable the interrupts and 533 * restart all the ports. 534 * 535 * Note that so far we don't support hot-plug during 536 * suspend/resume. 537 */ 538 if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) { 539 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp, 540 "Failed to initialize the controller " 541 "during DDI_RESUME", NULL); 542 return (DDI_FAILURE); 543 } 544 545 mutex_enter(&ahci_ctlp->ahcictl_mutex); 546 ahci_ctlp->ahcictl_flags &= ~ AHCI_SUSPEND; 547 mutex_exit(&ahci_ctlp->ahcictl_mutex); 548 549 return (DDI_SUCCESS); 550 551 default: 552 return (DDI_FAILURE); 553 } 554 555 attach_state = AHCI_ATTACH_STATE_NONE; 556 557 /* Allocate soft state */ 558 status = ddi_soft_state_zalloc(ahci_statep, instance); 559 if (status != DDI_SUCCESS) { 560 cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state", 561 instance); 562 goto err_out; 563 } 564 565 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 566 ahci_ctlp->ahcictl_flags |= AHCI_ATTACH; 567 ahci_ctlp->ahcictl_dip = dip; 568 569 /* Initialize the cport/port mapping */ 570 for (i = 0; i < AHCI_MAX_PORTS; i++) { 571 ahci_ctlp->ahcictl_port_to_cport[i] = 0xff; 572 ahci_ctlp->ahcictl_cport_to_port[i] = 0xff; 573 } 574 575 attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC; 576 577 /* 578 * Now map the AHCI base address; which includes global 579 * registers and port control registers 580 * 581 * According to the spec, the AHCI Base Address is BAR5, 582 * but BAR0-BAR4 are optional, so we need to check which 583 * rnumber is used for BAR5. 584 */ 585 586 /* 587 * search through DDI "reg" property for the AHCI register set 588 */ 589 if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip, 590 DDI_PROP_DONTPASS, "reg", (int **)®s, 591 (uint_t *)®s_length) != DDI_PROP_SUCCESS) { 592 cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property", 593 instance); 594 goto err_out; 595 } 596 597 /* AHCI Base Address is located at 0x24 offset */ 598 for (rnumber = 0; rnumber < regs_length; ++rnumber) { 599 if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M) 600 == AHCI_PCI_RNUM) 601 break; 602 } 603 604 ddi_prop_free(regs); 605 606 if (rnumber == regs_length) { 607 cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set", 608 instance); 609 goto err_out; 610 } 611 612 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber); 613 614 status = ddi_regs_map_setup(dip, 615 rnumber, 616 (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr, 617 0, 618 0, 619 &accattr, 620 &ahci_ctlp->ahcictl_ahci_acc_handle); 621 if (status != DDI_SUCCESS) { 622 cmn_err(CE_WARN, "!ahci%d: Cannot map register space", 623 instance); 624 goto err_out; 625 } 626 627 attach_state |= AHCI_ATTACH_STATE_REG_MAP; 628 629 /* Get the AHCI version information */ 630 ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 631 (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp)); 632 633 cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance, 634 (ahci_version & 0xffff0000) >> 16, 635 ((ahci_version & 0x0000ff00) >> 4 | 636 (ahci_version & 0x000000ff))); 637 638 /* We don't support controllers whose versions are lower than 1.0 */ 639 if (!(ahci_version & 0xffff0000)) { 640 cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower " 641 "than version 1.0", instance); 642 goto err_out; 643 } 644 645 /* Get the HBA capabilities information */ 646 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 647 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 648 649 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilites = 0x%x", 650 cap_status); 651 652 #if AHCI_DEBUG 653 /* Get the interface speed supported by the HBA */ 654 speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT; 655 if (speed == 0x01) { 656 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 657 "hba interface speed support: Gen 1 (1.5Gbps)", NULL); 658 } else if (speed == 0x10) { 659 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 660 "hba interface speed support: Gen 2 (3 Gbps)", NULL); 661 } else if (speed == 0x11) { 662 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 663 "hba interface speed support: Gen 3 (6 Gbps)", NULL); 664 } 665 #endif 666 667 /* Get the number of command slots supported by the HBA */ 668 ahci_ctlp->ahcictl_num_cmd_slots = 669 ((cap_status & AHCI_HBA_CAP_NCS) >> 670 AHCI_HBA_CAP_NCS_SHIFT) + 1; 671 672 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d", 673 ahci_ctlp->ahcictl_num_cmd_slots); 674 675 /* Get the bit map which indicates ports implemented by the HBA */ 676 ahci_ctlp->ahcictl_ports_implemented = 677 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 678 (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp)); 679 680 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x", 681 ahci_ctlp->ahcictl_ports_implemented); 682 683 /* 684 * According to the AHCI spec, CAP.NP should indicate the maximum 685 * number of ports supported by the HBA silicon, but we found 686 * this value of ICH8 chipset only indicates the number of ports 687 * implemented (exposed) by it. Therefore, the driver should calculate 688 * the potential maximum value by checking PI register, and use 689 * the maximum of this value and CAP.NP. 690 */ 691 ahci_ctlp->ahcictl_num_ports = max( 692 (cap_status & AHCI_HBA_CAP_NP) + 1, 693 ddi_fls(ahci_ctlp->ahcictl_ports_implemented)); 694 695 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d", 696 ahci_ctlp->ahcictl_num_ports); 697 698 /* Get the number of implemented ports by the HBA */ 699 ahci_ctlp->ahcictl_num_implemented_ports = 700 ahci_get_num_implemented_ports( 701 ahci_ctlp->ahcictl_ports_implemented); 702 703 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 704 "hba number of implemented ports: %d", 705 ahci_ctlp->ahcictl_num_implemented_ports); 706 707 /* Check whether HBA supports 64bit DMA addressing */ 708 if (!(cap_status & AHCI_HBA_CAP_S64A)) { 709 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 710 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 711 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 712 "hba does not support 64-bit addressing", NULL); 713 } 714 715 /* Checking for the support of Port Multiplier */ 716 if (cap_status & AHCI_HBA_CAP_SPM) { 717 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS; 718 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 719 "hba supports port multiplier (CBSS)", NULL); 720 721 /* Support FIS-based switching ? */ 722 if (cap_status & AHCI_HBA_CAP_FBSS) { 723 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS; 724 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 725 "hba supports FIS-based switching (FBSS)", NULL); 726 } 727 } 728 729 /* Checking for Support Command List Override */ 730 if (cap_status & AHCI_HBA_CAP_SCLO) { 731 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO; 732 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 733 "hba supports command list override.", NULL); 734 } 735 736 /* Checking for Asynchronous Notification */ 737 if (cap_status & AHCI_HBA_CAP_SSNTF) { 738 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF; 739 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 740 "hba supports asynchronous notification.", NULL); 741 } 742 743 if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle) 744 != DDI_SUCCESS) { 745 cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space", 746 instance); 747 goto err_out; 748 } 749 750 attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP; 751 752 /* 753 * Check the pci configuration space, and set caps. We also 754 * handle the hardware defect in this function. 755 * 756 * For example, force ATI SB600 to use 32-bit dma addressing 757 * since it doesn't support 64-bit dma though its CAP register 758 * declares it support. 759 */ 760 if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) { 761 cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed", 762 instance); 763 goto err_out; 764 } 765 766 /* 767 * Disable the whole controller interrupts before adding 768 * interrupt handlers(s). 769 */ 770 ahci_disable_all_intrs(ahci_ctlp); 771 772 /* Get supported interrupt types */ 773 if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) { 774 cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed", 775 instance); 776 goto err_out; 777 } 778 779 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 780 "ddi_intr_get_supported_types() returned: 0x%x", 781 intr_types); 782 783 if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) { 784 /* 785 * Try MSI first, but fall back to FIXED if failed 786 */ 787 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) == 788 DDI_SUCCESS) { 789 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI; 790 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 791 "Using MSI interrupt type", NULL); 792 goto intr_done; 793 } 794 795 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 796 "MSI registration failed, " 797 "trying FIXED interrupts", NULL); 798 } 799 800 if (intr_types & DDI_INTR_TYPE_FIXED) { 801 if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) == 802 DDI_SUCCESS) { 803 ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED; 804 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 805 "Using FIXED interrupt type", NULL); 806 goto intr_done; 807 } 808 809 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 810 "FIXED interrupt registration failed", NULL); 811 } 812 813 cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance); 814 815 goto err_out; 816 817 intr_done: 818 819 attach_state |= AHCI_ATTACH_STATE_INTR_ADDED; 820 821 /* Initialize the controller mutex */ 822 mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER, 823 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 824 825 attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT; 826 827 if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) { 828 ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER; 829 } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) { 830 ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER; 831 } 832 833 ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) + 834 (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) * 835 sizeof (ahci_prdt_item_t)); 836 837 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 838 "ahci_attach: ahci_dma_prdt_number set by user is 0x%x," 839 " ahci_cmd_table_size is 0x%x", 840 ahci_dma_prdt_number, ahci_cmd_table_size); 841 842 if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER) 843 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen = 844 ahci_dma_prdt_number; 845 846 ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr; 847 ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr; 848 ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr; 849 ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr; 850 851 /* 852 * enable 64bit dma for data buffer for SB600 if 853 * sb600_buf_64bit_dma_disable is B_FALSE 854 */ 855 if ((ahci_buf_64bit_dma == B_FALSE) || 856 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) && 857 !(sb600_buf_64bit_dma_disable == B_FALSE && 858 ahci_ctlp->ahcictl_venid == 0x1002 && 859 ahci_ctlp->ahcictl_devid == 0x4380))) { 860 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi = 861 0xffffffffull; 862 } 863 864 /* 865 * enable 64bit dma for command buffer for SB600/700/710/800 866 * if sbxxx_commu_64bit_dma_disable is B_FALSE 867 */ 868 if ((ahci_commu_64bit_dma == B_FALSE) || 869 ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) && 870 !(sbxxx_commu_64bit_dma_disable == B_FALSE && 871 ahci_ctlp->ahcictl_venid == 0x1002 && 872 (ahci_ctlp->ahcictl_devid == 0x4380 || 873 ahci_ctlp->ahcictl_devid == 0x4391)))) { 874 ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi = 875 0xffffffffull; 876 ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi = 877 0xffffffffull; 878 ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi = 879 0xffffffffull; 880 } 881 882 /* Allocate the ports structure */ 883 status = ahci_alloc_ports_state(ahci_ctlp); 884 if (status != AHCI_SUCCESS) { 885 cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure", 886 instance); 887 goto err_out; 888 } 889 890 attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC; 891 892 /* 893 * Initialize the controller and ports. 894 */ 895 status = ahci_initialize_controller(ahci_ctlp); 896 if (status != AHCI_SUCCESS) { 897 cmn_err(CE_WARN, "!ahci%d: HBA initialization failed", 898 instance); 899 goto err_out; 900 } 901 902 attach_state |= AHCI_ATTACH_STATE_HW_INIT; 903 904 /* Start one thread to check packet timeouts */ 905 ahci_ctlp->ahcictl_timeout_id = timeout( 906 (void (*)(void *))ahci_watchdog_handler, 907 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 908 909 attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED; 910 911 if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) { 912 cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed", 913 instance); 914 goto err_out; 915 } 916 917 ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH; 918 919 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL); 920 921 return (DDI_SUCCESS); 922 923 err_out: 924 if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) { 925 mutex_enter(&ahci_ctlp->ahcictl_mutex); 926 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 927 ahci_ctlp->ahcictl_timeout_id = 0; 928 mutex_exit(&ahci_ctlp->ahcictl_mutex); 929 } 930 931 if (attach_state & AHCI_ATTACH_STATE_HW_INIT) { 932 ahci_uninitialize_controller(ahci_ctlp); 933 } 934 935 if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) { 936 ahci_dealloc_ports_state(ahci_ctlp); 937 } 938 939 if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) { 940 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 941 } 942 943 if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) { 944 ahci_rem_intrs(ahci_ctlp); 945 } 946 947 if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) { 948 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 949 } 950 951 if (attach_state & AHCI_ATTACH_STATE_REG_MAP) { 952 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 953 } 954 955 if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) { 956 ddi_soft_state_free(ahci_statep, instance); 957 } 958 959 return (DDI_FAILURE); 960 } 961 962 /* 963 * The detach entry point for dev_ops. 964 */ 965 static int 966 ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 967 { 968 ahci_ctl_t *ahci_ctlp; 969 int instance; 970 int ret; 971 972 instance = ddi_get_instance(dip); 973 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 974 975 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL); 976 977 switch (cmd) { 978 case DDI_DETACH: 979 980 /* disable the interrupts for an uninterrupted detach */ 981 mutex_enter(&ahci_ctlp->ahcictl_mutex); 982 ahci_disable_all_intrs(ahci_ctlp); 983 mutex_exit(&ahci_ctlp->ahcictl_mutex); 984 985 /* unregister from the sata framework. */ 986 ret = ahci_unregister_sata_hba_tran(ahci_ctlp); 987 if (ret != AHCI_SUCCESS) { 988 mutex_enter(&ahci_ctlp->ahcictl_mutex); 989 ahci_enable_all_intrs(ahci_ctlp); 990 mutex_exit(&ahci_ctlp->ahcictl_mutex); 991 return (DDI_FAILURE); 992 } 993 994 mutex_enter(&ahci_ctlp->ahcictl_mutex); 995 996 /* stop the watchdog handler */ 997 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 998 ahci_ctlp->ahcictl_timeout_id = 0; 999 1000 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1001 1002 /* uninitialize the controller */ 1003 ahci_uninitialize_controller(ahci_ctlp); 1004 1005 /* remove the interrupts */ 1006 ahci_rem_intrs(ahci_ctlp); 1007 1008 /* deallocate the ports structures */ 1009 ahci_dealloc_ports_state(ahci_ctlp); 1010 1011 /* destroy mutex */ 1012 mutex_destroy(&ahci_ctlp->ahcictl_mutex); 1013 1014 /* teardown the pci config */ 1015 pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle); 1016 1017 /* remove the reg maps. */ 1018 ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle); 1019 1020 /* free the soft state. */ 1021 ddi_soft_state_free(ahci_statep, instance); 1022 1023 return (DDI_SUCCESS); 1024 1025 case DDI_SUSPEND: 1026 1027 /* 1028 * The steps associated with suspension must include putting 1029 * the underlying device into a quiescent state so that it 1030 * will not generate interrupts or modify or access memory. 1031 */ 1032 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1033 if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) { 1034 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1035 return (DDI_SUCCESS); 1036 } 1037 1038 ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND; 1039 1040 /* stop the watchdog handler */ 1041 if (ahci_ctlp->ahcictl_timeout_id) { 1042 (void) untimeout(ahci_ctlp->ahcictl_timeout_id); 1043 ahci_ctlp->ahcictl_timeout_id = 0; 1044 } 1045 1046 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1047 1048 /* 1049 * drain the taskq 1050 */ 1051 ahci_drain_ports_taskq(ahci_ctlp); 1052 1053 /* 1054 * Disable the interrupts and stop all the ports. 1055 */ 1056 ahci_uninitialize_controller(ahci_ctlp); 1057 1058 return (DDI_SUCCESS); 1059 1060 default: 1061 return (DDI_FAILURE); 1062 } 1063 } 1064 1065 /* 1066 * The info entry point for dev_ops. 1067 * 1068 */ 1069 static int 1070 ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, 1071 void *arg, void **result) 1072 { 1073 #ifndef __lock_lint 1074 _NOTE(ARGUNUSED(dip)) 1075 #endif /* __lock_lint */ 1076 1077 ahci_ctl_t *ahci_ctlp; 1078 int instance; 1079 dev_t dev; 1080 1081 dev = (dev_t)arg; 1082 instance = getminor(dev); 1083 1084 switch (infocmd) { 1085 case DDI_INFO_DEVT2DEVINFO: 1086 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 1087 if (ahci_ctlp != NULL) { 1088 *result = ahci_ctlp->ahcictl_dip; 1089 return (DDI_SUCCESS); 1090 } else { 1091 *result = NULL; 1092 return (DDI_FAILURE); 1093 } 1094 case DDI_INFO_DEVT2INSTANCE: 1095 *(int *)result = instance; 1096 break; 1097 default: 1098 break; 1099 } 1100 1101 return (DDI_SUCCESS); 1102 } 1103 1104 /* 1105 * Registers the ahci with sata framework. 1106 */ 1107 static int 1108 ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status) 1109 { 1110 struct sata_hba_tran *sata_hba_tran; 1111 1112 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 1113 "ahci_register_sata_hba_tran enter", NULL); 1114 1115 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1116 1117 /* Allocate memory for the sata_hba_tran */ 1118 sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP); 1119 1120 sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV; 1121 sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip; 1122 sata_hba_tran->sata_tran_hba_dma_attr = 1123 &ahci_ctlp->ahcictl_buffer_dma_attr; 1124 1125 /* Report the number of implemented ports */ 1126 sata_hba_tran->sata_tran_hba_num_cports = 1127 ahci_ctlp->ahcictl_num_implemented_ports; 1128 1129 /* Support ATAPI device */ 1130 sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI; 1131 1132 /* Get the data transfer capability for PIO command by the HBA */ 1133 if (cap_status & AHCI_HBA_CAP_PMD) { 1134 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ; 1135 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple " 1136 "DRQ block data transfer for PIO command protocol", NULL); 1137 } 1138 1139 /* 1140 * According to the AHCI spec, the ATA/ATAPI-7 queued feature set 1141 * is not supported by AHCI (including the READ QUEUED (EXT), WRITE 1142 * QUEUED (EXT), and SERVICE commands). Queued operations are 1143 * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA 1144 * QUEUED commands when the HBA and device support native command 1145 * queuing(NCQ). 1146 * 1147 * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the 1148 * CAP register of the HBA indicates NCQ is supported. 1149 * 1150 * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is 1151 * set because the previous register content of PxCI can be re-written 1152 * in the register write. 1153 */ 1154 if ((cap_status & AHCI_HBA_CAP_SNCQ) && 1155 !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) { 1156 sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ; 1157 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ; 1158 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native " 1159 "Command Queuing", NULL); 1160 } 1161 1162 /* Support port multiplier? */ 1163 if (cap_status & AHCI_HBA_CAP_SPM) { 1164 sata_hba_tran->sata_tran_hba_features_support |= 1165 SATA_CTLF_PORT_MULTIPLIER; 1166 1167 /* Support FIS-based switching for port multiplier? */ 1168 if (cap_status & AHCI_HBA_CAP_FBSS) { 1169 sata_hba_tran->sata_tran_hba_features_support |= 1170 SATA_CTLF_PMULT_FBS; 1171 } 1172 } 1173 1174 /* Report the number of command slots */ 1175 sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots; 1176 1177 sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port; 1178 sata_hba_tran->sata_tran_start = ahci_tran_start; 1179 sata_hba_tran->sata_tran_abort = ahci_tran_abort; 1180 sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport; 1181 sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops; 1182 #ifdef __lock_lint 1183 sata_hba_tran->sata_tran_selftest = ahci_selftest; 1184 #endif 1185 /* 1186 * When SATA framework adds support for pwrmgt the 1187 * pwrmgt_ops needs to be updated 1188 */ 1189 sata_hba_tran->sata_tran_pwrmgt_ops = NULL; 1190 sata_hba_tran->sata_tran_ioctl = NULL; 1191 1192 ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran; 1193 1194 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1195 1196 /* Attach it to SATA framework */ 1197 if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH) 1198 != DDI_SUCCESS) { 1199 kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t)); 1200 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1201 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1202 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1203 return (AHCI_FAILURE); 1204 } 1205 1206 return (AHCI_SUCCESS); 1207 } 1208 1209 /* 1210 * Unregisters the ahci with sata framework. 1211 */ 1212 static int 1213 ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp) 1214 { 1215 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1216 "ahci_unregister_sata_hba_tran enter", NULL); 1217 1218 /* Detach from the SATA framework. */ 1219 if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) != 1220 DDI_SUCCESS) { 1221 return (AHCI_FAILURE); 1222 } 1223 1224 /* Deallocate sata_hba_tran. */ 1225 kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran, 1226 sizeof (sata_hba_tran_t)); 1227 1228 mutex_enter(&ahci_ctlp->ahcictl_mutex); 1229 ahci_ctlp->ahcictl_sata_hba_tran = NULL; 1230 mutex_exit(&ahci_ctlp->ahcictl_mutex); 1231 1232 return (AHCI_SUCCESS); 1233 } 1234 1235 #define SET_PORTSTR(str, addrp) \ 1236 if (AHCI_ADDR_IS_PORT(addrp)) \ 1237 (void) sprintf((str), "%d", (addrp)->aa_port); \ 1238 else if (AHCI_ADDR_IS_PMULT(addrp)) \ 1239 (void) sprintf((str), "%d (pmult)", (addrp)->aa_port); \ 1240 else \ 1241 (void) sprintf((str), "%d:%d", (addrp)->aa_port, \ 1242 (addrp)->aa_pmport); 1243 1244 /* 1245 * ahci_tran_probe_port is called by SATA framework. It returns port state, 1246 * port status registers and an attached device type via sata_device 1247 * structure. 1248 * 1249 * We return the cached information from a previous hardware probe. The 1250 * actual hardware probing itself was done either from within 1251 * ahci_initialize_controller() during the driver attach or from a phy 1252 * ready change interrupt handler. 1253 */ 1254 static int 1255 ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd) 1256 { 1257 ahci_ctl_t *ahci_ctlp; 1258 ahci_port_t *ahci_portp; 1259 ahci_addr_t addr, pmult_addr; 1260 uint8_t cport = sd->satadev_addr.cport; 1261 char portstr[10]; 1262 uint8_t device_type; 1263 uint32_t port_state; 1264 uint8_t port; 1265 int rval = SATA_SUCCESS, rval_init; 1266 1267 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1268 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1269 1270 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1271 1272 mutex_enter(&ahci_portp->ahciport_mutex); 1273 1274 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 1275 ASSERT(AHCI_ADDR_IS_VALID(&addr)); 1276 SET_PORTSTR(portstr, &addr); 1277 1278 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1279 "ahci_tran_probe_port enter: port %s", portstr); 1280 1281 if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) && 1282 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1283 ahci_portp->ahciport_pmult_info == NULL)) { 1284 /* port mutliplier is removed. */ 1285 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1286 "ahci_tran_probe_port: " 1287 "pmult is removed from port %s", portstr); 1288 mutex_exit(&ahci_portp->ahciport_mutex); 1289 return (SATA_FAILURE); 1290 } 1291 1292 /* 1293 * The sata_device may refer to 1294 * 1. A controller port. 1295 * A controller port should be ready here. 1296 * 2. A port multiplier. 1297 * SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize 1298 * it and register the port multiplier to the framework. 1299 * SATA_ADDR_PMULT - check the status of all its device ports. 1300 * 3. A port multiplier port. 1301 * If it has not been initialized, initialized it. 1302 * 1303 * A port multiplier or a port multiplier port may require some 1304 * initialization because we cannot do these time-consuming jobs in an 1305 * interrupt context. 1306 */ 1307 if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) { 1308 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 1309 /* Initialize registers on a port multiplier */ 1310 rval_init = ahci_initialize_pmult(ahci_ctlp, 1311 ahci_portp, &pmult_addr, sd); 1312 if (rval_init != AHCI_SUCCESS) { 1313 AHCIDBG(AHCIDBG_PMULT, ahci_ctlp, 1314 "ahci_tran_probe_port: " 1315 "pmult initialization failed.", NULL); 1316 mutex_exit(&ahci_portp->ahciport_mutex); 1317 return (SATA_FAILURE); 1318 } 1319 } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) { 1320 /* Check pmports hotplug events */ 1321 (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr); 1322 } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT | 1323 SATA_ADDR_DPMPORT)) { 1324 if (ahci_probe_pmport(ahci_ctlp, ahci_portp, 1325 &addr, sd) != AHCI_SUCCESS) { 1326 rval = SATA_FAILURE; 1327 goto out; 1328 } 1329 } 1330 1331 /* Update port state and device type */ 1332 port_state = AHCIPORT_GET_STATE(ahci_portp, &addr); 1333 1334 switch (port_state) { 1335 1336 case SATA_PSTATE_FAILED: 1337 sd->satadev_state = SATA_PSTATE_FAILED; 1338 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1339 "ahci_tran_probe_port: port %s PORT FAILED", portstr); 1340 goto out; 1341 1342 case SATA_PSTATE_SHUTDOWN: 1343 sd->satadev_state = SATA_PSTATE_SHUTDOWN; 1344 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1345 "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr); 1346 goto out; 1347 1348 case SATA_PSTATE_PWROFF: 1349 sd->satadev_state = SATA_PSTATE_PWROFF; 1350 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1351 "ahci_tran_probe_port: port %s PORT PWROFF", portstr); 1352 goto out; 1353 1354 case SATA_PSTATE_PWRON: 1355 sd->satadev_state = SATA_PSTATE_PWRON; 1356 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1357 "ahci_tran_probe_port: port %s PORT PWRON", portstr); 1358 break; 1359 1360 default: 1361 sd->satadev_state = port_state; 1362 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1363 "ahci_tran_probe_port: port %s PORT NORMAL %x", 1364 portstr, port_state); 1365 break; 1366 } 1367 1368 device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1369 1370 switch (device_type) { 1371 1372 case SATA_DTYPE_ATADISK: 1373 sd->satadev_type = SATA_DTYPE_ATADISK; 1374 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1375 "ahci_tran_probe_port: port %s DISK found", portstr); 1376 break; 1377 1378 case SATA_DTYPE_ATAPI: 1379 /* 1380 * HBA driver only knows it's an ATAPI device, and don't know 1381 * it's CD/DVD, tape or ATAPI disk because the ATAPI device 1382 * type need to be determined by checking IDENTIFY PACKET 1383 * DEVICE data 1384 */ 1385 sd->satadev_type = SATA_DTYPE_ATAPI; 1386 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1387 "ahci_tran_probe_port: port %s ATAPI found", portstr); 1388 break; 1389 1390 case SATA_DTYPE_PMULT: 1391 ASSERT(AHCI_ADDR_IS_PORT(&addr)||AHCI_ADDR_IS_PMULT(&addr)); 1392 sd->satadev_type = SATA_DTYPE_PMULT; 1393 1394 /* Update the number of pmports. */ 1395 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1396 sd->satadev_add_info = ahci_portp-> 1397 ahciport_pmult_info->ahcipmi_num_dev_ports; 1398 1399 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1400 "ahci_tran_probe_port: port %s Port Multiplier found", 1401 portstr); 1402 break; 1403 1404 case SATA_DTYPE_UNKNOWN: 1405 sd->satadev_type = SATA_DTYPE_UNKNOWN; 1406 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1407 "ahci_tran_probe_port: port %s Unknown device found", 1408 portstr); 1409 break; 1410 1411 default: 1412 /* we don't support any other device types */ 1413 sd->satadev_type = SATA_DTYPE_NONE; 1414 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1415 "ahci_tran_probe_port: port %s No device found", portstr); 1416 break; 1417 } 1418 1419 out: 1420 /* Register update only fails while probing a pmult/pmport */ 1421 if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) { 1422 ahci_update_sata_registers(ahci_ctlp, port, sd); 1423 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 1424 if (port_state & SATA_STATE_READY) 1425 if (ahci_update_pmult_pscr(ahci_ctlp, 1426 &addr, sd) != AHCI_SUCCESS) 1427 rval = SATA_FAILURE; 1428 } 1429 1430 mutex_exit(&ahci_portp->ahciport_mutex); 1431 return (rval); 1432 } 1433 1434 /* 1435 * There are four operation modes in sata framework: 1436 * SATA_OPMODE_INTERRUPTS 1437 * SATA_OPMODE_POLLING 1438 * SATA_OPMODE_ASYNCH 1439 * SATA_OPMODE_SYNCH 1440 * 1441 * Their combined meanings as following: 1442 * 1443 * SATA_OPMODE_SYNCH 1444 * The command has to be completed before sata_tran_start functions returns. 1445 * Either interrupts or polling could be used - it's up to the driver. 1446 * Mode used currently for internal, sata-module initiated operations. 1447 * 1448 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS 1449 * It is the same as the one above. 1450 * 1451 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING 1452 * The command has to be completed before sata_tran_start function returns. 1453 * No interrupt used, polling only. This should be the mode used for scsi 1454 * packets with FLAG_NOINTR. 1455 * 1456 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS 1457 * The command may be queued (callback function specified). Interrupts could 1458 * be used. It's normal operation mode. 1459 */ 1460 /* 1461 * Called by sata framework to transport a sata packet down stream. 1462 */ 1463 static int 1464 ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt) 1465 { 1466 ahci_ctl_t *ahci_ctlp; 1467 ahci_port_t *ahci_portp; 1468 ahci_addr_t addr; 1469 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 1470 uint8_t port; 1471 char portstr[10]; 1472 1473 ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip)); 1474 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 1475 1476 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 1477 "ahci_tran_start enter: cport %d satapkt 0x%p", 1478 cport, (void *)spkt); 1479 1480 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 1481 1482 mutex_enter(&ahci_portp->ahciport_mutex); 1483 ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr); 1484 SET_PORTSTR(portstr, &addr); 1485 1486 /* Sanity check */ 1487 if (AHCI_ADDR_IS_PMPORT(&addr)) { 1488 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT || 1489 ahci_portp->ahciport_pmult_info == NULL) { 1490 1491 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1492 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1493 spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN; 1494 ahci_update_sata_registers(ahci_ctlp, port, 1495 &spkt->satapkt_device); 1496 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1497 "ahci_tran_start returning PORT_ERROR while " 1498 "pmult removed: port: %s", portstr); 1499 mutex_exit(&ahci_portp->ahciport_mutex); 1500 return (SATA_TRAN_PORT_ERROR); 1501 } 1502 1503 if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) & 1504 SATA_STATE_READY)) { 1505 if (!ddi_in_panic() || 1506 ahci_initialize_pmport(ahci_ctlp, 1507 ahci_portp, &addr) != AHCI_SUCCESS) { 1508 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1509 spkt->satapkt_device.satadev_type = 1510 AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr); 1511 spkt->satapkt_device.satadev_state = 1512 AHCIPORT_GET_STATE(ahci_portp, &addr); 1513 ahci_update_sata_registers(ahci_ctlp, port, 1514 &spkt->satapkt_device); 1515 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1516 "ahci_tran_start returning PORT_ERROR " 1517 "while sub-link is not initialized " 1518 "at port: %s", portstr); 1519 mutex_exit(&ahci_portp->ahciport_mutex); 1520 return (SATA_TRAN_PORT_ERROR); 1521 } 1522 } 1523 } 1524 1525 if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED || 1526 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN|| 1527 AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) { 1528 /* 1529 * In case the target driver would send the packet before 1530 * sata framework can have the opportunity to process those 1531 * event reports. 1532 */ 1533 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1534 spkt->satapkt_device.satadev_state = 1535 ahci_portp->ahciport_port_state; 1536 ahci_update_sata_registers(ahci_ctlp, port, 1537 &spkt->satapkt_device); 1538 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1539 "ahci_tran_start returning PORT_ERROR while " 1540 "port in FAILED/SHUTDOWN/PWROFF state: " 1541 "port: %s", portstr); 1542 mutex_exit(&ahci_portp->ahciport_mutex); 1543 return (SATA_TRAN_PORT_ERROR); 1544 } 1545 1546 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) { 1547 /* 1548 * ahci_intr_phyrdy_change() may have rendered it to 1549 * SATA_DTYPE_NONE. 1550 */ 1551 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 1552 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 1553 spkt->satapkt_device.satadev_state = 1554 ahci_portp->ahciport_port_state; 1555 ahci_update_sata_registers(ahci_ctlp, port, 1556 &spkt->satapkt_device); 1557 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1558 "ahci_tran_start returning PORT_ERROR while " 1559 "no device attached: port: %s", portstr); 1560 mutex_exit(&ahci_portp->ahciport_mutex); 1561 return (SATA_TRAN_PORT_ERROR); 1562 } 1563 1564 /* R/W PMULT command will occupy the whole HBA port */ 1565 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 1566 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1567 "ahci_tran_start returning BUSY while " 1568 "executing READ/WRITE PORT-MULT command: " 1569 "port: %s", portstr); 1570 spkt->satapkt_reason = SATA_PKT_BUSY; 1571 mutex_exit(&ahci_portp->ahciport_mutex); 1572 return (SATA_TRAN_BUSY); 1573 } 1574 1575 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 1576 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1577 "ahci_tran_start returning BUSY while " 1578 "hot-plug in progress: port: %s", portstr); 1579 spkt->satapkt_reason = SATA_PKT_BUSY; 1580 mutex_exit(&ahci_portp->ahciport_mutex); 1581 return (SATA_TRAN_BUSY); 1582 } 1583 1584 /* 1585 * SATA HBA driver should remember that a device was reset and it 1586 * is supposed to reject any packets which do not specify either 1587 * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE. 1588 * 1589 * This is to prevent a race condition when a device was arbitrarily 1590 * reset by the HBA driver (and lost it's setting) and a target 1591 * driver sending some commands to a device before the sata framework 1592 * has a chance to restore the device setting (such as cache enable/ 1593 * disable or other resettable stuff). 1594 */ 1595 /* 1596 * It is unnecessary to use specific flags to indicate 1597 * reset_in_progress for a pmport. While mopping, all command will be 1598 * mopped so that the entire HBA port is being dealt as a single 1599 * object. 1600 */ 1601 if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) { 1602 ahci_portp->ahciport_reset_in_progress = 0; 1603 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1604 "ahci_tran_start [CLEAR] the " 1605 "reset_in_progress for port: %d", port); 1606 } 1607 1608 if (ahci_portp->ahciport_reset_in_progress && 1609 ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset && 1610 ! ddi_in_panic()) { 1611 spkt->satapkt_reason = SATA_PKT_BUSY; 1612 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1613 "ahci_tran_start returning BUSY while " 1614 "reset in progress: port: %d", port); 1615 mutex_exit(&ahci_portp->ahciport_mutex); 1616 return (SATA_TRAN_BUSY); 1617 } 1618 1619 #ifdef AHCI_DEBUG 1620 if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) { 1621 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1622 "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d", 1623 spkt, port); 1624 } 1625 #endif 1626 1627 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 1628 spkt->satapkt_reason = SATA_PKT_BUSY; 1629 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1630 "ahci_tran_start returning BUSY while " 1631 "mopping in progress: port: %d", port); 1632 mutex_exit(&ahci_portp->ahciport_mutex); 1633 return (SATA_TRAN_BUSY); 1634 } 1635 1636 if (spkt->satapkt_op_mode & 1637 (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) { 1638 /* 1639 * If a SYNC command to be executed in interrupt context, 1640 * bounce it back to sata module. 1641 */ 1642 if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) && 1643 servicing_interrupt()) { 1644 spkt->satapkt_reason = SATA_PKT_BUSY; 1645 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1646 "ahci_tran_start returning BUSY while " 1647 "sending SYNC mode under interrupt context: " 1648 "port : %d", port); 1649 mutex_exit(&ahci_portp->ahciport_mutex); 1650 return (SATA_TRAN_BUSY); 1651 } 1652 1653 /* We need to do the sync start now */ 1654 if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, 1655 spkt) == AHCI_FAILURE) { 1656 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1657 "return QUEUE_FULL: port %d", port); 1658 mutex_exit(&ahci_portp->ahciport_mutex); 1659 return (SATA_TRAN_QUEUE_FULL); 1660 } 1661 } else { 1662 /* Async start, using interrupt */ 1663 if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt) 1664 == AHCI_FAILURE) { 1665 spkt->satapkt_reason = SATA_PKT_QUEUE_FULL; 1666 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start " 1667 "returning QUEUE_FULL: port %d", port); 1668 mutex_exit(&ahci_portp->ahciport_mutex); 1669 return (SATA_TRAN_QUEUE_FULL); 1670 } 1671 } 1672 1673 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start " 1674 "sata tran accepted: port %s", portstr); 1675 1676 mutex_exit(&ahci_portp->ahciport_mutex); 1677 return (SATA_TRAN_ACCEPTED); 1678 } 1679 1680 /* 1681 * SATA_OPMODE_SYNCH flag is set 1682 * 1683 * If SATA_OPMODE_POLLING flag is set, then we must poll the command 1684 * without interrupt, otherwise we can still use the interrupt. 1685 * 1686 * WARNING!!! ahciport_mutex should be acquired before the function 1687 * is called. 1688 */ 1689 static int 1690 ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1691 ahci_addr_t *addrp, sata_pkt_t *spkt) 1692 { 1693 int pkt_timeout_ticks; 1694 uint32_t timeout_tags; 1695 int rval; 1696 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1697 uint8_t port = addrp->aa_port; 1698 1699 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: " 1700 "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt); 1701 1702 if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) { 1703 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING; 1704 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1705 addrp, spkt)) == AHCI_FAILURE) { 1706 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_POLLING; 1707 return (rval); 1708 } 1709 1710 pkt_timeout_ticks = 1711 drv_usectohz((clock_t)spkt->satapkt_time * 1000000); 1712 1713 while (spkt->satapkt_reason == SATA_PKT_BUSY) { 1714 mutex_exit(&ahci_portp->ahciport_mutex); 1715 1716 /* Simulate the interrupt */ 1717 ahci_port_intr(ahci_ctlp, ahci_portp, port); 1718 1719 drv_usecwait(AHCI_10MS_USECS); 1720 1721 mutex_enter(&ahci_portp->ahciport_mutex); 1722 pkt_timeout_ticks -= AHCI_10MS_TICKS; 1723 if (pkt_timeout_ticks < 0) { 1724 cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start " 1725 "port %d satapkt 0x%p timed out\n", 1726 instance, port, (void *)spkt); 1727 timeout_tags = (0x1 << rval); 1728 mutex_exit(&ahci_portp->ahciport_mutex); 1729 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 1730 port, timeout_tags); 1731 mutex_enter(&ahci_portp->ahciport_mutex); 1732 } 1733 } 1734 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING; 1735 return (AHCI_SUCCESS); 1736 1737 } else { 1738 if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp, 1739 addrp, spkt)) == AHCI_FAILURE) 1740 return (rval); 1741 1742 #if AHCI_DEBUG 1743 /* 1744 * Note that the driver always uses the slot 0 to deliver 1745 * REQUEST SENSE or READ LOG EXT command 1746 */ 1747 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 1748 ASSERT(rval == 0); 1749 #endif 1750 1751 while (spkt->satapkt_reason == SATA_PKT_BUSY) 1752 cv_wait(&ahci_portp->ahciport_cv, 1753 &ahci_portp->ahciport_mutex); 1754 1755 return (AHCI_SUCCESS); 1756 } 1757 } 1758 1759 #define SENDUP_PACKET(ahci_portp, satapkt, reason) \ 1760 if (satapkt) { \ 1761 satapkt->satapkt_reason = reason; \ 1762 /* \ 1763 * We set the satapkt_reason in both sync and \ 1764 * non-sync cases. \ 1765 */ \ 1766 } \ 1767 if (satapkt && \ 1768 ! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1769 satapkt->satapkt_comp) { \ 1770 mutex_exit(&ahci_portp->ahciport_mutex); \ 1771 (*satapkt->satapkt_comp)(satapkt); \ 1772 mutex_enter(&ahci_portp->ahciport_mutex); \ 1773 } else { \ 1774 if (satapkt && \ 1775 (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) && \ 1776 ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING)) \ 1777 cv_broadcast(&ahci_portp->ahciport_cv); \ 1778 } 1779 1780 /* 1781 * Searches for and claims a free command slot. 1782 * 1783 * Returns value: 1784 * 1785 * AHCI_FAILURE returned only if 1786 * 1. No empty slot left 1787 * 2. Non-queued command requested while queued command(s) is outstanding 1788 * 3. Queued command requested while non-queued command(s) is outstanding 1789 * 4. HBA doesn't support multiple-use of command list while already a 1790 * non-queued command is oustanding 1791 * 5. Queued command requested while some queued command(s) has been 1792 * outstanding on a different port multiplier port. (AHCI spec 1.2, 1793 * 9.1.2) 1794 * 1795 * claimed slot number returned if succeeded 1796 * 1797 * NOTE: it will always return slot 0 for following commands to simplify the 1798 * algorithm. 1799 * 1. REQUEST SENSE or READ LOG EXT command during error recovery process 1800 * 2. READ/WRITE PORTMULT command 1801 * 1802 * WARNING!!! ahciport_mutex should be acquired before the function 1803 * is called. 1804 */ 1805 static int 1806 ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1807 ahci_addr_t *addrp, int command_type) 1808 { 1809 uint32_t port_cmd_issue; 1810 uint32_t free_slots; 1811 int slot; 1812 1813 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter " 1814 "ahciport_pending_tags = 0x%x " 1815 "ahciport_pending_ncq_tags = 0x%x", 1816 ahci_portp->ahciport_pending_tags, 1817 ahci_portp->ahciport_pending_ncq_tags); 1818 1819 /* 1820 * According to the AHCI spec, system software is responsible to 1821 * ensure that queued and non-queued commands are not mixed in 1822 * the command list. 1823 */ 1824 if (command_type == AHCI_NON_NCQ_CMD) { 1825 /* Non-NCQ command request */ 1826 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1827 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1828 "ahci_claim_free_slot: there is still pending " 1829 "queued command(s) in the command list, " 1830 "so no available slot for the non-queued " 1831 "command", NULL); 1832 return (AHCI_FAILURE); 1833 } 1834 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 1835 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 1836 "ahci_claim_free_slot: there is still pending " 1837 "read/write port-mult command(s) in command list, " 1838 "so no available slot for the non-queued command", 1839 NULL); 1840 return (AHCI_FAILURE); 1841 } 1842 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) && 1843 NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1844 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1845 "ahci_claim_free_slot: HBA cannot support multiple-" 1846 "use of the command list for non-queued commands", 1847 NULL); 1848 return (AHCI_FAILURE); 1849 } 1850 free_slots = (~ahci_portp->ahciport_pending_tags) & 1851 AHCI_SLOT_MASK(ahci_ctlp); 1852 } else if (command_type == AHCI_NCQ_CMD) { 1853 /* NCQ command request */ 1854 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 1855 AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp, 1856 "ahci_claim_free_slot: there is still pending " 1857 "non-queued command(s) in the command list, " 1858 "so no available slot for the queued command", 1859 NULL); 1860 return (AHCI_FAILURE); 1861 } 1862 1863 /* 1864 * NCQ commands cannot be sent to different port multiplier 1865 * ports in Command-Based Switching mode 1866 */ 1867 /* 1868 * NOTE: In Command-Based Switching mode, AHCI controller 1869 * usually reports a 'Handshake Error' when multiple NCQ 1870 * commands are outstanding simultaneously. 1871 */ 1872 if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) { 1873 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1874 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) && 1875 NCQ_CMD_IN_PROGRESS(ahci_portp) && 1876 AHCIPORT_NCQ_PMPORT(ahci_portp) != 1877 addrp->aa_pmport) { 1878 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1879 "ahci_claim_free_slot: there is still " 1880 "pending queued command(s) in the " 1881 "command list for another Port Multiplier " 1882 "port, so no available slot.", NULL); 1883 return (AHCI_FAILURE); 1884 } 1885 } 1886 1887 free_slots = (~ahci_portp->ahciport_pending_ncq_tags) & 1888 AHCI_NCQ_SLOT_MASK(ahci_portp); 1889 } else if (command_type == AHCI_ERR_RETRI_CMD) { 1890 /* Error retrieval command request */ 1891 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 1892 "ahci_claim_free_slot: slot 0 is allocated for REQUEST " 1893 "SENSE or READ LOG EXT command", NULL); 1894 slot = 0; 1895 goto out; 1896 } else if (command_type == AHCI_RDWR_PMULT_CMD) { 1897 /* 1898 * An extra check on PxCI. Sometimes PxCI bits may not be 1899 * cleared during hot-plug or error recovery process. 1900 */ 1901 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 1902 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port)); 1903 1904 if (port_cmd_issue != 0) { 1905 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 1906 "ahci_claim_free_slot: there is still pending " 1907 "command(s) in command list (0x%x/0x%x, PxCI %x)," 1908 "so no available slot for R/W PMULT command.", 1909 NON_NCQ_CMD_IN_PROGRESS(ahci_portp), 1910 NCQ_CMD_IN_PROGRESS(ahci_portp), 1911 port_cmd_issue); 1912 return (AHCI_FAILURE); 1913 } 1914 1915 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 1916 "ahci_claim_free_slot: slot 0 is allocated for " 1917 "READ/WRITE PORTMULT command", NULL); 1918 slot = 0; 1919 goto out; 1920 } 1921 1922 slot = ddi_ffs(free_slots) - 1; 1923 if (slot == -1) { 1924 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1925 "ahci_claim_free_slot: no empty slots", NULL); 1926 return (AHCI_FAILURE); 1927 } 1928 1929 /* 1930 * According to the AHCI spec, to allow a simple mechanism for the 1931 * HBA to map command list slots to queue entries, software must 1932 * match the tag number it uses to the slot it is placing the command 1933 * in. For example, if a queued command is placed in slot 5, the tag 1934 * for that command must be 5. 1935 */ 1936 if (command_type == AHCI_NCQ_CMD) { 1937 ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot); 1938 if (AHCI_ADDR_IS_PMPORT(addrp)) { 1939 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 1940 AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport; 1941 } 1942 } 1943 1944 ahci_portp->ahciport_pending_tags |= (0x1 << slot); 1945 1946 out: 1947 AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp, 1948 "ahci_claim_free_slot: found slot: 0x%x", slot); 1949 1950 return (slot); 1951 } 1952 1953 /* 1954 * Builds the Command Table for the sata packet and delivers it to controller. 1955 * 1956 * Returns: 1957 * slot number if we can obtain a slot successfully 1958 * otherwise, return AHCI_FAILURE 1959 * 1960 * WARNING!!! ahciport_mutex should be acquired before the function is called. 1961 */ 1962 static int 1963 ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 1964 ahci_addr_t *addrp, sata_pkt_t *spkt) 1965 { 1966 int cmd_slot; 1967 sata_cmd_t *scmd; 1968 ahci_fis_h2d_register_t *h2d_register_fisp; 1969 ahci_cmd_table_t *cmd_table; 1970 ahci_cmd_header_t *cmd_header; 1971 int ncookies; 1972 int i; 1973 int command_type = AHCI_NON_NCQ_CMD; 1974 int ncq_qdepth; 1975 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 1976 uint8_t port, pmport; 1977 #if AHCI_DEBUG 1978 uint32_t *ptr; 1979 uint8_t *ptr2; 1980 #endif 1981 1982 port = addrp->aa_port; 1983 pmport = addrp->aa_pmport; 1984 1985 spkt->satapkt_reason = SATA_PKT_BUSY; 1986 1987 scmd = &spkt->satapkt_cmd; 1988 1989 /* Check if the command is a NCQ command */ 1990 if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED || 1991 scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) { 1992 command_type = AHCI_NCQ_CMD; 1993 1994 /* 1995 * When NCQ is support, system software must determine the 1996 * maximum tag allowed by the device and the HBA, and it 1997 * must use a value not beyond of the lower bound of the two. 1998 * 1999 * Sata module is going to calculate the qdepth and send 2000 * down to HBA driver via sata_cmd. 2001 */ 2002 ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1; 2003 2004 /* 2005 * At the moment, the driver doesn't support the dynamic 2006 * setting of the maximum ncq depth, and the value can be 2007 * set either during the attach or after hot-plug insertion. 2008 */ 2009 if (ahci_portp->ahciport_max_ncq_tags == 0) { 2010 ahci_portp->ahciport_max_ncq_tags = ncq_qdepth; 2011 AHCIDBG(AHCIDBG_NCQ, ahci_ctlp, 2012 "ahci_deliver_satapkt: port %d the max tags for " 2013 "NCQ command is %d", port, ncq_qdepth); 2014 } else { 2015 if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) { 2016 cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt" 2017 " port %d the max tag for NCQ command is " 2018 "requested to change from %d to %d, at the" 2019 " moment the driver doesn't support the " 2020 "dynamic change so it's going to " 2021 "still use the previous tag value", 2022 instance, port, 2023 ahci_portp->ahciport_max_ncq_tags, 2024 ncq_qdepth); 2025 } 2026 } 2027 } 2028 2029 /* Check if the command is an error retrieval command */ 2030 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 2031 command_type = AHCI_ERR_RETRI_CMD; 2032 2033 /* Check if the command is an read/write pmult command */ 2034 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2035 command_type = AHCI_RDWR_PMULT_CMD; 2036 2037 /* Check if there is an empty command slot */ 2038 cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp, 2039 addrp, command_type); 2040 if (cmd_slot == AHCI_FAILURE) { 2041 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL); 2042 return (AHCI_FAILURE); 2043 } 2044 2045 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp, 2046 "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, " 2047 "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg, 2048 cmd_slot, port, (void *)spkt); 2049 2050 cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot]; 2051 bzero((void *)cmd_table, ahci_cmd_table_size); 2052 2053 /* For data transfer operations, it is the H2D Register FIS */ 2054 h2d_register_fisp = 2055 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 2056 2057 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 2058 2059 /* 2060 * PMP field only make sense when target is a port multiplier or a 2061 * device behind a port multiplier. Otherwise should set it to 0. 2062 */ 2063 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2064 SET_FIS_PMP(h2d_register_fisp, pmport); 2065 2066 SET_FIS_CDMDEVCTL(h2d_register_fisp, 1); 2067 SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg); 2068 SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg); 2069 SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb); 2070 2071 switch (scmd->satacmd_addr_type) { 2072 2073 case 0: 2074 /* 2075 * satacmd_addr_type will be 0 for the commands below: 2076 * ATAPI command 2077 * SATAC_IDLE_IM 2078 * SATAC_STANDBY_IM 2079 * SATAC_DOWNLOAD_MICROCODE 2080 * SATAC_FLUSH_CACHE 2081 * SATAC_SET_FEATURES 2082 * SATAC_SMART 2083 * SATAC_ID_PACKET_DEVICE 2084 * SATAC_ID_DEVICE 2085 * SATAC_READ_PORTMULT 2086 * SATAC_WRITE_PORTMULT 2087 */ 2088 /* FALLTHRU */ 2089 2090 case ATA_ADDR_LBA: 2091 /* FALLTHRU */ 2092 2093 case ATA_ADDR_LBA28: 2094 /* LBA[7:0] */ 2095 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2096 2097 /* LBA[15:8] */ 2098 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2099 2100 /* LBA[23:16] */ 2101 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2102 2103 /* LBA [27:24] (also called dev_head) */ 2104 SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg); 2105 2106 break; 2107 2108 case ATA_ADDR_LBA48: 2109 /* LBA[7:0] */ 2110 SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb); 2111 2112 /* LBA[15:8] */ 2113 SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb); 2114 2115 /* LBA[23:16] */ 2116 SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb); 2117 2118 /* LBA [31:24] */ 2119 SET_FIS_SECTOR_EXP(h2d_register_fisp, 2120 scmd->satacmd_lba_low_msb); 2121 2122 /* LBA [39:32] */ 2123 SET_FIS_CYL_LOW_EXP(h2d_register_fisp, 2124 scmd->satacmd_lba_mid_msb); 2125 2126 /* LBA [47:40] */ 2127 SET_FIS_CYL_HI_EXP(h2d_register_fisp, 2128 scmd->satacmd_lba_high_msb); 2129 2130 /* Set dev_head */ 2131 SET_FIS_DEV_HEAD(h2d_register_fisp, 2132 scmd->satacmd_device_reg); 2133 2134 /* Set the extended sector count and features */ 2135 SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp, 2136 scmd->satacmd_sec_count_msb); 2137 SET_FIS_FEATURES_EXP(h2d_register_fisp, 2138 scmd->satacmd_features_reg_ext); 2139 break; 2140 } 2141 2142 /* 2143 * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is 2144 * filled into features field, and sector count 8:15 is filled into 2145 * features (exp) field. The hba driver doesn't need to anything 2146 * special with regard to this, since sata framework has already 2147 * done so. 2148 * 2149 * However the driver needs to make sure TAG is filled into sector 2150 * field. 2151 */ 2152 if (command_type == AHCI_NCQ_CMD) { 2153 SET_FIS_SECTOR_COUNT(h2d_register_fisp, 2154 (cmd_slot << SATA_TAG_QUEUING_SHIFT)); 2155 } 2156 2157 ncookies = scmd->satacmd_num_dma_cookies; 2158 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2159 "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x", 2160 ncookies, ahci_dma_prdt_number); 2161 2162 ASSERT(ncookies <= ahci_dma_prdt_number); 2163 ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0; 2164 2165 /* *** now fill the scatter gather list ******* */ 2166 for (i = 0; i < ncookies; i++) { 2167 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr = 2168 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0]; 2169 cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper = 2170 scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1]; 2171 cmd_table->ahcict_prdt[i].ahcipi_descr_info = 2172 scmd->satacmd_dma_cookie_list[i].dmac_size - 1; 2173 ahci_portp->ahciport_prd_bytecounts[cmd_slot] += 2174 scmd->satacmd_dma_cookie_list[i].dmac_size; 2175 } 2176 2177 AHCIDBG(AHCIDBG_PRDT, ahci_ctlp, 2178 "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x", 2179 ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot); 2180 2181 /* The ACMD field is filled in for ATAPI command */ 2182 if (scmd->satacmd_cmd_reg == SATAC_PACKET) { 2183 bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd, 2184 SATA_ATAPI_MAX_CDB_LEN); 2185 } 2186 2187 /* Set Command Header in Command List */ 2188 cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot]; 2189 BZERO_DESCR_INFO(cmd_header); 2190 BZERO_PRD_BYTE_COUNT(cmd_header); 2191 2192 /* Set the number of entries in the PRD table */ 2193 SET_PRD_TABLE_LENGTH(cmd_header, ncookies); 2194 2195 /* Set the length of the command in the CFIS area */ 2196 SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH); 2197 2198 /* 2199 * PMP field only make sense when target is a port multiplier or a 2200 * device behind a port multiplier. Otherwise should set it to 0. 2201 */ 2202 if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp)) 2203 SET_PORT_MULTI_PORT(cmd_header, pmport); 2204 2205 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is " 2206 "sata_data_direction = 0x%x", 2207 scmd->satacmd_flags.sata_data_direction); 2208 2209 /* Set A bit if it is an ATAPI command */ 2210 if (scmd->satacmd_cmd_reg == SATAC_PACKET) 2211 SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI); 2212 2213 /* Set W bit if data is going to the device */ 2214 if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE) 2215 SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE); 2216 2217 /* 2218 * Set the prefetchable bit - this bit is only valid if the PRDTL 2219 * field is non-zero or the ATAPI 'A' bit is set in the command 2220 * header. This bit cannot be set when using native command 2221 * queuing commands or when using FIS-based switching with a Port 2222 * multiplier. 2223 */ 2224 if (command_type != AHCI_NCQ_CMD) 2225 SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE); 2226 2227 /* 2228 * Now remember the sata packet in ahciport_slot_pkts[]. 2229 * Error retrieval command and r/w port multiplier command will 2230 * be stored specifically for each port. 2231 */ 2232 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 2233 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 2234 ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt; 2235 2236 /* 2237 * We are overloading satapkt_hba_driver_private with 2238 * watched_cycle count. 2239 */ 2240 spkt->satapkt_hba_driver_private = (void *)(intptr_t)0; 2241 2242 #if AHCI_DEBUG 2243 if (ahci_debug_flags & AHCIDBG_ATACMD && 2244 scmd->satacmd_cmd_reg != SATAC_PACKET || 2245 ahci_debug_flags & AHCIDBG_ATAPICMD && 2246 scmd->satacmd_cmd_reg == SATAC_PACKET) { 2247 2248 /* Dump the command header and table */ 2249 ahci_log(ahci_ctlp, CE_WARN, "\n"); 2250 ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt " 2251 "0x%p cmd_reg 0x%x port %d", spkt, 2252 scmd->satacmd_cmd_reg, port); 2253 ptr = (uint32_t *)cmd_header; 2254 ahci_log(ahci_ctlp, CE_WARN, 2255 " Command Header:%8x %8x %8x %8x", 2256 ptr[0], ptr[1], ptr[2], ptr[3]); 2257 2258 /* Dump the H2D register FIS */ 2259 ptr = (uint32_t *)h2d_register_fisp; 2260 ahci_log(ahci_ctlp, CE_WARN, 2261 " Command FIS: %8x %8x %8x %8x", 2262 ptr[0], ptr[1], ptr[2], ptr[3]); 2263 2264 /* Dump the ACMD register FIS */ 2265 ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd); 2266 for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++) 2267 if (ahci_debug_flags & AHCIDBG_ATAPICMD) 2268 ahci_log(ahci_ctlp, CE_WARN, 2269 " ATAPI command: %2x %2x %2x %2x " 2270 "%2x %2x %2x %2x", 2271 ptr2[8 * i], ptr2[8 * i + 1], 2272 ptr2[8 * i + 2], ptr2[8 * i + 3], 2273 ptr2[8 * i + 4], ptr2[8 * i + 5], 2274 ptr2[8 * i + 6], ptr2[8 * i + 7]); 2275 2276 /* Dump the PRDT */ 2277 for (i = 0; i < ncookies; i++) { 2278 ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]); 2279 ahci_log(ahci_ctlp, CE_WARN, 2280 " Cookie %d: %8x %8x %8x %8x", 2281 i, ptr[0], ptr[1], ptr[2], ptr[3]); 2282 } 2283 } 2284 #endif 2285 2286 (void) ddi_dma_sync( 2287 ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot], 2288 0, 2289 ahci_cmd_table_size, 2290 DDI_DMA_SYNC_FORDEV); 2291 2292 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 2293 cmd_slot * sizeof (ahci_cmd_header_t), 2294 sizeof (ahci_cmd_header_t), 2295 DDI_DMA_SYNC_FORDEV); 2296 2297 /* Set the corresponding bit in the PxSACT.DS for queued command */ 2298 if (command_type == AHCI_NCQ_CMD) { 2299 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2300 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port), 2301 (0x1 << cmd_slot)); 2302 } 2303 2304 /* Indicate to the HBA that a command is active. */ 2305 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 2306 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 2307 (0x1 << cmd_slot)); 2308 2309 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt " 2310 "exit: port %d", port); 2311 2312 return (cmd_slot); 2313 } 2314 2315 /* 2316 * Called by the sata framework to abort the previously sent packet(s). 2317 * 2318 * Reset device to abort commands. 2319 */ 2320 static int 2321 ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag) 2322 { 2323 ahci_ctl_t *ahci_ctlp; 2324 ahci_port_t *ahci_portp; 2325 uint32_t slot_status = 0; 2326 uint32_t aborted_tags = 0; 2327 uint32_t finished_tags = 0; 2328 uint8_t cport = spkt->satapkt_device.satadev_addr.cport; 2329 uint8_t port; 2330 int tmp_slot; 2331 int instance = ddi_get_instance(dip); 2332 2333 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2334 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2335 2336 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2337 "ahci_tran_abort enter: port %d", port); 2338 2339 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2340 mutex_enter(&ahci_portp->ahciport_mutex); 2341 2342 /* 2343 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2344 * commands are being mopped, therefore there is nothing else to do 2345 */ 2346 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2347 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2348 "ahci_tran_abort: port %d is in " 2349 "mopping process, so just return directly ", port); 2350 mutex_exit(&ahci_portp->ahciport_mutex); 2351 return (SATA_SUCCESS); 2352 } 2353 2354 /* 2355 * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT 2356 * command is being executed so no other commands is outstanding, 2357 * nothing to do. 2358 */ 2359 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) { 2360 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2361 "ahci_tran_abort: port %d is reading/writing " 2362 "port multiplier, so just return directly ", port); 2363 mutex_exit(&ahci_portp->ahciport_mutex); 2364 return (SATA_SUCCESS); 2365 } 2366 2367 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2368 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2369 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2370 /* 2371 * In case the targer driver would send the request before 2372 * sata framework can have the opportunity to process those 2373 * event reports. 2374 */ 2375 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2376 spkt->satapkt_device.satadev_state = 2377 ahci_portp->ahciport_port_state; 2378 ahci_update_sata_registers(ahci_ctlp, port, 2379 &spkt->satapkt_device); 2380 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2381 "ahci_tran_abort returning SATA_FAILURE while " 2382 "port in FAILED/SHUTDOWN/PWROFF state: " 2383 "port: %d", port); 2384 mutex_exit(&ahci_portp->ahciport_mutex); 2385 return (SATA_FAILURE); 2386 } 2387 2388 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 2389 /* 2390 * ahci_intr_phyrdy_change() may have rendered it to 2391 * AHCI_PORT_TYPE_NODEV. 2392 */ 2393 spkt->satapkt_reason = SATA_PKT_PORT_ERROR; 2394 spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE; 2395 spkt->satapkt_device.satadev_state = 2396 ahci_portp->ahciport_port_state; 2397 ahci_update_sata_registers(ahci_ctlp, port, 2398 &spkt->satapkt_device); 2399 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2400 "ahci_tran_abort returning SATA_FAILURE while " 2401 "no device attached: port: %d", port); 2402 mutex_exit(&ahci_portp->ahciport_mutex); 2403 return (SATA_FAILURE); 2404 } 2405 2406 if (flag == SATA_ABORT_ALL_PACKETS) { 2407 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2408 aborted_tags = ahci_portp->ahciport_pending_tags; 2409 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2410 aborted_tags = ahci_portp->ahciport_pending_ncq_tags; 2411 2412 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets", 2413 instance, port); 2414 } else { 2415 aborted_tags = 0xffffffff; 2416 /* 2417 * Aborting one specific packet, first search the 2418 * ahciport_slot_pkts[] list for matching spkt. 2419 */ 2420 for (tmp_slot = 0; 2421 tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) { 2422 if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) { 2423 aborted_tags = (0x1 << tmp_slot); 2424 break; 2425 } 2426 } 2427 2428 if (aborted_tags == 0xffffffff) { 2429 /* request packet is not on the pending list */ 2430 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 2431 "Cannot find the aborting pkt 0x%p on the " 2432 "pending list", (void *)spkt); 2433 ahci_update_sata_registers(ahci_ctlp, port, 2434 &spkt->satapkt_device); 2435 mutex_exit(&ahci_portp->ahciport_mutex); 2436 return (SATA_FAILURE); 2437 } 2438 cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p", 2439 instance, port, (void *)spkt); 2440 } 2441 2442 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2443 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2444 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2445 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2446 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2447 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2448 2449 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2450 ahci_portp->ahciport_mop_in_progress++; 2451 2452 /* 2453 * To abort the packet(s), first we are trying to clear PxCMD.ST 2454 * to stop the port, and if the port can be stopped 2455 * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0', 2456 * then we just send back the aborted packet(s) with ABORTED flag 2457 * and then restart the port by setting PxCMD.ST and PxCMD.FRE. 2458 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we 2459 * perform a COMRESET. 2460 */ 2461 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 2462 ahci_portp, port, NULL, NULL); 2463 2464 /* 2465 * Compute which have finished and which need to be retried. 2466 * 2467 * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags 2468 * minus the slot_status. The aborted_tags has to be deducted by 2469 * finished_tags since we can't possibly abort a tag which had finished 2470 * already. 2471 */ 2472 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2473 finished_tags = ahci_portp->ahciport_pending_tags & 2474 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2475 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2476 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2477 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2478 2479 aborted_tags &= ~finished_tags; 2480 2481 ahci_mop_commands(ahci_ctlp, 2482 ahci_portp, 2483 slot_status, 2484 0, /* failed tags */ 2485 0, /* timeout tags */ 2486 aborted_tags, 2487 0); /* reset tags */ 2488 2489 ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device); 2490 mutex_exit(&ahci_portp->ahciport_mutex); 2491 2492 return (SATA_SUCCESS); 2493 } 2494 2495 /* 2496 * Used to do device reset and reject all the pending packets on a device 2497 * during the reset operation. 2498 * 2499 * NOTE: ONLY called by ahci_tran_reset_dport 2500 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2501 */ 2502 static int 2503 ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp, 2504 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2505 { 2506 uint32_t slot_status = 0; 2507 uint32_t reset_tags = 0; 2508 uint32_t finished_tags = 0; 2509 uint8_t port = addrp->aa_port; 2510 sata_device_t sdevice; 2511 int ret; 2512 2513 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2514 "ahci_reset_device_reject_pkts on port: %d", port); 2515 2516 /* 2517 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2518 * commands are being mopped, therefore there is nothing else to do 2519 */ 2520 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2521 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2522 "ahci_reset_device_reject_pkts: port %d is in " 2523 "mopping process, so return directly ", port); 2524 return (SATA_SUCCESS); 2525 } 2526 2527 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2528 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2529 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2530 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2531 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2532 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2533 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2534 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2535 } 2536 2537 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2538 != AHCI_SUCCESS) { 2539 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2540 "Try to do a port reset after software " 2541 "reset failed", port); 2542 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 2543 if (ret != AHCI_SUCCESS) { 2544 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2545 "ahci_reset_device_reject_pkts: port %d " 2546 "failed", port); 2547 return (SATA_FAILURE); 2548 } 2549 } 2550 /* Set the reset in progress flag */ 2551 ahci_portp->ahciport_reset_in_progress = 1; 2552 2553 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2554 ahci_portp->ahciport_mop_in_progress++; 2555 2556 /* Indicate to the framework that a reset has happened */ 2557 bzero((void *)&sdevice, sizeof (sata_device_t)); 2558 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2559 sdevice.satadev_addr.pmport = 0; 2560 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 2561 sdevice.satadev_state = SATA_DSTATE_RESET | 2562 SATA_DSTATE_PWR_ACTIVE; 2563 mutex_exit(&ahci_portp->ahciport_mutex); 2564 sata_hba_event_notify( 2565 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2566 &sdevice, 2567 SATA_EVNT_DEVICE_RESET); 2568 mutex_enter(&ahci_portp->ahciport_mutex); 2569 2570 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2571 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 2572 2573 /* Next try to mop the pending commands */ 2574 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2575 finished_tags = ahci_portp->ahciport_pending_tags & 2576 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2577 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2578 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2579 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2580 2581 reset_tags &= ~finished_tags; 2582 2583 ahci_mop_commands(ahci_ctlp, 2584 ahci_portp, 2585 slot_status, 2586 0, /* failed tags */ 2587 0, /* timeout tags */ 2588 0, /* aborted tags */ 2589 reset_tags); /* reset tags */ 2590 2591 return (SATA_SUCCESS); 2592 } 2593 2594 /* 2595 * Used to do device reset and reject all the pending packets on a device 2596 * during the reset operation. 2597 * 2598 * NOTE: ONLY called by ahci_tran_reset_dport 2599 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2600 */ 2601 static int 2602 ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp, 2603 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2604 { 2605 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 2606 uint8_t port = addrp->aa_port; 2607 uint8_t pmport = addrp->aa_pmport; 2608 sata_device_t sdevice; 2609 2610 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp, 2611 "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport); 2612 2613 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2614 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2615 "ahci_reset_pmdevice_reject_pkts: port %d is in " 2616 "mopping process, so return directly ", port); 2617 return (SATA_SUCCESS); 2618 } 2619 2620 /* Checking for outstanding commands */ 2621 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2622 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2623 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2624 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2625 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2626 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2627 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2628 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2629 } 2630 2631 /* Issue SOFTWARE reset command. */ 2632 if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp) 2633 != AHCI_SUCCESS) { 2634 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2635 "Try to do a port reset after software " 2636 "reset failed", port); 2637 return (SATA_FAILURE); 2638 } 2639 2640 /* Set the reset in progress flag */ 2641 ahci_portp->ahciport_reset_in_progress = 1; 2642 2643 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2644 ahci_portp->ahciport_mop_in_progress++; 2645 2646 /* Indicate to the framework that a reset has happened */ 2647 bzero((void *)&sdevice, sizeof (sata_device_t)); 2648 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 2649 sdevice.satadev_addr.pmport = pmport; 2650 if (AHCI_ADDR_IS_PMULT(addrp)) 2651 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 2652 else 2653 sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT; 2654 sdevice.satadev_state = SATA_DSTATE_RESET | 2655 SATA_DSTATE_PWR_ACTIVE; 2656 mutex_exit(&ahci_portp->ahciport_mutex); 2657 sata_hba_event_notify( 2658 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 2659 &sdevice, 2660 SATA_EVNT_DEVICE_RESET); 2661 mutex_enter(&ahci_portp->ahciport_mutex); 2662 2663 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 2664 "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET", 2665 port, pmport); 2666 2667 /* Next try to mop the pending commands */ 2668 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2669 finished_tags = ahci_portp->ahciport_pending_tags & 2670 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2671 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2672 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2673 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2674 reset_tags &= ~finished_tags; 2675 2676 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 2677 "reset_tags = %x, finished_tags = %x, slot_status = %x", 2678 reset_tags, finished_tags, slot_status); 2679 2680 /* 2681 * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we 2682 * try to reset a single device behind a port multiplier will 2683 * terminate all the commands on that HBA port. We need mop these 2684 * commands as well. 2685 */ 2686 ahci_mop_commands(ahci_ctlp, 2687 ahci_portp, 2688 slot_status, 2689 0, /* failed tags */ 2690 0, /* timeout tags */ 2691 0, /* aborted tags */ 2692 reset_tags); /* reset tags */ 2693 2694 return (SATA_SUCCESS); 2695 } 2696 2697 /* 2698 * Used to do port reset and reject all the pending packets on a port during 2699 * the reset operation. 2700 * 2701 * WARNING!!! ahciport_mutex should be acquired before the function is called. 2702 */ 2703 static int 2704 ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp, 2705 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 2706 { 2707 uint32_t slot_status = 0; 2708 uint32_t reset_tags = 0; 2709 uint32_t finished_tags = 0; 2710 uint8_t port = addrp->aa_port; 2711 2712 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2713 "ahci_reset_port_reject_pkts at port: %d", port); 2714 2715 /* 2716 * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending 2717 * commands are being mopped, therefore there is nothing else to do 2718 */ 2719 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2720 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2721 "ahci_reset_port_reject_pkts: port %d is in " 2722 "mopping process, so return directly ", port); 2723 return (SATA_SUCCESS); 2724 } 2725 2726 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2727 ahci_portp->ahciport_mop_in_progress++; 2728 2729 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2730 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2731 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2732 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2733 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2734 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 2735 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2736 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2737 } 2738 2739 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 2740 ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 2741 NULL) != AHCI_SUCCESS) { 2742 2743 /* Clear mop flag */ 2744 ahci_portp->ahciport_mop_in_progress--; 2745 if (ahci_portp->ahciport_mop_in_progress == 0) 2746 ahci_portp->ahciport_flags &= 2747 ~AHCI_PORT_FLAG_MOPPING; 2748 return (SATA_FAILURE); 2749 } 2750 2751 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2752 finished_tags = ahci_portp->ahciport_pending_tags & 2753 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 2754 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2755 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 2756 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 2757 2758 reset_tags &= ~finished_tags; 2759 2760 ahci_mop_commands(ahci_ctlp, 2761 ahci_portp, 2762 slot_status, 2763 0, /* failed tags */ 2764 0, /* timeout tags */ 2765 0, /* aborted tags */ 2766 reset_tags); /* reset tags */ 2767 2768 return (SATA_SUCCESS); 2769 } 2770 2771 /* 2772 * Used to do hba reset and reject all the pending packets on all ports 2773 * during the reset operation. 2774 */ 2775 static int 2776 ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp) 2777 { 2778 ahci_port_t *ahci_portp; 2779 uint32_t slot_status[AHCI_MAX_PORTS]; 2780 uint32_t reset_tags[AHCI_MAX_PORTS]; 2781 uint32_t finished_tags[AHCI_MAX_PORTS]; 2782 int port; 2783 int ret = SATA_SUCCESS; 2784 2785 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2786 "ahci_reset_hba_reject_pkts enter", NULL); 2787 2788 bzero(slot_status, sizeof (slot_status)); 2789 bzero(reset_tags, sizeof (reset_tags)); 2790 bzero(finished_tags, sizeof (finished_tags)); 2791 2792 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2793 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2794 continue; 2795 } 2796 2797 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2798 2799 mutex_enter(&ahci_portp->ahciport_mutex); 2800 ahci_portp->ahciport_reset_in_progress = 1; 2801 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2802 slot_status[port] = ddi_get32( 2803 ahci_ctlp->ahcictl_ahci_acc_handle, 2804 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 2805 reset_tags[port] = slot_status[port] & 2806 AHCI_SLOT_MASK(ahci_ctlp); 2807 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2808 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 2809 port, reset_tags[port], 2810 ahci_portp->ahciport_pending_tags); 2811 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 2812 slot_status[port] = ddi_get32( 2813 ahci_ctlp->ahcictl_ahci_acc_handle, 2814 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 2815 reset_tags[port] = slot_status[port] & 2816 AHCI_NCQ_SLOT_MASK(ahci_portp); 2817 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 2818 "port %d: reset_tags = 0x%x pending_tags = 0x%x", 2819 port, reset_tags[port], 2820 ahci_portp->ahciport_pending_tags); 2821 } 2822 mutex_exit(&ahci_portp->ahciport_mutex); 2823 } 2824 2825 if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) { 2826 ret = SATA_FAILURE; 2827 } 2828 2829 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 2830 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 2831 continue; 2832 } 2833 2834 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2835 2836 mutex_enter(&ahci_portp->ahciport_mutex); 2837 /* 2838 * To prevent recursive enter to ahci_mop_commands, we need 2839 * check AHCI_PORT_FLAG_MOPPING flag. 2840 */ 2841 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 2842 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2843 "ahci_reset_hba_reject_pkts: port %d is in " 2844 "mopping process, so return directly ", port); 2845 mutex_exit(&ahci_portp->ahciport_mutex); 2846 continue; 2847 } 2848 2849 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 2850 ahci_portp->ahciport_mop_in_progress++; 2851 2852 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 2853 finished_tags[port] = 2854 ahci_portp->ahciport_pending_tags & 2855 ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp); 2856 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 2857 finished_tags[port] = 2858 ahci_portp->ahciport_pending_ncq_tags & 2859 ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp); 2860 2861 reset_tags[port] &= ~finished_tags[port]; 2862 2863 ahci_mop_commands(ahci_ctlp, 2864 ahci_portp, 2865 slot_status[port], 2866 0, /* failed tags */ 2867 0, /* timeout tags */ 2868 0, /* aborted tags */ 2869 reset_tags[port]); /* reset tags */ 2870 mutex_exit(&ahci_portp->ahciport_mutex); 2871 } 2872 out: 2873 return (ret); 2874 } 2875 2876 /* 2877 * Called by sata framework to reset a port(s) or device. 2878 */ 2879 static int 2880 ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd) 2881 { 2882 ahci_ctl_t *ahci_ctlp; 2883 ahci_port_t *ahci_portp; 2884 ahci_addr_t addr; 2885 uint8_t cport = sd->satadev_addr.cport; 2886 uint8_t pmport = sd->satadev_addr.pmport; 2887 uint8_t port; 2888 int ret = SATA_SUCCESS; 2889 int instance = ddi_get_instance(dip); 2890 2891 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 2892 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 2893 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2894 2895 ahci_get_ahci_addr(ahci_ctlp, sd, &addr); 2896 2897 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 2898 "ahci_tran_reset_dport enter: cport %d", cport); 2899 2900 switch (sd->satadev_addr.qual) { 2901 case SATA_ADDR_PMPORT: 2902 /* 2903 * If we want to issue a COMRESET on a pmport, we need to 2904 * reject the outstanding commands on that pmport. According 2905 * to AHCI spec, PxCI register could only be cleared by 2906 * clearing PxCMD.ST, which will halt the controller port - as 2907 * well as other pmports. 2908 * 2909 * Therefore we directly reset the controller port for 2910 * simplicity. ahci_tran_probe_port() will handle reset stuff 2911 * like initializing the given pmport. 2912 */ 2913 /* FALLTHRU */ 2914 case SATA_ADDR_CPORT: 2915 /* Port reset */ 2916 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 2917 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2918 "port %d reset port", instance, port); 2919 2920 mutex_enter(&ahci_portp->ahciport_mutex); 2921 ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr); 2922 mutex_exit(&ahci_portp->ahciport_mutex); 2923 2924 break; 2925 2926 case SATA_ADDR_DPMPORT: 2927 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2928 "port %d:%d reset device", instance, port, pmport); 2929 /* FALLTHRU */ 2930 case SATA_ADDR_DCPORT: 2931 /* Device reset */ 2932 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT) 2933 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2934 "port %d reset device", instance, port); 2935 2936 mutex_enter(&ahci_portp->ahciport_mutex); 2937 /* 2938 * software reset request must be sent to SATA_PMULT_HOSTPORT 2939 * if target is a port multiplier: 2940 */ 2941 if (sd->satadev_addr.qual == SATA_ADDR_DCPORT && 2942 ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) 2943 AHCI_ADDR_SET_PMULT(&addr, port); 2944 2945 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED | 2946 ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN | 2947 ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) { 2948 /* 2949 * In case the targer driver would send the request 2950 * before sata framework can have the opportunity to 2951 * process those event reports. 2952 */ 2953 sd->satadev_state = ahci_portp->ahciport_port_state; 2954 ahci_update_sata_registers(ahci_ctlp, port, sd); 2955 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2956 "ahci_tran_reset_dport returning SATA_FAILURE " 2957 "while port in FAILED/SHUTDOWN/PWROFF state: " 2958 "port: %d", port); 2959 mutex_exit(&ahci_portp->ahciport_mutex); 2960 ret = SATA_FAILURE; 2961 break; 2962 } 2963 2964 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == 2965 SATA_DTYPE_NONE) { 2966 /* 2967 * ahci_intr_phyrdy_change() may have rendered it to 2968 * AHCI_PORT_TYPE_NODEV. 2969 */ 2970 sd->satadev_type = SATA_DTYPE_NONE; 2971 sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp, 2972 &addr); 2973 ahci_update_sata_registers(ahci_ctlp, port, sd); 2974 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 2975 "ahci_tran_reset_dport returning SATA_FAILURE " 2976 "while no device attached: port: %d", port); 2977 mutex_exit(&ahci_portp->ahciport_mutex); 2978 ret = SATA_FAILURE; 2979 break; 2980 } 2981 2982 if (AHCI_ADDR_IS_PORT(&addr)) { 2983 ret = ahci_reset_device_reject_pkts(ahci_ctlp, 2984 ahci_portp, &addr); 2985 } else { 2986 ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp, 2987 ahci_portp, &addr); 2988 } 2989 2990 mutex_exit(&ahci_portp->ahciport_mutex); 2991 break; 2992 2993 case SATA_ADDR_CNTRL: 2994 /* Reset the whole controller */ 2995 cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport " 2996 "reset the whole hba", instance); 2997 ret = ahci_reset_hba_reject_pkts(ahci_ctlp); 2998 break; 2999 3000 default: 3001 ret = SATA_FAILURE; 3002 } 3003 3004 return (ret); 3005 } 3006 3007 /* 3008 * Called by sata framework to activate a port as part of hotplug. 3009 * (cfgadm -c connect satax/y) 3010 * Support port multiplier. 3011 */ 3012 static int 3013 ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev) 3014 { 3015 ahci_ctl_t *ahci_ctlp; 3016 ahci_port_t *ahci_portp; 3017 ahci_addr_t addr; 3018 uint8_t cport = satadev->satadev_addr.cport; 3019 uint8_t pmport = satadev->satadev_addr.pmport; 3020 uint8_t port; 3021 int instance = ddi_get_instance(dip); 3022 3023 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3024 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3025 3026 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3027 "ahci_tran_hotplug_port_activate enter: cport %d", cport); 3028 3029 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3030 3031 mutex_enter(&ahci_portp->ahciport_mutex); 3032 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3033 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3034 3035 if (AHCI_ADDR_IS_PORT(&addr)) { 3036 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated", 3037 instance, port); 3038 3039 /* Enable the interrupts on the port */ 3040 ahci_enable_port_intrs(ahci_ctlp, port); 3041 3042 /* 3043 * Reset the port so that the PHY communication would be 3044 * re-established. But this reset is an internal operation 3045 * and the sata module doesn't need to know about it. 3046 * Moreover, the port with a device attached will be started 3047 * too. 3048 */ 3049 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, 3050 ahci_portp, port, 3051 AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, 3052 NULL); 3053 3054 /* 3055 * Need to check the link status and device status of the port 3056 * and consider raising power if the port was in D3 state 3057 */ 3058 ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON; 3059 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF; 3060 ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN; 3061 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3062 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated", 3063 instance, port, pmport); 3064 /* AHCI_ADDR_PMPORT */ 3065 AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON; 3066 AHCIPORT_PMSTATE(ahci_portp, &addr) &= 3067 ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN); 3068 } 3069 3070 satadev->satadev_state = ahci_portp->ahciport_port_state; 3071 3072 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3073 3074 mutex_exit(&ahci_portp->ahciport_mutex); 3075 return (SATA_SUCCESS); 3076 } 3077 3078 /* 3079 * Called by sata framework to deactivate a port as part of hotplug. 3080 * (cfgadm -c disconnect satax/y) 3081 * Support port multiplier. 3082 */ 3083 static int 3084 ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev) 3085 { 3086 ahci_ctl_t *ahci_ctlp; 3087 ahci_port_t *ahci_portp; 3088 ahci_addr_t addr; 3089 uint8_t cport = satadev->satadev_addr.cport; 3090 uint8_t pmport = satadev->satadev_addr.pmport; 3091 uint8_t port; 3092 uint32_t port_scontrol; 3093 int instance = ddi_get_instance(dip); 3094 3095 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 3096 port = ahci_ctlp->ahcictl_cport_to_port[cport]; 3097 3098 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 3099 "ahci_tran_hotplug_port_deactivate enter: cport %d", cport); 3100 3101 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3102 mutex_enter(&ahci_portp->ahciport_mutex); 3103 ahci_get_ahci_addr(ahci_ctlp, satadev, &addr); 3104 ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)); 3105 3106 if (AHCI_ADDR_IS_PORT(&addr)) { 3107 cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated", 3108 instance, port); 3109 3110 /* Disable the interrupts on the port */ 3111 ahci_disable_port_intrs(ahci_ctlp, port); 3112 3113 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 3114 3115 /* First to abort all the pending commands */ 3116 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3117 3118 /* Then stop the port */ 3119 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3120 ahci_portp, port); 3121 } 3122 3123 /* Next put the PHY offline */ 3124 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3125 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 3126 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE); 3127 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *) 3128 AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 3129 } else if (AHCI_ADDR_IS_PMPORT(&addr)) { 3130 cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated", 3131 instance, port, pmport); 3132 3133 ahci_disable_port_intrs(ahci_ctlp, port); 3134 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) 3135 != SATA_DTYPE_NONE) 3136 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 3137 3138 /* Re-enable the interrupts for the other pmports */ 3139 ahci_enable_port_intrs(ahci_ctlp, port); 3140 } 3141 3142 /* Update port state */ 3143 AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN); 3144 satadev->satadev_state = SATA_PSTATE_SHUTDOWN; 3145 3146 ahci_update_sata_registers(ahci_ctlp, port, satadev); 3147 3148 mutex_exit(&ahci_portp->ahciport_mutex); 3149 return (SATA_SUCCESS); 3150 } 3151 3152 /* 3153 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED 3154 * when a device is unplugged or a port is deactivated. 3155 * 3156 * WARNING!!! ahciport_mutex should be acquired before the function is called. 3157 */ 3158 static void 3159 ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp, 3160 ahci_port_t *ahci_portp, uint8_t port) 3161 { 3162 uint32_t slot_status = 0; 3163 uint32_t abort_tags = 0; 3164 3165 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 3166 "ahci_reject_all_abort_pkts at port: %d", port); 3167 3168 /* Read/write port multiplier command takes highest priority */ 3169 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 3170 slot_status = 0x1; 3171 abort_tags = 0x1; 3172 goto out; 3173 } 3174 3175 /* 3176 * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a 3177 * REQUEST SENSE command or READ LOG EXT command is delivered to HBA 3178 * to get the error data, if yes when the device is removed, the 3179 * command needs to be aborted too. 3180 */ 3181 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) { 3182 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 3183 slot_status = 0x1; 3184 abort_tags = 0x1; 3185 goto out; 3186 } else { 3187 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3188 "ahci_reject_all_abort_pkts return directly " 3189 "port %d no needs to reject any outstanding " 3190 "commands", port); 3191 return; 3192 } 3193 } 3194 3195 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3196 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3197 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 3198 abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 3199 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 3200 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3201 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 3202 abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 3203 } 3204 3205 out: 3206 /* No need to do mop when there is no outstanding commands */ 3207 if (slot_status != 0) { 3208 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 3209 ahci_portp->ahciport_mop_in_progress++; 3210 3211 ahci_mop_commands(ahci_ctlp, 3212 ahci_portp, 3213 slot_status, 3214 0, /* failed tags */ 3215 0, /* timeout tags */ 3216 abort_tags, /* aborting tags */ 3217 0); /* reset tags */ 3218 } 3219 } 3220 3221 #if defined(__lock_lint) 3222 static int 3223 ahci_selftest(dev_info_t *dip, sata_device_t *device) 3224 { 3225 return (SATA_SUCCESS); 3226 } 3227 #endif 3228 3229 /* 3230 * Allocate the ports structure, only called by ahci_attach 3231 */ 3232 static int 3233 ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp) 3234 { 3235 int port, cport = 0; 3236 3237 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3238 "ahci_alloc_ports_state enter", NULL); 3239 3240 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3241 3242 /* Allocate structures only for the implemented ports */ 3243 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3244 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3245 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3246 "hba port %d not implemented", port); 3247 continue; 3248 } 3249 3250 ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port; 3251 ahci_ctlp->ahcictl_port_to_cport[port] = 3252 (uint8_t)cport++; 3253 3254 if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) { 3255 goto err_out; 3256 } 3257 } 3258 3259 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3260 return (AHCI_SUCCESS); 3261 3262 err_out: 3263 for (port--; port >= 0; port--) { 3264 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3265 ahci_dealloc_port_state(ahci_ctlp, port); 3266 } 3267 } 3268 3269 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3270 return (AHCI_FAILURE); 3271 } 3272 3273 /* 3274 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach 3275 */ 3276 static void 3277 ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp) 3278 { 3279 int port; 3280 3281 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3282 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3283 /* if this port is implemented by the HBA */ 3284 if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) 3285 ahci_dealloc_port_state(ahci_ctlp, port); 3286 } 3287 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3288 } 3289 3290 /* 3291 * Drain the taskq. 3292 */ 3293 static void 3294 ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp) 3295 { 3296 ahci_port_t *ahci_portp; 3297 int port; 3298 3299 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3300 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3301 continue; 3302 } 3303 3304 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3305 3306 mutex_enter(&ahci_portp->ahciport_mutex); 3307 ddi_taskq_wait(ahci_portp->ahciport_event_taskq); 3308 mutex_exit(&ahci_portp->ahciport_mutex); 3309 } 3310 } 3311 3312 /* 3313 * Initialize the controller and all ports. And then try to start the ports 3314 * if there are devices attached. 3315 * 3316 * This routine can be called from three seperate cases: DDI_ATTACH, 3317 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from 3318 * other two cases; device signature probing are attempted only during 3319 * DDI_ATTACH case. 3320 * 3321 * WARNING!!! Disable the whole controller's interrupts before calling and 3322 * the interrupts will be enabled upon successfully return. 3323 */ 3324 static int 3325 ahci_initialize_controller(ahci_ctl_t *ahci_ctlp) 3326 { 3327 ahci_port_t *ahci_portp; 3328 ahci_addr_t addr; 3329 uint32_t ghc_control; 3330 int port; 3331 3332 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3333 "ahci_initialize_controller enter", NULL); 3334 3335 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3336 3337 /* 3338 * Indicate that system software is AHCI aware by setting 3339 * GHC.AE to 1 3340 */ 3341 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3342 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 3343 3344 ghc_control |= AHCI_HBA_GHC_AE; 3345 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3346 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), 3347 ghc_control); 3348 3349 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3350 3351 /* Initialize the implemented ports and structures */ 3352 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3353 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3354 continue; 3355 } 3356 3357 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3358 mutex_enter(&ahci_portp->ahciport_mutex); 3359 3360 /* 3361 * Ensure that the controller is not in the running state 3362 * by checking every implemented port's PxCMD register 3363 */ 3364 AHCI_ADDR_SET_PORT(&addr, (uint8_t)port); 3365 3366 if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr) 3367 != AHCI_SUCCESS) { 3368 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3369 "ahci_initialize_controller: failed to " 3370 "initialize port %d", port); 3371 /* 3372 * Set the port state to SATA_PSTATE_FAILED if 3373 * failed to initialize it. 3374 */ 3375 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 3376 } 3377 3378 mutex_exit(&ahci_portp->ahciport_mutex); 3379 } 3380 3381 /* Enable the whole controller interrupts */ 3382 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3383 ahci_enable_all_intrs(ahci_ctlp); 3384 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3385 3386 return (AHCI_SUCCESS); 3387 } 3388 3389 /* 3390 * Reverse of ahci_initialize_controller() 3391 * 3392 * We only need to stop the ports and disable the interrupt. 3393 */ 3394 static void 3395 ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp) 3396 { 3397 ahci_port_t *ahci_portp; 3398 int port; 3399 3400 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3401 "ahci_uninitialize_controller enter", NULL); 3402 3403 /* disable all the interrupts. */ 3404 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3405 ahci_disable_all_intrs(ahci_ctlp); 3406 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3407 3408 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 3409 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 3410 continue; 3411 } 3412 3413 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3414 3415 /* Stop the port by clearing PxCMD.ST */ 3416 mutex_enter(&ahci_portp->ahciport_mutex); 3417 3418 /* 3419 * Here we must disable the port interrupt because 3420 * ahci_disable_all_intrs only clear GHC.IE, and IS 3421 * register will be still set if PxIE is enabled. 3422 * When ahci shares one IRQ with other drivers, the 3423 * intr handler may claim the intr mistakenly. 3424 */ 3425 ahci_disable_port_intrs(ahci_ctlp, port); 3426 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3427 ahci_portp, port); 3428 mutex_exit(&ahci_portp->ahciport_mutex); 3429 } 3430 } 3431 3432 /* 3433 * ahci_alloc_pmult() 3434 * 1. Setting HBA port registers which are necessary for a port multiplier. 3435 * (Set PxCMD.PMA while PxCMD.ST is '0') 3436 * 2. Allocate ahci_pmult_info structure. 3437 * 3438 * NOTE: Must stop port before the function is called. 3439 * WARNING!!! ahciport_mutex should be acquired before the function is 3440 * called. 3441 */ 3442 static void 3443 ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3444 { 3445 uint32_t port_cmd_status; 3446 uint8_t port = ahci_portp->ahciport_port_num; 3447 3448 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3449 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3450 3451 /* The port must have been stopped before. */ 3452 ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST)); 3453 3454 if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) { 3455 /* set PMA bit */ 3456 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3457 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3458 port_cmd_status|AHCI_CMD_STATUS_PMA); 3459 3460 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3461 "ahci_alloc_pmult: " 3462 "PxCMD.PMA bit set at port %d.", port); 3463 } 3464 3465 /* Allocate port multiplier information structure */ 3466 if (ahci_portp->ahciport_pmult_info == NULL) { 3467 ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *) 3468 kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP); 3469 } 3470 3471 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 3472 } 3473 3474 /* 3475 * ahci_dealloc_pmult() 3476 * 1. Clearing related registers when a port multiplier is detached. 3477 * (Clear PxCMD.PMA while PxCMD.ST is '0') 3478 * 2. Deallocate ahci_pmult_info structure. 3479 * 3480 * NOTE: Must stop port before the function is called. 3481 * WARNING!!! ahciport_mutex should be acquired before the function is 3482 * called. 3483 */ 3484 static void 3485 ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 3486 { 3487 uint32_t port_cmd_status; 3488 uint8_t port = ahci_portp->ahciport_port_num; 3489 3490 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3491 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3492 3493 if (port_cmd_status & AHCI_CMD_STATUS_PMA) { 3494 /* Clear PMA bit */ 3495 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 3496 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 3497 (port_cmd_status & (~AHCI_CMD_STATUS_PMA))); 3498 3499 AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp, 3500 "ahci_dealloc_pmult: " 3501 "PxCMD.PMA bit cleared at port %d.", port); 3502 } 3503 3504 /* Release port multiplier information structure */ 3505 if (ahci_portp->ahciport_pmult_info != NULL) { 3506 kmem_free(ahci_portp->ahciport_pmult_info, 3507 sizeof (ahci_pmult_info_t)); 3508 ahci_portp->ahciport_pmult_info = NULL; 3509 } 3510 } 3511 3512 /* 3513 * The routine is to initialize a port. First put the port in NOTRunning 3514 * state, then enable port interrupt and clear Serror register. And under 3515 * AHCI_ATTACH case, find device signature and then try to start the port. 3516 * 3517 * Called by 3518 * 1. ahci_initialize_controller 3519 * 2. ahci_intr_phyrdy_change (hotplug) 3520 * 3521 * WARNING!!! ahciport_mutex should be acquired before the function is called. 3522 */ 3523 static int 3524 ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 3525 ahci_addr_t *addrp) 3526 { 3527 uint32_t port_sstatus, port_task_file, port_cmd_status; 3528 uint8_t port = addrp->aa_port; 3529 boolean_t pm_mode = B_TRUE; /* Power Management mode */ 3530 int ret; 3531 3532 ASSERT(mutex_owned(&ahci_portp->ahciport_mutex)); 3533 3534 /* AHCI_ADDR_PORT: We've no idea of the attached device here. */ 3535 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 3536 3537 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3538 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 3539 3540 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 3541 "ahci_initialize_port: port %d ", port); 3542 3543 /* 3544 * Check whether the port is in NotRunning state, if not, 3545 * put the port in NotRunning state 3546 */ 3547 if (port_cmd_status & 3548 (AHCI_CMD_STATUS_ST | 3549 AHCI_CMD_STATUS_CR | 3550 AHCI_CMD_STATUS_FRE | 3551 AHCI_CMD_STATUS_FR)) { 3552 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 3553 ahci_portp, port); 3554 } 3555 3556 /* Disable interrupt */ 3557 ahci_disable_port_intrs(ahci_ctlp, port); 3558 3559 /* Device is unknown at first */ 3560 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 3561 3562 /* Disable the interface power management */ 3563 ahci_disable_interface_pm(ahci_ctlp, port); 3564 3565 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3566 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 3567 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 3568 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 3569 3570 /* Check physcial link status */ 3571 if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM || 3572 SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM || 3573 3574 /* Check interface status */ 3575 port_task_file & AHCI_TFD_STS_BSY || 3576 port_task_file & AHCI_TFD_STS_DRQ || 3577 3578 /* Check whether port reset must be executed */ 3579 ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET) { 3580 3581 /* Something went wrong, we need do some reset things */ 3582 ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 3583 3584 /* Does port reset succeed on HBA port? */ 3585 if (ret != AHCI_SUCCESS) { 3586 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 3587 "ahci_initialize_port:" 3588 "port reset faild at port %d", port); 3589 return (AHCI_FAILURE); 3590 } 3591 3592 /* Is port failed? */ 3593 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 3594 SATA_PSTATE_FAILED) { 3595 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 3596 "ahci_initialize_port: port %d state 0x%x", 3597 port, ahci_portp->ahciport_port_state); 3598 return (AHCI_FAILURE); 3599 } 3600 } 3601 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 3602 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port); 3603 3604 /* 3605 * At the time being, only probe ports/devices and get the types of 3606 * attached devices during DDI_ATTACH. In fact, the device can be 3607 * changed during power state changes, but at the time being, we 3608 * don't support the situation. 3609 */ 3610 mutex_exit(&ahci_portp->ahciport_mutex); 3611 mutex_enter(&ahci_ctlp->ahcictl_mutex); 3612 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) 3613 pm_mode = B_FALSE; 3614 mutex_exit(&ahci_ctlp->ahcictl_mutex); 3615 mutex_enter(&ahci_portp->ahciport_mutex); 3616 3617 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) 3618 pm_mode = B_FALSE; 3619 3620 if (!pm_mode) { 3621 /* 3622 * Try to get the device signature if the port is 3623 * not empty. 3624 */ 3625 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 3626 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 3627 } else { 3628 3629 /* 3630 * During the resume, we need to set the PxCLB, PxCLBU, PxFB 3631 * and PxFBU registers in case these registers were cleared 3632 * during the suspend. 3633 */ 3634 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3635 "ahci_initialize_port: port %d " 3636 "reset the port during resume", port); 3637 (void) ahci_port_reset(ahci_ctlp, ahci_portp, addrp); 3638 3639 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3640 "ahci_initialize_port: port %d " 3641 "set PxCLB, PxCLBU, PxFB and PxFBU " 3642 "during resume", port); 3643 3644 /* Config Port Received FIS Base Address */ 3645 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 3646 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 3647 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 3648 3649 /* Config Port Command List Base Address */ 3650 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 3651 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 3652 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 3653 } 3654 3655 /* Return directly if no device connected */ 3656 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 3657 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3658 "No device connected to port %d", port); 3659 goto out; 3660 } 3661 3662 /* If this is a port multiplier, we need do some initialization */ 3663 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 3664 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 3665 "Port multiplier found at port %d", port); 3666 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 3667 } 3668 3669 /* Try to start the port */ 3670 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 3671 != AHCI_SUCCESS) { 3672 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3673 "failed to start port %d", port); 3674 return (AHCI_FAILURE); 3675 } 3676 out: 3677 /* Enable port interrupts */ 3678 ahci_enable_port_intrs(ahci_ctlp, port); 3679 3680 return (AHCI_SUCCESS); 3681 } 3682 3683 /* 3684 * Handle hardware defect, and check the capabilities. For example, 3685 * power management capabilty and MSI capability. 3686 */ 3687 static int 3688 ahci_config_space_init(ahci_ctl_t *ahci_ctlp) 3689 { 3690 ushort_t caps_ptr, cap_count, cap; 3691 #if AHCI_DEBUG 3692 ushort_t pmcap, pmcsr; 3693 ushort_t msimc; 3694 #endif 3695 uint8_t revision; 3696 3697 ahci_ctlp->ahcictl_venid = 3698 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3699 PCI_CONF_VENID); 3700 3701 ahci_ctlp->ahcictl_devid = 3702 pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3703 PCI_CONF_DEVID); 3704 3705 /* 3706 * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those 3707 * controllers with 0x00 revision id work on 4-byte aligned buffer, 3708 * which is a bug and was fixed after 0x00 revision id controllers. 3709 * 3710 * Moreover, VT8251 cannot use multiple command slots in the command 3711 * list for non-queued commands because the previous register content 3712 * of PxCI can be re-written in the register write, so a flag will be 3713 * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE. 3714 * 3715 * For VT8251, software reset also has the same defect as the below 3716 * AMD/ATI chipset. That is, software reset will get failed if 0xf 3717 * is filled in pmport field. Therefore, another software reset need 3718 * to be done with 0 filled in pmport field. 3719 */ 3720 if (ahci_ctlp->ahcictl_venid == VIA_VENID) { 3721 revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3722 PCI_CONF_REVID); 3723 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3724 "revision id = 0x%x", revision); 3725 if (revision == 0x00) { 3726 ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4; 3727 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3728 "change ddi_attr_align to 0x4", NULL); 3729 } 3730 3731 ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE; 3732 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3733 "VT8251 cannot use multiple command lists for " 3734 "non-queued commands", NULL); 3735 3736 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 3737 } 3738 3739 /* 3740 * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit 3741 * DMA addressing for communication memory descriptors though S64A bit 3742 * of CAP register declares it supports. Even though 64-bit DMA for 3743 * data buffer works on ASUS M2A-VM with newer BIOS, three other 3744 * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and 3745 * AHCI_CAP_COMMU_32BIT_DMA are set for this controller. 3746 * 3747 * Due to certain hardware issue, the chipset must do port reset during 3748 * initialization, otherwise, when retrieving device signature, 3749 * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag 3750 * need to set. 3751 * 3752 * For this chipset software reset will get failure if the pmport of 3753 * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port 3754 * multiplier is connected to the port. In order to fix the issue, 3755 * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software 3756 * reset got failure, the driver will try to do another software reset 3757 * with pmport 0. 3758 */ 3759 if (ahci_ctlp->ahcictl_venid == 0x1002 && 3760 ahci_ctlp->ahcictl_devid == 0x4380) { 3761 ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA; 3762 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3763 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 3764 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 3765 3766 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3767 "ATI SB600 cannot do 64-bit DMA for both data buffer and " 3768 "communication memory descriptors though CAP indicates " 3769 "support, so force it to use 32-bit DMA", NULL); 3770 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3771 "ATI SB600 need to do a port reset during initialization", 3772 NULL); 3773 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3774 "ATI SB600 will get software reset failure if pmport " 3775 "is set 0xf and no port multiplier is attached", NULL); 3776 } 3777 3778 /* 3779 * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same 3780 * vendor ID and device ID (0x1002,0x4391). 3781 * 3782 * SB700/750 AHCI chipset on some boards doesn't support 64-bit 3783 * DMA addressing for communication memory descriptors though S64A bit 3784 * of CAP register declares the support. However, it does support 3785 * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is 3786 * set for this controller. 3787 * 3788 * SB710 has the same initialization issue as SB600, so it also need 3789 * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it. 3790 * 3791 * SB700 also has the same issue about software reset, and thus 3792 * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed. 3793 */ 3794 if (ahci_ctlp->ahcictl_venid == 0x1002 && 3795 ahci_ctlp->ahcictl_devid == 0x4391) { 3796 ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA; 3797 ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET; 3798 ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT; 3799 3800 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3801 "ATI SB700/750 cannot do 64-bit DMA for communication " 3802 "memory descriptors though CAP indicates support, " 3803 "so force it to use 32-bit DMA", NULL); 3804 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3805 "ATI SB710 need to do a port reset during initialization", 3806 NULL); 3807 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 3808 "ATI SB700 will get software reset failure if pmport " 3809 "is set 0xf and no port multiplier is attached", NULL); 3810 } 3811 3812 /* 3813 * Check if capabilities list is supported and if so, 3814 * get initial capabilities pointer and clear bits 0,1. 3815 */ 3816 if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle, 3817 PCI_CONF_STAT) & PCI_STAT_CAP) { 3818 caps_ptr = P2ALIGN(pci_config_get8( 3819 ahci_ctlp->ahcictl_pci_conf_handle, 3820 PCI_CONF_CAP_PTR), 4); 3821 } else { 3822 caps_ptr = PCI_CAP_NEXT_PTR_NULL; 3823 } 3824 3825 /* 3826 * Walk capabilities if supported. 3827 */ 3828 for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) { 3829 3830 /* 3831 * Check that we haven't exceeded the maximum number of 3832 * capabilities and that the pointer is in a valid range. 3833 */ 3834 if (++cap_count > PCI_CAP_MAX_PTR) { 3835 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3836 "too many device capabilities", NULL); 3837 return (AHCI_FAILURE); 3838 } 3839 if (caps_ptr < PCI_CAP_PTR_OFF) { 3840 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 3841 "capabilities pointer 0x%x out of range", 3842 caps_ptr); 3843 return (AHCI_FAILURE); 3844 } 3845 3846 /* 3847 * Get next capability and check that it is valid. 3848 * For now, we only support power management. 3849 */ 3850 cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle, 3851 caps_ptr); 3852 switch (cap) { 3853 case PCI_CAP_ID_PM: 3854 3855 /* power management supported */ 3856 ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM; 3857 3858 /* Save PMCSR offset */ 3859 ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR; 3860 3861 #if AHCI_DEBUG 3862 pmcap = pci_config_get16( 3863 ahci_ctlp->ahcictl_pci_conf_handle, 3864 caps_ptr + PCI_PMCAP); 3865 pmcsr = pci_config_get16( 3866 ahci_ctlp->ahcictl_pci_conf_handle, 3867 ahci_ctlp->ahcictl_pmcsr_offset); 3868 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3869 "Power Management capability found PCI_PMCAP " 3870 "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr); 3871 if ((pmcap & 0x3) == 0x3) 3872 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3873 "PCI Power Management Interface " 3874 "spec 1.2 compliant", NULL); 3875 #endif 3876 break; 3877 3878 case PCI_CAP_ID_MSI: 3879 #if AHCI_DEBUG 3880 msimc = pci_config_get16( 3881 ahci_ctlp->ahcictl_pci_conf_handle, 3882 caps_ptr + PCI_MSI_CTRL); 3883 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3884 "Message Signaled Interrupt capability found " 3885 "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1); 3886 #endif 3887 AHCIDBG(AHCIDBG_MSI, ahci_ctlp, 3888 "MSI capability found", NULL); 3889 break; 3890 3891 case PCI_CAP_ID_PCIX: 3892 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3893 "PCI-X capability found", NULL); 3894 break; 3895 3896 case PCI_CAP_ID_PCI_E: 3897 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3898 "PCI Express capability found", NULL); 3899 break; 3900 3901 case PCI_CAP_ID_MSI_X: 3902 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3903 "MSI-X capability found", NULL); 3904 break; 3905 3906 case PCI_CAP_ID_SATA: 3907 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3908 "SATA capability found", NULL); 3909 break; 3910 3911 case PCI_CAP_ID_VS: 3912 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3913 "Vendor Specific capability found", NULL); 3914 break; 3915 3916 default: 3917 AHCIDBG(AHCIDBG_PM, ahci_ctlp, 3918 "unrecognized capability 0x%x", cap); 3919 break; 3920 } 3921 3922 /* 3923 * Get next capabilities pointer and clear bits 0,1. 3924 */ 3925 caps_ptr = P2ALIGN(pci_config_get8( 3926 ahci_ctlp->ahcictl_pci_conf_handle, 3927 (caps_ptr + PCI_CAP_NEXT_PTR)), 4); 3928 } 3929 3930 return (AHCI_SUCCESS); 3931 } 3932 3933 /* 3934 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE 3935 * PORTMULT command. SYNC & POLLING mode is used. 3936 * 3937 * WARNING!!! ahciport_mutex should be acquired before the function 3938 * is called. 3939 */ 3940 static int 3941 ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 3942 uint8_t regn, uint32_t *pregv, uint8_t type) 3943 { 3944 ahci_port_t *ahci_portp; 3945 ahci_addr_t pmult_addr; 3946 sata_pkt_t *spkt; 3947 sata_cmd_t *scmd; 3948 sata_device_t sata_device; 3949 uint8_t port = addrp->aa_port; 3950 uint8_t pmport = addrp->aa_pmport; 3951 uint8_t cport; 3952 uint32_t intr_mask; 3953 int rval; 3954 char portstr[10]; 3955 3956 SET_PORTSTR(portstr, addrp); 3957 cport = ahci_ctlp->ahcictl_port_to_cport[port]; 3958 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 3959 3960 ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp)); 3961 ASSERT(mutex_owned(&ahci_portp->ahciport_mutex)); 3962 3963 /* Check the existence of the port multiplier */ 3964 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) 3965 return (AHCI_FAILURE); 3966 3967 /* Request a READ/WRITE PORTMULT sata packet. */ 3968 bzero(&sata_device, sizeof (sata_device_t)); 3969 sata_device.satadev_addr.cport = cport; 3970 sata_device.satadev_addr.pmport = pmport; 3971 sata_device.satadev_addr.qual = SATA_ADDR_PMULT; 3972 sata_device.satadev_rev = SATA_DEVICE_REV; 3973 3974 /* 3975 * Make sure no command is outstanding here. All R/W PMULT requests 3976 * come from 3977 * 3978 * 1. ahci_attach() 3979 * The port should be empty. 3980 * 3981 * 2. ahci_tran_probe_port() 3982 * Any request from SATA framework (via ahci_tran_start) should be 3983 * rejected if R/W PMULT command is outstanding. 3984 * 3985 * If we are doing mopping, do not check those flags because no 3986 * command will be actually outstanding. 3987 * 3988 * If the port has been occupied by any other commands, the probe 3989 * function will return a SATA_RETRY. SATA framework will retry 3990 * later. 3991 */ 3992 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 3993 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 3994 "R/W PMULT failed: R/W PMULT in progress at port %d.", 3995 port, ahci_portp->ahciport_flags); 3996 return (AHCI_FAILURE); 3997 } 3998 3999 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && ( 4000 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 4001 NCQ_CMD_IN_PROGRESS(ahci_portp) || 4002 NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) { 4003 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4004 "R/W PMULT failed: port %d is occupied (flags 0x%x).", 4005 port, ahci_portp->ahciport_flags); 4006 return (AHCI_FAILURE); 4007 } 4008 4009 /* 4010 * The port multiplier is gone. This may happen when 4011 * 1. Cutting off the power of an enclosure. The device lose the power 4012 * before port multiplier. 4013 * 2. Disconnecting the port multiplier during hot-plugging a sub-drive. 4014 * 4015 * The issued command should be aborted and the following command 4016 * should not be continued. 4017 */ 4018 if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) { 4019 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4020 "READ/WRITE PMULT failed: " 4021 "port-mult is removed from port %d", port); 4022 return (AHCI_FAILURE); 4023 } 4024 4025 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT; 4026 4027 spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip, 4028 &sata_device, regn, *pregv, type); 4029 4030 /* 4031 * READ/WRITE PORTMULT command is intended to sent to the control port 4032 * of the port multiplier. 4033 */ 4034 AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port); 4035 4036 ahci_portp->ahciport_rdwr_pmult_pkt = spkt; 4037 4038 /* No interrupt here. Store the interrupt enable mask. */ 4039 intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4040 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 4041 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4042 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 4043 4044 rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt); 4045 4046 if (rval == AHCI_SUCCESS && 4047 spkt->satapkt_reason == SATA_PKT_COMPLETED) { 4048 if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) { 4049 scmd = &spkt->satapkt_cmd; 4050 *pregv = scmd->satacmd_lba_high_lsb << 24 | 4051 scmd->satacmd_lba_mid_lsb << 16 | 4052 scmd->satacmd_lba_low_lsb << 8 | 4053 scmd->satacmd_sec_count_lsb; 4054 } 4055 } else { 4056 /* Failed or not completed. */ 4057 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4058 "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s", 4059 type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write", 4060 AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr); 4061 rval = AHCI_FAILURE; 4062 } 4063 out: 4064 /* Restore the interrupt mask */ 4065 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4066 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask); 4067 4068 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT; 4069 ahci_portp->ahciport_rdwr_pmult_pkt = NULL; 4070 sata_free_rdwr_pmult_pkt(spkt); 4071 return (rval); 4072 } 4073 4074 static int 4075 ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4076 uint8_t regn, uint32_t *pregv) 4077 { 4078 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv, 4079 SATA_RDWR_PMULT_PKT_TYPE_READ); 4080 } 4081 4082 static int 4083 ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4084 uint8_t regn, uint32_t regv) 4085 { 4086 return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, ®v, 4087 SATA_RDWR_PMULT_PKT_TYPE_WRITE); 4088 } 4089 4090 #define READ_PMULT(addrp, r, pv, out) \ 4091 if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS) \ 4092 goto out; 4093 4094 #define WRITE_PMULT(addrp, r, v, out) \ 4095 if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS) \ 4096 goto out; 4097 4098 /* 4099 * Update sata registers on port multiplier, including GSCR/PSCR registers. 4100 * ahci_update_pmult_gscr() 4101 * ahci_update_pmult_pscr() 4102 * 4103 * WARNING!!! ahciport_mutex should be acquired before those functions 4104 * get called. 4105 */ 4106 static int 4107 ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4108 sata_pmult_gscr_t *sg) 4109 { 4110 READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err); 4111 READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err); 4112 READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err); 4113 READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err); 4114 4115 return (AHCI_SUCCESS); 4116 4117 err: /* R/W PMULT error */ 4118 return (AHCI_FAILURE); 4119 } 4120 4121 static int 4122 ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp, 4123 sata_device_t *sd) 4124 { 4125 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4126 4127 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err); 4128 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err); 4129 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err); 4130 READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err); 4131 4132 return (AHCI_SUCCESS); 4133 4134 err: /* R/W PMULT error */ 4135 return (AHCI_FAILURE); 4136 } 4137 4138 /* 4139 * ahci_initialize_pmult() 4140 * 4141 * Initialize a port multiplier, including 4142 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16) 4143 * 2. Redefine MASK register. (SATA Chap 16.?) 4144 * 4145 * WARNING!!! ahciport_mutex should be acquired before the function 4146 * is called. 4147 */ 4148 static int 4149 ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4150 ahci_addr_t *addrp, sata_device_t *sd) 4151 { 4152 sata_pmult_gscr_t sg; 4153 uint32_t gscr64; 4154 uint8_t port = addrp->aa_port; 4155 4156 ASSERT(mutex_owned(&ahci_portp->ahciport_mutex)); 4157 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4158 "[Initialize] Port-multiplier at port %d.", port); 4159 4160 /* 4161 * Enable features of port multiplier. Currently only 4162 * Asynchronous Notification is enabled. 4163 */ 4164 /* Check gscr64 for supported features. */ 4165 READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err); 4166 4167 if (gscr64 & SATA_PMULT_CAP_SNOTIF) { 4168 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4169 "port %d: Port Multiplier supports " 4170 "Asynchronous Notification.", port); 4171 4172 /* Write to gscr96 to enabled features */ 4173 WRITE_PMULT(addrp, SATA_PMULT_GSCR96, 4174 SATA_PMULT_CAP_SNOTIF, err); 4175 4176 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4177 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4178 AHCI_SNOTIF_CLEAR_ALL); 4179 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4180 "port %d: PMult PxSNTF cleared.", port); 4181 4182 } 4183 4184 /* 4185 * Now we need to update gscr33 register to enable hot-plug interrupt 4186 * for sub devices behind port multiplier. 4187 */ 4188 WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err); 4189 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4190 "port %d: gscr33 mask set to %x.", port, (0x1ffff)); 4191 4192 /* 4193 * Fetch the number of device ports of the port multiplier 4194 */ 4195 if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS) 4196 return (AHCI_FAILURE); 4197 4198 /* Register the port multiplier to SATA Framework. */ 4199 mutex_exit(&ahci_portp->ahciport_mutex); 4200 sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg); 4201 mutex_enter(&ahci_portp->ahciport_mutex); 4202 4203 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports = 4204 sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK; 4205 4206 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 4207 "port %d: pmult sub-port number updated to %x.", port, 4208 ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports); 4209 4210 /* Till now port-mult is successfully initialized */ 4211 ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT; 4212 return (AHCI_SUCCESS); 4213 4214 err: /* R/W PMULT error */ 4215 return (AHCI_FAILURE); 4216 } 4217 4218 /* 4219 * Initialize a port multiplier port. According to spec, firstly we need 4220 * issue a COMRESET, then a software reset to get its signature. 4221 * 4222 * NOTE: This function should only be called in ahci_probe_pmport() 4223 * WARNING!!! ahciport_mutex should be acquired before the function. 4224 */ 4225 static int 4226 ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4227 ahci_addr_t *addrp) 4228 { 4229 uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0; 4230 uint8_t port = addrp->aa_port; 4231 uint8_t pmport = addrp->aa_pmport; 4232 int ret = AHCI_FAILURE; 4233 4234 ASSERT(AHCI_ADDR_IS_PMPORT(addrp)); 4235 4236 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4237 "ahci_initialize_pmport: port %d:%d", port, pmport); 4238 4239 /* Check HBA port state */ 4240 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 4241 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4242 "ahci_initialize_pmport:" 4243 "port %d:%d Port Multiplier is failed.", 4244 port, pmport); 4245 return (AHCI_FAILURE); 4246 } 4247 4248 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) { 4249 return (AHCI_FAILURE); 4250 } 4251 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 4252 4253 /* Checking for outstanding commands */ 4254 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4255 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4256 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4257 reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4258 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 4259 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4260 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 4261 reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4262 } 4263 4264 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 4265 ahci_portp->ahciport_mop_in_progress++; 4266 4267 /* Clear status */ 4268 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN); 4269 4270 /* Firstly assume an unknown device */ 4271 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 4272 4273 ahci_disable_port_intrs(ahci_ctlp, port); 4274 4275 /* port reset is necessary for port multiplier port */ 4276 if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) { 4277 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp, 4278 "ahci_initialize_pmport:" 4279 "port reset failed at port %d:%d", 4280 port, pmport); 4281 goto out; 4282 } 4283 4284 /* Is port failed? */ 4285 if (AHCIPORT_GET_STATE(ahci_portp, addrp) & 4286 SATA_PSTATE_FAILED) { 4287 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4288 "ahci_initialize_pmport: port %d:%d failed. " 4289 "state = 0x%x", port, pmport, 4290 ahci_portp->ahciport_port_state); 4291 goto out; 4292 } 4293 4294 /* Is there any device attached? */ 4295 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp) 4296 == SATA_DTYPE_NONE) { 4297 /* Do not waste time on an empty port */ 4298 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 4299 "ahci_initialize_pmport: No device is found " 4300 "at port %d:%d", port, pmport); 4301 ret = AHCI_SUCCESS; 4302 goto out; 4303 } 4304 4305 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY); 4306 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4307 "port %d:%d is ready now.", port, pmport); 4308 4309 /* 4310 * Till now we can assure a device attached to that HBA port and work 4311 * correctly. Now try to get the device signature. This is an optional 4312 * step. If failed, unknown device is assumed, then SATA module will 4313 * continue to use IDENTIFY DEVICE to get the information of the 4314 * device. 4315 */ 4316 ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp); 4317 4318 ret = AHCI_SUCCESS; 4319 4320 out: 4321 /* Next try to mop the pending commands */ 4322 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) 4323 finished_tags = ahci_portp->ahciport_pending_tags & 4324 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 4325 else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) 4326 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 4327 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 4328 reset_tags &= ~finished_tags; 4329 4330 ahci_mop_commands(ahci_ctlp, 4331 ahci_portp, 4332 slot_status, 4333 0, /* failed tags */ 4334 0, /* timeout tags */ 4335 0, /* aborted tags */ 4336 reset_tags); /* reset tags */ 4337 4338 /* Clear PxSNTF register if supported. */ 4339 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 4340 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4341 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 4342 AHCI_SNOTIF_CLEAR_ALL); 4343 } 4344 4345 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 4346 ahci_enable_port_intrs(ahci_ctlp, port); 4347 return (ret); 4348 } 4349 4350 /* 4351 * ahci_probe_pmult() 4352 * 4353 * This function will be called to probe a port multiplier, which will 4354 * handle hotplug events on port multiplier ports. 4355 * 4356 * NOTE: Only called from ahci_tran_probe_port() 4357 * WARNING!!! ahciport_mutex should be acquired before the function is called. 4358 */ 4359 static int 4360 ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4361 ahci_addr_t *addrp) 4362 { 4363 sata_device_t sdevice; 4364 ahci_addr_t pmport_addr; 4365 uint32_t gscr32, port_hotplug_tags; 4366 uint32_t pmport_sstatus; 4367 int dev_exists_now = 0, dev_existed_previously = 0; 4368 uint8_t port = addrp->aa_port; 4369 int npmport; 4370 4371 /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */ 4372 READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err); 4373 port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp); 4374 4375 do { 4376 npmport = ddi_ffs(port_hotplug_tags) - 1; 4377 if (npmport == -1) 4378 /* no pending hot plug events. */ 4379 return (AHCI_SUCCESS); 4380 4381 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4382 "hot-plug event at port %d:%d", port, npmport); 4383 4384 AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport); 4385 4386 /* Check previous device at that port */ 4387 if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr) 4388 != SATA_DTYPE_NONE) 4389 dev_existed_previously = 1; 4390 4391 /* PxSStatus tells the presence of device. */ 4392 READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS, 4393 &pmport_sstatus, err); 4394 4395 if (SSTATUS_GET_DET(pmport_sstatus) == 4396 SSTATUS_DET_DEVPRE_PHYCOM) 4397 dev_exists_now = 1; 4398 4399 /* 4400 * Clear PxSERR is critical. The transition from 0 to 1 will 4401 * emit a FIS which generates an asynchronous notification 4402 * event at controller. If we fail to clear the PxSERR, the 4403 * Async Notif events will no longer be activated on this 4404 * pmport. 4405 */ 4406 WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR, 4407 AHCI_SERROR_CLEAR_ALL, err); 4408 4409 bzero((void *)&sdevice, sizeof (sata_device_t)); 4410 sdevice.satadev_addr.cport = ahci_ctlp-> 4411 ahcictl_port_to_cport[port]; 4412 sdevice.satadev_addr.qual = SATA_ADDR_PMPORT; 4413 sdevice.satadev_addr.pmport = (uint8_t)npmport; 4414 sdevice.satadev_state = SATA_PSTATE_PWRON; 4415 4416 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4417 "[Existence] %d -> %d", dev_existed_previously, 4418 dev_exists_now); 4419 4420 if (dev_exists_now) { 4421 if (dev_existed_previously) { 4422 /* Link (may) not change: Exist -> Exist * */ 4423 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 4424 "ahci_probe_pmult: port %d:%d " 4425 "device link lost/established", 4426 port, npmport); 4427 4428 mutex_exit(&ahci_portp->ahciport_mutex); 4429 sata_hba_event_notify( 4430 ahci_ctlp->ahcictl_sata_hba_tran-> 4431 sata_tran_hba_dip, 4432 &sdevice, 4433 SATA_EVNT_LINK_LOST| 4434 SATA_EVNT_LINK_ESTABLISHED); 4435 mutex_enter(&ahci_portp->ahciport_mutex); 4436 } else { 4437 /* Link change: None -> Exist */ 4438 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4439 "ahci_probe_pmult: port %d:%d " 4440 "device link established", port, npmport); 4441 4442 /* Clear port state */ 4443 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4444 SATA_STATE_UNKNOWN); 4445 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4446 "ahci_probe_pmult: port %d " 4447 "ahciport_port_state [Cleared].", port); 4448 4449 mutex_exit(&ahci_portp->ahciport_mutex); 4450 sata_hba_event_notify( 4451 ahci_ctlp->ahcictl_sata_hba_tran-> 4452 sata_tran_hba_dip, 4453 &sdevice, 4454 SATA_EVNT_LINK_ESTABLISHED); 4455 mutex_enter(&ahci_portp->ahciport_mutex); 4456 } 4457 } else { /* No device exists now */ 4458 if (dev_existed_previously) { 4459 4460 /* Link change: Exist -> None */ 4461 AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp, 4462 "ahci_probe_pmult: port %d:%d " 4463 "device link lost", port, npmport); 4464 4465 /* An existing device is lost. */ 4466 AHCIPORT_SET_STATE(ahci_portp, &pmport_addr, 4467 SATA_STATE_UNKNOWN); 4468 AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr, 4469 SATA_DTYPE_NONE); 4470 4471 mutex_exit(&ahci_portp->ahciport_mutex); 4472 sata_hba_event_notify( 4473 ahci_ctlp->ahcictl_sata_hba_tran-> 4474 sata_tran_hba_dip, 4475 &sdevice, 4476 SATA_EVNT_LINK_LOST); 4477 mutex_enter(&ahci_portp->ahciport_mutex); 4478 } 4479 } 4480 4481 CLEAR_BIT(port_hotplug_tags, npmport); 4482 } while (port_hotplug_tags != 0); 4483 4484 return (AHCI_SUCCESS); 4485 4486 err: /* R/W PMULT error */ 4487 return (AHCI_FAILURE); 4488 } 4489 4490 /* 4491 * Probe and initialize a port multiplier port. 4492 * A port multiplier port could only be initilaizer here. 4493 * 4494 * WARNING!!! ahcictl_mutex should be acquired before the function 4495 * is called. 4496 */ 4497 static int 4498 ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4499 ahci_addr_t *addrp, sata_device_t *sd) 4500 { 4501 uint32_t port_state; 4502 uint8_t port = addrp->aa_port; 4503 ahci_addr_t addr_pmult; 4504 4505 /* 4506 * Check the parent - port multiplier first. 4507 */ 4508 4509 /* 4510 * Parent port multiplier might have been removed. This event will be 4511 * ignored and failure. 4512 */ 4513 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 4514 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 4515 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4516 "ahci_tran_probe_port: " 4517 "parent device removed, ignore event.", NULL); 4518 4519 return (AHCI_FAILURE); 4520 } 4521 4522 /* The port is ready? */ 4523 port_state = ahci_portp->ahciport_port_state; 4524 if (!(port_state & SATA_STATE_READY)) { 4525 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4526 "ahci_tran_probe_port: " 4527 "parent port-mult is NOT ready.", NULL); 4528 4529 if (ahci_restart_port_wait_till_ready(ahci_ctlp, 4530 ahci_portp, port, AHCI_PORT_RESET, NULL) != 4531 AHCI_SUCCESS) { 4532 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 4533 "ahci_tran_probe_port: " 4534 "restart port-mult failed.", NULL); 4535 return (AHCI_FAILURE); 4536 } 4537 } 4538 4539 /* 4540 * If port-mult is restarted due to some reason, we need 4541 * re-initialized the PMult. 4542 */ 4543 if (!(port_state & SATA_DSTATE_PMULT_INIT)) { 4544 /* Initialize registers on a port multiplier */ 4545 AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port); 4546 if (ahci_initialize_pmult(ahci_ctlp, ahci_portp, 4547 &addr_pmult, sd) != AHCI_SUCCESS) 4548 return (AHCI_FAILURE); 4549 } 4550 4551 /* 4552 * Then we check the port-mult port 4553 */ 4554 /* Is this pmport initialized? */ 4555 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 4556 if (!(port_state & SATA_STATE_READY)) { 4557 4558 /* ahci_initialize_pmport() will set READY state */ 4559 if (ahci_initialize_pmport(ahci_ctlp, 4560 ahci_portp, addrp) != AHCI_SUCCESS) 4561 return (AHCI_FAILURE); 4562 } 4563 4564 return (AHCI_SUCCESS); 4565 } 4566 4567 /* 4568 * AHCI device reset ...; a single device on one of the ports is reset, 4569 * but the HBA and physical communication remain intact. This is the 4570 * least intrusive. 4571 * 4572 * When issuing a software reset sequence, there should not be other 4573 * commands in the command list, so we will first clear and then re-set 4574 * PxCMD.ST to clear PxCI. And before issuing the software reset, 4575 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be 4576 * cleared unless command list override (PxCMD.CLO) is supported. 4577 * 4578 * WARNING!!! ahciport_mutex should be acquired and PxCMD.FRE should be 4579 * set before the function is called. 4580 */ 4581 static int 4582 ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4583 ahci_addr_t *addrp) 4584 { 4585 ahci_fis_h2d_register_t *h2d_register_fisp; 4586 ahci_cmd_table_t *cmd_table; 4587 ahci_cmd_header_t *cmd_header; 4588 uint32_t port_cmd_status, port_cmd_issue, port_task_file; 4589 int slot, loop_count; 4590 uint8_t port = addrp->aa_port; 4591 uint8_t pmport = addrp->aa_pmport; 4592 int rval = AHCI_FAILURE; 4593 4594 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 4595 "port %d:%d device software resetting (FIS)", port, pmport); 4596 4597 /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */ 4598 if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 4599 port) != AHCI_SUCCESS) { 4600 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4601 "ahci_software_reset: cannot stop HBA port %d.", port); 4602 goto out; 4603 } 4604 4605 /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */ 4606 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4607 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 4608 4609 if (port_task_file & AHCI_TFD_STS_BSY || 4610 port_task_file & AHCI_TFD_STS_DRQ) { 4611 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) { 4612 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4613 "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), " 4614 "cannot issue a software reset.", port_task_file); 4615 goto out; 4616 } 4617 4618 /* 4619 * If HBA Support CLO, as Command List Override (CAP.SCLO is 4620 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in 4621 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ. 4622 */ 4623 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4624 "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL) 4625 4626 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4627 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4628 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4629 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4630 port_cmd_status|AHCI_CMD_STATUS_CLO); 4631 4632 /* Waiting till PxCMD.SCLO bit is cleared */ 4633 loop_count = 0; 4634 do { 4635 /* Wait for 10 millisec */ 4636 drv_usecwait(AHCI_10MS_USECS); 4637 4638 /* We are effectively timing out after 1 sec. */ 4639 if (loop_count++ > 100) { 4640 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4641 "SCLO time out. port %d is busy.", port); 4642 goto out; 4643 } 4644 4645 port_cmd_status = 4646 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4647 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4648 } while (port_cmd_status & AHCI_CMD_STATUS_CLO); 4649 4650 /* Re-check */ 4651 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4652 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 4653 if (port_task_file & AHCI_TFD_STS_BSY || 4654 port_task_file & AHCI_TFD_STS_DRQ) { 4655 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4656 "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)", 4657 port_task_file); 4658 goto out; 4659 } 4660 } 4661 4662 /* Then start port */ 4663 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 4664 != AHCI_SUCCESS) { 4665 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4666 "ahci_software_reset: cannot start AHCI port %d.", port); 4667 goto out; 4668 } 4669 4670 /* 4671 * When ahci_port.ahciport_mop_in_progress is set, A non-zero 4672 * ahci_port.ahciport_pending_ncq_tags may fail 4673 * ahci_claim_free_slot(). Actually according to spec, by clearing 4674 * PxCMD.ST there is no command outstanding while executing software 4675 * reseting. Hence we directly use slot 0 instead of 4676 * ahci_claim_free_slot(). 4677 */ 4678 slot = 0; 4679 4680 /* Now send the first H2D Register FIS with SRST set to 1 */ 4681 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 4682 bzero((void *)cmd_table, ahci_cmd_table_size); 4683 4684 h2d_register_fisp = 4685 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 4686 4687 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 4688 SET_FIS_PMP(h2d_register_fisp, pmport); 4689 SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST); 4690 4691 /* Set Command Header in Command List */ 4692 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 4693 BZERO_DESCR_INFO(cmd_header); 4694 BZERO_PRD_BYTE_COUNT(cmd_header); 4695 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 4696 SET_PORT_MULTI_PORT(cmd_header, pmport); 4697 4698 SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1); 4699 SET_RESET(cmd_header, 1); 4700 SET_WRITE(cmd_header, 1); 4701 4702 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4703 0, 4704 ahci_cmd_table_size, 4705 DDI_DMA_SYNC_FORDEV); 4706 4707 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 4708 slot * sizeof (ahci_cmd_header_t), 4709 sizeof (ahci_cmd_header_t), 4710 DDI_DMA_SYNC_FORDEV); 4711 4712 /* Indicate to the HBA that a command is active. */ 4713 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4714 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 4715 (0x1 << slot)); 4716 4717 loop_count = 0; 4718 4719 /* Loop till the first command is finished */ 4720 do { 4721 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4722 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4723 4724 /* We are effectively timing out after 1 sec. */ 4725 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 4726 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4727 "the first SRST FIS is timed out, " 4728 "loop_count = %d", loop_count); 4729 goto out; 4730 } 4731 /* Wait for 10 millisec */ 4732 drv_usecwait(AHCI_10MS_USECS); 4733 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 4734 4735 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 4736 "ahci_software_reset: 1st loop count: %d, " 4737 "port_cmd_issue = 0x%x, slot = 0x%x", 4738 loop_count, port_cmd_issue, slot); 4739 4740 /* According to ATA spec, we need wait at least 5 microsecs here. */ 4741 drv_usecwait(AHCI_1MS_USECS); 4742 4743 /* Now send the second H2D Register FIS with SRST cleard to zero */ 4744 cmd_table = ahci_portp->ahciport_cmd_tables[slot]; 4745 bzero((void *)cmd_table, ahci_cmd_table_size); 4746 4747 h2d_register_fisp = 4748 &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register); 4749 4750 SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE); 4751 SET_FIS_PMP(h2d_register_fisp, pmport); 4752 4753 /* Set Command Header in Command List */ 4754 cmd_header = &ahci_portp->ahciport_cmd_list[slot]; 4755 BZERO_DESCR_INFO(cmd_header); 4756 BZERO_PRD_BYTE_COUNT(cmd_header); 4757 SET_COMMAND_FIS_LENGTH(cmd_header, 5); 4758 SET_PORT_MULTI_PORT(cmd_header, pmport); 4759 4760 SET_WRITE(cmd_header, 1); 4761 4762 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot], 4763 0, 4764 ahci_cmd_table_size, 4765 DDI_DMA_SYNC_FORDEV); 4766 4767 (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle, 4768 slot * sizeof (ahci_cmd_header_t), 4769 sizeof (ahci_cmd_header_t), 4770 DDI_DMA_SYNC_FORDEV); 4771 4772 /* Indicate to the HBA that a command is active. */ 4773 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4774 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port), 4775 (0x1 << slot)); 4776 4777 loop_count = 0; 4778 4779 /* Loop till the second command is finished */ 4780 do { 4781 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4782 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 4783 4784 /* We are effectively timing out after 1 sec. */ 4785 if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) { 4786 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4787 "the second SRST FIS is timed out, " 4788 "loop_count = %d", loop_count); 4789 goto out; 4790 } 4791 4792 /* Wait for 10 millisec */ 4793 drv_usecwait(AHCI_10MS_USECS); 4794 } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot)); 4795 4796 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 4797 "ahci_software_reset: 2nd loop count: %d, " 4798 "port_cmd_issue = 0x%x, slot = 0x%x", 4799 loop_count, port_cmd_issue, slot); 4800 4801 rval = AHCI_SUCCESS; 4802 out: 4803 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4804 "ahci_software_reset: %s at port %d:%d", 4805 rval == AHCI_SUCCESS ? "succeed" : "failed", 4806 port, pmport); 4807 4808 return (rval); 4809 } 4810 4811 /* 4812 * AHCI port reset ...; the physical communication between the HBA and device 4813 * on a port are disabled. This is more intrusive. 4814 * 4815 * When an HBA or port reset occurs, Phy communication is going to 4816 * be re-established with the device through a COMRESET followed by the 4817 * normal out-of-band communication sequence defined in Serial ATA. AT 4818 * the end of reset, the device, if working properly, will send a D2H 4819 * Register FIS, which contains the device signature. When the HBA receives 4820 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates 4821 * the PxSIG register with the signature. 4822 * 4823 * Staggered spin-up is an optional feature in SATA II, and it enables an HBA 4824 * to individually spin-up attached devices. Please refer to chapter 10.9 of 4825 * AHCI 1.0 spec. 4826 */ 4827 /* 4828 * WARNING!!! ahciport_mutex should be acquired, and PxCMD.ST should be also 4829 * cleared before the function is called. 4830 */ 4831 static int 4832 ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 4833 ahci_addr_t *addrp) 4834 { 4835 ahci_addr_t pmult_addr; 4836 uint32_t cap_status, port_cmd_status; 4837 uint32_t port_scontrol, port_sstatus, port_serror; 4838 uint32_t port_intr_status, port_task_file; 4839 uint32_t port_state; 4840 uint8_t port = addrp->aa_port; 4841 4842 int loop_count; 4843 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 4844 4845 /* Target is a port multiplier port? */ 4846 if (AHCI_ADDR_IS_PMPORT(addrp)) 4847 return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp)); 4848 4849 /* Otherwise it must be an HBA port. */ 4850 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 4851 4852 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 4853 "Port %d port resetting...", port); 4854 ahci_portp->ahciport_port_state = 0; 4855 4856 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4857 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 4858 4859 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4860 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4861 4862 if (cap_status & AHCI_HBA_CAP_SSS) { 4863 /* 4864 * HBA support staggered spin-up, if the port has 4865 * not spin up yet, then force it to do spin-up 4866 */ 4867 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) { 4868 if (!(ahci_portp->ahciport_flags 4869 & AHCI_PORT_FLAG_SPINUP)) { 4870 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4871 "Port %d PxCMD.SUD is zero, force " 4872 "it to do spin-up", port); 4873 ahci_portp->ahciport_flags |= 4874 AHCI_PORT_FLAG_SPINUP; 4875 } 4876 } 4877 } else { 4878 /* 4879 * HBA doesn't support stagger spin-up, force it 4880 * to do normal COMRESET 4881 */ 4882 if (ahci_portp->ahciport_flags & 4883 AHCI_PORT_FLAG_SPINUP) { 4884 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 4885 "HBA does not support staggered spin-up " 4886 "force it to do normal COMRESET", NULL); 4887 ahci_portp->ahciport_flags &= 4888 ~AHCI_PORT_FLAG_SPINUP; 4889 } 4890 } 4891 4892 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_SPINUP)) { 4893 /* Do normal COMRESET */ 4894 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 4895 "ahci_port_reset: do normal COMRESET", port); 4896 4897 /* 4898 * According to the spec, SUD bit should be set here, 4899 * but JMicron JMB363 doesn't follow it, so remove 4900 * the assertion, and just print a debug message. 4901 */ 4902 #if AHCI_DEBUG 4903 if (!(port_cmd_status & AHCI_CMD_STATUS_SUD)) 4904 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 4905 "port %d SUD bit not set", port) 4906 #endif 4907 4908 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4909 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 4910 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 4911 4912 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4913 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 4914 port_scontrol); 4915 4916 /* Enable PxCMD.FRE to read device */ 4917 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4918 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4919 port_cmd_status|AHCI_CMD_STATUS_FRE); 4920 4921 /* 4922 * Give time for COMRESET to percolate, according to the AHCI 4923 * spec, software shall wait at least 1 millisecond before 4924 * clearing PxSCTL.DET 4925 */ 4926 drv_usecwait(AHCI_1MS_USECS*2); 4927 4928 /* Fetch the SCONTROL again and rewrite the DET part with 0 */ 4929 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4930 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 4931 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 4932 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4933 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 4934 port_scontrol); 4935 } else { 4936 /* Do staggered spin-up */ 4937 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4938 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 4939 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 4940 4941 /* PxSCTL.DET must be 0 */ 4942 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4943 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), 4944 port_scontrol); 4945 4946 port_cmd_status &= ~AHCI_CMD_STATUS_SUD; 4947 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4948 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4949 port_cmd_status); 4950 4951 /* 0 -> 1 edge */ 4952 drv_usecwait(AHCI_1MS_USECS*2); 4953 4954 /* Set PxCMD.SUD to 1 */ 4955 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4956 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 4957 port_cmd_status |= AHCI_CMD_STATUS_SUD; 4958 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4959 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4960 port_cmd_status); 4961 4962 /* Enable PxCMD.FRE to read device */ 4963 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 4964 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 4965 port_cmd_status|AHCI_CMD_STATUS_FRE); 4966 } 4967 4968 /* 4969 * The port enters P:StartComm state, and HBA tells link layer to 4970 * start communication, which involves sending COMRESET to device. 4971 * And the HBA resets PxTFD.STS to 7Fh. 4972 * 4973 * When a COMINIT is received from the device, then the port enters 4974 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 4975 * PxSSTS.DET to 1h to indicate a device is detected but communication 4976 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 4977 * a COMINIT has been received. 4978 */ 4979 /* 4980 * The DET field is valid only if IPM field indicates 4981 * that the interface is in active state. 4982 */ 4983 loop_count = 0; 4984 do { 4985 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 4986 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 4987 4988 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 4989 /* 4990 * If the interface is not active, the DET field 4991 * is considered not accurate. So we want to 4992 * continue looping. 4993 */ 4994 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 4995 } 4996 4997 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 4998 /* 4999 * We are effectively timing out after 0.1 sec. 5000 */ 5001 break; 5002 } 5003 5004 /* Wait for 10 millisec */ 5005 drv_usecwait(AHCI_10MS_USECS); 5006 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 5007 5008 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5009 "ahci_port_reset: 1st loop count: %d, " 5010 "port_sstatus = 0x%x port %d", 5011 loop_count, port_sstatus, port); 5012 5013 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 5014 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 5015 /* 5016 * Either the port is not active or there 5017 * is no device present. 5018 */ 5019 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5020 return (AHCI_SUCCESS); 5021 } 5022 5023 /* Now we can make sure there is a device connected to the port */ 5024 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5025 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 5026 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5027 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 5028 5029 /* 5030 * A COMINIT signal is supposed to be received 5031 * PxSERR.DIAG.X or PxIS.PCS should be set 5032 */ 5033 if (!(port_intr_status & AHCI_INTR_STATUS_PCS) && 5034 !(port_serror & SERROR_EXCHANGED_ERR)) { 5035 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 5036 "COMINIT signal from the device not received", 5037 instance, port); 5038 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED); 5039 return (AHCI_FAILURE); 5040 } 5041 5042 /* 5043 * According to the spec, when PxSCTL.DET is set to 0h, upon 5044 * receiving a COMINIT from the attached device, PxTFD.STS.BSY 5045 * shall be set to '1' by the HBA. 5046 * 5047 * However, we found JMicron JMB363 doesn't follow this, so 5048 * remove this check, and just print a debug message. 5049 */ 5050 #if AHCI_DEBUG 5051 port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5052 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5053 if (!(port_task_file & AHCI_TFD_STS_BSY)) { 5054 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 5055 "port %d BSY bit is not set after COMINIT signal " 5056 "is received", port); 5057 } 5058 #endif 5059 5060 /* 5061 * PxSERR.DIAG.X has to be cleared in order to update PxTFD with 5062 * the D2H FIS received by HBA. 5063 */ 5064 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5065 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5066 SERROR_EXCHANGED_ERR); 5067 5068 /* 5069 * Devices should return a FIS contains its signature to HBA after 5070 * COMINIT signal. Check whether a D2H FIS is received by polling 5071 * PxTFD.STS.ERR bit. 5072 */ 5073 loop_count = 0; 5074 do { 5075 /* Wait for 10 millisec */ 5076 drv_usecwait(AHCI_10MS_USECS); 5077 5078 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5079 /* 5080 * We are effectively timing out after 11 sec. 5081 */ 5082 cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d " 5083 "the device hardware has been initialized and " 5084 "the power-up diagnostics failed", 5085 instance, port); 5086 5087 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: " 5088 "port %d PxTFD.STS.ERR is not set, we need another " 5089 "software reset.", port); 5090 5091 /* Clear port serror register for the port */ 5092 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5093 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5094 AHCI_SERROR_CLEAR_ALL); 5095 5096 AHCI_ADDR_SET_PMULT(&pmult_addr, port); 5097 5098 /* Try another software reset. */ 5099 if (ahci_software_reset(ahci_ctlp, ahci_portp, 5100 &pmult_addr) != AHCI_SUCCESS) { 5101 AHCIPORT_SET_STATE(ahci_portp, addrp, 5102 SATA_PSTATE_FAILED); 5103 return (AHCI_FAILURE); 5104 } 5105 break; 5106 } 5107 5108 /* Wait for 10 millisec */ 5109 drv_usecwait(AHCI_10MS_USECS); 5110 5111 /* 5112 * The Error bit '1' means COMRESET is finished successfully 5113 * The device hardware has been initialized and the power-up 5114 * diagnostics successfully completed. The device requests 5115 * that the Transport layer transmit a Register - D2H FIS to 5116 * the host. (SATA spec 11.5, v2.6) 5117 */ 5118 port_task_file = 5119 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5120 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 5121 } while (((port_task_file & AHCI_TFD_ERR_MASK) 5122 >> AHCI_TFD_ERR_SHIFT) != AHCI_TFD_ERR_SGS); 5123 5124 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 5125 "ahci_port_reset: 2nd loop count: %d, " 5126 "port_task_file = 0x%x port %d", 5127 loop_count, port_task_file, port); 5128 5129 /* Clear port serror register for the port */ 5130 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5131 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 5132 AHCI_SERROR_CLEAR_ALL); 5133 5134 /* Set port as ready */ 5135 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5136 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5137 5138 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5139 "ahci_port_reset: succeed at port %d.", port); 5140 return (AHCI_SUCCESS); 5141 } 5142 5143 /* 5144 * COMRESET on a port multiplier port. 5145 * 5146 * NOTE: Only called in ahci_port_reset() 5147 */ 5148 static int 5149 ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5150 ahci_addr_t *addrp) 5151 { 5152 uint32_t port_scontrol, port_sstatus, port_serror; 5153 uint32_t port_cmd_status, port_intr_status; 5154 uint32_t port_state; 5155 uint8_t port = addrp->aa_port; 5156 uint8_t pmport = addrp->aa_pmport; 5157 int loop_count; 5158 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 5159 5160 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 5161 "port %d:%d: pmport resetting", port, pmport); 5162 5163 /* Initialize pmport state */ 5164 AHCIPORT_SET_STATE(ahci_portp, addrp, 0); 5165 5166 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5167 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET); 5168 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5169 5170 /* PxCMD.FRE should be set before. */ 5171 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5172 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5173 ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE); 5174 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) 5175 return (AHCI_FAILURE); 5176 5177 /* 5178 * Give time for COMRESET to percolate, according to the AHCI 5179 * spec, software shall wait at least 1 millisecond before 5180 * clearing PxSCTL.DET 5181 */ 5182 drv_usecwait(AHCI_1MS_USECS*2); 5183 5184 /* 5185 * Fetch the SCONTROL again and rewrite the DET part with 0 5186 * This will generate an Asychronous Notification events. 5187 */ 5188 READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err); 5189 SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION); 5190 WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err); 5191 5192 /* 5193 * The port enters P:StartComm state, and HBA tells link layer to 5194 * start communication, which involves sending COMRESET to device. 5195 * And the HBA resets PxTFD.STS to 7Fh. 5196 * 5197 * When a COMINIT is received from the device, then the port enters 5198 * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets 5199 * PxSSTS.DET to 1h to indicate a device is detected but communication 5200 * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate 5201 * a COMINIT has been received. 5202 */ 5203 /* 5204 * The DET field is valid only if IPM field indicates 5205 * that the interface is in active state. 5206 */ 5207 loop_count = 0; 5208 do { 5209 READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err); 5210 5211 if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) { 5212 /* 5213 * If the interface is not active, the DET field 5214 * is considered not accurate. So we want to 5215 * continue looping. 5216 */ 5217 SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV); 5218 } 5219 5220 if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) { 5221 /* 5222 * We are effectively timing out after 0.1 sec. 5223 */ 5224 break; 5225 } 5226 5227 /* Wait for 10 millisec */ 5228 drv_usecwait(AHCI_10MS_USECS); 5229 5230 } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM); 5231 5232 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5233 "ahci_pmport_reset: 1st loop count: %d, " 5234 "port_sstatus = 0x%x port %d:%d", 5235 loop_count, port_sstatus, port, pmport); 5236 5237 if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) || 5238 (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) { 5239 /* 5240 * Either the port is not active or there 5241 * is no device present. 5242 */ 5243 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5244 "ahci_pmport_reset: " 5245 "no device attached to port %d:%d", 5246 port, pmport); 5247 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE); 5248 return (AHCI_SUCCESS); 5249 } 5250 5251 /* Now we can make sure there is a device connected to the port */ 5252 /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */ 5253 READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err); 5254 5255 if (!(port_serror & (1 << 26))) { 5256 cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: " 5257 "COMINIT signal from the device not received port %d:%d", 5258 instance, port, pmport); 5259 5260 AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED); 5261 return (AHCI_FAILURE); 5262 } 5263 5264 /* 5265 * After clear PxSERR register, we will receive a D2H FIS. 5266 * Normally this FIS will cause a IPMS error according to AHCI spec 5267 * v1.2 because there is no command outstanding for it. So we need 5268 * to ignore this error. 5269 */ 5270 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS; 5271 WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err); 5272 5273 /* Now we need to check the D2H FIS by checking IPMS error. */ 5274 loop_count = 0; 5275 do { 5276 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5277 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 5278 5279 if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) { 5280 /* 5281 * No D2H FIS received. This is possible according 5282 * to SATA 2.6 spec. 5283 */ 5284 cmn_err(CE_WARN, "ahci_port_reset: port %d:%d " 5285 "PxIS.IPMS is not set, we need another " 5286 "software reset.", port, pmport); 5287 5288 break; 5289 } 5290 5291 /* Wait for 10 millisec */ 5292 mutex_exit(&ahci_portp->ahciport_mutex); 5293 delay(AHCI_10MS_TICKS); 5294 mutex_enter(&ahci_portp->ahciport_mutex); 5295 5296 } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS)); 5297 5298 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5299 "ahci_pmport_reset: 2st loop count: %d, " 5300 "port_sstatus = 0x%x port %d:%d", 5301 loop_count, port_sstatus, port, pmport); 5302 5303 /* Clear IPMS */ 5304 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5305 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 5306 AHCI_INTR_STATUS_IPMS); 5307 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5308 5309 /* This pmport is now ready for ahci_tran_start() */ 5310 port_state = AHCIPORT_GET_STATE(ahci_portp, addrp); 5311 AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY); 5312 5313 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5314 "ahci_pmport_reset: succeed at port %d:%d", port, pmport); 5315 return (AHCI_SUCCESS); 5316 5317 err: /* R/W PMULT error */ 5318 /* IPMS flags might be set before. */ 5319 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS; 5320 AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp, 5321 "ahci_pmport_reset: failed at port %d:%d", port, pmport); 5322 5323 return (AHCI_FAILURE); 5324 } 5325 5326 /* 5327 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled. 5328 * This is the most intrusive. 5329 * 5330 * When an HBA reset occurs, Phy communication will be re-established with 5331 * the device through a COMRESET followed by the normal out-of-band 5332 * communication sequence defined in Serial ATA. AT the end of reset, the 5333 * device, if working properly, will send a D2H Register FIS, which contains 5334 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS 5335 * and PxTFD.ERR register fields, and updates the PxSIG register with the 5336 * signature. 5337 * 5338 * Remember to set GHC.AE to 1 before calling ahci_hba_reset. 5339 */ 5340 static int 5341 ahci_hba_reset(ahci_ctl_t *ahci_ctlp) 5342 { 5343 ahci_port_t *ahci_portp; 5344 ahci_addr_t addr; 5345 uint32_t ghc_control; 5346 uint8_t port; 5347 int loop_count; 5348 int rval = AHCI_SUCCESS; 5349 5350 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting", 5351 NULL); 5352 5353 mutex_enter(&ahci_ctlp->ahcictl_mutex); 5354 5355 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5356 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5357 5358 /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */ 5359 ghc_control |= AHCI_HBA_GHC_HR; 5360 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5361 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5362 5363 /* 5364 * Wait until HBA Reset complete or timeout 5365 */ 5366 loop_count = 0; 5367 do { 5368 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5369 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5370 5371 if (loop_count++ > AHCI_POLLRATE_HBA_RESET) { 5372 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5373 "ahci hba reset is timing out, " 5374 "ghc_control = 0x%x", ghc_control); 5375 /* We are effectively timing out after 1 sec. */ 5376 break; 5377 } 5378 5379 /* Wait for 10 millisec */ 5380 drv_usecwait(AHCI_10MS_USECS); 5381 } while (ghc_control & AHCI_HBA_GHC_HR); 5382 5383 AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp, 5384 "ahci_hba_reset: 1st loop count: %d, " 5385 "ghc_control = 0x%x", loop_count, ghc_control); 5386 5387 if (ghc_control & AHCI_HBA_GHC_HR) { 5388 /* The hba is not reset for some reasons */ 5389 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5390 "hba reset failed: HBA in a hung or locked state", NULL); 5391 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5392 return (AHCI_FAILURE); 5393 } 5394 5395 /* 5396 * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE 5397 */ 5398 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5399 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 5400 ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE); 5401 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5402 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 5403 5404 mutex_exit(&ahci_ctlp->ahcictl_mutex); 5405 5406 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 5407 /* Only check implemented ports */ 5408 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 5409 continue; 5410 } 5411 5412 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 5413 mutex_enter(&ahci_portp->ahciport_mutex); 5414 5415 AHCI_ADDR_SET_PORT(&addr, port); 5416 5417 if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 5418 port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) != 5419 AHCI_SUCCESS) { 5420 rval = AHCI_FAILURE; 5421 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5422 "ahci_hba_reset: port %d failed", port); 5423 /* 5424 * Set the port state to SATA_PSTATE_FAILED if 5425 * failed to initialize it. 5426 */ 5427 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5428 } 5429 5430 mutex_exit(&ahci_portp->ahciport_mutex); 5431 } 5432 5433 return (rval); 5434 } 5435 5436 /* 5437 * This routine is only called from AHCI_ATTACH or phyrdy change 5438 * case. It first calls software reset, then stop the port and try to 5439 * read PxSIG register to find the type of device attached to the port. 5440 * 5441 * The caller should make sure a valid device exists on specified port and 5442 * physical communication has been established so that the signature could 5443 * be retrieved by software reset. 5444 * 5445 * WARNING!!! ahciport_mutex should be acquired before the function 5446 * is called. And the port interrupt is disabled. 5447 */ 5448 static void 5449 ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5450 ahci_addr_t *addrp) 5451 { 5452 ahci_addr_t dev_addr; 5453 uint32_t signature; 5454 uint8_t port = addrp->aa_port; 5455 uint8_t pmport = addrp->aa_pmport; 5456 ASSERT(AHCI_ADDR_IS_VALID(addrp)); 5457 int rval; 5458 5459 if (AHCI_ADDR_IS_PORT(addrp)) { 5460 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5461 "ahci_find_dev_signature enter: port %d", port); 5462 5463 /* 5464 * NOTE: when the ahci address is a HBA port, we do not know 5465 * it is a device or a port multiplier that attached. we need 5466 * try a software reset at port multiplier address (0xf 5467 * pmport) 5468 */ 5469 AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port); 5470 } else { 5471 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 5472 "ahci_find_dev_signature enter: port %d:%d", 5473 port, pmport); 5474 dev_addr = *addrp; 5475 } 5476 5477 /* Assume it is unknown. */ 5478 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5479 5480 /* Issue a software reset to get the signature */ 5481 rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr); 5482 if (rval != AHCI_SUCCESS) { 5483 5484 /* 5485 * Try to do software reset again with pmport set with 0 if 5486 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and 5487 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf) 5488 */ 5489 if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) && 5490 (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) { 5491 dev_addr.aa_pmport = 0; 5492 rval = ahci_software_reset(ahci_ctlp, ahci_portp, 5493 &dev_addr); 5494 } 5495 5496 if (rval != AHCI_SUCCESS) { 5497 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 5498 "ahci_find_dev_signature: software reset failed " 5499 "at port %d:%d, cannot get signature.", 5500 port, pmport); 5501 5502 AHCIPORT_SET_STATE(ahci_portp, addrp, 5503 SATA_PSTATE_FAILED); 5504 return; 5505 } 5506 } 5507 5508 /* 5509 * ahci_software_reset has started the port, so we need manually stop 5510 * the port again. 5511 */ 5512 if (AHCI_ADDR_IS_PORT(addrp)) { 5513 if (ahci_put_port_into_notrunning_state(ahci_ctlp, 5514 ahci_portp, port) != AHCI_SUCCESS) { 5515 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5516 "ahci_find_dev_signature: cannot stop port %d.", 5517 port); 5518 ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED; 5519 return; 5520 } 5521 } 5522 5523 /* Now we can make sure that a valid signature is received. */ 5524 signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5525 (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port)); 5526 5527 #ifdef AHCI_DEBUG 5528 if (AHCI_ADDR_IS_PMPORT(addrp)) { 5529 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 5530 "ahci_find_dev_signature: signature = 0x%x at port %d:%d", 5531 signature, port, pmport); 5532 } else { 5533 AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp, 5534 "ahci_find_dev_signature: signature = 0x%x at port %d", 5535 signature, port); 5536 } 5537 #endif 5538 5539 /* NOTE: Only support ATAPI device at controller port. */ 5540 if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp)) 5541 signature = SATA_DTYPE_UNKNOWN; 5542 5543 switch (signature) { 5544 5545 case AHCI_SIGNATURE_DISK: 5546 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK); 5547 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5548 "Disk is found at port: %d", port); 5549 break; 5550 5551 case AHCI_SIGNATURE_ATAPI: 5552 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI); 5553 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5554 "ATAPI device is found at port: %d", port); 5555 break; 5556 5557 case AHCI_SIGNATURE_PORT_MULTIPLIER: 5558 /* Port Multiplier cannot recursively attached. */ 5559 ASSERT(AHCI_ADDR_IS_PORT(addrp)); 5560 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT); 5561 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5562 "Port Multiplier is found at port: %d", port); 5563 break; 5564 5565 default: 5566 AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN); 5567 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 5568 "Unknown device is found at port: %d", port); 5569 } 5570 } 5571 5572 /* 5573 * According to the spec, to reliably detect hot plug removals, software 5574 * must disable interface power management. Software should perform the 5575 * following initialization on a port after a device is attached: 5576 * Set PxSCTL.IPM to 3h to disable interface state transitions 5577 * Set PxCMD.ALPE to '0' to disable aggressive power management 5578 * Disable device initiated interface power management by SET FEATURE 5579 * 5580 * We can ignore the last item because by default the feature is disabled 5581 */ 5582 static void 5583 ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port) 5584 { 5585 uint32_t port_scontrol, port_cmd_status; 5586 5587 port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5588 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port)); 5589 SCONTROL_SET_IPM(port_scontrol, SCONTROL_IPM_DISABLE_BOTH); 5590 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5591 (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol); 5592 5593 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5594 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5595 port_cmd_status &= ~AHCI_CMD_STATUS_ALPE; 5596 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5597 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 5598 } 5599 5600 /* 5601 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set 5602 * to 1, then set it firstly. 5603 * 5604 * Each port contains two major DMA engines. One DMA engine walks through 5605 * the command list, and is controlled by PxCMD.ST. The second DMA engine 5606 * copies received FISes into system memory, and is controlled by PxCMD.FRE. 5607 * 5608 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR 5609 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear 5610 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'. 5611 * 5612 * Software shall not set PxCMD.ST to '1' unless a functional device is 5613 * present on the port(as determined by PxTFD.STS.BSY = '0', 5614 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h). 5615 * 5616 * WARNING!!! ahciport_mutex should be acquired before the function 5617 * is called. 5618 */ 5619 static int 5620 ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 5621 { 5622 uint32_t port_cmd_status; 5623 5624 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port); 5625 5626 if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) { 5627 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5628 "the state for port %d is 0x%x", 5629 port, ahci_portp->ahciport_port_state); 5630 return (AHCI_FAILURE); 5631 } 5632 5633 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 5634 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed " 5635 "no device is attached at port %d", port); 5636 return (AHCI_FAILURE); 5637 } 5638 5639 /* First to set PxCMD.FRE before setting PxCMD.ST. */ 5640 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5641 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5642 5643 if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) { 5644 port_cmd_status |= AHCI_CMD_STATUS_FRE; 5645 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5646 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5647 port_cmd_status); 5648 } 5649 5650 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 5651 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 5652 5653 port_cmd_status |= AHCI_CMD_STATUS_ST; 5654 5655 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 5656 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), 5657 port_cmd_status); 5658 5659 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED; 5660 5661 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: " 5662 "PxCMD.ST set to '1' at port %d", port); 5663 5664 return (AHCI_SUCCESS); 5665 } 5666 5667 /* 5668 * Allocate the ahci_port_t including Received FIS and Command List. 5669 * The argument - port is the physical port number, and not logical 5670 * port number seen by the SATA framework. 5671 * 5672 * WARNING!!! ahcictl_mutex should be acquired before the function 5673 * is called. 5674 */ 5675 static int 5676 ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 5677 { 5678 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 5679 ahci_port_t *ahci_portp; 5680 char taskq_name[64] = "event_handle_taskq"; 5681 5682 ahci_portp = 5683 (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP); 5684 5685 ahci_ctlp->ahcictl_ports[port] = ahci_portp; 5686 ahci_portp->ahciport_port_num = port; 5687 5688 /* Initialize the port condition variable */ 5689 cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL); 5690 5691 /* Initialize the port mutex */ 5692 mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER, 5693 (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri); 5694 5695 mutex_enter(&ahci_portp->ahciport_mutex); 5696 5697 /* 5698 * Allocate memory for received FIS structure and 5699 * command list for this port 5700 */ 5701 if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 5702 goto err_case1; 5703 } 5704 5705 if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp, port) != AHCI_SUCCESS) { 5706 goto err_case2; 5707 } 5708 5709 (void) snprintf(taskq_name + strlen(taskq_name), 5710 sizeof (taskq_name) - strlen(taskq_name), 5711 "_port%d", port); 5712 5713 /* Create the taskq for the port */ 5714 if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip, 5715 taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) { 5716 cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event " 5717 "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip)); 5718 goto err_case3; 5719 } 5720 5721 /* Allocate the argument for the taskq */ 5722 ahci_portp->ahciport_event_args = 5723 kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP); 5724 5725 ahci_portp->ahciport_event_args->ahciea_addrp = 5726 kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP); 5727 5728 if (ahci_portp->ahciport_event_args == NULL) 5729 goto err_case4; 5730 5731 mutex_exit(&ahci_portp->ahciport_mutex); 5732 5733 return (AHCI_SUCCESS); 5734 5735 err_case4: 5736 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 5737 5738 err_case3: 5739 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 5740 5741 err_case2: 5742 ahci_dealloc_rcvd_fis(ahci_portp); 5743 5744 err_case1: 5745 mutex_exit(&ahci_portp->ahciport_mutex); 5746 mutex_destroy(&ahci_portp->ahciport_mutex); 5747 cv_destroy(&ahci_portp->ahciport_cv); 5748 5749 kmem_free(ahci_portp, sizeof (ahci_port_t)); 5750 5751 return (AHCI_FAILURE); 5752 } 5753 5754 /* 5755 * Reverse of ahci_dealloc_port_state(). 5756 * 5757 * WARNING!!! ahcictl_mutex should be acquired before the function 5758 * is called. 5759 */ 5760 static void 5761 ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port) 5762 { 5763 ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port]; 5764 5765 ASSERT(ahci_portp != NULL); 5766 5767 mutex_enter(&ahci_portp->ahciport_mutex); 5768 kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp, 5769 sizeof (ahci_addr_t)); 5770 ahci_portp->ahciport_event_args->ahciea_addrp = NULL; 5771 kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t)); 5772 ahci_portp->ahciport_event_args = NULL; 5773 ddi_taskq_destroy(ahci_portp->ahciport_event_taskq); 5774 ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp); 5775 ahci_dealloc_rcvd_fis(ahci_portp); 5776 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 5777 mutex_exit(&ahci_portp->ahciport_mutex); 5778 5779 mutex_destroy(&ahci_portp->ahciport_mutex); 5780 cv_destroy(&ahci_portp->ahciport_cv); 5781 5782 kmem_free(ahci_portp, sizeof (ahci_port_t)); 5783 5784 ahci_ctlp->ahcictl_ports[port] = NULL; 5785 } 5786 5787 /* 5788 * Allocates memory for the Received FIS Structure 5789 * 5790 * WARNING!!! ahciport_mutex should be acquired before the function 5791 * is called. 5792 */ 5793 static int 5794 ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5795 uint8_t port) 5796 { 5797 size_t rcvd_fis_size; 5798 size_t ret_len; 5799 uint_t cookie_count; 5800 5801 rcvd_fis_size = sizeof (ahci_rcvd_fis_t); 5802 5803 /* allocate rcvd FIS dma handle. */ 5804 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 5805 &ahci_ctlp->ahcictl_rcvd_fis_dma_attr, 5806 DDI_DMA_SLEEP, 5807 NULL, 5808 &ahci_portp->ahciport_rcvd_fis_dma_handle) != 5809 DDI_SUCCESS) { 5810 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5811 "rcvd FIS dma handle alloc failed", NULL); 5812 5813 return (AHCI_FAILURE); 5814 } 5815 5816 if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle, 5817 rcvd_fis_size, 5818 &accattr, 5819 DDI_DMA_CONSISTENT, 5820 DDI_DMA_SLEEP, 5821 NULL, 5822 (caddr_t *)&ahci_portp->ahciport_rcvd_fis, 5823 &ret_len, 5824 &ahci_portp->ahciport_rcvd_fis_acc_handle) != NULL) { 5825 5826 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5827 "rcvd FIS dma mem alloc fail", NULL); 5828 /* error.. free the dma handle. */ 5829 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 5830 return (AHCI_FAILURE); 5831 } 5832 5833 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle, 5834 NULL, 5835 (caddr_t)ahci_portp->ahciport_rcvd_fis, 5836 rcvd_fis_size, 5837 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 5838 DDI_DMA_SLEEP, 5839 NULL, 5840 &ahci_portp->ahciport_rcvd_fis_dma_cookie, 5841 &cookie_count) != DDI_DMA_MAPPED) { 5842 5843 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5844 "rcvd FIS dma handle bind fail", NULL); 5845 /* error.. free the dma handle & free the memory. */ 5846 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 5847 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 5848 return (AHCI_FAILURE); 5849 } 5850 5851 bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size); 5852 5853 /* Config Port Received FIS Base Address */ 5854 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 5855 (uint64_t *)AHCI_PORT_PxFB(ahci_ctlp, port), 5856 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 5857 5858 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 5859 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress); 5860 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 5861 ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address); 5862 5863 return (AHCI_SUCCESS); 5864 } 5865 5866 /* 5867 * Deallocates the Received FIS Structure 5868 * 5869 * WARNING!!! ahciport_mutex should be acquired before the function 5870 * is called. 5871 */ 5872 static void 5873 ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp) 5874 { 5875 /* Unbind the cmd list dma handle first. */ 5876 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle); 5877 5878 /* Then free the underlying memory. */ 5879 ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle); 5880 5881 /* Now free the handle itself. */ 5882 ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle); 5883 } 5884 5885 /* 5886 * Allocates memory for the Command List, which contains up to 32 entries. 5887 * Each entry contains a command header, which is a 32-byte structure that 5888 * includes the pointer to the command table. 5889 * 5890 * WARNING!!! ahciport_mutex should be acquired before the function 5891 * is called. 5892 */ 5893 static int 5894 ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 5895 uint8_t port) 5896 { 5897 size_t cmd_list_size; 5898 size_t ret_len; 5899 uint_t cookie_count; 5900 5901 cmd_list_size = 5902 ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t); 5903 5904 /* allocate cmd list dma handle. */ 5905 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 5906 &ahci_ctlp->ahcictl_cmd_list_dma_attr, 5907 DDI_DMA_SLEEP, 5908 NULL, 5909 &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) { 5910 5911 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5912 "cmd list dma handle alloc failed", NULL); 5913 return (AHCI_FAILURE); 5914 } 5915 5916 if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle, 5917 cmd_list_size, 5918 &accattr, 5919 DDI_DMA_CONSISTENT, 5920 DDI_DMA_SLEEP, 5921 NULL, 5922 (caddr_t *)&ahci_portp->ahciport_cmd_list, 5923 &ret_len, 5924 &ahci_portp->ahciport_cmd_list_acc_handle) != NULL) { 5925 5926 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5927 "cmd list dma mem alloc fail", NULL); 5928 /* error.. free the dma handle. */ 5929 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5930 return (AHCI_FAILURE); 5931 } 5932 5933 if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle, 5934 NULL, 5935 (caddr_t)ahci_portp->ahciport_cmd_list, 5936 cmd_list_size, 5937 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 5938 DDI_DMA_SLEEP, 5939 NULL, 5940 &ahci_portp->ahciport_cmd_list_dma_cookie, 5941 &cookie_count) != DDI_DMA_MAPPED) { 5942 5943 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 5944 "cmd list dma handle bind fail", NULL); 5945 /* error.. free the dma handle & free the memory. */ 5946 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 5947 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5948 return (AHCI_FAILURE); 5949 } 5950 5951 bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size); 5952 5953 /* Config Port Command List Base Address */ 5954 ddi_put64(ahci_ctlp->ahcictl_ahci_acc_handle, 5955 (uint64_t *)AHCI_PORT_PxCLB(ahci_ctlp, port), 5956 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 5957 5958 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx", 5959 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress); 5960 5961 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x", 5962 ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address); 5963 5964 if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) { 5965 goto err_out; 5966 } 5967 5968 return (AHCI_SUCCESS); 5969 5970 err_out: 5971 /* Unbind the cmd list dma handle first. */ 5972 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 5973 5974 /* Then free the underlying memory. */ 5975 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 5976 5977 /* Now free the handle itself. */ 5978 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 5979 5980 return (AHCI_FAILURE); 5981 } 5982 5983 /* 5984 * Deallocates the Command List 5985 * 5986 * WARNING!!! ahciport_mutex should be acquired before the function 5987 * is called. 5988 */ 5989 static void 5990 ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 5991 { 5992 /* First dealloc command table */ 5993 ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp); 5994 5995 /* Unbind the cmd list dma handle first. */ 5996 (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle); 5997 5998 /* Then free the underlying memory. */ 5999 ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle); 6000 6001 /* Now free the handle itself. */ 6002 ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle); 6003 } 6004 6005 /* 6006 * Allocates memory for all Command Tables, which contains Command FIS, 6007 * ATAPI Command and Physical Region Descriptor Table. 6008 * 6009 * WARNING!!! ahciport_mutex should be acquired before the function 6010 * is called. 6011 */ 6012 static int 6013 ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6014 { 6015 size_t ret_len; 6016 ddi_dma_cookie_t cmd_table_dma_cookie; 6017 uint_t cookie_count; 6018 int slot; 6019 6020 AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, 6021 "ahci_alloc_cmd_tables: port %d enter", 6022 ahci_portp->ahciport_port_num); 6023 6024 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6025 /* Allocate cmd table dma handle. */ 6026 if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip, 6027 &ahci_ctlp->ahcictl_cmd_table_dma_attr, 6028 DDI_DMA_SLEEP, 6029 NULL, 6030 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) != 6031 DDI_SUCCESS) { 6032 6033 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6034 "cmd table dma handle alloc failed", NULL); 6035 6036 goto err_out; 6037 } 6038 6039 if (ddi_dma_mem_alloc( 6040 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6041 ahci_cmd_table_size, 6042 &accattr, 6043 DDI_DMA_CONSISTENT, 6044 DDI_DMA_SLEEP, 6045 NULL, 6046 (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot], 6047 &ret_len, 6048 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 6049 NULL) { 6050 6051 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6052 "cmd table dma mem alloc fail", NULL); 6053 6054 /* error.. free the dma handle. */ 6055 ddi_dma_free_handle( 6056 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6057 goto err_out; 6058 } 6059 6060 if (ddi_dma_addr_bind_handle( 6061 ahci_portp->ahciport_cmd_tables_dma_handle[slot], 6062 NULL, 6063 (caddr_t)ahci_portp->ahciport_cmd_tables[slot], 6064 ahci_cmd_table_size, 6065 DDI_DMA_RDWR | DDI_DMA_CONSISTENT, 6066 DDI_DMA_SLEEP, 6067 NULL, 6068 &cmd_table_dma_cookie, 6069 &cookie_count) != DDI_DMA_MAPPED) { 6070 6071 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 6072 "cmd table dma handle bind fail", NULL); 6073 /* error.. free the dma handle & free the memory. */ 6074 ddi_dma_mem_free( 6075 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6076 ddi_dma_free_handle( 6077 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6078 goto err_out; 6079 } 6080 6081 bzero((void *)ahci_portp->ahciport_cmd_tables[slot], 6082 ahci_cmd_table_size); 6083 6084 /* Config Port Command Table Base Address */ 6085 SET_COMMAND_TABLE_BASE_ADDR( 6086 (&ahci_portp->ahciport_cmd_list[slot]), 6087 cmd_table_dma_cookie.dmac_laddress & 0xffffffffull); 6088 6089 #ifndef __lock_lint 6090 SET_COMMAND_TABLE_BASE_ADDR_UPPER( 6091 (&ahci_portp->ahciport_cmd_list[slot]), 6092 cmd_table_dma_cookie.dmac_laddress >> 32); 6093 #endif 6094 } 6095 6096 return (AHCI_SUCCESS); 6097 err_out: 6098 6099 for (slot--; slot >= 0; slot--) { 6100 /* Unbind the cmd table dma handle first */ 6101 (void) ddi_dma_unbind_handle( 6102 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6103 6104 /* Then free the underlying memory */ 6105 ddi_dma_mem_free( 6106 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6107 6108 /* Now free the handle itself */ 6109 ddi_dma_free_handle( 6110 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6111 } 6112 6113 return (AHCI_FAILURE); 6114 } 6115 6116 /* 6117 * Deallocates memory for all Command Tables. 6118 * 6119 * WARNING!!! ahciport_mutex should be acquired before the function 6120 * is called. 6121 */ 6122 static void 6123 ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp) 6124 { 6125 int slot; 6126 6127 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 6128 "ahci_dealloc_cmd_tables: %d enter", 6129 ahci_portp->ahciport_port_num); 6130 6131 for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) { 6132 /* Unbind the cmd table dma handle first. */ 6133 (void) ddi_dma_unbind_handle( 6134 ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6135 6136 /* Then free the underlying memory. */ 6137 ddi_dma_mem_free( 6138 &ahci_portp->ahciport_cmd_tables_acc_handle[slot]); 6139 6140 /* Now free the handle itself. */ 6141 ddi_dma_free_handle( 6142 &ahci_portp->ahciport_cmd_tables_dma_handle[slot]); 6143 } 6144 } 6145 6146 /* 6147 * Update SATA registers at controller ports 6148 * 6149 * WARNING!!! ahciport_mutex should be acquired before those functions 6150 * get called. 6151 */ 6152 static void 6153 ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port, 6154 sata_device_t *sd) 6155 { 6156 sd->satadev_scr.sstatus = 6157 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6158 (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port))); 6159 sd->satadev_scr.serror = 6160 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6161 (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port))); 6162 sd->satadev_scr.scontrol = 6163 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6164 (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port))); 6165 sd->satadev_scr.sactive = 6166 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6167 (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port))); 6168 } 6169 6170 /* 6171 * For poll mode, ahci_port_intr will be called to emulate the interrupt 6172 */ 6173 static void 6174 ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port) 6175 { 6176 uint32_t port_intr_status; 6177 uint32_t port_intr_enable; 6178 6179 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6180 "ahci_port_intr enter: port %d", port); 6181 6182 mutex_enter(&ahci_portp->ahciport_mutex); 6183 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) { 6184 /* For SATA_OPMODE_POLLING commands */ 6185 port_intr_enable = 6186 (AHCI_INTR_STATUS_DHRS | 6187 AHCI_INTR_STATUS_PSS | 6188 AHCI_INTR_STATUS_SDBS | 6189 AHCI_INTR_STATUS_UFS | 6190 AHCI_INTR_STATUS_PCS | 6191 AHCI_INTR_STATUS_PRCS | 6192 AHCI_INTR_STATUS_OFS | 6193 AHCI_INTR_STATUS_INFS | 6194 AHCI_INTR_STATUS_IFS | 6195 AHCI_INTR_STATUS_HBDS | 6196 AHCI_INTR_STATUS_HBFS | 6197 AHCI_INTR_STATUS_TFES); 6198 } else { 6199 /* 6200 * port_intr_enable indicates that the corresponding interrrupt 6201 * reporting is enabled. 6202 */ 6203 port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6204 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port)); 6205 } 6206 6207 /* IPMS error in port reset should be ignored according AHCI spec. */ 6208 if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS)) 6209 port_intr_enable |= AHCI_INTR_STATUS_IPMS; 6210 mutex_exit(&ahci_portp->ahciport_mutex); 6211 6212 /* 6213 * port_intr_stats indicates that the corresponding interrupt 6214 * condition is active. 6215 */ 6216 port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6217 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port)); 6218 6219 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6220 "ahci_port_intr: port %d, port_intr_status = 0x%x, " 6221 "port_intr_enable = 0x%x", 6222 port, port_intr_status, port_intr_enable); 6223 6224 port_intr_status &= port_intr_enable; 6225 6226 /* First clear the port interrupts status */ 6227 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6228 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 6229 port_intr_status); 6230 6231 /* Check the completed non-queued commands */ 6232 if (port_intr_status & (AHCI_INTR_STATUS_DHRS | 6233 AHCI_INTR_STATUS_PSS)) { 6234 (void) ahci_intr_cmd_cmplt(ahci_ctlp, 6235 ahci_portp, port); 6236 } 6237 6238 /* Check the completed queued commands */ 6239 if (port_intr_status & AHCI_INTR_STATUS_SDBS) { 6240 (void) ahci_intr_set_device_bits(ahci_ctlp, 6241 ahci_portp, port); 6242 } 6243 6244 /* Check the port connect change status interrupt bit */ 6245 if (port_intr_status & AHCI_INTR_STATUS_PCS) { 6246 (void) ahci_intr_port_connect_change(ahci_ctlp, 6247 ahci_portp, port); 6248 } 6249 6250 /* Check the device mechanical presence status interrupt bit */ 6251 if (port_intr_status & AHCI_INTR_STATUS_DMPS) { 6252 (void) ahci_intr_device_mechanical_presence_status( 6253 ahci_ctlp, ahci_portp, port); 6254 } 6255 6256 /* Check the PhyRdy change status interrupt bit */ 6257 if (port_intr_status & AHCI_INTR_STATUS_PRCS) { 6258 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, 6259 port); 6260 } 6261 6262 /* 6263 * Check the non-fatal error interrupt bits, there are four 6264 * kinds of non-fatal errors at the time being: 6265 * 6266 * PxIS.UFS - Unknown FIS Error 6267 * PxIS.OFS - Overflow Error 6268 * PxIS.INFS - Interface Non-Fatal Error 6269 * PxIS.IPMS - Incorrect Port Multiplier Status Error 6270 * 6271 * For these non-fatal errors, the HBA can continue to operate, 6272 * so the driver just log the error messages. 6273 */ 6274 if (port_intr_status & (AHCI_INTR_STATUS_UFS | 6275 AHCI_INTR_STATUS_OFS | 6276 AHCI_INTR_STATUS_IPMS | 6277 AHCI_INTR_STATUS_INFS)) { 6278 (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp, 6279 port, port_intr_status); 6280 } 6281 6282 /* 6283 * Check the fatal error interrupt bits, there are four kinds 6284 * of fatal errors for AHCI controllers: 6285 * 6286 * PxIS.HBFS - Host Bus Fatal Error 6287 * PxIS.HBDS - Host Bus Data Error 6288 * PxIS.IFS - Interface Fatal Error 6289 * PxIS.TFES - Task File Error 6290 * 6291 * The fatal error means the HBA can not recover from it by 6292 * itself, and it will try to abort the transfer, and the software 6293 * must intervene to restart the port. 6294 */ 6295 if (port_intr_status & (AHCI_INTR_STATUS_IFS | 6296 AHCI_INTR_STATUS_HBDS | 6297 AHCI_INTR_STATUS_HBFS | 6298 AHCI_INTR_STATUS_TFES)) 6299 (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp, 6300 port, port_intr_status); 6301 6302 /* Check the cold port detect interrupt bit */ 6303 if (port_intr_status & AHCI_INTR_STATUS_CPDS) { 6304 (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port); 6305 } 6306 6307 /* Second clear the corresponding bit in IS.IPS */ 6308 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6309 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port)); 6310 } 6311 6312 /* 6313 * Interrupt service handler 6314 */ 6315 static uint_t 6316 ahci_intr(caddr_t arg1, caddr_t arg2) 6317 { 6318 #ifndef __lock_lint 6319 _NOTE(ARGUNUSED(arg2)) 6320 #endif 6321 /* LINTED */ 6322 ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1; 6323 ahci_port_t *ahci_portp; 6324 int32_t global_intr_status; 6325 uint8_t port; 6326 6327 /* 6328 * global_intr_status indicates that the corresponding port has 6329 * an interrupt pending. 6330 */ 6331 global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6332 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp)); 6333 6334 if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) { 6335 /* The interrupt is not ours */ 6336 return (DDI_INTR_UNCLAIMED); 6337 } 6338 6339 /* Loop for all the ports */ 6340 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 6341 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 6342 continue; 6343 } 6344 if (!((0x1 << port) & global_intr_status)) { 6345 continue; 6346 } 6347 6348 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 6349 6350 /* Call ahci_port_intr */ 6351 ahci_port_intr(ahci_ctlp, ahci_portp, port); 6352 } 6353 6354 return (DDI_INTR_CLAIMED); 6355 } 6356 6357 /* 6358 * For non-queued commands, when the corresponding bit in the PxCI register 6359 * is cleared, it means the command is completed successfully. And according 6360 * to the HBA state machine, there are three conditions which possibly will 6361 * try to clear the PxCI register bit. 6362 * 1. Receive one D2H Register FIS which is with 'I' bit set 6363 * 2. Update PIO Setup FIS 6364 * 3. Transmit a command and receive R_OK if CTBA.C is set (software reset) 6365 * 6366 * Process completed non-queued commands when the interrupt status bit - 6367 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set. 6368 * 6369 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received 6370 * with the 'I' bit set. And the following commands will send thus 6371 * FIS with 'I' bit set upon the successful completion: 6372 * 1. Non-data commands 6373 * 2. DMA data-in command 6374 * 3. DMA data-out command 6375 * 4. PIO data-out command 6376 * 5. PACKET non-data commands 6377 * 6. PACKET PIO data-in command 6378 * 7. PACKET PIO data-out command 6379 * 8. PACKET DMA data-in command 6380 * 9. PACKET DMA data-out command 6381 * 6382 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received 6383 * with the 'I' bit set. And the following commands will send this 6384 * FIS upon the successful completion: 6385 * 1. PIO data-in command 6386 */ 6387 static int 6388 ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp, 6389 ahci_port_t *ahci_portp, uint8_t port) 6390 { 6391 uint32_t port_cmd_issue = 0; 6392 uint32_t finished_tags; 6393 int finished_slot; 6394 sata_pkt_t *satapkt; 6395 ahci_fis_d2h_register_t *rcvd_fisp; 6396 #if AHCI_DEBUG 6397 ahci_cmd_header_t *cmd_header; 6398 uint32_t cmd_dmacount; 6399 #endif 6400 6401 mutex_enter(&ahci_portp->ahciport_mutex); 6402 6403 if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6404 !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6405 !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6406 /* 6407 * Spurious interrupt. Nothing to be done. 6408 */ 6409 mutex_exit(&ahci_portp->ahciport_mutex); 6410 return (AHCI_SUCCESS); 6411 } 6412 6413 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6414 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6415 6416 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 6417 /* Slot 0 is always used during error recovery */ 6418 finished_tags = 0x1 & ~port_cmd_issue; 6419 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 6420 "ahci_intr_cmd_cmplt: port %d the sata pkt for error " 6421 "retrieval is finished, and finished_tags = 0x%x", 6422 port, finished_tags); 6423 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 6424 finished_tags = 0x1 & ~port_cmd_issue; 6425 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, 6426 "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w " 6427 "port multiplier is finished, and finished_tags = 0x%x", 6428 port, finished_tags); 6429 6430 } else { 6431 6432 finished_tags = ahci_portp->ahciport_pending_tags & 6433 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 6434 } 6435 6436 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6437 "ahci_intr_cmd_cmplt: pending_tags = 0x%x, " 6438 "port_cmd_issue = 0x%x finished_tags = 0x%x", 6439 ahci_portp->ahciport_pending_tags, port_cmd_issue, 6440 finished_tags); 6441 6442 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) && 6443 (finished_tags == 0x1)) { 6444 satapkt = ahci_portp->ahciport_err_retri_pkt; 6445 ASSERT(satapkt != NULL); 6446 6447 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6448 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6449 "with SATA_PKT_COMPLETED", (void *)satapkt); 6450 6451 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6452 goto out; 6453 } 6454 6455 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) && 6456 (finished_tags == 0x1)) { 6457 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 6458 ASSERT(satapkt != NULL); 6459 6460 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6461 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6462 "with SATA_PKT_COMPLETED", (void *)satapkt); 6463 6464 /* READ PORTMULT need copy out FIS content. */ 6465 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6466 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6467 ahcirf_d2h_register_fis); 6468 satapkt->satapkt_cmd.satacmd_status_reg = 6469 GET_RFIS_STATUS(rcvd_fisp); 6470 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 6471 } 6472 6473 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6474 goto out; 6475 } 6476 6477 while (finished_tags) { 6478 finished_slot = ddi_ffs(finished_tags) - 1; 6479 if (finished_slot == -1) { 6480 goto out; 6481 } 6482 6483 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 6484 ASSERT(satapkt != NULL); 6485 #if AHCI_DEBUG 6486 /* 6487 * For non-native queued commands, the PRD byte count field 6488 * shall contain an accurate count of the number of bytes 6489 * transferred for the command before the PxCI bit is cleared 6490 * to '0' for the command. 6491 * 6492 * The purpose of this field is to let software know how many 6493 * bytes transferred for a given operation in order to 6494 * determine if underflow occurred. When issuing native command 6495 * queuing commands, this field should not be used and is not 6496 * required to be valid since in this case underflow is always 6497 * illegal. 6498 * 6499 * For data reads, the HBA will update its PRD byte count with 6500 * the total number of bytes received from the last FIS, and 6501 * may be able to continue normally. For data writes, the 6502 * device will detect an error, and HBA most likely will get 6503 * a fatal error. 6504 * 6505 * Therefore, here just put code to debug part. And please 6506 * refer to the comment above ahci_intr_fatal_error for the 6507 * definition of underflow error. 6508 */ 6509 cmd_dmacount = 6510 ahci_portp->ahciport_prd_bytecounts[finished_slot]; 6511 if (cmd_dmacount) { 6512 cmd_header = 6513 &ahci_portp->ahciport_cmd_list[finished_slot]; 6514 AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp, 6515 "ahci_intr_cmd_cmplt: port %d, " 6516 "PRD Byte Count = 0x%x, " 6517 "ahciport_prd_bytecounts = 0x%x", port, 6518 cmd_header->ahcich_prd_byte_count, 6519 cmd_dmacount); 6520 6521 if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) { 6522 AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp, 6523 "ahci_intr_cmd_cmplt: port %d, " 6524 "an underflow occurred", port); 6525 } 6526 } 6527 #endif 6528 6529 /* 6530 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS 6531 * feature, sata_special_regs flag will be set, and the 6532 * driver should copy the status and the other corresponding 6533 * register values in the D2H Register FIS received (It's 6534 * working on Non-data protocol) from the device back to 6535 * the sata_cmd. 6536 * 6537 * For every AHCI port, there is only one Received FIS 6538 * structure, which contains the FISes received from the 6539 * device, So we're trying to copy the content of D2H 6540 * Register FIS in the Received FIS structure back to 6541 * the sata_cmd. 6542 */ 6543 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 6544 rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 6545 ahcirf_d2h_register_fis); 6546 satapkt->satapkt_cmd.satacmd_status_reg = 6547 GET_RFIS_STATUS(rcvd_fisp); 6548 ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp); 6549 } 6550 6551 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6552 "ahci_intr_cmd_cmplt: sending up pkt 0x%p " 6553 "with SATA_PKT_COMPLETED", (void *)satapkt); 6554 6555 CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot); 6556 CLEAR_BIT(finished_tags, finished_slot); 6557 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 6558 6559 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6560 } 6561 out: 6562 AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp, 6563 "ahci_intr_cmd_cmplt: pending_tags = 0x%x", 6564 ahci_portp->ahciport_pending_tags); 6565 6566 mutex_exit(&ahci_portp->ahciport_mutex); 6567 6568 return (AHCI_SUCCESS); 6569 } 6570 6571 /* 6572 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received 6573 * with the 'I' bit set and has been copied into system memory. It will 6574 * be sent under the following situations: 6575 * 6576 * 1. NCQ command is completed 6577 * 6578 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed 6579 * via the Set Device Bits FIS. When such event is generated, the software 6580 * needs to read PxSACT register and compares the current value to the 6581 * list of commands previously issue by software. ahciport_pending_ncq_tags 6582 * keeps the tags of previously issued commands. 6583 * 6584 * 2. Asynchronous Notification 6585 * 6586 * Asynchronous Notification is a feature in SATA spec 2.6. 6587 * 6588 * 1) ATAPI device will send a signal to the host when media is inserted or 6589 * removed and avoids polling the device for media changes. The signal 6590 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits 6591 * set to '1'. At the moment, it's not supported yet. 6592 * 6593 * 2) Port multiplier will send a signal to the host when a hot plug event 6594 * has occured on a port multiplier port. It is used when command based 6595 * switching is employed. This is handled by ahci_intr_pmult_sntf_events() 6596 */ 6597 static int 6598 ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp, 6599 ahci_port_t *ahci_portp, uint8_t port) 6600 { 6601 ahci_addr_t addr; 6602 6603 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6604 "ahci_intr_set_device_bits enter: port %d", port); 6605 6606 /* Initialize HBA port address */ 6607 AHCI_ADDR_SET_PORT(&addr, port); 6608 6609 /* NCQ plug handler */ 6610 (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr); 6611 6612 /* Check port multiplier's asynchronous notification events */ 6613 if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) { 6614 (void) ahci_intr_pmult_sntf_events(ahci_ctlp, 6615 ahci_portp, port); 6616 } 6617 6618 /* ATAPI events is not supported yet */ 6619 6620 return (AHCI_SUCCESS); 6621 } 6622 /* 6623 * NCQ interrupt handler. Called upon a NCQ command is completed. 6624 * Only be called from ahci_intr_set_device_bits(). 6625 * 6626 * WARNING!!! ahciport_mutex should be acquired before the function is 6627 * called. 6628 */ 6629 static int 6630 ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp, 6631 ahci_port_t *ahci_portp, ahci_addr_t *addrp) 6632 { 6633 uint32_t port_sactive; 6634 uint32_t port_cmd_issue; 6635 uint32_t issued_tags; 6636 int issued_slot; 6637 uint32_t finished_tags; 6638 int finished_slot; 6639 uint8_t port = addrp->aa_port; 6640 sata_pkt_t *satapkt; 6641 6642 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6643 "ahci_intr_set_device_bits enter: port %d", port); 6644 6645 mutex_enter(&ahci_portp->ahciport_mutex); 6646 if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) { 6647 mutex_exit(&ahci_portp->ahciport_mutex); 6648 return (AHCI_SUCCESS); 6649 } 6650 6651 /* 6652 * First the handler got which commands are finished by checking 6653 * PxSACT register 6654 */ 6655 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6656 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 6657 6658 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 6659 ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp); 6660 6661 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6662 "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x " 6663 "port_sactive = 0x%x", port, 6664 ahci_portp->ahciport_pending_ncq_tags, port_sactive); 6665 6666 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6667 "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags); 6668 6669 /* 6670 * For NCQ commands, the software can determine which command has 6671 * already been transmitted to the device by checking PxCI register. 6672 */ 6673 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6674 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 6675 6676 issued_tags = ahci_portp->ahciport_pending_tags & 6677 ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp); 6678 6679 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6680 "ahci_intr_set_device_bits: port %d pending_tags = 0x%x " 6681 "port_cmd_issue = 0x%x", port, 6682 ahci_portp->ahciport_pending_tags, port_cmd_issue); 6683 6684 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6685 "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags); 6686 6687 /* 6688 * Clear ahciport_pending_tags bit when the corresponding command 6689 * is already sent down to the device. 6690 */ 6691 while (issued_tags) { 6692 issued_slot = ddi_ffs(issued_tags) - 1; 6693 if (issued_slot == -1) { 6694 goto next; 6695 } 6696 CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot); 6697 CLEAR_BIT(issued_tags, issued_slot); 6698 } 6699 6700 next: 6701 while (finished_tags) { 6702 finished_slot = ddi_ffs(finished_tags) - 1; 6703 if (finished_slot == -1) { 6704 goto out; 6705 } 6706 6707 /* The command is certainly transmitted to the device */ 6708 ASSERT(!(ahci_portp->ahciport_pending_tags & 6709 (0x1 << finished_slot))); 6710 6711 satapkt = ahci_portp->ahciport_slot_pkts[finished_slot]; 6712 ASSERT(satapkt != NULL); 6713 6714 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp, 6715 "ahci_intr_set_device_bits: sending up pkt 0x%p " 6716 "with SATA_PKT_COMPLETED", (void *)satapkt); 6717 6718 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot); 6719 CLEAR_BIT(finished_tags, finished_slot); 6720 ahci_portp->ahciport_slot_pkts[finished_slot] = NULL; 6721 6722 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 6723 } 6724 out: 6725 AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp, 6726 "ahci_intr_set_device_bits: port %d " 6727 "pending_ncq_tags = 0x%x pending_tags = 0x%x", 6728 port, ahci_portp->ahciport_pending_ncq_tags, 6729 ahci_portp->ahciport_pending_tags); 6730 6731 mutex_exit(&ahci_portp->ahciport_mutex); 6732 6733 return (AHCI_SUCCESS); 6734 } 6735 6736 /* 6737 * Port multiplier asynchronous notification event handler. Called upon a 6738 * device is hot plugged/pulled. 6739 * 6740 * The async-notification event will only be recorded by ahcipmi_snotif_tags 6741 * here and will be handled by ahci_probe_pmult(). 6742 * 6743 * NOTE: called only from ahci_port_intr(). 6744 */ 6745 static int 6746 ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 6747 uint8_t port) 6748 { 6749 sata_device_t sdevice; 6750 6751 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 6752 "ahci_intr_pmult_sntf_events enter: port %d ", port); 6753 6754 /* no hot-plug while attaching process */ 6755 mutex_enter(&ahci_ctlp->ahcictl_mutex); 6756 if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) { 6757 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6758 return (AHCI_SUCCESS); 6759 } 6760 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6761 6762 mutex_enter(&ahci_portp->ahciport_mutex); 6763 if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 6764 mutex_exit(&ahci_portp->ahciport_mutex); 6765 return (AHCI_SUCCESS); 6766 } 6767 6768 ASSERT(ahci_portp->ahciport_pmult_info != NULL); 6769 6770 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags = 6771 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6772 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port)); 6773 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6774 (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port), 6775 AHCI_SNOTIF_CLEAR_ALL); 6776 6777 if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) { 6778 mutex_exit(&ahci_portp->ahciport_mutex); 6779 return (AHCI_SUCCESS); 6780 } 6781 6782 /* Port Multiplier sub-device hot-plug handler */ 6783 if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 6784 mutex_exit(&ahci_portp->ahciport_mutex); 6785 return (AHCI_SUCCESS); 6786 } 6787 6788 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) { 6789 /* Not allowed to re-enter. */ 6790 mutex_exit(&ahci_portp->ahciport_mutex); 6791 return (AHCI_SUCCESS); 6792 } 6793 6794 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF; 6795 6796 /* 6797 * NOTE: 6798 * Even if Asynchronous Notification is supported (and enabled) by 6799 * both controller and the port multiplier, the content of PxSNTF 6800 * register is always set to 0x8000 by async notification event. We 6801 * need to check GSCR[32] on the port multiplier to find out the 6802 * owner of this event. 6803 * This is not accord with SATA spec 2.6 and needs further 6804 * clarification. 6805 */ 6806 /* hot-plug will not reported while reseting. */ 6807 if (ahci_portp->ahciport_reset_in_progress == 1) { 6808 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 6809 "port %d snotif event ignored", port); 6810 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 6811 mutex_exit(&ahci_portp->ahciport_mutex); 6812 return (AHCI_SUCCESS); 6813 } 6814 6815 AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp, 6816 "PxSNTF is set to 0x%x by port multiplier", 6817 ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags); 6818 6819 /* 6820 * Now we need do some necessary operation and inform SATA framework 6821 * that link/device events has happened. 6822 */ 6823 bzero((void *)&sdevice, sizeof (sata_device_t)); 6824 sdevice.satadev_addr.cport = ahci_ctlp-> 6825 ahcictl_port_to_cport[port]; 6826 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 6827 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 6828 sdevice.satadev_state = SATA_PSTATE_PWRON; 6829 6830 /* Just reject packets, do not stop that port. */ 6831 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 6832 6833 mutex_exit(&ahci_portp->ahciport_mutex); 6834 sata_hba_event_notify( 6835 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 6836 &sdevice, 6837 SATA_EVNT_PMULT_LINK_CHANGED); 6838 mutex_enter(&ahci_portp->ahciport_mutex); 6839 6840 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF; 6841 mutex_exit(&ahci_portp->ahciport_mutex); 6842 6843 return (AHCI_SUCCESS); 6844 } 6845 6846 /* 6847 * 1=Change in Current Connect Status. 0=No change in Current Connect Status. 6848 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared 6849 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it 6850 * indicates a COMINIT signal was received. 6851 * 6852 * Hot plug insertion is detected by reception of a COMINIT signal from the 6853 * device. On reception of unsolicited COMINIT, the HBA shall generate a 6854 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall 6855 * begin the normal communication negotiation sequence as outlined in the 6856 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the 6857 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the 6858 * PxSSTS.DET field shall be set to 1h. When the communication negotiation 6859 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set 6860 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy 6861 * to handle hot plug insertion. In this interrupt handler, just do nothing 6862 * but print some log message and clear the bit. 6863 */ 6864 static int 6865 ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp, 6866 ahci_port_t *ahci_portp, uint8_t port) 6867 { 6868 #if AHCI_DEBUG 6869 uint32_t port_serror; 6870 #endif 6871 6872 mutex_enter(&ahci_portp->ahciport_mutex); 6873 6874 #if AHCI_DEBUG 6875 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6876 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 6877 6878 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6879 "ahci_intr_port_connect_change: port %d, " 6880 "port_serror = 0x%x", port, port_serror); 6881 #endif 6882 6883 /* Clear PxSERR.DIAG.X to clear the interrupt bit */ 6884 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6885 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6886 SERROR_EXCHANGED_ERR); 6887 6888 mutex_exit(&ahci_portp->ahciport_mutex); 6889 6890 return (AHCI_SUCCESS); 6891 } 6892 6893 /* 6894 * Hot Plug Operation for platforms that support Mechanical Presence 6895 * Switches. 6896 * 6897 * When set, it indicates that a mechanical presence switch attached to this 6898 * port has been opened or closed, which may lead to a change in the connection 6899 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP 6900 * are set to '1'. 6901 * 6902 * At the moment, this interrupt is not needed and disabled and we just log 6903 * the debug message. 6904 */ 6905 static int 6906 ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp, 6907 ahci_port_t *ahci_portp, uint8_t port) 6908 { 6909 uint32_t cap_status, port_cmd_status; 6910 6911 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6912 "ahci_intr_device_mechanical_presence_status enter, " 6913 "port %d", port); 6914 6915 cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6916 (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp)); 6917 6918 mutex_enter(&ahci_portp->ahciport_mutex); 6919 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 6920 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 6921 6922 if (!(cap_status & AHCI_HBA_CAP_SMPS) || 6923 !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) { 6924 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6925 "CAP.SMPS or PxCMD.MPSP is not set, so just ignore " 6926 "the interrupt: cap_status = 0x%x, " 6927 "port_cmd_status = 0x%x", cap_status, port_cmd_status); 6928 mutex_exit(&ahci_portp->ahciport_mutex); 6929 6930 return (AHCI_SUCCESS); 6931 } 6932 6933 #if AHCI_DEBUG 6934 if (port_cmd_status & AHCI_CMD_STATUS_MPSS) { 6935 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6936 "The mechanical presence switch is open: " 6937 "port %d, port_cmd_status = 0x%x", 6938 port, port_cmd_status); 6939 } else { 6940 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 6941 "The mechanical presence switch is close: " 6942 "port %d, port_cmd_status = 0x%x", 6943 port, port_cmd_status); 6944 } 6945 #endif 6946 6947 mutex_exit(&ahci_portp->ahciport_mutex); 6948 6949 return (AHCI_SUCCESS); 6950 } 6951 6952 /* 6953 * Native Hot Plug Support. 6954 * 6955 * When set, it indicates that the internal PHYRDY signal changed state. 6956 * This bit reflects the state of PxSERR.DIAG.N. 6957 * 6958 * There are three kinds of conditions to generate this interrupt event: 6959 * 1. a device is inserted 6960 * 2. a device is disconnected 6961 * 3. when the link enters/exits a Partial or Slumber interface power 6962 * management state 6963 * 6964 * If inteface power management is enabled for a port, the PxSERR.DIAG.N 6965 * bit may be set due to the link entering the Partial or Slumber power 6966 * management state, rather than due to a hot plug insertion or removal 6967 * event. So far, the interface power management is disabled, so the 6968 * driver can reliably get removal detection notification via the 6969 * PxSERR.DIAG.N bit. 6970 */ 6971 static int 6972 ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp, 6973 ahci_port_t *ahci_portp, uint8_t port) 6974 { 6975 uint32_t port_sstatus = 0; /* No dev present & PHY not established. */ 6976 sata_device_t sdevice; 6977 int dev_exists_now = 0; 6978 int dev_existed_previously = 0; 6979 ahci_addr_t port_addr; 6980 6981 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp, 6982 "ahci_intr_phyrdy_change enter, port %d", port); 6983 6984 /* Clear PxSERR.DIAG.N to clear the interrupt bit */ 6985 mutex_enter(&ahci_portp->ahciport_mutex); 6986 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 6987 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 6988 SERROR_PHY_RDY_CHG); 6989 mutex_exit(&ahci_portp->ahciport_mutex); 6990 6991 mutex_enter(&ahci_ctlp->ahcictl_mutex); 6992 if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) || 6993 (ahci_portp == NULL)) { 6994 /* The whole controller setup is not yet done. */ 6995 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6996 return (AHCI_SUCCESS); 6997 } 6998 mutex_exit(&ahci_ctlp->ahcictl_mutex); 6999 7000 mutex_enter(&ahci_portp->ahciport_mutex); 7001 7002 /* SStatus tells the presence of device. */ 7003 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7004 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 7005 7006 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 7007 dev_exists_now = 1; 7008 } 7009 7010 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) { 7011 dev_existed_previously = 1; 7012 } 7013 7014 if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) { 7015 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV; 7016 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7017 "ahci_intr_phyrdy_change: port %d " 7018 "AHCI_PORT_FLAG_NODEV is cleared", port); 7019 if (dev_exists_now == 0) 7020 dev_existed_previously = 1; 7021 } 7022 7023 bzero((void *)&sdevice, sizeof (sata_device_t)); 7024 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 7025 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 7026 sdevice.satadev_addr.pmport = 0; 7027 sdevice.satadev_state = SATA_PSTATE_PWRON; 7028 ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON; 7029 7030 AHCI_ADDR_SET_PORT(&port_addr, port); 7031 7032 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG; 7033 if (dev_exists_now) { 7034 if (dev_existed_previously) { /* 1 -> 1 */ 7035 /* Things are fine now. The loss was temporary. */ 7036 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7037 "ahci_intr_phyrdy_change port %d " 7038 "device link lost/established", port); 7039 7040 mutex_exit(&ahci_portp->ahciport_mutex); 7041 sata_hba_event_notify( 7042 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7043 &sdevice, 7044 SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED); 7045 mutex_enter(&ahci_portp->ahciport_mutex); 7046 7047 } else { /* 0 -> 1 */ 7048 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7049 "ahci_intr_phyrdy_change: port %d " 7050 "device link established", port); 7051 7052 /* 7053 * A new device has been detected. The new device 7054 * might be a port multiplier instead of a drive, so 7055 * we cannot update the signature directly. 7056 */ 7057 (void) ahci_initialize_port(ahci_ctlp, 7058 ahci_portp, &port_addr); 7059 7060 /* Try to start the port */ 7061 if (ahci_start_port(ahci_ctlp, ahci_portp, port) 7062 != AHCI_SUCCESS) { 7063 sdevice.satadev_state |= SATA_PSTATE_FAILED; 7064 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7065 "ahci_intr_phyrdy_change: port %d failed " 7066 "at start port", port); 7067 } 7068 7069 /* Clear the max queue depth for inserted device */ 7070 ahci_portp->ahciport_max_ncq_tags = 0; 7071 7072 mutex_exit(&ahci_portp->ahciport_mutex); 7073 sata_hba_event_notify( 7074 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7075 &sdevice, 7076 SATA_EVNT_LINK_ESTABLISHED); 7077 mutex_enter(&ahci_portp->ahciport_mutex); 7078 7079 } 7080 } else { /* No device exists now */ 7081 7082 if (dev_existed_previously) { /* 1 -> 0 */ 7083 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 7084 "ahci_intr_phyrdy_change: port %d " 7085 "device link lost", port); 7086 7087 ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port); 7088 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 7089 ahci_portp, port); 7090 7091 if (ahci_portp->ahciport_device_type == 7092 SATA_DTYPE_PMULT) { 7093 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 7094 } 7095 7096 /* An existing device is lost. */ 7097 ahci_portp->ahciport_device_type = SATA_DTYPE_NONE; 7098 ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN; 7099 7100 mutex_exit(&ahci_portp->ahciport_mutex); 7101 sata_hba_event_notify( 7102 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7103 &sdevice, 7104 SATA_EVNT_LINK_LOST); 7105 mutex_enter(&ahci_portp->ahciport_mutex); 7106 } 7107 } 7108 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG; 7109 7110 mutex_exit(&ahci_portp->ahciport_mutex); 7111 7112 return (AHCI_SUCCESS); 7113 } 7114 7115 /* 7116 * PxIS.UFS - Unknown FIS Error 7117 * 7118 * This interrupt event means an unknown FIS was received and has been 7119 * copied into system memory. An unknown FIS is not considered an illegal 7120 * FIS, unless the length received is more than 64 bytes. If an unknown 7121 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues 7122 * normal operation. If the unknown FIS is more than 64 bytes, then it 7123 * won't be posted to memory and PxSERR.ERR.P will be set, which is then 7124 * a fatal error. 7125 * 7126 * PxIS.IPMS - Incorrect Port Multiplier Status 7127 * 7128 * IPMS Indicates that the HBA received a FIS from a device that did not 7129 * have a command outstanding. The IPMS bit may be set during enumeration 7130 * of devices on a Port Multiplier due to the normal Port Multiplier 7131 * enumeration process. It is recommended that IPMS only be used after 7132 * enumeration is complete on the Port Multiplier (copied from spec). 7133 * 7134 * PxIS.OFS - Overflow Error 7135 * 7136 * Command list overflow is defined as software building a command table 7137 * that has fewer total bytes than the transaction given to the device. 7138 * On device writes, the HBA will run out of data, and on reads, there 7139 * will be no room to put the data. 7140 * 7141 * For an overflow on data read, either PIO or DMA, the HBA will set 7142 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a 7143 * non-fatal error when the HBA can continues. Sometimes, it will cause 7144 * a fatal error and need the software to do something. 7145 * 7146 * For an overflow on data write, setting PxIS.OFS is optional for both 7147 * DMA and PIO, and it's a fatal error, and a COMRESET is required by 7148 * software to clean up from this serious error. 7149 * 7150 * PxIS.INFS - Interface Non-Fatal Error 7151 * 7152 * This interrupt event indicates that the HBA encountered an error on 7153 * the Serial ATA interface but was able to continue operation. The kind 7154 * of error usually occurred during a non-Data FIS, and under this condition 7155 * the FIS will be re-transmitted by HBA automatically. 7156 * 7157 * When the FMA is implemented, there should be a stat structure to 7158 * record how many every kind of error happens. 7159 */ 7160 static int 7161 ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 7162 uint8_t port, uint32_t intr_status) 7163 { 7164 uint32_t port_serror; 7165 #if AHCI_DEBUG 7166 uint32_t port_cmd_status; 7167 uint32_t port_cmd_issue; 7168 uint32_t port_sactive; 7169 int current_slot; 7170 uint32_t current_tags; 7171 sata_pkt_t *satapkt; 7172 ahci_cmd_header_t *cmd_header; 7173 uint32_t cmd_dmacount; 7174 #endif 7175 7176 mutex_enter(&ahci_portp->ahciport_mutex); 7177 7178 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7179 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7180 7181 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp, 7182 "ahci_intr_non_fatal_error: port %d, " 7183 "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status); 7184 7185 ahci_log_serror_message(ahci_ctlp, port, port_serror, 1); 7186 7187 if (intr_status & AHCI_INTR_STATUS_UFS) { 7188 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 7189 "ahci port %d has unknown FIS error", port); 7190 7191 /* Clear the interrupt bit by clearing PxSERR.DIAG.F */ 7192 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7193 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 7194 SERROR_FIS_TYPE); 7195 } 7196 7197 #if AHCI_DEBUG 7198 if (intr_status & AHCI_INTR_STATUS_IPMS) { 7199 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d " 7200 "has Incorrect Port Multiplier Status error", port); 7201 } 7202 7203 if (intr_status & AHCI_INTR_STATUS_OFS) { 7204 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7205 "ahci port %d has overflow error", port); 7206 } 7207 7208 if (intr_status & AHCI_INTR_STATUS_INFS) { 7209 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7210 "ahci port %d has interface non fatal error", port); 7211 } 7212 7213 /* 7214 * Record the error occurred command's slot. 7215 */ 7216 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 7217 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7218 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7219 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7220 7221 current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7222 AHCI_CMD_STATUS_CCS_SHIFT; 7223 7224 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 7225 satapkt = ahci_portp->ahciport_err_retri_pkt; 7226 ASSERT(satapkt != NULL); 7227 ASSERT(current_slot == 0); 7228 } else { 7229 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7230 } 7231 7232 if (satapkt != NULL) { 7233 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7234 "ahci_intr_non_fatal_error: pending_tags = 0x%x " 7235 "cmd 0x%x", ahci_portp->ahciport_pending_tags, 7236 satapkt->satapkt_cmd.satacmd_cmd_reg); 7237 7238 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7239 "ahci_intr_non_fatal_error: port %d, " 7240 "satapkt 0x%p is being processed when error occurs", 7241 port, (void *)satapkt); 7242 7243 /* 7244 * PRD Byte Count field of command header is not 7245 * required to reflect the total number of bytes 7246 * transferred when an overflow occurs, so here 7247 * just log the value. 7248 */ 7249 cmd_dmacount = 7250 ahci_portp->ahciport_prd_bytecounts[current_slot]; 7251 if (cmd_dmacount) { 7252 cmd_header = &ahci_portp-> 7253 ahciport_cmd_list[current_slot]; 7254 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7255 "ahci_intr_non_fatal_error: port %d, " 7256 "PRD Byte Count = 0x%x, " 7257 "ahciport_prd_bytecounts = 0x%x", port, 7258 cmd_header->ahcich_prd_byte_count, 7259 cmd_dmacount); 7260 } 7261 } 7262 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7263 /* 7264 * For queued command, list those command which have already 7265 * been transmitted to the device and still not completed. 7266 */ 7267 port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7268 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 7269 7270 port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7271 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 7272 7273 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp, 7274 "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x " 7275 "port_sactive = 0x%x port_cmd_issue = 0x%x", 7276 ahci_portp->ahciport_pending_ncq_tags, 7277 port_sactive, port_cmd_issue); 7278 7279 current_tags = ahci_portp->ahciport_pending_ncq_tags & 7280 port_sactive & ~port_cmd_issue & 7281 AHCI_NCQ_SLOT_MASK(ahci_portp); 7282 7283 while (current_tags) { 7284 current_slot = ddi_ffs(current_tags) - 1; 7285 if (current_slot == -1) { 7286 goto out; 7287 } 7288 7289 satapkt = ahci_portp->ahciport_slot_pkts[current_slot]; 7290 AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, 7291 ahci_ctlp, "ahci_intr_non_fatal_error: " 7292 "port %d, satapkt 0x%p is outstanding when " 7293 "error occurs", port, (void *)satapkt); 7294 7295 CLEAR_BIT(current_tags, current_slot); 7296 } 7297 } 7298 out: 7299 #endif 7300 mutex_exit(&ahci_portp->ahciport_mutex); 7301 7302 return (AHCI_SUCCESS); 7303 } 7304 7305 /* 7306 * According to the AHCI spec, the error types include system memory 7307 * errors, interface errors, port multiplier errors, device errors, 7308 * command list overflow, command list underflow, native command 7309 * queuing tag errors and pio data transfer errors. 7310 * 7311 * System memory errors such as target abort, master abort, and parity 7312 * may cause the host to stop, and they are serious errors and needed 7313 * to be recovered with software intervention. When system software 7314 * has given a pointer to the HBA that doesn't exist in physical memory, 7315 * a master/target abort error occurs, and PxIS.HBFS will be set. A 7316 * data error such as CRC or parity occurs, the HBA aborts the transfer 7317 * (if necessary) and PxIS.HBDS will be set. 7318 * 7319 * Interface errors are errors that occur due to electrical issues on 7320 * the interface, or protocol miscommunication between the device and 7321 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS 7322 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that 7323 * causes PxIS.IFS/PxIS.INFS to be set are 7324 * 1. in PxSERR.ERR, P bit is set to '1' 7325 * 2. in PxSERR.DIAG, C or H bit is set to '1' 7326 * 3. PhyRdy drop unexpectly, N bit is set to '1' 7327 * If the error occurred during a non-data FIS, the FIS must be 7328 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If 7329 * the error occurred during a data FIS, the transfer will stop, so 7330 * the error is fatal and PxIS.IFS is set. 7331 * 7332 * When a FIS arrives that updates the taskfile, the HBA checks to see 7333 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA 7334 * stops processing any more commands. 7335 * 7336 * Command list overflow is defined as software building a command table 7337 * that has fewer total bytes than the transaction given to the device. 7338 * On device writes, the HBA will run out of data, and on reads, there 7339 * will be no room to put the data. For an overflow on data read, either 7340 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error. 7341 * For an overflow on data write, setting PxIS.OFS is optional for both 7342 * DMA and PIO, and a COMRESET is required by software to clean up from 7343 * this serious error. 7344 * 7345 * Command list underflow is defined as software building a command 7346 * table that has more total bytes than the transaction given to the 7347 * device. For data writes, both PIO and DMA, the device will detect 7348 * an error and end the transfer. And these errors are most likely going 7349 * to be fatal errors that will cause the port to be restarted. For 7350 * data reads, the HBA updates its PRD byte count, and may be 7351 * able to continue normally, but is not required to. And The HBA is 7352 * not required to detect underflow conditions for native command 7353 * queuing command. 7354 * 7355 * The HBA does not actively check incoming DMA Setup FISes to ensure 7356 * that the PxSACT register bit for that slot is set. Existing error 7357 * mechanisms, such as host bus failure, or bad protocol, are used to 7358 * recover from this case. 7359 * 7360 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final 7361 * DATA FIS must be an integral number of Dwords. If the HBA receives 7362 * a request which is not an integral number of Dwords, the HBA 7363 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until 7364 * software restarts the port. And the HBA ensures that the size 7365 * of the DATA FIS received during a PIO command matches the size in 7366 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the 7367 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then 7368 * stop running until software restarts the port. 7369 */ 7370 /* 7371 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES. 7372 * 7373 * PxIS.IFS indicates that the hba encountered an error on the serial ata 7374 * interface which caused the transfer to stop. 7375 * 7376 * PxIS.HBDS indicates that the hba encountered a data error 7377 * (uncorrectable ecc/parity) when reading from or writing to system memory. 7378 * 7379 * PxIS.HBFS indicates that the hba encountered a host bus error that it 7380 * cannot recover from, such as a bad software pointer. 7381 * 7382 * PxIS.TFES is set whenever the status register is updated by the device 7383 * and the error bit (bit 0) is set. 7384 */ 7385 static int 7386 ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp, 7387 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 7388 { 7389 uint32_t port_cmd_status; 7390 uint32_t port_serror; 7391 uint32_t task_file_status; 7392 int failed_slot; 7393 sata_pkt_t *spkt = NULL; 7394 uint8_t err_byte; 7395 ahci_event_arg_t *args; 7396 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 7397 7398 mutex_enter(&ahci_portp->ahciport_mutex); 7399 7400 /* 7401 * ahci_intr_phyrdy_change() may have rendered it to 7402 * SATA_DTYPE_NONE. 7403 */ 7404 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 7405 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 7406 "ahci_intr_fatal_error: port %d no device attached, " 7407 "and just return without doing anything", port); 7408 goto out0; 7409 } 7410 7411 if (intr_status & AHCI_INTR_STATUS_TFES) { 7412 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7413 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 7414 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7415 "ahci_intr_fatal_error: port %d " 7416 "task_file_status = 0x%x", port, task_file_status); 7417 } 7418 7419 /* 7420 * Here we just log the fatal error info in interrupt context. 7421 * Misc recovery processing will be handled in task queue. 7422 */ 7423 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 7424 /* 7425 * Read PxCMD.CCS to determine the slot that the HBA 7426 * was processing when the error occurred. 7427 */ 7428 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7429 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7430 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 7431 AHCI_CMD_STATUS_CCS_SHIFT; 7432 7433 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 7434 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 7435 "ahci_intr_fatal_error: spkt 0x%p is being processed when " 7436 "fatal error occurred for port %d", spkt, port); 7437 7438 /* A Task File Data error. */ 7439 if (intr_status & AHCI_INTR_STATUS_TFES) { 7440 err_byte = (task_file_status & AHCI_TFD_ERR_MASK) 7441 >> AHCI_TFD_ERR_SHIFT; 7442 7443 /* 7444 * Won't emit the error message if it is an IDENTIFY 7445 * DEVICE command sent to an ATAPI device. 7446 */ 7447 if ((spkt != NULL) && 7448 (spkt->satapkt_cmd.satacmd_cmd_reg == 7449 SATAC_ID_DEVICE) && 7450 (err_byte == SATA_ERROR_ABORT)) 7451 goto out1; 7452 7453 /* 7454 * Won't emit the error message if it is an ATAPI PACKET 7455 * command 7456 */ 7457 if ((spkt != NULL) && 7458 (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET)) 7459 goto out1; 7460 } 7461 } 7462 7463 /* print the fatal error type */ 7464 ahci_log_fatal_error_message(ahci_ctlp, port, intr_status); 7465 port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7466 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port)); 7467 7468 /* print PxSERR related error message */ 7469 ahci_log_serror_message(ahci_ctlp, port, port_serror, 0); 7470 7471 /* print task file register value */ 7472 if (intr_status & AHCI_INTR_STATUS_TFES) { 7473 cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status " 7474 "= 0x%x", instance, port, task_file_status); 7475 } 7476 7477 out1: 7478 /* Prepare the argument for the taskq */ 7479 args = ahci_portp->ahciport_event_args; 7480 args->ahciea_ctlp = (void *)ahci_ctlp; 7481 args->ahciea_portp = (void *)ahci_portp; 7482 args->ahciea_event = intr_status; 7483 AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port); 7484 7485 /* Start the taskq to handle error recovery */ 7486 if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq, 7487 ahci_events_handler, 7488 (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) { 7489 cmn_err(CE_WARN, "!ahci%d: ahci start taskq for event handler " 7490 "failed", instance); 7491 } 7492 out0: 7493 mutex_exit(&ahci_portp->ahciport_mutex); 7494 7495 return (AHCI_SUCCESS); 7496 } 7497 7498 /* 7499 * Hot Plug Operation for platforms that support Cold Presence Detect. 7500 * 7501 * When set, a device status has changed as detected by the cold presence 7502 * detect logic. This bit can either be set due to a non-connected port 7503 * receiving a device, or a connected port having its device removed. 7504 * This bit is only valid if the port supports cold presence detect as 7505 * indicated by PxCMD.CPD set to '1'. 7506 * 7507 * At the moment, this interrupt is not needed and disabled and we just 7508 * log the debug message. 7509 */ 7510 static int 7511 ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp, 7512 ahci_port_t *ahci_portp, uint8_t port) 7513 { 7514 uint32_t port_cmd_status; 7515 sata_device_t sdevice; 7516 7517 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7518 "ahci_intr_cold_port_detect enter, port %d", port); 7519 7520 mutex_enter(&ahci_portp->ahciport_mutex); 7521 7522 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7523 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7524 if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) { 7525 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7526 "port %d does not support cold presence detect, so " 7527 "we just ignore this interrupt", port); 7528 mutex_exit(&ahci_portp->ahciport_mutex); 7529 return (AHCI_SUCCESS); 7530 } 7531 7532 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7533 "port %d device status has changed", port); 7534 7535 bzero((void *)&sdevice, sizeof (sata_device_t)); 7536 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 7537 sdevice.satadev_addr.qual = SATA_ADDR_CPORT; 7538 sdevice.satadev_addr.pmport = 0; 7539 sdevice.satadev_state = SATA_PSTATE_PWRON; 7540 7541 if (port_cmd_status & AHCI_CMD_STATUS_CPS) { 7542 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7543 "port %d: a device is hot plugged", port); 7544 mutex_exit(&ahci_portp->ahciport_mutex); 7545 sata_hba_event_notify( 7546 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7547 &sdevice, 7548 SATA_EVNT_DEVICE_ATTACHED); 7549 mutex_enter(&ahci_portp->ahciport_mutex); 7550 7551 } else { 7552 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7553 "port %d: a device is hot unplugged", port); 7554 mutex_exit(&ahci_portp->ahciport_mutex); 7555 sata_hba_event_notify( 7556 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 7557 &sdevice, 7558 SATA_EVNT_DEVICE_DETACHED); 7559 mutex_enter(&ahci_portp->ahciport_mutex); 7560 } 7561 7562 mutex_exit(&ahci_portp->ahciport_mutex); 7563 7564 return (AHCI_SUCCESS); 7565 } 7566 7567 /* 7568 * Enable the interrupts for a particular port. 7569 * 7570 * WARNING!!! ahciport_mutex should be acquired before the function 7571 * is called. 7572 */ 7573 static void 7574 ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 7575 { 7576 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7577 "ahci_enable_port_intrs enter, port %d", port); 7578 7579 /* 7580 * Clear port interrupt status before enabling interrupt 7581 */ 7582 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7583 (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port), 7584 AHCI_PORT_INTR_MASK); 7585 7586 /* 7587 * Clear the pending bit from IS.IPS 7588 */ 7589 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7590 (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port)); 7591 7592 /* 7593 * Enable the following interrupts: 7594 * Device to Host Register FIS Interrupt (DHRS) 7595 * PIO Setup FIS Interrupt (PSS) 7596 * Set Device Bits Interrupt (SDBS) 7597 * Unknown FIS Interrupt (UFS) 7598 * Port Connect Change Status (PCS) 7599 * PhyRdy Change Status (PRCS) 7600 * Overflow Status (OFS) 7601 * Interface Non-fatal Error Status (INFS) 7602 * Interface Fatal Error Status (IFS) 7603 * Host Bus Data Error Status (HBDS) 7604 * Host Bus Fatal Error Status (HBFS) 7605 * Task File Error Status (TFES) 7606 */ 7607 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7608 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 7609 (AHCI_INTR_STATUS_DHRS | 7610 AHCI_INTR_STATUS_PSS | 7611 AHCI_INTR_STATUS_SDBS | 7612 AHCI_INTR_STATUS_UFS | 7613 AHCI_INTR_STATUS_DPS | 7614 AHCI_INTR_STATUS_PCS | 7615 AHCI_INTR_STATUS_PRCS | 7616 AHCI_INTR_STATUS_OFS | 7617 AHCI_INTR_STATUS_INFS | 7618 AHCI_INTR_STATUS_IFS | 7619 AHCI_INTR_STATUS_HBDS | 7620 AHCI_INTR_STATUS_HBFS | 7621 AHCI_INTR_STATUS_TFES)); 7622 } 7623 7624 /* 7625 * Enable interrupts for all the ports. 7626 * 7627 * WARNING!!! ahcictl_mutex should be acquired before the function 7628 * is called. 7629 */ 7630 static void 7631 ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp) 7632 { 7633 uint32_t ghc_control; 7634 7635 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL); 7636 7637 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7638 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 7639 7640 ghc_control |= AHCI_HBA_GHC_IE; 7641 7642 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7643 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 7644 } 7645 7646 /* 7647 * Disable interrupts for a particular port. 7648 * 7649 * WARNING!!! ahciport_mutex should be acquired before the function 7650 * is called. 7651 */ 7652 static void 7653 ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port) 7654 { 7655 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7656 "ahci_disable_port_intrs enter, port %d", port); 7657 7658 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7659 (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0); 7660 } 7661 7662 /* 7663 * Disable interrupts for the whole HBA. 7664 * 7665 * The global bit is cleared, then all interrupt sources from all 7666 * ports are disabled. 7667 * 7668 * WARNING!!! ahcictl_mutex should be acquired before the function 7669 * is called. 7670 */ 7671 static void 7672 ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp) 7673 { 7674 uint32_t ghc_control; 7675 7676 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter", 7677 NULL); 7678 7679 ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7680 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp)); 7681 7682 ghc_control &= ~ AHCI_HBA_GHC_IE; 7683 7684 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7685 (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control); 7686 } 7687 7688 /* 7689 * Handle FIXED or MSI interrupts. 7690 */ 7691 /* 7692 * According to AHCI spec, the HBA may support several interrupt modes: 7693 * * pin based interrupts (FIXED) 7694 * * single MSI message interrupts 7695 * * multiple MSI based message interrupts 7696 * 7697 * For pin based interrupts, the software interrupt handler need to check IS 7698 * register to find out which port has pending interrupts. And then check 7699 * PxIS register to find out which interrupt events happened on that port. 7700 * 7701 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and 7702 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts 7703 * in that software interrupt handler need to check IS register to determine 7704 * which port triggered the interrupts since it uses a single message for all 7705 * port interrupts. 7706 * 7707 * HBA may optionally support multiple MSI message for better performance. In 7708 * this mode, each port may have its own interrupt message, and thus generation 7709 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC 7710 * represents a power-of-2 wrapper on the number of implemented ports, and 7711 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the 7712 * maximum number of assigned interrupts. When the number of MSI messages 7713 * allocated is less than the number requested, then hardware may have two 7714 * implementation behaviors: 7715 * * assign each ports its own interrupt and then force all additional 7716 * ports to share the last interrupt message, and this condition is 7717 * indicated by clearing GHC.MRSM to '0' 7718 * * revert to single MSI mode, indicated by setting GHC.MRSM to '1' 7719 * When multiple-message MSI is enabled, hardware will still set IS register 7720 * as single message case. And this IS register may be used by software when 7721 * fewer than the requested number of messages is granted in order to determine 7722 * which port had the interrupt. 7723 * 7724 * Note: The current ahci driver only supports the first two interrupt modes: 7725 * pin based interrupts and single MSI message interrupts, and the reason 7726 * is indicated in below code. 7727 */ 7728 static int 7729 ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type) 7730 { 7731 dev_info_t *dip = ahci_ctlp->ahcictl_dip; 7732 int count, avail, actual; 7733 int i, rc; 7734 7735 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp, 7736 "ahci_add_intrs enter interrupt type 0x%x", intr_type); 7737 7738 /* get number of interrupts. */ 7739 rc = ddi_intr_get_nintrs(dip, intr_type, &count); 7740 if ((rc != DDI_SUCCESS) || (count == 0)) { 7741 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7742 "ddi_intr_get_nintrs() failed, " 7743 "rc %d count %d\n", rc, count); 7744 return (DDI_FAILURE); 7745 } 7746 7747 /* get number of available interrupts. */ 7748 rc = ddi_intr_get_navail(dip, intr_type, &avail); 7749 if ((rc != DDI_SUCCESS) || (avail == 0)) { 7750 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7751 "ddi_intr_get_navail() failed, " 7752 "rc %d avail %d\n", rc, avail); 7753 return (DDI_FAILURE); 7754 } 7755 7756 #if AHCI_DEBUG 7757 if (avail < count) { 7758 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7759 "ddi_intr_get_nintrs returned %d, navail() returned %d", 7760 count, avail); 7761 } 7762 #endif 7763 7764 /* 7765 * Note: So far Solaris restricts the maximum number of messages for 7766 * x86 to 2, that is avail is 2, so here we set the count with 1 to 7767 * force the driver to use single MSI message interrupt. In future if 7768 * Solaris remove the restriction, then we need to delete the below 7769 * code and try to use multiple interrupt routine to gain better 7770 * performance. 7771 */ 7772 if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) { 7773 AHCIDBG(AHCIDBG_INTR, ahci_ctlp, 7774 "force to use one interrupt routine though the " 7775 "HBA supports %d interrupt", count); 7776 count = 1; 7777 } 7778 7779 /* Allocate an array of interrupt handles. */ 7780 ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t); 7781 ahci_ctlp->ahcictl_intr_htable = 7782 kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP); 7783 7784 /* call ddi_intr_alloc(). */ 7785 rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable, 7786 intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL); 7787 7788 if ((rc != DDI_SUCCESS) || (actual == 0)) { 7789 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7790 "ddi_intr_alloc() failed, rc %d count %d actual %d " 7791 "avail %d\n", rc, count, actual, avail); 7792 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7793 ahci_ctlp->ahcictl_intr_size); 7794 return (DDI_FAILURE); 7795 } 7796 7797 /* use interrupt count returned */ 7798 #if AHCI_DEBUG 7799 if (actual < count) { 7800 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7801 "Requested: %d, Received: %d", count, actual); 7802 } 7803 #endif 7804 7805 ahci_ctlp->ahcictl_intr_cnt = actual; 7806 7807 /* 7808 * Get priority for first, assume remaining are all the same. 7809 */ 7810 if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0], 7811 &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) { 7812 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7813 "ddi_intr_get_pri() failed", NULL); 7814 7815 /* Free already allocated intr. */ 7816 for (i = 0; i < actual; i++) { 7817 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 7818 } 7819 7820 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7821 ahci_ctlp->ahcictl_intr_size); 7822 return (DDI_FAILURE); 7823 } 7824 7825 /* Test for high level interrupt. */ 7826 if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) { 7827 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7828 "ahci_add_intrs: Hi level intr not supported", NULL); 7829 7830 /* Free already allocated intr. */ 7831 for (i = 0; i < actual; i++) { 7832 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]); 7833 } 7834 7835 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7836 sizeof (ddi_intr_handle_t)); 7837 7838 return (DDI_FAILURE); 7839 } 7840 7841 /* Call ddi_intr_add_handler(). */ 7842 for (i = 0; i < actual; i++) { 7843 if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i], 7844 ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) { 7845 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7846 "ddi_intr_add_handler() failed", NULL); 7847 7848 /* Free already allocated intr. */ 7849 for (i = 0; i < actual; i++) { 7850 (void) ddi_intr_free( 7851 ahci_ctlp->ahcictl_intr_htable[i]); 7852 } 7853 7854 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7855 ahci_ctlp->ahcictl_intr_size); 7856 return (DDI_FAILURE); 7857 } 7858 } 7859 7860 if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0], 7861 &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) { 7862 AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp, 7863 "ddi_intr_get_cap() failed", NULL); 7864 7865 /* Free already allocated intr. */ 7866 for (i = 0; i < actual; i++) { 7867 (void) ddi_intr_free( 7868 ahci_ctlp->ahcictl_intr_htable[i]); 7869 } 7870 7871 kmem_free(ahci_ctlp->ahcictl_intr_htable, 7872 ahci_ctlp->ahcictl_intr_size); 7873 return (DDI_FAILURE); 7874 } 7875 7876 if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) { 7877 /* Call ddi_intr_block_enable() for MSI. */ 7878 (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable, 7879 ahci_ctlp->ahcictl_intr_cnt); 7880 } else { 7881 /* Call ddi_intr_enable() for FIXED or MSI non block enable. */ 7882 for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) { 7883 (void) ddi_intr_enable( 7884 ahci_ctlp->ahcictl_intr_htable[i]); 7885 } 7886 } 7887 7888 return (DDI_SUCCESS); 7889 } 7890 7891 /* 7892 * Removes the registered interrupts irrespective of whether they 7893 * were legacy or MSI. 7894 * 7895 * WARNING!!! The controller interrupts must be disabled before calling 7896 * this routine. 7897 */ 7898 static void 7899 ahci_rem_intrs(ahci_ctl_t *ahci_ctlp) 7900 { 7901 int x; 7902 7903 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL); 7904 7905 /* Disable all interrupts. */ 7906 if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) && 7907 (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) { 7908 /* Call ddi_intr_block_disable(). */ 7909 (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable, 7910 ahci_ctlp->ahcictl_intr_cnt); 7911 } else { 7912 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 7913 (void) ddi_intr_disable( 7914 ahci_ctlp->ahcictl_intr_htable[x]); 7915 } 7916 } 7917 7918 /* Call ddi_intr_remove_handler(). */ 7919 for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) { 7920 (void) ddi_intr_remove_handler( 7921 ahci_ctlp->ahcictl_intr_htable[x]); 7922 (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]); 7923 } 7924 7925 kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size); 7926 } 7927 7928 /* 7929 * This routine tries to put port into P:NotRunning state by clearing 7930 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h 7931 * and PxCMD.CR to '0'. 7932 * 7933 * WARNING!!! ahciport_mutex should be acquired before the function 7934 * is called. 7935 */ 7936 static int 7937 ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp, 7938 ahci_port_t *ahci_portp, uint8_t port) 7939 { 7940 uint32_t port_cmd_status; 7941 int loop_count; 7942 7943 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 7944 "ahci_put_port_into_notrunning_state enter: port %d", port); 7945 7946 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7947 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7948 7949 port_cmd_status &= ~AHCI_CMD_STATUS_ST; 7950 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 7951 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status); 7952 7953 /* Wait until PxCMD.CR is cleared */ 7954 loop_count = 0; 7955 do { 7956 port_cmd_status = 7957 ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 7958 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 7959 7960 if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) { 7961 AHCIDBG(AHCIDBG_INIT, ahci_ctlp, 7962 "clearing port %d CMD.CR timeout, " 7963 "port_cmd_status = 0x%x", port, 7964 port_cmd_status); 7965 /* 7966 * We are effectively timing out after 0.5 sec. 7967 * This value is specified in AHCI spec. 7968 */ 7969 break; 7970 } 7971 7972 /* Wait for 10 millisec */ 7973 drv_usecwait(AHCI_10MS_USECS); 7974 } while (port_cmd_status & AHCI_CMD_STATUS_CR); 7975 7976 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED; 7977 7978 if (port_cmd_status & AHCI_CMD_STATUS_CR) { 7979 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 7980 "ahci_put_port_into_notrunning_state: failed to clear " 7981 "PxCMD.CR to '0' after loop count: %d, and " 7982 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 7983 return (AHCI_FAILURE); 7984 } else { 7985 AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp, 7986 "ahci_put_port_into_notrunning_state: succeeded to clear " 7987 "PxCMD.CR to '0' after loop count: %d, and " 7988 "port_cmd_status = 0x%x", loop_count, port_cmd_status); 7989 return (AHCI_SUCCESS); 7990 } 7991 } 7992 7993 /* 7994 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY 7995 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a 7996 * stable state, then set PxCMD.ST to '1' to start the port directly. 7997 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a 7998 * COMRESET to the device to put it in an idle state. 7999 * 8000 * The fifth argument returns whether the port reset is involved during 8001 * the process. 8002 * 8003 * The routine will be called under following scenarios: 8004 * + To reset the HBA 8005 * + To abort the packet(s) 8006 * + To reset the port 8007 * + To activate the port 8008 * + Fatal error recovery 8009 * + To abort the timeout packet(s) 8010 * 8011 * WARNING!!! ahciport_mutex should be acquired before the function 8012 * is called. And ahciport_mutex will be released before the reset 8013 * event is reported to sata module by calling sata_hba_event_notify, 8014 * and then be acquired again later. 8015 * 8016 * NOTES!!! During this procedure, PxSERR register will be cleared, and 8017 * according to the spec, the clearance of three bits will also clear 8018 * three interrupt status bits. 8019 * 1. PxSERR.DIAG.F will clear PxIS.UFS 8020 * 2. PxSERR.DIAG.X will clear PxIS.PCS 8021 * 3. PxSERR.DIAG.N will clear PxIS.PRCS 8022 * 8023 * Among these three interrupt events, the driver needs to take care of 8024 * PxIS.PRCS, which is the hot plug event. When the driver found out 8025 * a device was unplugged, it will call the interrupt handler. 8026 */ 8027 static int 8028 ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp, 8029 ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag) 8030 { 8031 uint32_t port_sstatus; 8032 uint32_t task_file_status; 8033 sata_device_t sdevice; 8034 int rval; 8035 ahci_addr_t addr_port; 8036 ahci_pmult_info_t *pminfo = NULL; 8037 int dev_exists_begin = 0; 8038 int dev_exists_end = 0; 8039 uint32_t previous_dev_type = ahci_portp->ahciport_device_type; 8040 int npmport = 0; 8041 uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8042 8043 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8044 "ahci_restart_port_wait_till_ready: port %d enter", port); 8045 8046 AHCI_ADDR_SET_PORT(&addr_port, port); 8047 8048 if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) 8049 dev_exists_begin = 1; 8050 8051 /* First clear PxCMD.ST */ 8052 rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp, 8053 port); 8054 if (rval != AHCI_SUCCESS) 8055 /* 8056 * If PxCMD.CR does not clear within a reasonable time, it 8057 * may assume the interface is in a hung condition and may 8058 * continue with issuing the port reset. 8059 */ 8060 goto reset; 8061 8062 /* Then clear PxSERR */ 8063 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 8064 (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port), 8065 AHCI_SERROR_CLEAR_ALL); 8066 8067 /* Then get PxTFD */ 8068 task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8069 (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port)); 8070 8071 /* 8072 * Check whether the device is in a stable status, if yes, 8073 * then start the port directly. However for ahci_tran_reset_dport, 8074 * we may have to perform a port reset. 8075 */ 8076 if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) && 8077 !(flag & AHCI_PORT_RESET)) 8078 goto out; 8079 8080 reset: 8081 /* 8082 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue 8083 * a COMRESET to the device 8084 */ 8085 ahci_disable_port_intrs(ahci_ctlp, port); 8086 rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port); 8087 ahci_enable_port_intrs(ahci_ctlp, port); 8088 8089 #ifdef AHCI_DEBUG 8090 if (rval != AHCI_SUCCESS) 8091 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8092 "ahci_restart_port_wait_till_ready: port %d failed", 8093 port); 8094 #endif 8095 8096 if (reset_flag != NULL) 8097 *reset_flag = 1; 8098 8099 /* Indicate to the framework that a reset has happened. */ 8100 if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) && 8101 (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) && 8102 !(flag & AHCI_RESET_NO_EVENTS_UP)) { 8103 /* Set the reset in progress flag */ 8104 ahci_portp->ahciport_reset_in_progress = 1; 8105 8106 bzero((void *)&sdevice, sizeof (sata_device_t)); 8107 sdevice.satadev_addr.cport = 8108 ahci_ctlp->ahcictl_port_to_cport[port]; 8109 sdevice.satadev_addr.pmport = 0; 8110 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8111 8112 sdevice.satadev_state = SATA_DSTATE_RESET | 8113 SATA_DSTATE_PWR_ACTIVE; 8114 if (ahci_ctlp->ahcictl_sata_hba_tran) { 8115 mutex_exit(&ahci_portp->ahciport_mutex); 8116 sata_hba_event_notify( 8117 ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip, 8118 &sdevice, 8119 SATA_EVNT_DEVICE_RESET); 8120 mutex_enter(&ahci_portp->ahciport_mutex); 8121 } 8122 8123 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8124 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8125 } else { 8126 ahci_portp->ahciport_reset_in_progress = 0; 8127 } 8128 8129 out: 8130 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8131 8132 /* SStatus tells the presence of device. */ 8133 port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8134 (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port)); 8135 8136 if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) { 8137 dev_exists_end = 1; 8138 } 8139 8140 if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */ 8141 return (rval); 8142 8143 /* Check whether a hot plug event happened */ 8144 if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */ 8145 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8146 "ahci_restart_port_wait_till_ready: port %d " 8147 "device is removed", port); 8148 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV; 8149 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8150 "ahci_restart_port_wait_till_ready: port %d " 8151 "AHCI_PORT_FLAG_NODEV flag is set", port); 8152 mutex_exit(&ahci_portp->ahciport_mutex); 8153 (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port); 8154 mutex_enter(&ahci_portp->ahciport_mutex); 8155 8156 return (rval); 8157 } 8158 8159 8160 /* 0/1 -> 1 : device may change */ 8161 /* 8162 * May be called by ahci_fatal_error_recovery_handler, so 8163 * don't issue software if the previous device is ATAPI. 8164 */ 8165 if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 8166 return (rval); 8167 8168 /* 8169 * The COMRESET will make port multiplier enter legacy mode. 8170 * Issue a software reset to make it work again. 8171 */ 8172 ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port); 8173 8174 /* 8175 * Following codes are specific for the port multiplier 8176 */ 8177 if (previous_dev_type != SATA_DTYPE_PMULT && 8178 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8179 /* in case previous_dev_type is corrupt */ 8180 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8181 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8182 return (rval); 8183 } 8184 8185 /* Device change: PMult -> Non-PMult */ 8186 if (previous_dev_type == SATA_DTYPE_PMULT && 8187 ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) { 8188 /* 8189 * This might happen because 8190 * 1. Software reset failed. Port multiplier is not correctly 8191 * enumerated. 8192 * 2. Another non-port-multiplier device is attached. Perhaps 8193 * the port multiplier was replaced by another device by 8194 * whatever reason, but AHCI driver missed hot-plug event. 8195 * 8196 * Now that the port has been initialized, we just need to 8197 * update the port structure according new device, then report 8198 * and wait SATA framework to probe new device. 8199 */ 8200 8201 /* Force to release pmult resource */ 8202 ahci_dealloc_pmult(ahci_ctlp, ahci_portp); 8203 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8204 8205 bzero((void *)&sdevice, sizeof (sata_device_t)); 8206 sdevice.satadev_addr.cport = 8207 ahci_ctlp->ahcictl_port_to_cport[port]; 8208 sdevice.satadev_addr.pmport = 0; 8209 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8210 8211 sdevice.satadev_state = SATA_DSTATE_RESET | 8212 SATA_DSTATE_PWR_ACTIVE; 8213 8214 mutex_exit(&ahci_portp->ahciport_mutex); 8215 sata_hba_event_notify( 8216 ahci_ctlp->ahcictl_dip, 8217 &sdevice, 8218 SATA_EVNT_DEVICE_RESET); 8219 mutex_enter(&ahci_portp->ahciport_mutex); 8220 8221 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8222 "Port multiplier is [Gone] at port %d ", port); 8223 AHCIDBG(AHCIDBG_EVENT, ahci_ctlp, 8224 "port %d sending event up: SATA_EVNT_DEVICE_RESET", port); 8225 8226 return (AHCI_SUCCESS); 8227 } 8228 8229 /* Device change: Non-PMult -> PMult */ 8230 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 8231 8232 /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */ 8233 ahci_alloc_pmult(ahci_ctlp, ahci_portp); 8234 8235 (void) ahci_start_port(ahci_ctlp, ahci_portp, port); 8236 } 8237 pminfo = ahci_portp->ahciport_pmult_info; 8238 ASSERT(pminfo != NULL); 8239 8240 /* Device (may) change: PMult -> PMult */ 8241 /* 8242 * First initialize port multiplier. Set state to READY and wait for 8243 * probe entry point to initialize it 8244 */ 8245 ahci_portp->ahciport_port_state = SATA_STATE_READY; 8246 8247 /* 8248 * It's a little complicated while target is a port multiplier. we 8249 * need to COMRESET all pmports behind that PMult otherwise those 8250 * sub-links between the PMult and the sub-devices will be in an 8251 * inactive state (indicated by PSCR0/PxSSTS) and the following access 8252 * to those sub-devices will be rejected by Link-Fatal-Error. 8253 */ 8254 /* 8255 * The PxSNTF will be set soon after the pmult is plugged. While the 8256 * pmult itself is attaching, sata_hba_event_notfiy will fail. so we 8257 * simply mark every sub-port as 'unknown', then ahci_probe_pmport 8258 * will initialized it. 8259 */ 8260 for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++) 8261 pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN; 8262 8263 /* Report reset event. */ 8264 ahci_portp->ahciport_reset_in_progress = 1; 8265 8266 bzero((void *)&sdevice, sizeof (sata_device_t)); 8267 sdevice.satadev_addr.cport = cport; 8268 sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT; 8269 sdevice.satadev_addr.qual = SATA_ADDR_PMULT; 8270 sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE; 8271 sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice, 8272 SATA_EVNT_DEVICE_RESET); 8273 8274 return (rval); 8275 } 8276 8277 /* 8278 * This routine may be called under four scenarios: 8279 * a) do the recovery from fatal error 8280 * b) or we need to timeout some commands 8281 * c) or we need to abort some commands 8282 * d) or we need reset device/port/controller 8283 * 8284 * In all these scenarios, we need to send any pending unfinished 8285 * commands up to sata framework. 8286 * 8287 * WARNING!!! ahciport_mutex should be acquired before the function is called. 8288 */ 8289 static void 8290 ahci_mop_commands(ahci_ctl_t *ahci_ctlp, 8291 ahci_port_t *ahci_portp, 8292 uint32_t slot_status, 8293 uint32_t failed_tags, 8294 uint32_t timeout_tags, 8295 uint32_t aborted_tags, 8296 uint32_t reset_tags) 8297 { 8298 uint32_t finished_tags = 0; 8299 uint32_t unfinished_tags = 0; 8300 int tmp_slot; 8301 sata_pkt_t *satapkt; 8302 int ncq_cmd_in_progress = 0; 8303 int err_retri_cmd_in_progress = 0; 8304 int rdwr_pmult_cmd_in_progress = 0; 8305 8306 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8307 "ahci_mop_commands entered: port: %d slot_status: 0x%x", 8308 ahci_portp->ahciport_port_num, slot_status); 8309 8310 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8311 "ahci_mop_commands: failed_tags: 0x%x, " 8312 "timeout_tags: 0x%x aborted_tags: 0x%x, " 8313 "reset_tags: 0x%x", failed_tags, 8314 timeout_tags, aborted_tags, reset_tags); 8315 8316 #ifdef AHCI_DEBUG 8317 if (ahci_debug_flags & AHCIDBG_ERRS) { 8318 int i; 8319 char msg_buf[200] = {0, }; 8320 for (i = 0x1f; i >= 0; i--) { 8321 if (ahci_portp->ahciport_slot_pkts[i] != NULL) 8322 msg_buf[i] = 'X'; 8323 else 8324 msg_buf[i] = '.'; 8325 } 8326 msg_buf[0x20] = '\0'; 8327 cmn_err(CE_NOTE, "port[%d] slots: %s", 8328 ahci_portp->ahciport_port_num, msg_buf); 8329 cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ", 8330 (void *)ahci_portp->ahciport_err_retri_pkt, 8331 (void *)ahci_portp->ahciport_rdwr_pmult_pkt); 8332 } 8333 #endif 8334 8335 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8336 finished_tags = ahci_portp->ahciport_pending_tags & 8337 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 8338 8339 unfinished_tags = slot_status & 8340 AHCI_SLOT_MASK(ahci_ctlp) & 8341 ~failed_tags & 8342 ~aborted_tags & 8343 ~reset_tags & 8344 ~timeout_tags; 8345 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8346 ncq_cmd_in_progress = 1; 8347 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 8348 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 8349 8350 unfinished_tags = slot_status & 8351 AHCI_NCQ_SLOT_MASK(ahci_portp) & 8352 ~failed_tags & 8353 ~aborted_tags & 8354 ~reset_tags & 8355 ~timeout_tags; 8356 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8357 8358 /* 8359 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is 8360 * set, it means REQUEST SENSE or READ LOG EXT command doesn't 8361 * complete successfully due to one of the following three 8362 * conditions: 8363 * 8364 * 1. Fatal error - failed_tags includes its slot 8365 * 2. Timed out - timeout_tags includes its slot 8366 * 3. Aborted when hot unplug - aborted_tags includes its 8367 * slot 8368 * 8369 * Please note that the command is always sent down in Slot 0 8370 */ 8371 err_retri_cmd_in_progress = 1; 8372 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp, 8373 "ahci_mop_commands is called for port %d while " 8374 "REQUEST SENSE or READ LOG EXT for error retrieval " 8375 "is being executed slot_status = 0x%x", 8376 ahci_portp->ahciport_port_num, slot_status); 8377 ASSERT(ahci_portp->ahciport_mop_in_progress > 1); 8378 ASSERT(slot_status == 0x1); 8379 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 8380 rdwr_pmult_cmd_in_progress = 1; 8381 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp, 8382 "ahci_mop_commands is called for port %d while " 8383 "READ/WRITE PORTMULT command is being executed", 8384 ahci_portp->ahciport_port_num); 8385 8386 ASSERT(slot_status == 0x1); 8387 } 8388 8389 #ifdef AHCI_DEBUG 8390 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp, 8391 "ahci_mop_commands: finished_tags: 0x%x, " 8392 "unfinished_tags 0x%x", finished_tags, unfinished_tags); 8393 #endif 8394 8395 /* Send up finished packets with SATA_PKT_COMPLETED */ 8396 while (finished_tags) { 8397 tmp_slot = ddi_ffs(finished_tags) - 1; 8398 if (tmp_slot == -1) { 8399 break; 8400 } 8401 8402 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8403 ASSERT(satapkt != NULL); 8404 8405 AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: " 8406 "sending up pkt 0x%p with SATA_PKT_COMPLETED", 8407 (void *)satapkt); 8408 8409 /* 8410 * Cannot fetch the return register content since the port 8411 * was restarted, so the corresponding tag will be set to 8412 * aborted tags. 8413 */ 8414 if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) { 8415 CLEAR_BIT(finished_tags, tmp_slot); 8416 aborted_tags |= tmp_slot; 8417 continue; 8418 } 8419 8420 if (ncq_cmd_in_progress) 8421 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8422 tmp_slot); 8423 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8424 CLEAR_BIT(finished_tags, tmp_slot); 8425 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8426 8427 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_COMPLETED); 8428 } 8429 8430 /* Send up failed packets with SATA_PKT_DEV_ERROR. */ 8431 while (failed_tags) { 8432 if (err_retri_cmd_in_progress) { 8433 satapkt = ahci_portp->ahciport_err_retri_pkt; 8434 ASSERT(satapkt != NULL); 8435 ASSERT(failed_tags == 0x1); 8436 8437 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8438 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8439 (void *)satapkt); 8440 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8441 break; 8442 } 8443 if (rdwr_pmult_cmd_in_progress) { 8444 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8445 ASSERT(satapkt != NULL); 8446 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8447 "ahci_mop_commands: sending up " 8448 "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR", 8449 (void *)satapkt); 8450 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8451 break; 8452 } 8453 8454 tmp_slot = ddi_ffs(failed_tags) - 1; 8455 if (tmp_slot == -1) { 8456 break; 8457 } 8458 8459 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8460 ASSERT(satapkt != NULL); 8461 8462 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8463 "sending up pkt 0x%p with SATA_PKT_DEV_ERROR", 8464 (void *)satapkt); 8465 8466 if (ncq_cmd_in_progress) 8467 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8468 tmp_slot); 8469 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8470 CLEAR_BIT(failed_tags, tmp_slot); 8471 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8472 8473 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_DEV_ERROR); 8474 } 8475 8476 /* Send up timeout packets with SATA_PKT_TIMEOUT. */ 8477 while (timeout_tags) { 8478 if (err_retri_cmd_in_progress) { 8479 satapkt = ahci_portp->ahciport_err_retri_pkt; 8480 ASSERT(satapkt != NULL); 8481 ASSERT(timeout_tags == 0x1); 8482 8483 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8484 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 8485 (void *)satapkt); 8486 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8487 break; 8488 } 8489 if (rdwr_pmult_cmd_in_progress) { 8490 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8491 ASSERT(satapkt != NULL); 8492 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8493 "ahci_mop_commands: sending up " 8494 "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT", 8495 (void *)satapkt); 8496 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8497 break; 8498 } 8499 8500 tmp_slot = ddi_ffs(timeout_tags) - 1; 8501 if (tmp_slot == -1) { 8502 break; 8503 } 8504 8505 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8506 ASSERT(satapkt != NULL); 8507 8508 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8509 "sending up pkt 0x%p with SATA_PKT_TIMEOUT", 8510 (void *)satapkt); 8511 8512 if (ncq_cmd_in_progress) 8513 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8514 tmp_slot); 8515 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8516 CLEAR_BIT(timeout_tags, tmp_slot); 8517 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8518 8519 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_TIMEOUT); 8520 } 8521 8522 /* Send up aborted packets with SATA_PKT_ABORTED */ 8523 while (aborted_tags) { 8524 if (err_retri_cmd_in_progress) { 8525 satapkt = ahci_portp->ahciport_err_retri_pkt; 8526 ASSERT(satapkt != NULL); 8527 ASSERT(aborted_tags == 0x1); 8528 8529 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8530 "sending up pkt 0x%p with SATA_PKT_ABORTED", 8531 (void *)satapkt); 8532 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 8533 break; 8534 } 8535 if (rdwr_pmult_cmd_in_progress) { 8536 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8537 ASSERT(satapkt != NULL); 8538 ASSERT(aborted_tags == 0x1); 8539 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8540 "ahci_mop_commands: sending up " 8541 "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED", 8542 (void *)satapkt); 8543 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 8544 break; 8545 } 8546 8547 tmp_slot = ddi_ffs(aborted_tags) - 1; 8548 if (tmp_slot == -1) { 8549 break; 8550 } 8551 8552 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8553 ASSERT(satapkt != NULL); 8554 8555 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8556 "sending up pkt 0x%p with SATA_PKT_ABORTED", 8557 (void *)satapkt); 8558 8559 if (ncq_cmd_in_progress) 8560 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8561 tmp_slot); 8562 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8563 CLEAR_BIT(aborted_tags, tmp_slot); 8564 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8565 8566 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_ABORTED); 8567 } 8568 8569 /* Send up reset packets with SATA_PKT_RESET. */ 8570 while (reset_tags) { 8571 if (rdwr_pmult_cmd_in_progress) { 8572 satapkt = ahci_portp->ahciport_rdwr_pmult_pkt; 8573 ASSERT(satapkt != NULL); 8574 ASSERT(aborted_tags == 0x1); 8575 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8576 "ahci_mop_commands: sending up " 8577 "rdwr pmult pkt 0x%p with SATA_PKT_RESET", 8578 (void *)satapkt); 8579 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 8580 break; 8581 } 8582 8583 tmp_slot = ddi_ffs(reset_tags) - 1; 8584 if (tmp_slot == -1) { 8585 break; 8586 } 8587 8588 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8589 ASSERT(satapkt != NULL); 8590 8591 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8592 "sending up pkt 0x%p with SATA_PKT_RESET", 8593 (void *)satapkt); 8594 8595 if (ncq_cmd_in_progress) 8596 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8597 tmp_slot); 8598 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8599 CLEAR_BIT(reset_tags, tmp_slot); 8600 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8601 8602 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 8603 } 8604 8605 /* Send up unfinished packets with SATA_PKT_RESET */ 8606 while (unfinished_tags) { 8607 tmp_slot = ddi_ffs(unfinished_tags) - 1; 8608 if (tmp_slot == -1) { 8609 break; 8610 } 8611 8612 satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 8613 ASSERT(satapkt != NULL); 8614 8615 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: " 8616 "sending up pkt 0x%p with SATA_PKT_RESET", 8617 (void *)satapkt); 8618 8619 if (ncq_cmd_in_progress) 8620 CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, 8621 tmp_slot); 8622 CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot); 8623 CLEAR_BIT(unfinished_tags, tmp_slot); 8624 ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL; 8625 8626 SENDUP_PACKET(ahci_portp, satapkt, SATA_PKT_RESET); 8627 } 8628 8629 ahci_portp->ahciport_mop_in_progress--; 8630 ASSERT(ahci_portp->ahciport_mop_in_progress >= 0); 8631 8632 if (ahci_portp->ahciport_mop_in_progress == 0) 8633 ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING; 8634 } 8635 8636 /* 8637 * This routine is going to first request a READ LOG EXT sata pkt from sata 8638 * module, and then deliver it to the HBA to get the ncq failure context. 8639 * The return value is the exactly failed tags. 8640 */ 8641 static uint32_t 8642 ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 8643 uint8_t port) 8644 { 8645 sata_device_t sdevice; 8646 sata_pkt_t *rdlog_spkt, *spkt; 8647 ddi_dma_handle_t buf_dma_handle; 8648 ahci_addr_t addr; 8649 int loop_count; 8650 int rval; 8651 int failed_slot; 8652 uint32_t failed_tags = 0; 8653 struct sata_ncq_error_recovery_page *ncq_err_page; 8654 8655 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp, 8656 "ahci_get_rdlogext_data enter: port %d", port); 8657 8658 /* Prepare the sdevice data */ 8659 bzero((void *)&sdevice, sizeof (sata_device_t)); 8660 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8661 8662 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8663 sdevice.satadev_addr.pmport = 0; 8664 8665 /* Translate sata_device.satadev_addr -> ahci_addr */ 8666 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 8667 8668 /* 8669 * Call the sata hba interface to get a rdlog spkt 8670 */ 8671 loop_count = 0; 8672 loop: 8673 rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 8674 &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ); 8675 if (rdlog_spkt == NULL) { 8676 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 8677 /* Sleep for a while */ 8678 drv_usecwait(AHCI_10MS_USECS); 8679 goto loop; 8680 } 8681 /* Timed out after 1s */ 8682 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8683 "failed to get rdlog spkt for port %d", port); 8684 return (failed_tags); 8685 } 8686 8687 ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 8688 8689 /* 8690 * This flag is used to handle the specific error recovery when the 8691 * READ LOG EXT command gets a failure (fatal error or time-out). 8692 */ 8693 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT; 8694 8695 /* 8696 * This start is not supposed to fail because after port is restarted, 8697 * the whole command list is empty. 8698 */ 8699 ahci_portp->ahciport_err_retri_pkt = rdlog_spkt; 8700 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt); 8701 ahci_portp->ahciport_err_retri_pkt = NULL; 8702 8703 /* Remove the flag after READ LOG EXT command is completed */ 8704 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RDLOGEXT; 8705 8706 if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 8707 /* Update the request log data */ 8708 buf_dma_handle = *(ddi_dma_handle_t *) 8709 (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 8710 rval = ddi_dma_sync(buf_dma_handle, 0, 0, 8711 DDI_DMA_SYNC_FORKERNEL); 8712 if (rval == DDI_SUCCESS) { 8713 ncq_err_page = 8714 (struct sata_ncq_error_recovery_page *)rdlog_spkt-> 8715 satapkt_cmd.satacmd_bp->b_un.b_addr; 8716 8717 /* Get the failed tag */ 8718 failed_slot = ncq_err_page->ncq_tag; 8719 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8720 "ahci_get_rdlogext_data: port %d " 8721 "failed slot %d", port, failed_slot); 8722 if (failed_slot & NQ) { 8723 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8724 "the failed slot is not a valid tag", NULL); 8725 goto out; 8726 } 8727 8728 failed_slot &= NCQ_TAG_MASK; 8729 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 8730 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8731 "ahci_get_rdlogext_data: failed spkt 0x%p", 8732 (void *)spkt); 8733 if (spkt == NULL) { 8734 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8735 "the failed slot spkt is NULL", NULL); 8736 goto out; 8737 } 8738 8739 failed_tags = 0x1 << failed_slot; 8740 8741 /* Fill out the error context */ 8742 ahci_copy_ncq_err_page(&spkt->satapkt_cmd, 8743 ncq_err_page); 8744 ahci_update_sata_registers(ahci_ctlp, port, 8745 &spkt->satapkt_device); 8746 } 8747 } 8748 out: 8749 sata_free_error_retrieval_pkt(rdlog_spkt); 8750 8751 return (failed_tags); 8752 } 8753 8754 /* 8755 * This routine is going to first request a REQUEST SENSE sata pkt from sata 8756 * module, and then deliver it to the HBA to get the sense data and copy 8757 * the sense data back to the orignal failed sata pkt, and free the REQUEST 8758 * SENSE sata pkt later. 8759 */ 8760 static void 8761 ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 8762 uint8_t port, sata_pkt_t *spkt) 8763 { 8764 sata_device_t sdevice; 8765 sata_pkt_t *rs_spkt; 8766 sata_cmd_t *sata_cmd; 8767 ddi_dma_handle_t buf_dma_handle; 8768 ahci_addr_t addr; 8769 int loop_count; 8770 #if AHCI_DEBUG 8771 struct scsi_extended_sense *rqsense; 8772 #endif 8773 8774 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8775 "ahci_get_rqsense_data enter: port %d", port); 8776 8777 /* Prepare the sdevice data */ 8778 bzero((void *)&sdevice, sizeof (sata_device_t)); 8779 sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port]; 8780 8781 sdevice.satadev_addr.qual = SATA_ADDR_DCPORT; 8782 sdevice.satadev_addr.pmport = 0; 8783 8784 /* Translate sata_device.satadev_addr -> ahci_addr */ 8785 ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr); 8786 8787 sata_cmd = &spkt->satapkt_cmd; 8788 8789 /* 8790 * Call the sata hba interface to get a rs spkt 8791 */ 8792 loop_count = 0; 8793 loop: 8794 rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip, 8795 &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI); 8796 if (rs_spkt == NULL) { 8797 if (loop_count++ < AHCI_POLLRATE_GET_SPKT) { 8798 /* Sleep for a while */ 8799 drv_usecwait(AHCI_10MS_USECS); 8800 goto loop; 8801 8802 } 8803 /* Timed out after 1s */ 8804 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8805 "failed to get rs spkt for port %d", port); 8806 return; 8807 } 8808 8809 ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH); 8810 8811 /* 8812 * This flag is used to handle the specific error recovery when the 8813 * REQUEST SENSE command gets a faiure (fatal error or time-out). 8814 */ 8815 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE; 8816 8817 /* 8818 * This start is not supposed to fail because after port is restarted, 8819 * the whole command list is empty. 8820 */ 8821 ahci_portp->ahciport_err_retri_pkt = rs_spkt; 8822 (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt); 8823 ahci_portp->ahciport_err_retri_pkt = NULL; 8824 8825 /* Remove the flag after REQUEST SENSE command is completed */ 8826 ahci_portp->ahciport_flags &= ~ AHCI_PORT_FLAG_RQSENSE; 8827 8828 if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) { 8829 /* Update the request sense data */ 8830 buf_dma_handle = *(ddi_dma_handle_t *) 8831 (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle); 8832 (void) ddi_dma_sync(buf_dma_handle, 0, 0, 8833 DDI_DMA_SYNC_FORKERNEL); 8834 /* Copy the request sense data */ 8835 bcopy(rs_spkt-> 8836 satapkt_cmd.satacmd_bp->b_un.b_addr, 8837 &sata_cmd->satacmd_rqsense, 8838 SATA_ATAPI_MIN_RQSENSE_LEN); 8839 #if AHCI_DEBUG 8840 rqsense = (struct scsi_extended_sense *) 8841 sata_cmd->satacmd_rqsense; 8842 8843 /* Dump the sense data */ 8844 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL); 8845 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 8846 "Sense data for satapkt %p ATAPI cmd 0x%x", 8847 spkt, sata_cmd->satacmd_acdb[0]); 8848 AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, 8849 " es_code 0x%x es_class 0x%x " 8850 "es_key 0x%x es_add_code 0x%x " 8851 "es_qual_code 0x%x", 8852 rqsense->es_code, rqsense->es_class, 8853 rqsense->es_key, rqsense->es_add_code, 8854 rqsense->es_qual_code); 8855 #endif 8856 } 8857 8858 sata_free_error_retrieval_pkt(rs_spkt); 8859 } 8860 8861 /* 8862 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover, 8863 * the port must be restarted. When the HBA detects thus error, it may try 8864 * to abort a transfer. And if the transfer was aborted, the device is 8865 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both 8866 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows 8867 * that the device is in a stable status and transfers may be restarted without 8868 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set, 8869 * then the software will send the COMRESET to do the port reset. 8870 * 8871 * Software should perform the appropriate error recovery actions based on 8872 * whether non-queued commands were being issued or natived command queuing 8873 * commands were being issued. 8874 * 8875 * And software will complete the command that had the error with error mark 8876 * to higher level software. 8877 * 8878 * Fatal errors include the following: 8879 * PxIS.IFS - Interface Fatal Error Status 8880 * PxIS.HBDS - Host Bus Data Error Status 8881 * PxIS.HBFS - Host Bus Fatal Error Status 8882 * PxIS.TFES - Task File Error Status 8883 * 8884 * WARNING!!! ahciport_mutex should be acquired before the function is called. 8885 */ 8886 static void 8887 ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 8888 ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status) 8889 { 8890 uint32_t port_cmd_status; 8891 uint32_t slot_status = 0; 8892 uint32_t failed_tags = 0; 8893 int failed_slot; 8894 int reset_flag = 0, flag = 0; 8895 ahci_fis_d2h_register_t *ahci_rcvd_fisp; 8896 sata_cmd_t *sata_cmd = NULL; 8897 sata_pkt_t *spkt = NULL; 8898 #if AHCI_DEBUG 8899 ahci_cmd_header_t *cmd_header; 8900 #endif 8901 uint8_t port = addrp->aa_port; 8902 8903 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 8904 "ahci_fatal_error_recovery_handler enter: port %d", port); 8905 8906 /* Port multiplier error */ 8907 if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) { 8908 /* FBS code is neither completed nor tested. */ 8909 ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp, 8910 port, intr_status); 8911 8912 /* Force a port reset */ 8913 flag = AHCI_PORT_RESET; 8914 } 8915 8916 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 8917 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8918 8919 /* Read PxCI to see which commands are still outstanding */ 8920 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8921 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 8922 8923 /* 8924 * Read PxCMD.CCS to determine the slot that the HBA 8925 * was processing when the error occurred. 8926 */ 8927 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8928 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 8929 failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 8930 AHCI_CMD_STATUS_CCS_SHIFT; 8931 8932 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8933 spkt = ahci_portp->ahciport_err_retri_pkt; 8934 ASSERT(spkt != NULL); 8935 } else { 8936 spkt = ahci_portp->ahciport_slot_pkts[failed_slot]; 8937 if (spkt == NULL) { 8938 /* May happen when interface errors occur? */ 8939 goto next; 8940 } 8941 } 8942 8943 #if AHCI_DEBUG 8944 /* 8945 * Debugging purpose... 8946 */ 8947 if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) { 8948 cmd_header = 8949 &ahci_portp->ahciport_cmd_list[failed_slot]; 8950 AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 8951 "ahci_fatal_error_recovery_handler: port %d, " 8952 "PRD Byte Count = 0x%x, " 8953 "ahciport_prd_bytecounts = 0x%x", port, 8954 cmd_header->ahcich_prd_byte_count, 8955 ahci_portp->ahciport_prd_bytecounts[failed_slot]); 8956 } 8957 #endif 8958 8959 sata_cmd = &spkt->satapkt_cmd; 8960 8961 /* Fill out the status and error registers for PxIS.TFES */ 8962 if (intr_status & AHCI_INTR_STATUS_TFES) { 8963 ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis-> 8964 ahcirf_d2h_register_fis); 8965 8966 /* Copy the error context back to the sata_cmd */ 8967 ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp); 8968 } 8969 8970 /* The failed command must be one of the outstanding commands */ 8971 failed_tags = 0x1 << failed_slot; 8972 ASSERT(failed_tags & slot_status); 8973 8974 /* Update the sata registers, especially PxSERR register */ 8975 ahci_update_sata_registers(ahci_ctlp, port, 8976 &spkt->satapkt_device); 8977 8978 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 8979 /* Read PxSACT to see which commands are still outstanding */ 8980 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 8981 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 8982 } 8983 next: 8984 8985 #if AHCI_DEBUG 8986 /* 8987 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 8988 * set, it means a fatal error happened after REQUEST SENSE command 8989 * or READ LOG EXT command is delivered to the HBA during the error 8990 * recovery process. At this time, the only outstanding command is 8991 * supposed to be REQUEST SENSE command or READ LOG EXT command. 8992 */ 8993 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 8994 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 8995 "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE " 8996 "command or READ LOG EXT command for error data retrieval " 8997 "failed", port); 8998 ASSERT(slot_status == 0x1); 8999 ASSERT(failed_slot == 0); 9000 ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] == 9001 SCMD_REQUEST_SENSE || 9002 spkt->satapkt_cmd.satacmd_cmd_reg == 9003 SATAC_READ_LOG_EXT); 9004 } 9005 #endif 9006 9007 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9008 ahci_portp->ahciport_mop_in_progress++; 9009 9010 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9011 port, flag, &reset_flag); 9012 9013 /* 9014 * Won't retrieve error information: 9015 * 1. Port reset was involved to recover 9016 * 2. Device is gone 9017 * 3. IDENTIFY DEVICE command sent to ATAPI device 9018 * 4. REQUEST SENSE or READ LOG EXT command during error recovery 9019 */ 9020 if (reset_flag || 9021 ahci_portp->ahciport_device_type == SATA_DTYPE_NONE || 9022 spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE || 9023 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9024 goto out; 9025 9026 /* 9027 * Deliver READ LOG EXT to gather information about the error when 9028 * a COMRESET has not been performed as part of the error recovery 9029 * during NCQ command processing. 9030 */ 9031 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9032 failed_tags = ahci_get_rdlogext_data(ahci_ctlp, 9033 ahci_portp, port); 9034 goto out; 9035 } 9036 9037 /* 9038 * Deliver REQUEST SENSE for ATAPI command to gather information about 9039 * the error when a COMRESET has not been performed as part of the 9040 * error recovery. 9041 */ 9042 if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI) 9043 ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt); 9044 out: 9045 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9046 "ahci_fatal_error_recovery_handler: port %d fatal error " 9047 "occurred slot_status = 0x%x, pending_tags = 0x%x, " 9048 "pending_ncq_tags = 0x%x failed_tags = 0x%x", 9049 port, slot_status, ahci_portp->ahciport_pending_tags, 9050 ahci_portp->ahciport_pending_ncq_tags, failed_tags); 9051 9052 ahci_mop_commands(ahci_ctlp, 9053 ahci_portp, 9054 slot_status, 9055 failed_tags, /* failed tags */ 9056 0, /* timeout tags */ 9057 0, /* aborted tags */ 9058 0); /* reset tags */ 9059 } 9060 9061 /* 9062 * Used to recovery a PMULT pmport fatal error under FIS-based switching. 9063 * 1. device specific.PxFBS.SDE=1 9064 * 2. Non-Deivce specific. 9065 * Nothing will be done when Command-based switching is employed. 9066 * 9067 * Currently code is neither completed nor tested. 9068 * 9069 * WARNING!!! ahciport_mutex should be acquired before the function 9070 * is called. 9071 */ 9072 static void 9073 ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp, 9074 ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status) 9075 { 9076 #ifndef __lock_lint 9077 _NOTE(ARGUNUSED(intr_status)) 9078 #endif 9079 uint32_t port_fbs_ctrl; 9080 int loop_count = 0; 9081 ahci_addr_t addr; 9082 9083 /* Nothing will be done under Command-based switching. */ 9084 if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS)) 9085 return; 9086 9087 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9088 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9089 9090 if (!(port_fbs_ctrl & AHCI_FBS_EN)) 9091 /* FBS is not enabled. */ 9092 return; 9093 9094 /* Problem's getting complicated now. */ 9095 /* 9096 * If FIS-based switching is used, we need to check 9097 * the PxFBS to see the error type. 9098 */ 9099 port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9100 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port)); 9101 9102 /* Refer to spec(v1.2) 9.3.6.1 */ 9103 if (port_fbs_ctrl & AHCI_FBS_SDE) { 9104 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, 9105 "A Device Sepcific Error: port %d", port); 9106 /* 9107 * Controller has paused commands for all other 9108 * sub-devices until PxFBS.DEC is set. 9109 */ 9110 ahci_reject_all_abort_pkts(ahci_ctlp, 9111 ahci_portp, 0); 9112 9113 ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, 9114 (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port), 9115 port_fbs_ctrl | AHCI_FBS_DEC); 9116 9117 /* 9118 * Wait controller clear PxFBS.DEC, 9119 * then we can continue. 9120 */ 9121 loop_count = 0; 9122 do { 9123 port_fbs_ctrl = ddi_get32(ahci_ctlp-> 9124 ahcictl_ahci_acc_handle, (uint32_t *) 9125 AHCI_PORT_PxFBS(ahci_ctlp, port)); 9126 9127 if (loop_count++ > 1000) 9128 /* 9129 * Esclate the error. Follow 9130 * non-device specific error 9131 * procedure. 9132 */ 9133 return; 9134 9135 drv_usecwait(AHCI_100US_USECS); 9136 } while (port_fbs_ctrl & AHCI_FBS_DEC); 9137 9138 /* 9139 * Issue a software reset to ensure drive is in 9140 * a known state. 9141 */ 9142 (void) ahci_software_reset(ahci_ctlp, 9143 ahci_portp, &addr); 9144 9145 } else { 9146 9147 /* Process Non-Device Specific Error. */ 9148 /* This will be handled later on. */ 9149 cmn_err(CE_NOTE, "!FBS is not supported now."); 9150 } 9151 } 9152 /* 9153 * Handle events - fatal error recovery 9154 */ 9155 static void 9156 ahci_events_handler(void *args) 9157 { 9158 ahci_event_arg_t *ahci_event_arg; 9159 ahci_ctl_t *ahci_ctlp; 9160 ahci_port_t *ahci_portp; 9161 ahci_addr_t *addrp; 9162 uint32_t event; 9163 9164 ahci_event_arg = (ahci_event_arg_t *)args; 9165 9166 ahci_ctlp = ahci_event_arg->ahciea_ctlp; 9167 ahci_portp = ahci_event_arg->ahciea_portp; 9168 addrp = ahci_event_arg->ahciea_addrp; 9169 event = ahci_event_arg->ahciea_event; 9170 9171 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp, 9172 "ahci_events_handler enter: port %d intr_status = 0x%x", 9173 ahci_portp->ahciport_port_num, event); 9174 9175 mutex_enter(&ahci_portp->ahciport_mutex); 9176 9177 /* 9178 * ahci_intr_phyrdy_change() may have rendered it to 9179 * SATA_DTYPE_NONE. 9180 */ 9181 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9182 AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp, 9183 "ahci_events_handler: port %d no device attached, " 9184 "and just return without doing anything", 9185 ahci_portp->ahciport_port_num); 9186 goto out; 9187 } 9188 9189 if (event & (AHCI_INTR_STATUS_IFS | 9190 AHCI_INTR_STATUS_HBDS | 9191 AHCI_INTR_STATUS_HBFS | 9192 AHCI_INTR_STATUS_TFES)) 9193 ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp, 9194 addrp, event); 9195 9196 out: 9197 mutex_exit(&ahci_portp->ahciport_mutex); 9198 } 9199 9200 /* 9201 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they 9202 * detect there are some commands which are timed out. 9203 */ 9204 static void 9205 ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, 9206 uint8_t port, uint32_t tmp_timeout_tags) 9207 { 9208 uint32_t slot_status = 0; 9209 uint32_t finished_tags = 0; 9210 uint32_t timeout_tags = 0; 9211 9212 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp, 9213 "ahci_timeout_pkts enter: port %d", port); 9214 9215 mutex_enter(&ahci_portp->ahciport_mutex); 9216 9217 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) || 9218 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) || 9219 ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9220 /* Read PxCI to see which commands are still outstanding */ 9221 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9222 (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port)); 9223 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9224 /* Read PxSACT to see which commands are still outstanding */ 9225 slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9226 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9227 } 9228 9229 #if AHCI_DEBUG 9230 /* 9231 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is 9232 * set, it means a fatal error happened after REQUEST SENSE command 9233 * or READ LOG EXT command is delivered to the HBA during the error 9234 * recovery process. At this time, the only outstanding command is 9235 * supposed to be REQUEST SENSE command or READ LOG EXT command. 9236 */ 9237 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) { 9238 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9239 "ahci_timeout_pkts called while REQUEST SENSE " 9240 "command or READ LOG EXT command for error recovery " 9241 "timed out timeout_tags = 0x%x, slot_status = 0x%x, " 9242 "pending_tags = 0x%x, pending_ncq_tags = 0x%x", 9243 tmp_timeout_tags, slot_status, 9244 ahci_portp->ahciport_pending_tags, 9245 ahci_portp->ahciport_pending_ncq_tags); 9246 ASSERT(slot_status == 0x1); 9247 } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9248 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp, 9249 "ahci_timeout_pkts called while executing R/W PMULT " 9250 "command timeout_tags = 0x%x, slot_status = 0x%x", 9251 tmp_timeout_tags, slot_status); 9252 ASSERT(slot_status == 0x1); 9253 } 9254 #endif 9255 9256 ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING; 9257 ahci_portp->ahciport_mop_in_progress++; 9258 9259 (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp, 9260 port, AHCI_PORT_RESET, NULL); 9261 9262 /* 9263 * Re-identify timeout tags because some previously checked commands 9264 * could already complete. 9265 */ 9266 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9267 finished_tags = ahci_portp->ahciport_pending_tags & 9268 ~slot_status & AHCI_SLOT_MASK(ahci_ctlp); 9269 timeout_tags = tmp_timeout_tags & ~finished_tags; 9270 9271 AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp, 9272 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9273 "timeout_tags = 0x%x, port_cmd_issue = 0x%x, " 9274 "pending_tags = 0x%x ", 9275 port, finished_tags, timeout_tags, 9276 slot_status, ahci_portp->ahciport_pending_tags); 9277 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9278 finished_tags = ahci_portp->ahciport_pending_ncq_tags & 9279 ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp); 9280 timeout_tags = tmp_timeout_tags & ~finished_tags; 9281 9282 AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp, 9283 "ahci_timeout_pkts: port %d, finished_tags = 0x%x, " 9284 "timeout_tags = 0x%x, port_sactive = 0x%x, " 9285 "pending_ncq_tags = 0x%x ", 9286 port, finished_tags, timeout_tags, 9287 slot_status, ahci_portp->ahciport_pending_ncq_tags); 9288 } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9289 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9290 timeout_tags = tmp_timeout_tags; 9291 } 9292 9293 ahci_mop_commands(ahci_ctlp, 9294 ahci_portp, 9295 slot_status, 9296 0, /* failed tags */ 9297 timeout_tags, /* timeout tags */ 9298 0, /* aborted tags */ 9299 0); /* reset tags */ 9300 9301 mutex_exit(&ahci_portp->ahciport_mutex); 9302 } 9303 9304 /* 9305 * Watchdog handler kicks in every 5 seconds to timeout any commands pending 9306 * for long time. 9307 */ 9308 static void 9309 ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp) 9310 { 9311 ahci_port_t *ahci_portp; 9312 sata_pkt_t *spkt; 9313 uint32_t pending_tags; 9314 uint32_t timeout_tags; 9315 uint32_t port_cmd_status; 9316 uint32_t port_sactive; 9317 uint8_t port; 9318 int tmp_slot; 9319 int current_slot; 9320 uint32_t current_tags; 9321 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9322 /* max number of cycles this packet should survive */ 9323 int max_life_cycles; 9324 9325 /* how many cycles this packet survived so far */ 9326 int watched_cycles; 9327 9328 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9329 9330 AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, 9331 "ahci_watchdog_handler entered", NULL); 9332 9333 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 9334 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 9335 continue; 9336 } 9337 9338 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 9339 9340 mutex_enter(&ahci_portp->ahciport_mutex); 9341 if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) { 9342 mutex_exit(&ahci_portp->ahciport_mutex); 9343 continue; 9344 } 9345 9346 /* Skip the check for those ports in error recovery */ 9347 if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && 9348 !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) { 9349 mutex_exit(&ahci_portp->ahciport_mutex); 9350 continue; 9351 } 9352 9353 pending_tags = 0; 9354 port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle, 9355 (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port)); 9356 9357 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) || 9358 RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) { 9359 current_slot = 0; 9360 pending_tags = 0x1; 9361 } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9362 current_slot = 9363 (port_cmd_status & AHCI_CMD_STATUS_CCS) >> 9364 AHCI_CMD_STATUS_CCS_SHIFT; 9365 pending_tags = ahci_portp->ahciport_pending_tags; 9366 } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9367 port_sactive = ddi_get32( 9368 ahci_ctlp->ahcictl_ahci_acc_handle, 9369 (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port)); 9370 current_tags = port_sactive & 9371 ~port_cmd_status & 9372 AHCI_NCQ_SLOT_MASK(ahci_portp); 9373 pending_tags = ahci_portp->ahciport_pending_ncq_tags; 9374 } 9375 9376 timeout_tags = 0; 9377 while (pending_tags) { 9378 tmp_slot = ddi_ffs(pending_tags) - 1; 9379 if (tmp_slot == -1) { 9380 break; 9381 } 9382 9383 if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) 9384 spkt = ahci_portp->ahciport_err_retri_pkt; 9385 else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) 9386 spkt = ahci_portp->ahciport_rdwr_pmult_pkt; 9387 else 9388 spkt = ahci_portp->ahciport_slot_pkts[tmp_slot]; 9389 9390 if ((spkt != NULL) && spkt->satapkt_time && 9391 !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) { 9392 /* 9393 * We are overloading satapkt_hba_driver_private 9394 * with watched_cycle count. 9395 * 9396 * If a packet has survived for more than it's 9397 * max life cycles, it is a candidate for time 9398 * out. 9399 */ 9400 watched_cycles = (int)(intptr_t) 9401 spkt->satapkt_hba_driver_private; 9402 watched_cycles++; 9403 max_life_cycles = (spkt->satapkt_time + 9404 ahci_watchdog_timeout - 1) / 9405 ahci_watchdog_timeout; 9406 9407 spkt->satapkt_hba_driver_private = 9408 (void *)(intptr_t)watched_cycles; 9409 9410 if (watched_cycles <= max_life_cycles) 9411 goto next; 9412 9413 #if AHCI_DEBUG 9414 if (NCQ_CMD_IN_PROGRESS(ahci_portp)) { 9415 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9416 ahci_ctlp, "watchdog: the current " 9417 "tags is 0x%x", current_tags); 9418 } else { 9419 AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, 9420 ahci_ctlp, "watchdog: the current " 9421 "slot is %d", current_slot); 9422 } 9423 #endif 9424 9425 /* 9426 * We need to check whether the HBA has 9427 * begun to execute the command, if not, 9428 * then re-set the timer of the command. 9429 */ 9430 if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) && 9431 (tmp_slot != current_slot) || 9432 NCQ_CMD_IN_PROGRESS(ahci_portp) && 9433 ((0x1 << tmp_slot) & current_tags)) { 9434 spkt->satapkt_hba_driver_private = 9435 (void *)(intptr_t)0; 9436 } else { 9437 timeout_tags |= (0x1 << tmp_slot); 9438 cmn_err(CE_WARN, "!ahci%d: watchdog " 9439 "port %d satapkt 0x%p timed out\n", 9440 instance, port, (void *)spkt); 9441 } 9442 } 9443 next: 9444 CLEAR_BIT(pending_tags, tmp_slot); 9445 } 9446 9447 if (timeout_tags) { 9448 mutex_exit(&ahci_portp->ahciport_mutex); 9449 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9450 ahci_timeout_pkts(ahci_ctlp, ahci_portp, 9451 port, timeout_tags); 9452 mutex_enter(&ahci_ctlp->ahcictl_mutex); 9453 mutex_enter(&ahci_portp->ahciport_mutex); 9454 } 9455 9456 mutex_exit(&ahci_portp->ahciport_mutex); 9457 } 9458 9459 /* Re-install the watchdog timeout handler */ 9460 if (ahci_ctlp->ahcictl_timeout_id != 0) { 9461 ahci_ctlp->ahcictl_timeout_id = 9462 timeout((void (*)(void *))ahci_watchdog_handler, 9463 (caddr_t)ahci_ctlp, ahci_watchdog_tick); 9464 } 9465 9466 mutex_exit(&ahci_ctlp->ahcictl_mutex); 9467 } 9468 9469 /* 9470 * Fill the error context into sata_cmd for non-queued command error. 9471 */ 9472 static void 9473 ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 9474 { 9475 scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp); 9476 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 9477 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 9478 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 9479 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 9480 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 9481 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 9482 9483 if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) { 9484 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 9485 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 9486 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 9487 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 9488 } 9489 } 9490 9491 /* 9492 * Fill the ncq error page into sata_cmd for queued command error. 9493 */ 9494 static void 9495 ahci_copy_ncq_err_page(sata_cmd_t *scmd, 9496 struct sata_ncq_error_recovery_page *ncq_err_page) 9497 { 9498 scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext; 9499 scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count; 9500 scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext; 9501 scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number; 9502 scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext; 9503 scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low; 9504 scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext; 9505 scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high; 9506 scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head; 9507 scmd->satacmd_status_reg = ncq_err_page->ncq_status; 9508 scmd->satacmd_error_reg = ncq_err_page->ncq_error; 9509 } 9510 9511 /* 9512 * Put the respective register value to sata_cmd_t for satacmd_flags. 9513 */ 9514 static void 9515 ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp) 9516 { 9517 if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) 9518 scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp); 9519 if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) 9520 scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp); 9521 if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) 9522 scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp); 9523 if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) 9524 scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp); 9525 if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) 9526 scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp); 9527 if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) 9528 scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp); 9529 if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) 9530 scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp); 9531 if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) 9532 scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp); 9533 if (scmd->satacmd_flags.sata_copy_out_device_reg) 9534 scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp); 9535 if (scmd->satacmd_flags.sata_copy_out_error_reg) 9536 scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp); 9537 } 9538 9539 static void 9540 ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 9541 uint32_t intr_status) 9542 { 9543 int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip); 9544 9545 if (intr_status & AHCI_INTR_STATUS_IFS) 9546 cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal " 9547 "error", instance, port); 9548 9549 if (intr_status & AHCI_INTR_STATUS_HBDS) 9550 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error", 9551 instance, port); 9552 9553 if (intr_status & AHCI_INTR_STATUS_HBFS) 9554 cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error", 9555 instance, port); 9556 9557 if (intr_status & AHCI_INTR_STATUS_TFES) 9558 cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error", 9559 instance, port); 9560 9561 cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error " 9562 "recovery", instance, port); 9563 } 9564 9565 /* 9566 * Dump the serror message to the log. 9567 */ 9568 static void 9569 ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port, 9570 uint32_t port_serror, int debug_only) 9571 { 9572 static char err_buf[512]; 9573 static char err_msg_header[16]; 9574 char *err_msg = err_buf; 9575 9576 *err_buf = '\0'; 9577 *err_msg_header = '\0'; 9578 9579 if (port_serror & SERROR_DATA_ERR_FIXED) { 9580 err_msg = strcat(err_msg, 9581 "\tRecovered Data Integrity Error (I)\n"); 9582 } 9583 9584 if (port_serror & SERROR_COMM_ERR_FIXED) { 9585 err_msg = strcat(err_msg, 9586 "\tRecovered Communication Error (M)\n"); 9587 } 9588 9589 if (port_serror & SERROR_DATA_ERR) { 9590 err_msg = strcat(err_msg, 9591 "\tTransient Data Integrity Error (T)\n"); 9592 } 9593 9594 if (port_serror & SERROR_PERSISTENT_ERR) { 9595 err_msg = strcat(err_msg, 9596 "\tPersistent Communication or Data Integrity Error (C)\n"); 9597 } 9598 9599 if (port_serror & SERROR_PROTOCOL_ERR) { 9600 err_msg = strcat(err_msg, "\tProtocol Error (P)\n"); 9601 } 9602 9603 if (port_serror & SERROR_INT_ERR) { 9604 err_msg = strcat(err_msg, "\tInternal Error (E)\n"); 9605 } 9606 9607 if (port_serror & SERROR_PHY_RDY_CHG) { 9608 err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n"); 9609 } 9610 9611 if (port_serror & SERROR_PHY_INT_ERR) { 9612 err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n"); 9613 } 9614 9615 if (port_serror & SERROR_COMM_WAKE) { 9616 err_msg = strcat(err_msg, "\tComm Wake (W)\n"); 9617 } 9618 9619 if (port_serror & SERROR_10B_TO_8B_ERR) { 9620 err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n"); 9621 } 9622 9623 if (port_serror & SERROR_DISPARITY_ERR) { 9624 err_msg = strcat(err_msg, "\tDisparity Error (D)\n"); 9625 } 9626 9627 if (port_serror & SERROR_CRC_ERR) { 9628 err_msg = strcat(err_msg, "\tCRC Error (C)\n"); 9629 } 9630 9631 if (port_serror & SERROR_HANDSHAKE_ERR) { 9632 err_msg = strcat(err_msg, "\tHandshake Error (H)\n"); 9633 } 9634 9635 if (port_serror & SERROR_LINK_SEQ_ERR) { 9636 err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n"); 9637 } 9638 9639 if (port_serror & SERROR_TRANS_ERR) { 9640 err_msg = strcat(err_msg, 9641 "\tTransport state transition error (T)\n"); 9642 } 9643 9644 if (port_serror & SERROR_FIS_TYPE) { 9645 err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n"); 9646 } 9647 9648 if (port_serror & SERROR_EXCHANGED_ERR) { 9649 err_msg = strcat(err_msg, "\tExchanged (X)\n"); 9650 } 9651 9652 if (err_msg == NULL) 9653 return; 9654 9655 if (debug_only) { 9656 (void) sprintf(err_msg_header, "port %d", port); 9657 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL); 9658 AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL); 9659 } else if (ahci_ctlp) { 9660 cmn_err(CE_WARN, "!ahci%d: %s %s", 9661 ddi_get_instance(ahci_ctlp->ahcictl_dip), 9662 err_msg_header, err_msg); 9663 9664 /* sata trace debug */ 9665 sata_trace_debug(ahci_ctlp->ahcictl_dip, 9666 "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip), 9667 err_msg_header, err_msg); 9668 } else { 9669 cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg); 9670 9671 /* sata trace debug */ 9672 sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg); 9673 } 9674 } 9675 9676 /* 9677 * Translate the sata_address_t type into the ahci_addr_t type. 9678 * sata_device.satadev_addr structure is used as source. 9679 */ 9680 static void 9681 ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd, 9682 ahci_addr_t *ahci_addrp) 9683 { 9684 sata_address_t *sata_addrp = &sd->satadev_addr; 9685 ahci_addrp->aa_port = 9686 ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport]; 9687 ahci_addrp->aa_pmport = sata_addrp->pmport; 9688 9689 switch (sata_addrp->qual) { 9690 case SATA_ADDR_DCPORT: 9691 case SATA_ADDR_CPORT: 9692 ahci_addrp->aa_qual = AHCI_ADDR_PORT; 9693 break; 9694 case SATA_ADDR_PMULT: 9695 case SATA_ADDR_PMULT_SPEC: 9696 ahci_addrp->aa_qual = AHCI_ADDR_PMULT; 9697 break; 9698 case SATA_ADDR_DPMPORT: 9699 case SATA_ADDR_PMPORT: 9700 ahci_addrp->aa_qual = AHCI_ADDR_PMPORT; 9701 break; 9702 case SATA_ADDR_NULL: 9703 default: 9704 /* something went wrong */ 9705 ahci_addrp->aa_qual = AHCI_ADDR_NULL; 9706 break; 9707 } 9708 } 9709 9710 /* 9711 * This routine is to calculate the total number of ports implemented 9712 * by the HBA. 9713 */ 9714 static int 9715 ahci_get_num_implemented_ports(uint32_t ports_implemented) 9716 { 9717 uint8_t i; 9718 int num = 0; 9719 9720 for (i = 0; i < AHCI_MAX_PORTS; i++) { 9721 if (((uint32_t)0x1 << i) & ports_implemented) 9722 num++; 9723 } 9724 9725 return (num); 9726 } 9727 9728 #if AHCI_DEBUG 9729 static void 9730 ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...) 9731 { 9732 static char name[16]; 9733 va_list ap; 9734 9735 mutex_enter(&ahci_log_mutex); 9736 9737 va_start(ap, fmt); 9738 if (ahci_ctlp) { 9739 (void) sprintf(name, "ahci%d: ", 9740 ddi_get_instance(ahci_ctlp->ahcictl_dip)); 9741 } else { 9742 (void) sprintf(name, "ahci: "); 9743 } 9744 9745 (void) vsprintf(ahci_log_buf, fmt, ap); 9746 va_end(ap); 9747 9748 cmn_err(level, "%s%s", name, ahci_log_buf); 9749 9750 mutex_exit(&ahci_log_mutex); 9751 } 9752 #endif 9753 9754 /* 9755 * quiesce(9E) entry point. 9756 * 9757 * This function is called when the system is single-threaded at high 9758 * PIL with preemption disabled. Therefore, this function must not be 9759 * blocked. 9760 * 9761 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 9762 * DDI_FAILURE indicates an error condition and should almost never happen. 9763 */ 9764 static int 9765 ahci_quiesce(dev_info_t *dip) 9766 { 9767 ahci_ctl_t *ahci_ctlp; 9768 ahci_port_t *ahci_portp; 9769 int instance, port; 9770 9771 instance = ddi_get_instance(dip); 9772 ahci_ctlp = ddi_get_soft_state(ahci_statep, instance); 9773 9774 if (ahci_ctlp == NULL) 9775 return (DDI_FAILURE); 9776 9777 #if AHCI_DEBUG 9778 ahci_debug_flags = 0; 9779 #endif 9780 9781 /* disable all the interrupts. */ 9782 ahci_disable_all_intrs(ahci_ctlp); 9783 9784 for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) { 9785 if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) { 9786 continue; 9787 } 9788 9789 ahci_portp = ahci_ctlp->ahcictl_ports[port]; 9790 9791 /* 9792 * Stop the port by clearing PxCMD.ST 9793 * 9794 * Here we must disable the port interrupt because 9795 * ahci_disable_all_intrs only clear GHC.IE, and IS 9796 * register will be still set if PxIE is enabled. 9797 * When ahci shares one IRQ with other drivers, the 9798 * intr handler may claim the intr mistakenly. 9799 */ 9800 ahci_disable_port_intrs(ahci_ctlp, port); 9801 (void) ahci_put_port_into_notrunning_state(ahci_ctlp, 9802 ahci_portp, port); 9803 } 9804 9805 return (DDI_SUCCESS); 9806 } 9807