1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * Solaris SCSI RDMA Protocol Target (SRP) transport port provider 29 * module for the COMSTAR framework. 30 */ 31 32 #include <sys/cpuvar.h> 33 #include <sys/types.h> 34 #include <sys/conf.h> 35 #include <sys/stat.h> 36 #include <sys/file.h> 37 #include <sys/ddi.h> 38 #include <sys/sunddi.h> 39 #include <sys/modctl.h> 40 #include <sys/sysmacros.h> 41 #include <sys/sdt.h> 42 #include <sys/taskq.h> 43 44 #include <stmf.h> 45 #include <stmf_ioctl.h> 46 #include <portif.h> 47 48 #include "srp.h" 49 #include "srpt_impl.h" 50 #include "srpt_ioc.h" 51 #include "srpt_stp.h" 52 #include "srpt_cm.h" 53 #include "srpt_ioctl.h" 54 55 #define SRPT_NAME_VERSION "COMSTAR SRP Target" 56 57 /* 58 * srpt_send_msg_depth - Tunable parameter that specifies the 59 * maximum messages that could be in flight for a channel. 60 */ 61 uint16_t srpt_send_msg_depth = SRPT_DEFAULT_SEND_MSG_DEPTH; 62 63 /* 64 * srpt_errlevel -- determine which error conditions are logged 65 */ 66 uint_t srpt_errlevel = SRPT_LOG_DEFAULT_LEVEL; 67 68 srpt_ctxt_t *srpt_ctxt; 69 70 /* 71 * DDI entry points. 72 */ 73 static int srpt_drv_attach(dev_info_t *, ddi_attach_cmd_t); 74 static int srpt_drv_detach(dev_info_t *, ddi_detach_cmd_t); 75 static int srpt_drv_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **); 76 static int srpt_drv_open(dev_t *, int, int, cred_t *); 77 static int srpt_drv_close(dev_t, int, int, cred_t *); 78 static int srpt_drv_ioctl(dev_t, int, intptr_t, int, cred_t *, int *); 79 80 /* helper functions */ 81 static int srpt_disable_srp_services(void); 82 static int srpt_enable_srp_services(void); 83 static int srpt_ibdma_ops_load(srpt_ibdma_ops_t *); 84 static void srpt_ibdma_ops_unload(srpt_ibdma_ops_t *); 85 86 extern struct mod_ops mod_miscops; 87 88 static struct cb_ops srpt_cb_ops = { 89 srpt_drv_open, /* cb_open */ 90 srpt_drv_close, /* cb_close */ 91 nodev, /* cb_strategy */ 92 nodev, /* cb_print */ 93 nodev, /* cb_dump */ 94 nodev, /* cb_read */ 95 nodev, /* cb_write */ 96 srpt_drv_ioctl, /* cb_ioctl */ 97 nodev, /* cb_devmap */ 98 nodev, /* cb_mmap */ 99 nodev, /* cb_segmap */ 100 nochpoll, /* cb_chpoll */ 101 ddi_prop_op, /* cb_prop_op */ 102 NULL, /* cb_streamtab */ 103 D_MP, /* cb_flag */ 104 CB_REV, /* cb_rev */ 105 nodev, /* cb_aread */ 106 nodev, /* cb_awrite */ 107 }; 108 109 static struct dev_ops srpt_dev_ops = { 110 DEVO_REV, /* devo_rev */ 111 0, /* devo_refcnt */ 112 srpt_drv_getinfo, /* devo_getinfo */ 113 nulldev, /* devo_identify */ 114 nulldev, /* devo_probe */ 115 srpt_drv_attach, /* devo_attach */ 116 srpt_drv_detach, /* devo_detach */ 117 nodev, /* devo_reset */ 118 &srpt_cb_ops, /* devo_cb_ops */ 119 NULL, /* devo_bus_ops */ 120 NULL, /* devo_power */ 121 ddi_quiesce_not_needed, /* quiesce */ 122 }; 123 124 static struct modldrv modldrv = { 125 &mod_driverops, 126 SRPT_NAME_VERSION, 127 &srpt_dev_ops, 128 }; 129 130 static struct modlinkage srpt_modlinkage = { 131 MODREV_1, 132 &modldrv, 133 NULL, 134 }; 135 136 static char srpt_pp_name[] = "srpt"; 137 138 /* 139 * Prototypes 140 */ 141 static void srpt_pp_cb(stmf_port_provider_t *, int, void *, uint32_t); 142 143 /* 144 * _init() 145 */ 146 int 147 _init(void) 148 { 149 int status; 150 151 /* 152 * Global one time initialization. 153 */ 154 srpt_ctxt = kmem_zalloc(sizeof (srpt_ctxt_t), KM_SLEEP); 155 ASSERT(srpt_ctxt != NULL); 156 rw_init(&srpt_ctxt->sc_rwlock, NULL, RW_DRIVER, NULL); 157 158 /* Start-up state is DISABLED. SMF will tell us if we should enable. */ 159 srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; 160 161 status = mod_install(&srpt_modlinkage); 162 if (status != DDI_SUCCESS) { 163 cmn_err(CE_CONT, "_init, failed mod_install %d", status); 164 rw_destroy(&srpt_ctxt->sc_rwlock); 165 kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); 166 srpt_ctxt = NULL; 167 } 168 169 return (status); 170 } 171 172 /* 173 * _info() 174 */ 175 int 176 _info(struct modinfo *modinfop) 177 { 178 return (mod_info(&srpt_modlinkage, modinfop)); 179 } 180 181 /* 182 * _fini() 183 */ 184 int 185 _fini(void) 186 { 187 int status; 188 189 status = mod_remove(&srpt_modlinkage); 190 if (status != DDI_SUCCESS) { 191 return (status); 192 } 193 194 rw_destroy(&srpt_ctxt->sc_rwlock); 195 kmem_free(srpt_ctxt, sizeof (srpt_ctxt_t)); 196 srpt_ctxt = NULL; 197 198 return (status); 199 } 200 201 /* 202 * DDI entry points. 203 */ 204 205 /* 206 * srpt_getinfo() 207 */ 208 /* ARGSUSED */ 209 static int 210 srpt_drv_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **result) 211 { 212 213 switch (cmd) { 214 case DDI_INFO_DEVT2DEVINFO: 215 *result = srpt_ctxt->sc_dip; 216 return (DDI_SUCCESS); 217 218 case DDI_INFO_DEVT2INSTANCE: 219 *result = NULL; 220 return (DDI_SUCCESS); 221 222 default: 223 break; 224 } 225 return (DDI_FAILURE); 226 } 227 228 /* 229 * srpt_drv_attach() 230 */ 231 static int 232 srpt_drv_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 233 { 234 int status; 235 236 switch (cmd) { 237 case DDI_ATTACH: 238 break; 239 240 case DDI_RESUME: 241 return (DDI_SUCCESS); 242 243 default: 244 return (DDI_FAILURE); 245 } 246 247 /* 248 * We only allow a single instance. 249 */ 250 if (ddi_get_instance(dip) != 0) { 251 SRPT_DPRINTF_L1("drv_attach, error non-zero instance"); 252 return (DDI_FAILURE); 253 } 254 255 /* 256 * Create minor node that might ultimately be used to create 257 * targets outside of srpt. 258 */ 259 status = ddi_create_minor_node(dip, ddi_get_name(dip), 260 S_IFCHR, 0, DDI_PSEUDO, 0); 261 if (status != DDI_SUCCESS) { 262 SRPT_DPRINTF_L1("drv_attach, minor node creation error (%d)", 263 status); 264 return (DDI_FAILURE); 265 } 266 267 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 268 srpt_ctxt->sc_dip = dip; 269 rw_exit(&srpt_ctxt->sc_rwlock); 270 271 return (DDI_SUCCESS); 272 } 273 274 /* 275 * srpt_enable_srp_services() 276 * 277 * Caller must be holding the sc_rwlock as RW_WRITER. 278 */ 279 static int 280 srpt_enable_srp_services(void) 281 { 282 int status; 283 srpt_ioc_t *ioc; 284 285 ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); 286 287 SRPT_DPRINTF_L3("srpt_enable_srp_services"); 288 289 /* Register the port provider */ 290 srpt_ctxt->sc_pp = (stmf_port_provider_t *) 291 stmf_alloc(STMF_STRUCT_PORT_PROVIDER, 0, 0); 292 srpt_ctxt->sc_pp->pp_portif_rev = PORTIF_REV_1; 293 srpt_ctxt->sc_pp->pp_name = srpt_pp_name; 294 srpt_ctxt->sc_pp->pp_cb = srpt_pp_cb; 295 status = stmf_register_port_provider(srpt_ctxt->sc_pp); 296 if (status != STMF_SUCCESS) { 297 SRPT_DPRINTF_L1("enable_srp: SRP port_provider registration" 298 " failed(%d)", status); 299 goto err_exit_1; 300 } 301 302 /* 303 * Initialize IB resources, creating a list of SRP I/O Controllers. 304 */ 305 status = srpt_ioc_attach(); 306 if (status != DDI_SUCCESS) { 307 SRPT_DPRINTF_L1("enable_srp: error attach I/O" 308 " Controllers (%d)", status); 309 goto err_exit_2; 310 } 311 312 if (srpt_ctxt->sc_num_iocs == 0) { 313 SRPT_DPRINTF_L2("enable_srp: no IB I/O Controllers found"); 314 status = DDI_FAILURE; 315 goto err_exit_3; 316 } 317 318 /* 319 * For each I/O Controller register the default SCSI Target Port 320 * with STMF, and prepare profile and services. SRP will not 321 * start until the associated LPORT is brought on-line. 322 */ 323 ioc = list_head(&srpt_ctxt->sc_ioc_list); 324 325 while (ioc != NULL) { 326 rw_enter(&ioc->ioc_rwlock, RW_WRITER); 327 ioc->ioc_tgt_port = srpt_stp_alloc_port(ioc, ioc->ioc_guid); 328 if (ioc->ioc_tgt_port == NULL) { 329 SRPT_DPRINTF_L1("enable_srp: alloc SCSI" 330 " Target Port error on GUID(%016llx)", 331 (u_longlong_t)ioc->ioc_guid); 332 } 333 334 rw_exit(&ioc->ioc_rwlock); 335 ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); 336 } 337 338 return (DDI_SUCCESS); 339 340 err_exit_3: 341 srpt_ioc_detach(); 342 343 err_exit_2: 344 stmf_deregister_port_provider(srpt_ctxt->sc_pp); 345 346 err_exit_1: 347 stmf_free(srpt_ctxt->sc_pp); 348 srpt_ctxt->sc_pp = NULL; 349 350 return (status); 351 } 352 353 /* 354 * srpt_drv_detach() 355 * 356 * Refuse the detach request if we have channels open on 357 * any IOC. Users should use 'svcadm disable' to shutdown 358 * active targets. 359 */ 360 /*ARGSUSED*/ 361 static int 362 srpt_drv_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 363 { 364 switch (cmd) { 365 case DDI_DETACH: 366 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 367 if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { 368 rw_exit(&srpt_ctxt->sc_rwlock); 369 return (DDI_FAILURE); 370 } 371 372 ddi_remove_minor_node(dip, NULL); 373 srpt_ctxt->sc_dip = NULL; 374 375 rw_exit(&srpt_ctxt->sc_rwlock); 376 377 break; 378 379 case DDI_SUSPEND: 380 return (DDI_FAILURE); 381 382 default: 383 return (DDI_FAILURE); 384 } 385 386 return (DDI_SUCCESS); 387 } 388 389 /* 390 * srpt_disable_srp_services() 391 * 392 * Offlines all targets, deregisters all IOCs. Caller must hold 393 * the srpt_ctxt->sc_rwlock as RW_WRITER. 394 */ 395 static int 396 srpt_disable_srp_services(void) 397 { 398 stmf_status_t stmf_status; 399 stmf_change_status_t cstatus; 400 srpt_ioc_t *ioc; 401 srpt_target_port_t *tgt; 402 int ret_status = 0; 403 404 ASSERT((rw_read_locked(&srpt_ctxt->sc_rwlock)) == 0); 405 406 /* 407 * For each I/O Controller remove all SRP services and de-register 408 * with the associated I/O Unit's IB Device Management Agent. 409 */ 410 ioc = list_head(&srpt_ctxt->sc_ioc_list); 411 412 while (ioc != NULL) { 413 /* 414 * Notify STMF to take the I/O Controller SCSI Target Port(s) 415 * off-line after we mark them as disabled so that they will 416 * stay off-line. 417 */ 418 rw_enter(&ioc->ioc_rwlock, RW_WRITER); 419 420 tgt = ioc->ioc_tgt_port; 421 if (tgt != NULL) { 422 mutex_enter(&tgt->tp_lock); 423 tgt->tp_drv_disabled = 1; 424 mutex_exit(&tgt->tp_lock); 425 426 SRPT_DPRINTF_L2("disable_srp: unbind and de-register" 427 " services for GUID(%016llx)", 428 (u_longlong_t)ioc->ioc_guid); 429 430 cstatus.st_completion_status = STMF_SUCCESS; 431 cstatus.st_additional_info = NULL; 432 433 stmf_status = stmf_ctl(STMF_CMD_LPORT_OFFLINE, 434 tgt->tp_lport, &cstatus); 435 436 /* 437 * Wait for asynchronous target off-line operation 438 * to complete and then deregister the target 439 * port. 440 */ 441 mutex_enter(&tgt->tp_lock); 442 while (tgt->tp_state != SRPT_TGT_STATE_OFFLINE) { 443 cv_wait(&tgt->tp_offline_complete, 444 &tgt->tp_lock); 445 } 446 mutex_exit(&tgt->tp_lock); 447 ioc->ioc_tgt_port = NULL; 448 449 SRPT_DPRINTF_L3("disable_srp: IOC (0x%016llx) Target" 450 " SRP off-line complete", 451 (u_longlong_t)ioc->ioc_guid); 452 453 stmf_status = srpt_stp_deregister_port(tgt); 454 if (stmf_status != STMF_SUCCESS) { 455 /* Fails if I/O is pending */ 456 if (ret_status == 0) { 457 ret_status = EBUSY; 458 } 459 SRPT_DPRINTF_L1("disable_srp: could not" 460 " de-register LPORT, err(0x%llx)", 461 (u_longlong_t)stmf_status); 462 } else { 463 (void) srpt_stp_free_port(tgt); 464 } 465 } 466 467 rw_exit(&ioc->ioc_rwlock); 468 ioc = list_next(&srpt_ctxt->sc_ioc_list, ioc); 469 } 470 471 /* don't release IOCs until all ports are deregistered */ 472 if (ret_status != 0) { 473 return (ret_status); 474 } 475 476 /* 477 * Release I/O Controller(s) resources and detach. 478 */ 479 srpt_ioc_detach(); 480 481 /* De-register ourselves as an STMF port provider */ 482 stmf_deregister_port_provider(srpt_ctxt->sc_pp); 483 stmf_free(srpt_ctxt->sc_pp); 484 srpt_ctxt->sc_pp = NULL; 485 486 return (0); 487 } 488 489 /* 490 * srpt_drv_open() 491 */ 492 /* ARGSUSED */ 493 static int 494 srpt_drv_open(dev_t *devp, int flag, int otyp, cred_t *credp) 495 { 496 SRPT_DPRINTF_L3("drv_open, invoked"); 497 return (0); 498 } 499 500 /* 501 * srpt_drv_close() 502 */ 503 /* ARGSUSED */ 504 static int 505 srpt_drv_close(dev_t dev, int flag, int otyp, cred_t *credp) 506 { 507 SRPT_DPRINTF_L3("drv_close, invoked"); 508 return (0); 509 } 510 511 /* 512 * srpt_drv_ioctl() 513 */ 514 /* ARGSUSED */ 515 static int 516 srpt_drv_ioctl(dev_t drv, int cmd, intptr_t argp, int flag, cred_t *cred, 517 int *retval) 518 { 519 int ret = 0; 520 521 SRPT_DPRINTF_L3("drv_ioctl, invoked, cmd = %d", cmd); 522 523 if (drv_priv(cred) != 0) { 524 return (EPERM); 525 } 526 527 rw_enter(&srpt_ctxt->sc_rwlock, RW_WRITER); 528 529 switch (cmd) { 530 case SRPT_IOC_ENABLE_SVC: 531 if (srpt_ctxt->sc_svc_state != SRPT_SVC_DISABLED) { 532 break; 533 } 534 535 ret = srpt_ibdma_ops_load(&srpt_ctxt->sc_ibdma_ops); 536 if (ret != 0) { 537 break; 538 } 539 540 ret = srpt_enable_srp_services(); 541 if (ret == 0) { 542 srpt_ctxt->sc_svc_state = SRPT_SVC_ENABLED; 543 } 544 545 break; 546 547 case SRPT_IOC_DISABLE_SVC: 548 if (srpt_ctxt->sc_svc_state != SRPT_SVC_ENABLED) { 549 break; 550 } 551 552 ret = srpt_disable_srp_services(); 553 if (ret == 0) { 554 srpt_ctxt->sc_svc_state = SRPT_SVC_DISABLED; 555 } 556 557 srpt_ibdma_ops_unload(&srpt_ctxt->sc_ibdma_ops); 558 559 break; 560 561 default: 562 ret = EFAULT; 563 break; 564 } 565 566 rw_exit(&srpt_ctxt->sc_rwlock); 567 568 return (ret); 569 } 570 571 /* 572 * srpt_pp_cb() 573 */ 574 /* ARGSUSED */ 575 static void 576 srpt_pp_cb(stmf_port_provider_t *pp, int cmd, void *arg, uint32_t flags) 577 { 578 SRPT_DPRINTF_L3("srpt_pp_cb, invoked (%d)", cmd); 579 /* 580 * We don't currently utilize the port provider call-back, in the 581 * future we might use it to synchronize provider data via STMF. 582 */ 583 } 584 585 static int 586 srpt_ibdma_ops_load(srpt_ibdma_ops_t *ops) 587 { 588 int ibdma_err = 0; 589 590 ASSERT(ops != NULL); 591 592 ops->ibdmah = ddi_modopen("ibdma", KRTLD_MODE_FIRST, &ibdma_err); 593 if (ops->ibdmah == NULL) { 594 SRPT_DPRINTF_L0("failed to open ibdma driver, error = %d", 595 ibdma_err); 596 return (ibdma_err); 597 } 598 599 ops->ibdma_register = (ibdma_hdl_t (*)())ddi_modsym(ops->ibdmah, 600 "ibdma_ioc_register", &ibdma_err); 601 if (ops->ibdma_register == NULL) { 602 SRPT_DPRINTF_L0( 603 "failed to modsym ibdma_ioc_register, error = %d", 604 ibdma_err); 605 goto done; 606 } 607 608 ops->ibdma_unregister = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, 609 "ibdma_ioc_unregister", &ibdma_err); 610 if (ops->ibdma_unregister == NULL) { 611 SRPT_DPRINTF_L0( 612 "failed to modsym ibdma_ioc_unregister, error = %d", 613 ibdma_err); 614 goto done; 615 } 616 617 ops->ibdma_update = (ibdma_status_t (*)())ddi_modsym(ops->ibdmah, 618 "ibdma_ioc_update", &ibdma_err); 619 if (ops->ibdma_update == NULL) { 620 SRPT_DPRINTF_L0( 621 "failed to modsym ibdma_ioc_update, error = %d", 622 ibdma_err); 623 } 624 625 done: 626 if (ibdma_err != 0) { 627 srpt_ibdma_ops_unload(ops); 628 } 629 630 return (ibdma_err); 631 } 632 633 static void 634 srpt_ibdma_ops_unload(srpt_ibdma_ops_t *ops) 635 { 636 if (ops != NULL) { 637 if (ops->ibdmah != NULL) { 638 (void) ddi_modclose(ops->ibdmah); 639 } 640 bzero(ops, sizeof (srpt_ibdma_ops_t)); 641 } 642 } 643